2 * Copyright 2000-2018 JetBrains s.r.o.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package jetbrains.buildServer.buildTriggers.vcs.git.agent;
19 import com.intellij.openapi.util.Trinity;
20 import jetbrains.buildServer.BuildProblemData;
21 import jetbrains.buildServer.agent.AgentRunningBuild;
22 import jetbrains.buildServer.agent.BuildDirectoryCleanerCallback;
23 import jetbrains.buildServer.agent.BuildProgressLogger;
24 import jetbrains.buildServer.agent.SmartDirectoryCleaner;
25 import jetbrains.buildServer.buildTriggers.vcs.git.*;
26 import jetbrains.buildServer.buildTriggers.vcs.git.agent.command.*;
27 import jetbrains.buildServer.buildTriggers.vcs.git.agent.command.impl.CommandUtil;
28 import jetbrains.buildServer.buildTriggers.vcs.git.agent.command.impl.RefImpl;
29 import jetbrains.buildServer.buildTriggers.vcs.git.agent.errors.GitExecTimeout;
30 import jetbrains.buildServer.buildTriggers.vcs.git.agent.errors.GitIndexCorruptedException;
31 import jetbrains.buildServer.buildTriggers.vcs.git.agent.errors.GitOutdatedIndexException;
32 import jetbrains.buildServer.buildTriggers.vcs.git.agent.ssl.SSLInvestigator;
33 import jetbrains.buildServer.log.Loggers;
34 import jetbrains.buildServer.util.FileUtil;
35 import jetbrains.buildServer.util.StringUtil;
36 import jetbrains.buildServer.vcs.*;
37 import org.apache.log4j.Logger;
38 import org.eclipse.jgit.errors.ConfigInvalidException;
39 import org.eclipse.jgit.lib.*;
40 import org.eclipse.jgit.lib.Constants;
41 import org.eclipse.jgit.transport.URIish;
42 import org.jetbrains.annotations.NotNull;
43 import org.jetbrains.annotations.Nullable;
46 import java.io.FileFilter;
47 import java.io.IOException;
48 import java.net.URISyntaxException;
50 import java.util.regex.Matcher;
52 import static com.intellij.openapi.util.text.StringUtil.isEmpty;
53 import static jetbrains.buildServer.buildTriggers.vcs.git.GitUtils.*;
55 public class UpdaterImpl implements Updater {
57 private final static Logger LOG = Logger.getLogger(UpdaterImpl.class);
58 /** Git version which supports --progress option in the fetch command */
59 private final static GitVersion GIT_WITH_PROGRESS_VERSION = new GitVersion(1, 7, 1, 0);
60 //--force option in git submodule update introduced in 1.7.6
61 private final static GitVersion GIT_WITH_FORCE_SUBMODULE_UPDATE = new GitVersion(1, 7, 6);
62 public final static GitVersion GIT_WITH_SPARSE_CHECKOUT = new GitVersion(1, 7, 4);
63 public final static GitVersion BROKEN_SPARSE_CHECKOUT = new GitVersion(2, 7, 0);
64 public final static GitVersion MIN_GIT_SSH_COMMAND = new GitVersion(2, 3, 0);//GIT_SSH_COMMAND was introduced in git 2.3.0
66 * Git version supporting an empty credential helper - the only way to disable system/global/local cred helper
68 public final static GitVersion EMPTY_CRED_HELPER = new GitVersion(2, 9, 0);
69 /** Git version supporting [credential] section in config (the first version including a6fc9fd3f4b42cd97b5262026e18bd451c28ee3c) */
70 public final static GitVersion CREDENTIALS_SECTION_VERSION = new GitVersion(1, 7, 10);
72 private static final int SILENT_TIMEOUT = 24 * 60 * 60; //24 hours
74 protected final FS myFS;
75 private final SmartDirectoryCleaner myDirectoryCleaner;
76 protected final BuildProgressLogger myLogger;
77 protected final AgentPluginConfig myPluginConfig;
78 protected final GitFactory myGitFactory;
79 protected final File myTargetDirectory;
80 protected final String myRevision;
81 protected final AgentGitVcsRoot myRoot;
82 protected final String myFullBranchName;
83 protected final AgentRunningBuild myBuild;
84 protected final SSLInvestigator mySSLInvestigator;
85 private final CheckoutRules myRules;
86 private final CheckoutMode myCheckoutMode;
87 protected final MirrorManager myMirrorManager;
88 //remote repository refs, stored in field in order to not run 'git ls-remote' command twice
89 private Refs myRemoteRefs;
91 public UpdaterImpl(@NotNull FS fs,
92 @NotNull AgentPluginConfig pluginConfig,
93 @NotNull MirrorManager mirrorManager,
94 @NotNull SmartDirectoryCleaner directoryCleaner,
95 @NotNull GitFactory gitFactory,
96 @NotNull AgentRunningBuild build,
97 @NotNull VcsRoot root,
98 @NotNull String version,
99 @NotNull File targetDir,
100 @NotNull CheckoutRules rules,
101 @NotNull CheckoutMode checkoutMode) throws VcsException {
103 myPluginConfig = pluginConfig;
104 myDirectoryCleaner = directoryCleaner;
105 myGitFactory = gitFactory;
107 myLogger = build.getBuildLogger();
108 myRevision = GitUtils.versionRevision(version);
109 myTargetDirectory = targetDir;
110 myRoot = new AgentGitVcsRoot(mirrorManager, myTargetDirectory, root);
111 myFullBranchName = getBranch();
113 myCheckoutMode = checkoutMode;
114 myMirrorManager = mirrorManager;
115 mySSLInvestigator = new SSLInvestigator(myRoot.getRepositoryFetchURL(), myBuild.getAgentTempDirectory().getPath(),
116 myBuild.getAgentConfiguration().getAgentHomeDirectory().getPath());
120 private String getBranch() {
121 String defaultBranchName = GitUtils.expandRef(myRoot.getRef());
122 String rootBranchParam = GitUtils.getGitRootBranchParamName(myRoot.getOriginalRoot());
123 String customBranch = myBuild.getSharedConfigParameters().get(rootBranchParam);
124 return customBranch != null ? customBranch : defaultBranchName;
128 public void update() throws VcsException {
129 logInfo("Git version: " + myPluginConfig.getGitVersion());
130 logSshOptions(myPluginConfig.getGitVersion());
131 checkAuthMethodIsSupported();
133 checkNoDiffWithUpperLimitRevision();
136 private void logSshOptions(@NotNull GitVersion gitVersion) {
137 if (myPluginConfig.isUseNativeSSH()) {
138 logInfo("Will use native ssh (" + PluginConfigImpl.USE_NATIVE_SSH + "=true)");
139 if (myRoot.getAuthSettings().getAuthMethod() == AuthenticationMethod.TEAMCITY_SSH_KEY) {
140 if (gitVersion.isLessThan(UpdaterImpl.MIN_GIT_SSH_COMMAND)) {
141 logWarn("Git " + gitVersion + " doesn't support the GIT_SSH_COMMAND environment variable, uploaded SSH keys will not work. " +
142 "Required git version is " + UpdaterImpl.MIN_GIT_SSH_COMMAND);
143 } else if (!myPluginConfig.isUseGitSshCommand()) {
144 logWarn("Use of GIT_SSH_COMMAND is disabled (" + PluginConfigImpl.USE_GIT_SSH_COMMAND + "=false), uploaded SSH keys will not work.");
150 private void logInfo(@NotNull String msg) {
151 myLogger.message(msg);
152 Loggers.VCS.info(msg);
155 private void logWarn(@NotNull String msg) {
156 myLogger.warning(msg);
157 Loggers.VCS.warn(msg);
160 protected void doUpdate() throws VcsException {
161 String message = "Update checkout directory (" + myTargetDirectory.getAbsolutePath() + ")";
162 myLogger.activityStarted(message, GitBuildProgressLogger.GIT_PROGRESS_ACTIVITY);
166 removeRefLocks(new File(myTargetDirectory, ".git"));
170 myLogger.activityFinished(message, GitBuildProgressLogger.GIT_PROGRESS_ACTIVITY);
174 private void logStartUpdating() {
175 LOG.info("Starting update of root " + myRoot.getName() + " in " + myTargetDirectory + " to revision " + myRevision);
176 LOG.debug("Updating " + myRoot.debugInfo());
180 private void initGitRepository() throws VcsException {
181 if (!new File(myTargetDirectory, ".git").exists()) {
182 initDirectory(false);
185 configureRemoteUrl(new File(myTargetDirectory, ".git"));
186 setupExistingRepository();
187 configureSparseCheckout();
188 } catch (Exception e) {
189 LOG.warn("Do clean checkout due to errors while configure use of local mirrors", e);
193 mySSLInvestigator.setCertificateOptions(myGitFactory.create(myTargetDirectory));
194 removeOrphanedIdxFiles(new File(myTargetDirectory, ".git"));
197 protected void setupNewRepository() throws VcsException {
201 protected void setupExistingRepository() throws VcsException {
208 private void updateSources() throws VcsException {
209 final GitFacade git = myGitFactory.create(myTargetDirectory);
210 boolean branchChanged = false;
212 if (isRegularBranch(myFullBranchName)) {
213 String branchName = getShortBranchName(myFullBranchName);
214 Branches branches = git.listBranches();
215 if (branches.isCurrentBranch(branchName)) {
217 runAndFixIndexErrors(git, new VcsCommand() {
219 public void call() throws VcsException {
220 reset(git).setHard(true).setRevision(myRevision).call();
223 git.setUpstream(branchName, GitUtils.createRemoteRef(myFullBranchName)).call();
225 branchChanged = true;
226 if (!branches.contains(branchName)) {
229 .setStartPoint(GitUtils.createRemoteRef(myFullBranchName))
233 git.updateRef().setRef(myFullBranchName).setRevision(myRevision).call();
234 final String finalBranchName = branchName;
235 runAndFixIndexErrors(git, new VcsCommand() {
237 public void call() throws VcsException {
238 checkout(git).setForce(true).setBranch(finalBranchName).setTimeout(myPluginConfig.getCheckoutIdleTimeoutSeconds()).call();
241 if (branches.contains(branchName)) {
242 git.setUpstream(branchName, GitUtils.createRemoteRef(myFullBranchName)).call();
245 } else if (isTag(myFullBranchName)) {
246 final String shortName = myFullBranchName.substring("refs/tags/".length());
247 runAndFixIndexErrors(git, new VcsCommand() {
249 public void call() throws VcsException {
250 checkout(git).setForce(true).setBranch(shortName).setTimeout(myPluginConfig.getCheckoutIdleTimeoutSeconds()).call();
253 Ref tag = getRef(myTargetDirectory, myFullBranchName);
254 if (tag != null && !tag.getObjectId().name().equals(myRevision)) {
255 runAndFixIndexErrors(git, new VcsCommand() {
257 public void call() throws VcsException {
258 checkout(git).setBranch(myRevision).setForce(true).setTimeout(myPluginConfig.getCheckoutIdleTimeoutSeconds()).call();
262 branchChanged = true;
264 runAndFixIndexErrors(git, new VcsCommand() {
266 public void call() throws VcsException {
267 checkout(git).setForce(true).setBranch(myRevision).setTimeout(myPluginConfig.getCheckoutIdleTimeoutSeconds()).call();
270 branchChanged = true;
273 doClean(branchChanged);
274 if (myRoot.isCheckoutSubmodules()) {
275 checkoutSubmodules(myTargetDirectory);
280 private void runAndFixIndexErrors(@NotNull GitFacade git, @NotNull VcsCommand cmd) throws VcsException {
283 } catch (GitIndexCorruptedException e) {
284 File gitIndex = e.getGitIndex();
285 myLogger.message("Git index '" + gitIndex.getAbsolutePath() + "' is corrupted, remove it and repeat the command");
286 FileUtil.delete(gitIndex);
288 } catch (GitOutdatedIndexException e) {
289 myLogger.message("Refresh outdated git index and repeat the command");
290 updateIndex(git).reallyRefresh(true).quiet(true).call();
292 } catch (Exception e) {
293 if (e instanceof VcsException)
294 throw (VcsException) e;
295 throw new VcsException(e);
301 private UpdateIndexCommand updateIndex(final GitFacade git) {
302 UpdateIndexCommand result = git.updateIndex()
303 .setAuthSettings(myRoot.getAuthSettings())
304 .setUseNativeSsh(myPluginConfig.isUseNativeSSH());
305 configureLFS(result);
311 private ResetCommand reset(final GitFacade git) {
312 ResetCommand result = git.reset()
313 .setAuthSettings(myRoot.getAuthSettings())
314 .setUseNativeSsh(myPluginConfig.isUseNativeSSH());
315 configureLFS(result);
320 private CheckoutCommand checkout(final GitFacade git) {
321 CheckoutCommand result = git.checkout()
322 .setAuthSettings(myRoot.getAuthSettings())
323 .setUseNativeSsh(myPluginConfig.isUseNativeSSH());
324 configureLFS(result);
328 private void checkoutSubmodules(@NotNull final File repositoryDir) throws VcsException {
329 File dotGitModules = new File(repositoryDir, ".gitmodules");
331 Config gitModules = readGitModules(dotGitModules);
332 if (gitModules == null)
335 myLogger.message("Checkout submodules in " + repositoryDir);
336 GitFacade git = myGitFactory.create(repositoryDir);
337 git.submoduleInit().call();
338 git.submoduleSync().call();
340 addSubmoduleUsernames(repositoryDir, gitModules);
342 long start = System.currentTimeMillis();
343 SubmoduleUpdateCommand submoduleUpdate = git.submoduleUpdate()
344 .setAuthSettings(myRoot.getAuthSettings())
345 .setUseNativeSsh(myPluginConfig.isUseNativeSSH())
346 .setTimeout(SILENT_TIMEOUT)
347 .setForce(isForceUpdateSupported());
348 configureLFS(submoduleUpdate);
349 submoduleUpdate.call();
351 if (recursiveSubmoduleCheckout()) {
352 for (String submodulePath : getSubmodulePaths(gitModules)) {
353 checkoutSubmodules(new File(repositoryDir, submodulePath));
356 Loggers.VCS.info("Submodules update in " + repositoryDir.getAbsolutePath() + " is finished in " +
357 (System.currentTimeMillis() - start) + " ms");
359 } catch (IOException e) {
360 Loggers.VCS.error("Submodules checkout failed", e);
361 throw new VcsException("Submodules checkout failed", e);
362 } catch (ConfigInvalidException e) {
363 Loggers.VCS.error("Submodules checkout failed", e);
364 throw new VcsException("Submodules checkout failed", e);
369 private boolean isForceUpdateSupported() {
370 return !GIT_WITH_FORCE_SUBMODULE_UPDATE.isGreaterThan(myPluginConfig.getGitVersion());
374 private void addSubmoduleUsernames(@NotNull File repositoryDir, @NotNull Config gitModules)
375 throws IOException, VcsException {
376 if (!myPluginConfig.isUseMainRepoUserForSubmodules())
379 Loggers.VCS.info("Update submodules credentials");
381 AuthSettings auth = myRoot.getAuthSettings();
382 final String userName = auth.getUserName();
383 if (userName == null) {
384 Loggers.VCS.info("Username is not specified in the main VCS root settings, skip updating credentials");
388 Repository r = new RepositoryBuilder().setBare().setGitDir(getGitDir(repositoryDir)).build();
390 StoredConfig gitConfig = r.getConfig();
392 Set<String> submodules = gitModules.getSubsections("submodule");
393 if (submodules.isEmpty()) {
394 Loggers.VCS.info("No submodule sections found in " + new File(repositoryDir, ".gitmodules").getCanonicalPath()
395 + ", skip updating credentials");
398 File modulesDir = new File(r.getDirectory(), Constants.MODULES);
399 for (String submoduleName : submodules) {
400 //The 'git submodule sync' command executed before resolves relative submodule urls
401 //from .gitmodules and writes them into .git/config. We should use resolved urls in
402 //order to add parent repository username to submodules with relative urls.
403 String url = gitConfig.getString("submodule", submoduleName, "url");
405 Loggers.VCS.info(".git/config doesn't contain an url for submodule '" + submoduleName + "', use url from .gitmodules");
406 url = gitModules.getString("submodule", submoduleName, "url");
408 Loggers.VCS.info("Update credentials for submodule with url " + url);
409 if (url == null || !isRequireAuth(url)) {
410 Loggers.VCS.info("Url " + url + " does not require authentication, skip updating credentials");
414 URIish uri = new URIish(url);
415 String updatedUrl = uri.setUser(userName).toASCIIString();
416 gitConfig.setString("submodule", submoduleName, "url", updatedUrl);
417 String submodulePath = gitModules.getString("submodule", submoduleName, "path");
418 if (submodulePath != null && myPluginConfig.isUpdateSubmoduleOriginUrl()) {
419 File submoduleDir = new File(modulesDir, submodulePath);
420 if (submoduleDir.isDirectory() && new File(submoduleDir, Constants.CONFIG).isFile())
421 updateOriginUrl(submoduleDir, updatedUrl);
423 Loggers.VCS.debug("Submodule url " + url + " changed to " + updatedUrl);
424 } catch (URISyntaxException e) {
425 Loggers.VCS.warn("Error while parsing an url " + url + ", skip updating submodule credentials", e);
426 } catch (Exception e) {
427 Loggers.VCS.warn("Error while updating the '" + submoduleName + "' submodule url", e);
436 private void updateOriginUrl(@NotNull File repoDir, @NotNull String url) throws IOException {
437 Repository r = new RepositoryBuilder().setBare().setGitDir(repoDir).build();
438 StoredConfig config = r.getConfig();
439 config.setString("remote", "origin", "url", url);
445 private Config readGitModules(@NotNull File dotGitModules) throws IOException, ConfigInvalidException {
446 if (!dotGitModules.exists())
448 String content = FileUtil.readText(dotGitModules);
449 Config config = new Config();
450 config.fromText(content);
455 private boolean isRequireAuth(@NotNull String url) {
457 URIish uri = new URIish(url);
458 String scheme = uri.getScheme();
459 if (scheme == null || "git".equals(scheme)) //no auth for anonymous protocol and for local repositories
461 String user = uri.getUser();
462 //respect a user specified in config
464 } catch (URISyntaxException e) {
470 private Set<String> getSubmodulePaths(@NotNull Config config) {
471 Set<String> paths = new HashSet<String>();
472 Set<String> submodules = config.getSubsections("submodule");
473 for (String submoduleName : submodules) {
474 String submodulePath = config.getString("submodule", submoduleName, "path");
475 paths.add(submodulePath.replaceAll("/", Matcher.quoteReplacement(File.separator)));
480 private boolean recursiveSubmoduleCheckout() {
481 return SubmodulesCheckoutPolicy.CHECKOUT.equals(myRoot.getSubmodulesCheckoutPolicy()) ||
482 SubmodulesCheckoutPolicy.CHECKOUT_IGNORING_ERRORS.equals(myRoot.getSubmodulesCheckoutPolicy());
486 private void doClean(boolean branchChanged) throws VcsException {
487 if (myRoot.getCleanPolicy() == AgentCleanPolicy.ALWAYS ||
488 branchChanged && myRoot.getCleanPolicy() == AgentCleanPolicy.ON_BRANCH_CHANGE) {
489 myLogger.message("Cleaning " + myRoot.getName() + " in " + myTargetDirectory + " the file set " + myRoot.getCleanFilesPolicy());
490 myGitFactory.create(myTargetDirectory).clean().setCleanPolicy(myRoot.getCleanFilesPolicy()).call();
492 if (myRoot.isCheckoutSubmodules())
493 cleanSubmodules(myTargetDirectory);
498 private void cleanSubmodules(@NotNull File repositoryDir) throws VcsException {
499 File dotGitModules = new File(repositoryDir, ".gitmodules");
502 gitModules = readGitModules(dotGitModules);
503 } catch (Exception e) {
504 Loggers.VCS.error("Error while reading " + dotGitModules.getAbsolutePath() + ": " + e.getMessage());
505 throw new VcsException("Error while reading " + dotGitModules.getAbsolutePath(), e);
508 if (gitModules == null)
511 for (String submodulePath : getSubmodulePaths(gitModules)) {
512 File submoduleDir = new File(repositoryDir, submodulePath);
514 myLogger.message("Cleaning files in " + submoduleDir + " the file set " + myRoot.getCleanFilesPolicy());
515 myGitFactory.create(submoduleDir).clean().setCleanPolicy(myRoot.getCleanFilesPolicy()).call();
516 } catch (Exception e) {
517 Loggers.VCS.error("Error while cleaning files in " + submoduleDir.getAbsolutePath(), e);
519 if (recursiveSubmoduleCheckout())
520 cleanSubmodules(submoduleDir);
525 protected void removeUrlSections() throws VcsException {
528 r = new RepositoryBuilder().setWorkTree(myTargetDirectory).build();
529 StoredConfig config = r.getConfig();
530 Set<String> urlSubsections = config.getSubsections("url");
531 for (String subsection : urlSubsections) {
532 config.unsetSection("url", subsection);
535 } catch (IOException e) {
536 String msg = "Error while remove url.* sections";
538 throw new VcsException(msg, e);
546 private void removeLfsStorage() throws VcsException {
549 r = new RepositoryBuilder().setWorkTree(myTargetDirectory).build();
550 StoredConfig config = r.getConfig();
551 config.unsetSection("lfs", null);
553 } catch (IOException e) {
554 String msg = "Error while removing lfs.storage section";
556 throw new VcsException(msg, e);
564 protected void disableAlternates() {
565 FileUtil.delete(new File(myTargetDirectory, ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"));
569 private String getRemoteUrl() {
571 return myGitFactory.create(myTargetDirectory).getConfig().setPropertyName("remote.origin.url").call();
572 } catch (VcsException e) {
573 LOG.debug("Failed to read property", e);
580 protected Ref getRef(@NotNull File repositoryDir, @NotNull String ref) {
581 Map<String, Ref> refs = myGitFactory.create(repositoryDir).showRef().setPattern(ref).call().getValidRefs();
582 return refs.isEmpty() ? null : refs.get(ref);
587 * If some git process crashed in this repository earlier it can leave lock files for index.
588 * This method delete such lock file if it exists (with warning message), otherwise git operation will fail.
590 private void removeIndexLock() {
591 File indexLock = new File(myTargetDirectory, ".git" + File.separator + "index.lock");
592 if (indexLock.exists()) {
593 myLogger.warning("The .git/index.lock file exists. This probably means a git process crashed in this repository earlier. Deleting lock file");
594 FileUtil.delete(indexLock);
599 private void doFetch() throws VcsException {
600 boolean outdatedRefsFound = removeOutdatedRefs(myTargetDirectory);
601 ensureCommitLoaded(outdatedRefsFound);
605 protected void ensureCommitLoaded(boolean fetchRequired) throws VcsException {
606 fetchFromOriginalRepository(fetchRequired);
610 protected void fetchFromOriginalRepository(boolean fetchRequired) throws VcsException {
612 FetchHeadsMode fetchHeadsMode = myPluginConfig.getFetchHeadsMode();
613 switch (fetchHeadsMode) {
615 String msg = getForcedHeadsFetchMessage();
617 myLogger.message(msg);
620 if (!myFullBranchName.startsWith("refs/heads/")) {
621 remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
622 if (fetchRequired || remoteRef == null || !myRevision.equals(remoteRef.getObjectId().name()) || !hasRevision(myTargetDirectory, myRevision))
623 fetchDefaultBranch();
626 case BEFORE_BUILD_BRANCH:
627 remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
628 if (!fetchRequired && remoteRef != null && myRevision.equals(remoteRef.getObjectId().name()) && hasRevision(myTargetDirectory, myRevision))
630 myLogger.message("Commit '" + myRevision + "' is not found in local clone. Running 'git fetch'...");
632 if (!myFullBranchName.startsWith("refs/heads/")) {
633 remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
634 if (fetchRequired || remoteRef == null || !myRevision.equals(remoteRef.getObjectId().name()) || !hasRevision(myTargetDirectory, myRevision))
635 fetchDefaultBranch();
638 case AFTER_BUILD_BRANCH:
639 remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
640 if (!fetchRequired && remoteRef != null && myRevision.equals(remoteRef.getObjectId().name()) && hasRevision(myTargetDirectory, myRevision))
642 myLogger.message("Commit '" + myRevision + "' is not found in local clone. Running 'git fetch'...");
643 fetchDefaultBranch();
644 if (hasRevision(myTargetDirectory, myRevision))
646 myLogger.message("Commit still not found after fetching main branch. Fetching more branches.");
650 throw new VcsException("Unknown FetchHeadsMode: " + fetchHeadsMode);
653 if (hasRevision(myTargetDirectory, myRevision))
656 String msg = "Cannot find commit " + myRevision + " in the " + myRoot.getRepositoryFetchURL().toASCIIString() + " repository, " +
657 "possible reason: " + myFullBranchName + " branch was updated and the commit selected for the build is not reachable anymore";
658 throw new RevisionNotFoundException(msg);
662 protected String getForcedHeadsFetchMessage() {
663 return "Forced fetch of all heads (" + PluginConfigImpl.FETCH_ALL_HEADS + "=" + myBuild.getSharedConfigParameters().get(PluginConfigImpl.FETCH_ALL_HEADS) + ")";
667 private void fetchDefaultBranch() throws VcsException {
668 fetch(myTargetDirectory, getRefspecForFetch(), false);
671 private String getRefspecForFetch() {
672 return "+" + myFullBranchName + ":" + GitUtils.createRemoteRef(myFullBranchName);
675 private void fetchAllBranches() throws VcsException {
676 fetch(myTargetDirectory, "+refs/heads/*:refs/remotes/origin/*", false);
679 protected boolean hasRevision(@NotNull File repositoryDir, @NotNull String revision) {
680 return getRevision(repositoryDir, revision) != null;
683 private String getRevision(@NotNull File repositoryDir, @NotNull String revision) {
684 return myGitFactory.create(repositoryDir).log()
686 .setPrettyFormat("%H%x20%s")
687 .setStartPoint(revision)
691 protected void fetch(@NotNull File repositoryDir, @NotNull String refspec, boolean shallowClone) throws VcsException {
692 boolean silent = isSilentFetch();
693 int timeout = getTimeout(silent);
696 getFetch(repositoryDir, refspec, shallowClone, silent, timeout).call();
697 } catch (GitIndexCorruptedException e) {
698 File gitIndex = e.getGitIndex();
699 myLogger.message("Git index '" + gitIndex.getAbsolutePath() + "' is corrupted, remove it and repeat git fetch");
700 FileUtil.delete(gitIndex);
701 getFetch(repositoryDir, refspec, shallowClone, silent, timeout).call();
702 } catch (GitExecTimeout e) {
704 myLogger.error("No output from git during " + timeout + " seconds. Try increasing idle timeout by setting parameter '"
705 + PluginConfigImpl.IDLE_TIMEOUT +
706 "' either in build or in agent configuration.");
713 private FetchCommand getFetch(@NotNull File repositoryDir, @NotNull String refspec, boolean shallowClone, boolean silent, int timeout) {
714 FetchCommand result = myGitFactory.create(repositoryDir).fetch()
715 .setAuthSettings(myRoot.getAuthSettings())
716 .setUseNativeSsh(myPluginConfig.isUseNativeSSH())
719 .setFetchTags(myPluginConfig.isFetchTags());
722 result.setQuite(true);
724 result.setShowProgress(true);
732 protected void removeRefLocks(@NotNull File dotGit) {
733 File refs = new File(dotGit, "refs");
734 if (!refs.isDirectory())
736 Collection<File> locks = FileUtil.findFiles(new FileFilter() {
737 public boolean accept(File f) {
738 return f.isFile() && f.getName().endsWith(".lock");
741 for (File lock : locks) {
742 LOG.info("Remove a lock file " + lock.getAbsolutePath());
743 FileUtil.delete(lock);
745 File packedRefsLock = new File(dotGit, "packed-refs.lock");
746 if (packedRefsLock.isFile()) {
747 LOG.info("Remove a lock file " + packedRefsLock.getAbsolutePath());
748 FileUtil.delete(packedRefsLock);
752 private boolean isSilentFetch() {
753 GitVersion version = myPluginConfig.getGitVersion();
754 return version.isLessThan(GIT_WITH_PROGRESS_VERSION);
757 private int getTimeout(boolean silentFetch) {
759 return SILENT_TIMEOUT;
761 return myPluginConfig.getIdleTimeoutSeconds();
765 private void checkAuthMethodIsSupported() throws VcsException {
766 checkAuthMethodIsSupported(myRoot, myPluginConfig);
770 static void checkAuthMethodIsSupported(@NotNull GitVcsRoot root, @NotNull AgentPluginConfig config) throws VcsException {
771 if ("git".equals(root.getRepositoryFetchURL().getScheme()))
772 return;//anonymous protocol, don't check anything
773 AuthSettings authSettings = root.getAuthSettings();
774 switch (authSettings.getAuthMethod()) {
776 if ("http".equals(root.getRepositoryFetchURL().getScheme()) ||
777 "https".equals(root.getRepositoryFetchURL().getScheme())) {
778 GitVersion actualVersion = config.getGitVersion();
779 GitVersion requiredVersion = getMinVersionForHttpAuth();
780 if (actualVersion.isLessThan(requiredVersion)) {
781 throw new VcsException("Password authentication requires git " + requiredVersion +
782 ", found git version is " + actualVersion +
783 ". Upgrade git or use different authentication method.");
786 throw new VcsException("TeamCity doesn't support authentication method '" +
787 root.getAuthSettings().getAuthMethod().uiName() +
788 "' with agent checkout and non-http protocols. Please use different authentication method.");
791 case PRIVATE_KEY_FILE:
792 throw new VcsException("TeamCity doesn't support authentication method '" +
793 root.getAuthSettings().getAuthMethod().uiName() +
794 "' with agent checkout. Please use different authentication method.");
799 private static GitVersion getMinVersionForHttpAuth() {
800 //core.askpass parameter was added in 1.7.1, but
801 //experiments show that it works only in 1.7.3 on linux
802 //and msysgit 1.7.3.1-preview20101002.
803 return new GitVersion(1, 7, 3);
807 * Clean and init directory and configure remote origin
809 * @throws VcsException if there are problems with initializing the directory
811 private void initDirectory(boolean removeTargetDir) throws VcsException {
812 if (removeTargetDir) {
813 BuildDirectoryCleanerCallback c = new BuildDirectoryCleanerCallback(myLogger, LOG);
814 myDirectoryCleaner.cleanFolder(myTargetDirectory, c);
815 //noinspection ResultOfMethodCallIgnored
816 if (c.isHasErrors()) {
817 throw new VcsException("Unable to clean directory " + myTargetDirectory + " for VCS root " + myRoot.getName());
821 myTargetDirectory.mkdirs();
822 myLogger.message("The .git directory is missing in '" + myTargetDirectory + "'. Running 'git init'...");
823 final GitFacade gitFacade = myGitFactory.create(myTargetDirectory);
824 gitFacade.init().call();
826 configureRemoteUrl(new File(myTargetDirectory, ".git"));
828 URIish fetchUrl = myRoot.getRepositoryFetchURL();
829 URIish url = myRoot.getRepositoryPushURL();
830 String pushUrl = url == null ? null : url.toString();
831 if (pushUrl != null && !pushUrl.equals(fetchUrl.toString())) {
832 gitFacade.setConfig().setPropertyName("remote.origin.pushurl").setValue(pushUrl).call();
834 setupNewRepository();
835 configureSparseCheckout();
839 void configureRemoteUrl(@NotNull File gitDir) throws VcsException {
840 RemoteRepositoryConfigurator cfg = new RemoteRepositoryConfigurator();
841 cfg.setGitDir(gitDir);
842 cfg.setExcludeUsernameFromHttpUrls(myPluginConfig.isExcludeUsernameFromHttpUrl() && !myPluginConfig.getGitVersion().isLessThan(UpdaterImpl.CREDENTIALS_SECTION_VERSION));
843 cfg.configure(myRoot);
847 private void configureSparseCheckout() throws VcsException {
848 if (myCheckoutMode == CheckoutMode.SPARSE_CHECKOUT) {
849 setupSparseCheckout();
851 myGitFactory.create(myTargetDirectory).setConfig().setPropertyName("core.sparseCheckout").setValue("false").call();
855 private void setupSparseCheckout() throws VcsException {
856 myGitFactory.create(myTargetDirectory).setConfig().setPropertyName("core.sparseCheckout").setValue("true").call();
857 File sparseCheckout = new File(myTargetDirectory, ".git/info/sparse-checkout");
858 boolean hasIncludeRules = false;
859 StringBuilder sparseCheckoutContent = new StringBuilder();
860 for (IncludeRule rule : myRules.getIncludeRules()) {
861 if (isEmpty(rule.getFrom())) {
862 sparseCheckoutContent.append("/*\n");
864 sparseCheckoutContent.append("/").append(rule.getFrom()).append("\n");
866 hasIncludeRules = true;
868 if (!hasIncludeRules) {
869 sparseCheckoutContent.append("/*\n");
871 for (FileRule rule : myRules.getExcludeRules()) {
872 sparseCheckoutContent.append("!/").append(rule.getFrom()).append("\n");
875 FileUtil.writeFileAndReportErrors(sparseCheckout, sparseCheckoutContent.toString());
876 } catch (IOException e) {
877 LOG.warn("Error while writing sparse checkout config, disable sparse checkout", e);
878 myGitFactory.create(myTargetDirectory).setConfig().setPropertyName("core.sparseCheckout").setValue("false").call();
883 private void validateUrls() {
884 URIish fetch = myRoot.getRepositoryFetchURL();
885 if (isAnonymousGitWithUsername(fetch))
886 LOG.warn("Fetch URL '" + fetch.toString() + "' for root " + myRoot.getName() + " uses an anonymous git protocol and contains a username, fetch will probably fail");
887 URIish push = myRoot.getRepositoryPushURL();
888 if (!fetch.equals(push) && isAnonymousGitWithUsername(push))
889 LOG.warn("Push URL '" + push.toString() + "'for root " + myRoot.getName() + " uses an anonymous git protocol and contains a username, push will probably fail");
893 protected boolean removeOutdatedRefs(@NotNull File workingDir) throws VcsException {
894 boolean outdatedRefsRemoved = false;
895 GitFacade git = myGitFactory.create(workingDir);
896 ShowRefResult showRefResult = git.showRef().call();
897 Refs localRefs = new Refs(showRefResult.getValidRefs());
898 if (localRefs.isEmpty() && showRefResult.getInvalidRefs().isEmpty())
900 for (String invalidRef : showRefResult.getInvalidRefs()) {
901 git.updateRef().setRef(invalidRef).delete().call();
902 outdatedRefsRemoved = true;
904 final Refs remoteRefs;
906 remoteRefs = getRemoteRefs(workingDir);
907 } catch (VcsException e) {
908 if (CommandUtil.isCanceledError(e))
910 String msg = "Failed to list remote repository refs, outdated local refs will not be cleaned";
912 myLogger.warning(msg);
915 //We remove both outdated local refs (e.g. refs/heads/topic) and outdated remote
916 //tracking branches (refs/remote/origin/topic), while git remote origin prune
917 //removes only the latter. We need that because in some cases git cannot handle
918 //rename of the branch (TW-28735).
919 for (Ref localRef : localRefs.list()) {
920 Ref correspondingRemoteRef = createCorrespondingRemoteRef(localRef);
921 if (remoteRefs.isOutdated(correspondingRemoteRef)) {
922 git.updateRef().setRef(localRef.getName()).delete().call();
923 outdatedRefsRemoved = true;
926 return outdatedRefsRemoved;
931 private Refs getRemoteRefs(@NotNull File workingDir) throws VcsException {
932 if (myRemoteRefs != null)
934 GitFacade git = myGitFactory.create(workingDir);
935 myRemoteRefs = new Refs(git.lsRemote().setAuthSettings(myRoot.getAuthSettings())
936 .setUseNativeSsh(myPluginConfig.isUseNativeSSH())
937 .setTimeout(myPluginConfig.getLsRemoteTimeoutSeconds())
943 private boolean isRemoteTrackingBranch(@NotNull Ref localRef) {
944 return localRef.getName().startsWith("refs/remotes/origin");
948 private Ref createCorrespondingRemoteRef(@NotNull Ref localRef) {
949 if (!isRemoteTrackingBranch(localRef))
951 return new RefImpl("refs/heads" + localRef.getName().substring("refs/remotes/origin".length()),
952 localRef.getObjectId().name());
956 private void configureLFS(@NotNull BaseCommand command) {
957 if (!myPluginConfig.isProvideCredHelper())
959 Trinity<String, String, String> lfsAuth = getLfsAuth();
962 File credentialsHelper = null;
964 ScriptGen scriptGen = myGitFactory.create(new File(".")).getScriptGen();
965 final File credHelper = scriptGen.generateCredentialsHelper();
966 credentialsHelper = credHelper;
967 if (!myPluginConfig.getGitVersion().isLessThan(UpdaterImpl.EMPTY_CRED_HELPER)) {
968 //Specify an empty helper if it is supported in order to disable
969 //helpers in system-global-local chain. If empty helper is not supported,
970 //then the only workaround is to disable helpers manually in config files.
971 command.addConfig("credential.helper", "");
973 String path = credHelper.getCanonicalPath();
974 path = path.replaceAll("\\\\", "/");
975 command.addConfig("credential.helper", path);
976 CredentialsHelperConfig config = new CredentialsHelperConfig();
977 config.addCredentials(lfsAuth.first, lfsAuth.second, lfsAuth.third);
978 config.setMatchAllUrls(myPluginConfig.isCredHelperMatchesAllUrls());
979 for (Map.Entry<String, String> e : config.getEnv().entrySet()) {
980 command.setEnv(e.getKey(), e.getValue());
982 if (myPluginConfig.isCleanCredHelperScript()) {
983 command.addPostAction(new Runnable() {
986 FileUtil.delete(credHelper);
990 } catch (Exception e) {
991 if (credentialsHelper != null)
992 FileUtil.delete(credentialsHelper);
997 //returns (url, name, pass) for lfs or null if no authentication is required or
998 //root doesn't use http(s)
1000 private Trinity<String, String, String> getLfsAuth() {
1002 URIish uri = new URIish(myRoot.getRepositoryFetchURL().toString());
1003 String scheme = uri.getScheme();
1004 if (myRoot.getAuthSettings().getAuthMethod() == AuthenticationMethod.PASSWORD &&
1005 ("http".equals(scheme) || "https".equals(scheme))) {
1006 String lfsUrl = uri.setPass("").setUser("").toASCIIString();
1007 if (lfsUrl.endsWith(".git")) {
1008 lfsUrl += "/info/lfs";
1010 lfsUrl += lfsUrl.endsWith("/") ? ".git/info/lfs" : "/.git/info/lfs";
1012 return Trinity.create(lfsUrl, myRoot.getAuthSettings().getUserName(), myRoot.getAuthSettings().getPassword());
1014 } catch (Exception e) {
1015 LOG.debug("Cannot get lfs auth config", e);
1021 private interface VcsCommand {
1022 void call() throws VcsException;
1027 * Removes .idx files which don't have a corresponding .pack file
1028 * @param ditGitDir git dir
1030 void removeOrphanedIdxFiles(@NotNull File ditGitDir) {
1031 if ("false".equals(myBuild.getSharedConfigParameters().get("teamcity.git.removeOrphanedIdxFiles"))) {
1032 //looks like this logic is always needed, if no problems will be reported we can drop the option
1035 File packDir = new File(new File(ditGitDir, "objects"), "pack");
1036 File[] files = packDir.listFiles();
1037 if (files == null || files.length == 0)
1040 Set<String> packs = new HashSet<String>();
1041 for (File f : files) {
1042 String name = f.getName();
1043 if (name.endsWith(".pack")) {
1044 packs.add(name.substring(0, name.length() - 5));
1048 for (File f : files) {
1049 String name = f.getName();
1050 if (name.endsWith(".idx")) {
1051 if (!packs.contains(name.substring(0, name.length() - 4)))
1058 private void checkNoDiffWithUpperLimitRevision() {
1059 if ("false".equals(myBuild.getSharedConfigParameters().get("teamcity.git.checkDiffWithUpperLimitRevision"))) {
1063 String upperLimitRevision = getUpperLimitRevision();
1064 if (upperLimitRevision == null) {
1068 String message = "Check no diff with upper limit revision " + upperLimitRevision;
1069 myLogger.activityStarted(message, GitBuildProgressLogger.GIT_PROGRESS_ACTIVITY);
1071 if (!ensureCommitLoaded(upperLimitRevision)) {
1072 myLogger.warning("Failed to fetch " + upperLimitRevision + ", will not analyze diff with upper limit revision");
1076 if (myRevision.equals(upperLimitRevision)) {
1077 myLogger.message("Build revision is the same as the upper limit revision, skip checking diff");
1081 List<String> pathsMatchedByRules = getChangedFilesMatchedByRules(upperLimitRevision);
1082 if (!pathsMatchedByRules.isEmpty()) {
1083 StringBuilder msg = new StringBuilder();
1084 msg.append("Files matched by checkout rules changed between build revision and upper-limit revision\n");
1085 msg.append("Checkout rules: '").append(myRules.getAsString()).append("'\n");
1086 msg.append("Build revision: '").append(myRevision).append("'\n");
1087 msg.append("Upper limit revision: '").append(upperLimitRevision).append("'\n");
1088 msg.append("Files:\n");
1089 for (String path : pathsMatchedByRules) {
1090 msg.append("\t").append(path).append("\n");
1092 myLogger.error(msg.toString());
1093 String type = "UpperLimitRevisionDiff";
1094 myLogger.logBuildProblem(BuildProblemData.createBuildProblem(type + myRoot.getId(), type, "Diff with upper limit revision found"));
1096 myLogger.message("No diff matched by checkout rules found");
1099 myLogger.activityFinished(message, GitBuildProgressLogger.GIT_PROGRESS_ACTIVITY);
1103 private boolean ensureCommitLoaded(@NotNull String commit) {
1104 if (hasRevision(myTargetDirectory, commit))
1108 } catch (VcsException e) {
1109 LOG.warn("Error while fetching commit " + commit, e);
1112 return hasRevision(myTargetDirectory, commit);
1116 private List<String> getChangedFilesMatchedByRules(@NotNull String upperLimitRevision) {
1117 List<String> pathsMatchedByRules = new ArrayList<String>();
1118 List<String> changedFiles = getChangedFiles(upperLimitRevision);
1119 for (String file : changedFiles) {
1120 if (myRules.map(file) != null) {
1121 pathsMatchedByRules.add(file);
1124 return pathsMatchedByRules;
1128 private List<String> getChangedFiles(@NotNull String upperLimitRevision) {
1130 return myGitFactory.create(myTargetDirectory).diff()
1131 .setFormat("--name-only")
1132 .setCommit1(myRevision)
1133 .setCommit2(upperLimitRevision)
1135 } catch (VcsException e) {
1136 myLogger.warning("Error while computing changed files between build and upper limit revisions: " + e.toString());
1137 return Collections.emptyList();
1142 private String getUpperLimitRevision() {
1143 String rootExtId = getVcsRootExtId();
1144 return rootExtId != null ? myBuild.getSharedConfigParameters().get("teamcity.upperLimitRevision." + rootExtId) : null;
1148 private String getVcsRootExtId() {
1149 // We don't have vcs root extId on the agent, deduce it from vcs.number parameters
1150 String revisionParamPrefix = "build.vcs.number.";
1151 String vcsRootExtId = null;
1152 Map<String, String> params = myBuild.getSharedConfigParameters();
1153 for (Map.Entry<String, String> param : params.entrySet()) {
1154 if (param.getKey().startsWith(revisionParamPrefix) && myRevision.equals(param.getValue())) {
1155 String extId = param.getKey().substring(revisionParamPrefix.length());
1156 if (StringUtil.isNotEmpty(extId) && Character.isDigit(extId.charAt(0))) {
1157 // We have build.vcs.number.<extId> and build.vcs.number.<root number>, ignore the latter (extId cannot start with digit)
1160 if (vcsRootExtId != null) {
1161 LOG.debug("Build has more than one VCS root with same revision " + myRevision + ": " + vcsRootExtId + " and " +
1162 extId + ", cannot deduce VCS root extId");
1165 vcsRootExtId = extId;
1169 return vcsRootExtId;