Ability to fetch all heads only if build's commit is not found on the agent
authorDmitry Neverov <dmitry.neverov@gmail.com>
Mon, 23 Oct 2017 09:18:40 +0000 (11:18 +0200)
committerDmitry Neverov <dmitry.neverov@gmail.com>
Mon, 23 Oct 2017 09:18:40 +0000 (11:18 +0200)
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/AgentPluginConfig.java
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/FetchHeadsMode.java [new file with mode: 0644]
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/PluginConfigImpl.java
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/UpdaterImpl.java
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/UpdaterWithMirror.java
git-tests/src/jetbrains/buildServer/buildTriggers/vcs/git/tests/AgentVcsSupportTest.java

index 6d2f37e098d9e9115d2ab9aefdc50e3b1251f339..19df7348b8287729c2de7f001ddffbe5fa0fc1f4 100644 (file)
@@ -35,7 +35,8 @@ public interface AgentPluginConfig extends PluginConfig {
 
   boolean isDeleteTempFiles();
 
-  boolean isFetchAllHeads();
+  @NotNull
+  FetchHeadsMode getFetchHeadsMode();
 
   boolean isUseMainRepoUserForSubmodules();
 
diff --git a/git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/FetchHeadsMode.java b/git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/FetchHeadsMode.java
new file mode 100644 (file)
index 0000000..1d1fece
--- /dev/null
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jetbrains.buildServer.buildTriggers.vcs.git.agent;
+
+/**
+ * Specifies when all heads should be fetched on the agent.
+ */
+enum FetchHeadsMode {
+  /**
+   * If build revision is not found on the agent, the build's branch should
+   * be fetched first. If commit is still not found, then all heads should
+   * be fetched. This mode is used by default.
+   */
+  AFTER_BUILD_BRANCH,
+  /**
+   * If build revision is not found on the agent, all heads should be fetched.
+   * If commit is still not found and build's branch is not under refs/heads/, then
+   * the build's branch should be fetched as well.
+   */
+  BEFORE_BUILD_BRANCH,
+  /**
+   * Always fetch all branches, even if commit is found on the agent. If commit is
+   * not found after all branches fetch and build's branch is not under refs/heads/, then
+   * the build's branch should be fetched as well.
+   */
+  ALWAYS
+}
index 8b1206e5d968c53a3b1b7c73f4e27aa68c244a8d..281c5d6adea400ac78b1d4c88ed624690db54b20 100644 (file)
@@ -26,6 +26,7 @@ import org.apache.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
+import java.util.Map;
 
 /**
  * @author dmitry.neverov
@@ -168,11 +169,23 @@ public class PluginConfigImpl implements AgentPluginConfig {
   }
 
 
-  public boolean isFetchAllHeads() {
-    String value = myBuild.getSharedConfigParameters().get(FETCH_ALL_HEADS);
-    return Boolean.parseBoolean(value);
-  }
+  @NotNull
+  @Override
+  public FetchHeadsMode getFetchHeadsMode() {
+    Map<String, String> params = myBuild.getSharedConfigParameters();
+    String fetchAllHeads = params.get(FETCH_ALL_HEADS);
+    if (StringUtil.isEmpty(fetchAllHeads) || "false".equals(fetchAllHeads) || "afterBuildBranch".equals(fetchAllHeads))
+      return FetchHeadsMode.AFTER_BUILD_BRANCH;
 
+    if ("true".equals(fetchAllHeads) || "always".equals(fetchAllHeads))
+      return FetchHeadsMode.ALWAYS;
+
+    if ("beforeBuildBranch".equals(fetchAllHeads))
+      return FetchHeadsMode.BEFORE_BUILD_BRANCH;
+
+    LOG.warn("Unsupported value of the " + FETCH_ALL_HEADS + " parameter: '" + fetchAllHeads + "', treat it as false");
+    return FetchHeadsMode.AFTER_BUILD_BRANCH;
+  }
 
   public boolean isUseMainRepoUserForSubmodules() {
     String fromBuildConfiguration = myBuild.getSharedConfigParameters().get(USE_MAIN_REPO_USER_FOR_SUBMODULES);
index b8fe67e806e2529362e88d4938ce7f86c49a6054..343a5b6f042eec00712c1763dd903317b55c58aa 100644 (file)
@@ -580,28 +580,48 @@ public class UpdaterImpl implements Updater {
 
 
   protected void fetchFromOriginalRepository(boolean fetchRequired) throws VcsException {
-    if (myPluginConfig.isFetchAllHeads()) {
-      String msg = getForcedHeadsFetchMessage();
-      LOG.info(msg);
-      myLogger.message(msg);
-
-      fetchAllBranches();
-      if (!myFullBranchName.startsWith("refs/heads/")) {
-        Ref remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
-        if (fetchRequired || remoteRef == null || !myRevision.equals(remoteRef.getObjectId().name()) || !hasRevision(myTargetDirectory, myRevision))
-          fetchDefaultBranch();
-      }
-    } else {
-      Ref remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
-      if (!fetchRequired && remoteRef != null && myRevision.equals(remoteRef.getObjectId().name()) && hasRevision(myTargetDirectory, myRevision))
-        return;
-      myLogger.message("Commit '" + myRevision + "' is not found in local clone. Running 'git fetch'...");
-      fetchDefaultBranch();
-      if (hasRevision(myTargetDirectory, myRevision))
-        return;
-      myLogger.message("Commit still not found after fetching main branch. Fetching more branches.");
-      fetchAllBranches();
+    Ref remoteRef;
+    FetchHeadsMode fetchHeadsMode = myPluginConfig.getFetchHeadsMode();
+    switch (fetchHeadsMode) {
+      case ALWAYS:
+        String msg = getForcedHeadsFetchMessage();
+        LOG.info(msg);
+        myLogger.message(msg);
+
+        fetchAllBranches();
+        if (!myFullBranchName.startsWith("refs/heads/")) {
+          remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
+          if (fetchRequired || remoteRef == null || !myRevision.equals(remoteRef.getObjectId().name()) || !hasRevision(myTargetDirectory, myRevision))
+            fetchDefaultBranch();
+        }
+        break;
+      case BEFORE_BUILD_BRANCH:
+        remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
+        if (!fetchRequired && remoteRef != null && myRevision.equals(remoteRef.getObjectId().name()) && hasRevision(myTargetDirectory, myRevision))
+          return;
+        myLogger.message("Commit '" + myRevision + "' is not found in local clone. Running 'git fetch'...");
+        fetchAllBranches();
+        if (!myFullBranchName.startsWith("refs/heads/")) {
+          remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
+          if (fetchRequired || remoteRef == null || !myRevision.equals(remoteRef.getObjectId().name()) || !hasRevision(myTargetDirectory, myRevision))
+            fetchDefaultBranch();
+        }
+        break;
+      case AFTER_BUILD_BRANCH:
+        remoteRef = getRef(myTargetDirectory, GitUtils.createRemoteRef(myFullBranchName));
+        if (!fetchRequired && remoteRef != null && myRevision.equals(remoteRef.getObjectId().name()) && hasRevision(myTargetDirectory, myRevision))
+          return;
+        myLogger.message("Commit '" + myRevision + "' is not found in local clone. Running 'git fetch'...");
+        fetchDefaultBranch();
+        if (hasRevision(myTargetDirectory, myRevision))
+          return;
+        myLogger.message("Commit still not found after fetching main branch. Fetching more branches.");
+        fetchAllBranches();
+        break;
+      default:
+        throw new VcsException("Unknown FetchHeadsMode: " + fetchHeadsMode);
     }
+
     if (hasRevision(myTargetDirectory, myRevision))
       return;
 
index 137f5ec36d77b788239f20999fc57961e8c6c8f7..d49af6bbd0d6564b6181833a2b97d2a23182302c 100644 (file)
@@ -106,27 +106,39 @@ public class UpdaterWithMirror extends UpdaterImpl {
         }
       }
     }
+    FetchHeadsMode fetchHeadsMode = myPluginConfig.getFetchHeadsMode();
     Ref ref = getRef(bareRepositoryDir, myFullBranchName);
     if (ref == null)
       fetchRequired = true;
-    if (!fetchRequired && !myPluginConfig.isFetchAllHeads())
+    if (!fetchRequired && fetchHeadsMode != FetchHeadsMode.ALWAYS)
       return;
     if (!newMirror && optimizeMirrorBeforeFetch()) {
       GitFacade git = myGitFactory.create(bareRepositoryDir);
       git.gc().call();
       git.repack().call();
     }
-    if (myPluginConfig.isFetchAllHeads()) {
-      String msg = getForcedHeadsFetchMessage();
-      LOG.info(msg);
-      myLogger.message(msg);
 
-      fetchMirror(repeatFetchAttempt, bareRepositoryDir, "+refs/heads/*:refs/heads/*", false);
-    } else {
-      fetchMirror(repeatFetchAttempt, bareRepositoryDir, "+" + myFullBranchName + ":" + GitUtils.expandRef(myFullBranchName), false);
-      if (hasRevision(bareRepositoryDir, myRevision))
-        return;
-      fetchMirror(repeatFetchAttempt, bareRepositoryDir, "+refs/heads/*:refs/heads/*", false);
+    switch (fetchHeadsMode) {
+      case ALWAYS:
+        String msg = getForcedHeadsFetchMessage();
+        LOG.info(msg);
+        myLogger.message(msg);
+        fetchMirror(repeatFetchAttempt, bareRepositoryDir, "+refs/heads/*:refs/heads/*");
+        if (!myFullBranchName.startsWith("refs/heads/") && !hasRevision(bareRepositoryDir, myRevision))
+          fetchMirror(repeatFetchAttempt, bareRepositoryDir, "+" + myFullBranchName + ":" + GitUtils.expandRef(myFullBranchName));
+        break;
+      case BEFORE_BUILD_BRANCH:
+        fetchMirror(repeatFetchAttempt, bareRepositoryDir, "+refs/heads/*:refs/heads/*");
+        if (!myFullBranchName.startsWith("refs/heads/") && !hasRevision(bareRepositoryDir, myRevision))
+          fetchMirror(repeatFetchAttempt, bareRepositoryDir, "+" + myFullBranchName + ":" + GitUtils.expandRef(myFullBranchName));
+        break;
+      case AFTER_BUILD_BRANCH:
+        fetchMirror(repeatFetchAttempt, bareRepositoryDir, "+" + myFullBranchName + ":" + GitUtils.expandRef(myFullBranchName));
+        if (!hasRevision(bareRepositoryDir, myRevision))
+          fetchMirror(repeatFetchAttempt, bareRepositoryDir, "+refs/heads/*:refs/heads/*");
+        break;
+      default:
+        throw new VcsException("Unknown FetchHeadsMode: " + fetchHeadsMode);
     }
   }
 
@@ -138,11 +150,10 @@ public class UpdaterWithMirror extends UpdaterImpl {
 
   private void fetchMirror(boolean repeatFetchAttempt,
                            @NotNull File repositoryDir,
-                           @NotNull String refspec,
-                           boolean shallowClone) throws VcsException {
+                           @NotNull String refspec) throws VcsException {
     removeRefLocks(repositoryDir);
     try {
-      fetch(repositoryDir, refspec, shallowClone);
+      fetch(repositoryDir, refspec, false);
     } catch (VcsException e) {
       if (myPluginConfig.isFailOnCleanCheckout() || !repeatFetchAttempt || !shouldFetchFromScratch(e))
         throw e;
@@ -150,7 +161,7 @@ public class UpdaterWithMirror extends UpdaterImpl {
         GitFacade git = myGitFactory.create(repositoryDir);
         git.init().setBare(true).call();
         configureRemoteUrl(repositoryDir);
-        fetch(repositoryDir, refspec, shallowClone);
+        fetch(repositoryDir, refspec, false);
       } else {
         LOG.info("Failed to delete repository " + repositoryDir + " after failed checkout, clone repository in another directory");
         myMirrorManager.invalidate(repositoryDir);
index 80f4a58ff250c55800c0fee896d04dc7e8b3093f..f6689e12e606a87091a7c5eacf3b747997884b56 100644 (file)
@@ -921,6 +921,39 @@ public class AgentVcsSupportTest {
   }
 
 
+  @Test(dataProvider = "mirrors")
+  public void fetch_all_heads_before_build_branch(boolean useMirrors) throws Exception {
+    AgentRunningBuild build = createRunningBuild(map(PluginConfigImpl.USE_MIRRORS, String.valueOf(useMirrors),
+                                                     PluginConfigImpl.FETCH_ALL_HEADS, "beforeBuildBranch"));
+
+    myVcsSupport.updateSources(myRoot, CheckoutRules.DEFAULT, "465ad9f630e451b9f2b782ffb09804c6a98c4bb9", myCheckoutDir, build, false);
+
+    Repository remoteRepo = new RepositoryBuilder().setBare().setGitDir(myMainRepo).build();
+    Set<String> remoteHeads = remoteRepo.getRefDatabase().getRefs("refs/heads/").keySet();
+
+    //local repo should contain all heads since build's commit wasn't on the agent
+    Repository r = new RepositoryBuilder().setWorkTree(myCheckoutDir).build();
+    then(r.getRefDatabase().getRefs("refs/remotes/origin/").keySet()).containsAll(remoteHeads);
+  }
+
+
+  @Test(dataProvider = "mirrors")
+  public void fetch_all_heads_before_build_branch_commit_found(boolean useMirrors) throws Exception {
+    //run build to make sure commit is on the agent
+    AgentRunningBuild build = createRunningBuild(map(PluginConfigImpl.USE_MIRRORS, String.valueOf(useMirrors)));
+    myVcsSupport.updateSources(myRoot, CheckoutRules.DEFAULT, "465ad9f630e451b9f2b782ffb09804c6a98c4bb9", myCheckoutDir, build, false);
+
+    //run build with fetch_all_head before build's branch
+    build = createRunningBuild(map(PluginConfigImpl.USE_MIRRORS, String.valueOf(useMirrors),
+                                   PluginConfigImpl.FETCH_ALL_HEADS, "beforeBuildBranch"));
+    myVcsSupport.updateSources(myRoot, CheckoutRules.DEFAULT, "465ad9f630e451b9f2b782ffb09804c6a98c4bb9", myCheckoutDir, build, false);
+
+    //local repo shouldn't contain all heads since build commit was already on the agent and no fetch is required
+    Repository r = new RepositoryBuilder().setWorkTree(myCheckoutDir).build();
+    then(r.getRefDatabase().getRefs("refs/remotes/origin/").keySet()).containsOnly("master");
+  }
+
+
   private void removeTag(@NotNull File dotGitDir, @NotNull String tagName) {
     delete(tagFile(dotGitDir, tagName));
   }