TW-58811 Support 'update-ref --stdin'
authorVladislav Rassokhin <vladislav.rassokhin@jetbrains.com>
Thu, 17 Jan 2019 14:11:21 +0000 (17:11 +0300)
committerVladislav Rassokhin <vladislav.rassokhin@jetbrains.com>
Thu, 17 Jan 2019 14:13:25 +0000 (17:13 +0300)
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/GitFacade.java
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/NativeGitFacade.java
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/UpdaterImpl.java
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/command/UpdateRefBatchCommand.java [new file with mode: 0644]
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/command/impl/CommandUtil.java
git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/command/impl/UpdateRefBatchCommandImpl.java [new file with mode: 0644]

index ecc09c21251eee6a7a689deab8d4b6b0ce9cd9ff..1e6be01992df98205b3c19212f3842b0e2c665c2 100644 (file)
@@ -52,6 +52,9 @@ public interface GitFacade {
   UpdateRefCommand updateRef();
 
   @NotNull
+  UpdateRefBatchCommand updateRefBatch();
+
+  @NotNull
   CheckoutCommand checkout();
 
   @NotNull
index f2cd73dff9bc5a13a789397dfd89d7a98621258c..8285fea0c7a2d1df6159bb088a81520e0e724441 100644 (file)
@@ -140,6 +140,11 @@ public class NativeGitFacade implements GitFacade {
   }
 
   @NotNull
+  public UpdateRefBatchCommand updateRefBatch() {
+    return new UpdateRefBatchCommandImpl(createCommandLine());
+  }
+
+  @NotNull
   public CheckoutCommand checkout() {
     return new CheckoutCommandImpl(createCommandLine());
   }
index 63c4728019d62b05e51e7efc19820db92c2d7c69..8123a794017c5ae4d9567ae3d8d877f9b8883e71 100644 (file)
@@ -31,6 +31,7 @@ import jetbrains.buildServer.buildTriggers.vcs.git.agent.errors.GitIndexCorrupte
 import jetbrains.buildServer.buildTriggers.vcs.git.agent.errors.GitOutdatedIndexException;
 import jetbrains.buildServer.buildTriggers.vcs.git.agent.ssl.SSLInvestigator;
 import jetbrains.buildServer.log.Loggers;
+import jetbrains.buildServer.util.CollectionsUtil;
 import jetbrains.buildServer.util.FileUtil;
 import jetbrains.buildServer.util.StringUtil;
 import jetbrains.buildServer.vcs.*;
@@ -62,6 +63,7 @@ public class UpdaterImpl implements Updater {
   public final static GitVersion GIT_WITH_SPARSE_CHECKOUT = new GitVersion(1, 7, 4);
   public final static GitVersion BROKEN_SPARSE_CHECKOUT = new GitVersion(2, 7, 0);
   public final static GitVersion MIN_GIT_SSH_COMMAND = new GitVersion(2, 3, 0);//GIT_SSH_COMMAND was introduced in git 2.3.0
+  public final static GitVersion GIT_UPDATE_REFS_STDIN = new GitVersion(1, 8, 5); // update-refs with '--stdin' support
   /**
    * Git version supporting an empty credential helper - the only way to disable system/global/local cred helper
    */
@@ -895,10 +897,11 @@ public class UpdaterImpl implements Updater {
     GitFacade git = myGitFactory.create(workingDir);
     ShowRefResult showRefResult = git.showRef().call();
     Refs localRefs = new Refs(showRefResult.getValidRefs());
-    if (localRefs.isEmpty() && showRefResult.getInvalidRefs().isEmpty())
+    Set<String> invalidRefs = showRefResult.getInvalidRefs();
+    if (localRefs.isEmpty() && invalidRefs.isEmpty())
       return false;
-    for (String invalidRef : showRefResult.getInvalidRefs()) {
-      git.updateRef().setRef(invalidRef).delete().call();
+    if (!invalidRefs.isEmpty()) {
+      removeRefs(git, invalidRefs);
       outdatedRefsRemoved = true;
     }
     final Refs remoteRefs;
@@ -916,16 +919,38 @@ public class UpdaterImpl implements Updater {
     //tracking branches (refs/remote/origin/topic), while git remote origin prune
     //removes only the latter. We need that because in some cases git cannot handle
     //rename of the branch (TW-28735).
+    final List<String> localRefsToDelete = new ArrayList<String>();
     for (Ref localRef : localRefs.list()) {
       Ref correspondingRemoteRef = createCorrespondingRemoteRef(localRef);
       if (remoteRefs.isOutdated(correspondingRemoteRef)) {
-        git.updateRef().setRef(localRef.getName()).delete().call();
-        outdatedRefsRemoved = true;
+        localRefsToDelete.add(localRef.getName());
       }
     }
+    if (!localRefsToDelete.isEmpty()) {
+      removeRefs(git, localRefsToDelete);
+      outdatedRefsRemoved = true;
+    }
     return outdatedRefsRemoved;
   }
 
+  private void removeRefs(final GitFacade git, final Collection<String> invalidRefs) throws VcsException {
+    if (myPluginConfig.getGitVersion().isLessThan(UpdaterImpl.GIT_UPDATE_REFS_STDIN)) {
+      for (String invalidRef : invalidRefs) {
+        git.updateRef().setRef(invalidRef).delete().call();
+      }
+    } else {
+      List<List<String>> split = CollectionsUtil.split(new ArrayList<String>(invalidRefs), 1000);
+      for (final List<String> batch : split) {
+        if (batch.isEmpty()) continue;
+        UpdateRefBatchCommand command = git.updateRefBatch();
+        for (final String invalidRef : batch) {
+          command.delete(invalidRef, null);
+        }
+        command.call();
+      }
+    }
+  }
+
 
   @NotNull
   private Refs getRemoteRefs(@NotNull File workingDir) throws VcsException {
diff --git a/git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/command/UpdateRefBatchCommand.java b/git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/command/UpdateRefBatchCommand.java
new file mode 100644 (file)
index 0000000..46d115b
--- /dev/null
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2000-2019 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jetbrains.buildServer.buildTriggers.vcs.git.agent.command;
+
+import jetbrains.buildServer.vcs.VcsException;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Interface for 'update-ref --stdin'
+ * Requires at least git v1.8.5
+ */
+public interface UpdateRefBatchCommand extends BaseCommand {
+
+  // Supported commands
+  // update SP <ref> SP <newValue> [SP <oldValue>] LF
+  // create SP <ref> SP <newValue> LF
+  // delete SP <ref> [SP <oldValue>] LF
+  // verify SP <ref> [SP <oldValue>] LF
+  // option SP <opt> LF
+
+
+  @NotNull
+  UpdateRefBatchCommand update(@NotNull String ref, @NotNull String value, @Nullable String oldValue) throws VcsException;
+
+  @NotNull
+  UpdateRefBatchCommand create(@NotNull String ref, @NotNull String value) throws VcsException;
+
+  @NotNull
+  UpdateRefBatchCommand delete(@NotNull String ref, @Nullable String oldValue) throws VcsException;
+
+  @NotNull
+  UpdateRefBatchCommand verify(@NotNull String ref, @Nullable String oldValue) throws VcsException;
+
+  @NotNull
+  UpdateRefBatchCommand option(@NotNull String option) throws VcsException;
+
+  void call() throws VcsException;
+
+}
index 172e10845e2c71a9d7a0e7113b877577297eea85..1e277a7f4d704ec99db25e3ba74aef0eecb2969e 100644 (file)
@@ -104,7 +104,17 @@ public class CommandUtil {
     return runCommand(cli, DEFAULT_COMMAND_TIMEOUT_SEC, errorsLogLevel);
   }
 
-  public static ExecResult runCommand(@NotNull GitCommandLine cli, int timeoutSeconds, final String... errorsLogLevel) throws VcsException {
+  public static ExecResult runCommand(@NotNull GitCommandLine cli, byte[] input, final String... errorsLogLevel) throws VcsException {
+    return runCommand(cli, DEFAULT_COMMAND_TIMEOUT_SEC, input, errorsLogLevel);
+  }
+
+  public static ExecResult runCommand(@NotNull GitCommandLine cli, int timeoutSeconds, final String... errorsLogLevel)
+    throws VcsException {
+    return runCommand(cli, timeoutSeconds, null, errorsLogLevel);
+  }
+
+  public static ExecResult runCommand(@NotNull GitCommandLine cli, int timeoutSeconds, byte[] input, final String... errorsLogLevel)
+    throws VcsException {
     int attemptsLeft = 2;
     while (true) {
       try {
@@ -117,7 +127,8 @@ public class CommandUtil {
         cli.logStart(cmdStr);
         ByteArrayOutputStream stdoutBuffer = new ByteArrayOutputStream();
         ByteArrayOutputStream stderrBuffer = cli.createStderrBuffer();
-        ExecResult res = SimpleCommandLineProcessRunner.runCommandSecure(cli, cli.getCommandLineString(), null, new ProcessTimeoutCallback(timeoutSeconds), stdoutBuffer, stderrBuffer);
+        ExecResult res = SimpleCommandLineProcessRunner
+          .runCommandSecure(cli, cli.getCommandLineString(), input, new ProcessTimeoutCallback(timeoutSeconds), stdoutBuffer, stderrBuffer);
         cli.logFinish(cmdStr);
         CommandUtil.checkCommandFailed(cmdStr, res, errorsLogLevel);
         String out = res.getStdout().trim();
diff --git a/git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/command/impl/UpdateRefBatchCommandImpl.java b/git-agent/src/jetbrains/buildServer/buildTriggers/vcs/git/agent/command/impl/UpdateRefBatchCommandImpl.java
new file mode 100644 (file)
index 0000000..4337c85
--- /dev/null
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2000-2019 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jetbrains.buildServer.buildTriggers.vcs.git.agent.command.impl;
+
+import jetbrains.buildServer.ExecResult;
+import jetbrains.buildServer.buildTriggers.vcs.git.agent.GitCommandLine;
+import jetbrains.buildServer.buildTriggers.vcs.git.agent.command.UpdateRefBatchCommand;
+import jetbrains.buildServer.vcs.VcsException;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.springframework.util.FastByteArrayOutputStream;
+
+import java.io.IOException;
+
+import static org.apache.commons.codec.Charsets.UTF_8;
+
+
+public class UpdateRefBatchCommandImpl extends BaseCommandImpl implements UpdateRefBatchCommand {
+  private FastByteArrayOutputStream myInput = new FastByteArrayOutputStream();
+
+  public UpdateRefBatchCommandImpl(@NotNull GitCommandLine cmd) {
+    super(cmd);
+  }
+
+  @NotNull
+  @Override
+  public UpdateRefBatchCommand update(@NotNull final String ref, @NotNull final String value, @Nullable final String oldValue)
+    throws VcsException {
+    //update SP <ref> NUL <newValue> NUL [<oldValue>] NUL
+    try {
+      myInput.write("update".getBytes(UTF_8));
+      myInput.write(0x20);
+      myInput.write(ref.getBytes(UTF_8));
+      myInput.write(0x0);
+      myInput.write(value.getBytes(UTF_8));
+      myInput.write(0x0);
+      if (oldValue != null) {
+        myInput.write(oldValue.getBytes(UTF_8));
+      }
+      myInput.write(0x0);
+    } catch (IOException ignored) {
+      throw new VcsException("Failed to generate update-ref command binary input");
+    }
+    return this;
+  }
+
+
+  @NotNull
+  @Override
+  public UpdateRefBatchCommand create(@NotNull final String ref, @NotNull final String value) throws VcsException {
+    //create SP <ref> NUL <newValue> NUL
+    try {
+      myInput.write("create".getBytes(UTF_8));
+      myInput.write(0x20);
+      myInput.write(ref.getBytes(UTF_8));
+      myInput.write(0x0);
+    } catch (IOException ignored) {
+      throw new VcsException("Failed to generate update-ref command binary input");
+    }
+    return this;
+  }
+
+  @NotNull
+  @Override
+  public UpdateRefBatchCommand delete(@NotNull final String ref, @Nullable final String oldValue) throws VcsException {
+    //delete SP <ref> NUL [<oldValue>] NUL
+    try {
+      myInput.write("delete".getBytes(UTF_8));
+      myInput.write(0x20);
+      myInput.write(ref.getBytes(UTF_8));
+      myInput.write(0x0);
+      if (oldValue != null) {
+        myInput.write(oldValue.getBytes(UTF_8));
+      }
+      myInput.write(0x0);
+    } catch (IOException ignored) {
+      throw new VcsException("Failed to generate update-ref command binary input");
+    }
+    return this;
+  }
+
+  @NotNull
+  @Override
+  public UpdateRefBatchCommand verify(@NotNull final String ref, @Nullable final String oldValue) throws VcsException {
+    //verify SP <ref> NUL [<oldValue>] NUL
+    try {
+      myInput.write("verify".getBytes(UTF_8));
+      myInput.write(0x20);
+      myInput.write(ref.getBytes(UTF_8));
+      myInput.write(0x0);
+      if (oldValue != null) {
+        myInput.write(oldValue.getBytes(UTF_8));
+      }
+      myInput.write(0x0);
+    } catch (IOException ignored) {
+      throw new VcsException("Failed to generate update-ref command binary input");
+    }
+    return this;
+  }
+
+  @NotNull
+  @Override
+  public UpdateRefBatchCommand option(@NotNull final String option) throws VcsException {
+    //option SP <opt> NUL
+    try {
+      myInput.write("option".getBytes(UTF_8));
+      myInput.write(0x20);
+      myInput.write(option.getBytes(UTF_8));
+      myInput.write(0x0);
+    } catch (IOException ignored) {
+      throw new VcsException("Failed to generate update-ref command binary input");
+    }
+    return this;
+  }
+
+  public void call() throws VcsException {
+    GitCommandLine cmd = getCmd();
+    cmd.addParameter("update-ref");
+    cmd.addParameter("--stdin");
+    cmd.addParameter("-z");
+    byte[] input = myInput.toByteArray();
+    ExecResult r = CommandUtil.runCommand(cmd, input);
+    CommandUtil.failIfNotEmptyStdErr(cmd, r);
+  }
+}