2 * Copyright 2000-2018 JetBrains s.r.o.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package jetbrains.buildServer.buildTriggers.vcs.git;
19 import com.googlecode.javaewah.EWAHCompressedBitmap;
20 import com.intellij.openapi.diagnostic.Logger;
21 import com.intellij.openapi.util.SystemInfo;
22 import com.jcraft.jsch.*;
23 import gnu.trove.TObjectHashingStrategy;
24 import jetbrains.buildServer.agent.ClasspathUtil;
25 import jetbrains.buildServer.buildTriggers.vcs.git.patch.GitPatchProcess;
26 import jetbrains.buildServer.messages.serviceMessages.ServiceMessage;
27 import jetbrains.buildServer.serverSide.CachePaths;
28 import jetbrains.buildServer.serverSide.TeamCityProperties;
29 import jetbrains.buildServer.serverSide.crypt.EncryptUtil;
30 import jetbrains.buildServer.util.Dates;
31 import jetbrains.buildServer.util.DiagnosticUtil;
32 import jetbrains.buildServer.util.FileUtil;
33 import jetbrains.buildServer.vcs.AbstractPatchBuilder;
34 import jetbrains.buildServer.vcs.VcsException;
35 import jetbrains.buildServer.vcs.VcsPersonalSupport;
36 import jetbrains.buildServer.vcs.VcsRoot;
37 import jetbrains.buildServer.vcs.patches.LowLevelPatchBuilder;
38 import jetbrains.buildServer.vcs.patches.PatchBuilderImpl;
39 import org.apache.commons.codec.Decoder;
40 import org.apache.commons.logging.LogFactory;
41 import org.apache.http.HttpEntity;
42 import org.apache.http.client.HttpClient;
43 import org.apache.log4j.BasicConfigurator;
44 import org.apache.log4j.Layout;
45 import org.eclipse.jgit.lib.ProgressMonitor;
46 import org.eclipse.jgit.transport.http.apache.HttpClientConnectionFactory;
47 import org.jdom.Element;
48 import org.jetbrains.annotations.NotNull;
49 import org.jetbrains.annotations.Nullable;
50 import org.quartz.CronExpression;
53 import java.lang.management.ManagementFactory;
54 import java.lang.management.OperatingSystemMXBean;
55 import java.text.ParseException;
57 import java.util.concurrent.TimeUnit;
59 import static com.intellij.openapi.util.text.StringUtil.isEmpty;
60 import static com.intellij.openapi.util.text.StringUtil.isEmptyOrSpaces;
61 import static java.util.Arrays.asList;
62 import static jetbrains.buildServer.util.CollectionsUtil.setOf;
65 * @author dmitry.neverov
67 public class PluginConfigImpl implements ServerPluginConfig {
69 public static final String TEAMCITY_GIT_IDLE_TIMEOUT_SECONDS = "teamcity.git.idle.timeout.seconds";
70 public static final String MAP_FULL_PATH_PERSISTENT_CACHES = "teamcity.git.persistentCacheEnabled";
71 private static final String TEAMCITY_GIT_SSH_PROXY_TYPE = "teamcity.git.sshProxyType";
72 private static final String TEAMCITY_GIT_SSH_PROXY_HOST = "teamcity.git.sshProxyHost";
73 private static final String TEAMCITY_GIT_SSH_PROXY_PORT = "teamcity.git.sshProxyPort";
74 private static final String TEAMCITY_GIT_ALWAYS_CHECK_CIPHERS = "teamcity.git.always.check.ciphers";
75 private static final String HTTP_CONNECTION_FACTORY = "teamcity.git.httpConnectionFactory";
76 private static final String HTTP_CONNECTION_SSL_PROTOCOL = "teamcity.git.httpConnectionSslProtocol";
77 private static final String MONITORING_FILE_THRESHOLD_SECONDS = "teamcity.git.monitoringFileThresholdSeconds";
78 public static final String CREATE_NEW_CONNECTION_FOR_PRUNE = "teamcity.git.newConnectionForPrune";
79 public static final String IGNORE_MISSING_REMOTE_REF = "teamcity.git.ignoreMissingRemoteRef";
80 private static final String ACCESS_TIME_UPDATE_RATE_MINUTES = "teamcity.git.accessTimeUpdateRateMinutes";
81 private static final String MERGE_RETRY_ATTEMPTS = "teamcity.git.mergeRetryAttemps";
82 private static final String GET_REPOSITORY_STATE_TIMEOUT_SECONDS = "teamcity.git.repositoryStateTimeoutSeconds";
83 private final static Logger LOG = Logger.getInstance(PluginConfigImpl.class.getName());
84 private final static int GB = 1024 * 1024 * 1024;//bytes
85 private final File myCachesDir;
86 private final Set<String> myFetcherPropertyNames = setOf(TEAMCITY_GIT_IDLE_TIMEOUT_SECONDS,
87 TEAMCITY_GIT_SSH_PROXY_TYPE,
88 TEAMCITY_GIT_SSH_PROXY_HOST,
89 TEAMCITY_GIT_SSH_PROXY_PORT,
90 TEAMCITY_GIT_ALWAYS_CHECK_CIPHERS,
91 HTTP_CONNECTION_FACTORY,
92 HTTP_CONNECTION_SSL_PROTOCOL,
93 Constants.AMAZON_HOSTS,
94 MONITORING_FILE_THRESHOLD_SECONDS,
95 CREATE_NEW_CONNECTION_FOR_PRUNE,
96 GET_REPOSITORY_STATE_TIMEOUT_SECONDS,
97 IGNORE_MISSING_REMOTE_REF);
99 public PluginConfigImpl() {
103 public PluginConfigImpl(@NotNull final CachePaths paths) {
104 myCachesDir = paths.getCacheDirectory("git");
108 public static boolean isTeamcitySshKeysEnabled() {
109 return TeamCityProperties.getBooleanOrTrue("teamcity.git.enableTeamcitySshKeys");
114 public File getCachesDir() {
115 if (myCachesDir == null)
116 throw new IllegalStateException("Caches dir is not initialized");
121 public int getStreamFileThresholdMb() {
122 int defaultThreshold = 128;
123 int threshold = TeamCityProperties.getInteger("teamcity.git.stream.file.threshold.mb", defaultThreshold);
130 public String getFetchProcessJavaPath() {
131 final String jdkHome = System.getProperty("java.home");
132 File defaultJavaExec = new File(jdkHome.replace('/', File.separatorChar) + File.separator + "bin" + File.separator + "java");
133 return TeamCityProperties.getProperty("teamcity.git.fetch.process.java.exec", defaultJavaExec.getAbsolutePath());
137 public String getFetchProcessMaxMemory() {
138 String maxMemory = getExplicitFetchProcessMaxMemory();
139 if (!isEmpty(maxMemory))
142 Class.forName("com.sun.management.OperatingSystemMXBean");
143 } catch (ClassNotFoundException e) {
146 OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
147 if (osBean instanceof com.sun.management.OperatingSystemMXBean) {
148 long freeRAM = ((com.sun.management.OperatingSystemMXBean) osBean).getFreePhysicalMemorySize();
156 public String getGcProcessMaxMemory() {
157 String xmx = TeamCityProperties.getProperty("teamcity.git.gcXmx");
161 Class.forName("com.sun.management.OperatingSystemMXBean");
162 } catch (ClassNotFoundException e) {
165 OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
166 if (osBean instanceof com.sun.management.OperatingSystemMXBean) {
167 long freeRAM = ((com.sun.management.OperatingSystemMXBean) osBean).getFreePhysicalMemorySize();
175 public String getExplicitFetchProcessMaxMemory() {
176 return TeamCityProperties.getPropertyOrNull("teamcity.git.fetch.process.max.memory");
179 public boolean isSeparateProcessForFetch() {
180 return TeamCityProperties.getBooleanOrTrue("teamcity.git.fetch.separate.process");
184 public boolean isSeparateProcessForPatch() {
185 return TeamCityProperties.getBooleanOrTrue("teamcity.git.buildPatchInSeparateProcess");
188 public boolean isRunNativeGC() {
189 return TeamCityProperties.getBooleanOrTrue("teamcity.server.git.gc.enabled");
192 public boolean isRunJGitGC() {
193 return TeamCityProperties.getBoolean("teamcity.git.jgitGcEnabled");
196 public String getPathToGit() {
197 return TeamCityProperties.getProperty("teamcity.server.git.executable.path", "git");
200 public int getNativeGCQuotaMinutes() {
201 return TeamCityProperties.getInteger("teamcity.server.git.gc.quota.minutes", 60);
204 public String getFetchClasspath() {
205 Set<Class> classes = fetchProcessClasses();
206 return ClasspathUtil.composeClasspath(classes.toArray(new Class[classes.size()]), null, null);
210 public String getFetcherClassName() {
211 return Fetcher.class.getName();
214 public String getPatchClasspath() {
215 Set<Class> classes = fetchProcessClasses();
216 classes.add(AbstractPatchBuilder.class);
217 classes.add(PatchBuilderImpl.class);
218 classes.add(LowLevelPatchBuilder.class);
219 classes.add(org.slf4j.Logger.class);
220 classes.add(org.slf4j.impl.StaticLoggerBinder.class);
221 classes.add(EWAHCompressedBitmap.class);
222 return ClasspathUtil.composeClasspath(classes.toArray(new Class[classes.size()]), null, null);
225 public String getPatchBuilderClassName() {
226 return GitPatchProcess.class.getName();
229 public boolean passEnvToChildProcess() {
230 return TeamCityProperties.getBooleanOrTrue("teamcity.git.passEnvToChildProcess");
233 private Set<Class> fetchProcessClasses() {
234 Set<Class> result = new HashSet<Class>();
235 result.addAll(asList(
238 ProgressMonitor.class,
239 VcsPersonalSupport.class,
244 TObjectHashingStrategy.class,
246 DiagnosticUtil.class,
251 BasicConfigurator.class,
252 HttpClientConnectionFactory.class,
257 ServiceMessage.class,
258 org.slf4j.Logger.class,
259 org.slf4j.impl.StaticLoggerBinder.class,
260 EWAHCompressedBitmap.class
262 Collections.addAll(result, GitVcsSupport.class.getInterfaces());
266 public int getFixedSubmoduleCommitSearchDepth() {
267 return TeamCityProperties.getInteger("teamcity.server.git.fixed.submodule.commit.search.depth", 100);
270 public int getIdleTimeoutSeconds() {
271 return TeamCityProperties.getInteger(TEAMCITY_GIT_IDLE_TIMEOUT_SECONDS, DEFAULT_IDLE_TIMEOUT);
274 public int getFetchTimeout() {
275 int deprecatedFetchTimeout = TeamCityProperties.getInteger("teamcity.git.fetch.timeout", DEFAULT_IDLE_TIMEOUT);
276 int idleTimeout = getIdleTimeoutSeconds();
277 if (deprecatedFetchTimeout > idleTimeout)
278 return deprecatedFetchTimeout;
283 public int getPushTimeoutSeconds() {
284 return TeamCityProperties.getInteger("teamcity.git.pushIdleTimeoutSeconds", getIdleTimeoutSeconds());
287 public int getRepositoryStateTimeoutSeconds() {
288 return TeamCityProperties.getInteger(GET_REPOSITORY_STATE_TIMEOUT_SECONDS, 60);
291 public int getPatchProcessIdleTimeoutSeconds() {
292 return TeamCityProperties.getInteger("teamcity.git.patchProcessIdleTimeoutSeconds", 1800);
295 public long getMirrorExpirationTimeoutMillis() {
296 int days = TeamCityProperties.getInteger("teamcity.git.mirror.expiration.timeout.days", 7);
297 return days * Dates.ONE_DAY;
300 private void addProxySettingsForSeparateProcess(@NotNull List<String> options) {
301 addHttpProxyHost(options);
302 addHttpProxyPort(options);
303 addHttpNonProxyHosts(options);
304 addHttpsProxyHost(options);
305 addHttpsProxyPort(options);
306 addSshProxySettings(options);
310 public List<String> getOptionsForSeparateProcess() {
311 List<String> options = new ArrayList<String>();
312 addProxySettingsForSeparateProcess(options);
313 addSslTrustStoreSettingsForSeparateProcess(options);
314 addInheritedOption(options, "java.net.preferIPv6Addresses");
315 addInheritedOption(options, "jsse.enableSNIExtension");
319 private void addSslTrustStoreSettingsForSeparateProcess(@NotNull List<String> options) {
320 addInheritedOption(options, "javax.net.ssl.trustStore");
321 addInheritedOption(options, "javax.net.ssl.trustStorePassword");
324 private void addInheritedOption(@NotNull List<String> options, @NotNull String key) {
325 String value = System.getProperty(key);
327 options.add("-D" + key + "=" + value);
330 public Proxy getJschProxy() {
331 String sshProxyType = TeamCityProperties.getProperty(TEAMCITY_GIT_SSH_PROXY_TYPE);
332 if (isEmpty(sshProxyType))
334 String sshProxyHost = TeamCityProperties.getProperty(TEAMCITY_GIT_SSH_PROXY_HOST);
335 if (isEmpty(sshProxyHost))
337 int sshProxyPort = TeamCityProperties.getInteger(TEAMCITY_GIT_SSH_PROXY_PORT, -1);
338 if ("http".equals(sshProxyType)) {
339 return sshProxyPort != -1 ? new ProxyHTTP(sshProxyHost, sshProxyPort) : new ProxyHTTP(sshProxyHost);
341 if ("socks4".equals(sshProxyType)) {
342 return sshProxyPort != -1 ? new ProxySOCKS4(sshProxyHost, sshProxyPort) : new ProxySOCKS4(sshProxyHost);
344 if ("socks5".equals(sshProxyType)) {
345 return sshProxyPort != -1 ? new ProxySOCKS5(sshProxyHost, sshProxyPort) : new ProxySOCKS5(sshProxyHost);
350 private void addHttpProxyHost(@NotNull final List<String> proxySettings) {
351 String httpProxyHost = TeamCityProperties.getProperty("http.proxyHost");
352 if (!isEmpty(httpProxyHost))
353 proxySettings.add("-Dhttp.proxyHost=" + httpProxyHost);
356 private void addHttpProxyPort(List<String> proxySettings) {
357 int httpProxyPort = TeamCityProperties.getInteger("http.proxyPort", -1);
358 if (httpProxyPort != -1)
359 proxySettings.add("-Dhttp.proxyPort=" + httpProxyPort);
362 private void addHttpNonProxyHosts(List<String> proxySettings) {
363 String httpNonProxyHosts = TeamCityProperties.getProperty("http.nonProxyHosts");
364 if (!isEmpty(httpNonProxyHosts)) {
365 if (SystemInfo.isUnix) {
366 proxySettings.add("-Dhttp.nonProxyHosts=" + httpNonProxyHosts);
368 proxySettings.add("-Dhttp.nonProxyHosts=\"" + httpNonProxyHosts + "\"");
373 private void addHttpsProxyHost(List<String> proxySettings) {
374 String httpsProxyHost = TeamCityProperties.getProperty("https.proxyHost");
375 if (!isEmpty(httpsProxyHost))
376 proxySettings.add("-Dhttps.proxyHost=" + httpsProxyHost);
379 private void addHttpsProxyPort(List<String> proxySettings) {
380 int httpsProxyPort = TeamCityProperties.getInteger("https.proxyPort", -1);
381 if (httpsProxyPort != -1)
382 proxySettings.add("-Dhttps.proxyPort=" + httpsProxyPort);
385 private void addSshProxySettings(List<String> proxySettings) {
386 String sshProxyType = TeamCityProperties.getProperty(TEAMCITY_GIT_SSH_PROXY_TYPE);
387 if (!isEmpty(sshProxyType))
388 proxySettings.add("-Dteamcity.git.sshProxyType=" + sshProxyType);
389 String sshProxyHost = TeamCityProperties.getProperty(TEAMCITY_GIT_SSH_PROXY_HOST);
390 if (!isEmpty(sshProxyHost))
391 proxySettings.add("-Dteamcity.git.sshProxyHost=" + sshProxyHost);
392 int sshProxyPort = TeamCityProperties.getInteger(TEAMCITY_GIT_SSH_PROXY_PORT, -1);
393 if (sshProxyPort != -1)
394 proxySettings.add("-Dteamcity.git.sshProxyPort=" + sshProxyPort);
398 public String getMonitoringDirName() {
402 public int getMonitoringExpirationTimeoutHours() {
403 return TeamCityProperties.getInteger("teamcity.git.monitoring.expiration.timeout.hours", 24);
407 public long getMonitoringFileThresholdMillis() {
408 int thresholdSeconds = TeamCityProperties.getInteger(MONITORING_FILE_THRESHOLD_SECONDS, 60 * 10 /*10 minutes*/);
409 return thresholdSeconds * 1000L;
412 public boolean alwaysCheckCiphers() {
413 return TeamCityProperties.getBoolean(TEAMCITY_GIT_ALWAYS_CHECK_CIPHERS);
416 public boolean verboseGetContentLog() {
417 return TeamCityProperties.getBoolean("teamcity.git.verbose.get.content.log");
420 public boolean verboseTreeWalkLog() {
421 return TeamCityProperties.getBoolean("teamcity.git.verbose.tree.walk.log");
424 public int getMapFullPathRevisionCacheSize() {
425 return TeamCityProperties.getInteger("teamcity.git.map.full.path.revision.cache.size", 100);
428 public long getConnectionRetryIntervalMillis() {
429 return TeamCityProperties.getInteger("teamcity.git.connectionRetryIntervalSeconds", 4) * 1000L;
432 public int getConnectionRetryAttempts() {
433 return TeamCityProperties.getInteger("teamcity.git.connectionRetryAttempts", 3);
436 public boolean ignoreFetchedCommits() {
437 return TeamCityProperties.getBoolean("teamcity.git.mapFullPathIgnoresFetchedCommits");
441 public CronExpression getCleanupCronExpression() {
442 String cron = TeamCityProperties.getProperty("teamcity.git.cleanupCron", "0 0 2 * * ? *");
446 return new CronExpression(cron);
447 } catch (ParseException e) {
448 LOG.warn("Wrong cron expression " + cron, e);
454 public Map<String, String> getFetcherProperties() {
455 Map<String, String> fetcherProps = new HashMap<String, String>();
456 for (String propName : myFetcherPropertyNames) {
457 fetcherProps.put(propName, TeamCityProperties.getProperty(propName));
462 public boolean usePerBranchFetch() {
463 return TeamCityProperties.getBoolean("teamcity.git.usePerBranchFetch");
466 public int getHttpsSoLinger() {
467 return TeamCityProperties.getInteger("teamcity.git.httpsSoLinger", 0);
470 public int getListFilesTTLSeconds() {
471 return TeamCityProperties.getInteger("teamcity.git.listFilesTTLSeconds", 60);
475 public String getHttpConnectionFactory() {
476 return TeamCityProperties.getProperty(HTTP_CONNECTION_FACTORY, "httpClient");
480 public String getHttpConnectionSslProtocol() {
481 return TeamCityProperties.getProperty(HTTP_CONNECTION_SSL_PROTOCOL, "SSL");
484 public static boolean showKnownHostsDbOption() {
485 return TeamCityProperties.getBoolean("teamcity.git.showKnownHostsDbOption");
489 public List<String> getAmazonHosts() {
490 String amazonHosts = TeamCityProperties.getProperty(Constants.AMAZON_HOSTS);
491 if (isEmptyOrSpaces(amazonHosts))
492 return Collections.emptyList();
493 String[] hosts = amazonHosts.split(",");
494 return asList(hosts);
497 public boolean useTagPackHeuristics() {
498 return TeamCityProperties.getBoolean("teamcity.git.useTagPackHeuristics");
501 public boolean analyzeTagsInPackHeuristics() {
502 return TeamCityProperties.getBoolean("teamcity.git.tagPackHeuristicsAnalyzeTags");
505 public boolean checkLabeledCommitIsInRemoteRepository() {
506 return TeamCityProperties.getBooleanOrTrue("teamcity.git.tagPackHeuristicsCheckCommit");
510 public boolean failLabelingWhenPackHeuristicsFails() {
511 return TeamCityProperties.getBoolean("teamcity.git.failLabelingWhenPackHeuristicsFails");
515 public boolean persistentCacheEnabled() {
516 return TeamCityProperties.getBooleanOrTrue(MAP_FULL_PATH_PERSISTENT_CACHES);
520 public boolean logRemoteRefs() {
521 return TeamCityProperties.getBoolean("teamcity.git.logRemoteRefs");
525 public boolean createNewConnectionForPrune() {
526 return TeamCityProperties.getBooleanOrTrue(CREATE_NEW_CONNECTION_FOR_PRUNE);
530 public long getAccessTimeUpdateRateMinutes() {
531 return TeamCityProperties.getLong(ACCESS_TIME_UPDATE_RATE_MINUTES, 5);
534 public boolean ignoreMissingRemoteRef() {
535 return TeamCityProperties.getBoolean(IGNORE_MISSING_REMOTE_REF);
539 public int getMergeRetryAttempts() {
540 return TeamCityProperties.getInteger(MERGE_RETRY_ATTEMPTS, 2);
544 public boolean runInPlaceGc() {
545 return TeamCityProperties.getBoolean("teamcity.git.runInPlaceGc");
549 public int getRepackIdleTimeoutSeconds() {
550 return TeamCityProperties.getInteger("teamcity.git.repackIdleTimeoutSeconds", (int) TimeUnit.MINUTES.toSeconds(30));
554 public int getPackRefsIdleTimeoutSeconds() {
555 return TeamCityProperties.getInteger("teamcity.git.packRefsIdleTimeoutSeconds", (int) TimeUnit.MINUTES.toSeconds(5));
559 public boolean treatMissingBranchTipAsRecoverableError() {
560 return TeamCityProperties.getBooleanOrTrue("teamcity.git.treatMissingCommitAsRecoverableError");
564 public boolean reportPerParentChangedFiles() {
565 return TeamCityProperties.getBooleanOrTrue("teamcity.git.reportPerParentChangedFiles");
570 public List<String> getRecoverableFetchErrorMessages() {
571 String errorsStr = TeamCityProperties.getProperty("teamcity.git.recoverableFetchErrors");
572 if (isEmptyOrSpaces(errorsStr))
573 return Collections.emptyList();
574 String[] errors = errorsStr.split(",");
575 return asList(errors);