revert "move ktor to community build scripts, get rid of http jdk client in favour...
authorVladimir Krivosheev <vladimir.krivosheev@jetbrains.com>
Sun, 2 Oct 2022 07:58:51 +0000 (09:58 +0200)
committerintellij-monorepo-bot <intellij-monorepo-bot-no-reply@jetbrains.com>
Sun, 2 Oct 2022 07:59:38 +0000 (07:59 +0000)
GitOrigin-RevId: 00186f85a85ada65ea2732b320cad0c919744dfd

26 files changed:
.idea/libraries/ktor_client_encoding.xml [deleted file]
build/tasks/src/org/jetbrains/intellij/build/io/http.kt [new file with mode: 0644]
build/tasks/src/org/jetbrains/intellij/build/tasks/brokenPlugins.kt [moved from platform/build-scripts/src/org/jetbrains/intellij/build/impl/brokenPlugins.kt with 92% similarity]
build/tasks/test/org/jetbrains/intellij/build/tasks/TaskTest.kt [new file with mode: 0644]
platform/bootstrap/dev/src/org/jetbrains/intellij/build/devServer/DevMain.kt
platform/build-scripts/dev-server/intellij.platform.devBuildServer.iml
platform/build-scripts/dev-server/src/DevIdeaBuilder.kt
platform/build-scripts/dev-server/src/IdeBuilder.kt
platform/build-scripts/intellij.platform.buildScripts.iml
platform/build-scripts/src/org/jetbrains/intellij/build/ProprietaryBuildTools.kt
platform/build-scripts/src/org/jetbrains/intellij/build/SignTool.kt
platform/build-scripts/src/org/jetbrains/intellij/build/fus/StatisticsRecorderBundledMetadataProvider.kt
platform/build-scripts/src/org/jetbrains/intellij/build/http.kt [moved from platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/http.kt with 79% similarity]
platform/build-scripts/src/org/jetbrains/intellij/build/impl/BundledRuntime.kt
platform/build-scripts/src/org/jetbrains/intellij/build/impl/BundledRuntimeImpl.kt
platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt
platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDmgBuilder.kt
platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt
platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/CompilationPartsUtil.kt
platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/upload.kt
platform/build-scripts/src/org/jetbrains/intellij/build/kotlin/KotlinBinaries.kt
platform/build-scripts/src/org/jetbrains/intellij/build/kotlin/KotlinCompilerDependencyDownloader.kt
platform/build-scripts/src/org/jetbrains/intellij/build/ktor.kt [deleted file]
platform/build-scripts/src/org/jetbrains/intellij/build/striped.kt [deleted file]
platform/build-scripts/testFramework/src/org/jetbrains/intellij/build/testFramework/buildScriptTestUtils.kt
platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/BundledRuntimeTest.kt

diff --git a/.idea/libraries/ktor_client_encoding.xml b/.idea/libraries/ktor_client_encoding.xml
deleted file mode 100644 (file)
index 3b8f4e0..0000000
+++ /dev/null
@@ -1,12 +0,0 @@
-<component name="libraryTable">
-  <library name="ktor-client-encoding" type="repository">
-    <properties include-transitive-deps="false" maven-id="io.ktor:ktor-client-encoding-jvm:2.1.2" />
-    <CLASSES>
-      <root url="jar://$MAVEN_REPOSITORY$/io/ktor/ktor-client-encoding-jvm/2.1.2/ktor-client-encoding-jvm-2.1.2.jar!/" />
-    </CLASSES>
-    <JAVADOC />
-    <SOURCES>
-      <root url="jar://$MAVEN_REPOSITORY$/io/ktor/ktor-client-encoding-jvm/2.1.2/ktor-client-encoding-jvm-2.1.2-sources.jar!/" />
-    </SOURCES>
-  </library>
-</component>
\ No newline at end of file
diff --git a/build/tasks/src/org/jetbrains/intellij/build/io/http.kt b/build/tasks/src/org/jetbrains/intellij/build/io/http.kt
new file mode 100644 (file)
index 0000000..a38e74e
--- /dev/null
@@ -0,0 +1,56 @@
+// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.intellij.build.io
+
+import com.intellij.diagnostic.telemetry.use
+import org.jetbrains.intellij.build.tracer
+import java.io.ByteArrayOutputStream
+import java.lang.Thread.sleep
+import java.net.URI
+import java.net.http.HttpClient
+import java.net.http.HttpRequest
+import java.net.http.HttpResponse
+import java.time.Duration
+import java.util.zip.GZIPInputStream
+
+private val httpClient by lazy {
+  HttpClient.newBuilder()
+    .followRedirects(HttpClient.Redirect.NORMAL)
+    .connectTimeout(Duration.ofSeconds(5))
+    .build()
+}
+
+fun download(url: String): ByteArray {
+  tracer.spanBuilder("download").setAttribute("url", url).use {
+    var attemptNumber = 0
+    while (true) {
+      val request = HttpRequest.newBuilder(URI(url))
+        .header("Accept", "application/json")
+        .header("Accept-Encoding", "gzip")
+        .build()
+
+      val response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream())
+      val statusCode = response.statusCode()
+      val encoding = response.headers().firstValue("Content-Encoding").orElse("")
+      // readAllBytes doesn't work due to incorrect assert in HttpResponseInputStream
+      val byteOut = ByteArrayOutputStream()
+      (if (encoding == "gzip") GZIPInputStream(response.body()) else response.body()).use {
+        it.transferTo(byteOut)
+      }
+      val content = byteOut.toByteArray()
+
+      if (statusCode == 200) {
+        return content
+      }
+      else if (attemptNumber > 3 || statusCode < 500) {
+        // do not attempt again if client error
+        throw RuntimeException("Cannot download $url (status=${response.statusCode()}, content=${content.toString(Charsets.UTF_8)})")
+      }
+      else {
+        attemptNumber++
+        sleep(attemptNumber * 1_000L)
+      }
+    }
+  }
+
+  throw IllegalStateException("must be unreachable")
+}
\ No newline at end of file
similarity index 92%
rename from platform/build-scripts/src/org/jetbrains/intellij/build/impl/brokenPlugins.kt
rename to build/tasks/src/org/jetbrains/intellij/build/tasks/brokenPlugins.kt
index 6fab61f8c52fc11c214a52b151a1dfdf240ea1ed..961047950198eb99c4f58647e4dc9c01268ffe13 100644 (file)
@@ -1,11 +1,11 @@
-// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package org.jetbrains.intellij.build.impl
+// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.intellij.build.tasks
 
 import io.opentelemetry.api.trace.Span
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.builtins.ListSerializer
 import kotlinx.serialization.json.Json
-import org.jetbrains.intellij.build.downloadAsText
+import org.jetbrains.intellij.build.io.download
 import java.io.BufferedOutputStream
 import java.io.DataOutputStream
 import java.nio.file.Files
@@ -17,13 +17,11 @@ private const val MARKETPLACE_BROKEN_PLUGINS_URL = "https://plugins.jetbrains.co
 /**
  * Generate brokenPlugins.txt file using JetBrains Marketplace.
  */
-suspend fun buildBrokenPlugins(targetFile: Path, currentBuildString: String, isInDevelopmentMode: Boolean) {
+fun buildBrokenPlugins(targetFile: Path, currentBuildString: String, isInDevelopmentMode: Boolean) {
   val span = Span.current()
 
   val allBrokenPlugins = try {
-    val jsonFormat = Json { ignoreUnknownKeys = true }
-    val content = downloadAsText(MARKETPLACE_BROKEN_PLUGINS_URL)
-    jsonFormat.decodeFromString(ListSerializer(MarketplaceBrokenPlugin.serializer()), content)
+    downloadFileFromMarketplace(span)
   }
   catch (e: Exception) {
     if (isInDevelopmentMode) {
@@ -51,6 +49,12 @@ suspend fun buildBrokenPlugins(targetFile: Path, currentBuildString: String, isI
   span.setAttribute("pluginCount", result.size.toLong())
 }
 
+private fun downloadFileFromMarketplace(span: Span): List<MarketplaceBrokenPlugin> {
+  val jsonFormat = Json { ignoreUnknownKeys = true }
+  val content = download(MARKETPLACE_BROKEN_PLUGINS_URL).toString(Charsets.UTF_8)
+  return jsonFormat.decodeFromString(ListSerializer(MarketplaceBrokenPlugin.serializer()), content)
+}
+
 private fun storeBrokenPlugin(brokenPlugin: Map<String, Set<String>>, build: String, targetFile: Path) {
   Files.createDirectories(targetFile.parent)
   DataOutputStream(BufferedOutputStream(Files.newOutputStream(targetFile), 32_000)).use { out ->
diff --git a/build/tasks/test/org/jetbrains/intellij/build/tasks/TaskTest.kt b/build/tasks/test/org/jetbrains/intellij/build/tasks/TaskTest.kt
new file mode 100644 (file)
index 0000000..c7edc4d
--- /dev/null
@@ -0,0 +1,53 @@
+// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.intellij.build.tasks
+
+import com.intellij.testFramework.rules.InMemoryFsExtension
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.RegisterExtension
+import java.io.BufferedInputStream
+import java.io.DataInputStream
+import java.nio.file.Files
+import java.util.*
+
+class TaskTest {
+  @RegisterExtension
+  @JvmField
+  val fs = InMemoryFsExtension()
+
+  companion object {
+    @JvmStatic
+    val logger = object : System.Logger {
+      override fun getName() = ""
+
+      override fun isLoggable(level: System.Logger.Level) = true
+
+      override fun log(level: System.Logger.Level, bundle: ResourceBundle?, message: String, thrown: Throwable?) {
+        if (level == System.Logger.Level.ERROR) {
+          throw RuntimeException(message, thrown)
+        }
+        else {
+          println(message)
+        }
+      }
+
+      override fun log(level: System.Logger.Level, bundle: ResourceBundle?, message: String, vararg params: Any?) {
+        log(level, bundle = null, message = message, thrown = null)
+      }
+
+      override fun log(level: System.Logger.Level, message: String) {
+        log(level = level, bundle = null, message = message, thrown = null)
+      }
+    }
+  }
+
+  @Test
+  fun `broken plugins`() {
+    val targetFile = fs.root.resolve("result")
+    buildBrokenPlugins(targetFile, "2020.3", isInDevelopmentMode = false)
+    DataInputStream(BufferedInputStream(Files.newInputStream(targetFile), 32_000)).use { stream ->
+      assertThat(stream.readByte()).isEqualTo(2)
+      assertThat(stream.readUTF()).isEqualTo("2020.3")
+    }
+  }
+}
\ No newline at end of file
index 633a9b89c2bf377e0c693f45022619563c41b30e..9ceac91418438569b4a7c4dfc706a547c43f10f6 100644 (file)
@@ -12,7 +12,6 @@ fun main(rawArgs: Array<String>) {
       platformPrefix = System.getProperty("idea.platform.prefix") ?: "idea",
       additionalModules = emptyList(),
       homePath = Path.of(PathManager.getHomePath()),
-      keepHttpClient = false,
     ))
   }
 
index 3fa032ca2b8b399222ba6a38cd69edb0288ef7d1..1d4f50abe45bf4f8224a8f4c8b2d34e6dafdd82c 100644 (file)
@@ -47,6 +47,7 @@
     <orderEntry type="module" module-name="intellij.platform.util.classLoader" />
     <orderEntry type="module" module-name="intellij.platform.util.rt.java8" />
     <orderEntry type="library" scope="RUNTIME" name="xz" level="project" />
+    <orderEntry type="library" scope="RUNTIME" name="ktor-client" level="project" />
     <orderEntry type="library" scope="RUNTIME" name="ktor-client-auth" level="project" />
     <orderEntry type="library" scope="RUNTIME" name="ktor-client-cio" level="project" />
   </component>
index 4e7cae1b564a853ea8b0d5b86162fc5179ed1ab2..112b3bf8901316ac184b69ff16501f16cf46d81d 100644 (file)
@@ -7,7 +7,6 @@ import kotlinx.coroutines.runBlocking
 import org.jetbrains.intellij.build.ConsoleSpanExporter
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
 import org.jetbrains.intellij.build.TracerProviderManager
-import org.jetbrains.intellij.build.closeKtorClient
 import java.util.function.Supplier
 
 object DevIdeaBuilder {
@@ -35,9 +34,5 @@ suspend fun buildProductInProcess(request: BuildRequest) {
   spanBuilder("build ide").setAttribute("request", request.toString()).useWithScope2 {
     BuildServer(homePath = request.homePath, productionClassOutput = request.productionClassOutput)
       .buildProductInProcess(isServerMode = false, request = request)
-    // otherwise, thread leak in tests
-    if (!request.keepHttpClient) {
-      closeKtorClient()
-    }
   }
 }
\ No newline at end of file
index 4075e3e78bef37c3a9ab952dc1506aa6e444192d..5fa59eb540ea449c636c964c2a370eb6e14c40dd 100644 (file)
@@ -38,7 +38,6 @@ data class BuildRequest(
   @JvmField val homePath: Path,
   @JvmField val productionClassOutput: Path = Path.of(System.getenv("CLASSES_DIR")
                                                       ?: homePath.resolve("out/classes/production").toString()).toAbsolutePath(),
-  @JvmField val keepHttpClient: Boolean = true,
 )
 
 private suspend fun computeLibClassPath(targetFile: Path, homePath: Path, context: BuildContext) {
index 4e2bbc676da0191df727d1c9cd6ef212f625234a..e4edad4f555891568dfb7c1781213162a56ae5ae 100644 (file)
@@ -77,9 +77,5 @@
     <orderEntry type="library" name="jsch-agent-proxy" level="project" />
     <orderEntry type="library" name="jsch-agent-proxy-sshj" level="project" />
     <orderEntry type="library" name="commons-io" level="project" />
-    <orderEntry type="library" name="ktor-client" level="project" />
-    <orderEntry type="library" name="ktor-client-auth" level="project" />
-    <orderEntry type="library" name="ktor-client-cio" level="project" />
-    <orderEntry type="library" name="ktor-client-encoding" level="project" />
   </component>
 </module>
\ No newline at end of file
index 4ee27b9fc7f8a9ecc56d8a9e19dcdd5ab7eade96..3a5c1f3f1c7f4a83d9be7726a43eef7e94a33454 100644 (file)
@@ -62,7 +62,7 @@ class ProprietaryBuildTools(
           error("Must be not called if usePresignedNativeFiles is false")
         }
 
-        override suspend fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path? {
+        override fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path? {
           return null
         }
       },
index 119c47058772a2d61c16fde442387f2822ce557f..42de32158b4a865f35d9ddcb821c8d7c34128f40 100644 (file)
@@ -21,5 +21,5 @@ interface SignTool {
    */
   suspend fun getPresignedLibraryFile(path: String, libName: String, libVersion: String, context: BuildContext): Path?
 
-  suspend fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path?
+  fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path?
 }
\ No newline at end of file
index 0079de371e45924b4997cb897eb389cc76104bd1..825b2ae9f691aa36445b0b2f59a209c4f2b5e91c 100644 (file)
@@ -11,7 +11,6 @@ import kotlinx.coroutines.Job
 import org.jetbrains.intellij.build.BuildContext
 import org.jetbrains.intellij.build.BuildOptions
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
-import org.jetbrains.intellij.build.downloadAsBytes
 import org.jetbrains.intellij.build.impl.ModuleOutputPatcher
 import org.jetbrains.intellij.build.impl.createSkippableJob
 import java.util.concurrent.CancellationException
@@ -24,8 +23,8 @@ internal fun CoroutineScope.createStatisticsRecorderBundledMetadataProviderTask(
   val featureUsageStatisticsProperties = context.proprietaryBuildTools.featureUsageStatisticsProperties ?: return null
   return createSkippableJob(
     spanBuilder("bundle a default version of feature usage statistics"),
-    taskId = BuildOptions.FUS_METADATA_BUNDLE_STEP,
-    context = context
+    BuildOptions.FUS_METADATA_BUNDLE_STEP,
+    context
   ) {
     try {
       val recorderId = featureUsageStatisticsProperties.recorderId
@@ -53,12 +52,12 @@ private fun appendProductCode(uri: String, context: BuildContext): String {
   return if (uri.endsWith('/')) "$uri$name" else "$uri/$name"
 }
 
-private suspend fun download(url: String): ByteArray {
+private fun download(url: String): ByteArray {
   Span.current().addEvent("download", Attributes.of(AttributeKey.stringKey("url"), url))
-  return downloadAsBytes(url)
+  return org.jetbrains.intellij.build.io.download(url)
 }
 
-private suspend fun metadataServiceUri(featureUsageStatisticsProperties: FeatureUsageStatisticsProperties, context: BuildContext): String {
+private fun metadataServiceUri(featureUsageStatisticsProperties: FeatureUsageStatisticsProperties, context: BuildContext): String {
   val providerUri = appendProductCode(featureUsageStatisticsProperties.metadataProviderUri, context)
   Span.current().addEvent("parsing", Attributes.of(AttributeKey.stringKey("url"), providerUri))
   val appInfo = context.applicationInfo
similarity index 79%
rename from platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/http.kt
rename to platform/build-scripts/src/org/jetbrains/intellij/build/http.kt
index 1ae1453af2415783d31bccda32ad866ddaaf7058..f555d5d09c9d68ba840e46fdd4456f2349fdc2f9 100644 (file)
@@ -1,5 +1,5 @@
 // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package org.jetbrains.intellij.build.impl.compilation
+package org.jetbrains.intellij.build
 
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
@@ -29,7 +29,7 @@ internal inline fun <T> Response.useSuccessful(task: (Response) -> T): T {
   }
 }
 
-internal val httpClient: OkHttpClient by lazy {
+internal val httpClient by lazy {
   val timeout = 1L
   val unit = TimeUnit.MINUTES
   OkHttpClient.Builder()
@@ -57,14 +57,26 @@ internal val httpClient: OkHttpClient by lazy {
           if (error == null) {
             error = IOException("$maxTryCount attempts to ${request.method} ${request.url} failed")
           }
-          error.addSuppressed(e)
+          error?.addSuppressed(e)
           null
         }
         tryCount++
       }
-      while ((response == null || response.code >= 500) && tryCount < maxTryCount)
+      while ((response == null || response!!.code >= 500) && tryCount < maxTryCount)
       response ?: throw error ?: IllegalStateException()
     }
     .followRedirects(true)
     .build()
+}
+
+@Suppress("HttpUrlsUsage")
+internal fun toUrlWithTrailingSlash(serverUrl: String): String {
+  var result = serverUrl
+  if (!result.startsWith("http://") && !result.startsWith("https://")) {
+    result = "http://$result"
+  }
+  if (!result.endsWith('/')) {
+    result += '/'
+  }
+  return result
 }
\ No newline at end of file
index 9126246d524ecf174f4085f569e265f15a78d9e1..eb6e9a16401a4908c07835cdde2a148e7a683e05 100644 (file)
@@ -6,14 +6,14 @@ import org.jetbrains.intellij.build.OsFamily
 import java.nio.file.Path
 
 interface BundledRuntime {
-  suspend fun getHomeForCurrentOsAndArch(): Path
+  fun getHomeForCurrentOsAndArch(): Path
 
   /**
    * contract: returns a directory, where only one subdirectory is available: 'jbr', which contains specified JBR
    */
-  suspend fun extract(prefix: String, os: OsFamily, arch: JvmArchitecture): Path
+  fun extract(prefix: String, os: OsFamily, arch: JvmArchitecture): Path
 
-  suspend fun extractTo(prefix: String, os: OsFamily, destinationDir: Path, arch: JvmArchitecture)
+  fun extractTo(prefix: String, os: OsFamily, destinationDir: Path, arch: JvmArchitecture)
 
   fun archiveName(prefix: String, arch: JvmArchitecture, os: OsFamily, forceVersionWithUnderscores: Boolean = false): String
 
index c77f6a7aa428883c9c5e1e020a8230fe53b84521..dc1a4ca4122494040a5d154faa0ee303c91aea48 100644 (file)
@@ -7,10 +7,10 @@ import com.intellij.openapi.util.io.NioFiles
 import kotlinx.collections.immutable.persistentListOf
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.jetbrains.intellij.build.*
-import org.jetbrains.intellij.build.TraceManager.spanBuilder
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesExtractOptions
 import org.jetbrains.intellij.build.dependencies.DependenciesProperties
+import java.net.URI
 import java.nio.file.*
 import java.nio.file.attribute.BasicFileAttributes
 import java.nio.file.attribute.DosFileAttributeView
@@ -34,7 +34,7 @@ class BundledRuntimeImpl(
     options.bundledRuntimeBuild ?: dependenciesProperties.property("runtimeBuild")
   }
 
-  override suspend fun getHomeForCurrentOsAndArch(): Path {
+  override fun getHomeForCurrentOsAndArch(): Path {
     var prefix = "jbr_jcef-"
     val os = OsFamily.currentOs
     val arch = JvmArchitecture.currentJvmArch
@@ -59,7 +59,7 @@ class BundledRuntimeImpl(
   }
 
   // contract: returns a directory, where only one subdirectory is available: 'jbr', which contains specified JBR
-  override suspend fun extract(prefix: String, os: OsFamily, arch: JvmArchitecture): Path {
+  override fun extract(prefix: String, os: OsFamily, arch: JvmArchitecture): Path {
     val targetDir = paths.communityHomeDir.resolve("build/download/${prefix}${build}-${os.jbrArchiveSuffix}-$arch")
     val jbrDir = targetDir.resolve("jbr")
 
@@ -80,14 +80,14 @@ class BundledRuntimeImpl(
     return targetDir
   }
 
-  override suspend fun extractTo(prefix: String, os: OsFamily, destinationDir: Path, arch: JvmArchitecture) {
+  override fun extractTo(prefix: String, os: OsFamily, destinationDir: Path, arch: JvmArchitecture) {
     doExtract(findArchive(prefix, os, arch), destinationDir, os)
   }
 
-  private suspend fun findArchive(prefix: String, os: OsFamily, arch: JvmArchitecture): Path {
-    val archiveName = archiveName(prefix = prefix, arch = arch, os = os)
-    val url = "https://cache-redirector.jetbrains.com/intellij-jbr/$archiveName"
-    return downloadFileToCacheLocation(url = url, communityRoot = paths.communityHomeDirRoot)
+  private fun findArchive(prefix: String, os: OsFamily, arch: JvmArchitecture): Path {
+    val archiveName = archiveName(prefix, arch, os)
+    val url = URI("https://cache-redirector.jetbrains.com/intellij-jbr/$archiveName")
+    return BuildDependenciesDownloader.downloadFileToCacheLocation(paths.communityHomeDirRoot, url)
   }
 
   /**
@@ -145,7 +145,7 @@ private fun getArchSuffix(arch: JvmArchitecture): String {
 }
 
 private fun doExtract(archive: Path, destinationDir: Path, os: OsFamily) {
-  spanBuilder("extract JBR")
+  TraceManager.spanBuilder("extract JBR")
     .setAttribute("archive", archive.toString())
     .setAttribute("os", os.osName)
     .setAttribute("destination", destinationDir.toString())
index 10d9d57f79b2ae2d5f5c93163458e9ad13c507c0..df0189f052913dc4c4d36998383bdf545aac4356 100644 (file)
@@ -25,7 +25,7 @@ import org.jetbrains.intellij.build.impl.JdkUtils.readModulesFromReleaseFile
 import org.jetbrains.intellij.build.impl.compilation.CompiledClasses
 import org.jetbrains.intellij.build.impl.logging.BuildMessagesHandler
 import org.jetbrains.intellij.build.impl.logging.BuildMessagesImpl
-import org.jetbrains.intellij.build.kotlin.loadKotlinJpsPluginToClassPath
+import org.jetbrains.intellij.build.kotlin.KotlinBinaries
 import org.jetbrains.jps.model.*
 import org.jetbrains.jps.model.artifact.JpsArtifactService
 import org.jetbrains.jps.model.java.JpsJavaClasspathKind
@@ -234,7 +234,7 @@ class CompilationContextImpl private constructor(model: JpsModel,
           BuildDependenciesDownloader.TRACER = BuildDependenciesOpenTelemetryTracer.INSTANCE
         }
 
-        loadProject(projectHome = projectHome, communityHome, isCompilationRequired)
+        loadProject(projectHome = projectHome, kotlinBinaries = KotlinBinaries(communityHome, messages), isCompilationRequired)
       }
       val context = CompilationContextImpl(model = model,
                                            communityHome = communityHome,
@@ -271,26 +271,24 @@ class CompilationContextImpl private constructor(model: JpsModel,
     this.nameToModule = modules.associateByTo(HashMap(modules.size)) { it.name }
     val buildOut = options.outputRootPath ?: buildOutputRootEvaluator(project)
     val logDir = options.logPath?.let { Path.of(it).toAbsolutePath().normalize() } ?: buildOut.resolve("log")
-    paths = BuildPathsImpl(communityHome = communityHome, projectHome = projectHome, buildOut = buildOut, logDir = logDir)
+    paths = BuildPathsImpl(communityHome, projectHome, buildOut, logDir)
     dependenciesProperties = DependenciesProperties(paths.communityHomeDirRoot)
-    bundledRuntime = BundledRuntimeImpl(options = options,
-                                        paths = paths,
-                                        dependenciesProperties = dependenciesProperties,
-                                        error = messages::error,
-                                        info = messages::info)
+    bundledRuntime = BundledRuntimeImpl(options, paths, dependenciesProperties, messages::error, messages::info)
   }
 }
 
-private suspend fun loadProject(projectHome: Path, communityHome: BuildDependenciesCommunityRoot, isCompilationRequired: Boolean): JpsModel {
+private suspend fun loadProject(projectHome: Path, kotlinBinaries: KotlinBinaries, isCompilationRequired: Boolean): JpsModel {
   val model = JpsElementFactory.getInstance().createModel()
   val pathVariablesConfiguration = JpsModelSerializationDataService.getOrCreatePathVariablesConfiguration(model.global)
-  withContext(Dispatchers.IO) {
-    if (isCompilationRequired) {
-      val kotlinCompilerHome = loadKotlinJpsPluginToClassPath(communityHome)
-      System.setProperty("jps.kotlin.home", kotlinCompilerHome.toString())
-      pathVariablesConfiguration.addPathVariable("KOTLIN_BUNDLED", kotlinCompilerHome.toString())
-    }
+  if (isCompilationRequired) {
+    kotlinBinaries.loadKotlinJpsPluginToClassPath()
 
+    val kotlinCompilerHome = kotlinBinaries.kotlinCompilerHome
+    System.setProperty("jps.kotlin.home", kotlinCompilerHome.toString())
+    pathVariablesConfiguration.addPathVariable("KOTLIN_BUNDLED", kotlinCompilerHome.toString())
+  }
+
+  withContext(Dispatchers.IO) {
     spanBuilder("load project").useWithScope2 { span ->
       pathVariablesConfiguration.addPathVariable("MAVEN_REPOSITORY", FileUtilRt.toSystemIndependentName(
         Path.of(SystemProperties.getUserHome(), ".m2/repository").toString()))
index f41150244cd70a6757ab669c97ac0ea4ae20667d..b0d2a705978978b01a34c8a5d08e748f5bd123a1 100644 (file)
@@ -113,11 +113,11 @@ private suspend fun generateIntegrityManifest(sitFile: Path, sitRoot: String, co
   }
 }
 
-private suspend fun buildAndSignWithMacBuilderHost(sitFile: Path,
-                                                   macHostProperties: MacHostProperties,
-                                                   notarize: Boolean,
-                                                   customizer: MacDistributionCustomizer,
-                                                   context: BuildContext) {
+private fun buildAndSignWithMacBuilderHost(sitFile: Path,
+                                           macHostProperties: MacHostProperties,
+                                           notarize: Boolean,
+                                           customizer: MacDistributionCustomizer,
+                                           context: BuildContext) {
   val dmgImage = if (context.options.buildStepsToSkip.contains(BuildOptions.MAC_DMG_STEP)) {
     null
   }
index ee6a8330ff54be352df6ea8f110e64f7340c252c..3709928eec10ff01a72dc360e0588289e4e263e0 100644 (file)
@@ -15,7 +15,6 @@ import com.intellij.openapi.util.io.NioFiles
 import com.intellij.openapi.util.text.StringUtilRt
 import com.intellij.util.lang.UrlClassLoader
 import io.opentelemetry.api.common.AttributeKey
-import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.runBlocking
 import org.jetbrains.intellij.build.*
 import org.jetbrains.intellij.build.CompilationTasks.Companion.create
@@ -337,9 +336,9 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
       messages.info("Tests from bucket ${allSystemProperties[TestCaseLoader.TEST_RUNNER_INDEX_FLAG]} of ${numberOfBuckets} will be executed")
     }
     messages.block("Test classpath and runtime info") {
-      runBlocking(Dispatchers.IO) {
-        val runtime = getRuntimeExecutablePath().toString()
-        messages.info("Runtime: ${runtime}")
+      val runtime = runtimeExecutablePath().toString()
+      messages.info("Runtime: ${runtime}")
+      runBlocking {
         runProcess(args = listOf(runtime, "-version"), inheritOut = true)
       }
       messages.info("Runtime options: ${allJvmArgs}")
@@ -347,7 +346,6 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
       messages.info("Bootstrap classpath: ${bootstrapClasspath}")
       messages.info("Tests classpath: ${testClasspath}")
       modulePath?.let { mp ->
-        @Suppress("SpellCheckingInspection")
         messages.info("Tests modulepath: $mp")
       }
       if (!envVariables.isEmpty()) {
@@ -365,7 +363,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
     notifySnapshotBuilt(allJvmArgs)
   }
 
-  private suspend fun getRuntimeExecutablePath(): Path {
+  private fun runtimeExecutablePath(): Path {
     val runtimeDir: Path
     if (options.customRuntimePath != null) {
       runtimeDir = Path.of(options.customRuntimePath)
@@ -378,7 +376,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
     }
 
     var java = runtimeDir.resolve(if (SystemInfoRt.isWindows) "bin/java.exe" else "bin/java")
-    if (SystemInfoRt.isMac && Files.notExists(java)) {
+    if (SystemInfoRt.isMac && !Files.exists(java)) {
       java = runtimeDir.resolve("Contents/Home/bin/java")
     }
     check(Files.exists(java)) { "java executable is missing: ${java}" }
@@ -424,7 +422,6 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
     jvmArgs += options.jvmMemoryOptions?.split(Regex("\\s+")) ?: listOf("-Xms750m", "-Xmx750m")
 
     val tempDir = System.getProperty("teamcity.build.tempDir", System.getProperty("java.io.tmpdir"))
-    @Suppress("SpellCheckingInspection")
     mapOf(
       "idea.platform.prefix" to options.platformPrefix,
       "idea.home.path" to context.paths.projectHome.toString(),
@@ -444,11 +441,11 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
       "io.netty.leakDetectionLevel" to "PARANOID",
       "kotlinx.coroutines.debug" to "on",
       "sun.io.useCanonCaches" to "false",
-    ).forEach { (k, v) ->
+    ).forEach(BiConsumer { k, v ->
       if (v != null) {
         systemProperties.putIfAbsent(k, v)
       }
-    }
+    })
 
     System.getProperties().forEach(BiConsumer { key, value ->
       if ((key as String).startsWith("pass.")) {
@@ -676,7 +673,7 @@ internal class TestingTasksImpl(private val context: CompilationContext, private
     }
 
     val argFile = CommandLineWrapperUtil.createArgumentFile(args, Charset.defaultCharset())
-    val runtime = runBlocking(Dispatchers.IO) { getRuntimeExecutablePath ().toString() }
+    val runtime = runtimeExecutablePath().toString()
     context.messages.info("Starting tests on runtime ${runtime}")
     val builder = ProcessBuilder(runtime, "@" + argFile.absolutePath)
     builder.environment().putAll(envVariables)
@@ -749,7 +746,6 @@ private fun publishTestDiscovery(messages: BuildMessages, file: String?) {
       val map = LinkedHashMap<String, String>(7)
       map["teamcity-build-number"] = System.getProperty("build.number")
       map["teamcity-build-type-id"] = System.getProperty("teamcity.buildType.id")
-      @Suppress("SpellCheckingInspection")
       map["teamcity-build-configuration-name"] = System.getenv("TEAMCITY_BUILDCONF_NAME")
       map["teamcity-build-project-name"] = System.getenv("TEAMCITY_PROJECT_NAME")
       map["branch"] = System.getProperty("teamcity.build.branch")?.takeIf(String::isNotEmpty) ?: "master"
index 95ef8ca49b201ce3a9fea40828707544fd1ba0f5..8549523f9d3d22d132449686ca4057b8c988913d 100644 (file)
@@ -19,6 +19,7 @@ import org.jetbrains.annotations.VisibleForTesting
 import org.jetbrains.intellij.build.BuildMessages
 import org.jetbrains.intellij.build.CompilationContext
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
+import org.jetbrains.intellij.build.httpClient
 import org.jetbrains.intellij.build.io.AddDirEntriesMode
 import org.jetbrains.intellij.build.io.deleteDir
 import org.jetbrains.intellij.build.io.zip
index 62a05199711df5d23a239b00b63deac0e9a67e38..34a00ec4c477abdbfb8c5908d8d64ea9164fd0d6 100644 (file)
@@ -17,7 +17,10 @@ import okhttp3.*
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.RequestBody.Companion.toRequestBody
 import okio.*
+import org.jetbrains.intellij.build.MEDIA_TYPE_BINARY
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
+import org.jetbrains.intellij.build.httpClient
+import org.jetbrains.intellij.build.useSuccessful
 import java.nio.ByteBuffer
 import java.nio.channels.FileChannel
 import java.nio.file.Files
index 21b9423e18b4dba02a38e86bab0243fc22ca56e6..9ffcd0ea477dbbf8b843db5caced045acdb74013 100644 (file)
@@ -1,49 +1,52 @@
 // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.intellij.build.kotlin
 
-import io.opentelemetry.api.trace.Span
+import org.jetbrains.intellij.build.BuildMessages
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
-import org.jetbrains.intellij.build.impl.CompilationContextImpl
 import org.jetbrains.intellij.build.impl.addToClasspathAgent.AddToClasspathUtil
 import java.nio.file.Files
 import java.nio.file.Path
 
-private suspend fun getKotlinCompilerHome(communityHome: BuildDependenciesCommunityRoot): Path {
-  val compilerHome = KotlinCompilerDependencyDownloader.downloadAndExtractKotlinCompiler(communityHome)
-  val kotlinc = compilerHome.resolve("bin/kotlinc")
-  check(Files.exists(kotlinc)) { "Kotlin compiler home is missing under the path: $compilerHome" }
-  return compilerHome
-}
-
 /**
  * Sets up Kotlin compiler (downloaded from Marketplace) which is required for JPS to compile the repository
  */
-suspend fun loadKotlinJpsPluginToClassPath(communityHome: BuildDependenciesCommunityRoot): Path {
-  val required = KotlinCompilerDependencyDownloader.getKotlinJpsPluginVersion(communityHome)
-
-  val current = getCurrentKotlinJpsPluginVersionFromClassPath()
-  if (current != null) {
-    check(current == required) {
-      "Currently loaded Kotlin JPS plugin version is '$current', but required '$required'"
-    }
+internal class KotlinBinaries(private val communityHome: BuildDependenciesCommunityRoot, private val messages: BuildMessages) {
+  val kotlinCompilerHome: Path by lazy {
+    val compilerHome = KotlinCompilerDependencyDownloader.downloadAndExtractKotlinCompiler(communityHome)
+    val kotlinc = compilerHome.resolve("bin/kotlinc")
+    check(Files.exists(kotlinc)) { "Kotlin compiler home is missing under the path: $compilerHome" }
+    compilerHome
+  }
 
-    // already loaded
-    return getKotlinCompilerHome(communityHome)
+  private val kotlinJpsPlugin: Path by lazy {
+    val jpsPlugin = KotlinCompilerDependencyDownloader.downloadKotlinJpsPlugin(communityHome)
+    jpsPlugin
   }
 
-  val kotlinJpsPlugin = KotlinCompilerDependencyDownloader.downloadKotlinJpsPlugin(communityHome)
-  Span.current().addEvent("Loading Kotlin JPS plugin from $kotlinJpsPlugin")
-  AddToClasspathUtil.addToClassPathViaAgent(listOf(kotlinJpsPlugin))
+  suspend fun loadKotlinJpsPluginToClassPath() {
+    val required = KotlinCompilerDependencyDownloader.getKotlinJpsPluginVersion(communityHome)
 
-  val afterLoad = getCurrentKotlinJpsPluginVersionFromClassPath()
-  check(afterLoad == required) {
-    "Loaded Kotlin JPS plugin version '$afterLoad', but required '$required'"
-  }
+    val current = getCurrentKotlinJpsPluginVersionFromClassPath()
+    if (current != null) {
+      check(current == required) {
+        "Currently loaded Kotlin JPS plugin version is '$current', but required '$required'"
+      }
 
-  return getKotlinCompilerHome(communityHome)
-}
+      // already loaded
+      return
+    }
 
-private fun getCurrentKotlinJpsPluginVersionFromClassPath(): String? {
-  val stream = CompilationContextImpl::class.java.classLoader.getResourceAsStream("META-INF/compiler.version") ?: return null
-  return stream.use { stream.readAllBytes().decodeToString() }
+    messages.info("Loading Kotlin JPS plugin from $kotlinJpsPlugin")
+    AddToClasspathUtil.addToClassPathViaAgent(listOf(kotlinJpsPlugin))
+
+    val afterLoad = getCurrentKotlinJpsPluginVersionFromClassPath()
+    check(afterLoad == required) {
+      "Loaded Kotlin JPS plugin version '$afterLoad', but required '$required'"
+    }
+  }
+
+  private fun getCurrentKotlinJpsPluginVersionFromClassPath(): String? {
+    val stream = javaClass.classLoader.getResourceAsStream("META-INF/compiler.version") ?: return null
+    return stream.use { stream.readAllBytes().decodeToString() }
+  }
 }
\ No newline at end of file
index 6f85220e091d8d8ed2f0c5e0baa778974d1cd35d..bf55ffb499ef8bc56796868f95be6dfeb4f74032 100644 (file)
@@ -3,12 +3,9 @@ package org.jetbrains.intellij.build.kotlin
 
 import com.intellij.util.xml.dom.XmlElement
 import com.intellij.util.xml.dom.readXmlAsModel
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.runBlocking
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
-import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
+import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.*
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesManualRunOnly
-import org.jetbrains.intellij.build.downloadFileToCacheLocation
 
 import java.nio.file.Path
 
@@ -16,17 +13,17 @@ private const val MAVEN_REPOSITORY_URL = "https://cache-redirector.jetbrains.com
 private const val ARTIFACT_GROUP_ID = "org.jetbrains.kotlin"
 
 object KotlinCompilerDependencyDownloader {
-  suspend fun downloadAndExtractKotlinCompiler(communityRoot: BuildDependenciesCommunityRoot): Path {
+  fun downloadAndExtractKotlinCompiler(communityRoot: BuildDependenciesCommunityRoot): Path {
     val kotlinJpsPluginVersion = getKotlinJpsPluginVersion(communityRoot)
-    val kotlinDistUrl = BuildDependenciesDownloader.getUriForMavenArtifact(MAVEN_REPOSITORY_URL, ARTIFACT_GROUP_ID, "kotlin-dist-for-ide", kotlinJpsPluginVersion, "jar")
-    val kotlinDistJar = downloadFileToCacheLocation(kotlinDistUrl.toString(), communityRoot)
-    return BuildDependenciesDownloader.extractFileToCacheLocation(communityRoot, kotlinDistJar)
+    val kotlinDistUrl = getUriForMavenArtifact(MAVEN_REPOSITORY_URL, ARTIFACT_GROUP_ID, "kotlin-dist-for-ide", kotlinJpsPluginVersion, "jar")
+    val kotlinDistJar = downloadFileToCacheLocation(communityRoot, kotlinDistUrl)
+    return extractFileToCacheLocation(communityRoot, kotlinDistJar)
   }
 
-  suspend fun downloadKotlinJpsPlugin(communityRoot: BuildDependenciesCommunityRoot): Path {
+  fun downloadKotlinJpsPlugin(communityRoot: BuildDependenciesCommunityRoot): Path {
     val kotlinJpsPluginVersion = getKotlinJpsPluginVersion(communityRoot)
-    val kotlinJpsPluginUrl = BuildDependenciesDownloader.getUriForMavenArtifact(MAVEN_REPOSITORY_URL, ARTIFACT_GROUP_ID, "kotlin-jps-plugin-classpath", kotlinJpsPluginVersion, "jar")
-    return downloadFileToCacheLocation(kotlinJpsPluginUrl.toString(), communityRoot)
+    val kotlinJpsPluginUrl = getUriForMavenArtifact(MAVEN_REPOSITORY_URL, ARTIFACT_GROUP_ID, "kotlin-jps-plugin-classpath", kotlinJpsPluginVersion, "jar")
+    return downloadFileToCacheLocation(communityRoot, kotlinJpsPluginUrl)
   }
 
   fun getKotlinJpsPluginVersion(communityRoot: BuildDependenciesCommunityRoot): String {
@@ -44,11 +41,9 @@ object KotlinCompilerDependencyDownloader {
 
   @JvmStatic
   fun main(args: Array<String>) {
-    runBlocking(Dispatchers.IO) {
-      val path = downloadAndExtractKotlinCompiler(BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
-      println("Extracted Kotlin compiler is at $path")
-      val jpsPath = downloadKotlinJpsPlugin(BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
-      println("Download Kotlin Jps Plugin at $jpsPath")
-    }
+    val path = downloadAndExtractKotlinCompiler(BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
+    println("Extracted Kotlin compiler is at $path")
+    val jpsPath = downloadKotlinJpsPlugin(BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
+    println("Download Kotlin Jps Plugin at $jpsPath")
   }
 }
diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/ktor.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/ktor.kt
deleted file mode 100644 (file)
index fd51842..0000000
+++ /dev/null
@@ -1,250 +0,0 @@
-@file:Suppress("BlockingMethodInNonBlockingContext", "ReplaceGetOrSet")
-package org.jetbrains.intellij.build
-
-import com.intellij.diagnostic.telemetry.useWithScope2
-import com.intellij.util.concurrency.SynchronizedClearableLazy
-import io.ktor.client.HttpClient
-import io.ktor.client.call.body
-import io.ktor.client.engine.cio.CIO
-import io.ktor.client.plugins.HttpRequestRetry
-import io.ktor.client.plugins.UserAgent
-import io.ktor.client.plugins.auth.Auth
-import io.ktor.client.plugins.auth.providers.BasicAuthCredentials
-import io.ktor.client.plugins.auth.providers.BearerTokens
-import io.ktor.client.plugins.auth.providers.basic
-import io.ktor.client.plugins.auth.providers.bearer
-import io.ktor.client.plugins.compression.ContentEncoding
-import io.ktor.client.request.get
-import io.ktor.client.statement.bodyAsChannel
-import io.ktor.client.statement.bodyAsText
-import io.ktor.http.HttpHeaders
-import io.ktor.util.read
-import io.ktor.utils.io.*
-import io.ktor.utils.io.core.use
-import io.ktor.utils.io.jvm.nio.copyTo
-import io.opentelemetry.api.common.AttributeKey
-import io.opentelemetry.api.common.Attributes
-import io.opentelemetry.api.trace.Span
-import kotlinx.coroutines.*
-import org.jetbrains.intellij.build.TraceManager.spanBuilder
-import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
-import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
-import org.jetbrains.xxh3.Xx3UnencodedString
-import java.nio.channels.FileChannel
-import java.nio.charset.StandardCharsets
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.StandardCopyOption
-import java.nio.file.StandardOpenOption
-import java.nio.file.attribute.FileTime
-import java.time.Instant
-import java.util.*
-import java.util.concurrent.locks.*
-
-const val SPACE_REPO_HOST = "packages.jetbrains.team"
-
-private val httpClient = SynchronizedClearableLazy {
-  // HttpTimeout is not used - CIO engine handles that
-  HttpClient(CIO) {
-    // we have custom error handler
-    expectSuccess = true
-
-    install(ContentEncoding) {
-      deflate(1.0F)
-      gzip(0.9F)
-    }
-
-    install(HttpRequestRetry) {
-      retryOnExceptionOrServerErrors(maxRetries = 3)
-      exponentialDelay()
-    }
-
-    install(UserAgent) {
-      agent = "Build Script Downloader"
-    }
-  }
-}
-
-private val httpSpaceClient = SynchronizedClearableLazy {
-  httpClient.value.config {
-    // we have custom error handler
-    expectSuccess = false
-
-    val token = System.getenv("SPACE_PACKAGE_TOKEN")
-    if (!token.isNullOrEmpty()) {
-      install(Auth) {
-        bearer {
-          sendWithoutRequest { request ->
-            request.url.host == SPACE_REPO_HOST
-          }
-
-          loadTokens {
-            BearerTokens(token, "")
-          }
-        }
-      }
-    }
-    else {
-      val userName = System.getProperty("jps.auth.spaceUsername")
-      val password = System.getProperty("jps.auth.spacePassword")
-      if (userName != null && password != null) {
-        install(Auth) {
-          basic {
-            sendWithoutRequest { request ->
-              request.url.host == SPACE_REPO_HOST
-            }
-
-            credentials {
-              BasicAuthCredentials(username = userName, password = password)
-            }
-          }
-        }
-      }
-    }
-  }
-}
-
-fun closeKtorClient() {
-  httpClient.drop()?.close()
-  httpSpaceClient.drop()?.close()
-}
-
-private val fileLocks = StripedMutex()
-
-suspend fun downloadAsBytes(url: String): ByteArray {
-  return spanBuilder("download").setAttribute("url", url).useWithScope2 {
-    withContext(Dispatchers.IO) {
-      httpClient.value.get(url).body()
-    }
-  }
-}
-
-suspend fun downloadAsText(url: String): String {
-  return spanBuilder("download").setAttribute("url", url).useWithScope2 {
-    withContext(Dispatchers.IO) {
-      httpClient.value.get(url).bodyAsText()
-    }
-  }
-}
-
-suspend fun downloadFileToCacheLocation(url: String, context: BuildContext): Path {
-  return downloadFileToCacheLocation(url, context.paths.communityHomeDirRoot)
-}
-
-suspend fun downloadFileToCacheLocation(url: String, communityRoot: BuildDependenciesCommunityRoot): Path {
-  BuildDependenciesDownloader.cleanUpIfRequired(communityRoot)
-
-  val target = BuildDependenciesDownloader.getTargetFile(communityRoot, url)
-  val targetPath = target.toString()
-  val lock = fileLocks.getLock(Xx3UnencodedString.hashUnencodedString(targetPath).toInt())
-  lock.lock()
-  try {
-    val now = Instant.now()
-    if (Files.exists(target)) {
-      Span.current().addEvent("use asset from cache", Attributes.of(
-        AttributeKey.stringKey("url"), url,
-        AttributeKey.stringKey("target"), targetPath,
-      ))
-
-      // update file modification time to maintain FIFO caches i.e. in persistent cache folder on TeamCity agent
-      Files.setLastModifiedTime(target, FileTime.from(now))
-      return target
-    }
-
-    return spanBuilder("download").setAttribute("url", url).setAttribute("target", targetPath).useWithScope2 {
-      // save to the same disk to ensure that move will be atomic and not as a copy
-      val tempFile = target.parent
-        .resolve("${target.fileName}-${(now.epochSecond - 1634886185).toString(36)}-${now.nano.toString(36)}".take(255))
-      try {
-        val response = httpSpaceClient.value.get(url)
-        coroutineScope {
-          response.bodyAsChannel().copyAndClose(writeChannel(tempFile))
-        }
-        val statusCode = response.status.value
-        if (statusCode != 200) {
-          val builder = StringBuilder("Cannot download\n")
-          val headers = response.headers
-          headers.names()
-            .asSequence()
-            .sorted()
-            .flatMap { headerName -> headers.getAll(headerName)!!.map { value -> "Header: $headerName: $value\n" } }
-            .forEach(builder::append)
-          builder.append('\n')
-          if (Files.exists(tempFile)) {
-            Files.newInputStream(tempFile).use { inputStream ->
-              // yes, not trying to guess encoding
-              // string constructor should be exception free,
-              // so at worse we'll get some random characters
-              builder.append(inputStream.readNBytes(1024).toString(StandardCharsets.UTF_8))
-            }
-          }
-          throw BuildDependenciesDownloader.HttpStatusException(builder.toString(), statusCode, url)
-        }
-
-        val contentLength = response.headers.get(HttpHeaders.ContentLength)?.toLongOrNull() ?: -1
-        check(contentLength > 0) { "Header '${HttpHeaders.ContentLength}' is missing or zero for $url" }
-        val fileSize = Files.size(tempFile)
-        check(fileSize == contentLength) {
-          "Wrong file length after downloading uri '$url' to '$tempFile': expected length $contentLength " +
-          "from Content-Length header, but got $fileSize on disk"
-        }
-        Files.move(tempFile, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING)
-      }
-      finally {
-        Files.deleteIfExists(tempFile)
-      }
-
-      target
-    }
-  }
-  finally {
-    lock.unlock()
-  }
-}
-
-fun CoroutineScope.readChannel(file: Path): ByteReadChannel {
-  return writer(CoroutineName("file-reader") + Dispatchers.IO, autoFlush = false) {
-    @Suppress("BlockingMethodInNonBlockingContext")
-    FileChannel.open(file, StandardOpenOption.READ).use { fileChannel ->
-      @Suppress("DEPRECATION")
-      channel.writeSuspendSession {
-        while (true) {
-          val buffer = request(1)
-          if (buffer == null) {
-            channel.flush()
-            tryAwait(1)
-            continue
-          }
-
-          val rc = fileChannel.read(buffer)
-          if (rc == -1) {
-            break
-          }
-          written(rc)
-        }
-      }
-    }
-  }.channel
-}
-
-private val WRITE_NEW_OPERATION = EnumSet.of(StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
-
-internal fun CoroutineScope.writeChannel(file: Path): ByteWriteChannel {
-  return reader(CoroutineName("file-writer") + Dispatchers.IO, autoFlush = true) {
-    FileChannel.open(file, WRITE_NEW_OPERATION).use { fileChannel ->
-      channel.copyTo(fileChannel)
-    }
-  }.channel
-}
-
-@Suppress("HttpUrlsUsage")
-internal fun toUrlWithTrailingSlash(serverUrl: String): String {
-  var result = serverUrl
-  if (!result.startsWith("http://") && !result.startsWith("https://")) {
-    result = "http://$result"
-  }
-  if (!result.endsWith('/')) {
-    result += '/'
-  }
-  return result
-}
\ No newline at end of file
diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/striped.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/striped.kt
deleted file mode 100644 (file)
index c0fa258..0000000
+++ /dev/null
@@ -1,16 +0,0 @@
-// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package org.jetbrains.intellij.build
-
-import kotlinx.coroutines.sync.Mutex
-
-private const val MAX_POWER_OF_TWO = 1 shl Integer.SIZE - 2
-private const val ALL_SET = 0.inv()
-
-internal class StripedMutex(stripeCount: Int = 64) {
-  private val mask = if (stripeCount > MAX_POWER_OF_TWO) ALL_SET else (1 shl (Integer.SIZE - Integer.numberOfLeadingZeros(stripeCount - 1))) - 1
-  private val locks = Array(mask + 1) { Mutex() }
-
-  fun getLock(hash: Int): Mutex {
-    return locks[hash and mask]
-  }
-}
\ No newline at end of file
index 5f845179965d897ff7d5e7553afe42aad8bab381..15ffd39275b1b9665948a14977c38fe7914c7f11 100644 (file)
@@ -192,8 +192,6 @@ suspend fun runTestBuild(context: BuildContext, traceSpanName: String? = null, o
       }
   }
   finally {
-    closeKtorClient()
-
     // close debug logging to prevent locking of output directory on Windows
     (context.messages as BuildMessagesImpl).close()
 
index 27611b02e95c1714859186c2e8e2649b4f3acef0..9b1769cbae686fc22ab51846c578dfa09850280d 100644 (file)
@@ -3,8 +3,6 @@ package org.jetbrains.intellij.build.impl
 
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.io.NioFiles
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.runBlocking
 import org.jetbrains.intellij.build.*
 import org.jetbrains.intellij.build.dependencies.JdkDownloader
 import org.junit.Test
@@ -12,7 +10,7 @@ import java.nio.file.Files
 
 class BundledRuntimeTest {
   @Test
-  fun download(): Unit = runBlocking(Dispatchers.IO) {
+  fun download() {
     withCompilationContext { context ->
       val bundledRuntime = BundledRuntimeImpl(context.options, context.paths, context.dependenciesProperties, context.messages::error, context.messages::info)
       val currentJbr = bundledRuntime.getHomeForCurrentOsAndArch()
@@ -55,16 +53,8 @@ class BundledRuntimeTest {
   @Test
   fun currentArchDownload() {
     withCompilationContext { context ->
-      val currentJbrHome = runBlocking(Dispatchers.IO) {
-        BundledRuntimeImpl(
-          options = context.options,
-          paths = context.paths,
-          dependenciesProperties = context.dependenciesProperties,
-          error = context.messages::error,
-          info = context.messages::info
-        )
-          .getHomeForCurrentOsAndArch()
-      }
+      val currentJbrHome = BundledRuntimeImpl(context.options, context.paths, context.dependenciesProperties, context.messages::error, context.messages::info)
+        .getHomeForCurrentOsAndArch()
       val javaExe = JdkDownloader.getJavaExecutable(currentJbrHome)
       val process = ProcessBuilder(javaExe.toString(), "-version")
         .inheritIO()
@@ -76,7 +66,7 @@ class BundledRuntimeTest {
     }
   }
 
-  private inline fun withCompilationContext(block: (CompilationContext) -> Unit) {
+  private fun withCompilationContext(block: (CompilationContext) -> Unit) {
     val tempDir = Files.createTempDirectory("compilation-context-")
     try {
       val communityHome = IdeaProjectLoaderUtil.guessCommunityHome(javaClass)