get rid of intellij.build.toolbox.litegen parameter and use BuildOptions.TOOLBOX_LITE...
[idea/community.git] / platform / build-scripts / groovy / org / jetbrains / intellij / build / impl / DistributionJARsBuilder.groovy
1 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
2 package org.jetbrains.intellij.build.impl
3
4 import com.intellij.openapi.util.MultiValuesMap
5 import com.intellij.openapi.util.Pair
6 import com.intellij.openapi.util.io.FileUtil
7 import groovy.io.FileType
8 import org.apache.tools.ant.types.FileSet
9 import org.apache.tools.ant.types.resources.FileProvider
10 import org.jetbrains.annotations.Nullable
11 import org.jetbrains.intellij.build.*
12 import org.jetbrains.intellij.build.fus.StatisticsRecorderBundledWhiteListProvider
13 import org.jetbrains.jps.model.java.JpsJavaClasspathKind
14 import org.jetbrains.jps.model.java.JpsJavaExtensionService
15 import org.jetbrains.jps.model.library.JpsLibrary
16 import org.jetbrains.jps.model.library.JpsOrderRootType
17 import org.jetbrains.jps.model.module.JpsLibraryDependency
18 import org.jetbrains.jps.model.module.JpsModule
19 import org.jetbrains.jps.model.module.JpsModuleReference
20 import org.jetbrains.jps.util.JpsPathUtil
21
22 /**
23  * Assembles output of modules to platform JARs (in {@link org.jetbrains.intellij.build.BuildPaths#distAll distAll}/lib directory),
24  * bundled plugins' JARs (in {@link org.jetbrains.intellij.build.BuildPaths#distAll distAll}/plugins directory) and zip archives with
25  * non-bundled plugins (in {@link org.jetbrains.intellij.build.BuildPaths#artifacts artifacts}/plugins directory).
26  *
27  * @author nik
28  */
29 class DistributionJARsBuilder {
30   private static final boolean COMPRESS_JARS = false
31   private static final String RESOURCES_INCLUDED = "resources.included"
32   private static final String RESOURCES_EXCLUDED = "resources.excluded"
33   /**
34    * Path to file with third party libraries HTML content,
35    * see the same constant at com.intellij.ide.actions.AboutPopup#THIRD_PARTY_LIBRARIES_FILE_PATH
36    */
37   private static final String THIRD_PARTY_LIBRARIES_FILE_PATH = "license/third-party-libraries.html"
38   private final BuildContext buildContext
39   private final Set<String> usedModules = new LinkedHashSet<>()
40   private final PlatformLayout platform
41   private final File patchedApplicationInfo
42   private final LinkedHashMap<PluginLayout, PluginPublishingSpec> pluginsToPublish
43
44   DistributionJARsBuilder(BuildContext buildContext, File patchedApplicationInfo,
45                           LinkedHashMap<PluginLayout, PluginPublishingSpec> pluginsToPublish = [:]) {
46     this.patchedApplicationInfo = patchedApplicationInfo
47     this.buildContext = buildContext
48     this.pluginsToPublish = pluginsToPublish
49     buildContext.ant.patternset(id: RESOURCES_INCLUDED) {
50       include(name: "**/*Bundle*.properties")
51       include(name: "**/*Messages.properties")
52       include(name: "messages/**/*.properties")
53       include(name: "fileTemplates/**")
54       include(name: "inspectionDescriptions/**")
55       include(name: "intentionDescriptions/**")
56       include(name: "tips/**")
57       include(name: "search/**")
58     }
59
60     buildContext.ant.patternset(id: RESOURCES_EXCLUDED) {
61       exclude(name: "**/*Bundle*.properties")
62       exclude(name: "**/*Messages.properties")
63       exclude(name: "messages/**/*.properties")
64       exclude(name: "fileTemplates/**")
65       exclude(name: "fileTemplates")
66       exclude(name: "inspectionDescriptions/**")
67       exclude(name: "inspectionDescriptions")
68       exclude(name: "intentionDescriptions/**")
69       exclude(name: "intentionDescriptions")
70       exclude(name: "tips/**")
71       exclude(name: "tips")
72       exclude(name: "search/**")
73       exclude(name: "**/icon-robots.txt")
74     }
75
76     def productLayout = buildContext.productProperties.productLayout
77     def enabledPluginModules = getEnabledPluginModules()
78     buildContext.messages.debug("Collecting project libraries used by plugins: ")
79     List<JpsLibrary> projectLibrariesUsedByPlugins = getPluginsByModules(buildContext, enabledPluginModules).collectMany { plugin ->
80       plugin.getActualModules(enabledPluginModules).values().collectMany {
81         def module = buildContext.findRequiredModule(it)
82         def libraries = JpsJavaExtensionService.dependencies(module).includedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME).libraries.findAll { library ->
83           !(library.createReference().parentReference instanceof JpsModuleReference) && !plugin.includedProjectLibraries.any {
84             it.libraryName == library.name && it.relativeOutputPath == ""
85           }
86         }
87         if (!libraries.isEmpty()) {
88           buildContext.messages.debug(" plugin '$plugin.mainModule', module '$it': ${libraries.collect {"'$it.name'"}.join(",")}")
89         }
90         libraries
91       }
92     }
93
94     Set<String> allProductDependencies = (productLayout.getIncludedPluginModules(enabledPluginModules) + getIncludedPlatformModules(productLayout)).collectMany(new LinkedHashSet<String>()) {
95       JpsJavaExtensionService.dependencies(buildContext.findRequiredModule(it)).productionOnly().getModules().collect {it.name}
96     }
97
98     platform = PlatformLayout.platform(productLayout.platformLayoutCustomizer) {
99       productLayout.additionalPlatformJars.entrySet().each {
100         def jarName = it.key
101         it.value.each {
102           withModule(it, jarName)
103         }
104       }
105       getPlatformApiModules(productLayout).each {
106         withModule(it, "platform-api.jar")
107       }
108       getPlatformImplModules(productLayout).each {
109         withModule(it, "platform-impl.jar")
110       }
111       getProductApiModules(productLayout).each {
112         withModule(it, "openapi.jar")
113       }
114       getProductImplModules(productLayout).each {
115         withModule(it, productLayout.mainJarName)
116       }
117       productLayout.moduleExcludes.entrySet().each {
118         layout.moduleExcludes.putAll(it.key, it.value)
119       }
120       withModule("intellij.platform.util")
121       withModule("intellij.platform.util.rt", "util.jar")
122       withModule("intellij.platform.util.classLoader", "util.jar")
123       withModule("intellij.platform.util.ui")
124
125       withModule("intellij.platform.concurrency")
126       withModule("intellij.platform.core.ui")
127
128       withModule("intellij.platform.objectSerializer.annotations")
129       withModule("intellij.platform.objectSerializer")
130       withModule("intellij.platform.configurationStore.impl")
131
132       withModule("intellij.platform.extensions")
133       withModule("intellij.platform.bootstrap")
134       withModule("intellij.java.guiForms.rt")
135       withModule("intellij.platform.icons")
136       withModule("intellij.platform.boot", "bootstrap.jar")
137       withModule("intellij.platform.resources", "resources.jar")
138       withModule("intellij.platform.colorSchemes", "resources.jar")
139       withModule("intellij.platform.resources.en", productLayout.mainJarName)
140       withModule("intellij.platform.jps.model.serialization", "jps-model.jar")
141       withModule("intellij.platform.jps.model.impl", "jps-model.jar")
142
143       if (allProductDependencies.contains("intellij.platform.coverage")) {
144         withModule("intellij.platform.coverage", productLayout.mainJarName)
145       }
146
147       projectLibrariesUsedByPlugins.each {
148         if (!productLayout.projectLibrariesToUnpackIntoMainJar.contains(it.name) && !layout.excludedProjectLibraries.contains(it.name)) {
149           withProjectLibrary(it.name)
150         }
151       }
152       productLayout.projectLibrariesToUnpackIntoMainJar.each {
153         withProjectLibraryUnpackedIntoJar(it, productLayout.mainJarName)
154       }
155       withProjectLibrariesFromIncludedModules(buildContext)
156       removeVersionFromProjectLibraryJarNames("Trove4j")
157       removeVersionFromProjectLibraryJarNames("Log4J")
158       removeVersionFromProjectLibraryJarNames("jna")
159       removeVersionFromProjectLibraryJarNames("jetbrains-annotations-java5")
160       removeVersionFromProjectLibraryJarNames("JDOM")
161     }
162   }
163
164   private Set<String> getEnabledPluginModules() {
165     buildContext.productProperties.productLayout.allBundledPluginsModules + pluginsToPublish.keySet().
166       collect { it.mainModule } as Set<String>
167   }
168
169   List<String> getPlatformModules() {
170     (platform.moduleJars.values() as List<String>) + toolModules
171   }
172
173   static List<String> getIncludedPlatformModules(ProductModulesLayout modulesLayout) {
174     getPlatformApiModules(modulesLayout) + getPlatformImplModules(modulesLayout) + getProductApiModules(modulesLayout) +
175     getProductImplModules(modulesLayout) + modulesLayout.additionalPlatformJars.values()
176   }
177
178   /**
179    * @return module names which are required to run necessary tools from build scripts
180    */
181   static List<String> getToolModules() {
182     ["intellij.java.rt", "intellij.platform.main", /*required to build searchable options index*/ "intellij.platform.updater"]
183   }
184
185   static List<String> getPlatformApiModules(ProductModulesLayout productLayout) {
186     productLayout.platformApiModules.isEmpty() ? CommunityRepositoryModules.PLATFORM_API_MODULES : []
187   }
188
189   static List<String> getPlatformImplModules(ProductModulesLayout productLayout) {
190     productLayout.platformImplementationModules.isEmpty() ? CommunityRepositoryModules.PLATFORM_IMPLEMENTATION_MODULES : []
191   }
192
193   static List<String> getProductApiModules(ProductModulesLayout productLayout) {
194     productLayout.platformApiModules.isEmpty() ? productLayout.productApiModules : productLayout.platformApiModules
195   }
196
197   static List<String> getProductImplModules(ProductModulesLayout productLayout) {
198     productLayout.platformImplementationModules.isEmpty() ? productLayout.productImplementationModules : productLayout.platformImplementationModules
199   }
200
201   Collection<String> getIncludedProjectArtifacts() {
202     platform.includedArtifacts.keySet() + getPluginsByModules(buildContext, getEnabledPluginModules()).collectMany {it.includedArtifacts.keySet()}
203   }
204
205   void buildJARs() {
206     buildSearchableOptions()
207     buildLib()
208     buildBundledPlugins()
209     buildOsSpecificBundledPlugins()
210     buildNonBundledPlugins()
211     buildThirdPartyLibrariesList()
212
213     def loadingOrderFilePath = buildContext.productProperties.productLayout.classesLoadingOrderFilePath
214     if (loadingOrderFilePath != null) {
215       reorderJARs(loadingOrderFilePath)
216     }
217   }
218
219   /**
220    * Build index which is used to search options in the Settings dialog.
221    */
222   void buildSearchableOptions() {
223     buildContext.executeStep("Build searchable options index", BuildOptions.SEARCHABLE_OPTIONS_INDEX_STEP, {
224       def productLayout = buildContext.productProperties.productLayout
225       def modulesToIndex = productLayout.mainModules + getModulesToCompile(buildContext) + modulesForPluginsToPublish
226       modulesToIndex -= "intellij.clion.plugin" // TODO [AK] temporary solution to fix CLion build
227       def targetDirectory = getSearchableOptionsDir()
228       buildContext.messages.progress("Building searchable options for ${modulesToIndex.size()} modules")
229       buildContext.messages.debug("Searchable options are going to be built for the following modules: $modulesToIndex")
230       String targetFile = targetDirectory.absolutePath
231       FileUtil.delete(targetDirectory)
232       // Start the product in headless mode using com.intellij.ide.ui.search.TraverseUIStarter.
233       // It'll process all UI elements in Settings dialog and build index for them.
234       BuildTasksImpl.runApplicationStarter(buildContext, "$buildContext.paths.temp/searchableOptions", modulesToIndex, ['traverseUI', targetFile, 'true'])
235       def modules = targetDirectory.list()
236       if (modules == null || modules.length == 0) {
237         buildContext.messages.error("Failed to build searchable options index: $targetFile is empty")
238       }
239       else {
240         buildContext.messages.info("Searchable options are built successfully for $modules.length modules")
241         buildContext.messages.debug("The following modules contain searchable options: $modules")
242       }
243     })
244   }
245
246   static List<String> getModulesToCompile(BuildContext buildContext) {
247     def productLayout = buildContext.productProperties.productLayout
248     productLayout.getIncludedPluginModules(productLayout.allBundledPluginsModules) +
249     getPlatformApiModules(productLayout) +
250     getPlatformImplModules(productLayout) +
251     getProductApiModules(productLayout) +
252     getProductImplModules(productLayout) +
253     productLayout.additionalPlatformJars.values() +
254     toolModules + buildContext.productProperties.additionalModulesToCompile
255   }
256
257   List<String> getModulesForPluginsToPublish() {
258     def enabledModulesSet = enabledPluginModules
259     platformModules + (pluginsToPublish.collect { it.key.getActualModules(enabledModulesSet).values() }.flatten() as List<String>)
260   }
261
262   void reorderJARs(String loadingOrderFilePath) {
263     buildContext.messages.block("Reorder JARs") {
264       String targetDirectory = buildContext.paths.distAll
265       buildContext.messages.progress("Reordering *.jar files in $targetDirectory")
266       File ignoredJarsFile = new File(buildContext.paths.temp, "reorder-jars/required_for_dist.txt")
267       ignoredJarsFile.parentFile.mkdirs()
268       def moduleJars = platform.moduleJars.entrySet().collect(new HashSet()) { getActualModuleJarPath(it.key, it.value, platform.explicitlySetJarPaths) }
269       ignoredJarsFile.text = new File(buildContext.paths.distAll, "lib").list()
270         .findAll {it.endsWith(".jar") && !moduleJars.contains(it)}
271         .join("\n")
272
273       buildContext.ant.java(classname: "com.intellij.util.io.zip.ReorderJarsMain", fork: true, failonerror: true) {
274         arg(value: loadingOrderFilePath)
275         arg(value: targetDirectory)
276         arg(value: targetDirectory)
277         arg(value: ignoredJarsFile.parent)
278         classpath {
279           buildContext.getModuleRuntimeClasspath(buildContext.findRequiredModule("intellij.platform.util"), false).each {
280             pathelement(location: it)
281           }
282         }
283       }
284     }
285   }
286
287   void buildAdditionalArtifacts() {
288     def productProperties = buildContext.productProperties
289
290     if (productProperties.generateLibrariesLicensesTable && !buildContext.options.buildStepsToSkip.
291       contains(BuildOptions.THIRD_PARTY_LIBRARIES_LIST_STEP)) {
292       String artifactNamePrefix = productProperties.getBaseArtifactName(buildContext.applicationInfo, buildContext.buildNumber)
293       buildContext.ant.copy(file: getThirdPartyLibrariesHtmlFilePath(),
294                             tofile: "$buildContext.paths.artifacts/$artifactNamePrefix-third-party-libraries.html")
295       buildContext.ant.copy(file: getThirdPartyLibrariesJsonFilePath(),
296                             tofile: "$buildContext.paths.artifacts/$artifactNamePrefix-third-party-libraries.json")
297     }
298
299     if (productProperties.scrambleMainJar) {
300       createLayoutBuilder().layout("$buildContext.paths.buildOutputRoot/internal") {
301         jar("internalUtilities.jar") {
302           module("intellij.tools.internalUtilities")
303         }
304       }
305     }
306
307     if (productProperties.buildSourcesArchive) {
308       def archiveName = "${productProperties.getBaseArtifactName(buildContext.applicationInfo, buildContext.buildNumber)}-sources.zip"
309       BuildTasks.create(buildContext).zipSourcesOfModules(usedModules, "$buildContext.paths.artifacts/$archiveName")
310     }
311   }
312
313   private void buildThirdPartyLibrariesList() {
314     buildContext.executeStep("Generate table of licenses for used third-party libraries", BuildOptions.THIRD_PARTY_LIBRARIES_LIST_STEP) {
315       def generator = LibraryLicensesListGenerator.create(buildContext.messages,
316                                                           buildContext.project,
317                                                           buildContext.productProperties.allLibraryLicenses,
318                                                           usedModules)
319       generator.generateHtml(getThirdPartyLibrariesHtmlFilePath())
320       generator.generateJson(getThirdPartyLibrariesJsonFilePath())
321     }
322   }
323
324   private String getThirdPartyLibrariesHtmlFilePath() {
325     "$buildContext.paths.distAll/$THIRD_PARTY_LIBRARIES_FILE_PATH"
326   }
327
328   private String getThirdPartyLibrariesJsonFilePath() {
329     "$buildContext.paths.temp/third-party-libraries.json"
330   }
331
332   private void buildLib() {
333     def ant = buildContext.ant
334     def layoutBuilder = createLayoutBuilder()
335     def productLayout = buildContext.productProperties.productLayout
336
337     addSearchableOptions(layoutBuilder)
338
339     def applicationInfoFile = FileUtil.toSystemIndependentName(patchedApplicationInfo.absolutePath)
340     def applicationInfoDir = "$buildContext.paths.temp/applicationInfo"
341     ant.copy(file: applicationInfoFile, todir: "$applicationInfoDir/idea")
342     layoutBuilder.patchModuleOutput(buildContext.productProperties.applicationInfoModule, applicationInfoDir)
343
344     if (buildContext.productProperties.reassignAltClickToMultipleCarets) {
345       def patchedKeyMapDir = createKeyMapWithAltClickReassignedToMultipleCarets()
346       layoutBuilder.patchModuleOutput("intellij.platform.resources", FileUtil.toSystemIndependentName(patchedKeyMapDir.absolutePath))
347     }
348     if (buildContext.proprietaryBuildTools.featureUsageStatisticsProperties != null) {
349       def whiteList = StatisticsRecorderBundledWhiteListProvider.downloadWhiteList(buildContext)
350       layoutBuilder.patchModuleOutput('intellij.platform.ide.impl', whiteList.absolutePath)
351     }
352
353     buildContext.messages.block("Build platform JARs in lib directory") {
354       buildByLayout(layoutBuilder, platform, buildContext.paths.distAll, platform.moduleJars, [])
355     }
356
357     if (buildContext.proprietaryBuildTools.scrambleTool != null) {
358       def forbiddenJarNames = buildContext.proprietaryBuildTools.scrambleTool.namesOfJarsRequiredToBeScrambled
359       def packagedFiles = new File(buildContext.paths.distAll, "lib").listFiles()
360       def forbiddenJars = packagedFiles.findAll { forbiddenJarNames.contains(it.name) }
361       if (!forbiddenJars.empty) {
362         buildContext.messages.error( "The following JARs cannot be included into the product 'lib' directory, they need to be scrambled with the main jar: ${forbiddenJars}")
363       }
364       def modulesToBeScrambled = buildContext.proprietaryBuildTools.scrambleTool.namesOfModulesRequiredToBeScrambled
365       platform.moduleJars.keySet().each { jarName ->
366         if (jarName != productLayout.mainJarName) {
367           def notScrambled = platform.moduleJars.get(jarName).intersect(modulesToBeScrambled)
368           if (!notScrambled.isEmpty()) {
369             buildContext.messages.error("Module '${notScrambled.first()}' is included into $jarName which is not scrambled.")
370           }
371         }
372       }
373     }
374
375     usedModules.addAll(layoutBuilder.usedModules)
376   }
377
378   private void buildBundledPlugins() {
379     def layoutBuilder = createLayoutBuilder()
380     def allPlugins = getPluginsByModules(buildContext, buildContext.productProperties.productLayout.bundledPluginModules)
381     buildContext.messages.block("Build bundled plugins") {
382       buildPlugins(layoutBuilder, allPlugins.findAll { satisfiesBundlingRequirements(it, null) }, "$buildContext.paths.distAll/plugins")
383     }
384     usedModules.addAll(layoutBuilder.usedModules)
385   }
386
387   private boolean satisfiesBundlingRequirements(PluginLayout plugin, @Nullable OsFamily osFamily) {
388     def bundlingRestrictions = plugin.bundlingRestrictions
389     if (!buildContext.applicationInfo.isEAP && bundlingRestrictions.includeInEapOnly) {
390       return false
391     }
392     osFamily == null ? bundlingRestrictions.supportedOs == OsFamily.ALL
393                      : bundlingRestrictions.supportedOs != OsFamily.ALL && bundlingRestrictions.supportedOs.contains(osFamily)
394   }
395
396   private void buildOsSpecificBundledPlugins() {
397     def productLayout = buildContext.productProperties.productLayout
398     for (OsFamily osFamily in OsFamily.values()) {
399       List<PluginLayout> osSpecificPlugins =
400         getPluginsByModules(buildContext, productLayout.bundledOsPluginModules[osFamily] ?: []) +
401         getPluginsByModules(buildContext, productLayout.bundledPluginModules).findAll {
402           satisfiesBundlingRequirements(it, osFamily)
403         }
404
405       if (!osSpecificPlugins.isEmpty() && buildContext.shouldBuildDistributionForOS(osFamily.osId)) {
406         def layoutBuilder = createLayoutBuilder()
407         buildContext.messages.block("Build bundled plugins for $osFamily.osName") {
408           buildPlugins(layoutBuilder, osSpecificPlugins,
409                        "$buildContext.paths.buildOutputRoot/dist.$osFamily.distSuffix/plugins")
410         }
411         usedModules.addAll(layoutBuilder.usedModules)
412       }
413     }
414   }
415
416   void buildNonBundledPlugins() {
417     def productLayout = buildContext.productProperties.productLayout
418     def ant = buildContext.ant
419     def layoutBuilder = createLayoutBuilder()
420     buildContext.executeStep("Build non-bundled plugins", BuildOptions.NON_BUNDLED_PLUGINS_STEP) {
421       def pluginXmlFiles = new LinkedHashMap<PluginLayout, String>()
422
423       pluginsToPublish.each { pluginAndPublishing ->
424         def plugin = pluginAndPublishing.key
425
426         def moduleOutput = buildContext.getModuleOutputPath(buildContext.findRequiredModule(plugin.mainModule))
427         def pluginXmlPath = "$moduleOutput/META-INF/plugin.xml"
428         if (!new File(pluginXmlPath)) {
429           buildContext.messages.error("plugin.xml not found in $plugin.mainModule module: $pluginXmlPath")
430         }
431
432         pluginXmlFiles.put(plugin, pluginXmlPath)
433       }
434
435       if (buildContext.productProperties.setPluginAndIDEVersionInPluginXml) {
436         pluginsToPublish.each { pluginAndPublishing ->
437           def plugin = pluginAndPublishing.key
438           def publishingSpec = pluginAndPublishing.value
439
440           def pluginXmlPath = pluginXmlFiles[plugin]
441           def patchedPluginXmlDir = "$buildContext.paths.temp/patched-plugin-xml/$plugin.mainModule"
442           def patchedPluginXmlPath = "$patchedPluginXmlDir/META-INF/plugin.xml"
443           pluginXmlFiles.put(plugin, patchedPluginXmlPath)
444
445           ant.copy(file: pluginXmlPath, todir: "$patchedPluginXmlDir/META-INF")
446
447           CompatibleBuildRange compatibleBuildRange = publishingSpec.compatibleBuildRange
448           if (compatibleBuildRange == null) {
449             def includeInCustomRepository = productLayout.prepareCustomPluginRepositoryForPublishedPlugins && publishingSpec.includeInCustomPluginRepository
450             compatibleBuildRange = includeInCustomRepository ? CompatibleBuildRange.EXACT : CompatibleBuildRange.NEWER_WITH_SAME_BASELINE
451           }
452
453           setPluginVersionAndSince(patchedPluginXmlPath, getPluginVersion(plugin),
454                                    buildContext.buildNumber,
455                                    compatibleBuildRange)
456           layoutBuilder.patchModuleOutput(plugin.mainModule, patchedPluginXmlDir)
457         }
458       }
459
460       def pluginsToPublishDir = "$buildContext.paths.temp/${buildContext.applicationInfo.productCode}-plugins-to-publish"
461       def pluginsDirectoryName = "${buildContext.applicationInfo.productCode}-plugins"
462       buildPlugins(layoutBuilder, new ArrayList<PluginLayout>(pluginsToPublish.keySet()), pluginsToPublishDir)
463       def nonBundledPluginsArtifacts = "$buildContext.paths.artifacts/$pluginsDirectoryName"
464
465       def pluginsToIncludeInCustomRepository = new ArrayList<PluginRepositorySpec>()
466
467       pluginsToPublish.each { pluginAndPublishing ->
468         def plugin = pluginAndPublishing.key
469         def publishingSpec = pluginAndPublishing.value
470
471         def includeInCustomRepository = productLayout.prepareCustomPluginRepositoryForPublishedPlugins && publishingSpec.includeInCustomPluginRepository
472
473         def directory = getActualPluginDirectoryName(plugin, buildContext)
474         String suffix = includeInCustomRepository ? "" : "-${getPluginVersion(plugin)}"
475         def targetDirectory = publishingSpec.includeIntoDirectoryForAutomaticUploading ? "$nonBundledPluginsArtifacts/auto-uploading" : nonBundledPluginsArtifacts
476         def destFile = "$targetDirectory/$directory${suffix}.zip"
477
478         if (includeInCustomRepository) {
479           pluginsToIncludeInCustomRepository.add(new PluginRepositorySpec(pluginZip: destFile.toString(),
480                                                                           pluginXml: pluginXmlFiles[plugin]))
481         }
482
483         ant.zip(destfile: destFile) {
484           zipfileset(dir: "$pluginsToPublishDir/$directory", prefix: directory)
485         }
486         buildContext.notifyArtifactBuilt(destFile)
487       }
488
489       if (productLayout.prepareCustomPluginRepositoryForPublishedPlugins) {
490         new PluginRepositoryXmlGenerator(buildContext).generate(pluginsToIncludeInCustomRepository, nonBundledPluginsArtifacts)
491         buildContext.notifyArtifactBuilt("$nonBundledPluginsArtifacts/plugins.xml")
492       }
493     }
494   }
495
496   private String getPluginVersion(PluginLayout plugin) {
497     return plugin.versionEvaluator.apply(buildContext)
498   }
499
500   /**
501    * Returns name of directory in the product distribution where plugin will be placed. For plugins which use the main module name as the
502    * directory name return the old module name to temporary keep layout of plugins unchanged.
503    */
504   static String getActualPluginDirectoryName(PluginLayout plugin, BuildContext context) {
505     if (!plugin.directoryNameSetExplicitly && plugin.directoryName == BaseLayout.convertModuleNameToFileName(plugin.mainModule)
506                                            && context.getOldModuleName(plugin.mainModule) != null) {
507       context.getOldModuleName(plugin.mainModule)
508     }
509     else {
510       plugin.directoryName
511     }
512   }
513
514   static List<PluginLayout> getPluginsByModules(BuildContext buildContext, Collection<String> modules) {
515     def allNonTrivialPlugins = buildContext.productProperties.productLayout.allNonTrivialPlugins
516     def allOptionalModules = allNonTrivialPlugins.collectMany {it.optionalModules}
517     def nonTrivialPlugins = allNonTrivialPlugins.groupBy { it.mainModule }
518     (modules - allOptionalModules).collect { (nonTrivialPlugins[it] ?: nonTrivialPlugins[buildContext.findModule(it)?.name])?.first() ?: PluginLayout.plugin(it) }
519   }
520
521   private void buildPlugins(LayoutBuilder layoutBuilder, List<PluginLayout> pluginsToInclude, String targetDirectory) {
522     addSearchableOptions(layoutBuilder)
523     def enabledModulesSet = enabledPluginModules
524     pluginsToInclude.each { plugin ->
525       def actualModuleJars = plugin.getActualModules(enabledModulesSet)
526       checkOutputOfPluginModules(plugin.mainModule, actualModuleJars.values(), plugin.moduleExcludes)
527       List<Pair<File, String>> generatedResources = plugin.resourceGenerators.collectMany {
528         File resourceFile = it.first.generateResources(buildContext)
529         resourceFile != null ? [Pair.create(resourceFile, it.second)] : []
530       }
531       buildByLayout(layoutBuilder, plugin, "$targetDirectory/${getActualPluginDirectoryName(plugin, buildContext)}", actualModuleJars, generatedResources)
532     }
533   }
534
535   private void addSearchableOptions(LayoutBuilder layoutBuilder) {
536     if (!buildContext.options.buildStepsToSkip.contains(BuildOptions.SEARCHABLE_OPTIONS_INDEX_STEP)) {
537       def searchableOptionsDir = getSearchableOptionsDir()
538       if (!searchableOptionsDir.exists()) {
539         buildContext.messages.error("There are no searchable options available. " +
540                                     "Please ensure that you call DistributionJARsBuilder#buildSearchableOptions before this method.")
541       }
542       searchableOptionsDir.eachFile(FileType.DIRECTORIES) {
543         layoutBuilder.patchModuleOutput(it.name, FileUtil.toSystemIndependentName(it.absolutePath))
544       }
545     }
546   }
547
548   private File getSearchableOptionsDir() {
549     new File(buildContext.paths.temp, "searchableOptions/result")
550   }
551
552   private void checkOutputOfPluginModules(String mainPluginModule, Collection<String> moduleNames, MultiValuesMap<String, String> moduleExcludes) {
553     def modulesWithPluginXml = moduleNames.findAll { containsFileInOutput(it, "META-INF/plugin.xml", moduleExcludes.get(it)) }
554     if (modulesWithPluginXml.size() > 1) {
555       buildContext.messages.error("Multiple modules (${modulesWithPluginXml.join(", ")}) from '$mainPluginModule' plugin contain plugin.xml files so the plugin won't work properly")
556     }
557
558     moduleNames.each {
559       if (it != "intellij.java.guiForms.rt" && containsFileInOutput(it, "com/intellij/uiDesigner/core/GridLayoutManager.class", moduleExcludes.get(it))) {
560         buildContext.messages.error("Runtime classes of GUI designer must not be packaged to '$it' module in '$mainPluginModule' plugin, because they are included into a platform JAR. " +
561                                     "Make sure that 'Automatically copy form runtime classes to the output directory' is disabled in Settings | Editor | GUI Designer.")
562       }
563     }
564   }
565
566   private boolean containsFileInOutput(String moduleName, String filePath, Collection<String> excludes) {
567     def moduleOutput = new File(buildContext.getModuleOutputPath(buildContext.findRequiredModule(moduleName)))
568     def fileInOutput = new File(moduleOutput, filePath)
569     return fileInOutput.exists() && (excludes == null || excludes.every {
570       createFileSet(it, moduleOutput).iterator().every { !(it instanceof FileProvider && FileUtil.filesEqual(it.file, fileInOutput))}
571     })
572   }
573
574   /**
575    * Returns path to a JAR file in the product distribution where platform/plugin classes will be placed. If the JAR name corresponds to
576    * a module name and the module was renamed, return the old name to temporary keep the product layout unchanged.
577    */
578   private String getActualModuleJarPath(String relativeJarPath, Collection<String> moduleNames, Set<String> explicitlySetJarPaths) {
579     if (explicitlySetJarPaths.contains(relativeJarPath)) {
580       return relativeJarPath
581     }
582     for (String moduleName : moduleNames) {
583       if (relativeJarPath == "${BaseLayout.convertModuleNameToFileName(moduleName)}.jar" &&
584           buildContext.getOldModuleName(moduleName) !=
585           null) {
586         return "${buildContext.getOldModuleName(moduleName)}.jar"
587       }
588     }
589     return relativeJarPath
590   }
591
592   /**
593    * @param moduleJars mapping from JAR path relative to 'lib' directory to names of modules
594    * @param additionalResources pairs of resources files and corresponding relative output paths
595    */
596   private void buildByLayout(LayoutBuilder layoutBuilder, BaseLayout layout, String targetDirectory, MultiValuesMap<String, String> moduleJars,
597                              List<Pair<File, String>> additionalResources) {
598     def ant = buildContext.ant
599     def resourceExcluded = RESOURCES_EXCLUDED
600     def resourcesIncluded = RESOURCES_INCLUDED
601     def buildContext = buildContext
602     checkModuleExcludes(layout.moduleExcludes)
603     MultiValuesMap<String, String> actualModuleJars = new MultiValuesMap<>(true)
604     moduleJars.entrySet().each {
605       def modules = it.value
606       def jarPath = getActualModuleJarPath(it.key, modules, layout.explicitlySetJarPaths)
607       actualModuleJars.putAll(jarPath, modules)
608     }
609     layoutBuilder.layout(targetDirectory) {
610       dir("lib") {
611         actualModuleJars.entrySet().each {
612           def modules = it.value
613           def jarPath = it.key
614           jar(jarPath, true) {
615             modules.each { moduleName ->
616               modulePatches([moduleName]) {
617                 if (layout.localizableResourcesJarName(moduleName) != null) {
618                   ant.patternset(refid: resourceExcluded)
619                 }
620               }
621               module(moduleName) {
622                 if (layout.localizableResourcesJarName(moduleName) != null) {
623                   ant.patternset(refid: resourceExcluded)
624                 }
625                 else {
626                   ant.exclude(name: "**/icon-robots.txt")
627                 }
628
629                 layout.moduleExcludes.get(moduleName)?.each {
630                   //noinspection GrUnresolvedAccess
631                   ant.exclude(name: it)
632                 }
633               }
634             }
635             layout.projectLibrariesToUnpack.get(jarPath)?.each {
636               buildContext.project.libraryCollection.findLibrary(it)?.getFiles(JpsOrderRootType.COMPILED)?.each {
637                 ant.zipfileset(src: it.absolutePath)
638               }
639             }
640           }
641         }
642         def outputResourceJars = new MultiValuesMap<String, String>(true)
643         actualModuleJars.values().forEach {
644           def resourcesJarName = layout.localizableResourcesJarName(it)
645           if (resourcesJarName != null) {
646             outputResourceJars.put(resourcesJarName, it)
647           }
648         }
649         if (!outputResourceJars.empty) {
650           outputResourceJars.keySet().forEach { resourceJarName ->
651             jar(resourceJarName, true) {
652               outputResourceJars.get(resourceJarName).each { moduleName ->
653                 modulePatches([moduleName]) {
654                   ant.patternset(refid: resourcesIncluded)
655                 }
656                 module(moduleName) {
657                   layout.moduleExcludes.get(moduleName)?.each {
658                     //noinspection GrUnresolvedAccess
659                     ant.exclude(name: "$it/**")
660                   }
661                   ant.patternset(refid: resourcesIncluded)
662                 }
663               }
664             }
665           }
666         }
667         layout.includedProjectLibraries.each { libraryData ->
668           dir(libraryData.relativeOutputPath) {
669             projectLibrary(libraryData.libraryName, layout instanceof PlatformLayout && layout.projectLibrariesWithRemovedVersionFromJarNames.contains(libraryData.libraryName))
670           }
671         }
672         layout.includedArtifacts.entrySet().each {
673           def artifactName = it.key
674           def relativePath = it.value
675           dir(relativePath) {
676             artifact(artifactName)
677           }
678         }
679
680         //include all module libraries from the plugin modules added to IDE classpath to layout
681         actualModuleJars.entrySet().findAll { !it.key.contains("/") }.collectMany { it.value }
682                              .findAll {!layout.modulesWithExcludedModuleLibraries.contains(it)}.each { moduleName ->
683           findModule(moduleName).dependenciesList.dependencies.
684             findAll { it instanceof JpsLibraryDependency && it?.libraryReference?.parentReference?.resolve() instanceof JpsModule }.
685             findAll { JpsJavaExtensionService.instance.getDependencyExtension(it)?.scope?.isIncludedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME) ?: false }.
686             each {
687               jpsLibrary(((JpsLibraryDependency)it).library)
688             }
689         }
690
691         layout.includedModuleLibraries.each { data ->
692           dir(data.relativeOutputPath) {
693             moduleLibrary(data.moduleName, data.libraryName)
694           }
695         }
696       }
697       layout.resourcePaths.each {
698         def path = FileUtil.toSystemIndependentName(new File("${basePath(buildContext, it.moduleName)}/$it.resourcePath").absolutePath)
699         if (it.packToZip) {
700           zip(it.relativeOutputPath) {
701             if (new File(path).isFile()) {
702               ant.fileset(file: path)
703             }
704             else {
705               ant.fileset(dir: path)
706             }
707           }
708         }
709         else {
710           dir(it.relativeOutputPath) {
711             if (new File(path).isFile()) {
712               ant.fileset(file: path)
713             }
714             else {
715               ant.fileset(dir: path)
716             }
717           }
718         }
719       }
720       additionalResources.each {
721         File resource = it.first
722         dir(it.second) {
723           if (resource.isFile()) {
724             ant.fileset(file: resource.absolutePath)
725           }
726           else {
727             ant.fileset(dir: resource.absolutePath)
728           }
729         }
730       }
731     }
732   }
733
734   private void checkModuleExcludes(MultiValuesMap<String, String> moduleExcludes) {
735     moduleExcludes.entrySet().each { entry ->
736       String module = entry.key
737       entry.value.each { pattern ->
738         def moduleOutput = new File(buildContext.getModuleOutputPath(buildContext.findRequiredModule(module)))
739         if (!moduleOutput.exists()) {
740           buildContext.messages.error("There are excludes defined for module '$module', but the module wasn't compiled; " +
741                                       "most probably it means that '$module' isn't include into the product distribution so it makes no sense to define excludes for it.")
742         }
743         if (createFileSet(pattern, moduleOutput).size() == 0) {
744           buildContext.messages.error("Incorrect excludes for module '$module': nothing matches to $pattern in the module output at $moduleOutput")
745         }
746       }
747     }
748   }
749
750   private FileSet createFileSet(String pattern, File baseDir) {
751     def fileSet = new FileSet()
752     fileSet.setProject(buildContext.ant.antProject)
753     fileSet.setDir(baseDir)
754     fileSet.createInclude().setName(pattern)
755     return fileSet
756   }
757
758   static String basePath(BuildContext buildContext, String moduleName) {
759     JpsPathUtil.urlToPath(buildContext.findRequiredModule(moduleName).contentRootsList.urls.first())
760   }
761
762   private LayoutBuilder createLayoutBuilder() {
763     new LayoutBuilder(buildContext, COMPRESS_JARS)
764   }
765
766   private void setPluginVersionAndSince(String pluginXmlPath, String version, String buildNumber,
767                                         CompatibleBuildRange compatibleBuildRange) {
768     buildContext.ant.replaceregexp(file: pluginXmlPath,
769                                    match: "<version>[\\d.]*</version>",
770                                    replace: "<version>${version}</version>")
771     def sinceBuild
772     def untilBuild
773     if (compatibleBuildRange != CompatibleBuildRange.EXACT && buildNumber.matches(/(\d+\.)+\d+/)) {
774       if (compatibleBuildRange == CompatibleBuildRange.ANY_WITH_SAME_BASELINE) {
775         sinceBuild = buildNumber.substring(0, buildNumber.indexOf('.'))
776         untilBuild = buildNumber.substring(0, buildNumber.indexOf('.')) + ".*"
777       }
778       else {
779         if (buildNumber.matches(/\d+\.\d+/)) {
780           sinceBuild = buildNumber
781         }
782         else {
783           sinceBuild = buildNumber.substring(0, buildNumber.lastIndexOf('.'))
784         }
785         int end = compatibleBuildRange == CompatibleBuildRange.RESTRICTED_TO_SAME_RELEASE ? buildNumber.lastIndexOf('.') :
786                   buildNumber.indexOf('.')
787         untilBuild = buildNumber.substring(0, end) + ".*"
788       }
789     }
790     else {
791       sinceBuild = buildNumber
792       untilBuild = buildNumber
793     }
794     buildContext.ant.replaceregexp(file: pluginXmlPath,
795                                    match: "<idea-version\\s*since-build=\"\\d+\\.\\d+\"\\s*until-build=\"\\d+\\.\\d+\"",
796                                    replace: "<idea-version since-build=\"${sinceBuild}\" until-build=\"${untilBuild}\"")
797     buildContext.ant.replaceregexp(file: pluginXmlPath,
798                                    match: "<idea-version\\s*since-build=\"\\d+\\.\\d+\"",
799                                    replace: "<idea-version since-build=\"${sinceBuild}\"")
800     buildContext.ant.replaceregexp(file: pluginXmlPath,
801                                    match: "<change-notes>\\s*<\\!\\[CDATA\\[\\s*Plugin version: \\\$\\{version\\}",
802                                    replace: "<change-notes>\n<![CDATA[\nPlugin version: ${version}")
803     def file = new File(pluginXmlPath)
804     def text = file.text
805     def anchor = text.contains("</id>") ? "</id>" : "</name>"
806     if (!text.contains("<version>")) {
807       file.text = text.replace(anchor, "${anchor}\n  <version>${version}</version>")
808       text = file.text
809     }
810     if (!text.contains("<idea-version since-build")) {
811       file.text = text.replace(anchor, "${anchor}\n  <idea-version since-build=\"${sinceBuild}\" until-build=\"${untilBuild}\"/>")
812     }
813   }
814
815   private File createKeyMapWithAltClickReassignedToMultipleCarets() {
816     def sourceFile = new File("${buildContext.getModuleOutputPath(buildContext.findModule("intellij.platform.resources"))}/keymaps/\$default.xml")
817     String defaultKeymapContent = sourceFile.text
818     defaultKeymapContent = defaultKeymapContent.replace("<mouse-shortcut keystroke=\"alt button1\"/>",
819                                                         "<mouse-shortcut keystroke=\"to be alt shift button1\"/>")
820     defaultKeymapContent = defaultKeymapContent.replace("<mouse-shortcut keystroke=\"alt shift button1\"/>",
821                                                         "<mouse-shortcut keystroke=\"alt button1\"/>")
822     defaultKeymapContent = defaultKeymapContent.replace("<mouse-shortcut keystroke=\"to be alt shift button1\"/>",
823                                                         "<mouse-shortcut keystroke=\"alt shift button1\"/>")
824     def patchedKeyMapDir = new File(buildContext.paths.temp, "patched-keymap")
825     def targetFile = new File(patchedKeyMapDir, "keymaps/\$default.xml")
826     FileUtil.createParentDirs(targetFile)
827     targetFile.text = defaultKeymapContent
828     return patchedKeyMapDir
829   }
830 }