IDEA-130853 (HTTP redirects in library downloader)
[idea/community.git] / platform / lang-impl / src / com / intellij / util / download / impl / FileDownloaderImpl.java
1 /*
2  * Copyright 2000-2014 JetBrains s.r.o.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.intellij.util.download.impl;
17
18 import com.google.common.base.Throwables;
19 import com.google.common.util.concurrent.AtomicDouble;
20 import com.intellij.concurrency.SensitiveProgressWrapper;
21 import com.intellij.ide.IdeBundle;
22 import com.intellij.openapi.application.PathManager;
23 import com.intellij.openapi.application.Result;
24 import com.intellij.openapi.application.WriteAction;
25 import com.intellij.openapi.diagnostic.Logger;
26 import com.intellij.openapi.fileChooser.FileChooser;
27 import com.intellij.openapi.fileChooser.FileChooserDescriptor;
28 import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory;
29 import com.intellij.openapi.progress.EmptyProgressIndicator;
30 import com.intellij.openapi.progress.ProcessCanceledException;
31 import com.intellij.openapi.progress.ProgressIndicator;
32 import com.intellij.openapi.progress.ProgressManager;
33 import com.intellij.openapi.project.Project;
34 import com.intellij.openapi.util.Condition;
35 import com.intellij.openapi.util.Pair;
36 import com.intellij.openapi.util.Ref;
37 import com.intellij.openapi.util.io.FileUtil;
38 import com.intellij.openapi.util.text.StringUtil;
39 import com.intellij.openapi.vfs.*;
40 import com.intellij.util.io.HttpRequests;
41 import com.intellij.util.concurrency.BoundedTaskExecutor;
42 import com.intellij.util.containers.hash.LinkedHashMap;
43 import com.intellij.util.download.DownloadableFileDescription;
44 import com.intellij.util.download.FileDownloader;
45 import com.intellij.util.net.IOExceptionDialog;
46 import com.intellij.util.net.NetUtils;
47 import org.jetbrains.annotations.NotNull;
48 import org.jetbrains.annotations.Nullable;
49 import org.jetbrains.ide.PooledThreadExecutor;
50
51 import javax.swing.*;
52 import java.io.*;
53 import java.util.ArrayList;
54 import java.util.List;
55 import java.util.concurrent.Callable;
56 import java.util.concurrent.ExecutionException;
57 import java.util.concurrent.Future;
58 import java.util.concurrent.atomic.AtomicLong;
59
60 /**
61  * @author nik
62  */
63 public class FileDownloaderImpl implements FileDownloader {
64   private static final Logger LOG = Logger.getInstance(FileDownloaderImpl.class);
65   private static final String LIB_SCHEMA = "lib://";
66
67   private final List<? extends DownloadableFileDescription> myFileDescriptions;
68   private final JComponent myParentComponent;
69   @Nullable private final Project myProject;
70   private String myDirectoryForDownloadedFilesPath;
71   private final String myDialogTitle;
72
73   public FileDownloaderImpl(@NotNull List<? extends DownloadableFileDescription> fileDescriptions,
74                             @Nullable Project project,
75                             @Nullable JComponent parentComponent,
76                             @NotNull String presentableDownloadName) {
77     myProject = project;
78     myFileDescriptions = fileDescriptions;
79     myParentComponent = parentComponent;
80     myDialogTitle = IdeBundle.message("progress.download.0.title", StringUtil.capitalize(presentableDownloadName));
81   }
82
83   @Nullable
84   @Override
85   public List<VirtualFile> downloadFilesWithProgress(@Nullable String targetDirectoryPath,
86                                                      @Nullable Project project,
87                                                      @Nullable JComponent parentComponent) {
88     final List<Pair<VirtualFile, DownloadableFileDescription>> pairs = downloadWithProgress(targetDirectoryPath, project, parentComponent);
89     if (pairs == null) return null;
90
91     List<VirtualFile> files = new ArrayList<VirtualFile>();
92     for (Pair<VirtualFile, DownloadableFileDescription> pair : pairs) {
93       files.add(pair.getFirst());
94     }
95     return files;
96   }
97
98   @Nullable
99   @Override
100   public List<Pair<VirtualFile, DownloadableFileDescription>> downloadWithProgress(@Nullable String targetDirectoryPath,
101                                                                                    @Nullable Project project,
102                                                                                    @Nullable JComponent parentComponent) {
103     File dir;
104     if (targetDirectoryPath != null) {
105       dir = new File(targetDirectoryPath);
106     }
107     else {
108       VirtualFile virtualDir = chooseDirectoryForFiles(project, parentComponent);
109       if (virtualDir != null) {
110         dir = VfsUtilCore.virtualToIoFile(virtualDir);
111       }
112       else {
113         return null;
114       }
115     }
116
117     return downloadWithProcess(dir, project, parentComponent);
118   }
119
120   @Nullable
121   private List<Pair<VirtualFile,DownloadableFileDescription>> downloadWithProcess(final File targetDir,
122                                                                                   Project project,
123                                                                                   JComponent parentComponent) {
124     final Ref<List<Pair<File, DownloadableFileDescription>>> localFiles = Ref.create(null);
125     final Ref<IOException> exceptionRef = Ref.create(null);
126
127     boolean completed = ProgressManager.getInstance().runProcessWithProgressSynchronously(new Runnable() {
128       @Override
129       public void run() {
130         try {
131           localFiles.set(download(targetDir));
132         }
133         catch (IOException e) {
134           exceptionRef.set(e);
135         }
136       }
137     }, myDialogTitle, true, project, parentComponent);
138     if (!completed) {
139       return null;
140     }
141
142     @SuppressWarnings("ThrowableResultOfMethodCallIgnored") Exception exception = exceptionRef.get();
143     if (exception != null) {
144       final boolean tryAgain = IOExceptionDialog.showErrorDialog(myDialogTitle, exception.getMessage());
145       if (tryAgain) {
146         return downloadWithProcess(targetDir, project, parentComponent);
147       }
148       return null;
149     }
150
151     return findVirtualFiles(localFiles.get());
152   }
153
154   @NotNull
155   @Override
156   public List<Pair<File, DownloadableFileDescription>> download(@NotNull final File targetDir) throws IOException {
157     final List<Pair<File, DownloadableFileDescription>> downloadedFiles = new ArrayList<Pair<File, DownloadableFileDescription>>();
158     final List<Pair<File, DownloadableFileDescription>> existingFiles = new ArrayList<Pair<File, DownloadableFileDescription>>();
159     ProgressIndicator parentIndicator = ProgressManager.getInstance().getProgressIndicator();
160     if (parentIndicator == null) {
161       parentIndicator = new EmptyProgressIndicator();
162     }
163
164     try {
165       final ConcurrentTasksProgressManager progressManager = new ConcurrentTasksProgressManager(parentIndicator, myFileDescriptions.size());
166       parentIndicator.setText(IdeBundle.message("progress.downloading.0.files.text", myFileDescriptions.size()));
167       int maxParallelDownloads = Runtime.getRuntime().availableProcessors();
168       LOG.debug("Downloading " + myFileDescriptions.size() + " files using " + maxParallelDownloads + " threads");
169       long start = System.currentTimeMillis();
170       BoundedTaskExecutor executor = new BoundedTaskExecutor(PooledThreadExecutor.INSTANCE, maxParallelDownloads);
171       List<Future<Void>> results = new ArrayList<Future<Void>>();
172       final AtomicLong totalSize = new AtomicLong();
173       for (final DownloadableFileDescription description : myFileDescriptions) {
174         results.add(executor.submit(new Callable<Void>() {
175           @Override
176           public Void call() throws Exception {
177             SubTaskProgressIndicator indicator = progressManager.createSubTaskIndicator();
178             indicator.checkCanceled();
179
180             final File existing = new File(targetDir, description.getDefaultFileName());
181             final String url = description.getDownloadUrl();
182             if (url.startsWith(LIB_SCHEMA)) {
183               final String path = FileUtil.toSystemDependentName(StringUtil.trimStart(url, LIB_SCHEMA));
184               final File file = PathManager.findFileInLibDirectory(path);
185               existingFiles.add(Pair.create(file, description));
186             }
187             else if (url.startsWith(LocalFileSystem.PROTOCOL_PREFIX)) {
188               String path = FileUtil.toSystemDependentName(StringUtil.trimStart(url, LocalFileSystem.PROTOCOL_PREFIX));
189               File file = new File(path);
190               if (file.exists()) {
191                 existingFiles.add(Pair.create(file, description));
192               }
193             }
194             else {
195               File downloaded;
196               try {
197                 downloaded = downloadFile(description, existing, indicator);
198               }
199               catch (IOException e) {
200                 throw new IOException(IdeBundle.message("error.file.download.failed", description.getDownloadUrl(), e.getMessage()), e);
201               }
202               if (FileUtil.filesEqual(downloaded, existing)) {
203                 existingFiles.add(Pair.create(existing, description));
204               }
205               else {
206                 totalSize.addAndGet(downloaded.length());
207                 downloadedFiles.add(Pair.create(downloaded, description));
208               }
209             }
210             indicator.finished();
211             return null;
212           }
213         }));
214       }
215
216       for (Future<Void> result : results) {
217         try {
218           result.get();
219         }
220         catch (InterruptedException e) {
221           throw new ProcessCanceledException();
222         }
223         catch (ExecutionException e) {
224           Throwables.propagateIfInstanceOf(e.getCause(), IOException.class);
225           Throwables.propagateIfInstanceOf(e.getCause(), ProcessCanceledException.class);
226           LOG.error(e);
227         }
228       }
229       long duration = System.currentTimeMillis() - start;
230       LOG.debug("Downloaded " + StringUtil.formatFileSize(totalSize.get()) + " in " + StringUtil.formatDuration(duration) + "(" + duration + "ms)");
231
232       List<Pair<File, DownloadableFileDescription>> localFiles = new ArrayList<Pair<File, DownloadableFileDescription>>();
233       localFiles.addAll(moveToDir(downloadedFiles, targetDir));
234       localFiles.addAll(existingFiles);
235       return localFiles;
236     }
237     catch (ProcessCanceledException e) {
238       deleteFiles(downloadedFiles);
239       throw e;
240     }
241     catch (IOException e) {
242       deleteFiles(downloadedFiles);
243       throw e;
244     }
245   }
246
247   @Nullable
248   private static VirtualFile chooseDirectoryForFiles(Project project, JComponent parentComponent) {
249     FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor()
250       .withTitle(IdeBundle.message("dialog.directory.for.downloaded.files.title"))
251       .withDescription(IdeBundle.message("dialog.directory.for.downloaded.files.description"));
252     VirtualFile baseDir = project != null ? project.getBaseDir() : null;
253     return FileChooser.chooseFile(descriptor, parentComponent, project, baseDir);
254   }
255
256   private static List<Pair<File, DownloadableFileDescription>> moveToDir(List<Pair<File, DownloadableFileDescription>> downloadedFiles,
257                                                                          final File targetDir) throws IOException {
258     FileUtil.createDirectory(targetDir);
259     List<Pair<File, DownloadableFileDescription>> result = new ArrayList<Pair<File, DownloadableFileDescription>>();
260     for (Pair<File, DownloadableFileDescription> pair : downloadedFiles) {
261       final DownloadableFileDescription description = pair.getSecond();
262       final String fileName = description.generateFileName(new Condition<String>() {
263         @Override
264         public boolean value(String s) {
265           return !new File(targetDir, s).exists();
266         }
267       });
268       final File toFile = new File(targetDir, fileName);
269       FileUtil.rename(pair.getFirst(), toFile);
270       result.add(Pair.create(toFile, description));
271     }
272     return result;
273   }
274
275   @NotNull
276   private static List<Pair<VirtualFile, DownloadableFileDescription>> findVirtualFiles(List<Pair<File, DownloadableFileDescription>> ioFiles) {
277     List<Pair<VirtualFile,DownloadableFileDescription>> result = new ArrayList<Pair<VirtualFile, DownloadableFileDescription>>();
278     for (final Pair<File, DownloadableFileDescription> pair : ioFiles) {
279       final File ioFile = pair.getFirst();
280       VirtualFile libraryRootFile = new WriteAction<VirtualFile>() {
281         @Override
282         protected void run(@NotNull Result<VirtualFile> result) {
283           final String url = VfsUtil.getUrlForLibraryRoot(ioFile);
284           LocalFileSystem.getInstance().refreshAndFindFileByIoFile(ioFile);
285           result.setResult(VirtualFileManager.getInstance().refreshAndFindFileByUrl(url));
286         }
287
288       }.execute().getResultObject();
289       if (libraryRootFile != null) {
290         result.add(Pair.create(libraryRootFile, pair.getSecond()));
291       }
292     }
293     return result;
294   }
295
296   private static void deleteFiles(final List<Pair<File, DownloadableFileDescription>> pairs) {
297     for (Pair<File, DownloadableFileDescription> pair : pairs) {
298       FileUtil.delete(pair.getFirst());
299     }
300   }
301
302   @NotNull
303   private static File downloadFile(@NotNull final DownloadableFileDescription description,
304                                    @NotNull final File existingFile,
305                                    @NotNull final ProgressIndicator indicator) throws IOException {
306     final String presentableUrl = description.getPresentableDownloadUrl();
307     indicator.setText2(IdeBundle.message("progress.connecting.to.download.file.text", presentableUrl));
308     indicator.setIndeterminate(true);
309
310     return HttpRequests.request(description.getDownloadUrl()).connect(new HttpRequests.RequestProcessor<File>() {
311       @Override
312       public File process(@NotNull HttpRequests.Request request) throws IOException {
313         int size = request.getConnection().getContentLength();
314         if (existingFile.exists() && size == existingFile.length()) {
315           return existingFile;
316         }
317
318         File tempFile = FileUtil.createTempFile("download.", ".tmp");
319         boolean deleteFile = true;
320         OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile));
321         try {
322           indicator.setText2(IdeBundle.message("progress.download.file.text", description.getPresentableFileName(), presentableUrl));
323           indicator.setIndeterminate(size == -1);
324           NetUtils.copyStreamContent(indicator, request.getInputStream(), out, size);
325           deleteFile = false;
326           return tempFile;
327         }
328         finally {
329           out.close();
330           if (deleteFile) {
331             FileUtil.delete(tempFile);
332           }
333         }
334       }
335     });
336   }
337
338   @NotNull
339   @Override
340   public FileDownloader toDirectory(@NotNull String directoryForDownloadedFilesPath) {
341     myDirectoryForDownloadedFilesPath = directoryForDownloadedFilesPath;
342     return this;
343   }
344
345   @Nullable
346   @Override
347   public VirtualFile[] download() {
348     List<VirtualFile> files = downloadFilesWithProgress(myDirectoryForDownloadedFilesPath, myProject, myParentComponent);
349     return files != null ? VfsUtilCore.toVirtualFileArray(files) : null;
350   }
351
352   @Nullable
353   @Override
354   public List<Pair<VirtualFile, DownloadableFileDescription>> downloadAndReturnWithDescriptions() {
355     return downloadWithProgress(myDirectoryForDownloadedFilesPath, myProject, myParentComponent);
356   }
357
358   private static class ConcurrentTasksProgressManager {
359     private final ProgressIndicator myParent;
360     private final int myTasksCount;
361     private final AtomicDouble myTotalFraction;
362     private final Object myLock = new Object();
363     private final LinkedHashMap<SubTaskProgressIndicator, String> myText2Stack = new LinkedHashMap<SubTaskProgressIndicator, String>();
364
365     private ConcurrentTasksProgressManager(ProgressIndicator parent, int tasksCount) {
366       myParent = parent;
367       myTasksCount = tasksCount;
368       myTotalFraction = new AtomicDouble();
369     }
370
371     public void updateFraction(double delta) {
372       myTotalFraction.addAndGet(delta / myTasksCount);
373       myParent.setFraction(myTotalFraction.get());
374     }
375
376     public SubTaskProgressIndicator createSubTaskIndicator() {
377       return new SubTaskProgressIndicator(this);
378     }
379
380     public void setText2(@NotNull SubTaskProgressIndicator subTask, @Nullable String text) {
381       if (text != null) {
382         synchronized (myLock) {
383           myText2Stack.put(subTask, text);
384         }
385         myParent.setText2(text);
386       }
387       else {
388         String prev;
389         synchronized (myLock) {
390           myText2Stack.remove(subTask);
391           prev = myText2Stack.getLastValue();
392         }
393         if (prev != null) {
394           myParent.setText2(prev);
395         }
396       }
397     }
398   }
399
400   private static class SubTaskProgressIndicator extends SensitiveProgressWrapper {
401     private final AtomicDouble myFraction;
402     private final ConcurrentTasksProgressManager myProgressManager;
403
404     private SubTaskProgressIndicator(ConcurrentTasksProgressManager progressManager) {
405       super(progressManager.myParent);
406       myProgressManager = progressManager;
407       myFraction = new AtomicDouble();
408     }
409
410     @Override
411     public void setFraction(double newValue) {
412       double oldValue = myFraction.getAndSet(newValue);
413       myProgressManager.updateFraction(newValue - oldValue);
414     }
415
416     @Override
417     public void setIndeterminate(boolean indeterminate) {
418       if (myProgressManager.myTasksCount > 1) return;
419       super.setIndeterminate(indeterminate);
420     }
421
422     @Override
423     public void setText2(String text) {
424       myProgressManager.setText2(this, text);
425     }
426
427     @Override
428     public double getFraction() {
429       return myFraction.get();
430     }
431
432     public void finished() {
433       setFraction(1);
434       myProgressManager.setText2(this, null);
435     }
436   }
437 }