platform/lang-impl/src/com/intellij/util/download/impl/FileDownloaderImpl.java

   1 /*
   2  * Copyright 2000-2014 JetBrains s.r.o.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package com.intellij.util.download.impl;
  17
  18 import com.google.common.base.Throwables;
  19 import com.google.common.util.concurrent.AtomicDouble;
  20 import com.intellij.concurrency.SensitiveProgressWrapper;
  21 import com.intellij.ide.IdeBundle;
  22 import com.intellij.openapi.application.PathManager;
  23 import com.intellij.openapi.application.Result;
  24 import com.intellij.openapi.application.WriteAction;
  25 import com.intellij.openapi.diagnostic.Logger;
  26 import com.intellij.openapi.fileChooser.FileChooser;
  27 import com.intellij.openapi.fileChooser.FileChooserDescriptor;
  28 import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory;
  29 import com.intellij.openapi.progress.EmptyProgressIndicator;
  30 import com.intellij.openapi.progress.ProcessCanceledException;
  31 import com.intellij.openapi.progress.ProgressIndicator;
  32 import com.intellij.openapi.progress.ProgressManager;
  33 import com.intellij.openapi.project.Project;
  34 import com.intellij.openapi.util.Condition;
  35 import com.intellij.openapi.util.Pair;
  36 import com.intellij.openapi.util.Ref;
  37 import com.intellij.openapi.util.io.FileUtil;
  38 import com.intellij.openapi.util.text.StringUtil;
  39 import com.intellij.openapi.vfs.*;
  40 import com.intellij.util.io.HttpRequests;
  41 import com.intellij.util.concurrency.BoundedTaskExecutor;
  42 import com.intellij.util.containers.hash.LinkedHashMap;
  43 import com.intellij.util.download.DownloadableFileDescription;
  44 import com.intellij.util.download.FileDownloader;
  45 import com.intellij.util.net.IOExceptionDialog;
  46 import com.intellij.util.net.NetUtils;
  47 import org.jetbrains.annotations.NotNull;
  48 import org.jetbrains.annotations.Nullable;
  49 import org.jetbrains.ide.PooledThreadExecutor;
  50
  51 import javax.swing.*;
  52 import java.io.*;
  53 import java.util.ArrayList;
  54 import java.util.List;
  55 import java.util.concurrent.Callable;
  56 import java.util.concurrent.ExecutionException;
  57 import java.util.concurrent.Future;
  58 import java.util.concurrent.atomic.AtomicLong;
  59
  60 /**
  61  * @author nik
  62  */
  63 public class FileDownloaderImpl implements FileDownloader {
  64   private static final Logger LOG = Logger.getInstance(FileDownloaderImpl.class);
  65   private static final String LIB_SCHEMA = "lib://";
  66
  67   private final List<? extends DownloadableFileDescription> myFileDescriptions;
  68   private final JComponent myParentComponent;
  69   @Nullable private final Project myProject;
  70   private String myDirectoryForDownloadedFilesPath;
  71   private final String myDialogTitle;
  72
  73   public FileDownloaderImpl(@NotNull List<? extends DownloadableFileDescription> fileDescriptions,
  74                             @Nullable Project project,
  75                             @Nullable JComponent parentComponent,
  76                             @NotNull String presentableDownloadName) {
  77     myProject = project;
  78     myFileDescriptions = fileDescriptions;
  79     myParentComponent = parentComponent;
  80     myDialogTitle = IdeBundle.message("progress.download.0.title", StringUtil.capitalize(presentableDownloadName));
  81   }
  82
  83   @Nullable
  84   @Override
  85   public List<VirtualFile> downloadFilesWithProgress(@Nullable String targetDirectoryPath,
  86                                                      @Nullable Project project,
  87                                                      @Nullable JComponent parentComponent) {
  88     final List<Pair<VirtualFile, DownloadableFileDescription>> pairs = downloadWithProgress(targetDirectoryPath, project, parentComponent);
  89     if (pairs == null) return null;
  90
  91     List<VirtualFile> files = new ArrayList<VirtualFile>();
  92     for (Pair<VirtualFile, DownloadableFileDescription> pair : pairs) {
  93       files.add(pair.getFirst());
  94     }
  95     return files;
  96   }
  97
  98   @Nullable
  99   @Override
 100   public List<Pair<VirtualFile, DownloadableFileDescription>> downloadWithProgress(@Nullable String targetDirectoryPath,
 101                                                                                    @Nullable Project project,
 102                                                                                    @Nullable JComponent parentComponent) {
 103     File dir;
 104     if (targetDirectoryPath != null) {
 105       dir = new File(targetDirectoryPath);
 106     }
 107     else {
 108       VirtualFile virtualDir = chooseDirectoryForFiles(project, parentComponent);
 109       if (virtualDir != null) {
 110         dir = VfsUtilCore.virtualToIoFile(virtualDir);
 111       }
 112       else {
 113         return null;
 114       }
 115     }
 116
 117     return downloadWithProcess(dir, project, parentComponent);
 118   }
 119
 120   @Nullable
 121   private List<Pair<VirtualFile,DownloadableFileDescription>> downloadWithProcess(final File targetDir,
 122                                                                                   Project project,
 123                                                                                   JComponent parentComponent) {
 124     final Ref<List<Pair<File, DownloadableFileDescription>>> localFiles = Ref.create(null);
 125     final Ref<IOException> exceptionRef = Ref.create(null);
 126
 127     boolean completed = ProgressManager.getInstance().runProcessWithProgressSynchronously(new Runnable() {
 128       @Override
 129       public void run() {
 130         try {
 131           localFiles.set(download(targetDir));
 132         }
 133         catch (IOException e) {
 134           exceptionRef.set(e);
 135         }
 136       }
 137     }, myDialogTitle, true, project, parentComponent);
 138     if (!completed) {
 139       return null;
 140     }
 141
 142     @SuppressWarnings("ThrowableResultOfMethodCallIgnored") Exception exception = exceptionRef.get();
 143     if (exception != null) {
 144       final boolean tryAgain = IOExceptionDialog.showErrorDialog(myDialogTitle, exception.getMessage());
 145       if (tryAgain) {
 146         return downloadWithProcess(targetDir, project, parentComponent);
 147       }
 148       return null;
 149     }
 150
 151     return findVirtualFiles(localFiles.get());
 152   }
 153
 154   @NotNull
 155   @Override
 156   public List<Pair<File, DownloadableFileDescription>> download(@NotNull final File targetDir) throws IOException {
 157     final List<Pair<File, DownloadableFileDescription>> downloadedFiles = new ArrayList<Pair<File, DownloadableFileDescription>>();
 158     final List<Pair<File, DownloadableFileDescription>> existingFiles = new ArrayList<Pair<File, DownloadableFileDescription>>();
 159     ProgressIndicator parentIndicator = ProgressManager.getInstance().getProgressIndicator();
 160     if (parentIndicator == null) {
 161       parentIndicator = new EmptyProgressIndicator();
 162     }
 163
 164     try {
 165       final ConcurrentTasksProgressManager progressManager = new ConcurrentTasksProgressManager(parentIndicator, myFileDescriptions.size());
 166       parentIndicator.setText(IdeBundle.message("progress.downloading.0.files.text", myFileDescriptions.size()));
 167       int maxParallelDownloads = Runtime.getRuntime().availableProcessors();
 168       LOG.debug("Downloading " + myFileDescriptions.size() + " files using " + maxParallelDownloads + " threads");
 169       long start = System.currentTimeMillis();
 170       BoundedTaskExecutor executor = new BoundedTaskExecutor(PooledThreadExecutor.INSTANCE, maxParallelDownloads);
 171       List<Future<Void>> results = new ArrayList<Future<Void>>();
 172       final AtomicLong totalSize = new AtomicLong();
 173       for (final DownloadableFileDescription description : myFileDescriptions) {
 174         results.add(executor.submit(new Callable<Void>() {
 175           @Override
 176           public Void call() throws Exception {
 177             SubTaskProgressIndicator indicator = progressManager.createSubTaskIndicator();
 178             indicator.checkCanceled();
 179
 180             final File existing = new File(targetDir, description.getDefaultFileName());
 181             final String url = description.getDownloadUrl();
 182             if (url.startsWith(LIB_SCHEMA)) {
 183               final String path = FileUtil.toSystemDependentName(StringUtil.trimStart(url, LIB_SCHEMA));
 184               final File file = PathManager.findFileInLibDirectory(path);
 185               existingFiles.add(Pair.create(file, description));
 186             }
 187             else if (url.startsWith(LocalFileSystem.PROTOCOL_PREFIX)) {
 188               String path = FileUtil.toSystemDependentName(StringUtil.trimStart(url, LocalFileSystem.PROTOCOL_PREFIX));
 189               File file = new File(path);
 190               if (file.exists()) {
 191                 existingFiles.add(Pair.create(file, description));
 192               }
 193             }
 194             else {
 195               File downloaded;
 196               try {
 197                 downloaded = downloadFile(description, existing, indicator);
 198               }
 199               catch (IOException e) {
 200                 throw new IOException(IdeBundle.message("error.file.download.failed", description.getDownloadUrl(), e.getMessage()), e);
 201               }
 202               if (FileUtil.filesEqual(downloaded, existing)) {
 203                 existingFiles.add(Pair.create(existing, description));
 204               }
 205               else {
 206                 totalSize.addAndGet(downloaded.length());
 207                 downloadedFiles.add(Pair.create(downloaded, description));
 208               }
 209             }
 210             indicator.finished();
 211             return null;
 212           }
 213         }));
 214       }
 215
 216       for (Future<Void> result : results) {
 217         try {
 218           result.get();
 219         }
 220         catch (InterruptedException e) {
 221           throw new ProcessCanceledException();
 222         }
 223         catch (ExecutionException e) {
 224           Throwables.propagateIfInstanceOf(e.getCause(), IOException.class);
 225           Throwables.propagateIfInstanceOf(e.getCause(), ProcessCanceledException.class);
 226           LOG.error(e);
 227         }
 228       }
 229       long duration = System.currentTimeMillis() - start;
 230       LOG.debug("Downloaded " + StringUtil.formatFileSize(totalSize.get()) + " in " + StringUtil.formatDuration(duration) + "(" + duration + "ms)");
 231
 232       List<Pair<File, DownloadableFileDescription>> localFiles = new ArrayList<Pair<File, DownloadableFileDescription>>();
 233       localFiles.addAll(moveToDir(downloadedFiles, targetDir));
 234       localFiles.addAll(existingFiles);
 235       return localFiles;
 236     }
 237     catch (ProcessCanceledException e) {
 238       deleteFiles(downloadedFiles);
 239       throw e;
 240     }
 241     catch (IOException e) {
 242       deleteFiles(downloadedFiles);
 243       throw e;
 244     }
 245   }
 246
 247   @Nullable
 248   private static VirtualFile chooseDirectoryForFiles(Project project, JComponent parentComponent) {
 249     FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor()
 250       .withTitle(IdeBundle.message("dialog.directory.for.downloaded.files.title"))
 251       .withDescription(IdeBundle.message("dialog.directory.for.downloaded.files.description"));
 252     VirtualFile baseDir = project != null ? project.getBaseDir() : null;
 253     return FileChooser.chooseFile(descriptor, parentComponent, project, baseDir);
 254   }
 255
 256   private static List<Pair<File, DownloadableFileDescription>> moveToDir(List<Pair<File, DownloadableFileDescription>> downloadedFiles,
 257                                                                          final File targetDir) throws IOException {
 258     FileUtil.createDirectory(targetDir);
 259     List<Pair<File, DownloadableFileDescription>> result = new ArrayList<Pair<File, DownloadableFileDescription>>();
 260     for (Pair<File, DownloadableFileDescription> pair : downloadedFiles) {
 261       final DownloadableFileDescription description = pair.getSecond();
 262       final String fileName = description.generateFileName(new Condition<String>() {
 263         @Override
 264         public boolean value(String s) {
 265           return !new File(targetDir, s).exists();
 266         }
 267       });
 268       final File toFile = new File(targetDir, fileName);
 269       FileUtil.rename(pair.getFirst(), toFile);
 270       result.add(Pair.create(toFile, description));
 271     }
 272     return result;
 273   }
 274
 275   @NotNull
 276   private static List<Pair<VirtualFile, DownloadableFileDescription>> findVirtualFiles(List<Pair<File, DownloadableFileDescription>> ioFiles) {
 277     List<Pair<VirtualFile,DownloadableFileDescription>> result = new ArrayList<Pair<VirtualFile, DownloadableFileDescription>>();
 278     for (final Pair<File, DownloadableFileDescription> pair : ioFiles) {
 279       final File ioFile = pair.getFirst();
 280       VirtualFile libraryRootFile = new WriteAction<VirtualFile>() {
 281         @Override
 282         protected void run(@NotNull Result<VirtualFile> result) {
 283           final String url = VfsUtil.getUrlForLibraryRoot(ioFile);
 284           LocalFileSystem.getInstance().refreshAndFindFileByIoFile(ioFile);
 285           result.setResult(VirtualFileManager.getInstance().refreshAndFindFileByUrl(url));
 286         }
 287
 288       }.execute().getResultObject();
 289       if (libraryRootFile != null) {
 290         result.add(Pair.create(libraryRootFile, pair.getSecond()));
 291       }
 292     }
 293     return result;
 294   }
 295
 296   private static void deleteFiles(final List<Pair<File, DownloadableFileDescription>> pairs) {
 297     for (Pair<File, DownloadableFileDescription> pair : pairs) {
 298       FileUtil.delete(pair.getFirst());
 299     }
 300   }
 301
 302   @NotNull
 303   private static File downloadFile(@NotNull final DownloadableFileDescription description,
 304                                    @NotNull final File existingFile,
 305                                    @NotNull final ProgressIndicator indicator) throws IOException {
 306     final String presentableUrl = description.getPresentableDownloadUrl();
 307     indicator.setText2(IdeBundle.message("progress.connecting.to.download.file.text", presentableUrl));
 308     indicator.setIndeterminate(true);
 309
 310     return HttpRequests.request(description.getDownloadUrl()).connect(new HttpRequests.RequestProcessor<File>() {
 311       @Override
 312       public File process(@NotNull HttpRequests.Request request) throws IOException {
 313         int size = request.getConnection().getContentLength();
 314         if (existingFile.exists() && size == existingFile.length()) {
 315           return existingFile;
 316         }
 317
 318         File tempFile = FileUtil.createTempFile("download.", ".tmp");
 319         boolean deleteFile = true;
 320         OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile));
 321         try {
 322           indicator.setText2(IdeBundle.message("progress.download.file.text", description.getPresentableFileName(), presentableUrl));
 323           indicator.setIndeterminate(size == -1);
 324           NetUtils.copyStreamContent(indicator, request.getInputStream(), out, size);
 325           deleteFile = false;
 326           return tempFile;
 327         }
 328         finally {
 329           out.close();
 330           if (deleteFile) {
 331             FileUtil.delete(tempFile);
 332           }
 333         }
 334       }
 335     });
 336   }
 337
 338   @NotNull
 339   @Override
 340   public FileDownloader toDirectory(@NotNull String directoryForDownloadedFilesPath) {
 341     myDirectoryForDownloadedFilesPath = directoryForDownloadedFilesPath;
 342     return this;
 343   }
 344
 345   @Nullable
 346   @Override
 347   public VirtualFile[] download() {
 348     List<VirtualFile> files = downloadFilesWithProgress(myDirectoryForDownloadedFilesPath, myProject, myParentComponent);
 349     return files != null ? VfsUtilCore.toVirtualFileArray(files) : null;
 350   }
 351
 352   @Nullable
 353   @Override
 354   public List<Pair<VirtualFile, DownloadableFileDescription>> downloadAndReturnWithDescriptions() {
 355     return downloadWithProgress(myDirectoryForDownloadedFilesPath, myProject, myParentComponent);
 356   }
 357
 358   private static class ConcurrentTasksProgressManager {
 359     private final ProgressIndicator myParent;
 360     private final int myTasksCount;
 361     private final AtomicDouble myTotalFraction;
 362     private final Object myLock = new Object();
 363     private final LinkedHashMap<SubTaskProgressIndicator, String> myText2Stack = new LinkedHashMap<SubTaskProgressIndicator, String>();
 364
 365     private ConcurrentTasksProgressManager(ProgressIndicator parent, int tasksCount) {
 366       myParent = parent;
 367       myTasksCount = tasksCount;
 368       myTotalFraction = new AtomicDouble();
 369     }
 370
 371     public void updateFraction(double delta) {
 372       myTotalFraction.addAndGet(delta / myTasksCount);
 373       myParent.setFraction(myTotalFraction.get());
 374     }
 375
 376     public SubTaskProgressIndicator createSubTaskIndicator() {
 377       return new SubTaskProgressIndicator(this);
 378     }
 379
 380     public void setText2(@NotNull SubTaskProgressIndicator subTask, @Nullable String text) {
 381       if (text != null) {
 382         synchronized (myLock) {
 383           myText2Stack.put(subTask, text);
 384         }
 385         myParent.setText2(text);
 386       }
 387       else {
 388         String prev;
 389         synchronized (myLock) {
 390           myText2Stack.remove(subTask);
 391           prev = myText2Stack.getLastValue();
 392         }
 393         if (prev != null) {
 394           myParent.setText2(prev);
 395         }
 396       }
 397     }
 398   }
 399
 400   private static class SubTaskProgressIndicator extends SensitiveProgressWrapper {
 401     private final AtomicDouble myFraction;
 402     private final ConcurrentTasksProgressManager myProgressManager;
 403
 404     private SubTaskProgressIndicator(ConcurrentTasksProgressManager progressManager) {
 405       super(progressManager.myParent);
 406       myProgressManager = progressManager;
 407       myFraction = new AtomicDouble();
 408     }
 409
 410     @Override
 411     public void setFraction(double newValue) {
 412       double oldValue = myFraction.getAndSet(newValue);
 413       myProgressManager.updateFraction(newValue - oldValue);
 414     }
 415
 416     @Override
 417     public void setIndeterminate(boolean indeterminate) {
 418       if (myProgressManager.myTasksCount > 1) return;
 419       super.setIndeterminate(indeterminate);
 420     }
 421
 422     @Override
 423     public void setText2(String text) {
 424       myProgressManager.setText2(this, text);
 425     }
 426
 427     @Override
 428     public double getFraction() {
 429       return myFraction.get();
 430     }
 431
 432     public void finished() {
 433       setFraction(1);
 434       myProgressManager.setText2(this, null);
 435     }
 436   }
 437 }