Improved detection of Python integrated tools in a new project, including docstrings...

diff --git a/python/ide/src/com/jetbrains/python/PythonSourceRootConfigurator.java b/python/ide/src/com/jetbrains/python/PythonSourceRootConfigurator.java
index 8a5f4cb1f4808ce5d95fb1cc21fd715d3bbc81ff..f22bb9413beb873e267b827e1ffc8ef36c9ce1bf 100644 (file)
--- a/python/ide/src/com/jetbrains/python/PythonSourceRootConfigurator.java
+++ b/python/ide/src/com/jetbrains/python/PythonSourceRootConfigurator.java
@@ -40,11 +40,11 @@ public class PythonSourceRootConfigurator implements DirectoryProjectConfigurato
   @NonNls private static final String SETUP_PY = "setup.py";
 
   @Override
-  public void configureProject(Project project, @NotNull final VirtualFile baseDir, Ref<Module> moduleRef) {
+  public void configureProject(Project project, @NotNull VirtualFile baseDir, Ref<Module> moduleRef) {
     VirtualFile setupPy = baseDir.findChild(SETUP_PY);
     if (setupPy != null) {
       final CharSequence content = LoadTextUtil.loadText(setupPy);
-      PyFile setupPyFile = (PyFile) PsiFileFactory.getInstance(project).createFileFromText(SETUP_PY, content.toString());
+      PyFile setupPyFile = (PyFile) PsiFileFactory.getInstance(project).createFileFromText(SETUP_PY, PythonFileType.INSTANCE, content.toString());
       final SetupCallVisitor visitor = new SetupCallVisitor();
       setupPyFile.accept(visitor);
       String dir = visitor.getRootPackageDir();

diff --git a/python/src/META-INF/python-core.xml b/python/src/META-INF/python-core.xml
index 9a1cb3e3fb1e24812dd848378be17dc4d3443325..c6cef92852a89b6ff98db79faf53e7603e49a337 100644 (file)
--- a/python/src/META-INF/python-core.xml
+++ b/python/src/META-INF/python-core.xml
@@ -522,7 +522,8 @@
 
     <postStartupActivity implementation="com.jetbrains.python.sdk.PythonSdkUpdater"/>
     <postStartupActivity implementation="com.jetbrains.python.packaging.PyPIPackagesUpdater"/>
-    <postStartupActivity implementation="com.jetbrains.python.testing.PyTestRunnerUpdater"/>
+    <directoryProjectConfigurator implementation="com.jetbrains.python.testing.PyIntegratedToolsProjectConfigurator" id="integratedTools" order="after sdk"/>
+
 
     <macro implementation="com.jetbrains.python.sdk.InterpreterDirectoryMacro"/>
 
@@ -789,6 +790,10 @@
       <add-to-group group-id="XDebugger.ToolWindow.TopToolbar" relative-to-action="StepInto" anchor="after"/>
     </action>
 
+    <action class="com.jetbrains.python.codeInsight.PyProbeAction" id="PyProbeAction" text="Python Probe Action" internal="true">
+      <add-to-group group-id="Internal"/>
+    </action>
+
   </actions>
 
   <extensions defaultExtensionNs="com.intellij.spellchecker">

diff --git a/python/src/com/jetbrains/python/documentation/PyDocumentationSettings.java b/python/src/com/jetbrains/python/documentation/PyDocumentationSettings.java
index 26969482f8da1c301587fa1fd02fa61f3b36c61d..d876a1582e0010d40aa8b3c61457b49dfbd983dc 100644 (file)
--- a/python/src/com/jetbrains/python/documentation/PyDocumentationSettings.java
+++ b/python/src/com/jetbrains/python/documentation/PyDocumentationSettings.java
@@ -44,29 +44,19 @@ import java.util.List;
   storages = @Storage(file = StoragePathMacros.MODULE_FILE)
 )
 public class PyDocumentationSettings implements PersistentStateComponent<PyDocumentationSettings> {
+  public static final DocStringFormat DEFAULT_DOCSTRING_FORMAT = DocStringFormat.REST;
+
   public static PyDocumentationSettings getInstance(@NotNull Module module) {
     return ModuleServiceManager.getService(module, PyDocumentationSettings.class);
   }
 
-  @NotNull private DocStringFormat myDocStringFormat = DocStringFormat.REST;
+  @NotNull private DocStringFormat myDocStringFormat = DEFAULT_DOCSTRING_FORMAT;
   private boolean myAnalyzeDoctest = true;
 
-  public boolean isEpydocFormat(PsiFile file) {
-    return isFormat(file, DocStringFormat.EPYTEXT);
-  }
-
-  public boolean isReSTFormat(PsiFile file) {
-    return isFormat(file, DocStringFormat.REST);
-  }
-
   public boolean isNumpyFormat(PsiFile file) {
     return isFormat(file, DocStringFormat.NUMPY);
   }
 
-  public boolean isGoogleFormat(PsiFile file) {
-    return isFormat(file, DocStringFormat.GOOGLE);
-  }
-
   public boolean isPlain(PsiFile file) {
     return isFormat(file, DocStringFormat.PLAIN);
   }
@@ -77,6 +67,12 @@ public class PyDocumentationSettings implements PersistentStateComponent<PyDocum
 
   @NotNull
   public DocStringFormat getFormatForFile(@NotNull PsiFile file) {
+    final DocStringFormat fileFormat = getFormatFromDocformatAttribute(file);
+    return fileFormat != null && fileFormat != DocStringFormat.PLAIN ? fileFormat : myDocStringFormat;
+  }
+
+  @Nullable
+  public static DocStringFormat getFormatFromDocformatAttribute(@NotNull PsiFile file) {
     if (file instanceof PyFile) {
       final PyTargetExpression expr = ((PyFile)file).findTopLevelAttribute(PyNames.DOCFORMAT);
       if (expr != null) {
@@ -92,7 +88,7 @@ public class PyDocumentationSettings implements PersistentStateComponent<PyDocum
         }
       }
     }
-    return myDocStringFormat;
+    return null;
   }
 
   @Transient

diff --git a/python/src/com/jetbrains/python/documentation/docstrings/DocStringUtil.java b/python/src/com/jetbrains/python/documentation/docstrings/DocStringUtil.java
index 1bec2ce4472f4638ef0a0827400359c614389fc5..585d1d545a877e06bac7990470fdfa0e0014210f 100644 (file)
--- a/python/src/com/jetbrains/python/documentation/docstrings/DocStringUtil.java
+++ b/python/src/com/jetbrains/python/documentation/docstrings/DocStringUtil.java
@@ -164,11 +164,13 @@ public class DocStringUtil {
   }
 
   public static boolean isLikeSphinxDocString(@NotNull String text) {
-    return text.contains(":param ") || text.contains(":rtype") || text.contains(":type");
+    return text.contains(":param ") || 
+           text.contains(":return:") || text.contains(":returns:") || 
+           text.contains(":rtype") || text.contains(":type");
   }
 
   public static boolean isLikeEpydocDocString(@NotNull String text) {
-    return text.contains("@param ") || text.contains("@rtype") || text.contains("@type");
+    return text.contains("@param ") || text.contains("@return:") || text.contains("@rtype") || text.contains("@type");
   }
 
   public static boolean isLikeGoogleDocString(@NotNull String text) {

diff --git a/python/src/com/jetbrains/python/testing/PyTestRunnerUpdater.java b/python/src/com/jetbrains/python/testing/PyIntegratedToolsProjectConfigurator.java
similarity index 57%
rename from python/src/com/jetbrains/python/testing/PyTestRunnerUpdater.java
rename to python/src/com/jetbrains/python/testing/PyIntegratedToolsProjectConfigurator.java
index 67449b9b2d2a749034474324632745d9595194d1..5a2f4eba8b53acffaca777303ba3503c3e9abdc6 100644 (file)
--- a/python/src/com/jetbrains/python/testing/PyTestRunnerUpdater.java
+++ b/python/src/com/jetbrains/python/testing/PyIntegratedToolsProjectConfigurator.java
@@ -18,14 +18,15 @@ package com.jetbrains.python.testing;
 import com.intellij.openapi.application.Application;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.application.ModalityState;
+import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.module.Module;
 import com.intellij.openapi.module.ModuleManager;
 import com.intellij.openapi.module.ModuleType;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.projectRoots.Sdk;
-import com.intellij.openapi.startup.StartupActivity;
-import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.openapi.util.Ref;
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.platform.DirectoryProjectConfigurator;
 import com.intellij.psi.PsiFile;
 import com.intellij.psi.PsiManager;
 import com.intellij.psi.search.FilenameIndex;
@@ -36,6 +37,7 @@ import com.jetbrains.python.PythonFileType;
 import com.jetbrains.python.PythonModuleTypeBase;
 import com.jetbrains.python.documentation.PyDocumentationSettings;
 import com.jetbrains.python.documentation.docstrings.DocStringFormat;
+import com.jetbrains.python.documentation.docstrings.DocStringUtil;
 import com.jetbrains.python.packaging.PyPackageUtil;
 import com.jetbrains.python.psi.*;
 import com.jetbrains.python.sdk.PythonSdkType;
@@ -48,9 +50,11 @@ import java.util.List;
  * Detects test runner and docstring format
  *
  */
-public class PyTestRunnerUpdater implements StartupActivity {
+public class PyIntegratedToolsProjectConfigurator implements DirectoryProjectConfigurator {
+  private static final Logger LOG = Logger.getInstance(PyIntegratedToolsProjectConfigurator.class);
+
   @Override
-  public void runActivity(@NotNull final Project project) {
+  public void configureProject(Project project, @NotNull VirtualFile baseDir, Ref<Module> moduleRef) {
     final Application application = ApplicationManager.getApplication();
     if (application.isUnitTestMode()) {
       return;
@@ -65,6 +69,7 @@ public class PyTestRunnerUpdater implements StartupActivity {
   }
 
   private static void updateIntegratedTools(final Module module, final int delay) {
+    final PyDocumentationSettings docSettings = PyDocumentationSettings.getInstance(module);
     ApplicationManager.getApplication().executeOnPooledThread(new Runnable() {
       public void run() {
         if (delay > 0) {
@@ -74,41 +79,50 @@ public class PyTestRunnerUpdater implements StartupActivity {
           catch (InterruptedException ignore) {
           }
         }
+        
+        LOG.debug("Integrated tools configurator has started");
 
         ApplicationManager.getApplication().invokeLater(new Runnable() {
           @Override
           public void run() {
-            final TestRunnerService runnerService = TestRunnerService.getInstance(module);
-            if (runnerService == null) return;
-            final String configuration = runnerService.getProjectConfiguration();
-            if (!StringUtil.isEmptyOrSpaces(configuration))
-              return;
-
+            @NotNull DocStringFormat docFormat = DocStringFormat.PLAIN;
             //check setup.py
-            String testRunner = detectTestRunnerFromSetupPy(module);
+            @NotNull String testRunner = detectTestRunnerFromSetupPy(module);
+            if (!testRunner.isEmpty()) {
+              LOG.debug("Test runner '" + testRunner + "' was discovered from setup.py in the module '" + module.getModuleFilePath() + "'");
+            }
 
             //try to find test_runner import
-            final Collection<VirtualFile> filenames = FilenameIndex.getAllFilesByExt(module.getProject(), PythonFileType.INSTANCE.getDefaultExtension(),
-                                                                                     GlobalSearchScope.moduleScope(module));
-
-            for (VirtualFile file : filenames) {
+            final String extension = PythonFileType.INSTANCE.getDefaultExtension();
+            // Module#getModuleScope() and GlobalSearchScope#getModuleScope() search only in source roots
+            final GlobalSearchScope searchScope = module.getModuleContentScope();
+            final Collection<VirtualFile> pyFiles = FilenameIndex.getAllFilesByExt(module.getProject(), extension, searchScope);
+            for (VirtualFile file : pyFiles) {
               if (file.getName().startsWith("test")) {
-                if (testRunner.isEmpty()) testRunner = checkImports(file, module);   //find test runner import
+                if (testRunner.isEmpty()) {
+                  testRunner = checkImports(file, module); //find test runner import
+                  if (!testRunner.isEmpty()) {
+                    LOG.debug("Test runner '" + testRunner + "' was detected from imports in the file '" + file.getPath() + "'");
+                  }
+                }
               }
-              else {
-                // FIXME: Find a better way to run this updater only once when new project from existing sources is created
-                if (PyDocumentationSettings.getInstance(module).getFormat() == null) {
-                  checkDocstring(file, module);    // detect docstring type
+              else if (docFormat == DocStringFormat.PLAIN) {
+                docFormat = checkDocstring(file, module);    // detect docstring type
+                if (docFormat != DocStringFormat.PLAIN) {
+                  LOG.debug("Docstring format '" + docFormat + "' was detected from content of the file '" + file.getPath() + "'");
                 }
               }
-              if (!testRunner.isEmpty() && PyDocumentationSettings.getInstance(module).getFormat() != null) {
+              
+              if (!testRunner.isEmpty() && docFormat != DocStringFormat.PLAIN) {
                 break;
               }
             }
+            
+            // Check test runners available in the module SDK
             if (testRunner.isEmpty()) {
               //check if installed in sdk
               final Sdk sdk = PythonSdkType.findPythonSdk(module);
-              if (sdk != null && sdk.getSdkType() instanceof PythonSdkType && testRunner.isEmpty()) {
+              if (sdk != null && sdk.getSdkType() instanceof PythonSdkType) {
                 final Boolean nose = VFSTestFrameworkListener.isTestFrameworkInstalled(sdk, PyNames.NOSE_TEST);
                 final Boolean pytest = VFSTestFrameworkListener.isTestFrameworkInstalled(sdk, PyNames.PY_TEST);
                 final Boolean attest = VFSTestFrameworkListener.isTestFrameworkInstalled(sdk, PyNames.AT_TEST);
@@ -118,15 +132,27 @@ public class PyTestRunnerUpdater implements StartupActivity {
                   testRunner = PythonTestConfigurationsModel.PY_TEST_NAME;
                 else if (attest != null && attest)
                   testRunner = PythonTestConfigurationsModel.PYTHONS_ATTEST_NAME;
+                if (!testRunner.isEmpty()) {
+                  LOG.debug("Test runner '" + testRunner + "' was detected from SDK " + sdk);
+                }
               }
             }
 
-            if (StringUtil.isEmptyOrSpaces(testRunner))
-              testRunner = PythonTestConfigurationsModel.PYTHONS_UNITTEST_NAME;
+            final TestRunnerService runnerService = TestRunnerService.getInstance(module);
+            if (runnerService != null) {
+              if (testRunner.isEmpty()) {
+                runnerService.setProjectConfiguration(PythonTestConfigurationsModel.PYTHONS_UNITTEST_NAME);
+              }
+              else {
+                runnerService.setProjectConfiguration(testRunner);
+                LOG.info("Test runner '" + testRunner + "' was detected by project configurator");
+              }
+            }
 
-            runnerService.setProjectConfiguration(testRunner);
-            if (PyDocumentationSettings.getInstance(module).getFormat() == null) {
-              PyDocumentationSettings.getInstance(module).setFormat(DocStringFormat.PLAIN);
+            // Documentation settings should have meaningful default already
+            if (docFormat != DocStringFormat.PLAIN) {
+              docSettings.setFormat(docFormat);
+              LOG.info("Docstring format '" + docFormat + "' was detected by project configurator");
             }
           }
         }, ModalityState.any(), module.getDisposed());
@@ -134,15 +160,12 @@ public class PyTestRunnerUpdater implements StartupActivity {
     });
   }
 
-  private static String detectTestRunnerFromSetupPy(Module module) {
-    String testRunner = "";
-    if (!testRunner.isEmpty()) return  testRunner;
+  @NotNull
+  private static String detectTestRunnerFromSetupPy(@NotNull Module module) {
     final PyFile setupPy = PyPackageUtil.findSetupPy(module);
-    if (setupPy == null)
-      return  testRunner;
+    if (setupPy == null) return "";
     final PyCallExpression setupCall = PyPackageUtil.findSetupCall(setupPy);
-    if (setupCall == null)
-      return  testRunner;
+    if (setupCall == null) return "";
     for (PyExpression arg : setupCall.getArguments()) {
       if (arg instanceof PyKeywordArgument) {
         final PyKeywordArgument kwarg = (PyKeywordArgument)arg;
@@ -151,59 +174,48 @@ public class PyTestRunnerUpdater implements StartupActivity {
           if (value instanceof PyStringLiteralExpression) {
             final String stringValue = ((PyStringLiteralExpression)value).getStringValue();
             if (stringValue.contains(PyNames.NOSE_TEST)) {
-              testRunner = PythonTestConfigurationsModel.PYTHONS_NOSETEST_NAME;
-              break;
+              return PythonTestConfigurationsModel.PYTHONS_NOSETEST_NAME;
             }
             if (stringValue.contains(PyNames.PY_TEST)) {
-              testRunner = PythonTestConfigurationsModel.PY_TEST_NAME;
-              break;
+              return PythonTestConfigurationsModel.PY_TEST_NAME;
             }
             if (stringValue.contains(PyNames.AT_TEST_IMPORT)) {
-              testRunner = PythonTestConfigurationsModel.PYTHONS_ATTEST_NAME;
-              break;
+              return PythonTestConfigurationsModel.PYTHONS_ATTEST_NAME;
             }
           }
         }
       }
     }
-    return testRunner;
+    return "";
   }
 
-  private static void checkDocstring(VirtualFile file, Module module) {
-    final PyDocumentationSettings documentationSettings = PyDocumentationSettings.getInstance(module);
+  @NotNull
+  private static DocStringFormat checkDocstring(@NotNull VirtualFile file, @NotNull Module module) {
     final PsiFile psiFile = PsiManager.getInstance(module.getProject()).findFile(file);
     if (psiFile instanceof PyFile) {
-      if (documentationSettings.isEpydocFormat(psiFile))
-        documentationSettings.setFormat(DocStringFormat.EPYTEXT);
-      else if (documentationSettings.isReSTFormat(psiFile))
-        documentationSettings.setFormat(DocStringFormat.REST);
-      else {
-        final String fileText = psiFile.getText();
-        if (!fileText.contains(":param ") && !fileText.contains(":type ") && !fileText.contains(":rtype ") &&
-            !fileText.contains("@param ") && !fileText.contains("@type ") && !fileText.contains("@rtype ")) return;
-
-        final PyDocStringOwner[] childrens = PsiTreeUtil.getChildrenOfType(psiFile, PyDocStringOwner.class);
-        if (childrens != null) {
-          for (PyDocStringOwner owner : childrens) {
-            final PyStringLiteralExpression docStringExpression = owner.getDocStringExpression();
-            if (docStringExpression != null) {
-              String text = docStringExpression.getStringValue();
-              if (text.contains(":param ") || text.contains(":rtype") || text.contains(":type")) {
-                documentationSettings.setFormat(DocStringFormat.REST);
-                return;
-              }
-              else if (text.contains("@param ") || text.contains("@rtype") || text.contains("@type")) {
-                documentationSettings.setFormat(DocStringFormat.EPYTEXT);
-                return;
-              }
+      final DocStringFormat perFileFormat = PyDocumentationSettings.getFormatFromDocformatAttribute(psiFile);
+      if (perFileFormat != null) {
+        return perFileFormat;
+      }
+      // Why toplevel docstring owners only
+      final PyDocStringOwner[] children = PsiTreeUtil.getChildrenOfType(psiFile, PyDocStringOwner.class);
+      if (children != null) {
+        for (PyDocStringOwner owner : children) {
+          final PyStringLiteralExpression docStringExpression = owner.getDocStringExpression();
+          if (docStringExpression != null) {
+            final DocStringFormat guessed = DocStringUtil.guessDocStringFormat(docStringExpression.getStringValue());
+            if (guessed != DocStringFormat.PLAIN) {
+              return guessed;
             }
           }
         }
       }
     }
+    return DocStringFormat.PLAIN;
   }
 
-  private static String checkImports(VirtualFile file, Module module) {
+  @NotNull
+  private static String checkImports(@NotNull VirtualFile file, @NotNull Module module) {
     final PsiFile psiFile = PsiManager.getInstance(module.getProject()).findFile(file);
     if (psiFile instanceof PyFile) {
       final List<PyImportElement> importTargets = ((PyFile)psiFile).getImportTargets();