Overall more reliable docstring format detection

I added several convenient methods for detection docstring format
in DocStringUtil. One of them - parse(String, PsiElement?) fallbacks to
docstring format specified in settings if it was unable to detect
docstring format solely from text. I replaced existing usages of
#parse(String) with this method. For PLAIN format it also returns
special PlainDocString type that can only extract summary and
description, e.g. for documentation popups.

diff --git a/python/src/com/jetbrains/python/documentation/DocStringReferenceProvider.java b/python/src/com/jetbrains/python/documentation/DocStringReferenceProvider.java
index dcc56b3144a82830bd22d271eb9ebf7f18c14e64..958d43da5f613b88a6059b731e8b50c0c800bd96 100644 (file)
--- a/python/src/com/jetbrains/python/documentation/DocStringReferenceProvider.java
+++ b/python/src/com/jetbrains/python/documentation/DocStringReferenceProvider.java
@@ -59,7 +59,7 @@ public class DocStringReferenceProvider extends PsiReferenceProvider {
         final List<PsiReference> result = new ArrayList<PsiReference>();
         final int offset = ranges.get(0).getStartOffset();
         // XXX: It does not work with multielement docstrings
-        StructuredDocString docString = DocStringUtil.parse(text);
+        StructuredDocString docString = DocStringUtil.parse(text, element);
         if (docString instanceof TagBasedDocString) {
           final TagBasedDocString taggedDocString = (TagBasedDocString)docString;
           result.addAll(referencesFromNames(expr, offset, docString,

diff --git a/python/src/com/jetbrains/python/documentation/DocStringUtil.java b/python/src/com/jetbrains/python/documentation/DocStringUtil.java
index c7879624b5d927c9316bcc6308875e57aa8b9f90..6fa1a3c415cb429c64a35f23d827fabda7f162fd 100644 (file)
--- a/python/src/com/jetbrains/python/documentation/DocStringUtil.java
+++ b/python/src/com/jetbrains/python/documentation/DocStringUtil.java
@@ -51,24 +51,31 @@ public class DocStringUtil {
     return PyPsiUtils.strValue(owner.getDocStringExpression());
   }
 
+  /**
+   * Attempts to detect docstring format from given text and parses it into corresponding structured docstring.
+   * It's recommended to use more reliable {@link #parse(String, PsiElement)} that fallbacks to format specified in settings.
+   *
+   * @return structured docstring for one of supported formats or instance of {@link PlainDocString} if none was recognized.
+   * @see #parse(String, PsiElement) 
+   */
   @Nullable
-  public static StructuredDocString parse(@Nullable String text) {
-    if (text == null) {
-      return null;
-    }
-    if (isSphinxDocString(text)) {
-      return parseDocStringContent(DocStringFormat.REST, text);
-    }
-    if (isGoogleDocString(text)) {
-      return parseDocStringContent(DocStringFormat.GOOGLE, text);
-    }
-    if (isNumpyDocstring(text)) {
-      return parseDocStringContent(DocStringFormat.NUMPY, text);
-    }
-    return parseDocStringContent(DocStringFormat.EPYTEXT, text);
+  public static StructuredDocString parse(@NotNull String text) {
+    return parse(text, null);
   }
 
-
+  /**
+   * Attempts to detects docstring format first from given text, next from settings and parses text into corresponding structured docstring.
+   *
+   * @return structured docstring for one of supported formats or instance of {@link PlainDocString} if none was recognized.
+   * @see DocStringFormat#ALL_NAMES_BUT_PLAIN
+   * @see #guessDocStringFormat(String, PsiElement)
+   */
+  @NotNull
+  public static StructuredDocString parse(@NotNull String text, @Nullable PsiElement anchor) {
+    final DocStringFormat format = guessDocStringFormat(text, anchor);
+    return parseDocStringContent(format, text);
+  }
+  
   @NotNull
   public static StructuredDocString parseDocString(@NotNull DocStringFormat format,
                                                    @NotNull PyStringLiteralExpression literalExpression) {
@@ -104,7 +111,7 @@ public class DocStringUtil {
       case NUMPY:
         return new NumpyDocString(content);
       default:
-        throw new UnsupportedOperationException("Not supported for plain docstrings. Use PyDocStringUtil#ensureNotPlainDocstringFormat");
+        return new PlainDocString(content);
     }
   }
 
@@ -113,16 +120,58 @@ public class DocStringUtil {
     final TextRange contentRange = PyStringLiteralExpressionImpl.getNodeTextRange(text);
     return new Substring(text, contentRange.getStartOffset(), contentRange.getEndOffset());
   }
+  
+  /**
+   * @return docstring format inferred heuristically solely from its content. For more reliable result use anchored version 
+   * {@link #guessDocStringFormat(String, PsiElement)} of this method.
+   * @see #guessDocStringFormat(String, PsiElement) 
+   */
+  @NotNull
+  public static DocStringFormat guessDocStringFormat(@NotNull String text) {
+    if (isLikeNumpyDocstring(text)) {
+      return DocStringFormat.NUMPY;
+    }
+    if (isLikeGoogleDocString(text)) {
+      return DocStringFormat.GOOGLE;
+    }
+    if (isLikeEpydocDocString(text)) {
+      return DocStringFormat.EPYTEXT;
+    }
+    if (isLikeSphinxDocString(text)) {
+      return DocStringFormat.REST;
+    }
+    return DocStringFormat.PLAIN;
+  }
+
+  /**
+   * @return docstring inferred heuristically and if unsuccessful fallback to configured format retrieved from anchor PSI element 
+   * @see #getConfiguredDocStringFormat(PsiElement) 
+   */
+  @NotNull
+  public static DocStringFormat guessDocStringFormat(@NotNull String text, @Nullable PsiElement anchor) {
+    final DocStringFormat guessed = guessDocStringFormat(text);
+    return guessed == DocStringFormat.PLAIN && anchor != null ? getConfiguredDocStringFormat(anchor) : guessed;
+  }
 
-  public static boolean isSphinxDocString(@NotNull String text) {
+  /**
+   * @return docstring format configured for file or module containing given anchor PSI element
+   * @see PyDocumentationSettings#getFormatForFile(PsiFile)
+   */
+  @NotNull
+  public static DocStringFormat getConfiguredDocStringFormat(@NotNull PsiElement anchor) {
+    final PyDocumentationSettings settings = PyDocumentationSettings.getInstance(getModuleForElement(anchor));
+    return settings.getFormatForFile(anchor.getContainingFile());
+  }
+
+  public static boolean isLikeSphinxDocString(@NotNull String text) {
     return text.contains(":param ") || text.contains(":rtype") || text.contains(":type");
   }
 
-  public static boolean isEpydocDocString(@NotNull String text) {
+  public static boolean isLikeEpydocDocString(@NotNull String text) {
     return text.contains("@param ") || text.contains("@rtype") || text.contains("@type");
   }
 
-  public static boolean isGoogleDocString(@NotNull String text) {
+  public static boolean isLikeGoogleDocString(@NotNull String text) {
     for (@NonNls String title : StringUtil.findMatches(text, GoogleCodeStyleDocString.SECTION_HEADER, 1)) {
       if (SectionBasedDocString.SECTION_NAMES.contains(title.toLowerCase())) {
         return true;
@@ -131,7 +180,7 @@ public class DocStringUtil {
     return false;
   }
 
-  public static boolean isNumpyDocstring(@NotNull String text) {
+  public static boolean isLikeNumpyDocstring(@NotNull String text) {
     final String[] lines = StringUtil.splitByLines(text, false);
     for (int i = 0; i < lines.length; i++) {
       final String line = lines[i];
@@ -161,8 +210,10 @@ public class DocStringUtil {
     return null;
   }
 
-  public static StructuredDocString getStructuredDocString(PyDocStringOwner owner) {
-    return parse(owner.getDocStringValue());
+  @Nullable
+  public static StructuredDocString getStructuredDocString(@NotNull PyDocStringOwner owner) {
+    final String value = owner.getDocStringValue();
+    return value == null ? null : parse(value, owner);
   }
 
   public static boolean isDocStringExpression(@Nullable PyExpression expression) {
@@ -255,10 +306,4 @@ public class DocStringUtil {
     }
     return true;
   }
-
-  @NotNull
-  public static DocStringFormat getDocStringFormat(@NotNull PsiElement anchor) {
-    final PyDocumentationSettings settings = PyDocumentationSettings.getInstance(getModuleForElement(anchor));
-    return settings.getFormatForFile(anchor.getContainingFile());
-  }
 }

diff --git a/python/src/com/jetbrains/python/documentation/PlainDocString.java b/python/src/com/jetbrains/python/documentation/PlainDocString.java
new file mode 100644 (file)
index 0000000..c0b85ba
--- /dev/null
+++ b/python/src/com/jetbrains/python/documentation/PlainDocString.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2000-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.jetbrains.python.documentation;
+
+import com.jetbrains.python.psi.PyIndentUtil;
+import com.jetbrains.python.psi.StructuredDocString;
+import com.jetbrains.python.toolbox.Substring;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.List;
+
+/**
+ * Stub docstring that is capable only of extracting summary and description
+ * @author Mikhail Golubev
+ */
+public class PlainDocString extends DocStringLineParser implements StructuredDocString {
+  private final String mySummary;
+  private final String myDescription;
+
+  public PlainDocString(@NotNull Substring content) {
+    super(content);
+    if (!isEmpty(0) && isEmptyOrDoesNotExist(1)) {
+      mySummary = getLine(0).trim().toString();
+      final int next = skipEmptyLines(1);
+      if (next != 1) {
+        final String remaining = getLine(next).union(getLine(getLineCount() - 1)).toString();
+        myDescription = PyIndentUtil.removeCommonIndent(remaining, false);
+      }
+      else {
+        myDescription = "";
+      }
+    }
+    else {
+      mySummary = "";
+      myDescription = PyIndentUtil.removeCommonIndent(myDocStringContent.toString(), true);
+    }
+  }
+
+  @Override
+  public String getSummary() {
+    return mySummary;
+  }
+
+  @Override
+  public String getDescription() {
+    return myDescription;
+  }
+
+  @Override
+  protected boolean isBlockEnd(int lineNum) {
+    return false;
+  }
+
+  @NotNull
+  @Override
+  public String createParameterType(@NotNull String name, @NotNull String type) {
+    return "";
+  }
+
+  @Override
+  public List<String> getParameters() {
+    return null;
+  }
+
+  @Override
+  public List<Substring> getParameterSubstrings() {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public String getParamType(@Nullable String paramName) {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public Substring getParamTypeSubstring(@Nullable String paramName) {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public String getParamDescription(@Nullable String paramName) {
+    return null;
+  }
+
+  @Override
+  public List<String> getKeywordArguments() {
+    return null;
+  }
+
+  @Override
+  public List<Substring> getKeywordArgumentSubstrings() {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public String getKeywordArgumentDescription(@Nullable String paramName) {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public String getReturnType() {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public Substring getReturnTypeSubstring() {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public String getReturnDescription() {
+    return null;
+  }
+
+  @Override
+  public List<String> getRaisedExceptions() {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public String getRaisedExceptionDescription(@Nullable String exceptionName) {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public String getAttributeDescription() {
+    return null;
+  }
+}

diff --git a/python/src/com/jetbrains/python/documentation/PyDocstringGenerator.java b/python/src/com/jetbrains/python/documentation/PyDocstringGenerator.java
index 55badbdb97c05996deabc20d6c2c3f8567d9419d..1e42a31ba16f829d5eca55195353ba9cef42fba3 100644 (file)
--- a/python/src/com/jetbrains/python/documentation/PyDocstringGenerator.java
+++ b/python/src/com/jetbrains/python/documentation/PyDocstringGenerator.java
@@ -91,7 +91,7 @@ public class PyDocstringGenerator {
       indentation = PyIndentUtil.getElementIndent(((PyStatementListContainer)owner).getStatementList());
     }
     final String docStringText = owner.getDocStringExpression() == null ? null : owner.getDocStringExpression().getText();
-    return new PyDocstringGenerator(owner, docStringText, DocStringUtil.getDocStringFormat(owner), indentation);
+    return new PyDocstringGenerator(owner, docStringText, DocStringUtil.getConfiguredDocStringFormat(owner), indentation);
   }
   
   @NotNull
@@ -102,7 +102,7 @@ public class PyDocstringGenerator {
   @NotNull
   public static PyDocstringGenerator update(@NotNull PyStringLiteralExpression docString) {
     return new PyDocstringGenerator(PsiTreeUtil.getParentOfType(docString, PyDocStringOwner.class),
-                                    docString.getText(), DocStringUtil.getDocStringFormat(docString),
+                                    docString.getText(), DocStringUtil.getConfiguredDocStringFormat(docString),
                                     PyIndentUtil.getElementIndent(docString));
   }
 

diff --git a/python/src/com/jetbrains/python/documentation/PyDocumentationBuilder.java b/python/src/com/jetbrains/python/documentation/PyDocumentationBuilder.java
index e2785bc91e6694f6116eef3dccdd4e8e1f007b7c..f6dfd7118d206a7bc6eb94c1e92dbbfaa6a5107a 100644 (file)
--- a/python/src/com/jetbrains/python/documentation/PyDocumentationBuilder.java
+++ b/python/src/com/jetbrains/python/documentation/PyDocumentationBuilder.java
@@ -442,7 +442,7 @@ class PyDocumentationBuilder {
   }
 
   private static Pair<String, String> getTypeAndDescr(String docString, @NotNull PyNamedParameter followed) {
-    final StructuredDocString structuredDocString = DocStringUtil.parse(docString);
+    final StructuredDocString structuredDocString = docString != null ? DocStringUtil.parse(docString, followed) : null;
     String type = null;
     String desc = null;
     if (structuredDocString != null) {

diff --git a/python/src/com/jetbrains/python/documentation/PyStructuredDocstringFormatter.java b/python/src/com/jetbrains/python/documentation/PyStructuredDocstringFormatter.java
index cb1ddf6e0845b509d5969e286524b4fc33f247cf..ddfc9cd66aac8176d524c77b745b0a6eb812f63e 100644 (file)
--- a/python/src/com/jetbrains/python/documentation/PyStructuredDocstringFormatter.java
+++ b/python/src/com/jetbrains/python/documentation/PyStructuredDocstringFormatter.java
@@ -60,7 +60,6 @@ public class PyStructuredDocstringFormatter {
       module = modules[0];
     }
     if (module == null) return Lists.newArrayList();
-    final PyDocumentationSettings documentationSettings = PyDocumentationSettings.getInstance(module);
     final List<String> result = new ArrayList<String>();
 
     final String[] lines = PyDocumentationBuilder.removeCommonIndentation(docstring);
@@ -68,20 +67,21 @@ public class PyStructuredDocstringFormatter {
 
     final String formatter;
     final StructuredDocString structuredDocString;
-    if (documentationSettings.isGoogleFormat(element.getContainingFile()) || DocStringUtil.isGoogleDocString(preparedDocstring)) {
+    final DocStringFormat format = DocStringUtil.guessDocStringFormat(preparedDocstring, element);
+    if (format == DocStringFormat.GOOGLE) {
       formatter = PythonHelpersLocator.getHelperPath("google_formatter.py");
       structuredDocString = DocStringUtil.parseDocString(DocStringFormat.GOOGLE, preparedDocstring);
     }
-    else if (documentationSettings.isNumpyFormat(element.getContainingFile()) || DocStringUtil.isNumpyDocstring(preparedDocstring)) {
+    else if (format == DocStringFormat.NUMPY) {
       formatter = PythonHelpersLocator.getHelperPath("numpy_formatter.py");
       structuredDocString = DocStringUtil.parseDocString(DocStringFormat.NUMPY, preparedDocstring);
     }
-    else if (documentationSettings.isEpydocFormat(element.getContainingFile()) || DocStringUtil.isEpydocDocString(preparedDocstring)) {
+    else if (format == DocStringFormat.EPYTEXT) {
       formatter = PythonHelpersLocator.getHelperPath("epydoc_formatter.py");
       structuredDocString = DocStringUtil.parseDocString(DocStringFormat.EPYTEXT, preparedDocstring);
       result.add(formatStructuredDocString(structuredDocString));
     }
-    else if (documentationSettings.isReSTFormat(element.getContainingFile()) || DocStringUtil.isSphinxDocString(preparedDocstring)) {
+    else if (format == DocStringFormat.REST) {
       formatter = PythonHelpersLocator.getHelperPath("rest_formatter.py");
       structuredDocString = DocStringUtil.parseDocString(DocStringFormat.REST, preparedDocstring);
     }

diff --git a/python/src/com/jetbrains/python/documentation/PythonDocumentationProvider.java b/python/src/com/jetbrains/python/documentation/PythonDocumentationProvider.java
index 2ab161cd98235d23fd0d589a0968b6982fc9c341..ae6d3a1f862bff74ffce67fbdee68ab1e8855665 100644 (file)
--- a/python/src/com/jetbrains/python/documentation/PythonDocumentationProvider.java
+++ b/python/src/com/jetbrains/python/documentation/PythonDocumentationProvider.java
@@ -93,10 +93,8 @@ public class PythonDocumentationProvider extends AbstractDocumentationProvider i
       String summary = "";
       final PyStringLiteralExpression docStringExpression = func.getDocStringExpression();
       if (docStringExpression != null) {
-        final StructuredDocString docString = DocStringUtil.parse(docStringExpression.getStringValue());
-        if (docString != null) {
-          summary = docString.getSummary();
-        }
+        final StructuredDocString docString = DocStringUtil.parse(docStringExpression.getStringValue(), docStringExpression);
+        summary = docString.getSummary();
       }
       return $(cat.toString()).add(describeDecorators(func, LSame2, ", ", LSame1)).add(describeFunction(func, LSame2, LSame1))
                               .toString() + "\n" + summary;
@@ -112,10 +110,8 @@ public class PythonDocumentationProvider extends AbstractDocumentationProvider i
         }
       }
       if (docStringExpression != null) {
-        final StructuredDocString docString = DocStringUtil.parse(docStringExpression.getStringValue());
-        if (docString != null) {
-          summary = docString.getSummary();
-        }
+        final StructuredDocString docString = DocStringUtil.parse(docStringExpression.getStringValue(), docStringExpression);
+        summary = docString.getSummary();
       }
 
       return describeDecorators(cls, LSame2, ", ", LSame1).add(describeClass(cls, LSame2, false, false)).toString() + "\n" + summary;

diff --git a/python/src/com/jetbrains/python/inspections/PyDocstringInspection.java b/python/src/com/jetbrains/python/inspections/PyDocstringInspection.java
index 655981069d670e65024b168afee8fce094ad888e..fad9046bcc88316c48eb4d58c88d27896f9223b8 100644 (file)
--- a/python/src/com/jetbrains/python/inspections/PyDocstringInspection.java
+++ b/python/src/com/jetbrains/python/inspections/PyDocstringInspection.java
@@ -140,7 +140,7 @@ public class PyDocstringInspection extends PyInspection {
         return false;
       }
 
-      StructuredDocString docString = DocStringUtil.parse(text);
+      StructuredDocString docString = DocStringUtil.parse(text, node);
 
       if (docString == null) {
         return false;

diff --git a/python/src/com/jetbrains/python/inspections/PyDocstringTypesInspection.java b/python/src/com/jetbrains/python/inspections/PyDocstringTypesInspection.java
index cf3bb4034fe0c201835bd67518fd183ca772ad9d..e012ebf08176687e3a32e82b8196eab5f6b167ad 100644 (file)
--- a/python/src/com/jetbrains/python/inspections/PyDocstringTypesInspection.java
+++ b/python/src/com/jetbrains/python/inspections/PyDocstringTypesInspection.java
@@ -24,14 +24,14 @@ import com.jetbrains.python.debugger.PySignature;
 import com.jetbrains.python.debugger.PySignatureCacheManager;
 import com.jetbrains.python.debugger.PySignatureUtil;
 import com.jetbrains.python.documentation.DocStringUtil;
-import com.jetbrains.python.psi.StructuredDocString;
-import com.jetbrains.python.toolbox.Substring;
 import com.jetbrains.python.psi.PyElementGenerator;
 import com.jetbrains.python.psi.PyFunction;
 import com.jetbrains.python.psi.PyStringLiteralExpression;
+import com.jetbrains.python.psi.StructuredDocString;
 import com.jetbrains.python.psi.types.PyType;
 import com.jetbrains.python.psi.types.PyTypeChecker;
 import com.jetbrains.python.psi.types.PyTypeParser;
+import com.jetbrains.python.toolbox.Substring;
 import org.jetbrains.annotations.Nls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -88,7 +88,7 @@ public class PyDocstringTypesInspection extends PyInspection {
         return;
       }
 
-      StructuredDocString docString = DocStringUtil.parse(text);
+      StructuredDocString docString = DocStringUtil.parse(text, function);
       if (docString == null) {
         return;
       }

diff --git a/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java b/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java
index 43c15943f75f93222135d5a5a5c41c05ee3f1ebc..9c901130fdb533fd0a7e88812048f2adfb93a14e 100644 (file)
--- a/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java
+++ b/python/src/com/jetbrains/python/psi/impl/PyTargetExpressionImpl.java
@@ -294,7 +294,7 @@ public class PyTargetExpressionImpl extends PyBaseElementImpl<PyTargetExpression
   public static PyType getTypeFromComment(PyTargetExpressionImpl targetExpression) {
     String docComment = DocStringUtil.getAttributeDocComment(targetExpression);
     if (docComment != null) {
-      StructuredDocString structuredDocString = DocStringUtil.parse(docComment);
+      StructuredDocString structuredDocString = DocStringUtil.parse(docComment, targetExpression);
       if (structuredDocString != null) {
         String typeName = structuredDocString.getParamType(null);
         if (typeName == null) {

diff --git a/python/testSrc/com/jetbrains/python/PySectionBasedDocStringTest.java b/python/testSrc/com/jetbrains/python/PySectionBasedDocStringTest.java
index 0f1c61ada95b30c5ba0497b486ae549226068d2b..0838e0ecab8dc3f7cdfae526de60aeacb2f3a5d7 100644 (file)
--- a/python/testSrc/com/jetbrains/python/PySectionBasedDocStringTest.java
+++ b/python/testSrc/com/jetbrains/python/PySectionBasedDocStringTest.java
@@ -293,7 +293,7 @@ public class PySectionBasedDocStringTest extends PyTestCase {
 
   // PY-16766
   public void testGoogleDocStringContentDetection() {
-    assertTrue(DocStringUtil.isGoogleDocString(
+    assertTrue(DocStringUtil.isLikeGoogleDocString(
       "\n" +
       "    My Section:\n" +
       "        some user defined section\n" +