PY-19015: Check env has commands, skip fixers if has no
[idea/community.git] / python / testSrc / com / jetbrains / env / PyToxTest.java
1 /*
2  * Copyright 2000-2015 JetBrains s.r.o.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.jetbrains.env;
17
18 import com.google.common.collect.Sets;
19 import com.intellij.execution.testframework.sm.runner.SMTestProxy;
20 import com.intellij.execution.testframework.sm.runner.ui.MockPrinter;
21 import com.intellij.openapi.diagnostic.Logger;
22 import com.intellij.openapi.util.Pair;
23 import com.jetbrains.python.sdkTools.SdkCreationType;
24 import com.jetbrains.python.testing.tox.PyToxConfiguration;
25 import com.jetbrains.python.testing.tox.PyToxConfigurationFactory;
26 import org.hamcrest.Matchers;
27 import org.jetbrains.annotations.NotNull;
28 import org.jetbrains.annotations.Nullable;
29 import org.junit.Assert;
30 import org.junit.Test;
31
32 import java.io.IOException;
33 import java.util.*;
34
35 /**
36  * Ensure tox runner works
37  *
38  * @author Ilya.Kazakevich
39  */
40 public final class PyToxTest extends PyEnvTestCase {
41   public PyToxTest() {
42     super("tox");
43   }
44
45   /**
46    * Simply ensure tox runner works
47    */
48   @Test
49   public void testToxSimpleRun() throws IOException {
50     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxSimpleRun/", 2,
51                                                      new MyTestProcessRunner(),
52                                                      Arrays.asList(
53                                                        // Should fail, no skip in 26
54                                                        Pair.create("py26", new InterpreterExpectations(
55                                                          "AttributeError: 'module' object has no attribute 'skip'", false)),
56                                                        Pair.create("py27", new InterpreterExpectations("", true))
57                                                      )
58     ));
59   }
60
61   /**
62    * Check tox nose runner
63    */
64   @Test
65   public void testToxNose() throws IOException {
66     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxNose/", 1,
67                                                      new MyTestProcessRunner(),
68                                                      Arrays.asList(
69                                                        Pair.create("py26", new InterpreterExpectations("", true)),
70                                                        Pair.create("py27", new InterpreterExpectations("", true)),
71                                                        // Does not support 3.4
72                                                        Pair.create("py32", new InterpreterExpectations("SyntaxError", false)),
73                                                        Pair.create("py34", new InterpreterExpectations("SyntaxError", false))
74                                                      )
75                   )
76     );
77   }
78
79   /**
80    * Check tox pytest runner
81    */
82   @Test
83   public void testToxPyTest() throws IOException {
84     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxPyTest/", 1,
85                                                      new MyTestProcessRunner(),
86                                                      Arrays.asList(
87                                                        Pair.create("py26", new InterpreterExpectations("", true)),
88                                                        Pair.create("py27", new InterpreterExpectations("", true)),
89                                                        // Does not support 3.4
90                                                        Pair.create("py32", new InterpreterExpectations("SyntaxError", false)),
91                                                        Pair.create("py34", new InterpreterExpectations("SyntaxError", false))
92                                                      )
93                   )
94     );
95   }
96
97   /**
98    * Check tox unit runner
99    */
100   @Test
101   public void testToxUnitTest() throws IOException {
102     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxUnitTest/", 1,
103                                                      new MyTestProcessRunner(),
104                                                      Arrays.asList(
105                                                        Pair.create("py26", new InterpreterExpectations("", true)),
106                                                        Pair.create("py27", new InterpreterExpectations("", true)),
107                                                        // Does not support 3.4
108                                                        Pair.create("py32", new InterpreterExpectations("SyntaxError", false)),
109                                                        Pair.create("py34", new InterpreterExpectations("SyntaxError", false))
110                                                      )
111                   )
112     );
113   }
114
115   /**
116    * Checks empty envs for all but 2.7
117    */
118   @Test
119   public void textToxOneInterpreter() throws Exception {
120     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxOneInterpreter/", 0,
121                                                      new MyTestProcessRunner(),
122                                                      Arrays.asList(
123                                                        Pair.create("py26", new InterpreterExpectations("", true)),
124                                                        Pair.create("py27", new InterpreterExpectations("ython 2.7", true)),
125                                                        Pair.create("py32", new InterpreterExpectations("", true)),
126                                                        Pair.create("py34", new InterpreterExpectations("", true))
127                                                      )
128                   )
129     );
130
131
132   }
133
134   /**
135    * Big test which should run on any interpreter and check its output
136    */
137   @Test
138   public void testToxSuccessTest() throws IOException {
139     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxSuccess/", 1,
140                                                      new MyTestProcessRunner(),
141                                                      Arrays.asList(
142                                                        Pair.create("py26", new InterpreterExpectations("I am 2.6", true)),
143                                                        Pair.create("py27", new InterpreterExpectations("I am 2.7", true)),
144                                                        // Should have output
145                                                        Pair.create("py32", new InterpreterExpectations("I am 3.2", true)),
146                                                        Pair.create("py34", new InterpreterExpectations("I am 3.4", true))
147                                                      )
148                   )
149     );
150   }
151
152
153   private static final class MyPyProcessWithConsoleTestTask extends PyProcessWithConsoleTestTask<MyTestProcessRunner> {
154     @NotNull
155     private final Map<String, InterpreterExpectations> myInterpreters = new HashMap<>();
156     private final int myMinimumSuccessTestCount;
157     @NotNull
158     private final MyTestProcessRunner myRunner;
159
160     /**
161      * @param relativeTestDataPath    TODO: DOC
162      * @param minimumSuccessTestCount how many success tests should be
163      * @param interpreterExpectations interpreter_name -] expected result
164      */
165     private MyPyProcessWithConsoleTestTask(@Nullable final String relativeTestDataPath,
166                                            final int minimumSuccessTestCount,
167                                            @NotNull final MyTestProcessRunner runner,
168                                            @NotNull final Collection<Pair<String, InterpreterExpectations>> interpreterExpectations) {
169       super(relativeTestDataPath, SdkCreationType.EMPTY_SDK);
170       myMinimumSuccessTestCount = minimumSuccessTestCount;
171       myRunner = runner;
172       for (final Pair<String, InterpreterExpectations> interpreterExpectation : interpreterExpectations) {
173         myInterpreters.put(interpreterExpectation.first, interpreterExpectation.second);
174       }
175     }
176
177     @Override
178     protected void checkTestResults(@NotNull final MyTestProcessRunner runner,
179                                     @NotNull final String stdout,
180                                     @NotNull final String stderr,
181                                     @NotNull final String all) {
182
183       // Interpreters are used in tox.ini, so there should be such text
184       for (final String interpreterName : myInterpreters.keySet()) {
185         Assert.assertThat(String.format("No %s used from tox.ini", interpreterName), all, Matchers.containsString(interpreterName));
186       }
187
188
189       if (!stderr.isEmpty()) {
190         Logger.getInstance(PyToxTest.class).warn(stderr);
191       }
192
193
194       final Set<String> checkedInterpreters = new HashSet<>();
195       final Set<String> skippedInterpreters = new HashSet<>();
196       // Interpreter should either run tests or mentioned as NotFound
197       for (final SMTestProxy interpreterSuite : runner.getTestProxy().getChildren()) {
198         final String interpreterName = interpreterSuite.getName();
199         checkedInterpreters.add(interpreterName);
200
201         if (interpreterSuite.getChildren().size() == 1 && interpreterSuite.getChildren().get(0).getName().endsWith("ERROR")) {
202           // Interpreter failed to run
203           final String testOutput = getTestOutput(interpreterSuite.getChildren().get(0));
204           if (testOutput.contains("InterpreterNotFound")) {
205             // Skipped with out of "skip_missing_interpreters = True"
206             Logger.getInstance(PyToxTest.class).warn(String.format("Interpreter %s does not exit", interpreterName));
207             skippedInterpreters.add(interpreterName); // Interpreter does not exit
208             continue;
209           }
210           // Some other error?
211           final InterpreterExpectations expectations = myInterpreters.get(interpreterName);
212           Assert
213             .assertFalse(String.format("Interpreter %s should not fail, but failed: %s", interpreterName, getTestOutput(interpreterSuite)),
214                          expectations.myExpectedSuccess);
215           continue;
216         }
217
218         if (interpreterSuite.getChildren().size() == 1 && interpreterSuite.getChildren().get(0).getName().endsWith("SKIP")) {
219           // The only reason it may be skipped is it does not exist and skip_missing_interpreters = True
220           final String output = getTestOutput(interpreterSuite);
221           Assert.assertThat("Test marked skipped but not because interpreter not found", output, Matchers.containsString("InterpreterNotFound"));
222         }
223
224
225         // Interpretr run success,
226         //At least one interpreter tests should passed
227         Assert.assertThat(String.format("No test passed, should %s at least", myMinimumSuccessTestCount),
228                           new SMRootTestsCounter(interpreterSuite.getRoot()).getPassedTestsCount(),
229                           Matchers.greaterThanOrEqualTo(myMinimumSuccessTestCount));
230
231         // Check expected output
232         final String message = String.format("Interpreter %s does not have expected string in output. \n ", interpreterName) +
233                                String.format("All: %s \n", all) +
234                                String.format("Error: %s \n", stderr);
235
236
237         Assert
238           .assertThat(message,
239                       getTestOutput(interpreterSuite), Matchers.containsString(myInterpreters.get(interpreterName).myExpectedOutput));
240       }
241
242       Assert.assertThat("No all interpreters from tox.ini used", checkedInterpreters, Matchers.equalTo(myInterpreters.keySet()));
243       assert !skippedInterpreters.equals(myInterpreters.keySet()) : "All interpreters skipped (they do not exist on platform), " +
244                                                                     "we test nothing";
245     }
246
247     @NotNull
248     private static String getTestOutput(@NotNull final SMTestProxy test) {
249       final MockPrinter p = new MockPrinter();
250       test.printOn(p);
251       return p.getAllOut();
252     }
253
254     @NotNull
255     @Override
256     public Set<String> getTags() {
257       return Sets.newHashSet("tox");
258     }
259
260     @NotNull
261     @Override
262     protected MyTestProcessRunner createProcessRunner() throws Exception {
263       return myRunner;
264     }
265   }
266
267   private static final class MyTestProcessRunner extends PyAbstractTestProcessRunner<PyToxConfiguration> {
268     /**
269      * @param testPath testPath relative to community path
270      */
271     private MyTestProcessRunner() {
272       super(PyToxConfigurationFactory.INSTANCE, PyToxConfiguration.class, 0);
273     }
274   }
275
276   private static final class InterpreterExpectations {
277     @NotNull
278     private final String myExpectedOutput;
279     private final boolean myExpectedSuccess;
280
281     /**
282      * @param expectedOutput  expected test output
283      * @param expectedSuccess if test should be success
284      */
285     private InterpreterExpectations(@NotNull final String expectedOutput, final boolean expectedSuccess) {
286       myExpectedOutput = expectedOutput;
287       myExpectedSuccess = expectedSuccess;
288     }
289   }
290 }