3c84d6ce9225c5b92cf012e3f10ea4f3bae48417
[idea/community.git] / python / testSrc / com / jetbrains / env / PyToxTest.java
1 /*
2  * Copyright 2000-2015 JetBrains s.r.o.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.jetbrains.env;
17
18 import com.google.common.collect.Sets;
19 import com.intellij.execution.testframework.sm.runner.SMTestProxy;
20 import com.intellij.execution.testframework.sm.runner.ui.MockPrinter;
21 import com.intellij.openapi.diagnostic.Logger;
22 import com.intellij.openapi.util.Pair;
23 import com.jetbrains.python.sdkTools.SdkCreationType;
24 import com.jetbrains.python.testing.tox.PyToxConfiguration;
25 import com.jetbrains.python.testing.tox.PyToxConfigurationFactory;
26 import org.hamcrest.Matchers;
27 import org.jetbrains.annotations.NotNull;
28 import org.jetbrains.annotations.Nullable;
29 import org.junit.Assert;
30 import org.junit.Test;
31
32 import java.io.IOException;
33 import java.util.*;
34
35 /**
36  * Ensure tox runner works
37  *
38  * @author Ilya.Kazakevich
39  */
40 public final class PyToxTest extends PyEnvTestCase {
41   public PyToxTest() {
42     super("tox");
43   }
44
45   /**
46    * Simply ensure tox runner works
47    */
48   @Test
49   public void testToxSimpleRun() throws IOException {
50     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxSimpleRun/", 2,
51                                                      new MyTestProcessRunner(),
52                                                      Arrays.asList(
53                                                        // Should fail, no skip in 26
54                                                        Pair.create("py26", new InterpreterExpectations(
55                                                          "AttributeError: 'module' object has no attribute 'skip'", false)),
56                                                        Pair.create("py27", new InterpreterExpectations("", true))
57                                                      )
58     ));
59   }
60
61   /**
62    * Check tox nose runner
63    */
64   @Test
65   public void testToxNose() throws IOException {
66     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxNose/", 1,
67                                                      new MyTestProcessRunner(),
68                                                      Arrays.asList(
69                                                        Pair.create("py26", new InterpreterExpectations("", true)),
70                                                        Pair.create("py27", new InterpreterExpectations("", true)),
71                                                        // Does not support 3.4
72                                                        Pair.create("py32", new InterpreterExpectations("SyntaxError", false)),
73                                                        Pair.create("py34", new InterpreterExpectations("SyntaxError", false))
74                                                      )
75                   )
76     );
77   }
78
79   /**
80    * Check tox pytest runner
81    */
82   @Test
83   public void testToxPyTest() throws IOException {
84     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxPyTest/", 1,
85                                                      new MyTestProcessRunner(),
86                                                      Arrays.asList(
87                                                        Pair.create("py26", new InterpreterExpectations("", true)),
88                                                        Pair.create("py27", new InterpreterExpectations("", true)),
89                                                        // Does not support 3.4
90                                                        Pair.create("py32", new InterpreterExpectations("SyntaxError", false)),
91                                                        Pair.create("py34", new InterpreterExpectations("SyntaxError", false))
92                                                      )
93                   )
94     );
95   }
96
97   /**
98    * Check tox unit runner
99    */
100   @Test
101   public void testToxUnitTest() throws IOException {
102     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxUnitTest/", 1,
103                                                      new MyTestProcessRunner(),
104                                                      Arrays.asList(
105                                                        Pair.create("py26", new InterpreterExpectations("", true)),
106                                                        Pair.create("py27", new InterpreterExpectations("", true)),
107                                                        // Does not support 3.4
108                                                        Pair.create("py32", new InterpreterExpectations("SyntaxError", false)),
109                                                        Pair.create("py34", new InterpreterExpectations("SyntaxError", false))
110                                                      )
111                   )
112     );
113   }
114
115   /**
116    * Big test which should run on any interpreter and check its output
117    */
118   @Test
119   public void testToxSuccessTest() throws IOException {
120     runPythonTest(new MyPyProcessWithConsoleTestTask("/toxtest/toxSuccess/", 1,
121                                                      new MyTestProcessRunner(),
122                                                      Arrays.asList(
123                                                        Pair.create("py26", new InterpreterExpectations("I am 2.6", true)),
124                                                        Pair.create("py27", new InterpreterExpectations("I am 2.7", true)),
125                                                        // Should have output
126                                                        Pair.create("py32", new InterpreterExpectations("I am 3.2", true)),
127                                                        Pair.create("py34", new InterpreterExpectations("I am 3.4", true))
128                                                      )
129                   )
130     );
131   }
132
133
134   private static final class MyPyProcessWithConsoleTestTask extends PyProcessWithConsoleTestTask<MyTestProcessRunner> {
135     @NotNull
136     private final Map<String, InterpreterExpectations> myInterpreters = new HashMap<>();
137     private final int myMinimumSuccessTestCount;
138     @NotNull
139     private final MyTestProcessRunner myRunner;
140
141     /**
142      * @param relativeTestDataPath    TODO: DOC
143      * @param minimumSuccessTestCount how many success tests should be
144      * @param interpreterExpectations interpreter_name -] expected result
145      */
146     private MyPyProcessWithConsoleTestTask(@Nullable final String relativeTestDataPath,
147                                            final int minimumSuccessTestCount,
148                                            @NotNull final MyTestProcessRunner runner,
149                                            @NotNull final Collection<Pair<String, InterpreterExpectations>> interpreterExpectations) {
150       super(relativeTestDataPath, SdkCreationType.EMPTY_SDK);
151       myMinimumSuccessTestCount = minimumSuccessTestCount;
152       myRunner = runner;
153       for (final Pair<String, InterpreterExpectations> interpreterExpectation : interpreterExpectations) {
154         myInterpreters.put(interpreterExpectation.first, interpreterExpectation.second);
155       }
156     }
157
158     @Override
159     protected void checkTestResults(@NotNull final MyTestProcessRunner runner,
160                                     @NotNull final String stdout,
161                                     @NotNull final String stderr,
162                                     @NotNull final String all) {
163
164       // Interpreters are used in tox.ini, so there should be such text
165       for (final String interpreterName : myInterpreters.keySet()) {
166         Assert.assertThat(String.format("No %s used from tox.ini", interpreterName), all, Matchers.containsString(interpreterName));
167       }
168
169
170       if (!stderr.isEmpty()) {
171         Logger.getInstance(PyToxTest.class).warn(stderr);
172       }
173
174
175       final Set<String> checkedInterpreters = new HashSet<>();
176       final Set<String> skippedInterpreters = new HashSet<>();
177       // Interpreter should either run tests or mentioned as NotFound
178       for (final SMTestProxy interpreterSuite : runner.getTestProxy().getChildren()) {
179         final String interpreterName = interpreterSuite.getName();
180         checkedInterpreters.add(interpreterName);
181
182         if (interpreterSuite.getChildren().size() == 1 && interpreterSuite.getChildren().get(0).getName().endsWith("ERROR")) {
183           // Interpreter failed to run
184           final String testOutput = getTestOutput(interpreterSuite.getChildren().get(0));
185           if (testOutput.contains("InterpreterNotFound")) {
186             Logger.getInstance(PyToxTest.class).warn(String.format("Interpreter %s does not exit", interpreterName));
187             skippedInterpreters.add(interpreterName); // Interpreter does not exit
188             continue;
189           }
190           // Some other error?
191           final InterpreterExpectations expectations = myInterpreters.get(interpreterName);
192           Assert
193             .assertFalse(String.format("Interpreter %s should not fail, but failed: %s", interpreterName, getTestOutput(interpreterSuite)),
194                          expectations.myExpectedSuccess);
195           continue;
196         }
197
198         // Interpretr run success,
199         //At least one interpreter tests should passed
200         Assert.assertThat(String.format("No test passed, should %s at least", myMinimumSuccessTestCount),
201                           new SMRootTestsCounter(interpreterSuite.getRoot()).getPassedTestsCount(),
202                           Matchers.greaterThanOrEqualTo(myMinimumSuccessTestCount));
203
204         // Check expected output
205         final String message = String.format("Interpreter %s does not have expected string in output. \n ", interpreterName) +
206                                String.format("All: %s \n", all) +
207                                String.format("Error: %s \n", stderr);
208
209
210         Assert
211           .assertThat(message,
212                       getTestOutput(interpreterSuite), Matchers.containsString(myInterpreters.get(interpreterName).myExpectedOutput));
213       }
214
215       Assert.assertThat("No all interpreters from tox.ini used", checkedInterpreters, Matchers.equalTo(myInterpreters.keySet()));
216       assert !skippedInterpreters.equals(myInterpreters.keySet()) : "All interpreters skipped (they do not exist on platform), " +
217                                                                     "we test nothing";
218     }
219
220     @NotNull
221     private static String getTestOutput(@NotNull final SMTestProxy test) {
222       final MockPrinter p = new MockPrinter();
223       test.printOn(p);
224       return p.getAllOut();
225     }
226
227     @NotNull
228     @Override
229     public Set<String> getTags() {
230       return Sets.newHashSet("tox");
231     }
232
233     @NotNull
234     @Override
235     protected MyTestProcessRunner createProcessRunner() throws Exception {
236       return myRunner;
237     }
238   }
239
240   private static final class MyTestProcessRunner extends PyAbstractTestProcessRunner<PyToxConfiguration> {
241     /**
242      * @param testPath testPath relative to community path
243      */
244     private MyTestProcessRunner() {
245       super(PyToxConfigurationFactory.INSTANCE, PyToxConfiguration.class, 0);
246     }
247   }
248
249   private static final class InterpreterExpectations {
250     @NotNull
251     private final String myExpectedOutput;
252     private final boolean myExpectedSuccess;
253
254     /**
255      * @param expectedOutput  expected test output
256      * @param expectedSuccess if test should be success
257      */
258     private InterpreterExpectations(@NotNull final String expectedOutput, final boolean expectedSuccess) {
259       myExpectedOutput = expectedOutput;
260       myExpectedSuccess = expectedSuccess;
261     }
262   }
263 }