1 package com.jetbrains.edu.learning.stepic;
3 import com.google.gson.FieldNamingPolicy;
4 import com.google.gson.Gson;
5 import com.google.gson.GsonBuilder;
6 import com.intellij.ide.projectView.ProjectView;
7 import com.intellij.lang.Language;
8 import com.intellij.openapi.application.ApplicationManager;
9 import com.intellij.openapi.diagnostic.Logger;
10 import com.intellij.openapi.editor.Document;
11 import com.intellij.openapi.editor.Editor;
12 import com.intellij.openapi.project.Project;
13 import com.intellij.openapi.projectRoots.Sdk;
14 import com.intellij.openapi.ui.MessageType;
15 import com.intellij.openapi.ui.popup.Balloon;
16 import com.intellij.openapi.ui.popup.JBPopupFactory;
17 import com.intellij.openapi.util.Pair;
18 import com.intellij.openapi.util.io.FileUtil;
19 import com.intellij.openapi.util.text.StringUtil;
20 import com.intellij.openapi.vfs.VfsUtil;
21 import com.intellij.openapi.vfs.VirtualFile;
22 import com.intellij.openapi.vfs.VirtualFileManager;
23 import com.jetbrains.edu.learning.StudyTaskManager;
24 import com.jetbrains.edu.learning.StudyUtils;
25 import com.jetbrains.edu.learning.checker.StudyExecutor;
26 import com.jetbrains.edu.learning.core.EduNames;
27 import com.jetbrains.edu.learning.courseFormat.*;
28 import com.jetbrains.edu.learning.courseGeneration.StudyGenerator;
29 import com.jetbrains.edu.learning.courseGeneration.StudyProjectGenerator;
30 import com.jetbrains.edu.learning.editor.StudyEditor;
31 import com.jetbrains.edu.learning.ui.StudyToolWindow;
32 import org.apache.http.HttpEntity;
33 import org.apache.http.HttpStatus;
34 import org.apache.http.StatusLine;
35 import org.apache.http.client.config.RequestConfig;
36 import org.apache.http.client.methods.CloseableHttpResponse;
37 import org.apache.http.client.methods.HttpGet;
38 import org.apache.http.client.methods.HttpPost;
39 import org.apache.http.client.utils.URIBuilder;
40 import org.apache.http.entity.ContentType;
41 import org.apache.http.entity.StringEntity;
42 import org.apache.http.impl.client.CloseableHttpClient;
43 import org.apache.http.util.EntityUtils;
44 import org.jetbrains.annotations.NotNull;
45 import org.jetbrains.annotations.Nullable;
48 import java.io.IOException;
49 import java.io.UnsupportedEncodingException;
51 import java.net.URISyntaxException;
52 import java.util.HashMap;
53 import java.util.List;
55 import java.util.concurrent.TimeUnit;
57 import static com.jetbrains.edu.learning.stepic.EduStepicConnector.*;
59 public class EduAdaptiveStepicConnector {
60 public static final String PYTHON27 = "python27";
61 public static final String PYTHON3 = "python3";
62 public static final String PYCHARM_COMMENT = "# Posted from PyCharm Edu\n";
63 private static final Logger LOG = Logger.getInstance(EduAdaptiveStepicConnector.class);
64 private static final int CONNECTION_TIMEOUT = 60 * 1000;
67 public static Task getNextRecommendation(@NotNull final Project project, @NotNull Course course) {
69 final CloseableHttpClient client = getHttpClient(project);
70 final URI uri = new URIBuilder(EduStepicNames.STEPIC_API_URL + EduStepicNames.RECOMMENDATIONS_URL)
71 .addParameter(EduNames.COURSE, String.valueOf(course.getId()))
73 final HttpGet request = new HttpGet(uri);
74 setHeaders(request, EduStepicNames.CONTENT_TYPE_APPL_JSON);
77 final CloseableHttpResponse response = client.execute(request);
78 final StatusLine statusLine = response.getStatusLine();
79 final HttpEntity responseEntity = response.getEntity();
80 final String responseString = responseEntity != null ? EntityUtils.toString(responseEntity) : "";
82 if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
83 final Gson gson = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).create();
84 final StepicWrappers.RecommendationWrapper recomWrapper = gson.fromJson(responseString, StepicWrappers.RecommendationWrapper.class);
86 if (recomWrapper.recommendations.length != 0) {
87 final StepicWrappers.Recommendation recommendation = recomWrapper.recommendations[0];
88 final String lessonId = recommendation.lesson;
89 final StepicWrappers.LessonContainer
90 lessonContainer = getFromStepic(EduStepicNames.LESSONS + lessonId, StepicWrappers.LessonContainer.class);
91 if (lessonContainer.lessons.size() == 1) {
92 final Lesson realLesson = lessonContainer.lessons.get(0);
93 course.getLessons().get(0).setId(Integer.parseInt(lessonId));
95 viewAllSteps(client, realLesson.getId());
97 for (int stepId : realLesson.steps) {
98 final StepicWrappers.StepSource step = getStep(stepId);
99 if (step.block.name.equals("code")) {
100 return getTaskFromStep(project, stepId, step.block, realLesson.getName());
104 LOG.warn("Got a lesson without code part as a recommendation");
107 LOG.warn("Got unexpected number of lessons: " + lessonContainer.lessons.size());
112 throw new IOException("Stepic returned non 200 status code: " + responseString);
115 catch (IOException e) {
116 LOG.warn(e.getMessage());
118 final String connectionMessages = "Connection problems, Please, try again";
119 final Balloon balloon =
120 JBPopupFactory.getInstance().createHtmlTextBalloonBuilder(connectionMessages, MessageType.ERROR, null)
122 ApplicationManager.getApplication().invokeLater(() -> {
123 if (StudyUtils.getSelectedEditor(project) != null) {
124 StudyUtils.showCheckPopUp(project, balloon);
128 catch (URISyntaxException e) {
129 LOG.warn(e.getMessage());
134 private static void setTimeout(HttpGet request) {
135 final RequestConfig requestConfig = RequestConfig.custom()
136 .setConnectionRequestTimeout(CONNECTION_TIMEOUT)
137 .setConnectTimeout(CONNECTION_TIMEOUT)
138 .setSocketTimeout(CONNECTION_TIMEOUT)
140 request.setConfig(requestConfig);
143 private static void setTimeout(HttpPost request) {
144 final RequestConfig requestConfig = RequestConfig.custom()
145 .setConnectionRequestTimeout(CONNECTION_TIMEOUT)
146 .setConnectTimeout(CONNECTION_TIMEOUT)
147 .setSocketTimeout(CONNECTION_TIMEOUT)
149 request.setConfig(requestConfig);
152 private static void viewAllSteps(CloseableHttpClient client, int lessonId) throws URISyntaxException, IOException {
153 final URI unitsUrl = new URIBuilder(EduStepicNames.UNITS).addParameter(EduNames.LESSON, String.valueOf(lessonId)).build();
154 final StepicWrappers.UnitContainer unitContainer = getFromStepic(unitsUrl.toString(), StepicWrappers.UnitContainer.class);
155 if (unitContainer.units.size() != 1) {
156 LOG.warn("Got unexpected numbers of units: " + unitContainer.units.size());
160 final URIBuilder builder = new URIBuilder(EduStepicNames.ASSIGNMENT);
161 for (Integer step : unitContainer.units.get(0).assignments) {
162 builder.addParameter("ids[]", String.valueOf(step));
164 final URI assignmentUrl = builder.build();
165 final StepicWrappers.AssignmentsWrapper assignments = getFromStepic(assignmentUrl.toString(), StepicWrappers.AssignmentsWrapper.class);
166 if (assignments.assignments.size() > 0) {
167 for (StepicWrappers.Assignment assignment : assignments.assignments) {
168 final HttpPost post = new HttpPost(EduStepicNames.STEPIC_API_URL + EduStepicNames.VIEWS_URL);
169 final StepicWrappers.ViewsWrapper viewsWrapper = new StepicWrappers.ViewsWrapper(assignment.id, assignment.step);
170 post.setEntity(new StringEntity(new Gson().toJson(viewsWrapper)));
171 setHeaders(post, EduStepicNames.CONTENT_TYPE_APPL_JSON);
172 final CloseableHttpResponse viewPostResult = client.execute(post);
173 if (viewPostResult.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) {
174 LOG.warn("Error while Views post, code: " + viewPostResult.getStatusLine().getStatusCode());
179 LOG.warn("Got assignments of incorrect length: " + assignments.assignments.size());
183 public static boolean postRecommendationReaction(@NotNull final Project project, @NotNull final String lessonId,
184 @NotNull final String user, int reaction) {
185 final HttpPost post = new HttpPost(EduStepicNames.STEPIC_API_URL + EduStepicNames.RECOMMENDATION_REACTIONS_URL);
186 final String json = new Gson()
187 .toJson(new StepicWrappers.RecommendationReactionWrapper(new StepicWrappers.RecommendationReaction(reaction, user, lessonId)));
188 post.setEntity(new StringEntity(json, ContentType.APPLICATION_JSON));
189 final CloseableHttpClient client = getHttpClient(project);
190 setHeaders(post, EduStepicNames.CONTENT_TYPE_APPL_JSON);
193 final CloseableHttpResponse execute = client.execute(post);
194 if (execute.getStatusLine().getStatusCode() == HttpStatus.SC_CREATED) {
198 LOG.warn("Stepic returned non-201 status code: " + execute.getStatusLine().getStatusCode() + " " +
199 EntityUtils.toString(execute.getEntity()));
203 catch (IOException e) {
204 LOG.warn(e.getMessage());
209 public static void addNextRecommendedTask(@NotNull final Project project, int reaction) {
210 final StudyEditor editor = StudyUtils.getSelectedStudyEditor(project);
211 final Course course = StudyTaskManager.getInstance(project).getCourse();
212 if (course != null && editor != null && editor.getTaskFile() != null) {
213 final StepicUser user = StudyTaskManager.getInstance(project).getUser();
215 final boolean recommendationReaction =
216 postRecommendationReaction(project, String.valueOf(editor.getTaskFile().getTask().getLesson().getId()),
217 String.valueOf(user.getId()), reaction);
218 if (recommendationReaction) {
219 final Task task = getNextRecommendation(project, course);
222 final Lesson adaptive = course.getLessons().get(0);
223 final Task unsolvedTask = adaptive.getTaskList().get(adaptive.getTaskList().size() - 1);
224 if (reaction == 0 || reaction == -1) {
225 unsolvedTask.setName(task.getName());
226 unsolvedTask.setStepicId(task.getStepicId());
227 unsolvedTask.setText(task.getText());
228 unsolvedTask.getTestsText().clear();
229 unsolvedTask.setStatus(StudyStatus.Unchecked);
230 final Map<String, String> testsText = task.getTestsText();
231 for (String testName : testsText.keySet()) {
232 unsolvedTask.addTestsTexts(testName, testsText.get(testName));
234 final Map<String, TaskFile> taskFiles = task.getTaskFiles();
235 if (taskFiles.size() == 1) {
236 final TaskFile taskFile = editor.getTaskFile();
237 taskFile.text = ((TaskFile)taskFiles.values().toArray()[0]).text;
239 ApplicationManager.getApplication().invokeLater(() -> ApplicationManager.getApplication().runWriteAction(() -> {
240 final Document document = editor.getEditor().getDocument();
241 final String taskFileText = taskFiles.get(EduStepicNames.DEFAULT_TASKFILE_NAME).text;
242 document.setText(taskFileText);
246 LOG.warn("Got task without unexpected number of task files: " + taskFiles.size());
249 final File lessonDirectory = new File(course.getCourseDirectory(), EduNames.LESSON + String.valueOf(adaptive.getIndex()));
250 final File taskDirectory = new File(lessonDirectory, EduNames.TASK + String.valueOf(adaptive.getTaskList().size()));
251 StudyProjectGenerator.flushTask(task, taskDirectory);
252 StudyProjectGenerator.flushCourseJson(course, new File(course.getCourseDirectory()));
253 final VirtualFile lessonDir = project.getBaseDir().findChild(EduNames.LESSON + String.valueOf(adaptive.getIndex()));
255 if (lessonDir != null) {
256 createTestFiles(course, task, unsolvedTask, lessonDir);
258 final StudyToolWindow window = StudyUtils.getStudyToolWindow(project);
259 if (window != null) {
260 window.setTaskText(unsolvedTask.getText(), unsolvedTask.getTaskDir(project), project);
264 adaptive.addTask(task);
265 task.setIndex(adaptive.getTaskList().size());
266 final VirtualFile lessonDir = project.getBaseDir().findChild(EduNames.LESSON + String.valueOf(adaptive.getIndex()));
268 if (lessonDir != null) {
269 ApplicationManager.getApplication().invokeLater(() -> ApplicationManager.getApplication().runWriteAction(() -> {
271 final File lessonDirectory = new File(course.getCourseDirectory(), EduNames.LESSON + String.valueOf(adaptive.getIndex()));
272 final File taskDir = new File(lessonDirectory, EduNames.TASK + String.valueOf(task.getIndex()));
273 StudyProjectGenerator.flushTask(task, taskDir);
274 StudyProjectGenerator.flushCourseJson(course, new File(course.getCourseDirectory()));
275 StudyGenerator.createTask(task, lessonDir, new File(course.getCourseDirectory(), lessonDir.getName()), project);
276 adaptive.initLesson(course, true);
278 catch (IOException e) {
279 LOG.warn(e.getMessage());
285 ApplicationManager.getApplication().invokeLater(() -> {
286 VirtualFileManager.getInstance().refreshWithoutFileWatcher(false);
287 ProjectView.getInstance(project).refresh();
291 LOG.warn("Recommendation reactions weren't posted");
292 ApplicationManager.getApplication().invokeLater(() -> StudyUtils.showErrorPopupOnToolbar(project));
297 private static void createTestFiles(Course course, Task task, Task unsolvedTask, VirtualFile lessonDir) {
298 ApplicationManager.getApplication().invokeLater(() -> ApplicationManager.getApplication().runWriteAction(() -> {
300 final VirtualFile taskDir = VfsUtil
301 .findFileByIoFile(new File(lessonDir.getCanonicalPath(), EduNames.TASK + unsolvedTask.getIndex()), true);
302 final File resourceRoot = new File(course.getCourseDirectory(), lessonDir.getName());
303 File newResourceRoot = null;
304 if (taskDir != null) {
305 newResourceRoot = new File(resourceRoot, taskDir.getName());
306 File[] filesInTask = newResourceRoot.listFiles();
307 if (filesInTask != null) {
308 for (File file : filesInTask) {
309 String fileName = file.getName();
310 if (!task.isTaskFile(fileName)) {
311 File resourceFile = new File(newResourceRoot, fileName);
312 File fileInProject = new File(taskDir.getCanonicalPath(), fileName);
313 FileUtil.copy(resourceFile, fileInProject);
319 LOG.warn("Task directory is null");
322 catch (IOException e) {
323 LOG.warn(e.getMessage());
329 private static Task getTaskFromStep(Project project, int lessonID, @NotNull final StepicWrappers.Step step, @NotNull String name) {
330 final Task task = new Task();
332 task.setStepicId(lessonID);
333 task.setText(step.text);
334 task.setStatus(StudyStatus.Unchecked);
335 if (step.options.samples != null) {
336 final StringBuilder builder = new StringBuilder();
337 for (List<String> sample : step.options.samples) {
338 if (sample.size() == 2) {
339 builder.append("<b>Sample Input:</b><br>");
340 builder.append(StringUtil.replace(sample.get(0), "\n", "<br>"));
341 builder.append("<br>");
342 builder.append("<b>Sample Output:</b><br>");
343 builder.append(StringUtil.replace(sample.get(1), "\n", "<br>"));
344 builder.append("<br><br>");
347 task.setText(task.getText() + "<br>" + builder.toString());
350 if (step.options.executionMemoryLimit != null && step.options.executionTimeLimit != null) {
351 String builder = "<b>Memory limit</b>: " +
352 step.options.executionMemoryLimit + " Mb" +
354 "<b>Time limit</b>: " +
355 step.options.executionTimeLimit + "s" +
357 task.setText(task.getText() + builder);
360 if (step.options.test != null) {
361 for (StepicWrappers.TestFileWrapper wrapper : step.options.test) {
362 task.addTestsTexts(wrapper.name, wrapper.text);
366 if (step.options.samples != null) {
367 createTestFileFromSamples(task, step.options.samples);
371 task.taskFiles = new HashMap<String, TaskFile>(); // TODO: it looks like we don't need taskFiles as map anymore
372 if (step.options.files != null) {
373 for (TaskFile taskFile : step.options.files) {
374 task.taskFiles.put(taskFile.name, taskFile);
378 final TaskFile taskFile = new TaskFile();
379 taskFile.name = "code";
380 final String templateForTask = getCodeTemplateForTask(step.options.codeTemplates, task, project);
381 taskFile.text = templateForTask == null ? "# write your answer here \n" : templateForTask;
382 task.taskFiles.put("code.py", taskFile);
387 private static String getCodeTemplateForTask(@Nullable StepicWrappers.CodeTemplatesWrapper codeTemplates,
388 @NotNull final Task task, @NotNull final Project project) {
389 if (codeTemplates != null) {
390 final String languageString = getLanguageString(task, project);
391 if (languageString != null) {
392 return codeTemplates.getTemplateForLanguage(languageString);
400 public static Pair<Boolean, String> checkTask(@NotNull final Project project, @NotNull final Task task) {
403 attemptId = getAttemptId(project, task);
405 catch (IOException e) {
406 LOG.warn(e.getMessage());
408 if (attemptId != -1) {
409 final Editor editor = StudyUtils.getSelectedEditor(project);
410 String language = getLanguageString(task, project);
411 if (editor != null && language != null) {
412 final CloseableHttpClient client = getHttpClient(project);
413 StepicWrappers.ResultSubmissionWrapper wrapper = postResultsForCheck(client, attemptId, language, editor.getDocument().getText());
415 final StepicUser user = StudyTaskManager.getInstance(project).getUser();
416 final int id = user.getId();
417 wrapper = getCheckResults(attemptId, id, client, wrapper);
418 if (wrapper.submissions.length == 1) {
419 final boolean isSolved = !wrapper.submissions[0].status.equals("wrong");
420 return Pair.create(isSolved, wrapper.submissions[0].hint);
423 LOG.warn("Got a submission wrapper with incorrect submissions number: " + wrapper.submissions.length);
428 LOG.warn("Got an incorrect attempt id: " + attemptId);
430 return Pair.create(false, "");
434 private static StepicWrappers.ResultSubmissionWrapper postResultsForCheck(@NotNull final CloseableHttpClient client,
436 @NotNull final String language,
437 @NotNull final String text) {
438 final CloseableHttpResponse response;
440 final StepicWrappers.SubmissionToPostWrapper submissionToPostWrapper =
441 new StepicWrappers.SubmissionToPostWrapper(String.valueOf(attemptId), language, PYCHARM_COMMENT + text);
442 final HttpPost httpPost = new HttpPost(EduStepicNames.STEPIC_API_URL + EduStepicNames.SUBMISSIONS);
443 setHeaders(httpPost, EduStepicNames.CONTENT_TYPE_APPL_JSON);
444 setTimeout(httpPost);
446 httpPost.setEntity(new StringEntity(new Gson().toJson(submissionToPostWrapper)));
448 catch (UnsupportedEncodingException e) {
449 LOG.warn(e.getMessage());
451 response = client.execute(httpPost);
452 return new Gson().fromJson(EntityUtils.toString(response.getEntity()), StepicWrappers.ResultSubmissionWrapper.class);
454 catch (IOException e) {
455 LOG.warn(e.getMessage());
461 private static StepicWrappers.ResultSubmissionWrapper getCheckResults(int attemptId,
463 CloseableHttpClient client,
464 StepicWrappers.ResultSubmissionWrapper wrapper) {
466 while (wrapper.submissions.length == 1 && wrapper.submissions[0].status.equals("evaluation")) {
467 TimeUnit.MILLISECONDS.sleep(500);
468 final URI submissionURI = new URIBuilder(EduStepicNames.STEPIC_API_URL + EduStepicNames.SUBMISSIONS)
469 .addParameter("attempt", String.valueOf(attemptId))
470 .addParameter("order", "desc")
471 .addParameter("user", String.valueOf(id))
473 final HttpGet httpGet = new HttpGet(submissionURI);
474 setHeaders(httpGet, EduStepicNames.CONTENT_TYPE_APPL_JSON);
476 final CloseableHttpResponse httpResponse = client.execute(httpGet);
477 final String entity = EntityUtils.toString(httpResponse.getEntity());
478 wrapper = new Gson().fromJson(entity, StepicWrappers.ResultSubmissionWrapper.class);
481 catch (InterruptedException e) {
482 LOG.warn(e.getMessage());
484 catch (IOException e) {
485 LOG.warn(e.getMessage());
487 catch (URISyntaxException e) {
488 LOG.warn(e.getMessage());
494 private static String getLanguageString(@NotNull Task task, @NotNull Project project) {
495 final Language pythonLanguage = Language.findLanguageByID("Python");
496 if (pythonLanguage != null) {
497 final Sdk language = StudyExecutor.INSTANCE.forLanguage(pythonLanguage).findSdk(project);
498 if (language != null) {
499 final String versionString = language.getVersionString();
500 if (versionString != null) {
501 final List<String> versionStringParts = StringUtil.split(versionString, " ");
502 if (versionStringParts.size() == 2) {
503 return versionStringParts.get(1).startsWith("2") ? PYTHON27 : PYTHON3;
508 StudyUtils.showNoSdkNotification(task, project);
514 private static int getAttemptId(@NotNull final Project project, @NotNull Task task) throws IOException {
515 final StepicWrappers.AttemptToPostWrapper attemptWrapper = new StepicWrappers.AttemptToPostWrapper(task.getStepicId());
517 final HttpPost post = new HttpPost(EduStepicNames.STEPIC_API_URL + EduStepicNames.ATTEMPTS);
518 post.setEntity(new StringEntity(new Gson().toJson(attemptWrapper)));
520 final CloseableHttpClient client = getHttpClient(project);
521 setHeaders(post, EduStepicNames.CONTENT_TYPE_APPL_JSON);
523 final CloseableHttpResponse httpResponse = client.execute(post);
524 final String entity = EntityUtils.toString(httpResponse.getEntity());
525 final StepicWrappers.AttemptContainer container =
526 new Gson().fromJson(entity, StepicWrappers.AttemptContainer.class);
527 return (container.attempts != null && !container.attempts.isEmpty()) ? container.attempts.get(0).id : -1;
530 private static void createTestFileFromSamples(@NotNull final Task task,
531 @NotNull final List<List<String>> samples) {
533 String testText = "from test_helper import check_samples\n\n" +
534 "if __name__ == '__main__':\n" +
535 " check_samples(samples=" + new GsonBuilder().create().toJson(samples) + ")";
536 task.addTestsTexts("tests.py", testText);