1 package com.jetbrains.edu.learning.stepic;
3 import com.google.gson.FieldNamingPolicy;
4 import com.google.gson.Gson;
5 import com.google.gson.GsonBuilder;
6 import com.intellij.ide.projectView.ProjectView;
7 import com.intellij.lang.Language;
8 import com.intellij.openapi.application.ApplicationManager;
9 import com.intellij.openapi.diagnostic.Logger;
10 import com.intellij.openapi.editor.Document;
11 import com.intellij.openapi.editor.Editor;
12 import com.intellij.openapi.project.Project;
13 import com.intellij.openapi.projectRoots.Sdk;
14 import com.intellij.openapi.ui.MessageType;
15 import com.intellij.openapi.ui.popup.Balloon;
16 import com.intellij.openapi.ui.popup.JBPopupFactory;
17 import com.intellij.openapi.util.Pair;
18 import com.intellij.openapi.util.io.FileUtil;
19 import com.intellij.openapi.util.text.StringUtil;
20 import com.intellij.openapi.vfs.VfsUtil;
21 import com.intellij.openapi.vfs.VirtualFile;
22 import com.intellij.openapi.vfs.VirtualFileManager;
23 import com.jetbrains.edu.learning.StudyTaskManager;
24 import com.jetbrains.edu.learning.StudyUtils;
25 import com.jetbrains.edu.learning.checker.StudyExecutor;
26 import com.jetbrains.edu.learning.core.EduNames;
27 import com.jetbrains.edu.learning.courseFormat.*;
28 import com.jetbrains.edu.learning.courseGeneration.StudyGenerator;
29 import com.jetbrains.edu.learning.courseGeneration.StudyProjectGenerator;
30 import com.jetbrains.edu.learning.editor.StudyEditor;
31 import com.jetbrains.edu.learning.ui.StudyToolWindow;
32 import org.apache.http.HttpEntity;
33 import org.apache.http.HttpStatus;
34 import org.apache.http.StatusLine;
35 import org.apache.http.client.config.RequestConfig;
36 import org.apache.http.client.methods.CloseableHttpResponse;
37 import org.apache.http.client.methods.HttpGet;
38 import org.apache.http.client.methods.HttpPost;
39 import org.apache.http.client.utils.URIBuilder;
40 import org.apache.http.entity.ContentType;
41 import org.apache.http.entity.StringEntity;
42 import org.apache.http.impl.client.CloseableHttpClient;
43 import org.apache.http.util.EntityUtils;
44 import org.jetbrains.annotations.NotNull;
45 import org.jetbrains.annotations.Nullable;
48 import java.io.IOException;
49 import java.io.UnsupportedEncodingException;
51 import java.net.URISyntaxException;
52 import java.util.HashMap;
53 import java.util.List;
55 import java.util.concurrent.TimeUnit;
57 import static com.jetbrains.edu.learning.stepic.EduStepicConnector.*;
59 public class EduAdaptiveStepicConnector {
60 public static final String PYTHON27 = "python27";
61 public static final String PYTHON3 = "python3";
62 public static final String PYCHARM_COMMENT = "# Posted from PyCharm Edu\n";
63 private static final Logger LOG = Logger.getInstance(EduAdaptiveStepicConnector.class);
64 private static final int CONNECTION_TIMEOUT = 60 * 1000;
67 public static Task getNextRecommendation(@NotNull final Project project, @NotNull Course course) {
69 final CloseableHttpClient client = getHttpClient(project);
70 final URI uri = new URIBuilder(EduStepicNames.STEPIC_API_URL + EduStepicNames.RECOMMENDATIONS_URL)
71 .addParameter(EduNames.COURSE, String.valueOf(course.getId()))
73 final HttpGet request = new HttpGet(uri);
74 setHeaders(request, EduStepicNames.CONTENT_TYPE_APPL_JSON);
77 final CloseableHttpResponse response = client.execute(request);
78 final StatusLine statusLine = response.getStatusLine();
79 final HttpEntity responseEntity = response.getEntity();
80 final String responseString = responseEntity != null ? EntityUtils.toString(responseEntity) : "";
82 if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
83 final Gson gson = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).create();
84 final StepicWrappers.RecommendationWrapper recomWrapper = gson.fromJson(responseString, StepicWrappers.RecommendationWrapper.class);
86 if (recomWrapper.recommendations.length != 0) {
87 final StepicWrappers.Recommendation recommendation = recomWrapper.recommendations[0];
88 final String lessonId = recommendation.lesson;
89 final StepicWrappers.LessonContainer
90 lessonContainer = getFromStepic(EduStepicNames.LESSONS + lessonId, StepicWrappers.LessonContainer.class);
91 if (lessonContainer.lessons.size() == 1) {
92 final Lesson realLesson = lessonContainer.lessons.get(0);
93 course.getLessons().get(0).setId(Integer.parseInt(lessonId));
95 viewAllSteps(client, realLesson.getId());
97 for (int stepId : realLesson.steps) {
98 final StepicWrappers.StepSource step = getStep(stepId);
99 if (step.block.name.equals("code")) {
100 return getTaskFromStep(project, stepId, step.block, realLesson.getName());
103 final StudyEditor editor = StudyUtils.getSelectedStudyEditor(project);
104 if (editor != null && editor.getTaskFile() != null) {
105 final StepicUser user = StudyTaskManager.getInstance(project).getUser();
106 postRecommendationReaction(project, String.valueOf(editor.getTaskFile().getTask().getLesson().getId()),
107 String.valueOf(user.getId()), -1);
108 return getNextRecommendation(project, course);
113 LOG.warn("Got a lesson without code part as a recommendation");
116 LOG.warn("Got unexpected number of lessons: " + lessonContainer.lessons.size());
121 throw new IOException("Stepic returned non 200 status code: " + responseString);
124 catch (IOException e) {
125 LOG.warn(e.getMessage());
127 final String connectionMessages = "Connection problems, Please, try again";
128 final Balloon balloon =
129 JBPopupFactory.getInstance().createHtmlTextBalloonBuilder(connectionMessages, MessageType.ERROR, null)
131 ApplicationManager.getApplication().invokeLater(() -> {
132 if (StudyUtils.getSelectedEditor(project) != null) {
133 StudyUtils.showCheckPopUp(project, balloon);
137 catch (URISyntaxException e) {
138 LOG.warn(e.getMessage());
143 private static void setTimeout(HttpGet request) {
144 final RequestConfig requestConfig = RequestConfig.custom()
145 .setConnectionRequestTimeout(CONNECTION_TIMEOUT)
146 .setConnectTimeout(CONNECTION_TIMEOUT)
147 .setSocketTimeout(CONNECTION_TIMEOUT)
149 request.setConfig(requestConfig);
152 private static void setTimeout(HttpPost request) {
153 final RequestConfig requestConfig = RequestConfig.custom()
154 .setConnectionRequestTimeout(CONNECTION_TIMEOUT)
155 .setConnectTimeout(CONNECTION_TIMEOUT)
156 .setSocketTimeout(CONNECTION_TIMEOUT)
158 request.setConfig(requestConfig);
161 private static void viewAllSteps(CloseableHttpClient client, int lessonId) throws URISyntaxException, IOException {
162 final URI unitsUrl = new URIBuilder(EduStepicNames.UNITS).addParameter(EduNames.LESSON, String.valueOf(lessonId)).build();
163 final StepicWrappers.UnitContainer unitContainer = getFromStepic(unitsUrl.toString(), StepicWrappers.UnitContainer.class);
164 if (unitContainer.units.size() != 1) {
165 LOG.warn("Got unexpected numbers of units: " + unitContainer.units.size());
169 final URIBuilder builder = new URIBuilder(EduStepicNames.ASSIGNMENT);
170 for (Integer step : unitContainer.units.get(0).assignments) {
171 builder.addParameter("ids[]", String.valueOf(step));
173 final URI assignmentUrl = builder.build();
174 final StepicWrappers.AssignmentsWrapper assignments = getFromStepic(assignmentUrl.toString(), StepicWrappers.AssignmentsWrapper.class);
175 if (assignments.assignments.size() > 0) {
176 for (StepicWrappers.Assignment assignment : assignments.assignments) {
177 final HttpPost post = new HttpPost(EduStepicNames.STEPIC_API_URL + EduStepicNames.VIEWS_URL);
178 final StepicWrappers.ViewsWrapper viewsWrapper = new StepicWrappers.ViewsWrapper(assignment.id, assignment.step);
179 post.setEntity(new StringEntity(new Gson().toJson(viewsWrapper)));
180 setHeaders(post, EduStepicNames.CONTENT_TYPE_APPL_JSON);
181 final CloseableHttpResponse viewPostResult = client.execute(post);
182 if (viewPostResult.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) {
183 LOG.warn("Error while Views post, code: " + viewPostResult.getStatusLine().getStatusCode());
188 LOG.warn("Got assignments of incorrect length: " + assignments.assignments.size());
192 public static boolean postRecommendationReaction(@NotNull final Project project, @NotNull final String lessonId,
193 @NotNull final String user, int reaction) {
194 final HttpPost post = new HttpPost(EduStepicNames.STEPIC_API_URL + EduStepicNames.RECOMMENDATION_REACTIONS_URL);
195 final String json = new Gson()
196 .toJson(new StepicWrappers.RecommendationReactionWrapper(new StepicWrappers.RecommendationReaction(reaction, user, lessonId)));
197 post.setEntity(new StringEntity(json, ContentType.APPLICATION_JSON));
198 final CloseableHttpClient client = getHttpClient(project);
199 setHeaders(post, EduStepicNames.CONTENT_TYPE_APPL_JSON);
202 final CloseableHttpResponse execute = client.execute(post);
203 if (execute.getStatusLine().getStatusCode() == HttpStatus.SC_CREATED) {
207 LOG.warn("Stepic returned non-201 status code: " + execute.getStatusLine().getStatusCode() + " " +
208 EntityUtils.toString(execute.getEntity()));
212 catch (IOException e) {
213 LOG.warn(e.getMessage());
218 public static void addNextRecommendedTask(@NotNull final Project project, int reaction) {
219 final StudyEditor editor = StudyUtils.getSelectedStudyEditor(project);
220 final Course course = StudyTaskManager.getInstance(project).getCourse();
221 if (course != null && editor != null && editor.getTaskFile() != null) {
222 final StepicUser user = StudyTaskManager.getInstance(project).getUser();
224 final boolean recommendationReaction =
225 postRecommendationReaction(project, String.valueOf(editor.getTaskFile().getTask().getLesson().getId()),
226 String.valueOf(user.getId()), reaction);
227 if (recommendationReaction) {
228 final Task task = getNextRecommendation(project, course);
231 final Lesson adaptive = course.getLessons().get(0);
232 final Task unsolvedTask = adaptive.getTaskList().get(adaptive.getTaskList().size() - 1);
233 if (reaction == 0 || reaction == -1) {
234 unsolvedTask.setName(task.getName());
235 unsolvedTask.setStepicId(task.getStepicId());
236 unsolvedTask.setText(task.getText());
237 unsolvedTask.getTestsText().clear();
238 unsolvedTask.setStatus(StudyStatus.Unchecked);
239 final Map<String, String> testsText = task.getTestsText();
240 for (String testName : testsText.keySet()) {
241 unsolvedTask.addTestsTexts(testName, testsText.get(testName));
243 final Map<String, TaskFile> taskFiles = task.getTaskFiles();
244 if (taskFiles.size() == 1) {
245 final TaskFile taskFile = editor.getTaskFile();
246 taskFile.text = ((TaskFile)taskFiles.values().toArray()[0]).text;
248 ApplicationManager.getApplication().invokeLater(() -> ApplicationManager.getApplication().runWriteAction(() -> {
249 final Document document = editor.getEditor().getDocument();
250 final String taskFileText = taskFiles.get(EduStepicNames.DEFAULT_TASKFILE_NAME).text;
251 document.setText(taskFileText);
255 LOG.warn("Got task without unexpected number of task files: " + taskFiles.size());
258 final File lessonDirectory = new File(course.getCourseDirectory(), EduNames.LESSON + String.valueOf(adaptive.getIndex()));
259 final File taskDirectory = new File(lessonDirectory, EduNames.TASK + String.valueOf(adaptive.getTaskList().size()));
260 StudyProjectGenerator.flushTask(task, taskDirectory);
261 StudyProjectGenerator.flushCourseJson(course, new File(course.getCourseDirectory()));
262 final VirtualFile lessonDir = project.getBaseDir().findChild(EduNames.LESSON + String.valueOf(adaptive.getIndex()));
264 if (lessonDir != null) {
265 createTestFiles(course, task, unsolvedTask, lessonDir);
267 final StudyToolWindow window = StudyUtils.getStudyToolWindow(project);
268 if (window != null) {
269 window.setTaskText(unsolvedTask.getText(), unsolvedTask.getTaskDir(project), project);
273 adaptive.addTask(task);
274 task.setIndex(adaptive.getTaskList().size());
275 final VirtualFile lessonDir = project.getBaseDir().findChild(EduNames.LESSON + String.valueOf(adaptive.getIndex()));
277 if (lessonDir != null) {
278 ApplicationManager.getApplication().invokeLater(() -> ApplicationManager.getApplication().runWriteAction(() -> {
280 final File lessonDirectory = new File(course.getCourseDirectory(), EduNames.LESSON + String.valueOf(adaptive.getIndex()));
281 final File taskDir = new File(lessonDirectory, EduNames.TASK + String.valueOf(task.getIndex()));
282 StudyProjectGenerator.flushTask(task, taskDir);
283 StudyProjectGenerator.flushCourseJson(course, new File(course.getCourseDirectory()));
284 StudyGenerator.createTask(task, lessonDir, new File(course.getCourseDirectory(), lessonDir.getName()), project);
285 adaptive.initLesson(course, true);
287 catch (IOException e) {
288 LOG.warn(e.getMessage());
294 ApplicationManager.getApplication().invokeLater(() -> {
295 VirtualFileManager.getInstance().refreshWithoutFileWatcher(false);
296 ProjectView.getInstance(project).refresh();
300 LOG.warn("Recommendation reactions weren't posted");
301 ApplicationManager.getApplication().invokeLater(() -> StudyUtils.showErrorPopupOnToolbar(project));
306 private static void createTestFiles(Course course, Task task, Task unsolvedTask, VirtualFile lessonDir) {
307 ApplicationManager.getApplication().invokeLater(() -> ApplicationManager.getApplication().runWriteAction(() -> {
309 final VirtualFile taskDir = VfsUtil
310 .findFileByIoFile(new File(lessonDir.getCanonicalPath(), EduNames.TASK + unsolvedTask.getIndex()), true);
311 final File resourceRoot = new File(course.getCourseDirectory(), lessonDir.getName());
312 File newResourceRoot = null;
313 if (taskDir != null) {
314 newResourceRoot = new File(resourceRoot, taskDir.getName());
315 File[] filesInTask = newResourceRoot.listFiles();
316 if (filesInTask != null) {
317 for (File file : filesInTask) {
318 String fileName = file.getName();
319 if (!task.isTaskFile(fileName)) {
320 File resourceFile = new File(newResourceRoot, fileName);
321 File fileInProject = new File(taskDir.getCanonicalPath(), fileName);
322 FileUtil.copy(resourceFile, fileInProject);
328 LOG.warn("Task directory is null");
331 catch (IOException e) {
332 LOG.warn(e.getMessage());
338 private static Task getTaskFromStep(Project project, int lessonID, @NotNull final StepicWrappers.Step step, @NotNull String name) {
339 final Task task = new Task();
341 task.setStepicId(lessonID);
342 task.setText(step.text);
343 task.setStatus(StudyStatus.Unchecked);
344 if (step.options.samples != null) {
345 final StringBuilder builder = new StringBuilder();
346 for (List<String> sample : step.options.samples) {
347 if (sample.size() == 2) {
348 builder.append("<b>Sample Input:</b><br>");
349 builder.append(StringUtil.replace(sample.get(0), "\n", "<br>"));
350 builder.append("<br>");
351 builder.append("<b>Sample Output:</b><br>");
352 builder.append(StringUtil.replace(sample.get(1), "\n", "<br>"));
353 builder.append("<br><br>");
356 task.setText(task.getText() + "<br>" + builder.toString());
359 if (step.options.executionMemoryLimit != null && step.options.executionTimeLimit != null) {
360 String builder = "<b>Memory limit</b>: " +
361 step.options.executionMemoryLimit + " Mb" +
363 "<b>Time limit</b>: " +
364 step.options.executionTimeLimit + "s" +
366 task.setText(task.getText() + builder);
369 if (step.options.test != null) {
370 for (StepicWrappers.TestFileWrapper wrapper : step.options.test) {
371 task.addTestsTexts(wrapper.name, wrapper.text);
375 if (step.options.samples != null) {
376 createTestFileFromSamples(task, step.options.samples);
380 task.taskFiles = new HashMap<String, TaskFile>(); // TODO: it looks like we don't need taskFiles as map anymore
381 if (step.options.files != null) {
382 for (TaskFile taskFile : step.options.files) {
383 task.taskFiles.put(taskFile.name, taskFile);
387 final TaskFile taskFile = new TaskFile();
388 taskFile.name = "code";
389 final String templateForTask = getCodeTemplateForTask(step.options.codeTemplates, task, project);
390 taskFile.text = templateForTask == null ? "# write your answer here \n" : templateForTask;
391 task.taskFiles.put("code.py", taskFile);
396 private static String getCodeTemplateForTask(@Nullable StepicWrappers.CodeTemplatesWrapper codeTemplates,
397 @NotNull final Task task, @NotNull final Project project) {
398 if (codeTemplates != null) {
399 final String languageString = getLanguageString(task, project);
400 if (languageString != null) {
401 return codeTemplates.getTemplateForLanguage(languageString);
409 public static Pair<Boolean, String> checkTask(@NotNull final Project project, @NotNull final Task task) {
412 attemptId = getAttemptId(project, task);
414 catch (IOException e) {
415 LOG.warn(e.getMessage());
417 if (attemptId != -1) {
418 final Editor editor = StudyUtils.getSelectedEditor(project);
419 String language = getLanguageString(task, project);
420 if (editor != null && language != null) {
421 final CloseableHttpClient client = getHttpClient(project);
422 StepicWrappers.ResultSubmissionWrapper wrapper = postResultsForCheck(client, attemptId, language, editor.getDocument().getText());
424 final StepicUser user = StudyTaskManager.getInstance(project).getUser();
425 final int id = user.getId();
426 wrapper = getCheckResults(attemptId, id, client, wrapper);
427 if (wrapper.submissions.length == 1) {
428 final boolean isSolved = !wrapper.submissions[0].status.equals("wrong");
429 return Pair.create(isSolved, wrapper.submissions[0].hint);
432 LOG.warn("Got a submission wrapper with incorrect submissions number: " + wrapper.submissions.length);
437 LOG.warn("Got an incorrect attempt id: " + attemptId);
439 return Pair.create(false, "");
443 private static StepicWrappers.ResultSubmissionWrapper postResultsForCheck(@NotNull final CloseableHttpClient client,
445 @NotNull final String language,
446 @NotNull final String text) {
447 final CloseableHttpResponse response;
449 final StepicWrappers.SubmissionToPostWrapper submissionToPostWrapper =
450 new StepicWrappers.SubmissionToPostWrapper(String.valueOf(attemptId), language, PYCHARM_COMMENT + text);
451 final HttpPost httpPost = new HttpPost(EduStepicNames.STEPIC_API_URL + EduStepicNames.SUBMISSIONS);
452 setHeaders(httpPost, EduStepicNames.CONTENT_TYPE_APPL_JSON);
453 setTimeout(httpPost);
455 httpPost.setEntity(new StringEntity(new Gson().toJson(submissionToPostWrapper)));
457 catch (UnsupportedEncodingException e) {
458 LOG.warn(e.getMessage());
460 response = client.execute(httpPost);
461 return new Gson().fromJson(EntityUtils.toString(response.getEntity()), StepicWrappers.ResultSubmissionWrapper.class);
463 catch (IOException e) {
464 LOG.warn(e.getMessage());
470 private static StepicWrappers.ResultSubmissionWrapper getCheckResults(int attemptId,
472 CloseableHttpClient client,
473 StepicWrappers.ResultSubmissionWrapper wrapper) {
475 while (wrapper.submissions.length == 1 && wrapper.submissions[0].status.equals("evaluation")) {
476 TimeUnit.MILLISECONDS.sleep(500);
477 final URI submissionURI = new URIBuilder(EduStepicNames.STEPIC_API_URL + EduStepicNames.SUBMISSIONS)
478 .addParameter("attempt", String.valueOf(attemptId))
479 .addParameter("order", "desc")
480 .addParameter("user", String.valueOf(id))
482 final HttpGet httpGet = new HttpGet(submissionURI);
483 setHeaders(httpGet, EduStepicNames.CONTENT_TYPE_APPL_JSON);
485 final CloseableHttpResponse httpResponse = client.execute(httpGet);
486 final String entity = EntityUtils.toString(httpResponse.getEntity());
487 wrapper = new Gson().fromJson(entity, StepicWrappers.ResultSubmissionWrapper.class);
490 catch (InterruptedException e) {
491 LOG.warn(e.getMessage());
493 catch (IOException e) {
494 LOG.warn(e.getMessage());
496 catch (URISyntaxException e) {
497 LOG.warn(e.getMessage());
503 private static String getLanguageString(@NotNull Task task, @NotNull Project project) {
504 final Language pythonLanguage = Language.findLanguageByID("Python");
505 if (pythonLanguage != null) {
506 final Sdk language = StudyExecutor.INSTANCE.forLanguage(pythonLanguage).findSdk(project);
507 if (language != null) {
508 final String versionString = language.getVersionString();
509 if (versionString != null) {
510 final List<String> versionStringParts = StringUtil.split(versionString, " ");
511 if (versionStringParts.size() == 2) {
512 return versionStringParts.get(1).startsWith("2") ? PYTHON27 : PYTHON3;
517 StudyUtils.showNoSdkNotification(task, project);
523 private static int getAttemptId(@NotNull final Project project, @NotNull Task task) throws IOException {
524 final StepicWrappers.AttemptToPostWrapper attemptWrapper = new StepicWrappers.AttemptToPostWrapper(task.getStepicId());
526 final HttpPost post = new HttpPost(EduStepicNames.STEPIC_API_URL + EduStepicNames.ATTEMPTS);
527 post.setEntity(new StringEntity(new Gson().toJson(attemptWrapper)));
529 final CloseableHttpClient client = getHttpClient(project);
530 setHeaders(post, EduStepicNames.CONTENT_TYPE_APPL_JSON);
532 final CloseableHttpResponse httpResponse = client.execute(post);
533 final String entity = EntityUtils.toString(httpResponse.getEntity());
534 final StepicWrappers.AttemptContainer container =
535 new Gson().fromJson(entity, StepicWrappers.AttemptContainer.class);
536 return (container.attempts != null && !container.attempts.isEmpty()) ? container.attempts.get(0).id : -1;
539 private static void createTestFileFromSamples(@NotNull final Task task,
540 @NotNull final List<List<String>> samples) {
542 String testText = "from test_helper import check_samples\n\n" +
543 "if __name__ == '__main__':\n" +
544 " check_samples(samples=" + new GsonBuilder().create().toJson(samples) + ")";
545 task.addTestsTexts("tests.py", testText);