1 package com.jetbrains.edu.learning.courseGeneration;
3 import com.google.gson.*;
4 import com.google.gson.stream.JsonReader;
5 import com.intellij.facet.ui.ValidationResult;
6 import com.intellij.ide.projectView.ProjectView;
7 import com.intellij.openapi.application.ApplicationManager;
8 import com.intellij.openapi.application.PathManager;
9 import com.intellij.openapi.diagnostic.Logger;
10 import com.intellij.openapi.fileEditor.FileEditor;
11 import com.intellij.openapi.fileEditor.FileEditorManager;
12 import com.intellij.openapi.progress.ProgressManager;
13 import com.intellij.openapi.project.Project;
14 import com.intellij.openapi.startup.StartupManager;
15 import com.intellij.openapi.ui.Messages;
16 import com.intellij.openapi.util.io.FileUtil;
17 import com.intellij.openapi.util.text.StringUtil;
18 import com.intellij.openapi.vfs.LocalFileSystem;
19 import com.intellij.openapi.vfs.VirtualFile;
20 import com.intellij.openapi.vfs.VirtualFileManager;
21 import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
22 import com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl;
23 import com.intellij.platform.templates.github.ZipUtil;
24 import com.intellij.psi.PsiFile;
25 import com.intellij.psi.PsiManager;
26 import com.intellij.util.containers.ContainerUtil;
27 import com.jetbrains.edu.learning.StudySerializationUtils;
28 import com.jetbrains.edu.learning.StudyTaskManager;
29 import com.jetbrains.edu.learning.StudyUtils;
30 import com.jetbrains.edu.learning.core.EduNames;
31 import com.jetbrains.edu.learning.core.EduUtils;
32 import com.jetbrains.edu.learning.courseFormat.Course;
33 import com.jetbrains.edu.learning.courseFormat.Lesson;
34 import com.jetbrains.edu.learning.courseFormat.Task;
35 import com.jetbrains.edu.learning.courseFormat.TaskFile;
36 import com.jetbrains.edu.learning.editor.StudyEditor;
37 import com.jetbrains.edu.learning.statistics.EduUsagesCollector;
38 import com.jetbrains.edu.learning.stepic.CourseInfo;
39 import com.jetbrains.edu.learning.stepic.EduStepicConnector;
40 import com.jetbrains.edu.learning.stepic.StepicUser;
41 import org.apache.commons.codec.binary.Base64;
42 import org.jetbrains.annotations.NotNull;
43 import org.jetbrains.annotations.Nullable;
48 import static com.jetbrains.edu.learning.StudyUtils.execCancelable;
50 public class StudyProjectGenerator {
51 public static final String AUTHOR_ATTRIBUTE = "authors";
52 public static final String LANGUAGE_ATTRIBUTE = "language";
53 public static final String ADAPTIVE_COURSE_PREFIX = "__AdaptivePyCharmPython__";
54 public static final File OUR_COURSES_DIR = new File(PathManager.getConfigPath(), "courses");
55 private static final Logger LOG = Logger.getInstance(StudyProjectGenerator.class.getName());
56 private static final String COURSE_NAME_ATTRIBUTE = "name";
57 private static final String COURSE_DESCRIPTION = "description";
58 private static final String CACHE_NAME = "courseNames.txt";
59 private final List<SettingsListener> myListeners = ContainerUtil.newArrayList();
60 @Nullable public StepicUser myUser;
61 private List<CourseInfo> myCourses = new ArrayList<>();
62 private List<Integer> myEnrolledCoursesIds = new ArrayList<>();
63 protected CourseInfo mySelectedCourseInfo;
65 public void setCourses(List<CourseInfo> courses) {
69 public boolean isLoggedIn() {
70 return myUser != null && !StringUtil.isEmptyOrSpaces(myUser.getPassword()) && !StringUtil.isEmptyOrSpaces(myUser.getEmail());
73 public void setEnrolledCoursesIds(@NotNull final List<Integer> coursesIds) {
74 myEnrolledCoursesIds = coursesIds;
78 public List<Integer> getEnrolledCoursesIds() {
79 return myEnrolledCoursesIds;
82 public void setSelectedCourse(@NotNull final CourseInfo courseName) {
83 mySelectedCourseInfo = courseName;
86 public void generateProject(@NotNull final Project project, @NotNull final VirtualFile baseDir) {
88 StudyTaskManager.getInstance(project).setUser(myUser);
90 final Course course = getCourse(project);
92 LOG.warn("Course is null");
93 Messages.showWarningDialog("Some problems occurred while creating the course", "Error in Course Creation");
96 final File courseDirectory = StudyUtils.getCourseDirectory(project, course);
97 StudyTaskManager.getInstance(project).setCourse(course);
98 ApplicationManager.getApplication().runWriteAction(() -> {
99 StudyGenerator.createCourse(course, baseDir, courseDirectory, project);
100 course.setCourseDirectory(courseDirectory.getAbsolutePath());
101 VirtualFileManager.getInstance().refreshWithoutFileWatcher(true);
102 StudyUtils.registerStudyToolWindow(course, project);
103 openFirstTask(course, project);
104 EduUsagesCollector.projectTypeCreated(course.isAdaptive() ? EduNames.ADAPTIVE : EduNames.STUDY);
109 public Course getCourse(@NotNull final Project project) {
111 final File courseFile = new File(new File(OUR_COURSES_DIR, mySelectedCourseInfo.getName()), EduNames.COURSE_META_FILE);
112 if (courseFile.exists()) {
113 final Course course = readCourseFromCache(courseFile, false);
114 if (course != null && course.isUpToDate()) {
117 return getCourseFromStepic(project);
119 else if (myUser != null) {
120 final File adaptiveCourseFile = new File(new File(OUR_COURSES_DIR, ADAPTIVE_COURSE_PREFIX +
121 mySelectedCourseInfo.getName() + "_" +
122 myUser.getEmail()), EduNames.COURSE_META_FILE);
123 if (adaptiveCourseFile.exists()) {
124 return readCourseFromCache(adaptiveCourseFile, true);
127 return getCourseFromStepic(project);
130 private Course getCourseFromStepic(@NotNull Project project) {
131 return ProgressManager.getInstance().runProcessWithProgressSynchronously(() -> {
132 ProgressManager.getInstance().getProgressIndicator().setIndeterminate(true);
133 return execCancelable(() -> {
134 final Course course = EduStepicConnector.getCourse(project, mySelectedCourseInfo);
135 if (course != null) {
136 flushCourse(project, course);
137 course.initCourse(false);
141 }, "Creating Course", true, project);
145 private static Course readCourseFromCache(@NotNull File courseFile, boolean isAdaptive) {
146 Reader reader = null;
148 reader = new InputStreamReader(new FileInputStream(courseFile), "UTF-8");
150 new GsonBuilder().registerTypeAdapter(Course.class, new StudySerializationUtils.Json.CourseTypeAdapter(courseFile)).create();
151 final Course course = gson.fromJson(reader, Course.class);
152 course.initCourse(isAdaptive);
155 catch (UnsupportedEncodingException e) {
156 LOG.warn(e.getMessage());
158 catch (FileNotFoundException e) {
159 LOG.warn(e.getMessage());
162 StudyUtils.closeSilently(reader);
167 public static void openFirstTask(@NotNull final Course course, @NotNull final Project project) {
168 LocalFileSystem.getInstance().refresh(false);
169 final Lesson firstLesson = StudyUtils.getFirst(course.getLessons());
170 if (firstLesson == null) return;
171 final Task firstTask = StudyUtils.getFirst(firstLesson.getTaskList());
172 if (firstTask == null) return;
173 final VirtualFile taskDir = firstTask.getTaskDir(project);
174 if (taskDir == null) return;
175 final Map<String, TaskFile> taskFiles = firstTask.getTaskFiles();
176 VirtualFile activeVirtualFile = null;
177 for (Map.Entry<String, TaskFile> entry : taskFiles.entrySet()) {
178 final String name = entry.getKey();
179 final TaskFile taskFile = entry.getValue();
180 final VirtualFile virtualFile = ((VirtualDirectoryImpl)taskDir).refreshAndFindChild(name);
181 if (virtualFile != null) {
182 FileEditorManager.getInstance(project).openFile(virtualFile, true);
183 if (!taskFile.getAnswerPlaceholders().isEmpty()) {
184 activeVirtualFile = virtualFile;
188 if (activeVirtualFile != null) {
189 VirtualFile finalActiveVirtualFile = activeVirtualFile;
190 StartupManager.getInstance(project).registerPostStartupActivity(() -> {
191 final PsiFile file = PsiManager.getInstance(project).findFile(finalActiveVirtualFile);
192 ProjectView.getInstance(project).select(file, finalActiveVirtualFile, false);
193 final FileEditor[] editors = FileEditorManager.getInstance(project).getEditors(finalActiveVirtualFile);
194 if (editors.length == 0) {
197 final FileEditor studyEditor = editors[0];
198 if (studyEditor instanceof StudyEditor) {
199 StudyUtils.selectFirstAnswerPlaceholder((StudyEditor)studyEditor, project);
202 FileEditorManager.getInstance(project).openFile(finalActiveVirtualFile, true);
205 String first = StudyUtils.getFirst(taskFiles.keySet());
207 NewVirtualFile firstFile = ((VirtualDirectoryImpl)taskDir).refreshAndFindChild(first);
208 if (firstFile != null) {
209 FileEditorManager.getInstance(project).openFile(firstFile, true);
215 public static void flushCourse(@NotNull final Project project, @NotNull final Course course) {
216 final File courseDirectory = StudyUtils.getCourseDirectory(project, course);
217 FileUtil.createDirectory(courseDirectory);
218 flushCourseJson(course, courseDirectory);
221 for (Lesson lesson : course.getLessons()) {
222 if (lesson.getName().equals(EduNames.PYCHARM_ADDITIONAL)) {
223 flushAdditionalFiles(courseDirectory, lesson);
226 final File lessonDirectory = new File(courseDirectory, EduNames.LESSON + String.valueOf(lessonIndex));
227 flushLesson(lessonDirectory, lesson);
233 private static void flushAdditionalFiles(File courseDirectory, Lesson lesson) {
234 final List<Task> taskList = lesson.getTaskList();
235 if (taskList.size() != 1) return;
236 final Task task = taskList.get(0);
237 for (Map.Entry<String, String> entry : task.getTestsText().entrySet()) {
238 final String name = entry.getKey();
239 final String text = entry.getValue();
240 final File file = new File(courseDirectory, name);
241 FileUtil.createIfDoesntExist(file);
243 if (EduUtils.isImage(name)) {
244 FileUtil.writeToFile(file, Base64.decodeBase64(text));
247 FileUtil.writeToFile(file, text);
250 catch (IOException e) {
251 LOG.error("ERROR copying file " + name);
256 public static void flushLesson(@NotNull final File lessonDirectory, @NotNull final Lesson lesson) {
257 FileUtil.createDirectory(lessonDirectory);
259 for (Task task : lesson.taskList) {
260 final File taskDirectory = new File(lessonDirectory, EduNames.TASK + String.valueOf(taskIndex));
261 flushTask(task, taskDirectory);
266 public static void flushTask(@NotNull final Task task, @NotNull final File taskDirectory) {
267 FileUtil.createDirectory(taskDirectory);
268 for (Map.Entry<String, TaskFile> taskFileEntry : task.taskFiles.entrySet()) {
269 final String name = taskFileEntry.getKey();
270 final TaskFile taskFile = taskFileEntry.getValue();
271 final File file = new File(taskDirectory, name);
272 FileUtil.createIfDoesntExist(file);
275 if (EduUtils.isImage(taskFile.name)) {
276 FileUtil.writeToFile(file, Base64.decodeBase64(taskFile.text));
279 FileUtil.writeToFile(file, taskFile.text);
282 catch (IOException e) {
283 LOG.error("ERROR copying file " + name);
286 final Map<String, String> testsText = task.getTestsText();
287 for (Map.Entry<String, String> entry : testsText.entrySet()) {
288 final File testsFile = new File(taskDirectory, entry.getKey());
289 if (testsFile.exists()) {
290 FileUtil.delete(testsFile);
292 FileUtil.createIfDoesntExist(testsFile);
294 FileUtil.writeToFile(testsFile, entry.getValue());
296 catch (IOException e) {
297 LOG.error("ERROR copying tests file");
300 final File taskText = new File(taskDirectory, "task.html");
301 FileUtil.createIfDoesntExist(taskText);
303 FileUtil.writeToFile(taskText, task.getText());
305 catch (IOException e) {
306 LOG.error("ERROR copying tests file");
310 public static void flushCourseJson(@NotNull final Course course, @NotNull final File courseDirectory) {
311 final Gson gson = new GsonBuilder().setPrettyPrinting().
312 excludeFieldsWithoutExposeAnnotation().create();
313 final String json = gson.toJson(course);
314 final File courseJson = new File(courseDirectory, EduNames.COURSE_META_FILE);
315 final FileOutputStream fileOutputStream;
317 fileOutputStream = new FileOutputStream(courseJson);
318 OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream, "UTF-8");
320 outputStreamWriter.write(json);
322 catch (IOException e) {
323 Messages.showErrorDialog(e.getMessage(), "Failed to Generate Json");
328 outputStreamWriter.close();
330 catch (IOException e) {
335 catch (FileNotFoundException | UnsupportedEncodingException e) {
341 * Writes courses to cache file {@link StudyProjectGenerator#CACHE_NAME}
343 @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
344 public static void flushCache(List<CourseInfo> courses) {
345 flushCache(courses, true);
348 public static void flushCache(List<CourseInfo> courses, boolean preserveOld) {
349 File cacheFile = new File(OUR_COURSES_DIR, CACHE_NAME);
350 PrintWriter writer = null;
352 if (!createCacheFile(cacheFile)) return;
353 Gson gson = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).create();
355 final Set<CourseInfo> courseInfos = new HashSet<>();
356 courseInfos.addAll(courses);
358 courseInfos.addAll(getCoursesFromCache());
361 writer = new PrintWriter(cacheFile, "UTF-8");
363 for (CourseInfo courseInfo : courseInfos) {
364 final String json = gson.toJson(courseInfo);
365 writer.println(json);
369 StudyUtils.closeSilently(writer);
372 catch (IOException e) {
376 StudyUtils.closeSilently(writer);
380 private static boolean createCacheFile(File cacheFile) throws IOException {
381 if (!OUR_COURSES_DIR.exists()) {
382 final boolean created = OUR_COURSES_DIR.mkdirs();
384 LOG.error("Cannot flush courses cache. Can't create courses directory");
388 if (!cacheFile.exists()) {
389 final boolean created = cacheFile.createNewFile();
391 LOG.error("Cannot flush courses cache. Can't create " + CACHE_NAME + " file");
398 // Supposed to be called under progress
399 public List<CourseInfo> getCourses(boolean force) {
400 if (OUR_COURSES_DIR.exists()) {
401 myCourses = getCoursesFromCache();
403 if (force || myCourses.isEmpty()) {
404 myCourses = execCancelable(EduStepicConnector::getCourses);
405 flushCache(myCourses);
407 if (myCourses.isEmpty()) {
408 myCourses = getBundledIntro();
410 StudyUtils.sortCourses(myCourses);
415 public List<CourseInfo> getCoursesUnderProgress(boolean force, @NotNull final String progressTitle, @NotNull final Project project) {
417 return ProgressManager.getInstance()
418 .runProcessWithProgressSynchronously(() -> {
419 ProgressManager.getInstance().getProgressIndicator().setIndeterminate(true);
420 return getCourses(force);
421 }, progressTitle, true, project);
423 catch (RuntimeException e) {
424 return Collections.singletonList(CourseInfo.INVALID_COURSE);
428 public void addSettingsStateListener(@NotNull SettingsListener listener) {
429 myListeners.add(listener);
432 public interface SettingsListener {
433 void stateChanged(ValidationResult result);
436 public void fireStateChanged(ValidationResult result) {
437 for (SettingsListener listener : myListeners) {
438 listener.stateChanged(result);
442 public static List<CourseInfo> getBundledIntro() {
443 final File introCourse = new File(OUR_COURSES_DIR, "Introduction to Python");
444 if (introCourse.exists()) {
445 final CourseInfo courseInfo = getCourseInfo(introCourse);
447 return Collections.singletonList(courseInfo);
449 return Collections.emptyList();
452 public static List<CourseInfo> getCoursesFromCache() {
453 List<CourseInfo> courses = new ArrayList<>();
454 final File cacheFile = new File(OUR_COURSES_DIR, CACHE_NAME);
455 if (!cacheFile.exists()) {
459 final FileInputStream inputStream = new FileInputStream(cacheFile);
461 BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));
464 while ((line = reader.readLine()) != null) {
465 Gson gson = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).create();
466 final CourseInfo courseInfo = gson.fromJson(line, CourseInfo.class);
467 courses.add(courseInfo);
470 catch (IOException | JsonSyntaxException e) {
471 LOG.error(e.getMessage());
474 StudyUtils.closeSilently(reader);
477 catch (UnsupportedEncodingException e) {
478 LOG.error(e.getMessage());
481 StudyUtils.closeSilently(inputStream);
484 catch (FileNotFoundException e) {
485 LOG.error(e.getMessage());
491 * Adds course from zip archive to courses
493 * @return added course name or null if course is invalid
496 public CourseInfo addLocalCourse(String zipFilePath) {
497 File file = new File(zipFilePath);
499 String fileName = file.getName();
500 String unzippedName = fileName.substring(0, fileName.indexOf("."));
501 File courseDir = new File(OUR_COURSES_DIR, unzippedName);
502 ZipUtil.unzip(null, courseDir, file, null, null, true);
503 CourseInfo courseName = addCourse(myCourses, courseDir);
504 flushCache(myCourses);
505 if (courseName != null && !courseName.getName().equals(unzippedName)) {
506 //noinspection ResultOfMethodCallIgnored
507 File dest = new File(OUR_COURSES_DIR, courseName.getName());
509 FileUtil.delete(dest);
511 courseDir.renameTo(dest);
512 //noinspection ResultOfMethodCallIgnored
517 catch (IOException e) {
518 LOG.error("Failed to unzip course archive");
525 * Adds course to courses specified in params
528 * @param courseDir must be directory containing course file
529 * @return added course name or null if course is invalid
532 private static CourseInfo addCourse(List<CourseInfo> courses, File courseDir) {
533 if (courseDir.isDirectory()) {
534 File[] courseFiles = courseDir.listFiles((dir, name) -> name.equals(EduNames.COURSE_META_FILE));
535 if (courseFiles == null || courseFiles.length != 1) {
536 LOG.info("User tried to add course with more than one or without course files");
539 File courseFile = courseFiles[0];
540 CourseInfo courseInfo = getCourseInfo(courseFile);
541 if (courseInfo != null) {
542 courses.add(courseInfo);
550 * Parses course json meta file and finds course name
552 * @return information about course or null if course file is invalid
555 private static CourseInfo getCourseInfo(File courseFile) {
556 if (courseFile.isDirectory()) {
557 File[] courseFiles = courseFile.listFiles((dir, name) -> name.equals(EduNames.COURSE_META_FILE));
558 if (courseFiles == null || courseFiles.length != 1) {
559 LOG.info("More than one or without course files");
562 courseFile = courseFiles[0];
564 CourseInfo courseInfo = null;
565 BufferedReader reader = null;
567 if (courseFile.getName().equals(EduNames.COURSE_META_FILE)) {
568 reader = new BufferedReader(new InputStreamReader(new FileInputStream(courseFile), "UTF-8"));
569 JsonReader r = new JsonReader(reader);
570 JsonParser parser = new JsonParser();
571 JsonElement el = parser.parse(r);
572 String courseName = el.getAsJsonObject().get(COURSE_NAME_ATTRIBUTE).getAsString();
573 String courseDescription = el.getAsJsonObject().get(COURSE_DESCRIPTION).getAsString();
574 JsonArray courseAuthors = el.getAsJsonObject().get(AUTHOR_ATTRIBUTE).getAsJsonArray();
575 String language = el.getAsJsonObject().get(LANGUAGE_ATTRIBUTE).getAsString();
576 courseInfo = new CourseInfo();
577 courseInfo.setName(courseName);
578 courseInfo.setDescription(courseDescription);
579 courseInfo.setType("pycharm " + language);
580 final ArrayList<StepicUser> authors = new ArrayList<>();
581 for (JsonElement author : courseAuthors) {
582 final JsonObject authorAsJsonObject = author.getAsJsonObject();
583 final StepicUser stepicUser = new StepicUser();
584 stepicUser.setFirstName(authorAsJsonObject.get("first_name").getAsString());
585 stepicUser.setLastName(authorAsJsonObject.get("last_name").getAsString());
586 authors.add(stepicUser);
588 courseInfo.setAuthors(authors);
591 catch (Exception e) {
592 //error will be shown in UI
595 StudyUtils.closeSilently(reader);