1 package com.jetbrains.edu.learning.courseGeneration;
3 import com.google.gson.*;
4 import com.google.gson.stream.JsonReader;
5 import com.intellij.facet.ui.ValidationResult;
6 import com.intellij.ide.projectView.ProjectView;
7 import com.intellij.openapi.application.ApplicationManager;
8 import com.intellij.openapi.application.PathManager;
9 import com.intellij.openapi.diagnostic.Logger;
10 import com.intellij.openapi.fileEditor.FileEditor;
11 import com.intellij.openapi.fileEditor.FileEditorManager;
12 import com.intellij.openapi.progress.ProgressManager;
13 import com.intellij.openapi.project.Project;
14 import com.intellij.openapi.startup.StartupManager;
15 import com.intellij.openapi.ui.Messages;
16 import com.intellij.openapi.util.io.FileUtil;
17 import com.intellij.openapi.util.text.StringUtil;
18 import com.intellij.openapi.vfs.LocalFileSystem;
19 import com.intellij.openapi.vfs.VirtualFile;
20 import com.intellij.openapi.vfs.VirtualFileManager;
21 import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
22 import com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl;
23 import com.intellij.platform.templates.github.ZipUtil;
24 import com.intellij.psi.PsiFile;
25 import com.intellij.psi.PsiManager;
26 import com.intellij.util.containers.ContainerUtil;
27 import com.jetbrains.edu.learning.StudyProjectComponent;
28 import com.jetbrains.edu.learning.StudySerializationUtils;
29 import com.jetbrains.edu.learning.StudyTaskManager;
30 import com.jetbrains.edu.learning.StudyUtils;
31 import com.jetbrains.edu.learning.core.EduNames;
32 import com.jetbrains.edu.learning.core.EduUtils;
33 import com.jetbrains.edu.learning.courseFormat.Course;
34 import com.jetbrains.edu.learning.courseFormat.Lesson;
35 import com.jetbrains.edu.learning.courseFormat.Task;
36 import com.jetbrains.edu.learning.courseFormat.TaskFile;
37 import com.jetbrains.edu.learning.editor.StudyEditor;
38 import com.jetbrains.edu.learning.statistics.EduUsagesCollector;
39 import com.jetbrains.edu.learning.stepic.CourseInfo;
40 import com.jetbrains.edu.learning.stepic.EduStepicConnector;
41 import com.jetbrains.edu.learning.stepic.StepicUser;
42 import org.apache.commons.codec.binary.Base64;
43 import org.jetbrains.annotations.NotNull;
44 import org.jetbrains.annotations.Nullable;
49 import static com.jetbrains.edu.learning.StudyUtils.execCancelable;
51 public class StudyProjectGenerator {
52 public static final String AUTHOR_ATTRIBUTE = "authors";
53 public static final String LANGUAGE_ATTRIBUTE = "language";
54 public static final String ADAPTIVE_COURSE_PREFIX = "__AdaptivePyCharmPython__";
55 public static final File OUR_COURSES_DIR = new File(PathManager.getConfigPath(), "courses");
56 private static final Logger LOG = Logger.getInstance(StudyProjectGenerator.class.getName());
57 private static final String COURSE_NAME_ATTRIBUTE = "name";
58 private static final String COURSE_DESCRIPTION = "description";
59 private static final String CACHE_NAME = "courseNames.txt";
60 private final List<SettingsListener> myListeners = ContainerUtil.newArrayList();
61 @Nullable public StepicUser myUser;
62 private List<CourseInfo> myCourses = new ArrayList<>();
63 private List<Integer> myEnrolledCoursesIds = new ArrayList<>();
64 protected CourseInfo mySelectedCourseInfo;
66 public void setCourses(List<CourseInfo> courses) {
70 public boolean isLoggedIn() {
71 return myUser != null && !StringUtil.isEmptyOrSpaces(myUser.getPassword()) && !StringUtil.isEmptyOrSpaces(myUser.getEmail());
74 public void setEnrolledCoursesIds(@NotNull final List<Integer> coursesIds) {
75 myEnrolledCoursesIds = coursesIds;
79 public List<Integer> getEnrolledCoursesIds() {
80 return myEnrolledCoursesIds;
83 public void setSelectedCourse(@NotNull final CourseInfo courseName) {
84 mySelectedCourseInfo = courseName;
87 public void generateProject(@NotNull final Project project, @NotNull final VirtualFile baseDir) {
89 StudyTaskManager.getInstance(project).setUser(myUser);
91 final Course course = getCourse(project);
93 LOG.warn("Course is null");
94 Messages.showWarningDialog("Some problems occurred while creating the course", "Error in Course Creation");
97 final File courseDirectory = StudyUtils.getCourseDirectory(project, course);
98 StudyTaskManager.getInstance(project).setCourse(course);
99 ApplicationManager.getApplication().runWriteAction(() -> {
100 StudyGenerator.createCourse(course, baseDir, courseDirectory, project);
101 course.setCourseDirectory(courseDirectory.getAbsolutePath());
102 VirtualFileManager.getInstance().refreshWithoutFileWatcher(true);
103 StudyProjectComponent.getInstance(project).registerStudyToolWindow(course);
104 openFirstTask(course, project);
105 EduUsagesCollector.projectTypeCreated(course.isAdaptive() ? EduNames.ADAPTIVE : EduNames.STUDY);
110 public Course getCourse(@NotNull final Project project) {
112 final File courseFile = new File(new File(OUR_COURSES_DIR, mySelectedCourseInfo.getName()), EduNames.COURSE_META_FILE);
113 if (courseFile.exists()) {
114 final Course course = readCourseFromCache(courseFile, false);
115 if (course != null && course.isUpToDate()) {
118 return getCourseFromStepic(project);
120 else if (myUser != null) {
121 final File adaptiveCourseFile = new File(new File(OUR_COURSES_DIR, ADAPTIVE_COURSE_PREFIX +
122 mySelectedCourseInfo.getName() + "_" +
123 myUser.getEmail()), EduNames.COURSE_META_FILE);
124 if (adaptiveCourseFile.exists()) {
125 return readCourseFromCache(adaptiveCourseFile, true);
128 return getCourseFromStepic(project);
131 private Course getCourseFromStepic(@NotNull Project project) {
132 return ProgressManager.getInstance().runProcessWithProgressSynchronously(() -> {
133 ProgressManager.getInstance().getProgressIndicator().setIndeterminate(true);
134 return execCancelable(() -> {
135 final Course course = EduStepicConnector.getCourse(project, mySelectedCourseInfo);
136 if (course != null) {
137 flushCourse(project, course);
138 course.initCourse(false);
142 }, "Creating Course", true, project);
146 private static Course readCourseFromCache(@NotNull File courseFile, boolean isAdaptive) {
147 Reader reader = null;
149 reader = new InputStreamReader(new FileInputStream(courseFile), "UTF-8");
151 new GsonBuilder().registerTypeAdapter(Course.class, new StudySerializationUtils.Json.CourseTypeAdapter(courseFile)).create();
152 final Course course = gson.fromJson(reader, Course.class);
153 course.initCourse(isAdaptive);
156 catch (UnsupportedEncodingException e) {
157 LOG.warn(e.getMessage());
159 catch (FileNotFoundException e) {
160 LOG.warn(e.getMessage());
163 StudyUtils.closeSilently(reader);
168 public static void openFirstTask(@NotNull final Course course, @NotNull final Project project) {
169 LocalFileSystem.getInstance().refresh(false);
170 final Lesson firstLesson = StudyUtils.getFirst(course.getLessons());
171 if (firstLesson == null) return;
172 final Task firstTask = StudyUtils.getFirst(firstLesson.getTaskList());
173 if (firstTask == null) return;
174 final VirtualFile taskDir = firstTask.getTaskDir(project);
175 if (taskDir == null) return;
176 final Map<String, TaskFile> taskFiles = firstTask.getTaskFiles();
177 VirtualFile activeVirtualFile = null;
178 for (Map.Entry<String, TaskFile> entry : taskFiles.entrySet()) {
179 final String name = entry.getKey();
180 final TaskFile taskFile = entry.getValue();
181 final VirtualFile virtualFile = ((VirtualDirectoryImpl)taskDir).refreshAndFindChild(name);
182 if (virtualFile != null) {
183 FileEditorManager.getInstance(project).openFile(virtualFile, true);
184 if (!taskFile.getAnswerPlaceholders().isEmpty()) {
185 activeVirtualFile = virtualFile;
189 if (activeVirtualFile != null) {
190 VirtualFile finalActiveVirtualFile = activeVirtualFile;
191 StartupManager.getInstance(project).registerPostStartupActivity(() -> {
192 final PsiFile file = PsiManager.getInstance(project).findFile(finalActiveVirtualFile);
193 ProjectView.getInstance(project).select(file, finalActiveVirtualFile, false);
194 final FileEditor[] editors = FileEditorManager.getInstance(project).getEditors(finalActiveVirtualFile);
195 if (editors.length == 0) {
198 final FileEditor studyEditor = editors[0];
199 if (studyEditor instanceof StudyEditor) {
200 StudyUtils.selectFirstAnswerPlaceholder((StudyEditor)studyEditor, project);
203 FileEditorManager.getInstance(project).openFile(finalActiveVirtualFile, true);
206 String first = StudyUtils.getFirst(taskFiles.keySet());
208 NewVirtualFile firstFile = ((VirtualDirectoryImpl)taskDir).refreshAndFindChild(first);
209 if (firstFile != null) {
210 FileEditorManager.getInstance(project).openFile(firstFile, true);
216 public static void flushCourse(@NotNull final Project project, @NotNull final Course course) {
217 final File courseDirectory = StudyUtils.getCourseDirectory(project, course);
218 FileUtil.createDirectory(courseDirectory);
219 flushCourseJson(course, courseDirectory);
222 for (Lesson lesson : course.getLessons()) {
223 if (lesson.getName().equals(EduNames.PYCHARM_ADDITIONAL)) {
224 flushAdditionalFiles(courseDirectory, lesson);
227 final File lessonDirectory = new File(courseDirectory, EduNames.LESSON + String.valueOf(lessonIndex));
228 flushLesson(lessonDirectory, lesson);
234 private static void flushAdditionalFiles(File courseDirectory, Lesson lesson) {
235 final List<Task> taskList = lesson.getTaskList();
236 if (taskList.size() != 1) return;
237 final Task task = taskList.get(0);
238 for (Map.Entry<String, String> entry : task.getTestsText().entrySet()) {
239 final String name = entry.getKey();
240 final String text = entry.getValue();
241 final File file = new File(courseDirectory, name);
242 FileUtil.createIfDoesntExist(file);
244 if (EduUtils.isImage(name)) {
245 FileUtil.writeToFile(file, Base64.decodeBase64(text));
248 FileUtil.writeToFile(file, text);
251 catch (IOException e) {
252 LOG.error("ERROR copying file " + name);
257 public static void flushLesson(@NotNull final File lessonDirectory, @NotNull final Lesson lesson) {
258 FileUtil.createDirectory(lessonDirectory);
260 for (Task task : lesson.taskList) {
261 final File taskDirectory = new File(lessonDirectory, EduNames.TASK + String.valueOf(taskIndex));
262 flushTask(task, taskDirectory);
267 public static void flushTask(@NotNull final Task task, @NotNull final File taskDirectory) {
268 FileUtil.createDirectory(taskDirectory);
269 for (Map.Entry<String, TaskFile> taskFileEntry : task.taskFiles.entrySet()) {
270 final String name = taskFileEntry.getKey();
271 final TaskFile taskFile = taskFileEntry.getValue();
272 final File file = new File(taskDirectory, name);
273 FileUtil.createIfDoesntExist(file);
276 if (EduUtils.isImage(taskFile.name)) {
277 FileUtil.writeToFile(file, Base64.decodeBase64(taskFile.text));
280 FileUtil.writeToFile(file, taskFile.text);
283 catch (IOException e) {
284 LOG.error("ERROR copying file " + name);
287 final Map<String, String> testsText = task.getTestsText();
288 for (Map.Entry<String, String> entry : testsText.entrySet()) {
289 final File testsFile = new File(taskDirectory, entry.getKey());
290 if (testsFile.exists()) {
291 FileUtil.delete(testsFile);
293 FileUtil.createIfDoesntExist(testsFile);
295 FileUtil.writeToFile(testsFile, entry.getValue());
297 catch (IOException e) {
298 LOG.error("ERROR copying tests file");
301 final File taskText = new File(taskDirectory, "task.html");
302 FileUtil.createIfDoesntExist(taskText);
304 FileUtil.writeToFile(taskText, task.getText());
306 catch (IOException e) {
307 LOG.error("ERROR copying tests file");
311 public static void flushCourseJson(@NotNull final Course course, @NotNull final File courseDirectory) {
312 final Gson gson = new GsonBuilder().setPrettyPrinting().
313 excludeFieldsWithoutExposeAnnotation().create();
314 final String json = gson.toJson(course);
315 final File courseJson = new File(courseDirectory, EduNames.COURSE_META_FILE);
316 final FileOutputStream fileOutputStream;
318 fileOutputStream = new FileOutputStream(courseJson);
319 OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream, "UTF-8");
321 outputStreamWriter.write(json);
323 catch (IOException e) {
324 Messages.showErrorDialog(e.getMessage(), "Failed to Generate Json");
329 outputStreamWriter.close();
331 catch (IOException e) {
336 catch (FileNotFoundException | UnsupportedEncodingException e) {
342 * Writes courses to cache file {@link StudyProjectGenerator#CACHE_NAME}
344 @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
345 public static void flushCache(List<CourseInfo> courses) {
346 flushCache(courses, true);
349 public static void flushCache(List<CourseInfo> courses, boolean preserveOld) {
350 File cacheFile = new File(OUR_COURSES_DIR, CACHE_NAME);
351 PrintWriter writer = null;
353 if (!createCacheFile(cacheFile)) return;
354 Gson gson = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).create();
356 final Set<CourseInfo> courseInfos = new HashSet<>();
357 courseInfos.addAll(courses);
359 courseInfos.addAll(getCoursesFromCache());
362 writer = new PrintWriter(cacheFile, "UTF-8");
364 for (CourseInfo courseInfo : courseInfos) {
365 final String json = gson.toJson(courseInfo);
366 writer.println(json);
370 StudyUtils.closeSilently(writer);
373 catch (IOException e) {
377 StudyUtils.closeSilently(writer);
381 private static boolean createCacheFile(File cacheFile) throws IOException {
382 if (!OUR_COURSES_DIR.exists()) {
383 final boolean created = OUR_COURSES_DIR.mkdirs();
385 LOG.error("Cannot flush courses cache. Can't create courses directory");
389 if (!cacheFile.exists()) {
390 final boolean created = cacheFile.createNewFile();
392 LOG.error("Cannot flush courses cache. Can't create " + CACHE_NAME + " file");
399 // Supposed to be called under progress
400 public List<CourseInfo> getCourses(boolean force) {
401 if (OUR_COURSES_DIR.exists()) {
402 myCourses = getCoursesFromCache();
404 if (force || myCourses.isEmpty()) {
405 myCourses = execCancelable(EduStepicConnector::getCourses);
406 flushCache(myCourses);
408 if (myCourses.isEmpty()) {
409 myCourses = getBundledIntro();
411 StudyUtils.sortCourses(myCourses);
416 public List<CourseInfo> getCoursesUnderProgress(boolean force, @NotNull final String progressTitle, @NotNull final Project project) {
418 return ProgressManager.getInstance()
419 .runProcessWithProgressSynchronously(() -> {
420 ProgressManager.getInstance().getProgressIndicator().setIndeterminate(true);
421 return getCourses(force);
422 }, progressTitle, true, project);
424 catch (RuntimeException e) {
425 return Collections.singletonList(CourseInfo.INVALID_COURSE);
429 public void addSettingsStateListener(@NotNull SettingsListener listener) {
430 myListeners.add(listener);
433 public interface SettingsListener {
434 void stateChanged(ValidationResult result);
437 public void fireStateChanged(ValidationResult result) {
438 for (SettingsListener listener : myListeners) {
439 listener.stateChanged(result);
443 public static List<CourseInfo> getBundledIntro() {
444 final File introCourse = new File(OUR_COURSES_DIR, "Introduction to Python");
445 if (introCourse.exists()) {
446 final CourseInfo courseInfo = getCourseInfo(introCourse);
448 return Collections.singletonList(courseInfo);
450 return Collections.emptyList();
453 public static List<CourseInfo> getCoursesFromCache() {
454 List<CourseInfo> courses = new ArrayList<>();
455 final File cacheFile = new File(OUR_COURSES_DIR, CACHE_NAME);
456 if (!cacheFile.exists()) {
460 final FileInputStream inputStream = new FileInputStream(cacheFile);
462 BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));
465 while ((line = reader.readLine()) != null) {
466 Gson gson = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).create();
467 final CourseInfo courseInfo = gson.fromJson(line, CourseInfo.class);
468 courses.add(courseInfo);
471 catch (IOException | JsonSyntaxException e) {
472 LOG.error(e.getMessage());
475 StudyUtils.closeSilently(reader);
478 catch (UnsupportedEncodingException e) {
479 LOG.error(e.getMessage());
482 StudyUtils.closeSilently(inputStream);
485 catch (FileNotFoundException e) {
486 LOG.error(e.getMessage());
492 * Adds course from zip archive to courses
494 * @return added course name or null if course is invalid
497 public CourseInfo addLocalCourse(String zipFilePath) {
498 File file = new File(zipFilePath);
500 String fileName = file.getName();
501 String unzippedName = fileName.substring(0, fileName.indexOf("."));
502 File courseDir = new File(OUR_COURSES_DIR, unzippedName);
503 ZipUtil.unzip(null, courseDir, file, null, null, true);
504 CourseInfo courseName = addCourse(myCourses, courseDir);
505 flushCache(myCourses);
506 if (courseName != null && !courseName.getName().equals(unzippedName)) {
507 //noinspection ResultOfMethodCallIgnored
508 File dest = new File(OUR_COURSES_DIR, courseName.getName());
510 FileUtil.delete(dest);
512 courseDir.renameTo(dest);
513 //noinspection ResultOfMethodCallIgnored
518 catch (IOException e) {
519 LOG.error("Failed to unzip course archive");
526 * Adds course to courses specified in params
529 * @param courseDir must be directory containing course file
530 * @return added course name or null if course is invalid
533 private static CourseInfo addCourse(List<CourseInfo> courses, File courseDir) {
534 if (courseDir.isDirectory()) {
535 File[] courseFiles = courseDir.listFiles((dir, name) -> name.equals(EduNames.COURSE_META_FILE));
536 if (courseFiles == null || courseFiles.length != 1) {
537 LOG.info("User tried to add course with more than one or without course files");
540 File courseFile = courseFiles[0];
541 CourseInfo courseInfo = getCourseInfo(courseFile);
542 if (courseInfo != null) {
543 courses.add(courseInfo);
551 * Parses course json meta file and finds course name
553 * @return information about course or null if course file is invalid
556 private static CourseInfo getCourseInfo(File courseFile) {
557 if (courseFile.isDirectory()) {
558 File[] courseFiles = courseFile.listFiles((dir, name) -> name.equals(EduNames.COURSE_META_FILE));
559 if (courseFiles == null || courseFiles.length != 1) {
560 LOG.info("More than one or without course files");
563 courseFile = courseFiles[0];
565 CourseInfo courseInfo = null;
566 BufferedReader reader = null;
568 if (courseFile.getName().equals(EduNames.COURSE_META_FILE)) {
569 reader = new BufferedReader(new InputStreamReader(new FileInputStream(courseFile), "UTF-8"));
570 JsonReader r = new JsonReader(reader);
571 JsonParser parser = new JsonParser();
572 JsonElement el = parser.parse(r);
573 String courseName = el.getAsJsonObject().get(COURSE_NAME_ATTRIBUTE).getAsString();
574 String courseDescription = el.getAsJsonObject().get(COURSE_DESCRIPTION).getAsString();
575 JsonArray courseAuthors = el.getAsJsonObject().get(AUTHOR_ATTRIBUTE).getAsJsonArray();
576 String language = el.getAsJsonObject().get(LANGUAGE_ATTRIBUTE).getAsString();
577 courseInfo = new CourseInfo();
578 courseInfo.setName(courseName);
579 courseInfo.setDescription(courseDescription);
580 courseInfo.setType("pycharm " + language);
581 final ArrayList<StepicUser> authors = new ArrayList<>();
582 for (JsonElement author : courseAuthors) {
583 final JsonObject authorAsJsonObject = author.getAsJsonObject();
584 final StepicUser stepicUser = new StepicUser();
585 stepicUser.setFirstName(authorAsJsonObject.get("first_name").getAsString());
586 stepicUser.setLastName(authorAsJsonObject.get("last_name").getAsString());
587 authors.add(stepicUser);
589 courseInfo.setAuthors(authors);
592 catch (Exception e) {
593 //error will be shown in UI
596 StudyUtils.closeSilently(reader);