Merge pull request #2 from JetBrains/collect-personalization-data
authorVitaliy Bibaev <roenke54@gmail.com>
Thu, 14 Dec 2017 18:17:50 +0000 (21:17 +0300)
committerGitHub <noreply@github.com>
Thu, 14 Dec 2017 18:17:50 +0000 (21:17 +0300)
Collect long personalization data

45 files changed:
plugins/stats-collector/log-events/src/main/kotlin/com/intellij/stats/completion/events/CompletionStartedEvent.kt
plugins/stats-collector/log-events/src/test/kotlin/com/intellij/stats/completion/EventSerializeDeserializeTest.kt
plugins/stats-collector/log-events/src/test/kotlin/com/intellij/stats/completion/LogEventFixtures.kt
plugins/stats-collector/resources/META-INF/plugin.xml
plugins/stats-collector/src/com/intellij/completion/FeatureManagerImpl.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/sorting/MLCompletionSorter.kt
plugins/stats-collector/src/com/intellij/sorting/MLSorter.kt
plugins/stats-collector/src/com/intellij/stats/completion/CompletionLoggerImpl.kt
plugins/stats-collector/src/com/intellij/stats/completion/CompletionTrackerInitializer.kt
plugins/stats-collector/src/com/intellij/stats/completion/LookupCompletedTracker.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/completion/LookupStartedTracker.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/completion/TimeBetweenTypingTracker.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/DateUtil.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/Day.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/FactorReader.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/FactorUpdater.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/UserFactor.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorAccessors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorBase.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorDescription.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorDescriptions.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorStorage.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorsManager.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/ApplicationUserFactorStorage.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/BinaryFeatureFactors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/CategorialFeatureFactors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/CompletionFinishTypeFactors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/CompletionTypeFactors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/CompletionUsageFactors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/DailyAggregatedDoubleFactor.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/DayImpl.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/DoubleFeatureFactors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/FactorsUtil.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/ItemPositionFactors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/MnemonicsUsageFactors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/PrefixLengthFactor.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/ProjectUserFactorStorage.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/TimeBetweenTypingFactors.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/UserFactorStorageBase.kt [new file with mode: 0644]
plugins/stats-collector/src/com/intellij/stats/personalization/impl/UserFactorsManagerImpl.kt [new file with mode: 0644]
plugins/stats-collector/test/com/intellij/mocks/SortingMocks.kt
plugins/stats-collector/test/com/intellij/sorting/CompletionOrderWithFakeRankerTest.kt
plugins/stats-collector/test/com/intellij/sorting/UpdateExperimentStatusTest.kt
plugins/stats-collector/test/com/intellij/sorting/Utils.kt
plugins/stats-collector/test/com/intellij/stats/personalization/AggregatedFactorTest.kt [new file with mode: 0644]

index 78b9d0e008e61afc7d8161b75477aa913ce226c0..8df293d4d29b525a86bd1fc5cb658b87e851eae5 100644 (file)
@@ -31,6 +31,7 @@ class CompletionStartedEvent(
         @JvmField var performExperiment: Boolean,
         @JvmField var experimentVersion: Int,
         completionList: List<LookupEntryInfo>,
+        @JvmField var userFactors: Map<String, String?>,
         selectedPosition: Int)
 
     : LookupStateLogData(
index ac89d506624c4563457c23a276113c097c97be74..a1c383e20660b09c7303f775da37973dcf70ad5c 100644 (file)
@@ -35,6 +35,12 @@ object Fixtures {
             LookupEntryInfo(2, 7, relevance)
     )
 
+    val userFactors: Map<String, String> = mapOf(
+            "avgTimeToType" to "0.6",
+            "maxSelecterItem" to "10",
+            "explicitSelectCountToday" to "100"
+    )
+
     val history = mapOf(10 to ElementPositionHistory(listOf(StagePosition(0, 1))))
     
 }
@@ -49,7 +55,9 @@ class EventSerializeDeserializeTest {
 
     @Test
     fun `completion started event`() {
-        val event = CompletionStartedEvent("", "", "", Fixtures.userId, "xx", "Java", true, 1, Fixtures.lookupList, 0)
+        val event = CompletionStartedEvent("", "", "", Fixtures.userId,
+                "xx", "Java", true, 1, Fixtures.lookupList,
+                Fixtures.userFactors, 0)
         serializeDeserializeAndCheck(event)
     }
 
index ce8b7a4c2e5352ea19e3b476a2c431f0883ddc62..bdc56a1dedc1965624e3af35dd1206662548329b 100644 (file)
@@ -22,7 +22,9 @@ object LogEventFixtures {
     
     val sessionId = "session-id-xxx"
 
-    val completion_started_3_items_shown = CompletionStartedEvent("", "", "", "1", sessionId, "Java", true, 1, Fixtures.lookupList, 0)
+    val completion_started_3_items_shown = CompletionStartedEvent("", "", "",
+            "1", sessionId, "Java", true, 1, Fixtures.lookupList,
+            Fixtures.userFactors, 0)
 
     val completion_cancelled = CompletionCancelledEvent("1", sessionId)
 
index 7d142fa54799eef19097c8e08fe7d5def164ec8a..4c58170d6c414924ee0fa6dfab900ad9b672b83d 100644 (file)
@@ -1,7 +1,7 @@
 <idea-plugin>
   <id>com.intellij.stats.completion</id>
   <name>Completion Stats Collector</name>
-  <version>0.0.537</version>
+  <version>0.0.538</version>
   <vendor email="vitaliy.bibaev@jetbrains.com" url="http://www.jetbrains.com">JetBrains</vendor>
 
   <description><![CDATA[
     <component>
       <implementation-class>com.intellij.sorting.FeatureTransformerProvider</implementation-class>
     </component>
+
+    <component>
+      <implementation-class>
+        com.intellij.stats.personalization.impl.ApplicationUserFactorStorage
+      </implementation-class>
+    </component>
+
+    <component>
+      <implementation-class>com.intellij.completion.FeatureManagerImpl</implementation-class>
+      <interface-class>com.jetbrains.completion.ranker.features.FeatureManager</interface-class>
+    </component>
   </application-components>
 
+  <project-components>
+    <component>
+      <implementation-class>com.intellij.stats.personalization.impl.ProjectUserFactorStorage</implementation-class>
+    </component>
+
+    <component>
+      <interface-class>com.intellij.stats.personalization.UserFactorsManager</interface-class>
+      <implementation-class>com.intellij.stats.personalization.impl.UserFactorsManagerImpl</implementation-class>
+    </component>
+  </project-components>
+
 </idea-plugin>
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/completion/FeatureManagerImpl.kt b/plugins/stats-collector/src/com/intellij/completion/FeatureManagerImpl.kt
new file mode 100644 (file)
index 0000000..21b6fd6
--- /dev/null
@@ -0,0 +1,38 @@
+package com.intellij.completion
+
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.components.ApplicationComponent
+import com.jetbrains.completion.ranker.features.*
+import com.jetbrains.completion.ranker.features.impl.FeatureInterpreterImpl
+import com.jetbrains.completion.ranker.features.impl.FeatureManagerFactory
+import com.jetbrains.completion.ranker.features.impl.FeatureReader
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class FeatureManagerImpl : FeatureManager, ApplicationComponent {
+    companion object {
+        fun getInstance(): FeatureManager = ApplicationManager.getApplication().getComponent(FeatureManager::class.java)
+    }
+
+    private lateinit var manager: FeatureManager
+
+    override val binaryFactors: List<BinaryFeature> get() = manager.binaryFactors
+    override val doubleFactors: List<DoubleFeature> get() = manager.doubleFactors
+    override val categorialFactors: List<CatergorialFeature> get() = manager.categorialFactors
+    override val ignoredFactors: Set<String> get() = manager.ignoredFactors
+    override val completionFactors: CompletionFactors get() = manager.completionFactors
+    override val featureOrder: Map<String, Int> get() = manager.featureOrder
+
+    override fun createTransformer(): Transformer {
+        return manager.createTransformer()
+    }
+
+    override fun isUserFeature(name: String): Boolean = false
+
+    override fun initComponent() {
+        manager = FeatureManagerFactory().createFeatureManager(FeatureReader, FeatureInterpreterImpl())
+    }
+
+    override fun allFeatures(): List<Feature> = manager.allFeatures()
+}
\ No newline at end of file
index 79cee105eba0d2e765ba3481d99eb2850874a732..71309ce8766077482a462b9cc35c8f8436fb97c9 100644 (file)
  */
 package com.intellij.sorting
 
-import com.intellij.ide.util.PropertiesComponent
+import com.jetbrains.completion.ranker.features.FeatureManager
 import com.intellij.openapi.components.ApplicationComponent
 import com.intellij.openapi.components.ServiceManager
 import com.jetbrains.completion.ranker.CompletionRanker
-import com.jetbrains.completion.ranker.features.LookupElementInfo
-import com.jetbrains.completion.ranker.features.FeatureReader.binaryFactors
-import com.jetbrains.completion.ranker.features.FeatureReader.categoricalFactors
-import com.jetbrains.completion.ranker.features.FeatureReader.completionFactors
-import com.jetbrains.completion.ranker.features.FeatureReader.doubleFactors
-import com.jetbrains.completion.ranker.features.FeatureReader.featuresOrder
-import com.jetbrains.completion.ranker.features.FeatureReader.ignoredFactors
-import com.jetbrains.completion.ranker.features.FeatureTransformer
-import com.jetbrains.completion.ranker.features.IgnoredFactorsMatcher
+import com.jetbrains.completion.ranker.features.Transformer
 
 
 interface Ranker {
 
     /**
      * Items are sorted by descending order, so item with the highest rank will be on top
-     * @param state
      * @param relevance map from LookupArranger.getRelevanceObjects
      */
-    fun rank(state: LookupElementInfo, relevance: Map<String, Any?>): Double?
+    fun rank(relevance: Map<String, Any?>, userFactors: Map<String, Any?>): Double?
 
     companion object {
         fun getInstance(): Ranker = ServiceManager.getService(Ranker::class.java)
@@ -45,40 +36,17 @@ interface Ranker {
 }
 
 
-class FeatureTransformerProvider: ApplicationComponent.Adapter() {
-
-    lateinit var featureTransformer: FeatureTransformer
-        private set
-    
-    override fun initComponent() {
-        val binary = binaryFactors()
-        val double = doubleFactors()
-        val categorical = categoricalFactors()
-        val factors = completionFactors()
-        val order = featuresOrder()
-        val ignored = ignoredFactors()
-        
-        featureTransformer = FeatureTransformer(
-                binary, 
-                double, 
-                categorical, 
-                order, 
-                factors,
-                IgnoredFactorsMatcher(ignored)
-        )
-    }
-    
+class FeatureTransformerProvider(featureManager: FeatureManager) : ApplicationComponent.Adapter() {
+    val featureTransformer: Transformer = featureManager.createTransformer()
 }
 
-
-
-class MLRanker(val provider: FeatureTransformerProvider): Ranker {
+class MLRanker(val provider: FeatureTransformerProvider) : Ranker {
 
     private val featureTransformer = provider.featureTransformer
     private val ranker = CompletionRanker()
-    
-    override fun rank(state: LookupElementInfo, relevance: Map<String, Any?>): Double? {
-        val featureArray = featureTransformer.featureArray(state, relevance)
+
+    override fun rank(relevance: Map<String, Any?>, userFactors: Map<String, Any?>): Double? {
+        val featureArray = featureTransformer.featureArray(relevance, userFactors)
         if (featureArray != null) {
             return ranker.rank(featureArray)
         }
index 3c4461f42196c97255c961404bc275a4c922e1d8..66bb050806e8e9a9a6a984e81af2d8e02d90c4ef 100644 (file)
@@ -28,13 +28,12 @@ import com.intellij.openapi.util.Pair
 import com.intellij.plugin.ManualExperimentControl
 import com.intellij.plugin.ManualMlSorting
 import com.intellij.psi.util.PsiUtilCore
-import com.intellij.stats.experiment.WebServiceStatus
 import com.intellij.stats.completion.prefixLength
-import com.jetbrains.completion.ranker.features.FeatureUtils
-import com.jetbrains.completion.ranker.features.LookupElementInfo
+import com.intellij.stats.experiment.WebServiceStatus
+import com.intellij.stats.personalization.UserFactorsManager
+import com.jetbrains.completion.ranker.features.impl.FeatureUtils
 import java.util.*
 
-
 @Suppress("DEPRECATION")
 class MLSorterFactory : CompletionFinalSorter.Factory {
     override fun newSorter() = MLSorter()
@@ -42,7 +41,6 @@ class MLSorterFactory : CompletionFinalSorter.Factory {
 
 
 class MLSorter : CompletionFinalSorter() {
-
     private val webServiceStatus = WebServiceStatus.getInstance()
     private val ranker = Ranker.getInstance()
     private val cachedScore: MutableMap<LookupElement, ItemRankInfo> = IdentityHashMap()
@@ -55,11 +53,11 @@ class MLSorter : CompletionFinalSorter() {
         if (hasUnknownFeatures(items)) {
             return items.associate { it to listOf(Pair.create(FeatureUtils.ML_RANK, FeatureUtils.UNDEFINED as Any)) }
         }
-        
+
         if (!isCacheValid(items)) {
             return items.associate { it to listOf(Pair.create(FeatureUtils.ML_RANK, FeatureUtils.INVALID_CACHE as Any)) }
         }
-        
+
         return items.associate {
             val result = mutableListOf<Pair<String, Any>>()
             val cached = cachedScore[it]
@@ -70,11 +68,11 @@ class MLSorter : CompletionFinalSorter() {
             it to result
         }
     }
-    
+
     private fun isCacheValid(items: Iterable<LookupElement>): Boolean {
         return items.map { cachedScore[it]?.prefixLength }.toSet().size == 1
     }
-    
+
     private fun hasUnknownFeatures(items: Iterable<LookupElement>) = items.any {
         val score = cachedScore[it]
         score?.mlRank == null
@@ -89,10 +87,10 @@ class MLSorter : CompletionFinalSorter() {
         val startTime = System.currentTimeMillis()
         val sorted = sortByMLRanking(items, lookup, relevanceObjects) ?: return items
         val timeSpent = System.currentTimeMillis() - startTime
-        
+
         val elementsSorted = items.count()
         SortingTimeStatistics.registerSortTiming(elementsSorted, timeSpent)
-        
+
         return sorted
     }
 
@@ -120,13 +118,13 @@ class MLSorter : CompletionFinalSorter() {
      */
     private fun sortByMLRanking(items: MutableIterable<LookupElement>,
                                 lookup: LookupImpl,
-                                relevanceObjects: Map<LookupElement, List<Pair<String, Any?>>>): Iterable<LookupElement>?
-    {
+                                relevanceObjects: Map<LookupElement, List<Pair<String, Any?>>>): Iterable<LookupElement>? {
         val prefixLength = lookup.prefixLength()
+        val userFactors = lookup.getUserData(UserFactorsManager.USER_FACTORS_KEY) ?: emptyMap()
         return items
                 .mapIndexed { index, lookupElement ->
                     val relevance = relevanceObjects[lookupElement] ?: emptyList()
-                    val rank: Double = calculateElementRank(lookupElement, index, relevance, prefixLength) ?: return null
+                    val rank: Double = calculateElementRank(lookupElement, index, relevance, userFactors, prefixLength) ?: return null
                     lookupElement to rank
                 }
                 .sortedByDescending { it.second }
@@ -141,12 +139,11 @@ class MLSorter : CompletionFinalSorter() {
         return null
     }
 
-
     private fun calculateElementRank(element: LookupElement,
                                      position: Int,
                                      relevance: List<Pair<String, Any?>>,
-                                     prefixLength: Int): Double? 
-    {
+                                     userFactors: Map<String, Any?>,
+                                     prefixLength: Int): Double? {
         val cachedWeight = getCachedRankInfo(element, prefixLength, position)
         if (cachedWeight != null) {
             return cachedWeight.mlRank
@@ -154,26 +151,24 @@ class MLSorter : CompletionFinalSorter() {
 
         val elementLength = element.lookupString.length
 
-        val state = LookupElementInfo(position, query_length = prefixLength, result_length = elementLength)
+        val relevanceMap = mutableMapOf<String, Any?>()
+        relevance.forEach { p -> relevanceMap.put(p.first, p.second) }
 
-        val relevanceMap = relevance.associate { it.first to it.second }
-        val mlRank: Double? = ranker.rank(state, relevanceMap)
+        relevanceMap.put("position", position)
+        relevanceMap.put("query_length", prefixLength)
+        relevanceMap.put("result_length", elementLength)
+
+        val mlRank: Double? = ranker.rank(relevanceMap, userFactors)
         val info = ItemRankInfo(position, mlRank, prefixLength)
         cachedScore[element] = info
 
         return info.mlRank
     }
-
-
 }
 
-
 private data class ItemRankInfo(val positionBefore: Int, val mlRank: Double?, val prefixLength: Int)
 
-
-typealias WeightedElement = Pair<LookupElement, Double>
-
 fun CompletionParameters.language(): Language? {
     val offset = editor.caretModel.offset
-    return  PsiUtilCore.getLanguageAtOffset(originalFile, offset)
+    return PsiUtilCore.getLanguageAtOffset(originalFile, offset)
 }
\ No newline at end of file
index 74826ff02124549092e6653047bfc3d81278bafb..f8dfa4d8cc411d5c275b51b1a5bdcc9a6ce7da07 100644 (file)
@@ -23,6 +23,7 @@ import com.intellij.codeInsight.lookup.impl.LookupImpl
 import com.intellij.completion.tracker.LookupElementPositionTracker
 import com.intellij.ide.plugins.PluginManager
 import com.intellij.stats.completion.events.*
+import com.intellij.stats.personalization.UserFactorsManager
 
 
 class CompletionFileLogger(private val installationUID: String,
@@ -78,12 +79,14 @@ class CompletionFileLogger(private val installationUID: String,
         val pluginVersion = calcPluginVersion() ?: "pluginVersion"
         val mlRankingVersion = "NONE"
 
+        val userFactors = lookup.getUserData(UserFactorsManager.USER_FACTORS_KEY) ?: emptyMap()
+
         val event = CompletionStartedEvent(
                 ideVersion, pluginVersion, mlRankingVersion,
                 installationUID, completionUID,
                 language?.displayName,
                 isExperimentPerformed, experimentVersion,
-                lookupEntryInfos, selectedPosition = 0)
+                lookupEntryInfos, userFactors, selectedPosition = 0)
 
         event.isOneLineMode = lookup.editor.isOneLineMode
         event.fillCompletionParameters()
index 20715954970725e35e678f95040eef5183073bb4..04820e41c92ddab8c2458d3b63bfe75f7c7a9c79 100644 (file)
@@ -27,6 +27,9 @@ import com.intellij.openapi.project.ProjectManagerListener
 import com.intellij.stats.sender.isSendAllowed
 import com.intellij.stats.sender.isUnitTestMode
 import com.intellij.stats.experiment.WebServiceStatus
+import com.intellij.stats.personalization.UserFactorDescriptions
+import com.intellij.stats.personalization.UserFactorStorage
+import com.intellij.stats.personalization.UserFactorsManager
 import java.beans.PropertyChangeListener
 
 
@@ -36,7 +39,7 @@ class CompletionTrackerInitializer(experimentHelper: WebServiceStatus): Applicat
     }
 
     private val actionListener = LookupActionsListener()
-    
+
     private val lookupTrackerInitializer = PropertyChangeListener {
         val lookup = it.newValue
         if (lookup == null) {
@@ -45,13 +48,31 @@ class CompletionTrackerInitializer(experimentHelper: WebServiceStatus): Applicat
         else if (lookup is LookupImpl) {
             if (isUnitTestMode() && !isEnabledInTests) return@PropertyChangeListener
 
+            val globalStorage = UserFactorStorage.getInstance()
+            val projectStorage = UserFactorStorage.getInstance(lookup.project)
+
+            val userFactors = UserFactorsManager.getInstance(lookup.project).getAllFactors()
+            val userFactorValues = mutableMapOf<String, String?>()
+            userFactors.asSequence().map { "${it.id}Global" to it.compute(globalStorage) }.toMap(userFactorValues)
+            userFactors.asSequence().map { "${it.id}Project" to it.compute(projectStorage) }.toMap(userFactorValues)
+
+            lookup.putUserData(UserFactorsManager.USER_FACTORS_KEY, userFactorValues)
             val shownTimesTracker = PositionTrackingListener(lookup)
             lookup.setPrefixChangeListener(shownTimesTracker)
 
+            UserFactorStorage.applyOnBoth(lookup.project, UserFactorDescriptions.COMPLETION_USAGE) {
+                it.fireCompletionUsed()
+            }
+
             val tracker = actionsTracker(lookup, experimentHelper)
             actionListener.listener = tracker
             lookup.addLookupListener(tracker)
             lookup.setPrefixChangeListener(tracker)
+
+            // setPrefixChangeListener has addPrefixChangeListener semantics
+            lookup.setPrefixChangeListener(TimeBetweenTypingTracker(lookup.project))
+            lookup.addLookupListener(LookupCompletedTracker())
+            lookup.addLookupListener(LookupStartedTracker())
         }
     }
 
diff --git a/plugins/stats-collector/src/com/intellij/stats/completion/LookupCompletedTracker.kt b/plugins/stats-collector/src/com/intellij/stats/completion/LookupCompletedTracker.kt
new file mode 100644 (file)
index 0000000..514e265
--- /dev/null
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intellij.stats.completion
+
+import com.intellij.codeInsight.lookup.LookupAdapter
+import com.intellij.codeInsight.lookup.LookupElement
+import com.intellij.codeInsight.lookup.LookupEvent
+import com.intellij.codeInsight.lookup.impl.LookupImpl
+import com.intellij.completion.FeatureManagerImpl
+import com.intellij.stats.personalization.UserFactorDescriptions
+import com.intellij.stats.personalization.UserFactorStorage
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class LookupCompletedTracker : LookupAdapter() {
+    override fun lookupCanceled(event: LookupEvent?) {
+        val lookup = event?.lookup as? LookupImpl ?: return
+        val element = lookup.currentItem ?: return
+        if (isSelectedByTyping(lookup, element)) {
+            processTypedSelect(lookup, element)
+        }
+    }
+
+    override fun itemSelected(event: LookupEvent?) {
+        val lookup = event?.lookup as? LookupImpl ?: return
+        val element = event.item ?: return
+        processExplicitSelect(lookup, element)
+    }
+
+    private fun isSelectedByTyping(lookup: LookupImpl, element: LookupElement): Boolean =
+            element.lookupString == lookup.itemPattern(element)
+
+    private fun processElementSelected(lookup: LookupImpl, element: LookupElement) {
+        val relevanceObjects =
+                lookup.getRelevanceObjects(listOf(element), false)
+        val relevanceMap = relevanceObjects[element]!!.associate { it.first to it.second }
+        val project = lookup.project
+        val featureManager = FeatureManagerImpl.getInstance()
+        featureManager.binaryFactors.filter { !featureManager.isUserFeature(it.name) }.forEach { feature ->
+            UserFactorStorage.applyOnBoth(project, UserFactorDescriptions.binaryFeatureDescriptor(feature))
+            { updater ->
+                updater.update(relevanceMap[feature.name])
+            }
+        }
+
+        featureManager.doubleFactors.filter { !featureManager.isUserFeature(it.name) }.forEach { feature ->
+            UserFactorStorage.applyOnBoth(project, UserFactorDescriptions.doubleFeatureDescriptor(feature))
+            { updater ->
+                updater.update(relevanceMap[feature.name])
+            }
+        }
+
+        featureManager.categorialFactors.filter { !featureManager.isUserFeature(it.name) }.forEach { feature ->
+            UserFactorStorage.applyOnBoth(project, UserFactorDescriptions.categoriealFeatureDescriptor(feature))
+            { updater ->
+                updater.update(relevanceMap[feature.name])
+            }
+        }
+    }
+
+    private fun processExplicitSelect(lookup: LookupImpl, element: LookupElement) {
+        processElementSelected(lookup, element)
+
+        UserFactorStorage.applyOnBoth(lookup.project, UserFactorDescriptions.COMPLETION_FINISH_TYPE) { updater ->
+            updater.fireExplicitCompletionPerformed()
+        }
+
+        val prefixLength = lookup.getPrefixLength(element)
+        UserFactorStorage.applyOnBoth(lookup.project, UserFactorDescriptions.PREFIX_LENGTH_ON_COMPLETION) { updater ->
+            updater.fireCompletionPerformed(prefixLength)
+        }
+
+        val itemPosition = lookup.selectedIndex
+        if (itemPosition != -1) {
+            UserFactorStorage.applyOnBoth(lookup.project, UserFactorDescriptions.SELECTED_ITEM_POSITION) { updater ->
+                updater.fireCompletionPerformed(itemPosition)
+            }
+        }
+
+        if (prefixLength > 1) {
+            val pattern = lookup.itemPattern(element)
+            val isMmemonicsUsed = !element.lookupString.startsWith(pattern)
+            UserFactorStorage.applyOnBoth(lookup.project, UserFactorDescriptions.MNEMONICS_USAGE) { updater ->
+                updater.fireCompletionFinished(isMmemonicsUsed)
+            }
+        }
+    }
+
+    private fun processTypedSelect(lookup: LookupImpl, element: LookupElement) {
+        processElementSelected(lookup, element)
+
+        UserFactorStorage.applyOnBoth(lookup.project, UserFactorDescriptions.COMPLETION_FINISH_TYPE) { updater ->
+            updater.fireTypedSelectPerformed()
+        }
+    }
+}
diff --git a/plugins/stats-collector/src/com/intellij/stats/completion/LookupStartedTracker.kt b/plugins/stats-collector/src/com/intellij/stats/completion/LookupStartedTracker.kt
new file mode 100644 (file)
index 0000000..3ebf9f3
--- /dev/null
@@ -0,0 +1,32 @@
+package com.intellij.stats.completion
+
+import com.intellij.codeInsight.completion.CompletionProgressIndicator
+import com.intellij.codeInsight.completion.CompletionService
+import com.intellij.codeInsight.lookup.Lookup
+import com.intellij.codeInsight.lookup.LookupAdapter
+import com.intellij.codeInsight.lookup.LookupEvent
+import com.intellij.stats.personalization.UserFactorDescriptions
+import com.intellij.stats.personalization.UserFactorStorage
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class LookupStartedTracker : LookupAdapter() {
+    override fun currentItemChanged(event: LookupEvent?) {
+        val lookup = event?.lookup ?: return
+        if (processLookupStarted(lookup)) lookup.removeLookupListener(this)
+    }
+
+    private fun processLookupStarted(lookup: Lookup): Boolean {
+        // todo[bibaev]: avoid usage of deprecated methods
+        @Suppress("DEPRECATION")
+        val completionType =
+                (CompletionService.getCompletionService().currentCompletion as? CompletionProgressIndicator)
+                        ?.parameters?.completionType ?: return false
+        UserFactorStorage.applyOnBoth(lookup.project, UserFactorDescriptions.COMPLETION_TYPE) {
+            it.fireCompletionPerformed(completionType)
+        }
+
+        return true
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/completion/TimeBetweenTypingTracker.kt b/plugins/stats-collector/src/com/intellij/stats/completion/TimeBetweenTypingTracker.kt
new file mode 100644 (file)
index 0000000..17939c3
--- /dev/null
@@ -0,0 +1,37 @@
+package com.intellij.stats.completion
+
+import com.intellij.codeInsight.lookup.impl.PrefixChangeListener
+import com.intellij.openapi.project.Project
+import com.intellij.stats.personalization.UserFactorDescriptions
+import com.intellij.stats.personalization.UserFactorStorage
+import java.util.concurrent.TimeUnit
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class TimeBetweenTypingTracker(private val project: Project) : PrefixChangeListener {
+    private companion object {
+        val MAX_ALLOWED_DELAY = TimeUnit.SECONDS.toMillis(10)
+    }
+
+    private var lastTypingTime: Long = -1L
+
+    override fun beforeAppend(c: Char) = prefixChanged()
+    override fun beforeTruncate() = prefixChanged()
+
+    private fun prefixChanged() {
+        if (lastTypingTime == -1L) {
+            lastTypingTime = System.currentTimeMillis()
+            return
+        }
+
+        val currentTime = System.currentTimeMillis()
+        val delay = currentTime - lastTypingTime
+        if (delay > MAX_ALLOWED_DELAY) return
+        UserFactorStorage.applyOnBoth(project, UserFactorDescriptions.TIME_BETWEEN_TYPING) { updater ->
+            updater.fireTypingPerformed(delay.toInt())
+        }
+
+        lastTypingTime = currentTime
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/DateUtil.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/DateUtil.kt
new file mode 100644 (file)
index 0000000..060a4d0
--- /dev/null
@@ -0,0 +1,13 @@
+package com.intellij.stats.personalization
+
+import com.intellij.stats.personalization.impl.DayImpl
+import java.util.*
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+object DateUtil {
+    fun today(): Day = byDate(Date())
+
+    fun byDate(date: Date): Day = DayImpl(date)
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/Day.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/Day.kt
new file mode 100644 (file)
index 0000000..90c5e24
--- /dev/null
@@ -0,0 +1,10 @@
+package com.intellij.stats.personalization
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface Day : Comparable<Day> {
+    val dayOfMonth: Int
+    val month: Int
+    val year: Int
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/FactorReader.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/FactorReader.kt
new file mode 100644 (file)
index 0000000..71b57d5
--- /dev/null
@@ -0,0 +1,7 @@
+package com.intellij.stats.personalization
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface FactorReader {
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/FactorUpdater.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/FactorUpdater.kt
new file mode 100644 (file)
index 0000000..f7685f2
--- /dev/null
@@ -0,0 +1,7 @@
+package com.intellij.stats.personalization
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface FactorUpdater {
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactor.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactor.kt
new file mode 100644 (file)
index 0000000..8bc4aae
--- /dev/null
@@ -0,0 +1,12 @@
+package com.intellij.stats.personalization
+
+import com.intellij.openapi.project.Project
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface UserFactor {
+    val id: String
+
+    fun compute(storage: UserFactorStorage): String?
+}
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorAccessors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorAccessors.kt
new file mode 100644 (file)
index 0000000..9630f7b
--- /dev/null
@@ -0,0 +1,11 @@
+package com.intellij.stats.personalization
+
+import com.intellij.stats.personalization.impl.DailyAggregatedDoubleFactor
+import com.intellij.stats.personalization.impl.MutableDoubleFactor
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+abstract class UserFactorReaderBase(protected val factor: DailyAggregatedDoubleFactor) : FactorReader
+
+abstract class UserFactorUpdaterBase(protected val factor: MutableDoubleFactor) : FactorUpdater
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorBase.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorBase.kt
new file mode 100644 (file)
index 0000000..9893bc5
--- /dev/null
@@ -0,0 +1,12 @@
+package com.intellij.stats.personalization
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+abstract class UserFactorBase<in R : FactorReader>(override val id: String, private val descriptor: UserFactorDescription<*, R>) : UserFactor {
+    override final fun compute(storage: UserFactorStorage): String? {
+        return compute(storage.getFactorReader(descriptor))
+    }
+
+    abstract fun compute(reader: R): String?
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorDescription.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorDescription.kt
new file mode 100644 (file)
index 0000000..63f28d7
--- /dev/null
@@ -0,0 +1,14 @@
+package com.intellij.stats.personalization
+
+import com.intellij.stats.personalization.impl.DailyAggregatedDoubleFactor
+import com.intellij.stats.personalization.impl.MutableDoubleFactor
+import com.intellij.stats.personalization.impl.UserFactorStorageBase
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface UserFactorDescription<out U : FactorUpdater, out R : FactorReader> {
+    val factorId: String
+    val updaterFactory: (MutableDoubleFactor) -> U
+    val readerFactory: (DailyAggregatedDoubleFactor) -> R
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorDescriptions.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorDescriptions.kt
new file mode 100644 (file)
index 0000000..d66aa09
--- /dev/null
@@ -0,0 +1,39 @@
+package com.intellij.stats.personalization
+
+import com.intellij.stats.personalization.impl.*
+import com.jetbrains.completion.ranker.features.BinaryFeature
+import com.jetbrains.completion.ranker.features.CatergorialFeature
+import com.jetbrains.completion.ranker.features.DoubleFeature
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+object UserFactorDescriptions {
+    val COMPLETION_TYPE = Descriptor("completionType", ::CompletionTypeUpdater, ::CompletionTypeReader)
+    val COMPLETION_FINISH_TYPE =
+            Descriptor("completionFinishedType", ::CompletionFinishTypeUpdater, ::CompletionFinishTypeReader)
+    val COMPLETION_USAGE = Descriptor("completionUsage", ::CompletionUsageUpdater, ::CompletionUsageReader)
+    val PREFIX_LENGTH_ON_COMPLETION = Descriptor("prefixLength", ::PrefixLengthUpdater, ::PrefixLengthReader)
+    val SELECTED_ITEM_POSITION = Descriptor("itemPosition", ::ItemPositionUpdater, ::ItemPositionReader)
+    val TIME_BETWEEN_TYPING = Descriptor("timeBetweenTyping", ::TimeBetweenTypingUpdater, ::TimeBetweenTypingReader)
+    val MNEMONICS_USAGE = Descriptor("mnemonicsUsage", ::MnemonicsUsageUpdater, ::MnemonicsUsageReader)
+
+    fun binaryFeatureDescriptor(feature: BinaryFeature): Descriptor<BinaryFeatureUpdater, BinaryFeatureReader> {
+        return Descriptor("binaryFeature:${feature.name}", ::BinaryFeatureUpdater, ::BinaryFeatureReader)
+    }
+
+    fun doubleFeatureDescriptor(feature: DoubleFeature): Descriptor<DoubleFeatureUpdater, DoubleFeatureReader> {
+        return Descriptor("doubleFeature:${feature.name}", ::DoubleFeatureUpdater, ::DoubleFeatureReader)
+    }
+
+    fun categoriealFeatureDescriptor(feature: CatergorialFeature): Descriptor<CategoryFeatureUpdater, CategoryFeatureReader> {
+        return Descriptor("categorialFeature:${feature.name}",
+                { CategoryFeatureUpdater(feature.categories, it) },
+                ::CategoryFeatureReader)
+    }
+
+    class Descriptor<out U : FactorUpdater, out R : FactorReader>(
+            override val factorId: String,
+            override val updaterFactory: (MutableDoubleFactor) -> U,
+            override val readerFactory: (DailyAggregatedDoubleFactor) -> R) : UserFactorDescription<U, R>
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorStorage.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorStorage.kt
new file mode 100644 (file)
index 0000000..a02a31c
--- /dev/null
@@ -0,0 +1,26 @@
+package com.intellij.stats.personalization
+
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.project.Project
+import com.intellij.stats.personalization.impl.ApplicationUserFactorStorage
+import com.intellij.stats.personalization.impl.ProjectUserFactorStorage
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface UserFactorStorage {
+  companion object {
+    fun getInstance(): UserFactorStorage =
+        ApplicationManager.getApplication().getComponent(ApplicationUserFactorStorage::class.java)
+
+    fun getInstance(project: Project): UserFactorStorage = project.getComponent(ProjectUserFactorStorage::class.java)
+
+    fun <U : FactorUpdater> applyOnBoth(project: Project, description: UserFactorDescription<U, *>, updater: (U) -> Unit) {
+      updater(getInstance().getFactorUpdater(description))
+      updater(getInstance(project).getFactorUpdater(description))
+    }
+  }
+
+  fun <U : FactorUpdater> getFactorUpdater(description: UserFactorDescription<U, *>): U
+  fun <R : FactorReader> getFactorReader(description: UserFactorDescription<*, R>): R
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorsManager.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorsManager.kt
new file mode 100644 (file)
index 0000000..c037313
--- /dev/null
@@ -0,0 +1,20 @@
+package com.intellij.stats.personalization
+
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.Key
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface UserFactorsManager {
+  companion object {
+    val USER_FACTORS_KEY = Key.create<Map<String, String?>>("com.intellij.stats.personalization.userFactors")
+    fun getInstance(project: Project): UserFactorsManager = project.getComponent(UserFactorsManager::class.java)
+  }
+
+  fun getAllFactorIds(): List<String>
+
+  fun getAllFactors(): List<UserFactor>
+
+  fun getFactor(id: String): UserFactor
+}
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/ApplicationUserFactorStorage.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/ApplicationUserFactorStorage.kt
new file mode 100644 (file)
index 0000000..f587a02
--- /dev/null
@@ -0,0 +1,11 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.openapi.components.ApplicationComponent
+import com.intellij.openapi.components.State
+import com.intellij.openapi.components.Storage
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+@State(name = "ApplicationUserFactors", storages = arrayOf(Storage("completion.factors.user.xml")))
+class ApplicationUserFactorStorage : ApplicationComponent, UserFactorStorageBase()
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/BinaryFeatureFactors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/BinaryFeatureFactors.kt
new file mode 100644 (file)
index 0000000..5feebf2
--- /dev/null
@@ -0,0 +1,35 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.UserFactorBase
+import com.intellij.stats.personalization.UserFactorDescriptions
+import com.intellij.stats.personalization.UserFactorReaderBase
+import com.intellij.stats.personalization.UserFactorUpdaterBase
+import com.jetbrains.completion.ranker.features.BinaryFeature
+import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class BinaryFeatureReader(factor: DailyAggregatedDoubleFactor)
+    : UserFactorReaderBase(factor) {
+    fun calculateRatioByValue(): Map<String, Double> {
+        val sums = factor.aggregateSum()
+        val total = sums.values.sum()
+        if (total == 0.0) return emptyMap()
+        return sums.mapValues { e -> e.value / total }
+    }
+}
+
+class BinaryFeatureUpdater(factor: MutableDoubleFactor) : UserFactorUpdaterBase(factor) {
+    fun update(value: Any?) {
+        factor.incrementOnToday(value?.toString() ?: FeatureUtils.UNDEFINED)
+    }
+}
+
+class BinaryValueRatio(feature: BinaryFeature, private val valueName: String)
+    : UserFactorBase<BinaryFeatureReader>("BinaryValueRatio:${feature.name}:$valueName",
+        UserFactorDescriptions.binaryFeatureDescriptor(feature)) {
+    override fun compute(reader: BinaryFeatureReader): String {
+        return reader.calculateRatioByValue().getOrDefault(valueName, -1.0).toString()
+    }
+}
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/CategorialFeatureFactors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/CategorialFeatureFactors.kt
new file mode 100644 (file)
index 0000000..6b020b5
--- /dev/null
@@ -0,0 +1,52 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.UserFactorBase
+import com.intellij.stats.personalization.UserFactorDescriptions
+import com.intellij.stats.personalization.UserFactorReaderBase
+import com.intellij.stats.personalization.UserFactorUpdaterBase
+import com.jetbrains.completion.ranker.features.CatergorialFeature
+import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class CategoryFeatureReader(factor: DailyAggregatedDoubleFactor)
+    : UserFactorReaderBase(factor) {
+    fun calculateRatioByValue(): Map<String, Double> {
+        val sums = factor.aggregateSum()
+        val total = sums.values.sum()
+        if (total == 0.0) return emptyMap()
+        return sums.mapValues { e -> e.value / total }
+    }
+}
+
+class CategoryFeatureUpdater(private val knownCategories: Set<String>, factor: MutableDoubleFactor) : UserFactorUpdaterBase(factor) {
+    fun update(value: Any?) {
+        if (value == null) {
+            factor.incrementOnToday(FeatureUtils.UNDEFINED)
+        } else {
+            val category = value.toString()
+            if (category in knownCategories) {
+                factor.incrementOnToday(category)
+            } else {
+                factor.incrementOnToday(FeatureUtils.OTHER)
+            }
+        }
+    }
+}
+
+class CategoryRatio(feature: CatergorialFeature, private val categoryName: String)
+    : UserFactorBase<CategoryFeatureReader>("categoryFeature:${feature.name}$:categoryName",
+        UserFactorDescriptions.categoriealFeatureDescriptor(feature)) {
+    override fun compute(reader: CategoryFeatureReader): String {
+        return reader.calculateRatioByValue().getOrDefault(categoryName, -1.0).toString()
+    }
+}
+
+class MostFrequentCategory(feature: CatergorialFeature)
+    : UserFactorBase<CategoryFeatureReader>("mostFrequentCategory:${feature.name}",
+        UserFactorDescriptions.categoriealFeatureDescriptor(feature)) {
+    override fun compute(reader: CategoryFeatureReader): String? {
+        return reader.calculateRatioByValue().maxBy { it.value }?.key
+    }
+}
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/CompletionFinishTypeFactors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/CompletionFinishTypeFactors.kt
new file mode 100644 (file)
index 0000000..953a4e1
--- /dev/null
@@ -0,0 +1,39 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.*
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+private val explicitSelectKey = "explicitSelect"
+private val typedSelectKey = "typedSelect"
+
+class CompletionFinishTypeReader(private val factor: DailyAggregatedDoubleFactor) : FactorReader {
+    fun getTotalExplicitSelectCount(): Double =
+            factor.aggregateSum()[explicitSelectKey] ?: 0.0
+
+    fun getTotalTypedSelectCount(): Double =
+            factor.aggregateSum()[typedSelectKey] ?: 0.0
+
+}
+
+class CompletionFinishTypeUpdater(private val factor: MutableDoubleFactor) : FactorUpdater {
+    fun fireExplicitCompletionPerformed() {
+        factor.incrementOnToday(explicitSelectKey)
+    }
+
+    fun fireTypedSelectPerformed() {
+        factor.incrementOnToday(typedSelectKey)
+    }
+}
+
+class ExplicitCompletionRatio : UserFactor {
+    override val id: String = "explicitSelectRatio"
+
+    override fun compute(storage: UserFactorStorage): String? {
+        val factorReader = storage.getFactorReader(UserFactorDescriptions.COMPLETION_FINISH_TYPE)
+        val total = factorReader.getTotalExplicitSelectCount() + factorReader.getTotalTypedSelectCount()
+        if (total == 0.0) return null
+        return (factorReader.getTotalExplicitSelectCount() / total).toString()
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/CompletionTypeFactors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/CompletionTypeFactors.kt
new file mode 100644 (file)
index 0000000..5b3f6c5
--- /dev/null
@@ -0,0 +1,30 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.codeInsight.completion.CompletionType
+import com.intellij.stats.personalization.*
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class CompletionTypeReader(private val factor: DailyAggregatedDoubleFactor) : FactorReader {
+    fun getCompletionCountByType(type: CompletionType): Double =
+            factor.aggregateSum().getOrDefault(type.toString(), 0.0)
+
+    fun getTotalCompletionCount(): Double = factor.aggregateSum().values.sum()
+}
+
+class CompletionTypeUpdater(private val factor: MutableDoubleFactor) : FactorUpdater {
+    fun fireCompletionPerformed(type: CompletionType) {
+        factor.incrementOnToday(type.toString())
+    }
+}
+
+class CompletionTypeRatio(private val type: CompletionType) : UserFactor {
+
+    override val id: String = "CompletionTypeRatioOf$type"
+    override fun compute(storage: UserFactorStorage): String? {
+        val reader = storage.getFactorReader(UserFactorDescriptions.COMPLETION_TYPE)
+        val total = reader.getTotalCompletionCount()
+        return if (total == 0.0) "0.0" else (reader.getCompletionCountByType(type) / total).toString()
+    }
+}
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/CompletionUsageFactors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/CompletionUsageFactors.kt
new file mode 100644 (file)
index 0000000..6f8f9d4
--- /dev/null
@@ -0,0 +1,39 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.*
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class CompletionUsageReader(factor: DailyAggregatedDoubleFactor) : UserFactorReaderBase(factor) {
+    fun getTodayCount(): Double = factor.onToday().getOrDefault("count", 0.0)
+
+    fun getTotalCount(): Double = factor.aggregateSum().getOrDefault("count", 0.0)
+
+    fun getWeekAverage(): Double = factor.aggregateAverage().getOrDefault("count", 0.0)
+}
+
+class CompletionUsageUpdater(factor: MutableDoubleFactor) : UserFactorUpdaterBase(factor) {
+    fun fireCompletionUsed() {
+        factor.incrementOnToday("count")
+    }
+}
+
+class TodayCompletionUsageCount : CompletionUsageFactorBase("todayCompletionCount") {
+    override fun compute(reader: CompletionUsageReader): Double? = reader.getTodayCount()
+}
+
+class WeekAverageUsageCount : CompletionUsageFactorBase("weekAverageDailyCompletionCount") {
+    override fun compute(reader: CompletionUsageReader): Double? = reader.getWeekAverage()
+}
+
+class TotalUsageCount : CompletionUsageFactorBase("totalCompletionCountInLastDays") {
+    override fun compute(reader: CompletionUsageReader): Double? = reader.getTotalCount()
+}
+
+abstract class CompletionUsageFactorBase(override val id: String) : UserFactor {
+    override final fun compute(storage: UserFactorStorage): String? =
+            compute(storage.getFactorReader(UserFactorDescriptions.COMPLETION_USAGE))?.toString()
+
+    abstract fun compute(reader: CompletionUsageReader): Double?
+}
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/DailyAggregatedDoubleFactor.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/DailyAggregatedDoubleFactor.kt
new file mode 100644 (file)
index 0000000..0c1b973
--- /dev/null
@@ -0,0 +1,63 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.DateUtil
+import com.intellij.stats.personalization.Day
+
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface DailyAggregatedDoubleFactor {
+    fun availableDays(): List<Day>
+
+    fun onDate(date: Day): Map<String, Double>?
+}
+
+interface MutableDoubleFactor : DailyAggregatedDoubleFactor {
+    fun incrementOnToday(key: String): Boolean
+
+    fun updateOnDate(date: Day, updater: MutableMap<String, Double>.() -> Unit): Boolean
+}
+
+private fun DailyAggregatedDoubleFactor.aggregateBy(reduce: (Double, Double) -> Double): Map<String, Double> {
+    val result = mutableMapOf<String, Double>()
+    for (onDate in availableDays().mapNotNull(this::onDate)) {
+        for ((key, value) in onDate) {
+            result.compute(key, { _, old -> if (old == null) value else reduce(old, value) })
+        }
+    }
+
+    return result
+}
+
+fun MutableDoubleFactor.setOnDate(date: Day, key: String, value: Double): Boolean =
+        updateOnDate(date) { put(key, value) }
+
+fun DailyAggregatedDoubleFactor.onToday(): Map<String, Double> = onDate(DateUtil.today()) ?: emptyMap()
+
+fun DailyAggregatedDoubleFactor.aggregateMin(): Map<String, Double> = aggregateBy(::minOf)
+
+fun DailyAggregatedDoubleFactor.aggregateMax(): Map<String, Double> = aggregateBy(::maxOf)
+
+fun DailyAggregatedDoubleFactor.aggregateSum(): Map<String, Double> = aggregateBy({ d1, d2 -> d1 + d2 })
+
+fun DailyAggregatedDoubleFactor.aggregateAverage(): Map<String, Double> {
+    val result = mutableMapOf<String, Double>()
+    val counts = mutableMapOf<String, Int>()
+    for (onDate in availableDays().mapNotNull(this::onDate)) {
+        for ((key, value) in onDate) {
+            result.compute(key) { _, old ->
+                if (old != null) {
+                    val n = counts[key]!!.toDouble()
+                    counts.computeIfPresent(key) { _, value -> value + 1 }
+                    FactorsUtil.mergeAverage(n.toInt(), old, 1, value)
+                } else {
+                    counts[key] = 1
+                    value
+                }
+            }
+        }
+    }
+
+    return result
+}
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/DayImpl.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/DayImpl.kt
new file mode 100644 (file)
index 0000000..6b3a8b8
--- /dev/null
@@ -0,0 +1,57 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.Day
+import java.text.ParsePosition
+import java.text.SimpleDateFormat
+import java.util.*
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class DayImpl(date: Date) : Day {
+    override val dayOfMonth: Int
+    override val month: Int
+    override val year: Int
+
+    companion object {
+        private val DATE_FORMAT = SimpleDateFormat("dd-MM-yyyy")
+
+        fun fromString(str: String): Day? {
+            val position = ParsePosition(0)
+            val date = DATE_FORMAT.parse(str, position)
+            if (position.index == 0) return null
+            return DayImpl(date)
+        }
+    }
+
+    init {
+        val calendar = Calendar.getInstance()
+        calendar.time = date
+        dayOfMonth = calendar.get(Calendar.DAY_OF_MONTH)
+        month = calendar.get(Calendar.MONTH)
+        year = calendar.get(Calendar.YEAR)
+    }
+
+    override fun compareTo(other: Day): Int {
+        if (year == other.year) {
+            if (month == other.month) {
+                return dayOfMonth.compareTo(other.dayOfMonth)
+            }
+            return month.compareTo(other.month)
+        }
+        return year.compareTo(other.year)
+    }
+
+    override fun hashCode(): Int {
+        return Objects.hash(year, month, dayOfMonth)
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (other != null && other is Day) return compareTo(other) == 0
+        return false
+    }
+
+    override fun toString(): String {
+        return "$dayOfMonth-$month-$year"
+    }
+}
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/DoubleFeatureFactors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/DoubleFeatureFactors.kt
new file mode 100644 (file)
index 0000000..a1b02bd
--- /dev/null
@@ -0,0 +1,71 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.*
+import com.jetbrains.completion.ranker.features.DoubleFeature
+import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class DoubleFeatureReader(factor: DailyAggregatedDoubleFactor)
+    : UserFactorReaderBase(factor) {
+    fun calculateAverageValue(): Double? {
+        return FactorsUtil.calculateAverageByAllDays(factor)
+    }
+
+    fun min(): Double? {
+        return factor.aggregateMin()["min"]
+    }
+
+    fun max(): Double? {
+        return factor.aggregateMax()["max"]
+    }
+
+    fun undefinedRatio(): Double? {
+        val sums = factor.aggregateSum()
+        val total = sums["count"] ?: return null
+        if (total == 0.0) return null
+
+        return sums.getOrDefault(FeatureUtils.UNDEFINED, 0.0) / total
+    }
+}
+
+class DoubleFeatureUpdater(factor: MutableDoubleFactor) : UserFactorUpdaterBase(factor) {
+    fun update(value: Any?) {
+        if (value == null) {
+            factor.incrementOnToday(FeatureUtils.UNDEFINED)
+        } else {
+            val doubleValue = value.asDouble()
+            factor.updateOnDate(DateUtil.today()) {
+                FactorsUtil.updateAverageValue(this, doubleValue)
+                compute("max", { _, old -> if (old == null) doubleValue else maxOf(old, doubleValue) })
+                compute("min", { _, old -> if (old == null) doubleValue else minOf(old, doubleValue) })
+            }
+        }
+    }
+
+    private fun Any.asDouble(): Double {
+        if (this is Number) return this.toDouble()
+        return this.toString().toDouble()
+    }
+}
+
+abstract class DoubleFeatureUserFactorBase(prefix: String, feature: DoubleFeature) :
+        UserFactorBase<DoubleFeatureReader>("${prefix}DoubleFeature:${feature.name}$",
+                UserFactorDescriptions.doubleFeatureDescriptor(feature))
+
+class AverageDoubleFeatureValue(feature: DoubleFeature) : DoubleFeatureUserFactorBase("avg", feature) {
+    override fun compute(reader: DoubleFeatureReader): String? = reader.calculateAverageValue()?.toString()
+}
+
+class MinDoubleFeatureValue(feature: DoubleFeature) : DoubleFeatureUserFactorBase("min", feature) {
+    override fun compute(reader: DoubleFeatureReader): String? = reader.min()?.toString()
+}
+
+class MaxDoubleFeatureValue(feature: DoubleFeature) : DoubleFeatureUserFactorBase("max", feature) {
+    override fun compute(reader: DoubleFeatureReader): String? = reader.max()?.toString()
+}
+
+class UndefinedDoubleFeatureValueRatio(feature: DoubleFeature) : DoubleFeatureUserFactorBase("undefinedRatio", feature) {
+    override fun compute(reader: DoubleFeatureReader): String? = reader.undefinedRatio()?.toString()
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/FactorsUtil.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/FactorsUtil.kt
new file mode 100644 (file)
index 0000000..baf1a06
--- /dev/null
@@ -0,0 +1,46 @@
+package com.intellij.stats.personalization.impl
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+object FactorsUtil {
+    fun mergeAverage(n1: Int, avg1: Double, n2: Int, avg2: Double): Double {
+        if (n1 == 0 && n2 == 0) return 0.0
+        val total = (n1 + n2).toDouble()
+        return (n1 / total) * avg1 + (n2 / total) * avg2
+    }
+
+    fun updateAverageValue(map: MutableMap<String, Double>, valueToAdd: Double) {
+        val count = map["count"]?.toInt()
+        val avg = map["average"]
+        if (count != null && avg != null) {
+            val newAverage = mergeAverage(1, valueToAdd, count, avg)
+            update(map, 1 + count, newAverage)
+        } else {
+            update(map, 1, valueToAdd)
+        }
+    }
+
+    fun calculateAverageByAllDays(factor: DailyAggregatedDoubleFactor): Double? {
+        var totalCount = 0
+        var average = 0.0
+        var present = false
+        for (onDate in factor.availableDays().mapNotNull { factor.onDate(it) }) {
+            val avg = onDate["average"]
+            val count = onDate["count"]?.toInt()
+            if (avg != null && count != null && count > 0) {
+                present = true
+                average = FactorsUtil.mergeAverage(totalCount, average, count, avg)
+                totalCount += count
+            }
+        }
+
+        return if (present) average else average
+    }
+
+
+    private fun update(map: MutableMap<String, Double>, count: Int, avg: Double) {
+        map["count"] = count.toDouble()
+        map["average"] = avg
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/ItemPositionFactors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/ItemPositionFactors.kt
new file mode 100644 (file)
index 0000000..1f183b5
--- /dev/null
@@ -0,0 +1,50 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.UserFactorBase
+import com.intellij.stats.personalization.UserFactorDescriptions
+import com.intellij.stats.personalization.UserFactorReaderBase
+import com.intellij.stats.personalization.UserFactorUpdaterBase
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class ItemPositionReader(factor: DailyAggregatedDoubleFactor) : UserFactorReaderBase(factor) {
+    fun getCountsByPosition(): Map<Int, Double> {
+        return factor.aggregateSum().asIterable().associate { (key, value) -> key.toInt() to value }
+    }
+
+    fun getAveragePosition(): Double? {
+        val positionToCount = getCountsByPosition()
+        if (positionToCount.isEmpty()) return null
+
+        val positionsSum = positionToCount.asSequence().sumByDouble { it.key * it.value }
+        val completionCount = positionToCount.asSequence().sumByDouble { it.value }
+
+        if (completionCount == 0.0) return null
+        return positionsSum / completionCount
+    }
+}
+
+class ItemPositionUpdater(factor: MutableDoubleFactor) : UserFactorUpdaterBase(factor) {
+    fun fireCompletionPerformed(selectedItemOrder: Int) {
+        factor.incrementOnToday(selectedItemOrder.toString())
+    }
+}
+
+class AverageSelectedItemPosition()
+    : UserFactorBase<ItemPositionReader>("averageSelectedPosition", UserFactorDescriptions.SELECTED_ITEM_POSITION) {
+    override fun compute(reader: ItemPositionReader): String? = reader.getAveragePosition()?.toString()
+}
+
+class MaxSelectedItemPosition()
+    : UserFactorBase<ItemPositionReader>("maxSelectedItemPosition", UserFactorDescriptions.SELECTED_ITEM_POSITION) {
+    override fun compute(reader: ItemPositionReader): String? =
+            reader.getCountsByPosition().asSequence().filter { it.value != 0.0 }.maxBy { it.key }?.key?.toString()
+}
+
+class MostFrequentSelectedItemPosition()
+    : UserFactorBase<ItemPositionReader>("mostFrequentItemPosition", UserFactorDescriptions.SELECTED_ITEM_POSITION) {
+    override fun compute(reader: ItemPositionReader): String? =
+            reader.getCountsByPosition().maxBy { it.value }?.key?.toString()
+}
+
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/MnemonicsUsageFactors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/MnemonicsUsageFactors.kt
new file mode 100644 (file)
index 0000000..2c645a9
--- /dev/null
@@ -0,0 +1,32 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.*
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class MnemonicsUsageReader(factor: DailyAggregatedDoubleFactor) : UserFactorReaderBase(factor) {
+    fun mnemonicsUsageRatio(): Double? {
+        val sums = factor.aggregateSum()
+        val total = sums["total"]
+        val used = sums["withMnemonics"]
+        if (total == null || used == null || total < 1.0) return null
+        return used / total
+    }
+}
+
+class MnemonicsUsageUpdater(factor: MutableDoubleFactor) : UserFactorUpdaterBase(factor) {
+    fun fireCompletionFinished(isMnemonicsUsed: Boolean) {
+        factor.updateOnDate(DateUtil.today()) {
+            compute("total", { _, before -> if (before == null) 1.0 else before + 1 })
+            val valueBefore = computeIfAbsent("withMnemonics", { 0.0 })
+            if (isMnemonicsUsed) {
+                set("withMnemonics", valueBefore + 1.0)
+            }
+        }
+    }
+}
+
+class MnemonicsRatio : UserFactorBase<MnemonicsUsageReader>("mnemonicsUsageRatio", UserFactorDescriptions.MNEMONICS_USAGE) {
+    override fun compute(reader: MnemonicsUsageReader): String? = reader.mnemonicsUsageRatio()?.toString()
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/PrefixLengthFactor.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/PrefixLengthFactor.kt
new file mode 100644 (file)
index 0000000..7d2e845
--- /dev/null
@@ -0,0 +1,42 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.*
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class PrefixLengthReader(factor: DailyAggregatedDoubleFactor) : UserFactorReaderBase(factor) {
+    fun getCountsByPrefixLength(): Map<Int, Double> {
+        return factor.aggregateSum().asIterable().associate { (key, value) -> key.toInt() to value }
+    }
+
+    fun getAveragePrefixLength(): Double? {
+        val lengthToCount = getCountsByPrefixLength()
+        if (lengthToCount.isEmpty()) return null
+
+        val totalChars = lengthToCount.asSequence().sumByDouble { it.key * it.value }
+        val completionCount = lengthToCount.asSequence().sumByDouble { it.value }
+
+        if (completionCount == 0.0) return null
+        return totalChars / completionCount
+    }
+}
+
+class PrefixLengthUpdater(factor: MutableDoubleFactor) : UserFactorUpdaterBase(factor) {
+    fun fireCompletionPerformed(prefixLength: Int) {
+        factor.incrementOnToday(prefixLength.toString())
+    }
+}
+
+class MostFrequentPrefixLength : UserFactorBase<PrefixLengthReader>("mostFrequentPrefixLength",
+        UserFactorDescriptions.PREFIX_LENGTH_ON_COMPLETION) {
+    override fun compute(reader: PrefixLengthReader): String? {
+        return reader.getCountsByPrefixLength().maxBy { it.value }?.key?.toString()
+    }
+}
+
+class AveragePrefixLength : UserFactorBase<PrefixLengthReader>("", UserFactorDescriptions.PREFIX_LENGTH_ON_COMPLETION) {
+    override fun compute(reader: PrefixLengthReader): String? {
+        return reader.getAveragePrefixLength()?.toString()
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/ProjectUserFactorStorage.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/ProjectUserFactorStorage.kt
new file mode 100644 (file)
index 0000000..f085a6d
--- /dev/null
@@ -0,0 +1,11 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.openapi.components.ProjectComponent
+import com.intellij.openapi.components.State
+import com.intellij.openapi.components.Storage
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+@State(name = "ProjectUserFactors", storages = arrayOf(Storage("completion.factors.user.xml")))
+class ProjectUserFactorStorage : ProjectComponent, UserFactorStorageBase()
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/TimeBetweenTypingFactors.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/TimeBetweenTypingFactors.kt
new file mode 100644 (file)
index 0000000..91bfde4
--- /dev/null
@@ -0,0 +1,25 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.stats.personalization.*
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class TimeBetweenTypingReader(factor: DailyAggregatedDoubleFactor) : UserFactorReaderBase(factor) {
+    fun averageTime(): Double? {
+        return FactorsUtil.calculateAverageByAllDays(factor)
+    }
+}
+
+class TimeBetweenTypingUpdater(factor: MutableDoubleFactor) : UserFactorUpdaterBase(factor) {
+    fun fireTypingPerformed(delayMs: Int) {
+        factor.updateOnDate(DateUtil.today()) {
+            FactorsUtil.updateAverageValue(this, delayMs.toDouble())
+        }
+    }
+}
+
+class AverageTimeBetweenTyping
+    : UserFactorBase<TimeBetweenTypingReader>("averageTimeBetweenTyping", UserFactorDescriptions.TIME_BETWEEN_TYPING) {
+    override fun compute(reader: TimeBetweenTypingReader): String? = reader.averageTime()?.toString()
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/UserFactorStorageBase.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/UserFactorStorageBase.kt
new file mode 100644 (file)
index 0000000..44a5e6b
--- /dev/null
@@ -0,0 +1,162 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.openapi.components.PersistentStateComponent
+import com.intellij.stats.personalization.*
+import com.intellij.util.attribute
+import org.jdom.Element
+import java.util.*
+
+abstract class UserFactorStorageBase
+    : UserFactorStorage, PersistentStateComponent<Element> {
+
+    private val state = CollectorState()
+
+    override fun <U : FactorUpdater> getFactorUpdater(description: UserFactorDescription<U, *>): U =
+            description.updaterFactory.invoke(getAggregateFactor(description.factorId))
+
+    override fun <R : FactorReader> getFactorReader(description: UserFactorDescription<*, R>): R =
+            description.readerFactory.invoke(getAggregateFactor(description.factorId))
+
+    override fun getState(): Element {
+        val element = Element("component")
+        val start = System.currentTimeMillis()
+        state.writeState(element)
+        val end = System.currentTimeMillis()
+        println("saving of user factors took it in ${end - start}ms")
+        return element
+    }
+
+    override fun loadState(newState: Element) {
+        state.applyState(newState)
+    }
+
+    private fun getAggregateFactor(factorId: String): MutableDoubleFactor =
+            state.aggregateFactors.computeIfAbsent(factorId, { DailyAggregateFactor() })
+
+    private class CollectorState {
+        val aggregateFactors: MutableMap<String, DailyAggregateFactor> = HashMap()
+
+        fun applyState(element: Element) {
+            aggregateFactors.clear()
+            if (element.name == "userFactors") {
+                for (child in element.children) {
+                    val factorId = child.getAttributeValue("id")
+                    if (child.name == "factor" && factorId != null) {
+                        val factor = DailyAggregateFactor.restore(child)
+                        if (factor != null) aggregateFactors[factorId] = factor
+                    }
+                }
+            }
+        }
+
+        fun writeState(element: Element) {
+            for ((id, factor) in aggregateFactors.asSequence().sortedBy { it.key }) {
+                val factorElement = Element("factor")
+                factorElement.attribute("id", id)
+                factor.writeState(factorElement)
+                element.addContent(factorElement)
+            }
+        }
+    }
+
+    class DailyAggregateFactor private constructor(private val aggregates: SortedMap<Day, DailyData> = sortedMapOf())
+        : MutableDoubleFactor {
+        constructor() : this(sortedMapOf())
+
+        init {
+            ensureLimit()
+        }
+
+        companion object {
+            val DAYS_LIMIT = 10
+
+            fun restore(element: Element): DailyAggregateFactor? {
+                val data = sortedMapOf<Day, DailyData>()
+                for (child in element.children) {
+                    val date = child.getAttributeValue("date")
+                    val day = DayImpl.fromString(date)
+                    if (child.name == "dailyData" && day != null) {
+                        val dailyData = DailyData.restore(child)
+                        if (dailyData != null) data.put(day, dailyData)
+                    }
+                }
+
+                if (data.isEmpty()) return null
+                return DailyAggregateFactor(data)
+            }
+        }
+
+        fun writeState(element: Element) {
+            for ((day, data) in aggregates) {
+                val dailyDataElement = Element("dailyData")
+                dailyDataElement.attribute("date", day.toString())
+                data.writeState(dailyDataElement)
+                element.addContent(dailyDataElement)
+            }
+        }
+
+        override fun availableDays(): List<Day> = aggregates.keys.toList()
+
+        override fun incrementOnToday(key: String): Boolean {
+            return updateOnDate(DateUtil.today()) {
+                compute(key, { _, oldValue -> if (oldValue == null) 1.0 else oldValue + 1.0 })
+            }
+        }
+
+        override fun onDate(date: Day): Map<String, Double>? = aggregates[date]?.data
+
+        override fun updateOnDate(date: Day, updater: MutableMap<String, Double>.() -> Unit): Boolean {
+            val old = aggregates[date]
+            if (old != null) {
+                updater.invoke(old.data)
+                return true
+            }
+
+            if (aggregates.size < DAYS_LIMIT || aggregates.firstKey() < date) {
+                val data = DailyData()
+                updater.invoke(data.data)
+                aggregates.put(date, data)
+                ensureLimit()
+                return true
+            }
+
+            return false
+        }
+
+        private fun ensureLimit() {
+            while (aggregates.size > DAYS_LIMIT) {
+                aggregates.remove(aggregates.firstKey())
+            }
+        }
+    }
+
+    private class DailyData(val data: MutableMap<String, Double> = HashMap()) {
+        companion object {
+            fun restore(element: Element): DailyData? {
+                val data = mutableMapOf<String, Double>()
+                for (child in element.children) {
+                    if (child.name == "observation") {
+                        val dataKey = child.getAttributeValue("name")
+                        val dataValue = child.getAttributeValue("value")
+
+                        // skip all if any observation is inconsistent
+                        val value = dataValue.toDoubleOrNull() ?: return null
+                        data[dataKey] = value
+                    }
+                }
+
+                if (data.isEmpty()) return null
+                return DailyData(data)
+            }
+        }
+
+        fun writeState(element: Element) {
+            for ((key, value) in data.asSequence().sortedBy { it.key }) {
+                val observation = Element("observation")
+                observation.attribute("name", key)
+                observation.attribute("value", value.toString())
+                element.addContent(observation)
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/src/com/intellij/stats/personalization/impl/UserFactorsManagerImpl.kt b/plugins/stats-collector/src/com/intellij/stats/personalization/impl/UserFactorsManagerImpl.kt
new file mode 100644 (file)
index 0000000..ed919e5
--- /dev/null
@@ -0,0 +1,86 @@
+package com.intellij.stats.personalization.impl
+
+import com.intellij.codeInsight.completion.CompletionType
+import com.intellij.completion.FeatureManagerImpl
+import com.intellij.openapi.components.ProjectComponent
+import com.intellij.openapi.diagnostic.Logger
+import com.intellij.stats.personalization.UserFactor
+import com.intellij.stats.personalization.UserFactorsManager
+import com.jetbrains.completion.ranker.features.BinaryFeature
+import com.jetbrains.completion.ranker.features.CatergorialFeature
+import com.jetbrains.completion.ranker.features.DoubleFeature
+import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class UserFactorsManagerImpl : UserFactorsManager, ProjectComponent {
+    private companion object {
+        val LOG = Logger.getInstance(UserFactorsManagerImpl::class.java)
+    }
+    private val userFactors = mutableMapOf<String, UserFactor>()
+    init {
+        // user factors
+        register(ExplicitCompletionRatio())
+        register(CompletionTypeRatio(CompletionType.BASIC))
+        register(CompletionTypeRatio(CompletionType.SMART))
+        register(CompletionTypeRatio(CompletionType.CLASS_NAME))
+
+        register(TodayCompletionUsageCount())
+        register(TotalUsageCount())
+        register(WeekAverageUsageCount())
+
+        register(MostFrequentPrefixLength())
+        register(AveragePrefixLength())
+
+        register(AverageSelectedItemPosition())
+        register(MaxSelectedItemPosition())
+        register(MostFrequentSelectedItemPosition())
+
+        register(AverageTimeBetweenTyping())
+
+        register(MnemonicsRatio())
+
+        // feature-derived factors
+        val featureManager = FeatureManagerImpl.getInstance()
+        featureManager.binaryFactors.forEach(this::registerBinaryFeatureDerivedFactors)
+        featureManager.doubleFactors.forEach(this::registerDoubleFeatureDerivedFactors)
+        featureManager.categorialFactors.forEach(this::registerCategorialFeatureDerivedFactors)
+    }
+
+    private fun registerBinaryFeatureDerivedFactors(feature: BinaryFeature) {
+        register(BinaryValueRatio(feature, feature.availableValues.first))
+        register(BinaryValueRatio(feature, feature.availableValues.second))
+    }
+
+    private fun registerDoubleFeatureDerivedFactors(feature: DoubleFeature) {
+        register(MaxDoubleFeatureValue(feature))
+        register(MinDoubleFeatureValue(feature))
+        register(AverageDoubleFeatureValue(feature))
+        register(UndefinedDoubleFeatureValueRatio(feature))
+    }
+
+    private fun registerCategorialFeatureDerivedFactors(feature: CatergorialFeature) {
+        feature.categories.forEach { register(CategoryRatio(feature, it)) }
+        register(CategoryRatio(feature, FeatureUtils.OTHER))
+        register(MostFrequentCategory(feature))
+    }
+
+    override fun getAllFactors(): List<UserFactor> = userFactors.values.toList()
+
+    override fun getAllFactorIds(): List<String> = userFactors.keys.toList()
+
+    override fun getFactor(id: String): UserFactor = userFactors[id]!!
+
+    private fun register(factor: UserFactor) {
+        val old = userFactors.put(factor.id, factor)
+        if (old != null) {
+            if (old === factor) {
+                LOG.warn("The same factor was registered twice")
+            } else {
+                LOG.warn("Two different factors with the same id found: id = ${old.id}, " +
+                        "classes = ${listOf(factor.javaClass.canonicalName, old.javaClass.canonicalName)}")
+            }
+        }
+    }
+}
\ No newline at end of file
index 996d9b6fbd58a35578dc0e92b7e5063db0d9e543..b862a00c0ad5ed0b42ec9005550c9a704942209d 100644 (file)
@@ -21,18 +21,17 @@ import com.intellij.codeInsight.completion.CompletionWeigher
 import com.intellij.codeInsight.lookup.LookupElement
 import com.intellij.psi.PsiMethod
 import com.intellij.sorting.Ranker
-import com.jetbrains.completion.ranker.features.LookupElementInfo
 
 
-internal class FakeRanker: Ranker {
+internal class FakeRanker : Ranker {
 
     var isShortFirst = true
 
     /**
      * Items are sorted by descending order, so item with the highest rank will be on top
      */
-    override fun rank(state: LookupElementInfo, relevance: Map<String, Any?>): Double? {
-        val lookupElementLength = state.result_length!!.toDouble()
+    override fun rank(relevance: Map<String, Any?>, userFactors: Map<String, Any?>): Double? {
+        val lookupElementLength = relevance["result_length"]!!.toString().toDouble()
         return if (isShortFirst) -lookupElementLength else lookupElementLength
     }
 
index e45da6fa286f216bacf2303ebd601f24926afb71..bb7176d9de828c1ab31bae25281045f3d62b32ad 100644 (file)
@@ -31,7 +31,7 @@ import com.intellij.stats.experiment.WebServiceStatusProvider
 import com.intellij.stats.experiment.WebServiceStatus
 import com.intellij.stats.network.service.RequestService
 import com.intellij.stats.network.service.ResponseData
-import com.jetbrains.completion.ranker.features.FeatureUtils
+import com.jetbrains.completion.ranker.features.impl.FeatureUtils
 import com.nhaarman.mockito_kotlin.any
 import com.nhaarman.mockito_kotlin.mock
 import org.assertj.core.api.Assertions
index d755a56e86ccf435a3229b579125b4f95ecb41a6..e383b77b9bbc3dd29595c8a17752dc7287483c0f 100644 (file)
@@ -21,7 +21,7 @@ import com.intellij.codeInsight.lookup.impl.LookupImpl
 import com.intellij.ide.highlighter.JavaFileType
 import com.intellij.mocks.TestRequestService
 import com.intellij.stats.experiment.WebServiceStatus
-import com.jetbrains.completion.ranker.features.FeatureUtils
+import com.jetbrains.completion.ranker.features.impl.FeatureUtils
 import org.assertj.core.api.Assertions.assertThat
 
 
index d87612bf48b12e9683dc0130c4f2fa3494968d7b..21698af9f9af1c5cac3badbfaed837ebc77c8ef3 100644 (file)
@@ -19,8 +19,7 @@ package com.intellij.sorting
 import com.intellij.codeInsight.lookup.LookupElement
 import com.intellij.codeInsight.lookup.impl.LookupImpl
 import com.intellij.openapi.util.Pair
-import com.jetbrains.completion.ranker.features.FeatureUtils
-import com.jetbrains.completion.ranker.features.LookupElementInfo
+import com.jetbrains.completion.ranker.features.impl.FeatureUtils
 import org.assertj.core.api.Assertions
 
 
@@ -29,19 +28,21 @@ internal fun LookupImpl.checkMlRanking(ranker: Ranker, prefix_length: Int) {
     val lookupElements = getRelevanceObjects(items, false)
 
     lookupElements.forEach { element, relevance ->
-        val weights: Map<String, Any?> = relevance.associate { it.first to it.second }
-        val ml_rank = weights["ml_rank"]?.toString()
-        if (ml_rank == "UNDEFINED" || weights["before_rerank_order"] == null) {
+        val weights: MutableMap<String, Any?> = relevance.associate { it.first to it.second }.toMutableMap()
+        val mlRank = weights["ml_rank"]?.toString()
+        if (mlRank == "UNDEFINED" || weights["before_rerank_order"] == null) {
             throw UnsupportedOperationException("Ranking failed")
         }
 
-        val old_order = weights["before_rerank_order"].toString().toInt()
+        val oldOrder = weights["before_rerank_order"].toString().toInt()
 
-        val state = LookupElementInfo(old_order, prefix_length, element.lookupString.length)
+        weights.put("position", oldOrder)
+        weights.put("query_length", prefix_length)
+        weights.put("result_length", element.lookupString.length)
 
-        val calculated_ml_rank = ranker.rank(state, weights)
-        Assertions.assertThat(calculated_ml_rank).isEqualTo(ml_rank?.toDouble())
-                .withFailMessage("Calculated: $calculated_ml_rank Regular: ${ml_rank?.toDouble()}")
+        val calculatedMlRank = ranker.rank(weights, emptyMap())
+        Assertions.assertThat(calculatedMlRank).isEqualTo(mlRank?.toDouble())
+                .withFailMessage("Calculated: $calculatedMlRank Regular: ${mlRank?.toDouble()}")
     }
 }
 
@@ -58,10 +59,6 @@ internal fun LookupImpl.assertEachItemHasMlValue(value: String) {
 }
 
 
-
-
-
-
 internal object Samples {
 
     val callCompletionOnClass = """
diff --git a/plugins/stats-collector/test/com/intellij/stats/personalization/AggregatedFactorTest.kt b/plugins/stats-collector/test/com/intellij/stats/personalization/AggregatedFactorTest.kt
new file mode 100644 (file)
index 0000000..07d0fdd
--- /dev/null
@@ -0,0 +1,99 @@
+package com.intellij.stats.personalization
+
+import com.intellij.stats.personalization.impl.*
+import com.intellij.testFramework.UsefulTestCase
+import junit.framework.TestCase
+import org.junit.Assert
+import java.util.*
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class AggregatedFactorTest : UsefulTestCase() {
+    private companion object {
+        val DATE_1 = DateUtil.byDate(Calendar.Builder().setDate(2010, 1, 1).build().time)
+        val DATE_2 = DATE_1.update(1)
+        val DATE_3 = DATE_1.update(2)
+        val DATE_4 = DATE_1.update(3)
+
+        fun Day.update(count: Int): Day {
+            return Calendar.getInstance().let {
+                it.set(year, month, dayOfMonth)
+                it.add(Calendar.DATE, count)
+                DateUtil.byDate(it.time)
+            }
+        }
+    }
+
+    fun `test min is correct`() {
+        val aggregateFactor: MutableDoubleFactor = UserFactorStorageBase.DailyAggregateFactor()
+
+        aggregateFactor.setOnDate(DATE_1, "count", 10.0)
+        aggregateFactor.setOnDate(DATE_3, "count", 20.0)
+        aggregateFactor.setOnDate(DATE_4, "delay", 1000.0)
+
+        val mins = createFactorForTests().aggregateMin()
+        TestCase.assertEquals(2, mins.size)
+        UsefulTestCase.assertEquals(10.0, mins["count"])
+        UsefulTestCase.assertEquals(1000.0, mins["delay"])
+    }
+
+    fun `test max is correct`() {
+        val maximums = createFactorForTests().aggregateMax()
+        TestCase.assertEquals(2, maximums.size)
+        UsefulTestCase.assertEquals(20.0, maximums["count"])
+        UsefulTestCase.assertEquals(1000.0, maximums["delay"])
+    }
+
+    fun `test sum is correct`() {
+        val maximums = createFactorForTests().aggregateSum()
+        TestCase.assertEquals(2, maximums.size)
+        UsefulTestCase.assertEquals(30.0, maximums["count"])
+        UsefulTestCase.assertEquals(1000.0, maximums["delay"])
+    }
+
+    fun `test average is only on present`() {
+        val maximums = createFactorForTests().aggregateAverage()
+        TestCase.assertEquals(2, maximums.size)
+        UsefulTestCase.assertEquals(15.0, maximums["count"]!!, 1e-10)
+        UsefulTestCase.assertEquals(1000.0, maximums["delay"]!!, 1e-10)
+    }
+
+    fun `test average does not lose precision`() {
+        val factor: MutableDoubleFactor = UserFactorStorageBase.DailyAggregateFactor()
+        factor.setOnDate(DATE_1, "key1", Double.MAX_VALUE)
+        factor.setOnDate(DATE_2, "key1", Double.MAX_VALUE)
+
+        val avg = factor.aggregateAverage()
+        TestCase.assertEquals(1, avg.size)
+        Assert.assertNotEquals(Double.POSITIVE_INFINITY, avg["key1"]!!)
+    }
+
+    fun `test factor stores information only for the last 10 days`() {
+        val fieldName = "count"
+        val factor: MutableDoubleFactor = UserFactorStorageBase.DailyAggregateFactor()
+        for (i in 0 until 10) {
+            TestCase.assertTrue(factor.updateOnDate(DATE_1.update(i)) {
+                put(fieldName, i.toDouble())
+            })
+        }
+
+        TestCase.assertEquals(45.0, factor.aggregateSum()[fieldName]!!, 1e-10)
+        TestCase.assertNotNull(factor.onDate(DATE_1))
+        TestCase.assertFalse(factor.updateOnDate(DATE_1.update(-100)) { put(fieldName, 100.0) })
+        TestCase.assertEquals(9.0, factor.aggregateMax()[fieldName]!!, 1e-10)
+
+        TestCase.assertTrue(factor.updateOnDate(DATE_1.update(100)) { put(fieldName, 1.0) })
+        TestCase.assertEquals(46.0, factor.aggregateSum()[fieldName]!!, 1e-10)
+    }
+
+    private fun createFactorForTests(): DailyAggregatedDoubleFactor {
+        val aggregateFactor: MutableDoubleFactor = UserFactorStorageBase.DailyAggregateFactor()
+
+        aggregateFactor.setOnDate(DATE_1, "count", 10.0)
+        aggregateFactor.setOnDate(DATE_3, "count", 20.0)
+        aggregateFactor.setOnDate(DATE_4, "delay", 1000.0)
+
+        return aggregateFactor
+    }
+}
\ No newline at end of file