Extract module with features from the completion-ranking module
authorVitaliy.Bibaev <vitaliy.bibaev@jetbrains.com>
Tue, 10 Apr 2018 15:23:58 +0000 (18:23 +0300)
committerVitaliy.Bibaev <vitaliy.bibaev@jetbrains.com>
Tue, 10 Apr 2018 15:23:58 +0000 (18:23 +0300)
40 files changed:
plugins/stats-collector/features/build.gradle [new file with mode: 0644]
plugins/stats-collector/features/resources/features/all_features.json [new file with mode: 0644]
plugins/stats-collector/features/resources/features/binary.json [new file with mode: 0644]
plugins/stats-collector/features/resources/features/categorical.json [new file with mode: 0644]
plugins/stats-collector/features/resources/features/final_features_order.txt [new file with mode: 0644]
plugins/stats-collector/features/resources/features/float.json [new file with mode: 0644]
plugins/stats-collector/features/resources/features/ignored.json [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/BinaryFeature.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/CatergorialFeature.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/DoubleFeature.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/Feature.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/FeatureInterpreter.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/FeatureManager.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/Transformer.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/ex/FeatureDefaultValueNotFound.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/ex/FutureOrderNotFound.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/ex/UnexpectedBinaryValueException.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/BinaryFeatureImpl.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/CatergorialFeatureImpl.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/CompletionFactors.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/DoubleFeatureImpl.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureInterpreterImpl.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureManagerFactory.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureReader.kt [new file with mode: 0644]
plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureTransformer.kt [new file with mode: 0644]
plugins/stats-collector/features/test/com/jetbrains/completion/feature/FeaturesTest.kt [new file with mode: 0644]
plugins/stats-collector/features/test/com/jetbrains/completion/feature/ModelMetadataTest.kt [new file with mode: 0644]
plugins/stats-collector/resources/META-INF/plugin.xml
plugins/stats-collector/src/com/intellij/completion/FeatureManagerImpl.kt
plugins/stats-collector/src/com/intellij/sorting/MLCompletionSorter.kt
plugins/stats-collector/src/com/intellij/sorting/MLSorter.kt
plugins/stats-collector/src/com/intellij/stats/completion/LookupCompletedTracker.kt
plugins/stats-collector/src/com/intellij/stats/personalization/UserFactorDescriptions.kt
plugins/stats-collector/src/com/intellij/stats/personalization/impl/BinaryFeatureFactors.kt
plugins/stats-collector/src/com/intellij/stats/personalization/impl/CategorialFeatureFactors.kt
plugins/stats-collector/src/com/intellij/stats/personalization/impl/DoubleFeatureFactors.kt
plugins/stats-collector/src/com/intellij/stats/personalization/impl/UserFactorsManagerImpl.kt
plugins/stats-collector/test/com/intellij/sorting/CompletionOrderWithFakeRankerTest.kt
plugins/stats-collector/test/com/intellij/sorting/UpdateExperimentStatusTest.kt
plugins/stats-collector/test/com/intellij/sorting/Utils.kt

diff --git a/plugins/stats-collector/features/build.gradle b/plugins/stats-collector/features/build.gradle
new file mode 100644 (file)
index 0000000..e0e8ef2
--- /dev/null
@@ -0,0 +1,13 @@
+sourceSets {
+    main {
+        java.srcDirs = ['src']
+        resources.srcDirs = ['resources']
+    }
+    test {
+        java.srcDir('test')
+    }
+}
+
+dependencies {
+    compile 'com.google.code.gson:gson:2.8.0'
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/resources/features/all_features.json b/plugins/stats-collector/features/resources/features/all_features.json
new file mode 100644 (file)
index 0000000..ee11eb4
--- /dev/null
@@ -0,0 +1,61 @@
+{
+  "javaRelevance": [
+    "before_rerank_order",
+    "ml_rank",
+
+    "position",
+    "query_length",
+    "result_length",
+
+    "sorter",
+    "frozen",
+    "prefix",
+    "explicitProximity",
+    "priority",
+    "middleMatching",
+    "templates",
+    "stats",
+    "liftShorter",
+    "grouping",
+    "accessible",
+    "simple",
+    "shorter",
+    "nameEnd",
+    "sameWords",
+    "kind",
+    "expectedType",
+    "proximity",
+    "liftShorterClasses",
+    "recursion",
+    "com.intellij.psi.css.impl.util.completion.CssValuesCompletionWeigher",
+    "nonGeneric",
+    "statics",
+    "com.jetbrains.python.codeInsight.completion.PythonCompletionWeigher",
+    "scalaContainingClassWeigher",
+    "scalaSbtDefinitionWeigher",
+    "com.intellij.scala.play.language.routing.completion.Play2ControllerMethodsCompletionWeigher",
+    "mostUsed",
+    "invokationCount"
+  ],
+  "javaProximity": [
+    "explicitlyImported",
+    "goPackageProximity",
+    "groovyReferenceListWeigher",
+    "inResolveScope",
+    "javaInheritance",
+    "knownElement",
+    "openedInEditor",
+    "phpNamespace",
+    "phpSameFile",
+    "phpScope",
+    "referenceList",
+    "rubyOwnCode",
+    "sameDirectory",
+    "sameLogicalRoot",
+    "sameModule",
+    "samePsiMember",
+    "scalaClassObjectWeigher",
+    "scalaExplicitlyImportedWeigher",
+    "sdkOrLibrary"
+  ]
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/resources/features/binary.json b/plugins/stats-collector/features/resources/features/binary.json
new file mode 100644 (file)
index 0000000..82c19d8
--- /dev/null
@@ -0,0 +1,52 @@
+{
+  "liftShorterClasses": {
+    "true": 1,
+    "false": 0,
+    "default": 0
+  },
+  "liftShorter": {
+    "true": 1,
+    "false": 0,
+    "default": 0
+  },
+  "middleMatching": {
+    "true": 1,
+    "false": 0,
+    "default": 0
+  },
+  "templates": {
+    "true": 1,
+    "false": 0,
+    "default": 0
+  },
+  "nonGeneric": {
+    "0": 0,
+    "1": 1,
+    "default": 0
+  },
+  "prox_inResolveScope": {
+    "true": 1,
+    "false": 0,
+    "default": 1
+  },
+  "prox_openedInEditor": {
+    "true": 1,
+    "false": 0,
+    "default": 0
+  },
+  "prox_sameDirectory": {
+    "true": 1,
+    "false": 0,
+    "default": 0
+  },
+  "prox_sameLogicalRoot": {
+    "true": 1,
+    "false": 0,
+    "default": 0
+  },
+  "prox_sdkOrLibrary": {
+    "true": 1,
+    "false": 0,
+    "default": 0
+  }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/resources/features/categorical.json b/plugins/stats-collector/features/resources/features/categorical.json
new file mode 100644 (file)
index 0000000..e15a116
--- /dev/null
@@ -0,0 +1,58 @@
+{
+  "accessible": [
+    "NORMAL",
+    "UNDEFINED",
+    "DEPRECATED",
+    "INACCESSIBLE"
+  ],
+  "expectedType": [
+    "normal",
+    "expected",
+    "UNDEFINED",
+    "ofDefaultType",
+    "maybeExpected",
+    "unexpected"
+  ],
+  "kind": [
+    "classNameOrGlobalStatic",
+    "normal",
+    "suitableClass",
+    "UNDEFINED",
+    "variable",
+    "expectedTypeVariable",
+    "expectedTypeMethod",
+    "expectedTypeConstant",
+    "probableKeyword",
+    "collectionFactory",
+    "funExpr",
+    "improbableKeyword",
+    "annoMethod",
+    "qualifiedWithField",
+    "qualifiedWithGetter",
+    "unlikelyClass",
+    "superMethodParameters",
+    "expectedTypeArgument"
+  ],
+  "prox_groovyReferenceListWeigher": [
+    "unknown",
+    "UNDEFINED"
+  ],
+  "prox_javaInheritance": [
+    "true",
+    "null",
+    "false",
+    "UNDEFINED"
+  ],
+  "prox_referenceList": [
+    "unknown",
+    "inapplicable",
+    "applicableByKind",
+    "UNDEFINED"
+  ],
+  "recursion": [
+    "delegation",
+    "recursive",
+    "UNDEFINED",
+    "normal"
+  ]
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/resources/features/final_features_order.txt b/plugins/stats-collector/features/resources/features/final_features_order.txt
new file mode 100644 (file)
index 0000000..3f1fe31
--- /dev/null
@@ -0,0 +1,107 @@
+position
+position=UNDEFINED
+query_length
+query_length=UNDEFINED
+result_length
+result_length=UNDEFINED
+accessible=NORMAL
+accessible=UNDEFINED
+accessible=DEPRECATED
+accessible=INACCESSIBLE
+accessible=OTHER
+expectedType=normal
+expectedType=expected
+expectedType=UNDEFINED
+expectedType=ofDefaultType
+expectedType=maybeExpected
+expectedType=unexpected
+expectedType=OTHER
+explicitProximity
+explicitProximity=UNDEFINED
+grouping
+grouping=UNDEFINED
+kind=classNameOrGlobalStatic
+kind=normal
+kind=suitableClass
+kind=UNDEFINED
+kind=variable
+kind=expectedTypeVariable
+kind=expectedTypeMethod
+kind=expectedTypeConstant
+kind=probableKeyword
+kind=collectionFactory
+kind=funExpr
+kind=improbableKeyword
+kind=annoMethod
+kind=qualifiedWithField
+kind=qualifiedWithGetter
+kind=unlikelyClass
+kind=superMethodParameters
+kind=expectedTypeArgument
+kind=OTHER
+liftShorter
+liftShorter=UNDEFINED
+liftShorterClasses
+liftShorterClasses=UNDEFINED
+middleMatching
+middleMatching=UNDEFINED
+mostUsed
+mostUsed=UNDEFINED
+nameEnd
+nameEnd=UNDEFINED
+nonGeneric
+nonGeneric=UNDEFINED
+prefix
+prefix=UNDEFINED
+priority
+priority=UNDEFINED
+recursion=delegation
+recursion=recursive
+recursion=UNDEFINED
+recursion=normal
+recursion=OTHER
+sameWords
+sameWords=UNDEFINED
+shorter
+shorter=UNDEFINED
+simple
+simple=UNDEFINED
+sorter
+sorter=UNDEFINED
+statics
+statics=UNDEFINED
+stats
+stats=UNDEFINED
+templates
+templates=UNDEFINED
+prox_explicitlyImported
+prox_explicitlyImported=UNDEFINED
+prox_groovyReferenceListWeigher=unknown
+prox_groovyReferenceListWeigher=UNDEFINED
+prox_groovyReferenceListWeigher=OTHER
+prox_inResolveScope
+prox_inResolveScope=UNDEFINED
+prox_javaInheritance=true
+prox_javaInheritance=null
+prox_javaInheritance=false
+prox_javaInheritance=UNDEFINED
+prox_javaInheritance=OTHER
+prox_knownElement
+prox_knownElement=UNDEFINED
+prox_openedInEditor
+prox_openedInEditor=UNDEFINED
+prox_referenceList=unknown
+prox_referenceList=inapplicable
+prox_referenceList=applicableByKind
+prox_referenceList=UNDEFINED
+prox_referenceList=OTHER
+prox_sameDirectory
+prox_sameDirectory=UNDEFINED
+prox_sameLogicalRoot
+prox_sameLogicalRoot=UNDEFINED
+prox_sameModule
+prox_sameModule=UNDEFINED
+prox_samePsiMember
+prox_samePsiMember=UNDEFINED
+prox_sdkOrLibrary
+prox_sdkOrLibrary=UNDEFINED
\ No newline at end of file
diff --git a/plugins/stats-collector/features/resources/features/float.json b/plugins/stats-collector/features/resources/features/float.json
new file mode 100644 (file)
index 0000000..6774f9c
--- /dev/null
@@ -0,0 +1,21 @@
+{
+  "explicitProximity": 0,
+  "grouping": 0,
+  "mostUsed": -1,
+  "nameEnd": 0,
+  "position": 0,
+  "prefix": 0,
+  "priority": 0,
+  "prox_explicitlyImported": 0.0,
+  "prox_knownElement": 0,
+  "prox_sameModule": 0,
+  "prox_samePsiMember": 0,
+  "query_length": 0,
+  "result_length": 8,
+  "sameWords": 0,
+  "shorter": 0,
+  "simple": 0,
+  "sorter": 1,
+  "statics": -3,
+  "stats": 0
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/resources/features/ignored.json b/plugins/stats-collector/features/resources/features/ignored.json
new file mode 100644 (file)
index 0000000..432d0c3
--- /dev/null
@@ -0,0 +1,20 @@
+[
+  "cerp_length",
+  "frozen",
+  "prox_phpNamespace",
+  "prox_phpSameFile",
+  "prox_phpScope",
+  "com.jetbrains.python.codeInsight.completion.PythonCompletionWeigher",
+  "prox_scalaClassObjectWeigher",
+  "prox_scalaExplicitlyImportedWeigher",
+  "com.intellij.scala.play.language.routing.completion.Play2ControllerMethodsCompletionWeigher",
+  "scalaSbtDefinitionWeigher",
+  "scalaContainingClassWeigher",
+  "prox_rubyOwnCode",
+  "com.intellij.psi.css.impl.util.completion.CssValuesCompletionWeigher",
+  "prox_goPackageProximity",
+
+  "before_rerank_order",
+  "ml_rank",
+  "invokationCount"
+]
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/BinaryFeature.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/BinaryFeature.kt
new file mode 100644 (file)
index 0000000..060a070
--- /dev/null
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature
+
+import com.jetbrains.completion.feature.Feature
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface BinaryFeature : Feature {
+    val defaultValue: Double
+
+    val availableValues: Pair<String, String>
+
+    val index: Int
+
+    data class BinaryValueDescriptor(val key: String, val mapped: Double)
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/CatergorialFeature.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/CatergorialFeature.kt
new file mode 100644 (file)
index 0000000..46da37e
--- /dev/null
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature
+
+import com.jetbrains.completion.feature.Feature
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface CatergorialFeature : Feature {
+    val categories: Set<String>
+
+    val otherCatergoryIndex: Int
+
+    fun indexByCategory(category: String): Int
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/DoubleFeature.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/DoubleFeature.kt
new file mode 100644 (file)
index 0000000..1fd87d4
--- /dev/null
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature
+
+import com.jetbrains.completion.feature.Feature
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface DoubleFeature : Feature {
+    val defaultValue: Double
+
+    val index: Int
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/Feature.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/Feature.kt
new file mode 100644 (file)
index 0000000..e1b0361
--- /dev/null
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface Feature {
+    val name: String
+
+    val undefinedIndex: Int
+
+    fun process(value: Any, featureArray: DoubleArray)
+
+    fun setDefaults(featureArray: DoubleArray)
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/FeatureInterpreter.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/FeatureInterpreter.kt
new file mode 100644 (file)
index 0000000..aa2d919
--- /dev/null
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature
+
+import com.jetbrains.completion.feature.BinaryFeature
+import com.jetbrains.completion.feature.CatergorialFeature
+import com.jetbrains.completion.feature.DoubleFeature
+
+interface FeatureInterpreter {
+    fun binary(name: String, description: Map<String, Double>, order: Map<String, Int>): BinaryFeature
+    fun double(name: String, defaultValue: Double, order: Map<String, Int>): DoubleFeature
+    fun categorial(name: String, categories: Set<String>, order: Map<String, Int>): CatergorialFeature
+}
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/FeatureManager.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/FeatureManager.kt
new file mode 100644 (file)
index 0000000..23a609d
--- /dev/null
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2000-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature
+
+import com.jetbrains.completion.feature.impl.CompletionFactors
+import com.jetbrains.completion.feature.impl.FeatureReader
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface FeatureManager {
+    val binaryFactors: List<BinaryFeature>
+    val doubleFactors: List<DoubleFeature>
+    val categorialFactors: List<CatergorialFeature>
+    val ignoredFactors: Set<String>
+    val featureOrder: Map<String, Int>
+
+    val completionFactors: CompletionFactors
+
+    fun isUserFeature(name: String): Boolean
+    fun allFeatures(): List<Feature>
+
+    fun createTransformer(): Transformer
+
+    interface Factory {
+        fun createFeatureManager(reader: FeatureReader, interpreter: FeatureInterpreter): FeatureManager
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/Transformer.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/Transformer.kt
new file mode 100644 (file)
index 0000000..4cad171
--- /dev/null
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+interface Transformer {
+    fun featureArray(relevanceMap: Map<String, Any>,
+                     userFactors: Map<String, Any?>): DoubleArray
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/ex/FeatureDefaultValueNotFound.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/ex/FeatureDefaultValueNotFound.kt
new file mode 100644 (file)
index 0000000..a2adbbd
--- /dev/null
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.ex
+
+class FeatureDefaultValueNotFound(name: String)
+    : IllegalArgumentException("Feature default value not found. Feature name: $name")
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/ex/FutureOrderNotFound.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/ex/FutureOrderNotFound.kt
new file mode 100644 (file)
index 0000000..603d2a7
--- /dev/null
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.ex
+
+class FutureOrderNotFound(name: String)
+    : IllegalArgumentException("Information about feature order not found. Feature name = $name")
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/ex/UnexpectedBinaryValueException.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/ex/UnexpectedBinaryValueException.kt
new file mode 100644 (file)
index 0000000..069fb4f
--- /dev/null
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.ex
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class UnexpectedBinaryValueException(featureName: String, value: String, availableValues: Set<String>)
+    : IllegalArgumentException("Feature $featureName allows $availableValues but not $value")
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/BinaryFeatureImpl.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/BinaryFeatureImpl.kt
new file mode 100644 (file)
index 0000000..de38847
--- /dev/null
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.impl
+
+import com.jetbrains.completion.feature.BinaryFeature
+import com.jetbrains.completion.feature.ex.UnexpectedBinaryValueException
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class BinaryFeatureImpl(override val name: String,
+                        override val index: Int,
+                        override val undefinedIndex: Int,
+                        override val defaultValue: Double,
+                        private val firstValue: BinaryFeature.BinaryValueDescriptor,
+                        private val secondValue: BinaryFeature.BinaryValueDescriptor) : BinaryFeature {
+    private fun transform(value: String): Double = when (value) {
+        firstValue.key -> firstValue.mapped
+        secondValue.key -> secondValue.mapped
+        else -> throw UnexpectedBinaryValueException(name, value, setOf(firstValue.key, secondValue.key))
+    }
+
+    override val availableValues: Pair<String, String> = firstValue.key to secondValue.key
+
+    override fun process(value: Any, featureArray: DoubleArray) {
+        featureArray[undefinedIndex] = 0.0
+        featureArray[index] = transform(value.toString())
+    }
+
+    override fun setDefaults(featureArray: DoubleArray) {
+        featureArray[undefinedIndex] = 1.0
+        featureArray[index] = defaultValue
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/CatergorialFeatureImpl.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/CatergorialFeatureImpl.kt
new file mode 100644 (file)
index 0000000..797c9a3
--- /dev/null
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.impl
+
+import com.jetbrains.completion.feature.CatergorialFeature
+
+class CatergorialFeatureImpl(override val name: String,
+                             override val undefinedIndex: Int,
+                             override val otherCatergoryIndex: Int,
+                             private val categoryToIndex: Map<String, Int>)
+    : CatergorialFeature {
+    override fun indexByCategory(category: String): Int = categoryToIndex[category] ?: otherCatergoryIndex
+
+    override val categories: Set<String> = categoryToIndex.keys
+
+    override fun process(value: Any, featureArray: DoubleArray) {
+        setDefaults(featureArray)
+        featureArray[indexByCategory(value.toString())] = 1.0
+        featureArray[undefinedIndex] = 0.0
+    }
+
+    override fun setDefaults(featureArray: DoubleArray) {
+        categories.forEach { featureArray[indexByCategory(it)] = 0.0 }
+        featureArray[undefinedIndex] = 1.0
+        featureArray[otherCatergoryIndex] = 0.0
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/CompletionFactors.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/CompletionFactors.kt
new file mode 100644 (file)
index 0000000..a8a7ef0
--- /dev/null
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.impl
+
+
+class CompletionFactors(proximity: Set<String>, relevance: Set<String>) {
+
+    private val knownFactors: Set<String> = HashSet<String>().apply {
+        addAll(proximity.map { "prox_$it" })
+        addAll(relevance)
+    }
+
+    fun unknownFactors(factors: Set<String>): List<String> {
+        var result: MutableList<String>? = null
+        for (factor in factors) {
+            val normalized = factor.substringBefore('@')
+            if (normalized !in knownFactors) {
+                result = (result ?: mutableListOf()).apply { add(normalized) }
+            }
+        }
+
+        return if (result != null) result else emptyList()
+    }
+}
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/DoubleFeatureImpl.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/DoubleFeatureImpl.kt
new file mode 100644 (file)
index 0000000..0f3fd9a
--- /dev/null
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2000-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.impl
+
+import com.jetbrains.completion.feature.DoubleFeature
+
+class DoubleFeatureImpl(override val name: String,
+                        override val index: Int,
+                        override val undefinedIndex: Int,
+                        override val defaultValue: Double) : DoubleFeature {
+    private companion object {
+        private val MAX_VALUE = Math.pow(10.0, 10.0)
+    }
+
+    override fun process(value: Any, featureArray: DoubleArray) {
+        featureArray[undefinedIndex] = 0.0
+        featureArray[index] = Math.min(value.asDouble(), MAX_VALUE)
+    }
+
+    private fun Any.asDouble(): Double {
+        if (this is Number) return this.toDouble()
+        return this.toString().toDouble()
+    }
+
+    override fun setDefaults(featureArray: DoubleArray) {
+        featureArray[undefinedIndex] = 1.0
+        featureArray[index] = defaultValue
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureInterpreterImpl.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureInterpreterImpl.kt
new file mode 100644 (file)
index 0000000..11dfe37
--- /dev/null
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2000-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.impl
+
+import com.jetbrains.completion.feature.BinaryFeature
+import com.jetbrains.completion.feature.BinaryFeature.BinaryValueDescriptor
+import com.jetbrains.completion.feature.CatergorialFeature
+import com.jetbrains.completion.feature.DoubleFeature
+import com.jetbrains.completion.feature.FeatureInterpreter
+import com.jetbrains.completion.feature.ex.FeatureDefaultValueNotFound
+import com.jetbrains.completion.feature.ex.FutureOrderNotFound
+
+class FeatureInterpreterImpl : FeatureInterpreter {
+    override fun binary(name: String, description: Map<String, Double>, order: Map<String, Int>): BinaryFeature {
+        val index = extractIndex(name, order)
+        val undefinedIndex = extractUndefinedIndex(name, order)
+        val default = description[FeatureUtils.DEFAULT] ?: throw FeatureDefaultValueNotFound(name)
+        val values = extractBinaryValuesMapping(description)
+        return BinaryFeatureImpl(name, index, undefinedIndex, default, values.first, values.second)
+    }
+
+    override fun double(name: String, defaultValue: Double, order: Map<String, Int>): DoubleFeature {
+        val index = extractIndex(name, order)
+        val undefinedIndex = extractUndefinedIndex(name, order)
+        return DoubleFeatureImpl(name, index, undefinedIndex, defaultValue)
+    }
+
+    override fun categorial(name: String, categories: Set<String>, order: Map<String, Int>): CatergorialFeature {
+        val undefinedIndex = extractUndefinedIndex(name, order)
+        val otherCategoryIndex = extractIndex(FeatureUtils.getOtherCategoryFeatureName(name), order)
+        val categoryToIndex = categories.associate { it to extractIndex(combine(name, it), order) }
+        return CatergorialFeatureImpl(name, undefinedIndex, otherCategoryIndex, categoryToIndex)
+    }
+
+    private fun extractIndex(name: String, order: Map<String, Int>): Int {
+        return order[name] ?: throw FutureOrderNotFound(name)
+    }
+
+    private fun extractUndefinedIndex(name: String, order: Map<String, Int>): Int {
+        return extractIndex(FeatureUtils.getUndefinedFeatureName(name), order)
+    }
+
+    private fun extractBinaryValuesMapping(description: Map<String, Double>)
+            : Pair<BinaryValueDescriptor, BinaryValueDescriptor> {
+        val result = mutableListOf<BinaryValueDescriptor>()
+        for ((name, value) in description) {
+            if (name == FeatureUtils.DEFAULT) continue
+            result += BinaryValueDescriptor(name, value)
+        }
+
+        assert(result.size == 2, { "binary feature must contains 2 values, but found $result" })
+        result.sortBy { it.key }
+        return result[0] to result[1]
+    }
+
+    private fun combine(featureName: String, categoryName: String): String = "$featureName=$categoryName"
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureManagerFactory.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureManagerFactory.kt
new file mode 100644 (file)
index 0000000..efe5c79
--- /dev/null
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2000-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.impl
+
+import com.jetbrains.completion.feature.*
+
+class FeatureManagerFactory: FeatureManager.Factory {
+    override fun createFeatureManager(reader: FeatureReader, interpreter: FeatureInterpreter): FeatureManager {
+        val order = FeatureReader.featuresOrder()
+
+        val binaryFactors = FeatureReader.binaryFactors()
+                .map { (name, description) -> interpreter.binary(name, description, order) }
+        val doubleFactors = FeatureReader.doubleFactors()
+                .map { (name, defaultValue) -> interpreter.double(name, defaultValue, order) }
+        val categorialFactors = FeatureReader.categoricalFactors()
+                .map { (name, categories) -> interpreter.categorial(name, categories, order) }
+
+        val completionFactors = FeatureReader.completionFactors()
+
+        val ignoredFactors = FeatureReader.ignoredFactors()
+
+        return MyFeatureManager(binaryFactors, doubleFactors, categorialFactors, ignoredFactors, completionFactors, order)
+    }
+
+    private class MyFeatureManager(override val binaryFactors: List<BinaryFeature>,
+                                   override val doubleFactors: List<DoubleFeature>,
+                                   override val categorialFactors: List<CatergorialFeature>,
+                                   override val ignoredFactors: Set<String>,
+                                   override val completionFactors: CompletionFactors,
+                                   override val featureOrder: Map<String, Int>) : FeatureManager {
+        override fun isUserFeature(name: String): Boolean = false
+
+        override fun allFeatures(): List<Feature> = ArrayList<Feature>().apply {
+            addAll(binaryFactors)
+            addAll(doubleFactors)
+            addAll(categorialFactors)
+        }
+
+        override fun createTransformer(): Transformer {
+            val features = allFeatures().associate { it.name to it }
+            return FeatureTransformer(features, ignoredFactors, featureOrder.size)
+        }
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureReader.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureReader.kt
new file mode 100644 (file)
index 0000000..23f869b
--- /dev/null
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2000-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.impl
+
+
+import com.google.gson.Gson
+import com.google.gson.reflect.TypeToken
+
+
+typealias DoubleFeatureInfo = Map<String, Double>
+typealias CategoricalFeatureInfo = Map<String, Set<String>>
+typealias BinaryFeatureInfo = Map<String, Map<String, Double>>
+typealias IgnoredFeatureInfo = Set<String>
+
+
+object FeatureUtils {
+    const val UNDEFINED = "UNDEFINED"
+    const val INVALID_CACHE = "INVALID_CACHE"
+
+    const val OTHER = "OTHER"
+    const val NONE = "NONE"
+
+    const val ML_RANK = "ml_rank"
+    const val BEFORE_ORDER = "before_rerank_order"
+
+    const val DEFAULT = "default"
+
+    fun getOtherCategoryFeatureName(name: String) = "$name=$OTHER"
+    fun getUndefinedFeatureName(name: String) = "$name=$UNDEFINED"
+
+    fun prepareRevelanceMap(relevance: List<Pair<String, Any?>>, position: Int, prefixLength: Int, elementLength: Int)
+            : Map<String, Any> {
+        val relevanceMap = mutableMapOf<String, Any>()
+        for ((name, value) in relevance) {
+            if(value == null) continue
+            if (name == "proximity") {
+                val proximityMap = value.toString().toProximityMap()
+                relevanceMap.putAll(proximityMap)
+            } else {
+                relevanceMap[name] = value
+            }
+        }
+
+        relevanceMap["position"] = position
+        relevanceMap["query_length"] = prefixLength
+        relevanceMap["result_length"] = elementLength
+
+        return relevanceMap
+    }
+
+    /**
+     * Proximity features now came like [samePsiFile=true, openedInEditor=false], need to convert to proper map
+     */
+    private fun String.toProximityMap(): Map<String, Any> {
+        val items = replace("[", "").replace("]", "").split(",")
+
+        return items.map {
+            val (key, value) = it.trim().split("=")
+            "prox_$key" to value
+        }.toMap()
+    }
+}
+
+
+object FeatureReader {
+    private val gson = Gson()
+
+    fun completionFactors(): CompletionFactors {
+        val text = fileContent("features/all_features.json")
+        val typeToken = object : TypeToken<Map<String, Set<String>>>() {}
+        val map = gson.fromJson<Map<String, Set<String>>>(text, typeToken.type)
+
+        val relevance: Set<String> = map["javaRelevance"] ?: emptySet()
+        val proximity: Set<String> = map["javaProximity"] ?: emptySet()
+
+        return CompletionFactors(proximity, relevance)
+    }
+
+    fun binaryFactors(): BinaryFeatureInfo {
+        val text = fileContent("features/binary.json")
+        val typeToken = object : TypeToken<BinaryFeatureInfo>() {}
+        return gson.fromJson<BinaryFeatureInfo>(text, typeToken.type)
+    }
+
+    fun categoricalFactors(): CategoricalFeatureInfo {
+        val text = fileContent("features/categorical.json")
+        val typeToken = object : TypeToken<CategoricalFeatureInfo>() {}
+        return gson.fromJson<CategoricalFeatureInfo>(text, typeToken.type)
+    }
+
+
+    fun ignoredFactors(): IgnoredFeatureInfo {
+        val text = fileContent("features/ignored.json")
+        val typeToken = object : TypeToken<IgnoredFeatureInfo>() {}
+        return gson.fromJson<IgnoredFeatureInfo>(text, typeToken.type)
+    }
+
+    fun doubleFactors(): DoubleFeatureInfo {
+        val text = fileContent("features/float.json")
+        val typeToken = object : TypeToken<DoubleFeatureInfo>() {}
+        return gson.fromJson<DoubleFeatureInfo>(text, typeToken.type)
+    }
+
+    fun featuresOrder(): Map<String, Int> {
+        val text = fileContent("features/final_features_order.txt")
+
+        var index = 0
+        val map = mutableMapOf<String, Int>()
+        text.split("\n").forEach {
+            val featureName = it.trim()
+            map[featureName] = index++
+        }
+
+        return map
+    }
+
+    private fun fileContent(fileName: String): String {
+        val fileStream = gson.javaClass.classLoader.getResourceAsStream(fileName)
+        return fileStream.reader().readText()
+    }
+
+    fun jsonMap(fileName: String): List<MutableMap<String, Any>> {
+        val text = fileContent(fileName)
+        val typeToken = object : TypeToken<List<Map<String, Any>>>() {}
+        return gson.fromJson<List<MutableMap<String, Any>>>(text, typeToken.type)
+    }
+
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureTransformer.kt b/plugins/stats-collector/features/src/com/jetbrains/completion/feature/impl/FeatureTransformer.kt
new file mode 100644 (file)
index 0000000..d31b0e3
--- /dev/null
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature.impl
+
+import com.jetbrains.completion.feature.Feature
+import com.jetbrains.completion.feature.Transformer
+
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class FeatureTransformer(val features: Map<String, Feature>,
+                         private val ignoredFeatures: Set<String>,
+                         arraySize: Int)
+    : Transformer {
+    private val array = DoubleArray(arraySize)
+    override fun featureArray(relevanceMap: Map<String, Any>, userFactors: Map<String, Any?>): DoubleArray {
+        for ((name, feature) in features) {
+            val value = relevanceMap[name]
+            if (value == null || isFeatureIgnored(name)) {
+                feature.setDefaults(array)
+            } else {
+                feature.process(value, array)
+            }
+        }
+
+        return array
+    }
+
+    private fun isFeatureIgnored(name: String): Boolean {
+        val normalized = name.substringBefore('@')
+        return normalized in ignoredFeatures
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/test/com/jetbrains/completion/feature/FeaturesTest.kt b/plugins/stats-collector/features/test/com/jetbrains/completion/feature/FeaturesTest.kt
new file mode 100644 (file)
index 0000000..97c3277
--- /dev/null
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2000-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature
+
+import com.jetbrains.completion.feature.impl.BinaryFeatureImpl
+import com.jetbrains.completion.feature.impl.CatergorialFeatureImpl
+import com.jetbrains.completion.feature.impl.DoubleFeatureImpl
+import org.junit.Assert
+import org.junit.Test
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class FeaturesTest {
+    private companion object {
+        val BINARY_FEATURE = BinaryFeatureImpl("testBinaryFeature", 0, 2, -1.0,
+                BinaryFeature.BinaryValueDescriptor("no_true", 0.0),
+                BinaryFeature.BinaryValueDescriptor("true", 1.0))
+
+        val DOUBLE_FEATURE = DoubleFeatureImpl("testDoubleFeature", 2, 1, -100.0)
+
+        val CATEGORIAL_FEATURE = CatergorialFeatureImpl("testCategoryFeature", 0, 1,
+                mapOf("BASIC" to 2, "SMART" to 3, "CLASS" to 5))
+    }
+
+    @Test
+    fun `test binary feature procissing if value is present`() {
+        val featureArray = DoubleArray(3, { Double.MIN_VALUE })
+        BINARY_FEATURE.process("no_true", featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(0.0, Double.MIN_VALUE, 0.0), featureArray, 1e-10)
+        BINARY_FEATURE.process("true", featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(1.0, Double.MIN_VALUE, 0.0), featureArray, 1e-10)
+    }
+
+    @Test
+    fun `test binary feature procissing if value is absent`() {
+        val featureArray = DoubleArray(3, { Double.MIN_VALUE })
+        BINARY_FEATURE.setDefaults(featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(-1.0, Double.MIN_VALUE, 1.0), featureArray, 1e-10)
+    }
+
+    @Test
+    fun `test double feature procissing if value is present`() {
+        val featureArray = DoubleArray(3, { Double.MIN_VALUE })
+        DOUBLE_FEATURE.process(10, featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(Double.MIN_VALUE, 0.0, 10.0), featureArray, 1e-10)
+        DOUBLE_FEATURE.process(30L, featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(Double.MIN_VALUE, 0.0, 30.0), featureArray, 1e-10)
+    }
+
+    @Test
+    fun `test double feature procissing if value is absent`() {
+        val featureArray = DoubleArray(3, { Double.MIN_VALUE })
+        DOUBLE_FEATURE.setDefaults(featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(Double.MIN_VALUE, 1.0, -100.0), featureArray, 1e-10)
+    }
+
+    @Test(expected = IllegalArgumentException::class)
+    fun `test double feature shold throw exception if value cannot be interpreted as double`() {
+        val featureArray = DoubleArray(3, { Double.MIN_VALUE })
+        DOUBLE_FEATURE.process("NOT A NUMBER", featureArray)
+    }
+
+    @Test
+    fun `test double feature shold accept value as a string`() {
+        val featureArray = DoubleArray(3, { Double.MIN_VALUE })
+        DOUBLE_FEATURE.process("100", featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(Double.MIN_VALUE, 0.0, 100.0), featureArray, 1e-10)
+    }
+
+    @Test
+    fun `test categorial feature`() {
+        val featureArray = DoubleArray(6, { Double.MIN_VALUE })
+        CATEGORIAL_FEATURE.setDefaults(featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(1.0, 0.0, 0.0, 0.0, Double.MIN_VALUE, 0.0), featureArray, 1e-10)
+
+        CATEGORIAL_FEATURE.process("OTHER", featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(0.0, 1.0, 0.0, 0.0, Double.MIN_VALUE, 0.0), featureArray, 1e-10)
+
+        CATEGORIAL_FEATURE.process("BASIC", featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(0.0, 0.0, 1.0, 0.0, Double.MIN_VALUE, 0.0), featureArray, 1e-10)
+
+        CATEGORIAL_FEATURE.process("SMART", featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(0.0, 0.0, 0.0, 1.0, Double.MIN_VALUE, 0.0), featureArray, 1e-10)
+
+        CATEGORIAL_FEATURE.process("CLASS", featureArray)
+        Assert.assertArrayEquals(doubleArrayOf(0.0, 0.0, 0.0, 0.0, Double.MIN_VALUE, 1.0), featureArray, 1e-10)
+    }
+}
\ No newline at end of file
diff --git a/plugins/stats-collector/features/test/com/jetbrains/completion/feature/ModelMetadataTest.kt b/plugins/stats-collector/features/test/com/jetbrains/completion/feature/ModelMetadataTest.kt
new file mode 100644 (file)
index 0000000..a095707
--- /dev/null
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2000-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jetbrains.completion.feature
+
+import com.jetbrains.completion.feature.impl.FeatureInterpreterImpl
+import com.jetbrains.completion.feature.impl.FeatureManagerFactory
+import com.jetbrains.completion.feature.impl.FeatureReader
+import com.jetbrains.completion.feature.impl.FeatureUtils
+import org.junit.Assert
+import org.junit.Assert.*
+import org.junit.Test
+
+/**
+ * @author Vitaliy.Bibaev
+ */
+class ModelMetadataTest {
+    @Test
+    fun `validate feature order`() {
+        val manager = featureManager()
+        val featureIndex: Map<String, Feature> = mutableMapOf<String, Feature>().apply {
+            manager.binaryFactors.forEach { put(it.name, it) }
+            manager.doubleFactors.forEach { put(it.name, it) }
+            manager.categorialFactors.forEach { put(it.name, it) }
+        }
+
+        for ((name, order) in manager.featureOrder) {
+            val split = name.split('=')
+            if (split.size > 3) Assert.fail("line '$name' in feature order contains more than 1 symbol '='")
+            if (split.size == 2) {
+                val (featureName, value) = split
+                val feature = featureIndex[featureName]
+                        ?: throw AssertionError("feature declared in the featureOrder not found: $featureName")
+                when (feature) {
+                    is BinaryFeature -> {
+                        assertEquals(FeatureUtils.UNDEFINED, value)
+                        assertEquals(order, feature.undefinedIndex)
+                    }
+                    is DoubleFeature -> {
+                        assertEquals(FeatureUtils.UNDEFINED, value)
+                        assertEquals(order, feature.undefinedIndex)
+                    }
+                    is CatergorialFeature -> {
+                        when (value) {
+                            FeatureUtils.UNDEFINED -> assertEquals(order, feature.undefinedIndex)
+                            FeatureUtils.OTHER -> assertEquals(order, feature.otherCatergoryIndex)
+                            else -> assertEquals(order, feature.indexByCategory(value))
+                        }
+                    }
+                    else -> fail("unknown feature type found: ${feature.javaClass.canonicalName}")
+                }
+            } else {
+                val featureName = split.single()
+                val feature = featureIndex[featureName]
+                        ?: throw AssertionError("feature declared in the featureOrder not found: $featureName")
+                when (feature) {
+                    is BinaryFeature -> assertEquals(order, feature.index)
+                    is DoubleFeature -> assertEquals(order, feature.index)
+                    is CatergorialFeature -> fail("categorial feature claims 'name=category' in feature order list")
+                    else -> fail("unknown feature type found: ${feature.javaClass.canonicalName}")
+                }
+            }
+        }
+    }
+
+    @Test
+    fun `test all features have distinct names`() {
+        val manager = featureManager()
+        val featuresByName = mutableListOf<Feature>().apply {
+            addAll(manager.binaryFactors)
+            addAll(manager.doubleFactors)
+            addAll(manager.categorialFactors)
+        }.groupBy { it.name }
+
+        var failed = false
+        for ((name, features) in featuresByName) {
+            if (features.size > 1) {
+                println("${features.size} features with name '$name found'")
+                failed = true
+            }
+        }
+
+        assertFalse("features with the same name found (see output above)", failed)
+
+    }
+
+    @Test
+    fun `test features fill features array properly`() {
+        val manager = featureManager()
+        val revertedFeatureOrder = mutableMapOf<Int, Feature>()
+        fun MutableMap<Int, Feature>.checkAndPut(index: Int, feature: Feature) {
+            assertFeaturesNotStoreValueBySameIndex(index, put(index, feature), feature)
+        }
+        for (feature in manager.binaryFactors) {
+            revertedFeatureOrder.checkAndPut(feature.index, feature)
+            revertedFeatureOrder.checkAndPut(feature.undefinedIndex, feature)
+        }
+
+        for (feature in manager.doubleFactors) {
+            revertedFeatureOrder.checkAndPut(feature.index, feature)
+            revertedFeatureOrder.checkAndPut(feature.undefinedIndex, feature)
+        }
+
+        for (feature in manager.categorialFactors) {
+            revertedFeatureOrder.checkAndPut(feature.otherCatergoryIndex, feature)
+            feature.categories.forEach {
+                revertedFeatureOrder.checkAndPut(feature.indexByCategory(it), feature)
+            }
+        }
+
+        assertEquals("features should totally cover the features array", manager.featureOrder.size, revertedFeatureOrder.size)
+        assertEquals(0, revertedFeatureOrder.keys.min())
+        assertEquals(manager.featureOrder.size - 1, revertedFeatureOrder.keys.max())
+    }
+
+    private fun assertFeaturesNotStoreValueBySameIndex(index: Int, old: Feature?, new: Feature) {
+        if (old != null) {
+            fail("features store values to the same array element [index = $index, names = {${old.name}, ${new.name}}]")
+        }
+    }
+
+    private fun featureManager(): FeatureManager =
+            FeatureManagerFactory().createFeatureManager(FeatureReader, FeatureInterpreterImpl())
+}
\ No newline at end of file
index 7a45e1b91d81b653145c49c5ce077f6336f0940b..a5fcc8a0799ac2df8455d9156b9bd748f0a684da 100644 (file)
 
     <component>
       <implementation-class>com.intellij.completion.FeatureManagerImpl</implementation-class>
-      <interface-class>com.jetbrains.completion.ranker.features.FeatureManager</interface-class>
+      <interface-class>com.jetbrains.completion.feature.FeatureManager</interface-class>
     </component>
   </application-components>
 
index 541cde547d1a283bf63751e469ff86104cd1d190..865ee29bf5ac2faf17319e91f0e5c27e44ca636b 100644 (file)
@@ -18,10 +18,11 @@ package com.intellij.completion
 
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.components.ApplicationComponent
-import com.jetbrains.completion.ranker.features.*
-import com.jetbrains.completion.ranker.features.impl.FeatureInterpreterImpl
-import com.jetbrains.completion.ranker.features.impl.FeatureManagerFactory
-import com.jetbrains.completion.ranker.features.impl.FeatureReader
+import com.jetbrains.completion.feature.*
+import com.jetbrains.completion.feature.impl.CompletionFactors
+import com.jetbrains.completion.feature.impl.FeatureInterpreterImpl
+import com.jetbrains.completion.feature.impl.FeatureManagerFactory
+import com.jetbrains.completion.feature.impl.FeatureReader
 
 /**
  * @author Vitaliy.Bibaev
index 5e2434101453d18ce58b83a23da90ffb63cb8e83..5e0a535dd7a34594cc490ce0dcd5daccfdd04784 100644 (file)
  */
 package com.intellij.sorting
 
-import com.jetbrains.completion.ranker.features.FeatureManager
+import com.jetbrains.completion.feature.FeatureManager
 import com.intellij.openapi.components.ApplicationComponent
 import com.intellij.openapi.components.ServiceManager
 import com.jetbrains.completion.ranker.CompletionRanker
-import com.jetbrains.completion.ranker.features.Transformer
+import com.jetbrains.completion.feature.Transformer
 
 
 interface Ranker {
index 9e00d3bcb64d1fcd8dc9a48835ac05c7000ee50a..3e2e4bfa1970d84f42eec9d50a677d2fd1ac13e0 100644 (file)
@@ -32,7 +32,7 @@ import com.intellij.psi.util.PsiUtilCore
 import com.intellij.stats.completion.prefixLength
 import com.intellij.stats.experiment.WebServiceStatus
 import com.intellij.stats.personalization.UserFactorsManager
-import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+import com.jetbrains.completion.feature.impl.FeatureUtils
 import java.util.*
 
 @Suppress("DEPRECATION")
index f73e6120a3c796b0293df5bfcc5e6d587ca93d2e..93aa856a1f442bbd22f80a2be212c0b72c7916fd 100644 (file)
@@ -23,7 +23,7 @@ import com.intellij.codeInsight.lookup.impl.LookupImpl
 import com.intellij.completion.FeatureManagerImpl
 import com.intellij.stats.personalization.UserFactorDescriptions
 import com.intellij.stats.personalization.UserFactorStorage
-import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+import com.jetbrains.completion.feature.impl.FeatureUtils
 
 /**
  * @author Vitaliy.Bibaev
index fcba985f859bbac6637b5ef30abdc1ed27e265ba..367b941269fc547adff394fbc07dbf9ba935267a 100644 (file)
@@ -17,9 +17,9 @@
 package com.intellij.stats.personalization
 
 import com.intellij.stats.personalization.impl.*
-import com.jetbrains.completion.ranker.features.BinaryFeature
-import com.jetbrains.completion.ranker.features.CatergorialFeature
-import com.jetbrains.completion.ranker.features.DoubleFeature
+import com.jetbrains.completion.feature.BinaryFeature
+import com.jetbrains.completion.feature.CatergorialFeature
+import com.jetbrains.completion.feature.DoubleFeature
 
 /**
  * @author Vitaliy.Bibaev
index 12cca98b862fc165b33c81b5b6c6b85c4c8ea127..321a3d9888f7e75d3f90051e7d0684f9d4832078 100644 (file)
@@ -20,8 +20,8 @@ import com.intellij.stats.personalization.UserFactorBase
 import com.intellij.stats.personalization.UserFactorDescriptions
 import com.intellij.stats.personalization.UserFactorReaderBase
 import com.intellij.stats.personalization.UserFactorUpdaterBase
-import com.jetbrains.completion.ranker.features.BinaryFeature
-import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+import com.jetbrains.completion.feature.BinaryFeature
+import com.jetbrains.completion.feature.impl.FeatureUtils
 
 /**
  * @author Vitaliy.Bibaev
index 21bd8bba9b22999437e9b0ef71a7daa116d8fe9f..e3fc4533cdf32efcd60ce54096408e4267d9bda6 100644 (file)
@@ -20,8 +20,8 @@ import com.intellij.stats.personalization.UserFactorBase
 import com.intellij.stats.personalization.UserFactorDescriptions
 import com.intellij.stats.personalization.UserFactorReaderBase
 import com.intellij.stats.personalization.UserFactorUpdaterBase
-import com.jetbrains.completion.ranker.features.CatergorialFeature
-import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+import com.jetbrains.completion.feature.CatergorialFeature
+import com.jetbrains.completion.feature.impl.FeatureUtils
 
 /**
  * @author Vitaliy.Bibaev
index 2d7c638f1f9e2a3e622c104ca18f41e2bcedd0df..b9c4abda61981371af89b435a709d2ebfd5b499e 100644 (file)
@@ -17,8 +17,8 @@
 package com.intellij.stats.personalization.impl
 
 import com.intellij.stats.personalization.*
-import com.jetbrains.completion.ranker.features.DoubleFeature
-import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+import com.jetbrains.completion.feature.DoubleFeature
+import com.jetbrains.completion.feature.impl.FeatureUtils
 
 /**
  * @author Vitaliy.Bibaev
index 433d4c90120e9530e2f6763ebfa72e23b150c717..7cfb1e9ec4db25a3446c4310e52c06cda6db15fc 100644 (file)
@@ -22,10 +22,10 @@ import com.intellij.openapi.components.ProjectComponent
 import com.intellij.openapi.diagnostic.Logger
 import com.intellij.stats.personalization.UserFactor
 import com.intellij.stats.personalization.UserFactorsManager
-import com.jetbrains.completion.ranker.features.BinaryFeature
-import com.jetbrains.completion.ranker.features.CatergorialFeature
-import com.jetbrains.completion.ranker.features.DoubleFeature
-import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+import com.jetbrains.completion.feature.BinaryFeature
+import com.jetbrains.completion.feature.CatergorialFeature
+import com.jetbrains.completion.feature.DoubleFeature
+import com.jetbrains.completion.feature.impl.FeatureUtils
 
 /**
  * @author Vitaliy.Bibaev
index 1637ed0d94a9d8aa03e0a7b71c852a82c9eca74b..085fd646af122cce373da024b9e5184ea3e4c98c 100644 (file)
@@ -31,7 +31,7 @@ import com.intellij.stats.experiment.WebServiceStatusProvider
 import com.intellij.stats.experiment.WebServiceStatus
 import com.intellij.stats.network.service.RequestService
 import com.intellij.stats.network.service.ResponseData
-import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+import com.jetbrains.completion.feature.impl.FeatureUtils
 import com.nhaarman.mockito_kotlin.any
 import com.nhaarman.mockito_kotlin.mock
 import org.assertj.core.api.Assertions
index 3cf27dd6b1ab333fcc82bb69d3a9e4d74ef4d38b..8a92f7a3e644130fcc63f931056771f7372cf07c 100644 (file)
@@ -21,7 +21,7 @@ import com.intellij.codeInsight.lookup.impl.LookupImpl
 import com.intellij.ide.highlighter.JavaFileType
 import com.intellij.mocks.TestRequestService
 import com.intellij.stats.experiment.WebServiceStatus
-import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+import com.jetbrains.completion.feature.impl.FeatureUtils
 import org.assertj.core.api.Assertions.assertThat
 
 
index ae7cf8ddb4a8fa785edf35c95c2846238ea6470d..e317d0201b7d6795f83e707dc3f4782a3c25588d 100644 (file)
@@ -19,7 +19,7 @@ package com.intellij.sorting
 import com.intellij.codeInsight.lookup.LookupElement
 import com.intellij.codeInsight.lookup.impl.LookupImpl
 import com.intellij.openapi.util.Pair
-import com.jetbrains.completion.ranker.features.impl.FeatureUtils
+import com.jetbrains.completion.feature.impl.FeatureUtils
 import org.assertj.core.api.Assertions