Log filepath (#99)

* Added unique index keys

* changed insert statement, changed request behaviour

* deleted indexes

* [ML4SE-515] Log a path in the project to the file in DocumentLoggedData

* [ML4SE-515] fix server address

* [ML4SE-515] fix log files saving path

ij-plugin/src/main/kotlin/org/jetbrains/research/tasktracker/tracking/TaskDocumentListener.kt

@@ -3,10 +3,11 @@ package org.jetbrains.research.tasktracker.tracking
 import com.intellij.openapi.diagnostic.Logger
 import com.intellij.openapi.editor.event.DocumentEvent
 import com.intellij.openapi.editor.event.DocumentListener
+import com.intellij.openapi.project.Project
 import org.jetbrains.research.tasktracker.config.MainTaskTrackerConfig
 import org.jetbrains.research.tasktracker.tracking.logger.DocumentLogger
 
-class TaskDocumentListener : DocumentListener {
+class TaskDocumentListener(val project: Project) : DocumentListener {
     private val logger: Logger = Logger.getInstance(javaClass)
 
     init {
@@ -15,6 +16,6 @@ class TaskDocumentListener : DocumentListener {
 
     // Tracking documents changes before to be consistent with activity-tracker plugin
     override fun beforeDocumentChange(event: DocumentEvent) {
-        DocumentLogger.log(event.document)
+        DocumentLogger.log(project, event.document)
     }
 }

ij-plugin/src/main/kotlin/org/jetbrains/research/tasktracker/tracking/TaskFileHandler.kt

@@ -2,6 +2,7 @@ package org.jetbrains.research.tasktracker.tracking
 
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.diagnostic.Logger
+import com.intellij.openapi.editor.event.DocumentListener
 import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.module.ModuleManager
@@ -25,23 +26,25 @@ import java.io.File
 typealias ProjectTaskFileMap = MutableMap<Project, MutableMap<Task, MutableList<VirtualFile>>>
 typealias ProjectTaskIdFile = MutableMap<Project, MutableMap<Task, MutableMap<String, VirtualFile>>>
 
-@Suppress("UnusedPrivateMember")
+@Suppress("UnusedPrivateMember", "TooManyFunctions")
 object TaskFileHandler {
     private val logger: Logger = Logger.getInstance(TaskFileHandler.javaClass)
-    private val listener by lazy { TaskDocumentListener() }
+    private var listener: DocumentListener? = null
     private val projectTaskIdToFile: ProjectTaskIdFile = HashMap()
     val projectToTaskToFiles: ProjectTaskFileMap = HashMap()
 
     fun initProject(project: Project) {
-        TODO()
+        listener = TaskDocumentListener(project)
     }
 
+    private fun getListener() = listener ?: error("Listener is not define")
+
     fun initTask(project: Project, task: Task) {
         projectTaskIdToFile.putIfAbsent(project, mutableMapOf())
         projectTaskIdToFile[project]?.putIfAbsent(task, mutableMapOf())
         getOrCreateFiles(project, task).forEach { file ->
             file?.let {
-                addVirtualFileListener(it)
+                addVirtualFileListener(project, it)
                 projectToTaskToFiles.putIfAbsent(project, mutableMapOf())
                 projectToTaskToFiles[project]?.putIfAbsent(task, mutableListOf())
                 projectToTaskToFiles[project]?.get(task)?.add(it)
@@ -52,7 +55,7 @@ object TaskFileHandler {
     // TODO not forget to remove from document loggers hashmap, flush data
     fun disposeTask(project: Project, task: Task) {
         projectToTaskToFiles[project]?.let {
-            it[task]?.let { virtualFiles -> removeVirtualFileListener(virtualFiles) }
+            it[task]?.let { virtualFiles -> removeVirtualFileListener(project, virtualFiles) }
                 ?: logger.warn("attempt to dispose a uninitialized task: '$task'")
             it.remove(task)
             projectTaskIdToFile[project]?.remove(task)
@@ -63,24 +66,24 @@ object TaskFileHandler {
         } ?: logger.warn("attempt to dispose task: '$task' from uninitialized project: '$project'")
     }
 
-    private fun addVirtualFileListener(virtualFile: VirtualFile) {
+    private fun addVirtualFileListener(project: Project, virtualFile: VirtualFile) {
         ApplicationManager.getApplication().invokeAndWait {
             val document = FileDocumentManager.getInstance().getDocument(virtualFile)
             document?.let {
-                it.addDocumentListener(listener)
+                it.addDocumentListener(getListener())
                 // Log the first state
-                DocumentLogger.log(it)
+                DocumentLogger.log(project, it)
             } ?: logger.warn("attempt to add listener for non-existing document: '$document'")
         }
     }
 
-    private fun removeVirtualFileListener(virtualFiles: List<VirtualFile>) {
+    private fun removeVirtualFileListener(project: Project, virtualFiles: List<VirtualFile>) {
         virtualFiles.forEach { file ->
             ApplicationManager.getApplication().invokeAndWait {
                 val document = FileDocumentManager.getInstance().getDocument(file)
                 document?.let {
-                    DocumentLogger.removeDocumentLogPrinter(document)
-                    document.removeDocumentListener(listener)
+                    DocumentLogger.removeDocumentLogPrinter(project, document)
+                    document.removeDocumentListener(getListener())
                 }
             }
         }

ij-plugin/src/main/kotlin/org/jetbrains/research/tasktracker/tracking/logger/DocumentLogPrinter.kt

@@ -3,6 +3,7 @@ package org.jetbrains.research.tasktracker.tracking.logger
 import com.intellij.openapi.diagnostic.Logger
 import com.intellij.openapi.editor.Document
 import com.intellij.openapi.fileEditor.FileDocumentManager
+import com.intellij.openapi.project.Project
 import com.intellij.openapi.util.io.FileUtil
 import org.apache.commons.csv.CSVFormat
 import org.apache.commons.csv.CSVPrinter
@@ -25,27 +26,27 @@ class DocumentLogPrinter {
     /**
      * Gets the active logPrinter or creates a new one if there was none or the active one was full
      */
-    fun getActiveLogPrinter(document: Document): LogPrinter {
-        val activePrinter = getLastPrinter(document)
+    fun getActiveLogPrinter(project: Project, document: Document): LogPrinter {
+        val activePrinter = getLastPrinter(project, document)
         return if (activePrinter.isFull()) {
-            addLogPrinter(document)
+            addLogPrinter(project, document)
         } else {
             activePrinter
         }
     }
 
-    private fun getLastPrinter(document: Document) = if (logPrinters.isEmpty()) {
-        addLogPrinter(document)
+    private fun getLastPrinter(project: Project, document: Document) = if (logPrinters.isEmpty()) {
+        addLogPrinter(project, document)
     } else {
         logPrinters.last()
     }
 
-    private fun addLogPrinter(document: Document): LogPrinter {
+    private fun addLogPrinter(project: Project, document: Document): LogPrinter {
         logger.info("${MainTaskTrackerConfig.PLUGIN_NAME}: init printer")
         val logFile = createLogFile(document)
         val fileWriter = OutputStreamWriter(FileOutputStream(logFile), StandardCharsets.UTF_8)
         val csvPrinter = CSVPrinter(fileWriter, CSVFormat.DEFAULT)
-        csvPrinter.printRecord(DocumentLoggedData.headers)
+        csvPrinter.printRecord(DocumentLoggedData(project).headers)
         logPrinters.add(LogPrinter(csvPrinter, fileWriter, logFile))
         return logPrinters.last()
     }

ij-plugin/src/main/kotlin/org/jetbrains/research/tasktracker/tracking/logger/DocumentLogger.kt

@@ -2,6 +2,7 @@ package org.jetbrains.research.tasktracker.tracking.logger
 
 import com.intellij.openapi.diagnostic.Logger
 import com.intellij.openapi.editor.Document
+import com.intellij.openapi.project.Project
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.launch
 import org.jetbrains.research.tasktracker.requests.FileRequests
@@ -11,16 +12,16 @@ object DocumentLogger {
     private val myDocumentsToPrinters: HashMap<Document, DocumentLogPrinter> = HashMap()
     private val logger: Logger = Logger.getInstance(DocumentLogger.javaClass)
 
-    fun log(document: Document) {
+    fun log(project: Project, document: Document) {
         val docPrinter = myDocumentsToPrinters.getOrPut(document) { DocumentLogPrinter() }
-        val logPrinter = docPrinter.getActiveLogPrinter(document)
-        logPrinter.csvPrinter.printRecord(DocumentLoggedData.getData(document))
+        val logPrinter = docPrinter.getActiveLogPrinter(project, document)
+        logPrinter.csvPrinter.printRecord(DocumentLoggedData(project).getData(document))
     }
 
     fun getDocumentLogPrinter(document: Document): DocumentLogPrinter? = myDocumentsToPrinters[document]
 
-    fun removeDocumentLogPrinter(document: Document) {
-        log(document)
+    fun removeDocumentLogPrinter(project: Project, document: Document) {
+        log(project, document)
         val logFiles: List<File> = getDocumentLogPrinter(document)?.getLogFiles()
             ?: emptyList<File>().also {
                 logger.error("attempt to flush non-existing csv printer for document '$document'")

ij-plugin/src/main/kotlin/org/jetbrains/research/tasktracker/tracking/logger/LoggedData.kt

@@ -2,13 +2,15 @@ package org.jetbrains.research.tasktracker.tracking.logger
 
 import com.intellij.openapi.editor.Document
 import com.intellij.openapi.fileEditor.FileDocumentManager
+import com.intellij.openapi.project.Project
 import org.jetbrains.research.tasktracker.TaskTrackerPlugin
 import org.jetbrains.research.tasktracker.tracking.activity.ActivityEvent
 import org.jetbrains.research.tasktracker.tracking.fileEditor.FileEditorData
 import org.jetbrains.research.tasktracker.tracking.toolWindow.ToolWindowData
 import org.jetbrains.research.tasktracker.tracking.webcam.WebCamData
 import org.jetbrains.research.tasktracker.util.survey.SurveyData
 import org.joda.time.DateTime
+import kotlin.io.path.Path
 
 data class LoggedDataGetter<T, S>(val header: String, val getData: (T) -> S)
 
@@ -23,11 +25,17 @@ abstract class LoggedData<T, S> {
     }
 }
 
-object DocumentLoggedData : LoggedData<Document, String?>() {
+class DocumentLoggedData(project: Project) : LoggedData<Document, String?>() {
     override val loggedDataGetters: List<LoggedDataGetter<Document, String?>> = arrayListOf(
         LoggedDataGetter("date") { DateTime.now().toString() },
         LoggedDataGetter("timestamp") { it.modificationStamp.toString() },
-        LoggedDataGetter("filename") { FileDocumentManager.getInstance().getFile(it)?.name },
+        LoggedDataGetter("filename") {
+            FileDocumentManager.getInstance().getFile(it)?.toNioPath()?.let { filepath ->
+                project.basePath?.let { projectPath ->
+                    Path(projectPath).relativize(filepath)
+                } ?: filepath
+            }.toString()
+        },
         LoggedDataGetter("file_hash_code") { FileDocumentManager.getInstance().getFile(it)?.hashCode().toString() },
         LoggedDataGetter("document_hash_code") { it.hashCode().toString() },
         LoggedDataGetter("fragment") { it.text },

ij-plugin/src/main/kotlin/org/jetbrains/research/tasktracker/ui/main/panel/MainPluginPanelFactory.kt

@@ -50,6 +50,7 @@ class MainPluginPanelFactory : ToolWindowFactory {
     lateinit var project: Project
 
     override fun createToolWindowContent(project: Project, toolWindow: ToolWindow) {
+        TaskFileHandler.initProject(project)
         TaskTrackerPlugin.initPlugin()
         this.project = project
         mainWindow = project.getService(MainWindowService::class.java).mainWindow

ij-plugin/src/test/kotlin/org/jetbrains/research/tasktracker/tracking/TaskDocumentListenerTest.kt

@@ -22,7 +22,7 @@ class TaskDocumentListenerTest : CodeInsightFixtureTestCase<ModuleFixtureBuilder
             assert(document.text == text) { "Expected '$text' to be written, but found '${document.text}'" }
         }
         document.reWriteText("test")
-        DocumentLogger.getDocumentLogPrinter(document)?.getActiveLogPrinter(document)?.csvPrinter?.flush()
+        DocumentLogger.getDocumentLogPrinter(document)?.getActiveLogPrinter(project, document)?.csvPrinter?.flush()
         assert(logFile.exists()) { "log file with path '${logFile.path}' should have been created" }
         val changes = logFile.readLines()
             .filterIndexed { index, _ -> index > 1 }
@@ -46,7 +46,7 @@ class TaskDocumentListenerTest : CodeInsightFixtureTestCase<ModuleFixtureBuilder
         val logFileName =
             "${virtualFile.nameWithoutExtension}_${virtualFile.hashCode()}_${document.hashCode()}_0.csv"
         val logFile = File("${MainTaskTrackerConfig.logFilesFolder}/$logFileName")
-        document.addDocumentListener(TaskDocumentListener())
+        document.addDocumentListener(TaskDocumentListener(project))
         assert(!logFile.exists()) {
             "log file with path '${logFile.path}' should be created on first event in TaskDocumentListener"
         }

ij-plugin/src/test/kotlin/org/jetbrains/research/tasktracker/tracking/TaskFileHandlerTest.kt

@@ -51,6 +51,7 @@ class TaskFileHandlerTest : CodeInsightFixtureTestCase<ModuleFixtureBuilder<*>>(
     }
 
     private fun testInitTasks(tasks: List<TaskWithFiles>) {
+        TaskFileHandler.initProject(project)
         assertNoThrowable {
             tasks.forEach { task ->
                 TaskFileHandler.initTask(project, task)
@@ -84,6 +85,7 @@ class TaskFileHandlerTest : CodeInsightFixtureTestCase<ModuleFixtureBuilder<*>>(
     }
 
     private fun testDisposeTasks(tasks: List<Task>) {
+        TaskFileHandler.initProject(project)
         tasks.forEach { task ->
             TaskFileHandler.initTask(project, task)
         }

ij-server/src/main/kotlin/org/jetbrains/research/tasktracker/database/DatabaseFactory.kt

@@ -26,7 +26,7 @@ object DatabaseFactory {
                 SurveyData,
                 FileEditorData
             ).let {
-                SchemaUtils.create(tables = it)
+                SchemaUtils.createMissingTablesAndColumns(tables = it)
             }
             commit()
         }

ij-server/src/main/kotlin/org/jetbrains/research/tasktracker/database/models/data/ActivityData.kt

@@ -1,6 +1,6 @@
 package org.jetbrains.research.tasktracker.database.models.data
 
-import org.jetbrains.exposed.sql.statements.InsertStatement
+import org.jetbrains.exposed.sql.statements.UpdateBuilder
 
 /**
  * The table presenting all the activities that have occurred in the IDE.
@@ -26,12 +26,12 @@ object ActivityData : DataTable() {
     /**
      * Activity ID as provided by the IDE.
      */
-    val actionId = integer("action_id").nullable()
+    val actionId = integer("action_id").nullable().default(-1)
 
-    override fun insertData(insertStatement: InsertStatement<Number>, iterator: Iterator<String>, researchId: Int) {
-        insertStatement[type] = iterator.next()
-        insertStatement[info] = iterator.next()
-        insertStatement[selectedText] = iterator.next()
-        insertStatement[actionId] = iterator.next().toIntOrNull()
+    override fun insertData(updateBuilder: UpdateBuilder<*>, iterator: Iterator<String>, researchId: Int) {
+        updateBuilder[type] = iterator.next()
+        updateBuilder[info] = iterator.next()
+        updateBuilder[selectedText] = iterator.next()
+        updateBuilder[actionId] = iterator.next().toIntOrNull()
     }
 }

ij-server/src/main/kotlin/org/jetbrains/research/tasktracker/database/models/data/DataTable.kt

@@ -2,18 +2,18 @@ package org.jetbrains.research.tasktracker.database.models.data
 
 import org.jetbrains.exposed.dao.id.EntityID
 import org.jetbrains.exposed.dao.id.IntIdTable
-import org.jetbrains.exposed.sql.insert
+import org.jetbrains.exposed.sql.insertIgnore
 import org.jetbrains.exposed.sql.javatime.timestampWithTimeZone
-import org.jetbrains.exposed.sql.statements.InsertStatement
+import org.jetbrains.exposed.sql.statements.UpdateBuilder
 import org.jetbrains.research.tasktracker.database.models.Researches
 import java.time.OffsetDateTime
 
 abstract class DataTable : IntIdTable() {
-    val researchId = reference("research_id", Researches).index()
+    val researchId = reference("research_id", Researches)
     val date = timestampWithTimeZone("date")
 
     protected abstract fun insertData(
-        insertStatement: InsertStatement<Number>,
+        updateBuilder: UpdateBuilder<*>,
         iterator: Iterator<String>,
         researchId: Int
     )
@@ -22,7 +22,7 @@ abstract class DataTable : IntIdTable() {
         iterator: Iterator<String>,
         researchId: Int,
     ) {
-        insert {
+        insertIgnore {
             it[this.researchId] = EntityID(researchId, Researches)
             it[date] = OffsetDateTime.parse(iterator.next())
             insertData(it, iterator, researchId)

ij-server/src/main/kotlin/org/jetbrains/research/tasktracker/database/models/data/DocumentData.kt

@@ -1,6 +1,6 @@
 package org.jetbrains.research.tasktracker.database.models.data
 
-import org.jetbrains.exposed.sql.statements.InsertStatement
+import org.jetbrains.exposed.sql.statements.UpdateBuilder
 
 /**
  * The table represents information about changes to the file used in the research.
@@ -17,12 +17,12 @@ object DocumentData : DataTable() {
      */
     val testMode = text("test_mode")
 
-    override fun insertData(insertStatement: InsertStatement<Number>, iterator: Iterator<String>, researchId: Int) {
-        insertStatement[timestamp] = iterator.next().toLong()
-        insertStatement[filename] = iterator.next()
-        insertStatement[fileHashCode] = iterator.next().toInt()
-        insertStatement[documentHashCode] = iterator.next().toInt()
-        insertStatement[fragment] = iterator.next()
-        insertStatement[testMode] = iterator.next()
+    override fun insertData(updateBuilder: UpdateBuilder<*>, iterator: Iterator<String>, researchId: Int) {
+        updateBuilder[timestamp] = iterator.next().toLong()
+        updateBuilder[filename] = iterator.next()
+        updateBuilder[fileHashCode] = iterator.next().toInt()
+        updateBuilder[documentHashCode] = iterator.next().toInt()
+        updateBuilder[fragment] = iterator.next()
+        updateBuilder[testMode] = iterator.next()
     }
 }

ij-server/src/main/kotlin/org/jetbrains/research/tasktracker/database/models/data/FileEditorData.kt

@@ -1,6 +1,6 @@
 package org.jetbrains.research.tasktracker.database.models.data
 
-import org.jetbrains.exposed.sql.statements.InsertStatement
+import org.jetbrains.exposed.sql.statements.UpdateBuilder
 
 /**
  *
@@ -22,9 +22,9 @@ object FileEditorData : DataTable() {
      */
     val newFile = text("new_file").nullable()
 
-    override fun insertData(insertStatement: InsertStatement<Number>, iterator: Iterator<String>, researchId: Int) {
-        insertStatement[action] = iterator.next()
-        insertStatement[oldFile] = iterator.next()
-        insertStatement[newFile] = iterator.next()
+    override fun insertData(updateBuilder: UpdateBuilder<*>, iterator: Iterator<String>, researchId: Int) {
+        updateBuilder[action] = iterator.next()
+        updateBuilder[oldFile] = iterator.next()
+        updateBuilder[newFile] = iterator.next()
     }
 }

ij-server/src/main/kotlin/org/jetbrains/research/tasktracker/database/models/data/SurveyData.kt

@@ -1,6 +1,6 @@
 package org.jetbrains.research.tasktracker.database.models.data
 
-import org.jetbrains.exposed.sql.statements.InsertStatement
+import org.jetbrains.exposed.sql.statements.UpdateBuilder
 
 /**
  * The table represents information about a completed survey.
@@ -19,10 +19,10 @@ object SurveyData : DataTable() {
     val option = text("option").nullable()
     val answer = text("answer")
 
-    override fun insertData(insertStatement: InsertStatement<Number>, iterator: Iterator<String>, researchId: Int) {
-        insertStatement[questionId] = iterator.next().toInt()
-        insertStatement[question] = iterator.next()
-        insertStatement[option] = iterator.next()
-        insertStatement[answer] = iterator.next()
+    override fun insertData(updateBuilder: UpdateBuilder<*>, iterator: Iterator<String>, researchId: Int) {
+        updateBuilder[questionId] = iterator.next().toInt()
+        updateBuilder[question] = iterator.next()
+        updateBuilder[option] = iterator.next()
+        updateBuilder[answer] = iterator.next()
     }
 }