Merge pull request #6 from ckanz/cypher-transactions

Cypher transactions

Author: ckanz

.gitignore

@@ -12,3 +12,4 @@
 *.out
 
 code2cypher
+*/node_modules/

cypherGenerator.go

@@ -27,7 +27,10 @@ func getLabelForFileNode(currentFile fileInfo) string {
 
 // fileInfoToCypher returns a cypher statement to create a node for a given file
 func fileInfoToCypher(currentFile fileInfo, label string) string {
-  properties := ("{ name: '" + currentFile.Name + "', url: '" + currentFile.Url + "'")
+  properties := ("{ name: '" + currentFile.Name +
+    "', path: '" + currentFile.Path +
+    "', url: '" + currentFile.Url +
+    "', _tempId: '" + currentFile.Id + "'")
 
   if (!currentFile.IsDir) {
     properties += (", " + "size: " + strconv.FormatInt(currentFile.Size, 10) + ", " +
@@ -42,17 +45,19 @@ func fileInfoToCypher(currentFile fileInfo, label string) string {
 
 // contributerToCypher returns a cypher statement to create node for a given contributer
 func contributerToCypher(contributerId, contributerName, contributerEmail string) string {
-  return ("CREATE (" + contributerId + ":" + "person" + " { name: '" + contributerName + "', email: '" + contributerEmail + "' })")
+  return ("MERGE (" + contributerId + ":" + "person" + " { _tempId: '" + contributerId + "', name: '" + contributerName + "', email: '" + contributerEmail + "' })")
 }
 
 // contributerToCypherUpdate returns a cypher statement to update a given contributer's commitCount
 func contributerToCypherUpdate(contributerId string, commitCount int) string {
-  return ("SET " + contributerId + ".commitCount = " + strconv.Itoa(commitCount))
+  return ("MATCH (c:person { _tempId: '" + contributerId + "' }) " +
+  "SET c.commitCount = " + strconv.Itoa(commitCount)) + " " +
+  "REMOVE c._tempId"
 }
 
 // contributionToCypher returns to cypher statement to create a relationship between a file and a contributer
 func contributionToCypher(fileId, contributerId string, contributionId string) string {
-  return "CREATE (" + fileId + ")<-[" + contributionId + ":EDITED]-(" + contributerId + ")"
+  return "CREATE (" + fileId + ")<-[" + contributionId + ":EDITED { _tempId: '" + contributionId +"' }]-(" + contributerId + ")"
 }
 
 // contributionToCypher returns to cypher statement to create a relationship between a file and a contributer
@@ -63,10 +68,16 @@ func commitToCypher(fileId, contributerId string, contribution fileContribution)
 
 // contributionToCypherUpdate returns a cypher statement to update a given contribution's commitCount
 func contributionToCypherUpdate(contributionId string, commitCount int) string {
-  return "SET " + contributionId + ".commitCount = " + strconv.Itoa(commitCount)
+  return "MATCH (c:person)-[e:EDITED { _tempId: '" + contributionId + "' }]->(f:file) " +
+  "SET e.commitCount = " + strconv.Itoa(commitCount)
 }
 
 // folderStructureToCypher returns to cypher statement to create a relationship between a file and its parent folder
 func folderStructureToCypher(currentFile fileInfo) string {
-  return "CREATE (" + currentFile.Id + ")-[:IN_FOLDER]->(" + currentFile.ParentId + ")"
+  return "Match (a:directory { path: '" + currentFile.ParentPath +"' }) Match (b { path: '" + currentFile.Path +"' }) CREATE (b)-[:IN_FOLDER]->(a)"
+}
+
+// returns a cypher statement that removes a given property from all nodes
+func removeProperty(propertyName string) string {
+  return "MATCH (a) REMOVE a." + propertyName
 }

cypherGenerator_test.go

@@ -30,17 +30,17 @@ func TestFileInfoToCypher(t *testing.T) {
     {
       fileInfo { Id: "fileId", Name: "someDir", IsDir: true, },
       "testLabel",
-      "CREATE (fileId:testLabel { name: 'someDir', url: '' })",
+      "CREATE (fileId:testLabel { name: 'someDir', path: '', url: '', _tempId: 'fileId' })",
     },
     {
       fileInfo { Name: "someDir", IsDir: true, Url: "https://github.com/someName/someRepo/tree/master/someDir" },
       "testLabel",
-      "CREATE (:testLabel { name: 'someDir', url: 'https://github.com/someName/someRepo/tree/master/someDir' })",
+      "CREATE (:testLabel { name: 'someDir', path: '', url: 'https://github.com/someName/someRepo/tree/master/someDir', _tempId: '' })",
     },
     {
       fileInfo { Id: "fileId", Name: "someFile", IsDir: false, Size: 42, CommitCount: 23, ModTime: 111222333, Extension: "go" },
       "testLabel",
-      "CREATE (fileId:testLabel { name: 'someFile', url: '', size: 42, commitCount: 23, lastModifiedDateTime: datetime({ epochseconds: 111222333 }), lastModifiedTimestamp: 111222333, extension: 'go' })",
+      "CREATE (fileId:testLabel { name: 'someFile', path: '', url: '', _tempId: 'fileId', size: 42, commitCount: 23, lastModifiedDateTime: datetime({ epochseconds: 111222333 }), lastModifiedTimestamp: 111222333, extension: 'go' })",
     },
   }
   for _, table := range testTables {
@@ -62,7 +62,7 @@ func TestContributerToCypher(t *testing.T) {
       "someId",
       "William T. Riker",
       "[email protected]",
-      "CREATE (someId:person { name: 'William T. Riker', email: '[email protected]' })",
+      "MERGE (someId:person { _tempId: 'someId', name: 'William T. Riker', email: '[email protected]' })",
     },
   }
   for _, table := range testTables {
@@ -84,7 +84,7 @@ func TestContributionToCypher(t *testing.T) {
       "someFile_java",
       "William_T__Riker",
       "someFile_java__William_T__Riker",
-      "CREATE (someFile_java)<-[someFile_java__William_T__Riker:EDITED]-(William_T__Riker)",
+      "CREATE (someFile_java)<-[someFile_java__William_T__Riker:EDITED { _tempId: 'someFile_java__William_T__Riker' }]-(William_T__Riker)",
     },
   }
   for _, table := range testTables {
@@ -101,8 +101,8 @@ func TestFolderStructureToCypher(t *testing.T) {
     cypherResult string
   }{
     {
-      fileInfo { Id: "someFileId", ParentId: "someParentId" },
-      "CREATE (someFileId)-[:IN_FOLDER]->(someParentId)",
+      fileInfo { Path: "someFilePath", ParentPath: "someParentPath" },
+      "Match (a:directory { path: 'someParentPath' }) Match (b { path: 'someFilePath' }) CREATE (b)-[:IN_FOLDER]->(a)",
     },
   }
   for _, table := range testTables {

code2cypher.go renamed to git2cypher.go

@@ -7,28 +7,27 @@ import (
   "os"
   "log"
   "strings"
-  "strconv"
 )
 
 type fileInfo struct {
   Name string
+  Path string
   Url string
   Size int64
   Level int
   IsDir bool
   Id string
   Extension string
   ModTime int64
-  ParentName string
-  ParentId string
+  ParentPath string
   Contributions []fileContribution
   CommitCount int
 }
 var nodes []fileInfo
 var processedFiles = make(map[string]bool)
 var processedNodes = make(map[string]bool)
-var processedContributers = make(map[string]int)
-var processedContributions = make(map[string]int)
+var processedContributersSum = make(map[string]int)
+var processedContributionsSum = make(map[string]int)
 var verbose bool
 var simplified bool
 var repoPath string
@@ -42,12 +41,6 @@ func initFlags() {
   flag.Parse()
 }
 
-// getUniqueNameString creates a unique string for a file based on its nested depth in the folder and its name
-// TODO: instead of depth, modified timestamp might be a better value to create unique variable names with
-func getUniqueNameString(index int, element string) string {
-  return strconv.Itoa(index) + "-" + element
-}
-
 // getFileExtension returns the extension for a given file's full name
 func getFileExtension (info os.FileInfo) string {
   if (info.IsDir() == false) {
@@ -85,9 +78,10 @@ func main() {
       fileDepth := len(pathSegments) - 1
       fileName := info.Name()
       verboseLog("fileName: " + fileName)
-      uniqueNameString := getUniqueNameString(fileDepth, fileName)
 
-      if (processedFiles[uniqueNameString] != true) {
+      if (processedFiles[path] != true) {
+        parentPath := strings.Join(pathSegments[:len(pathSegments)-1], "/")
+        verboseLog("parentPath: " + parentPath)
         parentDepth := fileDepth - 1
         if (parentDepth < 0) {
           parentDepth = 0
@@ -97,19 +91,19 @@ func main() {
 
         nodes = append(nodes, fileInfo{
           Name: fileName,
+          Path: path,
           Url: buildGitHubUrl(gitRepoUrl, path, info.IsDir()),
           Size: info.Size(),
           Level: fileDepth,
           Extension: getFileExtension(info),
-          Id: createCypherFriendlyVarName(fileName, fileDepth),
+          Id: createCypherFriendlyVarName(path, fileDepth),
           IsDir: info.IsDir(),
           ModTime: info.ModTime().Unix(),
-          ParentName: pathSegments[parentDepth],
-          ParentId : createCypherFriendlyVarName(pathSegments[parentDepth], parentDepth),
+          ParentPath: parentPath,
           Contributions: contributions,
           CommitCount: len(contributions),
         })
-        processedFiles[uniqueNameString] = true
+        processedFiles[path] = true
       }
     }
 
@@ -121,11 +115,14 @@ func main() {
   verboseLog("")
 
   for _, currentFile := range nodes {
+    var processedContributers = make(map[string]int)
+    var processedContributions = make(map[string]int)
+    fmt.Println(":BEGIN")
     label := getLabelForFileNode(currentFile)
 
-    if (!processedNodes[currentFile.Id]) {
+    if (!processedNodes[currentFile.Path]) {
       fmt.Println(fileInfoToCypher(currentFile, label))
-      processedNodes[currentFile.Id] = true
+      processedNodes[currentFile.Path] = true
     }
 
     if (label == "file") {
@@ -136,33 +133,51 @@ func main() {
           processedContributers[contributerId] = 0
         }
         processedContributers[contributerId] += 1
+        processedContributersSum[contributerId] += 1
 
         contributionId := currentFile.Id + "__" + contributerId
-        contributionCypherStatement := contributionToCypher(currentFile.Id, contributerId, contributionId)
         if (processedContributions[contributionId] < 1) {
-          fmt.Println(contributionCypherStatement)
+          fmt.Println(contributionToCypher(currentFile.Id, contributerId, contributionId))
           processedContributions[contributionId] = 0
         }
         if (simplified != true) {
           fmt.Println(commitToCypher(currentFile.Id, contributerId, contribution))
         }
         processedContributions[contributionId] += 1
+        processedContributionsSum[contributionId] += 1
       }
     }
 
-    if (currentFile.Id != currentFile.ParentId) {
+    fmt.Println(";")
+    fmt.Println(":COMMIT")
+
+    if (len(currentFile.ParentPath) > 0) {
+      fmt.Println(":BEGIN")
       fmt.Println(folderStructureToCypher(currentFile))
+      fmt.Println(";")
+      fmt.Println(":COMMIT")
     }
   }
 
-  for contributerId, contributionCount := range processedContributers {
+  for contributerId, contributionCount := range processedContributersSum {
+    fmt.Println(":BEGIN")
     fmt.Println(contributerToCypherUpdate(contributerId, contributionCount))
+    fmt.Println(";")
+    fmt.Println(":COMMIT")
   }
-  for contributionId, commitCount := range processedContributions {
+
+  for contributionId, commitCount := range processedContributionsSum {
+    fmt.Println(":BEGIN")
     fmt.Println(contributionToCypherUpdate(contributionId, commitCount))
+    fmt.Println(";")
+    fmt.Println(":COMMIT")
   }
 
+  fmt.Println(":BEGIN")
+  fmt.Println(removeProperty("_tempId"))
   fmt.Println(";")
+  fmt.Println(":COMMIT")
+
 
   if err != nil {
     log.Println(err)

gitFunctions.go

@@ -57,7 +57,7 @@ func getGitLog(path string, repoPath string) []fileContribution {
     splitLog := strings.Split(contribution, "||")
     if (len(splitLog) > 1) {
       fileContribs = append(fileContribs, fileContribution{
-        Name: splitLog[0],
+        Name: strings.Replace(splitLog[0], "'", " ", -1),
         Email: splitLog[1],
         Commit: splitLog[2],
         AbbreviatedHash: splitLog[3],

js/acorn2cypher.js

@@ -0,0 +1,69 @@
+const acorn = require('acorn')
+const fs = require('fs')
+const path = require('path')
+
+const SystemPath = path.join(__dirname, '/')
+
+const getAllFiles = function (dirPath, arrayOfFiles) {
+  const files = fs.readdirSync(dirPath)
+
+  arrayOfFiles = arrayOfFiles || []
+
+  files.forEach(function (file) {
+    if (fs.statSync(dirPath + '/' + file).isDirectory()) {
+      arrayOfFiles = getAllFiles(dirPath + '/' + file, arrayOfFiles)
+    } else {
+      const extension = file.split('.').pop()
+      if (extension === 'js') arrayOfFiles.push(path.join(__dirname, dirPath, '/', file))
+    }
+  })
+
+  return arrayOfFiles
+}
+
+const toCypher = (type, size, name, varName, index, parentIndex, rawNode) => {
+  console.log(`CREATE (${varName}_${parentIndex !== undefined ? parentIndex : ''})-[:DECLARES]->(${parentIndex === undefined ? varName + '_' + index : ''}:${type} { size: ${size}, name: '[${name}]', ${Object.entries(rawNode).map(e => `_${e[0]}:'# ${e[1]}'`)} })`)
+}
+
+const allFiles = getAllFiles('src/')
+
+allFiles.forEach((f, i) => {
+  try {
+    console.log(':BEGIN')
+    const varName = `f_${i}`
+    console.log(`MERGE (${varName}_:file { path: '${f.split(SystemPath)[1]}' })`)
+    const file = fs.readFileSync(f, 'utf8')
+
+    const r = acorn.parse(file, { ecmaVersion: 2020, sourceType: 'module' })
+
+    const logNode = (n, index, parentIndex) => {
+      if (n.key) {
+        toCypher(n.type, n.end - n.start, n.key.name, varName, index, parentIndex, n)
+        return
+      }
+      const d = n.declarations ? n.declarations[0] : n.declaration
+      if (!d) return
+      if (d.id) {
+        toCypher(n.type, n.end - n.start, d.id.name, varName, index, parentIndex, n)
+      } else if (d.declarations) {
+        toCypher(n.type, n.end - n.start, d.declarations[0].id.name, varName, index, parentIndex, n)
+        if (d.declarations[0].body) logNodeBody(d.declarations[0].body.body, index)
+      }
+      if (d.body) logNodeBody(d.body.body, index)
+    }
+
+    const logNodeBody = (nb, parentIndex) => {
+      nb.forEach((n, index) => {
+        logNode(n, index, parentIndex)
+        // if (n.body) logNodeBody(n.body.body, index)
+      })
+    }
+
+    logNodeBody(r.body)
+    console.log(';')
+    console.log(':COMMIT')
+  } catch (e) {
+    console.log(';')
+    console.log(':ROLLBACK')
+  }
+})

js/dependencytree2cypher.js

@@ -0,0 +1,31 @@
+const dependencyTree = require('dependency-tree')
+const path = require('path')
+
+const SystemPath = path.join(__dirname, '/')
+
+const tree = dependencyTree({
+  filename: 'src/index.js',
+  directory: '.',
+  webpackConfig: 'webpack.config.js'
+})
+
+const createFileCypher = (fileName, varName = '') => {
+  return `MERGE (${varName}:file { path: '${fileName.split(SystemPath)[1]}' })`
+}
+
+const listFileImports = file => {
+  Object.entries(file).forEach(([currentFile, fileImports]) => {
+    console.log(':BEGIN')
+    console.log(createFileCypher(currentFile, 'f'))
+    Object.keys(fileImports).forEach((fileImport, index) => {
+      const varName = `i${index}`
+      console.log(createFileCypher(fileImport, varName))
+      console.log(`MERGE (f)-[:IMPORTS]->(${varName})`)
+    })
+    console.log(';')
+    console.log(':COMMIT')
+    listFileImports(fileImports)
+  })
+}
+
+listFileImports(tree)