WIP: add javascript tokenizer

tokenizers/javascript/.gitignore

@@ -0,0 +1,6 @@
+*.swp
+*.swo
+*.log
+
+node_modules
+.DS_Store

tokenizers/javascript/index.js

@@ -0,0 +1,88 @@
+const esprima = require('esprima')
+const escodegen = require('escodegen')
+const fs = require('fs-extra-promise')
+const tokenizer = require('./tokenizer')
+
+const immutable = require('immutable')
+const walk = require('esprima-walk')
+
+const { base64FileName } = require('./util')
+
+const estools = require('estools')
+
+const TOKENIZER_SCOPE_FILE = 'file-scope'
+const TOKENIZER_SCOPE_FUNCTION = 'function-scope'
+
+const TOKENIZER_SCOPE = TOKENIZER_SCOPE_FILE
+
+// TODO: estools map / filter / traverse (instead of walk)
+// - filter subfunctions from fuction asts somehow
+// - test on SCC
+
+// TODO: get rid of the function block and indentation
+const regenerateFunctionCode = function(functionAst) {
+  codegenOptions = { // NOTE: doesn't help
+    format: {
+      parentheses: false
+    }
+  }
+
+  // NOTE: functionAst.body ommits the function signature (returns block only)
+  return escodegen.generate(functionAst.body, {})
+}
+
+const processFile = function(fileName, data) {
+  //let parentId = base64FileName(fileName) // TODO: incorporate repo name / hash
+  let parentId = fileName
+  let blockId = 1
+
+  if (TOKENIZER_SCOPE === TOKENIZER_SCOPE_FILE) {
+    return immutable.List.of(tokenizer(data, parentId, blockId))
+  }
+
+  options = {
+    loc: true,
+    range: true,
+    comment: true,
+    attachComment: true
+  }
+  fileAst = esprima.parse(data, {});
+
+  let functions = immutable.List()
+  let functionTokens = immutable.List()
+  walk(fileAst, (node) => {
+    if (node.type == 'FunctionExpression') {
+      // const functionAstShallow = estools.map(node, (subNode) => {
+      //   if (subNode === undefined || subNode.type === undefined) return
+      //   if (subNode.type == 'FunctionExpression')
+      //     return {}
+      //   else return subNode
+      // })
+      //console.log(functionAstShallow)
+      //process.exit(1)
+      const functionAstShallow = node
+      const functionCode = regenerateFunctionCode(functionAstShallow)
+      functions = functions.push(functionCode)
+      const tokenizedFunction = tokenizer(functionCode, parentId, blockId++)
+      functionTokens = functionTokens.push(tokenizedFunction)
+    }
+  })
+
+  return functionTokens
+}
+
+
+const outputFile = function(functionTokens) {
+  functionTokens.forEach((f) => {
+    //console.log("===")
+    console.log(f)
+    //console.log("===")
+  })
+}
+
+// TODO: check input
+const fileName = process.argv[2]
+
+fs.readFileAsync(fileName).then((data) => {
+  outputFile(processFile(process.argv[3], data))
+});

tokenizers/javascript/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "jstokenizer",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Jakub Žitný <jakubzitny@avocode.com> (https://github.com/jakubzitny)",
+  "license": "ISC",
+  "dependencies": {
+    "escodegen": "^1.8.0",
+    "esprima": "^2.7.2",
+    "esprima-ast-utils": "0.0.6",
+    "esprima-walk": "^0.1.0",
+    "estools": "^2.1.0",
+    "fs-extra-promise": "^0.3.1",
+    "immutable": "^3.8.1",
+    "lodash": "^4.13.1"
+  }
+}

tokenizers/javascript/readme.md

@@ -0,0 +1,5 @@
+# JavaScript tokenizer for SourcererCC
+
+- use `node 6.*`
+- run as `node index.js /path/to/file.js`
+- (carefully) use `batch.sh` to apply to larger dataset

tokenizers/javascript/tokenizer.js

@@ -0,0 +1,65 @@
+const _ = require('lodash')
+const immutable = require('immutable')
+const fs = require('fs-extra-promise')
+const esprima = require('esprima')
+
+const MAIN_DELIMITER = '@#@'
+const COUNT_DELIMITER = '@@::@@'
+const TOKEN_DELIMITER = ','
+
+const filterTokens = function (type, token) {
+  return token.type == type
+}
+
+
+// NOTE: http://esprima.org/doc/#usage
+const tokenTypes = immutable.List.of(
+  'Boolean',
+  'Identifier',
+  'Keyword',
+  'Null',
+  'Numeric', 
+  'Punctuator',
+  'String',
+  'RegularExpression'
+)
+
+const tokenFilters = tokenTypes.map((tokenType) => {
+  return _.partial(filterTokens, tokenType)
+})
+
+const tokenizer = function(code, parentId, blockId) {
+  const options = { }
+  const tokens = immutable.List(esprima.tokenize(code, options))
+
+  // TODO: reduce to map
+  // const filteredTokens = tokenFilters.map((tokenFilter) => {
+  //  return tokens.filter(tokenFilter)
+  // })
+
+  let uniqueTokens = immutable.Map()
+  tokens.forEach((token) => {
+    if (uniqueTokens.has(token.value)) {
+      newUniqueTokens = uniqueTokens.updateIn(
+        [ token.value ],
+        (count) => {
+          return count + 1
+        })
+    } else {
+      newUniqueTokens = uniqueTokens.set(token.value, 1)
+    }
+    uniqueTokens = newUniqueTokens
+  })
+
+  const tokenPairs = uniqueTokens.map((count, token) => {
+    return `${token}${COUNT_DELIMITER}${count}`
+  })
+
+  const lhs = `${parentId},${blockId},`
+  const rhs = tokenPairs.join(TOKEN_DELIMITER)
+  const output = `${lhs}${MAIN_DELIMITER}${rhs}`
+
+  return output
+};
+
+module.exports = tokenizer

tokenizers/javascript/util.js

@@ -0,0 +1,10 @@
+
+const base64FileName = function(fileName) {
+  const fileNameBuffer = Buffer.from(fileName)
+  return fileNameBuffer.toString('base64')
+}
+
+
+module.exports = {
+  base64FileName
+}