Skip to content

Commit b914e87

Browse files
brentlintnerbrentlintner
committed
v0.4.0
1 parent 200b928 commit b914e87

File tree

14 files changed

+495
-6
lines changed

14 files changed

+495
-6
lines changed

‎CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Synt Changelog
2+
3+
Please see the GithHub [releases](https://github.com/brentlintner/synt/releases) section.

‎README.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# synt [![Circle CI](https://circleci.com/gh/brentlintner/synt.svg?style=shield)](https://circleci.com/gh/brentlintner/synt) [![score-badge](https://vile.io/api/v0/projects/synt/badges/score?token=USryyHar5xQs7cBjNUdZ)](https://vile.io/~brentlintner/synt) [![security-badge](https://vile.io/api/v0/projects/synt/badges/security?token=USryyHar5xQs7cBjNUdZ)](https://vile.io/~brentlintner/synt) [![coverage-badge](https://vile.io/api/v0/projects/synt/badges/coverage?token=USryyHar5xQs7cBjNUdZ)](https://vile.io/~brentlintner/synt) [![dependency-badge](https://vile.io/api/v0/projects/synt/badges/dependency?token=USryyHar5xQs7cBjNUdZ)](https://vile.io/~brentlintner/synt) [![npm version](https://badge.fury.io/js/synt.svg)](https://badge.fury.io/js/synt)
22

3+
![demo image](https://user-images.githubusercontent.com/93340/26853130-c50f2724-4ade-11e7-905e-6923af2a759d.png)
4+
35
Find similar functions and classes in your JavaScript/TypeScript code.
46

57
## Supported Languages
@@ -34,7 +36,9 @@ synt -h
3436
*example*
3537

3638
```sh
37-
synt analyze lib
39+
git clone https://github.com/brentlintner/synt.git
40+
cd synt
41+
synt analyze src
3842
```
3943

4044
### Library

‎lib/cli.js

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
"use strict";
2+
var program = require("commander");
3+
var similar = require("./similar");
4+
var fs_collector = require("./cli/file_collector");
5+
var pkg = require("./../package.json");
6+
var compare = function (targets, opts) {
7+
var files = fs_collector.files(targets);
8+
var nocolors = !!opts.disablecolors;
9+
fs_collector.print(files, nocolors);
10+
var _a = similar.compare(files, opts), js = _a.js, ts = _a.ts;
11+
similar.print(js, nocolors);
12+
similar.print(ts, nocolors);
13+
};
14+
var configure = function () {
15+
program
16+
.version(pkg.version)
17+
.command("analyze [paths...]")
18+
.alias("a")
19+
.option("-s, --similarity [number]", "Lowest % similarity to look for " +
20+
("[default=" + similar.DEFAULT_THRESHOLD + "]."))
21+
.option("-m, --minlength [number]", "Default token length a function needs to be to compare it " +
22+
("[default=" + similar.DEFAULT_TOKEN_LENGTH + "]."))
23+
.option("-n, --ngram [number]", "Specify ngram length for comparing token sequences. " +
24+
("[default=" + similar.DEFAULT_NGRAM_LENGTH + ",2,3...]"))
25+
.option("-d, --disablecolors", "Disable color output")
26+
.action(compare);
27+
program.on("--help", function () {
28+
console.log(" Command specific help:");
29+
console.log("");
30+
console.log(" {cmd} -h, --help");
31+
console.log("");
32+
console.log(" Examples:");
33+
console.log("");
34+
console.log(" $ synt analyze lib");
35+
console.log(" $ synt analyze -s 90 foo.js bar.js baz.js");
36+
console.log("");
37+
});
38+
};
39+
var interpret = function (argv) {
40+
configure();
41+
program.parse(argv);
42+
};
43+
module.exports = { interpret: interpret };

‎lib/cli/file_collector.js

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
"use strict";
2+
var fs = require("fs");
3+
var path = require("path");
4+
var _ = require("lodash");
5+
var chalk = require("chalk");
6+
var walk_sync = require("walk-sync");
7+
var all_files = function (target) {
8+
if (fs.statSync(target).isDirectory()) {
9+
var dirs = walk_sync(target, { directories: false });
10+
return _.map(dirs, function (dir) { return path.join(target, dir); });
11+
}
12+
else {
13+
return [target];
14+
}
15+
};
16+
var normalize_cli_targets = function (targets) {
17+
targets = _.concat([], targets);
18+
var files = _.uniq(_.reduce(targets, function (paths, target) {
19+
return _.concat(paths, all_files(target));
20+
}, []));
21+
files = _.map(files, function (file) {
22+
return path.relative(process.cwd(), file);
23+
});
24+
return _.filter(files, function (file) {
25+
return /\.(js|ts)$/.test(file);
26+
});
27+
};
28+
var print_found = function (files, nocolors) {
29+
return _.each(files, function (file) {
30+
if (nocolors) {
31+
console.log("found:", file);
32+
}
33+
else {
34+
console.log(chalk.gray("found:"), chalk.green(file));
35+
}
36+
});
37+
};
38+
module.exports = {
39+
files: normalize_cli_targets,
40+
print: print_found
41+
};

‎lib/index.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"use strict";
2+
var similar = require("./similar");
3+
module.exports = similar;

‎lib/similar.js

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
"use strict";
2+
var _ = require("lodash");
3+
var ngram = require("./similar/ngram");
4+
var parse_js = require("./similar/javascript");
5+
var parse_ts = require("./similar/typescript");
6+
var print = require("./similar/print");
7+
var DEFAULT_NGRAM_LENGTH = 1;
8+
var DEFAULT_THRESHOLD = 70;
9+
var DEFAULT_TOKEN_LENGTH = 10;
10+
var similarity = function (src, cmp) {
11+
var a = _.uniq(src);
12+
var b = _.uniq(cmp);
13+
var i = _.intersection(a, b);
14+
var u = _.union(a, b);
15+
return _.toNumber(_.toNumber((i.length / u.length) * 100)
16+
.toFixed(0));
17+
};
18+
var parse_token_length = function (str) {
19+
return _.isEmpty(str) ?
20+
DEFAULT_TOKEN_LENGTH :
21+
_.toNumber(str);
22+
};
23+
var parse_ngram_length = function (str) {
24+
return _.isEmpty(str) ?
25+
DEFAULT_NGRAM_LENGTH :
26+
_.toNumber(str);
27+
};
28+
var parse_threshold = function (str) {
29+
var threshold = _.toNumber(str);
30+
return threshold || DEFAULT_THRESHOLD;
31+
};
32+
var is_ts_ancestor = function (src, cmp) {
33+
var match = false;
34+
var last = src.ast;
35+
while (true) {
36+
var parent_1 = last.parent;
37+
if (parent_1 === cmp.ast)
38+
match = true;
39+
if (!parent_1 || last === parent_1 || match)
40+
break;
41+
last = parent_1;
42+
}
43+
return match;
44+
};
45+
var false_positive = function (src, cmp, t_len) {
46+
var same_node = function () { return cmp.ast === src.ast; };
47+
var size_is_too_different = function () {
48+
var l1 = src.tokens.length;
49+
var l2 = cmp.tokens.length;
50+
return l1 * 2 < l2 || l2 * 2 < l1;
51+
};
52+
var one_is_too_short = function () {
53+
return src.tokens.length < t_len ||
54+
cmp.tokens.length < t_len;
55+
};
56+
var subset_of_other = function () {
57+
var is_eithers_ancestor = function () {
58+
return is_ts_ancestor(src, cmp) ||
59+
is_ts_ancestor(cmp, src);
60+
};
61+
var is_eithers_middle = function () {
62+
var src_j = src.tokens.join("");
63+
var cmp_j = cmp.tokens.join("");
64+
return src_j !== cmp_j &&
65+
(_.includes(src_j, cmp_j) ||
66+
_.includes(cmp_j, src_j));
67+
};
68+
return is_eithers_ancestor() || is_eithers_middle();
69+
};
70+
var both_are_not_classes = function () {
71+
return (src.is_class && !cmp.is_class) ||
72+
(!src.is_class && cmp.is_class);
73+
};
74+
return same_node() ||
75+
both_are_not_classes() ||
76+
subset_of_other() ||
77+
one_is_too_short() ||
78+
size_is_too_different();
79+
};
80+
var each_pair = function (items, callback) {
81+
_.each(items, function (src) {
82+
_.each(items, function (cmp) {
83+
callback(src, cmp);
84+
});
85+
});
86+
};
87+
var filter_redundencies = function (group) {
88+
_.each(group, function (results, sim) {
89+
group[sim] = _.reduce(results, function (new_arr, result) {
90+
var already_added = _.some(new_arr, function (result_two) {
91+
return _.xor(result, result_two).length === 0;
92+
});
93+
if (!already_added) {
94+
new_arr.push(result);
95+
}
96+
return new_arr;
97+
}, []);
98+
});
99+
return group;
100+
};
101+
var _compare = function (files, ftype, n_len, t_len, sim_min) {
102+
var is_ts = ftype === "ts";
103+
var is_file = is_ts ? /\.ts$/ : /\.js$/;
104+
files = _.filter(files, function (file) { return is_file.test(file); });
105+
var parse = is_ts ? parse_ts : parse_js;
106+
var items = parse.find(files);
107+
var group = {};
108+
each_pair(items, function (src, cmp) {
109+
if (false_positive(src, cmp, t_len))
110+
return;
111+
var src_grams = ngram.generate(src.tokens, n_len);
112+
var cmp_grams = ngram.generate(cmp.tokens, n_len);
113+
var val = similarity(src_grams, cmp_grams);
114+
if (val < sim_min)
115+
return;
116+
if (_.isEmpty(group[val]))
117+
group[val] = [];
118+
group[val].push([src, cmp]);
119+
});
120+
return filter_redundencies(group);
121+
};
122+
var compare = function (files, opts) {
123+
if (opts === void 0) { opts = {}; }
124+
files = _.concat([], files);
125+
var t_len = parse_token_length(_.toString(opts.minlength));
126+
var n_len = parse_ngram_length(_.toString(opts.ngram));
127+
var threshold = parse_threshold(_.toString(opts.similarity));
128+
var js_group = _compare(files, "js", n_len, t_len, threshold);
129+
var ts_group = _compare(files, "ts", n_len, t_len, threshold);
130+
return { js: js_group, ts: ts_group };
131+
};
132+
module.exports = {
133+
DEFAULT_NGRAM_LENGTH: DEFAULT_NGRAM_LENGTH,
134+
DEFAULT_THRESHOLD: DEFAULT_THRESHOLD,
135+
DEFAULT_TOKEN_LENGTH: DEFAULT_TOKEN_LENGTH,
136+
compare: compare,
137+
print: print.print
138+
};

‎lib/similar/javascript.js

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
"use strict";
2+
var fs = require("fs");
3+
var _ = require("lodash");
4+
var codegen = require("escodegen");
5+
var esprima = require("esprima");
6+
var estraverse = require("estraverse");
7+
var FUNCTION_OR_CLASS_NODE = [
8+
"ArrowFunctionExpression",
9+
"ClassDeclaration",
10+
"FunctionDeclaration",
11+
"FunctionExpression"
12+
];
13+
var normalize = function (token_list) {
14+
return _.map(token_list, function (t) { return t.value; });
15+
};
16+
var tokenize = function (code) {
17+
return normalize(esprima.tokenize(code));
18+
};
19+
var astify = function (code) {
20+
return esprima.parse(code, { loc: true });
21+
};
22+
var ast_to_code = function (node) {
23+
var opts = { format: { indent: { style: " " } } };
24+
return codegen.generate(node, opts);
25+
};
26+
var is_a_method_or_class = function (node) {
27+
return _.some(FUNCTION_OR_CLASS_NODE, function (type) { return type === node.type; });
28+
};
29+
var line_info = function (node) { return node.loc; };
30+
var parse_methods_and_classes = function (root_node, filepath) {
31+
var entries = [];
32+
estraverse.traverse(root_node, {
33+
enter: function (node, parent) {
34+
if (!is_a_method_or_class(node))
35+
return;
36+
var method = ast_to_code(node);
37+
var tokens = tokenize(method);
38+
var result = {
39+
ast: node,
40+
code: method,
41+
is_class: node.type === "ClassDeclaration",
42+
path: filepath,
43+
pos: line_info(node),
44+
tokens: tokens,
45+
type: node.type
46+
};
47+
entries.push(result);
48+
}
49+
});
50+
return entries;
51+
};
52+
var find_similar_methods_and_classes = function (filepaths) {
53+
return _.flatMap(filepaths, function (filepath) {
54+
var code = fs.readFileSync(filepath).toString();
55+
var node = astify(code);
56+
return parse_methods_and_classes(node, filepath);
57+
});
58+
};
59+
module.exports = {
60+
find: find_similar_methods_and_classes
61+
};

‎lib/similar/ngram.js

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
"use strict";
2+
var generate = function (arr, len) {
3+
if (len === void 0) { len = 1; }
4+
if (len > arr.length)
5+
len = 1;
6+
if (len == 1)
7+
return arr;
8+
var sets = [];
9+
arr.forEach(function (token, index) {
10+
var s_len = index + len;
11+
if (s_len <= arr.length) {
12+
sets.push(arr.slice(index, s_len).join(""));
13+
}
14+
});
15+
return sets;
16+
};
17+
module.exports = { generate: generate };

‎lib/similar/print.js

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
"use strict";
2+
var _ = require("lodash");
3+
var chalk = require("chalk");
4+
var cardinal = require("cardinal");
5+
var print = function (group, nocolors) {
6+
_.each(group, function (results, sim) {
7+
_.each(results, function (result) {
8+
var src = result[0], cmp = result[1];
9+
console.log("");
10+
var match_sim = sim + "% match";
11+
if (nocolors) {
12+
console.log(match_sim);
13+
}
14+
else {
15+
console.log(chalk.white.bgRed.bold(match_sim));
16+
}
17+
console.log("");
18+
if (nocolors) {
19+
console.log("in: " + src.path);
20+
}
21+
else {
22+
console.log(chalk.gray("in: ") + chalk.green(src.path));
23+
}
24+
console.log("");
25+
if (nocolors) {
26+
console.log(src.code);
27+
}
28+
else {
29+
console.log(cardinal.highlight(src.code, {
30+
firstline: src.pos.start.line,
31+
linenos: true
32+
}));
33+
}
34+
console.log("");
35+
if (src.path !== cmp.path) {
36+
if (nocolors) {
37+
console.log("in: " + cmp.path);
38+
}
39+
else {
40+
console.log(chalk.gray("in: ") + chalk.green(cmp.path));
41+
}
42+
console.log("");
43+
}
44+
if (nocolors) {
45+
console.log(cmp.code);
46+
}
47+
else {
48+
console.log(cardinal.highlight(cmp.code, {
49+
firstline: cmp.pos.start.line,
50+
linenos: true
51+
}));
52+
}
53+
});
54+
});
55+
};
56+
module.exports = { print: print };

0 commit comments

Comments
 (0)