From c9b4ed528e7e48c09fe86dc4a27bbe2f97872ae9 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 26 Dec 2018 16:10:23 +0100 Subject: [PATCH 1/9] exclude build artifacts from git Signed-off-by: Alexander Bezzubov --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 57b21f4e..c2cb0a9f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ benchmarks/output Makefile.main .shared .idea -.docsrv-resources \ No newline at end of file +.docsrv-resources +build/ +java/lib/ From 2a66ddd3e2d62399d96a564e4711246b17de6f80 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 26 Dec 2018 16:10:56 +0100 Subject: [PATCH 2/9] build: simplify building by using src-d/ci Signed-off-by: Alexander Bezzubov --- Makefile | 9 --------- README.md | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 8ae908dd..b9151af0 100644 --- a/Makefile +++ b/Makefile @@ -23,12 +23,6 @@ $(DOCS_PATH)/Makefile.inc: LINGUIST_PATH = .linguist -# build CLI -LOCAL_TAG := $(shell git describe --tags --abbrev=0) -LOCAL_COMMIT := $(shell git rev-parse --short HEAD) -LOCAL_BUILD := $(shell date +"%m-%d-%Y_%H_%M_%S") -LOCAL_LDFLAGS = -s -X main.version=$(LOCAL_TAG) -X main.build=$(LOCAL_BUILD) -X main.commit=$(LOCAL_COMMIT) - # shared objects RESOURCES_DIR=./.shared LINUX_DIR=$(RESOURCES_DIR)/linux-x86-64 @@ -63,9 +57,6 @@ benchmarks-slow: $(LINGUST_PATH) mkdir -p benchmarks/output && go test -run=NONE -bench=. -slow -benchtime=100ms -timeout=100h >benchmarks/output/enry_samples.bench && \ benchmarks/linguist-samples.rb 5 >benchmarks/output/linguist_samples.bench -build-cli: - go build -o enry -ldflags "$(LOCAL_LDFLAGS)" cmd/enry/main.go - linux-shared: $(LINUX_SHARED_LIB) darwin-shared: $(DARWIN_SHARED_LIB) diff --git a/README.md b/README.md index e9cc08f4..968f1c3d 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ go get gopkg.in/src-d/enry.v1/... To build enry's CLI you must run - make build-cli + make build this will generate a binary in the project's root directory called `enry`. You can then move this binary to anywhere in your `PATH`. From 9e0eee1795c57aa16f40649cc81e84bc295a33b3 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 26 Dec 2018 17:33:48 +0100 Subject: [PATCH 3/9] bench: simplify shell runners Signed-off-by: Alexander Bezzubov --- benchmarks/parse.sh | 9 +++++---- benchmarks/run-benchmarks.sh | 7 ++++--- benchmarks/run.sh | 10 +++++++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/benchmarks/parse.sh b/benchmarks/parse.sh index 3baa900a..ac5cedac 100755 --- a/benchmarks/parse.sh +++ b/benchmarks/parse.sh @@ -1,5 +1,6 @@ -#!/bin/sh - -cd benchmarks/output && go run ../parser/main.go -outdir ../csv && \ -cd ../csv && go run ../parser/main.go -distribution +#!/usr/bin/env bash +cd benchmarks/output +go run ../parser/main.go -outdir ../csv +cd ../csv +go run ../parser/main.go -distribution diff --git a/benchmarks/run-benchmarks.sh b/benchmarks/run-benchmarks.sh index aa6e1ef2..7df766c8 100755 --- a/benchmarks/run-benchmarks.sh +++ b/benchmarks/run-benchmarks.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/usr/bin/env bash -mkdir -p benchmarks/output && go test -run NONE -bench=. -benchtime=120s -timeout=100h >benchmarks/output/enry_total.bench && \ -benchmarks/linguist-total.rb 5 >benchmarks/output/linguist_total.bench +mkdir -p benchmarks/output +go test -run NONE -bench=. -benchtime=120s -timeout=100h > benchmarks/output/enry_total.bench +benchmarks/linguist-total.rb 5 > benchmarks/output/linguist_total.bench diff --git a/benchmarks/run.sh b/benchmarks/run.sh index 1ae74cbc..1d13b855 100755 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -1,4 +1,8 @@ -#!/bin/sh +#!/usr/bin/env bash -benchmarks/run-benchmarks.sh && make benchmarks-slow && \ -benchmarks/parse.sh && benchmarks/plot-histogram.gp +set -e + +benchmarks/run-benchmarks.sh +make benchmarks-slow +benchmarks/parse.sh +benchmarks/plot-histogram.gp From 9b690f51ae619af1ecd722a13ab281e2f9d94934 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 26 Dec 2018 20:22:34 +0100 Subject: [PATCH 4/9] build: simplify benchmarks* targets Signed-off-by: Alexander Bezzubov --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index b9151af0..5b051739 100644 --- a/Makefile +++ b/Makefile @@ -48,14 +48,17 @@ code-generate: $(LINGUIST_PATH) go run internal/code-generator/main.go benchmarks: $(LINGUIST_PATH) - go test -run=NONE -bench=. && benchmarks/linguist-total.sh + go test -run=NONE -bench=. + benchmarks/linguist-total.rb benchmarks-samples: $(LINGUIST_PATH) - go test -run=NONE -bench=. -benchtime=5us && benchmarks/linguist-samples.rb + go test -run=NONE -bench=. -benchtime=5us + benchmarks/linguist-samples.rb benchmarks-slow: $(LINGUST_PATH) - mkdir -p benchmarks/output && go test -run=NONE -bench=. -slow -benchtime=100ms -timeout=100h >benchmarks/output/enry_samples.bench && \ - benchmarks/linguist-samples.rb 5 >benchmarks/output/linguist_samples.bench + mkdir -p benchmarks/output + go test -run=NONE -bench=. -slow -benchtime=100ms -timeout=100h > benchmarks/output/enry_samples.bench + benchmarks/linguist-samples.rb 5 > benchmarks/output/linguist_samples.bench linux-shared: $(LINUX_SHARED_LIB) From 1aa05d38d88b7d8c697e0c6a8bad039be97201af Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 26 Dec 2018 20:23:31 +0100 Subject: [PATCH 5/9] test: remove dependency on single test suite Signed-off-by: Alexander Bezzubov --- common_test.go | 2 +- utils_test.go | 33 +++++++++++++++++---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/common_test.go b/common_test.go index b2a5f153..5270aabc 100644 --- a/common_test.go +++ b/common_test.go @@ -24,7 +24,7 @@ type EnryTestSuite struct { cloned bool } -func TestEnryTestSuite(t *testing.T) { +func Test_EnryTestSuite(t *testing.T) { suite.Run(t, new(EnryTestSuite)) } diff --git a/utils_test.go b/utils_test.go index 928fe409..c75f8065 100644 --- a/utils_test.go +++ b/utils_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -func (s *EnryTestSuite) TestIsAuxiliaryLanguage() { +func TestIsAuxiliaryLanguage(t *testing.T) { type testType struct { name string lang string @@ -30,11 +30,12 @@ func (s *EnryTestSuite) TestIsAuxiliaryLanguage() { for _, test := range tests { is := IsAuxiliaryLanguage(test.lang) - assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) + assert.Equal(t, is, test.expected, + fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) } } -func (s *EnryTestSuite) TestIsVendor() { +func TestIsVendor(t *testing.T) { tests := []struct { name string path string @@ -53,11 +54,11 @@ func (s *EnryTestSuite) TestIsVendor() { for _, test := range tests { is := IsVendor(test.path) - assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) + assert.Equal(t, is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) } } -func (s *EnryTestSuite) TestIsDocumentation() { +func TestIsDocumentation(t *testing.T) { tests := []struct { name string path string @@ -69,11 +70,11 @@ func (s *EnryTestSuite) TestIsDocumentation() { for _, test := range tests { is := IsDocumentation(test.path) - assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) + assert.Equal(t, is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) } } -func (s *EnryTestSuite) TestIsImage() { +func TestIsImage(t *testing.T) { tests := []struct { name string path string @@ -88,11 +89,11 @@ func (s *EnryTestSuite) TestIsImage() { for _, test := range tests { is := IsImage(test.path) - assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) + assert.Equal(t, is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) } } -func (s *EnryTestSuite) TestGetMimeType() { +func TestGetMimeType(t *testing.T) { tests := []struct { name string path string @@ -106,11 +107,11 @@ func (s *EnryTestSuite) TestGetMimeType() { for _, test := range tests { is := GetMimeType(test.path, test.lang) - assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) + assert.Equal(t, is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) } } -func (s *EnryTestSuite) TestIsConfiguration() { +func TestIsConfiguration(t *testing.T) { tests := []struct { name string path string @@ -123,11 +124,11 @@ func (s *EnryTestSuite) TestIsConfiguration() { for _, test := range tests { is := IsConfiguration(test.path) - assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) + assert.Equal(t, is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) } } -func (s *EnryTestSuite) TestIsBinary() { +func TestIsBinary(t *testing.T) { tests := []struct { name string data []byte @@ -140,11 +141,11 @@ func (s *EnryTestSuite) TestIsBinary() { for _, test := range tests { is := IsBinary(test.data) - assert.Equal(s.T(), is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) + assert.Equal(t, is, test.expected, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) } } -func (s *EnryTestSuite) TestIsDotFile() { +func TestIsDotFile(t *testing.T) { tests := []struct { name string path string @@ -156,7 +157,7 @@ func (s *EnryTestSuite) TestIsDotFile() { for _, test := range tests { is := IsDotFile(test.path) - assert.Equal(s.T(), test.expected, is, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) + assert.Equal(t, test.expected, is, fmt.Sprintf("%v: is = %v, expected: %v", test.name, is, test.expected)) } } From b2d9831005c649e670b618a26bf38c20ea7fe349 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 26 Dec 2018 20:24:20 +0100 Subject: [PATCH 6/9] doc: rel image link + linguist cli difference highlight Signed-off-by: Alexander Bezzubov --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 968f1c3d..acf1c5ce 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ expression](https://github.com/github/linguist/blob/master/lib/linguist/heuristi for `*.sql` files right. This expression doesn't comply with the pattern for the rest in [heuristics.rb](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.rb). +`enry` [CLI tool](#cli) does not require a full Git repository to be present in filesystem in order to report languages. Benchmarks ------------ @@ -203,7 +204,7 @@ Enry's language detection has been compared with Linguist's one. In order to do We got these results: -![histogram](https://raw.githubusercontent.com/src-d/enry/master/benchmarks/histogram/distribution.png) +![histogram](benchmarks/histogram/distribution.png) The histogram represents the number of files for which spent time in language detection was in the range of the time interval indicated in the x axis. From 5e330682416f3d645320462ba90a586fc990855a Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Wed, 26 Dec 2018 21:20:49 +0100 Subject: [PATCH 7/9] suggestions from code review Co-Authored-By: bzz --- Makefile | 2 +- benchmarks/parse.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5b051739..e80eb1e9 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ benchmarks-samples: $(LINGUIST_PATH) go test -run=NONE -bench=. -benchtime=5us benchmarks/linguist-samples.rb -benchmarks-slow: $(LINGUST_PATH) + benchmarks-slow: $(LINGUIST_PATH) mkdir -p benchmarks/output go test -run=NONE -bench=. -slow -benchtime=100ms -timeout=100h > benchmarks/output/enry_samples.bench benchmarks/linguist-samples.rb 5 > benchmarks/output/linguist_samples.bench diff --git a/benchmarks/parse.sh b/benchmarks/parse.sh index ac5cedac..bf04ed2b 100755 --- a/benchmarks/parse.sh +++ b/benchmarks/parse.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - + set -e cd benchmarks/output go run ../parser/main.go -outdir ../csv cd ../csv From 002aaec3fbe825d387d8da6f424f0d27bee74373 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 26 Dec 2018 21:34:06 +0100 Subject: [PATCH 8/9] bench: add fail fast to all shell runners Signed-off-by: Alexander Bezzubov --- benchmarks/parse.sh | 3 ++- benchmarks/run-benchmarks.sh | 1 + benchmarks/run.sh | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/parse.sh b/benchmarks/parse.sh index bf04ed2b..77b6f924 100755 --- a/benchmarks/parse.sh +++ b/benchmarks/parse.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash - set -e +set -e + cd benchmarks/output go run ../parser/main.go -outdir ../csv cd ../csv diff --git a/benchmarks/run-benchmarks.sh b/benchmarks/run-benchmarks.sh index 7df766c8..d74bee55 100755 --- a/benchmarks/run-benchmarks.sh +++ b/benchmarks/run-benchmarks.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -e mkdir -p benchmarks/output go test -run NONE -bench=. -benchtime=120s -timeout=100h > benchmarks/output/enry_total.bench diff --git a/benchmarks/run.sh b/benchmarks/run.sh index 1d13b855..67d32775 100755 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - set -e benchmarks/run-benchmarks.sh From 8f80533c463f615f2262d32c5248aacdb1c0eb1a Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 26 Dec 2018 21:34:20 +0100 Subject: [PATCH 9/9] bench: fix makefile targets Signed-off-by: Alexander Bezzubov --- Makefile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index e80eb1e9..9c60ff26 100644 --- a/Makefile +++ b/Makefile @@ -44,20 +44,20 @@ clean-shared: clean: clean-linguist clean-shared code-generate: $(LINGUIST_PATH) - mkdir -p data + mkdir -p data && \ go run internal/code-generator/main.go benchmarks: $(LINGUIST_PATH) - go test -run=NONE -bench=. + go test -run=NONE -bench=. && \ benchmarks/linguist-total.rb benchmarks-samples: $(LINGUIST_PATH) - go test -run=NONE -bench=. -benchtime=5us + go test -run=NONE -bench=. -benchtime=5us && \ benchmarks/linguist-samples.rb - benchmarks-slow: $(LINGUIST_PATH) - mkdir -p benchmarks/output - go test -run=NONE -bench=. -slow -benchtime=100ms -timeout=100h > benchmarks/output/enry_samples.bench +benchmarks-slow: $(LINGUIST_PATH) + mkdir -p benchmarks/output && \ + go test -run=NONE -bench=. -slow -benchtime=100ms -timeout=100h > benchmarks/output/enry_samples.bench && \ benchmarks/linguist-samples.rb 5 > benchmarks/output/linguist_samples.bench linux-shared: $(LINUX_SHARED_LIB) @@ -73,3 +73,5 @@ $(LINUX_SHARED_LIB): mkdir -p $(LINUX_DIR) && \ GOOS=linux GOARCH=amd64 go build -buildmode=c-shared -o $(LINUX_SHARED_LIB) $(NATIVE_LIB) && \ mv $(LINUX_DIR)/$(HEADER_FILE) $(RESOURCES_DIR)/$(HEADER_FILE) + +.PHONY: benchmarks benchmarks-samples benchmarks-slow