From 62ebd7ab75cb89b0c5b673032bc78a04512311e5 Mon Sep 17 00:00:00 2001 From: Tim Ramlot <42113979+inteon@users.noreply.github.com> Date: Mon, 10 Apr 2023 00:17:45 +0200 Subject: [PATCH 1/4] remove unused git logic Signed-off-by: Tim Ramlot <42113979+inteon@users.noreply.github.com> --- go.mod | 14 ------ go.sum | 66 ---------------------------- licenses/git.go | 100 ------------------------------------------- licenses/git_test.go | 79 ---------------------------------- 4 files changed, 259 deletions(-) delete mode 100644 licenses/git.go delete mode 100644 licenses/git_test.go diff --git a/go.mod b/go.mod index 13917c68..733f72da 100644 --- a/go.mod +++ b/go.mod @@ -13,30 +13,16 @@ require ( golang.org/x/net v0.9.0 golang.org/x/text v0.9.0 golang.org/x/tools v0.8.0 - gopkg.in/src-d/go-git.v4 v4.13.1 k8s.io/klog/v2 v2.90.1 ) require ( - github.com/Microsoft/go-winio v0.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emirpasic/gods v1.12.0 // indirect github.com/go-logr/logr v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/martian/v3 v3.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/src-d/gcfg v1.4.0 // indirect - github.com/xanzy/ssh-agent v0.3.0 // indirect - golang.org/x/crypto v0.1.0 // indirect golang.org/x/sys v0.7.0 // indirect - gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect - gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index f28bacd4..60636549 100644 --- a/go.sum +++ b/go.sum @@ -41,15 +41,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= -github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -59,13 +50,9 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -73,9 +60,6 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -163,58 +147,31 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= -github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/otiai10/copy v1.10.0 h1:znyI7l134wNg/wDktoVQPxPkgvhDfGCYUasey+h0rDQ= github.com/otiai10/copy v1.10.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -222,9 +179,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= -github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= -github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -240,16 +194,12 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -358,9 +308,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -369,7 +317,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -391,7 +338,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -410,7 +356,6 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -439,7 +384,6 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -596,17 +540,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/licenses/git.go b/licenses/git.go deleted file mode 100644 index c8b8a7ff..00000000 --- a/licenses/git.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2019 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package licenses - -import ( - "fmt" - "net/url" - "path" - "path/filepath" - "regexp" - "strings" - - git "gopkg.in/src-d/go-git.v4" - "k8s.io/klog/v2" -) - -var ( - gitRegexp = regexp.MustCompile(`^\.git$`) - - // TODO(RJPercival): Support replacing "master" with Go Module version - gitRepoPathPrefixes = map[string]string{ - "github.com": "blob/master/", - "bitbucket.org": "src/master/", - "go.googlesource.com": "+/refs/heads/master/", - "code.googlesource.com": "+/refs/heads/master/", - } -) - -// GitRepo represents a Git repository that exists on disk locally. -type GitRepo struct { - dotGitPath string -} - -// FindGitRepo finds the Git repository that contains the specified filePath -// by searching upwards through the directory tree for a ".git" directory. -func FindGitRepo(filePath string) (*GitRepo, error) { - // TODO(Bobgy): the "/" is used just to fix the test. git.go is not - // currently used, but I plan to bring it back to detect version of the - // main module in following up PRs. - path, err := findUpwards(filepath.Dir(filePath), gitRegexp, "/", nil) - if err != nil { - return nil, err - } - return &GitRepo{dotGitPath: path}, nil -} - -// FileURL returns the URL of a file stored in a Git repository. -// It uses the URL of the specified Git remote repository to construct this URL. -// It supports repositories hosted on github.com, bitbucket.org and googlesource.com. -func (g *GitRepo) FileURL(filePath string, remote string) (*url.URL, error) { - relFilePath, err := filepath.Rel(filepath.Dir(g.dotGitPath), filePath) - if err != nil { - return nil, err - } - repoURL, err := gitRemoteURL(g.dotGitPath, remote) - if err != nil { - return nil, err - } - repoURL.Host = strings.TrimSuffix(repoURL.Host, ".") - repoURL.Path = strings.TrimSuffix(repoURL.Path, ".git") - if prefix, ok := gitRepoPathPrefixes[repoURL.Host]; ok { - repoURL.Path = path.Join(repoURL.Path, prefix, filepath.ToSlash(relFilePath)) - } else { - return nil, fmt.Errorf("unrecognised Git repository host: %q", repoURL) - } - - return repoURL, nil -} - -func gitRemoteURL(repoPath string, remoteName string) (*url.URL, error) { - repo, err := git.PlainOpen(repoPath) - if err != nil { - return nil, err - } - remote, err := repo.Remote(remoteName) - if err != nil { - return nil, err - } - for _, urlStr := range remote.Config().URLs { - u, err := url.Parse(urlStr) - if err != nil { - klog.Warningf("Error parsing %q as URL from remote %q in Git repo at %q: %s", urlStr, remoteName, repoPath, err) - continue - } - return u, nil - } - return nil, fmt.Errorf("the Git remote %q does not have a valid URL", remoteName) -} diff --git a/licenses/git_test.go b/licenses/git_test.go deleted file mode 100644 index eae8dce0..00000000 --- a/licenses/git_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2019 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package licenses - -import ( - "os" - "path/filepath" - "testing" - - git "gopkg.in/src-d/go-git.v4" -) - -func TestGitFileURL(t *testing.T) { - t.Parallel() - - dir, err := os.MkdirTemp("", "git_test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - cloneOpts := git.CloneOptions{ - URL: "https://github.com/google/trillian", - Depth: 1, - } - if _, err := git.PlainClone(dir, false, &cloneOpts); err != nil { - t.Fatal(err) - } - - for _, test := range []struct { - desc string - file string - remote string - wantURL string - wantErr error - }{ - { - desc: "License URL", - file: filepath.Join(dir, "LICENSE"), - remote: "origin", - wantURL: "https://github.com/google/trillian/blob/master/LICENSE", - }, - { - desc: "Non-existent remote", - file: filepath.Join(dir, "LICENSE"), - remote: "foo", - wantErr: git.ErrRemoteNotFound, - }, - } { - t.Run(test.desc, func(t *testing.T) { - repo, err := FindGitRepo(test.file) - if err != nil { - t.Fatalf("FindGitRepo(%q) = (_, %q), want (_, nil)", test.file, err) - } - url, err := repo.FileURL(test.file, test.remote) - if err != nil { - if err != test.wantErr { - t.Fatalf("repo.FileURL(%q, %q) = (_, %q), want (_, %q)", test.file, test.remote, err, test.wantErr) - } - return - } - if url.String() != test.wantURL { - t.Fatalf("repo.FileURL(%q, %q) = (%q, nil), want (%q, nil)", test.file, test.remote, url, test.wantURL) - } - }) - } -} From da83c6d61893bb2e9309cac0da1fd93cbf7615b1 Mon Sep 17 00:00:00 2001 From: Tim Ramlot <42113979+inteon@users.noreply.github.com> Date: Wed, 26 Apr 2023 17:40:53 +0200 Subject: [PATCH 2/4] only Identify once & optimise logic Signed-off-by: Tim Ramlot <42113979+inteon@users.noreply.github.com> --- check.go | 9 +- go.mod | 1 + licenses/classifier.go | 7 +- licenses/find.go | 55 +++---- licenses/find_test.go | 178 +++++++++------------- licenses/library.go | 313 ++++++++++++++++++++++++++------------- licenses/library_test.go | 108 +++++++++----- licenses/module.go | 4 +- report.go | 113 +++++++++----- save.go | 17 +-- 10 files changed, 465 insertions(+), 340 deletions(-) diff --git a/check.go b/check.go index 79739dca..4c00125a 100644 --- a/check.go +++ b/check.go @@ -81,18 +81,13 @@ func checkMain(_ *cobra.Command, args []string) error { found := false for _, lib := range libs { - if lib.LicensePath == "" { + if lib.LicenseFile == "" { fmt.Fprintf(os.Stderr, "Did not find license for library '%v'.\n", lib) found = true continue } - licenses, err := classifier.Identify(lib.LicensePath) - if err != nil { - return err - } - - for _, license := range licenses { + for _, license := range lib.Licenses { if hasLicenseNames && !isAllowedLicenseName(license.Name, allowedLicenseNames) { fmt.Fprintf(os.Stderr, "Not allowed license '%s' found for library '%v'.\n", license.Name, lib) found = true diff --git a/go.mod b/go.mod index 733f72da..bcf4d2c1 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( go.opencensus.io v0.24.0 golang.org/x/mod v0.10.0 golang.org/x/net v0.9.0 + golang.org/x/sync v0.1.0 golang.org/x/text v0.9.0 golang.org/x/tools v0.8.0 k8s.io/klog/v2 v2.90.1 diff --git a/licenses/classifier.go b/licenses/classifier.go index 0a4415f3..3550990b 100644 --- a/licenses/classifier.go +++ b/licenses/classifier.go @@ -15,7 +15,6 @@ package licenses import ( - "fmt" "os" licenseclassifier "github.com/google/licenseclassifier/v2" @@ -83,9 +82,5 @@ func (c *googleClassifier) Identify(licensePath string) ([]License, error) { }) } - if len(matches.Matches) > 0 { - return licenses, nil - } else { - return nil, fmt.Errorf("no license found") - } + return licenses, nil } diff --git a/licenses/find.go b/licenses/find.go index 05d1bad2..1406041b 100644 --- a/licenses/find.go +++ b/licenses/find.go @@ -15,7 +15,6 @@ package licenses import ( - "errors" "fmt" "os" "path/filepath" @@ -27,63 +26,50 @@ var ( licenseRegexp = regexp.MustCompile(`^(?i)((UN)?LICEN(S|C)E|COPYING|README|NOTICE).*$`) ) -// Find returns the file path of the license for this package. +// FindCandidates returns the candidate file path of the license for this package. // // dir is path of the directory where we want to find a license. // rootDir is path of the module containing this package. Find will not search out of the // rootDir. -func Find(dir string, rootDir string, classifier Classifier) (string, error) { +func FindCandidates(dir string, rootDir string) ([]string, error) { dir, err := filepath.Abs(dir) if err != nil { - return "", err + return nil, err } + rootDir, err = filepath.Abs(rootDir) if err != nil { - return "", err + return nil, err } + if !strings.HasPrefix(dir, rootDir) { - return "", fmt.Errorf("licenses.Find: rootDir %s should contain dir %s", rootDir, dir) - } - found, err := findUpwards(dir, licenseRegexp, rootDir, func(path string) bool { - // TODO(RJPercival): Return license details - if _, err := classifier.Identify(path); err != nil { - return false - } - return true - }) - if err != nil { - if errors.Is(err, errNotFound) { - return "", fmt.Errorf("cannot find a known open source license for %q whose name matches regexp %s and locates up until %q", dir, licenseRegexp, rootDir) - } - return "", fmt.Errorf("finding a known open source license: %w", err) + return nil, fmt.Errorf("licenses.Find: rootDir %s should contain dir %s", rootDir, dir) } - return found, nil + + return findAllUpwards(dir, licenseRegexp, rootDir) } -var errNotFound = fmt.Errorf("file/directory matching predicate and regexp not found") +func findAllUpwards(dir string, r *regexp.Regexp, stopAt string) ([]string, error) { + var foundPaths []string -func findUpwards(dir string, r *regexp.Regexp, stopAt string, predicate func(path string) bool) (string, error) { - // Dir must be made absolute for reliable matching with stopAt regexps - dir, err := filepath.Abs(dir) - if err != nil { - return "", err - } - start := dir // Stop once we go out of the stopAt dir. for strings.HasPrefix(dir, stopAt) { dirContents, err := os.ReadDir(dir) if err != nil { - return "", err + return nil, err } + for _, f := range dirContents { + if f.IsDir() { + continue + } + if r.MatchString(f.Name()) { path := filepath.Join(dir, f.Name()) - if predicate != nil && !predicate(path) { - continue - } - return path, nil + foundPaths = append(foundPaths, path) } } + parent := filepath.Dir(dir) if parent == dir { // Can't go any higher up the directory tree. @@ -91,5 +77,6 @@ func findUpwards(dir string, r *regexp.Regexp, stopAt string, predicate func(pat } dir = parent } - return "", fmt.Errorf("findUpwards(dir=%q, regexp=%q, stopAt=%q, predicate=func): %w", start, r, stopAt, errNotFound) + + return foundPaths, nil } diff --git a/licenses/find_test.go b/licenses/find_test.go index 6b95184f..263cde59 100644 --- a/licenses/find_test.go +++ b/licenses/find_test.go @@ -17,152 +17,120 @@ package licenses import ( "os" "path/filepath" - "regexp" "testing" + + "github.com/google/go-cmp/cmp" ) -func TestFind(t *testing.T) { +func TestFindCandidates(t *testing.T) { wd, err := os.Getwd() if err != nil { t.Fatalf("Cannot get working directory: %v", err) } - classifier := classifierStub{ - licenses: map[string][]License{ - "testdata/LICENSE": { - { - Name: "foo", - Type: Notice, - }, - }, - "testdata/MIT/LICENSE.MIT": { - { - Name: "MIT", - Type: Notice, - }, - }, - "testdata/licence/LICENCE": { - { - Name: "foo", - Type: Notice, - }, - }, - "testdata/copying/COPYING": { - { - Name: "foo", - Type: Notice, - }, - }, - "testdata/notice/NOTICE.txt": { - { - Name: "foo", - Type: Notice, - }, - }, - "testdata/readme/README.md": { - { - Name: "foo", - Type: Notice, - }, - }, - "testdata/lowercase/license": { - { - Name: "foo", - Type: Notice, - }, - }, - "testdata/license-apache-2.0/LICENSE-APACHE-2.0.txt": { - { - Name: "foo", - Type: Notice, - }, - }, - "testdata/unlicense/UNLICENSE": { - { - Name: "unlicense", - Type: Unencumbered, - }, - }, - }, - } - for _, test := range []struct { - desc string - dir string - rootDir string - wantLicensePath string - wantErr *regexp.Regexp + desc string + dir string + rootDir string + wantLicensePathCandidates []string }{ { - desc: "licenSe", - dir: "testdata", - wantLicensePath: filepath.Join(wd, "testdata/LICENSE"), + desc: "licenSe", + dir: "testdata", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/LICENSE"), + }, }, { - desc: "licenCe", - dir: "testdata/licence", - wantLicensePath: filepath.Join(wd, "testdata/licence/LICENCE"), + desc: "licenCe", + dir: "testdata/licence", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/licence/LICENCE"), + filepath.Join(wd, "testdata/LICENSE"), + }, }, { - desc: "LICENSE.MIT", - dir: "testdata/MIT", - wantLicensePath: filepath.Join(wd, "testdata/MIT/LICENSE.MIT"), + desc: "LICENSE.MIT", + dir: "testdata/MIT", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/MIT/LICENSE.MIT"), + filepath.Join(wd, "testdata/LICENSE"), + }, }, { - desc: "COPYING", - dir: "testdata/copying", - wantLicensePath: filepath.Join(wd, "testdata/copying/COPYING"), + desc: "COPYING", + dir: "testdata/copying", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/copying/COPYING"), + filepath.Join(wd, "testdata/LICENSE"), + }, }, { - desc: "NOTICE", - dir: "testdata/notice", - wantLicensePath: filepath.Join(wd, "testdata/notice/NOTICE.txt"), + desc: "NOTICE", + dir: "testdata/notice", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/notice/NOTICE.txt"), + filepath.Join(wd, "testdata/LICENSE"), + }, }, { - desc: "README", - dir: "testdata/readme", - wantLicensePath: filepath.Join(wd, "testdata/readme/README.md"), + desc: "README", + dir: "testdata/readme", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/readme/README.md"), + filepath.Join(wd, "testdata/LICENSE")}, }, { - desc: "parent dir", - dir: "testdata/internal", - wantLicensePath: filepath.Join(wd, "testdata/LICENSE"), + desc: "parent dir", + dir: "testdata/internal", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/LICENSE"), + }, }, { - desc: "lowercase", - dir: "testdata/lowercase", - wantLicensePath: filepath.Join(wd, "testdata/lowercase/license"), + desc: "lowercase", + dir: "testdata/lowercase", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/lowercase/license"), + filepath.Join(wd, "testdata/LICENSE"), + }, }, { - desc: "license-apache-2.0.txt", - dir: "testdata/license-apache-2.0", - wantLicensePath: filepath.Join(wd, "testdata/license-apache-2.0/LICENSE-APACHE-2.0.txt"), + desc: "license-apache-2.0.txt", + dir: "testdata/license-apache-2.0", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/license-apache-2.0/LICENSE-APACHE-2.0.txt"), + filepath.Join(wd, "testdata/LICENSE"), + }, }, { desc: "proprietary-license", dir: "testdata/proprietary-license", rootDir: "testdata/proprietary-license", - wantErr: regexp.MustCompile(`cannot find a known open source license for.*testdata/proprietary-license.*whose name matches regexp.*and locates up until.*testdata/proprietary-license`), + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/proprietary-license/LICENSE"), + }, }, { - desc: "UNLICENSE", - dir: "testdata/unlicense", - wantLicensePath: filepath.Join(wd, "testdata/unlicense/UNLICENSE"), + desc: "UNLICENSE", + dir: "testdata/unlicense", + wantLicensePathCandidates: []string{ + filepath.Join(wd, "testdata/unlicense/UNLICENSE"), + filepath.Join(wd, "testdata/LICENSE"), + }, }, } { t.Run(test.desc, func(t *testing.T) { if test.rootDir == "" { test.rootDir = "./testdata" } - licensePath, err := Find(test.dir, test.rootDir, classifier) - if test.wantErr != nil { - if err == nil || !test.wantErr.Match([]byte(err.Error())) { - t.Fatalf("Find(%q) = %q, %q, want (%q, %q)", test.dir, licensePath, err, "", test.wantErr) - } - return + licensePathCandidates, err := FindCandidates(test.dir, test.rootDir) + if err != nil { + t.Fatalf("FindCandidates(%q) = (%#v, %q), want (%q, nil)", test.dir, licensePathCandidates, err, test.wantLicensePathCandidates) } - if err != nil || licensePath != test.wantLicensePath { - t.Fatalf("Find(%q) = (%#v, %q), want (%q, nil)", test.dir, licensePath, err, test.wantLicensePath) + + if diff := cmp.Diff(test.wantLicensePathCandidates, licensePathCandidates); diff != "" { + t.Fatalf("FindCandidates(%q) = %q, %q, want (%q, nil); diff (-want +got): %s", test.dir, licensePathCandidates, err, test.wantLicensePathCandidates, diff) } }) } diff --git a/licenses/library.go b/licenses/library.go index dbbe877d..073a64b5 100644 --- a/licenses/library.go +++ b/licenses/library.go @@ -21,22 +21,24 @@ import ( "path/filepath" "sort" "strings" - "time" "github.com/google/go-licenses/internal/third_party/pkgsite/source" + "golang.org/x/sync/errgroup" "golang.org/x/tools/go/packages" "k8s.io/klog/v2" ) // Library is a collection of packages covered by the same license file. type Library struct { - // LicensePath is the path of the file containing the library's license. - LicensePath string + // LicenseFile is the path of the file containing the library's license. + LicenseFile string // Packages contains import paths for Go packages in this library. // It may not be the complete set of all packages in the library. Packages []string // Parent go module. module *Module + // List of licenses for found at the LicenseFile. + Licenses []License } // PackagesError aggregates all Packages[].Errors into a single error. @@ -71,134 +73,240 @@ func Libraries(ctx context.Context, classifier Classifier, includeTests bool, ig return nil, err } - pkgs := map[string]*packages.Package{} - pkgsByLicense := make(map[string][]*packages.Package) - pkgErrorOccurred := false - otherErrorOccurred := false - packages.Visit(rootPkgs, func(p *packages.Package) bool { - if len(p.Errors) > 0 { - pkgErrorOccurred = true - return false - } - if isStdLib(p) { - // No license requirements for the Go standard library. - return false + vendoredSearch := []*Module{} + for _, parentPkg := range rootPkgs { + if parentPkg.Module == nil { + continue } - if includeTests && isTestBinary(p) { - // A test binary only imports the standard library, so we do not need to check its license. - // Moreover, Find below will return an error because pkgDir is not under p.Module.Dir - // as pkgDir is under GOCACHE instead. - return false + + module := newModule(parentPkg.Module) + if module.Dir == "" { + continue } - for _, i := range ignoredPaths { - if strings.HasPrefix(p.PkgPath, i) { - // Marked to be ignored. + + vendoredSearch = append(vendoredSearch, module) + } + + type pkgInfo struct { + // pkgPath is the import path of the package. + pkgPath string + // modulePath is the module path of the package. + modulePath string + + // pkgDir is the directory containing the package's source code. + pkgDir string + // moduleDir is the directory containing the module's source code. + moduleDir string + } + + allModules := map[string]*Module{} + allPackages := []pkgInfo{} + { + pkgErrorOccurred := false + otherErrorOccurred := false + packages.Visit(rootPkgs, func(p *packages.Package) bool { + if len(p.Errors) > 0 { + pkgErrorOccurred = true + return false + } + if isStdLib(p) { + // No license requirements for the Go standard library. + return false + } + if includeTests && isTestBinary(p) { + // A test binary only imports the standard library, so we do not need to check its license. + // Moreover, Find below will return an error because pkgDir is not under p.Module.Dir + // as pkgDir is under GOCACHE instead. + return false + } + for _, i := range ignoredPaths { + if strings.HasPrefix(p.PkgPath, i) { + // Marked to be ignored. + return true + } + } + + if len(p.OtherFiles) > 0 { + klog.Warningf("%q contains non-Go code that can't be inspected for further dependencies:\n%s", p.PkgPath, strings.Join(p.OtherFiles, "\n")) + } + + var pkgDir string + switch { + case len(p.GoFiles) > 0: + pkgDir = filepath.Dir(p.GoFiles[0]) + case len(p.CompiledGoFiles) > 0: + pkgDir = filepath.Dir(p.CompiledGoFiles[0]) + case len(p.OtherFiles) > 0: + pkgDir = filepath.Dir(p.OtherFiles[0]) + default: + // This package is empty - nothing to do. return true } - } - if len(p.OtherFiles) > 0 { - klog.Warningf("%q contains non-Go code that can't be inspected for further dependencies:\n%s", p.PkgPath, strings.Join(p.OtherFiles, "\n")) - } - var pkgDir string - switch { - case len(p.GoFiles) > 0: - pkgDir = filepath.Dir(p.GoFiles[0]) - case len(p.CompiledGoFiles) > 0: - pkgDir = filepath.Dir(p.CompiledGoFiles[0]) - case len(p.OtherFiles) > 0: - pkgDir = filepath.Dir(p.OtherFiles[0]) - default: - // This package is empty - nothing to do. + if p.Module == nil { + otherErrorOccurred = true + klog.Errorf("Package %s does not have module info. Non go modules projects are no longer supported. For feedback, refer to https://github.com/google/go-licenses/issues/128.", p.PkgPath) + return false + } + + module := newModule(p.Module) + + if module.Dir == "" { + // A known cause is that the module is vendored, so some information is lost. + isVendored := strings.Contains(pkgDir, "/vendor/") + if !isVendored { + klog.Warningf("module %s does not have dir and it's not vendored, cannot discover the license URL. Report to go-licenses developer if you see this.", module.Path) + } else { + // This is vendored. Handle this known special case. + + // Extra note why we identify a vendored package like this. + // + // For a normal package: + // * if it's not in a module, lib.module == nil + // * if it's in a module, lib.module.Dir != "" + // Only vendored modules will have lib.module != nil && lib.module.Path != "" && lib.module.Dir == "" as far as I know. + // So the if condition above is already very strict for vendored packages. + // On top of it, we checked the lib.LicensePath contains a vendor folder in it. + // So it's rare to have a false positive for both conditions at the same time, although it may happen in theory. + // + // These assumptions may change in the future, + // so we need to keep this updated with go tooling changes. + for _, parentModule := range vendoredSearch { + if strings.HasPrefix(pkgDir, parentModule.Dir) { + module = parentModule + break + } + } + + if module.Dir == "" { + klog.Warningf("cannot find parent package of vendored module %s", module.Path) + } + } + } + + allPackages = append(allPackages, pkgInfo{ + pkgPath: p.PkgPath, + modulePath: module.Path, + pkgDir: pkgDir, + moduleDir: module.Dir, + }) + allModules[module.Path] = module + return true + }, nil) + if pkgErrorOccurred { + return nil, PackagesError{ + pkgs: rootPkgs, + } } - if p.Module == nil { - otherErrorOccurred = true - klog.Errorf("Package %s does not have module info. Non go modules projects are no longer supported. For feedback, refer to https://github.com/google/go-licenses/issues/128.", p.PkgPath) - return false + if otherErrorOccurred { + return nil, fmt.Errorf("some errors occurred when loading direct and transitive dependency packages") } - licensePath, err := Find(pkgDir, p.Module.Dir, classifier) + } + + pkgCandidates := map[string][]string{} + allCandidates := map[string]struct{}{} + for _, pkg := range allPackages { + candidates, err := FindCandidates(pkg.pkgDir, pkg.moduleDir) if err != nil { - klog.Errorf("Failed to find license for %s: %v", p.PkgPath, err) + return nil, err } - pkgs[p.PkgPath] = p - pkgsByLicense[licensePath] = append(pkgsByLicense[licensePath], p) - return true - }, nil) - if pkgErrorOccurred { - return nil, PackagesError{ - pkgs: rootPkgs, + + pkgCandidates[pkg.pkgDir] = candidates + for _, candidate := range candidates { + allCandidates[candidate] = struct{}{} } } - if otherErrorOccurred { - return nil, fmt.Errorf("some errors occurred when loading direct and transitive dependency packages") + + group, _ := errgroup.WithContext(ctx) + foundLicenseSlice := make([]struct { + candidate string + licenes []License + }, len(allCandidates)) + counter := 0 + for candidate := range allCandidates { + idx := counter + counter++ + candidate := candidate + + group.Go(func() error { + licenses, err := classifier.Identify(candidate) + if err != nil { + klog.Errorf("Failed to parse %s: %v", candidate, err) + return nil // Continue even if one LICENSE file fails to parse. + } + + foundLicenseSlice[idx] = struct { + candidate string + licenes []License + }{ + candidate: candidate, + licenes: licenses, + } + return nil + }) + } + + if err := group.Wait(); err != nil { + return nil, err + } + + foundLicenses := map[string][]License{} + for _, found := range foundLicenseSlice { + if len(found.licenes) == 0 { + continue + } + + foundLicenses[found.candidate] = found.licenes + } + + pkgsByLicense := make(map[string][]pkgInfo) + for _, pkg := range allPackages { + candidates := pkgCandidates[pkg.pkgDir] + + bestCandidate := "" + for _, candidate := range candidates { + if _, ok := foundLicenses[candidate]; ok { + bestCandidate = candidate + break + } + } + + pkgsByLicense[bestCandidate] = append(pkgsByLicense[bestCandidate], pkg) } var libraries []*Library - for licensePath, pkgs := range pkgsByLicense { - if licensePath == "" { + for licenseFile, pkgs := range pkgsByLicense { + if licenseFile == "" { // No license for these packages - return each one as a separate library. for _, p := range pkgs { libraries = append(libraries, &Library{ - Packages: []string{p.PkgPath}, - module: newModule(p.Module), + Packages: []string{p.pkgPath}, + module: allModules[p.modulePath], }) } continue } + lib := &Library{ - LicensePath: licensePath, - } - for _, pkg := range pkgs { - lib.Packages = append(lib.Packages, pkg.PkgPath) - if lib.module == nil && pkg.Module != nil { - // All the sub packages should belong to the same module. - lib.module = newModule(pkg.Module) - } + LicenseFile: licenseFile, + Licenses: foundLicenses[licenseFile], + Packages: make([]string, len(pkgs)), + module: allModules[pkgs[0].modulePath], } - if lib.module != nil && lib.module.Path != "" && lib.module.Dir == "" { - // A known cause is that the module is vendored, so some information is lost. - splits := strings.SplitN(lib.LicensePath, "/vendor/", 2) - if len(splits) != 2 { - klog.Warningf("module %s does not have dir and it's not vendored, cannot discover the license URL. Report to go-licenses developer if you see this.", lib.module.Path) - } else { - // This is vendored. Handle this known special case. - - // Extra note why we identify a vendored package like this. - // - // For a normal package: - // * if it's not in a module, lib.module == nil - // * if it's in a module, lib.module.Dir != "" - // Only vendored modules will have lib.module != nil && lib.module.Path != "" && lib.module.Dir == "" as far as I know. - // So the if condition above is already very strict for vendored packages. - // On top of it, we checked the lib.LicensePath contains a vendor folder in it. - // So it's rare to have a false positive for both conditions at the same time, although it may happen in theory. - // - // These assumptions may change in the future, - // so we need to keep this updated with go tooling changes. - parentModDir := splits[0] - var parentPkg *packages.Package - for _, rootPkg := range rootPkgs { - if rootPkg.Module != nil && rootPkg.Module.Dir == parentModDir { - parentPkg = rootPkg - break - } - } - if parentPkg == nil { - klog.Warningf("cannot find parent package of vendored module %s", lib.module.Path) - } else { - // Vendored modules should be commited in the parent module, so it counts as part of the - // parent module. - lib.module = newModule(parentPkg.Module) - } - } + + for i, p := range pkgs { + lib.Packages[i] = p.pkgPath } + libraries = append(libraries, lib) } + // Sort libraries to produce a stable result for snapshot diffing. sort.Slice(libraries, func(i, j int) bool { return libraries[i].Name() < libraries[j].Name() }) + return libraries, nil } @@ -234,7 +342,7 @@ func (l *Library) String() string { // FileURL attempts to determine the URL for a file in this library using // go module name and version. -func (l *Library) FileURL(ctx context.Context, filePath string) (string, error) { +func (l *Library) FileURL(ctx context.Context, cl *source.Client, filePath string) (string, error) { if l == nil { return "", fmt.Errorf("library is nil") } @@ -248,8 +356,7 @@ func (l *Library) FileURL(ctx context.Context, filePath string) (string, error) if m.Dir == "" { return "", wrap(fmt.Errorf("empty go module dir")) } - client := source.NewClient(time.Second * 20) - remote, err := source.ModuleInfo(ctx, client, m.Path, m.Version) + remote, err := source.ModuleInfo(ctx, cl, m.Path, m.Version) if err != nil { return "", wrap(err) } diff --git a/licenses/library_test.go b/licenses/library_test.go index 41e91868..13af573e 100644 --- a/licenses/library_test.go +++ b/licenses/library_test.go @@ -18,12 +18,18 @@ import ( "context" "os" "testing" + "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/go-licenses/internal/third_party/pkgsite/source" ) func TestLibraries(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Cannot get working directory: %v", err) + } + classifier := classifierStub{ licenses: map[string][]License{ "testdata/LICENSE": { @@ -47,30 +53,45 @@ func TestLibraries(t *testing.T) { }, } + type wantedLibInfo struct { + Name string + LicenseFile string + Licenses []License + } + + testdataLibInfo := wantedLibInfo{ + Name: "github.com/google/go-licenses/licenses/testdata", + LicenseFile: wd + "/testdata/LICENSE", + Licenses: classifier.licenses["testdata/LICENSE"], + } + directLibInfo := wantedLibInfo{ + Name: "github.com/google/go-licenses/licenses/testdata/direct", + LicenseFile: wd + "/testdata/direct/LICENSE", + Licenses: classifier.licenses["testdata/direct/LICENSE"], + } + indirectLibInfo := wantedLibInfo{ + Name: "github.com/google/go-licenses/licenses/testdata/indirect", + LicenseFile: wd + "/testdata/indirect/LICENSE", + Licenses: classifier.licenses["testdata/indirect/LICENSE"], + } + for _, test := range []struct { desc string importPath string goflags string includeTests bool ignore []string - wantLibs []string + wantLibs []wantedLibInfo }{ { desc: "Detects direct dependency", importPath: "github.com/google/go-licenses/licenses/testdata/direct", - wantLibs: []string{ - "github.com/google/go-licenses/licenses/testdata/direct", - "github.com/google/go-licenses/licenses/testdata/indirect", - }, + wantLibs: []wantedLibInfo{directLibInfo, indirectLibInfo}, }, { desc: "Detects transitive dependency", importPath: "github.com/google/go-licenses/licenses/testdata", - wantLibs: []string{ - "github.com/google/go-licenses/licenses/testdata", - "github.com/google/go-licenses/licenses/testdata/direct", - "github.com/google/go-licenses/licenses/testdata/indirect", - }, + wantLibs: []wantedLibInfo{testdataLibInfo, directLibInfo, indirectLibInfo}, }, { desc: "Ignores a package path", @@ -78,35 +99,44 @@ func TestLibraries(t *testing.T) { ignore: []string{ "github.com/google/go-licenses/licenses/testdata/direct", }, - wantLibs: []string{ - "github.com/google/go-licenses/licenses/testdata", - "github.com/google/go-licenses/licenses/testdata/indirect", - }, + wantLibs: []wantedLibInfo{testdataLibInfo, indirectLibInfo}, }, { desc: "Detects the dependencies only imported in testing code", importPath: "github.com/google/go-licenses/licenses/testdata/testlib", includeTests: true, - wantLibs: []string{ - "github.com/google/go-licenses/licenses/testdata/testlib", - "github.com/google/go-licenses/licenses/testdata/indirect", + wantLibs: []wantedLibInfo{ + indirectLibInfo, + { + Name: "github.com/google/go-licenses/licenses/testdata/testlib", + LicenseFile: wd + "/testdata/LICENSE", + Licenses: classifier.licenses["testdata/LICENSE"], + }, }, }, { desc: "Should not detect the dependencies only imported in testing code", importPath: "github.com/google/go-licenses/licenses/testdata/testlib", includeTests: false, - wantLibs: []string{ - "github.com/google/go-licenses/licenses/testdata/testlib", + wantLibs: []wantedLibInfo{ + { + Name: "github.com/google/go-licenses/licenses/testdata/testlib", + LicenseFile: wd + "/testdata/LICENSE", + Licenses: classifier.licenses["testdata/LICENSE"], + }, }, }, { desc: "Build tagged package", importPath: "github.com/google/go-licenses/licenses/testdata/tags", goflags: "-tags=tags", - wantLibs: []string{ - "github.com/google/go-licenses/licenses/testdata/tags", - "github.com/google/go-licenses/licenses/testdata/indirect", + wantLibs: []wantedLibInfo{ + indirectLibInfo, + { + Name: "github.com/google/go-licenses/licenses/testdata/tags", + LicenseFile: wd + "/testdata/LICENSE", + Licenses: classifier.licenses["testdata/LICENSE"], + }, }, }, } { @@ -119,12 +149,21 @@ func TestLibraries(t *testing.T) { if err != nil { t.Fatalf("Libraries(_, %q) = (_, %q), want (_, nil)", test.importPath, err) } - var gotLibNames []string - for _, lib := range gotLibs { - gotLibNames = append(gotLibNames, lib.Name()) + + for i, gotLib := range gotLibs { + if diff := cmp.Diff(test.wantLibs[i].Name, gotLib.Name()); diff != "" { + t.Errorf("Libraries(_, %q)[%d].Name() diff (-want +got): %s", test.importPath, i, diff) + } + if diff := cmp.Diff(test.wantLibs[i].LicenseFile, gotLib.LicenseFile); diff != "" { + t.Errorf("Libraries(_, %q)[%d].LicenseFile diff (-want +got): %s", test.importPath, i, diff) + } + if diff := cmp.Diff(test.wantLibs[i].Licenses, gotLib.Licenses); diff != "" { + t.Errorf("Libraries(_, %q)[%d].Licenses diff (-want +got): %s", test.importPath, i, diff) + } } - if diff := cmp.Diff(test.wantLibs, gotLibNames, cmpopts.SortSlices(func(x, y string) bool { return x < y })); diff != "" { - t.Errorf("Libraries(_, %q): diff (-want +got)\n%s", test.importPath, diff) + + if diff := cmp.Diff(len(test.wantLibs), len(gotLibs)); diff != "" { + t.Errorf("len(Libraries(_, %q)) diff (-want +got): %s", test.importPath, diff) } }) } @@ -193,7 +232,7 @@ func TestLibraryFileURL(t *testing.T) { "github.com/google/trillian", "github.com/google/trillian/crypto", }, - LicensePath: "/go/src/github.com/google/trillian/LICENSE", + LicenseFile: "/go/src/github.com/google/trillian/LICENSE", module: &Module{ Path: "github.com/google/trillian", Dir: "/go/src/github.com/google/trillian", @@ -210,7 +249,7 @@ func TestLibraryFileURL(t *testing.T) { "bitbucket.org/user/project/pkg", "bitbucket.org/user/project/pkg2", }, - LicensePath: "/foo/bar/bitbucket.org/user/project/LICENSE", + LicenseFile: "/foo/bar/bitbucket.org/user/project/LICENSE", module: &Module{ Path: "bitbucket.org/user/project", Dir: "/foo/bar/bitbucket.org/user/project", @@ -227,7 +266,7 @@ func TestLibraryFileURL(t *testing.T) { "example.com/user/project/pkg", "example.com/user/project/pkg2", }, - LicensePath: "/foo/bar/example.com/user/project/LICENSE", + LicenseFile: "/foo/bar/example.com/user/project/LICENSE", module: &Module{ Path: "example.com/user/project", Dir: "/foo/bar/example.com/user/project", @@ -244,7 +283,7 @@ func TestLibraryFileURL(t *testing.T) { "github.com/google/trillian", "github.com/google/trillian/crypto", }, - LicensePath: "/go/src/github.com/google/trillian/LICENSE", + LicenseFile: "/go/src/github.com/google/trillian/LICENSE", module: &Module{ Path: "github.com/google/trillian", Dir: "/go/src/github.com/google/trillian", @@ -259,7 +298,7 @@ func TestLibraryFileURL(t *testing.T) { Packages: []string{ "k8s.io/api/core/v1", }, - LicensePath: "/go/modcache/k8s.io/api/LICENSE", + LicenseFile: "/go/modcache/k8s.io/api/LICENSE", module: &Module{ Path: "k8s.io/api", Dir: "/go/modcache/k8s.io/api", @@ -271,7 +310,8 @@ func TestLibraryFileURL(t *testing.T) { }, } { t.Run(test.desc, func(t *testing.T) { - fileURL, err := test.lib.FileURL(context.Background(), test.path) + client := source.NewClient(time.Second * 20) + fileURL, err := test.lib.FileURL(context.Background(), client, test.path) if gotErr := err != nil; gotErr != test.wantErr { t.Fatalf("FileURL(%q) = (_, %q), want err? %t", test.path, err, test.wantErr) } else if gotErr { diff --git a/licenses/module.go b/licenses/module.go index eaff1b57..99abc16b 100644 --- a/licenses/module.go +++ b/licenses/module.go @@ -33,9 +33,6 @@ type Module struct { } func newModule(mod *packages.Module) *Module { - if mod == nil { - return nil - } // Example of a module with replace directive: k8s.io/kubernetes => k8s.io/kubernetes v1.11.1 // { // "Path": "k8s.io/kubernetes", @@ -58,6 +55,7 @@ func newModule(mod *packages.Module) *Module { if tmp.Replace != nil { tmp = *tmp.Replace } + // The +incompatible suffix does not affect module version. // ref: https://golang.org/ref/mod#incompatible-versions tmp.Version = strings.TrimSuffix(tmp.Version, "+incompatible") diff --git a/report.go b/report.go index 3d67bdde..c40f76f4 100644 --- a/report.go +++ b/report.go @@ -17,11 +17,15 @@ package main import ( "context" "encoding/csv" + "fmt" "os" "text/template" + "time" + "github.com/google/go-licenses/internal/third_party/pkgsite/source" "github.com/google/go-licenses/licenses" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" "k8s.io/klog/v2" ) @@ -49,16 +53,24 @@ func init() { } type libraryData struct { + Name string + Version string + LicensePath string + LicenseURL string + LicenseNames []string +} + +type libraryDataFlat struct { Name string + Version string + LicensePath string LicenseURL string LicenseName string - LicensePath string - Version string } // LicenseText reads and returns the contents of LicensePath, if set // or an empty string if not. -func (lib libraryData) LicenseText() (string, error) { +func (lib libraryDataFlat) LicenseText() (string, error) { if lib.LicensePath == "" { return "", nil } @@ -80,56 +92,83 @@ func reportMain(_ *cobra.Command, args []string) error { return err } - var reportData []libraryData - for _, lib := range libs { - version := lib.Version() - if len(version) == 0 { - version = UNKNOWN - } - libData := libraryData{ - Name: lib.Name(), - Version: version, - LicenseURL: UNKNOWN, - LicenseName: UNKNOWN, + reportData := make([]libraryData, len(libs)) + client := source.NewClient(time.Second * 20) + group, gctx := errgroup.WithContext(context.Background()) + for idx, lib := range libs { + idx := idx + lib := lib + + reportData[idx] = libraryData{ + Name: lib.Name(), + Version: UNKNOWN, + LicensePath: UNKNOWN, + LicenseURL: UNKNOWN, + LicenseNames: nil, } - if lib.LicensePath == "" { - reportData = append(reportData, libData) - continue + if version := lib.Version(); version != "" { + reportData[idx].Version = version } - libData.LicensePath = lib.LicensePath + if lib.LicenseFile != "" { + reportData[idx].LicensePath = lib.LicenseFile + } - url, err := lib.FileURL(context.Background(), lib.LicensePath) - if err == nil { - libData.LicenseURL = url - } else { - klog.Warningf("Error discovering license URL: %s", err) + for _, license := range lib.Licenses { + reportData[idx].LicenseNames = append(reportData[idx].LicenseNames, license.Name) } - licenses, err := classifier.Identify(lib.LicensePath) - if err != nil { - klog.Errorf("Error identifying license in %q: %v", lib.LicensePath, err) - reportData = append(reportData, libData) - continue + if lib.LicenseFile != "" { + group.Go(func() error { + url, err := lib.FileURL(gctx, client, lib.LicenseFile) + if err == nil { + reportData[idx].LicenseURL = url + } else { + klog.Warningf("Error discovering license URL: %s", err) + } + return nil + }) } + } - // Add each license to the report data, licenses is already sorted by - // location in the LICENSE file. - for _, license := range licenses { - libData.LicenseName = license.Name - reportData = append(reportData, libData) + if err := group.Wait(); err != nil { + return err + } + + // Flatten the report data + reportDataFlat := make([]libraryDataFlat, 0, len(reportData)) + for _, lib := range reportData { + if len(lib.LicenseNames) == 0 { + klog.Errorf("Error identifying license in %q: %v", lib.LicensePath, fmt.Errorf("no license found")) + reportDataFlat = append(reportDataFlat, libraryDataFlat{ + Name: lib.Name, + Version: lib.Version, + LicensePath: lib.LicensePath, + LicenseURL: lib.LicenseURL, + LicenseName: UNKNOWN, + }) + } else { + for _, licenseName := range lib.LicenseNames { + reportDataFlat = append(reportDataFlat, libraryDataFlat{ + Name: lib.Name, + Version: lib.Version, + LicensePath: lib.LicensePath, + LicenseURL: lib.LicenseURL, + LicenseName: licenseName, + }) + } } } if templateFile == "" { - return reportCSV(reportData) + return reportCSV(reportDataFlat) } else { - return reportTemplate(reportData) + return reportTemplate(reportDataFlat) } } -func reportCSV(libs []libraryData) error { +func reportCSV(libs []libraryDataFlat) error { writer := csv.NewWriter(os.Stdout) for _, lib := range libs { if err := writer.Write([]string{lib.Name, lib.LicenseURL, lib.LicenseName}); err != nil { @@ -140,7 +179,7 @@ func reportCSV(libs []libraryData) error { return writer.Error() } -func reportTemplate(libs []libraryData) error { +func reportTemplate(libs []libraryDataFlat) error { templateBytes, err := os.ReadFile(templateFile) if err != nil { return err diff --git a/save.go b/save.go index 55f5a3dc..1faa4004 100644 --- a/save.go +++ b/save.go @@ -91,14 +91,9 @@ func saveMain(_ *cobra.Command, args []string) error { libsWithBadLicenses := make(map[licenses.Type][]*licenses.Library) for _, lib := range libs { libSaveDir := filepath.Join(savePath, unvendor(lib.Name())) - // Detect what type of license this library has and fulfill its requirements, e.g. copy license, copyright notice, source code, etc. - licenseList, err := classifier.Identify(lib.LicensePath) - if err != nil { - return err - } - licenseTypes := make([]licenses.Type, 0, len(licenseList)) - for _, license := range licenseList { + licenseTypes := make([]licenses.Type, 0, len(lib.Licenses)) + for _, license := range lib.Licenses { licenseTypes = append(licenseTypes, license.Type) } @@ -107,23 +102,23 @@ func saveMain(_ *cobra.Command, args []string) error { switch restrictiveness { case licenses.RestrictionsShareCode: // Copy the entire source directory for the library. - libDir := filepath.Dir(lib.LicensePath) + libDir := filepath.Dir(lib.LicenseFile) if err := copySrc(libDir, libSaveDir); err != nil { return err } case licenses.RestrictionsShareLicense: // Just copy the license and copyright notice. - if err := copyNotices(lib.LicensePath, libSaveDir); err != nil { + if err := copyNotices(lib.LicenseFile, libSaveDir); err != nil { return err } default: - if len(licenseList) == 0 { + if len(lib.Licenses) == 0 { // If we can't identify the license, we can't fulfill its requirements. libsWithBadLicenses[licenses.Unknown] = append(libsWithBadLicenses[licenses.Unknown], lib) } else { // Register all bad licenses, so we can print them out at the end. FindAllBadLicences: - for _, license := range licenseList { + for _, license := range lib.Licenses { switch license.Type { case licenses.Notice, licenses.Permissive, licenses.Unencumbered, licenses.Restricted, licenses.Reciprocal: // these are allowed From e3146eb3fa59542a46571d7a252cce0d70b59510 Mon Sep 17 00:00:00 2001 From: Tim Ramlot <42113979+inteon@users.noreply.github.com> Date: Mon, 21 Aug 2023 16:45:46 +0200 Subject: [PATCH 3/4] add comment explaining how Libraries functions Signed-off-by: Tim Ramlot <42113979+inteon@users.noreply.github.com> --- licenses/library.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/licenses/library.go b/licenses/library.go index 073a64b5..3deedd2c 100644 --- a/licenses/library.go +++ b/licenses/library.go @@ -62,6 +62,17 @@ func (e PackagesError) Error() string { // Packages not covered by a license will be returned as individual libraries. // Standard library packages will be ignored. func Libraries(ctx context.Context, classifier Classifier, includeTests bool, ignoredPaths []string, importPaths ...string) ([]*Library, error) { + // These are the steps we take to find libraries: + // 1. we list all modules and all packages + // 2. for each package, we find a list of candidates + // 3. we deduplicate all candidates + // 4. for each candidate, we classify if the candidate is a license file + // 5. for each package, we select the first candidates that is a license + // file & add the package to a list of packages for that license file + // 6. we return an array of libraries (which are the license files, the + // found licenses in that file, all the packages that had that file as + // its first candidate and the module in which those packages live) + cfg := &packages.Config{ Context: ctx, Mode: packages.NeedImports | packages.NeedDeps | packages.NeedFiles | packages.NeedName | packages.NeedModule, From 0465735fece6491fbfb6daba4ba3923a2d1055a6 Mon Sep 17 00:00:00 2001 From: Tim Ramlot <42113979+inteon@users.noreply.github.com> Date: Fri, 25 Aug 2023 22:42:16 +0200 Subject: [PATCH 4/4] improve error message in case the LicensePath is UNKNOWN Signed-off-by: Tim Ramlot <42113979+inteon@users.noreply.github.com> --- report.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/report.go b/report.go index c40f76f4..888be488 100644 --- a/report.go +++ b/report.go @@ -140,7 +140,13 @@ func reportMain(_ *cobra.Command, args []string) error { reportDataFlat := make([]libraryDataFlat, 0, len(reportData)) for _, lib := range reportData { if len(lib.LicenseNames) == 0 { - klog.Errorf("Error identifying license in %q: %v", lib.LicensePath, fmt.Errorf("no license found")) + if lib.LicensePath != UNKNOWN { + klog.Errorf("Error identifying license in %q: %v", lib.LicensePath, fmt.Errorf("no license found")) + } else if lib.Version != UNKNOWN { + klog.Errorf("Error identifying license for version %q of %q: %v", lib.Version, lib.Name, fmt.Errorf("no license found")) + } else { + klog.Errorf("Error identifying license for %q: %v", lib.Name, fmt.Errorf("no license found")) + } reportDataFlat = append(reportDataFlat, libraryDataFlat{ Name: lib.Name, Version: lib.Version,