Resolve #194

SCC-OUTPUT-REPORT.html

@@ -12,12 +12,12 @@
 	<tbody><tr>
 		<th>Go</th>
 		<th>34</th>
-		<th>7983</th>
-		<th>1304</th>
-		<th>339</th>
-		<th>6340</th>
+		<th>8005</th>
+		<th>1309</th>
+		<th>342</th>
+		<th>6354</th>
 		<th>1320</th>
-		<th>318569</th>
+		<th>319179</th>
 	</tr><tr>
 		<th>Java</th>
 		<th>23</th>
@@ -36,6 +36,15 @@
 		<th>175</th>
 		<th>53</th>
 		<th>796847</th>
+	</tr><tr>
+		<th>Python</th>
+		<th>9</th>
+		<th>350</th>
+		<th>20</th>
+		<th>20</th>
+		<th>310</th>
+		<th>32</th>
+		<th>10072</th>
 	</tr><tr>
 		<th>Markdown</th>
 		<th>8</th>
@@ -45,15 +54,6 @@
 		<th>849</th>
 		<th>0</th>
 		<th>45221</th>
-	</tr><tr>
-		<th>Python</th>
-		<th>8</th>
-		<th>343</th>
-		<th>18</th>
-		<th>19</th>
-		<th>306</th>
-		<th>29</th>
-		<th>9985</th>
 	</tr><tr>
 		<th>CSS</th>
 		<th>5</th>
@@ -93,12 +93,12 @@
 	</tr><tr>
 		<th>Shell</th>
 		<th>3</th>
-		<th>919</th>
-		<th>129</th>
+		<th>932</th>
+		<th>130</th>
 		<th>37</th>
-		<th>753</th>
-		<th>83</th>
-		<th>32787</th>
+		<th>765</th>
+		<th>85</th>
+		<th>33171</th>
 	</tr><tr>
 		<th>JavaServer Pages</th>
 		<th>2</th>
@@ -350,7 +350,7 @@
 		<th>0</th>
 		<th>563</th>
 		<th>0</th>
-		<th>8262</th>
+		<th>8263</th>
 	</tr><tr>
 		<th>JSON</th>
 		<th>1</th>
@@ -552,12 +552,12 @@
 	</tr></tbody>
 	<tfoot><tr>
 		<th>Total</th>
-		<th>160</th>
-		<th>24311</th>
-		<th>2758</th>
-		<th>1531</th>
-		<th>20022</th>
-		<th>2233</th>
-    	<th>1733039</th>
+		<th>161</th>
+		<th>24353</th>
+		<th>2766</th>
+		<th>1535</th>
+		<th>20052</th>
+		<th>2238</th>
+    	<th>1734121</th>
 	</tr></tfoot>
 	</table></body></html>

examples/symlink/link.py

@@ -0,0 +1 @@
+test.py

examples/symlink/test.py

@@ -0,0 +1,7 @@
+# this is a comment.
+
+import os
+
+for e in os.scandir('.'):
+	if e.is_file():
+		print(e)

main.go

@@ -241,6 +241,12 @@ func main() {
 		false,
 		"ignore files over certain byte and line size set by max-line-count and max-byte-count",
 	)
+	flags.BoolVar(
+		&processor.IncludeSymLink,
+		"include-symlink",
+		false,
+		"if set will count symlink files",
+	)
 	flags.Int64Var(
 		&processor.LargeLineCount,
 		"large-line-count",

processor/file.go

@@ -77,7 +77,7 @@ func NewDirectoryWalker(output chan<- *FileJob) *DirectoryWalker {
 func (dw *DirectoryWalker) Start(root string) error {
 	root = filepath.Clean(root)
 
-	fileInfo, err := os.Stat(root)
+	fileInfo, err := os.Lstat(root)
 	if err != nil {
 		return err
 	}
@@ -177,18 +177,7 @@ DIRENTS:
 				},
 			)
 		} else {
-			fileInfo := dirent
-			if dirent.Mode()&os.ModeSymlink != 0 {
-				linkName, err := os.Readlink(path)
-				if err != nil {
-					continue DIRENTS
-				}
-				fileInfo, err = os.Lstat(linkName)
-				if err != nil {
-					continue DIRENTS
-				}
-			}
-			fileJob := newFileJob(path, name, fileInfo)
+			fileJob := newFileJob(path, name, dirent)
 			if fileJob != nil {
 				dw.output <- fileJob
 			}
@@ -222,6 +211,22 @@ func newFileJob(path, name string, fileInfo os.FileInfo) *FileJob {
 		}
 	}
 
+	var symPath = ""
+	// Check if the file is a symlink and if we want to count those then work out its path and rejig
+	// everything so we can count the real file to ensure the counts are correct
+	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
+		if !IncludeSymLink {
+			if Verbose {
+				printWarn(fmt.Sprintf("skipping symlink file: %s", name))
+			}
+
+			return nil
+		}
+
+		symPath, _ = filepath.EvalSymlinks(path)
+		fileInfo, _ = os.Lstat(symPath)
+	}
+
 	language, extension := DetectLanguage(name)
 
 	if len(language) != 0 {
@@ -247,6 +252,7 @@ func newFileJob(path, name string, fileInfo os.FileInfo) *FileJob {
 
 		return &FileJob{
 			Location:          path,
+			Symlocation:       symPath,
 			Filename:          name,
 			Extension:         extension,
 			PossibleLanguages: language,

processor/processor.go

@@ -135,6 +135,9 @@ var isLazy = false
 // NoLarge if set true will ignore files over a certain number of lines or bytes
 var NoLarge = false
 
+// IncludeSymLink if set true will count symlink files
+var IncludeSymLink = false
+
 // LargeLineCount number of lines before being counted as a large file based on https://github.com/pinpt/ripsrc/blob/master/ripsrc/fileinfo/fileinfo.go#L44
 var LargeLineCount int64 = 40000
 
@@ -405,6 +408,7 @@ func processFlags() {
 		printDebug(fmt.Sprintf("Cocomo: %t", !Cocomo))
 		printDebug(fmt.Sprintf("Minified/Generated Detection: %t/%t", Minified, Generated))
 		printDebug(fmt.Sprintf("Ignore Minified/Generated: %t/%t", IgnoreMinified, IgnoreGenerated))
+		printDebug(fmt.Sprintf("IncludeSymLink: %t", IncludeSymLink))
 	}
 }
 

processor/structs.go

@@ -65,6 +65,7 @@ type FileJob struct {
 	Filename           string
 	Extension          string
 	Location           string
+	Symlocation        string
 	Content            []byte `json:"-"`
 	Bytes              int64
 	Lines              int64

processor/workers.go

@@ -425,7 +425,7 @@ func CountStats(fileJob *FileJob) {
 		digest = blake2b.New256()
 	}
 
-	for index := checkBomSkip(fileJob); index < len(fileJob.Content); index++ {
+	for index := checkBomSkip(fileJob); index < int(fileJob.Bytes); index++ {
 		// Based on our current state determine if the state should change by checking
 		// what the character is. The below is very CPU bound so need to be careful if
 		// changing anything in here and profile/measure afterwards!
@@ -633,8 +633,13 @@ func fileProcessorWorker(input chan *FileJob, output chan *FileJob) {
 			for job := range input {
 				atomic.CompareAndSwapInt64(&startTime, 0, makeTimestampMilli())
 
+				loc := job.Location
+				if job.Symlocation != "" {
+					loc = job.Symlocation
+				}
+
 				fileStartTime := makeTimestampNano()
-				content, err := reader.ReadFile(job.Location, int(job.Bytes))
+				content, err := reader.ReadFile(loc, int(job.Bytes))
 				atomic.AddInt64(&fileCount, 1)
 
 				if atomic.LoadInt64(&gcEnabled) == 0 && atomic.LoadInt64(&fileCount) >= int64(GcFileCount) {

test-all.sh

@@ -404,6 +404,19 @@ else
     echo -e "${GREEN}PASSED minified ignored check"
 fi
 
+a=$(./scc ./examples/symlink/)
+b=$(./scc --include-symlink ./examples/symlink/)
+if [ "$a" == "$b" ]; then
+    echo "$a"
+    echo "$b"
+    echo -e "${RED}======================================================="
+    echo -e "FAILED symlink check"
+    echo -e "=================================================${NC}"
+    exit
+else
+    echo -e "${GREEN}PASSED minified ignored check"
+fi
+
 if ./scc ./examples/minified/ --no-min-gen | grep -q "0.000000"; then
     echo -e "${GREEN}PASSED removed min"
 else