start to add in uloc

README.md

@@ -287,6 +287,7 @@ Flags:
   -s, --sort string                  column to sort by [files, name, lines, blanks, code, comments, complexity] (default "files")
       --sql-project string           use supplied name as the project identifier for the current run. Only valid with the --format sql or sql-insert option
   -t, --trace                        enable trace output (not recommended when processing multiple files)
+      --uloc                         flip into uloc mode and count the number of unique lines of code
   -v, --verbose                      verbose output
       --version                      version for scc
   -w, --wide                         wider output with additional statistics (implies --complexity)

SCC-OUTPUT-REPORT.html

@@ -12,12 +12,12 @@
 	<tbody><tr>
 		<th>Go</th>
 		<th>30</th>
-		<th>9131</th>
-		<th>1434</th>
-		<th>445</th>
-		<th>7252</th>
-		<th>1485</th>
-		<th>389947</th>
+		<th>9219</th>
+		<th>1449</th>
+		<th>448</th>
+		<th>7322</th>
+		<th>1503</th>
+		<th>392214</th>
 	</tr><tr>
 		<th>Java</th>
 		<th>24</th>
@@ -48,12 +48,12 @@
 	</tr><tr>
 		<th>Markdown</th>
 		<th>10</th>
-		<th>1397</th>
+		<th>1398</th>
 		<th>332</th>
 		<th>0</th>
-		<th>1065</th>
+		<th>1066</th>
 		<th>0</th>
-		<th>57928</th>
+		<th>58030</th>
 	</tr><tr>
 		<th>YAML</th>
 		<th>8</th>
@@ -467,7 +467,7 @@
 		<th>0</th>
 		<th>797</th>
 		<th>0</th>
-		<th>11575</th>
+		<th>11574</th>
 	</tr><tr>
 		<th>Hare</th>
 		<th>1</th>
@@ -787,11 +787,11 @@
 	<tfoot><tr>
 		<th>Total</th>
 		<th>191</th>
-		<th>93807</th>
-		<th>4516</th>
-		<th>5937</th>
-		<th>83354</th>
-		<th>2871</th>
-    	<th>3132026</th>
+		<th>93896</th>
+		<th>4531</th>
+		<th>5940</th>
+		<th>83425</th>
+		<th>2889</th>
+    	<th>3134394</th>
 	</tr></tfoot>
 	</table></body></html>

main.go

@@ -52,6 +52,12 @@ func main() {
 
 	flags := rootCmd.PersistentFlags()
 
+	flags.BoolVar(
+		&processor.UlocMode,
+		"uloc",
+		false,
+		"flip into uloc mode and count the number of unique lines of code",
+	)
 	flags.BoolVar(
 		&processor.DisableCheckBinary,
 		"binary",

processor/formatters.go

@@ -673,6 +673,18 @@ create table t        (
 	return str.String()
 }
 
+// var tabularWideFormatBody = "%-33s %9d %9d %8d %9d %8d %10d %16.2f\n"
+var tabularUlocFormatBody = "Total Unique Source Lines of Code (ULOC) %38d\n"
+
+func ulocDisplay(input int) string {
+	var str strings.Builder
+	str.WriteString(getTabularShortBreak())
+	str.WriteString(fmt.Sprintf(tabularUlocFormatBody, input))
+	str.WriteString(getTabularShortBreak())
+
+	return str.String()
+}
+
 func fileSummarize(input chan *FileJob) string {
 	if FormatMulti != "" {
 		return fileSummarizeMulti(input)

processor/processor.go

@@ -102,6 +102,9 @@ var CountIgnore = false
 // DisableCheckBinary toggles checking for binary files using NUL bytes
 var DisableCheckBinary = false
 
+// UlocMode toggles checking for binary files using NUL bytes
+var UlocMode = false
+
 // SortBy sets which column output in formatter should be sorted by
 var SortBy = ""
 
@@ -610,13 +613,17 @@ func Process() {
 		close(fileListQueue)
 	}()
 
-	go fileProcessorWorker(fileListQueue, fileSummaryJobQueue)
-
-	result := fileSummarize(fileSummaryJobQueue)
-	if FileOutput == "" {
-		fmt.Println(result)
+	if UlocMode {
+		fileProcessorWorkerUloc(fileListQueue, fileSummaryJobQueue)
 	} else {
-		_ = os.WriteFile(FileOutput, []byte(result), 0644)
-		fmt.Println("results written to " + FileOutput)
+		go fileProcessorWorker(fileListQueue, fileSummaryJobQueue)
+
+		result := fileSummarize(fileSummaryJobQueue)
+		if FileOutput == "" {
+			fmt.Println(result)
+		} else {
+			_ = os.WriteFile(FileOutput, []byte(result), 0644)
+			fmt.Println("results written to " + FileOutput)
+		}
 	}
 }

processor/workers.go

@@ -699,7 +699,70 @@ func fileProcessorWorker(input chan *FileJob, output chan *FileJob) {
 			printDebug(fmt.Sprintf("milliseconds reading files into memory: %d", makeTimestampMilli()-startTime))
 		}
 	}()
+}
+
+// Processes files using ULOC - Unique lines of code
+func fileProcessorWorkerUloc(input chan *FileJob, output chan *FileJob) {
+	var startTime int64
+	var fileCount int64
+	var gcEnabled int64
+	var wg sync.WaitGroup
+
+	var ulocMutex = sync.Mutex{}
+	uloc := map[string]struct{}{}
+
+	for i := 0; i < FileProcessJobWorkers; i++ {
+		wg.Add(1)
+		go func() {
+			reader := NewFileReader()
+
+			for job := range input {
+				atomic.CompareAndSwapInt64(&startTime, 0, makeTimestampMilli())
+
+				loc := job.Location
+				if job.Symlocation != "" {
+					loc = job.Symlocation
+				}
+
+				fileStartTime := makeTimestampNano()
+				content, err := reader.ReadFile(loc, int(job.Bytes))
+				atomic.AddInt64(&fileCount, 1)
+
+				if atomic.LoadInt64(&gcEnabled) == 0 && atomic.LoadInt64(&fileCount) >= int64(GcFileCount) {
+					debug.SetGCPercent(gcPercent)
+					atomic.AddInt64(&gcEnabled, 1)
+					if Verbose {
+						printWarn("read file limit exceeded GC re-enabled")
+					}
+				}
+
+				if Trace {
+					printTrace(fmt.Sprintf("nanoseconds read into memory: %s: %d", job.Location, makeTimestampNano()-fileStartTime))
+				}
+
+				if err == nil {
+					ulocMutex.Lock()
+					for _, l := range strings.Split(string(content), "\n") {
+						uloc[l] = struct{}{}
+					}
+					ulocMutex.Unlock()
+				} else {
+					if Verbose {
+						printWarn(fmt.Sprintf("error reading: %s %s", job.Location, err))
+					}
+				}
+			}
+
+			wg.Done()
+		}()
+	}
+
+	wg.Wait()
+	if Debug {
+		printDebug(fmt.Sprintf("milliseconds reading files into memory: %d", makeTimestampMilli()-startTime))
+	}
 
+	fmt.Println(ulocDisplay(len(uloc)))
 }
 
 // Process a single file