Skip to content

Commit

Permalink
attempt 7: use preprocessed min, max and sum temps as map value
Browse files Browse the repository at this point in the history
  • Loading branch information
shraddhaag committed Jan 25, 2024
1 parent 7812da4 commit e5213a8
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 54 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,5 @@
|3|Decouple reading and processing of file content. A buffered goroutine is used to communicate between the two processes.|5:22.83|+57.24|[2babf7d](https://github.com/shraddhaag/1brc/commit/2babf7dda72d92c72722b220b8b663e747075bd7)|
|4|Instead of sending each line to the channel, now sending 100 lines chunked together. Also, to minimise garbage collection, not freeing up memory when resetting a slice. |3:41.76|-161.07|[b7b1781](https://github.com/shraddhaag/1brc/commit/b7b1781f58fd258a06940bd6c05eb404c8a14af6)|
|5|Read file in chunks of 100 MB instead of reading line by line. |3:32.62|-9.14|[c26fea4](https://github.com/shraddhaag/1brc/commit/c26fea40019552a7e4fc1c864236f433b1b686f0)|
|6|Convert temperature from `string` to `int64`, process in `int64` and convert to `float64` at the end. |2:51.50|-41.14||
|6|Convert temperature from `string` to `int64`, process in `int64` and convert to `float64` at the end. |2:51.50|-41.14|[7812da4](https://github.com/shraddhaag/1brc/commit/7812da4d0be07dd4686d5f9b9df1e93b08cd0dd1)|
|7|In the city <> temperatures map, replaced the value for each key (city) to preprocessed min, max, count and sum of all temperatures instead of storing all recorded temperatures for the city.|1:39.81|-71.79||
91 changes: 38 additions & 53 deletions main.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package main

import (
"cmp"
"errors"
"flag"
"fmt"
Expand All @@ -12,11 +11,9 @@ import (
"runtime"
"runtime/pprof"
"runtime/trace"
"sort"
"strconv"
"strings"
"sync"

"slices"
)

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
Expand Down Expand Up @@ -71,66 +68,44 @@ type result struct {
}

func evaluate(input string) string {
// mapOfTemp, err := readFileLineByLineIntoAMap("./test_cases/measurements-rounding.txt")
// mapOfTemp, err := readFileLineByLineIntoAMap("measurements.txt")
mapOfTemp, err := readFileLineByLineIntoAMap(input)
if err != nil {
panic(err)
}

var resultArr []result
var wg sync.WaitGroup
var mx sync.Mutex

updateResult := func(city, temp string) {
mx.Lock()
defer mx.Unlock()

resultArr = append(resultArr, result{city, temp})
resultArr := make([]string, len(mapOfTemp))
var count int
for city, _ := range mapOfTemp {
resultArr[count] = city
count++
}

for city, temps := range mapOfTemp {
wg.Add(1)
go func(city string, temps []int64) {
defer wg.Done()
var min, max, avg int64
min, max = math.MaxInt64, math.MinInt64

for _, temp := range temps {
if temp < min {
min = temp
}

if temp > max {
max = temp
}
avg += temp
}

updateResult(city, fmt.Sprintf("%.1f/%.1f/%.1f", round(float64(min)/10.0), round(float64(avg)/10.0/float64(len(temps))), round(float64(max)/10.0)))

}(city, temps)
}

wg.Wait()
slices.SortFunc(resultArr, func(i, j result) int {
return cmp.Compare(i.city, j.city)
})
sort.Strings(resultArr)

var stringsBuilder strings.Builder
for _, i := range resultArr {
stringsBuilder.WriteString(fmt.Sprintf("%s=%s, ", i.city, i.temp))
stringsBuilder.WriteString(fmt.Sprintf("%s=%.1f/%.1f/%.1f, ", i,
round(float64(mapOfTemp[i].min)/10.0),
round(float64(mapOfTemp[i].sum)/10.0/float64(mapOfTemp[i].count)),
round(float64(mapOfTemp[i].max)/10.0)))
}
return stringsBuilder.String()[:stringsBuilder.Len()-2]
}

func readFileLineByLineIntoAMap(filepath string) (map[string][]int64, error) {
type cityTemperatureInfo struct {
count int64
min int64
max int64
sum int64
}

func readFileLineByLineIntoAMap(filepath string) (map[string]cityTemperatureInfo, error) {
file, err := os.Open(filepath)
if err != nil {
panic(err)
}

mapOfTemp := make(map[string][]int64)
mapOfTemp := make(map[string]cityTemperatureInfo)

chanOwner := func() <-chan []string {
resultStream := make(chan []string, 100)
Expand Down Expand Up @@ -170,21 +145,31 @@ func readFileLineByLineIntoAMap(filepath string) (map[string][]int64, error) {
}
city := text[:index]
temp := convertStringToInt64(text[index+1:])
if _, ok := mapOfTemp[city]; ok {
mapOfTemp[city] = append(mapOfTemp[city], temp)
if val, ok := mapOfTemp[city]; ok {
val.count++
val.sum += temp
if temp < val.min {
val.min = temp
}

if temp > val.max {
val.max = temp
}
mapOfTemp[city] = val
} else {
mapOfTemp[city] = []int64{temp}
mapOfTemp[city] = cityTemperatureInfo{
count: 1,
min: temp,
max: temp,
sum: temp,
}
}
}
}
// fmt.Println(mapOfTemp)
return mapOfTemp, nil
}

type cityTemp struct {
city string
temp float64
}

func convertStringToInt64(input string) int64 {
input = input[:len(input)-2] + input[len(input)-1:]
output, _ := strconv.ParseInt(input, 10, 64)
Expand Down
Binary file added profiles/cpu-int64.prof
Binary file not shown.
Binary file added profiles/cpu-preprocess.prof
Binary file not shown.

0 comments on commit e5213a8

Please sign in to comment.