Skip to content

Commit

Permalink
continue to improve badges
Browse files Browse the repository at this point in the history
  • Loading branch information
boyter committed Aug 3, 2023
1 parent 86547ec commit 3d993f7
Show file tree
Hide file tree
Showing 374 changed files with 198,730 additions and 28 deletions.
68 changes: 44 additions & 24 deletions cmd/badges/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ import (
"context"
"errors"
"fmt"
"log"
"github.com/rs/zerolog/log"
"math"
"net/http"
"net/url"
"os"
"os/exec"
"regexp"
Expand All @@ -15,22 +16,31 @@ import (
"time"
)

var uniqueCode = "unique_code"
var cache = NewSimpleCache(1000)

func main() {
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, err := processUrlPath(r.URL.Path)
loc, err := processUrlPath(r.URL.Path)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte("you be invalid"))
return
}

process(1, loc)

category := strings.TrimSpace(strings.ToLower(r.URL.Query().Get("category")))

title := "Total lines"
title := ""

switch category {
case "codes":
fallthrough
case "code":
title = "Code lines"
case "blank":
fallthrough
case "blanks":
title = "Blank lines"
case "comment":
Expand All @@ -41,6 +51,8 @@ func main() {
title = "COCOMO $"
case "lines": // lines is the default
fallthrough
case "line": // lines is the default
fallthrough
default:
//
title = "Total lines"
Expand All @@ -53,6 +65,7 @@ func main() {
textLength = "200"
}

log.Info().Str(uniqueCode, "42c5269c").Str("loc", loc.String()).Str("category", category).Send()
w.Header().Set("Content-Type", "image/svg+xml;charset=utf-8")
_, _ = w.Write([]byte(`<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="100" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="100" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h69v20H0z"/><path fill="#4c1" d="M69 0h31v20H69z"/><path fill="url(#b)" d="M0 0h100v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="355" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="590">` + title + `</text><text x="355" y="140" transform="scale(.1)" textLength="590">` + title + `</text><text x="835" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="` + textLength + `">` + s + `</text><text x="835" y="140" transform="scale(.1)" textLength="` + textLength + `">` + s + `</text></g> </svg>`))
})
Expand All @@ -61,14 +74,14 @@ func main() {
}

type location struct {
Location string
Provider string
User string
Repo string
}

func (l *location) String() string {
//url.Parse("https://"+l.Location)
return ""
parse, _ := url.Parse("https://" + l.Provider + ".com/" + l.User + "/" + l.Repo + ".git")
return parse.String()
}

// processUrlPath takes in a string path and returns a struct
Expand All @@ -81,11 +94,11 @@ func processUrlPath(path string) (location, error) {
path = strings.TrimSuffix(path, "/")
s := strings.Split(path, "/")
if len(s) != 3 {
return location{}, errors.New("")
return location{}, errors.New("invalid path part")
}

return location{
Location: s[0],
Provider: s[0],
User: s[1],
Repo: s[2],
}, nil
Expand Down Expand Up @@ -123,8 +136,12 @@ func formatCount(count float64) string {
return fmt.Sprintf("%v", math.Round(count))
}

func process(id int, s string) {
fmt.Println("processing", s)
func process(id int, s location) {
_, ok := cache.Get(s.String())
if ok {
// TODO real thing here
return
}

// Clean target just to be sure
cmdArgs := []string{
Expand All @@ -136,7 +153,7 @@ func process(id int, s string) {
err := cmd.Run()

if err != nil {
fmt.Println("rm start", err.Error())
log.Error().Err(err).Str(uniqueCode, "41b5460d").Str("loc", s.String()).Send()
return
}

Expand All @@ -145,23 +162,23 @@ func process(id int, s string) {
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()

cmd = exec.CommandContext(ctx, "git", "clone", "--depth=1", s+".git", "/tmp/scc-tmp-path-"+strconv.Itoa(id))
cmd = exec.CommandContext(ctx, "git", "clone", "--depth=1", s.String(), "/tmp/scc-tmp-path-"+strconv.Itoa(id))

cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
resp, err := cmd.Output()

if ctx.Err() == context.DeadlineExceeded {
fmt.Println("git clone timed out")
log.Error().Str(uniqueCode, "8f8ccc64").Str("loc", s.String()).Msg("git clone timed out")
return
}

if err != nil {
fmt.Println("git clone non-zero exit code", string(resp))
log.Error().Err(err).Str(uniqueCode, "f28fb388").Str("loc", s.String()).Str("resp", string(resp)).Msg("git clone non-zero exit code")
return
}

// Run scc against what we just cloned
fileName := processPath(s)
fileName := processPath(s.String())

if fileName == "" {
return
Expand All @@ -179,14 +196,17 @@ func process(id int, s string) {
err = cmd.Run()

if err != nil {
fmt.Println("scc", err.Error())
log.Error().Err(err).Str(uniqueCode, "3a74fde3").Str("loc", s.String()).Str("resp", string(resp)).Send()
return
}

//err = uploadS3File("sloccloccode", fileName, "/tmp/"+fileName)
//if err != nil {
// fmt.Println("s3 upload", err.Error())
//}
//fmt.Println("uploaded now cleaning up")
file, err := os.ReadFile("/tmp/" + fileName)
if err != nil {
log.Error().Err(err).Str(uniqueCode, "b16570de").Str("loc", s.String()).Str("resp", string(resp)).Send()
return
}
cache.Add(s.String(), file)
//map[string]processor.LanguageSummary{}

// Cleanup
cmdArgs = []string{
Expand All @@ -198,7 +218,7 @@ func process(id int, s string) {
err = cmd.Run()

if err != nil {
fmt.Println("rm cleanup filename", err.Error())
log.Error().Err(err).Str(uniqueCode, "ed40409c").Str("loc", s.String()).Str("resp", string(resp)).Send()
return
}

Expand All @@ -211,7 +231,7 @@ func process(id int, s string) {
err = cmd.Run()

if err != nil {
fmt.Println("rm cleanup", err.Error())
log.Error().Err(err).Str(uniqueCode, "2bca46a1").Str("loc", s.String()).Str("resp", string(resp)).Send()
return
}
}
Expand Down Expand Up @@ -241,7 +261,7 @@ func processPath(s string) string {
func cleanString(s string) string {
reg, err := regexp.Compile("[^a-z0-9-._]+")
if err != nil {
log.Fatal(err)
log.Fatal().Err(err).Send()
}

processedString := reg.ReplaceAllString(s, "")
Expand Down
4 changes: 2 additions & 2 deletions cmd/badges/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func Test_processPath(t *testing.T) {
path: "/github/boyter/really-cheap-chatbot/",
},
want: location{
Location: "github",
Provider: "github",
User: "boyter",
Repo: "really-cheap-chatbot",
},
Expand All @@ -101,7 +101,7 @@ func Test_processPath(t *testing.T) {
path: "github/boyter/really-cheap-chatbot",
},
want: location{
Location: "github",
Provider: "github",
User: "boyter",
Repo: "really-cheap-chatbot",
},
Expand Down
86 changes: 86 additions & 0 deletions cmd/badges/simplecache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package main

import (
"math"
"sync"
)

type cacheEntry struct {
entry []byte
hits int
}

type SimpleCache struct {
maxItems int
items map[string]cacheEntry
lock sync.Mutex
}

func NewSimpleCache(maxItems int) *SimpleCache {
cache := SimpleCache{
maxItems: maxItems,
items: map[string]cacheEntry{},
lock: sync.Mutex{},
}

return &cache
}

func (cache *SimpleCache) Add(cacheKey string, entry []byte) {
cache.expireItems()

cache.lock.Lock()
defer cache.lock.Unlock()

cache.items[cacheKey] = cacheEntry{
entry: entry,
hits: 1,
}
}

func (cache *SimpleCache) Get(cacheKey string) ([]byte, bool) {
cache.lock.Lock()
defer cache.lock.Unlock()

item, ok := cache.items[cacheKey]

if ok {
if item.hits < 100 {
item.hits++
}
cache.items[cacheKey] = item
return item.entry, true
}

return nil, false
}

// ExpireItems is called before any insert operation because we need to ensure we have less than
// the total number of items
func (cache *SimpleCache) expireItems() {
cache.lock.Lock()
defer cache.lock.Unlock()

count := 10
if len(cache.items) >= cache.maxItems {
lfuKey := ""
lfuLowestCount := math.MaxInt64

for k, v := range cache.items {
v.hits--
cache.items[k] = v
if v.hits < lfuLowestCount {
lfuKey = k
lfuLowestCount = v.hits
}

// we only want to process X random elements so we don't spin forever
count--
if count <= 0 {
break
}
}

delete(cache.items, lfuKey)
}
}
52 changes: 52 additions & 0 deletions cmd/badges/simplecache_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package main

import (
"fmt"
"testing"
)

func TestSimpleCache_Add(t *testing.T) {
simpleCache := NewSimpleCache(5)

for i := 0; i < 5; i++ {
simpleCache.Add(fmt.Sprintf("%d", i), []byte{})
}

for i := 0; i < 4; i++ {
for j := 0; j < 5; j++ {
simpleCache.Get(fmt.Sprintf("%d", i))
}
}

simpleCache.Add("10", []byte{})
}

func TestSimpleCache_Multiple(t *testing.T) {
simpleCache := NewSimpleCache(10)

for i := 0; i < 500; i++ {
simpleCache.Add(fmt.Sprintf("%d", i), []byte{})
}

simpleCache.Add("10", []byte{})

if len(simpleCache.items) != 10 {
t.Errorf("expected 10 items got %v", len(simpleCache.items))
}
}

func TestSimpleCache_MultipleLarge(t *testing.T) {
simpleCache := NewSimpleCache(10)

for i := 0; i < 5000; i++ {
simpleCache.Add(fmt.Sprintf("%d", i), []byte{})
simpleCache.Add("10", []byte{})
simpleCache.Get(fmt.Sprintf("%d", i))
simpleCache.Get("10")
simpleCache.Get("10")
}

if len(simpleCache.items) != 9 {
t.Errorf("expected 9 items got %v", len(simpleCache.items))
}
}
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,19 @@ require (
github.com/json-iterator/go v1.1.12
github.com/mattn/go-runewidth v0.0.14
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
github.com/rs/zerolog v1.30.0
github.com/spf13/cobra v1.6.1
golang.org/x/text v0.8.0
gopkg.in/yaml.v2 v2.4.0
)

require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/sys v0.10.0 // indirect
)
Loading

0 comments on commit 3d993f7

Please sign in to comment.