Skip to content

Commit

Permalink
fix: split large compute file up
Browse files Browse the repository at this point in the history
This is a followup to googleapis#2544. That change reduced the size of
the compute client from 9.4M to 6.7. This change now splits the
compute client into three files ranging from 2.7-1.8M each. This
should allow editors and tooling to better be able to process the
client.
  • Loading branch information
codyoss committed Apr 25, 2024
1 parent 2f2505b commit bd51b4e
Showing 1 changed file with 103 additions and 8 deletions.
111 changes: 103 additions & 8 deletions google-api-go-generator/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ const (
googleDiscoveryURL = "https://www.googleapis.com/discovery/v1/apis"
googleDefaultUniverse = "googleapis.com"
universeDomainPlaceholder = "UNIVERSE_DOMAIN"

splitFileSeperator = `// =*=*"`
)

var (
Expand Down Expand Up @@ -75,6 +77,10 @@ var skipAPIGeneration = map[string]bool{
"datalineage:v1": true,
}

var apisToSplit = map[string]bool{
"compute": true,
}

// API represents an API to generate, as well as its state while it's
// generating.
type API struct {
Expand Down Expand Up @@ -226,7 +232,7 @@ func getAPIs() []*API {
} else {
bytes = slurpURL(*apisURL)
if *publicOnly {
if err := writeFile(apiListFile, bytes); err != nil {
if err := writeFile(apiListFile, "", bytes); err != nil {
log.Fatal(err)
}
}
Expand Down Expand Up @@ -300,7 +306,7 @@ func apiFromFile(file string) (*API, error) {
func checkAndUpdateSpecFile(file string, contents []byte) error {
// check if file exists
if _, err := os.Stat(file); os.IsNotExist(err) {
return writeFile(file, contents)
return writeFile(file, "", contents)
}
existing, err := os.ReadFile(file)
if err != nil {
Expand All @@ -309,7 +315,7 @@ func checkAndUpdateSpecFile(file string, contents []byte) error {
if err := isNewerRevision(existing, contents); err != nil {
return err
}
return writeFile(file, contents)
return writeFile(file, "", contents)
}

// isNewerRevision returns nil if the contents of new has a newer revision than
Expand All @@ -331,7 +337,7 @@ func isNewerRevision(old []byte, new []byte) error {
return nil
}

func writeFile(file string, contents []byte) error {
func writeFile(file, pkg string, contents []byte) error {
// Don't write it if the contents are identical.
existing, err := os.ReadFile(file)
if err == nil && (bytes.Equal(existing, contents) || basicallyEqual(existing, contents)) {
Expand All @@ -341,7 +347,42 @@ func writeFile(file string, contents []byte) error {
if err = os.MkdirAll(outdir, 0755); err != nil {
return fmt.Errorf("failed to Mkdir %s: %v", outdir, err)
}
return os.WriteFile(file, contents, 0644)
// Don't try to split spec files, json or non-allowlisted packages
if pkg == "" || !apisToSplit[pkg] {
return os.WriteFile(file, contents, 0644)
}

// Split generated file out into multiple
bs := bytes.Split(contents, []byte(splitFileSeperator))
for i, b := range bs {
var name string
var newB []byte
if i == 0 {
// For the base case, use the provided inputs as is
name = file
newB = b
} else {
// determine the new file name
base := filepath.Dir(file)
fileNum := i + 1
name = base + string(filepath.Separator) + pkg + fmt.Sprint(fileNum) + "-gen.go"

// prepend file header, package, and imports
var buf bytes.Buffer
// Size the buffer so there is room for the header and contents
buf.Grow(len(b) + 500)
splitFileHeading(&buf, pkg)
_, err := buf.Write(b)
if err != nil {
return err
}
newB = buf.Bytes()
}
if err := os.WriteFile(name, newB, 0644); err != nil {
return err
}
}
return nil
}

var ignoreLines = regexp.MustCompile(`(?m)^\s+"(?:etag|revision)": ".+\n`)
Expand Down Expand Up @@ -589,7 +630,7 @@ func (a *API) WriteGeneratedCode() error {
}

code, err := a.GenerateCode()
errw := writeFile(genfilename, code)
errw := writeFile(genfilename, a.Package(), code)
if err == nil {
err = errw
}
Expand Down Expand Up @@ -862,8 +903,12 @@ func (a *API) GenerateCode() ([]byte, error) {
for _, meth := range a.APIMethods() {
meth.generateCode()
}

for _, res := range a.doc.Resources {
a.insertSplitFileComment()
rCnt := len(a.doc.Resources) / 2
for i, res := range a.doc.Resources {
if i == rCnt {
a.insertSplitFileComment()
}
a.generateResourceMethods(res)
}

Expand All @@ -874,6 +919,56 @@ func (a *API) GenerateCode() ([]byte, error) {
return clean, nil
}

func (a *API) insertSplitFileComment() {
if apisToSplit[a.Package()] {
a.pn("")
a.pn(splitFileSeperator)
a.pn("")
}
}

// splitFileHeading writes the file preamble used when generating a split file
// client like compute.
func splitFileHeading(w io.Writer, pkg string) {
pn := func(format string, args ...interface{}) {
_, err := fmt.Fprintf(w, format+"\n", args...)
if err != nil {
panic(err)
}
}

pn(`// Copyright %s Google LLC.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Code generated file. DO NOT EDIT.
`, *copyrightYear)
pn("")
pn("package %s", pkg)
pn("")
pn("import (")
for _, imp := range []string{
"context",
"fmt",
"io",
"net/http",
} {
pn(" %q", imp)
}
pn("")
for _, imp := range []struct {
pkg string
lname string
}{
{*gensupportPkg, "gensupport"},
{*googleapiPkg, "googleapi"},
} {
pn(" %s %q", imp.lname, imp.pkg)
}
pn(")")
pn("")
}

func (a *API) generateScopeConstants() {
scopes := a.doc.Auth.OAuth2Scopes
if len(scopes) == 0 {
Expand Down

0 comments on commit bd51b4e

Please sign in to comment.