Skip to content

Commit

Permalink
Rename module dependencies
Browse files Browse the repository at this point in the history
Remove the term declared from the dependencies in a ModuleData that refer
to the full transitive list of dependencies, those in the buf.lock file.
Declared dependencies are the direct dependencies, those in the buf.yaml
file.
  • Loading branch information
emcfarlane committed Feb 11, 2025
1 parent abfbd3d commit e0aa28e
Show file tree
Hide file tree
Showing 7 changed files with 103 additions and 104 deletions.
46 changes: 23 additions & 23 deletions private/bufpkg/bufmodule/added_module.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,27 +161,27 @@ func (a *addedModule) ToModule(
}
return moduleData.V1Beta1OrV1BufLockObjectData()
}
// getDeclaredDepModuleKeysB5 gets the declared dependencies for the specific Module.
// getDepModuleKeysB5 gets the dependencies for the specific Module.
//
// This is needed to calculate the digest for the Module. A Module constructed from this
// ModuleData as the target will require all Modules referenced by its DeclaredDepModuleKeys
// to be present in the ModuleSet.
// ModuleData as the target will require all Modules referenced by its DepModuleKeys to
// be present in the ModuleSet.
//
// Modules that depend on this remote Module will include this Module and its data.
// However all the dependencies of the remote Module may not be present in the parents ModuleSet.
// As the target Module will use its direct dependencies to resolve the dependencies required.
// The digest of the remote Module is however, unchanged. It is calculated based on the contents
// and its declared dependencies, not the dependencies of the parent ModuleSet.
// and its dependencies, not the dependencies of the parent ModuleSet.
//
// In contrast, a local Module dependency can be thought of as a ModuleKey at the latest commit.
// It will always use the bucket and declared dependencies, which may be resolved recursively
// for dependencies on other local Modules, to calculate its digest.
// It will always use the bucket and dependencies, which may be resolved recursively for
// dependencies on other local Modules, to calculate its digest.
// This is the difference between the ModuleData digest calculation and the Module
// digest calculation. As remote Modules are required to have all their dependencies
// declared as ModuleKeys they can calculate their digest directly from the contents
// and declared dependencies, they do not need to recursively resolve digests.
// digest calculation. As remote Modules are required to have all their dependencies as
// ModuleKeys, they can calculate their digest directly from the contents and dependencies,
// without needing to recursively resolve the digest as local Modules do.
//
// For example, consider the following modules at commits with their declared dependencies:
// For example, consider the following modules at commits with their dependencies:
// ```
// X:C1 (X has no dependencies)
// A:C1 -> X:C1 (A depends on X)
Expand All @@ -196,42 +196,42 @@ func (a *addedModule) ToModule(
// The ModuleSet for B:C1 will include A:C1 and X:C1.
// When calculating the digest for B:C1 in the ModuleSet of C:C1, the ModuleSet
// ModuleDeps cannot be used to resolve the dependencies of B:C1. It must use
// the declared dependencies of B:C1, which are A:C1 and X:C1, not A:C2.
// the dependencies of B:C1, which are A:C1 and X:C1, not A:C2.
//
// This is used for digest calculations. It is not used otherwise.
getDeclaredDepModuleKeysB5 := func() ([]ModuleKey, error) {
getDepModuleKeysB5 := func() ([]ModuleKey, error) {
moduleData, err := getModuleData()
if err != nil {
return nil, err
}
declaredDepModuleKeys, err := moduleData.DeclaredDepModuleKeys()
depModuleKeys, err := moduleData.DepModuleKeys()
if err != nil {
return nil, err
}
if len(declaredDepModuleKeys) == 0 {
if len(depModuleKeys) == 0 {
return nil, nil
}
var digestType DigestType
for i, moduleKey := range declaredDepModuleKeys {
for i, moduleKey := range depModuleKeys {
digest, err := moduleKey.Digest()
if err != nil {
return nil, err
}
if i == 0 {
digestType = digest.Type()
} else if digestType != digest.Type() {
return nil, syserror.Newf("multiple digest types found in DeclaredDepModuleKeys: %v, %v", digestType, digest.Type())
return nil, syserror.Newf("multiple digest types found in DepModuleKeys: %v, %v", digestType, digest.Type())
}
}
switch digestType {
case DigestTypeB4:
// The declared ModuleKey dependencies for a commit may be stored in v1 buf.lock file,
// The ModuleKey dependencies for a commit may be stored in v1 buf.lock file,
// in which case they will use B4 digests. B4 digests aren't allowed to be used as
// input to the B5 digest calculation, so we perform a call to convert all ModuleKeys
// from B4 to B5 by using the commit provider.
commitKeysToFetch := make([]CommitKey, len(declaredDepModuleKeys))
for i, declaredDepModuleKey := range declaredDepModuleKeys {
commitKey, err := NewCommitKey(declaredDepModuleKey.FullName().Registry(), declaredDepModuleKey.CommitID(), DigestTypeB5)
commitKeysToFetch := make([]CommitKey, len(depModuleKeys))
for i, depModuleKey := range depModuleKeys {
commitKey, err := NewCommitKey(depModuleKey.FullName().Registry(), depModuleKey.CommitID(), DigestTypeB5)
if err != nil {
return nil, err
}
Expand All @@ -248,8 +248,8 @@ func (a *addedModule) ToModule(
return commit.ModuleKey()
}), nil
case DigestTypeB5:
// No need to fetch b5 digests - we've already got them stored in the module's declared dependencies.
return declaredDepModuleKeys, nil
// No need to fetch b5 digests - we've already got them stored in the module's dependencies.
return depModuleKeys, nil
default:
return nil, syserror.Newf("unsupported digest type: %v", digestType)
}
Expand All @@ -265,7 +265,7 @@ func (a *addedModule) ToModule(
false,
getV1BufYAMLObjectData,
getV1BufLockObjectData,
getDeclaredDepModuleKeysB5,
getDepModuleKeysB5,
a.remoteTargetPaths,
a.remoteTargetExcludePaths,
"",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func (a *moduleDataProvider) getIndexedModuleDatasForRegistryAndIndexedModuleKey
digestType bufmodule.DigestType,
) ([]slicesext.Indexed[bufmodule.ModuleData], error) {
// This returns the graph with pruned dependencies for the given indexedModuleKeys.
// We must resolve the declared dependencies for each ModuleKey.
// We must resolve the direct and transitive dependencies for each ModuleKey.
graph, err := a.graphProvider.GetGraphForModuleKeys(ctx, slicesext.IndexedToValues(indexedModuleKeys))
if err != nil {
return nil, err
Expand Down
10 changes: 5 additions & 5 deletions private/bufpkg/bufmodule/bufmodulestore/module_data_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,9 +262,9 @@ func (p *moduleDataStore) getModuleDataForModuleKey(
// We don't want to do this lazily (or anything else in this function) as we want to
// make sure everything we have is valid before returning so we can auto-correct
// the cache if necessary.
declaredDepModuleKeys, err := slicesext.MapError(
depModuleKeys, err := slicesext.MapError(
externalModuleData.Deps,
getDeclaredDepModuleKeyForExternalModuleDataDep,
getDepModuleKeyForExternalModuleDataDep,
)
if err != nil {
return nil, err
Expand Down Expand Up @@ -316,7 +316,7 @@ func (p *moduleDataStore) getModuleDataForModuleKey(
), nil
},
func() ([]bufmodule.ModuleKey, error) {
return declaredDepModuleKeys, nil
return depModuleKeys, nil
},
func() (bufmodule.ObjectData, error) {
return v1BufYAMLObjectData, nil
Expand Down Expand Up @@ -461,7 +461,7 @@ func (p *moduleDataStore) putModuleData(
}
}
// Proceed to writing module data.
depModuleKeys, err := moduleData.DeclaredDepModuleKeys()
depModuleKeys, err := moduleData.DepModuleKeys()
if err != nil {
return err
}
Expand Down Expand Up @@ -653,7 +653,7 @@ func getModuleDataStoreTarPath(moduleKey bufmodule.ModuleKey) (string, error) {
), nil
}

func getDeclaredDepModuleKeyForExternalModuleDataDep(dep externalModuleDataDep) (bufmodule.ModuleKey, error) {
func getDepModuleKeyForExternalModuleDataDep(dep externalModuleDataDep) (bufmodule.ModuleKey, error) {
if dep.Name == "" {
return nil, errors.New("no module name specified")
}
Expand Down
4 changes: 2 additions & 2 deletions private/bufpkg/bufmodule/bufmoduletesting/bufmoduletesting.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ func (o *omniProvider) getModuleDataForModuleKey(
if err != nil {
return nil, err
}
declaredDepModuleKeys, err := slicesext.MapError(
depModuleKeys, err := slicesext.MapError(
moduleDeps,
func(moduleDep bufmodule.ModuleDep) (bufmodule.ModuleKey, error) {
return bufmodule.ModuleToModuleKey(moduleDep, digest.Type())
Expand All @@ -327,7 +327,7 @@ func (o *omniProvider) getModuleDataForModuleKey(
return bufmodule.ModuleReadBucketToStorageReadBucket(module), nil
},
func() ([]bufmodule.ModuleKey, error) {
return declaredDepModuleKeys, nil
return depModuleKeys, nil
},
func() (bufmodule.ObjectData, error) {
return module.V1Beta1OrV1BufYAMLObjectData()
Expand Down
88 changes: 44 additions & 44 deletions private/bufpkg/bufmodule/module.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,17 +230,17 @@ func ModuleDirectModuleDeps(module Module) ([]ModuleDep, error) {
type module struct {
ModuleReadBucket

ctx context.Context
getBucket func() (storage.ReadBucket, error)
bucketID string
description string
moduleFullName bufparse.FullName
commitID uuid.UUID
isTarget bool
isLocal bool
getV1BufYAMLObjectData func() (ObjectData, error)
getV1BufLockObjectData func() (ObjectData, error)
getDeclaredDepModuleKeysB5 func() ([]ModuleKey, error)
ctx context.Context
getBucket func() (storage.ReadBucket, error)
bucketID string
description string
moduleFullName bufparse.FullName
commitID uuid.UUID
isTarget bool
isLocal bool
getV1BufYAMLObjectData func() (ObjectData, error)
getV1BufLockObjectData func() (ObjectData, error)
getDepModuleKeysB5 func() ([]ModuleKey, error)

moduleSet ModuleSet

Expand All @@ -261,7 +261,7 @@ func newModule(
isLocal bool,
getV1BufYAMLObjectData func() (ObjectData, error),
getV1BufLockObjectData func() (ObjectData, error),
getDeclaredDepModuleKeysB5 func() ([]ModuleKey, error),
getDepModuleKeysB5 func() ([]ModuleKey, error),
targetPaths []string,
targetExcludePaths []string,
protoFileTargetPath string,
Expand Down Expand Up @@ -304,17 +304,17 @@ func newModule(
}

module := &module{
ctx: ctx,
getBucket: syncOnceValuesGetBucketWithStorageMatcherApplied,
bucketID: bucketID,
description: description,
moduleFullName: moduleFullName,
commitID: commitID,
isTarget: isTarget,
isLocal: isLocal,
getV1BufYAMLObjectData: sync.OnceValues(getV1BufYAMLObjectData),
getV1BufLockObjectData: sync.OnceValues(getV1BufLockObjectData),
getDeclaredDepModuleKeysB5: sync.OnceValues(getDeclaredDepModuleKeysB5),
ctx: ctx,
getBucket: syncOnceValuesGetBucketWithStorageMatcherApplied,
bucketID: bucketID,
description: description,
moduleFullName: moduleFullName,
commitID: commitID,
isTarget: isTarget,
isLocal: isLocal,
getV1BufYAMLObjectData: sync.OnceValues(getV1BufYAMLObjectData),
getV1BufLockObjectData: sync.OnceValues(getV1BufLockObjectData),
getDepModuleKeysB5: sync.OnceValues(getDepModuleKeysB5),
}
moduleReadBucket, err := newModuleReadBucketForModule(
ctx,
Expand Down Expand Up @@ -398,17 +398,17 @@ func (m *module) ModuleSet() ModuleSet {
func (m *module) withIsTarget(isTarget bool) (Module, error) {
// We don't just call newModule directly as we don't want to double sync.OnceValues stuff.
newModule := &module{
ctx: m.ctx,
getBucket: m.getBucket,
bucketID: m.bucketID,
description: m.description,
moduleFullName: m.moduleFullName,
commitID: m.commitID,
isTarget: isTarget,
isLocal: m.isLocal,
getV1BufYAMLObjectData: m.getV1BufYAMLObjectData,
getV1BufLockObjectData: m.getV1BufLockObjectData,
getDeclaredDepModuleKeysB5: m.getDeclaredDepModuleKeysB5,
ctx: m.ctx,
getBucket: m.getBucket,
bucketID: m.bucketID,
description: m.description,
moduleFullName: m.moduleFullName,
commitID: m.commitID,
isTarget: isTarget,
isLocal: m.isLocal,
getV1BufYAMLObjectData: m.getV1BufYAMLObjectData,
getV1BufLockObjectData: m.getV1BufLockObjectData,
getDepModuleKeysB5: m.getDepModuleKeysB5,
}
moduleReadBucket, ok := m.ModuleReadBucket.(*moduleReadBucket)
if !ok {
Expand Down Expand Up @@ -452,26 +452,26 @@ func newGetDigestFuncForModuleAndDigestType(module *module, digestType DigestTyp
}
return getB4Digest(module.ctx, bucket, v1BufYAMLObjectData, v1BufLockObjectData)
case DigestTypeB5:
// B5 digests are calculated from the Module's content and its declared dependencies.
// B5 digests are calculated from the Module's content and its dependencies.
//
// Declared dependencies are all direct and transitive dependencies of the Module.
// For local Modules, the declared dependencies are resolved to the current ModuleSet.
// For remote Modules, the declared dependencies are the ModuleKeys of
// Dependencies are all direct and transitive dependencies of the Module.
// For local Modules, the dependencies are resolved to the current ModuleSet.
// For remote Modules, the dependencies are the ModuleKeys of
// the direct and transitive dependencies of the Module at the specific commit.
//
// This is effectively the same. The local Modules have a declared dependency at the current commit,
// and the remote Modules have a declared dependency at a specific commit. Pushing up a local Module
// This is effectively the same. The local Modules have a dependency at the current commit,
// and the remote Modules have a dependency at a specific commit. Pushing up a local Module
// as a remote Module will result in the same Digest.
//
// See the comment on getDeclaredDepModuleKeysB5 for more information.
// See the comment on getDepModuleKeysB5 for more information.
if !module.isLocal {
declaredDepModuleKeys, err := module.getDeclaredDepModuleKeysB5()
depModuleKeys, err := module.getDepModuleKeysB5()
if err != nil {
return nil, err
}
return getB5DigestForBucketAndDepModuleKeys(module.ctx, bucket, declaredDepModuleKeys)
return getB5DigestForBucketAndDepModuleKeys(module.ctx, bucket, depModuleKeys)
}
// ModuleDeps returns the declared dependent Modules resolved for the target Module in the ModuleSet.
// ModuleDeps returns the dependent Modules resolved for the target Module in the ModuleSet.
// A local Module will recursively resolve the Digest of other dependent local Modules.
// This is done by calling Module.Digest(DigestTypeB5) on each ModuleDep.
moduleDeps, err := module.ModuleDeps()
Expand Down
Loading

0 comments on commit e0aa28e

Please sign in to comment.