From 0af225ccf1256e95794c2d8c27c5e0900aa82b2f Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Mon, 13 May 2024 18:10:42 +0600 Subject: [PATCH] fix(conda): add support `pip` deps for `environment.yml` files (#6675) --- .../parser/conda/environment/parse.go | 58 +++++++++++++++---- .../parser/conda/environment/parse_test.go | 39 ++++++++++++- .../conda/environment/testdata/happy.yaml | 3 + .../environment/testdata/wrong-deps-type.yaml | 7 +++ .../testdata/wrong-nested-dep-type.yaml | 9 +++ .../testdata/wrong-nested-type.yaml | 7 +++ 6 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 pkg/dependency/parser/conda/environment/testdata/wrong-deps-type.yaml create mode 100644 pkg/dependency/parser/conda/environment/testdata/wrong-nested-dep-type.yaml create mode 100644 pkg/dependency/parser/conda/environment/testdata/wrong-nested-type.yaml diff --git a/pkg/dependency/parser/conda/environment/parse.go b/pkg/dependency/parser/conda/environment/parse.go index f8bdcfb49a92..be04d828b40a 100644 --- a/pkg/dependency/parser/conda/environment/parse.go +++ b/pkg/dependency/parser/conda/environment/parse.go @@ -15,7 +15,11 @@ import ( ) type environment struct { - Dependencies []Dependency `yaml:"dependencies"` + Entries []Entry `yaml:"dependencies"` +} + +type Entry struct { + Dependencies []Dependency } type Dependency struct { @@ -42,13 +46,15 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc } var pkgs ftypes.Packages - for _, dep := range env.Dependencies { - pkg := p.toPackage(dep) - // Skip empty pkgs - if pkg.Name == "" { - continue + for _, entry := range env.Entries { + for _, dep := range entry.Dependencies { + pkg := p.toPackage(dep) + // Skip empty pkgs + if pkg.Name == "" { + continue + } + pkgs = append(pkgs, pkg) } - pkgs = append(pkgs, pkg) } sort.Sort(pkgs) @@ -96,8 +102,40 @@ func (*Parser) parseDependency(line string) (string, string) { return name, parts[1] } -func (d *Dependency) UnmarshalYAML(node *yaml.Node) error { - d.Value = node.Value - d.Line = node.Line +func (e *Entry) UnmarshalYAML(node *yaml.Node) error { + var dependencies []Dependency + // cf. https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/resolve.go#L70-L81 + switch node.Tag { + case "!!str": + dependencies = append(dependencies, Dependency{ + Value: node.Value, + Line: node.Line, + }) + case "!!map": + if node.Content != nil { + // Map key is package manager (e.g. pip). So we need to store only map values (dependencies). + // e.g. dependencies: + // - pip: + // - pandas==2.1.4 + if node.Content[1].Tag != "!!seq" { // Conda supports only map[string][]string format. + return xerrors.Errorf("unsupported dependency type %q on line %d", node.Content[1].Tag, node.Content[1].Line) + } + + for _, depContent := range node.Content[1].Content { + if depContent.Tag != "!!str" { + return xerrors.Errorf("unsupported dependency type %q on line %d", depContent.Tag, depContent.Line) + } + + dependencies = append(dependencies, Dependency{ + Value: depContent.Value, + Line: depContent.Line, + }) + } + } + default: + return xerrors.Errorf("unsupported dependency type %q on line %d", node.Tag, node.Line) + } + + e.Dependencies = dependencies return nil } diff --git a/pkg/dependency/parser/conda/environment/parse_test.go b/pkg/dependency/parser/conda/environment/parse_test.go index 109f53a405bd..6127054b2552 100644 --- a/pkg/dependency/parser/conda/environment/parse_test.go +++ b/pkg/dependency/parser/conda/environment/parse_test.go @@ -31,6 +31,16 @@ func TestParse(t *testing.T) { }, }, }, + { + Name: "asgiref", + Version: "3.8.1", + Locations: ftypes.Locations{ + { + StartLine: 21, + EndLine: 21, + }, + }, + }, { Name: "blas", Version: "1.0", @@ -61,6 +71,16 @@ func TestParse(t *testing.T) { }, }, }, + { + Name: "django", + Version: "5.0.6", + Locations: ftypes.Locations{ + { + StartLine: 22, + EndLine: 22, + }, + }, + }, { Name: "ld_impl_linux-aarch64", Locations: ftypes.Locations{ @@ -167,9 +187,24 @@ func TestParse(t *testing.T) { }, }, { - name: "invalid_json", + name: "invalid yaml file", input: "testdata/invalid.yaml", - wantErr: "unable to decode conda environment.yml file", + wantErr: "cannot unmarshal !!str `invalid` into environment.environment", + }, + { + name: "`dependency` field uses unsupported type", + input: "testdata/wrong-deps-type.yaml", + wantErr: `unsupported dependency type "!!int" on line 5`, + }, + { + name: "nested field uses unsupported type", + input: "testdata/wrong-nested-type.yaml", + wantErr: `unsupported dependency type "!!str" on line 5`, + }, + { + name: "nested dependency uses unsupported type", + input: "testdata/wrong-nested-dep-type.yaml", + wantErr: `unsupported dependency type "!!map" on line 6`, }, } for _, tt := range tests { diff --git a/pkg/dependency/parser/conda/environment/testdata/happy.yaml b/pkg/dependency/parser/conda/environment/testdata/happy.yaml index f36e8bf990eb..29ba800e5bb6 100644 --- a/pkg/dependency/parser/conda/environment/testdata/happy.yaml +++ b/pkg/dependency/parser/conda/environment/testdata/happy.yaml @@ -17,5 +17,8 @@ dependencies: - liblapack=3.9.*=22_linuxaarch64_openblas - libnsl=2.0.1=h31becfc_0 - bzip2=1.0.8=h998d150_5 + - pip: + - asgiref==3.8.1 + - django==5.0.6 prefix: /opt/conda/envs/test-env diff --git a/pkg/dependency/parser/conda/environment/testdata/wrong-deps-type.yaml b/pkg/dependency/parser/conda/environment/testdata/wrong-deps-type.yaml new file mode 100644 index 000000000000..e7257be4abe5 --- /dev/null +++ b/pkg/dependency/parser/conda/environment/testdata/wrong-deps-type.yaml @@ -0,0 +1,7 @@ +name: test-env +channels: + - defaults +dependencies: + - 1 + +prefix: /opt/conda/envs/test-env diff --git a/pkg/dependency/parser/conda/environment/testdata/wrong-nested-dep-type.yaml b/pkg/dependency/parser/conda/environment/testdata/wrong-nested-dep-type.yaml new file mode 100644 index 000000000000..0c032b8a6d40 --- /dev/null +++ b/pkg/dependency/parser/conda/environment/testdata/wrong-nested-dep-type.yaml @@ -0,0 +1,9 @@ +name: test-env +channels: + - defaults +dependencies: + - pip: + - wrongType: + - asgiref==3.8.1 + +prefix: /opt/conda/envs/test-env diff --git a/pkg/dependency/parser/conda/environment/testdata/wrong-nested-type.yaml b/pkg/dependency/parser/conda/environment/testdata/wrong-nested-type.yaml new file mode 100644 index 000000000000..2e8c01cad787 --- /dev/null +++ b/pkg/dependency/parser/conda/environment/testdata/wrong-nested-type.yaml @@ -0,0 +1,7 @@ +name: test-env +channels: + - defaults +dependencies: + - pip: asgiref==3.8.1 + +prefix: /opt/conda/envs/test-env