Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expose repo include through API #751

Merged
merged 2 commits into from
Jun 19, 2018
Merged

Expose repo include through API #751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f1882cf
Expose repo include through API
Jun 18, 2018
9509629
Add changes test to increase coverage
Jun 18, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions api/repos.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/utils"
"github.com/gin-gonic/gin"
)
Expand Down Expand Up @@ -367,3 +368,77 @@ func apiReposPackageFromDir(c *gin.Context) {
"FailedFiles": failedFiles,
})
}

// POST /repos/:name/include/:dir/:file
func apiReposIncludePackageFromFile(c *gin.Context) {
// redirect all work to dir method
apiReposIncludePackageFromDir(c)
}

// POST /repos/:name/include/:dir
func apiReposIncludePackageFromDir(c *gin.Context) {
forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1"
acceptUnsigned := c.Request.URL.Query().Get("acceptUnsigned") == "1"
ignoreSignature := c.Request.URL.Query().Get("ignoreSignature") == "1"

repoTemplateString := c.Params.ByName("name")

if !verifyDir(c) {
return
}

fileParam := c.Params.ByName("file")
if fileParam != "" && !verifyPath(fileParam) {
c.AbortWithError(400, fmt.Errorf("wrong file"))
return
}

var (
err error
verifier = context.GetVerifier()
sources, changesFiles []string
failedFiles, failedFiles2 []string
reporter = &aptly.RecordingResultReporter{
Warnings: []string{},
AddedLines: []string{},
RemovedLines: []string{},
}
)

if fileParam == "" {
sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
} else {
sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
}

localRepoCollection := context.CollectionFactory().LocalRepoCollection()
localRepoCollection.Lock()
defer localRepoCollection.Unlock()

changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
_, failedFiles2, err = deb.ImportChangesFiles(
changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
repoTemplateString, context.Progress(), localRepoCollection, context.CollectionFactory().PackageCollection(),
context.PackagePool(), context.CollectionFactory().ChecksumCollection(), nil, query.Parse)
failedFiles = append(failedFiles, failedFiles2...)

if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to import changes files: %s", err))
return
}

if !noRemoveFiles {
// atempt to remove dir, if it fails, that's fine: probably it's not empty
os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
}

if failedFiles == nil {
failedFiles = []string{}
}

c.JSON(200, gin.H{
"Report": reporter,
"FailedFiles": failedFiles,
})
}
3 changes: 3 additions & 0 deletions api/router.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ func Router(c *ctx.AptlyContext) http.Handler {
root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)

root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)

root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
}

Expand Down
153 changes: 8 additions & 145 deletions cmd/repo_include.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
package cmd

import (
"bytes"
"fmt"
"os"
"path/filepath"
"text/template"

"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/utils"
"github.com/smira/commander"
"github.com/smira/flag"
)
Expand All @@ -35,11 +30,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)

repoTemplate, err := template.New("repo").Parse(context.Flags().Lookup("repo").Value.Get().(string))
if err != nil {
return fmt.Errorf("error parsing -repo template: %s", err)
}
repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)

uploaders := (*deb.Uploaders)(nil)
uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
Expand All @@ -59,143 +50,15 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {

reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}

var changesFiles, failedFiles, processedFiles []string
var changesFiles, failedFiles, failedFiles2 []string

changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)

for _, path := range changesFiles {
var changes *deb.Changes

changes, err = deb.NewChanges(path)
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", path, err)
continue
}

err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
changes.Cleanup()
continue
}

err = changes.Prepare()
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
changes.Cleanup()
continue
}

repoName := &bytes.Buffer{}
err = repoTemplate.Execute(repoName, changes.Stanza)
if err != nil {
return fmt.Errorf("error applying template to repo: %s", err)
}

context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)

var repo *deb.LocalRepo
repo, err = context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
changes.Cleanup()
continue
}

currentUploaders := uploaders
if repo.Uploaders != nil {
currentUploaders = repo.Uploaders
for i := range currentUploaders.Rules {
currentUploaders.Rules[i].CompiledCondition, err = query.Parse(currentUploaders.Rules[i].Condition)
if err != nil {
return fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
}
}
}

if currentUploaders != nil {
if err = currentUploaders.IsAllowed(changes); err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
changes.ChangesName, changes.SignatureKeys, err)
changes.Cleanup()
continue
}
}

err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
if err != nil {
return fmt.Errorf("unable to load repo: %s", err)
}

var list *deb.PackageList
list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}

packageFiles, otherFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)

var restriction deb.PackageQuery

restriction, err = changes.PackageQuery()
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
changes.Cleanup()
continue
}

var processedFiles2, failedFiles2 []string

processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
context.CollectionFactory().PackageCollection(), reporter, restriction, context.CollectionFactory().ChecksumCollection())

if err != nil {
return fmt.Errorf("unable to import package files: %s", err)
}

repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))

err = context.CollectionFactory().LocalRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to save: %s", err)
}

err = changes.Cleanup()
if err != nil {
return err
}

for _, file := range failedFiles2 {
failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
}

for _, file := range processedFiles2 {
processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
}

for _, file := range otherFiles {
processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
}

processedFiles = append(processedFiles, path)
}

if !noRemoveFiles {
processedFiles = utils.StrSliceDeduplicate(processedFiles)

for _, file := range processedFiles {
err = os.Remove(file)
if err != nil {
return fmt.Errorf("unable to remove file: %s", err)
}
}
}
_, failedFiles2, err = deb.ImportChangesFiles(
changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplateString,
context.Progress(), context.CollectionFactory().LocalRepoCollection(), context.CollectionFactory().PackageCollection(),
context.PackagePool(), context.CollectionFactory().ChecksumCollection(),
uploaders, query.Parse)
failedFiles = append(failedFiles, failedFiles2...)

if len(failedFiles) > 0 {
context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
Expand Down
Loading