use lists for repo-actions
This commit is contained in:
parent
42e95d0b75
commit
f009e9ff00
118
main.go
118
main.go
|
|
@ -189,7 +189,7 @@ func (b *BuildManager) buildWorker(id int, march string) {
|
|||
pkg.DbPackage.Update().SetStatus(dbpackage.StatusFailed).ClearSkipReason().SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).SetHash(pkg.Hash).ExecX(context.Background())
|
||||
|
||||
// purge failed package from repo
|
||||
b.repoPurge[pkg.FullRepo] <- pkg
|
||||
b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
|
||||
|
||||
err = cleanBuildDir(buildDir)
|
||||
if err != nil {
|
||||
|
|
@ -251,7 +251,7 @@ func (b *BuildManager) buildWorker(id int, march string) {
|
|||
}
|
||||
|
||||
log.Infof("[%s/%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, pkg.Version, time.Since(start))
|
||||
b.repoAdd[pkg.FullRepo] <- pkg
|
||||
b.repoAdd[pkg.FullRepo] <- []*BuildPackage{pkg}
|
||||
|
||||
err = cleanBuildDir(buildDir)
|
||||
if err != nil {
|
||||
|
|
@ -318,7 +318,7 @@ func (b *BuildManager) parseWorker() {
|
|||
pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).
|
||||
SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status).
|
||||
SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).SaveX(context.Background())
|
||||
b.repoPurge[pkg.FullRepo] <- pkg
|
||||
b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
|
||||
b.parseWG.Done()
|
||||
continue
|
||||
} else {
|
||||
|
|
@ -347,13 +347,13 @@ func (b *BuildManager) parseWorker() {
|
|||
case MultiplePKGBUILDError:
|
||||
log.Infof("Skipped %s: Multiple PKGBUILDs for dependency found: %v", pkg.Srcinfo.Pkgbase, err)
|
||||
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("multiple PKGBUILD for dep. found").SaveX(context.Background())
|
||||
b.repoPurge[pkg.FullRepo] <- pkg
|
||||
b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
|
||||
b.parseWG.Done()
|
||||
continue
|
||||
case UnableToSatisfyError:
|
||||
log.Infof("Skipped %s: unable to resolve dependencies: %v", pkg.Srcinfo.Pkgbase, err)
|
||||
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("unable to resolve dependencies").SaveX(context.Background())
|
||||
b.repoPurge[pkg.FullRepo] <- pkg
|
||||
b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
|
||||
b.parseWG.Done()
|
||||
continue
|
||||
}
|
||||
|
|
@ -374,7 +374,7 @@ func (b *BuildManager) parseWorker() {
|
|||
// Worst case would be clients downloading a package update twice, once from their official mirror,
|
||||
// and then after build from ALHP. Best case we prevent a not buildable package from staying in the repos
|
||||
// in an outdated version.
|
||||
b.repoPurge[pkg.FullRepo] <- pkg
|
||||
b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
|
||||
b.parseWG.Done()
|
||||
continue
|
||||
}
|
||||
|
|
@ -541,65 +541,69 @@ func (b *BuildManager) htmlWorker() {
|
|||
func (b *BuildManager) repoWorker(repo string) {
|
||||
for {
|
||||
select {
|
||||
case pkg := <-b.repoAdd[repo]:
|
||||
args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
|
||||
args = append(args, pkg.PkgFiles...)
|
||||
cmd := exec.Command("repo-add", args...)
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil && cmd.ProcessState.ExitCode() != 1 {
|
||||
log.Panicf("%s while repo-add: %v", string(res), err)
|
||||
}
|
||||
case pkgL := <-b.repoAdd[repo]:
|
||||
for _, pkg := range pkgL {
|
||||
args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
|
||||
args = append(args, pkg.PkgFiles...)
|
||||
cmd := exec.Command("repo-add", args...)
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil && cmd.ProcessState.ExitCode() != 1 {
|
||||
log.Panicf("%s while repo-add: %v", string(res), err)
|
||||
}
|
||||
|
||||
pkg.toDbPackage(true)
|
||||
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background())
|
||||
pkg.toDbPackage(true)
|
||||
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background())
|
||||
|
||||
cmd = exec.Command("paccache",
|
||||
"-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch),
|
||||
"-k", "1")
|
||||
res, err = cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
check(err)
|
||||
updateLastUpdated()
|
||||
b.buildWG.Done()
|
||||
case pkg := <-b.repoPurge[repo]:
|
||||
if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil {
|
||||
continue
|
||||
cmd = exec.Command("paccache",
|
||||
"-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch),
|
||||
"-k", "1")
|
||||
res, err = cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
check(err)
|
||||
updateLastUpdated()
|
||||
b.buildWG.Done()
|
||||
}
|
||||
if len(pkg.PkgFiles) == 0 {
|
||||
if err := pkg.findPkgFiles(); err != nil {
|
||||
log.Warningf("[%s/%s] Unable to find files: %v", pkg.FullRepo, pkg.Pkgbase, err)
|
||||
continue
|
||||
} else if len(pkg.PkgFiles) == 0 {
|
||||
case pkgL := <-b.repoPurge[repo]:
|
||||
for _, pkg := range pkgL {
|
||||
if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(pkg.PkgFiles) == 0 {
|
||||
if err := pkg.findPkgFiles(); err != nil {
|
||||
log.Warningf("[%s/%s] Unable to find files: %v", pkg.FullRepo, pkg.Pkgbase, err)
|
||||
continue
|
||||
} else if len(pkg.PkgFiles) == 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
var realPkgs []string
|
||||
for _, filePath := range pkg.PkgFiles {
|
||||
realPkgs = append(realPkgs, Package(filePath).Name())
|
||||
}
|
||||
var realPkgs []string
|
||||
for _, filePath := range pkg.PkgFiles {
|
||||
realPkgs = append(realPkgs, Package(filePath).Name())
|
||||
}
|
||||
|
||||
b.repoWG.Add(1)
|
||||
args := []string{"-s", "-v", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
|
||||
args = append(args, realPkgs...)
|
||||
cmd := exec.Command("repo-remove", args...)
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil && cmd.ProcessState.ExitCode() == 1 {
|
||||
log.Warningf("Error while deleting package %s: %s", pkg.Pkgbase, string(res))
|
||||
}
|
||||
b.repoWG.Add(1)
|
||||
args := []string{"-s", "-v", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
|
||||
args = append(args, realPkgs...)
|
||||
cmd := exec.Command("repo-remove", args...)
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil && cmd.ProcessState.ExitCode() == 1 {
|
||||
log.Warningf("Error while deleting package %s: %s", pkg.Pkgbase, string(res))
|
||||
}
|
||||
|
||||
if pkg.DbPackage != nil {
|
||||
_ = pkg.DbPackage.Update().ClearRepoVersion().Exec(context.Background())
|
||||
}
|
||||
if pkg.DbPackage != nil {
|
||||
_ = pkg.DbPackage.Update().ClearRepoVersion().Exec(context.Background())
|
||||
}
|
||||
|
||||
for _, file := range pkg.PkgFiles {
|
||||
_ = os.Remove(file)
|
||||
_ = os.Remove(file + ".sig")
|
||||
for _, file := range pkg.PkgFiles {
|
||||
_ = os.Remove(file)
|
||||
_ = os.Remove(file + ".sig")
|
||||
}
|
||||
updateLastUpdated()
|
||||
b.repoWG.Done()
|
||||
}
|
||||
updateLastUpdated()
|
||||
b.repoWG.Done()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -784,8 +788,8 @@ func main() {
|
|||
buildManager = &BuildManager{
|
||||
build: make(map[string]chan *BuildPackage),
|
||||
parse: make(chan *BuildPackage, 10000),
|
||||
repoPurge: make(map[string]chan *BuildPackage),
|
||||
repoAdd: make(map[string]chan *BuildPackage),
|
||||
repoPurge: make(map[string]chan []*BuildPackage),
|
||||
repoAdd: make(map[string]chan []*BuildPackage),
|
||||
exit: false,
|
||||
}
|
||||
|
||||
|
|
|
|||
14
utils.go
14
utils.go
|
|
@ -67,8 +67,8 @@ type BuildPackage struct {
|
|||
type BuildManager struct {
|
||||
build map[string]chan *BuildPackage
|
||||
parse chan *BuildPackage
|
||||
repoPurge map[string]chan *BuildPackage
|
||||
repoAdd map[string]chan *BuildPackage
|
||||
repoPurge map[string]chan []*BuildPackage
|
||||
repoAdd map[string]chan []*BuildPackage
|
||||
exit bool
|
||||
buildWG sync.WaitGroup
|
||||
parseWG sync.WaitGroup
|
||||
|
|
@ -710,7 +710,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
|
|||
PkgFiles: []string{path},
|
||||
March: mPackage.MArch(),
|
||||
}
|
||||
buildManager.repoPurge[pkg.FullRepo] <- pkg
|
||||
buildManager.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -742,7 +742,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
|
|||
if err != nil || pkgResolved.DB().Name() != pkg.DbPackage.Repository.String() || pkgResolved.DB().Name() != pkg.Repo.String() {
|
||||
// package not found on mirror/db -> not part of any repo anymore
|
||||
log.Infof("[HK/%s/%s] not included in repo", pkg.FullRepo, pkg.Pkgbase)
|
||||
buildManager.repoPurge[pkg.FullRepo] <- pkg
|
||||
buildManager.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
|
||||
err = db.DbPackage.DeleteOne(pkg.DbPackage).Exec(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -757,7 +757,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
|
|||
}
|
||||
if !valid {
|
||||
log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase)
|
||||
buildManager.repoPurge[pkg.FullRepo] <- pkg
|
||||
buildManager.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -861,8 +861,8 @@ func syncMarchs() {
|
|||
for _, repo := range conf.Repos {
|
||||
fRepo := fmt.Sprintf("%s-%s", repo, march)
|
||||
repos = append(repos, fRepo)
|
||||
buildManager.repoAdd[fRepo] = make(chan *BuildPackage, conf.Build.Worker)
|
||||
buildManager.repoPurge[fRepo] = make(chan *BuildPackage, 10000)
|
||||
buildManager.repoAdd[fRepo] = make(chan []*BuildPackage, conf.Build.Worker)
|
||||
buildManager.repoPurge[fRepo] = make(chan []*BuildPackage, 10000)
|
||||
go buildManager.repoWorker(fRepo)
|
||||
|
||||
if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, fRepo, "os", conf.Arch))); os.IsNotExist(err) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue