ALHP
/
ALHP.GO
6
34
Fork 9
Code Issues 29 Pull Requests Projects Releases Wiki Activity

added automatic LTO fail detection for some cases based on build output

This commit is contained in:
Giovanni Harting 2021-11-25 01:50:49 +01:00
parent d5e9832b33
commit 743b08bba1
6 changed files with 48 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ent/dbpackage/dbpackage.go
View File

@ -140,9 +140,10 @@ const DefaultLto = LtoUnknown
// Lto values.
const (
LtoEnabled Lto = "enabled"
LtoUnknown Lto = "unknown"
LtoDisabled Lto = "disabled"
LtoEnabled Lto = "enabled"
LtoUnknown Lto = "unknown"
LtoDisabled Lto = "disabled"
LtoAutoDisabled Lto = "auto_disabled"
)
func (l Lto) String() string {
@ -152,7 +153,7 @@ func (l Lto) String() string {
// LtoValidator is a validator for the "lto" field enum values. It is called by the builders before save.
func LtoValidator(l Lto) error {
switch l {
case LtoEnabled, LtoUnknown, LtoDisabled:
case LtoEnabled, LtoUnknown, LtoDisabled, LtoAutoDisabled:
return nil
default:
return fmt.Errorf("dbpackage: invalid enum value for lto field: %q", l)

2
ent/migrate/schema.go
View File

@ -23,7 +23,7 @@ var (
{Name: "build_time_end", Type: field.TypeTime, Nullable: true},
{Name: "updated", Type: field.TypeTime, Nullable: true},
{Name: "hash", Type: field.TypeString, Nullable: true},
{Name: "lto", Type: field.TypeEnum, Nullable: true, Enums: []string{"enabled", "unknown", "disabled"}, Default: "unknown"},
{Name: "lto", Type: field.TypeEnum, Nullable: true, Enums: []string{"enabled", "unknown", "disabled", "auto_disabled"}, Default: "unknown"},
}
// DbPackagesTable holds the schema information for the "db_packages" table.
DbPackagesTable = &schema.Table{

2
ent/schema/dbpackage.go
View File

@ -25,7 +25,7 @@ func (DbPackage) Fields() []ent.Field {
field.Time("build_time_end").Optional(),
field.Time("updated").Optional(),
field.String("hash").Optional(),
field.Enum("lto").Values("enabled", "unknown", "disabled").Default("unknown").Optional(),
field.Enum("lto").Values("enabled", "unknown", "disabled", "auto_disabled").Default("unknown").Optional(),
}
}

56
main.go
View File

@ -61,7 +61,7 @@ func (b *BuildManager) buildWorker(id int) {
pkg.toDbPackage(true)
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuilding).ClearSkipReason().SaveX(context.Background())
err := importKeys(pkg)
err := pkg.importKeys()
if err != nil {
log.Warningf("[%s/%s] Failed to import pgp keys: %v", pkg.FullRepo, pkg.Pkgbase, err)
}
@ -84,15 +84,12 @@ func (b *BuildManager) buildWorker(id int) {
}
pkg.PkgFiles = []string{}
ltoDisabled := false
// default to LTO
makepkgFile := "makepkg-%s-lto.conf"
if contains(conf.Blacklist.LTO, pkg.Pkgbase) {
if pkg.DbPackage.Lto == dbpackage.LtoDisabled || pkg.DbPackage.Lto == dbpackage.LtoAutoDisabled {
// use non-lto makepkg.conf if LTO is blacklisted for this package
makepkgFile = "makepkg-%s.conf"
ltoDisabled = true
pkg.DbPackage.Update().SetLto(dbpackage.LtoDisabled).ExecX(context.Background())
}
cmd := exec.Command("sh", "-c",
"cd "+filepath.Dir(pkg.Pkgbuild)+"&&makechrootpkg -c -D "+conf.Basedir.Makepkg+" -l worker-"+strconv.Itoa(id)+" -r "+conf.Basedir.Chroot+" -- "+
@ -124,6 +121,14 @@ func (b *BuildManager) buildWorker(id int) {
continue
}
if pkg.DbPackage.Lto != dbpackage.LtoAutoDisabled && pkg.DbPackage.Lto != dbpackage.LtoDisabled && reLdError.Match(out.Bytes()) {
log.Infof("[%s/%s] ld error detected, disabling LTO", pkg.FullRepo, pkg.Pkgbase)
pkg.DbPackage.Update().SetStatus(dbpackage.StatusQueued).SetSkipReason("non-LTO rebuild").SetLto(dbpackage.LtoAutoDisabled).ExecX(context.Background())
gitClean(pkg)
b.buildWG.Done()
continue
}
log.Warningf("[%s/%s] Build failed (%d)", pkg.FullRepo, pkg.Pkgbase, cmd.ProcessState.ExitCode())
check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), 0755))
@ -181,10 +186,10 @@ func (b *BuildManager) buildWorker(id int) {
check(os.Remove(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log")))
}
if !ltoDisabled {
if pkg.DbPackage.Lto != dbpackage.LtoDisabled && pkg.DbPackage.Lto != dbpackage.LtoAutoDisabled {
pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoEnabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background())
} else {
pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoDisabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background())
pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background())
}
log.Infof("[%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, time.Now().Sub(start))
@ -241,9 +246,9 @@ func (b *BuildManager) parseWorker() {
}
if skipping {
pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).
pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).
SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status).
SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).ExecX(context.Background())
SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).SaveX(context.Background())
b.repoPurge[pkg.FullRepo] <- pkg
b.parseWG.Done()
continue
@ -251,6 +256,10 @@ func (b *BuildManager) parseWorker() {
pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetVersion(pkg.Version).SaveX(context.Background())
}
if contains(conf.Blacklist.LTO, pkg.Pkgbase) {
pkg.DbPackage = pkg.DbPackage.Update().SetLto(dbpackage.LtoDisabled).SaveX(context.Background())
}
repoVer, err := pkg.repoVersion()
if err != nil {
pkg.DbPackage = pkg.DbPackage.Update().ClearRepoVersion().SaveX(context.Background())
@ -309,19 +318,20 @@ func (b *BuildManager) parseWorker() {
func (b *BuildManager) htmlWorker() {
type Pkg struct {
Pkgbase string
Status string
Class string
Skip string
Version string
Svn2GitVersion string
BuildDate string
BuildDuration time.Duration
Checked string
Log string
LTO bool
LTOUnknown bool
LTODisabled bool
Pkgbase string
Status string
Class string
Skip string
Version string
Svn2GitVersion string
BuildDate string
BuildDuration time.Duration
Checked string
Log string
LTO bool
LTOUnknown bool
LTODisabled bool
LTOAutoDisabled bool
}
type Repo struct {
@ -394,6 +404,8 @@ func (b *BuildManager) htmlWorker() {
addPkg.LTO = true
case dbpackage.LtoDisabled:
addPkg.LTODisabled = true
case dbpackage.LtoAutoDisabled:
addPkg.LTOAutoDisabled = true
}
addRepo.Packages = append(addRepo.Packages, addPkg)

3
tpl/packages.html
View File

@ -82,6 +82,9 @@
title="build with LTO"></i>{{end}}
{{if $pkg.LTODisabled}}<i class="fa fa-times" style="color: var(--bs-danger)"
title="LTO explicitly disabled"></i>{{end}}
{{if $pkg.LTOAutoDisabled}}<i class="fa fa-times-circle-o"
style="color: var(--bs-danger)"
title="LTO automatically disabled"></i>{{end}}
{{if $pkg.LTOUnknown}}<i class="fa fa-hourglass-o" title="not build with LTO yet"></i>{{end}}
</td>
<td>{{$pkg.Svn2GitVersion}}</td>

7
utils.go
View File

@ -39,6 +39,7 @@ var (
rePkgSource = regexp.MustCompile(`(?msU)^source.*=.*\((.+)\)$`)
rePkgSum = regexp.MustCompile(`(?msU)^sha256sums.*=.*\((.+)\)$`)
rePkgFile = regexp.MustCompile(`^(.+)(?:-.+){2}-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$`)
reLdError = regexp.MustCompile(`(?mi)^\s*collect2: error: ld returned (\d+) exit status$`)
)
type BuildPackage struct {
@ -354,10 +355,10 @@ func packages2slice(pkgs interface{}) []string {
}
}
func importKeys(pkg *BuildPackage) error {
if pkg.Srcinfo.ValidPGPKeys != nil {
func (p *BuildPackage) importKeys() error {
if p.Srcinfo.ValidPGPKeys != nil {
args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"}
args = append(args, pkg.Srcinfo.ValidPGPKeys...)
args = append(args, p.Srcinfo.ValidPGPKeys...)
cmd := exec.Command("gpg", args...)
_, err := cmd.CombinedOutput()