diff --git a/buildmanager.go b/buildmanager.go index 59046d3..3691c8f 100644 --- a/buildmanager.go +++ b/buildmanager.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/Jguer/go-alpm/v2" "github.com/c2h5oh/datasize" "github.com/prometheus/client_golang/prometheus" "github.com/sethvargo/go-retry" @@ -464,14 +465,52 @@ func (b *BuildManager) genQueue(ctx context.Context) ([]*ProtoPackage, error) { } } - if pkg.DBPackage.TagRev != nil && *pkg.DBPackage.TagRev == state.TagRev { + // Cross-check against the live pacman DB: state.git lags + // reality for pkgrel-only rebuilds and arch-any moves leave + // stale x86_64 state files behind. SyncPkg was resolved by + // isAvailable above; reuse it. + if pkg.SyncPkg != nil { + if pkg.SyncPkg.Architecture() == "any" && pkg.Arch == "x86_64" { + // Already marked from a prior cycle: nothing to redo. + if pkg.DBPackage.Status == dbpackage.StatusSkipped && + pkg.DBPackage.SkipReason == SkipReasonAnyArchMoved { + continue + } + log.Infof("[QG] %s->%s upstream moved to any-arch, dropping", pkg.FullRepo, pkg.Pkgbase) + pkg.DBPackage, err = pkg.DBPackage.Update(). + SetStatus(dbpackage.StatusSkipped). + SetSkipReason(SkipReasonAnyArchMoved). + SetTagRev(state.TagRev). + SetVersion(state.PkgVer). + Save(ctx) + if err != nil { + log.Warningf("[QG] error updating dbpackage %s: %v", state.Pkgbase, err) + continue + } + buildManager.repoPurge[pkg.FullRepo] <- []*ProtoPackage{pkg} + continue + } + if alpm.VerCmp(pkg.SyncPkg.Version(), state.PkgVer) > 0 { + log.Infof("[QG] %s->%s upstream version drift detected (state: %s < upstream: %s), building from %s", + pkg.FullRepo, pkg.Pkgbase, state.PkgVer, pkg.SyncPkg.Version(), upstreamDefaultGitBranch) + pkg.UseLatest = true + } + } + + if !pkg.UseLatest && pkg.DBPackage.TagRev != nil && *pkg.DBPackage.TagRev == state.TagRev { continue } - // try download .SRCINFO from repo - srcInfo, err := downloadSRCINFO(pkg.DBPackage.Pkgbase, state.TagRev) + srcRef := state.TagRev + if pkg.UseLatest { + srcRef = upstreamDefaultGitBranch + } + srcInfo, err := downloadSRCINFO(pkg.DBPackage.Pkgbase, srcRef) if err == nil { pkg.Srcinfo = srcInfo + if pkg.UseLatest { + pkg.Version = constructVersion(srcInfo.Pkgver, srcInfo.Pkgrel, srcInfo.Epoch) + } } if !pkg.isEligible(ctx) { diff --git a/housekeeping.go b/housekeeping.go index b627482..b2941ee 100644 --- a/housekeeping.go +++ b/housekeeping.go @@ -2,6 +2,7 @@ package main import ( "context" + "github.com/Jguer/go-alpm/v2" log "github.com/sirupsen/logrus" "os" "os/exec" @@ -146,6 +147,26 @@ func housekeeping(ctx context.Context, repo, march string, wg *sync.WaitGroup) e return err } } + + // detect upstream version drift: Arch sometimes ships a pkgrel + // rebuild without updating state.git, so state.TagRev never + // changes and genQueue never re-queues. Compare directly against + // the pacman sync DB (always current) and force a re-queue. + // Only act on settled packages; a Queued/Building/Delayed row is + // already moving and our SetStatus(Queued) would race it. + if pkg.DBPackage.Status == dbpackage.StatusLatest && + pkg.DBPackage.Version != "" && pkgResolved.Version() != "" && + alpm.VerCmp(pkgResolved.Version(), pkg.DBPackage.Version) > 0 { + log.Infof("[HK] %s->%s upstream version drift detected (db: %s < upstream: %s), requeuing", + pkg.FullRepo, pkg.Pkgbase, pkg.DBPackage.Version, pkgResolved.Version()) + pkg.DBPackage, err = pkg.DBPackage.Update(). + SetStatus(dbpackage.StatusQueued). + ClearTagRev(). + Save(ctx) + if err != nil { + return err + } + } } // check all packages from db for existence diff --git a/proto_package.go b/proto_package.go index 3a7cbe7..68df56e 100644 --- a/proto_package.go +++ b/proto_package.go @@ -24,6 +24,14 @@ import ( "time" ) +// Skip reasons surfaced from the build pipeline. Hoisted so the same string +// is used wherever a package is skipped for that reason. +const ( + SkipReasonAnyArch = "arch = any" + SkipReasonAnyArchMoved = "arch = any (moved upstream)" + upstreamDefaultGitBranch = "main" +) + type ProtoPackage struct { Pkgbase string Srcinfo *srcinfo.Srcinfo @@ -36,6 +44,13 @@ type ProtoPackage struct { DBPackage *ent.DBPackage Pkgbuild string State *StateInfo + // SyncPkg is the alpm package resolved by isAvailable; cached so + // later checks (drift, arch-any-move) don't re-do FindSatisfier. + SyncPkg alpm.IPackage + // UseLatest causes SRCINFO/PKGBUILD to be fetched from + // upstreamDefaultGitBranch (main) instead of state.TagVer/state.TagRev. + // Set when upstream Arch ships a rebuild that state.git did not record. + UseLatest bool } var ( @@ -47,7 +62,7 @@ func (p *ProtoPackage) isEligible(ctx context.Context) bool { switch { case p.Arch == "any": log.Debugf("skipped %s: any-package", p.Pkgbase) - p.DBPackage.SkipReason = "arch = any" + p.DBPackage.SkipReason = SkipReasonAnyArch p.DBPackage.Status = dbpackage.StatusSkipped skipping = true case MatchGlobList(p.Pkgbase, conf.Blacklist.Packages): @@ -365,7 +380,7 @@ func (p *ProtoPackage) setupBuildDir(ctx context.Context) (string, error) { gr = retry.WithMaxRetries(conf.MaxCloneRetries, gr) if err := retry.Do(ctx, gr, func(ctx context.Context) error { - cmd := exec.CommandContext(ctx, "git", "clone", "--depth", "1", "--branch", p.State.TagVer, //nolint:gosec + cmd := exec.CommandContext(ctx, "git", "clone", "--depth", "1", "--branch", p.cloneBranch(), //nolint:gosec fmt.Sprintf("https://gitlab.archlinux.org/archlinux/packaging/packages/%s.git", gitlabPath), buildDir) res, err := cmd.CombinedOutput() log.Debug(string(res)) @@ -541,9 +556,21 @@ func (p *ProtoPackage) isAvailable(ctx context.Context, h *alpm.Handle) bool { return false } + p.SyncPkg = pkg return true } +// cloneBranch returns the git ref to use when cloning the upstream +// packaging repo. The default ref is the tag recorded in state.git; +// UseLatest overrides that to track the package repo's main branch +// (used when state.git lags real upstream). +func (p *ProtoPackage) cloneBranch() string { + if p.UseLatest { + return upstreamDefaultGitBranch + } + return p.State.TagVer +} + func (p *ProtoPackage) GitVersion(h *alpm.Handle) (string, error) { if p.Pkgbase == "" { return "", errors.New("invalid arguments") diff --git a/proto_package_test.go b/proto_package_test.go index 2f9b282..8dfb1f5 100644 --- a/proto_package_test.go +++ b/proto_package_test.go @@ -96,6 +96,35 @@ package() { # vim:set sw=2 et: ` +func TestCloneBranch(t *testing.T) { //nolint:paralleltest + for _, tc := range []struct { + name string + useLatest bool + tagVer string + want string + }{ + {"state-tag-default", false, "3.3-4", "3.3-4"}, + {"main-when-drift", true, "3.3-4", "main"}, + {"main-ignores-empty-tag", true, "", "main"}, + } { + t.Run(tc.name, func(t *testing.T) { + p := &ProtoPackage{ + UseLatest: tc.useLatest, + State: &StateInfo{TagVer: tc.tagVer}, + } + if got := p.cloneBranch(); got != tc.want { + t.Errorf("cloneBranch() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestSkipReasonsDistinct(t *testing.T) { //nolint:paralleltest + if SkipReasonAnyArch == SkipReasonAnyArchMoved { + t.Fatal("any-arch skip reasons must remain distinguishable: callers and operators inspect skip_reason to tell why a package was dropped") + } +} + func TestIncreasePkgRel(t *testing.T) { //nolint:paralleltest pkgbuild, err := os.CreateTemp(t.TempDir(), "") if err != nil {