From 352ce97b88970d79140b376e47b7b6341a292693 Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Sun, 17 Dec 2023 21:02:44 +0100 Subject: [PATCH] removed status page code, provided by api now --- README.md | 6 +- buildmanager.go | 186 ----------------------------------------------- config_dist.yaml | 10 --- housekeeping.go | 3 +- main.go | 1 - utils.go | 24 ------ 6 files changed, 4 insertions(+), 226 deletions(-) diff --git a/README.md b/README.md index 846a726..6fd3d1d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # ALHP [![](https://img.shields.io/badge/license-GPL-blue?style=flat-square)](https://somegit.dev/anonfunc/ALHP.GO/src/branch/master/LICENSE) -[![](https://img.shields.io/badge/package-status-informational?style=flat-square)](https://alhp.dev/packages.html) +[![](https://img.shields.io/badge/package-status-informational?style=flat-square)](https://status.alhp.dev) [![](https://goreportcard.com/badge/somegit.dev/ALHP/ALHP.GO?style=flat-square)](https://goreportcard.com/report/somegit.dev/ALHP/ALHP.GO) [![](https://pkg.go.dev/badge/somegit.dev/ALHP/ALHP.GO)](https://pkg.go.dev/somegit.dev/ALHP/ALHP.GO) [![](https://img.shields.io/liberapay/patrons/anonfunc.svg?logo=liberapay&style=flat-square)](https://liberapay.com/anonfunc/) @@ -159,7 +159,7 @@ so, [see alhp-mirrorlist for how to become one](https://somegit.dev/ALHP/alhp-mi Packages [excluded](https://www.reddit.com/r/archlinux/comments/oflged/alhp_archlinux_recompiled_for_x8664v3_experimental/h4fkinu?utm_source=share&utm_medium=web2x&context=3) from building (besides all 'any' architecture packages) are being listed in issue #16. -Also [package status page](https://alhp.dev/packages.html) (search for `blacklisted`). +Also [package status page](https://status.alhp.dev) (search for `blacklisted`). ### Why is package X not up-to-date @@ -170,7 +170,7 @@ This leads to packages being delayed if the current batch contains many packages build (e.g. `chromium`). You can always check on the progress of the current build-cycle on -the [package status page](https://alhp.dev/packages.html). +the [package status page](https://status.alhp.dev). Please refrain from opening issues caused by packages currently in queue/not yet build/not yet moved to the repo. Please keep in mind that large rebuilds such as `openssl` or `python` can take days to complete on our current build hardware. diff --git a/buildmanager.go b/buildmanager.go index b52a307..7ae0698 100644 --- a/buildmanager.go +++ b/buildmanager.go @@ -7,11 +7,9 @@ import ( "github.com/c2h5oh/datasize" "github.com/sethvargo/go-retry" log "github.com/sirupsen/logrus" - "html/template" "os" "os/exec" "path/filepath" - "somegit.dev/ALHP/ALHP.GO/ent" "somegit.dev/ALHP/ALHP.GO/ent/dbpackage" "strings" "sync" @@ -149,190 +147,6 @@ func (b *BuildManager) buildQueue(ctx context.Context, queue []*ProtoPackage) er return nil } -func (b *BuildManager) htmlWorker(ctx context.Context) { - type Pkg struct { - Pkgbase string - Status string - Class string - Skip string - Version string - Svn2GitVersion string - BuildDate string - BuildDuration time.Duration - BuildMemory *string - Checked string - Log string - LTO bool - LTOUnknown bool - LTODisabled bool - LTOAutoDisabled bool - DebugSym bool - DebugSymNotAvailable bool - DebugSymUnknown bool - } - - type Repo struct { - Name string - Packages []Pkg - } - - type March struct { - Name string - Repos []Repo - } - - type tpl struct { - March []March - Generated string - Latest int - Failed int - Skipped int - Queued int - LTOEnabled int - LTOUnknown int - LTODisabled int - } - - for { - gen := &tpl{} - - for _, march := range conf.March { - addMarch := March{ - Name: march, - } - - for _, repo := range conf.Repos { - addRepo := Repo{ - Name: repo, - } - - pkgs := db.DBPackage.Query().Order(ent.Asc(dbpackage.FieldPkgbase)). - Where(dbpackage.MarchEQ(march), dbpackage.RepositoryEQ(dbpackage.Repository(repo))).AllX(ctx) - - for _, pkg := range pkgs { - addPkg := Pkg{ - Pkgbase: pkg.Pkgbase, - Status: strings.ToUpper(pkg.Status.String()), - Class: statusID2string(pkg.Status), - Skip: pkg.SkipReason, - Version: pkg.RepoVersion, - Svn2GitVersion: pkg.Version, - } - - if pkg.STime != nil && pkg.UTime != nil { - addPkg.BuildDuration = time.Duration(*pkg.STime+*pkg.UTime) * time.Second - } - - if !pkg.BuildTimeStart.IsZero() { - addPkg.BuildDate = pkg.BuildTimeStart.UTC().Format(time.RFC1123) - } - - if !pkg.Updated.IsZero() { - addPkg.Checked = pkg.Updated.UTC().Format(time.RFC1123) - } - - if pkg.Status == dbpackage.StatusFailed { - addPkg.Log = fmt.Sprintf("%s/%s/%s.log", logDir, pkg.March, pkg.Pkgbase) - } - - if pkg.MaxRss != nil { - hrSize := (datasize.ByteSize(*pkg.MaxRss) * datasize.KB).HumanReadable() - addPkg.BuildMemory = &hrSize - } - - switch pkg.Lto { - case dbpackage.LtoUnknown: - if pkg.Status != dbpackage.StatusSkipped && pkg.Status != dbpackage.StatusFailed { - addPkg.LTOUnknown = true - } - case dbpackage.LtoEnabled: - addPkg.LTO = true - case dbpackage.LtoDisabled: - addPkg.LTODisabled = true - case dbpackage.LtoAutoDisabled: - addPkg.LTOAutoDisabled = true - } - - switch pkg.DebugSymbols { - case dbpackage.DebugSymbolsUnknown: - if pkg.Status != dbpackage.StatusSkipped && pkg.Status != dbpackage.StatusFailed { - addPkg.DebugSymUnknown = true - } - case dbpackage.DebugSymbolsAvailable: - addPkg.DebugSym = true - case dbpackage.DebugSymbolsNotAvailable: - addPkg.DebugSymNotAvailable = true - } - - addRepo.Packages = append(addRepo.Packages, addPkg) - } - addMarch.Repos = append(addMarch.Repos, addRepo) - } - gen.March = append(gen.March, addMarch) - } - - gen.Generated = time.Now().UTC().Format(time.RFC1123) - - var v []struct { - Status dbpackage.Status `json:"status"` - Count int `json:"count"` - } - - db.DBPackage.Query().GroupBy(dbpackage.FieldStatus).Aggregate(ent.Count()).ScanX(ctx, &v) - - for _, c := range v { - switch c.Status { //nolint:exhaustive - case dbpackage.StatusFailed: - gen.Failed = c.Count - case dbpackage.StatusSkipped: - gen.Skipped = c.Count - case dbpackage.StatusLatest: - gen.Latest = c.Count - case dbpackage.StatusQueued: - gen.Queued = c.Count - } - } - - var v2 []struct { - Status dbpackage.Lto `json:"lto"` //nolint:tagliatelle - Count int `json:"count"` - } - - db.DBPackage.Query().Where(dbpackage.StatusNEQ(dbpackage.StatusSkipped)). - GroupBy(dbpackage.FieldLto).Aggregate(ent.Count()).ScanX(ctx, &v2) - - for _, c := range v2 { - switch c.Status { - case dbpackage.LtoUnknown: - gen.LTOUnknown = c.Count - case dbpackage.LtoDisabled, dbpackage.LtoAutoDisabled: - gen.LTODisabled += c.Count - case dbpackage.LtoEnabled: - gen.LTOEnabled = c.Count - } - } - - statusTpl, err := template.ParseFiles("tpl/packages.html") - if err != nil { - log.Warningf("[HTML] error parsing template file: %v", err) - continue - } - - f, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, "packages.html"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) - if err != nil { - log.Warningf("[HTML] error opening output file: %v", err) - continue - } - err = statusTpl.Execute(f, gen) - if err != nil { - log.Warningf("[HTML] error filling template: %v", err) - } - _ = f.Close() - - time.Sleep(time.Minute * 5) - } -} - func (b *BuildManager) repoWorker(repo string) { for { select { diff --git a/config_dist.yaml b/config_dist.yaml index 7ed94e5..6dfae1f 100644 --- a/config_dist.yaml +++ b/config_dist.yaml @@ -33,16 +33,6 @@ blacklist: - llvm - rust -status: - class: - skipped: "secondary" - queued: "warning" - latest: "primary" - failed: "danger" - signing: "success" - building: "info" - unknown: "dark" - build: # number of workers total worker: 4 diff --git a/housekeeping.go b/housekeeping.go index 3ba1ea0..a51c79f 100644 --- a/housekeeping.go +++ b/housekeeping.go @@ -64,7 +64,6 @@ func housekeeping(repo, march string, wg *sync.WaitGroup) error { pkgResolved.Architecture() != pkg.Arch || pkgResolved.Name() != mPackage.Name() || Contains(conf.Blacklist.Packages, pkg.Pkgbase) { - switch { case err != nil: log.Infof("[HK] %s->%s not included in repo (resolve error: %v)", pkg.FullRepo, mPackage.Name(), err) @@ -310,7 +309,7 @@ func debugHK() { for _, march := range conf.March { if _, err := os.Stat(filepath.Join(conf.Basedir.Debug, march)); err == nil { log.Debugf("[DHK/%s] start cleanup debug packages", march) - cleanCmd := exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Debug, march), "-k", "1") + cleanCmd := exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Debug, march), "-k", "1") //nolint:gosec res, err := cleanCmd.CombinedOutput() if err != nil { log.Warningf("[DHK/%s] cleanup debug packages failed: %v (%s)", march, err, string(res)) diff --git a/main.go b/main.go index 57e7e47..e5664be 100644 --- a/main.go +++ b/main.go @@ -120,7 +120,6 @@ func main() { go func() { _ = buildManager.syncWorker(ctx) }() - go buildManager.htmlWorker(ctx) killLoop: for { diff --git a/utils.go b/utils.go index 602b0f0..fe9c87e 100644 --- a/utils.go +++ b/utils.go @@ -80,11 +80,6 @@ type Conf struct { Housekeeping struct { Interval string } - Status struct { - Class struct { - Skipped, Queued, Latest, Failed, Signing, Building, Unknown string - } - } MaxCloneRetries uint64 `yaml:"max_clone_retries"` } @@ -112,25 +107,6 @@ func updateLastUpdated() error { return nil } -func statusID2string(s dbpackage.Status) string { - switch s { //nolint:exhaustive - case dbpackage.StatusSkipped: - return conf.Status.Class.Skipped - case dbpackage.StatusQueued: - return conf.Status.Class.Queued - case dbpackage.StatusLatest: - return conf.Status.Class.Latest - case dbpackage.StatusFailed: - return conf.Status.Class.Failed - case dbpackage.StatusSigning: - return conf.Status.Class.Signing - case dbpackage.StatusBuilding, dbpackage.StatusDelayed: - return conf.Status.Class.Building - default: - return conf.Status.Class.Unknown - } -} - func cleanBuildDir(dir, chrootDir string) error { if stat, err := os.Stat(dir); err == nil && stat.IsDir() { rmCmd := exec.Command("sudo", "rm_chroot.py", dir)