removed status page code, provided by api now
This commit is contained in:
parent
1c1f1fba17
commit
352ce97b88
|
@ -1,7 +1,7 @@
|
||||||
# ALHP
|
# ALHP
|
||||||
|
|
||||||
[![](https://img.shields.io/badge/license-GPL-blue?style=flat-square)](https://somegit.dev/anonfunc/ALHP.GO/src/branch/master/LICENSE)
|
[![](https://img.shields.io/badge/license-GPL-blue?style=flat-square)](https://somegit.dev/anonfunc/ALHP.GO/src/branch/master/LICENSE)
|
||||||
[![](https://img.shields.io/badge/package-status-informational?style=flat-square)](https://alhp.dev/packages.html)
|
[![](https://img.shields.io/badge/package-status-informational?style=flat-square)](https://status.alhp.dev)
|
||||||
[![](https://goreportcard.com/badge/somegit.dev/ALHP/ALHP.GO?style=flat-square)](https://goreportcard.com/report/somegit.dev/ALHP/ALHP.GO)
|
[![](https://goreportcard.com/badge/somegit.dev/ALHP/ALHP.GO?style=flat-square)](https://goreportcard.com/report/somegit.dev/ALHP/ALHP.GO)
|
||||||
[![](https://pkg.go.dev/badge/somegit.dev/ALHP/ALHP.GO)](https://pkg.go.dev/somegit.dev/ALHP/ALHP.GO)
|
[![](https://pkg.go.dev/badge/somegit.dev/ALHP/ALHP.GO)](https://pkg.go.dev/somegit.dev/ALHP/ALHP.GO)
|
||||||
[![](https://img.shields.io/liberapay/patrons/anonfunc.svg?logo=liberapay&style=flat-square)](https://liberapay.com/anonfunc/)
|
[![](https://img.shields.io/liberapay/patrons/anonfunc.svg?logo=liberapay&style=flat-square)](https://liberapay.com/anonfunc/)
|
||||||
|
@ -159,7 +159,7 @@ so, [see alhp-mirrorlist for how to become one](https://somegit.dev/ALHP/alhp-mi
|
||||||
|
|
||||||
Packages [excluded](https://www.reddit.com/r/archlinux/comments/oflged/alhp_archlinux_recompiled_for_x8664v3_experimental/h4fkinu?utm_source=share&utm_medium=web2x&context=3)
|
Packages [excluded](https://www.reddit.com/r/archlinux/comments/oflged/alhp_archlinux_recompiled_for_x8664v3_experimental/h4fkinu?utm_source=share&utm_medium=web2x&context=3)
|
||||||
from building (besides all 'any' architecture packages) are being listed in issue #16.
|
from building (besides all 'any' architecture packages) are being listed in issue #16.
|
||||||
Also [package status page](https://alhp.dev/packages.html) (search for `blacklisted`).
|
Also [package status page](https://status.alhp.dev) (search for `blacklisted`).
|
||||||
|
|
||||||
### Why is package X not up-to-date
|
### Why is package X not up-to-date
|
||||||
|
|
||||||
|
@ -170,7 +170,7 @@ This leads to packages being delayed if the current batch contains many packages
|
||||||
build (e.g. `chromium`).
|
build (e.g. `chromium`).
|
||||||
|
|
||||||
You can always check on the progress of the current build-cycle on
|
You can always check on the progress of the current build-cycle on
|
||||||
the [package status page](https://alhp.dev/packages.html).
|
the [package status page](https://status.alhp.dev).
|
||||||
Please refrain from opening issues caused by packages currently in queue/not yet build/not yet moved to the repo.
|
Please refrain from opening issues caused by packages currently in queue/not yet build/not yet moved to the repo.
|
||||||
Please keep in mind that large rebuilds such as `openssl` or `python` can take days to complete on our current build
|
Please keep in mind that large rebuilds such as `openssl` or `python` can take days to complete on our current build
|
||||||
hardware.
|
hardware.
|
||||||
|
|
186
buildmanager.go
186
buildmanager.go
|
@ -7,11 +7,9 @@ import (
|
||||||
"github.com/c2h5oh/datasize"
|
"github.com/c2h5oh/datasize"
|
||||||
"github.com/sethvargo/go-retry"
|
"github.com/sethvargo/go-retry"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"html/template"
|
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"somegit.dev/ALHP/ALHP.GO/ent"
|
|
||||||
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
|
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
@ -149,190 +147,6 @@ func (b *BuildManager) buildQueue(ctx context.Context, queue []*ProtoPackage) er
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *BuildManager) htmlWorker(ctx context.Context) {
|
|
||||||
type Pkg struct {
|
|
||||||
Pkgbase string
|
|
||||||
Status string
|
|
||||||
Class string
|
|
||||||
Skip string
|
|
||||||
Version string
|
|
||||||
Svn2GitVersion string
|
|
||||||
BuildDate string
|
|
||||||
BuildDuration time.Duration
|
|
||||||
BuildMemory *string
|
|
||||||
Checked string
|
|
||||||
Log string
|
|
||||||
LTO bool
|
|
||||||
LTOUnknown bool
|
|
||||||
LTODisabled bool
|
|
||||||
LTOAutoDisabled bool
|
|
||||||
DebugSym bool
|
|
||||||
DebugSymNotAvailable bool
|
|
||||||
DebugSymUnknown bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type Repo struct {
|
|
||||||
Name string
|
|
||||||
Packages []Pkg
|
|
||||||
}
|
|
||||||
|
|
||||||
type March struct {
|
|
||||||
Name string
|
|
||||||
Repos []Repo
|
|
||||||
}
|
|
||||||
|
|
||||||
type tpl struct {
|
|
||||||
March []March
|
|
||||||
Generated string
|
|
||||||
Latest int
|
|
||||||
Failed int
|
|
||||||
Skipped int
|
|
||||||
Queued int
|
|
||||||
LTOEnabled int
|
|
||||||
LTOUnknown int
|
|
||||||
LTODisabled int
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
|
||||||
gen := &tpl{}
|
|
||||||
|
|
||||||
for _, march := range conf.March {
|
|
||||||
addMarch := March{
|
|
||||||
Name: march,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, repo := range conf.Repos {
|
|
||||||
addRepo := Repo{
|
|
||||||
Name: repo,
|
|
||||||
}
|
|
||||||
|
|
||||||
pkgs := db.DBPackage.Query().Order(ent.Asc(dbpackage.FieldPkgbase)).
|
|
||||||
Where(dbpackage.MarchEQ(march), dbpackage.RepositoryEQ(dbpackage.Repository(repo))).AllX(ctx)
|
|
||||||
|
|
||||||
for _, pkg := range pkgs {
|
|
||||||
addPkg := Pkg{
|
|
||||||
Pkgbase: pkg.Pkgbase,
|
|
||||||
Status: strings.ToUpper(pkg.Status.String()),
|
|
||||||
Class: statusID2string(pkg.Status),
|
|
||||||
Skip: pkg.SkipReason,
|
|
||||||
Version: pkg.RepoVersion,
|
|
||||||
Svn2GitVersion: pkg.Version,
|
|
||||||
}
|
|
||||||
|
|
||||||
if pkg.STime != nil && pkg.UTime != nil {
|
|
||||||
addPkg.BuildDuration = time.Duration(*pkg.STime+*pkg.UTime) * time.Second
|
|
||||||
}
|
|
||||||
|
|
||||||
if !pkg.BuildTimeStart.IsZero() {
|
|
||||||
addPkg.BuildDate = pkg.BuildTimeStart.UTC().Format(time.RFC1123)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !pkg.Updated.IsZero() {
|
|
||||||
addPkg.Checked = pkg.Updated.UTC().Format(time.RFC1123)
|
|
||||||
}
|
|
||||||
|
|
||||||
if pkg.Status == dbpackage.StatusFailed {
|
|
||||||
addPkg.Log = fmt.Sprintf("%s/%s/%s.log", logDir, pkg.March, pkg.Pkgbase)
|
|
||||||
}
|
|
||||||
|
|
||||||
if pkg.MaxRss != nil {
|
|
||||||
hrSize := (datasize.ByteSize(*pkg.MaxRss) * datasize.KB).HumanReadable()
|
|
||||||
addPkg.BuildMemory = &hrSize
|
|
||||||
}
|
|
||||||
|
|
||||||
switch pkg.Lto {
|
|
||||||
case dbpackage.LtoUnknown:
|
|
||||||
if pkg.Status != dbpackage.StatusSkipped && pkg.Status != dbpackage.StatusFailed {
|
|
||||||
addPkg.LTOUnknown = true
|
|
||||||
}
|
|
||||||
case dbpackage.LtoEnabled:
|
|
||||||
addPkg.LTO = true
|
|
||||||
case dbpackage.LtoDisabled:
|
|
||||||
addPkg.LTODisabled = true
|
|
||||||
case dbpackage.LtoAutoDisabled:
|
|
||||||
addPkg.LTOAutoDisabled = true
|
|
||||||
}
|
|
||||||
|
|
||||||
switch pkg.DebugSymbols {
|
|
||||||
case dbpackage.DebugSymbolsUnknown:
|
|
||||||
if pkg.Status != dbpackage.StatusSkipped && pkg.Status != dbpackage.StatusFailed {
|
|
||||||
addPkg.DebugSymUnknown = true
|
|
||||||
}
|
|
||||||
case dbpackage.DebugSymbolsAvailable:
|
|
||||||
addPkg.DebugSym = true
|
|
||||||
case dbpackage.DebugSymbolsNotAvailable:
|
|
||||||
addPkg.DebugSymNotAvailable = true
|
|
||||||
}
|
|
||||||
|
|
||||||
addRepo.Packages = append(addRepo.Packages, addPkg)
|
|
||||||
}
|
|
||||||
addMarch.Repos = append(addMarch.Repos, addRepo)
|
|
||||||
}
|
|
||||||
gen.March = append(gen.March, addMarch)
|
|
||||||
}
|
|
||||||
|
|
||||||
gen.Generated = time.Now().UTC().Format(time.RFC1123)
|
|
||||||
|
|
||||||
var v []struct {
|
|
||||||
Status dbpackage.Status `json:"status"`
|
|
||||||
Count int `json:"count"`
|
|
||||||
}
|
|
||||||
|
|
||||||
db.DBPackage.Query().GroupBy(dbpackage.FieldStatus).Aggregate(ent.Count()).ScanX(ctx, &v)
|
|
||||||
|
|
||||||
for _, c := range v {
|
|
||||||
switch c.Status { //nolint:exhaustive
|
|
||||||
case dbpackage.StatusFailed:
|
|
||||||
gen.Failed = c.Count
|
|
||||||
case dbpackage.StatusSkipped:
|
|
||||||
gen.Skipped = c.Count
|
|
||||||
case dbpackage.StatusLatest:
|
|
||||||
gen.Latest = c.Count
|
|
||||||
case dbpackage.StatusQueued:
|
|
||||||
gen.Queued = c.Count
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var v2 []struct {
|
|
||||||
Status dbpackage.Lto `json:"lto"` //nolint:tagliatelle
|
|
||||||
Count int `json:"count"`
|
|
||||||
}
|
|
||||||
|
|
||||||
db.DBPackage.Query().Where(dbpackage.StatusNEQ(dbpackage.StatusSkipped)).
|
|
||||||
GroupBy(dbpackage.FieldLto).Aggregate(ent.Count()).ScanX(ctx, &v2)
|
|
||||||
|
|
||||||
for _, c := range v2 {
|
|
||||||
switch c.Status {
|
|
||||||
case dbpackage.LtoUnknown:
|
|
||||||
gen.LTOUnknown = c.Count
|
|
||||||
case dbpackage.LtoDisabled, dbpackage.LtoAutoDisabled:
|
|
||||||
gen.LTODisabled += c.Count
|
|
||||||
case dbpackage.LtoEnabled:
|
|
||||||
gen.LTOEnabled = c.Count
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
statusTpl, err := template.ParseFiles("tpl/packages.html")
|
|
||||||
if err != nil {
|
|
||||||
log.Warningf("[HTML] error parsing template file: %v", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, "packages.html"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
|
||||||
if err != nil {
|
|
||||||
log.Warningf("[HTML] error opening output file: %v", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
err = statusTpl.Execute(f, gen)
|
|
||||||
if err != nil {
|
|
||||||
log.Warningf("[HTML] error filling template: %v", err)
|
|
||||||
}
|
|
||||||
_ = f.Close()
|
|
||||||
|
|
||||||
time.Sleep(time.Minute * 5)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *BuildManager) repoWorker(repo string) {
|
func (b *BuildManager) repoWorker(repo string) {
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
|
|
|
@ -33,16 +33,6 @@ blacklist:
|
||||||
- llvm
|
- llvm
|
||||||
- rust
|
- rust
|
||||||
|
|
||||||
status:
|
|
||||||
class:
|
|
||||||
skipped: "secondary"
|
|
||||||
queued: "warning"
|
|
||||||
latest: "primary"
|
|
||||||
failed: "danger"
|
|
||||||
signing: "success"
|
|
||||||
building: "info"
|
|
||||||
unknown: "dark"
|
|
||||||
|
|
||||||
build:
|
build:
|
||||||
# number of workers total
|
# number of workers total
|
||||||
worker: 4
|
worker: 4
|
||||||
|
|
|
@ -64,7 +64,6 @@ func housekeeping(repo, march string, wg *sync.WaitGroup) error {
|
||||||
pkgResolved.Architecture() != pkg.Arch ||
|
pkgResolved.Architecture() != pkg.Arch ||
|
||||||
pkgResolved.Name() != mPackage.Name() ||
|
pkgResolved.Name() != mPackage.Name() ||
|
||||||
Contains(conf.Blacklist.Packages, pkg.Pkgbase) {
|
Contains(conf.Blacklist.Packages, pkg.Pkgbase) {
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case err != nil:
|
case err != nil:
|
||||||
log.Infof("[HK] %s->%s not included in repo (resolve error: %v)", pkg.FullRepo, mPackage.Name(), err)
|
log.Infof("[HK] %s->%s not included in repo (resolve error: %v)", pkg.FullRepo, mPackage.Name(), err)
|
||||||
|
@ -310,7 +309,7 @@ func debugHK() {
|
||||||
for _, march := range conf.March {
|
for _, march := range conf.March {
|
||||||
if _, err := os.Stat(filepath.Join(conf.Basedir.Debug, march)); err == nil {
|
if _, err := os.Stat(filepath.Join(conf.Basedir.Debug, march)); err == nil {
|
||||||
log.Debugf("[DHK/%s] start cleanup debug packages", march)
|
log.Debugf("[DHK/%s] start cleanup debug packages", march)
|
||||||
cleanCmd := exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Debug, march), "-k", "1")
|
cleanCmd := exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Debug, march), "-k", "1") //nolint:gosec
|
||||||
res, err := cleanCmd.CombinedOutput()
|
res, err := cleanCmd.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warningf("[DHK/%s] cleanup debug packages failed: %v (%s)", march, err, string(res))
|
log.Warningf("[DHK/%s] cleanup debug packages failed: %v (%s)", march, err, string(res))
|
||||||
|
|
1
main.go
1
main.go
|
@ -120,7 +120,6 @@ func main() {
|
||||||
go func() {
|
go func() {
|
||||||
_ = buildManager.syncWorker(ctx)
|
_ = buildManager.syncWorker(ctx)
|
||||||
}()
|
}()
|
||||||
go buildManager.htmlWorker(ctx)
|
|
||||||
|
|
||||||
killLoop:
|
killLoop:
|
||||||
for {
|
for {
|
||||||
|
|
24
utils.go
24
utils.go
|
@ -80,11 +80,6 @@ type Conf struct {
|
||||||
Housekeeping struct {
|
Housekeeping struct {
|
||||||
Interval string
|
Interval string
|
||||||
}
|
}
|
||||||
Status struct {
|
|
||||||
Class struct {
|
|
||||||
Skipped, Queued, Latest, Failed, Signing, Building, Unknown string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MaxCloneRetries uint64 `yaml:"max_clone_retries"`
|
MaxCloneRetries uint64 `yaml:"max_clone_retries"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -112,25 +107,6 @@ func updateLastUpdated() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func statusID2string(s dbpackage.Status) string {
|
|
||||||
switch s { //nolint:exhaustive
|
|
||||||
case dbpackage.StatusSkipped:
|
|
||||||
return conf.Status.Class.Skipped
|
|
||||||
case dbpackage.StatusQueued:
|
|
||||||
return conf.Status.Class.Queued
|
|
||||||
case dbpackage.StatusLatest:
|
|
||||||
return conf.Status.Class.Latest
|
|
||||||
case dbpackage.StatusFailed:
|
|
||||||
return conf.Status.Class.Failed
|
|
||||||
case dbpackage.StatusSigning:
|
|
||||||
return conf.Status.Class.Signing
|
|
||||||
case dbpackage.StatusBuilding, dbpackage.StatusDelayed:
|
|
||||||
return conf.Status.Class.Building
|
|
||||||
default:
|
|
||||||
return conf.Status.Class.Unknown
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func cleanBuildDir(dir, chrootDir string) error {
|
func cleanBuildDir(dir, chrootDir string) error {
|
||||||
if stat, err := os.Stat(dir); err == nil && stat.IsDir() {
|
if stat, err := os.Stat(dir); err == nil && stat.IsDir() {
|
||||||
rmCmd := exec.Command("sudo", "rm_chroot.py", dir)
|
rmCmd := exec.Command("sudo", "rm_chroot.py", dir)
|
||||||
|
|
Loading…
Reference in New Issue