2021-06-10 21:32:11 +02:00
package main
import (
2021-07-13 18:07:29 +02:00
"ALHP.go/ent"
"ALHP.go/ent/dbpackage"
2021-07-26 16:38:12 +02:00
"ALHP.go/ent/migrate"
2021-06-14 15:52:53 +02:00
"bytes"
2021-07-13 18:07:29 +02:00
"context"
2021-09-12 17:25:33 +02:00
"flag"
2021-06-10 21:32:11 +02:00
"fmt"
"github.com/Jguer/go-alpm/v2"
"github.com/Morganamilo/go-srcinfo"
2021-07-13 18:07:29 +02:00
_ "github.com/mattn/go-sqlite3"
2021-06-10 21:32:11 +02:00
log "github.com/sirupsen/logrus"
2021-08-30 11:05:11 +02:00
"github.com/wercker/journalhook"
2021-06-10 21:32:11 +02:00
"gopkg.in/yaml.v2"
2021-07-13 18:07:29 +02:00
"html/template"
2021-06-10 21:32:11 +02:00
"os"
"os/exec"
"os/signal"
"path/filepath"
"regexp"
2021-07-01 21:40:49 +02:00
"runtime"
2021-06-10 21:32:11 +02:00
"strconv"
"strings"
"sync"
"syscall"
"time"
)
var (
2021-09-12 17:35:45 +02:00
conf = Conf { }
repos [ ] string
alpmHandle * alpm . Handle
reMarch = regexp . MustCompile ( ` (-march=)(.+?) ` )
rePkgRel = regexp . MustCompile ( ` (?m)^pkgrel\s*=\s*(.+)$ ` )
rePkgFile = regexp . MustCompile ( ` ^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$ ` )
buildManager BuildManager
db * ent . Client
dbLock sync . RWMutex
journalLog = flag . Bool ( "journal" , false , "Log to systemd journal instead of stdout" )
checkInterval = flag . Int ( "interval" , 5 , "How often svn2git should be checked in minutes (default: 5)" )
2021-06-10 21:32:11 +02:00
)
func ( b * BuildManager ) buildWorker ( id int ) {
2021-06-30 17:41:50 +02:00
err := syscall . Setpriority ( syscall . PRIO_PROCESS , 0 , 18 )
if err != nil {
log . Warningf ( "[worker-%d] Failed to drop priority: %v" , id , err )
}
2021-06-14 16:28:00 +02:00
2021-06-10 21:32:11 +02:00
for {
select {
2021-07-09 20:37:00 +02:00
case pkg := <- b . build :
2021-06-10 21:32:11 +02:00
if b . exit {
2021-06-14 15:52:53 +02:00
log . Infof ( "Worker %d exited..." , id )
return
2021-06-10 21:32:11 +02:00
} else {
2021-07-01 21:13:48 +02:00
b . buildWG . Add ( 1 )
2021-06-10 21:32:11 +02:00
}
start := time . Now ( )
log . Infof ( "[%s/%s] Build starting" , pkg . FullRepo , pkg . Pkgbase )
2021-07-13 18:07:29 +02:00
dbPkg := getDbPackage ( pkg )
dbLock . Lock ( )
2021-08-04 17:42:42 +02:00
dbPkg . Update ( ) . SetStatus ( BUILDING ) . SetBuildTime ( time . Now ( ) . UTC ( ) ) . SetSkipReason ( "" ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
2021-07-27 02:43:30 +02:00
err := importKeys ( pkg )
if err != nil {
log . Warningf ( "[%s/%s] Failed to import pgp keys: %v" , pkg . FullRepo , pkg . Pkgbase , err )
}
err = increasePkgRel ( pkg )
if err != nil {
log . Errorf ( "[%s/%s] Failed to increase pkgrel: %v" , pkg . FullRepo , pkg . Pkgbase , err )
b . buildWG . Done ( )
continue
}
2021-06-10 21:32:11 +02:00
pkg . PkgFiles = [ ] string { }
2021-07-08 16:30:03 +02:00
cmd := exec . Command ( "sh" , "-c" ,
2021-06-10 21:32:11 +02:00
"cd " + filepath . Dir ( pkg . Pkgbuild ) + "&&makechrootpkg -c -D " + conf . Basedir . Makepkg + " -l worker-" + strconv . Itoa ( id ) + " -r " + conf . Basedir . Chroot + " -- " +
"--config " + filepath . Join ( conf . Basedir . Makepkg , fmt . Sprintf ( "makepkg-%s.conf" , pkg . March ) ) )
2021-06-14 15:52:53 +02:00
var out bytes . Buffer
cmd . Stdout = & out
cmd . Stderr = & out
check ( cmd . Start ( ) )
b . buildProcMutex . Lock ( )
2021-06-14 16:14:17 +02:00
b . buildProcesses = append ( b . buildProcesses , cmd . Process )
2021-06-14 15:52:53 +02:00
b . buildProcMutex . Unlock ( )
2021-07-13 18:07:29 +02:00
err = cmd . Wait ( )
2021-06-14 15:52:53 +02:00
b . buildProcMutex . Lock ( )
for i := range b . buildProcesses {
2021-06-14 16:14:17 +02:00
if b . buildProcesses [ i ] . Pid == cmd . Process . Pid {
2021-06-14 15:52:53 +02:00
b . buildProcesses = append ( b . buildProcesses [ : i ] , b . buildProcesses [ i + 1 : ] ... )
break
}
}
b . buildProcMutex . Unlock ( )
2021-06-10 21:32:11 +02:00
if err != nil {
2021-06-12 00:04:33 +02:00
if b . exit {
gitClean ( pkg )
2021-07-01 21:13:48 +02:00
b . buildWG . Done ( )
2021-06-12 00:04:33 +02:00
continue
}
2021-06-10 21:32:11 +02:00
log . Warningf ( "[%s/%s] Build failed, exit code %d" , pkg . FullRepo , pkg . Pkgbase , cmd . ProcessState . ExitCode ( ) )
b . failedMutex . Lock ( )
2021-07-09 14:12:43 +02:00
f , err := os . OpenFile ( filepath . Join ( conf . Basedir . Repo , pkg . FullRepo + "_failed.txt" ) , os . O_WRONLY | os . O_APPEND | os . O_CREATE | os . O_SYNC , os . ModePerm )
2021-06-10 21:32:11 +02:00
check ( err )
2021-07-27 02:43:30 +02:00
_ , err = f . WriteString ( fmt . Sprintf ( "%s==%s\n" , pkg . Pkgbase , constructVersion ( pkg . Srcinfo . Pkgver , pkg . Srcinfo . Pkgrel , pkg . Srcinfo . Epoch ) ) )
check ( err )
2021-06-12 17:57:32 +02:00
check ( f . Close ( ) )
2021-06-10 21:32:11 +02:00
b . failedMutex . Unlock ( )
check ( os . MkdirAll ( filepath . Join ( conf . Basedir . Repo , "logs" ) , os . ModePerm ) )
2021-06-14 15:52:53 +02:00
check ( os . WriteFile ( filepath . Join ( conf . Basedir . Repo , "logs" , pkg . Pkgbase + ".log" ) , out . Bytes ( ) , os . ModePerm ) )
2021-06-10 21:32:11 +02:00
2021-07-13 18:07:29 +02:00
dbPkg := getDbPackage ( pkg )
dbLock . Lock ( )
2021-08-31 05:08:50 +02:00
dbPkg . Update ( ) . SetStatus ( FAILED ) . SetBuildTime ( time . Now ( ) ) . SetBuildDuration ( uint64 ( time . Now ( ) . Sub ( start ) . Milliseconds ( ) ) ) . SetHash ( pkg . Hash ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
2021-09-07 12:49:01 +02:00
// purge failed package from repo
b . repoPurge [ pkg . FullRepo ] <- pkg
2021-06-10 21:32:11 +02:00
gitClean ( pkg )
2021-07-01 21:13:48 +02:00
b . buildWG . Done ( )
2021-06-10 21:32:11 +02:00
continue
}
pkgFiles , err := filepath . Glob ( filepath . Join ( filepath . Dir ( pkg . Pkgbuild ) , "*.pkg.tar.zst" ) )
check ( err )
log . Debug ( pkgFiles )
if len ( pkgFiles ) == 0 {
log . Warningf ( "No packages found after building %s. Abort build." , pkg . Pkgbase )
gitClean ( pkg )
2021-07-01 21:13:48 +02:00
b . buildWG . Done ( )
2021-06-10 21:32:11 +02:00
continue
}
for _ , file := range pkgFiles {
2021-07-08 16:30:03 +02:00
cmd = exec . Command ( "gpg" , "--batch" , "--detach-sign" , file )
2021-06-10 21:32:11 +02:00
res , err := cmd . CombinedOutput ( )
log . Debug ( string ( res ) )
if err != nil {
log . Warningf ( "Failed to sign %s: %s" , pkg . Pkgbase , err )
2021-07-01 21:13:48 +02:00
b . buildWG . Done ( )
2021-06-10 21:32:11 +02:00
continue
}
}
copyFiles , err := filepath . Glob ( filepath . Join ( filepath . Dir ( pkg . Pkgbuild ) , "*.pkg.tar.zst*" ) )
check ( err )
for _ , file := range copyFiles {
_ , err = copyFile ( file , filepath . Join ( conf . Basedir . Repo , pkg . FullRepo , "os" , conf . Arch , filepath . Base ( file ) ) )
if err != nil {
check ( err )
2021-07-01 21:13:48 +02:00
b . buildWG . Done ( )
2021-06-10 21:32:11 +02:00
continue
}
if filepath . Ext ( file ) != ".sig" {
pkg . PkgFiles = append ( pkg . PkgFiles , filepath . Join ( conf . Basedir . Repo , pkg . FullRepo , "os" , conf . Arch , filepath . Base ( file ) ) )
}
}
if _ , err := os . Stat ( filepath . Join ( conf . Basedir . Repo , "logs" , pkg . Pkgbase + ".log" ) ) ; err == nil {
check ( os . Remove ( filepath . Join ( conf . Basedir . Repo , "logs" , pkg . Pkgbase + ".log" ) ) )
}
2021-07-13 18:07:29 +02:00
dbPkg = getDbPackage ( pkg )
dbLock . Lock ( )
2021-08-04 17:42:42 +02:00
dbPkg . Update ( ) . SetStatus ( BUILD ) . SetBuildDuration ( uint64 ( time . Now ( ) . Sub ( start ) . Milliseconds ( ) ) ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
2021-06-10 21:32:11 +02:00
log . Infof ( "[%s/%s] Build successful (%s)" , pkg . FullRepo , pkg . Pkgbase , time . Now ( ) . Sub ( start ) )
2021-07-13 18:07:29 +02:00
b . repoAdd [ pkg . FullRepo ] <- pkg
gitClean ( pkg )
2021-06-10 21:32:11 +02:00
}
}
}
func ( b * BuildManager ) parseWorker ( ) {
for {
if b . exit {
return
}
select {
2021-07-09 20:37:00 +02:00
case pkg := <- b . parse :
2021-07-08 16:30:03 +02:00
cmd := exec . Command ( "sh" , "-c" , "cd " + filepath . Dir ( pkg . Pkgbuild ) + "&&" + "makepkg --printsrcinfo" )
2021-06-10 21:32:11 +02:00
res , err := cmd . Output ( )
if err != nil {
2021-06-14 15:52:53 +02:00
log . Warningf ( "Failed generate SRCINFO for %s: %v" , pkg . Pkgbase , err )
2021-07-01 21:13:48 +02:00
b . parseWG . Done ( )
2021-06-10 21:32:11 +02:00
continue
}
info , err := srcinfo . Parse ( string ( res ) )
if err != nil {
2021-06-14 15:52:53 +02:00
log . Warningf ( "Failed to parse SRCINFO for %s: %v" , pkg . Pkgbase , err )
2021-07-01 21:13:48 +02:00
b . parseWG . Done ( )
2021-06-10 21:32:11 +02:00
continue
}
pkg . Srcinfo = info
2021-07-27 02:43:30 +02:00
pkg . Version = constructVersion ( pkg . Srcinfo . Pkgver , pkg . Srcinfo . Pkgrel , pkg . Srcinfo . Epoch )
2021-06-10 21:32:11 +02:00
2021-07-13 18:07:29 +02:00
dbPkg := getDbPackage ( pkg )
dbLock . Lock ( )
2021-07-13 19:18:35 +02:00
dbPkg = dbPkg . Update ( ) . SetUpdated ( time . Now ( ) ) . SetVersion ( pkg . Version ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
skipping := false
if contains ( info . Arch , "any" ) {
2021-07-22 23:55:06 +02:00
log . Debugf ( "Skipped %s: any-Package" , info . Pkgbase )
2021-07-13 18:07:29 +02:00
dbLock . Lock ( )
2021-08-30 11:02:06 +02:00
dbPkg = dbPkg . Update ( ) . SetStatus ( SKIPPED ) . SetSkipReason ( "arch = any" ) . SetHash ( pkg . Hash ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
skipping = true
2021-07-27 02:43:30 +02:00
} else if contains ( conf . Blacklist . Packages , info . Pkgbase ) {
2021-07-22 23:55:06 +02:00
log . Debugf ( "Skipped %s: blacklisted package" , info . Pkgbase )
2021-07-13 18:07:29 +02:00
dbLock . Lock ( )
2021-08-30 11:02:06 +02:00
dbPkg = dbPkg . Update ( ) . SetStatus ( SKIPPED ) . SetSkipReason ( "blacklisted" ) . SetHash ( pkg . Hash ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
skipping = true
} else if contains ( info . MakeDepends , "ghc" ) || contains ( info . MakeDepends , "haskell-ghc" ) || contains ( info . Depends , "ghc" ) || contains ( info . Depends , "haskell-ghc" ) {
// Skip Haskell packages for now, as we are facing linking problems with them,
2021-07-27 16:24:30 +02:00
// most likely caused by not having a dependency check implemented yet and building at random.
2021-07-13 18:07:29 +02:00
// https://git.harting.dev/anonfunc/ALHP.GO/issues/11
2021-07-22 23:55:06 +02:00
log . Debugf ( "Skipped %s: haskell package" , info . Pkgbase )
2021-07-13 18:07:29 +02:00
dbLock . Lock ( )
2021-08-30 11:02:06 +02:00
dbPkg = dbPkg . Update ( ) . SetStatus ( SKIPPED ) . SetSkipReason ( "blacklisted (haskell)" ) . SetHash ( pkg . Hash ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
skipping = true
} else if isPkgFailed ( pkg ) {
2021-07-22 23:55:06 +02:00
log . Debugf ( "Skipped %s: failed build" , info . Pkgbase )
2021-07-13 18:07:29 +02:00
dbLock . Lock ( )
2021-08-30 11:02:06 +02:00
dbPkg = dbPkg . Update ( ) . SetStatus ( FAILED ) . SetSkipReason ( "" ) . SetHash ( pkg . Hash ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
skipping = true
2021-07-09 14:46:50 +02:00
}
2021-07-13 18:07:29 +02:00
if skipping {
2021-07-09 20:37:00 +02:00
b . repoPurge [ pkg . FullRepo ] <- pkg
2021-07-01 21:13:48 +02:00
b . parseWG . Done ( )
2021-06-10 21:32:11 +02:00
continue
}
repoVer := getVersionFromRepo ( pkg )
2021-07-13 18:07:29 +02:00
dbLock . Lock ( )
dbPkg = dbPkg . Update ( ) . SetRepoVersion ( repoVer ) . SaveX ( context . Background ( ) )
dbLock . Unlock ( )
2021-07-13 19:18:35 +02:00
if repoVer != "" && alpm . VerCmp ( repoVer , pkg . Version ) > 0 {
log . Debugf ( "Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)" , info . Pkgbase , pkg . Version , repoVer )
2021-07-13 18:07:29 +02:00
dbLock . Lock ( )
2021-08-30 11:02:06 +02:00
dbPkg = dbPkg . Update ( ) . SetStatus ( LATEST ) . SetSkipReason ( "" ) . SetHash ( pkg . Hash ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
2021-07-01 21:13:48 +02:00
b . parseWG . Done ( )
2021-06-10 21:32:11 +02:00
continue
}
2021-07-29 16:53:15 +02:00
isLatest , local , syncVersion , err := isMirrorLatest ( alpmHandle , pkg )
2021-07-27 18:42:37 +02:00
if err != nil {
2021-08-26 12:21:20 +02:00
switch err . ( type ) {
default :
log . Warningf ( "[%s/%s] Problem solving dependencies: %v" , pkg . FullRepo , info . Pkgbase , err )
case MultiplePKGBUILDError :
log . Debugf ( "Skipped %s: Multiple PKGBUILDs for dependency found: %v" , info . Pkgbase , err )
dbLock . Lock ( )
dbPkg = dbPkg . Update ( ) . SetStatus ( SKIPPED ) . SetSkipReason ( "multiple PKGBUILD for dep. found" ) . SaveX ( context . Background ( ) )
dbLock . Unlock ( )
b . repoPurge [ pkg . FullRepo ] <- pkg
b . parseWG . Done ( )
continue
2021-08-26 12:59:23 +02:00
case UnableToSatisfyError :
log . Debugf ( "Skipped %s: unable to resolve dependencies: %v" , info . Pkgbase , err )
dbLock . Lock ( )
dbPkg = dbPkg . Update ( ) . SetStatus ( SKIPPED ) . SetSkipReason ( "unable to resolve dependencies" ) . SaveX ( context . Background ( ) )
dbLock . Unlock ( )
b . repoPurge [ pkg . FullRepo ] <- pkg
b . parseWG . Done ( )
continue
2021-08-26 12:21:20 +02:00
}
2021-07-27 18:42:37 +02:00
}
2021-07-13 18:07:29 +02:00
dbLock . Lock ( )
dbPkg = dbPkg . Update ( ) . SetStatus ( QUEUED ) . SaveX ( context . Background ( ) )
dbLock . Unlock ( )
2021-07-27 02:43:30 +02:00
if ! isLatest {
2021-07-29 16:53:15 +02:00
if local != nil {
log . Infof ( "Delayed %s: not all dependencies are up to date (local: %s==%s, sync: %s==%s)" , info . Pkgbase , local . Name ( ) , local . Version ( ) , local . Name ( ) , syncVersion )
2021-07-29 20:45:45 +02:00
dbLock . Lock ( )
dbPkg = dbPkg . Update ( ) . SetSkipReason ( fmt . Sprintf ( "waiting for %s==%s" , local . Name ( ) , syncVersion ) ) . SaveX ( context . Background ( ) )
dbLock . Unlock ( )
2021-07-27 18:51:35 +02:00
} else {
2021-08-30 17:26:34 +02:00
log . Infof ( "Delayed %s: not all dependencies are up to date or resolvable" , info . Pkgbase )
2021-07-29 20:45:45 +02:00
dbLock . Lock ( )
dbPkg = dbPkg . Update ( ) . SetSkipReason ( "waiting for mirror" ) . SaveX ( context . Background ( ) )
dbLock . Unlock ( )
2021-07-27 18:51:35 +02:00
}
2021-07-29 20:45:45 +02:00
2021-09-12 17:40:17 +02:00
// Purge delayed packages in case delay is caused by inconsistencies in svn2git.
// Worst case would be clients downloading a package update twice, once from their official mirror,
// and then after build from ALHP. Best case we prevent a not buildable package from staying in the repos
// in an outdated version.
b . repoPurge [ pkg . FullRepo ] <- pkg
2021-07-27 02:43:30 +02:00
b . parseWG . Done ( )
continue
2021-06-10 21:32:11 +02:00
}
2021-07-27 02:43:30 +02:00
b . parseWG . Done ( )
b . build <- pkg
2021-06-10 21:32:11 +02:00
}
}
2021-07-13 18:07:29 +02:00
}
func ( b * BuildManager ) htmlWorker ( ) {
type Pkg struct {
Pkgbase string
Status string
Class string
Skip string
Version string
Svn2GitVersion string
BuildDate string
BuildDuration time . Duration
Checked string
2021-07-14 19:21:58 +02:00
Log string
2021-07-13 18:07:29 +02:00
}
type Repo struct {
Name string
Packages [ ] Pkg
}
type March struct {
Name string
Repos [ ] Repo
}
type tpl struct {
2021-07-14 19:21:58 +02:00
March [ ] March
Generated time . Time
2021-07-13 18:07:29 +02:00
}
for {
gen := & tpl { }
for _ , march := range conf . March {
addMarch := March {
Name : march ,
}
for _ , repo := range conf . Repos {
addRepo := Repo {
Name : repo ,
}
dbLock . RLock ( )
2021-07-13 19:34:06 +02:00
pkgs := db . DbPackage . Query ( ) . Order ( ent . Asc ( dbpackage . FieldPkgbase ) ) . Where ( dbpackage . MarchEQ ( march ) , dbpackage . RepositoryEQ ( repo ) ) . AllX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . RUnlock ( )
for _ , pkg := range pkgs {
status , class := statusId2string ( pkg . Status )
addPkg := Pkg {
Pkgbase : pkg . Pkgbase ,
Status : status ,
Class : class ,
Skip : pkg . SkipReason ,
Version : pkg . RepoVersion ,
Svn2GitVersion : pkg . Version ,
}
if pkg . BuildDuration > 0 {
duration , err := time . ParseDuration ( strconv . Itoa ( int ( pkg . BuildDuration ) ) + "ms" )
check ( err )
addPkg . BuildDuration = duration
}
if ! pkg . BuildTime . IsZero ( ) {
2021-07-13 19:51:09 +02:00
addPkg . BuildDate = pkg . BuildTime . UTC ( ) . Format ( time . RFC3339 )
2021-07-13 18:07:29 +02:00
}
if ! pkg . Updated . IsZero ( ) {
2021-07-13 19:51:09 +02:00
addPkg . Checked = pkg . Updated . UTC ( ) . Format ( time . RFC3339 )
2021-07-13 18:07:29 +02:00
}
2021-07-14 19:21:58 +02:00
if pkg . Status == FAILED {
addPkg . Log = fmt . Sprintf ( "logs/%s.log" , pkg . Pkgbase )
}
2021-07-13 18:07:29 +02:00
addRepo . Packages = append ( addRepo . Packages , addPkg )
}
addMarch . Repos = append ( addMarch . Repos , addRepo )
}
gen . March = append ( gen . March , addMarch )
}
2021-07-14 19:21:58 +02:00
gen . Generated = time . Now ( ) . UTC ( )
2021-07-22 23:41:24 +02:00
statusTpl , err := template . ParseFiles ( "tpl/packages.html" )
2021-07-13 18:07:29 +02:00
check ( err )
2021-07-22 23:41:24 +02:00
f , err := os . OpenFile ( filepath . Join ( conf . Basedir . Repo , "packages.html" ) , os . O_WRONLY | os . O_CREATE | os . O_TRUNC , os . ModePerm )
2021-07-13 18:07:29 +02:00
check ( statusTpl . Execute ( f , gen ) )
check ( f . Close ( ) )
time . Sleep ( time . Minute )
}
}
2021-07-09 20:37:00 +02:00
func ( b * BuildManager ) repoWorker ( repo string ) {
2021-06-10 21:32:11 +02:00
for {
select {
2021-07-09 20:37:00 +02:00
case pkg := <- b . repoAdd [ repo ] :
2021-06-10 21:32:11 +02:00
args := [ ] string { "-s" , "-v" , "-p" , "-n" , filepath . Join ( conf . Basedir . Repo , pkg . FullRepo , "os" , conf . Arch , pkg . FullRepo ) + ".db.tar.xz" }
args = append ( args , pkg . PkgFiles ... )
2021-07-08 16:30:03 +02:00
cmd := exec . Command ( "repo-add" , args ... )
2021-06-10 21:32:11 +02:00
res , err := cmd . CombinedOutput ( )
log . Debug ( string ( res ) )
2021-07-09 14:12:43 +02:00
if err != nil && cmd . ProcessState . ExitCode ( ) != 1 {
log . Panicf ( "%s while repo-add: %v" , string ( res ) , err )
2021-06-10 22:28:54 +02:00
}
2021-06-10 21:32:11 +02:00
2021-07-13 18:07:29 +02:00
dbPkg := getDbPackage ( pkg )
dbLock . Lock ( )
2021-08-30 11:02:06 +02:00
dbPkg = dbPkg . Update ( ) . SetStatus ( LATEST ) . SetSkipReason ( "" ) . SetRepoVersion ( pkg . Version ) . SetHash ( pkg . Hash ) . SaveX ( context . Background ( ) )
2021-07-13 18:07:29 +02:00
dbLock . Unlock ( )
2021-07-08 16:30:03 +02:00
cmd = exec . Command ( "paccache" ,
2021-06-10 21:32:11 +02:00
"-rc" , filepath . Join ( conf . Basedir . Repo , pkg . FullRepo , "os" , conf . Arch ) ,
"-k" , "1" )
res , err = cmd . CombinedOutput ( )
log . Debug ( string ( res ) )
check ( err )
2021-07-01 21:13:48 +02:00
b . buildWG . Done ( )
2021-07-09 20:37:00 +02:00
case pkg := <- b . repoPurge [ repo ] :
2021-06-10 21:32:11 +02:00
if _ , err := os . Stat ( filepath . Join ( conf . Basedir . Repo , pkg . FullRepo , "os" , conf . Arch , pkg . FullRepo ) + ".db.tar.xz" ) ; err != nil {
continue
}
if len ( pkg . PkgFiles ) == 0 {
findPkgFiles ( pkg )
2021-07-13 20:01:42 +02:00
if len ( pkg . PkgFiles ) == 0 {
continue
}
2021-06-10 21:32:11 +02:00
}
var realPkgs [ ] string
for _ , realPkg := range pkg . Srcinfo . Packages {
realPkgs = append ( realPkgs , realPkg . Pkgname )
}
2021-07-09 14:12:43 +02:00
b . repoWG . Add ( 1 )
2021-06-10 21:32:11 +02:00
args := [ ] string { "-s" , "-v" , filepath . Join ( conf . Basedir . Repo , pkg . FullRepo , "os" , conf . Arch , pkg . FullRepo ) + ".db.tar.xz" }
args = append ( args , realPkgs ... )
2021-07-08 16:30:03 +02:00
cmd := exec . Command ( "repo-remove" , args ... )
2021-06-10 21:32:11 +02:00
res , err := cmd . CombinedOutput ( )
log . Debug ( string ( res ) )
if err != nil && cmd . ProcessState . ExitCode ( ) == 1 {
log . Debugf ( "Deleteing package %s failed: Package not found in database" , pkg . Pkgbase )
2021-07-09 14:50:25 +02:00
b . repoWG . Done ( )
2021-06-10 21:32:11 +02:00
continue
}
2021-07-18 23:33:16 +02:00
dbPkg := getDbPackage ( pkg )
dbLock . Lock ( )
dbPkg = dbPkg . Update ( ) . SetRepoVersion ( "" ) . SaveX ( context . Background ( ) )
dbLock . Unlock ( )
2021-06-10 21:32:11 +02:00
for _ , file := range pkg . PkgFiles {
check ( os . Remove ( file ) )
check ( os . Remove ( file + ".sig" ) )
}
2021-07-09 14:12:43 +02:00
b . repoWG . Done ( )
2021-06-10 21:32:11 +02:00
}
}
}
func ( b * BuildManager ) syncWorker ( ) {
check ( os . MkdirAll ( conf . Basedir . Upstream , os . ModePerm ) )
for i := 0 ; i < conf . Build . Worker ; i ++ {
go b . buildWorker ( i )
2021-07-01 21:40:49 +02:00
}
for i := 0 ; i < runtime . NumCPU ( ) ; i ++ {
2021-06-10 21:32:11 +02:00
go b . parseWorker ( )
}
for {
2021-07-01 21:13:48 +02:00
b . buildWG . Wait ( )
2021-07-26 16:38:12 +02:00
2021-06-10 21:32:11 +02:00
for gitDir , gitURL := range conf . Svn2git {
gitPath := filepath . Join ( conf . Basedir . Upstream , gitDir )
if _ , err := os . Stat ( gitPath ) ; os . IsNotExist ( err ) {
2021-07-08 16:30:03 +02:00
cmd := exec . Command ( "git" , "clone" , "--depth=1" , gitURL , gitPath )
2021-06-10 21:32:11 +02:00
res , err := cmd . CombinedOutput ( )
log . Debug ( string ( res ) )
check ( err )
} else if err == nil {
2021-07-08 16:30:03 +02:00
cmd := exec . Command ( "sudo" , "git_clean.sh" , gitPath )
2021-06-10 21:32:11 +02:00
res , err := cmd . CombinedOutput ( )
log . Debug ( string ( res ) )
2021-07-08 05:08:47 +02:00
if err != nil {
log . Warningf ( "Failed to execute %s: %v" , cmd . String ( ) , err )
}
2021-06-10 21:32:11 +02:00
2021-07-08 16:30:03 +02:00
cmd = exec . Command ( "sh" , "-c" , "cd " + gitPath + " && git reset --hard" )
2021-06-10 21:32:11 +02:00
res , err = cmd . CombinedOutput ( )
log . Debug ( string ( res ) )
check ( err )
2021-07-08 16:30:03 +02:00
cmd = exec . Command ( "sh" , "-c" , "cd " + gitPath + " && git pull" )
2021-06-10 21:32:11 +02:00
res , err = cmd . CombinedOutput ( )
log . Debug ( string ( res ) )
check ( err )
}
}
2021-07-03 22:13:39 +02:00
// fetch updates between sync runs
2021-08-31 21:13:22 +02:00
b . alpmMutex . Lock ( )
2021-07-29 00:45:45 +02:00
check ( alpmHandle . Release ( ) )
2021-07-03 22:13:39 +02:00
setupChroot ( )
2021-07-29 00:43:22 +02:00
var err error
alpmHandle , err = initALPM ( filepath . Join ( conf . Basedir . Chroot , pristineChroot ) , filepath . Join ( conf . Basedir . Chroot , pristineChroot , "/var/lib/pacman" ) )
check ( err )
2021-08-31 21:13:22 +02:00
b . alpmMutex . Unlock ( )
2021-07-03 22:13:39 +02:00
2021-07-27 02:43:30 +02:00
pkgBuilds , err := Glob ( filepath . Join ( conf . Basedir . Upstream , "/**/PKGBUILD" ) )
2021-06-10 21:32:11 +02:00
check ( err )
for _ , pkgbuild := range pkgBuilds {
if b . exit {
return
}
sPkgbuild := strings . Split ( pkgbuild , "/" )
repo := sPkgbuild [ len ( sPkgbuild ) - 2 ]
2021-07-27 02:43:30 +02:00
if repo == "trunk" || ! contains ( conf . Repos , strings . Split ( repo , "-" ) [ 0 ] ) || containsSubStr ( repo , conf . Blacklist . Repo ) {
2021-06-10 21:32:11 +02:00
continue
}
2021-08-30 11:02:06 +02:00
// compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs
// reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly
dbLock . RLock ( )
2021-08-30 17:26:34 +02:00
dbPkg , dbErr := db . DbPackage . Query ( ) . Where ( dbpackage . And ( dbpackage . Pkgbase ( sPkgbuild [ len ( sPkgbuild ) - 4 ] ) , dbpackage . Repository ( strings . Split ( repo , "-" ) [ 0 ] ) ) ) . Only ( context . Background ( ) )
2021-08-30 11:02:06 +02:00
dbLock . RUnlock ( )
if dbErr != nil {
switch dbErr . ( type ) {
case * ent . NotFoundError :
2021-08-30 17:26:34 +02:00
log . Debugf ( "[%s/%s] Package not found in database" , strings . Split ( repo , "-" ) [ 0 ] , sPkgbuild [ len ( sPkgbuild ) - 4 ] )
2021-08-30 11:02:06 +02:00
break
default :
2021-08-30 17:26:34 +02:00
log . Errorf ( "[%s/%s] Problem querying db for package: %v" , strings . Split ( repo , "-" ) [ 0 ] , sPkgbuild [ len ( sPkgbuild ) - 4 ] , dbErr )
2021-08-30 11:02:06 +02:00
}
}
b3s , err := b3sum ( pkgbuild )
check ( err )
if dbPkg != nil && b3s == dbPkg . Hash {
2021-08-30 17:26:34 +02:00
log . Debugf ( "[%s/%s] Skipped: PKGBUILD hash matches db (%s)" , strings . Split ( repo , "-" ) [ 0 ] , sPkgbuild [ len ( sPkgbuild ) - 4 ] , b3s )
2021-08-30 11:02:06 +02:00
continue
}
// send to parse
2021-06-10 21:32:11 +02:00
for _ , march := range conf . March {
2021-07-01 21:13:48 +02:00
b . parseWG . Add ( 1 )
2021-07-09 20:37:00 +02:00
b . parse <- & BuildPackage {
2021-06-10 21:32:11 +02:00
Pkgbuild : pkgbuild ,
Pkgbase : sPkgbuild [ len ( sPkgbuild ) - 4 ] ,
Repo : strings . Split ( repo , "-" ) [ 0 ] ,
March : march ,
FullRepo : strings . Split ( repo , "-" ) [ 0 ] + "-" + march ,
2021-08-30 11:02:06 +02:00
Hash : b3s ,
2021-06-10 21:32:11 +02:00
}
}
}
2021-07-01 21:13:48 +02:00
b . parseWG . Wait ( )
2021-09-12 17:35:45 +02:00
time . Sleep ( time . Duration ( * checkInterval ) * time . Minute )
2021-06-10 21:32:11 +02:00
}
}
func main ( ) {
killSignals := make ( chan os . Signal , 1 )
signal . Notify ( killSignals , syscall . SIGINT , syscall . SIGTERM )
2021-09-12 17:25:33 +02:00
flag . Parse ( )
2021-06-10 21:32:11 +02:00
confStr , err := os . ReadFile ( "config.yaml" )
check ( err )
err = yaml . Unmarshal ( confStr , & conf )
check ( err )
lvl , err := log . ParseLevel ( conf . Logging . Level )
check ( err )
log . SetLevel ( lvl )
2021-09-12 17:25:33 +02:00
if * journalLog {
journalhook . Enable ( )
}
2021-06-10 21:32:11 +02:00
2021-06-30 17:41:50 +02:00
err = syscall . Setpriority ( syscall . PRIO_PROCESS , 0 , 5 )
if err != nil {
log . Warningf ( "Failed to drop priority: %v" , err )
}
2021-06-14 13:04:06 +02:00
2021-07-14 19:21:58 +02:00
err = os . MkdirAll ( conf . Basedir . Repo , os . ModePerm )
check ( err )
2021-07-13 18:07:29 +02:00
db , err = ent . Open ( "sqlite3" , "file:" + conf . Basedir . Db + "?_fk=1&cache=shared" )
if err != nil {
log . Panicf ( "Failed to open database %s: %v" , conf . Basedir . Db , err )
}
defer func ( dbSQLite * ent . Client ) {
check ( dbSQLite . Close ( ) )
} ( db )
2021-07-26 16:38:12 +02:00
if err := db . Schema . Create ( context . Background ( ) , migrate . WithDropIndex ( true ) , migrate . WithDropColumn ( true ) ) ; err != nil {
2021-07-13 18:07:29 +02:00
log . Panicf ( "Automigrate failed: %v" , err )
}
2021-06-10 21:32:11 +02:00
buildManager = BuildManager {
2021-07-09 20:44:37 +02:00
build : make ( chan * BuildPackage , 10000 ) ,
parse : make ( chan * BuildPackage , 10000 ) ,
repoPurge : make ( map [ string ] chan * BuildPackage ) ,
repoAdd : make ( map [ string ] chan * BuildPackage ) ,
exit : false ,
2021-06-10 21:32:11 +02:00
}
setupChroot ( )
syncMarchs ( )
2021-07-28 02:31:53 +02:00
alpmHandle , err = initALPM ( filepath . Join ( conf . Basedir . Chroot , pristineChroot ) , filepath . Join ( conf . Basedir . Chroot , pristineChroot , "/var/lib/pacman" ) )
2021-07-27 02:43:30 +02:00
check ( err )
2021-06-10 21:32:11 +02:00
go buildManager . syncWorker ( )
2021-07-13 18:07:29 +02:00
go buildManager . htmlWorker ( )
2021-06-10 21:32:11 +02:00
<- killSignals
buildManager . exit = true
2021-06-14 15:56:21 +02:00
buildManager . buildProcMutex . RLock ( )
2021-06-14 15:52:53 +02:00
for _ , p := range buildManager . buildProcesses {
2021-06-14 16:14:17 +02:00
pgid , err := syscall . Getpgid ( p . Pid )
check ( err )
check ( syscall . Kill ( - pgid , syscall . SIGTERM ) )
2021-06-14 15:52:53 +02:00
}
2021-06-14 15:56:21 +02:00
buildManager . buildProcMutex . RUnlock ( )
2021-07-01 21:13:48 +02:00
buildManager . buildWG . Wait ( )
2021-07-09 14:12:43 +02:00
buildManager . repoWG . Wait ( )
2021-07-27 02:43:30 +02:00
check ( alpmHandle . Release ( ) )
2021-06-10 21:32:11 +02:00
}