Added pre-build dependency version check (#32)
Fixes #15. Uses ALPM's `satisfies` to resolve dependencies and then compares resolved dependency from local sync database with svn2git's version. Reviewed-on: https://git.harting.dev/anonfunc/ALHP.GO/pulls/32 Co-authored-by: Giovanni Harting <539@idlegandalf.com> Co-committed-by: Giovanni Harting <539@idlegandalf.com>
This commit is contained in:
parent
17b415f8f3
commit
e28d85b10d
16
config.yaml
16
config.yaml
@ -19,10 +19,16 @@ march:
|
||||
- x86-64-v3
|
||||
|
||||
blacklist:
|
||||
- pacman
|
||||
- tensorflow
|
||||
- tensorflow-cuda
|
||||
- gcc
|
||||
packages:
|
||||
- pacman
|
||||
- tensorflow
|
||||
- tensorflow-cuda
|
||||
- gcc
|
||||
repo:
|
||||
- testing
|
||||
- i686
|
||||
- staging
|
||||
- unstable
|
||||
|
||||
build:
|
||||
worker: 4
|
||||
@ -39,4 +45,4 @@ status:
|
||||
unknown: "dark"
|
||||
|
||||
logging:
|
||||
level: DEBUG
|
||||
level: INFO
|
||||
2
go.mod
2
go.mod
@ -5,11 +5,11 @@ go 1.16
|
||||
require (
|
||||
entgo.io/ent v0.8.0
|
||||
github.com/Jguer/go-alpm/v2 v2.0.5
|
||||
github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5
|
||||
github.com/Morganamilo/go-srcinfo v1.0.0
|
||||
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.6
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3
|
||||
github.com/yargevad/filepathx v1.0.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
||||
19
go.sum
19
go.sum
@ -19,6 +19,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
|
||||
github.com/Jguer/go-alpm/v2 v2.0.5 h1:1TZxkvCIfTOhjhxGy/Z1FNSeuY9DXBKF5qxUoj0IZ0A=
|
||||
github.com/Jguer/go-alpm/v2 v2.0.5/go.mod h1:zU4iKCtNkDARfj5BrKJXYAQ5nIjtZbySfa0paboSmTQ=
|
||||
github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5 h1:TMscPjkb1ThXN32LuFY5bEYIcXZx3YlwzhS1GxNpn/c=
|
||||
github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5/go.mod h1:Hk55m330jNiwxRodIlMCvw5iEyoRUCIY64W1p9D+tHc=
|
||||
github.com/Morganamilo/go-srcinfo v1.0.0 h1:Wh4nEF+HJWo+29hnxM18Q2hi+DUf0GejS13+Wg+dzmI=
|
||||
github.com/Morganamilo/go-srcinfo v1.0.0/go.mod h1:MP6VGY1NNpVUmYIEgoM9acix95KQqIRyqQ0hCLsyYUY=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
@ -60,7 +62,6 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4=
|
||||
github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4=
|
||||
github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
@ -127,10 +128,6 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
|
||||
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
@ -151,9 +148,7 @@ github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
|
||||
github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
|
||||
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
@ -173,7 +168,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
|
||||
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
@ -208,11 +202,9 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
|
||||
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@ -263,7 +255,6 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@ -334,12 +325,10 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
@ -390,10 +379,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
|
||||
gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
|
||||
gorm.io/gorm v1.20.7 h1:rMS4CL3pNmYq1V5/X+nHHjh1Dx6dnf27+Cai5zabo+M=
|
||||
gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
423
main.go
423
main.go
@ -4,7 +4,6 @@ import (
|
||||
"ALHP.go/ent"
|
||||
"ALHP.go/ent/dbpackage"
|
||||
"ALHP.go/ent/migrate"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
@ -13,10 +12,8 @@ import (
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/wercker/journalhook"
|
||||
"github.com/yargevad/filepathx"
|
||||
"gopkg.in/yaml.v2"
|
||||
"html/template"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"os/exec"
|
||||
@ -24,7 +21,6 @@ import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@ -39,19 +35,11 @@ const (
|
||||
orgChrootName = "root"
|
||||
)
|
||||
|
||||
const (
|
||||
SKIPPED = iota
|
||||
FAILED = iota
|
||||
BUILD = iota
|
||||
QUEUED = iota
|
||||
BUILDING = iota
|
||||
LATEST = iota
|
||||
UNKNOWN = iota
|
||||
)
|
||||
|
||||
var (
|
||||
conf = Conf{}
|
||||
repos []string
|
||||
alpmHandle *alpm.Handle
|
||||
alpmLock sync.RWMutex
|
||||
reMarch = regexp.MustCompile(`(-march=)(.+?) `)
|
||||
rePkgRel = regexp.MustCompile(`(?m)^pkgrel\s*=\s*(.+)$`)
|
||||
rePkgFile = regexp.MustCompile(`^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$`)
|
||||
@ -60,231 +48,6 @@ var (
|
||||
dbLock sync.RWMutex
|
||||
)
|
||||
|
||||
type BuildPackage struct {
|
||||
Pkgbase string
|
||||
Pkgbuild string
|
||||
Srcinfo *srcinfo.Srcinfo
|
||||
PkgFiles []string
|
||||
Repo string
|
||||
March string
|
||||
FullRepo string
|
||||
Version string
|
||||
}
|
||||
|
||||
type BuildManager struct {
|
||||
build chan *BuildPackage
|
||||
parse chan *BuildPackage
|
||||
repoPurge map[string]chan *BuildPackage
|
||||
repoAdd map[string]chan *BuildPackage
|
||||
exit bool
|
||||
buildWG sync.WaitGroup
|
||||
parseWG sync.WaitGroup
|
||||
repoWG sync.WaitGroup
|
||||
failedMutex sync.RWMutex
|
||||
buildProcesses []*os.Process
|
||||
buildProcMutex sync.RWMutex
|
||||
}
|
||||
|
||||
type Conf struct {
|
||||
Arch string
|
||||
Repos, March, Blacklist []string
|
||||
Svn2git map[string]string
|
||||
Basedir struct {
|
||||
Repo, Chroot, Makepkg, Upstream, Db string
|
||||
}
|
||||
Build struct {
|
||||
Worker int
|
||||
Makej int
|
||||
}
|
||||
Logging struct {
|
||||
Level string
|
||||
}
|
||||
Status struct {
|
||||
Class struct {
|
||||
Skipped, Queued, Latest, Failed, Signing, Building, Unknown string
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func check(e error) {
|
||||
if e != nil {
|
||||
panic(e)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s interface{}, str string) bool {
|
||||
switch v := s.(type) {
|
||||
case []string:
|
||||
if i := find(v, str); i != -1 {
|
||||
return true
|
||||
}
|
||||
case []srcinfo.ArchString:
|
||||
var n []string
|
||||
for _, as := range v {
|
||||
n = append(n, as.Value)
|
||||
}
|
||||
|
||||
if i := find(n, str); i != -1 {
|
||||
return true
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func find(s []string, str string) int {
|
||||
for i, v := range s {
|
||||
if v == str {
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) (int64, error) {
|
||||
sourceFileStat, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if !sourceFileStat.Mode().IsRegular() {
|
||||
return 0, fmt.Errorf("%s is not a regular file", src)
|
||||
}
|
||||
|
||||
source, err := os.Open(src)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func(source *os.File) {
|
||||
check(source.Close())
|
||||
}(source)
|
||||
|
||||
destination, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func(destination *os.File) {
|
||||
check(destination.Close())
|
||||
}(destination)
|
||||
nBytes, err := io.Copy(destination, source)
|
||||
return nBytes, err
|
||||
}
|
||||
|
||||
//goland:noinspection SpellCheckingInspection
|
||||
func setupMakepkg(march string) {
|
||||
lMakepkg := filepath.Join(conf.Basedir.Makepkg, fmt.Sprintf("makepkg-%s.conf", march))
|
||||
|
||||
check(os.MkdirAll(conf.Basedir.Makepkg, os.ModePerm))
|
||||
t, err := os.ReadFile(makepkgConf)
|
||||
check(err)
|
||||
makepkgStr := string(t)
|
||||
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, "-mtune=generic", "")
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, "-O2", "-O3")
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, " check ", " !check ")
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, " color ", " !color ")
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, "#MAKEFLAGS=\"-j2\"", "MAKEFLAGS=\"-j"+strconv.Itoa(conf.Build.Makej)+"\"")
|
||||
makepkgStr = reMarch.ReplaceAllString(makepkgStr, "${1}"+march)
|
||||
|
||||
check(os.WriteFile(lMakepkg, []byte(makepkgStr), os.ModePerm))
|
||||
}
|
||||
|
||||
func syncMarchs() {
|
||||
files, err := os.ReadDir(conf.Basedir.Repo)
|
||||
check(err)
|
||||
|
||||
var eRepos []string
|
||||
for _, file := range files {
|
||||
if file.Name() != "." && file.Name() != logDir && file.IsDir() {
|
||||
eRepos = append(eRepos, file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
for _, march := range conf.March {
|
||||
setupMakepkg(march)
|
||||
for _, repo := range conf.Repos {
|
||||
tRepo := fmt.Sprintf("%s-%s", repo, march)
|
||||
repos = append(repos, tRepo)
|
||||
buildManager.repoAdd[tRepo] = make(chan *BuildPackage, conf.Build.Worker)
|
||||
buildManager.repoPurge[tRepo] = make(chan *BuildPackage, 10000)
|
||||
go buildManager.repoWorker(tRepo)
|
||||
|
||||
if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch))); os.IsNotExist(err) {
|
||||
log.Debugf("Creating path %s", filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch))
|
||||
check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch), os.ModePerm))
|
||||
}
|
||||
|
||||
if i := find(eRepos, tRepo); i != -1 {
|
||||
eRepos = append(eRepos[:i], eRepos[i+1:]...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("Repos: %s", repos)
|
||||
|
||||
for _, repo := range eRepos {
|
||||
log.Infof("Removing old repo %s", repo)
|
||||
check(os.RemoveAll(filepath.Join(conf.Basedir.Repo, repo)))
|
||||
}
|
||||
}
|
||||
|
||||
func importKeys(pkg *BuildPackage) {
|
||||
if pkg.Srcinfo.ValidPGPKeys != nil {
|
||||
args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"}
|
||||
args = append(args, pkg.Srcinfo.ValidPGPKeys...)
|
||||
cmd := exec.Command("gpg", args...)
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
|
||||
if err != nil {
|
||||
log.Warningf("Unable to import keys: %s", string(res))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func packages2string(pkgs []srcinfo.Package) []string {
|
||||
var sPkgs []string
|
||||
for _, p := range pkgs {
|
||||
sPkgs = append(sPkgs, p.Pkgname)
|
||||
}
|
||||
|
||||
return sPkgs
|
||||
}
|
||||
|
||||
func increasePkgRel(pkg *BuildPackage) {
|
||||
f, err := os.OpenFile(pkg.Pkgbuild, os.O_RDWR, os.ModePerm)
|
||||
check(err)
|
||||
defer func(f *os.File) {
|
||||
check(f.Close())
|
||||
}(f)
|
||||
|
||||
fStr, err := io.ReadAll(f)
|
||||
check(err)
|
||||
|
||||
nStr := rePkgRel.ReplaceAllLiteralString(string(fStr), "pkgrel="+pkg.Srcinfo.Pkgrel+".1")
|
||||
_, err = f.Seek(0, 0)
|
||||
check(err)
|
||||
check(f.Truncate(0))
|
||||
|
||||
_, err = f.WriteString(nStr)
|
||||
check(err)
|
||||
|
||||
pkg.Version = pkg.Version + ".1"
|
||||
}
|
||||
|
||||
func gitClean(pkg *BuildPackage) {
|
||||
cmd := exec.Command("sudo", "git_clean.sh", filepath.Dir(pkg.Pkgbuild))
|
||||
res, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Warningf("git clean failed with %v:\n%s", err, res)
|
||||
} else {
|
||||
log.Debug(string(res))
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BuildManager) buildWorker(id int) {
|
||||
err := syscall.Setpriority(syscall.PRIO_PROCESS, 0, 18)
|
||||
if err != nil {
|
||||
@ -310,8 +73,17 @@ func (b *BuildManager) buildWorker(id int) {
|
||||
dbPkg.Update().SetStatus(BUILDING).SaveX(context.Background())
|
||||
dbLock.Unlock()
|
||||
|
||||
importKeys(pkg)
|
||||
increasePkgRel(pkg)
|
||||
err := importKeys(pkg)
|
||||
if err != nil {
|
||||
log.Warningf("[%s/%s] Failed to import pgp keys: %v", pkg.FullRepo, pkg.Pkgbase, err)
|
||||
}
|
||||
|
||||
err = increasePkgRel(pkg)
|
||||
if err != nil {
|
||||
log.Errorf("[%s/%s] Failed to increase pkgrel: %v", pkg.FullRepo, pkg.Pkgbase, err)
|
||||
b.buildWG.Done()
|
||||
continue
|
||||
}
|
||||
pkg.PkgFiles = []string{}
|
||||
|
||||
cmd := exec.Command("sh", "-c",
|
||||
@ -351,13 +123,8 @@ func (b *BuildManager) buildWorker(id int) {
|
||||
f, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE|os.O_SYNC, os.ModePerm)
|
||||
check(err)
|
||||
|
||||
if pkg.Srcinfo.Epoch != "" {
|
||||
_, err := f.WriteString(fmt.Sprintf("%s==%s:%s-%s\n", pkg.Pkgbase, pkg.Srcinfo.Epoch, pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel))
|
||||
check(err)
|
||||
} else {
|
||||
_, err := f.WriteString(fmt.Sprintf("%s==%s-%s\n", pkg.Pkgbase, pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel))
|
||||
check(err)
|
||||
}
|
||||
_, err = f.WriteString(fmt.Sprintf("%s==%s\n", pkg.Pkgbase, constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch)))
|
||||
check(err)
|
||||
check(f.Close())
|
||||
b.failedMutex.Unlock()
|
||||
|
||||
@ -430,17 +197,6 @@ func (b *BuildManager) buildWorker(id int) {
|
||||
}
|
||||
}
|
||||
|
||||
func getDbPackage(pkg *BuildPackage) *ent.DbPackage {
|
||||
dbLock.Lock()
|
||||
dbPkg, err := db.DbPackage.Query().Where(dbpackage.Pkgbase(pkg.Pkgbase)).Only(context.Background())
|
||||
if err != nil {
|
||||
dbPkg = db.DbPackage.Create().SetPkgbase(pkg.Pkgbase).SetMarch(pkg.March).SetPackages(packages2string(pkg.Srcinfo.Packages)).SetRepository(pkg.Repo).SaveX(context.Background())
|
||||
}
|
||||
dbLock.Unlock()
|
||||
|
||||
return dbPkg
|
||||
}
|
||||
|
||||
func (b *BuildManager) parseWorker() {
|
||||
for {
|
||||
if b.exit {
|
||||
@ -463,11 +219,7 @@ func (b *BuildManager) parseWorker() {
|
||||
continue
|
||||
}
|
||||
pkg.Srcinfo = info
|
||||
if pkg.Srcinfo.Epoch == "" {
|
||||
pkg.Version = pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
|
||||
} else {
|
||||
pkg.Version = pkg.Srcinfo.Epoch + ":" + pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
|
||||
}
|
||||
pkg.Version = constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch)
|
||||
|
||||
dbPkg := getDbPackage(pkg)
|
||||
dbLock.Lock()
|
||||
@ -481,7 +233,7 @@ func (b *BuildManager) parseWorker() {
|
||||
dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("arch = any").SaveX(context.Background())
|
||||
dbLock.Unlock()
|
||||
skipping = true
|
||||
} else if contains(conf.Blacklist, info.Pkgbase) {
|
||||
} else if contains(conf.Blacklist.Packages, info.Pkgbase) {
|
||||
log.Debugf("Skipped %s: blacklisted package", info.Pkgbase)
|
||||
dbLock.Lock()
|
||||
dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted").SaveX(context.Background())
|
||||
@ -527,114 +279,21 @@ func (b *BuildManager) parseWorker() {
|
||||
dbPkg = dbPkg.Update().SetStatus(QUEUED).SaveX(context.Background())
|
||||
dbLock.Unlock()
|
||||
|
||||
isLatest, err := isMirrorLatest(alpmHandle, pkg)
|
||||
check(err)
|
||||
|
||||
if !isLatest {
|
||||
log.Infof("Delayed %s: not all dependencies are up to date", info.Pkgbase)
|
||||
b.parseWG.Done()
|
||||
continue
|
||||
}
|
||||
|
||||
b.parseWG.Done()
|
||||
b.build <- pkg
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func findPkgFiles(pkg *BuildPackage) {
|
||||
pkgs, err := os.ReadDir(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch))
|
||||
check(err)
|
||||
|
||||
var fPkg []string
|
||||
for _, file := range pkgs {
|
||||
if !file.IsDir() && !strings.HasSuffix(file.Name(), ".sig") {
|
||||
matches := rePkgFile.FindStringSubmatch(file.Name())
|
||||
|
||||
var realPkgs []string
|
||||
for _, realPkg := range pkg.Srcinfo.Packages {
|
||||
realPkgs = append(realPkgs, realPkg.Pkgname)
|
||||
}
|
||||
|
||||
if len(matches) > 1 && contains(realPkgs, matches[1]) {
|
||||
fPkg = append(fPkg, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, file.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pkg.PkgFiles = fPkg
|
||||
}
|
||||
|
||||
func getVersionFromRepo(pkg *BuildPackage) string {
|
||||
findPkgFiles(pkg)
|
||||
|
||||
if len(pkg.PkgFiles) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
fNameSplit := strings.Split(pkg.PkgFiles[0], "-")
|
||||
return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2]
|
||||
}
|
||||
|
||||
func isPkgFailed(pkg *BuildPackage) bool {
|
||||
buildManager.failedMutex.Lock()
|
||||
defer buildManager.failedMutex.Unlock()
|
||||
|
||||
file, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_RDWR|os.O_CREATE|os.O_SYNC, os.ModePerm)
|
||||
check(err)
|
||||
defer func(file *os.File) {
|
||||
check(file.Close())
|
||||
}(file)
|
||||
|
||||
failed := false
|
||||
var newContent []string
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
splitPkg := strings.Split(line, "==")
|
||||
|
||||
if splitPkg[0] == pkg.Pkgbase {
|
||||
var pkgVer string
|
||||
if pkg.Srcinfo.Epoch == "" {
|
||||
pkgVer = pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
|
||||
} else {
|
||||
pkgVer = pkg.Srcinfo.Epoch + ":" + pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
|
||||
}
|
||||
|
||||
// try to build new versions of previously failed packages
|
||||
if alpm.VerCmp(splitPkg[1], pkgVer) < 0 {
|
||||
failed = false
|
||||
} else {
|
||||
failed = true
|
||||
newContent = append(newContent, line+"\n")
|
||||
}
|
||||
} else {
|
||||
newContent = append(newContent, line+"\n")
|
||||
}
|
||||
}
|
||||
|
||||
check(scanner.Err())
|
||||
sort.Strings(newContent)
|
||||
|
||||
_, err = file.Seek(0, 0)
|
||||
check(err)
|
||||
check(file.Truncate(0))
|
||||
_, err = file.WriteString(strings.Join(newContent, ""))
|
||||
check(err)
|
||||
|
||||
return failed
|
||||
}
|
||||
|
||||
func statusId2string(status int) (string, string) {
|
||||
switch status {
|
||||
case SKIPPED:
|
||||
return "SKIPPED", "table-" + conf.Status.Class.Skipped
|
||||
case QUEUED:
|
||||
return "QUEUED", "table-" + conf.Status.Class.Queued
|
||||
case LATEST:
|
||||
return "LATEST", "table-" + conf.Status.Class.Latest
|
||||
case FAILED:
|
||||
return "FAILED", "table-" + conf.Status.Class.Failed
|
||||
case BUILD:
|
||||
return "SIGNING", "table-" + conf.Status.Class.Signing
|
||||
case BUILDING:
|
||||
return "BUILDING", "table-" + conf.Status.Class.Building
|
||||
default:
|
||||
return "UNKNOWN", "table-" + conf.Status.Class.Unknown
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BuildManager) htmlWorker() {
|
||||
type Pkg struct {
|
||||
Pkgbase string
|
||||
@ -731,26 +390,6 @@ func (b *BuildManager) htmlWorker() {
|
||||
}
|
||||
}
|
||||
|
||||
func setupChroot() {
|
||||
if _, err := os.Stat(filepath.Join(conf.Basedir.Chroot, orgChrootName)); err == nil {
|
||||
//goland:noinspection SpellCheckingInspection
|
||||
cmd := exec.Command("arch-nspawn", filepath.Join(conf.Basedir.Chroot, orgChrootName), "pacman", "-Syuu", "--noconfirm")
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
check(err)
|
||||
} else if os.IsNotExist(err) {
|
||||
err := os.MkdirAll(conf.Basedir.Chroot, os.ModePerm)
|
||||
check(err)
|
||||
|
||||
cmd := exec.Command("mkarchroot", "-C", pacmanConf, filepath.Join(conf.Basedir.Chroot, orgChrootName), "base-devel")
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
check(err)
|
||||
} else {
|
||||
check(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BuildManager) repoWorker(repo string) {
|
||||
for {
|
||||
select {
|
||||
@ -874,7 +513,7 @@ func (b *BuildManager) syncWorker() {
|
||||
// fetch updates between sync runs
|
||||
setupChroot()
|
||||
|
||||
pkgBuilds, err := filepathx.Glob(filepath.Join(conf.Basedir.Upstream, "/**/PKGBUILD"))
|
||||
pkgBuilds, err := Glob(filepath.Join(conf.Basedir.Upstream, "/**/PKGBUILD"))
|
||||
check(err)
|
||||
|
||||
// Shuffle pkgbuilds to spread out long-running builds, otherwise pkgBuilds is alphabetically-sorted
|
||||
@ -889,7 +528,7 @@ func (b *BuildManager) syncWorker() {
|
||||
sPkgbuild := strings.Split(pkgbuild, "/")
|
||||
repo := sPkgbuild[len(sPkgbuild)-2]
|
||||
|
||||
if repo == "trunk" || !contains(conf.Repos, strings.Split(repo, "-")[0]) || strings.Contains(repo, "i686") || strings.Contains(repo, "testing") || strings.Contains(repo, "staging") {
|
||||
if repo == "trunk" || !contains(conf.Repos, strings.Split(repo, "-")[0]) || containsSubStr(repo, conf.Blacklist.Repo) {
|
||||
continue
|
||||
}
|
||||
|
||||
@ -956,6 +595,11 @@ func main() {
|
||||
setupChroot()
|
||||
syncMarchs()
|
||||
|
||||
alpmLock.Lock()
|
||||
alpmHandle, err = initALPM(filepath.Join(conf.Basedir.Chroot, "root"), filepath.Join(conf.Basedir.Chroot, "/root/var/lib/pacman"))
|
||||
check(err)
|
||||
alpmLock.Unlock()
|
||||
|
||||
go buildManager.syncWorker()
|
||||
go buildManager.htmlWorker()
|
||||
|
||||
@ -972,4 +616,5 @@ func main() {
|
||||
buildManager.buildProcMutex.RUnlock()
|
||||
buildManager.buildWG.Wait()
|
||||
buildManager.repoWG.Wait()
|
||||
check(alpmHandle.Release())
|
||||
}
|
||||
|
||||
591
utils.go
Normal file
591
utils.go
Normal file
@ -0,0 +1,591 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"ALHP.go/ent"
|
||||
"ALHP.go/ent/dbpackage"
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/Jguer/go-alpm/v2"
|
||||
paconf "github.com/Morganamilo/go-pacmanconf"
|
||||
"github.com/Morganamilo/go-srcinfo"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
SKIPPED = iota
|
||||
FAILED = iota
|
||||
BUILD = iota
|
||||
QUEUED = iota
|
||||
BUILDING = iota
|
||||
LATEST = iota
|
||||
UNKNOWN = iota
|
||||
)
|
||||
|
||||
type BuildPackage struct {
|
||||
Pkgbase string
|
||||
Pkgbuild string
|
||||
Srcinfo *srcinfo.Srcinfo
|
||||
PkgFiles []string
|
||||
Repo string
|
||||
March string
|
||||
FullRepo string
|
||||
Version string
|
||||
}
|
||||
|
||||
type BuildManager struct {
|
||||
build chan *BuildPackage
|
||||
parse chan *BuildPackage
|
||||
repoPurge map[string]chan *BuildPackage
|
||||
repoAdd map[string]chan *BuildPackage
|
||||
exit bool
|
||||
buildWG sync.WaitGroup
|
||||
parseWG sync.WaitGroup
|
||||
repoWG sync.WaitGroup
|
||||
failedMutex sync.RWMutex
|
||||
buildProcesses []*os.Process
|
||||
buildProcMutex sync.RWMutex
|
||||
}
|
||||
|
||||
type Conf struct {
|
||||
Arch string
|
||||
Repos, March []string
|
||||
Svn2git map[string]string
|
||||
Basedir struct {
|
||||
Repo, Chroot, Makepkg, Upstream, Db string
|
||||
}
|
||||
Build struct {
|
||||
Worker int
|
||||
Makej int
|
||||
}
|
||||
Logging struct {
|
||||
Level string
|
||||
}
|
||||
Status struct {
|
||||
Class struct {
|
||||
Skipped, Queued, Latest, Failed, Signing, Building, Unknown string
|
||||
}
|
||||
}
|
||||
Blacklist struct {
|
||||
Packages []string
|
||||
Repo []string
|
||||
}
|
||||
}
|
||||
|
||||
type Globs []string
|
||||
|
||||
func check(e error) {
|
||||
if e != nil {
|
||||
panic(e)
|
||||
}
|
||||
}
|
||||
|
||||
func containsSubStr(str string, subList []string) bool {
|
||||
for _, checkStr := range subList {
|
||||
if strings.Contains(str, checkStr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func statusId2string(status int) (string, string) {
|
||||
switch status {
|
||||
case SKIPPED:
|
||||
return "SKIPPED", "table-" + conf.Status.Class.Skipped
|
||||
case QUEUED:
|
||||
return "QUEUED", "table-" + conf.Status.Class.Queued
|
||||
case LATEST:
|
||||
return "LATEST", "table-" + conf.Status.Class.Latest
|
||||
case FAILED:
|
||||
return "FAILED", "table-" + conf.Status.Class.Failed
|
||||
case BUILD:
|
||||
return "SIGNING", "table-" + conf.Status.Class.Signing
|
||||
case BUILDING:
|
||||
return "BUILDING", "table-" + conf.Status.Class.Building
|
||||
default:
|
||||
return "UNKNOWN", "table-" + conf.Status.Class.Unknown
|
||||
}
|
||||
}
|
||||
|
||||
func getVersionFromRepo(pkg *BuildPackage) string {
|
||||
findPkgFiles(pkg)
|
||||
|
||||
if len(pkg.PkgFiles) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
fNameSplit := strings.Split(pkg.PkgFiles[0], "-")
|
||||
return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2]
|
||||
}
|
||||
|
||||
func gitClean(pkg *BuildPackage) {
|
||||
cmd := exec.Command("sudo", "git_clean.sh", filepath.Dir(pkg.Pkgbuild))
|
||||
res, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Warningf("git clean failed with %v:\n%s", err, res)
|
||||
} else {
|
||||
log.Debug(string(res))
|
||||
}
|
||||
}
|
||||
|
||||
func increasePkgRel(pkg *BuildPackage) error {
|
||||
f, err := os.OpenFile(pkg.Pkgbuild, os.O_RDWR, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func(f *os.File) {
|
||||
err := f.Close()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}(f)
|
||||
|
||||
fStr, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
nStr := rePkgRel.ReplaceAllLiteralString(string(fStr), "pkgrel="+pkg.Srcinfo.Pkgrel+".1")
|
||||
_, err = f.Seek(0, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = f.Truncate(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = f.WriteString(nStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pkg.Version = pkg.Version + ".1"
|
||||
return nil
|
||||
}
|
||||
|
||||
func packages2slice(pkgs interface{}) []string {
|
||||
switch v := pkgs.(type) {
|
||||
case []srcinfo.Package:
|
||||
var sPkgs []string
|
||||
for _, p := range v {
|
||||
sPkgs = append(sPkgs, p.Pkgname)
|
||||
}
|
||||
|
||||
return sPkgs
|
||||
case []srcinfo.ArchString:
|
||||
var sPkgs []string
|
||||
for _, p := range v {
|
||||
sPkgs = append(sPkgs, p.Value)
|
||||
}
|
||||
|
||||
return sPkgs
|
||||
default:
|
||||
return []string{}
|
||||
}
|
||||
}
|
||||
|
||||
func importKeys(pkg *BuildPackage) error {
|
||||
if pkg.Srcinfo.ValidPGPKeys != nil {
|
||||
args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"}
|
||||
args = append(args, pkg.Srcinfo.ValidPGPKeys...)
|
||||
cmd := exec.Command("gpg", args...)
|
||||
_, err := cmd.CombinedOutput()
|
||||
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func constructVersion(pkgver string, pkgrel string, epoch string) string {
|
||||
if epoch == "" {
|
||||
return pkgver + "-" + pkgrel
|
||||
} else {
|
||||
return epoch + ":" + pkgver + "-" + pkgrel
|
||||
}
|
||||
}
|
||||
|
||||
func initALPM(root string, dbpath string) (*alpm.Handle, error) {
|
||||
h, err := alpm.Initialize(root, dbpath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
PacmanConfig, _, err := paconf.ParseFile(filepath.Join(root, "/etc/pacman.conf"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, repo := range PacmanConfig.Repos {
|
||||
db, err := h.RegisterSyncDB(repo.Name, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetServers(repo.Servers)
|
||||
|
||||
if len(repo.Usage) == 0 {
|
||||
db.SetUsage(alpm.UsageAll)
|
||||
}
|
||||
for _, usage := range repo.Usage {
|
||||
switch usage {
|
||||
case "Sync":
|
||||
db.SetUsage(alpm.UsageSync)
|
||||
case "Search":
|
||||
db.SetUsage(alpm.UsageSearch)
|
||||
case "Install":
|
||||
db.SetUsage(alpm.UsageInstall)
|
||||
case "Upgrade":
|
||||
db.SetUsage(alpm.UsageUpgrade)
|
||||
case "All":
|
||||
db.SetUsage(alpm.UsageAll)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return h, nil
|
||||
}
|
||||
|
||||
func getSVN2GITVersion(pkg *BuildPackage) (string, error) {
|
||||
if pkg.Pkgbuild == "" && pkg.Pkgbase == "" {
|
||||
return "", fmt.Errorf("invalid arguments")
|
||||
}
|
||||
|
||||
// upstream/upstream-core-extra/extra-cmake-modules/repos/extra-any/PKGBUILD
|
||||
pkgBuilds, _ := Glob(filepath.Join(conf.Basedir.Upstream, "**/"+pkg.Pkgbase+"/repos/*/PKGBUILD"))
|
||||
|
||||
var fPkgbuilds []string
|
||||
for _, pkgbuild := range pkgBuilds {
|
||||
sPkgbuild := strings.Split(pkgbuild, "/")
|
||||
repo := sPkgbuild[len(sPkgbuild)-2]
|
||||
|
||||
if repo == "trunk" || containsSubStr(repo, conf.Blacklist.Repo) {
|
||||
continue
|
||||
}
|
||||
|
||||
if !contains(fPkgbuilds, pkgbuild) {
|
||||
fPkgbuilds = append(fPkgbuilds, pkgbuild)
|
||||
}
|
||||
}
|
||||
|
||||
if len(fPkgbuilds) > 1 {
|
||||
return "", fmt.Errorf("%s: multiple PKGBUILD found: %s", pkg.Pkgbase, fPkgbuilds)
|
||||
} else if len(fPkgbuilds) == 0 {
|
||||
return "", fmt.Errorf("%s: no matching PKGBUILD found (searched: %s, canidates: %s)", pkg.Pkgbase, filepath.Join(conf.Basedir.Upstream, "**/"+pkg.Pkgbase+"/repos/*/PKGBUILD"), pkgBuilds)
|
||||
}
|
||||
|
||||
cmd := exec.Command("sh", "-c", "cd "+filepath.Dir(fPkgbuilds[0])+"&&"+"makepkg --printsrcinfo")
|
||||
res, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
info, err := srcinfo.Parse(string(res))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return constructVersion(info.Pkgver, info.Pkgrel, info.Epoch), nil
|
||||
}
|
||||
|
||||
func isPkgFailed(pkg *BuildPackage) bool {
|
||||
buildManager.failedMutex.Lock()
|
||||
defer buildManager.failedMutex.Unlock()
|
||||
|
||||
file, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_RDWR|os.O_CREATE|os.O_SYNC, os.ModePerm)
|
||||
check(err)
|
||||
defer func(file *os.File) {
|
||||
check(file.Close())
|
||||
}(file)
|
||||
|
||||
failed := false
|
||||
var newContent []string
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
splitPkg := strings.Split(line, "==")
|
||||
|
||||
if splitPkg[0] == pkg.Pkgbase {
|
||||
pkgVer := constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch)
|
||||
|
||||
// try to build new versions of previously failed packages
|
||||
if alpm.VerCmp(splitPkg[1], pkgVer) < 0 {
|
||||
failed = false
|
||||
} else {
|
||||
failed = true
|
||||
newContent = append(newContent, line+"\n")
|
||||
}
|
||||
} else {
|
||||
newContent = append(newContent, line+"\n")
|
||||
}
|
||||
}
|
||||
|
||||
check(scanner.Err())
|
||||
sort.Strings(newContent)
|
||||
|
||||
_, err = file.Seek(0, 0)
|
||||
check(err)
|
||||
check(file.Truncate(0))
|
||||
_, err = file.WriteString(strings.Join(newContent, ""))
|
||||
check(err)
|
||||
|
||||
return failed
|
||||
}
|
||||
|
||||
func setupChroot() {
|
||||
if _, err := os.Stat(filepath.Join(conf.Basedir.Chroot, orgChrootName)); err == nil {
|
||||
//goland:noinspection SpellCheckingInspection
|
||||
cmd := exec.Command("arch-nspawn", filepath.Join(conf.Basedir.Chroot, orgChrootName), "pacman", "-Syuu", "--noconfirm")
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
check(err)
|
||||
} else if os.IsNotExist(err) {
|
||||
err := os.MkdirAll(conf.Basedir.Chroot, os.ModePerm)
|
||||
check(err)
|
||||
|
||||
cmd := exec.Command("mkarchroot", "-C", pacmanConf, filepath.Join(conf.Basedir.Chroot, orgChrootName), "base-devel")
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
check(err)
|
||||
} else {
|
||||
check(err)
|
||||
}
|
||||
}
|
||||
|
||||
func findPkgFiles(pkg *BuildPackage) {
|
||||
pkgs, err := os.ReadDir(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch))
|
||||
check(err)
|
||||
|
||||
var fPkg []string
|
||||
for _, file := range pkgs {
|
||||
if !file.IsDir() && !strings.HasSuffix(file.Name(), ".sig") {
|
||||
matches := rePkgFile.FindStringSubmatch(file.Name())
|
||||
|
||||
var realPkgs []string
|
||||
for _, realPkg := range pkg.Srcinfo.Packages {
|
||||
realPkgs = append(realPkgs, realPkg.Pkgname)
|
||||
}
|
||||
|
||||
if len(matches) > 1 && contains(realPkgs, matches[1]) {
|
||||
fPkg = append(fPkg, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, file.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pkg.PkgFiles = fPkg
|
||||
}
|
||||
|
||||
func getDbPackage(pkg *BuildPackage) *ent.DbPackage {
|
||||
dbLock.Lock()
|
||||
dbPkg, err := db.DbPackage.Query().Where(dbpackage.Pkgbase(pkg.Pkgbase)).Only(context.Background())
|
||||
if err != nil {
|
||||
dbPkg = db.DbPackage.Create().SetPkgbase(pkg.Pkgbase).SetMarch(pkg.March).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetRepository(pkg.Repo).SaveX(context.Background())
|
||||
}
|
||||
dbLock.Unlock()
|
||||
|
||||
return dbPkg
|
||||
}
|
||||
|
||||
func syncMarchs() {
|
||||
files, err := os.ReadDir(conf.Basedir.Repo)
|
||||
check(err)
|
||||
|
||||
var eRepos []string
|
||||
for _, file := range files {
|
||||
if file.Name() != "." && file.Name() != logDir && file.IsDir() {
|
||||
eRepos = append(eRepos, file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
for _, march := range conf.March {
|
||||
setupMakepkg(march)
|
||||
for _, repo := range conf.Repos {
|
||||
tRepo := fmt.Sprintf("%s-%s", repo, march)
|
||||
repos = append(repos, tRepo)
|
||||
buildManager.repoAdd[tRepo] = make(chan *BuildPackage, conf.Build.Worker)
|
||||
buildManager.repoPurge[tRepo] = make(chan *BuildPackage, 10000)
|
||||
go buildManager.repoWorker(tRepo)
|
||||
|
||||
if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch))); os.IsNotExist(err) {
|
||||
log.Debugf("Creating path %s", filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch))
|
||||
check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch), os.ModePerm))
|
||||
}
|
||||
|
||||
if i := find(eRepos, tRepo); i != -1 {
|
||||
eRepos = append(eRepos[:i], eRepos[i+1:]...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("Repos: %s", repos)
|
||||
|
||||
for _, repo := range eRepos {
|
||||
log.Infof("Removing old repo %s", repo)
|
||||
check(os.RemoveAll(filepath.Join(conf.Basedir.Repo, repo)))
|
||||
}
|
||||
}
|
||||
|
||||
//goland:noinspection SpellCheckingInspection
|
||||
func setupMakepkg(march string) {
|
||||
lMakepkg := filepath.Join(conf.Basedir.Makepkg, fmt.Sprintf("makepkg-%s.conf", march))
|
||||
|
||||
check(os.MkdirAll(conf.Basedir.Makepkg, os.ModePerm))
|
||||
t, err := os.ReadFile(makepkgConf)
|
||||
check(err)
|
||||
makepkgStr := string(t)
|
||||
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, "-mtune=generic", "")
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, "-O2", "-O3")
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, " check ", " !check ")
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, " color ", " !color ")
|
||||
makepkgStr = strings.ReplaceAll(makepkgStr, "#MAKEFLAGS=\"-j2\"", "MAKEFLAGS=\"-j"+strconv.Itoa(conf.Build.Makej)+"\"")
|
||||
makepkgStr = reMarch.ReplaceAllString(makepkgStr, "${1}"+march)
|
||||
|
||||
check(os.WriteFile(lMakepkg, []byte(makepkgStr), os.ModePerm))
|
||||
}
|
||||
|
||||
func isMirrorLatest(h *alpm.Handle, buildPkg *BuildPackage) (bool, error) {
|
||||
alpmLock.Lock()
|
||||
dbs, err := h.SyncDBs()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
allDepends := buildPkg.Srcinfo.Depends
|
||||
allDepends = append(allDepends, buildPkg.Srcinfo.MakeDepends...)
|
||||
|
||||
for _, dep := range allDepends {
|
||||
pkg, err := dbs.FindSatisfier(dep.Value)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
svn2gitVer, err := getSVN2GITVersion(&BuildPackage{
|
||||
Pkgbase: pkg.Base(),
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if svn2gitVer != "" && alpm.VerCmp(svn2gitVer, pkg.Version()) != 0 {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
alpmLock.Unlock()
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func contains(s interface{}, str string) bool {
|
||||
switch v := s.(type) {
|
||||
case []string:
|
||||
if i := find(v, str); i != -1 {
|
||||
return true
|
||||
}
|
||||
case []srcinfo.ArchString:
|
||||
var n []string
|
||||
for _, as := range v {
|
||||
n = append(n, as.Value)
|
||||
}
|
||||
|
||||
if i := find(n, str); i != -1 {
|
||||
return true
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func find(s []string, str string) int {
|
||||
for i, v := range s {
|
||||
if v == str {
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) (int64, error) {
|
||||
sourceFileStat, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if !sourceFileStat.Mode().IsRegular() {
|
||||
return 0, fmt.Errorf("%s is not a regular file", src)
|
||||
}
|
||||
|
||||
source, err := os.Open(src)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func(source *os.File) {
|
||||
check(source.Close())
|
||||
}(source)
|
||||
|
||||
destination, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func(destination *os.File) {
|
||||
check(destination.Close())
|
||||
}(destination)
|
||||
nBytes, err := io.Copy(destination, source)
|
||||
return nBytes, err
|
||||
}
|
||||
|
||||
func Glob(pattern string) ([]string, error) {
|
||||
if !strings.Contains(pattern, "**") {
|
||||
return filepath.Glob(pattern)
|
||||
}
|
||||
return Globs(strings.Split(pattern, "**")).Expand()
|
||||
}
|
||||
|
||||
func (globs Globs) Expand() ([]string, error) {
|
||||
var matches = []string{""}
|
||||
for _, glob := range globs {
|
||||
var hits []string
|
||||
var hitMap = map[string]bool{}
|
||||
for _, match := range matches {
|
||||
paths, err := filepath.Glob(match + glob)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, path := range paths {
|
||||
err = filepath.WalkDir(path, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return fs.SkipDir
|
||||
}
|
||||
if _, ok := hitMap[path]; !ok {
|
||||
hits = append(hits, path)
|
||||
hitMap[path] = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
matches = hits
|
||||
}
|
||||
|
||||
if globs == nil && len(matches) > 0 && matches[0] == "" {
|
||||
matches = matches[1:]
|
||||
}
|
||||
|
||||
return matches, nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user