ALHP/ALHP.GO
5
39
Fork 10
Code Issues 32 Pull Requests Projects Releases Wiki Activity

added separate hash for srcinfo cache

Browse Source
This commit is contained in:
Giovanni Harting 2022-08-14 15:02:24 +02:00
parent 041e7762f7
commit f30442d100
11 changed files with 380 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ent/dbpackage.go
View File

@ -59,6 +59,8 @@ type DbPackage struct {
IoOut *int64 `json:"io_out,omitempty"`
// Srcinfo holds the value of the "srcinfo" field.
Srcinfo *string `json:"srcinfo,omitempty"`
// SrcinfoHash holds the value of the "srcinfo_hash" field.
SrcinfoHash string `json:"srcinfo_hash,omitempty"`
}
// scanValues returns the types for scanning values from sql.Rows.
@ -70,7 +72,7 @@ func (*DbPackage) scanValues(columns []string) ([]interface{}, error) {
values[i] = new([]byte)
case dbpackage.FieldID, dbpackage.FieldMaxRss, dbpackage.FieldUTime, dbpackage.FieldSTime, dbpackage.FieldIoIn, dbpackage.FieldIoOut:
values[i] = new(sql.NullInt64)
case dbpackage.FieldPkgbase, dbpackage.FieldStatus, dbpackage.FieldSkipReason, dbpackage.FieldRepository, dbpackage.FieldMarch, dbpackage.FieldVersion, dbpackage.FieldRepoVersion, dbpackage.FieldHash, dbpackage.FieldLto, dbpackage.FieldLastVersionBuild, dbpackage.FieldDebugSymbols, dbpackage.FieldSrcinfo:
case dbpackage.FieldPkgbase, dbpackage.FieldStatus, dbpackage.FieldSkipReason, dbpackage.FieldRepository, dbpackage.FieldMarch, dbpackage.FieldVersion, dbpackage.FieldRepoVersion, dbpackage.FieldHash, dbpackage.FieldLto, dbpackage.FieldLastVersionBuild, dbpackage.FieldDebugSymbols, dbpackage.FieldSrcinfo, dbpackage.FieldSrcinfoHash:
values[i] = new(sql.NullString)
case dbpackage.FieldBuildTimeStart, dbpackage.FieldUpdated, dbpackage.FieldLastVerified:
values[i] = new(sql.NullTime)
@ -229,6 +231,12 @@ func (dp *DbPackage) assignValues(columns []string, values []interface{}) error
dp.Srcinfo = new(string)
*dp.Srcinfo = value.String
}
case dbpackage.FieldSrcinfoHash:
if value, ok := values[i].(*sql.NullString); !ok {
return fmt.Errorf("unexpected type %T for field srcinfo_hash", values[i])
} else if value.Valid {
dp.SrcinfoHash = value.String
}
}
}
return nil
@ -331,6 +339,9 @@ func (dp *DbPackage) String() string {
builder.WriteString("srcinfo=")
builder.WriteString(*v)
}
builder.WriteString(", ")
builder.WriteString("srcinfo_hash=")
builder.WriteString(dp.SrcinfoHash)
builder.WriteByte(')')
return builder.String()
}

3
ent/dbpackage/dbpackage.go
View File

@ -53,6 +53,8 @@ const (
FieldIoOut = "io_out"
// FieldSrcinfo holds the string denoting the srcinfo field in the database.
FieldSrcinfo = "srcinfo"
// FieldSrcinfoHash holds the string denoting the srcinfo_hash field in the database.
FieldSrcinfoHash = "srcinfo_hash"
// Table holds the table name of the dbpackage in the database.
Table = "db_packages"
)
@ -81,6 +83,7 @@ var Columns = []string{
FieldIoIn,
FieldIoOut,
FieldSrcinfo,
FieldSrcinfoHash,
}
// ValidColumn reports if the column name is valid (part of the table columns).

120
ent/dbpackage/where.go
View File

@ -192,6 +192,13 @@ func Srcinfo(v string) predicate.DbPackage {
})
}
// SrcinfoHash applies equality check predicate on the "srcinfo_hash" field. It's identical to SrcinfoHashEQ.
func SrcinfoHash(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.EQ(s.C(FieldSrcinfoHash), v))
})
}
// PkgbaseEQ applies the EQ predicate on the "pkgbase" field.
func PkgbaseEQ(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
@ -1892,6 +1899,119 @@ func SrcinfoContainsFold(v string) predicate.DbPackage {
})
}
// SrcinfoHashEQ applies the EQ predicate on the "srcinfo_hash" field.
func SrcinfoHashEQ(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.EQ(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashNEQ applies the NEQ predicate on the "srcinfo_hash" field.
func SrcinfoHashNEQ(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.NEQ(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashIn applies the In predicate on the "srcinfo_hash" field.
func SrcinfoHashIn(vs ...string) predicate.DbPackage {
v := make([]interface{}, len(vs))
for i := range v {
v[i] = vs[i]
}
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.In(s.C(FieldSrcinfoHash), v...))
})
}
// SrcinfoHashNotIn applies the NotIn predicate on the "srcinfo_hash" field.
func SrcinfoHashNotIn(vs ...string) predicate.DbPackage {
v := make([]interface{}, len(vs))
for i := range v {
v[i] = vs[i]
}
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.NotIn(s.C(FieldSrcinfoHash), v...))
})
}
// SrcinfoHashGT applies the GT predicate on the "srcinfo_hash" field.
func SrcinfoHashGT(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.GT(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashGTE applies the GTE predicate on the "srcinfo_hash" field.
func SrcinfoHashGTE(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.GTE(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashLT applies the LT predicate on the "srcinfo_hash" field.
func SrcinfoHashLT(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.LT(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashLTE applies the LTE predicate on the "srcinfo_hash" field.
func SrcinfoHashLTE(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.LTE(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashContains applies the Contains predicate on the "srcinfo_hash" field.
func SrcinfoHashContains(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.Contains(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashHasPrefix applies the HasPrefix predicate on the "srcinfo_hash" field.
func SrcinfoHashHasPrefix(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.HasPrefix(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashHasSuffix applies the HasSuffix predicate on the "srcinfo_hash" field.
func SrcinfoHashHasSuffix(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.HasSuffix(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashIsNil applies the IsNil predicate on the "srcinfo_hash" field.
func SrcinfoHashIsNil() predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.IsNull(s.C(FieldSrcinfoHash)))
})
}
// SrcinfoHashNotNil applies the NotNil predicate on the "srcinfo_hash" field.
func SrcinfoHashNotNil() predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.NotNull(s.C(FieldSrcinfoHash)))
})
}
// SrcinfoHashEqualFold applies the EqualFold predicate on the "srcinfo_hash" field.
func SrcinfoHashEqualFold(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.EqualFold(s.C(FieldSrcinfoHash), v))
})
}
// SrcinfoHashContainsFold applies the ContainsFold predicate on the "srcinfo_hash" field.
func SrcinfoHashContainsFold(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.ContainsFold(s.C(FieldSrcinfoHash), v))
})
}
// And groups predicates with the AND operator between them.
func And(predicates ...predicate.DbPackage) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {

22
ent/dbpackage_create.go
View File

@ -282,6 +282,20 @@ func (dpc *DbPackageCreate) SetNillableSrcinfo(s *string) *DbPackageCreate {
return dpc
}
// SetSrcinfoHash sets the "srcinfo_hash" field.
func (dpc *DbPackageCreate) SetSrcinfoHash(s string) *DbPackageCreate {
dpc.mutation.SetSrcinfoHash(s)
return dpc
}
// SetNillableSrcinfoHash sets the "srcinfo_hash" field if the given value is not nil.
func (dpc *DbPackageCreate) SetNillableSrcinfoHash(s *string) *DbPackageCreate {
if s != nil {
dpc.SetSrcinfoHash(*s)
}
return dpc
}
// Mutation returns the DbPackageMutation object of the builder.
func (dpc *DbPackageCreate) Mutation() *DbPackageMutation {
return dpc.mutation
@ -609,6 +623,14 @@ func (dpc *DbPackageCreate) createSpec() (*DbPackage, *sqlgraph.CreateSpec) {
})
_node.Srcinfo = &value
}
if value, ok := dpc.mutation.SrcinfoHash(); ok {
_spec.Fields = append(_spec.Fields, &sqlgraph.FieldSpec{
Type: field.TypeString,
Value: value,
Column: dbpackage.FieldSrcinfoHash,
})
_node.SrcinfoHash = value
}
return _node, _spec
}

66
ent/dbpackage_update.go
View File

@ -422,6 +422,26 @@ func (dpu *DbPackageUpdate) ClearSrcinfo() *DbPackageUpdate {
return dpu
}
// SetSrcinfoHash sets the "srcinfo_hash" field.
func (dpu *DbPackageUpdate) SetSrcinfoHash(s string) *DbPackageUpdate {
dpu.mutation.SetSrcinfoHash(s)
return dpu
}
// SetNillableSrcinfoHash sets the "srcinfo_hash" field if the given value is not nil.
func (dpu *DbPackageUpdate) SetNillableSrcinfoHash(s *string) *DbPackageUpdate {
if s != nil {
dpu.SetSrcinfoHash(*s)
}
return dpu
}
// ClearSrcinfoHash clears the value of the "srcinfo_hash" field.
func (dpu *DbPackageUpdate) ClearSrcinfoHash() *DbPackageUpdate {
dpu.mutation.ClearSrcinfoHash()
return dpu
}
// Mutation returns the DbPackageMutation object of the builder.
func (dpu *DbPackageUpdate) Mutation() *DbPackageMutation {
return dpu.mutation
@ -812,6 +832,19 @@ func (dpu *DbPackageUpdate) sqlSave(ctx context.Context) (n int, err error) {
Column: dbpackage.FieldSrcinfo,
})
}
if value, ok := dpu.mutation.SrcinfoHash(); ok {
_spec.Fields.Set = append(_spec.Fields.Set, &sqlgraph.FieldSpec{
Type: field.TypeString,
Value: value,
Column: dbpackage.FieldSrcinfoHash,
})
}
if dpu.mutation.SrcinfoHashCleared() {
_spec.Fields.Clear = append(_spec.Fields.Clear, &sqlgraph.FieldSpec{
Type: field.TypeString,
Column: dbpackage.FieldSrcinfoHash,
})
}
_spec.Modifiers = dpu.modifiers
if n, err = sqlgraph.UpdateNodes(ctx, dpu.driver, _spec); err != nil {
if _, ok := err.(*sqlgraph.NotFoundError); ok {
@ -1226,6 +1259,26 @@ func (dpuo *DbPackageUpdateOne) ClearSrcinfo() *DbPackageUpdateOne {
return dpuo
}
// SetSrcinfoHash sets the "srcinfo_hash" field.
func (dpuo *DbPackageUpdateOne) SetSrcinfoHash(s string) *DbPackageUpdateOne {
dpuo.mutation.SetSrcinfoHash(s)
return dpuo
}
// SetNillableSrcinfoHash sets the "srcinfo_hash" field if the given value is not nil.
func (dpuo *DbPackageUpdateOne) SetNillableSrcinfoHash(s *string) *DbPackageUpdateOne {
if s != nil {
dpuo.SetSrcinfoHash(*s)
}
return dpuo
}
// ClearSrcinfoHash clears the value of the "srcinfo_hash" field.
func (dpuo *DbPackageUpdateOne) ClearSrcinfoHash() *DbPackageUpdateOne {
dpuo.mutation.ClearSrcinfoHash()
return dpuo
}
// Mutation returns the DbPackageMutation object of the builder.
func (dpuo *DbPackageUpdateOne) Mutation() *DbPackageMutation {
return dpuo.mutation
@ -1646,6 +1699,19 @@ func (dpuo *DbPackageUpdateOne) sqlSave(ctx context.Context) (_node *DbPackage,
Column: dbpackage.FieldSrcinfo,
})
}
if value, ok := dpuo.mutation.SrcinfoHash(); ok {
_spec.Fields.Set = append(_spec.Fields.Set, &sqlgraph.FieldSpec{
Type: field.TypeString,
Value: value,
Column: dbpackage.FieldSrcinfoHash,
})
}
if dpuo.mutation.SrcinfoHashCleared() {
_spec.Fields.Clear = append(_spec.Fields.Clear, &sqlgraph.FieldSpec{
Type: field.TypeString,
Column: dbpackage.FieldSrcinfoHash,
})
}
_spec.Modifiers = dpuo.modifiers
_node = &DbPackage{config: dpuo.config}
_spec.Assign = _node.assignValues

1
ent/migrate/schema.go
View File

@ -32,6 +32,7 @@ var (
{Name: "io_in", Type: field.TypeInt64, Nullable: true},
{Name: "io_out", Type: field.TypeInt64, Nullable: true},
{Name: "srcinfo", Type: field.TypeString, Nullable: true, Size: 2147483647},
{Name: "srcinfo_hash", Type: field.TypeString, Nullable: true},
}
// DbPackagesTable holds the schema information for the "db_packages" table.
DbPackagesTable = &schema.Table{

75
ent/mutation.go
View File

@ -59,6 +59,7 @@ type DbPackageMutation struct {
io_out *int64
addio_out *int64
srcinfo *string
srcinfo_hash *string
clearedFields map[string]struct{}
done bool
oldValue func(context.Context) (*DbPackage, error)
@ -1258,6 +1259,55 @@ func (m *DbPackageMutation) ResetSrcinfo() {
delete(m.clearedFields, dbpackage.FieldSrcinfo)
}
// SetSrcinfoHash sets the "srcinfo_hash" field.
func (m *DbPackageMutation) SetSrcinfoHash(s string) {
m.srcinfo_hash = &s
}
// SrcinfoHash returns the value of the "srcinfo_hash" field in the mutation.
func (m *DbPackageMutation) SrcinfoHash() (r string, exists bool) {
v := m.srcinfo_hash
if v == nil {
return
}
return *v, true
}
// OldSrcinfoHash returns the old "srcinfo_hash" field's value of the DbPackage entity.
// If the DbPackage object wasn't provided to the builder, the object is fetched from the database.
// An error is returned if the mutation operation is not UpdateOne, or the database query fails.
func (m *DbPackageMutation) OldSrcinfoHash(ctx context.Context) (v string, err error) {
if !m.op.Is(OpUpdateOne) {
return v, errors.New("OldSrcinfoHash is only allowed on UpdateOne operations")
}
if m.id == nil || m.oldValue == nil {
return v, errors.New("OldSrcinfoHash requires an ID field in the mutation")
}
oldValue, err := m.oldValue(ctx)
if err != nil {
return v, fmt.Errorf("querying old value for OldSrcinfoHash: %w", err)
}
return oldValue.SrcinfoHash, nil
}
// ClearSrcinfoHash clears the value of the "srcinfo_hash" field.
func (m *DbPackageMutation) ClearSrcinfoHash() {
m.srcinfo_hash = nil
m.clearedFields[dbpackage.FieldSrcinfoHash] = struct{}{}
}
// SrcinfoHashCleared returns if the "srcinfo_hash" field was cleared in this mutation.
func (m *DbPackageMutation) SrcinfoHashCleared() bool {
_, ok := m.clearedFields[dbpackage.FieldSrcinfoHash]
return ok
}
// ResetSrcinfoHash resets all changes to the "srcinfo_hash" field.
func (m *DbPackageMutation) ResetSrcinfoHash() {
m.srcinfo_hash = nil
delete(m.clearedFields, dbpackage.FieldSrcinfoHash)
}
// Where appends a list predicates to the DbPackageMutation builder.
func (m *DbPackageMutation) Where(ps ...predicate.DbPackage) {
m.predicates = append(m.predicates, ps...)
@ -1277,7 +1327,7 @@ func (m *DbPackageMutation) Type() string {
// order to get all numeric fields that were incremented/decremented, call
// AddedFields().
func (m *DbPackageMutation) Fields() []string {
fields := make([]string, 0, 21)
fields := make([]string, 0, 22)
if m.pkgbase != nil {
fields = append(fields, dbpackage.FieldPkgbase)
}
@ -1341,6 +1391,9 @@ func (m *DbPackageMutation) Fields() []string {
if m.srcinfo != nil {
fields = append(fields, dbpackage.FieldSrcinfo)
}
if m.srcinfo_hash != nil {
fields = append(fields, dbpackage.FieldSrcinfoHash)
}
return fields
}
@ -1391,6 +1444,8 @@ func (m *DbPackageMutation) Field(name string) (ent.Value, bool) {
return m.IoOut()
case dbpackage.FieldSrcinfo:
return m.Srcinfo()
case dbpackage.FieldSrcinfoHash:
return m.SrcinfoHash()
}
return nil, false
}
@ -1442,6 +1497,8 @@ func (m *DbPackageMutation) OldField(ctx context.Context, name string) (ent.Valu
return m.OldIoOut(ctx)
case dbpackage.FieldSrcinfo:
return m.OldSrcinfo(ctx)
case dbpackage.FieldSrcinfoHash:
return m.OldSrcinfoHash(ctx)
}
return nil, fmt.Errorf("unknown DbPackage field %s", name)
}
@ -1598,6 +1655,13 @@ func (m *DbPackageMutation) SetField(name string, value ent.Value) error {
}
m.SetSrcinfo(v)
return nil
case dbpackage.FieldSrcinfoHash:
v, ok := value.(string)
if !ok {
return fmt.Errorf("unexpected type %T for field %s", value, name)
}
m.SetSrcinfoHash(v)
return nil
}
return fmt.Errorf("unknown DbPackage field %s", name)
}
@ -1745,6 +1809,9 @@ func (m *DbPackageMutation) ClearedFields() []string {
if m.FieldCleared(dbpackage.FieldSrcinfo) {
fields = append(fields, dbpackage.FieldSrcinfo)
}
if m.FieldCleared(dbpackage.FieldSrcinfoHash) {
fields = append(fields, dbpackage.FieldSrcinfoHash)
}
return fields
}
@ -1813,6 +1880,9 @@ func (m *DbPackageMutation) ClearField(name string) error {
case dbpackage.FieldSrcinfo:
m.ClearSrcinfo()
return nil
case dbpackage.FieldSrcinfoHash:
m.ClearSrcinfoHash()
return nil
}
return fmt.Errorf("unknown DbPackage nullable field %s", name)
}
@ -1884,6 +1954,9 @@ func (m *DbPackageMutation) ResetField(name string) error {
case dbpackage.FieldSrcinfo:
m.ResetSrcinfo()
return nil
case dbpackage.FieldSrcinfoHash:
m.ResetSrcinfoHash()
return nil
}
return fmt.Errorf("unknown DbPackage field %s", name)
}

1
ent/schema/dbpackage.go
View File

@ -34,6 +34,7 @@ func (DbPackage) Fields() []ent.Field {
field.Int64("io_in").Optional().Nillable(),
field.Int64("io_out").Optional().Nillable(),
field.Text("srcinfo").Optional().Nillable(),
field.String("srcinfo_hash").Optional(),
}
}

80
main.go
View File

@ -21,7 +21,6 @@ import (
"os/exec"
"os/signal"
"path/filepath"
"runtime"
"strings"
"sync"
"syscall"
@ -319,85 +318,6 @@ func (b *BuildManager) repoWorker(repo string) {
}
}
func (b *BuildManager) refreshSRCINFOs(ctx context.Context, path string) error {
pkgBuilds, err := Glob(path)
if err != nil {
return fmt.Errorf("error scanning for PKGBUILDs: %w", err)
}
step := int(float32(len(pkgBuilds)) / float32(runtime.NumCPU()))
cur := 0
wg := sync.WaitGroup{}
for i := 0; i < runtime.NumCPU(); i++ {
if i == runtime.NumCPU()-1 {
step = len(pkgBuilds) - cur
}
wg.Add(1)
go func(pkgBuilds []string) {
defer wg.Done()
for _, pkgbuild := range pkgBuilds {
mPkgbuild := PKGBUILD(pkgbuild)
if mPkgbuild.FullRepo() == "trunk" || !Contains(conf.Repos, mPkgbuild.Repo()) || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) {
continue
}
for _, march := range conf.March {
dbPkg, dbErr := db.DbPackage.Query().Where(
dbpackage.And(
dbpackage.Pkgbase(mPkgbuild.PkgBase()),
dbpackage.RepositoryEQ(dbpackage.Repository(mPkgbuild.Repo())),
dbpackage.March(march),
),
).Only(context.Background())
if ent.IsNotFound(dbErr) {
log.Debugf("[%s/%s] Package not found in database", mPkgbuild.Repo(), mPkgbuild.PkgBase())
} else if err != nil {
log.Errorf("[%s/%s] Problem querying db for package: %v", mPkgbuild.Repo(), mPkgbuild.PkgBase(), dbErr)
}
// compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs
// reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly
b3s, err := b3sum(pkgbuild)
if err != nil {
log.Fatalf("Error hashing PKGBUILD: %v", err)
}
if dbPkg != nil && b3s == dbPkg.Hash {
log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", mPkgbuild.Repo(), mPkgbuild.PkgBase(), b3s)
continue
} else if dbPkg != nil && b3s != dbPkg.Hash {
log.Debugf("[%s/%s] srcinfo cleared", mPkgbuild.Repo(), mPkgbuild.PkgBase())
dbPkg = dbPkg.Update().ClearSrcinfo().SaveX(context.Background())
}
proto := &ProtoPackage{
Pkgbuild: pkgbuild,
Pkgbase: mPkgbuild.PkgBase(),
Repo: dbpackage.Repository(mPkgbuild.Repo()),
March: march,
FullRepo: mPkgbuild.Repo() + "-" + march,
Hash: b3s,
DbPackage: dbPkg,
}
_, err = proto.isEligible(ctx)
if err != nil {
log.Infof("Unable to determine status for package %s: %v", proto.Pkgbase, err)
b.repoPurge[proto.FullRepo] <- []*ProtoPackage{proto}
}
}
}
}(pkgBuilds[cur : cur+step])
cur += step
}
wg.Wait()
return nil
}
func (b *BuildManager) syncWorker(ctx context.Context) error {
err := os.MkdirAll(filepath.Join(conf.Basedir.Work, upstreamDir), 0755)
if err != nil {

2
proto_package.go
View File

@ -715,7 +715,7 @@ func (p *ProtoPackage) genSrcinfo() error {
p.Srcinfo = info
if p.DbPackage != nil {
p.DbPackage = p.DbPackage.Update().SetSrcinfo(string(res)).SaveX(context.Background())
p.DbPackage = p.DbPackage.Update().SetSrcinfoHash(p.Hash).SetSrcinfo(string(res)).SaveX(context.Background())
}
return nil

80
utils.go
View File

@ -19,6 +19,7 @@ import (
"os/exec"
"path/filepath"
"regexp"
"runtime"
"sort"
"strconv"
"strings"
@ -117,6 +118,85 @@ func updateLastUpdated() error {
return nil
}
func (b *BuildManager) refreshSRCINFOs(ctx context.Context, path string) error {
pkgBuilds, err := Glob(path)
if err != nil {
return fmt.Errorf("error scanning for PKGBUILDs: %w", err)
}
step := int(float32(len(pkgBuilds)) / float32(runtime.NumCPU()))
cur := 0
wg := sync.WaitGroup{}
for i := 0; i < runtime.NumCPU(); i++ {
if i == runtime.NumCPU()-1 {
step = len(pkgBuilds) - cur
}
wg.Add(1)
go func(pkgBuilds []string) {
defer wg.Done()
for _, pkgbuild := range pkgBuilds {
mPkgbuild := PKGBUILD(pkgbuild)
if mPkgbuild.FullRepo() == "trunk" || !Contains(conf.Repos, mPkgbuild.Repo()) || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) {
continue
}
for _, march := range conf.March {
dbPkg, dbErr := db.DbPackage.Query().Where(
dbpackage.And(
dbpackage.Pkgbase(mPkgbuild.PkgBase()),
dbpackage.RepositoryEQ(dbpackage.Repository(mPkgbuild.Repo())),
dbpackage.March(march),
),
).Only(context.Background())
if ent.IsNotFound(dbErr) {
log.Debugf("[%s/%s] Package not found in database", mPkgbuild.Repo(), mPkgbuild.PkgBase())
} else if err != nil {
log.Errorf("[%s/%s] Problem querying db for package: %v", mPkgbuild.Repo(), mPkgbuild.PkgBase(), dbErr)
}
// compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs
// reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly
b3s, err := b3sum(pkgbuild)
if err != nil {
log.Fatalf("Error hashing PKGBUILD: %v", err)
}
if dbPkg != nil && b3s == dbPkg.Hash {
log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", mPkgbuild.Repo(), mPkgbuild.PkgBase(), b3s)
continue
} else if dbPkg != nil && b3s != dbPkg.Hash && dbPkg.SrcinfoHash != b3s {
log.Debugf("[%s/%s] srcinfo cleared", mPkgbuild.Repo(), mPkgbuild.PkgBase())
dbPkg = dbPkg.Update().ClearSrcinfo().SaveX(context.Background())
}
proto := &ProtoPackage{
Pkgbuild: pkgbuild,
Pkgbase: mPkgbuild.PkgBase(),
Repo: dbpackage.Repository(mPkgbuild.Repo()),
March: march,
FullRepo: mPkgbuild.Repo() + "-" + march,
Hash: b3s,
DbPackage: dbPkg,
}
_, err = proto.isEligible(ctx)
if err != nil {
log.Infof("Unable to determine status for package %s: %v", proto.Pkgbase, err)
b.repoPurge[proto.FullRepo] <- []*ProtoPackage{proto}
}
}
}
}(pkgBuilds[cur : cur+step])
cur += step
}
wg.Wait()
return nil
}
func statusId2string(s dbpackage.Status) string {
switch s {
case dbpackage.StatusSkipped: