Hash each PKGBUILD and compare before parsing, fixes #25

This will speed things up significantly. See #25 for more information and discussion.
This commit is contained in:
Giovanni Harting 2021-08-30 11:02:06 +02:00
parent 1977181409
commit b78b09aeaa
12 changed files with 375 additions and 28 deletions

View File

@ -39,6 +39,8 @@ type DbPackage struct {
BuildDuration uint64 `json:"build_duration,omitempty"`
// Updated holds the value of the "updated" field.
Updated time.Time `json:"updated,omitempty"`
// Hash holds the value of the "hash" field.
Hash string `json:"hash,omitempty"`
}
// scanValues returns the types for scanning values from sql.Rows.
@ -50,7 +52,7 @@ func (*DbPackage) scanValues(columns []string) ([]interface{}, error) {
values[i] = new([]byte)
case dbpackage.FieldID, dbpackage.FieldStatus, dbpackage.FieldBuildDuration:
values[i] = new(sql.NullInt64)
case dbpackage.FieldPkgbase, dbpackage.FieldSkipReason, dbpackage.FieldRepository, dbpackage.FieldMarch, dbpackage.FieldVersion, dbpackage.FieldRepoVersion:
case dbpackage.FieldPkgbase, dbpackage.FieldSkipReason, dbpackage.FieldRepository, dbpackage.FieldMarch, dbpackage.FieldVersion, dbpackage.FieldRepoVersion, dbpackage.FieldHash:
values[i] = new(sql.NullString)
case dbpackage.FieldBuildTime, dbpackage.FieldUpdated:
values[i] = new(sql.NullTime)
@ -144,6 +146,12 @@ func (dp *DbPackage) assignValues(columns []string, values []interface{}) error
} else if value.Valid {
dp.Updated = value.Time
}
case dbpackage.FieldHash:
if value, ok := values[i].(*sql.NullString); !ok {
return fmt.Errorf("unexpected type %T for field hash", values[i])
} else if value.Valid {
dp.Hash = value.String
}
}
}
return nil
@ -194,6 +202,8 @@ func (dp *DbPackage) String() string {
builder.WriteString(fmt.Sprintf("%v", dp.BuildDuration))
builder.WriteString(", updated=")
builder.WriteString(dp.Updated.Format(time.ANSIC))
builder.WriteString(", hash=")
builder.WriteString(dp.Hash)
builder.WriteByte(')')
return builder.String()
}

View File

@ -29,6 +29,8 @@ const (
FieldBuildDuration = "build_duration"
// FieldUpdated holds the string denoting the updated field in the database.
FieldUpdated = "updated"
// FieldHash holds the string denoting the hash field in the database.
FieldHash = "hash"
// Table holds the table name of the dbpackage in the database.
Table = "db_packages"
)
@ -47,6 +49,7 @@ var Columns = []string{
FieldBuildTime,
FieldBuildDuration,
FieldUpdated,
FieldHash,
}
// ValidColumn reports if the column name is valid (part of the table columns).

View File

@ -162,6 +162,13 @@ func Updated(v time.Time) predicate.DbPackage {
})
}
// Hash applies equality check predicate on the "hash" field. It's identical to HashEQ.
func Hash(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.EQ(s.C(FieldHash), v))
})
}
// PkgbaseEQ applies the EQ predicate on the "pkgbase" field.
func PkgbaseEQ(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
@ -1230,6 +1237,131 @@ func UpdatedNotNil() predicate.DbPackage {
})
}
// HashEQ applies the EQ predicate on the "hash" field.
func HashEQ(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.EQ(s.C(FieldHash), v))
})
}
// HashNEQ applies the NEQ predicate on the "hash" field.
func HashNEQ(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.NEQ(s.C(FieldHash), v))
})
}
// HashIn applies the In predicate on the "hash" field.
func HashIn(vs ...string) predicate.DbPackage {
v := make([]interface{}, len(vs))
for i := range v {
v[i] = vs[i]
}
return predicate.DbPackage(func(s *sql.Selector) {
// if not arguments were provided, append the FALSE constants,
// since we can't apply "IN ()". This will make this predicate falsy.
if len(v) == 0 {
s.Where(sql.False())
return
}
s.Where(sql.In(s.C(FieldHash), v...))
})
}
// HashNotIn applies the NotIn predicate on the "hash" field.
func HashNotIn(vs ...string) predicate.DbPackage {
v := make([]interface{}, len(vs))
for i := range v {
v[i] = vs[i]
}
return predicate.DbPackage(func(s *sql.Selector) {
// if not arguments were provided, append the FALSE constants,
// since we can't apply "IN ()". This will make this predicate falsy.
if len(v) == 0 {
s.Where(sql.False())
return
}
s.Where(sql.NotIn(s.C(FieldHash), v...))
})
}
// HashGT applies the GT predicate on the "hash" field.
func HashGT(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.GT(s.C(FieldHash), v))
})
}
// HashGTE applies the GTE predicate on the "hash" field.
func HashGTE(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.GTE(s.C(FieldHash), v))
})
}
// HashLT applies the LT predicate on the "hash" field.
func HashLT(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.LT(s.C(FieldHash), v))
})
}
// HashLTE applies the LTE predicate on the "hash" field.
func HashLTE(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.LTE(s.C(FieldHash), v))
})
}
// HashContains applies the Contains predicate on the "hash" field.
func HashContains(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.Contains(s.C(FieldHash), v))
})
}
// HashHasPrefix applies the HasPrefix predicate on the "hash" field.
func HashHasPrefix(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.HasPrefix(s.C(FieldHash), v))
})
}
// HashHasSuffix applies the HasSuffix predicate on the "hash" field.
func HashHasSuffix(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.HasSuffix(s.C(FieldHash), v))
})
}
// HashIsNil applies the IsNil predicate on the "hash" field.
func HashIsNil() predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.IsNull(s.C(FieldHash)))
})
}
// HashNotNil applies the NotNil predicate on the "hash" field.
func HashNotNil() predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.NotNull(s.C(FieldHash)))
})
}
// HashEqualFold applies the EqualFold predicate on the "hash" field.
func HashEqualFold(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.EqualFold(s.C(FieldHash), v))
})
}
// HashContainsFold applies the ContainsFold predicate on the "hash" field.
func HashContainsFold(v string) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {
s.Where(sql.ContainsFold(s.C(FieldHash), v))
})
}
// And groups predicates with the AND operator between them.
func And(predicates ...predicate.DbPackage) predicate.DbPackage {
return predicate.DbPackage(func(s *sql.Selector) {

View File

@ -142,6 +142,20 @@ func (dpc *DbPackageCreate) SetNillableUpdated(t *time.Time) *DbPackageCreate {
return dpc
}
// SetHash sets the "hash" field.
func (dpc *DbPackageCreate) SetHash(s string) *DbPackageCreate {
dpc.mutation.SetHash(s)
return dpc
}
// SetNillableHash sets the "hash" field if the given value is not nil.
func (dpc *DbPackageCreate) SetNillableHash(s *string) *DbPackageCreate {
if s != nil {
dpc.SetHash(*s)
}
return dpc
}
// Mutation returns the DbPackageMutation object of the builder.
func (dpc *DbPackageCreate) Mutation() *DbPackageMutation {
return dpc.mutation
@ -354,6 +368,14 @@ func (dpc *DbPackageCreate) createSpec() (*DbPackage, *sqlgraph.CreateSpec) {
})
_node.Updated = value
}
if value, ok := dpc.mutation.Hash(); ok {
_spec.Fields = append(_spec.Fields, &sqlgraph.FieldSpec{
Type: field.TypeString,
Value: value,
Column: dbpackage.FieldHash,
})
_node.Hash = value
}
return _node, _spec
}

View File

@ -199,6 +199,26 @@ func (dpu *DbPackageUpdate) ClearUpdated() *DbPackageUpdate {
return dpu
}
// SetHash sets the "hash" field.
func (dpu *DbPackageUpdate) SetHash(s string) *DbPackageUpdate {
dpu.mutation.SetHash(s)
return dpu
}
// SetNillableHash sets the "hash" field if the given value is not nil.
func (dpu *DbPackageUpdate) SetNillableHash(s *string) *DbPackageUpdate {
if s != nil {
dpu.SetHash(*s)
}
return dpu
}
// ClearHash clears the value of the "hash" field.
func (dpu *DbPackageUpdate) ClearHash() *DbPackageUpdate {
dpu.mutation.ClearHash()
return dpu
}
// Mutation returns the DbPackageMutation object of the builder.
func (dpu *DbPackageUpdate) Mutation() *DbPackageMutation {
return dpu.mutation
@ -430,6 +450,19 @@ func (dpu *DbPackageUpdate) sqlSave(ctx context.Context) (n int, err error) {
Column: dbpackage.FieldUpdated,
})
}
if value, ok := dpu.mutation.Hash(); ok {
_spec.Fields.Set = append(_spec.Fields.Set, &sqlgraph.FieldSpec{
Type: field.TypeString,
Value: value,
Column: dbpackage.FieldHash,
})
}
if dpu.mutation.HashCleared() {
_spec.Fields.Clear = append(_spec.Fields.Clear, &sqlgraph.FieldSpec{
Type: field.TypeString,
Column: dbpackage.FieldHash,
})
}
if n, err = sqlgraph.UpdateNodes(ctx, dpu.driver, _spec); err != nil {
if _, ok := err.(*sqlgraph.NotFoundError); ok {
err = &NotFoundError{dbpackage.Label}
@ -621,6 +654,26 @@ func (dpuo *DbPackageUpdateOne) ClearUpdated() *DbPackageUpdateOne {
return dpuo
}
// SetHash sets the "hash" field.
func (dpuo *DbPackageUpdateOne) SetHash(s string) *DbPackageUpdateOne {
dpuo.mutation.SetHash(s)
return dpuo
}
// SetNillableHash sets the "hash" field if the given value is not nil.
func (dpuo *DbPackageUpdateOne) SetNillableHash(s *string) *DbPackageUpdateOne {
if s != nil {
dpuo.SetHash(*s)
}
return dpuo
}
// ClearHash clears the value of the "hash" field.
func (dpuo *DbPackageUpdateOne) ClearHash() *DbPackageUpdateOne {
dpuo.mutation.ClearHash()
return dpuo
}
// Mutation returns the DbPackageMutation object of the builder.
func (dpuo *DbPackageUpdateOne) Mutation() *DbPackageMutation {
return dpuo.mutation
@ -876,6 +929,19 @@ func (dpuo *DbPackageUpdateOne) sqlSave(ctx context.Context) (_node *DbPackage,
Column: dbpackage.FieldUpdated,
})
}
if value, ok := dpuo.mutation.Hash(); ok {
_spec.Fields.Set = append(_spec.Fields.Set, &sqlgraph.FieldSpec{
Type: field.TypeString,
Value: value,
Column: dbpackage.FieldHash,
})
}
if dpuo.mutation.HashCleared() {
_spec.Fields.Clear = append(_spec.Fields.Clear, &sqlgraph.FieldSpec{
Type: field.TypeString,
Column: dbpackage.FieldHash,
})
}
_node = &DbPackage{config: dpuo.config}
_spec.Assign = _node.assignValues
_spec.ScanValues = _node.scanValues

View File

@ -22,6 +22,7 @@ var (
{Name: "build_time", Type: field.TypeTime, Nullable: true},
{Name: "build_duration", Type: field.TypeUint64, Nullable: true},
{Name: "updated", Type: field.TypeTime, Nullable: true},
{Name: "hash", Type: field.TypeString, Nullable: true},
}
// DbPackagesTable holds the schema information for the "db_packages" table.
DbPackagesTable = &schema.Table{

View File

@ -45,6 +45,7 @@ type DbPackageMutation struct {
build_duration *uint64
addbuild_duration *uint64
updated *time.Time
hash *string
clearedFields map[string]struct{}
done bool
oldValue func(context.Context) (*DbPackage, error)
@ -658,6 +659,55 @@ func (m *DbPackageMutation) ResetUpdated() {
delete(m.clearedFields, dbpackage.FieldUpdated)
}
// SetHash sets the "hash" field.
func (m *DbPackageMutation) SetHash(s string) {
m.hash = &s
}
// Hash returns the value of the "hash" field in the mutation.
func (m *DbPackageMutation) Hash() (r string, exists bool) {
v := m.hash
if v == nil {
return
}
return *v, true
}
// OldHash returns the old "hash" field's value of the DbPackage entity.
// If the DbPackage object wasn't provided to the builder, the object is fetched from the database.
// An error is returned if the mutation operation is not UpdateOne, or the database query fails.
func (m *DbPackageMutation) OldHash(ctx context.Context) (v string, err error) {
if !m.op.Is(OpUpdateOne) {
return v, fmt.Errorf("OldHash is only allowed on UpdateOne operations")
}
if m.id == nil || m.oldValue == nil {
return v, fmt.Errorf("OldHash requires an ID field in the mutation")
}
oldValue, err := m.oldValue(ctx)
if err != nil {
return v, fmt.Errorf("querying old value for OldHash: %w", err)
}
return oldValue.Hash, nil
}
// ClearHash clears the value of the "hash" field.
func (m *DbPackageMutation) ClearHash() {
m.hash = nil
m.clearedFields[dbpackage.FieldHash] = struct{}{}
}
// HashCleared returns if the "hash" field was cleared in this mutation.
func (m *DbPackageMutation) HashCleared() bool {
_, ok := m.clearedFields[dbpackage.FieldHash]
return ok
}
// ResetHash resets all changes to the "hash" field.
func (m *DbPackageMutation) ResetHash() {
m.hash = nil
delete(m.clearedFields, dbpackage.FieldHash)
}
// Op returns the operation name.
func (m *DbPackageMutation) Op() Op {
return m.op
@ -672,7 +722,7 @@ func (m *DbPackageMutation) Type() string {
// order to get all numeric fields that were incremented/decremented, call
// AddedFields().
func (m *DbPackageMutation) Fields() []string {
fields := make([]string, 0, 11)
fields := make([]string, 0, 12)
if m.pkgbase != nil {
fields = append(fields, dbpackage.FieldPkgbase)
}
@ -706,6 +756,9 @@ func (m *DbPackageMutation) Fields() []string {
if m.updated != nil {
fields = append(fields, dbpackage.FieldUpdated)
}
if m.hash != nil {
fields = append(fields, dbpackage.FieldHash)
}
return fields
}
@ -736,6 +789,8 @@ func (m *DbPackageMutation) Field(name string) (ent.Value, bool) {
return m.BuildDuration()
case dbpackage.FieldUpdated:
return m.Updated()
case dbpackage.FieldHash:
return m.Hash()
}
return nil, false
}
@ -767,6 +822,8 @@ func (m *DbPackageMutation) OldField(ctx context.Context, name string) (ent.Valu
return m.OldBuildDuration(ctx)
case dbpackage.FieldUpdated:
return m.OldUpdated(ctx)
case dbpackage.FieldHash:
return m.OldHash(ctx)
}
return nil, fmt.Errorf("unknown DbPackage field %s", name)
}
@ -853,6 +910,13 @@ func (m *DbPackageMutation) SetField(name string, value ent.Value) error {
}
m.SetUpdated(v)
return nil
case dbpackage.FieldHash:
v, ok := value.(string)
if !ok {
return fmt.Errorf("unexpected type %T for field %s", value, name)
}
m.SetHash(v)
return nil
}
return fmt.Errorf("unknown DbPackage field %s", name)
}
@ -931,6 +995,9 @@ func (m *DbPackageMutation) ClearedFields() []string {
if m.FieldCleared(dbpackage.FieldUpdated) {
fields = append(fields, dbpackage.FieldUpdated)
}
if m.FieldCleared(dbpackage.FieldHash) {
fields = append(fields, dbpackage.FieldHash)
}
return fields
}
@ -966,6 +1033,9 @@ func (m *DbPackageMutation) ClearField(name string) error {
case dbpackage.FieldUpdated:
m.ClearUpdated()
return nil
case dbpackage.FieldHash:
m.ClearHash()
return nil
}
return fmt.Errorf("unknown DbPackage nullable field %s", name)
}
@ -1007,6 +1077,9 @@ func (m *DbPackageMutation) ResetField(name string) error {
case dbpackage.FieldUpdated:
m.ResetUpdated()
return nil
case dbpackage.FieldHash:
m.ResetHash()
return nil
}
return fmt.Errorf("unknown DbPackage field %s", name)
}

View File

@ -24,6 +24,7 @@ func (DbPackage) Fields() []ent.Field {
field.Time("build_time").Optional(),
field.Uint64("build_duration").Positive().Optional(),
field.Time("updated").Optional(),
field.String("hash").Optional(),
}
}

1
go.mod
View File

@ -12,4 +12,5 @@ require (
github.com/sirupsen/logrus v1.8.1
github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3
gopkg.in/yaml.v2 v2.4.0
lukechampine.com/blake3 v1.1.5
)

14
go.sum
View File

@ -62,6 +62,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4=
github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4=
github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@ -137,6 +138,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@ -148,6 +151,7 @@ github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
@ -168,6 +172,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@ -202,9 +207,11 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -220,8 +227,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3 h1:shC1HB1UogxN5Ech3Yqaaxj1X/P656PPCB4RbojIJqc=
github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3/go.mod h1:XCsSkdKK4gwBMNrOCZWww0pX6AOt+2gYc5Z6jBRrNVg=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@ -255,6 +260,7 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -325,10 +331,12 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@ -384,4 +392,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
lukechampine.com/blake3 v1.1.5 h1:hsACfxWvLdGmjYbWGrumQIphOvO+ZruZehWtgd2fxoM=
lukechampine.com/blake3 v1.1.5/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

57
main.go
View File

@ -11,10 +11,8 @@ import (
"github.com/Morganamilo/go-srcinfo"
_ "github.com/mattn/go-sqlite3"
log "github.com/sirupsen/logrus"
"github.com/wercker/journalhook"
"gopkg.in/yaml.v2"
"html/template"
"math/rand"
"os"
"os/exec"
"os/signal"
@ -222,13 +220,13 @@ func (b *BuildManager) parseWorker() {
if contains(info.Arch, "any") {
log.Debugf("Skipped %s: any-Package", info.Pkgbase)
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("arch = any").SaveX(context.Background())
dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("arch = any").SetHash(pkg.Hash).SaveX(context.Background())
dbLock.Unlock()
skipping = true
} else if contains(conf.Blacklist.Packages, info.Pkgbase) {
log.Debugf("Skipped %s: blacklisted package", info.Pkgbase)
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted").SaveX(context.Background())
dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted").SetHash(pkg.Hash).SaveX(context.Background())
dbLock.Unlock()
skipping = true
} else if contains(info.MakeDepends, "ghc") || contains(info.MakeDepends, "haskell-ghc") || contains(info.Depends, "ghc") || contains(info.Depends, "haskell-ghc") {
@ -237,13 +235,13 @@ func (b *BuildManager) parseWorker() {
// https://git.harting.dev/anonfunc/ALHP.GO/issues/11
log.Debugf("Skipped %s: haskell package", info.Pkgbase)
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted (haskell)").SaveX(context.Background())
dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted (haskell)").SetHash(pkg.Hash).SaveX(context.Background())
dbLock.Unlock()
skipping = true
} else if isPkgFailed(pkg) {
log.Debugf("Skipped %s: failed build", info.Pkgbase)
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(FAILED).SetSkipReason("").SaveX(context.Background())
dbPkg = dbPkg.Update().SetStatus(FAILED).SetSkipReason("").SetHash(pkg.Hash).SaveX(context.Background())
dbLock.Unlock()
skipping = true
}
@ -261,7 +259,7 @@ func (b *BuildManager) parseWorker() {
if repoVer != "" && alpm.VerCmp(repoVer, pkg.Version) > 0 {
log.Debugf("Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)", info.Pkgbase, pkg.Version, repoVer)
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SaveX(context.Background())
dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SetHash(pkg.Hash).SaveX(context.Background())
dbLock.Unlock()
b.parseWG.Done()
continue
@ -429,7 +427,7 @@ func (b *BuildManager) repoWorker(repo string) {
dbPkg := getDbPackage(pkg)
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SetRepoVersion(pkg.Version).SaveX(context.Background())
dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background())
dbLock.Unlock()
cmd = exec.Command("paccache",
@ -495,17 +493,6 @@ func (b *BuildManager) syncWorker() {
for {
b.buildWG.Wait()
/* TODO: Use `v` to print rudimentary stats
var v []struct {
Status int `json:"status"`
Count int `json:"count"`
}
dbLock.RLock()
db.DbPackage.Query().GroupBy(dbpackage.FieldStatus).Aggregate(ent.Count()).ScanX(context.Background(), &v)
dbLock.RUnlock()
*/
for gitDir, gitURL := range conf.Svn2git {
gitPath := filepath.Join(conf.Basedir.Upstream, gitDir)
@ -544,10 +531,6 @@ func (b *BuildManager) syncWorker() {
pkgBuilds, err := Glob(filepath.Join(conf.Basedir.Upstream, "/**/PKGBUILD"))
check(err)
// Shuffle pkgbuilds to spread out long-running builds, otherwise pkgBuilds is alphabetically-sorted
rand.Seed(time.Now().UnixNano())
rand.Shuffle(len(pkgBuilds), func(i, j int) { pkgBuilds[i], pkgBuilds[j] = pkgBuilds[j], pkgBuilds[i] })
for _, pkgbuild := range pkgBuilds {
if b.exit {
return
@ -560,6 +543,31 @@ func (b *BuildManager) syncWorker() {
continue
}
// compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs
// reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly
dbLock.RLock()
dbPkg, dbErr := db.DbPackage.Query().Where(dbpackage.And(dbpackage.Pkgbase(sPkgbuild[len(sPkgbuild)-4]), dbpackage.Repository(repo))).Only(context.Background())
dbLock.RUnlock()
if dbErr != nil {
switch dbErr.(type) {
case *ent.NotFoundError:
log.Debugf("[%s/%s] Package not found in database", repo, sPkgbuild[len(sPkgbuild)-4])
break
default:
log.Errorf("[%s/%s] Problem querying db for package: %v", repo, sPkgbuild[len(sPkgbuild)-4], dbErr)
}
}
b3s, err := b3sum(pkgbuild)
check(err)
if dbPkg != nil && b3s == dbPkg.Hash {
log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", repo, sPkgbuild[len(sPkgbuild)-4], b3s)
continue
}
// send to parse
for _, march := range conf.March {
b.parseWG.Add(1)
b.parse <- &BuildPackage{
@ -568,6 +576,7 @@ func (b *BuildManager) syncWorker() {
Repo: strings.Split(repo, "-")[0],
March: march,
FullRepo: strings.Split(repo, "-")[0] + "-" + march,
Hash: b3s,
}
}
}
@ -590,7 +599,7 @@ func main() {
lvl, err := log.ParseLevel(conf.Logging.Level)
check(err)
log.SetLevel(lvl)
journalhook.Enable()
// journalhook.Enable()
err = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 5)
if err != nil {

View File

@ -5,6 +5,7 @@ import (
"ALHP.go/ent/dbpackage"
"bufio"
"context"
"encoding/hex"
"fmt"
"github.com/Jguer/go-alpm/v2"
paconf "github.com/Morganamilo/go-pacmanconf"
@ -12,6 +13,7 @@ import (
log "github.com/sirupsen/logrus"
"io"
"io/fs"
"lukechampine.com/blake3"
"os"
"os/exec"
"path/filepath"
@ -44,6 +46,7 @@ type BuildPackage struct {
March string
FullRepo string
Version string
Hash string
}
type BuildManager struct {
@ -100,6 +103,22 @@ func check(e error) {
}
}
func b3sum(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer func(file *os.File) {
check(file.Close())
}(file)
hash := blake3.New(32, nil)
if _, err := io.Copy(hash, file); err != nil {
return "", err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
func containsSubStr(str string, subList []string) bool {
for _, checkStr := range subList {
if strings.Contains(str, checkStr) {