diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 066e7e2cda4bc..18985e04e9549 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1719,6 +1719,12 @@ PATH = ;RUN_AT_START = false ;; Notice if not success ;NO_SUCCESS_NOTICE = true +;; Limit the number of mirrors added to the queue to this number +;; (negative values mean no limit, 0 will result in no result in no mirrors being queued effectively disabling pull mirror updating.) +;PULL_LIMIT=50 +;; Limit the number of mirrors added to the queue to this number +;; (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating) +;PUSH_LIMIT=50 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 10ba325ddcc39..3e9b27961418f 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -828,6 +828,8 @@ NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take ef - `SCHEDULE`: **@every 10m**: Cron syntax for scheduling update mirrors, e.g. `@every 3h`. - `NO_SUCCESS_NOTICE`: **true**: The cron task for update mirrors success report is not very useful - as it just means that the mirrors have been queued. Therefore this is turned off by default. +- `PULL_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling pull mirror updating). +- `PUSH_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating). #### Cron - Repository Health Check (`cron.repo_health_check`) diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 35685b3220962..e28d0d25174d4 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -119,6 +119,7 @@ func MirrorsIterate(f func(idx int, bean interface{}) error) error { return db.GetEngine(db.DefaultContext). Where("next_update_unix<=?", time.Now().Unix()). And("next_update_unix!=0"). + OrderBy("updated_unix ASC"). Iterate(new(Mirror), f) } diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go index c6207bae6df69..38a1a66947f9d 100644 --- a/models/repo_pushmirror.go +++ b/models/repo_pushmirror.go @@ -107,5 +107,6 @@ func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { return db.GetEngine(db.DefaultContext). Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()). And("`interval` != 0"). + OrderBy("last_update ASC"). Iterate(new(PushMirror), f) } diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index 57fb399d4e19c..219173ccf0140 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -18,13 +18,24 @@ import ( ) func registerUpdateMirrorTask() { - RegisterTaskFatal("update_mirrors", &BaseConfig{ - Enabled: true, - RunAtStart: false, - Schedule: "@every 10m", - NoSuccessNotice: true, - }, func(ctx context.Context, _ *models.User, _ Config) error { - return mirror_service.Update(ctx) + type UpdateMirrorTaskConfig struct { + BaseConfig + PullLimit int + PushLimit int + } + + RegisterTaskFatal("update_mirrors", &UpdateMirrorTaskConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 10m", + NoSuccessNotice: true, + }, + PullLimit: 50, + PushLimit: 50, + }, func(ctx context.Context, _ *models.User, cfg Config) error { + umtc := cfg.(*UpdateMirrorTaskConfig) + return mirror_service.Update(ctx, umtc.PullLimit, umtc.PushLimit) }) } diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index eb37639beff8e..dae6f2807b6bf 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -45,15 +45,19 @@ func doMirrorSync(ctx context.Context, req *SyncRequest) { } } +var errLimit = fmt.Errorf("reached limit") + // Update checks and updates mirror repositories. -func Update(ctx context.Context) error { +func Update(ctx context.Context, pullLimit, pushLimit int) error { if !setting.Mirror.Enabled { log.Warn("Mirror feature disabled, but cron job enabled: skip update") return nil } log.Trace("Doing: Update") - handler := func(idx int, bean interface{}) error { + requested := 0 + + handler := func(idx int, bean interface{}, limit int) error { var item SyncRequest if m, ok := bean.(*models.Mirror); ok { if m.Repo == nil { @@ -78,21 +82,49 @@ func Update(ctx context.Context) error { return nil } + // Check we've not been cancelled select { case <-ctx.Done(): - return fmt.Errorf("Aborted") + return fmt.Errorf("aborted") default: - return mirrorQueue.Push(&item) } + + // Check if this request is already in the queue + has, err := mirrorQueue.Has(&item) + if err != nil { + return err + } + if has { + return nil + } + + // Push to the Queue + if err := mirrorQueue.Push(&item); err != nil { + return err + } + + requested++ + if limit > 0 && requested > limit { + return errLimit + } + return nil } - if err := models.MirrorsIterate(handler); err != nil { - log.Error("MirrorsIterate: %v", err) - return err + if pullLimit != 0 { + if err := models.MirrorsIterate(func(idx int, bean interface{}) error { + return handler(idx, bean, pullLimit) + }); err != nil && err != errLimit { + log.Error("MirrorsIterate: %v", err) + return err + } } - if err := models.PushMirrorsIterate(handler); err != nil { - log.Error("PushMirrorsIterate: %v", err) - return err + if pushLimit != 0 { + if err := models.PushMirrorsIterate(func(idx int, bean interface{}) error { + return handler(idx, bean, pushLimit) + }); err != nil && err != errLimit { + log.Error("PushMirrorsIterate: %v", err) + return err + } } log.Trace("Finished: Update") return nil