manual gc every crawl

hydrionz · Sep 5, 2020 · f2275c7 · f2275c7
1 parent 5b59998
commit f2275c7
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 3 deletions.
diff --git a/internal/app/task.go b/internal/app/task.go
@@ -5,9 +5,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/zu1k/proxypool/internal/database"
-
 	"github.com/zu1k/proxypool/internal/cache"
+	"github.com/zu1k/proxypool/internal/database"
 	"github.com/zu1k/proxypool/pkg/provider"
 	"github.com/zu1k/proxypool/pkg/proxy"
 )

diff --git a/internal/cron/cron.go b/internal/cron/cron.go
@@ -1,6 +1,8 @@
 package cron
 
 import (
+	"runtime"
+
 	"github.com/jasonlvhit/gocron"
 	"github.com/zu1k/proxypool/internal/app"
 )
@@ -13,4 +15,6 @@ func Cron() {
 func crawlTask() {
 	_ = app.InitConfigAndGetters("")
 	app.CrawlGo()
+	app.Getters = nil
+	runtime.GC()
 }
diff --git a/internal/database/db.go b/internal/database/db.go
@@ -8,6 +8,7 @@ import (
 
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
 )
 
 var DB *gorm.DB
@@ -20,7 +21,9 @@ func connect() (err error) {
 	if url := os.Getenv("DATABASE_URL"); url != "" {
 		dsn = url
 	}
-	DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{})
+	DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
 	if err == nil {
 		fmt.Println("DB connect success: ", DB.Name())
 	}

diff --git a/internal/database/proxy.go b/internal/database/proxy.go
@@ -26,6 +26,9 @@ func InitTables() {
 }
 
 func SaveProxyList(pl proxy.ProxyList) {
+	if DB == nil {
+		return
+	}
 	proxies := make([]Proxy, pl.Len())
 	for i, p := range pl {
 		proxies[i] = Proxy{