Tidy up logging

tsdb/engine/tsm1/engine.go

@@ -109,6 +109,8 @@ func NewEngine(path string, walPath string, opt tsdb.EngineOptions) tsdb.Engine
 
 	e := &Engine{
 		path:              path,
+		logger:            log.New(os.Stderr, "[tsm1] ", log.LstdFlags),
+		logOutput:         os.Stderr,
 		measurementFields: make(map[string]*tsdb.MeasurementFields),
 
 		WAL:   w,
@@ -127,7 +129,6 @@ func NewEngine(path string, walPath string, opt tsdb.EngineOptions) tsdb.Engine
 		enableCompactionsOnOpen:       true,
 		stats: &EngineStatistics{},
 	}
-	e.SetLogOutput(os.Stderr)
 
 	return e
 }
@@ -302,13 +303,16 @@ func (e *Engine) Close() error {
 	return e.WAL.Close()
 }
 
-// SetLogOutput sets the logger used for all messages. It must not be called
-// after the Open method has been called.
+// SetLogOutput sets the logger used for all messages. It is safe for concurrent
+// use.
 func (e *Engine) SetLogOutput(w io.Writer) {
-	e.logger = log.New(w, "[tsm1] ", log.LstdFlags)
+	e.logger.SetOutput(w)
 	e.WAL.SetLogOutput(w)
 	e.FileStore.SetLogOutput(w)
+
+	e.mu.Lock()
 	e.logOutput = w
+	e.mu.Unlock()
 }
 
 // LoadMetadataIndex loads the shard metadata into memory.

tsdb/engine/tsm1/file_store.go

@@ -110,7 +110,8 @@ type FileStore struct {
 
 	files []TSMFile
 
-	Logger       *log.Logger
+	logger       *log.Logger
+	logOutput    io.Writer
 	traceLogging bool
 
 	stats *FileStoreStatistics
@@ -143,15 +144,20 @@ func NewFileStore(dir string) *FileStore {
 	return &FileStore{
 		dir:          dir,
 		lastModified: time.Now(),
-		Logger:       log.New(os.Stderr, "[filestore] ", log.LstdFlags),
+		logger:       log.New(os.Stderr, "[filestore] ", log.LstdFlags),
+		logOutput:    os.Stderr,
 		stats:        &FileStoreStatistics{},
 	}
 }
 
-// SetLogOutput sets the logger used for all messages. It must not be called
-// after the Open method has been called.
+// SetLogOutput sets the logger used for all messages. It is safe for concurrent
+// use.
 func (f *FileStore) SetLogOutput(w io.Writer) {
-	f.Logger = log.New(w, "[filestore] ", log.LstdFlags)
+	f.logger.SetOutput(w)
+
+	f.mu.Lock()
+	f.logOutput = w
+	f.mu.Unlock()
 }
 
 // FileStoreStatistics keeps statistics about the file store.
@@ -367,7 +373,7 @@ func (f *FileStore) Open() error {
 			start := time.Now()
 			df, err := NewTSMReader(file)
 			if f.traceLogging {
-				f.Logger.Printf("%s (#%d) opened in %v", file.Name(), idx, time.Now().Sub(start))
+				f.logger.Printf("%s (#%d) opened in %v", file.Name(), idx, time.Now().Sub(start))
 			}
 
 			if err != nil {

tsdb/engine/tsm1/wal.go

@@ -82,8 +82,8 @@ type WAL struct {
 	closing chan struct{}
 
 	// WALOutput is the writer used by the logger.
-	LogOutput io.Writer
 	logger    *log.Logger
+	logOutput io.Writer
 
 	// SegmentSize is the file size at which a segment file will be rotated
 	SegmentSize int
@@ -101,19 +101,23 @@ func NewWAL(path string) *WAL {
 		path: path,
 
 		// these options should be overriden by any options in the config
-		LogOutput:   os.Stderr,
 		SegmentSize: DefaultSegmentSize,
-		logger:      log.New(os.Stderr, "[tsm1wal] ", log.LstdFlags),
 		closing:     make(chan struct{}),
 		stats:       &WALStatistics{},
 		limiter:     limiter.NewFixed(defaultWaitingWALWrites),
+		logger:      log.New(os.Stderr, "[tsm1wal] ", log.LstdFlags),
+		logOutput:   os.Stderr,
 	}
 }
 
-// SetLogOutput sets the location that logs are written to. It must not be
-// called after the Open method has been called.
+// SetLogOutput sets the location that logs are written to. It is safe for
+// concurrent use.
 func (l *WAL) SetLogOutput(w io.Writer) {
-	l.logger = log.New(w, "[tsm1wal] ", log.LstdFlags)
+	l.logger.SetOutput(w)
+
+	l.mu.Lock()
+	l.logOutput = w
+	l.mu.Unlock()
 }
 
 // WALStatistics maintains statistics about the WAL.

tsdb/shard.go

@@ -100,9 +100,9 @@ type Shard struct {
 	statTags models.Tags
 
 	logger *log.Logger
+	// used by logger. Referenced so it can be passed down to new caches.
+	logOutput io.Writer
 
-	// The writer used by the logger.
-	LogOutput    io.Writer
 	EnableOnOpen bool
 }
 
@@ -128,22 +128,24 @@ func NewShard(id uint64, index *DatabaseIndex, path string, walPath string, opti
 		database:        db,
 		retentionPolicy: rp,
 
-		LogOutput:    os.Stderr,
+		logger:       log.New(os.Stderr, "[shard] ", log.LstdFlags),
+		logOutput:    os.Stderr,
 		EnableOnOpen: true,
 	}
-
-	s.SetLogOutput(os.Stderr)
 	return s
 }
 
-// SetLogOutput sets the writer to which log output will be written. It must
-// not be called after the Open method has been called.
+// SetLogOutput sets the writer to which log output will be written. It is safe
+// for concurrent use.
 func (s *Shard) SetLogOutput(w io.Writer) {
-	s.LogOutput = w
-	s.logger = log.New(w, "[shard] ", log.LstdFlags)
+	s.logger.SetOutput(w)
 	if err := s.ready(); err == nil {
 		s.engine.SetLogOutput(w)
 	}
+
+	s.mu.Lock()
+	s.logOutput = w
+	s.mu.Unlock()
 }
 
 // SetEnabled enables the shard for queries and write.  When disabled, all
@@ -215,7 +217,7 @@ func (s *Shard) Open() error {
 		}
 
 		// Set log output on the engine.
-		e.SetLogOutput(s.LogOutput)
+		e.SetLogOutput(s.logOutput)
 
 		// Disable compactions while loading the index
 		e.SetEnabled(false)

tsdb/store.go

@@ -60,14 +60,17 @@ func NewStore(path string) *Store {
 	}
 }
 
-// SetLogOutput sets the writer to which all logs are written. It must not be
-// called after Open is called.
+// SetLogOutput sets the writer to which all logs are written. It is safe for
+// concurrent use.
 func (s *Store) SetLogOutput(w io.Writer) {
-	s.Logger = log.New(w, "[store] ", log.LstdFlags)
-	s.logOutput = w
+	s.Logger.SetOutput(w)
 	for _, s := range s.shards {
 		s.SetLogOutput(w)
 	}
+
+	s.mu.Lock()
+	s.logOutput = w
+	s.mu.Unlock()
 }
 
 func (s *Store) Statistics(tags map[string]string) []models.Statistic {