Skip to content

Commit

Permalink
Merge pull request tuna#56 from tuna/dev
Browse files Browse the repository at this point in the history
Dev
  • Loading branch information
bigeagle authored Dec 9, 2016
2 parents 81a15e7 + 7601e57 commit 5474038
Show file tree
Hide file tree
Showing 10 changed files with 319 additions and 42 deletions.
10 changes: 9 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
sudo: required

language: go
go:
- 1.6
Expand All @@ -11,8 +13,14 @@ before_install:
os:
- linux

services:
- docker

before_script:
- sudo cgcreate -t travis -a travis -g memory:tunasync
- lssubsys -am
- sudo cgcreate -a $USER -t $USER -g cpu:tunasync
- sudo cgcreate -a $USER -t $USER -g memory:tunasync
- docker pull alpine

script:
- ./.testandcover.bash
Expand Down
10 changes: 9 additions & 1 deletion worker/base_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ type baseProvider struct {

cgroup *cgroupHook
zfs *zfsHook
hooks []jobHook
docker *dockerHook

hooks []jobHook
}

func (p *baseProvider) Name() string {
Expand Down Expand Up @@ -87,6 +89,8 @@ func (p *baseProvider) AddHook(hook jobHook) {
p.cgroup = v
case *zfsHook:
p.zfs = v
case *dockerHook:
p.docker = v
}
p.hooks = append(p.hooks, hook)
}
Expand All @@ -103,6 +107,10 @@ func (p *baseProvider) ZFS() *zfsHook {
return p.zfs
}

func (p *baseProvider) Docker() *dockerHook {
return p.docker
}

func (p *baseProvider) prepareLogFile() error {
if p.LogFile() == "/dev/null" {
p.cmd.SetLogFile(nil)
Expand Down
30 changes: 14 additions & 16 deletions worker/cgroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,35 +15,31 @@ import (
"github.com/codeskyblue/go-sh"
)

var cgSubsystem = "cpu"

type cgroupHook struct {
emptyHook
provider mirrorProvider
basePath string
baseGroup string
created bool
subsystem string
memLimit string
}

func initCgroup(basePath string) {
if _, err := os.Stat(filepath.Join(basePath, "memory")); err == nil {
cgSubsystem = "memory"
return
}
logger.Warning("Memory subsystem of cgroup not enabled, fallback to cpu")
}

func newCgroupHook(p mirrorProvider, basePath, baseGroup string) *cgroupHook {
func newCgroupHook(p mirrorProvider, basePath, baseGroup, subsystem, memLimit string) *cgroupHook {
if basePath == "" {
basePath = "/sys/fs/cgroup"
}
if baseGroup == "" {
baseGroup = "tunasync"
}
if subsystem == "" {
subsystem = "cpu"
}
return &cgroupHook{
provider: p,
basePath: basePath,
baseGroup: baseGroup,
subsystem: subsystem,
}
}

Expand All @@ -52,13 +48,15 @@ func (c *cgroupHook) preExec() error {
if err := sh.Command("cgcreate", "-g", c.Cgroup()).Run(); err != nil {
return err
}
if cgSubsystem != "memory" {
if c.subsystem != "memory" {
return nil
}
if c.provider.Type() == provRsync || c.provider.Type() == provTwoStageRsync {
if c.memLimit != "" {
gname := fmt.Sprintf("%s/%s", c.baseGroup, c.provider.Name())
return sh.Command(
"cgset", "-r", "memory.limit_in_bytes=512M", gname,
"cgset", "-r",
fmt.Sprintf("memory.limit_in_bytes=%s", c.memLimit),
gname,
).Run()
}
return nil
Expand All @@ -76,7 +74,7 @@ func (c *cgroupHook) postExec() error {

func (c *cgroupHook) Cgroup() string {
name := c.provider.Name()
return fmt.Sprintf("%s:%s/%s", cgSubsystem, c.baseGroup, name)
return fmt.Sprintf("%s:%s/%s", c.subsystem, c.baseGroup, name)
}

func (c *cgroupHook) killAll() error {
Expand All @@ -87,7 +85,7 @@ func (c *cgroupHook) killAll() error {

readTaskList := func() ([]int, error) {
taskList := []int{}
taskFile, err := os.Open(filepath.Join(c.basePath, cgSubsystem, c.baseGroup, name, "tasks"))
taskFile, err := os.Open(filepath.Join(c.basePath, c.subsystem, c.baseGroup, name, "tasks"))
if err != nil {
return taskList, err
}
Expand Down
18 changes: 12 additions & 6 deletions worker/cgroup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,14 @@ sleep 30
provider, err := newCmdProvider(c)
So(err, ShouldBeNil)

initCgroup("/sys/fs/cgroup")
cg := newCgroupHook(provider, "/sys/fs/cgroup", "tunasync")
cg := newCgroupHook(provider, "/sys/fs/cgroup", "tunasync", "cpu", "")
provider.AddHook(cg)

err = cg.preExec()
if err != nil {
logger.Errorf("Failed to create cgroup")
return
}
So(err, ShouldBeNil)

go func() {
Expand Down Expand Up @@ -129,12 +132,15 @@ sleep 30
provider, err := newRsyncProvider(c)
So(err, ShouldBeNil)

initCgroup("/sys/fs/cgroup")
cg := newCgroupHook(provider, "/sys/fs/cgroup", "tunasync")
cg := newCgroupHook(provider, "/sys/fs/cgroup", "tunasync", "cpu", "512M")
provider.AddHook(cg)

cg.preExec()
if cgSubsystem == "memory" {
err = cg.preExec()
if err != nil {
logger.Errorf("Failed to create cgroup")
return
}
if cg.subsystem == "memory" {
memoLimit, err := ioutil.ReadFile(filepath.Join(cg.basePath, "memory", cg.baseGroup, provider.Name(), "memory.limit_in_bytes"))
So(err, ShouldBeNil)
So(strings.Trim(string(memoLimit), "\n"), ShouldEqual, strconv.Itoa(512*1024*1024))
Expand Down
20 changes: 17 additions & 3 deletions worker/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type Config struct {
Server serverConfig `toml:"server"`
Cgroup cgroupConfig `toml:"cgroup"`
ZFS zfsConfig `toml:"zfs"`
Docker dockerConfig `toml:"docker"`
Include includeConfig `toml:"include"`
Mirrors []mirrorConfig `toml:"mirrors"`
}
Expand Down Expand Up @@ -69,9 +70,16 @@ type serverConfig struct {
}

type cgroupConfig struct {
Enable bool `toml:"enable"`
BasePath string `toml:"base_path"`
Group string `toml:"group"`
Enable bool `toml:"enable"`
BasePath string `toml:"base_path"`
Group string `toml:"group"`
Subsystem string `toml:"subsystem"`
}

type dockerConfig struct {
Enable bool `toml:"enable"`
Volumes []string `toml:"volumes"`
Options []string `toml:"options"`
}

type zfsConfig struct {
Expand Down Expand Up @@ -111,6 +119,12 @@ type mirrorConfig struct {
Username string `toml:"username"`
Password string `toml:"password"`
Stage1Profile string `toml:"stage1_profile"`

MemoryLimit string `toml:"memory_limit"`

DockerImage string `toml:"docker_image"`
DockerVolumes []string `toml:"docker_volumes"`
DockerOptions []string `toml:"docker_options"`
}

// LoadConfig loads configuration
Expand Down
98 changes: 98 additions & 0 deletions worker/docker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package worker

import (
"fmt"
"os"
)

type dockerHook struct {
emptyHook
provider mirrorProvider
image string
volumes []string
options []string
}

func newDockerHook(p mirrorProvider, gCfg dockerConfig, mCfg mirrorConfig) *dockerHook {
volumes := []string{}
volumes = append(volumes, gCfg.Volumes...)
volumes = append(volumes, mCfg.DockerVolumes...)

options := []string{}
options = append(options, gCfg.Options...)
options = append(options, mCfg.DockerOptions...)

return &dockerHook{
provider: p,
image: mCfg.DockerImage,
volumes: volumes,
options: options,
}
}

func (d *dockerHook) preExec() error {
p := d.provider
logFile := p.LogFile()
workingDir := p.WorkingDir()

if _, err := os.Stat(workingDir); os.IsNotExist(err) {
logger.Debugf("Making dir %s", workingDir)
if err = os.MkdirAll(workingDir, 0755); err != nil {
return fmt.Errorf("Error making dir %s: %s", workingDir, err.Error())
}
}

logFileNew := "/log_latest"
workingDirNew := "/data"

// Override workingDir
ctx := p.EnterContext()
ctx.Set(_WorkingDirKey, workingDirNew)
ctx.Set(_LogFileKey+":docker", logFileNew)
ctx.Set(
"volumes", []string{
fmt.Sprintf("%s:%s", logFile, logFileNew),
fmt.Sprintf("%s:%s", workingDir, workingDirNew),
},
)
return nil
}

func (d *dockerHook) postExec() error {
// sh.Command(
// "docker", "rm", "-f", d.Name(),
// ).Run()
d.provider.ExitContext()
return nil
}

// Volumes returns the configured volumes and
// runtime-needed volumes, including mirror dirs
// and log files
func (d *dockerHook) Volumes() []string {
vols := make([]string, len(d.volumes))
copy(vols, d.volumes)

p := d.provider
ctx := p.Context()
if ivs, ok := ctx.Get("volumes"); ok {
vs := ivs.([]string)
vols = append(vols, vs...)
}
return vols
}

func (d *dockerHook) LogFile() string {
p := d.provider
ctx := p.Context()
if iv, ok := ctx.Get(_LogFileKey + ":docker"); ok {
v := iv.(string)
return v
}
return p.LogFile()
}

func (d *dockerHook) Name() string {
p := d.provider
return "tunasync-job-" + p.Name()
}
Loading

0 comments on commit 5474038

Please sign in to comment.