Merge branch 'develop' into config-validation

.github/workflows/dev.yml

@@ -122,15 +122,12 @@ jobs:
           push: true
           tags: semaphoreui/semaphore:develop
 
+      - name: Build and push runner
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./deployment/docker/prod/runner.buildx.Dockerfile
+          push: true
+          tags: semaphoreui/runner:develop
 
-  # test-docker:
-  #   runs-on: [ubuntu-latest]
-  #   steps:
-  #     - uses: actions/setup-go@v3
-  #       with: { go-version: 1.19 }
-
-  #     - run: go install github.com/go-task/task/v3/cmd/task@latest
-
-  #     - uses: actions/checkout@v3
-
-  #     - run: context=prod task docker:test

.github/workflows/release.yml

@@ -61,4 +61,13 @@ jobs:
           platforms: linux/amd64,linux/arm64
           file: ./deployment/docker/prod/buildx.Dockerfile
           push: true
-          tags: semaphoreui/semaphore:latest,semaphoreui/semaphore:${{ github.ref_name }}
+          tags: semaphoreui/semaphore:latest,semaphoreui/semaphore:${{ github.ref_name }}
+
+      - name: Build and push runner
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./deployment/docker/prod/runner.buildx.Dockerfile
+          push: true
+          tags: semaphoreui/runner:latest,semaphoreui/runner:${{ github.ref_name }}

api/projects/tasks.go

@@ -130,7 +130,15 @@ func StopTask(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err := helpers.TaskPool(r).StopTask(targetTask)
+	var stopObj struct {
+		Force bool `json:"force"`
+	}
+
+	if !helpers.Bind(w, r, &stopObj) {
+		return
+	}
+
+	err := helpers.TaskPool(r).StopTask(targetTask, stopObj.Force)
 	if err != nil {
 		helpers.WriteError(w, err)
 		return

api/runners/runners.go

@@ -51,7 +51,7 @@ func GetRunner(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 
-		if tsk.Task.Status == db.TaskRunningStatus {
+		if tsk.Task.Status == db.TaskStartingStatus {
 
 			data.NewJobs = append(data.NewJobs, runners.JobData{
 				Username:        tsk.Username,
@@ -64,13 +64,29 @@ func GetRunner(w http.ResponseWriter, r *http.Request) {
 			})
 
 			if tsk.Inventory.SSHKeyID != nil {
+				err := tsk.Inventory.SSHKey.DeserializeSecret()
+				if err != nil {
+					// TODO: return error
+				}
 				data.AccessKeys[*tsk.Inventory.SSHKeyID] = tsk.Inventory.SSHKey
 			}
 
 			if tsk.Inventory.BecomeKeyID != nil {
+				err := tsk.Inventory.BecomeKey.DeserializeSecret()
+				if err != nil {
+					// TODO: return error
+				}
 				data.AccessKeys[*tsk.Inventory.BecomeKeyID] = tsk.Inventory.BecomeKey
 			}
 
+			if tsk.Template.VaultKeyID != nil {
+				err := tsk.Template.VaultKey.DeserializeSecret()
+				if err != nil {
+					// TODO: return error
+				}
+				data.AccessKeys[*tsk.Template.VaultKeyID] = tsk.Template.VaultKey
+			}
+
 			data.AccessKeys[tsk.Repository.SSHKeyID] = tsk.Repository.SSHKey
 
 		} else {
@@ -112,6 +128,8 @@ func UpdateRunner(w http.ResponseWriter, r *http.Request) {
 		for _, logRecord := range job.LogRecords {
 			tsk.Log2(logRecord.Message, logRecord.Time)
 		}
+
+		tsk.SetStatus(job.Status)
 	}
 
 	w.WriteHeader(http.StatusNoContent)
@@ -135,7 +153,7 @@ func RegisterRunner(w http.ResponseWriter, r *http.Request) {
 	}
 
 	runner, err := helpers.Store(r).CreateRunner(db.Runner{
-		State: db.RunnerActive,
+		//State: db.RunnerActive,
 	})
 
 	if err != nil {

db/Migration.go

@@ -60,6 +60,7 @@ func GetMigrations() []Migration {
 		{Version: "2.8.57"},
 		{Version: "2.8.58"},
 		{Version: "2.8.91"},
+		{Version: "2.9.6"},
 	}
 }
 

db/Runner.go

@@ -2,14 +2,16 @@ package db
 
 type RunnerState string
 
-const (
-	RunnerOffline RunnerState = "offline"
-	RunnerActive  RunnerState = "active"
-)
+//const (
+//	RunnerOffline RunnerState = "offline"
+//	RunnerActive  RunnerState = "active"
+//)
 
 type Runner struct {
-	ID        int         `db:"id" json:"-"`
-	Token     string      `db:"token" json:"-"`
-	ProjectID *int        `db:"project_id" json:"project_id"`
-	State     RunnerState `db:"state" json:"state"`
+	ID        int    `db:"id" json:"-"`
+	Token     string `db:"token" json:"-"`
+	ProjectID *int   `db:"project_id" json:"project_id"`
+	//State            RunnerState `db:"state" json:"state"`
+	Webhook          string `db:"webhook" json:"webhook"`
+	MaxParallelTasks int    `db:"max_parallel_tasks" json:"max_parallel_tasks"`
 }

db/Store.go

@@ -314,7 +314,7 @@ var ViewProps = ObjectProps{
 	DefaultSortingColumn: "position",
 }
 
-var RunnerProps = ObjectProps{
+var GlobalRunnerProps = ObjectProps{
 	TableName:         "runner",
 	Type:              reflect.TypeOf(Runner{}),
 	PrimaryColumnName: "id",

db/Task.go

@@ -7,14 +7,19 @@ import (
 type TaskStatus string
 
 const (
-	TaskRunningStatus  TaskStatus = "running"
 	TaskWaitingStatus  TaskStatus = "waiting"
+	TaskStartingStatus TaskStatus = "starting"
+	TaskRunningStatus  TaskStatus = "running"
 	TaskStoppingStatus TaskStatus = "stopping"
 	TaskStoppedStatus  TaskStatus = "stopped"
 	TaskSuccessStatus  TaskStatus = "success"
 	TaskFailStatus     TaskStatus = "error"
 )
 
+func (s TaskStatus) IsFinished() bool {
+	return s == TaskStoppedStatus || s == TaskSuccessStatus || s == TaskFailStatus
+}
+
 // Task is a model of a task which will be executed by the runner
 type Task struct {
 	ID         int `db:"id" json:"id"`

db/bolt/runner.go

@@ -18,13 +18,13 @@ func (d *BoltDb) DeleteRunner(projectID int, runnerID int) (err error) {
 }
 
 func (d *BoltDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
-	err = d.getObject(0, db.RunnerProps, intObjectID(runnerID), &runner)
+	err = d.getObject(0, db.GlobalRunnerProps, intObjectID(runnerID), &runner)
 
 	return
 }
 
 func (d *BoltDb) GetGlobalRunners() (runners []db.Runner, err error) {
-	err = d.getObjects(0, db.RunnerProps, db.RetrieveQueryParams{}, nil, &runners)
+	err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, nil, &runners)
 	return
 }
 
@@ -39,7 +39,7 @@ func (d *BoltDb) UpdateRunner(runner db.Runner) (err error) {
 func (d *BoltDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
 	runner.Token = util.RandString(12)
 
-	res, err := d.createObject(0, db.RunnerProps, runner)
+	res, err := d.createObject(0, db.GlobalRunnerProps, runner)
 	if err != nil {
 		return
 	}

db/sql/SqlDb.go

@@ -211,8 +211,13 @@ func (d *SqlDb) getObject(projectID int, props db.ObjectProps, objectID int, obj
 
 func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
 	q := squirrel.Select("*").
-		From(props.TableName+" pe").
-		Where("pe.project_id=?", projectID)
+		From(props.TableName + " pe")
+
+	if props.IsGlobal {
+		q = q.Where("pe.project_id is null")
+	} else {
+		q = q.Where("pe.project_id=?", projectID)
+	}
 
 	orderDirection := "ASC"
 	if params.SortInverted {
@@ -240,11 +245,18 @@ func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.Retrie
 }
 
 func (d *SqlDb) deleteObject(projectID int, props db.ObjectProps, objectID int) error {
-	return validateMutationResult(
-		d.exec(
-			"delete from "+props.TableName+" where project_id=? and id=?",
-			projectID,
-			objectID))
+	if props.IsGlobal {
+		return validateMutationResult(
+			d.exec(
+				"delete from "+props.TableName+" where id=?",
+				objectID))
+	} else {
+		return validateMutationResult(
+			d.exec(
+				"delete from "+props.TableName+" where project_id=? and id=?",
+				projectID,
+				objectID))
+	}
 }
 
 func (d *SqlDb) Close(token string) {

db/sql/access_key.go

@@ -8,10 +8,6 @@ import (
 func (d *SqlDb) GetAccessKey(projectID int, accessKeyID int) (key db.AccessKey, err error) {
 	err = d.getObject(projectID, db.AccessKeyProps, accessKeyID, &key)
 
-	if err != nil {
-		return
-	}
-
 	return
 }
 

db/sql/migrations/v2.9.6.sql

@@ -0,0 +1,10 @@
+create table runner
+(
+    id                  integer primary key autoincrement,
+    project_id          int,
+    token               varchar(255) not null,
+    webhook             varchar(1000) not null default '',
+    max_parallel_tasks  int not null default 0,
+
+    foreign key (`project_id`) references project(`id`) on delete cascade
+);

db/sql/runner.go

@@ -1,6 +1,10 @@
 package sql
 
-import "github.com/ansible-semaphore/semaphore/db"
+import (
+	"encoding/base64"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/gorilla/securecookie"
+)
 
 func (d *SqlDb) GetRunner(projectID int, runnerID int) (runner db.Runner, err error) {
 	return
@@ -15,21 +19,47 @@ func (d *SqlDb) DeleteRunner(projectID int, runnerID int) (err error) {
 }
 
 func (d *SqlDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
+	err = d.getObject(0, db.GlobalRunnerProps, runnerID, &runner)
 	return
 }
 
 func (d *SqlDb) GetGlobalRunners() (runners []db.Runner, err error) {
+	err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, &runners)
 	return
 }
 
 func (d *SqlDb) DeleteGlobalRunner(runnerID int) (err error) {
+	err = d.deleteObject(0, db.GlobalRunnerProps, runnerID)
 	return
 }
 
 func (d *SqlDb) UpdateRunner(runner db.Runner) (err error) {
+	_, err = d.exec(
+		"update runner set webhook=?, max_parallel_tasks=? where id=?",
+		runner.Webhook,
+		runner.MaxParallelTasks,
+		runner.ID)
+
 	return
 }
 
 func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
+	token := base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
+
+	insertID, err := d.insert(
+		"id",
+		"insert into runner (project_id, token, webhook, max_parallel_tasks) values (?, ?, ?, ?)",
+		runner.ProjectID,
+		token,
+		runner.Webhook,
+		runner.MaxParallelTasks)
+
+	if err != nil {
+		return
+	}
+
+	newRunner = runner
+	newRunner.ID = insertID
+	newRunner.Token = token
 	return
 }

deployment/docker/ci/bin/install

@@ -9,6 +9,7 @@ EOF
 
 echo "--> Install Semaphore entrypoint wrapper script"
 cp ./deployment/docker/common/semaphore-wrapper /usr/local/bin/semaphore-wrapper
+cp ./deployment/docker/common/runner-wrapper /usr/local/bin/runner-wrapper
 task deps
 task compile
 task build:local GOOS= GOARCH=

deployment/docker/common/runner-wrapper

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -e
+
+echoerr() { printf "%s\n" "$*" >&2; }
+
+
+SEMAPHORE_CONFIG_PATH="${SEMAPHORE_CONFIG_PATH:-/etc/semaphore}"
+SEMAPHORE_RUNNER_CONFIG_FILE="${SEMAPHORE_RUNNER_CONFIG_FILE:-/var/lib/semaphore/runner.json}"
+SEMAPHORE_TMP_PATH="${SEMAPHORE_TMP_PATH:-/tmp/semaphore}"
+
+SEMAPHORE_REGISTRATION_TOKEN="${SEMAPHORE_REGISTRATION_TOKEN:-}"
+SEMAPHORE_API_URL="${SEMAPHORE_API_URL:-}"
+SEMAPHORE_RUNNER_ONE_OFF="${SEMAPHORE_RUNNER_ONE_OFF:-false}"
+
+# Create a config if it does not exist in the current config path
+if [ ! -f "${SEMAPHORE_CONFIG_PATH}/config.json" ]; then
+    echoerr "Generating ${SEMAPHORE_CONFIG_PATH}/config.json ..."
+
+    cat << EOF > "${SEMAPHORE_CONFIG_PATH}/config.json"
+{
+ 	"tmp_path": "${SEMAPHORE_TMP_PATH}",
+	"runner": {
+		"registration_token": "${SEMAPHORE_REGISTRATION_TOKEN}",
+		"config_file": "${SEMAPHORE_RUNNER_CONFIG_FILE}",
+		"api_url": "${SEMAPHORE_API_URL}",
+		"one_off": ${SEMAPHORE_RUNNER_ONE_OFF}
+	}
+}
+EOF
+
+    echoerr "Run Semaphore with semaphore runner --config ${SEMAPHORE_CONFIG_PATH}/config.json"
+fi
+
+# run our command
+exec "$@"

deployment/docker/dev/bin/install

@@ -11,5 +11,6 @@ EOF
 
 echo "--> Install Semaphore entrypoint wrapper script"
 mv ./deployment/docker/common/semaphore-wrapper /usr/local/bin/semaphore-wrapper
+mv ./deployment/docker/common/runner-wrapper /usr/local/bin/runner-wrapper
 task deps
 chmod -R 0777 /go