Skip to content

Commit

Permalink
all schemas new
Browse files Browse the repository at this point in the history
  • Loading branch information
@iamlouk
iamlouk committed Dec 17, 2021
1 parent 8933366 commit 5403177
Show file tree
Hide file tree
Showing 20 changed files with 2,911 additions and 2,119 deletions.
7 changes: 2 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# ClusterCockpit with a Golang backend

__*DOES NOT WORK WITH CURRENT FRONTEND*__

[![Build](https://github.com/ClusterCockpit/cc-jobarchive/actions/workflows/test.yml/badge.svg)](https://github.com/ClusterCockpit/cc-jobarchive/actions/workflows/test.yml)

### Run server
Expand All @@ -11,11 +13,6 @@ git clone --recursive [email protected]:ClusterCockpit/cc-jobarchive.git
# Prepare frontend
cd ./cc-jobarchive/frontend
yarn install
export CCFRONTEND_ROLLUP_INTRO='
const JOBVIEW_URL = job => `/monitoring/job/${job.id}`;
const USERVIEW_URL = userId => `/monitoring/user/${userId}`;
const TAG_URL = tag => `/monitoring/jobs/?tag=${tag.id}`;
'
yarn build

cd ..
Expand Down
147 changes: 75 additions & 72 deletions api/rest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,15 @@ package api

import (
"context"
"database/sql"
"encoding/json"
"fmt"
"log"
"net/http"
"strings"

"github.com/ClusterCockpit/cc-jobarchive/config"
"github.com/ClusterCockpit/cc-jobarchive/graph"
"github.com/ClusterCockpit/cc-jobarchive/graph/model"
"github.com/ClusterCockpit/cc-jobarchive/metricdata"
"github.com/ClusterCockpit/cc-jobarchive/schema"
sq "github.com/Masterminds/squirrel"
"github.com/gorilla/mux"
"github.com/jmoiron/sqlx"
Expand All @@ -33,18 +31,6 @@ func (api *RestApi) MountRoutes(r *mux.Router) {
r.HandleFunc("/api/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch)
}

// TODO/FIXME: UPDATE API!
type StartJobApiRequest struct {
JobId int64 `json:"jobId"`
UserId string `json:"userId"`
ClusterId string `json:"clusterId"`
StartTime int64 `json:"startTime"`
MetaData string `json:"metaData"`
ProjectId string `json:"projectId"`
Nodes []string `json:"nodes"`
NodeList string `json:"nodeList"`
}

type StartJobApiRespone struct {
DBID int64 `json:"id"`
}
Expand All @@ -53,15 +39,12 @@ type StopJobApiRequest struct {
// JobId, ClusterId and StartTime are optional.
// They are only used if no database id was provided.
JobId *string `json:"jobId"`
ClusterId *string `json:"clusterId"`
Cluster *string `json:"clusterId"`
StartTime *int64 `json:"startTime"`

// Payload
StopTime int64 `json:"stopTime"`
}

type StopJobApiRespone struct {
DBID string `json:"id"`
StopTime int64 `json:"stopTime"`
State schema.JobState `json:"jobState"`
}

type TagJobApiRequest []*struct {
Expand Down Expand Up @@ -110,7 +93,7 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) {
}

for _, tag := range req {
var tagId string
var tagId int64
if err := sq.Select("id").From("tag").
Where("tag.tag_type = ?", tag.Type).Where("tag.tag_name = ?", tag.Name).
RunWith(api.DB).QueryRow().Scan(&tagId); err != nil {
Expand All @@ -123,10 +106,10 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) {
return
}

job.Tags = append(job.Tags, &model.JobTag{
ID: tagId,
TagType: tag.Type,
TagName: tag.Name,
job.Tags = append(job.Tags, &schema.Tag{
ID: tagId,
Type: tag.Type,
Name: tag.Name,
})
}

Expand All @@ -136,31 +119,25 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) {
}

func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) {
req := StartJobApiRequest{}
req := schema.JobMeta{BaseJob: schema.JobDefaults}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
return
}

if config.GetClusterConfig(req.ClusterId) == nil {
http.Error(rw, fmt.Sprintf("cluster '%s' does not exist", req.ClusterId), http.StatusBadRequest)
if config.GetClusterConfig(req.Cluster) == nil {
http.Error(rw, fmt.Sprintf("cluster '%s' does not exist", req.Cluster), http.StatusBadRequest)
return
}

if req.Nodes == nil {
req.Nodes = strings.Split(req.NodeList, "|")
if len(req.Nodes) == 1 {
req.Nodes = strings.Split(req.NodeList, ",")
}
}
if len(req.Nodes) == 0 || len(req.Nodes[0]) == 0 || len(req.UserId) == 0 {
if len(req.Resources) == 0 || len(req.User) == 0 || req.NumNodes == 0 {
http.Error(rw, "required fields are missing", http.StatusBadRequest)
return
}

// Check if combination of (job_id, cluster_id, start_time) already exists:
rows, err := api.DB.Query(`SELECT job.id FROM job WHERE job.job_id = ? AND job.cluster_id = ? AND job.start_time = ?`,
req.JobId, req.ClusterId, req.StartTime)
rows, err := api.DB.Query(`SELECT job.id FROM job WHERE job.job_id = ? AND job.cluster = ? AND job.start_time = ?`,
req.JobID, req.Cluster, req.StartTime)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
Expand All @@ -173,9 +150,12 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) {
return
}

res, err := api.DB.Exec(
`INSERT INTO job (job_id, user_id, project_id, cluster_id, start_time, duration, job_state, num_nodes, node_list, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`,
req.JobId, req.UserId, req.ProjectId, req.ClusterId, req.StartTime, 0, model.JobStateRunning, len(req.Nodes), strings.Join(req.Nodes, ","), req.MetaData)
req.RawResources, err = json.Marshal(req.Resources)
if err != nil {
log.Fatal(err)
}

res, err := api.DB.NamedExec(schema.JobInsertStmt, req)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
Expand All @@ -187,7 +167,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) {
return
}

log.Printf("new job (id: %d): clusterId=%s, jobId=%d, userId=%s, startTime=%d, nodes=%v\n", id, req.ClusterId, req.JobId, req.UserId, req.StartTime, req.Nodes)
log.Printf("new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d\n", id, req.Cluster, req.JobID, req.User, req.StartTime)
rw.Header().Add("Content-Type", "application/json")
rw.WriteHeader(http.StatusCreated)
json.NewEncoder(rw).Encode(StartJobApiRespone{
Expand All @@ -203,66 +183,89 @@ func (api *RestApi) stopJob(rw http.ResponseWriter, r *http.Request) {
}

var err error
var job *model.Job
var sql string
var args []interface{}
id, ok := mux.Vars(r)["id"]
if ok {
job, err = graph.ScanJob(sq.Select(graph.JobTableCols...).From("job").Where("job.id = ?", id).RunWith(api.DB).QueryRow())
sql, args, err = sq.Select(schema.JobColumns...).From("job").Where("job.id = ?", id).ToSql()
} else {
job, err = graph.ScanJob(sq.Select(graph.JobTableCols...).From("job").
sql, args, err = sq.Select(schema.JobColumns...).From("job").
Where("job.job_id = ?", req.JobId).
Where("job.cluster_id = ?", req.ClusterId).
Where("job.start_time = ?", req.StartTime).
RunWith(api.DB).QueryRow())
Where("job.cluster = ?", req.Cluster).
Where("job.start_time = ?", req.StartTime).ToSql()
}
if err != nil {
http.Error(rw, err.Error(), http.StatusNotFound)
http.Error(rw, err.Error(), http.StatusBadRequest)
return
}
job, err := schema.ScanJob(api.DB.QueryRowx(sql, args...))
if err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
return
}

if job == nil || job.StartTime.Unix() >= req.StopTime || job.State != model.JobStateRunning {
if job == nil || job.StartTime.Unix() >= req.StopTime || job.State != schema.JobStateRunning {
http.Error(rw, "stop_time must be larger than start_time and only running jobs can be stopped", http.StatusBadRequest)
return
}

doArchiving := func(job *model.Job, ctx context.Context) error {
job.Duration = int(req.StopTime - job.StartTime.Unix())
if req.State != "" && !req.State.Valid() {
http.Error(rw, fmt.Sprintf("invalid job state: '%s'", req.State), http.StatusBadRequest)
return
} else {
req.State = schema.JobStateCompleted
}

doArchiving := func(job *schema.Job, ctx context.Context) error {
job.Duration = int32(req.StopTime - job.StartTime.Unix())
jobMeta, err := metricdata.ArchiveJob(job, ctx)
if err != nil {
log.Printf("archiving job (id: %s) failed: %s\n", job.ID, err.Error())
log.Printf("archiving job (dbid: %d) failed: %s\n", job.ID, err.Error())
return err
}

getAvg := func(metric string) sql.NullFloat64 {
stats, ok := jobMeta.Statistics[metric]
if !ok {
return sql.NullFloat64{Valid: false}
stmt := sq.Update("job").
Set("job_state", req.State).
Set("duration", job.Duration).
Where("job.id = ?", job.ID)

for metric, stats := range jobMeta.Statistics {
switch metric {
case "flops_any":
stmt = stmt.Set("flops_any_avg", stats.Avg)
case "mem_used":
stmt = stmt.Set("mem_used_max", stats.Max)
case "mem_bw":
stmt = stmt.Set("mem_bw_avg", stats.Avg)
case "load":
stmt = stmt.Set("load_avg", stats.Avg)
case "net_bw":
stmt = stmt.Set("net_bw_avg", stats.Avg)
case "file_bw":
stmt = stmt.Set("file_bw_avg", stats.Avg)
}
return sql.NullFloat64{Valid: true, Float64: stats.Avg}
}

if _, err := api.DB.Exec(
`UPDATE job SET
job_state = ?, duration = ?,
flops_any_avg = ?, mem_bw_avg = ?, net_bw_avg = ?, file_bw_avg = ?, load_avg = ?
WHERE job.id = ?`,
model.JobStateCompleted, job.Duration,
getAvg("flops_any"), getAvg("mem_bw"), getAvg("net_bw"), getAvg("file_bw"), getAvg("load"),
job.ID); err != nil {
log.Printf("archiving job (id: %s) failed: %s\n", job.ID, err.Error())
sql, args, err := stmt.ToSql()
if err != nil {
log.Printf("archiving job (dbid: %d) failed: %s\n", job.ID, err.Error())
return err
}

if _, err := api.DB.Exec(sql, args...); err != nil {
log.Printf("archiving job (dbid: %d) failed: %s\n", job.ID, err.Error())
return err
}

log.Printf("job stopped and archived (id: %s)\n", job.ID)
log.Printf("job stopped and archived (dbid: %d)\n", job.ID)
return nil
}

log.Printf("archiving job... (id: %s): clusterId=%s, jobId=%d, userId=%s, startTime=%s\n", job.ID, job.Cluster, job.JobID, job.User, job.StartTime)
log.Printf("archiving job... (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%s\n", job.ID, job.Cluster, job.JobID, job.User, job.StartTime)
if api.AsyncArchiving {
rw.Header().Add("Content-Type", "application/json")
rw.WriteHeader(http.StatusOK)
json.NewEncoder(rw).Encode(StopJobApiRespone{
DBID: job.ID,
})
json.NewEncoder(rw).Encode(job)
go doArchiving(job, context.Background())
} else {
err := doArchiving(job, r.Context())
Expand Down
8 changes: 4 additions & 4 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ func Init(usersdb *sqlx.DB, authEnabled bool, uiConfig map[string]interface{}, j
cluster.FilterRanges.StartTime.To = time.Unix(0, 0)
}

if cluster.ClusterID != de.Name() {
return fmt.Errorf("the file '%s/cluster.json' contains the clusterId '%s'", de.Name(), cluster.ClusterID)
if cluster.Name != de.Name() {
return fmt.Errorf("the file '%s/cluster.json' contains the clusterId '%s'", de.Name(), cluster.Name)
}

Clusters = append(Clusters, &cluster)
Expand Down Expand Up @@ -149,7 +149,7 @@ func ServeConfig(rw http.ResponseWriter, r *http.Request) {

func GetClusterConfig(cluster string) *model.Cluster {
for _, c := range Clusters {
if c.ClusterID == cluster {
if c.Name == cluster {
return c
}
}
Expand All @@ -158,7 +158,7 @@ func GetClusterConfig(cluster string) *model.Cluster {

func GetMetricConfig(cluster, metric string) *model.MetricConfig {
for _, c := range Clusters {
if c.ClusterID == cluster {
if c.Name == cluster {
for _, m := range c.MetricConfig {
if m.Name == metric {
return m
Expand Down
28 changes: 13 additions & 15 deletions gqlgen.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,21 +55,19 @@ models:
- github.com/99designs/gqlgen/graphql.Int64
- github.com/99designs/gqlgen/graphql.Int32
Job:
model: "github.com/ClusterCockpit/cc-jobarchive/schema.Job"
fields:
Tags:
tags:
resolver: true
JobMetric:
model: "github.com/ClusterCockpit/cc-jobarchive/schema.JobMetric"
JobMetricSeries:
model: "github.com/ClusterCockpit/cc-jobarchive/schema.MetricSeries"
JobMetricStatistics:
model: "github.com/ClusterCockpit/cc-jobarchive/schema.MetricStatistics"
NullableFloat:
model: "github.com/ClusterCockpit/cc-jobarchive/schema.Float"
JobMetricScope:
model: "github.com/ClusterCockpit/cc-jobarchive/schema.MetricScope"
JobResource:
model: "github.com/ClusterCockpit/cc-jobarchive/schema.JobResource"
Accelerator:
model: "github.com/ClusterCockpit/cc-jobarchive/schema.Accelerator"
NullableFloat: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.Float" }
MetricScope: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.MetricScope" }
JobStatistics: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.JobStatistics" }
Tag: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.Tag" }
Resource: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.Resource" }
JobState: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.JobState" }
JobMetric: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.JobMetric" }
Series: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.Series" }
MetricStatistics: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.MetricStatistics" }
StatsSeries: { model: "github.com/ClusterCockpit/cc-jobarchive/schema.StatsSeries" }


Loading

0 comments on commit 5403177

Please sign in to comment.