free parameters

Agapanthus · Dec 27, 2019 · 0323b4e · 0323b4e
1 parent 6e32add
commit 0323b4e
Show file tree

Hide file tree

Showing 11 changed files with 221 additions and 145 deletions.
diff --git a/conf.jl b/conf.jl
@@ -17,35 +17,8 @@ const DRAW_NIGHT_BACKGROUND = true
 # List of all commands and keywords to generate graphs for
 global QUERY = Array{QWidget,1}([
 
-    #=
-    QPlot(
-        "Demoplot",
-        days = 5,
-        data = [
-            QGroup(
-                "packets/s",
-                log = true,
-                data = [
-                    QStack([
-                        QStyle(Sysstat("wtps"), Dict("color" => :red)),
-                        QStyle(Sysstat("rtps"), Dict("color" => :royalblue))
-                    ]), 
-                    QStyle(Sysstat("tps"), Dict("color" => :navy))
-                ],
-            ),
-            QStyle(
-                QGroup(
-                    "blocks/s",
-                    data = [Sysstat("bread/s"), Sysstat("bwrtn/s")]
-                ),
-                Dict("fillrange" => 0, "fillalpha" => 0.2),
-            ),
-        ],
-    ),
-    =#
-
-
 
+    #=
 
     QPlot(
         "I/O",
@@ -77,8 +50,7 @@ global QUERY = Array{QWidget,1}([
     QPlot(
         "CPU",
         days = 3,
-        data = [QGroup(
-            "%",
+        data = [QGroup("%",
             min = 0,
             max = 100,
             data = [
@@ -93,6 +65,28 @@ global QUERY = Array{QWidget,1}([
         )],
     ),
 
+    =#
+
+    QPlot(
+        "Network",
+        days = 3,
+        data = [
+            QGroup("kB",
+                data= [
+                    QStack([
+                        Sysstat("rxpck/s"),
+                        Sysstat("txpck/s"),
+                    ]),
+                ]),
+            QGroup("%",
+                min = 0,
+                max = 100,
+                data = [
+                    Sysstat("%ifutil"),
+                ]),
+        ]),
+
+
     # TODO: Monthly view
 
 

diff --git a/src/ServerDiary.jl b/src/ServerDiary.jl
@@ -26,9 +26,9 @@ function writeDiary()
         # TODO: Debug
         #logException(_->begin
             local results = renderWidget(widget, today, joinpath(BASE_PATH, "stats", Dates.format(Dates.now(), "yyyy-mm-dd")) )
-            for (pngPath, header, description, specialization) in results
+            for (pngPath, titles, title, description) in results
                 path = pngPath
-                doc = appendGraph(doc, basename(pngPath), header, description, widget.title * specialization)
+                doc = appendGraph(doc, basename(pngPath), titles, string(description), title)
             end
         #end, "gettings sar data for $(widget.title)")
     end

diff --git a/src/output/html.jl b/src/output/html.jl
@@ -16,7 +16,6 @@ function appendGraph(doc::StatsDocument, path::String, titles::Array{Tuple{Strin
         doc.parts["overview"] *= "<img src=\"$path\" title=\"$title\" alt=\"$title\" height=\"$OVERVIEW_HEIGHT\" width=\"$OVERVIEW_WIDTH\">"
     end
 
-
     if title ∉ keys(doc.parts)
         doc.parts[title] = ""
         push!(doc.order, title)

diff --git a/src/providers/data.jl b/src/providers/data.jl
@@ -9,18 +9,17 @@ function fetchData!(store::DataStore, attribute::DataAttribute)
     throw("Unknown data provider $(typeof(attribute))")
 end
 
-function fetchData!(
+
+function fetchContext!(
     store::DataStore,
     attribute::Sysstat,
     from::DateTime,
     to::DateTime,
-)::Tuple{Array{DateTime,1},Array{Float64,1}, Tuple{String,String}}
-
-    @assert "$(typeof(attribute))" == "Sysstat"
+)::Dict{String, DataAttributeContext}
 
     local fqn = ("$(typeof(attribute))", attribute.property)
     if fqn in keys(store.data)
-        return store.data[fqn]["$(attribute.context)"]..., fqn
+        return Dict(k=>v[1] for (k,v) in store.data[fqn])
     end
 
     local cmd, keyword = getCommand(attribute)
@@ -40,28 +39,59 @@ function fetchData!(
         end
     end
 
+    local srcCtx = Dict{String, DataAttributeContext}()
+    local localCtx
     for i in 2:length(header)
         fqn = ("$(typeof(attribute))", header[i][1])
         store.data[fqn] = Dict()
         for data in datas
-            # TODO: Context is never used! Implement it! Reiterate until all elements of product exhausted!
             local values = pseudoColumn(data[2], i)
             if length(header[i][3]) == 0
                 values = convert(Array{Float64,1}, values)
             else
                 logger("", "Scipping text column $(header[i][1])", true)
                 continue
             end
-            store.data[fqn]["$(DataAttributeContext(Set()))"] =
-                (pseudoColumn(data[2], 1), values)
+
+            localCtx = DataAttributeContext(Set(map(
+                x -> ("$(typeof(attribute))", "$(x[1])", "$(x[3])"),
+                data[1],
+            )))
+            srcCtx["$localCtx"] = localCtx
+
+            store.data[fqn]["$localCtx"] =
+                (localCtx, pseudoColumn(data[2], 1), values)
         end
     end
 
-    fqn = ("$(typeof(attribute))", attribute.property)
+    return srcCtx
+end
+
+
+function fetchData!(
+    store::DataStore,
+    attribute::Sysstat,
+    ctx::DataAttributeContext,
+    from::DateTime,
+    to::DateTime,
+)::Tuple{Array{DateTime,1},Array{Float64,1},Tuple{String,String}}
+
+    @assert "$(typeof(attribute))" == "Sysstat"
+
+    local fqn = ("$(typeof(attribute))", attribute.property)
+
+    @assert fqn in keys(store.data)
+
+    local myData = store.data[fqn]["$ctx"]
 
-    return store.data[fqn]["$(attribute.context)"]..., fqn
+    myData = filter(x -> from <= x[1] <= to, collect(zip(myData[2], myData[3])))
+
+    return pseudoColumn(myData, 1), pseudoColumn(myData, 2), fqn
 end
 
 function getPoints(store::DataStore, from::DateTime, to::DateTime)
-    return filter( v->length(last(v)) > 0, Dict( key => filter(x -> from <= x <= to, value) for (key, value) in store.points))
+    return filter(
+        v -> length(last(v)) > 0,
+        Dict(key => filter(x -> from <= x <= to, value) for (key, value) in store.points),
+    )
 end
diff --git a/src/providers/sysstat.jl b/src/providers/sysstat.jl
@@ -17,7 +17,9 @@ function sarExe(cmd)
 end
 
 # calculates the cross product over the text-columns
-function recursiveCross(data, header, startAt = 2)
+function recursiveCross(data, header, startAt = 2)::Array{Tuple{Any, Array{Array{Any,1},1}}}
+    data = convert(Array{Array{Any,1}}, data)
+
     for i in startAt:length(header)
         length(header[i][3]) > 1 || continue
         local classes = Dict{String,Tuple{Int,String,Any}}()
@@ -107,6 +109,8 @@ function getSarDay(command::String, keyword::String, useDate::DateTime)
         x
     end, data)
 
+    @assert length(data) > 0
+
     # The dates are always increasing, but might continue on the next day. So fix this.
     local lastDate = data[1][1]
     for i in 1:length(data)
@@ -137,7 +141,7 @@ function getSarDay(command::String, keyword::String, useDate::DateTime)
     # filter duplicate title rows
     data = filter(x -> x[2:end] != header[2:end], data)
 
-    return description, header, data, myCommands, points
+    return description, header, convert(Array{Array{Any,1},1}, data) , myCommands, points
 end
 
 # extracts sar data for the given command and keyword for the last days
@@ -149,17 +153,22 @@ end
 # - description, what the command did
 function collectData(command::String, keyword::String, days::Int, today::DateTime)
     # Collect sar data for several days
-    local description, header, data, myCommands, points = getSarDay(command, keyword, today)
-    for day in 1:days-1
+    local description, header, myCommands = nothing, nothing, nothing
+    local data = []
+    local points = []
+    for day in 0:days-1
         logException(
             _ -> begin
                 local description2, header2, data2, myCommands2, points2 =
                     getSarDay(command, keyword, today - Dates.Day(day))
-                @assert header[2:end] == header2[2:end] "$header $header2"
-                @assert description == description2
-                @assert myCommands == myCommands2
+                @assert header == nothing || header[2:end] == header2[2:end] "$header $header2"
+                @assert description == nothing || description == description2
+                @assert myCommands == nothing || myCommands == myCommands2
                 data = [data2..., data...]
                 points = [points2..., points...]
+                description = description2
+                header = header2
+                myCommands = myCommands2
             end,
             "getting sar data from $day days before",
         )

diff --git a/src/structures.jl b/src/structures.jl
@@ -6,14 +6,14 @@ mutable struct QPlot <: QWidget
     title::String
     days::Int64
     data::Array{<:QueryStruct,1}
-    size::Union{Tuple{Int64,Int64}, Nothing}
+    size::Union{Tuple{Int64,Int64},Nothing}
 end
 # line plot of the given structure
 QPlot(
     title::String;
     days::Int64 = 3,
     data::Array{<:QueryStruct,1} = [],
-    size::Union{Tuple{Int64,Int64}, Nothing} = nothing,
+    size::Union{Tuple{Int64,Int64},Nothing} = nothing,
 ) = QPlot(title, days, data, size)
 
 mutable struct QStack <: QueryStruct
@@ -55,15 +55,15 @@ QGroup(
 ############ Providers
 
 struct DataAttributeContext
-    ctx::Set{Tuple{String,String}}
+    ctx::Set{Tuple{String,String,String}}
 end
 
 
 struct Sysstat <: DataAttribute
     property::String
-    context::DataAttributeContext
+    #context::DataAttributeContext
 end
-Sysstat(name::String) = Sysstat(name, DataAttributeContext(Set()))
+#Sysstat(name::String) = Sysstat(name, DataAttributeContext(Set()))
 
 struct Aws <: DataAttribute
     property::String
@@ -78,7 +78,7 @@ end
 using Dates
 
 mutable struct DataStore
-    data::Dict{Tuple{String,String},Dict{String,Tuple{Array{DateTime,1},Array{Float64,1}}}}
+    data::Dict{Tuple{String,String},Dict{String,Tuple{DataAttributeContext,Array{DateTime,1},Array{Float64,1}}}}
     points::Dict{Tuple{String,String},Array{DateTime,1}}
     descriptions::Dict{Tuple{String,String},String}
 end
@@ -105,13 +105,11 @@ mutable struct PlotContext
 end
 PlotContext(;
     plot = nothing,
-    palette = nothing,
+    palette = Palette(nothing, 1),
     styles::Dict{String,Any} = defaultStyles(),
     today::DateTime = Dates.now(),
     store::DataStore = DataStore(),
     maxDate::DateTime = Dates.now(),
     minDate::DateTime = Dates.now(),
-    group::Int = 0
+    group::Int = 0,
 ) = PlotContext(plot, palette, styles, today, store, maxDate, minDate, group)
-
-
diff --git a/src/util/util.jl b/src/util/util.jl
@@ -35,8 +35,7 @@ function logException(f, operation)
     end
 end
 
-
-function pseudoColumn(data::Array{Array{Any,1},1}, n::Int)::Array{Any,1}
+function pseudoColumn(data::Array{<:Any,1}, n::Int)::Array{Any,1}
     local column = []
     for d in data
         push!(column, d[n])

diff --git a/src/widgets/qAttributes.jl b/src/widgets/qAttributes.jl
@@ -1,12 +1,16 @@
 include("base.jl")
 include("../providers/data.jl")
 
-function analyze(attr::DataAttribute, ctx::PlotContext)
-    local dates, values, title = fetchData!(ctx.store, attr, ctx.minDate, ctx.maxDate)
-    # TODO: Multiply along text columns!
-    local srcCtx = []
-
-    return dates, values, srcCtx, minimum(values), maximum(values), [title]
+function analyze(attr::DataAttribute, ctx::PlotContext)       
+    local srcCtx = fetchContext!(ctx.store, attr, ctx.minDate, ctx.maxDate)
+    @assert length(srcCtx) > 0
+    for (k,lctx) in srcCtx
+        if "source context" in keys(ctx.styles)
+            lctx = ctx.styles["source context"] 
+        end
+        local dates, values, title = fetchData!(ctx.store, attr, lctx, ctx.minDate, ctx.maxDate)
+        return dates, values, srcCtx, minimum(values), maximum(values), [title]
+    end
 end
 
 import Base.+
@@ -24,9 +28,8 @@ end
 
 function renderWidget!(attr::DataAttribute, ctx::PlotContext)
 
-    local dates, values, srcCtx, _ = analyze(attr, ctx)
-    @assert length(srcCtx) == 0 "Ambiguos attribute."
-
+    local dates, values, _ = fetchData!(ctx.store, attr, ctx.styles["source context"], ctx.minDate, ctx.maxDate)
+
     local label = attr.property
     if "stacked" in keys(ctx.styles)
         label *= " (stacked)"

diff --git a/src/widgets/qGroups.jl b/src/widgets/qGroups.jl
@@ -3,7 +3,7 @@ include("base.jl")
 function analyze(widget::QGroup, ctx::PlotContext)
     local dates = Array{DateTime,1}()
     local values = nothing
-    local srcCtx = Array{String,1}()
+    local srcCtx = Dict{String, DataAttributeContext}()
     local mini = typemax(Float64)
     local maxi = typemin(Float64)
     local titles = Array{String,1}()
@@ -12,13 +12,11 @@ function analyze(widget::QGroup, ctx::PlotContext)
         local _dates, _values, _srcCtx, _mini, _maxi, _titles = analyze(attr, ctx)
 
         @assert length(_dates) == length(_values)
+        for (k,v) in _srcCtx
+            srcCtx[k] = v
+        end   
+        values = _values
         dates = _dates
-        for c in _srcCtx
-            push!(srcCtx, c)
-        end
-        if values === nothing
-            values = _values
-        end
         mini = min(mini, minimum(values), _mini)
         maxi = max(maxi, maximum(values), _maxi)
         titles = [titles..., _titles...]