html generator fix

README.md

@@ -1,5 +1,5 @@
-# server-diary
-generates very detailed e-mails with graphs and statistics on what was happening on you server during the last days
+# ServerDiary
+Generates very detailed e-mails with graphs and statistics on what was happening on you server during the last days
 
 ### WARNING: This is a prototype. Don't run it on your server yet!
 
@@ -12,19 +12,15 @@ sudo add-apt-repository ppa:jonathonf/julialang
 sudo apt-get update
 sudo apt install julia
 ```
+You also need to install pngquant to compress the pngs and send smaller emails: `sudo apt install pngquant`
 
-Optionally, you can also install pngquant to compress the pngs and send smaller emails: `sudo apt install pngquant`
-
-To install and build the dependencies of julia, run `julia install.jl`. If you want to run this package as cronjob, make sure to run the `install.jl` from the same user as the cron job.
+To install this package and its julia dependencies enter your Julia CLI and type `]` to enter the Pkg-manager. Now enter `add https://github.com/Agapanthus/ServerDiary.jl`.
 
 Currently, the only supported backend is `sysstat`. So make sure `sysstat` and `sar` are installed and properly configured.
 
 ## Usage
 
-Run `julia ServerDiary.jl`. It will generate a file `stats.email` which is a multipart-html-email with images. You can send it using `sendmail -i -t < stats.email`.
-
-Old graphs are archived in the `stats` folder. Feel free to delete them if they become too many.
-
-## Notes
+Run `julia ServerDiary/run.jl`. It will generate a file `stats.email` which is a multipart-html-email with images. You can send it using `sendmail -i -t < stats.email`.
+The script might take a minute to start because it imports the Plots.jl package.
 
-To delete Julia's precompiled cache, run `rm -rf ~/.julia/compiled`
+Old graphs are archived in the `stats` folder. Feel free to delete them if they become too many.

conf.jl

@@ -43,10 +43,13 @@ global QUERY = Array{QWidget,1}([
         ],
     ),
     =#
+
+
+
 
     QPlot(
         "I/O",
-        days = 4,
+        days = 3,
         data = [
             QGroup(
                 "packets/s",
@@ -71,10 +74,9 @@ global QUERY = Array{QWidget,1}([
         ],
     ),
 
-    #=
     QPlot(
         "CPU",
-        days = 2,
+        days = 3,
         data = [QGroup(
             "%",
             min = 0,
@@ -90,10 +92,6 @@ global QUERY = Array{QWidget,1}([
             ],
         )],
     ),
-    =#
-
-
-
 
     # TODO: Monthly view
 
@@ -147,7 +145,8 @@ const MAKE_OVERVIEW = true
 const OVERVIEW_WIDTH = DEFAULT_SIZE[1] ÷ 3
 const OVERVIEW_HEIGHT = DEFAULT_SIZE[2] ÷ 3
 
-
 # Sysstat Corruption detection
 const SYSSTAT_CORRUPTION_THRESHOLD = 1_000_000_000_000
-const SYSSTAT_CORRUPTION_MARGIN = 4
+const SYSSTAT_CORRUPTION_MARGIN = 4
+
+const USE_PNGQUANT = true

run.jl

@@ -0,0 +1,3 @@
+include("src/ServerDiary.jl")
+
+writeDiary()

src/ServerDiary.jl

@@ -18,20 +18,17 @@ function writeDiary()
     logger("", "Writing Diary...", true)
 
     local today = Dates.now()
-    local doc = HTML.StatsDocument()
+    local doc = StatsDocument()
     local args = getArguments()
     local path = ""
 
     for widget in QUERY
         # TODO: Debug
         #logException(_->begin
             local results = renderWidget(widget, today, joinpath(BASE_PATH, "stats", Dates.format(Dates.now(), "yyyy-mm-dd")) )
-            @show results
-
             for (pngPath, header, description, specialization) in results
-                @show pngPath, header, description
                 path = pngPath
-                doc = HTML.appendGraph(doc, basename(pngPath), header, description, widget.title * specialization)
+                doc = appendGraph(doc, basename(pngPath), header, description, widget.title * specialization)
             end
         #end, "gettings sar data for $(widget.title)")
     end

src/output/html.jl

@@ -7,7 +7,7 @@ mutable struct StatsDocument
 end
 StatsDocument() = StatsDocument(Dict(), [], "")
 
-function appendGraph(doc::StatsDocument, path::String, header, description::String, title::String)::StatsDocument
+function appendGraph(doc::StatsDocument, path::String, titles::Array{Tuple{String,String,String},1}, description::String, title::String)::StatsDocument
     if MAKE_OVERVIEW
         if "overview" ∉ keys(doc.parts)
             doc.parts["overview"] = ""
@@ -23,7 +23,7 @@ function appendGraph(doc::StatsDocument, path::String, header, description::Stri
     end
     doc.parts[title] *= "<p>" * replace("$description", "\n"=>"\n<br>") * 
             "</p><p><img src=\"$path\" title=\"$title\" alt=\"$title\"></p><p><ul>\n" *
-            reduce(*, map(x->"<li>$(x[1]) - $(x[2])</li>\n", header[2:end])) * "</ul></p>"
+            reduce(*, map(x->"<li><i><b>$(x[1])</b> ($(x[2]))</i> $(x[3])</li>\n", titles)) * "</ul></p>"
 
     return doc
 end

src/providers/data.jl

@@ -14,13 +14,13 @@ function fetchData!(
     attribute::Sysstat,
     from::DateTime,
     to::DateTime,
-)::Tuple{Array{DateTime,1},Array{Float64,1}}
+)::Tuple{Array{DateTime,1},Array{Float64,1}, Tuple{String,String}}
 
     @assert "$(typeof(attribute))" == "Sysstat"
 
     local fqn = ("$(typeof(attribute))", attribute.property)
     if fqn in keys(store.data)
-        return store.data[fqn]["$(attribute.context)"]
+        return store.data[fqn]["$(attribute.context)"]..., fqn
     end
 
     local cmd, keyword = getCommand(attribute)
@@ -35,7 +35,9 @@ function fetchData!(
     end
 
     for h in header[2:end]
-        store.descriptions[("$(typeof(attribute))", h[1])] = h[2]
+        if length(h[2]) > 0
+            store.descriptions[("$(typeof(attribute))", h[1])] = h[2]
+        end
     end
 
     for i in 2:length(header)
@@ -56,8 +58,8 @@ function fetchData!(
     end
 
     fqn = ("$(typeof(attribute))", attribute.property)
-
-    return store.data[fqn]["$(attribute.context)"]
+    
+    return store.data[fqn]["$(attribute.context)"]..., fqn
 end
 
 function getPoints(store::DataStore, from::DateTime, to::DateTime)

src/providers/sysstat.jl

@@ -171,7 +171,7 @@ function collectData(command::String, keyword::String, days::Int, today::DateTim
         header[i] in keys(myCommands) ||
         logger("", "coudln't find $(header[i]) in $(keys(myCommands))")
         if header[i] ∉ keys(myCommands)
-            myCommands[header[i]] = "unknown"
+            myCommands[header[i]] = ""
         end
 
         if isNumericString(data[1][i])

src/providers/sysstatDB.jl

@@ -643,23 +643,26 @@ global SAR_DB = Dict(
         "-u [ ALL ]",
         "Report CPU utilization. The ALL keyword indicates that all the CPU fields should be displayed. The report may show the following fields:",
         Dict(
+            "%system" =>
+                    "Percentage of CPU utilization that occurred while executing at the system level (kernel). Note that this field includes time spent servicing hardware and software interrupts.",
+            "%user" =>
+                    "Percentage of CPU utilization that occurred while executing at the user level (application). Note that this field includes time spent running virtual processors.",        
+            "%nice" =>
+                    "Percentage of CPU utilization that occurred while executing at the user level with nice priority.",
+            "%idle" =>
+                    "Percentage of time that the CPU or CPUs were idle and the system did not have an outstanding disk I/O request.",
+            "%steal" =>
+                    "Percentage of time spent in involuntary wait by the virtual CPU or CPUs while the hypervisor was servicing another virtual processor.",
+            "%iowait" =>
+                "Percentage of time that the CPU or CPUs were idle during which the system had an outstanding disk I/O request.",
+
             "ALL" => (
                 "",
                 Dict(
-                    "%user" =>
-                        "Percentage of CPU utilization that occurred while executing at the user level (application). Note that this field includes time spent running virtual processors.",
                     "%usr" =>
                         "Percentage of CPU utilization that occurred while executing at the user level (application). Note that this field does NOT include time spent running virtual processors.",
-                    "%nice" =>
-                        "Percentage of CPU utilization that occurred while executing at the user level with nice priority.",
-                    "%system" =>
-                        "Percentage of CPU utilization that occurred while executing at the system level (kernel). Note that this field includes time spent servicing hardware and software interrupts.",
                     "%sys" =>
                         "Percentage of CPU utilization that occurred while executing at the system level (kernel). Note that this field does NOT include time spent servicing hardware and software interrupts.",
-                    "%iowait" =>
-                        "Percentage of time that the CPU or CPUs were idle during which the system had an outstanding disk I/O request.",
-                    "%steal" =>
-                        "Percentage of time spent in involuntary wait by the virtual CPU or CPUs while the hypervisor was servicing another virtual processor.",
                     "%irq" =>
                         "Percentage of time spent by the CPU or CPUs to service hardware interrupts.",
                     "%soft" =>
@@ -668,9 +671,7 @@ global SAR_DB = Dict(
                         "Percentage of time spent by the CPU or CPUs to run a virtual processor.",
                     "%gnice" =>
                         "Percentage of time spent by the CPU or CPUs to run a niced guest.",
-                    "%idle" =>
-                        "Percentage of time that the CPU or CPUs were idle and the system did not have an outstanding disk I/O request.",
-
+
                 ),
             ),
         ),

src/widgets/qAttributes.jl

@@ -2,11 +2,11 @@ include("base.jl")
 include("../providers/data.jl")
 
 function analyze(attr::DataAttribute, ctx::PlotContext)
-    local dates, values = fetchData!(ctx.store, attr, ctx.minDate, ctx.maxDate)
+    local dates, values, title = fetchData!(ctx.store, attr, ctx.minDate, ctx.maxDate)
     # TODO: Multiply along text columns!
     local srcCtx = []
 
-    return dates, values, srcCtx, minimum(values), maximum(values), [attr.property]
+    return dates, values, srcCtx, minimum(values), maximum(values), [title]
 end
 
 import Base.+

src/widgets/qPlots.jl

@@ -199,9 +199,34 @@ function renderWidget(widget::QPlot, today::DateTime, saveTo::String)
     mkpath(saveTo)
     local path = joinpath(saveTo, "$(sanitizeFile(widget.title)).png")
     savefig(ctx.plot, path)
-    return path
+
+    # TODO: Add specialization
+    local title = widget.title
+
+    # TODO: Generate Description from Specialization data!
+    local description = ""
+
+    titles = map(titles) do x
+        local str = ""
+        if x in keys(globalDataStore.descriptions)
+            str = globalDataStore.descriptions[x]
+        end
+        (x..., str)
+    end
+
+
+    @assert isfile("$path") "Error saving the Plot"
+
+    # Apply strong png compression to make e-mail smaller
+    if USE_PNGQUANT
+        exe(`pngquant --quality=60-80 --force --output $path $path`)
+    end
+
+    return [
+        (path, titles, title, description),
+    ]
 end
 
 
-renderWidget(QUERY[1], Dates.now(), joinpath(BASE_PATH, "stats"))
+# renderWidget(QUERY[1], Dates.now(), joinpath(BASE_PATH, "stats"))
 

src/widgets/qWidgets.jl

@@ -10,46 +10,6 @@ function renderWidget(widget::QWidget, today::DateTime, saveTo::String) #::Array
     logger(widget, "Unknwon widget type $(typeof(widget))", true)
 
     throw("Unknwon widget type $(typeof(widget))")
-
-    #=
-    local datas, points, header, description
-    local results = []
-    if length(keyword) == 0
-        datas, points, header, description = collectData(command, "", SHOW_DAYS, today)
-    else
-        datas, points, header, description = collectData(command, keyword, SHOW_DAYS, today)
-    end
-    for data in datas
-        local title =
-            "-$(command) $(keyword) " * reduce(*, map(d -> "$(d[1])=$(d[3]) ", data[1]))
-        local specialization = reduce(*, map(d -> " $(d[1])=$(d[3])", data[1]))
-        local exDescription =
-            description *
-            "\n" *
-            reduce(*, map(d -> "$(d[3]) ($(d[1])) - $(d[2])\n", data[1]))
-
-        local pngPath =
-            joinpath(BASE_PATH, "stats", Dates.format(Dates.now(), "yyyy-mm-dd"))
-        mkpath(pngPath)
-        local pngPlace = joinpath(pngPath, title[2:end])
-
-        # Generate the image
-        makeDiagram(data[2], points, header, exDescription, title, pngPlace)
-
-        pngPlace *= ".png"
-        @assert isfile("$pngPlace") "Error saving the Plot"
-
-        # Apply strong png compression to make e-mail smaller
-        exe(`pngquant --quality=60-80 --force --output $pngPlace $pngPlace`)
-
-        # cleanup
-        # rm("$pngPlace")
-        push!(results, ("$pngPlace", header, exDescription, specialization))
-
-    end
-
-    return results
-    =#
 end