Add support for curl upload (krzysztofslusarski#19)

* Add support for curl upload * Slightly change kubernetes command Co-authored-by: Artur Owczarek <[email protected]>
mwwojcik · Jan 10, 2021 · 4dd5be5 · 4dd5be5
1 parent 1da30df
commit 4dd5be5
Show file tree

Hide file tree

Showing 11 changed files with 104 additions and 22 deletions.
diff --git a/pom.xml b/pom.xml
@@ -36,6 +36,7 @@
         <miglayout-swing.version>5.0</miglayout-swing.version>
         <spock.version>1.3-groovy-2.5</spock.version>
         <gmavenplus-plugin.version>1.6.2</gmavenplus-plugin.version>
+        <axios.version>0.21.1</axios.version>
     </properties>
 
     <dependencyManagement>
@@ -90,6 +91,11 @@
                 <artifactId>miglayout-swing</artifactId>
                 <version>${miglayout-swing.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.webjars.npm</groupId>
+                <artifactId>axios</artifactId>
+                <version>${axios.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.spockframework</groupId>
                 <artifactId>spock-core</artifactId>

diff --git a/...commons/src/main/java/pl/ks/profiling/safepoint/analyzer/commons/shared/StatsService.java b/...commons/src/main/java/pl/ks/profiling/safepoint/analyzer/commons/shared/StatsService.java
@@ -110,7 +110,9 @@ public JvmLogFile createAllStatsUnifiedLogger(InputStream inputStream, String or
                 tlabLogFileParser.parseLine(line);
                 stringDedupLogFileParser.parseLine(line);
                 line = reader.readLine();
-                notifyProgress.accept(new ParsingProgress(numberOfLine++, false));
+                if (numberOfLine / 1000 == 0) {
+                    notifyProgress.accept(new ParsingProgress(numberOfLine++, false));
+                }
             }
         } catch (Exception ex) {
             throw new RuntimeException(ex);

diff --git a/safepoint-analyzer-web/pom.xml b/safepoint-analyzer-web/pom.xml
@@ -48,6 +48,10 @@
             <groupId>org.hibernate</groupId>
             <artifactId>hibernate-validator</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.webjars.npm</groupId>
+            <artifactId>axios</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...nt-analyzer-web/src/main/java/pl/ks/profiling/safepoint/analyzer/web/ParsingExecutor.java b/...nt-analyzer-web/src/main/java/pl/ks/profiling/safepoint/analyzer/web/ParsingExecutor.java
@@ -16,8 +16,9 @@
 package pl.ks.profiling.safepoint.analyzer.web;
 
 import java.io.InputStream;
+import java.util.function.Function;
 
 public interface ParsingExecutor {
-    ParsingStatus enqueue(InputStream inputStream, String originalFilename);
+    ParsingStatus enqueue(InputStream inputStream, String originalFilename, Function<String, String> resultLocationFactory);
     ParsingStatus getParsingStatus(String parsingId);
 }
diff --git a/...lyzer-web/src/main/java/pl/ks/profiling/safepoint/analyzer/web/ParsingExecutorSimple.java b/...lyzer-web/src/main/java/pl/ks/profiling/safepoint/analyzer/web/ParsingExecutorSimple.java
@@ -28,6 +28,7 @@
 import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.function.Function;
 
 @Slf4j
 @Component
@@ -36,15 +37,17 @@ public class ParsingExecutorSimple implements ParsingExecutor {
     private final StatsRepository statsRepository;
     private final Cache<String, ParsingStatus> statuses;
     private final ExecutorService executor;
+    private final ParsingProperties parsingProperties;
 
     public ParsingExecutorSimple(StatsService statsService, StatsRepository statsRepository, ParsingProperties parsingProperties) {
         this.statsService = statsService;
         this.statsRepository = statsRepository;
+        this.parsingProperties = parsingProperties;
         this.statuses = CacheBuilder.newBuilder().expireAfterAccess(parsingProperties.results.expiration).build();
         this.executor = Executors.newFixedThreadPool(parsingProperties.workerThreads, new CustomizableThreadFactory("parsing-"));
     }
 
-    public ParsingStatus enqueue(InputStream inputStream, String originalFilename) {
+    public ParsingStatus enqueue(InputStream inputStream, String originalFilename, Function<String, String> resultLocationFactory) {
         String parsingId = UUID.randomUUID().toString();
         executor.submit(() -> {
             log.info("Submitting parsing {} to parser", parsingId);
@@ -54,18 +57,37 @@ public ParsingStatus enqueue(InputStream inputStream, String originalFilename) {
                     (ParsingProgress p) -> updateParsingProgress(parsingId, p),
                     (JvmLogFile f) -> storeInRepo(parsingId, f));
         });
-        return new ParsingStatus(parsingId, 0, false);
+        ParsingStatus parsingStatus = createParsingInitialParsingStatus(resultLocationFactory, parsingId);
+        statuses.put(parsingId, parsingStatus);
+        return parsingStatus;
     }
 
     private void updateParsingProgress(String parsingId, ParsingProgress progress) {
-        statuses.put(parsingId, new ParsingStatus(parsingId, progress.getProcessedLines(), progress.isCompleted()));
+        ParsingStatus current = statuses.getIfPresent(parsingId);
+        if (current != null) {
+            ParsingStatus updated = current
+                    .withFinished(progress.isCompleted())
+                    .withProcessedLines(progress.getProcessedLines());
+            statuses.put(parsingId, updated);
+        } else {
+            log.warn("Status for parsing {} is not available", parsingId);
+        }
     }
 
     private void storeInRepo(String parsingId, JvmLogFile f) {
         log.info("Storing parsing {} in repository", parsingId);
         statsRepository.put(parsingId, f);
     }
 
+    private ParsingStatus createParsingInitialParsingStatus(Function<String, String> resultLocationFactory, String parsingId) {
+        return new ParsingStatus(
+                parsingId,
+                resultLocationFactory.apply(parsingId),
+                this.parsingProperties.results.expiration.toMinutes(),
+                0,
+                false);
+    }
+
     public ParsingStatus getParsingStatus(String parsingId) {
         return statuses.getIfPresent(parsingId);
     }

diff --git a/...oint-analyzer-web/src/main/java/pl/ks/profiling/safepoint/analyzer/web/ParsingStatus.java b/...oint-analyzer-web/src/main/java/pl/ks/profiling/safepoint/analyzer/web/ParsingStatus.java
@@ -15,13 +15,16 @@
  */
 package pl.ks.profiling.safepoint.analyzer.web;
 
-import lombok.AllArgsConstructor;
-import lombok.Data;
+import lombok.Value;
+import lombok.With;
 
-@AllArgsConstructor
-@Data
+@Value
 public class ParsingStatus {
     String parsingId;
+    String progressUrl;
+    long readyReportExpirationMinutes;
+    @With
     long processedLines;
+    @With
     boolean finished;
 }
diff --git a/...nt-analyzer-web/src/main/java/pl/ks/profiling/safepoint/analyzer/web/StatsController.java b/...nt-analyzer-web/src/main/java/pl/ks/profiling/safepoint/analyzer/web/StatsController.java
@@ -27,7 +27,10 @@
 import pl.ks.profiling.io.StorageUtils;
 import pl.ks.profiling.safepoint.analyzer.commons.shared.JvmLogFile;
 import pl.ks.profiling.web.commons.WelcomePage;
+
+import javax.servlet.http.HttpServletRequest;
 import java.io.InputStream;
+
 import static org.springframework.http.HttpStatus.NOT_FOUND;
 
 @Slf4j
@@ -40,6 +43,7 @@ class StatsController {
     @Value("${spring.servlet.multipart.max-file-size}")
     private DataSize maxFileSize;
 
+    private final ParsingProperties parsingProperties;
     private final StatsRepository statsRepository;
     private final ParsingExecutor parsingExecutor;
 
@@ -49,32 +53,41 @@ String indexDefault() {
     }
 
     @GetMapping("/upload")
-    String upload(Model model) {
+    String upload(Model model, HttpServletRequest request) {
+        String enqueueUrl = serverUrl(request) + "/enqueue";
+        model.addAttribute("enqueueUrl", enqueueUrl);
+        model.addAttribute("parsingProperties", parsingProperties);
         model.addAttribute("maxFileSize", maxFileSize);
         return "upload";
     }
 
     @PostMapping("/enqueue")
     @ResponseBody
-    ParsingStatus enqueue(@RequestParam("file") MultipartFile file) throws Exception {
+    ParsingStatus enqueue(@RequestParam("file") MultipartFile file, HttpServletRequest request) throws Exception {
         String originalFilename = file.getOriginalFilename();
         log.info("New request to enqueue file {}. Copying to persistent storage", originalFilename);
         log.debug("Copying file {} to persistent storage.", originalFilename);
         InputStream inputStream = StorageUtils.createCopy(INPUTS_PATH, originalFilename, file.getInputStream());
         log.debug("File {} has been copied. Enqueuing.", originalFilename);
-        ParsingStatus initialStatus = parsingExecutor.enqueue(inputStream, originalFilename);
+        ParsingStatus initialStatus = parsingExecutor.enqueue(
+                inputStream,
+                originalFilename,
+                (String parsingId) -> createParsingProgressUrl(request, parsingId));
         log.debug("File {} has received status {}", originalFilename, initialStatus);
         return initialStatus;
     }
 
     @PostMapping("/enqueue-plain-text")
     @ResponseBody
-    ParsingStatus enqueue(String text) throws Exception {
+    ParsingStatus enqueue(String text, HttpServletRequest request) throws Exception {
         log.info("New request to enqueue logs of length {} characters.", text.length());
         log.debug("Saving text to persistent storage");
         InputStream inputStream = StorageUtils.savePlainText(INPUTS_PATH, text);
         log.debug("Enqueuing logs for parsing.");
-        ParsingStatus initialStatus = parsingExecutor.enqueue(inputStream, "plain-text.log");
+        ParsingStatus initialStatus = parsingExecutor.enqueue(
+                inputStream,
+                "plain-text.log",
+                (String parsingId) -> createParsingProgressUrl(request, parsingId));
         log.debug("Logs have received status {}", initialStatus);
         return initialStatus;
     }
@@ -105,4 +118,12 @@ String getParsing(Model model, @PathVariable String parsingId) {
                 .build());
         return "welcome";
     }
+
+    private String createParsingProgressUrl(HttpServletRequest request, String parsingId) {
+        return serverUrl(request) + "/parsings/" + parsingId + "/progress";
+    }
+
+    private static String serverUrl(HttpServletRequest request) {
+        return request.getScheme() + "://" + request.getServerName() + (request.getServerPort() != 80 ? ":" + request.getServerPort() : "");
+    }
 }
diff --git a/safepoint-analyzer-web/src/main/resources/templates/upload.html b/safepoint-analyzer-web/src/main/resources/templates/upload.html
@@ -57,7 +57,7 @@ <h5 class="card-title">Current limitations:</h5>
 
     <div class="card mt-3 shadow rounded collapse" id="real-import">
         <h5 class="card-header">
-            Import from file...
+            Upload file:
         </h5>
         <div class="card-body">
             <form id="fileUploadForm">
@@ -77,7 +77,7 @@ <h5 class="card-header">
             </form>
         </div>
         <h5 class="card-header">
-            ...or paste log here:
+            Upload logs from clipboard:
         </h5>
         <div class="card-body">
             <form id="textUploadForm">
@@ -91,6 +91,31 @@ <h5 class="card-header">
                 <input type="submit" value="Submit logs" class="btn btn-primary"/>
             </form>
         </div>
+        <h5 class="card-header">
+            Upload logs with curl:
+        </h5>
+        <div class="card-body">
+            <div class="mb-3">
+                <p>
+                    Compress your logs:<br/>
+                    <kbd>zip -r logs.zip logs_file.log*</kbd><br/>
+                    <small class="text-muted">Make sure your logs are not bigger than <span
+                            th:text="${maxFileSize.toMegabytes()}"/>MB after compression.</small>
+                </p>
+                <p>
+                    Upload logs archive to the server:<br/>
+                    <kbd>curl -X POST --form file=@./logs.zip [[${enqueueUrl}]]</kbd><br/>
+                    <small class="text-muted">In response you'll receive link to your report. The report will be available for [[${parsingProperties.results.expiration.toMinutes()}]] minutes.</small>
+                </p>
+                <h6>Kubernetes</h6>
+                <p>
+                    <kbd>kubectl exec -t &lt;pod where logs are located&gt; -- curl -X POST --form
+                        file=@&lt;path to your logs on container&gt; [[${enqueueUrl}]]</kbd><br/>
+                    <small class="text-muted">Remember that <kbd>curl</kbd> command must be available in the
+                        container</small>
+                </p>
+            </div>
+        </div>
     </div>
 
     <div class="card mt-3">
@@ -215,6 +240,7 @@ <h4 class="alert-heading">Terms of service:</h4>
 
     function uploadLogs(payload, parentFormSelector, endpoint) {
         const parentElement = $(parentFormSelector);
+
         function toggleProgressBarVisible(visible) {
             const element = parentElement.find('.progress');
             if (visible) {

diff --git a/web-commons/pom.xml b/web-commons/pom.xml
@@ -25,7 +25,6 @@
         <dependency>
             <groupId>org.webjars.npm</groupId>
             <artifactId>axios</artifactId>
-            <version>0.21.1</version>
         </dependency>
         <dependency>
             <groupId>pl.ks.profiling</groupId>

diff --git a/web-commons/src/main/resources/templates/waiting-for-results.html b/web-commons/src/main/resources/templates/waiting-for-results.html
@@ -33,14 +33,13 @@ <h6 class="card-subtitle mb-2 text-muted">Parsing id:
 </div>
 <script type="text/javascript">
     $(document).ready(function () {
-        const keepQuerying = setInterval(keepAskingForStatus, 1000);
         const parsingId = "[[${initialStatus.parsingId}]]";
+        askForStatus();
+        const keepQuerying = setInterval(askForStatus, 1000);
 
-        function keepAskingForStatus() {
+        function askForStatus() {
             axios.get(`/parsings/${parsingId}/status`, {})
                 .then(function (response) {
-                    console.log(response);
-                    console.log(response.finished);
                     const responseData = response.data;
                     if (responseData.finished) {
                         clearInterval(keepQuerying);

diff --git a/web-commons/src/main/resources/templates/welcome.html b/web-commons/src/main/resources/templates/welcome.html
@@ -184,7 +184,6 @@ <h3 th:if="${content.title != null}">[[${content.title}]]</h3>
                                 var id = /*[[${pagesStatus.index}]]*/;
                                 var contentId = /*[[${contentStatus.index}]]*/;
                                 var common = {...commonHighChartsLineChartsOptions(/*[[${content.xAxisLabel}]]*/, /*[[${content.yAxisLabel}]]*/, /*[[${content.forceZeroMinValue}]]*/)};
-                                console.log(common);
                                 Highcharts.chart('container-' + id + '-' + contentId, {
                                     ...common,
                                     chart: {