Cypress flaky (streamlit#1209)

* parameterize cypress tests
* separate out flaky files and update scripts to handle flakes
* cleanup circle config
* clean up yarn script names
* update nvm version

e2e/Dockerfile

@@ -28,7 +28,7 @@ RUN sudo apt-get update && sudo apt-get install -y \
     && sudo rm -rf /var/lib/apt/lists/*
 
 # install nvm
-RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash
+RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.35.2/install.sh | bash
 
 # node vars
 ARG NVM_DIR=$HOME/.nvm

e2e/scripts/st_reuse_label.py

@@ -21,7 +21,7 @@
 if hasattr(st, "test_run_count"):
     st.test_run_count += 1
 else:
-    st.test_run_count = 0 if st.get_option("server.headless") else -1
+    st.test_run_count = -1 if st.get_option("server.headless") else -2
 
 if st.test_run_count < 1:
     w1 = st.slider("label", 0, 100, 25, 1)

e2e/specs/st_add_rows.spec.ts

@@ -59,10 +59,4 @@ describe("st.add_rows", () => {
       7
     );
   });
-
-  it("correctly adds rows to charts", () => {
-    cy.get(".element-container .stVegaLiteChart").each((el, i) => {
-      return cy.get(el).matchImageSnapshot(`stVegaLiteChart-${i}`);
-    });
-  });
 });

e2e/specs/st_bokeh_chart.spec.ts

@@ -21,12 +21,4 @@ describe("st.bokeh_chart", () => {
   before(() => {
     cy.visit("http://localhost:3000/");
   });
-
-  it("displays a bokeh chart", () => {
-    cy.get(".element-container .stBokehChart").should(
-      "have.css",
-      "height",
-      "600px"
-    );
-  });
 });

e2e/specs/st_empty.spec.ts

@@ -24,8 +24,4 @@ describe("st.empty", () => {
     // Make the ribbon decoration line disappear
     cy.get(".decoration").invoke("css", "display", "none");
   });
-
-  it("matches the snapshot", () => {
-    cy.get(".block-container").matchImageSnapshot("stEmpty");
-  });
 });

e2e/specs/st_graphviz_chart.spec.ts

@@ -28,16 +28,4 @@ describe("st.graphviz_chart", () => {
   beforeEach(() => {
     return cy.get(".stGraphVizChart").should("have.length", 3);
   });
-
-  it("displays a graph with two connected nodes", () => {
-    cy.get("#graphviz-chart-0 svg").matchImageSnapshot("graphviz-chart-0");
-  });
-
-  it("displays a colorful node within a cluster within a graph", () => {
-    cy.get("#graphviz-chart-1 svg").matchImageSnapshot("graphviz-chart-1");
-  });
-
-  it("displays a graph representing a finite state machine", () => {
-    cy.get("#graphviz-chart-2 svg").matchImageSnapshot("graphviz-chart-2");
-  });
 });

e2e/specs/st_in_cache_warning.spec.ts

@@ -21,18 +21,4 @@ describe("st calls within cached functions", () => {
   beforeEach(() => {
     cy.visit("http://localhost:3000/");
   });
-
-  it("displays expected results", () => {
-    // We should have two alerts
-    cy.get(".element-container > .alert-warning").should("have.length", 2);
-
-    // One button
-    cy.get(".element-container > .stButton").should("have.length", 1);
-
-    // And two texts
-    cy.get(".element-container > .markdown-text-container").should(
-      "have.length",
-      3
-    );
-  });
 });

e2e/specs/st_pyplot_kwargs.spec.ts

@@ -29,11 +29,4 @@ describe("st.pyplot with kwargs", () => {
     // Make the ribbon decoration line disappear
     cy.get(".decoration").invoke("css", "display", "none");
   });
-
-  it("draws long text strings correctly", () => {
-    cy.get(".stImage")
-      .find("img")
-      .should("have.attr", "src");
-    cy.get(".stImage > img").matchImageSnapshot("pyplot-long-text-strings");
-  });
 });

e2e_flaky/scripts/st_add_rows.py

@@ -0,0 +1 @@
+../../e2e/scripts/st_add_rows.py

e2e_flaky/scripts/st_bokeh_chart.py

@@ -0,0 +1 @@
+../../e2e/scripts/st_bokeh_chart.py

e2e_flaky/scripts/st_empty.py

@@ -0,0 +1 @@
+../../e2e/scripts/st_empty.py

e2e_flaky/scripts/st_graphviz_chart.py

@@ -0,0 +1 @@
+../../e2e/scripts/st_graphviz_chart.py

e2e_flaky/scripts/st_in_cache_warning.py

@@ -0,0 +1 @@
+../../e2e/scripts/st_in_cache_warning.py

e2e_flaky/scripts/st_pyplot_kwargs.py

@@ -0,0 +1 @@
+../../e2e/scripts/st_pyplot_kwargs.py

e2e_flaky/specs/st_add_rows.spec.ts

@@ -0,0 +1,48 @@
+/**
+ * @license
+ * Copyright 2018-2020 Streamlit Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/// <reference types="cypress" />
+
+describe("st.add_rows", () => {
+  // Doesn't have to run before each, since these tests are stateless.
+  before(() => {
+    cy.visit("http://localhost:3000/");
+
+    // Rerun the script because we want to test that JS-side coalescing works.
+    cy.get(".stApp .decoration").trigger("keypress", {
+      keyCode: 82, // "r"
+      which: 82 // "r"
+    });
+
+    // Wait for 'stale-element' class to go away, so the snapshot looks right.
+    cy.get(".element-container").should("not.have.class", "stale-element");
+
+    // Make the ribbon decoration line disappear
+    cy.get(".decoration").invoke("css", "display", "none");
+  });
+
+  beforeEach(() => {
+    // Check that the app is fully loaded
+    return cy.get(".element-container").should("have.length", 26);
+  });
+
+  it("correctly adds rows to charts", () => {
+    cy.get(".element-container .stVegaLiteChart").each((el, i) => {
+      return cy.get(el).matchImageSnapshot(`stVegaLiteChart-${i}`);
+    });
+  });
+});

e2e_flaky/specs/st_bokeh_chart.spec.ts

@@ -0,0 +1,32 @@
+/**
+ * @license
+ * Copyright 2018-2020 Streamlit Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/// <reference types="cypress" />
+
+describe("st.bokeh_chart", () => {
+  before(() => {
+    cy.visit("http://localhost:3000/");
+  });
+
+  it("displays a bokeh chart", () => {
+    cy.get(".element-container .stBokehChart").should(
+      "have.css",
+      "height",
+      "600px"
+    );
+  });
+});

e2e_flaky/specs/st_empty.spec.ts

@@ -0,0 +1,31 @@
+/**
+ * @license
+ * Copyright 2018-2020 Streamlit Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/// <reference types="cypress" />
+
+describe("st.empty", () => {
+  before(() => {
+    cy.visit("http://localhost:3000/");
+
+    // Make the ribbon decoration line disappear
+    cy.get(".decoration").invoke("css", "display", "none");
+  });
+
+  it("matches the snapshot", () => {
+    cy.get(".block-container").matchImageSnapshot("stEmpty");
+  });
+});

e2e_flaky/specs/st_graphviz_chart.spec.ts

@@ -0,0 +1,43 @@
+/**
+ * @license
+ * Copyright 2018-2020 Streamlit Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/// <reference types="cypress" />
+
+describe("st.graphviz_chart", () => {
+  before(() => {
+    cy.visit("http://localhost:3000/");
+
+    // Make the ribbon decoration line disappear
+    cy.get(".decoration").invoke("css", "display", "none");
+  });
+
+  beforeEach(() => {
+    return cy.get(".stGraphVizChart").should("have.length", 3);
+  });
+
+  it("displays a graph with two connected nodes", () => {
+    cy.get("#graphviz-chart-0 svg").matchImageSnapshot("graphviz-chart-0");
+  });
+
+  it("displays a colorful node within a cluster within a graph", () => {
+    cy.get("#graphviz-chart-1 svg").matchImageSnapshot("graphviz-chart-1");
+  });
+
+  it("displays a graph representing a finite state machine", () => {
+    cy.get("#graphviz-chart-2 svg").matchImageSnapshot("graphviz-chart-2");
+  });
+});

e2e_flaky/specs/st_in_cache_warning.spec.ts

@@ -0,0 +1,38 @@
+/**
+ * @license
+ * Copyright 2018-2020 Streamlit Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/// <reference types="cypress" />
+
+describe("st calls within cached functions", () => {
+  beforeEach(() => {
+    cy.visit("http://localhost:3000/");
+  });
+
+  it("displays expected results", () => {
+    // We should have two alerts
+    cy.get(".element-container > .alert-warning").should("have.length", 2);
+
+    // One button
+    cy.get(".element-container > .stButton").should("have.length", 1);
+
+    // And two texts
+    cy.get(".element-container > .markdown-text-container").should(
+      "have.length",
+      3
+    );
+  });
+});

e2e_flaky/specs/st_pyplot_kwargs.spec.ts

@@ -0,0 +1,39 @@
+/**
+ * @license
+ * Copyright 2018-2020 Streamlit Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/// <reference types="cypress" />
+
+describe("st.pyplot with kwargs", () => {
+  before(() => {
+    cy.visit("http://localhost:3000/");
+
+    // Wait for the site to be fully loaded
+    cy.contains("Done!", { timeout: 100000 }).should($els => {
+      expect($els).to.have.length.of.at.least(1);
+    });
+
+    // Make the ribbon decoration line disappear
+    cy.get(".decoration").invoke("css", "display", "none");
+  });
+
+  it("draws long text strings correctly", () => {
+    cy.get(".stImage")
+      .find("img")
+      .should("have.attr", "src");
+    cy.get(".stImage > img").matchImageSnapshot("pyplot-long-text-strings");
+  });
+});

frontend/cypress.json

@@ -1,5 +1,4 @@
 {
-  "projectId": "r53s63",
   "reporter": "mocha-multi-reporters",
   "reporterOptions": {
     "configFile": "reporter-config.json"

frontend/package.json

@@ -11,8 +11,12 @@
     "test:coverage": "export NODE_OPTIONS=--max_old_space_size=4096 && craco test --env=jsdom --coverage --watchAll false --watch false ./",
     "cy:open": "unset NODE_OPTIONS && cypress open --env devicePixelRatio=1",
     "cy:run": "unset NODE_OPTIONS && cypress run",
-    "cy:run-all": "../scripts/run_e2e_tests.sh -a true -c .. -r",
-    "cy:serve-and-run-all": "NODE_OPTIONS=--max_old_space_size=4096 start-server-and-test start http://localhost:3000 cy:run-all",
+    "cy:run-flaky": "yarn cy:run --config integrationFolder=../e2e_flaky/specs",
+    "cy:run-all": "../scripts/run_e2e_tests.sh -a true -c ..",
+    "cy:run-all-flaky": "yarn cy:run-all -f",
+    "_cy:serve-and-run": "NODE_OPTIONS=--max_old_space_size=4096 start-server-and-test start http://localhost:3000",
+    "cy:serve-and-run-all": "yarn _cy:serve-and-run -- cy:run-all",
+    "cy:serve-and-run-all-flaky": "yarn _cy:serve-and-run -- cy:run-all-flaky",
     "typecheck": "tsc --noEmit"
   },
   "jest": {