Skip to content

Commit

Permalink
build: Skip loading example data from configs in CI (apache#12610)
Browse files Browse the repository at this point in the history
* Skip loading example data from configs in CI

* Use .test suffix for example data required by e2e tests

* Address linting error with regex
  • Loading branch information
robdiciuccio authored Feb 2, 2021
1 parent c781ab8 commit 68dae80
Show file tree
Hide file tree
Showing 5 changed files with 11 additions and 7 deletions.
8 changes: 4 additions & 4 deletions superset/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ def load_examples_run(
print("Loading [Birth names]")
examples.load_birth_names(only_metadata, force)

print("Loading [Tabbed dashboard]")
examples.load_tabbed_dashboard(only_metadata)

if not load_test_data:
print("Loading [Random time series data]")
examples.load_random_time_series_data(only_metadata, force)
Expand Down Expand Up @@ -164,11 +167,8 @@ def load_examples_run(
print("Loading DECK.gl demo")
examples.load_deck_dash()

print("Loading [Tabbed dashboard]")
examples.load_tabbed_dashboard(only_metadata)

# load examples that are stored as YAML config files
examples.load_from_configs(force)
examples.load_from_configs(force, load_test_data)


@with_appcontext
Expand Down
10 changes: 7 additions & 3 deletions superset/examples/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import re
from pathlib import Path
from typing import Any, Dict

Expand All @@ -24,13 +25,13 @@
YAML_EXTENSIONS = {".yaml", ".yml"}


def load_from_configs(force_data: bool = False) -> None:
contents = load_contents()
def load_from_configs(force_data: bool = False, load_test_data: bool = False) -> None:
contents = load_contents(load_test_data)
command = ImportExamplesCommand(contents, overwrite=True, force_data=force_data)
command.run()


def load_contents() -> Dict[str, Any]:
def load_contents(load_test_data: bool = False) -> Dict[str, Any]:
"""Traverse configs directory and load contents"""
root = Path("examples/configs")
resource_names = resource_listdir("superset", str(root))
Expand All @@ -39,13 +40,16 @@ def load_contents() -> Dict[str, Any]:
contents: Dict[Path, str] = {}
while queue:
path_name = queue.pop()
test_re = re.compile(r"\.test\.|metadata\.yaml$")

if resource_isdir("superset", str(path_name)):
queue.extend(
path_name / child_name
for child_name in resource_listdir("superset", str(path_name))
)
elif path_name.suffix.lower() in YAML_EXTENSIONS:
if load_test_data and test_re.search(str(path_name)) is None:
continue
contents[path_name] = (
resource_stream("superset", str(path_name)).read().decode("utf-8")
)
Expand Down

0 comments on commit 68dae80

Please sign in to comment.