Update docs

README.md

@@ -15,7 +15,6 @@ YiVal stands at the intersection of flexibility and adaptability in the AI devel
     - [Demo](#demo)
         - [Basic Interactive Mode](#basic-interactive-mode)
         - [Question Answering with expected result evaluator](#question-answering-with-expected-result-evaluator)
-    - [Architecture](#architecture)
 
 ## Overview
 
@@ -125,7 +124,7 @@ Once started, navigate to the following address in your web browser:
 
 </details>
 
-For more details on this demo, check out the [Basic Interactive Mode Demo](https://github.com/YiVal/YiVal/blob/master/docs/docs/basic_interactive_mode.md#demo).
+For more details on this demo, check out the [Basic Interactive Mode Demo](https://github.com/YiVal/YiVal/blob/master/docs/basic_interactive_mode.md#demo).
 
 ### Question Answering with expected result evaluator
 
@@ -142,9 +141,4 @@ Once started, navigate to the following address in your web browser:
 
 </details>
 
-For more details on this demo, check out the [Question Answering with expected result evaluator](https://github.com/YiVal/YiVal/blob/master/docs/docs/qa_expected_results.md#demo).
-
-## Architecture
-
-For detailed information about Yival's architecutre, refer to this [doc](https://github.com/YiVal/YiVal/blob/master/docs/architecture.md#demo).
-
+For more details on this demo, check out the [Question Answering with expected result evaluator](https://github.com/YiVal/YiVal/blob/master/docs/qa_expected_results.md#demo).

auto_prompts_config.yml

@@ -0,0 +1,176 @@
+# This is a generated template. Modify the values as needed.
+
+custom_function: /Users/taofeng/YiVal/demo.headline_generation.headline_generation
+dataset:
+  data_generators:
+    openai_prompt_data_generator:
+      chunk_size: 1000
+      input_function:
+        description:
+          Given an tech startup business, generate one corresponding landing
+          page headline
+        name: headline_generation_for_business
+        parameters:
+          tech_startup_business: str
+      number_of_examples: 10
+      openai_model_name: gpt-4
+      output_path: generated_examples.pkl
+  source_type: machine_generated
+description: Generated experiment config
+
+human_rating_configs:
+  - name: clarity
+    instructions: Rate whether the headline clearly state what the start is doing
+    scale: [1, 5]
+
+  - name: relevance
+    instructions: Rate whether the headline is relevant
+    scale: [1, 5]
+
+evaluators:
+  # - evaluator_type: all
+  #   input_description:
+  #     Given an tech startup business, generate one corresponding landing
+  #     page headline
+  #   metric_calculators: []
+  #   name: openai_elo_evaluator
+  #   openai_model_name: gpt-4
+  - evaluator_type: individual
+    metric_calculators:
+      - method: AVERAGE
+    name: openai_prompt_based_evaluator
+    prompt: |-
+      You are assessing a submitted answer on a given task based on a criterion. Here is the data:
+      - Task: Given an tech startup business, generate one corresponding landing page headline
+      - Does the headline clearly communicate what the startup does or what problem it solves?
+        It should be immediately clear to anyone who reads the headline what the startup's purpose is.
+        A lack of clarity can lead to confusion and may discourage potential users or investors.
+      [Input]: {tech_startup_business}
+      [Result]: {raw_output}
+      Answer the question by selecting one of the following options:
+      A It fails to meet the criterion at all.
+      B It somewhat meets the criterion, but there is significant room for improvement.
+      C It meets the criterion to a satisfactory degree.
+      D It meets the criterion very well.
+      E It meets the criterion exceptionally well, with little to no room for improvement.
+    display_name: clarity
+    choices: ["A", "B", "C", "D", "E"]
+    description: Does the headline clearly communicate what the startup does or what problem it solves?
+    scale_description: "0-4"
+    choice_scores:
+      A: 0
+      B: 1
+      C: 2
+      D: 3
+      E: 4
+  - evaluator_type: individual
+    metric_calculators:
+      - method: AVERAGE
+    name: openai_prompt_based_evaluator
+    prompt: |-
+      You are assessing a submitted answer on a given task based on a criterion. Here is the data:
+      - Task: Given an tech startup business, generate one corresponding landing page headline
+      - Is the headline relevant to the target audience? The headline should speak directly to the
+        startup's intended users or customers, highlighting the benefits or value proposition that 
+        the startup offers.
+      [Input]: {tech_startup_business}
+      [Result]: {raw_output}
+      Answer the question by selecting one of the following options:
+      A It fails to meet the criterion at all.
+      B It somewhat meets the criterion, but there is significant room for improvement.
+      C It meets the criterion to a satisfactory degree.
+      D It meets the criterion very well.
+      E It meets the criterion exceptionally well, with little to no room for improvement.
+    display_name: relevance
+    description: Is the headline relevant to the target audience?
+    scale_description: "0-4"
+    choices: ["A", "B", "C", "D", "E"]
+    choice_scores:
+      A: 0
+      B: 1
+      C: 2
+      D: 3
+      E: 4
+
+  - evaluator_type: individual
+    metric_calculators:
+      - method: AVERAGE
+    name: openai_prompt_based_evaluator
+    prompt: |-
+      You are assessing a submitted answer on a given task based on a criterion. Here is the data:
+      - Task: Given an tech startup business, generate one corresponding landing page headline
+      - Is the headline catchy or memorable? While it's important to be clear and relevant,
+        a good headline should also be engaging and memorable. 
+        This can help the startup stand out in a crowded market.
+      [Input]: {tech_startup_business}
+      [Result]: {raw_output}
+      Answer the question by selecting one of the following options:
+      A It fails to meet the criterion at all.
+      B It somewhat meets the criterion, but there is significant room for improvement.
+      C It meets the criterion to a satisfactory degree.
+      D It meets the criterion very well.
+      E It meets the criterion exceptionally well, with little to no room for improvement.
+    display_name: catchiness
+    description: Is the headline catchy or memorable?
+    scale_description: "0-4"
+    choices: ["A", "B", "C", "D", "E"]
+    choice_scores:
+      A: 0
+      B: 1
+      C: 2
+      D: 3
+      E: 4
+
+selection_strategy:
+  ahp_selection:
+    criteria:
+      - openai_elo_evaluator
+      - average_token_usage
+      - average_latency
+      - "openai_prompt_based_evaluator: clarity"
+      - "openai_prompt_based_evaluator: relevance"
+      - "openai_prompt_based_evaluator: catchiness"
+    criteria_maximization:
+      openai_elo_evaluator: true
+      average_latency: false
+      average_token_usage: false
+    criteria_weights:
+      openai_elo_evaluator: 0.3
+      average_latency: 0.2
+      average_token_usage: 0.2
+      "openai_prompt_based_evaluator: clarity": 0.1
+      "openai_prompt_based_evaluator: relevance": 0.1
+      "openai_prompt_based_evaluator: catchiness": 0.1
+
+# wrapper_configs: {}
+
+# Variations allow for dynamic content during experiments.
+# They are identified by a globally unique name. For example, in your code,
+# you might reference a variation by its name, like:
+# variation = StringWrapper("hello", 'test_experiment')
+# In this config, you would define the variations associated with that name, e.g.
+
+variations:
+- generator_config:
+    input_description:
+      Given an tech startup business, generate one corresponding landing
+      page headline
+    input_test_cases: ["AI law firm", "AI sales agent"]
+    number_of_variations: 5
+    openai_model_name: gpt-4
+    output_path: generated_prompt.pkl
+  generator_name: openai_prompt_based_variation_generator
+name: task
+variations:
+  - instantiated_value: Generate landing page headline for
+    value: Generate landing page headline for
+    value_type: str
+    variation_id: null
+
+variations:
+  - name: task
+    variations:
+      - instantiated_value: Generate landing page headline for
+        value: Generate landing page headline for
+        value_type: str
+        variation_id: null

docs/code/combination_improvers/base_combination_improver.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+::: yival.combination_improvers.base_combination_improver

docs/code/data/base_reader.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+::: yival.data.base_reader

docs/code/data_generators/base_data_generator.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+::: yival.data_generators.base_data_generator

docs/code/evaluators/base_evaluator.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+::: yival.evaluators.base_evaluator

docs/code/result_selectors/selection_strategy.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+::: yival.result_selectors.selection_strategy

docs/code/states/experiment_state.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+::: yival.states.experiment_state

docs/code/variation_generators/base_variation_generator.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+::: yival.variation_generators.base_variation_generator

docs/code/wrappers/base_wrapper.md

@@ -0,0 +1,2 @@
+<!-- markdownlint-disable MD041 -->
+::: yival.wrappers.base_wrapper

mkdocs.yml

@@ -2,9 +2,19 @@
 site_name: YiVal
 nav:
   - Index: index.md
+  - Eng Spec:
+      - Arctitecture: architecture.md
   - Contributing Guide: contributing.md
   - Code Reference:
-    - yival.configs.config_utils: code/configs/config_utils.md
+      - yival.configs.config_utils: code/configs/config_utils.md
+      - yival.data_generators.base_data_generator: code/data_generators/base_data_generator.md
+      - yival.combination_improvers.base_combination_improver: code/combination_improvers/base_combination_improver.md
+      - yival.data.base_reader: code/data/base_reader.md
+      - yival.evaluators.base_evaluator: code/evaluators/base_evaluator.md
+      - yival.result_selectors.selection_strategy: code/result_selectors/selection_strategy.md
+      - yival.states.experiment_state: code/states/experiment_state.md
+      - yival.variation_generators.base_variation_generator: code/variation_generators/base_variation_generator.md
+      - yival.wrappers.base_wrapper: code/wrappers/base_wrapper.md
   - Change Log: changelog.md
 
 theme:
@@ -13,7 +23,7 @@ theme:
     cheme: slate
     primary: lightgreen
     accent: purple
-  logo: fontawesome/brands/github
+  logo: https://avatars.githubusercontent.com/u/134003593?s=400&u=34efa0d11568c1b5956f2e48dec7d15d95252a73&v=4
   favicon: fontawesome/brands/github
   icon:
     repo: fontawesome/brands/github

pyproject.toml

@@ -26,7 +26,6 @@ pydantic = "^2.1.1"
 tqdm = "^4.65.0"
 rich = "^13.4.2"
 types-tqdm = "^4.65.0.2"
-npyscreen = "^4.10.5"
 numpy = "^1.25.1"
 dash = "^2.11.1"
 pandas = "^2.0.3"

src/yival/combination_improvers/base_combination_improver.py

@@ -1,3 +1,11 @@
+"""
+This module defines the base class for combination improvers.
+
+Combination improvers are responsible for improving the combination of
+experiments based on their experiment results.
+
+"""
+
 from abc import ABC, abstractmethod
 from typing import Any, Dict, Optional, Type
 
@@ -16,10 +24,11 @@
 class BaseCombinationImprover(ABC):
     """
     Abstract base class for all combination improvers.
-    
     Attributes:
-        _registry (Dict[str, Dict[str, Any]]): A registry to keep track of combination improvers.
-        default_config (Optional[BaseCombinationImproverConfig]): Default configuration for the combination improver.
+        _registry (Dict[str, Dict[str, Any]]): A registry to keep track of
+        combination improvers.
+        default_config (Optional[BaseCombinationImproverConfig]): Default
+        configuration for the combination improver.
     """
 
     _registry: Dict[str, Dict[str, Any]] = {}
@@ -36,7 +45,8 @@ def get_combination_improver(
     def get_default_config(
         cls, name: str
     ) -> Optional[BaseCombinationImproverConfig]:
-        """Retrieve the default configuration of a combination improver by its name."""
+        """Retrieve the default configuration of a combination improver by its
+           name."""
         return cls._registry.get(name, {}).get("default_config")
 
     def __init__(self, config: BaseCombinationImproverConfig):
@@ -46,7 +56,8 @@ def __init__(self, config: BaseCombinationImproverConfig):
     def get_config_class(
         cls, name: str
     ) -> Optional[Type[BaseCombinationImproverConfig]]:
-        """Retrieve the configuration class of a combination imporver by its name."""
+        """Retrieve the configuration class of a combination imporver by its
+           name."""
         return cls._registry.get(name, {}).get("config_cls")
 
     @classmethod
@@ -56,7 +67,8 @@ def register_combination_improver(
         combination_improver_cls: Type['BaseCombinationImprover'],
         config_cls: Optional[Type[BaseCombinationImproverConfig]] = None
     ):
-        """Register a new combination improver along with its default configuration and configuration class."""
+        """Register a new combination improver along with its default
+           configuration and configuration class."""
         cls._registry[name] = {
             "class": combination_improver_cls,
             "default_config": combination_improver_cls.default_config,
@@ -74,10 +86,9 @@ def improve(
         Args:
             experiment (Experiment): The experiment with its results.
             config (ExperimentConfig): The original experiment configuration.
-            evaluator (Evaluator): A utility class to evaluate the ExperimentResult.
-            token_logger (TokenLogger): Logs the token usage.
+            evaluator (Evaluator): A utility class to evaluate the
+            ExperimentResult. token_logger (TokenLogger): Logs the token usage.
 
         Returns:
             ImproverOutput
         """
-        pass