fix prose

pages/2_👩🏼‍🔬 Describe Domain.py

@@ -1,281 +0,0 @@
-import json
-
-import streamlit as st
-
-from hub import push_dataset_to_hub, pull_seed_data_from_repo
-from infer import query
-from defaults import (
-    N_PERSPECTIVES,
-    N_TOPICS,
-    SEED_DATA_PATH,
-    PIPELINE_PATH,
-    DATASET_REPO_ID,
-)
-from utils import project_sidebar, create_seed_terms, create_application_instruction
-
-
-st.set_page_config(
-    page_title="Domain Data Grower",
-    page_icon="🧑‍🌾",
-)
-project_sidebar()
-
-
-################################################################################
-# HEADER
-################################################################################
-
-st.header("🧑‍🌾 Domain Data Grower")
-st.divider()
-st.subheader(
-    "Step 2. Define the specific domain that you want to generate synthetic data for.",
-)
-st.write(
-    "Define the project details, including the project name, domain, and API credentials"
-)
-
-
-################################################################################
-# LOAD EXISTING DOMAIN DATA
-################################################################################
-
-DATASET_REPO_ID = (
-    f"{st.session_state['hub_username']}/{st.session_state['project_name']}"
-)
-SEED_DATA = pull_seed_data_from_repo(
-    DATASET_REPO_ID, hub_token=st.session_state["hub_token"]
-)
-DEFAULT_DOMAIN = SEED_DATA.get("domain", "")
-DEFAULT_PERSPECTIVES = SEED_DATA.get("perspectives", [""])
-DEFAULT_TOPICS = SEED_DATA.get("topics", [""])
-DEFAULT_EXAMPLES = SEED_DATA.get("examples", [{"question": "", "answer": ""}])
-DEFAULT_SYSTEM_PROMPT = SEED_DATA.get("domain_expert_prompt", "")
-
-################################################################################
-# Domain Expert Section
-################################################################################
-
-(
-    tab_domain_expert,
-    tab_domain_perspectives,
-    tab_domain_topics,
-    tab_examples,
-    tab_raw_seed,
-) = st.tabs(
-    tabs=[
-        "👩🏼‍🔬 Domain Expert",
-        "🔍 Domain Perspectives",
-        "🕸️ Domain Topics",
-        "📚 Examples",
-        "🌱 Raw Seed Data",
-    ]
-)
-
-with tab_domain_expert:
-    st.text("Define the domain expertise that you want to train a language model")
-    st.info(
-        "A domain expert is a person who is an expert in a particular field or area. For example, a domain expert in farming would be someone who has extensive knowledge and experience in farming and agriculture."
-    )
-
-    domain = st.text_input("Domain Name", DEFAULT_DOMAIN)
-
-    domain_expert_prompt = st.text_area(
-        label="Domain Expert Definition",
-        value=DEFAULT_SYSTEM_PROMPT,
-        height=200,
-    )
-
-################################################################################
-# Domain Perspectives
-################################################################################
-
-with tab_domain_perspectives:
-    st.text("Define the different perspectives from which the domain can be viewed")
-    st.info(
-        """
-    Perspectives are different viewpoints or angles from which a domain can be viewed. 
-    For example, the domain of farming can be viewed from the perspective of a commercial 
-    farmer or an independent family farmer."""
-    )
-
-    perspectives = st.session_state.get(
-        "perspectives",
-        [DEFAULT_PERSPECTIVES[0]],
-    )
-    perspectives_container = st.container()
-
-    perspectives = [
-        perspectives_container.text_input(
-            f"Domain Perspective {i + 1}", value=perspective
-        )
-        for i, perspective in enumerate(perspectives)
-    ]
-
-    if st.button("Add Perspective", key="add_perspective"):
-        n = len(perspectives)
-        value = DEFAULT_PERSPECTIVES[n] if n < N_PERSPECTIVES else ""
-        perspectives.append(
-            perspectives_container.text_input(f"Domain Perspective {n + 1}", value="")
-        )
-
-    st.session_state["perspectives"] = perspectives
-
-
-################################################################################
-# Domain Topics
-################################################################################
-
-with tab_domain_topics:
-    st.text("Define the main themes or subjects that are relevant to the domain")
-    st.info(
-        """Topics are the main themes or subjects that are relevant to the domain. For example, the domain of farming can have topics like soil health, crop rotation, or livestock management."""
-    )
-    topics = st.session_state.get(
-        "topics",
-        [DEFAULT_TOPICS[0]],
-    )
-    topics_container = st.container()
-    topics = [
-        topics_container.text_input(f"Domain Topic {i + 1}", value=topic)
-        for i, topic in enumerate(topics)
-    ]
-
-    if st.button("Add Topic", key="add_topic"):
-        n = len(topics)
-        value = DEFAULT_TOPICS[n] if n < N_TOPICS else ""
-        topics.append(topics_container.text_input(f"Domain Topics {n + 1}", value=""))
-
-    st.session_state["topics"] = topics
-
-
-################################################################################
-# Examples Section
-################################################################################
-
-with tab_examples:
-    st.text(
-        "Add high-quality questions and answers that can be used to generate synthetic data"
-    )
-    st.info(
-        """
-    Examples are high-quality questions and answers that can be used to generate 
-    synthetic data for the domain. These examples will be used to train the language model
-    to generate questions and answers.
-    """
-    )
-
-    examples = st.session_state.get(
-        "examples",
-        [
-            {
-                "question": "",
-                "answer": "",
-            }
-        ],
-    )
-
-    for n, example in enumerate(examples, 1):
-        question = example["question"]
-        answer = example["answer"]
-        examples_container = st.container()
-        question_column, answer_column = examples_container.columns(2)
-
-        if st.button(f"Generate Answer {n}"):
-            if st.session_state["hub_token"] is None:
-                st.error("Please provide a Hub token to generate answers")
-            else:
-                answer = query(question, st.session_state["hub_token"])
-        with question_column:
-            question = st.text_area(f"Question {n}", value=question)
-
-        with answer_column:
-            answer = st.text_area(f"Answer {n}", value=answer)
-        examples[n - 1] = {"question": question, "answer": answer}
-        st.session_state["examples"] = examples
-        st.divider()
-
-    if st.button("Add Example"):
-        examples.append({"question": "", "answer": ""})
-        st.session_state["examples"] = examples
-        st.rerun()
-
-################################################################################
-# Save Domain Data
-################################################################################
-
-perspectives = list(filter(None, perspectives))
-topics = list(filter(None, topics))
-
-domain_data = {
-    "domain": domain,
-    "perspectives": perspectives,
-    "topics": topics,
-    "examples": examples,
-    "domain_expert_prompt": domain_expert_prompt,
-    "application_instruction": create_application_instruction(domain, examples),
-    "seed_terms": create_seed_terms(topics, perspectives),
-}
-
-with open(SEED_DATA_PATH, "w") as f:
-    json.dump(domain_data, f, indent=2)
-
-with tab_raw_seed:
-    st.code(json.dumps(domain_data, indent=2), language="json", line_numbers=True)
-
-################################################################################
-# Setup Dataset on the Hub
-################################################################################
-
-st.divider()
-
-
-if st.button("🤗 Push Dataset Seed") and all(
-    (
-        domain,
-        domain_expert_prompt,
-        perspectives,
-        topics,
-        examples,
-    )
-):
-    if all(
-        (
-            st.session_state.get("project_name"),
-            st.session_state.get("hub_username"),
-            st.session_state.get("hub_token"),
-        )
-    ):
-        project_name = st.session_state["project_name"]
-        hub_username = st.session_state["hub_username"]
-        hub_token = st.session_state["hub_token"]
-    else:
-        st.error(
-            "Please create a dataset repo on the Hub before pushing the dataset seed"
-        )
-        st.stop()
-
-    push_dataset_to_hub(
-        domain_seed_data_path=SEED_DATA_PATH,
-        project_name=project_name,
-        domain=domain,
-        hub_username=hub_username,
-        hub_token=hub_token,
-        pipeline_path=PIPELINE_PATH,
-    )
-
-    st.success(
-        f"Dataset seed created and pushed to the Hub. Check it out [here](https://huggingface.co/datasets/{hub_username}/{project_name})"
-    )
-
-    st.write("You can now move on to runnning your distilabel pipeline.")
-
-    st.page_link(
-        page="pages/3_🌱 Generate Dataset.py",
-        label="Generate Dataset",
-        icon="🌱",
-    )
-
-else:
-    st.info(
-        "Please fill in all the required domain fields to push the dataset seed to the Hub"
-    )

pages/3_🌱 Generate Dataset.py

@@ -1,205 +0,0 @@
-import streamlit as st
-
-from defaults import ARGILLA_URL
-from hub import push_pipeline_params
-from utils import project_sidebar
-
-st.set_page_config(
-    page_title="Domain Data Grower",
-    page_icon="🧑‍🌾",
-)
-
-project_sidebar()
-
-################################################################################
-# HEADER
-################################################################################
-
-st.header("🧑‍🌾 Domain Data Grower")
-st.divider()
-st.subheader("Step 3. Run the pipeline to generate synthetic data")
-st.write("Define the distilabel pipeline for generating the dataset.")
-
-hub_username = st.session_state.get("hub_username")
-project_name = st.session_state.get("project_name")
-hub_token = st.session_state.get("hub_token")
-
-###############################################################
-# CONFIGURATION
-###############################################################
-
-st.divider()
-
-st.markdown("## 🧰 Pipeline Configuration")
-
-st.write(
-    "Now we need to define the configuration for the pipeline that will generate the synthetic data."
-)
-st.write(
-    "⚠️ Model and parameter choices significantly affect the quality of the generated data. \
-    We reccomend that you start with generating a few samples and review the data. Then scale up from there. \
-    You can run the pipeline multiple times with different configurations and append it to the same Argilla dataset."
-)
-
-
-st.markdown("#### 🤖 Inference configuration")
-
-st.write(
-    "Add the url of the Huggingface inference API or endpoint that your pipeline should use. You can find compatible models here:"
-)
-
-with st.expander("🤗 Recommended Models"):
-    st.write("All inference endpoint compatible models can be found via the link below")
-    st.link_button(
-        "🤗 Inference compaptible models on the hub",
-        "https://huggingface.co/models?pipeline_tag=text-generation&other=endpoints_compatible&sort=trending",
-    )
-    st.write("🔋Projects with sufficient resources could take advantage of LLama3 70b")
-    st.code(
-        "https://api-inference.huggingface.co/models/meta-llama/Meta-Llama-3-8B-Instruct"
-    )
-
-    st.write("🪫Projects with less resources could take advantage of LLama 3 8b")
-    st.code(
-        "https://api-inference.huggingface.co/models/meta-llama/Meta-Llama-3-8B-Instruct"
-    )
-
-    st.write("🍃Projects with even less resources could use Phi-3-mini-4k-instruct")
-    st.code(
-        "https://api-inference.huggingface.co/models/microsoft/Phi-3-mini-4k-instruct"
-    )
-
-    st.write("Note Hugggingface Pro gives access to more compute resources")
-    st.link_button(
-        "🤗 Huggingface Pro",
-        "https://huggingface.co/pricing",
-    )
-
-
-self_instruct_base_url = st.text_input(
-    label="Model base URL for instruction generation",
-    value="https://api-inference.huggingface.co/models/microsoft/Phi-3-mini-4k-instruct",
-)
-domain_expert_base_url = st.text_input(
-    label="Model base URL for domain expert response",
-    value="https://api-inference.huggingface.co/models/microsoft/Phi-3-mini-4k-instruct",
-)
-
-st.divider()
-st.markdown("#### 🧮 Parameters configuration")
-
-self_intruct_num_generations = st.slider(
-    "Number of generations for self-instruction", 1, 10, 2
-)
-domain_expert_num_generations = st.slider(
-    "Number of generations for domain expert response", 1, 10, 2
-)
-self_instruct_temperature = st.slider("Temperature for self-instruction", 0.1, 1.0, 0.9)
-domain_expert_temperature = st.slider("Temperature for domain expert", 0.1, 1.0, 0.9)
-
-st.divider()
-st.markdown("#### 🔬 Argilla API details to push the generated dataset")
-argilla_url = st.text_input("Argilla API URL", ARGILLA_URL)
-argilla_api_key = st.text_input("Argilla API Key", "owner.apikey")
-argilla_dataset_name = st.text_input("Argilla Dataset Name", project_name)
-st.divider()
-
-###############################################################
-# LOCAL
-###############################################################
-
-st.markdown("## Run the pipeline")
-
-st.markdown(
-    "Once you've defined the pipeline configuration above, you can run the pipeline from your local machine."
-)
-
-
-if all(
-    [
-        argilla_api_key,
-        argilla_url,
-        self_instruct_base_url,
-        domain_expert_base_url,
-        self_intruct_num_generations,
-        domain_expert_num_generations,
-        self_instruct_temperature,
-        domain_expert_temperature,
-        hub_username,
-        project_name,
-        hub_token,
-        argilla_dataset_name,
-    ]
-) and st.button("💾 Save Pipeline Config"):
-    with st.spinner("Pushing pipeline to the Hub..."):
-        push_pipeline_params(
-            pipeline_params={
-                "argilla_api_key": argilla_api_key,
-                "argilla_api_url": argilla_url,
-                "argilla_dataset_name": argilla_dataset_name,
-                "self_instruct_base_url": self_instruct_base_url,
-                "domain_expert_base_url": domain_expert_base_url,
-                "self_instruct_temperature": self_instruct_temperature,
-                "domain_expert_temperature": domain_expert_temperature,
-                "self_intruct_num_generations": self_intruct_num_generations,
-                "domain_expert_num_generations": domain_expert_num_generations,
-            },
-            hub_username=hub_username,
-            hub_token=hub_token,
-            project_name=project_name,
-        )
-
-    st.success(
-        f"Pipeline configuration pushed to the dataset repo {hub_username}/{project_name} on the Hub."
-    )
-
-    st.markdown(
-        "To run the pipeline locally, you need to have the `distilabel` library installed. You can install it using the following command:"
-    )
-
-    st.code(
-        f"""
-        
-        # Install the distilabel library
-        pip install distilabel
-        """
-    )
-
-    st.markdown("Next, you'll need to clone your dataset repo and run the pipeline:")
-
-    st.code(
-        f"""
-        git clone https://github.com/huggingface/data-is-better-together
-        cd data-is-better-together/domain-specific-datasets/pipelines
-        pip install -r requirements.txt
-        """
-    )
-
-    st.markdown("Finally, you can run the pipeline using the following command:")
-
-    st.code(
-        f"""
-        huggingface-cli login
-        python domain_expert_pipeline.py {hub_username}/{project_name}""",
-        language="bash",
-    )
-    st.markdown(
-        "👩‍🚀 If you want to customise the pipeline take a look in `pipeline.py` and teh [distilabel docs](https://distilabel.argilla.io/)"
-    )
-
-    st.markdown(
-        "🚀 Once you've run the pipeline your records will be available in the Argilla space"
-    )
-
-    st.link_button("🔗 Argilla Space", argilla_url)
-
-    st.markdown("Once you've reviewed the data, you can publish it on the next page:")
-
-    st.page_link(
-        page="pages/4_🔍 Review Generated Data.py",
-        label="Review Generated Data",
-        icon="🔍",
-    )
-
-else:
-    st.info("Please fill all the required fields.")

pages/4_🔍 Review Generated Data.py

@@ -1,48 +0,0 @@
-import streamlit as st
-
-from defaults import PROJECT_NAME, ARGILLA_URL, DATASET_REPO_ID
-from utils import project_sidebar
-from hub import push_argilla_dataset_to_hub
-
-st.set_page_config(
-    page_title="Domain Data Grower",
-    page_icon="🧑‍🌾",
-)
-
-project_sidebar()
-
-################################################################################
-# HEADER
-################################################################################
-
-st.header("🧑‍🌾 Domain Data Grower")
-st.divider()
-
-st.write(
-    """Once you have reviewed the synthetic data in Argilla, you can publish the 
-    generated dataset to the Hub."""
-)
-
-
-################################################################################
-# Configuration
-################################################################################
-
-st.divider()
-st.write("🔬 Argilla API details to push the generated dataset")
-argilla_url = st.text_input("Argilla API URL", ARGILLA_URL)
-argilla_api_key = st.text_input("Argilla API Key", "owner.apikey")
-argilla_dataset_name = st.text_input("Argilla Dataset Name", PROJECT_NAME)
-dataset_repo_id = st.text_input("Dataset Repo ID", DATASET_REPO_ID)
-st.divider()
-
-if st.button("🚀 Publish the generated dataset"):
-    with st.spinner("Publishing the generated dataset..."):
-        push_argilla_dataset_to_hub(
-            name=argilla_dataset_name,
-            repo_id=dataset_repo_id,
-            url=argilla_url,
-            api_key=argilla_api_key,
-            workspace="admin",
-        )
-    st.success("The generated dataset has been published to the Hub.")