Upload common.py with huggingface_hub

common.py

@@ -5,7 +5,7 @@ from .collections import ItemPicker, RandomPicker
 from .operator import SourceOperator
 from .recipe import Recipe, SequentialRecipe
 from .schema import ToUnitxtGroup
-from .splitters import RandomSampler, SliceSplit, SpreadSplit
+from .splitters import RandomSampler, SeparateSplit, SliceSplit, SpreadSplit
 from .stream import MultiStream
 from .templates import RenderTemplatedICL
 
@@ -13,6 +13,7 @@ from .templates import RenderTemplatedICL
 class CommonRecipe(Recipe, SourceOperator):
     card: TaskCard
     demos_pool_name: str = "demos_pool"
+    demos_taken_from: str = "train"
     demos_pool_size: int = None
     demos_field: str = "demos"
     num_demos: int = None
@@ -21,7 +22,7 @@ class CommonRecipe(Recipe, SourceOperator):
     template_item: Union[str, int] = None
 
     def verify(self):
-        self.sampler_type in ["random"]
+        assert self.sampler_type in ["random"], f"Uknown sampler type {self.sampler_type}"
 
     def prepare(self):
         steps = [
@@ -35,13 +36,10 @@ class CommonRecipe(Recipe, SourceOperator):
 
         if self.demos_pool_size is not None:
             steps.append(
-                SliceSplit(
-                    slices={
-                        self.demos_pool_name: f"train[:{int(self.demos_pool_size)}]",
-                        "train": f"train[{int(self.demos_pool_size)}:]",
-                        "validation": "validation",
-                        "test": "test",
-                    }
+                SeparateSplit(
+                    from_split=self.demos_taken_from,
+                    to_split_names=[self.demos_pool_name, self.demos_taken_from],
+                    to_split_sizes=[int(self.demos_pool_size)],
                 )
             )
 
@@ -58,8 +56,8 @@ class CommonRecipe(Recipe, SourceOperator):
             )
 
         if self.card.instructions is not None:
-            if self.instruction_item is None:
-                picker = ItemPicker(self.instruction_item)
+            if not self.instruction_item is None:
+                picker = ItemPicker(int(self.instruction_item))
             else:
                 picker = RandomPicker()
             instruction = picker(self.card.instructions)
@@ -68,9 +66,9 @@ class CommonRecipe(Recipe, SourceOperator):
 
         if self.card.templates is not None:
             if self.template_item is None:
-                picker = ItemPicker(self.template_item)
-            else:
                 picker = RandomPicker()
+            else:
+                picker = ItemPicker(self.template_item)
             template = picker(self.card.templates)
         else:
             template = None