Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -30,41 +30,48 @@ def get_user_models(hf_username, task):
|
|
| 30 |
dataset = 'marsyas/gtzan'
|
| 31 |
case "automatic-speech-recognition":
|
| 32 |
dataset = 'PolyAI/minds14'
|
|
|
|
|
|
|
| 33 |
case _:
|
| 34 |
print("Unsupported task")
|
| 35 |
|
| 36 |
dataset_specific_models = []
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
continue
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
dataset_specific_models.append(model)
|
| 45 |
-
except: continue
|
| 46 |
-
return dataset_specific_models
|
| 47 |
|
| 48 |
def calculate_best_result(user_models, task):
|
| 49 |
"""
|
| 50 |
Calculate the best results of a unit for a given task
|
| 51 |
:param user_model_ids: models of a user
|
| 52 |
"""
|
| 53 |
-
|
| 54 |
best_model = ""
|
| 55 |
-
|
| 56 |
if task == "audio-classification":
|
| 57 |
best_result = -100
|
| 58 |
larger_is_better = True
|
| 59 |
elif task == "automatic-speech-recognition":
|
| 60 |
best_result = 100
|
| 61 |
-
larger_is_better = False
|
| 62 |
|
| 63 |
for model in user_models:
|
| 64 |
meta = get_metadata(model)
|
| 65 |
if meta is None:
|
| 66 |
-
continue
|
| 67 |
-
metric = parse_metrics(model, task)
|
| 68 |
|
| 69 |
if larger_is_better:
|
| 70 |
if metric > best_result:
|
|
@@ -76,7 +83,7 @@ def calculate_best_result(user_models, task):
|
|
| 76 |
best_model = meta['model-index'][0]["name"]
|
| 77 |
|
| 78 |
return best_result, best_model
|
| 79 |
-
|
| 80 |
|
| 81 |
def get_metadata(model_id):
|
| 82 |
"""
|
|
@@ -97,19 +104,19 @@ def extract_metric(model_card_content, task):
|
|
| 97 |
:param model_card_content: model card content
|
| 98 |
"""
|
| 99 |
accuracy_pattern = r"Accuracy: (\d+\.\d+)"
|
| 100 |
-
wer_pattern = r"Wer: (\d+\.\d+)"
|
| 101 |
-
|
| 102 |
if task == "audio-classification":
|
| 103 |
pattern = accuracy_pattern
|
| 104 |
elif task == "automatic-speech-recognition":
|
| 105 |
pattern = wer_pattern
|
| 106 |
-
|
| 107 |
match = re.search(pattern, model_card_content)
|
| 108 |
if match:
|
| 109 |
metric = match.group(1)
|
| 110 |
return float(metric)
|
| 111 |
else:
|
| 112 |
-
return None
|
| 113 |
|
| 114 |
|
| 115 |
def parse_metrics(model, task):
|
|
@@ -133,16 +140,16 @@ def certification(hf_username):
|
|
| 133 |
},
|
| 134 |
{
|
| 135 |
"unit": "Unit 5: Automatic Speech Recognition",
|
| 136 |
-
"task": "automatic-speech-recognition",
|
| 137 |
"baseline_metric": 0.37,
|
| 138 |
"best_result": 0,
|
| 139 |
"best_model_id": "",
|
| 140 |
"passed_": False
|
| 141 |
},
|
| 142 |
{
|
| 143 |
-
"unit": "Unit 6:
|
| 144 |
-
"task": "
|
| 145 |
-
"baseline_metric": 0
|
| 146 |
"best_result": 0,
|
| 147 |
"best_model_id": "",
|
| 148 |
"passed_": False
|
|
@@ -155,7 +162,7 @@ def certification(hf_username):
|
|
| 155 |
"best_model_id": "",
|
| 156 |
"passed_": False
|
| 157 |
},
|
| 158 |
-
]
|
| 159 |
|
| 160 |
for unit in results_certification:
|
| 161 |
unit["passed"] = pass_emoji(unit["passed_"])
|
|
@@ -167,31 +174,39 @@ def certification(hf_username):
|
|
| 167 |
best_result, best_model_id = calculate_best_result(user_ac_models, task = "audio-classification")
|
| 168 |
unit["best_result"] = best_result
|
| 169 |
unit["best_model_id"] = best_model_id
|
| 170 |
-
if unit["best_result"] >= unit["baseline_metric"]:
|
| 171 |
unit["passed_"] = True
|
| 172 |
unit["passed"] = pass_emoji(unit["passed_"])
|
| 173 |
-
except: print("Either no relevant models found, or no metrics in the model card for audio classificaiton")
|
| 174 |
case "automatic-speech-recognition":
|
| 175 |
try:
|
| 176 |
user_asr_models = get_user_models(hf_username, task = "automatic-speech-recognition")
|
| 177 |
best_result, best_model_id = calculate_best_result(user_asr_models, task = "automatic-speech-recognition")
|
| 178 |
unit["best_result"] = best_result
|
| 179 |
unit["best_model_id"] = best_model_id
|
| 180 |
-
if unit["best_result"] <= unit["baseline_metric"]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
unit["passed_"] = True
|
| 182 |
unit["passed"] = pass_emoji(unit["passed_"])
|
| 183 |
-
except: print("Either no relevant models found, or no metrics in the model card for automatic speech recognition")
|
| 184 |
-
case "TBD":
|
| 185 |
print("Evaluation for this unit is work in progress")
|
| 186 |
case _:
|
| 187 |
print("Unknown task")
|
| 188 |
-
|
| 189 |
print(results_certification)
|
| 190 |
-
|
| 191 |
df = pd.DataFrame(results_certification)
|
| 192 |
df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']]
|
| 193 |
return df
|
| 194 |
-
|
| 195 |
with gr.Blocks() as demo:
|
| 196 |
gr.Markdown(f"""
|
| 197 |
# π Check your progress in the Audio Course π
|
|
|
|
| 30 |
dataset = 'marsyas/gtzan'
|
| 31 |
case "automatic-speech-recognition":
|
| 32 |
dataset = 'PolyAI/minds14'
|
| 33 |
+
case "text-to-speech":
|
| 34 |
+
dataset = ""
|
| 35 |
case _:
|
| 36 |
print("Unsupported task")
|
| 37 |
|
| 38 |
dataset_specific_models = []
|
| 39 |
+
|
| 40 |
+
if dataset == "":
|
| 41 |
+
return user_model_ids
|
| 42 |
+
else:
|
| 43 |
+
for model in user_model_ids:
|
| 44 |
+
meta = get_metadata(model)
|
| 45 |
+
if meta is None:
|
| 46 |
+
continue
|
| 47 |
+
try:
|
| 48 |
+
if meta["datasets"] == [dataset]:
|
| 49 |
+
dataset_specific_models.append(model)
|
| 50 |
+
except:
|
| 51 |
continue
|
| 52 |
+
return dataset_specific_models
|
| 53 |
+
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
def calculate_best_result(user_models, task):
|
| 56 |
"""
|
| 57 |
Calculate the best results of a unit for a given task
|
| 58 |
:param user_model_ids: models of a user
|
| 59 |
"""
|
| 60 |
+
|
| 61 |
best_model = ""
|
| 62 |
+
|
| 63 |
if task == "audio-classification":
|
| 64 |
best_result = -100
|
| 65 |
larger_is_better = True
|
| 66 |
elif task == "automatic-speech-recognition":
|
| 67 |
best_result = 100
|
| 68 |
+
larger_is_better = False
|
| 69 |
|
| 70 |
for model in user_models:
|
| 71 |
meta = get_metadata(model)
|
| 72 |
if meta is None:
|
| 73 |
+
continue
|
| 74 |
+
metric = parse_metrics(model, task)
|
| 75 |
|
| 76 |
if larger_is_better:
|
| 77 |
if metric > best_result:
|
|
|
|
| 83 |
best_model = meta['model-index'][0]["name"]
|
| 84 |
|
| 85 |
return best_result, best_model
|
| 86 |
+
|
| 87 |
|
| 88 |
def get_metadata(model_id):
|
| 89 |
"""
|
|
|
|
| 104 |
:param model_card_content: model card content
|
| 105 |
"""
|
| 106 |
accuracy_pattern = r"Accuracy: (\d+\.\d+)"
|
| 107 |
+
wer_pattern = r"Wer: (\d+\.\d+)"
|
| 108 |
+
|
| 109 |
if task == "audio-classification":
|
| 110 |
pattern = accuracy_pattern
|
| 111 |
elif task == "automatic-speech-recognition":
|
| 112 |
pattern = wer_pattern
|
| 113 |
+
|
| 114 |
match = re.search(pattern, model_card_content)
|
| 115 |
if match:
|
| 116 |
metric = match.group(1)
|
| 117 |
return float(metric)
|
| 118 |
else:
|
| 119 |
+
return None
|
| 120 |
|
| 121 |
|
| 122 |
def parse_metrics(model, task):
|
|
|
|
| 140 |
},
|
| 141 |
{
|
| 142 |
"unit": "Unit 5: Automatic Speech Recognition",
|
| 143 |
+
"task": "automatic-speech-recognition",
|
| 144 |
"baseline_metric": 0.37,
|
| 145 |
"best_result": 0,
|
| 146 |
"best_model_id": "",
|
| 147 |
"passed_": False
|
| 148 |
},
|
| 149 |
{
|
| 150 |
+
"unit": "Unit 6: Text-to-Speech",
|
| 151 |
+
"task": "text-to-speech",
|
| 152 |
+
"baseline_metric": 0,
|
| 153 |
"best_result": 0,
|
| 154 |
"best_model_id": "",
|
| 155 |
"passed_": False
|
|
|
|
| 162 |
"best_model_id": "",
|
| 163 |
"passed_": False
|
| 164 |
},
|
| 165 |
+
]
|
| 166 |
|
| 167 |
for unit in results_certification:
|
| 168 |
unit["passed"] = pass_emoji(unit["passed_"])
|
|
|
|
| 174 |
best_result, best_model_id = calculate_best_result(user_ac_models, task = "audio-classification")
|
| 175 |
unit["best_result"] = best_result
|
| 176 |
unit["best_model_id"] = best_model_id
|
| 177 |
+
if unit["best_result"] >= unit["baseline_metric"]:
|
| 178 |
unit["passed_"] = True
|
| 179 |
unit["passed"] = pass_emoji(unit["passed_"])
|
| 180 |
+
except: print("Either no relevant models found, or no metrics in the model card for audio classificaiton")
|
| 181 |
case "automatic-speech-recognition":
|
| 182 |
try:
|
| 183 |
user_asr_models = get_user_models(hf_username, task = "automatic-speech-recognition")
|
| 184 |
best_result, best_model_id = calculate_best_result(user_asr_models, task = "automatic-speech-recognition")
|
| 185 |
unit["best_result"] = best_result
|
| 186 |
unit["best_model_id"] = best_model_id
|
| 187 |
+
if unit["best_result"] <= unit["baseline_metric"]:
|
| 188 |
+
unit["passed_"] = True
|
| 189 |
+
unit["passed"] = pass_emoji(unit["passed_"])
|
| 190 |
+
except: print("Either no relevant models found, or no metrics in the model card for automatic speech recognition")
|
| 191 |
+
case "text-to-speech":
|
| 192 |
+
try:
|
| 193 |
+
user_tts_models = get_user_models(hf_username, task = "text-to-speech")
|
| 194 |
+
if user_tts_models:
|
| 195 |
+
unit["best_result"] = 0
|
| 196 |
+
unit["best_model_id"] = user_tts_models[0]
|
| 197 |
unit["passed_"] = True
|
| 198 |
unit["passed"] = pass_emoji(unit["passed_"])
|
| 199 |
+
except: print("Either no relevant models found, or no metrics in the model card for automatic speech recognition")
|
|
|
|
| 200 |
print("Evaluation for this unit is work in progress")
|
| 201 |
case _:
|
| 202 |
print("Unknown task")
|
| 203 |
+
|
| 204 |
print(results_certification)
|
| 205 |
+
|
| 206 |
df = pd.DataFrame(results_certification)
|
| 207 |
df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']]
|
| 208 |
return df
|
| 209 |
+
|
| 210 |
with gr.Blocks() as demo:
|
| 211 |
gr.Markdown(f"""
|
| 212 |
# π Check your progress in the Audio Course π
|