Commit
·
60132ec
1
Parent(s):
7095a34
bugfixes and added auth
Browse files- .gitignore +1 -1
- app.py +80 -73
- results/audios/temp +0 -0
- results/results.csv +1 -0
.gitignore
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
|
|
| 1 |
*.json
|
| 2 |
mapping
|
| 3 |
*.ipynb
|
| 4 |
test.py
|
| 5 |
-
results/
|
| 6 |
.notebook/
|
| 7 |
__pycache__/
|
|
|
|
| 1 |
+
.env
|
| 2 |
*.json
|
| 3 |
mapping
|
| 4 |
*.ipynb
|
| 5 |
test.py
|
|
|
|
| 6 |
.notebook/
|
| 7 |
__pycache__/
|
app.py
CHANGED
|
@@ -25,12 +25,13 @@ if not os.path.exists("results"):
|
|
| 25 |
os.mkdir("results")
|
| 26 |
|
| 27 |
if not os.path.exists(SAVE_PATH):
|
| 28 |
-
open(SAVE_PATH,"w")
|
|
|
|
| 29 |
|
| 30 |
if not os.path.exists(TEMP_DIR):
|
| 31 |
os.mkdir(TEMP_DIR)
|
| 32 |
|
| 33 |
-
CREATE_TASK_URL =
|
| 34 |
|
| 35 |
def decode_audio_array(base64_string):
|
| 36 |
bytes_data = base64.b64decode(base64_string)
|
|
@@ -41,7 +42,10 @@ def decode_audio_array(base64_string):
|
|
| 41 |
return audio_array
|
| 42 |
|
| 43 |
def send_task(payload):
|
| 44 |
-
|
|
|
|
|
|
|
|
|
|
| 45 |
response = response.json()
|
| 46 |
|
| 47 |
if payload["task"] == "transcribe_with_fastapi":
|
|
@@ -591,76 +595,79 @@ def dashboard():
|
|
| 591 |
st.title('Model Arena Scoreboard')
|
| 592 |
|
| 593 |
df = pd.read_csv(SAVE_PATH)
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
|
|
|
| 604 |
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
|
|
|
|
|
|
| 664 |
else:
|
| 665 |
st.write('You have not entered your email and name yet')
|
| 666 |
st.write('Please Navigate to login page in the dropdown menu')
|
|
@@ -732,7 +739,7 @@ def validate_name(name):
|
|
| 732 |
return re.match(pattern, name) is not None
|
| 733 |
|
| 734 |
def create_login_page():
|
| 735 |
-
st.title("Welcome to the
|
| 736 |
|
| 737 |
if 'logged_in' not in st.session_state:
|
| 738 |
st.session_state.logged_in = False
|
|
|
|
| 25 |
os.mkdir("results")
|
| 26 |
|
| 27 |
if not os.path.exists(SAVE_PATH):
|
| 28 |
+
with open(SAVE_PATH,"w") as f:
|
| 29 |
+
f.write("""email,path,Ori Apex_score,Ori Apex XT_score,deepgram_score,Ori Swift_score,Ori Prime_score,Ori Apex_appearance,Ori Apex XT_appearance,deepgram_appearance,Ori Swift_appearance,Ori Prime_appearance,Ori Apex_duration,Ori Apex XT_duration,deepgram_duration,Ori Swift_duration,Ori Prime_duration,azure_score,azure_appearance,azure_duration\n""")
|
| 30 |
|
| 31 |
if not os.path.exists(TEMP_DIR):
|
| 32 |
os.mkdir(TEMP_DIR)
|
| 33 |
|
| 34 |
+
CREATE_TASK_URL = os.getenv("CREATE_TASK_URL")
|
| 35 |
|
| 36 |
def decode_audio_array(base64_string):
|
| 37 |
bytes_data = base64.b64decode(base64_string)
|
|
|
|
| 42 |
return audio_array
|
| 43 |
|
| 44 |
def send_task(payload):
|
| 45 |
+
header = {
|
| 46 |
+
"Authorization": f"Bearer {os.getenv('CREATE_TASK_API_KEY')}"
|
| 47 |
+
}
|
| 48 |
+
response = requests.post(CREATE_TASK_URL,json=payload,headers=header)
|
| 49 |
response = response.json()
|
| 50 |
|
| 51 |
if payload["task"] == "transcribe_with_fastapi":
|
|
|
|
| 595 |
st.title('Model Arena Scoreboard')
|
| 596 |
|
| 597 |
df = pd.read_csv(SAVE_PATH)
|
| 598 |
+
if len(df) != 0:
|
| 599 |
+
metrics = calculate_metrics(df)
|
| 600 |
+
|
| 601 |
+
MODEL_DESCRIPTIONS = {
|
| 602 |
+
"Ori Prime": "Foundational, large, and stable.",
|
| 603 |
+
"Ori Swift": "Lighter and faster than Ori Prime.",
|
| 604 |
+
"Ori Apex": "The top-performing model, fast and stable.",
|
| 605 |
+
"Ori Apex XT": "Enhanced with more training, though slightly less stable than Ori Apex.",
|
| 606 |
+
"DG" : "Deepgram Nova-2 API",
|
| 607 |
+
"Azure" : "Azure Speech Services API"
|
| 608 |
+
}
|
| 609 |
|
| 610 |
+
st.header('Model Descriptions')
|
| 611 |
+
|
| 612 |
+
cols = st.columns(2)
|
| 613 |
+
for idx, (model, description) in enumerate(MODEL_DESCRIPTIONS.items()):
|
| 614 |
+
with cols[idx % 2]:
|
| 615 |
+
st.markdown(f"""
|
| 616 |
+
<div style='padding: 1rem; border: 1px solid #e1e4e8; border-radius: 6px; margin-bottom: 1rem;'>
|
| 617 |
+
<h3 style='margin: 0; margin-bottom: 0.5rem;'>{model}</h3>
|
| 618 |
+
<p style='margin: 0; color: #6e7681;'>{description}</p>
|
| 619 |
+
</div>
|
| 620 |
+
""", unsafe_allow_html=True)
|
| 621 |
+
|
| 622 |
+
st.header('Overall Performance')
|
| 623 |
+
|
| 624 |
+
col1, col2, col3= st.columns(3)
|
| 625 |
+
|
| 626 |
+
with col1:
|
| 627 |
+
create_metric_container("Total Matches", len(df))
|
| 628 |
+
|
| 629 |
+
best_model = max(metrics.items(), key=lambda x: x[1]['win_rate'])[0]
|
| 630 |
+
with col2:
|
| 631 |
+
create_metric_container(
|
| 632 |
+
"Best Model",
|
| 633 |
+
get_model_abbreviation(best_model),
|
| 634 |
+
full_name=best_model
|
| 635 |
+
)
|
| 636 |
+
|
| 637 |
+
most_appearances = max(metrics.items(), key=lambda x: x[1]['appearances'])[0]
|
| 638 |
+
with col3:
|
| 639 |
+
create_metric_container(
|
| 640 |
+
"Most Used",
|
| 641 |
+
get_model_abbreviation(most_appearances),
|
| 642 |
+
full_name=most_appearances
|
| 643 |
+
)
|
| 644 |
+
|
| 645 |
+
st.header('Win Rates')
|
| 646 |
+
win_rate_chart = create_win_rate_chart(metrics)
|
| 647 |
+
st.plotly_chart(win_rate_chart, use_container_width=True)
|
| 648 |
+
|
| 649 |
+
st.header('Appearance Distribution')
|
| 650 |
+
appearance_chart = create_appearance_chart(metrics)
|
| 651 |
+
st.plotly_chart(appearance_chart, use_container_width=True)
|
| 652 |
+
|
| 653 |
+
st.header('Head-to-Head Analysis')
|
| 654 |
+
matrix_chart = create_head_to_head_matrix(df)
|
| 655 |
+
st.plotly_chart(matrix_chart, use_container_width=True)
|
| 656 |
+
|
| 657 |
+
st.header('Detailed Metrics')
|
| 658 |
+
metrics_df = pd.DataFrame.from_dict(metrics, orient='index')
|
| 659 |
+
metrics_df['win_rate'] = metrics_df['win_rate'].round(2)
|
| 660 |
+
metrics_df.drop(["avg_response_time","response_time_std"],axis=1,inplace=True)
|
| 661 |
+
# metrics_df['avg_response_time'] = metrics_df['avg_response_time'].round(3)
|
| 662 |
+
metrics_df.index = [get_model_abbreviation(model) for model in metrics_df.index]
|
| 663 |
+
st.dataframe(metrics_df)
|
| 664 |
+
|
| 665 |
+
st.header('Full Dataframe')
|
| 666 |
+
df = df.drop('path', axis=1)
|
| 667 |
+
df = df.drop(['Ori Apex_duration', 'Ori Apex XT_duration', 'deepgram_duration', 'Ori Swift_duration', 'Ori Prime_duration','azure_duration','email'],axis=1)
|
| 668 |
+
st.dataframe(df)
|
| 669 |
+
else:
|
| 670 |
+
st.write("No Data to show")
|
| 671 |
else:
|
| 672 |
st.write('You have not entered your email and name yet')
|
| 673 |
st.write('Please Navigate to login page in the dropdown menu')
|
|
|
|
| 739 |
return re.match(pattern, name) is not None
|
| 740 |
|
| 741 |
def create_login_page():
|
| 742 |
+
st.title("Welcome to the Speech-To-Text Arena")
|
| 743 |
|
| 744 |
if 'logged_in' not in st.session_state:
|
| 745 |
st.session_state.logged_in = False
|
results/audios/temp
ADDED
|
File without changes
|
results/results.csv
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
email,path,Ori Apex_score,Ori Apex XT_score,deepgram_score,Ori Swift_score,Ori Prime_score,Ori Apex_appearance,Ori Apex XT_appearance,deepgram_appearance,Ori Swift_appearance,Ori Prime_appearance,Ori Apex_duration,Ori Apex XT_duration,deepgram_duration,Ori Swift_dur
|