feat: Optimise colour mapping for visible models only

app.py

@@ -127,6 +127,8 @@ paper](https://aclanthology.org/2023.nodalida-1.20):
 
 UPDATE_FREQUENCY_MINUTES = 5
 MIN_COLOUR_DISTANCE_BETWEEN_MODELS = 200
+DEFAULT_LANGUAGES = ["Danish"]
+DEFAULT_MODELS = ["gpt-4-0613", "mistralai/Mistral-7B-v0.1"]
 
 
 class Task(BaseModel):
@@ -170,12 +172,14 @@ INFORMATION_EXTRACTION = Task(name="information extraction", metric="micro_f1_no
 ALL_TASKS = [obj for obj in globals().values() if isinstance(obj, Task)]
 
 DANISH = Language(code="da", name="Danish")
-NORWEGIAN = Language(code="no", name="Norwegian")
-SWEDISH = Language(code="sv", name="Swedish")
-ICELANDIC = Language(code="is", name="Icelandic")
-GERMAN = Language(code="de", name="German")
 DUTCH = Language(code="nl", name="Dutch")
 ENGLISH = Language(code="en", name="English")
+FAROESE = Language(code="fo", name="Faroese")
+FRENCH = Language(code="fr", name="French")
+GERMAN = Language(code="de", name="German")
+ICELANDIC = Language(code="is", name="Icelandic")
+NORWEGIAN = Language(code="no", name="Norwegian")
+SWEDISH = Language(code="sv", name="Swedish")
 ALL_LANGUAGES = {
     obj.name: obj for obj in globals().values() if isinstance(obj, Language)
 }
@@ -187,6 +191,9 @@ DATASETS = [
     Dataset(name="sb10k", language=GERMAN, task=TEXT_CLASSIFICATION),
     Dataset(name="dutch-social", language=DUTCH, task=TEXT_CLASSIFICATION),
     Dataset(name="sst5", language=ENGLISH, task=TEXT_CLASSIFICATION),
+    Dataset(name="fosent", language=FAROESE, task=TEXT_CLASSIFICATION),
+    Dataset(name="allocine", language=FRENCH, task=TEXT_CLASSIFICATION),
+
     Dataset(name="suc3", language=SWEDISH, task=INFORMATION_EXTRACTION),
     Dataset(name="dansk", language=DANISH, task=INFORMATION_EXTRACTION),
     Dataset(name="norne-nb", language=NORWEGIAN, task=INFORMATION_EXTRACTION),
@@ -195,6 +202,9 @@ DATASETS = [
     Dataset(name="germeval", language=GERMAN, task=INFORMATION_EXTRACTION),
     Dataset(name="conll-nl", language=DUTCH, task=INFORMATION_EXTRACTION),
     Dataset(name="conll-en", language=ENGLISH, task=INFORMATION_EXTRACTION),
+    Dataset(name="fone", language=FAROESE, task=INFORMATION_EXTRACTION),
+    Dataset(name="eltec", language=FRENCH, task=INFORMATION_EXTRACTION),
+
     Dataset(name="scala-sv", language=SWEDISH, task=GRAMMAR),
     Dataset(name="scala-da", language=DANISH, task=GRAMMAR),
     Dataset(name="scala-nb", language=NORWEGIAN, task=GRAMMAR),
@@ -203,6 +213,9 @@ DATASETS = [
     Dataset(name="scala-de", language=GERMAN, task=GRAMMAR),
     Dataset(name="scala-nl", language=DUTCH, task=GRAMMAR),
     Dataset(name="scala-en", language=ENGLISH, task=GRAMMAR),
+    Dataset(name="scala-fo", language=FAROESE, task=GRAMMAR),
+    Dataset(name="scala-fr", language=FRENCH, task=GRAMMAR),
+
     Dataset(name="scandiqa-da", language=DANISH, task=READING_COMPREHENSION),
     Dataset(name="norquad", language=NORWEGIAN, task=READING_COMPREHENSION),
     Dataset(name="scandiqa-sv", language=SWEDISH, task=READING_COMPREHENSION),
@@ -210,6 +223,9 @@ DATASETS = [
     Dataset(name="germanquad", language=GERMAN, task=READING_COMPREHENSION),
     Dataset(name="squad", language=ENGLISH, task=READING_COMPREHENSION),
     Dataset(name="squad-nl", language=DUTCH, task=READING_COMPREHENSION),
+    Dataset(name="foqa", language=FAROESE, task=READING_COMPREHENSION),
+    Dataset(name="fquad", language=FRENCH, task=READING_COMPREHENSION),
+
     Dataset(name="nordjylland-news", language=DANISH, task=SUMMARISATION),
     Dataset(name="mlsum", language=GERMAN, task=SUMMARISATION),
     Dataset(name="rrn", language=ICELANDIC, task=SUMMARISATION),
@@ -217,6 +233,8 @@ DATASETS = [
     Dataset(name="wiki-lingua-nl", language=DUTCH, task=SUMMARISATION),
     Dataset(name="swedn", language=SWEDISH, task=SUMMARISATION),
     Dataset(name="cnn-dailymail", language=ENGLISH, task=SUMMARISATION),
+    Dataset(name="orange-sum", language=FRENCH, task=SUMMARISATION),
+
     Dataset(name="danish-citizen-tests", language=DANISH, task=KNOWLEDGE),
     Dataset(name="danske-talemaader", language=DANISH, task=KNOWLEDGE),
     Dataset(name="mmlu-no", language=NORWEGIAN, task=KNOWLEDGE),
@@ -225,6 +243,8 @@ DATASETS = [
     Dataset(name="mmlu-de", language=GERMAN, task=KNOWLEDGE),
     Dataset(name="mmlu-nl", language=DUTCH, task=KNOWLEDGE),
     Dataset(name="mmlu", language=ENGLISH, task=KNOWLEDGE),
+    Dataset(name="mmlu-fr", language=FRENCH, task=KNOWLEDGE),
+
     Dataset(name="hellaswag-da", language=DANISH, task=REASONING),
     Dataset(name="hellaswag-no", language=NORWEGIAN, task=REASONING),
     Dataset(name="hellaswag-sv", language=SWEDISH, task=REASONING),
@@ -232,6 +252,7 @@ DATASETS = [
     Dataset(name="hellaswag-de", language=GERMAN, task=REASONING),
     Dataset(name="hellaswag-nl", language=DUTCH, task=REASONING),
     Dataset(name="hellaswag", language=ENGLISH, task=REASONING),
+    Dataset(name="hellaswag-fr", language=FRENCH, task=REASONING),
 ]
 
 
@@ -254,7 +275,8 @@ def main() -> None:
     global colour_mapping
     global seed
     seed = 4242
-    update_colour_mapping(results_dfs=results_dfs)
+    colour_mapping = dict()
+    update_colour_mapping(model_ids=DEFAULT_MODELS)
 
     with gr.Blocks(theme=gr.themes.Monochrome()) as demo:
         gr.Markdown(INTRO_MARKDOWN)
@@ -266,7 +288,7 @@ def main() -> None:
                         choices=all_languages,
                         multiselect=True,
                         label="Languages",
-                        value=["Danish"],
+                        value=DEFAULT_LANGUAGES,
                         interactive=True,
                         scale=2,
                     )
@@ -274,7 +296,7 @@ def main() -> None:
                         choices=danish_models,
                         multiselect=True,
                         label="Models",
-                        value=["gpt-4-0613", "mistralai/Mistral-7B-v0.1"],
+                        value=DEFAULT_MODELS,
                         interactive=True,
                         scale=2,
                     )
@@ -310,11 +332,6 @@ def main() -> None:
                         interactive=True,
                         scale=1,
                     )
-                    update_colours_button = gr.Button(
-                        value="Update colours",
-                        interactive=True,
-                        scale=1,
-                    )
                 with gr.Row():
                     plot = gr.Plot(
                         value=produce_radial_plot(
@@ -339,7 +356,7 @@ def main() -> None:
             fn=partial(update_model_ids_dropdown, results_dfs=results_dfs),
             inputs=[language_names_dropdown, model_ids_dropdown],
             outputs=model_ids_dropdown,
-        )
+        ).then(fn=update_colour_mapping, inputs=model_ids_dropdown)
 
         # Update plot when anything changes
         update_plot_kwargs = dict(
@@ -357,16 +374,23 @@ def main() -> None:
             ],
             outputs=plot,
         )
-        language_names_dropdown.change(**update_plot_kwargs)
-        model_ids_dropdown.change(**update_plot_kwargs)
-        use_rank_score_checkbox.change(**update_plot_kwargs)
-        show_scale_checkbox.change(**update_plot_kwargs)
-        plot_width_slider.change(**update_plot_kwargs)
-        plot_height_slider.change(**update_plot_kwargs)
-
-        # Update colours when the button is clicked
-        update_colours_button.click(
-            fn=partial(update_colour_mapping, results_dfs=results_dfs),
+        language_names_dropdown.change(
+            fn=update_colour_mapping, inputs=model_ids_dropdown
+        ).then(**update_plot_kwargs)
+        model_ids_dropdown.change(
+            fn=update_colour_mapping, inputs=model_ids_dropdown
+        ).then(**update_plot_kwargs)
+        use_rank_score_checkbox.change(
+            fn=update_colour_mapping, inputs=model_ids_dropdown
+        ).then(**update_plot_kwargs)
+        show_scale_checkbox.change(
+            fn=update_colour_mapping, inputs=model_ids_dropdown
+        ).then(**update_plot_kwargs)
+        plot_width_slider.change(
+            fn=update_colour_mapping, inputs=model_ids_dropdown
+        ).then(**update_plot_kwargs)
+        plot_height_slider.change(
+            fn=update_colour_mapping, inputs=model_ids_dropdown
         ).then(**update_plot_kwargs)
 
     demo.launch()
@@ -703,29 +727,23 @@ def fetch_results() -> dict[Language, pd.DataFrame]:
     return results_dfs
 
 
-def update_colour_mapping(results_dfs: dict[Language, pd.DataFrame]) -> None:
+def update_colour_mapping(model_ids: list[str]) -> None:
     """Get a mapping from model ids to RGB triplets.
 
     Args:
-        results_dfs:
-            The results dataframes for each language.
+        model_ids:
+            The model ids to update the colour
     """
     global colour_mapping
     global seed
     seed += 1
 
-    gr.Info(f"Updating colour mapping...")
-
-    # Get distinct RGB values for all models
-    all_models = list(
-        {model_id for df in results_dfs.values() for model_id in df.index}
-    )
-    colour_mapping = dict()
-
     for i in it.count():
         min_colour_distance = MIN_COLOUR_DISTANCE_BETWEEN_MODELS - i
-        retries_left = 10 * len(all_models)
-        for model_id in all_models:
+        retries_left = 10 * len(model_ids)
+        for model_id in model_ids:
+            if model_id in colour_mapping:
+                continue
             random.seed(hash(model_id) + i + seed)
             r, g, b = 0, 0, 0
             too_bright, similar_to_other_model = True, True