fix: remove llama3.0 from models and defaults (#1054)

* fix: notebook that used llama3.0
* fix: overlapping ports between studio and fastapi test in notebook runner

.github/workflows/sdk-tests.yml

@@ -221,7 +221,7 @@ jobs:
       studio-backend:
         image: registry.gitlab.aleph-alpha.de/product/studio/backend:latest
         ports:
-          - "8000:8000"
+          - "8001:8000"
         env:
           DATABASE_URL: "postgres:5432"
           POSTGRES_DB: "il_sdk"
@@ -264,6 +264,6 @@ jobs:
           ARGILLA_API_URL: "http://localhost:6900/"
           ARGILLA_API_KEY: "argilla.apikey"
           CLIENT_URL: ${{ secrets.CLIENT_URL }}
-          STUDIO_URL: "http://localhost:8000"
+          STUDIO_URL: "http://localhost:8001"
         run: |
           ./scripts/notebook_runner.sh

CHANGELOG.md

@@ -7,7 +7,7 @@
 ### Deprecations 
 ...
 ### Breaking Changes
-...
+ - The default model for `Llama3InstructModel` is now `llama-3.1-8b-instruct` instead of `llama-3-8b-instruct`. We also removed the llama3.0 models from the recommended models of the `Llama3InstructModel`.
 
 ## 6.0.0
 

scripts/fastapi_example_test.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env -S bash -eu -o pipefail
-
+echo "Testing the fastapi app"
 # start the server in the background
 hypercorn src/documentation/fastapi_example:app --bind localhost:8000 &
 server_pid=$!

src/documentation/elo_qa_eval.ipynb

@@ -285,7 +285,7 @@
    "outputs": [],
    "source": [
     "elo_qa_evaluation_logic = EloQaEvaluationLogic(\n",
-    "    model=Llama3InstructModel(name=\"llama-3-8b-instruct\")\n",
+    "    model=Llama3InstructModel(name=\"llama-3.1-8b-instruct\")\n",
     ")\n",
     "\n",
     "evaluator = IncrementalEvaluator(\n",
@@ -312,6 +312,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# check that the evaluation did not crash\n",
+    "failed_evals = evaluation_repository.failed_example_evaluations(\n",
+    "    evaluation_overview.id, Matches\n",
+    ")\n",
+    "assert (\n",
+    "    len(failed_evals) == 0\n",
+    "), f\"There are failed evaluations when there should not be:\\n {failed_evals}\"\n",
+    "\n",
     "# ensure that for each example there are evaluated comparisons\n",
     "for example_evaluation in evaluation_repository.example_evaluations(\n",
     "    evaluation_overview.id, Matches\n",
@@ -598,7 +606,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.12.2"
   }
  },
  "nbformat": 4,

src/intelligence_layer/core/model.py

@@ -513,7 +513,7 @@ class Llama3InstructModel(ControlModel):
 
     Args:
         name: The name of a valid llama-3 model.
-            Defaults to `llama-3-8b-instruct`
+            Defaults to `llama-3.1-8b-instruct`
         client: Aleph Alpha client instance for running model related API calls.
             Defaults to :class:`LimitedConcurrencyClient`
     """
@@ -529,15 +529,13 @@ class Llama3InstructModel(ControlModel):
     )
 
     RECOMMENDED_MODELS: ClassVar[list[str]] = [
-        "llama-3-8b-instruct",
-        "llama-3-70b-instruct",
         "llama-3.1-8b-instruct",
         "llama-3.1-70b-instruct",
     ]
 
     def __init__(
         self,
-        name: str = "llama-3-8b-instruct",
+        name: str = "llama-3.1-8b-instruct",
         client: Optional[AlephAlphaClientProtocol] = None,
     ) -> None:
         super().__init__(name, client)

tests/core/test_model.py

@@ -116,7 +116,7 @@ def test_models_know_their_context_size(client: AlephAlphaClientProtocol) -> Non
     )
     assert AlephAlphaModel(client=client, name="luminous-base").context_size == 2048
     assert (
-        Llama3InstructModel(client=client, name="llama-3-8b-instruct").context_size
+        Llama3InstructModel(client=client, name="llama-3.1-8b-instruct").context_size
         == 8192
     )