{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":633420468,"defaultBranch":"jp-stable","name":"lm-evaluation-harness","ownerLogin":"Stability-AI","currentUserCanPush":false,"isFork":true,"isEmpty":false,"createdAt":"2023-04-27T13:09:46.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/100950301?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1708937757.0","currentOid":""},"activityList":{"items":[{"before":"d8e6ada167de1e48887af34188b29d54a282241d","after":"080419ef6d1c9a94245196a061ef687aa2f16554","ref":"refs/heads/lm-bench","pushedAt":"2024-09-13T18:13:17.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add olmoe-1b-7b-0924","shortMessageHtmlLink":"feat(results): add olmoe-1b-7b-0924"}},{"before":"d61a32ddccfc2f40b97a9d8dfa7b66d09535b6e3","after":"d8e6ada167de1e48887af34188b29d54a282241d","ref":"refs/heads/lm-bench","pushedAt":"2024-05-17T18:21:58.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add falcon-11b and xverse","shortMessageHtmlLink":"feat(results): add falcon-11b and xverse"}},{"before":"9883885a6c38e1c1d787f4db54329dd483762bd6","after":"d61a32ddccfc2f40b97a9d8dfa7b66d09535b6e3","ref":"refs/heads/lm-bench","pushedAt":"2024-04-18T17:19:17.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): update table","shortMessageHtmlLink":"feat(results): update table"}},{"before":"d26acea69b4e2a2f823fbe5eefb174e8aaee8bac","after":"9883885a6c38e1c1d787f4db54329dd483762bd6","ref":"refs/heads/lm-bench","pushedAt":"2024-04-18T17:18:25.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add `llama-3-8b`","shortMessageHtmlLink":"feat(results): add <code>llama-3-8b</code>"}},{"before":"083b9edd512716e0b2d463e209cc082dd5dfb96e","after":"d26acea69b4e2a2f823fbe5eefb174e8aaee8bac","ref":"refs/heads/lm-bench","pushedAt":"2024-04-18T16:29:11.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): update benchmark results table","shortMessageHtmlLink":"feat(results): update benchmark results table"}},{"before":"1e5b66f6035be6593027840ea8603e936bbff843","after":"083b9edd512716e0b2d463e209cc082dd5dfb96e","ref":"refs/heads/lm-bench","pushedAt":"2024-04-18T15:58:27.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add `stablelm-2-12b`","shortMessageHtmlLink":"feat(results): add <code>stablelm-2-12b</code>"}},{"before":"b131deb57e516169aed4fc46f0c53105eebbeaf8","after":"522a828f7cfcdb71b228938f1ac8ee8078add8da","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-04-14T17:08:03.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add `mistralai/Mistral-7B-v0.1`","shortMessageHtmlLink":"feat(results): add <code>mistralai/Mistral-7B-v0.1</code>"}},{"before":"1040fa9979c06db1ffdc2c76b25bf73cdf665f60","after":"b131deb57e516169aed4fc46f0c53105eebbeaf8","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-04-14T05:31:31.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"format","shortMessageHtmlLink":"format"}},{"before":"4dc2b6d3f45ea25a28b4ed79a036ddc73d3a6c3c","after":"1040fa9979c06db1ffdc2c76b25bf73cdf665f60","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-04-14T05:30:08.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add stablelm-2-12b","shortMessageHtmlLink":"feat(results): add stablelm-2-12b"}},{"before":"fee2b90cc1c61df6f7a2567df749f3793942f8f4","after":"4dc2b6d3f45ea25a28b4ed79a036ddc73d3a6c3c","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-04-14T05:28:44.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(mmlu): add mmlu-only results view script","shortMessageHtmlLink":"feat(mmlu): add mmlu-only results view script"}},{"before":"6b726f5f972c1fde7dfa27fe502541a65b9fb75b","after":"fee2b90cc1c61df6f7a2567df749f3793942f8f4","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-04-14T05:27:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"update .gitignore to drop `.env` tracking","shortMessageHtmlLink":"update .gitignore to drop <code>.env</code> tracking"}},{"before":"aeb6aea1d70e924a8b5e002a9b1c0d46079d2ecd","after":"6b726f5f972c1fde7dfa27fe502541a65b9fb75b","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-04-05T05:22:01.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat: add multilingual `x` benchmarks","shortMessageHtmlLink":"feat: add multilingual <code>x</code> benchmarks"}},{"before":"2e54151516e14ba43d364c1829d294598bdfbffc","after":"1e5b66f6035be6593027840ea8603e936bbff843","ref":"refs/heads/lm-bench","pushedAt":"2024-04-05T04:01:19.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat: add `interlm2-base-20b` results","shortMessageHtmlLink":"feat: add <code>interlm2-base-20b</code> results"}},{"before":"e5cc65d4c3b4a97b6a9ff74522a48aa936e110b3","after":"aeb6aea1d70e924a8b5e002a9b1c0d46079d2ecd","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-04-05T03:00:03.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat: add multilingual `x` benchmarks","shortMessageHtmlLink":"feat: add multilingual <code>x</code> benchmarks"}},{"before":"9e17c4713445e6ba80d133743e7626d32e0044a6","after":"2e54151516e14ba43d364c1829d294598bdfbffc","ref":"refs/heads/lm-bench","pushedAt":"2024-04-05T00:52:02.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat: add yi-9B","shortMessageHtmlLink":"feat: add yi-9B"}},{"before":"5f2592f491f92c4a55727fb70ab2ee030006ade4","after":"9e17c4713445e6ba80d133743e7626d32e0044a6","ref":"refs/heads/lm-bench","pushedAt":"2024-04-04T20:32:25.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"fix(llama-2-13b): run with bs=8","shortMessageHtmlLink":"fix(llama-2-13b): run with bs=8"}},{"before":"6b9eb876148fbe245522fd75efbcc595db7709b5","after":"e5cc65d4c3b4a97b6a9ff74522a48aa936e110b3","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-03-29T21:38:11.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add Qwen-1.5 14B","shortMessageHtmlLink":"feat(results): add Qwen-1.5 14B"}},{"before":"619fea9b09777d7afdbf48354a1dd538d0700394","after":"6b9eb876148fbe245522fd75efbcc595db7709b5","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-03-29T21:37:34.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add gemma-7b","shortMessageHtmlLink":"feat(results): add gemma-7b"}},{"before":"b9b5de3d829ba3d1dc725db08be383d397e28e0c","after":"619fea9b09777d7afdbf48354a1dd538d0700394","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-03-08T16:08:28.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"Merge branch 'stablelm-2/multilingual-bench' of https://github.com/Stability-AI/lm-evaluation-harness into stablelm-2/multilingual-bench","shortMessageHtmlLink":"Merge branch 'stablelm-2/multilingual-bench' of <a href=\"https://github.com/Stability-AI/lm-evaluation-harness\">https://github.com/St…</a>"}},{"before":"530fa8afa14a7f2898611c5f4a5295ab27f9a7b1","after":"b9b5de3d829ba3d1dc725db08be383d397e28e0c","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-03-05T03:24:21.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"fix(bench): add check to avoid overwriting json results","shortMessageHtmlLink":"fix(bench): add check to avoid overwriting json results"}},{"before":"db86e4e510fc567419ba8b0d2c541a991d0251b3","after":"530fa8afa14a7f2898611c5f4a5295ab27f9a7b1","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-03-04T22:40:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add mistral 7b instruct v0.2","shortMessageHtmlLink":"feat(results): add mistral 7b instruct v0.2"}},{"before":"81d63842e56138b0a1f81ba121c8a0d81ee2e22b","after":"5f2592f491f92c4a55727fb70ab2ee030006ade4","ref":"refs/heads/lm-bench","pushedAt":"2024-02-29T16:44:30.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add `stablelm-2-1.6b`","shortMessageHtmlLink":"feat(results): add <code>stablelm-2-1.6b</code>"}},{"before":"783de5e3b6519247e7b10454f2e80445ce8833a3","after":"81d63842e56138b0a1f81ba121c8a0d81ee2e22b","ref":"refs/heads/lm-bench","pushedAt":"2024-02-28T05:55:48.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add `qwen-1.5-14b`","shortMessageHtmlLink":"feat(results): add <code>qwen-1.5-14b</code>"}},{"before":"8e46c9a062200f001e56f0865821378168b8bdb8","after":"783de5e3b6519247e7b10454f2e80445ce8833a3","ref":"refs/heads/lm-bench","pushedAt":"2024-02-28T05:51:06.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add `gemma-7b`","shortMessageHtmlLink":"feat(results): add <code>gemma-7b</code>"}},{"before":"3eedcbb203fa5fb01aa3d2299c618ce65b760b5f","after":"db86e4e510fc567419ba8b0d2c541a991d0251b3","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-02-27T14:53:47.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"fix: view acc norm for arc-c fr","shortMessageHtmlLink":"fix: view acc norm for arc-c fr"}},{"before":"4425dbaf612a1be50916e13c936039c1901998f6","after":"3eedcbb203fa5fb01aa3d2299c618ce65b760b5f","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-02-27T05:15:50.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(results): add Qwen-1.5 1.8B","shortMessageHtmlLink":"feat(results): add Qwen-1.5 1.8B"}},{"before":"8d589a0351c12028867009c71151f16502bc80a9","after":"4425dbaf612a1be50916e13c936039c1901998f6","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-02-26T22:45:31.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"fix(mmlu_average.py): provide proper arg helper","shortMessageHtmlLink":"fix(mmlu_average.py): provide proper arg helper"}},{"before":"bf111bad64d792c536d61eafe1af208c1723fd25","after":"8d589a0351c12028867009c71151f16502bc80a9","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-02-26T13:50:48.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"marcobellagente93","name":null,"path":"/marcobellagente93","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/56817594?s=80&v=4"},"commit":{"message":"special tokens support","shortMessageHtmlLink":"special tokens support"}},{"before":"e167f5daaafd9d3ff5ef3682787ae47f16c11bdb","after":"bf111bad64d792c536d61eafe1af208c1723fd25","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-02-26T13:47:29.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"marcobellagente93","name":null,"path":"/marcobellagente93","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/56817594?s=80&v=4"},"commit":{"message":"Merge branch 'stablelm-2/multilingual-bench' of https://github.com/Stability-AI/lm-evaluation-harness into stablelm-2/multilingual-bench","shortMessageHtmlLink":"Merge branch 'stablelm-2/multilingual-bench' of <a href=\"https://github.com/Stability-AI/lm-evaluation-harness\">https://github.com/St…</a>"}},{"before":"4c08e11e81b4efc49c23ae1b6445d6fc6f94f6dc","after":"e167f5daaafd9d3ff5ef3682787ae47f16c11bdb","ref":"refs/heads/stablelm-2/multilingual-bench","pushedAt":"2024-02-26T13:36:02.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"jon-tow","name":"Jonathan Tow","path":"/jon-tow","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/41410219?s=80&v=4"},"commit":{"message":"feat(stablelm_2): add multilingual results for tech report","shortMessageHtmlLink":"feat(stablelm_2): add multilingual results for tech report"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"startCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOS0xM1QxODoxMzoxNy4wMDAwMDBazwAAAAS1xPTy","endCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wMi0yNlQxMzozNjowMi4wMDAwMDBazwAAAAQFavWv"}},"title":"Activity · Stability-AI/lm-evaluation-harness"}