00_retrievers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.llms import LlamaCpp\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.chains import LLMChain\n",
    "from langchain.callbacks.manager import CallbackManager\n",
    "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import RetrievalQA\n",
    "from utils import LlamaEmbeddingFunction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import TextLoader\n",
    "loader = TextLoader('./data/state_of_union/state_of_the_union.txt', encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "model_name = \"BAAI/bge-large-en-v1.5\"\n",
    "model_kwargs = {'device': 'cuda'}\n",
    "encode_kwargs = {'normalize_embeddings': True} # set True to compute cosine similarity\n",
    "model = HuggingFaceBgeEmbeddings(\n",
    "    model_name=model_name,\n",
    "    model_kwargs=model_kwargs,\n",
    "    encode_kwargs=encode_kwargs,\n",
    "    query_instruction=\"为这个句子生成表示以用于检索相关文章：\"\n",
    ")\n",
    "model.query_instruction = \"为这个句子生成表示以用于检索相关文章：\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_model_loader: loaded meta data with 19 key-value pairs and 363 tensors from /Users/Shared/Models/llama-2-13b.Q5_K_M.gguf (version GGUF V2 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q5_K     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:           blk.0.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:            blk.0.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:            blk.0.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:         blk.0.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:              blk.0.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.1.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.1.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:            blk.1.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:         blk.1.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:              blk.1.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:          blk.10.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:           blk.10.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:           blk.10.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:             blk.10.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:           blk.10.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:             blk.10.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:        blk.10.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:             blk.10.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:             blk.10.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:          blk.11.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:           blk.11.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:           blk.11.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:             blk.11.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:           blk.11.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:             blk.11.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:        blk.11.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:             blk.11.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:             blk.11.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:          blk.12.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:           blk.12.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:           blk.12.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:             blk.12.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:           blk.12.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:             blk.12.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:        blk.12.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:             blk.12.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:             blk.12.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:          blk.13.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:           blk.13.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:           blk.13.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:             blk.13.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:           blk.13.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:             blk.13.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:        blk.13.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:             blk.13.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:             blk.13.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:          blk.14.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:           blk.14.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:           blk.14.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:             blk.14.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:           blk.14.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:             blk.14.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:        blk.14.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:             blk.14.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:             blk.14.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:             blk.15.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:             blk.15.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:           blk.2.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:            blk.2.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:            blk.2.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:              blk.2.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.2.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:              blk.2.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:         blk.2.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:              blk.2.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:              blk.2.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:           blk.3.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:            blk.3.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:            blk.3.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:              blk.3.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.3.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:              blk.3.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:         blk.3.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:              blk.3.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:              blk.3.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:           blk.4.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:            blk.4.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:            blk.4.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:              blk.4.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.4.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:              blk.4.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:         blk.4.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:              blk.4.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:              blk.4.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:           blk.5.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:            blk.5.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:            blk.5.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:              blk.5.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:            blk.5.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:              blk.5.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:         blk.5.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:              blk.5.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:              blk.5.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:           blk.6.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:            blk.6.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:            blk.6.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:              blk.6.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:            blk.6.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:              blk.6.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:         blk.6.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:              blk.6.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:              blk.6.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:           blk.7.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:            blk.7.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:            blk.7.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:              blk.7.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:            blk.7.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:              blk.7.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:         blk.7.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:              blk.7.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:              blk.7.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:           blk.8.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:            blk.8.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:            blk.8.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:              blk.8.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:            blk.8.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:              blk.8.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:         blk.8.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:              blk.8.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:              blk.8.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:           blk.9.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:            blk.9.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:            blk.9.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:              blk.9.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:            blk.9.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:              blk.9.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:         blk.9.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:              blk.9.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:              blk.9.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:          blk.15.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:           blk.15.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.15.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.15.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:        blk.15.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.16.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.16.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:           blk.16.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.16.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.16.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:        blk.16.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:             blk.16.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.17.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.17.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:           blk.17.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:           blk.17.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:             blk.17.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:        blk.17.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:             blk.17.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.18.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.18.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:           blk.18.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:           blk.18.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:             blk.18.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:        blk.18.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:             blk.18.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.19.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.19.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:           blk.19.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:           blk.19.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:             blk.19.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:        blk.19.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:             blk.19.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.20.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.20.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:           blk.20.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:           blk.20.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:             blk.20.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:        blk.20.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:             blk.20.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.21.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.21.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:           blk.21.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:           blk.21.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:             blk.21.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:        blk.21.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:             blk.21.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.22.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.22.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:           blk.22.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:           blk.22.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:             blk.22.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:        blk.22.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:             blk.22.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.23.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.23.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:           blk.23.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:           blk.23.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:             blk.23.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:        blk.23.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:             blk.23.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.24.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.24.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:           blk.24.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:           blk.24.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:             blk.24.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:        blk.24.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:             blk.24.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.25.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.25.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:           blk.25.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:           blk.25.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:             blk.25.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:        blk.25.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:             blk.25.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.26.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.26.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:           blk.26.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:           blk.26.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:             blk.26.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:        blk.26.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:             blk.26.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.27.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.27.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:           blk.27.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:           blk.27.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:             blk.27.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:        blk.27.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:             blk.27.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.28.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.28.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:           blk.28.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:           blk.28.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:             blk.28.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:        blk.28.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:             blk.28.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.29.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.29.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:           blk.29.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:           blk.29.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:             blk.29.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:        blk.29.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:             blk.29.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:           blk.30.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:             blk.30.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:        blk.30.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:             blk.30.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:                    output.weight q6_K     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:          blk.30.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:           blk.30.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:           blk.30.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:          blk.31.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:           blk.31.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:           blk.31.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:           blk.31.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:             blk.31.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:        blk.31.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:             blk.31.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:          blk.32.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  291:           blk.32.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  292:           blk.32.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  293:             blk.32.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  294:           blk.32.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  295:             blk.32.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  296:        blk.32.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  297:             blk.32.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  298:             blk.32.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  299:          blk.33.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  300:           blk.33.ffn_down.weight q5_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  301:           blk.33.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  302:             blk.33.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  303:           blk.33.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  304:             blk.33.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  305:        blk.33.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  306:             blk.33.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  307:             blk.33.attn_v.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  308:          blk.34.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  309:           blk.34.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  310:           blk.34.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  311:             blk.34.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  312:           blk.34.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  313:             blk.34.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  314:        blk.34.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  315:             blk.34.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  316:             blk.34.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  317:          blk.35.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  318:           blk.35.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  319:           blk.35.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  320:             blk.35.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  321:           blk.35.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  322:             blk.35.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  323:        blk.35.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  324:             blk.35.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  325:             blk.35.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  326:          blk.36.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  327:           blk.36.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  328:           blk.36.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  329:             blk.36.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  330:           blk.36.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  331:             blk.36.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  332:        blk.36.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  333:             blk.36.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  334:             blk.36.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  335:          blk.37.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  336:           blk.37.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  337:           blk.37.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  338:             blk.37.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  339:           blk.37.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  340:             blk.37.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  341:        blk.37.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  342:             blk.37.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  343:             blk.37.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  344:          blk.38.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  345:           blk.38.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  346:           blk.38.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  347:             blk.38.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  348:           blk.38.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  349:             blk.38.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  350:        blk.38.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  351:             blk.38.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  352:             blk.38.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  353:          blk.39.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  354:           blk.39.ffn_down.weight q6_K     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  355:           blk.39.ffn_gate.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  356:             blk.39.ffn_up.weight q5_K     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  357:           blk.39.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  358:             blk.39.attn_k.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  359:        blk.39.attn_output.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  360:             blk.39.attn_q.weight q5_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  361:             blk.39.attn_v.weight q6_K     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  362:               output_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str     \n",
      "llama_model_loader: - kv   1:                               general.name str     \n",
      "llama_model_loader: - kv   2:                       llama.context_length u32     \n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32     \n",
      "llama_model_loader: - kv   4:                          llama.block_count u32     \n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32     \n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32     \n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32     \n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32     \n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32     \n",
      "llama_model_loader: - kv  10:                          general.file_type u32     \n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str     \n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr     \n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr     \n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr     \n",
      "llama_model_loader: - kv  15:                tokenizer.ggml.bos_token_id u32     \n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.eos_token_id u32     \n",
      "llama_model_loader: - kv  17:            tokenizer.ggml.unknown_token_id u32     \n",
      "llama_model_loader: - kv  18:               general.quantization_version u32     \n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q5_K:  241 tensors\n",
      "llama_model_loader: - type q6_K:   41 tensors\n",
      "llm_load_print_meta: format           = GGUF V2 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = mostly Q5_K - Medium\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 8.60 GiB (5.67 BPW) \n",
      "llm_load_print_meta: general.name   = LLaMA v2\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 2 '</s>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.12 MB\n",
      "llm_load_tensors: mem required  = 8801.75 MB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 2048\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: kv self size  = 1600.00 MB\n",
      "llama_new_context_with_model: compute buffer total size = 199.88 MB\n",
      "llama_new_context_with_model: max tensor size =   128.17 MB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  8802.34 MB, ( 8802.97 / 49152.00)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =  1602.00 MB, (10404.97 / 49152.00)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =   194.02 MB, (10598.98 / 49152.00)\n",
      "AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n"
     ]
    }
   ],
   "source": [
    "# from langchain.llms import LlamaCpp\n",
    "llm = LlamaCpp(\n",
    "    model_path=\"/Users/Shared/Models/llama-2-13b.Q5_K_M.gguf\",\n",
    "    n_gpu_layers=99,\n",
    "    n_batch=512,\n",
    "    n_ctx=2048,\n",
    "    f16_kv=True,  \n",
    "    callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]),\n",
    "    verbose=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "text_splitter = CharacterTextSplitter(chunk_size=1024, chunk_overlap=512)\n",
    "texts = text_splitter.split_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /embedding (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x16a8407d0>: Failed to establish a new connection: [Errno 61] Connection refused'))\n",
      "PAYLOAD: {'content': 'Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans.  \\n\\nLast year COVID-19 kept us apart. This year we are finally together again. \\n\\nTonight, we meet as Democrats Republicans and Independents. But most importantly as Americans. \\n\\nWith a duty to one another to the American people to the Constitution. \\n\\nAnd with an unwavering resolve that freedom will always triumph over tyranny. \\n\\nSix days ago, Russia’s Vladimir Putin sought to shake the foundations of the free world thinking he could make it bend to his menacing ways. But he badly miscalculated. \\n\\nHe thought he could roll into Ukraine and the world would roll over. Instead he met a wall of strength he never imagined. \\n\\nHe met the Ukrainian people. \\n\\nFrom President Zelenskyy to every Ukrainian, their fearlessness, their courage, their determination, inspires the world.'}\n"
     ]
    },
    {
     "ename": "UnboundLocalError",
     "evalue": "cannot access local variable 'response' where it is not associated with a value",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mConnectionRefusedError\u001b[0m                    Traceback (most recent call last)",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/connection.py:203\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    202\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 203\u001b[0m     sock \u001b[39m=\u001b[39m connection\u001b[39m.\u001b[39;49mcreate_connection(\n\u001b[1;32m    204\u001b[0m         (\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_dns_host, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mport),\n\u001b[1;32m    205\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtimeout,\n\u001b[1;32m    206\u001b[0m         source_address\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msource_address,\n\u001b[1;32m    207\u001b[0m         socket_options\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msocket_options,\n\u001b[1;32m    208\u001b[0m     )\n\u001b[1;32m    209\u001b[0m \u001b[39mexcept\u001b[39;00m socket\u001b[39m.\u001b[39mgaierror \u001b[39mas\u001b[39;00m e:\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     84\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> 85\u001b[0m     \u001b[39mraise\u001b[39;00m err\n\u001b[1;32m     86\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m     87\u001b[0m     \u001b[39m# Break explicitly a reference cycle\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     72\u001b[0m     sock\u001b[39m.\u001b[39mbind(source_address)\n\u001b[0;32m---> 73\u001b[0m sock\u001b[39m.\u001b[39;49mconnect(sa)\n\u001b[1;32m     74\u001b[0m \u001b[39m# Break explicitly a reference cycle\u001b[39;00m\n",
      "\u001b[0;31mConnectionRefusedError\u001b[0m: [Errno 61] Connection refused",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mNewConnectionError\u001b[0m                        Traceback (most recent call last)",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/connectionpool.py:790\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m    789\u001b[0m \u001b[39m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[0;32m--> 790\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_make_request(\n\u001b[1;32m    791\u001b[0m     conn,\n\u001b[1;32m    792\u001b[0m     method,\n\u001b[1;32m    793\u001b[0m     url,\n\u001b[1;32m    794\u001b[0m     timeout\u001b[39m=\u001b[39;49mtimeout_obj,\n\u001b[1;32m    795\u001b[0m     body\u001b[39m=\u001b[39;49mbody,\n\u001b[1;32m    796\u001b[0m     headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[1;32m    797\u001b[0m     chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[1;32m    798\u001b[0m     retries\u001b[39m=\u001b[39;49mretries,\n\u001b[1;32m    799\u001b[0m     response_conn\u001b[39m=\u001b[39;49mresponse_conn,\n\u001b[1;32m    800\u001b[0m     preload_content\u001b[39m=\u001b[39;49mpreload_content,\n\u001b[1;32m    801\u001b[0m     decode_content\u001b[39m=\u001b[39;49mdecode_content,\n\u001b[1;32m    802\u001b[0m     \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mresponse_kw,\n\u001b[1;32m    803\u001b[0m )\n\u001b[1;32m    805\u001b[0m \u001b[39m# Everything went great!\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/connectionpool.py:496\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[1;32m    495\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 496\u001b[0m     conn\u001b[39m.\u001b[39;49mrequest(\n\u001b[1;32m    497\u001b[0m         method,\n\u001b[1;32m    498\u001b[0m         url,\n\u001b[1;32m    499\u001b[0m         body\u001b[39m=\u001b[39;49mbody,\n\u001b[1;32m    500\u001b[0m         headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[1;32m    501\u001b[0m         chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[1;32m    502\u001b[0m         preload_content\u001b[39m=\u001b[39;49mpreload_content,\n\u001b[1;32m    503\u001b[0m         decode_content\u001b[39m=\u001b[39;49mdecode_content,\n\u001b[1;32m    504\u001b[0m         enforce_content_length\u001b[39m=\u001b[39;49menforce_content_length,\n\u001b[1;32m    505\u001b[0m     )\n\u001b[1;32m    507\u001b[0m \u001b[39m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[1;32m    508\u001b[0m \u001b[39m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[1;32m    509\u001b[0m \u001b[39m# With this behaviour, the received response is still readable.\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/connection.py:395\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[1;32m    394\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mputheader(header, value)\n\u001b[0;32m--> 395\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mendheaders()\n\u001b[1;32m    397\u001b[0m \u001b[39m# If we're given a body we start sending that in chunks.\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/http/client.py:1281\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1280\u001b[0m     \u001b[39mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1281\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_output(message_body, encode_chunked\u001b[39m=\u001b[39;49mencode_chunked)\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/http/client.py:1041\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1040\u001b[0m \u001b[39mdel\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1041\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(msg)\n\u001b[1;32m   1043\u001b[0m \u001b[39mif\u001b[39;00m message_body \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m   1044\u001b[0m \n\u001b[1;32m   1045\u001b[0m     \u001b[39m# create a consistent interface to message_body\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/http/client.py:979\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    978\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mauto_open:\n\u001b[0;32m--> 979\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnect()\n\u001b[1;32m    980\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/connection.py:243\u001b[0m, in \u001b[0;36mHTTPConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    242\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mconnect\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 243\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msock \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_new_conn()\n\u001b[1;32m    244\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_tunnel_host:\n\u001b[1;32m    245\u001b[0m         \u001b[39m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/connection.py:218\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    217\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mOSError\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m--> 218\u001b[0m     \u001b[39mraise\u001b[39;00m NewConnectionError(\n\u001b[1;32m    219\u001b[0m         \u001b[39mself\u001b[39m, \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mFailed to establish a new connection: \u001b[39m\u001b[39m{\u001b[39;00me\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m    220\u001b[0m     ) \u001b[39mfrom\u001b[39;00m \u001b[39me\u001b[39;00m\n\u001b[1;32m    222\u001b[0m \u001b[39m# Audit hooks are only available in Python 3.8+\u001b[39;00m\n",
      "\u001b[0;31mNewConnectionError\u001b[0m: <urllib3.connection.HTTPConnection object at 0x16a8407d0>: Failed to establish a new connection: [Errno 61] Connection refused",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mMaxRetryError\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/requests/adapters.py:486\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    485\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 486\u001b[0m     resp \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49murlopen(\n\u001b[1;32m    487\u001b[0m         method\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mmethod,\n\u001b[1;32m    488\u001b[0m         url\u001b[39m=\u001b[39;49murl,\n\u001b[1;32m    489\u001b[0m         body\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mbody,\n\u001b[1;32m    490\u001b[0m         headers\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mheaders,\n\u001b[1;32m    491\u001b[0m         redirect\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    492\u001b[0m         assert_same_host\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    493\u001b[0m         preload_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    494\u001b[0m         decode_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    495\u001b[0m         retries\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_retries,\n\u001b[1;32m    496\u001b[0m         timeout\u001b[39m=\u001b[39;49mtimeout,\n\u001b[1;32m    497\u001b[0m         chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[1;32m    498\u001b[0m     )\n\u001b[1;32m    500\u001b[0m \u001b[39mexcept\u001b[39;00m (ProtocolError, \u001b[39mOSError\u001b[39;00m) \u001b[39mas\u001b[39;00m err:\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/connectionpool.py:844\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m    842\u001b[0m     new_e \u001b[39m=\u001b[39m ProtocolError(\u001b[39m\"\u001b[39m\u001b[39mConnection aborted.\u001b[39m\u001b[39m\"\u001b[39m, new_e)\n\u001b[0;32m--> 844\u001b[0m retries \u001b[39m=\u001b[39m retries\u001b[39m.\u001b[39;49mincrement(\n\u001b[1;32m    845\u001b[0m     method, url, error\u001b[39m=\u001b[39;49mnew_e, _pool\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m, _stacktrace\u001b[39m=\u001b[39;49msys\u001b[39m.\u001b[39;49mexc_info()[\u001b[39m2\u001b[39;49m]\n\u001b[1;32m    846\u001b[0m )\n\u001b[1;32m    847\u001b[0m retries\u001b[39m.\u001b[39msleep()\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/urllib3/util/retry.py:515\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    514\u001b[0m     reason \u001b[39m=\u001b[39m error \u001b[39mor\u001b[39;00m ResponseError(cause)\n\u001b[0;32m--> 515\u001b[0m     \u001b[39mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[39mfrom\u001b[39;00m \u001b[39mreason\u001b[39;00m  \u001b[39m# type: ignore[arg-type]\u001b[39;00m\n\u001b[1;32m    517\u001b[0m log\u001b[39m.\u001b[39mdebug(\u001b[39m\"\u001b[39m\u001b[39mIncremented Retry for (url=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m): \u001b[39m\u001b[39m%r\u001b[39;00m\u001b[39m\"\u001b[39m, url, new_retry)\n",
      "\u001b[0;31mMaxRetryError\u001b[0m: HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /embedding (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x16a8407d0>: Failed to establish a new connection: [Errno 61] Connection refused'))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
      "File \u001b[0;32m~/Developer/babyagi/utils/__init__.py:9\u001b[0m, in \u001b[0;36mget_embedding\u001b[0;34m(text, url, **kwargs)\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m----> 9\u001b[0m     response \u001b[39m=\u001b[39m requests\u001b[39m.\u001b[39;49mpost(\u001b[39mf\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39m{\u001b[39;49;00murl\u001b[39m}\u001b[39;49;00m\u001b[39m/embedding\u001b[39;49m\u001b[39m\"\u001b[39;49m, json\u001b[39m=\u001b[39;49mpayload, headers\u001b[39m=\u001b[39;49mheaders)\n\u001b[1;32m     10\u001b[0m     \u001b[39mreturn\u001b[39;00m response\u001b[39m.\u001b[39mjson()[\u001b[39m\"\u001b[39m\u001b[39membedding\u001b[39m\u001b[39m\"\u001b[39m]\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/requests/api.py:115\u001b[0m, in \u001b[0;36mpost\u001b[0;34m(url, data, json, **kwargs)\u001b[0m\n\u001b[1;32m    104\u001b[0m \u001b[39m\u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[1;32m    105\u001b[0m \n\u001b[1;32m    106\u001b[0m \u001b[39m:param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[39m:rtype: requests.Response\u001b[39;00m\n\u001b[1;32m    113\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 115\u001b[0m \u001b[39mreturn\u001b[39;00m request(\u001b[39m\"\u001b[39;49m\u001b[39mpost\u001b[39;49m\u001b[39m\"\u001b[39;49m, url, data\u001b[39m=\u001b[39;49mdata, json\u001b[39m=\u001b[39;49mjson, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[39mwith\u001b[39;00m sessions\u001b[39m.\u001b[39mSession() \u001b[39mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m     \u001b[39mreturn\u001b[39;00m session\u001b[39m.\u001b[39;49mrequest(method\u001b[39m=\u001b[39;49mmethod, url\u001b[39m=\u001b[39;49murl, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    588\u001b[0m send_kwargs\u001b[39m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(prep, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49msend_kwargs)\n\u001b[1;32m    591\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[39m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[39m=\u001b[39m adapter\u001b[39m.\u001b[39;49msend(request, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    705\u001b[0m \u001b[39m# Total elapsed time of the request (approximately)\u001b[39;00m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/requests/adapters.py:519\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    517\u001b[0m         \u001b[39mraise\u001b[39;00m SSLError(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[0;32m--> 519\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mConnectionError\u001b[39;00m(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[1;32m    521\u001b[0m \u001b[39mexcept\u001b[39;00m ClosedPoolError \u001b[39mas\u001b[39;00m e:\n",
      "\u001b[0;31mConnectionError\u001b[0m: HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /embedding (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x16a8407d0>: Failed to establish a new connection: [Errno 61] Connection refused'))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mUnboundLocalError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/Users/dev/Developer/babyagi/00_retrievers.ipynb Cell 8\u001b[0m line \u001b[0;36m2\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/dev/Developer/babyagi/00_retrievers.ipynb#W6sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mlangchain\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mvectorstores\u001b[39;00m \u001b[39mimport\u001b[39;00m Chroma\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/dev/Developer/babyagi/00_retrievers.ipynb#W6sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m db \u001b[39m=\u001b[39m Chroma\u001b[39m.\u001b[39;49mfrom_documents(texts, LlamaEmbeddingFunction())\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/langchain/vectorstores/chroma.py:646\u001b[0m, in \u001b[0;36mChroma.from_documents\u001b[0;34m(cls, documents, embedding, ids, collection_name, persist_directory, client_settings, client, collection_metadata, **kwargs)\u001b[0m\n\u001b[1;32m    644\u001b[0m texts \u001b[39m=\u001b[39m [doc\u001b[39m.\u001b[39mpage_content \u001b[39mfor\u001b[39;00m doc \u001b[39min\u001b[39;00m documents]\n\u001b[1;32m    645\u001b[0m metadatas \u001b[39m=\u001b[39m [doc\u001b[39m.\u001b[39mmetadata \u001b[39mfor\u001b[39;00m doc \u001b[39min\u001b[39;00m documents]\n\u001b[0;32m--> 646\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39;49m\u001b[39m.\u001b[39;49mfrom_texts(\n\u001b[1;32m    647\u001b[0m     texts\u001b[39m=\u001b[39;49mtexts,\n\u001b[1;32m    648\u001b[0m     embedding\u001b[39m=\u001b[39;49membedding,\n\u001b[1;32m    649\u001b[0m     metadatas\u001b[39m=\u001b[39;49mmetadatas,\n\u001b[1;32m    650\u001b[0m     ids\u001b[39m=\u001b[39;49mids,\n\u001b[1;32m    651\u001b[0m     collection_name\u001b[39m=\u001b[39;49mcollection_name,\n\u001b[1;32m    652\u001b[0m     persist_directory\u001b[39m=\u001b[39;49mpersist_directory,\n\u001b[1;32m    653\u001b[0m     client_settings\u001b[39m=\u001b[39;49mclient_settings,\n\u001b[1;32m    654\u001b[0m     client\u001b[39m=\u001b[39;49mclient,\n\u001b[1;32m    655\u001b[0m     collection_metadata\u001b[39m=\u001b[39;49mcollection_metadata,\n\u001b[1;32m    656\u001b[0m     \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs,\n\u001b[1;32m    657\u001b[0m )\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/langchain/vectorstores/chroma.py:610\u001b[0m, in \u001b[0;36mChroma.from_texts\u001b[0;34m(cls, texts, embedding, metadatas, ids, collection_name, persist_directory, client_settings, client, collection_metadata, **kwargs)\u001b[0m\n\u001b[1;32m    582\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Create a Chroma vectorstore from a raw documents.\u001b[39;00m\n\u001b[1;32m    583\u001b[0m \n\u001b[1;32m    584\u001b[0m \u001b[39mIf a persist_directory is specified, the collection will be persisted there.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    599\u001b[0m \u001b[39m    Chroma: Chroma vectorstore.\u001b[39;00m\n\u001b[1;32m    600\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    601\u001b[0m chroma_collection \u001b[39m=\u001b[39m \u001b[39mcls\u001b[39m(\n\u001b[1;32m    602\u001b[0m     collection_name\u001b[39m=\u001b[39mcollection_name,\n\u001b[1;32m    603\u001b[0m     embedding_function\u001b[39m=\u001b[39membedding,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    608\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs,\n\u001b[1;32m    609\u001b[0m )\n\u001b[0;32m--> 610\u001b[0m chroma_collection\u001b[39m.\u001b[39;49madd_texts(texts\u001b[39m=\u001b[39;49mtexts, metadatas\u001b[39m=\u001b[39;49mmetadatas, ids\u001b[39m=\u001b[39;49mids)\n\u001b[1;32m    611\u001b[0m \u001b[39mreturn\u001b[39;00m chroma_collection\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/baby311/lib/python3.11/site-packages/langchain/vectorstores/chroma.py:188\u001b[0m, in \u001b[0;36mChroma.add_texts\u001b[0;34m(self, texts, metadatas, ids, **kwargs)\u001b[0m\n\u001b[1;32m    186\u001b[0m texts \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(texts)\n\u001b[1;32m    187\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_embedding_function \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 188\u001b[0m     embeddings \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_embedding_function\u001b[39m.\u001b[39;49membed_documents(texts)\n\u001b[1;32m    189\u001b[0m \u001b[39mif\u001b[39;00m metadatas:\n\u001b[1;32m    190\u001b[0m     \u001b[39m# fill metadatas with empty dicts if somebody\u001b[39;00m\n\u001b[1;32m    191\u001b[0m     \u001b[39m# did not specify metadata for all texts\u001b[39;00m\n\u001b[1;32m    192\u001b[0m     length_diff \u001b[39m=\u001b[39m \u001b[39mlen\u001b[39m(texts) \u001b[39m-\u001b[39m \u001b[39mlen\u001b[39m(metadatas)\n",
      "File \u001b[0;32m~/Developer/babyagi/utils/__init__.py:43\u001b[0m, in \u001b[0;36mLlamaEmbeddingFunction.embed_documents\u001b[0;34m(self, texts)\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39membed_documents\u001b[39m(\u001b[39mself\u001b[39m, texts: Documents) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Embeddings:\n\u001b[0;32m---> 43\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m(texts)\n",
      "File \u001b[0;32m~/Developer/babyagi/utils/__init__.py:38\u001b[0m, in \u001b[0;36mLlamaEmbeddingFunction.__call__\u001b[0;34m(self, texts)\u001b[0m\n\u001b[1;32m     36\u001b[0m embeddings \u001b[39m=\u001b[39m []\n\u001b[1;32m     37\u001b[0m \u001b[39mfor\u001b[39;00m t \u001b[39min\u001b[39;00m texts:\n\u001b[0;32m---> 38\u001b[0m     e \u001b[39m=\u001b[39m get_embedding(t, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mLLAMA_API_PATH)\n\u001b[1;32m     39\u001b[0m     embeddings\u001b[39m.\u001b[39mappend(e)\n\u001b[1;32m     40\u001b[0m \u001b[39mreturn\u001b[39;00m embeddings\n",
      "File \u001b[0;32m~/Developer/babyagi/utils/__init__.py:14\u001b[0m, in \u001b[0;36mget_embedding\u001b[0;34m(text, url, **kwargs)\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[39mprint\u001b[39m(e)\n\u001b[1;32m     13\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mPAYLOAD:\u001b[39m\u001b[39m\"\u001b[39m, payload)\n\u001b[0;32m---> 14\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mRESPONSE:\u001b[39m\u001b[39m\"\u001b[39m, response)\n\u001b[1;32m     15\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n",
      "\u001b[0;31mUnboundLocalError\u001b[0m: cannot access local variable 'response' where it is not associated with a value"
     ]
    }
   ],
   "source": [
    "from langchain.vectorstores import Chroma\n",
    "db = Chroma.from_documents(texts, LlamaEmbeddingFunction())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever = db.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "qa = RetrievalQA.from_chain_type(llm=llm, chain_type=\"stuff\", retriever=retriever)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"What did the president say about Ketanji Brown Jackson\"\n",
    "qa.run(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "baby311",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}