07_factors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to DataFrames\n",
    "**[Bogumił Kamiński](http://bogumilkaminski.pl/about/), February 13, 2023**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using DataFrames # load package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "using CategoricalArrays # CategoricalArrays.jl is independent from DataFrames.jl but it is often used in combination"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working with CategoricalArrays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Constructor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element CategoricalArray{String,1,UInt32}:\n",
       " \"A\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"C\""
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = categorical([\"A\", \"B\", \"B\", \"C\"]) # unordered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element CategoricalArray{String,1,UInt32}:\n",
       " \"A\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"C\""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = categorical([\"A\", \"B\", \"B\", \"C\"], ordered=true) # ordered, by default order is sorting order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element CategoricalArray{Union{Missing, String},1,UInt32}:\n",
       " \"A\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"C\"\n",
       " missing"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z = categorical([\"A\",\"B\",\"B\",\"C\", missing]) # unordered with missings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10-element CategoricalArray{String,1,UInt32}:\n",
       " \"Q1: [1.0, 2.8)\"\n",
       " \"Q1: [1.0, 2.8)\"\n",
       " \"Q2: [2.8, 4.6)\"\n",
       " \"Q2: [2.8, 4.6)\"\n",
       " \"Q3: [4.6, 6.4)\"\n",
       " \"Q3: [4.6, 6.4)\"\n",
       " \"Q4: [6.4, 8.2)\"\n",
       " \"Q4: [6.4, 8.2)\"\n",
       " \"Q5: [8.2, 10.0]\"\n",
       " \"Q5: [8.2, 10.0]\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = cut(1:10, 5) # ordered, into equal counts, possible to rename labels and give custom breaks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(we will cover grouping later, but let us here use it to analyze the results, we use Chain.jl for chaining)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div style = \"float: left;\"><span>10×2 DataFrame</span></div><div style = \"clear: both;\"></div></div><div class = \"data-frame\" style = \"overflow-x: scroll;\"><table class = \"data-frame\" style = \"margin-bottom: 6px;\"><thead><tr class = \"header\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">Row</th><th style = \"text-align: left;\">x</th><th style = \"text-align: left;\">nrow</th></tr><tr class = \"subheader headerLastRow\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\"></th><th title = \"CategoricalValue{String, UInt32}\" style = \"text-align: left;\">Cat…</th><th title = \"Int64\" style = \"text-align: left;\">Int64</th></tr></thead><tbody><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">1</td><td style = \"text-align: left;\">Q1: [-4.37425549047856, -1.2786607637653769)</td><td style = \"text-align: right;\">10000</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">2</td><td style = \"text-align: left;\">Q2: [-1.2786607637653769, -0.8430443474683171)</td><td style = \"text-align: right;\">10000</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">3</td><td style = \"text-align: left;\">Q3: [-0.8430443474683171, -0.5258259568979863)</td><td style = \"text-align: right;\">10000</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">4</td><td style = \"text-align: left;\">Q4: [-0.5258259568979863, -0.2593167185316153)</td><td style = \"text-align: right;\">10000</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">5</td><td style = \"text-align: left;\">Q5: [-0.2593167185316153, -0.0018062175416992735)</td><td style = \"text-align: right;\">10000</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">6</td><td style = \"text-align: left;\">Q6: [-0.0018062175416992735, 0.25137476303234196)</td><td style = \"text-align: right;\">10000</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">7</td><td style = \"text-align: left;\">Q7: [0.25137476303234196, 0.5222753668380758)</td><td style = \"text-align: right;\">10000</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">8</td><td style = \"text-align: left;\">Q8: [0.5222753668380758, 0.838323113350727)</td><td style = \"text-align: right;\">10000</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">9</td><td style = \"text-align: left;\">Q9: [0.838323113350727, 1.2837083260172553)</td><td style = \"text-align: right;\">10000</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">10</td><td style = \"text-align: left;\">Q10: [1.2837083260172553, 4.269831619555335]</td><td style = \"text-align: right;\">10000</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & nrow\\\\\n",
       "\t\\hline\n",
       "\t& Cat… & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & Q1: [-4.37425549047856, -1.2786607637653769) & 10000 \\\\\n",
       "\t2 & Q2: [-1.2786607637653769, -0.8430443474683171) & 10000 \\\\\n",
       "\t3 & Q3: [-0.8430443474683171, -0.5258259568979863) & 10000 \\\\\n",
       "\t4 & Q4: [-0.5258259568979863, -0.2593167185316153) & 10000 \\\\\n",
       "\t5 & Q5: [-0.2593167185316153, -0.0018062175416992735) & 10000 \\\\\n",
       "\t6 & Q6: [-0.0018062175416992735, 0.25137476303234196) & 10000 \\\\\n",
       "\t7 & Q7: [0.25137476303234196, 0.5222753668380758) & 10000 \\\\\n",
       "\t8 & Q8: [0.5222753668380758, 0.838323113350727) & 10000 \\\\\n",
       "\t9 & Q9: [0.838323113350727, 1.2837083260172553) & 10000 \\\\\n",
       "\t10 & Q10: [1.2837083260172553, 4.269831619555335] & 10000 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m10×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x                                 \u001b[0m\u001b[1m nrow  \u001b[0m\n",
       "     │\u001b[90m Cat…                              \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────────────────────────────────\n",
       "   1 │ Q1: [-4.37425549047856, -1.27866…  10000\n",
       "   2 │ Q2: [-1.2786607637653769, -0.843…  10000\n",
       "   3 │ Q3: [-0.8430443474683171, -0.525…  10000\n",
       "   4 │ Q4: [-0.5258259568979863, -0.259…  10000\n",
       "   5 │ Q5: [-0.2593167185316153, -0.001…  10000\n",
       "   6 │ Q6: [-0.0018062175416992735, 0.2…  10000\n",
       "   7 │ Q7: [0.25137476303234196, 0.5222…  10000\n",
       "   8 │ Q8: [0.5222753668380758, 0.83832…  10000\n",
       "   9 │ Q9: [0.838323113350727, 1.283708…  10000\n",
       "  10 │ Q10: [1.2837083260172553, 4.2698…  10000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@chain DataFrame(x=cut(randn(100000), 10)) begin\n",
    "      groupby(:x)\n",
    "      combine(nrow) # just to make sure cut works right\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element CategoricalArray{Int64,1,UInt32}:\n",
       " 1\n",
       " 2\n",
       " 2\n",
       " 3\n",
       " 3"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = categorical([1,2,2,3,3]) # contains integers not strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Vector{Union{Missing, String}}:\n",
       " \"A\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"C\"\n",
       " missing"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Vector{Union{String, Missing}}(z) # sometimes you need to convert back to a standard vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Managing levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Vector{CategoricalVector{T, UInt32} where T}:\n",
       " CategoricalValue{String, UInt32}[\"A\", \"B\", \"B\", \"C\"]\n",
       " CategoricalValue{String, UInt32}[\"A\", \"B\", \"B\", \"C\"]\n",
       " Union{Missing, CategoricalValue{String, UInt32}}[\"A\", \"B\", \"B\", \"C\", missing]\n",
       " CategoricalValue{String, UInt32}[\"Q1: [1.0, 2.8)\", \"Q1: [1.0, 2.8)\", \"Q2: [2.8, 4.6)\", \"Q2: [2.8, 4.6)\", \"Q3: [4.6, 6.4)\", \"Q3: [4.6, 6.4)\", \"Q4: [6.4, 8.2)\", \"Q4: [6.4, 8.2)\", \"Q5: [8.2, 10.0]\", \"Q5: [8.2, 10.0]\"]\n",
       " CategoricalValue{Int64, UInt32}[1, 2, 2, 3, 3]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr = [x,y,z,c,v]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element BitVector:\n",
       " 0\n",
       " 1\n",
       " 0\n",
       " 1\n",
       " 0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isordered.(arr) # chcek if categorical array is orderd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalValue{String, UInt32}[\"A\", \"B\", \"B\", \"C\"], true)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordered!(x, true), isordered(x) # make x ordered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalValue{String, UInt32}[\"A\", \"B\", \"B\", \"C\"], false)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordered!(x, false), isordered(x) # and unordered again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Vector{Vector}:\n",
       " [\"A\", \"B\", \"C\"]\n",
       " [\"A\", \"B\", \"C\"]\n",
       " [\"A\", \"B\", \"C\"]\n",
       " [\"Q1: [1.0, 2.8)\", \"Q2: [2.8, 4.6)\", \"Q3: [4.6, 6.4)\", \"Q4: [6.4, 8.2)\", \"Q5: [8.2, 10.0]\"]\n",
       " [1, 2, 3]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels.(arr) # list levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Vector{Vector}:\n",
       " [\"A\", \"B\", \"C\"]\n",
       " [\"A\", \"B\", \"C\"]\n",
       " Union{Missing, String}[\"A\", \"B\", \"C\", missing]\n",
       " [\"Q1: [1.0, 2.8)\", \"Q2: [2.8, 4.6)\", \"Q3: [4.6, 6.4)\", \"Q4: [6.4, 8.2)\", \"Q5: [8.2, 10.0]\"]\n",
       " [1, 2, 3]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique.(arr) # missing will be included"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[1] < y[2] # can compare as y is ordered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "ArgumentError: Unordered CategoricalValue objects cannot be tested for order using <. Use isless instead, or call the ordered! function on the parent array to change this",
     "output_type": "error",
     "traceback": [
      "ArgumentError: Unordered CategoricalValue objects cannot be tested for order using <. Use isless instead, or call the ordered! function on the parent array to change this",
      "",
      "Stacktrace:",
      " [1] <(x::CategoricalValue{Int64, UInt32}, y::CategoricalValue{Int64, UInt32})",
      "   @ CategoricalArrays C:\\Users\\bogum\\.julia\\packages\\CategoricalArrays\\tJ8hD\\src\\value.jl:166",
      " [2] top-level scope",
      "   @ In[18]:1"
     ]
    }
   ],
   "source": [
    "v[1] < v[2] # not comparable, v is unordered although it contains integers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "ArgumentError: cannot compare a `CategoricalValue` to value `v` of type `String`: wrap `v` using `CategoricalValue(v, catvalue)` or `CategoricalValue(v, catarray)` first",
     "output_type": "error",
     "traceback": [
      "ArgumentError: cannot compare a `CategoricalValue` to value `v` of type `String`: wrap `v` using `CategoricalValue(v, catvalue)` or `CategoricalValue(v, catarray)` first",
      "",
      "Stacktrace:",
      " [1] <(x::CategoricalValue{String, UInt32}, y::String)",
      "   @ CategoricalArrays C:\\Users\\bogum\\.julia\\packages\\CategoricalArrays\\tJ8hD\\src\\value.jl:173",
      " [2] top-level scope",
      "   @ In[19]:1"
     ]
    }
   ],
   "source": [
    "y[2] < \"A\" # comparison against type underlying categorical value is not allowed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[2] < CategoricalValue(\"A\", y) # you need to explicitly convert a value to a level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "ArgumentError: level Z not found in source pool",
     "output_type": "error",
     "traceback": [
      "ArgumentError: level Z not found in source pool",
      "",
      "Stacktrace:",
      " [1] CategoricalValue(value::String, source::CategoricalVector{String, UInt32, String, CategoricalValue{String, UInt32}, Union{}})",
      "   @ CategoricalArrays C:\\Users\\bogum\\.julia\\packages\\CategoricalArrays\\tJ8hD\\src\\value.jl:13",
      " [2] top-level scope",
      "   @ In[21]:1"
     ]
    }
   ],
   "source": [
    "y[2] < CategoricalValue(\"Z\", y) # but it is treated as a level, and thus only valid levels are allowed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element CategoricalArray{String,1,UInt32}:\n",
       " \"A\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"C\""
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels!(y, [\"C\", \"B\", \"A\"]) # you can reorder levels, mostly useful for ordered CategoricalArrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[1] < y[2] # observe that the order is changed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "ArgumentError: cannot remove level \"C\" as it is used at position 4 and allowmissing=false.",
     "output_type": "error",
     "traceback": [
      "ArgumentError: cannot remove level \"C\" as it is used at position 4 and allowmissing=false.",
      "",
      "Stacktrace:",
      " [1] levels!(A::CategoricalVector{Union{Missing, String}, UInt32, String, CategoricalValue{String, UInt32}, Missing}, newlevels::Vector{String}; allowmissing::Bool, allow_missing::Nothing)",
      "   @ CategoricalArrays C:\\Users\\bogum\\.julia\\packages\\CategoricalArrays\\tJ8hD\\src\\array.jl:859",
      " [2] levels!(A::CategoricalVector{Union{Missing, String}, UInt32, String, CategoricalValue{String, UInt32}, Missing}, newlevels::Vector{String})",
      "   @ CategoricalArrays C:\\Users\\bogum\\.julia\\packages\\CategoricalArrays\\tJ8hD\\src\\array.jl:793",
      " [3] top-level scope",
      "   @ In[24]:1"
     ]
    }
   ],
   "source": [
    "levels!(z, [\"A\", \"B\"]) # you have to specify all levels that are present"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element CategoricalArray{Union{Missing, String},1,UInt32}:\n",
       " \"A\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " missing\n",
       " missing"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels!(z, [\"A\", \"B\"], allowmissing=true) # unless the underlying array allows for missings and force removal of levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element CategoricalArray{Union{Missing, String},1,UInt32}:\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " missing\n",
       " missing"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z[1] = \"B\"\n",
    "z # now z has only \"B\" entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2-element Vector{String}:\n",
       " \"A\"\n",
       " \"B\""
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels(z) # but it remembers the levels it had (the reason is mostly performance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1-element Vector{String}:\n",
       " \"B\""
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "droplevels!(z) # this way we can clean it up\n",
    "levels(z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data manipulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalValue{String, UInt32}[\"A\", \"B\", \"B\", \"C\"], [\"A\", \"B\", \"C\"])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, levels(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalValue{String, UInt32}[\"A\", \"0\", \"B\", \"C\"], [\"A\", \"B\", \"C\", \"0\"])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[2] = \"0\"\n",
    "x, levels(x) # new level added at the end (works only for unordered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalValue{Int64, UInt32}[1, 2, 2, 3, 3], [1, 2, 3])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v, levels(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "MethodError: no method matching +(::CategoricalValue{Int64, UInt32}, ::CategoricalValue{Int64, UInt32})\n\n\u001b[0mClosest candidates are:\n\u001b[0m  +(::Any, ::Any, \u001b[91m::Any\u001b[39m, \u001b[91m::Any...\u001b[39m)\n\u001b[0m\u001b[90m   @\u001b[39m \u001b[90mBase\u001b[39m \u001b[90m\u001b[4moperators.jl:578\u001b[24m\u001b[39m\n",
     "output_type": "error",
     "traceback": [
      "MethodError: no method matching +(::CategoricalValue{Int64, UInt32}, ::CategoricalValue{Int64, UInt32})\n\n\u001b[0mClosest candidates are:\n\u001b[0m  +(::Any, ::Any, \u001b[91m::Any\u001b[39m, \u001b[91m::Any...\u001b[39m)\n\u001b[0m\u001b[90m   @\u001b[39m \u001b[90mBase\u001b[39m \u001b[90m\u001b[4moperators.jl:578\u001b[24m\u001b[39m\n",
      "",
      "Stacktrace:",
      " [1] top-level scope",
      "   @ In[32]:1"
     ]
    }
   ],
   "source": [
    "v[1] + v[2] # even though the underlying data is Int, we cannot operate on it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Vector{Int64}:\n",
       " 1\n",
       " 2\n",
       " 2\n",
       " 3\n",
       " 3"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Vector{Int}(v) # you have either to retrieve the data by conversion (may be expensive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unwrap(v[1]) + unwrap(v[2]) # or get a single value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Vector{Int64}:\n",
       " 1\n",
       " 2\n",
       " 2\n",
       " 3\n",
       " 3"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unwrap.(v) # this will work for arrays witout missings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Vector{Union{Missing, String}}:\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " missing\n",
       " missing"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unwrap.(z) # also works on missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Vector{Union{Missing, String}}:\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " missing\n",
       " missing"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Vector{Union{String, Missing}}(z) # or do the conversion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6-element Vector{Union{Missing, Int64}}:\n",
       " 10\n",
       "  2\n",
       "  3\n",
       "  4\n",
       "  5\n",
       "   missing"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recode([1,2,3,4,5,missing], 1=>10) # recode some values in an array; has also in place recode! equivalent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6-element Vector{Union{Missing, Int64, String}}:\n",
       " 10\n",
       " 20\n",
       "   \"a\"\n",
       "   \"a\"\n",
       "   \"a\"\n",
       "   missing"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recode([1,2,3,4,5,missing], \"a\", 1=>10, 2=>20) # here we provided a default value for not mapped recodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6-element Vector{Union{Int64, String}}:\n",
       " 10\n",
       "  2\n",
       "  3\n",
       "  4\n",
       "  5\n",
       "   \"missing\""
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recode([1,2,3,4,5,missing], 1=>10, missing=>\"missing\") # to recode Missing you have to do it explicitly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Union{Missing, CategoricalValue{Int64, UInt32}}[1, 2, 3, 4, 5, missing], [1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = categorical([1:5; missing])\n",
    "t, levels(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Union{Missing, CategoricalValue{Int64, UInt32}}[2, 2, 2, 4, 5, missing], [2, 4, 5])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recode!(t, [1,3]=>2)\n",
    "t, levels(t) # note that the levels are dropped after recode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3-element Vector{Int64}:\n",
       "  3\n",
       "  0\n",
       " -1"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = categorical([1,2,3], ordered=true)\n",
    "levels(recode(t, 2=>0, 1=>-1)) # and if you introduce a new levels they are added at the end in the order of appearance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3-element Vector{Int64}:\n",
       " 100\n",
       " 200\n",
       " 300"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = categorical([1,2,3,4,5], ordered=true) # when using default it becomes the last level\n",
    "levels(recode(t, 300, [1,2]=>100, 3=>200))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparisons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4×4 Matrix{Bool}:\n",
       " 1  1  1  1\n",
       " 1  1  1  1\n",
       " 1  1  1  1\n",
       " 1  1  1  1"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = categorical([1,2,3])\n",
    "xs = [x, categorical(x), categorical(x, ordered=true), categorical(x, ordered=true)]\n",
    "levels!(xs[2], [3,2,1])\n",
    "levels!(xs[4], [2,3,1])\n",
    "[a == b for a in xs, b in xs] # all are equal - comparison only by contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4×4 Matrix{Bool}:\n",
       " 1  0  0  0\n",
       " 0  1  0  0\n",
       " 0  0  1  0\n",
       " 0  0  0  1"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "signature(x::CategoricalArray) = (x, levels(x), isordered(x)) # this is actually the full signature of CategoricalArray\n",
    "# all are different, notice that x[1] and x[2] are unordered but have a different order of levels\n",
    "[signature(a) == signature(b) for a in xs, b in xs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "ArgumentError: Unordered CategoricalValue objects cannot be tested for order using <. Use isless instead, or call the ordered! function on the parent array to change this",
     "output_type": "error",
     "traceback": [
      "ArgumentError: Unordered CategoricalValue objects cannot be tested for order using <. Use isless instead, or call the ordered! function on the parent array to change this",
      "",
      "Stacktrace:",
      " [1] <(x::CategoricalValue{Int64, UInt32}, y::CategoricalValue{Int64, UInt32})",
      "   @ CategoricalArrays C:\\Users\\bogum\\.julia\\packages\\CategoricalArrays\\tJ8hD\\src\\value.jl:166",
      " [2] top-level scope",
      "   @ In[47]:1"
     ]
    }
   ],
   "source": [
    "x[1] < x[2] # you cannot compare elements of unordered CategoricalArray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[1] < t[2] # but you can do it for an ordered one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isless(x[1], x[2]) # isless works within the same CategoricalArray even if it is not ordered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = deepcopy(x) # but not across categorical arrays\n",
    "isless(x[1], y[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isless(unwrap(x[1]), unwrap(y[2])) # you can use get to make a comparison of the contents of CategoricalArray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[1] == y[2] # equality tests works OK across CategoricalArrays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical columns in a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div style = \"float: left;\"><span>3×3 DataFrame</span></div><div style = \"clear: both;\"></div></div><div class = \"data-frame\" style = \"overflow-x: scroll;\"><table class = \"data-frame\" style = \"margin-bottom: 6px;\"><thead><tr class = \"header\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">Row</th><th style = \"text-align: left;\">x</th><th style = \"text-align: left;\">y</th><th style = \"text-align: left;\">z</th></tr><tr class = \"subheader headerLastRow\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\"></th><th title = \"Int64\" style = \"text-align: left;\">Int64</th><th title = \"Char\" style = \"text-align: left;\">Char</th><th title = \"String\" style = \"text-align: left;\">String</th></tr></thead><tbody><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">1</td><td style = \"text-align: right;\">1</td><td style = \"text-align: left;\">a</td><td style = \"text-align: left;\">a</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">2</td><td style = \"text-align: right;\">2</td><td style = \"text-align: left;\">b</td><td style = \"text-align: left;\">b</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">3</td><td style = \"text-align: right;\">3</td><td style = \"text-align: left;\">c</td><td style = \"text-align: left;\">c</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& x & y & z\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Char & String\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & a & a \\\\\n",
       "\t2 & 2 & b & b \\\\\n",
       "\t3 & 3 & c & c \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×3 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x     \u001b[0m\u001b[1m y    \u001b[0m\u001b[1m z      \u001b[0m\n",
       "     │\u001b[90m Int64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m String \u001b[0m\n",
       "─────┼─────────────────────\n",
       "   1 │     1  a     a\n",
       "   2 │     2  b     b\n",
       "   3 │     3  c     c"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = DataFrame(x = 1:3, y = 'a':'c', z = [\"a\",\"b\",\"c\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert all `String` columns to categorical in-place"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div style = \"float: left;\"><span>3×3 DataFrame</span></div><div style = \"clear: both;\"></div></div><div class = \"data-frame\" style = \"overflow-x: scroll;\"><table class = \"data-frame\" style = \"margin-bottom: 6px;\"><thead><tr class = \"header\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">Row</th><th style = \"text-align: left;\">x</th><th style = \"text-align: left;\">y</th><th style = \"text-align: left;\">z</th></tr><tr class = \"subheader headerLastRow\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\"></th><th title = \"Int64\" style = \"text-align: left;\">Int64</th><th title = \"Char\" style = \"text-align: left;\">Char</th><th title = \"CategoricalValue{String, UInt32}\" style = \"text-align: left;\">Cat…</th></tr></thead><tbody><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">1</td><td style = \"text-align: right;\">1</td><td style = \"text-align: left;\">a</td><td style = \"text-align: left;\">a</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">2</td><td style = \"text-align: right;\">2</td><td style = \"text-align: left;\">b</td><td style = \"text-align: left;\">b</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">3</td><td style = \"text-align: right;\">3</td><td style = \"text-align: left;\">c</td><td style = \"text-align: left;\">c</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccc}\n",
       "\t& x & y & z\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Char & Cat…\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & a & a \\\\\n",
       "\t2 & 2 & b & b \\\\\n",
       "\t3 & 3 & c & c \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×3 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x     \u001b[0m\u001b[1m y    \u001b[0m\u001b[1m z    \u001b[0m\n",
       "     │\u001b[90m Int64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Cat… \u001b[0m\n",
       "─────┼───────────────────\n",
       "   1 │     1  a     a\n",
       "   2 │     2  b     b\n",
       "   3 │     3  c     c"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transform!(df, names(df, String) => categorical, renamecols=false)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div style = \"float: left;\"><span>3×7 DataFrame</span></div><div style = \"clear: both;\"></div></div><div class = \"data-frame\" style = \"overflow-x: scroll;\"><table class = \"data-frame\" style = \"margin-bottom: 6px;\"><thead><tr class = \"header\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">Row</th><th style = \"text-align: left;\">variable</th><th style = \"text-align: left;\">mean</th><th style = \"text-align: left;\">min</th><th style = \"text-align: left;\">median</th><th style = \"text-align: left;\">max</th><th style = \"text-align: left;\">nmissing</th><th style = \"text-align: left;\">eltype</th></tr><tr class = \"subheader headerLastRow\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\"></th><th title = \"Symbol\" style = \"text-align: left;\">Symbol</th><th title = \"Union{Nothing, Float64}\" style = \"text-align: left;\">Union…</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Union{Nothing, Float64}\" style = \"text-align: left;\">Union…</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Int64\" style = \"text-align: left;\">Int64</th><th title = \"DataType\" style = \"text-align: left;\">DataType</th></tr></thead><tbody><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">1</td><td style = \"text-align: left;\">x</td><td style = \"text-align: left;\">2.0</td><td style = \"text-align: left;\">1</td><td style = \"text-align: left;\">2.0</td><td style = \"text-align: left;\">3</td><td style = \"text-align: right;\">0</td><td style = \"text-align: left;\">Int64</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">2</td><td style = \"text-align: left;\">y</td><td style = \"font-style: italic; text-align: left;\"></td><td style = \"text-align: left;\">a</td><td style = \"font-style: italic; text-align: left;\"></td><td style = \"text-align: left;\">c</td><td style = \"text-align: right;\">0</td><td style = \"text-align: left;\">Char</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">3</td><td style = \"text-align: left;\">z</td><td style = \"font-style: italic; text-align: left;\"></td><td style = \"text-align: left;\">a</td><td style = \"font-style: italic; text-align: left;\"></td><td style = \"text-align: left;\">c</td><td style = \"text-align: right;\">0</td><td style = \"text-align: left;\">CategoricalValue{String, UInt32}</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccccc}\n",
       "\t& variable & mean & min & median & max & nmissing & eltype\\\\\n",
       "\t\\hline\n",
       "\t& Symbol & Union… & Any & Union… & Any & Int64 & DataType\\\\\n",
       "\t\\hline\n",
       "\t1 & x & 2.0 & 1 & 2.0 & 3 & 0 & Int64 \\\\\n",
       "\t2 & y &  & a &  & c & 0 & Char \\\\\n",
       "\t3 & z &  & a &  & c & 0 & CategoricalValue\\{String, UInt32\\} \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×7 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m variable \u001b[0m\u001b[1m mean   \u001b[0m\u001b[1m min \u001b[0m\u001b[1m median \u001b[0m\u001b[1m max \u001b[0m\u001b[1m nmissing \u001b[0m\u001b[1m eltype                   \u001b[0m ⋯\n",
       "     │\u001b[90m Symbol   \u001b[0m\u001b[90m Union… \u001b[0m\u001b[90m Any \u001b[0m\u001b[90m Union… \u001b[0m\u001b[90m Any \u001b[0m\u001b[90m Int64    \u001b[0m\u001b[90m DataType                 \u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ x         2.0     1    2.0     3           0  Int64                     ⋯\n",
       "   2 │ y        \u001b[90m        \u001b[0m a   \u001b[90m        \u001b[0m c           0  Char\n",
       "   3 │ z        \u001b[90m        \u001b[0m a   \u001b[90m        \u001b[0m c           0  CategoricalValue{String,\n",
       "\u001b[36m                                                                1 column omitted\u001b[0m"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "describe(df)"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Julia 1.9.0",
   "language": "julia",
   "name": "julia-1.9"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}