02_hyperopt.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"02_hyperopt.ipynb","provenance":[],"mount_file_id":"1i07fs6IevE0IkBDnZKpyZYUTu7wzf9Fr","authorship_tag":"ABX9TyP5v4Db8uggbSvCcw1ksilG"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","metadata":{"id":"gyOKkYqTO304"},"source":["!pip install --upgrade tables\n","!pip install eli5\n","!pip install xgboost\n","!pip install hyperopt"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Q1jo7Iz-PESY","executionInfo":{"status":"ok","timestamp":1637951010854,"user_tz":-60,"elapsed":243,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}}},"source":["import pandas as pd\n","import numpy as np\n","\n","import xgboost as xgb\n","\n","from sklearn.metrics import mean_absolute_error as mae\n","from sklearn.model_selection import cross_val_score\n","\n","from hyperopt import hp, fmin, tpe, STATUS_OK\n","\n","import eli5\n","from eli5.sklearn import PermutationImportance"],"execution_count":26,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"sJREhjbxQROC","executionInfo":{"status":"ok","timestamp":1637951339075,"user_tz":-60,"elapsed":248,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}},"outputId":"391c0012-2c2d-44cf-89e6-adb173de1418"},"source":["cd \"/content/drive/MyDrive/Colab Notebooks/dw_transform/transform_two/dw_car/\""],"execution_count":33,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/drive/MyDrive/Colab Notebooks/dw_transform/transform_two/dw_car\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Vm8YcipsQZgS","executionInfo":{"status":"ok","timestamp":1637951342821,"user_tz":-60,"elapsed":2372,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}},"outputId":"80f02dc2-a842-4b32-d0f6-a4539ba5b139"},"source":["df = pd.read_hdf(\"data/car.h5\")\n","df.shape"],"execution_count":34,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(106494, 155)"]},"metadata":{},"execution_count":34}]},{"cell_type":"markdown","metadata":{"id":"RJHVrk7rQm0t"},"source":["## Feature Engineering"]},{"cell_type":"code","metadata":{"id":"liPBqB0rQvSV","executionInfo":{"status":"ok","timestamp":1637951349482,"user_tz":-60,"elapsed":1661,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}}},"source":["SUFFIX_CAT = \"__cat\"\n","for feat in df.columns:\n","  if isinstance(df[feat][0], list): continue\n","\n","  factorized_values = df[feat].factorize()[0]\n","  if SUFFIX_CAT in feat:\n","    df[feat] = factorized_values\n","  else:\n","    df[feat + SUFFIX_CAT] = factorized_values"],"execution_count":35,"outputs":[]},{"cell_type":"code","metadata":{"id":"2troFNhaQ36M","executionInfo":{"status":"ok","timestamp":1637951353306,"user_tz":-60,"elapsed":1269,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}}},"source":["df[\"param_rok-produkcji\"] = df[\"param_rok-produkcji\"].map(lambda x: -1 if str(x) == \"None\" else int(x))\n","df[\"param_moc\"] = df[\"param_moc\"].map(lambda x: -1 if str(x) == \"None\" else int(x.split(\" \")[0]))\n","df[\"param_pojemność-skokowa\"] = df[\"param_pojemność-skokowa\"].map(lambda x: -1 if str(x) == \"None\" else int(str(x).split(\"cm\")[0].replace(\" \", \"\")))"],"execution_count":36,"outputs":[]},{"cell_type":"code","metadata":{"id":"hVjj0R4CRo77","executionInfo":{"status":"ok","timestamp":1637951355384,"user_tz":-60,"elapsed":235,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}}},"source":["def run_model(model, feats):\n","\n","  X = df[feats].values\n","  y = df[\"price_value\"].values\n","\n","  scores = cross_val_score(model, X, y, cv=3, scoring=\"neg_mean_absolute_error\")\n","  return np.mean(scores), np.std(scores)"],"execution_count":37,"outputs":[]},{"cell_type":"code","metadata":{"id":"iSTOZxPWR4PB","executionInfo":{"status":"ok","timestamp":1637951358717,"user_tz":-60,"elapsed":216,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}}},"source":["feats = [\"param_napęd__cat\",\n","\"param_rok-produkcji\",\n","\"param_stan__cat\",\n","\"param_skrzynia-biegów__cat\",\n","\"param_faktura-vat__cat\",\n","\"param_moc\",\n","\"param_marka-pojazdu__cat\",\n","\"feature_kamera-cofania__cat\",\n","\"param_typ__cat\",\n","\"param_pojemność-skokowa\",\n","\"seller_name__cat\",\n","\"feature_wspomaganie-kierownicy__cat\",\n","\"param_model-pojazdu__cat\",\n","\"param_wersja__cat\",\n","\"param_kod-silnika__cat\",\n","\"feature_system-start-stop__cat\",\n","\"feature_asystent-pasa-ruchu__cat\",\n","\"feature_czujniki-parkowania-przednie__cat\",\n","\"feature_łopatki-zmiany-biegów__cat\",\n","\"feature_regulowane-zawieszenie__cat\"]"],"execution_count":38,"outputs":[]},{"cell_type":"code","metadata":{"id":"NOMxSOj1SG76","executionInfo":{"status":"ok","timestamp":1637951361836,"user_tz":-60,"elapsed":348,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}}},"source":["xgb_params = {\n","    \"max_depth\" : 5,\n","    \"n_estimators\": 50,\n","    \"learning_rate\": 0.1,\n","    \"seed\": 0\n","}"],"execution_count":39,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"j9E0sgdESMc2","executionInfo":{"status":"ok","timestamp":1637951377730,"user_tz":-60,"elapsed":13772,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}},"outputId":"2a58c25c-a99e-45bb-c69d-0d5f76cfb5df"},"source":["run_model(xgb.XGBRegressor(**xgb_params), feats)"],"execution_count":40,"outputs":[{"output_type":"stream","name":"stdout","text":["[18:29:23] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n","[18:29:28] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n","[18:29:32] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"]},{"output_type":"execute_result","data":{"text/plain":["(-9569.227198767323, 72.83561801421891)"]},"metadata":{},"execution_count":40}]},{"cell_type":"markdown","metadata":{"id":"4YLCovJ6UVaY"},"source":["## Hyperopt"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"lv3v4e58Sc6Q","executionInfo":{"status":"ok","timestamp":1637952923515,"user_tz":-60,"elapsed":1540107,"user":{"displayName":"Paulina Rakowska","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14502376587314830749"}},"outputId":"20012b19-8351-489a-8886-4ed39119ef33"},"source":["def obj_func(params):\n","  print(\"Training with params: \")\n","  print(params)\n","\n","  mean_mae, score_std = run_model(xgb.XGBRegressor(**params), feats)\n","\n","  return {\"loss\": np.abs(mean_mae), \"status\": STATUS_OK}\n","\n","#space\n","xgb_reg_params = {\n","    \"learning_rate\":    hp.choice(\"learning_rate\",     np.arange(0.05, 0.31, 0.05)),\n","    \"max_depth\":        hp.choice(\"max_depth\",         np.arange(5, 16, 1, dtype=int)),\n","    \"subsample\":        hp.quniform(\"subsample\", 0.5, 1, 0.05),\n","    \"colsample_bytree\": hp.quniform(\"colsample_bytree\", 0.5, 1, 0.05),\n","    \"objective\":        \"reg:squarederror\",\n","    \"n_estimators\":     100,\n","    \"seed\": 0,\n","}\n","\n","#run\n","best = fmin(obj_func, xgb_reg_params, algo=tpe.suggest, max_evals=25)\n","\n","best"],"execution_count":41,"outputs":[{"output_type":"stream","name":"stdout","text":["Training with params: \n","{'colsample_bytree': 0.65, 'learning_rate': 0.05, 'max_depth': 8, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.8}\n","Training with params: \n","{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.1, 'max_depth': 14, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.6000000000000001}\n","Training with params: \n","{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.15000000000000002, 'max_depth': 7, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.7000000000000001}\n","Training with params: \n","{'colsample_bytree': 0.55, 'learning_rate': 0.1, 'max_depth': 13, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.9}\n","Training with params: \n","{'colsample_bytree': 0.5, 'learning_rate': 0.05, 'max_depth': 11, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.65}\n","Training with params: \n","{'colsample_bytree': 0.9, 'learning_rate': 0.15000000000000002, 'max_depth': 6, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.5}\n","Training with params: \n","{'colsample_bytree': 0.8, 'learning_rate': 0.2, 'max_depth': 7, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.9500000000000001}\n","Training with params: \n","{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.15000000000000002, 'max_depth': 14, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.9}\n","Training with params: \n","{'colsample_bytree': 0.8, 'learning_rate': 0.1, 'max_depth': 7, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.9500000000000001}\n","Training with params: \n","{'colsample_bytree': 1.0, 'learning_rate': 0.05, 'max_depth': 13, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.9500000000000001}\n","Training with params: \n","{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.2, 'max_depth': 15, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.55}\n","Training with params: \n","{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.15000000000000002, 'max_depth': 15, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.8}\n","Training with params: \n","{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.05, 'max_depth': 13, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.5}\n","Training with params: \n","{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.1, 'max_depth': 12, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.65}\n","Training with params: \n","{'colsample_bytree': 0.7000000000000001, 'learning_rate': 0.1, 'max_depth': 5, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.6000000000000001}\n","Training with params: \n","{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.25, 'max_depth': 15, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.9500000000000001}\n","Training with params: \n","{'colsample_bytree': 0.9, 'learning_rate': 0.1, 'max_depth': 7, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 1.0}\n","Training with params: \n","{'colsample_bytree': 0.6000000000000001, 'learning_rate': 0.1, 'max_depth': 13, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.8}\n","Training with params: \n","{'colsample_bytree': 0.55, 'learning_rate': 0.2, 'max_depth': 7, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.7000000000000001}\n","Training with params: \n","{'colsample_bytree': 0.8, 'learning_rate': 0.25, 'max_depth': 6, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.7000000000000001}\n","Training with params: \n","{'colsample_bytree': 0.9, 'learning_rate': 0.3, 'max_depth': 12, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.5}\n","Training with params: \n","{'colsample_bytree': 1.0, 'learning_rate': 0.3, 'max_depth': 12, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.55}\n","Training with params: \n","{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.05, 'max_depth': 9, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.6000000000000001}\n","Training with params: \n","{'colsample_bytree': 0.9500000000000001, 'learning_rate': 0.05, 'max_depth': 10, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.55}\n","Training with params: \n","{'colsample_bytree': 0.8500000000000001, 'learning_rate': 0.3, 'max_depth': 12, 'n_estimators': 100, 'objective': 'reg:squarederror', 'seed': 0, 'subsample': 0.65}\n","100%|██████████| 25/25 [25:39<00:00, 61.59s/it, best loss: 7600.182506180269]\n"]},{"output_type":"execute_result","data":{"text/plain":["{'colsample_bytree': 0.9500000000000001,\n"," 'learning_rate': 1,\n"," 'max_depth': 7,\n"," 'subsample': 0.65}"]},"metadata":{},"execution_count":41}]},{"cell_type":"code","metadata":{"id":"u3yaBA2ha6ni"},"source":[""],"execution_count":null,"outputs":[]}]}