GRU KFolds - Lzc.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np  # linear algebra\nimport pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-10-26T06:07:05.917591Z","iopub.execute_input":"2023-10-26T06:07:05.917877Z","iopub.status.idle":"2023-10-26T06:07:06.328418Z","shell.execute_reply.started":"2023-10-26T06:07:05.917849Z","shell.execute_reply":"2023-10-26T06:07:06.327288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset, SubsetRandomSampler, Subset\nfrom tqdm.auto import tqdm\nfrom math import pi, sqrt, exp\nfrom timm.scheduler import CosineLRScheduler\nfrom torch import nn\nfrom sklearn.model_selection import KFold\nimport gc\nimport torch\nimport json\nimport matplotlib.pyplot as plt\nimport joblib\nimport random\nimport warnings\nimport scipy\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:06.330226Z","iopub.execute_input":"2023-10-26T06:07:06.330670Z","iopub.status.idle":"2023-10-26T06:07:12.510010Z","shell.execute_reply.started":"2023-10-26T06:07:06.330639Z","shell.execute_reply":"2023-10-26T06:07:12.508829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    EPOCHS = 15\n    WARMUP_RATIO = 0.2\n    BATCH_SIZE = 1\n    WORKERS = 4\n    TRAIN_RATIO = 0.9\n    MAX_CHUNK_SIZE = 150_000\n    DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    SAMPLE_FREQ = 12\n    SIGMA = 720\n\n\nif CFG.DEVICE == \"cpu\":\n    torch.set_num_interop_threads(CFG.WORKERS)\n    torch.set_num_threads(CFG.WORKERS)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:12.511406Z","iopub.execute_input":"2023-10-26T06:07:12.511798Z","iopub.status.idle":"2023-10-26T06:07:12.547221Z","shell.execute_reply.started":"2023-10-26T06:07:12.511766Z","shell.execute_reply":"2023-10-26T06:07:12.546332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize(y):\n    mean = y[:, 0].mean().item()\n    std = y[:, 0].std().item()\n    y[:, 0] = (y[:, 0] - mean) / (std + 1e-16)\n    mean = y[:, 1].mean().item()\n    std = y[:, 1].std().item()\n    y[:, 1] = (y[:, 1] - mean) / (std + 1e-16)\n    return y","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:12.551198Z","iopub.execute_input":"2023-10-26T06:07:12.551552Z","iopub.status.idle":"2023-10-26T06:07:12.572029Z","shell.execute_reply.started":"2023-10-26T06:07:12.551522Z","shell.execute_reply":"2023-10-26T06:07:12.571128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, model_path=\".\", show=True):\n    epochs = range(1, len(history[\"train_loss\"]) + 1)\n\n    plt.figure()\n    plt.plot(epochs, history[\"train_loss\"], label=\"Training Loss\")\n    plt.plot(epochs, history[\"valid_loss\"], label=\"Validation Loss\")\n    plt.title(\"Loss\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"Loss\")\n    plt.legend()\n    plt.savefig(os.path.join(model_path, \"loss.png\"))\n    if show:\n        plt.show()\n    plt.close()\n\n    plt.figure()\n    plt.plot(epochs, history[\"lr\"])\n    plt.title(\"Learning Rate\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"LR\")\n    plt.savefig(os.path.join(model_path, \"lr.png\"))\n    if show:\n        plt.show()\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:12.573501Z","iopub.execute_input":"2023-10-26T06:07:12.574100Z","iopub.status.idle":"2023-10-26T06:07:12.583655Z","shell.execute_reply.started":"2023-10-26T06:07:12.574068Z","shell.execute_reply":"2023-10-26T06:07:12.582728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, max_chunk_size, loader, device, criterion):\n    model.eval()\n    valid_loss = 0.0\n#     y_true = torch.FloatTensor([]).half()\n#     y_pred = torch.FloatTensor([]).half()\n\n    for X, y in tqdm(loader, desc=\"Eval\", unit=\"batch\"):\n        y = y.to(device)\n        pred = torch.zeros(y.shape).to(device).half()\n\n        h = None\n\n        seq_len = X.shape[1]\n        for i in range(0, seq_len, max_chunk_size):\n            X_chunk = X[:, i:i + max_chunk_size].float().to(device)\n            y_pred, h = model(X_chunk, h)\n            h = [hi.detach() for hi in h]\n            pred[:, i:i + max_chunk_size] = y_pred.half()\n            del X_chunk\n            gc.collect()\n\n        loss = criterion(pred.float(), y.float())\n        valid_loss += loss.item()\n        del pred, loss\n        gc.collect()\n\n    valid_loss /= len(loader)\n\n#     y_true = y_true.squeeze(0)\n#     y_pred = y_pred.squeeze(0)\n    gc.collect()\n    return valid_loss","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:12.585004Z","iopub.execute_input":"2023-10-26T06:07:12.585301Z","iopub.status.idle":"2023-10-26T06:07:12.598321Z","shell.execute_reply.started":"2023-10-26T06:07:12.585269Z","shell.execute_reply":"2023-10-26T06:07:12.597201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResidualBiGRU(nn.Module):\n    def __init__(self, hidden_size, n_layers=1, bidir=True):\n        super(ResidualBiGRU, self).__init__()\n\n        self.hidden_size = hidden_size\n        self.n_layers = n_layers\n\n        self.gru = nn.GRU(\n            hidden_size,\n            hidden_size,\n            num_layers=n_layers,\n            batch_first=True,\n            bidirectional=bidir\n        )\n\n        dir_factor = 2 if bidir else 1\n\n        self.fc1 = nn.Linear(hidden_size * dir_factor,\n                             hidden_size * dir_factor * 2)\n        self.ln1 = nn.LayerNorm(hidden_size * dir_factor * 2)\n        self.fc2 = nn.Linear(hidden_size * dir_factor * 2, hidden_size)\n        self.ln2 = nn.LayerNorm(hidden_size)\n\n    def forward(self, x, h=None):\n        res, new_h = self.gru(x, h)\n\n        res = self.fc1(res)\n        res = self.ln1(res)\n        res = nn.functional.relu(res)\n\n        res = self.fc2(res)\n        res = self.ln2(res)\n        res = nn.functional.relu(res)\n\n        res = res + x\n        return res, new_h","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:12.599639Z","iopub.execute_input":"2023-10-26T06:07:12.599970Z","iopub.status.idle":"2023-10-26T06:07:12.611199Z","shell.execute_reply.started":"2023-10-26T06:07:12.599934Z","shell.execute_reply":"2023-10-26T06:07:12.610396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MultiResidualBiGRU(nn.Module):\n    def __init__(self, input_size, hidden_size, output_size, n_layers, bidir=True):\n        super(MultiResidualBiGRU, self).__init__()\n\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.output_size = output_size\n        self.n_layers = n_layers\n\n        self.fc_in = nn.Linear(input_size, hidden_size)\n        self.ln = nn.LayerNorm(hidden_size)\n        self.res_bigrus = nn.ModuleList([\n            ResidualBiGRU(hidden_size, n_layers=1, bidir=bidir) for _ in range(n_layers)\n        ])\n        self.fc_out = nn.Linear(hidden_size, output_size)\n\n    def forward(self, x, h=None):\n        if h is None:\n            h = [None for _ in range(self.n_layers)]\n\n        x = self.fc_in(x)\n        x = self.ln(x)\n        x = nn.functional.relu(x)\n\n        new_h = []\n        for i, res_bigru in enumerate(self.res_bigrus):\n            x, new_hi = res_bigru(x, h[i])\n            new_h.append(new_hi)\n\n        x = self.fc_out(x)\n        return x, new_h","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:12.613072Z","iopub.execute_input":"2023-10-26T06:07:12.613448Z","iopub.status.idle":"2023-10-26T06:07:12.628091Z","shell.execute_reply.started":"2023-10-26T06:07:12.613408Z","shell.execute_reply":"2023-10-26T06:07:12.626961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SleepDataset(Dataset):\n    def __init__(self, file):\n        self.targets, self.data, self.ids = joblib.load(file)\n\n    def downsample_seq_generate_features(self, feat, downsample_factor=CFG.SAMPLE_FREQ):\n        if len(feat) % downsample_factor == 0:\n            feat = np.concatenate([feat, np.zeros(downsample_factor - ((len(feat)) % downsample_factor)) + feat[-1]])\n        feat = np.reshape(feat, (-1, downsample_factor))\n        feat_mean = np.mean(feat, 1)\n        feat_std = np.std(feat, 1)\n        feat_median = np.median(feat, 1)\n        feat_max = np.max(feat, 1)\n        feat_min = np.min(feat, 1)\n        feat_iqr = np.percentile(feat, 75, axis=1) - np.percentile(feat, 25, axis=1)\n#         feat_skew = scipy.stats.skew(feat, axis=1)\n#         feat_kurt = scipy.stats.kurtosis(feat, axis=1)\n\n        return np.dstack([feat_mean, feat_std, feat_median, feat_max, feat_min, feat_iqr])[0]\n\n    def downsample_seq(self, feat, downsample_factor=CFG.SAMPLE_FREQ):\n        if len(feat) % downsample_factor == 0:\n            feat = np.concatenate([feat, np.zeros(downsample_factor - ((len(feat)) % downsample_factor)) + feat[-1]])\n        feat = np.reshape(feat, (-1, downsample_factor))\n        feat_mean = np.mean(feat, 1)\n        return feat_mean\n\n    def gauss(self, n=CFG.SIGMA, sigma=CFG.SIGMA * 0.15):\n        r = range(-int(n / 2), int(n / 2) + 1)\n        return [1 / (sigma * sqrt(2 * pi)) * exp(-float(x) ** 2 / (2 * sigma ** 2)) for x in r]\n\n    def __len__(self):\n        return len(self.targets)\n\n    def __getitem__(self, index):\n        X = self.data[index][[\"anglez\", \"enmo\"]]\n#         print(\"X shape:\", X.shape)\n        y = self.targets[index]\n#         print(\"y len:\", len(y))\n\n        target_gaussian = np.zeros((len(X), 2))\n        for s, e in y:\n            st1, st2 = max(0, s - CFG.SIGMA // 2), s + CFG.SIGMA // 2 + 1\n            ed1, ed2 = e - CFG.SIGMA // 2, min(len(X), e + CFG.SIGMA // 2 + 1)\n            target_gaussian[st1:st2, 0] = self.gauss()[st1 - (s - CFG.SIGMA // 2):]\n            target_gaussian[ed1:ed2, 1] = self.gauss()[:CFG.SIGMA + 1 - ((e + CFG.SIGMA // 2 + 1) - ed2)]\n            gc.collect()\n\n        y = target_gaussian\n#         print(\"Target_gaussian shape:\", y.shape)\n#         print(y)\n        gc.collect()\n\n        X = np.concatenate([self.downsample_seq_generate_features(X.values[:, i], CFG.SAMPLE_FREQ) for i in range(X.shape[1])], -1)\n#         print(\"X shape:\", X.shape)\n#         print(X)\n        gc.collect()\n\n        y = np.dstack([self.downsample_seq(y[:, i], CFG.SAMPLE_FREQ) for i in range(y.shape[1])])[0]\n#         print(\"y shape:\", y.shape)\n#         print(y)\n        gc.collect()\n\n        y = normalize(torch.from_numpy(y))\n        X = torch.from_numpy(X)\n#         print(\"X shape:\", X.shape)\n#         print(\"y shape:\", y.shape)\n        return X, y","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:12.629874Z","iopub.execute_input":"2023-10-26T06:07:12.630263Z","iopub.status.idle":"2023-10-26T06:07:12.655614Z","shell.execute_reply.started":"2023-10-26T06:07:12.630226Z","shell.execute_reply":"2023-10-26T06:07:12.654355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = SleepDataset(\"/kaggle/input/detect-sleep-states-train-data/train_data.pkl\")","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:12.659973Z","iopub.execute_input":"2023-10-26T06:07:12.660400Z","iopub.status.idle":"2023-10-26T06:07:23.465697Z","shell.execute_reply.started":"2023-10-26T06:07:12.660357Z","shell.execute_reply":"2023-10-26T06:07:23.464669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds[0]","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:23.467271Z","iopub.execute_input":"2023-10-26T06:07:23.467647Z","iopub.status.idle":"2023-10-26T06:07:27.270044Z","shell.execute_reply.started":"2023-10-26T06:07:23.467617Z","shell.execute_reply":"2023-10-26T06:07:27.268835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_size = int(CFG.TRAIN_RATIO * len(train_ds))\n# valid_size = len(train_ds) - train_size\n\n# indices = torch.randperm(len(train_ds))\n# train_sampler = SubsetRandomSampler(indices[:train_size])\n# valid_sampler = SubsetRandomSampler(indices[train_size:train_size + valid_size])\n\n# steps = train_size * CFG.EPOCHS\n# warmup_steps = int(steps * CFG.WARMUP_RATIO)\n\nmodel = MultiResidualBiGRU(\n    input_size=12,\n    hidden_size=64,\n    output_size=2,\n    n_layers=5\n).to(CFG.DEVICE)\n\n# scheduler = CosineLRScheduler(\n#     optimizer=optimizer,\n#     t_initial=steps,\n#     warmup_t=warmup_steps,\n#     warmup_lr_init=1e-6,\n#     lr_min=2e-8\n# )\n\ncriterion = torch.nn.MSELoss()\n\nmodel_path = \".\"\n\nhistory = {\n    \"train_loss\": [],\n    \"valid_loss\": [],\n    \"valid_mAP\": [],\n    \"lr\": []\n}\n\nbest_valid_loss = np.inf","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:27.271625Z","iopub.execute_input":"2023-10-26T06:07:27.272046Z","iopub.status.idle":"2023-10-26T06:07:27.295704Z","shell.execute_reply.started":"2023-10-26T06:07:27.272007Z","shell.execute_reply":"2023-10-26T06:07:27.294494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model)\ntotal_params = sum(p.numel() for p in model.parameters())\nprint(f\"Total parameters: {total_params}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:27.297157Z","iopub.execute_input":"2023-10-26T06:07:27.297600Z","iopub.status.idle":"2023-10-26T06:07:27.304337Z","shell.execute_reply.started":"2023-10-26T06:07:27.297563Z","shell.execute_reply":"2023-10-26T06:07:27.303384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_dataloader = DataLoader(\n#     train_ds,\n#     batch_size=CFG.BATCH_SIZE,\n#     sampler=train_sampler,\n#     pin_memory=True,\n#     num_workers=CFG.WORKERS\n# )\n\n# valid_dataloader = DataLoader(\n#     train_ds,\n#     batch_size=1,\n#     sampler=valid_sampler,\n#     pin_memory=True,\n#     num_workers=CFG.WORKERS\n# )","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:27.305652Z","iopub.execute_input":"2023-10-26T06:07:27.306210Z","iopub.status.idle":"2023-10-26T06:07:27.316256Z","shell.execute_reply.started":"2023-10-26T06:07:27.306180Z","shell.execute_reply":"2023-10-26T06:07:27.315417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_folds = 7\nkf = KFold(n_splits=num_folds, shuffle=True, random_state=42)\n\nfor fold, (train_index, val_index) in enumerate(kf.split(train_ds)):\n    train_subset = Subset(train_ds, train_index)\n    val_subset = Subset(train_ds, val_index)\n\n    train_loader = DataLoader(train_subset, batch_size=CFG.BATCH_SIZE, shuffle=True)\n    val_loader = DataLoader(val_subset, batch_size=CFG.BATCH_SIZE)\n\n    steps = len(train_index) * CFG.EPOCHS\n    warmup_steps = int(steps * CFG.WARMUP_RATIO)\n    \n    init_lr = 8e-4\n\n    if fold == 0:\n        optimizer = torch.optim.Adam(\n            model.parameters(),\n            lr=init_lr,\n            weight_decay=0.001\n        )\n    else:\n        current_learning_rate = optimizer.param_groups[0][\"lr\"]\n        print(\"current learning rate is:\", current_learning_rate)\n        optimizer = torch.optim.Adam(\n            model.parameters(),\n            lr=init_lr / (1.5 ** fold),\n            weight_decay=0.001\n        )\n\n#     optimizer = torch.optim.Adam(\n#         model.parameters(),\n#         lr=5e-4,\n#         weight_decay=0.001\n#     )\n    \n    scheduler = CosineLRScheduler(\n        optimizer=optimizer,\n        t_initial=steps,\n        warmup_t=warmup_steps,\n        warmup_lr_init=1e-6,\n        lr_min=2e-8\n    )\n\n    for epoch in range(1, CFG.EPOCHS + 1):\n        train_loss = 0.0\n        n_tot_chunks = 0\n        pbar = tqdm(train_loader, desc=\"Training\", unit=\"batch\")\n        model.train()\n\n        for step, (X, y) in enumerate(pbar):\n            y = y.to(CFG.DEVICE)\n            pred = torch.zeros(y.shape).to(CFG.DEVICE)\n            optimizer.zero_grad()\n            scheduler.step(step + len(train_index) * epoch)\n\n            h = None\n            seq_len = X.shape[1]\n            for i in range(0, seq_len, CFG.MAX_CHUNK_SIZE):\n                X_chunk = X[:, i:i + CFG.MAX_CHUNK_SIZE].float()\n                X_chunk = X_chunk.to(CFG.DEVICE)\n                y_pred, h = model(X_chunk, h)\n                h = [hi.detach() for hi in h]\n                pred[:, i:i + CFG.MAX_CHUNK_SIZE] = y_pred\n                del X_chunk, y_pred\n\n            loss = criterion(normalize(pred).float(), y.float())\n            loss.backward()\n            train_loss += loss.item()\n            n_tot_chunks += 1\n            pbar.set_description(\n                f\"Training: loss = {(train_loss / n_tot_chunks):.6f}\")\n            nn.utils.clip_grad_norm_(model.parameters(), max_norm=1e-1)\n            optimizer.step()\n            del pred, loss, y, X, h\n            gc.collect()\n\n        train_loss /= len(train_loader)\n        del pbar\n        gc.collect()\n\n        if epoch % 1 == 0:\n            valid_loss = evaluate(model, CFG.MAX_CHUNK_SIZE,\n                                  val_loader, CFG.DEVICE, criterion)\n            history[\"train_loss\"].append(train_loss)\n            history[\"valid_loss\"].append(valid_loss)\n            history[\"lr\"].append(optimizer.param_groups[0][\"lr\"])\n\n            if valid_loss < best_valid_loss:\n                best_valid_loss = valid_loss\n                torch.save(model.state_dict(), os.path.join(\n                    model_path, f\"model_best_fold_{fold + 1}_epoch_{epoch}.pth\"))\n\n            print(\n                f\"Fold {fold + 1} -- \"\n                f\"Epoch: {epoch} / {CFG.EPOCHS} -- \",\n                f\"train_loss = {train_loss:.6f} -- \",\n                f\"valid_loss = {valid_loss:.6f} -- \"\n            )","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:07:27.317428Z","iopub.execute_input":"2023-10-26T06:07:27.318022Z","iopub.status.idle":"2023-10-26T06:08:29.196928Z","shell.execute_reply.started":"2023-10-26T06:07:27.317992Z","shell.execute_reply":"2023-10-26T06:08:29.195344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, model_path=model_path)\nhistory_path = os.path.join(model_path, \"history.json\")\nwith open(history_path, \"w\", encoding=\"utf-8\") as f:\n    json.dump(history, f, ensure_ascii=False, indent=4)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T06:08:29.198575Z","iopub.status.idle":"2023-10-26T06:08:29.199428Z","shell.execute_reply.started":"2023-10-26T06:08:29.199128Z","shell.execute_reply":"2023-10-26T06:08:29.199155Z"},"trusted":true},"execution_count":null,"outputs":[]}]}