am24-taskC-DynamicsClass.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task C: Dynamics classification (p,mf,f)\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dataset.aGPTset.ExpressiveGuitarTechniquesDataset as agptset\n",
    "import os\n",
    "import librosa\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import multiprocessing as mp\n",
    "import pandas as pd\n",
    "import sklearn as sk\n",
    "import imblearn\n",
    "\n",
    "import am24utils\n",
    "from am24utils import Run\n",
    "\n",
    "\n",
    "dataset = agptset.import_db()\n",
    "\n",
    "DOTEST = False\n",
    "VERBOSE = False\n",
    "DB_PATH = 'dataset/aGPTset'\n",
    "printVerbose = lambda x: print(x) if VERBOSE else None\n",
    "printVerboseLevel1 = lambda x,vl: print(x) if vl > 1 else None\n",
    "printVerboseLevel2 = lambda x,vl: print(x) if vl > 2 else None\n",
    "printVerboseLevel3 = lambda x,vl: print(x) if vl > 3 else None\n",
    "MULTIPROCESSING = False\n",
    "\n",
    "if MULTIPROCESSING:\n",
    "    import multiprocessing as mp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DYNAMICS=['forte', 'mezzoforte', 'piano']\n",
    "dynamics_str_to_int = lambda x: {DYNAMICS[0]:0, DYNAMICS[1]:1, DYNAMICS[2]:2}[x]\n",
    "dynamics_int_to_str = lambda x: DYNAMICS[x] if (x < 3 and x>=0 and type(x)==int) else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "print(\"Filtering the Dataset...\")\n",
    "\n",
    "# Filter the db to keep only pitched notes\n",
    "filtered_notes_db, filtered_files_db = am24utils.filter_files_db(dataset)\n",
    "#make sure that no audio_file_path contains \"impro\"\n",
    "assert filtered_notes_db.index.get_level_values(1).str.contains('impro').sum() == 0, \"Some audio_file_path contain 'impro' (%i)\"%(filtered_notes_db.index.get_level_values(1).str.contains('impro').sum())\n",
    "print(\"Done (%i notes in the filtered db).\"%len(filtered_notes_db))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_onsetlist_filename_dynamics(filtered_notes_db:pd.DataFrame, filtered_files_db:pd.DataFrame):\n",
    "    onsetlist = []\n",
    "    filenames = []\n",
    "    players = []\n",
    "    dynamics = []\n",
    "    for file in filtered_notes_db.index.get_level_values(1).unique():\n",
    "        if file in filtered_files_db.index:\n",
    "            afp = filtered_files_db[filtered_files_db.index == file].full_audiofile_path.values\n",
    "            assert len(afp) == 1, \"More than one audio file path for file %s\"%file\n",
    "            filenames.append(afp[0])\n",
    "            cur_onset_list = filtered_notes_db.loc[filtered_notes_db.index.get_level_values(1) == file].onset_label_samples.values\n",
    "            cur_dynamics = filtered_notes_db.loc[filtered_notes_db.index.get_level_values(1) == file].playing_intensity.values\n",
    "            assert len(cur_onset_list) == len(cur_dynamics), \"Onset list and cur_dynamics have different lengths\"\n",
    "            # print('%s has %i onsets and %i cur_dynamics'%(file,len(cur_onset_list),len(cur_dynamics)))\n",
    "            cur_onset_list = [int(x) for x in cur_onset_list]\n",
    "            onsetlist.append(cur_onset_list)\n",
    "            cur_dynamics = [dynamics_str_to_int(x) for x in cur_dynamics]\n",
    "            dynamics.append(cur_dynamics)\n",
    "            cur_player = filtered_files_db[filtered_files_db.index == file].player_id.values\n",
    "            assert len(cur_player) == 1, \"More than one player for file %s\"%file\n",
    "            cur_player = int(cur_player[0])\n",
    "            players.append(cur_player)\n",
    "        else:\n",
    "            raise ValueError(\"File %s not found in the files db\"%file)\n",
    "        \n",
    "    return onsetlist, filenames, dynamics, players\n",
    "        \n",
    "\n",
    "onsetlist,filenames,dynamics,playerlist  = get_onsetlist_filename_dynamics(filtered_notes_db,filtered_files_db)\n",
    "assert len(onsetlist) == len(filenames) == len(dynamics) == len(playerlist), \"Different lengths for onsetlist, filenames, dynamics and playerlist\"\n",
    "packedData = (onsetlist,filenames,dynamics,playerlist)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_and_compute_features_for_file(cur_filename, \n",
    "                                       cur_onsetlist, \n",
    "                                       cur_dynamicslist, \n",
    "                                       cur_player,\n",
    "                                       window_size_samples,\n",
    "                                       onset_perturbation_distribution,\n",
    "                                       onset_perturbation_max_samples, \n",
    "                                       onset_perturbation_min_samples):\n",
    "    # print('Processing file %s'%cur_filename)\n",
    "    assert len(cur_onsetlist) == len(cur_dynamicslist), \"onsetlist and cur_dynamicslist have different lengths (%i != %i)\"%(len(cur_onsetlist), len(cur_dynamicslist))\n",
    "    if onset_perturbation_distribution is not None:\n",
    "        # print('Applying onset perturbation to file %s'%cur_filename)\n",
    "        cur_onsetlist = am24utils.apply_onset_perturbation(cur_onsetlist, onset_perturbation_distribution, onset_perturbation_max_samples, onset_perturbation_min_samples)\n",
    "\n",
    "    # print('Computing features for file %s'%cur_filename)\n",
    "    Xfn, yfn = am24utils.get_Xy(cur_filename, cur_onsetlist, cur_dynamicslist, window_size_samples)\n",
    "    \n",
    "    assert len(Xfn) == len(yfn), \"Xfn and yfn have different lengths (%i != %i)\"%(len(Xfn), len(yfn))\n",
    "\n",
    "    print('.',end='',flush=True)\n",
    "    playerlist = [cur_player]*len(Xfn)\n",
    "    return Xfn, yfn,playerlist\n",
    "\n",
    "def run_taskC(runs, packedData, classifier='KNN'):\n",
    "    onsetlist_list,filenames_list,dynamics_list, player_list = packedData\n",
    "    assert len(onsetlist_list) == len(filenames_list) == len(dynamics_list) == len(player_list), \"Different lengths for onsetlist_list, filenames_list, dynamics_list and player_list\"\n",
    "    for ridx,run in enumerate(runs):\n",
    "        print('Running task B for Run:%s [%i,%i]'%(run.name,ridx+1,len(runs)), end='\\r')\n",
    "        print('+--%s--Arguments--------------+'%(run.name))\n",
    "        print('| Window size: %i'%run.window_size_samples)\n",
    "        print('| Onset perturbation distribution: %s'%run.onset_perturbation_distribution)\n",
    "        print('| Onset perturbation max samples: %i'%run.onset_perturbation_max_samples)\n",
    "        print('| Onset perturbation min samples: %i'%run.onset_perturbation_min_samples)\n",
    "        print('+-------------------------------------+')\n",
    "\n",
    "        '''\n",
    "        # for i in range(len(onsetlist_list)):\n",
    "        #     Xfn, yfn, playervec = load_and_compute_features_for_file(os.path.join(DB_PATH,filenames_list[i]),\n",
    "        #                                                              onsetlist_list[i], \n",
    "        #                                                              dynamics_list[i], \n",
    "        #                                                              player_list[i])\n",
    "        #     X.extend(Xfn)\n",
    "        #     y.extend(yfn)\n",
    "        #     group.extend(playervec) # Add the player id to the group list, repeated for each onset\n",
    "        '''\n",
    "\n",
    "        X,y,group = [],[],[]\n",
    "        if MULTIPROCESSING:\n",
    "            # replace previous commented block with parallel processing\n",
    "            pool = mp.Pool(mp.cpu_count())\n",
    "            # results = [pool.apply_async(load_and_compute_features_for_file, args=(os.path.join(DB_PATH,filenames_list[i]), onsetlist_list[i], dynamics_list[i], player_list[i])) for i in range(len(onsetlist_list))]\n",
    "            results = [pool.apply_async(load_and_compute_features_for_file, \n",
    "                        args=(os.path.join(DB_PATH,filenames_list[i]), \n",
    "                            onsetlist_list[i], \n",
    "                            dynamics_list[i], \n",
    "                            player_list[i], \n",
    "                            run.window_size_samples, \n",
    "                            run.onset_perturbation_distribution, \n",
    "                            run.onset_perturbation_max_samples, \n",
    "                            run.onset_perturbation_min_samples)) for i in range(len(onsetlist_list))]\n",
    "            pool.close()\n",
    "            pool.join()\n",
    "\n",
    "            print('All files processed.')\n",
    "            \n",
    "            X,y,group = [],[],[]\n",
    "            for r in results:\n",
    "                Xfnret, yfnret, playervecret = r.get()\n",
    "                X.extend(Xfnret)\n",
    "                y.extend(yfnret)\n",
    "                group.extend(playervecret)\n",
    "        else:\n",
    "            for i in range(len(onsetlist_list)):\n",
    "                Xfn, yfn, playervec = load_and_compute_features_for_file(os.path.join(DB_PATH,filenames_list[i]),\n",
    "                                                                        onsetlist_list[i], \n",
    "                                                                        dynamics_list[i], \n",
    "                                                                        player_list[i],\n",
    "                                                                        run.window_size_samples, \n",
    "                                                                        run.onset_perturbation_distribution, \n",
    "                                                                        run.onset_perturbation_max_samples, \n",
    "                                                                        run.onset_perturbation_min_samples)\n",
    "                X.extend(Xfn)\n",
    "                y.extend(yfn)\n",
    "                group.extend(playervec) # Add the player id to the group list, repeated for each onset\n",
    "\n",
    "        \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        \n",
    "        assert len(X) == len(y), \"X and y have different lengths (%i != %i)\"%(len(X), len(y))\n",
    "        assert len(X) == len(group), \"X and group have different lengths (%i != %i)\"%(len(X), len(group))\n",
    "        # print('X',X)\n",
    "        # print('y',y)\n",
    "        # print(group)\n",
    "        \n",
    "\n",
    "        from sklearn.model_selection import StratifiedGroupKFold\n",
    "        N_FOLDS = 3\n",
    "        skf = StratifiedGroupKFold(n_splits=N_FOLDS)\n",
    "\n",
    "        skf.get_n_splits(X, y, group)\n",
    "        run.results = []\n",
    "        for fold_idx, (train_index, test_index) in enumerate(skf.split(X, y, group)):\n",
    "            X_train, X_test = np.array(X)[train_index], np.array(X)[test_index]\n",
    "            y_train, y_test = np.array(y)[train_index], np.array(y)[test_index]\n",
    "            group_train, group_test = np.array(group)[train_index], np.array(group)[test_index]\n",
    "            print('Fold %i/%i'%(fold_idx+1,N_FOLDS))\n",
    "            print('Train:', len(X_train), len(y_train),'Groups: ', sorted(list(set(group_train))))\n",
    "            print('Test:', len(X_test), len(y_test),'Groups: ', sorted(list(set(group_test))))\n",
    "            \n",
    "            # For each test set, print the percentage of each dynamic (0,1,2)\n",
    "            eachdyn = []\n",
    "            for dynamic in range(len(DYNAMICS)):\n",
    "                eachdyn.append(100*sum(y_test==dynamic)/len(y_test))\n",
    "                print('Dynamic %s in test set: %.2f%%'%(DYNAMICS[dynamic], eachdyn[-1]))\n",
    "                \n",
    "\n",
    "            # If any of the dynamics percentage is more than 10%than the equal split, enable smote\n",
    "            if any([abs(x-100/len(DYNAMICS))>10 for x in eachdyn]):\n",
    "                # Use SMOTE to balance the classes\n",
    "                from imblearn.over_sampling import SMOTE\n",
    "                smote = SMOTE()\n",
    "                print('Balancing classes with SMOTE ...')\n",
    "                X_train, y_train = smote.fit_resample(X_train, y_train)\n",
    "        \n",
    "            if classifier.upper()=='KNN':\n",
    "                from sklearn.neighbors import KNeighborsClassifier as KNN\n",
    "                # Train KNN\n",
    "                print('Training', classifier, '...')\n",
    "                knn = KNN(n_neighbors=3)\n",
    "                knn.fit(X_train, y_train)\n",
    "                \n",
    "                # Test\n",
    "                y_pred = knn.predict(X_test)\n",
    "            elif classifier.upper()=='SVM':\n",
    "                from sklearn.svm import SVC\n",
    "                # Train SVM\n",
    "                svm = SVC()\n",
    "                print('Training', classifier, '...')\n",
    "                svm.fit(X_train, y_train)\n",
    "                \n",
    "                # Test\n",
    "                y_pred = svm.predict(X_test)\n",
    "                \n",
    "            # TODO: uncomment NN classifier and test it    \n",
    "            # elif classifier.upper()=='RESNET':\n",
    "            #     import torch\n",
    "            #     from am24utils import AudioResNet\n",
    "            #     # Train ResNet for audio classification\n",
    "            #     device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
    "            #     resnet = AudioResNet(num_classes=3).to(device)\n",
    "            #     print('Training', classifier, '...')\n",
    "            #     resnet.fit(X_train, y_train)\n",
    "                \n",
    "            #     # Test\n",
    "            #     y_pred = resnet.predict(X_test)\n",
    "                \n",
    "            # elif classifier.upper()=='SQUEEZENET':\n",
    "            #     import torch\n",
    "            #     from am24utils import AudioSqueezeNet\n",
    "            #     # Train ResNet for audio classification\n",
    "            #     device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
    "            #     squeezenet = AudioSqueezeNet(num_classes=3).to(device)\n",
    "            #     print('Training', classifier, '...')\n",
    "            #     resnet.fit(X_train, y_train)\n",
    "                \n",
    "            #     # Test\n",
    "            #     y_pred = resnet.predict(X_test)\n",
    "                \n",
    "            \n",
    "            # # Metrics\n",
    "            classification_report = sk.metrics.classification_report(y_test, y_pred, labels=[dynamics_str_to_int(x) for x in DYNAMICS],target_names=DYNAMICS)\n",
    "            print('-Classification FOLD %i/%i-'%(fold_idx+1,N_FOLDS))\n",
    "            print('\\t\\t\\t'+classification_report.replace('\\n','\\n\\t\\t\\t'))\n",
    "            print('\\n\\n')\n",
    "\n",
    "            confusion_matrix = sk.metrics.confusion_matrix(y_test, y_pred, labels=[dynamics_str_to_int(x) for x in DYNAMICS] )\n",
    "            print('-Confusion Matrix FOLD %i/%i-'%(fold_idx+1,N_FOLDS))\n",
    "            print('\\t\\t\\t'+str(confusion_matrix).replace('\\n','\\n\\t\\t\\t'))\n",
    "            print('\\n\\n')\n",
    "            print('\\n\\n')\n",
    "            \n",
    "        \n",
    "            run.foldresults.append({\n",
    "                'classification_report': classification_report,\n",
    "                'confusion_matrix': confusion_matrix,\n",
    "                'accuracy': sk.metrics.accuracy_score(y_test, y_pred),\n",
    "                'dictclassifiction_report' : sk.metrics.classification_report(y_test, y_pred, output_dict=True)\n",
    "            })\n",
    "        print('Run %s done.'%run.name)\n",
    "\n",
    "        print('Average classification report over %i folds:',len(run.foldresults))\n",
    "        mean_dict = am24utils.report_average([r['dictclassifiction_report'] for r in run.foldresults])\n",
    "        mean_accuracy = sum([r['accuracy'] for r in run.foldresults])/len(run.foldresults)\n",
    "        run.results = {'mean_classification_report_dict': mean_dict,'mean_classification_report_str': am24utils.classification_report_dict2print(mean_dict), 'mean_accuracy': mean_accuracy, 'number_folds': N_FOLDS}\n",
    "        print(run.results['mean_classification_report_str'])\n",
    "       \n",
    "\n",
    "\n",
    "# TEST_WINDOWSIZES_VALUES = [2400]\n",
    "# TEST_PERTURBATION_DISTRIBUTIONS = ['normal']\n",
    "# TEST_PERTURBATION_MAXSAMPLES = [2400, 0]\n",
    "\n",
    "# to_run = am24utils.get_run_list(winsizes = TEST_WINDOWSIZES_VALUES,\n",
    "#                  pert_distributions = TEST_PERTURBATION_DISTRIBUTIONS,\n",
    "#                  pert_maxsamples  = TEST_PERTURBATION_MAXSAMPLES)       \n",
    "       \n",
    "       \n",
    "# Create the runs        \n",
    "to_run = am24utils.get_run_list()\n",
    "\n",
    "for ridx,run in enumerate(to_run):\n",
    "    print(run.name)\n",
    "\n",
    "run_taskC(to_run, packedData=packedData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle, datetime\n",
    "\n",
    "\n",
    "resdir_path = os.path.join('results','task-C','date_%s'%(datetime.datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")))\n",
    "os.makedirs(resdir_path)\n",
    "\n",
    "bakfilename  = 'taskC_KNN_results.pickle'\n",
    "\n",
    "with open(os.path.join(resdir_path,bakfilename), 'wb') as f:\n",
    "    pickle.dump(to_run, f)\n",
    "    \n",
    "am24utils.plot_runs(to_run, 'mean_f1')\n",
    "plt.savefig(os.path.join(resdir_path,'accuracy_KNN.png'))\n",
    "plt.savefig(os.path.join(resdir_path,'accuracy_KNN.pdf'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST WITH RESNET AND SQUEEZENET\n",
    "\n",
    "def load_and_compute_features_for_file(cur_filename, \n",
    "                                       cur_onsetlist, \n",
    "                                       cur_dynamicslist, \n",
    "                                       cur_player,\n",
    "                                       window_size_samples,\n",
    "                                       onset_perturbation_distribution,\n",
    "                                       onset_perturbation_max_samples, \n",
    "                                       onset_perturbation_min_samples):\n",
    "    # print('Processing file %s'%cur_filename)\n",
    "    assert len(cur_onsetlist) == len(cur_dynamicslist), \"onsetlist and cur_dynamicslist have different lengths (%i != %i)\"%(len(cur_onsetlist), len(cur_dynamicslist))\n",
    "    if onset_perturbation_distribution is not None:\n",
    "        # print('Applying onset perturbation to file %s'%cur_filename)\n",
    "        cur_onsetlist = am24utils.apply_onset_perturbation(cur_onsetlist, onset_perturbation_distribution, onset_perturbation_max_samples, onset_perturbation_min_samples)\n",
    "\n",
    "    # print('Computing features for file %s'%cur_filename)\n",
    "    Xfn, yfn = am24utils.get_Xy(cur_filename, cur_onsetlist, cur_dynamicslist, window_size_samples, features=[\"log-mel\",\"mfcc\"])\n",
    "    \n",
    "    assert len(Xfn) == len(yfn), \"Xfn and yfn have different lengths (%i != %i)\"%(len(Xfn), len(yfn))\n",
    "\n",
    "    print('.',end='',flush=True)\n",
    "    playerlist = [cur_player]*len(Xfn)\n",
    "    return Xfn, yfn,playerlist\n",
    "\n",
    "def run_taskC(runs, packedData, classifier='RESNET'):\n",
    "    onsetlist_list,filenames_list,dynamics_list, player_list = packedData\n",
    "    assert len(onsetlist_list) == len(filenames_list) == len(dynamics_list) == len(player_list), \"Different lengths for onsetlist_list, filenames_list, dynamics_list and player_list\"\n",
    "    for ridx,run in enumerate(runs):\n",
    "        print('Running task B for Run:%s [%i,%i]'%(run.name,ridx+1,len(runs)), end='\\r')\n",
    "        print('+--%s--Arguments--------------+'%(run.name))\n",
    "        print('| Window size: %i'%run.window_size_samples)\n",
    "        print('| Onset perturbation distribution: %s'%run.onset_perturbation_distribution)\n",
    "        print('| Onset perturbation max samples: %i'%run.onset_perturbation_max_samples)\n",
    "        print('| Onset perturbation min samples: %i'%run.onset_perturbation_min_samples)\n",
    "        print('+-------------------------------------+')\n",
    "\n",
    "        '''\n",
    "        # for i in range(len(onsetlist_list)):\n",
    "        #     Xfn, yfn, playervec = load_and_compute_features_for_file(os.path.join(DB_PATH,filenames_list[i]),\n",
    "        #                                                              onsetlist_list[i], \n",
    "        #                                                              dynamics_list[i], \n",
    "        #                                                              player_list[i])\n",
    "        #     X.extend(Xfn)\n",
    "        #     y.extend(yfn)\n",
    "        #     group.extend(playervec) # Add the player id to the group list, repeated for each onset\n",
    "        '''\n",
    "\n",
    "        X,y,group = [],[],[]\n",
    "        if MULTIPROCESSING:\n",
    "            # replace previous commented block with parallel processing\n",
    "            pool = mp.Pool(mp.cpu_count())\n",
    "            # results = [pool.apply_async(load_and_compute_features_for_file, args=(os.path.join(DB_PATH,filenames_list[i]), onsetlist_list[i], dynamics_list[i], player_list[i])) for i in range(len(onsetlist_list))]\n",
    "            results = [pool.apply_async(load_and_compute_features_for_file, \n",
    "                        args=(os.path.join(DB_PATH,filenames_list[i]), \n",
    "                            onsetlist_list[i], \n",
    "                            dynamics_list[i], \n",
    "                            player_list[i], \n",
    "                            run.window_size_samples, \n",
    "                            run.onset_perturbation_distribution, \n",
    "                            run.onset_perturbation_max_samples, \n",
    "                            run.onset_perturbation_min_samples)) for i in range(len(onsetlist_list))]\n",
    "            pool.close()\n",
    "            pool.join()\n",
    "\n",
    "            print('All files processed.')\n",
    "            \n",
    "            X,y,group = [],[],[]\n",
    "            for r in results:\n",
    "                Xfnret, yfnret, playervecret = r.get()\n",
    "                X.extend(Xfnret)\n",
    "                y.extend(yfnret)\n",
    "                group.extend(playervecret)\n",
    "        else:\n",
    "            for i in range(len(onsetlist_list)):\n",
    "                Xfn, yfn, playervec = load_and_compute_features_for_file(os.path.join(DB_PATH,filenames_list[i]),\n",
    "                                                                        onsetlist_list[i], \n",
    "                                                                        dynamics_list[i], \n",
    "                                                                        player_list[i],\n",
    "                                                                        run.window_size_samples, \n",
    "                                                                        run.onset_perturbation_distribution, \n",
    "                                                                        run.onset_perturbation_max_samples, \n",
    "                                                                        run.onset_perturbation_min_samples)\n",
    "                X.extend(Xfn)\n",
    "                y.extend(yfn)\n",
    "                group.extend(playervec) # Add the player id to the group list, repeated for each onset\n",
    "\n",
    "        \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        \n",
    "        assert len(X) == len(y), \"X and y have different lengths (%i != %i)\"%(len(X), len(y))\n",
    "        assert len(X) == len(group), \"X and group have different lengths (%i != %i)\"%(len(X), len(group))\n",
    "        # print('X',X)\n",
    "        # print('y',y)\n",
    "        # print(group)\n",
    "        \n",
    "\n",
    "        from sklearn.model_selection import StratifiedGroupKFold, KFold\n",
    "        N_FOLDS = 3\n",
    "        skf = StratifiedGroupKFold(n_splits=N_FOLDS, shuffle=True, random_state=42)\n",
    "        kf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=42)\n",
    "\n",
    "        skf.get_n_splits(X, y, group)\n",
    "        run.results = []\n",
    "        # for fold_idx, (train_index, test_index) in enumerate(skf.split(X, y, group)):\n",
    "        for fold_idx, (train_index, test_index) in enumerate(kf.split(X, y, group)):\n",
    "            X_features = [x[1:] for x in X]\n",
    "            X_specs = [x[0] for x in X]\n",
    "            X_train, X_test = np.array(X_features)[train_index], np.array(X_features)[test_index]\n",
    "            S_train, S_test = np.array(X_specs)[train_index], np.array(X_specs)[test_index]\n",
    "            y_train, y_test = np.array(y)[train_index], np.array(y)[test_index]\n",
    "            group_train, group_test = np.array(group)[train_index], np.array(group)[test_index]\n",
    "            print('Fold %i/%i'%(fold_idx+1,N_FOLDS))\n",
    "            print('Train:', len(X_train), len(y_train),'Groups: ', sorted(list(set(group_train))))\n",
    "            print('Test:', len(X_test), len(y_test),'Groups: ', sorted(list(set(group_test))))\n",
    "            \n",
    "            # For each test set, print the percentage of each dynamic (0,1,2)\n",
    "            eachdyn = []\n",
    "            for dynamic in range(len(DYNAMICS)):\n",
    "                eachdyn.append(100*sum(y_test==dynamic)/len(y_test))\n",
    "                print('Dynamic %s in test set: %.2f%%'%(DYNAMICS[dynamic], eachdyn[-1]))\n",
    "                \n",
    "\n",
    "            # If any of the dynamics percentage is more than 10%than the equal split, enable smote\n",
    "            if any([abs(x-100/len(DYNAMICS))>10 for x in eachdyn]):\n",
    "                # Use SMOTE to balance the classes\n",
    "                from imblearn.over_sampling import SMOTE\n",
    "                smote = SMOTE()\n",
    "                print('Balancing classes with SMOTE ...')\n",
    "                shape_train = S_train.shape[1:]\n",
    "                X_resampled, y_train = smote.fit_resample(S_train.reshape(-1, shape_train[0]*shape_train[1]*shape_train[2]), y_train)\n",
    "                # Reshape the data\n",
    "                S_train = X_resampled.reshape(-1, shape_train[0], shape_train[1], shape_train[2])\n",
    "\n",
    "\n",
    "            if classifier.upper()=='KNN':\n",
    "                from sklearn.neighbors import KNeighborsClassifier as KNN\n",
    "                # Train KNN\n",
    "                print('Training', classifier, '...')\n",
    "                knn = KNN(n_neighbors=3)\n",
    "                knn.fit(X_train, y_train)\n",
    "                \n",
    "                # Test\n",
    "                y_pred = knn.predict(X_test)\n",
    "            elif classifier.upper()=='SVM':\n",
    "                from sklearn.svm import SVC\n",
    "                # Train SVM\n",
    "                svm = SVC()\n",
    "                print('Training', classifier, '...')\n",
    "                svm.fit(X_train, y_train)\n",
    "                \n",
    "                # Test\n",
    "                y_pred = svm.predict(X_test)\n",
    "                \n",
    "            # TODO: uncomment NN classifier and test it    \n",
    "            elif classifier.upper()=='RESNET':\n",
    "                import torch\n",
    "                from am24utils import AudioResNet\n",
    "                # Train ResNet for audio classification\n",
    "                device = 'cuda:1' if torch.cuda.is_available() else 'cpu'\n",
    "                resnet = AudioResNet(num_classes=3, fine_tuning=True).to(device)\n",
    "                #am24utils.save_or_reset_weights(resnet, 'resnet_weights_starting_weights.pth')\n",
    "                print('Training', classifier, '...')\n",
    "                resnet.fit(S_train, y_train)\n",
    "                \n",
    "                # Save the weights\n",
    "                #am24utils.save_or_reset_weights(resnet, f'squeezenet_weights_f{fold_idx+1}/{N_FOLDS}.pth')\n",
    "                \n",
    "                # Test\n",
    "                _, y_pred = resnet.test(S_test, y_test)\n",
    "                \n",
    "                resnet.cpu()\n",
    "                resnet = None\n",
    "                \n",
    "                del resnet\n",
    "                # Free memory\n",
    "                torch.cuda.empty_cache()\n",
    "                # Use garbage collector\n",
    "                import gc\n",
    "                gc.collect()\n",
    "                \n",
    "            elif classifier.upper()=='SQUEEZENET':\n",
    "                import torch\n",
    "                from am24utils import AudioSqueezeNet\n",
    "                # Train ResNet for audio classification\n",
    "                device = 'cuda:1' if torch.cuda.is_available() else 'cpu'\n",
    "                squeezenet = AudioSqueezeNet(num_classes=3).to(device)\n",
    "                #am24utils.save_or_reset_weights(squeezenet, 'squeezenet_weights_starting_weights.pth')\n",
    "                print('Training', classifier, '...')\n",
    "                squeezenet.fit(S_train, y_train)\n",
    "                \n",
    "                # Save the weights\n",
    "                #am24utils.save_or_reset_weights(squeezenet, f'squeezenet_weights_f{fold_idx+1}/{N_FOLDS}.pth')\n",
    "                \n",
    "                # Test\n",
    "                _, y_pred = squeezenet.test(S_test, y_test)\n",
    "                \n",
    "                squeezenet.cpu()\n",
    "                squeezenet = None\n",
    "                \n",
    "                del squeezenet\n",
    "                # Free memory\n",
    "                torch.cuda.empty_cache()\n",
    "                # Use garbage collector\n",
    "                import gc\n",
    "                gc.collect()\n",
    "                \n",
    "            \n",
    "            # # Metrics\n",
    "            classification_report = sk.metrics.classification_report(y_test, y_pred, labels=[dynamics_str_to_int(x) for x in DYNAMICS],target_names=DYNAMICS)\n",
    "            print('-Classification FOLD %i/%i-'%(fold_idx+1,N_FOLDS))\n",
    "            print('\\t\\t\\t'+classification_report.replace('\\n','\\n\\t\\t\\t'))\n",
    "            print('\\n\\n')\n",
    "\n",
    "            confusion_matrix = sk.metrics.confusion_matrix(y_test, y_pred, labels=[dynamics_str_to_int(x) for x in DYNAMICS] )\n",
    "            print('-Confusion Matrix FOLD %i/%i-'%(fold_idx+1,N_FOLDS))\n",
    "            print('\\t\\t\\t'+str(confusion_matrix).replace('\\n','\\n\\t\\t\\t'))\n",
    "            print('\\n\\n')\n",
    "            print('\\n\\n')\n",
    "            \n",
    "        \n",
    "            run.foldresults.append({\n",
    "                'classification_report': classification_report,\n",
    "                'confusion_matrix': confusion_matrix,\n",
    "                'accuracy': sk.metrics.accuracy_score(y_test, y_pred),\n",
    "                'dictclassifiction_report' : sk.metrics.classification_report(y_test, y_pred, output_dict=True)\n",
    "            })\n",
    "        print('Run %s done.'%run.name)\n",
    "\n",
    "        print('Average classification report over %i folds:',len(run.foldresults))\n",
    "        mean_dict = am24utils.report_average([r['dictclassifiction_report'] for r in run.foldresults])\n",
    "        mean_accuracy = sum([r['accuracy'] for r in run.foldresults])/len(run.foldresults)\n",
    "        run.results = {'mean_classification_report_dict': mean_dict,'mean_classification_report_str': am24utils.classification_report_dict2print(mean_dict), 'mean_accuracy': mean_accuracy, 'number_folds': N_FOLDS}\n",
    "        print(run.results['mean_classification_report_str'])\n",
    "       \n",
    "\n",
    "\n",
    "# TEST_WINDOWSIZES_VALUES = [4800]\n",
    "# TEST_PERTURBATION_DISTRIBUTIONS = ['normal']\n",
    "# TEST_PERTURBATION_MAXSAMPLES = [0]\n",
    "\n",
    "# to_run = am24utils.get_run_list(winsizes = TEST_WINDOWSIZES_VALUES,\n",
    "#                  pert_distributions = TEST_PERTURBATION_DISTRIBUTIONS,\n",
    "#                  pert_maxsamples  = TEST_PERTURBATION_MAXSAMPLES)       \n",
    "       \n",
    "       \n",
    "# Create the runs        \n",
    "to_run_NN = am24utils.get_run_list()\n",
    "\n",
    "for ridx,run in enumerate(to_run_NN):\n",
    "    print(run.name)\n",
    "\n",
    "run_taskC(to_run_NN, packedData=packedData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Check memory usage\n",
    "# import os\n",
    "# import psutil\n",
    "# process = psutil.Process(os.getpid())\n",
    "# print(process.memory_info().rss / 1024**2)  # in bytes\n",
    "\n",
    "# # Check GPU memory usage\n",
    "# import torch\n",
    "# print(torch.cuda.memory_allocated())\n",
    "# print(torch.cuda.memory_reserved())\n",
    "\n",
    "# # Clear GPU memory\n",
    "# torch.cuda.empty_cache()\n",
    "\n",
    "# # Check GPU memory usage\n",
    "# print(torch.cuda.memory_allocated())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle, datetime\n",
    "\n",
    "\n",
    "#resdir_path = os.path.join('results','task-C','date_%s'%(datetime.datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")))\n",
    "#os.makedirs(resdir_path)\n",
    "\n",
    "# resdir_path = \"results/task-C/date_2024-04-30_15-08-43\"\n",
    "\n",
    "bakfilename  = 'taskC_ResNet_results.pickle'\n",
    "\n",
    "with open(os.path.join(resdir_path,bakfilename), 'wb') as f:\n",
    "    pickle.dump(to_run_NN, f)\n",
    "    \n",
    "    \n",
    "am24utils.plot_runs(to_run_NN, 'mean_f1')\n",
    "plt.savefig(os.path.join(resdir_path,'accuracy_ResNet.png'))\n",
    "plt.savefig(os.path.join(resdir_path,'accuracy_ResNet.pdf'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# get_window_size_from_run = lambda run: run.window_size_samples\n",
    "# get_onset_perturbation_distribution_from_run = lambda run: run.onset_perturbation_distribution\n",
    "# get_onset_perturbation_max_samples_from_run = lambda run: run.onset_perturbation_max_samples\n",
    "\n",
    "\n",
    "# all_sizes = sorted(list(set([get_window_size_from_run(run) for run in to_run])))\n",
    "# runs_grouped_by_size = [[run for run in to_run if get_window_size_from_run(run) == size] for size in all_sizes]\n",
    "# # print('Runs grouped by size:',[[ga.name for ga in g] for g in runs_grouped_by_size])\n",
    "\n",
    "# runs_grouped_size_x_maxsamp = []\n",
    "# for rungroup in runs_grouped_by_size:\n",
    "#     all_maxsamp = sorted(list(set([get_onset_perturbation_max_samples_from_run(run) for run in rungroup])))\n",
    "#     runs_grouped_size_x_maxsamp.append([[run for run in rungroup if get_onset_perturbation_max_samples_from_run(run) == maxsamp] for maxsamp in all_maxsamp])\n",
    "\n",
    "# # print('Runs grouped by size and maxsamp:',[[[(ga.name,ga.window_size_samples,ga.onset_perturbation_max_samples) for ga in g] for g in gg] for gg in runs_grouped_size_x_maxsamp])\n",
    "\n",
    "# # window_size_samples\n",
    "# # onset_perturbation_max_samples\n",
    "\n",
    "\n",
    "# fig, ax = plt.subplots(figsize=(20,6))\n",
    "# barWidth = 0.25\n",
    "# group1_spacing = 0.1\n",
    "# group2_spacing = 0.05\n",
    "# # runs_grouped_size_x_maxsamp\n",
    "\n",
    "# xticks_positions = []\n",
    "# xticks_text = []\n",
    "# max_group2_width = max([max([len(e)*barWidth+(group2_spacing*(len(e)-1)) for e in ee]) for ee in runs_grouped_size_x_maxsamp])\n",
    "\n",
    "# max_window_group_width = max([len(e)*barWidth+(group1_spacing*(len(e)-1)) for e in runs_grouped_size_x_maxsamp])\n",
    "# all_rects = []\n",
    "# for idx,run_wsize_group in enumerate(runs_grouped_size_x_maxsamp):\n",
    "#     for idx2,run_maxsamp_group in enumerate(run_wsize_group):\n",
    "#         accuracies_toplot = [run.results['mean_accuracy'] for run in run_maxsamp_group]\n",
    "#         runnames_labels = [run.name for run in run_maxsamp_group]\n",
    "#         r1 = np.arange(len(accuracies_toplot))\n",
    "#         x_bars_pos = r1*barWidth + idx2*max_group2_width+ idx*max_window_group_width\n",
    "#         xticklabs = [run.onset_perturbation_distribution if not run.onset_perturbation_distribution == None else '' for run in run_maxsamp_group]\n",
    "\n",
    "#         color_from_probdist = lambda probdist: {'':'blue','normal':'orange','normal':'red'}[probdist]\n",
    "#         label_from_probdist = lambda probdist: {'':'Dataset labels','normal':'Normal Pert.','normal':'Normal'}[probdist]\n",
    "\n",
    "#         cur_rects = ax.bar(x_bars_pos, \n",
    "#                accuracies_toplot, \n",
    "#                width = barWidth,\n",
    "#                label = [label_from_probdist(e) for e in xticklabs],\n",
    "#                color = [color_from_probdist(e) for e in xticklabs])\n",
    "#         all_rects.append(cur_rects)\n",
    "#         # ax.bar(r1 + idx*barWidth, accuracies_toplot, width = barWidth, label = 'w%i_p%s'%(run_maxsamp_group[0].window_size_samples, run_maxsamp_group[0].onset_perturbation_distribution))\n",
    "#         # xticks_positions.extend(x_bars_pos)\n",
    "#         # xticks_text.extend(xticklabs)\n",
    "#         # print(xticks_text,end='\\n\\n')\n",
    "\n",
    "# ax.set_xticks([r + barWidth for r in range(len(r1))])\n",
    "# # ax.set_xticks(xticks_positions)\n",
    "# # ax.set_xticklabels(xticks_text)\n",
    "\n",
    "# plt.xticks(rotation=45)\n",
    "# plt.ylabel('Accuracy')\n",
    "\n",
    "# ###Curstom legend\n",
    "\n",
    "# handles = []\n",
    "# labels = []\n",
    "# for rects in all_rects:\n",
    "#     handle, label = rects[0], rects[0].get_label()\n",
    "#     handles.append(handle)\n",
    "#     labels.append(label)\n",
    "\n",
    "# # Creating the legend with combined entries for each group of bars\n",
    "# ax.legend(list(set(handles)), list(set(labels)))\n",
    "\n",
    "\n",
    "\n",
    "# plt.savefig(os.path.join(resdir_path,'accuracy.png'))\n",
    "# plt.savefig(os.path.join(resdir_path,'accuracy.pdf'))\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}