From 1630e7e8b7bf22bfdcd7c263e0c84d81e74d428a Mon Sep 17 00:00:00 2001 From: Michael Fuest Date: Thu, 3 Oct 2024 18:40:23 -0400 Subject: [PATCH] cleaned up logging --- config/model_config.yaml | 5 +- datasets/openpower.py | 1 - datasets/pecanstreet.py | 1 - datasets/timeseries_dataset.py | 42 ++++ eval/evaluator.py | 213 ++++++------------ ...{generative_model.py => data_generator.py} | 21 +- generator/diffusion_ts/gaussian_diffusion.py | 43 ++-- generator/gan/acgan.py | 1 - generator/options.py | 2 +- main.py | 10 +- 10 files changed, 154 insertions(+), 185 deletions(-) create mode 100644 datasets/timeseries_dataset.py rename generator/{generative_model.py => data_generator.py} (86%) diff --git a/config/model_config.yaml b/config/model_config.yaml index 5521ddb..647fa5f 100644 --- a/config/model_config.yaml +++ b/config/model_config.yaml @@ -1,3 +1,4 @@ +device: 1 # 0, cpu seq_len: 96 # should not be changed for the current datasets input_dim: 2 # or 1 depending on user, but is dynamically set noise_dim: 256 @@ -66,7 +67,7 @@ diffusion_ts: acgan: batch_size: 32 - n_epochs: 200 + n_epochs: 10 lr_gen: 3e-4 lr_discr: 1e-4 - warm_up_epochs: 50 + warm_up_epochs: 5 diff --git a/datasets/openpower.py b/datasets/openpower.py index 0b6e4ed..186f7d3 100644 --- a/datasets/openpower.py +++ b/datasets/openpower.py @@ -12,7 +12,6 @@ warnings.filterwarnings("ignore", category=pd.errors.SettingWithCopyWarning) ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") class OpenPowerDataset(Dataset): diff --git a/datasets/pecanstreet.py b/datasets/pecanstreet.py index 7163a95..542601c 100644 --- a/datasets/pecanstreet.py +++ b/datasets/pecanstreet.py @@ -15,7 +15,6 @@ warnings.filterwarnings("ignore", category=pd.errors.SettingWithCopyWarning) ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") class PecanStreetDataManager: diff --git a/datasets/timeseries_dataset.py b/datasets/timeseries_dataset.py new file mode 100644 index 0000000..edd9904 --- /dev/null +++ b/datasets/timeseries_dataset.py @@ -0,0 +1,42 @@ +import torch +from torch.utils.data import Dataset + + +class TimeSeriesDataset(Dataset): + def __init__(self, dataframe, time_series_column_name, conditioning_vars=None): + """ + Initializes the TimeSeriesDataset. + + Args: + dataframe (pd.DataFrame): The input DataFrame containing the time series data and optional conditioning variables. + time_series_column_name (str): The name of the column containing the time series data. + conditioning_vars (list of str, optional): List of column names to be used as conditioning variables. + """ + self.data = dataframe.reset_index(drop=True) + self.conditioning_vars = conditioning_vars or [] + self.time_series_column_name = time_series_column_name + + if self.time_series_column_name not in self.data.columns: + raise ValueError( + f"Time series column '{self.time_series_column_name}' not found in DataFrame." + ) + + for var in self.conditioning_vars: + if var not in self.data.columns: + raise ValueError( + f"Conditioning variable '{var}' not found in DataFrame." + ) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + time_series = self.data.iloc[idx][self.time_series_column] + time_series = torch.tensor(time_series, dtype=torch.float32) + + conditioning_vars_dict = {} + for var in self.conditioning_vars: + value = self.data.iloc[idx][var] + conditioning_vars_dict[var] = torch.tensor(value, dtype=torch.long) + + return time_series, conditioning_vars_dict diff --git a/eval/evaluator.py b/eval/evaluator.py index 308e89f..d4f0cd4 100644 --- a/eval/evaluator.py +++ b/eval/evaluator.py @@ -1,3 +1,5 @@ +import datetime +import os from typing import Any from typing import Dict from typing import List @@ -13,9 +15,6 @@ from eval.metrics import calculate_mmd from eval.metrics import calculate_period_bound_mse from eval.metrics import dynamic_time_warping_dist -from eval.metrics import plot_range_with_syn_values -from eval.metrics import plot_syn_with_closest_real_ts -from eval.metrics import visualization from eval.predictive_metric import predictive_score_metrics from generator.diffcharge.diffusion import DDPM from generator.diffusion_ts.gaussian_diffusion import Diffusion_TS @@ -37,12 +36,11 @@ def __init__(self, real_dataset: Any, model_name: str, log_dir: str = "runs"): Args: real_dataset: The real dataset used for evaluation. model_name (str): The name of the generative model being evaluated. - log_dir (str): Directory for storing TensorBoard logs. + log_dir (str): Base directory for storing TensorBoard logs. """ self.real_dataset = real_dataset self.model_name = model_name - self.log_dir = log_dir - self.writer = SummaryWriter(f"{log_dir}/{model_name}") + self.base_log_dir = os.path.join(log_dir, model_name) self.metrics: Dict[str, List] = { "dtw": [], "mmd": [], @@ -52,7 +50,12 @@ def __init__(self, real_dataset: Any, model_name: str, log_dir: str = "runs"): "predictive": [], } - def evaluate_model(self, user_id: int = None, distinguish_rare: bool = False): + def evaluate_model( + self, + user_id: int = None, + distinguish_rare: bool = False, + data_label: str = None, + ): """ Evaluate the model for a specific user or the entire dataset. @@ -60,25 +63,14 @@ def evaluate_model(self, user_id: int = None, distinguish_rare: bool = False): user_id (int, optional): The ID of the user to evaluate. If None, evaluate on the entire dataset. distinguish_rare (bool): Whether to distinguish between rare and non-rare data samples. """ - # Prepare dataset and logging if user_id is not None: dataset = self.real_dataset.create_user_dataset(user_id) - log_dir = f"{self.log_dir}/{self.model_name}/user_{user_id}" + data_label = f"user_{user_id}" else: dataset = self.real_dataset - if dataset.include_generation: - dataset_type = "pv_users" - else: - dataset_type = "non_pv_users" - log_dir = f"{self.log_dir}/{self.model_name}/{dataset_type}" model = self.get_trained_model(dataset) - if model.sparse_conditioning_loss_weight != 0.5: - log_dir += "/rare_upweight" - - writer = SummaryWriter(log_dir) - print("----------------------") if user_id is not None: print(f"Starting evaluation for user {user_id}") @@ -86,10 +78,11 @@ def evaluate_model(self, user_id: int = None, distinguish_rare: bool = False): print("Starting evaluation for all users") print("----------------------") - self.run_evaluation(dataset, model, writer, distinguish_rare) + # Pass data_label to run_evaluation + self.run_evaluation(dataset, model, distinguish_rare, data_label) def run_evaluation( - self, dataset: Any, model: Any, writer: SummaryWriter, distinguish_rare: bool + self, dataset: Any, model: Any, distinguish_rare: bool, data_label: str ): """ Run the evaluation process. @@ -97,23 +90,28 @@ def run_evaluation( Args: dataset: The dataset to evaluate. model: The trained model. - writer (SummaryWriter): TensorBoard writer for logging results. distinguish_rare (bool): Whether to distinguish between rare and non-rare data samples. + data_label (str): Label for the dataset (used in log directory). """ if distinguish_rare: rare_indices, non_rare_indices = self.identify_rare_combinations( dataset, model ) - self.evaluate_subset(dataset, model, writer, rare_indices, data_type="rare") self.evaluate_subset( - dataset, model, writer, non_rare_indices, data_type="non_rare" + dataset, model, rare_indices, data_type="rare", data_label=data_label + ) + self.evaluate_subset( + dataset, + model, + non_rare_indices, + data_type="non_rare", + data_label=data_label, ) else: all_indices = dataset.data.index.to_numpy() - self.evaluate_subset(dataset, model, writer, all_indices, data_type="all") - - writer.flush() - writer.close() + self.evaluate_subset( + dataset, model, all_indices, data_type="all", data_label=data_label + ) def identify_rare_combinations( self, dataset: Any, model: Any @@ -141,7 +139,6 @@ def identify_rare_combinations( model.conditioning_module.compute_mahalanobis_distance(embeddings) ) - # Define rarity threshold (e.g., top 20% as rare) rarity_threshold = torch.quantile(mahalanobis_distances, 0.8) rare_mask = mahalanobis_distances > rarity_threshold @@ -154,9 +151,9 @@ def evaluate_subset( self, dataset: Any, model: Any, - writer: SummaryWriter, indices: np.ndarray, data_type: str, + data_label: str, ): """ Evaluate the model on a subset of the data. @@ -164,10 +161,15 @@ def evaluate_subset( Args: dataset: The dataset containing real data. model: The trained model to generate data. - writer (SummaryWriter): TensorBoard writer for logging results. indices (np.ndarray): Indices of data to use. data_type (str): Label for the data subset ("rare", "non_rare", or "all"). + data_label (str): Label for the dataset (used in log directory). """ + # Generate the timestamp here to create a unique subdirectory under data_type + timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + log_dir = os.path.join(self.base_log_dir, data_label, data_type, timestamp) + writer = SummaryWriter(log_dir=log_dir) + real_data_subset = dataset.data.iloc[indices].reset_index(drop=True) conditioning_vars = { name: torch.tensor( @@ -193,22 +195,18 @@ def evaluate_subset( syn_data_array = np.stack(syn_data_inv["timeseries"]) # Compute metrics - self.compute_metrics( - real_data_array, syn_data_array, real_data_subset, writer, data_type - ) + self.compute_metrics(real_data_array, syn_data_array, real_data_subset, writer) - # Create visualizations - self.create_visualizations( - real_data_inv, syn_data_inv, dataset, model, writer, data_type - ) + # Close the writer + writer.flush() + writer.close() def compute_metrics( self, real_data: np.ndarray, syn_data: np.ndarray, - dataset: Any, + real_data_subset: pd.DataFrame, writer: SummaryWriter, - data_type: str, ): """ Compute evaluation metrics and log them. @@ -216,120 +214,41 @@ def compute_metrics( Args: real_data (np.ndarray): Real data array. syn_data (np.ndarray): Synthetic data array. - dataset (Any): The original dataset. + real_data_subset (pd.DataFrame): Real data subset DataFrame. writer (SummaryWriter): TensorBoard writer for logging results. - data_type (str): Label for the data subset ("rare", "non_rare", or "all"). """ # DTW dtw_mean, dtw_std = dynamic_time_warping_dist(real_data, syn_data) - writer.add_scalar(f"DTW/{data_type}/mean", dtw_mean) - writer.add_scalar(f"DTW/{data_type}/std", dtw_std) - self.metrics["dtw"].append((data_type, dtw_mean, dtw_std)) + writer.add_scalar("DTW/mean", dtw_mean) + writer.add_scalar("DTW/std", dtw_std) + self.metrics["dtw"].append((dtw_mean, dtw_std)) # MMD mmd_mean, mmd_std = calculate_mmd(real_data, syn_data) - writer.add_scalar(f"MMD/{data_type}/mean", mmd_mean) - writer.add_scalar(f"MMD/{data_type}/std", mmd_std) - self.metrics["mmd"].append((data_type, mmd_mean, mmd_std)) + writer.add_scalar("MMD/mean", mmd_mean) + writer.add_scalar("MMD/std", mmd_std) + self.metrics["mmd"].append((mmd_mean, mmd_std)) # MSE - mse_mean, mse_std = calculate_period_bound_mse(dataset, syn_data) - writer.add_scalar(f"MSE/{data_type}/mean", mse_mean) - writer.add_scalar(f"MSE/{data_type}/std", mse_std) - self.metrics["mse"].append((data_type, mse_mean, mse_std)) + mse_mean, mse_std = calculate_period_bound_mse(real_data_subset, syn_data) + writer.add_scalar("MSE/mean", mse_mean) + writer.add_scalar("MSE/std", mse_std) + self.metrics["mse"].append((mse_mean, mse_std)) # FID fid_score = Context_FID(real_data, syn_data) - writer.add_scalar(f"FID/{data_type}/score", fid_score) - self.metrics["fid"].append((data_type, fid_score)) + writer.add_scalar("FID/score", fid_score) + self.metrics["fid"].append(fid_score) # Discriminative Score discr_score, _, _ = discriminative_score_metrics(real_data, syn_data) - writer.add_scalar(f"Discriminative/{data_type}/score", discr_score) - self.metrics["discriminative"].append((data_type, discr_score)) + writer.add_scalar("Discriminative/score", discr_score) + self.metrics["discriminative"].append(discr_score) # Predictive Score pred_score = predictive_score_metrics(real_data, syn_data) - writer.add_scalar(f"Predictive/{data_type}/score", pred_score) - self.metrics["predictive"].append((data_type, pred_score)) - - def create_visualizations( - self, - real_data_df: pd.DataFrame, - syn_data_df: pd.DataFrame, - dataset: Any, - model: Any, - writer: SummaryWriter, - data_type: str, - num_samples: int = 100, - num_runs: int = 3, - ): - """ - Create various visualizations for the evaluation results. - - Args: - real_data_df (pd.DataFrame): Inverse-transformed real data. - syn_data_df (pd.DataFrame): Inverse-transformed synthetic data. - dataset (Any): The dataset object. - model (Any): The trained model. - writer (SummaryWriter): TensorBoard writer for logging visualizations. - data_type (str): Label for the data subset ("rare", "non_rare", or "all"). - num_samples (int): Number of samples to generate for visualization. - num_runs (int): Number of visualization runs. - """ - for i in range(num_runs): - # Sample a conditioning variable combination from real data - sample_row = real_data_df.sample(n=1).iloc[0] - conditioning_vars_sample = { - var_name: torch.tensor( - [sample_row[var_name]] * num_samples, - dtype=torch.long, - device=device, - ) - for var_name in model.categorical_dims.keys() - } - - generated_samples = model.generate(conditioning_vars_sample).cpu().numpy() - if generated_samples.ndim == 2: - generated_samples = generated_samples.reshape( - generated_samples.shape[0], -1, generated_samples.shape[1] - ) - - generated_samples_df = pd.DataFrame( - { - var_name: [sample_row[var_name]] * num_samples - for var_name in model.categorical_dims.keys() - } - ) - generated_samples_df["timeseries"] = list(generated_samples) - generated_samples_df["dataid"] = sample_row[ - "dataid" - ] # required for inverse transform - generated_samples_df = dataset.inverse_transform(generated_samples_df) - - # Extract month and weekday for plotting - month = sample_row.get("month", None) - weekday = sample_row.get("weekday", None) - - # Visualization 1: Plot range with synthetic values - range_plot = plot_range_with_syn_values( - real_data_df, generated_samples_df, month, weekday - ) - writer.add_figure(f"Visualizations/{data_type}/Range_Plot_{i}", range_plot) - - # Visualization 2: Plot closest real signals with synthetic values - closest_plot = plot_syn_with_closest_real_ts( - real_data_df, generated_samples_df, month, weekday - ) - writer.add_figure( - f"Visualizations/{data_type}/Closest_Real_TS_{i}", closest_plot - ) - - # Visualization 3: KDE plots for real and synthetic data - real_data_array = np.stack(real_data_df["timeseries"]) - syn_data_array = np.stack(syn_data_df["timeseries"]) - kde_plot = visualization(real_data_array, syn_data_array, "kernel") - writer.add_figure(f"Visualizations/{data_type}/KDE", kde_plot) + writer.add_scalar("Predictive/score", pred_score) + self.metrics["predictive"].append(pred_score) def get_trained_model(self, dataset: Any) -> Any: """ @@ -373,18 +292,14 @@ def get_summary_metrics(self) -> Dict[str, float]: """ metrics_summary = {} for metric_name, values in self.metrics.items(): - metric_values = [value[1] for value in values] # value[0] is data_type - metrics_summary[metric_name] = np.mean(metric_values) + if values: + if isinstance(values[0], tuple): + # For metrics with mean and std + metric_values = [value[0] for value in values] # value[0] is mean + else: + metric_values = values + metrics_summary[metric_name] = np.mean(metric_values) + else: + metrics_summary[metric_name] = None return metrics_summary - - def log_final_results(self): - """ - Log the final evaluation results. - """ - metrics_summary = self.get_summary_metrics() - for metric_name, mean_value in metrics_summary.items(): - self.writer.add_scalar(f"Final_Results/{metric_name}", mean_value) - print(f"{metric_name}: {mean_value:.4f}") - - self.writer.close() diff --git a/generator/generative_model.py b/generator/data_generator.py similarity index 86% rename from generator/generative_model.py rename to generator/data_generator.py index c836eec..e74fca5 100644 --- a/generator/generative_model.py +++ b/generator/data_generator.py @@ -1,13 +1,14 @@ import pandas as pd import torch +from datasets.timeseries_dataset import TimeSeriesDataset from generator.diffcharge import DDPM from generator.diffusion_ts import Diffusion_TS from generator.gan.acgan import ACGAN from generator.options import Options -class GenerativeModel: +class DataGenerator: """ A wrapper class for generative models. """ @@ -23,14 +24,12 @@ def __init__(self, model_name: str, model_params: dict = None): self.model_name = model_name self.model_params = model_params if model_params is not None else {} self.model = None - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.opt = Options(model_name) # Update opt with parameters from model_params for key, value in self.model_params.items(): setattr(self.opt, key, value) - # Initialize the model class self._initialize_model() def _initialize_model(self): @@ -54,13 +53,11 @@ def fit(self, X): Args: X: Input data. Should be a compatible dataset object or pandas DataFrame. - y: Not used. Included for compatibility with sklearn API. """ - # If X is a pandas DataFrame, convert it to the expected dataset format if isinstance(X, pd.DataFrame): dataset = self._prepare_dataset(X) else: - dataset = X # Assuming X is already a dataset in the required format + dataset = X self.model.train_model(dataset) def generate(self, conditioning_vars): @@ -118,4 +115,14 @@ def _prepare_dataset(self, df: pd.DataFrame): Returns: dataset: The dataset in the required format. """ - pass + if isinstance(df, torch.utils.data.Dataset): + return df + elif isinstance(df, pd.DataFrame): + dataset = TimeSeriesDataset( + dataframe=df, + conditioning_vars=self.conditioning_vars, + time_series_column="timeseries", + ) + return dataset + else: + raise TypeError("Input X must be a Dataset or a DataFrame.") diff --git a/generator/diffusion_ts/gaussian_diffusion.py b/generator/diffusion_ts/gaussian_diffusion.py index 35d616d..f9aedbd 100644 --- a/generator/diffusion_ts/gaussian_diffusion.py +++ b/generator/diffusion_ts/gaussian_diffusion.py @@ -32,10 +32,8 @@ from generator.diffusion_ts.model_utils import identity from generator.diffusion_ts.transformer import Transformer -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -def linear_beta_schedule(timesteps): +def linear_beta_schedule(timesteps, device): scale = 1000 / timesteps beta_start = scale * 0.0001 beta_end = scale * 0.02 @@ -44,7 +42,7 @@ def linear_beta_schedule(timesteps): ) -def cosine_beta_schedule(timesteps, s=0.004): +def cosine_beta_schedule(timesteps, device, s=0.004): steps = timesteps + 1 x = torch.linspace(0, timesteps, steps, dtype=torch.float32).to(device) alphas_cumprod = torch.cos(((x / timesteps) + s) / (1 + s) * math.pi * 0.5) ** 2 @@ -90,15 +88,17 @@ def __init__(self, opt): ) if opt.beta_schedule == "linear": - betas = linear_beta_schedule(opt.n_steps) + betas = linear_beta_schedule(opt.n_steps, self.device) elif opt.beta_schedule == "cosine": - betas = cosine_beta_schedule(opt.n_steps) + betas = cosine_beta_schedule(opt.n_steps, self.device) else: raise ValueError(f"unknown beta schedule {opt.beta_schedule}") alphas = 1.0 - betas - alphas_cumprod = torch.cumprod(alphas, dim=0).to(device) - alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.0).to(device) + alphas_cumprod = torch.cumprod(alphas, dim=0).to(self.device) + alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.0).to( + self.device + ) (timesteps,) = betas.shape self.num_timesteps = int(timesteps) @@ -139,7 +139,7 @@ def __init__(self, opt): posterior_variance = ( betas * (1.0 - alphas_cumprod_prev) / (1.0 - alphas_cumprod) - ).to(device) + ).to(self.device) # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t) @@ -352,7 +352,7 @@ def _train_loss( train_loss = self.loss_fn(model_out, target, reduction="none") - fourier_loss = torch.tensor([0.0]).to(device) + fourier_loss = torch.tensor([0.0]).to(self.device) if self.use_ff: fft1 = torch.fft.fft(model_out.transpose(1, 2), norm="forward") fft2 = torch.fft.fft(target.transpose(1, 2), norm="forward") @@ -380,7 +380,7 @@ def forward(self, x, conditioning_vars=None, **kwargs): ) def train_model(self, train_dataset): - self.to(device) + self.to(self.device) train_loader = DataLoader( train_dataset, @@ -397,25 +397,27 @@ def train_model(self, train_dataset): betas=[0.9, 0.96], ) self.ema = EMA( - self, beta=self.opt.ema_decay, update_every=self.opt.ema_update_interval - ).to(device) + self, + beta=self.opt.ema_decay, + update_every=self.opt.ema_update_interval, + device=self.device, + ).to(self.device) self.scheduler = ReduceLROnPlateau( self.optimizer, **self.opt.lr_scheduler_params ) self.conditioning_module.to(self.device) - # Training loop for epoch in tqdm(range(self.opt.n_epochs), desc="Training"): total_loss = 0.0 for i, (time_series_batch, conditioning_vars_batch) in enumerate( train_loader ): - time_series_batch = time_series_batch.to(device) + time_series_batch = time_series_batch.to(self.device) for key in conditioning_vars_batch: conditioning_vars_batch[key] = conditioning_vars_batch[key].to( - device + self.device ) current_batch_size = time_series_batch.size(0) @@ -429,7 +431,7 @@ def train_model(self, train_dataset): embeddings = self.conditioning_module(conditioning_vars_batch) rare_mask = ( self.conditioning_module.is_rare(embeddings) - .to(device) + .to(self.device) .float() ) @@ -443,8 +445,8 @@ def train_model(self, train_dataset): rare_indices = (rare_mask == 1.0).nonzero(as_tuple=True)[0] non_rare_indices = (rare_mask == 0.0).nonzero(as_tuple=True)[0] - loss_rare = torch.tensor(0.0).to(device) - loss_non_rare = torch.tensor(0.0).to(device) + loss_rare = torch.tensor(0.0).to(self.device) + loss_non_rare = torch.tensor(0.0).to(self.device) if len(rare_indices) > 0: time_series_batch_rare = time_series_batch[rare_indices] @@ -502,12 +504,13 @@ def train_model(self, train_dataset): class EMA(nn.Module): - def __init__(self, model, beta, update_every): + def __init__(self, model, beta, update_every, device): super(EMA, self).__init__() self.ema_model = copy.deepcopy(model).eval().to(device) self.beta = beta self.update_every = update_every self.step = 0 + self.device = device for param in self.ema_model.parameters(): param.requires_grad = False diff --git a/generator/gan/acgan.py b/generator/gan/acgan.py index c0a3af8..c6d898a 100644 --- a/generator/gan/acgan.py +++ b/generator/gan/acgan.py @@ -16,7 +16,6 @@ import torch import torch.nn as nn import torch.optim as optim -from tensorboardX import SummaryWriter from tqdm import tqdm from datasets.utils import prepare_dataloader diff --git a/generator/options.py b/generator/options.py index 1942583..eb08731 100644 --- a/generator/options.py +++ b/generator/options.py @@ -28,7 +28,7 @@ def __init__(self, model_name: str): config = load_model_config() self.seed = 42 self.model_name = model_name - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.device = config.device self.seq_len = config.seq_len self.input_dim = config.input_dim diff --git a/main.py b/main.py index f940281..620f9b2 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,5 @@ +from torch.utils.tensorboard import SummaryWriter + from datasets.pecanstreet import PecanStreetDataManager from eval.evaluator import Evaluator @@ -35,8 +37,10 @@ def evaluate_single_dataset_model( non_pv_user_evaluator = Evaluator(non_pv_user_dataset, model_name) # evaluator.evaluate_all_users() # evaluator.evaluate_all_non_pv_users() - pv_user_evaluator.evaluate_model(None, distinguish_rare=True) - non_pv_user_evaluator.evaluate_model(None, distinguish_rare=True) + non_pv_user_evaluator.evaluate_model( + None, distinguish_rare=True, data_label="non_pv_users" + ) + pv_user_evaluator.evaluate_model(None, distinguish_rare=True, data_label="pv_users") def main(): @@ -45,7 +49,7 @@ def main(): # evaluate_individual_user_models("acgan", include_generation=False, normalization_method="date") evaluate_single_dataset_model( "diffusion_ts", - geography="austin", + geography="california", include_generation=False, normalization_method="group", )