From a3df205567ca6a7b3692c4adf81dd2a71528acc1 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Mon, 25 Mar 2024 17:58:37 +0000 Subject: [PATCH 01/76] enable W&B --- recipes/configs/llama2/7B_lora_single_device.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/recipes/configs/llama2/7B_lora_single_device.yaml b/recipes/configs/llama2/7B_lora_single_device.yaml index 6c950d4e94..757f353945 100644 --- a/recipes/configs/llama2/7B_lora_single_device.yaml +++ b/recipes/configs/llama2/7B_lora_single_device.yaml @@ -21,6 +21,8 @@ # This config works only for training on single device. +# export PATH="/home/ubuntu/miniforge3/envs/pt/bin:$PATH" + # Model Arguments model: _component_: torchtune.models.llama2.lora_llama2_7b @@ -74,9 +76,11 @@ gradient_accumulation_steps: 1 # Logging output_dir: /tmp/lora_finetune_output metric_logger: - _component_: torchtune.utils.metric_logging.DiskLogger - log_dir: ${output_dir} -log_every_n_steps: null + _component_: torchtune.utils.metric_logging.WandBLogger + project: torchtune + kwarg1: 1 + kwarg2: 2 +log_every_n_steps: 1 # Environment device: cuda From 3f4123418ff844405ce0892f1fc5bfd62d578ec3 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Mon, 25 Mar 2024 18:03:24 +0000 Subject: [PATCH 02/76] add default project --- torchtune/utils/metric_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 4366a54a7a..eb8856ad2c 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -147,7 +147,7 @@ class WandBLogger(MetricLoggerInterface): def __init__( self, - project: str, + project: str = "torchtune", entity: Optional[str] = None, group: Optional[str] = None, **kwargs, From 914a901ef1f19c2d999d335c816847f9b3b7e47a Mon Sep 17 00:00:00 2001 From: Kartikay Khandelwal <47255723+kartikayk@users.noreply.github.com> Date: Wed, 27 Mar 2024 09:34:33 -0700 Subject: [PATCH 03/76] Delete old checkpoint code (#601) --- docs/source/api_ref_utilities.rst | 10 - .../test_lora_finetune_single_device.py | 2 +- tests/torchtune/utils/test_checkpoint.py | 138 ------------ torchtune/utils/__init__.py | 6 +- torchtune/utils/_checkpointing/__init__.py | 2 +- .../_checkpointing/_checkpointer_utils.py | 30 +++ torchtune/utils/checkpoint.py | 212 ------------------ 7 files changed, 33 insertions(+), 367 deletions(-) delete mode 100644 tests/torchtune/utils/test_checkpoint.py delete mode 100644 torchtune/utils/checkpoint.py diff --git a/docs/source/api_ref_utilities.rst b/docs/source/api_ref_utilities.rst index 1a761f3683..6cc9065b46 100644 --- a/docs/source/api_ref_utilities.rst +++ b/docs/source/api_ref_utilities.rst @@ -65,16 +65,6 @@ Data checkpointable_dataloader.CheckpointableDataLoader collate.padded_collate -Checkpoint saving & loading ---------------------------- - -.. autosummary:: - :toctree: generated/ - :nosignatures: - - checkpoint.save_checkpoint - checkpoint.load_checkpoint - .. _gen_label: Generation diff --git a/tests/recipes/test_lora_finetune_single_device.py b/tests/recipes/test_lora_finetune_single_device.py index 6a4a3a2d01..6cdbe2f4cd 100644 --- a/tests/recipes/test_lora_finetune_single_device.py +++ b/tests/recipes/test_lora_finetune_single_device.py @@ -123,7 +123,7 @@ def test_loss_qlora(self, tmpdir, monkeypatch): loss_values = get_loss_values_from_metric_logger(log_file) expected_loss_values = self._fetch_expected_loss_values(run_qlora=True) torch.testing.assert_close( - loss_values, expected_loss_values, rtol=1e-5, atol=1e-5 + loss_values, expected_loss_values, rtol=1e-4, atol=1e-4 ) @pytest.mark.integration_test diff --git a/tests/torchtune/utils/test_checkpoint.py b/tests/torchtune/utils/test_checkpoint.py deleted file mode 100644 index 0c8f5091bf..0000000000 --- a/tests/torchtune/utils/test_checkpoint.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import tempfile - -import pytest -import torch -import torch.distributed as dist - -from tests.test_utils import skip_if_cuda_not_available -from torch.distributed import launcher -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP -from torchtune.utils.checkpoint import load_checkpoint, save_checkpoint - - -class TestCheckpoint: - def _save_and_load(self, checkpoint, model, optimizer): - with tempfile.NamedTemporaryFile() as f: - save_checkpoint(ckpt_dict=checkpoint, output_loc=f.name) - if torch.distributed.is_initialized(): - # All ranks wait for 0 to finish saving. - dist.barrier() - # Broadcast rank 0's saved filename so other ranks know - # where to load from. - file_list = [f.name if dist.get_rank() == 0 else None] - dist.broadcast_object_list(file_list, src=0) - file_name = file_list[0] - else: - file_name = f.name - checkpoint = load_checkpoint(file_name, model, optimizer) - model.load_state_dict(checkpoint["model"]) - optimizer.load_state_dict(checkpoint["optimizer"]) - # Have rank 0 wait for all ranks to finish loading before exiting - # context manager which would trigger file destruction. - if torch.distributed.is_initialized(): - dist.barrier() - - def _get_model_and_optim(self, zero_model, fsdp): - model = torch.nn.Linear(10, 10) - if zero_model: - with torch.no_grad(): - for p in model.parameters(): - p.zero_() - if fsdp: - model = FSDP(model, device_id=torch.cuda.current_device()) - optim = torch.optim.SGD(model.parameters(), lr=0.01) - return model, optim - - def _validate_dicts(self, d1, d2): - assert len(d1) == len(d2) - for k, v in d1.items(): - assert k in d2 - if isinstance(v, dict): - self._validate_dicts(v, d2[k]) - else: - if isinstance(v, torch.Tensor): - torch.testing.assert_close(v, d2[k]) - else: - assert v == d2[k] - - def test_local_checkpoint_save_load(self) -> None: - model, optim = self._get_model_and_optim(zero_model=False, fsdp=False) - # Create dummy optim states to verify they can be loaded. - for p in model.parameters(): - p.grad = torch.rand_like(p) - optim.step() - checkpoint = {"model": model, "optimizer": optim, "lr": 0.01} - model_new, optim_new = self._get_model_and_optim(zero_model=True, fsdp=False) - # Saves checkpoint, calls load_checkpoint and loads model + optim states into new model/optim. - self._save_and_load(checkpoint, model_new, optim_new) - # model_new and model params should match - for p1, p2 in zip(model.parameters(), model_new.parameters()): - torch.testing.assert_close(p1, p2) - # optim state_dicts should match - self._validate_dicts(optim.state_dict(), optim_new.state_dict()) - - def test_no_model_key_save(self) -> None: - checkpoint = {"lr": 0.03} - with pytest.raises( - RuntimeError, - match="Expected `ckpt_dict` to contain a `model` key, but it does not.", - ): - with tempfile.NamedTemporaryFile() as f: - save_checkpoint(checkpoint, f.name) - - def test_no_model_key_load(self) -> None: - model = torch.nn.Linear(1, 1) - with tempfile.NamedTemporaryFile() as f: - torch.save({"lr": 0.01}, f.name) - with pytest.raises( - RuntimeError, - match="Expected loaded checkpoint to contain a `model` key.*", - ): - load_checkpoint(f.name, model) - - def test_no_optim_key_load(self) -> None: - model = torch.nn.Linear(1, 1) - optim = torch.optim.SGD(model.parameters(), lr=0.01) - with tempfile.NamedTemporaryFile() as f: - save_checkpoint({"model": model}, f.name) - with pytest.raises( - RuntimeError, - match="Expected loaded checkpoint to contain an `optimizer` key.*", - ): - load_checkpoint(f.name, model, optim) - - def _test_distributed_save_load(self) -> None: - torch.distributed.init_process_group(backend="nccl") - torch.cuda.set_device(torch.distributed.get_rank()) - torch.distributed.barrier() - model, optim = self._get_model_and_optim(zero_model=False, fsdp=True) - for p in model.parameters(): - p.grad = torch.rand_like(p) - optim.step() - checkpoint = {"model": model, "optimizer": optim, "lr": 0.01} - model_new, optim_new = self._get_model_and_optim(zero_model=True, fsdp=True) - # Saves checkpoint, calls load_checkpoint and loads model + optim states into new model/optim. - self._save_and_load(checkpoint, model_new, optim_new) - - # Verify model - with FSDP.summon_full_params(model_new): - with FSDP.summon_full_params(model): - for p1, p2 in zip(model.parameters(), model_new.parameters()): - torch.testing.assert_close(p1, p2) - # Verify optim state_dicts - self._validate_dicts( - FSDP.optim_state_dict(model_new, optim_new), - FSDP.optim_state_dict(model, optim), - ) - torch.distributed.barrier() - - @skip_if_cuda_not_available - def test_distributed_save_load(self, get_pet_launch_config) -> None: - lc = get_pet_launch_config(nproc=min(4, torch.cuda.device_count())) - launcher.elastic_launch(lc, entrypoint=self._test_distributed_save_load)() diff --git a/torchtune/utils/__init__.py b/torchtune/utils/__init__.py index 0f6f89dbd0..0e92efdc42 100644 --- a/torchtune/utils/__init__.py +++ b/torchtune/utils/__init__.py @@ -9,6 +9,7 @@ FullModelMetaCheckpointer, FullModelTorchTuneCheckpointer, ModelType, + transform_opt_state_dict, ) from ._device import get_device from ._distributed import ( # noqa @@ -22,11 +23,6 @@ wrap_fsdp, ) from .argparse import TuneArgumentParser -from .checkpoint import ( # noqa - save_checkpoint, - transform_opt_state_dict, - validate_checkpoint, -) from .checkpointable_dataloader import CheckpointableDataLoader from .collate import padded_collate from .constants import ( # noqa diff --git a/torchtune/utils/_checkpointing/__init__.py b/torchtune/utils/_checkpointing/__init__.py index 3002b0dfd2..7fb33f8981 100644 --- a/torchtune/utils/_checkpointing/__init__.py +++ b/torchtune/utils/_checkpointing/__init__.py @@ -9,4 +9,4 @@ FullModelMetaCheckpointer, FullModelTorchTuneCheckpointer, ) -from ._checkpointer_utils import ModelType # noqa +from ._checkpointer_utils import ModelType, transform_opt_state_dict # noqa diff --git a/torchtune/utils/_checkpointing/_checkpointer_utils.py b/torchtune/utils/_checkpointing/_checkpointer_utils.py index 19109385b4..3f52e8c968 100644 --- a/torchtune/utils/_checkpointing/_checkpointer_utils.py +++ b/torchtune/utils/_checkpointing/_checkpointer_utils.py @@ -9,10 +9,16 @@ from typing import Any, Dict import torch +import torch.nn as nn +import torch.optim as optim +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP + +from torchtune.utils._distributed import contains_fsdp class ModelType(Enum): LLAMA2 = "llama2" + MISTRAL = "mistral" def get_path(input_dir: Path, filename: str, missing_ok: bool = False) -> Path: @@ -54,3 +60,27 @@ def safe_torch_load(checkpoint_path: Path) -> Dict[str, Any]: except Exception as e: raise ValueError(f"Unable to load checkpoint from {checkpoint_path}. ") from e return state_dict + + +def transform_opt_state_dict( + opt_state_dict: Dict[str, Any], model: nn.Module, optimizer: optim.Optimizer +) -> Dict[str, Any]: + """ + Transforms the optimizer state dict for FSDP using the ``optim_state_dict_to_load`` + from distributed library within PyTorch. If FSDP is not used, the optimizer state dict is returned as is. + + Args: + opt_state_dict (Dict[str, Any]): Optimizer state dict extracted from the checkpoint + model (nn.Module): Model that checkpoint will be loaded into. + optimizer (optim.Optimizer): Optimizer that optimizer state checkpoints will be loaded into. + + Returns: + ckpt_dict (Dict[str, Any]): Transformed optimizer state dict. + """ + optim_state_dict_to_load = ( + FSDP.optim_state_dict_to_load(model, optimizer, opt_state_dict) + if contains_fsdp(model) + else opt_state_dict + ) + + return optim_state_dict_to_load diff --git a/torchtune/utils/checkpoint.py b/torchtune/utils/checkpoint.py deleted file mode 100644 index 12a4dd1217..0000000000 --- a/torchtune/utils/checkpoint.py +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -from typing import Any, Callable, Dict, Optional - -import torch -import torch.nn as nn -import torch.optim as optim -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP -from torchtune.utils._distributed import get_world_size_and_rank - -from torchtune.utils.constants import ( - EPOCHS_KEY, - MAX_STEPS_KEY, - MODEL_KEY, - OPT_KEY, - SEED_KEY, - TOTAL_EPOCHS_KEY, -) - - -def _contains_fsdp(model: nn.Module) -> bool: - """ - Checks if the model contains FSDP. - - Args: - model (nn.Module): Model to check. - - Returns: - bool: True if the model contains FSDP, False otherwise. - """ - return any( - isinstance(m, torch.distributed.fsdp.FullyShardedDataParallel) - for m in model.modules() - ) - - -def save_checkpoint( - ckpt_dict: Dict[str, Any], - output_loc: str, - model_key_filter: Optional[Callable[[str], bool]] = None, -) -> None: - """ - Saves `ckpt_dict` to `output_loc`. `ckpt_dict` is expected to have at least a key `model` which represents - the model to be checkpointed. This function will call `state_dict` in a distributed-aware fashion on checkpointable objects - (currently only objects specified by "model" and "optimizer" keys). For distributed jobs, only rank 0 - will write out a checkpoint. - Only full (unsharded) checkpoints are supported currently, i.e. full checkpoints are taken even if model and optimizer - are sharded with FSDP. - - Args: - ckpt_dict (Dict[str, Any]): Dictionary containing the checkpoint to be saved. Must have at least `model` key. - output_loc (str): Local path to save the checkpoint to. - model_key_filter (Optional[Callable[[str], bool]]): Optional function to filter the keys in the model state dict. - This function should return True if the key is intended to be included in the saved checkpoint - and False otherwise. - Raises: - RuntimeError: If `ckpt_dict` does not contain a `model` key. - - Example: - >>> output_loc = "/tmp/output.pt" - >>> ckpt_dict = {"model": model.state_dict(), "optimizer": optimizer.state_dict()} - >>> torchtune.utils.checkpoint.save_checkpoint(ckpt_dict, output_loc) - """ - if MODEL_KEY not in ckpt_dict: - raise RuntimeError( - "Expected `ckpt_dict` to contain a `model` key, but it does not." - ) - if OPT_KEY in ckpt_dict: - optimizer_state_dict = ( - FSDP.optim_state_dict( - ckpt_dict[MODEL_KEY], - ckpt_dict[OPT_KEY], - ) - if _contains_fsdp(ckpt_dict[MODEL_KEY]) - else ckpt_dict[OPT_KEY].state_dict() - ) - ckpt_dict[OPT_KEY] = optimizer_state_dict - - model_state_dict = ckpt_dict[MODEL_KEY].state_dict() - if model_key_filter: - model_state_dict = { - k: v for k, v in model_state_dict.items() if model_key_filter(k) - } - ckpt_dict[MODEL_KEY] = model_state_dict - _, rank = get_world_size_and_rank() - if rank == 0: - torch.save(ckpt_dict, output_loc) - - -def load_checkpoint( - ckpt_path: str, - model: nn.Module, - optimizer: Optional[optim.Optimizer] = None, -) -> Dict[str, Any]: - """ - Loads a checkpoint from `ckpt_path` into `model` and optionally `optimizer`. This function is meant to be used in tandem with - `save_checkpoint` and assumes the checkpoint was saved as such. At minimum, the checkpoint needs to contain a `model` key that - maps to the model's states. - - NOTE: `load_checkpoint` does NOT load model and optimizer states into the model and optimizer respectively. - `load_checkpoint` handles the appropriate transformations (i.e. related to FSDP), but user is expected to - call `load_state_dict` on the returned results. - - Args: - ckpt_path (str): String indicating local path to saved checkpoint file. - model (nn.Module): Model that checkpoint will be loaded into. - optimizer (Optional[optim.Optimizer]): Optimizer that optimizer state checkpoints will be loaded into. If not specified, - "optimizer" key in `ckpt_dict` will be ignored, if present. Default: `None`. - - Returns: - ckpt_dict (Dict[str, Any]): Dictionary containing loaded objects. Objects in this dictionary can be used - to restore model, optimizer, and any other checkpointed states. - - Raises: - RuntimeError: If `ckpt_dict` does not contain a `model` key. - RuntimeError: If `ckpt_dict` does not contain an `optimizer` key and an optimizer was passed in. - - Example: - >>> ckpt_dict = torchtune.utils.checkpoint.load_checkpoint(ckpt_path, model, optimizer) - >>> model.load_state_dict(ckpt_dict["model"]) - >>> optimizer.load_state_dict(ckpt_dict["optimizer"]) - """ - - ckpt_dict = torch.load(ckpt_path, map_location="cpu", weights_only=True) - if MODEL_KEY not in ckpt_dict: - raise RuntimeError( - """Expected loaded checkpoint to contain a `model` key, but it does not. Ensure checkpoint was saved - with `save_checkpoint`.""" - ) - if optimizer is not None and OPT_KEY not in ckpt_dict: - raise RuntimeError( - """Expected loaded checkpoint to contain an `optimizer` key since an optimizer was passed in, but it does not. - Ensure checkpoint was saved with `save_checkpoint`.""" - ) - - # Transform optimizer states if using FSDP and overwrite ckpt_dict["optimizer"] with the transformed optimizer state. - if optimizer is not None: - optim_state_dict_to_load = ( - FSDP.optim_state_dict_to_load(model, optimizer, ckpt_dict[OPT_KEY]) - if _contains_fsdp(model) - else ckpt_dict[OPT_KEY] - ) - - ckpt_dict[OPT_KEY] = optim_state_dict_to_load - - return ckpt_dict - - -def transform_opt_state_dict( - opt_state_dict: Dict[str, Any], model: nn.Module, optimizer: optim.Optimizer -) -> Dict[str, Any]: - """ - Transforms the optimizer state dict for FSDP using the ``optim_state_dict_to_load`` - from distributed library within PyTorch. If FSDP is not used, the optimizer state dict is returned as is. - - Args: - opt_state_dict (Dict[str, Any]): Optimizer state dict extracted from the checkpoint - model (nn.Module): Model that checkpoint will be loaded into. - optimizer (optim.Optimizer): Optimizer that optimizer state checkpoints will be loaded into. - - Returns: - ckpt_dict (Dict[str, Any]): Transformed optimizer state dict. - """ - optim_state_dict_to_load = ( - FSDP.optim_state_dict_to_load(model, optimizer, opt_state_dict) - if _contains_fsdp(model) - else opt_state_dict - ) - - return optim_state_dict_to_load - - -def validate_checkpoint(ckpt_dict: Dict[str, Any], resume_from_checkpoint: bool): - """ - Validates the checkpoint dict. This includes validating the recipe state in case we're resuming - training from a checkpoint. - - Args: - ckpt_dict (Dict[str, Any]): Dictionary with recipe state, extracted from the checkpoint - resume_from_checkpoint (bool): Boolean flag specifying whether training is being resumed from a checkpoint. - - Raises: - RuntimeError: If ``ckpt_dict`` does not contain a ``model`` key. - RuntimeError: If ``resume_from_checkpoint`` is `True` and `ckpt_dict` does not contain - either "optimizer", "epochs_run", "seed", "total_epochs" or "max_steps_per_epoch" keys. - """ - if MODEL_KEY not in ckpt_dict: - raise RuntimeError( - """Expected loaded checkpoint to contain a `model` key, but it does not. Ensure checkpoint was saved - with `save_checkpoint`.""" - ) - - if resume_from_checkpoint: - - # If the correct state is not available, fail. Training will not be - # meaningful - if ( - OPT_KEY not in ckpt_dict - or EPOCHS_KEY not in ckpt_dict - or SEED_KEY not in ckpt_dict - or TOTAL_EPOCHS_KEY not in ckpt_dict - or MAX_STEPS_KEY not in ckpt_dict - ): - raise ValueError( - f"Checkpoint does not contain the required keys needed to resume training correctly.\n" - f"Expected Keys: {OPT_KEY}, {EPOCHS_KEY}, {SEED_KEY}, {TOTAL_EPOCHS_KEY}, {MAX_STEPS_KEY}, {MODEL_KEY}\n" - f"Found Keys: {ckpt_dict.keys()}." - ) From 54a5e2a7d9d74cae174d88797026fea7e69a0c30 Mon Sep 17 00:00:00 2001 From: yechenzhi <136920488@qq.com> Date: Fri, 29 Mar 2024 00:10:29 +0800 Subject: [PATCH 04/76] fix typo (#606) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b66a516b2b..1df884e6e0 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ Again, the argument to `--nproc_per_node` can be varied subject to memory constr An example to run QLoRA on a single device can be achieved with the following: ``` -tune lora_finetune_single_device --config recipes/configs/llama2/7B_qlora_single_device +tune lora_finetune_single_device --config llama2/7B_qlora_single_device ```   From de155dce24a740a25b97dd81f527cc8b23b48c6c Mon Sep 17 00:00:00 2001 From: Rafi Ayub <33648637+RdoubleA@users.noreply.github.com> Date: Thu, 28 Mar 2024 12:52:39 -0700 Subject: [PATCH 05/76] Chat dataset + SlimOrca refactor + more templates (#576) --- docs/source/api_ref_datasets.rst | 2 +- docs/source/examples/configs.rst | 4 +- tests/test_utils.py | 10 + .../{test_utils.py => test_config_utils.py} | 32 +++ tests/torchtune/data/test_data_utils.py | 84 ++++++++ tests/torchtune/data/test_templates.py | 108 ++++++++++ .../torchtune/datasets/test_alpaca_dataset.py | 4 +- tests/torchtune/datasets/test_chat_dataset.py | 185 ++++++++++++++++++ .../datasets/test_grammar_dataset.py | 4 +- .../datasets/test_instruct_dataset.py | 44 +---- .../torchtune/datasets/test_samsum_dataset.py | 4 +- .../datasets/test_slimorca_dataset.py | 87 ++------ torchtune/config/_utils.py | 36 ++++ torchtune/data/__init__.py | 14 ++ torchtune/{datasets => data}/_common.py | 0 torchtune/data/_templates.py | 163 +++++++++++++-- torchtune/data/_transforms.py | 53 +++++ torchtune/data/_types.py | 17 ++ torchtune/data/_utils.py | 73 +++++++ torchtune/datasets/__init__.py | 10 +- torchtune/datasets/_chat.py | 182 +++++++++++++++++ torchtune/datasets/_instruct.py | 64 +----- torchtune/datasets/_samsum.py | 2 +- torchtune/datasets/_slimorca.py | 127 +++--------- 24 files changed, 1014 insertions(+), 295 deletions(-) rename tests/torchtune/config/{test_utils.py => test_config_utils.py} (77%) create mode 100644 tests/torchtune/data/test_data_utils.py create mode 100644 tests/torchtune/datasets/test_chat_dataset.py rename torchtune/{datasets => data}/_common.py (100%) create mode 100644 torchtune/data/_transforms.py create mode 100644 torchtune/data/_types.py create mode 100644 torchtune/data/_utils.py create mode 100644 torchtune/datasets/_chat.py diff --git a/docs/source/api_ref_datasets.rst b/docs/source/api_ref_datasets.rst index d6199623de..c39b9cf4bc 100644 --- a/docs/source/api_ref_datasets.rst +++ b/docs/source/api_ref_datasets.rst @@ -13,4 +13,4 @@ torchtune.datasets alpaca_dataset grammar_dataset samsum_dataset - SlimOrcaDataset + slimorca_dataset diff --git a/docs/source/examples/configs.rst b/docs/source/examples/configs.rst index ef564fa1a3..1a8fb5294a 100644 --- a/docs/source/examples/configs.rst +++ b/docs/source/examples/configs.rst @@ -243,5 +243,5 @@ name directly. Any nested fields in the components can be overridden with dot no .. code-block:: bash - # Change to SlimOrcaDataset and set train_on_input to False - tune full_finetune --config my_config.yaml dataset=torchtune.datasets.SlimOrcaDataset dataset.train_on_input=False + # Change to slimorca_dataset and set train_on_input to False + tune full_finetune --config my_config.yaml dataset=torchtune.datasets.slimorca_dataset dataset.train_on_input=False diff --git a/tests/test_utils.py b/tests/test_utils.py index 6e2b6fbde1..1c785a09d2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -32,6 +32,16 @@ } +class DummyTokenizer: + def encode(self, text, **kwargs): + words = text.split() + return [len(word) for word in words] + + @property + def eos_id(self): + return -1 + + def get_assets_path(): return Path(__file__).parent / "assets" diff --git a/tests/torchtune/config/test_utils.py b/tests/torchtune/config/test_config_utils.py similarity index 77% rename from tests/torchtune/config/test_utils.py rename to tests/torchtune/config/test_config_utils.py index b1332b3139..b89e78f4fa 100644 --- a/tests/torchtune/config/test_utils.py +++ b/tests/torchtune/config/test_config_utils.py @@ -9,9 +9,11 @@ import pytest from torchtune.config._utils import ( _get_component_from_path, + _get_template, _merge_yaml_and_cli_args, InstantiationError, ) +from torchtune.data import AlpacaInstructTemplate from torchtune.utils.argparse import TuneArgumentParser _CONFIG = { @@ -107,3 +109,33 @@ def test_merge_yaml_and_cli_args(self, mock_load): ValueError, match="Command-line overrides must be in the form of key=value" ): _ = _merge_yaml_and_cli_args(yaml_args, cli_args) + + def test_get_template(self): + # Test valid template class + template = _get_template("AlpacaInstructTemplate") + assert isinstance(template, AlpacaInstructTemplate) + + # Test invalid template class + with pytest.raises( + ValueError, + match="Must be a PromptTemplate class or a string with placeholders.", + ): + _ = _get_template("InvalidTemplate") + + # Test valid template strings + valid_templates = [ + "Instruction: {instruction}\nInput: {input}", + "Instruction: {instruction}", + "{a}", + ] + for template in valid_templates: + assert _get_template(template) == template + + # Test invalid template strings + invalid_templates = ["hello", "{}", "a}{b"] + for template in invalid_templates: + with pytest.raises( + ValueError, + match="Must be a PromptTemplate class or a string with placeholders.", + ): + _ = _get_template(template) diff --git a/tests/torchtune/data/test_data_utils.py b/tests/torchtune/data/test_data_utils.py new file mode 100644 index 0000000000..bc6974a3f9 --- /dev/null +++ b/tests/torchtune/data/test_data_utils.py @@ -0,0 +1,84 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from tests.test_utils import DummyTokenizer +from torchtune.data import tokenize_prompt_and_response, truncate +from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX + + +def test_tokenize_prompt_and_response(): + tokenizer = DummyTokenizer() + prompt = "Instruction:\nThis is an instruction.\n\nInput:\nThis is an input.\n\nResponse: " + response = "I always know what I'm doing, do you?" + prompt_length = 11 + expected_tokenized_prompt = [ + 12, + 4, + 2, + 2, + 12, + 6, + 4, + 2, + 2, + 6, + 9, + 1, + 6, + 4, + 4, + 3, + 6, + 2, + 4, + ] + expected_tokenized_label = [CROSS_ENTROPY_IGNORE_IDX] * prompt_length + [ + 1, + 6, + 4, + 4, + 3, + 6, + 2, + 4, + ] + + tokenized_prompt, tokenized_label = tokenize_prompt_and_response( + tokenizer, prompt, response + ) + assert tokenized_prompt == expected_tokenized_prompt + assert tokenized_label == expected_tokenized_label + + tokenized_prompt, tokenized_label = tokenize_prompt_and_response( + tokenizer, prompt, response, train_on_input=True + ) + assert tokenized_prompt == expected_tokenized_prompt + assert tokenized_label == expected_tokenized_prompt + + +def test_truncate(): + prompt_tokens = [1, 2, 3, 4, -1] + label_tokens = [1, 2, 3, 4, -1] + + # Test no truncation + truncated_prompt_tokens, truncated_label_tokens = truncate( + tokenizer=DummyTokenizer(), + prompt_tokens=prompt_tokens, + label_tokens=label_tokens, + max_seq_len=5, + ) + assert truncated_prompt_tokens == prompt_tokens + assert truncated_label_tokens == label_tokens + + # Test truncated + truncated_prompt_tokens, truncated_label_tokens = truncate( + tokenizer=DummyTokenizer(), + prompt_tokens=prompt_tokens, + label_tokens=label_tokens, + max_seq_len=4, + ) + assert truncated_prompt_tokens == [1, 2, 3, -1] + assert truncated_label_tokens == [1, 2, 3, -1] diff --git a/tests/torchtune/data/test_templates.py b/tests/torchtune/data/test_templates.py index c3d695ad57..6e0f828773 100644 --- a/tests/torchtune/data/test_templates.py +++ b/tests/torchtune/data/test_templates.py @@ -4,12 +4,24 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import pytest from torchtune.data import ( AlpacaInstructTemplate, + ChatMLTemplate, GrammarErrorCorrectionTemplate, + Llama2ChatTemplate, + MistralChatTemplate, SummarizeTemplate, ) +# Taken from Open-Orca/SlimOrca-Dedup on HuggingFace: +# https://huggingface.co/datasets/Open-Orca/SlimOrca-Dedup +CHAT_SAMPLE = { + "system": "You are an AI assistant. User will you give you a task. Your goal is to complete the task as faithfully as you can. While performing the task think step-by-step and justify your steps.", # noqa: B950 + "user": "Please briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? How about on an icy road? Well one father in Russia did just that, and recorded the entire thing. To her credit, the child seemed to be doing a great job. (0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\nSummary:", # noqa: B950 + "assistant": "A father in Russia allowed his 8-year-old child to drive his car on an icy road and recorded the event. The child appeared to be handling the situation well, showcasing their driving skills despite the challenging conditions.", # noqa: B950 +} + class TestAlpacaInstructTemplate: samples = [ @@ -144,3 +156,99 @@ def test_format_with_column_map(self): actual = self.template.format(modified_sample, column_map=column_map) assert actual == expected_prompt + + +class TestLlama2ChatTemplate: + expected_prompt = ( + "[INST] <>\nYou are an AI assistant. User will you give you a task. " + "Your goal is to complete the task as faithfully as you can. While performing " + "the task think step-by-step and justify your steps.\n<>\n\nPlease " + "briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old " + "Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? " + "How about on an icy road? Well one father in Russia did just that, and recorded " + "the entire thing. To her credit, the child seemed to be doing a great job. " + "(0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\n" + "Summary: [/INST] " + ) + + template = Llama2ChatTemplate() + + def test_format(self): + actual = self.template.format(CHAT_SAMPLE) + assert actual == self.expected_prompt + + def test_format_with_column_map(self): + column_map = {"system": "not_system"} + modified_sample = CHAT_SAMPLE.copy() + modified_sample["not_system"] = modified_sample["system"] + del modified_sample["system"] + + actual = self.template.format(modified_sample, column_map=column_map) + + assert actual == self.expected_prompt + + +class TestMistralChatTemplate: + expected_prompt = ( + "[INST] Please briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old " + "Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? " + "How about on an icy road? Well one father in Russia did just that, and recorded " + "the entire thing. To her credit, the child seemed to be doing a great job. " + "(0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\n" + "Summary: [/INST] " + ) + + template = MistralChatTemplate() + + def test_format(self): + no_system_sample = CHAT_SAMPLE.copy() + del no_system_sample["system"] + actual = self.template.format(no_system_sample) + assert actual == self.expected_prompt + + def test_format_with_system_prompt_raises(self): + with pytest.raises( + ValueError, match="System prompts are not supported in MistralChatTemplate" + ): + _ = self.template.format(CHAT_SAMPLE) + + def test_format_with_column_map(self): + column_map = {"user": "not_user"} + modified_sample = CHAT_SAMPLE.copy() + modified_sample["not_user"] = modified_sample["user"] + del modified_sample["system"] + del modified_sample["user"] + + actual = self.template.format(modified_sample, column_map=column_map) + + assert actual == self.expected_prompt + + +class TestChatMLTemplate: + expected_prompt = ( + "<|im_start|>system\nYou are an AI assistant. User will you give you a task. " + "Your goal is to complete the task as faithfully as you can. While performing " + "the task think step-by-step and justify your steps.<|im_end|>\n<|im_start|>user\nPlease " + "briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old " + "Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? " + "How about on an icy road? Well one father in Russia did just that, and recorded " + "the entire thing. To her credit, the child seemed to be doing a great job. " + "(0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\n" + "Summary:<|im_end|>\n<|im_start|>assistant\n" + ) + + template = ChatMLTemplate() + + def test_format(self): + actual = self.template.format(CHAT_SAMPLE) + assert actual == self.expected_prompt + + def test_format_with_column_map(self): + column_map = {"system": "not_system"} + modified_sample = CHAT_SAMPLE.copy() + modified_sample["not_system"] = modified_sample["system"] + del modified_sample["system"] + + actual = self.template.format(modified_sample, column_map=column_map) + + assert actual == self.expected_prompt diff --git a/tests/torchtune/datasets/test_alpaca_dataset.py b/tests/torchtune/datasets/test_alpaca_dataset.py index 107f68f0f0..a11fd68e89 100644 --- a/tests/torchtune/datasets/test_alpaca_dataset.py +++ b/tests/torchtune/datasets/test_alpaca_dataset.py @@ -9,9 +9,9 @@ import pytest from tests.test_utils import get_assets_path +from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX -from torchtune.datasets._alpaca import alpaca_dataset -from torchtune.datasets._common import CROSS_ENTROPY_IGNORE_IDX +from torchtune.datasets import alpaca_dataset from torchtune.modules.tokenizer import Tokenizer diff --git a/tests/torchtune/datasets/test_chat_dataset.py b/tests/torchtune/datasets/test_chat_dataset.py new file mode 100644 index 0000000000..ee9e122b79 --- /dev/null +++ b/tests/torchtune/datasets/test_chat_dataset.py @@ -0,0 +1,185 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from unittest import mock + +import pytest +from tests.test_utils import DummyTokenizer +from torchtune.data import Message +from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX + +from torchtune.datasets import ChatDataset + + +class DummyTemplate: + def __init__(self): + self.template = { + "system": "System:\n{system}\nUser:\n{user}\nAssistant:\n", + "no_system": "User:\n{user}\nAssistant:\n", + } + + def format(self, sample, column_map=None): + if "system" in sample: + return self.template["system"].format(**sample) + else: + return self.template["no_system"].format(**sample) + + +def _are_messages_equal(messages_a, messages_b): + for ma, mb in zip(messages_a, messages_b): + if ma.role != mb.role: + return False + if ma.content != mb.content: + return False + return True + + +class TestChatDataset: + @pytest.fixture + def template(self): + return DummyTemplate() + + @pytest.fixture + def dialogue(self): + return [ + { + "dialogue": [ + Message(role="system", content="You are an AI assistant."), + Message(role="user", content="What is the meaning of life?"), + Message(role="assistant", content="The meaning of life is 42."), + Message(role="user", content="That's ridiculous."), + Message(role="assistant", content="I agree."), + ], + }, + ] + + @mock.patch("torchtune.datasets._chat.load_dataset") + def test_get_turns(self, mock_load_dataset, template, dialogue): + ds = ChatDataset( + tokenizer=DummyTokenizer(), + source="iam/agoofy/goober", + convert_to_dialogue=lambda x: x, + template=template, + max_seq_len=100, + train_on_input=False, + ) + + # Test a normal multiturn dialogue + prompts, responses = zip( + *[(p, l) for p, l in ds._get_turns(dialogue[0]["dialogue"])] + ) + assert prompts[0] == { + "system": "You are an AI assistant.", + "user": "What is the meaning of life?", + } + assert responses[0] == "The meaning of life is 42." + assert prompts[1] == {"user": "That's ridiculous."} + assert responses[1] == "I agree." + + # Test without system prompt + prompts, responses = zip( + *[(p, l) for p, l in ds._get_turns(dialogue[0]["dialogue"][1:])] + ) + assert prompts[0] == {"user": "What is the meaning of life?"} + assert responses[0] == "The meaning of life is 42." + assert prompts[1] == {"user": "That's ridiculous."} + assert responses[1] == "I agree." + + # Test a missing user message + with pytest.raises( + ValueError, match="Missing a user message before assistant message" + ): + for _ in ds._get_turns( + [dialogue[0]["dialogue"][0]] + dialogue[0]["dialogue"][2:] + ): + pass + + # Test a missing user message and no system message + with pytest.raises( + ValueError, match="Missing a user message before assistant message" + ): + for _ in ds._get_turns(dialogue[0]["dialogue"][2:]): + pass + + # Test repeated messages + with pytest.raises(ValueError, match="Duplicate"): + for _ in ds._get_turns( + dialogue[0]["dialogue"][:2] + dialogue[0]["dialogue"][3:] + ): + pass + with pytest.raises(ValueError, match="Duplicate"): + for _ in ds._get_turns( + [dialogue[0]["dialogue"][0]] + [dialogue[0]["dialogue"][0]] + ): + pass + + # Test incomplete turn + with pytest.raises(ValueError, match="Incomplete turn in dialogue"): + for _ in ds._get_turns(dialogue[0]["dialogue"][:2]): + pass + + @mock.patch("torchtune.datasets._chat.load_dataset") + def test_get_item(self, mock_load_dataset, template, dialogue): + mock_load_dataset.return_value = dialogue + expected_tokenized_prompts = [ + [ + 7, + 3, + 3, + 2, + 2, + 10, + 5, + 4, + 2, + 3, + 7, + 2, + 5, + 10, + 3, + 7, + 2, + 4, + 2, + 3, + 5, + 6, + 11, + 10, + 1, + -1, + ] + ] + prompt_lengths = (14, 4) + expected_labels = [ + [CROSS_ENTROPY_IGNORE_IDX] * prompt_lengths[0] + + [ + 3, + 7, + 2, + 4, + 2, + 3, + ] + + [CROSS_ENTROPY_IGNORE_IDX] * prompt_lengths[1] + + [1, -1] + ] + + ds = ChatDataset( + tokenizer=DummyTokenizer(), + source="iam/agoofy/goober", + convert_to_dialogue=lambda x: x["dialogue"], + template=template, + max_seq_len=100, + train_on_input=False, + ) + assert len(ds) == 1 + mock_load_dataset.assert_called_once() + + prompt, label = ds[0] + assert prompt == expected_tokenized_prompts[0] + assert label == expected_labels[0] diff --git a/tests/torchtune/datasets/test_grammar_dataset.py b/tests/torchtune/datasets/test_grammar_dataset.py index 9c9c0b8cce..5fb41d39eb 100644 --- a/tests/torchtune/datasets/test_grammar_dataset.py +++ b/tests/torchtune/datasets/test_grammar_dataset.py @@ -9,9 +9,9 @@ import pytest from tests.test_utils import get_assets_path -from torchtune.datasets._common import CROSS_ENTROPY_IGNORE_IDX +from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX -from torchtune.datasets._grammar import grammar_dataset +from torchtune.datasets import grammar_dataset from torchtune.modules.tokenizer import Tokenizer diff --git a/tests/torchtune/datasets/test_instruct_dataset.py b/tests/torchtune/datasets/test_instruct_dataset.py index b7c836240d..56f3a61b43 100644 --- a/tests/torchtune/datasets/test_instruct_dataset.py +++ b/tests/torchtune/datasets/test_instruct_dataset.py @@ -6,17 +6,11 @@ from unittest import mock -import pytest -from torchtune.data import AlpacaInstructTemplate -from torchtune.datasets._common import CROSS_ENTROPY_IGNORE_IDX +from tests.test_utils import DummyTokenizer -from torchtune.datasets._instruct import _get_template, InstructDataset +from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX - -class DummyTokenizer: - def encode(self, text, **kwargs): - words = text.split() - return [len(word) for word in words] +from torchtune.datasets import InstructDataset def dummy_transform(sample): @@ -77,7 +71,6 @@ def test_get_item_no_train_on_input(self, mock_load_dataset): for i in range(len(dataset)): prompt, label = dataset[i] - print(prompt, label) assert prompt == self.expected_tokenized_prompts[i] assert label == expected_labels[i] @@ -100,34 +93,3 @@ def test_get_item_train_on_input(self, mock_load_dataset): prompt, label = dataset[i] assert prompt == self.expected_tokenized_prompts[i] assert label == expected_labels[i] - - -def test_get_template(): - # Test valid template class - template = _get_template("AlpacaInstructTemplate") - assert isinstance(template, AlpacaInstructTemplate) - - # Test invalid template class - with pytest.raises( - ValueError, - match="Must be a PromptTemplate class or a string with placeholders.", - ): - _ = _get_template("InvalidTemplate") - - # Test valid template strings - s = [ - "Instruction: {instruction}\nInput: {input}", - "Instruction: {instruction}", - "{a}", - ] - for t in s: - assert _get_template(t) == t - - # Test invalid template strings - s = ["hello", "{}", "a}{b"] - for t in s: - with pytest.raises( - ValueError, - match="Must be a PromptTemplate class or a string with placeholders.", - ): - _ = _get_template(t) diff --git a/tests/torchtune/datasets/test_samsum_dataset.py b/tests/torchtune/datasets/test_samsum_dataset.py index 71eea59937..972b8bbb25 100644 --- a/tests/torchtune/datasets/test_samsum_dataset.py +++ b/tests/torchtune/datasets/test_samsum_dataset.py @@ -9,9 +9,9 @@ import pytest from tests.test_utils import get_assets_path -from torchtune.datasets._common import CROSS_ENTROPY_IGNORE_IDX +from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX -from torchtune.datasets._samsum import samsum_dataset +from torchtune.datasets import samsum_dataset from torchtune.modules.tokenizer import Tokenizer diff --git a/tests/torchtune/datasets/test_slimorca_dataset.py b/tests/torchtune/datasets/test_slimorca_dataset.py index 786dbf4f41..b40df911bd 100644 --- a/tests/torchtune/datasets/test_slimorca_dataset.py +++ b/tests/torchtune/datasets/test_slimorca_dataset.py @@ -8,10 +8,13 @@ import pytest from tests.test_utils import get_assets_path +from torchtune.data import Llama2ChatTemplate -from torchtune.datasets._slimorca import _Llama2ChatFormatConstants, SlimOrcaDataset +from torchtune.datasets import slimorca_dataset from torchtune.modules.tokenizer import Tokenizer +LLAMA_TEMPLATE = Llama2ChatTemplate() + class TestSlimOrcaDataset: @pytest.fixture @@ -20,81 +23,15 @@ def tokenizer(self): # spm.SentencePieceTrainer.train('--input= --model_prefix=m --vocab_size=2000') return Tokenizer.from_file(str(get_assets_path() / "m.model")) - @patch("torchtune.datasets._slimorca.load_dataset") - def test_prompt_label_generation(self, load_dataset, tokenizer): - load_dataset.return_value = [] - dataset = SlimOrcaDataset(tokenizer=tokenizer) - sample = [ - { - "from": "system", - "value": "hi", - }, - { - "from": "human", - "value": "mid", - }, - { - "from": "gpt", - "value": "lo", - }, - ] - prompt, label = dataset._generate_prompt_label(sample) - assert ( - prompt - == f"{_Llama2ChatFormatConstants.B_INST} {_Llama2ChatFormatConstants.B_SYS}hi{_Llama2ChatFormatConstants.E_SYS}mid {_Llama2ChatFormatConstants.E_INST}" # noqa: B950 - ) - assert label == " lo " - - sample = [ - { - "from": "human", - "value": "mid", - }, - { - "from": "gpt", - "value": "lo", - }, - ] - prompt, label = dataset._generate_prompt_label(sample) - assert ( - prompt - == f"{_Llama2ChatFormatConstants.B_INST} mid {_Llama2ChatFormatConstants.E_INST}" - ) - assert label == " lo " - - @patch("torchtune.datasets._slimorca.load_dataset") - def test_token_generation(self, load_dataset, tokenizer): - load_dataset.return_value = [] - dataset = SlimOrcaDataset(tokenizer=tokenizer, max_token_length=4096) - input, label = dataset._generate_tokens("Hello ", "world!") - assert input == [tokenizer.bos_id, 12, 1803, 1024, 103, tokenizer.eos_id] - assert label == ([-100] * 3 + [1024, 103, tokenizer.eos_id]) - - @patch("torchtune.datasets._slimorca.load_dataset") - def test_truncated_token_generation(self, load_dataset, tokenizer): - load_dataset.return_value = [] - dataset = SlimOrcaDataset(tokenizer=tokenizer, max_token_length=5) - # 5 is enough for full prompt, but not for label - input, label = dataset._generate_tokens("Hello ", "world!") - assert input == [tokenizer.bos_id, 12, 1803, 1024, tokenizer.eos_id] - assert label == ([-100] * 3 + [1024, tokenizer.eos_id]) - - # 4 is not enough for full prompt nor response but truncation - # is still feasible - dataset = SlimOrcaDataset(tokenizer=tokenizer, max_token_length=4) - input, label = dataset._generate_tokens("Hello ", "world!") - assert input == [tokenizer.bos_id, 12, 1024, tokenizer.eos_id] - assert label == ([-100] * 2 + [1024, tokenizer.eos_id]) - - @patch("torchtune.datasets._slimorca.load_dataset") + @patch("torchtune.datasets._chat.load_dataset") def test_value_error(self, load_dataset, tokenizer): load_dataset.return_value = [] with pytest.raises(ValueError): - SlimOrcaDataset(tokenizer=tokenizer, max_token_length=3) + slimorca_dataset(tokenizer=tokenizer, max_seq_len=3) - @patch("torchtune.datasets._slimorca.load_dataset") - @pytest.mark.parametrize("max_token_length", [128, 512, 1024, 4096]) - def test_dataset_get_item(self, load_dataset, tokenizer, max_token_length): + @patch("torchtune.datasets._chat.load_dataset") + @pytest.mark.parametrize("max_seq_len", [128, 512, 1024, 4096]) + def test_dataset_get_item(self, load_dataset, tokenizer, max_seq_len): # Sample data from slimorca dataset load_dataset.return_value = [ { @@ -114,10 +51,10 @@ def test_dataset_get_item(self, load_dataset, tokenizer, max_token_length): ] } ] - ds = SlimOrcaDataset(tokenizer=tokenizer, max_token_length=max_token_length) + ds = slimorca_dataset(tokenizer=tokenizer, max_seq_len=max_seq_len) input, label = ds[0] - assert len(input) <= max_token_length - assert len(label) <= max_token_length + assert len(input) <= max_seq_len + assert len(label) <= max_seq_len assert len(input) == len(label) assert input[0] == tokenizer.bos_id assert input[-1] == tokenizer.eos_id diff --git a/torchtune/config/_utils.py b/torchtune/config/_utils.py index 16c5b30d1f..9f54a54adb 100644 --- a/torchtune/config/_utils.py +++ b/torchtune/config/_utils.py @@ -4,6 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import re from argparse import Namespace from importlib import import_module from types import ModuleType @@ -12,6 +13,7 @@ from omegaconf import DictConfig, OmegaConf from torchtune.config._errors import InstantiationError +from torchtune.data._templates import PromptTemplate def _has_component(node: Union[Dict[str, Any], DictConfig]) -> bool: @@ -148,3 +150,37 @@ def _merge_yaml_and_cli_args(yaml_args: Namespace, cli_args: List[str]) -> DictC # CLI takes precedence over yaml args return OmegaConf.merge(yaml_conf, cli_conf) + + +def _get_template(template: str) -> PromptTemplate: + """ + Get the prompt template class from the template string. + + String should either be the PromptTemplate class name directly, or a raw + string with 1 or more placeholders. If none of these apply, then raise an + error. + + Args: + template (str): class name of template, or string with placeholders + + Returns: + PromptTemplate: the prompt template class or the same verified string + + Raises: + ValueError: if the template is not a PromptTemplate class or a proper + template string + """ + path = "torchtune.data." + template + try: + template_class = _get_component_from_path(path) + return template_class() + except InstantiationError: + # Verify that string can be used as a template, should have variable + # placeholders + pattern = r"\{\w+\}" + if not re.search(pattern, template): + raise ValueError( + f"Invalid template '{template}': " + + "Must be a PromptTemplate class or a string with placeholders." + ) from None + return template diff --git a/torchtune/data/__init__.py b/torchtune/data/__init__.py index b18b942778..d67021f879 100644 --- a/torchtune/data/__init__.py +++ b/torchtune/data/__init__.py @@ -6,14 +6,28 @@ from torchtune.data._templates import ( AlpacaInstructTemplate, + ChatMLTemplate, GrammarErrorCorrectionTemplate, + Llama2ChatTemplate, + MistralChatTemplate, PromptTemplate, SummarizeTemplate, ) +from torchtune.data._transforms import sharegpt_to_llama2_dialogue +from torchtune.data._types import Dialogue, Message +from torchtune.data._utils import tokenize_prompt_and_response, truncate __all__ = [ "AlpacaInstructTemplate", "GrammarErrorCorrectionTemplate", "PromptTemplate", "SummarizeTemplate", + "Llama2ChatTemplate", + "MistralChatTemplate", + "ChatMLTemplate", + "sharegpt_to_llama2_dialogue", + "truncate", + "tokenize_prompt_and_response", + "Dialogue", + "Message", ] diff --git a/torchtune/datasets/_common.py b/torchtune/data/_common.py similarity index 100% rename from torchtune/datasets/_common.py rename to torchtune/data/_common.py diff --git a/torchtune/data/_templates.py b/torchtune/data/_templates.py index 783d1554d1..549edf0306 100644 --- a/torchtune/data/_templates.py +++ b/torchtune/data/_templates.py @@ -70,12 +70,9 @@ def format( Returns: The formatted prompt """ - if column_map is not None: - key_input = column_map["input"] - key_instruction = column_map["instruction"] - else: - key_input = "input" - key_instruction = "instruction" + column_map = column_map or {} + key_input = column_map.get("input", "input") + key_instruction = column_map.get("instruction", "instruction") if key_input in sample and sample[key_input]: prompt = self.template["prompt_input"].format( @@ -110,10 +107,8 @@ def format( Returns: The formatted prompt """ - if column_map is not None and "sentence" in column_map: - key_sentence = column_map["sentence"] - else: - key_sentence = "sentence" + column_map = column_map or {} + key_sentence = column_map.get("sentence", "sentence") prompt = self.template.format(sentence=sample[key_sentence]) return prompt @@ -141,10 +136,150 @@ def format( Returns: The formatted prompt """ - if column_map is not None and "dialogue" in column_map: - key_dialogue = column_map["dialogue"] - else: - key_dialogue = "dialogue" + column_map = column_map or {} + key_dialogue = column_map.get("dialogue", "dialogue") prompt = self.template.format(dialogue=sample[key_dialogue]) return prompt + + +class Llama2ChatTemplate(PromptTemplate): + """ + Prompt template that formats human and system prompts with appropriate tags + used in LLaMA2 pre-training. Taken from Meta's official LLaMA inference + repository at https://github.com/meta-llama/llama/blob/main/llama/generation.py. + The response is tokenized outside of this template. + + Example: + "[INST] <> + You are a helpful, respectful and honest assistant. + <> + + I am going to Paris, what should I see? [/INST] Paris, the capital of France, is known for its stunning architecture..." + """ + + B_INST, E_INST = "[INST]", "[/INST]" + B_SYS, E_SYS = "<>\n", "\n<>\n\n" + template = { + "system": f"{B_INST} {B_SYS}{{system}}{E_SYS}{{user}} {E_INST} ", + "no_system": f"{B_INST} {{user}} {E_INST} ", + } + + def format( + self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None + ) -> str: + """ + Generate prompt from a user message and optional system prompt. + + Args: + sample (Mapping[str, Any]): a single data sample, expects role keys "system" (optional) + and "user" in the sample. + column_map (Optional[Dict[str, str]]): a mapping from the expected + role names in the template to the actual role names in the sample. + If None, assume these are "system" and "user". + + Returns: + The formatted prompt + """ + column_map = column_map or {} + key_system = column_map.get("system", "system") + key_user = column_map.get("user", "user") + + if key_system in sample: + return self.template["system"].format( + system=sample[key_system], user=sample[key_user] + ) + else: + return self.template["no_system"].format(user=sample[key_user]) + + +class MistralChatTemplate(PromptTemplate): + """ + Prompt template that formats according to Mistral's instruct model: + https://docs.mistral.ai/models/ + + It is identical to `Llama2ChatTemplate`, except it does not support system + prompts. + + Example: + "[INST] I am going to Paris, what should I see? [/INST] Paris, the capital + of France, is known for its stunning architecture..." + """ + + B_INST, E_INST = "[INST]", "[/INST]" + template = f"{B_INST} {{user}} {E_INST} " + + def format( + self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None + ) -> str: + """ + Generate prompt from a user message + + Args: + sample (Mapping[str, Any]): a single data sample, expects only "user" in the sample. + column_map (Optional[Dict[str, str]]): a mapping from the expected + role names in the template to the actual role names in the sample. + If None, assume these are "user". + + Returns: + The formatted prompt + + Raises: + ValueError: if the sample contains a "system" key + """ + if "system" in sample: + raise ValueError("System prompts are not supported in MistralChatTemplate") + + column_map = column_map or {} + key_user = column_map.get("user", "user") + + return self.template.format(user=sample[key_user]) + + +class ChatMLTemplate(PromptTemplate): + """ + OpenAI's Chat Markup Language used by their chat models: + https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/ai-services/openai/includes/chat-markup-language.md + It is the default template used by HuggingFace models. + + Example: + <|im_start|>system + Provide some context and/or instructions to the model.<|im_end|> + <|im_start|>user + The user’s message goes here<|im_end|> + <|im_start|>assistant + The assistant’s response goes here<|im_end|> + """ + + IM_START, IM_END = "<|im_start|>", "<|im_end|>" + template = { + "system": f"{IM_START}system\n{{system}}{IM_END}\n{IM_START}user\n{{user}}{IM_END}\n{IM_START}assistant\n", + "no_system": f"{IM_START}user\n{{user}}{IM_END}\n{IM_START}assistant\n", + } + + def format( + self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None + ) -> str: + """ + Generate prompt from a user message and optional system prompt. + + Args: + sample (Mapping[str, Any]): a single data sample, expects role keys "system" (optional) + and "user" in the sample. + column_map (Optional[Dict[str, str]]): a mapping from the expected + role names in the template to the actual role names in the sample. + If None, assume these are "system" and "user". + + Returns: + The formatted prompt + """ + column_map = column_map or {} + key_system = column_map.get("system", "system") + key_user = column_map.get("user", "user") + + if key_system in sample: + return self.template["system"].format( + system=sample[key_system], user=sample[key_user] + ) + else: + return self.template["no_system"].format(user=sample[key_user]) diff --git a/torchtune/data/_transforms.py b/torchtune/data/_transforms.py new file mode 100644 index 0000000000..155b123caf --- /dev/null +++ b/torchtune/data/_transforms.py @@ -0,0 +1,53 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Any, Mapping + +from torchtune.data._types import Dialogue, Message + + +def sharegpt_to_llama2_dialogue(sample: Mapping[str, Any]) -> Dialogue: + """ + Convert a chat sample adhering to the ShareGPT format to the LLaMA2 format. + + ShareGPT follows: + { + "conversations": [ + { + "from": , + "value": , + }, + ... + ] + } + + LLaMA2 follows: + [ + { + "role": , + "content": , + }, + ... + ] + + Args: + sample (Mapping[str, Any]): a single data sample with "conversations" field pointing + to a list of dict messages. + + Returns: + Dialogue: a list of messages with "role" and "content" fields. See `torchtune.datasets._types.Message` + and `torchtune.datasets._types.Dialogue` for more details. + """ + role_map = {"system": "system", "human": "user", "gpt": "assistant"} + conversations = sample["conversations"] + + dialogue = [] + for message in conversations: + role = role_map[message["from"]] + content = message["value"] + dialogue.append(Message(role=role, content=content)) + + return dialogue diff --git a/torchtune/data/_types.py b/torchtune/data/_types.py new file mode 100644 index 0000000000..ec30de7cfc --- /dev/null +++ b/torchtune/data/_types.py @@ -0,0 +1,17 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import List, Literal, TypedDict + +Role = Literal["system", "user", "assistant"] + + +class Message(TypedDict): + role: Role + content: str + + +Dialogue = List[Message] diff --git a/torchtune/data/_utils.py b/torchtune/data/_utils.py new file mode 100644 index 0000000000..207369ed80 --- /dev/null +++ b/torchtune/data/_utils.py @@ -0,0 +1,73 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import copy +from typing import List, Tuple + +from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX +from torchtune.modules import Tokenizer + + +def tokenize_prompt_and_response( + tokenizer: Tokenizer, + prompt: str, + response: str, + train_on_input: bool = False, +) -> Tuple[List[int], List[int]]: + """ + Tokenize a prompt and response pair. + + Args: + tokenizer (Tokenizer): The tokenizer to use. + prompt (str): The prompt to tokenize. + response (str): The response to tokenize. + train_on_input (bool): Whether to train on prompt or mask it out. Default + is False. + + Returns: + The tokenized prompt and response. + """ + prompt_with_response = prompt + response + encoded_prompt = tokenizer.encode(text=prompt, add_bos=True, add_eos=False) + encoded_prompt_with_response = tokenizer.encode( + text=prompt_with_response, add_bos=True, add_eos=True + ) + labels = copy.deepcopy(encoded_prompt_with_response) + + if not train_on_input: + labels[: len(encoded_prompt)] = [CROSS_ENTROPY_IGNORE_IDX] * len(encoded_prompt) + + assert len(encoded_prompt_with_response) == len(labels) + + return encoded_prompt_with_response, labels + + +def truncate( + tokenizer: Tokenizer, + prompt_tokens: List[int], + label_tokens: List[int], + max_seq_len: int, +) -> Tuple[List[int], List[int]]: + """ + Truncate a prompt and label pair if longer than max sequence length. + + Args: + tokenizer (Tokenizer): The tokenizer to use. + prompt_tokens (List[int]): The prompt + response tokens. + label_tokens (List[int]): The label tokens. + max_seq_len (int): The maximum sequence length. + + Returns: + The truncated prompt and label. + """ + prompt_tokens_truncated = prompt_tokens[:max_seq_len] + label_tokens_truncated = label_tokens[:max_seq_len] + if prompt_tokens_truncated[-1] != tokenizer.eos_id: + prompt_tokens_truncated[-1] = tokenizer.eos_id + if label_tokens_truncated[-1] != tokenizer.eos_id: + label_tokens_truncated[-1] = tokenizer.eos_id + + return prompt_tokens_truncated, label_tokens_truncated diff --git a/torchtune/datasets/__init__.py b/torchtune/datasets/__init__.py index 787eb558a3..765723d84d 100644 --- a/torchtune/datasets/__init__.py +++ b/torchtune/datasets/__init__.py @@ -5,15 +5,19 @@ # LICENSE file in the root directory of this source tree. from torchtune.datasets._alpaca import alpaca_dataset +from torchtune.datasets._chat import ChatDataset from torchtune.datasets._grammar import grammar_dataset -from torchtune.datasets._instruct import InstructDataset +from torchtune.datasets._instruct import instruct_dataset, InstructDataset from torchtune.datasets._samsum import samsum_dataset -from torchtune.datasets._slimorca import SlimOrcaDataset +from torchtune.datasets._slimorca import slimorca_dataset __all__ = [ "alpaca_dataset", "grammar_dataset", "samsum_dataset", - "SlimOrcaDataset", "InstructDataset", + "slimorca_dataset", + "ChatDataset", + "instruct_dataset", + "chat_dataset", ] diff --git a/torchtune/datasets/_chat.py b/torchtune/datasets/_chat.py new file mode 100644 index 0000000000..6390c19ca0 --- /dev/null +++ b/torchtune/datasets/_chat.py @@ -0,0 +1,182 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Any, Callable, Dict, Generator, List, Mapping, Tuple + +from datasets import load_dataset +from torch.utils.data import Dataset +from torchtune.config._utils import _get_template + +from torchtune.data import ( + Dialogue, + PromptTemplate, + sharegpt_to_llama2_dialogue, + tokenize_prompt_and_response, + truncate, +) +from torchtune.modules import Tokenizer + + +class ChatDataset(Dataset): + """ + Class that supports any custom dataset with multiturn conversations. + + The general flow from loading a sample to tokenized prompt is: + load sample -> apply transform -> foreach turn{format into template -> tokenize} + + If the column/key names differ from the expected names in the `PromptTemplate`, + then the `column_map` argument can be used to provide this mapping. + + Use `convert_to_dialogue` to prepare your dataset into the llama conversation format + and roles: + [ + { + "role": , + "content": , + }, + ... + ] + + This class supports multi-turn conversations. If a tokenizer sample with multiple + turns does not fit within `max_seq_len` then it is truncated. + + Args: + tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. + source (str): path string of dataset, anything supported by HuggingFace's `load_dataset` + (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) + convert_to_dialogue (Callable[[Mapping[str, Any]], Dialogue]): function that keys into the desired field in the sample + and converts to a list of `Messages` that follows the llama format with the expected keys + template (PromptTemplate): template used to format the prompt. If the placeholder variable + names in the template do not match the column/key names in the dataset, use `column_map` to map them. + max_seq_len (int): Maximum number of tokens in the returned input and label token id lists. + train_on_input (bool): Whether the model is trained on the prompt or not. Default is False. + **load_dataset_kwargs (Dict[str, Any]): additional keyword arguments to pass to `load_dataset`. + """ + + def __init__( + self, + tokenizer: Tokenizer, + source: str, + convert_to_dialogue: Callable[[Mapping[str, Any]], Dialogue], + template: PromptTemplate, + max_seq_len: int, + train_on_input: bool = False, + **load_dataset_kwargs: Dict[str, Any], + ) -> None: + self._tokenizer = tokenizer + self._data = load_dataset(source, **load_dataset_kwargs) + self._convert_to_dialogue = convert_to_dialogue + self.template = template + self.max_seq_len = max_seq_len + self.train_on_input = train_on_input + + def __len__(self): + return len(self._data) + + def __getitem__(self, index: int) -> Tuple[List[int], List[int]]: + sample = self._data[index] + return self._prepare_sample(sample) + + def _prepare_sample(self, sample: Mapping[str, Any]) -> Tuple[List[int], List[int]]: + dialogue = self._convert_to_dialogue(sample) + + prompt_tokens = [] + label_tokens = [] + for prompt, label in self._get_turns(dialogue): + formatted_prompt = self.template.format(prompt) + encoded_prompt_with_response, labels = tokenize_prompt_and_response( + tokenizer=self._tokenizer, + prompt=formatted_prompt, + response=label, + train_on_input=self.train_on_input, + ) + prompt_tokens.extend(encoded_prompt_with_response) + label_tokens.extend(labels) + + if len(prompt_tokens) >= self.max_seq_len: + break + + prompt_tokens, label_tokens = truncate( + self._tokenizer, prompt_tokens, label_tokens, self.max_seq_len + ) + + assert len(prompt_tokens) == len(label_tokens) + + return prompt_tokens, label_tokens + + def _get_turns( + self, dialogue: Dialogue + ) -> Generator[Tuple[Dict[str, str], str], None, None]: + prompt_messages = {} + for message in dialogue: + # If we are at the assistant message, we are at the end of a turn, yield. + if message["role"] == "assistant": + if "user" not in prompt_messages: + raise ValueError( + f"Missing a user message before assistant message: {message['content']}" + ) + yield prompt_messages, message["content"] + prompt_messages = {} + # Otherwise, continue to add to the turn's messages + else: + if message["role"] in prompt_messages: + raise ValueError( + f"Duplicate {message['role']} message in dialogue: {message['content']}" + ) + prompt_messages[message["role"]] = message["content"] + + # If we never yielded, then the last turn was incomplete + if prompt_messages: + raise ValueError( + f"Incomplete turn in dialogue, current turn: {prompt_messages}" + ) + + +def chat_dataset( + tokenizer: Tokenizer, + source: str, + conversation_format: str, + template: str, + max_seq_len: int, + train_on_input: bool = False, + **load_dataset_kwargs: Dict[str, Any], +) -> ChatDataset: + """ + Build a configurable dataset with conversations. This method should be + used to configure a custom chat dataset from the yaml config instead of + using `ChatDataset` directly, as it is made to be config friendly. + + Args: + tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. + source (str): path string of dataset, anything supported by HuggingFace's `load_dataset` + (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) + conversation_format (str): string specifying expected format of conversations in the dataset + for automatic conversion to the llama format. Supported formats are: "sharegpt" + template (str): class name of template used to format the prompt. + max_seq_len (int): Maximum number of tokens in the returned input and label token id lists. + train_on_input (bool): Whether the model is trained on the prompt or not. Default is False. + **load_dataset_kwargs (Dict[str, Any]): additional keyword arguments to pass to `load_dataset`. + + Returns: + ChatDataset: the configured ChatDataset + + Raises: + ValueError: if the conversation format is not supported + """ + if conversation_format == "sharegpt": + convert_to_dialogue = sharegpt_to_llama2_dialogue + else: + raise ValueError(f"Unsupported conversation format: {conversation_format}") + + return ChatDataset( + tokenizer=tokenizer, + source=source, + convert_to_dialogue=convert_to_dialogue, + template=_get_template(template), + max_seq_len=max_seq_len, + train_on_input=train_on_input, + **load_dataset_kwargs, + ) diff --git a/torchtune/datasets/_instruct.py b/torchtune/datasets/_instruct.py index f9e987b80d..1e3e13de55 100644 --- a/torchtune/datasets/_instruct.py +++ b/torchtune/datasets/_instruct.py @@ -4,17 +4,14 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import copy -import re from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple from datasets import load_dataset from torch.utils.data import Dataset -from torchtune.config._errors import InstantiationError -from torchtune.config._utils import _get_component_from_path -from torchtune.data import PromptTemplate -from torchtune.datasets._common import CROSS_ENTROPY_IGNORE_IDX +from torchtune.config._utils import _get_template + +from torchtune.data import PromptTemplate, tokenize_prompt_and_response from torchtune.modules import Tokenizer @@ -82,24 +79,13 @@ def _prepare_sample(self, sample: Mapping[str, Any]) -> Tuple[List[int], List[in if self._column_map and "output" in self._column_map else "output" ) - prompt_with_response = prompt + sample[key_output] - encoded_prompt = self._tokenizer.encode( - text=prompt, add_bos=True, add_eos=False - ) - encoded_prompt_with_response = self._tokenizer.encode( - text=prompt_with_response, add_bos=True, add_eos=True + return tokenize_prompt_and_response( + tokenizer=self._tokenizer, + prompt=prompt, + response=transformed_sample[key_output], + train_on_input=self.train_on_input, ) - labels = copy.deepcopy(encoded_prompt_with_response) - - if not self.train_on_input: - labels[: len(encoded_prompt)] = [CROSS_ENTROPY_IGNORE_IDX] * len( - encoded_prompt - ) - - assert len(encoded_prompt_with_response) == len(labels) - - return encoded_prompt_with_response, labels def instruct_dataset( @@ -137,37 +123,3 @@ def instruct_dataset( train_on_input=train_on_input, **load_dataset_kwargs, ) - - -def _get_template(template: str) -> PromptTemplate: - """ - Get the prompt template class from the template string. - - String should either be the PromptTemplate class name directly, or a raw - string with 1 or more placeholders. If none of these apply, then raise an - error. - - Args: - template (str): class name of template, or string with placeholders - - Returns: - PromptTemplate: the prompt template class or the same verified string - - Raises: - ValueError: if the template is not a PromptTemplate class or a proper - template string - """ - path = "torchtune.data." + template - try: - template_class = _get_component_from_path(path) - return template_class() - except InstantiationError: - # Verify that string can be used as a template, should have variable - # placeholders - pattern = r"\{.+?\}" - if not re.search(pattern, template): - raise ValueError( - f"Invalid template '{template}': " - + "Must be a PromptTemplate class or a string with placeholders." - ) from None - return template diff --git a/torchtune/datasets/_samsum.py b/torchtune/datasets/_samsum.py index 3dc7ea819a..e715940eec 100644 --- a/torchtune/datasets/_samsum.py +++ b/torchtune/datasets/_samsum.py @@ -5,7 +5,7 @@ # LICENSE file in the root directory of this source tree. from torchtune.data import SummarizeTemplate -from torchtune.datasets._instruct import InstructDataset +from torchtune.datasets import InstructDataset from torchtune.modules import Tokenizer diff --git a/torchtune/datasets/_slimorca.py b/torchtune/datasets/_slimorca.py index 15aa37c135..7084579183 100644 --- a/torchtune/datasets/_slimorca.py +++ b/torchtune/datasets/_slimorca.py @@ -4,27 +4,16 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from typing import Dict, List, Tuple +from torchtune.data import Llama2ChatTemplate, sharegpt_to_llama2_dialogue -from datasets import load_dataset -from torch.utils.data import Dataset +from torchtune.datasets._chat import ChatDataset -from torchtune.datasets._common import CROSS_ENTROPY_IGNORE_IDX - -# Not ideal to import this type here but it's needed for the transform function from torchtune.modules import Tokenizer -class _Llama2ChatFormatConstants: - """ - Contains constants that are used in Llama2 Chat Format. - """ - - B_INST, E_INST = "[INST]", "[/INST]" - B_SYS, E_SYS = "<>\n", "\n<>\n\n" - - -class SlimOrcaDataset(Dataset): +def slimorca_dataset( + tokenizer: Tokenizer, max_seq_len: int = 1024, train_on_input: bool = False +) -> ChatDataset: """ PyTorch Representation of the SlimOrca Dataset https://huggingface.co/datasets/Open-Orca/SlimOrca-Dedup @@ -35,7 +24,7 @@ class SlimOrcaDataset(Dataset): The base Llama2 Model doesn't prescribe a particular format. The returned data is a tuple of input token id list and label token id - list. If `max_token_length` keyword argument is provided, the returned + list. If `max_seq_len` keyword argument is provided, the returned input token id list is ensured (by truncation if necessary) to be within that length. @@ -43,92 +32,38 @@ class SlimOrcaDataset(Dataset): Args: tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. - **kwargs: Additional keyword arguments to pass to the SlimOrca Dataset. - - Keyword Arguments: - max_token_length (int): Maximum number of tokens in the returned input and label token id lists. This value needs to be at least 4 though it is generally set to max sequence length accepted by the model. Default is 1024. + max_seq_len (int): Maximum number of tokens in the returned input and label token id lists. + This value needs to be at least 4 though it is generally set to max sequence length accepted by the model. + Default is 1024. + train_on_input (bool): Whether the model is trained on the prompt or not. Default is False. Raises: - ValueError: If `max_token_length` is less than 4. + ValueError: If `max_seq_len` is less than 4. + + Returns: + ChatDataset: dataset configured with SlimOrca source data and LLaMA2 chat template Example: - >>> ds = SlimOrcaDataset(tokenizer=tokenizer, max_token_length=10) + >>> ds = slimorca_dataset(tokenizer=tokenizer, max_seq_len=10) >>> for input, label in ds: >>> print(input) >>> print(label) >>> - >>> Sample Ouput: + >>> Sample Output: >>> [1, 351, 82, 391, 221, 220, 193, 12, 471, ..., 2] >>> [-100, -100, -100, -100, -100, -100, -100, -100, 471, ..., 2] - """ # noqa - - def __init__(self, tokenizer: Tokenizer, **kwargs) -> None: - self._data = load_dataset("Open-Orca/SlimOrca-Dedup", split="train") - self._tokenizer = tokenizer - self._max_token_length = kwargs.get("max_token_length", 1024) - if self._max_token_length < 4: - # Input token needs to have 1 bos, 1 eos, - # and 1 token from prompt, 1 from label - raise ValueError("max_token_length must be at least 4") - - def __len__(self): - return len(self._data) - - def __getitem__(self, index: int) -> Tuple[List[int], List[int]]: - data = self._data[index]["conversations"] - prompt, label = self._generate_prompt_label(data) - return self._generate_tokens(prompt, label) - - def _generate_tokens(self, prompt: str, label: str) -> Tuple[List[int], List[int]]: - """ - Given a prompt string and label string, generate input and label token id lists. - - Tokenizer is used to tokenize both the strings. - The prompt token list is truncated to `max_token_length` - 2 - (so that there is at least one label token, as EOS takes one token). - - The label token list is truncated to `max_token_length` - len(prompt_token_list) - - Finally input token list is the concatenation of prompt and label token lists. - - Label token list is padded with cross entropy ignore idx value to match the length of input token list. - """ - prompt_tokens = self._tokenizer.encode(prompt, add_bos=True, add_eos=False) - # Truncate to max token length - 2 (so that there is at least one label token) - prompt_tokens = prompt_tokens[: self._max_token_length - 2] - - # Calculate space left for label tokens - label_tokens_length = self._max_token_length - len(prompt_tokens) - label_tokens = self._tokenizer.encode(label, add_bos=False, add_eos=True) - - # Truncate label tokens - label_tokens = label_tokens[: label_tokens_length - 1] - if label_tokens[-1] != self._tokenizer.eos_id: - label_tokens.append(self._tokenizer.eos_id) - - input = prompt_tokens + label_tokens - label = [ - CROSS_ENTROPY_IGNORE_IDX for _ in range(len(prompt_tokens)) - ] + label_tokens - return input, label - - def _generate_prompt_label(self, data: List[Dict[str, str]]) -> Tuple[str, str]: - """ - Construct prompt and label strings adhering to Llama2 Chat Format. - This method supports only back-and-forth conversation per sample (as it is sufficient for SlimOrca dataset). - """ - agent_text_dict = {} - # agents can be {system, human, gpt} - for conversation in data: - agent = conversation["from"] - text = conversation["value"] - agent_text_dict[agent] = text - - # Llama2 Chat Format - https://github.com/facebookresearch/llama/blob/main/llama/generation.py#L284 - if "system" in agent_text_dict: - prompt = f"{_Llama2ChatFormatConstants.B_INST} {_Llama2ChatFormatConstants.B_SYS}{agent_text_dict['system']}{_Llama2ChatFormatConstants.E_SYS}{agent_text_dict['human']} {_Llama2ChatFormatConstants.E_INST}" # noqa: B950 - else: - prompt = f"{_Llama2ChatFormatConstants.B_INST} {agent_text_dict['human']} {_Llama2ChatFormatConstants.E_INST}" - - response = f" {agent_text_dict['gpt']} " - return prompt, response + """ + if max_seq_len < 4: + # Input token needs to have 1 bos, 1 eos, + # and 1 token from prompt, 1 from label + raise ValueError("max_seq_len must be at least 4") + + return ChatDataset( + tokenizer=tokenizer, + source="Open-Orca/SlimOrca-Dedup", + convert_to_dialogue=sharegpt_to_llama2_dialogue, + template=Llama2ChatTemplate(), + max_seq_len=max_seq_len, + train_on_input=train_on_input, + split="train", + ) From 97b994d995b6a2949325a353e9db7f1c5112c7dd Mon Sep 17 00:00:00 2001 From: Kartikay Khandelwal <47255723+kartikayk@users.noreply.github.com> Date: Thu, 28 Mar 2024 20:26:42 -0700 Subject: [PATCH 06/76] Add Acknowledgements (#613) --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 1df884e6e0..5b66773157 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,18 @@ TorchTune provides well-tested components with a high-bar on correctness. The li   +## Acknowledgements + +The Llama2 code in this repository is inspired by the original [Llama2 code](https://github.com/meta-llama/llama/blob/main/llama/model.py). We'd also like to give a huge shoutout to some awesome libraries and tools in the ecosystems! + +- EleutherAI's [LM Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness) +- Hugging Face for the [Datasets Repository](https://github.com/huggingface/datasets) +- [gpt-fast](https://github.com/pytorch-labs/gpt-fast) for performant LLM inference techniques which we've adopted OOTB +- [lit-gpt](https://github.com/Lightning-AI/litgpt), [axolotl](https://github.com/OpenAccess-AI-Collective/axolotl) [transformers](https://github.com/huggingface/transformers) and [llama recipes](https://github.com/meta-llama/llama-recipes) for reference implementations and pushing forward the LLM finetuning community +- [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) + +  + ## Contributing We welcome any feature requests, bug reports, or pull requests from the community. See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out. From 08fae5e21a377af1cce8b12f20772f0c19061540 Mon Sep 17 00:00:00 2001 From: Rohan Varma Date: Thu, 28 Mar 2024 22:38:09 -0700 Subject: [PATCH 07/76] Full finetune < 16GB (#527) --- README.md | 5 +- .../configs/llama2/7B_full_single_device.yaml | 1 + .../7B_full_single_device_low_memory.yaml | 76 +++++++++++ recipes/full_finetune_single_device.py | 84 +++++++++--- .../test_full_finetune_single_device.py | 7 +- tests/torchtune/utils/test_optim_utils.py | 91 +++++++++++++ torchtune/utils/__init__.py | 6 + torchtune/utils/memory.py | 125 +++++++++++++++++- 8 files changed, 374 insertions(+), 21 deletions(-) create mode 100644 recipes/configs/llama2/7B_full_single_device_low_memory.yaml create mode 100644 tests/torchtune/utils/test_optim_utils.py diff --git a/README.md b/README.md index 5b66773157..d64ffb3ea5 100644 --- a/README.md +++ b/README.md @@ -55,12 +55,15 @@ experience different peak memory utilization based on changes made in configurat | 1 x RTX 4090 | QLoRA | [qlora_finetune_single_device](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_qlora_single_device.yaml) | Llama-7B | 9.29 GB * | | 2 x RTX 4090 | LoRA | [lora_finetune_distributed](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_lora.yaml) | Llama-7B | 14.17 GB * | | 1 x RTX 4090 | LoRA | [lora_finetune_single_device](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_lora_single_device.yaml) | Llama-7B | 17.18 GB * | -| 1 x A6000 | Full finetune | [full_finetune_single_device](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_full_single_device.yaml) | Llama-7B | 27.15 GB * | +| 1 x A6000 | Full finetune | [full_finetune_single_device](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_full_single_device_low_memory.yaml) | Llama-7B | 15.97 GB * ^ | | 4 x RTX 4090 | Full finetune | [full_finetune_distributed](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_full.yaml) | Llama-7B | 12.01 GB * | NOTE: * indicates an estimated metric based on experiments conducted on A100 GPUs with GPU memory artificially limited using [torch.cuda.set_per_process_memory_fraction API](https://pytorch.org/docs/stable/generated/torch.cuda.set_per_process_memory_fraction.html). Peak memory per GPU is as reported by `torch.cuda.max_memory_reserved()`. Please file an issue if you are not able to reproduce these results when running TorchTune on certain hardware. +NOTE: ^ indicates the required use of third-party dependencies that are not installed with torchtune by default. In particular, for the most memory efficient full finetuning [configuration](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_full_single_device_low_memory.yaml), [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) is required and can be installed via `pip install bitsandbytes`, after which the configuration +can be run successfully. +   --- diff --git a/recipes/configs/llama2/7B_full_single_device.yaml b/recipes/configs/llama2/7B_full_single_device.yaml index e657e5fc7f..25c48bbe58 100644 --- a/recipes/configs/llama2/7B_full_single_device.yaml +++ b/recipes/configs/llama2/7B_full_single_device.yaml @@ -56,6 +56,7 @@ loss: _component_: torch.nn.CrossEntropyLoss max_steps_per_epoch: null gradient_accumulation_steps: 1 +optimizer_in_bwd: False # Training environment diff --git a/recipes/configs/llama2/7B_full_single_device_low_memory.yaml b/recipes/configs/llama2/7B_full_single_device_low_memory.yaml new file mode 100644 index 0000000000..f507aa4ca3 --- /dev/null +++ b/recipes/configs/llama2/7B_full_single_device_low_memory.yaml @@ -0,0 +1,76 @@ +# Config for single device full finetuning in full_finetune_single_device.py +# using a Llama2 7B model +# +# This config assumes that you've run the following command before launching +# this run: +# tune download --repo-id meta-llama/Llama-2-7b \ +# --hf-token \ +# --output-dir /tmp/llama2 +# +# To launch on a single device, run the following command from root: +# tune --nnodes 1 --nproc_per_node 1 full_finetune_single_device \ +# --config llama2/7B_full_single_device_low_memory \ +# +# You can add specific overrides through the command line. For example +# to override the checkpointer directory while launching training +# you can run: +# tune --nnodes 1 --nproc_per_node 1 full_finetune_single_device \ +# --config llama2/7B_full_single_device_low_memory \ +# checkpointer.checkpoint_dir= +# +# This config works only for training on single device. + + +# Tokenizer +tokenizer: + _component_: torchtune.models.llama2.llama2_tokenizer + path: /tmp/llama2/tokenizer.model + +# Dataset +dataset: + _component_: torchtune.datasets.alpaca_dataset + train_on_input: True +seed: null +shuffle: True + +# Model Arguments +model: + _component_: torchtune.models.llama2.llama2_7b + +checkpointer: + _component_: torchtune.utils.FullModelMetaCheckpointer + checkpoint_dir: /tmp/llama2 + checkpoint_files: [consolidated.00.pth] + recipe_checkpoint: null + output_dir: /tmp/llama2 + model_type: LLAMA2 +resume_from_checkpoint: False + +# Fine-tuning arguments +batch_size: 2 +epochs: 1 +optimizer: + _component_: bitsandbytes.optim.PagedAdamW + lr: 2e-5 +optimizer_in_bwd: True +loss: + _component_: torch.nn.CrossEntropyLoss +max_steps_per_epoch: null +gradient_accumulation_steps: 1 + + +# Training environment +device: cuda + +# Memory management +enable_activation_checkpointing: True + +# Reduced precision +dtype: bf16 + +# Logging +metric_logger: + _component_: torchtune.utils.metric_logging.DiskLogger + log_dir: ${output_dir} +output_dir: /tmp/alpaca-llama2-finetune +log_every_n_steps: null diff --git a/recipes/full_finetune_single_device.py b/recipes/full_finetune_single_device.py index 3d1f7c8f17..3e6dd2fb1f 100644 --- a/recipes/full_finetune_single_device.py +++ b/recipes/full_finetune_single_device.py @@ -44,6 +44,8 @@ class FullFinetuneRecipeSingleDevice(FTRecipeInterface): hood. Setting up the env variables is handled by TorchRun. - Training happens on CUDA (CPU training is not supported) - Checkpoints are ONLY saved at epoch boundaries. Mid-epoch checkpointing is NOT supported. + - User can only use ONE of gradient accumulation or optimizer in backward. These features + currently do not work together. - Datasets are Map-style and data fits in memory (not streamed). The following configs can be used to run this recipe: @@ -55,8 +57,9 @@ class FullFinetuneRecipeSingleDevice(FTRecipeInterface): cfg (DictConfig): OmegaConf object parsed from yaml file Raises: - ValueError: If ``dtype`` is set to fp16. + RuntimeError: If ``dtype`` is set to fp16. RuntimeError: If ``dtype`` is set to bf16 and the hardware does not support bf16. + RuntimeError: If ``gradient_accumulation_steps > 1`` and ``optimizer_in_bwd`` is `True`. """ def __init__(self, cfg: DictConfig) -> None: @@ -65,7 +68,7 @@ def __init__(self, cfg: DictConfig) -> None: # Disable for fp16, as we haven't validated "full" fp16 with this recipe, nor # enabled necessary features such as gradient scaling. if self._dtype == torch.float16: - raise ValueError( + raise RuntimeError( "full fp16 training is not supported with this recipe. Please use bf16 or fp32 instead." ) @@ -84,7 +87,14 @@ def __init__(self, cfg: DictConfig) -> None: # Training cfg self._resume_from_checkpoint = cfg.resume_from_checkpoint self._gradient_accumulation_steps = cfg.gradient_accumulation_steps - + self._optimizer_in_bwd = cfg.optimizer_in_bwd + # TODO: find a better place / way to perform validation of args that don't yet + # compose with each other. + if self._gradient_accumulation_steps > 1 and self._optimizer_in_bwd: + raise RuntimeError( + "Gradient accumulation is not supported with optimizer in bwd." + "Please set gradient_accumulation_steps=1, or optimizer_in_bwd=False." + ) # These are public properties which are updated by the checkpoint loader # when ``resume_from_checkpoint`` is `True` or validated in tests self.seed = utils.set_seed(seed=cfg.seed) @@ -158,6 +168,7 @@ def setup(self, cfg: DictConfig) -> None: # checkpoint. Transforming the opt state dict is handled by this method self._optimizer = self._setup_optimizer( cfg_optimizer=cfg.optimizer, + optimizer_in_bwd=cfg.optimizer_in_bwd, opt_state_dict=( ckpt_dict[utils.OPT_KEY] if self._resume_from_checkpoint else None ), @@ -221,18 +232,46 @@ def _setup_model( return model def _setup_optimizer( - self, cfg_optimizer: DictConfig, opt_state_dict: Optional[Dict[str, Any]] = None - ) -> Optimizer: + self, + cfg_optimizer: DictConfig, + optimizer_in_bwd: bool = False, + opt_state_dict: Optional[Dict[str, Any]] = None, + ) -> Optional[Optimizer]: """ Set up the optimizer. This method also handles loading the optimizer state_dict, if specified. """ - optimizer = config.instantiate(cfg_optimizer, self._model.parameters()) - - if opt_state_dict: - optimizer.load_state_dict(opt_state_dict) - - log.info("Optimizer is initialized.") - return optimizer + if optimizer_in_bwd: + # Maintain a dict of optims for every parameter. + optim_dict = { + p: config.instantiate(cfg_optimizer, [p]) + for p in self._model.parameters() + } + # Register optimizer step hooks on the model to run optimizer in backward. + utils.register_optim_in_bwd_hooks(model=self._model, optim_dict=optim_dict) + # Create a wrapper for checkpoint save/load of optimizer states when running in backward. + self._optim_ckpt_wrapper = utils.create_optim_in_bwd_wrapper( + model=self._model, optim_dict=optim_dict + ) + # Load optimizer states. If optimizer states are being restored in an optimizer in backward + # run, these need to have been saved with the same setting. Cannot restore from runs that did not + # use optimizer in backward. + if opt_state_dict is not None: + try: + self._optim_ckpt_wrapper.load_state_dict(opt_state_dict) + except BaseException as e: + raise RuntimeError( + "Failed loading in-backward optimizer checkpoints." + "Please make sure run being restored from was using in-backward optimizer." + ) from e + log.info("In-backward optimizers are set up.") + return None + else: + optimizer = config.instantiate(cfg_optimizer, self._model.parameters()) + + if opt_state_dict: + optimizer.load_state_dict(opt_state_dict) + log.info("Optimizer is initialized.") + return optimizer def _setup_data( self, @@ -281,13 +320,16 @@ def save_checkpoint(self, epoch: int) -> None: if epoch + 1 < self.total_epochs: ckpt_dict.update( { - utils.OPT_KEY: self._optimizer.state_dict(), utils.SEED_KEY: self.seed, utils.EPOCHS_KEY: self.epochs_run, utils.TOTAL_EPOCHS_KEY: self.total_epochs, utils.MAX_STEPS_KEY: self.max_steps_per_epoch, } ) + if not self._optimizer_in_bwd: + ckpt_dict[utils.OPT_KEY] = self._optimizer.state_dict() + else: + ckpt_dict[utils.OPT_KEY] = self._optim_ckpt_wrapper.state_dict() self._checkpointer.save_checkpoint( ckpt_dict, epoch=epoch, @@ -311,8 +353,8 @@ def train(self) -> None: ``max_steps_per_epoch``. """ # zero out the gradients before starting training - self._optimizer.zero_grad() - + if not self._optimizer_in_bwd: + self._optimizer.zero_grad() # self.epochs_run should be non-zero when we're resuming from a checkpoint for curr_epoch in range(self.epochs_run, self.total_epochs): # Update the sampler to ensure data is correctly shuffled across epochs @@ -344,7 +386,13 @@ def train(self) -> None: self._metric_logger.log_dict( { "loss": loss.item(), - "lr": self._optimizer.param_groups[0]["lr"], + # NOTE: for optim in backward, this assumes all optimizers have the same LR. This is currently + # true since we don't expose the ability to configure this yet. + "lr": ( + self._optim_ckpt_wrapper.get_optim_key("lr") + if self._optimizer_in_bwd + else self._optimizer.param_groups[0]["lr"] + ), "gpu_resources": torch.cuda.memory_allocated(), }, step=self.total_training_steps, @@ -352,12 +400,14 @@ def train(self) -> None: loss = loss / self._gradient_accumulation_steps loss.backward() - if self._should_update_weights(idx): + if not self._optimizer_in_bwd and self._should_update_weights(idx): self._optimizer.step() self._optimizer.zero_grad(set_to_none=True) # Update the number of steps when the weights are updated self.total_training_steps += 1 + elif self._optimizer_in_bwd: + self.total_training_steps += 1 # Log peak memory for iteration if self.total_training_steps % self._log_peak_memory_every_n_steps == 0: diff --git a/tests/recipes/test_full_finetune_single_device.py b/tests/recipes/test_full_finetune_single_device.py index 1d58a140e3..a97ce71f82 100644 --- a/tests/recipes/test_full_finetune_single_device.py +++ b/tests/recipes/test_full_finetune_single_device.py @@ -51,7 +51,10 @@ def _fetch_expected_loss_values(self): return [10.5074, 10.5563, 10.5152, 10.4851] @pytest.mark.integration_test - def test_loss(self, tmpdir, monkeypatch): + @pytest.mark.parametrize( + "config", ["full_single_device_low_memory", "full_single_device"] + ) + def test_loss(self, config, tmpdir, monkeypatch): ckpt = "small_test_ckpt_meta" ckpt_path = Path(CKPT_MODEL_PATHS[ckpt]) ckpt_dir = ckpt_path.parent @@ -59,7 +62,7 @@ def test_loss(self, tmpdir, monkeypatch): cmd = f""" tune full_finetune_single_device - --config llama2/7B_full_single_device \ + --config llama2/7B_{config} \ output_dir={tmpdir} \ checkpointer._component_=torchtune.utils.FullModelMetaCheckpointer checkpointer.checkpoint_dir='{ckpt_dir}' \ diff --git a/tests/torchtune/utils/test_optim_utils.py b/tests/torchtune/utils/test_optim_utils.py new file mode 100644 index 0000000000..0d491acc80 --- /dev/null +++ b/tests/torchtune/utils/test_optim_utils.py @@ -0,0 +1,91 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import pytest +import torch +from torchtune.utils import create_optim_in_bwd_wrapper, register_optim_in_bwd_hooks + + +def _run_dummy_step(model, wrapper): + with torch.no_grad(): + for p in model.parameters(): + p.grad = torch.rand_like(p) + for v in wrapper.optim_map.values(): + v.step() + v.zero_grad() + + +def _validate_dicts(d1, d2): + if len(d1) != len(d2): + return False + for k, v in d1.items(): + if k not in d2: + return False + if isinstance(v, dict): + return _validate_dicts(v, d2[k]) + else: + if isinstance(v, torch.Tensor): + if not torch.allclose(v, d2[k]): + return False + elif v != d2[k]: + return False + return True + + +@pytest.fixture +def model(): + return torch.nn.Linear(10, 1) + + +@pytest.fixture +def optim_dict(model): + return {p: torch.optim.AdamW([p], lr=0.01) for p in model.parameters()} + + +@pytest.fixture +def wrapper(model, optim_dict): + return create_optim_in_bwd_wrapper(model, optim_dict) + + +class TestOptimInBackward: + def test_state_dict_save_load(self, model, wrapper): + # Run a dummy step to create optimizer states + _run_dummy_step(model, wrapper) + + sd = wrapper.state_dict() + new_optim_dict = create_optim_in_bwd_wrapper( + model, {p: torch.optim.AdamW([p], lr=0.01) for p in model.parameters()} + ) + assert not _validate_dicts(sd, new_optim_dict.state_dict()) + new_optim_dict.load_state_dict(sd) + assert _validate_dicts(sd, new_optim_dict.state_dict()) + + def test_missing_unexpected_param_load_raises(self, model, wrapper): + # Run a dummy step to create optimizer states + _run_dummy_step(model, wrapper) + sd = wrapper.state_dict() + new_optim_dict = create_optim_in_bwd_wrapper( + model, {p: torch.optim.AdamW([p], lr=0.01) for p in model.parameters()} + ) + with pytest.raises(RuntimeError, match="Expected to load optimizer state"): + sd.pop(next(iter(sd.keys()))) + new_optim_dict.load_state_dict(sd) + + sd = wrapper.state_dict() + sd["new_key"] = 1234 + with pytest.raises(RuntimeError, match="unexpected param"): + new_optim_dict.load_state_dict(sd) + + +class TestRegisterOptimHooks: + def test_register_optim_in_bwd_hooks(self, model, optim_dict): + register_optim_in_bwd_hooks(model, optim_dict) + # Ensure backward() updates the parameters and sets grads to None + orig_params = [p.clone().detach() for p in model.parameters()] + model(torch.rand(2, 10)).sum().backward() + for p, orig_p in zip(model.parameters(), orig_params): + assert not p.grad + assert not torch.allclose(p, orig_p) diff --git a/torchtune/utils/__init__.py b/torchtune/utils/__init__.py index 0e92efdc42..15e749e9f1 100644 --- a/torchtune/utils/__init__.py +++ b/torchtune/utils/__init__.py @@ -37,7 +37,10 @@ from .logging import get_logger from .memory import ( # noqa cleanup_before_training, + create_optim_in_bwd_wrapper, memory_stats_log, + OptimizerInBackwardWrapper, + register_optim_in_bwd_hooks, set_activation_checkpointing, ) from .precision import ( @@ -73,4 +76,7 @@ "validate_expected_param_dtype", "TuneArgumentParser", "CheckpointableDataLoader", + "OptimizerInBackwardWrapper", + "create_optim_in_bwd_wrapper", + "register_optim_in_bwd_hooks", ] diff --git a/torchtune/utils/memory.py b/torchtune/utils/memory.py index 0067b783ee..5713c73e98 100644 --- a/torchtune/utils/memory.py +++ b/torchtune/utils/memory.py @@ -6,7 +6,7 @@ import gc -from typing import Optional, Set +from typing import Any, Dict, Optional, Set import torch @@ -37,6 +37,129 @@ def cleanup_before_training() -> None: torch.cuda.reset_peak_memory_stats() +class OptimizerInBackwardWrapper: + """ + A bare-bones class meant for checkpoint save and load for optimizers running + in backward. Usage is limited to the following: + + optim_dict = { + p: config.instantiate(cfg_optimizer, [p]) + for p in self._model.parameters() + } + # Save checkpoint + ckpt = OptimizerInBackwardWrapper(optim_dict).state_dict() + torch.save("/tmp/optim_ckpt", ckpt) + # Load checkpoint + placeholder_optim_dict = { + p: config.instantiate(cfg_optimizer, [p]) + for p in self._model.parameters() + } + wrapper = OptimInBackwardWrapper(placeholder_optim_dict) + # load_state_dict expects a dict produced by this class's + # state_dict method. + wrapper.load_state_dict(torch.load("/tmp/optim_ckpt")) + # placeholder_optim_dict now has updated optimizer states. + + NOTE: This wrapper is only meant to be used for single-device use cases. + Distributed use cases such as FSDP, which require specialized + optimizer state checkpointing, are not supported. + + Args: + optim_map (Dict[str, torch.optim.Optimizer]): Mapping from parameter names to optimizers. + + """ + + def __init__(self, optim_map: Dict[str, torch.optim.Optimizer]): + self.optim_map = optim_map + + def state_dict(self) -> Dict[str, Any]: + """ + Returns a state dict mapping parameter names to optimizer states. This + state_dict is only loadable by this same class. + + Returns: + Dict[str, Any]: state dict mapping parameter names to optimizer states. + """ + return {p: opt.state_dict() for p, opt in self.optim_map.items()} + + def load_state_dict(self, optim_ckpt_map: Dict[str, Any]): + """ + Load optimizer states from a state dict produced by this class's + state_dict method. + + Args: + optim_ckpt_map (Dict[str, Any]): state dict mapping parameter names to optimizer states. + + Raises: + RuntimeError: If the optimizer state dict does not contain all the expected parameters. + """ + params_covered = set() + for param_name in optim_ckpt_map.keys(): + if param_name not in self.optim_map: + raise RuntimeError( + f"Trying to load optimizer state for unexpected param {param_name}" + ) + self.optim_map[param_name].load_state_dict(optim_ckpt_map[param_name]) + params_covered.add(param_name) + # Ensure all params have been loaded into, report missing params + missing_params = set(self.optim_map.keys()) - params_covered + if missing_params: + raise RuntimeError( + f"Expected to load optimizer state for params {missing_params}!" + ) + + def get_optim_key(self, key: str) -> Any: + """ + Returns value of key from an arbitrary optimizer running in backward. Note that + this assumes all optimizer in backwards have the same value for the key, i.e., + are initialized with the same hyperparameters. + """ + return list(self.optim_map.values())[0].param_groups[0][key] + + +def create_optim_in_bwd_wrapper( + model: torch.nn.Module, optim_dict: Dict[torch.nn.Parameter, torch.optim.Optimizer] +) -> OptimizerInBackwardWrapper: + """ + Create a wrapper for optimizer step running in backward. + + Args: + model (torch.nn.Module): Model that contains parameters that are being optimized. For now, + it is assumed that all parameters being optimized belong to a single top-level model. + `named_parameters` attribute of `model` will be accessed to look up parameter names for + parameters being optimized. + optim_dict (Dict[torch.nn.Parameter, torch.optim.Optimizer]): Mapping from + parameters to optimizers. + + Returns: + ``OptimizerInBackwardWrapper``: Wrapper for optimizer states running in backward. + """ + return OptimizerInBackwardWrapper( + {n: optim_dict[p] for n, p in model.named_parameters()} + ) + + +def register_optim_in_bwd_hooks( + model: torch.nn.Module, optim_dict: Dict[torch.nn.Parameter, torch.optim.Optimizer] +) -> None: + """ + Register hooks for optimizer step running in backward. + + Args: + model (torch.nn.Module): Model whose parameters will be optimized. Note that currently + hooks for ALL parameters in the model will be registered. + optim_dict (Dict[torch.nn.Parameter, torch.optim.Optimizer]): Mapping from + parameters to optimizers. + """ + + def optim_step(param) -> None: + optim_dict[param].step() + optim_dict[param].zero_grad() + + for p in model.parameters(): + p.register_post_accumulate_grad_hook(optim_step) + + def memory_stats_log( prefix: str, device: torch.device, reset_stats: bool = True ) -> None: From ae600b29488911bba90f574d1a66e645b5c813ef Mon Sep 17 00:00:00 2001 From: Rohan Varma Date: Fri, 29 Mar 2024 03:06:42 -0700 Subject: [PATCH 08/76] Small fix to README for full finetune (#615) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d64ffb3ea5..ddee064112 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,13 @@ experience different peak memory utilization based on changes made in configurat | 1 x RTX 4090 | QLoRA | [qlora_finetune_single_device](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_qlora_single_device.yaml) | Llama-7B | 9.29 GB * | | 2 x RTX 4090 | LoRA | [lora_finetune_distributed](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_lora.yaml) | Llama-7B | 14.17 GB * | | 1 x RTX 4090 | LoRA | [lora_finetune_single_device](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_lora_single_device.yaml) | Llama-7B | 17.18 GB * | -| 1 x A6000 | Full finetune | [full_finetune_single_device](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_full_single_device_low_memory.yaml) | Llama-7B | 15.97 GB * ^ | +| 1 x RTX 4090 | Full finetune | [full_finetune_single_device](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_full_single_device_low_memory.yaml) | Llama-7B | 15.97 GB * ^ | | 4 x RTX 4090 | Full finetune | [full_finetune_distributed](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_full.yaml) | Llama-7B | 12.01 GB * | NOTE: * indicates an estimated metric based on experiments conducted on A100 GPUs with GPU memory artificially limited using [torch.cuda.set_per_process_memory_fraction API](https://pytorch.org/docs/stable/generated/torch.cuda.set_per_process_memory_fraction.html). Peak memory per GPU is as reported by `torch.cuda.max_memory_reserved()`. Please file an issue if you are not able to reproduce these results when running TorchTune on certain hardware. -NOTE: ^ indicates the required use of third-party dependencies that are not installed with torchtune by default. In particular, for the most memory efficient full finetuning [configuration](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_full_single_device_low_memory.yaml), [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) is required and can be installed via `pip install bitsandbytes`, after which the configuration +NOTE: ^ indicates the required use of third-party dependencies that are not installed with ``torchtune`` by default. In particular, for the most memory efficient full finetuning [configuration](https://github.com/pytorch/torchtune/blob/main/recipes/configs/llama2/7B_full_single_device_low_memory.yaml), [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) is required and can be installed via `pip install bitsandbytes`, after which the configuration can be run successfully.   From 290beb5992a6902c437b18df4d491c9b430a904a Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Fri, 29 Mar 2024 16:06:41 -0400 Subject: [PATCH 09/76] Add `tune run` and refactor CLI (#586) --- README.md | 71 ++++--- docs/source/api_ref_utilities.rst | 2 +- docs/source/examples/recipe_deepdive.rst | 2 +- recipes/eleuther_eval.py | 3 +- tests/recipes/test_alpaca_generate.py | 12 +- tests/recipes/test_eleuther_eval.py | 28 +-- .../recipes/test_full_finetune_distributed.py | 9 +- .../test_full_finetune_single_device.py | 24 +-- .../recipes/test_lora_finetune_distributed.py | 10 +- .../test_lora_finetune_single_device.py | 22 +-- .../torchtune/_cli/test_convert_checkpoint.py | 158 --------------- tests/torchtune/_cli/test_cp.py | 62 ++++-- tests/torchtune/_cli/test_download.py | 59 ++++-- tests/torchtune/_cli/test_ls.py | 23 +-- tests/torchtune/_cli/test_run.py | 75 ++++++++ tests/torchtune/_cli/test_tune.py | 31 ++- tests/torchtune/_cli/test_validate.py | 37 ++-- tests/torchtune/config/test_config_utils.py | 4 +- tests/torchtune/config/test_parse.py | 2 +- tests/torchtune/utils/test_argparse.py | 4 +- torchtune/__init__.py | 33 ---- torchtune/_cli/__init__.py | 7 - torchtune/_cli/convert_checkpoint.py | 121 ------------ torchtune/_cli/cp.py | 175 +++++++++-------- torchtune/_cli/download.py | 149 +++++++++------ torchtune/_cli/ls.py | 106 ++++++----- torchtune/_cli/run.py | 180 ++++++++++++++++++ torchtune/_cli/subcommand.py | 17 ++ torchtune/_cli/tune.py | 146 +++----------- torchtune/_cli/validate.py | 77 +++++--- torchtune/_recipe_registry.py | 95 +++++++++ torchtune/config/_parse.py | 4 +- torchtune/utils/__init__.py | 4 +- torchtune/utils/argparse.py | 4 +- 34 files changed, 911 insertions(+), 845 deletions(-) delete mode 100644 tests/torchtune/_cli/test_convert_checkpoint.py create mode 100644 tests/torchtune/_cli/test_run.py delete mode 100644 torchtune/_cli/convert_checkpoint.py create mode 100644 torchtune/_cli/run.py create mode 100644 torchtune/_cli/subcommand.py create mode 100644 torchtune/_recipe_registry.py diff --git a/README.md b/README.md index ddee064112..2aaee6bcc8 100644 --- a/README.md +++ b/README.md @@ -83,14 +83,20 @@ pip install -e . To confirm that the package is installed correctly, you can run the following command: ``` -tune +tune --help ``` And should see the following output: ``` -usage: tune [options] [recipe_args] -tune: error: the following arguments are required: recipe, recipe_args +usage: tune [-h] {ls,cp,download,run,validate} ... + +Welcome to the TorchTune CLI! + +options: + -h, --help show this help message and exit + +... ```   @@ -109,9 +115,9 @@ Follow the instructions on the official [`meta-llama`](https://huggingface.co/me You can find your token at https://huggingface.co/settings/tokens ``` -tune download --repo-id meta-llama/Llama-2-7b \ +tune download meta-llama/Llama-2-7b \ +--output-dir /tmp/llama2 \ --hf-token \ ---output-dir /tmp/llama2 ``` Note: While the ``tune download`` command allows you to download *any* model from the hub, there's no guarantee that the model can be finetuned with TorchTune. Currently supported models can be found [here](#introduction) @@ -120,62 +126,51 @@ Note: While the ``tune download`` command allows you to download *any* model fro #### Running recipes -TorchTune contains recipes for: +TorchTune contains built-in recipes for: - Full finetuning on [single device](https://github.com/pytorch/torchtune/blob/main/recipes/full_finetune_single_device.py) and on [multiple devices with FSDP](https://github.com/pytorch/torchtune/blob/main/recipes/full_finetune_distributed.py) - LoRA finetuning on [single device](https://github.com/pytorch/torchtune/blob/main/recipes/lora_finetune_single_device.py) and on [multiple devices with FSDP](https://github.com/pytorch/torchtune/blob/main/recipes/lora_finetune_distributed.py). - QLoRA finetuning on [single device](https://github.com/pytorch/torchtune/blob/main/recipes/lora_finetune_single_device.py), with a QLoRA specific [configuration](https://github.com/pytorch/torchtune/blob/main/recipes/configs/7B_qlora_single_device.yaml) -To run a full finetune on two devices on the Alpaca dataset using the Llama2 7B model and FSDP: - +To run a LoRA finetune on a single device using the [Alpaca Dataset](https://huggingface.co/datasets/tatsu-lab/alpaca): ``` -tune --nnodes 1 --nproc_per_node 2 \ -full_finetune_distributed \ ---config llama2/7B_full +tune run lora_finetune_single_device --config llama2/7B_lora_single_device ``` -The argument passed to `--nproc_per_node` can be varied depending on how many GPUs you have. A full finetune can be memory-intensive, so make sure you are running on enough devices. See [this table](https://github.com/pytorch/torchtune/blob/main/README.md#finetuning-resource-requirements) for resource requirements on common hardware setups. +TorchTune integrates with [`torchrun`](https://pytorch.org/docs/stable/elastic/run.html) for easily running distributed training. See below for an example of running a Llama2 7B full-finetune on two GPUs. -Similarly, you can finetune with LoRA on the Alpaca dataset using the Llama2 13B model on two devices via the following. +> Make sure to place any torchrun commands **before** the recipe specification b/c any other CLI args will +overwrite the config, not affect distributed training. ``` -tune --nnodes 1 --nproc_per_node 2 \ -lora_finetune_distributed \ ---config llama2/13B_lora +tune run --nproc_per_node 2 full_finetune_distributed --config llama2/7B_full_distributed ``` -Again, the argument to `--nproc_per_node` can be varied subject to memory constraints of your device(s). - -An example to run QLoRA on a single device can be achieved with the following: - +You can easily overwrite some config properties as follows, but you can also easily copy a built-in config and +modify it following the instructions in the [next section](#copy-and-edit-a-custom-recipe-or-config). ``` -tune lora_finetune_single_device --config llama2/7B_qlora_single_device +tune run lora_finetune_single_device --config llama2/7B_lora_single_device batch_size=8 ```   -#### Copy and edit a custom recipe +#### Copy and edit a custom recipe or config -To copy a recipe to customize it yourself and then run -``` -tune cp full_finetune_distributed.py my_recipe/full_finetune_distributed.py -tune cp llama2/7B_full.yaml my_recipe/7B_full.yaml -tune my_recipe/full_finetune_distributed.py --config my_recipe/7B_full.yaml ``` +tune cp full_finetune_distributed my_custom_finetune_recipe.py +Copied to ./my_custom_finetune_recipe.py -  - -#### Command Utilities - -``tune`` provides functionality for launching torchtune recipes as well as local -recipes. Aside from torchtune recipe utilties, it integrates with ``torch.distributed.run`` -to support distributed job launching by default. ``tune`` offers everyting that ``torchrun`` -does with the following additional functionalities: +tune cp llama2/7B_full . +Copied to ./7B_full.yaml +``` -1. ``tune `` will launch a torchrun job +Then, you can run your custom recipe by directing the `tune run` command to your local files: +``` +tune run my_custom_finetune_recipe.py --config 7B_full.yaml +``` -2. ```` and recipe arg ```` can both be passed in as names instead of paths if they're included in torchtune +  -3. ``tune ls`` and ``tune cp`` commands provide utilities for listing and copying packaged recipes and configs +Check out `tune --help` for all possible CLI commands and options.   diff --git a/docs/source/api_ref_utilities.rst b/docs/source/api_ref_utilities.rst index 6cc9065b46..0194b05cc5 100644 --- a/docs/source/api_ref_utilities.rst +++ b/docs/source/api_ref_utilities.rst @@ -85,7 +85,7 @@ Miscellaneous :toctree: generated/ :nosignatures: - argparse.TuneArgumentParser + argparse.TuneRecipeArgumentParser logging.get_logger get_device seed.set_seed diff --git a/docs/source/examples/recipe_deepdive.rst b/docs/source/examples/recipe_deepdive.rst index f1a0bcc376..887514079b 100644 --- a/docs/source/examples/recipe_deepdive.rst +++ b/docs/source/examples/recipe_deepdive.rst @@ -82,7 +82,7 @@ An example script looks something like this: .. code-block:: python # Launch using TuneCLI which uses TorchRun under the hood - parser = utils.TuneArgumentParser(...) + parser = utils.TuneRecipeArgumentParser(...) # Parse and validate the params args, _ = parser.parse_known_args() diff --git a/recipes/eleuther_eval.py b/recipes/eleuther_eval.py index 1d6705c411..a08f5017a7 100644 --- a/recipes/eleuther_eval.py +++ b/recipes/eleuther_eval.py @@ -135,8 +135,7 @@ def setup(self) -> None: self._limit = self._cfg.limit self._tasks = list(self._cfg.tasks) - seed = utils.set_seed(seed=self._cfg.seed) - logger.info(f"Random seed set to {seed}.") + utils.set_seed(seed=self._cfg.seed) ckpt_dict = self.load_checkpoint(self._cfg.checkpointer) self._model = self._setup_model( diff --git a/tests/recipes/test_alpaca_generate.py b/tests/recipes/test_alpaca_generate.py index e772403551..7a09751fac 100644 --- a/tests/recipes/test_alpaca_generate.py +++ b/tests/recipes/test_alpaca_generate.py @@ -4,23 +4,15 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import logging - import runpy import sys import pytest from tests.common import TUNE_PATH -from tests.recipes.common import RECIPE_TESTS_DIR from tests.recipes.utils import llama2_test_config from tests.test_utils import CKPT_MODEL_PATHS -_CONFIG_PATH = RECIPE_TESTS_DIR / "alpaca_generate_test_config.yaml" - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - class TestAlpacaGenerateRecipe: @pytest.mark.integration_test @@ -28,7 +20,7 @@ def test_alpaca_generate(self, tmpdir, monkeypatch): ckpt = "small_test_ckpt_tune" model_checkpoint = CKPT_MODEL_PATHS[ckpt] cmd = f""" - tune alpaca_generate + tune run alpaca_generate \ --config alpaca_generate \ model_checkpoint={model_checkpoint} \ tokenizer.path=/tmp/test-artifacts/tokenizer.model \ @@ -39,5 +31,5 @@ def test_alpaca_generate(self, tmpdir, monkeypatch): cmd += model_config monkeypatch.setattr(sys, "argv", cmd) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") diff --git a/tests/recipes/test_eleuther_eval.py b/tests/recipes/test_eleuther_eval.py index ccc76593ad..a6aa80baf7 100644 --- a/tests/recipes/test_eleuther_eval.py +++ b/tests/recipes/test_eleuther_eval.py @@ -5,8 +5,9 @@ # LICENSE file in the root directory of this source tree. import builtins +import math +import re import runpy - import sys from pathlib import Path @@ -25,10 +26,10 @@ def test_torchune_checkpoint_eval_results(self, caplog, monkeypatch, tmpdir): ckpt_dir = ckpt_path.parent cmd = f""" - tune eleuther_eval \ + tune run eleuther_eval \ --config eleuther_eval \ output_dir={tmpdir} \ - checkpointer=torchtune.utils.FullModelTorchTuneCheckpointer + checkpointer=torchtune.utils.FullModelTorchTuneCheckpointer \ checkpointer.checkpoint_dir='{ckpt_dir}' \ checkpointer.checkpoint_files=[{ckpt_path}]\ checkpointer.output_dir={tmpdir} \ @@ -43,11 +44,14 @@ def test_torchune_checkpoint_eval_results(self, caplog, monkeypatch, tmpdir): cmd = cmd + model_config monkeypatch.setattr(sys, "argv", cmd) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") - log_out = caplog.messages[-1] - assert "'acc,none': 0.3" in log_out + err_log = caplog.messages[-1] + log_search_results = re.search(r"'acc,none': (\d+\.\d+)", err_log) + assert log_search_results is not None + acc_result = float(log_search_results.group(1)) + assert math.isclose(acc_result, 0.3, abs_tol=0.05) @pytest.fixture def hide_available_pkg(self, monkeypatch): @@ -60,18 +64,18 @@ def mocked_import(name, *args, **kwargs): monkeypatch.setattr(builtins, "__import__", mocked_import) - @pytest.mark.usefixtures("hide_available_pkg") @pytest.mark.integration_test + @pytest.mark.usefixtures("hide_available_pkg") def test_eval_recipe_errors_without_lm_eval(self, caplog, monkeypatch, tmpdir): ckpt = "small_test_ckpt_tune" ckpt_path = Path(CKPT_MODEL_PATHS[ckpt]) ckpt_dir = ckpt_path.parent cmd = f""" - tune eleuther_eval \ + tune run eleuther_eval \ --config eleuther_eval \ output_dir={tmpdir} \ - checkpointer=torchtune.utils.FullModelTorchTuneCheckpointer + checkpointer=torchtune.utils.FullModelTorchTuneCheckpointer \ checkpointer.checkpoint_dir='{ckpt_dir}' \ checkpointer.checkpoint_files=[{ckpt_path}]\ checkpointer.output_dir={tmpdir} \ @@ -83,8 +87,8 @@ def test_eval_recipe_errors_without_lm_eval(self, caplog, monkeypatch, tmpdir): """.split() monkeypatch.setattr(sys, "argv", cmd) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match="1"): runpy.run_path(TUNE_PATH, run_name="__main__") - log_out = caplog.messages[0] - assert "Recipe requires EleutherAI Eval Harness v0.4" in log_out + err_log = caplog.messages[-1] + assert "Recipe requires EleutherAI Eval Harness v0.4" in err_log diff --git a/tests/recipes/test_full_finetune_distributed.py b/tests/recipes/test_full_finetune_distributed.py index 3afd37c1c1..5c9c80025a 100644 --- a/tests/recipes/test_full_finetune_distributed.py +++ b/tests/recipes/test_full_finetune_distributed.py @@ -4,8 +4,6 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import logging - import runpy import sys @@ -23,9 +21,6 @@ gpu_test, ) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - class TestFullFinetuneDistributedRecipe: def _get_test_config_overrides(self): @@ -58,10 +53,10 @@ def test_loss(self, tmpdir, monkeypatch): write_hf_ckpt_config(ckpt_dir) cmd = f""" - tune --nnodes 1 --nproc_per_node 2 full_finetune_distributed + tune run --nnodes 1 --nproc_per_node 2 full_finetune_distributed \ --config llama2/7B_full \ output_dir={tmpdir} \ - checkpointer._component_=torchtune.utils.FullModelHFCheckpointer + checkpointer._component_=torchtune.utils.FullModelHFCheckpointer \ checkpointer.checkpoint_dir='{ckpt_dir}' \ checkpointer.checkpoint_files=[{ckpt_path}]\ checkpointer.output_dir={tmpdir} \ diff --git a/tests/recipes/test_full_finetune_single_device.py b/tests/recipes/test_full_finetune_single_device.py index a97ce71f82..55a95d2b55 100644 --- a/tests/recipes/test_full_finetune_single_device.py +++ b/tests/recipes/test_full_finetune_single_device.py @@ -4,7 +4,6 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import logging import os import runpy @@ -26,9 +25,6 @@ get_loss_values_from_metric_logger, ) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - class TestFullFinetuneSingleDeviceRecipe: def _get_test_config_overrides(self): @@ -61,7 +57,7 @@ def test_loss(self, config, tmpdir, monkeypatch): log_file = gen_log_file_name(tmpdir) cmd = f""" - tune full_finetune_single_device + tune run full_finetune_single_device \ --config llama2/7B_{config} \ output_dir={tmpdir} \ checkpointer._component_=torchtune.utils.FullModelMetaCheckpointer @@ -76,7 +72,7 @@ def test_loss(self, config, tmpdir, monkeypatch): cmd = cmd + self._get_test_config_overrides() + model_config monkeypatch.setattr(sys, "argv", cmd) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") loss_values = get_loss_values_from_metric_logger(log_file) @@ -107,7 +103,7 @@ def test_training_state_on_resume(self, tmpdir, monkeypatch): # Train for two epochs cmd_1 = f""" - tune full_finetune_single_device + tune run full_finetune_single_device \ --config llama2/7B_full_single_device \ output_dir={tmpdir} \ checkpointer._component_=torchtune.utils.FullModelHFCheckpointer \ @@ -121,12 +117,12 @@ def test_training_state_on_resume(self, tmpdir, monkeypatch): cmd_1 = cmd_1 + self._get_test_config_overrides() + model_config monkeypatch.setattr(sys, "argv", cmd_1) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") # Resume training cmd_2 = f""" - tune full_finetune_single_device + tune run full_finetune_single_device \ --config llama2/7B_full_single_device \ output_dir={tmpdir} \ checkpointer._component_=torchtune.utils.FullModelHFCheckpointer \ @@ -142,7 +138,7 @@ def test_training_state_on_resume(self, tmpdir, monkeypatch): cmd_2 = cmd_2 + self._get_test_config_overrides() + model_config monkeypatch.setattr(sys, "argv", cmd_2) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") expected_loss_values = self._fetch_expected_loss_values()[2:] @@ -189,7 +185,7 @@ def test_gradient_accumulation(self, tmpdir, monkeypatch): grad_accum_log_file = gen_log_file_name(tmpdir, suffix="grad_accum") cmd_1 = f""" - tune full_finetune_single_device \ + tune run full_finetune_single_device \ --config llama2/7B_full_single_device \ checkpointer._component_=torchtune.utils.FullModelTorchTuneCheckpointer \ checkpointer.checkpoint_dir={ckpt_dir} \ @@ -206,7 +202,7 @@ def test_gradient_accumulation(self, tmpdir, monkeypatch): cmd_1 = cmd_1 + self._get_test_config_overrides() + model_config monkeypatch.setattr(sys, "argv", cmd_1) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") no_accum_loss = get_loss_values_from_metric_logger(no_grad_accum_log_file)[ @@ -215,7 +211,7 @@ def test_gradient_accumulation(self, tmpdir, monkeypatch): # Update the cmd with new values for gradient accumulation cmd_2 = f""" - tune full_finetune_single_device \ + tune run full_finetune_single_device \ --config llama2/7B_full_single_device \ checkpointer._component_=torchtune.utils.FullModelTorchTuneCheckpointer \ checkpointer.checkpoint_dir={ckpt_dir} \ @@ -230,7 +226,7 @@ def test_gradient_accumulation(self, tmpdir, monkeypatch): cmd_2 = cmd_2 + self._get_test_config_overrides() + model_config monkeypatch.setattr(sys, "argv", cmd_2) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") accum_loss = np.mean(get_loss_values_from_metric_logger(grad_accum_log_file)) diff --git a/tests/recipes/test_lora_finetune_distributed.py b/tests/recipes/test_lora_finetune_distributed.py index 7027a4008f..3f56bd154f 100644 --- a/tests/recipes/test_lora_finetune_distributed.py +++ b/tests/recipes/test_lora_finetune_distributed.py @@ -56,10 +56,10 @@ def test_loss(self, tmpdir, monkeypatch): ckpt_dir = ckpt_path.parent log_file = gen_log_file_name(tmpdir) cmd = f""" - tune --nnodes 1 --nproc_per_node 2 lora_finetune_distributed + tune run --nnodes 1 --nproc_per_node 2 lora_finetune_distributed --config llama2/7B_lora \ output_dir={tmpdir} \ - checkpointer=torchtune.utils.FullModelTorchTuneCheckpointer + checkpointer=torchtune.utils.FullModelTorchTuneCheckpointer \ checkpointer.checkpoint_dir='{ckpt_dir}' \ checkpointer.checkpoint_files=[{ckpt_path}]\ checkpointer.output_dir={tmpdir} \ @@ -109,7 +109,7 @@ def test_training_state_on_resume(self, tmpdir, monkeypatch): # Train for two epochs cmd_1 = f""" - tune --nnodes 1 --nproc_per_node 2 lora_finetune_distributed + tune run --nnodes 1 --nproc_per_node 2 lora_finetune_distributed \ --config llama2/7B_lora \ output_dir={tmpdir} \ checkpointer=torchtune.utils.FullModelHFCheckpointer \ @@ -133,7 +133,7 @@ def test_training_state_on_resume(self, tmpdir, monkeypatch): # Resume training cmd_2 = f""" - tune --nnodes 1 --nproc_per_node 2 lora_finetune_distributed + tune run --nnodes 1 --nproc_per_node 2 lora_finetune_distributed \ --config llama2/7B_lora \ output_dir={tmpdir} \ checkpointer=torchtune.utils.FullModelHFCheckpointer \ @@ -166,7 +166,7 @@ def test_save_and_load_merged_weights(self, tmpdir, monkeypatch): ckpt_path = Path(CKPT_MODEL_PATHS[ckpt]) ckpt_dir = ckpt_path.parent cmd = f""" - tune --nnodes 1 --nproc_per_node 2 lora_finetune_distributed + tune run --nnodes 1 --nproc_per_node 2 lora_finetune_distributed \ --config llama2/7B_lora \ output_dir={tmpdir} \ model=torchtune.models.lora_small_test_model \ diff --git a/tests/recipes/test_lora_finetune_single_device.py b/tests/recipes/test_lora_finetune_single_device.py index 6cdbe2f4cd..7fc590ac17 100644 --- a/tests/recipes/test_lora_finetune_single_device.py +++ b/tests/recipes/test_lora_finetune_single_device.py @@ -57,10 +57,10 @@ def test_loss(self, tmpdir, monkeypatch): log_file = gen_log_file_name(tmpdir) cmd = f""" - tune lora_finetune_single_device + tune run lora_finetune_single_device \ --config llama2/7B_lora_single_device \ output_dir={tmpdir} \ - checkpointer=torchtune.utils.FullModelMetaCheckpointer + checkpointer=torchtune.utils.FullModelMetaCheckpointer \ checkpointer.checkpoint_dir='{ckpt_dir}' \ checkpointer.checkpoint_files=[{ckpt_path}]\ checkpointer.output_dir={tmpdir} \ @@ -78,7 +78,7 @@ def test_loss(self, tmpdir, monkeypatch): cmd = cmd + self._get_test_config_overrides(dtype_str="fp32") + model_config monkeypatch.setattr(sys, "argv", cmd) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") loss_values = get_loss_values_from_metric_logger(log_file) @@ -95,7 +95,7 @@ def test_loss_qlora(self, tmpdir, monkeypatch): log_file = gen_log_file_name(tmpdir) cmd = f""" - tune lora_finetune_single_device + tune run lora_finetune_single_device --config llama2/7B_qlora_single_device \ output_dir={tmpdir} \ checkpointer=torchtune.utils.FullModelMetaCheckpointer @@ -148,7 +148,7 @@ def test_training_state_on_resume(self, tmpdir, monkeypatch): # Train for two epochs cmd_1 = f""" - tune lora_finetune_single_device + tune run lora_finetune_single_device \ --config llama2/7B_lora_single_device \ output_dir={tmpdir} \ checkpointer=torchtune.utils.FullModelHFCheckpointer \ @@ -168,12 +168,12 @@ def test_training_state_on_resume(self, tmpdir, monkeypatch): cmd_1 = cmd_1 + self._get_test_config_overrides() + model_config monkeypatch.setattr(sys, "argv", cmd_1) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") # Resume training cmd_2 = f""" - tune lora_finetune_single_device + tune run lora_finetune_single_device \ --config llama2/7B_lora_single_device \ output_dir={tmpdir} \ checkpointer=torchtune.utils.FullModelHFCheckpointer \ @@ -188,7 +188,7 @@ def test_training_state_on_resume(self, tmpdir, monkeypatch): """.split() cmd_2 = cmd_2 + self._get_test_config_overrides() + model_config monkeypatch.setattr(sys, "argv", cmd_2) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") # Second epoch only @@ -206,10 +206,10 @@ def test_save_and_load_merged_weights(self, tmpdir, monkeypatch): ckpt_dir = ckpt_path.parent cmd = f""" - tune lora_finetune_single_device + tune run lora_finetune_single_device \ --config llama2/7B_lora_single_device \ output_dir={tmpdir} \ - checkpointer=torchtune.utils.FullModelTorchTuneCheckpointer + checkpointer=torchtune.utils.FullModelTorchTuneCheckpointer \ checkpointer.checkpoint_dir='{ckpt_dir}' \ checkpointer.checkpoint_files=[{ckpt_path}]\ checkpointer.output_dir={tmpdir} \ @@ -226,7 +226,7 @@ def test_save_and_load_merged_weights(self, tmpdir, monkeypatch): cmd = cmd + self._get_test_config_overrides() + model_config monkeypatch.setattr(sys, "argv", cmd) - with pytest.raises(SystemExit): + with pytest.raises(SystemExit, match=""): runpy.run_path(TUNE_PATH, run_name="__main__") # Next load both the merged weights in a Llama2 base model diff --git a/tests/torchtune/_cli/test_convert_checkpoint.py b/tests/torchtune/_cli/test_convert_checkpoint.py deleted file mode 100644 index c6dfbbdac7..0000000000 --- a/tests/torchtune/_cli/test_convert_checkpoint.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import runpy -import sys -import tempfile - -from pathlib import Path -from unittest.mock import patch - -import pytest -import torch - -from tests.common import TUNE_PATH -from tests.test_utils import assert_expected -from tests.torchtune.models.llama2.scripts.compare_decoder import Transformer -from torchtune.models.llama2 import llama2 - -ASSETS = Path(__file__).parent.parent.parent / "assets" - -# Generating `tiny_state_dict_with_one_key.pt` -# >>> import torch -# >>> state_dict = {"test_key": torch.randn(10, 10)} -# >>> torch.save(state_dict, "tiny_state_dict_with_one_key.pt") - -# Generating `tiny_fair_checkpoint.pt` -# >>> from tests.torchtune.models.llama2.scripts.compare_decoder import Transformer -# >>> from tests.test_utils import init_weights_with_constant -# >>> import torch -# >>> tiny_fair_transfomer = Transformer( -# vocab_size=500, -# n_layers=2, -# n_heads=4, -# dim=32, -# max_seq_len=64, -# n_kv_heads=4, -# ) -# >>> init_weights_with_constant(tiny_fair_transfomer, constant=0.2) -# >>> torch.save(tiny_fair_transfomer.state_dict(), "tiny_fair_checkpoint.pt") - - -class TestTuneCLIWithConvertCheckpointScript: - def test_convert_checkpoint_errors_on_bad_conversion(self, capsys): - incorrect_state_dict_loc = ASSETS / "tiny_state_dict_with_one_key.pt" - testargs = ( - f"tune convert_checkpoint --checkpoint-path {incorrect_state_dict_loc} --model llama2 --train-type full" - ).split() - with patch.object(sys, "argv", testargs): - with pytest.raises( - Exception, match=r".*Error converting the original Llama2.*" - ) as e: - runpy.run_path(TUNE_PATH, run_name="__main__") - - def _tiny_fair_transformer(self, ckpt): - tiny_fair_transfomer = Transformer( - vocab_size=500, - n_layers=2, - n_heads=4, - dim=32, - max_seq_len=64, - n_kv_heads=4, - ) - tiny_fair_state_dict = torch.load(ckpt, weights_only=True) - tiny_fair_transfomer.load_state_dict(tiny_fair_state_dict, strict=True) - return tiny_fair_transfomer - - def _tiny_native_transformer(self, ckpt): - tiny_native_transfomer = llama2( - vocab_size=500, - num_layers=2, - num_heads=4, - embed_dim=32, - max_seq_len=64, - num_kv_heads=4, - ) - tiny_native_state_dict = torch.load(ckpt, weights_only=True) - tiny_native_transfomer.load_state_dict( - tiny_native_state_dict["model"], strict=False - ) - return tiny_native_transfomer - - def _llama2_7b_fair_transformer(self, ckpt): - llama2_7b_fair_transformer = Transformer( - vocab_size=32_000, - n_layers=32, - n_heads=32, - dim=4096, - max_seq_len=2048, - n_kv_heads=32, - ) - llama2_7b_fair_state_dict = torch.load(ckpt, weights_only=True) - llama2_7b_fair_transformer.load_state_dict( - llama2_7b_fair_state_dict, strict=False - ) - llama2_7b_fair_transformer.eval() - return llama2_7b_fair_transformer - - def _llama2_7b_native_transformer(self, ckpt): - llama2_7b_native_transformer = llama2( - vocab_size=32_000, - num_layers=32, - num_heads=32, - embed_dim=4096, - max_seq_len=2048, - num_kv_heads=32, - ) - llama2_7b_native_state_dict = torch.load(ckpt, weights_only=True) - llama2_7b_native_transformer.load_state_dict( - llama2_7b_native_state_dict["model"], strict=True - ) - llama2_7b_native_transformer.eval() - return llama2_7b_native_transformer - - def _generate_toks_for_tiny(self): - return torch.randint(low=0, high=500, size=(16, 64)) - - def _generate_toks_for_llama2_7b(self): - return torch.randint(low=0, high=32_000, size=(16, 128)) - - def test_convert_checkpoint_matches_fair_model(self, caplog, pytestconfig): - is_large_scale_test = pytestconfig.getoption("--large-scale") - - if is_large_scale_test: - ckpt = "/tmp/test-artifacts/llama2-7b-fair" - fair_transformer = self._llama2_7b_fair_transformer(ckpt) - else: - ckpt = ASSETS / "tiny_fair_checkpoint.pt" - fair_transformer = self._tiny_fair_transformer(ckpt) - - output_path = tempfile.NamedTemporaryFile(delete=True).name - testargs = ( - f"tune convert_checkpoint --checkpoint-path {ckpt} --output-path {output_path} --model llama2 --train-type lora" - ).split() - with patch.object(sys, "argv", testargs): - runpy.run_path(TUNE_PATH, run_name="__main__") - - output = caplog.text - assert "Succesfully wrote PyTorch-native model checkpoint" in output - - native_transformer = ( - self._llama2_7b_native_transformer(output_path) - if is_large_scale_test - else self._tiny_native_transformer(output_path) - ) - - with torch.no_grad(): - for i in range(10): - toks = ( - self._generate_toks_for_llama2_7b() - if is_large_scale_test - else self._generate_toks_for_tiny() - ) - fair_out = fair_transformer(toks) - native_out = native_transformer(toks) - assert_expected(fair_out.sum(), native_out.sum()) diff --git a/tests/torchtune/_cli/test_cp.py b/tests/torchtune/_cli/test_cp.py index 499ec4bce7..0b6ae3b944 100644 --- a/tests/torchtune/_cli/test_cp.py +++ b/tests/torchtune/_cli/test_cp.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # @@ -10,11 +9,12 @@ from pathlib import Path import pytest - from tests.common import TUNE_PATH class TestTuneCLIWithCopyScript: + """This class tests the `tune cp` command.""" + @pytest.mark.parametrize("already_exists", (True, False)) def test_copy_successful(self, capsys, monkeypatch, tmpdir, already_exists): tmpdir_path = Path(tmpdir) @@ -23,7 +23,7 @@ def test_copy_successful(self, capsys, monkeypatch, tmpdir, already_exists): if already_exists: dest.touch() - args = f"tune cp llama2/7B_full.yaml {dest}".split() + args = f"tune cp llama2/7B_full {dest}".split() monkeypatch.setattr(sys, "argv", args) runpy.run_path(TUNE_PATH, run_name="__main__") @@ -32,7 +32,28 @@ def test_copy_successful(self, capsys, monkeypatch, tmpdir, already_exists): out = captured.out.rstrip("\n") assert dest.exists(), f"Expected {dest} to exist" - assert out == "" + assert f"Copied file to {dest}" in out + + def test_copy_successful_with_cwd_as_path(self, capsys, monkeypatch, tmpdir): + tmpdir_path = Path(tmpdir) + + # Needed so we can run test from tmpdir + tune_path_as_absolute = Path(TUNE_PATH).absolute() + + # Change cwd to tmpdir + monkeypatch.chdir(tmpdir_path) + + args = "tune cp llama2/7B_full .".split() + monkeypatch.setattr(sys, "argv", args) + runpy.run_path(str(tune_path_as_absolute), run_name="__main__") + + captured = capsys.readouterr() + out = captured.out.rstrip("\n") + + dest = tmpdir_path / "7B_full.yaml" + + assert dest.exists() + assert "Copied file to ./7B_full.yaml" in out def test_copy_skips_when_dest_already_exists_and_no_clobber_is_true( self, capsys, monkeypatch, tmpdir @@ -41,7 +62,7 @@ def test_copy_skips_when_dest_already_exists_and_no_clobber_is_true( existing_file = tmpdir_path / "existing_file.yaml" existing_file.touch() - args = f"tune cp llama2/7B_full_single_device.yaml {existing_file} -n".split() + args = f"tune cp llama2/7B_full_single_device {existing_file} -n".split() monkeypatch.setattr(sys, "argv", args) runpy.run_path(TUNE_PATH, run_name="__main__") @@ -51,23 +72,38 @@ def test_copy_skips_when_dest_already_exists_and_no_clobber_is_true( err = captured.err.rstrip("\n") assert err == "" - assert ( - "not overwriting" in out - ), f"Expected 'not overwriting' message, got '{out}'" + assert "not overwriting" in out + + def test_adds_correct_suffix_to_dest_when_no_suffix_is_provided( + self, capsys, monkeypatch, tmpdir + ): + tmpdir_path = Path(tmpdir) + dest = tmpdir_path / "my_custom_finetune" + + args = f"tune cp llama2/7B_full_single_device {dest}".split() + + monkeypatch.setattr(sys, "argv", args) + runpy.run_path(TUNE_PATH, run_name="__main__") + + captured = capsys.readouterr() + out = captured.out.rstrip("\n") + + assert dest.with_suffix(".yaml").exists(), f"Expected {dest} to exist" + assert f"Copied file to {dest}.yaml" in out @pytest.mark.parametrize( "tune_command,expected_error_message", [ ( - "tune cp non_existent_recipe.py .", - "error: Invalid file name: non_existent_recipe.py. Try `tune ls` to see all available files to copy.", + "tune cp non_existent_recipe .", + "error: Invalid file name: non_existent_recipe. Try `tune ls` to see all available files to copy.", ), ( - "tune cp non_existent_config.yaml .", - "error: Invalid file name: non_existent_config.yaml. Try `tune ls` to see all available files to copy.", + "tune cp non_existent_config .", + "error: Invalid file name: non_existent_config. Try `tune ls` to see all available files to copy.", ), ( - "tune cp full_finetune_single_device.py /home/mr_bean/full_finetune_single_device.py", + "tune cp full_finetune_single_device /home/mr_bean/full_finetune_single_device.py", "error: Cannot create regular file: '/home/mr_bean/full_finetune_single_device.py'. No such file or directory.", ), ( diff --git a/tests/torchtune/_cli/test_download.py b/tests/torchtune/_cli/test_download.py index 913094d728..b9e8eb512d 100644 --- a/tests/torchtune/_cli/test_download.py +++ b/tests/torchtune/_cli/test_download.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # @@ -7,27 +6,51 @@ import runpy import sys -import tempfile -from unittest.mock import patch import pytest - from tests.common import TUNE_PATH -class TestTuneCLIWithDownloadScript: - def test_download_no_hf_token_set_for_gated_model(self, capsys): - model = "meta-llama/Llama-2-7b" - testargs = f"tune download --repo-id {model}".split() - with patch.object(sys, "argv", testargs): - with pytest.raises(ValueError) as e: - runpy.run_path(TUNE_PATH, run_name="__main__") +class TestTuneDownloadCommand: + """This class tests the `tune download` command.""" + + @pytest.fixture + def snapshot_download(self, mocker, tmpdir): + + from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError + + yield mocker.patch( + "torchtune._cli.download.snapshot_download", + return_value=tmpdir, + # Side effects are iterated through on each call + side_effect=[ + GatedRepoError("test"), + RepositoryNotFoundError("test"), + mocker.DEFAULT, + ], + ) - def test_download_calls_snapshot(self, capsys): + def test_download_calls_snapshot(self, capsys, monkeypatch, snapshot_download): model = "meta-llama/Llama-2-7b" - with tempfile.TemporaryDirectory() as tmpdir: - testargs = f"tune download --repo-id {model} --output-dir {tmpdir} --hf-token ABCDEF".split() - with patch.object(sys, "argv", testargs): - with patch("huggingface_hub.snapshot_download") as snapshot: - runpy.run_path(TUNE_PATH, run_name="__main__") - snapshot.assert_called_once() + testargs = f"tune download {model}".split() + monkeypatch.setattr(sys, "argv", testargs) + + # Call the first time and get GatedRepoError + with pytest.raises(SystemExit, match="2"): + runpy.run_path(TUNE_PATH, run_name="__main__") + err = capsys.readouterr().err + assert "It looks like you are trying to access a gated repository." in err + + # Call the second time and get RepositoryNotFoundError + with pytest.raises(SystemExit, match="2"): + runpy.run_path(TUNE_PATH, run_name="__main__") + err = capsys.readouterr().err + assert "not found on the HuggingFace Hub" in err + + # Call the third time and get the expected output + runpy.run_path(TUNE_PATH, run_name="__main__") + output = capsys.readouterr().out + assert "Successfully downloaded model repo" in output + + # Make sure it was called twice + assert snapshot_download.call_count == 3 diff --git a/tests/torchtune/_cli/test_ls.py b/tests/torchtune/_cli/test_ls.py index cd7f2fad97..8ab70c950d 100644 --- a/tests/torchtune/_cli/test_ls.py +++ b/tests/torchtune/_cli/test_ls.py @@ -1,22 +1,20 @@ -#!/usr/bin/env python3 # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. - import runpy import sys from tests.common import TUNE_PATH -from torchtune import list_configs, list_recipes +from torchtune._recipe_registry import get_all_recipes -from torchtune._cli.ls import _NULL_VALUE +class TestTuneListCommand: + """This class tests the `tune ls` command.""" -class TestTuneCLIWithListScript: - def test_ls_lists_all_models(self, capsys, monkeypatch): + def test_ls_lists_all_recipes_and_configs(self, capsys, monkeypatch): testargs = "tune ls".split() monkeypatch.setattr(sys, "argv", testargs) @@ -24,11 +22,8 @@ def test_ls_lists_all_models(self, capsys, monkeypatch): captured = capsys.readouterr() output = captured.out.rstrip("\n") - for recipe in list_recipes(): - assert recipe in output, f"{recipe} was not found in output" - all_configs = list_configs(recipe) - if len(all_configs) > 0: - for config in list_configs(recipe): - assert config in output, f"{config} was not found in output" - else: - assert _NULL_VALUE in output, f"{_NULL_VALUE} was not found in output" + + for recipe in get_all_recipes(): + assert recipe.name in output + for config in recipe.configs: + assert config.name in output diff --git a/tests/torchtune/_cli/test_run.py b/tests/torchtune/_cli/test_run.py new file mode 100644 index 0000000000..076ff07704 --- /dev/null +++ b/tests/torchtune/_cli/test_run.py @@ -0,0 +1,75 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import runpy +import sys + +import pytest + +from tests.common import TUNE_PATH + + +class TestTuneRunCommand: + def test_run_calls_distributed_run_for_distributed_recipe( + self, capsys, monkeypatch, mocker + ): + testargs = "tune run --nproc_per_node 4 full_finetune_distributed --config llama2/7B_full".split() + + monkeypatch.setattr(sys, "argv", testargs) + distributed_run = mocker.patch("torchtune._cli.tune.Run._run_distributed") + runpy.run_path(TUNE_PATH, run_name="__main__") + distributed_run.assert_called_once() + + def test_run_calls_single_device_run_for_single_device_recipe( + self, capsys, monkeypatch, mocker + ): + testargs = "tune run full_finetune_single_device --config llama2/7B_full_single_device".split() + + monkeypatch.setattr(sys, "argv", testargs) + single_device_run = mocker.patch("torchtune._cli.tune.Run._run_single_device") + runpy.run_path(TUNE_PATH, run_name="__main__") + single_device_run.assert_called_once() + + def test_run_fails_when_called_with_distributed_args_for_single_device_recipe( + self, capsys, monkeypatch + ): + testargs = "tune run --nproc_per_node 4 full_finetune_single_device --config llama2/7B_full_single_device".split() + + monkeypatch.setattr(sys, "argv", testargs) + with pytest.raises(SystemExit, match="2"): + runpy.run_path(TUNE_PATH, run_name="__main__") + + output = capsys.readouterr() + assert "does not support distributed training" in output.err + + def test_run_fails_when_config_not_passed_in(self, capsys, monkeypatch): + testargs = "tune run full_finetune_single_device batch_size=3".split() + + monkeypatch.setattr(sys, "argv", testargs) + with pytest.raises(SystemExit, match="2"): + runpy.run_path(TUNE_PATH, run_name="__main__") + + output = capsys.readouterr() + assert "The '--config' argument is required" in output.err + + def test_run_succeeds_with_local_recipe_file_and_default_config( + self, capsys, monkeypatch, mocker + ): + testargs = "tune run my_custom_recipe.py --config llama2/7B_full".split() + monkeypatch.setattr(sys, "argv", testargs) + local_file_run = mocker.patch("torchtune._cli.tune.Run._run_single_device") + runpy.run_path(TUNE_PATH, run_name="__main__") + local_file_run.assert_called_once() + + def test_run_calls_local_file_run_for_local_file_recipe( + self, capsys, monkeypatch, mocker + ): + testargs = "tune run my_custom_recipe.py --config custom_config.yaml".split() + + monkeypatch.setattr(sys, "argv", testargs) + local_file_run = mocker.patch("torchtune._cli.tune.Run._run_single_device") + runpy.run_path(TUNE_PATH, run_name="__main__") + local_file_run.assert_called_once() diff --git a/tests/torchtune/_cli/test_tune.py b/tests/torchtune/_cli/test_tune.py index 5138b4c071..0e294fcbe7 100644 --- a/tests/torchtune/_cli/test_tune.py +++ b/tests/torchtune/_cli/test_tune.py @@ -5,27 +5,20 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import os -from pathlib import Path +import runpy +import sys -import torchtune - -from torchtune import list_configs, list_recipes +from tests.common import TUNE_PATH class TestTuneCLI: - def test_recipe_paths(self): - recipes = list_recipes() - for recipe in recipes: - pkg_path = str(Path(torchtune.__file__).parent.parent.absolute()) - recipe_path = os.path.join(pkg_path, "recipes", recipe) - assert os.path.exists(recipe_path), f"{recipe_path} must exist" + def test_tune_without_args_returns_help(self, capsys, monkeypatch): + testargs = ["tune"] + + monkeypatch.setattr(sys, "argv", testargs) + runpy.run_path(TUNE_PATH, run_name="__main__") + + captured = capsys.readouterr() + output = captured.out.rstrip("\n") - def test_config_paths(self): - recipes = list_recipes() - for recipe in recipes: - configs = list_configs(recipe) - for config in configs: - pkg_path = str(Path(torchtune.__file__).parent.parent.absolute()) - config_path = os.path.join(pkg_path, "recipes", "configs", config) - assert os.path.exists(config_path), f"{config_path} must exist" + assert "Welcome to the TorchTune CLI!" in output diff --git a/tests/torchtune/_cli/test_validate.py b/tests/torchtune/_cli/test_validate.py index be4e2150fc..c215a2b128 100644 --- a/tests/torchtune/_cli/test_validate.py +++ b/tests/torchtune/_cli/test_validate.py @@ -7,18 +7,22 @@ import runpy import sys -import pytest +from pathlib import Path +import pytest from tests.common import TUNE_PATH -from torchtune.config._errors import ConfigError -VALID_CONFIG_PATH = "tests/assets/valid_dummy_config.yaml" -INVALID_CONFIG_PATH = "tests/assets/invalid_dummy_config.yaml" +ASSETS = Path(__file__).parent.parent.parent / "assets" + + +class TestTuneValidateCommand: + """This class tests the `tune validate` command.""" + VALID_CONFIG_PATH = ASSETS / "valid_dummy_config.yaml" + INVALID_CONFIG_PATH = ASSETS / "invalid_dummy_config.yaml" -class TestTuneCLIWithValidateScript: def test_validate_good_config(self, capsys, monkeypatch): - args = f"tune validate --config {VALID_CONFIG_PATH}".split() + args = f"tune validate {self.VALID_CONFIG_PATH}".split() monkeypatch.setattr(sys, "argv", args) runpy.run_path(TUNE_PATH, run_name="__main__") @@ -28,23 +32,14 @@ def test_validate_good_config(self, capsys, monkeypatch): assert out == "Config is well-formed!" - def test_validate_bad_config(self, monkeypatch): - args = f"tune validate --config {INVALID_CONFIG_PATH}".split() + def test_validate_bad_config(self, monkeypatch, capsys): + args = f"tune validate {self.INVALID_CONFIG_PATH}".split() monkeypatch.setattr(sys, "argv", args) - with pytest.raises( - ConfigError, match="got an unexpected keyword argument 'dummy'" - ): + with pytest.raises(SystemExit): runpy.run_path(TUNE_PATH, run_name="__main__") - def test_validate_bad_override(self, monkeypatch, tmpdir): - args = f"\ - tune validate --config {VALID_CONFIG_PATH} \ - test._component_=torchtune.utils.get_dtype \ - test.dtype=fp32 test.dummy=3".split() + captured = capsys.readouterr() + err = captured.err.rstrip("\n") - monkeypatch.setattr(sys, "argv", args) - with pytest.raises( - ConfigError, match="got an unexpected keyword argument 'dummy'" - ): - runpy.run_path(TUNE_PATH, run_name="__main__") + assert "got an unexpected keyword argument 'dummy'" in err diff --git a/tests/torchtune/config/test_config_utils.py b/tests/torchtune/config/test_config_utils.py index b89e78f4fa..739f6d06bc 100644 --- a/tests/torchtune/config/test_config_utils.py +++ b/tests/torchtune/config/test_config_utils.py @@ -14,7 +14,7 @@ InstantiationError, ) from torchtune.data import AlpacaInstructTemplate -from torchtune.utils.argparse import TuneArgumentParser +from torchtune.utils.argparse import TuneRecipeArgumentParser _CONFIG = { "a": 1, @@ -47,7 +47,7 @@ def test_get_component_from_path(self): @mock.patch("torchtune.utils.argparse.OmegaConf.load", return_value=_CONFIG) def test_merge_yaml_and_cli_args(self, mock_load): - parser = TuneArgumentParser("test parser") + parser = TuneRecipeArgumentParser("test parser") yaml_args, cli_args = parser.parse_known_args( [ "--config", diff --git a/tests/torchtune/config/test_parse.py b/tests/torchtune/config/test_parse.py index f0a62b75da..9ce1436e35 100644 --- a/tests/torchtune/config/test_parse.py +++ b/tests/torchtune/config/test_parse.py @@ -24,7 +24,7 @@ def func(cfg): assert cfg.b != b with patch( - "torchtune.config._parse.TuneArgumentParser.parse_known_args", + "torchtune.config._parse.TuneRecipeArgumentParser.parse_known_args", return_value=(_CONFIG, []), ) as mock_parse_args: with pytest.raises(SystemExit): diff --git a/tests/torchtune/utils/test_argparse.py b/tests/torchtune/utils/test_argparse.py index 6d51f6c139..67018d848a 100644 --- a/tests/torchtune/utils/test_argparse.py +++ b/tests/torchtune/utils/test_argparse.py @@ -11,7 +11,7 @@ import pytest from omegaconf import OmegaConf -from torchtune.utils import TuneArgumentParser +from torchtune.utils import TuneRecipeArgumentParser _CONFIG = {"a": 1, "b": 2} @@ -19,7 +19,7 @@ class TestArgParse: @pytest.fixture def parser(self): - parser = TuneArgumentParser("Test parser") + parser = TuneRecipeArgumentParser("Test parser") return parser @mock.patch("torchtune.utils.argparse.OmegaConf.load", return_value=_CONFIG) diff --git a/torchtune/__init__.py b/torchtune/__init__.py index 00754930ee..f0a1aea443 100644 --- a/torchtune/__init__.py +++ b/torchtune/__init__.py @@ -6,37 +6,4 @@ from torchtune import datasets, models, modules, utils -_RECIPE_LIST = [ - "full_finetune_single_device.py", - "full_finetune_distributed.py", - "alpaca_generate.py", - "lora_finetune_single_device.py", - "lora_finetune_distributed.py", - "eleuther_eval.py", -] -_CONFIG_LISTS = { - "full_finetune_single_device.py": ["llama2/7B_full_single_device.yaml"], - "full_finetune_distributed.py": ["llama2/7B_full.yaml", "llama2/13B_full.yaml"], - "lora_finetune_single_device.py": [ - "llama2/7B_lora_single_device.yaml", - "llama2/7B_qlora_single_device.yaml", - ], - "lora_finetune_distributed.py": ["llama2/7B_lora.yaml", "llama2/13B_lora.yaml"], - "alpaca_generate.py": ["alpaca_generate.yaml"], - "eleuther_eval.py": ["eleuther_eval.yaml"], -} - - -def list_recipes(): - """List of recipes available from the CLI""" - return _RECIPE_LIST - - -def list_configs(recipe: str): - """List of configs available from the CLI given a recipe""" - if recipe not in _CONFIG_LISTS: - raise ValueError(f"Unknown recipe: {recipe}") - return _CONFIG_LISTS[recipe] - - __all__ = [datasets, models, modules, utils] diff --git a/torchtune/_cli/__init__.py b/torchtune/_cli/__init__.py index d7740adc37..2e41cd717f 100644 --- a/torchtune/_cli/__init__.py +++ b/torchtune/_cli/__init__.py @@ -3,10 +3,3 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. - -_SCRIPTS = ["download", "convert_checkpoint", "ls", "cp", "eval", "validate"] - - -def list_scripts(): - """List of available scripts.""" - return _SCRIPTS diff --git a/torchtune/_cli/convert_checkpoint.py b/torchtune/_cli/convert_checkpoint.py deleted file mode 100644 index 7c5e57ab4c..0000000000 --- a/torchtune/_cli/convert_checkpoint.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -"""This file converts download checkpoints to a format compatible with Torchtune.""" - -import argparse -from pathlib import Path -from typing import Optional - -import torch - -from torchtune.models.llama2 import convert_llama2_fair_format -from torchtune.utils import get_logger -from torchtune.utils.constants import MODEL_KEY - -_PYTORCH_MODEL_FILENAME = "native_pytorch_model.pt" - -log = get_logger("DEBUG") - - -def convert_checkpoint( - checkpoint_path: Path, - model: str, - output_path: Optional[Path] = None, - train_type: str = "full", - output_numerical_validation: bool = False, -): - """Convert model checkpoint to a PyTorch-native format compatible with Torchtune. - - Args: - checkpoint_path (Path): Path to the checkpoint path. - model (str): Model name - output_path (Optional[Path]): Path to the output checkpoint. - train_type (str): Type of finetuning - output_numerical_validation (bool): Whether to run numerical validation on the converted checkpoint. - - Raises: - Exception: If unsupported model is provided. - """ - # Load the original state dict - original_state_dict = torch.load( - checkpoint_path, map_location="cpu", weights_only=True - ) - log.info(msg="Loaded original state dict") - - # Convert checkpoint - if model == "llama2": - state_dict = convert_llama2_fair_format( - original_state_dict, output_numerical_validation - ) - else: - raise NotImplementedError(f"Model {model} is not supported in TorchTune.") - - # Save the state dict - if output_path is None: - checkpoint_dir = checkpoint_path.parent - output_path = checkpoint_dir / _PYTORCH_MODEL_FILENAME - - output_state_dict = {} - if train_type == "lora": - output_state_dict[MODEL_KEY] = state_dict - else: - output_state_dict = state_dict - torch.save(output_state_dict, output_path) - - log.info(msg=f"Succesfully wrote PyTorch-native model checkpoint to {output_path}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--checkpoint-path", type=Path, help="Path to the checkpoint to convert." - ) - parser.add_argument( - "--output-path", - type=Path, - help="Where to write the converted checkpoint. " - "Will default to the same directory as the original checkpoint if no arg is provided" - f"under the filename {_PYTORCH_MODEL_FILENAME}.", - required=False, - default=None, - ) - parser.add_argument( - "--model", - type=str, - help="model name", - choices=["llama2"], - required=True, - ) - parser.add_argument( - "--train-type", - type=str, - help="Type of finetuning. Currently Full-Finetuning and LoRA have slightly different formats. " - "This will be resolved soon.", - choices=["full", "lora"], - required=True, - ) - parser.add_argument( - "--output-numerical-validation", - action="store_true", - help="Whether to load the original checkpoint and the converted checkpoint and compare" - "the numerical output of a forward pass to ensure that the conversion was successful." - "Prints results to stdout. This additional check is only available for Llama2 7B." - "This will take awhile and may consume lots of memory. If you see an OOM error," - "please disable this flag. Note: All our checkpoints conversions are already validated" - "in unit tests for smaller checkpoints and integration tests for larger checkpoints." - "This flag is primarily for debugging purposes.", - required=False, - default=False, - ) - args = parser.parse_args() - convert_checkpoint( - args.checkpoint_path, - args.model, - args.output_path, - args.train_type, - args.output_numerical_validation, - ) diff --git a/torchtune/_cli/cp.py b/torchtune/_cli/cp.py index a403527de3..8c8d466828 100644 --- a/torchtune/_cli/cp.py +++ b/torchtune/_cli/cp.py @@ -3,101 +3,114 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. - -"""This script copies a built-in recipe or config to a local path.""" import argparse import shutil import textwrap from pathlib import Path import torchtune -from torchtune import list_configs, list_recipes +from torchtune._cli.subcommand import Subcommand +from torchtune._recipe_registry import get_all_recipes +ROOT = Path(torchtune.__file__).parent.parent -def _get_absolute_path(file_name: str) -> Path: - pkg_path = Path(torchtune.__file__).parent.parent.absolute() - recipes_path = pkg_path / "recipes" - if file_name.endswith(".yaml"): - path = recipes_path / "configs" / file_name - else: - assert file_name.endswith(".py"), f"Expected .py file, got {file_name}" - path = recipes_path / file_name - return path +class Copy(Subcommand): + """Holds all the logic for the `tune cp` subcommand.""" -def main(parser): - args = parser.parse_args() - destination = args.destination + def __init__(self, subparsers): + super().__init__() + self._parser = subparsers.add_parser( + "cp", + prog="tune cp", + usage="tune cp destination [OPTIONS]", + help="Copy a built-in recipe or config to a local path.", + description="Copy a built-in recipe or config to a local path.", + epilog=textwrap.dedent( + """\ + examples: + $ tune cp lora_finetune_distributed . + Copied file to ./lora_finetune_distributed.py - # Check if recipe/config is valid - all_recipes_and_configs = list_recipes() + [ - config for recipe in list_recipes() for config in list_configs(recipe) - ] - if args.file not in all_recipes_and_configs: - parser.error( - f"Invalid file name: {args.file}. Try `tune ls` to see all available files to copy." - ) + $ tune cp llama2/7B_full ./new_dir/my_custom_lora.yaml --make-parents + Copyied file to ./new_dir/my_custom_lora.yaml - # Get file path - file_name = args.file - src = _get_absolute_path(file_name) + Need to see all possible recipes/configs to copy? Try running `tune ls`. + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + self._add_arguments() + self._parser.set_defaults(func=self._cp_cmd) - # Copy file - try: - if args.no_clobber and destination.exists(): - print(f"File already exists at {destination.absolute()}, not overwriting.") - else: - if args.make_parents: - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(src, destination) - except FileNotFoundError: - parser.error( - f"Cannot create regular file: '{destination}'. No such file or directory. " - "If the specified destination's parent directory does not exist and you would " - "like to create it on-the-fly, use the --make-parents flag." + def _add_arguments(self) -> None: + """Add arguments to the parser.""" + self._parser.add_argument( + "file", + type=str, + help="Recipe/config to copy. For a list of all possible options, run `tune ls`", + ) + self._parser.add_argument( + "destination", + type=Path, + help="Location to copy the file to", ) + self._parser.add_argument( + "-n", + "--no-clobber", + action="store_true", + help="Do not overwrite destination if it already exists", + default=False, + ) + self._parser.add_argument( + "--make-parents", + action="store_true", + help="Create parent directories for destination if they do not exist. " + "If not set to True, will error if parent directories do not exist", + default=False, + ) + + def _cp_cmd(self, args: argparse.Namespace): + """Copy a recipe or config to a new location.""" + destination: Path = args.destination + src = None + + # Iterate through all recipes and configs + for recipe in get_all_recipes(): + if recipe.name == args.file: + src = ROOT / "recipes" / recipe.file_path + proper_suffix = ".py" + break + for config in recipe.configs: + if config.name == args.file: + src = ROOT / "recipes" / "configs" / config.file_path + proper_suffix = ".yaml" + break + # Fail if no file exists + if src is None: + self._parser.error( + f"Invalid file name: {args.file}. Try `tune ls` to see all available files to copy." + ) -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="tune cp", - usage="tune cp destination [OPTIONS]", - description="Copy a built-in recipe or config to a local path.", - epilog=textwrap.dedent( - """\ - examples: - $ tune cp llama2/7B_lora.yaml ./my_custom_llama2_lora.yaml - $ tune cp full_finetune_distributed.py ./my_custom_full_finetune.py - $ tune cp full_finetune_distributed.py ./new_dir/my_custom_full_finetune.py --make-parents + # Attach proper suffix if needed + if destination.name != "" and destination.suffix != proper_suffix: + destination = destination.with_suffix(proper_suffix) - Need to see all possible recipes/configs to copy? Try running `tune ls`. - And as always, you can also run `tune cp --help` for more information. - """ - ), - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "file", - type=str, - help="Recipe/config to copy. For a list of all possible options, run `tune ls`", - ) - parser.add_argument( - "destination", - type=Path, - help="Location to copy the file to", - ) - parser.add_argument( - "-n", - "--no-clobber", - action="store_true", - help="Do not overwrite destination if it already exists", - default=False, - ) - parser.add_argument( - "--make-parents", - action="store_true", - help="Create parent directories for destination if they do not exist. " - "If not set to True, will error if parent directories do not exist", - default=False, - ) - main(parser) + # Copy file + try: + if args.no_clobber and destination.exists(): + print( + f"File already exists at {destination.absolute()}, not overwriting." + ) + else: + if args.make_parents: + destination.parent.mkdir(parents=True, exist_ok=True) + output = shutil.copy(src, destination) + print(f"Copied file to {output}") + except FileNotFoundError: + self._parser.error( + f"Cannot create regular file: '{destination}'. No such file or directory. " + "If the specified destination's parent directory does not exist and you would " + "like to create it on-the-fly, use the --make-parents flag." + ) diff --git a/torchtune/_cli/download.py b/torchtune/_cli/download.py index 66b2b9b78c..eb039aedad 100644 --- a/torchtune/_cli/download.py +++ b/torchtune/_cli/download.py @@ -4,70 +4,111 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""This script downloads a model from the HuggingFace hub.""" - import argparse import os +import textwrap + from pathlib import Path from huggingface_hub import snapshot_download +from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError +from torchtune._cli.subcommand import Subcommand -def download(repo_id: str, output_dir: Path, hf_token: str) -> None: - """Downloads a model from the Hugging Face Hub. +class Download(Subcommand): + """Holds all the logic for the `tune download` subcommand.""" - Args: - repo_id (str): Name of the repository on Hugging Face Hub. - output_dir (Path): Directory in which to save the model. - hf_token (str): Hugging Face API token. + def __init__(self, subparsers: argparse._SubParsersAction): + super().__init__() + self._parser = subparsers.add_parser( + "download", + prog="tune download", + usage="tune download [OPTIONS]", + help="Download a model from the HuggingFace Hub.", + description="Download a model from the HuggingFace Hub.", + epilog=textwrap.dedent( + """\ + examples: + # Download a model from the HuggingFace Hub with a Hugging Face API token + $ tune download meta-llama/Llama-2-7b-hf --hf-token --output-dir /tmp/model + Successfully downloaded model repo and wrote to the following locations: + ./model/config.json + ./model/README.md + ./model/consolidated.00.pth + ... - Raises: - ValueError: If the model is not supported. - """ - if "meta-llama" in repo_id and hf_token is None: - raise ValueError( - "You need to provide a Hugging Face API token to download gated models." - "You can find your token by visiting https://huggingface.co/settings/tokens" - ) + # Download an ungated model from the HuggingFace Hub + $ tune download mistralai/Mistral-7B-Instruct-v0.2 + Successfully downloaded model repo and wrote to the following locations: + ./model/config.json + ./model/README.md + ./model/model-00001-of-00002.bin + ... - # Download the tokenizer and PyTorch model files - snapshot_download( - repo_id, - local_dir=output_dir, - resume_download=True, - token=hf_token, - ) + For a list of all models, visit the HuggingFace Hub https://huggingface.co/models. + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + self._add_arguments() + self._parser.set_defaults(func=self._download_cmd) - print( - "Succesfully downloaded model repo and wrote to the following locations:", - *list(output_dir.iterdir()), - sep="\n", - ) + def _add_arguments(self) -> None: + """Add arguments to the parser.""" + self._parser.add_argument( + "repo_id", + type=str, + help="Name of the repository on HuggingFace Hub.", + ) + self._parser.add_argument( + "--output-dir", + type=Path, + required=False, + default="./model", + help="Directory in which to save the model.", + ) + self._parser.add_argument( + "--hf-token", + type=str, + required=False, + default=os.getenv("HF_TOKEN", None), + help="HuggingFace API token. Needed for gated models like Llama2.", + ) + self._parser.add_argument( + "--ignore-patterns", + type=str, + required=False, + default="*.safetensors", + help="If provided, files matching any of the patterns are not downloaded. Defaults to ignoring " + "safetensors files as those are not currently supported in TorchTune.", + ) + def _download_cmd(self, args: argparse.Namespace) -> None: + """Downloads a model from the HuggingFace Hub.""" + # Download the tokenizer and PyTorch model files + try: + true_output_dir = snapshot_download( + args.repo_id, + local_dir=args.output_dir, + ignore_patterns=args.ignore_patterns, + token=args.hf_token, + ) + except GatedRepoError: + self._parser.error( + "It looks like you are trying to access a gated repository. Please ensure you " + "have access to the repository and have provided the proper HuggingFace API token " + "using the option `--hf-token` or by running `huggingface-cli login`." + "You can find your token by visiting https://huggingface.co/settings/tokens" + ) + except RepositoryNotFoundError: + self._parser.error( + f"Repository '{args.repo_id}' not found on the HuggingFace Hub." + ) + except Exception as e: + self._parser.error(e) -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Download a model from the Hugging Face Hub." - ) - parser.add_argument( - "--repo-id", - type=str, - required=True, - help="Name of the repository on Hugging Face Hub.", - ) - parser.add_argument( - "--output-dir", - type=Path, - required=False, - default="/tmp/model", - help="Directory in which to save the model.", - ) - parser.add_argument( - "--hf-token", - type=str, - required=False, - default=os.getenv("HF_TOKEN", None), - help="Hugging Face API token. Needed for gated models like Llama2.", - ) - args = parser.parse_args() - download(args.repo_id, args.output_dir, args.hf_token) + print( + "Successfully downloaded model repo and wrote to the following locations:", + *list(Path(true_output_dir).iterdir()), + sep="\n", + ) diff --git a/torchtune/_cli/ls.py b/torchtune/_cli/ls.py index 3e7f24c0e0..9f0abdbd11 100644 --- a/torchtune/_cli/ls.py +++ b/torchtune/_cli/ls.py @@ -4,57 +4,61 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""This script lists all built-in recipes/configs""" - import argparse import textwrap -from torchtune import list_configs, list_recipes - -_NULL_VALUE = "<>" - - -def main(): - # Print table header - header = f"{'RECIPE':<40} {'CONFIG':<40}" - print(header) - - # Print recipe/config pairs - for recipe in list_recipes(): - configs = list_configs(recipe) - # If there are no configs for a recipe, print a blank config - if len(configs) == 0: - row = f"{recipe:<40} {_NULL_VALUE:<40}" - print(row) - for i, config in enumerate(configs): - # If there are multiple configs for a single recipe, omit the recipe name - # on latter configs - if i > 0: - recipe = "" - row = f"{recipe:<40} {config:<40}" - print(row) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="List all built-in recipes and configs", - usage="tune ls", - epilog=textwrap.dedent( - """\ - examples: - $ tune ls - RECIPE CONFIG - full_finetune_distributed.py llama2/7B_full, - llama2/13B_full - lora_finetune_distributed.py llama2/7B_lora, - llama2/13B_lora - alpaca_generate.py alpaca_generate.yaml - - To run one of these recipes: - $ tune full_finetune_single_device --config llama2/7B_full_single_device - """ - ), - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.parse_args() - main() +from torchtune._cli.subcommand import Subcommand + +from torchtune._recipe_registry import get_all_recipes + + +class List(Subcommand): + """Holds all the logic for the `tune ls` subcommand.""" + + NULL_VALUE = "<>" + + def __init__(self, subparsers: argparse._SubParsersAction): + super().__init__() + self._parser = subparsers.add_parser( + "ls", + prog="tune ls", + help="List all built-in recipes and configs", + description="List all built-in recipes and configs", + epilog=textwrap.dedent( + """\ + examples: + $ tune ls + RECIPE CONFIG + full_finetune_single_device llama2/7B_full_single_device + full_finetune_distributed llama2/7B_full + llama2/13B_full + ... + + To run one of these recipes: + $ tune run full_finetune_single_device --config full_finetune_single_device + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + self._parser.set_defaults(func=self._ls_cmd) + + def _ls_cmd(self, args: argparse.Namespace) -> None: + """List all available recipes and configs.""" + # Print table header + header = f"{'RECIPE':<40} {'CONFIG':<40}" + print(header) + + # Print recipe/config pairs + for recipe in get_all_recipes(): + # If there are no configs for a recipe, print a blank config + recipe_str = recipe.name + if len(recipe.configs) == 0: + row = f"{recipe_str:<40} {self.NULL_VALUE:<40}" + print(row) + for i, config in enumerate(recipe.configs): + # If there are multiple configs for a single recipe, omit the recipe name + # on latter configs + if i > 0: + recipe_str = "" + row = f"{recipe_str:<40} {config.name:<40}" + print(row) diff --git a/torchtune/_cli/run.py b/torchtune/_cli/run.py new file mode 100644 index 0000000000..fc44b4fe60 --- /dev/null +++ b/torchtune/_cli/run.py @@ -0,0 +1,180 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import argparse +import runpy +import sys +import textwrap + +from pathlib import Path +from typing import Optional + +import torchtune + +from torch.distributed.run import get_args_parser as get_torchrun_args_parser, run +from torchtune._cli.subcommand import Subcommand +from torchtune._recipe_registry import Config, get_all_recipes, Recipe + +ROOT = Path(torchtune.__file__).parent.parent + + +class Run(Subcommand): + """Holds all the logic for the `tune run` subcommand.""" + + def __init__(self, subparsers): + super().__init__() + self._parser = subparsers.add_parser( + "run", + prog="tune run", + help="Run a recipe. For distributed recipes, this supports all torchrun arguments.", + description="Run a recipe. For distributed recipes, this supports all torchrun arguments.", + usage="tune run [TORCHRUN-OPTIONS] --config [RECIPE-OPTIONS]", + epilog=textwrap.dedent( + """\ + examples: + + # Run a finetuning recipe on a single device w/ default values + $ tune run lora_finetune_single_device --config llama2/7B_lora_single_device + + # Run a finetuning recipe in a distributed fashion using torchrun w/ default values + $ tune run --nproc_per_node 4 full_finetune_distributed --config llama2/7B_full_finetune_distributed + + # Override a parameter in the config file and specify a number of GPUs for torchrun + $ tune run --nproc_per_node 2 \ + lora_finetune_single_device \ + --config llama2/7B_lora_single_device \ + model.lora_rank=16 \ + + Remember, you can use `tune cp` to copy a default recipe/config to your local dir and modify the values. + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + self._add_arguments() + self._parser.set_defaults(func=self._run_cmd) + + def _add_arguments(self) -> None: + """Add arguments to the parser. + + This is a bit hacky since we need to add the torchrun arguments to our parser. + This grabs the argparser from torchrun, iterates over it's actions, and adds them + to our parser. We rename the training_script and training_script_args to recipe and recipe_args + respectively. In addition, we leave out the help argument since we add it manually to ours. + """ + torchrun_argparser = get_torchrun_args_parser() + for action in torchrun_argparser._actions: + if action.dest == "training_script": + action.dest = "recipe" + action.help = """ + Name or path to recipe to be launched followed by args. + For a list of all possible recipes, run `tune ls`.""" + elif action.dest == "training_script_args": + action.dest = "recipe_args" + action.help = "Args to be passed to the recipe." + elif action.dest == "help": + continue + self._parser._add_action(action) + + def _run_distributed(self, args: argparse.Namespace): + """Run a recipe with torchrun.""" + # TODO (rohan-varma): Add check that nproc_per_node <= cuda device count. Currently, + # we don't do this since we test on CPUs for distributed. Will update once multi GPU CI is supported. + print("Running with torchrun...") + # Have to reset the argv so that the recipe can be run with the correct arguments + args.training_script = args.recipe + args.training_script_args = args.recipe_args + run(args) + + def _run_single_device(self, args: argparse.Namespace): + """Run a recipe on a single device.""" + sys.argv = [str(args.recipe)] + args.recipe_args + runpy.run_path(str(args.recipe), run_name="__main__") + + def _is_distributed_args(self, args: argparse.Namespace): + """Check if the user is trying to run a distributed recipe.""" + total = len(sys.argv) - 2 # total args minus "tune run" + script_args = len(args.recipe_args) + 1 # script args + 1 for script name + return total > script_args + + def _get_recipe(self, recipe_str: str) -> Optional[Recipe]: + """Get a recipe from the name or path. + + Args: + recipe_str (str): The name or path of the recipe. + + Returns: + The recipe if it's found in built-in recipes, otherwise None. + """ + for recipe in get_all_recipes(): + if recipe.name == recipe_str: + return recipe + + def _get_config( + self, config_str: str, specific_recipe: Optional[Recipe] + ) -> Optional[Config]: + """Get a config from the name or path. + + Args: + config_str (str): The name or path of the config. + specific_recipe (Optional[Recipe]): The specific recipe to search through. + + Returns: + The config if it's found in built-in configs, otherwise None. + """ + # If a specific recipe is provided, search through it + if specific_recipe is not None: + for config in specific_recipe.configs: + if config.name == config_str: + return config + + # If not, search through all recipes + for recipe in get_all_recipes(): + for config in recipe.configs: + if config.name == config_str: + return config + + def _run_cmd(self, args: argparse.Namespace): + """Run a recipe.""" + # We have to assume that the recipe supports distributed training + supports_distributed = True + recipe_path, config_path = None, None + + # Try to find config string in args + try: + config_idx = args.recipe_args.index("--config") + 1 + config_str = args.recipe_args[config_idx] + except ValueError: + self._parser.error("The '--config' argument is required.") + + # Get recipe path + recipe = self._get_recipe(args.recipe) + if recipe is None: + recipe_path = args.recipe + else: + recipe_path = str(ROOT / "recipes" / recipe.file_path) + supports_distributed = recipe.supports_distributed + + # Get config path + config = self._get_config(config_str, recipe) + if config is None: + config_path = config_str + else: + config_path = str(ROOT / "recipes" / "configs" / config.file_path) + + # Prepare args + args.recipe = recipe_path + args.recipe_args[config_idx] = config_path + + # Execute recipe + if self._is_distributed_args(args): + if not supports_distributed: + self._parser.error( + f"Recipe {recipe.name} does not support distributed training." + "Please run without torchrun commands." + ) + self._run_distributed(args) + else: + self._run_single_device(args) diff --git a/torchtune/_cli/subcommand.py b/torchtune/_cli/subcommand.py new file mode 100644 index 0000000000..db298a0b03 --- /dev/null +++ b/torchtune/_cli/subcommand.py @@ -0,0 +1,17 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +class Subcommand: + def __init__(self, *args, **kwargs): + pass + + @classmethod + def create(cls, *args, **kwargs): + return cls(*args, **kwargs) + + def _add_arguments(self): + pass diff --git a/torchtune/_cli/tune.py b/torchtune/_cli/tune.py index f6a5c43fcf..87fb778016 100644 --- a/torchtune/_cli/tune.py +++ b/torchtune/_cli/tune.py @@ -4,133 +4,49 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -""" -Launcher and utilities for torchtune recipes - -``tune`` provides functionality for launching torchtune recipes as well as local -recipes. Aside from torchtune recipe utilties it integrates with ``torch.distributed.run`` -to support distributed job launching by default. ``tune`` offers everyting that ``torchrun`` -does with the following additional functionalities: - -1. ``tune `` with no optional ``torchrun`` options launches a single python process - -2. ```` and recipe arg ```` can both be passed in as names instead of paths if they're included in torchtune - -3. ``tune `` can be used to launch local recipes - -4. ``tune `` will launch a torchrun job - -.. note:: ``tune`` is a python - `console script `_ - to the main module - `scripts.cli_utils.tune `_ - declared in the ``scripts`` configuration in - `setup.py `_. - It is equivalent to invoking ``python -m scripts.cli_utils.tune``. -""" import argparse -import runpy -import sys -from pathlib import Path -import torchtune -from torch.distributed.run import get_args_parser, run -from torchtune import list_recipes -from torchtune._cli import list_scripts -from torchtune.utils._distributed import _valid_distributed_single_node_nnodes +from torchtune._cli.cp import Copy +from torchtune._cli.download import Download +from torchtune._cli.ls import List +from torchtune._cli.run import Run +from torchtune._cli.validate import Validate -def _update_parser_help(parser): - parser.description = "Torch Tune Recipe Launcher" - parser.usage = "tune [options] [recipe_args]" - parser.formatter_class = argparse.RawDescriptionHelpFormatter +class TuneCLIParser: + """Holds all information related to running the CLI""" - # Update torchrun argparse name for more accurate CLI help - actions = [a.dest for a in parser._actions] - # Update training_script help to be recipe - idx = actions.index("training_script") - parser._actions[idx].dest = "recipe" - parser._actions[idx].help = "Name or path to recipe to be launched followed by args" - - # Update training_script_args help to be recipe_args - idx = actions.index("training_script_args") - parser._actions[idx].dest = "recipe_args" - - -def _is_distributed_args(args): - total = len(sys.argv) - 1 # total args minus "tune" - script_args = len(args.recipe_args) + 1 # script args + 1 for script name - return total > script_args - - -def _validate_distributed_args(args): - """ - Validates nnodes and nproc_per_node are appropriately set for distributed training - runs. - """ - if not hasattr(args, "nnodes"): - raise RuntimeError("Expect --nnodes to be specified for distributed runs") - - if args.nnodes not in _valid_distributed_single_node_nnodes: - raise RuntimeError( - f"Expect --nnodes to be one of {_valid_distributed_single_node_nnodes}" + def __init__(self): + # Initialize the top-level parser + self._parser = argparse.ArgumentParser( + prog="tune", + description="Welcome to the TorchTune CLI!", + add_help=True, ) + # Default command is to print help + self._parser.set_defaults(func=lambda args: self._parser.print_help()) - if not hasattr(args, "nproc_per_node"): - raise RuntimeError( - "Expect --nproc_per_node to be specified for distributed runs" - ) + # Add subcommands + subparsers = self._parser.add_subparsers(title="subcommands") + Download.create(subparsers) + List.create(subparsers) + Copy.create(subparsers) + Run.create(subparsers) + Validate.create(subparsers) + + def parse_args(self) -> argparse.Namespace: + """Parse CLI arguments""" + return self._parser.parse_args() - # TODO (rohan-varma): Add check that nproc_per_node <= cuda device count. Currently, - # we don't do this since we test on CPUs for distributed. Will update once multi GPU - # CI is supported. + def run(self, args: argparse.Namespace) -> None: + """Execute CLI""" + args.func(args) def main(): - parser = get_args_parser() - _update_parser_help(parser) + parser = TuneCLIParser() args = parser.parse_args() - - distributed_args = _is_distributed_args(args) - cmd = args.recipe - if not cmd.endswith(".py"): - pkg_path = Path(torchtune.__file__).parent.absolute() - if f"{cmd}.py" in list_recipes(): - recipes_pkg_path = pkg_path.parent / "recipes" - cmd = recipes_pkg_path / f"{cmd}.py" - args.recipe = str(cmd) - - # Replace config name with package path if provided - if "--config" in args.recipe_args: - cfg_idx = args.recipe_args.index("--config") + 1 - config = args.recipe_args[cfg_idx] - if not config.endswith(".yaml"): - args.recipe_args[cfg_idx] = str( - recipes_pkg_path / "configs" / f"{config}.yaml" - ) - elif cmd in list_scripts(): - cmd = pkg_path / "_cli" / f"{cmd}.py" - args.recipe = str(cmd) - assert not distributed_args, "You can't use distributed args with scripts" - else: - parser.error( - f"Unrecognized command '{cmd}'\nTry 'tune --help' for more information." - ) - - if distributed_args: - _validate_distributed_args(args) - args.training_script = str(cmd) # arg names expected by torchrun - args.training_script_args = args.recipe_args - run(args) - else: - sys.argv = [str(cmd)] + args.recipe_args - runpy.run_path(str(cmd), run_name="__main__") + parser.run(args) if __name__ == "__main__": diff --git a/torchtune/_cli/validate.py b/torchtune/_cli/validate.py index 1c054d8662..7efac13090 100644 --- a/torchtune/_cli/validate.py +++ b/torchtune/_cli/validate.py @@ -6,33 +6,54 @@ import argparse import textwrap +from pathlib import Path + +from omegaconf import OmegaConf -from omegaconf import DictConfig from torchtune import config -from torchtune.config._utils import _merge_yaml_and_cli_args -from torchtune.utils import TuneArgumentParser - - -def main(cfg: DictConfig): - config.validate(cfg) - print("Config is well-formed!") - - -if __name__ == "__main__": - parser = TuneArgumentParser( - description="Validate a config and ensure that it is well-formed.", - usage="tune validate", - epilog=textwrap.dedent( - """\ - examples: - $ tune validate --config recipes/configs/llama2/7B_lora.yaml - Config is well-formed! - """ - ), - formatter_class=argparse.RawTextHelpFormatter, - ) - # Get user-specified args from config and CLI and create params for recipe - yaml_args, cli_args = parser.parse_known_args() - conf = _merge_yaml_and_cli_args(yaml_args, cli_args) - - main(conf) +from torchtune._cli.subcommand import Subcommand +from torchtune.config._errors import ConfigError + + +class Validate(Subcommand): + """Holds all the logic for the `tune validate` subcommand.""" + + def __init__(self, subparsers: argparse._SubParsersAction): + super().__init__() + self._parser = subparsers.add_parser( + "validate", + prog="tune validate", + help="Validate a config and ensure that it is well-formed.", + description="Validate a config and ensure that it is well-formed.", + usage="tune validate ", + epilog=textwrap.dedent( + """\ + examples: + + $ tune validate recipes/configs/full_finetune_distributed.yaml + Config is well-formed! + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + self._add_arguments() + self._parser.set_defaults(func=self._validate_cmd) + + def _add_arguments(self) -> None: + """Add arguments to the parser.""" + self._parser.add_argument( + "config", + type=Path, + help="Path to a config to validate.", + ) + + def _validate_cmd(self, args: argparse.Namespace): + """Validate a config file.""" + cfg = OmegaConf.load(args.config) + + try: + config.validate(cfg) + except ConfigError as e: + self._parser.error(str(e)) + + print("Config is well-formed!") diff --git a/torchtune/_recipe_registry.py b/torchtune/_recipe_registry.py new file mode 100644 index 0000000000..57ba562cbb --- /dev/null +++ b/torchtune/_recipe_registry.py @@ -0,0 +1,95 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from dataclasses import dataclass +from typing import List + + +@dataclass +class Config: + name: str + file_path: str + + +@dataclass +class Recipe: + name: str + file_path: str + configs: List[Config] + supports_distributed: bool + + +_ALL_RECIPES = [ + Recipe( + name="full_finetune_single_device", + file_path="full_finetune_single_device.py", + configs=[ + Config( + name="llama2/7B_full_single_device", + file_path="llama2/7B_full_single_device.yaml", + ), + Config( + name="llama2/7B_full_single_device_low_memory", + file_path="llama2/7B_full_single_device_low_memory.yaml", + ), + ], + supports_distributed=False, + ), + Recipe( + name="full_finetune_distributed", + file_path="full_finetune_distributed.py", + configs=[ + Config(name="llama2/7B_full", file_path="llama2/7B_full.yaml"), + Config(name="llama2/13B_full", file_path="llama2/13B_full.yaml"), + ], + supports_distributed=True, + ), + Recipe( + name="lora_finetune_single_device", + file_path="lora_finetune_single_device.py", + configs=[ + Config( + name="llama2/7B_lora_single_device", + file_path="llama2/7B_lora_single_device.yaml", + ), + Config( + name="llama2/7B_qlora_single_device", + file_path="llama2/7B_qlora_single_device.yaml", + ), + ], + supports_distributed=False, + ), + Recipe( + name="lora_finetune_distributed", + file_path="lora_finetune_distributed.py", + configs=[ + Config(name="llama2/7B_lora", file_path="llama2/7B_lora.yaml"), + Config(name="llama2/13B_lora", file_path="llama2/13B_lora.yaml"), + ], + supports_distributed=True, + ), + Recipe( + name="alpaca_generate", + file_path="alpaca_generate.py", + configs=[ + Config(name="alpaca_generate", file_path="alpaca_generate.yaml"), + ], + supports_distributed=False, + ), + Recipe( + name="eleuther_eval", + file_path="eleuther_eval.py", + configs=[ + Config(name="eleuther_eval", file_path="eleuther_eval.yaml"), + ], + supports_distributed=False, + ), +] + + +def get_all_recipes(): + """List of recipes available from the CLI.""" + return _ALL_RECIPES diff --git a/torchtune/config/_parse.py b/torchtune/config/_parse.py index 58ec0c5e5f..dc35f41c8e 100644 --- a/torchtune/config/_parse.py +++ b/torchtune/config/_parse.py @@ -10,7 +10,7 @@ from omegaconf import DictConfig from torchtune.config._utils import _merge_yaml_and_cli_args -from torchtune.utils.argparse import TuneArgumentParser +from torchtune.utils.argparse import TuneRecipeArgumentParser from torchtune.utils.logging import get_logger @@ -40,7 +40,7 @@ def parse(recipe_main: Recipe) -> Callable[[Recipe], Any]: @functools.wraps(recipe_main) def wrapper(*args: Any, **kwargs: Any) -> Any: - parser = TuneArgumentParser( + parser = TuneRecipeArgumentParser( description=recipe_main.__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) diff --git a/torchtune/utils/__init__.py b/torchtune/utils/__init__.py index 15e749e9f1..623ad474fb 100644 --- a/torchtune/utils/__init__.py +++ b/torchtune/utils/__init__.py @@ -22,7 +22,7 @@ validate_no_params_on_meta_device, wrap_fsdp, ) -from .argparse import TuneArgumentParser +from .argparse import TuneRecipeArgumentParser from .checkpointable_dataloader import CheckpointableDataLoader from .collate import padded_collate from .constants import ( # noqa @@ -74,7 +74,7 @@ "set_default_dtype", "set_seed", "validate_expected_param_dtype", - "TuneArgumentParser", + "TuneRecipeArgumentParser", "CheckpointableDataLoader", "OptimizerInBackwardWrapper", "create_optim_in_bwd_wrapper", diff --git a/torchtune/utils/argparse.py b/torchtune/utils/argparse.py index 79e34bd92f..55acb3b23c 100644 --- a/torchtune/utils/argparse.py +++ b/torchtune/utils/argparse.py @@ -11,9 +11,9 @@ from omegaconf import OmegaConf -class TuneArgumentParser(argparse.ArgumentParser): +class TuneRecipeArgumentParser(argparse.ArgumentParser): """ - TuneArgumentParser is a helpful utility subclass of the argparse ArgumentParser that + TuneRecipeArgParser is a helpful utility subclass of the argparse ArgumentParser that adds a builtin argument "config". The config argument takes a file path to a yaml file and will load in argument defaults from the yaml file. The yaml file must only contain argument names and their values and nothing more, it does not have to include all of the From dc6e54d89a3fa155106f19d3c969966c2aa31d15 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Fri, 29 Mar 2024 16:34:50 -0400 Subject: [PATCH 10/76] Fix typos in Acknowledgements section (#617) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2aaee6bcc8..c84b97fda3 100644 --- a/README.md +++ b/README.md @@ -205,12 +205,12 @@ TorchTune provides well-tested components with a high-bar on correctness. The li ## Acknowledgements -The Llama2 code in this repository is inspired by the original [Llama2 code](https://github.com/meta-llama/llama/blob/main/llama/model.py). We'd also like to give a huge shoutout to some awesome libraries and tools in the ecosystems! +The Llama2 code in this repository is inspired by the original [Llama2 code](https://github.com/meta-llama/llama/blob/main/llama/model.py). We'd also like to give a huge shoutout to some awesome libraries and tools in the ecosystem! - EleutherAI's [LM Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness) - Hugging Face for the [Datasets Repository](https://github.com/huggingface/datasets) - [gpt-fast](https://github.com/pytorch-labs/gpt-fast) for performant LLM inference techniques which we've adopted OOTB -- [lit-gpt](https://github.com/Lightning-AI/litgpt), [axolotl](https://github.com/OpenAccess-AI-Collective/axolotl) [transformers](https://github.com/huggingface/transformers) and [llama recipes](https://github.com/meta-llama/llama-recipes) for reference implementations and pushing forward the LLM finetuning community +- [lit-gpt](https://github.com/Lightning-AI/litgpt), [axolotl](https://github.com/OpenAccess-AI-Collective/axolotl), [transformers](https://github.com/huggingface/transformers) and [llama recipes](https://github.com/meta-llama/llama-recipes) for reference implementations and pushing forward the LLM finetuning community - [bitsandbytes](https://github.com/TimDettmers/bitsandbytes)   From a804c232f9651378d656240d5d76818c4469a347 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Fri, 29 Mar 2024 16:57:50 -0400 Subject: [PATCH 11/76] HuggingFace --> Hugging Face (#618) --- README.md | 4 ++-- .../examples/first_finetune_tutorial.rst | 6 +++--- docs/source/overview.rst | 4 ++-- requirements.txt | 2 +- tests/torchtune/_cli/test_download.py | 2 +- tests/torchtune/data/test_templates.py | 2 +- torchtune/_cli/download.py | 20 +++++++++---------- torchtune/data/_templates.py | 2 +- torchtune/datasets/_alpaca.py | 2 +- torchtune/datasets/_chat.py | 4 ++-- torchtune/datasets/_grammar.py | 2 +- torchtune/datasets/_instruct.py | 4 ++-- torchtune/datasets/_samsum.py | 2 +- torchtune/modules/lr_schedulers.py | 2 +- 14 files changed, 29 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index c84b97fda3..02f241d3db 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ The library provides: - Support for checkpoints in various formats, including checkpoints in HF format - Training recipes for popular fine-tuning techniques with reference benchmarks and comprehensive correctness checks - Evaluation of trained models with EleutherAI Eval Harness -- Integration with HuggingFace Datasets for training +- Integration with Hugging Face Datasets for training - Support for distributed training using FSDP from PyTorch Distributed - YAML configs for easily configuring training runs - [Upcoming] Support for lower precision dtypes and quantization techniques from [TorchAO](https://github.com/pytorch-labs/ao) @@ -182,7 +182,7 @@ TorchTune embodies PyTorch’s design philosophy [[details](https://pytorch.org/ #### Native PyTorch -TorchTune is a native-PyTorch library. While we provide integrations with the surrounding ecosystem (eg: HuggingFace Datasets, EluetherAI Eval Harness), all of the core functionality is written in PyTorch. +TorchTune is a native-PyTorch library. While we provide integrations with the surrounding ecosystem (eg: Hugging Face Datasets, EluetherAI Eval Harness), all of the core functionality is written in PyTorch. #### Simplicity and Extensibility diff --git a/docs/source/examples/first_finetune_tutorial.rst b/docs/source/examples/first_finetune_tutorial.rst index cdb86d350f..fa4b52dec8 100644 --- a/docs/source/examples/first_finetune_tutorial.rst +++ b/docs/source/examples/first_finetune_tutorial.rst @@ -25,13 +25,13 @@ job using TorchTune. Downloading a model ------------------- First, you need to download a model. TorchTune's supports an integration -with the `HuggingFace Hub `_ - a collection of the latest and greatest model weights. +with the `Hugging Face Hub `_ - a collection of the latest and greatest model weights. For this tutorial, you're going to use the `Llama2 model from Meta `_. Llama2 is a "gated model", meaning that you need to be granted access in order to download the weights. Follow `these instructions `_ on the official Meta page -hosted on HuggingFace to complete this process. (This should take less than 5 minutes.) +hosted on Hugging Face to complete this process. (This should take less than 5 minutes.) -Once you have authorization, you will need to authenticate with HuggingFace Hub. The easiest way to do so is to provide an +Once you have authorization, you will need to authenticate with Hugging Face Hub. The easiest way to do so is to provide an access token to the download script. You can find your token `here `_. Then, it's as simple as: diff --git a/docs/source/overview.rst b/docs/source/overview.rst index 77f4b7c4c3..46f72176a5 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -22,7 +22,7 @@ TorchTune provides: - Modular native-PyTorch implementations of popular LLMs - Interoperability with popular model zoos through checkpoint-conversion utilities - Training recipes for a variety of fine-tuning techniques -- Integration with `HuggingFace Datasets `_ for training and `EleutherAI's Eval `_ Harness for evaluation +- Integration with `Hugging Face Datasets `_ for training and `EleutherAI's Eval `_ Harness for evaluation - Support for distributed training using `FSDP `_ - Yaml configs for easily configuring training runs @@ -55,7 +55,7 @@ TorchTune embodies `PyTorch’s design philosophy --output-dir /tmp/model Successfully downloaded model repo and wrote to the following locations: ./model/config.json @@ -37,7 +37,7 @@ def __init__(self, subparsers: argparse._SubParsersAction): ./model/consolidated.00.pth ... - # Download an ungated model from the HuggingFace Hub + # Download an ungated model from the Hugging Face Hub $ tune download mistralai/Mistral-7B-Instruct-v0.2 Successfully downloaded model repo and wrote to the following locations: ./model/config.json @@ -45,7 +45,7 @@ def __init__(self, subparsers: argparse._SubParsersAction): ./model/model-00001-of-00002.bin ... - For a list of all models, visit the HuggingFace Hub https://huggingface.co/models. + For a list of all models, visit the Hugging Face Hub https://huggingface.co/models. """ ), formatter_class=argparse.RawTextHelpFormatter, @@ -58,7 +58,7 @@ def _add_arguments(self) -> None: self._parser.add_argument( "repo_id", type=str, - help="Name of the repository on HuggingFace Hub.", + help="Name of the repository on Hugging Face Hub.", ) self._parser.add_argument( "--output-dir", @@ -72,7 +72,7 @@ def _add_arguments(self) -> None: type=str, required=False, default=os.getenv("HF_TOKEN", None), - help="HuggingFace API token. Needed for gated models like Llama2.", + help="Hugging Face API token. Needed for gated models like Llama2.", ) self._parser.add_argument( "--ignore-patterns", @@ -84,7 +84,7 @@ def _add_arguments(self) -> None: ) def _download_cmd(self, args: argparse.Namespace) -> None: - """Downloads a model from the HuggingFace Hub.""" + """Downloads a model from the Hugging Face Hub.""" # Download the tokenizer and PyTorch model files try: true_output_dir = snapshot_download( @@ -96,13 +96,13 @@ def _download_cmd(self, args: argparse.Namespace) -> None: except GatedRepoError: self._parser.error( "It looks like you are trying to access a gated repository. Please ensure you " - "have access to the repository and have provided the proper HuggingFace API token " + "have access to the repository and have provided the proper Hugging Face API token " "using the option `--hf-token` or by running `huggingface-cli login`." "You can find your token by visiting https://huggingface.co/settings/tokens" ) except RepositoryNotFoundError: self._parser.error( - f"Repository '{args.repo_id}' not found on the HuggingFace Hub." + f"Repository '{args.repo_id}' not found on the Hugging Face Hub." ) except Exception as e: self._parser.error(e) diff --git a/torchtune/data/_templates.py b/torchtune/data/_templates.py index 549edf0306..8c99f9e75c 100644 --- a/torchtune/data/_templates.py +++ b/torchtune/data/_templates.py @@ -240,7 +240,7 @@ class ChatMLTemplate(PromptTemplate): """ OpenAI's Chat Markup Language used by their chat models: https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/ai-services/openai/includes/chat-markup-language.md - It is the default template used by HuggingFace models. + It is the default template used by Hugging Face models. Example: <|im_start|>system diff --git a/torchtune/datasets/_alpaca.py b/torchtune/datasets/_alpaca.py index 68ca723f27..184007174c 100644 --- a/torchtune/datasets/_alpaca.py +++ b/torchtune/datasets/_alpaca.py @@ -15,7 +15,7 @@ def alpaca_dataset( use_clean: bool = False, ) -> InstructDataset: """ - Support for the Alpaca dataset and its variants from HuggingFace Datasets. + Support for the Alpaca dataset and its variants from Hugging Face Datasets. https://huggingface.co/datasets/tatsu-lab/alpaca Data input format: https://huggingface.co/datasets/tatsu-lab/alpaca#data-instances diff --git a/torchtune/datasets/_chat.py b/torchtune/datasets/_chat.py index 6390c19ca0..19c6b50557 100644 --- a/torchtune/datasets/_chat.py +++ b/torchtune/datasets/_chat.py @@ -45,7 +45,7 @@ class ChatDataset(Dataset): Args: tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. - source (str): path string of dataset, anything supported by HuggingFace's `load_dataset` + source (str): path string of dataset, anything supported by Hugging Face's `load_dataset` (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) convert_to_dialogue (Callable[[Mapping[str, Any]], Dialogue]): function that keys into the desired field in the sample and converts to a list of `Messages` that follows the llama format with the expected keys @@ -151,7 +151,7 @@ def chat_dataset( Args: tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. - source (str): path string of dataset, anything supported by HuggingFace's `load_dataset` + source (str): path string of dataset, anything supported by Hugging Face's `load_dataset` (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) conversation_format (str): string specifying expected format of conversations in the dataset for automatic conversion to the llama format. Supported formats are: "sharegpt" diff --git a/torchtune/datasets/_grammar.py b/torchtune/datasets/_grammar.py index dfd3c2ba2b..ffef93424a 100644 --- a/torchtune/datasets/_grammar.py +++ b/torchtune/datasets/_grammar.py @@ -14,7 +14,7 @@ def grammar_dataset( train_on_input: bool = False, ) -> InstructDataset: """ - Support for the Grammar dataset and its variants from HuggingFace Datasets. + Support for the Grammar dataset and its variants from Hugging Face Datasets. https://huggingface.co/datasets/liweili/c4_200m Data input format: https://huggingface.co/datasets/liweili/c4_200m#description diff --git a/torchtune/datasets/_instruct.py b/torchtune/datasets/_instruct.py index 1e3e13de55..8c146ab871 100644 --- a/torchtune/datasets/_instruct.py +++ b/torchtune/datasets/_instruct.py @@ -34,7 +34,7 @@ class InstructDataset(Dataset): Args: tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. - source (str): path string of dataset, anything supported by HuggingFace's `load_dataset` + source (str): path string of dataset, anything supported by Hugging Face's `load_dataset` (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) template (PromptTemplate): template used to format the prompt. If the placeholder variable names in the template do not match the column/key names in the dataset, use `column_map` to map them. @@ -103,7 +103,7 @@ def instruct_dataset( Args: tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. - source (str): path string of dataset, anything supported by HuggingFace's `load_dataset` + source (str): path string of dataset, anything supported by Hugging Face's `load_dataset` (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) template (str): class name of template used to format the prompt. If the placeholder variable names in the template do not match the column/key names in the dataset, use `column_map` to map them. diff --git a/torchtune/datasets/_samsum.py b/torchtune/datasets/_samsum.py index e715940eec..5f35431d28 100644 --- a/torchtune/datasets/_samsum.py +++ b/torchtune/datasets/_samsum.py @@ -14,7 +14,7 @@ def samsum_dataset( train_on_input: bool = False, ) -> InstructDataset: """ - Support for the Summarize dataset and its variants from HuggingFace Datasets. + Support for the Summarize dataset and its variants from Hugging Face Datasets. https://huggingface.co/datasets/samsum Data input format: https://huggingface.co/datasets/samsum#data-fields diff --git a/torchtune/modules/lr_schedulers.py b/torchtune/modules/lr_schedulers.py index 53b938f32e..6f7dd18dc7 100644 --- a/torchtune/modules/lr_schedulers.py +++ b/torchtune/modules/lr_schedulers.py @@ -22,7 +22,7 @@ def get_cosine_schedule_with_warmup( 0.0 to lr over num_warmup_steps, then decreases to 0.0 on a cosine schedule over the remaining num_training_steps-num_warmup_steps (assuming num_cycles = 0.5). - This is based on the HuggingFace implementation + This is based on the Hugging Face implementation https://github.com/huggingface/transformers/blob/v4.23.1/src/transformers/optimization.py#L104. Args: From 0217bfa7d9fadbec18f2877438095740fd9fa34f Mon Sep 17 00:00:00 2001 From: Rafi Ayub <33648637+RdoubleA@users.noreply.github.com> Date: Sat, 30 Mar 2024 16:00:49 -0700 Subject: [PATCH 12/76] Configure max_seq_len in InstructDataset (#620) --- tests/test_utils.py | 13 ++++++- tests/torchtune/data/test_data_utils.py | 5 ++- tests/torchtune/datasets/test_chat_dataset.py | 9 ++++- .../datasets/test_instruct_dataset.py | 38 ++++++++++++++++--- torchtune/datasets/_alpaca.py | 6 +++ torchtune/datasets/_instruct.py | 23 ++++++++++- 6 files changed, 82 insertions(+), 12 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1c785a09d2..8f1be9a569 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -33,14 +33,23 @@ class DummyTokenizer: - def encode(self, text, **kwargs): + def encode(self, text, add_bos=True, add_eos=True, **kwargs): words = text.split() - return [len(word) for word in words] + tokens = [len(word) for word in words] + if add_bos: + tokens = [self.bos_id] + tokens + if add_eos: + tokens = tokens + [self.eos_id] + return tokens @property def eos_id(self): return -1 + @property + def bos_id(self): + return 0 + def get_assets_path(): return Path(__file__).parent / "assets" diff --git a/tests/torchtune/data/test_data_utils.py b/tests/torchtune/data/test_data_utils.py index bc6974a3f9..475c0cba00 100644 --- a/tests/torchtune/data/test_data_utils.py +++ b/tests/torchtune/data/test_data_utils.py @@ -13,8 +13,9 @@ def test_tokenize_prompt_and_response(): tokenizer = DummyTokenizer() prompt = "Instruction:\nThis is an instruction.\n\nInput:\nThis is an input.\n\nResponse: " response = "I always know what I'm doing, do you?" - prompt_length = 11 + prompt_length = 12 expected_tokenized_prompt = [ + 0, 12, 4, 2, @@ -34,6 +35,7 @@ def test_tokenize_prompt_and_response(): 6, 2, 4, + -1, ] expected_tokenized_label = [CROSS_ENTROPY_IGNORE_IDX] * prompt_length + [ 1, @@ -44,6 +46,7 @@ def test_tokenize_prompt_and_response(): 6, 2, 4, + -1, ] tokenized_prompt, tokenized_label = tokenize_prompt_and_response( diff --git a/tests/torchtune/datasets/test_chat_dataset.py b/tests/torchtune/datasets/test_chat_dataset.py index ee9e122b79..66deb2c2c6 100644 --- a/tests/torchtune/datasets/test_chat_dataset.py +++ b/tests/torchtune/datasets/test_chat_dataset.py @@ -126,6 +126,7 @@ def test_get_item(self, mock_load_dataset, template, dialogue): mock_load_dataset.return_value = dialogue expected_tokenized_prompts = [ [ + 0, 7, 3, 3, @@ -146,15 +147,18 @@ def test_get_item(self, mock_load_dataset, template, dialogue): 4, 2, 3, + -1, + 0, 5, 6, 11, 10, 1, + 6, -1, ] ] - prompt_lengths = (14, 4) + prompt_lengths = (15, 5) expected_labels = [ [CROSS_ENTROPY_IGNORE_IDX] * prompt_lengths[0] + [ @@ -164,9 +168,10 @@ def test_get_item(self, mock_load_dataset, template, dialogue): 4, 2, 3, + -1, ] + [CROSS_ENTROPY_IGNORE_IDX] * prompt_lengths[1] - + [1, -1] + + [1, 6, -1] ] ds = ChatDataset( diff --git a/tests/torchtune/datasets/test_instruct_dataset.py b/tests/torchtune/datasets/test_instruct_dataset.py index 56f3a61b43..b31e8a2aec 100644 --- a/tests/torchtune/datasets/test_instruct_dataset.py +++ b/tests/torchtune/datasets/test_instruct_dataset.py @@ -32,8 +32,34 @@ class TestInstructDataset: "Instruction:\n{instruction}\n\nInput:\n{input}\n\nResponse: " ) expected_tokenized_prompts = [ - [12, 4, 2, 3, 2, 12, 10, 6, 4, 2, 3, 2, 6, 10, 9, 1, 5, 4, 4, 3, 6, 2, 4], - [12, 4, 2, 2, 12, 10, 6, 4, 2, 2, 6, 10, 9, 1, 6, 4, 4, 3, 6, 2, 4], + [ + 0, + 12, + 4, + 2, + 3, + 2, + 12, + 10, + 6, + 4, + 2, + 3, + 2, + 6, + 10, + 9, + 1, + 5, + 4, + 4, + 3, + 6, + 2, + 4, + -1, + ], + [0, 12, 4, 2, 2, 12, 10, 6, 4, 2, 2, 6, 10, 9, 1, 6, 4, 4, 3, 6, 2, 4, -1], ] def get_samples(self): @@ -53,10 +79,12 @@ def get_samples(self): @mock.patch("torchtune.datasets._instruct.load_dataset") def test_get_item_no_train_on_input(self, mock_load_dataset): mock_load_dataset.return_value = self.get_samples() - prompt_lengths = (15, 13) + prompt_lengths = (16, 14) expected_labels = [ - [CROSS_ENTROPY_IGNORE_IDX] * prompt_lengths[0] + [1, 5, 4, 4, 3, 6, 2, 4], - [CROSS_ENTROPY_IGNORE_IDX] * prompt_lengths[1] + [1, 6, 4, 4, 3, 6, 2, 4], + [CROSS_ENTROPY_IGNORE_IDX] * prompt_lengths[0] + + [1, 5, 4, 4, 3, 6, 2, 4, -1], + [CROSS_ENTROPY_IGNORE_IDX] * prompt_lengths[1] + + [1, 6, 4, 4, 3, 6, 2, 4, -1], ] dataset = InstructDataset( diff --git a/torchtune/datasets/_alpaca.py b/torchtune/datasets/_alpaca.py index 184007174c..399fbf83aa 100644 --- a/torchtune/datasets/_alpaca.py +++ b/torchtune/datasets/_alpaca.py @@ -13,6 +13,7 @@ def alpaca_dataset( tokenizer: Tokenizer, train_on_input: bool = True, use_clean: bool = False, + max_seq_len: int = 512, ) -> InstructDataset: """ Support for the Alpaca dataset and its variants from Hugging Face Datasets. @@ -39,6 +40,10 @@ def alpaca_dataset( tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. train_on_input (bool): Whether the model is trained on the prompt or not. Default is True. use_clean (bool): Whether to use the cleaned version of the dataset or not. Default is False. + max_seq_len (int): Maximum number of tokens in the returned input and label token id lists. + Default is 512, as set by Stanford Alpaca (https://github.com/tatsu-lab/stanford_alpaca?tab=readme-ov-file#fine-tuning), + but we recommend setting this to the highest you can fit in memory and is supported by the model. + For example, llama2-7B supports up to 4096 for sequence length. Returns: InstructDataset: dataset configured with Alpaca source data and template @@ -56,5 +61,6 @@ def alpaca_dataset( source="yahma/alpaca-cleaned" if use_clean else "tatsu-lab/alpaca", template=AlpacaInstructTemplate(), train_on_input=train_on_input, + max_seq_len=max_seq_len, split="train", ) diff --git a/torchtune/datasets/_instruct.py b/torchtune/datasets/_instruct.py index 8c146ab871..e0c14e88b6 100644 --- a/torchtune/datasets/_instruct.py +++ b/torchtune/datasets/_instruct.py @@ -11,7 +11,7 @@ from torchtune.config._utils import _get_template -from torchtune.data import PromptTemplate, tokenize_prompt_and_response +from torchtune.data import PromptTemplate, tokenize_prompt_and_response, truncate from torchtune.modules import Tokenizer @@ -43,6 +43,9 @@ class InstructDataset(Dataset): column_map (Optional[Dict[str, str]]): a mapping from the expected placeholder names in the template to the column/key names in the sample. If None, assume these are identical. train_on_input (bool): Whether the model is trained on the prompt or not. Default is False. + max_seq_len (Optional[int]): Maximum number of tokens in the returned input and label token id lists. + Default is None, disabling truncation. We recommend setting this to the highest you can fit in memory + and is supported by the model. For example, llama2-7B supports up to 4096 for sequence length. **load_dataset_kwargs (Dict[str, Any]): additional keyword arguments to pass to `load_dataset`. """ @@ -54,6 +57,7 @@ def __init__( transform: Optional[Callable] = None, column_map: Optional[Dict[str, str]] = None, train_on_input: bool = False, + max_seq_len: Optional[int] = None, **load_dataset_kwargs: Dict[str, Any], ) -> None: self._tokenizer = tokenizer @@ -62,6 +66,7 @@ def __init__( self._transform = transform self._column_map = column_map self.train_on_input = train_on_input + self.max_seq_len = max_seq_len def __len__(self): return len(self._data) @@ -80,13 +85,22 @@ def _prepare_sample(self, sample: Mapping[str, Any]) -> Tuple[List[int], List[in else "output" ) - return tokenize_prompt_and_response( + prompt_tokens, label_tokens = tokenize_prompt_and_response( tokenizer=self._tokenizer, prompt=prompt, response=transformed_sample[key_output], train_on_input=self.train_on_input, ) + if self.max_seq_len is not None: + prompt_tokens, label_tokens = truncate( + self._tokenizer, prompt_tokens, label_tokens, self.max_seq_len + ) + + assert len(prompt_tokens) == len(label_tokens) + + return prompt_tokens, label_tokens + def instruct_dataset( tokenizer: Tokenizer, @@ -94,6 +108,7 @@ def instruct_dataset( template: str, column_map: Optional[Dict[str, str]] = None, train_on_input: bool = False, + max_seq_len: Optional[int] = None, **load_dataset_kwargs: Dict[str, Any], ) -> InstructDataset: """ @@ -110,6 +125,9 @@ def instruct_dataset( column_map (Optional[Dict[str, str]]): a mapping from the expected placeholder names in the template to the column/key names in the sample. If None, assume these are identical. train_on_input (bool): Whether the model is trained on the prompt or not. Default is False. + max_seq_len (Optional[int]): Maximum number of tokens in the returned input and label token id lists. + Default is None, disabling truncation. We recommend setting this to the highest you can fit in memory + and is supported by the model. For example, llama2-7B supports up to 4096 for sequence length. **load_dataset_kwargs (Dict[str, Any]): additional keyword arguments to pass to `load_dataset`. Returns: @@ -121,5 +139,6 @@ def instruct_dataset( template=_get_template(template), column_map=column_map, train_on_input=train_on_input, + max_seq_len=max_seq_len, **load_dataset_kwargs, ) From 98ae83044ed01395bfc9188c4105773453beb049 Mon Sep 17 00:00:00 2001 From: Kartikay Khandelwal <47255723+kartikayk@users.noreply.github.com> Date: Sun, 31 Mar 2024 16:52:02 -0700 Subject: [PATCH 13/76] Inference (#619) --- docs/source/api_ref_utilities.rst | 10 - recipes/alpaca_generate.py | 86 ------ recipes/configs/alpaca_generate.yaml | 22 -- recipes/configs/generate.yaml | 31 +++ recipes/generate.py | 101 ++++++++ tests/recipes/test_alpaca_generate.py | 35 --- tests/recipes/utils.py | 2 - tests/torchtune/generation/test_generation.py | 213 ++------------- tests/torchtune/modules/test_attention.py | 56 ++-- .../modules/test_position_embeddings.py | 21 +- .../modules/test_transformer_decoder.py | 7 +- .../torchtune/utils/test_logits_transforms.py | 86 ------ torchtune/_recipe_registry.py | 6 +- .../models/llama2/_component_builders.py | 38 +-- torchtune/models/llama2/_model_builders.py | 10 +- .../models/mistral/_component_builders.py | 6 + torchtune/modules/attention.py | 23 +- torchtune/modules/kv_cache.py | 56 ++-- torchtune/modules/position_embeddings.py | 14 +- torchtune/modules/transformer.py | 87 +++++-- torchtune/utils/__init__.py | 1 + torchtune/utils/_generation.py | 139 ++++++++++ torchtune/utils/generation.py | 245 ------------------ torchtune/utils/logits_transforms.py | 95 ------- 24 files changed, 483 insertions(+), 907 deletions(-) delete mode 100644 recipes/alpaca_generate.py delete mode 100644 recipes/configs/alpaca_generate.yaml create mode 100644 recipes/configs/generate.yaml create mode 100644 recipes/generate.py delete mode 100644 tests/recipes/test_alpaca_generate.py delete mode 100644 tests/torchtune/utils/test_logits_transforms.py create mode 100644 torchtune/utils/_generation.py delete mode 100644 torchtune/utils/generation.py delete mode 100644 torchtune/utils/logits_transforms.py diff --git a/docs/source/api_ref_utilities.rst b/docs/source/api_ref_utilities.rst index 0194b05cc5..50f548716e 100644 --- a/docs/source/api_ref_utilities.rst +++ b/docs/source/api_ref_utilities.rst @@ -67,16 +67,6 @@ Data .. _gen_label: -Generation ----------- - -.. autosummary:: - :toctree: generated/ - :nosignatures: - - generation.GenerationUtils - generation.generate_from_prompt - Miscellaneous ------------- diff --git a/recipes/alpaca_generate.py b/recipes/alpaca_generate.py deleted file mode 100644 index cffe2a753f..0000000000 --- a/recipes/alpaca_generate.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -import sys - -import torch -from omegaconf import DictConfig - -from torchtune import config -from torchtune.utils import get_device, get_logger, set_seed -from torchtune.utils.generation import GenerationUtils - -# From https://github.com/tatsu-lab/stanford_alpaca/blob/761dc5bfbdeeffa89b8bff5d038781a4055f796a/train.py#L31 -PROMPT_DICT = { - "prompt_input": ( - "Below is an instruction that describes a task, paired with an input that provides further context. " - "Write a response that appropriately completes the request.\n\n" - "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:" - ), - "prompt_no_input": ( - "Below is an instruction that describes a task. " - "Write a response that appropriately completes the request.\n\n" - "### Instruction:\n{instruction}\n\n### Response:" - ), -} - - -def recipe( - cfg: DictConfig, -): - logger = get_logger("DEBUG") - - # Inference setup - tokenizer = config.instantiate(cfg.tokenizer) - - example = {"instruction": cfg.instruction} - if cfg.input != "": - example["input"] = cfg.input - prompt = PROMPT_DICT["prompt_input"].format_map(example) - else: - prompt = PROMPT_DICT["prompt_no_input"].format_map(example) - - token_for_generation = [tokenizer.encode(prompt, add_eos=False)] - - set_seed() - - device = get_device() - - with device: - decoder = config.instantiate(cfg.model, max_batch_size=1) - - # Load state_dict into decoder - native_state_dict = torch.load(cfg.model_checkpoint, weights_only=True) - missing, unexpected = decoder.load_state_dict(native_state_dict, strict=False) - - decoder.eval() - - with torch.no_grad(): - generations, _ = GenerationUtils( - decoder_lm=decoder, - eos_id=tokenizer.eos_id, - pad_id=tokenizer.pad_id, - ).generate( - prompt_tokens=token_for_generation, - incremental_decode=True, - min_gen_len=1, - max_gen_len=cfg.max_gen_len, - top_p=0, - top_k=1, - temperature=1.0, - device=device, - ) - - generated_tokens = tokenizer.decode(generations.tolist()) - logger.info(msg=generated_tokens[0]) - - -@config.parse -def main(cfg: DictConfig) -> None: - recipe(cfg) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/recipes/configs/alpaca_generate.yaml b/recipes/configs/alpaca_generate.yaml deleted file mode 100644 index a6ba65a3e4..0000000000 --- a/recipes/configs/alpaca_generate.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Config alpaca_generate.py recipe -# -# To launch, run the following command from root: -# tune alpaca_generate \ -# --config generate \ -# model_checkpoint= \ -# tokenizer_checkpoint= - -# Model arguments -model: - _component_: torchtune.models.llama2.llama2_7b -model_checkpoint: /tmp/llama2_native - -# Tokenizer arguments -tokenizer: - _component_: torchtune.models.llama2.llama2_tokenizer - path: /tmp/llama2/tokenizer.model - -# Generation arguments -instruction: "Answer the question." -input: "What is some cool music from the 1920s?" -max_gen_len: 64 diff --git a/recipes/configs/generate.yaml b/recipes/configs/generate.yaml new file mode 100644 index 0000000000..2865f33d42 --- /dev/null +++ b/recipes/configs/generate.yaml @@ -0,0 +1,31 @@ + +# Model arguments +model: + _component_: torchtune.models.llama2.llama2_13b + +checkpointer: + _component_: torchtune.utils.FullModelHFCheckpointer + checkpoint_dir: /tmp/Llama-2-13b-hf/ + checkpoint_files: [ + pytorch_model-00001-of-00003.bin, + pytorch_model-00002-of-00003.bin, + pytorch_model-00003-of-00003.bin + ] + output_dir: /tmp/Llama-2-13b-hf/ + model_type: LLAMA2 + +device: cuda +dtype: bf16 + +seed: 1234 + +# Tokenizer arguments +tokenizer: + _component_: torchtune.models.llama2.llama2_tokenizer + path: /tmp/Llama-2-13b-hf/tokenizer.model + +# Generation arguments; defaults taken from gpt-fast +prompt: "Hello, my name is" +max_new_tokens: 300 +temperature: 0.8 +top_k: 300 diff --git a/recipes/generate.py b/recipes/generate.py new file mode 100644 index 0000000000..8a3043229d --- /dev/null +++ b/recipes/generate.py @@ -0,0 +1,101 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +import sys +import time +from typing import Any, Dict + +import torch +from omegaconf import DictConfig + +from torch import nn + +from torchtune import config, utils + +logger = utils.get_logger("DEBUG") + + +class InferenceRecipe: + """ + Recipe for generating tokens from a dense Transformer-based LLM. + + Currently this recipe support single-GPU generation only. Speculative + decoding is not supported. + """ + + def __init__(self, cfg: DictConfig) -> None: + self._device = utils.get_device(device=cfg.device) + self._dtype = utils.get_dtype(dtype=cfg.dtype) + + utils.set_seed(seed=cfg.seed) + + def load_checkpoint(self, checkpointer_cfg: DictConfig) -> Dict[str, Any]: + checkpointer = config.instantiate(checkpointer_cfg) + checkpoint_dict = checkpointer.load_checkpoint() + return checkpoint_dict + + def setup(self, cfg: DictConfig) -> None: + ckpt_dict = self.load_checkpoint(cfg.checkpointer) + self._model = self._setup_model( + model_cfg=cfg.model, + model_state_dict=ckpt_dict[utils.MODEL_KEY], + ) + self._tokenizer = config.instantiate(cfg.tokenizer) + + def _setup_model( + self, + model_cfg: DictConfig, + model_state_dict: Dict[str, Any], + ) -> nn.Module: + with utils.set_default_dtype(self._dtype), self._device: + model = config.instantiate(model_cfg) + + model.load_state_dict(model_state_dict) + + # Validate model was loaded in with the expected dtype. + utils.validate_expected_param_dtype(model.named_parameters(), dtype=self._dtype) + logger.info(f"Model is initialized with precision {self._dtype}.") + + # Ensure the cache is setup on the right device + with self._device: + model.setup_caches(max_batch_size=1, dtype=self._dtype) + + return model + + @torch.no_grad() + def generate(self, cfg: DictConfig): + tokens = self._tokenizer.encode(cfg.prompt, add_bos=True, add_eos=False) + prompt = torch.tensor(tokens, dtype=torch.int, device=self._device) + + t0 = time.perf_counter() + generated_tokens = utils.generate( + model=self._model, + prompt=prompt, + max_generated_tokens=cfg.max_new_tokens, + temperature=cfg.temperature, + top_k=cfg.top_k, + eos_id=self._tokenizer.eos_id, + ) + t = time.perf_counter() - t0 + + logger.info(self._tokenizer.decode(generated_tokens)) + + tokens_generated = len(generated_tokens) - prompt.size(0) + tokens_sec = tokens_generated / t + logger.info( + f"Time for inference: {t:.02f} sec total, {tokens_sec:.02f} tokens/sec" + ) + logger.info(f"Memory used: {torch.cuda.max_memory_allocated() / 1e9:.02f} GB") + + +@config.parse +def main(cfg: DictConfig) -> None: + recipe = InferenceRecipe(cfg=cfg) + recipe.setup(cfg=cfg) + recipe.generate(cfg=cfg) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/recipes/test_alpaca_generate.py b/tests/recipes/test_alpaca_generate.py deleted file mode 100644 index 7a09751fac..0000000000 --- a/tests/recipes/test_alpaca_generate.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import runpy -import sys - -import pytest - -from tests.common import TUNE_PATH -from tests.recipes.utils import llama2_test_config -from tests.test_utils import CKPT_MODEL_PATHS - - -class TestAlpacaGenerateRecipe: - @pytest.mark.integration_test - def test_alpaca_generate(self, tmpdir, monkeypatch): - ckpt = "small_test_ckpt_tune" - model_checkpoint = CKPT_MODEL_PATHS[ckpt] - cmd = f""" - tune run alpaca_generate \ - --config alpaca_generate \ - model_checkpoint={model_checkpoint} \ - tokenizer.path=/tmp/test-artifacts/tokenizer.model \ - output_dir={tmpdir} \ - """.split() - - model_config = llama2_test_config() - cmd += model_config - - monkeypatch.setattr(sys, "argv", cmd) - with pytest.raises(SystemExit, match=""): - runpy.run_path(TUNE_PATH, run_name="__main__") diff --git a/tests/recipes/utils.py b/tests/recipes/utils.py index 01b9343825..ce5767ed27 100644 --- a/tests/recipes/utils.py +++ b/tests/recipes/utils.py @@ -48,7 +48,6 @@ def llama2_test_config(max_batch_size: Optional[int] = None) -> List[str]: "model.max_seq_len=2048", "model.norm_eps=1e-5", "model.num_kv_heads=8", - f"model.max_batch_size={max_batch_size if max_batch_size else 'null'}", ] @@ -75,7 +74,6 @@ def lora_llama2_test_config( "model.max_seq_len=2048", "model.norm_eps=1e-5", "model.num_kv_heads=8", - f"model.max_batch_size={max_batch_size if max_batch_size else 'null'}", f"model.lora_rank={lora_rank}", f"model.lora_alpha={lora_alpha}", "model.lora_dropout=0.0", diff --git a/tests/torchtune/generation/test_generation.py b/tests/torchtune/generation/test_generation.py index 6f01122709..bddc99eca9 100644 --- a/tests/torchtune/generation/test_generation.py +++ b/tests/torchtune/generation/test_generation.py @@ -12,11 +12,11 @@ import pytest import torch -from tests.test_utils import assert_expected, init_weights_with_constant, set_dtype +from tests.test_utils import init_weights_with_constant +from torchtune import utils from torchtune.models.llama2 import llama2 -from torchtune.utils.generation import GenerationUtils -from torchtune.utils.seed import set_seed +from torchtune.utils._generation import sample @pytest.fixture(autouse=True) @@ -41,7 +41,7 @@ def prevent_leaking_rng(): @pytest.fixture(autouse=True) def random_seed(): - set_seed(42) + utils.set_seed(42) _test_pad_id = -1 @@ -67,39 +67,21 @@ class TestTextGenerate: Test class for text generation functionality. """ - @property - def _batch_size(self): - return 2 - - def _get_generation_model(self, use_kv_cache): + @pytest.fixture + def generation_model(self, dtype=torch.float32): model = llama2( vocab_size=4_000, embed_dim=128, num_layers=2, num_heads=4, - num_kv_heads=None, + num_kv_heads=4, max_seq_len=2048, - max_batch_size=None if not use_kv_cache else 2, ) init_weights_with_constant(model) + model.setup_caches(max_batch_size=1, dtype=dtype) model.eval() return model - @pytest.fixture - def generation_model(self): - """ - A dummy model to test `generate` API - """ - return self._get_generation_model(use_kv_cache=False) - - @pytest.fixture - def generation_model_kv_cache(self): - """ - A dummy model to test incremental decoding portion of `generate` API - w/kv-caching enabled. - """ - return self._get_generation_model(use_kv_cache=True) - @pytest.fixture def prompt_tokens(self) -> List[int]: """ @@ -107,186 +89,43 @@ def prompt_tokens(self) -> List[int]: Returns: A list of prompt tokens. """ - return [list(range(2, 10)) for _ in range(self._batch_size)] + return torch.arange(2, 10) - def test_different_len_prompts_in_batch(self, generation_model): + def test_sample_consistency(self): """ - Test to check if the `generate` function can handle prompts of different lengths in a batch. + Test token sampling produces the right output. """ - prompt_tokens = [ - [1], - [8, 9], - [4, 5, 6], - [7, 8, 9, 20], - ] - min_gen_len = 1 - max_gen_len = 1 - temperature = 1.0 - top_p = 1.0 - top_k = 0 - generate = _make_generate(generation_model) - outputs_actual, _ = generate( - prompt_tokens=prompt_tokens, - min_gen_len=min_gen_len, - max_gen_len=max_gen_len, - temperature=temperature, - top_p=top_p, - top_k=top_k, - incremental_decode=False, - ) - # Since keep_prompt=True by default, each generation should have - # its prompt at the beginning. - expected_prompt_lens = [len(prompt) for prompt in prompt_tokens] - assert len(expected_prompt_lens) == len(outputs_actual) - for i, (expected_len, generation) in enumerate( - zip(expected_prompt_lens, outputs_actual) - ): - generation_tokens = generation.tolist() - expected_prompt = generation_tokens[:expected_len] - assert_expected(expected_prompt, prompt_tokens[i]) - for tok in generation_tokens: - assert tok not in (_test_pad_id,) + # set all probabilities except for token_id=100 to 0 + logits = torch.zeros(2000) + logits[100] = 1 - def test_no_keep_prompt(self, generation_model): - """ - Test to check if the `generate` function works correctly when `keep_prompt` is set to False. - """ - prompt_tokens = [ - [1], - [8, 9], - [4, 5, 6], - [7, 8, 9, 20], - ] - min_gen_len = 1 - max_gen_len = 1 - temperature = 1.0 - top_p = 1.0 - top_k = 0 - generate = _make_generate(generation_model) - outputs_actual, _ = generate( - prompt_tokens=prompt_tokens, - min_gen_len=min_gen_len, - max_gen_len=max_gen_len, - temperature=temperature, - top_p=top_p, - top_k=top_k, - keep_prompt=False, - incremental_decode=False, - ) - for generation in outputs_actual: - generation = generation.tolist() - assert_expected(len(generation), max_gen_len) - - @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires cuda") - def test_cuda_device(self, generation_model, prompt_tokens): - """ - Test to check if the `generate` function reutnrs outputs on the expected CUDA device. - """ - min_gen_len = 1 - max_gen_len = 1 - generation_model.cuda() - generate = _make_generate(generation_model) - outputs = generate( - prompt_tokens=prompt_tokens, - min_gen_len=min_gen_len, - max_gen_len=max_gen_len, - logprobs=True, - device="cuda", - incremental_decode=False, - ) - assert outputs[0].device == torch.device("cuda", 0) - assert outputs[1].device == torch.device("cuda", 0) - - def test_token_logprobs(self, generation_model, prompt_tokens): - """ - Test to check if the `generate` function returns expected type for token_logprobs. - """ - min_gen_len = 1 - max_gen_len = 1 - generate = _make_generate(generation_model) - outputs = generate( - prompt_tokens=prompt_tokens, - min_gen_len=min_gen_len, - max_gen_len=max_gen_len, - logprobs=True, - incremental_decode=False, - ) - assert_expected(outputs[0].shape, outputs[1].shape) - assert isinstance(outputs[1], torch.FloatTensor) - - @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) - def test_kv_cache_incremental_decode_parity(self, prompt_tokens, dtype): - """ - Test to check if the `generate` function produces the same output when run with and without - incremental decoding, where we use a kv-caching model with incremental_decode=True. - """ - with set_dtype(dtype): - min_gen_len = 1 - max_gen_len = 20 - temperature = 1.0 - top_p = 1.0 - top_k = 0 - gen_model = self._get_generation_model(use_kv_cache=False) - gen_model_kv = self._get_generation_model(use_kv_cache=True) - generate = _make_generate(gen_model) - generate_kv_cache = _make_generate(gen_model_kv) - outputs, _ = generate( - prompt_tokens=prompt_tokens, - min_gen_len=min_gen_len, - max_gen_len=max_gen_len, - temperature=temperature, - top_p=top_p, - top_k=top_k, - keep_prompt=False, - incremental_decode=False, - ) - outputs_kv_cache, _ = generate_kv_cache( - prompt_tokens=prompt_tokens, - min_gen_len=min_gen_len, - max_gen_len=max_gen_len, - temperature=temperature, - top_p=top_p, - top_k=top_k, - keep_prompt=False, - incremental_decode=True, - ) - assert outputs.tolist() == outputs_kv_cache.tolist() + token = sample(logits, temperature=1, top_k=1) + assert token.item() == 100 def test_reproducibility(self, generation_model, prompt_tokens): """ Test to check if the `generate` function produces the same output when run with the same inputs and a fixed seed. """ - min_gen_len = 1 - max_gen_len = 20 - # Use real values to test reproducibility of some of the transforms temperature = 0.6 - top_p = 0.9 - top_k = 0 - generate = _make_generate(generation_model) + top_k = 100 torch.manual_seed(42) - outputs_first, _ = generate( - prompt_tokens=prompt_tokens, - min_gen_len=min_gen_len, - max_gen_len=max_gen_len, + outputs_first = utils.generate( + model=generation_model, + prompt=prompt_tokens, + max_generated_tokens=10, temperature=temperature, - top_p=top_p, top_k=top_k, - keep_prompt=False, - incremental_decode=False, ) torch.manual_seed(42) - outputs_second, _ = generate( - prompt_tokens=prompt_tokens, - min_gen_len=min_gen_len, - max_gen_len=max_gen_len, + outputs_second = utils.generate( + model=generation_model, + prompt=prompt_tokens, + max_generated_tokens=10, temperature=temperature, - top_p=top_p, top_k=top_k, - keep_prompt=False, - incremental_decode=False, ) - assert outputs_first.tolist() == outputs_second.tolist() + assert outputs_first == outputs_second diff --git a/tests/torchtune/modules/test_attention.py b/tests/torchtune/modules/test_attention.py index 5c90e0c424..34f5d637fe 100644 --- a/tests/torchtune/modules/test_attention.py +++ b/tests/torchtune/modules/test_attention.py @@ -32,12 +32,6 @@ class TestCausalSelfAttention: https://github.com/facebookresearch/llama/blob/main/llama/model.py#L450 """ - def _get_mask(self, inpt: Tensor) -> Tensor: - seq_len = inpt.shape[1] - mask = torch.full((1, 1, seq_len, seq_len), float("-inf"), device=inpt.device) - mask = torch.triu(mask, diagonal=1).type_as(inpt) - return mask - @pytest.fixture def input_params(self) -> Tuple[int, int, int]: batch_size = 4 @@ -129,8 +123,9 @@ def gqa_kv_cache( kv_cache = KVCache( max_batch_size=4, max_seq_len=max_seq_len, - n_kv_heads=num_heads, + num_heads=num_heads, head_dim=head_dim, + dtype=torch.float32, ) rope = RotaryPositionalEmbeddings(dim=head_dim, max_seq_len=max_seq_len) attn = CausalSelfAttention( @@ -182,8 +177,9 @@ def mha_kv_cache( kv_cache = KVCache( max_batch_size=4, max_seq_len=max_seq_len, - n_kv_heads=num_heads, + num_heads=num_heads, head_dim=head_dim, + dtype=torch.float32, ) rope = RotaryPositionalEmbeddings(dim=head_dim, max_seq_len=max_seq_len) attn = CausalSelfAttention( @@ -241,8 +237,9 @@ def mqa_kv_cache( kv_cache = KVCache( max_batch_size=4, max_seq_len=max_seq_len, - n_kv_heads=num_heads, + num_heads=num_heads, head_dim=head_dim, + dtype=torch.float32, ) rope = RotaryPositionalEmbeddings(dim=head_dim, max_seq_len=max_seq_len) attn = CausalSelfAttention( @@ -271,14 +268,18 @@ def test_forward_gqa(self, input: Tensor, gqa: CausalSelfAttention) -> None: assert_expected(output.shape, input.shape) def test_forward_gqa_kv_cache( - self, input: Tensor, gqa_kv_cache: CausalSelfAttention + self, input: Tensor, gqa_kv_cache: CausalSelfAttention, attn_params_gqa ) -> None: - # seq_len = input.shape[1] - # mask = torch.full((1, 1, seq_len, seq_len), float("-inf"), device=input.device) - # mask = torch.triu(mask, diagonal=1).type_as(input) - mask = self._get_mask(input) + + _, _, _, max_seq_len = attn_params_gqa + _, seq_len, _ = input.shape + + causal_mask = torch.tril(torch.ones(max_seq_len, max_seq_len, dtype=torch.bool)) + input_pos = torch.arange(seq_len) + mask = causal_mask[None, None, input_pos] + with torch.no_grad(): - output = gqa_kv_cache(input, mask=mask, curr_pos=0) + output = gqa_kv_cache(input, mask=mask, input_pos=input_pos) assert_expected( output.mean(), torch.tensor(-2545.42236328125), atol=1e-8, rtol=1e-3 ) @@ -293,11 +294,18 @@ def test_forward_mha(self, input: Tensor, mha: CausalSelfAttention) -> None: assert_expected(output.shape, input.shape) def test_forward_mha_kv_cache( - self, input: Tensor, mha_kv_cache: CausalSelfAttention + self, input: Tensor, mha_kv_cache: CausalSelfAttention, attn_params_mha ) -> None: - mask = self._get_mask(input) + + _, _, _, max_seq_len = attn_params_mha + _, seq_len, _ = input.shape + + causal_mask = torch.tril(torch.ones(max_seq_len, max_seq_len, dtype=torch.bool)) + input_pos = torch.arange(seq_len) + mask = causal_mask[None, None, input_pos] + with torch.no_grad(): - output = mha_kv_cache(input, mask=mask, curr_pos=0) + output = mha_kv_cache(input, mask=mask, input_pos=input_pos) assert_expected( output.mean(), torch.tensor(-2597.248046875), atol=1e-8, rtol=1e-3 ) @@ -312,11 +320,17 @@ def test_forward_mqa(self, input: Tensor, mqa: CausalSelfAttention) -> None: assert_expected(output.shape, input.shape) def test_forward_mqa_kv_cache( - self, input: Tensor, mqa_kv_cache: CausalSelfAttention + self, input: Tensor, mqa_kv_cache: CausalSelfAttention, attn_params_mqa ) -> None: - mask = self._get_mask(input) + _, _, _, max_seq_len = attn_params_mqa + _, seq_len, _ = input.shape + + causal_mask = torch.tril(torch.ones(max_seq_len, max_seq_len, dtype=torch.bool)) + input_pos = torch.arange(seq_len) + mask = causal_mask[None, None, input_pos] + with torch.no_grad(): - output = mqa_kv_cache(input, mask=mask, curr_pos=0) + output = mqa_kv_cache(input, mask=mask, input_pos=input_pos) assert_expected( output.mean(), torch.tensor(-2108.076660156255), atol=1e-8, rtol=1e-3 ) diff --git a/tests/torchtune/modules/test_position_embeddings.py b/tests/torchtune/modules/test_position_embeddings.py index 626810c741..5c449289d5 100644 --- a/tests/torchtune/modules/test_position_embeddings.py +++ b/tests/torchtune/modules/test_position_embeddings.py @@ -65,12 +65,21 @@ def test_forward(self, input: tensor, rope: RotaryPositionalEmbeddings) -> None: def test_forward_with_curr_pos( self, input: tensor, rope: RotaryPositionalEmbeddings ) -> None: - x_out = rope(input, curr_pos=10) - - # check the numerics of the computed tensor - assert_expected(x_out.mean(), tensor(0.0002), atol=1e-4) - assert_expected(x_out.sum(), tensor(5158.3159)) - assert_expected(x_out.max(), tensor(5.4543)) + ( + _, + seq_len, + _, + _, + ) = input.shape + x_out = rope(input, input_pos=torch.arange(seq_len)) + + # these values should be exactly the same as test_forward + # since in this case input_pos covers the entire input + # sequence. This tests that input_pos works as expected i.e. + # extracts the embeddings for the relevant positions + assert_expected(x_out.mean(), tensor(6.4543e-05), atol=1e-4) + assert_expected(x_out.sum(), tensor(2165.7053)) + assert_expected(x_out.max(), tensor(5.4546)) # check shapes assert_expected(x_out.shape, input.shape) diff --git a/tests/torchtune/modules/test_transformer_decoder.py b/tests/torchtune/modules/test_transformer_decoder.py index f900ac8d7c..ccf177f4eb 100644 --- a/tests/torchtune/modules/test_transformer_decoder.py +++ b/tests/torchtune/modules/test_transformer_decoder.py @@ -201,11 +201,11 @@ def decoder_with_kv_cache_enabled( num_kv_heads=num_kv_heads, embed_dim=embed_dim, max_seq_len=max_seq_len, - max_batch_size=4, ) # TODO: fix weight initialization to use fixed_init_model init_weights_with_constant(decoder, constant=0.2) decoder.eval() + decoder.setup_caches(max_batch_size=4, dtype=torch.float32) return decoder def test_forward( @@ -234,8 +234,11 @@ def test_kv_cache( decoder_with_kv_cache_enabled: TransformerDecoder, decoder: TransformerDecoder, ) -> None: + _, seq_len = input.shape + input_pos = torch.arange(seq_len) + with torch.no_grad(): - output_cache = decoder_with_kv_cache_enabled(input, 0) + output_cache = decoder_with_kv_cache_enabled(input, input_pos=input_pos) output_no_cache = decoder(input) assert_expected(output_cache.mean(), output_no_cache.mean()) diff --git a/tests/torchtune/utils/test_logits_transforms.py b/tests/torchtune/utils/test_logits_transforms.py deleted file mode 100644 index 683a5bf24e..0000000000 --- a/tests/torchtune/utils/test_logits_transforms.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -import pytest - -import torch - -from tests.test_utils import assert_expected - -from torchtune.utils.logits_transforms import ( - TemperatureTransform, - TopKTransform, - TopPTransform, -) - - -class TestTemperatureTransform: - def test_invalid_temperature(self): - """ - Test if ValueError is raised when temperature is set to 0.0 - """ - with pytest.raises(ValueError): - _ = TemperatureTransform(temperature=0.0) - - def test_default(self): - """ - Test if the TemperatureTransform correctly transforms the logits - """ - temperature = 0.6 - transform = TemperatureTransform(temperature=temperature) - logits = torch.arange(0, 1, 0.1)[None] - - expected = logits / temperature - actual = transform(logits) - assert_expected(actual, expected) - - -class TestTopPTransform: - def test_invalid_p(self): - """ - Test if ValueError is raised when prob is set to -1.0, 0.0, or 2.0 - """ - for prob in (-1.0, 0.0, 2.0): - with pytest.raises(ValueError): - _ = TopPTransform(prob=prob) - - def test_default(self): - """ - Test if the TopPTransform correctly transforms the logits - """ - prob = 0.5 - transform = TopPTransform(prob=prob) - logits = torch.arange(0.1, 0.5, 0.1)[None] - - expected = torch.tensor([[0.0, 0.0, 0.3, 0.4]]) / 0.7 - actual = transform(logits) - assert_expected(actual, expected) - - -class TestTopKTransform: - def test_invalid_k(self): - """ - Test if ValueError is raised when top_k is set to -1 or TypeError is raised when top_k is set to 0.5 - """ - with pytest.raises(ValueError): - _ = TopKTransform(top_k=-1) - - with pytest.raises(TypeError): - _ = TopKTransform(top_k=0.5) - - def test_default(self): - """ - Test if the TopKTransform correctly transforms the logits - """ - top_k = 3 - transform = TopKTransform(top_k=top_k) - logits = torch.arange(0, 0.5, 0.1)[None] - - expected = torch.tensor([[0.0, 0.0, 0.2, 0.3, 0.4]]) / 0.9 - actual = transform(logits) - assert_expected(actual, expected) diff --git a/torchtune/_recipe_registry.py b/torchtune/_recipe_registry.py index 57ba562cbb..ae25713652 100644 --- a/torchtune/_recipe_registry.py +++ b/torchtune/_recipe_registry.py @@ -72,10 +72,10 @@ class Recipe: supports_distributed=True, ), Recipe( - name="alpaca_generate", - file_path="alpaca_generate.py", + name="generate", + file_path="generate.py", configs=[ - Config(name="alpaca_generate", file_path="alpaca_generate.yaml"), + Config(name="generate", file_path="generate.yaml"), ], supports_distributed=False, ), diff --git a/torchtune/models/llama2/_component_builders.py b/torchtune/models/llama2/_component_builders.py index 22d5a96023..476fc45bb7 100644 --- a/torchtune/models/llama2/_component_builders.py +++ b/torchtune/models/llama2/_component_builders.py @@ -49,7 +49,6 @@ def llama2( max_seq_len: int, attn_dropout: float = 0.0, intermediate_dim: Optional[int] = None, - max_batch_size: Optional[int] = None, norm_eps: float = 1e-5, ) -> TransformerDecoder: """ @@ -74,7 +73,6 @@ def llama2( Default: 0.0 intermediate_dim (Optional[int]): intermediate dimension for MLP. If not specified, this is computed using :func:`~torchtune.modules.scale_hidden_dim_for_mlp` - max_batch_size (Optional[int]): maximum batch size to be passed to :func:`~torchtune.modules.KVCache` norm_eps (float): epsilon in RMS norms. Returns: @@ -82,16 +80,7 @@ def llama2( """ head_dim = embed_dim // num_heads num_kv_heads = num_kv_heads if num_kv_heads else num_heads - kv_cache = ( - KVCache( - max_batch_size=max_batch_size, - max_seq_len=max_seq_len, - n_kv_heads=num_heads, - head_dim=head_dim, - ) - if max_batch_size is not None - else None - ) + rope = RotaryPositionalEmbeddings(dim=head_dim, max_seq_len=max_seq_len) self_attn = CausalSelfAttention( embed_dim=embed_dim, @@ -103,7 +92,7 @@ def llama2( v_proj=nn.Linear(embed_dim, num_kv_heads * head_dim, bias=False), output_proj=nn.Linear(embed_dim, embed_dim, bias=False), pos_embeddings=rope, - kv_cache=kv_cache, + kv_cache=None, max_seq_len=max_seq_len, attn_dropout=attn_dropout, ) @@ -121,6 +110,9 @@ def llama2( tok_embeddings=tok_embeddings, layer=layer, num_layers=num_layers, + max_seq_len=max_seq_len, + num_heads=num_heads, + head_dim=head_dim, norm=RMSNorm(embed_dim, eps=norm_eps), output=output_proj, ) @@ -153,7 +145,6 @@ def lora_llama2( max_seq_len: int, intermediate_dim: Optional[int] = None, attn_dropout: float = 0.0, - max_batch_size: Optional[int] = None, norm_eps: float = 1e-5, # LoRA args lora_rank: int, @@ -188,7 +179,6 @@ def lora_llama2( Default: 0.0 intermediate_dim (Optional[int]): intermediate dimension for MLP. If not specified, this is computed using :func:`~torchtune.modules.scale_hidden_dim_for_mlp` - max_batch_size (Optional[int]): maximum batch size to be passed to :func:`~torchtune.modules.KVCache` norm_eps (float): epsilon in RMS norms. lora_rank (int): rank of each low-rank approximation lora_alpha (float): scaling factor for the low-rank approximation @@ -210,7 +200,6 @@ def lora_llama2( num_kv_heads=num_kv_heads, max_seq_len=max_seq_len, attn_dropout=attn_dropout, - max_batch_size=max_batch_size, lora_rank=lora_rank, lora_alpha=lora_alpha, lora_dropout=lora_dropout, @@ -248,6 +237,9 @@ def lora_llama2( tok_embeddings=tok_embeddings, layer=layer, num_layers=num_layers, + max_seq_len=max_seq_len, + num_heads=num_heads, + head_dim=(embed_dim // num_heads), norm=RMSNorm(embed_dim, eps=norm_eps), output=output_proj, ) @@ -271,7 +263,6 @@ def lora_llama2_self_attention( num_kv_heads: int, max_seq_len: int, attn_dropout: float = 0.0, - max_batch_size: Optional[int] = None, # LoRA args lora_rank: int, lora_alpha: float, @@ -296,7 +287,6 @@ def lora_llama2_self_attention( by :func:`~torchtune.modules.KVCache` attn_dropout (float): dropout value passed onto scaled_dot_product_attention. Default: 0.0 - max_batch_size (Optional[int]): maximum batch size to be passed to :func:`~torchtune.modules.KVCache` lora_rank (int): rank of each low-rank approximation lora_alpha (float): scaling factor for the low-rank approximation lora_dropout (float): LoRA dropout probability. Default: 0.0 @@ -317,16 +307,6 @@ def lora_llama2_self_attention( head_dim = embed_dim // num_heads num_kv_heads = num_kv_heads if num_kv_heads else num_heads - kv_cache = ( - KVCache( - max_batch_size=max_batch_size, - max_seq_len=max_seq_len, - n_kv_heads=num_heads, - head_dim=head_dim, - ) - if max_batch_size is not None - else None - ) q_proj = ( LoRALinear( embed_dim, @@ -382,7 +362,7 @@ def lora_llama2_self_attention( v_proj=v_proj, output_proj=output_proj, pos_embeddings=rope, - kv_cache=kv_cache, + kv_cache=None, max_seq_len=max_seq_len, attn_dropout=attn_dropout, ) diff --git a/torchtune/models/llama2/_model_builders.py b/torchtune/models/llama2/_model_builders.py index 87f58db769..62ddb8e55d 100644 --- a/torchtune/models/llama2/_model_builders.py +++ b/torchtune/models/llama2/_model_builders.py @@ -22,7 +22,7 @@ """ -def llama2_7b(max_batch_size: Optional[int] = None) -> TransformerDecoder: +def llama2_7b() -> TransformerDecoder: """ Builder for creating a Llama2 model initialized w/ the default 7b parameter values from https://arxiv.org/abs/2307.09288 @@ -40,7 +40,6 @@ def llama2_7b(max_batch_size: Optional[int] = None) -> TransformerDecoder: num_kv_heads=32, embed_dim=4096, max_seq_len=4096, - max_batch_size=max_batch_size, attn_dropout=0.0, norm_eps=1e-5, ) @@ -59,7 +58,6 @@ def lora_llama2_7b( apply_lora_to_output: bool = False, lora_rank: int = 8, lora_alpha: float = 16, - max_batch_size: Optional[int] = None, quantize_base: bool = False, ) -> TransformerDecoder: """ @@ -95,7 +93,6 @@ def lora_llama2_7b( num_kv_heads=32, embed_dim=4096, max_seq_len=4096, - max_batch_size=max_batch_size, attn_dropout=0.0, norm_eps=1e-5, lora_rank=lora_rank, @@ -113,7 +110,7 @@ def lora_llama2_7b( """ -def llama2_13b(max_batch_size: Optional[int] = None) -> TransformerDecoder: +def llama2_13b() -> TransformerDecoder: """ Builder for creating a Llama2 model initialized w/ the default 13b parameter values from https://arxiv.org/abs/2307.09288 @@ -132,7 +129,6 @@ def llama2_13b(max_batch_size: Optional[int] = None) -> TransformerDecoder: embed_dim=5120, intermediate_dim=13824, max_seq_len=4096, - max_batch_size=max_batch_size, attn_dropout=0.0, norm_eps=1e-5, ) @@ -144,7 +140,6 @@ def lora_llama2_13b( apply_lora_to_output: bool = False, lora_rank: int = 8, lora_alpha: float = 16, - max_batch_size: Optional[int] = None, quantize_base: bool = False, ) -> TransformerDecoder: """ @@ -182,7 +177,6 @@ def lora_llama2_13b( embed_dim=5120, max_seq_len=4096, intermediate_dim=13824, - max_batch_size=max_batch_size, attn_dropout=0.0, norm_eps=1e-5, lora_rank=lora_rank, diff --git a/torchtune/models/mistral/_component_builders.py b/torchtune/models/mistral/_component_builders.py index 9ac4a2f02f..b9b3777ba9 100644 --- a/torchtune/models/mistral/_component_builders.py +++ b/torchtune/models/mistral/_component_builders.py @@ -105,6 +105,9 @@ def mistral( tok_embeddings=tok_embeddings, layer=layer, num_layers=num_layers, + max_seq_len=max_seq_len, + num_heads=num_heads, + head_dim=head_dim, norm=RMSNorm(embed_dim, eps=norm_eps), output=output_proj, ) @@ -217,6 +220,9 @@ def lora_mistral( tok_embeddings=tok_embeddings, layer=layer, num_layers=num_layers, + max_seq_len=max_seq_len, + num_heads=num_heads, + head_dim=(embed_dim // num_heads), norm=RMSNorm(embed_dim, eps=norm_eps), output=output_proj, ) diff --git a/torchtune/modules/attention.py b/torchtune/modules/attention.py index 09dffbe5c9..03ac3e7aee 100644 --- a/torchtune/modules/attention.py +++ b/torchtune/modules/attention.py @@ -122,14 +122,16 @@ def forward( self, x: Tensor, mask: Optional[Tensor] = None, - curr_pos: int = 0, + input_pos: Optional[Tensor] = None, ) -> Tensor: """ Args: x (Tensor): input tensor with shape [batch_size x seq_length x embed_dim] - mask (Optional[Tensor]): boolean mask, defaults to None. - curr_pos (int): current position in the sequence, defaults to 0. + mask (Optional[Tensor]): Optional tensor which contains the mask. + Only used during inference. Default is None. + input_pos (Optional[Tensor]): Optional tensor which contains the position + of the current token. This is only used during inference. Default is None Returns: Tensor: output tensor with attention applied @@ -149,7 +151,6 @@ def forward( - Return the attention weights - Make application of positional embeddings optional """ - # input has shape [b, s, d] bsz, seq_len, _ = x.shape @@ -190,20 +191,18 @@ def forward( v = v.reshape(bsz, seq_len, -1, self.head_dim) # Apply positional embeddings - q = self.pos_embeddings(q, curr_pos) - k = self.pos_embeddings(k, curr_pos) - - # Update key-value cache - if self.kv_cache is not None: - k, v = self.kv_cache.update( - bsz=bsz, seq_len=seq_len, curr_pos=curr_pos, k_val=k, v_val=v - ) + q = self.pos_embeddings(q, input_pos) + k = self.pos_embeddings(k, input_pos) # [b, n_h, s, h_d] q = q.transpose(1, 2) k = k.transpose(1, 2) v = v.transpose(1, 2) + # Update key-value cache + if self.kv_cache is not None: + k, v = self.kv_cache.update(input_pos, k, v) + # Flash attention from https://pytorch.org/blog/accelerating-large-language-models/ output = nn.functional.scaled_dot_product_attention( q, diff --git a/torchtune/modules/kv_cache.py b/torchtune/modules/kv_cache.py index cfc8e731ce..da8e9d0246 100644 --- a/torchtune/modules/kv_cache.py +++ b/torchtune/modules/kv_cache.py @@ -17,50 +17,38 @@ class KVCache(nn.Module): Args: max_batch_size (int): maximum batch size model will be run with max_seq_len (int): maximum sequence length model will be run with - n_kv_heads (int): number of kv heads + num_heads (int): number of heads. We take num_heads instead of num_kv_heads because + the cache is created after we've expanded the key and value tensors to have the + same shape as the query tensor. See attention.py for more details head_dim (int): per-attention head embedding dimension + dtype (torch.dtype): dtype for the caches """ def __init__( self, max_batch_size: int, max_seq_len: int, - n_kv_heads: int, + num_heads: int, head_dim: int, - ): + dtype: torch.dtype, + ) -> None: super().__init__() - cache_shape = (max_batch_size, max_seq_len, n_kv_heads, head_dim) - self.register_buffer("k_cache", torch.zeros(cache_shape), persistent=False) - self.register_buffer("v_cache", torch.zeros(cache_shape), persistent=False) + cache_shape = (max_batch_size, num_heads, max_seq_len, head_dim) + self.register_buffer( + "k_cache", torch.zeros(cache_shape, dtype=dtype), persistent=False + ) + self.register_buffer( + "v_cache", torch.zeros(cache_shape, dtype=dtype), persistent=False + ) self.max_batch_size = max_batch_size - def update( - self, bsz: int, seq_len: int, curr_pos: int, k_val: Tensor, v_val: Tensor - ) -> Tuple[Tensor, Tensor]: - """ - Updates the kv-cache at curr_pos with the given k_val and v_val. - - Args: - bsz (int): Batch size. - seq_len (int): Sequence length. - curr_pos (int): Current position in sequence. - k_val (Tensor): New k value. - v_val (Tensor): New v value. + def update(self, input_pos, k_val, v_val) -> Tuple[Tensor, Tensor]: + # input_pos: [S], k_val: [B, H, S, D] + assert input_pos.shape[0] == k_val.shape[2] - Raises: - ValueError: if bsz is greater than the ``max_batch_size`` supported by the model + k_out = self.k_cache + v_out = self.v_cache + k_out[:, :, input_pos] = k_val + v_out[:, :, input_pos] = v_val - Returns: - Tuple[Tensor, Tensor]: the key-cache and value-cache - """ - if bsz > self.max_batch_size: - raise ValueError( - f"Batch size {bsz} greater than max batch size {self.max_batch_size}" - ) - - self.k_cache[:bsz, curr_pos : curr_pos + seq_len] = k_val - self.v_cache[:bsz, curr_pos : curr_pos + seq_len] = v_val - return ( - self.k_cache[:bsz, : curr_pos + seq_len], - self.v_cache[:bsz, : curr_pos + seq_len], - ) + return k_out, v_out diff --git a/torchtune/modules/position_embeddings.py b/torchtune/modules/position_embeddings.py index 63fb3dd2b7..62c78b03e1 100644 --- a/torchtune/modules/position_embeddings.py +++ b/torchtune/modules/position_embeddings.py @@ -4,6 +4,8 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +from typing import Optional + import torch from torch import nn, Tensor @@ -70,12 +72,13 @@ def build_rope_cache(self, max_seq_len: int = 4096) -> None: cache = torch.stack([torch.cos(idx_theta), torch.sin(idx_theta)], dim=-1) self.register_buffer("cache", cache, persistent=False) - def forward(self, x: Tensor, curr_pos: int = 0) -> Tensor: + def forward(self, x: Tensor, input_pos: Optional[Tensor] = None) -> Tensor: """ Args: x (Tensor): input tensor with shape [bsz, seq_len, num_heads, head_dim] - curr_pos (int): current position in the sequence, defualts to 0. + input_pos (Optional[Tensor]): Optional tensor which contains the position + of the current token. This is only used during inference. Default is None Returns: Tensor: output tensor with RoPE applied @@ -91,7 +94,12 @@ def forward(self, x: Tensor, curr_pos: int = 0) -> Tensor: """ # input tensor has shape [b, s, n_h, n_d] seq_len = x.size(1) - rope_cache = self.cache[curr_pos : curr_pos + seq_len] + + # extract the values based on whether input_pos is set or not. When + # input_pos is provided, we're in infernce mode + rope_cache = ( + self.cache[:seq_len] if input_pos is None else self.cache[input_pos] + ) # reshape input; the last dimension is used for computing the output. # Cast to float to match the reference implementation diff --git a/torchtune/modules/transformer.py b/torchtune/modules/transformer.py index b7475efe88..9ac6524b60 100644 --- a/torchtune/modules/transformer.py +++ b/torchtune/modules/transformer.py @@ -9,7 +9,7 @@ import torch from torch import nn, Tensor -from torchtune.modules import CausalSelfAttention +from torchtune.modules import CausalSelfAttention, KVCache class TransformerDecoderLayer(nn.Module): @@ -39,14 +39,16 @@ def forward( self, x: Tensor, mask: Optional[Tensor] = None, - curr_pos: int = 0, + input_pos: Optional[Tensor] = None, ) -> Tensor: """ Args: x (Tensor): input tensor with shape [batch_size x seq_length x embed_dim] - mask (Optional[Tensor]): mask tensor, defaults to None. - curr_pos (int): current position in the seq, defaults to 0. + mask (Optional[Tensor]): Optional tensor which contains the mask. + Only used during inference. Default is None. + input_pos (Optional[Tensor]): Optional tensor which contains the position + of the current token. This is only used during inference. Default is None Returns: Tensor: output tensor with same shape as input @@ -63,7 +65,7 @@ def forward( # Input tensor and attention output have the same shape # [b, s, d] # Norm applied before self-attention - attn_out = self.attn(self.sa_norm(x), mask, curr_pos) + attn_out = self.attn(self.sa_norm(x), mask, input_pos) # Residual connection; shape: [b, s, d] h = attn_out + x @@ -96,11 +98,21 @@ class TransformerDecoder(nn.Module): Transformer Decoder derived from the Llama2 architecture. Args: - tok_embeddings (nn.Embedding): PyTorch embedding layer, to be used to move tokens to an embedding space. + tok_embeddings (nn.Embedding): PyTorch embedding layer, to be used to move + tokens to an embedding space. layer (TransformerDecoderLayer): Transformer Decoder layer. num_layers (int): Number of Transformer Decoder layers. - norm (nn.Module): Callable that applies normalization to the output of the decoder, before final MLP. - output (nn.Linear): Callable that applies a linear transformation to the output of the decoder. + max_seq_len (int): maximum sequence length the model will be run with, as used + by :func:`~torchtune.modules.KVCache` + num_heads (int): number of query heads. For MHA this is also the + number of heads for key and value. This is used to setup the + :func:`~torchtune.modules.KVCache` + head_dim (int): embedding dimension for each head in self-attention. This is used + to setup the :func:`~torchtune.modules.KVCache` + norm (nn.Module): Callable that applies normalization to the output of the decoder, + before final MLP. + output (nn.Linear): Callable that applies a linear transformation to the output of + the decoder. Note: Arg values are checked for correctness (eg: ``attn_dropout`` belongs to [0,1]) @@ -113,33 +125,63 @@ def __init__( tok_embeddings: nn.Embedding, layer: TransformerDecoderLayer, num_layers: int, + max_seq_len: int, + num_heads: int, + head_dim: int, norm: nn.Module, output: nn.Linear, ) -> None: super().__init__() + self.tok_embeddings = tok_embeddings self.layers = _get_clones(layer, num_layers) self.norm = norm self.output = output + self.max_seq_len = max_seq_len + self.num_heads = num_heads + self.head_dim = head_dim + self.causal_mask = None - def forward( - self, tokens: Tensor, mask: Optional[Tensor] = None, curr_pos: int = 0 - ) -> Tensor: + def setup_caches(self, max_batch_size: int, dtype: torch.dtype) -> None: + for layer in self.layers: + layer.attn.kv_cache = KVCache( + max_batch_size=max_batch_size, + max_seq_len=self.max_seq_len, + num_heads=self.num_heads, + head_dim=self.head_dim, + dtype=dtype, + ) + + # causal_mask is used during inference to ensure we're attending + # to the right tokens + self.causal_mask = torch.tril( + torch.ones(self.max_seq_len, self.max_seq_len, dtype=torch.bool) + ) + + def forward(self, tokens: Tensor, input_pos: Optional[Tensor] = None) -> Tensor: """ Args: tokens (Tensor): input tensor with shape [b x s] - mask (Optional[Tensor]): attention mask tensor, defaults to None. - curr_pos (int): current position in the seq, defaults to 0. - Only relevant when incrementally decoding. + input_pos (Optional[Tensor]): Optional tensor which contains the position + of the current token. This is only used during inference. Default is None + + Note: At the very first step of inference, when the model is provided with a prompt, + ``input_pos`` would contain the positions of all of the tokens in the prompt + (eg: ``torch.arange(prompt_length)``). This is because we will need to compute the + KV values for each position. Returns: Tensor: output tensor with shape [b x s x v] + Raises: + ValueError: if causal_mask is set but input_pos is None + Notation used for tensor shapes: - b: batch size - s: sequence length - v: vocab size - d: embed dim + - m_s: max seq len """ # input tensor of shape [b, s] bsz, seq_len = tokens.shape @@ -147,16 +189,19 @@ def forward( # shape: [b, s, d] h = self.tok_embeddings(tokens) - # TODO: Fix the masking logic to not rely on checking kv_cache - if seq_len > 1 and self.layers[0].attn.kv_cache is not None: - mask = torch.full( - (1, 1, seq_len, seq_len), float("-inf"), device=tokens.device - ) - mask = torch.triu(mask, diagonal=curr_pos + 1) + mask = None + if self.causal_mask is not None: + if input_pos is None: + raise ValueError( + "Caches are setup, but the position of input token is missing" + ) + # shape: [1, input_pos_len, m_s] + # in most cases input_pos_len should be 1 + mask = self.causal_mask[None, None, input_pos] for layer in self.layers: # shape: [b, s, d] - h = layer(h, mask, curr_pos) + h = layer(h, mask, input_pos) # shape: [b, s, d] h = self.norm(h) diff --git a/torchtune/utils/__init__.py b/torchtune/utils/__init__.py index 623ad474fb..666711a34c 100644 --- a/torchtune/utils/__init__.py +++ b/torchtune/utils/__init__.py @@ -22,6 +22,7 @@ validate_no_params_on_meta_device, wrap_fsdp, ) +from ._generation import generate # noqa from .argparse import TuneRecipeArgumentParser from .checkpointable_dataloader import CheckpointableDataLoader from .collate import padded_collate diff --git a/torchtune/utils/_generation.py b/torchtune/utils/_generation.py new file mode 100644 index 0000000000..4e42ae3be0 --- /dev/null +++ b/torchtune/utils/_generation.py @@ -0,0 +1,139 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Optional + +import torch + +from torchtune.modules import TransformerDecoder + + +def multinomial_sample_one(probs): + q = torch.empty_like(probs).exponential_(1) + return torch.argmax(probs / q, dim=-1, keepdim=True).to(dtype=torch.int) + + +def sample( + logits: torch.Tensor, temperature: float = 1.0, top_k: Optional[int] = None +) -> torch.Tensor: + # scale the logits based on temperature + logits = logits / max(temperature, 1e-5) + + # keep only the top_k logits if this is specified + if top_k is not None: + v, _ = torch.topk(logits, min(top_k, logits.size(-1))) + # select the very last value from the top_k above as the pivot + pivot = v.select(-1, -1).unsqueeze(-1) + # set everything smaller than pivot value to inf since these + # should be pruned + logits = torch.where(logits < pivot, -float("Inf"), logits) + + # compute the probabilities + probs = torch.nn.functional.softmax(logits, dim=-1) + + # sample the next token + token = multinomial_sample_one(probs) + return token + + +def generate_next_token( + model: TransformerDecoder, + input_pos: torch.Tensor, + x: torch.Tensor, + temperature: float = 1.0, + top_k: Optional[int] = None, +) -> torch.Tensor: + # x: [1, s] + # input_pos: [s] + logits = model(x, input_pos) + + # logits: [1, s, v] where v is vocab_size + # for sampling we extract the logits for the + # last token and convert to shape: [v] + logits = logits[0, -1] + + # sample the next token + token = sample(logits, temperature, top_k) + return token + + +@torch.inference_mode() +def generate( + model: TransformerDecoder, + prompt: torch.Tensor, + max_generated_tokens: int, + temperature: float = 1.0, + top_k: Optional[int] = None, + eos_id: Optional[int] = None, +) -> torch.Tensor: + """ + Generate tokens from a model conditioned on a prompt. + + Args: + model (TransformerDecoder): model used for generation + prompt (torch.Tensor): tensor with the token IDs associated with the given + prompt. This is the output of the relevant tokenizer + max_generated_tokens (int): number of tokens to be generated. This is the max + since we can stop early based on whether the eos token is respected or not + temperature (float): value to scale the predicted logits by. Default is 1.0 + top_k (Optional[int]): If specified, we prune the sampling to only token ids within + the top_k probabilities. Default is None + eos_id (Optional[int]): If specified, generation is stopped when the eos token is + generated. Default is None + + Returns: + List: list of generated tokens + + Raises: + ValueError: if max_seq_len supported by the model is smaller than the number of tokens + requested + """ + + prompt_length = prompt.size(0) + + if model.max_seq_len < (prompt_length + max_generated_tokens) - 1: + raise ValueError( + f"Models maximum seq length {model.max_seq_len} should be >= " + f"{(prompt_length + max_generated_tokens)} - 1" + ) + + # generated_tokens is a list of tensors where each tensor contains tokens + # needed for the output + generated_tokens = [prompt] + + # generate the first token by conditioning on the input prompt + token = generate_next_token( + model=model, + input_pos=torch.arange(0, prompt_length, device=prompt.device), + # convert the input into [B, S] shape as expected by the model + x=prompt.view(1, -1), + temperature=temperature, + top_k=top_k, + ).clone() + + generated_tokens.append(token) + + # generation starts at position=prompt_length and continues till + # we get the requested number of tokens or we hit eos_id + input_pos = torch.tensor([prompt_length], device=prompt.device) + for _ in range(max_generated_tokens - 1): + token = generate_next_token( + model=model, + input_pos=input_pos, + x=token.view(1, -1), + temperature=temperature, + top_k=top_k, + ).clone() + + generated_tokens.append(token) + + if eos_id is not None and token == eos_id: + break + + # update the position before we generate the next token + input_pos += 1 + + return torch.cat(generated_tokens).tolist() diff --git a/torchtune/utils/generation.py b/torchtune/utils/generation.py deleted file mode 100644 index c4791102ed..0000000000 --- a/torchtune/utils/generation.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import functools -from typing import Callable, List, Optional, Tuple - -import torch -import torch.nn.functional as F -from torch import nn, Tensor - -from torchtune.modules import Tokenizer, TransformerDecoder -from torchtune.utils.logits_transforms import ( - LogitsTransform, - TemperatureTransform, - TopKTransform, - TopPTransform, -) - - -class GenerationUtils: - """Utility class for generating text from a decoder-style LLM. - - Args: - decoder_lm (nn.Module): Transformer-Decoder based language model. - pad_id (int): Padding token ID. - eos_id (int): End-of-sequence token ID. - - NOTE: - Currently, `decoder_lm` assumes a forward API with the signature - `def forward(x: torch.Tensor, curr_pos: int)` as the index of the - current token is passed in for kv-caching during incremental decoding. - If `decoder_lm` does not support this interface, please set - `incremental_decode` to `False` when calling `generate` function. - """ - - def __init__(self, decoder_lm: nn.Module, pad_id: int, eos_id: int): - self.decoder_lm = decoder_lm - self.pad_id = pad_id - self.eos_id = eos_id - - def _get_logits_transforms( - self, - temperature: float, - top_p: float, - top_k: int, - ) -> List[LogitsTransform]: - """Returns a list of parameterized logits transforms that can be chained. - - Args: - temperature (float): Sampling temperature. - top_p (float): Probability threshold for nucleus sampling. - top_k (int): Number of tokens kept for top-k filtering. - - Returns: - List of LogitsTransform objects. - """ - logits_transforms = [] - if temperature > 0: - logits_transforms.append(TemperatureTransform(temperature)) - if top_p > 0: - logits_transforms.append(TopPTransform(top_p)) - if top_k > 1: - logits_transforms.append(TopKTransform(top_k)) - - return logits_transforms - - def _apply_logits_transforms( - self, logits_transforms: List[LogitsTransform], logits: torch.FloatTensor - ) -> torch.FloatTensor: - """Applies a chained list of logits transforms. - - Args: - logits_transforms (List[LogitsTransform]): List of LogitsTransform objects. - logits (torch.FloatTensor): Raw logits tensor. - - Returns: - Transformed logits tensor. - """ - output_logits = ( - functools.reduce(lambda x, f: f(x), logits_transforms, logits) - if logits_transforms - else logits - ) - return output_logits - - @torch.no_grad() - def generate( - self, - prompt_tokens: List[List[int]], - min_gen_len: int, - max_gen_len: int, - temperature: float = 0.6, - top_p: float = 0.9, - top_k: int = 1, - keep_prompt: bool = True, - logprobs: bool = False, - incremental_decode: bool = True, - logits_accessor: Optional[Callable] = None, - device: Optional[torch.device] = torch.device("cpu"), - ) -> Tuple[Tensor, Optional[Tensor]]: - """ - Interface for generation supporting temperature, top-k, and top-p sampling. - - Args: - prompt_tokens (List[List[int]]): List of tokenized per-batch prompts. - min_gen_len (int): Minimum generated sequence length. - max_gen_len (int): Maximum generated sequence length. - temperature (float): Temperature value to control sampling randomness. Defaults to 0.6. - top_p (float): Probability threshold for nucleus sampling. Defaults to 0.9. - top_k (int): Number of tokens kept for top-k filtering. Defaults to 1. - keep_prompt (bool): Whether to keep prompt tokens in the output tensor(s). Defaults to True. - logprobs (bool): Whether to compute log probabilities. Defaults to False. - incremental_decode (bool): Whether to decode incrementally or not. Defaults to True. - logits_accessor (Optional[Callable]): Function to transform logits before sampling. Defaults to None. - device (Optional[torch.device]): Device on which to initialize prompt token tensors (should match device of model). - Defaults to torch.device("cpu"). - - Returns: - Tuple[Tensor, Optional[Tensor]]: Tuple of generated tokens and optional log probabilities if `logprobs=True`, - where the dimensions of each tensor are (batch_size, max_gen_length) - - Example: - >>> LLaMA = GenerationUtils(model, pad_id = tokenizer.pad_id, eos_id = tokenizer.eos_id) - >>> tokens = LLaMA.generate( - ... [tokenizer.encode(["I love to eat"])], - ... min_gen_len=5, - ... max_gen_len=20, - ... temperature=0.8, - ... top_p=0.7, - ... keep_prompt=True, - ... ) - >>> print(tokens) - ["I love to eat ice cream"] - """ - torch.manual_seed(1337) - - batch_size = len(prompt_tokens) - max_prompt_len = max(len(p) for p in prompt_tokens) - min_prompt_len = min(len(p) for p in prompt_tokens) - total_gen_len = max_gen_len + max_prompt_len - tokens = torch.full( - (batch_size, total_gen_len), self.pad_id, dtype=torch.long, device=device - ) - for i, prompt in enumerate(prompt_tokens): - tokens[i, : len(prompt)] = torch.tensor( - prompt, dtype=torch.long, device=device - ) - if logprobs: - token_logprobs = torch.full_like( - tokens, float("-inf"), dtype=torch.float, device=device - ) - else: - token_logprobs = None - # mask to ensure we don't overwrite the prompt for prompts > min_prompt_len. - prompt_mask = tokens != self.pad_id - logits_transforms = self._get_logits_transforms(temperature, top_p, top_k) - # TODO: generalize the LLM's behavior - for example, models may not take in - # a start_pos. - prev_pos = 0 - eos_reached = torch.zeros(batch_size, dtype=torch.bool, device=device) - for cur_pos in range(min_prompt_len, total_gen_len): - input_ids = tokens[:, prev_pos:cur_pos] - if incremental_decode: - outputs = self.decoder_lm(input_ids, curr_pos=prev_pos) - else: - outputs = self.decoder_lm(input_ids) - if logits_accessor: - logits = logits_accessor(outputs) - else: - logits = outputs - next_token_logits = logits[:, -1] - - # Convert to probability distribution, then sample - next_token_probs = next_token_logits.softmax(dim=-1) - next_token_probs = self._apply_logits_transforms( - logits_transforms, next_token_probs - ) - next_token = torch.multinomial(next_token_probs, num_samples=1).squeeze(1) - # Record positions of any EOS tokens across batches - eos_reached_cur = next_token.eq(self.eos_id) - eos_reached |= eos_reached_cur - # Avoid overwriting the prompt for prompts that are longer than min_prompt_len. - tokens[:, cur_pos] = torch.where( - prompt_mask[:, cur_pos], - tokens[:, cur_pos], - next_token, - ) - if token_logprobs is not None: - token_logprobs[:, cur_pos].masked_scatter_( - ~eos_reached, - -F.cross_entropy( - next_token_logits, - tokens[:, cur_pos], - reduction="none", - ignore_index=self.pad_id, - ), - ) - - if incremental_decode: - prev_pos = cur_pos - - if eos_reached.all().item(): - break - - if not keep_prompt: - tokens = tokens[:, max_prompt_len:] - if token_logprobs is not None: - token_logprobs = token_logprobs[:, max_prompt_len:] - - return tokens, token_logprobs if logprobs else None - - -def generate_from_prompt( - prompt: str, tokenizer: Tokenizer, decoder: TransformerDecoder -) -> Tuple[str, List[int]]: - """ - Generate a response from a prompt and a decoder. - Args: - prompt (str): Prompt to generate from. - tokenizer (Tokenizer): Tokenizer to use for generation. - decoder (TransformerDecoder): Model to use for generation. - - Returns: - Tuple[str, List[int]]: Generated response and corresponding tokenized response. - """ - prompt_tokens = [tokenizer.encode(prompt, add_eos=False)] - with torch.no_grad(): - generations_no_kv_cache, _ = GenerationUtils( - decoder_lm=decoder, - eos_id=tokenizer.eos_id, - pad_id=tokenizer.pad_id, - ).generate( - prompt_tokens=prompt_tokens, - incremental_decode=False, - min_gen_len=1, - max_gen_len=256, - top_k=3, - device=torch.cuda.current_device(), - ) - gens = generations_no_kv_cache.tolist()[0] - gen_str = tokenizer.decode(gens) - return gens, gen_str diff --git a/torchtune/utils/logits_transforms.py b/torchtune/utils/logits_transforms.py deleted file mode 100644 index 51bc413cf2..0000000000 --- a/torchtune/utils/logits_transforms.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -import abc - -import torch - - -class LogitsTransform(abc.ABC): - """Interface for a logits transformation.""" - - @abc.abstractmethod - def __call__(self, scores: torch.FloatTensor) -> torch.FloatTensor: - pass - - -class TemperatureTransform(LogitsTransform): - """Controls randomness of predicted tokens via a temperature value. - Args: - temperature (float): The parameter controlling distribution randomness. - Raises: - ValueError: If `temperature` is less than or equal to zero. - """ - - def __init__(self, temperature: float): - if temperature <= 0: - raise ValueError(f"Expected 0 < `temperature` but got {temperature=}") - - self.temperature = temperature - - def __call__(self, scores: torch.FloatTensor) -> torch.FloatTensor: - scores /= self.temperature - return scores - - -class TopPTransform(LogitsTransform): - """Filters the distribution to cover the fewest tokens whose cumulative mass - exceeds `prob`. - Args: - prob (float): The minimum cumulative probability mass that the kept tokens - must cover. - Raises: - ValueError: If `prob` is less than or equal to zero or greater than one. - """ - - def __init__(self, prob: float): - if prob <= 0 or prob > 1: - raise ValueError(f"Expected 0 < `prob` <= 1 but got {prob=}") - - self.prob = prob - - def __call__(self, scores: torch.FloatTensor) -> torch.FloatTensor: - scores_sort, scores_index = torch.sort(scores, dim=-1, descending=True) - scores_cumulative = scores_sort.cumsum(dim=-1) - - # Ignore tokens introducing more probability mass than needed - discard_mask = scores_cumulative - scores_sort > self.prob - scores_sort[discard_mask] = 0.0 - - scores_sort.div_(scores_sort.sum(dim=-1, keepdim=True)) # renormalize - scores.scatter_(-1, scores_index, scores_sort) - return scores - - -class TopKTransform(LogitsTransform): - """Filters the distribution to include the top-k highest probability tokens. - Args: - top_k (int): The number of highest probability tokens to keep. - Raises: - ValueError: If `top_k` is less than or equal to zero. - TypeError: If `top_k` is not an integer. - """ - - def __init__(self, top_k: int): - if top_k <= 0: - raise ValueError(f"Expected 0 `top_k` > but got {top_k=}") - if not isinstance(top_k, int): - raise TypeError(f"Expected `top_k` to be int but got {type(top_k)=}") - - self.top_k = top_k - - def __call__(self, scores: torch.FloatTensor) -> torch.FloatTensor: - top_k = min(self.top_k, scores.size(-1)) - scores_topk, _ = scores.topk(top_k) - - discard_mask = scores < scores_topk[..., -1] - scores.masked_fill_(discard_mask, 0.0) - - scores.div_(scores.sum(dim=-1, keepdim=True)) # renormalize - return scores From f60ebb2c6a5f051104cb744ca8ec85b2bfb05ec8 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Sun, 31 Mar 2024 19:53:41 -0400 Subject: [PATCH 14/76] Print out "Ignoring patterns" for download (#625) --- tests/torchtune/_cli/test_download.py | 19 +++++++++++++++---- torchtune/_cli/download.py | 1 + 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/torchtune/_cli/test_download.py b/tests/torchtune/_cli/test_download.py index 965bf2e4ba..5dbd695226 100644 --- a/tests/torchtune/_cli/test_download.py +++ b/tests/torchtune/_cli/test_download.py @@ -38,18 +38,29 @@ def test_download_calls_snapshot(self, capsys, monkeypatch, snapshot_download): # Call the first time and get GatedRepoError with pytest.raises(SystemExit, match="2"): runpy.run_path(TUNE_PATH, run_name="__main__") - err = capsys.readouterr().err - assert "It looks like you are trying to access a gated repository." in err + out_err = capsys.readouterr() + assert ( + "Ignoring files matching the following patterns: *.safetensors" + in out_err.out + ) + assert ( + "It looks like you are trying to access a gated repository." in out_err.err + ) # Call the second time and get RepositoryNotFoundError with pytest.raises(SystemExit, match="2"): runpy.run_path(TUNE_PATH, run_name="__main__") - err = capsys.readouterr().err - assert "not found on the Hugging Face Hub" in err + out_err = capsys.readouterr() + assert ( + "Ignoring files matching the following patterns: *.safetensors" + in out_err.out + ) + assert "not found on the Hugging Face Hub" in out_err.err # Call the third time and get the expected output runpy.run_path(TUNE_PATH, run_name="__main__") output = capsys.readouterr().out + assert "Ignoring files matching the following patterns: *.safetensors" in output assert "Successfully downloaded model repo" in output # Make sure it was called twice diff --git a/torchtune/_cli/download.py b/torchtune/_cli/download.py index 3830a1b848..d30f4491af 100644 --- a/torchtune/_cli/download.py +++ b/torchtune/_cli/download.py @@ -86,6 +86,7 @@ def _add_arguments(self) -> None: def _download_cmd(self, args: argparse.Namespace) -> None: """Downloads a model from the Hugging Face Hub.""" # Download the tokenizer and PyTorch model files + print(f"Ignoring files matching the following patterns: {args.ignore_patterns}") try: true_output_dir = snapshot_download( args.repo_id, From ba93269202ce9062e7168381ed78e2aa63ce434d Mon Sep 17 00:00:00 2001 From: Botao Chen Date: Mon, 1 Apr 2024 09:34:34 -0700 Subject: [PATCH 15/76] [Fix] Update the tune command to kick off training in yaml files (#628) --- recipes/configs/llama2/13B_full.yaml | 2 +- recipes/configs/llama2/13B_lora.yaml | 2 +- recipes/configs/llama2/7B_full.yaml | 2 +- recipes/configs/llama2/7B_full_single_device.yaml | 2 +- recipes/configs/llama2/7B_full_single_device_low_memory.yaml | 2 +- recipes/configs/llama2/7B_lora.yaml | 2 +- recipes/configs/llama2/7B_lora_single_device.yaml | 2 +- recipes/configs/llama2/7B_qlora_single_device.yaml | 4 ++-- recipes/configs/mistral/7B_full.yaml | 2 +- recipes/configs/mistral/7B_lora.yaml | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/recipes/configs/llama2/13B_full.yaml b/recipes/configs/llama2/13B_full.yaml index 872533ad6d..abbd9c45c5 100644 --- a/recipes/configs/llama2/13B_full.yaml +++ b/recipes/configs/llama2/13B_full.yaml @@ -8,7 +8,7 @@ # --output-dir /tmp/llama2-13b-hf # # To launch on 4 devices, run the following command from root: -# tune --nnodes 1 --nproc_per_node 4 full_finetune_distributed \ +# tune run --nproc_per_node 4 full_finetune_distributed \ # --config llama2/13B_full \ # # You can add specific overrides through the command line. For example diff --git a/recipes/configs/llama2/13B_lora.yaml b/recipes/configs/llama2/13B_lora.yaml index 2a338272dd..bd69fdc92d 100644 --- a/recipes/configs/llama2/13B_lora.yaml +++ b/recipes/configs/llama2/13B_lora.yaml @@ -8,7 +8,7 @@ # --output-dir /tmp/llama2-13b-hf # # To launch on 4 devices, run the following command from root: -# tune --nnodes 1 --nproc_per_node 4 lora_finetune_distributed \ +# tune run --nproc_per_node 4 lora_finetune_distributed \ # --config llama2/13B_lora \ # # You can add specific overrides through the command line. For example diff --git a/recipes/configs/llama2/7B_full.yaml b/recipes/configs/llama2/7B_full.yaml index 74b5d5a8f3..16f3dcb3ec 100644 --- a/recipes/configs/llama2/7B_full.yaml +++ b/recipes/configs/llama2/7B_full.yaml @@ -8,7 +8,7 @@ # --output-dir /tmp/llama2 # # To launch on 4 devices, run the following command from root: -# tune --nnodes 1 --nproc_per_node 4 full_finetune_distributed \ +# tune run --nproc_per_node 4 full_finetune_distributed \ # --config llama2/7B_full \ # # You can add specific overrides through the command line. For example diff --git a/recipes/configs/llama2/7B_full_single_device.yaml b/recipes/configs/llama2/7B_full_single_device.yaml index 25c48bbe58..1d297a28ec 100644 --- a/recipes/configs/llama2/7B_full_single_device.yaml +++ b/recipes/configs/llama2/7B_full_single_device.yaml @@ -8,7 +8,7 @@ # --output-dir /tmp/llama2 # # To launch on a single device, run the following command from root: -# tune --nnodes 1 --nproc_per_node 1 full_finetune_single_device \ +# tune run full_finetune_single_device \ # --config llama2/7B_full_single_device \ # # You can add specific overrides through the command line. For example diff --git a/recipes/configs/llama2/7B_full_single_device_low_memory.yaml b/recipes/configs/llama2/7B_full_single_device_low_memory.yaml index f507aa4ca3..c1bfd5cb6f 100644 --- a/recipes/configs/llama2/7B_full_single_device_low_memory.yaml +++ b/recipes/configs/llama2/7B_full_single_device_low_memory.yaml @@ -8,7 +8,7 @@ # --output-dir /tmp/llama2 # # To launch on a single device, run the following command from root: -# tune --nnodes 1 --nproc_per_node 1 full_finetune_single_device \ +# tune run full_finetune_single_device \ # --config llama2/7B_full_single_device_low_memory \ # # You can add specific overrides through the command line. For example diff --git a/recipes/configs/llama2/7B_lora.yaml b/recipes/configs/llama2/7B_lora.yaml index 25175af971..6a248b2740 100644 --- a/recipes/configs/llama2/7B_lora.yaml +++ b/recipes/configs/llama2/7B_lora.yaml @@ -8,7 +8,7 @@ # --output-dir /tmp/llama2 # # To launch on 4 devices, run the following command from root: -# tune --nnodes 1 --nproc_per_node 4 lora_finetune_distributed \ +# tune run --nproc_per_node 4 lora_finetune_distributed \ # --config llama2/7B_lora \ # # You can add specific overrides through the command line. For example diff --git a/recipes/configs/llama2/7B_lora_single_device.yaml b/recipes/configs/llama2/7B_lora_single_device.yaml index 757f353945..8c2281df8e 100644 --- a/recipes/configs/llama2/7B_lora_single_device.yaml +++ b/recipes/configs/llama2/7B_lora_single_device.yaml @@ -8,7 +8,7 @@ # --output-dir /tmp/llama2 # # To launch on a single device, run the following command from root: -# tune --nnodes 1 --nproc_per_node 1 lora_finetune_single_device \ +# tune run lora_finetune_single_device \ # --config llama2/7B_lora_single_device \ # # You can add specific overrides through the command line. For example diff --git a/recipes/configs/llama2/7B_qlora_single_device.yaml b/recipes/configs/llama2/7B_qlora_single_device.yaml index 1ae29e57af..b3c874e8d9 100644 --- a/recipes/configs/llama2/7B_qlora_single_device.yaml +++ b/recipes/configs/llama2/7B_qlora_single_device.yaml @@ -8,8 +8,8 @@ # --output-dir /tmp/llama2 # # To launch on a single device, run the following command from root: -# tune --nnodes 1 --nproc_per_node 1 lora_finetune_single_device \ -# --config 7B_qlora_single_device \ +# tune run lora_finetune_single_device \ +# --config llama2\7B_qlora_single_device \ # # You can add specific overrides through the command line. For example # to override the checkpointer directory while launching training diff --git a/recipes/configs/mistral/7B_full.yaml b/recipes/configs/mistral/7B_full.yaml index 024c8a4222..211c5526ac 100644 --- a/recipes/configs/mistral/7B_full.yaml +++ b/recipes/configs/mistral/7B_full.yaml @@ -3,7 +3,7 @@ # from the paper # # Run this config on 4 GPUs using the following: -# tune --nnodes 1 --nproc_per_node 4 full_finetune_distributed --config mistral/7B_full +# tune run --nproc_per_node 4 full_finetune_distributed --config mistral/7B_full # Tokenizer tokenizer: diff --git a/recipes/configs/mistral/7B_lora.yaml b/recipes/configs/mistral/7B_lora.yaml index caa2cbaf8c..53f926de74 100644 --- a/recipes/configs/mistral/7B_lora.yaml +++ b/recipes/configs/mistral/7B_lora.yaml @@ -3,7 +3,7 @@ # from the paper # # Run this config on 4 GPUs using the following: -# tune --nnodes 1 --nproc_per_node 4 lora_finetune_distributed --config mistral/7B_lora +# tune run --nproc_per_node 4 lora_finetune_distributed --config mistral/7B_lora # Tokenizer From ee2f82b5352755c107b4b9fa1b5e54be24282061 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Mon, 1 Apr 2024 12:53:29 -0400 Subject: [PATCH 16/76] Remove conversion script (#629) --- .github/workflows/conversion_script_test.yaml | 69 ------------------- 1 file changed, 69 deletions(-) delete mode 100644 .github/workflows/conversion_script_test.yaml diff --git a/.github/workflows/conversion_script_test.yaml b/.github/workflows/conversion_script_test.yaml deleted file mode 100644 index 64d0edd4c4..0000000000 --- a/.github/workflows/conversion_script_test.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# This tests a conversion of Llama2 7b FAIR checkpoint to a TorchTune-supported native checkpoint -# It runs on a daily cadence at the end of each day - -name: large scale conversion script test - -on: - schedule: - - cron: "0 0 * * *" # Run at the end of every day - workflow_dispatch: - - -concurrency: - group: conversion-script-test-${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -defaults: - run: - shell: bash -l -eo pipefail {0} - -jobs: - conversion_script_test: - runs-on: 32-core-ubuntu - strategy: - matrix: - python-version: ['3.11'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: configure aws credentials - id: aws_creds - uses: aws-actions/configure-aws-credentials@v1.7.0 - with: - role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_torchtune_pytorch-multimodal - aws-region: us-east-1 - - name: Download files from S3 - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - retry_on: error - timeout_seconds: 14400 - command: | - sudo swapoff -a - sudo fallocate -l 64G /swapfile - sudo chmod 600 /swapfile - sudo mkswap /swapfile - sudo swapon /swapfile - python -m pip install awscli==1.32.6 - mkdir -p /tmp/test-artifacts - aws s3 cp s3://pytorch-multimodal/llama2-7b/consolidated.00.pth /tmp/test-artifacts/llama2-7b-fair - - name: Install dependencies - run: | - python -m pip install torch - python -m pip install -e ".[dev]" - - name: Run conversion test - run: | - pytest tests/torchtune/_cli/test_convert_checkpoint.py --large-scale True From 83660cdb9bd92a2a665772c18462de13d6e6f49c Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Mon, 1 Apr 2024 14:09:28 -0400 Subject: [PATCH 17/76] Add Mistral models to recipe registry (#631) --- torchtune/_recipe_registry.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/torchtune/_recipe_registry.py b/torchtune/_recipe_registry.py index ae25713652..20e578e306 100644 --- a/torchtune/_recipe_registry.py +++ b/torchtune/_recipe_registry.py @@ -35,6 +35,10 @@ class Recipe: name="llama2/7B_full_single_device_low_memory", file_path="llama2/7B_full_single_device_low_memory.yaml", ), + Config( + name="mistral/7B_full", + file_path="mistral/7B_full.yaml", + ), ], supports_distributed=False, ), @@ -44,6 +48,7 @@ class Recipe: configs=[ Config(name="llama2/7B_full", file_path="llama2/7B_full.yaml"), Config(name="llama2/13B_full", file_path="llama2/13B_full.yaml"), + Config(name="mistral/7B_full", file_path="mistral/7B_full.yaml"), ], supports_distributed=True, ), @@ -59,6 +64,10 @@ class Recipe: name="llama2/7B_qlora_single_device", file_path="llama2/7B_qlora_single_device.yaml", ), + Config( + name="mistral/7B_lora", + file_path="mistral/7B_lora.yaml", + ), ], supports_distributed=False, ), @@ -68,6 +77,7 @@ class Recipe: configs=[ Config(name="llama2/7B_lora", file_path="llama2/7B_lora.yaml"), Config(name="llama2/13B_lora", file_path="llama2/13B_lora.yaml"), + Config(name="mistral/7B_lora", file_path="mistral/7B_lora.yaml"), ], supports_distributed=True, ), From 97381a7913491fe50aadcc8690e424dc24d741be Mon Sep 17 00:00:00 2001 From: Kartikay Khandelwal <47255723+kartikayk@users.noreply.github.com> Date: Mon, 1 Apr 2024 17:38:37 -0700 Subject: [PATCH 18/76] Fix first_finetune tutorial (#634) --- docs/source/examples/first_finetune_tutorial.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/examples/first_finetune_tutorial.rst b/docs/source/examples/first_finetune_tutorial.rst index fa4b52dec8..3a628042ba 100644 --- a/docs/source/examples/first_finetune_tutorial.rst +++ b/docs/source/examples/first_finetune_tutorial.rst @@ -39,7 +39,7 @@ Then, it's as simple as: .. code-block:: bash tune download \ - --repo-id meta-llama/Llama-2-7b \ + meta-llama/Llama-2-7b \ --output-dir /tmp/llama2 \ --hf-token @@ -78,7 +78,7 @@ It looks like there's already a config called :code:`alpaca_llama_full_finetune` .. code-block:: bash - tune cp llama2/7B_full.yaml custom_config.yaml + tune cp llama2/7B_full custom_config.yaml Now you can update the custom YAML config to point to your model and tokenizer. While you're at it, you can make some other changes, like setting the random seed in order to make replication easier, @@ -138,7 +138,7 @@ run using two GPUs, it's as easy as: .. code-block:: bash - tune --nnodes 1 --nproc_per_node 2 full_finetune_distributed.py --config custom_config.yaml + tune run --nnodes 1 --nproc_per_node 2 full_finetune_distributed --config custom_config.yaml You should see some immediate output and see the loss going down, indicating your model is training succesfully. From 86c6ee41f9bd83168a49fb491b60e9999392d6ad Mon Sep 17 00:00:00 2001 From: Kartikay Khandelwal <47255723+kartikayk@users.noreply.github.com> Date: Mon, 1 Apr 2024 18:53:33 -0700 Subject: [PATCH 19/76] update license (#635) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02f241d3db..c6d9292935 100644 --- a/README.md +++ b/README.md @@ -223,4 +223,4 @@ We welcome any feature requests, bug reports, or pull requests from the communit ## License -TorchTune is released under the [BSD 3 license](./LICENSE). +TorchTune is released under the [BSD 3 license](./LICENSE). However you may have other legal obligations that govern your use of other content, such as the terms of service for third-party models. From ec3d93ea012543672923e8910cff079237e8796f Mon Sep 17 00:00:00 2001 From: Rohan Varma Date: Tue, 2 Apr 2024 00:33:03 -0700 Subject: [PATCH 20/76] Remove _copy_tensor from usage (#633) --- tests/torchtune/utils/test_tensor_utils.py | 23 ------------------- torchtune/modules/low_precision/nf4_linear.py | 2 ++ torchtune/modules/peft/lora.py | 3 +-- torchtune/utils/tensor_utils.py | 21 ----------------- 4 files changed, 3 insertions(+), 46 deletions(-) delete mode 100644 tests/torchtune/utils/test_tensor_utils.py delete mode 100644 torchtune/utils/tensor_utils.py diff --git a/tests/torchtune/utils/test_tensor_utils.py b/tests/torchtune/utils/test_tensor_utils.py deleted file mode 100644 index 14886e20f8..0000000000 --- a/tests/torchtune/utils/test_tensor_utils.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - - -import torch -from torchtune.utils.tensor_utils import _copy_tensor - - -class TestTensorUtils: - def test_copy_tensor(self): - x = torch.rand(10, 10) - y = _copy_tensor(x) - assert x.shape == y.shape - assert x.dtype == y.dtype - assert x.device == y.device - assert x.requires_grad == y.requires_grad - assert x.grad_fn == y.grad_fn - assert x.is_leaf == y.is_leaf diff --git a/torchtune/modules/low_precision/nf4_linear.py b/torchtune/modules/low_precision/nf4_linear.py index 32abdead9c..0638297e47 100644 --- a/torchtune/modules/low_precision/nf4_linear.py +++ b/torchtune/modules/low_precision/nf4_linear.py @@ -12,6 +12,8 @@ from torch import Tensor from torchao.dtypes.nf4tensor import linear_nf4, to_nf4 +# TESTING + class FrozenNF4Linear(nn.Linear): """ diff --git a/torchtune/modules/peft/lora.py b/torchtune/modules/peft/lora.py index 35ba9307a2..b2acd28b29 100644 --- a/torchtune/modules/peft/lora.py +++ b/torchtune/modules/peft/lora.py @@ -15,7 +15,6 @@ FrozenNF4Linear, ) from torchtune.modules.peft.peft_utils import AdapterModule -from torchtune.utils.tensor_utils import _copy_tensor class LoRALinear(nn.Module, AdapterModule): @@ -101,7 +100,7 @@ def _create_weight_and_bias(self): raise NotImplementedError( "Quantized LoRALinear does not support bias at the moment." ) - bias = _copy_tensor(linear.bias) + bias = linear.bias return weight, bias def adapter_params(self) -> List[str]: diff --git a/torchtune/utils/tensor_utils.py b/torchtune/utils/tensor_utils.py deleted file mode 100644 index af20904239..0000000000 --- a/torchtune/utils/tensor_utils.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - - -import torch -from torch import Tensor - - -def _copy_tensor(t: Tensor) -> Tensor: - """ - A torch.clone-free way to clone a torch.tensor. We implement without - torch.clone for better compatibility with copy.deepcopy. - """ - ret_tensor = torch.empty_like(t).requires_grad_(t.requires_grad) - with torch.no_grad(): - ret_tensor.copy_(t) - - return ret_tensor From 4c6460f2c5682991a747790bdf9506b1fbb13faa Mon Sep 17 00:00:00 2001 From: Rohan Varma Date: Tue, 2 Apr 2024 00:33:24 -0700 Subject: [PATCH 21/76] Add fp32 support for QLoRA (#595) --- requirements.txt | 2 +- .../test_lora_finetune_single_device.py | 20 ++++--- tests/torchtune/models/test_lora_llama2.py | 39 +++++++------ .../modules/low_precision/test_nf4_linear.py | 55 +++++++++---------- tests/torchtune/modules/peft/test_lora.py | 32 ++++++----- .../models/llama2/_component_builders.py | 23 ++++++-- torchtune/models/llama2/_model_builders.py | 4 +- torchtune/modules/low_precision/__init__.py | 4 +- .../low_precision/_state_dict_hooks.py | 14 +++-- torchtune/modules/low_precision/nf4_linear.py | 18 +----- torchtune/modules/peft/lora.py | 1 + 11 files changed, 112 insertions(+), 100 deletions(-) diff --git a/requirements.txt b/requirements.txt index 91e253c532..ffcecfe799 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ tqdm omegaconf # Quantization -torchao-nightly==2024.3.25 +torchao-nightly==2024.3.29 diff --git a/tests/recipes/test_lora_finetune_single_device.py b/tests/recipes/test_lora_finetune_single_device.py index 7fc590ac17..c81945abe8 100644 --- a/tests/recipes/test_lora_finetune_single_device.py +++ b/tests/recipes/test_lora_finetune_single_device.py @@ -43,11 +43,13 @@ def _get_test_config_overrides(self, dtype_str: str = "fp32"): "log_every_n_steps=1", ] - def _fetch_expected_loss_values(self, run_qlora: bool = False): - if run_qlora: + def _fetch_expected_loss_values(self): + return [10.5074, 10.5614, 10.5205, 10.4918] + + def _fetch_qlora_expected_loss_values(self, dtype): + if dtype == "bf16": return [10.5057, 10.5575, 10.5179, 10.4898] - else: - return [10.5074, 10.5614, 10.5205, 10.4918] + return [10.5059, 10.5571, 10.5181, 10.4897] @pytest.mark.integration_test def test_loss(self, tmpdir, monkeypatch): @@ -82,13 +84,14 @@ def test_loss(self, tmpdir, monkeypatch): runpy.run_path(TUNE_PATH, run_name="__main__") loss_values = get_loss_values_from_metric_logger(log_file) - expected_loss_values = self._fetch_expected_loss_values(run_qlora=False) + expected_loss_values = self._fetch_expected_loss_values() torch.testing.assert_close( loss_values, expected_loss_values, rtol=1e-5, atol=1e-5 ) @pytest.mark.integration_test - def test_loss_qlora(self, tmpdir, monkeypatch): + @pytest.mark.parametrize("dtype", ["fp32", "bf16"]) + def test_loss_qlora(self, dtype, tmpdir, monkeypatch): ckpt = "small_test_ckpt_meta" ckpt_path = Path(CKPT_MODEL_PATHS[ckpt]) ckpt_dir = ckpt_path.parent @@ -114,14 +117,13 @@ def test_loss_qlora(self, tmpdir, monkeypatch): lora_alpha=16, ) - # TODO (rohan-varma): QLoRA only supported with bf16 for now - cmd = cmd + self._get_test_config_overrides(dtype_str="bf16") + model_config + cmd = cmd + self._get_test_config_overrides(dtype_str=dtype) + model_config monkeypatch.setattr(sys, "argv", cmd) with pytest.raises(SystemExit): runpy.run_path(TUNE_PATH, run_name="__main__") loss_values = get_loss_values_from_metric_logger(log_file) - expected_loss_values = self._fetch_expected_loss_values(run_qlora=True) + expected_loss_values = self._fetch_qlora_expected_loss_values(dtype=dtype) torch.testing.assert_close( loss_values, expected_loss_values, rtol=1e-4, atol=1e-4 ) diff --git a/tests/torchtune/models/test_lora_llama2.py b/tests/torchtune/models/test_lora_llama2.py index 9509455f0e..3452b7a2e5 100644 --- a/tests/torchtune/models/test_lora_llama2.py +++ b/tests/torchtune/models/test_lora_llama2.py @@ -203,8 +203,9 @@ def test_lora_linear_quantize_base(self): if isinstance(module, LoRALinear): assert module._quantize_base - def test_qlora_llama2_parity(self, inputs): - with utils.set_default_dtype(torch.bfloat16): + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_qlora_llama2_parity(self, dtype, inputs): + with utils.set_default_dtype(dtype): model_ref = self.get_lora_llama2( lora_modules=["q_proj", "v_proj", "k_proj", "output_proj"], apply_lora_to_mlp=True, @@ -212,7 +213,7 @@ def test_qlora_llama2_parity(self, inputs): vocab_size=50, quantize_base=False, embed_dim=512, - dtype=torch.bfloat16, + dtype=dtype, ) qlora = self.get_lora_llama2( lora_modules=["q_proj", "v_proj", "k_proj", "output_proj"], @@ -221,7 +222,7 @@ def test_qlora_llama2_parity(self, inputs): vocab_size=50, quantize_base=True, embed_dim=512, - dtype=torch.bfloat16, + dtype=dtype, ) qlora_sd = qlora.state_dict() model_ref.load_state_dict(qlora_sd) @@ -232,8 +233,9 @@ def test_qlora_llama2_parity(self, inputs): output = qlora(inputs) torch.testing.assert_close(ref_output, output) - def test_qlora_llama2_state_dict(self): - with utils.set_default_dtype(torch.bfloat16): + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_qlora_llama2_state_dict(self, dtype): + with utils.set_default_dtype(dtype): model_ref = self.get_lora_llama2( lora_modules=["q_proj", "v_proj", "k_proj", "output_proj"], apply_lora_to_mlp=True, @@ -241,11 +243,11 @@ def test_qlora_llama2_state_dict(self): vocab_size=50, quantize_base=False, embed_dim=512, - dtype=torch.bfloat16, + dtype=dtype, ) - bf16_sd = model_ref.state_dict() - for v in bf16_sd.values(): - assert v.dtype == torch.bfloat16 + high_prec_sd = model_ref.state_dict() + for v in high_prec_sd.values(): + assert v.dtype == dtype # ensure quantized LoRA can load a bf16 state_dict qlora = self.get_lora_llama2( @@ -255,9 +257,9 @@ def test_qlora_llama2_state_dict(self): vocab_size=50, quantize_base=True, embed_dim=512, - dtype=torch.bfloat16, + dtype=dtype, ) - qlora.load_state_dict(bf16_sd) + qlora.load_state_dict(high_prec_sd) # LoRALinear base weights should be nf4 still for module in qlora.modules(): if isinstance(module, LoRALinear): @@ -265,10 +267,11 @@ def test_qlora_llama2_state_dict(self): # saved state_dict should have bf16 weights. qlora_sd = qlora.state_dict() for v in qlora_sd.values(): - assert v.dtype == torch.bfloat16 + assert v.dtype == dtype - def test_qlora_llama2_merged_state_dict(self): - with utils.set_default_dtype(torch.bfloat16): + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_qlora_llama2_merged_state_dict(self, dtype): + with utils.set_default_dtype(dtype): qlora = self.get_lora_llama2( lora_modules=["q_proj", "v_proj", "k_proj", "output_proj"], apply_lora_to_mlp=True, @@ -276,7 +279,7 @@ def test_qlora_llama2_merged_state_dict(self): vocab_size=50, quantize_base=True, embed_dim=512, - dtype=torch.bfloat16, + dtype=dtype, reset_norm=False, # to ensure norm.scale key exists ) @@ -286,10 +289,10 @@ def test_qlora_llama2_merged_state_dict(self): for v in merged_ckpt.values(): # paranoid check for both, as NF4Tensor had issue where NF4Tensor.dtype would return bf16 assert not isinstance(v, NF4Tensor) - assert v.dtype == torch.bfloat16 + assert v.dtype == dtype # Ensure checkpoint can be loaded into non-LoRA model - with utils.set_default_dtype(torch.bfloat16): + with utils.set_default_dtype(dtype): llama2 = self.get_ref_llama2(vocab_size=50, embed_dim=512) llama2.load_state_dict(merged_ckpt) diff --git a/tests/torchtune/modules/low_precision/test_nf4_linear.py b/tests/torchtune/modules/low_precision/test_nf4_linear.py index 1f35092216..668defa915 100644 --- a/tests/torchtune/modules/low_precision/test_nf4_linear.py +++ b/tests/torchtune/modules/low_precision/test_nf4_linear.py @@ -48,57 +48,53 @@ def test_bias_unsupported(self): with pytest.raises(RuntimeError, match="does not currently support biases"): _ = FrozenNF4Linear(1, 1, bias=True) - def test_non_bf16_unsupported(self): - with pytest.raises(RuntimeError, match="only supported with bf16"): - _ = FrozenNF4Linear(1, 1, dtype=torch.float32) - - def test_parameters(self): - nf4_linear = FrozenNF4Linear(512, 512, device="cpu", dtype=torch.bfloat16) + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_parameters(self, dtype): + nf4_linear = FrozenNF4Linear(512, 512, device="cpu", dtype=dtype) params = list(nf4_linear.parameters()) assert len(params) == 1 assert isinstance(params[0], NF4Tensor) - def test_state_dict(self): - nf4_linear = FrozenNF4Linear(512, 512, device="cpu", dtype=torch.bfloat16) + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_state_dict(self, dtype): + nf4_linear = FrozenNF4Linear(512, 512, device="cpu", dtype=dtype) state_dict = nf4_linear.state_dict() assert len(state_dict) == 1 assert isinstance(state_dict["weight"], NF4Tensor) - def test_frozen_nf4_linear(self): - nf4_linear = FrozenNF4Linear(512, 512, device="cpu", dtype=torch.bfloat16) - assert isinstance(nf4_linear.weight, NF4Tensor) - assert torch.bfloat16 == nf4_linear.weight.get_original_weight().dtype - - def test_output_bf16(self): - # Test to ensure W4 A16 produces A16 - nf4_linear = FrozenNF4Linear(512, 512, device="cpu", dtype=torch.bfloat16) - inp = torch.randn(2, 512, dtype=torch.bfloat16, requires_grad=True) + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_output_dtype(self, dtype): + # Test to ensure W4 A16 produces A16 / W4A32 produces A32 + nf4_linear = FrozenNF4Linear(512, 512, device="cpu", dtype=dtype) + inp = torch.randn(2, 512, dtype=dtype, requires_grad=True) out = nf4_linear(inp) - assert out.dtype == torch.bfloat16 + assert out.dtype == dtype - def test_backward_bf16(self): + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_backward_dtype(self, dtype): # Test to ensure backward pass gives activation a bf16 gradient and no gradient # to the linear's weight, as it is frozen. - nf4_linear = FrozenNF4Linear(512, 512, device="cpu", dtype=torch.bfloat16) - inp = torch.randn(2, 512, dtype=torch.bfloat16, requires_grad=True) + nf4_linear = FrozenNF4Linear(512, 512, device="cpu", dtype=dtype) + inp = torch.randn(2, 512, dtype=dtype, requires_grad=True) nf4_linear(inp).sum().backward() - assert inp.grad is not None and inp.grad.dtype == torch.bfloat16 + assert inp.grad is not None and inp.grad.dtype == dtype assert nf4_linear.weight.grad is None @pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA available") - def test_nf4_reconstruction_vs_bnb(self): + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_nf4_reconstruction_vs_bnb(self, dtype): """ Ensures a BNB NF4 linear and our FrozenNF4Linear have low error when reconstructing the respective original weights. """ dim = 512 - nf4_linear = FrozenNF4Linear(dim, dim, device="cuda", dtype=torch.bfloat16) + nf4_linear = FrozenNF4Linear(dim, dim, device="cuda", dtype=dtype) orig_weight = nf4_linear.weight.get_original_weight().clone().detach() bnb_nf4_linear = _build_bnb_linear(input_weight=orig_weight) # From https://github.com/drisspg/transformer_nuggets/blob/f05afad68ad9086d342268f46a7f344617a02314/test/test_qlora.py#L65 bnb_reconstruction = bnb_nf4_linear( - torch.eye(dim, dim, dtype=torch.bfloat16, device="cuda") + torch.eye(dim, dim, dtype=dtype, device="cuda") ) # Ensure nf4_linear and bnb reconstructions are close to each other. diff = ( @@ -107,18 +103,19 @@ def test_nf4_reconstruction_vs_bnb(self): assert diff.item() < 1e-2 @pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA available") - def test_nf4_bnb_linear(self): + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_nf4_bnb_linear(self, dtype): """ This test ensures that nf4_linear is "no worse" than BNB by ensuring the error compared to a bf16 linear is not more than BNB's implementation. """ dim = 512 - nf4_linear = FrozenNF4Linear(dim, dim, device="cuda", dtype=torch.bfloat16) + nf4_linear = FrozenNF4Linear(dim, dim, device="cuda", dtype=dtype) orig_weight = nf4_linear.weight.get_original_weight().clone().detach() bnb_nf4_linear = _build_bnb_linear(input_weight=orig_weight) - bf16_linear = torch.nn.Linear(dim, dim, device="cuda", dtype=torch.bfloat16) + bf16_linear = torch.nn.Linear(dim, dim, device="cuda", dtype=dtype) - inp = torch.randn(2, 512, dtype=torch.bfloat16, device="cuda") + inp = torch.randn(2, 512, dtype=dtype, device="cuda") out_nf4 = nf4_linear(inp) out_bnb = bnb_nf4_linear(inp) diff --git a/tests/torchtune/modules/peft/test_lora.py b/tests/torchtune/modules/peft/test_lora.py index e87fc072bd..80c253f04c 100644 --- a/tests/torchtune/modules/peft/test_lora.py +++ b/tests/torchtune/modules/peft/test_lora.py @@ -13,7 +13,7 @@ from torch import nn from torchao.dtypes.nf4tensor import NF4Tensor, to_nf4 from torchtune import utils -from torchtune.modules.low_precision import reparametrize_as_bf16_state_dict_post_hook +from torchtune.modules.low_precision import reparametrize_as_dtype_state_dict_post_hook from torchtune.modules.peft import LoRALinear from torchtune.utils.seed import set_seed @@ -111,8 +111,9 @@ def test_quantize_with_bias_raises(self): quantize_base=True, ) - def test_qlora_parity(self): - with utils.set_default_dtype(torch.bfloat16): + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_qlora_parity(self, dtype): + with utils.set_default_dtype(dtype): qlora_linear = LoRALinear( in_dim=512, out_dim=512, @@ -130,20 +131,21 @@ def test_qlora_parity(self): quantize_base=False, ) - # set weight of lora_linear to unquantized bf16 of qlora_linear and check + # set weight of lora_linear to unquantized weight of qlora_linear and check # parity. - lora_linear.weight.data = qlora_linear.weight.get_original_weight() + lora_linear.weight.data = qlora_linear.weight.to(dtype) # Ensure forward passes are the same. This is because LoRALinear should use a special - # quantized linear operator that runs compute in bf16 (but only saves the 4 bit quantized tensor) + # quantized linear operator that runs compute in higher prec (but only saves the 4 bit quantized tensor) # for autograd. - inputs = torch.randn(BSZ, SEQ_LEN, 512, dtype=torch.bfloat16) + inputs = torch.randn(BSZ, SEQ_LEN, 512, dtype=dtype) lora_linear_out = lora_linear(inputs) qlora_linear_out = qlora_linear(inputs) torch.testing.assert_close(lora_linear_out, qlora_linear_out) - def test_quantized_state_dict_bf16(self): - with utils.set_default_dtype(torch.bfloat16): + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + def test_quantized_state_dict(self, dtype): + with utils.set_default_dtype(dtype): lora_linear = LoRALinear( in_dim=512, out_dim=512, @@ -154,12 +156,16 @@ def test_quantized_state_dict_bf16(self): ) lora_linear._register_state_dict_hook( - partial(reparametrize_as_bf16_state_dict_post_hook, offload_to_cpu=False) + partial( + reparametrize_as_dtype_state_dict_post_hook, + dtype=dtype, + offload_to_cpu=False, + ) ) sd = lora_linear.state_dict() - # No nf4 tensors, all bf16 + # No nf4 tensors, all have type dtype for v in sd.values(): - assert v.dtype == torch.bfloat16 + assert v.dtype == dtype assert not isinstance(v, NF4Tensor) # Load back in results in re-quant and creates the same nf4 tensor. @@ -177,7 +183,7 @@ def test_quantized_state_dict_bf16(self): to_nf4( torch.zeros_like( lora_linear.weight.get_original_weight(), - dtype=torch.bfloat16, + dtype=dtype, device=lora_linear.weight.device, ) ) diff --git a/torchtune/models/llama2/_component_builders.py b/torchtune/models/llama2/_component_builders.py index 476fc45bb7..9abd305c6e 100644 --- a/torchtune/models/llama2/_component_builders.py +++ b/torchtune/models/llama2/_component_builders.py @@ -21,7 +21,7 @@ TransformerDecoderLayer, ) -from torchtune.modules.low_precision import reparametrize_as_bf16_state_dict_post_hook +from torchtune.modules.low_precision import reparametrize_as_dtype_state_dict_post_hook from torchtune.modules.peft import LORA_ATTN_MODULES, LoRALinear @@ -40,6 +40,7 @@ # ------------------ Vanilla Llama2 ------------------ + def llama2( vocab_size: int, num_layers: int, @@ -96,7 +97,9 @@ def llama2( max_seq_len=max_seq_len, attn_dropout=attn_dropout, ) - hidden_dim = intermediate_dim if intermediate_dim else scale_hidden_dim_for_mlp(embed_dim) + hidden_dim = ( + intermediate_dim if intermediate_dim else scale_hidden_dim_for_mlp(embed_dim) + ) mlp = llama2_mlp(dim=embed_dim, hidden_dim=hidden_dim) layer = TransformerDecoderLayer( attn=self_attn, @@ -117,6 +120,7 @@ def llama2( output=output_proj, ) + def llama2_mlp(dim: int, hidden_dim: int) -> FeedForward: """ Build the MLP layer associated with the Llama model. @@ -127,7 +131,6 @@ def llama2_mlp(dim: int, hidden_dim: int) -> FeedForward: return FeedForward(gate_proj=gate_proj, down_proj=down_proj, up_proj=up_proj) - # ------------------ LoRA Llama2 ------------------ @@ -206,7 +209,9 @@ def lora_llama2( quantize_base=quantize_base, ) - hidden_dim = intermediate_dim if intermediate_dim else scale_hidden_dim_for_mlp(embed_dim) + hidden_dim = ( + intermediate_dim if intermediate_dim else scale_hidden_dim_for_mlp(embed_dim) + ) if apply_lora_to_mlp: mlp = lora_llama2_mlp( dim=embed_dim, @@ -245,10 +250,16 @@ def lora_llama2( ) if quantize_base: - # For QLoRA, we reparametrize 4-bit tensors to bf16, and offload to CPU on the fly + # For QLoRA, we reparametrize 4-bit tensors to higher precision, and offload to CPU on the fly # so as to not increase peak memory model._register_state_dict_hook( - partial(reparametrize_as_bf16_state_dict_post_hook, offload_to_cpu=True) + partial( + reparametrize_as_dtype_state_dict_post_hook, + # TODO this is clowny, figure out a better way to get what precision the rest + # of the model is in + dtype=tok_embeddings.weight.dtype, + offload_to_cpu=True, + ) ) return model diff --git a/torchtune/models/llama2/_model_builders.py b/torchtune/models/llama2/_model_builders.py index 62ddb8e55d..59fd4d1a8e 100644 --- a/torchtune/models/llama2/_model_builders.py +++ b/torchtune/models/llama2/_model_builders.py @@ -182,5 +182,7 @@ def lora_llama2_13b( lora_rank=lora_rank, lora_alpha=lora_alpha, lora_dropout=0.05, - quantize_base=False, + quantize_base=quantize_base, ) + +qlora_llama2_13b = partial(lora_llama2_13b, quantize_base=True) diff --git a/torchtune/modules/low_precision/__init__.py b/torchtune/modules/low_precision/__init__.py index b68bdc6611..2847a98ccd 100644 --- a/torchtune/modules/low_precision/__init__.py +++ b/torchtune/modules/low_precision/__init__.py @@ -4,8 +4,8 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from ._state_dict_hooks import reparametrize_as_bf16_state_dict_post_hook +from ._state_dict_hooks import reparametrize_as_dtype_state_dict_post_hook from .nf4_linear import FrozenNF4Linear -__all__ = ["FrozenNF4Linear", "reparametrize_as_bf16_state_dict_post_hook"] +__all__ = ["FrozenNF4Linear", "reparametrize_as_dtype_state_dict_post_hook"] diff --git a/torchtune/modules/low_precision/_state_dict_hooks.py b/torchtune/modules/low_precision/_state_dict_hooks.py index c79985b3ce..9133451608 100644 --- a/torchtune/modules/low_precision/_state_dict_hooks.py +++ b/torchtune/modules/low_precision/_state_dict_hooks.py @@ -6,24 +6,27 @@ from typing import Any, Dict, Tuple +import torch + import torch.nn as nn from torchao.dtypes.nf4tensor import NF4Tensor -def reparametrize_as_bf16_state_dict_post_hook( +def reparametrize_as_dtype_state_dict_post_hook( model: nn.Module, state_dict: Dict[str, Any], *args: Tuple[Any, ...], + dtype: torch.dtype = torch.bfloat16, offload_to_cpu: bool = True, **kwargs: Dict[Any, Any], ): """ A state_dict hook that replaces nf4 tensors with their restored - bf16 weight and optionally offloads the restored weight to CPU. + higher-precision weight and optionally offloads the restored weight to CPU. This function is meant to be used with PyTorch's ``nn.Module._register_state_dict_hook``, i.e. >>> m = MyModule() - >>> m._register_state_dict_hook(reparametrize_as_bf16_state_dict_post_hook) + >>> m._register_state_dict_hook(reparametrize_as_dtype_state_dict_post_hook) If the hook is registered per the above process, this hook will be called _after_ the module's ``state_dict`` method is called. The hook will replace all ``NF4Tensor`` instances by unquantizing @@ -33,11 +36,12 @@ def reparametrize_as_bf16_state_dict_post_hook( model (nn.Module): the model to take ``state_dict()`` on state_dict (Dict[str, Any]): the state dict to modify *args (Tuple[Any, ...]): Unused args passed when running this as a state_dict hook. - offload_to_cpu (bool): whether to offload the restored weight to CPU + dtype (torch.dtype): the dtype to restore the weight to. Default is ``torch.bfloat16``. + offload_to_cpu (bool): whether to offload the restored weight to CPU. Default is ``True``. **kwargs (Dict[Any, Any]): Unused keyword args passed when running this as a state_dict hook. """ for k, v in state_dict.items(): if isinstance(v, NF4Tensor): - state_dict[k] = v.get_original_weight() + state_dict[k] = v.to(dtype) if offload_to_cpu: state_dict[k] = state_dict[k].cpu() diff --git a/torchtune/modules/low_precision/nf4_linear.py b/torchtune/modules/low_precision/nf4_linear.py index 0638297e47..236503c40f 100644 --- a/torchtune/modules/low_precision/nf4_linear.py +++ b/torchtune/modules/low_precision/nf4_linear.py @@ -22,8 +22,6 @@ class FrozenNF4Linear(nn.Linear): and is meant to be used as the base Linear layer for modeling use cases such as QLoRA where base model parameters are frozen. NOTE: biases are currently not supported. - NOTE: This class always creates the underlying full precision weight as bf16 dtypte. Note that - this will override the default PyTorch dtype that is set via `torch.set_default_dtype`. Args: in_dim (int): input dimension @@ -34,7 +32,6 @@ class FrozenNF4Linear(nn.Linear): Raises: RuntimeError: if ``bias`` is set to ``True`` - RuntimeError: if ``dtype`` is not set to ``torch.bfloat16`` """ def __init__( @@ -43,27 +40,16 @@ def __init__( if "bias" in kwargs and kwargs.pop("bias"): raise RuntimeError("FrozenNF4Linear does not currently support biases!") - if "dtype" in kwargs: - kwargs_dtype = kwargs.pop("dtype") - if kwargs_dtype != torch.bfloat16: - raise RuntimeError( - "FrozenNF4Linear is only supported with bf16 parameter currently." - ) - super().__init__( - in_dim, out_dim, device=device, dtype=torch.bfloat16, bias=False, **kwargs - ) + super().__init__(in_dim, out_dim, device=device, bias=False, **kwargs) self.weight.requires_grad_(False) self.nf4_weight = to_nf4(self.weight.data) # re-register self.weight as the nf4 weight, so that the nf4 weight # shows up as expected in .parameters, state_dict, etc. self.weight = torch.nn.Parameter(self.nf4_weight, requires_grad=False) - # TODO: likely need to handle state_dict save & load via hooks to properly manage - # types. - def forward(self, input: Tensor) -> Tensor: """ - Runs linear operation with input tensor as given by `input`. Computation happens in bf16 + Runs linear operation with input tensor as given by `input`. Computation happens in higher precision, though only the nf4 weight is saved for backward for gradient computation to ensure additional memory is not used. Args: diff --git a/torchtune/modules/peft/lora.py b/torchtune/modules/peft/lora.py index b2acd28b29..3e09a5006d 100644 --- a/torchtune/modules/peft/lora.py +++ b/torchtune/modules/peft/lora.py @@ -9,6 +9,7 @@ import torch.nn.functional as F from torch import nn, Tensor + from torchao.dtypes.nf4tensor import linear_nf4 from torchtune.modules.low_precision import ( # noqa: F401 _register_nf4_dispatch_ops, From 72dd372f2a687c3002e97edf407503ab38a48b12 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 2 Apr 2024 14:55:19 +0100 Subject: [PATCH 22/76] Add link to docs from README (#636) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c6d9292935..df1de940f7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Stay *tuned* for the first release in the coming weeks! # TorchTune -[**Introduction**](#introduction) | [**Installation**](#installation) | [**Get Started**](#get-started) | [**Design Principles**](#design-principles) | [**Contributing**](#contributing) | [**License**](#license) +[**Introduction**](#introduction) | [**Installation**](#installation) | [**Get Started**](#get-started) | [**Documentation**](https://pytorch.org/torchtune) | [**Design Principles**](#design-principles) | [**Contributing**](#contributing) | [**License**](#license)   From d876889aa014f3d6b1fb5af304f86c7bedf52849 Mon Sep 17 00:00:00 2001 From: ebsmothers Date: Tue, 2 Apr 2024 08:36:55 -0700 Subject: [PATCH 23/76] Refactor datasets and tokenizer (#624) --- tests/test_utils.py | 7 +- tests/torchtune/config/test_config_utils.py | 32 -- tests/torchtune/data/test_chat_formats.py | 130 ++++++++ tests/torchtune/data/test_data_utils.py | 83 +---- ...emplates.py => test_instruct_templates.py} | 100 ------ tests/torchtune/datasets/test_chat_dataset.py | 128 +++----- .../datasets/test_slimorca_dataset.py | 9 +- tests/torchtune/modules/test_tokenizer.py | 240 +++++++++++++++ torchtune/config/_utils.py | 36 --- torchtune/data/__init__.py | 32 +- torchtune/data/_chat_formats.py | 215 +++++++++++++ torchtune/data/_instruct_templates.py | 147 +++++++++ torchtune/data/_templates.py | 285 ------------------ torchtune/data/_transforms.py | 21 +- torchtune/data/_types.py | 10 +- torchtune/data/_utils.py | 71 +---- torchtune/datasets/_alpaca.py | 2 +- torchtune/datasets/_chat.py | 99 ++---- torchtune/datasets/_grammar.py | 2 +- torchtune/datasets/_instruct.py | 39 ++- torchtune/datasets/_samsum.py | 2 +- torchtune/datasets/_slimorca.py | 6 +- torchtune/modules/tokenizer.py | 109 ++++++- 23 files changed, 984 insertions(+), 821 deletions(-) create mode 100644 tests/torchtune/data/test_chat_formats.py rename tests/torchtune/data/{test_templates.py => test_instruct_templates.py} (61%) create mode 100644 torchtune/data/_chat_formats.py create mode 100644 torchtune/data/_instruct_templates.py delete mode 100644 torchtune/data/_templates.py diff --git a/tests/test_utils.py b/tests/test_utils.py index 8f1be9a569..2ca3fcfee6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -18,7 +18,7 @@ import torch from torch import nn - +from torchtune.modules import Tokenizer skip_if_cuda_not_available = unittest.skipIf( not torch.cuda.is_available(), "CUDA is not available" @@ -31,8 +31,11 @@ "llama2_7b": "/tmp/test-artifacts/llama2-7b-torchtune.pt", } +# Inherit from tokenizer class to reuse its tokenize_messages method +class DummyTokenizer(Tokenizer): + def __init__(self): + self.encodes_whitespace = False -class DummyTokenizer: def encode(self, text, add_bos=True, add_eos=True, **kwargs): words = text.split() tokens = [len(word) for word in words] diff --git a/tests/torchtune/config/test_config_utils.py b/tests/torchtune/config/test_config_utils.py index 739f6d06bc..1f7f01d4e5 100644 --- a/tests/torchtune/config/test_config_utils.py +++ b/tests/torchtune/config/test_config_utils.py @@ -9,11 +9,9 @@ import pytest from torchtune.config._utils import ( _get_component_from_path, - _get_template, _merge_yaml_and_cli_args, InstantiationError, ) -from torchtune.data import AlpacaInstructTemplate from torchtune.utils.argparse import TuneRecipeArgumentParser _CONFIG = { @@ -109,33 +107,3 @@ def test_merge_yaml_and_cli_args(self, mock_load): ValueError, match="Command-line overrides must be in the form of key=value" ): _ = _merge_yaml_and_cli_args(yaml_args, cli_args) - - def test_get_template(self): - # Test valid template class - template = _get_template("AlpacaInstructTemplate") - assert isinstance(template, AlpacaInstructTemplate) - - # Test invalid template class - with pytest.raises( - ValueError, - match="Must be a PromptTemplate class or a string with placeholders.", - ): - _ = _get_template("InvalidTemplate") - - # Test valid template strings - valid_templates = [ - "Instruction: {instruction}\nInput: {input}", - "Instruction: {instruction}", - "{a}", - ] - for template in valid_templates: - assert _get_template(template) == template - - # Test invalid template strings - invalid_templates = ["hello", "{}", "a}{b"] - for template in invalid_templates: - with pytest.raises( - ValueError, - match="Must be a PromptTemplate class or a string with placeholders.", - ): - _ = _get_template(template) diff --git a/tests/torchtune/data/test_chat_formats.py b/tests/torchtune/data/test_chat_formats.py new file mode 100644 index 0000000000..f0ba051824 --- /dev/null +++ b/tests/torchtune/data/test_chat_formats.py @@ -0,0 +1,130 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import pytest +from torchtune.data import ChatMLFormat, Llama2ChatFormat, Message, MistralChatFormat + +# Taken from Open-Orca/SlimOrca-Dedup on HuggingFace: +# https://huggingface.co/datasets/Open-Orca/SlimOrca-Dedup +CHAT_SAMPLE = [ + Message( + role="system", + content="You are an AI assistant. User will you give you a task. " + "Your goal is to complete the task as faithfully as you can. " + "While performing the task think step-by-step and justify your steps.", + ), + Message( + role="user", + content="Please briefly summarize this news article:\n\nAOL.com Video - " + "Father Lets 8-Year-Old Drive On Icy Road\n\nDescription:Would you let your " + "8-year-old drive your car? How about on an icy road? Well one father in " + "Russia did just that, and recorded the entire thing. To her credit, the " + "child seemed to be doing a great job. (0:44)\n\nTags: 8-year-old driver , " + "caught on camera , child driver , pix11\n\nSummary:", + ), + Message( + role="assistant", + content="A father in Russia allowed his 8-year-old child to drive his car " + "on an icy road and recorded the event. The child appeared to be handling the " + "situation well, showcasing their driving skills despite the challenging conditions.", + ), +] + + +def _assert_dialogue_equal(actual, expected): + assert len(actual) == len(expected) + for i in range(len(actual)): + assert actual[i].role == expected[i].role + assert actual[i].content == expected[i].content + + +class TestLlama2ChatFormat: + expected_dialogue = [ + Message( + role="user", + content="[INST] <>\nYou are an AI assistant. User will you give you a task. " + "Your goal is to complete the task as faithfully as you can. While performing " + "the task think step-by-step and justify your steps.\n<>\n\nPlease " + "briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old " + "Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? " + "How about on an icy road? Well one father in Russia did just that, and recorded " + "the entire thing. To her credit, the child seemed to be doing a great job. " + "(0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\n" + "Summary: [/INST] ", + ), + Message( + role="assistant", + content="A father in Russia allowed his 8-year-old child to drive his car on an " + "icy road and recorded the event. The child appeared to be handling the situation well, " + "showcasing their driving skills despite the challenging conditions.", + ), + ] + + def test_format(self): + actual = Llama2ChatFormat.format(CHAT_SAMPLE) + _assert_dialogue_equal(actual, self.expected_dialogue) + + +class TestMistralChatFormat: + expected_dialogue = [ + Message( + role="user", + content="[INST] Please briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old " + "Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? " + "How about on an icy road? Well one father in Russia did just that, and recorded " + "the entire thing. To her credit, the child seemed to be doing a great job. " + "(0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\n" + "Summary: [/INST] ", + ), + Message( + role="assistant", + content="A father in Russia allowed his 8-year-old child to drive his car on an " + "icy road and recorded the event. The child appeared to be handling the situation well, " + "showcasing their driving skills despite the challenging conditions.", + ), + ] + + def test_format(self): + no_system_sample = CHAT_SAMPLE[1:] + actual = MistralChatFormat.format(no_system_sample) + _assert_dialogue_equal(actual, self.expected_dialogue) + + def test_format_with_system_prompt_raises(self): + with pytest.raises( + ValueError, match="System prompts are not supported in MistralChatFormat" + ): + _ = MistralChatFormat.format(CHAT_SAMPLE) + + +class TestChatMLFormat: + expected_dialogue = [ + Message( + role="system", + content="<|im_start|>system\nYou are an AI assistant. User will you give you a task. " + "Your goal is to complete the task as faithfully as you can. While performing " + "the task think step-by-step and justify your steps.<|im_end|>\n", + ), + Message( + role="user", + content="<|im_start|>user\nPlease " + "briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old " + "Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? " + "How about on an icy road? Well one father in Russia did just that, and recorded " + "the entire thing. To her credit, the child seemed to be doing a great job. " + "(0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\n" + "Summary:<|im_end|>\n", + ), + Message( + role="assistant", + content="<|im_start|>assistant\nA father in Russia allowed his 8-year-old child to drive his car on an " + "icy road and recorded the event. The child appeared to be handling the situation well, " + "showcasing their driving skills despite the challenging conditions.<|im_end|>", + ), + ] + + def test_format(self): + actual = ChatMLFormat.format(CHAT_SAMPLE) + _assert_dialogue_equal(actual, self.expected_dialogue) diff --git a/tests/torchtune/data/test_data_utils.py b/tests/torchtune/data/test_data_utils.py index 475c0cba00..599f9cc840 100644 --- a/tests/torchtune/data/test_data_utils.py +++ b/tests/torchtune/data/test_data_utils.py @@ -4,84 +4,21 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from tests.test_utils import DummyTokenizer -from torchtune.data import tokenize_prompt_and_response, truncate -from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX - - -def test_tokenize_prompt_and_response(): - tokenizer = DummyTokenizer() - prompt = "Instruction:\nThis is an instruction.\n\nInput:\nThis is an input.\n\nResponse: " - response = "I always know what I'm doing, do you?" - prompt_length = 12 - expected_tokenized_prompt = [ - 0, - 12, - 4, - 2, - 2, - 12, - 6, - 4, - 2, - 2, - 6, - 9, - 1, - 6, - 4, - 4, - 3, - 6, - 2, - 4, - -1, - ] - expected_tokenized_label = [CROSS_ENTROPY_IGNORE_IDX] * prompt_length + [ - 1, - 6, - 4, - 4, - 3, - 6, - 2, - 4, - -1, - ] - - tokenized_prompt, tokenized_label = tokenize_prompt_and_response( - tokenizer, prompt, response - ) - assert tokenized_prompt == expected_tokenized_prompt - assert tokenized_label == expected_tokenized_label - - tokenized_prompt, tokenized_label = tokenize_prompt_and_response( - tokenizer, prompt, response, train_on_input=True - ) - assert tokenized_prompt == expected_tokenized_prompt - assert tokenized_label == expected_tokenized_prompt +from torchtune.data import truncate def test_truncate(): - prompt_tokens = [1, 2, 3, 4, -1] - label_tokens = [1, 2, 3, 4, -1] + tokens = [1, 2, 3, 4, -1] # Test no truncation - truncated_prompt_tokens, truncated_label_tokens = truncate( - tokenizer=DummyTokenizer(), - prompt_tokens=prompt_tokens, - label_tokens=label_tokens, + truncated_tokens = truncate( + tokens=tokens, max_seq_len=5, + eos_id=-1, ) - assert truncated_prompt_tokens == prompt_tokens - assert truncated_label_tokens == label_tokens + assert truncated_tokens == tokens - # Test truncated - truncated_prompt_tokens, truncated_label_tokens = truncate( - tokenizer=DummyTokenizer(), - prompt_tokens=prompt_tokens, - label_tokens=label_tokens, - max_seq_len=4, - ) - assert truncated_prompt_tokens == [1, 2, 3, -1] - assert truncated_label_tokens == [1, 2, 3, -1] + masks = [True, True, False, True, False] + # Test truncated mask + truncated_masks = truncate(tokens=masks, max_seq_len=4, eos_id=False) + assert truncated_masks == [True, True, False, False] diff --git a/tests/torchtune/data/test_templates.py b/tests/torchtune/data/test_instruct_templates.py similarity index 61% rename from tests/torchtune/data/test_templates.py rename to tests/torchtune/data/test_instruct_templates.py index 8132894fbd..be8f34ebd8 100644 --- a/tests/torchtune/data/test_templates.py +++ b/tests/torchtune/data/test_instruct_templates.py @@ -4,13 +4,9 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import pytest from torchtune.data import ( AlpacaInstructTemplate, - ChatMLTemplate, GrammarErrorCorrectionTemplate, - Llama2ChatTemplate, - MistralChatTemplate, SummarizeTemplate, ) @@ -156,99 +152,3 @@ def test_format_with_column_map(self): actual = self.template.format(modified_sample, column_map=column_map) assert actual == expected_prompt - - -class TestLlama2ChatTemplate: - expected_prompt = ( - "[INST] <>\nYou are an AI assistant. User will you give you a task. " - "Your goal is to complete the task as faithfully as you can. While performing " - "the task think step-by-step and justify your steps.\n<>\n\nPlease " - "briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old " - "Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? " - "How about on an icy road? Well one father in Russia did just that, and recorded " - "the entire thing. To her credit, the child seemed to be doing a great job. " - "(0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\n" - "Summary: [/INST] " - ) - - template = Llama2ChatTemplate() - - def test_format(self): - actual = self.template.format(CHAT_SAMPLE) - assert actual == self.expected_prompt - - def test_format_with_column_map(self): - column_map = {"system": "not_system"} - modified_sample = CHAT_SAMPLE.copy() - modified_sample["not_system"] = modified_sample["system"] - del modified_sample["system"] - - actual = self.template.format(modified_sample, column_map=column_map) - - assert actual == self.expected_prompt - - -class TestMistralChatTemplate: - expected_prompt = ( - "[INST] Please briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old " - "Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? " - "How about on an icy road? Well one father in Russia did just that, and recorded " - "the entire thing. To her credit, the child seemed to be doing a great job. " - "(0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\n" - "Summary: [/INST] " - ) - - template = MistralChatTemplate() - - def test_format(self): - no_system_sample = CHAT_SAMPLE.copy() - del no_system_sample["system"] - actual = self.template.format(no_system_sample) - assert actual == self.expected_prompt - - def test_format_with_system_prompt_raises(self): - with pytest.raises( - ValueError, match="System prompts are not supported in MistralChatTemplate" - ): - _ = self.template.format(CHAT_SAMPLE) - - def test_format_with_column_map(self): - column_map = {"user": "not_user"} - modified_sample = CHAT_SAMPLE.copy() - modified_sample["not_user"] = modified_sample["user"] - del modified_sample["system"] - del modified_sample["user"] - - actual = self.template.format(modified_sample, column_map=column_map) - - assert actual == self.expected_prompt - - -class TestChatMLTemplate: - expected_prompt = ( - "<|im_start|>system\nYou are an AI assistant. User will you give you a task. " - "Your goal is to complete the task as faithfully as you can. While performing " - "the task think step-by-step and justify your steps.<|im_end|>\n<|im_start|>user\nPlease " - "briefly summarize this news article:\n\nAOL.com Video - Father Lets 8-Year-Old " - "Drive On Icy Road\n\nDescription:Would you let your 8-year-old drive your car? " - "How about on an icy road? Well one father in Russia did just that, and recorded " - "the entire thing. To her credit, the child seemed to be doing a great job. " - "(0:44)\n\nTags: 8-year-old driver , caught on camera , child driver , pix11\n\n" - "Summary:<|im_end|>\n<|im_start|>assistant\n" - ) - - template = ChatMLTemplate() - - def test_format(self): - actual = self.template.format(CHAT_SAMPLE) - assert actual == self.expected_prompt - - def test_format_with_column_map(self): - column_map = {"system": "not_system"} - modified_sample = CHAT_SAMPLE.copy() - modified_sample["not_system"] = modified_sample["system"] - del modified_sample["system"] - - actual = self.template.format(modified_sample, column_map=column_map) - - assert actual == self.expected_prompt diff --git a/tests/torchtune/datasets/test_chat_dataset.py b/tests/torchtune/datasets/test_chat_dataset.py index 66deb2c2c6..3397a8e244 100644 --- a/tests/torchtune/datasets/test_chat_dataset.py +++ b/tests/torchtune/datasets/test_chat_dataset.py @@ -14,18 +14,28 @@ from torchtune.datasets import ChatDataset -class DummyTemplate: - def __init__(self): - self.template = { - "system": "System:\n{system}\nUser:\n{user}\nAssistant:\n", - "no_system": "User:\n{user}\nAssistant:\n", - } - - def format(self, sample, column_map=None): - if "system" in sample: - return self.template["system"].format(**sample) - else: - return self.template["no_system"].format(**sample) +class DummyChatFormat: + + B_SYS, E_SYS = "System:\n", "\n" + B_INST, E_INST = "User:\n", "\nAssistant:\n" + B_ASST, E_ASST = "", "" + system = f"{B_SYS}{{content}}{E_SYS}" + user = f"{B_INST}{{content}}{E_INST}" + assistant = f"{B_ASST}{{content}}{E_ASST}" + + @classmethod + def format( + cls, + messages, + ): + formats = {"system": cls.system, "user": cls.user, "assistant": cls.assistant} + formatted_dialogue = [] + for message in messages: + content = formats.get(message.role).format(content=message.content) + formatted_dialogue.append( + Message(role=message.role, content=content, masked=message.masked), + ) + return formatted_dialogue def _are_messages_equal(messages_a, messages_b): @@ -39,90 +49,33 @@ def _are_messages_equal(messages_a, messages_b): class TestChatDataset: @pytest.fixture - def template(self): - return DummyTemplate() + def chat_format(self): + return DummyChatFormat() @pytest.fixture def dialogue(self): return [ { "dialogue": [ - Message(role="system", content="You are an AI assistant."), - Message(role="user", content="What is the meaning of life?"), - Message(role="assistant", content="The meaning of life is 42."), - Message(role="user", content="That's ridiculous."), - Message(role="assistant", content="I agree."), + Message( + role="system", content="You are an AI assistant.", masked=True + ), + Message( + role="user", content="What is the meaning of life?", masked=True + ), + Message( + role="assistant", + content="The meaning of life is 42.", + masked=False, + ), + Message(role="user", content="That's ridiculous.", masked=True), + Message(role="assistant", content="I agree.", masked=False), ], }, ] @mock.patch("torchtune.datasets._chat.load_dataset") - def test_get_turns(self, mock_load_dataset, template, dialogue): - ds = ChatDataset( - tokenizer=DummyTokenizer(), - source="iam/agoofy/goober", - convert_to_dialogue=lambda x: x, - template=template, - max_seq_len=100, - train_on_input=False, - ) - - # Test a normal multiturn dialogue - prompts, responses = zip( - *[(p, l) for p, l in ds._get_turns(dialogue[0]["dialogue"])] - ) - assert prompts[0] == { - "system": "You are an AI assistant.", - "user": "What is the meaning of life?", - } - assert responses[0] == "The meaning of life is 42." - assert prompts[1] == {"user": "That's ridiculous."} - assert responses[1] == "I agree." - - # Test without system prompt - prompts, responses = zip( - *[(p, l) for p, l in ds._get_turns(dialogue[0]["dialogue"][1:])] - ) - assert prompts[0] == {"user": "What is the meaning of life?"} - assert responses[0] == "The meaning of life is 42." - assert prompts[1] == {"user": "That's ridiculous."} - assert responses[1] == "I agree." - - # Test a missing user message - with pytest.raises( - ValueError, match="Missing a user message before assistant message" - ): - for _ in ds._get_turns( - [dialogue[0]["dialogue"][0]] + dialogue[0]["dialogue"][2:] - ): - pass - - # Test a missing user message and no system message - with pytest.raises( - ValueError, match="Missing a user message before assistant message" - ): - for _ in ds._get_turns(dialogue[0]["dialogue"][2:]): - pass - - # Test repeated messages - with pytest.raises(ValueError, match="Duplicate"): - for _ in ds._get_turns( - dialogue[0]["dialogue"][:2] + dialogue[0]["dialogue"][3:] - ): - pass - with pytest.raises(ValueError, match="Duplicate"): - for _ in ds._get_turns( - [dialogue[0]["dialogue"][0]] + [dialogue[0]["dialogue"][0]] - ): - pass - - # Test incomplete turn - with pytest.raises(ValueError, match="Incomplete turn in dialogue"): - for _ in ds._get_turns(dialogue[0]["dialogue"][:2]): - pass - - @mock.patch("torchtune.datasets._chat.load_dataset") - def test_get_item(self, mock_load_dataset, template, dialogue): + def test_get_item(self, mock_load_dataset, chat_format, dialogue): mock_load_dataset.return_value = dialogue expected_tokenized_prompts = [ [ @@ -173,12 +126,11 @@ def test_get_item(self, mock_load_dataset, template, dialogue): + [CROSS_ENTROPY_IGNORE_IDX] * prompt_lengths[1] + [1, 6, -1] ] - ds = ChatDataset( tokenizer=DummyTokenizer(), source="iam/agoofy/goober", - convert_to_dialogue=lambda x: x["dialogue"], - template=template, + convert_to_messages=lambda x, y: x["dialogue"], + chat_format=chat_format, max_seq_len=100, train_on_input=False, ) diff --git a/tests/torchtune/datasets/test_slimorca_dataset.py b/tests/torchtune/datasets/test_slimorca_dataset.py index b40df911bd..725b60d49d 100644 --- a/tests/torchtune/datasets/test_slimorca_dataset.py +++ b/tests/torchtune/datasets/test_slimorca_dataset.py @@ -8,13 +8,10 @@ import pytest from tests.test_utils import get_assets_path -from torchtune.data import Llama2ChatTemplate from torchtune.datasets import slimorca_dataset from torchtune.modules.tokenizer import Tokenizer -LLAMA_TEMPLATE = Llama2ChatTemplate() - class TestSlimOrcaDataset: @pytest.fixture @@ -51,7 +48,11 @@ def test_dataset_get_item(self, load_dataset, tokenizer, max_seq_len): ] } ] - ds = slimorca_dataset(tokenizer=tokenizer, max_seq_len=max_seq_len) + ds = slimorca_dataset( + tokenizer=tokenizer, + max_seq_len=max_seq_len, + train_on_input=(max_seq_len == 128), + ) input, label = ds[0] assert len(input) <= max_seq_len assert len(label) <= max_seq_len diff --git a/tests/torchtune/modules/test_tokenizer.py b/tests/torchtune/modules/test_tokenizer.py index 9d940925ea..5ac4255e01 100644 --- a/tests/torchtune/modules/test_tokenizer.py +++ b/tests/torchtune/modules/test_tokenizer.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +from torchtune.data._types import Message from torchtune.modules.tokenizer import Tokenizer ASSETS = Path(__file__).parent.parent.parent / "assets" @@ -59,3 +60,242 @@ def test_token_ids(self, tokenizer): def test_tokenizer_vocab_size(self, tokenizer): assert tokenizer.vocab_size == 2000 + + def test_encode_without_leading_whitespace(self, tokenizer): + s1 = "Hello" + s2 = "I'm an outgoing and friendly person." + s1_tokens = tokenizer.encode(s1, add_bos=False, add_eos=False) + s2_tokens = tokenizer.encode(s2, add_bos=False, add_eos=False) + # Set prefix="pre" since "\n" is not in the test tokenizer's vocab + s2_tokens_no_whitespace = tokenizer.encode( + s2, add_bos=False, add_eos=False, trim_leading_whitespace=True, prefix="pre" + ) + s1s2_tokens = tokenizer.encode(s1 + s2, add_bos=False, add_eos=False) + assert (s1_tokens + s2_tokens) != s1s2_tokens + assert (s1_tokens + s2_tokens_no_whitespace) == s1s2_tokens + + def test_tokenize_messages(self, tokenizer): + messages = [ + Message( + role="user", + content="Below is an instruction that describes a task. Write a response " + "that appropriately completes the request.\n\n### Instruction:\nGenerate " + "a realistic dating profile bio.\n\n### Response:\n", + masked=True, + ), + Message( + role="assistant", + content="I'm an outgoing and friendly person who loves spending time with " + "friends and family. I'm also a big-time foodie and love trying out new " + "restaurants and different cuisines. I'm a big fan of the arts and enjoy " + "going to museums and galleries. I'm looking for someone who shares my " + "interest in exploring new places, as well as someone who appreciates a " + "good conversation over coffee.", + ), + ] + tokens, mask = tokenizer.tokenize_messages(messages) + expected_tokens = [ + 1, + 323, + 418, + 202, + 31, + 128, + 15, + 120, + 47, + 88, + 584, + 23, + 1665, + 182, + 9, + 434, + 295, + 85, + 4, + 780, + 47, + 636, + 9, + 1094, + 213, + 23, + 9, + 69, + 69, + 164, + 1153, + 299, + 35, + 961, + 132, + 237, + 7, + 5, + 761, + 4, + 12, + 0, + 313, + 120, + 47, + 88, + 584, + 166, + 493, + 171, + 54, + 299, + 9, + 906, + 244, + 19, + 186, + 767, + 303, + 671, + 92, + 209, + 24, + 190, + 52, + 38, + 4, + 12, + 0, + 1243, + 7, + 69, + 135, + 213, + 166, + 6, + 21, + 45, + 128, + 71, + 58, + 38, + 14, + 10, + 652, + 35, + 462, + 101, + 1306, + 7, + 341, + 171, + 20, + 14, + 127, + 26, + 652, + 7, + 10, + 1268, + 4, + 6, + 21, + 45, + 591, + 9, + 566, + 22, + 994, + 913, + 38, + 20, + 52, + 24, + 10, + 1306, + 734, + 14, + 71, + 365, + 1382, + 7, + 10, + 801, + 105, + 88, + 244, + 985, + 7, + 4, + 6, + 21, + 45, + 9, + 566, + 126, + 180, + 11, + 5, + 1137, + 7, + 10, + 1089, + 151, + 8, + 1156, + 213, + 342, + 7, + 10, + 384, + 104, + 54, + 470, + 4, + 6, + 21, + 45, + 287, + 14, + 33, + 125, + 135, + 24, + 101, + 512, + 66, + 7, + 28, + 822, + 15, + 542, + 69, + 59, + 110, + 14, + 365, + 229, + 7, + 3, + 36, + 267, + 36, + 125, + 135, + 24, + 101, + 1503, + 182, + 9, + 222, + 1661, + 191, + 332, + 92, + 92, + 24, + 24, + 4, + 2, + ] + expected_mask = [True] * 75 + [False] * 125 + assert expected_tokens == tokens + assert expected_mask == mask diff --git a/torchtune/config/_utils.py b/torchtune/config/_utils.py index 9f54a54adb..16c5b30d1f 100644 --- a/torchtune/config/_utils.py +++ b/torchtune/config/_utils.py @@ -4,7 +4,6 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import re from argparse import Namespace from importlib import import_module from types import ModuleType @@ -13,7 +12,6 @@ from omegaconf import DictConfig, OmegaConf from torchtune.config._errors import InstantiationError -from torchtune.data._templates import PromptTemplate def _has_component(node: Union[Dict[str, Any], DictConfig]) -> bool: @@ -150,37 +148,3 @@ def _merge_yaml_and_cli_args(yaml_args: Namespace, cli_args: List[str]) -> DictC # CLI takes precedence over yaml args return OmegaConf.merge(yaml_conf, cli_conf) - - -def _get_template(template: str) -> PromptTemplate: - """ - Get the prompt template class from the template string. - - String should either be the PromptTemplate class name directly, or a raw - string with 1 or more placeholders. If none of these apply, then raise an - error. - - Args: - template (str): class name of template, or string with placeholders - - Returns: - PromptTemplate: the prompt template class or the same verified string - - Raises: - ValueError: if the template is not a PromptTemplate class or a proper - template string - """ - path = "torchtune.data." + template - try: - template_class = _get_component_from_path(path) - return template_class() - except InstantiationError: - # Verify that string can be used as a template, should have variable - # placeholders - pattern = r"\{\w+\}" - if not re.search(pattern, template): - raise ValueError( - f"Invalid template '{template}': " - + "Must be a PromptTemplate class or a string with placeholders." - ) from None - return template diff --git a/torchtune/data/__init__.py b/torchtune/data/__init__.py index d67021f879..010798e2d0 100644 --- a/torchtune/data/__init__.py +++ b/torchtune/data/__init__.py @@ -4,30 +4,32 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from torchtune.data._templates import ( +from torchtune.data._chat_formats import ( + ChatFormat, + ChatMLFormat, + Llama2ChatFormat, + MistralChatFormat, +) +from torchtune.data._instruct_templates import ( AlpacaInstructTemplate, - ChatMLTemplate, GrammarErrorCorrectionTemplate, - Llama2ChatTemplate, - MistralChatTemplate, - PromptTemplate, + InstructTemplate, SummarizeTemplate, ) -from torchtune.data._transforms import sharegpt_to_llama2_dialogue -from torchtune.data._types import Dialogue, Message -from torchtune.data._utils import tokenize_prompt_and_response, truncate +from torchtune.data._transforms import sharegpt_to_llama2_messages +from torchtune.data._types import Message +from torchtune.data._utils import truncate __all__ = [ "AlpacaInstructTemplate", + "ChatFormat", "GrammarErrorCorrectionTemplate", - "PromptTemplate", + "InstructTemplate", "SummarizeTemplate", - "Llama2ChatTemplate", - "MistralChatTemplate", - "ChatMLTemplate", - "sharegpt_to_llama2_dialogue", + "Llama2ChatFormat", + "MistralChatFormat", + "ChatMLFormat", + "sharegpt_to_llama2_messages", "truncate", - "tokenize_prompt_and_response", - "Dialogue", "Message", ] diff --git a/torchtune/data/_chat_formats.py b/torchtune/data/_chat_formats.py new file mode 100644 index 0000000000..6071827233 --- /dev/null +++ b/torchtune/data/_chat_formats.py @@ -0,0 +1,215 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from abc import ABC, abstractmethod +from typing import List + +from torchtune.data._types import Message + + +class ChatFormat(ABC): + """ + Interface for chat formats. Each chat format should include template + prompts with placeholders for the data inputs. There should be a template + for each role: system, user, assistant. + """ + + system = "" + user = "" + assistant = "" + + @classmethod + @abstractmethod + def format( + cls, + sample: List[Message], + ) -> List[Message]: + """ + Format each role's message(s) according to the chat format + + Args: + sample (List[Message]): a single conversation, structured as a list + of `Message` objects + + Returns: + The formatted list of messages + """ + pass + + +class Llama2ChatFormat(ChatFormat): + """ + Chat format that formats human and system prompts with appropriate tags + used in LLaMA2 pre-training. Taken from Meta's official LLaMA inference + repository at https://github.com/meta-llama/llama/blob/main/llama/generation.py. + + Example: + "[INST] <> + You are a helpful, respectful and honest assistant. + <> + + I am going to Paris, what should I see? [/INST] Paris, the capital of France, is known for its stunning architecture..." + """ + + B_INST, E_INST = "[INST]", "[/INST]" + B_SYS, E_SYS = "<>\n", "\n<>\n\n" + + system = f"{B_SYS}{{content}}{E_SYS}" + user = f"{B_INST} {{system_message}}{{content}} {E_INST} " + assistant = "" + + @classmethod + def format( + cls, + sample: List[Message], + ) -> List[Message]: + """ + Format user and system messages with appropriate tags. + + Args: + sample (List[Message]): a single conversation, structured as a list + of `Message` objects + + Returns: + The formatted list of messages + """ + system_message = "" + formatted_dialogue = [] + for message in sample: + content = "" + if message.role == "system": + content = cls.system.format(content=message.content) + system_message = content + # Incorporate the system message in the user message - LLaMA2 only + # looks for the <> tags and not the explicit role so this will + # be treated the same as an actual system message. We do this because + # of the nesting of the system prompt in the user message. + continue + elif message.role == "user": + content = cls.user.format( + system_message=system_message, content=message.content + ) + elif message.role == "assistant": + # No special formatting needed for assistant message + content = message.content + assert content != "" + formatted_dialogue.append( + Message(role=message.role, content=content, masked=message.masked), + ) + return formatted_dialogue + + +class MistralChatFormat(ChatFormat): + """ + Formats according to Mistral's instruct model: + https://docs.mistral.ai/models/ + + It is identical to `Llama2ChatFormat`, except it does not support system + prompts. + + Example: + "[INST] I am going to Paris, what should I see? [/INST] Paris, the capital + of France, is known for its stunning architecture..." + + """ + + B_INST, E_INST = "[INST]", "[/INST]" + system = None + user = f"{B_INST} {{content}} {E_INST} " + assistant = "" + + @classmethod + def format( + cls, + sample: List[Message], + ) -> List[Message]: + """ + Format user and system messages with appropriate tags. + + Args: + sample (List[Message]): a single conversation, structured as a list + of `Message` objects + + Returns: + The formatted list of messages + + Raises: + ValueError: If system prompts are provided + """ + formatted_dialogue = [] + for message in sample: + content = "" + if message.role == "system": + raise ValueError( + "System prompts are not supported in MistralChatFormat" + ) + elif message.role == "user": + content = cls.user.format( + content=message.content, + ) + elif message.role == "assistant": + # No special formatting needed for assistant message + content = message.content + assert content != "" + formatted_dialogue.append( + Message(role=message.role, content=content, masked=message.masked), + ) + return formatted_dialogue + + +class ChatMLFormat(ChatFormat): + """ + OpenAI's Chat Markup Language used by their chat models: + https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/ai-services/openai/includes/chat-markup-language.md + It is the default chat format used by HuggingFace models. + + Example: + <|im_start|>system + Provide some context and/or instructions to the model.<|im_end|> + <|im_start|>user + The user’s message goes here<|im_end|> + <|im_start|>assistant + The assistant’s response goes here<|im_end|> + """ + + IM_START, IM_END = "<|im_start|>", "<|im_end|>" + system = f"{IM_START}system\n{{content}}{IM_END}\n" + user = f"{IM_START}user\n{{content}}{IM_END}\n" + assistant = f"{IM_START}assistant\n{{content}}{IM_END}" + + @classmethod + def format( + cls, + sample: List[Message], + ) -> List[Message]: + """ + Format user and system messages with appropriate tags. + + Args: + sample (List[Message]): a single conversation, structured as a list + of `Message` objects + + Returns: + The formatted list of messages + """ + formatted_dialogue = [] + for message in sample: + content = "" + if message.role == "system": + content = cls.system.format(content=message.content) + elif message.role == "user": + content = cls.user.format( + content=message.content, + ) + elif message.role == "assistant": + content = cls.assistant.format( + content=message.content, + ) + assert content != "" + formatted_dialogue.append( + Message(role=message.role, content=content, masked=message.masked), + ) + return formatted_dialogue diff --git a/torchtune/data/_instruct_templates.py b/torchtune/data/_instruct_templates.py new file mode 100644 index 0000000000..fe26d0c9a6 --- /dev/null +++ b/torchtune/data/_instruct_templates.py @@ -0,0 +1,147 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from abc import ABC, abstractmethod +from typing import Any, Dict, Mapping, Optional + + +class InstructTemplate(ABC): + """ + Interface for instruction templates. Each template should include the template + prompt with placeholders for the data inputs. + """ + + template = "" + + @classmethod + @abstractmethod + def format( + cls, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None + ) -> str: + """ + Format the prompt template with the given arguments. + + Args: + sample (Mapping[str, Any]): a single data sample with various fields + column_map (Optional[Dict[str, str]]): a mapping from the expected + placeholder names in the template to the column names in the sample. + If None, assume these are identical. Note: if the sample output is not named + as "output" in the dataset, you always need to map it to "output" in column_map. + + Returns: + The formatted prompt + """ + pass + + +class AlpacaInstructTemplate(InstructTemplate): + """ + Prompt template for the Alpaca dataset. Template prompt changes slightly depending + on if there's an instruction + input or just an instruction. + """ + + template = { + "prompt_input": ( + "Below is an instruction that describes a task, paired with an input that provides further context. " + "Write a response that appropriately completes the request.\n\n" + "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n" + ), + "prompt_no_input": ( + "Below is an instruction that describes a task. " + "Write a response that appropriately completes the request.\n\n" + "### Instruction:\n{instruction}\n\n### Response:\n" + ), + } + + @classmethod + def format( + cls, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None + ) -> str: + """ + Generate prompt from instruction and input. + + Args: + sample (Mapping[str, Any]): a single data sample with instruction + column_map (Optional[Dict[str, str]]): a mapping from the expected + placeholder names in the template to the column names in the sample. + If None, assume these are identical. + + Returns: + The formatted prompt + """ + column_map = column_map or {} + key_input = column_map.get("input", "input") + key_instruction = column_map.get("instruction", "instruction") + + if key_input in sample and sample[key_input]: + prompt = cls.template["prompt_input"].format( + instruction=sample[key_instruction], input=sample[key_input] + ) + else: + prompt = cls.template["prompt_no_input"].format( + instruction=sample[key_instruction] + ) + return prompt + + +class GrammarErrorCorrectionTemplate(InstructTemplate): + """ + Prompt template for the Grammar dataset. + """ + + template = "Correct this to standard English: {sentence}\n---\nCorrected: " + + @classmethod + def format( + cls, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None + ) -> str: + """ + Generate prompt from sentence. + + Args: + sample (Mapping[str, Any]): a single data sample with sentence + column_map (Optional[Dict[str, str]]): a mapping from the expected + placeholder names in the template to the column names in the sample. + If None, assume these are identical. + + Returns: + The formatted prompt + """ + column_map = column_map or {} + key_sentence = column_map.get("sentence", "sentence") + + prompt = cls.template.format(sentence=sample[key_sentence]) + return prompt + + +class SummarizeTemplate(InstructTemplate): + """ + Prompt template to format datasets for summarization tasks. + """ + + template = "Summarize this dialogue:\n{dialogue}\n---\nSummary:\n" + + @classmethod + def format( + cls, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None + ) -> str: + """ + Generate prompt from dialogue. + + Args: + sample (Mapping[str, Any]): a single data sample with dialog + column_map (Optional[Dict[str, str]]): a mapping from the expected + placeholder names in the template to the column names in the sample. + If None, assume these are identical. + + Returns: + The formatted prompt + """ + column_map = column_map or {} + key_dialogue = column_map.get("dialogue", "dialogue") + + prompt = cls.template.format(dialogue=sample[key_dialogue]) + return prompt diff --git a/torchtune/data/_templates.py b/torchtune/data/_templates.py deleted file mode 100644 index 8c99f9e75c..0000000000 --- a/torchtune/data/_templates.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -from abc import ABC, abstractmethod -from typing import Any, Dict, Mapping, Optional - - -class PromptTemplate(ABC): - """ - Interface for prompt templates. Each template should include the template - prompt with placeholders for the data inputs. - """ - - template = "" - - @abstractmethod - def format( - self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None - ) -> str: - """ - Format the prompt template with the given arguments. - - Args: - sample (Mapping[str, Any]): a single data sample with various fields - column_map (Optional[Dict[str, str]]): a mapping from the expected - placeholder names in the template to the column names in the sample. - If None, assume these are identical. Note: if the sample output is not named - as "output" in the dataset, you always need to map it to "output" in column_map. - - Returns: - The formatted prompt - """ - pass - - -class AlpacaInstructTemplate(PromptTemplate): - """ - Prompt template for the Alpaca dataset. Template prompt changes slightly depending - on if there's an instruction + input or just an instruction. - """ - - template = { - "prompt_input": ( - "Below is an instruction that describes a task, paired with an input that provides further context. " - "Write a response that appropriately completes the request.\n\n" - "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n" - ), - "prompt_no_input": ( - "Below is an instruction that describes a task. " - "Write a response that appropriately completes the request.\n\n" - "### Instruction:\n{instruction}\n\n### Response:\n" - ), - } - - def format( - self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None - ) -> str: - """ - Generate prompt from instruction and input. - - Args: - sample (Mapping[str, Any]): a single data sample with instruction - column_map (Optional[Dict[str, str]]): a mapping from the expected - placeholder names in the template to the column names in the sample. - If None, assume these are identical. - - Returns: - The formatted prompt - """ - column_map = column_map or {} - key_input = column_map.get("input", "input") - key_instruction = column_map.get("instruction", "instruction") - - if key_input in sample and sample[key_input]: - prompt = self.template["prompt_input"].format( - instruction=sample[key_instruction], input=sample[key_input] - ) - else: - prompt = self.template["prompt_no_input"].format( - instruction=sample[key_instruction] - ) - return prompt - - -class GrammarErrorCorrectionTemplate(PromptTemplate): - """ - Prompt template for the Grammar dataset. - """ - - template = "Correct this to standard English: {sentence}\n---\nCorrected: " - - def format( - self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None - ) -> str: - """ - Generate prompt from sentence. - - Args: - sample (Mapping[str, Any]): a single data sample with sentence - column_map (Optional[Dict[str, str]]): a mapping from the expected - placeholder names in the template to the column names in the sample. - If None, assume these are identical. - - Returns: - The formatted prompt - """ - column_map = column_map or {} - key_sentence = column_map.get("sentence", "sentence") - - prompt = self.template.format(sentence=sample[key_sentence]) - return prompt - - -class SummarizeTemplate(PromptTemplate): - """ - Prompt template to format datasets for summarization tasks. - """ - - template = "Summarize this dialogue:\n{dialogue}\n---\nSummary:\n" - - def format( - self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None - ) -> str: - """ - Generate prompt from dialogue. - - Args: - sample (Mapping[str, Any]): a single data sample with dialog - column_map (Optional[Dict[str, str]]): a mapping from the expected - placeholder names in the template to the column names in the sample. - If None, assume these are identical. - - Returns: - The formatted prompt - """ - column_map = column_map or {} - key_dialogue = column_map.get("dialogue", "dialogue") - - prompt = self.template.format(dialogue=sample[key_dialogue]) - return prompt - - -class Llama2ChatTemplate(PromptTemplate): - """ - Prompt template that formats human and system prompts with appropriate tags - used in LLaMA2 pre-training. Taken from Meta's official LLaMA inference - repository at https://github.com/meta-llama/llama/blob/main/llama/generation.py. - The response is tokenized outside of this template. - - Example: - "[INST] <> - You are a helpful, respectful and honest assistant. - <> - - I am going to Paris, what should I see? [/INST] Paris, the capital of France, is known for its stunning architecture..." - """ - - B_INST, E_INST = "[INST]", "[/INST]" - B_SYS, E_SYS = "<>\n", "\n<>\n\n" - template = { - "system": f"{B_INST} {B_SYS}{{system}}{E_SYS}{{user}} {E_INST} ", - "no_system": f"{B_INST} {{user}} {E_INST} ", - } - - def format( - self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None - ) -> str: - """ - Generate prompt from a user message and optional system prompt. - - Args: - sample (Mapping[str, Any]): a single data sample, expects role keys "system" (optional) - and "user" in the sample. - column_map (Optional[Dict[str, str]]): a mapping from the expected - role names in the template to the actual role names in the sample. - If None, assume these are "system" and "user". - - Returns: - The formatted prompt - """ - column_map = column_map or {} - key_system = column_map.get("system", "system") - key_user = column_map.get("user", "user") - - if key_system in sample: - return self.template["system"].format( - system=sample[key_system], user=sample[key_user] - ) - else: - return self.template["no_system"].format(user=sample[key_user]) - - -class MistralChatTemplate(PromptTemplate): - """ - Prompt template that formats according to Mistral's instruct model: - https://docs.mistral.ai/models/ - - It is identical to `Llama2ChatTemplate`, except it does not support system - prompts. - - Example: - "[INST] I am going to Paris, what should I see? [/INST] Paris, the capital - of France, is known for its stunning architecture..." - """ - - B_INST, E_INST = "[INST]", "[/INST]" - template = f"{B_INST} {{user}} {E_INST} " - - def format( - self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None - ) -> str: - """ - Generate prompt from a user message - - Args: - sample (Mapping[str, Any]): a single data sample, expects only "user" in the sample. - column_map (Optional[Dict[str, str]]): a mapping from the expected - role names in the template to the actual role names in the sample. - If None, assume these are "user". - - Returns: - The formatted prompt - - Raises: - ValueError: if the sample contains a "system" key - """ - if "system" in sample: - raise ValueError("System prompts are not supported in MistralChatTemplate") - - column_map = column_map or {} - key_user = column_map.get("user", "user") - - return self.template.format(user=sample[key_user]) - - -class ChatMLTemplate(PromptTemplate): - """ - OpenAI's Chat Markup Language used by their chat models: - https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/ai-services/openai/includes/chat-markup-language.md - It is the default template used by Hugging Face models. - - Example: - <|im_start|>system - Provide some context and/or instructions to the model.<|im_end|> - <|im_start|>user - The user’s message goes here<|im_end|> - <|im_start|>assistant - The assistant’s response goes here<|im_end|> - """ - - IM_START, IM_END = "<|im_start|>", "<|im_end|>" - template = { - "system": f"{IM_START}system\n{{system}}{IM_END}\n{IM_START}user\n{{user}}{IM_END}\n{IM_START}assistant\n", - "no_system": f"{IM_START}user\n{{user}}{IM_END}\n{IM_START}assistant\n", - } - - def format( - self, sample: Mapping[str, Any], column_map: Optional[Dict[str, str]] = None - ) -> str: - """ - Generate prompt from a user message and optional system prompt. - - Args: - sample (Mapping[str, Any]): a single data sample, expects role keys "system" (optional) - and "user" in the sample. - column_map (Optional[Dict[str, str]]): a mapping from the expected - role names in the template to the actual role names in the sample. - If None, assume these are "system" and "user". - - Returns: - The formatted prompt - """ - column_map = column_map or {} - key_system = column_map.get("system", "system") - key_user = column_map.get("user", "user") - - if key_system in sample: - return self.template["system"].format( - system=sample[key_system], user=sample[key_user] - ) - else: - return self.template["no_system"].format(user=sample[key_user]) diff --git a/torchtune/data/_transforms.py b/torchtune/data/_transforms.py index 155b123caf..95c6688ee2 100644 --- a/torchtune/data/_transforms.py +++ b/torchtune/data/_transforms.py @@ -4,12 +4,14 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from typing import Any, Mapping +from typing import Any, List, Mapping -from torchtune.data._types import Dialogue, Message +from torchtune.data._types import Message -def sharegpt_to_llama2_dialogue(sample: Mapping[str, Any]) -> Dialogue: +def sharegpt_to_llama2_messages( + sample: Mapping[str, Any], train_on_input: bool = False +) -> List[Message]: """ Convert a chat sample adhering to the ShareGPT format to the LLaMA2 format. @@ -36,18 +38,17 @@ def sharegpt_to_llama2_dialogue(sample: Mapping[str, Any]) -> Dialogue: Args: sample (Mapping[str, Any]): a single data sample with "conversations" field pointing to a list of dict messages. - + train_on_input (bool): whether the prompt should remain unmasked. Default: False Returns: - Dialogue: a list of messages with "role" and "content" fields. See `torchtune.datasets._types.Message` + List[Message]: a list of messages with "role" and "content" fields. See `torchtune.datasets._types.Message` and `torchtune.datasets._types.Dialogue` for more details. """ role_map = {"system": "system", "human": "user", "gpt": "assistant"} conversations = sample["conversations"] - - dialogue = [] + messages = [] for message in conversations: role = role_map[message["from"]] content = message["value"] - dialogue.append(Message(role=role, content=content)) - - return dialogue + masked = (role != "assistant") and (not train_on_input) + messages.append(Message(role=role, content=content, masked=masked)) + return messages diff --git a/torchtune/data/_types.py b/torchtune/data/_types.py index ec30de7cfc..4aba199e3c 100644 --- a/torchtune/data/_types.py +++ b/torchtune/data/_types.py @@ -4,14 +4,14 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from typing import List, Literal, TypedDict +from dataclasses import dataclass +from typing import Literal Role = Literal["system", "user", "assistant"] -class Message(TypedDict): +@dataclass +class Message: role: Role content: str - - -Dialogue = List[Message] + masked: bool = False diff --git a/torchtune/data/_utils.py b/torchtune/data/_utils.py index 207369ed80..04318d9652 100644 --- a/torchtune/data/_utils.py +++ b/torchtune/data/_utils.py @@ -4,70 +4,15 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import copy -from typing import List, Tuple - -from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX -from torchtune.modules import Tokenizer - - -def tokenize_prompt_and_response( - tokenizer: Tokenizer, - prompt: str, - response: str, - train_on_input: bool = False, -) -> Tuple[List[int], List[int]]: - """ - Tokenize a prompt and response pair. - - Args: - tokenizer (Tokenizer): The tokenizer to use. - prompt (str): The prompt to tokenize. - response (str): The response to tokenize. - train_on_input (bool): Whether to train on prompt or mask it out. Default - is False. - - Returns: - The tokenized prompt and response. - """ - prompt_with_response = prompt + response - encoded_prompt = tokenizer.encode(text=prompt, add_bos=True, add_eos=False) - encoded_prompt_with_response = tokenizer.encode( - text=prompt_with_response, add_bos=True, add_eos=True - ) - labels = copy.deepcopy(encoded_prompt_with_response) - - if not train_on_input: - labels[: len(encoded_prompt)] = [CROSS_ENTROPY_IGNORE_IDX] * len(encoded_prompt) - - assert len(encoded_prompt_with_response) == len(labels) - - return encoded_prompt_with_response, labels +from typing import Any, List def truncate( - tokenizer: Tokenizer, - prompt_tokens: List[int], - label_tokens: List[int], + tokens: List[Any], max_seq_len: int, -) -> Tuple[List[int], List[int]]: - """ - Truncate a prompt and label pair if longer than max sequence length. - - Args: - tokenizer (Tokenizer): The tokenizer to use. - prompt_tokens (List[int]): The prompt + response tokens. - label_tokens (List[int]): The label tokens. - max_seq_len (int): The maximum sequence length. - - Returns: - The truncated prompt and label. - """ - prompt_tokens_truncated = prompt_tokens[:max_seq_len] - label_tokens_truncated = label_tokens[:max_seq_len] - if prompt_tokens_truncated[-1] != tokenizer.eos_id: - prompt_tokens_truncated[-1] = tokenizer.eos_id - if label_tokens_truncated[-1] != tokenizer.eos_id: - label_tokens_truncated[-1] = tokenizer.eos_id - - return prompt_tokens_truncated, label_tokens_truncated + eos_id: Any, +) -> List[Any]: + tokens_truncated = tokens[:max_seq_len] + if tokens_truncated[-1] != eos_id: + tokens_truncated[-1] = eos_id + return tokens_truncated diff --git a/torchtune/datasets/_alpaca.py b/torchtune/datasets/_alpaca.py index 399fbf83aa..29e8ef9b93 100644 --- a/torchtune/datasets/_alpaca.py +++ b/torchtune/datasets/_alpaca.py @@ -59,7 +59,7 @@ def alpaca_dataset( return InstructDataset( tokenizer=tokenizer, source="yahma/alpaca-cleaned" if use_clean else "tatsu-lab/alpaca", - template=AlpacaInstructTemplate(), + template=AlpacaInstructTemplate, train_on_input=train_on_input, max_seq_len=max_seq_len, split="train", diff --git a/torchtune/datasets/_chat.py b/torchtune/datasets/_chat.py index 19c6b50557..2858b45eb4 100644 --- a/torchtune/datasets/_chat.py +++ b/torchtune/datasets/_chat.py @@ -4,19 +4,14 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from typing import Any, Callable, Dict, Generator, List, Mapping, Tuple +from typing import Any, Callable, Dict, List, Mapping, Tuple + +import numpy as np from datasets import load_dataset from torch.utils.data import Dataset -from torchtune.config._utils import _get_template - -from torchtune.data import ( - Dialogue, - PromptTemplate, - sharegpt_to_llama2_dialogue, - tokenize_prompt_and_response, - truncate, -) +from torchtune.data import ChatFormat, Message, sharegpt_to_llama2_messages +from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX from torchtune.modules import Tokenizer @@ -27,10 +22,10 @@ class ChatDataset(Dataset): The general flow from loading a sample to tokenized prompt is: load sample -> apply transform -> foreach turn{format into template -> tokenize} - If the column/key names differ from the expected names in the `PromptTemplate`, + If the column/key names differ from the expected names in the `ChatFormat`, then the `column_map` argument can be used to provide this mapping. - Use `convert_to_dialogue` to prepare your dataset into the llama conversation format + Use `convert_to_messages` to prepare your dataset into the llama conversation format and roles: [ { @@ -47,9 +42,9 @@ class ChatDataset(Dataset): tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. source (str): path string of dataset, anything supported by Hugging Face's `load_dataset` (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) - convert_to_dialogue (Callable[[Mapping[str, Any]], Dialogue]): function that keys into the desired field in the sample + convert_to_messages (Callable[[Mapping[str, Any]], List[Message]]): function that keys into the desired field in the sample and converts to a list of `Messages` that follows the llama format with the expected keys - template (PromptTemplate): template used to format the prompt. If the placeholder variable + chat_format (ChatFormat): template used to format the chat. If the placeholder variable names in the template do not match the column/key names in the dataset, use `column_map` to map them. max_seq_len (int): Maximum number of tokens in the returned input and label token id lists. train_on_input (bool): Whether the model is trained on the prompt or not. Default is False. @@ -60,16 +55,16 @@ def __init__( self, tokenizer: Tokenizer, source: str, - convert_to_dialogue: Callable[[Mapping[str, Any]], Dialogue], - template: PromptTemplate, + convert_to_messages: Callable[[Mapping[str, Any]], List[Message]], + chat_format: ChatFormat, max_seq_len: int, train_on_input: bool = False, **load_dataset_kwargs: Dict[str, Any], ) -> None: self._tokenizer = tokenizer self._data = load_dataset(source, **load_dataset_kwargs) - self._convert_to_dialogue = convert_to_dialogue - self.template = template + self._convert_to_messages = convert_to_messages + self.chat_format = chat_format self.max_seq_len = max_seq_len self.train_on_input = train_on_input @@ -81,65 +76,23 @@ def __getitem__(self, index: int) -> Tuple[List[int], List[int]]: return self._prepare_sample(sample) def _prepare_sample(self, sample: Mapping[str, Any]) -> Tuple[List[int], List[int]]: - dialogue = self._convert_to_dialogue(sample) - - prompt_tokens = [] - label_tokens = [] - for prompt, label in self._get_turns(dialogue): - formatted_prompt = self.template.format(prompt) - encoded_prompt_with_response, labels = tokenize_prompt_and_response( - tokenizer=self._tokenizer, - prompt=formatted_prompt, - response=label, - train_on_input=self.train_on_input, - ) - prompt_tokens.extend(encoded_prompt_with_response) - label_tokens.extend(labels) - - if len(prompt_tokens) >= self.max_seq_len: - break - - prompt_tokens, label_tokens = truncate( - self._tokenizer, prompt_tokens, label_tokens, self.max_seq_len + messages = self._convert_to_messages(sample, self.train_on_input) + messages = self.chat_format.format(messages) + tokens, mask = self._tokenizer.tokenize_messages( + messages, max_seq_len=self.max_seq_len ) + # Wherever mask == True, set to CROSS_ENTROPY_IGNORE_IDX. Otherwise keep as tokens + labels = list(np.where(mask, CROSS_ENTROPY_IGNORE_IDX, tokens)) + assert len(tokens) == len(labels) - assert len(prompt_tokens) == len(label_tokens) - - return prompt_tokens, label_tokens - - def _get_turns( - self, dialogue: Dialogue - ) -> Generator[Tuple[Dict[str, str], str], None, None]: - prompt_messages = {} - for message in dialogue: - # If we are at the assistant message, we are at the end of a turn, yield. - if message["role"] == "assistant": - if "user" not in prompt_messages: - raise ValueError( - f"Missing a user message before assistant message: {message['content']}" - ) - yield prompt_messages, message["content"] - prompt_messages = {} - # Otherwise, continue to add to the turn's messages - else: - if message["role"] in prompt_messages: - raise ValueError( - f"Duplicate {message['role']} message in dialogue: {message['content']}" - ) - prompt_messages[message["role"]] = message["content"] - - # If we never yielded, then the last turn was incomplete - if prompt_messages: - raise ValueError( - f"Incomplete turn in dialogue, current turn: {prompt_messages}" - ) + return tokens, labels def chat_dataset( tokenizer: Tokenizer, source: str, conversation_format: str, - template: str, + chat_format: ChatFormat, max_seq_len: int, train_on_input: bool = False, **load_dataset_kwargs: Dict[str, Any], @@ -155,7 +108,7 @@ def chat_dataset( (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) conversation_format (str): string specifying expected format of conversations in the dataset for automatic conversion to the llama format. Supported formats are: "sharegpt" - template (str): class name of template used to format the prompt. + chat_format (ChatFormat): Template class used to format the chat. max_seq_len (int): Maximum number of tokens in the returned input and label token id lists. train_on_input (bool): Whether the model is trained on the prompt or not. Default is False. **load_dataset_kwargs (Dict[str, Any]): additional keyword arguments to pass to `load_dataset`. @@ -167,15 +120,15 @@ def chat_dataset( ValueError: if the conversation format is not supported """ if conversation_format == "sharegpt": - convert_to_dialogue = sharegpt_to_llama2_dialogue + convert_to_messages = sharegpt_to_llama2_messages else: raise ValueError(f"Unsupported conversation format: {conversation_format}") return ChatDataset( tokenizer=tokenizer, source=source, - convert_to_dialogue=convert_to_dialogue, - template=_get_template(template), + convert_to_messages=convert_to_messages, + chat_format=chat_format, max_seq_len=max_seq_len, train_on_input=train_on_input, **load_dataset_kwargs, diff --git a/torchtune/datasets/_grammar.py b/torchtune/datasets/_grammar.py index ffef93424a..e8f3d053b7 100644 --- a/torchtune/datasets/_grammar.py +++ b/torchtune/datasets/_grammar.py @@ -48,7 +48,7 @@ def grammar_dataset( return InstructDataset( tokenizer=tokenizer, source="liweili/c4_200m", - template=GrammarErrorCorrectionTemplate(), + template=GrammarErrorCorrectionTemplate, column_map={"sentence": "input"}, train_on_input=train_on_input, split="train", diff --git a/torchtune/datasets/_instruct.py b/torchtune/datasets/_instruct.py index e0c14e88b6..598d69d865 100644 --- a/torchtune/datasets/_instruct.py +++ b/torchtune/datasets/_instruct.py @@ -6,12 +6,13 @@ from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple +import numpy as np from datasets import load_dataset from torch.utils.data import Dataset -from torchtune.config._utils import _get_template +from torchtune.data import InstructTemplate, Message -from torchtune.data import PromptTemplate, tokenize_prompt_and_response, truncate +from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX from torchtune.modules import Tokenizer @@ -23,7 +24,7 @@ class InstructDataset(Dataset): The general flow from loading a sample to tokenized prompt is: load sample -> apply transform -> format into template -> tokenize - If the column/key names differ from the expected names in the `PromptTemplate`, + If the column/key names differ from the expected names in the `InstructTemplate`, then the `column_map` argument can be used to provide this mapping. Masking of the prompt during training is controlled by the `train_on_input` flag, which is @@ -36,7 +37,7 @@ class InstructDataset(Dataset): tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. source (str): path string of dataset, anything supported by Hugging Face's `load_dataset` (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) - template (PromptTemplate): template used to format the prompt. If the placeholder variable + template (InstructTemplate): template used to format the prompt. If the placeholder variable names in the template do not match the column/key names in the dataset, use `column_map` to map them. transform (Optional[Callable]): transform to apply to the sample before formatting to the template. Default is None. @@ -53,7 +54,7 @@ def __init__( self, tokenizer: Tokenizer, source: str, - template: PromptTemplate, + template: InstructTemplate, transform: Optional[Callable] = None, column_map: Optional[Dict[str, str]] = None, train_on_input: bool = False, @@ -84,28 +85,26 @@ def _prepare_sample(self, sample: Mapping[str, Any]) -> Tuple[List[int], List[in if self._column_map and "output" in self._column_map else "output" ) + messages = [ + Message(role="user", content=prompt, masked=(not self.train_on_input)), + Message(role="assistant", content=transformed_sample[key_output]), + ] - prompt_tokens, label_tokens = tokenize_prompt_and_response( - tokenizer=self._tokenizer, - prompt=prompt, - response=transformed_sample[key_output], - train_on_input=self.train_on_input, + tokens, mask = self._tokenizer.tokenize_messages( + messages, max_seq_len=self.max_seq_len ) - if self.max_seq_len is not None: - prompt_tokens, label_tokens = truncate( - self._tokenizer, prompt_tokens, label_tokens, self.max_seq_len - ) + # Wherever mask == True, set to CROSS_ENTROPY_IGNORE_IDX. Otherwise keep as tokens + labels = list(np.where(mask, CROSS_ENTROPY_IGNORE_IDX, tokens)) + assert len(tokens) == len(labels) - assert len(prompt_tokens) == len(label_tokens) - - return prompt_tokens, label_tokens + return tokens, labels def instruct_dataset( tokenizer: Tokenizer, source: str, - template: str, + template: InstructTemplate, column_map: Optional[Dict[str, str]] = None, train_on_input: bool = False, max_seq_len: Optional[int] = None, @@ -120,7 +119,7 @@ def instruct_dataset( tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. source (str): path string of dataset, anything supported by Hugging Face's `load_dataset` (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) - template (str): class name of template used to format the prompt. If the placeholder variable + template (InstructTemplate): class used to format the prompt. If the placeholder variable names in the template do not match the column/key names in the dataset, use `column_map` to map them. column_map (Optional[Dict[str, str]]): a mapping from the expected placeholder names in the template to the column/key names in the sample. If None, assume these are identical. @@ -136,7 +135,7 @@ def instruct_dataset( return InstructDataset( tokenizer=tokenizer, source=source, - template=_get_template(template), + template=template, column_map=column_map, train_on_input=train_on_input, max_seq_len=max_seq_len, diff --git a/torchtune/datasets/_samsum.py b/torchtune/datasets/_samsum.py index 5f35431d28..5f61eccc65 100644 --- a/torchtune/datasets/_samsum.py +++ b/torchtune/datasets/_samsum.py @@ -48,7 +48,7 @@ def samsum_dataset( return InstructDataset( tokenizer=tokenizer, source="samsum", - template=SummarizeTemplate(), + template=SummarizeTemplate, column_map={"output": "summary"}, train_on_input=train_on_input, split="train", diff --git a/torchtune/datasets/_slimorca.py b/torchtune/datasets/_slimorca.py index 7084579183..58228dc33c 100644 --- a/torchtune/datasets/_slimorca.py +++ b/torchtune/datasets/_slimorca.py @@ -4,7 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from torchtune.data import Llama2ChatTemplate, sharegpt_to_llama2_dialogue +from torchtune.data import Llama2ChatFormat, sharegpt_to_llama2_messages from torchtune.datasets._chat import ChatDataset @@ -61,8 +61,8 @@ def slimorca_dataset( return ChatDataset( tokenizer=tokenizer, source="Open-Orca/SlimOrca-Dedup", - convert_to_dialogue=sharegpt_to_llama2_dialogue, - template=Llama2ChatTemplate(), + convert_to_messages=sharegpt_to_llama2_messages, + chat_format=Llama2ChatFormat, max_seq_len=max_seq_len, train_on_input=train_on_input, split="train", diff --git a/torchtune/modules/tokenizer.py b/torchtune/modules/tokenizer.py index 8cc274e69c..f158e05c83 100644 --- a/torchtune/modules/tokenizer.py +++ b/torchtune/modules/tokenizer.py @@ -4,9 +4,13 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from typing import List +from typing import List, Optional, Tuple from sentencepiece import SentencePieceProcessor +from torchtune.data._types import Message +from torchtune.data._utils import truncate + +WHITESPACE_CHARS = [" ", "\n", "\t", "\r", "\v"] class Tokenizer: @@ -41,6 +45,13 @@ def __init__( self.eos_id = eos_id self.pad_id = pad_id + # This is used in tokenize_messages: if the tokenizer does not + # encode whitespace, then we can more easily split strings + # on whitespace characters and encode them separately. + self.encodes_whitespace = any( + [self.spm_model.encode(c) for c in WHITESPACE_CHARS] + ) + @classmethod def from_file(cls, path: str) -> "Tokenizer": """Initialize a `Tokenizer` instance from a SentencePiece model file. @@ -60,6 +71,8 @@ def encode( text: str, add_bos: bool = True, add_eos: bool = True, + trim_leading_whitespace: bool = False, + prefix: Optional[str] = None, ) -> List[int]: """Encode text into token IDs. @@ -67,17 +80,37 @@ def encode( text (str): The input text to be encoded, unbatched. add_bos (bool): Whether to prepend BOS to the input, defaults to True. add_eos (bool): Whether to append EOS to the input, defaults to True. - + trim_leading_whitespace (bool): Whether to trim leading whitespace from + underlying sentencepiece tokenization. Sentencepiece normally prepends + whitespace to any tokenized text, which can cause differences where + encode(s1) + encode(s2) != encode(s1 + s2) due to leading whitespace + added to s2. Default: False + prefix (Optional[str]): Optional string to encode for trimming leading + whitespaces. Used only if trim_leading_whitespace=True. Default: None Returns: List[int]: The encoded token IDs. """ - assert type(text) == str, f"Expected string but got {type(text)}" - return self.spm_model.encode( - text, - add_bos=add_bos, - add_eos=add_eos, - out_type=int, - ) + if trim_leading_whitespace: + # Can define our own custom prefix depending on vocab if needed + if not hasattr(self, "prefix"): + self.prefix = prefix or "\n" + self.encoded_prefix = self.spm_model.encode( + self.prefix, add_bos=False, add_eos=False + ) + start_idx = len(self.encoded_prefix) + int(add_bos) + return self.spm_model.encode( + self.prefix + text, + add_bos=add_bos, + add_eos=add_eos, + out_type=int, + )[start_idx:] + else: + return self.spm_model.encode( + text, + add_bos=add_bos, + add_eos=add_eos, + out_type=int, + ) def decode(self, ids: List[int]) -> str: """Decode token IDs to strings. @@ -89,3 +122,61 @@ def decode(self, ids: List[int]) -> str: str: The decoded text. """ return self.spm_model.decode(ids) + + def tokenize_messages( + self, messages: List[Message], max_seq_len: Optional[int] = None + ) -> Tuple[List[int], List[bool]]: + start_of_turn = True + end_of_turn = False + prev_ends_with_space = False + tokenized_messages = [] + mask = [] + for message in messages: + # If assistant message, this is the end of a turn + end_of_turn = message.role == "assistant" + + # Prepend BOS on start of new turns + if start_of_turn: + tokenized_messages.append(self.bos_id) + mask.append(message.masked) + + # We want to trim leading whitespace on the next message when + # (a) it is a continuation of the turn (i.e. not the first message) + # (b) the vocabulary explicitly encodes whitespace characters, and + # (c) the previous message did not end with a space + trim_leading_whitespace = ( + (not start_of_turn) + and self.encodes_whitespace + and not prev_ends_with_space + ) + + # Tokenize current message, append with masks + tokens = self.encode( + message.content.rstrip(" "), + add_bos=False, + add_eos=False, + trim_leading_whitespace=trim_leading_whitespace, + ) + prev_ends_with_space = message.content.endswith(" ") + tokenized_messages.extend(tokens) + mask.extend([message.masked] * len(tokens)) + + # If assistant message, append EOS at end + if end_of_turn: + tokenized_messages.append(self.eos_id) + mask.append(message.masked) + end_of_turn = False + start_of_turn = True + else: + start_of_turn = False + + # Break out early if we reach max_seq_len + if max_seq_len and len(tokenized_messages) >= max_seq_len: + break + + # Finally, truncate if necessary + if max_seq_len: + tokenized_messages = truncate(tokenized_messages, max_seq_len, self.eos_id) + mask = truncate(mask, max_seq_len, message.masked) + + return tokenized_messages, mask From 0770781ad75fc733455725227e27dfa2c7213f24 Mon Sep 17 00:00:00 2001 From: Rafi Ayub <33648637+RdoubleA@users.noreply.github.com> Date: Tue, 2 Apr 2024 14:36:04 -0700 Subject: [PATCH 24/76] Split alpaca_dataset to alpaca + alpaca_cleaned (#639) --- docs/source/api_ref_datasets.rst | 1 + docs/source/examples/configs.rst | 4 +- docs/source/examples/finetune_llm.rst | 12 +--- recipes/configs/llama2/13B_lora.yaml | 3 +- recipes/configs/llama2/7B_lora.yaml | 3 +- .../configs/llama2/7B_lora_single_device.yaml | 3 +- .../llama2/7B_qlora_single_device.yaml | 3 +- .../recipes/test_lora_finetune_distributed.py | 2 +- .../test_lora_finetune_single_device.py | 2 +- .../torchtune/datasets/test_alpaca_dataset.py | 4 +- torchtune/datasets/__init__.py | 3 +- torchtune/datasets/_alpaca.py | 63 ++++++++++++++++--- 12 files changed, 71 insertions(+), 32 deletions(-) diff --git a/docs/source/api_ref_datasets.rst b/docs/source/api_ref_datasets.rst index c39b9cf4bc..c371883abf 100644 --- a/docs/source/api_ref_datasets.rst +++ b/docs/source/api_ref_datasets.rst @@ -11,6 +11,7 @@ torchtune.datasets :nosignatures: alpaca_dataset + alpaca_cleaned_dataset grammar_dataset samsum_dataset slimorca_dataset diff --git a/docs/source/examples/configs.rst b/docs/source/examples/configs.rst index 1a8fb5294a..921ac4bdc3 100644 --- a/docs/source/examples/configs.rst +++ b/docs/source/examples/configs.rst @@ -119,7 +119,7 @@ keyword arguments not specified in the config if we'd like: def alpaca_dataset( tokenizer: Tokenizer, train_on_input: bool = True, - use_clean: bool = False, + max_seq_len: int = 512, ) -> InstructDataset: from torchtune import config @@ -132,7 +132,7 @@ keyword arguments not specified in the config if we'd like: dataset = config.instantiate( cfg.dataset, tokenizer, - use_clean=True, + train_on_input=False, ) Note that additional keyword arguments will overwrite any duplicated keys in the diff --git a/docs/source/examples/finetune_llm.rst b/docs/source/examples/finetune_llm.rst index 140aa85f06..bc454e1415 100644 --- a/docs/source/examples/finetune_llm.rst +++ b/docs/source/examples/finetune_llm.rst @@ -88,17 +88,9 @@ from Stanford. The following parameters are related to the data: # This is the default value train_on_input: True - # Train on the raw data, not the cleaned version - # This is the default value - use_clean: False + # Truncate after a maximum sequence length to limit memory usage + max_seq_len: 512 - # Shuffle the data between epochs - # This is set in the config - shuffle: True - -.. note:: - Shuffling the data after every epoch is a good practice. This helps makes sure the model does not learn - spurious patterns related to the how the data is sequenced. .. note:: Set ``train_on_input`` to False if you want to learn on the label only i.e. mask out the prompt. The resulting loss diff --git a/recipes/configs/llama2/13B_lora.yaml b/recipes/configs/llama2/13B_lora.yaml index bd69fdc92d..947faf7c6a 100644 --- a/recipes/configs/llama2/13B_lora.yaml +++ b/recipes/configs/llama2/13B_lora.yaml @@ -54,9 +54,8 @@ tokenizer: # Dataset and Sampler dataset: - _component_: torchtune.datasets.alpaca_dataset + _component_: torchtune.datasets.alpaca_cleaned_dataset train_on_input: True - use_clean: True seed: null shuffle: True batch_size: 32 diff --git a/recipes/configs/llama2/7B_lora.yaml b/recipes/configs/llama2/7B_lora.yaml index 6a248b2740..d9035e64c9 100644 --- a/recipes/configs/llama2/7B_lora.yaml +++ b/recipes/configs/llama2/7B_lora.yaml @@ -49,9 +49,8 @@ tokenizer: # Dataset and Sampler dataset: - _component_: torchtune.datasets.alpaca_dataset + _component_: torchtune.datasets.alpaca_cleaned_dataset train_on_input: True - use_clean: True seed: null shuffle: True batch_size: 2 diff --git a/recipes/configs/llama2/7B_lora_single_device.yaml b/recipes/configs/llama2/7B_lora_single_device.yaml index 8c2281df8e..ec8498b9c7 100644 --- a/recipes/configs/llama2/7B_lora_single_device.yaml +++ b/recipes/configs/llama2/7B_lora_single_device.yaml @@ -49,9 +49,8 @@ tokenizer: # Dataset and Sampler dataset: - _component_: torchtune.datasets.alpaca_dataset + _component_: torchtune.datasets.alpaca_cleaned_dataset train_on_input: True - use_clean: True seed: null shuffle: True batch_size: 2 diff --git a/recipes/configs/llama2/7B_qlora_single_device.yaml b/recipes/configs/llama2/7B_qlora_single_device.yaml index b3c874e8d9..26510f1642 100644 --- a/recipes/configs/llama2/7B_qlora_single_device.yaml +++ b/recipes/configs/llama2/7B_qlora_single_device.yaml @@ -47,9 +47,8 @@ tokenizer: # Dataset and Sampler dataset: - _component_: torchtune.datasets.alpaca_dataset + _component_: torchtune.datasets.alpaca_cleaned_dataset train_on_input: True - use_clean: True seed: null shuffle: True batch_size: 2 diff --git a/tests/recipes/test_lora_finetune_distributed.py b/tests/recipes/test_lora_finetune_distributed.py index 3f56bd154f..c3fb06ba65 100644 --- a/tests/recipes/test_lora_finetune_distributed.py +++ b/tests/recipes/test_lora_finetune_distributed.py @@ -33,8 +33,8 @@ def _get_test_config_overrides(self): "batch_size=4", "enable_activation_checkpointing=False", "tokenizer.path=/tmp/test-artifacts/tokenizer.model", + "dataset=torchtune.datasets.alpaca_dataset", "dataset.train_on_input=False", - "dataset.use_clean=False", "seed=9", "epochs=2", "dtype=fp32", diff --git a/tests/recipes/test_lora_finetune_single_device.py b/tests/recipes/test_lora_finetune_single_device.py index c81945abe8..b74d7d8faf 100644 --- a/tests/recipes/test_lora_finetune_single_device.py +++ b/tests/recipes/test_lora_finetune_single_device.py @@ -34,8 +34,8 @@ def _get_test_config_overrides(self, dtype_str: str = "fp32"): f"dtype={dtype_str}", "enable_activation_checkpointing=False", "tokenizer.path=/tmp/test-artifacts/tokenizer.model", + "dataset=torchtune.datasets.alpaca_dataset", "dataset.train_on_input=False", - "dataset.use_clean=False", "seed=9", "epochs=2", "max_steps_per_epoch=2", diff --git a/tests/torchtune/datasets/test_alpaca_dataset.py b/tests/torchtune/datasets/test_alpaca_dataset.py index a11fd68e89..9b9cb56b07 100644 --- a/tests/torchtune/datasets/test_alpaca_dataset.py +++ b/tests/torchtune/datasets/test_alpaca_dataset.py @@ -11,7 +11,7 @@ from tests.test_utils import get_assets_path from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX -from torchtune.datasets import alpaca_dataset +from torchtune.datasets import alpaca_cleaned_dataset, alpaca_dataset from torchtune.modules.tokenizer import Tokenizer @@ -103,7 +103,7 @@ def test_alpaca_clean(self, load_dataset, tokenizer): } ] - alpaca_ds = alpaca_dataset(tokenizer=tokenizer, use_clean=True) + alpaca_ds = alpaca_cleaned_dataset(tokenizer=tokenizer) input, labels = alpaca_ds[0] assert len(input) == len(labels) diff --git a/torchtune/datasets/__init__.py b/torchtune/datasets/__init__.py index 765723d84d..4d96e1194e 100644 --- a/torchtune/datasets/__init__.py +++ b/torchtune/datasets/__init__.py @@ -4,7 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from torchtune.datasets._alpaca import alpaca_dataset +from torchtune.datasets._alpaca import alpaca_cleaned_dataset, alpaca_dataset from torchtune.datasets._chat import ChatDataset from torchtune.datasets._grammar import grammar_dataset from torchtune.datasets._instruct import instruct_dataset, InstructDataset @@ -13,6 +13,7 @@ __all__ = [ "alpaca_dataset", + "alpaca_cleaned_dataset", "grammar_dataset", "samsum_dataset", "InstructDataset", diff --git a/torchtune/datasets/_alpaca.py b/torchtune/datasets/_alpaca.py index 29e8ef9b93..cebdbef474 100644 --- a/torchtune/datasets/_alpaca.py +++ b/torchtune/datasets/_alpaca.py @@ -12,11 +12,10 @@ def alpaca_dataset( tokenizer: Tokenizer, train_on_input: bool = True, - use_clean: bool = False, max_seq_len: int = 512, ) -> InstructDataset: """ - Support for the Alpaca dataset and its variants from Hugging Face Datasets. + Support for the Alpaca dataset from Hugging Face Datasets. https://huggingface.co/datasets/tatsu-lab/alpaca Data input format: https://huggingface.co/datasets/tatsu-lab/alpaca#data-instances @@ -32,14 +31,64 @@ def alpaca_dataset( contributes to the loss. - If `train_on_input` is False, the prompt is masked out (tokens replaced with -100) - The version of the dataset used is controlled by the `use_clean` flag which set to False by default. - - If `use_clean` is True, then https://huggingface.co/datasets/yahma/alpaca-cleaned is used - - If `use_clean` is False, then https://huggingface.co/datasets/tatsu-lab/alpaca is used + Args: + tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. + train_on_input (bool): Whether the model is trained on the prompt or not. Default is True. + max_seq_len (int): Maximum number of tokens in the returned input and label token id lists. + Default is 512, as set by Stanford Alpaca (https://github.com/tatsu-lab/stanford_alpaca?tab=readme-ov-file#fine-tuning), + but we recommend setting this to the highest you can fit in memory and is supported by the model. + For example, llama2-7B supports up to 4096 for sequence length. + + Returns: + InstructDataset: dataset configured with Alpaca source data and template + + + Example: + >>> alpaca_ds = alpaca_dataset(tokenizer=tokenizer) + >>> for batch in Dataloader(alpaca_ds, batch_size=8): + >>> print(f"Batch size: {len(batch)}") + >>> Batch size: 8 + """ + + return InstructDataset( + tokenizer=tokenizer, + source="tatsu-lab/alpaca", + template=AlpacaInstructTemplate, + train_on_input=train_on_input, + max_seq_len=max_seq_len, + split="train", + ) + + +def alpaca_cleaned_dataset( + tokenizer: Tokenizer, + train_on_input: bool = True, + max_seq_len: int = 512, +) -> InstructDataset: + """ + Support for the Alpaca cleaned dataset from Hugging Face Datasets. + https://huggingface.co/datasets/yahma/alpaca-cleaned + + Data input format: https://huggingface.co/datasets/tatsu-lab/alpaca#data-instances + + The input is created using the prompt template from the original alpaca codebase: + https://github.com/tatsu-lab/stanford_alpaca/blob/761dc5bfbdeeffa89b8bff5d038781a4055f796a/train.py#L31 + + where `instruction`, `input`, and `output` are fields from the dataset. + + Masking of the prompt during training is controlled by the `train_on_input` flag, which is + set to `True` by default (ref: https://github.com/tloen/alpaca-lora/blob/main/finetune.py#L49) + - If `train_on_input` is True, the prompt is used during training and + contributes to the loss. + - If `train_on_input` is False, the prompt is masked out (tokens replaced with -100) + + This is the cleaned version of the original Alpaca dataset, which removes hallucinations, + poorly formed instructions/inputs/outputs, wrong answers, and other errors. See more details + on the Hugging Face dataset card. Args: tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. train_on_input (bool): Whether the model is trained on the prompt or not. Default is True. - use_clean (bool): Whether to use the cleaned version of the dataset or not. Default is False. max_seq_len (int): Maximum number of tokens in the returned input and label token id lists. Default is 512, as set by Stanford Alpaca (https://github.com/tatsu-lab/stanford_alpaca?tab=readme-ov-file#fine-tuning), but we recommend setting this to the highest you can fit in memory and is supported by the model. @@ -58,7 +107,7 @@ def alpaca_dataset( return InstructDataset( tokenizer=tokenizer, - source="yahma/alpaca-cleaned" if use_clean else "tatsu-lab/alpaca", + source="yahma/alpaca-cleaned", template=AlpacaInstructTemplate, train_on_input=train_on_input, max_seq_len=max_seq_len, From f085a77ee66b6d13ae95e4410f410d07a5c3f3a7 Mon Sep 17 00:00:00 2001 From: Kartikay Khandelwal <47255723+kartikayk@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:03:14 -0700 Subject: [PATCH 25/76] Add weights_only flag to torchtune checkpointer (#642) --- tests/torchtune/utils/test_checkpointer.py | 28 +++++++++++++++++++ .../utils/_checkpointing/_checkpointer.py | 13 +++++++-- .../_checkpointing/_checkpointer_utils.py | 7 +++-- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/tests/torchtune/utils/test_checkpointer.py b/tests/torchtune/utils/test_checkpointer.py index d70166f551..4616b07486 100644 --- a/tests/torchtune/utils/test_checkpointer.py +++ b/tests/torchtune/utils/test_checkpointer.py @@ -292,3 +292,31 @@ def test_save_load_checkpoint_multiple_file( assert len(output_state_dict_1.keys()) + 1 == len(orig_state_dict_1.keys()) assert len(output_state_dict_2.keys()) + 1 == len(orig_state_dict_2.keys()) + + +class TestCheckpointerUtils: + @pytest.fixture + def model_checkpoint(self, tmp_path): + """ + Fixture which creates a checkpoint file for testing checkpointer utils. + """ + checkpoint_file = tmp_path / "model_checkpoint_01.pt" + + state_dict = { + "token_embeddings.weight": torch.ones(1, 10), + "output.weight": torch.ones(1, 10), + } + + torch.save(state_dict, checkpoint_file) + + return checkpoint_file + + @pytest.mark.parametrize("weights_only", [True, False]) + def test_safe_torch_load(self, model_checkpoint, weights_only): + state_dict = safe_torch_load(Path(model_checkpoint), weights_only) + + assert "token_embeddings.weight" in state_dict + assert "output.weight" in state_dict + + assert state_dict["token_embeddings.weight"].shape[1] == 10 + assert state_dict["output.weight"].shape[0] == 1 diff --git a/torchtune/utils/_checkpointing/_checkpointer.py b/torchtune/utils/_checkpointing/_checkpointer.py index 6c273a11be..4b6486d84c 100644 --- a/torchtune/utils/_checkpointing/_checkpointer.py +++ b/torchtune/utils/_checkpointing/_checkpointer.py @@ -148,7 +148,7 @@ def __init__( ) self._recipe_checkpoint = get_path(self._checkpoint_dir, recipe_checkpoint) - def load_checkpoint(self) -> Dict[str, Any]: + def load_checkpoint(self, weights_only: bool = True) -> Dict[str, Any]: """ Load TorchTune checkpoint from file. Currently only loading from a single file is supported. @@ -162,9 +162,18 @@ def load_checkpoint(self) -> Dict[str, Any]: "optimizer": ..., ... } + + Args: + weights_only (bool): flag passed down to torch.load. We expose this, because quantized models + cannot be loaded with weights_only=True + + Returns: + Dict[str, Any]: state_dict from the input checkpoint """ state_dict: Dict[str:Any] = {} - state_dict[utils.MODEL_KEY] = safe_torch_load(self._checkpoint_path) + state_dict[utils.MODEL_KEY] = safe_torch_load( + self._checkpoint_path, weights_only=weights_only + ) if self._adapter_checkpoint: adapter_state_dict = safe_torch_load(self._adapter_checkpoint) diff --git a/torchtune/utils/_checkpointing/_checkpointer_utils.py b/torchtune/utils/_checkpointing/_checkpointer_utils.py index 3f52e8c968..d1ae4d2374 100644 --- a/torchtune/utils/_checkpointing/_checkpointer_utils.py +++ b/torchtune/utils/_checkpointing/_checkpointer_utils.py @@ -47,7 +47,7 @@ def get_path(input_dir: Path, filename: str, missing_ok: bool = False) -> Path: return file_path -def safe_torch_load(checkpoint_path: Path) -> Dict[str, Any]: +def safe_torch_load(checkpoint_path: Path, weights_only: bool = True) -> Dict[str, Any]: """ Utility to load a checkpoint file in a safe manner. """ @@ -55,7 +55,10 @@ def safe_torch_load(checkpoint_path: Path) -> Dict[str, Any]: # convert the path into a string since pathlib Path and mmap don't work # well together state_dict = torch.load( - str(checkpoint_path), map_location="cpu", mmap=True, weights_only=True + str(checkpoint_path), + map_location="cpu", + mmap=True, + weights_only=weights_only, ) except Exception as e: raise ValueError(f"Unable to load checkpoint from {checkpoint_path}. ") from e From 34accd91d49632f41cf87238666f941c324f405d Mon Sep 17 00:00:00 2001 From: ebsmothers Date: Tue, 2 Apr 2024 15:38:34 -0700 Subject: [PATCH 26/76] add missing tokenize_messages docstring (#643) --- torchtune/modules/tokenizer.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/torchtune/modules/tokenizer.py b/torchtune/modules/tokenizer.py index f158e05c83..07c0268fb4 100644 --- a/torchtune/modules/tokenizer.py +++ b/torchtune/modules/tokenizer.py @@ -126,6 +126,40 @@ def decode(self, ids: List[int]) -> str: def tokenize_messages( self, messages: List[Message], max_seq_len: Optional[int] = None ) -> Tuple[List[int], List[bool]]: + r"""Tokenize a list of messages one at a time then concatenate them, + returning a list of tokens and a list of masks. + + Note: llama2 sentencepiece has problems where in general + encode(s1 + s2) != encode(s1) + encode(s2) due to whitespace handling. + We can get around this by prepending s2 with a known token and slicing the + beginning off the tokenized s2. + + Example: + >>> tokenizer = Tokenizer.from_file(tokenizer_path) + >>> messages = [ + Message(role="system", content="system message\n", masked=True), + Message(role="user", content="user prompt\n", masked=True), + Message(role="assistant", content="assistant response\n"), + ] + # tokenize_messages encodes messages separately and concats + >>> tokenizer.tokenize_messages(messages, max_seq_len)[0] + [1, 1788, 2643, 13, 1792, 9508, 13, 465, 22137, 2933, 2] + + + # Same result as encoding the full string in one go + >>> tokenizer.encode(''.join([message.content for message in messages])) + [1, 1788, 2643, 13, 1792, 9508, 13, 465, 22137, 2933, 2] + + + Args: + messages (List[Message]): A list of messages, each containing role, content, + and masked attributes. + max_seq_len (Optional[int]): A max sequence length to truncate tokens to. + Default: None + + Returns: + Tuple[List[int], List[bool]]: The tokenized messages + """ start_of_turn = True end_of_turn = False prev_ends_with_space = False From 07d381328e735b39be969e7d299a4a35bd6a440e Mon Sep 17 00:00:00 2001 From: Rafi Ayub <33648637+RdoubleA@users.noreply.github.com> Date: Wed, 3 Apr 2024 10:20:39 -0700 Subject: [PATCH 27/76] Add string to InstructTemplate, ChatFormat getters (#641) --- tests/torchtune/config/test_config_utils.py | 32 +++++++++++++ torchtune/config/_utils.py | 52 +++++++++++++++++++++ torchtune/datasets/_chat.py | 17 +++---- torchtune/datasets/_instruct.py | 7 +-- 4 files changed, 97 insertions(+), 11 deletions(-) diff --git a/tests/torchtune/config/test_config_utils.py b/tests/torchtune/config/test_config_utils.py index 1f7f01d4e5..b1c77f362c 100644 --- a/tests/torchtune/config/test_config_utils.py +++ b/tests/torchtune/config/test_config_utils.py @@ -8,10 +8,14 @@ import pytest from torchtune.config._utils import ( + _get_chat_format, _get_component_from_path, + _get_instruct_template, _merge_yaml_and_cli_args, + _try_get_component, InstantiationError, ) +from torchtune.data import AlpacaInstructTemplate, Llama2ChatFormat from torchtune.utils.argparse import TuneRecipeArgumentParser _CONFIG = { @@ -107,3 +111,31 @@ def test_merge_yaml_and_cli_args(self, mock_load): ValueError, match="Command-line overrides must be in the form of key=value" ): _ = _merge_yaml_and_cli_args(yaml_args, cli_args) + + def test_try_get_component(self): + # Test a valid classname + template = _try_get_component( + module_path="torchtune.data._instruct_templates", + component_name="AlpacaInstructTemplate", + class_type="InstructTemplate", + ) + assert template == AlpacaInstructTemplate + + # Test an invalid class + with pytest.raises( + ValueError, + match="Invalid InstructTemplate class", + ): + _ = _try_get_component( + module_path="torchtune.data._instruct_templates", + component_name="InvalidTemplate", + class_type="InstructTemplate", + ) + + def test_get_instruct_template(self): + assert ( + _get_instruct_template("AlpacaInstructTemplate") == AlpacaInstructTemplate + ) + + def test_get_chat_format(self): + assert _get_chat_format("Llama2ChatFormat") == Llama2ChatFormat diff --git a/torchtune/config/_utils.py b/torchtune/config/_utils.py index 16c5b30d1f..b76ebdd879 100644 --- a/torchtune/config/_utils.py +++ b/torchtune/config/_utils.py @@ -12,6 +12,7 @@ from omegaconf import DictConfig, OmegaConf from torchtune.config._errors import InstantiationError +from torchtune.data import ChatFormat, InstructTemplate def _has_component(node: Union[Dict[str, Any], DictConfig]) -> bool: @@ -148,3 +149,54 @@ def _merge_yaml_and_cli_args(yaml_args: Namespace, cli_args: List[str]) -> DictC # CLI takes precedence over yaml args return OmegaConf.merge(yaml_conf, cli_conf) + + +def _try_get_component(module_path: str, component_name: str, class_type: str) -> Any: + """ + Try-except wrapper around `_get_component_from_path`, used to quickly retrieve + a class from a name string with better error handling. + + Args: + module_path (str): path string of the file the class resides in + component_name (str): name of the class + class_type (str): type of the class, only used for more descriptive error message + + + Returns: + Any: the class + + Raises: + ValueError: if the string is not a valid class + """ + try: + return _get_component_from_path(module_path + "." + component_name) + except InstantiationError: + raise ValueError(f"Invalid {class_type} class: '{component_name}'") from None + + +def _get_instruct_template(template: str) -> InstructTemplate: + """ + Get the instruct template class from the template string. + + Args: + template (str): class name of template, or string with placeholders + + Returns: + InstructTemplate: the prompt template class or the same verified string + """ + return _try_get_component( + "torchtune.data._instruct_templates", template, "InstructTemplate" + ) + + +def _get_chat_format(chat_format: str) -> ChatFormat: + """ + Get the chat format class from a string. + + Args: + chat_format (str): class name of the ChatFormat + + Returns: + ChatFormat: the chat format class + """ + return _try_get_component("torchtune.data._chat_formats", chat_format, "ChatFormat") diff --git a/torchtune/datasets/_chat.py b/torchtune/datasets/_chat.py index 2858b45eb4..9f5d8e9524 100644 --- a/torchtune/datasets/_chat.py +++ b/torchtune/datasets/_chat.py @@ -10,6 +10,7 @@ from datasets import load_dataset from torch.utils.data import Dataset +from torchtune.config._utils import _get_chat_format from torchtune.data import ChatFormat, Message, sharegpt_to_llama2_messages from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX from torchtune.modules import Tokenizer @@ -91,8 +92,8 @@ def _prepare_sample(self, sample: Mapping[str, Any]) -> Tuple[List[int], List[in def chat_dataset( tokenizer: Tokenizer, source: str, - conversation_format: str, - chat_format: ChatFormat, + conversation_style: str, + chat_format: str, max_seq_len: int, train_on_input: bool = False, **load_dataset_kwargs: Dict[str, Any], @@ -106,9 +107,9 @@ def chat_dataset( tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. source (str): path string of dataset, anything supported by Hugging Face's `load_dataset` (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) - conversation_format (str): string specifying expected format of conversations in the dataset - for automatic conversion to the llama format. Supported formats are: "sharegpt" - chat_format (ChatFormat): Template class used to format the chat. + conversation_style (str): string specifying expected style of conversations in the dataset + for automatic conversion to the llama style. Supported styles are: "sharegpt" + chat_format (str): name of ChatFormat class used to format the messages. max_seq_len (int): Maximum number of tokens in the returned input and label token id lists. train_on_input (bool): Whether the model is trained on the prompt or not. Default is False. **load_dataset_kwargs (Dict[str, Any]): additional keyword arguments to pass to `load_dataset`. @@ -119,16 +120,16 @@ def chat_dataset( Raises: ValueError: if the conversation format is not supported """ - if conversation_format == "sharegpt": + if conversation_style == "sharegpt": convert_to_messages = sharegpt_to_llama2_messages else: - raise ValueError(f"Unsupported conversation format: {conversation_format}") + raise ValueError(f"Unsupported conversation style: {conversation_style}") return ChatDataset( tokenizer=tokenizer, source=source, convert_to_messages=convert_to_messages, - chat_format=chat_format, + chat_format=_get_chat_format(chat_format), max_seq_len=max_seq_len, train_on_input=train_on_input, **load_dataset_kwargs, diff --git a/torchtune/datasets/_instruct.py b/torchtune/datasets/_instruct.py index 598d69d865..19c61aa7e6 100644 --- a/torchtune/datasets/_instruct.py +++ b/torchtune/datasets/_instruct.py @@ -9,6 +9,7 @@ import numpy as np from datasets import load_dataset from torch.utils.data import Dataset +from torchtune.config._utils import _get_instruct_template from torchtune.data import InstructTemplate, Message @@ -104,7 +105,7 @@ def _prepare_sample(self, sample: Mapping[str, Any]) -> Tuple[List[int], List[in def instruct_dataset( tokenizer: Tokenizer, source: str, - template: InstructTemplate, + template: str, column_map: Optional[Dict[str, str]] = None, train_on_input: bool = False, max_seq_len: Optional[int] = None, @@ -119,7 +120,7 @@ def instruct_dataset( tokenizer (Tokenizer): Tokenizer used to encode data. Tokenize must implement an `encode` and `decode` method. source (str): path string of dataset, anything supported by Hugging Face's `load_dataset` (https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset.path) - template (InstructTemplate): class used to format the prompt. If the placeholder variable + template (str): class used to format the prompt. If the placeholder variable names in the template do not match the column/key names in the dataset, use `column_map` to map them. column_map (Optional[Dict[str, str]]): a mapping from the expected placeholder names in the template to the column/key names in the sample. If None, assume these are identical. @@ -135,7 +136,7 @@ def instruct_dataset( return InstructDataset( tokenizer=tokenizer, source=source, - template=template, + template=_get_instruct_template(template), column_map=column_map, train_on_input=train_on_input, max_seq_len=max_seq_len, From 7fab51f0690c094506b636863eff19b203689bc6 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Wed, 3 Apr 2024 18:03:59 -0400 Subject: [PATCH 28/76] Add ``include_package_data`` to setuptools (#649) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0de506a8ae..5f70e81dfe 100644 --- a/setup.py +++ b/setup.py @@ -30,4 +30,5 @@ def read_requirements(file): long_description_content_type="text/markdown", url="https://github.com/pytorch/torchtune", extras_require={"dev": read_requirements("dev-requirements.txt")}, + include_package_data=True, ) From 96ecf281ef4415b2a87b5b53e2c091624bbec93a Mon Sep 17 00:00:00 2001 From: Mengtao Yuan Date: Wed, 3 Apr 2024 17:40:58 -0700 Subject: [PATCH 29/76] Add verification of llama model access in first_finetune_tutorial.rst (#650) Co-authored-by: Kartikay Khandelwal <47255723+kartikayk@users.noreply.github.com> --- docs/source/examples/first_finetune_tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/examples/first_finetune_tutorial.rst b/docs/source/examples/first_finetune_tutorial.rst index 3a628042ba..beee903559 100644 --- a/docs/source/examples/first_finetune_tutorial.rst +++ b/docs/source/examples/first_finetune_tutorial.rst @@ -29,7 +29,7 @@ with the `Hugging Face Hub `_ - a coll For this tutorial, you're going to use the `Llama2 model from Meta `_. Llama2 is a "gated model", meaning that you need to be granted access in order to download the weights. Follow `these instructions `_ on the official Meta page -hosted on Hugging Face to complete this process. (This should take less than 5 minutes.) +hosted on Hugging Face to complete this process. (This should take less than 5 minutes.) To verify that you have the access, go to the `model page `_. You should be able to see the model files. If not, you may need to accept the agreement to complete the signup process. Once you have authorization, you will need to authenticate with Hugging Face Hub. The easiest way to do so is to provide an access token to the download script. You can find your token `here `_. From 77eb6953e011d3938296a0f6cd99c1e75154be3e Mon Sep 17 00:00:00 2001 From: ebsmothers Date: Wed, 3 Apr 2024 19:19:04 -0700 Subject: [PATCH 30/76] grad accum in LoRA distributed recipe (#644) --- recipes/configs/llama2/7B_lora.yaml | 1 + recipes/full_finetune_distributed.py | 13 +--------- recipes/full_finetune_single_device.py | 16 +++--------- recipes/lora_finetune_distributed.py | 36 ++++++++++++++++++-------- recipes/lora_finetune_single_device.py | 12 +-------- 5 files changed, 32 insertions(+), 46 deletions(-) diff --git a/recipes/configs/llama2/7B_lora.yaml b/recipes/configs/llama2/7B_lora.yaml index d9035e64c9..16053b7168 100644 --- a/recipes/configs/llama2/7B_lora.yaml +++ b/recipes/configs/llama2/7B_lora.yaml @@ -70,6 +70,7 @@ loss: # Training epochs: 1 max_steps_per_epoch: null +gradient_accumulation_steps: 1 # Logging output_dir: /tmp/lora_finetune_output diff --git a/recipes/full_finetune_distributed.py b/recipes/full_finetune_distributed.py index 7bcb590857..db1e3ce6f3 100644 --- a/recipes/full_finetune_distributed.py +++ b/recipes/full_finetune_distributed.py @@ -378,17 +378,6 @@ def save_checkpoint(self, epoch: int) -> None: intermediate_checkpoint=(epoch + 1 < self.total_epochs), ) - def _should_update_weights(self, current_iteration: int) -> bool: - """ - Determines whether the weights should be updated on the current iteration or not. - True is returned either if we've accumulated gradients for enough steps or if this - is the last step in the epoch. - """ - should_update_weights = ( - current_iteration + 1 - ) % self._gradient_accumulation_steps == 0 - return should_update_weights - def train(self) -> None: """ The core training loop. Supports training on subsets of the dataset using the @@ -450,7 +439,7 @@ def train(self) -> None: loss = loss / self._gradient_accumulation_steps loss.backward() - if self._should_update_weights(idx): + if (idx + 1) % self._gradient_accumulation_steps == 0: self._optimizer.step() self._optimizer.zero_grad(set_to_none=True) diff --git a/recipes/full_finetune_single_device.py b/recipes/full_finetune_single_device.py index 3e6dd2fb1f..3c276369d4 100644 --- a/recipes/full_finetune_single_device.py +++ b/recipes/full_finetune_single_device.py @@ -336,17 +336,6 @@ def save_checkpoint(self, epoch: int) -> None: intermediate_checkpoint=(epoch + 1 < self.total_epochs), ) - def _should_update_weights(self, current_iteration: int) -> bool: - """ - Determines whether the weights should be updated on the current iteration or not. - True is returned either if we've accumulated gradients for enough steps or if this - is the last step in the epoch. - """ - should_update_weights = ( - current_iteration + 1 - ) % self._gradient_accumulation_steps == 0 - return should_update_weights - def train(self) -> None: """ The core training loop. Supports training on subsets of the dataset using the @@ -400,7 +389,10 @@ def train(self) -> None: loss = loss / self._gradient_accumulation_steps loss.backward() - if not self._optimizer_in_bwd and self._should_update_weights(idx): + if ( + not self._optimizer_in_bwd + and (idx + 1) % self._gradient_accumulation_steps == 0 + ): self._optimizer.step() self._optimizer.zero_grad(set_to_none=True) diff --git a/recipes/lora_finetune_distributed.py b/recipes/lora_finetune_distributed.py index d400277185..1ac626d378 100644 --- a/recipes/lora_finetune_distributed.py +++ b/recipes/lora_finetune_distributed.py @@ -104,6 +104,7 @@ def __init__(self, cfg: DictConfig) -> None: self.total_training_steps = 0 self._resume_from_checkpoint = cfg.resume_from_checkpoint + self._gradient_accumulation_steps = cfg.gradient_accumulation_steps def load_checkpoint(self, cfg_checkpointer: DictConfig) -> Dict[str, Any]: """ @@ -205,18 +206,21 @@ def setup(self, cfg: DictConfig) -> None: # by the dataloader and the max_steps_per_epoch param set by the user and is used # for logging and tracking training state. This should be computed after the dataloader # has been setup - steps_per_epoch = len(self._dataloader) - if self.max_steps_per_epoch is not None and self.max_steps_per_epoch < len( - self._dataloader + self._steps_per_epoch = ( + len(self._dataloader) // self._gradient_accumulation_steps + ) + if ( + self.max_steps_per_epoch is not None + and self.max_steps_per_epoch < self._steps_per_epoch ): - steps_per_epoch = self.max_steps_per_epoch - self.total_training_steps = self.epochs_run * steps_per_epoch + self._steps_per_epoch = self.max_steps_per_epoch + self.total_training_steps = self.epochs_run * self._steps_per_epoch # Learning rate scheduler can only be set up after number of steps # has been computed self._lr_scheduler = self._setup_lr_scheduler( cfg_lr_scheduler=cfg.lr_scheduler, - num_training_steps=self.total_epochs * steps_per_epoch, + num_training_steps=self.total_epochs * self._steps_per_epoch, last_epoch=self.total_training_steps - 1, ) @@ -477,6 +481,9 @@ def train(self) -> None: _, rank = utils.get_world_size_and_rank() + # zero out the gradients before starting training + self._optimizer.zero_grad() + # self.epochs_run should be non-zero when we're resuming from a checkpoint for curr_epoch in range(self.epochs_run, self.total_epochs): @@ -489,11 +496,10 @@ def train(self) -> None: ): if ( self.max_steps_per_epoch is not None - and idx == self.max_steps_per_epoch + and (idx // self._gradient_accumulation_steps) + == self.max_steps_per_epoch ): break - self.total_training_steps += 1 - self._optimizer.zero_grad() input_ids, labels = batch input_ids = input_ids.to(self._device) @@ -521,9 +527,17 @@ def train(self) -> None: step=self.total_training_steps, # Each step is unique, not limited to each epoch ) + loss = loss / self._gradient_accumulation_steps loss.backward() - self._optimizer.step() - self._lr_scheduler.step() + + if (idx + 1) % self._gradient_accumulation_steps == 0: + self._optimizer.step() + self._optimizer.zero_grad(set_to_none=True) + self._lr_scheduler.step() + + # Update the number of steps when the weights are updated + self.total_training_steps += 1 + if ( self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero diff --git a/recipes/lora_finetune_single_device.py b/recipes/lora_finetune_single_device.py index 37e36cd9ef..5f09bf8f13 100644 --- a/recipes/lora_finetune_single_device.py +++ b/recipes/lora_finetune_single_device.py @@ -370,16 +370,6 @@ def save_checkpoint(self, epoch: int) -> None: intermediate_checkpoint=(epoch + 1 < self.total_epochs), ) - def _should_update_weights(self, current_iteration: int) -> bool: - """ - Determines whether the weights should be updated on the current iteration or not. - True is returned either if we've accumulated gradients for enough steps. - """ - should_update_weights = ( - current_iteration + 1 - ) % self._gradient_accumulation_steps == 0 - return should_update_weights - def train(self) -> None: """ The core training loop. @@ -422,7 +412,7 @@ def train(self) -> None: ) loss = loss / self._gradient_accumulation_steps loss.backward() - if self._should_update_weights(idx): + if (idx + 1) % self._gradient_accumulation_steps == 0: self._optimizer.step() self._optimizer.zero_grad(set_to_none=True) self._lr_scheduler.step() From 76c21b75400d4d2872880ffeaa570bdc9ca0934b Mon Sep 17 00:00:00 2001 From: solitude-alive <44771751+solitude-alive@users.noreply.github.com> Date: Fri, 5 Apr 2024 02:35:00 +0800 Subject: [PATCH 31/76] Gemma (#630) Co-authored-by: ebsmothers --- recipes/configs/gemma/2B_full.yaml | 86 +++ recipes/gemma_full_finetune_distributed.py | 497 ++++++++++++++++++ requirements.txt | 1 + torchtune/_recipe_registry.py | 8 + torchtune/models/__init__.py | 2 +- torchtune/models/convert_weights.py | 6 +- torchtune/models/gemma/__init__.py | 8 + torchtune/models/gemma/_component_builders.py | 131 +++++ torchtune/models/gemma/_model_builders.py | 47 ++ torchtune/models/gemma/rms_norm.py | 26 + torchtune/models/gemma/transformer.py | 133 +++++ torchtune/modules/feed_forward.py | 5 +- torchtune/modules/position_embeddings.py | 2 +- .../utils/_checkpointing/_checkpointer.py | 16 + .../_checkpointing/_checkpointer_utils.py | 54 +- 15 files changed, 1012 insertions(+), 10 deletions(-) create mode 100644 recipes/configs/gemma/2B_full.yaml create mode 100644 recipes/gemma_full_finetune_distributed.py create mode 100644 torchtune/models/gemma/__init__.py create mode 100644 torchtune/models/gemma/_component_builders.py create mode 100644 torchtune/models/gemma/_model_builders.py create mode 100644 torchtune/models/gemma/rms_norm.py create mode 100644 torchtune/models/gemma/transformer.py diff --git a/recipes/configs/gemma/2B_full.yaml b/recipes/configs/gemma/2B_full.yaml new file mode 100644 index 0000000000..ebabb23c24 --- /dev/null +++ b/recipes/configs/gemma/2B_full.yaml @@ -0,0 +1,86 @@ +# Config for multi-device full finetuning in full_finetune_distributed.py +# using a gemma 2B model +# +# This config assumes that you've run the following command before launching +# this run: +# tune download --repo-id google/gemma-2b \ +# --hf-token \ +# --output-dir /tmp/gemma2 +# +# To launch on 4 devices, run the following command from root: +# tune --nnodes 1 --nproc_per_node 4 full_finetune_distributed \ +# --config gemma/2B_full \ +# +# You can add specific overrides through the command line. For example +# to override the checkpointer directory while launching training +# you can run: +# tune --nnodes 1 --nproc_per_node 4 full_finetune_distributed \ +# --config gemma/2B_full \ +# checkpointer.checkpoint_dir= +# +# This config works only when the model is being fine-tuned on 2+ GPUs. + + +# Tokenizer +tokenizer: + _component_: torchtune.models.gemma.gemma_tokenizer + path: /tmp/gemma/tokenizer.model + +# Dataset +dataset: + _component_: torchtune.datasets.alpaca_dataset + train_on_input: True +seed: null +shuffle: True + +# Model Arguments +model: + _component_: torchtune.models.gemma.gemma_2b + +checkpointer: + _component_: torchtune.utils.FullModelHFCheckpointer + checkpoint_dir: /tmp/gemma/ + checkpoint_files: [ + model-00001-of-00002.safetensors, + model-00002-of-00002.safetensors, + ] + recipe_checkpoint: null + output_dir: /tmp/gemma + model_type: GEMMA +share_weights: + share_weights: True + weight_tying_config: { + "output": "tok_embeddings" + } +resume_from_checkpoint: False + +# Fine-tuning arguments +batch_size: 2 +epochs: 3 +optimizer: + _component_: torch.optim.AdamW + lr: 2e-5 +loss: + _component_: torch.nn.CrossEntropyLoss +max_steps_per_epoch: null +gradient_accumulation_steps: 1 + + +# Training env +device: cuda + +# Distributed +cpu_offload: False + +# Memory management +enable_activation_checkpointing: True + +# Reduced precision +dtype: bf16 + +# Logging +metric_logger: + _component_: torchtune.utils.metric_logging.DiskLogger + log_dir: ${output_dir} +output_dir: /tmp/alpaca-gemma-finetune +log_every_n_steps: null diff --git a/recipes/gemma_full_finetune_distributed.py b/recipes/gemma_full_finetune_distributed.py new file mode 100644 index 0000000000..dad0ded7bb --- /dev/null +++ b/recipes/gemma_full_finetune_distributed.py @@ -0,0 +1,497 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import sys +import time + +from functools import partial +from typing import Any, Dict, Optional, Tuple +from warnings import warn + +import torch +from omegaconf import DictConfig + +from torch import nn +from torch.distributed import init_process_group +from torch.distributed.fsdp import ( + FullOptimStateDictConfig, + FullStateDictConfig, + FullyShardedDataParallel as FSDP, + StateDictType, +) +from torch.distributed.fsdp.wrap import ModuleWrapPolicy +from torch.optim import Optimizer +from torch.utils.data import DataLoader, DistributedSampler + +from torchtune import config, modules, utils + +from torchtune.recipe_interfaces import FTRecipeInterface + +from tqdm import tqdm + + +log = utils.get_logger("DEBUG") + + +class FullFinetuneRecipeDistributed(FTRecipeInterface): + """ + Full finetuning recipe for dense transformer-based LLMs such as Llama2. + + This recipe supports: + - FSDP and activation checkpointing. AC is disabled by default but can be enabled using + the ``activation_checkpointing`` flag. DDP is not supported. + - Full fp32 and bf16 training are supported + - Checkpointing of model weights, optimizer state and the recipe state (epoch and seed). + - Resuming from checkpoints saved using the ``save_checkpoint`` functionality. + - Logging to terminal. WandB and TensorBoard. + + Assumptions: + - Training is launched with the Tune CLI (recommended) which uses TorchRun under the + hood. Setting up the env variables is handled by TorchRun. + - Training is on multiple GPUs (--nproc_per_node > 1). ``world_size=1`` is currently supported + on CPU for our unit tests. This will change soon + - Checkpoints are ONLY saved at epoch boundaries. Mid-epoch checkpointing is NOT supported. + - Datasets are Map-style and data fits in memory (not streamed) + + The following configs can be used to run this recipe: + >>> tune ls + RECIPE CONFIG + full_finetune_distributed Gemma/2B_full + + Args: + cfg (DictConfig): OmegaConf object parsed from yaml file + + Raises: + ValueError: If ``dtype`` is set to fp16. + """ + + def __init__(self, cfg: DictConfig) -> None: + + self._device = utils.get_device(device=cfg.device) + self._dtype = utils.get_dtype(cfg.dtype, device=self._device) + + if self._dtype == torch.float16: + raise ValueError( + "full fp16 training is not supported with this recipe. Please use bf16 or fp32 instead." + ) + + # logging attributes + self._output_dir = cfg.output_dir + self._log_every_n_steps = cfg.log_every_n_steps if cfg.log_every_n_steps else 1 + self._log_peak_memory_every_n_steps = 100 + + # _is_rank_zero is used primarily for logging. In the future, the logger + # should directly take care of this + _, rank = utils.get_world_size_and_rank() + self._is_rank_zero = rank == 0 + + # Training cfg + self._resume_from_checkpoint = cfg.resume_from_checkpoint + self._gradient_accumulation_steps = cfg.gradient_accumulation_steps + + # These are public properties which are updated by the checkpoint loader + # when ``resume_from_checkpoint`` is `True` or validated in tests + self.seed = utils.set_seed(seed=cfg.seed) + self.epochs_run = 0 + self.total_epochs = cfg.epochs + self.max_steps_per_epoch = cfg.max_steps_per_epoch + self.total_training_steps = 0 + + def load_checkpoint(self, cfg: DictConfig) -> Dict[str, Any]: + """ + Extract the checkpoint state from file and validate. If resume_from_checkpoint + is True, this also includes the recipe state. + """ + self._checkpointer = config.instantiate( + cfg, + resume_from_checkpoint=self._resume_from_checkpoint, + ) + checkpoint_dict = self._checkpointer.load_checkpoint() + + if self._resume_from_checkpoint: + self._update_recipe_state(checkpoint_dict) + return checkpoint_dict + + def _update_recipe_state(self, ckpt_dict: Dict[str, Any]) -> None: + """ + Updates the recipe state from checkpoint. + """ + # If seed, total_epoch or max_steps_per_epoch don't match, + # warn the user and overwrite + try: + if ( + self.seed != ckpt_dict[utils.SEED_KEY] + or self.total_epochs != ckpt_dict[utils.TOTAL_EPOCHS_KEY] + or self.max_steps_per_epoch != ckpt_dict[utils.MAX_STEPS_KEY] + ): + warn( + message="""Configured value for seed, epochs or max_steps_per_epoch + does not match the value stored in checkpoint.""" + ) + self.seed = utils.set_seed(seed=ckpt_dict[utils.SEED_KEY]) + self.epochs_run = ckpt_dict[utils.EPOCHS_KEY] + self.total_epochs = ckpt_dict[utils.TOTAL_EPOCHS_KEY] + self.max_steps_per_epoch = ckpt_dict[utils.MAX_STEPS_KEY] + except KeyError as e: + raise KeyError from e( + "Checkpoint does not contain the required keys needed for updating recipe state." + "Are you sure you passed in the right recipe checkpoint?" + ) + + def setup(self, cfg: DictConfig) -> None: + """ + Sets up the recipe state correctly. This includes setting recipe attributes based + on the ``resume_from_checkpoint`` flag. + """ + self._metric_logger = config.instantiate(cfg.metric_logger) + + ckpt_dict = self.load_checkpoint(cfg.checkpointer) + + # TODO: This is a temporary fix to handle the model tie for GEMMA models. + if cfg.checkpointer.model_type == "GEMMA": + shared = cfg.share_weights + else: + shared = None + + # ``_setup_model`` handles initialization and loading the state dict. This method + # should be called before ``_setup_optimizer`` since transforming the optimizer + # state dict requires the model + self._model = self._setup_model( + cfg_model=cfg.model, + enable_activation_checkpointing=cfg.enable_activation_checkpointing, + model_state_dict=ckpt_dict[utils.MODEL_KEY], + shared=shared, + ) + + self._tokenizer = config.instantiate(cfg.tokenizer) + + # _setup_optimizer should take in ckpt_dict only if training is resumed from + # checkpoint. Transforming the opt state dict is handled by this method + self._optimizer = self._setup_optimizer( + cfg_optimizer=cfg.optimizer, + opt_state_dict=ckpt_dict[utils.OPT_KEY] + if self._resume_from_checkpoint + else None, + ) + + self._loss_fn = config.instantiate(cfg.loss) + + # sampler and dataloader depend on the tokenizer and loss_fn and should be + # setup after both of these are initialized + self._sampler, self._dataloader = self._setup_data( + cfg_dataset=cfg.dataset, + shuffle=cfg.shuffle, + batch_size=cfg.batch_size, + ) + + # Finally update the recipe state which can only be correctly set after all of the + # other components have been initialized and updated. + # + # Number of training steps in each epoch depends on the number of batches produced + # by the dataloader, the max_steps_per_epoch param set by the user and the + # gradient_accumulation_steps param. This value is used for logging and tracking + # training state. The computation should happen after the dataloader has been setup + self._steps_per_epoch = ( + len(self._dataloader) // self._gradient_accumulation_steps + ) + if ( + self.max_steps_per_epoch is not None + and self.max_steps_per_epoch < self._steps_per_epoch + ): + self._steps_per_epoch = self.max_steps_per_epoch + self.total_training_steps = self.epochs_run * self._steps_per_epoch + + def _setup_model( + self, + cfg_model: DictConfig, + enable_activation_checkpointing: bool, + model_state_dict: Dict[str, Any], + shared: Dict[str, str] = None, + ) -> nn.Module: + """ + Model initialization has some important considerations: + a. Initialize the model not on the meta device. This is because the model + is wrapped with FSDP which will shard the model across all available GPUs. + b. Shared weights are tied between the output layer and the token embeddings. + This is a temporary fix for GEMMA models. + """ + init_start = time.perf_counter() + + with utils.set_default_dtype(self._dtype): + model = config.instantiate(cfg_model) + + log.info( + f"Model instantiation took {time.perf_counter() - init_start:.2f} secs" + ) + # Load both the model weights + model.load_state_dict(model_state_dict) + + # Tie the weights of the model + if shared is not None: # Tie the weights of the model if required + for k, v in shared.weight_tying_config.items(): + out = getattr(model, v) + tok = getattr(model, k) + out.weight = tok.weight + log.info(f"Model weights are shared between {k} and {v}.") + + if self._dtype == torch.bfloat16: + model = model.to(torch.bfloat16) + + # Wrap the model with FSDP. This will ensure that the model is sharded + # across all available GPUs. + model = FSDP( + module=model, + auto_wrap_policy=ModuleWrapPolicy({modules.TransformerDecoderLayer}), + sharding_strategy=torch.distributed.fsdp.ShardingStrategy.FULL_SHARD, + device_id=self._device, + # this recipe does not currently support mixed precision training + mixed_precision=None, + # Ensure we broadcast params and buffers from rank 0 + sync_module_states=False, + # Initialize empty modules on all non-zero ranks + param_init_fn=None, + ) + + # Ensure no params and buffers are on meta device + utils.validate_no_params_on_meta_device(model) + + if enable_activation_checkpointing: + utils.set_activation_checkpointing( + model, auto_wrap_policy={modules.TransformerDecoderLayer} + ) + if self._is_rank_zero: + log.info( + utils.memory_stats_log( + "Memory Stats after model init", device=self._device + ) + ) + + # synchronize before training begins + torch.distributed.barrier() + + return model + + def _setup_optimizer( + self, cfg_optimizer: DictConfig, opt_state_dict: Optional[Dict[str, Any]] = None + ) -> Optimizer: + """ + Set up the optimizer. This method also handles transforing the state dict + for FSDP. + """ + optimizer = config.instantiate(cfg_optimizer, self._model.parameters()) + + if opt_state_dict: + opt_state_dict = utils.transform_opt_state_dict( + opt_state_dict, self._model, optimizer + ) + optimizer.load_state_dict(opt_state_dict) + + if self._is_rank_zero: + log.info("Optimizer is initialized.") + return optimizer + + def _setup_data( + self, + cfg_dataset: DictConfig, + shuffle: bool, + batch_size: int, + ) -> Tuple[DistributedSampler, DataLoader]: + """ + All data related setup happens here. Currently this recipe only supports the + DistributedSamplers with Map-style Datasets which fit into memory. Other samplers, + iterable datasets and streaming datasets are not supported. + """ + world_size, rank = utils.get_world_size_and_rank() + ds = config.instantiate( + cfg_dataset, + tokenizer=self._tokenizer, + ) + sampler = DistributedSampler( + ds, + num_replicas=world_size, + rank=rank, + shuffle=shuffle, + seed=0, + ) + dataloader = DataLoader( + dataset=ds, + batch_size=batch_size, + sampler=sampler, + collate_fn=partial( + utils.padded_collate, + padding_idx=self._tokenizer.pad_id, + ignore_idx=self._loss_fn.ignore_index, + ), + ) + + if self._is_rank_zero: + log.info("Dataset and Sampler are initialized.") + + return sampler, dataloader + + def save_checkpoint(self, epoch: int) -> None: + """ + Save state dict to file. The recipe save_checkpoint method is responsible for + correctly creating the checkpoint dict and passing to the checkpointer. + """ + checkpoint_dict = {} + + # To prevent GPU memory from spiking during checkpoint save, + # we consolidate the full model and optim state dicts on CPU for rank 0 + with FSDP.state_dict_type( + self._model, + StateDictType.FULL_STATE_DICT, + FullStateDictConfig(offload_to_cpu=True, rank0_only=True), + FullOptimStateDictConfig(offload_to_cpu=True, rank0_only=True), + ): + cpu_state_dict = self._model.state_dict() + opt_state_dict = FSDP.optim_state_dict(self._model, self._optimizer) + + # Now that we have the model and opt state dict, create the actual checkpoint dict + # to be sent to the checkpointer and ultimately written to file + if self._is_rank_zero: + + checkpoint_dict.update({utils.MODEL_KEY: cpu_state_dict}) + + # if training is in-progress, checkpoint the optimizer state as well + if epoch + 1 < self.total_epochs: + checkpoint_dict.update( + { + utils.OPT_KEY: opt_state_dict, + utils.SEED_KEY: self.seed, + utils.EPOCHS_KEY: self.epochs_run, + utils.TOTAL_EPOCHS_KEY: self.total_epochs, + utils.MAX_STEPS_KEY: self.max_steps_per_epoch, + } + ) + + self._checkpointer.save_checkpoint( + checkpoint_dict, + epoch=epoch, + intermediate_checkpoint=(epoch + 1 < self.total_epochs), + ) + + def _should_update_weights(self, current_iteration: int) -> bool: + """ + Determines whether the weights should be updated on the current iteration or not. + True is returned either if we've accumulated gradients for enough steps or if this + is the last step in the epoch. + """ + should_update_weights = ( + current_iteration + 1 + ) % self._gradient_accumulation_steps == 0 + return should_update_weights + + def train(self) -> None: + """ + The core training loop. Supports training on subsets of the dataset using the + ``max_steps_per_epoch``. + """ + # clean up before training begins + utils.cleanup_before_training() + + _, rank = utils.get_world_size_and_rank() + + # zero out the gradients before starting training + self._optimizer.zero_grad() + + # self.epochs_run should be non-zero when we're resuming from a checkpoint + for curr_epoch in range(self.epochs_run, self.total_epochs): + + # Update the sampler to ensure data is correctly shuffled across epochs + # in case shuffle is True + self._sampler.set_epoch(curr_epoch) + + for idx, batch in enumerate( + pbar := tqdm(self._dataloader, disable=not (rank == 0)) + ): + if ( + self.max_steps_per_epoch is not None + and (idx // self._gradient_accumulation_steps) + == self.max_steps_per_epoch + ): + break + + input_ids, labels = batch + input_ids = input_ids.to(self._device) + labels = labels.to(self._device) + + logits = self._model(input_ids) + # Shift so that tokens < n predict n + logits = logits[..., :-1, :].contiguous() + labels = labels[..., 1:].contiguous() + logits = logits.transpose(1, 2) + # Compute loss + loss = self._loss_fn(logits, labels) + + # Note: We're always logging the loss before normalizing it + # Check if this is the norm or not + if ( + self.total_training_steps % self._log_every_n_steps == 0 + and self._is_rank_zero + ): + pbar.set_description(f"{curr_epoch+1}|{idx+1}|Loss: {loss.item()}") + self._metric_logger.log_dict( + { + "loss": loss.item(), + "lr": self._optimizer.param_groups[0]["lr"], + "gpu_resources": torch.cuda.memory_allocated(), + }, + step=self.total_training_steps, + ) + + loss = loss / self._gradient_accumulation_steps + loss.backward() + + if self._should_update_weights(idx): + self._optimizer.step() + self._optimizer.zero_grad(set_to_none=True) + + # Update the number of steps when the weights are updated + self.total_training_steps += 1 + + # Log peak memory for iteration + if ( + self.total_training_steps % self._log_peak_memory_every_n_steps == 0 + and self._is_rank_zero + ): + log.info( + utils.memory_stats_log("Memory Stats", device=self._device) + ) + + self.epochs_run += 1 + self.save_checkpoint(epoch=curr_epoch) + + def cleanup(self) -> None: + self._metric_logger.close() + torch.distributed.destroy_process_group() + + +@config.parse +def recipe_main(cfg: DictConfig) -> None: + """ + Entry point for the recipe. + + Configurable parameters are read in the following order: + - Parameters specified in config (see available configs through ``tune ls``) + - Overwritten by arguments from the command-line + """ + if not utils.is_distributed(): + raise RuntimeError( + "Distributed finetune recipe should be run via a distributed launcher." + "If using tune CLI, please specify --nnodes 1 and --nproc_per_node [num_gpus]" + ) + + init_process_group(backend="gloo" if cfg.device == "cpu" else "nccl") + + recipe = FullFinetuneRecipeDistributed(cfg=cfg) + recipe.setup(cfg=cfg) + recipe.train() + recipe.cleanup() + + +if __name__ == "__main__": + sys.exit(recipe_main()) diff --git a/requirements.txt b/requirements.txt index ffcecfe799..74b3d34d33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # Hugging Face Integration Reqs datasets huggingface_hub +safetensors # Misc sentencepiece diff --git a/torchtune/_recipe_registry.py b/torchtune/_recipe_registry.py index 20e578e306..5a07cb1f78 100644 --- a/torchtune/_recipe_registry.py +++ b/torchtune/_recipe_registry.py @@ -97,6 +97,14 @@ class Recipe: ], supports_distributed=False, ), + Recipe( + name="gemma_full_finetune_distributed", + file_path="gemma_full_finetune_distributed.py", + configs=[ + Config(name="gemma/2B_full", file_path="gemma/2B_full.yaml"), + ], + supports_distributed=True, + ), ] diff --git a/torchtune/models/__init__.py b/torchtune/models/__init__.py index 8b19aba836..f57ff7e27b 100644 --- a/torchtune/models/__init__.py +++ b/torchtune/models/__init__.py @@ -4,4 +4,4 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from torchtune.models import convert_weights, llama2, mistral # noqa +from torchtune.models import convert_weights, gemma, llama2, mistral # noqa diff --git a/torchtune/models/convert_weights.py b/torchtune/models/convert_weights.py index 9ac65b0b2d..2dcff16b7c 100644 --- a/torchtune/models/convert_weights.py +++ b/torchtune/models/convert_weights.py @@ -115,6 +115,7 @@ def hf_to_tune( num_heads: int = 32, num_kv_heads: int = 32, dim: int = 4096, + head_dim: int = None, ) -> Dict[str, torch.Tensor]: """ Convert a state dict from HF's format to TorchTune's format. State dicts @@ -129,12 +130,15 @@ def hf_to_tune( num_heads (int): Number of heads in the model. num_kv_heads (int): Number of heads in the key/value projection layers. dim (int): Dimension of the model. + head_dim (int): Dimension of the head. If not provided, it will be calculated + as dim // num_heads. Returns: Dict[str, torch.Tensor]: State dict in TorchTune's format. """ converted_state_dict = {} - head_dim = dim // num_heads + if head_dim is None: + head_dim = dim // num_heads def _permute(t, n_heads): return ( diff --git a/torchtune/models/gemma/__init__.py b/torchtune/models/gemma/__init__.py new file mode 100644 index 0000000000..b017e0ae42 --- /dev/null +++ b/torchtune/models/gemma/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from ._component_builders import gemma # noqa +from ._model_builders import gemma_2b, gemma_tokenizer # noqa diff --git a/torchtune/models/gemma/_component_builders.py b/torchtune/models/gemma/_component_builders.py new file mode 100644 index 0000000000..4386564c62 --- /dev/null +++ b/torchtune/models/gemma/_component_builders.py @@ -0,0 +1,131 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from functools import partial +from typing import List, Literal, Optional + +from torch import nn + +from torchtune.modules import ( + CausalSelfAttention, + FeedForward, + KVCache, + RotaryPositionalEmbeddings, + TransformerDecoderLayer, +) +from torchtune.models.gemma.rms_norm import GemmaRMSNorm +from torchtune.models.gemma.transformer import GemmaTransformerDecoder + +from torchtune.modules.peft import LORA_ATTN_MODULES, LoRALinear + +""" +Component builders for the Gemma 2B models and popular variants such as LoRA. + +TorchTune provides composable building blocks. Builder functions help +stitch these building blocks into higher-level components. This design has +two benefits: +- The building blocks themselves are very flexible. For example, ``CausalSelfAttention`` +can take either nn.Linear or nn.LoRALinear for ``q_proj``. +- Builder functions expose a set of configurable params which keep the constructors of +the building blocks simple. +""" + + +def gemma( + vocab_size: int, + num_layers: int, + num_heads: int, + head_dim: int, + num_kv_heads: int, + embed_dim: int, + intermediate_dim: int, + max_seq_len: int, + attn_dropout: float = 0.0, + norm_eps: float = 1e-6, + rope_base: int = 10_000, + norm_embeddings: bool = True, +) -> GemmaTransformerDecoder: + """ + Build the decoder associated with the gemma model. This includes: + - Token embeddings + - num_layers number of TransformerDecoderLayer blocks + - RMS Norm layer applied to the output of the transformer + - Final projection into token space + + This does NOT currently include inference-time optimizations such as + sliding-window attention + + Args: + vocab_size (int): number of tokens in vocabulary. + num_layers (int): number of layers in the transformer decoder. + num_heads (int): number of query heads. For MHA this is also the + number of heads for key and value + head_dim (int): dimension of head + num_kv_heads (int): number of key and value heads. + embed_dim (int): embedding dimension for self-attention + intermediate_dim (int): intermediate dimension for MLP + max_seq_len (int): maximum sequence length the model will be run with, + attn_dropout (float): dropout value passed onto scaled_dot_product_attention. + Default: 0.0 + norm_eps (float): epsilon in RMS norms Default: 1e-6 + rope_base (int): base for the rotary positional embeddings. Default: 10_000 + norm_embeddings (bool): whether to apply layer norm before the self-attention + and mlp layers. Default: True + + Returns: + GemmaTransformerDecoder: Instantiation of gemma model. + """ + rope = RotaryPositionalEmbeddings(dim=head_dim, max_seq_len=max_seq_len, base=rope_base) + self_att = CausalSelfAttention( + embed_dim=embed_dim, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + q_proj=nn.Linear(embed_dim, num_heads * head_dim, bias=False), + k_proj=nn.Linear(embed_dim, num_kv_heads * head_dim, bias=False), + v_proj=nn.Linear(embed_dim, num_kv_heads * head_dim, bias=False), + output_proj=nn.Linear(num_heads * head_dim, embed_dim, bias=False), + pos_embeddings=rope, + kv_cache=None, + max_seq_len=max_seq_len, + attn_dropout=attn_dropout, + ) + mlp = gemma_mlp(dim=embed_dim, hidden_dim=intermediate_dim) + layer = TransformerDecoderLayer( + attn=self_att, + mlp=mlp, + sa_norm=GemmaRMSNorm(embed_dim, eps=norm_eps), + mlp_norm=GemmaRMSNorm(embed_dim, eps=norm_eps), + ) + tok_embeddings = nn.Embedding(vocab_size, embed_dim) + output_proj = nn.Linear(embed_dim, vocab_size, bias=False) + model = GemmaTransformerDecoder( + tok_embeddings=tok_embeddings, + layer=layer, + num_layers=num_layers, + max_seq_len=max_seq_len, + num_heads=num_heads, + head_dim=head_dim, + norm=GemmaRMSNorm(embed_dim, eps=norm_eps), + output=output_proj, + norm_embeddings=norm_embeddings, + ) + return model + + +def gemma_mlp(dim: int, hidden_dim: int) -> FeedForward: + """ + Build the MLP layer associated with the Gemma model. + + Args: + dim (int): input dimension to the MLP + hidden_dim (int): hidden dimension of the MLP + """ + gate_proj = nn.Linear(dim, hidden_dim, bias=False) + down_proj = nn.Linear(hidden_dim, dim, bias=False) + up_proj = nn.Linear(dim, hidden_dim, bias=False) + activation = nn.GELU(approximate="tanh") + return FeedForward(gate_proj=gate_proj, down_proj=down_proj, up_proj=up_proj, activation=activation) diff --git a/torchtune/models/gemma/_model_builders.py b/torchtune/models/gemma/_model_builders.py new file mode 100644 index 0000000000..a972dc4224 --- /dev/null +++ b/torchtune/models/gemma/_model_builders.py @@ -0,0 +1,47 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +from typing import List, Optional +from functools import partial + +from torch import nn + +from torchtune.models.gemma._component_builders import gemma + +from torchtune.modules import Tokenizer, TransformerDecoder +from torchtune.modules.peft import LORA_ATTN_MODULES + +""" +Model builders build specific instantiations using component builders. For example +the ``gemma_2b`` model builder uses the ``gemma`` component builder. +""" + + +def gemma_2b() -> TransformerDecoder: + """ + Builder for creating a Gemma 2B model initialized w/ the default 2b parameter values + from: https://blog.google/technology/developers/gemma-open-models/ + + Returns: + TransformerDecoder: Instantiation of Gemma 2B model + """ + return gemma( + vocab_size=256_000, + num_layers=18, + num_heads=8, + head_dim=256, + num_kv_heads=1, + embed_dim=2048, + intermediate_dim=16384, + max_seq_len=8192, + attn_dropout=0.0, + norm_eps=1e-6, + ) + + +def gemma_tokenizer(path: str) -> Tokenizer: + tokenizer = Tokenizer.from_file(path) + tokenizer.pad_id = 0 + return tokenizer diff --git a/torchtune/models/gemma/rms_norm.py b/torchtune/models/gemma/rms_norm.py new file mode 100644 index 0000000000..c803b5f454 --- /dev/null +++ b/torchtune/models/gemma/rms_norm.py @@ -0,0 +1,26 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import torch +from torch import nn + + +class GemmaRMSNorm(nn.Module): + # Copied from https://github.com/google/gemma_pytorch/blob/main/gemma/model.py + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.scale = nn.Parameter(torch.zeros(dim)) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + output = self._norm(x.float()) + # Llama does x.to(float16) * w whilst Gemma is (x * w).to(float16) + # See https://github.com/huggingface/transformers/pull/29402 + output = output * (1.0 + self.scale.float()) + return output.type_as(x) diff --git a/torchtune/models/gemma/transformer.py b/torchtune/models/gemma/transformer.py new file mode 100644 index 0000000000..4a427f64a2 --- /dev/null +++ b/torchtune/models/gemma/transformer.py @@ -0,0 +1,133 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Optional + +import torch +import torch.nn as nn +from torch import Tensor +from torchtune.modules import KVCache + +from torchtune.modules.transformer import _get_clones, TransformerDecoderLayer + + +class GemmaTransformerDecoder(nn.Module): + """ + Transformer Decoder derived from the Llama2 architecture. + + Args: + tok_embeddings (nn.Embedding): PyTorch embedding layer, to be used to move + tokens to an embedding space. + layer (TransformerDecoderLayer): Transformer Decoder layer. + num_layers (int): Number of Transformer Decoder layers. + max_seq_len (int): maximum sequence length the model will be run with, as used + by :func:`~torchtune.modules.KVCache` + num_heads (int): number of query heads. For MHA this is also the + number of heads for key and value. This is used to setup the + :func:`~torchtune.modules.KVCache` + head_dim (int): embedding dimension for each head in self-attention. This is used + to setup the :func:`~torchtune.modules.KVCache` + norm (nn.Module): Callable that applies normalization to the output of the decoder, + before final MLP. + output (nn.Linear): Callable that applies a linear transformation to the output of + the decoder. + norm_embeddings (bool): Whether to normalize the embeddings before passing them + through the decoder layers. Defaults to False. + + Note: + Arg values are checked for correctness (eg: ``attn_dropout`` belongs to [0,1]) + in the module where they are used. This helps reduces the number of raise + statements in code and improves readability. + """ + + def __init__( + self, + tok_embeddings: nn.Embedding, + layer: TransformerDecoderLayer, + num_layers: int, + max_seq_len: int, + num_heads: int, + head_dim: int, + norm: nn.Module, + output: nn.Linear, + norm_embeddings: bool = False, + ) -> None: + super().__init__() + self.tok_embeddings = tok_embeddings + self.layers = _get_clones(layer, num_layers) + self.norm = norm + self.output = output + self.max_seq_len = max_seq_len + self.num_heads = num_heads + self.head_dim = head_dim + self.causal_mask = None + self.norm_embeddings = norm_embeddings + + def setup_caches(self, max_batch_size: int, dtype: torch.dtype) -> None: + for layer in self.layers: + layer.attn.kv_cache = KVCache( + max_batch_size=max_batch_size, + max_seq_len=self.max_seq_len, + num_heads=self.num_heads, + head_dim=self.head_dim, + dtype=dtype, + ) + + # causal_mask is used during inference to ensure we're attending + # to the right tokens + self.causal_mask = torch.tril( + torch.ones(self.max_seq_len, self.max_seq_len, dtype=torch.bool) + ) + + def forward(self, tokens: Tensor, input_pos: Optional[Tensor] = None) -> Tensor: + """ + Args: + tokens (Tensor): input tensor with shape [b x s] + input_pos (Optional[Tensor]): Optional tensor which contains the position + of the current token. This is only used during inference. Default is None + + Returns: + Tensor: output tensor with shape [b x s x v] + + Raises: + ValueError: if causal_mask is set but input_pos is None + + Notation used for tensor shapes: + - b: batch size + - s: sequence length + - v: vocab size + - d: embed dim + """ + # input tensor of shape [b, s] + bsz, seq_len = tokens.shape + + # shape: [b, s, d] + h = self.tok_embeddings(tokens) + + mask = None + if self.causal_mask is not None: + if input_pos is None: + raise ValueError( + "Caches are setup, but the position of input token is missing" + ) + # shape: [1, input_pos_len, m_s] + # in most cases input_pos_len should be 1 + mask = self.causal_mask[None, None, input_pos] + + if self.norm_embeddings: + hidden_dim = h.size(-1) + h = h * torch.tensor(hidden_dim**0.5, dtype=h.dtype) + + for layer in self.layers: + # shape: [b, s, d] + h = layer(h, mask, None) + + # shape: [b, s, d] + h = self.norm(h) + + # shape: [b, s, v] + output = self.output(h).float() + return output diff --git a/torchtune/modules/feed_forward.py b/torchtune/modules/feed_forward.py index 31a76e0dfe..c69cd17ae6 100644 --- a/torchtune/modules/feed_forward.py +++ b/torchtune/modules/feed_forward.py @@ -4,7 +4,6 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import torch.nn.functional as F from torch import nn, Tensor @@ -17,6 +16,7 @@ class FeedForward(nn.Module): down_proj (nn.Module): Final projection to output dim. up_proj (nn.Module): Projection from input dim to hidden dim, multiplied by activation(gate_proj). + activation (nn.Module): Activation function to use. Default is nn.SiLU(). """ def __init__( @@ -25,12 +25,13 @@ def __init__( gate_proj: nn.Module, down_proj: nn.Module, up_proj: nn.Module, + activation: nn.Module = nn.SiLU(), ): super().__init__() self.w1 = gate_proj self.w2 = down_proj self.w3 = up_proj - self.activation = F.silu + self.activation = activation def forward(self, x: Tensor) -> Tensor: return self.w2(self.activation(self.w1(x)) * self.w3(x)) diff --git a/torchtune/modules/position_embeddings.py b/torchtune/modules/position_embeddings.py index 62c78b03e1..3bcdd3d81d 100644 --- a/torchtune/modules/position_embeddings.py +++ b/torchtune/modules/position_embeddings.py @@ -96,7 +96,7 @@ def forward(self, x: Tensor, input_pos: Optional[Tensor] = None) -> Tensor: seq_len = x.size(1) # extract the values based on whether input_pos is set or not. When - # input_pos is provided, we're in infernce mode + # input_pos is provided, we're in inference mode rope_cache = ( self.cache[:seq_len] if input_pos is None else self.cache[input_pos] ) diff --git a/torchtune/utils/_checkpointing/_checkpointer.py b/torchtune/utils/_checkpointing/_checkpointer.py index 4b6486d84c..7c4e589a64 100644 --- a/torchtune/utils/_checkpointing/_checkpointer.py +++ b/torchtune/utils/_checkpointing/_checkpointer.py @@ -17,8 +17,11 @@ from torchtune.models import convert_weights from torchtune.utils._checkpointing._checkpointer_utils import ( get_path, + load_shared_weight_utils, ModelType, safe_torch_load, + save_config, + save_shared_weight_utils, ) from torchtune.utils.logging import get_logger @@ -309,6 +312,9 @@ def __init__( Path.joinpath(self._checkpoint_dir, "config.json").read_text() ) + # save config.json to output_dir + save_config(self._output_dir, self._config) + # recipe_checkpoint contains the recipe state. This should be available if # resume_from_checkpoint is True self._recipe_checkpoint = None @@ -392,6 +398,11 @@ def load_checkpoint(self) -> Dict[str, Any]: dim=self._config["hidden_size"], ) + if self._model_type == "GEMMA": + converted_state_dict[utils.MODEL_KEY] = load_shared_weight_utils( + converted_state_dict[utils.MODEL_KEY] + ) + if self._adapter_checkpoint: adapter_state_dict = safe_torch_load(self._adapter_checkpoint) converted_state_dict[utils.ADAPTER_KEY] = adapter_state_dict @@ -440,6 +451,11 @@ def save_checkpoint( dim=self._config["hidden_size"], ) + if self._model_type == "GEMMA": + save_shared_weight_utils( + weight_map=self._weight_map, state_dict=state_dict[utils.MODEL_KEY] + ) + # split the state_dict into separate dicts, one for each output checkpoint file split_state_dicts: Dict[str, Dict[str, torch.Tensor]] = {} for key, weight in state_dict[utils.MODEL_KEY].items(): diff --git a/torchtune/utils/_checkpointing/_checkpointer_utils.py b/torchtune/utils/_checkpointing/_checkpointer_utils.py index d1ae4d2374..c04f448600 100644 --- a/torchtune/utils/_checkpointing/_checkpointer_utils.py +++ b/torchtune/utils/_checkpointing/_checkpointer_utils.py @@ -4,6 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import json from enum import Enum from pathlib import Path from typing import Any, Dict @@ -11,6 +12,7 @@ import torch import torch.nn as nn import torch.optim as optim +from safetensors import safe_open from torch.distributed.fsdp import FullyShardedDataParallel as FSDP from torchtune.utils._distributed import contains_fsdp @@ -19,6 +21,7 @@ class ModelType(Enum): LLAMA2 = "llama2" MISTRAL = "mistral" + GEMMA = "gemma" def get_path(input_dir: Path, filename: str, missing_ok: bool = False) -> Path: @@ -54,12 +57,19 @@ def safe_torch_load(checkpoint_path: Path, weights_only: bool = True) -> Dict[st try: # convert the path into a string since pathlib Path and mmap don't work # well together - state_dict = torch.load( - str(checkpoint_path), - map_location="cpu", - mmap=True, - weights_only=weights_only, + is_safetensors_file = ( + True if str(checkpoint_path).endswith(".safetensors") else False ) + if is_safetensors_file: + result = {} + with safe_open(checkpoint_path, framework="pt", device="cpu") as f: + for k in f.keys(): + result[k] = f.get_tensor(k) + state_dict = result + else: + state_dict = torch.load( + str(checkpoint_path), map_location="cpu", mmap=True, weights_only=True + ) except Exception as e: raise ValueError(f"Unable to load checkpoint from {checkpoint_path}. ") from e return state_dict @@ -87,3 +97,37 @@ def transform_opt_state_dict( ) return optim_state_dict_to_load + + +def load_shared_weight_utils(state_dict): + if "output.weight" not in state_dict.keys(): + state_dict["output.weight"] = state_dict["tok_embeddings.weight"] + return state_dict + + +def save_shared_weight_utils(weight_map, state_dict): + if ( + "lm_head.weight" not in weight_map.keys() + and "lm_head.weight" in state_dict.keys() + ): + if torch.equal( + state_dict["lm_head.weight"], + state_dict["model.embed_tokens.weight"], + ): + del state_dict["lm_head.weight"] + + +def save_config(path: Path, config: Dict[str, Any]) -> None: + """ + Save a configuration dictionary to a file. + + Args: + path (Path): Path to save the configuration file. + config (Dict[str, Any]): Configuration dictionary to save. + """ + if not path.is_dir(): + path.mkdir(exist_ok=True) + file_path = Path.joinpath(path, "config.json") + if not file_path.exists(): + with open(file_path, "w") as f: + json.dump(config, f) From cba056085161385110944dc5a6fc6a6a2c7da394 Mon Sep 17 00:00:00 2001 From: Rafi Ayub <33648637+RdoubleA@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:02:46 -0700 Subject: [PATCH 32/76] Validate messages (#647) --- tests/torchtune/data/test_data_utils.py | 77 ++++++++++++++++++- torchtune/data/__init__.py | 5 +- .../data/{_transforms.py => _converters.py} | 0 torchtune/data/_utils.py | 48 ++++++++++++ torchtune/datasets/_chat.py | 8 +- torchtune/datasets/_instruct.py | 4 +- 6 files changed, 137 insertions(+), 5 deletions(-) rename torchtune/data/{_transforms.py => _converters.py} (100%) diff --git a/tests/torchtune/data/test_data_utils.py b/tests/torchtune/data/test_data_utils.py index 599f9cc840..c592e00da0 100644 --- a/tests/torchtune/data/test_data_utils.py +++ b/tests/torchtune/data/test_data_utils.py @@ -4,7 +4,8 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from torchtune.data import truncate +import pytest +from torchtune.data import Message, truncate, validate_messages def test_truncate(): @@ -22,3 +23,77 @@ def test_truncate(): # Test truncated mask truncated_masks = truncate(tokens=masks, max_seq_len=4, eos_id=False) assert truncated_masks == [True, True, False, False] + + +def test_validate_messages(): + messages = [ + Message(role="system", content="hello"), + Message(role="user", content="hello"), + Message(role="assistant", content="world"), + ] + + # Test valid conversation with system + validate_messages(messages) + + # Test valid conversation without system + validate_messages(messages[1:]) + + # Test system not first + messages = [ + Message(role="user", content="hello"), + Message(role="system", content="hello"), + Message(role="assistant", content="world"), + ] + with pytest.raises( + ValueError, + match="System message at index 1 in messages, but system messages must come first", + ): + validate_messages(messages) + + # Test empty message + messages = [ + Message(role="system", content="hello"), + Message(role="user", content=""), + Message(role="assistant", content="world"), + ] + with pytest.raises(ValueError, match="Message at index 1 in messages is empty"): + validate_messages(messages) + + # Test empty assistant message + messages = [ + Message(role="system", content="hello"), + Message(role="user", content="world"), + Message(role="assistant", content=""), + ] + validate_messages(messages) + + # Test single message + messages = [ + Message(role="user", content="hello"), + ] + with pytest.raises( + ValueError, match="Messages must be at least length 2, but got 1 messages" + ): + validate_messages(messages) + + # Test repeated user message + messages = [ + Message(role="user", content="hello"), + Message(role="user", content="world"), + Message(role="assistant", content="world"), + ] + with pytest.raises( + ValueError, match="Two consecutive user messages at index 1 and 0 in messages" + ): + validate_messages(messages) + + # Test assistant message comes first + messages = [ + Message(role="assistant", content="hello"), + Message(role="user", content="world"), + ] + with pytest.raises( + ValueError, + match="Assistant message before expected user message at index 0 in messages", + ): + validate_messages(messages) diff --git a/torchtune/data/__init__.py b/torchtune/data/__init__.py index 010798e2d0..3ad19f04da 100644 --- a/torchtune/data/__init__.py +++ b/torchtune/data/__init__.py @@ -10,15 +10,15 @@ Llama2ChatFormat, MistralChatFormat, ) +from torchtune.data._converters import sharegpt_to_llama2_messages from torchtune.data._instruct_templates import ( AlpacaInstructTemplate, GrammarErrorCorrectionTemplate, InstructTemplate, SummarizeTemplate, ) -from torchtune.data._transforms import sharegpt_to_llama2_messages from torchtune.data._types import Message -from torchtune.data._utils import truncate +from torchtune.data._utils import truncate, validate_messages __all__ = [ "AlpacaInstructTemplate", @@ -32,4 +32,5 @@ "sharegpt_to_llama2_messages", "truncate", "Message", + "validate_messages", ] diff --git a/torchtune/data/_transforms.py b/torchtune/data/_converters.py similarity index 100% rename from torchtune/data/_transforms.py rename to torchtune/data/_converters.py diff --git a/torchtune/data/_utils.py b/torchtune/data/_utils.py index 04318d9652..a94bdd029d 100644 --- a/torchtune/data/_utils.py +++ b/torchtune/data/_utils.py @@ -6,6 +6,8 @@ from typing import Any, List +from torchtune.data._types import Message + def truncate( tokens: List[Any], @@ -16,3 +18,49 @@ def truncate( if tokens_truncated[-1] != eos_id: tokens_truncated[-1] = eos_id return tokens_truncated + + +def validate_messages( + messages: List[Message], +) -> None: + """ + Given a list of messages, ensure that messages form a valid + back-and-forth conversation. An error will be raised if: + - There is a system message that's not the first message + - There are two consecutive user messages + - An assistant message comes before the first user message + - The message is empty + - Messages are shorter than length of 2 (min. one user-assistant turn) + + Args: + messages (List[Message]): the messages to validate. + + Raises: + ValueError: If the messages are invalid. + """ + if len(messages) < 2: + raise ValueError( + f"Messages must be at least length 2, but got {len(messages)} messages" + ) + + last_turn = "assistant" + for i, message in enumerate(messages): + if message.role == "assistant" and last_turn != "user": + raise ValueError( + f"Assistant message before expected user message at index {i} in messages" + ) + if message.role == "user" and last_turn == "user": + raise ValueError( + f"Two consecutive user messages at index {i} and {i - 1} in messages" + ) + if message.role == "system" and i > 0: + raise ValueError( + f"System message at index {i} in messages, but system messages must come first" + ) + # Assistant messages can be empty because they will not be tokenized and + # will not contribute to the loss, assuming the entire batch is not full + # of empty assistant messages. The alpaca dataset is an example of the + # output assistant message being empty sometimes. + if not message.content and message.role != "assistant": + raise ValueError(f"Message at index {i} in messages is empty") + last_turn = message.role diff --git a/torchtune/datasets/_chat.py b/torchtune/datasets/_chat.py index 9f5d8e9524..b054b42dab 100644 --- a/torchtune/datasets/_chat.py +++ b/torchtune/datasets/_chat.py @@ -11,7 +11,12 @@ from datasets import load_dataset from torch.utils.data import Dataset from torchtune.config._utils import _get_chat_format -from torchtune.data import ChatFormat, Message, sharegpt_to_llama2_messages +from torchtune.data import ( + ChatFormat, + Message, + sharegpt_to_llama2_messages, + validate_messages, +) from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX from torchtune.modules import Tokenizer @@ -79,6 +84,7 @@ def __getitem__(self, index: int) -> Tuple[List[int], List[int]]: def _prepare_sample(self, sample: Mapping[str, Any]) -> Tuple[List[int], List[int]]: messages = self._convert_to_messages(sample, self.train_on_input) messages = self.chat_format.format(messages) + validate_messages(messages) tokens, mask = self._tokenizer.tokenize_messages( messages, max_seq_len=self.max_seq_len ) diff --git a/torchtune/datasets/_instruct.py b/torchtune/datasets/_instruct.py index 19c61aa7e6..ff5633a3c3 100644 --- a/torchtune/datasets/_instruct.py +++ b/torchtune/datasets/_instruct.py @@ -11,7 +11,7 @@ from torch.utils.data import Dataset from torchtune.config._utils import _get_instruct_template -from torchtune.data import InstructTemplate, Message +from torchtune.data import InstructTemplate, Message, validate_messages from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX from torchtune.modules import Tokenizer @@ -91,6 +91,8 @@ def _prepare_sample(self, sample: Mapping[str, Any]) -> Tuple[List[int], List[in Message(role="assistant", content=transformed_sample[key_output]), ] + validate_messages(messages) + tokens, mask = self._tokenizer.tokenize_messages( messages, max_seq_len=self.max_seq_len ) From 98f82e5cd9dcab52956f6da9197e11da389196bf Mon Sep 17 00:00:00 2001 From: Botao Chen Date: Thu, 4 Apr 2024 13:02:10 -0700 Subject: [PATCH 33/76] [Perf Tools] Torch profiler component (#627) --- .../configs/llama2/7B_lora_single_device.yaml | 7 ++ .../llama2/7B_qlora_single_device.yaml | 7 ++ recipes/lora_finetune_single_device.py | 100 ++++++++++-------- torchtune/utils/__init__.py | 2 + torchtune/utils/_profiler.py | 50 +++++++++ 5 files changed, 123 insertions(+), 43 deletions(-) create mode 100644 torchtune/utils/_profiler.py diff --git a/recipes/configs/llama2/7B_lora_single_device.yaml b/recipes/configs/llama2/7B_lora_single_device.yaml index ec8498b9c7..76bf68e38e 100644 --- a/recipes/configs/llama2/7B_lora_single_device.yaml +++ b/recipes/configs/llama2/7B_lora_single_device.yaml @@ -85,3 +85,10 @@ log_every_n_steps: 1 device: cuda dtype: bf16 enable_activation_checkpointing: True + +# Show case the usage of pytorch profiler +# Set enabled to False as it's only needed for debugging training +profiler: + _component_: torchtune.utils.profiler + enabled: False + output_dir: /tmp/alpaca-llama2-finetune/torchtune_perf_tracing.json diff --git a/recipes/configs/llama2/7B_qlora_single_device.yaml b/recipes/configs/llama2/7B_qlora_single_device.yaml index 26510f1642..644541b1bf 100644 --- a/recipes/configs/llama2/7B_qlora_single_device.yaml +++ b/recipes/configs/llama2/7B_qlora_single_device.yaml @@ -81,3 +81,10 @@ log_every_n_steps: 1 device: cuda dtype: bf16 enable_activation_checkpointing: True + +# Show case the usage of pytorch profiler +# Set enabled to False as it's only needed for debugging training +profiler: + _component_: torchtune.utils.profiler + enabled: False + output_dir: /tmp/alpaca-llama2-finetune/torchtune_perf_tracing.json diff --git a/recipes/lora_finetune_single_device.py b/recipes/lora_finetune_single_device.py index 5f09bf8f13..c37f92d866 100644 --- a/recipes/lora_finetune_single_device.py +++ b/recipes/lora_finetune_single_device.py @@ -207,6 +207,9 @@ def setup(self, cfg: DictConfig) -> None: last_epoch=self.total_training_steps - 1, ) + self._profiler_enabled = cfg.profiler.enabled + self._profiler = config.instantiate(cfg.profiler) + def _setup_model( self, cfg_model: DictConfig, @@ -380,49 +383,60 @@ def train(self) -> None: # Update the sampler to ensure data is correctly shuffled across epochs # in case shuffle is True self._sampler.set_epoch(curr_epoch) - for idx, batch in enumerate(pbar := tqdm(self._dataloader)): - if ( - self.max_steps_per_epoch is not None - and (idx // self._gradient_accumulation_steps) - == self.max_steps_per_epoch - ): - break - - input_ids, labels = batch - input_ids = input_ids.to(self._device) - labels = labels.to(self._device) - - logits = self._model(input_ids) - # Shift so that tokens < n predict n - logits = logits[..., :-1, :].contiguous() - labels = labels[..., 1:].contiguous() - logits = logits.transpose(1, 2) - # Compute loss - loss = self._loss_fn(logits, labels) - - if self.total_training_steps % self._log_every_n_steps == 0: - pbar.set_description(f"{curr_epoch+1}|{idx+1}|Loss: {loss.item()}") - self._metric_logger.log_dict( - { - "loss": loss.item(), - "lr": self._optimizer.param_groups[0]["lr"], - "gpu_resources": torch.cuda.memory_allocated(), - }, - step=self.total_training_steps, # Each step is unique, not limited to each epoch - ) - loss = loss / self._gradient_accumulation_steps - loss.backward() - if (idx + 1) % self._gradient_accumulation_steps == 0: - self._optimizer.step() - self._optimizer.zero_grad(set_to_none=True) - self._lr_scheduler.step() - # Update the number of steps when the weights are updated - self.total_training_steps += 1 - # Log peak memory for iteration - if self.total_training_steps % self._log_peak_memory_every_n_steps == 0: - log.info( - utils.memory_stats_log("Memory Stats:", device=self._device) - ) + + # Optionally profile the training loop + with self._profiler: + for idx, batch in enumerate(pbar := tqdm(self._dataloader)): + if ( + self.max_steps_per_epoch is not None + and (idx // self._gradient_accumulation_steps) + == self.max_steps_per_epoch + ): + break + + if self._profiler_enabled: + self._profiler.step() + + input_ids, labels = batch + input_ids = input_ids.to(self._device) + labels = labels.to(self._device) + + logits = self._model(input_ids) + # Shift so that tokens < n predict n + logits = logits[..., :-1, :].contiguous() + labels = labels[..., 1:].contiguous() + logits = logits.transpose(1, 2) + # Compute loss + loss = self._loss_fn(logits, labels) + + if self.total_training_steps % self._log_every_n_steps == 0: + pbar.set_description( + f"{curr_epoch+1}|{idx+1}|Loss: {loss.item()}" + ) + self._metric_logger.log_dict( + { + "loss": loss.item(), + "lr": self._optimizer.param_groups[0]["lr"], + "gpu_resources": torch.cuda.memory_allocated(), + }, + step=self.total_training_steps, # Each step is unique, not limited to each epoch + ) + loss = loss / self._gradient_accumulation_steps + loss.backward() + if (idx + 1) % self._gradient_accumulation_steps == 0: + self._optimizer.step() + self._optimizer.zero_grad(set_to_none=True) + self._lr_scheduler.step() + # Update the number of steps when the weights are updated + self.total_training_steps += 1 + # Log peak memory for iteration + if ( + self.total_training_steps % self._log_peak_memory_every_n_steps + == 0 + ): + log.info( + utils.memory_stats_log("Memory Stats:", device=self._device) + ) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/torchtune/utils/__init__.py b/torchtune/utils/__init__.py index 666711a34c..c05494b54e 100644 --- a/torchtune/utils/__init__.py +++ b/torchtune/utils/__init__.py @@ -23,6 +23,7 @@ wrap_fsdp, ) from ._generation import generate # noqa +from ._profiler import profiler from .argparse import TuneRecipeArgumentParser from .checkpointable_dataloader import CheckpointableDataLoader from .collate import padded_collate @@ -80,4 +81,5 @@ "OptimizerInBackwardWrapper", "create_optim_in_bwd_wrapper", "register_optim_in_bwd_hooks", + "profiler", ] diff --git a/torchtune/utils/_profiler.py b/torchtune/utils/_profiler.py new file mode 100644 index 0000000000..429aa74ce0 --- /dev/null +++ b/torchtune/utils/_profiler.py @@ -0,0 +1,50 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import contextlib + +from typing import ContextManager, Optional + +import torch +from torch.profiler import profile + + +def profiler( + enabled: Optional[bool] = False, + output_dir: Optional[str] = "./torchtune_perf_tracing.json", +) -> ContextManager: + """ + Utility component that wraps around `torch.profiler` to profile model's operators. + See https://pytorch.org/docs/stable/profiler.html for more details. + The schedule for this profiler is wait 100 steps, warmup 5 steps, trace 5 steps + Note: Enabling pytorch profiler may have training speed reduction. + + Args: + enabled (Optional[bool]): Enable pytorch profiler. Default is False. + output_dir (Optional[str]): Tracing file output path. Default is "./torchtune_perf_tracing.json". + + Returns: + ContextManager: pytorch profiler context manager + """ + + def trace_handler(prof) -> None: + prof.export_chrome_trace(output_dir) + + return ( + profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + schedule=torch.profiler.schedule(wait=100, warmup=5, active=5, repeat=1), + on_trace_ready=trace_handler, + record_shapes=True, + profile_memory=False, + with_stack=False, + ) + if enabled + else contextlib.nullcontext() + ) From e97720a72d946c9b8b0f2737a2ac299b7417fa87 Mon Sep 17 00:00:00 2001 From: Jerry Zhang Date: Thu, 4 Apr 2024 19:58:59 -0700 Subject: [PATCH 34/76] Adding quantization support in torchtune (#632) --- recipes/README.md | 115 +++++++++++++++++++++++++++++ recipes/configs/eleuther_eval.yaml | 3 + recipes/configs/generate.yaml | 2 + recipes/configs/quantize.yaml | 52 +++++++++++++ recipes/eleuther_eval.py | 21 ++++-- recipes/generate.py | 21 ++++-- recipes/quantize.py | 92 +++++++++++++++++++++++ requirements.txt | 2 +- torchtune/_recipe_registry.py | 8 ++ torchtune/utils/__init__.py | 2 + torchtune/utils/quantization.py | 45 +++++++++++ 11 files changed, 349 insertions(+), 14 deletions(-) create mode 100644 recipes/configs/quantize.yaml create mode 100644 recipes/quantize.py create mode 100644 torchtune/utils/quantization.py diff --git a/recipes/README.md b/recipes/README.md index 53b0a2ffdb..8652cda175 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -111,3 +111,118 @@ python3 convert.py /tmp/llama2/meta_model_0.pt --ctx 4096 ``` This will output a gguf file in the same precision which can be used for running inference. + +### Architecture Optimization + +TorchTune integrates with `torchao`(https://github.com/pytorch-labs/ao/) for architecture optimization techniques including quantization and sparsity. Currently only some quantization techniques are integrated, see `receipes/configs/quantize.yaml` for more details. + +#### Quantize +To quantize a model (default is int4 weight only quantization): +``` +tune run quantize --config quantize +``` + +#### Eval +To evaluate a quantized model, add the following to `receipes/configs/eleuther_eval.yaml`: + + +``` +# make sure to change the checkpointer component +checkpointer: + _component_: torchtune.utils.FullModelTorchTuneCheckpointer + +# Quantization specific args +quantizer: + _component_: torchtune.utils.quantization.Int4WeightOnlyQuantizer + groupsize: 256 +``` + +and run the eval command: +``` +tune run eleuther_eval --config eleuther_eval +``` + +#### Generate +Changes in `receipes/configs/generate.yaml` +``` +# Model arguments +checkpointer: +# make sure to change the checkpointer component +checkpointer: + _component_: torchtune.utils.FullModelTorchTuneCheckpointer + checkpoint_files: [meta_model_0.4w.pt] + +# Quantization Arguments +quantizer: + _component_: torchtune.utils.quantization.Int4WeightOnlyQuantizer + groupsize: 256 +``` + +and run generate command: +``` +tune run generate --config generate +``` + +#### GPTQ + +GPTQ is an algorithm to improve the accuracy of quantized model through optimizing the loss of (activation * weight) together, here are the changes that's needed to use it for int4 weight only quantization + +`receipes/configs/quantize.yaml` + +We'll publish doc pages for different quantizers in torchao a bit later. Please check `receipes/configs/quantized.yaml for how to use them for now. + +``` +quantizer: + _component_: torchtune.utils.quantization.Int4WeightOnlyGPTQQuantizer + blocksize: 128 + percdamp: 0.01 + groupsize: 256 + +tokenizer: + _component_: torchtune.models.llama2.llama2_tokenizer + path: /tmp/llama2/tokenizer.model +``` + +`receipes/quantize.py` + +``` +def quantize(self, cfg: DictConfig): + from torchao.quantization.GPTQ import InputRecorder + tokenizer = config.instantiate(cfg.tokenizer) + calibration_seq_length = 100 + calibration_tasks = ['wikitext'] + inputs = InputRecorder( + tokenizer, + calibration_seq_length, + vocab_size=self._model.tok_embeddings.weight.shape[0], + device="cpu", + ).record_inputs( + calibration_tasks, + 5, # calibration_limit + ).get_inputs() + t0 = time.perf_counter() + self._model = self._quantizer.quantize(self._model, inputs) + .... +``` + +Run quantize +``` +tune run quantize --config quantize +``` + +`recipes/eleuther_eval.py` + +``` +# To skip running the full GPTQ quantization process that typically takes a long time, +# change model = quantizer.quantize(model) to: +model = quantizer._convert_for_runtime(model) +``` + +`recipes/configs/eleuther_eval.yaml` +``` +quantizer: + _component_: torchtune.utils.quantization.Int4WeightOnlyGPTQQuantizer + blocksize: 128 + percdamp: 0.01 + groupsize: 256 +``` diff --git a/recipes/configs/eleuther_eval.yaml b/recipes/configs/eleuther_eval.yaml index e0e70f2556..748e812810 100644 --- a/recipes/configs/eleuther_eval.yaml +++ b/recipes/configs/eleuther_eval.yaml @@ -28,3 +28,6 @@ seed: 217 tasks: ["truthfulqa_mc2"] limit: null max_seq_length: 4096 + +# Quantization specific args +quantizer: null diff --git a/recipes/configs/generate.yaml b/recipes/configs/generate.yaml index 2865f33d42..dffec77852 100644 --- a/recipes/configs/generate.yaml +++ b/recipes/configs/generate.yaml @@ -29,3 +29,5 @@ prompt: "Hello, my name is" max_new_tokens: 300 temperature: 0.8 top_k: 300 + +quantizer: null diff --git a/recipes/configs/quantize.yaml b/recipes/configs/quantize.yaml new file mode 100644 index 0000000000..ea11cdd72b --- /dev/null +++ b/recipes/configs/quantize.yaml @@ -0,0 +1,52 @@ +# Config for QuantizationRecipe in quantize.py +# +# To launch, run the following command from root torchtune directory: +# tune quantize --config quantize +# +# Supported quantization modes are: +# 8w: +# torchtune.utils.quantization.Int8WeightOnlyQuantizer +# int8 weight only per axis group quantization +# +# 4w: +# torchtune.utils.quantization.Int4WeightOnlyQuantizer +# int4 weight only per axis group quantization +# Args: +# `groupsize` (int): a parameter of int4 weight only quantization, +# it refers to the size of quantization groups which get independent quantization parameters +# e.g. 32, 64, 128, 256, smaller numbers means more fine grained and higher accuracy +# +# 4w-gptq: +# torchtune.utils.quantization.Int4WeightOnlyGPTQQuantizer +# int4 weight only per axis group quantization with GPTQ +# Args: +# `groupsize`: see description in `4w` +# `blocksize`: GPTQ is applied to a 'block' of columns at a time, +# larger blocks trade off memory for perf, recommended to be a constant +# multiple of groupsize. +# `percdamp`: GPTQ stablization hyperparameter, recommended to be .01 +# +# future note: blocksize and percdamp should not have to be 'known' by users by default. +# Similar to momentum constant in MovingAverageObserver, it can be tuned, +# but 99% of users don't need to pay attention to it. blocksize should probably be set at +# max(`groupsize`, 128) and percdamp at .01 + +# +# Model arguments +model: + _component_: torchtune.models.llama2.llama2_7b + +checkpointer: + _component_: torchtune.utils.FullModelMetaCheckpointer + checkpoint_dir: /tmp/llama2/ + checkpoint_files: [meta_model_0.pt] + output_dir: /tmp/llama2/ + model_type: LLAMA2 + +device: cuda +dtype: bf16 +seed: 1234 + +quantizer: + _component_: torchtune.utils.quantization.Int4WeightOnlyQuantizer + groupsize: 256 diff --git a/recipes/eleuther_eval.py b/recipes/eleuther_eval.py index a08f5017a7..786c5856df 100644 --- a/recipes/eleuther_eval.py +++ b/recipes/eleuther_eval.py @@ -124,20 +124,25 @@ class EleutherEvalRecipe(EvalRecipeInterface): def __init__(self, cfg: DictConfig) -> None: self._cfg = cfg - def load_checkpoint(self, checkpointer_cfg: DictConfig) -> Dict[str, Any]: - checkpointer = config.instantiate(checkpointer_cfg) - checkpoint_dict = checkpointer.load_checkpoint() - return checkpoint_dict - def setup(self) -> None: self._device = utils.get_device(device=self._cfg.device) self._dtype = utils.get_dtype(dtype=self._cfg.dtype) self._limit = self._cfg.limit self._tasks = list(self._cfg.tasks) + self._quantizer = config.instantiate(self._cfg.quantizer) + self._quantization_mode = utils.get_quantizer_mode(self._quantizer) utils.set_seed(seed=self._cfg.seed) - ckpt_dict = self.load_checkpoint(self._cfg.checkpointer) + checkpointer = config.instantiate(self._cfg.checkpointer) + if self._quantization_mode is None: + ckpt_dict = checkpointer.load_checkpoint() + else: + # weights_only needs to be False when loading a quantized model + # currently loading a quantized model is only supported with the + # FullModelTorchTuneCheckpointer + ckpt_dict = checkpointer.load_checkpoint(weights_only=False) + self._model = self._setup_model( model_cfg=self._cfg.model, model_state_dict=ckpt_dict[utils.MODEL_KEY], @@ -152,9 +157,11 @@ def _setup_model( ) -> nn.Module: with utils.set_default_dtype(self._dtype), self._device: model = config.instantiate(model_cfg) + if self._quantization_mode is not None: + model = self._quantizer.quantize(model) + model = model.to(device=self._device, dtype=self._dtype) model.load_state_dict(model_state_dict) - # Validate model was loaded in with the expected dtype. utils.validate_expected_param_dtype(model.named_parameters(), dtype=self._dtype) logger.info(f"Model is initialized with precision {self._dtype}.") diff --git a/recipes/generate.py b/recipes/generate.py index 8a3043229d..43db6a3147 100644 --- a/recipes/generate.py +++ b/recipes/generate.py @@ -28,16 +28,21 @@ class InferenceRecipe: def __init__(self, cfg: DictConfig) -> None: self._device = utils.get_device(device=cfg.device) self._dtype = utils.get_dtype(dtype=cfg.dtype) + self._quantizer = config.instantiate(cfg.quantizer) + self._quantization_mode = utils.get_quantizer_mode(self._quantizer) utils.set_seed(seed=cfg.seed) - def load_checkpoint(self, checkpointer_cfg: DictConfig) -> Dict[str, Any]: - checkpointer = config.instantiate(checkpointer_cfg) - checkpoint_dict = checkpointer.load_checkpoint() - return checkpoint_dict - def setup(self, cfg: DictConfig) -> None: - ckpt_dict = self.load_checkpoint(cfg.checkpointer) + checkpointer = config.instantiate(cfg.checkpointer) + if self._quantization_mode is None: + ckpt_dict = checkpointer.load_checkpoint() + else: + # weights_only needs to be False when loading a quantized model + # currently loading a quantized model is only supported with the + # FullModelTorchTuneCheckpointer + ckpt_dict = checkpointer.load_checkpoint(weights_only=False) + self._model = self._setup_model( model_cfg=cfg.model, model_state_dict=ckpt_dict[utils.MODEL_KEY], @@ -52,6 +57,10 @@ def _setup_model( with utils.set_default_dtype(self._dtype), self._device: model = config.instantiate(model_cfg) + if self._quantization_mode is not None: + model = self._quantizer.quantize(model) + model = model.to(device=self._device, dtype=self._dtype) + model.load_state_dict(model_state_dict) # Validate model was loaded in with the expected dtype. diff --git a/recipes/quantize.py b/recipes/quantize.py new file mode 100644 index 0000000000..88eb45bb4a --- /dev/null +++ b/recipes/quantize.py @@ -0,0 +1,92 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +import sys +import time +from typing import Any, Dict + +import torch +from omegaconf import DictConfig + +from torch import nn + +from torchtune import config, utils + +logger = utils.get_logger("DEBUG") + + +class QuantizationRecipe: + """ + Recipe for quantizing a Transformer-based LLM. + Uses quantizer classes from torchao to quantize a model. + Please refer to `receipes/configs/quantize.yaml` for supported quantizers and how to use them. + """ + + def __init__(self, cfg: DictConfig) -> None: + self._device = utils.get_device(device=cfg.device) + self._dtype = utils.get_dtype(dtype=cfg.dtype) + self._quantizer = config.instantiate(cfg.quantizer) + self._quantization_mode = utils.get_quantizer_mode(self._quantizer) + utils.set_seed(seed=cfg.seed) + + def load_checkpoint(self, checkpointer_cfg: DictConfig) -> Dict[str, Any]: + self._checkpointer = config.instantiate(checkpointer_cfg) + checkpoint_dict = self._checkpointer.load_checkpoint() + return checkpoint_dict + + def setup(self, cfg: DictConfig) -> None: + ckpt_dict = self.load_checkpoint(cfg.checkpointer) + self._model = self._setup_model( + model_cfg=cfg.model, + model_state_dict=ckpt_dict[utils.MODEL_KEY], + ) + + def _setup_model( + self, + model_cfg: DictConfig, + model_state_dict: Dict[str, Any], + ) -> nn.Module: + with utils.set_default_dtype(self._dtype), self._device: + model = config.instantiate(model_cfg) + + model.load_state_dict(model_state_dict, assign=True) + + # Validate model was loaded in with the expected dtype. + utils.validate_expected_param_dtype(model.named_parameters(), dtype=self._dtype) + logger.info(f"Model is initialized with precision {self._dtype}.") + return model + + @torch.no_grad() + def quantize(self, cfg: DictConfig): + t0 = time.perf_counter() + self._model = self._quantizer.quantize(self._model) + t = time.perf_counter() - t0 + logger.info(f"Time for quantization: {t:.02f} sec") + logger.info(f"Memory used: {torch.cuda.max_memory_allocated() / 1e9:.02f} GB") + + def save_checkpoint(self, cfg: DictConfig): + ckpt_dict = self._model.state_dict() + file_name = cfg.checkpointer.checkpoint_files[0].split(".")[0] + quantized_file_name = ( + cfg.checkpointer.output_dir + + file_name + + "." + + self._quantization_mode + + ".pt" + ) + torch.save(ckpt_dict, quantized_file_name) + logger.info(f"Saved quantized model to {quantized_file_name}") + + +@config.parse +def main(cfg: DictConfig) -> None: + recipe = QuantizationRecipe(cfg=cfg) + recipe.setup(cfg=cfg) + recipe.quantize(cfg=cfg) + recipe.save_checkpoint(cfg=cfg) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/requirements.txt b/requirements.txt index 74b3d34d33..9c37618f2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ tqdm omegaconf # Quantization -torchao-nightly==2024.3.29 +torchao==0.1 diff --git a/torchtune/_recipe_registry.py b/torchtune/_recipe_registry.py index 5a07cb1f78..8b46dbdd43 100644 --- a/torchtune/_recipe_registry.py +++ b/torchtune/_recipe_registry.py @@ -105,6 +105,14 @@ class Recipe: ], supports_distributed=True, ), + Recipe( + name="quantize", + file_path="quantize.py", + configs=[ + Config(name="quantize", file_path="quantize.yaml"), + ], + supports_distributed=False, + ), ] diff --git a/torchtune/utils/__init__.py b/torchtune/utils/__init__.py index c05494b54e..27365deb56 100644 --- a/torchtune/utils/__init__.py +++ b/torchtune/utils/__init__.py @@ -53,6 +53,7 @@ set_default_dtype, validate_expected_param_dtype, ) +from .quantization import get_quantizer_mode from .seed import set_seed __all__ = [ @@ -82,4 +83,5 @@ "create_optim_in_bwd_wrapper", "register_optim_in_bwd_hooks", "profiler", + "get_quantizer_mode", ] diff --git a/torchtune/utils/quantization.py b/torchtune/utils/quantization.py new file mode 100644 index 0000000000..9d4d8b26e8 --- /dev/null +++ b/torchtune/utils/quantization.py @@ -0,0 +1,45 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Any, Callable, Optional + +import torch +from torchao.quantization.quant_api import ( + apply_weight_only_int8_quant, + Int4WeightOnlyGPTQQuantizer, + Int4WeightOnlyQuantizer, + Quantizer, +) + +__all__ = [ + "Int4WeightOnlyQuantizer", + "Int4WeightOnlyGPTQQuantizer", + "Int8WeightOnlyQuantizer", + "get_quantizer_mode", +] + + +class Int8WeightOnlyQuantizer(Quantizer): + def quantize( + self, model: torch.nn.Module, *args: Any, **kwargs: Any + ) -> torch.nn.Module: + apply_weight_only_int8_quant(model) + return model + + +_quantizer_to_mode = { + Int4WeightOnlyQuantizer: "4w", + Int8WeightOnlyQuantizer: "8w", + Int4WeightOnlyGPTQQuantizer: "4w-gptq", +} + + +def get_quantizer_mode(quantizer: Optional[Callable]) -> Optional[str]: + """Given a quantizer object, returns a string that specifies the type of quantization e.g. + 4w, which means int4 weight only quantization. + If the quantizer is not recognized as a known quantizer, we'll return None + """ + return _quantizer_to_mode.get(type(quantizer), None) From 1162295107d5f0ebc30e9b1e43f853d9467689ab Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 5 Apr 2024 12:15:44 +0000 Subject: [PATCH 35/76] tiny llama --- dev.ipynb | 314 +++++++++++++++++++++ recipes/configs/llama2/tiny_llama.yaml | 87 ++++++ recipes/lora_finetune_single_device.py | 4 +- torchtune/models/llama2/__init__.py | 2 + torchtune/models/llama2/_model_builders.py | 22 ++ 5 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 dev.ipynb create mode 100644 recipes/configs/llama2/tiny_llama.yaml diff --git a/dev.ipynb b/dev.ipynb new file mode 100644 index 0000000000..88b9d36922 --- /dev/null +++ b/dev.ipynb @@ -0,0 +1,314 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from torchtune.models.llama2 import llama2\n", + "from torchtune.modules import TransformerDecoder\n", + "from torchtune.utils.checkpoint import save_checkpoint, load_checkpoint" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def llama2_tiny(max_batch_size = None) -> TransformerDecoder:\n", + " \"\"\"\n", + " Builder for creating a Llama2 model initialized w/ the default 7b parameter values\n", + " from https://arxiv.org/abs/2307.09288\n", + "\n", + " Args:\n", + " max_batch_size (Optional[int]): Maximum batch size to be passed to KVCache.\n", + "\n", + " Returns:\n", + " TransformerDecoder: Instantiation of Llama2 7B model\n", + " \"\"\"\n", + " return llama2(\n", + " vocab_size=32_000,\n", + " num_layers=2,\n", + " num_heads=4,\n", + " num_kv_heads=4,\n", + " embed_dim=128,\n", + " max_seq_len=4096,\n", + " max_batch_size=max_batch_size,\n", + " attn_dropout=0.0,\n", + " norm_eps=1e-5,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "m = llama2_tiny()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "chpt_dict = {\"model\": m}\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "torch.save(m.state_dict(), \"tiny_llama/model.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "save_checkpoint(chpt_dict, 'tiny_llama/model.pth')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'model': OrderedDict([('tok_embeddings.weight',\n", + " tensor([[ 0.6721, -0.0063, 0.2206, ..., -0.4694, 1.8257, 2.1110],\n", + " [-0.3021, 0.1397, 0.0167, ..., 1.3691, -0.1057, 0.4563],\n", + " [ 0.2236, 2.0759, 3.3434, ..., 0.3655, 0.8593, 0.4055],\n", + " ...,\n", + " [ 2.0139, -0.4313, -0.6845, ..., -0.1600, 0.2825, -0.6131],\n", + " [ 0.1583, 0.9916, 1.0918, ..., -2.1604, -1.0594, 0.0648],\n", + " [-1.4329, -0.8049, -0.5369, ..., -1.2593, 2.4386, 1.2828]])),\n", + " ('layers.0.sa_norm.scale',\n", + " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1.])),\n", + " ('layers.0.attn.q_proj.weight',\n", + " tensor([[ 0.0280, 0.0101, -0.0457, ..., -0.0786, 0.0836, -0.0700],\n", + " [-0.0022, 0.0369, -0.0879, ..., -0.0854, 0.0129, -0.0333],\n", + " [-0.0572, -0.0465, -0.0008, ..., -0.0680, 0.0245, 0.0512],\n", + " ...,\n", + " [ 0.0769, 0.0790, -0.0056, ..., -0.0442, 0.0624, -0.0328],\n", + " [ 0.0110, 0.0400, -0.0696, ..., -0.0524, 0.0883, 0.0866],\n", + " [-0.0589, -0.0402, -0.0066, ..., 0.0207, 0.0333, 0.0478]])),\n", + " ('layers.0.attn.k_proj.weight',\n", + " tensor([[-0.0351, -0.0071, 0.0104, ..., -0.0403, 0.0620, 0.0435],\n", + " [-0.0071, 0.0844, 0.0140, ..., 0.0817, -0.0595, 0.0528],\n", + " [ 0.0559, 0.0310, 0.0526, ..., 0.0125, 0.0205, 0.0430],\n", + " ...,\n", + " [ 0.0870, -0.0838, 0.0671, ..., -0.0014, -0.0662, -0.0745],\n", + " [-0.0127, 0.0005, -0.0680, ..., -0.0583, -0.0266, -0.0288],\n", + " [-0.0371, 0.0380, 0.0256, ..., -0.0173, 0.0089, 0.0115]])),\n", + " ('layers.0.attn.v_proj.weight',\n", + " tensor([[ 0.0030, -0.0682, -0.0049, ..., 0.0430, -0.0171, 0.0023],\n", + " [-0.0876, -0.0240, -0.0790, ..., 0.0093, 0.0397, 0.0405],\n", + " [-0.0364, 0.0719, -0.0348, ..., 0.0146, -0.0119, -0.0518],\n", + " ...,\n", + " [ 0.0278, 0.0330, -0.0836, ..., -0.0189, 0.0710, -0.0296],\n", + " [-0.0140, -0.0416, 0.0520, ..., -0.0156, 0.0559, 0.0683],\n", + " [ 0.0519, 0.0505, 0.0572, ..., -0.0004, -0.0767, 0.0242]])),\n", + " ('layers.0.attn.output_proj.weight',\n", + " tensor([[ 0.0324, 0.0520, -0.0132, ..., 0.0285, -0.0848, 0.0720],\n", + " [ 0.0236, -0.0741, -0.0754, ..., 0.0153, 0.0187, 0.0066],\n", + " [ 0.0456, 0.0669, 0.0566, ..., -0.0149, 0.0764, 0.0644],\n", + " ...,\n", + " [ 0.0577, 0.0264, -0.0095, ..., 0.0436, 0.0627, 0.0722],\n", + " [ 0.0871, 0.0220, -0.0865, ..., -0.0504, -0.0218, -0.0323],\n", + " [ 0.0582, 0.0574, -0.0780, ..., 0.0021, -0.0865, 0.0793]])),\n", + " ('layers.0.mlp_norm.scale',\n", + " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1.])),\n", + " ('layers.0.mlp.w1.weight',\n", + " tensor([[-0.0878, -0.0187, 0.0862, ..., -0.0095, -0.0032, -0.0611],\n", + " [-0.0174, -0.0744, -0.0868, ..., 0.0549, 0.0327, 0.0868],\n", + " [-0.0107, 0.0538, -0.0477, ..., 0.0769, 0.0007, -0.0623],\n", + " ...,\n", + " [ 0.0022, 0.0863, 0.0487, ..., 0.0201, -0.0106, 0.0311],\n", + " [ 0.0260, -0.0131, -0.0065, ..., -0.0812, 0.0281, 0.0416],\n", + " [-0.0521, 0.0364, -0.0264, ..., -0.0464, -0.0031, 0.0752]])),\n", + " ('layers.0.mlp.w2.weight',\n", + " tensor([[ 4.4035e-02, -1.5159e-02, 1.4231e-02, ..., -3.8606e-02,\n", + " 2.3046e-02, -4.1586e-03],\n", + " [-2.4230e-02, -1.7738e-02, -3.4205e-02, ..., -5.8932e-03,\n", + " 2.8201e-02, -1.3607e-02],\n", + " [ 9.5524e-03, -2.8478e-02, -1.6222e-02, ..., 1.5925e-03,\n", + " 4.3839e-02, 4.0368e-02],\n", + " ...,\n", + " [ 3.2933e-02, 1.2597e-05, -1.7801e-02, ..., -3.5409e-02,\n", + " -1.4612e-02, -4.5030e-03],\n", + " [ 1.1489e-02, -3.2748e-02, 4.1240e-02, ..., 2.2262e-02,\n", + " 4.2146e-02, 1.0142e-02],\n", + " [ 1.6441e-02, 3.9053e-02, -5.2180e-03, ..., -1.7018e-02,\n", + " -2.9996e-02, -4.1561e-02]])),\n", + " ('layers.0.mlp.w3.weight',\n", + " tensor([[-0.0253, 0.0373, 0.0508, ..., 0.0006, 0.0654, -0.0867],\n", + " [-0.0258, 0.0112, 0.0398, ..., 0.0495, -0.0066, -0.0329],\n", + " [ 0.0832, 0.0729, 0.0340, ..., -0.0801, -0.0719, 0.0561],\n", + " ...,\n", + " [-0.0681, -0.0745, 0.0475, ..., 0.0271, -0.0758, 0.0660],\n", + " [ 0.0268, -0.0854, 0.0629, ..., -0.0278, -0.0001, 0.0677],\n", + " [ 0.0325, -0.0836, 0.0125, ..., 0.0498, 0.0376, 0.0281]])),\n", + " ('layers.1.sa_norm.scale',\n", + " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1.])),\n", + " ('layers.1.attn.q_proj.weight',\n", + " tensor([[ 0.0280, 0.0101, -0.0457, ..., -0.0786, 0.0836, -0.0700],\n", + " [-0.0022, 0.0369, -0.0879, ..., -0.0854, 0.0129, -0.0333],\n", + " [-0.0572, -0.0465, -0.0008, ..., -0.0680, 0.0245, 0.0512],\n", + " ...,\n", + " [ 0.0769, 0.0790, -0.0056, ..., -0.0442, 0.0624, -0.0328],\n", + " [ 0.0110, 0.0400, -0.0696, ..., -0.0524, 0.0883, 0.0866],\n", + " [-0.0589, -0.0402, -0.0066, ..., 0.0207, 0.0333, 0.0478]])),\n", + " ('layers.1.attn.k_proj.weight',\n", + " tensor([[-0.0351, -0.0071, 0.0104, ..., -0.0403, 0.0620, 0.0435],\n", + " [-0.0071, 0.0844, 0.0140, ..., 0.0817, -0.0595, 0.0528],\n", + " [ 0.0559, 0.0310, 0.0526, ..., 0.0125, 0.0205, 0.0430],\n", + " ...,\n", + " [ 0.0870, -0.0838, 0.0671, ..., -0.0014, -0.0662, -0.0745],\n", + " [-0.0127, 0.0005, -0.0680, ..., -0.0583, -0.0266, -0.0288],\n", + " [-0.0371, 0.0380, 0.0256, ..., -0.0173, 0.0089, 0.0115]])),\n", + " ('layers.1.attn.v_proj.weight',\n", + " tensor([[ 0.0030, -0.0682, -0.0049, ..., 0.0430, -0.0171, 0.0023],\n", + " [-0.0876, -0.0240, -0.0790, ..., 0.0093, 0.0397, 0.0405],\n", + " [-0.0364, 0.0719, -0.0348, ..., 0.0146, -0.0119, -0.0518],\n", + " ...,\n", + " [ 0.0278, 0.0330, -0.0836, ..., -0.0189, 0.0710, -0.0296],\n", + " [-0.0140, -0.0416, 0.0520, ..., -0.0156, 0.0559, 0.0683],\n", + " [ 0.0519, 0.0505, 0.0572, ..., -0.0004, -0.0767, 0.0242]])),\n", + " ('layers.1.attn.output_proj.weight',\n", + " tensor([[ 0.0324, 0.0520, -0.0132, ..., 0.0285, -0.0848, 0.0720],\n", + " [ 0.0236, -0.0741, -0.0754, ..., 0.0153, 0.0187, 0.0066],\n", + " [ 0.0456, 0.0669, 0.0566, ..., -0.0149, 0.0764, 0.0644],\n", + " ...,\n", + " [ 0.0577, 0.0264, -0.0095, ..., 0.0436, 0.0627, 0.0722],\n", + " [ 0.0871, 0.0220, -0.0865, ..., -0.0504, -0.0218, -0.0323],\n", + " [ 0.0582, 0.0574, -0.0780, ..., 0.0021, -0.0865, 0.0793]])),\n", + " ('layers.1.mlp_norm.scale',\n", + " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1.])),\n", + " ('layers.1.mlp.w1.weight',\n", + " tensor([[-0.0878, -0.0187, 0.0862, ..., -0.0095, -0.0032, -0.0611],\n", + " [-0.0174, -0.0744, -0.0868, ..., 0.0549, 0.0327, 0.0868],\n", + " [-0.0107, 0.0538, -0.0477, ..., 0.0769, 0.0007, -0.0623],\n", + " ...,\n", + " [ 0.0022, 0.0863, 0.0487, ..., 0.0201, -0.0106, 0.0311],\n", + " [ 0.0260, -0.0131, -0.0065, ..., -0.0812, 0.0281, 0.0416],\n", + " [-0.0521, 0.0364, -0.0264, ..., -0.0464, -0.0031, 0.0752]])),\n", + " ('layers.1.mlp.w2.weight',\n", + " tensor([[ 4.4035e-02, -1.5159e-02, 1.4231e-02, ..., -3.8606e-02,\n", + " 2.3046e-02, -4.1586e-03],\n", + " [-2.4230e-02, -1.7738e-02, -3.4205e-02, ..., -5.8932e-03,\n", + " 2.8201e-02, -1.3607e-02],\n", + " [ 9.5524e-03, -2.8478e-02, -1.6222e-02, ..., 1.5925e-03,\n", + " 4.3839e-02, 4.0368e-02],\n", + " ...,\n", + " [ 3.2933e-02, 1.2597e-05, -1.7801e-02, ..., -3.5409e-02,\n", + " -1.4612e-02, -4.5030e-03],\n", + " [ 1.1489e-02, -3.2748e-02, 4.1240e-02, ..., 2.2262e-02,\n", + " 4.2146e-02, 1.0142e-02],\n", + " [ 1.6441e-02, 3.9053e-02, -5.2180e-03, ..., -1.7018e-02,\n", + " -2.9996e-02, -4.1561e-02]])),\n", + " ('layers.1.mlp.w3.weight',\n", + " tensor([[-0.0253, 0.0373, 0.0508, ..., 0.0006, 0.0654, -0.0867],\n", + " [-0.0258, 0.0112, 0.0398, ..., 0.0495, -0.0066, -0.0329],\n", + " [ 0.0832, 0.0729, 0.0340, ..., -0.0801, -0.0719, 0.0561],\n", + " ...,\n", + " [-0.0681, -0.0745, 0.0475, ..., 0.0271, -0.0758, 0.0660],\n", + " [ 0.0268, -0.0854, 0.0629, ..., -0.0278, -0.0001, 0.0677],\n", + " [ 0.0325, -0.0836, 0.0125, ..., 0.0498, 0.0376, 0.0281]])),\n", + " ('norm.scale',\n", + " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", + " 1., 1.])),\n", + " ('output.weight',\n", + " tensor([[ 0.0388, 0.0253, -0.0010, ..., 0.0023, -0.0367, -0.0049],\n", + " [-0.0090, 0.0756, 0.0523, ..., 0.0191, -0.0657, 0.0881],\n", + " [ 0.0492, -0.0446, 0.0090, ..., 0.0346, 0.0222, -0.0203],\n", + " ...,\n", + " [-0.0378, -0.0467, 0.0259, ..., 0.0030, 0.0568, 0.0595],\n", + " [ 0.0258, -0.0561, -0.0195, ..., -0.0867, -0.0714, -0.0340],\n", + " [-0.0815, 0.0689, -0.0411, ..., -0.0726, -0.0856, 0.0440]]))])}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "load_checkpoint('tiny_llama/model.pth', m)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/recipes/configs/llama2/tiny_llama.yaml b/recipes/configs/llama2/tiny_llama.yaml new file mode 100644 index 0000000000..d7d339d115 --- /dev/null +++ b/recipes/configs/llama2/tiny_llama.yaml @@ -0,0 +1,87 @@ +# Config for multi-device full finetuning in full_finetune_distributed.py +# using a Llama2 7B model +# +# This config assumes that you've run the following command before launching +# this run: +# tune download --repo-id meta-llama/Llama-2-7b \ +# --hf-token \ +# --output-dir /tmp/llama2 +# +# To launch on 4 devices, run the following command from root: +# tune --nnodes 1 --nproc_per_node 4 full_finetune_distributed \ +# --config llama2/7B_full \ +# +# You can add specific overrides through the command line. For example +# to override the checkpointer directory while launching training +# you can run: +# tune --nnodes 1 --nproc_per_node 4 full_finetune_distributed \ +# --config llama2/7B_full \ +# checkpointer.checkpoint_dir= +# +# This config works best when the model is being fine-tuned on 2+ GPUs. +# Single device full finetuning requires more memory optimizations. It's +# best to use 7B_full_single_device.yaml for those cases + + +# Tokenizer +tokenizer: + _component_: torchtune.models.llama2.llama2_tokenizer + path: /tmp/llama2/tokenizer.model + +# Dataset +dataset: + _component_: torchtune.datasets.alpaca_dataset + train_on_input: True +seed: null +shuffle: True + +# Model Arguments +model: + _component_: torchtune.models.llama2.llama2_tiny + +checkpointer: + _component_: torchtune.utils.FullModelTorchTuneCheckpointer + checkpoint_dir: tiny_llama + checkpoint_files: [model.pt] + recipe_checkpoint: null + output_dir: tiny_llama + model_type: LLAMA2 +resume_from_checkpoint: False + +# Fine-tuning arguments +batch_size: 2 +epochs: 3 +optimizer: + _component_: torch.optim.AdamW + lr: 2e-5 +loss: + _component_: torch.nn.CrossEntropyLoss +max_steps_per_epoch: null +gradient_accumulation_steps: 1 + + +# Training env +device: cuda + +# Distributed +cpu_offload: False + +# Memory management +enable_activation_checkpointing: True + +# Reduced precision +dtype: bf16 + +# Logging +# output_dir: /tmp/full_finetune_output +# metric_logger: +# _component_: torchtune.utils.metric_logging.WandBLogger +# project: torchtune +# log_every_n_steps: 1 + +# Logging +output_dir: /tmp/full_finetune_output +metric_logger: + _component_: torchtune.utils.metric_logging.DiskLogger + log_dir: ${output_dir} +log_every_n_steps: null \ No newline at end of file diff --git a/recipes/lora_finetune_single_device.py b/recipes/lora_finetune_single_device.py index c37f92d866..e6ae4a1ffa 100644 --- a/recipes/lora_finetune_single_device.py +++ b/recipes/lora_finetune_single_device.py @@ -145,7 +145,9 @@ def setup(self, cfg: DictConfig) -> None: Setup the recipe state. This includes recipe state (if resume_from_checkpoint is True), model, tokenizer, loss, optimizer, learning rate scheduler, sampler, and dataloader. """ - self._metric_logger = config.instantiate(cfg.metric_logger) + self._metric_logger = config.instantiate(cfg.metric_logger, **cfg) + print(f"#########\nConfig: \n{cfg}\n#########\n") + breakpoint() checkpoint_dict = self.load_checkpoint(cfg=cfg.checkpointer) diff --git a/torchtune/models/llama2/__init__.py b/torchtune/models/llama2/__init__.py index b66f6d9fca..90481c48b5 100644 --- a/torchtune/models/llama2/__init__.py +++ b/torchtune/models/llama2/__init__.py @@ -10,6 +10,7 @@ from ._model_builders import ( # noqa llama2_13b, llama2_7b, + llama2_tiny, llama2_tokenizer, lora_llama2_13b, lora_llama2_7b, @@ -20,6 +21,7 @@ __all__ = [ "convert_llama2_fair_format", "llama2", + "llama2_tiny", "llama2_7b", "llama2_tokenizer", "lora_llama2", diff --git a/torchtune/models/llama2/_model_builders.py b/torchtune/models/llama2/_model_builders.py index 59fd4d1a8e..6a030255d0 100644 --- a/torchtune/models/llama2/_model_builders.py +++ b/torchtune/models/llama2/_model_builders.py @@ -21,6 +21,28 @@ llama2 7B model. """ +def llama2_tiny(max_batch_size: Optional[int] = None) -> TransformerDecoder: + """ + Creates a very small Llama2 model for testing purposes. + + Args: + max_batch_size (Optional[int]): Maximum batch size to be passed to KVCache. + + Returns: + TransformerDecoder: Instantiation of Llama2 7B model + """ + return llama2( + vocab_size=32_000, + num_layers=2, + num_heads=4, + num_kv_heads=4, + embed_dim=128, + max_seq_len=4096, + max_batch_size=max_batch_size, + attn_dropout=0.0, + norm_eps=1e-5, + ) + def llama2_7b() -> TransformerDecoder: """ From 99283aef29fa8443d41e0ac7c797a6d016ab2ed1 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 5 Apr 2024 12:27:13 +0000 Subject: [PATCH 36/76] fix tiny config --- recipes/configs/llama2/tiny_llama.yaml | 1 + torchtune/models/llama2/_model_builders.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/configs/llama2/tiny_llama.yaml b/recipes/configs/llama2/tiny_llama.yaml index d7d339d115..ee71b3e213 100644 --- a/recipes/configs/llama2/tiny_llama.yaml +++ b/recipes/configs/llama2/tiny_llama.yaml @@ -58,6 +58,7 @@ loss: _component_: torch.nn.CrossEntropyLoss max_steps_per_epoch: null gradient_accumulation_steps: 1 +optimizer_in_bwd: False # Training env diff --git a/torchtune/models/llama2/_model_builders.py b/torchtune/models/llama2/_model_builders.py index 6a030255d0..c2e6a41791 100644 --- a/torchtune/models/llama2/_model_builders.py +++ b/torchtune/models/llama2/_model_builders.py @@ -21,7 +21,7 @@ llama2 7B model. """ -def llama2_tiny(max_batch_size: Optional[int] = None) -> TransformerDecoder: +def llama2_tiny() -> TransformerDecoder: """ Creates a very small Llama2 model for testing purposes. @@ -38,7 +38,6 @@ def llama2_tiny(max_batch_size: Optional[int] = None) -> TransformerDecoder: num_kv_heads=4, embed_dim=128, max_seq_len=4096, - max_batch_size=max_batch_size, attn_dropout=0.0, norm_eps=1e-5, ) From 6e1bfccf6e1ef2ce35a44b3dce4c6153c35f5cd8 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 5 Apr 2024 12:31:11 +0000 Subject: [PATCH 37/76] updated recipe tiny --- dev.ipynb | 228 +------------------------ recipes/configs/llama2/tiny_llama.yaml | 20 +-- 2 files changed, 13 insertions(+), 235 deletions(-) diff --git a/dev.ipynb b/dev.ipynb index 88b9d36922..48cf9d64e4 100644 --- a/dev.ipynb +++ b/dev.ipynb @@ -7,8 +7,7 @@ "outputs": [], "source": [ "from torchtune.models.llama2 import llama2\n", - "from torchtune.modules import TransformerDecoder\n", - "from torchtune.utils.checkpoint import save_checkpoint, load_checkpoint" + "from torchtune.modules import TransformerDecoder" ] }, { @@ -17,7 +16,7 @@ "metadata": {}, "outputs": [], "source": [ - "def llama2_tiny(max_batch_size = None) -> TransformerDecoder:\n", + "def llama2_tiny() -> TransformerDecoder:\n", " \"\"\"\n", " Builder for creating a Llama2 model initialized w/ the default 7b parameter values\n", " from https://arxiv.org/abs/2307.09288\n", @@ -35,7 +34,6 @@ " num_kv_heads=4,\n", " embed_dim=128,\n", " max_seq_len=4096,\n", - " max_batch_size=max_batch_size,\n", " attn_dropout=0.0,\n", " norm_eps=1e-5,\n", " )" @@ -61,233 +59,13 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import torch\n", "torch.save(m.state_dict(), \"tiny_llama/model.pt\")" ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "save_checkpoint(chpt_dict, 'tiny_llama/model.pth')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'model': OrderedDict([('tok_embeddings.weight',\n", - " tensor([[ 0.6721, -0.0063, 0.2206, ..., -0.4694, 1.8257, 2.1110],\n", - " [-0.3021, 0.1397, 0.0167, ..., 1.3691, -0.1057, 0.4563],\n", - " [ 0.2236, 2.0759, 3.3434, ..., 0.3655, 0.8593, 0.4055],\n", - " ...,\n", - " [ 2.0139, -0.4313, -0.6845, ..., -0.1600, 0.2825, -0.6131],\n", - " [ 0.1583, 0.9916, 1.0918, ..., -2.1604, -1.0594, 0.0648],\n", - " [-1.4329, -0.8049, -0.5369, ..., -1.2593, 2.4386, 1.2828]])),\n", - " ('layers.0.sa_norm.scale',\n", - " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1.])),\n", - " ('layers.0.attn.q_proj.weight',\n", - " tensor([[ 0.0280, 0.0101, -0.0457, ..., -0.0786, 0.0836, -0.0700],\n", - " [-0.0022, 0.0369, -0.0879, ..., -0.0854, 0.0129, -0.0333],\n", - " [-0.0572, -0.0465, -0.0008, ..., -0.0680, 0.0245, 0.0512],\n", - " ...,\n", - " [ 0.0769, 0.0790, -0.0056, ..., -0.0442, 0.0624, -0.0328],\n", - " [ 0.0110, 0.0400, -0.0696, ..., -0.0524, 0.0883, 0.0866],\n", - " [-0.0589, -0.0402, -0.0066, ..., 0.0207, 0.0333, 0.0478]])),\n", - " ('layers.0.attn.k_proj.weight',\n", - " tensor([[-0.0351, -0.0071, 0.0104, ..., -0.0403, 0.0620, 0.0435],\n", - " [-0.0071, 0.0844, 0.0140, ..., 0.0817, -0.0595, 0.0528],\n", - " [ 0.0559, 0.0310, 0.0526, ..., 0.0125, 0.0205, 0.0430],\n", - " ...,\n", - " [ 0.0870, -0.0838, 0.0671, ..., -0.0014, -0.0662, -0.0745],\n", - " [-0.0127, 0.0005, -0.0680, ..., -0.0583, -0.0266, -0.0288],\n", - " [-0.0371, 0.0380, 0.0256, ..., -0.0173, 0.0089, 0.0115]])),\n", - " ('layers.0.attn.v_proj.weight',\n", - " tensor([[ 0.0030, -0.0682, -0.0049, ..., 0.0430, -0.0171, 0.0023],\n", - " [-0.0876, -0.0240, -0.0790, ..., 0.0093, 0.0397, 0.0405],\n", - " [-0.0364, 0.0719, -0.0348, ..., 0.0146, -0.0119, -0.0518],\n", - " ...,\n", - " [ 0.0278, 0.0330, -0.0836, ..., -0.0189, 0.0710, -0.0296],\n", - " [-0.0140, -0.0416, 0.0520, ..., -0.0156, 0.0559, 0.0683],\n", - " [ 0.0519, 0.0505, 0.0572, ..., -0.0004, -0.0767, 0.0242]])),\n", - " ('layers.0.attn.output_proj.weight',\n", - " tensor([[ 0.0324, 0.0520, -0.0132, ..., 0.0285, -0.0848, 0.0720],\n", - " [ 0.0236, -0.0741, -0.0754, ..., 0.0153, 0.0187, 0.0066],\n", - " [ 0.0456, 0.0669, 0.0566, ..., -0.0149, 0.0764, 0.0644],\n", - " ...,\n", - " [ 0.0577, 0.0264, -0.0095, ..., 0.0436, 0.0627, 0.0722],\n", - " [ 0.0871, 0.0220, -0.0865, ..., -0.0504, -0.0218, -0.0323],\n", - " [ 0.0582, 0.0574, -0.0780, ..., 0.0021, -0.0865, 0.0793]])),\n", - " ('layers.0.mlp_norm.scale',\n", - " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1.])),\n", - " ('layers.0.mlp.w1.weight',\n", - " tensor([[-0.0878, -0.0187, 0.0862, ..., -0.0095, -0.0032, -0.0611],\n", - " [-0.0174, -0.0744, -0.0868, ..., 0.0549, 0.0327, 0.0868],\n", - " [-0.0107, 0.0538, -0.0477, ..., 0.0769, 0.0007, -0.0623],\n", - " ...,\n", - " [ 0.0022, 0.0863, 0.0487, ..., 0.0201, -0.0106, 0.0311],\n", - " [ 0.0260, -0.0131, -0.0065, ..., -0.0812, 0.0281, 0.0416],\n", - " [-0.0521, 0.0364, -0.0264, ..., -0.0464, -0.0031, 0.0752]])),\n", - " ('layers.0.mlp.w2.weight',\n", - " tensor([[ 4.4035e-02, -1.5159e-02, 1.4231e-02, ..., -3.8606e-02,\n", - " 2.3046e-02, -4.1586e-03],\n", - " [-2.4230e-02, -1.7738e-02, -3.4205e-02, ..., -5.8932e-03,\n", - " 2.8201e-02, -1.3607e-02],\n", - " [ 9.5524e-03, -2.8478e-02, -1.6222e-02, ..., 1.5925e-03,\n", - " 4.3839e-02, 4.0368e-02],\n", - " ...,\n", - " [ 3.2933e-02, 1.2597e-05, -1.7801e-02, ..., -3.5409e-02,\n", - " -1.4612e-02, -4.5030e-03],\n", - " [ 1.1489e-02, -3.2748e-02, 4.1240e-02, ..., 2.2262e-02,\n", - " 4.2146e-02, 1.0142e-02],\n", - " [ 1.6441e-02, 3.9053e-02, -5.2180e-03, ..., -1.7018e-02,\n", - " -2.9996e-02, -4.1561e-02]])),\n", - " ('layers.0.mlp.w3.weight',\n", - " tensor([[-0.0253, 0.0373, 0.0508, ..., 0.0006, 0.0654, -0.0867],\n", - " [-0.0258, 0.0112, 0.0398, ..., 0.0495, -0.0066, -0.0329],\n", - " [ 0.0832, 0.0729, 0.0340, ..., -0.0801, -0.0719, 0.0561],\n", - " ...,\n", - " [-0.0681, -0.0745, 0.0475, ..., 0.0271, -0.0758, 0.0660],\n", - " [ 0.0268, -0.0854, 0.0629, ..., -0.0278, -0.0001, 0.0677],\n", - " [ 0.0325, -0.0836, 0.0125, ..., 0.0498, 0.0376, 0.0281]])),\n", - " ('layers.1.sa_norm.scale',\n", - " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1.])),\n", - " ('layers.1.attn.q_proj.weight',\n", - " tensor([[ 0.0280, 0.0101, -0.0457, ..., -0.0786, 0.0836, -0.0700],\n", - " [-0.0022, 0.0369, -0.0879, ..., -0.0854, 0.0129, -0.0333],\n", - " [-0.0572, -0.0465, -0.0008, ..., -0.0680, 0.0245, 0.0512],\n", - " ...,\n", - " [ 0.0769, 0.0790, -0.0056, ..., -0.0442, 0.0624, -0.0328],\n", - " [ 0.0110, 0.0400, -0.0696, ..., -0.0524, 0.0883, 0.0866],\n", - " [-0.0589, -0.0402, -0.0066, ..., 0.0207, 0.0333, 0.0478]])),\n", - " ('layers.1.attn.k_proj.weight',\n", - " tensor([[-0.0351, -0.0071, 0.0104, ..., -0.0403, 0.0620, 0.0435],\n", - " [-0.0071, 0.0844, 0.0140, ..., 0.0817, -0.0595, 0.0528],\n", - " [ 0.0559, 0.0310, 0.0526, ..., 0.0125, 0.0205, 0.0430],\n", - " ...,\n", - " [ 0.0870, -0.0838, 0.0671, ..., -0.0014, -0.0662, -0.0745],\n", - " [-0.0127, 0.0005, -0.0680, ..., -0.0583, -0.0266, -0.0288],\n", - " [-0.0371, 0.0380, 0.0256, ..., -0.0173, 0.0089, 0.0115]])),\n", - " ('layers.1.attn.v_proj.weight',\n", - " tensor([[ 0.0030, -0.0682, -0.0049, ..., 0.0430, -0.0171, 0.0023],\n", - " [-0.0876, -0.0240, -0.0790, ..., 0.0093, 0.0397, 0.0405],\n", - " [-0.0364, 0.0719, -0.0348, ..., 0.0146, -0.0119, -0.0518],\n", - " ...,\n", - " [ 0.0278, 0.0330, -0.0836, ..., -0.0189, 0.0710, -0.0296],\n", - " [-0.0140, -0.0416, 0.0520, ..., -0.0156, 0.0559, 0.0683],\n", - " [ 0.0519, 0.0505, 0.0572, ..., -0.0004, -0.0767, 0.0242]])),\n", - " ('layers.1.attn.output_proj.weight',\n", - " tensor([[ 0.0324, 0.0520, -0.0132, ..., 0.0285, -0.0848, 0.0720],\n", - " [ 0.0236, -0.0741, -0.0754, ..., 0.0153, 0.0187, 0.0066],\n", - " [ 0.0456, 0.0669, 0.0566, ..., -0.0149, 0.0764, 0.0644],\n", - " ...,\n", - " [ 0.0577, 0.0264, -0.0095, ..., 0.0436, 0.0627, 0.0722],\n", - " [ 0.0871, 0.0220, -0.0865, ..., -0.0504, -0.0218, -0.0323],\n", - " [ 0.0582, 0.0574, -0.0780, ..., 0.0021, -0.0865, 0.0793]])),\n", - " ('layers.1.mlp_norm.scale',\n", - " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1.])),\n", - " ('layers.1.mlp.w1.weight',\n", - " tensor([[-0.0878, -0.0187, 0.0862, ..., -0.0095, -0.0032, -0.0611],\n", - " [-0.0174, -0.0744, -0.0868, ..., 0.0549, 0.0327, 0.0868],\n", - " [-0.0107, 0.0538, -0.0477, ..., 0.0769, 0.0007, -0.0623],\n", - " ...,\n", - " [ 0.0022, 0.0863, 0.0487, ..., 0.0201, -0.0106, 0.0311],\n", - " [ 0.0260, -0.0131, -0.0065, ..., -0.0812, 0.0281, 0.0416],\n", - " [-0.0521, 0.0364, -0.0264, ..., -0.0464, -0.0031, 0.0752]])),\n", - " ('layers.1.mlp.w2.weight',\n", - " tensor([[ 4.4035e-02, -1.5159e-02, 1.4231e-02, ..., -3.8606e-02,\n", - " 2.3046e-02, -4.1586e-03],\n", - " [-2.4230e-02, -1.7738e-02, -3.4205e-02, ..., -5.8932e-03,\n", - " 2.8201e-02, -1.3607e-02],\n", - " [ 9.5524e-03, -2.8478e-02, -1.6222e-02, ..., 1.5925e-03,\n", - " 4.3839e-02, 4.0368e-02],\n", - " ...,\n", - " [ 3.2933e-02, 1.2597e-05, -1.7801e-02, ..., -3.5409e-02,\n", - " -1.4612e-02, -4.5030e-03],\n", - " [ 1.1489e-02, -3.2748e-02, 4.1240e-02, ..., 2.2262e-02,\n", - " 4.2146e-02, 1.0142e-02],\n", - " [ 1.6441e-02, 3.9053e-02, -5.2180e-03, ..., -1.7018e-02,\n", - " -2.9996e-02, -4.1561e-02]])),\n", - " ('layers.1.mlp.w3.weight',\n", - " tensor([[-0.0253, 0.0373, 0.0508, ..., 0.0006, 0.0654, -0.0867],\n", - " [-0.0258, 0.0112, 0.0398, ..., 0.0495, -0.0066, -0.0329],\n", - " [ 0.0832, 0.0729, 0.0340, ..., -0.0801, -0.0719, 0.0561],\n", - " ...,\n", - " [-0.0681, -0.0745, 0.0475, ..., 0.0271, -0.0758, 0.0660],\n", - " [ 0.0268, -0.0854, 0.0629, ..., -0.0278, -0.0001, 0.0677],\n", - " [ 0.0325, -0.0836, 0.0125, ..., 0.0498, 0.0376, 0.0281]])),\n", - " ('norm.scale',\n", - " tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", - " 1., 1.])),\n", - " ('output.weight',\n", - " tensor([[ 0.0388, 0.0253, -0.0010, ..., 0.0023, -0.0367, -0.0049],\n", - " [-0.0090, 0.0756, 0.0523, ..., 0.0191, -0.0657, 0.0881],\n", - " [ 0.0492, -0.0446, 0.0090, ..., 0.0346, 0.0222, -0.0203],\n", - " ...,\n", - " [-0.0378, -0.0467, 0.0259, ..., 0.0030, 0.0568, 0.0595],\n", - " [ 0.0258, -0.0561, -0.0195, ..., -0.0867, -0.0714, -0.0340],\n", - " [-0.0815, 0.0689, -0.0411, ..., -0.0726, -0.0856, 0.0440]]))])}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "load_checkpoint('tiny_llama/model.pth', m)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/recipes/configs/llama2/tiny_llama.yaml b/recipes/configs/llama2/tiny_llama.yaml index ee71b3e213..a538710d9b 100644 --- a/recipes/configs/llama2/tiny_llama.yaml +++ b/recipes/configs/llama2/tiny_llama.yaml @@ -73,16 +73,16 @@ enable_activation_checkpointing: True # Reduced precision dtype: bf16 -# Logging -# output_dir: /tmp/full_finetune_output -# metric_logger: -# _component_: torchtune.utils.metric_logging.WandBLogger -# project: torchtune -# log_every_n_steps: 1 - # Logging output_dir: /tmp/full_finetune_output metric_logger: - _component_: torchtune.utils.metric_logging.DiskLogger - log_dir: ${output_dir} -log_every_n_steps: null \ No newline at end of file + _component_: torchtune.utils.metric_logging.WandBLogger + project: torchtune +log_every_n_steps: 1 + +# # Logging +# output_dir: /tmp/full_finetune_output +# metric_logger: +# _component_: torchtune.utils.metric_logging.DiskLogger +# log_dir: ${output_dir} +# log_every_n_steps: null \ No newline at end of file From f73b4d7ee80805d856e95a1586ee752feb0b3b9b Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Tue, 9 Apr 2024 16:44:31 +0200 Subject: [PATCH 38/76] grab yaml used on input --- torchtune/utils/metric_logging.py | 37 ++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index eb8856ad2c..160fd5146a 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -6,6 +6,8 @@ import os import sys import time +from shutil import copyfile +from tempfile import NamedTemporaryFile from pathlib import Path from typing import Mapping, Optional, Union @@ -18,7 +20,6 @@ Scalar = Union[Tensor, ndarray, int, float] - class MetricLoggerInterface(Protocol): """Abstract metric logger.""" @@ -169,6 +170,39 @@ def __init__( config=kwargs, ) + # Save the config used with `tune` to wandb/Files + yaml_configs = self._grab_yaml_configs() + for yaml_file in yaml_configs: + self._log_yaml_config(yaml_file) + + def _log_yaml_config(self, config_file: str) -> None: + """Log the yaml config file to wandb in files.""" + try: + with NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False, prefix="torchtune_config_" + ) as temp_file: + copyfile(config_file, temp_file.name) + print(f"Logging {temp_file.name} to wandb under Files") + temp_config = Path(temp_file.name) + self._wandb.save(temp_config, base_path=temp_config.parent) + except Exception as e: + print(f"Error saving {config_file} to wandb: {e}") + + def _grab_yaml_configs(self, extensions: tuple[str, ...] = (".yaml", ".yml")) -> list[str]: + """Grab the yaml config files used with `tune` by inspecting the command line args.""" + def _find_yaml_files(args: list[str]) -> list[str]: + yaml_files = [] + for arg in args: + if arg.lower().endswith(extensions): + yaml_files.append(arg) + return yaml_files + + # We could have more than one yaml config in the future... + command_line_args = sys.argv[1:] + yaml_files = _find_yaml_files(command_line_args) + + return yaml_files + def log(self, name: str, data: Scalar, step: int) -> None: self._wandb.log({name: data}, step=step) @@ -243,3 +277,4 @@ def close(self) -> None: if self._writer: self._writer.close() self._writer = None + From 6a888acddda51a03c873909616578729ba08de22 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Tue, 9 Apr 2024 17:14:00 +0200 Subject: [PATCH 39/76] update the wandb.config with the re-loaded yaml --- torchtune/utils/metric_logging.py | 40 ++++++++++++++++++------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 160fd5146a..ee7437bedf 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -6,6 +6,7 @@ import os import sys import time +import yaml from shutil import copyfile from tempfile import NamedTemporaryFile from pathlib import Path @@ -171,9 +172,18 @@ def __init__( ) # Save the config used with `tune` to wandb/Files - yaml_configs = self._grab_yaml_configs() - for yaml_file in yaml_configs: - self._log_yaml_config(yaml_file) + yaml_config = self._grab_yaml_config() + self._log_yaml_config(yaml_config) # only the first file + + # not ideal, we re-load the configs into a dict and then log them + try: + yaml_config = self._load_config(yaml_config) + self._wandb.config.update(yaml_config) + except Exception as e: + print(f"Error loading {yaml_config} into wandb: {e}") + + def _load_config(self, config_file: Path) -> dict: + return yaml.safe_load(config_file.read_text()) def _log_yaml_config(self, config_file: str) -> None: """Log the yaml config file to wandb in files.""" @@ -188,20 +198,18 @@ def _log_yaml_config(self, config_file: str) -> None: except Exception as e: print(f"Error saving {config_file} to wandb: {e}") - def _grab_yaml_configs(self, extensions: tuple[str, ...] = (".yaml", ".yml")) -> list[str]: - """Grab the yaml config files used with `tune` by inspecting the command line args.""" - def _find_yaml_files(args: list[str]) -> list[str]: - yaml_files = [] - for arg in args: - if arg.lower().endswith(extensions): - yaml_files.append(arg) - return yaml_files + def _grab_yaml_config(self, extensions: tuple[str, ...] = (".yaml", ".yml")) -> list[Path]: + """Grab the yaml config file passed to the --config arg.""" + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=str, required=True, help="Path to config file") + args, _ = parser.parse_known_args() - # We could have more than one yaml config in the future... - command_line_args = sys.argv[1:] - yaml_files = _find_yaml_files(command_line_args) - - return yaml_files + config_file = Path(args.config) + if not config_file.is_file() or config_file.suffix not in extensions: + raise ValueError(f"Invalid config file: {config_file}") + + return config_file def log(self, name: str, data: Scalar, step: int) -> None: self._wandb.log({name: data}, step=step) From 90ebe5332d12fdae1fb205ef5609175223aa02a3 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Tue, 9 Apr 2024 17:23:31 +0200 Subject: [PATCH 40/76] undo changes --- recipes/lora_finetune_single_device.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/recipes/lora_finetune_single_device.py b/recipes/lora_finetune_single_device.py index e6ae4a1ffa..c37f92d866 100644 --- a/recipes/lora_finetune_single_device.py +++ b/recipes/lora_finetune_single_device.py @@ -145,9 +145,7 @@ def setup(self, cfg: DictConfig) -> None: Setup the recipe state. This includes recipe state (if resume_from_checkpoint is True), model, tokenizer, loss, optimizer, learning rate scheduler, sampler, and dataloader. """ - self._metric_logger = config.instantiate(cfg.metric_logger, **cfg) - print(f"#########\nConfig: \n{cfg}\n#########\n") - breakpoint() + self._metric_logger = config.instantiate(cfg.metric_logger) checkpoint_dict = self.load_checkpoint(cfg=cfg.checkpointer) From c0f81ae9438f14cd4acede8f9c1a2cc2befd9915 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Tue, 9 Apr 2024 18:04:53 +0200 Subject: [PATCH 41/76] some clean up logic --- torchtune/utils/metric_logging.py | 42 ++++++++++++++++--------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index ee7437bedf..f449756a14 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -3,6 +3,7 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import argparse import os import sys import time @@ -171,22 +172,26 @@ def __init__( config=kwargs, ) - # Save the config used with `tune` to wandb/Files - yaml_config = self._grab_yaml_config() - self._log_yaml_config(yaml_config) # only the first file - - # not ideal, we re-load the configs into a dict and then log them + yaml_config = self._maybe_grab_yaml_config() + + if yaml_config is not None: + # we log the config to wandb/Files + self._log_yaml_config_to_files(yaml_config) + + # update the wandb config with the yaml config + self._update_wandb_config(yaml_config) + + def _update_wandb_config(self, config_file: Path) -> None: try: - yaml_config = self._load_config(yaml_config) + yaml_config = yaml.safe_load(config_file.read_text()) self._wandb.config.update(yaml_config) except Exception as e: - print(f"Error loading {yaml_config} into wandb: {e}") + print(f"Error loading {config_file} into wandb: {e}") - def _load_config(self, config_file: Path) -> dict: - return yaml.safe_load(config_file.read_text()) - def _log_yaml_config(self, config_file: str) -> None: - """Log the yaml config file to wandb in files.""" + def _log_yaml_config_to_files(self, config_file: str) -> None: + """Log the yaml config file to wandb in files. We copy the original config to avoid + conflicting with the wandb config.""" try: with NamedTemporaryFile( mode="w", suffix=".yaml", delete=False, prefix="torchtune_config_" @@ -198,18 +203,16 @@ def _log_yaml_config(self, config_file: str) -> None: except Exception as e: print(f"Error saving {config_file} to wandb: {e}") - def _grab_yaml_config(self, extensions: tuple[str, ...] = (".yaml", ".yml")) -> list[Path]: + def _maybe_grab_yaml_config(self) -> Optional[Path]: """Grab the yaml config file passed to the --config arg.""" - import argparse parser = argparse.ArgumentParser() - parser.add_argument("--config", type=str, required=True, help="Path to config file") + parser.add_argument("--config", type=str) args, _ = parser.parse_known_args() - config_file = Path(args.config) - if not config_file.is_file() or config_file.suffix not in extensions: - raise ValueError(f"Invalid config file: {config_file}") - - return config_file + config_file = getattr(args, "config", None) + if config_file is None: + return None + return Path(config_file) def log(self, name: str, data: Scalar, step: int) -> None: self._wandb.log({name: data}, step=step) @@ -223,7 +226,6 @@ def __del__(self) -> None: def close(self) -> None: self._wandb.finish() - class TensorBoardLogger(MetricLoggerInterface): """Logger for use w/ PyTorch's implementation of TensorBoard (https://pytorch.org/docs/stable/tensorboard.html). From c80c7ab0a7b3475a4dae9097c16c2f51d3fd6cd9 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Tue, 9 Apr 2024 18:24:25 +0200 Subject: [PATCH 42/76] walrus :) --- torchtune/utils/metric_logging.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index f449756a14..c7510d63a5 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -172,9 +172,7 @@ def __init__( config=kwargs, ) - yaml_config = self._maybe_grab_yaml_config() - - if yaml_config is not None: + if yaml_config := self._maybe_grab_yaml_config(): # we log the config to wandb/Files self._log_yaml_config_to_files(yaml_config) @@ -201,7 +199,7 @@ def _log_yaml_config_to_files(self, config_file: str) -> None: temp_config = Path(temp_file.name) self._wandb.save(temp_config, base_path=temp_config.parent) except Exception as e: - print(f"Error saving {config_file} to wandb: {e}") + print(f"Error saving {config_file} to wandb.\nError: \n{e}") def _maybe_grab_yaml_config(self) -> Optional[Path]: """Grab the yaml config file passed to the --config arg.""" From 6880cba7f592db592b4cf7a052d2645f631994b0 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Thu, 11 Apr 2024 14:09:16 +0200 Subject: [PATCH 43/76] remove nb --- dev.ipynb | 92 ------------------------------------------------------- 1 file changed, 92 deletions(-) delete mode 100644 dev.ipynb diff --git a/dev.ipynb b/dev.ipynb deleted file mode 100644 index 48cf9d64e4..0000000000 --- a/dev.ipynb +++ /dev/null @@ -1,92 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from torchtune.models.llama2 import llama2\n", - "from torchtune.modules import TransformerDecoder" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "def llama2_tiny() -> TransformerDecoder:\n", - " \"\"\"\n", - " Builder for creating a Llama2 model initialized w/ the default 7b parameter values\n", - " from https://arxiv.org/abs/2307.09288\n", - "\n", - " Args:\n", - " max_batch_size (Optional[int]): Maximum batch size to be passed to KVCache.\n", - "\n", - " Returns:\n", - " TransformerDecoder: Instantiation of Llama2 7B model\n", - " \"\"\"\n", - " return llama2(\n", - " vocab_size=32_000,\n", - " num_layers=2,\n", - " num_heads=4,\n", - " num_kv_heads=4,\n", - " embed_dim=128,\n", - " max_seq_len=4096,\n", - " attn_dropout=0.0,\n", - " norm_eps=1e-5,\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "m = llama2_tiny()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "chpt_dict = {\"model\": m}\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "torch.save(m.state_dict(), \"tiny_llama/model.pt\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pt", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 1ddbe7a99a45a783a545f224e5deb3907a57c901 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Thu, 11 Apr 2024 15:04:18 +0200 Subject: [PATCH 44/76] add docs --- docs/source/examples/wandb_logging.rst | 92 ++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/source/examples/wandb_logging.rst diff --git a/docs/source/examples/wandb_logging.rst b/docs/source/examples/wandb_logging.rst new file mode 100644 index 0000000000..b13b697b41 --- /dev/null +++ b/docs/source/examples/wandb_logging.rst @@ -0,0 +1,92 @@ +.. _wandb_logging: + +=========================== +Logging to Weights & Biases +=========================== + +Torchtune supports logging your training runs to [Weights & Biases](https://wandb.ai). + +.. note:: + + You will need to install the `wandb`` package to use this feature. + You can install it via pip: + + .. code-block:: bash + + pip install wandb + +An example config for enabling Weights & Biases logging on the full llama2 7b finetune recipe is as follows: + +.. code-block:: yaml + + # Tokenizer + tokenizer: + _component_: torchtune.models.llama2.llama2_tokenizer + path: /tmp/tokenizer.model + + # Dataset + dataset: + _component_: torchtune.datasets.alpaca_dataset + shuffle: True + + # Model Arguments + model: + _component_: torchtune.models.llama2.llama2_7b + + checkpointer: + _component_: torchtune.utils.FullModelMetaCheckpointer + checkpoint_dir: /tmp/llama2 + checkpoint_files: [consolidated.00.pth] + recipe_checkpoint: null + output_dir: /tmp/llama2 + model_type: LLAMA2 + resume_from_checkpoint: False + + # Fine-tuning arguments + batch_size: 2 + epochs: 3 + optimizer: + _component_: torch.optim.SGD + lr: 2e-5 + loss: + _component_: torch.nn.CrossEntropyLoss + output_dir: /tmp/alpaca-llama2-finetune + + device: cuda + dtype: bf16 + + enable_activation_checkpointing: True + + log_every_n_steps: 1 + + metric_logger: + _component_: torchtune.utils.metric_logging.WandBLogger + project: torchtune + log_model: checkpoint + + + +Metric Logger +------------- + +The only change you need to make is to add the metric logger to your config. Weights & Biases will log the metrics and model checkpoints for you. + +.. code-block:: python + # enable logging to the built-in WandBLogger + metric_logger: + _component_: torchtune.utils.metric_logging.WandBLogger + # the W&B project to log to + project: torchtune + # How often to log the model. Options are "null", "checkpoint" and "end" + # "null" means no logging + # "checkpoint" means log the model at each checkpoint (managed by the checkpointer) + # "end" means log the model at the end of the run + log_model: checkpoint + +We automatically grab the config from the recipe you are running and log it to W&B. You can find it in the W&B overview tab and the actual file in the `Files` tab. + +.. note:: + + Click on this sample [project to see the W&B workspace](https://wandb.ai/capecape/torchtune) + The config used to train the models can be found [here](https://wandb.ai/capecape/torchtune/runs/6053ofw0/files/torchtune_config_j67sb73v.yaml) + From ad42a35bbe957b5e4502bf31dc29fa981acf3a2b Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Thu, 11 Apr 2024 16:10:53 +0200 Subject: [PATCH 45/76] refactor memory so it's loggable --- recipes/full_finetune_single_device.py | 4 ++- torchtune/utils/memory.py | 49 +++++++++++++++++++------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/recipes/full_finetune_single_device.py b/recipes/full_finetune_single_device.py index 3658fa8205..546ab11b14 100644 --- a/recipes/full_finetune_single_device.py +++ b/recipes/full_finetune_single_device.py @@ -415,8 +415,10 @@ def train(self) -> None: # Log peak memory for iteration if self.total_training_steps % self._log_peak_memory_every_n_steps == 0: + memory_stats = utils.memory_stats_log("Memory Stats:", device=self._device) + self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) log.info( - utils.memory_stats_log("Memory Stats:", device=self._device) + utils.print_memory_stats_log("Memory Stats:", memory_stats) ) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/torchtune/utils/memory.py b/torchtune/utils/memory.py index 5713c73e98..bda0702843 100644 --- a/torchtune/utils/memory.py +++ b/torchtune/utils/memory.py @@ -160,22 +160,20 @@ def optim_step(param) -> None: p.register_post_accumulate_grad_hook(optim_step) -def memory_stats_log( - prefix: str, device: torch.device, reset_stats: bool = True -) -> None: +def memory_stats_log(device: torch.device, reset_stats: bool = True) -> None: """ - Print a memory summary for the passed in device. If ``reset_stats`` is ``True``, this will + Computes a memory summary for the passed in device. If ``reset_stats`` is ``True``, this will also reset CUDA's peak memory tracking. This is useful to get data around relative use of peak memory (i.e. peak memory during model init, during forward, etc) and optimize memory for individual sections of training. Args: - prefix (str): Prefix to prepend to the printed summary. device (torch.device): Device to get memory summary for. Only CUDA devices are supported. reset_stats (bool): Whether to reset CUDA's peak memory tracking. Returns: - None + Dict[str, float]: A dictionary containing the peak memory active, peak memory allocated, + and peak memory reserved. This dict is useful for logging memory stats. """ if device.type != "cuda": return @@ -183,14 +181,39 @@ def memory_stats_log( peak_mem_alloc = torch.cuda.max_memory_allocated(device) / 1e9 peak_mem_reserved = torch.cuda.max_memory_reserved(device) / 1e9 - ret = f""" - {prefix}: - GPU peak memory allocation: {peak_mem_alloc:.2f} GB - GPU peak memory reserved: {peak_mem_reserved:.2f} GB - GPU peak memory active: {peak_memory_active:.2f} GB - """ - if reset_stats: torch.cuda.reset_peak_memory_stats(device) + memory_stats = { + "peak_memory_active": peak_memory_active, + "peak_memory_alloc": peak_mem_alloc, + "peak_memory_reserved": peak_mem_reserved, + } + return memory_stats + +def print_memory_stats_log( + prefix: str, memory_stats: dict +) -> str: + """ + Print a memory summary for the passed in device. If ``reset_stats`` is ``True``, this will + also reset CUDA's peak memory tracking. This is useful to get data around relative use of peak + memory (i.e. peak memory during model init, during forward, etc) and optimize memory for + individual sections of training. + + Args: + prefix (str): Prefix to prepend to the printed summary. + memory_stats (dict): A dictionary containing the peak memory active, peak memory allocated, + and peak memory reserved. + + Returns: + str: A string containing the memory summary. + """ + + ret = f""" + {prefix}: + GPU peak memory allocation: {memory_stats["peak_memory_alloc"]:.2f} GB + GPU peak memory reserved: {memory_stats["peak_memory_reserved"]:.2f} GB + GPU peak memory active: {memory_stats["peak_memory_active"]:.2f} GB + """ + return ret From 2af532a810d7c05d666d2e31cc66f168d968d902 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Thu, 11 Apr 2024 17:05:04 +0200 Subject: [PATCH 46/76] put checkpoint logic as a tutorial --- docs/source/examples/wandb_logging.rst | 55 +++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/docs/source/examples/wandb_logging.rst b/docs/source/examples/wandb_logging.rst index b13b697b41..5489797114 100644 --- a/docs/source/examples/wandb_logging.rst +++ b/docs/source/examples/wandb_logging.rst @@ -77,11 +77,6 @@ The only change you need to make is to add the metric logger to your config. Wei _component_: torchtune.utils.metric_logging.WandBLogger # the W&B project to log to project: torchtune - # How often to log the model. Options are "null", "checkpoint" and "end" - # "null" means no logging - # "checkpoint" means log the model at each checkpoint (managed by the checkpointer) - # "end" means log the model at the end of the run - log_model: checkpoint We automatically grab the config from the recipe you are running and log it to W&B. You can find it in the W&B overview tab and the actual file in the `Files` tab. @@ -90,3 +85,53 @@ We automatically grab the config from the recipe you are running and log it to W Click on this sample [project to see the W&B workspace](https://wandb.ai/capecape/torchtune) The config used to train the models can be found [here](https://wandb.ai/capecape/torchtune/runs/6053ofw0/files/torchtune_config_j67sb73v.yaml) +Logging Model Checkpoints to W&B +------------------------------- + +You can also log the model checkpoints to W&B by modifying the desired script `save_checkpoint` method. + +A suggested approach would be something like this: + +.. code-block:: python + + def save_checkpoint(self, epoch: int) -> None: + ckpt_dict = {utils.MODEL_KEY: self._model.state_dict()} + # if training is in-progress, checkpoint the optimizer state as well + if epoch + 1 < self.total_epochs: + ckpt_dict.update( + { + utils.SEED_KEY: self.seed, + utils.EPOCHS_KEY: self.epochs_run, + utils.TOTAL_EPOCHS_KEY: self.total_epochs, + utils.MAX_STEPS_KEY: self.max_steps_per_epoch, + } + ) + if not self._optimizer_in_bwd: + ckpt_dict[utils.OPT_KEY] = self._optimizer.state_dict() + else: + ckpt_dict[utils.OPT_KEY] = self._optim_ckpt_wrapper.state_dict() + self._checkpointer.save_checkpoint( + ckpt_dict, + epoch=epoch, + intermediate_checkpoint=(epoch + 1 < self.total_epochs), + ) + ## Let's save the checkpoint to W&B + ## depending on the Checkpointer Class the file will be named differently + ## Here it is an example for the full_finetune case + checkpoint_file = Path.joinpath( + self._checkpointer._output_dir, f"torchtune_model_{epoch}" + ).with_suffix(".pt") + wandb_at = wandb.Artifact( + name=f"torchtune_model_{epoch}", + type="model", + description="Model checkpoint", + metadata={ + utils.SEED_KEY: self.seed, + utils.EPOCHS_KEY: self.epochs_run, + utils.TOTAL_EPOCHS_KEY: self.total_epochs, + utils.MAX_STEPS_KEY: self.max_steps_per_epoch, + } + ) + wandb_at.add_file(checkpoint_file) + wandb.log_artifact(wandb_at) + From c8c771e9b6b265ecbdbdfd744163bdbf03ffccc3 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 14:12:21 +0200 Subject: [PATCH 47/76] refactor memory logging --- recipes/full_finetune_single_device.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/recipes/full_finetune_single_device.py b/recipes/full_finetune_single_device.py index 546ab11b14..55d0a2b23b 100644 --- a/recipes/full_finetune_single_device.py +++ b/recipes/full_finetune_single_device.py @@ -150,6 +150,8 @@ def setup(self, cfg: DictConfig) -> None: """ self._metric_logger = config.instantiate(cfg.metric_logger) + + ckpt_dict = self.load_checkpoint(cfg.checkpointer) # ``_setup_model`` handles initialization and loading the state dict. This method @@ -231,11 +233,8 @@ def _setup_model( if compile_model: log.info("Compiling model with torch.compile...") model = utils.wrap_compile(model) - log.info( - utils.memory_stats_log( - "Memory Stats after model init:", device=self._device - ) - ) + memory_stats = utils.memory_stats_log(device=self._device) + log.info(f"Memory Stats:\n{memory_stats}") return model def _setup_optimizer( @@ -415,11 +414,9 @@ def train(self) -> None: # Log peak memory for iteration if self.total_training_steps % self._log_peak_memory_every_n_steps == 0: - memory_stats = utils.memory_stats_log("Memory Stats:", device=self._device) + memory_stats = utils.memory_stats_log(device=self._device) self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) - log.info( - utils.print_memory_stats_log("Memory Stats:", memory_stats) - ) + log.info(f"Memory Stats:\n{memory_stats}") self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) From 8518e268db6d3c76ffc8839d674018d8492fc901 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 14:14:08 +0200 Subject: [PATCH 48/76] dump omegaconf as ground true --- recipes/full_finetune_single_device.py | 3 +- torchtune/utils/metric_logging.py | 55 ++++++++++---------------- 2 files changed, 22 insertions(+), 36 deletions(-) diff --git a/recipes/full_finetune_single_device.py b/recipes/full_finetune_single_device.py index 55d0a2b23b..7462091aaf 100644 --- a/recipes/full_finetune_single_device.py +++ b/recipes/full_finetune_single_device.py @@ -150,7 +150,8 @@ def setup(self, cfg: DictConfig) -> None: """ self._metric_logger = config.instantiate(cfg.metric_logger) - + # log config with parameter override + self._metric_logger.save_config(cfg) ckpt_dict = self.load_checkpoint(cfg.checkpointer) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index c7510d63a5..910e9edaad 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -16,6 +16,7 @@ from numpy import ndarray from torch import Tensor +from omegaconf import OmegaConf from torchtune.utils._distributed import get_world_size_and_rank from typing_extensions import Protocol @@ -169,48 +170,32 @@ def __init__( group=group, reinit=True, resume="allow", - config=kwargs, + **kwargs, ) - if yaml_config := self._maybe_grab_yaml_config(): - # we log the config to wandb/Files - self._log_yaml_config_to_files(yaml_config) + def save_config(self, config: OmegaConf) -> None: + "Logs the config to W&B. Also updates config on overview tab." + resolved = OmegaConf.to_container(config, resolve=True) + self._wandb.config.update(resolved) + self._log_yaml_config_to_files(config) - # update the wandb config with the yaml config - self._update_wandb_config(yaml_config) - - def _update_wandb_config(self, config_file: Path) -> None: - try: - yaml_config = yaml.safe_load(config_file.read_text()) - self._wandb.config.update(yaml_config) - except Exception as e: - print(f"Error loading {config_file} into wandb: {e}") - - - def _log_yaml_config_to_files(self, config_file: str) -> None: + def _log_yaml_config_to_files(self, config: OmegaConf) -> None: """Log the yaml config file to wandb in files. We copy the original config to avoid conflicting with the wandb config.""" - try: - with NamedTemporaryFile( - mode="w", suffix=".yaml", delete=False, prefix="torchtune_config_" - ) as temp_file: - copyfile(config_file, temp_file.name) - print(f"Logging {temp_file.name} to wandb under Files") - temp_config = Path(temp_file.name) - self._wandb.save(temp_config, base_path=temp_config.parent) - except Exception as e: - print(f"Error saving {config_file} to wandb.\nError: \n{e}") - def _maybe_grab_yaml_config(self) -> Optional[Path]: - """Grab the yaml config file passed to the --config arg.""" - parser = argparse.ArgumentParser() - parser.add_argument("--config", type=str) - args, _ = parser.parse_known_args() + output_config_fname = Path(os.path.join( + config.checkpointer.checkpoint_dir, + f"torchtune_config_{self._wandb.run.id}.yaml" + ) + ) - config_file = getattr(args, "config", None) - if config_file is None: - return None - return Path(config_file) + try: + OmegaConf.save(config, output_config_fname) + print(f"Logging {output_config_fname} to W&B under Files") + self._wandb.save(output_config_fname, base_path=output_config_fname.parent) + + except Exception as e: + print(f"Error saving {output_config_fname} to W&B.\nError: \n{e}") def log(self, name: str, data: Scalar, step: int) -> None: self._wandb.log({name: data}, step=step) From b2ab64c1d1656da81820d720086ccdaf84a5b591 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 14:14:41 +0200 Subject: [PATCH 49/76] revert to return a dict --- torchtune/utils/memory.py | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/torchtune/utils/memory.py b/torchtune/utils/memory.py index bda0702843..1581f38dfa 100644 --- a/torchtune/utils/memory.py +++ b/torchtune/utils/memory.py @@ -189,31 +189,4 @@ def memory_stats_log(device: torch.device, reset_stats: bool = True) -> None: "peak_memory_alloc": peak_mem_alloc, "peak_memory_reserved": peak_mem_reserved, } - return memory_stats - -def print_memory_stats_log( - prefix: str, memory_stats: dict -) -> str: - """ - Print a memory summary for the passed in device. If ``reset_stats`` is ``True``, this will - also reset CUDA's peak memory tracking. This is useful to get data around relative use of peak - memory (i.e. peak memory during model init, during forward, etc) and optimize memory for - individual sections of training. - - Args: - prefix (str): Prefix to prepend to the printed summary. - memory_stats (dict): A dictionary containing the peak memory active, peak memory allocated, - and peak memory reserved. - - Returns: - str: A string containing the memory summary. - """ - - ret = f""" - {prefix}: - GPU peak memory allocation: {memory_stats["peak_memory_alloc"]:.2f} GB - GPU peak memory reserved: {memory_stats["peak_memory_reserved"]:.2f} GB - GPU peak memory active: {memory_stats["peak_memory_active"]:.2f} GB - """ - - return ret + return memory_stats \ No newline at end of file From c251b5917eb5fd833c9f3f3232493d30c7d16f38 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 14:28:33 +0200 Subject: [PATCH 50/76] rename to log_config --- recipes/full_finetune_single_device.py | 2 +- torchtune/utils/metric_logging.py | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/recipes/full_finetune_single_device.py b/recipes/full_finetune_single_device.py index 7462091aaf..702f6dce73 100644 --- a/recipes/full_finetune_single_device.py +++ b/recipes/full_finetune_single_device.py @@ -151,7 +151,7 @@ def setup(self, cfg: DictConfig) -> None: self._metric_logger = config.instantiate(cfg.metric_logger) # log config with parameter override - self._metric_logger.save_config(cfg) + self._metric_logger.log_config(cfg) ckpt_dict = self.load_checkpoint(cfg.checkpointer) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 910e9edaad..4d93628c0a 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -40,7 +40,13 @@ def log( step (int): step value to record """ pass - + def log_config(self, config: OmegaConf) -> None: + """Logs the config + + Args: + config (OmegaConf): config to log + """ + pass def log_dict(self, payload: Mapping[str, Scalar], step: int) -> None: """Log multiple scalar values. @@ -83,10 +89,10 @@ def __init__(self, log_dir: str, filename: Optional[str] = None, **kwargs): self._file_name = self.log_dir / filename self._file = open(self._file_name, "a") print(f"Writing logs to {self._file_name}") - + def path_to_log_file(self) -> Path: return self._file_name - + def log(self, name: str, data: Scalar, step: int) -> None: self._file.write(f"Step {step} | {name}:{data}\n") @@ -173,24 +179,20 @@ def __init__( **kwargs, ) - def save_config(self, config: OmegaConf) -> None: + def log_config(self, config: OmegaConf) -> None: "Logs the config to W&B. Also updates config on overview tab." + resolved = OmegaConf.to_container(config, resolve=True) self._wandb.config.update(resolved) - self._log_yaml_config_to_files(config) - - def _log_yaml_config_to_files(self, config: OmegaConf) -> None: - """Log the yaml config file to wandb in files. We copy the original config to avoid - conflicting with the wandb config.""" - + output_config_fname = Path(os.path.join( config.checkpointer.checkpoint_dir, f"torchtune_config_{self._wandb.run.id}.yaml" ) ) - + OmegaConf.save(config, output_config_fname) try: - OmegaConf.save(config, output_config_fname) + print(f"Logging {output_config_fname} to W&B under Files") self._wandb.save(output_config_fname, base_path=output_config_fname.parent) From a9f43fe91ebc826092d12b7ebb926db03bc88439 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 17:18:31 +0200 Subject: [PATCH 51/76] undo lora stuff --- recipes/configs/llama2/7B_lora_single_device.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/recipes/configs/llama2/7B_lora_single_device.yaml b/recipes/configs/llama2/7B_lora_single_device.yaml index 0212fc408d..a303afbbe6 100644 --- a/recipes/configs/llama2/7B_lora_single_device.yaml +++ b/recipes/configs/llama2/7B_lora_single_device.yaml @@ -21,8 +21,6 @@ # This config works only for training on single device. -# export PATH="/home/ubuntu/miniforge3/envs/pt/bin:$PATH" - # Model Arguments model: _component_: torchtune.models.llama2.lora_llama2_7b @@ -78,11 +76,9 @@ compile: False # Logging output_dir: /tmp/lora_finetune_output metric_logger: - _component_: torchtune.utils.metric_logging.WandBLogger - project: torchtune - kwarg1: 1 - kwarg2: 2 -log_every_n_steps: 1 + _component_: torchtune.utils.metric_logging.DiskLogger + log_dir: ${output_dir} +log_every_n_steps: null # Environment device: cuda From 7b66488ddcc423dd2ce6fbf0eeffb0638d646e30 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 17:54:09 +0200 Subject: [PATCH 52/76] remove tiny llama --- torchtune/models/llama2/__init__.py | 2 -- torchtune/models/llama2/_model_builders.py | 21 --------------------- 2 files changed, 23 deletions(-) diff --git a/torchtune/models/llama2/__init__.py b/torchtune/models/llama2/__init__.py index 90481c48b5..b66f6d9fca 100644 --- a/torchtune/models/llama2/__init__.py +++ b/torchtune/models/llama2/__init__.py @@ -10,7 +10,6 @@ from ._model_builders import ( # noqa llama2_13b, llama2_7b, - llama2_tiny, llama2_tokenizer, lora_llama2_13b, lora_llama2_7b, @@ -21,7 +20,6 @@ __all__ = [ "convert_llama2_fair_format", "llama2", - "llama2_tiny", "llama2_7b", "llama2_tokenizer", "lora_llama2", diff --git a/torchtune/models/llama2/_model_builders.py b/torchtune/models/llama2/_model_builders.py index e19f0a655f..483eab09c9 100644 --- a/torchtune/models/llama2/_model_builders.py +++ b/torchtune/models/llama2/_model_builders.py @@ -21,27 +21,6 @@ llama2 7B model. """ -def llama2_tiny() -> TransformerDecoder: - """ - Creates a very small Llama2 model for testing purposes. - - Args: - max_batch_size (Optional[int]): Maximum batch size to be passed to KVCache. - - Returns: - TransformerDecoder: Instantiation of Llama2 7B model - """ - return llama2( - vocab_size=32_000, - num_layers=2, - num_heads=4, - num_kv_heads=4, - embed_dim=128, - max_seq_len=4096, - attn_dropout=0.0, - norm_eps=1e-5, - ) - def llama2_7b() -> TransformerDecoder: """ From 498575843ffff17fbc5a79e37241ec7f28dda258 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 17:59:30 +0200 Subject: [PATCH 53/76] add distributed training logics --- torchtune/utils/metric_logging.py | 80 +++++++++++++++++++------------ 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 4d93628c0a..d76dcfda77 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -8,16 +8,15 @@ import sys import time import yaml -from shutil import copyfile -from tempfile import NamedTemporaryFile from pathlib import Path from typing import Mapping, Optional, Union from numpy import ndarray from torch import Tensor -from omegaconf import OmegaConf +from omegaconf import OmegaConf, DictConfig +from torchtune.utils._device import _get_local_rank from torchtune.utils._distributed import get_world_size_and_rank from typing_extensions import Protocol @@ -40,7 +39,7 @@ def log( step (int): step value to record """ pass - def log_config(self, config: OmegaConf) -> None: + def log_config(self, config: DictConfig) -> None: """Logs the config Args: @@ -136,6 +135,8 @@ class WandBLogger(MetricLoggerInterface): project (str): WandB project name entity (Optional[str]): WandB entity name group (Optional[str]): WandB group name + log_strategy (Optional[str]): Strategy to use for logging. Options are "main", "node", "all". + Default: "main" **kwargs: additional arguments to pass to wandb.init Example: @@ -160,6 +161,7 @@ def __init__( project: str = "torchtune", entity: Optional[str] = None, group: Optional[str] = None, + log_strategy: Optional[str] = "main", **kwargs, ): try: @@ -170,46 +172,62 @@ def __init__( "Alternatively, use the ``StdoutLogger``, which can be specified by setting metric_logger_type='stdout'." ) from e self._wandb = wandb - self._wandb.init( - project=project, - entity=entity, - group=group, - reinit=True, - resume="allow", - **kwargs, - ) - - def log_config(self, config: OmegaConf) -> None: - "Logs the config to W&B. Also updates config on overview tab." - resolved = OmegaConf.to_container(config, resolve=True) - self._wandb.config.update(resolved) + # logging strategy options are "main", "node", "all" + self.log_strategy = log_strategy + self.world_size, self.rank = get_world_size_and_rank() + self.local_rank = _get_local_rank() + self.local_rank = 0 if self.local_rank is None else self.local_rank - output_config_fname = Path(os.path.join( - config.checkpointer.checkpoint_dir, - f"torchtune_config_{self._wandb.run.id}.yaml" + if ((self.log_strategy == "main" and self.rank == 0) + or self.log_strategy == "all" + or (self.log_strategy == "node" and self.local_rank == 0) + ): + self._wandb.init( + project=project, + entity=entity, + group=group, + reinit=True, + resume="allow", + **kwargs, + ) - ) - OmegaConf.save(config, output_config_fname) - try: + + def log_config(self, config: DictConfig) -> None: + "Logs the config to W&B. Also updates config on overview tab." + if self._wandb.run: + resolved = OmegaConf.to_container(config, resolve=True) + self._wandb.config.update(resolved) - print(f"Logging {output_config_fname} to W&B under Files") - self._wandb.save(output_config_fname, base_path=output_config_fname.parent) + output_config_fname = Path(os.path.join( + config.checkpointer.checkpoint_dir, + f"torchtune_config_{self._wandb.run.id}.yaml" + ) + ) + OmegaConf.save(config, output_config_fname) + try: + + print(f"Logging {output_config_fname} to W&B under Files") + self._wandb.save(output_config_fname, base_path=output_config_fname.parent) - except Exception as e: - print(f"Error saving {output_config_fname} to W&B.\nError: \n{e}") + except Exception as e: + print(f"Error saving {output_config_fname} to W&B.\nError: \n{e}") def log(self, name: str, data: Scalar, step: int) -> None: - self._wandb.log({name: data}, step=step) + if self._wandb.run: + self._wandb.log({name: data}, step=step) def log_dict(self, payload: Mapping[str, Scalar], step: int) -> None: - self._wandb.log(payload, step=step) + if self._wandb.run: + self._wandb.log(payload, step=step) def __del__(self) -> None: - self._wandb.finish() + if self._wandb.run: + self._wandb.finish() def close(self) -> None: - self._wandb.finish() + if self._wandb.run: + self._wandb.finish() class TensorBoardLogger(MetricLoggerInterface): """Logger for use w/ PyTorch's implementation of TensorBoard (https://pytorch.org/docs/stable/tensorboard.html). From 3ed657c4bb94d5c0e4f32ba1961d3e1c1f6c18e8 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 17:59:37 +0200 Subject: [PATCH 54/76] compute number of params --- recipes/full_finetune_single_device.py | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/full_finetune_single_device.py b/recipes/full_finetune_single_device.py index 702f6dce73..03359b7df3 100644 --- a/recipes/full_finetune_single_device.py +++ b/recipes/full_finetune_single_device.py @@ -236,6 +236,7 @@ def _setup_model( model = utils.wrap_compile(model) memory_stats = utils.memory_stats_log(device=self._device) log.info(f"Memory Stats:\n{memory_stats}") + log.info(f"Model trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad)/1e6:,.2f}M") return model def _setup_optimizer( From 5cc577a7bb178b589cbad998d8ee7fa677939126 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 18:03:09 +0200 Subject: [PATCH 55/76] use provided tiny llama --- recipes/configs/llama2/tiny_llama.yaml | 56 +++++++++++++------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/recipes/configs/llama2/tiny_llama.yaml b/recipes/configs/llama2/tiny_llama.yaml index a538710d9b..6cf9bb61af 100644 --- a/recipes/configs/llama2/tiny_llama.yaml +++ b/recipes/configs/llama2/tiny_llama.yaml @@ -1,32 +1,20 @@ -# Config for multi-device full finetuning in full_finetune_distributed.py -# using a Llama2 7B model +# Config for single-device full finetuning +# using a Llama2 20M param tiny model # # This config assumes that you've run the following command before launching -# this run: -# tune download --repo-id meta-llama/Llama-2-7b \ -# --hf-token \ -# --output-dir /tmp/llama2 +# You can gran the model checkpoint using +# wget https://ossci-datasets.s3.amazonaws.com/torchtune/small-ckpt-tune-03082024.pt +# place it in the tiny_llama directory. # -# To launch on 4 devices, run the following command from root: -# tune --nnodes 1 --nproc_per_node 4 full_finetune_distributed \ -# --config llama2/7B_full \ +# and use the same tokenizer as llama2 # -# You can add specific overrides through the command line. For example -# to override the checkpointer directory while launching training -# you can run: -# tune --nnodes 1 --nproc_per_node 4 full_finetune_distributed \ -# --config llama2/7B_full \ -# checkpointer.checkpoint_dir= -# -# This config works best when the model is being fine-tuned on 2+ GPUs. -# Single device full finetuning requires more memory optimizations. It's -# best to use 7B_full_single_device.yaml for those cases - +# tune run full_finetune_single_device \ +# --config recipes/configs/llama2/tiny_llama.yaml # Tokenizer tokenizer: _component_: torchtune.models.llama2.llama2_tokenizer - path: /tmp/llama2/tokenizer.model + path: tiny_llama/tokenizer.model # Dataset dataset: @@ -37,32 +25,40 @@ shuffle: True # Model Arguments model: - _component_: torchtune.models.llama2.llama2_tiny + _component_: torchtune.models.llama2.llama2 + vocab_size: 32000 + num_layers: 4 + num_heads: 16 + embed_dim: 256 + max_seq_len: 2048 + norm_eps: 1e-5 + num_kv_heads: 8 checkpointer: _component_: torchtune.utils.FullModelTorchTuneCheckpointer checkpoint_dir: tiny_llama - checkpoint_files: [model.pt] + checkpoint_files: ["small-ckpt-tune-03082024.pt"] recipe_checkpoint: null - output_dir: tiny_llama + output_dir: tiny_llama/checkpoints model_type: LLAMA2 resume_from_checkpoint: False # Fine-tuning arguments batch_size: 2 -epochs: 3 +epochs: 2 optimizer: _component_: torch.optim.AdamW lr: 2e-5 loss: _component_: torch.nn.CrossEntropyLoss -max_steps_per_epoch: null +max_steps_per_epoch: 20 gradient_accumulation_steps: 1 optimizer_in_bwd: False +compile: False # Training env -device: cuda +device: mps # Distributed cpu_offload: False @@ -71,10 +67,12 @@ cpu_offload: False enable_activation_checkpointing: True # Reduced precision -dtype: bf16 - +dtype: fp32 # Logging output_dir: /tmp/full_finetune_output + + + metric_logger: _component_: torchtune.utils.metric_logging.WandBLogger project: torchtune From b64300f2c88a9d92ca799b0d55121331983e6193 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 18:11:20 +0200 Subject: [PATCH 56/76] add logging --- torchtune/utils/metric_logging.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index d76dcfda77..44d9b610c3 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -16,12 +16,14 @@ from torch import Tensor from omegaconf import OmegaConf, DictConfig +from torchtune.utils import get_logger from torchtune.utils._device import _get_local_rank from torchtune.utils._distributed import get_world_size_and_rank from typing_extensions import Protocol Scalar = Union[Tensor, ndarray, int, float] +log = get_logger("DEBUG") class MetricLoggerInterface(Protocol): """Abstract metric logger.""" @@ -179,9 +181,10 @@ def __init__( self.local_rank = _get_local_rank() self.local_rank = 0 if self.local_rank is None else self.local_rank - if ((self.log_strategy == "main" and self.rank == 0) - or self.log_strategy == "all" - or (self.log_strategy == "node" and self.local_rank == 0) + if ( + (self.log_strategy == "main" and self.rank == 0) + or (self.log_strategy == "node" and self.local_rank == 0) + or self.log_strategy == "all" ): self._wandb.init( project=project, @@ -207,11 +210,11 @@ def log_config(self, config: DictConfig) -> None: OmegaConf.save(config, output_config_fname) try: - print(f"Logging {output_config_fname} to W&B under Files") + log.info(f"Logging {output_config_fname} to W&B under Files") self._wandb.save(output_config_fname, base_path=output_config_fname.parent) except Exception as e: - print(f"Error saving {output_config_fname} to W&B.\nError: \n{e}") + log.warning(f"Error saving {output_config_fname} to W&B.\nError: \n{e}") def log(self, name: str, data: Scalar, step: int) -> None: if self._wandb.run: From 3d18e6958ec7040fcfe4e963f3fd94a6e24cf2ec Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 18:13:06 +0200 Subject: [PATCH 57/76] typos --- torchtune/utils/metric_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 44d9b610c3..4c9638cef5 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -45,7 +45,7 @@ def log_config(self, config: DictConfig) -> None: """Logs the config Args: - config (OmegaConf): config to log + config (DictConfig): config to log """ pass def log_dict(self, payload: Mapping[str, Scalar], step: int) -> None: From 0b7c13a81a9d5b475ad5aa019441ed4703314708 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 18:17:12 +0200 Subject: [PATCH 58/76] remove big yaml --- docs/source/examples/wandb_logging.rst | 76 ++------------------------ 1 file changed, 4 insertions(+), 72 deletions(-) diff --git a/docs/source/examples/wandb_logging.rst b/docs/source/examples/wandb_logging.rst index 5489797114..aa5dc16b67 100644 --- a/docs/source/examples/wandb_logging.rst +++ b/docs/source/examples/wandb_logging.rst @@ -15,56 +15,6 @@ Torchtune supports logging your training runs to [Weights & Biases](https://wand pip install wandb -An example config for enabling Weights & Biases logging on the full llama2 7b finetune recipe is as follows: - -.. code-block:: yaml - - # Tokenizer - tokenizer: - _component_: torchtune.models.llama2.llama2_tokenizer - path: /tmp/tokenizer.model - - # Dataset - dataset: - _component_: torchtune.datasets.alpaca_dataset - shuffle: True - - # Model Arguments - model: - _component_: torchtune.models.llama2.llama2_7b - - checkpointer: - _component_: torchtune.utils.FullModelMetaCheckpointer - checkpoint_dir: /tmp/llama2 - checkpoint_files: [consolidated.00.pth] - recipe_checkpoint: null - output_dir: /tmp/llama2 - model_type: LLAMA2 - resume_from_checkpoint: False - - # Fine-tuning arguments - batch_size: 2 - epochs: 3 - optimizer: - _component_: torch.optim.SGD - lr: 2e-5 - loss: - _component_: torch.nn.CrossEntropyLoss - output_dir: /tmp/alpaca-llama2-finetune - - device: cuda - dtype: bf16 - - enable_activation_checkpointing: True - - log_every_n_steps: 1 - - metric_logger: - _component_: torchtune.utils.metric_logging.WandBLogger - project: torchtune - log_model: checkpoint - - Metric Logger ------------- @@ -95,26 +45,7 @@ A suggested approach would be something like this: .. code-block:: python def save_checkpoint(self, epoch: int) -> None: - ckpt_dict = {utils.MODEL_KEY: self._model.state_dict()} - # if training is in-progress, checkpoint the optimizer state as well - if epoch + 1 < self.total_epochs: - ckpt_dict.update( - { - utils.SEED_KEY: self.seed, - utils.EPOCHS_KEY: self.epochs_run, - utils.TOTAL_EPOCHS_KEY: self.total_epochs, - utils.MAX_STEPS_KEY: self.max_steps_per_epoch, - } - ) - if not self._optimizer_in_bwd: - ckpt_dict[utils.OPT_KEY] = self._optimizer.state_dict() - else: - ckpt_dict[utils.OPT_KEY] = self._optim_ckpt_wrapper.state_dict() - self._checkpointer.save_checkpoint( - ckpt_dict, - epoch=epoch, - intermediate_checkpoint=(epoch + 1 < self.total_epochs), - ) + ... ## Let's save the checkpoint to W&B ## depending on the Checkpointer Class the file will be named differently ## Here it is an example for the full_finetune case @@ -124,7 +55,9 @@ A suggested approach would be something like this: wandb_at = wandb.Artifact( name=f"torchtune_model_{epoch}", type="model", + # description of the model checkpoint description="Model checkpoint", + # you can add whatever metadata you want as a dict metadata={ utils.SEED_KEY: self.seed, utils.EPOCHS_KEY: self.epochs_run, @@ -133,5 +66,4 @@ A suggested approach would be something like this: } ) wandb_at.add_file(checkpoint_file) - wandb.log_artifact(wandb_at) - + wandb.log_artifact(wandb_at) From 80bd373dfbc0e0737576e5540f83a7773889dda2 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 18:19:35 +0200 Subject: [PATCH 59/76] add tok --- recipes/configs/llama2/tiny_llama.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipes/configs/llama2/tiny_llama.yaml b/recipes/configs/llama2/tiny_llama.yaml index 6cf9bb61af..173f9e0eb0 100644 --- a/recipes/configs/llama2/tiny_llama.yaml +++ b/recipes/configs/llama2/tiny_llama.yaml @@ -5,6 +5,8 @@ # You can gran the model checkpoint using # wget https://ossci-datasets.s3.amazonaws.com/torchtune/small-ckpt-tune-03082024.pt # place it in the tiny_llama directory. +# The tokenizer can be downloaded from: +# wget https://ossci-datasets.s3.amazonaws.com/torchtune/tokenizer.model # # and use the same tokenizer as llama2 # From 29a5252b1268695556400e638a68370c8bff047c Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 19:05:26 +0200 Subject: [PATCH 60/76] update memory logging logic --- recipes/full_finetune_single_device.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/recipes/full_finetune_single_device.py b/recipes/full_finetune_single_device.py index 03359b7df3..2b84d09829 100644 --- a/recipes/full_finetune_single_device.py +++ b/recipes/full_finetune_single_device.py @@ -235,8 +235,7 @@ def _setup_model( log.info("Compiling model with torch.compile...") model = utils.wrap_compile(model) memory_stats = utils.memory_stats_log(device=self._device) - log.info(f"Memory Stats:\n{memory_stats}") - log.info(f"Model trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad)/1e6:,.2f}M") + log.info(f"Memory Stats after model init:\n{memory_stats}") return model def _setup_optimizer( @@ -418,7 +417,6 @@ def train(self) -> None: if self.total_training_steps % self._log_peak_memory_every_n_steps == 0: memory_stats = utils.memory_stats_log(device=self._device) self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) - log.info(f"Memory Stats:\n{memory_stats}") self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) From c040735155ee5bb131f4cb44f553da7598b19898 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 19:05:49 +0200 Subject: [PATCH 61/76] fix output class --- torchtune/utils/memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchtune/utils/memory.py b/torchtune/utils/memory.py index 1581f38dfa..f853b1edb3 100644 --- a/torchtune/utils/memory.py +++ b/torchtune/utils/memory.py @@ -160,7 +160,7 @@ def optim_step(param) -> None: p.register_post_accumulate_grad_hook(optim_step) -def memory_stats_log(device: torch.device, reset_stats: bool = True) -> None: +def memory_stats_log(device: torch.device, reset_stats: bool = True) -> dict: """ Computes a memory summary for the passed in device. If ``reset_stats`` is ``True``, this will also reset CUDA's peak memory tracking. This is useful to get data around relative use of peak From 706949873c24b09a5060b997edb75202445c24d4 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 19:07:01 +0200 Subject: [PATCH 62/76] remove unused imports --- torchtune/utils/metric_logging.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 4c9638cef5..39fcb20f99 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -3,11 +3,9 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import argparse import os import sys import time -import yaml from pathlib import Path from typing import Mapping, Optional, Union From f319fbf181dccb7a6bf79d704e1f55bf81b72e9c Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 19:11:41 +0200 Subject: [PATCH 63/76] better docstrings --- torchtune/utils/metric_logging.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 39fcb20f99..8f431d269f 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -135,7 +135,10 @@ class WandBLogger(MetricLoggerInterface): project (str): WandB project name entity (Optional[str]): WandB entity name group (Optional[str]): WandB group name - log_strategy (Optional[str]): Strategy to use for logging. Options are "main", "node", "all". + log_strategy (Optional[str]): Strategy to use for logging. Options are "main", "node", "all". In case of "main" + only the main process will log to W&B. In case of "node" only the node's main process will log to W&B. In + case of "all" all processes will log to W&B. If you only have one node, "node" and "all" will have the same + effect. Default: "main" **kwargs: additional arguments to pass to wandb.init @@ -195,7 +198,13 @@ def __init__( ) def log_config(self, config: DictConfig) -> None: - "Logs the config to W&B. Also updates config on overview tab." + """Saves the config locally and also logs the config to W&B. The config is + stored in the same directory as the checkpoint. You can + see an example of the logged config to W&B in the following link: + https://wandb.ai/capecape/torchtune/runs/6053ofw0/files/torchtune_config_j67sb73v.yaml + Raises: + RuntimeError: If W&B run is not initialized. + """ if self._wandb.run: resolved = OmegaConf.to_container(config, resolve=True) self._wandb.config.update(resolved) From 8202bc9b5b1353b4e4d8edf45965c5b378b36755 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 19:24:28 +0200 Subject: [PATCH 64/76] integrate logging to other recipes --- recipes/full_finetune_distributed.py | 19 ++++++++++--------- recipes/gemma_full_finetune_distributed.py | 20 ++++++++++---------- recipes/lora_dpo_single_device.py | 16 ++++++++-------- recipes/lora_finetune_distributed.py | 16 ++++++++-------- recipes/lora_finetune_single_device.py | 17 +++++++++-------- 5 files changed, 45 insertions(+), 43 deletions(-) diff --git a/recipes/full_finetune_distributed.py b/recipes/full_finetune_distributed.py index 75e513defd..31aec92997 100644 --- a/recipes/full_finetune_distributed.py +++ b/recipes/full_finetune_distributed.py @@ -147,7 +147,11 @@ def setup(self, cfg: DictConfig) -> None: Sets up the recipe state correctly. This includes setting recipe attributes based on the ``resume_from_checkpoint`` flag. """ - self._metric_logger = config.instantiate(cfg.metric_logger) + if self._is_rank_zero: + self._metric_logger = config.instantiate(cfg.metric_logger) + + # log config with parameter override + self._metric_logger.log_config(cfg) ckpt_dict = self.load_checkpoint(cfg.checkpointer) @@ -267,11 +271,8 @@ def _setup_model( model, auto_wrap_policy={modules.TransformerDecoderLayer} ) if self._is_rank_zero: - log.info( - utils.memory_stats_log( - "Memory Stats after model init", device=self._device - ) - ) + memory_stats = utils.memory_stats_log(device=self._device) + log.info(f"Memory Stats after model init:\n{memory_stats}") # synchronize before training begins torch.distributed.barrier() @@ -451,9 +452,9 @@ def train(self) -> None: self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero ): - log.info( - utils.memory_stats_log("Memory Stats", device=self._device) - ) + # Log peak memory for iteration + memory_stats = utils.memory_stats_log(device=self._device) + self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/recipes/gemma_full_finetune_distributed.py b/recipes/gemma_full_finetune_distributed.py index cd918a9ebc..4b498fc697 100644 --- a/recipes/gemma_full_finetune_distributed.py +++ b/recipes/gemma_full_finetune_distributed.py @@ -146,7 +146,11 @@ def setup(self, cfg: DictConfig) -> None: Sets up the recipe state correctly. This includes setting recipe attributes based on the ``resume_from_checkpoint`` flag. """ - self._metric_logger = config.instantiate(cfg.metric_logger) + if self._is_rank_zero: + self._metric_logger = config.instantiate(cfg.metric_logger) + + # log config with parameter override + self._metric_logger.log_config(cfg) ckpt_dict = self.load_checkpoint(cfg.checkpointer) @@ -263,12 +267,8 @@ def _setup_model( model, auto_wrap_policy={modules.TransformerDecoderLayer} ) if self._is_rank_zero: - log.info( - utils.memory_stats_log( - "Memory Stats after model init", device=self._device - ) - ) - + memory_stats = utils.memory_stats_log(device=self._device) + log.info(f"Memory Stats after model init:\n{memory_stats}") # synchronize before training begins torch.distributed.barrier() @@ -458,9 +458,9 @@ def train(self) -> None: self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero ): - log.info( - utils.memory_stats_log("Memory Stats", device=self._device) - ) + # Log peak memory for iteration + memory_stats = utils.memory_stats_log(device=self._device) + self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/recipes/lora_dpo_single_device.py b/recipes/lora_dpo_single_device.py index 2c501f069b..cc68073fdf 100644 --- a/recipes/lora_dpo_single_device.py +++ b/recipes/lora_dpo_single_device.py @@ -148,6 +148,9 @@ def setup(self, cfg: DictConfig) -> None: """ self._metric_logger = config.instantiate(cfg.metric_logger) + # log config with parameter override + self._metric_logger.log_config(cfg) + checkpoint_dict = self.load_checkpoint(cfg_checkpointer=cfg.checkpointer) self._model = self._setup_model( @@ -252,11 +255,8 @@ def _setup_model( ) log.info(f"Model is initialized with precision {self._dtype}.") - log.info( - utils.memory_stats_log( - "Memory Stats after model init:", device=self._device - ) - ) + memory_stats = utils.memory_stats_log(device=self._device) + log.info(f"Memory Stats after model init:\n{memory_stats}") return model def _setup_optimizer( @@ -491,9 +491,9 @@ def train(self) -> None: self.total_training_steps += 1 # Log peak memory for iteration if self.total_training_steps % self._log_peak_memory_every_n_steps == 0: - log.info( - utils.memory_stats_log("Memory Stats:", device=self._device) - ) + # Log peak memory for iteration + memory_stats = utils.memory_stats_log(device=self._device) + self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/recipes/lora_finetune_distributed.py b/recipes/lora_finetune_distributed.py index ec89ee424a..810c627f56 100644 --- a/recipes/lora_finetune_distributed.py +++ b/recipes/lora_finetune_distributed.py @@ -168,6 +168,9 @@ def setup(self, cfg: DictConfig) -> None: if self._is_rank_zero: self._metric_logger = config.instantiate(cfg.metric_logger) + # log config with parameter override + self._metric_logger.log_config(cfg) + checkpoint_dict = self.load_checkpoint(cfg_checkpointer=cfg.checkpointer) self._model = self._setup_model( @@ -324,11 +327,8 @@ def _setup_model( model, auto_wrap_policy={modules.TransformerDecoderLayer} ) if self._is_rank_zero: - log.info( - utils.memory_stats_log( - "Memory Stats after model init:", device=self._device - ) - ) + memory_stats = utils.memory_stats_log(device=self._device) + log.info(f"Memory Stats after model init:\n{memory_stats}") # synchronize before training begins torch.distributed.barrier() @@ -542,9 +542,9 @@ def train(self) -> None: self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero ): - log.info( - utils.memory_stats_log("Memory Stats:", device=self._device) - ) + # Log peak memory for iteration + memory_stats = utils.memory_stats_log(device=self._device) + self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/recipes/lora_finetune_single_device.py b/recipes/lora_finetune_single_device.py index 197b166655..d2c4fdefa4 100644 --- a/recipes/lora_finetune_single_device.py +++ b/recipes/lora_finetune_single_device.py @@ -146,6 +146,10 @@ def setup(self, cfg: DictConfig) -> None: model, tokenizer, loss, optimizer, learning rate scheduler, sampler, and dataloader. """ self._metric_logger = config.instantiate(cfg.metric_logger) + + # log config with parameter override + self._metric_logger.log_config(cfg) + self._model_compile = cfg.compile checkpoint_dict = self.load_checkpoint(cfg_checkpointer=cfg.checkpointer) @@ -263,11 +267,8 @@ def _setup_model( if compile_model: log.info("Compiling model with torch.compile...") model = utils.wrap_compile(model) - log.info( - utils.memory_stats_log( - "Memory Stats after model init:", device=self._device - ) - ) + memory_stats = utils.memory_stats_log(device=self._device) + log.info(f"Memory Stats after model init:\n{memory_stats}") return model def _setup_optimizer( @@ -447,9 +448,9 @@ def train(self) -> None: self.total_training_steps % self._log_peak_memory_every_n_steps == 0 ): - log.info( - utils.memory_stats_log("Memory Stats:", device=self._device) - ) + # Log peak memory for iteration + memory_stats = utils.memory_stats_log(device=self._device) + self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) From 09f59e88c49b69fed7f496623e25bb8c33a68f20 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 23:01:59 +0200 Subject: [PATCH 65/76] remove tiny llama --- recipes/configs/llama2/tiny_llama.yaml | 88 -------------------------- 1 file changed, 88 deletions(-) delete mode 100644 recipes/configs/llama2/tiny_llama.yaml diff --git a/recipes/configs/llama2/tiny_llama.yaml b/recipes/configs/llama2/tiny_llama.yaml deleted file mode 100644 index 173f9e0eb0..0000000000 --- a/recipes/configs/llama2/tiny_llama.yaml +++ /dev/null @@ -1,88 +0,0 @@ -# Config for single-device full finetuning -# using a Llama2 20M param tiny model -# -# This config assumes that you've run the following command before launching -# You can gran the model checkpoint using -# wget https://ossci-datasets.s3.amazonaws.com/torchtune/small-ckpt-tune-03082024.pt -# place it in the tiny_llama directory. -# The tokenizer can be downloaded from: -# wget https://ossci-datasets.s3.amazonaws.com/torchtune/tokenizer.model -# -# and use the same tokenizer as llama2 -# -# tune run full_finetune_single_device \ -# --config recipes/configs/llama2/tiny_llama.yaml - -# Tokenizer -tokenizer: - _component_: torchtune.models.llama2.llama2_tokenizer - path: tiny_llama/tokenizer.model - -# Dataset -dataset: - _component_: torchtune.datasets.alpaca_dataset - train_on_input: True -seed: null -shuffle: True - -# Model Arguments -model: - _component_: torchtune.models.llama2.llama2 - vocab_size: 32000 - num_layers: 4 - num_heads: 16 - embed_dim: 256 - max_seq_len: 2048 - norm_eps: 1e-5 - num_kv_heads: 8 - -checkpointer: - _component_: torchtune.utils.FullModelTorchTuneCheckpointer - checkpoint_dir: tiny_llama - checkpoint_files: ["small-ckpt-tune-03082024.pt"] - recipe_checkpoint: null - output_dir: tiny_llama/checkpoints - model_type: LLAMA2 -resume_from_checkpoint: False - -# Fine-tuning arguments -batch_size: 2 -epochs: 2 -optimizer: - _component_: torch.optim.AdamW - lr: 2e-5 -loss: - _component_: torch.nn.CrossEntropyLoss -max_steps_per_epoch: 20 -gradient_accumulation_steps: 1 -optimizer_in_bwd: False -compile: False - - -# Training env -device: mps - -# Distributed -cpu_offload: False - -# Memory management -enable_activation_checkpointing: True - -# Reduced precision -dtype: fp32 -# Logging -output_dir: /tmp/full_finetune_output - - - -metric_logger: - _component_: torchtune.utils.metric_logging.WandBLogger - project: torchtune -log_every_n_steps: 1 - -# # Logging -# output_dir: /tmp/full_finetune_output -# metric_logger: -# _component_: torchtune.utils.metric_logging.DiskLogger -# log_dir: ${output_dir} -# log_every_n_steps: null \ No newline at end of file From ce42cc8a3545e50428630daea347c297ab177a53 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 23:04:15 +0200 Subject: [PATCH 66/76] destroy on rank zero --- recipes/full_finetune_distributed.py | 3 ++- recipes/gemma_full_finetune_distributed.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/recipes/full_finetune_distributed.py b/recipes/full_finetune_distributed.py index 31aec92997..7c371ec958 100644 --- a/recipes/full_finetune_distributed.py +++ b/recipes/full_finetune_distributed.py @@ -460,7 +460,8 @@ def train(self) -> None: self.save_checkpoint(epoch=curr_epoch) def cleanup(self) -> None: - self._metric_logger.close() + if self._is_rank_zero: + self._metric_logger.close() torch.distributed.destroy_process_group() diff --git a/recipes/gemma_full_finetune_distributed.py b/recipes/gemma_full_finetune_distributed.py index 4b498fc697..ac09fd2bc8 100644 --- a/recipes/gemma_full_finetune_distributed.py +++ b/recipes/gemma_full_finetune_distributed.py @@ -466,7 +466,8 @@ def train(self) -> None: self.save_checkpoint(epoch=curr_epoch) def cleanup(self) -> None: - self._metric_logger.close() + if self._is_rank_zero: + self._metric_logger.close() torch.distributed.destroy_process_group() From cf8a948de359febd27deb8b0f4b291de79398916 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 23:05:41 +0200 Subject: [PATCH 67/76] missing tab --- recipes/full_finetune_distributed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/full_finetune_distributed.py b/recipes/full_finetune_distributed.py index 7c371ec958..0576c0f6fb 100644 --- a/recipes/full_finetune_distributed.py +++ b/recipes/full_finetune_distributed.py @@ -452,7 +452,7 @@ def train(self) -> None: self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero ): - # Log peak memory for iteration + # Log peak memory for iteration memory_stats = utils.memory_stats_log(device=self._device) self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) From bfb8e98ef18445b9bae2e67745ced355e18c617d Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 23:06:19 +0200 Subject: [PATCH 68/76] missing tab --- recipes/lora_finetune_single_device.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipes/lora_finetune_single_device.py b/recipes/lora_finetune_single_device.py index d2c4fdefa4..cfee7cc5bd 100644 --- a/recipes/lora_finetune_single_device.py +++ b/recipes/lora_finetune_single_device.py @@ -448,9 +448,9 @@ def train(self) -> None: self.total_training_steps % self._log_peak_memory_every_n_steps == 0 ): - # Log peak memory for iteration - memory_stats = utils.memory_stats_log(device=self._device) - self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) + # Log peak memory for iteration + memory_stats = utils.memory_stats_log(device=self._device) + self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) From 820bef162a6838c24ec7a495896d52d20920958d Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 23:07:44 +0200 Subject: [PATCH 69/76] literal typing --- torchtune/utils/metric_logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 8f431d269f..9fd487a148 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -8,7 +8,7 @@ import time from pathlib import Path -from typing import Mapping, Optional, Union +from typing import Mapping, Optional, Union, Literal from numpy import ndarray from torch import Tensor @@ -164,7 +164,7 @@ def __init__( project: str = "torchtune", entity: Optional[str] = None, group: Optional[str] = None, - log_strategy: Optional[str] = "main", + log_strategy: Literal["main", "node", "all"] = "main", **kwargs, ): try: From 8fee701743d12f85ac7595c7cfdea1e6366e90dd Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 23:12:28 +0200 Subject: [PATCH 70/76] add wandb workspace screenshot --- .../source/_static/img/torchtune_workspace.png | Bin 0 -> 660860 bytes docs/source/examples/wandb_logging.rst | 9 +++++++++ 2 files changed, 9 insertions(+) create mode 100644 docs/source/_static/img/torchtune_workspace.png diff --git a/docs/source/_static/img/torchtune_workspace.png b/docs/source/_static/img/torchtune_workspace.png new file mode 100644 index 0000000000000000000000000000000000000000..4a94a53b7206738e378c3689e641edf3c76946e0 GIT binary patch literal 660860 zcmb@t1ymecn=Xt7_XG(V+})*dcXtU+aCc8|cXtmGECgxXonXNev~g?PZs(kDX1@FX zYi6?6xxK2os(01iwQIlryl+RUC`qFszeR?Efc-&`}@3&C^dx5V3}aEBUw%heZcn@^KnQ=9F6u zn=Az%R&*eak^UR5hD3K*Z4|9X7IA$e?RBS)ips)3h@p2fOxp>=hQJ2T&GSt@#|xk9 z$tpWkMeffCiKPmDD861nXSW1WAvU^9r2}aE0vNQQAXOrV+6V#yf>#A-xi1c<4kE(V z+Q+4OFGnwRWj$MaF3=+6@050STwKWjh)_loZz3Alb*m^`#;7k&puvPGkP81HKx0LZvH3cp zrZ67Sv=TsZg^&yGvMEEib>ZFb(*fXR(%qrBNG|1MzF6@+x9ph9hned1mLNm{XzC3jU^?@s2f)gb&&Q`F~k){=}(?-a*P>2F<$>KSV}7hq+z zh|~*T_(P|(KxS3&1ocRm>SWwgc>Uqx)yVbY4q6Hoy;cUIuNeG>r5PjBb{)~3IgRnR zX%>erX8VI@kYAE=RtxV;v)SQ}oZ#2<)3FE{>o67ucZM&*U3-`>KaGETM;zc_`eWL_ z%kKs1Znd7Ila*Pj@KQ8o508m7T3uo(5jk)*C7+mbNAaD<gk5-{%S`l=tTF}|Av?;5alf712yk-awWvyHS%apKhz^nxMi z5iW8y+3NLce4l*%;u{e0G|Bo7r_6*>9BcoBcwRqLkuX~ZzD!fkm9yh24o0Z7@sAv* zS0@0@_|xZ)v!*+NR^r0_(EYHh zB5^H`?YF(X+iKFPsnEUYe4+Y*AJ9jP9@ql$#654Ji3Q@g-XH%^r6VyiR5t%?ORLw! zn9o>2aVyQ+ z27w>Q1O0)KN&Y;Tx!0$wy~0pu`~Y5FuUi2;7os44hqH6X=Sf|TpXK5NL9P=9k-Wx} zzrMplcT&y$OfEx+2_pCscIgaL+l_M@0)+`fF2X1Sll=agz$rJn$BmVc5o(~|bv2Z& z2-Oe7uR+8nJU=kr_Fy?-=Ro82SU3@{e)?Qa*a7wQllV4<3=yf2oKpgWO^OjypdaZy z*}4oAc9;Z(tPFBe93mDHJE3kw3b}<0RRck#$h+ijevLi!5a z4-tAr#j3AHCB$cR+_1Cal*Cb{$=wL_f)@(fr(c07eXxLW9)&E^(5NP(8R(WFOJ=As zpCnC2bly@A?wV5R@M(q4n=zdMj0f7j0FEe>`fYfLTf!N5@#FhiZ#n&#J73rJd;Uc8 zA`EyF7;&?;fv6Dz-4pjy)C30`Y5(K?M^iDr_k_PNe{uSywIQ8GNED&ZM1fxA4^Wy? zJL5XjISV;soWs>8-)8WNkSBs=JSbVz;3CCGDc&S6Bzh#qCD$gj$&#j|q{gJ?q`;&m zrGAwAB)20ckQzs4sOnP&&a77YO6HqDF)VnYaN%;H<$`jD-J1QFStMFt?FADFw#yQlF>``7xU`ZV4vUO}T}4dTZl#uCMrz7^n_v;|uyaWZkia(ZV;X6bV( z+W^Ymb8@_m9xP=lu`kUnbGOB6u5S9Fm#J5(*WS$Ebl=R?oZGzBY-O!&Lpp!*y~mBw z{RjUN|Csyr;p+4qXRi~SD_mfI64n((ZH!l)fZC#VdYyhzyYx-l35`&jQIJvEx*Sjn z_;CNNvnC+r+3gAKzWnMz>O(A;XoOgo-Q{Ii;8jI%jc8L3V<^+B<5#d@QemQD?o7?>|v zFPW9v4t%BmQq|aIrPzF3pJ!=k=06c;-S=grLD$iB3Afq3e&S0_1JFEtxpf)oXko=Y zjy%SNl!{1`>BTv^#ud8*aHC^;s zG~#h>dEN4%B^#AViqed3D@UMbZOT9I0qZ87q?W{+)2nXThH!j_fzoW1#ki4{U-dpJ6HVj+3Yhi#|_8R1L6WqyTk)dbxErE zs&Sq(0uurw1Eqs3MGQnNxUc zGWr#A)(X}Zpkme)R%e|tFE_v8Tg|Q0Px>EG`UojR-~>!H`=4Y!58EV$7e}Zjs%ok# z7Y|8H#Q_M#qcgD^Id%=JjgKyVv^y_2Cw*J+YQBv;Z$3Z8dBhpTZo`(QnUw9wD9NXk z`5;G4HALemPm+Pj`hMc$NhC8)G|hU3IhV~~$z}2tjg>NhU(z1gYBcz@n4H+?WC}|T zq7s@FS!%9c?(&p{^P-FKnV8~ZRa9!PV^zlfDPmYg{Z$eD*ecBiMs+ly4S_EDmp<_AK7yt7IOe2vzKwwE$hZ}JFnrfUua&b^^CWjB z(B!(0Z=5fCkK9##@9QK-wl-&-Z@}~;F$yh81MZ^{NJmHO*5iF|ZyI@D(Vb*`Akuwl ztxmI!fuC#QUQulH`9R13vHT#jZp*elQr)&#rJ0TQetPL@DP92arR_jn=-Br(cScXc z&iC-npPU9_QvnQ~DK;r@QiP1S0yO(R+#e*(J1#2dx^}L%`fcfr>WwVdMtzAYbKgDG zs(w(*RP_dq)_qWhD43*!n?vuzI=izqm97~lv;9HkY`AO(d-Ua16*FD+&QIT{0>@@+ z&h@9B2XAXUvY)?xw>NGga%^Qo+vpF6yu?2DtC`&# z<&*4FB*J^{C&S+HmlJsScyC$SjO?sEZG^$?W2fQ#1_E!HTMa+79R;+Ox*F`&GXS)e zjIO$TcHcfM4;=nl6maZc`{wStIZ>iluWpl@|Wz{UCBqZ$v>&qgXf7bEv2{f?R zxhc%`TpeKZbn(-;GqCxF=GwChyX&nV5FERnee9UK@fBSHUC=B2^5pU%3B!lf!_VAL z@PY2vY~$2yS3;+TU-RAa1rvA$*pt&EiX}as_=IforO?I(s$(4;iVF$q?P|U%_3%v~ zyZE-R76Hr)(%4M|yDNMF$#a+VRZmxTHq_A?a*MHwu{}9dq6U=d8Z^JhkrUza=aK7F zHHu#U?y`uwO1{dU8yS96cjD{9`N10V1do2tc*4cP(Eg#URi1x*VD`&T;*6jZnk6x_e=QGy(Qe-a@3?>>JYVUxn35Fo#> zA-h)r%zxaCGF<@sALp;yAlINo)x~9HAxCvH7YhpqS8GQ%Dk2;(qyfoEM%NVz3YY5l z4lS!jc?KE(hmD4gn~tIaznP;w^LukgQwwG(1X=lu7s|90vB+f~!m z!bRNC9`aB(z<&|!U!DK`;=eiyviyGce_M;c7y7SO$Vvl{1zG-+Y5-)eZr4W$A&G1x zlrz#2lnX#YM!&fzAI0qLzPP*5UJvJ#>iUeHHC1QUaS*5{{_S}g{fdEfeN z>YF%4j(+f4M9PnFrb!7Z*fQ-UYm}?6J0Ft`nD>z=6EMGsiz$mC0?klg#eCj&SULdK zlxQwp15c|bXO`yWjAR*XsDxnGIXJ8=C&wKvv#&GvGUT`&!I=sYxy=tq&~PG9|8y%O z4E~ryWp26FQE5IamA|-Gd=dsbYafG;ffxk!Pqzdi&&Z+zI50H_5vZQ}|G{Kwpale` zN|LGElm`LHsm?QY4O(eS%;6eL)!oq<$d?ZcI$e>{U(Z;HX z)NjS|t}zoH?8{^Sn`(ycnA6I$t%G0w=~_Uh{3RMzh;WJl8VDIa7jt?F1RvZJ`VUFg z?M9VA3*$qQQ)zxFl~22$d*xJ+Uz;539Q;oQR-h(B2V*NxW$TDcVIhLvUG0LTfsI&HJzz)e%!~jy2@yxO z#-M8ZXk3MF}H#S#*0FXL}O4M1!T4j{|{yx2Y1veKleqU zl(FE(E3%F7T6xIXRNx!Jd zfqq(7WGvvpau*X3UUCgN{Bu33fEm2^y|#e(TAdp7;T!7jnJYNL(6|Go)0+TXK!nJd ze^;Ff-Z0lWRMmN>c+;1OZ3n+Zgm z_hEfRrbU7wV=YGd(Y86VOe~)UV8|4DZin(_GP>w*8-BAP4ZJ=fm_P_c_)FmMNnw!4 z3-(Ot*E(isw7^P;prLS1J@`Ks#)!8tPI>WJl_Js*IV*by%{!yqCp(ZJLe>i>!ArX6 zK-0LwZa21$k8xGvCMqM$NRv8J;Jf{oz7B!6obX=JP&(LZ)&%Fu%kevtdH zKA@Eh1)AnE4avVDbW%rKXv^t%aN`%<1?y^ap>=V-hUE!rSXoOl+Am zE23c3-~LV+bg-Kz&YIfIXB$>*REQ%;rUqwcc%CwyBUrYZLx6tJ>=|E)rtu20;K7v= z1@k$QS6^EniMivK%^E8ElBCwsC=WOW60=Ps{2cTIcs?yK4Hxj%g;o?N`W?@2+wzMU zackairz@8JQqnO#Zk5jvy)@gwqfT;YHFQ>+ef z(}RY46VPLO5}+UWDcAsc+MDx@sVsW=Mb27MMVd5_(_`F{;4Wsoj`gtUr;W)Lu3>D zZLuQIVOq4;NQj9=Ss+uT&K03t-D?&de~st!Stt~FVWS!?dpRDcSVJBl2^CCBq2jf$ zIE=t|lW*8Q6&#I1I+L4p8V z=*)4pxxI?w_3H!B!B5_6%ZYR&a$B*sHZiL!hvl1_DU3EfUW-?nS?>s%>rqCIbQLD% zNj_LxleMf!I%^TSgRj->J)qs03}hkJQlD3Ob(adn?~{5l2J#26Wd4vfRI|LmJOqt6 zG^)rI#j>km(^AYU*^D5iq~I$%b2F&UqhFl(#rW;4XfSXbi^74iQ-X{NdV-D4yJ7#- z)&zBPQb9pKapvlvqCmv$08vDQ)LlIUJ-EB%U)+nGc=$K>zE1o~z9zE}#ZY;6R2SQ} zbfJAxwCg`rbq^8zw_6`IW90J{;mbE#-DEcPxp?*M}0R^k#vou7U9OA8ar_)qiRC415@f_l0U@ z01sCN6Wlavw}5H8ImveYW4m~vH;j#&9WdXxXrSmi`7&8SD17}GU+H|>rQweVl8$v2 zqqnSk=6#dvPncQLx1n2DbHqgNdmX-g;*gyXpg1Vmi!~IkZpnTR&|S1MZb;;h5>_gG zz)<%q4%KR2n3;qvojdeUXNcZwzqC|$v?TGyP9Eu>ynLP@2zdYarLN=&9Za3vl@dQo zUr~a%?{EHVQN_19l6@%=Aj}bLxl^L*VhW?{A*29a&!<5d`DnbD*ZvVM{;yEz?BHWi zKran6-rpwdk8A|*?}Z?;H9&%BI7^)0qBJEeg#jZ4X(qHmB@Pb?u;!j|88rW@a{k6s zRqtFkv~By+M<;gd`R6MdAG$dg&>Q}XxfX=fd}e}Q5nB*lVd#a*6W5vIJ=d^l=~9ZD zyfQD{ev>|%vx_TdUtLhF*{qS6BInb#384LjfLw!fqH38FuoUv42V1b#fS&)cgeN3c{ z5M80N5SMYhq-j-Res7^osS5Dgu*EX+`dP7ZM9S(v!BTC@<6X>n#~h)&O^CVjZ9H>|2H z>Pwr~hP;==!knqOW)L#47O*5hD#U#8<>&g|)dd~OKz|g2P*>S#SgZ;j%INH;Tl$R* zQ(2*w_l>u-J#yPu_rv?-{Ea*B_2!-eR2XI%=pX?wMDK6|6{rw=u*$7opNX-3-2EUu z7Xc~;2EMY32>HVxyF2C`L=(kien5Qv%F|~c*R2^6-Tw8l-F6yDi_9V z+FU+6O86{`o4oJF!hPvNWAV*5AKkT+G#aeB_jo94LazzaMQ{Vq5<1x6Cra0U zAhvkB47oPdUc#H5G)1i}1sjD_x;64nGBP?>S)d^t^YZEih&pVk{U`yU1Ko+XOEfl^wYVuLx_i8 zbkLy=keB^rJCfI!(cI>7U{h1Vyb#NM;vGe_6KZ(g{8eo)SC)_F4!)aMQ=rmv1-2SG zt~7^BTm7JegVtu@t;>yDQt_tHhxHejW@3N!&vGM@H0nJBJnE*;^i)BoHZ^K{3>%Mz zT_7Wtx0v<))GE6d@lL=uuk&>k&i5NrA3T)x-?-PWw6kJ`-xP*K)HP}uRIdKCulz)L z9EsD>_+FTg-c#i%PFi=I^jR;G(vU+4V#tB=mh)XAqB-bl$gOH&-|r z@zGWJ8MQ99Ii>sZaR0cZc3JHM;0%T4|7+N0#zrwYRRlQlTb3%kzY$ zRa~qB^Dj%Nw^Do@t|h!2hc-Z1Sqf5XU{%H%j9xe7PM%v_RNFt7D)IKLEB%8WPgV{>FYTTY6wgz1Ir>6rdkXA?{A_Lr(dF1=H)4^0!& z?m!8nEkU&|SRf*({qEEIs*l!q71F*hWut<{`G4vpV#-4FyL1xuGR$w^yYRPgauVi1 z2WQHVl~N~pjM2B`N`A$m+4y=YNY{(+<|4}P=o zj&iq7gZb9khI|g~jo)w2pvvoM@tx$(W*J~FQB}5nB4w1ZIl5byc)UYMdbW!}Xa?o)Q53Mi}L-O!8llad9Bx^RBt1gi!en>Bk!hF^_2PB*;P zH^B7R(HQ{t7RO|G<2ma*@CEZt?gLs>BnnWqG+@t6qT{H42!`jqJIe%Ha+NhDYI`D<(S|SvNQ?I1n$~fC|h0dCrS>{6rYfR;>7^_)~<( zZ{j|n76wAtVfGUg9C+4(nhhN@0yS9X+qAOejdvkLxPj$Ul|*yl*|AynfrSv8g+XsR zWhf`Tq^U|dHg=um(uah}SaTKY-Rv{vx>uNYV z1$baKcqHnK)!jnm#YQH~&6lpVcB{POB<$2XRCE<53ghftu<=yRur$)?SU@|pYXtTm z)$=aRqcJ^6RqGwR|>5E*z;bc^Hf*G=U-%zPQu)5 z(Yr=22&LnEnsqLZJ1?bSx$%cKB%C!*2ZZ-wu+n$FZXnW^0hmoAO=@O)?l-xvgzg%r zMa!O~L=`*=jQ%t58jp!D^YTQ2=QML)9!PzBN)p*(<5_S&0QjpsRIe6<)vP8Q)~yrd zmTUn^cjapZo)c?pO6{jOi+<&KoQ57J62dzll5m)otkiaT2~+x7-6Xfq09ez}+iQD;gJoLu zjwhugQ8(?Agxo9=O@~F8dt@R0`G}x9en5P9M@||**VNv;ewkOF+W;>5x^#+h;l>$& zayn(ZEDJA$H8}5Y zU=QNapWY3C%J85k5(O%qY5=TE$~hHO;O50Dv%2E2Cc#v;-td_SPxA|bek$BWzS@)(NN!B}PFJzyE z3`^I9(3daE8Gf9v6{e%$+Ih~WHqbtC<6iI_T71VdmKjKYQBaOH0Uj-ps*LrdUkN&Q z82gW*v?X`&&YyNziKx`QNdd?P`sj@|Gd@{2TTLIZ2=<|0X{FMjUvHY%!Rxkvb_mdu z|E@D|@?E8ez!>pIwbRdfeVLDU#b{;(`34+bfCEo})SJ5J9QVn*Am$0+Eka83CbCf7 zV`{sXcprri|6Yl$^|kkgnvL^0%hO@`$dZ?YFiWC&fT51xgul=Fi7!jBQTWT12)i<6 z97eeU<$a;2Qxe5mq$}uW73mjR!|e;~{;@R{bDnz4wSBkm=ROF{tvZ5^^|yx8OPK+y zVqFpI8?c>GZE5K|YT(57x^&YmoV(WkIB*Y)&pu!D)5uWK!&jeC+7;#WvMz;@bHlBL z`A9n*A`sPIiIAV;=Fq^1Jr3lgH<-tTt;<;MwbyzZPJbT1;qyWKI_XJNuHV702`5}x zTaXbB#@G-0kvsm7`l#@$FifOuk==_i%_Ve?Dw1d?fg_a3gph_ouAUBOV7S0zTolHz zYOB!dCiD&jntS~hAc~zpn)BQG)09U09Iy41muc{0!`x@t->A>0`M!#6ck0uoJ)Jwe zdIVzlg|ubI9qdbLZ%jBGmsd6|u1U9_08?`J^sg@UN=%*<8L?lA(n>;={W2jaQ*n7H z{0ag|dn)|)>A@Zqc6P?Z=ItK1BTc++IXu5R5mk1fy#8Q%G&!2;&oMu zZ|_H*BkVME9#R+G@(}WGk?Uexu1pgl#etksjm(0j!XFeSo7rV}8xSN0K(?P+g zXFzO)y4S_DJH%GdNt)0jgQV$caiQ7ZuD044lTj8`)e<+}04uj`5J40{`ucmtSQFuG zk2;^g3nw}aVBL|UbK9E^To0)@_e6$H$5E6$E~72Ff+?X);@zoE#rD_7fw$Bil^ccS zi>voA0vZonT8`}<@PEmMzX6K>DMQr&&LI65|Z7}u(jXAEmf>@RjV@mYXYILWvE<@ei z*sUP*&c}`6I*NcjueEr677l}wv+6pvbvjt`r>FQ;g@#HR?Dw2oC{}phzl{8@e;yK z&kDQsZD++A*`D<-IXs!V_>OFepW=4|!tc z{y){}CI`R4n1iWzQV`#Ol?zdU3i?S0et?`Tja855q>`-|XG1;{&-rB$4!jA#z`L)X z2z3@0bCQ2+%w+8Pu=w$MITyd_ZhF*as2*{@nY--J*#I7FTUT1-+ErQ1febR+OxmaP zKwlJtC}z-bS649=Y#jNfk?Uc!kI`kP)jo)0V{!Hu(pN>d(Uw;ya;zyq0qdon39K`{ z5K@-t(x^_#<`*=|)}Anp>NA=@Crkw*dCA3~S=_ARY{rN@?j83Zc3 z*3w!j6KyNuUozh%8hwEn1U2>jKrq;v+>F53>+;uWYBjVj3bxP6XPX;;Qq}XHSB30- zUKLB%c%6j3Yd$0O$RLV&44`=OYv*Mv^^xzV=kr2x*mW^$cfnVe^bsVjot3flE5Pl> zWj@D-kC)HSIUv>5Jw+prDxUhF&gMwVW3Su>f0AZCllez%xhT)PipUCp-(P0|nFlX(9W@POphoFR_4@;Q>k4TQ>NsKJ%w z1$e3@)R^dDpOGRSmTj-p>*>73)53#vhByyozGV}9v3b88AztII$XQo&<)n|czFIJ! z?6Iu`jTj6^Eid9IKUu74jF0IZ{Zs$rMq$W6nuKlFeOo}))$pO)ZT`meZvs)!6io5$ zzs>SrZvh|5_b)j`^h<6mIVChK`h~24@G33~;lR&P)Em=Bqe+ zlhn0&Z%-|b@&rmZp8`ORIiy4M1@RifV01;b62F1PNAbw!Gr(0$0P`%HenetVSmTM=aiVoF!zNYmwzEg4UFiEeIfMxb~Tjz zq+zGOD-O2~Nn3tn6>L_q6;eBNtJn*U0PVa*41h_dD!`jRS*`=4rK_?P&x5ks=qd-8o04423F~7%t&Uw}N6$|*piW(J+V$hU9PUQ@RfiKr zB0fZjL|?C-cDKW1Sg!@zziZ?&atFTO`wOk7AqpE?Qpj6}&OOnA2RA5D%*5k(_z|JF znOB!xXl_u)4=FNxl6tQ)vp;k&=Iuo{V6GirsoG|T#_{+Ln<=XG-ixlif8W;CTfmW= z-1DT`>%t+hd*d~%*O0o)rIk2=EpA%X<#sJ9C;uTkr&IE0)%7Y>NkVkc`XSYUL&jTl zu=fB#ie?5jwy2qfz&wc!y0=3SI|NL`8Gv4d{tJ%(-;wHIYe*8)zi9&T9u}ew4Ur@o zg8(TyzKkKJSXc-s`|rrm11-W`q!vHelH0YSK&`%T(f=Y9uemz^;S{vjedfyU4da8` z+F;xEtz6!TPv3fxl!YpPh^wOSLNLd3Bp&=o@$!t`Oyno498U8iE+#0TM>xD0A^$sj|7r2bj{bxw(xr->84Dp7#lc z_!gqEzArW9eX$aze!nIVR43RFp!`E|EB^}h8Ar7W}0P6 zNjpaVtzH?LEh^k!$%F_kXn$H!XUhH)Ow4!(U^wv$m(U*L2ZJl4vX z0XZrrE$Vf>#;-jVyI1@6AnT8(<)PFz`%3xB?IKd4?ZqvgU> z)OSL+sQDIxE__uJq=@pjm^Fd6y37`zu6)YTUk(Z9d~V%^gdjjyQd21m!`GWjV&wkw zK+1Ozhgk+}M(H1CzBIXQx(qx!Wc0B*Z%5n^B5^Q+n5JjW&)e-eYi>=0vLu9od+PVD zy5dhel9F{P;2f$D2iegX{mYPb<0B1| z(o_COx&)_ff7Q%l2+E5@{9kZn1?*sJ?CEIh=jAs8C6Rkvl<_G_JF)_MaLifbHBAr|*yI91VggCcMIui{J#SPy~1y7qpXzSQ) z^?j5uk20D`)m^E7H~jhBcup6nU#iIXD&O_j4Xy2t^wlKKkxxim7nnncDe-S2*R|c* zd9G)SL>G6sk!zZ_rtq@u#z3>`dTcHYgtD zN_`F=&3h^|vO3x#6eOJt3w3)O$XBznmwjZYZXiE z4dTIvd$`27BMS);7C%K_NT#70Byd=?5gag}>*@_VbtXV>&eOfl6DOIbl~E(<`_fDF zD(YXgLERCE!+7Z>z0LP{89zsf->hh+nSE^62wf5V-?H_Ca%6gNVDg+UOYie9eOT|F z`h4S2^UHJ4XvV605?&iU+3;-#a=JVx7ChIpx<5M= z>WsU#RH+&kIg~IHZCTnKI3QjY8a>?;Yoo|A5e}i#Mwzgrei z@yV{4U#?zfcDHHXBE;YvK`g;R#WGd`P2`a#sd_f{E2Z0E)n^huLJSUei~*~0i<9z% zrMJ?MxRkz`bcJ6f#VlI*PMV8FuWUTk&VY5jhb)KP>^R)xDN^u$T#Mgjgko-OkGds# z;Ih`};Mn8d5Ttk{rff-7A#Ajw1mw=dz=_@!sHka((PM~9DlSYCJW^6-%xlqMGOzzJ zoR_Frc_*2or!&r9?Y1DVpOOs>G-$Yc@Ac#+F;tgxuE==5cleb{{kpa$Woy6iC(i!T;4Vqz;qx3ad4biD`CnCYc4`^O=-?SmG9H($ounyDJSjb`+KwQTH30&pgPnC%E^eG!(B6$rDGt*Qh1#0<1z(6RbOpFkWf z>e@`v6QN>XOdh(;S{jD$*9z1UoeSb2l@?>d55r{d+C11GsB6`;K*OGHJoI_q z=SliDkAasHD{Z~80r=d$f}y?OrL_DzaROcYG*7`(j*qckiy&i5h! z?JfP{D!_FbXf%aV9#cJA4;JtG0VidEGFd)2+a*7NRx>xdV_xsa@^Yv;z7(G~Ud`a> z&Ra3qle;;A$BetEeY`GpRVt9WdKoWzF2x+&7=);f6Xno(U`iMWss7dD?cJDyWKE&No!ncu=sN^ zM&{Z=aJiu|$SAhnoC5`PhpgTc{2!KXtoU+ZLwh6RSFzq5s&0K9b3u>}s0y#qF>Kdf7Rp!W|O+Cr~j*gSy*Mk(K;rWP)3 zzY2A?rEMJUJB?a@l3u~6d6pmn;=LqfS>}qTT5!z`hg0!TCauE znytIw&J3M3rcrJ5*@Y0y{CHT~39rH-E4aBI{=QRC)F9MEzAkqSC(t@!$#1D#Y(!s- z;f`jV750H;`Qo_s8Ev6V4R_kNLt+i>p|~bK^gcN&Sw>R65n+drgZ`RJMc==ijiK7B zr&mD^k_fWUKjg2xS5eowiup9y7&v%Hh7Jj6fQS*i-0si1?+|t_TSvwmV_a-0J(UGE zn6}B!w-dIxP5q=6=09Nql2|MEY@67G^X_3Mp;Z)uq4#{91H>(um> z^Ychfhj-(1<>|iuP&F)DS`yC>Jh)Pwmn@BZN0zOKYZ=iY&`_wWqWXbNA|$7boHC7*kIh$&WX{xVQ5rUl{zq75R;a z2Lg)DRf*;#!S;&7Dq@b#d5|K#Id{#EKhFsm*_9Yd@)nR!4zcRhNu7mjt`*ifzsN-P zw(F1TrWhy!;%)nnQ4M=a?(y?A6>61m<#~w6>K^*@DTdfPO`xBi!yQ$~y3oksln|6`Yh6Q)c6ZVNlBrXDK`HyWVEi%&mE4xI37(JD~nbA)pm* z3qYOAkI2|^RaQ6CMx9U5RCS@!X@(tX9$h=R^H z5?f{Jk(T~&s=Obvx-!~zG z=1D??FdV?=mQV*((E-pvl?jH)g6iIsgdjqOmC9?GpLlBt@S*IIg88ZmLq=Y))@G>G4@HTl0!XP1pRMo!Dd$s*_dyk1 z;a4+#3ZK||Zfeppjx?jHEN9OaRY(cO(DHUnU7de0$%xW96+t3m`>bPVj%H8Bdr0!$Ew>4!|9RUAxV)_m_?5u?F@`H>~a zMSikq=O_{&Iag6Z0CFg|*NXU(E5D)wrJXZHHJdi1clu=lw29}YIY;qNTUoyf<75&J zk)G&b;I@TVS=ni)nSIMvIU~xDQZ_=RszQWA0bvMwpzw_rU)E}>!ZU~D?knagJ}9+E z^Xsovwl{`$9GNmN^U23Vj^pB-zi1fFD%bpdhSn!N=mTHSqFPoeO^$Vsw5HI!TWhgF@@zH4m4jE`vYKeGc21CU(YWT8bjIF_ zx6Xz{n%Z&8LC_>L&Q`z0k6x1;441x@n_XXpcLOz;oPAe~^TSP-pu8mI-!N?nb-Cl> zPOYd7W7!rhd2{n$ojKC3^EGovP&WkS-Owf2oI>haIlfKOkr$+oV3ptejm1^Ay72o{ zwhpqeD&yxuchcJ6zeoG^oVf+3>I5VL7yM}rc3%@eR=oKyb*`Ux1 zs_oFG1QZq)Fv8Dptp*ZVZD2~*Z@EJL@Y|9v#{`f!gn_vqPFV1$iSMWDxgGGOyy`tZ zKVSjK*UR!%@xp-%Dxey zuV=9A4EP(lR;7a_qCn+IK}J7N@zN|yXQn2FHCfnaVD4LOMqU@UDjxvJ?$^VTX@%D7 zqOCy(WiP(A>A$Ll53YUob@Yb{{LCc~pmC-Slk+lo<%8@OpLJ zTRi%^QAf3kJ9AAAcs*L$nKLh9XoHBW>eSgBIsIbC!+Znq2hM&?=GNN_B9v`LiRBlk zc0=?~4P!wdk6ya~)5Cm5W3b1bQD5AHQrbwXcMQ3mm26wY-clhjTk>$87;rHVQQtz_ zywgZlt~vM8de(2^H#fHF=c$ZGRkHWxQXu-WwW_Hwq>1j>S#jl_XQi3QGb8; z9Ri2kU>rk2$rw`TmMO7$|D}`<`voZ(Jq|cqo;$&5q3~dP>2*pgoDGkN&?P%6j?>#H z#mUnCRg2#^n|6x+b`z1Gt}+EXXD}oLO)+RvdXs?+bFZXK#MM4aJG3n3yt`hAz%6T) zlxd|VJeBy|Bl@7Wrdpp%lu9b$0M{XYxn1YD6`S>l6bt$1WZ?|;gr2y3?Neg_HjEE3 zJo$g;?m^l)KK@oK@R#a~M%C8Qw}ukn!Vp#3I*5+(CQsSW-zb6uC$W(GD9Y5OV$3^G zUP3ZTExL?zhvyOv`o$j(#Tzd{*@xQu?$Q(m@zM1v@d6EIt)~}_v^1)#?;>+A+d=YE zb~6w7N)i$)md|^piBHxc2lTl!O`CN%V`X9fcILbW5mwq0B|NP8FY52SBp2jh$&{Qf2_knP zJx4ICG)+3iUuA9o!SKEvaZeM`t-!O@XMfmMSzU48H>D+oXLB-3x&E56cCR>33BjFH zLv?E=bWYD`@X1;A*Qk5bc9Ja}Gu7CJea+JI!WHadyz`AJGvVR&5_Uywdtxq$xbD7l zhio+D_j#r}#g?dn8W@*T7SY|KW2dRYrc&=8i*~oxgo`m|gJ3b%&FiHs zm877)|B@{M=3dU9RDU`jqU&1*ge)i0y=5$L6av46MCoWBRw21BT?Sz(wsX3Job-*1 z4;%goTMe*`IhfDU`Y}4TPi>A9cHD%i~cXLGXW%%;QIia3X>NUDoN^eh8(Q z`F&A&{sj!!qB0BkBJ+RvdatOaqPAUBKtV)6KtL&?D56vq=^a4?q)3%+p;xH^0t8f= zNN>`MAW|c}q=`!JH9{bvC)7|v3nba`J7=GZeZKwQtgDQSHP%@3o%5*+9pokFd|KT4 z`7TA1mN@WHuKgh0vC`GKFgKih=*4NDpR4YCb!DtC?{c61^P~pmQ@Vi^8-pCq)bY$14*@h-*p`oGybIf+wFb$G!`@)8QfF3+@2MMm4Y;u=t&keT zw#xJFmm`AeHst*@c9%dm_Q3^^)n5|~-%SNdCkptzN^+;gv29hWGs?fO+QaHgfo`0P z=p;a);xj6o{7z_VRi=yQ>FpwJHlmr{>S+DuV-W*GC#;{8nViC!i=Lvgc}V)H9KoCY zku&#Dq1IxomnvU2WDeG9h>f^lEYO!mDd|wUAEQrKt7R&-Z z>jq0DJpAnFDO4|t>(0hjKqVTscE#KtRRsh$cB6V-OFz2(B%DP2Zevm|&L(UGv>v;(B{rkqg>}eZ z`Lgz(m<{I-_e1fCFO8pP#R`r>W_3#?3Kulzx8M02wa6Fv?(N-E70>rD7i~L{3UGH8 zHj`~i{XN}Hd$`jzFICr|gjYYPADTp=)K%c=ak9QSmOTd5AV=vUw2}5(W-}OtYtR-E zxoF}$-DV*JCX!Z~d{xNRTl2m#n4nLUfiS~hrmV60e(-JJlo&z>UaY+0CfRPjRz>*T zy-mE!?MQtwbJ?^Xo(0QABiu_HH;Wg?RM$ygF0oNDPcDLer2j7FKZXlQj=O?HYGObd zP!j+(x>b-g6Truj^*ruGv^VFA&eX@tLPdb1MiYDI+X3cyK?{rdEIo?a^Q(pONsGqL z_Zht1LwOL%e^SfA@v}qb>)&V9VTmIkkGj9?I)7T&HQz}lV(x}7qV_G`0i&+wrVkF^ zk!n-O{oVr8Z3+;42Z66-K@U{-Xb!f}AyzlJG5Hru@?(DrX8Y5bi^_n+- zLdcfV@tbBtYqqSj#!kb+^Zr4jc{25oe&5gFs~p!t-wY?58B%EK_?eZerab%DUVm3y z1`-n9)1E2QzP_w1NKkC)>}v_Me3ywfHH=S}oE-09YYJpCufir~@E>S>kqBE4nAjy^ z*t+gsvXhmMzbE0@%&u>}nEEmZ)$RG%*Iu-u!ATM@S#`dxx53bCL-)%?2F5)l85N8& z%yuG>Fc{(0cQ=a4v9=}2R~Cfc94Yu}d%*A={8gnWw3K5OID?UnnVHpjalZr!pMN^5 z6@K-m6xBU4)iPj4RnrJuRKtF-{$-DkRnvz>R`wy0o1idv4HPdSB80~@c@X6O!wM$9O}!s zc1~Z>1(7u1`>2*n^QHJjIPnY@hm(}TAnWm0lO=6jJ9!E9 z+Jz@7Es8;DqwkZ*sov@T$C&WaGv`dy&1I<{$Q4wczsbM2w!HG5)q-3EoS@1YlYjj13Q0E%p=Q~f>5_fa4y%73LT{m(Cd#^3UXk^|4m zAQDRY8EG*Bh+P^_u&e2s9@o`TZsbWq=A9)5Fw79Lakh8}l*ANio>k62VC_gGT;KH} zG>u!FzxXv_a;2qA+p?N`k*O+UymWW@;^J+p(m3?I3}s+1zx>64%!=U4Uf#~O9CN!w zhV-HBO^hl~d{ElMZlte$`a5o@C2$ldMr2{(2oPr*D(uuauF4uLx9lZjcs`s8x@IrM z9P)h=KFnAvM}+qIwnxa`&bxPBaV*Kvv)6BXh{-^RZIqSMf#bI~dQR_A))GIJfL2A9 zEOPivgj;9$UKs%|DdvOCdb*y>*9=CboCMivL+l#)y8UW+WlMU3wCghy0=oT{G#dx# zh%~eu>gQ;QaYagJ(&)Sc=5_t-D$Qj<-=m%U?@AtLxb=-veJ^F!bWToE2 z-0JgEX)%baO@aPn1@%0Yo?FX}v19US{g*k`duP>7(L4m)lR>$7bi3jQ~dnpleC%R=-#WNNmMCp5lM)` zu06;(QXVeZ^;OP?<3iD5kL=6%SMUwVu#ffBIEhT6OO)oz8Ph!h@rv_{RA)enc#Y98 zQ({ln7piG^UPN`)q)i`%$KYMv!yQXd5W`6#K@S^`#` znz>qRvfuN&nRKeM1C^4w#jAO!>>u`%{=J#3awoV&n0&itG$ovBs(rI?DjhNoA-{`b zU)#7&N=ZY&^+5*xew2|kd!~DD&K9REVYw`%4PMjGX+t_0rE8*%0>Bs+m0icT=4wh* z7sVig(ES7s>T&v5j}0%iYoHYo{7>la{|3gM|C15>8^(3c?>@P}79YP4qr-z&;;kAKyYoDYd%O3KQtyK<6jw>*`Wbw3NxdnK~hnY=3!Kri3X z-?m{c;gGVg6j!-lc!I53UTX>X+!&z}na>3p=B3lbRbTKi@!ni7rscBiuDe~zMQ#lf zP)6cEf>#oj<=AOp9)eeP8!b)6Na`{|{tf-515qB&l5Yy;fmK>EL@v{=OPwU1!zjEB z0Etu7kK<5Kxf}+#j8pd0O#igHoee35Ns_;&ByCTMPL4h+OL{-U8N)+x z-_^9$J5NBgEnm*zZ}f6xp0O-dQ44v19HkZ@)HgeXgZy@{(+t-o+GY9n6y~RvdTY9F zZx^EqfQs{8tR1trGomBVhQ3N#gDE*xac<5fFP~2Y-2G(TiX08^&f2@p)*3SouiHxb zG=HmzHReHI#Lu3Ku?3@z?cZ>9cA;Lr=@JYMe0UT*H-Vlq z$UMHI5VoRQk2(fME;O`-em~fA-Xw80nOcpi!6X7zxSfw83^O9%zG74YjXZv_QOCZH zI#g;y@l;fm$A{L{rN4yc)Ehuz52U7T;#9NuuxAk)7i6UG3Dq7poubP>fX`xlS0&3X zU~Sc>jx{GgewU^rc9NbqTf2;&UG3|8zK}q}^=YXtezfuH>a{`OYNq;@w%^lG8O4$K z2spJLs-w%RN(vRJk|WF%w$E{eI|#TPAr;Zp+1>JUHG4K3d?gAu_4)if~k80npVGb}IzL9v{9w&EQ%xG#oHb}+2Q@iyn;sJ7e zyhE}mOl-&5BGglsawRM!@EaY2&wQ3XTao!(HM-%OU95y(vc$sY7m^MU+5T4u<*YwY@vq4es2_yLeeeVF$ec9Myt{vwjy9y?tz&nGdmF1T_vN!KE4~TE%NnGm8h+iMlAhs_bDNTm z7C=Vj&BylzPmqZx_|I+&m?{;}*qe5b1&;uD)Eoh^L&(*Qt$pe!- z>OlX|wA22H8Oc@m-fxUwKZNW!u~kT?Ag!_rwJ@sg;#=+>Wgr2i+An?3zNd_q5CL1N z0cpq_ZL7tI<{8{Bce)h^1-g8ir;8 ze`ZxQNAo(si{2LD^l+|w!>sEO8x3uU>PoiADoaz>-A|pGw!#zFgry8m`R$2UsHA`0 z+-M^+HR4oz$%dCo7eg;9WdIl~*@Mz5C&6HjVL%c}YjDdzx38~kJWHZPlNa){0j{OF zv#v_MIhd*>vVxa7t>QUWe6L!l$2_fnznF~uBpwrJfqx5mmG>w@2S;si*6#Mji;SKmGUFp|H9`slx@O)!sm~7F5Ko>y)|>n z&HVZ47=zE?pv~cH*r$t5oyE=NCtrNenr6mdegP^3wD8Hc4xh$%mkoaVT*jRy_Zu*^ zSqx7M5o+XM^LnH>E?j@pk|}4$T(AFOdIWkL)%gn!_)bkQ=^g)VgT#)l7M8`kmla$* zS+NS4=l=cAJR)e*%kiRCZqVIvU`npU&EeYZv8D1iasx3}5%L&3cu`q(k*hRyB!ln4I6B!oN}e ztrcj)Lb6p0J5ocAg}}wtgRO4MDZEV{WZRp4k$6ag2-E}yX1(;SgIAQmJVXPgSk7Lj zG5pFklt2~=_OmcHns zFPI@*c~oZTeEeBwFPfr+9h)s^l=LprFQZ&yPrL_qsREN<)U$GUVP|9qCWh4Nh!t&za!AVU+|O+K`XK01gi^ zZ5^6a&Y44T7(AMr4Dm399*TyO6bIf11+%usVuWi;sF8-NKcwVds!1~UxWAb@W!>Gt zay5lh1+@QU_)wV;`OVhTP-StjlyCpH_flRgf8}m!1OQqJLAGRL5GBKTLt^D=WWD(<*@uzau+R^gci0g4O82R8s$SQmVXqu;Azv1( zrZ?t1-9e2485+8$1iSVU^tkTFOSOH3U_{Hflg!=9Cm&{BH~~S{LsP$gwD(3;F>RW4 zUA*y&b0q#1qG6dAm+vIxpBHY%fo+=TF zH%%%$2Fy7-q-m!|ytz$|$sr-Yx--}Q6o(riQ1I&OmWo~PhGuDr;Bj-DCBSn|-@9(Z z<=wT7nbZT(jh9RZn`J55lmpyz=}=WOvc!>jRe?FQc;8aNf4 ztyV^@*BrTxKIH$L3qX@BZRe!RHqYtSAY6-%-&4epT1JhNt8 z{JjO2L%3@k%quLqh!2gi|LJJnCL!prGE#5g3gYA^(}zC|jFwFIJs0Fd=3dc2TUWmg zC_c&O_Ia~Z0QxcVXk-h0+^K(Q%98>_?4;T+eq&@{)9rss8-8FeHj@a|u~i4FKAM7> zsFrt$MOm@JWN!YjXzDt8U7|%;pvuLhcpxNwQp0fHr-`enk&h}H<#FwXj>DZbRO~fl z3t@X-i8ehmT)S=T=#7M&dudT^GJ!)=8aoCn@6Hp}pz@lUKiM_&Fh5RpwsZB|jrpko zEMwzQ1y1f_KN17$26kne!}sWdcx+KyFoj7yETf^m_Dq1MvEKJ;Oz2wq`DrC1s;iKb zHT;?NJDGSt?h21-=q=h6my4rRlzhzlMJhS}v z?4+nJW?j<@K4q$B!{CiK0ph9?JS@~lEk~xt_e&BhX`HWBAeLuNEsIot+~53wDX2lj zxNHr>j$_D*#RTU~DE56aDPw#|s3;5amXC~;r4~h~Z)DEQ*RM`-`vbzKt4-?HtVmGJ z<%S4?&ih&MqGfR5x9u)9ZlPd)%h17}N?Y)wN2v)s|MT==`?5|Oa$Wl#8%z!v<6DF5 zQohlhatSBkti4I4W^PPyvgyUUSTWf%`^r-IAWrseL5T`;TfXA~n?{mDIIH8H5_x|* z2wd+<3K>g2$=T+T3{JT^!rM8Q7~XQ$fq2d~=QGrMClSM5Fl+HNcRC>S@huw~Qqqu} zw)@_rNg?!W^&twJ^O*O2UhL~z1DIL`L-}$&Vj{9-JaOi3T)~6b+t7(gjqeYyYi*rQ zFPte#BC0r%N%ks^^0p5^nFTBb?(%hVQW@LOSM#+~Ntv-}A@l9oJ??Ay{DOkK+>MGX z&r+%cfcNJU=@u^Gy4#!Ix4|s!`gxUfCv4fHal}^VCu(?YkFx~nWxF6|jGI;V#!u%l=(+SxmOO(JR~PECe>T+E^ah z|ALdf0+J046_?y_td6Vx$~oF1*Ljq6ZDQ)(bsGz}_+z`o8(!mc(;YeaFFz$W&WD`;$ zp~1q1bWiAEvPV}SHzek$1e2q>Q<;LYt=yfuCD2};$>bMk?Tx^j=mDRDN2RZgwA+>< zJ^c3+kFUJ7k3D z$2=yk5k-;av{sv#ON$@HS@x}5&=ja|5przi1s`0>;_|GMP>_lN)xZ9N>{^J#k39Hm ziv|fz;{;hwMRs@tsK5y~tmRWqHH*%HhV%{~1vlB)VtA?h9)pyt>Y)t8I%~&~9z6OE z%wOS~CSD>-ey#17fXrLgmg@SO9-6BEgk5lfKt&rW4?#lWbH@?rmG&*x``kgwUsd14 z=@(7fhZpH{4M{+;We=$ z@N}T6nP;1`QT{(?H2&61+`m9DUd6{$uktNbA*9-0+4Tc+T*Ue%*^I7sZFF- zS&Vp$zwfxcCNdp@Gi!=dQ!M9Sk>GYm?Bz)OQ9b@+ahxda?OyKlKc=8RaZ#akWDJMq zHC3DBKm${!R^nTE?)%g<^^(>da1I%W{%~{V$bF; zJv{v(##5OgmaN#<+#!2weaDgX@^=kz+wdEEZF3%1`wJfi(^Y?;))yN(YPX#*vDbMR zSK*-FXS4`-6!+71VmmxrS^MeLbcE88R{QN}i0)^}N?)By2DbeMUgcaZR4D&rRn=&Z z&?E4*DlJ=k;zj^4o$X)^1utLM&tPDq_G-sb4Md6=e7M`>}wzD86otucw8YPouuG_;*A%- zlxr5=T_6}!8J>)*xto+HvC!|P4cF`w*?kS+2J)M_v(rIxrAC%Rall#|llO900h2_^ zPbpiuQc0zx#fDFl$2R%;6EFCQNm{$t_y2m%QM%K%zddr3KTfTbN@3*?Dq-asDRlYl zTmCVVPBI!=r9q`xsXa)274oLJU2Vy>u7Z6TvQ+(%(#Y#i1N8-Aj`Hj_sRH_!S95#=&pTwJ&AdAF{&4QVqnD+(ZOJN%HI6Hx=TwN7r-_4-VsmD)Mrepp#W&=>P% z_Nm5Ks;bQ~#{_p5`Q6D%2%7F?ALYffT#1vM;kot}@;0D^)j~}Y@bin`!j=^`ZhpzW z_?;R zb(*LT?9`oVCF~n+6GfOQK2I8@PBE`XSBCI&O$Gw-np3lTO8AmNjziqmCg4jP%szFcG?_3}tV9S?< z@-p3{cv&7dje-zYi7l}lO>Bt~Du4OOnx1Z00ZH*CMOwbGREJf}fa;%II*I!dr9zfu zOw~b}dWxsn0fEv05ZLj_$L8Y;E6&lwKAg{emclOd0`@nx+Kp4MPDs%p@1@REqKOBF z;5+VBf4|TMGv`r1t#2=!DAX+^5f^XcsF&&3pIO~!r}gQ)4#BP*jg|i(B`0<599;>^ z4$$+plR?<`4WA>jHgZ#x4=z4{OCYu!XZeAHKZyQv43MaQp?$ z*l{DPswVRsj357oA)ZN^K;Pz>zrup&o*7=T8rrB!Ry(vI+18Qrm)2wUf03iDSx~M$3Kb&_%)|h z%x*jn8}dS z1jxUhyx$Gn8_M-ud~@jCOsdWbZsS~3v>!Fv>e^T1dSHK}NZ(rRO^G(*j+H?0H3Q(Md5)%Gij| zqm;{fVc+c~5}@#9TOaLA$;ZnD;ICe>;l;NHssG>&J8fy8_uoA2QEXyA>kYLXFCyU; z=|pC;!QscZusQzrKsv&2XZwr~g~yfv^Qo^e?b^vooL6o|NV}ryQQgQnm@U>CPplbO z@8l}Nob1?#&<{8=;QW}S805P-AZc1CDUfWp@xaq_H0PzrU}Pd zC9Q@SLFf%JxWVY#_ODyUwi`3)&U|fJxilhq0MGkiDP+$jrAA$T+i$$wGmOTWsZU5v z1+s7?#N(CvZAK){{b$$@YbHikf_ns9!&r!oxud&DAxy*1`xaxyG(zBUtXB$v)Q0Mv zz+SVG1{4l86e5jt4&w7wg(kJ_Q$LN;?7B{dfOO***FkkgFX=aA`uxIR12rqa_*$klBxJ+$RQ?95XW+QMf}{>yVfrnfCb&WY*4^=i1JkaHGoqamjFo#thh4I|J%39&0)bSh46mak8POovpD zz}Z^ZOCwmc@~wtNKEw9IaCAj|;VHRJQSRlrU=>dlnh;d&Rzt`}f^nr;^?1wg>=|nj zwzL~Cx`?P#i;{dE%spq}F|$^{)1hZ3F5z^aBua<9BH$zI`(6k5hy3pzSnMatP%D`{$?Sk>k`77xUK^mwR)brvhOu=1HoH@O0`g13z1keq}w{ zn8r@K+mn9}Y_t_zDMAX&`d!98^1~Je6TJz*UG2FnCBcYuv=?vw|1YA(k!aq)mUY~I z#T`U>n}p~+?+xpWgEra|^beA8{^82y5${ToECg^zKl{na3OLllfc3Isu@}>*HSi3# zvA4io&dp*oaE>G}7oKm^@{k1JValYeGytW-a@Kmk&>M@*f^O9$pMyo2s~o&NioH-? zW7&Bq$sarytb)o4-x~jZI5@-L)jG>)1L+5C9bXrA@ch6zdl*#+#e}tc(GmiY9}w;M z)#pLz_xqINU+TdNHd$omXNYMF(Pa8F=xQvw1g%+EYelrlA zbHAoyBpA7OiP1d*6K>w5UT>mb=wa>o&D|=-wq@=t#f6`oe)~3dCp! zrP_?OfI0ua_8faWMMqi;2iRW5`{JyM8SD8o`%?n?bJjS->z(g5B%<)QK(q71f>x1s z*E?J6MMrq2USbynafge6KHbv1WzcW_QwfQLr2e?9;lcLHQd#Cj^#^VhGl3>f!d_lI| z;;hjg5=dUyKb^H5I(9MZNaHbYZkFNGT$cpKr%!qLB;J>q6@qCaR+I|2^mGyT@4Wfi zAYPB#um82b5I${544I0k?woy1T@-kn$)g$R0LY({O?MRs8M!l@4CexPmpLh$K{lXb7< zG_^tr@pq4W8`kHjK3lwECsS3g#)>Qj>Fz zq^dH1*{ydA!;IMdFmTjDM#4q4RgbDHf{gIBdp4_0rw+}R2(G`K zCmz^NmhZ0&KZCG*&#mfw3N$AqviTh(o&2|De(~myGunPz>RKFu{7fU($!Ol0 z03n#!+T6-)a6G9sk0xZ})E)zBuqoxdQA673k6B^jVW<2>;Zyrklclx*h?6|N^>x1j zAmS=}5oIN}Hd)pilWw^MD+K9=bbpIMWBd+;_73N`<@-AuuS`KyPsQWh9DB>$qR!(y zF_a~^Y@b2RN3X32J6;4B`K)+QLs9_LZJX`3VwkTa3L<@21NGY_(GXi~>-EuqtumIf z3vp=5vD8>C&28_7qr4-SoVXgo^Xu6RFL#1AdmL^EX< zLhMmf8XxAF)`oqnH%KVR7C0G!AD)l|V>ITVbUK;7^M9Uq|MI8q*01iLhg*=6`P@RP zqETs&e8{DD7=U*;<8{U~HpQ*~a8HHCckOICKh#^A9-fVAO26}zzHD>e&lqqh&G>R9 zcQ=mfBtD7q14Yp2C6~t{^?hv6%E!+zlql8a|l1Z=L%=;mS{QT>-;y z4lpl}`!MTr>Y?x@MW|VQx>-@n>UrFx?r-S0_8V|2T*6ZJa5nq}v2>`Ofmf&wzn0b@ z)=^!?_2bhv7a<^*bW=H~{fG6enFrLkQZnRih#5%@7qn(FUWt@6k3e%@?5Af=Ao@=v zH_aydEM<6Sy>lF2hVJvIkxTZWBa-f&#a0YYOmRgQ-76e4M6*Gr0$X204;rg?pymZF zBA7MaQZao23lTN@rh~f`v*{|_fe!iLQ1dspmB(2gwa`MD&($SYqgF4Cpnz|d^zoB; zIU>I)Dir;DTP3HK_=3^wv7Drd!uL{AXOi=U?70=EkO7vFjaYe-*cZe7lMluXB(Au} zZz4gx=bAOM&Nv`#Fj%eiw#6@3WQw1S zKQd1Qjh=%vWuYJ|OTeUFN~LXT$~L25ob6Jv zGYbFijCy`f!N48Y@$Q9x0nf={6h^>di7FRQQOmnV{&H!izB5%r@`yYo%HJj4y!b4a z!D~>A*fNaI5;kza#L4sR^gD;2p$8tRx@;t~D)0NQHYo=`XOJ?@iq3bHN$#M3yxa_M zFFORd&d+)xN&xIZw)i{qo`al@Z>Z?F>N~~I>R=9)>28nk+nDs-U*&Q}D!*OYYbc~N zx9|?_7mm7PwUvM+O!|{0MeuO-QRnUQWqZ3EyLsP-wKd5Y@Pcr>g1)yv$iBX> z<=YymrDp)d8OHGJ6;UHz$+77I4hG62#h%RaJcWrTm=fQs+tNR90l zs-4zLkafQ+c|VNv;1RKl;g3ZvU1C6vlsrd)A-<(vNwCGHS$|#hm%&Yp&fXd~wp@G8 zOGDO)hWOiaj(xI^U>>8wS(seu0C_yIHhj{o?3_#d-fy_AdDy5bkxP%Ived7&5lVWN z>bCB1x+KdBAt|FQw1g8J;|SKo673q9ddanIdoImeB@AO3bXaDnYb**Ko?Mq`)9&LL zL*Ra_?{7(-p&HG;#l)(9y|suQ@a=e3^VoiWDI3W(!>dI4J>Y(%;}vzh;CJdUzud6x z!HhwRtniL?sGgo%Q`tF&KAouLjG6t>${hVepxHuiKN_3+8|LeiP4ex5mxaHp2Rx(@!f!EfHJ zI-(03k_FYwO|aCKPl4MZ@w+q3!>FcKsj<8I#TCQfJO7eBP&?!g&B>sFTGI%42#R+( zRp1mdQ2g>9`qyroxT&461%l!}#9ARUy#1{H$R1tXIE{GT=Z6NF>i;M3 z{tph1G!y?ZOA{hXfG4l-OMGv+MDloj)_HjgayoR>I!ie@^?8|x_ zyGfsgcz2v0s#!ILb*Jv+c;uxyf!5c1-fit{RlmrHCJn&Y`&$K0oP3m1W!c_ZB9F71 z2DMCcpJCbBY7_-=FrpOrA03*Xpw|MgCw3`Wf5D7XkxPK1&>nylt|M4n)Ii4sq;wwH z<(sD<+V6;Y&(4b3E7k%$3O)2!9QPJl=C`E>V<*QU7Okmb{x!8 z0mM|In)eSaGHe%+|MImiMJV}mq7D>nV*G)`KK|Kqt*eGztja&!rLmBATvd4}?CgHO zWK({!RgIcGuRvC1pKnD)W#+Te=NeXjruGIntk%L^f}YXSGdKkKYu+U!zErk8dbar~ zS5rf1Xt=^)WR5JpbRsWZs(!Ws@JJQcZ(_ep@1IFx>cWb7$Imn;t*K4J$`u`-Q<&dg zi1~UuY>el6>R7!|NZQaX9GkpoX%hC(PavhwKc%xgKhEPk2Kb*79n22rc`cSYP7 zvd@yAS*!613^@$9D|y0yXqsy%TI?Q0tdU(G_rE#yEfD?)7=hM6&@n> z-2v!|Ni6jn!Ow)=7(}@TNqjagjb{fvP+mvVlBhUh&9oAbRi_R5{#;B2hzkqzMxgM~P1 zuUX=clC@?{4yo18YN{lg(o)Lu(I1fk$l#FrR%eSN`t~7uBjiAp0Gv1d+nRhX%yo;a z0-2_c%xUY%?W(iG4U<*D4j28vs{QRm&h0B>;)%TIQ&ka zbcWqmWUA;9zn@;W+Z#gkUMBvau|6dqbO(%#PY);@I#3x4zuA5CFwQxY zh-Cq?<_8l&dZ(*kqK^v;O`2&Q`%Olq3!cfYHtm$Uv%W@}^s=XnG z5Gg+fw$F_M$+rtngs?y`48H}O`DTs5OKdAiI7XrsPjV34XN=vtHb{bA@LvdawC$;I{>RXB%em0Au!eBseLK)Ym)x*to;|1ZMV-ShP#OgD>vJ>SVN0jJ)y_YI3r@@a(F{G5-I^B13&oJeN|GoRQf5Xs0@~Z+eiXS} zV^v22p+WKHktFa{o+sXUpqH=YYl#;pXxIAFr%iw#V6m~ z!c0WbI=9S{qHn^*qIdHZOwXcFfQLVgR6f+7y?sl^?TcOcr^}7K(F%pf0Y(wPYk`lw zM0Gy*gn5Ff&mgC}{@Yy0^N?r%^a89yX&C7O&OJxeq2E!zp`dY`L9q;R`!w@)L+#H2 zA+{eQZ5Iy2K5jRsqeH}G#UT3b&(}=@>7(*_#KbP zpn2ImeB-q}}lSH;x$Yu!futZF-YEb<=sas6S3?nGl{ zGyVIUQ&$*=QN`6Z5q1puEArgCbu_TM!2DoKWV0Q3=;l@Cnh@(-G$Gb#+8W0MZfEnR zqhd2kV$#9J&s26{f|YSo2jjyQgbN`1TECdPp)Y>#)j0c$FaZ{TBAubeO5g@B;!amg z>C%d!^RJO+n^K?hiS73O0w*i~?^V{oa>Z^v;UL@@n++7PNdgAk#sTK_j)M7eaV12CST_ zh?X^3OJV=gKRngsSW@TL#b!Fq3B$+Xs-9E5Fh~ikjf$bR(RD{TlZ{Vs2x-E9Ct1)Odz-%&M*eB zvK%-E%O4g6ZSPxlbMt#3ho=Q%nKk70rtSNnM;=Cf(gMeoQW2e7jdN`s^gq*aMb~(7 z4ud4<53JtxqLwXL+XI&AK)k@;@!rfaMRg_oWw!=J?Js7NnY{T&`=i6;Pg3`KY*V%W zi>C4KX!Y~_zb(Xi9yTIn;_?7KvT|t-|Gd9OGlCqMZ@tV7F1+;E8lw>1z03_K5T4Ja zQ{{=~J;}MCsM)Wa*Y}%LL-u0!)*X7LR}rsG8bWuv`a3Jbho|W8mhAc22U&@(I$Q7- zdh+9;RJlsLXO9CYktqQ2&^B+yK$MS8n6T~3!v*2gwnKpJgb0Edt(=W)Gh}hS29)E; z;r>5;p%a zG!$>T3ukI;)@^$K{Q?Q!?(&sP>f&|Hf%gj=H1=Xt5Fp}bp)d7TRFwt+YL9_k%c$G; z1W22n10?wf-GJQ{bsY`qmvmwQJsb*fdRs+^1N{6M)sDo6$CF7x@qQnll-nnFNSR=y zE-8C>M}#Z4a&1Oc&!3UBaFKB~u}z4^epoZR3z$ET6F$nAfFwxI&H3ymFF@J87p=_(6=}Id2$3_0K+`{E<$e)?evd!kt-52J!SX1P z#o*a+TL;sncJJ0!J04l*n8LYzWPp>}kO1 z(GY`bhY0kh2SpmSWmBqZvL8G%pGHm_b~8I%<9u|9dBoafdZ=y7yVmKoop(U6HTqZk zB=?-yia1ek)H3|bk9UwT`oF5@rSy@W@YU|UaIU|v^7e;GRUCO>*fERq&QmAY^#THa zm@XY85c#pp-75$U*T#Q>N*r{ounwa}gFzK9o<+cQRSoBcCvK6n2{x4E0{o#fhj;5H z%`|zwaK(0c_K0FR13zWW4o>N^tzW7E)fQ4E#qn>dIy$HEg{e}8qzoxvDF8W?x^*UE zs31rISwbq3H)$A#eJwnl{5zV?Ll%6dldY4>Ex#Z=S?+4 zkZ`yVErIJL1&6{5N+aHdd&#^pK-XEH>;n1&ePweMUD*Syj0k7~p$f{r_%>>ZCH z8+NMSoSxRgCK$)9Bc@UG(sGo^x8*DG+IGnK-~mNQ zep+8}2Hsuhm|`gU3y6xcvsP@>UAt|umEO@Tz^BDKdxA^socTonRL0+}>7-}QgUIt+C zWNWdjBED3%=>S&%yb%ObI1k^2egBiT2A^#dzIPzqfITqR2Cr$=UF4I5m=}CM$?n}= zAGWw(wNgtNpT)V@m;2`{2zBgPRbZf&eS#}%t-1U7Yp0eT>li8YlD^wb_}Up`o5a+{ zS?C&?AY$8T2Pm#m0FG|`HTO`%>cM^GQNDF|np`$LKHQ)Cf`EuazGhOX(uBJQSZi2E zR_9VcRePyHy!*oRw=qWX&s7Jmon+Ws;Dn4k&(9lPe8--KKp#I4swC2vQoe-O)XmK^w6)Jf4c`ohds!l)pfwTBuShTRPCJueN&E z%I#M?wHMBp=+-wedZbiedcUz{6j%{R!M{~J(ZZ{L?6kLV$G1{>qg)@M=BHO$POvH( z20-N9KGo2Ta{hemF0X*5VsTM{#75xJK+!_&?$X%99co{XIsIclPz($q1UpLAQka3| zHVwck-Gy)p^B4kyIxli)Q)FA3v@w_xX;JE2C|*ww__395mr*Sw z$k#l54!raFP)n2ifpLimX`P9ubU=2aXKe@3#4YbVpX&q2J!n}Gz|34m+Mt!-AuUh# zYRHrcR+Rr@wx)TYp;0Pfq1%k)6(`-$d@^I4`N{Na|Z8Lr_5 z7;?tP2BTD#`xxDs^?ovpZa#r${>qH#cy`mG8XeB}$>GQm$_^|AX-{002UZ4dapf)N zt4)vf7Uar+{GkqhxuF|O7bSZiV+V5){rQ34hr(>~77wwlFVF*jvV*zmoY1Em=8k-K zu~zpB#fxZ>>)e2*DolN)R6OV1v(ld+a!tNz_Vv?ITeOGXyK>MKRq@m(8^f3Yfb69Q zUOfS1vsDHvzttUrp}YU5vf>7XWO?1dkn3TlG0b+dN2}%R;*%#=K&eHBiNUJxk4p@8 z2}pgjWNKf3_v`&a?67=Khc&3)573YiE8X*sV~z3&uNP6P>yv4CWdU!h8s!K=3hTY73!(cu$v7sA$%g_h9P za{Ns14)65Me&sEr?8c3D@8Vf*ov+Reua|j_eVYn>OS4-pJR;{Nm5%RKz+!l-EJQQH zXHk5!1Gi2y@qi`iw9=QEo>ab>Eqli+yN2mI^4J=4le@W*^$`gOdd;=?^lH!DMiC?35I^!(7}4%`oFm@`UK8f$~dY{HG~*QjETu&~0*yO)-f&t}|qN zry=mmH8r#{?tBtHphPa^EoP9%M^P9@NA<7m^0=tf$#wcP{UF`Kf;rMM!9~wsTAaf4 zB~8m=^rH2Y^s5Bkxo!<-w;f&B$l{_^H}8A2DKZ*i`q)E|(RM+mU=k?+ZtNOy+PesejU@xDhO3AO zsXWsXvq_d&2`cY1tY6=zp+&IAhC+tE@KdmtdX%gbJpJI< z(S1p}R0w)>uXx<9Lp<~2lQRSX?}AgXA{fidJ|kkHdgNg>&6Qu8&i95LqtMIl?kcBT zHT1#}Z-*C-0x9_!L+ajb^u$7kpgJ{dmTv)K>%mXAa&mjQ1qEDU9dJCAO0hO3eUD7z zJyd$+d~F_lT{Ilvm)NVYS+u59Cc;VHs-NGx)Z4fiZLkR8WS0hY*x2~DIlZ88TiP9a zh)z*<=Vt)NO2+pscA-%^mbP6NqaQh{^WgEptouh*!QA~L>=J=5atu5Y8WdC>U$=*@ zZiKfGoF-F{bJxtyeNm3y{d$bo=57qx`B86y?y7jKO70QWb1~B`InKim=K0!(W%F6? zB14#Ch4RglBLq2L50<Ydov|NQ*&eOue-4dR&_pFYGbJ3(oiS)Z+{6rOuH_qbJf0Bw4LT?Aa| z9n#q}b3EAJxNDfXM&@9&X=(i7wG*5F?W1Yvg=c_g1|IH6%N7}8gComki~Yr&tYNXT zuvM}1MBI;W4Il9Y*53Y^gpWGiCQ8GBkI{U@#xbak27kZ&Rrdwina8o(*X*y9HDs?? z+qUF_+9rF#vGOmr?|)2@m(K{>D1~!sTm|X6PTC#w&0`6u>O@W^KK%x1>Oq6IQDmtJ zEuOmt!JcmknO%e3Slch?v$ryp5v%IUckWjfd9JBAvXv@%o8?`6{9dNniMuA;B<%li zlJI)rd~8&kD&}XZ%Ypr4@q7z!Hm9^*Hy>Bw9PE{X5vC(2-+DE6dq5&F42DJ7m$@G6 zWz9Pe$g)(jWQ=>{1sbci`Z-NlC;6I7r96?(v+j#yB;jghw;vsp#{&PAcsFD zrz@T+Rh8pLWg{7E#!fHdwkE^GzHJgph)j^J z`20ORKd`}6YL+pLGRLeh_yG)h-D2b3hHv;P+P4CwttDL)=I1W&;W+k~!%+bPXP66@ zjq(_$E}Sdwy6jqKh@`*TzRx6guQ)GsS&Jy+ULvf4@BYIFNvQ>7#kLM{>H^#B$y8pG zyt%|RO_eGY_qXDRpyMfx*G&dirS>45P4^4KPu|QF$>`SZF3ksz%9)LFNH-iS$?CXT z1yKV`nC5KeDzGkmg$L!W{iN;e6zhomwO(}Cl^Nc%1d8_6i? zI!$r-_;h|SoLH_N>zNamQ;Fb{5!{Z_0 zppNtx_8$Bwd-G|z=<9_XVoy@8S{i&S3OK6nFs|U$bA9Gb@y(#zanWuyD=!?T%!0b; zsi*$r-7P^?DOD5~NRONRLBuU$_f7@B(~jiu%BX9QyY$#AqR9+nAP^3Buu5JbNDREuMSV>Sj$2nay9x zQ9D176#@)v;+_56IL&dnjzQX%pfIydw1)aw0o``Ark@?*S^ zAoDn79fiMCEk->>n;QCoDwbGcOzPF0CZ8-7if#YmVy~GJowsTrlfsuYf%;l-?7oux za*MP)$^3Ye(`{&0hysZBE>-f>*DPh_?tEye6yJBOd`PyTxa%bmJIh$aIliWb$tv93 z?WH=F`^Pk4FCZr2QoW za@<*{{}$4a-)WFt>7TOix9C>tr0{xqQ|onCNy}5cI;rs)7!7H;p2}92{*c0^=w+ns z*hfHe8HLjSs9Hczy{D{)k}|}VMAoXs_kXJe|4~|0wTp=}TzR2{f&$dXF$UKYW|=9M z%nqOWR1Av7m!4K^G$=y|{VOaX)-Qfs`G7OGwkT%WW6rPU#LQm78f<2&?S0!C0V1>J zdS)dVuy6M*+iv34G00oXi(+Q;2v^XBm3Gdb<=q!eO`XN~0z;^=ueW-iVga@__$)x1 zUEkDfIhnEor*BwZ@r3Gq|JHgOky_r$TB%fxrK?b=ICoN9GQsCnV;fD`9xbBpz3$DX z?-@NRKM61|_6nsoWeCm0euwnrw}>-XHk-7pOLkgkL(#-I$ zQkzHw!7SGMxpDmJPu5qpH&=P~dI3(&|1sg^o&vH}gbp3?F!5E0Hh;rQ9fTb&R+o=n zM8X@n@YPYsSdFM4J73)wNw{jQ1SG2E{v=DGzlET7i%d@9@uoNV{`h9B(+}L0Q@DIj zF|p>|54{Bwvbf1%-PE__z1=e$rmCfW^1V#-HEDphX`#tMIQcV?Q<&S%$7PzZni;`| zOcIsr(qDx10x=P(YqzFd493bL3JE+E1AF$%`d88v7DU0f#*~8=ybwa|!KWd5S(_!E z@2e1VOrq@n@)2EWt04r)FzFqSe(87zU zgpZQ2#h>^FI)jb|oj(d5L)Y4{Tqo=7M^k+ZG5NJY7T%4kx@n$;Gjx;i1X)1c#_JJ3 zHwJVWTENkb;--UiiUOTL%292ZcQR~VD2uVM9rv0TPD83Wg&lUoKHHc2xy+YaoF9&9 zFumNZ0MYv6Z4tagK^_z`bxUrZ8y1Hh-GKcDsz!sy^9VP$FtKy@5z&^{F30&Dwu#%P z)Y1KT7h}S;gn@EunyTp^100g@AzKCeb%e+cVrTDTc}d2cc-rTUTA4llyl_wmmP?xb z^Rb;2zNatJRyl|4F*voe*y;?#X7{`FEJ^<=EE&Z7V*I-K`?%E?8g9Nt&{6EO!$DUrCogrqzWi3P}7 z;&Fx%I?>3mpAATn-3v5OG6F^J67k1xDZ6$pF<5|gUres@7C8-+v>54MS(v~YPFMG& z^Go?fhe|p?_V@xbg7b-PtfQdsG3TQE;k6COOUS1aTUKY#vrD_35S^SoDtS=<$#EE) zl*{JHw%tAqr43UzKHgxd*$%4DaCz{qxJ)BgcPeoIgavPAO|0I>d9*KTh81QM>Q|a6 z{*tt{-r&>OVvAz>vInDbfV0Xr1FBEu?V)7Jg{D62)%*zG0!JbL#DY4fR$_3n)gzYG z;@X;g3kmusjO#z-p2|~Cp2nFeKD|d>Tpa%vVqlnyUc>u@)gAH)gPvBSTyCE9gM1oU z8&z3Zd_vgz&W}mYpjx#VHN?jw?scV;Su4#(elbp|HgxXcan8k+x3J(G%}22@-OJ7_ zEF6jQI{Dunf3WKu4=fSbe=x&Fa}YQYoc7z+5}e_LB*(7JsTTTUKlG)yrdG_A0no0g z5d#7dspIo{l&(@1n<;O7b^%i-vvJ47d4}%*yGD+DTBiA0xX^(Q|0X8<%PN$A{=yvp zCSss7#YzENka7rVB+N((ZO1)a8iM!}2g\G&GoOlN<>G1iaRfaSNw;j`q-h`uu` znPYno?9~#bQJp+rM7$xDn4O?bwc*3peh~?(1Hca^Z)fy4g{@&mzATArmhwbfsqFrIny{qDV{L0utB6ud?Fqo+N8H!V_ znD~*VGG`le7V3QQheDm7*EPDMdz8wa+ci|EP0rDbTcQK@l0h)^t}zDUGvlsGCApVd zx1XKB>r)jskINU&R7ltJD)?B%GG@o8nYbWs#T*ZyMOf>(++xy1zcbg-gbce%ClSShMC%$^%239d@`6%Q5Whs==fSL} z6NXhfb)(Jl*XEj0VAZ#w&~gLo^TxURgBTNF?YaWu5fgl$pfrZtJVy5`DGj8tpb4D zM}6FQq#L&!lZKvNhK1GIK*DxJ;}se5M5bwHC|Vdu6no!<{Wyg*>me?}>(Tu58K39J zDSnUODq1|2(0@(a79tJQ_fX^z{N%W|4V9o_4mOv6AC3)g+a(7h*K1E}WjEcK6p&d8 z8-XJL+9zHns=tPZFc(}zsN7ckN*#}|!{5~VnU$zn{(>vu{ep0GQY+Uaihog=b|cmB z(1*O)v4bH^QCNd|j!rcMbEAB%8MG%ytzdBf`gw^2bn10^3q>BJp4Glq)CL>SrgzJ1 zTmOyI-ncZgDKNF4eRnqmfeL|S6ja2LkQtl|;AU#=lv!ZM(q{u-<_j&{Ktag&YAW)3 z5<+hS_696i#)8@!8RA(ADOxcXD7P*T7aaebpx4J_RYQ}RUVN-J!eh8T19L*O{-w-j zdKu#K*P%>>C9U&*^}67BKK~Z3hu>M@@j5Gw4me*;_*F13+GtN-l#5*_@v9t9@67i= z1UPjgVL#d$-&k8(UO}dufmeuxQ(Zlq`&ZL zI`U9^pv7kTJ*4OOxhg~M=sh8h^2lYOB1X>23;`CKq09;I9Mbt?Rl+Mq+wwNe=R~7h z{?dr2IO1eV3d-wD^>)pe>;6|+ShF0`m8f1!(qNcl^AA5sDY$c?53mhNt@{i8le?sW z+?*;LcWqx5#f-{Jhpfbdc?|E1=c8Uq=wlxzBq|wrhZ1M0+0IuuTzF1%pB=LIXdBn| z@k)Teb%69|3rK%u>$l(@KiQ?tW;89-@^1isD_sZqzYnrIH$hf-J&x`fO7J*`L{NY` z7P;wwb&JJ{D0?QJ0l{i+3_iWnrxtarrI^NBE=aEmGn% zueoF2Dn1pXwKSBuGS*8=2i|g@S~%I-7EpBd8gV~oPX*i-=$@msCYF5x9&SE#F4wLc z-*8_%unOTcWME)mDK5SxuTrlACJ@$HLsER+rj}F)6y*4>j!<;6k$O5p7-GVryhSDv2 zJWLtBMvOejM4^RGuQr4UVCA}AZKG&Ix?=Q|3G>C-9_!cVE%D0oG#;uAlOFHIt#me` z74+o7d9uFf7Ttc@OKJp4?32%c)Qs?=-hXgEq01(7JOb{R8V|nRzoioT zxMil!?zC`_!RosDjlTMvqvtGPoMZWBK;Z5>-1`&?r<)C1k~RKE@MVA$7@LYEaMD7n zN4QR5L>~OB7{_K);7prEU%bWjhL5bnM#I-*X5g52FMY0CJTX>&%+WGQgJuN0`DoRy zA5+Z&!(ZCBHBPhdkQlbXw)={8WPK~TtqU2P(3<^1BFwTd-@ZsIib|Mu6(HJTZo#RV2P%Iyl#lD4cVm%yQ39YEYe=uO z9gp<1kARIj1t#jdY4mZ;@l5=A21_mvWr^9<=%xh9eJ z`R*CJPr+DgS`-aj`2AnrHz=w|NeQ`u@5K|{kEi?sPGAcMXTa8Zx$|4SJ$(p+F*h7Z zy5zA*HzvO}tv>#D==f$a00>Bq*9Y$qMlfHq<`tm_n-5=kG96fK&@sa1* z0JY^xm$0(WeVi-fZXtU+XcvmL!9j1PwVC`g{pZO=^yk%3^i9PBSXOJJTIxt5b(1K& z`*=L9@lFRIxp*Hoxq-{22xiHW3Yv@0@o}F$fX&^d+XErS(9tR`%`o*Y%=1CK>^l-7``(9e`a#?TYWOX zfcrbYsoqQ!t;CQF5LLEx%mHzN=K^t>;{stb8JGj=P(vY2rlm^uCoO@@4037=d751- zjV53dTnJDwJa4Y}FwjSmP366@kA?)_Q>^aSn;HHJ-sk4VvOQy{GyKM+E^968j~msJ zcW{JIFM(qB2LT+MN#gY$K!s`GvehaAP7(|Fj<9e1Vd7pM+wjbZ2Z-P+u#i(;k`YJu zb9fBwQR5AiZone0W(?6?2J$xpW)7cXBpGG#46~}~iLskl3#Uu?N`%0Z9g5;lO>1>7 zi{HTUPx?oW!dLazoo%RZr0EugN6ARxe9p=ERqZpu&7L@UX+uwk~-8p7a> z?s$Y3)Fo$5g~$F>LR=&@4iq5%^UbP_rQg{uQ8_*6*_0a_y#)01CNw|r1VW2XQ*>l~ zfOmgd=kxKkxflLEOVdTSZx>pA+I8Er%+<|kWZhfbHokrh)K$Q*`*<`qM``>a6Qc{s zgAehEwp?z;bstg=b+wmpYGN!5Ip_NZ46XVuEzpm1rdS$E6A*mq0 zhw9SRo9^dNY=v`aeUP9kkh~%t!xF*I3({vyxjziA<%U4Qgj4)_)lUyVG+^ z5n5t#DpkoqPmy-3{NkzjOy09&0PiW<$1^QfYymP%qvPov`c;{7zWI@i1h3A5yfkZh zuDOQ^rJT4gaROLV_68?gD_Y%{IptKp)K1h|SI&~g^mE;bIOb~fmfF@;nD0Dq+F6c{ zR$|Z0^Z{7G^oC$VY;^(<%9MZ9L!I6yAnL%+nxsoNu6u}yxr<0Qf#XpeO#m51;Ql3O zxC2$(I_x~4R-BPfqel~>krqdJPW_9Lu7)bm{l^ALN#6 zPL*38h|RO4#UrC0FZz$JT7UMGf!j^bR9$5WDKEH*_MGg!ilE(tloRXF*Y9bpQ1}q<<}lB8l@)T$u#)1$ZPE91wvjQMq~%Gh;zAqTxyimDj`43*G; z(cZqo=RP*K%GIfH5DbJ95%A{pp{v64GspVLulZ?}$?A{GuFtL6GZT_$npdRfQ|@#g z^z$-Q_Kri^MpFga9C{ikJhcJ9GcaV2iRh_AYZC)>4m%MQIPU*w=fC8|=G7Qtb(=%< zf(F^3>V{sAKazS4<|RVm!q+h{MKo6LzhP7DEqece&EnZdNM0kWtQwu1?6E~wY%SEcEqR=)%wPLijta3O7hHNoTS9WF8*U=3U|_z4tD8?cE(DM>uk{gJqg%{%<)VO z)2And>%WKQZ-27aUMQ_6=v7+fuNcW)t^!&TJWl&+=j+j75`?QPYH;|kOA;bzt7KWH zo7bndn1~=Vs|^v-4N)7a0CA@ z7&RXyMm)7w=1A_iCns&C?GE+*I6iP5DX%^x#;;`2B zp9DTw5Cb@|0R|%7I0fu5Pj<6Mh+wGtR_!m$AR%)=uo_>hoGII!!6kO*-5UZ_DsO&q z_T;>GH@YI{;RxwJAA^cfgIR{w#>*tJccc|8mKOdO3;p|PFbGnF+n;Z{H~?v57EL?T z%*pu#&=CIdA6@}RnF38u-D|hh4K}r^TOJN|2zUQ4v-zv7{qOIX6@zORIcgd&DFk(D zOb{XNaU|tpa)0Ep6s`73a0>5%=``O#sgFkZo0#f-)tN!`) zvO0+-ovg{uBcN2g5)}c^C^mo9De>QF+aIcdwtkRlrN$`p2e|t(_#`~!_imPjmPqNL zq{%jPaRkUW%$`K#jHG>1nGpwSp|9_71 zulo1rcM{!b@BlF826aZ6#Qd|MOg!}8yN~$}t~8KAuvc%WQgL2V4mE=W^kk4*?#=H_ zB@IBk(BMlmg`^whB*<67q#I3~|8hM3zH9##KPNJSFgMTx@Q3%18+0sStPOxcoT&bO z%QP!c5ebrKC*lRDKK_!JKAeBeBL6`fgX@`rWH2MhK933@`?yCpg5Z*~i}G)p*OqZR z8k~jW+1|{>h@55s^sBt^4OmyX0qeix;;AG>zyUR*Xz+`N<0x%vRw;mzCNlDXB}x7I zESS&X=?C4UO|2hsy?JNIEB9#44=6oqna<7r2Qq2zt1sa9f6F8ET~H$j-UHh)%pS$c z-_100*<+YNMMW^=T6g_Uud8G+W^;~IF@5+|pt4bpCRA9!icKElmQgtX&HF5E z@coXJv8Mr!UM`4=u;OM3fCf%`3`UWu0h;2HK@G$2^iO{VO$=BcEh7;v7Xah~@^$ar z$={m0Ry5sE7!Y+@TdYCvSpaGI3GX!7@AR-t29-&qGITuyDC>xgk)Wv}i*_BLCPqc@ z21W=<{8k_7fjQ(NT2XWiLqRI`)|&m!O_=#m0YI3P%hfPAW_A}1ZjAXm!xSW8fuUP8V#<2C_ssgWHb5&V)1`cTW~!Sut70H6D~IYL%7-f zl{6p+bSSJ5{tC$d-#0UVhSqB}YA}yGJc9?Fhi2i|9QNI;EZz4!`7J;GeABe|l{4@u zV0YV|;{My;3I^YRTHOE-u#R{#3bI68#*OfIVy~_~O`@WpD6q+{lj(VO3zGy?{n^AX z=qvwwh){x$Y0PPTZg3Xk=e^U)a7$Cy9{Hevfeg(%IKR`dB>A9Qa3zcu<1_#q8INxN z7G{gkX)hTvqLVNyvw2lB4~+RG#;E`AcrB2j@P;q^#10tZdlN*nNecn1fPovKkg7ZR1nbO$&x;%94&Bn2k9mj)jKB8D?@U7!-uCxE~vkq^4h>Wx!Kkq)= zbqhTq!@n*06izl|50p{3R&4z3hJ(NJK=lu3@1!-q zeEAYly@j{yYGCkD*YwoJu}5-hE1xKEB6th0R1!44@xERV)dv~JI)R=w{|IBF>DGS~ z@BX+rNz+_Z7wjo|q?2tHJ-%)dU9TPrD`a}C^DP%;Hon@F8h5yIl4r>IKjz&1bdQCE z7|AHiq6g5HW91DdvWZH24E_uh zy55`^t3&D^WD25<$w2^{^~fe&fr)j8Y;gRi5JCM$lgAzQt(<-iFp7AE>e*J{0B#Ps zJ<-RRV_w29e8OH^}eBxXQ|$N3Uq=-uieRc?ev0v zfk6K8Msf_Sf`jCiP@`DCNz$+2pU(^uL76CRrx6?zC~dyC0r|qe!}0(Z$iNU++2}uW zmHrOe$V=rfOgY@y(KnAY&qiZdj7dm(!%+6ezyCY^{>z_&mq`8`xYdexD59hrZH0B8 zGKsUeDvo48|9HRv8ZkvGH5GW+?&#h4tO_CwD3@R*yp2V#Y50hNqM(rW-*cjy;v^qn z5r?foA06)WBJ&m;R%`HP2zQu#Q6abg&)8AM9qjdpmX2)LfkF~v2pn0SGS#XY{b41IZe|()i8y9wW?{v&%ER8M=OYT5zMh!g7P8hg)mE_O% zkPs`+p7n2Kt#htJe!4T3J!&OaR)YzPwLg=acqFsIwPKSGqNi3nbH<$h6>RG7l2qYaX)6$=&Ah*yO8u7ykIfu+wo5||7cerZP3M0uT~+4 z_j30;m62ST+BdzyGz)qd(%!Jjw+cQ$yKx*aq~D2E-Ay{gyy?b2dh-h(IQP&bv&hlf zys<*H(b{%H9u04W#MHUVria(~6@1VQ#SLB>`>(SAyj3d11D5DF{pTiQD&&DaiN@8C z0^@f5X#eN9eX7`4SDdJSbDegTgqTvw0}N3#;2l$i9}HX!Sa-*3*?5cedWaaYUk8~H zo-Iccz!5vbP^;LwKPDIh7j3aCG#>=TC#82+VE7Lw9S1EyN&*KCnAcFs&})gx)r|ov zyuk~a4*r+3ouksx_v3L_y!9LozkhrR*nrY#Oo=Se`2NRXpF#j)KYWG6-UR-_S2nc} zYiXc_8n@BBcPyS2>A-fW$FxooS89}(Ta{3aq-}5Zwo0=`zT@5T0iY+(qJs!GLLMFs zwS9ql>4dCvnFj98C0qeED}f_R((sDZ5t)R8}ZQ2zBn@srq z13~WbQls~VPwkb>aNm8s^9a|K7!3q#&3?LBYp7`d;}fm>&S=N~>_c)TY&2Z_`f|2~ z1xbv75lr|&g4u&P@?HmGlorENE|v37r~L)XC&a-iGk_*O6b+D^_a~3C(Pd``mQ( z%-@95qaj1E3Fhvg7Ohu5|MJSWDOhI#1QpF~pc0BNT(?Lr{hH1~7fH$UI!7*g&E=s2 zGtL<%>g?BedNupr(PnENyZMTG2IV`i9|EBPTXYTV8tH|)|8t;asOkMEKj189%7kzC zP4vAj&Od(0ca!=>=Lv3?Fi-7~Y<4{r!{Z4ppt5h2JiOFo&qJG zV}8n$9E}z}>c9}c*9Dhmy64A}W*RGR7q7W4I5Wt8&CbDhKlxtMs-)Go%YUy{A&z*! zlH<%Sd`*pWmMjr{&f~u3hn}Z9@KfW%ON~g&>AS#9!FO?#(b_(WfNA}OQlADS3`L$} zRMQ4a1YUNt_?%oeNitO_z-d295Cloy4u&bW-?`mc_fZ-wsD*|0Oq@Q2aN`}CrPwFn z6_&QztyY4~&PPd&IbC76Qnq6S1#rKSYeZ?~MtAR+8{(UR&&5f3EE)eB8N3%-Md~$f zF{lySpGbq*r;mm!p;s>w1hWFE8D({Oqpa7*XlO7HY9SZv`di@yfusGC)O&1TDe6k& zHd!t+NR8WSKNoa`0RhosMV}9(mf1}sbDQ-Xu2$kgv|}I;?)I8t@Cj1oS-t*34%_vL zwz?N{vJs?gSngZ1S7Ur@*L5dv7rPDwF9)t|_wr|cEhsMB4puSp9O>xliQLEuJ=Dxh zH;7zrhZ$5^@y@ke2OX~U$xhe4xV`-|&yVS5xUWxv(9&I8yV*N7PvQ)*9&3hwCN~o( zX4nWhr4o_M6oD>(j+7c_O~gfD$w$5p;=;io3l^h~Sw~fory0)8cqWr`&5i?Cj1apw zhx0h-#SxeH!*g?vR9urXkpUb4Bd^ew}9Eo=~0V#ZG=5EdViKoVEOrj1^(?Y zxiK4`;b9AyXbuH`iJFN}ePLya0H^^7L-Ap^sT5q-II`|N$r<~h)4vf;4dG{s^?%l0 zj+0QTpE#H)3Q7>{u@qYE%@`9%%1{ddqOmysF4(fXGDYM%QvnW+S4rg#;WTSnt5bR< z6ryDn?AOF%_nkVUARY&L6MvwVjDnX@5-%Bt!b9u832vtu4ktMj38T#;+!&e2q8SO-Mnj5kD`RTi zf(ryaw|Q0neN$Y_Azj#gD*r`l?|y&=^Q3zy2`M5*I?K~`L99zU8Z46iI#g-2#dGfi zEc6alYS^7!HS7Ka?#^wGhP|oJeT7e0WDc!7M#RF}E|8u9UoT#FzB!-hal3kVd9erm z`1U+FD|3%CB;r-~+q<(m>8$Id*XFm#Im`I5F`?S8ArSwIs7KN`d&v%*h5kqcZz#bY zq$cnhNwB+@9H49BG~F!f2&)-*c^c(N$^7i{>`S$pFt9%cdr^#t&p{s?7kc)oJtg;r zdM4DXqrXpv8(Tx2Xan8;!JJ^v;{NHFg26KcE+T zHo*U6l*>R;+E67cnwW_%UwP3Te3_=_=Z#$B5cNkSIKdF1!=AI)LiHeysFzL;bZCcL z?kcnN$zcS%C3>RI5^y%9|BmFBVpOn%+(L58scmzQw+I%}@NP8GtwQ zMc)ehkS+~^K^TCPf;+3qhKmynv(4_HdEOi0?zY=$4SXF^9m6<9x8MXMwMRqfvA_h$ z`J$&uQ)}YF+p!98;HE7dN7vQMOvTTZ$x569XgfnB_MX^e_DyD)QYGtb3>=4Ikoo}k z8S=Jw(rLM4@^TIlxkh6**+5!vJ3dQ97K2WRak@Px$E2EBP^wp13~f2xW$V<~o~mZH zjqBEK*2ZPksreK=-nxo?3G>IkXS>uBEmv;Dr`1D1EtW%CIx$>r$+WT3s|D}5I86|6 zex#Vphfm68m~yz*YrW!Xjh&hPK8ZbdtZ-;_lgEZs<6dS4ixz!h0*jXVj^Q|qPu

I|el&7&8&Q^jG_c;vNH2vCUO za-&8T&F;4ee2#XC$($SjlAIZOgP;=j9NQK3B~5zC!|uHFCV^GvR$qp2A%!pLMi;b0 zN^ba@fvff{b`|%7S=>Ua=im6hZ89k74e!iAt89+761hxUKi!)DUaYCPsz`ddP3U|I z6Q$ts$jMJ6?Mu!!7If9`gm*YY#Le&6UZm=|+!L}2(h7e(OFRgM7RA?cyUUiCme)9M ze{wyV;4rLqiR=sUMqT1tV_s8d-KAU8N}sBxs`VBYb3J)5kkL4d!)2I9xJVEt77c#< zalQMF$9J#NFA&*!*6;O7*ra=UXrgNm)-tDSUi3=qlxlNh7bAnBEY-gq$VF4L^+c7H zQDjv(ia>+MJm>v;)-4}FbmJK=qjp zYeLQl@rz;5rRZbk-fe%Z4|z}ntH;b+2_vUb?H>f+Z=P^@vOL0j?qRzpb|>jH^)+9& z<&`FosL25r;Mmqoqb7XC1Z4KiQz_zBhJhHdj9d2l&^y-h;0dEvVf5Fj5`iI2@2l2_ z%!X^%htOveAANi0Qh5F)+s(!#SVOmpX*ex$#Y6`@DM8CsCQ~ab^flj6UNakDK1x!E zUZUV2qoTJu)(@>V{e2r)&|XSVVvX-brZo%$pkU)iV7D!}kKL@|_VINSdFeq|t;(AG zL?jt6tM|jHeEB}9@JP*Wot6(YnGK-H2*djQM>8||Pr`S8v?Vut?Byp(afhd!K6^{W zV|Qxc%PiwN9__^1b>vMQ&X(lm z`r=iO0M^s!i`RtDGgruFiD2^_DBT~D9WmZPQzI67Dgv5>nPBX8{|LBnV7#J_>>>zlRop5&Vp;YMw(KX!Cg^QCaagC}U#L#ZS3hYO3A z%^hs!&}C4iJocn)Z+c$x%-FMuQ%`co)Y8SJ znzdDB(A)!x=BW2YWKSQ8Xf9dMfWCZq7`fE*KKC(-P0z=ZD zi2!J{8#jUYwI!r$JAHrqvl3!$t1=-Z$g=hPOssMJ+qn`evrU6?#8;=No-37#Yof`( zJz2EB&C))EP zWwJc^245asoKnABhlNb#pE)n9ALhOL4q1kVM`sQf*OF_$TDxt#A-BdU@WbqzjgZSW z3GotW4A}BgT#b!YZTUG%Ii1O^NSzh$%h`655?t6dGb%%%bj^!CGlZoH{6o_J2%ofX zFZI)=!9((I$zI~03rKp2N*@I+R}#@!qBCo@Mb zN1w7kCs&EYM%?B5aJuI$F#ihHQDdd7Bwl5npNk516h19pNco`D{1i4sEiD;)%?TE_ zfFP3&*L!#MKRm#Gf(DQ{P#B68qMd#en<4n&inmG@|LGH&i;vs|h0L*Yjn9?j8HSfX zqLFRm!ojb<-TCaxJYqAL5S%4Rovwld@9=h$kM(_a$ZND7wNvC>`(=$La3|!cX@RkI zz%blr>Q#M=B<9f=yjV98lrb4qmKMvE%A}e;s^B-PTcREnMo@b)`6PsHbK(ii?{YQS zjCx39h2Bx!HR=(Zd2I!;S=R+g z`(?nY!7Oo1T6Q~o0oNu>&ds6mpIe)F{I&bTUHA}@S1r$J)+&u5Q!L(Y7f~kP>%8rC zw0(3t&6;KvmE|zr?yR!vy7wALHQIJpb!b;h8u)K)H8>%^)LjP^#&QB6u+DM@Yk8_w ztISF;U(xB#7Ob9_*J%YQ_axz)>q?E!u;WZkCQuN6Dpr84fmYCJ0f}EsruvYMs}ZRwx+ubneP2_GwG`*^$|TH z0heBW@R3>Gs9@fnNzjxu02B&uga*r8|LHEg;QG=eA@@uPkA!d+4M^#Wb*sf)*P-zS z4Eni#JDS=y0Aq zXTTug%8`E8&8FL(<6yvZ^#1WhBHKc1?R7h?`$J38`>j@QyKH%0$d~HIIXmD*0ZdY{ z(2Idwz#|(pWY)T{+Ne5~$Tqpr8(&x=uD+qxe=vBPtFP)c{?FE)Kz$2FGU2FpHcIjI z?QdKd2tXJbS?*E$ev80>+h$+)L1t5>8B`(mkzCG_Oq=fdW2R=`(}z?7Ui#(38YNa5 zt_eTHd)VHa=SK^-8Z&*NJY!r<7SVt5Wx8Kd?~C()QPj^|mCf!~8yNYND6_$V%mKan zVS4B1YiX742|xEvGFmpppxqr0x|HPW*g%=-e$-Lsd0m451RZZ{Rqyb#t54&x86rlw z#RuX9_Pf%+)SQ{+qRK9bc9#3~Tv1cP_xqp&s@kV#3?w3hizpujG|*U$nW53D4gy?J zRN^wpmBu=j(KP`}i2ZNR`E`rPcZB`d9(34l*#{JKw#piod63)-BM-hyHzgc^CF`$Q z@N2vd!wkcz`p22FF4c7$l;Ct@n;{+N^+{Qil6LM)qn(w~sLV<#z7eVSy$k>5df=5n zZ$DMXzj!io&L7b=|2CDkB$CG3_k=oUr{!czYKB98Zk)~N{O(y;z2C9yRoGD{AHm!2 ziT%lLKb#1=lPmP)_A+pWg1e0&%1>m9`1M}J_g)EmP-mHa3b0Aa?CtxCCkbtsySU>l zRvFOn(pc_po;z^b1Li3n#^Eb=v8i%Py!Uh{Z%2$8_h^Ucm>$#?FEd{{RvE5xlf2g> zYUB~1Dx0o~9DHm$GZ@F9Q1o^;h4@M%i<-xo5s8@Lxtbu2PvtGME=HR_4U0Nl!s~pJ zC9T0WDb0jc?28+XU*mq4%KN0r~el@j>+9Fa~%wEh1Wd#C6;2#VW81bG+iGmvw)tM&&;E|TeU_t)k&)ahYt8wa^H0OWEfi0q7OhkyQ#kPN zDrz+n)6l@mIR1OHj;Y;jf}X+Spv$SDLZ$9EZ?oJjhilqq{N^VNUn?xR{$r`F|LAuT z&32Rh8CgMSRqyD^<%X9y<}lX7ybs40nRGUB(!iNe9lYeC+I?D8a|Uv*2J@eE%Zpho zR&>0sx5-Lm@tZSvnLEPdQ`nU4l$^M1m9GxWDie>A>5C)R zB(ufSOx;HdG;0q^_ARel^@KM0tKA!um_ANkOAY-LWpm#2h zr8Oe%xyR>?Tw8LEPXjkM?=!@B0P&=MHAJq3EsJcni?iKT$rxVit;$qh(2LBW6HTut z>UG(1yP>A97Y*cdml%is9=pv_VuSUH0j1H*^ISZQCYp4z4{}h{7g^<^i5er2y~nW% zW%43k5S?yYUqFpVqAPB>X0uAn87|4`P|H3`(`JEWMrcC#24l!Gv~S;q*+>!%Vf6Xg zue(&I&@PKe2u&Wn4xV^IL7tWoXa>NnQ9Rj9+&4V zqZDeta)FejkN0C-h#B^0%IH)$Y z%p0tAxRv6>5Jv?EI!KxqgCA{xNCqU_&di>)I>y!%BcEyHDNB~*tuwTreu&auyIV4t z3y(yX4Q!G|NP90l? zsqFIBp<|g-2Pu~@s!qJ#qG1@hlrl6AVX|3Af*NOCzsMu;H(`Dc?GxUYdW)sh?)|w% zhGR(LF_HO`-w&o{@!P(G&6Mh8&HXmVctri!XaFmqr!NGJ3WUjB6c~hftTb`16Mnmo zCl1aVT>0^OL%*}P7ZP6A1Y=l!2~s7<6%Q*6t>z}jgS#E_w;XFYjsO$MDq&#O){1Pm2{iY_3;o@M;`|5MhbXgx%SV(8$P=?T149B z9f>-~04J2UC5~LW6rX8=ff57KD0%G_G2B636>I1tv)#3S!SdveI=bw0?6wZkz`NN&J&+{h={X}! zoqq)GKrH2IamP?BDOC>a?1ut`J1Gyzxck4FFHr_=M^0hNp3; zj+F)7S!&~AjAyoX@t+^yOni1V2uS+HK;izWHl8vT$10P{i%n2fww_wXx>U2un28~tE$nGqQ*I&mrHldNTvPE*?_eVsbuJ^0Yj0NCs)iqI$ zrxQ-3e09q#{9lp4TiH%cP3&l_)T=gfpUPw3k$q-hHpmR0XQifaH`E1i`Nnn~7`VEw z%$D*a*`XXf4ZpE3342+Fqr7vgA;`=A)j-7Cs1O_lUGsH-8Z5M;<^uTbEeKOt+3b7QY=JK2seK^6j)P31h7OQd2+CVY*IYz8A7lc{`gE`Xz<75 z6^LSV!d-w|K1+Eg{G#p)%qXLHFD@>>8{18vx4Pn&!M;(ap5!P6lxm!$Z?l!9()K1_4&u{0K z{Wc+>=X1!%z&T&}BjJ`2+qywP60uUPSc1~~u-({v{i${^gGyB!t;P28`Ob29P-KRG z{X>w|QYDY8-m+TAoJLv>3!O4PPmu?fxzPtha>1Mi`2;#_NhKbkaajQ)BWxURGf-gfR zW>!$mr49-KTRAHWM$4}^5J9{bgkj~bYBjjqdp;CtX;us8?X#t_FuiV{j_%LbO}e#< zd?rGBJ?`DzPzT?R$hFClDblm}9Mnv69L;AaSba!-yIczYI&TZCGdXkD;_zuu?)PxO zzu}VzC>T)#^fXNLI^xzy+wTP@w&5Q~=pY*W>EIK7eb}*-MgejXZgSsV@kqr8t-ElR z#gONnC>G0o41}dC4*o@W>Hrz2y?HD9v@(iwqywzrhQi4i$o3)1G z?UjtUNXX|a^rFc{yLnYoXrx1F)>2iwU!h#i3;*PcEDkrS2z+M2t)&Jh1hRJ%(`|FJ zJbssCqtzyJ>9B1+;)O~ru@Yqu0tk-a_%emLTnpq}tC4-#64{geV`Hf69hik&(h)PT zxY8uL)B~$`=z(UCJH*OAMOGavJ)KeYy*5!OX4>fO~j12K`A#Sq{%A;@nNLV~v z^}A{Ggy1{W6dwPr+RYt8{;BJgcW&gFR>|#i7MZSp__^C51tv|jI_l8JCeSB`h3t1b zdxS3&+)<7(Sf)Z()zHMI`?LNUSR0f9?N=q44t@7dP*}Wl4{M7ih(`f=@H*t?o}9mqJG ziMloZqQ0+gj~(&RStUQ2plyuhh`X(MxLE?+Q;5|yo1Cy|?3d7)1wy#RM8I1)bYP{;sw@}D6-73=^W_1pc%W#V#LzSBEc0TLQL|H?Vs~f$q2ba;J;1CKJ!hxQ%M<477Tc7?}D8Otm=~!k{Qc2M7nO zM?ok2@uUTt)oj%=c7=D(W?DIHhXZ;w{BzeoYVBI2z0`O&?pwLW@E7~CIQE>9W2#WD z1PjLmxvm+FV01F@{~K4}ixT!pVoq3GR_Uiwq{X}f80eepEP+pU1MRoPd&X=?cO43e ziJhWRHBcbJZjNVj1+f~}TGp$VEp8d`zX$h6#PS^VCA-iGJ3Go%Qq^+y&@|lRicxP1 zL4Upi($L?sd22P%`EspHB?hpSUxZj6!vXfi#oWtFH%W*1afLT~f6pWGAO$Polud6Q zG8$=2zQ%Adn13yOU(OCd+H>#o)vz)J7=-MnF!j>nLPQI#XA|p2vf#9lPh0N8qfb&` ze5UuKD!CHg4t2zjy_(?Ast*=2OfYUgaGHLKKYE-eo_SzSka3_LGM-xAWA>{7pil>Z{dKY`)ZQj zdz}{*g`C%B3LpH7adK((O7c-s&su7oHk5A7o?r5r?_v(vbf)j4)T|TBjn+mfHtq@F z9H_VF^Yhp3-}YoO8Ra=Kva1#}Nmn8XeL@1SuxK1tbkB%^l$>Tsk~R87Vi5M zO9|sJsYdFgi)bCQV$FB#G6$lVBd)+d&v-O_MuA!EWpm8!HhQ5K)Vho?ZIc@Sr?NCY zT&~p4=6hR)DAEczMn}Simg4gfBgX~GWRbO`D*lc-=?RlI8}I!T5ZpKmRZ5}b`vT&| z7p_;^(?L$uw8(N?yz#C#yVOD&v2f7fA0c@#DT{X%ay`h;4<#B`H9<)v#Fu%7XQ_M8 zB_k5A8%T7XuEn*?t1Q`S7ccdq9ABpta@Qqjdvhb7>tCQvQ2Ak8?rjqJ0Fj#+6iFUiwc2XDnF zVb(f6TWG{a7c6W25kw8AC|^j^Oml#!@{Wf`G5Xj;Wb}~PE_OwoM4x+mH2rOGky5!M zRP@Sg@^@CsCygZuh;5T8G-FzauK{NjY4d2a2Y6@031upwA@77l7XG=H%X1@bsH_>N zhl01d925T4!+y-Q+~i}B|7Vi9I3$GSUSx=+^f~HzliH0TntM8 z;K+s^JevE_c-(TfYMWUgQ%Qqs>l>Ok68h^bDRn_5Me)$sX+S6mb)oOS?)%= z%LY!4HhNV|C3IYHt?5W^Z87b>+<~>%Y<-QQKNpsZf4TJ1!d2Igh5 zK_rmrVAV>Wyo={6tlD8LK#1j3OvOGn9UUNHMZQ#EmE6MSjnEjwkNjfdAJ_d`kOTvc zK*SYgyWThXBeE)q@Q8akRP|(5ip3W4WUaQA)cq#$tnInWS&jKAmCMCUm=QN|K!ksc z-~P|>nP;-|<8~E65ii65ZtK%=jNc)lZs0|+#89QlTrQrWfa0=jn zw66ZyI@nfFx<>fdpQh?;v^lIUJmipJkRyYv81D9;Ahh=CH$Ukad0*AKQfbwJW27`+mB^UtERq|GwSf)l8l94x7_rt{ooM zR%m5jcmT*t7(un?=v=?n^ZZGjN!Fcy1=j+^}V+{D*}V$8qM0xL`|gD=e@hjVn)YVPSx zX`lSCM$uRw^-$5bRpNHv9_ehuylxGTJCf&%4C%a1eK>Z28jm3Ks|d*|qrr*FqZyHaa{vuF(1~K( zvKNZnX?YZ@<|QA_u-KLH1;%*Sa=nJuYOO&STQH5iyf9xf?#tlLfPAhUdYgnVSG8)G z$!w07rPty8+0zX+2I|U>%i9w4zlwNxSI)cJWl3%E_G3?Xd=h?|4!f$M4P2jxUk8yc z7C;u+0o-v5*0R6d1eoG8H(hI$pk`f$lfr$y&iHI=t9W#vP{Yu@52w-}-d7K2FX|uaJY*v#@_EH)> zjE`Z^TBrc8D_FY7S%CB6I}fXu7tZDabeBwJdbb0bRR8#>&dwJNhgHqivmWKGmLC4L zde%+jboaYzE@X~O2#K^1;Qegc%&OYvy0`d)OlibhqNxM9Z22TMR?HnsoYSh#kKClw zHKaMGa@=b_-WNeMYoKB;C$~~^PeTpqyJ;)v2?7RrW_quTg1>A=o#JC^N(j9wXMVk~ zpnX|rf)CphezkzV-edjHzMNJ1xx%jv8_O`Jlth62-gkdhl`JoLcjVtprpUrAY* zOw)4ayNx&u7+#bWXq;EWZHwNSu0-(bllyULg-&OP^XY2u2sl};8f~gZjrqTMN3s*H z>2P$~j1W)mf>iqBY6l^zUs~;QBr*@u-m}xUH(pC%;yfd~C3_cH^y6?U4>fpYSzsT! zgc|@jRvV%gJn*8N?T^ z1__R^wYSB|T7Kvx%^aL}_OeW^Qa{ae?)a`QO+6%}kDd4$MKp`q?s4BvFE6zj(JEfmnMu%a3wXh)u~K%KHjk<4CURoSaRHIkgD>x#G*fH zgDj^M*~v?z+q1b?K=%TEt3}6qEtAi~H?eBHxWg;;%q+F&`3a`O(DZ0AlPlf@jb#$Y_nKmmEWZvlTJ+J`pU)Qb}s~dC! zOku)DHwk&EGuhVMprsn^!P!b~ZDezr0DrL!wyEk=^;FyuW|bp;R>)aI@}@PL><@8` zw0*?njq54%{&b`ljj7G4og$2PZmfut#pz&ZB;hY0AI^M%^>m&nhi`V+bUFp1Bl8M{ zx8@o&bO;eC9C3jgLdp|#Ry5SaGU>wyXg(hk@kEFT^g5$MA0;jwdrrwI zAEh@WQ6#?W2~05;%9{FUhg0M((Hlhj3$ah8)zMa-4|9=~T^2ae0sQ}jR2?Y6(%YY!b#tjck&MV6o7_`W|!q^aaD z4#K#`pA~;W5}o&i;&cD94Fbd1BN2g(+HHXD;Sc^;&hA4Jb+$65R>^UY8JM{rW8bh% zfobD}I&}T87958%|IFbRjDS|Lbc#`hdZBL5cwJ|`62oS_?5ER9m(FXTxPGzN%pC{b z?oCLf*bD7cV(M~$+-k-zF|Zs92(fCglY`z%&d1aKY8ASce`D#azFG8^-YXzFFL)VU z#qQ14K1_)5=D1SQad_$@^sbJl(>9`ZnjT=0LEJN_X$~bY!;Gad_XfMDP{E+iICL)b z2Fca^dXnm)f-~uD4t1DMF}MSLt3TGvB@f3DC6(47SUiKh{l_lRWwhhZ(m-+GE@s3L zPO4XGWDYaBz0X&yh(&&v`L6TF(YNa0ck2LS^?uV7Q5xPW3XJ*vJ$+|2_$^*4n-vy4 z)cv%C`2nHd4b(xz(!Q&VDe=W1fyq9H=}Xr2s>3{)dan;~^w&uFmOc4-a8(FgD03Uu z)HpxY6wtu$y&&V}8dIw&f8*=KVK28#!&~r*4pJ_V>882`l#NtZogy{XdZScSgJsQH z>pUfT5WL$st|n#(KGml4{pFOQvn@_RZZ_IE4IajEjtT)QRawZx;V+B=$@QsR(E?qv z?dF)@F3-0(G80*y3d843YA;4ry6yYeZc&dZXQ&xmUWwLM@XXHe``=*H;cl0Sx?^BB z8b449;=!h0A46n9JH3Tb`vfsS<}_QupFogSYnoH|!>0-ZFayp(2{2+2ckYx>;&3<2 zAnQ)~o#H=pkk7mx$diBo91rikAa_Q=(|KA~i+Xzc`ykVHt4B#tR}|t50f((zM+tKu z-uE3?$XG1zJ)N?kzOFAJ%=16)G3?Lwf}$S3UP5PZxM|HX7W>c(v*M4F`Ui!pZ1J`@ zUl#J&EKHuaf3fj)JBd^Nq{j8zK`Kek3egm^wdVC(kZBk}uBQsmm8G{N{r@ucV8Mw9 zz%FeB4xx{@)PZ}oO~TtI!%yj6R&8`(xSi05TMNJ)KMZ32dixJjpjO2Hjf>99Z7^IJ zkiE|)m-kVHG)76N_j;RO26~A))iJ8 z^nCeCE}JEdjL)Zbd}BL58KjC-y)0N@PfF>v96!AEDZqoHvdb3=nrN)Mmz)B;V~uCd z*ekRmB@O;cFM7SG8kV-lz(i2LLMn)!YAx1k)8qbFpJkmP9YnFNvYDM4LnPfje98~kd6H7A78(j zD{=dZjJvCEnJQq@q0uz0aRUWBt-d3r3-F?TvH5mTzMoUsgQxoWR&grBK`kZF^yV`Ddx(sio&i)re1F`f%~+6y~o<5hx4?Zc8g8-O%{(%}6(0g^MMJIbrz?ZepVxFm;+D?XP{>%(ci5dpgi zWCERP51NU~W~JTda?bNb6O>)`${)hXg9qbTMH@>iYbo71hI66)iDu%axFYzxby427 z+eg-ae{nqmTVKi7_hqm9ZOk$VR~?8~RRM;ve7M^f649T5_EmGZ0$NchM1D$4QFz=~ z1rnLFX$})Ldu8Sy-NI6;;AMh~Un^f8r?5dIpNKTl$aG&sa0lGefqeYIDb$W&f z;9i0D1?JQ7DaX3ssAkz`^DM=Cs2H80$ssA`^nMm`e>oTBtT8PRD!;G9gBX1R7S5d% zSV$+0jrFz9P)N9j;nkt=1ITg_%eRkPUv@xt0o{x}hU4I@2f)vkuV$uzD)*jZ*Vo)X zvowFngdjcBy#*ObeqK}6@f@2g61rKrosauJ+F-u0KmL0iYGv}66OS~at!1pZoHNSr z`*Qu7E64BG2cxQmBK3V&e`kYcwbD4p)hq_=9*d17eu@U+W;|jk{s%MpN3vqXz6awn zw(6l(9lSoOjSa@0yrn0)mbBUa#Q5r_F1WT~D2x0OROZ6nL5%e&d); z&lQ)RrDJc-i8w=-xVMofi;C|%<)jZ)IG=Gl6ns+lO8L44o4BZ*L%PqsAhF)KR0{$& zeL-1y@=2-NXtL1L_1>TFjU*?}V%g$glru(k(ONV4lA@MsbgaXf|1mIj>ThFF4+rs> z71tD1@kI9~l9EiuE{T(#{+N+)!nWLC)0j-}5GBo};?3PV8N3r=3$J+}GSHL$)QNOU z7dVtjAw%yT;UIKAdX*%L47yiGwVnRthPP|S-P;a{gu&b5Qg}F(yF_Y7Nn`V#i$xw? zsl&xUS(P)qa&X{Vr=No@$>mas%i?nXxnLC;%Ew9?d%`ekEd03ozpzE>J z8=lUV&-_MWq}0)Vyxck({oDII>5TAEYeYFi9c7l0tZm2(M=a5DBqfNx&fKFfO3@fZQoHJslW2YYGs{Vvc>7Y&yLH^*z zH0ne+siAvo)uALu4z1nuZr=52>$_y|=XpqX*)HdLbi#7~Xne+>kh)BlFa)*zyV!yW z)QQZ~O+sz+AzQy7_Z|TOkn$&EkKiGU;E%E#OJnEg8bK3h6WjDurZHhrZ!0r z_pE#ZhoVc`El34)2gNyeOoc`wRdFA=s3eNo1R^r+>3 z3e*E`TUA$o_j(Gi78bE@>cF`f_NRySQz%*S2^;qksdQq17TZnbkDn@C+!Bq(CrID} z3~iSI-+1G1$2wbqA*9#o9XBAGUOIfV(Fh~DUm$7V`ov{0&Eh_AOlYw9^}BN6Z9I)R zS|}cMq0%7;0(?{^d!cD(Z}kv${2+aJ=wsA%@ZcpJc&0U@L#SB^W3XFMWbnx}EU6Kd z#C6Q&V>gNCe~Ut)T%&5#TQg^HIF0W`?v5oygkscP8~DWv_<%xQXEZqR)tUsXhm*Kl zqpb8g+(|O{JOm4l=ISFVggAry04i05>P3r^9bit&6@WCX@%U!NE0FP=29Pif!YND9 zhhl#3iSKT(EMlhH_fd`)?!^KY;!@~PXVnP6w$JX~r#)g5T4 zo6rxPJT7jIr$9(q*$&uMUp69$dJ-wUzESDaR5#8+`a$^OJDeW)=E2?&8u&HMW;?5{ z;jNrqDVv(#wa`*kn#^6t6Knn?!As{i8NJ$m%*KrexpCUBvW&pPh(~7SI-T00Tof(N zHhCi^`1J1MIR+nijeeObHjZl%OX)Trq?rwO=8QdX((X3yCr~*y*z6&uZHU~rNQr#% zJLdR~a@Y-L{*=XWlJmyB1|P9+c__41zALvf@G7c7hLswDwj@n4k+Ij>S`N73Fb@|9 z9e>^|$?TeNvVSKY=rMT!!QFS&c4RZXw>UlEOF`PhYZmL}Sm*O)!xGz-#JhNkoCW_Z z(jEdBg`d{5`JDV_cz221|I#3Sb6U!U5^@($-A?PTqKPafKN})1k$v&8WAiDezaqO) zPC1d%@&Z`>Q%?$&5m` zr)x|={YzD=)H;bD{lN3!i82*+6emY43gi-TyjV&~q|-JCNDVm>4ND?Ip)~+vIC=MH zS~G7i$P_LeJz~$!R@#bbYGVl`BFmqbG$fxRP#bL~*vPO4m?Hlu7kXlH`K+J7P)6px zKa$|xulnl{AR2dizb#8ZO!E!Bm^iTeuK#G*>7YiU=aLr~oZ^Rc-`H=mUWv?TEdd<) z#1r`K%QnBgK&4A3#09n=3+HQ{RkbmAz8x~n!;bGg;>a!(6T-gh^jJT}c{}Tt((h@} zeKvqpKJKmN=}VG8XNu$);M*%N+yVCEBOHQy@IoY%P>knA2SlCehM?0}06cH-hASm_ z|0SSi22c6J)G%@Dai4t-eNq@7R007yA!{tKN%_FuHZFD}JN3picrGc$)7#L^k?A~E zuDz^EW$TSoO9BbgzhyQ?d8G-|40b+z8Jr%O*%DC%$*xQD1s(+OcvLK42q+Upau1sh zeE9X2GeYdr_1Jx``3t)qB-3>_!6Enl?hx=-FP09tz|2m%40MIWQ<(B|t_#96m$zLRyzYsg12J!}XsbPn5+kP)cYr%`Z>}MV zD#1k`qD-R!(pVyCSmMEt6K{qWi;Cv*?T(w9y;g%FU2;>!&n@r?irnwoxNSUYLxbhl zZNS&zoKKP8a~1`SUgMW;AO9+cp@S5wPd%9+g}{Bi$!;)+$ddmP6JDn>!BN-*Py0_w zST|#6M$$zkF_$~`vglN(SepaVy#-F|~GQcw0 zUGN;S*q382816YUQ!U}t)6GG-B6ThWHfiM&065+Mxq7NCVFJjc*X5koTEmr^dEWr= zVg**a!IQLX=$Pd;F`%u-^0H%vP4N6%R- z5??&7NT$(9ayb8qtKLFElWa5)5l5}xmjbb;?%xo)m}@1V)v6i&!Gp)$meujW7mAul z;sO@-?7AfcML~`J{Dbv%vs9@$x9qQ%%mWVf2Ayt8I2OI89P*Ok{FFehUabWjY4F8E z+EY+OT8V6{oLG!P?E_DBSi6Fr0$_xOV}b)xQxO3Fl{#|XV6ZI504$^y$AoajC*K6Z zI$sWC@b)~X)71>MDos;J|6)XOh^%<$Gl{uK2LdQ)GG9L6BE{O~$CsL~(v(Z?tW5R! z&lFPM2R_~&C2k`PSjIll>y|2ILcmE%MsaCLty)Mdn4+Bq3?Us5U7hutScZtzTP>P| zE!P80hTYa?FNptW@1&6~|Ffp#t8_NM*M2&&4F~~SEvE5pTjCi0YWZuZ3FejV8aJxM zg@$5*WmJ}GB5+PAQ#pIq)&_L13QrhJPoZ`;TO5{qezY|SX>DKx>pDh)|FXc?0 zn*rukR;O)9;aIZlAH%`i%8X?t04Yg8^J%0zizqFUN$Wqj#RT7a>kFBwb9#}ZQY>e{ zJm9ZI{K}S!f9W+C6K*Z%aJvYX+G$(T5*hW4Z_=LE<=@DmKp(!ED|KDt=3O$kyNbm; zhBB$b&w#rb!dz)H=C<1IyaY7u8@U6!K!HxzBdP0MRV5bZA%wwfbLRcxYaondW#=u> z#ti;;vS3pze(^3Lgm|)0=xDB7=9Tr1XwqhvRi#kLQuP?OQj~8U750TMy`Qn`3{%4Q ze_V^oQJZ@5AKk**`W`IQt0ex$es^462aHDZ!E*?z4$(L+4C(>#nm2c~njk;!a!U{v zhHvN6;?kmI?cotngMq92r&%*^c|G>`D1s7wxvejc3~YnFgkz9F9{CLQJDs80%)H$` zt%L_KHk8T`$ipY3Hk>@wp6A^Nc)g|YZxYcYmiQTp^1aHU%E&KxZIc_$R*O0ah;Sjz z#2o?wJ~W}%^t}joy+zY26#q$=*U&QB7gaFR(8NOD$u)pU`QNkydlPk~y9)mytwk9*FQQ z<&S^9SobL&KA!&*C~8LAgmRcvv7C*|+VS&U1~62I$YW`2KjG`$l&*St)UiKWd@F~N zR+}lHUXYluYd;p63oW3OEsfe35)aRivi($|UdHs+w~+wz8K>w`@Xk^_M}(%042uT^ z_8ii^_Sv)d*{N93zr`Ag4j?z!t=EZPy8$ig{JVQrC89MHUo`xSpDNTxAh?Mw+eh(a zTFB*i;z0FDty-4&YO}K1PoTD1 z-;`hwdk6S!G}yIBtYpY%AY!5T6go$vdVjQ*aXj}VF#x0_XjF@hylmJ1SowDb&$Z-o z-F=(!_0N2cZkHpO{<}jE6#=W~e2V)i6pJ!{_B3bSx&cty*v)3c?@!lkR6?5VkDHP} zg-GQa)g?+y^MeJ_3E6grVu!u*HG%L965ECYviU$4h4!*Sj;H1~PS0x6(FA|sH4w%9 zUv8gsd9HE!YqBn*XYx~$AmE42Oy%&|aB)Y!KQb6}>@|@DG;h@0i33n-=$`rGB+&lN z@E>2(bf6*2AiAp z*95M6M{9huaM)b1yF~_^5l-DEGs+!7l(7Ja0YA4|nm)I+f25pkq;FjK5}S?2896z` zE@I?CX0trsS+Dx_f+@Y16dLgkFmrr*%iL}WIMUmneTo2zLSf7A>Bo5|b)V&?)x)m} z%=xX!$r?99-L>z3N?mT7Bt4Nnwq?zA*pK^1l5QP7a-{6qJ)4zO2=(ys`pZ^N3s_Is zsF?U9o}A}>HW}_>tAyOh_-h?98sOpzXH+{Pi9Xv!HhA}%EH@aki*8|CygfTMinRLq zlR^-9>Wli)t-X|;qHv3b772|lA#w=%-iH&0r!>`yI_>YcoXl2Yw{J#-YAZIH z0voJ0`iC)dN_;l(Jqch9Bh{Qff8`U4=>V(%L4cR5?Q5O(j-Y+sXjE63uv4`#=Mw5ia*hdS~ZsflU7M#Z>5Kt z^4D$x#?Qe}NgZwv7pXqx!~7E6kq?8{No1?8_w)9bHWAa$2@BR8$90R=i%qJW%p&?w zsH3$E2Ia14Gc6t;APQ8g338VOlD+>aVCg4tDWcz>8sT_63RNJsR_j1b0CY4Z$R~4Y zJAIcQFITBk(9Kg$Rf+taD0{3l`h_5H#gP~#ZBQn?WaxAp_Ew7&$Iae&LZ&}#cVUTk z1whTvB#!lW>#f&+=Zd~{qg5dT=}0kzq)IhVvx8C2&%m1_3cbevQ*_#!tQ1a>Zq6F1 zCeoosj+P^|+k_lcEavJmtvCTd@nbKYUmoxFlrgyfB|mFg6p)ozpB4`JG+P{^HMt?#HaK|10{jzXBglU?>6#_*|hnZ`Tt-Put*Y!EyS<*bR4noTC!~Kgi0KY|?ViZ4>v)@eKt# zFw2TbzQ-I2A2-8U!<-_lKP5@tLQ{$aLf873u)1|dZwcmt>do$_M#hUOpOPFelv8~1 zuw!9wk?~#K#+%X6WelTz{AyK8mEt5ji)dv}r%QNQEsvMmAkDcwA1@PMIc&dI35B?? zS(Ldqk7EEs?Z2UZM^y`MKmr=hpa!8r7o7j=G#Z+W{O&>WCyW~c8mVB(;XqqSK8?vY zd*~e7;XAv3yw~-jp!O(4oB(oo#LHx_V zk|Y3arkc*HWCwDDbq;B&Z`y-T4Nl+1C(@Yxk??s&LtATA)CN^yaMigqnmtj^*4v}V z}(U!rO)CZ@oz6D%Y>2mGdiic4<;F>cTze!w*rZ5-D zn_#K7l2V2Xf+$8Vv?yWO=JqQ zF(J%eWSyL1oM4!KL#b#bHdBbfF<_$y~ zf1_91EUq^DYtJ<-O%Mcshj@27|5pfK+^ zVgFwQ#e27o5LIPR&L+Wk+VQ^x8dxI9X^;5taPL?IODe=z7vbICSB&gB?-G)`gRPk4dUMGT%u zt+tVi^5THZ{#6I__*>6+MX8^@w-%5+6T6L1&;heZat5q{*m81q%1#mHS zDue)TwXf>+j5;!o5tr4Qi+9InAm<53i~Zb^0tRe+r)Zpt)myg9@%v;R0f1B&=7&^l ze0CyX&A}a%#hJ~|+V*^%2e~;!%^msbU07SS&sB_Engc54n+KKz&IpGADU|+djkyjy z_WaB7YT1Gn(I!~gKg0COi zVD~3>@94M&n0l3>!{;^e3p|Xt{#X07*6Lh?(QbU= zY%x`uL0bM_6i!-0A8%9KP=g@e^nhNYJe}L|E%_vgKE0-2xyTNf$25S@lj3}_Bn?Vl zgz2?FFgd+gZI*t5$6^jWS!zeXp>Cww(56RgH6`xEUe^d(*SK$Z9~b8`XSHVU@|{N zqQv;$FvQzGx%JT(Z;rjpsq58TyR!QoH?RSN8iqG&vIesZ^Ut@wo5!o|Qfw~u6qa`c zy*ve*m>#HN$G=evRPpT|6W#^ik$uRi?V;qQ5*^GX1m#!+x(^B-Op}?9R1Z)9(F7Hg=gTe({C8EnT_` z9+IPx(b<)+JFL}0m5QmGKaP|D$rDFBe`*nk@S}GAoYT@p-OtuSs}*mI3|)-mgc|&G zi;R?Mw=lG(pJXgB&O8srFra-r`m zltXtVo!HJFE?AP)<-q>p?|RT*yK}U7zXiE3CGqKe^OkGNKevp8bjPT1J3(^3T&}c5 z8li7dAV2PIbA8_qv~bDN)tQComq7ku%C!CkC%+f>U(ZXsQK09jRrJXpE^swT+ayt= zYo2Ga|FH?@BMbINi+u(b&6N_#L&&a`nCZ^E&aT^I6THQIaRYJB&Zjb!s4pi4v4}P9 z384ikhd5cHLwu92NI|x6x(&o;Jm}ryC+r2o2%(_4T5x_w{gD^6Q!r1 zG48|3sW>V+yPiiG0s7qFHxvY~QHv*q1)s;mkWd$uM`$pQAfTvlBEXws?O-(XFG{>n zQ0oIe!;?4BTkpoKfCjXk<_eIWFzYPBt^%ieUDDvh-I%(JcpIqTE{E)pMV)nW&#Lyi zzE%GhLL`;VVEXF2nGV>z&al<$#^OCmuNBs(`I^Vv*nK&nb~sFbw0L2!7(8JDz+Zrs zwpeIs8y+3=YNHdqXoAw72T5?TQl;a2T(OVR#z!@H7f{a;{!S#T4?wtAJY0_(JbpLO z7!SPxPihehKy<_mVDBKkQ6(N3#KGSHi}h$T$55)fLu0G$b_aQ*%<=IdwV%>-R9EH@ z!Ieyhi-nv+&`i7400)oPP75!QPBD0p-hbI8U5~4MZ5|KOJ>COoD$%O@6;TQ0QPX@8d~6^~@nlmlV4*(1x|XiS1M3{%DJj{gqPu@CauZ=yn@O{!1F z`Fj^`5OKT4r63`o$I`Ai)}id^2<8xl6|h+-EF|Q=OQEYky0eUz1Mxva!OV&HDerz* z^^r^I78LpHgS4H_Wr#6)Jmlj#6HNEnGdVOb{Y1P)4me-FNA$z1P4!L#<>khY?8YO7 zDmhwmu(SaLi~mv>H+aZ+L<3HdP~ErA!w=5U#6fJI&+!m;uH5hrcQloh zS@sKir~)WM$B=XZsHFwPrUaXC;YkDmr5<@V2lR)8^?}M=i;BIp0pcR7ogcrw&-sIw z8!g8m>x{(ki{t#;tQ#~>^W*rlRIi>-mu_ZjWIa?PIV(s)vCocW4bkoWTWnu|aNHus295-v=Oy#@7#nCs5@~gD%}}QD?d`PNA)ji3i=)l?Qp=bZL`j?z`5WXiGJmF# z;ns7EXRWg}n{N;V}E+HkwitS7OCO7ABX*lZi$yX9Ndw=>C#HylVo4Ge-}VzodN}=C%Wl8PGYDFIY0~I;vYjrMY0xQI6VIM z`evcFpj;k_?SehjhXW8;b$?RN*yjoO?Y_H z_dSQO)F@QgZCL^(WZRR14Lt`fYuvS|Y2X^iQ z(>=F3kFgfOm94rVDhGU-07hpPlm%egm(FfG{cv=Krc})>PGyGxg{#;Ba8W)8y$2w< zVdY4cFJ)qA%zu`u{7K&v#oxXnh8(^>BvGT948pFoRiOaNLIj%@?B3VB-|o~gIJd=t z2E#;>r~lE1*(I+2H3uy^4yD*Go-{92u-3S~3bZDfjO*ma?ip>ng%CeQ&{(E$c;7Z* zGk*0Yr1dY3wBb_tQl(o!v-ZB1sr2I^L;%^ISz?^QQ~Sv`WVZ9h#Uz?ao3)~-=L-2p zL_&G2_xm;`2@QA{=ERYZEh=l<;4s9r#);Mo;#U(h%HU#f+8kT1j7Y} z@IiRBp@qI*BELNeg8uGzL1Be;LiE^y4IlUlW`myj<1hcf`>_>I^=^SDhR;n=Cs0d5 zfTGj~O~84Zpa1|+Klj5tkRGk;53$Lp+3iA>4`ZTy_{C6tnYxN`iO@UHGx&tEvj_&Z zg8=dWSbOiVCbMpR7!U z=)Fp>p#+cs$+uDGT=SlpIWzBkzklYMNV1>(?7jBA?p5|$_H^HGH6qLw1}xhVPZDu3 zYCn3x)rkeUnq7CANyxOXzbIiyDXStyOx}&E3FS2X9DKGQH!6D@FxGvtQRqMP=WxuQSy>K}g9mE1g6d11b<8Slgln1$eZtHxZ+LUMT)3@|CRTEtgKb zDk#BW)|r?A%J4kutLS*C$J(_b0UQ7sx}wD;7@I5APD?jzhRerCW2Keo2~DoWjc9>; z&lcdGTlBjP#V)J%<>IBDY$ghAKPqB9ahLUUn^9QaMpPTM*4AQxqv8~&L2mr!>+A-j z#$dTSBvdS~Eetk#bU{VPbiMc1p$v@g%z#65ncxLlr(!L2o~MB1fKgjBU;CwlgHM{J zch@#W!Q;=dam2YR$8ILK)HC|k>h^b11FE?%^mMdNSL?nj5Vm}z!k}G#pN029wf}SO zP{R8+;q5GK&rVZbL(YJ-$4&C0SEc6NH`tmavxiHv`OkG%fA-CK%s`fUPq*CDWQIs% zcU3D+SNopf8Rren!^dZW!kHTGa@rZ{se@2jE)I!6r=^r%f}<~&&{oa_F*L7SAN#0G zx5v`XP-6Iy;Jq?lgC()g^-m9IzdCJv^u~0JUYyZMS9hC3n1vAz7lAY$Py$i9+G(G} zX2H!ZG)cETI{tKKel(xPteT_9Zh${D+#QMuoCzIf3u91vO3*dmXf9N=O@uqe3vAKFPqKf36>A4nGw3bfAgTy+`^eYmb9si!-m+*W?+->iJX)$TE z8-w4fJ;guvK6Xk6N5D)!XeUDdm^LAap}m#f??DQeNuJPV;%6`7OVhG><0u}bRAgiV zYFbgd33Wq)@7u{oSJo9OpY9^ntIx)ok?%ieElgn?6~{grT=E=|Zov-=H1OFLl+tWM zJCi|^2#s)WQZ(&DzZ{mYB&axJE>cg|e2`z7sBk*}4|ASV*a;Zj$?5qYcrhy{UZ?NR zf7qB6=1&2MrF#E~2d}6a#FK+F!`d0!WV8-9eDXG`Kbjx%Is$SAoWwh%{I>t#e~>VymR>z~(ade3qgMooj18dZB2PP_;oaCIb@aE;cFUViz$ zdXV4kFi1RAFWt&O)0C(6P;Dge`kHP)oUSC|JIpmpgV>QhM)Jb)5eQ9d=9{f}GMbwy zro@xXA+xSfP8pxOMCjfJ3$Nd~=QKacd)w^@Avn-?+}5RPt5%F)i$Du7E{r zY24iie5UT267Gy!w`wVYU-s4+r4RXJmdiV|g4=h+U|DGqSMO$u()UtnY2^fMx$n#O z_n%h~YahP&O!CmrGp_?q=$?cezDUjm>M&++&9#asq&MrawyAFjoKj;8b<9hKJiX4P zP%~v=YTM`QjW}y!L1FdkgTtPTMzC|^#?!zcPaAIlh@KFev=<1&HIi4$RYsV&lKHNw zH0CP-UJ}a5+#Cf94!0nC`M^osi&XgLt4_4zX2x}Ci5H&?#{2$Dl*X0uY$>lTm)9u~sWO7Te}H#;27Y z3ohheq3;6`BU{*Nz2{Cen>us}q%a|17<3>=tH-+^7*zUX{sG*yew}I>vR7G`QLMsF^^PVs@)#O3_!M{%AK*Gb!vgziZ=+%B|ahzI|}+ z&XYH!zGS5HhvVnNYMwmR@O?bY{B=JF#mqI$`EHk}?<)wFj9g$p_6~lC?)}DS`n+7q z4U?d%N3=OEJVr+=Fj~ZeFRAg( zntT1^xuQ#B^z9x+I4RK3!0srThY$moLCaC3Usx3E3Ht(jcJTZj>YZ5lA#i)C%`0C1 zK%1^ejApW|{O-5R;k+~w3Cad9IyCg=C@3@pl4ah0&$(`XTwjz!%}nH!PFN(Hk~DMR z*>uAOSuR#*x+PBF&6vwU9)=?h@WRtAM@6ui&_%>v7y&0i@J$Q<3sh7OKKQ+OvQQ?h z-bH=1`r;AAd$=C{Nvc)f3lDDH4@QTf$(W#@Vab%Han>$sLJ3MPdp?DvE-tNc0`+lc zPggC()mwibA09Y(SYPCU9mYj=CW*Q7$?timR0*ibpi;t9Vr}E#mocF@ue~u=P#ATy zY^d`9U9x%OTdtW24jT#c!suwp%!ZSD+>RHP>MdzSUluS)t*W|JZ?Q8dAH&JoK<_c6 zk?DhN?32B|RIPQk%Sv#iMEhn;N{-TA1*tghuyA2`J8Q*kAX_PciCprAVtFNfR8+T6 zL8qwB-B8a7Z-L})_qu3{lLuxgv>XPWqM|bFW;z9Dq+Hs~MGxLz?0oO5UzT32p|NLs z0$!sL2yv`b~QwysaL@8Y}b$);@BPX@@`;c77(KnD#RtXK6)^fP&-3cgK2w115opTjV4)$AE# z%Wl*h6+;M!3E7~&5QNT>YNNm*at^I?{cl>N`MoC zzC6`CULYUU>M)!|-BwU>raY>hfo|8p4HpnDk#8&rg)_~O@+~qo&@&h|RIR^u!&hS} zeZJo=DfrDERNsxyLA_Tzyx~|bvAs%KY`ssC(qG5*@cp$x-EkzgMLt$nX0{>4Rkd_d zCWtJH)?Y)Wer@2>SG0pp=%mN*rU8xWCXR5t@>pC37pHBHAM4tqoRz{7K zagDpUhmDj%WNi^{Pva9#^%TT5UiQHu9^DP~eu&lGh(R7v-3(u;#^lxQR`({5IgHHX zs-UHo)i95v{s%o}3TCL`U9Ws0x{jo_(H`zz+WU10Vxk?jArp(^ei=%xxQ1{kDwJc^ z=+ZvT^(o8CH=p_Henu)~)khW4#;f25&Ris2$`3svVT-CvGjr*;2e9`_av5T+>~IXmLlx z+E<@J_&hD`3Z(Eu-0s6r2v@-1M^6!=g(M>uDC*v`1*>$oD}DYaNjowE%SRZ^&UvXv zoSsj9lJ`}!K%2FVDC?GTK|JEg94Ch@XUvKcOH_wZK=G&xYJ#T&X`5hL;;~M1wPMUO zBPoY!f|~nqRKpIk(--J~u~1@RVq)sR40}1E7axePXDP(AOlLOqqg+C2)Hd1*?37D|%Gt zls4DRU#MHAeKEcjo9j0vP~u#7blGao*10bMk5edV%U4%fUPj!49m0yOMGFB8rzxHN`yxXmA9OM7WXk%}7OJS98jc9Vuu+xZ&s@|4R*BRb<_XWovZUP>Pgx2;wd z7ayj&aZjh+o)ms+V&XAy3(s!~^t6;(dX)Lp|3b&N88H>x#=mP|tVKOkWmQqTvGu6t z(=boGJUm&iq>kU}s;@6aD(G^Gz4Xk=cS1Dq5@oU!{XLPsCC2(%O#VF-eTte1n!KKwmiCkxj)I3vArY<9c;?9;CLR(S3spyU#WFdG*2bwomh<_=TIE_B0MR zUz{MKWZk^Zgt@|0%ugVCA@*=bU8}|ganr&f&&k?n>m`los?e$spYF9=!Bjj~TxLu^ z{@L$e&+ZVT4W(ng27hqa-4xw~TxO~h#kmFDvaSii;~ql{RGOy@Ju7eu;+LP=1*4F2|Z6PmSJi&(sE2xK5AAz;_3?8mZBM^ ze7ts#dWt4IL_m=h(<~Bx*|Puj z%E6wJK)5h5Fld7VuoXuH$@IJ8nCO+o6n3mynb$|!ra?f4` zn8~T^tNB_<_$0?b?>c|qI0q^q-)BrH2B(?vHM8HhJFjBp8BRL(5z+ol@4wd(9}_3% z+tM3x%^-g<(`iXnwZcVd>8UN(%U7Wt<^s`7zP{u?4|U9)%2SI;MlTjg(4DvLZgJsb z?m=qOq=dkn1lV=2NGQ0srV!}P>~Gf+isc#2o3?^pMr{!pvM1L3*FaB1KW@?s-1niu z7kOAV?d`QD0;zuXtNTOm(ciD#mc~z&Ms?{NzI)CU#KFf{vg7GY+b4fEc}ut$0Lvz( zCPKbR-paunbj3>WiPAk(?KvmvBqA3^r_gDkX&nw)&}CA-2oI0E`usc#lV9HU($iSK zzwJvm>-g)X%|WYKHC0uXXxTTFNUI@5%2e_MHL8mzSfLEHy|*hxT%*x%rA}6U-Cl4z zJ@78Pua5FwvG_Ol_%lZd>_nv`Hq@eHKI+w8S3AC~))+XnaolNiY;q@`apHp^>@! zr;QAMzsq2$e*f|Fy`P&tN7IS#E=dN~qvUcR;3o|mnf%88ZL3Mf5JKWE%2b#^;kk3E z6{9C7t|uXRddBDeNO2^yUcjuColYU+U?+Y@L*FTw{9R|50{ zZ~Xth!#$Ljl!cYGr8}d^!HsXf?LIxvXMum)V$W5g-zK@D)~q&DvY_cwwHc>SCEloG7TWS zSCGk8?SzYZmlUu5PfJL~fB1j1!O)u;X?qX7CO(CJXuk6K<8Y^Z7Zc330*0#iYFX%Y~QlNz0aN<8Mo65O~F= z+#x_4dJ|j*^!=Ys`sep3?h!S8_(Iy1 zFE#wvy>mb7GKz}c38NFY%gkDt4e4 z`ZD?c+XH(H1uxX`Y14&$=)(I>gapo9g)*Eqi!-|YA0Y7`r}Fpc@gINidrbVr{P>*S z0Vgz|o5AbZBk+zdJ`MjJ#r@L_eU{}rkQ^?jtPD|hg>WNN=zw0{kc5R?`OD}3AAgW$ ze+!l7My zue_q;)mh@Mh38@i|7!J~o`1UB*<^mwoiaUUXlUq4xWR!I!aa0#1R+av^BsGZd~~~g zO4{mU9QMdznpLye?}*5_-HK0c+te)3VGn=p6`mK=+WOhQ5vK<%src?Z5)5zgjv|<4 z=blIKwJlKAz0J&IAvZ!)I~#vDB5zyD5!`KWLsG5`*u5ro-zk#u91A&k^NOz>*-&Q0 z3{63)oICNa-dp10PgxFM$i#R)d6KT{G5&bpY}mo$UVq-hCbtkQTC-&0Y7uF~1 z4aA%g_sJL_+`?ouj1`maYVn~Ug`Nvk7%OhGtMv9AvmaHu;q_JVvLMEpLpAe5?ZLpM zgl{k*3?{*Dt^kH*&$(&M$eirj7sr$04Y>uu7}vhnN8II9zs8p&mdFJxh#J74%m?Mf z9BvoZx~IP7)zki32a(smB&uDPcvvF2i`gq*u^uVa zTCd!Oh)cTnIjUBvvWg#VqYaGFTt=CXiqdyolgOmLFRH~|w}0cb+NEq;WAZ^jd=yT& zSfcdx)Sm*zpTOkrzekZpAesl2^1%`;`FL5HV}o)dGoLh)EL~!t5z}S!k-Vdw>=hkC zyIs6GUZWzGI4A>tXDd?r^8(I(Ht0t=xb68cjO-tK(k5JO-s;-V;b|448E7{K zQ|o(gzQ=u2fZwYrFPQ6(G5B!HXH9mrl0Ty(R%d7l9KIzh0t}aCVKLQ)phUexzh#Z0 z9ZeUcEZFCubKGQaU9$YV50ZXibPjlL!6YM<&Oy1*@%gl-5Z8`k2uudy*}F~wfFg&MNegf zCmO5MY3b16OKbe;bG=R3!7JXyX<;a?rw!gxSus&|;L}LB))vFX{^}xMM1CP1%X3od z;--hSuJ%JBICbA?e*P_lFan2a;8D*|!|Fr*2vJqhj#ZrE+}Nf(!7djhUGlV@$XjIAu>PF!q|MsJ=OZ;X0!ri9Iy2XuXwLjt*@qZUuE` zYqxSR^Uk(_$>A5d8>^e_S+|bdZ(w`(Y=-I`2T5l~@()^YknfMgV3vG*kUgJb5Ol6O zY{gpGa&9=qcN{Mpp^@{yS~hLJW4A|P*5U~Z5Kt_T*w`@2?(s-$$E{JPFnJb0;13tA z)ZuIfDwwf#Hh%sGoojvLrv~aV-_&xoN&+Pgn`dq;<=f5Hd0?3bb^8YkVR9+?#&fL? zg;EVl@0~+ZkY5ooKe2}3UFN{iVP8y6)tH?S;Ge=Im`of>%@w7QHv{);G+*7JJpJWe zgQcNq-OD@Q1Y91%SUuRG@cx-mXOT%>xK@UHywO;lQO*^Srytcmd2i4A%pE4j3^|)B z;gn4eZ?sKZ+sMn+#-SaVO;E{*8#4>JTixs_ZE7k~eA+wSUns+?gW}dKz%sEJo%mPF zB#mDnZO7F` zBzrR@b8^J)LU~j-pCmi3>HvR#Gsj{Jtro5jZmVZA2t23S2yCq__3&IrLK4Sw5>{7o znZo*YZPENY0^*aGw`t*_Z&2Rf<-V$X?+Khw@$i1@DY|c^6Nl}nm?(+b5j=zop6u?m2)$(QUp~@GWkv+sK)!na+dR#AvgYwF8!jcPa z;x+>IEXH(MU+jH5ggdLpCMs-O_5@7o-$gz1C+DTQfJK($uyrXf-!# zSNEDTiYsdvQe#KlWu_Uj^WR#`$DEZ>`=o=qpMEz4GwbNKg-b5E1#-;kpKyH!7!(}1 zV27Bo)hcrjRWDSRWAoY1Wr^LW?hT`t~d* z=2ejUeBby|y?mKqR|8W0<+o%=-2qm%>Yy$t zOXi>)P#Es2bYxcgCy$*)mrq$eMGaM^SjAk`tYXDDiHQ=uR7%3d6H!Lu|LV{P;nSwV zTDyrHHh{>MGwlsMuC_HOH9yCplN|!CAa=WGzBhkBt7({Ql$(3k%DO zQ_DV>buJE_@(?pgxkh7Zs&D`vl+kzOOGW ze)*zlpb|wCL{@U3ATM7$(?fyiJ=`e)rC+KlVN$~-FnB@VWcH4zYPCAY{@Nbsp3&Po zx0FKSUZ3mn`rdhSJIp1q9SzIF9eH83+Ga{3z&Q8fv9 zdvfVs?cXi(&i0D0nDil|d35GpvZoXYle~*5myhCXbv}Yea~)Xezkm5#Er4D7>JRW3 zgJFYcUOm$sAMCUhhi3WvByaP@r*Vfn7sQWzm;)7#`Oi|~F}q=Rw|@0+Y@cq%DdTw? zUq~#``~DF?W_g^K`-=NV0{Sb+T-U01&NDFZfy?DB52mEpje0hY;>%X7vkpO8!dzy4 zs(^4h z_BmeQNymofe>`>IQ{n6i+K&T{>v{Iq(?4xLibev$#fj*EZT!@t_=XO9?#gfvuD zRll5Cbe1;j`m6#xD&La??uPslxm+-0)%kxsFDRorL43@hlzJ+8lq>IA5S|zBryibP z9<|wOJba8UyD8B>o*v`W`4=x=GC@&W#BFlGJ?A!Z>E^G&E&-Rm?tfvZQOE;eRspV+ zLdw#O3&2B@yb$`}H=7yXz3`uJCIxW;6!;3hu&}d_;z=@Ly1o%v*@5TAB2d}iNN08{z>un6A+5Ctytw40>T!+Bk%LuGID}F zPJ3he+p_*Pk^e({C@3kJcf&bke015(r;p!I0!f?kTg>7dkeHMI$B+Nl_!OM@9qkNK z-V$X5h|}E)*4EbVc00WNuYplt#5z6ybwHRv2!TP>)+O-Q=f52vs!r<4*_CkN$R7CQ zJ$OLqV$!jx`6m#*Z~<@J?S1m;;olDP+vA>TFhU^uPu@gEhO2o&N;f`@(c)tsTlP@@ z+Sg~Q;6z_58~*L{ACURCf&S+Y-8_$%I~9&49QX=*Z!7fl{=Yy4Wqshwmnl=*rSt^v z_(?cR_7}Y(4h5*tB~n8D&*>Hgfc&yE0AsfE{qTV29vXNY7z9VGL zh(P;!G6~uZkD*=fd@6Z1o7b;H3%v$VIY>3(x6glAr$26yFUb&|m%My%4vQrBG%Jvk z+D__>Uw(hTNFdxQ+Sh-Rn?L^g4-cr~dJJM9z+Ve&RlENO;(QDW-JiG13B2C9w0isV zfeC)kM!)#=+_ZB*kZH_z{|mBl6)aqnl7g~wbY-Q8?&teJ(k><64E#-ZXn@EcHNii| z7vg8~P_8a35Q};D#nD2FaMAH}wCyi~z&LoUBE|T{|1-Gw2T1uH^!QN{X#43Wob5`u ziY{k4dTCPo3}gx%hu+WprqO#q;83OE6YW2jyyiQ8H>r+8$Y0_jUL9nv=u$93Q%K)A zz5KO^Ovj6O-qhv4Sj4-~SDj5D!mE)g%i?(~a;Pe!*`LyJy9P|yyq-YB`ybCxnmzU8 zLfM9Ah+Gc?`MB+{+W%rjKtB^MdMU8eiEQhzVui0p>Y(Wfw2&MF<&1vkA40#3 zNnpiFSw>w}`5!JsuC}YPD%{&lf1Z_^8&BD=w`7s#8uW)__Uj~4^b^F}i#|!7uf{!2 zUmdPNYG07Q_nKr*{!SSw(NBE&_0u@gMI_~j^^ZtY;%%?_gyOs2=Ybrgbe=N$ zC(h2!_Tfn~PIH0P^hDAz|5UY%BS36&^zr4gv`1d0Fnd3hcL(2B<43_hgx7cERY?SLLW^8zORrVvh4lGI)Ngr@u6Q zm4BZ8JX3DJrzg#n)#84(@vhj5t}>%1PrhLJvli8?x@8yr?qYzU5RMjT-~0)s6jeax z)Q|;gUUlj%=T*ZPG_LOXFZ?yxbx?X7bb5GgxF(zvUB#x@^mE?Nl&3CPTdh%>c#*vz&?bC^!#V2%-ojS9P*y<`b zn^j&g?6G>E+n(|y+0yU?S}aY#e3~8X&SB?e=3l(SPaP<{T(T8m8v)9uEZYLh-E$9g z8GgSR|BHCMmc&<+P}L|HHx~+ypn1q;Y-cp^+gP4VATOqEOW1{nyfnO7&aPXU} z*==o*LXz%#4>J*Tb6!t3ay;JgQu;ccr*g-StKUAs=$C(8D&h>ok8|#K4G`-T_ny2rJ|a-|_Q&)3aC_-z`mJBr>9Ngm_!bH) zlQzoaAKPq%>kD1cS|7i)c{oQnEl@h0)j`D(gicEWUBMqVKeNL{9kI;V)+e(X&4 zoh<0HE2Us1@iD}rX;)7D#E1thK!|pXDO1DKUnR@@W$r+;AlSKEzL3EeOP79bfiC5u z05(T=B_Zq_^kcXIrsS8{5zq1+gVh2v-w8A*z7f-JC7fAveYr1>x6G>lo^Z+IC-J|cACE^ zS+}?p-uYBywb#w}aj~)2Ha9m_Q=uElE)<-?NA8(3eA=bv?_65Ee+b;xF{X2R>Ju*d zBnl~b$3wV!1q_;=EsB@ffowfB4vjyo)UORqSD6Cr!(n6UR<~vj6_@sxxVOPntsdDj z{L{1SfO$zLc>T13D0Gi)l$}gr!s4qG1^Dh|-JUjz4h~zJ(0n9iGP0F*-Y4ntv&E$2 zeLH}KePvDY-mT)+FIMkFIV6|bj&oWQU&QXjC}NkvZHBtUGj)ddhe8tket_MNy8-mDw>TiI;9BcX<5ec3-3H`fxa4Gph6yYitf-_6KtxZcHQY z72kjPl&mWp1WHTIC)DGG#RZr(K&t04TcNNAu*JOo)&XkBc>E$fml&w2+4;5oCT@0L zhq;VSd|~OPkmPyl_kHLQhn5J_6xk11BvQO&Pis3~u9WP7y>p=cLZ4di+S*8<%8*TW zdgE572d-pcHt^PC0O$sXmOSWWv~+(&jt}orswFjhFv#TK~ctKc~1> z9KFsbVt~Hg=mwkH?_29VwJLKi6ShS;O_MsfaY#EeQBFM=8c-;3XQA-bDk%?j*!tIm zpdtN9*R?F&#Ux&Ew)_s+uiIf$T<-g{>r7qNt(3+b7<>)ojX{EZsq*hnHG7E9;+S8r%gQ;3%OB_?Vy|sTtAm;Q}@k zT`j}rkXLeD>|cpF#S00)`ineeL=SUZl)4|+Y9vlp+XO8@1#iNq-C^bvC0&APqZW>K z`M2C**C)%|!`PD!Z;PB-9G`Y0@~1#BN0e!vjS$1fxH*~`+b|CtYlqK|p!M^j`0l6% z;wB#%;x3PrI}^;W4}Jiw;B#czNe~dq@3h)+)7~H~B|&IpAZ>wqX;;C=7%X%~A)8d% zd(tjkyAN#zk#n;O358DEZfTN>+_70}k6J1|#+O;mMQIvTkn&Yk#*|u&x9$bA&oB3~ z&v&HowM7@aF3A481+pbFZv*xS5;<};%JNDD`&TH>7h|72dyLMR_CcJ6jz%3}(hdBW=;Q@5&EHS?)q%88-UeS>ai0qv%RK&@D}Yy9A3PjIky zD4BuVO6IK;k9hxL+4-ZLqXV&5t{u#Npj%N3 zve#@BVc>L2Ggp=4Mv8ibY^{aM!FI?nxB|l|j>^6@n5lI!%!&wc}fIyr`OxoPKO8jmlU=AOi$tTGFT z+5F9|Q+}_BZ(I55+n1$UK&ckVV+|_V=@iIXP6tOPv3zNgE5egL1LzhwQm~ZCXV0X2 zjt+T+O4pK+P6H<=jwTPb_9kByt9yAjd=0%}_?bE#j*ZtSs-OKvI#xwJyt50ru2XAYcnoq@c2CQ?Z=3YDMrGbldp|iv-3)0kOxP#A*z%pPzDS1d zW@<`bp_b&8K)Pt7E4W7C8$Q!l#dn{26UYQto%0?OXK1NDn~>=}K5}Df8G@e@4hny? zI`V}UfwmxBchOb^fc^h(bb>NE5GLXwPR*0kOBj=LUrK|)SPvef7kJiZ**|kYQFOje z(&03TK1O4syA-FpJyI+tQ9y0D7}_opj-@Pw?9%Y0SmcORpQ=pqt?*WE3h`>%dZzY4 zJYzdfvn{?tVsL$AaA&seV#CtQD%c7yc{4T&rTS zvBQ?!b?$75VJJtY}Z(I4+rJfdbUUBh`&*ZkL&ZHXSo#p;w$3b4?uCT7u^@c9f4+jaNR^^_Xb1XHu8dPYl%UbML5uGEiFO&_3>%zj|x>WDj`XtKl`D1*1jQZ>tN!P;NeplEFNJYRG*>L(O z15=zS_K;UL?8>HY1m?(LxY#7q8!^(8r6w--qKEy`xzs(~^Hq>@9QYVerxZk(PPiN> z4a$LXtVy7dAuBY6hgah__wVmE?z8g8(_P{x$V<0sOArTw!ZK)O@2k2XH{Cbm6l2+q zvQx@s7E|3G`%|=qj%qj7fGcl;^jQTH)l(n+qz{#F{7SnwFVU80WnX=|#1FOGg31fe z)jsGnB}~SHMCc3N+@$Z|qs>f}% zDT?Q-x4dycj>Fe8)nuiRa(!nzE%7i zWktA1Nsm3ZLcyb#ckJe!D|)izxL2RR992R3kwmd?X3O?#c5G-TCXgM3mP8guB2%~%v$>auJqALqQ zgSEt6G2H6+Eu*-3gE~;%K|7xK#k>Ql*A5l6HDN}-P_-rWv^2#l>c5|uj_a9Fw z3D7j~Y0&r#nD#pfF*MW5S8Wp=tjbcg$E4Mdx>`0 zLvsa4&6Q8q!FOk0guL@H$kofg+tq1jtC=>)pgQ4>qU^RD9??#8t1h)19+Tk63ArNa zpe_VzNn3Rk1KtypF`r#uY$D(GTIbZ=@jVh?IlIW2jXcOCT-<@~oO*b+D@H)Yexsj^ zUW`QrB0j=PBeB5=uAJt<`=8+pem>AfK9 zG+~3ZL+EutcD%{EYGkt3r`~(iiBl|*$yX1Of4O$Pd~+d!N=mxM0*+Ua#k)Bd8Rguw zOOu2G7NBX$12ajxtoHJ4eJS0E8mXAhST8-Z&KvqFB;Sk0t8O}TW~_XjESq0vu1|V( z|KwtVCNJOkvSv(mmcqT^tL(2@Ecq)MM_pQSUTL&^E)XAAS7fFrI%ZFM>Z!u@P^o0= z_0$$i5ofz6Z>|WWUDVz<`K?3mEGsFyalX+4(Z{FcT`RYI4qI~O3kUXhzpGgFJi-`w zD3x=kyUo#wyENk@w-Z?x%|C;4=d|6nxiVIj{S9}!=5Q61n4Rd6ddoLyF=F1{1vFkAJs!MuHKNdp}#%0$#~K}pgs)~cyfZ>#?; zL&uh~YMrhYs_`5y8CcQE_ADnC1=L1CPFsj4>&}4>uskh_{w&}A<2hm$DE#4yW&#DG ze3cNDgrW%FAOgbIKV*tS;Z4idk&~ap>N;hv&5P1k^EGwHq`-Hi`Rkqz z1&w~a42SJF8LWs*bPCmGgI#KEzAM!>Qi{PHTME4sDIVei?KlS?-Xxipr9u8RweeCD z^emy7&opLP{q-O+OCx14$9%B}w0hWHZim&o2Q4PmC0g-vc~u^?tu2j_by44J)Paka zq@mxX*l&vR+!OR(qS@boQ@ydyZYZM)c@>4^MJ-?U#=Vqkq32xnkl32Jt)Ez84l-g+ zy4wsFLhDz$40XzQrSzfG5H>`9TF6Ra_WLX0+b15g5H6Y>-cSRHT&?sj9BOyG-i326 zSBLe|B)RA1-j26_WQk27o&D%{!sSQGGf=`w9lfZrtA#yfm%b1vf}WmYW(!jr~1<31;Aq;gO;qvdS^NJ+cNKd4F# zmpU)zM6_sksY}JOAARIy%W}m^pF!s4$;CmNAr`yZ9Y0p2yKh~AljWosT!r%B&%)8o zGJe4AAvX3{GYuG!$6|$2uL{VHxRkjA8}sXg5k(93jiG-*Rhol}1d6 z#@L0ZvW$U!Z1w$2d54|h=qf$T7LJNFDQ7?>P$Z_78l?>HP8W|I33hn2`(4Se2f5&F z9uoU&hy!g0oJZ;@Ok6)4hH-0`@2zw4vgs7ERY0xqb9v1Vh*e!q<+HmH*C3Y6+An`W zN%$}$JJ!jM(olQvv+z7m1Uv=~Lz3h#$u|Z!$C4sh&QKPQQX}(OqN^UkBxvJ*)IkMY8ne_xCV}U>>CM_>_ z;ygd!-rpIla!E1_CyRR07`(5CYfABn=1{9EYeQO(s25fj!zZ1`LF&Ju;gs`HZvpKp zHj!>|Fe@ks!YLa@AEibk^!C=E!2trBksb9*hGU~Y0-VT4`m8bDBPsG&!C#=cA0doMldoI?`TlX=PjtXJ+Q;LN8Q5a;nRy=7{JVk{PV!zJk!@p>xY8LBbK9=>dimA5F7`;L_BPGJ2&(*n z=6alMl-~S&C&^r!{6oj>`8czi(cltFrq(1~YQ065pen-B#Ia6S2%vEw1Kvf9om9Bf+a{bn!c!0JIf*o z`=0$aG2D8*IBo-0kZ>0Rx^l_`1)k~9NYb;~-kW*1j1y9!aIvY!&e@Eg`jU=kZWld-@oH~kgUc1}T%RN%91iiiMUw^S}o@bQh>y^#@g)-`eLYQ!8 z`I`7#xD>9qB>tx5;-D?2-5~T~y1GD9_5p09)L_0j?Ms7dxgv+f@V8l?_q=-LJ^E6( z_#>Q6XNk~wq3dR(l;)yy>R>?mZ`N6_*ns0`Y5QhXQBu-(s~M0f4ZnlCXFoGkxHIgs zF4d;l`d-v$h&G|^Eida4FNzh!FzxM3>v>J~4#nsei*T4x&*3Mwv;9a9t?jt{$F@UZ zufiQQir_`_rYp93VwrI{cE#Ieccmul{wss{ORN}Z;-MW`A z(iESNa%*o8dH5|R0Utv&n#8f7p1&x%2ORHn8TFd2pLxVh){>7@KOfS6@7v1Cw2O3C zEm60R!=cFBo?s;8PS22_LtA5_b2_b-L`@0M^ueCu5!4IZz;NkAeFMphjf6Cbm*e#4 z+%BgY6g%f|3mGI{v>T2XzJ5Y`qEgjTBu7PUvc8t=U9Yw6g${hoO9M}i5>sT+6Hrns zk_k2zt;`2pShzD)G_i7shWFS}^h#aq&^E$6&U&;4Rh8 zE194n^VatdvLzQJ9PE(C*=4UO{hpF`trP|7ya23anM&Rlvc~Q_k>b<#OipdJy}Y3F z&AOg=wjL8LsuT{-K74G~ygi}VFofV;S?MhlgWL7&Gwcc1H(F>p1)aP>#f{_+GFGb!y1AGLVXJsq&u7u@Xpb8}m3Smh%4x^p)xaEsN>Q~ouieDd z(>=d0#T4VdP-c&<#PFz%OmM3f-?rbgMxyz*Qyx!xOLaIp2AX|(dHSsJ-kM_sqUy6p zoKf1gmt?o$aB%-t%R(f}Qu8^K(%J#YFCVeGb7JRm*>FA95Sdme@Ew!y6*#j~t*&Cz zk&w(5vn!9=ZkTS6+nO}HJDj4&yXu{c5dCQT80+pbyHae1f>5r)nF4f$2WxlgRgC+W zojWZd27IfA7%{FLi7KQL(l+T%_~s0f?WB{f=URJJ-Hzu3Y@uUol!0`%4PL+fK2Nwr z!U0Et6ikZT1`YGf%)+Ydn+_Hvkz{(57*Px=W3PHX78i@0AEY_TRkgd_li-B5>h@{h zS@T-6Z1$ezvKv8`R_9CoiLa*_PSA<-8OYUB3#L-iK{J-{FBt2f>P~lgyt>`w0VD1^ zc;P7Exr~56^+pxmE%-D$-=UiSP_0brrdj`K1#w}%XQuF~dTMrVsqcaK$q9mqRy;MJ z$j`zU-EP=zai6AmS~KqY{W%TaI;}Z}tw%u@=|9s2k+nq%%(vx%UPs%BFa7O{9JDg3 zw;H9aW3P^eTvEnOT31B(+Rpr|($lK& zsN(M$;T$bjB5UA@$tB$DVjY&iY8jj)wHjzRwb=W@c#92AfKK6g#FoON8{M+#Us8#@ ze3lh1jRs$ndh(f#FT_*MB{na0BwpBx?Ff@io*i@*YH-{aYSXdetP!B1RpxU$!15U0 zvl@6Gp#nJ?)2QjQUk+NFCPE2++BuIAQaUi1=iYbdF&2kE{^Wln6(CvutQIpeQ55?$!aP^cMsU1=kBI<^boA zdoS8aw2qK%jeYzEei6_NuS&bTN>p-9mh|(j&a3*zp#oj%rvzxwUm;4zmVOoH1FsYM zs-d>-)zYY1Vf*!A?2*fxPXsx*cFrzwy=Q7afI^EIiqb{o!VeVZ79!vw0xJ0OXPK=& zXrCCHh|2D@jlw2rObyB(j#$RYZTiP0Pa{-MRG1;DRHtEBLZ~no(qv|qAy2Z0$ ze6FDS zZlV0~(d1yBmfk?&=womGhkYRx<$c#5eukkYxEvJ)ZI1LG=+r|H5(lLMH}=~^0|~c} zmL3?T8-90pwv;<^FSB{LaW(-Pb&~B0d}eSUO^<8oi8OY-yiq-YL2{n%z$;H97;cs~ zZB+)NTVPW&^vup>^X`+dx%SoCsflNBH27eBZJfylarqVx`k)7lIZ|Z)STCSWT&WRe zgO&b@Ng=aKa<%j?%u!+JvRrQL=*V1t60`i>Ole{ruOB)7XCS7nq>d4rKx?DEkhWnBeDVBeop_xrZv0@wY z*M+JG%W=9@vk#nVD=pr{Q93{gSVcPR7*PKwXt+SVArmv)u<3eObWLzHTp3L^%m*5C z!yabqUBPNnY%OaOmHH|axu_k1mE`enR3x7^d-nvPfpJ3<-VC#=;PU9f3k8;pGbFB* zl;G$NnK63Zyn~hsJ}hjf+_#@_>7vTDHir^WXv1xE(kc*JB1~uJfT~IH6P?uZT~MXu zBp+rw^d&|8=sm-moBm%Z8t7{ZAN+F9yRWvrfB3Mb+wnm0%p&hIpc*e7?wa;SZARdn zci_$bz5$87-+-H7zWk)0K_LDWxp+j|TGd{grq5Jq2NxIbwuOUXoeOxE2)lpuS5@%0 z(HqsOjIc9CSfH@(EN~X~T1?nw|5lG!g3NVN{Fa-%57bvXdI7|g)0Hpti-ZaMguLc> z)N5_8OWO*^2i%%wTDk{EzEKHJCe4Il@jJ*OPiwXXAxf>Fzt0`b>H^zh} z{G_4NlI6>}>U^fF%@}R8LQSSOJ2U@Sy6x!S=fjf4p!$BhdJ#&^rOIeaOM%-UVS3a~ z(AcMn5cxlKN2Ln!c=2V~111YvO-Pe{q4_Ty+oeYAVX(v`^g>izD(-kSDI;giR=djlLuXal`8S3u zmq>KNOVe_AWq|OUd;;OF7763|L&y8blcey52rC1 z!R+@}?+OpuW-v)*E}gQ)Jlr==R=eh9dutz0r+UYWs`NQM83Psbm`ip!J;u)O=z@^R zrm&pAVhH)c6HTvxnQ#ySDpL4!eD)QytL$^v9zr2sxtrhL`nv7^JVb%v+SMb)|4_j&IV-)s zEg+t)+h%5x*-HJvm*TI-&H;D~ZW%2A_>q38tZ}0&{6km2eudZ3L$`+3_^B*XYtJH~ zp&whhCjDW@!ep`n#!ly&_H1$C)(=$rN6!aq6r!q-bkaAt<66(QPgQQUN1KWF!tzgY zb1and836j|#O>k-)8T^j!WNZsbZ`V+HkiJ?lWOazRPF`_zB+7X&WQ2ceyja4 zRw<7Z{uPIdf)+J*n2-MIJrVZ?sVvoSTNpYmR!<-WWlIkDI9Wt`-h&krp`h6B#|&eb zQfUPj>qb7rD+d#42T?S42Y5nUgG?Pf zBs{UDjrE>+>nCC-pz`8~*3b@e$6KRMEuN0pv2}Cb7x5F{b((s`?%Y?~sM{365>k8U z)cldLPV?_y2w8fIXT|@&59wbs(*ORk^2Azq6X(3cte}l5ITbzh#RAa%m|KFSbOI%I zk!KNY{zo4M*X#WCbFbsez1T(Jd?T$Po~>w{bt7qXLj!=b5_cE&aNE*^^ml9n{Y*)WYH>b%v+CxpcFPoJ(a>x#us8Lt}Xf{Q|brogOZ^m#PcikA2rPJ zw`W$Rl}29s$b{P7l|R5E#0Uc&bcJ{r_@3Lo=&9Bu9_>cA{XO?GxtMCEB^t5$)NsMy zX$__hxTUPW#wc9a?Qrw8dxMoRPC0Qb)?~KwO!9K;+Q`Ie`DRv~iaz!cY_|FP%5eV3 z@7v^FZtxa&INADS$aQ^(u?{r3DfJ0;U8J2rMte5?{e338IE=CAr%xrT!Zc~4HEPX8 zq@(?n2{xT7+67x;a6>VBeii&tChr=f->g8B$tuNi=Rc6p*!B@~O1&-_e zN#%YTe=t-u)^hhqYb|VIcUHaDo{b60I9-Kt%>vzQ>?+Vz^#ENJ<99vY^72V+HMq?c zo*=j}-m1?ne@5TN$pP|=4$Y3b0oC3pw+mQ78>2EXgE>@ZBh-qm@`Z%;T_3OSY|YZR zkpGX+FwCPM(&bsZ<*I}2sn3P^2BAIusS%eH&#jPcKMIG%044A;S@O#URaI5H2=Hse z7SNvmVlZ0Ob_1}f^HPugRTX=Uj5hsiX1ZJcyQ%5sBVu$t>)NXkY~#wxH(QsmKm!p(1-CgZZTC-o%A@e!`DuHM#5LJYvR}Bq^74Ro~f7&6Ml`! zI~7{%U9HedN(nETm-V6>M<~U_<~z3!v)Z`I{)>~U6FVn7#vw2FU34ojyGfiN0-K#Y z13kM*5TSb~Ca@A6Q8zls!qWG$z9&&o8B-}J;1d(jn(FLcY`)-~_elQ(2`Rf3TEnt% zviYL4_oTl@5=96v7}AP+)WAKliQp5TC$oUK(cM1yQmj+&A~~{9I~novQ!PR**w_bt zyu_%wIPB~?(3LKrTcOw`*0I&{oi(S$bDM33y!6ARSsO%PE8NR@eORL;7~rx?Ufoyw z2G61rolB?heS3C<9LoTdmoL}ez>fzSm;v}(DVVtN`pLPZ;yD*3qC=Hfg7Pn@X`rCo z<#>m?pKpMc(A4@n+Q}`*2m=v5EOk+4qfeEsBc2uFLR#$SS=NzcPaEZ~3AQo-J@;iW z^~BcMewX8%w5{OYc|yJ)TQ&L}2zC+JNO21}d8m0;1EDy{`+6wGat$^3*8@F~vCGTQk z{ky9=GQDO~yh84qHiM0VX9`6h+iTwwK1`BTD)B8jOV{&9l|9-W8U%os$dNoS#metN z!g)Z&!#kbb8F7c)75)2SUv5NICM3Ym_lsF!P3fAx#(V887Rzvh6&$?x4pOG)O(p`> zVDIwGVKc!a*6ZtNK7P3T5ZSD}FOv(A`^giy$Gf1{4dfr_{L&bRBlZCR%~R}=0Pyfm zj*&z0RK-cvOF&4=H#Xxt#1s{9zVG?wcMfx2z;t>tu-FSL`gO|_3esFLV{onBbiG!n z;5z`3(F+O-qus~N+SpHvDZN*~j_SPNvI|1}X2!t?%^{Qzs##u z8G7(OHa?TTQLrpg4t7U>2viIn|02;*@=4%oD|?~W?z0^pa`cI$Kx1^rmK>{swMOqw z_cgLg%Yl2(rDn$}=IgO1C?YL=TA7>iKZ&BztM5D%;Bm6eFXZRwFtX9J_4>s3>#TQDae2rPMqCR2GP3b6JmwCC9QRL;v zX3|y*koBLTtW(Pe{-fiG*aQGwC@mU5uOnT&<^g<=vy;7h2Rhm8V-+N$!8+>WInrle zFkESFY$@tszm&O9+HCt!;L-5u+CHWw=2C1lC=~Nz5KRQed|T*H(ynPTju6!x_5tyy zDsV}^FoYI&t5;IiM|D2MOFk#d_N(CHzWlUR%Q_WJ+M9<7uix+B_zF!0j~1+|G#r|z zIUH|`tN0xU@|4}5sYqQv)*Dy0w@v$2amTv#Wcg6b*#*|o#!Vs@Y?GZQE*&!ut2gm~UkovaF8Ul=q$XwY_UtXJl)v zFc(`M-CuobdhIUvg7Sop>+v(xF!IT9i@@TdEw>JPXb#X>0gdAu&}k|_d?uhgUZDi+ zo(l8Do#l6$uUfO_T4ovqL*3l-C*LiH9v2T=NoVK(9X~_4B<^mvY#qC~)_` zigMK^U<3zhi7eFuVTh)UL4#*Q3XpVu#b3a!rQE7eBDp)E#Sczc;5Cp-DTQ9!TNV27 zLH2)!UVLfs82-wG6nugPCy*g^G!fu_3_Bk|Q#4U__|7Rp#ZAsabx>Vy-+9${)IKR$!e}DA z-}g#`w-jlsfXbXegh{CnN$njlAS*XrzvFVuXL3NueEFpo#bwxA!Th=Z#IO;dd#(WC z{@`fkwKoM3EkB$xD(aiVC;+-5yk6mbL*+=DjB?mL?;Oi}_?oMIu}8w(Hgo<=&r}oj z2ARKldUf9XKp0*kX_TVpCB%FUx}{aVi)v+oslmB?_Sso zc)wl|`nlzuZyjVj;KM}o@%cSY0~5nn_Nq0!ZQJ&XYvp|U8L~7H@jP+X0pJWFYDSRL zBRnbUMEsiA%v>1dc@Gj|RVq7x>YRp#7f!Em^GE4s2pYjVC26Un3ZPF^&Zn_~DcH(a zJT@1hyC-`)6Ej+Go(l?9gR;?mB9W7DOX}?aD^wpofH7I@yO~?8qW2%r*$0=bC92n? z$$+u^XfMReznBAF`c=A%V1<6dQg1Sou=1|}X2ciX((bk6Hz}hZJO&MnVD@dB(?2T> z0>0&}po)eUppUgm6wwEyA^3bOD8-xNc@gi$D%i!*|DcV4+o&%-YxPNm_2kyE?p5|y z-IZoPxFwTd*5B%brkV8~nICPXCT=Aa`D4V|k#G1DIDW!HD5|}w7Z`Uw#syx^l0OtH z)914~&x`8wbKD$Ep1LE0w+gsVxD36fgDpAZ&{`V5z_ZN1ixy6re8O!(kQ_SSPe)J4 zWR<78eE@r0aS6s;E4^>{lGPXObGhIbx5bBXOngp(kw=$g2L@`#?42Z-c@(P^(F@l} z?tgp_#a}NDRlke4e`+kIs5tkPO1;Yt=ZS9v$lK~sVyqHAh`SbYTb4aPL}&*p~``_9vzDuoUg>_l!;e>E>5$(xil1tse$G2^qt~EOWuQu)wXN2me#iP4$ zj`7qyp8d{la8j10HBy=yK5+{-{I{_AChKYK6Ki~4Xx3q;H^4C7@-LtC%!v(9-KVd1 z4WLpN<%&pnw+E7&^GfkB+qfN;9=oR`b}zp+K!X%k4!YFLX^zZZP@7 z-mMp^C@~-?LncBY!TS&Zt!j&#*Fq<7Gs_)5rTVlQ%HB?cX!%C`b0o~8StF93p$lH&;jkoEB?EGT*Ca2zh3YD$*~$wH1xHBFXo52LWx7c;=xStRQD5S9*qNFxlYCd zpf1Lwb@8rWcRDry1PpejTnEflU)kQMxNHErl6L=C8FY6EFES1IKSDn;T@?4wSv_rg z@V*0Q_w~qez6?1ppLMBOFNg^2{cKa*v%!`c*lz~2ayxC0WFXF81{O2uY+?-65*l5$ zw+4M{HEHVZ>Uybwt$qdHdor7=oGA7UPqG|x#u~z0F>Gls6!`ZRJe^Q=q?vs0Ipj*p zaLTr4R{E|?T48mRYV8IQ%qPjF=?*DewJwpR3d;#yeAMrv8`JJ_f(+e!uJDG7#5aL> zJrqU5d6wDDE*3a!mKi`*y%eT36zyD(+rBaBpZL!vLumR5a-*~^>FLDlVXGWnOGND z!+?I;fF2?oA?hxtw%&T{$nYCfzLe-$wwc2(Ql`|kQnn{R6}jsRG(JVbt@g_Wz2I0h zq6#5#IORBQC@y`P^0HFRP`}4On(rvZ^qVUccvnj|X&Y8QD+3!xR|FeE&OVdO7wOw}@9b*~} zN&F|qO-kgk^3R@On9rUvhA5l9xc66AJ~yw#>k?kbuf{!Egwp;Pa4%tS%-*U7ve&P~ zBms)LNvd0zbf79YX|lCH-Pd$KFb@9OFW;YK4!bzE)RNXEJ(>7*oR!F z@m|()L&k4d~+7U)r)l~=1H zSvUSRg3pYYL4kg12ys!gQM67GP4;(ljjO`}i+9cSm5HA;&Z`MdOaL#?8IHhC4>`e^ z-vowbLuHlRDvmODavs@pCBQwp{hM4`73F<}ob%_shZ4p5hd`o9NJmWz z5CwGx0=7hWNGGwyL!otUM%^)Kd#S>#!9PfcuPctx)Om<{8Ni+%JVB&qu&LeKRTDI< z-R$i$AE_qjUc4B5F{4K!qMwpqT_UEC-$9IAX4il3r_XTJ-fRM}+#au1hJ)d~a#rHw z7(XhjcE1YmKuHqb*Y|gSk*g7K$9Oc}nPT=hK5Db#$u_Zm9&$40HxwBJ z4Op8pn->0YM+kn%Tzs3>(m8#tY-}swC*YDx!`=I)ZD=kNz4JDoG@+r9o)LQo+`}5P zLhWcSrk7F$vG09Ie&g-_sCGV9j`8UW0Fk!3B`2D;RVI#Fo%4oqk2L|F%PmT}FU}>4 zDLUu?O-|^Vx6Kl>NkE8Qv5KxeeH59(r&6yso=rrzHNJN&^2@tb{90G&*I@-b&IED- z#YcUas_5+MaNOi&9N7d>@G)kU(0_ju#}V!kChFv;pR39@lXQ>mTn;R@bWY$cQmfs5 z(VyLjo-N3$4|C;j7bl`*u9(qKo;fe{LRx5nzrj|yhpmv;kJ)Tn<@#Xp=(Wm`k0(NisYbwpN{Q?+yeEH>jh~GRt1(eRPF0`B-k<|{nH`W zDWOVFkp_-|f^*UqV5K?2ShejDp`TQs3G)91^IlTpmr3{s0dwVrG2YQyZJjLmZ z+Oq9)Ho+G_JNCGBNKG%SGCSJkaa+y^io#sAmz85{L9LJ{NLr>6$J)`;%+ZL|ZQo9^rQLc0 zu?bJd`L)%f11N&in{qW>N}0zoswitei78=jHQPmw~LDAO&4 z3`+J1YY?=$Y4aM4TN%9OOVa=+4ucmS8{5Rey}ENsffR@nIKI&qj#~;o=OK~ZjUnA# ze<|8kxjn4f)efvPiKoGebth+YyfsMn$1Ra>^7Az$37os0ry2wL^P8eDX67t16Y!s{ zz!gDy8w-o#;sE7j)(K2Hp;Qj`_#liH#QFdzj2ViAkAy;SCV|}phgxl{ac%#4#!L4C z=R+y8`J-qn@FRU;Cm_nwXaxqLZa=-#pk{!_=yxQ*oRVW!E*p78SDX%2T6c*go>`{` z2v3#6e}@a@%ATgBG&Y6~U#)GADJvS=&7Ne%hf{&rpa;n!w_NnXNTH+gceC4H*qOKN zu-s8nXQcsF!NeLVvf%wb$F3g>SLR5j8Pi16@yH3OYxJi^UXUBLr%YAO9kxa)v}cqf zCapd^zf0QjF@RwjPbXzC@K_HA(Oc<9w$B$v3roidc!!7U!1q}6baA#%=jDMlwu<35 zxbKFI?NX<$Suu^#gm8WmF7bwdZP#ti-*HI>ZT9(2AiCYhC~RC!QnXa=ohWk6xs*t2=^jc*!C9 zNO9VH46l+(>Vb--^rsZdTf!$Zf&E^2Y}`RVY$pS1jw|j871nu#HpApKvnzBl}s(# z`OAm-^KS~qV%2MOVV>S^ydAu0u^+r`(i%gwph{z>zBm7lc|$25?MJRH73}ZbB<1@k zs@fzjX~m%=psfw{7K6pC_j~$RVkpy3N(M5%XwkP$M z!ix+&4G{-cFkq*k0iVr0N990`z6h(C6B-QV5_rRJqv#}9tQ%LT)Mdg(-TEuwPt`A$ zG45U0rT~xhVdU-_WZ3CzB|H_EgFw=2``bPFu{$fRk&wYqW;ddEIjvsV;7-t4PV?UD zhfV{%@r%V<(VZB#cZAC2N3#B~Tcgf`7Ujr`^_BY?@SC{mo%M=WB zg}B;_#OxIz9BhP8AqN-j?H#%F>kuD%QMEohn~RRHwi*%5TM`*LAh&K!Bqkv6xsu6Q z0mpI_og4hebX^TdW|nU>{#5N2kk!{}4xeBjW$>fTIwZeoY9q^^`Z|5l9vwTaSec9fRZ;&LcSj810Ru?rCcbR=P zqY}vZem1}%HTIH1im5rOcJBa)=Kb0JftfYyL0>E!Of>yLY|j^+*5etk6v!G{vpDZJ zE)jVR2L|A*gk&2P9mubhe86PUNtU9jcvW{d}h~AM=*Y@uY;1#4i+qdD!auizA>hZYAE)BbC%Gr6?=E`JS;Q0}`dbIT^4oP5E!$N( z*B)4H=h0$=`E7HS)ccq>tpu)x=<^+AZGw;SCeFJFqns+U=G7IVl`m$!-51G&>dL32 z)rY!QAJhTq9?z|1v_)unA5pFKqLCbJh)UGBX0A}Kr_x(^4rFj0sW9KD03@X=>)JZ! zie*Rao3H4rIO&EBHj!4A`nHlCHjzN-O0F{%^@4=v;U0-#%Z4z_p;cuBE>q>vP&>U% zJ$!D=5f_mqVy~zS(|QtIEI`T|6PmAa97273B;WnE3gr3}(@*pg?6j@mx}6TX5a;v@ zQLpII@D%`vK|KG@rDUIr^$A=h`Ss#Cm7mAn>0}mLK<-h51(Dn5kZmhb^TdiQ>}Wf? zIDR!Ar~}iXFx+_geSWZ5GZ)Cay{-0{ZcPpCNKVpQUuUDCYXxge;kcgv0wV58RMe-M z8D<2!o$BxB*R>Ac+l3gmWTy>hWoFJG<%l`&zCe5?QM8DO2HRq&c&nKv)^|j~2O@H~ zfw}$Ao3~1lI4T*01R>bMrJiqunH<-uMUB*LvM5+v+8o0T`zL z7|q`909OB1x>j}_fx(XBYnhN=hvY--v#mA1(Z;VFklw_gs1#dqM_l>|Yl|o^_FUV~ zK=~40;r&Zn=3B`vqJDL0HsXt>msoPkDJieo}`(q!?Pl0{` zz`m^5B5yu)G;-PM00eiEK3J3W%vf^AtY&H&W-h2ML#Sq5kXab-Au2h`HJFsL5@JJp zSwFn#;Cdj=6qqDQO*EK>pfc(mRgZ2KZcVDc*#P|e>k0cOF8JKk0E(q+ePhZ2lJM|I zPc_Mku6vJaE^l%Gnf_v~mgI1vY=VXqi9A zw6S3*J(VppF>-GRBb~*W(PETe6PB>#Ex*DO>K|`wMYi5h7-lTEdmGxtNW}T_WP-oVc3#%-Hh=e^NF+9---F$dma$mftAlYiZRizm zeGjpq%v&Q_&V>{yqzF$=jQoD=NEvj}E$(<;UfxzQw)z7oH0Gipc5xo&^wH>sF1e)c1I}JSh5Wu z&vq^ELiHr8ac451W)OD}CK}n^cCWFT1nDqiy(&eQy{m9W)_erYl4ql42d(5&^5@It zWEW17=<1U`9AkJPs?RCIB=G${B=kpra0(O8Yx@Tq`QF}hy8G<2ph^2$#>9{LtKt=| z`HkRkk0pYbh^^uTU+X+MgjMm;o7Xf6cp4Xc#5!W%TR|%I((RMh$SFQA=iZn0_<1#V zyG+FkMa8~#@+A=a>iJjc_6&-@UCyfOSc*iLG716 zjE{qNHr4IE6XHvJ_ik9;4+%jj8|7zy|oxJXvMc^yF0Oy+>W%Uh5R;T$H zT(_Fyf{VrtAFyA`bq-p}hzk4I{)w%oM=%>lP7OO2)OM-cXa_A`=s(7xoAaP{r z)=#1nn)d79VQd`>Zvpx6M^ysfI3w?I%1cvg^n;HBvyJh?t4h_uNm1q+(u-T= z9k<-YE@PiyL{}>Y{qm@yS&7=!&8t2ycy@Xp0h;js!5oaO(IL(*HXK&7Hq&6e`3F2N zWGp&qtvftULQ?DSKJ;-zv+$pA>}*0sQ6Xv*4;VwtKFsk^G`ON81EyzXHv}h(r(l+0 z;b~nBr3o?wVUstN_OqE%pe3ke6q`l70*vJG?$WN8gbbc#6YVng0{Jx6&eN5M%G#ht z+?%n^W$yP3`gy}M<_oVQ)S`^KaL;-msQ@oWE8BM$+sR_mqI@(*WYzz>B409{qN)ZS*yfAT}d{7m_!>kp$;nH@mly2HQdV+B_G?Xf)z z5qJ91!#Ap5uYR`+%!P`GguU9X9gLe#r&5KXB`oIUtml)5h!p9<0;9LHq5iH*=Bp8Pm_xIZ%CfPn&m)bQyk~p4QQFG44wjY&s`FtDSbL0VZ5C@ znMF0U(mk=b;F>xla@>h~+UT4<3T-(-C$)N*YjIr`%)cU{}Qkia)={ zGmNldHjUK{Tj#(`;xQ+4ZiCFv52XC>!o$BgDkyRuGR2DKyOwpr+ z-TQ;(dti6aZ4-x1K;{7B%b(5xa7$=el1NZ`cn1VvUs`+vh6nTm23xFr$B6q!XV|9W zj*(=vdEudinnf}Bkqg>sE|~Zc)1a?VT5vk#rVT!i8qNU5d%bt$cYrWM(eeRVKV-rh z8G5xEY(aX4B>$lJP1q!30ASwVPVHbWP$n|mwh?wt7MV7&5B%mJ1A2JT7TAt~$|W=; zE*r@FPQY7}Jh4~Gb2~nzJt}g0y#c*i7x_fW4z|+DaQwR>OP>IvwP(U{-B>{Vz3d3-r8O)+% zQ+_`s4_bXj_^5HV-^@~U6zSV9%#CTL6;aEz>ROWx?liU~KK}P4)pYWZ=(!Wt(!$=L z_9w<>E0_5!_bSEHMBP`B zj$z}z;Xwnw?el3>i^GcRjCUm)&`jQwKp@~;t-y)^bbedA%9|8bt2nQoY|T*A^YRVQ zD>$T2ub^y;$jj`#jc4o4pZiwtiN^T_?yr#sCzK7JxM2yN6*|VeT_%ke-B~9&6A4rA z3FqKlrSRy4XgmJOYQ!z;(TX6vuxZT$`w6TK!i=I0kV5wCc7T^PmijKF%b;D}R}hPvdWk*0r@ zIoF06vY1gc=s7V`8Gz!rr-&jv~k7X|Mx?iqhdd? zbwpFkI5DP_Hb_dW`UyiNrJJ*-@nqMXKw6Yj7!UsY{*$5oC4}F}cd2qefF>sBv8B=` z&O;NH(o-00xzgN*^UkdK)gblc-moom(nku>4!VG!p{g5hpFJLJt{~~nO<&0XlW%Ew zB^>B{MXCzyQHeoJ=6xa#c6k4&30$Tgdv4*qm& zn!0~_D7ih&l~fb(d_j+ze;J(;!21LcJ;rr00HdaSl%(lec^p`L!nGXz5EtdpDtdDY zr;!;Fj!9&Gf6`Rhf*X;X9<3?5lPc`g*SquBRr9_RpL5x)_${ErKgQ^mLeVEW_)g8+ z#;OXQGCiieHCsc6ce18`cumO!UDU8f`d-nOIRSiX*`v`@yJ22_(>A$9->TxteYJ7` zlg?Yj2(;U*Ddv2iE{Y}<)C4^r8bEk zH3yVnVchxdP=Jg+s=#dr!ES2e^g(2=Swm=;!1QRAWqtts`mbtGO^AYUyA=sxm?7mS;MQ&eb@bx;IltLla({#Nw-In9^?~@EKrL zynDp$LFN4o*#<`YN2+gl0}Ir2EZSl-qwy=(4n8j5fPPVboaPR9E6tt(+*^G|3lq!A zal>>sRTm$mk4k8Xk!FkMEzDR@q=YZQ1i&#RBGT<*%P9!Xa6#rDWV=7|P^0Qy@i>#D z_aA#5A9x@1I>d|QRvkQs-h|$E`Kz~-*`lJ`4)8BoSV%7FBJbo|*RPGi|BQfJ&B~O& z+9P*fKazT_xCu^Ja?W@gVJotXuCfO2MAaD1zZhN$-1)J0wHsVx3_S-#2 zI87_xB2t#W(m6f-knH+DSA6CjS^r0ZG zdo=7PCe$0B)e46*xMSxKtpcibrt%UkBa}ztb|o`(vo6{70Ule zYir$P%hM1%Q-2 zLBP2lJcI0~I{T9_GQ0WVB_Jg==u&C3An|+I(N>`o*!H};Yd_H09I#1~@+UzgsfMS= zT8B?Wjb^*IDC+-tG=XOkC~s~(+A%E);Nl|F^{qj#hv~9|MiCP~w~eO{!6BX^lbP-E zL8HGux!qh1vsKXOD%psB?}QBJlXrcXV&5pQTqCL~cKyZHN1%d_F^Wm1y~#=F_R^TZ zRySG6y#>yG;TU(aA>k~kg(k%298xQ* zb)7HTTLECYK1fzNJsB?2oI=0C_3zJK*hE|!&~0^d$5+i8TS*o?O`?uYf^61EZ84`i zM+n;{;P+OKq2i4JRMlqmwuXF3k1KZK=`{0H=n@)6@uB9=z5aTGFt{+IgI)pX%~KU! zC8JcXX=J`z3f#Xq;3G+k&@;sxtAi>Js_&bV+5Op&H@sPC45@CEf$oV=)N#D$Vi++V zkkc43+2dBpL%q$L_i9ea>XV^T+(_88RxW!i7z5%J^51hu zTV=UxTo137AX`}-?r^9YmddmtM2GLZ%TsV*P60h{U9-{DJsI=RHDYscGN!WP0&`19 z(X6IJio5Ddc<$wN=@c;!4|q;|fhn5o56EFo+V!|~VV2rJ5}8cuQxs;(C{3jo+o-FOpjT6~y!O+B~p}w(Ze)G&SFj z*hKq{KRb|8>0u5*=v%4V zR=7s`i&UTi!6HAi`Bo-h-?(Xe+OdI@{*`xW<%j*lJU>T|E{v`eNugEsQ}%}`1*n?b zZUGnER7>)~c=^DlwNc%p?%pSg9OrXs)GETdhP+SSoQPUGZ8H6AxW*M&fOgctz_ls2 zKn}CM5-R^~{I+mVb$59CR6;OttJfyk{NYhK6d3l_<0X?AG|)-;{Yn=9PpItijysgP zuZj5hk!>d)6(DW=d({La*31c)usKJpqk-`zFvsGRiaCe#JPIUZq$W$GTIS3| zv#z3R}Xr#=|?UN1n1pudF zQ3453CuCNt=@yA7Od@)|h$rYQ1m-4r1@cu-+@7q8-rF*3o?2e~+N(4&28YncYz>9A z#Vq4=%teVsqA_3pMng<1fA-uy{^Vv@gXVmHsb@3mb!xdC44|iOvXm3BYu$>R3}_{o zir?P3x~-Bcd&6tVC3O(I4; z^rO52zx>I9%`fsmZ-RUK`aTh!CNZvmig}1Wb9j(=ZRHm{0?!5?|D06wWUtYW|1dkw z9Q!4Urklow5cDMLR#z;MmK+{xB125B{|KEKZH6RO|M3^a3Z;|j%dG?*nT?)26Rr^> zE^2!riA_pzZiYjkFlbIgyi&1@d-Qhr<%!t&2kl2+pI2L?p;=# znMVB~>VWgume}3s$;lfOp21Z@3oD78po#+;GMBFdV2sNqEA%s#dB zh0~PCIG`dEp(frV8eR`FJZE-1FcYLrapx zZ;JG)v(=B#b*zze5$;;0)w}aHMSY_O3YW9ANOhe!ou2E_v5WbCRp*6Ipl+$OzIWdq z5es$)o2VHKBz;w{pUJJ-PzZq~Fo*hOHYu>X1aqqy_#20_BL^CGNpN$1O_5?u)-=2Vfs# zp0{I<(vHX{>fHBZe`h|tf)L_l*8pr%a>aV>_1q0;pJ#y3@vNd{zGC5eeSibt350dI zimrgyX5B%Puq1F&Q~zs-B!nY28YUU)tUMC&imaj`5?^Eg2vo!(QPA`c#}d`Bzz z6$B?E+5&00vazsihCaZ{<3S`M5*3FUx+S2S7yq z^Thjnj4B{^5bO4rWXx^-iaV_F3m7aQ{^)y;nFU3McArQ&0ngjXveJIz0*PsKa_Ne| z0&niChyYw&6)}#@U6uRJufgj9it7K74f+!lC^;&%23W;)AMQ0jar_cXxzIi9%a*0h zjn9DF?Ldav&$$O}q+7P?UJDUuYYV%T1f^Xpa!vX(Wk?ZESHw4wR_P3qPB)#F?yQUIWwknCm!c6TGp0$N#bXL+We`GlwaAw*#w+GSzn z|0_RA;GfKoyHDq27-J@n5XjacNS9St%?P+w|W#xV0FUFgl$9knXBGZCF z(q?1(i(1Hg#;Tvj+X-{S!@_dO-lu#}js~B3co^xpi+!Yb#w3X-99|F@~-vuwO*t$xwgY=LY}hyv2 z#X11k#&hljD;6lB3FL;u!Uyi=lr^rXl~$;2leTND+lzsCC_VcwLyh*~91`eAui2*U zI&c)2wSAmy5kB(+w)`&bb@?o+p9ggzLt$B31d<~zN`;jdSbPZT$Z(x`T0RXbJ{X$h z9a`z-_VUqzb`U4a92++`aESkf_)Hgnt%ql1h^{NYn8A#~-3YXs?a5}-drnHn!T?rt z!=mmeJPY6p7h92Be~EAW*#)=bQ8GEG;pWs~A)Z(ME^-myj0Yf6e*)QyRz)X+pKn0Z zp8Z{-6YL(>ZG2!8=h(D))Sh_hIi02h&=sqwfpzTM)5{kw37cBsO|xV{Cwy=B2p;oB zkE-~bM}ii-)&Z?xbWj;Ne@6KK*OT7iZf(O+dJL}YLgkpHBora&L2PflrOeQ^TEZ`A zi-*?MxzP@(tc8Yc`ynx`!%XWkRuuTUS99s?p2smuCN%Opc9DFpOy}GWr!E}Ncq1ut zQGE#7hCCvZ@Th(sQ%F_T08?|y+?#96I1FXZN!!l-IN!U(b<(|8IkCo_@=q~Gfc_;z z|Fhq;Qo+J+u7-Md`Q}G{SyJ`Fy~VLe&`2-|?xr7ca}}X5W0VgKIcSNc9+-C1$Ly(h zxb@@u=Lgt=TxjT=&*r%{7CP8On$^~MCAY4vS#Go;jZ9b4ky)+EE=H{XU;xLWbJ91! zzFB%r&vnj|P_40#s`k(TvYMVoPYpJGVOstHT)&Y0oO1UDR;RHeTG3u29+O!u zBiyU%&B$rrlgH-{%(cAHx}gz)W?P{Ixpbud=+nt94HabDdI}bH`qurIVT24J!ZHA%PUSSTZ?%g~_KCPqr!eR6*r#D4=Ad@w#sn4occfF@*P-30 zwDBW@#Ek+Tvi0EKC=N4q13KsalY?9J^z>$?uvUkIgM*u{0)SJm{eE(T^#Vmu(lHQJ z^c>wwOA)fNH8x)B$S5Av|NGAylVT0*RDhHLD{oMSdtWFU$QS*vQGAFBW-qL zZLBk78T(+yZuq{Ybbro$-{;(?-{W`wIFH8gUar^mx|Y{o*~K4!OtfXkSRPc z*GE%ii9rBWGhmf_VR@`mMi>@aFkIZ2(r`yJ00ozxpYg?!bW?eeZ%@KH-iOTfv9JYKp=l2 zESDVBbf2}T|Z++%Rt8d8aKG) z)<)DrX}H-^n!gje>J!b7RN zEL9DRec^m<`3!9#~vWe;*QClP`NV9n9CSn{zc!i%u&8o8~l>r za~uN$A)C?;p*3UD5KIQ+Ps*A;$4S05ic|$TV`4R)L@fIV5IvOn?a`l|A*)O2zN@I+ z3Toy7GBvmRGp^?$=Nzd!nZ`1Ua~U^GTQoO)*3RBlQYzD$F86#`AWWOTgNKIQwYJWC zJl}bvsBS7xmsx!L7RmGQJO%Gf$3ng(Yba?;Vmf0OmZ1X(JqRg@y3EX^7tkany;rY+q?hfe!6Oh2 zdh5UN`BE|RfUdg|uAy;daZsC3ejVKZyPAOBwxsvQJL+_qD>F0lW*|v)YS_enX(P8@ zf;y0O3aI&3&#B}ZNr;($?STAH;U`Q$g|C9y)SdEAr~Xgc|M{;CnrHwp{I8i)|878^ zi6~2&Sz@&qQMjvKWeWzW>ldX)lg&TYL%{RKUDEjaJR0>~6Vrb{kFIHkbIFrhUjiX$RQ!!iL8Ws1WhXg#`xn#wu){ z3BopmW$#9AdDj5s)Mk|VJn3-kZ3O3egs2;n*Ue)kNW6YeF=QQSRX`28N16Lz2ONpE8 z3QN&CZR4CE_j8A1mE3IQn&Umb0yg_+q4jYvcqz8KVYZ{*)H-ORk0Z`R%2=L+4WSfZku z9%&^~Epa;0`?$*bq3LK!_bqPPEx?Mpo&=5oxW_Cp>0 z>A_#oac~)N(5I0^Ff@w?JLD-=bvhAZ02-Q-!Dl_=0|Z1nYxp|TJJf12FTrDVY{=pQ z8Z@3g72MvxxW6Uf{lkeJ0d%`#)}}}QP3nK%!^I@pyKhMLpPZ;XBn~M>`~bE;L@|v- z473mU9ncW=jI6F^%iFkb`~c_|Em8SHBwaTS^!uNjIk3Nt9^UW+@gqOrRSV3Ix%wBV z*FT@(Cu5+@&6jjP-?tV>9BcRK1m(wzY5>d;O+(;T0_lGA#>R%<%Qu%Hd!Syv!RnSa zw(*2_V0%IGr4V&z(DAsUA6o_VsN?~a@lrWNzV)`6pZVzDuJcP0MQs;DJ+x%cWuDsozfQH@8|qMbA}VUWZ*U*@$%(SE6w`QXg(bg8sy2Fe}Q1LubS-D zRNXyc&Dn(#7dSX>%rZHTeV&ee_;`QWaz5BWcX@`6oqN&IaFv%*Aq`U3a(87ozD{v8 zg3y=eHQ)doY?;Tz?%|;uPL&;pF?Khsc3%bnvicQaB4{TYAP`J1i^ef7#>aKhAW^3WVQRQRt(Yd@}o*45nsS@Qbqa2kqLn z>2)U1@CVF`_%C5(P4ng*HGY ze|JH@k9}V`{ZBJ(G|p$0JZ?-mEkpAx3__rJT%@6~h{K3^9y;wGVFflUXt{|ZkRNmo zjbGZi!H|&?Xq;9;jf}g-v2`E#-TP&_O#3U7aTmDm#nLfAVNv%~?AJ5=#V#FHO;i7J zA@^QkN&W4%{py4N>62+EU`GLE>{$kith$w2=SX2868=bONy+ECy%KyAt9gkF*W8xD zf~kGdWTve^@Bpsh)I**>;t&wjHvyW>bzG8t-W14wdNtq3a51kntER9G6XETK|D7oR z(sqSJNYaJZ-=9t@M|=#vJDHUfVChou3k$#_f$tacr$X$9gt|kfQ_mJkyiw8v9vg2> z6JCEOj`q@IB0g#bnq*uDs!p90#{<-~RRmtpu=-mP^1Z5;65>hPYj z<@H0XUUqmbh(n%s9Imip^emK9<*W76PDah*14+FXe2uRAPp38v97@Rc(RaRozofr} zv#*y5VSZDStI%}5PQI9DSdFlA3>se$gr>lyJLBKKXDV!#WN>VK1Pc)uVUTz|1P*JMF@WAHcclV6S$g@Fu=%}O93%;M{41(ZM*?o8PlHFeagQT^nZR4 z%>{_+ku5Fa=?u56STbaie1@ORgmkM~4QTUq=?LVLR-(~WrZ=)ZQa|^J!eJoQGx4gFXhU->pN(6I@y$J&DM7^vI;;!;7 zZyrEbuJ(U)<+n(zj~D*G(3NKyq_s7ltpuTDd}oairzU?wi;TT2p;1uiqR`w^(}X0S z=cQZeEY?liT{R5fSE2FOFWI#0a3?Hz8#b+fp8=GQukX+N%2qy-0=D_j^S=zme`wWT zeN>^bG&Xj|lVr&)%bFrD<^+_L)1p0P|Ciazz~`HDcOCvDn7@_S&o9C&A+?uY+Lz!T z@^xbCSQImifKbX(IllfPFZQ6OPeUp`+x-1Y(p&ojj_%?;CFU&#@f#t^B#rSvpT4%Q zG0*AKu)KeAvj6b@{&%P?&38Y)U;qT*fL+q8CHh4#VnFpUQ1&4DsG?tQo7XY#=`$cSKSfDGvkj7e0R!FEwmAX3rQLzFOKpd-WEWXb> z;r|CI<>0(xv>)95w=MiHWq=$5+s5GlK`eHoRxBm{qI@V@rE`9)K%B`;gh}J1yLPN5 zeg^TZF6b`!XyC zGE|3ciS2*>%SHHiH8{_!Hf4&g`#?iDB4QWkmEBPiusu3BrY@CR>y~DMPQG#p#EM|_ zvDs{7&;zTgU!uP&;r1d)T|h5|`B8$K4-Xf}Kr}(KVo$&?!$B_r^uh#dcvXGhfc^U& z|LUO>p7?2}3%sL%&32gxKK$Oe=FV1s|J`F7-A0wUoe1+?YJ8E-xaW#5mX?qpDPn@~ zF1D_FIp3>qzkunJkU0q$d7XK|x!Fv^FJu3lm9zt#i%wtqZJT-V_J^O5$)&qE+=nZ^ zc#VNaOYdku+3*+-C9N7vG$1X8W-?1&tTewquKxAHO8Ll2gHDMj*0^fB%-BhL$lNt$ z`_l^smKvrmN?-Q&A~51y>I68WnWs)OsKky4SfC8yG)_z3X7}E`Xt>&DNc>AQ195Xw_5Fd<=#W~9%g7k zy;7Uh=&Q~Ks3Hzg!s3MmB{mg_*!WA8K1=1A`D!ZXgcxy+NF!cok+Hxe1H;4uAF@>l zBC{KmwbJu-q5GY_SrzUjT7(~N;%>L=R0<28SC!y(;ngW)B)}K??pbRU+;`hV!hNCv zVqR07d1AbcXp5+gNK41fpe&DeYr!H1i8jqkl-@>;m>+Pdwva5Hir4CK_&6L{5 zC^+3kRXF(so$BClhn8cgWv)a)D{v8?8bcMMShVJ=ZyH=%E-JO1yAsyi0aL%iVXTrH z$6pCr;G1J2Yv(3J0hh82>ppp^A4K>FkoQ#Sr~bdv$-n#=j~<1-qwx{s5DkHfbx`7n zPD1!PQiOkR=Pj605>?$&ugE?Z*;-T1L<`4s!}DLx4(pSsz;x=RluwMeHszSed*mZ* zo)e3i@pc~&PE{opVF5FZwtX(NwRH9a{T)4>+wYdy`WNILTVITdEvUfPi01KyCCfB9 z`0bX)%uO`M%e+|Ib`X-L?XvOmoTAu#Jn10v8vX4lp)%)OMGOHx-(_4h=Mrqxm>Uc2 z((FQod#z@!M0yGvCGiqeX=RkfzvY{uU=GW?UIkln2Nw1HeJ~MkTnAtlgoqw}kI)+y zmVvwVu&bz|H>djeqD32aI7Qg^dIN9HKdMm{9`@bnzbY2(GmMs88l3zzzuez}-vqp~ zTW_X@vlDVoWiq#pHwv{KcTW)yN2b5~L6;0%jqDYA?iW*Bp8+MxT}FJV+-8(aK%lp1 z;5=~79Ao`Q{~`1DzmX^Yw*fAzuRWWMs^BC#0i}_R{z%dOjRzF zNn&pkcdD7Y)i)-Ot9NyF65FrC4RO%`Yp1J^rHwN5}+JlPe|T5KKv+Ek5FdXjsHsf1Uh%q|oBki5x7?lXr)E;6lk@V9Vb48>#>I zPVn!fu>@%(xfbZ#-h8XWJw%JHnw1=Qk4vNNh1!3_$GO;We^OR6%CGl*OblG9$kmqV z(#)61F>>A<4^MGDQ%C{t8!d-9E;t%Y8s)gf2hYbLu4-{lrljYz#M>nd(pPDxp{5o; zb^O5w%0rcKNmct4Ld?8aB$w~$;hP;oH@w_xzPGEI)#N@YBBHzN^D)7T02)yAo%C3v z*-HKG`hJCto<^e!o{jzvJv3VyIy+qO*Dic9iqWcHUzX*2@3wk^34%>m&yQIailr=C zFwydu<-W|jtGp{MX4vOFHO{577S7HeH9iGR40gE5fk~4}d(1`oZb_}pd~nJV)80;D zrtISD>w{?gG3`Z?+~Gn$^t@Z!Fq!+3W5T>lxV40;pz4=gsHj+(1h1(BNO`Cm&3`+% zVW%(ii?RT`PqaE>LMyX(0~T4__fQ@IrhOe!D}y_eD@f-ebIGwsK>O`|7J$aO!;6fI1Xd@ zbO`&F2R^t5Y(E<;fLC6Yty-HuVjbVnQb$f3&e)S7Djk# z@irTn8}25UWQd;AROQ7P-EOSzbMGxwzXw44fRa*8C%52VP*bZ{x+-6?2>sU>W*0}e zz5ULRz0Wg=ln9P4^%sD%<+0(Jmw$nd{~xY#|D%65d-xj)SS+=jgwTq(J8hKB+Z}C< zW?0u0>xxN>_NIH=VS~OA>X#TdIkq!rQEJ*JC%DU9O+Mn?$I}N4qt@@dX^|@zL+b)|A5d#3p_oU&;qoj_8zZTek zeBrNtKJ7Mke0=rl{(p+9lw>A{jGNW>BQ?K4G68H=qIT!oO&?1G1 zn~kBZy=UZtpAD5;U2~-!uyH3cFjTAZCR$>1&Pj1`a{KUex7A1-Tn)!dlpW@UUi60ZPCc0dlg|gRn4r}-1Xu4Od;x?GNS{hI#I~Jt= zmnGR8{d{)6+O5Q?)<+%G;%kmQ5Hv9i!m$p}<_Y%-&20VxS6;qiqgz-~>$CX>?wn`> z?~-V(=!mAh3`b==FP`NiYU*hd0|*vAD6uA+a=pj|7tP5vmRmGWyQTKhZTg};?-X1ko46dk2jmFRtJfJYvepgQl z&6*InrhA(sLD1HAum=Y+!vKas!``xUT+6ni_5dP@ zjJO3Nu&?X&;rykG(^u>!vJpn}E1$R~>MyT`7b?MHmK#%qt3dA^Lz4sNBiGBvX}2Xz zq{aAP1mo$)_yH1R8P%a&FOuqWw%*11~O&V<`zT8rqI++~E zcdm)^_MYCNEMQoOS#y8h{8edfmnZA}F|B6@-v{c*ssG;AC zJM>rxhKrTpLWT#MF=WcWsa7xd_J~g`x$Vj-e*UI1&#o~;Flq{vG4&5@v@5L=v?SJ_ zYd4aPec|zE$gLAM2Zc}W4J{Hy%HXh~Ydx2;gYyqgmIJc&q0i-y3OH z8R{_)<&+khJ6OD?3j`c;6(J(Ha2e%zt4~A7S7&((qw^6$&yXGtkThA5fkw(X?2vvY%>hKuwF zLcpZsX1zJ>#4~jAExt{r-VAO?ee`jr$Ztde1Gw7R8ddwv`}thyiFMb#YB1Lt1^d zg?v@D#UX>+GU#EWQO#sG_?(grqZjQ5oqRQPd<-`K9IoO((5?Il8+PLZE4za#J=h1% zCjLd^Pv?k*SP21r3db(NV^Fpq5c%K*Aol*2n+Rn;>Gi4y$A0_S4?mSB9jbsc70`OR zG=E;i6OcOVtWL9|)@Dzi32=QS`13wYv{Cy)3J-#md1G8D2p&hH27X(~!1Fax@@l?i zf0TW?x)8!vVr=q`697tyS#_QGe5Adr^T^LfzG%R%k8of}6Bs}-%l|qb`^URLgsDM8 z9r{>K@1be74cm%>kvdh9cVcp}XHdrSq!r&%B)g*j^*B8X-_KHuS_@Uij8Al&Pn&SPh!<)bQ8KjLm`jUoRNlA$$m?f08=zOTk{zFQk{SpB$PycYJ{!K@2 zwXWNNlD^FP{PF+rv48u~zneGB!-J?Z%h%bsMMZy^{JkZ!d_k(}*PqV@xj+e?T7-P- zPLkc)UvzQ*-P4Cc3=PkcdmiG{hQ7H-#&mi7c^|&%{m<0kc_wK&xgc!Kr5%Xh3lT_< z|MTobWt#G`2vx?-^@9-RA4eFHl9^d{s0At5mg6#FH|GG{&&b8a1l zL&_*e&tE%6+m3vF4Mm9+J$2Y?FwIBsv12_2%7p6Yi-6KtCc=SfLctHg&Jp|D4%ZLz zmhwA@0Zp-_FiA7*{`Ve(@KApL{{8c=klD2AzZ7cmQSHiBjaXg>#<4wX?ddh=Ky0$mTdi4C zmTk-N3RF1pb=1)Hx#i#=8x$gZc$)TMd!on*Urf)(YY0bQ%(2JAl+y=d9!mZGg7W0~ znNO)ET_4MZILH#zvz6jA)|%J8u#@(oW9T>{hz;Lz@XXWoLjAoaC~j?yvH3;ekZaST zl$RM24U0_iQOeWGKM&wEz^MM5Vj#^Vr&#pcfjIqXru0l2#tWBnWqV(S_><%kancc? zdV9<2(%24y^(EWCm0*F#^B6Vc4>NyR;qx;>zN$`IQS(g9)Y=@$^ckO3TJny-s*xmEf!jGM9 zgwe^Et3H>|)QuK88En1gzv#Q?uj#S-MQ75|vzqn`oy4556{aG_&6$`qRA&KKBF@`v z+7S;*)T=D$lbUw3KyxN~Rbhpad@Iu1RZN7Eeu;Z@S9-$R0Z-d0fphV)VD*JDx)@&v zo7NnJ$w%IN75VQS^H*Mp^fR^7KT8dPGk$g%H%U$@{ala)WCN({G3E+}76)`$@yn{I z?(o&Qp{`zJeJmSRAd(V<{e|O25rfU;-eBUrT)a*@7iGOVc;$?gp?HOZNsi*yq5l)S z>IWiGIc{$KjZ60TT8ASR?#Ankv$!F|I2{2*5Rfe0pRG!N>*D_WMKG^gwi5FY72|XW zuU^}bJaE#z?iE@0QhS%F5^?3p|5pfNQ44Bwgi4}K3%#|m$@&%TSlEz^z z&y-hIZ1zg=2Ul^IEsn9iHu2y`G`F>tg}rZbmbIS?zXM=1Ys~lelEd= zHMtnpctx-Fah2^z#ZA}w;mhULec{=!&M4%cQa&O)SnHBl_`Ke1&UK((M(f0h6PXGz z3f=E+NX-n?l?oR?1yIu0`el5_)D3R?0`QE9rU>Eq-Bl&MZG+(6d;I?UGJ~Z72+kumFs$xR`vO7|b`j@bWqJiT+}f?hh|> zt)iZF!&Ox;nILqjdZq*n&jZAZp-0E2hgt_=%d7lC6?TH}Fu56c?VXaZyFEVgVb^t% z8lB#+JF~B*O}h?0t^t$sbs^ZDr6}90*o?d52c4|Lib|YO=k(sW$J?T7wOnxn+~qbX z!6KLT;`MM6L{O!(>ij^BDuQTVWQ#5_T3Z+rm~4rQ5-*R+x=|-C=zgE=60|7E%LNb! zrBPD9uV)hrVqDC5bA7uDrRIZiT|YA~46NxF-M{TO`ZqH$Oksm6in(l1aR)TUnwpta zF5!C^Z^McBwJ%2j?D^LB?FIcPW7=0 zz;#Hj=O1ixs9I*pHO$R$0|6{5ZyO-pMBbj7BJ(W-2v7K<@3#%0SQwa89D*ilw`aM* z?N`3lu!U%UL5Vzel+hTeO0D~PHm+V~gS(ysR(Ef#iivx0p$Gu7mDvxp0fehcH1Up) zUY+)1?_^agZXE0nt#0KdOl4o5-c9@WYwFJPVmAra+2d_(<3S>I=2vRm-}k}|8Y>k+ zA_94Nr#|5l2AS+o^R4r}HV3XBp`8ANAXLA11Cg*?w~!x^dm58( zp_8lirn=A_0WjvyeE$|WxM%hzXlONwu6n0^4`zxHxz@LEh;I5a%QV`TT}#A%@I1@+ zZ{hDN9-*Kdug+Y1*R$0Cn)l1M#IqBswc#jwcR;6_b??(b!Oxm^P!-ng4(4oZgI2UK zH{9S|eG!NAjiGFDG1)EKVPF!Dfm>yQW<-MAv#yyPjEK)vh89j3mzukDwah#F8ji9K zy(?9T^7;BlCpdN}SKmd~ZQH#w3NzqqO6^WBEO)}_ssT73^jXKH&Ibm$zJ65^={Jbt zQ&to+C8h&32FlRJf(9?zQ3>UIP*cc4O}@X{opbE1-niUqVF22Fd9`hz;=6AjZ_H!| zCak<~C)o}lUHbPkcIM$pGqsoi0uNz3Nd)T^sWOM+$_d=M#19C!Sz2&Ii#`F#AlNH7 zJ@;FIr5{aO!nM|C*lYB|*~(|`s;I<*%DW@TCFCI?bhyECMQyn80x^kM*fJccpM@5T?X#pif1H=*+RR0Zl?HHyAuwGQM518 z&XvD$VfT{NEo|J41OmzR>=7CjYqPioegi++B= z?A@9P!4K1QZ=5{B!>gq^ziY<>XT5QYUhO2G>pck7jdybXTJ z57}1LY$s#==T)klvCiD4&vvHfeH+}|j6@F?%~@Ss$+#W-K133#0YKf(!5&6iT0)1a z-YRk(*SuOwmngy@(6OZ;5%!YLvI>_Q2qTOQgA)+ z+>jTWk2%`-`_pYzyTN{=iOoDa8SFe~d%W{KtV;LyUM}-^fQLCsvyZX)VUpPBd9$s+WS&al^wiCO*`k6r#a`%1^RpK$qQwnSWFAv^THxhQWSza zXY_S-c(Y-Pr7d-6bB2L@YiGLn^6x}8%a=yAQyu#9Vl;~k#iWm4x2B|^@2~YJ<=1`p zLDKsK9L$nX)TwBu(T!4;STPt-I5x}&o#E2NHlC{$njT)4+R3!SKeK@^QC~d>liw3@ zpYD91S8V-ir`)MrL(9ExzQ3v!SEOHQ$z^n9db)I;I!Am5;0WH%M?!s;1EpJQF-ksj zu+4E~Ova6Eub^WD`9G!fSV9|Q0WR|RZ?u4_-F^9?6NWWFupBPrvo|i_F`i7cM<~kgk1r3c0XD07y+fg#KUsXa6hB@ZUK3THGZs zQ#{Z@!fjD$d3r;V>iR4hRoQ7mPO6V}@UH7j_r(v|8D2u*bT*2*Q{wjYm*nFkL5#k2 zmEgdo&ByYs*T;N^#D=4;O(Qi6YESXb&Iiw3cURjruDP+RU#uT{8OpJ}hFiCF-35To zi9PzS*Y)r1j_rAM)T+43PBoX?xPNQic-WWVQewRdSg~)ek8lMFQhwdr#Iott*az4S zf&gs+J-#zAw=)3ZAbUWZXaryz47TsiWU8mQj4;8KQXZZ=#-sT*T+$sGE=iR5esDTb zR6_5{qEpIHCQ=8m);(}0uFrHGT3phL9XTTkffm^^SHYHE`RI@&t^Q@jNpWwExOlVC z^t%o2wHeF{da2w-TNJ*U{yfgu=hMr;nz2VQya`u}5)FqW3)K3rn#ls!#N_CE<#?I{ z{dPe6@xPs&`1AP#M2%8%e%yJ4S(a*&yOrV_uQpF(#X;_ z>ggl-WF$;Y>xCO1akf~F7lKo^*r3P-BP;ie9Cf(NY4Gd2onat? z4eJoNjvMN=x>rjVi%l9$*M@PE206hs&!&Y`NHup7h+hP!d}$%X7sq-a>(=sjX3;nc zxjNGdxG|t(o&&M$}6Ldc^*&~(xe1HpB+3D9&!W03{dHV-$b^wj>nW&eG>rLnj=Nc(ur4FHFHrxuWNHYf4*g4fM~~ zacu22)^a`I?+^0W-IDtNVy}1GSG#f{^)1vVXHT}J)tqUgGt0tE`W8Cz);V0|*r>25 zi+pOB#FDDCt$u1}qg2m%v>kOfW}LS<`?7%8qU6FSHq~GU414NZrVOnzJY9(KvhsA)K*WJ}eP|B3Fz(F#S}Yd{72f()S0KBB~q*`{CwI6R|R)tMBB#~n;Wgv}Wc!w% zw3=AO?;J$3`%rzCB0F2piHtaOlO0{}TlS&+R0;qNeOY?xiCsU4ZCqu_Sa6iwx8NEj z8ZkOIKW`>~)PnhR)B+4a9cN{?L&J{?H4D8BLye$2b2!1t0M&NSx}x{DSgyV0)|c8H z8{E&?^pgiMx+v(rG%kAz}^{ z_Cp*YKAT#I&Ba4PYyOf+;l;{7iV)0HPl(PECfijo&@Nx8UhsRz_eGG}Lw-hw5a;Qi zLu!5ZELS}WWx?Nvmo{eD*;0Q9%wR+Nub)XGgRyeoVVj;9(dNlV+U(T^%<1bH@fBc& z+mobJ&5=ZOfe0ZsG1E4_dula`)A(fJ9pid?)~X`Zq@D==lWKiawz5H%Qv8LkcXcXg zJ9DiY#`%plj2_AQDYKesxTH!Ng8k&qE$oXV35PlEi;B^I*m>-gn_#ba*b^x4y6iSo zy>;fcdSuN|dcB)`%K@=>FWdL(5Mgq0Xj?Hia;Cuthwv4LWqa>^IxobhOTKjy+kDYZ zr)Rl6>ChPh1Nn|Jn=;YR$FQv5+vx+$?bu2sV6_AT4}Mg#%gz$v9iXT5)+@W;+K3K6 zHsboUGhvgFv=M)Lhr|nYsV%*mx6QXZkC3!UONL8aYg$LFZ`y@Mc;g&6sv3ATk02hJ z7`P|Ud@tuo?ZGiG=!j8E9ADxj7m1mW>1!P?i?G1GjQ7J5kJBuSL_ZX|j2~LZamMg! zFY!A==ZDLOD0{EHE_?u^U6>;PjFDzHJd?r`0JuJshENA~-Hic>(sM>cz!AG0wR9IC z^{yianb%j&m~V$#R6JVqt%N!fR$MFAjlYV^W~vM$VI2 z-6SNOL849Tw*^7hkETMQ;sUzlo_2FH5#y$)MXBzBnY&I_l}8?HXImvOEbe`y-o$E- z$n-pGBh~AjIOy|IhpPI^k-U`CuL?S97HY1=1wmUnJB z=E_Nbtl!=XtYRXrB-ZXN3zuHcr&+6BdgUe^joI61-8loqgwL93=B`#2@7vgmtjzZ2 zXIX?pD=Rc^*VkC+0oX{u#2E7+OeQCA^wrF2N6y)|$-7?fea6i__TE@cr5{{pYrTJP zT4tAUq)U~y6Q6FeZa>9vQ1XwCkXMP7O_|NOghvho!<(jlNA|W&$#ARg;N-;h@O*!< zB4XDSf%j93Od1(GDj^2CroTuNMFH4qwNd@EB(H`X=;D@=t2A>#BgO|fhYa6g$BH6Hxw$E)8#$$tbJYT|SGF~>y_a$=N2LMV z9xt0tmu3_yc-5f7R%q-o?N{i)_(LoA9bc_p7b5Wc%EG2_jpB^k7pqj12No%%KwM$| zWSl#-w@LFY$v&$$ZOY$>OZ8oh&z|+LKS0pG3+tZ4i7zY)Z(q{OczV{&kZPdRBGn@0 zV9z1+K=m)x3-&MA#4>rLW}1+c0;sjf~h35M=s! zn%ZfplcN-qa4nA+FKnTy4cFt|%B2vuD&M^!x-LQ||C)IRj0k_DZcD%y;vwCYDL#{{ zHw2K+9{3!e1sg=_ECrz|=3$-X$fiFJE7x^}NU;_bHCBaD&bn!((29v+El1k&Z-n$+n+Eox@mdL)Kzt8h|?Vwh}b zOu1GXz?_CU=)y0vJFmFuwczPiDGd#S$MG|zxWyv5k8Kou04n@WmVF$_1&ZKjPJ#d7 zAKGnG5xyzNw11`TIMi*jE~hNRp#&BBjU%rOn$KQj{W_qVenQy~m--k^39cvc$kssO zY{TmCS?B9O$qQaOAmk^$xuJ((Xxq3vkV!x z1QF>QEpf<^`niSyult2h@mN<^DKJTpe5^G?86tyL{Df~v621jbg?xVcKkW5>HDoE3 zjC^#D-#rjw{KIOOo=(7l0XJB>(CZ9Ul6APSmHuEu%jVnWQGKTc7WKyb^L5CNzL;C* zZ{grM(|0vBGmr+mGML)yd6tjAsvR$r5(X=Hyg z*ph1DQiY`=Sx2nT#BEH?6XF1CM^kHd!Tw~e=8^stXHZ4gOqF|t0bs#cBU@QQ5he7W z#t_(3s1B1g;-{VGHyKB_u^2L#wY7Bu*Gznd29P*-!{;6MGEqMl=&8wc2< zn8T+Zw>rFc_?`Su>5$f&bgMVG;q=Dy5l4|VyHBT$sDDK2|EqQek))v~3vIlIeI0O- z)lqdP`N= zHC;|0x|dWw@9h~!-8tmgza?CDa3=mALpd^okmv)KZ_7aOHP6^VEGoHjpOJolf|l8(sRj>2c(v^KzyGC1xwT z+qe+W6o^A?ml*WnN|=Srn?42}$#Pv7?9SH_sGs|cZ`;IUTifD=vcL_T@a>}V7jKgy zdD0vm^`%Rfte#@uteh1|uJ|F-47Uq2`=g~}`!bzx^af~@g7!PN^r{JN4A0h$Yh*W9 zoB%ZL9pt%@c4hdwAPh7%y}7fCkyuxrI>@_snF>x-7uKUdQ2JjB{KJYsB&k2Eleza~ zD`#wyCC{SZ0v8oy+fZs(jH)%!o^zL@E}j;#)lC^n0EaQ)pbHWi6_ur$D(!)m{wBIH zmFRGbP4KO5amKJy>*aeM?~4gUh9)x04p|}gYzi1V?Ymd3KUXS<^^L5+&sBeC@KYa4 zr9on{sShx{^~4-UA>a7K6!j}Cuk6*}&RsI9E<<{DolZmLcnK`{1ZHNvkd)rJXVpLI zo@x?2ktm#_vA2U)MbxAEi`fHy2Kn=II-RN(fA}WR%O@c22dj{x-3kJy zxO{~1&r^MRB{Ltcb~{B+w~1OF>NAklT{hF~B>P6r91X9(98-lyM9c_D zi{cIuYVLYIt{=`RwmjMi*ihuFjlFZlLa)A&Z@J5zGVQDs;T@Gai)TUX5lXEDv2z|P zjVqH*8%qzIdB8DVl%hb@oh!57w1Q7wI~2e;!7Fb4vsV)RQl` zQ^GSdFMt3m3>;+Y9XxjSHT)hR`4B*y73G4Yaz?E3S(+P{p3SDH)0XWWrP3f!gOsT> zi>*ua31Po|YiZv^P8CB}<{Sham*`8=3N*5Iy5!GAe&ybzi}bzb6}q!AE?b2U9TL)F z5BmywG|T5QQRQ{*#Y3mkz|^xRJ{o1!7|J>oMR{tstVt-_C2EdDT44I=8Lhc-6c=}E zkLnKe!@7y5NsiX^?^ulEq&Iv*b;NYGH`i&g3fltG4pXq6f~HOMxoYXMA8y|DlfGYV zXm`g?T1QD6ZkmU*0qi7)Bjx#6gMS>(qSciL!+}Jt7yt@%)ZjQQ=Y&LB-6Eh7%wTwuJLNIw|M?6|`?S0y){d^b8WSqa* z>eWA}yAY!w_+T;L)0hJkE88l%>b%xl3gG6gbSmZBt15kj1KV_Wl)34RG1LKWIG=dU zw(rgBuk8XUk3aqZC7)Z(`5sdDsJR&GhNp2#Y~x}Opy~_yCec&;=AhV zk(!12S}HQc$6Y4P*N>+mc}~D^t2?NqK+qcR@K*OUEa0uwz5{bUIkP`#J;n%pvg?Hd zgofH(T5SWjVV5xejWPMI#1DYKTshR)KKMZLFd1w}#G)y@;lS;HK}BC^Y;G0E^P5e= zBrmilrY2YokWALXC0vUa&X)(2^4rP`Eut@L$er_YiUJ2~kHFaV9Ts2zM%+UGJP^zl zc%ZCtpePVsWK@%YQKVy&HUwbQwiE=G%ym&Kxv_jWB2vtG>NUVLHSlgV!At8~Ywt37wN$et`31ndoBfYQyJN8a*7-#lI} zFkD*;`al=%bv~jXBpuk_1mPAS#_nXh?g&9_FG^f8s3=HsYPm#yC@hRs!rUIQHb20* zFjPtRp4E-)nr%_KbK$)7qn+z+t`I0FFD>YogOjU~5~PO{@iT*J_&1d-{P!=xA=(bm z3Gbam6_?rGSRH6engA@DPG(D2?xi=n&8GjIzwje_xiezWRk=rFA5$kvXN1~H6k>Q= zh8&bc8hmjFYYz~@XuWh(oCgQThP|y_JPJ?M!-ia$JNL{tm&R3-ayn26iPO+z>jiNu zfSvYm&R=J`(_s?1I+aBQx>en|_hnGI)4R!r!-e_7;uo;o zDUX*sxWGXpT%zZLSB>j}k{aTrMzjNQ zYjx^|KY&_`H>)a0%#x+?K4hn>Fq(e!%9_3@(`)KI>92MoeUdh-^Z(l?Q2vpV_N zQXMy_PE|j)(m1WgH=S?x;>5)X3vPyVI+khNHmJ|dPh8aZa^Ov%I(^ML+VZ(EoA`8h zR!Hu3xL!D$&ql-3t=09VYN#DD`EX~=iLPy}kdw2wsJ5nAZ}vEYc#2OO?zR=hfqLro*?u({*neVm@}A^`Ih`4)&=5)(6n8Tk^zs9SAT?du+N zhY1apn1xi`=n30tu{nd@*t$ii!W!}9c5IZ{49T5y$@_@4a5_L3e1>kKbpW4?eR!ukLV)&2!@3O!ua&RbNRUQV@opwSl*O*5QiZbQ=-Vydt?L z2@YRdB_PlY_eWTTd#0a-RK+=Vo-Oi9Los`!oXnd$mO>6~(-BVU&vH6OVgzuHvTPsO z@63T%hK~X2WZ{LF4IqD|Dt@WwRjEawB%-tU*cZacI=-Fz!1Ve=sj3u*@{Z;*veidu zSN?`^g_+SmMzHJ&lo?Wx`k_lt*V}}UuVt;)(Oz)u`cMdyK&%@EhNAo)bd)*L>W(lj z?SXTlI;PgcxoSEiM6giE0WHa}yT+6TEq)lNlpx$}`=+liQN(Vxw9ZF-n-n)ttbr;> zdH4flid2?$U!L^BY!2F{9t_w%n?TC!d8bYSj}pzRm1o+LGP^9jIT4kSdKLrRx{}s3 z*w?x0S($*#R~>L@r@BHtCK>f+PLAi#ot&H-si{PkfUqTg{tiIK&jOkjzb0ijD6Elm zk4>$!Wor@GnwaN{@GasEpdsgO+VFPn)%lcKzJDv%H=Yz$#B+8s@9}=p*Uxy}Y2t0c z(Cc$sMOt-Ex9Eh`!GtLPTvX3iS+R--ZI@RNb~RVCF1^55%9oC_z_47w6YBeBMit3d ztyrJf4NYVw-AH z_rb8N1BSyUiFsaso81emI@B8cfU)Sa8 z>N<|+x$n>3AGQMrj+~(Rk(+kK@}-!rXooWA?S6TuQiTXXmGf+mU;x7)g3GK`WV>S2 zj2u1L77#uZFyxS6URthIJ=6vAwV^DlmxXKtNiGYnCxi4g5Gs}lhFauNPchHt`U{qR zQtIdI;- zqH-+^HT$aIRvZo*aT>9mM2A?5tw&egjcV%j8LBf|qKxdKl11L+5es-oU$!~8DNl}z z8DJ18)QOCms02yQw(Pg>t#QyhS4DOrOHz5f5IT+@(4}*iT#}v&rJ+>1YEGi07nBr? zSm>jWy#`wISjf=;g+R(k+Nmtz(J)XkZxfc3Ha+TeyH7pQso!F0>+2ay%`Y++!-)QD z0z6lja^D(6&Lb6v(BY1ts-OXvIx=L#9!y!FwKv%>#s427clQ~|X0eAUb(~uD&vTTn zKM??yBXVZo+Z52ApRAe$sttNTUyVMn1lzver!V;PEy*dUz&!I*i`dTkXr{}qbHVV- zQ)B4%ZAt{Rh`LCJR0Qn{ru!CZt!{ifu>vdvMu6-aF3wwvl92P4ZP+SaXfr*Ll}Z&o z-x}+|ZPwpy5K^j^ZRqE;@E!j_@}nCC^QCBOhrDH=?eGlmF}a3xPsWw24Uo(M%>f$4t{ zP#Q?{N#`mqX2?VvAEOFbZ6ZEj7E@iIg{Zw_Ss(4zGpE4XI2|yV^e`nzC8?|CPe;mVdIz_bpt`7}6 zrf8klw713q6@kiXK#pEkUQ30$fHCb3Ga1zl&sBAhIoyUVO>cU3I1BO=9=C8A=)GtXePn3EM$Ep_ zCQAb@j_~IQEl<%vfY|C0ddaxmzlu^PG@)=np`Zh^Yth7<@?*DiI<3$Pgs4DO;3BEm&h091Do(_!v;c1{uZe`Z zpOiKqxial#29qz8>mZyE64VV-(e3k`22K7J3Mv@-rfwZ%$4AS7998_{QzFiG!Z>E` z#Q$=*vD~hm@tnZg?k}(!_u?$A`3Tl8Z$oM^(k9AjNxBEjQnsGyzAtD!IUF*-eI*HN z+xxPRex>J`XC0-;O_>z+oMOJ^ViFZsZ7;NZIDhMm!DP#4&_Z*uS=690lrhVonRnj! z*H=2=Z1D)V%QP{y8Mh!oX~QkzmfeL4(xqCas9lA&wYkM4{t&0qV~IlM`Ls#a|FW|844@WA2GftTAy_qcJF;% zU~jU%APh`Hy1 z(u%A{pshu(nuPsI(^0%y%H0`pxupw8J{{+Al@N5Vemnm5qfC<7K4~kaui9y5nZIn! z;x~yeu+{Hwam@fqXt702>};=z1vbYkei|rRvkwpD6;PWiTcb%-X68o~MABlS?_7&d zLp23R(p)zCF`rg?^A-!OohH1fH6gYj>XfJVhkyBCFrpJN0`T1*r#L}TL)ciLl7fZk zY)QP%HVDxwYfJ8Uoo@px1Dl=ZDb-F)uGkGha+4X>RpMwT z0L+?_eZ)Dp#@t!@j!zrhg25#f9CGv*D(pWV9 zWLL5pJ^59+7nv!qU0PD9SVlOzKWWP?(zH&b7Ik_wSF))f zu$jz-+xy`SYJjTP^^z|Azs=KkcriTid94#Wt9|C5K$roe()D~s{iD{WX~Z0Q&~d&N z7!*%luJH}p#|rhKj30us->Jle$eU5ZL#0SZ0aUn^Q?AvJJ1ARfm87kO;_AT5y9q1{ zJG-0D8ZPwYTf|yVwcP=Mlkt%GQi40KJHy{!-E>-ik#%a;Yn3?vXx9X?Sf7MYHPU{U zG;85aHam-QofYk=C>Jz>m%IJTfTQ3!v|atM+x8cYE)MM3I8ck(-+Y3h`tHBNP-Gu) zz?SbQE(7m|aDjG;UsU`rbc4{LO7=S&sKzB=<*R*DO*7Lr#?pE_DfheU#E!OXAQ|;A` z7C~tPzK-=l>|a3NNJwWN*x@O>+`u(qH-BJ52HRM>kczHHZz!)Pr54Iu;(0!nmBP^_ zv#4%U23!J5prh8=T*mw)$0l22+hQe2jr#L5fFl-L{jEB=V|Ko5*Ewq3jAs|KGF_k3 zr9GX+R>X%lnbrQ*`>C0mI4~^hJm%g{PKuUTpD-bADcT-oJ@mqTKf0O%`IB(68ZZen zIN~y%|B9eM;761e?ZBE%*P4ylR%fUWo8xGXs%5fxndzo&;-k+m``l5+&bVk0m%6%r z^7PGFVi)uoBM=;j%q`Kuf20>Hv**wv^9jZDZL*~=-{#+*~^MGW8Y`l7&(^U^o?s4~_vZlN;Ho_@GO=VmYm3HPoaxTiV~>f2>o(d}T)I)AXX@ z)15cOR5EGWCDuV9)r=UR$PQ^How5Q6btO{E%i5 zDOi@fMm+JXY@Qf7_9rP7-FH{p;E%v((#xo)y})Wfm+su{$#W^|Z%N3rE?sbfOv8k? zzznG0r9nIb9Ur4k)S6F2xGkLEx9pMS727`eilfEdQr|z;CFFa@he3MJG>Da+vh8sAb0e`g*^j zSUb-wV&hvqVs(aDeR04sRLCyXY4IoJvmjB5=*@vS3i4k&6wD7Q$IfJneJ5#|CP6hv zakeKY@(K7o0Djad;;Q-nf_CK+RYl?z@wWi zA&d{0VB^LWcNGo(cE->PjO%adVLV`VygTKcVChx3_O?Y3ZespVq7m8;QWjaan%8xgo(_ffvn4QYk3tS;LMSQ8)KNgf9kz&g*U5K%`zw zqtv;;$fUn+&!_0Iye=Lk+6m_IM~K=~O}@UN*XVL`lK7Dp>Ta3#fId_d$EKZGnXMKl z_1M9Ai~wYhW~VXgNYQYV0Su}C?;sGzL7{63()W0Fibpi@jDhZhZSAt7<6zN%1q$T?&RzokXI3DR2H(xb(H_&n~c@mlEaD>Or!Q6>N~_OCq^& zy_`8z_2&Q^OX=OwHO1?Xc}~UYffV@vmKG)eEkrx&QM`Sz1oZQ7LhYjN6X8@3iL)IQ z*)_O7;qTZ*F$nOP4JrXa-`ko0>1qv?dC9EN!zgD(Yw=+!UNi1xbs@gD=U4(iDT;-i z2fbvxn1%Fe{Z7>^`Av+w(}zDD`rB_lD6FiW(r=8OzOMu0#B

A}?rRT<%5{KbZ zE(#q~*Ob_4#oIkZ%gb)?2GsDHomTTY@1HpDeo-Qo)pp1WMu{5}!0c=~kn*w)6$3LM zstCJKr>=0T)Hd@DhuI>Mzkj0V!|}Z!L{|#VUurSn0s$JIb{>sS{^S09H0D>~Bzx44 z9ud*XUs7 z+cfs%huK`7o|w2YSY?@hR#I*YhMiV?CHzBC6}^VX>4aS}KR$9IfM`bmj{f?>(o$xJ zi=6@w>MZDG$?iRV>EQ}d$WIf*?9ofGU;A>^Qb{U+H8i%xSEZN53y{0DG|%wNuE>Q< zu&az?R)1%N#^I4R-w>;=7_(kFEZal{OcDuyHM^o@@aNhotno1*-k(pL;AUKFZ*e%Q z_nwI7V;b79+!4wrP(pAqgP+1-(3%_^5z&dkCS+P#Z#m;=u+YjDhkDu1wze&L= ztJq{#@wCD+iJ*D~ z;F3mQtnq=Q;9wz0{hm!i@EY=M@O1v!vyF|_Stz7DYd*k#p!DdG0&ZuA20{PKBIE98 zbP!zq9X3F@WS>JbzV;8b>ZdjkV+kf?m${96@b=u9KGA5pCI&T%B!Y~0kag^nxA6Zh zLVH)qS>kpk{3E$ac1PwoVWRC^yrJOw2im%zNH2%u5Xj=4v{P_9hE!x2~N*1_VMOXaa5*H8mF#XY)YytQNN4vJWf4@Pe{|x?2 zHh#?NEv~HvAwG-!4mW-9JntusMszieerF}}FyX^~Z+()i0$*x&B&|`;!e_AoS{526 zMVN*~K3n=k6uEUl1g*mhcDCu@@*iXvYsRUFt}^Sxzl`L!X#p9?=BrFUar}9^ zei^6&`Loy|q5Po5<2t4EN&ieJi9BlM)x=HD1)}(1(}`TG&~DGg+KS7@E3mKG}TNhn9J=nh|h7G4y@#87#X{7 zO@l6ebPprzS)595S0qK@s!aI3sN!NJdOTP!$|+5mL7Q~GZbLua$A*y4G3bS9ik;v0 zn#5FQ6wIT(y$L-bAb;XYj)rEcTx~<$-qFFN7!l-2Me*jU+x$nSqPm#DLWk`zJX}#G&i_ILTXB#=6ruo>w%8fPY6WX$av?E7l2oL z2YyfimKIAPUM`PCgk1$$tBi}JzaSQH6cS4^`sFBO3LkV{T~|s^gyNW9Uu9LhUxZF= zqzWaaqpz+YLx(ccl#hxUc;wCO2}iPlgr!;3ukt@7EFJ0ngh_y^VhbDOd9C{9tK-d7 zXT|weE5++ehP(@`=+AG+NW#|RcXT*}F6sU?Lcz1a{Z%M|+JxrE$G!MgiZx4I_#;F%z$`x0p5|y zjbYRxor&+6C4zqzh`?F`3(<=bM%F|*>AyrAm6@(5ro#M^u#bsb{AAtyquwt!DZj&m z1Lf>y)Z^6o5TQs#1P@At+8P(YZ}_5H8@%+n2 z*%;w9Ks+9C>f&yDetsui<0!TEpKANZe5qXVm+qs*dg9t@)idy)61UP==2lM+Y?sc- zynFvh*@(d&f0w*?rfRRXH9+s1V#v;L))OX`UpZE=Xp}JghQuK)-&458&QH>m$bt0|0uTIdq|B?d;W3QQH)g*2>NAYyldcN;GAR5 z^*F*hKX1fP94^2fqaDKdaVKDjnU}iFUzkdD#FYlE5D9!=K)@ znG;+`@l%sp_LrxY&p$raA^Z_V#*5eyDn;_R0sM9lRD;>0ox*Y21}tCgwwYqGsml%N zlYaX~|N1ZvS$#xeyN{?~hu>PDh`^REktUt(3+Q()N(6S4K0EW@CSx@Qz=uUZ2va(7 zWcA&rMR6eG;S}y^A*@hd`ur#4_ML-~0g*hED5s6^;hv@OPcFfz<%c+CNyMfHQ?$L3 zn)Tum5?TDe$0!Aw`^d+wke7SuD!Uo6W*q)3F>N9Bg20)cKbD)VUHN)|@(II((0W@W z*AidAfoB@BzLx9=U9r7^RoEUFa(Z4`h8=pjqDn`&qTou!z2O_5f7z;DtcJZHH|eRU zJ9WZU`l5&t5xQ!qnRpB{r2Ar;yFqLedSAFY{o|wg3~cms=r^ z=cS(4PV-_%?_jx^#JjEW@3V*LSJhT9*V+QtJ1{#Vky%i~wL?VpHdfU81`@LVe#%5> zLKg@6VGbFFLoWlxE!AffVOC$qEjpO&NKXI5g zT@0$xW$SXXk43>zVs$2EgdK~jp|a1A!aEXNPrt=5rK76tRm;(3C5^3RhJaHO#lw~M zND6(mn{=4(Wx z^&PJW8e9L=77E1UmOT7)?D@zixWB*MbjzX6D|xXniD$?1RqeYhp~G%{%fY(i#a*KD z9ic57Jp@;WTdcatk3v>@0!}pEkpKG0J{tI_PWMw!=k^k9JnpyHBLO80lWdxtmBZew zw7uiEnF0o7zsTI}Tw%n9u{#nsEf_ezek(1lsJAd@|FQ2aL{tI>QSU}}imeGL3GYU} zXD%)-r5=}%7vEk;J=EekgfPZ9T?M~}9n>t3nq^TH;-(jUZrFDzO)y{(*Ud zsehvw6A|^kH~3-TIm#n#9|n5nkM8i)y(}QK+%V^6nZrJJlSV)7+KpJ6U4&4UiYFxzW(!E_X+O*+d`Q}H)-gg_pNW!L+Ndm{NcMd<*>@^sN%m!87-TmX+gOI-xkmKq z{(ZjR=Qw`PaXkOr|J?U&uJ?7F*Ll9q*ZDfnOB=~G!Cd@Sw}klvib;Uu_dbBlP*(mV z2p(=MQyX{Mn*S_N>0ngG zbiuk#owZdafCsp)+VK_GyJmJ7 z)v1aA@#dWg9d*o7O~d&q=X?F#Y4=v{4ip$Z7PL4p#d8!nySGg)sytowbimG+``m5p z!OyihG%BozZaS>Qi`JIHiq{a~9+KVmdhHC;SOXco%Hrwi(%`uJbGl_F_Q|S=P&(Db zGJ?jlp9V0oKCT&ElErnXhIR^ZUx&c|G zJWE2RKAY`;d$~4C%a^4acNcDQ2U;u)Sy9;qk@-{|J!5sJj4j-#vbut`llhNDDJlbt zs`RgvJ0FFH^QgwbA1eA`-q37EllhF?pYz|dIx-3s+M8vJ_WD21@@U7t;ZS;k{p^f~ zPX#mgMy<5Tm}o@a2;rgmP!+qN(qu8O^`B|#JvxUA<)*H{+0qaeU=*e?^ffjfD zP@p;FQ1UYDbr zM6I?+*IhT2oAw2#{%nySp2F2*NO8zvzUEHPX2UJ|LiMRv2 zB<0FIY!mH2NglmL!uHRmf~`4ak@0%PNLS74H-fhPyEwsT)2C2)R8%<0%Aj%6E2>5q zcf(`(-J_OCm))jMeidF96n!G5>bEnY*&Y>?Zr?iptZ&~9g9V+xa3>0Nyl0-FYFu7M zWY*h)o7_|BeEAeQ-gth{k-&L8w)@7P^ZtDFW*Ie~n^PS~POzdD4aXEJ7KgJSz8mWI z+hRHMOkfu!1|J2ZJc}4(B|oYbMrys?uCy5ZfRWFvbCx%6X@brV>f4%cl{-iGo}xz8 za0W=U3NBy%E-@>VW>+fYBqM&{5UIS6Y{Q4{Z4%mXR`mWUEyj1q#%^P+C&8i{f7?D~CxN&!xwV)tBh7Zshrf$_o79X#CJHZU2W z)TUMw0)FNbFMDF`nkWxX09j2iQtawEXQ2>=+*{;?0qzLPjWkA$R#735vQx%Nf6grM zy`LF)3fCd@C--`P*sUoRz9SthCqK)9nAH|sa}d)0#H=ebD*eSauW!6xEPu;CBmZQp zFAr_Cy}|3$cUvnBK5X&|ZP|ggvKEXW=9n)Y?SOb!C7d3A$z@2->*4ADR-^9vC2_>X zY2P21rICRdC-;@X^DzR$i`u+qITm5)0`dbhr?;aZg)~w^llj9-2#)$1^WvSfH#>46KWO@XY~N3>y|0)Nubb?h-DvpZ z{2%AO!AO(4vpgIkW%t`Tfsx?kefW5{P_SDq|Giruqd3kiswlGWK> zr8XJd!?JFZLdy!^^%1GWck`*vQG7#v6|JZ7zCOa(<=A=?t6_+a=b>p@!$DBXiIyHp zGaXu_CJadnSWf;_7*_9lWqq{&OF{Osheew5H&qpf#)H`6r#KDnd=krt;U~;K=2p-n z9Wij0&a+24)ZWIlJ)ysRZN(O9F(4)hzfl!(EdjhlZp-D#wilO@j%0>S6%`f5efFSO zP4akQ2S?W>LGn-0=~bzIyx=fFQ4>)78u{*-u2p>xOY46&<(UZxp5@O|QhZX{Btjy| zYB(4MMac;xgFk8a@CrHAHgMBhWxNX?!03u5i_qYAug>s&r=%N93BHz)jLY7UBlcTo ze1CB~#&YD-j&9k)%r|?H>3NYdc;wTZ3)eM@!W`_~4^MX+Yrjo(vaYnZ!2igH5Y#aR zX1;Vvho%ht;;9d;<%#H8LMe`{20O$^ zC}zkNrdTx)oZ(pF9;{msUp0R~zWGd4qnYx6N^&1M}1;sv|DT<~BDJSkWkVZq*dsARtL z>FvVB67}$NQM|1V-h+9^QDikkstFP&uE;oN&6$W8A6<41R%x4hHm;xbxXmJ5|3ZC5 z(2)ZWI--B_BPmK%gxT~`u3q|N%Qc%Pf0;mc1Dx6&^M|Y+zgzVt#$BAS7%LGOLj=K`o)paEeh2fC z(g9TC1+EY1BYA%Tq8pzCkt~^LPS<81S4$A6G$>YIC_ieanttkRA+*2BMIptZh2aYw zVs*Q<`tlnioVPf;asmi$abV8G^(aB;D}u!8BZWvGVQAm(a?Q9UBuevekP}w>(6n=j zDdF>}M+fZOl?Z3|*XJ;jnjyGNm7~kWa9iz5oV*4?;f2r?S%rgMyaR8YaNgJLG@w zi2nYLc=p#wi-KTuXfGI9g9M?XfN|fHHy9Hg*5Rk-KGb-L;^}>YnP4!>ggf1M3}6L-&{*jMEJxdjrIcM;la`~GUHQh?>4^b8a~!a*oT~?REPAz z$1N<=R~e@BZkXzSzC|O0IpR`U8~CqDDARXxptqn03MWsB^$&5L|FqIE_*$R{K3F!aH|PILjo8 z9oQe4f8f!d6E2i4UdlpMP-_6?a@iH(*QwTT_{mF3tX2{AWE(4;H$VoFu{7OFdM|}Eaw?Gnx zXcKuNc1aOox^=nP1|%;rd^hY4Hc-*>&~K+-h#D%A0Jl5x)c<UH>B> zZTs;$FYnY%f9c=4^7u~R2EYOMU0gUBc{QvC?5(FsvJ zQpSpsJUop|u@V%^wbxb}q{nW5&axRBmA2=noe~9}EQ&p(!Yz6a3RnctrnZUs{FwLq z8!BDehe|bb=5b6ewFOW9SC9U(R4k{KsB9=?oEdrzC!{!BY-U}NXo7qoBNoE==cY~U zeV`AAe!51xfz9Whh%CdL&{sQ@XW11YmziXYxJC#$4ewoj47+Q6gXn9dAENLL)!3pB zlO&Y?wbGBl2}Jkq{1uhwQ}4pCYcHY;e!Qn!Cw z%4SaV;=fEfMhLVrS-&PTr}6AxA^fx;y%EvC-6RQF(tM(Ngh#fsUL*JP9R!IgV!Uaq(L(F`&4PxoxU^MKJL?)Fl}!q|5kemAp*E5I=5 z=PBYpyINrIw}s|*j(2Uv7@la5wix&*JRma3x%4oK{RUTC$w&EzyDxn2Qp4+AR6nX7 znohg%LBHx_(eco5rGqO)MSn5h&&*r`YAjYZ@#7CV-oJn{4`w!b?H^B3@KNZwk_A4u0H;X{UFwmdZhus@ z7fjOb-tdyI$`=I*kY}d72k8$K2#y#RF?D-hUB^w2G!#!2IXRl$|54?cg%9(+`CjJh zkzenKwxs5Jj8Kz(4Y#LCtR4Yg1^ zz_-;TGxUPo9*^SU;<|2XZXVB;Sa$x(nN7Wr8nz=HjTo`u3DMuHloxA9Q6JQx1uO*h3yvMpW6nn14u9e(~i^96*Pm#{;?1If7M>uxe2duv! zU0-J1>HoVU^f(S4b%Du+upWH=;#m;J7kn(k%7`~9QN*(@nmJZ4e-M3%!h)$wBmmTTjjTZi#D|X2hB9^(ZsfAuam>$<`RMPb$#3(*g{+E??Lr zc+11%T})_8OA8;nEabMgs28UqJpF>bipD4C&mkkjo+4>g3zqhoFCoJ&313PrSi$AJ zlcwlcKbMc*{2(5=>E$Y?X2F883i|*SeVw%E5U^;Wrq;;vQ;l52AkRrx#XkXtjDq3J z$334EXzq$^3z8l-x=P|a769k*$YJjJ{_=JJi$+8%uQ1OpKB)CPCvy-S@jnwNg7zQ9VDhn&=P3XVl4I{;nvihT^5%kk+ZrEl8LZ3vJfUcyZ&tPos!?4#l(x}=2Q$; z#}8ww^Cy>Nm|go>O`BP7u~O+|xF0_s&F&#{!qLp?w>v4vm0)*V+`rofJy=K}^C8*! z5OjLTCA!Xrd#bX{pljF(W&fT8K$EpT!SW_S!x?6{*l{5r8*CNkOA6L#`9Qs!iSqA8 zch_jvJv#|e*9hy9BAAYKvb$#wZVZhP`$q!b$Ry?9=RE`p<~7jYF~fPp+??`y^-NsH zPyUL+8tF#>*~hl$<^f0nZr;yD%B37x6=d}%^s|G1>;FK-*5qBZu9Th+m8XER(9+UU z*ldjNjne?loWG!T`}~N|;zH%1%Yfu{SWA$LENNRaUil&EtFa;|8|W!Si&@#ruf=9h zE{92?z7ReVSqRTrtB<3WUeq2JIvVio!1|uClj3iIVHpk`#f>i86?pmbNLS`9$^_^| znXPBJCS_N}9R~Tql(n}6eEZwt;_vch$JejTdOJS{sUBASz??vRt-zD+Wz!3l(XagZf+}%)&LQe%ro+SAVCyJdfi~fS4 zgA56CFG0f0V0Se(0{Zg#V__hpK96y&z5*1 zw3n~NV?T%Zlm!sUI(mKtVD*cR12Geag5===EewOg03ZQoVvcu2=rS^-^rK6@wcg;o z>H1w}aHJor{OT@OPdPG{n%%|9qF?sN0>B_0`9w0T((NcVKw_1)`!4Cef>Qz!*M}5Y z=rvuocoMvMXVr+5S2Z*Wp?>Da%SG@1;1ZCnQveklyfZmR`u>&vLNv}jswp;r|UaS zd~U8n%1NAI>(6=&;0$j$vV9xMm$WY+@$2+?6yJ`2V|1&!E<*~C7QBI{X1to9i0UMq z=JWp;=qd?fD*^r;)F0m*Ic5F^%)J0}R?u9)WT1Y&o682vvpI0}97BOLwNQt)80)Dv zGZD{Kt}H0-Mg^``Ca#53_-Y(HUxeJ#SbV@&;2jU8;Wr~B$G(Ti z1CNteRVP7cU8>gXi}%c?#w#@FpQS|kyoGSbG1bZ;e!YxZUOwm!x6+G2fsBOs(NgXC zI(zg9CMl;(Fgmc=-b~Uu6ErGwE5h@64)QBM9}F(^dF7BcnB!3%mw&jCRPNY2+=DXOQ9*s;T1-K}4s5PqVl^ zlWMO{f`@SC{lP%448T#pFv>(T@^4&|Cq7Oxb=I=BLFEU_E`Fj>zj0&rXote~;#hc0 zOir-=?lY7Gsx4+c%X{gYL|2uYj?`3a(Ro+kg<>?d{^d6*NLSXqZ?w0on@f8u^bf1y zr}!A0FL)YfI?MYEX9s#z6DJMFl@qCG)>btE~3`^X|!)!6XLS ztLIQyTYT{p3t?u)AXlD?@COLd(_AIxpUE{2(3A$9_lwm%SrHsyb|&iSlG2M~Q75^> z2Psx<-x97qWqa4tqakA3a}&w8K413QvM<-8E6;?{23=Z$B0Lfx-CjWV{GYdXF}EJF zHGZTbw|TeuIOF5cUcJc!2Sn!o{w&D@xnAv~Tm_=zd&kt_QIunnuXsQ?%3b^6L5y2F zm3Z0B^c0%f@#s-yP1Hu6{}&QLNBK`@!^&~+&h?HEtKGg&BO@QC@b87yqA$a<0M=D@ z9?HMA1Zb34%liuvYHUpBfpDu!=#7cr&TyqXpIsdokY^|&V%z+b4&$|aZKrAn1&eAj zf!E{&-$A?-5BK3g;&~X|wKr>#A#8G!6-n)zl+-6G1i&o&Ty%9^G$8Q_CN3I6Jc@t}9N)t$DgL zBGq9?6!2`mZT7)0(=?&qi*4Z8`OLLzKJI)GUw^blt89ydp9%^rQSFR zEzh(5>vJ)CT&|%m;Bx(^@c*VEUhd!1zN5g=ZKp^DTFkSKSGl#5o#5=)a=pj<$5#vN z`RY8EJ;M3dSH{$`K5?Fp1zoMi_?rH&-WvG&2};)!8eRa^uBsy zNEBAVoQ10V6R2_^s=Pwa>O;qXbME?BkgNK5e*_wS2Hw!$${r_PiyOXkLA~ zzbf%lfu(d5x@G9O!$+4MrF$mc))7{4U4St`u2_I2R^E|5`#od#H)t=W04 zHwu(zDdv-Ezled^c_>%@q6$z|jDfMApxZZsZPMpHNEQnmTmNEQ4RY;ojD8CTp_M#> zgkw33v1``x9_rwQV*aeGcd3@m;SNyi#*jJ{qS96T>(p$G2Gh)zKF`wZQrp2 z5!si)tB|~_=UNFAg^FbbZ*n6d}5DtrM&#CoTF_gQU5Ozxw zreN7b=HuIw={O+21&Rr`!oz>bcU1u*JNVjg9RhCCm73JFw3uF-6j9n3YAN3#V4xi zR($FKT?VpfyY)|e!)XB>IjZqc&WH9n11XERJx34{VVg&pS%@xJUYakt=op@RK z4!*selWU|q0`Q%Pb($@b~htyDq1A@ooZAMK)(T=g?KY55PddEhl3VkHpnO1NI;(^cT0-pV!FyaGtvsmUsuU2Ck_<99V^vtU!Gm7u| zD$dc8%1&1Cy~+VKL2(w8m5!EglDX}{id^&%#VVUi4Z->;m}}dTtfJ8J@)XZ%&R;qd zPnb$5(dS{Jz$ITZT!n&5ZN~KnE8rHcSnO{WFxsI7tTeF81=o0$-XWDdp@yw_HK*nA zcV|o{zT&)6p;O(1m5AkUsMQhgUtuDIWh_n}+UB5#wr9D2aM{VJOu_M*iOq@Rfw$&x z{-*%vdgjOh=X!8p6@DP8F;x1wbKED#pJFcIj+Wn@hkvr^pQfa^q}^L7CnTg(3g0yW ztt=O`YcwDx#u$8anqZ(Rr zDmqZ!zs9btIlF(jIsY9Cw}KEX`^t|uFQ8@Nt3|V_!_AMr?yu}7Kk15plm2rNu^tF56m2UkU!9N;}-7W3C5s5rHc7fcI_UmqYxBhKHHXrCz} z1v=}k^d%N>^^=laNB2)U_H?3yf(-+$9?ZVwEzFH=ku_21-9;zE?8j9}1+lQaYy7bb zydEcV z&fi!1Dt+UiSNNUazrYub5nupFMeaV#5wNd>f9P}mNOseyQWtpDW zLRsATGZATywoDaicaN93{7?@4)MkaXa!qi2&h{warCZC)E-Jah-sm7Uck+-RbY7(9 zjT@)&^EsY=|N8|_05tPD$7klh(h2&H#=i?6c{Qg1ZpQ#}5+I{$>JxEI@9Lws z2DRraj^EbZ%DleoiszE&XtQS(?c`kimV*3V*^b~{^|OoBgH_fN?j)fylk`7W8U4ZXBsl-Cmp3PKL9K|cB5uH0 z330NyEyA{>MH3ky`1X}e+F+mPDOQSZWHl^gPVVOT=Mx)vgO{&E|El-DnB$_Arc;U=xSasoJ!>8ktZg!heHlC*Cps2qO~6`uTG*d54&hbYl~*3CiIzLDnj z?XEInpnPfU*l3YA5!7G-SDf?@2INp;3vgDwp`zOoqjq#!ZMeE8FKtBUqNNx*)=vm|~ z=H(JbPB8Z62X{@7|Luwms~oL+&M8D_^vhTVusk4o@S? zlfsJU&sK~rOW!<8C9v3=DPf6k++C^IwaJUo%&^AcD?*;{drkTw0I=(ZPQ2Q4Lv@N; zi-Fo8vUPe4N!sqO>0>*~tRal{iR7d?`G1=#Nkme%rVY|VRKmZ~!!?gOg~-{p>S|dg zAOZxcztFe5dd2`SF%_8Dk7dg(?`YOBG9}{h9nZTKV^;%N8Cj$G78_S}tiJs~PAo3) z*NnbhPD@)ViPC5xN4-Zg{7L%lfm+PBid7Nl`M9+CD)D4;9G)_!;h#eN|D_jOqbKE+ z0Y;um!zbL%erA?2Jrwe%et#J_5K^JKwJQB{v2q+F=)d|w91!_l^ zX;Z!PE_HgHJSTNv8k)f2dlj4z4`N~R*es`*XK3(!ApHYqq3Vy_SUETgaS)ZL1xP+Y z_`f%rRNDC;30T!ci7mFDwLSqro%7_1U7t~@!_bBPexCDeiNmG730#qFE;3@AilD3a zs;7ceL6415zrCF3{%?XywYHa~8}fZ?Y{Uj6?z8rPYqjhS3?USNwJ~FQz--5J8Cm@0 z&M6FO_o}{CC2kY|dQoN>7nv3e+~GX95rAK!A@LzOD*EQ;6O&`bzpDHet9zq3H+GKvHDICfA^V5Q4jOMrS%n>_F$EGv{w=|6dnM<* zQaEwlHT_&0hEFZTdN$yuG`GQbE@)&|b5{8KHxIwv?c7I?ku)F}{3O-Gwt(d1KfEFt zdCu?B#?XL_Kgfa*|H^_cQ`FpE|5R!(WBKLTRGN0&F{Ue-b)e6s`3*;Xn(|97iq-oZ z-6sy#`sgS)BCv}>{@F!%i2XkV3bD_e(g2O4cPl3t$G?~U;4XBSatrr(pqE4)Yg`%v zRF5xvH0(K+9A7?nXwmq69LZ)N-E%x-5BbV_k$(Q&(_OT{AXWtnJ32Ew)&8A0~L;1L7I>G>(yGBD{?B=Vt`Zs0P*_m)WJT2+9`_T1J+fHVzOID zfhRh8YdlQt#r28r?^g95({=<9e7EadHi0tMd&xsU=F!L-bm8(uOJ(a;46i|UYMlZ9 zAI3{t82m9A^GENZg*t5Q{?PZXvf3e8G{Ci~klikqR#1TM3@%kX{E2^huH6xF_jb?S zG_H5=M6*jHRmJjqc80mG!`?Dt_h4AE-Y3DU|K#973AVOZk$g?(;|0ab;(vbcAz&JR z1F?($6A)`HrlU|+;gg{;{s5ob{MZRy;m@#T#csIX)0@3Yf3Po`yNl(Uucaeol^+sJfp#* zZv5O*oiBku*fuIcGbTgbw&LVx$`>~6oc+In`jsfnQ&-p2q%F;QkA_ck)?4B*H?qL%53;MD-n^GlsET`IR=o@6k4CTl zaF*7=p&h~u8DS44E`?B&qxaw&1Ibkz{9n0hIy{^EWOS5e{fpM{560@?70j8w!4Ou& zYi5ojPyAz_&5oq?nH2|?=Qf~FdE?`iKiA2Ki-g7nq1oM5&%E8WopJEkY7tm=n-m^WIW&sx<>wUB)zk?7&YgDOnAg5%^W?yTRUXSm zj76bO`>p;Y`zqJ1ySF}j;WP8?y;`hFCJN6Qlx*FGonBJVci#!<)hqFR9kxR^J^d@y z<>^l3i|LHo*k?%qGb#uG^D>~I|ErYc=hsW3{76dYI7LvT zQ20NJ6ak)e)ti$DtKnMygw}_cYJ>I_OUs@v6djWY`@K!J*+&Uc&kn$ipJLB%-h{lD zTE7?YJ|;ewS0UAk#njVo_gSMdFlGWdrRRDW!XtjA5i5GfD>ZF3N?xfTh*1N(PjRI! z)^+o5=VHBUf*Y77#yeS9uJ_)XgH%_q$Ey{JPQ0ENyTRgtDWVoTNj%faM)K@&n{l^VZx3K^^mmpyT=IgE~Rz5@{w z*A&sk`%)y;%d8^x_J;y>qlA?eXRFcg>Lhn#w3TIK%yPG`nK)3v^P&U-3r?^lCxGcZ z71?%f^Ho$wEq`q|c7f4#Fn)qa6;&mX?M_hM$Jb>{%(NrMz2ZC;>QC+T9dkNc_UB};yz z$(~+PYaKj*kKv23vYnsrPWx_?vvZ&NRsZX)-?=B2OjtBkdvzU7%7}zEm#-VPz)a{W z1fZhF?V{cVcwKqJoW(uVXW#FluUcE9QHqmG_py&|YKmb|&$%!Pk+YsCJydAxHIj0@ ziq%`#ni(~BQOnll(q!MHy=(TN>2=(&(7cRU8fC3Gp4(=i_$BPdzqW!vq&{(dSZ0e6 zdvqRj&#cK5DQe3IDBZ7?BlWa!^$=h83bSy~tCHZOAju~I_V529r5vrM2I>sGsM3Ab zfy`eIz*SrsTEz^gYQrLfIcDaa?wnIwqDmG9^8%V3JUl0D-7a_z>~7V7<^Ea z{h(0&uqn6g6%PcRm@RP{6P&g=b7oNd7Uw{y7d&`-6`Ib)^R4BX%m?+9ASVa(d#MQ6 z8$QUcylKdkJ2kO&uJHjMcUJCWDeK~|jVi9E33>vGb)eI(B5A5sT87+nY@w*p8H9n? ze1v(o;gX7B{H=iAoA5P-V3}N}R*te8 zKAghpSBI&IJe9k2{_cPD=81}T@KC>C>gtL;=~Tc*CHtpJ%!+E#f8j$vIl!6wm<1T^ z5q3$n5Xc=fZPo>>pas(b#G(%boj&ZwVFQK`z+rC$t)80Pv ztxz17-hzCKU9l%>UMU(=xDLHt?cjM)i>eylQdy?1IlG+#&=?vZw_y3DedVZBddR*d@S@jlU(oSa&@((`_l9EM|EhoO$ zZ^d@R0|2(16Hk1V7z5MEh@E2|PV2OYrc${^t`u9|Xps?qWZ2ZZreYn5X0KrfK5^|( zO*{4B6E4-IP|nWb^)B68NyK!Q?=7Ol7%zK^h9}SYcr+%HwC-Oqm=o7+=?iQ5AbWFe z{&m%80wOOGS-WFwm}{CA;DSZv@K}q%1&69!-bF)xgfp6Z--c&@BqRD$Qctuzhc*97 zK!nZ9l(vl4tBn^24d|9Rn9cN$c>?sX;vs2=%8$@d*u4frr8%of)r72z8SvF@uF2f4 zy_Z%yBu7cmy!K=(fJc9LR`!F0-K8k=F4bXe)|SE0$!5ZFmetKl>v-Nc>Cz~GE^^ze zc9!asAy#AQTFD_(Imo(BsG09=nJm+aLg%90?g9Js61XhIk;m|%QZ4!JHs`eaxvCY& zy#6a4u`+J67nuf%>(sQhLw*n*g=`bB@IouBI4DumyZA7RKV*lx&G3Wf{N)O4B_3-v zxm9G@o!(k#LVE|5Z=g48vt`zmUgByWvpLA~<~fOC2yz>KSGQxF0>kxaWEpR{Ow64X z7+-D~AP`+61_w=Vr6daxmJo%YbEZG4L?_!Y@6znR#nF!)nV+&YZAbShG5Wx}ds*Bm8MQ>A4z}!1_+v@NK7Ok(S^oz|JL)dO zpZ*{^2FU-~zGC0AVQf=NSx1?^&0%-7=BmlTEC9E2f^2@J1~tr9<7r!MYZb(hn#S=q zYXnNI8sg@q=hNerz;d|avV@Sy{aq2RrWz%5-qvOWw=joPvFgr{$(9)Vu8&)K^%eXX z;vuAUN=95HhZeI$skYui-O|3X+3-mK|+6fAJ%GVcr!t z*;w$;YQ4X7>!@Q5F{7>}PTSAamq!OfSLbVY?Q~Ns?qJm8Ea$rsyJEk z?OV~3o3}R~U>X~2r-}&NKkJ5ByReZlaV^)-RkV3JKGB zAt$R(*TWK=RR@>v%m>=;;c&!GzYV{ummvajyb{MIm%2duGs)FTHH^XXh@A;2)8GF4&Bq$nxZ=2<1mDM_7#VoCXZv^qG?=qf31EN zr+IcBRBbvq15bT78?QnBmQPem+&gmM(E)%2+yjuaSCy0<`yYr6NY1*SHEZo+fJ&QQ z-fcOvHo2VWw*pFUFfYGQ9sG?u&H#}NJBUU6T|@0Worsg3ML2C3U4N6WX% z3e_n}hi=?*a$51pUoW}LD{MZg;8DQ#W4&ajL`S^&2d=dFwzdA7mu@fQf0f!ns6%!H z5(HSrDlgvao{yo6SWe51=^7So*Xn$V%N9O!Y@uUE!=|tJPP-Og<5+1DVs-alNL@Hk>XwMBh$TE$FBtKGQhuRVMKiC=7ax(G^jyTF08J z>FR2H&sNy&QFE=E{z8inTkM@C(8_P>D+r2(ugBgN9a4=#z*uY$E0Y3+zGJGlYrUWQ zy4x17^=P~cQtGYW%!g21yws8Bs(*LQ23eO?mZa)imu?5OS6({SU+Po!qnBdG*3tv- z?z*gO$7DpoHU zRc}}`ZGTcCc!+1hV04oSUTyuJG%&oF8cW!2kf?T)4YK;&V&G(UXiao<*UF^1)8fcg zqRAm?m$^ir>Cy92zOp8eyT59pArE#TQ{T~W?FeQu+kXPMu_*W(6ED!i{v{_Pqh)g{ zf#*0&Pn5<7Tmpi~jjLU+?Q3wn1$nfALtwhp%-j~Xamu$aNtoB!EMn3-DfN)8RhD1tft*?xUUsjvv?}JK;zSa zx{i!On0v3NqiVpUi0=R$LL;{P9|@nP6)uyU-S9?FJBdwrY!fX=iOj;TC>J>|r?prp z*e3Nx+_g_&spJm|4UGs*C^rg^IG1qL@phVy_V%4T?=`e`7D@b6m=HEAx6Zq;9g;gy z_yEuMZD_D(-dJp!N9@vz9e=;$EaKk<%J7{}zW{U+)f~~FsX9O4&`3bz5KYqsW0q6e zje(=J6sr}+M#jOa7<{&1SK-7VhpikYv6FJ^I)IwC{gpJ?&AFa|s)H^=3jbnDHMVClqs{0IcJYDF{2v+E+ zt=n0vGEF?xX$H}Gz`2wYhUpEUnrz!q?74`~4;3&e)zE#(b&F%|4%;l6rbjM`ZLA?A zQd7ugtnWvXQh;(?8DWy5y^&U^mcKXKAR-^?wnSk67M|8Jo?nIQb-M)_|3~WH`>6%p zw{Ds1_dwD(_-L#(S#?LJvzBS(<*iWln7+J0@HF5s0y|Y_*{thRf%F=%4`9Dv3LoY` z;it0@G+&||Z)v@#MBJJ$Vj~wNrP!}Pc3B>yT{^n?0zBXqA%5de-0Zw~5I5!iD{lT4 zG{(D>Tl<;Exy1z;cg<3)BXU9v=-Lk2{OAK+_xxH(O4v{jFeE>lzYM94IC0j)YxFFU zN^Q7L*;xEB-72#+#<~k$=(2vxqO@shz(Av!Wjz7wT=fkxA2HeuoyER*jO1f&rxWC# zD{{0S9YEBEamQG1ny~gw2xwOs&ZZ@tbCmtrw!kWBM0Y2L2=m!gvFXdrYERc6F+J4Y z3%nX;N?VYY4kCY^RT`ES&?MvXK#22&!fc`8>f(k)N_sYhp*|MTF`gUY zbW^^m?Y_IM*LZIwKP%8c`0CPz2z5Len;jK;>^;4B*JiljHRy(Geg`$Z5E|-zV;F;{ z6Qx{TKf#t3xP4EIAFur4`jv>~_gtw+~wP=<0} zfY`_N`%1FvGIV(0vYIo^Dh?nhw^(k(d3tkzpaO(IZio|YmsllE5IU>>t2&_BiCUo6}<1JNZ<%_HlT%_{O;`{XV(vT&@dkU8+I5y~14{j+O%n zP%XCd(s)$*)`_?`GVz;cB#NO=i?0Qw(H{1vW}iHu>ww64>JOM>%&jGH)y@-});wF> zN5mA7%!cGktTL`mQr2Z}7!%xr791Y`6T&dfxVur)S&edz${(@jT34= zT4{O^vO3Kn%$L{PeApBm3%w<+^LJkF^f#ZHsCI65G-F*}g~AY=^{PQV=!z6Lq@rfs zo}%H_4J3@gm`Awt3K;yl#+^U_T)Kp^4E!Z$QwkRu0`8&+u5@?8z$%br$@WLw44 zSo`h#TpH_|n6*=WPTX3rQgW!mpM?dCxZTd&+OmIthUPs#fGuPt7W5=#Z=vhFYV(i5 z3?IAeAFez!f?d+Th+99=)F#h??X!Oa=Cx_2X`D&(C5mzjRwI82J@;ze zhEKEoANGC%(v-Z~lN@UhEl@1WzSM)+tx#BaEd`=fa$v$K zS$^mTy@?4unLqhQh^B5EE6Coi3|t%`8N58zoL4qDUO8Yd4WvmR5jvY{B7~K2==t)P z#j3tRtMFmJla%AHXMi${jNT=Nnrwxg=os$G3C%UBGw;b<6vf0ZxboXJ3PwH$K;C_OjfgaHnB{e&n}+kBMcu;pJ{U)tqqQo{dsXP%Wh9p0 z|NMA2Wv~SLuWOSktZAGiN@)Y_jN{Y%TO99esK)yFqN$g5k z>+^Ddz^?qva#&Y4);F`q6UeZh?arzH8^uCByLKKEO7~H4J-42CXj%yF^;|#4v;oC^ z_Yc$Z=oM%(M=u?|36!+4?Bt#@=5b^pyaN=gvpqdDtDwnU>^w28wR=1Xi`9Jj)xE69 zkP{GD)2}^2(|a!spjS>+Y>Mm<#g^v_<-*_7^XM%&aNup9O??~iPeFh|77gXDkb;@D z)&TWQ-Cm_x_~(p-nbC*6O+PzzuA_IWdf;B)j(5N@)hhA?_ge4l68mR~1}Um{j1Cop zer}GU`?c6UXeuS}(yQ;p?D}5FDeiVwhfZW=@RY~d-4bOTfbnvw9dmX_|8e1lr|P6l z{w!meXKHF{lhypdcRb1^%F_iaUFH&JRA`wcD||n9d!@Y@{#bJ5hiLI4(ZC6BKw^=b zU7T-MIm8WmqURl2Os!>~4pzD0!ooroI5Z;#ogECN#0;bXBS6Zvb@lk9qP(={hU;Kw zy_chXwM|MLT=EV5TTXIoTn9j-c5bYqsDvS?I~Lok>`1x#dGNK2+d>Z-N;E|inLX|_ z-4UBvxn<UJU02lk)hy*6(Z7`If0iTsTh2 zeZ?`qk%~9HtxKET>{ULkmIhzkdfHv zfBf2m-=%SuZ1R_+@&Q(z@J01`G2y|?FjDc#;pyIH@1-rl=+r~wpg5I!ciw4P8L6sU zaN|hXis?2=UO>4{bq&(yXp(>b(p&X#YGjTO^Qi4rJ6Fx@kBCNJqGZZ{85Q-xCwG&+wl?fU4%BbK;*mR8#3IH%&Ua@n~B zI*t^pNG&fVbA)qc?dzjga};+rr{eg4LXy`-TSkkeU|aBZ&HYy1Gl>Xn+-_5g9;{cZ zt2;;CJ9aBT`X2N z=3m=(o@_fM)>XBX86Pjm`9kizfN`u9f0x=guD1&5jB);@0I%cCKrOKyA);}$QT>p% zuwmcbdZtAogY37NaIu>lkJ}q}eU14;kFg{Lj2qn98FX$VVdE48{m8L-7V`9(rS$Ff zp=g4`=fl%aYZ!NpYm7aI7d8xEV9JIr5f>J5!2Zn5GN$4YpI3VDn@$too2_XlH-Y`> zLo9DX9^6DtO)#ea^ZGRr7~Z#qz~c-ww@5{T2cQMoQ|q#I8^yevj$W1?PyPIC?K&o) z@ImK2(j!F9QL|lYbeO;QqnnYBB9=`wA%1u)0iIdS!lN@CIKGyRLwb6-|0zEc~(*^ zv@X0r{bxL5e(c5Q0P*K}KCt5%k$Id^VjiDR*?y5(>}wwxIPpGgEIXNx=Q=RB9hF1V zu(-9#>ZM5&!p=~Y87C1p$7j_N>0*Z~m3NDE0+mNW#75_6WxWafMg#$mt%#-)O;Lp- z5Pc7$#k@k!$-3W!C+jty@MN$3xOR4T`}MYU0|K8|HK6@`5EerSmp_WQyLHBGs6i{n zvLVQAVnvAY6|)#jPX9WLHD+nUX6c^C^yuC2hzQQs7_o>(RoP2o*;R;$FK-#mkMSx( zzSm?vM7vdLD42?c?<}f8#7_)iD?k`THIVa50-R@xsg202i-FJfD=pM`DUx};J|MO^ zNP>K_6pp|m*LlxqkqLu&T#&Sg`=VJJpMuM5Org^ROvJKj$cZqskhHZlmeE9#TZ}g) z=~g&x*ollK%93IpylL@2b+!zW&{|n(h%F9U?yja42b0bC<)GqMTdw;sV80mm?M$MB zP`>_S`>ERjUWcbqR)+8Lw)7@eXL*;6klo4HaY^F?WHxe|)Lh^o<^w9?AQh1!jpZVV zkNqE<=*TI;EVeT+ZT4O2k9ILt%hby&Hbqya7dsf=ru)puo$_T}A@?U=k?&8z7+1DV z38bO03LCE1sO`FtS47{#ejTFTX5YvHYsVyHwcs3)kkO z1vNq?uI~0l+f*-9BfxpzpXexbGYaf1d7%s(y7%Ma?C25`T*u8M(09+t6i0I0dnFEO_D#mKT+h5dYMdbmOrSO>&gfuaBkx z%3BkIfWL6YCvwmQ9$mKX$rsq8o+E1NR5;#})iAB!L${z?I#bkq%yp(-J~#!)#9i>x zugIfCq9;^GUS3zQE;r7uOhr4O5)sDwk(}ZJ$(SyajLE7+Jo@se2dRT$6^AG^ci$Lo z#M5bMyUihPGPQ+lnT`2X!sFbtImcoT=8Uk@i|)z&Y;V4U zcOr_DrfGBcW`nQ~0M6(KAbTd_o|~-FdppFBx!%xzg;I@1@=w`vHbL{=@Eere^!_HI z9-bC_y-t%BjT=Z1sI%exx%<%xSzBkzBg~;qH$3cg`$Z;^6W=7%^ez+u5o~lmp_1D2 zHnp&8BDQmYw)^XwEj{u%KAFdTPkTyid*7}P$Bmxx6_w`YSFCgDy)@&MVR2<(X{KEvF% zr0NT<8vg=CxXDvWl-;;QV*qzcBJ zD^!kW1-G>)+sxSGq|G>3r6uLZ(|wV2%&VEHfHJ?8^1^fI_^@x)_4}UHZVeKg}pVuag9A$w>}nomQg6n?PjE{jcUolE|7Ad=}NxaVs9# zbi?{4%sqk=&aAWtMLTk;IUsfi-L~gXs`+6Aqt2KAKeWAfSkqgxHY|b^ktPb#5fnre zq)D$*EEE+H>7pP_h;#`AL_{p~D$=D(Q>v6CDpI8PPNLLMLJfo#@~xn_ea`cobA9ji zp7+{+x{d3%)~uO(?zv~y_=~^;aObH6Pp%jvG$d}(aGpu3*vFYCGm<$&r(3eJTt`Q? z;JuV+(`_dc-n??Zntwt0;%i%^6X!nI80aAg; z(89FK{$DXkTrYHw5hL0NSBejU|T(&%iBrYHhY}t+E${$4OFYPU{VE ztGS)jflnNYx;AWfQdU+LI$#>urpRMb{xBvjj6SM0p#qbqlq_%3)zvj}QMhF|*xMOU z0pfrXSW_}MgX(wY2Kg5^_eIm#b4KqWHV>Zo4$VQDN~KXf!!Ss_yvvXso71#Kk2WBP zdzI#xF6aYLlV*qK)%94X0exj9?iWm_IAhUChzrQc8XMABsw$_`` zi6d!Hs{65>ZaM+7x7&Zv^F-5WE>9>?VDyYH?Bkaypqx$^x2?GS3+3dA=*BoXjt~|5 zouvE|*Va(#0y;|QasvcIhyd|Xho?|O>XpSwQCz)k$(4i_1q^@#yaPZJdQ?uyRz?I3hO;X?l zGc%K>UFRH!oE>5VEP%YXs%0IXVfdDvK)IxWvHNJqV z&)^e*1^cF_KX}9&Z{}iDT~12Q(uFq{t0go-RtmYBHe(Z@a{b3`)DA86QVH~I;(8E5 zDlik3%&UR{TlU8`uOZ*J-TPuH zi?1g>M{s_ZOXNS^eCo<(*A>LYk3NGD^V*yDHf6Af#ydZ70)pbEi$+OMrkl%qKsG(9 zK5U=&;{(XT=(l)wg30!;Y;k^t-u-zQVGl8gAEYtO&2P`8~+FLWoE<*KrX1 z!X|#3KgY0dC=tKxv&@=mvC-R4Dyb67f5?P_B>%1lTL=aG&) zdd*-TQwI0EnMfg_o1k}?REIN#8Q5S0CeVf(gYzPC34VZ)52TiFLvDK=FV_PwpG zPx;~90c}oYUD`p5jKzw3byvN7a3>V+6tU#Yc^PXyzm<2}V9p_7^{u31*PSc`?kV@( ze$R7LJZC=vvuQmcI7;(_U2lEJ=#tnMnaM!nb#XiV$0| zq?jGV!+l0*2pv>E{<|3&OKvr@(#RhWQr89^CfU+>(DEjeV2;$5^Qih-IpUzUEq`gd zC$4`iinb?V+1Xg}hf?v*_q7H6Wd2^m{5v($i~2VsM>+11MiA|dPLQ!a#JLt{ri%u_ z9=w?}*Z)4JYG`)t=n!m@ZRDg1&k!-~UwZ)=jbYa7vS1FBzGTy^i-r3}8%!`m{btRr zvks_Kzq+3Lq=CnK0F(DHX!yfIJ9~b71EZ^Q&aCHLgWqqr36UT4F-*Ni=~&SAmjn~kd~u_@0b;dj(yw9WACu|fOI%5io8(wCWeILNAYw$Uwf(6tP3qY zrZtD2Z9Vxqyx7tajPi7H$T#F4%DcPUs_0!60vc+I0K01r-83+&dKjP6xoE~iH`^cQ zk4K%KePhZq3$2k}g~iK55IY4S8HES2^_O60e%5mo#+g^3NO~D@7OI_m(=6m#D7UY4 zVP)s#>T;qW`NFK=Ecv12utO<6XWNlWktQ;V=g;dP5iltJ=<$w44=ml56R2fscqO=t zE{v32Bc^^Xo)k{xxqF>Q(>4S4;(p@~{~=fM%0ex@8|bmS51t^}xd;>O%ET_D!4E*K zd9NIo<(b{p@c2wg{RY(Pq^(6(f_)d*^m2K|}cV zgyv=7JsZtJ{6{nDU`N{0*Ku} zw+C`AOAV)7&lBO-ZZUwA1@Yr4~kox8hY#GUFZ!&k0q9*;=l>Epx$PZrGvncb_{ zcqj^9P^S4@49YY>SRM~X=-%)P8gXtY=*l;`pJM0%8sCc{XkXAB6+Mi>+25Dis$maA z_z_WH%*>^f?V$UlHkM#*wtvqDJ@SY7J&)%vz9dy@z^LrMDmD94MbHr65wG`j4yHn_ z_7;<%g>~PZkml6yb$s2Vp`kgN9r6Mk-s{6oLqDI$QbIr!LMJ@aY zaNLY*R3gm7b=oltv1v8%p>s&OX`*r9BY%ea%g;*T(+|(QpxL8ssr(y7{?N7|q(R_c zoZPEJL~f|l2ZElL2mfN0jK2#}o^%~~u1?9$A(A=1M)}D~f6-})zv#5S#^wgQpRot? z7@qG2u!2^F8naLM762QSJ$*ztJP5oXH$Bv2cJ-!_Y`VRJT?Wh^kQidAu)!01JLL~X zPJ-?+A^Ahjw`_O8#M-M5rj#nTWla;j?`NX^=c~Z1GQ2n#R`yTF4=B8!Ly^lD z-@d%F{N5>hiY-;|>KUQ_L}~Nwdfej*?3+LR5{w@J(*$@!V?ia<4dm>eruuxwHC2Z- z4Vo;_DD)*q!y{$qsp(*BrFtKBM^~J|@oed8PqBN3Q`xnQ+xE0C!}w(^&aepYo##7q zkXAU;V2^&2eY_N|4H)|Q(F7surs&u#9ggPZ@rKi2%p}&_p(h7XYUdi^I@l8LIuXR< z8(HPv9=_TRISI167zn~BqeATQ(t#^F1(=bGN0})c_|mj_{(Stz-r<5ixQPjDFEfx+ z^#}Y*#~=`}EOXx}U*pe>bW0n~C1LX>qJ!l1_t8bu6DG=um>}H0aM)WGWLxI<^|@mc2z7E zM8?L>j)>eOecovuOYdh*z~D=1hY&|VaiFAKm&}LS!23FPF>ELP{GwH$%AI76!y1tGCV@$nGf)IF9J%dP zFe2q!DC`y_5@NM6W46O}z;1s9$GFIF>tBagAk)NaZ{zCa>iZ{D{2rlycHPTs3#58- zT=mxicn|m=JjKWYSF|k!&6i`lGM1P;$=a9{ibr4y2JC*{{?9!VO7|#|e#mzgS_UHGaX)?0>GS4a1(OZgP(kpB>OvB%qFDeRPHqH6s zN5@rC*p#7{e*NEHRS&@7SumiQ)_~}#idua1K;&B(NK=;dgD0BD3tc*N`|C@8{2 z1)iXsx~Z?3!=~~$Em)-w!pGG`Gcz+S<2hC@`yZgk34MMi^y>r;98BY5WYKz)n)VEp z$|IS2tfAqffW}?Ppx-BS;NVF?#@J~}-1=RH!(cmioKx{}f?a#32UbU|_n8yt;is2w z$bgsYSVWFb7Q2^l@6x}L+&??8b@@!xui=UgwAwcxNrI(lSOk*8B74*Zj!E?vu9=LkZ2llgmQCoa`?7a?AU4}_pzt^x=Y!1%|~h|VVszX4=1UBjsH5sl?akyt{6cqDqUg} zwlMnQHfMCx&+I$m6x8S5R83Jj#NMy4y*#?%(ri_y3&wn56%Qf#HDX^WT!< zii^dCI;nAOr|oy9nv@U9bWEd@#qRH~j*sGc3qR+|Ad_IQ&7ea^gTGR6^|!?RLw=&N zLh*HFB)kY(t(kxXB`d4Zsx=o(+9j*$r0vV;nsk$MMcK{QMBxIuPq^4F&b!QXq?x(t z<3h^PHA9>er{fx(8@PnLv1QfgjBbkPDQyOyGcy)pHGNc$o9PxFQetJp`S=Nqt1OD@ zW$f?h6LMHuG8t}3ZPcjm(C=MT`FNCMNVdE67AW`s$y1~hh01(MPTokr-x1dSoPGZ* zdBel@zk{;fYcj^~GSXN63Vy-te(2|tcEmXzpQV@c6=;;p%J)m@?iMhq){r~OcR!oE zGzm3mG_#7Bo~l4AX+m)Wx+2x{o}DYERvoK^%wTdjQXztJ|D@@0fl$hhT3UA-Bi8*Z zR(F;6-1>UzW?XD+l|Ju5SciPX&kdOl8|v_dea}Imd7w_=sgebk^Feu;vLnH&NEVe_ zzwbY|LJIug5-qU05}HZ}36Foov#( zE#F5fSFI-+ExW4Rn{87b+JKdw{{Oww3rqcHaO$ZHwYpXps5mhaPqYm8_wXRe6@RgK z5h|m~eeTGXYM0H!{XgRUp}coH-=UOj+Dt2_lc_TFQp{a`22QI>LpGL7Ci*S>+-IGR zubws02V>|VO~~D~!bI0LyW(M|>NjKg*_h2;zIuO@Me^l<{VPwkrKU)Z`59PM^2e_I z@1163@%Fy6QB9AFiHW(V))i!ZuZ3S0*tB57%iH@K`Yi0gD=#ZOwZ#|u2B$;&wRgy) zu~6swIEr!a2hUZ}+uxt zc3-_ZTsih?F4G_{)@8M|wtT*ih}*pqr4-I;;bRagzi>wO#}mT)UWm}rl(@VhaN^1T z=EQj=2u+JpsTG1(B!|~=LlgHQdM@dyMhEb60xw+Z)4A23TP_ z_}}+t8-MUxExO_k-iR{NuI?)^sUK;m&NZpft$F`Wq`@DNQ;|~l+d_-g4!{+K?~t>; zf-5eazx&4nseCp(XxRz6m2Raha`Gg&-OkQoSmg$0 z0ug?Fr00&}Z%|S}VO(k(J4`0QNe5TF^yDOVp|_tDi!vmvjrWfLEc`no^=njei)qQA z0J}e;sS;O~Z*6To$jLBh#0Nev_A?*L?~U-!XI2pl;}wzAO3dd;G^YHVYC+5U{v-(j zk)nf2G85^nKy#agpQ0Hz3@2pQ`#$psk8h=J$m zJbpXz!92uwoW2{X3t>Wf(rcQrVS)`Dxk`iW5^pa$a_U*#h--gSVZH#fOlwZn%BFPb zeTjYjmg@EI!2T>k1p#AXgewMTpIMc)gtRA~Jo=Bne?$FD1HiJEnyLM-sd1`{>d(gk zA~~s2weOJ_Pms=QOU>mQ&HM!;^7|bXcf$k!*xn%(UeZN+SJ56XVOSpuQ^%1%%_~Bv z4^==+u+gTzo9-a9yf1i^K-W8q^7W1c^CF&ez|7M-rf1dK@o_j$<)WDC`gZrgg^tXK zBg*jCNOIo&u;>4d{|Uwh3x5Xmoy9JU`2)M&YMgjzJE%v<0$#g9=Cv%cHhXc+lM9mL)xXUR@aQpd`5H%3Va>xE9lJPzT z%gsKZSaSg%W3V6IuGdg}eOEyRCl_PF2n!33zK~GWd$p`D2<&FI1w*!P7&6qUZ?x9a z7&)E4I=|y9`3&At8oev>#|C*_4_bOTHG0C~H9nPqfvGJ9?|K@iHz0Rb=ER-Hyj>P5 zvzrU67dS`N>Atzv(p5BP=_E{s%4=SRL*3Ke2ir)f;>3SN6$AxxT2_%r((FR7O>xqE zQSf|>{#$9t)HLIjW;F&q_TzN_*seLa1t2Ehv7ZwW7Jdza#(xjsQx-{K`Q0oC%SCO^ zj{f6++xNn9JG5G^6};fhUr632*1#!3#U&~8Gvazs(lCUF->ljrR=8*f-jl1};0Rjp zqz)3Eka<_?ZqNj7n7#zS?)rZPcKGol`KFauN!KY}!p1516D8ej^GiW)VZ!zzi+=KJ z4&0LI^#@1(S%v&T%a0Y)PIXdY8B7>3VeRGk6$=Ab=BukD#Ju_8S_{+ub$&}amYioO zCM_P;t5@xZzFsyepVQzm+{5{lFa}1afA*(|;$}DUCQHh_Ab?ya@=yr-Ui(TChMMVe zu^V|TJZP7V0*z~wgEd~RExPj<^)gNw+Z5YD*LFDHB9i#=A(BvDJe^FiS{}<>0S~-< z5-+S-;R6mrqqHC)7hB1z0av>Hi5iNCyZitTFhzYB>X&`^KVk$KgB{O`0i@q{^?gN= zNrgGRqT7r48dn~>o}4rv!kU}cOlE(^yLW2O>8efTN+AjHa3u{mr(#W(z9!o~cwxL8 zcmV2sw$!EfHV`?^DbV98`I0l8LFRMx&bR%1Cfxm?rSr4zTWSt$yf{jN@f1(F{gExp zVWuhaePBc(zK3CadWW-hwspCEH;7R?wXaRxs@+eDNVpgM^-|) zhMT>nlY|_HN?(;X8&E2uExKenwwsy%a zgVb4nE?^T07∨G6_!7wcnsRvL$=4^};{*Pm5%@;C1b4&8kFch=1wBLAAG%c8Td` zMM_7YzU~6?(I8KtQQX?!rt8cg-F%y~p?~pyeVDk@NP020d~~u&*l#hc<5Xs9kRZ*g z2mk!K-kuj`kK1aQo0RJ?ynXxCat;IZUT_LKzytc@8V>um`}Gn4kHxHYvoE3wqF$&f zn^;BA{@M1YCx7Q}>*(ND$&kw%QNTYjXl3j>y$|4cw^>+xr>9!th4CY+PXH*tCQ`G+ ziL4Wh4&qK`WQh4^NsGUhWSS)Bt{%|vi5{n__k6jVjP=mcAUR0<2mnj z{GBd7QRyJ<*b@}@{1p}8>6PhmyDRGK`=-W+!4AqFf*=)*1hhATIGw+JaT&0rdH_jf zKb0QTe;|-=|ATqM8}|HX$cKy!^${MyW3W>!unviPJEhcM!&xrM?fY5AgCM7j(D}^N zfUbR^ivAl~9#mxpQI{O*{Xvin7}xH<8W;GP0*MdK;{t3__Q51S;#9M2Z`YkwCVI&c zwc~Kp1r+;Ox(nUsV#V;wPP2Ul%tfzU{w(i+{{fn3btrD$r*G0;s_+Ex6P>_)K`P#_ z-``4gk_(af@nU=tLJ=zU_a-n>za^!L9m4^NGyoB=H2haY{4mL%1UdzDwp5-UVLNjv zhj%#DrD0^mF**WsW^ z`)MVNyx663Z#9e+>qmlB9c8a z_%Qf7YM!Khub1NwJO>iKUq&oMAz0#GlHG=S9f(i69Rc7pc; zi88>Yd49`%e^f~5rJruGnP}TaNsn^lt#jNzpOvFchTLIXn&S#Qg@PF}XEU*xG3z6drDLktl1( z&WQsC>DF}SPoi=#7kFE}z}q6ju`<3FKzbedn5F)!zY^_rra#w0m-<;9lYroBoeP&S zbXzo2-1TRHj~qEZMe*mdcKs$>aQKd}WXz`T^$|XDs;%?q8utzQ2Ll=4TV&ox{|A6V zU`~E_3b%C3zmxNy^G{R(dwAil3l|Az{72I21+YY2vQU}b-HUhsr`wuB9Q5>MJ-#*J zF0a07wGX_WsK|lc>Q1{2*YnzRsU+?OS#Pc!C#B5aL4Y=ZMAsh0EpdK*aOThD2qz~& zHP;V4#zALH`@JHXSjD5)Hpa}3i{K59{ zpUql%vVT#|fs3*VE{b2)<=<@4F$v&lTRbh2ra%y75)!o#XyA1H#Oc=Yqb2ac88StM zzpMywT>N|F-nn`UL`GBP)8r%L*?I@4ws3=;%=%hj_yR?6ZA zy_oX;&W7mtJE!WUa4GIP0qR%)w4W&hyGp)vWb1$MaR1@pzK+0fOJ7F}Z||>PWf@REC~h2m zbWF_4ad}ZEimg<^+Kn0J&;Cj<58OUQ$a{z64|Zk7HoWP@Fn@yu6<*t4UW;M<6hOW0 zJXBC(Jl}6bcfAFeFs&=TvzFG?!sz&ZII6qGdk*T|A%FemO}>RN=I4m35~iD1fL-j2 zNI#(aZH7;%UwKU&16VNo{nbRlpbf2y!)s%~qBQmRmLmZOD(Z@VHqQEB?u>=dYrT(` z_WZkqkI3a4MR;yzoR4Tk%c67YUpd&9qX4_TREBz=hkCz)%0eIOWCx4ypEGaxVPM?1 zY$x9rY_%B5B1D`hpban0uU@;>GlN_`a6snK6?N5Hu2L7rH!dHbg7R4M9p1O%M6iwg z+PbV}wRh8`G5r^&FR)s6Fsv?IkpAb#8JnenbedZKi>E1hSdj@+`+=FQg^X(J$dAtB z`?jk>^1w?Z4=nb7>VZXDMzt73TVU1(`L6pVM9me{l9qf$+DxF~u9lpFwp!Tu$bNkb<2h0qu#sNT|E}V{hZ2QDRr^F>ru2VsJeTiqT0TX+ z?%==$P)3h!oWSW*Xu9s74KJh1zQ0TTKlOL1WYbbhH5}l~zhCg(0%raH-mR2mS?Z(S}Is^F9z@WM}q)$Y!w0eD2G!Aq%; z>-*h@|A}%ZU9*P&t!oCpT;L#~#%nd>o7j$_W5uY)O=DyBiNq7~IwYq;6Y8{oPbhRr zspTP3L_PL*MD@Q#ee3pZF!NsD8vN~8IrrI&u9)&N#yiy>Nj^jZL-Aape1BK*()-$9 z>E0k9_EJdc-mf^~?{o4ADjA-n24S5UEU{TXAb64uu1IrPNK%zSkBmjAmV zOqhJly|7>Y0*hHOkcN?_GY*w5PV|`+@ymG7X-(I2XnhB77+d$`aAX>lWJ#62uy_WC zf8xTdS>-SjsCazN08!eGOK(ToB@LvV?y;>6q@-oR7k{OE`P!O&yq*(6+5`IpoAS9! zkPc5o8P>;IUg`P6{?`(SQzy_85=XZ7>(nZ#6g@vHYkZ7ux(+-@aB`W52>Rq`0bA4A z>_?(vz=?cU#EZ;BpPt>%!x81oobEt1$miU8~0CDeiPU+dUIxvf%%=xRp zV{pA(YU$EPFuQ^&3X;v;*K~+YD^{oWc8N`iH6y3ryZtzc$eJP%S#69oV5@$Dv6pvDEQ`){& z@vtvnb_MWISHif5ztaeEa&?R3OQU_|FLC=UiEWN9wnZXf-ZqYJN!7-h?S*ZZr7u*M ziM!R-j*03|*Lo|{wZfuQ1J)z>LuF)c=v&N@s1Troi5T%1UR!XmgB|C-XStVuGpvXI zi)U}N0Dxt8kqashttMulm3@i9es{iFejNI^IK%c=Ev1q-yb#4rIVx$O#14wLo~Wo{ z2vU_qS=r;?*`~ZChadckN>+v_x7GXi-CYGsiH$62SE`;}Zs75JdR2f2jQ3+!djKWH zgfI^HzD-dllWgj#fd7BFRZox<`?U+ONssuN(X&NMPcjvkKBpHyUcOm58T<1PF}4|q zQ;l{v8kE7j;Jwn5i>5jH!K2ky&*ros`tP~pzHKZgY*P1tTEk!}=kfIEZ3a*a|ApVr zaFSr!$2bhOa#RbqKC;r*Ym50=$=52g=fcA2{ndJhrJY7j&2pl$+v&kO@e!%=GfP4^ zrFglH`w&rcs;&2DVYEduWNE&FrvoVI-oKS`zW2rNgF%iTBS{{RWNr4#8Tx+(xmWUk9NiTkg`m0G{L`^?6zI-O38LG5(K~Q{WcUO%zt#qav85P70RhXv7y&5AG zY6nZE8^ONcl3XA8g*>;LU;Ldkraqzu=(^tgUB$Eq56F_6N+?=TYgzHtHm|1g`>tR@&pVrT47k4k1KuUX9KoL;jb z67X@6p#QJ`84`W44`M#E9JCb6fyCdqAPdm)Byf^{2s1yL zQw|*|%0yPLxdP7&lZv*&sPgzgAv6RDx30!kHG40@PLV3+Qb!j*Tw-98dBkIfMik_P zEOzS=`7L~wW6s@KdhLekFb`E+e_Z`bwR+?#?)?0-T+FhFtA5Q z?dr$U^{K=lWz(^i%4^ys?aC=mUintLbHDWq`lW_YlT^sgw5_L z6UeB**%F8Twl-tRx)&Rm^!5p$*kT#20Hq-rg^Gw;dim>s5#;FB?cZG=jZl1S=DvV+4C3(d*x;WMvN_B8l4xK47gbXjpo1qwU*Q=Z)kE zOSR{`z?2E_@_P!Hz_Hbpc}-AuXDPtc+x=Ef%ng(E>P3X8>vUV3`&^%V%5k@7#dVAz zVQa0-Sfe<_XQk4Cm<;G`Ai28Jqv_jtlIk+x6!XfYyi-${PdOl^)qFRtBOH`dq#Qz5 zE7fvWGoL!OvC~I`h@{OTVrtszuXJ2e=^B(!AH5&09{~y6k-3)B_ zr?e|Iy@;FBUhn9~W{)B7)tzm418;MtfSm6)bzy z!oc|eDux$*@y@=h2~mm5C@{VwuBy0a1_la@QiT?5;Mun}sd-`PMQgqR+fq-nIL#sA zI1f)WP_XtAIz`D&*gS?MEezVEOuZT?wJT9r{`MF=^fUvec++b6vUW1)_-@VEcp?iw zjeW(ilAk?$b^_(i%v%MmC$odEatNJ=DDF*>!I$v5u7( zgCtV@uvIZk>mJ^5ulPyzMC#tRQVxl(D9x)l z$q?WtfyAeHjQ9H8yl?)9&ljU7f0s*My&y?1RQVe~Md5D%2|`_@6ox}<^D#Y6PB`S) z>M(k@-nqkRh8@um;@#uk(f48!G!#dyw#nDJryANUf~a8npk<@;<*vATsN8Hg=3URn z-9D2=XMRM^ml1Wzey`mbJwNBfj=0A26{dnA9D*M20a579mA&ZFz*GZO=e&QSUsdXB zI|$8K_W9l`zBtUsP=D`$Ic-#WL%D7SetdKXpvB3B>V~RNB0yyb+txeOshC#mc z=R)=3n9Kmrc$-NQZmpS`-A*eV$pb1bH;i)SIub(JUt%WXEIiTl7u+yRSu$@C>q90^ z>V6?(VE45&xPD0Sg|TllC_MVGok@mW`mMRZ<~z$*PDHMzlj-*On=8LqLL5>_-yCu8 zCxDmoe#xjB&P{0lu5MWW3u${e=m4-PU{|xqc3@W$%|YD2u6|ct0sRN;O0Pa^J_tL+ z;9YIh4J0=|WGYiQN-A>)BBeG$hc={a+rVHvwz6f1XPes#-UEbr74#X0LQuwT`sD2A z@eofpndqt;#pO9abK7y?gDR+>T$juVljEL~vi!>7r2g2wssDgV2)mS9bG9y_iLOr{ zX#RjkhC%F$PYbtgtrZq8Y*^@`ggk+jYtMuGb)>>ZoN*EignOTdA$lw!0!sEhA)Oy@ zT4sFxP*HvP^%ReLVt?Z*JEz_JfV0b5ub~DfW_ukg41U&~q0;V_CVScJgyN_RC24`>(k_L$)$`cCatKiY zdO`21&V_WY#9*c|lC61PL>kxwue&9X>kiQl6 z_#-mj=jY!-5Y|lu-*v7#J%+l!lSZEq;#SzKyvd2FMM4bPL`R-$lvt0BK&)4rPSXgQ zRv2;E^;E`5+76pt-NhFaJmbpw&3#XZ`F}Xk6QFqlh>xcSKQAx53KJS$Yn$s&r1{2! z)(`kc@RlOt!x+8?05lWopL^4!_?C_z3kXyB*3KzVwwMUSTsm=Y|FUh;1oAD0A{RR>M4k%LyY4 zJT-**ZE)dd=B=rwL*B6t)G1BrcO!i`KWX0hRj^zlsU=X}CpuQ)>f%U1Fu6Xz48q#s zxF?-~5`-~f==LeY1}=riD!31+6Kqy~^&VxcB9H#3YDWY7!xfk2QI&HOKt%q6ZW-S0gD> zEzW#es1nu)IP6vQmJ<_YZPol0sJ+jO)NFm5V^x++l68$`e)PWL)~#EboS3>Bkqk;3 zcc**565TikI+G8xUa`yv!t14l`v%s5`ALMB`~*DwmXmQxm-@9isc6k4Xw&V{ZMjgK zwa{9o*zba<#a`vo)*RFfsOZkrEsJ5Yse`9}C0P0-FxKsyDy=yMmJ@f*XjW5Wu_G>1^kY$?z1!)c)WEh%S0Ora!lEx5l7Bd z>nCq04|y>E7P_{ zB?CvLqtkt*1{JLq;dsP+Njpm9kYheK-t?!y^&EQKWZuQ^8AOqY8ReqL)56kCk;gf# z>xNppeDbQgvbl znvTQfR8ZUC$q#_5Bf)AQi{XRoTL7V0P5v)2Or@BTU@Fs5WDc)? zIpUHxo-z9QWof7If?BoP3`WyGZ1{8Tr>$9zBty7IVdtmh#tX@_L5y-ik5|TG98%sM zSu#$hquQEI_SQi85%1%54!oQ{q{RcX4OUcDE`6i0dC3bu%Ak&=BL(<(N4SizmyQxX zt0>D1_-;Fb#`gAZ>tj2|f1Q6A`@fyP`35}>kSj#QyNd00$s=$Cv;% z3A2$8H7p#@w69)oMmas~11(A67(E?B$MOkL#72?A?#5%1dc=0=&Ex)L?{(Moo`iLI zY-zgbqc4NYZ>uLY!@%P_>FVTPdfd#sv6rRq1Z$Q+zyl1!rQ51df3xM6e};`i~+?plh*O7Hyfw#~exYFF({2!6_?!^xyswYof}_@-j> z7jurdwwM@njbKcI^rm=3SJhgEQHuXfr27uB!-wsR&dFJoLO(&tdr9cHT{>f=FMKu4 zdF_Sc_fRR=R)zQ!;-k~5Yn##~%K2(YvDtX}b^`~b$zperW}18>&L z1a%ovy7i0O02(r}9K#eNr)!VCEyqd!MYHdmz8j1H6jJyvhy%=I?gvoB&ijb@lyJ~! zraNVLkGgdyIrW2xT=@0{@FBO9pI!gC7iyHG*nX7xER1Zr;79K#QSWe(geYZarr3e} zS=x99kTDaQ&!uCS1rgKg!G2~yQZs?L+8^y8=e44>@|Bi(A}5{Roo|lkuHosJegeq$ z^{l@>lQtz^DC;+$(rlAXYMpcQM8jC8k5^~Mk3H4>O?Zo4`kU~+E(0pewV;C(mCS=Q z!C*H(>gBc_Ra7vzSGuQV@i!#_++RlIaJj()8?>kC4CeI8#w{Wf5}h^gDVIJjw+a#Q zYv95+(aS#+jWc!eEie)H+ZA=0PQuGV3>@cVGA)umz(4via4V?j%KXIGJN6f$c|rOc zH8&>>#kF>BsppRmE)ti5m_qq?G;hdCZPE+4q)L-F+_#$jSix~LFk#llVoSgpE{QI4 zFr214Li>)^qbB6qIqAu>vRjz+B!+jp+>TF}VaGJiNCf^+DGFU#^r^g)yo={Fx{g3i z=7%mR%1OnB7O-D!=%dTjPteru*@Ws!Zs~~o%|PQTtdd$hhuYp$X!b0d7t!kuOVH2r zQLQ`7$+%T($P%@ZrsMQr^1|cg7|*+yL=4*3=G}-FUU6agtP^5JbI@iH?qjmeIo+%f zXW?8HEovfQleEbpyEx+76-gM3Zl5d`<}1oZ{;aZe>50(FINjg|ovNG$V|kS0hi&D? zp3hCZsAYJKiu%PF50Tgf^KI>t;e(*6v-y`$?H9k`I;o-z6nf%7sprzEFbM3%Tzg;%Pe}3D4h5&Ix;#>85}2 z24U>x4z1|U=*yeM6^U+L9dCRkY}#T$l0qWy_zcb5ork_$0oghFopJ-W_cPANKTd&R)U z(|cG21qtvu*9Bhri5rRp|BQPc)`)8!J19O z3b-5_;2#V~vZqhj5JI*4+-<pvfJ$rB_ylUOgaz(*w zZZ)vj*>xg;4_bOL4qFRhgdc?=8@Z;Q7P(N3RB!w?}Vg) z2~jtF{k|+tJRwju=MR>QLIkDkIFsSaj{L>vY{%yBn`q-Yj zUgZompjz8w$O#O}EzuwE6VKVtBUYNw*GeAf+?Sbg(B+WK8C6;xdtKC&afL^we&grD zYVr9~5ta%`3@67(0Lk^G&69ox3ZN6Fl7?yET?C7A!OKlB0?}xl(ofe5l)XI zK~jbF0gKVbrP}8PVWz;^Q$`Wy)1JrI=ZcV3;&VN@<5^pE0d%^@l9ClGTo+rF#~YBm zJbnrqd!1<@4b=I#3hfk~$jwfULGP|hcNJNnZM{y~{FZnM{olF3IJjB*7m{k(sT#X= zA`#^IA+;NA^7shXHd&ou#Ubij32p(DYpn&_nmrZh1rp1=J~qRGx^OA zXo1$?2L|4>d0X9fkK%!3ZnEo|AM5>9-EVTqgk3mZDRw%=Peablc6vjChbggPEF0FLJ?<8Euo+6y`8gB|0M$oTJeE~K@WmNss(C$u=b|^dCIjeJ{wzy%>hvyU%Q*H`gg;(A#96cxMcL7Z>LguQK+h5lzl=Jmw z>!}E_i|j3ZU#u3Zvl&FfUA@jE+;tu=qjvRbbFrLm@mIkef;CXwcOt4L(A_%gDKxmGU#Pu>mLRdEl?g{qE7reW@wJR8X_g)posbppI*V z9h!YYM&P>d=518%Q2@%wVWbk`wEM@}d%=@ZOq*k`iB02SVV)LLc}0O`MIg+(DLs23 zC~98Wb~>5Nd~SEWVtO)4Psx*RetTgUO}IOjSCmzWcb|=}mQ63O+IEW4UeE z$-qmhD9in1;QSQ!Dq7=ptjw1y+-{ES@(Tm{Q2fGa6UW}`Bv#vJYsN3Jac#Tt3)SKx zyW=9lJD)>CYZ7p6NzCf0FMYPLcqe&+iVFX(q+RhO?eK>ze(QsAleG-W?kIJ+CK?fTns4ty$4JCi`0JsZ>9!`++@D8}~EWOUzbYL08} z$+eJOpVf3(uA82T9(4dsvEoqNo&NICEFFhI7~YJt+KsuB8f#BPgZ|P9kj;Azix`*Q z0aZ9E4f4))?63(=!XHqp@)xn~U?gtLI}eZUb-%IGBroqBl0gC?yAa)iMA@|{u#d82 z21r-s>gz$i?nBb_xLonBN=t~e{4JT>t%`)4)wi>mZL47Ll)G}na-rm`3Q?H|CLv$E zsj9{09_{*=xPTom{}O>&Gl)(8bG`At}(OuHqm!|?2U!6*li|i%vnuzcUz&l*4o%u*ftZmtW{F_jAMEo zT3d39AN0i;an({oYFb6Q9uNCYU0k}wmAZCK8g<6qS6e$hDh{9tt|2YH&zbz)dQ@A| z=&kG#m*F#>?CFbDmnaul9aOY!d^d-tlU^>$(c}0>A!mFqc-8EalXeFLGY$1LE2mA7Z@mCl>N@8s)-BSBA?W1F8 zm{E^HS|7~V`+#hRkeEVNcw;EKKbk$r1r{<3@pZka9rf4k&=nxv4j^zC*No~nY@f}f z-0&rMMC$9x>Y77mq&9y9=zW`0DZ*xH$DbuG65$xcn)YI^$U9wZ{evn{`s!qmLu^hB z+p04cbTcs1EAQN$LBU^pM1diBO%6ryNnh%ZU^sY%HgTq};I%|hS{G>2v5jZ%N)7tcp$vi zife08U7VLFt4L_4YLgkannC+X^Itq|QtWY45bDmDA~|!Zv!d{A9&EeZ1=OiN(a&UQ z>pQNlG#6Eq*|a3w>J@xMlB%-Ug zavsnAl33&iYbz_^u)&C!$;m>T6j2{X{bq+2LAQ8l0NhOI6aqAH2_{=Z%b=P#*Ar3i zqnY&-Iya9@m0@Jpu7(1EeV`m?-X+(GS5}}NbgEqupSb$|$7PDsA(|FIZ*$yM>6keD zvggoLcYPVbTe{umX52n&V75LZ?^cSUEBlQ=t?)4X~CVyU;6t)gn zW!DI~kQQKvPjZ3QBv6Ydt{~v4bF^S zy@LXbFr1R3mwq_#`J?kiI;dQ*#W#u`RJC1qQDKT>5WS;5Us^;zkm(46+{muX_xnI3 zR(Hr3bXc_|*!0U*IVBil@v}TNkAB|6<8v+rc{n{XFiB48Z(O5g*%?DwEA)0uoC>k1_FWzaZaW&hBs}UxzEtyU?uOnm zg8H|rtDOGz5lS~R4^lC3pJPzOg|D9d$?aqUI!>vnP}3%`P*W1r~be0t~9FYD~rz=+d8e5Olyk2!w4~SsJ#C0TQ-1%sF}QzPtSL-hID&-v(Q1SwrVQG7(LKj!!`k7z)JIksVE+=8LkkH&TkHd2-1| z8|M_={0fi`SO{mh$LxG%+ZPmF9W#gMcp;*BSagKL7ZLihVsiO~vGv^7728+cjSsmZ z1Hsa{$uX2|z<%x^=3mHwL} zQ};e9j*lc(pAXu!J4AF@Rq(f_sp!Ou$$0TsJQ&_m4W1IK{Sq zD83z+w@`3^LBWnsDOJIv^7UjjxYo|?#+b6h4wkFSeR|SPw8iPOfg5RxvCJ%~-WfBv z)Nm%bQaJP`XjS6r{1AG5pbxkqoMzM@zoGBDwoyOdgKWTDhUrMkMO_v{`f`n*U{3Em zm_>Y&j9+`lX|N4N0F&Y|xA_@*rNU~Cx9$1Zg~0SQcgy6))(vuH>fmtGciOTX;*t`O zlAjBhXx$5Q4UhCotg{f#Qt86WbG{k$qlutc>jz<_p~;LZePhWs-zgCttOj;K&U3!~ z_L8chUPzlxE6WbH7ma%GsfO;9wDZ=O-|J;b_bcFc`&$fEC_VHz8F+1^AZ7fOm5kH3 zGv?(twLU#L+CY6K(6Fi|?kp+5>@s014>B%%F&>U?CEK)%=?-h!)6voH&C$q=!3$h< zV+BZlzdu5@{d5=hrA?T?5tU$>X*PT?tyf9?;qWyqGnN!_%ZKLKvy#gRHLR}CKn3H1 z{)$Y#>7G3iG-X_R!U*pQ9jt_q?Ay6i~tT#x8CzHY9Zh3hJ#wxTXe?tAW-v$vxM!zfYuCnT=UIYVkc5&S z2Jxp@6xY5MzI>caNRX9}c=pAluJ8>?Dut*)N9>oE<9N)j~a1|vzCN?bX9dVM%q2%r6$d^7mOnql&MUS z&PGzo!}Vw=ROd^~@}mb%&W_d|-x?Ky0;h=J53#s7dv0kUf%woWu_p~y>F9R`_R2fd zbK59x(fMQL>5m9oj6OPagLT<=1q7Mn;8V>Zj=3pNU~?!H`s95ioA4*fSR;Fed*VRc zyVfp>wmjfY;gmq%v2|pqzvG`HGhEPlk{aEiWbZ0UEBqJ{%WJ6~UepEP;t;K=jH4f_ zndR*t@87T2B}wHvZ4b{!&H8mf?wCblhF3=~0nm!BV|Jo&HU6Z5_5u*74ZhkQCD>Su z`a=B>5aW&&;%ccZl+nFicdW8=c+M#8eM!3S4!`8g_eLu?cdofDSz%#Z+o1kdmXS->HN z#6SSTT)*4r3#YBK($|?40&U?%+k|x4HjTkk?hB(Ny*tc>g(i}&N!~2?pg*A zxyG$#TRoRLlezEe3lA$*c3;$uD*-OL{^k2r&AD-!Ho3Q#th%}~2uco1N>w0>hc=BipA8u@S09c>28<3gFdK!3d` zA>!vy9p0{b#6u#PU4)IH`=$iv%SxCoZL%PdTny_ zy5V-b Date: Fri, 12 Apr 2024 23:13:07 +0200 Subject: [PATCH 71/76] Update docs/source/examples/wandb_logging.rst Co-authored-by: ebsmothers --- docs/source/examples/wandb_logging.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/examples/wandb_logging.rst b/docs/source/examples/wandb_logging.rst index 3f9c40a9c8..5c07bd15da 100644 --- a/docs/source/examples/wandb_logging.rst +++ b/docs/source/examples/wandb_logging.rst @@ -57,7 +57,7 @@ A suggested approach would be something like this: ... ## Let's save the checkpoint to W&B ## depending on the Checkpointer Class the file will be named differently - ## Here it is an example for the full_finetune case + ## Here is an example for the full_finetune case checkpoint_file = Path.joinpath( self._checkpointer._output_dir, f"torchtune_model_{epoch}" ).with_suffix(".pt") From f3fe9e57b2b0be7a4a9f1ca0993abc5f548fd321 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 23:13:49 +0200 Subject: [PATCH 72/76] Update torchtune/utils/metric_logging.py Co-authored-by: Rafi Ayub <33648637+RdoubleA@users.noreply.github.com> --- torchtune/utils/metric_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 9fd487a148..b51c5ab26d 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -180,7 +180,7 @@ def __init__( self.log_strategy = log_strategy self.world_size, self.rank = get_world_size_and_rank() self.local_rank = _get_local_rank() - self.local_rank = 0 if self.local_rank is None else self.local_rank + self.local_rank = self.local_rank or 0 if ( (self.log_strategy == "main" and self.rank == 0) From fffba109e38d2587ffc966d491e201bc47472b02 Mon Sep 17 00:00:00 2001 From: Thomas Capelle Date: Fri, 12 Apr 2024 23:21:33 +0200 Subject: [PATCH 73/76] remove multi-node logi --- torchtune/utils/metric_logging.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 9fd487a148..a05320c3ea 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -135,11 +135,6 @@ class WandBLogger(MetricLoggerInterface): project (str): WandB project name entity (Optional[str]): WandB entity name group (Optional[str]): WandB group name - log_strategy (Optional[str]): Strategy to use for logging. Options are "main", "node", "all". In case of "main" - only the main process will log to W&B. In case of "node" only the node's main process will log to W&B. In - case of "all" all processes will log to W&B. If you only have one node, "node" and "all" will have the same - effect. - Default: "main" **kwargs: additional arguments to pass to wandb.init Example: @@ -164,7 +159,6 @@ def __init__( project: str = "torchtune", entity: Optional[str] = None, group: Optional[str] = None, - log_strategy: Literal["main", "node", "all"] = "main", **kwargs, ): try: @@ -176,17 +170,9 @@ def __init__( ) from e self._wandb = wandb - # logging strategy options are "main", "node", "all" - self.log_strategy = log_strategy - self.world_size, self.rank = get_world_size_and_rank() - self.local_rank = _get_local_rank() - self.local_rank = 0 if self.local_rank is None else self.local_rank + _, self.rank = get_world_size_and_rank() - if ( - (self.log_strategy == "main" and self.rank == 0) - or (self.log_strategy == "node" and self.local_rank == 0) - or self.log_strategy == "all" - ): + if self.rank == 0: self._wandb.init( project=project, entity=entity, @@ -196,7 +182,7 @@ def __init__( **kwargs, ) - + def log_config(self, config: DictConfig) -> None: """Saves the config locally and also logs the config to W&B. The config is stored in the same directory as the checkpoint. You can From 1256087a12fa087417b75ec89aed4c7149134185 Mon Sep 17 00:00:00 2001 From: RdoubleA Date: Sun, 14 Apr 2024 19:07:23 -0700 Subject: [PATCH 74/76] run linter, only log memory stats with cuda --- docs/source/examples/wandb_logging.rst | 4 +-- recipes/full_finetune_distributed.py | 7 ++-- recipes/full_finetune_single_device.py | 14 +++++--- recipes/gemma_full_finetune_distributed.py | 7 ++-- recipes/lora_dpo_single_device.py | 16 ++++++--- recipes/lora_finetune_distributed.py | 9 +++-- recipes/lora_finetune_single_device.py | 12 ++++--- torchtune/utils/memory.py | 12 +++++-- torchtune/utils/metric_logging.py | 42 ++++++++++++---------- 9 files changed, 80 insertions(+), 43 deletions(-) diff --git a/docs/source/examples/wandb_logging.rst b/docs/source/examples/wandb_logging.rst index 5c07bd15da..73ccba56e6 100644 --- a/docs/source/examples/wandb_logging.rst +++ b/docs/source/examples/wandb_logging.rst @@ -47,7 +47,7 @@ We automatically grab the config from the recipe you are running and log it to W Logging Model Checkpoints to W&B ------------------------------- -You can also log the model checkpoints to W&B by modifying the desired script `save_checkpoint` method. +You can also log the model checkpoints to W&B by modifying the desired script `save_checkpoint` method. A suggested approach would be something like this: @@ -75,4 +75,4 @@ A suggested approach would be something like this: } ) wandb_at.add_file(checkpoint_file) - wandb.log_artifact(wandb_at) + wandb.log_artifact(wandb_at) diff --git a/recipes/full_finetune_distributed.py b/recipes/full_finetune_distributed.py index 0576c0f6fb..7624de609c 100644 --- a/recipes/full_finetune_distributed.py +++ b/recipes/full_finetune_distributed.py @@ -270,7 +270,7 @@ def _setup_model( utils.set_activation_checkpointing( model, auto_wrap_policy={modules.TransformerDecoderLayer} ) - if self._is_rank_zero: + if self._is_rank_zero and self._device == torch.device("cuda"): memory_stats = utils.memory_stats_log(device=self._device) log.info(f"Memory Stats after model init:\n{memory_stats}") @@ -451,10 +451,13 @@ def train(self) -> None: if ( self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero + and self._device == torch.device("cuda") ): # Log peak memory for iteration memory_stats = utils.memory_stats_log(device=self._device) - self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) + self._metric_logger.log_dict( + memory_stats, step=self.total_training_steps + ) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/recipes/full_finetune_single_device.py b/recipes/full_finetune_single_device.py index 2b84d09829..ed530694e5 100644 --- a/recipes/full_finetune_single_device.py +++ b/recipes/full_finetune_single_device.py @@ -234,8 +234,9 @@ def _setup_model( if compile_model: log.info("Compiling model with torch.compile...") model = utils.wrap_compile(model) - memory_stats = utils.memory_stats_log(device=self._device) - log.info(f"Memory Stats after model init:\n{memory_stats}") + if self._device == torch.device("cuda"): + memory_stats = utils.memory_stats_log(device=self._device) + log.info(f"Memory Stats after model init:\n{memory_stats}") return model def _setup_optimizer( @@ -414,9 +415,14 @@ def train(self) -> None: self.total_training_steps += 1 # Log peak memory for iteration - if self.total_training_steps % self._log_peak_memory_every_n_steps == 0: + if ( + self.total_training_steps % self._log_peak_memory_every_n_steps == 0 + and self._device == torch.device("cuda") + ): memory_stats = utils.memory_stats_log(device=self._device) - self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) + self._metric_logger.log_dict( + memory_stats, step=self.total_training_steps + ) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/recipes/gemma_full_finetune_distributed.py b/recipes/gemma_full_finetune_distributed.py index ac09fd2bc8..e73369d896 100644 --- a/recipes/gemma_full_finetune_distributed.py +++ b/recipes/gemma_full_finetune_distributed.py @@ -266,7 +266,7 @@ def _setup_model( utils.set_activation_checkpointing( model, auto_wrap_policy={modules.TransformerDecoderLayer} ) - if self._is_rank_zero: + if self._is_rank_zero and self._device == torch.device("cuda"): memory_stats = utils.memory_stats_log(device=self._device) log.info(f"Memory Stats after model init:\n{memory_stats}") # synchronize before training begins @@ -457,10 +457,13 @@ def train(self) -> None: if ( self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero + and self._device == torch.device("cuda") ): # Log peak memory for iteration memory_stats = utils.memory_stats_log(device=self._device) - self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) + self._metric_logger.log_dict( + memory_stats, step=self.total_training_steps + ) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/recipes/lora_dpo_single_device.py b/recipes/lora_dpo_single_device.py index cc68073fdf..d89fabb1ce 100644 --- a/recipes/lora_dpo_single_device.py +++ b/recipes/lora_dpo_single_device.py @@ -150,7 +150,7 @@ def setup(self, cfg: DictConfig) -> None: # log config with parameter override self._metric_logger.log_config(cfg) - + checkpoint_dict = self.load_checkpoint(cfg_checkpointer=cfg.checkpointer) self._model = self._setup_model( @@ -255,8 +255,9 @@ def _setup_model( ) log.info(f"Model is initialized with precision {self._dtype}.") - memory_stats = utils.memory_stats_log(device=self._device) - log.info(f"Memory Stats after model init:\n{memory_stats}") + if self._device == torch.device("cuda"): + memory_stats = utils.memory_stats_log(device=self._device) + log.info(f"Memory Stats after model init:\n{memory_stats}") return model def _setup_optimizer( @@ -490,10 +491,15 @@ def train(self) -> None: # Update the number of steps when the weights are updated self.total_training_steps += 1 # Log peak memory for iteration - if self.total_training_steps % self._log_peak_memory_every_n_steps == 0: + if ( + self.total_training_steps % self._log_peak_memory_every_n_steps == 0 + and self._device == torch.device("cuda") + ): # Log peak memory for iteration memory_stats = utils.memory_stats_log(device=self._device) - self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) + self._metric_logger.log_dict( + memory_stats, step=self.total_training_steps + ) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/recipes/lora_finetune_distributed.py b/recipes/lora_finetune_distributed.py index 810c627f56..110d02ac42 100644 --- a/recipes/lora_finetune_distributed.py +++ b/recipes/lora_finetune_distributed.py @@ -170,7 +170,7 @@ def setup(self, cfg: DictConfig) -> None: # log config with parameter override self._metric_logger.log_config(cfg) - + checkpoint_dict = self.load_checkpoint(cfg_checkpointer=cfg.checkpointer) self._model = self._setup_model( @@ -326,7 +326,7 @@ def _setup_model( utils.set_activation_checkpointing( model, auto_wrap_policy={modules.TransformerDecoderLayer} ) - if self._is_rank_zero: + if self._is_rank_zero and self._device == torch.device("cuda"): memory_stats = utils.memory_stats_log(device=self._device) log.info(f"Memory Stats after model init:\n{memory_stats}") @@ -541,10 +541,13 @@ def train(self) -> None: if ( self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero + and self._device == torch.device("cuda") ): # Log peak memory for iteration memory_stats = utils.memory_stats_log(device=self._device) - self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) + self._metric_logger.log_dict( + memory_stats, step=self.total_training_steps + ) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/recipes/lora_finetune_single_device.py b/recipes/lora_finetune_single_device.py index cfee7cc5bd..cb41293d5b 100644 --- a/recipes/lora_finetune_single_device.py +++ b/recipes/lora_finetune_single_device.py @@ -146,7 +146,7 @@ def setup(self, cfg: DictConfig) -> None: model, tokenizer, loss, optimizer, learning rate scheduler, sampler, and dataloader. """ self._metric_logger = config.instantiate(cfg.metric_logger) - + # log config with parameter override self._metric_logger.log_config(cfg) @@ -267,8 +267,9 @@ def _setup_model( if compile_model: log.info("Compiling model with torch.compile...") model = utils.wrap_compile(model) - memory_stats = utils.memory_stats_log(device=self._device) - log.info(f"Memory Stats after model init:\n{memory_stats}") + if self._device == torch.device("cuda"): + memory_stats = utils.memory_stats_log(device=self._device) + log.info(f"Memory Stats after model init:\n{memory_stats}") return model def _setup_optimizer( @@ -447,10 +448,13 @@ def train(self) -> None: if ( self.total_training_steps % self._log_peak_memory_every_n_steps == 0 + and self._device == torch.device("cuda") ): # Log peak memory for iteration memory_stats = utils.memory_stats_log(device=self._device) - self._metric_logger.log_dict(memory_stats, step=self.total_training_steps) + self._metric_logger.log_dict( + memory_stats, step=self.total_training_steps + ) self.epochs_run += 1 self.save_checkpoint(epoch=curr_epoch) diff --git a/torchtune/utils/memory.py b/torchtune/utils/memory.py index f853b1edb3..a767ea51c6 100644 --- a/torchtune/utils/memory.py +++ b/torchtune/utils/memory.py @@ -172,11 +172,17 @@ def memory_stats_log(device: torch.device, reset_stats: bool = True) -> dict: reset_stats (bool): Whether to reset CUDA's peak memory tracking. Returns: - Dict[str, float]: A dictionary containing the peak memory active, peak memory allocated, + Dict[str, float]: A dictionary containing the peak memory active, peak memory allocated, and peak memory reserved. This dict is useful for logging memory stats. + + Raises: + ValueError: If the passed in device is not CUDA. """ if device.type != "cuda": - return + raise ValueError( + f"Logging memory stats is only supported on CUDA devices, got {device}" + ) + peak_memory_active = torch.cuda.memory_stats().get("active_bytes.all.peak", 0) / 1e9 peak_mem_alloc = torch.cuda.max_memory_allocated(device) / 1e9 peak_mem_reserved = torch.cuda.max_memory_reserved(device) / 1e9 @@ -189,4 +195,4 @@ def memory_stats_log(device: torch.device, reset_stats: bool = True) -> dict: "peak_memory_alloc": peak_mem_alloc, "peak_memory_reserved": peak_mem_reserved, } - return memory_stats \ No newline at end of file + return memory_stats diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index a05320c3ea..6dc96cac68 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -8,20 +8,21 @@ import time from pathlib import Path -from typing import Mapping, Optional, Union, Literal +from typing import Mapping, Optional, Union from numpy import ndarray +from omegaconf import DictConfig, OmegaConf from torch import Tensor -from omegaconf import OmegaConf, DictConfig from torchtune.utils import get_logger -from torchtune.utils._device import _get_local_rank from torchtune.utils._distributed import get_world_size_and_rank from typing_extensions import Protocol Scalar = Union[Tensor, ndarray, int, float] log = get_logger("DEBUG") + + class MetricLoggerInterface(Protocol): """Abstract metric logger.""" @@ -39,13 +40,15 @@ def log( step (int): step value to record """ pass + def log_config(self, config: DictConfig) -> None: """Logs the config - + Args: config (DictConfig): config to log """ pass + def log_dict(self, payload: Mapping[str, Scalar], step: int) -> None: """Log multiple scalar values. @@ -88,10 +91,10 @@ def __init__(self, log_dir: str, filename: Optional[str] = None, **kwargs): self._file_name = self.log_dir / filename self._file = open(self._file_name, "a") print(f"Writing logs to {self._file_name}") - + def path_to_log_file(self) -> Path: return self._file_name - + def log(self, name: str, data: Scalar, step: int) -> None: self._file.write(f"Step {step} | {name}:{data}\n") @@ -171,7 +174,7 @@ def __init__( self._wandb = wandb _, self.rank = get_world_size_and_rank() - + if self.rank == 0: self._wandb.init( project=project, @@ -180,31 +183,34 @@ def __init__( reinit=True, resume="allow", **kwargs, - ) - + def log_config(self, config: DictConfig) -> None: """Saves the config locally and also logs the config to W&B. The config is stored in the same directory as the checkpoint. You can see an example of the logged config to W&B in the following link: https://wandb.ai/capecape/torchtune/runs/6053ofw0/files/torchtune_config_j67sb73v.yaml - Raises: - RuntimeError: If W&B run is not initialized. + + Args: + config (DictConfig): config to log """ if self._wandb.run: resolved = OmegaConf.to_container(config, resolve=True) self._wandb.config.update(resolved) - - output_config_fname = Path(os.path.join( - config.checkpointer.checkpoint_dir, - f"torchtune_config_{self._wandb.run.id}.yaml" + + output_config_fname = Path( + os.path.join( + config.checkpointer.checkpoint_dir, + f"torchtune_config_{self._wandb.run.id}.yaml", ) ) OmegaConf.save(config, output_config_fname) try: - + log.info(f"Logging {output_config_fname} to W&B under Files") - self._wandb.save(output_config_fname, base_path=output_config_fname.parent) + self._wandb.save( + output_config_fname, base_path=output_config_fname.parent + ) except Exception as e: log.warning(f"Error saving {output_config_fname} to W&B.\nError: \n{e}") @@ -225,6 +231,7 @@ def close(self) -> None: if self._wandb.run: self._wandb.finish() + class TensorBoardLogger(MetricLoggerInterface): """Logger for use w/ PyTorch's implementation of TensorBoard (https://pytorch.org/docs/stable/tensorboard.html). @@ -286,4 +293,3 @@ def close(self) -> None: if self._writer: self._writer.close() self._writer = None - From 97e8aa395f287cd8d4b4076d42d1e567bbad27e3 Mon Sep 17 00:00:00 2001 From: RdoubleA Date: Sun, 14 Apr 2024 19:19:34 -0700 Subject: [PATCH 75/76] fix docs, link deep dive in index --- docs/source/examples/wandb_logging.rst | 27 +++++++++++++------------- docs/source/index.rst | 1 + torchtune/utils/metric_logging.py | 3 +-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/source/examples/wandb_logging.rst b/docs/source/examples/wandb_logging.rst index 73ccba56e6..e787269e9a 100644 --- a/docs/source/examples/wandb_logging.rst +++ b/docs/source/examples/wandb_logging.rst @@ -29,7 +29,8 @@ Metric Logger The only change you need to make is to add the metric logger to your config. Weights & Biases will log the metrics and model checkpoints for you. -.. code-block:: python +.. code-block:: yaml + # enable logging to the built-in WandBLogger metric_logger: _component_: torchtune.utils.metric_logging.WandBLogger @@ -45,7 +46,7 @@ We automatically grab the config from the recipe you are running and log it to W The config used to train the models can be found [here](https://wandb.ai/capecape/torchtune/runs/6053ofw0/files/torchtune_config_j67sb73v.yaml) Logging Model Checkpoints to W&B -------------------------------- +-------------------------------- You can also log the model checkpoints to W&B by modifying the desired script `save_checkpoint` method. @@ -53,7 +54,7 @@ A suggested approach would be something like this: .. code-block:: python - def save_checkpoint(self, epoch: int) -> None: + def save_checkpoint(self, epoch: int) -> None: ... ## Let's save the checkpoint to W&B ## depending on the Checkpointer Class the file will be named differently @@ -62,16 +63,16 @@ A suggested approach would be something like this: self._checkpointer._output_dir, f"torchtune_model_{epoch}" ).with_suffix(".pt") wandb_at = wandb.Artifact( - name=f"torchtune_model_{epoch}", - type="model", - # description of the model checkpoint - description="Model checkpoint", - # you can add whatever metadata you want as a dict - metadata={ - utils.SEED_KEY: self.seed, - utils.EPOCHS_KEY: self.epochs_run, - utils.TOTAL_EPOCHS_KEY: self.total_epochs, - utils.MAX_STEPS_KEY: self.max_steps_per_epoch, + name=f"torchtune_model_{epoch}", + type="model", + # description of the model checkpoint + description="Model checkpoint", + # you can add whatever metadata you want as a dict + metadata={ + utils.SEED_KEY: self.seed, + utils.EPOCHS_KEY: self.epochs_run, + utils.TOTAL_EPOCHS_KEY: self.total_epochs, + utils.MAX_STEPS_KEY: self.max_steps_per_epoch, } ) wandb_at.add_file(checkpoint_file) diff --git a/docs/source/index.rst b/docs/source/index.rst index b9fdef3c6b..9e16acd5d0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -107,6 +107,7 @@ TorchTune tutorials. examples/checkpointer examples/configs examples/recipe_deepdive + examples/wandb_logging .. toctree:: :glob: diff --git a/torchtune/utils/metric_logging.py b/torchtune/utils/metric_logging.py index 6dc96cac68..59b862303f 100644 --- a/torchtune/utils/metric_logging.py +++ b/torchtune/utils/metric_logging.py @@ -189,7 +189,7 @@ def log_config(self, config: DictConfig) -> None: """Saves the config locally and also logs the config to W&B. The config is stored in the same directory as the checkpoint. You can see an example of the logged config to W&B in the following link: - https://wandb.ai/capecape/torchtune/runs/6053ofw0/files/torchtune_config_j67sb73v.yaml + https://wandb.ai/capecape/torchtune/runs/6053ofw0/files/torchtune_config_j67sb73v.yaml Args: config (DictConfig): config to log @@ -206,7 +206,6 @@ def log_config(self, config: DictConfig) -> None: ) OmegaConf.save(config, output_config_fname) try: - log.info(f"Logging {output_config_fname} to W&B under Files") self._wandb.save( output_config_fname, base_path=output_config_fname.parent From 55513f758d87abd00e0ebe765fc3e3d7d9f20a9e Mon Sep 17 00:00:00 2001 From: RdoubleA Date: Sun, 14 Apr 2024 19:55:49 -0700 Subject: [PATCH 76/76] address comments --- docs/source/examples/wandb_logging.rst | 8 +++++++- recipes/full_finetune_distributed.py | 3 +-- recipes/gemma_full_finetune_distributed.py | 3 +-- recipes/lora_finetune_distributed.py | 3 +-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/source/examples/wandb_logging.rst b/docs/source/examples/wandb_logging.rst index e787269e9a..1faa9e1c26 100644 --- a/docs/source/examples/wandb_logging.rst +++ b/docs/source/examples/wandb_logging.rst @@ -16,13 +16,19 @@ Torchtune supports logging your training runs to [Weights & Biases](https://wand .. note:: - You will need to install the `wandb`` package to use this feature. + You will need to install the `wandb` package to use this feature. You can install it via pip: .. code-block:: bash pip install wandb + Then you need to login with your API key using the W&B CLI: + + .. code-block:: bash + + wandb login + Metric Logger ------------- diff --git a/recipes/full_finetune_distributed.py b/recipes/full_finetune_distributed.py index 7624de609c..28b882ea85 100644 --- a/recipes/full_finetune_distributed.py +++ b/recipes/full_finetune_distributed.py @@ -270,7 +270,7 @@ def _setup_model( utils.set_activation_checkpointing( model, auto_wrap_policy={modules.TransformerDecoderLayer} ) - if self._is_rank_zero and self._device == torch.device("cuda"): + if self._is_rank_zero: memory_stats = utils.memory_stats_log(device=self._device) log.info(f"Memory Stats after model init:\n{memory_stats}") @@ -451,7 +451,6 @@ def train(self) -> None: if ( self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero - and self._device == torch.device("cuda") ): # Log peak memory for iteration memory_stats = utils.memory_stats_log(device=self._device) diff --git a/recipes/gemma_full_finetune_distributed.py b/recipes/gemma_full_finetune_distributed.py index e73369d896..5114fefb9c 100644 --- a/recipes/gemma_full_finetune_distributed.py +++ b/recipes/gemma_full_finetune_distributed.py @@ -266,7 +266,7 @@ def _setup_model( utils.set_activation_checkpointing( model, auto_wrap_policy={modules.TransformerDecoderLayer} ) - if self._is_rank_zero and self._device == torch.device("cuda"): + if self._is_rank_zero: memory_stats = utils.memory_stats_log(device=self._device) log.info(f"Memory Stats after model init:\n{memory_stats}") # synchronize before training begins @@ -457,7 +457,6 @@ def train(self) -> None: if ( self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero - and self._device == torch.device("cuda") ): # Log peak memory for iteration memory_stats = utils.memory_stats_log(device=self._device) diff --git a/recipes/lora_finetune_distributed.py b/recipes/lora_finetune_distributed.py index 110d02ac42..0b6c76e825 100644 --- a/recipes/lora_finetune_distributed.py +++ b/recipes/lora_finetune_distributed.py @@ -326,7 +326,7 @@ def _setup_model( utils.set_activation_checkpointing( model, auto_wrap_policy={modules.TransformerDecoderLayer} ) - if self._is_rank_zero and self._device == torch.device("cuda"): + if self._is_rank_zero: memory_stats = utils.memory_stats_log(device=self._device) log.info(f"Memory Stats after model init:\n{memory_stats}") @@ -541,7 +541,6 @@ def train(self) -> None: if ( self.total_training_steps % self._log_peak_memory_every_n_steps == 0 and self._is_rank_zero - and self._device == torch.device("cuda") ): # Log peak memory for iteration memory_stats = utils.memory_stats_log(device=self._device)

2< zyQFzgVpniNzKYU%Uh4Zl`j-d4fM=hAN~JEBZ8tH+Oea))vJ)Kv%N-m-xXC|%?99oJ zKH2Yejy330BeF9>*ddcmr&Lot&x}1n$d1?HRLZYX$2BwZhP9n++bH6aK|>O_&z;TB z8+hADdwwSN-h@pt+nHc$yMBup6?9f;HdfTb&Ms*&Y`7R9EHoY}&uC^RkLHucTz}Z{ zKA!yNYU$bNJf!<1?Ey>bGwM3JiY7UG7?+;>QLSjN7PP(S!E|A6uE=S_(lTRt%q~q^ z_1(#7E(O3tv{q*8SLK}4Kqv)Qam9uhX}q!)DdGrx;z;`T_-5bgC?`Qf;2%2eg5_k) zRw(4Cm6Uu~I9qI_V`oF(c)zZ3S?R`vP;1v>W-f*;f znBalp*lMMF?GAqNw2s49|76|J^Lq_Ozp_(P6do3A6`X~^qgbzIwFRYUjDslZO_h4E z%k3J-37?XN_v`p@$Cb7v#yF4FJT`J_llBh`95c^}ZtBZ54NKPivMKyWI7Lg|fBM&r z>PQQwz6hqyjynmVJkR-Qy(W}v?L?omDhCYm88ITLu8JtkjhI{@#DHS_|Jg?lbB%`^BW(J)~e)O6upup)guX4u8sO`GvCUo>pN|y$#u&yAxC|yPr>n zf;|y}8tVJAN3g&UL{O%k!kOZ67YyUL6z4M*_kH7@=ryuhJ-wm-`4Hf@a0LjiO|?w@ z^zqv&3`=9Q24VYd7Vgz*hnjqp$NA zv4-RLoo*$MBt=POasJDEO1}>q#4XSLK!}a#yUVb-v#=d7r-yWPZs;Zy{fN@wR!k#H z8(1UhaBOaNp3KtuGw_VteUh-nKrE8&U_u(UBbWf)C&Y_pGjNjvziUPbb0q(B?o0+J z9IHF^#?Deb=hJ~go3`}F5&}S2&iBB2s_KQ6wc=+5$|A@x1w{X-n{{HsKr0>9>g$U)PU$JJ~xPpGm|xuiHvOA z^JXypR|6`T5Cnt=e`wXa(q2&91P)QwBKA-0qhsSJJy z<5{wnulY!ci;IJabK56+2wS{BiNb82@L_wjt0}mhDb*QF0Il0S<=BKjPR@7TP5<=< z*h8o??@x`T<_OtdWnPKWjT`a-XkG;Ym6uF#Mc2WqdGAFm`tN z>?oWAy~IF695nBn$r4<& zpP=-%N~w>s&ekkQ@9roEANGT^!oXDaOLyOjo@5yY-z>uz*Aq85x~$~UKs^!v$`iDK zn_69kk>x|>$N3H$cuXy49vM}Z&svo*0uE0ES2L-+bij|g31DN2Q1xr=f6F}rm~IwS z!D&iEg?Uy6J)Gj&z4Fukge?j2@$ZBx;y*8wwB#8M%7p?Ek>GgBMV+D0TVdO*@VkMU z^H4Fr*-N9BMCgr=`i_T;Sq`ju#4bm}AvYa<{=dDVTQbxj6rOWhmaIk%Y7pY)Oe@%M zhiX-*DwYM^2eA<@w^l`wn_)JfB)t9=&v^Eb?kv-V*?VrH7v5zba-|>GWJF77v-8dlvIaa2}&)0);4)1!fB_z*F4`9I3&+MGmj6~TXcho`h%;Ws{Wq> z4;)~xj8J#;t~<>gP#o*B%fP_kxiK2}S+SjxR{_lDpg9MYqEKa=JdkqT%;4Eg7N1rB zt7o%;5=Gsx>Nll5oO}Enq9`jXn+5{(lV9L&IGrEBVv}$WcPwPn-M=Es76|Y>E6vHn zwO9Y6#ld?@4kVGd*Ln-A44_471xaXP>?FJg4<3xQD~j=iEP?{UM;&=DY^s1Bk#0>m z6TSAdnK^&H$X0&{EPqKBb37awUk;=wzFBq9e{-$Mr1CBu7xp5%`Jjg>>e8U!ZSP;z^W1DAut zR&ewOS%ppFUhkJJWY`!OtPSq~&bMy^ZF0N2=f9ejexE)4$xkA+LhD3A>^!6PH!SG; zBB9oznmFpn9S~o+K^g_sXFLGz@Qc~rZ`=GfsK&C3gkTL<2$-AG$2(4;&jjZLyS{R> zf_4rKIWqtd)WdfN_0Ruc`Erf&6!El=*;HT}$2r!A3{vyNmBx5G=URrhK&(?A1gEYF z)6RFG2B6Z7K12`@rw;o6FN8o*xf?FN{>YElSIJBqb*5->atRFWOC6Rc9>m-x)F`A_ zb%x}^OaKwudD4n#ZnKBsNfy{X`Tl6%l_+#Qa^J>(32jIYcv`uEOVxmgf<7>w{X;`9 zp)OI2;)I;wz8)DE$N-#<0L03lr`mKRD8q9ImX^(n_Bd$gd%=4v;2`4%PYoc`_m!tno; zZuW@Xf>{@oAf7-l2}i}7n}k~uWs*lz(!tF~Lu8{4V?2C7-&w_03jR0Tu}?r`r?J`j z(_W+G!CETr3-5TZoQtOAvV94OV1YOt7qH<=;*@$A*qC!yTH@fTWY55UfXDxLmYq~) zxF+BrL%F`h#YsJR5@SQ{N7TY6`XwjBa34zVZ8!!mgZ=X<-w*fZcwPnXXysN%|FgpV zP|LYZo^K&S3#w_kX$u8w1(zre?ZnW5JnKLXdx~(HjtyiCwgFqT!B}JW3fz3UrsdG+ ztAX}3DTuQjh?xHZ?QXh+V1##soa-R7IOp7Z-C5Y9+4mQPzPtAhta7u>E10!t4LLU3 zeL}4y**Uh7MRtwS!=t+HKRetxlzs?t zQ)JE=z0gk3Rn8(+W8&flwavH$<_A0odk#z<$;NxF_y0#FeEd!^+VE3N#v{PKkd8XP zB1vA253Ck%{3wZoNy~MB%Oy9k+71Eq4>1OqJw^%+T34;XL~9D-(6ZFC(Q667Al!$i z=ShfouFIhR`&p6QWDxO$V3kO>F)>tD`Tgv<57rxGVWVE`U)qxnJn{y2Fg4LXT%;q; z3pxb~qnXH~6@h)uci-HhY)TtC*G0Jfe&+}l&V70Sh39YHd0+Xb=XJ<>D>z5Jd%tMq zscC^F%}?V*a(N$_*T4o(+HqW|wf{z3?g82J3N&QyhD-PZ^iU4Vez!s1h7qXitR>e1vpma3lSez|;1^Ehn~~D<79FOL0nTA?H$(8@ zLd#N)nj6ju1|A+2Iah6+$y}Le#qzkqMZMzu!fMyGfR($|f@*}&?mJV+U z4&ug7c}+e0D^$NqbdE*;HD1^3!_&ACNA#F3=Q?d$f6{&Kn7uPBS)My>E2ee2kFBwO znwF)-|sj43U&8-Ej%h+vR2T2LukquC@Dh+Z|aiBayco;)Vnsx+&-%@a| zl9{10om{g477vokXSWr%LBdLnuH2^RTm_uz=$mY0!-WGcZ$+OvA~}LFnC`NTBZ(I2 z#*>sL(D*&p@Oc0m^x0sll%`#_bHu^ih61ojkTuyuD^>2I(XX$cL28yWdGkX@guKH% zQVp{?-&Xik7K(vnAcCoZ%UtORu!eR6sq6%kzFc{3;i&L%xfh~$o6Jd$ma@TIa4@u+ zpq}C;9R^g|$$;?jQkSl1-bA5Dl&gvNU-OwP07o;gdMw@Y-qD$FT-EKfDIzWg{jBA2 zQ>Wr5!D3e0h|yz|vP@%SDwuXVNA!nB+^5eq%{o#s*)j;N(dR3nyPZnC2 z-s314yVC5a4Phh~Y=KdvJ(KtD%fk*i8JSGe;7QVB9D9ZwT4X%Al1Z#GSzzYVZB*`k z#3^i)0H90QF^)%vBjCs3)ynI;4g<5q{C)lXpWvX~ADUk9FJ7ZNkBgo!-F0gSTT6)( zP>H|7^N~YI9W&$<0BDEtffhlZv)JyF_~MmSp$FfTnT^^fxWSke)gLQT@}X9skTxaI zymUhwjAUaHNt>jLXqvr{tLO9OG7`P0yfHbdc=PgEI*c3x)Qar|q-_Fe6%;_n!23Wy zhSZ9KSCe8Pv|ac>hdW@WyvwqldZ6@TWPQ}f$Iv6+%;%bz(2+t*e*~!33*RI4nCrHX z<|^Mc>qkSu#~C&3Zo^$sNAU$G<3cp;bwf{T{ui*{^p+hTRciz;H+$QQi4VAel4>}w zSwBM_$wdZv1yO-tMaz9VnPNfpVVusIsqJ%V8h4N-rnVTYJ-g^S{u z6qpYeiqO8Qi9e#F8=U1fjt!(Y@b^{}?@4M*Nh#tDl(zYPn+Hx0Y>%KRg8bFDt^Q#F1{<0Es!MSsSn{+iYp zS_UvritZJDZMISS_}Wt!;MjZ_54wdr_BJ8XufoIU!n^6z0(C)aL&|A3Y7l0&2sc8Gdh!Eh7{u#FAW z`xJ749&4aG-1;(}7Q!E5W5C#`+HX8AVITdCyZk{#S!mbbGA<9=tl1PSt7rAwS^1>n z?8jLe&Pb_k?-p(^PCxW|x0Gu6H~+>+_3OxmEzsTsQxtS_;Dhh>XDQaV4SlNh1ljK> zyj9PPU^@C|N_>dz&n>61zaP2R>M{6#+ZOTDjpJZ#hcnag*IW_W={V7&eQCrr*@Hf4$J2^ zRyXIt6oC(4%mxadnN?wxG;Qf^Sdz!h593YNe_~ym?{tE;$Xwwpf9-&|Kvwy7t3ZnM zNZ!wY{)keyXclXV4ecyIKdC{`=Df3 znK!KjT+(-Nw!oAu+K#cUjHJNy>kMc_Q!xB>Z?1UMcIb6a@iJv}^D~?CkbF6FZup0A zL=;TnwxWdXk?_|o*hhwjb4<(8AYnI8q89@+_nnV@(G{M$cYYNEiG2S=Q7|bhLe~;R zAjo7q@~7+xTf{cdS)iJtUOXn`9_-d!q25<+lNU}>y?G_A$sp7_$A^v30WAlmG{q8N zz7YjD3{sQucjd%J(YYGe9Tl&y4Xe@(FXrphzfB;HW@Y#wQnDRLB=ZV*Z$G5yK}@ay z;(FZMWYtULALou>aa(yG7W9Zq=~jQK2)iCwT6s>clg9@cc<*!l={=o+dKD$*jc)zf zx--fMr&=PND$1S6S=)jOO%Fu&RYiZ#t1I2!eA-q& zI-q8;4F-H^l8WK`!21baJdPCog=g`5A`rE|jDC-paX_Cgp!rp-lRol?)Qyez`>yNs zYW~INBou#X9`^18fRPXPW1tn}DCJzi@oy6>GOf&7~MYA4& zegu8=*E8`lW#C}M0sbm`D(|a8(Xi&rTm|>|EO#a;{*HfbRRdIE0 zk8jMh7`>aN0Hs1@KGy94iLj*4st;^Wmx*&>gwIby8~HrtAw^UD5vE`gIYfOFY=gfz zP&E9b(yT}1lfeu`n~C?qTKj!PqX8+JzL@7Yd;VikT;3NwG`$_pB6ajo@l!80^=v5+ zhqLk+PgW^)au7(dVO5eA>7x3%{tk$|3S>7J|MC;`@go*MROE@0>#1e51*0c<7>ZBw zZ`=e%Mtz_+n(W(-JS4aY#M@z`b;tOwGH8#29`baca&St>BNf~Ec;$?Yj6l^kO;jqF zgdimobZvyohi{j#(#mybXIJFfG+&&m)UF}}O7#`~>ZfOo3bhVD`PG05iNKXVjB%1_ zLSPah{EfF(9K&-AFTh@a2&h{yjsMco$HUOUG!@m4O`C#IKw$Ft-M!Qnzc(xsv?Ocu zo*pk0vYi1H?VL5)UQnwU;fG6gJSC~VSxqKUMh?Qwo+zyCZi~^UJ1=RMxVKK6Iv0pP zSETy9Guba;_K5>npJVF{$%DP9IsC;5pr+OsJjwf)-*q0*^Ap*9{?nT-{%v^l@k>^K z%NHU>hZ~o{Z`|$?bXyM4ywj_f4Fe>u(6N>s7)Svkstxt4bZT9o*#wkPQgCW(g4xH+ zRK?%ki}B82PxT5|)rc}&e)sb3r~o{zA`9rERz;4Q*}y<$8JUH*Cq*>@rcq51Oi7(v zrk(PsTC!kBrLneS^P15$QuE?ID?Tdj^@mX5i5$YASco;X3^;bvBdFmso>Y5gP$*2H zKYWS$+FyMN2bTCAmy{1@5FULB*nEJ7PomN7n2OHWqlc@%pxBLuoAXlKVq3PfUVAMl zOR6`u>!&~Ef>8r?H7s1B+vN?URXPO@_QSvy_5-cEC-v-ZnbQyAqHp1Us5qHL)Iv&Q z88$Oe)a*GafgdaRfeg7+<>(x(iGfx8|Ht6;Dc=Mhfl!b(t9=pIm7W{(L}-v+m~9VF zEAzW0@g{T*DP|rY$fow-8Inh`>U{+4i!4(uSoLRLIA^}&7b`hlr;Uu03u(K#`WiL% z(}H~5D@3OjRLqR8V^;LLE7oKyPF6eig3cAs_cY7?PD@kVJq1>ulv5t4spmas#>K3B z1kN&|(+hnH?8l&Sn;dchmKRhKRP}+X5gWw4Rle~L;R-r5^Ks#<*B6P{S>d2lZ(~&d zKUR>AaVkib{p=3C|K)lP_-^l5-dLi6qzo8)px@MB6kkT`gnlc$zVL!sr!+hSwK_1g zRpCJ@>(wq8jDJEwFvD*P(>L!Q9DGgPO*=uI$GGN~PrN-1gZ+AI^?1pem_O*xmH`bG zzM#yc?k>|kC#9jmo9VmqbE#K$fF7yzfc+5yf>pF1|9mqx=BI2>O`mxgD=KP8x;MvE z52ObtQJm|yM0f1Jfh7DL1>=FXd9o7zRYtXi3LpP)O-Y*kUG1{z>(GLF5S{~4&JX6P zSAS9ZSd7kuV&4LzY9_%PCh37z^}_0!&3%uzeH0{s!;B%>pEdW`KFF}uq0uLd2tCsC zw8?RiGI}V-F|R>qA@;rf+=-rShb-$^CpTfhrR~9G(Zw7Kd_!P7_7Z4Gs9ktW`tIbE zj^j4FgPAg71!Fhl+^F<=Cp-7@Q&3KR6EW7_R;opHKfT9lBD9OhpdQT6Loz@W%jSv> zimUT|C@yYHEfMIN3b1Mx=ra&C-yzQ*iFcl8{CcKYX7TsW9SyQ7RWKOA=QQX+cep$Q2#mwfUx1fI=oFbnI0;w< zg?D@|`IV?tJulb*wZfa6fh;Ztg5TYiciWDw0HFdJ8hAznRavhNlc&In?Y;s68~>M3 zXpMS#(DZ7u zq@@h~`n7E%C4lYAw2xN_rP3S{&E8I}jYf9)yKz`kak;pH4`TefOJ)1T9f9b zcCOAZ+ZB?68bxNg%?AeYR-zD~4mC|llFMZxS-`S-V9o1cRp<5H9s^t7!7i;?cQv0i z6N3sMSf*mr%?1#yEQ6ajm7^&^EWiGILjBycSBJ9zWPIv5^B(6X&s5NYuH%tegGO>5 z%+zU{=}w#Ovp+h!ptKBTg-UK^!*Q-q0?ZAe%tiKP$EqYftW|1$zu-P8q9|}q$~XHL zce@I4GwDU)nho@Oa+5!TNXR3?IS)A$-Y|vo1Tv~4)luoT?@&~KudSjx`WO#_R=Ksw zk*n+i%=o_E?(Z@kwm8PvKH#v{=2ermB9^RC}KLeOPe2SO91g0RAvF@%_ zay^KXIAf%oYMx+##ViZC3^A( z`2x~uy~5Htp+x0;Rs-(p!(22V8lnRm>*r+K)gzKghIA?`eXYe& zK(aD(Pp9&BHWVv>86Baa&wq^g%>|Z$PIK3J+uC!Por%|-zdNAn$amK~6T+0l!Z_9j zchf1@9^E_dB-E$;uIu$Yav?)lcxLx@Qg?HRKw~nklnE$=6fBA-)Dl+L&npcX`dpRUa^gY*9i$}ZPfOWcqKsh@HlH?Z1zDGx zG|0AoiD$9ocr`=h=qhZdx-EwQ^4OxDQd1p$>Pr@axXZ{u! z3XH{>!p6*~5)VCa!A1-eo$NBFLikM8Os9E&y2`}In?|{{5ZnKv9kz_<5B9Uf7^Q&V zuY6mPxyv$-7&ZoqKr}`??*;*9o>`a80T84rCu0V~8yj{fShqKa5w_zfC z4^X&+-xUOmw{ng>!~7M@TMNZlaG{(A{mE#O%8@z9`8{&iWfKp_jT zI5(#?_)(1e$^00_v^%s-B?cL3z>zIPq#1A4{bnrN(|UZ`VTOV(3&V5*MM{)CWiDk7 zI(cT9$=W4%K~erym)#QHVIs7bvo>3a`6S96RFHn&3Ic;9)k3ycgh9c#m!|zKCWoyI zk1YO|3(a88ytCD+C!8Oz)4cl{mqDZds%f88grLoj4B1tP;@q`>2DG+($^TevrTb`S z4V;2-A43!o+}1~#IoJ%ioJU0X;-WzTXF%7GlcaXoR*=b1M#YR| z6mS0Umeu%<gsgrBT3W7H{*^2jT)CE7c7Q3MLV z&=~=l!T@S(~FpzKqAzzHR!R={lFi{6;?n_k3dPBjk z&i&OR!=}#`*kTF4zG@F7O6drHHN}s;frI{jFOK-Y)9`eM2>lP9#TARy48hXZB6AHd8Z8bPIWeb>=gUcyv=BSL={Yv?xj!$QvZ+>qRt zaqwg!$bHVlvv{&{8kK*{gP_x`hhhxhFUa>EStp(<*r|PbHu?e_M<;p&#Z?DRwu4*{ zC^ih}m4fZ~s0{ddz2VsmhJV0UG?!bpaQkB~>8&%S>PHJ>xGM|T6+j|i({ zK`-o0C<2goiv>)aJPRCJ5Cb49A(rr869q*KMHe*=#G~uwJ=KeQi(05j5^*G5brTM5 z;^5rE`u8T-LI5RsdeoDL6b3z|2Qa}vq!w*OMnhvgQ=EVjzio}EIF!rdPR%}yrGBOa z02=`D;mKy6)jdCa-x)7CaEwY&WFklEoS+2Qke~UUksiDD7#R9Cy}Css>72)TFWS}U z2i~NTlG4uzzo9ohqQy|z-gM*PLO7T`bSn)IWEvE>BYX6a45oe1W!RZn+&=pFJ2{9} z*au}F8L`lgGAPK@u{fTq)mU$ACQ8T1$oOHw$*gBa!w0JbsVDAp0DG7n^e&+@m9+yK zKh%#Ny2!l$u0@9(i(f|{7eMc_Q%F7v7JzW=Ua%(4ZB7UvD}1In7({F4PH&QGf=uJE zt;u1%l^+yH8~{Y8-z>3uERdLYfGT8U3y6>$20d#aqEmA@_C>J9agXTabtDQD#Km8N z?#GK5Zsd202mrlUtNzJ@$uk6=fPI|LlU0rs9b4@Sipcgu`O`O^9Qm+VK-5{09O+t^ zZrPr)4Qgv}Qdd_;fclKvoF|AVJguX;U9iA=aN&A~35;}c$gg+|{EDAbQpYoS2a#P_ zMY6&pFzLPqH1VIwM>tGl&##!(zi=MZuLvc(a`XRH%QOM|fHF%%) z+y!S8qgtlEKOW8G2iObgW-@2IYtf`-H}t%uUGb#$eqx}vPQd9%TS+iv91AOdOIAbof&GIbp8 zhS&`yPx4-en%=)J2k#1(FK3=U**n`5$z*c*=xV-jum;$Sl-;FM;qV!IceEwo|7n)j z-l-ce9m<`^5=k@pN?HKBF5XoX81^;PR7Q>+%AgCQ>c2Ba5ma?Fp<8(_Al<&ufQvWL&;wAmp&tkI{plXXZxh0 z*pTImWIBeng*&dcMV#VJ{|U{{Ov4mKBEJ{{*BtDl`6`$5UR01pLL|~`p_lX*5VsA3 zC7W@GEY|yulNlUmkdXiB0K8~m^THzGv0Gz-yP>F$!xn~(f^J6h{J{%;((2_pT;x}( z3Jfk292+@26C7|riBhm^Jn*LF1Tv=VW#63R*WwRzRq7=rx*N4J)<0F~-H z$q|Ro*rVwgGVnIDgp}`cqf2m?dICQ4!9Y-Sa3V`I=rRKiMGHP5n-}9Da0mn;?0GU* z5@M9~P8=uf$8!6m`B?aExApP2Mw5fLT303?^3XoZ{03jQbqp6eTa7d9M&m{^IZANz zR}XVE=?X!*^WmSn!6AOTDoLjh&E>gqoa4^o?lT+PyEv+6#bVI#i)_ld?szadAop6RT;IIRqBC}gAG&2jinY#j)$pSSQds zi%*{aeJ%>(J(2SizxfRmG8vCQwTXU@9c)pJ#JQfwZc{FRXP75sz}9{C??a&fEZhYE zEy`PER2B|zVgl2+dU28BDs_n8QQ(z}&4G-6p~P?+!eKabrF8H{mHKh0{s`EooC>R* z^p4|aA(Q}6{qf=%`|%GM$S5IEHrnQ~UUSN0`Gs&)<-N9{cRo03cMS>dfy$P-%-$E4&OwO;e_!~9kN@9e*riT5AZB;dtP=$`#PmyotJ zf~0A4Aeuz_S*}|VOTK_*Z z1GUUPSCB-&OwJ!Whbujx8N15s>1X~v1o{sdD|EXI-kaDo;#?4&-oJl8=G+S5J-#GQ zNkFO9^m6C#97<9w29L;s`ufElKr(LWjQ5Jna0{Hq-#y<&Nr(p2ePnrKs?+^q zpPG#K0hC8WTXh^hzE}V)C}Bt(yH(2SRyReoWmxf1y5=MWDZ%GszMq2r`o(z&NeZ9+ z-GOR5eMff|sPF4$4S(@MhJc8#+EqNfrTI=S1O_KUzz(H))j^P$25$bIfs!ZmMD93D zd@lf1p!hzvNfi1PoEs3C067GAm=mPNNqI8jqyuD>l>@I~i*S&A|F?AldjU!BCU1c~ zAZrrr=^qizXQ%uAS{e{4hyW@iZbjHYaB$@Uj646{LYEbKaXATeQQO{24csmDqqHI| z_yLwUf%4c$9Qd4BOZAk+;2+=g93e)0;3DR>?SI{4G!pThzn z!fywvFEGp320+VQ$PJE%?n>1P0P~D611t5asZyc85=~JOBDDcbsJAY;Z*R~+FM34& zqOQ=*wrFMjdJs5+3y$jPqO-PFc1~PM)hXj(eh^2%?O3zaV`K+gEiB6w#Og1!ku~ej zKl8xb);>CA=Q?1GtC>=n`nA6j255snC2l{O?`Hor1%P#y6x`>nL2{4Mo-hWo5xYby ztUF6#a9=g!LE;%qVfPu*l!4QpJ7;r>^7M!_pyCB;zTD8mo59`RIH1|0$vn!4_rw7| zb`cH+3?!|k9A&Bl17OBmMk@SoNSuTsPuxcRk1p|kMde_Z?Uu?{Ip5F*d#7ZZqQp2s z$AZ8A-BOSrk*Po5_{4M6CyJj=ulmEM6_6I0YENLa`KJ6J+5QGJ)HK_8**#7%s>xW( zY2RMaJuoplM=KtIt2p98p%=X>XF3h%Z54$`Va9ds)E6V-Z?r;NQ(QBj&IctF+1$4f z+gcB32rGkYtvTd=8sqfUxvCh#BDTBbHPc(tmoCp$r}afvwyP;3F8{F=;>{G-v$}5` z#wykwENe2Gwx$FAAT<6+bbUcjwI3yMa)Z6l_9?GcaNqV1D`Xtl?~$oHaz$-xqkE;{g0+DX`k>8qspI9HrD|`nu^+F!{KZ^EnXS6C+Lov0K5*22Ix12-l3ewR0r0^W@89mw1vn*+coPzKFe8h;~|UP54vlkae{!|GorB%Xl?}iby(ZuX)AYT59J*;eq#H7C+i5S z=#nk-1E(9J9=Fv3%eO8HA>89`-d-ZuT%+~%)+($8vqT5lrl>sUJ%~5t-koQUiCy29 zGrF(#tqRlA=K~-fJiG2NAb4@%H!Du_yV4)qJm2c8X`Q3}zW8!5@Mcjvd)#^!F?g~a{@&u40eLz*=ekQB6;kB#_B%S+xU1n<7)eN z7E z7bm>HXqnoEDJ4dY&SaKL+lcMPyfHgHtCuR}S5GxCH-8`iA@$FybJtL}ByR8ic6EMu z=cYG=FE*eV@+yBwL83*90)oL!E|tpio^5D3c(y&-?brR-^aNh zKfT(D$0h1c(O*e=nD}I?NOfmtlUD61fgbFc8>+hLa+KxxRBwrgZ1#g>!~2OEdASU- zp~_%;D_dFeP_!^V$wlx*0;Z+?_ZWKdf$N>~v_Sc8kN*7iSEDyU8Xds~ z%-N=JeOd6dA3kss9E|KIj@nVvs6eFzg+y;2(|##!hmAX3o1_FX+K6$U0*lc%h@DzV zTB#3zTD66`K+>ROQ#9>2U18J>(|3l7c=jHdPsUjiRhFGq)fUWj5}fI8C~PQM=rl&K z3s~ygVZZh)l^PKTLyVuPQgcFAaK)QBv#+j87N4*HYO%vPb)LhYRCMdn9dT+ghu@T` zcw+1t9GqpC!b0=@ub+z-ds$Nr@)syt#8wTB9pVj0KYg+%n0%b2U1TdiKHU*plx^PC zsq%{dl_!31HX>yoIEkef8?>5*U)K8&|{$#%(|v+A0gNF*H?2$0?oDWs@X;V4?X zK)-IBMBj|Q$n~@aRbjzQwnTb)D>0Bo?ZLn(Ug8d}&I2l)GuU(+`8Q%UbDf=tCRJWn zk=iGkFISGfaUI&GNCr6yjnd5>{Ew%@7)7^+k$m%;^+9RAp=(`a1lwoQH?YFU$Y8d# z=vv8C^33$6sb4uoX9q@whSk~=E{bmnv&?>N)qDDTjQLMt$18Sy-QB*(i45;vP>KJX zJVU0O7Q&ov0M@K+Sak4v2yj^Y6u-h);r*APOvR7suz= z-vRrG+O13RAAtpTKDTZ>^eIYCzcsP-dp2Z)Y?Mfzu10z$n&Oov$4qpXxk@O}czwO+ zV~JlYCney!GcGs?f{_Hda59Phn6o=;xHypJf*GcIt{?i(6pcKdd)&*mfG z<4L{8gGG9H(tJj>4~*BY=AD>Jr=#LEN>vbfX|S4hnsjMG_k7C`((d9#q37-!8HK#3 zQXjkgAnEwdjDz9RaZ$O%V-g)Uj^Xb6%}F)bs&PTPt?@Bf%+a8<{<*}Jf zOYxRtR~KPpQoL)Pydo@3)C}fmU-1!b?X(f^sCg{$#c9KFLF%^#+{ZblED~FLK_mD1 z?7P*f#*gLsaKx;L%Utr5Ka*lMTsz_7+E2WTe9kWmkw8JVfd)_;k>?HkA^b*}g{k*T zLY>fMYg$w%t+k!^%<3}WvTw@1pW3s;EeY0Hqx1Bp?~+_zb%(HZ=;f3PU{l93lcGPH zot`EQcRk;tQ)!rrW+~kmg9|G-%mf z)E<%ihT7@4(i}_&YwQSJ5{-A}U*|jj&bSO0@YQ9f{2Fak_Kr@7cFqvtL++H8;XBoGFX0< z#C^zR=;{Ch?GI+pBszwByKbl@tm67#mA`bZdXLdS0JC>?=d2zA(qEp`I6DNBdK&b% zNzNP~Q+Cf~B^=TEh%w0jT=sP3H_I<#{q0rYS(*EgEB*Dn`9FccjLWz01z>b$gpicU z#XTV%2%*vdvG(yp7;jRLo#BuILU@4vE++}K@HHQi)~0jj=d;R3$6YCPg1|`Tg2++4 zvOgW)>e>eiW&EzZ)>_PWO^-)$m7I7QDv{{VIG#u{`{iNdW@Bsd)jfTiE?v&diDr(@ z6VR2r3xvX5-HIytC6g$WvSzMHkG|W;QyQ97viRn3Mu|6fy?&?gI`y7u*)2AU5MKL% zr9$JJyguca6GGmlK-S_&zEe@rg=<4qvohV)svL!#AkH2x*7dQ8R%5$sD7UU%BXk zz8{shAwfsxhqra6#Sx%{;ezcuh`*@!45P2D!q-u+YPI{Q;w}Ts762zn! zsYy7Q10Slz!9f0S1cF$Lre#3T%t9$$7dr&#Z8HgB`LRYNcBuH!ZvGfBr; z^0Io{LPyS5r_=lemDl@=Ts5ALqh!He#{RbDy?Z^OhZl+P3OWIjgX8$g1Gl$^MrRe+ zwy@0o*oMkDl16)y_}n%-(d7R@DP?25#zBDJ3X!V`0hI8Q1C;Cz;?vdv!(MrB?4#-JpM=&GVu;hy5(YFbJS)Pfo9#I)N^&e$Nc>(oYEgU$nh{M1Z$(|t;vW~s?-kbY*G`^qTeP8!=-T&Ozb*?Ts z2j{%suh;YSd_JC!=i}jiFkBP#*io$|{C4^}kJu08t?SLqj|B%ZURn8fgwEz9ySpyVQ-maj3$Z;dBQ7uLLZj_L@^=s=(#~*zVgJj?HUOC=JJ9y4LrFPu} z!E*kBm|f=4dfbOqLHQ zel1f%JeSRvGl<$Ktgq_@y~ z%;Y`|c}~yl#$|3yEKb$caxhM~W!cq{;Sb4@^s8_pLV+=QXW;ZJBstpJ1}J_q=_)s0 z8_Rbuk&aMqSr0RpeZHSQPc7h z1C3E&FucS%gY8)Im`5oBtz#$S@Gn(k`Vb>L24=_b8|HimypbK5=pKjTcqn`P{{+0t zw~enGw!|v-&=PoZO5&sB&DzUfG_@O6OvQ(G5f%OlnrHw`O~6Mz>ZQF9a`vRY8qXtn zj#yK*vNHRv>Z$moU4(`#-9H~V#1^~u5D(>V%Q@#cf0M2;HMr|BC1(@4JMo-?+xdgP zNk_Iv>CSHYWc{C0YWXhal-k&Z4Nr|*R}ZVv0?+R9Pu#fK4yA7{n==v$1=1G_r*Z}& zPky{jW-Kl28OC$_k2clb~ii?^JT844cLPcf7WdK)VID5+$7je!Yzri`kd=Y$@y)=b8F1!P^4QP{w$hNd-I8Ep{)5#*T=kQcb9!HsT&6UA^twG+M*01YpwpqkAl*p9aH2B%N9&LNAv|)LJzrc(UL*Ei#_-vm`i41TC|jr> zb?^TU&)Px%!t-WbKq`di_k^FHl-=7Ua+VqhV|OYP*pxKAW@J|UAWk64|3$W-&8G*A zV{e&n<~888Wap528lp0f8P?Gil?AH#P)@JD9Pvx*2r{SIeGYI!pe?*EAajZ`?4rJ7FJ8dmQ0P#U;u(aqzH16}|Ze+e5f-B@x2xG5keND~f?*dFC@7w)}<*zb_*qIT^`zOQ<@ zvv`5|&lrbaE2+$##*d5pc?KQ(h+B_nhzEqXwNF>=!;Ezr=0sPx&f zly??cG@dwgNyP~ib?Vjq&Y)4W!(rsRjt9^qi15(81r-V-Hfq&%Jcm0|G4wOZ-F3~h zr-WnqAACMYjor&+Fi_)xq2B8`rL0wMTCmgbKSwCfTxd)4#X!H1TA? z*%Rx27ADy>5Eak;Zk0S<{^^I9j^!k=GQvHvZslJ88FPLgKWG^fPF%gvs)(fi<=4;% zv7-82C17b2J%cS6HTOrt!*{H}){1KvkHbHy?zOE$jD#ojP^4e%ZhVkiG8_+FpV1rl zh&1w~eeGlKfNE`{wV7=GRWzpft8uv_l1(*#>Y+d%faWy_Mf}uHdc>;n&|F_yV8TVVLAH+jtX96>^rYogJ7u*6Q#3# z1?8AT0o0;})7Vgr)!!wuDt7&mVwM+W)$Bbn$lGeRE~!dl5g#MX!3ZHoY33& z?gKjDJ9M<+$SrXQbW5ZuWsLf-m2R6jG}pA5~hC^*Fm7c8-Da|0Z8=cnQE`PyXa((;z4mRBXR$ zL^F>^X$p1}IGBrKyNpH>%`?K1x4{r@>`%d6t|zbRWh_&?TxW--M`D)=%0Bqlj~o{G zTh9E9jx_@;5XpPq9r+X#Ful=}?_r$oq4i|LNWl(qGt5P)4BM7#SK(OTpe&;v3W=>x zo#m2)k-v$b>P5OPKA!e$_~#H|NfJR)*Xhm6OC7=<52@|E{>`27jr|vwbArg@0r>bz zdYnK}Z-?}1_bly{TTjInb$pUVzg9WRzqUM5vwA=zd~suI6h+L_}hGWtyGPr)p&-xw(`?7y*=ng0#)v=Ggi*1l~Kmd zpD8KPpu>Lt`x{!(&y2GL1a`fOsU7X|cAB6*Y!x0)KTLiK?>SvAD4NUkQ5I9Y^SW?4 z0|t4U$gm$&Y@vB}WIfAZYfje6{2}5V>Z`EbzL&mQnK+YN+=F4NSdZhTfN6_!x&am43*8l*0@*rpQOOW$dJzTzd!%%#-D=q z)O4(X8k!rr;~oGMOAVY1P)dUBME62yry<|Nf3dpqc#+MJ_1@ICG31%M8oY zW9AdXiK1yMGL$?aY2I(jLDgZFzlgo!kOeHP!n4j2t=Fm8hTJwKqqu4Pb^Lp%#781{ z{~PZLZaDo^uKASRjq`w(L~PR2i-CR!(0M^euQ z`|2G)-Ey8L@Arp>7ka(^d!-wgXjy-QHosTJGl73sno3w7jU;(?!2{FKtpq3t3jEsM z%u|6M=~z|X``IPc%Zlbs4>yic~^@&C%>{DUJU#`^sqth>tZ7&KKYOY3Ta z{i>Nx-~Vn2K)gUcbV_N+)BNw-CCSFD{E`DQd}c<0)|!>lRU~x`%VQG}4A+A8YTmiO zFZR&>d!vZz_I0EF`@Vz!RaLtUkt*{i{7V+1msf_t;NQ35Gx!?)LqH_fpX*KiyDjhq?1&OUz5V+p5BpJ;#UZ z-}{sNZxIcq-h8?6-{3>Q2v~gLhnC|1Bdk>-FkyD{31#Uc*w1f)eJPLQs7^!otHxJ; zOm{(dLi!@!QS~(lak5F@zWfVuk$;bj=&2J~|K~3R=I5Ufw>G`}-&KsMJ%KUy;O~3= zy-fZRc!4#g^&ZAxb;b;!rI&0zSz)Pa$ZwuPssO?Ny!OGrD+D}^-_e!m`8h6XLb1yp zFu~ja0`_>-r4OQRh(Fm98KfS=6T;Zj09OtPCa_H4_8JDQ137>?kl`;G?(ew-Rsc0TUOv8W0)&wz*APhM1$nQN(5+wr&=1<3$Hu*eo zNEY#~ONB5FI22HWfO9d7K?>rru&msE6elyDce8TvYWO{z;D)>13hR=-u z$BA9`fmW!S5f?k07Qm4##FxH)xYlM7sR$Myr-unUSFccG^{?W=79~#&%QT-R*~R`pgZ_p{B+fkAUTI- z*cpl&4dBSiG~~P%Fc;N{PVre;9-I6Ty$a6S=>Zo&Am2xeQM35Dl-)d(1WTwX|t5sm4*n8_%L?0@Z>8f#yWg>_p<*hzS<24 z`b2T8FmSxZ{wO~Wu=^HP1d|^Fa~4?*DNEe5&#I(^4EG@UioHFX-v?LK^j_|5_$7xJ zn)sqz^ozILm?4SU?Nw#MX?j&LOO;EOJQ~?28Qxd}d)w>s+P{#<>+vU*DI=Yv2R8hQ zls0B4-L#9X>inl`ik}`0jw^nYW~>jQCijgxkMV*v1}H_XV1z9JY(ERlz%dq_Wtd^? z&w4I`)FNL}2;2sLfjV#@lSY}NlJe)riM118{v85pfL{fJS||yIm>)zYa6d#gSn{}Y zh#ejs?qxONtdy2}UoAIo|70TT5bkq_$7b@ugg&(mvCR@FSTr^X$?{ctckn9Ig^YvwW|>->Q9^Gu)Z#gzW!X$Pxa=Bb*?DR-`yU}ZWJK+lc@UCzt(4! zg4o5D0~Sf2YJY(03+aB%qAeATktR4L_1m-vDx@kfQe_C$s3MoT&f^clvbqP3F6Kp6 zw6xJq!-XU!vM%iqw~jS=1W@vv(tLh*eW<`xh0Cd+zRo?E zUgDI{TxU8nP`6YxG{9&D{LrO8E)15H^zA>8B>Q|Eo#+1-eis|Ks0ewZVRsn@5@D9k z&zLB)i;!(o!FpMEsCyZ%;_qRN$JHN_2V_eGbqp?>!axkcNlZkpnvNFMuf9v+Nzy7I zE`CkU^zZ_)tntFBv4u}I3PaZt?0*?$|BqpQ_&1T+G92V&C*+N-OjS${LsNK8mb>gd z$t)<#sDcGII4vR&6tSz{7$!g1`$9!=o)P?B6v2vgIKmyl2+nXrjZUS>^(beLTVU^X z$7%C3UmF%P<#e`w6WPvU$rn|^5~9~?**0w!$peV_J9VoNS3M;lZCgg?J|&TC#zZ8Wb>qTz3;W6D zr|A%K{x@WSi!E55CuhAj`9tlA40ZD9ZH&PUvPx4B0)Xn>_XTr9uEO(A} zrl@>Bf;8jL|@pOB# z3S0AmNB0J(Q5J379f}cl)8wl9>aO?cK@>x^QZ&t9Edao=XRTnCn$hrQ4A`CN@2u2Q zE2+1Cl2NL#Jp{#%rmTv?G_NhpSlK{{J!{Ysvuw=6--rA1V`lSs@P`f;W zfFe9euJ_fbjqWZbwbMM83c`|nlB>4glb1Rg8%z|hE%X*Bx_GJV0ZZbV+}69CB7v|K zkmXF{HfRd}u_3wE@|HE?UHVnMxvmV8MXj&T?&iQ&{PeUXZs4timbPZqX<=?$ZCCB< z30L+rAAjuL!PGRwn*VZsqO6^ot`K(&%7;P;YN@LL3iP zmCTe_VF;CHMP}GS@JIrYubqe0-!D?RLXusiPG42l1lW3ChfF(>qn!Cqczuo`2e z#+ds>Et0IYMr|96NZ8=c%5&pt63@9CE?-@wpwadrxMG|$?DONy3KtIEH})s_536lr ziHvClyems|YP|WLxQGyxgL#YXS*{Az>3{~ z4|dDN3z?^D5lhHLbIbVsy!o_&2BZX)-@EyC{=n8X3p2kMh!A8^{PZBlkpW!5vSA$i z5lt?}yh{3}b|n@MwxHM&49Up5c=A#nbli2a^ht3YEOlVvvz_hmzv;DZB#gI9#rOQ- zsDq8BhWBBu!7HShC@KM8Ykefq2cbZD=^@i(Rl?mD9qOpXqN$GL%+v9a$)et8u@;vG;JkLv@qw?jKK!c>I;`TsLGho;4xRdASxZ>*G;hOhyEsxcMk&@E}Z;H2`M(}Mt z4Sgy_?Hl%xY_&9n#3 zp=QC$sXYBV@zfqAC)qHTP@{d!E$!cA4KK$KLxf0?xlF?S!r8 zVs5z?(8~(=))rjYZGfSj0MI`6aU4j%^v#*SP&6Fgy8(4V7o`>^9@VEl9ON$lWf3%9 z;XhAFSQ`LE!q*i&3s_N@?5EvgYR~S72VSss$n#Q8k|yVv7cgs8X`lA&QF!gP)oHDV zvm9lj1KHXgztcm`-b+SN!3+sL4_$k@+HWi**I3Ulv7RFIS{AoS4i!*Uyl*k46=!D5 zpynNBy}N(=j^m{Nl{+q~ftTW)0yRkpmd-CWhdk-+`pH_*q<^y(m&IS0&b<;CyZYb! zFE)l6fb9+au4OplI;zAgn{ZH+_K7D?lO!QAm&?(?RvJGsm#PaP;^%?S)O*0>(+=Nq zXlbrawJcrcV_R&s_YI)Qm5vuGo1@@>&w{}+(krn2{9eMGudME6+s)g*2(<&sEC%@W ziH46{y};g1k+7Wh)8?1ME;{`W#=ZHe(cRC`H2{h$>kID9=QZA|JxjI;W~~q7NI~y7 zt#+5C%}qF))@mM|)7ZpQJ4xJ4PlG@-tib!1Ys0a~C&zK{eQUqipNcmO2hU#Ctzht* zAjI2*hE;G^hFZz!0>Sjb4Ed?9>gTA4t$yTbWi$=e`3kC*1(>=DzirwDQm*Sr?!df~2BTW_ay2sE#RMocB z9TNi{k;bmV?%6F-YE`|4Xi z0xu{1VpHdoyMZ@s&e?T}Zke{Wftbfk&C z6vtu90@V+JG8yPA!#ks$f$oFxu9z~4aHs7P{6IbWIo5e$ym9a{QyJV1zWyaa>jOSZ zf=}Tt_-OXZe8`-uxrc{}@Ts@?tfG6mmv~gNcmZYC*e&K^~yy~4mHMR-%pQ!;c zasn$}tYtemYbV$6TYgXq{V-lHh>PYGOaK)+zTSnKLU`-AC(k@{XzPAS&2RSrx)h(e zr7P9S&n@3L8VeF;k=uw=L!+7jH6o7sbJl%bxqSm>2;Y8%#S(;YGsl$kCGK%>pl#M3 zsJtzc`wW4JxhvSPLW*Vt((@n#88{4M#`ZSRC)S;ZT%9tPfeJW$JUm*ccA=OxraH2g z7lIpKf7O9 zE^w7XdoF}SOQ^F{_;6d4jQz<6v06|G$~C*e9R;4)x4p3(4t6(i_$fsynPlFjls=&u zDzUfpa7QptArmw{KjzwShUn91u(^JU8fNB{Djr5nwt-#j_pocBXWGV6&(1vqUly>}DnULbgcBnnz14g)wr%0(pd z#hsUYBYsr%xpdlkc-)fOHIt!oLaCzyZo^`@F=pC>Z$_f-4b9~&Gkp>%)+U_(k<4-C z3hzT6sh(d@j3`E9WgAqZtDVXQeza$LqJVxfjQR(*^1C)>dth?l-i4%&JzGm|>IJ(! z#aEadYcKERH7Ocv%sPz8c`7GaIdgn$cH4w4^&OD6uDO)N31ae{-&A1Z#xMyv^&0vEctIG? zF3?|C5=lo%SrX*Xyu{ZjW9O}N=o2JVM|$0~a2(gqim;+&zB#k2<}$p$BuUfux04|E zJT#t4HXmMoLbO-v1zAJuZ32?XVQsU%Mx)z(##y#D}^|yvPM$ z5_xYT>det*PaD+y&gd(Su&q^upkvC^@_N9JkHih8OPgrPQ%keiStI6D(MQ7zgI7lB z5(;O@8&9&Dg=}0X6Gd0Ra3iBKz9|1)n^m^c4pkS-k<051o&^N5-u^PkAZH&UYO%GE5S1ryxIg4Y8uTK;zDKZ$RV;pimymjKABHqYDEH+av}}04)eW z&~sAkwsb)0^riEK>d7Pw)9rsoenbKn^d;;(t zOGfR|pDSA_<7)H7Dh)pT_A=caff(I;y{nO_85D9#{AQ!t!hPx74~v>*wC|R7c9ic} zZKlbU-nscP6nd!9(z>-qj z{74((Zef98OPqAV8ICzPNR>24er({0B8j4{cGqX5mxC0e0x6>3jvOGk@60o|1msO_ zqBKoM9J5cX7i)gjgE*Y(H4kpaAy7}JOB7SB?1)f%TA6tvk3k9E(Op=^ z_C#<-FhV+es5T4xsgWfrc)r%CaLAyFmBg3fxE)w|L36sM4!2Chsziw|oi`mjJNhS! zWy)x^Jo0F%Gy0c(2lpN4-Q4k(;Cn&~Pnf)?9F}922ku6d&8C>=2ldCR@p1cJQ+Ua| zOI~L|>bu!=;j5_7taZe~jRmF-oZYG2&2S8YcV5toN=r7%=$d~4Yl6?4;rjUS@4IIy zCzgZht!G9V8Ee^<^L+(&jZDA8cxgA2Y#g7=^_U86U-HW^d;YTX{`}5qvPS>6>3D-V zc?REEI@6Sc>k&?3zbMQ*M+!64^fzN1yd|~;4Af>NB_FuK|(|A7|B!gw7$E|0fE$Sg;AvRS8Q=8f~ zSUA66xcR=Y^}PVmLF&@~ZKtCrh4v_W@U$iFW@|!UD~k$aDoygj=~RG=Us#5w za}DB(kn`8vnr0Dmn+bq%LZC=aZ4bN5V`1j{!K?!fiXcJ7kZI9r4WSm?5u_kEB~{s{L3z{^8u z1Lo?~oGdtmnC;(pGo2DPnM1Y|uFX)tmIj&ZId>n&&(sLF{sW_t?F+3*LZ5GEJ7Ae6 zx*qHnl^)G~Sq26+%|5~3?k$LK`m!?t>np!6Jyj;vfV<({`xEV&m7p8ZiIf_=bjsei_RMpZm)4(`!;*GZe{RjML0FS zEpTKD5*kzvq|_wZ6C6_U>(+AKQ37dsx7YbVlD&?}Q~l$49sEg-oR~Zd>$e77+m{y? z2iEw+)_^)6M~la|7^N{Fyg5z(tIX68!y2Bju?+4lafW8|2@#|8Z0iYWMo({=E4pXX z-6OV%QKH&eWyZ75iAzLjf+}7te=^?KB1W8{C2NejnT$RkS$UJ0n+0?zL>#p9Srog8 zM5#UJFeuPs*_I|1aJ((;{Ls*vobm4_Up681AQf9t#-_9LEnG>(6&pf7bJs42LMFdJ zQR&ko&Bm^Ed2Bnk#0i4Ugl5);=$k-NY|iU(Re|n8Uk*k4gQo0oCcO+Pvvsf!MEO7C zOcIoOI#TjDpZDCQeY|$&l1;OcuMe>y8) z8}nZHxnl*siiKV?9~B`bl9Jmjg&RGSsrP~ruLErqaYJ8vroSk!_gQK@J_`(wnW0^R zgmzQNS=>;9|lrBPa${V(*y=x}Q zE?xwP_f8$y(QTztWE*36I5Fv{D(L?7t$^zI{|KHBx;Fu z`(1QHkV8OwO?GNN)XHf=z7l=V?@8>^`x*qW>%7DAi!$?bfNMp-(n|l418pN7;ZY2z z6#2J*Ie_>?>i2^Qn-nUx3eYBAVXjRaKnNBY6~@19OVT?vZCn_AxV;>LM;8NtCjqs# zT@aoK^0)&mxUgLwyVSR#KFkcivHw`+&Pi@m*>hEh2<8b~Hu!pAGQcP8-(EmZOKx(s7MlQ*unjp0$ zDnDKimCvDe^~HWxyE(^@vhhJcJI*dfeYg91?9;CKq8nzll2Ru7ApwZn>e3fD7L`B1 zj@r*6YcLcQQgM(DXQQ1svx2%dMtX0gE+G2p_g|JBc~|b_HoZ1hWD}oxg4Fs({CJ*I zAW&356EreJ&uEA{e2*7)^Vruz4cxngK02}E(r35Rdli_`0;@1HHT!k1Grhaf`$(+g z?x*(WQy?^AvK}g1Fq*%gj{^6{UW^RBm`&aM08nJJ>=Ay*9$KvU!Df}9fer8!ZLa;e^>NU2dw}+fsz6a{ zory`sU15i z(-0UMK~r3BaFVkjnkVat)-d8I;yoL>Oipkk>{ak&;$A?mJo)4)ZsHn5J+H370v?=k z`q~;Ns?N-hv#I7@N{+nv@C+x>D@s${+86ho38r&0LA~R3ukRVPE4GQU&Gw&PX3JI^ zpGN!sz@U4Ff6%?s1GmeEA9`>}B-P>Qs*eX?&uM+>+`?ugjw7fl%`LGw-G_tl4nB0P zJntUw$a2f6y-n?XbDb!LH<6hANVIWjc+bY!GEJb>r|D7hGtbh9k=v4? z)3p)dUw4O>SJSha-W3R^n~9!u(M4^QNRTHA>3p`E?hro*N(K67mKLeqw#%cnDgDS` zwkqD0TTFqnI-EjF)@sdnMme%RKy~|^M=;OQ>utT0I%fqa>COuHh?c$=Z(rWWxl8QD zbdmLmQ(L#@bQ_+p4b62A;AP;vHt+kA>UtMFnnA^~;wH@LjbS2@P~7cboVh5U{L)?w zW$k0=N+8gTXwf5W)l2*eiIUP8?z6}gy>pkyW1bEd&CmdnsC}-5OM7A_7O_wn(a0YV zKuMsCv-%aJFsd|Nr&mrk;lDnMXMbr;3qOAk5M>3ZF|s|74}$V!^W2VHcNPkVI(PxT z9Flo+q74oFgA^~Y{2;#KV1ceQxES-;eaBb3(@lfCRyLBEK<3>ozc#Yiz<37Ye>f$* zZD5|3Uj)`+kXC`oTfQm+%}O1WIV>(0z#{v(WoHec#vjC6ZwTTW_Cp{l!xjNmBr?V#zuG>0{37g?3 zSBit;G<#Rh;Mir0U*gnuH>L7$`iz@Ri(~2sYz#V$N&S4+y=+lhq)$c`uGt_!n5m&(n<))9Y@Bs7@F*!vvAr;3 ziR%|(I%CVu6;8k6GoP`;uWO{?&HIXK5a;MAp5(^TqfeW<@+v^Mv2yY(t6cbF#o?_p zwzK6keDc&^Ie~{I2DF>wX>w3k)6>i$XTXA8v}7Mm0Li4~-1**xLatvH)EU}&M6TplwXlLrtsv0O$O%kXcI3=$`6E%eM6v8J>q7 zUSgtI{@)s|94%w`mgkYx$kl}S&32urFZ0Q%|fD6S@gKeKUR&k2-GTZTH|CrS$HU zX;~`%21U6qTMym~v{tkpiC;W&9uivBZkn_(W|E6#8Y_=Xs9hmFL$u-W0MTJveO0u1 z?E`I>un`J(T{%eB^spmtWMzvGXBUlg7K9yGn$!KRi>gDoyHK#lbol76J08M^8qrd2?>;W6*+*c@7hXfn6ywC9zAU;T`rD`S@oQY!&jPs(bII`4BRhTHfAft)`h z713vO7;kt^6Vf>m6u_B(%|IvXXV8rtdEB^L8J?SLL54Ym(^ZCT@25#G52p^L2`W$A z48)^*35jB)TMOMs%foe1n*`!|YLf@|aYjV3;mU2Ciq7tL2TAxgy?Kxit0dRyGk}+j3SWn zqk9S2nk0qP<_nNV2IN6|O!WWpiepmwTE`IMFWfR?hN}rYRRp6G0nvry-&WijsPQ=@ zz4O28*WR39XrgM1EC11XF0MkQyzdIT5NEwpiN!-&pLKI3-ny|04)ut*8rRUZqro?` z_WHpph(fZ`x`t^9m=P6aIQm2z-aWE&h0_qe-%d+!jI+g5IZand4cho1{3Xrp%8T2qAm#Y+V$?A#MnKro^ z@3VRWTd$98Y=u`R4T+N~L%(v!pTZ*tEK1(5g z1SJSvv3&Iq@T5y3;n1igo)k?88osuw9IWZe5ay2PtWmt?C5SG?a-D?SUyz%wGM$m^ z4h&N&*^1=rookBVAiuP?ynkwGWw09h*7`%`E?lzgk8^ZiuDbgbbtK%y&C+EQw3l}^ z!*LYw*^>9IgwpwAt&p!Cv_b!HYoQH#W2y&3UfILb?t?l>tw%EZV+Ky`8cb#=MAPqW z%o)D*vI1%pJkBGL6?23e#dou`ph1Ykbg=}JRajzj2uf)(APZwAhG0pciD>yT&tM8; ziRIy3Q_PPKGh;Z3uH5{Cp{6p4B&{{1QGvc&??zLRe?iS3xqr~8HeA8ScRL21#+5fz z%O&qQ6pCKcJ6EhI;Bxp*z#Nr$v^6WUg->R0tjHKD@4dBSM$un(=dz-jW^BIbmwj7V zYkgV9_$@Zp83pFNtF#vKa1QNlE=9;{6th}=?pfO{-O>6!oLKYd<16xJMa@-R(<<8< z){!_tr>9GUF1g_8#5g~&yT@I%*b8{#_YNsi-KT~03GsKF0?&Xr$HWFxLWT84+AX7w zM;|c*+3QZ_S?SkN;g;yHQGBysESC&F6s`2DMcy_IL}qI>E>8UqFHn zwa{<*np&)fVEK&eQrXt-71p=g71T!L0?lKRIq)V~X$y9Cz__^cYjuerHv#I9DiBg* zS6Ernpv#fv?-lWlPTXJ8Vhj(fOteuwqWoOAJHq89C)GmHvk}i-0h`I!t4N}jNH7QL zlzA9mMckUDQKIb7xjw0ZIO>LOk=TI{NGB-Aat?Q0ZT|s1PsZg{qM$h7LOd62)3tAA zjxic$2LZIB%|6RcDZ;Bn6E^^B!i;Vtl2v=XE5dL)QY1V#VD5_756esgZO{8XRsGIf zXkGS*Ro7=#lXktnFKyATu^sqdULG^uX(_({XGX z6A5xSX4rp)r|=^iy65OieW?t*_`#xK=xs-1r!7XZ#n zhdmH+(<8ygI!!X8{e&07`M9OKmLuxm8u07D*6?8IhJVz6}tp|?AVCfvU(dS>NQ_~ogEc0Ib zXPw4VbP|E{Hw-7A^Gk_4skXeKeSzVbn8pF*s0!kssRFSz`~Wc!ptapOcxb)qhwk_e zkIV>~_vTyZFK`P?i?YRnlG4T%PfNVAoC( z9AOxUrT_GVm6x80eD}c!(!N1n#lUflE?viyoRm6{mEA(37vzdjKXPGkWaG2M)^(6Hy*}$$n_`9GDA<}abNqTu zP07x(`6O5r;g#o(AaFxlOM5#Ps8|595VG^DY0g|)^3(dY{^`u4HEPE9!cH^AY62EF zKHjHwQM(xIbpNTflilX1t;1fEu>Mrys}MG05_%b(R3|yR!doj2q|xq&rL>$D?VD0n zhJTW)1^~dv3wQ<fTUf0P0fX@Q67pJ)d&uZq9A5bk;6lw^=**+h#* zi3m%=Q27DT%*q4H&U>g?Sv}uEmlFhs!Ux-Kh6|xX0LAGJSl5d(@OhXQn%D?H7<2I zgeFHWKzbW!wm01M-Dyhqhwl#F#AHzze;~!dE!y5L>;+cv`9@BgV`LhKV8*0Z-F=ld zvL0>F5~tx8^UMUxjtisOV6IN^$sIA4XV!CLLRE;nWtmqA%Uq@hF-3rKN2+N$HOPO_?pGT`ViLV)o;XnTGySXjjR*6xAUbRkR>}^+3MMmyvH!lM-oGu zmkIZq0q~|6p1?Dt3$1x$$EO)4EiX|^5_yK3L7)FYhd#lF6PmdtBU|iq{UL)IhhMb2 z7CgS4tzF9@Twu@jVq@t)v5r+}yrt^kHfd4hyzvRseCzH0!5qLk!HHKSrLGSAp1cFg z9-H-$D}pxS)`l_}rQcO5SYMH}&3Vat6T8jSh2EH`7qvbdq6}eDy5X<`t&N<=53F*3 zl2XS*@+mI_lZvE*t}9Fd)XdChbWj!89BFA8cbfdm~YkYV(Z@#z5t2uS-HhCE&x8GzC3a(Uxs zJ>$3EGu;sjKoL)5c@Li6@8yKESwgKCi2eQ&Z0N0A^W-ph@b6V7o`TOl)&~$@|(V6N!f#*(eq4cddyrfCs!7(y z|%rdHCGICyc zB}+Q)rXq_&hB@K27}kuec`Mh&`$)I7Ntv0Bk0tYc6fC-U}4sR|5lXT5#PJxA^e`Ox~dL|yy1%1acq4m8QoQ14;a zypbx`0NOYSSFiE!A(H7P_R`V44kva<=^$7porS5Pkf}YQN245AT-NMxt{n>Fp%xHY)=Jqx=$1E}1Sf1TN zFNh|2U;35_8J`1nkW53s1Bk_;AkC4!4@p`Wfc@#FP$E~$0&)75Q7iJB>H+8JSm;@@ zLt{Qq+JM4vkN~UgcInUjM8VFoOi6XFBegoy^h{C&D#-o!|u0Z|qH_lWq8VByF zqOpKmJ`sSHRq`1xOc)rITzH{NX9Bgni2rUV^IR&uV*_oM{RhtKL z=d#J?G}Y2l-Q!}`jL#~%8O0gdoM2{ZjU>2=bAhZi&%K;0!wJDks*Ur&osbiosS+>J z-4(BS!n=I+;3bfpeo#jk{O>6&Qll$)ycU*M)|!)^{`cD{LwL=M7Cg7be>E$H;5d>{ zv-mqI+JDyF8|T$(lPTAYDt43$+?ZUH9D0Md-Y8JVFrnMCh&Rvo7RUPkV(YD=qH4eQ zVL_0RE~QJ57Lji0E@7lgx;uvuq`MUu8U&P(ZfWTb>F#a@hItP@_5G~h`;QBUGpuvY zeeb&Vb?rO3%<@NNg#`Taur4=Ko*0fqeF;S&lnhlXe9@L zW?|33&iGR%MEY!Qv>Mm?h)h9GytueHH7a3`W5bM(o1vkoon2kMORhjiNCP(1=(K?j zuw&Rm#1U=qjIHH4I5>E{&c6yu(s(#gO*1gAS^WO-!p6oX&$^+cg!u_3NnYibVx;r+ z24&~9)zzE}FQ*s=Vo$Gm)F_Z%s$%Iq-7do@H{YuI^9u=p&9$HEGFNyPjtKp5(is5k z!d`ioK@7(io9k*v%gc}O(n=V1MTCU>@%WCd@rO0Je4&k!3bF%Af|h~E&6+7+1;Y+^ zVhj}}uXbIF%L22qsDLAeHDFbO1%pHYF{2Z(j16xlantJj?QIJ(5no+l-0WfoDvQSC zvs6)Wsw`BMDj1dp>RQRpGXgA@<z*wjQL^NB?daU@xWiV}hgSNXLuUDqjEDLx2Bv zTi$=o4IJj^`LM74`-Y(p;wD}#so(k+)_A)gb#-+LqGg_aK8tZ4_a%}`U#i3tGQH-% zNuFb>ik2yO1UMuvInyHxDACbgYO|FV5@%(79WaITz2~X+FO1;&*tGwuJQoETT;DM( z6MZH}nKbOuaoeIHZ6y-Y%{8+_(#e^YXuOCdU z^eE@@0!&Qk)YQ}?^elAL+-NznRSH-abei-=4e>G-;FDF!onOFs!0c>`rIuzVZFIQo zQzaTOGc&UbFyQO81|bg0uCar=yZiakl7V<^slMiP1x+G-j}xAkx;LQyd4#s>`a%!U zi3IK5*)Oyg5(+yz2vrCHco#w} zEDHR;)jjbB)nR>}OzKJg_W(UTG%6~I6xD(j0dO%hf4i8+U=y8NLH+-`1p>a{ze*v6 z_FWO%UKLsild$#v-|g`eiG&^j*iM^Gr(gfit^qzKLU0b7Q&dbE2;G3cefoY=(tel1 z(lQhITs!OCFRNdOP;~Jm`|(PQxMV#fP)LD;WBHh7C!Hr3;fVSc5zC!+_;OOTCNPug zKBug;i@go8h zDy393nU<+u1Je400e_)GK6HP7KQA<6NHvo&&LUm+W+&Y%$1xs5=tK?gpqNHK(V{tS z+Z&K?oPs40giImAU~TjRQ-qU1?k6Lu1S`B~S7G>kHP-kesZO1}Mt)@fM;AoR~yW$P-N zvw+5TMNws}izVMHp?iYTgYd7PJDY%$GiA%{_nu z^-7yXKI3?cUR|xHtaQFEr}XhQ{Pv^FyaLZ;CLAeffC>AVznFT_}Z+)1RF9DYmMq9STM?mx1DS9%C7q(AJLlYYx zpQCNx8Od$$f~Rbh1)9whB}I2Z+qN|62JnS^@%!!3-27vG9i8+ZB$C$%lK$<8UPL22 z>#jp4jU0$w$S+>@v3|_2M*cE8DU8)vq5rO>;bqyguA8nL<>VauDK)C5wCPkNIp~20O_2k>n`L_ zq2cku^WVc;1ZkvH42>Q+;F*l4T7H`_cgied8+{)}7?t!sZqHY-RK1KxM8u#*aeiT8 z;i3fsRfxV>e{{6DX%^oqStaWa6~#KOLh!$ybZN0%L)Mcod*i$Meq4eS$Ko-Ptv1!D z5a1Y}q5YeCmBPP7)>q{b#FdzgbdDji+N0yelo$1CV3rKzM>&RGuL9z3Zt+@k{?x%` zrF$2wt^4S=)3OJ+Z=E~B9?~^KIz;5>jG+tX0=nV?ZI$YNwAFosuMo!Z{&ufKBWZ3z zsU~}5wU5ba)7JXdg@Jc#D2E9ACtkSLmg(V#aKTy~b>Ii*dHwO~_6@+}S_nQQd4kS^ zw5$wwWZF$&HSUY^jUee!L^z{2~gK13*#6Er_Qi?2e~BfBi*alAYW>IA(;;H1go zoxldPLi|%{s?&AM3O_B72oTUQwzWjifFscJ~na(Clg zY7lR)`FN}kKeDH#vATi(V;^siwxJG91_^Ng-nUTYR{^ASv7q<6toSZzhcM%?_Qs~eSPj@>O-MhI`3y^jIO7>JL#FvJo%+f zGSf((`>EOyl~T+HeY<{n;)0J-7gSZ|7AWE9%ceFgNp-M__6C>7(^CDDPMTeL`H%Xz z($tOWf46X@E+^jr!3priMs z&-J$*G>7>iM}Z}K9@ti_w~7$p7d^&=7H}wAqu z9zWnRGg?U^&-ao{BgA;%C34Z+HL2h^S`9+W^Q|#9vaa5vf{t7;Z##uAEZB3MuFL3X z;&#NR6NcJ+_L;OahIu4Y^t1@ZTeIv+kjKz5HbwDb)mcrO1>!_Bg}+W+kw3h4U`Z|_>w+?0LZ$wh1z`E_@*J4Q>Xbo%Z%uN( zc0r4t!&IC&zi^z_hmobq&+CJg_gaDoMp6CJJdPqGOz23O%Ajr(J97>@`W~yeXuP&*_pPuY;uHGVq|A`3sr!Jt92&tH zXDWV_CmnC+f#JTLVmAEVPr>4|s|ayrd9airdB^5ceb|MB&r>D3PQwrHi6i$W0N6#Frf09 zyh0`ODg5OJT6>K*c$DEK5`N}1avI%xyu}Hj(H6=`n6Bv8SOj%#Sr#2V0j~ zBLW`Fx?Q)G4_8KBUR+5D>+bHZn9wLyy|tcQK{VqL>~~a5Oh`loMtXYsgz^aozJ<#_ z4$J;qm*4KzSyoR~{5Bse|6^5alQ8Fx@3LJ$dE`MpT~qAQ=5ck3mKE$isojZ)Gs6X6znX+uznbV3|%0-BCHsYjKo z_*|oom3^Zcrs;ENbT)76`)KML1*Wl_*Q&ZIPp1vdgBD29r;~?BzP~g%-d_$#Hu;EC z%dC2uQ*q@}kwC_vC+bJZe(x;%dN{KGqcf-U8`ABV0^|K#5#eI$ERoBF=j&*P!v-HKr`S`B)HX@wW3CT&S%;0S2vQ3nC%=71vb`C0ygX%b zR9OExz|M^mQRLEzcHfIQPHXzSFQ3bS9`U7R(LO5s1|nq`#% z$pMlp9Nqmsp5U)3s?jO?OTmF0#K(Qdb2#gZ4m@(1unm*-d|k0Q6#anjA(^jzZoIrs zykGuOklgdu_iv8b#35(_a*8UQcX)^DvZKr!debH)?e>!!UZ{5;vpIssk(+F2OAl5@naT}LK{KP(lq zm1g4{p{i~N_oh`U-^zZBjiM*NS|q%cHef%myg(4GUi^#}A_!OZoGPP>R^>XO0SPNn+YLm>O$KtELIu)cl*J&DBgfBpUfJ9G40HRooX z9{h-etk&8!&{^bl;{M9a-v>G}N>-fI{avN7lNAc+YPf zsiBoPG*UX$QUmdbc22zR=ng*@udwA$8Uka;UvKNmGQRDp7TpRnA~}#M4eJV{9709# z48z&{-Dh3oqi-d=FVRff^6U|_w;}>-SevYb9V^l}0?a4QhoA6yO67-UAd;U}uLW@* zA05!{={a6`z2|#xG+)-iH9I8R*R}XNB#!7awd7_gt6pp-Bxaz_Mg>42=M-6(G48pT zaP7~@#ZP~~+lMU!B8=0GXSzWSkZ$=LcMN8Xr#@vV1SiY~IHV+jnKM~9BR#|+n-nPu z7M$%P%C`Z}h&Kt0GM{1Cnw8^#3n~cAh-uVOe}$Qpz8BfWI$!ToI@~9xungpO*Ce`Q ziY#K+Cl^v722Z`{!Au58_tAGwkAN&i!xSHz-e}kxcl-Rk zPE`Zf`@e!8kB15kH=pj1j93fK&Z5N+-1bwmp~G^OB|fzdxtK1!cAiF_U)Hj^cJ?NL zU-v+9pHm{Fy|Bf~h`xVNA!Min^BDd27sQ=w4Nk_m&;IJZm|mGdzxyM(HHTX0A^y1fClBE#km#XPcLOK4 z0+y=}@VIS)k+-gjt|%bRu~*Iql-u+{S1}ipcjOnN!KQV|NdYl-lm;7{Tfno^o!zr` zI2gnrOmb=MoppI?{Sv06_RS=oZot}jYwd0@gPep>lytYNuM8F;PpQ58gcYMKy}T^& zuIo%Kq5PoU=oCB`qAwmXid`~97~$G<(<@wlhW^t(2A*^?9dp#49Yfzyc&p+9wk$2S zku?(L7My(yA-`?ss={Y0eNT7qTNfIot^X@LhQNe94)L-EyM?JNy9s0*0Nl#;ZW{+)wb-JHle(-UVHQr>2Cd- z@OcS2iQPW&Jrps{_*+Ki4XQ8LE`FLEzKdD!>90FSeivwTuo@8gRsCgOe=ulTZ2P{I z{oMtShqe+xLzLl?;a3_Aonn@jXf;9tPbKTMo;4UY&Wvc*JYSz3CFgB1L=O*gSZr2w ze?!>HtTs$3Wn{+R-rX9OwQ^MqQr~cNd;7+%g&V7-SvtGlInyF>;cTecSX`4On059$ z38$?L@7XvCs<#LFml!XVTpw{?6E>dnq$74rEm0s}x?zwFtAEH@^|T24zAV@2QL9}r z_Z&Q=u4TBkK7y?z%5GEb-qcJvtyDmjRmSesG$MPM zQQGKrO%l|HC9KD#SCquh+_Ee11g~)%b9<^5xuenfGu7u^LD(E}`VEZ6!shYBru6g2 zdV(wxd24{?8ioi6Nz4YSy?7z*Kv z?E4?naAvV#Rb|B#{8WWE8ZkD>X;^kpyK;iVDap)gv~7HB)VAAOTn7x-wyY~}>qyny z1j5A;!7=k72$iVP*&FvNEO$i_Ivz;momJNSw=CTPd(VvqQm{~_6QNGd@Vo_A@m|Q2 zknC!lzTLD%gDc`+o{-KRI;TPj#4DiclI?Mo{GfCrLOBy`Nc{ubev~8`UXL0pE`YYz zI||-p#r`Q38l3dTdLgrprgLK$J-~-F^G6}^hmEb3wW$6LZ7=I{J(AK>ID|L!qJ0vq zQi4@iXjo)!#PkyeC0QEq@y9|FwU+hTJc;3~g zH7~ykH`Xjc?kTl_m3TC z7HR43+3fsP{4NRJNJ1TiY^k~7spwaSA_fF+#j!)4!`SHS$0E3ysw<&oc?axKA0^eT zKQQRO&3xX9hU<5u&UZOX#@@Ln{p!5`pqxlKGRwts0gnJR1ZKEGV*dhCE*i$o6#E_$ z1f@H4x7DcWWi*4a^HzSSacM3reEhXAu@aZ}coB(lfFVFQAF(3NVvCulfAGnI!L3}~ zYHiZZCoM}}-douZ24QVk%F{(q8fDG1wGrur&Zfg;tjlb71){SpCJ31W4$6`rS+=_{ zrk9g{{LnoVjqUplu27*5W(*csB{eC!3PW1b`_RE=O78BUXn|fl-`5$;^{0GG31-Ox z{KLNbpCWqJmvWahWKTz5C#|{{$ehALS^4bB>f8VTRYu~c&)EEeC5t*U*iGY`bk7cc=Z!*@5~*G<5HxYK9} zvQ+1H?1i2h`CNh{Tu<~;66Ei`gz0?xIf_^d=aLJ(?w4aS$#N&;AyA><+bPSe%E00{ zEfE`w0}jYL>iG*^4GQ*$DA{`X5|5d@=J1NE>#iNZO+8KWdt>`jp zO?PUjTaPuZUDQCB}520nwnI4V!m%FFCWo*bJU2(SV?a2&}3mdxDv0m zXqc>t);K)#1K%i@SN`bkaaR}_6+igBKGo{5<(1w@f~`~WJlnwH*<}PKk}VjKL)a&D z-XjYpN5#rOst$rN@w^ojSvS+`dK&L{Q|KP+dYC1)ney$9prl+9|}usYA|Hg;+)OE z15HT~T^dyCwHA+Ht;+-xQd5#5#dsrI`pU7!j!s{WIM>`O5Wep-qV{!p7Yb1=97y|A zD%&mQ_(+N>uiD{x?j;PtxKTz+h;v~k>Z3N>3#V#MhD#@{72DSn-fzp{SlUZ4vbWtr z)WdNC_nQp2?HpMzphM8g*#e7PL=L+byB5Ho_#ue4S}V-+?r!n2Eo+#IM=SmQ1l(zN zr^r8*;4JiFLl#lb;EqgKs1HI$r5sAxCxtv4DoToH^<`W3afk?*sF&d?4AT+If_vX? zIz4KBDXp_X_hcaHhDR<4?$7K!EnDcruS|+M8|5e8@SAyqw6El^F4>z8 z2tt72W|BnDaq%GH6GqSVPX)I)*0pT(Yu8s~{z(;Vksaqtqrxu*fpW`Pb@)$>b#am9 zJK)&JAJrhVS=SuN{$S>N(7c^Lego79U3dOdQC+k@R8+HPymr$=l0qqKZqaj-(}B>6 z`p$!0@arx|e52GCaANNBe3zd^zIcWRU|2fOA=kgf)uTThj0dYv23^WsIf{StW>mS) z)BRAYoDuHtd%Xo)!Q0Ps+!-oDi3QLc^u@`H$)b@(Y%E%%rAfJV_bBwvw6QU@mFP5D z-Q}{iZJxsXpWs9XY|vPMa`fuyki=6c6&d8t?2zD)ULF6@A-{=6xa?(?2uslQ z=-bexjUqP-zJmE|Q2n8lehFz{O@nj7SuAeaCvYWp?#r_G{1?oKPNq+*PF9jLL*yC! zF8D)H98a*X#4>priZbJS8j#MnQY2S8W41&a=MDQodyA70AB5Kasaaa(tDrX*Cno3gUBHy^y@zjkL8|xMO~UowFABji zZ&a^pix>Tr;q{>aBq9^y>GY*6^iaohVBFiTie&PlZJ5pfyTQ#7 z#blTj*5$B=L_Bc=oi;QissDeOcyrkoox#*7+CQ z+q`|JlW5!D_cLDY@Aa8PrJk{rlK+mTXy;l0x>_2_C3qtYhgyLq82Tw#XX|eaI2aMn zsLyyhy1(0)q|#&^U2M8L*n?WazO7czE++WjE$=fd6AP1MWWwGP)i>Q>r|S!1h!Y>Q z-|t|{$9a_1`o?Z0p0=4#5+2^LTbHP9CJ}66Wcv{q^T49(Z2NJ)>F&C?j`|;uKuPcbby}LNHU;+&$YPtx^`~88ZC%Oj`IjqMaCmSfhkz?_t<2^mt389c z=`nZeuDa>LI5mYWH)Tnw$+tg+QV2nV zY`R^j#OkHl<2x%24Gl!7(2*q7iDA4Zpa$UUaqyHzY%g>X7p0_(s)@h|$8EhuXR{3# z@4InXsWM>En~@KfGu$$Y;8;rGdonIb7c2bZm|%8c3Ek)_Da?&g1OSTD4CYz)3b+&V zLWh~rw`-eQh=C5`D+f(`4_60JILJ$}IW^j32~w1}w9)J4fXPUlvd8&EON%kqMezA3 zDH}$`l`FvqLUhpT55?aum7WvrVh-ob_)qsp0>_HRTlIf_&L`0GtE>Mex}#97Aq@JGl|x3gIYnZ`h0bT zItEA$o_)2qIm1T3N5Q{v?Ix6J+@6g!&_1cLV$WDeIz87WMuajl`@Yrmg}4kbFnG$z z)4B!N&6E_1@?g)8196*boxfEkbV~q3RTyoPk(_`OLoWTg3c2m9oVRhxxdi z1cO0WWE!P^qxf%6A_M6*p-cu-jJ`kUe<#Y}H+xjT_C~H6`{EHS@Bv`136Kwrske zDg23xx2JGZBc|pg2p}=tQ99#Z4A&ZqzYm>R*SN8KR^J=;OTGosb)}vrt8Cjv855#^ zMuncW1#ijlIuKz68}b1H9qWWR<5X#@07xY`Y%zKI|zjN}>(n=?3^p_t$RYxfWcopOP7=MC1`&HCdjhs;Z z=g&DD-fOJSittdeRnJTc@zIlBt=H^EM^-oI2*#vN{U#RRBl%*cOGQ6aOXBFZl73Ma z@CxS$K1!lQ^!b+W?K|F)7>Sgv&6mG1S0{Dtwwad7rw^W}%q+}8XwXq$y+HeqV}c?Sp?n4!-1hN+*D(Qp0l*!F(MJys#gz3J=GL+_ zbTxP53pB++zO=i(Tq$*W?$}YzQX8V|_XzrT-dGshQGpA`-^=|Mj}_{r{y{nve)9g+ zG9qp9U*u&jhH5@#%^Gfr4^CGi_=zW8QA6r-)jnTi8?} zX|{ZgZJtzHJgtO3*Ut68P&|4jW7B#C>!<@L47y(gvb&B&Jy3bTTU63EUPe4W@P1t2 zjZng%Z)iY0S!+Qx7GUr4;+Vbwz;$FSom?X53dFJK-IDtp;I?j|UxGc54k&O)2@YA8 zvJAI{)#t+f7xhu+o#Q+2+%FOp0Dc9{-9_m;3HsUhxQ(o0o+V!7*ZQBC&+-QyLsQ)m z$$M(W_5|K6d`t3sM579ejp>6lWZ8Ra1Y#ajok9t};U8|8mv;7!R-XWqtid5|Vu}N$ zyy*fM9CjWK)yc3=Mtyrv9dW%*GL9o1;vW%3{yH}iGNZ+y?G$&V&8 z0D-L1hPWWOgupLmZ$=){n(js3|B1K7xFOe?5-D7mkx>~nw71@O?*?CO#Eu>603f#K zXm+^-9AEq)x>xv2sM}#n89Bo*jL&+`CtKw!;QMnFqYnKH)c5n>;UBr#fm$YVy>$1^ z59R#jEGU_kC=J`@xa~VONN~Y$sZbj7bI}s}XKe}&uZRdr$`or`y@7Ds$R~j=dNT8U zuLDg+3ZNR`S;49V)m4aujm$AnpDR(%z}cUxiz;(zg00He zF-XJ084Z8(?yx8$-jc&f1KamqDvS(62(Fn;F%5e>F(m~>gKzlp zu1eedl{MSiYU|Z>Tg<|u5UYYMI>g~InpWv0NA`a)x$~(pwg*kWJIw2IVb84Gl1O4iwXM%a zonE#{9q7Tx3nu|)#eF7Y^xTEH=2F>mU9-)TKR{-SUvd%r50waD8$6UA;|;*q;L2q{ zT>-tCKL)+Aoh5ka-^>37FCIHYY0v`JF8O87=wCAVQ$X3JZ!l?a`{gSZsEtc2?olo-Bvx#pK|q@if{Gw27{4caSI(q^Pv%E7ZT2<)eoP zZKfoXYHumohG~P4F3<9Fs+FM8E5`t~pzo&@z94&E3i-+gMm{Ijl#q|}>rYF|?Z-cu zC`stiKm)=81&2ZCFdTo05`Y*48L3oXd<8Lax?~pVONNT~h>0VPoOLP9TpWrq<{aI+ zUCBAiZdB#NC@Cf9OC_QdQFb?`gew%$&+YYJKjj{p0{XIEdYBiyN~q&-qV{tPSu_%w z1n~;u3XdbAs>{osJcL{$Ry+TfJGJ5h)#%&3dVbws@_+U9i;wy#ZLB*j>4Z+pzOgIZ zJXfH(62U&|W+nHQL-EPC6K_^CxRii&0a%}^uJ*@isrhI#INv1uY&``7p7HTJxa#(^ zEXFLZJe*iFg*lO8`9aA})e=VBH`Vy|IYq$!Gf}N3+c|o5Rh!bWCe020 zY3P8>Q8u3N#KxwRorrg95Ph^xD*-s}vwfg}2rW405X+59-FYkK^oY{T*`?3}lpkaW z#*ula^!ADGgJp}21Ge5U>(o-OV;JeRQ{l zWkY|7!8>zmaepwyGLGD0m@f>naxq_vFwmT4h5S&3z^=(0^WO3gR7E=wadOp zAk&67K2cq-Yd+O}t%3EhrPHhqCJXccN>O3AqcfRGgVkcL0dz5ej&lznRn;QG@+WZRPQr3wW*=(S$7Nbg1cXwR>; z{|kDF|M4VTEq;+g*XLUfiw=%aodZujR7}Z$A{L-W%)Fdm6F{v!SymO3iBJfM0va+f zvLGOkr58@*0#s4xGzKrw&_lev&AuwpOI-D8oiclBO}P}@lqeW^3l4I)t)8%`Y{=GF3!0lz^*PLIWVyo*M}HRywS>xBa~GHks^5TOruaa408Q=345mJ*M1Bf|k$DCX8 zg{p|`OSs{1z&3tPeVpmp@3r^7Hko4SEG4^w_5H)zSA{quwRnh7>ZI_o1ca9P7AbzX z$b_{+{+U3KH=MG)%gwfA*Z zCRflp3G(FP{8xWr0csKf17>FncXX>L=T<)^vQ77h!baX-nldWNjQcB$KkUCBfpMv8 z6%swqY3**E((lZev=^at+6%CBrc5jVB4dt*SE>Fi+A^TTZp_FJ5yA(Yt$k<(2!jrU98IN)lV6uMZCv_>C;@@ltrSatp&SGwdmu4+_8q z(_8!U@#(DLD;56D@$QJ=#D8@`uB{CKa`a)qQ9vc_h(sc8&26yy!>&6}U=9Nfunm@h zi_o>pV)J02=Oqk$SP4XkbV(Dw)jB=rKL(=bEIG0C=W+m@{(mojl~UK-BwfH~b(-lJ zPcV{JfH7Z84b+>9I0W@r{}~6tf5vfP4j2c(^O-q5$J#*`-n^x5KbU-rm7Dh!>7jS@ zYmy_0Y@2qvb9p;we4D2l#i!9*L%;^uSa{p&bD!!Zx6h<_uCFdtCpFaGe_ZD*I^JE+ z?i+ViN#lsE6s{K8ANO^z-T77w9oyt3yV)i93RA^E2~$KcHFnT5QDMtFt-__|;lo~l z9QZHe4koI%FNhF(FA4-j&anK=D^tl($NCi~*v^8|TThZF2uj+ZeqRbzrZS_8#wcqIL&6nyH^3L!O zVoh<$axhK<1Y8x_eRu>;{}G|F=S54Ykn6J&Ws0|3KZ!hYmSi+8de+!`D+p5(C+htn zkJmgi)73+aa*N{0pDd#lZ}mUaV9}!kG%mxZOy_cEQy_~?wPiz-^H>N z+2`BW={iB#N8WxGRg(5tO(M9Ah1ft-iJjJCKbK@nzVl1|hwMw2U{XK>0-iG3EJM$9XJ=dAk8Z^XnR* z69i!dVB?2SW5E1ic@``H>|;Ij<{@%fmxe-5q93r{y&k#A1^Is(M*318*W64c9|0uG zl&fEFq=|@3p;&HNJeYog)qMbQKYKm=KOGJ&z6#PymMn9G=89wpwf%O2hYKu^oC@?D zuLI@$B8ld>PA}}`02>~h7#$ZvD5G1Jw^yiE4jY7x$qHJ;w&Fh0bNX)B{Q?CxA=` zP*%|5z(Fox_FpWWGhN~v6;Ox0oZ*Hb-Y(>>nd^O}b-E3shXN^EO=|@u zFw#)Pxap}gQ*#DLyXzw;(XN18iHSD%)I<~ zO(B+z>T>44;m4z76ZAf_^X7YV*H(>XJ^uCNz?*^JRd@hD#$s#w=>hCxkWMou-Y1On z`|Y!5GsQwu!5Z+yP_@y^kUhARuJ#XtuG&j1oP*y1y7QJ~mpMZ`(hJB8and_pnorEW zX@%pRcz?C7#6fECdZ-flS^}~U%5UaR_=!iZ@tc?;mKdpq!V9bUT}$Uq3L9j(vwfhhob}Nt&kc^ zX{0+mQo8f#Ck7R@+uu|b=EC(7AQS>U)3?0Bh+Us<01h!3BmVW4tyVqA|K^{jv0h z0)SaQt=-yaWc0QDHW2Oxm~LUMy>_ea6$~<>Wvm&0^?EQhquDIz0d@e*E#k>7)y9Nx zo<1)Vgsog^Bem?n>!PUySI1%fj2K8#>G>L}_CskZ+Crddb?)C?;71pi#wU3Rr;6eL zc!LAL$R{5V<&k1OKSF^U(nZv;+tcIv#_reVUpJ4&#{UKI0>Y?65HZlS1+vaEm^l{4Aj=COAY=8C z`jde=COvT2D+~%el-bVrMeWf~pV%ZIpFaue)%ZtfV~8mA7m;Kb(aVHYP<(NuP`FUi z%n;5X9hVf#8Zt65g+~{8^eUUxtp)su&Q}xIFxg;AiZ~i~&wyk~x^4!Lr&Cp#oTze& zz5s+p;=J(PfjXqQmYiQ`Rudh%19p{N;sDTAW4N^RLqAA!q4Nh?)2BJ1`tQty3fYF+DSI|U zV;Z(A*{HFqOG2TRoSo&v(6+a)ikVd5LoCgqU%4u%0-EB}1Ib2JzvYK3GXq7Knt{0I zi&5_SG6vK(L9}5;EgkNz`GbHo}&!fO7`Y=JuNisS!qaMvDZa zqbZZ^>l^N9Wx!f3$AKiE>dCTpUR=2zu_M7C4^lG_G(QJ8_-L`Gm(bmHkGbpI{+q5b z0pP`gB<}lT4j)&jh_}@3GwbTsh;CkFHj_mMWRq-RvSt4xg-G+?D;AD}FHXKA~2?`Gy=q}}hK=UG$;?QWLbxuBL z#BV$S)`_otx-w@?*B~e&ovaRzEB7W;g7eCwk1}R<9=Gv`g#Q$C54FAp$Z+Pm{DDID zOSC0c;G+l<(#I2fg(g;O^Y)n^(+`p)+P!CwM|>v4)$YpHL057y(Lz^qWcBokKNLHU zssVx)I>Sw`+Sjsm=CaQfTaGqud%y5VpC_(Qy5jH)-j|Y<#ZRl~5B?)lcWwOvM81L4 zwaZ!XG>~fowVEbStC{8jjQ6SVlC%eg#20`LvFeT6b3hUiwC09M7PX4mbC6W{`5_!K zm!Y2x0TyNB%WKDTj(o+&xoQSEx4WdEsC!gLawEkVpHJrDF1^@euV3CYky z^t|#~pi^15X101>MjV@|>VG)wDuJVssG4J@Ebrs-50d6_WE3$E)3+EG+VHy z37@)9j`41UK)P(lA-Mr5BRkw;6Rot%-Mr!IozQ4&8Ntk?*vELk1a9U!x~r|O3;_j| zMa`nw7DkATQ$}s|54^)uwsJhBfmt;(eYW@@=bGA=xx97O%+hS9&b6iSSf?>pqyYnW zx&%aovK&J7^OR8$-Px%T4wdem13DC6-yx&@2JB%&{e@44o}^!2#x65qDNSJBeQ&#w zun-Rlq@8Q2Ye?UG2XRycuT=f{ZVF?mvyOM_D;WiM3}N*pSD(yE5Nv7lXHzt zX3NFt4anUURLX*zCXi>SgStn4Cm!VNJK0*8xTkQU&B9$9E8kLmy+3!$T~^7q9>##a zuG`GQU5?y7wXhXD%P6Bvq+An*t8_CoCcF67X10ipFqA?61A|ltM^?UOHrRD~I_2yi z4Hp#p!|Ba!k@5f_*P?Hf`{Oa2$8tt8+8hObIyN3&w3(kX06*LhY@-7Ll^(^V5+LPD zd6qIQ1*j%kmP&2)Cg_!yZnvPxWr~>tH$Z{0~yn3VN8p18k(|Iw;`k-HAH7L#ad)&e!{I3FkF_i%kMWcq9Fjq=)tU8p) zBd9)mIBlWIEmHTFti;k`fF^1qXx#Dr9&$RMmiBK$wGr^c^CSNq$e4SW$aYA`%*Mjj zo=7O4w{~mpt{BwiH#AS!f8zaDwe(65;U9W-^UBsty48B!Lb_StZ95VZAYN;JjX(4U zflI{zR3d|>^d!}xz!z<7W>$c3$T?GmKcrLuqG~%Zn(-(zK-1DGvIuXTt(!4|pIapv$ zSs?@Nb2suQC2nK++Nx#K>pSkMryVqdAm~14wx|_@Q_XDf3ot;han4R8EL_@oPdUUci^;+1QA$9(sq{9XAfe|uYZl^ zFmTE${BVhN{`vFgUwuQt)k$1}f-|2gR@(ukmL$y`9M-Lj? z`bV|a$XfpbcLN##E$JWrEU?dS`FLD004UAtNgbG8g$uadlFbhDKicmC8o>IBe(@~2 z0lK~>=V971%7-mTdDhts?69m8+&^1zolr5DU<3o-D~22a#{i@ZLim@zqGIr`2DhbjW%g1)gq8BSWFWEm{gRC}AjBb$VYKP{%*5~?5$;bU1@WK% z0EhNpTn)oD0hfU6YL|dE5ecjl*IH-(pOxyS10IdT(phI0cp^|?TaUueCSW((>7TVV z|9MF`f94wqk&2$<3z$^PF^mw}*v1c-wlWSw~a+|56I`+?a1 z_SjQmH21%s>91>kt^>Eqb3@>O@;~266|H{QRb@4~uDbr*o^kqm{3FwjI!VRKN=j`( zJq&AO`OiFumX@%E7xj6;n`EbFXSj=v3@uu`1`yzu?}uBHO}f8fJI;OQG1+Xi2ayEU ze*oYeU9*q3ZT}kF_P=ffT1w6~{O{+0?s>%H&&-iOB2d!d*|oU;DAB-UO+4s7xy-x% zcg6tV3{Zhg*sd7ROT@1TJ|9k#5QSe(e13m(WW~$Jw~~u(mgUzf{KMe)Z@$#C8y`Uq zN5MNEuqgzfmn?o~Ch`N0p(-gd@pW{paGA0n%L{vb$Ye0N2QzM)et$lAHHp3*5gtAQ z%uZ(7``(`bY;ySq-T#lVw~ULb``(9@kO8Dyx;s_6r3C4gl2+;N?nVLW5&=QFySux) zyE|t7XS~Dv`};gEo;Ndo#>1RBd#`=1Yh7#YwYmOST>QVQDt&?u$V?_tz-h8*k-T`iq;ozGQXl8&6_8EqOBPx`RK_e^qA)iIQ(wOZRNeY?sV# zZ;Xg|E#(dwN-l1lo|dE!O!ow%FhVpV-pb2j8+eeq^P93nlJNO(>ADe^ahZ$SUr_(Z z$!VRPMHCcVI;#^$-fHU~$3hHIKMA30HeL4I*xDkeqQcLTcn)l}BK3`kh{#7VBm!m{ zKeiU-Um-7p<=<{ZEK_*X|I-X)gzY~w-Bq`y_*avnfAH}eFgAgky_J7R7}TtN>d2t` z)^);V(|LP+(f|ojhS~!~XYTTon3g7HQH(~a|4#B0B#Yd0T1^*Y#Lc;eQb6F%+?KM} zu9q}v!!?T~3Ez8??d@&DnV(ulJcMRuH@ElXEG#4&n538sF#V7_)yXuaW8&(F_9|k& zFJHeh>?a6aWAR_NF`m_|MWL9fx6Jn=J1w1~Q)GQ#-P#JBDipQZI^Z!fG4X-B@@R8Y zZ77gi2AVI)o!V~xvknu4i823{Ry>GVUKS<&jmW}&R0m#`QwswSA7I9 z+wDKaj}ulfUEvlHvV94L>otor<^6|=U%i3sW{#1tMGUweXl69vvy_nPVtaNzQ-A!V zE0Q2!wme6DzJkL3c&wK48xvSU)f0fe(`~(ek+o|f&qu|$Voza0Sc8Bba9*ySu;?02 z19(rydB?}i7$X~K!=d$Xc`yHQiC^kbX$NyeSs<%xfu>*1<j%Ky&cAt zvTZBp4@CR@V@Zu7c)03aQ0V+g?(@*GFSYE2V0ZF#N*uY& ziOZ{zT21Ym#jL}&ZcJ~LR6X_e$)_x4{F~z+{bFe4 z$(u8T9gcdtX;*4ZX?m~g_EeGfF-OL4Zo z%^rrQJb#x2fhi~~IB8jN8>O#-Me-+nJW_T$*)G~F`DoxX8#O5QQ%_0@tXO9SB`KADDpQCk%U4tsDgN7_wRqXfHzp>g;J#Ior27<$7N}|xf~@_Tsq@G^AiY#pud^n{ES7U2{GvGe z4~#qD6vjZ?(;Xa`18l%om*+p;I71NFGy*h)+HdFG6a-k>h&4$2?t)8Os}i}jSO9K# z)I$YUIb}=7wNi3_R{qzSa+L&`?bbCYg&WlqEmCRo{3#Y2wa=hr^=g0LNCoYk9@ZbZpd%rzBu&JA2j5(bj=w zll7C%tQxl#Vo`0cC?FtchkGsyCG9}Bg^`#2m%o2;J`I5NK;IVgN1$ni+hYRlQMNvm zpW{brBSU8+qX`3mqozUpYjr}vlU6FOHUvty05!78=hTh-k5={wt}kx(t=@GeIS{U_ ze279bk?oEafu)N2VK&WfamQ3)i4z*H>nyRg3$zCZ2T|XcFU5n9&HSKFK$jG7tWk0) z;bzmt`_vifg6FUgmFCoWEnpe2+sj98iDVOn`DGZ>}Fl-J0|>smFi6+%e;y{ z_YRcQzB*Rl_?akMyfZkd5??O$&8o*dnGum-(1t9+7nciuCSvo|sb=H{5|RiSUd{kL zy=t^Pz9_P71EQol-qunJWO82;L^CTVRl-glmzs-S){>HPk&|C&d1t#BY|L1tb`W8# z{Txu@8HkM2oiUH98GV!#3};5_^vZ3|QCqj>IlQ_u#?3}y$?3UDXr6AloY>l4nT7T4 z_|r5>w_j(!_~@jEMc1qdqaAo(+=leF@hsvE3AC@cO)K=uKDy3!^%3{*kTRAZYH|5p z7s9Siu&XpC?A37EMOWQ(7g_OoAG~C@HYa^Ptk@{>-AvfDUE5F&}S z+OC)MKkjX0=vMTtBU`bG&ZVvIT}1A!T8Nv*h*Bo)@1sQHm;^=WHGJtjp}!;)8)G zl^Urwf~^4Qv>Q`bKvZY;FZQvaA9~1lDQ#S0o^*W3{P%Vtd+gvx?Sdu4AZtw~*<= zw=iKy6)BnC5gAiYh37`-=3jkCAF#8rqstybE4jA(XlA|Brr$wJOE&WI2`t}*Uql@9 z7E|)CYHzS#HtBlHEr`k0Qm{d%@l{gt&8<=IYJaZKc+d2eiARS7llD1rVDPPr=Yo-i zICm5xemE2tfsA8oPAp@75Zq_1m$uNpNDQYjjedkxoOUg#9G8>0!8h zK*a+({``bnwqjlHxSltw(%s7kdN%=1te6?t?egiwMh7ZqH5PSi#@r=GkJjG=)r=)! za~cs~_6c2ej0cC%-y?Y7;hENO^HE;erkKqk2i(Krfs!Hq1>+X50rcfM4b^W7< zVs!MCs1scQ1q}^W*b>=AzWy>)fbXs@kf$v@@DAG&aWRx3H-bf4OKT| z{dUmeSMKgSKA!iUTOUnH^cJ1Hrd%sn5Ed5oWGsE!!qP!n%dW0K*R|%NOp4MfR4MDe z)8C{c!_K-E$B8=PWaPD)LUJ<0{K8_=1oh)Xh-imgn_YhO%4eh`jo0G$0mc1;D5c2Y zz{;So1?%W^Yn>`Q-u1UAQ?YM-Z;_9gM0F2TOG{fZu3DcOP>S=1j+On0Zkim({Xn>p zwP*E75o3t|n!aqfvqgI1c%n9L=AAY1tf}D#!j%A2p$0#iZ*iU7`A8F}*dp%dc*lO% zHDxG|_I2Ehc#@WGvX(0!U;jE-db&vqE+es7e)Ll-r)gI;_hd7vvwn>^%~7;y9fK>& zG)n(d;Gpo(OjTWJ?zQ{PbZ#&vg@YC&VZ{Zq$a^{7wPM?}7y98}X)B6hf)Ok;4P36>WpY4 z90()d8j{A()^+7vB;HN(mf(OFN}or5KRz}`KmEUx zKMy&PP(Ib|UF%v-{nxc?8-K0^kOgNN1iXZt+}A;67B`*9(i{hwr;XMstl6-JxlmP( z_gbbvvz}ykm!QlAb08+{UQ1c`%bH8y1hqY6wL5*Yb_I9vboeYg|A_T&%k81+f|ZIk zw`FJub)D4n3DER2;kr|_k!Yv)lV`qlyWZ{aXVt|GmgPJ{w*kEpsjFnM^VpxMAyIO(BN64|oDtobfk@dWAhB@+EbPTiXT{A9ShSfwgbT;Cf~eR8p7Vy_g2&Y4(JK8U=w zP7?|<`1(mh*Hso%;&-9n`Ch|)(3qwJSx#*C`W~mz&V$o#}7)#iCkov_d zqGky3s?;$}z|EfJ<;D-Miy?X?^QGG;&DV4G!gAcSGy`Ia5}YukqzKLTlHV$s{r5MF zjyW^JtjC;UT#%5>b@$L^5D~u z3*ewaALlw@T;H@tWL>VyI5}{|tv4rmMe4b)ghxD{QaWdE1aaqVZP~Kgk!Z(q9<}zp zV}4HMT)JGZbgNABJCVGbVf0T5^WX&Tj}1$ziucN1N%R4G9__B{JgXgY6t2=#__-66 z#utiTo9fs+jwo;co`O_x8QNo+C^fomu?>Q*IR>4*)^9JkxbR&kU7NehZ%9Y zxjl=IYFhUkEu)sXx-t5I0x%!-Z`zN@BO4Z5{R! zza2_fflH`Qtuuog+AjXC7+;Q->wLu39MNp0z`0rfYG3}^q@|K2FmwFD` z#NU^T8l(@*Cjy06Xj_L#A5UyPlYgLZ5U9LM))1JQn(D#E!lHz~*g_v=j5t5NjAJ=X zgBRF{3hHw(6cSHA(WseMcA4ni@XuKJ!0)z&62DNpx|bxWgN=BQj@13&w1(j6v?^g` zM(E|{;$VP90_ds!E4HKIPbgbJY%svv2PL?KgkLMhbdIXOj}jSj zRYi9P^fpDEb(d0hu}T3cM@p<7fx zD2T3+Mr~X8K#n0I<*A69HFK+y(d+dr@%G}{R_R=LeO5_7SlSZU>1b1yUK8KFP4LPd z)X+514)VlIRwh=9yfR4H&f7Y<(?i%^BcWqX*66^M9{<+nX$4m)iX{v6_>W>^2&`82Fy+U z>jyk?VnG2qeaiLQ6M4Ct&fM#OI6ocRWNcQ*8%TFTiQDIf{Vv^~D~hOZnqMSr96)r1 zd|aAYzOo#L-3_QB&tfr1VT`ELX`^DzMTNvOk$-Wy*IX19Js+f;YU$!#J%$qxKl-pu z2ER?kKmDB6GOaB|2YGe(J^H!Z;6%9e$xU0MOhTChjUabEG!gvpNN&RhWsyDaNyim$IMA3oDs%J-M_D@@DakPoGrA&&jq5aX<0E zMf=(r!>lrJL6z3d$?( zUE@ts@QAOEa`lk+!k6y`-$x1@u_`daC@~2c2Or!AKvW1%Wn3&jETyF=ZZvd}wKNca zSSm{5fvy9>GvYR2dGHNlZ5;+KFsF2>fY_|KaLTlcvMK*sHYhIhip=NUTdBZt|tOdq3rll`{18VTU?OvFHtffO6o&(uNNx#en z=RF86wH6WLXIfE2z1@*5g0c!qFVV6QHC`r#Ab*q@KXB;F5+2U6KCzxmRKXb0GIbQ& z^srmYuH`bJ_m$I;JSBLHF09KZ4i)B8fBa0xUnkSRPnFg*=gI#He)E`OY0Y4LwWN2? zKa?-GpC3b))gShO3Ojj>uyxz=Ervt3N4Y+=Baa&%{TQ-MjWUK)tnQs(9jbWn@tK2# zGpbo}6WF;4y&ZBzD$R_`R}`mKYy686i#tR73C8e)f*Q9gpX-(^8O{$-ia*$qWmZu? z*|PN#XEhw67Tm;Wp30=XZ`W6*Wc0j)G2mS9dB=T%)6ctUip!y=(uSU5&Q?gv+tsUO-I~W~wXgX?uwecG$#Txn0NBb7m>|?M*O^Dtw%4A|3Ig!c zamh9?k-0FnoHJRNeEi}iS$lZPRG0!s7_Wub^^s?HG(GSQ+Q8NBEXREIdMA0q;Mf@Ep0fkv!?4$&auEUp(u6X+N?A(7?vehaJ9?#=JM?Gm35J;tN9{+w#J(d0fP zZF0v>4lE8Vj9{h^cg1|b!yGy(ukT*616GG#a;{}+!1*=7@W{)PFkN9An0Nx^*}(f9p{{a~r>0P540F30*w0xLb$MmN`;J z;6NHkR$L;}Usb3cBc()uEkm+8wb4J6xJzd-fDSFQLEeOaI|AsN1O;?e5Fy)yrwFGE zeuR@6IPf|t!1gB8=?Mx~yK=Ptq@Z)yloosT{ms%!>km~>t+OP5qRH2_Yj@%%u`P~( z%Ft2T3@ahJs5r_*WQ z+_}+96`ytcI=9&mzN0#7<}+X~Q)BVLv@=*Ne?6|Cl{eUFdV!g1lYZmkJaVS zCYEoz%aw7G5oxlBWU{t6B{+MP_X~GRSroIv?Xlrasw58`cLq*MC{A>)U^3d;Z2q!> zb+mOd-kOtb-}T}tmwPqox3-nGEI!KaZk)IRM9;odRNj~t0jzwY1X?G4DYjA!jEcDZ zI|yexYsYf%!klKoOU*3(&S!LAs%sqmpPEumhsVq9Qx&l&D?khg#D$A@bBZ)&MJdil zP84>Dye>x_d_K~su)N1$76ju`FILVM$cM{0IHb)hBQ>g{>QcZj0sP2dvehAJFuGfy z`5?4S6i#0MA+WdR>gJi}oh2$#P0e~st)Is2_WM<~W|#>R+f`ag58J8cvy&Gz=1HMj z4oTnJO-sXQG53<2IER-ipU_&Rb$ukCDpqRp#yn1|QJ4_1~ zv#AFtOq)>{)zKIq;P9VHcmr?tl)%~vKt{H)A4Y3RvO#zDIDDFBbpzYFZPhCm18V5G zIch`HrLN+X5SDMVZ}xz+W$7W;a2Drcz3U`MUD&d)Gl~1;UIh*?Z9V`x?eWVBwKm%H zpH3a)^f8f!{8Z1ZS#YOXct0)av$Tx#^h)r?oHrQr{295KxP{O0`6a?iqoYu0_+em5 zx~j;|S?mq(=dS-&IWN%mNqmCY3fDuT?@1Dmg-8U_6rh>w0+7_J-eQ$7;XJNIV)m?5=rf z{-r_1b6sr4Xt;o&@chO#D$n-$1fQY;4>U4qry&nsv6OLGVrGt~PUE-yH% zAmxPdq`GnOeRvFE`hy7MVTQOaMS#hM@PfE*xkMKlO?qsNoQ48}1fxo>P9zfb&gG|v z`K0k&>aO;CJv6+G%I7|L-Cgm3K&a7=sIFMyMdz+rp9&>*w3{2zGpss?AXj8X{hrcD zJL(U$5}`MFg85vQBh(qw)5F_7g7`J&3EFUMMTb&_6vqP1QKVcy7cSyaG=dA{@{h#} z#a_@x>G8Kl>q$_Uek0OgzcH=d5ur>%Bk;i%{~OYGN^JE%q5EvuO6VZ#B?9NhgP=&> zosJWzqrW5wJ21aKjI(uk%D&2&^9BI*-IC3(1U^5YK47`JJC*gJNq~i7OTZK`P@U=L zMoUG=I87!Da7+Q!$@yM^-^?s5P%=~1c}576LdG>ofYzqf4eYD9?Ef+yZ+#FOyZUN} zC{gKS4MeY6H|jB^5w$YY_%IlALCa@_Tx z2#pqDHt6=2lMJLjAO5E5h4GJ<;Bn*nfcjgESRI~ckpbtQvSkm%FmBc71YRn#JKbb$ z*DQH3q#$ieCdOqs;JRc@+xsti(X7*_7JWGQZINDWNDX(}WkiNA_G`#@I#g?&MeImA zWY1a7m>;gwJH|Jg`%>ZhmKDzN#CE3_Ne4Kg)_u%qE-0Q%H zC|xC#kofq)qGENbLX;pq@i=zv0}YSM9t{wP_q_4 zI;-xj{;juoPs}2lkjkLs#D$Ez(6EI@26l??2<6SRmVtWNp^ice^^@-9 zs1LJqD9H-K6^$YMr-_JNpq)3xh~3vB=lbcaTz!QQjKmIH^Ca?<;&Nwh4`L{n{CfVZ zTOy>;gteN88UD1>-y0%vOfZ(suXV}6F*u$r3Ti&LQdxLStUe$4=G&RLqPq!X=HZ3p zbr*}|$!zk^3b$vLSE&K-IH$>->k!xH$_FZc3K5Wlk;50-xU%- zgS*0UIN=oqV~F-Bu0?CJ$$P^tvOE~qIhz)@!Q-CA3>u_}ou8VG{GNXWdI1lqgcN$) z#p|7By3S2upC0gkWW(&l~0>f zdAwgQvq~1H-(|gPRl;AkjO7&~{_(>be2{uhhW7x&a-?yyc;JkB3`_-t+Y8kEQ2OGr zX*c37ZtqiNsvPG-BYeI5L1#;sheB~1Er%cjjkMVpZ$eFQQuKd&Vd)6E_!ts&w!oKrm?JNL8{1<`g8;b^W)sww>bKT(2NTSwsvvgXdUgZ|*Zt%oj$q3OBf-MHixfIzbP{SjcQk}+{yn0JD8RQ{GwiVxyA&@* zZSSLzxfPbt@s{SkA9xhxeQ`}&1XEk5T%SiQEtSVEmG3Rcd-J-;Aq{R0F{5Ka#FpH5 zEN6O}bnPk{9_Sz;TkPSq^_AR48S9HZp^s|0h*xQM zI@|-rfqsRJ@A87zwLBM|^L6<84qftPd@8i6kX9V~$h!q(vj!1s*Z$#_J5%k7h0Zvd zH2N|gXwO{;hS|a5(WE&Rsqm~2)0>7f$Vj!me4!(i{r8#!ydf>n%R0F+%&z7L&oJ`l zaDS&2yN3lQ!L!XiWX#=TzH(AZE!4Q}q)o8HDd^jQp3g(=nZAu)=}HJJMkOpz1R;x7MQ;W~X!N6-W&bhh zSg1G8>=z|c9^zn`SHf_XPKuG>?W{(6BOf=0!J>n^-XCG_N1c*ccZhP^QKd3B+{4r*OKDZUn_v{IL#wb$0k_ z)l9EYz&unPWBpgfvCNPfZQ@u41yQ7W`9)q>OuLvBUU^460;_Ou@Wr8BLaiZ-t?^)p ze4j`7cNgS#))~@n|Afa=WY&tuAJp$BEm->1J9d!5snlq>Zq!A0hgk~HxkfwZ2N(wOVn z?vWn!QM;VH&a-Haz*p>y5vX$Hg z0H-KI%baocTxKa*xjrn#YI`RMHT?5?$Cr!l+J%7j;apGsJSbwOQs8%p<(OXWzsgpG z0%WmJxrRFb(y5{H#FnIYNbTq68o`F(+8$;$G?Ke*sq(y%{COf`h}AsT{$x z7-wDcWQkUdrvjVQnGfr=$>{D!-|A44nTD*7C&iF<=w%nsi7#HnCq>2Xo;!eF-12cv zNQ)lrWX2!AHNB9oH~BfA7q`>idyeAOBgh+nA@+$}my01des+UIOhY^X=KYwYuC`~s~%yVFCsvaseojy&n+@iF~ zeRR&@*q~ju`ORSey=X}hoFUvr>^0!@XKda1(i)r@rs#rIktk-6udj0|UjTiohhwnd z-~oFnx@c_LV}V=U|FO>BIXgS^R{e%k)Vlab?tWlkbP2kB;=I<>a-Ah|#qEy(wJDo@ z+PA09YPkg(eD<~2Rc16&pPmZDc_=X{WppBEu4a}o(7FMIv6VhI*YK`P2gycHAM3A= ztxwwY%>|mUCiJL{Yz8Q7z-4>eB~^eJvz|FjY0g`P;?#J&?%;gzsKlhFXn)%P8@4?u zb?QU^cdjLe$e3mRE5J5}5~97+?tXKGA~zs(Vo|x!Gh9N2vfCnv+$wgCFz_M0!X^j7 z({?Bblc_y4>A-Xz&!eQU(!A>e3>aV?E-9BS;Wd-^O@Ajo*_nnZ8r-L@-;qie7K4A*UaFwGE0wO`b8NUo zo`Opgyt0#smuPO#SyFfQqYHm0U%eI=2RpPU*RzB*TCzE;!P!fMy8-37?;An9nXSGY z?3fFAhW=~nrMpIdowSq^r~bs71@v7?F+Rg3e;MsUB2}#jn#Q$?R6nulVVJb`*ODX~ zeH3gmtLUekFj0>U{>TlmXPX1aTB|oj1i-FR#HML9E%K~1ggdea3+^6O9p#6@L{?_f zabK9_cvk;&j|BV1>OSoHmi8IX(RyLqTQm@=uw^Mag*+!-}+ zVDbXIv-!E$tmF$Zwq2QUd?r@Hp-X>11u`)2(Dwb9;@r^#l)cOT22ntT?LSB1~}ns4J2BOKn+omKG$=PC$sM8E=d#Ots;f8YIt}9Afs1sHvyg! zgf>U4lu!PG<+O-!;XB}WxZM#dAAQ+`-rTm!SE~*50~h$k%E+IJf-riVf$1C_m6c#o=l={+@s|;vHsvs;{HZa7Soi`G@-cV6KTcfVoJXxN8XxFKjo% zg+nROcd@(yg>!xIkk*+H9q#^tg|6J4svwm}se*(h9HXZPbb|aC&VRRX9S$rj=MV8N&<~DNhIU$jXnWCrn&-S^@2qMfCrcs^NRwA85uQHPQ;|MB?uQ@13s1~B8@bL z8>58z);2#FQD~n|i{MJ0<|R){G)4o!hC zS?~WXZ2tZB??C)P!M}b^AQ|+=(N#$t5pzD^Mi6kIbxBnfr?H22RS2{Te2Dj^!G9R+ z29&{y42TlN$iFUv3a5W=Lx%GA?Z;23)^h&$J_$rXqJt(Vf$_h80R8i?5m4+la-vFv zkA@Y1cg&EdiQXC-f^oHrdNMuy$B?_Ah#dgUZKR5%ME&o5DS=0TKK9Et^L4>L-~oL; zeCYfeepz?_KjSUD_d890=~_@v18kQhgTH9=C+UMMsj^cV+8FWQD`!!b`6wEm0stZygn;DOOkjmr&R7G1 z&U0G1xKrqQIqgbuOR<3hf=zdtnLUv2y#If9I^Y&Mw?iDo0SH5M1rJ z0{6K{fX&c^QN-Iue+&kwZwIDS2!KsKwA!n1pO#8;*wecqi&(ve=R*NVrr~nA2CP}W zzAN<*E+cwP%-LTmGM!1a`4zq`zSPYtz2m0%f+ehHn+`93&Q>2UwV%BvR4B5ix8I+c z%HaPbFxo+2hmK9;i6n_3M<#NOt zvz@$lW+qXH$ZvWg_Cbv<#8Z)S)-3Oz&oZiyzH95I_kPkm!u7nP0xSwComS>|e}K!( zEdJ6>%_SYr#DWWBBwKRc5Ta>q8<(E_;9x zFIH#(^Z(xg0mfx8Has3kZgvsc9ocmK&S-Y0LDQPDXxYHU=Gec3WAhxN29@7n%3nJi zgA9bmg94bn1;F_>y>QXxe?Y=}7OM96rUh`DB>QUH@QSuTj@z9d`jxQn^(*8~Ujh$v z_#tSiE+i-nmrScu`s83z@)t)PiSwGCie@!|+EtzS!Fz(QHfNk{2@WP-kB+{dE?4~-yhhpGLSx2RxbJSj5Az_p%yuxg zhwNdyGgIS%$D&h9!?(Aw5nRG%b38knxmg{dm~C+D{DcT{Om%etQQog}YiigTZ$zdO zLXZ%~`b)bl;$Z+PBz~)5>((-P+JFy7>b=lH1#GD68?>UR-9)-ern5m{J%Z<&`w_a# z8Vvh^mz&&xIiP-mQyNOK2sxEGbQykwL>kc!z0}i2l`+#kqa#k6M+s;>1f}pg(|=$B z0}2x;umDUL@Y+(2alj&%D`O|mS!5PBwpH@ zHj7V{ZDQQE;mNBNi=AwS^*p{6C?zwDrB6=I?AyY-3q)~x?~w`FUd`fm-k|G?V+jaE z*=v6tg`9SOW3YKQ==`{a5{Pzy3{*Nzkfk)+!zcT%2<;=TY=RCl}H?VJ$ zW3r^zGm`b^y98ek(f3^aQMNP})f;pX7q1BrGqd82Wu_0cHIzCJ-Q> zF@bow|MBkqB2fR%1O%)+c3-u=I$}7NnNX)4FBGW@>*=3zaN{tfIPxVu)ABsLf9mXU z9oY+iVWrp95fEwNKi|N}Q2d^C$+05|ZXt~gwRk91r1mx}y4D#ngrauho|qRjOwu^< z-8Izto&z0^x6*H>98HzWBFN(E;1jw-zZnN$@aFGm22;Y+MOqWsK695D4;m=sl;g4J z_YdhQ{(6s7@p(!tb^B`3O{2-g54+N7e@4w?5gTEAi$zR$MJAkp6TjAWOF`Y{Vi{}3 zX@%^C$0fYrulp}|cPZ)LTBe6d4TI3Wt+`MeEPgwzalhn!c5xKhmIfc!|IiC$0@Aht z&lU+q{#w6YfRT37yw>V!SY)gC&!RqSN?!&BytZ3F&IQU!`E%%1DS%Kwl*1yv9e~7B zr~Bj z(@{@C!h1#dhw)IN8>L?tK5uG1j)`2Z-S-Ggd2MFaIm%gFJd`5Qw8cF6-I&@3vib(Z zf66M=VV=y)#}Q8#@=)l1GfN%yRqT<#`oLfX2W6v zPDTD?vOm}vOr+Tt8=w=J!ci>q$m&n%;owrhJ)*ANB)yaWvh_OZvCa4ab4gr+OlG%C zRBVxuQ&YuYGNFWx|K^h*pj{~1pC~YsSB?F|BR$);ntd@O9!4LJ1;#coZ*E3yJw6ec zm8&P>F}gqZm?w&a=U>|lyKsAYY@sErRoNvb)>12_xM8sX9Hd}I<4H;!;Z|-+W_F8Y zZ8ih)fso*RKM}$J9}tA1K8n=j$1Brb^W6lsJO}~O3Fw}`=zGDLwbe9LElSS_}cC@nbf`$`xn!wNq2F5e|KD%+{yh-xXFZh(-`CJc)R~TOVm%78OD8{ccGng`8c4R$ z{^<8xg?uNN=d`&ASq}RXqMf$J_E)S73`mlZkf3`s(pFZoCoZSa4sV9SJ!5#~16i z$}*+V>E&R5o*n_GjWMWnBQ_Gv*I49Yn|rDJdN!of5N^h0xte!vw=oolMIDCIY5HBy zCW1DS2!b~(mUNdsB@T9Xsg$PCmP-)~!=$j1B{z>Ebl1CCJ@x{(8y4r5Er^o9pL9a@ z5L@qE8_?Tqp{+~XxINdfEti^*g}l6K>S{7N>Qv$Zf99?xl%YVzudj&9*Kr@$d4ZO)D%QPrb_(6|fD6eJ{wMP|5tD zwz-A)J5lY%CiRjG5<VQ8}8xmFv| z9iQnn(ZTohTI5us=bHS|&}25?tZ49Fm$b2(vagxACk;g<%#1r!8fH;Kms`8VT&MZi zNl&qnPgvb{<@fV3)tTPisY_+|nH7d2 zgY8+_VcAa>iJA?^VugxldPg3QpC2T**suI5;Kp*Fd~XXg$}aFWYK5wbY$2me$T`B~?}R}k<-L(jRv9nt;j z&?gQ(L8;!bjnbbR9B3}tY0S79&6QpUvXyQYa-%WuuMzr%UZBOtLwT{~AQs@CVVw@P z#u0U)Zg9KDHEnu)#`AeXw1}5<72GgO(HPT_@c^hAg!Rb>%ZrKQ9R`8QW2pjmr(el@ z!25jFt12T{#{TP-WzODx)CE#TdIv$H-y1XyMnx;IZ)yE;HZ$RID;$_2bV5q8b})~I zQv>32<@2|TI(GdS-DhXd1Ee6YEyM8{#K9ARH9okS3+X}#M;3FL!rd%3S?lRx>V(?? z{Z(}i`fcB@!CO7&i+r2-;~M+O)t*Ft8Ry#ymaPUnl9$=?pZOaDPjS~AFuiHH zW)MqV!3~yY@bXIiN^pBrPp(}~VYvLAb?bt{s>OUnqvbN)x*h4X>VBw=dV-&7RO*+b z+aQpY*I1p`)gnEuN>H|1B8!r-Obily?FSsPpiuR^XsL$2^Ngj|&c{i#&Y=-rz`QG?fd1ZA>Z0yq2oDNec%uF6hfPW}N1&iwJ1OgbM*5(#J4RXUj~) z6|Akl!d<9b2Qu?nKTcVcYEba;Bh{?O-N?C}T@akrZHYj17gslaK_7W?ZA0wvo;RcPM*qPm1VVSJ)8T`m zfc{rydSCEKE(_pX!vYeZwD6_X^yMX4F_iHPH3+oM`%#WzKy|Ir4>{|<>jH+L*(o;z zJ)6I-mOnsUt-0`pDs3{YFR_q!n2qVe%@6>Ytv$l-KUF5TP?_xoEFcqkvAnlM?omw9 zu2E*dJ`w`e3WBNLc8UkVt!vk-jmDJNjqU*Q2*Ki&YlZPXOjok#2Fx zOmtqK)%H~{mPw(?liKtGSk!Gab|wUPH_JjbLZ&nvi0o{&9rK$5*+W?%o1WK;x}a_3 zOG3JUT!5A5b2}n3$b9OwYo_nbM_N!s~?)BKOTCq4I_QBknPrk0D zA~cMQyAMTZjD0_4gp0AAtHiNJtE?wxD&<+o6pF^THU1Eds7$fXyx zQ-d3AK6`vuWHAvDO-yeiy>2>?X@^qeZJYc#Ch`<<0s_L^l>8_Za@}wJLCuEA->mP+ zJi&)iZ^p<|Q+Dsec=F&dA*|S8f@vV2HQP@s2LI8GA>rxk5dLnk0D0$(-~Gh|CUCfx zt1Z3NWC0>WEHm@^+nz=Vrn1bc4hLfOIXrb?c?V|9Wvq>*B@xgjqq@Rjg|O)LVe?UY z;yGTx&&DkDO2QI8{~h4qrd+|VH_3feKD+_mE%6#OTeQq{ALrM!REm|}>gEbTYe_`k zuv-j?(x*YHMyGEyxR;bOIEnae-x9gy|u|$_S#`XC4EF7oi3v9R z8-8j#pY+e6WCWmO1)+YeM^Vs6KqdadM_OSuNuCT`1+2mjd$FslV zVZ#9~L$}@r0kzzy257FOog`zO(m<}4E#UV-ZCi8 zb%_=Z?j9_-I|O$L?oMz>&=B0+-CcqvNN@A1&2Nq#Isk(hom;qrftT2eg4R+~ zxYEDN%l|UOzcs+&%o}r{^5%tcd}Fk&OoBrgeA5LOS)Yq~OuIg5jLi+{U5J7_{`#l&bEiZ^su335C3U>8hn}Atk+Qvpgm37BDt- zmg#e;qEXEvRiA)Ek^HHBJ&IdBqH0r31Pyl*p9BJ9AW~j!mYf%##u9j9?=2~KA93x+ zRGoMLtoa4UdqtOJHCnfP@vhWIJ2}!1orc7BI~pzs>z1!#(AkrUU_u{@{2a=aghYmh zLr>uQlTPaL@aHFUdUYKOCsyk!=*9qXgcGl9m;iPatX#JW>@3PP+zNtm(x6jow`ita z%YG9`sA~K=itB=fE=C9 zkd6kq)C0d~CYWcwPrgs~0;&+J05ht&W#Y8odyx6w^wAuS#FJM$*eEI?pL|=I@h^}N?gGV6mE#WNJ*eIi33|fIo{Ag|Obwg= z?R6VZlB!N5Amzulu7!LVYdF^F3k8$dnwG0pw>cz`~GyKayD%{uHYySraD4^dw<9$ws8jk7$b(xNFK0Y2t& zxxo&a%W@`|J_xM+P&O%AgSjs&0EEos&gYrm7bUt49D2O*&xS&d{lsguWcf5-{b}%K z6>;;ukqg4mXPowmt&97D0&Ux_377#Fr!$W7g#_d(?nWe`|FjQ9G$9}dPln))U8E;S zp1C%k#S0}8j%3ru|8d*)5_jbt>G9pKJ{w$HC7Z+^&9i50z2>*h>pJsRqjIiLJ_!{6 za-~iX!bPj?qZ)DI|2Qoi0rD^?i7~c28ZcwH#hC!(B=~>>102q0{=z>AU;Wt5i~+<3 zfIl(6KD~E?i|iHs0`NTX7R(oJxNiVu@ZOyd7{GEzBGX6y4|$KUx3x$1znaYlMl*l> zM^ZM{yPktvXIN2WrTTuG(Av(8WFLphO=u>f9{&>Mdeyl9Qk2ZE0hX>T@FPWBsIn@G z)oe5jI2d+;0ar-CJ@{a5g}Q}ri^l5X+EdjgSkRe~vKB<314qj!^Ml1H6*)L?F6%dS6cx-?I=$Ros)Y^-;9oKF~U58BO zQ@L`Xqu2tp@$Ss~)p>^Lr2;x(>=3FN@LFgq>(M8Vg5!%wCq0 zO0X#Fjati@_JetBC$;4wHG`Ai{jzU%n+vzpaK3n%7Gr!;Ulfp@wnF_30$bH52g!=b z&yVy+bA?EG$g*$cN|b_*X1k?2EE*W))vt=zKPW37ZO&K00`U&*f}KBrhSH}Dvq>*1 zb||kpudx-7qI%zWvK+0k!@7Fk0V-VZU97c;?4z7o@1UAxJR5A zCASc7vc8GUczqAT9f9PXd^oEk$X_5#&`uX2T0k7Klb?Z&87 zRj*-GPvOHM9D}_Djgg!O5N6ym0lQ|73jg4RCk(^l_lr6A6IlwuR4a2!wsU`0xclp> z8t6an_Ee7Fbxj$9+*u7I8Ke(5FSpUms@0X4^)eW%0eD2TGg{K<+w@*fKyC4y1jPe! z$}i31Eb+;Ye9D8LVt>pazc5BcQ7jMheLwPUX}7nt1^Mdkik=|BVvY!5;PYeja5--T z94=$b;Fp!>LE|%ss@t`X=aH>g05JlSj10d~eb2kWE9+CeI_z&v_7wU>!}RRiGCI*7 zhHHkJDf8SPVa59l*k+1F7dxyfm~^8IsM0Sl(yJ-tXTB_cV8V0MtkgkQM!ICGIllr; zlJT0;8i7573A-Z~pL*6~;KX;IsN*IL>*8co+E1rzQ`P5{q1|bAaVD#m>d~~TR>bb# zkEu1N!+l}-cFkaNA4IzDsT(dK5LqoJreflbOx-BzzLq33`@nfB_{#*zm+g{aLg$R+ zbXzF{GZjOE=gd%s0&w9vSpYYIc>|x=)-}d~VRBfZ9?AS4V+ z`QmiNq&U=ikm0y5*HB;(QZhD-0l(C{PN>(8#fu4F&(xL8AlmOd5TBJc!HNE_vN6hU z>NWg=of49fS+#bkT`^R2yh0zfy(q zl?*VLnP{*=XTmdz?64=2u7=JNOs5A^L#w(o8=*nsqD z_Y~@N6Z)fGT2nQ_<-7?`d5phoPID*PJE7$hJHvJH0pQk0-qw{*5A@B|sqtIn5EgLP z8YpfI$dx>8Kb@SG`Z+Nnz7?!_&yKFTjOy7-H&ou&uOJ#M#aM{7}S{Id$HgH-8owL;($%!YQT@ zq7)y&6)uH_@P7PA0Kmn=-kaS~xec==$7f&%AXCtVw6QOgVceMpz0$&QJy(K=A_>tnE?;ybca~-W6Uk@pc_6tP3|dB%qXZPY91L znC>8veR&D_?>%V>%Kku8W4WwcmkAhFTATjb0R2{SJC64*7l{D5DlLHhi=jC-x{kWo z1f`b(K~iB8g)q{t=`v_wE!vOIWeu)M1esDp0Tej8J&ZpbH&MRXwUcW2(H7UAJ9}AN zoiFhDD!sYm7(Kr`U!%}WpN=Ty$N-wvTnjq?H3Jt6zBnto{@gm^Z-z0b(gk+VH^$=x=3knw4K1LNA`LMdsci)oBqW z_v`T#BoEJhYDQ%DJ%H~zH3RlvWB?3f|48!l+W3r|xT>g}KuL14dmwtw+HxbPb)o}& z)K_3GN@c(C{gisjBO8+{yK15l@?M}Q8PgbP`}HKkld}P0d9l2ums1H?poDB0{8M~$jdp>guEp24BH4uAE!?i? zb8~vYYj651Y>XrBC;huT;AGuUcaX6j*ZA0#^^qbqs93@958`7}o^HbX0;3e{A4UuJ z9A39sDc(EQx3;Dp@UHb&3hpkKk_@M*AFY0sWL!@X;4UwaO14iKA>jYWNZBVW;<`wG zrub&XS2I1}X&y}X0IK1I4;6K9bn5KO*~%Cm%YFZMKif&Ngq3hO&TPlpIdB!-M)_-Uw@$bt!!f@}Z@m z}xyGz{u3WVowY;>~ zbwhebP=Z6U%vMLzcNa@xyj%`4N3r<@DIBgBJyD5;B!k#3W)Qg`-#=!;eYd)Je|mK# zHI~W!<3rW(7}uCv7|}RlA=Jyd2>8fQKC2J4PB=8sZ0HYh;iAtW0FNfII)oG9b%nio ztwaGCv(Cm}x96pZYZ^+zRpG6`Jzhg8LIEj~KZPtd#}SF521+dZl-!Q&dqBH9gE2Oq z7Ig>ps)8^9`0GE93EZK4BiCyTK)g)|eFymrff`@KJ>Xhlb@`0qt<9T4!26E4DOkTp zntrAj{F9|XR3fDAF*dO$p+L_SkgTQBoHh7Z5sX`6)NzU_y?=f78kwd0s{OWj>DBN! zmfbPD5IteA)?}!2Gq)eA(`;ERwkh4GhAcPM2y`^5Ut@wFvRJa-XFILeS@M!r>(DxS zxI(BO&>tD_7;FM#ZpzCj+6urD*e>j_4DS6*Q`KQ4K4EpBGosM6u+D^Q2}!JugQ-bhn2+T@5&R?5I#Wvx!G2Uz-MBa z1^#_L&I>od=lA(L0EvzXp1A%U`uXb#d@=9>rJxSz+xwG&wgL=R*=Gg@)HHkfK!0A zSW*JE6?r@Ypx3wbcG9Oy_n06(GmMVK3S~kgFMixRBh&Eb(IGjn8YBm#i*5v6Ffs7PrH)nLW_z@B zG=c!s2mmT{J5M#~7;A&}24uPLm-FI(L8}EJN3DXE2V7?+K3;P8UiGsyyY7(g`|IJO5zT1tw0J*5rB&(CEiDag86r`4 z(subuN>W3v4&O|Kiik63IQ(eRPC9thG}=uL0IS@L>-i!hqxY+fykFB&1?IwzLX!HT z%SHq}HVK}D9^8eRe6icBm)g-x1_nH!y`;!<=eRK-(K-0p#HNP3@qp%gk3ovpb_N=N zbzHF+$&{LvI?7_i>r4>zKCnvMjfsMUA_s8yp$ixtIl_S z&|Hot;kQOo($a^%_&+5Xi=i;e+T2ymz zv}eJ~zz}?vH(yM(C=1GNyz$jQyW!0#E#;dyx7knEYJPWYS3qx7s#x=86>G_c^v?&Ih7E%0d*yMx=0u7vHrQ#nt3nn*d(6)AAf?-p&$ z^{YRHS7uj(G5Zosox{=%(EWv}MLpdo%WdH&5OlE&OoMuxWmJi9RON7x;v;JfjPFnO z;!xrP9vjISgsDsmB)h6A0+#HDa_#amPu&B)mb(Vz4IJstm=~i{;Dk6ar!t_X4j~#~ zn`k-Z#v@_Gv$+_9@s<~wI^_|y>$EGFNftKVV?G@+RACTOlN>nDR>az*G6wfvJjtN1 z*^0d~06)A5Mfl{*B>{W+ZYLB55^wr}rMJXW#Qs(I!*MtFvQ~53T!PYD>DN{D!s{)( ztiBF8jT1*f!`30ep2P^BXez5F21e35Emug*SdGN_>gNIr!V!#CI?%v9;h4CW$mZY6 z4Q^7~?VBJQzY{`pk=Hypu3mbL&KU37xfN1=z-l9FU$pO zE_)vK$uub4?YW)Lr^XJ}GX!_~jcR4#e~+#IfBZ-2-swV4Q*c@-b=NWuF7DcMatJru zqhg&k1>M)r!cnMiqJ)gOCZp!u#YOoHNoG{W$J$NkLJqacD3cq|f>?9IC#u^%Y^?^I zqofadzB3mFNJW+Rl?i=G3*gFmehNJyx(HqxJuO@k9%!B~oYsa4Y%hnpxWl;F2REw` zMX@iM2HSbZ*}2bo1^=gKuH?q1qA@@auw=-Jom`FTN7c#+88y1dnPER0cDNY|)b0Ql ziyrt(to3`HH9@4UAtlb}POxve1-(&yMC+)6cGbs%PcWhTT8wg7kAm#(xlsb+#;rje z;)uqR1pxxSCJ4hQzBEcLKZ{fgf6$4Gi#s0wG!F^ukPfXipDgmVripw&^|?cy?o}qb zbc$VHy4dQ2Sv!7Ooib&}7Ns@V-4K|>5vjv_LIt=z)jTF`dJMrDXJ{634`zcuIepMy zFdeHu1H24}Q!Cw$qEmFW<~L%1Ye-+eWD*ZiM!^GMXB6F66;#nO|Fv4&_1}C3%*ga% zKWUi#kUhQa~YEvLOomb*j$XKuiM&%M03@WKKDJo)w! zgEO11I0KL}3V?M4o^3RX@rz--FCz`P9!yGP3jKPQMCmSHd3{|Cf=e-F%b_7F6Sp7M zs&8b8pl%l9GUiKBky{0_pUtlTW3ZuGJ7!`t8Klfw}1`{Fkd+7O^$fg@&^t%@qI_T zXL3ezGF=GZy2BBdzPVZgA9q#oUznLK#fyq3_s1qs$)GQS{mHCnE5qaC*|I|%EH1(t z9Gsn_wPl?y2AC4q-a=bQzLaPmK(%(ncKRhJy*gD()4k`5QE_;vYP(=}SNd9f{CxAR z$~RTOi*f(G2D|7=PW%o*bS%Bw;!cd*L~inEs;KStq%{p5T^QnSY4!k9T9@TX_5NYY zfnSsfmGp?SspQ&zP;b}Rl&CC}cL!dj>JMv-F?!Iz=I}|~iT3T`9j-kL%*@`A(@vP> z#7W^nD`Y(!=D9#-l_vYd$}tezrp{bqnRcufY+po>T}1@QS1$FtYikptFM_WJn-T0F zc}Vl`K4|!x>zsrlGlpN>XLAG<*f{G64$nXnYO)i^v@d{sLsWt{&4dror|E5&X4qA8 zTDXx^PtT=Wg1qr+-5%0E@v;+ z_nXRx#D#~(CGVmLeJD1-hOw7mG|%~T0bF}0C%aVajsGQnae^TM5o}YM^l3W3blfmw z*7kSYh<0_4QpB+g>M`2|y}xV%*s#7!|J0Hu!P*P3Af5GD=8?BGTcRQX7OjeP?-HQ( zJ3Tn@mmTmNwq|m(Bp-MHP;&D6Rr6L3zdx4X%@aCfF!aaMBVJ=Fe-Yl8JOQTMfZU%U zk#Ms5ZVypJO(zl6tY_+yA<%aha)-*f!tu$L<9z_8$z+e!zRlymuH0>ApV9|m$)%mhDSy@JEuofZ$Rmh?w63=q33Q3(&BdiF1#7*3CMiG_~U z->81`RWs704CRf#)l{Dsd&j1_N+6z!k9sj)2jqb>R?}w?vE(23g%H&=Hejy+geEOO z@m+E{aR4f7&BtXw8P;VQt{0E#W9Ag#ixe)Hcy$bB|68g7%VUP%0LKM@m19v}PGOp7 zxKlaT%dxBaP3ufNUDxnH=SS_Wg77j@Vx=bLf15;RVb{jez|kPTg)M%I{gd6u8MaCY zwR@hMSIq7l3C^i9?`H!lOq{Em?67z z1^%|D=4-LC%*_uO49B8p_pdof@-bNnpBy2SR(7GHS}YE3E^M!?Yxoj_tuV=-_UZ8Y zHnA>}4F=HAQYJ*-sxsTS1%q@Fl66io5L3jl=7d^L_D&-ZyYe0jy#-rZ>0HyLwn9Z( zz$%2i>O7(QF__WS7tz;yO!3X2_>w%>TmD)i+CRnepB#C&MshfdyXF&DMaWN!x@-%) zF4trpi`^#M-f7po3cy1nAQ~(U-luv8QaCnnx_RYm4u?|woREN7vrArWRB@HsceaQB z&xQ}b9fBKX$GtYpn%zMMNN9kY_DHT zB&LFX=8JDAX*MIM$&w4B(Uun4klKiuEqNilV;FE!jtF|yy5-OrMWH2T;x&eOaV*?l z(GqvzZcLF?$KtoI&7P|dvVd$7-)CR+G&DeRP?9xd7TdR4Q^b}(d#)R8J4<&mBN{Y& zAU(pPCb`BFazn>Aw6jJd9V#&fnIdKD>OZw~abp0QEi7CXX z#IVN`Uyz)Z+4w(lIBAwAX6N&WtmVi3B<=`Am0DAl_;eV+AB{PXp zI8JDWAYfeWb)cF(@VCDb??qA!0T46})JPu(G(*J8w_uZ_O<;erzs%rSIsgS1ecV97 z2Y6ls10mIKq0_=%lr!fI%B7vqh+JU7CuyYFY+f4d`-KJa0Tu6Aq*K!=ceSP-a z3!$!>^LUwgufx)=!TP)US6)6^^hRDz`DD$T9J2MsRjNFr&sE~>TnvtDquC>fey=mJ zr3ekA8^v{1O?9M#r|Hs8w<{NJ4^FW*v6X#E{xqb@RU5X&RW7)%rbQI6MT>_bfAd|K zEzx9XRMxv|Hr-(*e%>)|_qoCjmCAg>T~^>ag6D<5DRg$;y+-A~(H*MbL6P0K0`d>f zmx}so^PA(--1i2ffj0WJ>=x%ay99^(wH^Rix`?fHhZe2!*%@0SQMg=v*4o;doK&qa zYBM68du>~(2GU#H)Y~am$?Mvyvvk}!78bhkW9=I+l27dh?ggS~thU95*a50yQRGAb`ZEa_Zp_V9c zDt0Y~5FN_zTi#k&wqQ2=Kk&}V;oKpa6ZLPTmHa2tqV0f4JLnIs=73In2~34IXm%)6 zAHDz;MkU7)*!&~F=giUyOhZ%=;dcxp>R`WE9$27oh=_>Huor~eEpI#(2+%kFBmfZ^ zeIwkNi?jUG(XYrMmTh&tux{Gdzcr$5x!SdVuT@9L--4M?M_2oGJ52YKbr;;8hiHK} z|DIPh=eIln%gf6ILN0u=ZU0=Yf+;ogTHk>NZ*EZA@TfgML z*dB4)m;YdC87>0y&acG0r{ku^51yQ>uJIgspAaF)%&cg6#qYG)SwDp1iag8BKHp%< zB>UmXa0MzNB4X+6%FTii^*qGh>*ynen zpRZSeY528cThcB+iwA*_S%zZW<&IQ|Ci_!&Zg4m{N;_R-C`H`ueYNNoC0^0w=C=0m zn#%uw)6HAx8pL*?JF!sV&5(ui5f)V{z<8G^>UFUx?C3vZZM-YQtH6MB2v9+Qg;5ON09`k{K2${b zvg2XHVZ$Is>@zTgB+Nav-AWKx4-7|N&k(Y0AC(4k&L!~yL5VXEZb}HH@Q+K)Q?BBEwls%E*x8+qQ} z`Z|#?jLzYL*E^|f1fO2xMw<;}8*RDLutWYF?ASXv?MoG0(9xM5v8N{T8&_~P)R;3s zd+Etu^zB@o?Q}^1z`w`xN+htYO>i8si>+l;u zhbjJYyC>DO67KrZv+sKqQtKx|mh(s7%Z%4^$uK^9Kogno;n?nThaLVTz)Z5m;QRRO z8^(}K`^h48V6%3)RDoCM=Iq_ELCf3C`*+=j9hsQ^;TxS~2sI|ZJ|_tJR%UXmwbTx? zfKBes>9wmWMynaVE6a+h1f5?45Ncw=c$>knjCz0eQAc1?)bCs}YF`dm0dDGj`B>%N z&a>Z^V1Vj&4AF|ZFD3n~FGbPG%96dh=lyWla?wrW2et{a&J_Z#vyRnaC?e!y&*1kh zqql~FhJiWre0=`)X^PT>8RqRLG0{6w#9{ZtThXsP(?JNCR4G-TQ-{x=37hOzx}a)a zQ5i`1R0BrBhvZ_#o+r_#ls07ITVJ+XvvI}rjZicq-nE^g5pxpo(Qg)f7uEdD_0*Xs zT7VRExkH9Wg^)_#f`MzvV#ovqj6^RDJ?^hrG8;a{z6$j0L*AA5AQ8u3sQ_2F>L#6b>Iu#u#<$qdL<(9^xig6 zPDspImLE1h$Y>1^qa&~kIGiJsP7jw+ltYn=4d4v0sXmZH_ny#=X28hQ2+n1E3`QI6 z8v^Ex$+`c@n5F#t2KZDT2!QBB>tWyr=Dz3 z`!W>}{iF@!XmzIqqyF6|NgkxoQ{+=+hF^iWn;O(%a|$O5Yin&?ZV*JdtHruGopAA$5cAAWXlN$Ij9Cm095>c?RP00=~x*efzG{iOE}G zREG|(FrZ-@z^$8>QbNtn$tgPZtHE@fHVJE{;NZZnTyP6GiN<%pG)a=f=`T1QAhV*O z+PP(zwpjgHG;;2>C@=wB`yS9GVYbLe!Dmk&vePP_ZYS(`KTc9Zp3mr~_f{o`y2XR0 z2&`!(4(-()@JE*QVQVD4XF`-^vQ=gVjc0Wgj6?VK^QcG5eDHB9R!?~8N%#S;-B%TG z;JN?La&6}H=r`+lcmCJoFd|W9(d!7}C!iZjl0vy^=$}Qw`7T8fE`<@~T9RU7h3-&$ zQJ1vzL6@wcn>Tuh)D*w%j~DED+J@}*X$7;gcj6C6%iW(&%t4&|C(33E2uJZd)T{E) zE~8_Y484w9jGOb+V$lD6Nz;8uxrc`wNgFfN$HU47{QK%mh23pxOA;BNg~BHP?J z*4wBRQXcDqsbpp%URmdoU9F7RG#N>9+8=iWUszX3*LxQJ!L><5;Qu_2 zJWBQRTSLH8!EEJZoA$(qC4*KU!o0?l!gEf`@xtD-FB}%A?@#KgmK-6Y{<1j&+>M{6 z`g^j_iTNT#9udkcR(ynE(YO*@UH4}T=#;aCLb3HU?|yR@>*JT2MvfakJ_<1EwSwqx zXH7nPLqn2_mA~o7v!#}_n+>_MJ45O}m`b`ze?k=}GS0u8v!2(9tdpu>&38R88xuRH z>28q}XiCRp)eHLVyi=NQm*@U2{QBrnOC#~b`_Y)1OjC?4@qmceHgF)GtU$dFIpebD zrVE0J<#&;0tJwka#F*`5q~bDKYcblJ%?&W(!S-2xL>bIN*gQgw-kx|@rde81f%v@E z@%U_VI{EQ)?j{S=c|W`-OmFkqiZ$iqPC^rmW|ijD$ALuKlOLvd31VD0Urp`cl_`|D z*o4q!{?D_eUVRT63MRQ-s2fuz6~TB z{7Ft|&~%NoefTl;SN{Wh@n?Rzvz04+*)Loo+t)0yTthgpFg%X2c?I`u{+h=mb@yc%_vccZNHLL* z56xe-X6|-tR^GwhC0!r?#yv({0yM3ky|ZIl_YVcWwq2<=IcyR@*EXxP;P`;Y>3%*Yc;)SA=;2#f)#Iv4E@eSPEgH-mbE2^o- zVnxWiFYG3p^fn6}8ivn{x||8T_PIYYcYZ7<)8+2|F2?}yKWRlAsDZFv{ok|6_;NNk zfwTDwd9(Sy2NU?0`sB^#7bfC~d5-E60rI#OeQ@*1yNLTe*|o&wHLF z?oXz?7#mj5Rl=p3AhEYQ50pFhqLAj(sn2Y<=caRh$+bAX450ekmd~SOT9K~`Wx*!D#t9%b0*KEi0-5DI>Jl|PD z!_c<)BtsM`-?REPJN)`US6D9@XfG>`IyXZPZ*MAUsK?7ZOOw5G<6oFrPq!pi(|`O& z0_=Je*w_r(J&?4k47yUw=g1R}Cvt_0qq4Z~Fpbd;tF^WAykH?W!)aah49T{`o01(P zWxpya^Sv*MX+-5QX?f!Scjs?X0M8*(i2U>mQ&FPsSPS&ctW1M%Ty-cps>Akqm~ zNK`u(y)!sG^Nn5~a&kd|u82^nDLW+lo0MBs!4|Rm1iTh0u4+FvPS24KuJ&Qza3@E+ z(p*{g7S?)HZ-NE9v_^`-V6dKd8frProqu_Fc(@k&UsuDMb8aDFVRn~O?fqB;>$%4` z=bediuSgskb$fJ7(rX|R77mgM?z*MBcxsO>l|$H_k`XYHBKRKS3D?poXHVa z1z4|0Nj9v#@^snwYCT-BAP+~|1lgA_lIbN4USJ0<3*2>}b%to+|Jd22>b-~vwlYrX z71Xt*GSZ)mAFY{N(w-N+)-h|1a9D^kP8Y9b{un=V^y%vl#-)NWVKqdR#F8>_JX>*@yHz#s^ zdUQ8ctNe_s@Wl_4(C8z`$d}m~$~Wsh%anbmHlCwT&Yy<2`wMG8Fzjdthp(T2o^{46 zF+OWnhn@7tHJXtKyAk`|V`%2z)fC8-t!{icc&B%RvST)H&~W2}NrK?fLu)2YE|2Q0)kX%C#1j zCmTKcVn*^3|8#U@g4S4BohCXykOk$d!Mg>-9TkudyU&?!cH12^spEzi>~({JCF<<5 zF5#2N)BG=ca9t~kw&;mU6=HlyV`62`udZh%nc{%ufdPbIuE6_17=&NDr5eTWu|Nv1 z*}@*~&MCFZG(Et|!dez8V$Y{*R4%)VQqhZ#uW6$22jq6+4%TIK+F)*+ldypvf#Zx0XH&4jZ$!7VqjZr)J=~>3iGn8v*c4K#-HNYtU zW<XvP$_YHjVjS}PzYf)KZ8jMfAl1RyXb=KFoD2N1G9f5w{%sMvkEs~HU_%Dc=@1X6( z0{q&D6qAw_3s;V*6Ao$RktXfq)!Ppj0|1anfu8wt3#^U*C>S@Cs-6j&JfYpp^t z4uB<&FHuk|!_yEiGe6Bx=L=!P5285sSizJF1_rbt?{59XFh1?wHQ!mSjGWY!C%Cqna(w1$T+*sX>( zktKcVKDYmaC;l6{5!xHn!6}NNEEKpiXCzQJqIxg?MUa0O&4YKw>tiV>udqnH-_Gqnpo{Pw2CG9ecs;Q z8)dW(c*Pp9o1Ff+-+OP%sLIK{MV4OC@9A<__P25~Rq6@sOcJ7Qk7mWgq2MzGXYyJ{ z0oX7wVxm$bTscolPwe^dH|LD)q>d}o1((h4hXV?mxE{){{->uDnOxST+qS*D#sx-( z7edjp#=Vg1?FGVi@hs;olqqEwC{V8P`4V;pk2y!j?k^c&E$tVMwK;Qz8GX!4fscm^UZ!FKZ_JBoA#pd*;Be9Pr*z!i-Zd7@&r=5GzdMSr(8Ye{@%JhLf5m|wP1^4 zdiL91(DK8*sKQjO9oY=?3gfWsp26vIcvJjcz>YX+&)Y8~8IB8})?ki-2xchAQ6kOY^|3S%nHQEZz}uHv8B@%!D|RuS z5$H_2O)?1Paag+0(q{`)-Xw{(Y*@Q5-HMr9Ycog?WfyKK)w}ww$(eui^JOgM7eNj6 z`jR9xt*9am%G=4)s9V32y#bxN8=k5q|DrNW`5Z*51^jo*qf=BLTHJ3bl-XJ_GfS0O z*k-@I;z%D`O%W+j5cA&#KWmz~utN?gf+tHp27kB?mlAuPp5XjpbBC-6l#%7k)u+SX z&W!5rfQe15ZT9}tQdwv${fbw@WbcfzKPPnC(_*FVdPKCLV577wwXLwxu7qq zJ4MB1_eVT(zsR3(KtL)j{Sx<2wthU_z$`R`_;;P@a`qS5L`pbT2K_G!Fa$eEZa5&mh)rID8{CJvVk99?6#~*dE4|v3r7t!IeIB~F7yRb2@@HN6da%PZ)b+qH%jpm zp8{zc{ZB^`Wa<&Gfo%8ws7OsHo=@KA*{U)-;d&|+dO7p-=7+?o?;a701g&iDE3JX{ zVx;s@aBm`sCjOSZC}**X#8A>T)xz9t$O0RvU9~~pT#-O@{P+i_#aMrmaaw7ulK(?o zy%o&LEAl&aR;!&x7TGlBTlnxv*vo-9GHuF3_vNAZK9ud!6Bex0=vBjKbqyH4Tdd3s zuc>vK3~tNZ&yP|!RkxSPjp$xcI&Nl!Iw-^D=w_^kG?qxaQg8j%x~2Num-cZJb`P%A zb>J@jZ5BfuSA63oBe6d&Y@^NBn={cqyhS49C>%Jk)-HFfxQ}OM+heNdZz5HRbA)hko{a*gx8f@^`_|CsqTqt2u{URp3fHy60CJeBSL(E?Li&vteMH$E5o z%Fz5_Gd>np{34`=E7RyZ$?l;HElm!@qY%pLb%KxF{pduL5V7Z{A*Oj@H>6syHexD5 zeI&_`ZjaP98Mgl5=c={G#GHf9|=4swefJWoRJaR#Xy<4>lu8KY~^K9cLEQC}?Y7pR=xo|(2Mi;Cin=fPma z|BzWtJ*d{GhBB*c`urRE5pr33Y3u9o2tlj2oqH$McJi}@s?}w`Bnp7>UZ|wW0ctNW zegZa>Pg*yTeI+hJWHSEF%p>$zd3DU0&0xuQam}3AxE6Q9PDrd^&o{9^982rutWRI( zap)8ESi+lz0hcMt>7D*$g{HHe5~=;V>-mXOG6rHf>hDA7%sR?XWM*quWMxYK92~kRLgc+vC0hh!24V`_36C zz?QqL!jtyy>0hFj+Z5_`W+M z_5!e2S(5VS$^8lYb+%6k5eDc?Hk#+;o=1#L8fZ)tjiTlAI3=((*%C=pqY9&EBQEeH zD$t?1?$MP5BJp`a6;3(+8I&>{?Q`G|Jl|zctZSekgYBQeyP}iKoqk{ow9BJM8UPp- z!+CP8ou@KkIzA*83@}d~BJExwo&m^STdhT*qr7G^eQ+ig1{Zmf)AK z;QW@PWS~4^Avp_xMHLT-Yw@I{?`0nhih;9WwmQW%TE3ACuf$2@@-5SG=xdk6eq~1{ z5`VQso9~d!SaPQ1jgmC>SiJEk7v~d)C3$Tc>xUu#i%}Fn;T7jl(s`cRpNhcIj#eZ? zncgUNKO39D8%*F4l=f@+X^ZG1+JsWI!mj}3%V)vdO1k^(?Sqq6K^8q+L{wl~(6#pC zt%sIp-08-8?&qp-SN;51t5dy3ej@jYp6_71ucmca9@V!9jiExELjUB=2lYsHcp(1R zHHJ`-5CCMj7GSQG>eq%FL*NIb&@dGT)D|Jg5xW_sC*en0F|hDxi4HqFK%tS=3>B`B zvfCZcNBIGYoZeyGZ+gGB@}cN$&7f6vv6KBT5u3N{YGU>nLl@e99{XG2K;5P?|9N{| z)+}^#mCVcc_KEMIeMLHw206ayH*! zdE_LSs8@{TL>?2xgN&`-T5qJr?a86;JkDgaYTLA>A%)rm%@Drs`Q{L&%)N@bkdjWG zbTo3Fh_Ap*i4ocf*cc}g2ksm>Iz#UX^Z6o1SEs|I2$H1v*dJX?ITf8?Ai0%PYcUqp zQfEP~l=(yEn>&1Pp5=Hyn?gDR(WcDy=oc<@QamhgK>zyHA0i5Pl!2)H`Fw^$$8NU! zmRrV&RgOnQXl`4v_vDO=hq5e>(>_oDLrDyLljlpo_bH-<>ZsTTR;T6=f28Fqkkcd7 ze7tf1RY{D^6@0jaA0AIE)g6LV~^Gq_F?mh7xJlNQ@7ehz=;nhp4n=D2V2Zd*f%Gq@>Y)Cv!X(!6( z3g;-aPU#%!Lw})5WwU!nm21?7uPEm4&~NW?(M|LVzNxS0xoTr(*s$4oQ%K&&fat_e z9Ra)ae)mCXP?d{U-tRLW$rGm)jghcVR1`>`+%wIk3sL@_v;2l!`xiF-%&zAjKl8dqgIJUGlKN@o>aXG1kI(2Z|K`{!Qu@Rc_q7R&z1KmP3XCO3;!QoUmX?I z_q{D$N()Lih=d~DNK2PADBXxi!_W;PEg+4gbmxE|rIa)%L)XyFz`%P)KPunfS}%Xi zg1Ovt@7cYd{hUeBUV{Tl&CN;daCL^-4WSw$s3^5@L+}$@S6iP9$$qnM*u86fBAFpGd2Z^-32B$ghg&?dZ)$34 zMfqE8_;zgSk~dXVmhjJHN|R4B=GOz18r%-B=i2rqnvexlTLvz8b6laI&(2%B7bFj7 zqK09|qg{LSu{72%qz-Kn!PY)L>9By(Qm5PU4-LRX-vci zY4M_Z5MLmf-LjjlULHOEEz@hMA`lp2P@)SICaZ8aoca;zzWQ134>w(kU% zIPGk{6h3H)26BgN>(ZS~4M&>AEd@U1`Ljk5_oYMZU6SJbo%p;K-EGcyyjm7PjL7hh z(4PIt1(#><(^*~;F88YA-zjmCN!3g_${s7|Blh!Eu?va1P_2F-UB%s-@)6%g%PLTD z!+Cn_ZgbtPesNB&S66iCRtiKS*o{2P3N*VblP2S;yd1-pcC8m?>vp9v$wCpaQXa-? zo~l!(<>)Gz_LaWCZGIJw({Q~W;tEeuiE1#j&`T34&{Q}miHtD`H@xZ=})B4m~ zv^VJi^`bfdou)ZS8@trCtAS#as&-!=OW5MJ?7;x8gRKV`OLxa1*w*5=| zWo77I5QBe?D>A@ZE(hX~J++6V7|8=}2OkAgu_lIL~(M=(_(m#&g}12g4H_FaUX}l?}*viOrwr$A9Ab@x21Q&Vpr~F z2$0f$rpp35Eis8&L)7f=vFO*`2BmUMXTwjhmwnsx>+Q7 zNdZwzpLmL0G9c0813kYCG;F7+3Nm_2Nk89P4m(13sLLl~p>nqAk919z3pH;yJrZc4 zG@5g|*W6}Uh!yCgm2gh1JNhuY>GE0ngQDSZU{* zyXR6{Q`=GMaMSp~fdN=y(il2lX+1>9dB&quLJ@Nqf)yPfncUQ_C8b8_6P%d{#X^OM`GIp>l}c{^)x z^jaC0Iy<(r(}LP(4RdeiXfEM!E*|4Pu6&7;gJ3mhcZ~71C)=rfr)h5sQeTNWZ7$2D zaX!p2XbYTFF=q9Gs_cZ1rDhp@FOeOpDBQWl@bx{}Hs5vD!4Wl(O@`RLco&V4?y6K7 zUIu!#y35byNWU3%L2)kj$Z>J@d_dS>%3FL~A^qVK!UB0Dg0ZhW%f;#xck++EH+OHk znvS7Ikwk{lnT`l2c?;mznZ)mwOs>}~$#$v4sU||s)`(qir(uZMYe*%0+%^5mJ#lXG-z{re|4a7!OlTL#5(7VT=I{kdZl1h+nt^V|k(R;aLalxJG_{K;FKW@Y z^2f(eF|9_-!Cz znV?%L|BiaS8vQU5L>?T!cFOv)@eSRgn|O2V#CJdCYVJy(IaMHlPWK+oK$~IO%InQM zG*k*&4E&mUX88WdPUwDP^<6c1UyPPMNHYginfoH6Y7Jb&&(rAV&fIgSgc66BOlNn|(p|nb z`EWn>oJaAg)UYGfk7rOW!&*BmS-OZRD~ZPUnN^wO6(O#UdWccwAu;m->?!57Z7$4M zbqYFEPXp`ACDH}zX~ag0b51_CvvFonNhRy_V$+(mHVsOTw+0_|1l!6!&L7gto7j5s zS`1mH(GlnagKU`#zZ}g%(WH@K+j_SMVB1M965yW!PbmQ#y(|Gv2YJogV9pZ2Pb!=u ztZUcndur~x{5&3ZEPxEIZO z%-j@ho4=|h;KVZF5CGXOz>dk?_YWAWEQdmXk|%xaxIWR->L*p(lF-Z*2K;f<_4jF5 z8ev4^57{>_Nc}4@qtg>@qIUw2kH3aC3+?s{Rn}~%GIEr2&C@GzyM%xCaUZ1SuX8UU z444ICc1#1h$hEPq3-e}P8A2tNQf4n?&+t!eIRo{qaqcA_A?sS>1(++OW1EqveeqfF zd@PdKI?NR5VYZPk507o_rx{*>Iz<^_oa1dAtVcF4Hxx(<6vj@mTT^p&O2}BhH-+tJ z`wGg0_mrZ+l0s-HbWU~6#mR}S%cP6m0`oLNF(%Ng6g|}Apm$=!a#ELWvlITZ0q(4u zaB2~z61gMFdX=hhld@CNBmXuj&+AsG6vNoPFn4{rIt4;w|HA%r0tXFb^W zK+#&W%V%2kg?wsxbN-UsyWRsv(c+I&U&2d%f^E+)@;~v)rcLtFa~pGyefE@wI`a6# z?Q7unZ0ak$RvR|*xgC39)GCpwzSQi=w|LtnFobmxWn&(48}CDN_&pshFCw2WnzKCJhge`xJ*ftPb`}?>s=&x zfjOKe)|SjlZJ2ywJqIsW^a7-<&@6khL}pe|end5|&M4mz0tNBVZEG59^pa+lMO(dXdI7!3QA~@jcfGuYo+g@-Jo<>;860 z>{fFJ3obaQn&aI^By=TGiIu3pgG+M9sDGQ$u!3{);bdt+r(S|b<+L90RRIYc-pCtD ztqIc{2p#h|StRHl+WurO?jGfzZaIqyp#+M(x5XTf`DQ}GVrrF;{T&CU8Z6uDMqade zYDko;6s+eL4%NsuN|_Qka%wAO7(3Q`;d2U}qgBSYLqdDZKBG6F; z0!V4ArxZm%J@ZvAJKWU|Fl@Ujc7iz&sx%8BoAN}E5F)-jn(aJf2WqD8NZ_t9;`eAN z_C364D1f+=%>;lzUb<>OAsuL8V9dfB&e_$Gm(YFl;5y%m+gekiq zgdCBnL!v3|%gGLZW9bYO5w#U$mM*9Lr*A&*5r!MmRS7-R+WLviI2+9VEa3RPS2st8 zFZ4=gfk{hsDeYuJdehwr?!V``H|BuqNib~5seLcn@(5>Q5WtGDZqRM3T%RdNj%Cb! zS0>HX$0?FNYWuV?DHqR-qsKv+AD%iO)O0Lk%Pmn?K!SF42rjU7D_R}+*~I@-P_Nm@ zIB16MvhY**ytF3OgxSXEAQPIQD!bY$<4lW--tj(*aMLTIDqR|%><>wFt>7%YBk6lr zTiqxvVpT-gyq~O!9CyV@hZW2`2wEc7=L8L>jyPR$GacQk1ICD4@vfRJo=kw0bU0(V zr7zjjP#HwP$M_0qx3!L!+0hO{{J*D@6>?1Ol)Xa~`Anpnr|m ze{2=Ol|JATMgvm&c_o)h%QIttIhWrkH6hk&Qy&5RS198Kw05qGs#ha7LCMFk-KbJU zA>oo+HOfxoK|;KOKHkfVN#{MIdOWq0YMBn3U5d(Eq`W(_DI9my%Djja8Y_P6eTmt$ zzf+Z+w@@>e#AQi&=>tOF5gDH|Sk2RHp3;_!XM1HBQakJDYm-a&-p!5|c_PQr`nvZJ zH$=tuJ{z)+oR9C5S~LdrCMoZsv`*n6bulhv*t3sh3qAuWtKA)B1v~vGlq6d&s0_io!26LGzxSrpx% zH&WoAilHDcxe^A8~;DeAB!D41=WOg~hn3bY9^1rZg29kgprU(V|m+ATrQXDUr3+)ee-BpHiwdB0=nKvH@ zDe7Icf9@Swb6&kMoiqe=$KK{N3K0$~NL~0Alu871rsZmewompukPDM1yHUuLn5Ry) zWa*B1IYA}1&OFN^9&yI52IhmA*?Kk-eP_ZhD8olTZij!It=#R=vN5M=%ia_ z&}`^nkrHZEO1q%_{YNO;Zq$;2)L^Ny;#HQP>JH^%9z^Y*@n9t^SCn;1RZ_wAJ3}xX z(&5Nn{jNEQ(VdYS zG_J4KOfKA}136Z@vv83N-FCf>|9s6TU3{h5rH^LxD4+Pa}`f^@I3Dw>Ua-Rwb4&^gyINZZ#fSnx+S&kH)%iZ-+rxpN|G0Ej3%fe%pkL<@- zPKqyI0nuyAhsYwVEuzn*0CZ2H&t6_POB@*4vo1Z)&HP*#^qgp`oUGVzvVO*8TteIr zT)Ye9B-EN$#I>y6c{ykLWIB}Jj08}b2jL&X;U!t}rBuBM9GB#jdCfmDv?X&E&6G3A zOQENd{y#pJ%3z(V(#=79-%$Ee8sQLBYIDG8XL!lXtcj_RTBWMM7kkBv3riPzM3s)i zIp5Ow>Zt{)i4G@k_cR9=7qMI=Azk^--v)>|ZD-PONlv90**&kBqcJcj>82W|_^nJN z?_!Cqrx6EeS%hR`{etN|J7r}F<#Kl$}k#g0{oQd13XsArQNdJE4&$G&XC z`J|yR_=KzLrN*lATZ-{28q>a`)QT<&k0)K|m4&Z>WZ*rw2R0$&vmkczNqVQnF5Gg^ z#oP3ISKBOFqUHxu37n@HolR`Nk!r6fkpx#QPKXy99PS9(E^FguE@t+X%tKcwH#mqG z6axC;dee6=L&h4x5dCjOV<@k^CS9%--MeB)KY3aqdqP9MPecPj3S%lfF2tBNok=*U zcPow|?mR#bh`UT}yjxtOKBl3JPA;r}9*|2HdP4Y7&Zch?59o7@G; zpYL`XFC}*f_QR(kSKa4#E{<7~hAeQ9rITon8;gdx4Ip0qWqw9o*Hu_qb9j0+N2&fJ z-?xw5pzXeAl94YmYTqDnpM%AJnnb0e0K7m}OI@kBOM5^amOD)hGX*k0{V5a}^;&pc zu=}o3&Y?!OSg*)S<8cTA|rASI}beBTO` z2nunkeuaNf%nZ%pis`O{{secH0+!tUakfps^dp#)9rU$FCX8G) zDaR(5RFp4`E?yYb0^Zz|tY-F+s7_eS<%2}yRv(%&S08AzQjpzA z;qce}5aia(J&>rU)BUyG9dW-tu@O8?6{3iBvyG`vNIGGvu!;h#y=K95}C8%E7t*@88v*^pED4y8z z;+)qijjy)DNIKwnADTTe=gN)sy5mJv;M+QbgRj(OZdGS+@YU= z|Aomto>+a)5YD!A*JHhYsJ0O(%u-#>27)OYlvj`b@YPXHWwv)U>hF4~byN<`SS1d>lz zLt}9wI7ZA!%p-6zrq*|+Ag{}sh}KDPJM&~Cc90w zb=2J_wAF| zP!GC1^~!yb`)me1dSdeq(m}po$fzg1(qm=J;Y^$(r%PSYf!?+K zG`klc6Ysdi7^^!HW3pyxle5^IO6Mf>4LdBI7L$j2`szjMK!DPu*#qmjTYCTskHqSj zkU=TRKq~^;iFToA%u+yjEY~qd4f+!Q)y-}*?5yqe^#rfS>(}~`beza%ds6R`<<4KH z#|=78g&IOQq-s0t)oRS|B$AGT*44%%ZPX^g6+|u~K;xJrK{=L8X5uKv%v2DVJ?8qn$UH>JSro6 zM=jhaZU4OYJuikJH~CuwT^#+*^u{C3Rjb(Dl~Ux75o1z)Psd{oiIyVo;7oub0W3NE z()*F^%8L)J-l}PkmO`B>+5;PMjw&m)y`Djv7z`DM8O(KUhm0tZ(1|RdL8IENz8B4Y zf_lWaYuG-sEv9QXiKhn|Ca?0Ru}ZC}5aNLFhZ7jH$>tw529RI~O-^pg-a*f@nYQTY zPgxNC4}Rz~5f#h+5z*o(fRtUo{q8s^&wrH%Y5R#B3c9#$cWqhZ8uQ{Ji32FKF@8ot zL4XeZ7p>BRvTERr>8Etra}B$a1$W{|W^oZKe+T zqF5d7lW1IIb7!=xPvh(-2{{Q`c`T}2dnp!xSnX%H_V)HRITlL~)K_7O=0o+LP6R(v z;vAnof;A=^dXbEiXZ5#IiPkFA7T4j#bzoFYEi0{GWU?y|xyrGdkbgSt|B+ zzRo;mJAMEUr!3Ik6j%)2HXf?&OOB_6eA6%o($IXg0bJGX+5z&-siuP#vR0*P z);E0TV3(`IFv;r4hvY{t)*=o%Z?7p@b!x%qyD1f#ZeIoYc0N3DAy@0*_UvYM-S$0O zu4APZElpH9&(@gLx>}Yteh*Z7XA{`rrY)?aQb)Y!dZ?a><Hjz9rCW_%W;tQ-?0gNB5FqqRBi9Uf}Ts5558U=MFS z@x)XWrxy7j5aEOg&};M^!SUi1kI8+kSdEK{r4@~VvAt(f*Gps8`51V1X!_li8A-n< zA9N5CaC7`w&J`A&ri5Xb{&%T;N~FMmg|C_wW<@;`HJZy$S!qt0ctc@%aO!27s=uu; zx_M$6811_@C2gL@I@*>0Ql3`Co3F~xh&1ooAU{$*@;%k&M<6~C6Lt$Usdql}ZuZ$G zYgA2)wE7|FVbpAC?Pjv!IvQ1#nc{8STBoUdzRxn+A0v~xL&Y~VFcO@`ZvHyU@h$(} zp+*(pj;`c|MA2ninke15G8Y zUX$uWa3?HaRaeE|bdh0KOJb_<)}yMQEDR_>CVSD!gH!DlQ)*o`$?<*!my5$lD8DRQ{*E=`zU@ty;^7aiJY&L5-^;R+zHX5Ple#zbgGgL=9g zo1w^_r2cCaAv83`Ccu&1Pegpk3@Bb=qFAH;8kP|c1B(9|;?#@CFJCrbt>+x9u!pA2 z3HzFl8g-2jIvJpT9gGs^3<&;{wIDsrj1kFS=wiJw?k50Jb!?ElTg%#mTKnj6(VRAw_iEb6#excb*ja&UcZF zhq?rv0&Y9?)9ka>9+llx&!|s%IbB~pbiR)~Y1VwB25sW9*Kq698*)tLjzSie~h{!yzd${g4_V&D+0a5z; ziEFB!qqQR?ermo^FO*UHZU+r`*5na{*t)e~M{7I!}JCx#iV5ByH}qu--+ z4__^>KX%dm+3YGY&Gpe_hOrph=)Ii;`<5)^71bhYMEqyYOZ*$-E7*~>ixV0)aabei z{g8*vuNDG|wLa-oJ{q&QOZZArJn~k|-BR$EYVy-qE-&=u>v<6Xz=O|slHMISsvY~t zT-{v^^cGj2na&!fne=v|hB_3(vMlF!BYpn#F5hWpM+j#DjmmLcA>Rg$f)IQj24Su& z+EtPJASl4SAS&pP7Sv@O2mO+yQfX@+y1Tdf<>7Ln4sjzZsWz{NQDfvG`Dv~@G?isM zJL4lr&NW@PmE1SlIK)k?DDat&zV1wrE53SW8cebCxV2!|T}I4|Q+q1EgwlBE%a>9N zwcSvl%==_-?z@^VXiik;xX+4*wCJ^&RGXlAEVtn^pbLHCD_m`EE49A` zFi=?se!MLEYV2uF$MKQ+xZ;uGF%48y#iidymlW!q9NemuWLJx z6|%58YF*ylViYZle*H5|r{956I8w}0Q6?ZSycV>SGg|f_jEHYKVPANwF8V>rDV1U8 znLzpi^c4Nvxh>LUSMzhpB+x)o>fw~6%{IG9aSWiCa}9ee0In!vpBa@- zp~hzq-d1blN^BC}-!t0Bd-p_vR@lQ84@22u#I7MKgr_M>qE?xNoPtu=_xv72(Y7e! zVM1ZdkLd_;oKSz$Yj_bqNv|PU>#S0s@a!n+_C1j$qBD#$+rvZJ12fK$*;CGi_9Au! zgn&x8t;lX;$}Y&fVQK9|eD;z5W#hBOTo}qb0Z9j~y5-YK8;2^3Z;{x$yy8C^sQ7)H zsDTv3e_39m^Y9!cYO@x5W7Mqrh|&4Mc2^RYwCzLvU9qOJdrZ&XNg4XCe2*Rh8f*$_ z6;u?0wRdJL@Jbpx613kS@6?xbAOsxv4A?qb;8t`&WFFYR|tfZQv+A>IWMeX+}e z>T2?U+8~gV{Lc9$uf?!z8v+1embU8Jdo_8Wf42T+qy7*$0`A#a3+4QwtEgi#1Vhwu z5`6c}+6*z}65ZY_c`@rn;ma=g;qv7}rto!XVKL_uVe)ReoZS)jGwKt2*2V=av?B+8 zYwnak1FGT#Ks08ANxZm6`Z)#F+sNH?vL1(<_DudgvSKk5z%}M{^KCo1{)7zYL3g56Q?Q0D@{IOQC|*pW_7MTlp*J1RzGEgzN zMbH@8U=SC!pio<0B6aCp;)4auxD#ZT!=~6?mF}54r$rMWCIPuZNX|9vcZREjmBM3e zvm5HN#d-aM_y&qDgBKyXv&8RMw78lF%t2Qv4alVR6kYn%##*wxM8wzYJ}r~Sl4%2| zu0mn7DQ?t8Us`w|^K+fHn-Rjoi`nk}{bvmw+V6qf6`7 z8@TGyHYPRem@O@e$CNcm_fb-64JbYrZ1zNxlX_H>$~H4~zJd?xelvdZ!y4Ih`WB7J zUgYEe(p|Im7&2U`@$1NqYvBq)x&u8reQgQW8L&94EHRyW8AoG$-U;wR^ zbz7}YhcdB1nztttLt6qy7+aSi#R6FjG-A=p%g;b#Clp1Lwwm$Ed&z0EKmljrKq9WM zBn(mWA|ahr=gUJOfS1ocHZ)A0dLU0)RR5j#es2i6Ncgb#msNRpc(;Q2N4X>tmzsS9 zkQ1~7MI{;rAArgRT=0%DsHQb((r_5C6t<&L_^fyf9Q3r@oE#s}?d9j6CUA4p489z4x0FYQRXziMUaI*(B z;4UMQ8WXKxT-pcdDB=0kS{vh?1@52=0(CQ`7_<8tvg40BV2S4*uQ!r-IrnR=j#Esd zUx~7Wks1L5FB^ttUlXY5_Hfzd^IXDtGauXI-i;QnfvNATz!jy%NjqAlvSW%FVj6`r zAw8p{c(IH=R1ETV7c#fYwTJf3rVY)8j&RL7#fKTlQK&>4OR;)0ZDXO%#l71S(DgOW z7U8?0nth}6_Ql`y5?n6=+zFNy#?FKEh)-@?SJQ)9!hF`k-Hqw)w;rErY*9KT>@Uag z5V{z3<0uezcoZ*hyi>ohOsDdg^Z$gWOO60M+eu8uge z#D4?L@4^>~G=M2mH*A@LH$doScMsUtXbsJApkgYQ^d~aQMbvT6sT!@5 z33NQWqXAY8erYXspjK-3D{VnUeAvJ8*VXK6B^9i76l7$RZ`9RN!kl!T@;Ix5sy&|E zu+bj8hScWK9!pfu0($e%zsw#ZQa~^AK6fzxNX z1^SmXsld`>n1(4rZv4^@l}-Xd_`$%@@jyvhoU`ei|LW?B$xvxGA;WO04+tl-B3F*X z4ktW_hnru$=I8!*CsbO*>}Jr@4>UM|1%KCt`0xTqrVx}PrA)#97L51SL=DL3H~kWouAY1&1!acLCueDxd_>okpZ5^*=Y4z(SB3*l54W><3^;n!BY<^Q~5unBoPHTbXiXe>6DdI=Qdm zGRCekBDd!47&d3g)^j8wv1HX}%_IGL)_W1RD4h@AsD_yqn#-I1vjD&>j?C{3xKsDY z3isay@BFe6GtgsihVR{68)1b4BKb93%6U9W=x?MklJ-Maob$5>fI4E;&nE`p)h;=H zbE)Vtzq{aw8@2NzjLcfVQ;MZe@_J8iywMlJzY&EFEwvJnH`f#;2~1f#tp@y8Z29l4 z`OzaH1j=dB^z;42FOn3#zP|CzGMV)xLpTk9=%X&lBKbUG_;!PcLh=`886Df{b|uAs zdH8SM3O`^g5xfJ<5~P>|H$;W-5h}3rA@9-2ByOxz9GR97!BdnIJEB_nB^{ty!B_;C zOV}lzes9(XxJ&4Ak{J;1-)C3~$w!q=nE9V5nf_V<1|WNtRMu!WU+?4pR`7Tdqx64P zfR2#8_ZWKrX9d4_u;QXe)bgP#iGHGj=xLiP+) zL>>6MH<)~g?yDK`VvyHH#myy%v(f$bjg?j-`|3BB@mUElkqxnxH@|%gOG*e~7Dqq} zuILFYM-KP>jkjA<6U59NNLXWnSg#SPiNczRV`YCambgX zRhBCWGXMKq;6r8lZ}sdsN$iE**ik~vaKJFVTd#=v-?xD1^`B-?9Zryx zhuyeQy9Y21DHOx#byH+${0kA|4I;&5oS)#u>j0OI5z$cEau;QsR2*q`Gefu_=g3-m zY)-MU4Q73i$80|<+={@}K=1AQ| z=t&jvqSQFHDfp&*KKB1lK7+=)Y;V4sFjfTBS_Iq0P5w0+q&0*A*=czgp`!xG2Y|4) zl~~WVl={@XZf#DGYevEz^i;hdKVZ=q}!uN`Xhrh-x1cF zKs}6T<6{@@O=&TC^m{xqrXw;cS?0#R;9CO=qK|$@dDHv76Hh?!wNR~)lJ(!>B|ro; zS(ZWH2Uot(xlc;(o@_|KmBze4D5 zx4Ap*+q&uJv-VJThiD_*0Z`Bjv9Eoh(9Zo`K3}o3iw+3h6^Ngq{TkfuaQ%7Vb66bm zFR})_l{P<6VBM7mWZ_J~?;0`>o8QIo6{8@SKScpkpTXzX>rrA)$|+7M?lJ;_ncVpL z(5PfJ=XYovHuGspi?I`5Hc;~jhOv;E+lCLij(%9 zQCR2k|3*L@zp916(31fw#m$hdeGfs61B$SsxuHM^gASCc3!WkmD$Jkrr^Xr86?PR8 z19VfI$&p5n+(HOPnjhZ6UjON`YQ^{WI=~!|W#NZ{ac;^R!~A=M!2K9;1p8hiaTv8k zFslg1KG3j#N#^s13&4|@Rr6Mn`o}l7(-{BfT6_sW1Uk_NM-Lx3K-wbR_k~p(UUgLP zL`aoZTXN2C&(^zo$)$2XI9MGpua8Q(dF^9hoE^Z>jwY@j(xu0^0FZZK1h-0Kv%685 zKeA_b`(M8{{}v&dmIIW*{5uVER!{yu4-n{jAp+e%q4VvbOmO=Lfk>bu3LGDePqDwePDTc`A;KY z#nEj(&9m&LOgHW){_S_1`UVKf0OYUsGZ?>vsuI z_fVNR+iK#he>vU#>4m9Ypd9RZY=idXk#xi|i(ZXtg}22`-R&|!ST#c>9qz-p1V4b> z;Z{@5+<1t9sW)OG0pV+1}o2-2KYvzFe z-ta3N{sBA@#x)pThXpW8OYZy-uwS+R65(>TmWfiR{t;QP-oZwro9M5WAU=fM27CeS zXtU1+F6iu~U<4^=KpLN8M^6u5KiUzevFFR)cWNv;kHkDrF4@dbbZ=@C{zC*!nH!JY z^cX9-a-!edcsIWn;voozBEyWn2Q)sRAy_wkn&T(R^o_3k4u zVsmpf+KBl@NyWWcPNSA6>(Nyj$GIQtv-N-qCN153+AbPEAt+VyF`bzd--m?aQ7@!9 zq8FPMZL=V^7OZL5&7Lrq88&AJolI9-O4QmUz7z8`>Is~B;ITJPD-)9uR%Nb`?Q_BF zYV#=ak7Y)rZFRqlyv+`Pee;JENasE7$#JWdR;CaG8g@V^@?O9zgDl%?l4E_OG?GZx z*6wfxr6w%RM7%gj__AMUh*jQj^yY z*>0oWW0(tcJ620a_z-A0)Q1pKH&#du;K78C)<^D7SDC9Nvn-x(jII@9p3s9Sqs;o^ zr<>n6XCxYPnq2BuldvNB03?A<`-^wGV%GdW>4A4!we_NjKGBz)bo6V6R8FM(kf+?R z0Khma+jSqy(5#_|Bw)&<#l>$W_!;3c(C0HifdqV#VbOzNTfi?cJ6ttol;t4p+yJJ6V;jZp#fO~5LwCx9qi;)FfaWu?FUl@}WSdrCCA zT&D+t07-}OTc8|pc6CwYe$YiE8FD1q^u}8ABM2TceRud82G!e|b;Ac}JaHMH5!Y_) zZA~A(e{AYc@}?X$Dw`gL1RM{Y}X zfq!gC`yC)V^=~Q}X>7*t8NbZ2xy-`51gPW0jv)dcE_XkD$y3n}-p0G1gsv=j54@GX z8IJ79jAYLD*I;lkBC+d!|6H(iK14BEd#`V+wm zgXvzLm9T`mQazr^J{(Uvg7u#03j*L!45{6@pMcrS$kNK$`f!S-7Lr8y&jO3T|3y$4 zU#Vsw|Erp&__}>L1C=GqZO9{c{O@BRG9%VkmbA)>Mm+=`@4&AQ)y_N(>t7TZwz6`W z%Q*_fl%W8n4^)`6B(}`tJBrXhFVGJ)=eIuts41P6Nb;4_Kho)(>_UiAo>)dzHDVVn zg`G{(9`|tl*&{^Bz!XvV5uz5m;zt3S|OrGIEM!3{pwoa1siEi3gCwVhzInrn&8T}|>1e|kM>e33BKW-}g< zbXh1m?<1i}WO4oX>3*5?vu#@a`yw9?nDY5&b^usRL71p#mXH@Q_IkQf0{m}MDn*L$ z4BvNHyTZ#{7K@$e4rq*npr_QL>P%u@@x+0%7P1`_^0{zAL2oysXn7=`n%p(Nzjspm zl}YZb*NcHJQRA85pSt)Fmf%YyX?VbWZ2e?{{qoLiaMbv7z9yqLb)cb722IbSwZsmRhFzAWv)>NRy0GS@@ zx$>r9mxH@8J{=XwKS8^hb~G7@zLi(V9y71>#{%jC^L^RYe^*S4$Vt1G-ah?+&&(S_ zEVC?e)vN2fv;ob--*3soM)NBh$PvSC;mC4J^b-qVLo!1DV;lT_m4G{bUY;%X z(C1+KYXjXp@uo+;7Bt&tn`6@rZ#J+!Pj@1KSyy!Sw^hz_q*C|wTowQQ&zHXn0*c)- zFF!UoEfvoJ6(fLytKY3cW}&ugv+O}u%=qX~CbB}ru&D+Jl7!Ur6<>~S z0KwZk7B#vwVOi$W9g+4H4{(-Y^TaUyPpAuEBK@T!ljyaI6Yhz6XECj+WRXgrGwju^jfh1CmOACekY1@oZMugx=Y`!bSxdsaq@0sse~`lN~%BZcChnQR{daQe?t1>;b@JblET1I zd7V{iQOdS?qn1t5g;Ajg`#=nG8W_j@<^IJ@S37q3ku5ZZ@1j`M20$lI1il< zzV=-c1g(kg!kH#hFGYJ}-iCkaXV0!uaZ8r+ARbwl6`6c6>11cv2vVV#xTa!m2F_TR zcr#7?pLqRXNA1E(T3x)p`xw1-i+t;gBNzln7rlJNvA9^4Wgx)EZT%hdsa1SZl10_) zJLKqD_Olqf3A)fp$VQ(X2l%RXT3JW#&(;AERTix{*bt@jeJuvqXUo$ZvaDMY9C?`EMN<0(NT{DRX%x4`}lYGGXqgvuuh=SSPcQ^|ClHE4Ea&R}$ z+kW}<*>LQaaac!94!z?_9?cjM{BSs4)ciMk4Jigi-x2>Y4O(pf<(JJ2w1;IVDo|lr@sKrt?a{Y#L7V4D ztx3dgI5_`3!?1zU`93a3S>bR47TFOSMdf!f*-n~Oy6+;E$8Qbu4cL#MKR&1KO#Q7#`JqOn`x%`eI!MiTfnC(+Bh&xq=5JvU z^8ltsTIq*q&r(~v=wx*D-`tu-0QG3Go|I3`6xCB&IumeNPMoqGJq3Pk^IKDKe;j$1 zJ0fM>n=HX+77?NBiOG!PoIyv2t$9|5t@AJ|OVO}y5*BNhNBG^kMwYE?1az@Wr9VT6 z^rO^xk>5GRNW>1Jd4TsjZ=foa+=I-o2r|8Fl3G8B)HrS9f(yW64@wCH?O8<7*5A}@{ z5a_)nk?he}XcO;=>@50w@sGQRr-dT90!R5QWMP_e-S;uV`>IwU0PN8t2fM#@RMktw z9!-clKBb^d#lsQufA=mBUyJ~{vEg5&!My6;DJAux`$uEsE9mp$k?!&FyX7qhIdu7_ zPm*-w&et+?fdC#&U0q!f7&ilBz%Ncis;jHzJzi!2n|*az)J)*PrmSqezFc?au*1Rd zMWFYvZVK1YKt{81ra`9+*BhGrG{?jJ^`t#|I!`P>a9i7M6a578+9uPf*{J!jSeXYBOo9l zB_S#xASDeF(n!b5&>hm!El5h|FiI*&Nq55_-Ob$J===0 zea-=DNg}#^)>gN*V(KSPYAqTS97y~0o9IH1rq48kRkng8ebX?p55xc?0jQKfSM8&4 z!FCSiMNXQtPA(>#v#iw6P(t^EpB!iP!0?^J^^<3^&G3z?C_9(ZSZzvy;x6Ecn>e^I z&F7QY=*jm+mwdcVCr zU=n7fXo^D&sfI>oO&z8-M7>)NXdQj7Gm##q#p(WPGA-cGKf;^=H-Go*((JsP^MhL z-N!ZS_{$QsJp(eSH*(tVVQ{eg*RNl_h)a9l?--VVP$NZSelxHM-!H!DlA#pcVS=&5 zh*Hh6n@VCQRB5hRzGb3$VN|vNuzCAEm{czNEya$jiVBJJ%d@QKKYh=yqAxiEp}S>r z4)n0qPX6{&EU1U!%7^ZU>7HGhth`Y>(+^W$P{*Z^_BU_cP~7qfQ%P4_@jKI%1)uRzL-dl7F-U zG&ul_%I~kwO@QE0vvt5(Fy^5D;$@D|e}vNaY#Efusq^)alFFAKRo>p0&436m`}30E z?)&@1rT&SQWLN-xzV_9fd|s!)SvUB$_#;5AbaxllgI~}Crm2piqYqz$_I_H&q@;vh zC;h+yTHu`DxJx>_J+u6{SWt)q8E|mabXBw75qG@W+P?|v@&4uGj~X!UuFB^g@fwI> z0Y(hJlPk!%VJJ$i5a*06Juq|sjs`+2T z`o7y`LoM@=N5*1w{|-lIc1g zI2V1($37+C(tW4ibAJSus=ZivOTp#)-?R8%!Tcrq6@RXm9?KV|f3xM4>#;Y0y5L#6 zFvX#oo&JkazcKN@-&<6afWSO9EAG|5x|`7pyfvEWCRD)klj_?5hs?!Ze(T?7?LCi~ zQ!JrdDwWa*j9`vTZ#7Hjk|4U`IZGsS($QfV*{6wam#3|F#pyr8#|qbuh)G(; zFW3z5)XHBA2|Joc&d&CX*{lKtX1F+dk}Z26l>MMC_d4lxeDg_}v7AwVR_;TWO`m=p z=^V!U8R_8WQj2>PtPTK05t_utmiMGVZQ|zVDHf34CPOW<4PiQw78cfeeMu>Yl?c0E z^MrKu_=LGu1|1qIe$RJc=~?-6&)3570hVi5isWtm2L#u?zTKMOu@fZ;)LznwdQnQj zA;gvyA$QOH(J-A@XDCj5j{ecPcUM{2kC*r;8JZuDKF=SPWBo;bPer|dnPG?`vX%b?L0q7Z3}G<2S*<=vIcd$=v-XG0?W6r2^kr~TfZhfZoYVQV8yyhL z*aX6!2(CZ$VMqc1Sg*nn1opcmeOuEPlU*1LNfGb=A1qLBZfTujhMc2Z^55KC2_?Qg=|V-Ay%4* zl9KHr4QlGVY1zeaVNGZWlB|<3s_6-jp+rG4XTe=%l+3Q9i(Hqf&|Zuz8XBf=PTcz| zxn6|MyH{BxrbbrN@BPhvlvuX5N)7b}7`XAXW}?f;^q_^>dsoG*&s}!H0UMoz`(#9%3MDxFoZTkuRRYC<2b*#{SjCef1M`J z@6%)y-iJ_d>x2L;tl2JGHe>Pdo%y9vB>IYMB*Qg0x_<2EkWXdhM2OcV(cQGIA4~R* zKlk%ru|xtT4u1p#k=#tvqrPfu&X{_il>DfrJ&Quw0(}H04JhG<9e*29>EWT|g$4Im z60A~~hK-vFJd9U!C}o#C$<(edenM}PrpI+A5vM=zVYy1me<)Nk zjAZU(-!SpYUf-^!_+jav?NSo9zo9uL$XkyYNT{1}pWb>Bpo($6F8}#Z+Aw817DCVe zq1$y76I1Ve%Pr)){`qu|&5u?`{gZY23!`Or)Iz`p7!hqm^=!D{hpqva8K|oEy*-b^ zbANKq;Y@{Ob<{|yT=i))#|y?qIoUkJFT=FNS^L-cr|uV-Wzg+*mOS3J8hO6FQcNTV zgeS5_ZLl-)tpSZq8~r`PTF&yM{z#UaIx)yT!(=qt?2^_oBkoDU1>Qv|4** zqh_Whj&sKqeeuT5nLD{yPkZ|dp8o9qYDLJ#%i7xavaaGTo7kqkmt@^Z{9oSPxmwvx zz+Vb|hqRc$GL}99VA(f&f3NA>4-g&86C(^%tII!V$L32s_T2)rI3tUK8e5JGixg%kd%GU7G0|M}wg&<}cX)6kFp?q0x+}KOf7>CL7XltRTAP$w zKKYL$>JZqM1^~Cyjl=Q}uh6f)Ct|yMXbJVi{EOJUaP9$7ZHiaXrGfki*X4}<^Is1S zyA8Jc&O~UkuRP9aSlEqx+_pzN5$PDx(aR#uFRl(Uavkg@4#dQ>ks5joS&h|}t{x-i8QtuVJ)7}!KG@}nM$nT(AhHz} zw|%)9ScY0NK@F}dJFf25%BPAdkFSiw4=E_(%o!IMkB{_Q^${VhnzLnx8!z&rC>0gY zt}$#Rb+PX=KG)?ubKL7}-Hy8E4Yl!|4ASwZ&5Ng27&PDGd-eQ5{Fh9!h`r zr)V$Ts#_M)J7EpgJ=svgNp*2_d?20CYim5d76@F%Q!T=0rqS_H5wtZbb{FcrHaQ=@ zvMzN!%$CxkjD@V-$Ww`kcYI9y&tJ+8!qlSi%B8pVxj=ssi}MD96d<^TA1%$dbpgF^ zST`8XZ@j8BH@!!xyjEj#f$Q{hu=(O>CO8DPX9{LjO2cw-aS5Y6Dkt%u)~W_2$481^ z5`aK?Jns8;$7drvhiaqL8@}g+c_oJ3E9lNU2wKzm%-gqI1JabieHxM$FNl|wCahC> zipU>@pQP|of7Gx)IRbrcOl4M(COaMIOs9eBk;NB|(aUe@-iH@t+Jc}kRBW}7({+H2 zudSj4!}u$og?(#XguJCVkYfmsf#58T@^JonlTR|22^G3OS=JJE0w*G&r;`^lFd0crh|Xn;V;-{cF7wP#L|5 z#;_hmC24*SJ&poq7r)^Pl?@KQ1oAVaGoB0J*ORDh+sx8dxn^3)acgXH=LeKl<^V;{ zb=Lc~%&Z=zp|NU!tqAUqjIa^qPw!0jrnIw2~(OiHhi8? z&TJqK1B;5=^x^)q$2&1kU!E>Mbv(`U*NS4<@bxVAYEp)@9Nt~_5$R<6ko~xKS+mBS zlEm1nKD@CXF5Lrm*;rIHIs{*CTS@GYSFo;h9<^kTG2`1eaP+vSWlgNG`T@f3tt?}s zcR*>}q*VF2qja6eIXNaD-cevYS|&^{)v&C@$Z*$hOtwBQg!Wb8tAqm^ z$t(7I>#~y};`g%q6PydLC2&~JsUs@N0s&HELDux%WTQ6_f~2Ig1QX&*bd2vY^wm&` zy|;MivWFcI`GC;*rWt&m>&Kbee(JMSk7GIM`Qix zsGod>VIWs}6-Q6vTm2U)`$7VL%Z4kM81KR5MAm^5`Lqn@^SpE@gIokd z3!yKb(f54NI7jF%S)aC_yoRhsNuJTloF<)!3w_y$&7H%aTsjKRAF~LuxoU6_ZiSfi zf0D(mwEaocOJL&2@4}Nv`|<85g*{oBykJpMa*d^%sCo^kGvOOLlu4cF`@j6r3mi-g zyPZIGCt|w}qFaAM9LW)4CuW9T-(8}rZD=qf9kH~ntF7f!k%pZ92sx+{@K5bsmAup! zzYK6gzl>@bagLOvkJO|-&?T$9Rgeh5La>Yw)02j7h`2l(;9auF)L)@6P5x-)%CCZ; ziPED=CY>$>3LU5+Au3xcvZYkUiz+1eZy4VN=I5UUL<1#}2bR^~Z8r3{>{P|S0T&~o zcpCw2CW!ZxpA`dEnm#L1gbe(Fxa?)WVVOZ@BBeY5ijSA~wPVM8+DpX+Rf%*`>I`O? z45i+$tr1swMk(B>(J4&@=h9L=BgMl(M`D{Fb+6NFzm)CpfX-@mJ_z|}p%L3rF;X%+ zGGJI62b*ARiFdjeE@vSq(H)W|uo{{A4<7c?!%+T3z%CQ_IwpmcOtG&@(~TPHgKlNX zqG>6I;i3%API7O<9TU+kSy7_bFz}U@WwC)MhPbcymGeyv)ydbTGl~S?hdmbiAAV0a zB_11jgYwT*9XD!I%ET8>U~ju<@xTRt3dls5pJVvhk^)oY7N^jhn@op74yL7Q$}Tyq^B| zGkA-M*pH61C}7qA8yl6DW<%UZd1*HsZUC^7I$_z_*<;?7AQWXc8vCfqDl@2pZz-K7 zWX9@oE-=bwg80*Z(o-McT$bQMWWFpDz4P;NuY+IV&#h^2;)yF!9=p|LB-Ip>I5mZl zgo}|Y+kdMiPWEGOJ8bn&CT{kKD&VNTDm*9=xGctA!uXPS{)8*4mhd zW&Zl_wagism>e*hN1Uf=+d_w@!MHf1vz=YJLwk%9-7Cuk8<8|lw&c#WZ_gAJ$fK6t zAU)F%1QpN{6@SOaf*#i~zkI9M3P9aiZ7pSlWwgLks2iRw&e0oAnWL=mC*y8!tcSQg zIXNNO)wbDbk0Sy^(6r*8|+j5HG%oSr9wLn zFbSY`v|e!eZjX;D=dVNAcC*zb(ETY|qdRN1qkCuRDs*t$iVv=9-NE^<+D0DwYuDJ! z?bYp-+3zU@MJl~>x-|)Blx^AvbC`h&sc6Yn+oy5&`>)@9fAaWYL>hwodb|^dzoKLl zYM_M1b1HQ&hhw~S+zCtjah5$N*W+=)0xhgkae914!toU$39%ahA~HBABTkk{>lLmH zlAdWWBnDzi^0VP;Q8w>$l3LeQl4`39>Zqu^xUDa)FH804FE0=JmcI4a+4sP1g=}?q z1B>ft4Eu^LDIW?6Ej7^70uKowk^rV`?a7@8u`RO$$tI2rmycSw+*O_mIQu`p9}V4X zJ|t9Rm2#@;p^yV9h#f3bkHra13L=sn#r}#xA2J)o%ScoTI_S#o07|GpJz-Kzw z%l;KsHWe{kG!VTvFI6urt0@0}D4ncAr%z^B7qK7tJ~>~=%6E0-@sf~`yh}<70QkOz zGnUX76(1yVvdH@iGmj9Wk3h7PY!mM8?go3OaqJo;DWd7ZPzv=@ty^X?%;P!skELaF zTic%XQytD5G>RH&_7_LsGL2Pu!A8W(j_tbTnLkVhJfoJ|x7uJsbJS0P229!emz1QZ zT0e#g#^vvQvWSvk}o>>OdHiIlL9zaymogkt|8Co1=M| zo^ANhbN_wiK*&$zl+AHl-uNbuiUgd$Z5zE^6LUj(UqQI%Z8`UA(qVs^L;b7*Z+$Pa zfg}8%No_y&_d_Wj&r*`TQ;=YUtv+ndQk1ZUtoy7)<^@fYG4}qGyECZpgIqaX(i+HH zaGnPYc41b4GMr(@sCB`z01}=Ecc0@r!=Fm*Nd{&KR5Fts^3g5i)cDHDr_Cln&70Ki z`eUE*j#l_AjQ%!kwZGdqH9QhGbNMum!yl;bo?s!q1_7;74j>d9?EulE5t3Cr8HhAp zm^-w#-&8 zBl@s--0>Y0dijm6(sF_Z`~+<);?mU!1nkQGu0R9}ty{;C3!TREs@~gsjy!K14&Rb|xF-}sX#Y)Rg@0)*zwN7%T9!C{jl|0x%aT5((jmgSSr1lX1%#U1NHjGYYQ)o;Q#r|O%hrs(q& zQ=0`t6K11#d7}yoIh*uPi+3Nn%dxv2jPu{17M{$CYI`O>?@w`z8zS{mo{Q$G6I*%sLU9ZV&`es^8H85HYeYt`rpQiEOe9WA-3+c$E2 z-RjSxE$}xrm^oMYVe=X#W$a=47Xe~efYXiPsZfA6Ih4XeJoSPZEf#UEY>JHN0_(ho! zu2xCMEf2E3t%p>;N9umhc03EOrG*fSNyvtc>^`D%Be#V)kb#pt73HOi)5U5*Ny8ss z8-c;&y{}+&aDQD%tvnT1bel)tC)2rN!vGQ}Y!&Gdl@f2k8`&&uZ_nY2rNAHoR8f$7 ztuEF`d3kx%{%gUd1W}d#lVHX7=gVuX_iU!0w^m%myK}m0B6|9@#r253#ick{nSo8j zWuv`yWF{7k#r0m0GMq->dp2$%@Y0_0V&cb8F0ov?OWo;h{h$^9Aid;}#}eX}V?uuE z&!kvZqydGbMS7jvZ^uH#e|v6)gvWBsO^Z&mg&fR#;k=Yj2xZnu}7~dU8_oK)C|2)b$Jc~_C5zst0EUlN z!f;pNcU${c5GBxeaL#?^A2(C(D-ObU4HCtLHfW6#XFjvdzFO-kt)SwxiUom|)UXqE@{piOV0GymZKG756rE=uo-b%U!L+t{TEk!R;I=pf}Jg- z&(2mt(@Xer82A=mG;ONxDse6RfzGp=z|-+%xmvC$Dx>5QLGMqdx^+qdO0C7FbZ zLf&&vzNP7Mp~*6N)qMQ%aENCyczo=+4D+e7!!606!{ot|pvVBhZd2n>AE!AyfRU-^ zsxPm-#aKgt0+!CT9lB9}$jvnb=Gu-T#c|AfQ{`5FMBwM^GZN9@WC1RIJ13{^GQvCc z(^Vd~E+4%mv$wamb8vcmaeAVo_eAMdAUa+(34%X|KN8iv zT`CVB%X3j$(qJig&3WsQ9dX0a`j;rCj(6|gWrS@-e)c*=O%+mh#yNDE>c7Lb4;Qzc zDA^VYY;9#;63H%U{>qpt;^Lz}R4`V5eo&Jd9h^_awIx7?IpN*q)>y>s2BraCt|lgIIQ$$8~KOiZ|N%${IDDBPo$V$~|X@5kWNhl5= zkpnKPDMZ2$^27i8ou2IP2i;s;qb#>lm?7^6BedFL;RD3iYCA5pkic^kfx6{et{H1^KjP|!1) zOPL*UuwwmwZ%(e(ZGf0ZZDLsGval{aTWi20Oik+f<;f(Z%xKBNI8lJfZ(~a_y&!0n zoL>to=9neKo|M&aYZ}>oc-wF1BHfMVaiAzfzfw8ek60J~hfJq}`Sezznf>9q&;yH6 z;!q0DTmeA_UylPBfU#x`V+<51W^Bz7oaQCi3zvP`-qwb@)$K`nRCVJA#R@0#XbVck z=bU1-w?vtq=~l50Nj-10#{2L*m9^u{=}cYo6y8zx$qoQsI!%+zC8|0QQVcN#0IkWZ zWb>%K8L45s6B9)p>GKlbK7ysvbz$bNR`S}xA|H!hCyv3;#L=xDz1M(CDu3YXGud~P zQH$L&QD}poubA7Zzzridc+%}2T_+0E)9MT|FVEt%rDIQ}Ux!;jZVdZdHP^rJ6*pFt zhWwN%h^F(Q`4(uR=lK$9h2TK~C3X;jkd5fAG;szP*K=qEAE;VgZvS?iDZ zH~Hu{S!A!ahkExhXFe@VxQVy`cD|}}e%YaaY&w?!*q?Hw<1Vi<$st%3mSKmPWbZ;^ zFm?IH8@?O^r-R+x?W0mo|HRUWWM-Lte#_y{?NwbS(0};TGb;4B`3=v*H;Lko$NHj2 z=l91pQ)^~kg6W;3mL{svJ{c58%`GjuY3Q95y&H@|vcZ+>AqXUzqicWH5V+m5HR#sv z#R+`aCl0+$z21Ax`(e)=-PUg9O!eH?wk_%V=y=x(wsPn0cR>y0Y~Fxo*_vq5P_duT zXdtyQ$&d7~kTfXGcBkNHT2JUz{bkCa?AO#(1a)@^IJz?>khvQXmdwY;t#Q8$L?t@QG zoql&qbZ*z{fokhl3sHd+El9mbCSj__OCvL%R*uB`3e9eX!8g=mHm$ib;HWR^#NSCs zO0w)1WC`~0V$IPdyr)RlL`h~)9enIZ8|&WJ-eApjof1lXi>m`7AOX^Ky5Z9FTcH+`3&Hd z1;3acX(o$r6mEKC*rz9|E{}tzWoU$Ri^1+k?SSLe?ij_RrH)V}9><$Hk)bhUbh~%6 zL@$vT+dV3Y19E$2Am4jV`eab{BO^dNo5IU(=61&WWM^N!`GE#!w|O+hOK)TGC~@FL zp8pp{kjePhomcjaX2q}fH6O~$#!=3nU4tFi8yi17>_+nS%si%_t%9^rMd|Ik%=DF* z3*BoFzx(t39|ZDL3}Z8RYf*pYMBr!kG49MX^~XpD;ISYrTN3|17W&_h zWkV(R>9@O2u%NQC0rHEB^GP~y%SpJ-bCrpk4}opMEfZaY39(-Qi-~9A4?^EdFD&G&&{&NT!$kSm=oil?HS1O0IbqDVzgHMn%Bh?E= z&L*^aHR%Nr!)`PoU^5vE3UlZn0g+#tKz$~?MJeD%e9URTc85&W_-J@Bz7vH&9avaKbJpo3ew_DSDS?p*iJSE-+|oh7;{+J z-dDZ2e;OC{LjcM6<h^o&PDk`ud9U?e|LV#^s)6lF}KJQCpUGk&AbtHA@p>^*M=c4WI}E1{~67 zKl}-mQG0`Y#4{68j*!-)Ujo~0Ovyco_bU_l*5RE<-QRlEHS)Sy>SZZwajhO##gZ~- zyoUcaxYKj#+O?Uw>|NeW;-SJfV>&W+jI@M;Z6;yL4-ReO==%z+Z!>rv_dvejC&OLZ zzOgbKiDW&ka!ibTBIf@!HilP!EVF4B{L{5cJ7(e1jW!JX*h{Vi%w2wOyirU`HZi@< z?dFf=fG>OHzE%!zUUKMJzr30zIlyB#9a{dJGoG@4Wnq&OcP0P%gNy#xR3Rt@`|W?H zUTkvw%imS=t>|lyF#QkLH42!K;rysL0JhQ$^@fcE@cj{qYeg`oafr>hlmx^v0fT4~ z3zC{@*;v|09p^55P*?M@Q-;=wl~sU{Ots32vBp6f_AJ{Tt=?q8EB zD8_#@lXfn=X+1q?UO<47ITK_J?^yh}_0+JG*t^<#k9ay&V2u5W;4VG z+4n;g{d8B@7+R8_PP|>}q&^#Z+@6NB|ExN_Et=kprFzwuX2JU!gDFT1Es5XL#++Ep zr=gi$&e;Ux`l>^I7|^!RZA*lGgkkbdnm9~12wolAe#bqxzkDf~tUa`u`SYMJ=PiHX z>FC*!SQxd4;^EE?-qwOQ-eygKxb*_0DJ3v>h8kQ`s*O+~SRHcT*;=2w8QBG|@x>!* z{X`j7Ie^}`L=ST#%XS6yQZ0ZwpvjYm#48SIv8wYY54gYQMDbyC_uQ};7*2l2B(cGHoF zla6%w9KVb-f8K<3XYFdF)4CUR#kA^sebuJO8JV>} zhRWKRNb!psT^dC`N=u!ZHKw^cGVGxu8FzrrfnXMl5?NVKIUmGWq~cJ9y7})cfb%)J znCUf#6xU7ic(Jd@Pp+KNy?W89!R-5hvB89NY(*@3sS130j_G}o9NVn|95=UF54A1V z4TeL~+8e)G&3tUcrz2(RQryCho~L^^Bpa>*3A4DnRaN zy~Ts7YcNcpx$)Hp*)R>6U5mPhJq7fk$VYmg>-)mXk1tQI?I0h50A0*tmeR{xt=IFzJ&L*p)1;mPf(A45ln9Dgu9xm&o0(P>ka>-0=@H! z0`55zfkFU4U&0!ydiM;cpdC=}WmA(s4GOc|cMMX|uPvXryG`4yt##VMBO)SRRPMb4 zq4p+f+1J|^!A!jw5K{d^&*r*?cB0=n_glQuKr{_jI%AW$$cCH~vMH`}I?#*82G z#k>p<-4NNxY3R}kkoW!KCAq?!z)$OgUO;~Anmp?c>6DoovK;CASRwhm>Fo(Gu)0i_ zY=!gD5d2CH`wRqV2S2vhG}}JdDa%I`%*6RS7 zVW4ZMVOH;fOE7RBM+i_uFwt@Tz&vnHjT3=*LXcKh4{by0WH;eV_Y{l&juVZ$A&P83 z-%V>rm0da+5IV{go>Og)$Lh5og0sUdN(q*@Q-!@zK{0@>`qL`v((hqLy)AvEt;b4z zW&cX<7dT@E2_c?TJ~^@bqLghzW1!IXYhN0S zv`Dpg=J)+%(DR?a;k?|jYuCu$@=tbnLrSZil2S(#SPTllKdMGS#G#G_6I4|kZcBN= zS`#iGdOt^MS9c17?lih=rQxfC-zPxQ_qte?g+r9VS!oa5NI+3hon4Tqhwk@oNR4v+ z*neZ2-I5(il{{7NL`cuDb&n}!g}FfNyjPs+@JCm9cL-zC<8B)MunkJc#Z7$}KO>mq zETF0|d}*1NUgXjB9ZoVv=zZFG7Obe&XAkCC=pE!e8AI~dEKYawxLjAJ5_boI;!Zbh zJq!kQ&wj-ZEVx-OW5?YGgW+=ic~7A3N6nU{5duAT#YZ@3=m`ls{N!!FCIsrGAB$5% z;oaj0_=L9-p!E+$7Pyvj1LYWOAiaC181%f|uC1CgR-PNGT%)+rnDi+P=gBNL7!xi> zF`55EZ?Eq;XT!}qfRGwPc=sOYf@%p@IciduWXB-Gm#5itK$gf}Df=3$gC>&grpV)- zEdLp5Et?)$=_f-Nb0^;ijlhCisyJ@quNk2lnps#lEV zf2R8lev`H7ySjE4^tGos|9XTm@K*|ezk2FYGkdd55;JQpV}Iaptmkh>US(g9DF@gX z7~+M0?-?M=PCCpVe9w^afoHv_K!;|fR!5gF^cPc6R zJ7kiCm$*bPHRWPX6J(owd;@PX-Odne(t0R9;nEk#2oH%XRBr(~qmWh;;F)7(k@vh# zG_dLF$B(K|$L%@SQR7SUZyG$?I6Ti+%}ghP^&5{*^14}_EQd>4RveI00OI$5DL+rR ze|h#QOQV;;eBksBsrEq*a3=w(-!&)uP6%8b3koaY-~TMxcslOt#@Kk)^Rm)<;1z=I zd}l@f_&gL6uDlv}(+#Wl7OY1zU3J;_mUw#szvUf0G2*qaXv??Ywznj;BELLdz>2!l zMl>PN&dr&FQH6)nx-L0IhV?cMQ$516P5_6=+*_+6vVmmM4#J&t?j@z5c^MMyvqMRK zg%7vEgYYU=It^8_okjXMCUrQK{-W zy@LoZJ+k}}^^c*9r%I_wX2`jO2xDCoE48vd)e%1LIfqIb(VJgAKo~p0yYSzFLj{*Y z69K`z{)*)ln*Y&7$O%N!wvZW{Yk()lCO(Apk42t6^LOQ@mx+#rhlGg5V4u$GMfgP8 zMgo_%YC;OKa8>4ZdRkQQHGU3*aR;I~y^^r&K?uX0kf6(lCDYIO?ozVFbqX1JreB$4 zL>a0i*zu9aP%=Iqty{CZw{+#sr>)WEj@I^gwl~zPnvQmnTn&}ArIblE-l})yLO1q! zZ-~R_IMqu6f~Wvp27JA&f;cr2Q?BQ)XKl{sKBGG6G}oyo<5E;^PzUa;tK0+p8-75k z^OSk-e%RTJw3eBo#4k2WWD^b~GUBXA>H}VY#?84{W`s9~X;qj^aWctep!u$O#}>U5 zlSQ6l@VS1EymWUBGeW@m27{FeF+6zfe6gapTcGeoCU*5@EswUYjt-rA(JNn|=5HM; z)CkASs%V*rnUNlLL&-?ku2zKvonPSHYzC$w@GSa&q1rt+Sl}<927QCC*Is{Ux1Di* z>vg}6oB8PN7sUb|bPrLuCtWx`0}1JHi{Su&OiqVja(Y!BAXlNU5$xU;Uw!@{E8Z%K z@}(yBDbUFDEr0j=w-RE|g8#N|4zNg%=~KXJSDaRMS7fJTpEzJyU(b$q0H++K#kuXw zg_7C{J3UQn5QB=BBi0|u)ObxdgxT+;H+?&y=9y2mttPx43s{)~GfuPMLr zLBr>FKwWJ&)AWPL@vV}#S=TE2+O+2Q(ms#F9^El>z!MKcCh#!2V!5acJ$tfHOwkAN zaKh)XB^K0wJomoA!MO;z(4rqOqNiP`yxLp;3ab-!f^Y6dNZWgF zwIYw;&(0NwN|M%RQ+L@Y5Y7YLuHgSscagDRx?SZbU>);KpJ5{reE)X%W3*6A z|EIUXE7?Ndm4ON~hb%w2cMIzvZE8!{M8v&orr*C_ z=N1;Gd8|(p{QkYBtDF0L>POYvI6ubs&luqVdretA!>tuky3F;iTYB-x-D&M9&_28p zx{qSIWqev$ldGvsaekC3o9E)-&=Qdj?^G|Xs&W$J;vy2a9t$y=jBz;e<_e{xdn~3to>0h(E>FP&#g{y)1VJ&Ipsl!cR-6(9fzGS|0JY;!V?| zu3@OLp*jQ`UEYPc9I_C_GO`!gDW6;y%r%$(DrKCU!kabYlu2t+McjvXdX3D2%>9d^ zopH`tin`y@@9TK@{9gfH2KS(}m^SWz@LeGxK!ZmX*c$MV>Kn|gb&UN9siWe*8jBQE zf<{8#*yX0pp7*CZOoPQnG~RrrO$4vtyqCN%A$s+`Wn7CR^NnlU zN9>JXPmNRrZKhk_Yjn(^cw{7VbGm6q zd6?;?IGxz}Ee3cI6BrD(=Ky0bbtOEW5_a0zSB3U}H~=<%6tK|>*ff7Xnoh_ANVS^) zN&TWQWSt#WXWSFWXuMH>@b>!~TcvhmdJ(D)w8Tg4D$`=oUWIr6OL{P|DmC@-IPiBh zoSRg#_Cl8VeAapvAj#DEdenP zKVc`^BO0JdJgO;M@f*LOJ66Bf%jqkQF~17Vw3Mj1~OkCzl^PX#I#TIA%j8`$xo06;JVu z#|lA*GI6}TaVO{o%5QHz%JAlds#4!D-@Wowi0qPobAr)F*|@!P&E0mn!F*)+_IDaS zXS%~n#I1vii?YQD8Pie`2-Zcj%#6X@$V`U*%(Sc1KKoc;(B|gi;_GU0YeP-fhu4q> zLGhRCd`b&TQII!JGq2sxFu@y#WWm((Fxw)7$9g`Qj&HmdE_`NN=UjRq>_WkQbb;Kq(Rvwd6q30nx3K zjS$bUEu~v2`)7Q~8X!;!Z-(4QBZ-WSEZ$7&UEqZ@5Me4}3MAVoxc~0ehV*1ie z^#se-`bsPe>)C?U@{ z1KW#S2;ki+l9$dt3UijI>U&;WYc_vEzkH!@En+=mA@pDS*=_jmB0CM~)McUj0xfJQ z{@(%5BzY{yRdJ~Zu>h{El^?ztLn z-!@MD3Us#9XY0|2E_4tnzX}z>Mrc-aq`8uS%V9uC#p4_{mwPQno|6rjxF&=4_I;O@ z7I6o1qa9#%%=mfI3L)}AK1*HkS%5{YUHF&9O9C@IyY?2qn=4)FaZD<0244ftcP#2U~-ss57ZZ!{;ajgWX0r+`_{|mHqx_q z?jrO{x}8R&d#NUMK@5y#)-2;U`ObgF5ch_P7o&8ggTxb5)W8Lz;-i z^`qb3pwsQlw%D>Ch4HnsUt$bw(M>bqg$*QOhZf6QnNRoKmQsVRCG*a~;BKOPM@&^5 z9SguN(y*u#HKsd1H1r8xbXhr(g%MQ1wYy_3ewzqJ39)vh67QUbFqjr${4s~3hq!K& z_kgs!hl>9@v5eV{5;M%jW@oi#Wv)j1Oq6kW%4L%BeS@XepP9Tb%CBp#h>s!sC9 zoYoWkNDq?g#ox!hlb0>PA}>*DL=IX-Kcopv4K%hFV7*@UwDM$dg4%+=GnO0I78n@N z-C1HjGe~=jf0yG&&|Z76Iq&LIveSb(JW#UZV^xc@BA^)p2Tvd{J3OlwzB@*FYG;ESDv*H1x1d=ib@$?G(n3m1|QKcse>d zubB_~`mU-t!gr#xcuygTz?P0I>FSx%pd+0~TDW;PJ}geF{#ht=G$Z4?8k6}~LX*Dvl*n?Zj*z8>=eD2Ez>@+FBMftUL% zA)XcJNp|%ZcRW9xP?S+c-Ih6*!YO41sfpEUFyc47U7EbVv2mBz$U{Yi7pQ-3>BAc| zL;9gmD2SX~Z5@PaYZO1(cR2rKl-r1NCxY?V3Lbk`RTjZ4EuGe$Y^&hyYA-QK8j94y z(Pv~d2G)H7%4}9&Xyrjics?*RLY*`XKt_{Gm`)L!D^ucq$Y)yvJ^*Q6zD7LnLLWPb zAu7!pOa<6#fq@L?bPR!Sr$4Xyb-o`(3Ij6=66xin3y*dVr4&IwWth|?(vykxN2h`` zxzW;1KaJPgBneO^vM2bV!iHoyE6f=hxLZt*DGR!vN4zR)3>q!jP^Bu!0wy=*fPY3# zHjzv_*<`=!YTv$*#9e5^FcZ1=Y+9xJQ=XBG&a~D|M6%pB-oPIwb~!Doqps_M+2O?f zY_Ma}5_%=Rcp1P*FbU0_r6hRJIPZmJxlFtEw#&Tqv1NkO<#@f%autM!OcsrsO8M_d z8?WMjy#?f4`Ecv0{Qug#m|wgEQaL60n=^*MGwd@x@t%Jnz?09H*8vS_gi!u&2BnlL zKQls)wcSlqHiMY})vbF1aN5k1&wV|F*j_q1@S3Ttn+U3?emd2}x5=I1j;)IY+DxX2 zd|xz$RVbWNqBmDOk9h#k5qow-sA5+FB=Ym;E0gt3&twCY|r&krTd@< zf{(A<7#Jv0pjH^rlgiKmJ*xVC_^8TdYsS^A{kcvZ31sC?RzIO+lkr`-lt(U#dV%?< zF(xWBt)Tk&;)-!gy8cuSqrWS(IFuFT?D07Qgb42zeLEoTJKo8l+;cyu+SzTp%Jb%nzVzYI_V(wK zy_KC_q}-x=aFsh6k+|!BWuYzcTTsHS)@S(h=!6-U3xM?h{=g7$34hkbbXyyK!D#d0 z`h9pO@@83}s@I{7>{*_-wU%_+UDCb6}JIO%*m7G6AN83BQO1fd)iNW_HvNnQ1i;Ri4 z8g|&B5x2%6@@{XJ`y{9@UH1W&d{kh#Ghx=nYw2_MFZTI6>5&aTkXTt@+uLou#D*H- zvn8_MD8m<^d~@V$ULi-dvx)xc)Ms6s+2nnpu$HWdE>^9TbPe(NLCZQ-3ES&DWvii@UD1cID$IMg@4XK1iwm`k1rHp2Orr#N$fAc@`z&xevN$VLw z=6r3}W1u9p+Bo}%<-%sAx?kc|(G;DI>G>l#{G9INjaxB!iiVpBd-kYpL+Wl_5eWl{ zE3!yllI~=)$Q1WQl=Al!er;J6jfvX2y17g|ZNtGXrrD(>|JgJsG4zD@`*jpwTza5~ z$$lZeGI&@!x}LLtiA{kjy?Y-RPIkP#p!CV|h#}txKJ-M|Wee>n9#lR44Vku?{u;M2&K21--wQSdDv?caT)NDw)R+ zYiTy|TidZ(tf9xebM|9cX;snoOJRinhqJc;in9H}hXp}OX+Z@^1w}$cK)OYxMQIjk z1Oe$T0qIhtmoBBG8w^T;Wyz(HTwqy{jwQZ(^%ebn^To`6X2)>`o#i>txvz7b>%pd~K4-0=Y56vn=waIai z<)9tkwX>_=$=IPQ5Mt@YN#^g2m<8K6+cT`wLMjczDAT>Y`8C8F*(a_PV$FUAUL%O` z2!}bfi%XqU;0jo8dT8I_G9crVj&2f`fx)!S))}EO#eY4-e*Q+846P!5t4$L|hmK-#u90ULNGX zbqx{8x*?s-mDwM*xI0lN8q%8uxewOi=ruYkC50X+#B)zSk;QX9+$iAva`?*4AoJnu zeeL?UNF#0|hL}8+uZ@qdgU$@PcN`jbt#ski6mAlSdqg7*W)WBNbt;Kv2QUq9YKqnK zSY9d#X_P=NfP`h!GBfO8cvve1Y<5N1jz@3O>;~+ao?*jU<0JOQf^%=EXNH#}%d8~; zQ%hr%I((-eMBa=_|JHKEP9`AKvm{G6n`LYSuTYa2@6VS~Hj~rg4j3n*`xf zb&lhTbHfE@%j&z&TYr2Q76qx%yJ$hXTgSolU&`klW4YC@vYT!3k?y00EZZ8$dq^l6 zAMjZ{X+PTJGZmj?2iLUYXI9F1W%(XAW#~JE$MVo=?=iPvS3%t^b&7T6AN7i&naF-- zK$kJ~%cEW<+u3}0KsV`XMZCY;D~5wq99k z4A0Y4DM%>e{guZ0f_Yak42cg8RV{lZU+KcHCaFvH-yf3u7_MtvOM5DvX8zFoY?`kX z*A!>Yhzi1|99MAsQC4a)fsWYzOJ{!;P72(0r;OM6=-ph*?7&?a2q-{`aPk^pQ}(gI zOM16H(6X#ENa-sTb9>I~-m74F99QWEv=C<=ysf1Wgvp#6|G_UM>cs9tP>s?}HZP*y z<_GOQMJAL&23?(34C`vn@n31=KT1&CX{LHA$)jw|1rroZHWmF(a00IgB=PFhVAX{Y*hd2) zx&=1&O+NAs92OPhJ4?U={n}d-_CQ&o8}Wc!*%aPUz>%c+9!H+wtHh9}Q@X>fAbT?x zYv(SypU$}bQp>uKlD2HD&%y}OWIJVqulFD$$o`J0&~IZyu~$oI-6po%a+}rJ1oe@0 zLH6LDD{hlHW*4&(Aaey^Q>V%e<~1HUEhWuylqyBUfn!I1)9nit6f?=@L-L9E?ej0C zC8z8jTf3f9;^~^Exadxg3TonM|4BkD`LkiY=CI#0fP0b|s3QbE+5Pz1)S7i?k4NZj z9#6PGpl?8336-5YY-qu-MTAYuu5EUbge3S3(Xwn>a8{Iw-%7>oq}j$CBg zpy(!e=YJJpe5@iYHFDhkqv_KzB3kp0C9gf3`W?CCuuE2Mrfa@}%W&OT4Y4~sq)7J_ z_h|_IDl9|MCx5MF=7AzLH#?-qiCEm`BD|)Iakj@DxtbkTxDWV#56zf||7T#<+ zoZ+h+Ua=cm5z=ZlWCcZppPMSlKbDs*ha$0(J^1X%V_8J^4eVHgRX5n(>FDNw>u%i$ z=8V9emGf>PtL2X$<8P4#7|1KDt(I5aEOaOet_OlRhhi=3BgUGtv)>V3A0%bMI+in= zC8vD#?uc3=!QrxbWp2AGiGqsCw3r&-qeI5Hz@n5f^)tv6yIT%LZO* zIsS4ZQ=6^MMJ0O2bRzkOpMW|u-Uy!nEXLG``)ytL+x>BIyr`cR*NqzQ*itx64pAU0 zjjcwK!(b(qgT{~sPeUSg1*DCYR8!7uU*s=ED*YvPc>}gT)~Tuu@CK9rArC6TXaAH$^ z)uuwelt)4|yVPMOhY_H-5}vsBt+dVs6-pZ0w)G-FHq)2iSbVP#ov(~jy z0XMk}6=A9fF#q*Y@FVUPQ+f}_9ovsL`)?>V+gfM>wkD=Iw7^mM<)vD)$uO>iSz8&l zqj-T#g6L)}Ej|QuM|C&)EVo#EsDo+eqh#hLQj0M!@e1o>hGpL?uxfbk5gUzu-E4f* zq_rN{fM9ook`}xi_+vEqFtbqz%(=((4SL&qgh)1+q$|aR9#&e1@~ME`gMG+htUE>1 z;k2H}a333hN=SzC-!>f&t3%MHWA+cAjDu?qtCMwI!S0eh^pJQ@kkjtfUHpYrow&ah z=oh#{`zlptS@B48RZ}t8-TVqB0;&-#aAY+vdRqo84L&NwH#ZW(S;ya+#4vVrY-GN3 z^~-}y)cxGp2yo3{U!IWz1EjJ0#1}b!yo=$tP#V zSm6FIBGM~H+%AbMEb1qof2!7CnMaHobU>@2XOk43KT&MD;K1Tlp6ObOD<~L&@~mT6&WMqt(GhWIzt0>n47OWhd}c<;t-*ul{=Y@JFRC1A+gB{qu#P! zLrt>U8gw3%@PgflS`;NhBl;TNzL<_9Kc-Jw>cIHFwE3^@03D>=-LvM=imS8Bs^&ex7M3v%2XU+eKco9o$`dPE2gQ@b|1F+?dvkX((_fqzgW}1FRmZK< z&Yq8;9|F6-C=;j%TLsi`+TBN5LgM&a8n`A76XXw+mG66cijGfAOfN5gNOrSh`Hj7S zSnT!V3cMHS5~bZ=&QG=hwwPiIR(3ktavQakd;*W}0Di>P$GHq^omtZOpO00LTzoPf6z2uD)#~J^9bVT~t-q1uVkv*M<|s``#ewjPM<=KSXZtxle65?>egNb`7i? zA8Z=d*MmzzG^aiUE>9zDGgnHow!?>*&fuPYYMFe!Jh*SrdNaCm-#8R%A(T9W_xnSf z?0XZlQxtd0rt_P1{1MEqUHB@`GvRO|*tRqDZiL6+*4I9YCd|F z-Fis9J%XZ4Etfijd+Z#lFAHx=gUIL&$^c4i%bIpf9m@ zUFp+sfH&2SUjsJ)gFB63(mQV%YVQXzEb4@{x2DlR!}c`shF4J~f9vRbE@^Gw}=!z_1%sx(6P5KyS!-n&>DAHx zcbbry|8H{_2sDsgU|>qUE0vp@0u!n=qQtby zIcCbM_B;yKRBTYY$IX%a_dDoqZYOBhP7c{a* z<6_CBp9TNJ0`NNwoWnZvW_OZ-N#*gi$Gz+hy+{$3tuzW8!bQF>ukW8i&pE6k3lRmf zweUV$n!6SNjB7>FK)6`>b3)~3({#N(2c6(IhP(4;WK_ z71}VuzU!Se!CAFX!tBb$iJQ63n?1ZuuAiX;j?bTJ#l;R!b_lVhdcf?5yY$VEGL|zn$xAK_&I&TxyuVIc* zK@4>p4;5G=}BUBh8H^{O*C{Qx>3Ua!Fq<$yMu=ZxNY@ZNKG!lxHp zJ|J~fxI)3im+{SGT4Z;f6YBXQoMIqHLB4LKOvB7HG@cEozs)EEi#3GmzySy+#eU)zeZgU{k5i zT<;X;nO@%69;?a}K<-q10E>5ZOnAiY-Dsq0f|Il*ec*EJ+1-R|7gnm1XkhHW) zjD`KsA$gT!z~iF_6JPaSq%%0YIV!BrH-pu(wrdIVJ zlpT|3?fw?qGd9Zrkg&D2^@+Cj!#Lkmo|-fA(-gkGO+OdTQ~3V@|7%3gj8o^H9TFT= zI)1)?#N;zSce(0WD*jK;7kV`k-!a>XoNSHxx`7dyMA1!70qH~ns6^H8zSK3 z+&1GDnJm{S>|amltG(Cx<=}Rv?KG1(savc>Hq(cxYO14|H&k4`wKNVQfg^pBYn)9Z zY-2MfT#F0Ghb>5YT1FkMHTqi!@HHs!S@Jg`vv6K}MVlZ4y7!&ud>BL+-IeNE%`Fol zY2WOgoD7{+HmM%d>GQ^*m1qq-A&3f(AQ_wZ{7F%BV3W5N4He8X`+(Kl&XEZR(h3(DpXQ`$Cq*p!4ak#1R7P@*#)tXcH+qGx zXVot;AZtBZQAJm*TP+?FbIkogx0>|AWqUvU_L+SG2G_0wjrY@E zgU%hVKwJ07Bo4E$r`z`D4ji0swwa*h;IdXCU>yIt+`cL*Ppc;3(xaBJc<%3uvE;;2 z-$ApTSr7vkRql~y$1kjsE^PBw4CFq~52BMn1Xe{tz*upeb z0#&H~JYL^f`T5pLIZuB(<4OFv3&KI9_nu9~*4Z{CzT2Ze4v}TsZz7WNEQu8^N$wF( zzCYV0X8v^3e5ZlHpw#H%I&ux{zI|xY%=n;4w|dKx;RAgSe#Xnsm;S!6jr*Tsy*wt* zPg;@3U`2oI8v$r;^MlTusO+sizF2}2%DZ*cU z%GsNZrX-=)$h`K9`8H4Zx0Fv^L3TGdi98n_1?_e%B!+=QF(sXa}P?C z>=ZJ0=WCJe>%n&^fyZ?yn;e7V4RYblE_KIuNbCqTCD;&aG38bN7zy*xc1{+s&IE}1 z?ie%mK16Y=XmQl)MDqIe>#5?eG=9`F-MBe?`nwF=1(A<-HGD(7=0>ShHQp9^FYSV? z^#ASUGNs4!KT9GHk_bkYyqPE4l#gr}6Fhs=$)l1@CcYPZj#M-ZTzrdw#!+!q%yZrXLKyn{8QMB8Rkd21ER17l zXXVwP8McZ+>zY(D3Z&(fhFk~ZxJ2{E+uqA~3X4-Yrskb83GuWk`e0!6$HeozHGTpy zHhtL6Xj+$0JOPxp>YcxV;tziV`yVp9Ru_K+_2Om1d;|!Pdet_~h>;b>CaLsjLdjq% zud1?}>GHYX7B|n5ol;aLRVu<~Q>orG)wWYM*VCuxc&EH}#jd)Hs|_^!*cMC=73nU0 zD6q){H0^P+K#@MQJMH8uy3xh_%AGmff>a5Ao?E(aj5@7VB#yFU?;tOP?E#|(#c}K3 z2W&b)*m=DFIpqNZIEyhEIDd)CTp%{{Za5Ed;&9*3Y+>`N=V&+e!!n0YZ0Dy>)5c*u za2hjjgW>|!)@+m&)#W+cO=|dN(2C&rPxf2Jq$J})x()Pnf;NgKUoR^wCEd*g1SfTF zT4uQswz3HmS<5o*jO*_fG%(gLx6UA56w>t^F5FZW5T5lE#t}mVeS5jK(z)dUH560N zcu!L(90<1~w7%YUTiRNXxwc%f9X+pmu;6AQ&f*&Hrqmve`N69H+P+G`oYm_9x_zm? zmCfO+u{lLEJxgr7JGaZmdI6P$xB1n);fKxkQRTJ2UW0tDPf2Z>V>7`>IDV7VCr2!W zPN=gbyW=xL4JTMivrf5ulZ1;;9d7r9^e@7XO1J{L$#tLcogG2X03TE zqZ^*VN;%b`)ARk(+Prm?*rO#xv-p<}-b*<|6|Qz4e48pND&I{^7Bt2>&>VbwD^b<5 zBTv6hauarNxd&fcKAiM#p7bdmyaeYdCI?k*J-rv(MnsE3U$d#_(rq0nSV1OZ96+Rb z>LxQh`u(JYekq1;~!gGyEa}5%|^xtiThm~Dosqh>GCI95Z~KgU6g54 zqvXk+(CyK=ONZXgn-^IR@Nv|Cg|p=*u?|rwNyY%m7k@2v1}~HRQW_l}v;V7>V) zFLMkI!#aOW+uQeyL!IR|O*dlZ?L=BO6OFojz!dhT1VQkrD8R&rO>1fsIa(gFr>BW| za8LN07&{Id67;TlyXaKP{Tw!)#b&gbsFM0uqbm@dqDk%_JS`j?nT*~yvg%js(nhOo z6;5`2{EG^=58qXatf(R+&K~ukt$ld)6E>v8!+v(QDSZ(5Fv^{7&crEz))~E7aY+lW zSGm-ZURsx;#Y{%Br|BpF4cl|db=H=Ibfs7mY_0Gp^`#1Niys;}2vaE6jfW*hMxag4 zdvbYM(jgm`o4yb)I_a*ekd|oWOjeXHu$EpNy>xB$qq*45$4$nnl8-yE?RmX=!=uj( zcr({O8EENHmvjLAlsxi`z`{can^$^?`E980 zad}T}FOYP4dE4n9vO^skY%KLpW*K>UeU3LT+c_V2?V@#xAhbXIzX9A;xTRT~nITh> zxzL@gGw6r6HLB)|8Z7WR|4gm|++0mv5B?4F{3I&Z@GG?O!yRse+Z#)k)^v>CXZVFY zeRPg`Pb|9S4Yal&YjoGVrfv{MJGna@Sl!GLD)o>MUjNi|({#yZHN}yvTSqjg80B%@ z$!6r6K)gM7M@z{)BMHP#?PHt%)a{xY196iS!Acacco6+X=TPKmP5L=s7p3ydIsJ%(GLs@DOzl{FR=p&k{3 ziTOGb6r1cmbpBeKG@XCT)ty4bx#~fbcwtEm8oEj8?A3Oa>~N497Z9qO`8dgdT4K+l z$vH|mfgSbdbJB@~KjGAWA(Ge^aQ^pi0q6;d?fsZ_Tu97&g7Iu9d_fd(W!v}q@MT`3 z8~iro9hP0MoM}ME?PPdy!v>}0K)zvUzf!V*TBG~+Kbg8udO>K$Xc_LQGUnMOAJx||zgg&E4z;HK@#^Tj&|&78xQA3?#iU%q zTD*3io&lHB`lI;HHM=(pPtVizFrK(UFS4Ip^4hq%;{+#>n3|E0}?{Jq{BV*I&CBRK>eAYE5rA6!*}Wh%W57aiJ=)u>Ygi>r#Y+y6(cY^Nho0QIcUa z-{LQpBV-yV1#TuL8H$e0(f5z1CY1A+)lBYSlKFn1)LSC<8{RoSdQ;$uA7< z?M@fzpp($t(~N(8M3N~83~k?N~>xDr$ulX%e0WX-4loMoW0Q)J*w?OuiFJW2;|9qIYFvigIn!wsyWQAlSP- zR=`H^@Io25RWA3u!{fREOal5#DnvZzVf#xS+&gK*En0MBB3h|L4CAp1EIRbRJ z2TGA$8#k|hpfk@sU{iXd?He0{GL1fIcxk<;USqM6Twyo;hB_>_u}-}lGGX!6W&VkP z&6}s1&DOUp9c!lmCh1|{+C5bkUCYW(0DzUPMKf2s^fn85YU&l%Lrh*Hrcbop9 zR_Kxegia15C%UGIdrEdRGxukv+0MFO^N9yx`lL7XHgPA+8PJn;%G>j`p9j3^>CiXTbJ_1a5=Uv6#)BkZ9FSC6jQAKW`FE4#+lQxW zl&_>WR+@yKfO|uG_ts0moiv-z2R{r};h3kafhIBGEqH2N^hELA{woPhpZRx!!4g^V z#vtj-%!(8iT}NvA@wJs`Z)VLS$83>Q2A!pt)f~sR%V#+an`7sf!;!D_V(v?v@+PJq z${_2k_i(JeRmlgJkxUZ-@(t1dt9%1q?%_Vk(rP_pwm+$GG)!!_dN8glfma1CIMtPz zY}@qOd8|mT(*0%)GL`qP?WD^yQuA%ebrJ#r3OX#U~lhp0*u+ z%?7oq2awm=?#jTSi*uYrTw2~fR=gT?p`wIjnUrXP$`1z9Irv0j@s}>8?GO-H;1~8uyq$hwidnA|s?`!BJ18Q=F)#{hpFk(acS4 z7EJZ`uo8y=KtC?8w3%?%sdK!{Z#^22boV*^qJs?$_EN$(NrJZbIqRbAerW0&# z0rUHf$b$ulYc1=n@2$rsmXfTvORCas8n;PxYV9vhI3Wn9N@Be>zWHREO!Y=KVOBj* z5|lM$;>C5A57JMx;No{01(y6h1#uc_zFUDmhRlU`JgY(B#6eHzJet})C?q*>3U(f? z)5Rr?zAdg)I{3z=_?#OfnW=BzQUsgE8# z$`9s)vqz>7scne;Od68F+v2yr&&%+ine)YdS+DY!_1q(mL4_qpE{vw%x_aV4#1;Mu zT3fgMGTD(E%j31I`ri9Y^`a}SPc6595LwX0P8CwlNDIkOeM&}-wJDMdPqmh zUD`$;!^wg9ljegookog{x#RVQ0LlJxiei~EX>p*Ir}>uE6T4XoJfbV3>wPRf#S%fHKa&7u z0g-uRJQ*yPr!XLfXdOR7s)Vkws=tz~EV6&U<9=hL!Xd%Y1hq$sRC*QpZ zyx@gXpLS!#rlyS#rBhTSpwbRVSUK)knJI}<+VORC%`E9a(U|v+un>?Ki%QC7USTUA zK;Td$q2+UNOt{}(g)Ma}gSc*~1+=3mgsJA34*RD0z48jviG zKw3U|qNzVSXvg{oju-~isvY!MyD74=KQZ-GPN@SVryc-8&s&8-hTwp_w=|9gIqcyLempdKB37GH`A)y#flXV?P%Oh(t83@ zq^z0J1ugE76aRUqk~gq06d8RE4W`^0kf{>tpxeMYIf72GmrCUj3a%`R zL4wqXSPoKXE>G5SkWN;FN#N%+!np*Nu{!(kUQO>3wru z;eAELQ|Wun{>YhMk@0z7c2dn`xl}3#)*xYXP+a%>+2vKJa5QId$yAl;Gm1tUxPyde zaQbACK_^oiqkGDXKs>HHg<`puFd~TNz=io4K3@VlijZ|)A4v_ao6!ND{p7*+Y(_>k@$#pD){EKK$MNx zvOsR*RyyKs3xlJcQ4}BH01Z~<8o)-Ap@`=gl&76!^XYMAQdP(DFLiT)NxSJY8M%u%lMAQjvGuejl9QJC4QfG6*kQyj!O=2Z>LN>O{we^u=N#NLOc8V z)mJ-aGX(n}j665>ccF?^xEiH`2>S>D{}t4U6YccEYK~l6wetd-Rs_L?$*{~-wsfR> z-UI!%AJ;YF4;;+}DeB@cX+^!WzOQ`AwALkZyd}M-R5#`QFq0b6@ivCI?8M3bJbeb;=H?2Bsyqg z8j>$TmJ^Z&P%l>ht9&by3@!!H#YQBXRFW*STCc88(1XCV>4=&buzPuPut zo4MVx4+`+NY>W&knCe3yl{duRHaU4Dvhu{#st>F*#P5veMLul%%yagh+-jdj8ozFJ zg5ybFB*x*udGo8OzK(O%9i)TaY@Tw(oe>d(bL7qY_fYG0dT*XjMaYTIbeg5XN*xSx z8)Sjw+=ip{zAK7ty1QWNTLxSyG!3c18N)^Pz#{%S85x$roHfq{^$%*aheXHW7o~KIQLR^1 z%+G&9Yg2Q}*M%R#hI62r^y9Q)Q`Z&hZA#=OAoDX*3eR}9S$=Mp_s_;T!^*n?`^m;a zIv_79p6Py1IkxvIG5*kv>ps1@V;?dwzfp*&-m?0h6ytgb+rZ45`wNp}Wx}2d!`1T1 zYE-+3-unUn)76#);5zACaUx+xk_=&xYxR*A|qY ze}9pcz>|{c9CW=t2B(pM#`mgu zE3Rv#tm(+A@#^4pr~B?ZtRp`_YXm!8w|-b>Yidtl+qVaM7If&S&TZE{aYlJ1FnHj} z*Y3#U{4s(-WNOp!OUg_74G&gS93ss#83&Q<*&<c=m0(-!O5|e|FL3V_PlBiXbdH zeJX$rcY$!ue?iZK;**0P?d8~i(moqPfGs`U7L6X$zxw?SWGUU>;I6UdgPT!71s6vA z7sUysvCwjz(C$@X-1u0+0nkZj0H> zjPCouvFU!E`)vJ>COKDF#GQ-#WzX)tt*Z0Mixz#+X2YdJ+NVLXC|3wO@8}lID}?4( zpoV(7}-En7cKWIQ9G1A}b0@|hW$^3jF(%;z0th+}+m084e z2S&^Cs*p>}dFf7v4yA6&1W||QoQHM2d{AG~18mW5Wh|?RTOI2)f&b$*(q}Nv%zovJ zIHI(By;4{Iglj;-(y*HJO4LAeQ0)1wckkX^!$0m;wYmaMjZ!j2%W1KlJQjOu9=DO$ zQSyMv_^Rzxjeb-k>uwS6w!bF~K3S+Kh#0QigO*oei<@P$;ak6(JT}P$K~S15j{wb2 z`4dG~$J(40n_A4Rze)1UM!1`;JF{hK@YvmJr;^PtoTb4L3Tr3WT1?x?!Q(PQ7&&rz zJGXuE>R_II=Utm3*QW!xERVGlXzW059|FQX`jL_nBe5}ZGJnbM*V)Q*<9<-itvF$jM?^RHH$*+V4ZLWA zjR3QA7qgR}hE%U`3PR#mKMlL{2W$M2l$b`3Z`ugtdQLLz)_ggb&E5o+$#3Vk4C%g3 z(t}u)%tsrH_Xoh4=&@aP4dEw}65|y^m#ow!I?QA0HFsRO#Lasf2p8mhi}82#*1Ake z@VDw9nC>cHNc)YjhwUt;_dKYcWvu(A9yMO@w5)*Zbj`rhVe8!-DkcZEBZ=4O8fa2b z#}#`cH>F*uYOG=^X+k%(QqZ-EyQqq3VR5s#?)Po#yG8gX8vbcM7RgM~wi5y{E3x}# zo#Q0DXB$&>r>R~S*sP`SnPnS9lb?WZBTuzy)|@O-J&T+89|(6Lqji(&nj-}`D*g&s zIOOD@;z~l4a)d>oDAMKAxGc$p!6J~kPxi6c%P(%qq6V?5eVeyP?-9p>bA~T$M@aX_ zEWUu}aN`o%bsd`C02ovW*H^#TTTj#B&`5}CjVm8h7;@LoI|}7_PPKa(_h3;rE7-5{ znLdkMXhe=^e>tITZR6fTLc0nyf#rh=(tu>3HLAawuyrIp9v+V|i)jen@82%K{M;;< zC1KrCM)(Z7Go+c!#M|P*ci%Kh6i(@liPSor7}hNBltE0+i)!xwU~0CdC)n+AXq9IU z-Ve+}ti9tWKRn}i37b*_T37Lq>sL6+`X?L>a05FaiuYlT6(FOIr5`ApZ!-ulSbjsmZ8El)G;$I+Z$>}bAk z#&F9t6DCfS^*DGewMRwl-IhAfRr+$pbATGBk?~~iL4o(3v(($*Yfif!AdnBZC%pR(sKf9nn{HP{ubLo;Xj&}$l;@{pa=g zzt|D#sv9t8w@-t^6P1L&@#eFxY#-!q%TU3jf(6<%YGa%nj=JxDo+Ukgo1I7Hr{IO3 z%qX=XHE~&#j%10!UhshkyJwkO-2ROKt@3X;O%C$^foHcHGd8qb`X{ucJ5zx6T<30= z+^kjysoXjEKjqpPAPh8#rS_Vv2J&x$R`P@I-yc!Rc`b1F-eJTx+!8o#5&fZg5Fviv)76-g4(Dx^y@Jk*Lzid zW)sEtg7G8SeP6kDmV3(aMH8J_gl}shBv3im&iSR5YWg5T=m>;SXOa8Z>GGJRT3Lk> zd%C#@*9)4dzRB2fMmlw)a6ZyTn-<%;+FsOWt4Y&h=5&?E7wU>l#QTm&JN2f|rmFMUMbLNVATOGT zn+qDxZtmL8vW1edq$781#N`ed-3XSr1+BfszB0ZgT71WF8!z36jYEF} zRX8tFD5cP)1Q~@4K!zhirmhTHwOBK3D=?I?m5GxN`Z*}_b-ro#IoOp>M9#fQ%%1)3 z_h3cJd)J=wp78tiKOFw?EzlNNq4qy;z<`B=)Y`~9k40%^GlHs4pPGprijA?0tRV{- z`T4}CS` z*x4O)?~r;1_V`@+_xS993VP#Cms4!aOf3_!Sj9h*?IIQ?KRsXm7T zRexu$B1>#VI6nyJqe36h_hkm^(LlSXOEj-z6s|uiM>eKucB3^Cm?AQ zvTAHl{Djzebq;~5?AWcnx0N*K#I`Q0?VvWhQ2ANDj#m7dGvQ(urC_ZXQgdihwm`=jbVDWDtZT(76|Tp8;J6wW(UrFZCRD#^G8*Z0%#YAN7nfyt{G zN!W1n7@s9Zm4FN2frj#p626H~`{RZvpdPf7Da=Nv;pH0EcJzRHG{An8&HG zrz2iDLDwg^92%VIgkkS2 z4pxK)hwKh(B9z3*2RdTW`;Uxg$q1OM)l`c49V0M;BYna8)Ndr2#IY^(vvg_;siC3Z zp!LwFUFKp=7c^!d$N!TENV;N!a-vi)l{b1WQ;yjux<`78N&83TFoe&^s-Sg8uc!UP zw|=yl9^0-BuKQ|%RDE1kW0Tf)nx;i}?87Ie3##?5h+_|U-HCOj+z~l^%%aRa1AlE? z?TKOGPQ3C;Q7i04XOIcuF6Cy~aC{WDB@T8dgZIf+qUV~RH*$CaZ%gzz zorn9SkIojuoGF2g`h%dtB1bD#FyLFM|6ulh;ldwT!73*jn;b>{lN_mvfIc{g$b!)F zjdeZpcTA)Iq>gv7mdlD@NgaywZ7M6X7cuy8LlJw>78;l>Pu-IRoU?*6()4^jbj150 z8ybg5Fl*QX;=(h28;nzT;=1kdymF%$7vbwb!L)r1nyou6Sq@rxCLc<`T+>2BU2@*k z$!jZBX>wy7=8fkF2c4yd?Nw45zjCr|9(vHe3MOt@OB%r>j z_4TgoC7$%10>+e#;QLc7Y8b z39$bs3Gf@00?Pd;Z;zUM2=Y}vUN6HR#0DKx6rk$CU`@SV!F}^)MrLRzv4SKSg_o$R z0~(C;=uyf0oJK~wU!oPIbV`lJlLl|B?P%cDLe(xzOfTxOm1)q2gp%e;N6dRp*bPe- zJiyo!PipI}aq4cbyXqXYv1;fF;)szf*QSt}>lTnplu1kCRV>|6HSeochZhERg5f2S zRk<7J>u<~q`58x}Xm%cj+wkC8hzOprj5=H0^6rPW5{%V`3^<+d(uG`l=24wcqlcOE z5Ny`t4yr;_)umWFK`Z#@3ytURnd-mvgrtlcxg7MO+PWu;6nnH2nkGCGwb+_Z=#!`R zEQJZEkhxXuCcgurh7t1`2!!7*GR&kVC5C@6e2&#jij}C$}|}8P7AC zdM~q{@9S5jdM?FD;vJ`?tK?IR1DlelCiAE{w~ikq=mY6y&@l-&Ctv^1S0&NdHa+`Q z<`&*fm-J|f4&;lqrLrCpCocE+JA$QkpqoDoUCE;lTYp*Qo;S7cb5%Au{^XgHuAaF8 z2lq&OU2)-F5=XU{T%983?L9jfDGo)OnT#r${TYp&7zuKpki}*}fUk2Qd-%~|Qe-?~ z>eyy)&eJlLiTij*5bMe!MCINOcb?QQ0hk!q;YltwZUnA5%>=?-HiGYZhTpSe;+sa*!i6HS;%IZ5^O~`swa%1C&)YGvz4{E zh{j%t{vXy=cWVx?u9*K}U8Gpn73mf!{0OkF>%lvxzQsfyfn{A5S;3QaH8sQbpvw_E zdPtmE4=W;|>5$m!n2mcWuGkZ=mqOi3TbICqdvLp)RQL2yzr`kJDHjOs6m{M`Eggs> z+mIWF@Y}DwSWA(cbh9w5rx`)mj(vx{;z3tf&o5lK>(TP9N`xye$ak{sO*{#Op1BM) zidve0;u}WlB^4b?JiCY{ulz@o)pK@(-r5RdUOVcTyO`-qv(z%1dz{Vl<<9{NE3#^4 z{1R+EKJ&~+%587^ZMU#ko*nVxye{dz9Y-FPj=hRcqTfeEqJ`P({v7GuGg)%PN%8P z;0|Ji_!0(f8wD+%p1OT-d_&QJH&RvDQxMw7>#SfJkbyJXT({I^DA}6~r*VldVDB{2 zG+<43&A5R)sQKfE{0^%tA&Motk>l05Ri4o$a%;HRfalvM+S4#MP+P``#y>ZBoQHpK z6Ze2C*<$DC+m@$~s@w<{tvTIl$?XdHblqZ~$+i@=*4hzmja1XTwkqXs=bVA)GgR2x zP(7Vv45S-nBkD++6M@7ZQnu{9rz(8Z43>$tzizm{2a!b?9%@}ZI+O_;&pM1;)G~=^ zo7Zhm{#N<6`uNDmr*1OJhs`T1;vwE1iGtUTj}YnLFYT;flr~$Nw$juGdhYCpe<;R% z;AHimTbjhWo5Yrp`~SZMvX93=e7=zZWxsXSYq+fzY{ZTs2+6;=`v zuUu~qGz~{J(W*5LbG!b$Gg`5yi>k_AReSQJp}TUu z<)anNk@t!Jg~8j{Dp0~^94dcZFWPEmk?e2ic#L&El?aYq7rBsYZ+>^yc^WQfv~CzE zpTus7wbMVjwTfsjumr}O2t;8`D{F9JF3Ids!18FMdAE{9 ziS=+i&A5nHN#X$w*FI8g2-nwlGG*#4%?^ws%{l}&8~TXEzcnv;FF|fIMLz|TrY$gb z=iSP9A9sH_4QJTLR4=FJBO`pBqU~=lP?WcPIbRp&{AkG^GpBgu=}MGP^u2}5?VE3+ z+_zZ9T$Lq}c=JYYEy*5&s6;Y$iGrUvW*wY z)xV9^4io_{gTzN+BXPI2c~;8X{LUIPh6{ATVar8&h}~9!S_o_wVr$g z=sNm0pfW{2!j_9HpVT%RcOUMwRP$+HUZi4i=iugsc~RrI53sn$*edt_TLMYFaAkQdh#t%^3h_`ixm|S-dysd^s%0kfjNVVi&~o zjeQBvaUBuVvzwU`zFZP#z-O)rXYO{B`z0=~^2(w%UYJT+q+>%*lGE+2OwVEf+>Y15 zv+i?`-Xkv4aeOl{PFO|E4b#R$6+DmZjqJ)LwRl1ZUUV0632#yroc8&)?{Vr-P6P8m zdkt@WZe*yBJ3U=$%A6TFbwr79BR9@4#nhb!yIr~!#zgC_mdTnsnC5p{fg3rze+CS+ z^9uNF3I^dv_lEFqaON-k=U031=dKn@^wjAjiz~$T|EeVMSC!v*lecIxj6;xBzg{dB5hZ0B_z8wY|{&_`Rve2Gn$ zVu3Y$?KM}Xd#^ZLV2Ti~aZ3 zY;~}JwVX59MejbRE3;+1x46yQ%D67pb+<78bD}4y@gBd@(|&1Mhyc=&=?L|YrAwox z2)~CGTxM=dztb#f!Nzw3wKLRqMXOuQep5@qMmht%`;|@{l3HG6B2u0zQ@iVE8DVG6 z#-WXA2p`YWaJ?NWiu2_LnK!%PF^ESH{lLA52iK;+K_91grQN)G(zX9gvrz(FwN?=N zh;4rA9_X`w-~wAp`IsxO>Hjd7$aHvpHJyQkIgEa$31x9IoSX z!wX8vYzJn#co`*XN4yv?QEe<<6AXRB2fWFQRFR8+=r+Kd3&i=_Eg;Sjc2{v8Hg`#Y zeox)rX`=_0ps{8Je5^cdp(kXENX7M&vgE! z3=CjqV4Y|{Nhhjxw=n8aA?J%P3D8Eq%^`EMJbnD`6sEsphPe~Kg5oWA z%na9`XBlhlh4ID;*kMyq($M{%To(11$w$teqqQ}$N7)1<%d&kHT@|K#dXT9bh*Oqm zxGN>1@W0_4KK}nNoE4YT3ch~5`(*JJIwar)Yimf5x+HD|t?Jumwl7Mua1>J%oL)WS ztRG7XEN;^2+y7a2&dXn|wCIT(FSHb}o7U5Wr_Aq2nIn&@7icJq`Mf#(kPQwYuKNF_ z32ID&fu0U+u%T%u*e<(8pu^Z36db;^$r$``L^&13%6A;57}lKEnqrTMv0bRph5>87 zucvw-6)j<#*6^V1+WASvzshwzj=G>VhGcA0b8w#XfzF?%Yo>q}R3xXR8pJMdG$IPB zsymwT*mpi#K5yt8xa?)(b?kGx1+O||*HC?L^&HFLp5pjLq}7cdRn z?IUWq3JdO>sVdvqBcGss! z4w^nw)XoUw&5C;H&j{nmujcy?qQa3^w_TAlB>hir0yOz?=dcu^R9Jgirmn_s)P?{sq5&O46217t1eq- zU*L27yOuKo%aKHOg$F$C3O@Rpi>Tt3f?2fJZnv+^)eGcchi7WW78Fo+RT~1tW1mSY zU{X8KW}RZZ*2+_*_xAUDJ*s6VEnD%K_8vV4jyD&9tM8XYZ7K^>_5=+9{#Bee@bM%h z7HYL=1*yC^?ZijMdM8pRyJQQwkIqUzj~ncaSJXv0mFZf>)J6WZjcin9pn!4yl&71V zapl}n?`k{#?Drzc%lFVfT>a3&G^X6>Vm{Ru|7;hp9wtA-@@fm$4aTj_J+jFI=q!Lp zZncrI|K`n-L(9Pav#%x<*w%tQTojWk|Q#HrMu3QN!ldc_q}AMDyFX7=(WDqH}-)Q ztv9YIHM#W3b-7K-ZaItenzBm^R+m=m*+vfFc1o2PzX7SIU4qPVp^TY-|D@&4y>Bf~*M-tK}sS|D8L!rd0 z9%JJ@TptCVu3s`BwW5TK^d<(JnDS-Rt@8el*``xr(2X^6GUx{ zHles4j|xBwNo=48lw9c~2eJ&8{}3-fN`Afi zwm$HnZ@%dO5xD(u`ZPb9w0`SQH2o3&*&|!$-**8NXZQ{Rfw9%M>R4X0>IYYh)E$+) zd|qDdOkwTgU~D~yy7i=3T~I^+{@?gS)*nL@b+X08#`iW$gr}*G)A>8Q94&dF8ItMe z2PFWNB69L0Dbo+;$s}{<(rEy5R&{7#091Qv0DT=G%F8h@cNT&Vji144275sKor*KY1X!v5?oz(5iGdS2# z!|gO&>=p94iO-+FW-#mUoF47GtQEXH3hXxj`$w{$1z7p^r;L0ECxTy}%)FWLua}p4 z^>yaS!ChY|2Amf?fdS90sXIcOXTJr_owUKzcq09{3Ql+=8hsLl$eO-GW6;{F-E0t# z(7?OwsnuUMHs&m_hZ-yPLS~eD5g+oOOW5Xhij3t^MY!iuMY!d*9$UVWXBhmH{CtKf zq5IlVt?QAR9(v2Rrs#tmqc=~ER~~&)f0E$aS+-`+q+v@<{lL!lA}u1ZWGa-#u#$m3@2VzGw7uWa+;}g*#@i_+HkXc4A=RYrbSs4$%pO zzld5~GcIiLMcu^B=pDtt->CN+LB?X-1$zekZifps)U@;&7OY1}sn0<>+TheRZ=pc{ znZrq;J3I?8yVwdFG0=fB`DrLC?d4|UqtX4O|Lo2w_5Odozchb8@5REx!pyHM;(BXj_%LXEp^YR<3ZhnW?H zGb+0(`f~QFhOcq@-@nw^tCGgRL``s*h>Zz`=HW`h9XZyHQ=Gu(qiTtqD>#;t7Eu6_ z{44D1nTR@S1VHaaKy*K*L}LM+5E<^XdObJaMv9b~gFh@p2$ADK^PN_&F2YkI zvtJ44#LJT@S08-A6A?%y6$noXR}V95mFZ48d7c+R_1j5ll8h52J`>o$%)EwOB13rh z>ylLhmVn-sMDN>77hWDGaW3$tOCH$XP}?0#s9C5+G_=cgv6?$!^qPkyrZ*aL4cWk_ zhWGqiA~|MXxQg1@u*P~+jj_!(4QRm{4@wXz8_o+I?SIgWR*OeELKwu)7eD{CC-_76 z{VeR<7(ola-nbGm_s_H^`2%p;@NMQVH$XZA{IQ$D*IKE%s)^b1oBc61C<}Wv zU1hSt>NnU{nqJEp5fZEYo(L)uauQR*pFx9{q%xxX^YK}wr*2n=Vh z&=zu+IQ( zRuqu>9tIxJkcTIQ%c4SZq2?cx&++HL8h-_PCt4xmez z=f2}|qZ;z!VA8lG7&c`O@8VcVM)I@(Hj91B}wb)Z3x1z(}eR{d}OCbEm zBn@dbuqtKl=S)#HV%Fyhb)&Gd>EuLPK%5AE_i@)bxt&ok33dqs21@4RK!=}p-DqlG zh$xIvU{O8Fu;6Wf*p$YD#b6Do`K>&ipK=b^^m2-rkifcge?layomlu{RPuve4LvoID*L% z>5~qSrXEbFn3I8cx>5PII+V$RyP7&io5700W!kjXletEE?`NRtFj`SOZ`ie|JV3+F%d3u@?WaB9b)gXF< ze?7(CVtClRw@`6ypaw4PJ@S+AC>Jo4E3)|b>yhLSfNPY<&WmB;_TsWF-bXpjpvE&<-=2Ze0t!w$? z|KZ^E3t|vMC;zUv0Au&uZ3NsMezIrqu2Yl|SWf;2y7Hd_jLoj(H;k;;e9g^S6{Mio zNTy&^gIB38v;Tw0ECVpeDys0X;p~NGFAogPtVRTiVK0f(E=tr;syS(!Q;r&JRlV@X z`l?87ql|80*iergveE^-8F!+d+O5XZ6$KO-2)4QJtlClW`nE>J9YfdG@|(2~65ijN zp67LC=-C|Yq5P^w-R%M+D|u`8p;--$g3z~GBH`;7-?Z5W4RdxY1AFUdd|2qc`-aHH zyaXX_7%eSiy7Wp9?!Az!g zl^gVG`nGRMjnZ29d?@Mamew6$11Q{;)>&m4!HFG1%mav|3s)THE$Sh`uKBen)ROq8 zaMs`me(qMe+WWdb*G;yUm+zIL#1m7X2;a?#ns>9ka?>ajBX;&fLmx_?#6?spaoyv< zvH!4Z=A?Uh`tAX_^Psr7ZI$mHvaKB(Q7rCGGNoO&^brYzJLE-5|MR>a{D-gj?Ei(YnDP#H8fWDK zpnvIECn1Zj(*a%#uT(_^I!B^5C_g9rV{38TS>9dn+H}~|vkKKV_+-%3&N$?D@_vppDNDOsrO4+&z~sJzA%`1}=fQ zw%ekygt)5WshY=m*P7i6W+~QmtJem>8LC;p!NIpnoe>)uJmT`VALL?vSI|L#yO;mF ztD@q>w|&i*svn*z6qK)&q+&CE@d|Jk%3ElwI8Gh)Jfs=~Y@2cbCDDYy#qsh=0_p4E z4aIFfSQrRmpco}C98bl4!PHS|)JnfmGoq6H^3Fz#T6!2QP@{NOl&+&`p&SKOjT8ih~rT2nf+xYY#=kuiSmrQ(%*}14Fiy~i<&rMR78#?@r ztFBA;vP4WB4^tF;>}*EM`pP68lia19Cep`T+56dl@eqq|AVAMl090Z(@h}L?5#?b< zHx;CSkXmia^=Ru|}C9U%h3iF*CpK zVJF_-_~@I$s}1E^3J4G(5H!mYvx>nM-_87FJ`Q5=O!dONi2ia5I`~|iYOp#DvA`O6 zweWDGpn_K$W-$OfA&C1KU(?d5ZoCKfdwKs|>5~DiP7g6c<38h;f?}u79R)lq0;Tea&KDfS(_OpNB#?g8{cMfdmm$zg|k~_8`C$JJucEMQ(u_KhXPiR|GwZ+i& zfE)U_b;H9hj*Ylj!ralr{s2-jX;s`MO-ak{zOUuFE8uqMp3H=e7W07IP$VK{p6 z-Mbd^2*}ypSD?6pT|Tj+9n|w-nNe$Jy2ihZqE0L_?+Wj>@LN^1PK}i6I$Lc`td+v# z?OXPX+k53BJ{a3kr`lq7n!5%8*P|l$-JFe&*D%U0g&G-GmZ4s!D()|Dr0&GgktJN*CXERluvDjs)KVog2YjNHTDH>w9k8Q*jHbF;rX{00N0+|nR{pU zQUXe8(U~Fw#n=Dx%rQNn)Ul4Z8tZz?&v`BU^KR>EEb5uO1!quuKFZb2xj%mT#hUy@ zq66kZ0C8mp`Agv33BVP|-=1F50X3riFI~B8ckCpgmyrdF##OpMp2_q6m%N+nSCfKiMw3rVtu|n`Dwt|DcXx>Zy zn9ko$+6Z}IJEx~0Ch|N>f^;U~b*MV*ZnErQz)t$>l@h_cyxA%RF|Vy+hW7Vfz8V^K z1+*C#q4|Bf$`NkGipPdiWqNg9wxE@>E}pr94xSmB4%zj(GF4do1NuGG;I_3>KraRU zCY(ZgGk|ncpQM?TMpY)XbHZG>n5Rs3GLi};Me>8UP^~z z{Gb&$)tvZQl-R1}*~p&0+NLAGp^p~3>SHF8+1gJx3T}@p%3pc##<1*S`X%A`$N#dR zvfzpTe^^lMxcww$UFw3V%r@`JZ%@}N^Lp+yLWW_u%g0xaf`A5n%=qJL-pJD%pye(S zm;cJ*daVHCu_E1y;-f|Oa=7b_voUjyjp_Ye^(9S|`d_8CGavOB4oRNOdLPF*%>Z}+ z*VEPR+KKw_a*On3&f)KAWJdeznp#tcQcx@Rfo-#E(RQk|Ay@Vtj!dezg6AMW9$S() z!(mk66k<5wCCW^cAiSmrV@MxPF#!x>^=QW0Mifk*h9XzSB zO?X`Yp^GvBm;Cgq|5ci<@2Zt^S|L0~{h#{zXb@1`)`GmVjzZTyNvH6(ll+s5M*avo z1hoew;X$mK^>Jv~BIZ>e!tQL3fv0f!3C?{ZTbpE$^vQc)0exov%_g+J6}s&pV$`kv zh6^&{3gm+N?pIZBe>D+OHob95x!@;w;@$Vl1-NweKlyKDbNUnle>!Q#FxmyLCWndQ+U$$O15Ylj(I$h$qJLgJX?m3AhoT92odPiz1e#{vy4D zZ>ot7KLs0Kc+LFM4qDTMi-W(x^OMYfCvg8NmD-k|MkJ?9-*+Z|S$8(b6Z;BeGXACJ z_wgh!i2$ydr|B7)m!=TwDRnz)NU8*G=^ zU;MgFabEO}FJ|`>B&J<`$UnP=vSg$H^;LT5E>FPL_qIFex#`$KJL&FSQLIy!VLOlR zUKq1`)a%R6VEzHo>Et#BLbv+y#A>b`V1-uoXN{<@{`tiQ0eWRcaieZpZUlPnmiIHD z1ZF1ag}zn;c)LW*Afety-W%p+c#DMcwBK@GUQhGoBXmZB_x=G$A5X$xl=4UHP($R`cjpl z1g0XV^F$ursXL@FwrT$En;RyDK>4LI+Ze7Q-)76#bRSOj7sDFFW1KtfmKr_7B z_xX1WpmSQDl=|=w3S8^|2O_Vw;Cjd&QFG<%&{+ZjiqU%y@n2?H?i7QF_N~Ldyl4V6 z`|KG=59Ualh7KEdnGD&XN$0q$3p$^@;I*CA`rSt20-P;Ws<_-ph<}PK`)O*=nYqS} zlbjWPmj9M|d)MNtNC+*6|$f<0dN~40gDR z8Vl^gU2ZuyE%r!E`OZX>MP$r>jkrklryc}uf^8(qU=D#P-FiBReDHYVl9oeJ7`M_+ zvrnoLd?si&@ruI8JA#VqNwrV9SI=KfXT==0ICFDzbe}|e^$t4}`-K&abVX31ON=jm z)fFK##SQ(e4yHL99Ds9YF5|oK>H(3R29`^#JD8lLAndMj@6S&c>=_RfX$5VwFEO)0 z4BpF3(@_S&K;KrTnj(-Ue(n=GhRqCIPhS$-nJ_KwL%-j3$AXzhuxU9vbxI7r>rzt? z+@)HdkoRt7>-lhge+t6Zze_)0~YC^luD^qm-JqhBs+KRfV?eHR?QgFZX*m)T%C>1+b>x zAbZ|Mm+p5qV!0c4uJnVvf6$AO|JXDE6@LSJ4$5h}Ba-NM+`Nf|9{kU)5BBLqwm~V< z9hs(7(Q+XCU!zeaHf}z<0MY|~xwWaM>Ud%zDI$vs)8HeGKyZq<$zLBmS$QqjxG}M=WjPh5DS|uMe}3^7fy8`|HAy zT=W!m38nzMaAo+o^t3qy?Rv}Z$Zx<^UxHlpe7@2O4>DM1VtNoJxYLvjxB(h@g_8|} zp`QS|9_5>f9heLDw6uy?I?byB(g(Xph_9Gb zx+Jxg)xNd9{@U_z{Pm{9mT9pP(MAA6M~Cb)$95ho0K6<$mTP?xG?%KTLQj#j2|DQ3 z*qwSRxzLo9F`NxUKKQ}s7b^fpzHWALrG8Gh9dUgWg6*Y`KW#H*$h0HOg}!sZ1(fdPl-?9 zXuxU8C0b>5fmnN0ul}BAcC;5hjs_WiS^~b2M^G4>?1l<06y!Ml%FA>(Wwh|^OE2^S z@2Fdpm|7q7(v2a_{U@VhjrisOY_P)bv^UO+d~g>)3;sBb)oL5dvXj1e+`+bfo1{s1 zqQ47h<0RKeMFsOVx^Sm~SdV_!hZkx1p1c#9Kv zUcvXK*VxnP06+pOh)U<*Jl5uscj!D5zf^gMrTPTzI?hV5SF&iRr^Wd=BCnCMRn17K z_L&0XOn#rT#`pmAVh7}P^7g?w8p#!l$DS5_a(@1PaqHA)ubyR3@kU-=9y+QZgW1nf zI&xzKJjj}G`LpQ~m~_@BOe5>cPxWtzEBd^Ynr$bc2WAyTvm<2n6bV~AlOM2MleuC< z-OVThtrt)XLNJrh9tiFT39lxyY~;p9R@6~p_oEWlC2*&GJt}bB%{GPA}Ay!BAFrw(;FrbHXJuD8QEQ)A`)dI&Heigwy{9r?a6V5C{wqf zU6ZQ?2H70yJUy^FZ@Rn8hh#?u9xvTI70BGYNO`eJI+jE2m! z-u@}^@G`qD)8y7?LXi~B$E*@6e!lC%n2bo^ri7qCNW$7-BqEmQRr9Ip+K9c_iR0kq zYfi*b>#&9ETCVq2ZD#MJRz(uBmL%?_D1m*sQJLc zT%!Kf#57xfu6%E|Xl)=({%{Q<4_Jj{4Wu21+8S>oD)i|C`Rp{_{KNhMcx_i@Y|i;N za#a{S@P9)`1rO}VO#1a+Y47@PFE;{2fXWi|Wo@FuO zpYdRiC2Z(sbi(q|2IS`a*p(AclE3L-7!aq^PPgVbOTyWGS$ejl$SX!KjI z?4m$2J{{n*o%qg_mgwP_>DyfAGTl90WEIPXUS}~abS)W$UGwk}RS`d-CDftwMD0ym zVn1EiO1hYT?%w8#P&?k4%_RZ@2{YrPZFjR_@ET*#T2fT0sc7x#VEhwp2@*#)#-PV*xxedrNKI#VUG-;fPwgZ|k448&x8j;adu z#FqL6_soYRe>$Ump)Pbv`5e+M`}|$Gk597?w8Nw?DC(6qW|GHZyB9v-7T++x|4M>h z&n1&7$T+v;HyoVknE%2R&EYC+5!oA2VH7koz-6(Y0IyyxUK%tVUCB4(2V)IgQm)aX z5~CPuu`2U?MzA0^R~%hg@oo0HLa0PZF?{>m71Fgcy*=r$73L`cLHb4w11|9&C9>BciP?R-zG%4$FYw#2D5;X)LaukK=SfG<^+f$ zNv6O5UdafyKX$bb{71njfC6oDOCQcmNK_Ln460YvlWxAZ&!X(Yk^xvPXm-{V(x zUehmBsXsr=L~2#0VNcen(MRq!MMEmC8-=Urd)1qsjF6oEYS@FD!QGm1`560&gp+QzR# z!u*;9TB_>m!n7z}(iufUw_R>cj1XXy2HgD11!s8A^)wuHwm-p3*j`=>JC-=1o^=>` zYCTs<(<)%%^SYL3U3VP^r_QM7n=H5=*g^v|is<3qrLw;5UFebA>l??&ORPb%AOuhO zK8{c9=_-@gd7Ar1XfID5Kg9dXZ0|CuL^kTnoaL;l&&rAnRV9_1VRV?fibQNrPmh5y zRrIZst+n(a_hFMDt;@@9b7RQiZNsOQPmT3-aYE`)I>+x+#!_%&-^=HQm>>>dQK3AR z+~9i+pSJa2lc4Ex;BEZnn6{w;DccKs6`x?2AJ}tURI0>aGM{Vl(N{bi`(f*Gcpa2H zQ;+Zu@*I4HJb$<{7iQ;6`{>sh&a<)Km66X9bwQctwMpDm<(*;}FhunbQXd<+> zqAgPS!KRzRD!O9HdHhns8-ZseJ5-C*7-o9-N?KZ8ni6lif+#ri-dpnAR}_>|Z^_8W zWFIrD<%SZFy(P$1d`m!1{uuOLD((T1@?EA}av}n?ZWx%yncsbPr`213V!rP%Iw?&E zSU8Hjw~6BDJj?X4046KK9x9!Zd6hp3UKh&Xzpso z57Fgk9^q0u?Y2S4#5IoJ#h17|9~Q!#7Qy?jv*iMAHAvNZp}gGOMVgJLo7YHZ0!Bw& zeD*u|tROt}muF(6V7m}C8%6tc#-4TA*WJVf1o-%P{vRbJ85zkQB@qR?Hm@|q)Sm*D z(DR#PbTxLDj~+ZL49d)u9Gp8&evZt{%>0W*?w0iDo9UCNhc<-~QUP+zqAj*qSC6zgQH!Cd1;y_W$?NJx`BdA30Sx$;nLl4v9wU$s zRqLYO^?B!%x_l4Mk*64}$CVr`VL8~ziR<%iLXrw?-#_%r}kM{R@skf(E zY3=WT9!ta$dL789`U_}IiJjJL@m`|FIQEXFToW&bo^1rIT_*CnhH}1?$nQ`5{H;VQ zCv@4$_jI^p)u&d*$SB>#)}8sTYI6%$ ziG`Jh%57;|&Le-TusyTEu%B2@-Xv?>PjNNYr)zhjw6RVRohiBgOVx87H64L88ICLZ zT7N+nPRsv*kd&c)%9Ylv;xd&%i$JY|Bs_Ro z)G;GwI>ezop`m0P%F?~p?xBBw %~+ID~0diXaH+9zYnB=>Wfp0~AU%k9^n1da2z z_>ro^C8Z_fR2f}SL33v%w#+TFWU zkPuqppoLgx%IiOw_KMU=U5!16{lquA#j6<*Bgac}bCf4Hx)`4}(MXqw>C?@QelT53 zt$jql8t@M1b8lv^vxCkm>WveD@b`=(6EkI2m3zy^1Cmpil?I;^NH;9*25fkROm9s} z=GpEIWHg_OWlgEmSDXGJ0;Eo$PY%t+V{x;B-yj z04^H*>l12^r07#-A{9|NIxuIoN!G8)g}+2Qle&i>+H^$P%=q|cp5k>6n~!Uq(5DjA zMH1XU^8!R-5*$^G&V$-#gM#zZ7C1LS zvG;C#|BFg53a7i(j$FMYe*T|{${GHGeHj}(jmRo?mC65;MXz#Qc=$)1dRk8x^t9Z7 ziz0!4dQ*GsH2uCz1@#>xi4qnf^ey7!&Sn2}(VaGm0|rfIt6kqR-5hJx$*G(*)Edm8_#6=6UmI)Z8uPl3ml!|e_yMgj@#7Gu$?!E zdtxkGP*XQi0lSR!w@(b-1;JCf>7Q9~lBk|I+!gR?e#@pz;psJ&T*2!)b!%p%f!qR~ z%+yO(S?=15;TzJ8nAqXQ7gOcawh0O!dI{h(X!9jktWLGH(EEox7o6)%gQO6Y?bbwN zm}X@#$ZGBu$ZroS#KX=mUzgUs?VG;)+0WPbYp(zVcHOiyLcUp`Y=5V)-n>f0GnbTz z=m%+v;HNvHZ{88!Pv0ITe@RIC^)OCCB||*m=qoi2m@${$YsEKCAP_Evf!&+df<)1b z5ke8ZmX?nMe4FQ#=n9lG?$C=Y`Mkc_^Iz>LN?6DeOEH3-K*F>WqsG515kKtq#px%6wXXyoJQg?QD;YP3Id4nR|;4 zCGVDKi5q8tK&(S)3F?Z6XWt%L`n42t8{3tHn-K?1@#d$jj(&IRw16tto=R^|kqoQ@ zR>iFtgCZN%`huR{7B}_kpY54;eVAXn(aZ1kA@2A_PWkj$r~S?hoOKOf{;65_Um7JA z3gbzlo>T+r%AM;&EMzoADt2#1HmuW|#}U8Ju|Fo0)E9D`u9=uvGG;R*ygGc6Aph-& zKB+lvR1C&f>D;6o(-m{Z51sfJoar<6NnODi1p7WMg5Mz~Z0y%y8BTl`Hq?rpvaV-wGbPOXyJ&A;23 zc{llYlj)ih=wm8pK}7|Ev%w#403zUY;~INyTDFc=d!as{?|`Qk&~g^0`7!USo|jN`mI>C{0g4TxgIDUrM)0Fq;5(g z0XTq^w20;So%?1e4S=0w1`0%Pev*X0afGz2MBP*RMs#pp@aEfQwfmh%XBr;!)V#NxxfqG}B*np}lGrZdN5ka^RcR%J~h-{{514-+BdIKtk0+;N-P;d`uYAzT14vU=@&eqgbox zQ-YD-b$N=r&xp}McR-zCzRE!|`KoCj*fI`J1K*}i)%MvoU$aS$)h;ijSPyF9B}x-c z#v(7hcj%g2P)t!a7hAetse_vP-M-7HsqvkWdKh9DRp$=0D|FamH;OLivtIJuh{Ia{ zX4WXt`}S4K|BkTe@PVU{vBG@)T`HQZmfVgYyeg}F`W-3pgKxp6QIe_-O3BYC6cli; zjX05&C*Gdei|dbfS=u{hLlDM@wMjNHF$#W*_JH8iW!Bgh^SPe+$cPJlD;W8-7R#Wq zm@mU`W4o{Yw@c|FCg#e@l(b90tjnvn<~Xq5&fFJw3UW!~;=z8iGudz0$s9oV)`7ZU zqCU|iorv}R;AS(S828%lbsuOl!|_qeD+(l=ssqp0r+FTwiMATY>!$>|nObo!C-do; z9_wUZ&5?rAQh7PuAhAQ!?1xf^X3k&NdTqbOa2l5jXn&Y2{ah=SHd@{~AAU#w{-jM5 z$QyhoZDM3M#0i6YL+YR2@2EPlV3j84Y5#Py$?!G~=q*W{I3kP?$`rB-CgSXFk`A1zfGm7_#&^XY3opWr>EZ14$4Y`T9#% ze6bGI=hi_FR=7fns3`6B19a+9pud_vyY|FIy{#~^MeicJvYF`v4QnhLfz$NlTi32# zb69MDywSX&>9TWCvsF8n{V9UbEo>_%Nw_(qRA zlxa5DY_?)>(@+W*%F=haB|;EnR6RKgqnW!ugtJQFi=-N6LiEaGEVT|vc%mM>gndL; z6+X_xuwdpc&0PMuK-(tifT zr_A{8XYJbGI?l0yKp+pwd-u8tyjJ9z4z0!X)fKYc%P#e$Z_F8c5olLXGjev5`Z836yB*6anr&r&7LHkS zXHe?cU;gkgZ-r5SkIen`+%l+KTZL_iKP=CgapD9e{^ z2OsQ&tHOP+34x=U)bk7t{V8p-UPEH3J(_dA@TyKsykDx@ z*jJi$IYJ-vByBxi$oI((Y}ZIBO`M`ct3oPGxHj(^*I}O%r^!QnvB~z9T8g1mqO}16 zF#<37BW|Ordl~KU*w*He1%d-UoLNw0g98&K?Rfc5diS_k%}553J4pME~#Kq|34U?9SiOB`aUM0(46Cu#^>QDnQoZ$a_Mg2-W*CnbB@D^|N?urj`~s$l-FY zGt!SVN@841rJw-ZO-QkB^6SL$0Pk9(djDH+PLL$bw2PtCpiv9scequpn&g;$8;kUt2Fvy^IbE#wOT0FES~8D4SFWFX z!`hR*szXV&l38Kl?$2L8i zqD1|%1IL^a?>(Drvy2(%?+|z_@@}ZJ$WGe?u`9=xe)TDN_ynXN&gNN!WGq=xK(#RB z-SHM!=PD4;VI$JY{@F?32<`{mo)XLi1!qmPLYEdiuA+VWSj1Ov#48dj^wSI@LK9?N!=} zHLZe(gSmI3uFXz9xtD$|`<>H_?D(dgi5N$Zr0r>UlcUYs9;-9%jfeBjg3O`#h4Cyufn)`lQCXa@;mRA=1yV z{+q(LzffcdYC_vPaDQE)boz+2bG%H#bhatN60yTuV`X|~p-{_XJkZ(TsQxg>4eD_; zM$JlSNokfRlb#`Vu6{bbKT*j-ewUfI-IaiOX=GbNCbM9=JBnU}X#*4U;x9@Eq;JHE zn@gc%N$^1{sAsK4da!H<@x2c988j*;XLXwZ!qW=S;n}0UHsyL>`?0JONNq> z!6d@IXU6fvK;jcmuM@9@Vlb~rApIy0I9X**E8?bHL6GUb!{w_a z0((+hX5_AD^p`P_SclCEN?p+>qJNosBV>;(eR(1=Q{T1Ij}QVE%}!Nfy!G+g>!Ngd zv^svG4tf4799kxwrBUXjE7N`x;kyEmC}NPxuMt(Tkoqqv;x4g%tCVTv4d*Qi2XV;LgXs&)i0BR`a@m@>{q2cX zd(VnuzZ;(@YOE~m`r*>gS6fFYL|cLLTH?whDUc#CK~XIWwJM{!c+U(K?Nya@Nd5hO z{(@J|SHsOKE`gh=Mo`C(F`LF-S)@i@L&+MecS@5Y>fQ^m|1QCn(u_QlB@U7Y*<9Ng zrwmJ>T33B{C=ZIYf46?9TL~g_AILr!pWR2)4CY=gV0E50ITfQLwsu)4^{GY1)k5DPu{BNauA_aezHBG zA;L?SQL@-z1#vY0h_x>YXIDM6C6n?3aM$@)RUzF2vGtzxb-~b{I(kp7ubD;V50Z?F z)cXaD65?BZ#Ec5KvR{munAyFq{&6r-%~9#3lxUY8ta5hhx9_`f=qchs<98?emT~9i zp=S$&{3=sL0DAL?2!5=o>V4tCao#y?H~I`!p=JE-g|%m~pfxVh@TwsiZb~!Bp&D5j z1sERxOrJ8qjd0c9IV*f2G9yX{(vt(lGefslnkJ1d%S7CJm?hE+Il7{>By!JuEF(hR zysgjKIPBPFMviJTM{^==z7(|u#m}J;k>Sr0idgOBPg{BPzk*NvRJ{(aVaZtp3U?j( zn3#C>Oq~7SUnMBA!z?1a{qt%flGoqjkfg6k(F;;}`_k2O@oBt+u|S(cduvJ^Hy=>M zNGOmacU!j6Nfre&?;U@4ywkcqT~7_*$>6NLy@i}aesEUHmx9`s^gUF?`QA!GulZZ2 zcB%&tIJeut-N+ag=3he<#vqNt`5A0cmG8b$CcqT$Dlz@nI z3dk0aj=gCF>24*ZyIbknba!`mt03Kyo9^z;_vRdr=Xu9FzGsZ{{dOFW`@XKV)?9PW zYpuE)$rbz+bz1Cgdvp%*zOS{l-T8hW(r5(8dTC$tl9KaK{Gc~$IZqL>9_I}qTs#M# zbsI$}qEkw&I4Oxr{_Jw1N`^>JY7I7eb*K+^QC{4@k0OW5xz9z!UW(07IE8_&`3SGxN_`XxlVaQ|8)yrnKsJ(HMP`Uz|Bsd zj?oA^+F z-%Dm257#Us&kugH*h)Hm-0sD795N)o;GI6ujBxrGkx2ghdVlVj5ZBKmjkyt!G`rWG z8y+4W;8pq*qM}0kQ##Jykc%f|(c&2sOpti9n}s(VuUq~$jm%|z$8@?qUI5Y=uTDW53kk}SG**@z!}yinDMHX%bm8lnYmP{ zatzqXPHj3h@sWsSTr6kHDMamd0TaQ0@+!NtUv}^h4gJ1_+soje^1-(gV)Bex6$>3` z{I9Uy_V_lq2k}xEQIbD{rPYp&ZjOS(ct$Vcg{`eiJ}rKy5)%`P!k`F;G!`&8m(^ELPd)f2UgFnrI zNQbqeNiC&+FWo+?6t}&q><9 zAZcg3k3=4--eyr+4K<*h=s(*+hi@hwI=1nC_GFU0H|qHpig_BoH#_)<=6{t{#65Bb zS}J$2|2)P`@K6GVxOB`Szdj|r8>poq9Uv-8EI2mUXd7??o4z}2e7v&nHU_jp)gu89 zf=C)@_0oPA{uLI*&$e*4qeA6UozWbUhkmHNV=OF@Y@HRbK6TDDdx)iMWHPy4K*lUG z`>31C9YL6g&h90XTrU_~UUuW7afmsQH?u082cWlNG&ec%{( zMAPmWK_g0;3M`>qbr2o_VPm>W~kg~uziCyV2xD)BS^ zCYQ@rpvMRO1)J?#JdmbuRMd0R={o*1u7Q(m0D=oykpGm7KOysN;(P@M2Y2!5SOlG% zC=w(Ee~a#Ry?C%lL2Z2*BZ(5VBlq}7VD7X&#TpFk7AeykUT-L^({144PotUDoxgvS zNI7_K`;jb7;k9Yhv$w7n->x0A>a&sI$V*bgOB$$e=A9g>cCeD0#@Xw(84WUC-JhP% zZ=Cm-SSK_MM&di`PgY-hI9l{^46UJR!bDfBiXnQgjf)N|?Wrz?1OnQ3(4 zyvya3;?Znezw$IAsYv4Hftk_}d?@iHr{7PfeQB14d=#;VHCbjDazoB-*5y_t9Xvel z$*qKMqoUAJFfc&m6ZAbnbaNl-)y%VHw0*6AXk%`iE*eM#(&Qj3g~oL68f$GvbN$+S ztsga*#V&dzP%4s^ThZ1dn=rSTY$QF^;)sVbV~psi^JXdo2sxA7m#E6`$HxI8-RvOC zfS1x*OAERye(k>?btaRCOXUP=&23H^KmKZ44snj6v=xES+-4m;?qZUEhQ= zZEZB04F^kA8f;moYs{$bSQo8b-iAOlr=o+FwVDq?t)*(x~8+g91F66x$ytpVua_jcUG%uvF5%GPnv;$AXbkqkgI{ok-;`y)1@sf z3#YN>2Pjf^MqFi9A7gDjFAjvYSeY}>WY3ybQv`t(YBOU6(LS&4(`I0qx6jo1yU<-P zF3HxuK77{aWTfsMc%S%R8@hvC%RZrD52)VpDm(w3e?qQ=Zj=MKF{TijWkYO#F^c4@ zHxC@AtaelI@*X>d)_i(*2@8M>mQg`Z8ZojzU2Era*eue?QCS?>nd~DjCN? zK-h8#4DFow;e~)~Wf<~OrRj5e^wwT;N@oBLjn#zass29T`m_Wad|<_jjQ-y7J}vd# zDn*fIEkE00KiSve{7fTVPpM5mU*81DYo|`B;1@x76nwX|wR zB(mv8aHhUKcd}(-~t?s!b_U34o5^3F^8ot4;$A>Inh(-e{FKrPj+?t|opHFWcWh zn4Rj7Ie25=wNzNrtF2hj6Dy?~o>xf6C5G9WjbgsSut0Q0laDswo4Bezr*Us}kR4CNp{XF{1UGh|1=(OKWvRE7l~lDKTfKFRX;>Exa)X4bg68x%1IZ zpQYI=L!bnEQJv2q2dh0*756ue^~?89O`y}9&M9l&n?DhnZ4$?ngPdT|%inu(sBhr# z)tCDwSmdT@_n%T3&GLO75;Fob2ptJo?9KvDm3LKY_qeC7d7RQgB?~2|rzy%>z7`@0 zIaApg=vY(ZLM_T@NF98k%yu8x+30$CPI-*xxD-%8(=a8IbrEwE=X1vo#a-D=e-Wt) zwxfY^svonb^F}NhnAIn2KDjeu=brK3W!p+#mN{ckuR8M?X)W@E5Pmr0fF%j`kb@YO2t#U5WHtnU=~G$zQPPoBxeUrG7&Y^qWhC3WTJm z!t9f>WUo(}zI9!$`-{BBd_m|(SK3!at36Xs!k9c{pGF{)gDLIC^eh}#mtiXa`M1vo z6XgXKT)8>>QXYq4WbBHN+#QY6%$qE46e=7qSU9AV4O>h8W*!y3j1-J%_X zZ=VRY1DZEekb<2 z(^0}pajZZ%IzC=)uwD50qk+{w*#GUevP9jN>IA;(CWXziZ$Y5*P9O9fs0F}fMDbAl z3gdnEO8P?-hN}!OS5n!*Xnn!layL3Se50Elj6}XXkta)Jcz20RKoD=Ob?a=$WO&9f z2E)^XI~06ypuB*~G|l5StxKejq9hMZY&8FDw>R0vZi{RYZq?cOp75CcNAmuuS1#%a zQ}eWnW5PO2*S;)nC%vr~^IHnd1j|FCAr22;f*D(VB}%#VECkk@Qt5P5!Yjat0GQAH77#}kSIK_O%L6jX^z{&kFMt5sOm zT_uwaUA`omvA1Z20j+ zRAg*DE`smXYM}%yveScg{@szCDvs4zht&JWD?TkA_k3l-eOBi)mJ?T8PV*!j+%|Ns zS%hU#>2Kd*%0jihER6I~WeQB)qRe>Iu4tD`G&dHL2((QhLSaVIl2Z!S1^9C>KmHKR ztN4O2I5CjiHU*lOr>eh>X7u}8mF-^iMKJEqnCf)98)FS-^9{YZC8jFI|r``bmq zK16S?^2D<{%9kDOyG`nV4wM}4AOBaI2b!0@i_)jjdG$WGHD1-4ywNB9#;i=S1WaV( zyVIEwF)^X}@?d`ZGu)G1n1pqdlCq*>yFb&&?y^(F&b5-%oC1AaSiSqL!^NWCEasJ6L#Rml(~>-aK&aiTvv-vt~LvS6rH9|w;f@SH+{22w~_{bNUFqR zilA|=4*N@z<;hoCks~IuzBL@C)c||+h(rGtWkoFQB6SJq;kXI1BIlTZ!z(|38Wc9q z6qT|&@sBLqGo>ET;8|wRw87_GlpG^m{;&Vz>x2KpkU^u|o8!2hC7O3gg3=iZYSUy= z`ZXr9&?=BNA=0KoHHF%`w6ioEcIQw(@pcnz+L5A*h3$~)a%w%@C4aqa221Q-9_34T zLasvpBSqW&+d{f9C`JO$jS(j#+WJHz3DMaY+BK-#8Pd7BlOgy-=p=gYMObbKHJGDgDn2mO=Ya;e>EfR&R5!jp{JJd{^BE%&i=>j5@4_J@h46UP!lEy9nx zSV26!;PExoOEjzkmX#2%jT=)MaWy0^(zh0O@C@4ZJs(_i?-YyWka!y$1Yv;1WB2F& zTleomlN3XdN9=yksJC-(YHjwL&lhx2m3hCZ7M-%`-mgA-gnsm1h+kpDfO$nTrD+SK zS@SAtP0-U8l2N$wx>Xsv4k5m-hQEE2^%d6ySct1MPE{G#+6s0)-3jGuc2RXIVMh<} z2EOgW^S^8uJ&x>3bnk^;vD?EA3TDc~EiRrK}c?;UhY; zDhO%47x5GuW1#KcW|$w2+oW_W!L+_?5MKf~n%Ip($}sx(mo-=K+b!{`wv0BFE2Cre zot@-b)rhEbY1RTJgFX9eyKA#^pnBabd{`a%tv5Esv;nRj(1utpfP*z-o;{m3I2F_4 z9RJ`~tWw-sv$8vtSyWgBJ^!^A+$?^3(g)?#E@_Vd=^AZeObPLG@-=C>0J*^VUYm(9 z7?W_dIi0c?)_QlPZVIbG05uH}0ei0BVo)#brGCAr&6e-4s=9b!#+@|)ZE z1Eg-9beQoi|Cc{Z@aV$GNtb&5ky8eUBPNvoKB4ItS=8klH=~?m$~r%`(eFBUAx#b; zep5>{=M{-B^@V3>BA__PzzQS7y~EkJQ?Lj&=12RtW=#RuX=c^a4EhtGx}DgETZN!B zsbsY+G@`C(RQ{3LPt|X2)_7il-QmAJK8<`8si=%|D3;PVhmMxDzi_?0PI|r%UUa#{ zMiWPBD-@Ab*dBTMDSX0mHPyn3_c+aV@|f#Jpn#M%2k8qv?_xOLlsp%&J8u(wvkbMT zqpD9=%idRsSTr8{u$Y~6!b~|2jV}_&4& zyqaJ%)G2M%+Y?sGw^y{9YJ?clXN??XIjx4a+ijaHhASlsB^szqxO5yqJYNdVZReW< zZ!K3~Pj~`ChTdRal#+jpdU~oJ;_;q*X~wuHu+j~1E#c?(Q>7PY2`7fz{Io58>*P1c zX+sQP;n#^Jl#R!?R#A|_Y1F^X1^opoeEKpjoVkPQlQ;@g;*M$TnN-;DaMvi9^|46J zlx`_yAMB#rwGI7S#7EvaeKh=*qAC7`t$L8Bo#ee#*e=EQ%V{uM7ahT699H6cU-;hK zcct?0KRe`jWJMwogME`JxW=|vW@~hO%e6+>KZ$rPiWnJyvUicTaFAMBQfizSq9zNg z^V@&m0HhJ$0o%s6uZvQrQMZd`>+2-*k0oFuKlFlM{f@oCROxRQhw`!jP5s~ia?<|F zGZ=|{MJ-7j*3I3)xu@f%ne;iR=6~3vF;Sj_A9U~#&hRzEzPO;1lP5f<3hGtHV>4?0 zy<`I1STG6-iWR%2qr^SdO+0GrS2;9E+_7Jf(D5>rOBs(`;I!Az=g*geKrIn1*|9%e z_Mb*R8^1d4AeIa|hlhTrk9uG;|A_8oL7uYXyY^4tJciTO)_xG(pGWu^V2M-1e}P!^ zS@rgQ;2)_ttc!7-p+eX!fK8fbF5qj;QSoL}{9BIvJyqfoMY1Wetb#e9Jo=H7UFT0K zc)XHPfNo~Lmb|R6dWXR)#g=LCC`xHdc3Ke8nQss;KlLQw;o3eYgXn4DRq@mxduvk{ zQequ5_b@AozFC9iT8o(0=6Z!Myus7mZ7&5wfz&?`#}k;CZF>U#9ACd2Pj1X z`F~F4;hbktUg*#MW%L;_ljP&$J0Qn6kqg7`A~u%DJ0%59d7s}>Q$aH>)vqd5d10xP zFg!JRGmLFM%rdIJl!!dh&MVg9P&0ccEL`BhX8m`bVI(KTTcds?@e-@E;}eK!tV5*5 z(3eh=yziKJeT=8Au(~gDI>1p{$U)tUrsw#`l@Dus?4)3I%2*=m;Y~oz`xbAEa8IYf zIUWO`R!Se#h{mm*9vEpMQ^X&yx+c(&MecY%6-I^=lL~@Z6h5WPgiz{_J8%kK$?mqZIZ*4;E14*VWc7D#Df|@JjZ3hOIF)_hWngX0wCzW{#A*R8=8KSeBfwkwbw}@3 zwlZWjoNt`H$0p0em@6rN5V!**pzmMcj|(i&36->`Xx#aZZImVOp%kN=q3%qY9k>qx zoK*}n8uZwmwkS85&>AZCe6FLDaX2E;EK{N(Wc4()o~XQqO^^)8J%3s+6hw0g#P)_+ zYZVM4uE2{6pO|<$X$)RlFfbzJv+NPf;=0Uy)}gDXL2n@UYZ;X1sO+RPz#Mvr4>IU> z#>SEUGQyxx^x`X{CYKm{rLvA{2VT(m7oD$a?yq;_4}Zj~!|#_pLNoN~N!xTwTq20tNp3 zlcQ`A+rQ7e`$84sFF&%LtaS|>rdHOg$Ys0r+%*m+2WM|Bjzep*s}yS+3Nw3T74!ql6K-tQwA&5q1Y z6zzbJNt?fIzxh~=lIk`HP_gx)k+sGjIMNjq5cRQikxc>73Nv$L3O zbc_?mZ;spa=WF$oBE!w7$n6m9a}559?)LIhJ4X>baK2#=E{)j`tz5i z2~hyCl(WdfgC%;T@mf;lO@N;-=8haCCv41o`ewGs ztXid5Q=QVNCAhkCdja-w#gq>!JsMZ|sasb}$h?}(i8qQ`^ZBn6Sw4$Q-bS$$`1z98 z$O-M%N+o#mu;&||qJA#Zg#jA_@;ZXI)S%0gd+!P9OQDk7HWCAwvw)CvErg4!(6Q)o zkFxT+Q4`O^6(EfQ4t@JM^qt^Z6>S)t*z^CZ^Z@;Gxw-P!4@I!uBADjvpr6UHR1HLc&IkjmfvWYtX z(6+hOBU*J|&QPv6ulm3LW#1t(wBd~o8 zI{HehCEI=SRN&k;K5o%=p|B*j8Riz;iX& zWj_8S_wW>Z7Hf@9q|h8OJUSM|^a_DPVeA%&%~nRTnaQ?Cyu_!UcIeV~`Ad)}*z z>(=ia~C9F2DR7?OZ&0t1tJyG!(Q9C?Z>5mHcR+p z#v$9od2=}5u`LL@-;4|6<`rgo;}muZKtL?>$8lRK1z?7v*j>Ff=P}WTXZ7CLG98^q zq3Yp`LGJzW_09fnif|PtK2RxOh#%itjna1uJO<=F+=0)a^|mh;mfy|C*lS(e)3EM7 zdaP(8g$bB&=V%PN_nz@Y;m~tC{JN~i9kL-|RPreH+*V9xN-m{KGo;^YT50HU+pcmq)~0a}chaaTwsSIv%+Iiyt`6H@WPe!Z8dHOcgbD8&>(I$TCe_SmJJ%i5Xw<>%lIw) z00XIIF?EhFkoVMSflw;sT5`?m(&vmWSl0#cdas=gQ+E)jN3>6Akq#>ux94{@3d}0& zRBN6(oG_PK+V`IW7pk{KX`6Jvo;TsL3xlGrm2M<<$)aT4YRsfWK1c(7fT#5x`@!Lt z(YYnQDRhMRU6yUVFb(in$>ogsowcj^{?E{2 zmCGeJ&GnZ1FJQ91Rby@Pmy2ma@X4QPhL7N8QAy1w+_{!g?aJw%&65N$!bDqW~eT_A5t1Kfg- znydwZ(__wj&}`2+hG6B}MVIUGBK#kTuz0uc!2A^m1n~`LnWHjZ<%RNip=c#(+^t>yvba(@P>!dWEMDK^iMWN+2PF2l9a;uh7chZlnQXGs*yTw0k3(z||&7T0_ z_G+}GwaLP6g-@=1|6~HL=q#O!8U|G|9oV@e7{u6{dk zprIG;5>}d-99gCKOH?z35VO`r-Kl+I?joK2pU3HsP&T9Dw$72X(l=K*Hz%>M3goRg><{kva?EP=0yNtqVtNP)Ye7T{mJ_2nyP5(jD+QG;Le zZU@LH&ZqC@lh<=a8#krG&~HyxDw0QKaUCR>TDC|GjSS1#@QI-?38s3^$cE@QxrfDj zh0BS;GfKd)1*vNhGx~Lt)vs#!10>a&9I&l9w!!z+SbH<((h0Ybs>ibi)9Xx%_gNTH z?geg1brOjUuYQM^C~3>tPmJb25dwiY!284Ow8?mHe%N>sZ$AS48PF!q2&cJZ->U4% zGGv1V?@WQp+l|&r^y4BN3%D!HTL10>OT&1$E7ibVN#)IV{(^r=7c!YrsN-lfB@Ndc zotAq0#Z#-~St#j8>9!!!2?lb5@>+ZsdmVk3eFTRYl<8tuy9yJB`NB8!!wyw!#imo2 zL?wJP7r+lSS*2X5*&-RhZ+ZBg7922hv%Isu^(~&uACE8Lz8WWPSK_ZBHYNxk*k8-v zYKA9|G3g~cD_nC}DPM0|mi#V7aR0FsT{pv%5p1BNp@}J7?7sL34Ss}p5`OqU*4DV8 zkiNwz@;MWDwXCbYY@&W0l6mdsGWcWeThL9JBhPHG2xJhUX#hN&nPrJ;!BsU=rsKCxhfR zo|sjuwX?RHI;|85ZHoB9>G9rIQ=v{NU%8*yzIma18@4_|Qad%P-)bp(CTq~VZZAj5 z*eMeaJ6yT_W|Cfl07Ij>iMbAEX=qdh(Z}#S#<%m}LLEimMY~rp%R5$Bx4;`pTsJiD zC3hXOuv4G`D|+-T=h?%ri`<%^~^j#_xfj{|=Z(fm4NZF47ILivozz*#x#f3qfNiAN2n*tHS5_g5|Hlc zY!p=n&gGM~HEzaaPHV^7{GEwlSpKEWhk_RW4x}AQezasit1O0bf1r`p$<+q@ir>@NBP1ZP zb+h}p3*N^`j(puRBgZt)Xi>!Zqr{YM-TVs3<~`q%YxW=`z^RzZQG9g>&EsfD1^>f4)Ue znmt)$lk3`1Y%dBdrJ}KG(dhK4tgW<6>Wud3m{@}nW;LszZ76yaIp$|rDylHvM)4i8 z4*IQsZ?R#z2GQN$rcXzo4iEJ12m(jM{j9)jlZIr~xxh}zMsDARWKKRJ@=_PB@lI(w zV|r}&9|-jLze6Bkosh&jS=zY^6;CLz`BT($;}3FQAOzA7+zIB=MGexAXTkWFpAH>c z>dlW{{RPZJ3E`TjX*BE|6(NHHF4d{C9L6M@3Uy3+gs*^P2#>hq{eAJG%-_*y4{f+z ze3Gc7bhKIQrW_bO`652(d82wjQ?l#Z`C4pdQFg=h?lsWO?|{FY|2Pdu;qDl4!29L4^VrJ)N)NHgKZ*xAai! z+&;W#u_kdMr_LH8#>vSCb2}J>o}OAC+c7XvBp|p_y}kzNWksMs3iMLW!sh3eg#vnw zhl=GjrWBPf5>_@0-ks%o9elkJ@iF`bGV0^U;S&PQhP7-X#IV4eJex}FA&G0##7-HQ zdU3cj7#psMOL?!8D(QlE+RFPAKG<=tiR8y+8rn?u`{;l%H*>?X$j5K2GigX6K zax<6b&of&vnE^TVb6<$3)~@b^7H+RP>c+0nA?6TrEIbVK)o|X&4br7UlA!QdAWS25wnWa-YVxvR6L}ND^saidslW5 zhcs>NTIrc$W@Tjp8sa+1Rn_CG`_VS^n5jXIK!`l;oVm%#Jc0sNg*}ugJhxUp=0?TR z-P=mnHeoN|a6Hp*-#@WtwX>6O-LX8~C$oo965G)vs&8;@Nq%X0o;54P@%!SK^Skbb zkU0Nar9Jn6D!ttz;CulTao%w_tv}^9N&~(D=&_0*atuf(rIHB(gTWrMeY=67yGFnK z{dfm$rd?P*_X4!~t*^O4xsLGQS>X4!)WZ7=P8)EhLA>PVm*c^p?J&%tPFzs%pZ{Td zIFNE4Y3%xN+%y*-o)()8-?kJlFIoVNsUx4(ege|T<}6J1Qg_>%wJ>l-@f+{N#Alm| z4U-99mOh*OiDhQ1clU3v5#XBRF-!Q{5DARdYJ{?vX14~KhlIL!P=K*7|7YRJ9VB^K0Tg^Yh>ut*2J$R5$!r% zsFta$^urlC=Fm`S+c&_-zn&OitPLk|6q9XCN}R}?(Z1pLiccR?p%W!m-xIB#>r>YB zNXfjx{1%U~u8xXx@09H7efu|{KG0YwEAcNSdbG&)Z;K)H0cbIBncH`tSv5|aU#^(V zFV?piPh>p%r;kW;#5uqNcO--`?!$C>OT~7nXlUg5`QcFEcvCHXCPsFzQmSmHtm&c? z#Pxe((w$NVa}<_mB#OnaLE;?7)WpzJA@f(%Gw*@40D2wi>!H2q#-$P{J9M+5S6g z64jlbR=VtJ?$#j}y6~6$z9)XrwetFqQNNdcs{I1{0D^YYlLZSMRNQAZXYl;K)&>7P zA9|i1-GB@THttnZMPs$*;uF|=v$ni4ji@wUvTpzN2uU<|bR_eK)Q)t6I_;O9=%is)Dt8kq9Df z{U1O_Ggh?3kY~W(%y7em0DT>l9Hub`J=>_lQ1BRIatu-VwYw4<={A}JL_e%TM-g~prVjaC1hF^f6=BhH%m^guP~ z(4?gvttmJk`lmryc{!{866ysR!#mqNg*h(=4>+G_>dZGP4-i4H>u3Np7))~ z-&xSPT@0I2WV$U38nScMkY&e+XQK~V2s)fJDxPp#E<9>J7!kTUIR>9shIU~55cL2+Dh~wtFHA-ymHC_!DfR#sC zmbKINTeKmoJ<-{F>>3iyE~hBd3ph2$Ngl!`Ptbq4 zbI*nt2XvH7;AHk~YN?m?$xU?mQ76}ZH501@k6{|f@MMb1BHU8YAXuB-DU>Wqn?#Q0 zcI9sIXSj~5IcT@Lg#*}asDdLS+{R2;V<)k#rZ_~eE~G1ZW{3Tz*M5!SJhCI{)7r%&Zx78J9G+MD{$veIFTTH`5G($`PvN36&Fd(n{mkjb+5{AT{-}=K&=ad1$!^LfZ=%1elIivvyj}inU zvNU$)5|Txn($$}d=II?^5ZUYJ`y;2$1qON<_gj@v;Q|+<`;1<$VPIRx<|6WbZi%v8 z_?TV^3E!4g!rfipf}7LGW#F@S){}zmOP3}VJHmh>ihZ{ILg4h&iEyJUoZ?rJ>f>hC zOGP?GI?{}LgFBD>`DBDV^VxS?jtjuP+lFYzfh4mO^lK)Uz1%9p!IY0g6NLmPu{T%T znufuhK0b%Qapl-;TX&o)D%uv#;meo7Yu;cCinI zJfPmd1)mmkZk7cMt(EOs#sbf|9U}Ct?bEto1-RxuaSL-3ho^AOL5g%L~8swv7d1BuI|COvEQX-|I@`(TdppWNVkiYwK zPQUGNHjyyi+2t``I&-#!y!x{9@=wW=X?oC*QEm;|zyw27EPK(G2Rn1Kw#}?$Mu4j4 zT~+v-yVLg;fu`q?+KG^-glw3qD;8UJgl6;Qf_mxu4(b-e%4i#xOG%Z3S&Je{H#c8g z1riN^!!_eqliznWQ3 zHJx4iD~l2rs|&S2oOPIVKKq*LjufGN5|0WFw7g^Ft&!=e*J;Pkn-Z^V_#`B)>B3o@2?v zD@%ZHJQ*OEFRgDt=Y6+wmv$@a3l^VZF$^$0wgLCxofdey4?~ZA7LqN3-Hdr8-H*+H zG)L`rxar4F=rC&ocDo##uw5g_9i|6`V2l^iAU`9{>Im{kpL5&892?DTeor8o=x>9v zp(w|J{7)P7z3DQV)&Ls4Aj%4A2>4aNB#?*(h(#n1wxY9d%)Q6yPKtqp)VsHQP+ z6bPl(w-OT8Kd;w1^%n~*%_tjW)?$|z)=X)geNRd&)M{TcXf%J}tBYyd!x4Tx&=+vs z%h|&=16}{rvgT|)NJ4A5$g1q706XqP0ovjZviFb~VpgtUtpB&O9$Iw+FWEOcpaGhc z7K!5@@Go=oph;aza&dC1){!EfAR!@9(q~u|O}x3?=tKe!XUA4$9}jXX$EnnPW|H*q z$id9@U}LBGJTFc=NK~}7i+Gas>2Bs2gu5I}cucElZKZBC0Oq@LWaK?qkg>XLy-bzw zK*JTo9lCEiXyw<=X=eW_5<)CkZ9*cL+Ru6AQN(-stMc7yf4=R}dzukoQ1+AQH7mou zu5?wB{v5FKy>OpoOaM+wCW{nUQR5-|FQEL>;XQ&sdnj6yK;FA0K0JP=wQ~J_-rZDj z#oRo4u9;|6DxRn)Q~}beKc_Pdl&lRBqxpq24mdAtu{QCY9XtedkJz;m%zo8ZZ-jrC zU#nLKi5wIK1}idEs_hGVzqwU#lkW^apCwl0HYULf1@-FY^a@wgQJyoWaQTipaU9k) zC{L1%Npla^s_)U^upry`y5w@^o%h=-g2(K)oH!mEb9aBe)o)a~?$@BXE+b+${dI>q zRCtofEjmL?#6E?7SWW3g@lS<%gz{V(zM*Mfbf)C|?z@fG+&04gzq}&&FMj%crB=%6 z{Ci+@p=gu;dae1a2)@P)pU?;Ml$ogar82#d_Y?)i-bb!K3g)CBtyas_#x6z+gHEj{ zH>0+|+L*_FH`a6PwiB~n!RN<-Zw`PBmAFH#+-{FUT&v1FwG48UkxuIC?}sFtujRDn z9vjC@MNMnF6R>i&Fr98rXb1PlZ*l7K;-hRJJ-+!Pv^?@EdpL!I&-VhU_a7DIB5u{T z+~?0no{tgx z;Ad6*7n(oNzx-B7C$8kPwly>3A3b}~zhtbMp`Set$}up(jMK;ks+KMqP*fVp zdG6b5QXN9K#&(@kW7Dgtud}uw3@I(aANQU2dpH1)`DEO+4cNiw z{-aUc#b9!HT=mFbvJ#*M{0O9Iv47+Z`;&o5F_{28PN@sCmb-mdYkobl)(vPU3pInd z{xFwIl5OfY&rR4Gnk=BFP2|Ne-`nq%4*cW8FQRac6(q1rYma2L7@8{Uu={Myp7 zyaF9F@tImC9E18fP5zJF^mSMOy96n`MZc}veboGPS-2%-I}`ZN-)fh9N>1C*JdN`? z1&E~_CC(H<$i6?yQ%GBcC*9B~xkAl5V&QD{!wO?HT&hAHb31eM_MU=%$5e?=?1l%i zr*(KBsZX^l@rgsbvUP{vuxqd=lK%!DS!6vJSF*n)zh=goy|nR!Y(|^S z-{6(e#3t>89}HOSvDy-u)OT0*l!aodv#)P}q*AiczF}hfZ#qfpzp?`ErH{5-JHQlA zZ?X6SYAhe0-QVqB3kcZPup1rS56choNE_L04bfB@29i0SYzczjT*a%^8o$S(;StTx z&j)54myQJCXyKkK!ttVSsj3XFc6W*n>&I$Q4qSF3U_8r)@})>TCH$4^Fqa^NORtng z#BDD(vF$trzAwsUOy(XwNEZ~kRzcago^xmKzu$NGb^Fs~vjxtp==iuL1ys?0ObXPv zi>}VkSHTz?bt0W3$SiYMd|f0rvTc#eDd{-sNs#lCR!t>QH*ZfqW7~~CNpj&YatFy1 zvvWPDY(ST%aeyD zi_Z#vdA#} zu28WPGrS)6+;VryUln2zd#ZW$Q?$_P*YzP^EMG5%bl;tdRiqoG`X)tfwD2u2Gc3vs zdpGZI0gD^?|1DhqtCs>0u!`crIyZLI;a+o?!}8YF*6n=dADbm%_T6RnLGu+c<2D{Is;d_6ZL7srpR=KKE0-uK08?gI7dm+*!z%WN*QWko&_V>CZz8qV1FY%!x0j zF1JSn1@|aY_P=64(~>X{;t%^(Plf$ZJOFl^?CrZhkYvdB0q4sB>~??56Dvf}k)Y*g zoASTQ8y&?oL&!$ZY0xD zZ7UqsPKy?=+i&r>ghTHlBuGg?~mm zAz68mags{w7P1Bnc*-{*Wz++-f+Bk{rTH>;f&XYx$7Se z=Wh=vn)x>SG2wXGRc?;48`kHLz2BykCbIUXIn1Z@KlBXo69ByZq%C-w_aKMHPHFBKWBB;UW#JnpZd75Dg^od7Q z9F$>@mnCMv!sc0jT)f~95grjEBfE{|q|K;^2=YWBJ0SGvOg9k!!6#{O54@@{AQpe^ z|GWsVwW1CgP76dj(a3U485}D*aqSnzokTXCx%Kv(BciE-_fo-@@oAM>GG<>c`0}Z; zi$lW$@1u=T%-p-Ur^y{XNmQB!R<+pXewCY@yozEap(Lp2)u}L>V=ja7Zj_3s42zmo zgOL6VtG(Rm+644L7uA%$8j^(JlxuZdyHYW%Mn`zjCpZ#^X|xz(>P;n|v|dT6HfVM@YLTM& z(&Y^ipKIJpHdBbcchZVHiA}c0&2$)OU@(WBG@vf# zEA=Ok|7bBCBzX?(bV(MErlTs%;1ht5uyWb{sZR&-4-o+m@n1d`sUjc-9htRVX!OHh zSwG#D0Vr2Q-hga?f{tb}Erh>afN%ghkv}?|;>EUz#_o#ua}=9K6@PEV`yHWvDZY*? zzMwqSNl*~o);K@JqT{_Le&Mw>qrK?sSFz{Grk2;)_4M9(e1(di_?wLfjpp-P8mHuE zQp}D-5mk%1#beQlR=IjkZ7L_S=GQI@w@U)I>zf&pzG)3~^z-O}i9#EA*h$ULYX~v_ ze;9vYoR?16hH30*R^rZ4?GOC@QZWI~ALaw)K>=Ej!S?0=#z&-UHrf+tX;ukpaHH2`W%76(Jz#lrL6eb-RrI7UZg3Cd#_;x2ta_tYz3B&Cin$k%AGDCWbPJ z>XAvLf#zNB!65YAT&REeYAZXHOWH^@ut0J6T~JqjUi_*4wF+@ENB&*HtscqjMQ-RK zrL)7dvYbEh$4Gv;cJmG)BIPUdvC!>bR+rO87}hSYD`J-qW{QOFO`+(O@2=BB3(+#YS!m+gNIO1dW572_cJV*u980<{3I6J2Z&#q=S0j-R%Zv^GW9QoTp=XFq zIF|G`cH9xbZMp_-Q+8hbZoF`>#=5fX`fIm?Q80~EL94Z9(p8sJcP5?ldV{5aRjyn1 zv^Dm!=u$4%pygdRXFj~MDh+fe^F{)}n9b#^li|XFn}id;{;@T~EEKpm&nN1d^TtUI z?PSA$a{LQX7cc;bdS@wJC3~n5_L=^%#~-UpAmLyALOnGN4LW|9*W7;e0S6!A2{Bv- z_dorHr^N$mA2I>8Gu^30AFdbdoCoyZzLuX$Cvm@cos|lho;I{U-ng6$TlqS=vDIKTx4gK06&ijhA%jn2w9ly#(9moX0L^`T7bfhcb4P|U1_2F6?5A(pgetOy>eRB z<6L@0-vPX+?&ynMeS^=aqW`H@9fw#srmreE?8AW)yp!F$>TGPy#pl&rXYG{2mJS^d z?6T{wT^nN2?(v-ZE;5oa~1L{X&NSKcu|{P}JYw zHY^BAh?IiTAt0@UbV#bCbfbWDH!P_lAOd30CDKTDETB?KcelXOwXndl%X8NH{pS5Y z@BP02xo38qaTa#i^F8Mi*XO#f;~PnFzUc34Nlp$>InMp0au^JMj6`hDs)5-X#nr_& zD8us|K$!a2%G-3RP~5FuXsqY@>v8V)XX#^d7#iu<+3qPM^2m@YCMmkOX z?Gcq!SwJ@Tu(a%^b^Y-OG%XJR#NEXa`q^8Inw8SZIOuYPT z`saiOFDL*KhcRW5jI^n`x-=gPlqc#lu@??Qx>q}X;osGSGXqqsgq7uP7{6J|+X~R1 zmXY_V#or1KKzXneU2_(r^)mYUx?9w;%Nvs%UR z#>`ZS;*UCosyt<;w1Ak%H+1{@Z%%badLcMqch6&T+D#2?jnhv_xuXw)j4~g6xx^aM zXG_kWPM<>F(dXEhlN2=b(us)xHtch2-x^U`m)#Jn;DIJwF#`l z7kt?M{yvy|gsNjkFk>ZmL+&i;ns(=`4!Y$k#YMT#hpbT4(H9ed{XjW>G`7^Wi(&ml z%1YDA1(TE9!^5y*8lt2h#45B#*sKNlR7it2`J4`hpG!*vTa{oh|7&=!jc zYg?jQDD9@nbXe8RR6Nw=)}d!AnKf##IEi~TruqpNPGmchBY{Np#ZDWJi@#UpR(ap4 zo#6`{w7!q%s((!+l7kjWx&K}i#JT@|QE0lP16yu#8Qi~re|M7Y6+JE>c6~yfx{hv$EGc62EuG z(-*cq3AC6OG-(V7*?BmU=YPX=JZkd!egZO}1jhV$?PcsHw+NSElg3o9vYm$Y)|8M{ zf9OI$PAqMh!;x{9WT}*ipVskS#G}Ql&dx8NT*C5ak(D;BwP(rK`9O6bjPT7jXne5K z=*R{#;JfEF5~+5?AC$#czx-?Ze53{r;F;ZPJi57B22ve-$-mM4iWO$oAHctS32yX1 zCcroxMpa=f|39JFW+46qv8`z>#ba{H2|Zc6BD#{ypKDmn6SAdf@HpTuy-i@wVb+5z zQNhnBVbZa~g^OQYZKo5p^7hAFsBkU8axUDRYjrXqwM3>s?8Lhi9o&YOrB*|Hcq7p5 z^n>y4`@Obi`MqnX$w~N!Ho<7k0yAGn!7WW)Nm80;qesz^1@{UUlZ<9-6q<{bl1y(D z)zZc}ODaL8wgxOI2dA>q_SkEUb84FcDK@^MeC(Sd#@P-PdpaYd8!nXX7=(Vh|Hfq{ zH$XBzxbY2R`ie5Vkh(JSPJ%olE+8|_5mQTmm7Ly81QK4F(3~t<^2qD(zvL+Z>Giz> zm6i&n0)I8&>5V^aru`e@$rLd>J*S;y?=Zl?a=XB!KVNNrO^Oe5;@Suc|CSzbMof^W z070G{V31b5CR75}*RpbQ`sY4}@DrbH2TJuBI;rGz@-~51t?n5jiLY}KAo=;lI^D)P zBV|uI#}efHlkq<#iH9B*>Z<|3FcY-3QTx`nh(QUVgdOcK9Lu+Ep{}zYRVqMDZn!~B zX^i4ma~(ff5)fsZEBAG}mlOOhA@+k#D}2g!irt^}LVJhFt%%qq?wrrG7cQ)SQjJFp zOf4q}B`{e~O+4bRpj@~{KS^ae zIpIV@iBNn+18#VA%!pwQRj;(wmUXy}Le5G&=g`i#w#W#)1y6N37|${7?;@6@7+(<~ z^#8CR5b)vh7RL_LEG=7#fkQ%dd{vJj4i;230^3x&nFWjDvRnT{>1|hn2?0V<6;6{p zGtVqh%baDG?hN;8J1*QskR_f^6EE-T3HP>chF;b&J8-`BBlOBt8&_YP+ad%ANEA>X zWHKdpy=}wgq)kODD>}q^`TO>FM+kj6mW7S|S)|>^s7e{Zq^PW<3qn zz~z?u9Z+u*K+_RlZEHXLEpNhuEk2{vxyw+5lX>nswl5++;kuRkiv4E3-ZU3i97TkN zw9|WB$q~CxJ%hI5sns-t9Q)8oWyG5_{bZi?C1F15#ySdHYN3^lVoEU`Nxqyo$-wtj zo8xo~PesoA`&QNsZN4gynSi-}=ya34A2zvIyA@Wvc%QQB@A;8#4g@T9l<70MopZgo z;Uy1tNdHo4gvl}Dn55fm$tx}U_g}n8m2#23uMU=h4ff^pKWi%Uh;9rA9KgjJ>1nqY zWlt6&it}`f*H_>_bLte;qObs+00V!;wLi9uX_`YDeRj?!^o80Wk|v3J3YZxVAk06soC}Ja?U_FBWj9kpWAIexxDN$WLMQWzF%um zxduB54yTR&dK_w_cW8YA>6}RS5j>A_X{S*fO&fl%@kaA0!b1!=e}1^pvbk#nX&K$k zB>dMbVonOv=oQ~(@(x;gDyPSC^cUS#PyRW*>yG~=T=V^kCJQ#%T|+|(26RF0@;JAcK1Qt$>wcJ zRs+!WEx7|!u0o{k=PG#@Oly2sF8ZeSKpXiP89to6m3IUsRU*QmMN_ zvZ{D4`>_nq*U0MwJF|phSQ4`{rCt}Ix6il@YJ6j?x%vCjEG<&D$I586Z)(n)a{AI? zYOyF6$U_W%kdYZ#`nT_pPKN=N5U=lYxG+;HA(_kY9RJ@!-Q`!It~t^LgdZnQpJKmx z>H9w^KW;|JBm>uierhLWqWbZZgLV1tB&!RPJLeyG9)`ZyK|@5nchYmMg8jE=rt@?P zsXlxD1`tH676F=Tm|6m^icROmqQ!;ht6AP{O2N=jvn8^6<=nJ81PMp=1=66-6x`kK z6DGuBu52lCNE6?7tixk_6TKw7RuVHISZ$hEpP_yMIpuKHm)IQ`foN_YvA^e$Kco{n zbO@N$FBYywqK=^ii%3s z>FH@u(2SGStpGSCC#BN*{;jBvlrUlisprBUkgw@AMhcOk7)0qkhxR3?{&QXL7}Mt^ z=E6fDjK21ld?Egr07qKd6x(vXmP5~qv63tKTS~;0_njx+G}>uefyr%VE~0T8AtIl% zV-Dl|hcE3|-mQ&ma#eqa?Yn$!&ISJUoZAHy!_B4V{zkEEQa51Mep)3=9a>8n?)T$0 zf0L>C{(Ca@y$27Nc6&SE*E;04s}-+FabT*K5Tca7TSb5YQ@!AkELfLZ=Qe7N@FTp5 zua1l5kIhA91VE=Sd@SuxJr}e;`+&*-|GF%c(v`W5GGDDfialg41|H|{Et;<`9*ceA z>=rRBynNeRlCQjx8cU*DH-eV3QY9n#_joGmza38j(kdAaypBuWJpk+?S|K?0ADRNd z#sC6JWzE|%zwHoN?Bo!!g>axH4m=MOTl=4lf7Ne+__=0whjLewku2%D?b@jD7gu_F z9ooFbKGYcd3!t6p5ARb93l*){Z)5r&-O+c)F1u>dy~jFVS^~e+XCOhg0#a9uD26-u ze=UkJadqVbmK}gcyN|D%euECQJ>*W^bAK;ul@0>m=-fq+#pC*jA}Ehp+~;rqOm6oW zux*#!1o!JWELK*%ua?)#XPhjy$8Y2$<{1-ACS?g-)l`??=J*I|zPc1Mqsdj4^yqah z<~u+gCCk*l`5PP6EVoq2)5_BzExyYo%Fq~vD?j$nEE~%oL<%nxYN>_2*+5g(P-6Nj z_abn$<510RX)IzvJACD-cTC9dRmH5W^Utz4rFz${)z=3ddI`R3Zge9S5cbKsuHpPS zeGHc|SX(-48426vNg3+5fTQ27(Melw>4 zOytv6L~3?paF5D%+ObP}nt2_NESNzCIU#Uf4M!eK4_5dY3t!AkW8B{ymWiZFMfr zV!Dof1VQ@u<}ZeoMK2Vf4$HCzJQ0O!yX)_fTOUEmvJ~dn*L7b^8K=+dog7gb@J1hj`ObO8Y?e`hc%d^eVP*7;N#2@RkB& zdbF6?rE`C#R3vN|n$sf>$OrnIB~{^a@0c`ar3;~$Dk{B&^!3xNtlCbS2WjD-j|(zf zi{P?dTdd@MeUFLREcb?jcJVyzIbvoUjvUwaSp@1vg?t@y=I zCmZeUm$N;OPGLYpPA?$_|2sL@iNi#hGdT*N4|%pACJJccalD^?B}H$xbb=-A7;(au zC;Ex?Upk~`hi-G$+ix%Bo_r+ef{wYbQLA2Z^6D*`vEiye_92BPwObe_-%pE3T$-8c zS)1TWkE6e&SE*7V3Q`rBh@%(hQ25G>v>&ZnUCF-))o9=ou+?zEabeFi(D=bywkB{1 zAYEo#BsFkedSWx2KMd%hCTkOw{3(#Fgr4)L$%S62(LPNzsI-e* zl>-)tTf#E-%Lt<)ALnE<8h(jQ9Kf*9AQ_C%oXEM+Bt{M{(>@|dLV&P3?)(zf{M zHiklVm)87O&X<#szUp3peR9vfrL2%1d&}9(yB^YilD8L2k2e-cN?0VG%H`Ms@9$BH z`mhgXzc1vXCa3W{7N8eNOxu%gZT#y`Y69^`Y~SnU&&@lPt(B*>Ho9#I>W89R-xq-C zeA-QP{?N|+vpjIQT+g%X+Qo1*hc@5z|z2RL9)PuLW;2p3&yEg?-M zG{Ym${%89z5MZA8!y39lo=^_ag5~n`YMBzV^4|1EJ#xNHeGe^B<;3O5{0Ci!x5%U#0hzHPPP| z6Q{f*PlE%$9OV4V z+@aYEn=9S~aDtCwhVz-9WIs9DdP{$~gki>UE9Pb2*JCFZb=}sZ?g9P|mFy#D^IQbe z!Sn|GBnJ;t_SgPNc`mv(69J#15mKC>-~8Ljfd~BwKQoKg++TLkpk=d%aPdkUF}>sq z&&>&W^R`QDZe+?mGE&xkw?vT}gR2vE#qM-n)}>zCCW#VAH_+){e_}zyd$KXxlr{sD z*e7UYhyv@0K;0R-KCD&NgfNw`flMy&PxuIyV+pE}xUI|c?H&_5`nnA)dI$f385vN_ zoq;i*sdRmysmf%a(F5eD1HroUxNl|dT3&aq*}lTNcW-E_xPR}UAn6S9zNL`5SsHTv zutlMI?yQQA7?xv+c%9RKzu95{ZdvNPGw zcblIEaln@+0b)lQ0aY@)FMtJID>mWjUp~4bF-}$=6!&qCZ}58I20-1DWp6i!AYHqJ25pO-RMVGh1mVL_{b4^m{BfCtq6< z^85M)qJdS=B$LO*NK)RT_WQvF-(U9Ul>8#G-B3Pc=pxbXcGqb4i}0U=cMeY&<7eXK zF3>}Sp}1T{kph58SuAQfq97=sv*#InxHk!gX!07!roCuAczC|Tzn3xJ;KP;Se1jLZ z+U&C3)LPm5YWebl9C1<1^Dt%liN>UnCIl|6UK5Mx`Ljcgw>r-TiW*-(&@P01yDa)U zc>bX;l;+Xd(>S? zN?d%X0`zwmhIBT-jAV>iJam?!tsPzPm-C}_`ceV_*gsVRnhd0_u32^-dVua?XoVna zT}R-=)BEk*_CHzG9Rhb!rZdd?T$nMOt8EsO)%#a9!;2{{p#x*6~^l8TNG#tk&zRgr4{Y)VL0cEx53m(a=J6SiV zuS&*AaE@x29FA(0oIO$&zt3ZsxMq=}cr|e-I$Yw~A6VP?r-gwWNOA7tO1nFtFIv2N z_0F%*^73hrzO+>4U~?|+qWrv*c>5y?F8dWG-0l2(#zJ@Uo!$`Qz)AW#p>s>H8$-Kh zlMbdWqmK7K{FRlQ!@I$XWjupeKW&@_YaxZcYEZgByf#`)Dd?2C>cf1kRv?_;&fvXO znty^$k&!$KiTJ>(LS-lzt^}+D7M@;J_$S0Q3u5y2=;ii2#`f>*?CkGauEQeDsD;~Y z{Nf`?gBKXmtv;NtLK89t2^R`}k|0oIfE2SGLfMcM{-g$V7{~(!b|1p6F5TTHiGyam&2* z7YsXxO-c$*TXe^&us|$*A_0~^&Bj*?Qqp_wSpG9rDEQdW6Zm9e%2T}rjc@My>!R)+ zNxoV+iThva5rIxKqKV}U(U)mBNbaaw>m|G}O1fWZ43A#oqXkC}9kBE){z`r0KHs37ysHK8;1mO2MrBB}bvty1o@rH*|W8H7R* z0~;c44%-Dn6Q(mg2ZVf;^gVZ_e>$vizUh#mnLgCwLV?l0-oSJ2pHVtE1JjmxPlPk2 z_u3BUsUMy?^PBIU!jelf@U040u(6=cB<1wsSE`Fmpf8qt&v=9pX@TfSJ$tvkZ)_R~ zd{{1BA(W|#tE)Dj@9aw6{m1V^h=;2=AX=x$_j}aG{{fy5F z_fW;^Zd8@oIDoE8Gw}`o#tQ}#5llnUArb8m4P37tlU#?HcUE>3vXlh{%GuBvrMx>+)L4DHy=^`|eVrs3^U^?j?zUpdPTU?l{x}n zF4ZGOaXi9j_8i8nuYyd%*Q=do88=RM+tXTB%7%*5R7Z{8 z%D2-^O6$-{aG;`f@~wUvn`p)UqlLM79jBpHt<*BJ6M3lD%Si?Kv?TkB2gCLdP&Naa zQd7H4?w_7&7h8%@&7g2>_3Hib#;3TEuwAd%{(&a1kfD9~@(b@_!Hx;201uQf$cDL%<&bg8m+c zS?Ef4g`;uN(c70$$GUJ;dl&4@vD#;ii3lS9*W@HR?$8qsDrbaoNV0Lok<}UmnUc{6 ze7@fqa8#;O>+h~{lIPmK=h!1tB)O-D>3fxO1_@EmrxxTbnnuvfy^sx8KO4Ah|%iHSWVcdCM z+0O*OgU3)wLjv({WCWqh<*8T0Z91QOqq{5xFERSP=(tw==JdFC6W*6O8&>{p@@kKP zRQ?LS`+$qKY@)LQvVB%x^~|s?RpzRgcz?sq6zVy$;FYB%k<~*FP$UyS1F-FZ{4db1(ATCgt10A_uos;!Z!Ak0ZY7>ex4A(KCi8Ts*GfIyEYBh`~ll`x`Ik#q8vBt zM(CClMfdF9kj(eVs#%wZYg(L2i0h(@VD19c3_|FgkI^8VT;PyS|0%%GdkE}3rp5ut zwo=#3`MZ!0doG=U4_lfV+kWB`9L>N)Iv^{}i{G{qRx?el`=876M;pujf7Qlr3E8L$ z_Zmoop70m^Pj+9r$tzHD8hxr(;W;P?jwwLRNu6^Z&;qijil;V-P7XzE8Cqwnd z)~r@rT?w#lUzgjZ)@LmW+OXA0RaXc8H!ti=&t4xEPhPoC+3V0bQXqc-@z}9fqyqQJ zk*K7_s7Rq}f<{iEu^O71H$U2F+KzC&ErN{&e~;tkO}Y^6q?{@v=DC$|xXJTz5&X~I z{4lu|{w*i1f7WhPTa+3q%LZgx&Fg5qOi?qz0656rjH!Y1tM zOyoUmNAFZR2PObSlD<1twu#Eku+)rYbE@_&A3uMt&%ULMvvHo8ypnnUjwX=1CTi-ViQvA?p?&M$h+%xqr{-HmvF+8Z}n(nwl9c zvpIuoWo-u@#S}raIS-oFqh8-}QcF=-ObZPD{+?6OG~kZV4?4j8ts%ph$>=xqjk<;Y z)cRl-|89;L$gq>c1Km{_3LwC%)9kkIq0Hj6?3U}oI-pf3m_>D6U&-RW! zHC92EbFz7(&5QN*J?xVeH=X`2CpY=FW8KJ4y9ahxDW2@VLPnm2>-^aAOg(MlPR!QL zq}E%V=6_IF?BCzNkPwmV5aw18IIUwiW8^Jm$LwcWFbvg&HbNcx)KH$*YDqi?Do zpFgNa@tjzpe3di>ZMAYBC$sXDmC##10#-QdHB3zCi0YgeFltqSqtBrs=@OYSE}zZj z*ytedRK+SQk9Br@zr8WF8S7AF7ZMWE%SkQq_8Vg+e)#OzfrGbhple0s9xURce)OahHPK4{d#p8GQ)+lPMJLWf)3(-LnY;qM?wtk)9?8u zqId8c8>+pGUN_3em8LuX7)v`(BSG?ef{kC$Yum{?Q<2&Q=uFb+sO+T|B~$rB`Pvb5 z0nW~hvHg2*%})`5vp|k=iJ79=@cpY-+3>mZ&z`57K#RA62i2tBUzkChL)H3LPRJ!! zi)3fIH;HLOL)=JiR$)5PSPEYF!PLV4GkSsEi$3qsjJOUH zKJ55BmVQHz;l%FxA1Tne|6U3N+@T2h`jAjEx^}f2SpF-I&$6bGmo-<)*9K^HX`Dp% zPoFXm&Cffn_RV8=B6Z$pX}n$i8e>KHxN#U2PG6jDvp+KMEwW%>%y3h_PscxH?Xu&P zf*os@yDdc=lE`UYIV(DEdHj&u|Ab&30xlhZ?Rmy&B)Roweb>&vrau?gj~wR@m!FP~ zqEU6=ZD6qAVo#Mse&UZ*Ug)%$tlAvPXvKFQNw8AKWj04O6gQDBcx(+F)hNJ6lHSUm zbVh79B2zC+3Eexr|7t*?3K2y+q1pLXE#v&f%VB7@sq1tW8P~%GI>iX+F;}^-4IC3p z0H(V~4U>|^$hO6ifW;0&rP5CF**nGd9%GEb#k=9!w@2v=-uzF=DLy0W%bTu6*Qd-b zi`!1;UmY&;zl3UZXJ6_%QSaK%D^=@Duu3Ma*a)Ggo`1DEI^8F0QD+q_X@y)NW;)!K zl0KD+-&Th79{GzaYqCNO!jh)WH!K5m^PT7RxNJ_9qsmIMKPyn$Kg(`PbJ&P*_@cXv z=;T#`eT|mP29x#vkd-vGb{fryMzm#PM@LU&lwAw*2T`Z}SwN4ba&v36%{I8!J&l&P z>>%3b^x?Cw? zW~n6=BnzKJ6P9)hW1d`W|Qt%x%@nbd?C7 zw&N16!Ut7}@>U3eXW;(JtDJqXhs?5kAKgqT+RAqlBD5*ID889O{JmmOspZwt{tGUQ zbQH8`)GkP){eAPyAth5BAN2MA2FYohmf@Kq)dBOZYsRSc->% z{k3lUBT@v4_!6o1V2(M@+(Tf*Kp4PfO)q%ja(iU) zqayx%qv)+bqmt$Aro2{sw%p+bqv`V)YYn!vYbtE06__H&DJ5=E?UsV5J>Jff+LP}* z?$M>aKXi>wo1_p#R5JTbC>??MuFZxX+g(U%KJR=FF`f85#1w>{`_v|8jME=E^+4Bj zSpF0=+AHT}>B#6~jeNh!3;4m9LzE@N-G%*6##hYHR%sp-{``TGsQq5we63nCLHoWS zRVkFf+9SU?hViR>K?EfCq%oaH#0AEv)FevYy#pgk?_bTpoU-)QNVj!4N;R?GvBr?^ zPx;Fr-7Sti(o&);xsN0$lFKQ(2nbe|K4Cm~CH*$gd)%Kp^^Pb6mcL?X{Gd$cmi+Iv zc$sKbJeWR=gY(q$(xRdQLktdFQ_@&gQ__%`Rd_9%!d;H-wJ1>Y~U zzsnijc5v{BXf&*QPv^U+>*Cb>UIn%{IKVY2Z&(X(W1GRLAZc*}>i)w5kJVMJ1Nd^k zCnc&8iN06|X_j_Dch#`1zT#K)a!`Aj0|xa9yplqxQ4&>;KQzvw5rOW-j2SC@-hKY= z@;QBD~2zdt#otl_=^u4>VE%f!~dStwKc%eJV9VWcn-@5iz zS3UBOM5BWrqa_$|7PzkHcJeQ#ktd7{4y>sk!o8kVQ*fgbPG$mfYUo7#ZW{YNoSc*J zVM4kd9uiwNP!0zw&Blv|ur}=F-x{yiRa^6SQ?rrWxJ_|tlDxyG;tdpIl2I?uNr}ndD92(E&71siLp&4j&@cCRNot4*j z0($vdu7?s*AZLZFOOEuOCAn6iigb-JGtvy9S>Gg8oAEcPK5O@PKRh45!vBb3{=p!t ziuG7us9Y7=dDEE~hb#mGOFXavIvz=EzQ(G`S)IDsYJ~U(&mHh z#+x9tGn%vx@jh*}y2(@HJwcZBf<$4iZ`MG_qo@=|i&tm2i`AjVtxs}BJ_PFua=Sy; zi=Mf64ieh5r^>o-g$vrih-O-rLoVFnATyS3a+thhJ33EryOO_G_*tsA{kyD^QQIvI z@$RQ{l)!r_AHUErPD$6;t+2+5<k8n}Bev$kC75^3G+*hk93 z(?#O7q0(m^XdZ|C-NbdrbU`~b>+(Ll=Jp!5%S>(O@HrMIh8XDNYlT~0Lnb5AwS9aR3gB%& zgz~=awhwxYn#f+iuF`sur~qxG;-u!UXi>Ei02+6>w`@>bB+~2&wFg5wCq}aRz8@XZ zSVVoZCtzMaAQ|hc_)iR1dJ5yV`7&01wR_uOhT5EG;Be z8FaYS>xHA)XiosWlP)-%ufdwh9VjdTt7L%>es_4M1LabI1rliAOrjcKrJ#hwo~fKE zBgHAQnW`Ql605WrKi+LYb0bW(K9K-<+Ly513Sf&3%3rt3UsY=uDz^>jfQrWo`I(9T)uRpiCv&1Zs1< zca;e#ZuKGVN!Qy&>$WapSWM`PzNJkba(cp&Ido+>IqdmzFfj+lt1GT@`zFrO@2XMS zld?T;a3;&DU5Xb6QEu78&{M_JxvdoUuy_Wm6p=Y<^1(78VUGf6Z6nS)E^xjhM-$%) z{M-^EV&vSP)soQJD}v;r5?gl3%0k(EI@+A9j`p4Pr7pdfq_l6D;I1-SbRl$~T~P3J z`&E*}t%DRB_=X!V-lQk?2^da9K$|1WM7=WZB5?Vz_bSu&e?;>EQBs$q9>wz8Obd(U zDm5u24OEIb2?M-Md#un3KzfXJqea|A7U6@|Kf)eYF`ZP}jgp?V;uJIyFeCUU=atc? zpU)HE-4Mm>tgFv~$z%#Jto1D-Z1}}t0}Lw5`zs?RWrg+B+%5)r*ar(Y({)Gv-FwF7 zzT7x5W$eT3D2THXAW|NQ{###t4jUgR(jZ^B-WNwL)Hds$UZeSEl6Mc|DxCFblhyA0 zZx7ux0QjWxSYcTF$ua2Dx3>K#WBy!sTUR{%9_ii$SPFqv>dGa!t%!F`P+LQNa^r~) zc~*jNKMAb&*6_!#O$EP@*B zLvd3AaBt(~5pE)qIE3AE`Sx@bbaZSz%-_uDl>#t1=)+gVK7cbQQRZ1jQVju=+-!O% z`odG9SL8yFM-%mfL=Sx!sOwXL(0z=H#d68BI`KfGhe~F%Lab~cJVT^@%n^&$?3|^~ zxNZBxrx8->>pD4oZ`(pimR`*=^I)XjqiibK*T?xo7@AC=?o?-8np#5KDpOv_XMt&T zMjnwR?bxVr-liW_oD;!8zq-i_zy`YdCU7~c<=Fu=)1R~329iEw>049~}QfoP3AO(8M8_;BtoH4hA~SuhhkX2($o1l+s;t~&Ml zJGE#{`+4(1<`Y*##&pUDzAO<%vptXf5IAHHrF+Yf7V{UVtnhy96Go{6Y}-aIdReQ` zZ1r_`V6}SzV}_~fq!&Fw-bzINRi%D^U8}y;1jX`TH&oU1#nqT^O@jax*LdJcR@#hS z&;j6q(StvzM1%v|$DgQ#Qk<*p@(H?ZsWZH_2PB594uW3VW2q~&Vih^aEohGV9ao`+ zRcGuImL{}9H5@-|*{Qx#XnN=)M9(xd@|ry88JQ;hL{~n(zI4oWx&^TCg&DUOw~mK2 z4F^vSeBOCz>zd8`ffA@yg5p;6b>x*4zkxt&fHXwZfm=b9wp9Yt`1leBevF!4^;o8t z4UNvMP4jyI20!tjyULt%plB>Vm)d&7^<9@RV(%+HOk`3bjzuXT%g}M1tp_HbJ04$(^8oV@q;{M zLRkCKHE$^t06HFGI*ENXOEJNAf0DjbOnF2$AJUj=c=g}4^*L-4Ja6D`+bjWGOXjf& zhxLD?>a)ojSygVqS_S(@xzg#U=3%DRZss)DzCyrU;aQD@ujdr`f1(oqU(Q;Ig8QJw zYfMj<#(RSDBv@2bWdK7H@j?1Hr>qz!?kwoVy*$@rjiLR47enX@0drnf+`A>-=p<92 z{_j)M6^{Fo`MTvag15`QrA=nUlJs}Jz~wdz<_izVwYYNi?q=O&fZoH$k%0*^&QB(|^%m3yh(UY~5lW6;o@-|*^$@SE zaj)rR4-d2N6+%BCn#oAtX{(MDvFZYQWoE2Fl5}&&+gB;!!^vev9lC=&8&~mPpToS4 z9yC9M7sb(!EX${5)JM2hVS7R8b~&EI8(&U4bgm<-4v?BTg+YzRk#`K5I6CqenbVJ_a`MsDHh@f&1qz=@XXpm$Qx) zxX%LwSXf$_(uL~y?PmkAEh+jU9@2~Z;C`eyi#&20WqHmg@0$J<+Ul~Mf}4dmLw=pH z@yw?EI@j5%@a1;XnDK}ooiA@kYu~NS=pjv)7+&c;Rua3+6fhi(zH(5n9OH>#98Fm| z<=IyGtQ5z)cpS{Tux(s#ao9WZN#-*)?k9ODsag(8I_1?$$4NSTe0)#R`9h+xjWDqQ zgDRVda;x4CI^9&>b|=(#thno+nbNT`H?FT7gdG$|1>h3jv9a$iMj*RwG@VyywKKy^ zr{n}=Ba8y0DQah?M`{g;PxpIC3#8AW2&v|-akq!)s>z+1g^;b6KJRB5hGm7lwrKLF z0>ow#q}mJkm^5!4*V@7Xt<>AHSvo9<<3a4r;OMx3M{z1v`cMPGE?-F9v%zLS=O(p5Ikduku@i7&p zNW2{OGLv-slo>C9%TRLle>{2}Ek{G0Csl@Emj!n!tYe@Zsyg#9PB;5i>Bl zQvW8rcw=ea3;xU^SGNnLnB;eaG`R+ED`j6Ycz66XH&6aGZ?X^}fnCOtp9ueaGDQsf z9cz`82l~PtJJOB72_J=6$^(m6QEwKIbdgGT-w}^`aRRBLIe#i7;5E(YdU2%USL9Y@)I-WkIV`D%H_`)Lb~=v$h_e*hk5r=gU^LD* za^m0>aJ0*I>AI*9tH=7p_1?p6D~Q2Xu~8+}+iYdoC>jw9_v278cbU^XO+Eu?m_e}< z69+U*b@n4fXzFA5ZrX5LoQRv{r?WM}5lfwtGTS*wbL1f{ z?982BKh8YBG)kTx-KQ7f@VQ~U=Hr8_#V;?uO{>4K(v=b|uQc;EM}7!xRO_l=?#qAi zDKLfgO&-C@DIF~69ULp&cM`BWVNht|`{K|;By}1V7pmVgIbkr>PeDt&S(Pm80`A4c zTV8naLbP%_xCZ|@HGX``Ia;>}T=tfSZme#MrYiL%x;Lb;F|RpJEW?Mj4*G#O%T=Tc z4(oet1`Q5A?I`V}Q8TqulZ2@v1-oMArOBw_v|W)y+4tt!MbxnBi+7k|rWf0yv+n=d z2lu2su;G*1)vu{giypyLo{xC{Zjk|AEbUWIZ7vddz`)MlDW;EMNFP;1_vRJop%-0F zYnZW^${MN-54rklwtKF**AxpspzPUlC{15hI(Y}m!<00s>RLR6pE!oosdqdBFIu*crI_k0$k#_O}skcb1WJr7j% z?Bf>Bk54MR`j;DUw`(G8oeXytS|h9v`o03K&YLewcX=P-6{`*-e{tXG|Ka=!POQ^;@q9My^5+tYRHfCLcK_X`4`!kAGT5@j} zFBBAyGHYflzsd2wIvHQO$TAsca_y$Rm=-T_D-q3t!sSwlv#D~})GBFhZ{Uqbme}rH z+#{MMVa!|A!&{9W^?3m^4()@IpF>V7j(50z$K?2sAd-*86J|b7<4EeZqss9HQCH)o zM62Tl`<`*9BXse$okVi5?&8$k`+DZaLo#sX=YD|H@a)_ogqcFlwG95=SeU)Vk9D$H z^t#^(F;{)DE|q@nfT@_!sM{br=4YY-nL+Lj_z8{)T^MhOr8WVqHHxQ0tF(OMEd)E} z2G}F)Oao_oi0k%8oLGvgrt(iI93)pQ^8=_7_${ShQM_-|`mkZ`u6t=sm;KsE*#1b= z*v8aAh9bS;k%(ima>acjv}0@ruKL2aSOfTR1+z9}vW(}%LA^;bb#;_7ODV35LgH=w zP5J5e=ypc6)G57k+mMu!QR>|aaOah^{TlV)?zg?GWyYl1kR2u7biMuzjpTrxImQN2 zN9@DRS>9ek7Z+sc5_$(+6L_Icwg3qUxclx5g;0id6D+DZS=j)LM zZci+ETsiJ9j`|vB>L1AzLf8pAl-ZDUI#jMdK}nD-o|;X^m3bdeI)2@GESs5Y>~|+3 zY*EHt0fTdpm)!j7|?o*qE91H)?f zDLXQ^jO*tT&}yA1<9fv0w^Zm7+M)Wr?CgvLi<1PCzuAI*SE9tlv@fpg8UB-1Mm{|mM=X($y||mGLfW*TBY(4ip&+zEi%g0 z%42ezstNW-9xj4GPgx=RTh@%bC|}P_Ul3muqN zU;3mcS$|^TBywHYU-WKvcwlL1sW;?=+L3w6kl!^FM_DE)s zWDHoC*UtN(!d&rdGG6a-;SklW;f&4V_hImR$j?zx+ni^Qe*7*cMlu6wuSvzS{`{Sv zzd4NG=cGF=V+)TS#kCy^hW`3E@bA4Q9If3+?UJCrObzMcr!3eoSzpq=i$OO($ro=( zy&$JAB)*^2S|^|owa4<*1c5$(u>v--zq4W*csJ~wgAnOmNOL*rVd2Dd)i>8!aKweM zeW7NgRzt-3`Re&l`cms%h_LexgIuQj!}UmZUlAAcdub+WALREfc~+8YK0MxPz3dcmmCPo9_6 zwA`}$rK7RvVyR-ik_&YDKHck}aQwd8)z_kBXaw1AQ|=8{5JKdCpP@!F62QH-%kD7y zO!9V8A#@Z;cIk1WN!&;2!24;(C+x+h&m=61sV_fN_?7f6ADxT_@0#2Ee#qddsHdDm zc@A@DDx7&6r@x>bBmDDLFdSUm(`1ZGcn=Fo5pohcHgBnleWkC!N{BHd+VsG@kEuVR zV_$w^nv!g{fm?z!20gaBDD1uS`jxOrnH+vg%N4mADf;JO-o2^Njtu#^BE8b5U;--@ zIkP=nO%FiH@Cn4w7r0RNi%()Q(oIfhlGa#r4@+fmuiSh+TTq2P&&HHPFYb;9Li>jj z)-hh~vgfX+9(eiDn&PeVuu^kjU-&gwJ2lUnS@Ic(L)488XIS6k&`&dvGq6E2trVBP5B)&vU^* zMN1o*3#4@H>BQCdba2nb^j!poN)={%$DP8*-@IyE_-?0Pwq!RrCr@8XkBNJ^bQG5_ z2g`;Me?)#%>*!c8SBR7wyqJWOV>Tj#*&nO#FeT%BH&+X(T9;nK5i@%_#qF>8bN1gp;`-=`Zt~j3;xd3U@)*~>;?n1{Q{<8}@QgorZMVPXHq=>> zVN{-%3Ba2#v;Dmkc+6PFJ3P;~3aUQEa9?C5^Ur-DjRob`2Rp56qgqUpD3AcPzQb=7 z@lQEHq6#X^ulFY#9CquP8>-Cz4_|KqRrR*D4<8VuMA)<<(jk&kN;fD{QqtWm9h>e( zkd#JILZq9G2!b>!-5?-zo-tUex#yMv=+_io)p83pY&eeMbKB@eD7`^KH zIh`6j| zJi$qb4Eo<^ccXcv!C^c>G+qC<>*Myn;pB83p60SL=`rHKV_Ge z=S{})%_{h^;`Bj@pcIBpkgS_nz5=IGMsj=!L}+l$FHU8F5d|*FL;fc8nawW?_Z+^~ zJt#e}x{a;~rr4DDh}1<^>ceqb8p~pX&UhaB2Me9Fx*SG5Mn?O?mDhX^v52Vd-NQJ> zxf>t5*!s(2VEs$iE!*w4)M%>wGzd@iMhAgb0Rgb@J|`rcH=5P;J|!dfs=hv`TB82$%}an$Bjvd)AEn>QX_oq-f=V%NR#l}eMBx86?F5UQ$--PM6pOyHt-d1G?ZsSJ+!3$W;} zghxdMp)WB|_Z@2_9S%@}qvfsClcx&z32_{&HpkL}c}?(sbaz^reg*lApd>7ByZo*a zPv`+9(JXg}3gw|Yn+><6IIDH?Kyg|be9^&WGpul2xStOobI{9-bj=rb6eAa##a~yz9f~)y4Kyip{``Cj~@Gi zoL0}t!Fahy)SZzM?{kr^BzF0`Jhqd=hGd_*ngUPsz8Pu%iKu!^pz^O>dQ5@ruBJE% ziaX+2mlAC2-E7a@y$tKAlW_hGYnhquybdRS_0zpgjZUZ~ym+E)=GJA7N5Tt10TG${ zQ&<-HwPGyM-X!MV3?ky+!WsyV`@U@Wum_{NzOSMk9i9?$_tpa=bg8c)I^{iBtLg?wp8u{#p-f+PZ-#(} zV3E!9-ihj1qX}ZZWDe6(K4+I0J{LKcyd=5H_^F5Mn@6-)W?3U!k>DzQykj~>3L}qe zbp8BcX`1MoVu`d{9j7;r_-TgCu@(`r)6qBiu|^dp4J7|mp^Z|~hF!ZlH#tuA7D&n3PFRl)800HZL`l~#T6pc7F0cXyZTZND%F2Yl0 z5VFn3!WLUc0(VT=ksCPZ-u`>WK2wM|+?T8#Qs#Mh+iG}|V|$@HMoh>!AHnbTf|`AciAAfLp%cE;j{(?`&F7P2H=?lM zlLlTqP^e@wfnD`1|I>2enUipeP_F`68C`OPfzc{($95B!x9Rz9@eJI!6> z3&VuTgI+Hu>?zPHcc)a}-8q^VYg<{aXV(0vCs@1!`r5zKn##wV9CDrVcZ-%Ngh$ftov6akaJW91F#bB_|#cp$2|7j}s zq#@iH;^ehH-der#ff3iwtpr@5!|k~@q6YX`{uEC0fZ)3p`bb36-fivM>-i25%B)`& zy6LF49j-u{LU@y9hU};H>V{IK&!hDrPbBUiO?(od-#2JZx#v8e2v4?kxzp}zefrAq zzJXZ*al?n6%0dR3j;exw-pnoc30HIuL~b*I2?Q)`)mmU0*GslW{A)FOPk^tkHo0(*A-W9WI~Hz ze`};6vk!HY>bE;A0gu((5}Nbvb@r_JbHguBo58gk#ya+~*|uP$>J$8zgWdMed+?1Z zd>~HilMg(3g>^_N)ph;PK~q?vd<*x{`KN-c;Gxa$QlHsbw(k9l3EX>Lb&B}DXBCP+ zM_B%fp1~k`ZrX8s+^=V0t3w^}z6vL0_z1!{CFet8B3MVum-FP<1iu%{22>3XJce!kAWQe|sFLrauzCI>z^%+>-nxwYXPGD$YjTH?}u0gbsJlQ?=F! z2Wg^R40Fke(HAC}J$3gX;f7_V_nN@{g^P@j_)eSAJ~>nCy39=TfaV-L{#A8s`DrtL z&Pl}eVHx3{FRhRa$|S?09MJ1vLZuK3x>1T1A$(p;+r^Mhtexj`rh9EW@GA5iAidzJOY#PFRAuOrmPLz8Rn@TGv!eP#Kwldt$&BtrTrbeY8;FqZiED#|>D zbh=*e-dz6iK6HJ8k2ftid@qgTFugg@b1Jz6<T`@i%P1 z`)S##*_Vbhz$1i<)xQ+Zy0O0px{9r$R>3QJ&{a&BkTQY?ifw#%o*4d}3Y6G) z*I0|WhR#ld*<9A^z>Q{Tl99Kq8XVM&aASW3;(}YtzrEA03u2;PXY)RsXP18bct8g& zQ|}YbSigm}P9OU$NwmSZb4kz-< z*WyVUvE4Uixn6hUbVU!}7|#TQ8DWoAwin7y-~Cckt&JIphFREH_%3V;M| zv+=;6AEoz?FT!^!Yjb-&R~ie_9> zm`{GTHyN%V(Ip1PvlD5OhD}Jge1qbh?l}sf)+0BqAQ!W-TQg+CDMzG78VMTXTgv7wjt0g^v_Wx+?!+j1+jmHz-kICHO>BY=@ikqw} z)ZkG?2iMrWCHFdHC@xp;Y|>*TTO#~Rd%8d!m%tr9t#%wV7r z>#O_pIv#6;iDdQm)EMfeMwRI@a&m(x@Xv~+Mq56p@B;2)Z0QplKDmAo%%vg|OxLz9 zFOMdF98qv>tp7i;Dp%R`IR?pEl_p;^+5O+KNmDla=gnfQcup+7OA?5ZS{No^WOn-D zIXI&YB$W4hK{swuAoq5n!$tBrmZcI5Y#7b}@BP7UnJ#>Uu2+3NVS5Br zyB-#RdJ4vxWFXM~?7)h5+aupzyXDEHNj7>O#yoPMOFR_U&XcE?{^a@eDH3xER5I~U zG_5`GL;>IA^p6?g>L6D(ZlP@|^?ur`TJKetm3Gxj*$74i$pYSXKJi+u=7 zAB-xboWH~-yq}P09MA+V+=z0X_deZq0lOh?wzQ^+pPd{kGHAF z9PcNPWt=D$__S!cI^~TE)CLfgdDGt5 z@ZwX`wwG6~+k;ZSHD)bSq+TYc343}YWr}5avfhpK>}bW7f4f}_muKK;wrXsT`PrU| z`bOT=hrNhdC*RRM{f*01ISg#umG2o^7g2yQm(7cs;O*8(Cc6a8T-2D zcHJ*&!=)t$3)R_d#_eh{;^QM+rcVFib6yn%O|o3{v(*uYgT;&-2Gv6JbnnLEU30!l zaEnE9=S*`{rv_P-LcMEYy}shO6@!hq=koiz@1DJRSvZBLBLZ9-K|!j_sL{_I{%a4^ zRe;DE{*m>c#P3J?>BuP?O@klrhU|WQdzE>(mc1@7bc#IXS%w6vz-ADE%D1VI$Mtr|Jvu=7cPfweDo^ zM2WgVk|!0#SzESKtDXn!a#<=9x8@G(tec$^^<)HUa54XKPf6Cfm&p@XF|2m!{79Mz z`{rb5DM=$-UIoolGR=?lmnO@Aq>BR2E}HTh^V^>~Co!X4tNCm);gSEOKgP9 zs}+!RkC=8gUY%8Vk3SE)!V>~W@?Ld@Uk%%4{T8aiYn_l9s$(n7zaK-l$H1T=1Q-iNF_EP0I7aKJ7t&sJ;rVbp~-eQ4(mZT8SkwFg%n`0#LKcb7q9 z-b;|@vwXo10fd&q0nnnwXkY`?pETu-fevYCbiq{oBS5^SmI%SnxZ&KG#8m!PG}OP>e0C+ zKIA8;Wc7-K(HA6ITp`vvQ~aSzyXZUaj_!&qcr7~ycL@5hFvtHbOQaEHi3MnwS@Mio zSg`CXJ>A_|_efv3#qu*RG?@O6Ia6rkO3TT80ar4BdtSXhC36r_LIG{m*We+g71d~D z95ZJkR~5?yD*GUbaKV?51&Lg@wzW8U)wy7DuhxSt>KJ-?8#fK&K9{9F$*?`LPEZry z1bmc$Mui1qj@D>8PShp38`C5C?Uw5^wN0P30IWm8mA@*v4rD2qVcG{9M+pf&g5zFw zlX*=#zpCnH0Wah~b$7{K5INPQp2d#KGFkFZ3H{3-fL*TD!VY)9$cp#5*?+Gf=p&uD zrMAY0xhQd0GppWE*?r7`su*{8#n0{ZwRM_%O1tGkB(gh)W;FCqkn@Ejh>zAYa87yO z%{u~594%aLUP3f2TwJX8V5xjtUd9J8(3-4y2UZ3;ghVnkI^@x|^tY}{gb|2>9T903 zLZEnSnF?nW^#Aroreh}^1uk@Ua9{Aum1}vR4h>vH^ReK&s<7^(N~+NTGi+O`^)Z#~ zVP8<8!pE+qlE90v&9!~IivQ^tNndUE`ET#cXHMA&rrzo|)FCDjNwfaLnfYh{mm#&p z0WM>BPx$(4V&BDA1bt(e6HW>rsJW6qykvKZ%iLC5-4k?kjHN_o&A<$=(U4MO8EW7;Hdz(s z-@Du=<9xh62W6B1DuB~W0WFPB$VzQ7zuU3|z$wd7z@>uK&~G5aH-SX!pdWTKf?Ve6 zA-l>q-SDd0&taFByC`7_1WZ9crxH5M_a_RX(*qbalY6SGnSZc$zT2K80Drz1{#qvvPHT6V+QdE$ps3s^vp4#(u)(z?)5>|) z1nz(z**r)Cw6CpWDaC&Z&%g~bUDH*^+*G>`1*MbnFzvOP<`v|Mwa=4(3Jl?D&~3!w zu?M7rzzKDgEjy^p9sor3=A|HGhhVU;RIQI7^Hx1>4;G|;*b~yf^7#c}zmAaUTX5xW zw-NK+tE(8B01`45Hp|jj52h1nf)__I599xPbX)B``1Eqwu3OI&yVoZha5zj^gj>Vr zw^qv(E^>rH=`GE5?B!?%)p$fG{PdRp{OKPNKb^b3 zX+^2k7RQsucYuO(#~=>bon6sM0;)G~Jr5q&aP(NQ{$01CxDX&PmlS`xSdAH3_Yp&& zx#Ahb{&_x=yr|V;m$t0;-~tBD`8B5CC;msw_f^1uUh7Ef1ppeS<(|(;nKnlQVCNQk zv;)9i%$clUjm*b!lBIm0jB1Uc%z5+B>su~+djfNTHjimnQaJSZX14WMg4uX=*2JT& zB#jc|uB85y4_HfJ8dVO#TwUN}%_mjnTzqt?!#kv8fY7YG96>Ui)r~6G8i5 zh9Te!Al{mk(HZq0b5Hy)b4SQhKGEHFg#vT;ji&i{yT$eafN4ZXP(2k?SUDeieUYn9 z`7(*br1qnRU~`O}U49b}a3hfwk)ih$FXMwM8xBWvGNhTMw2hN>5zAHwCHQxRC6qKn?k-a#*g<( z_=~-|!kq~?9htOwa?yj4k&ed1G zp~9&sZ72EPsh6efrI32;t+w@~)x`q6+6|IEsY+vJ_(o59oCk;%8mm!b_mzSw+#s*Q zR2{oa@-G!%*8ipWTE$43u&pAOHM-Mp8-0T>$eo~`1)WQOQ2vieSb*I5AwfXg7SWHQ zqfRGcIxkm%Cp?Es?zd+_rP%8~o6~RAdrg+SHbgm=#_v}WF4jyhzKXUc)Ciu69idXJ z`OkO};NLXgsYW5bN@d{*qQ7OFX!RNFeByrVdLgXY@g^(O0EAA0P)?qa1H!Xe|u z10hyQS;M|fY?gRJL5@2A2J~Jw2o%!;G zsC6#drDD($aWWu^av`1GwtRpJS0zB&0G@M$^1n=q1Bv5!ttW@wJB)GV6h1bPE(5f# zV}h$>e?U=yAE3Py6n0^9AU~W-;f}vl%m;!eHnP@Ds%}MG)^B%s+FnE%Q#}@!M{3B~ zb$fc{SVRajN_gfaqC0L|5L!2Glb_)-wpA`~M=4}2$WQ8pytRdaJNxzIaEAi)2WB?@ z(WF5lMb4AY=+Cyxvw)PBflBH%w`sKuXo|-qx!!#I`0J>tp~)J@1KU163Zi}aG~P&l zx9v<}U(aXo^$!rri0EjcNU#Kn*#fF))Dho*Z>2kx$9`74zA#TZR#YSXshpc5(kXRx zxy0w7UAs9+yxCc~Y`sq>qaG}YN?v0D)96bQ^{+TjgvPFa{&*qmamE=!y}N*XYKi0H zs&NY$Tq>WKQ3nY4ml?+$juB*(NN8YZg2hZT>O8*f9l4ztlz0le^>I|tFA_UyZBE3V zH{pf}JY)j>s98|$U*V3U{Y0C(YzeGib)K}oAMqc0pHYEv@+bs1Oh;EfrKC@Ohm?~H z>69na^KhHOQuXY*5$`ogh2UbXnlNz3TIb4B`-WSqkzO7jLyvSa7+2OB_xY=7#c^r2 zQlkj^!pUMYXBQsdU_Ijh)4%}*gA`y3=GpjPg%dyoP~nU{8hfEvNq`jbpyK0o`}V&i zL6U;84k2dBaX624`9Wa#859w-t^{t&1pn0!&=9soq0<(g$`T{xwD{4K#{%|sjt!Xk zJ9B9sE9P+ts&xFZIq!Mm2``TeHiSgkLRiMpQ z)qUt6e-9t)wuag5>YSf_8_V(0q5q^tjjjMb$f@#Il+PI{E-HK-on1f+MU*)cXXv5T zSk{z{%r(HcR&A&pLd5d<=m5>yXxaV2Z|GvVG_S;(OCtnv~0 zI1@h;d@+4Da3JYYY|)?^mW;a)6!UtbFl6G8TUHW~Ae}nALOyDGSn6nop&(4I(U?E* zp~GDW$V^Ovpx*EeR)JT@?rA8xkrbf42>CJcmJj*JR+yl#j=BYI2g&pi)XI&tlmcAF zL*(;{JA{PBH<22aN^?SVLgpn>*63Pxs3x2pm`v|yR}oItF<6TFcS3OdD)vL^regffWpCv}Coe|-bdQ-RanQ%r* zio48elD-4M=M`urVe5){WpUIz{G{?1_+EH(f|vCeXj!Hs0E!`|y{OR8iMcP<;hBPA zvI1Tou{_X=&>9b}F@tw+pl$V>p}-AAJ_02-qK*(*8SGYod0A($Jrmx*XUTYw3L<^9 zP#{Rjl%RJMMi448-$5A5V|zE_3t3?o;GQI2?gpd)Yt5qTI`WqgKG%BVBluigU7i75 z9AGJ2m;gb26?t) z|6#>cio)`+39mh%0UX=!NXkL9J>KNz0=bd81e|KN9v8-ELseVaRL)&59*VZpY+foRvk zL!YGBv>K%Kl?KLk=UrlGC9=Ds5fDo0Bq;NwBDz}JW2{=$Az+u*Tzjmca}d}ch;`R3 zm)TnRwK%~1UI=9{BdpLLOASu*z%WpWd8rartiZmswf zacjqJ`G>hBS|Im3%b}a#hNlEBW-UGt@+R-1!g(w*HG@&TlrRLHLx zD#X*XE8&pg1HeUc=iBbKVpVzfQqRF?<9&8d5W9*7(6>%jYp{Nn6tRKKt0NpOT}+<( z3%Jz(;m)6*ir&#YmN)|bOg+)OK$_9$`UI)r(26)z;H{4!UXjHlyP~kIEg$C6nj6Ya zJCj#k2)B^+D3ccA29Fd0pStzhYa?Z*pQFg?J{q)Ym?1f_8MXgIa9@=s$(4Ul;(U~v z%0wXmm`fOm-^8Z$g}F>pGn%l)fIqw5%#&#j;92)Vb23Nh()UICaeJwGJIB`1AYeH33c|&4(*+O^Q+WJ%#~h`E&#^-QMG=s6 zmj!rMaCo`>8bUiN)%5KJAUl78V2{oQ|h^bslel@$5Y%uaR7(P;S6IU;@ zBIrwPeYQ$}=}|t^h!8=g)#vj1mc<=F3SskoBtC~+_|thb#(o5F_y+sck<#Cq8-Adl z=8A^{m@slzw!h%Y7G|Muqo)=qD#-A02`|9d6AI)^OKT_Lk* zE);r>h>Jvz2k|XHh6JjCEV}NaYd}~5rGR&-sRBnTJ6N0fxVg9H-AS*+^{4E!ejs>8 z@Fg-e43p62vqSH0<=bFGDMVVx{vQqnK8M(RZd4Jp;G%rq7-uCC*B&x3&vr4Fg)Ta| zMj5yYXxG<uA0zzGySXx+MT=b*e{ z0(p03&C-E4+j}h_03!?!WD>>CA2sDr;JrXOp4P`ELWy{`qa^S#ZAcUfOyK#-mcv4Q zI2bZD?8`i3(A=9!wLITFrk73e1zZ@L#SdHprlfPB`1B80xWo0|VKDghd;c@~iiz;D zo!=W4cxK!nFg&+i(FkJBVY%up-^rn7#(5XoY+xQ%Q+j#kw*TRzzSiy9dkM z2v?+LU<=5le`04j?tR`MwYZ~Aq;E% zvQ=Gp&`DjS23(+=gU_;lG5hi9| z#(wqe;y^=oJp)N)d+d}{#O{f9%?m=Ypa@ATE5#dRynXM#MUr~Ap^-CzGE(+uhEfnN z-DWLCfSo$m<=F}kBUl!`?M%*ht!Yw@j-1)zDDKw&=#OoK)}p&Lq01&TVGGts0Hw?L zA79y|2^B~ltr}W=YA%CdmdvDXNhOiNg9zRC-~c7Wa{o`E=d0XWN`uSh`#rChf|i!> z(F-G~_3=m7K13772veW{qKNX|J@sR&u0p8f0(LHAf=av(z$lD|Yg8#Gq4;~?Cahrt zv#G|4I6_2_`SpEK7^UBBCeT!&+)O`@f945DKtNnckBg|*+5k{%fBh)V`TOHW;d}#i zJoI}=ryLb_Gk)6~g5V~r7kHa-O*Ib0cBecI z2k1*Mr1{Q7DJ;*XDW7Jkbmm8RxI8o3^{(nVreQ0xHnCC{ilEY39UaW2$n_gz`xzh# zo|PH?3+xOLdXY`F$03HR5emglEI$iOQTmA!sbEci@6T) z^LHP7-#pMYG5yh>Loa&39Y+`G?2h>=M36PRZ-4C=WxYD*#dE%5?_fn?a@AF!Ru zZDIYF`o94jPZx^SfiAFUsWasZAxt^?Lu&5t5}7==Ut6pJb|gh&nEc8#Z4tjJ=9x}N zjzs<7UA6hiLO=^lIYgTRkthVh5fJNx0BcZ^tg?Gmz`4^mO3vN#nB(WmJbbo1GqUTD{B2_AUY-ssp@OZG?9Wtld}h3IdEAL|$+Nd!S&-*!)R>ZrAIX&uw4xYGjkx zo(a0OjpE3!b$;{JW?i1r2Y&nBp^XCkT2hqs?pm7Pb&d@G(;L*_RK$={y*wD+UC)t& z1g1ll{6)Ss@d7M|E0}uK_(u=%VH8dBxef_v20a2a{RopHKj14BT1I=Ch>BbA!J1{I zB7mpd;S$c?_cEicj>GzaDoKwHrX8{{<>?T$7Z6nDx|Br(bPBO}_grB|F5EGHoGDuJh3r2Vh zJl`?mbTbMZ>!iH_%hfa9MF@(ATL22*HBmJ4XbC*&+csWn{0%erBS{^CgAh+*rbFWt zfj7O!yOf`RHXwk6fa_{5H!rhWcuMtRq05(VqtI_1obUbDQAwKxdocD(`v z6oZb9=VzvFMQnC@LLi8;u96MGK7-7{=gz$Zu!%bw1wUu}Z`L6w#W&Ityjk>#IbztG z)BF}xM?7<=Wv8kIiN^upxXIVfhQ(JI=SL`wc{y_zRSSDc9AwRbateLf>N^<{!d)*A zJYtL_IF5U4TO53hd_b4C&MQKmYV%>oe> z`mVinQV+jRNC}j@-0FkGPr)0Z#l1apLi|8UK>ju7t7Ob9W_XJ|d-ihG2n2cf>LG)H z0Vc^ebHo#M7uzgTBn2-TYnuK5d>>JC@r)DvYP^PUS%OQKWxbG2NV){A9m|7ncs^1> zbD0i&N)S-+AXNf{A}jW{AQX{)PAjUVyC^a@QgD<$BRGN0Z4*m#K$Hkt3&z^1UbJ<6 zd+9(Elfwu*3&_=>OSj4br*VJ(kel*@c()NJ5>$|1@y%9k;6Z{H0tp>#;kRc9k-wPl zFgG*Z-CzS@it8Hb5Fmtl%`@=;B}M^YYUz`sRV-9^tgQ>8&HGrBy3f+(^(0=;3rEND zeyG8*7sBvy%$P3bK@jOF69rNBa43Z2v^YABU;w1k(k$+ez7(DZ{MhIpZc8cfi}~*G zvn2t8x=u=Qqr2e61dx~58GeV|d`)SIE`V@upc6(3zHl$7ooi`AANUkOPA!$9S6+4b z35vzKASHl^JVMv78-jcUH?gm1E4W}mMxCGFH01aQZWk3^*b&56$Li1EUj{(KNTm<2 z4q6{^(0xiFhQkE#VK(Az0V;zawm)Y`8XCcZiv+8_}`=`z-%sDzRIO13b%4FSFAgJ(P-Ln?a zY8Q28#ajGbcPZfIn(7GI(KjmOhG9TuV2R=gd|HYsflh&^@VAg3PaznvGvUk(EsCac zyn6vyhf(6LR$xCU!sMxonkt>Zw^^Y%$p1~FiJBdQB!$?tu=ezv*Dk1#5)OU^j(}k$ z?j7VKQ&1#+n)#|BDu$q!>{KdL0llOl+2HjH5p)YoQ{jWuAXTNd4%{uiAO$VP?g;=K zf7D)8ikP9vU8CyDd;3-tQ4L#7@qkZ0{t{mEuXj=+Sb41Zz=xyVm0*}|ojz)Labo|q z>Ax`02Sp37??fnf=DdapzgGNWbPUV~qfZd%bzfl|@INd78@U7qdex!91p0P8-*Ihd zbtC;d8XN=qsXw+el-0ow0ZFPQPhgX@$PN8z`p0L$IVQcGnI2zJtauc@oZ!>t@p0t;9e4nL-3RYtOeBaBxhK47M=|B4&iHIo zcP_$6-p>qMU%eiL<)_G)_6H_c`>JiMS?zx9XUYx+poWqC-QHvrmn|7p0Kr!<-cg z4K96rS;SxCM*n~X0toa7C}zR9;eQpo;2gnwOx^uUKKc7+`T^l4VC-|YU_@ET#Hj=q zCZ+b{SeI7jinqEvO>)U` zzwL_{(p_vbeG`!Osk$u*s_x7r?=Zxrwem zPr(P_;1J;8dJjsm{{GS(c%b0*^{OjX3Lv|Za)QqTuQ+u^jc8)|w=zW7CmP}x-mSe= z`CSnNs7NR2+u1&XG|$!VT|Ge(1ZEIc{d>y)?yT1g=TYy@&(DW5du!N@LI-El$Nn2Q zRVW9iDUGLCRru62^uH~4rq;*HpZnZxV&+WDXNvSSxu^H$=}go=LgWXBO}%I?8&OR| z#^XbG!Z_e#4Y~P8?zMSm{To9!x_g=T%hIM+{*M>fuEs*C`nTQGhlzHCp?M#s$A8@~-8saZ7xp?#Bug$c;T-ZA*8F28V8b6^u=zpY zK^jhyxQa&zbE8oB{m^iJykz?O`G8S4FlSNf^oaY?)PM-dSDNwyg~$Vf>vC-TB6|d7 zxps6bTTKxJN9x{f=|467ZdGD*c=B$n@J{|h8pfA2T87}ZmH6$lxW#t1D_|OOK&okap z)7$g+-zk?MzgX;fgM97#zAv)7ldLPpBSKQujn=oeH)pm3Nd=uaP1Bj)ea~gOT~QPg zk<5_0k8zjdw<#wDA*}5^LkoL%ys^~ZM|FeZA{shhi|D2M`_G?>0qZt@+8XtDG|V+b z7#m}fFy<1t2gx=xO~G2@G^uJ@n@I~U7!aakmEFLXul(g*tQeTPJyhUIV_A+=MuN_^ z(jfLV)J5rt`_T!?KlHw~SWH808}&5gYbq*xO4G^S%bNbB4}A2g?6tJ?_km9a!clT~2eTm0YmSsW&klLN8q5ZigoK7Z z=Fuwe^@OjF=|%w{I=Tcdy!MKmNOcdoK^6r{QAI_M<@RY}w8Md$SRf99tr@`jj-Q3# zkL#`O=$%-8>i!n3*a$iW)%8H$T_~0y)hjp|&hSFbcH>@#)52JphzGxqi zjO|J6IXo)YU(1wa68HpGC5aXe;lP7ggY5tl$1~w?L2~ku!=hpT%YCCL!VD$}FgI`hGpbN&cIUl_Sm&b6j)`mqz3+{EOP|Rf z-z%X~Ep2T%?iChEyL*nXQGH4f1KQ~jC`K34p2{QL!SU~S>a{>@;vNm6Z};9xbU``>Jzl2L$l^ z6M6Kvh5DpTWF9(PhaL;C?&ry-`+!SJI!l68Kj3_FPf2!^gGWH83T?w|=fGR$n-ABZgLn$np%IevaLE+3}?I7Y~AT z0oPIVcT7an-(9gu?A4%<<})GgYs;vM8WJ4!hed0}iWh7boUEUaC;Lv zSXhi>^`ev$BbNg~C|}RRrB9RfyqS2(cmfA+qW_%+_P<(n{f&TPdH@+JZ+d;v?mt4_ z;|t=*RaBVOaYs=DWlysbCW zJTJpr0%nRv>9J>(GJO`*mC9jy^EID~PP(lEjKu_Y8Z=Ij9Pgnumq{5vkIIu_?gpK3Iqcs}N5-8`6gQX4jQGou0f0 zhd9IToNnv%(>~f1z#`Su0EBo6CLQj4nyA02SbG6U-L#ulvd;(S(k9nWbmsO`l3_y+ zy&r*p(gX*KFg(4Jj!*Tuj9i_Usyx0XZ`^l)6`Ki-Mtk`m4)*4kG!@JJDZmYOD zovbiuzU36uu2Ehfp7jcX022PbVb!$w~ znA1zYLq+Of3-h9i2Rv*?14El&SAC9rG5yw4*koy8hyqM+#z6UlG7`sBN>Gv?YRS%q z_R^l2JebrWoGLsf0O`q(C6hR?*4Yc?8X7zp_qo?*e_PT+^OVAs)7Bk(wE?~o(+xrX zZ-WxLJK>QOoqZa#R%Z zKSGUmt6MCSsvU8h$s2wK6uCmqivOODyFHvE zP)=I+WVV_p(5tm$V7S&g^|dfQeS}y}(7qd2GEx1`hdt=_2QF5G>}s46sTfh6WCG4} zu0n#EKBxNo!RAcP_S>;{?{0vNop->%Qr2OjIl3g3>A{^)B$TMT0#Q4Qtku>v76Xhd zy{$kv-Uvmj$g9y7t$FdLqDF%VOozCLm3E3gr9mP_X^Bg@dFNpSpw|&fP2xvqGTTm* zp?TP0ZnueD@Mxb90DZzd=rP(+gX_oZlcNoa>^3G(*YkyEn%PHnDf4z-N6Se+js1HqWmopZsD{G|-w82cTg=8unyyH}*K^|1nA&-8GBM8I`m`ha~!=Wz3U6S3m2Ubzw z-qwdIULhe}L&dsMYe)N?()+#em7LI>jHb}IEwj<=qQaH=PyT(sVuRl#O=`HIyL2&R z>cZAIb#yDG!WLIlSu!zZ?stT2RVJN%qpFE^-M0-f+g|5}RCtZ!tJozHy_#^whpf@F zOoT_);m1mZ{}8NEQyJG`tg=qXT+bVS+G=?^hydCmMMK)gk9Lec^{ID5(%Aj$n4az< zny+yB%!^GlW*J+eoG_z)K4%`gP}2IKo8%aftVYm;d|v1NW|w1XLK)BRgx+H})+PHD ziwb7&`C%|*Y2uRsFGj8iC_+(h20k=@ig!&eR3ZSV8Coi>`BpWGmGbDo{(~0xB7K*a zXf(JKzK2{a8F4^}f{P5nw`=sJ=X<_>5 z6;A@5yRs>qVyx@%QP6P(yWcal!D?Qzn@VKw5G39N%p9zU5VBz+yp4QvhK{W^xGdxw zM=x)_GFZ2{n88P}kT?7#iflnuZ*IjjzxeQ!979I(fv?@c@~q_0EslkmAMd}b#*4zh z@)NWlMVg%{U&tx|2ctG7D7has^;lAq;!|*r1}s^*dzc#Cm;Glx z-EKs{TaXm0o#QgXhW{NEi85uX!GoOJc9PED6+_@h5<6=h*>R=__0J9Q4-ew+t6jf_ zCTQ&vCRl&*8vxS9meq0y>ovdSt25qBv;F{Mp6>Rs@tp0!wEvq2?)7Fe;#Rs_TgMNU z>mL$PIZ{e`dmIsfVMz=e{$Wgc{z0-t&XRiEui^D6>*P2tt3lZkVbd#41rqE}U7?JE z8b^q<9*BIn(+R|+T7BevcY{HP?O?m?)kE(Yi|Ru8af*)z4LI{&jrDFo{O-Vi_#Efnm-tZ1BU5fL5(938 zD=MmveI1rUxUGMJ??S=r52cZ(OxfVF#+vQb-j^n#h= z#Ta@ix~?VfDy4Gwu@m1_+v7u$goa%y3Y7z`%I{?r3PmCq8J#6QflViFbvG zBAwtjO-qI z6wroKj#gUV`tadnnxnYhXnC?mKn1ny&)7nuVz(oKZE9Hu=6@TE}ayl;4Y$GjVIX-}GfaN>PYgE(RY zW4dyz!_46H^u>DEZsyEqG;;@@XzgQuFyP>s7Hz=NaJuJti#^YMx9)WlA*;>}TA4E! z?Y6_EfOch!3pQ)tcC-(+7fmV?81xTJNc8ioiX(ZwuP@&mobDGn6N1Ty{aucMp%Z7a z86=Do8HyhuH#C@^9K26)6-2@j8hCK-L!ma;~4IC$7TEzy+f+L=r>y) z|5p(!)Udb+!wenGw;LHc%(ko5JT@m{-zB^ZhC<6x;Ei6~N41i+Ly)_$FqHbz7=P)bRkQ3x#^n*RKyXh^3wzs$9?`(thtr@)xqXz9MIVTU-_V~R>G85tO| zo=)pva?AQGYmk|JeIV9(9@9M9{=V)R_t@K?UNfSBY$iY0+=kreOstJGB9{kZxGahC z>g5Ht-bFk=|K1Dh5DdFf30Z2jk8bmr>4iCGI<)PP2*PP&X?P&!-u*9eHpezn=k3$^ zYqrj^4;i#;f50C3og2_0oer$rTRPuUy(3c{^enK4;Lzy+J%xBX+w}WfAfsQ}iZ}dCWovK3j?<#)mK zqrv3q!FTyl?dm#CwDmiT$_&NbX_!lWjRK(YIAf8dlSc+t+mP|O#2x0zCI`4XG0O{< z6bX<29N0Xg+qUWWS_}*FApKD#U|l6-`-$)Q!Meekz)S4TL})+>zRqW`k_+@rr9#nY zrpXYbhaADIE01n->QZgOxU=l}tilMw#=oNe*cZbI&%vt*dAwFG%xXD@ySbK{S z*EC$B^fS8odmd~-r2?P65d-qx4!%U`9E(BBa`KO6)fsJ$aEKZ1xHJVbvH`QS=Z8>P z@obQOj8h;>&@`9(s)em!urq)@eJSiV*LtP%d|n2(0T7ft;H@qQd~PYZo_im;u}5l4 zonCN9qsk`ett_*=x(pV{qM?T44j=LDcyMZevP|5a64ydk8>J|T$Cmvz=R5zNH1bM=$Y`NAMQySJUaYm) zy#hmHDLrIPi0nDUEx8|Ykv*Z6AbUbbCzcekrgRPt3^Xg2Io!$$`H&ifG9nv4^pke` zuaU|KpPW&9x@{Q+VBo*~O3}9tIs*J+!Hta92KjfMvPP2umxsiDy9G4HDn2yGUJ~Xm zG2@tTT+hZ2cW@yiWK}lNf`My%XF)}{UK46ldYhfp{h40c1VhKQ+4N0{~3V{-lNAHP7AFDaSQ8-Fj(&|z?{cAq;Tb|G+68-04h z8k_-D{^_Z#hDe0vQVell#SXQc3Sv~7wSgZQjSKQ(ELcN-bqONp=J!euxb&FiO@m!l z`vYF;F82?$1xg`&4%iJ-<5bj>@}A3bw{|Cu zC57V4AW%lor^MuUxoDmPsEmx>LHY@;hreoI{C2t8oSKg)r zd&8fRBrF`Ru?#kYcs$r4HT?yPajwguZD~(JQ<*fXDm>|#wAgT}EagS)D?;}>z%*EU z4cByyX&WqQ7Tq|^|MKtjtKxk6; z*su+&hLw3KD2U#)L}t%@!QOwUaoUdnOzBEyk1L z1g*x_D%!dHZn2Bv?1u|N^*GkJmMUV0?5W{g^`TD!qOMg^2~AF7-njcdyg{5YzX6o* zX7E3)3_b2_DzecP{Doa#1~xB6Hh>&JBa|x8>;N=0)2JbizmwQ=^1~pW|7*%02K4_x zYyWO$&k4X)vUf9B?Pn!~xp%_~fiI9xFiBAAMqv2tiNUrC-(XGs4Yp5NTNBmxVM}+4 zZBv_=@Kc|LFsmhb`aQ3AZWP}df7>}`E`KwGn&Jy%O1%fvs*L!3)ki-1cgt(MuOkIN zLJ?n45Q{@2JQVUKpm~C!z@~M}XZd~o2~`dLqWxE`yhEmLPi_K_9s5Ud*^iwnLZ_SF zzSJq4zLwJ|dmBxNXad;sJZ0PB){IsBpoM_QNlm$*np-f`xXi9q|DAO)eJwpCxIwo|R^O*V&j*S6HK z15fwr#qOnE9^5n4pDnD{!Ml#C^)g?xY#?Fuk?1<_3E}lxEa+oV81imP^`~0UKwH`! z_pGa%uKP~e%Lf)id1|$%o2h|wOtti;7FX3gwz4y+&*!vsy9SD`1q3K2 zC|#b^NG^WLs{wv&4cZ;&Q4{m4papecVf_K|oxr*(!XB(#3OP`{2}y8TcRCnA@1@^- z=XJ_nEmyNrgtAM?^|8@+Qe@nR5R@A7syx*FQ0ug{sJ(aK(9$!#wS+x$r1zzk`<_Z^ zmNxlH-SHKJ(pb*#knS8JN=?CbOEw>PQgh<+`?pwl$uYZ74!fq~OX z%XHfgjE)x7RC~B6wdGkEKI1j%#_@W8XQDzEDR()=4P`*FPV zj+WnywKE5L?Z>-s&8g}a0 zWm0P9@+g42_=n$B!*Py?+|gjzi=Ytkjak33c|9*I6M6EZ&47OCn+s?FeCaHPK)vMb z1oD!UO$4-$xOK=muY)SRVxK25j|83)#?s$S2yX^$X}|R0!QSJ4UyjB93Loxc&G?o*K-}5d0iORy)?e;aF~8dZQ0|3ZntKeBHNf_J|0H2Zx;}-8Gigas=7i0W&^P*!`L? z8)hWTOaQSI@SzLkbPi_QSEq2xerQ!_#*DU`NpwBit<{CHWiR5RxlUDx+uT}U#KZod zdou0)lmf20R~jwH)XSW&=_J0al1u{)l5g&A>D4U0dD4O0`aF#fsyaHIU3lJo9s{1A zuQm2AN}u+(=UV;m7s{1hLF~BRDL%?#F-VW97uzw9IvBD?hV3Y0$GVtuqJNCl5S`mH zQ>0H;l5|OiE;d(QCd_}b1-eEyl1oBe_|qpE`jv|ZcwLDZP3`wL!k%*!W8A|2{PQfl9KfjL!Gq^mb-Qe?BN=Gzej5w<}$fg2YCid17I*EI*S)m zO?P#WHGIit##_;_HBx#|F_LuAO*yaz#o2)b#es1EjQ!Me$~g>mh8@oK&A0buSYo-M zezg$$d2FT4tEuXQ%tG4)qs(*I_UAZYG|u%SiWE+jp9;jU5o_M^+FZU90)Z{lG4tsT zn;{hiW2j3OI>JlM`rB&djK|(=Tuy?tH9o=MBhb^cdT|H8Iklfo@`^t>uR1Q-zPJf$ zcSz&r*L;Af+FOEW{aB z7{NZCRLclEkoQ@lfNHt*X*+hS0x@@_XB?$lPEaCVtO!lcs|hs zmsFC4m6W86WC%XE`z9`M_tqC~>xk}aeq&HUkDIzK)=k7GL{rFb^>qUYT&NIK1gtK@ zSyR9UYO)e`XKJOWx>{3dZ{5xByVQsuZ&;&TL)nXOV(O))Rka8O?Q9+7X4qG<(xcW& zep9^oN!~7(kVzl21|Jjy5BhtIQVRb(ki+?i+%06C(LY+FAr!@Vg}gu+1u7*4g}^7p zW_lly<7fgC+CX5uJkW32@`f<4L4=J<5~{LHWz2Dmmk#@@sVPBL=MKjMx=T4egsA{@qWI#EUdicEi%$ymIW<{HR;awuVk!mAFv&r&s z_w_=`FE@k)A)VCGHM>C>H-~0Wvu*3QHP^~+l_=vTRxh)R@N@UKc5pKAFx5K^t1odL zF+&SXkopRlwlnBmYqv*B_3K^Pn?sx?+?^CUhAnl(DDr3dLhCu#Ad3S}O^hMo8c-FsN!wT6>gXLWj^=y6968K&KkN;|OP7i6 z8&bv%RLp!9IbtW++wMvvBxu-!;`Gz3`9$sZgteifYV`DA+MDv`I>EMTDZ#@#o+o>2 z9ycLqizzF)O4%tk3%*6TmTHMvj1+;&`2~qQ?LzQeXC?H@m_4~u`yUD)6F6Aas==L{ za))Dw9`f)NoV@n0d-Y@@TyLV{LA>z?$PX$ndWMg6c9B-LGjo#KQra!OpT9^Kn{E=s zW7eC6HuY5*xMZ@gjzTp@OINbs_5^!mbe^#Lq&U|%Bs{&U-nSyWxSGgC{2ml*QT7{_ zy7weO<`GLJtXAV2r!c$q)yacxk~H1=X&yFc+jHyKklvHA-B$Mx$nB#8k1no?!#&F6 z!NG-2FstAeM2^Q=>!Z7(0%%L%e~Ufn3K1jZjlz4-Z}grZvh*pBgE<}DV0_=qu{$os zaf0a6cox>DS3{M=+-!!&D0OCsWby)MBNg1@E(!mLr^H>*ZsjG>9?OSt!TpMknCoAL z0$~-1z3femN+(~P z7~n*j<^6z<9L-+0SIJYmzGOOnK&8Aq;0*2G?p;5&aTXZO59}|UHD;RF zvZ>jdz2v=Fw#kWlWFuL-=^%9q)juWg3Un;IC$Lo%V5WqWofPCLC)nXniy?yv>SU1@ zdf>t-)BQHD&q}TRM0f`VB=%Piel=96Z$~-gJzCg^$q`slr#C=4&04eD)R%k?=M=at zHqKp`AInvHX>YmhpdDVgGw|{B=EB|sm>PeVv$7XwVcCLTebFA}qrQD!chsfyu12o@ z2&OT=8lvuD$dn>v#+Q@ZY%*B}tlV?@l-_TQ|FNvsLi!rNq#L|z0SyKTU343c-Ag@f zI#KqAv(DB}a`_?sd9AAB6sF)6Z!wSH8%N~Ko&o`Z-z-vZO@gDoSvYwRo3?ZG99coV z0ztgSqFM)Ea~Vrs?=)*>5`TtG?>Dkt(WjI{Tl6?MP^Hm;9b!krQr{T1n;x)^!j_Ov#hUh+=vPVTxO8(Btei@OHyw?-bgJ)L zHX|8G3dTd4r5A#{o=G)52>pTCN)57?N4r}aeM$U%HI{EeTph>m`>^9{7_nPVmiy;- z>)vT=fp*WD*NMq3r8!WdX+aagH~1XOctr24@U&a=UV1$f;UtuKj3 zdUKQ8A>wgUTLex}e#Oq}Q2tw70{^jv>oiTbAO^fI##vr?Zepr_sLXW=@Kp426vnra za&m#GThnW1cJ-&6G{=Kvk&Nz5mHb=M2U|w71wWmxki3EjFi`C9RM^ZKJ8iW#)Am)) zrpKhD$bz0&iLGaL!)7yOgsP5{JoAFIud zaA%TA@l*ZCUZ_pYNZ|9S!)6zSsZ~YWc0Z)zRxf9BT{BKnU_X;>~s5jm7&9?0HO0ib9L8I_;AcyEp&3=Z_w=b=OCvvkHj)vrV} z0w)+$3&nGv5ybVSUk(j)Q3-45h{)3)`YKlIQ@gX7SZq1T-5Sc|FSDki)0er4e0rO= zf+Dgi)uWgrHZ8EznV*sF!CisWI(Bq7BN~eX40`qU!4;9gWw{k9{ct)hBb*A^Gn|-B z+1Vqv&>7DJ+OQNAeA=r#@wlBPSC8s^&@wGD-)gSBQ=6k}R5={Io-05N54cHet$Bp$ zAM==V{iV(GPsH~bxBX;xhs+iVCi-%o@ghqVy?6Wa%nVj?90SjkB$Wlx=ed^(=%`1- zZPqGzVL%Klr>vtbVuA5346)TQCl%|OlCgLF6I2CVuM16Xcfz)2$&#JhVoruA}aYjHTe~Nit9aS>s`(7q6Xp( zKj4z*`)2XlWOZ-G1zz^SZ00F(wbC13M;?6LA9Z0X$M!>kQ+nBCDvSTYzpx@sK9Pnh z&%)+qhTuj`>uY_e-0<3d8l6-@B*Hhi*FCuI(b!1i$R>xLun^|W3aqfn0srj;!{u+G zGd#^d1RKe>Jl8%0^W>cLLHo^b8H5|Oo-pN1Wv!u)*JtNI6Qia{J4PSq&2Ke*R}$up zNh`24WK0q{^{ z$Xg+~LK&hJoNi4r2aZE8$$RoyUB$)3cm}GbYvA*10!T!$I;98g-XtW^66q-{GB?H^ zfegHT!|H*(iI|bDs``(?f%0b$%qEK8CkVREeXKpsq5A=qOdU>>UI=im%B5tmsA|(G zSl{^k1zS|rXOx|x@3KCwoVu_%W= zl(G>{iJikcO`VnNz9@$zw`&=j0~3S5G_e&b>r7rfMthQ9@bL(I8jp-g!`aL9>@A(Z+m5l_pC{e<9qd>8I1Axz;&TK57*_fy=$$Z z^pURwstMUnozIE8>YTXa&WO7t)5zEJ7da|InYwL+8VFH!0|%93C#s>T@FdQcYLkxI zYBjIlGHO>pF-cTaUySZxYN(tTZ>A4|&hp<6IxJn2@hqxmBZ@fCXaEAGqR=1aE)HKo zCf7|7Nywl;M=e&!-|M_0Tf7aCrqxP`i>2H_DSYN__R4KxoYYO88?8GQs z5Z9eN25P{J*P+>KAYX`yXKWAe_D~fFws7#gUX2>?IADh&+ppc0=w2^eIS@P@rM4W? zeDjmH)Pja7hO@t28+r4wVN{+5GG7O3JjmG;X->LRdqq}Nc-cIL%RNphM@_h!_bB)| z3bgID`~0VEl*5FHnx)nGAs*=EB-Jl4hDFV1I13(kF|W*3sLY05MabP!KQ1J#=$3DO zH5w*aUOn4YiIcRa%%3yy4sCI*J6?<1u4yf!e_ zb&iL)@(X_E6+fU)9in;4ELuw(%z2r*pJ3qc%K?+B@r_X2X_t>sw|a>iWc7!XRQMr0 zMR<@o{Jy%8jQetS&en`?1Wq;Dgzl)^2^t{W;MP#V2#_Eub+pTbz&+h+=ZNNo)D2|G zD=!b)@sV^V)>AQ{Bl-8{hAR|gPM)kL7h;TK3Ol;MKywQib^s+cCJu3s4z#*yC8IjE z88p`N#|ix)5%V~EHs3_JjTiu+>VBtFq~un&ptrwF#Drfr?BLyYZ+z6jaO}Jt`s{0X zH&N3tEVk(B6o%s|b{s#GCU`-Q%Tdg-rBeVpS_1aIb6&wH+2cj+S?jO`qk$~priE{h zaVMUSEGQEn;*x8$q2rCiKL^nNB-9(Xsx#1!px8_>jGz$~%tP}-<`eK8D*nY3-EOJt zQv?z(QIN$(4-7kFu5E!i9+ErDa*OdMS1Gj?gvMbpM&lF}x{}E82rq%5sDU-&JoUKH zYMRnWE-$@nNvbYTqd*(YHK&@nWq;Gwu!m1Hh8XX4t8^J?<>Bj#$bZ=uJSL($3eb*7 z%`B8WAu~&l3VwF7iFL;TA6k}VLxXwnGjf*IVu!lQaVH{8n*1*E_9Sz-6xXF6n(aqh zHn#QGY#k@n0cVsKxO_4SmHn1e#k64N1$u`njzX62G_`xB$;N9)h249oTByTbTuMHm zI(p-wl0=3pmvFIR>&)}{%1uIhsuGR;!M?`L;^8q8Sn-bP(rBmcQ6mfr3yNHZCaHob zv!qf_bxcCxQ5?dlrXm5dlj_#|uoxLA_zQl_^EXabn zH04=-DxkY@h={jwR~$WP(;5k44wAul-jfY&Vt9SW&f5HxTk|zvtw?OZ!y(poCBJPj z|N0SvGzT8iQ-2zZ`|Nb%n_RKpmq>*Qt7qBj+xNX4_=KsbQ6a+k5IY!j59Y}2jL?-= z#~9R3$?Nwa%=9yY-%jN9l^iax%=dK?ZMb{p=fl*7z6Ra76=Ci!AG6>*%>m@bM|Gjw zy-l~n*o+Uxf#^Q!2L}G0@ylslRG8rLiu9Tf;VYm#|3>#GH*=ZQx61N#PoZE=xLR8z zvqh>>;Rxt|6&lNFy;QTBE=BZ0v)PXu0G!zE%BW=3TKeeedug=-k9Ospn#W(dtUmOF zjSV-|90b5#-bK(qIs^AL&Yep96oT$Ct7Cf=HVaw13VBg<+4l_g8_1+}%gljAT)4Yn zwIWZtI4FAQu!iVy>+3IMLWS6;0`5;Qb&XKdcWDo&ytm7YycD<)3GC7y;f?huY#D+t|pE6sNwLv2pqpk14tx^($3aSXIlJ+TQ*smy6=(zcO`V@lI9ub@!8gD|RYBYFzx1@Wxe`HJf zgHyN}jAsp2BCb(djGyC`lDJzON8ZuU(xw9D z1ZK}s4#l9#6I28PO>NuN!VJtrH*>>`TJi3q!)7q8Y_9PV^NM+MKzYqtg?!XZY) z@-+qRIsEI_sdMYKEiiP(Usw5Su-NUA(L7p=y1}IIqeaWQs{Kik{@$&TMfkH3DCFaO>oVkPZX|o1B|H=NtB7C@@Mq zuOQAcRFv`$!B8TF-WRH}34W}_c5*!nv$58%d)4RjF$I8tIasR@OnVkNpe=m9s3*E;1MA8!{PlC_1k+N$!Tr`54NC%JApUa1a}k?7V&*(;`P z1V-<@v29}1U1YP`J3+i~U4xQp0(G{BcRHd_Vtt_xCRZ`bXd$Q`euHvxTxME^{9=<| z^27rJ%ImGOJGL3np(UautrOg8dx17CpDDOc_hR>GPjC0XNd+!%al8afF`x;M6Scow z-g>)nDxG(y9(tbaZ-%lOG|~6vjXZFcv6}Q)+1EvO+BA6TgfJ*GKx0^Hp$Z$Fu|O-B za$CINB^OHUQKt1;zqdxuh`)-8Fo#=>e+rfxqUaOes7N)~!OaZ7W-sno2>K*;=|D?e zLHZ%hJpqdk*xV+DX@X`LV~KnoBIUP8S3UMuN?|&dHzQOl9#z6G+IjzVFkl2L zUr-q-Nh)CfIX9&*?|txu%cMX&(sJ-qA+5oOtL4Q*NtT1KOgobmA;U4LnrL|4sW8kr zvrwn6vNyM{%#MC+6bz@A?i}r2@;dniv8KUpr03Bt zpvEh>7wW_cYAldSIkC_I3g-h6|4Y%9=B*3?R7 zGTE)JS6g*5c-LTXJ+RguDi@(Y0KMqvlH0-!k1JV*8u{*cg2Ggw2qZ9O9A6dFzPP^g zv>}c=Q5v-1_Dkv(NBq$L_TicB<_w+LSUG8huk2TYV1gJP(4SQ%o=1A%=`FC?P!~AU z9pK)*5uWVQpV7(V2opG}9TatoW_LeUg4Q|jH2eGV7PkOT@*L=oEcWwYS?rSYFvOsE z=nD;+bfh~?~Q8gG;lrF3Rgwy%dZJD^c>8yM?YU;rPOc;`z3 zk-5`$6UIR58u-h8>7qu5R_C-zRA|CSW7}5 zP$E8-hX;Hlkx0H2yT)T5;dZpQ!rVdZ${~kUKBD^AwB?a{*`X#>0y8^TZ>KW!%8V-N zQKszEC1qcdu1Hs>?Ixz5z1zP%NySjuBK3~0GYjnXG7h{8q%H@<^X>?^?JkkscMLQr zcW_B8x73haiC`jImRCiugQaGINQf#kgjmR9apQ#(fMb?wrKZ=TH+z-jy{2|Uo%2r{ zxL~@BT4^M_piL66Q@ntkG6=@hzpK7?hmo;k5q_iUwC-p?KT)nzv#j@Z&PiZ~dlp-J zv*lcq|Lwj?b)0y~I@c;&R;RxG%BNZa^qw%fYY*^WJvej48Uj}=QLwSr+)rDn4=c*X zh7K#-7J6nsJEG^pRdohNUe0pUenUcji0^j87f$oXdW-cNE1EC?kOtlsp0~w3ml$%l ze0LJUrNZMU=a+sI0hm0*NB**~EmJa7Bf)jc#%Wi*2IoGvySt$Km*Yb)viBYmvyR6j z?%BBw!jrePSX*NCBFv-U1AbdDcdEkdGGC1j}uUH7fow(W1uQ1*G%n$B< z+lO;&2aXMuQwEJ~Wlfm3#Gkt9Cf#+AuRBB+LLNHM`EJ6<_x83&%T@}yo0YCtC+VJ<_s+&M^AWbpT|Qm~HZb_eJ{hhQTf;j3`zS-y$NO%w8|RL6lv8cPk% z@RqnP_-Nkcw++ne3j{04av>+YkAI%WXTB`ti^E^s|3NRQL!uA_Qq#pD?rw$J(7SB@ z@bP%}$m}{^DPeb6-QFLk7|jkOUAjSaNJblZS)|{4cJqHiUwONWW%8Cb&6D#GghPVt zXOE#Z@>>-iUpJqrxDWAC{h*ww+y+^86^(N7f)JgEhnn|~_9%n4o2p*MD{V3FaBe+m zn)ICsu+R>e_OCG!hgF$=*T7pyjD;&(K9^)Wi!Jx9=C7y>aOa&^WwZ6vJZ!pEf2DhW|+<##TKvh;P(B%h_>J~(uI8F zXOCJ-gj1X6HQUt&r=v!J#BNnuUwblzXJ6f-UU=-rRk7}*k{&+Wd|L#UuEJ_}b`b77 z%it;#d*m;ZUf2POXjfLdP=NZ-E#A|MQ{Kw@W+o{uC34 zS|nb$Q>hN}!=bcr_%nl+B(IdK1-B@B+)w67qPblHj<>t1X>OZYaRZkK7fzDu&iz!P z2dYas%AXzU>);Z>Ul|B>^HyO{ae+g}jNoX52^3Tb6AKka0Q9<|jRi(0)BDzcJoQF# zxc-($hvA2CJn!nnuz}+;1E{9+G zl7DjnYj)-PZH2eiz}v9lPg91W(l9oxWZjPS8g9zZjXmCJkzXyc+naj23>O`UPmg*d z4Rqcs@c0_)p|Vcz^=TN&f@??rvG{7<{qwb0C763uB-veO)8(M4&+b z$M`e9>50qH)=>4PHLnQyqI3q1V3b24N*~ekL^AZn_+YI{Tj?(CIy!YWQHh@jo8mXz zV^k>ZT~D>cFI(q_T8QZ)S<=~ek9PJFLh^#^I*SX@iZ#nD3*TBP)eriJ8$`PSxQ zm9MIiQPBSR!iD|F3Nkfy`TWPTKV;l^)LzNo!^I#buWc%h;+7OBdS^`TJ}Sb; zP^!2SV$Dl!Cx$dVwmm49q_NV9*>e|Ne~>r^nu2Gq&8n9e9r{I641zv`EYTcJA;2CR zO0K=)Nx^TN_wsO)zRF>x-WOZRc$25hTxz%=eijqPtMqE2HD_?L&08^pwBoIwty=z@ zoXOP@5eAie@XzBTp_A@M+8_KKhlzU%+OpjFwC1U`y|3K!Sbrah$f&!Ll*sKCvwC`D z9nET4o%M2WND{N3)3Wc!{^Y$cS5p|&bc~vHD;<3t6p?3OV#&l%SCw>rfm!VyrA*W! ztEZ0TtL>l9oIF;sifsqYy=Q|2Jp{}~QmXhgYa;v}7w0o;o`kofIO`6Uc$`l59qd?@ z<6KrUhB#0Qk4tJpxpv=f9^#@dn_C@88ckRqk3QIX6C5p26wNUB6rJ#{+dYqyvTt>l z`3upO2j7$8MKBRs$Jby2M+TujKUx9iO)%F5`@4->z~@W9uL)e(FDH9f)yLmgMCIve zf!efIV5#=@uV1NSbg)|Cm3m9H<=rJ0Dx_Ibe)mwZ=Q6&Kb)87Yn?(|t(U}q%1jF|6 z18s%_CPz>vtjzPHR;|co{w+(Y!`=1%OgZU}_*|5(L_y{wKcU5EVi5-b?^$dFkdI1k zXFQ9W$(=f=$0O?*S->HANc;W@v3)cX%nTDkoB+Jpui-G7Y{m2t0=iGq88XoX4D0q2 zBP9!l!|M|~D=znGgfxtr_T~|tpTc=8>T>K32U-=X5$K0)X(HaZ?EvDZOH6-N_&mjb z`BGQ&T438uAzC-{&A>C;ZKm;4BhyT;9Z`_SR0HVbUN?u_lkSQOrm!Tbc&@)9ztBx7 zHs#Flt!@B4T~zz|l(|C1Q*PJ&kLB$#biOwpDyS73cCF^@Ycto|FNGW%kfnm`Z;5rS zDg;Y)?Dk3lZ^H*{Dm_9Fb4jMcZ++Yzh zNp>@Yu(N8XEZdY7wcXOKNT4G;+DiAO6W?7e_|(Qu!{!)ok~kbFr;^J+4GxnzO;u7GF&%*e#NUG>7QD^?>T|?(LMVc2Y6}S$9 zk%VFl!Wa8#C`1Fc$B_d)e-x_foA|oTHGN zDw278BPW%VH8X-ijwIN8M@qu+-Tdq8I?7WorsmiY6EY?es1OJx5(Kaw0fIic0S>;G4!=3kgw%oNj}XGB@l+S!d}M{U z%jR=(@#}p>Pn#M+hwR&bwuyo|i5+D$z8(v$I?{OZqpsw#m4jT-%dJ=WDGC`<0+(_2bf;>|2ysZ5gINp&gC3jp zwX_vRN{*JA<>$`uyVn@!U%mNQl=)O|^+;hpxoxv8trT! z3EAcDL!0eGOKuVe<*vBeI}5Vfy|;}zqjidxPM2tvV_tPLY{_ST6Ax&~^^cK~k?{+2 zPYSD@sENG6r13p@7i7h^&d}+$`9+t^@2-t)DL_fmAcdS{ZD|AaVD$qu-R?AC^%!9F zv|#mvq)++(TD|=F>YGb^-Jbj*m5uPT)k`Yi{k8fN!n4(TxGbu7CMPHJ7^`4`WIOdT z&-?or*hzSG2FB$Sg<54Apqe{d7hh@e%8eFgfD$7HvK86^Je}K^rm+R`_A(w`B#Hmj z0mVf1mb1+Q6@id-c}esMm8a0>%a0yakg=fIyjkl=ufJHLixyg$94@2#Xg4fF3%)Ss1D5&*U&Nyc{9`V_hT!@m8kTBOc>W!i&%Hb5B72?)8 zi-`jI=t4uY*|+?2p7=ryW&i9A;^z0a3IfE@^%rMibTI1y4?Jnw*RNk`X?b`wuzCnZ ztv`s&xKUy!?SXWlb1>3CcD9vfFCj9!xIhHjpC>fLq5TtHh+DvdUJ&6UZ}1*FGAw7W zKZoim^yEy0(GN7IshRjk;`y*=q?rbB9McG)Ks~gwiOwH+%na;$eHp5N({6T+i|t_R zoJ;xI1$=(-3u3Pa?*ZOcv_Hg!g62bXhJ=caf`6=uh=@24@puZ{y(^Z7L!ikrMxPJF zjpWKRi*p*C;zhmmlUN$f#uXwfPx$8{LR)}g&J68V__%)H1s-#y2Z0nw#K60YU3u1@ zsr0?!6DN(rPm!uN+tj_iy+s(^E^WqjwirFJ@TeY|^PAt0ISZEh2$6Mnkw{%!dz#EY zTT5W=7To=PrQrV90DMT`mcR1ntWms#&ZvPx0)6ncnp)J&qU{v69;|A%Qja|}jGjp- zxhC+`pY1_N?@ZnuU;ci1)k$LIf*pSn|Ff%~0&Oxew_17-mf?aw3nu5UV78hrS zuQ%Ss{fE{8JUYPqg@wXCMF;x(ml=ZD6^nF_A3yGKOF`D)Q+b{|?D_GV6J)3XQP|$Z z{?5|4Btd`iOqp{=dglD5hv^T0p(+A+%-Q%NiSVyQ(cS}e?(!yH=1q9{RfO0LH%34R zRO^{{0htpr4xXQ!=j7SReFU=hC|eBu;^V@M|8VSET~Vs$-@O3#X;c0>Tz3+nv?2Pp zE@)PWs`nSgJBzzM7j|^4d?zX{-W=iv!-_%8d4qC@@}-~g{1Rw2&N7pOgyKXKrIaqt zD98`9<19PZg;k*S2m7D>d5PjYvXXmDO>|)pP)$M3k0KTMjZT6%1eQQt8=n$~7JI%#W;=f}0XN%gDG(&$qxS=xM?hm_xy=~*1k)Y{) zi`rV;Bs_Z))6;c;K&vXKr@i36VAD?o&nGqEKZ-<@?nZ&u z{#f>tAVDlFvFUEND2emA{vct-)D0?60j|K? zP_gryy^glXWb!3h#Cz^=XGP(&o^Wf}eZtI3f8Gs_Yy3K>u}#U^w2gUehH&H#dIYp# zw%phSix?AwREVFcB9nj7)MxA{vvG<=gA{_UT%0TRXBj^qcHufeebrPLNS~iEb{>$) z#HS@D-BLj)UY5u{;#_)D!)>Cx+z+tg?eqp@1F3sG9g009lsNr{63dQ4{k{*jBL)I7ghp(*h5 zX*VpUQvtJgp(9$6)v!f~3hBYuadDz)RH!>oPe7CYc`%v~1a7X5EaV^ije(?(3))Jv z2ZV4Rc~$T9;&R>yH*|oDILy9^Fu#NDMiux5Z}hGw4C|xPZykwx9Y{UFT`=3Rwf^+v z0cZtuYE5l-al+VXuTY8j5;?7Pd_I1B^d4jOwIbOip5LDa1`;cH>s?+dd#nrk1Jy#t zw<~y;^h1~wK!80BebJv5p|2fe$e%==h>Z{CT>UXKyimZ;l$1!|&$nIbqpPx+{gB1N zbD5qR5yXs zy2`?5v;)LWr24J3gR(S6xLN@Z$l2Q7I+q-%lt56yX(Vzkp!i?H^lu?_@$3pThGZik z?dZ^{>S!L&re|dY>SsZ4*qqBf62BBNqh0#6dP*zD;>H7JeE6425u@3 zi0G=$AcL&z+uw{z1FS_i3`zF?!!uQcj1pCow}A^GofQH;-s$H0M*&w7Q+dM5uKlek z|7F(=VAJT(7<&|GKpNiVO-r1C2wDID=r6PRpKs2S643D;I)m~BN&d^WJv|jsiP#E( zQv!fYnxT)*nd$1L8epl_qE^pHt=kr*J%g;ZFzqLyr-w8S5Ods@2S5MB0>vc#hL6$dL~L|<&))p z-;fWuv4nf_f4gTcDCR$Y=XxI>pn7KR(7K0ci=nVM;`1#waDw$7H3e(@+eR;_vjHk( zYMS-^`}dEyNvdq-sQ*d*qBwV9{Ks|t$JajD{A{*T7xrKc}T>tBLK?pQpJy*-0^9ZKoB$0-UA2@%XUIWeZ zQ^!sNKYI;iTK?ND{5b@L2+zOm1}yj&dQSnzK=N}dhbkQhUnsG!8&tLbw|DH7Mu?4r zLrC*l2WZ?hi--tUCSYR~KF!8Ee?o{9(7cjY!?+3MF1d<4}JIH!(HXE(pix+x%yF3`J9V`(#}N1eNO)i-SRI&A}9s( zCF!F7y+bv$#kYG}0OS1sJVqjH;PD7rTwGKu?xts|Wd<+-78x_}{+voA?*k8jg85&b zjsKN=fR4GZ+0eqv{Nlw6f)v)C+o4%u2$?VR;2|62Y|eh?A0`IJz)kqChxmsWsIoEg zHn#w_%)Ygw${U*56?DJf*ip+Z{nNa|NOV8FORc{;4sg{B(TLQaKeZtq5b63mwZMN{ z?}du1Bf!3PZBQB0?b0FW{v?@O9>3m}Zg9lWmn{-oGdje0`?`?zS^n$qH)w)gK#5ZM z{)UkM`}NU^2^l5hRbNFg=; z7v6Y_a!AD`VSoMnAmV^&3J-h#{$k1h_T7{?>NzKeA>3a+=v9_W;1S7?iJdN`ur6+; zR|@fNH)e(O-N(yAp0D}TA($iwZ58lr zEm+6S<}+v)e9|$d)S@g5@*xI+kCeBRX(Zbq_ zSE$49FC2GKU_yd3A;pdA-!%ApL;oW^2(c19mt6Sv^(!OkVrN|-z@7q6z?X8C*_f3I ztCl6{%=v-FegY%DocpW>xIP_*$D9=XqG=ok6-(!s-~9~kG3WhdUH=dFez6wWf2dF3 z`^U6EhhmtLFLn~djSGCSG>BlYFtBQ8_Bds*faFCY>G5cPZ`~k)BT>qUC$c z_oX=RA5(zp2Y_^VyEGDi*-!t%Y}gd2;chzby9k9S7ox%+sy=02`cda^zua@TOJkcK zRDioSe;-X9>#dm2om6Q#jGt_MZ$Xmp%kSI~#}TT;tggHqkxMiE0;A96of!3*Ui3;L z1R7)!mz4asmHe;TlFAfQ>5}tVY>8Zwl0bhn9vC|-BB5I7i``na_ivYrw|0Fap6XER|%0E#3CojU@I==J?>t78xyq^4NO^69eqh>8P~$w~cmj7ys$IRz;!4j$FxKpzR}j3e2xY8U?zN z8WM-Lm};PJv{tUTwN^+!ANY2`Uz4^HX1#LDVM09mtNPs174G?^ttwi^#~UPXj@|e1 zqRUSVs%&R-%3^(-WfFPfOABEd6t$T$Da$P1nmDatVfeF2Penzel6f;$-#XjWywwCi@K)I=ae{AzehON-~84=(UoZUI>*;6yIB``^QMHk^0f-qL52ay(%j z6@#;S^R8tY5PO?Lv6jNN*h#8hTPyB@baumJAs*dY z{ofhz>l{z6hA_Kmv_{h8&jreu7{<#P8K&0XZ8Wd-X=+4alAc$;|_IS>g&iP+!Wm!E2y8Vb?LkJ?^}~lk#oV` zO!XFZ_E&bRamfW3C#vhIO}Y}oe|E-5fHVN~YU$t26x={drMkKHSk5G5vJD2lg!>r) zsyAq(_^)M^o(gI>hQ=vpF*(r8!+0)npsWbVV;%dNm=;tGR($e#;4w4qKzeXsHDa@v zQgF{fG;1IyO7oUVCv#OvhK**`sQWZT+-Pu!&~{eE@1bsv-rc3`QMZuxw`&s|Q%}A6 zBfBI{8%10roCK`;83>U*hv>B8O{+Gln2p=A+SRS)lkWX)`6@e?ogm*oCoMu0UfVWx zsI$(_s#mw36@)Gm=`N8hr-7VX%@g2gaXX?cHtP?x-AMEhod`~PHT^^(!`>|W#E`w> z$QktWRj;v%e|b79=fmTs0RqH~U}$zI10FwZ_ygrqIL|gV75`>k%d1ZqbhPjxto zyL9_U7JcX2sT%!Hlf@#OSL~H?)e?5g!Pe+`zshbyU&$H5lSEG|Z>;l$x7_e5r~QCo}WiJO*jm2Ua*g9E)| z(W=+T>i)HMel?X#w;&uX@ed;VcNl>jwF8LAx@-1H;lD`E3*ashaAx2T00#i5ukOsk zG>%^_={P+Y`{_6_UhJS*YXnBHC>6QZ>SOdY(`>fCDfAF{I9oqXzkGsqV3!vDrBh1s zi`Ej^D9`JkF}1uN6{|e1tuLyaZBO}0=hsx>p{^tSJbAqKTRB(bYD5AO>WneWeXqV$ zP@r(3@)jZW;5)Egu|3!tER>Zy|4K; zr&UXIi3y!`0v{@9t|PB|2~3<6e2&$fJA2yJer;}3oH87L)@@;L(&Ma6N%)GGo~sW* zqtQ=<*&$ic?(A8tRd({^EXCGY9`sL5e{!(3C6pavdvffgUO&w{QUgiUf#~`Wu)2*J z`h47n-yyT$Kg;{(K65oJ*d(j|N80*>1pfLft@tnF9n0;Kvo0kWuyB}e)z_8P$A|0@ zuMU0L#+yu>z<$S^%+@p(Iqp7Hi-;`~QQ~k?4Q;*L zm=$V*DgOnrt7tEihlt;ad^zDF%bFP&bUB{VFESri=c@nHV|=efMq7}EMcEN*>Ah?@ z7p#xx1SP*sHRa!cAZ42eX-+$L&vTxgORDl${Q^x|LFAnl)U6b6tr(Th@3J8;T@5K) z8IlGSVJ%Z$=&P!vBxC;%Wp5r2_1?daMr3dx>*r?O?=cVkE;5g~*u z+1JLtkEvAlosn&j-Pnz72J?N*eLDB&{{9~K=X?77an8f((Rqw{zh3X>ay_r>dVcF@ zKE~zxl|P~GEY<3Id4NYtsDa zl!lpPly7HtO}TxWjczh24XG+L7q{?Fh&b78M?_ML37U&nMxh_xv0$|1zg*#!vnuER znL*#*tQB7Wx-BGN!_N#g_>4io?A^R2^k(4RY63iPUqw-|4y=nl)>;2jkwqLo?98{C zFX}kb4!Ga*Un*>JR_J0n!GPO(D=xRfVYim>irlbl{%{Tm^jxz#3b!2`Hg#kG!#^Xv z7XPuvvT9Ck^ocdUoY%oCkepQ_etmx4oh|xB-^Njcqz=%9AYm;Am|poa>Fu(e1_Dj~gCsrI z@|)%w`^f?O-45U3T1QMnVt1-)o_eAbry0;0+V-RuCmH$|R7mU9$F=P3p~tJ8Z`u#$ z^!G#}%y?7p9AD4R(o_R_xtw?IT_&{-HyDL;e1D=w>AcO5rIvz#|D}+Xl~uJLX4A=J zJdJLD_9D`HAY2z?8$q7QcI)Y@Aiy_ycRPBvi;+zN{qixcut0qWxTD0^SIYXBaDRLO z7>c9=Ndm_qpfZy9m{kI<1s#Fp^&h>?Ki`oDM8pK70Dz~Zra907; z2g7in=x~hV(PCX5`uL{Bhon!+?EurBGY{Xo5%7u7;k50C`{vkihnLOq1ej}-hrKjPCqhz+cdUa zuFMv@uM98dJGA|-$|xC>lm43!m6=|9BJRGO#x zy+lbq)RC)pNfpq=ekCx=J&r4o$MGkNbauEVj$=KK4#@q>zjQVjL;0=o7919ueAfi&%s->d`F zVjn}CUh#qmR$8M#8V%y6G$C#-8uiirdE9Ia5e>fm?W}l|D6WV5qC9?rlFB=8rExU; z+5W}1Kr!g`^WCWcGU^Q3La`;~`k=L%p+HWUm;a1KzPEDgw+_!^vG&q&Az2uKDXS7pV4;byQY|ZKYde;{Yd2)xP-|g*O?WbsFi1 zI(fbCZ;)=(D;F8UxC^i8sC;?$<0yXE%O803IFI`d8TaYqyd6f~6~)L2BuAavnQZip z>?k|gpCYq9==-v*x69)VL9!ub3sy0_A?BoEhM)Abi=;7ct$WkWeO4!K@EMj~N`RYd z0Fiw>^%{%dhjQfndbuk;RI(Am#Ib`*(VzN8NrDH8{ejX}<{AfV(@vr^exkBw5t(dz z(-eIUzw#vHvBDW+24;rS{;cw_^|P538{y&MksYrrN81dbYb&A&GATn|Dq*KJaT)EBkK zq5~h~>B>NDr)if1=5UaOb+*rP?<;;jDIAa zRUt@O(MM`j7X7JB6J?>8)su#ng%weUPNb#xf^Rte4?0J-A&-Q9I20kfy~IHyb&=rF zq7X{LQN8mAO@RUsf;I9*zs;leq#!*l^j?`=;1vXboZhO>9gqcVvPpup4I zuCtcrts!%f{7H)U3e%!r6hW0PZMiRDI*0<6L~mYS6yOKe`F0Z;9JQs?`gp!KDN9o1 z1@d@T!{Aa8#5l!e5g_2ryJ@m43z{~=AJL4tfDMUGSm)laO8>tM-tTeuHr zfFPg|$LnQckmi}-`XvybiQ2cq*MJ4m7fY?7~*izGAFQCY}bb^it(m?k@Q>@wH@Ue zZgY2AkSjda8#Rjg{#8kMFN{1Op(Esz@W((YB_$=5$$-SwCFk*~IOI)JalMZfCB;tY z!94Z2y(^7NAtwn_{0!)ua(XjB9C95#wVqvT+HgSGo`yqA`?`Xdz1d8W4sQR+i&IGX z;NiBfhZ!oN#E0I2NH+7v-984+f<-os!-qg2>})>2DegVyl`OMQ2j?#bM;K>6XxCOU z>gf$h7Pg6FNQd#gprjuP#g10kCdA}vBsTe9IJu(Vs?B&`DJ>CrPcBPQ7f_N~1n%0{ zk2fR~3Z5C1>H(Y*H`=l@eNy~$|4H2chqGj#CL^NvwExx<*(1LZj)j?tK_fI4{ExO%XdWv$6}buduxtrMf}cl0DKGAe4zl;0ROTk2JjX^=ay zZF{;kYPsm=sG3;rTAk(m*96mTzhxHTG8GjbQyv%XGO|8Pcz}L-KweD`>iFTjs&Mec zWtjYG*a%PXEdyT0^NN;1{P~ubb^NJ)ukUN5{EI_wR!?Tr!Ko(S%nJpjB%H?oP1JfM;=n9DaE40) zg01RGmT0+}jsG2IIiV*(6;Hn`Qfv6r$xr2ZtJx10<+ki0c{&kTllx6ls&a1vwVz8a z5NzW{3TJbpQ;XY2h%nH#pI6N-Hm(8=Ma)P}gw03PNb|?I?E?W~`@y_nl~{$S+nDmb(b(>~Fao~HOK>0CyGL5>dl{i2 zmnfL1k8(606F+}tUl4$P_A)-jW^mTEy07h(UiM$-X4C1pggqU?tINx;xk8TeoyL7N zx2@GUbQ=?rF!o=sbR7I@uYuNdk`?nDX7R$%OBR)p?*#Rb=(Sa`heX}ZW8OoN78?V` zOz^$e#}2hS=^*vKzCe*_h+ac?0GZ0|r<-v@BEs+Ny54J-`bdnrCV>L7NZuyK(0EP_ z1zfJx5&Zhx5{cQH2VD_DBq(d6Z%@S><|&g?VVMi`6eD9Sipj#m!wYzBzcm zRn%u0xM^f%Ae_{Re4pRl`_8OIgO<^|Loja>^=sojZfanv7459K+q+zcNaA z%)L5lKBj@J>IIrhdfyW65A>xT=d}e1mG~MkQ4xGR@%=gZzsF|(Tz_xohX87#=(8yA zGXeA@_H67cy)QTNTmVF0Olk@Yj_49XEo`z38JCL_-n>HzCcV|0`A98QDBM@9j#=kU z@CS6O+6)&|bgZ`JYAn@wHm5YdBDa?S&-WdhGQnS!huL;9Dj6@rf($am=-)gQf3R)W zx!BFU)gb`HD+Kt^;^d?t7W6_fz=W}n*!9^6FncIw-|M&f5z&$U$>PJ%J8li8!Yw^- z|N6&}8x|$+2S51@h9{Pew||;!B$zS$tEt6043*pjo65>%YBTRJX(3E4{{k9|2 zcIQ*&ecEGI3Fu*Bi^ndf+RL-c!)4EG%Y2e*W8vdLhSy^d1uL zw&vr(%;iC2vU!kkZmTzB$RRE#!mX=mW9x@6MNc{|qJZ6AR3lYbr6ci%r`;Pn8Wahv zkutNkrY^9T)p+y!HRIY7D1m|xH!`2r-LJRH1*i?D^u(4XF$(k1I1(7wOv$##a)bxL zog%7L{2P#!p!N96OG+gSD>&9sBwM&mXqYZ_r>cbeB#vYU>{=!QkxkpccgqLB;h10y z-Cq1ARnod>YIAab+b1I!`z&{u50Mzf$RfD<;PojiFk(vrhHOZj6_hB%nQYCL5Nim| zRmVW!=dA(sassd>q@v4Y1D7th##@hh)+)y@!hx72q2<2%;#mEYDp0#?T)KH+MmE^W zxAY`5fCi1)(a>fPD~J@h1lg*eYfGP0vcCir56NIiCpOrhDfmyx@!ta}PACz5I1}iV zuQ)EO?jTzpHcUVnBlI6NQsnFp=MXOr-?kY$XPQ9LH-7!-PNhw+F{7kwk{SH`&Ex^4 z2c0RT^P?U9QPaO@O;dUOPQnpc)>Gx39vF;hdj?18MUjejxfA&u>a5yJQ9{I9h{O51 zE)&QD0Sh?SvQD@P{>Q#r?uBTgJ>+(C5X*1aB}u=b6pb=dvX+-od+A>Lg6{ezf@-Q= z3W7slIE~$n1vXk-4<%1{!Fw%31x}iPs~kS*e_~v9pjHovBM}Q|BgtU({zO|CqMN&PaLL zf02uyU;Q4Kmst%gFLgH|0CU0ZtlPew<#F;nYUG$2c@AR`j)<|04fXVCi|`!T&H7qD zs20|`(>a^#xy0`0C}9}GdvFpir`4Io7TxiMF7eziFvN)c(ygY_FTHH;yf8Q~w~jN; z`-I>4MVVo+o>f6TJl@9fozv9>f68^LEEmfwq|B+7Oak&4E6gcDW-?*Pzvr+?30`m8&D4U789I5GBkY zl=#%f!E)7icS=o6Qe*a)B7yGw?z}3!%~vw3+3LsUZosa_erz-3Q;xhM_WllyskX;^lmj-Df}i$cB^60HNr}!YF0X46eP`Fdcvrh2XLhM)HhC(;4msHfpT+vP%I!4OrKKJA zLQi(m&Dmb!dR$p$2bAL2Eu4xwf=q5W=)diKD;#QFc5kaN?niv5ffCl!;OEIiLSw~b z(IdoR2BmG_B)7{4BkrK79J4!>aDm9z%c1zw%hHYcX7J-W)d1@a5v8m-;i9cu5zJC9 zb-ugPUrygJ{8$0R^U6AT>Nh$s3BFZ~xbULRQi`RN`KZBLw`}}{s5~s~WDn=F+*cU6 z4E>oOmfKYf6W^mB5Q@qWHwHL{eGPiDIA5>Wy?mUdr0r${JD$!suqVJYsa`Bv(`N3M?0k9p{{Wl+?^@?oJ8`*H>z(jLQTnCR=~1t! z`w@DW7a@(CulI$n%DXDy1J@JzQC>QCj@NihjAe6lf04M)4KEI=Od3|Ui7-B+8GTvR z+gD>+1IgvbS?NC&qtwNj2pYtjC>4j>oBA?qq{;&CnEH8^cwXzbow^CG9Ui~db3v!{{h~SOK~zWD#D4Y z?3F~0isWl%B--9IX$NMR-9lX%BD#S8ZU>_tbbMptJr8zu<;5MBNbJmkQ?A%cO^6d# zqELo{ZLM*dcBK+uW32skHq$d~>pqBfZ zp2e!hj+#aK`eZ=R^~GLXvXa`R?xe0{;n8A+qXrwaL2-sf9>nd-9&k2j&u@8|>NEQz zI`*0pMe1QocuX5^P3l4To^!54LWj$KaYlykcvXT-;C}LMu^~TUcSM78CooLhD= zq-V1w$zK$@eq z;R|vuPAYc#xxI<9=kOO(rix+ZYBx|ATe|FS9&FGC=D`@_B*N_Z}Idhkt4`+pU0&ydT2 z7pnQmNYgJVLk|FofCc@OsR729yi3@(O~Mg!p+898G!cD)(0I{jB?k})@BdVcAaCyCYP91k3B!c zpWxUCiuNoHOh75?#U?JG>%JyS0>Oa-oy1> zb)zUOwbm)2Ipf@0&=E2BZdpR?T_pPw*!GVVoAEQsgP%NdUk?<_n8tA^`|MaiStJ7V znlB&c))OgAjTBgD1njPUs#FJ}5>jH4CHOR_km4WXyHDPSkoVzmxJdIj*_T%=qvSrk zzntmoJManBB$(tnk))?GtMZz2rkG{@6aJNv&kR;+xzajhqA8H?S_9>favVv7&7-vD zxI4xv^60OP^2KkU6N_{b!M~no^HF8u>0M7lCpyzR7>54FIw8pF_hu7V&r2&u36o-`nH5f*WBBJjav{xc>>+ zPZ|u^5y8g|%L?PmR}{KaMPv6(5ZSIRGGH&_nZu>o=IG$tbyHhmKGFMoQD^!WZyG&C zRPz?r#@906aclBTk0?3?U-z8J`ko~wARTmYU{4beJZNmYcziV1pRG)=KP%KN&Qnj9 z>^>({LKHChvv;L(xY3zqksCdzrS{28LWBMUEP+pEK38Sl;I(YMlC!m-KZVXAIub&P zGu$-B`i1W5%CRjE=F%f58Ro&1BznG={yO_?Gt+EM10wr8kc3ZbgHw`O@w#>S&rhi; zq)-+{zf`)p!U1WSOCxT+khz~&xwE)sF6^*om zaj`qyT&k$!JhBw;LfnU!o$F1~YVcWK(t%ir)wG}`*^sB;F2(U@3K6Vg+9yY7LEVU> zt$|j#orQL*ugglWFZx(GEvyM~s=j}SEU|b@JVnWHQ|OChL%=gW- z#0lG|wma)n(C`keHkm7SqoMCLKSMf8+a>$%4@PR3lG zdFofGn=R|yzF>nsRP?No>PE<#B1;HBb9&Z=c-glsky?aITsYYM<&Q-dw57z+6c_`e zz5bSP8U+@MD5Q4fjy!m7r(^xTJ);~E?0COBJ7!8@~#1woGC*LsO{?!R6-m|8ljhoDA!#9M*3T1X}26hvJz zm4)(;6I%MsO!_5nVWNQIv+$pk(t$st1Ts$%VMDH^@1KBb?r?3BZTongZN9Ib;|ujI zyO*mvpX7~iB+e>DFsDujl^NkJRYGJrRjUa7hBVMZ!|l=;?*C27{>P3{Uj3q5q5Q_i zhTz___kLwUhwsn!1&)luJ@KE+>Vog~ST?c;Ba1bc<8l+c+Dqk-j$i%Ao6sTBkE+0Z zp60wo5N@zXfkSFJ+iLrdpZ0HhCanX}O-VK$%@3{gOA2sNWDfLq=N6_viluIXZcBYvN8y3B zg?S)F+CSS_Pd0iT5*hOHN3GN7lY1Caii>YQNO(4s>?eqoKly%Pra4ro3q1?SWhQyj{244cvjMiq z3k!6XpYFlzQYbJMC;mM08}mGGXZ|9xPPc&9334#!x%4Mz`CR+aaFPBm@B-@>8{}ex zE>n$miIG-O9Ip%)bNQ_P)G0L)%~0&K8o>v@KLzhuBIe$pW0&86EJmIGE%W?r1+G%i zg+IHlRm5N`@~u?hPR>!4I>1Vl!LYfz5wkGAO9R5@Q875EPA&G9zbpwphl6v%ZMHA= z{YeNun;ilp3u$TlVSDy6QM24Na1_^oA@dX&F$NWUhEeXXC*I5lKEk(GM)^;O`0_@i zXHSLeN3YS+&fs?Nr{D+#oE*Ta zz#VrZ=D`PXcTOb8R~jMdw8v03YS$DF`3()b$nw7>eU;#&?5Gz0XnYJ2Y`3OPW(;b}f=5>73K( zsf?fHR?OGH>%70u5cnm2`d;M;vUJ>IA7!4T5yzuut2))lKd12?9A0>GG}Z6loIM5n zxAG48qy7KoR(?n;fAF^#z`x@>0g?3kNH|#d<~yJ^yYSa-5fx)YgZDn>8izD&F<^SS z*Lvnxu0X!|egZHJDb)Me1b`u;JTLg!JkA*rZwoMZ56C!IrkP~-2YB%>KjLc(h3E{EWbEUF#qTZvUWI3MG{g+~_ z=<~B1wXNY0>$O_6&F1H39k|2%*$ug>D;F|p2p8qA5B{tR5?f$L@*|f1tC^wx@a$3Q z9_9JVtGD2&iIN@AEBzb1{qH}8h>;%zocfvCLvOV!O=O8b`I#JHWbkxW+xi9f>Z#*S zuGj6~+^xFu<#ln)cU%%R&E;-ygi&NUzURE1uvKjd=bKiE-z(OAfJ5%xNpu-0(Pthl zpY45g8uM{7e2*C!aSCEI|C9Rn8<1rR60cVV(H@&QalqOWOA*D9%d5vL-r5g7Wv}iv z{c}mze{&j5G`+NEhev^*rr5flJC-ZxhFY{RU!{5%zEIUj**88u0&PrQoouvBxgDSu z;Dxpp1VpUWcm0;p@iKvXD$kyi&wao4Vm_OuQRN4K-JJoAjfc-5n4{F>j;IaB&Uj;= zO`OwM*e(uK$4Jtv_W%zfIQ}5)8NL`ngpBmT33;0P0&3NW3a9G0Qd+Keh8{sE{#LHKUHz z{j{p-6ZLazwJWLsj%2~TT4E%|oWH3G{e5z@`rsAKR!uqfdmUooYww4m5#| z4Q5%aP0y+eqj$ePcs9=r5j;szmH%$IlOg$Y!=0f=xGKg^Yy3jO>!MZsQT-I6YeAXJsU!7wXCQP>M#()F7vPHn5XavZa!8c84=%B@!gY4 zBW0*3Rq3ZR5%z3$&fkVFubzNBqkXPtJxl*{d{GY>>0ME5<{v1k2r-N3k(MMw)OQ@o z!O5%kh}Mc{EHT%L2OfQ?fVd0DZdG8;?mR5T^LBrw@XU z$LQm(YhSB(+se{|?IteQ+Q+{fDD?!hn*5gg9ZjT>T>2gFDxQo6?KqGvt;Klm3tY+7 zTw^j2J^UGW>+!wHI={jZjfc>`!hua=VONTXr7ZK-3()ZzUg}D7Nz9N5)a=2TyA^tD z{A7P-3t}rR=j88KU;Z5>ZDx#zhtFjVViaw9t^L{l@eBX>)`jGQ)U-Dw(o)xS_wxus zE{0X6!ehq$n_lLKm`R*$7LSxX>}u4()@=-ONCi}_&A-q>-lZ~YYhRO#R_QvH<7 zn_*-+tF#|Gk7l~C{z|n8T=OpJ63h}IklWE@U(`>-IHXmiU&!f$VCu4Lwoacc^)JISvl9%)UY6M2k1!{p1I=zsSb$H+a7QR=p(`#;<>NHxRdfZT*^Ej;G z0=Sm)$BIM5AaHQ0$o(BSh6#bgm`Lgz6;gCEqP$DN@V{KRugoj z9pgs2RINbO#$Um^+?!VWF#060Yc(4LT?5ghtvCR2?#O0A!7YBOjs3f9FCeri`U9&x zk|amR$KAVky;h~N_6lS8D+O6NinOr*JTd#qM@TE1*mAfl<>qnu*PZewGSq#_NaZh| z@yyXb_fo-4pI@1rM3b&f4HhyV@tN|S762Z=s2!j-wPY!|L8bWY158$;aDJO{n zAFN2XJlj_5DXo8QYI7fydk+Svq;N(UL+4lY{dad?b30At`_dGf1g+7ME?Ej3znrM^ z*FA$l->h~p5P{U=wgz8n44`MV=72lBu-_xgLW}B(M%x*S`ysLu&%ORm>iwLc$Mo}~ zdXxcpp(ipvW(4%z-(xW@MkceHFfzq-wo7Vv2#zl8@$ohIIz>cOQiR~?2k8LPv;vW0c*tY@C3KPvhC@g>!nf&Hr_G`((wSFd@Y-<>%ai%?O=Xyn=1l;HIkvrL$A2@G-m7}Uk3v( zrirEY-WnlxjYqS{Dx&xekzME*A*^lf%r^76pgN#vu2>1sX-=1T-B&h3LUCJ!EZG6I;@`pq2>3fHwxW0=ID3&~Cu zPW>#pH!YF`jcau7Ll3LXnuC<4QZfTZM=TRG*{Tx20jJpbgjlq!fHPu8Ll4^dwZMcY zXree=ujFA*Q!VuAE1NG_x!ez+z|3BGTobs`PL7GL+FmV^htB*`0u$E8qvKxI%Cvhf zqno8*!x~cW$4D9C%KQG*Q?jf5#vRS9&G<}-GfTEFPr;clUiz(k3^6=Qo9Qc~VN%`X zIkJw_edCi%ug_~3*Wm4-g3QlI97n!M2XIOfqJ)R_rcssAx{3ZurIFUd6d~u=-zvX$ z_e&gCJ5*K5B`uY|1S|Z#P@-$Ngrc6h~z`|GI)_C^2K{t%bGOM77 zHxi5M)E-~HDpjncimn?-B#KtEmhOX0!Lp$sqCk)Bd)j~LpW5eZrtEnnORh!;4%RzP z?>PY?O;~Yjq7H-L_GW%@$g>GF^SG#ooJ1(HPJPx6MNuTl4ZKwb9k#SXpIwQv&O}}C z;PZvq)|f@%J5us^oqh%|?r43CQ!}XUu`9jR{8H*!%IycxqiwuWTd*xldCTLjm>l4L2Fa_Q%-(+^Ii)UJ1w(a!FFchG*Eu&(G9g$~D)|)aR z@-HO>7kLWIwPBL-anG*h(pVYFi;M(zt{_L5Xnn@E>Sbm~KmI^IC$ra{K1YPOZ8)^aLGug3A&X%-r7 zX|!X8bfarJ!QNU&Ls+{Y!Uc4``>IZSX67)LE3E-(py7k49OK#Rb|^|~5dP?1AP};s z51Ox7mh)XV)5$l`09b8=r*7*}GXysu&IT1ewns=FR|eMdn>6X|txXA*m*8b^t0LTr z7IhErnmgwaVBfghPNkU6UFb>klBIU#X_T!%S z-h_lb^4Ar*9{jJJnb4v>-;GfjD_Ww&5{*S1v zm?f`%w$HNBN8=?8o4^g7oVTDcZP!LG&w;oc3raM#aOx}R1-^4_=MT{^#2jc=@|*z% zp6j>u3rW|r(E7@OwZCT2%NFEUS3rm zyG-7v%yje0ZxmLZlTy(xME061`dUrraA*M^z_ zKDxzq9mU($SBAX6Ql&z3SZHA)s(B5kgFwypH5zDc3^&F&;nsX>Q&+y*0A)?7R99;1 z^dWklS6Yalb@P4*NDZBJ@;*V!iiZnb zX@mkqsC8z-C=4}=I2pjWcQlkC-45I1^FO4%KfHjHV>x$4_|`pG80M=9tn`|d!*Ii! zBS+Fzso*}8UZF3lHPCsy6Zj|sRP012%TH)g|NI@;NTfsOKNnq!wS=4EnBZHwXHjA# zR0#H49-8mGy{a(RlP1M254#)S8*h6t_-IRcwCu4$=2Vj7)wK@>q0&^@=%z*r4B$89 z2pU&>Bbhk82fOPz346H>;HO|@iE+ggZ;)a66D2yu5!%4dDk5awXk-F0DMek$0L78x z$S2Xe!GSNL&0}BW0p9NhTuT27% zbUcy;mQ=#?RnVyHbGfTbFr4V_e(4wJ6<0t-WLfhD*_$%lu(^My_>c_1t{ohY`aqDC zVf)+lY||igiK5R@X*&aYVoZ?HDV3D@8!3EYIx`X}NL8A2Lzlbj_~+ur*YA-7IrfzG z{^&$wy?h@Hdt^>Tj(YM}mkX=tky)M&z`*NP+RQriER7uVav67=KL9C4=<31u$OW}A z{FIQ);{j_r)sZ)VpmGCnrhrRy+tqMtw#1A(PI0A)#-&Wdxt{s0z6~)6fs>nsvq|${|0;T7f2HzRFw7 z7htG1UnYzf1OgX(RzY{!AZWNwb?9=uAJ^X?FJSdX1f=II`ibg}CQVQ+CkR}TdN6fz zT1oIEk|9v#HC#^*v;bu%S05#})Dd(|SVx-=zI31b;pAVMU(pyUyWDY^%q!icp;FFm zAzRD2e`*oPDN=cNEn}`&O^Uzm>_*O5=HTVe1B&mauOiKV1fc(pt;&m&KDieX41R0w z0@Sw-QUCz9zT!8l>zkvsfF67MgPbzK>pa;o{QSGO4G<;ef|?%KTbt-soAZ6aA1=;L zwUg0j0z{{VycZn+@YGlU9noF0LpqQpkJ$cw>Rw9I*y8815NRn~=+4m*Kh8#C*K2)-XC2|D%B;T>gvZ|M< zY-y+qI;mgm+8XP`?9Mi4wB>c1cD(K;TU3e5Xn;>m+BoZhF8uhjDkyH?b4-W(wQG4oD&VCoVa??(M zc3Qs6atm1HEFLY65S4i5#du$$J|4tAJRuvNcLMCRaIz zHICm{IEfoHElG}_En4<_1_2jOkrF#!_L|IrK5%h4oGqBPI~uIz)i>Rjbhv9m>(2iE*D7M-3F@1Lvrhbzh1I~GI=k+J`?;Bu$Ak} zjJ^o&jHjQ)hUze72fx8XiL5*jYTi9=QBIMsn#_Os%@TSnT4-2XqyJdC7B1fB*eWoo z{$jsGOYUZ+ohD_~2^fj%Wc$eYA8dZ~LXR028J0d7TjJ>MFf4f@K+7Z=q=pLIPm>=% zQ_apv6hPIAn9NBwtM||^G94al^d=Ab2s?)1PskX>4|I2ys;Z|-yvSRn$d=F}#T=j! zW_$X;%Ed95vE>HIOsGyB7D_Ci3(y9g;MbJ`?W(Dmc3* z&v&0Ta8_8)1SP3%VreeB8WcJ7m|nw}*7KR2qF^A~E(Ed-a1;U%*ytZ3h3zxwWZ`&1a5CWgfarF_i!`HPwF^ zEfW$O8nyf6fdXbPyA$*E{zgwUTCaOz(%*w2eS}gCz8TfF3`W%i)P}$qe|j+C&ud}4KezmwKF|L zSRuuLt6*(&NZT;C5p#1l&RD6B0 z-KOK)mR8W~P_zFzw$l<$=Xkt@xq7-q2^>q{WKa|t#dWJP9}i^73>kPbjI zlDxrMn|J1oJ_N}_j&}Ba0raXEpc8{4+Clg%kUA#{)?`b}x1X~2nj-jU#MgkgcA|i}L;3`&(bu(g z1uXvRd@p3x~;0)fYL>^2$^0F$}Kc10YJ zC1kc(?);Tfgyh*JPh&drp=7e|zn(8~^kJSXsa(YtQ1nV=n7|v&!cdV({PCdVlr&hL zZ9TKQ!K3uLPF8d6B#g}9eM4v=#-iAkYI$Pobn6|-!PyqncCmN<0hoJsmT8eSmUvPZ z@Tk6fz-w5!b)BM)Qt%1PRHHRbV#W&)tSWq4p6nF_UO`++s^jIu!x!fio4z9 zR{H4k>B-4C_Z5u2=Z*r%+P6H|06Mq zwvZUbl8FGc-zXM|frEFv`l=UrG*_Q^gGNZgNd_XLk#c*W-qSA0+p0A>0pO6T%LBQ& zPrkn<9C*)N@6rkvt=iIGPGMwnJ>*~;%s1Y?#8!*>Z6;$PEcid#0YKZ;4CtZ1wgUsG zZlm2@r<%?8)+hOBN>;-(u{d|C3N%*Id&P(`;+k#!@vI6_KzA6wN!0@aO3em6Du*XO zwICEe+~3&xruy4b`k#e2xKj1;Jb3%L1dufN=hAle7CQh%VTbl|HJHXPdj8!DqIhX1 zi2}R_X^#Qo+kIv1%ZsIK4(W{`3-1q8CB{6uc^b96v6jNcIyn!ndY1C8G3EKDue#OA z?l~pXQ^(kuKV0$JLEbF1$gX+X)~vN*q0>CU+>@GoXv1gFaX=ogDEPr!CqC(Q^K}5n zC+XasfUw4$Iu5z3Fo$Ccz72VaQ3jeO$KBy*$ni;fP%#X zgYTtS)$gc-;Y1{ZU`K&=w)!BDJD6%k_bxOgTgg@sW0Xtt3$&7jzpv&4Pjmv&Qv~K8 z1>WKg(86k5-6YhACr5|#h56~iFKEfAn5@&=SYwH>gv!t0lbLaP4{nvQViqZy?KGcafXGa<>OY2jm_!Oq&r+SXKK=;|>ETaCpX^*0PNKr&7oK_dH+?Je!*`EW=f%M%^V~i@+ul-!1J4_;y=yy6ai<_7 z>uf1Pj;Ev`k|S&8lAZyBg;s&Z#1Qw^3*enX+o`zvyn1Ed8l47aFU<=*SZcPac&6c^ zmxZRKnp0K&e3fh8wk%bmflb9wjBGi7f%4LeLWMj3%m(>-eY%?n3u<*$u+7e@sL-xE zWw`V+P+{-~2M2F=@WCE{tjXeB^{mmr1G=1>lym$ZO6UpO-DJXmV*0`IjMBHy?)l-s!a zvtGBO3H$oo>QryZ2GHyr0bsc4#DsBq3Z|9mOf1mHOooIsS}cHFb6~`9ySUO0YS4V* zQPs&Ycke>x2*nZr`QkZ|*^7XVeV$tm9F3!&g`C<?>!Oh1R9bkMMUMNV_>-g#q(9TybL?Gfw8HR3*RTt-|CUo4K*OEKt zlsnoCGls3L+Hd{(zP*?MeSKzwP|F0CcRD^K3UIz1bl_GWuv-MEjrTi}1UD+uj=90B z)R`ohv}RB)U&=rA$y}C4#h%X`nfgw{DEw_zp*72j zVp%anHb*CYT(goUW#bl6K-$%FGxa=tb>Q#OXommosXEx2~`Ar=N+utXk}BW7weF`L#YxHJiH<*95lCRn>s3z zXUwppB~?SS93@5&Sw)i3us+AUov0b)IqS8KG#3EFyYG>3&eO8Xd?xjBC4&SfuSv4B zt_6a$=iLpR=Y@P%6ueO47xL~_p8qRp|L?u?w+Dm+as63#w;6vDNRY-sU5Te z1mbBd09nYe=2a&>rhH#FYJ+<}Oa?UgcP8F6uc2A#E-|j`y7T=teLScb$sF zxUH--?_Vkz4Cn;y@lw8JcIzqLkBx^xk*im&YO~y5s+PI8UrOtn7C<%Ekx&eR8k_3M zP+_iGEo^^1ltuGPAvjSYN+m@1<~!{icxuI*71(qwp^ z&QyQlv?gwk?XB;)G%9b#ba(!Ee`1Q9%}r`?AN=woTMi)>ceY#U6#AdV941o68Eh*qNlYG6LIZqnUsQj*zVGuK_(L zec~wyfLI9tzL*s<{{uQ_<}g{rt^@qT>AyWkD1lmKy(dA>t1cyTU&bI%(tkULAAoL` zmnvRS_n|c?EFrrIg^Csc2q}E{@tKub10ryLy&XhH4#B3)XU*Q~(4(zze$yIt9i#7P z3^$4Eho)T|T`UrN47drgS^a$sMA)bDc1geQJ?&NE^vjterw1u}z)ts+#_Zi^0=!>$ z2x_|hu^JKmR#RY?<|lEq~W?@$QyPa{Cwa)PjmOI;2NHp;CAO1?`iTx-~sx|yqzU3&!*1va5C z^jJPZ#T+;lTnsEMuu9ue-;h1F-bpOYE5g^on%_0~KSOAL_BT!22RKu3FoxXnV;V?Z zjxQD)Bd19UtwE2rQ4yDxi;S36^}Ng?z)|j) zV^&cbD1Q8Xw@K1^P1kGXcz2Gr`z8Myaq@qLG+7TmGoSv9{YJE(GKDoYDI~4q22=pFXqX{eNhC z>!>Qbt#23*6i`Y;3>pyu0YT~7B1j5Iij*QDtu&k1V9=e?-6bU*3eug@NO$)pzICbG z?(>fCIp;iMyzd{5n|pJyueIiyYvymxiFh8Znc2IPgteKk*Jk)_vH^593wI{tF>eH7 z;+vw2pwrQQ3M=#K9*Flx9)NQFPbwM%Ao|n{;1#i~;0<_&RPTo?RFsS-j4r(k`^Zv8L?!mzCU} z%hjx>M=iPCAjf5~k&I-OqpA*OQR-Ht9*8Sdb~@b~f0h5HOZ&i}_OS^KRxCoM-eIHE zE-P4-aH0VO)svp~%+6lhW-+a-K|0jw%WsPglqXshWcQ2OMcgc}bgDKzXVk9Y8~ ztZ(NmK78cb?CKNgU0ae{FYdJTZaBtie}n$Mwd3jI!GUC8E9*%EfR!&veet=2LqBw% z$f5@gpYA=xFxb*GN>?uOoWeInEG5{0R)ae26$hK~B)d+F3wu9oE0%;5Kj;B4 zNoXN_&yfdlQZabfWaz={PJf%4c|%!?&_pk&Wleux8VL2Y5lhP zmvN91#ql+QsnZ`jz5!ESA$v`)9I)R92ACGe*ChA%q=H6#LTwYuBf5Ierk_C;VB2fm6iC%&CzoO@0oK(g|H*5jlElPD6VmHnFFmPJMQ~+> zJ9kP)cytOoJXe>ykL_`p4!3QV#RM4VbjWNPGdj>u3*;ywDn2S3R)ba_II~7XCG<~t zuU1l1fkb#9aBb;1{ML8CA2Y9wSF?gB8)0?=Wp5G>PIh2!kRUqMiyFbS6@J}Pfd5`5 zle@blwLZ>F90LHDV(t(O{*$NcAB8aS5+jBp&JUj@`&*Sxle}uBUN)J)p;;XDYr2Em zJ~t#+hKk5SM$d7WKacBMMD9?&24h1t6nH^G%piMdS$FB1)n;%yfXsFzY_%&yJV>93 zR%T9?|BickxUDDCSB=!w@o+aRv7h#5RO;2X2uZKmC()!$MMeWU+hJO&ejkP}rlw6|n;sdoFdr!U+@U8h#PY~>`zOV_VvTt> zr1QdmEN|e~gplwt*6)_l96|L(5`1SdhLgXZ7zW`B0Z0bK`35{gci*AORP4L~SY8UJ zch@NXn2DG`HVTU0C=jmgOqGrOvbtYoJgO1h%XDr-|r0A%G01y^X7q8EYbtI8B^1A%Vo_ovUlo($^eL&)#+1iRXrVs@{(7(nP zzE7T)CD%YJO!O7;jNcA$wX7hYH{PG`dIt(+D5GTRvU+~0G^x;5cAI{0_l00h;k$9@ z@XGnkdWl>j4p_&3P6OCRKs68oD-Hl}PhT|u_2`BJWhw(;e!J7|753&{3^dl z^yd{Rr-4z@-1ak=1|=ZR^0Yk;UMYz<{Qv4TjO~Cuh7vIH3~CgxyCOsdy2S?Y6r@Nc zh#l?XKM&+$l{x>D<>K<1aD4~u%>(~}mTOR945b(r zoJsW59VEn@@t@PFO?*rK{1Lxs>jM*9`Aa+|ee0KhU#AID}krX|{-ec7UcD)1@ zSZ?_P@ih+xu)UJ>A!6A}ul`f8!373Ec=0j6VPFbV*X)ViG94+o+=BatKjF~fjvx<3 ze=u65TQs5T*|-zxtAHJ~GYC13R`ymlw^*y20m_#S!htN#gwcV>3H4CurT6;~d$>!Nv|SBO!`;SrjozX>p=+haEE*Vz8B7(ZA* z2mRMDtnC}pOx;VFCmzjBPGsS(bEWv+l6JfQvF4PB8ba^7-%BDGCxH$UW+D?{upD<mI*7jjA}T4+L>LLjn=B|p&b4%icg@sJ~3aY zKR(6^ohmbD8dTJKQ~>H3|1VXi0Z_SeRs%X*ka&y<7=`8S(WlXT<^yXLBLX%ut1&|Q z=VPfH&rA3VR_@Thuao)IS&r}C^?T)n_<*cGYhEe*r|3V#y$)dh>1V$fTYNrYQ2M~H z_kd*n`1trciv%TgndN0&2s!S}+sX&uo^Vk=+|lQ$HuV|dwx#;v6gb<=yuhR!+Dj)X znN{-ky-CZTuu+8ufOi$gKCJ(5%3Wc-7YWgP;TF5sEHI&}l#C7|a6|Y{#mDdcs-f!; zF!Ng-59?_O^Bg}kn>p!Yiofx?;tB(u#sO|%x}J##39Dw>m*hVTGzRK<$JE-dOr1?Q z{C5anE!S%4%Wge#|M>%68 z)l4=s&27j)b_sroFm{(DsQ}of{u7p55=nZ4z=pjNzgmszE%W`YpXOVQKi~K;OYD*b zCQ8{z__H{c9KL+@do}z|0e$FWNd9xp4v-Aa`WK<1tbz{w#4SsxD2to1Qz$>hjet@T zf-N3Lu?ccvfEFdEou>f~)l(G?dks8;R@2}Q)D8~ zk{Qet`0&9qO7bArIRYn65#ityf;P15bIpBIpa>$K?c&2>m-nuXTt)-oq1C3NSKcwd zo?O6a@TCm%jV^2cE)|xf+Ec$N({eRNP^aF_Sr+Jsy@j5Rum36O3dtD4+b*De4v4&~ zBmrz$Als|@dplnrWA7)N73e|%c7PW4X1)CEV6p-I6|j8_D|azsd)Fu5@9v@<54PA$ z_t+tI_VSEo@7{7JRb#x`c4wzqC%`-XvfxJR{H_;gOt95(NAoE1B*oFixM?QQ)sF5F z_7}t?zHrw>&bZxRPl4J8r5J(|qim5_#1CwU4{MYv2oiy+x%bU(T@T=`gPIgl!ijDJ zsBD!+L9@}x=?X{+fX4GTcx>nWqj;?(yHjQ9I|PwgiVg)w#HR^RA^$K4_;VVM+y!;x zYz?sj$bvvtGff7qRXmG^xC^nIuCxEk`r--ykX)#I8V5a$MI(Z7`=zDM{qf770AB5j zF&kIxl{<1U^V!w~HMCQq=V73vprU|uKgpGpvXJY-JlE=GP)8V9aHI`|c*6eMB^umQ(I$)_b4O- zKfFzridL6TdBIyQkyIsEhS1vGJr*~m;2}##N(2hHP2apHqWvA&aYx=}J~?`Cf1P%) zQr{2sb$+%uka=_ZDTwt2)*Gpiaofrb6&}MLcx&d$m@Et>E5yAz*vi%DE|=BvxJjRr z_3R6G=i&Ub&!C^jb-_&w(0NWiZj@g7lN`VqkQ@AnbBePnn>ex)>goW!HIbt^9g*~Y z+b3zM)0m{Nu;~M}nd+(FNSJ>a+$p8G)IwN{HZl!W}`>UB(U>ty$|5#=_)q5XC zbi8)t!9kM~lU9nX>H5s#mSK+Q+!QEF1sHY3Xv|&W4UDlHaaxrt-rkzlorhGOIueoM zhg{55gY37g(Rr;lzFdl~mffYOUd>k@pKghe7Ia|B)sYEm)E&*UC{7x09W~$lW|FyB zC|+HEki2bSpfMy;AF1GVwQ=G5MViDVn4@0CFpPb`i10Pf#S`M!Kho04KxzVt8-V_5 zO$a88RuJ+=yZmTutkPbEMYHTycnrLGh3I&FN>e`P{5spGy5P9h{;@_#0 zGv}v5VPdu|GT#wtO^m(UPWz%4Tcpz2bt1>-hP%fI;bb-U5@9-fru6X=1gGR(_QDpqq zUJI)|tK>V|JYCzRcY#7`=3$&f2eV!8FKns|+dztvXwbyp@w-%jJg%d_Q*G6sTDSKw zLNfdOLCk?T&Zc8(3;du(9g9b~z=nRxE0X?ir_Wom^)urf(SavY>t(`{2F9&Kd{ojz*U ztePt9JvOda#L<-xBNeUPZtr9(6X1vIVaGl~s+i{sj?!o9c^0?{Idf zNgY3sp6WZupJ9qv+!83W3IOgDFF{``C7-qWjf&A9t*jA3DltyHUWsjB@Q`kE??U(P z@!qYc-#$PYK9iv@VM3=vbRe)D(#I6ao2yOX@|qNwUf9nrT+{b`Fa*??W zGYi~Sb`EYrVpcG~5KHZk47jY_qSniq!5t1|Gr4#ngFjdQ#pX~VJEGQuFxI8>uxN9^P{(+!X3Ft_h+B?rU+*0zn&xNQgTO%M=%vn-+b_(Ze^6C z0)#_aKXEirOP?os+%jc0Q$J%3=Ny{e4+ zW4(8wC*OLdI3M&@WX`o8r>*C2!)rq6$LJdI7x4KlUTn8wK?QZM#?(1zE8+=WMn^nDp_})zcHEj%z@! zKC)=(7KrWUFQl%unXgzJc!rqIufN<946|xb3TY-uwoI4zU9p@69Rt{%zTL8WQ7H%@ z?V!w_M1yR4yT{chza@j6LP(RJczK}H={0?JhLHXkKbTrn4ca8jcPGivf-HF;sppk| z&6#Fi#-MT#bM}eM*PdZK(BCt)z_%O*xbK$C+~Cm3@0#Yt9;#O_BhT+Fl^N%TbPYHn zqnCdiqz-B=Kg(;Y0$uQLfgl&YdDL!I#!aI zz+gl?#_o1xi{J_ugy2l?(T^ym^CY$4`9dcZ*3_3dpaV*58&?JFTe)xa=IWanYTm3K zm4j?JsjTJ|-b(qhqzw+Ucl+_NTmoDj&tGL|W*ZV_kFfSnWt=M07Hn=2#g-mZES`#m zF{o)m$mCi#M?;^cor3wQxzkXY7~UtqPu07crM2gCODc4^I;54Funm%w2biDj{~Jm9 zOK&-d!?tXB!_g{T$yY=w39i?3^FlNVID^JpDYyJkFoF4@Td#4^KY7BuOSyTl%>Q$BjI<5ZHa^;}qb3~kD z^A!f26)q;a7u*gri)K2#q_3p!@pRVrW8g{KEQ~x2yd^PdS_BZCTJ13cOqKRK!2_C1 zv}=W?w7`C}DO~BSiY8&HGRpAeyIK~`fIKu}hcI6^_+YVj4df5>`pmDhnFr55J0}C@ zf8JK58q}avv$8X4zZ=-5_C*pzL3CCYNdoVE+KMNsD+(-ejpe{3A6l5xatiT7I?;fH zhk1Fr>rZ>cj~>j$bDW)>4&zSera{wGbd~TuGcIE)li><|#{3J$gqQ6{$6{F2EAAX$ zxD+^EeEO2r+>Uuy{wYYn`$1W5-O{SOy}kl_E@E{E*bmCZR$(i>_Mk1NC=!=Dn#V%f zurJ%}>JlF1TH(=Jil2OLm8q!%c8Sjxva~?Iu}3vuyy!-eXl6O7;|A7szT5Nk((@aJ zYR5u-zT`tvwdeizRz~#SY#ox@ulR7a?PiY~Fl0Y}K+rn^@^l$}S+7E6qxqk1&b8c3 zd-CSuRhH*>W7WomqTe0jUnmw>B6ODqzHC6c)cBvU)fwvN-~8{yT|zEX^&r{@=((zp>><`vC(g~Z(p@6x+F{A8y`KC zF!F7F=R=?L#y6OKa`fG5#0R^<72<}OSkvfl6^^!?4yJXaf%6MlKNp1IdkgeJNBcoI zPDZ+mw#rtPa|U2Wi&R6q2DukN)GV(uW^&io3cF0D^pgml+oLR?bAKS0jcGJTkZY@vVy<2_3E;IK89A|Z zQhFAb04%d8!@2ry$FtlUV<&oT&xuWyBXzw&ZvFFUhMYqGe%|Gp1gLu z+4<^I$=b(H+7eujk*a-@<8L&N>+J9hsz*!0O9!bzqa$U}Bgxw@0H{=>YklDe%PVWG z^&BYH1(O4#29vADnKNZH$|FxF^u=Q}BREX%+HK7iG3IIIeLEFu={{9?ZZ?z@fYViO zY+Q-sQ7K3QO9i>!mSF(19{}cuc+g|B$gL|a@ad);4Ma=f!u{zlRy|Ey9VGq|#P#7O zQHUQ%Mx{+%1-esE9<>?OryT1WCky*gVo+e$>}(oDi!5N*@4qzH*5&{JcSNy%mrvbJ zW}C3!_g%)gm3tPp;>Zh)2i3b%Bxtpqc8XfV`Nl^sCB$14S#TLBA%FJ!kPZzr>w0Mj zQmzh_tMvne(ZU!cf&!a}8>CWFW6TXwl=hJXWXAPH#!bS-<3wS27!?aWxz$+!4Q$c# zZfySR2WNVyKU`A5*JuB;f&PEQ58Vebp34m$tY=7>@j_;J-!L<8=<=~M>74i@8h`LZkC?pe4$x4nx2gzR>@p|qAoWpuiRJ`Wfjo^jc*?4M{TC3ttq-oB?mS~qZ`?V5 zj!z^jq>7_(m4yl_D@~Uxjj(Dznn@(E7YF5c&>jQjg}!CVYRL2tseErnn$MwMR*o zjngYoRUdwGArB;vk|m}dw?C2I*D}NCTo|=^F41zejcjS>laXrQMn|-eSw_6y!K4L* zn>FasZ~h<=L>HhGugSE1K_z|6j?dm{hs#K((?5Q8rIY&obP;>*P-bgp%(?8wQ*{>% zmKKngF~b^)YyuICnS;65pr^F1Qr$^8N+*BJV1xQX0K1k5x?Lni8J`D@9m@hmQUEr{ zLfn909?UNrn|M3l`O;u4qlo|Ww-Z`Hq#`JDnX%R?lWXA7>4b$bZY_>}FMIiOK&v=> zWyDeMJ!nD0hg?te(*<)6QzlPBeT0U*G&K9NoN6D>&U#D;icwlF8)vLk%O$xx2v`?N zQBpfv0K}(V_2>ojlZsf-)T_&(Y9|Ff8dR~~X1;qL?`UsvaCGLpzyGdQ5uCXz@*Y@# zL8;IQ*+)yYA!@=wpnYl8KArrh+}D|_0vhYZp@F8vp0Ii8)CqyeYf#FG1 zaO2#K-l7FH-w~FEj`93)!3_?h!#egNqo>6yPJn#`V@~{32a;u2($!mw@(YH2g!9%% zBKpi7>U-skiQYab;51^m&Ui|yOd!<(fNk>GdgKKLOE%9XGq1L3lo zIow-X%3ovXNoh%HQfOn#`OFw%^l{%gXeBLV^Mb=>cQy$`Y=2}x6syT)p&n zE|DGW-;O=av$^_4nPi0$5z2yYASw!b2-{8`BPFRrBccoTmoXp=zI+i=R9FO7R-Rm^ zT=)1p##5Y|io!S#F)>fJT90E}N^RD5M=R%6of-)24LA7;R`1Up&0ZfR7bg8mNHsx+ zfjwW@d$Sh{gU}fblL`$T8;16ue_V)!foyn|X*ce?5-gnyyGx8J0(1K`hKK%G-+^Hm z!OwVLt{)OkpcsHbs^f*_xXd`kb`muHYSeG3ex=W|$vH(=Ti|3XSD|}mPs>8H*;K^rD|W>&gr{+`gmF% z8NaREY)3eqZcEJE@$v@`2;f#a9XOe9SlSU(p_lM&i`Q!H^TDWt0`qmANgTx3SNcUf z^~!ffwr?iTN|8A*>U`QPWrFQY6_#R(_V-(Fa)OxR#*a?;WUl7c86C# zozmMyq*cPL!@gvzVI`@&v<%W?UapkW95_eQ@OERizIZyI!F4D#US##{t)Sg#-WFeS zzL|bC(@-MWOI}P7-sF2(M25Kmau}y|`LXLGI;voBZQA~EGHL&J)&BZo&Zb&F{5(D) z<7d^ksbUnA9mYoXmIs-hL>3n=0PN$PG9wqh4+^l;`pBCzUDSTV%X6(!EHf=(7Nm6T z`1qh8oy|m(3Fu%v+nC^sJI$y2>-3^6m++yo{)zABzrGdLpyag_k0>@|0!2I_Mmgzs z?x)D$lla&Zy5B*^+$Q=i`}2Qu{Dh5in2M~qr|U4vnNf--7;w?-gxytQ!GeZn70o+| zYuR^L^k=_t;gj(&?D-DUAWkiAe;sQIp)F`#RZ+s7YoQ?A8{wg?i>Z7fIBX@(!s!1f zWheV?sf7-XJF%}oc190qqmG;R&f7ukKU zlX-eFRdazd&}Q6T3v@VP0C>l}@qr*wsnBL1Yi8bEWcf^EG?Vo+>Ut7mgfCOto#Bmb z&BC%z^%JywPQe^5QZrQ!X*&(YLP$(EWr-66#gQwmaKqw0L zwp8NQDcxDc_s|{Rb2-`UY5Di0!deAoD;q*Yfx zUgu4*G%kI_jkOJB&iNk9YX_!0Sq()$rg%e22aRAQ5FEM7rH+W^-`akCT9CAzefw~$ zx5P0`6U`&g z5*M2)ZcpA!j`Y705oi3Aguuv1e*_rin9KwN{ym#wzK^&Po>aXs^o@N!CO?o_Y62g7 z(M40Bi~dUQRs}rH#!cZgCWMX2x-o^7Ks>aSjXh6=jzNLnk5BSPyX?_sfddDX+%ox> zR^rC_#k?uHx#Tc9LZ+V$l!$!M!;v-ODj}|)st`iy$#!pJSH4rN*X9pef~IJj-k^|< z+)({`pRf!i#+D?HDU&ErRoA>?!}(cz{i&<~16N;Jm)69P)sk-Sr@UP_02`PeTlF1x z2uF{|&siTJ*Q)7)i&yHfGo$$x9>LaZ&Hbccxip3(3P#wwo#UlO1P6vlYY^wH;^+TC zDmJw;QY1BGRiYyM$o2chIjIGCy5o-BH6f};Zg^oG)%)FuT)D6B5-yS{ifev;fqv>J za4-`l(48e3$93e4P0iwYm4RldKMU73@Y!xVJ{GzMafz+d3_F9VHTN4~?dEQ_gjb@E z3@^?XUD7y;^P<|EX<@qApG~BXUI*1m{4l%2m`};M&6aMjM($(!Nd8c{vd(6{R}_@> zn~|o>W026#w-W}XN)wZG8+DGychQRNFHO{4v{KKvER~L$-r^pwGP$|XRcVy=dgMK< z4u>kv&z4l~zf%uqXRmVyLm6gPF&tx2L8hP9pBhyEIMG?Nt9o}q(5@bPx+6+}eukxm zsdhLl?BbTr#rcr7@goC~ZNBOCydLzu9U4ST7W-#a)q`;*mxP~>8f2RM542jigeXH= z!?i`QcC9k5_=E_mbG4sr`8f;;8qmzV3DM9!x!e>?E9P0i2MQQ_OOIui(kmVzt?OH? z^dnPg^)!)3xD3i!pN{sYWV;ir)R@Ms`1{P@Nb>7P>xunLyP2$J{uei+w9@V&e$s^V znz*(Z)%P};w{eh6WgFF&N4k5~<(4EA-i;9AqRLSSYYt&-3YH+oh{4hl5r*Yzegkuy zSgO~C>GvKOF&?k3y~<2pG3YHx*LV@SjxLfo6^i1ZcgZx`5m+x4!rKd}-S6T`GM zr;;1M9P6aF)n{L7J_FE|<*I8l@a&hvD#H-=ceeuOa*3>&ypRGZ!0abG^q##e+u~=_O)hfQB&8YY$+yZ9rOB<@b=Bzf zuRX%As(2Ya-D($OCVyx>J)f#OdPsV!DlePn@S|7QGwHpZMTTs(qBs5@`4?q~79BN_ z$@O>xEuWss(luE)HcFS`xF~{I8G<1-fz1E4!SIVC2m@hIM`G~rzB0_uUGNrd$GEpL zdgZ%gorc)Wx=FL?wIKalW;t2id8Rm}Th*l3Joum_(}N83 zi15q;xJ$#$)&gRknT)w)hXZqV{aH%{zknSw+Gggf8%5sUI|0iJA`eIxznAwjTZw#} zFEXnXjUhX+1qTE~1!e$YrxBeCaW$hGCGaiq||HjQiXbB=3Ebj)fG26}F})#aP@R3$Et9Y+%x zk5j#B@Rl~`iJ$w=#=q*gFzLYL5( zy;v2()InGYZ)r<@Z9gLp`gPi{30YvpDJKZkkYNMBm~C5i_% zTJ6NLf}jra$g|LBcnHqFW^1N$Fgt9P{HXj(ccmIBIM%l!`|vSPEVM9mpecZNoosFQ z@UvQ2I~cuE?riHpq|ApyLl-yj5zFTWSAiE=%uOK~03Bp(|Mp>a??K3g!^4VhzMV@0 zPeqp)KM4YZRo!|a8T5J7M)dF8MX2H_6mVKW#4u_BD$reH%t1*>VnB=wMo1A zzq+;Er09JU2A?tDJBQ07;#zIer`aCX=#{&jO+EU$KW_@D5*=0RJ1p_!>$mAtw2}s9 z)W3J=IvPw)Bl@oOi9VBzonm3{dfstWExoSS#*5Qcl?x{DHhQVqYfM#UYULkZ9~R=y zdGD>U+FOV8h~cM71gLJ!gc+r_`hU8s`b{}TtwzwSU0OA7)6T95d#1183;c)mx6Is1 zmPc*^)FjPeVGKdHpREn*kzTadES0CDj}TZqRaqQyN+r2Y${_#4Hec(i{B~wQZ1laS zp}Eg=djx(v!>#{614EGRng-ujWZDNcdD4)Ur|{*1l8sRt=e?hJw8zA0^~5Qifw%IqN*l#|AfkMy!ih{E@3`^rX{Gw5%r2Nasp?xfHh6>x53l zSB4ioPVfbeI^SE5)P?AQQAo3U<{{Iq0guKWc&WcT)m!q#1g(LJtXhNUiw~ls?&#)J zu;(n75TJJu!eDFRIdL>+D*M0j{s_hQFcs}n5Bsmtj_{G*t&`fzmKDR3CcuEF@;Qy{ z6=%Wq{VEDpoYV9V>Ri!}ew6;i-K4p@Jk|!!v5Dbnh;=$qg{&}{7=8v7l{6}l0?qrT zH&;qQ%rx0wa(eCQ=9A*FlvK`Y)%z(!TXI6*nBz_-YTH6RaDVh!oYMN?g-nua-R;O# zznZZxGLAgn8=sliLVRa%?>8SC*$)L{_(oT$;*Jv^ik<>cb$X+YD$hYxoS8jpmz9kX(S@%;ZvSX+w0c0v}n`{8l2HrXvOJV{w_ zuEV{w$H-xVW?+&x$|6rg&(#kfr6W3c{pi#}rNK2zW_@}qY{b&!Wof){{DU3twOse5 z>8K64<2HY{nf<--(pA^f2R7;5@}~O=mx+b5V!FFyZg4#Aum1dyQMuyz7ZappPQ@gy zDmOwV%UG!q3Gn$X90Q8M*=ofSJBz158y)=QOfh4fO1$K&D>R_sF27^A$b>mt?MpD2 z`rKSyZ*e;7Cm8w@hyz1a!6f+T z$4h1v41RdzdP}OQyV>l<^RK?JED{NN9ZX#_WNT!tUJkH-{^iS*VnZ&=Z9CH=?sVns zjQ5PHVxxJma8SJ|V;NA3R^Xfr{EL$cd`cdg#K%I+kz-B`EKb+>^b~9G@q|t{y|$P8 z#PClx+)QfY(2tkbW;b3ntY}{zkD)kOUY7W{IkTLD>0(p*4yFVAKEF#gk{|#3_z zJ{A(_AM5TDX1pe3zU_YHuAJP}*2opV8-N@1`1MyE!r7J_t^+qeOrB2autj>&12@qP z4hJ|R z>oAaDA^3r8HW*zDkx!GO1}26CBN7doh3}PWeQ`Jp zuT1v3#+ju?Y((VD=cL`C-xkhPpx05<6{2EC`(_~e;w~jORT*sZ6H%?v{78DDWmR#a zfQ=J(Yt&L58HgyEwunosPr|5*AZPE%zUe^LE3Nty_zEw}K2OEFQ3kCc_Nn72;zbr^K5%hW>5{&mw<<_^l5~2`Fz9URDXg6`xf!7WP2wfhwn)<*R%Q2{d%icZ=cmwWSKNVuM^KcXWP|EGt(8@x zxixcCb>QOtgv!k>%py-!hr+P9MBM+YsUN0Q^M}93w$6*bY@NGDxbyz+XU@*<1IxSO zyB>?zR)lCD2)H>525JQ4=M#0NdTSXbDH9zNJf0?Jk=H#3JZa*JfOtz{gAJ zL^9F5x(BDb!)CLu_C(D~8DvS>Z0Y951vm)}^#eOv!z_1sDGar!PfseiGv&?{Br|QMVnV0J z5n%}q=3$J?$w<6lNM_!lyqrpozLPJcx6I81SFbcIC3%>tu@D>_fk|O!oR&s^_Cn`z1V77(z<5F*k*P7v7Hr0aL49uWn zdjfRK>jpg<0J6w`-x?#X}W#&%jB#L4YSnp0)n zCuSP#u17ExgY>zBqNRSdj;{rJ(m|UuIi4}+<$AbrjHto0PfI&7^JL;7OM(J61| zGPynm1+dILZiAh;8P2swCtQ5aq?qQscOc?NK_mP(W21wKaag5WuG*W!TF?TfA$8q; zRV;Qz=;NcmkJx@fvo3ezbIW-6_cPeUDUNNEsh?%4PIGXRGCIL_z^ccUP(NqEk7zyI z2ZM-{0EpX~9Ff!0CyrmXmqk4#cWFdhs}lklKx48TqVLHcY77cmYUWd811_?Zzyy!C z?+=z5g|+2?n$zy0txATbh(e;?$LD$U1$&{H_CQIaBlDal3p&J$43%ir`8?fV zht$wvg@LXYpXo_JLn@(YDJj&f6ya?Q&2X+4#ct`3n$DxJm&WB9Q`xXl$wZWgyRW0? z{#enh$?&ybhpl~#5r0Td*qMq(Awlggi~&VBas=Q97F{6{Alwzas=)`%%R1iU@est= znq~DjxUYA>6vJm@YWL;^qBL8^$xaQs-n}~bRW}XN>%5`4|b5@zf`?0cM z`8dU0y&hCU zAartvwelzfN^wbsRE)rHS7#UxdgIVW7s0p0C|i*ATSuUQEhwjFmrtz6lkZ)=@({e^ zn-p5@NCPpojBNu+Dar{avHqr8YVKQ6BNQ5e=gS2u_?z(l(h?0L{b3E0wY^~>GlSjp zwXH>$G55KS1uSY_$S!DiO6)(|w)=}u$j9iyWKK6xzLAQM3q7~>=I7z5Roa+;aVN{8 z*-%9s`Wi$93IqAYH=5)fYJlf*qz!|GbcgOnu7t`81yPLhN)=^%C)!^gl28e{x#qd>Mjk$po`9Kyzt$idmuP$Guax(X2PTpJ{A%jP8T9~do(e-tNP3%o z0gks&f6h_;o81ooX;Wy&QN<3gfL)Rpn6395aNX~(ac75KiT}u*Sod7mBFgrX7=BZ{ zr*N17Ws6kJ*e{C3RNbCxpzz}*iC?e$nF(GAg7Z zYagkCUI|Z@^%I5z=}gX6_h;9z8Yawpt&RdNgbhrzfpC=LwGH^q|C(gvy*Pi#44(fy zT-iZv|9ryH?V!YA zaLz@rJ4C|pK!T+G!W<=|BVah)_Ev_kPX!P@&SP5|B3RUC)PIlWT1s$9IEwo`cJbMV z(5-%fZL2%xJlg&SWkt@g5wPzmy;|bGuSUnV{phR$G{=^l+t#A3VEvti4$(4p4`!j@ zbpb)YEu5L*bSc@Vx!bG$fkz_;NqsyRhlMmxc+U9cLnK&$-yPwoH-2|FQJ7v__7B6! zCVGBBIT7%@wS9?4Amv})1oGk6_V?973(5MB6_Rs_k71N|V+g`JHL$i7Qaf-7yWN=9 zlB^^)wQ1vIDY)d%LtXsE;_)yEi;3~n4lGb?u< z?m{1JCGi)>szIKeZ6fr_)N;4xe^x{Q$pEqs+#8oJQ=EMW-lk*wThi(8$77@YrM`r^f z{D-kVxIqU&D;%}%O>ge<*DMx~{c{kpQ(np_D+143+uL#nUPL_z`5=fFdW~OcZ?*^0 zc>x2jo<2@i`ki>gTd(21i^oyn3g4b5GxX>+^)3_`$D{`!o-r*oUn1CMxfb=$g;0Vn z#OOyt`V3d*5ETDh{M8?6|F%Ja?C)D)ibOT0ei^F?`6xuV9{h_br~EuTh>#s633~?# zewIF$?`R3sX1b4&fBWe(#vEm{srlYKc z=ECicNtGz(-^bG_iN{{c-HOncuN=HEz{H#j94k7L<}+uSDhfLG2Sk$}EEEGs|I!i$ zr=19~J7;YVzo}Yy;U9K|f#*LD*GdFcpj=xje_qTA?c!9KrNB@lX=&+g?0F{T3{y2O z{n_AW7<+QxY)QWE&ChP#S;Jl#wcS>|J-!;r|&2klH>&KL!ngIu<+z-qXhU z$&QRc;xNy_ftuxKMatWY8^w-QJXOa0TbIx)^~>IQo{2ht5g?ibUtqrtgAw8U*5fiLCgD>a%h+|K(krfV@6P9HiGn|6!@o~ zi8BHlWpT#fDl|P76aLE_=-5fW3|52w32t2T?M=2hX?I%RInxVX^L;lR|A!naV=QX`Ppzw9Hl_dd5# zZssjWRpfV}E*uTR;4ws!2o7%FcTnrC*x$$Vy_87Ip<^V5-6S+^ev~|LkxlT&RK;xa zH@h}5zpg8Uoni`RPgTx#HS&v72`5X$u;$KW26S%Nw|;Z#i|RN3l1k(P!3Hk8_OpFJ z>UH7?hGyO{?Gv$`&5TGt;0L#sz9#A@kEc@+r(FK7xh0)}Hm>0(9t&8;l0Z)I>`vrucFOgag_oX~=CW;PPn@;~Nw=G4>ewuZpI!FXQlG&T+3du^fD>4qSNcavNI+C5 z0lZ=GgAmmxeoyq9H{Rp%N1zIiTYxttzKQDmT^>5N$OlMFyFVCHdMz#O{R2^tcg!QO z0Z)6DUY4DJJEpWxK_-kd-B(cI${c%Gw#Yo%Ut+Y5ypL z4MO0?e0}In+(Gq|=!|9{Z0kz@`(RutJ<%ZQpBX=p3gdrci}{y>(a`+}f7$P38R1}_ zL{!JQN}?_)01KqiNVRRT!Q5tq=RQw1zbm^P@eFJItG1m@%3GW{4qtf8T5ocy5HAK? z2=^b;))|%oE$F4Fdk|x-mo&=xcjNrAZJV4EuA`h|x-f8Dls*SC{If=|+;5T)%Rajq z;@x^gG4JDfqW%DTest_-QRU*pDt((+w}XjYBXvB5RHQEsSW67Ma zBqomH>;<<%AQgX2I4a4PY!bzJ*17e^xJY^`riQZ0KuNHg^g4F}6#BRn(oc|-3#_P7 zl-Jz_l@yRnX}23iRw(SeAgUab_4uJ7&|dy=i=nL}UH>`7|4SKvIW}{Hj}^Q>^x6*0 z>|L7F;cFZQz_^mRnu`$QGGia@h``pwzW$?`pFReDuR|tsT1!r6rNb@g-9*r1u<)*RsCBKZ#^#K{s5!>Un zeN|CB_hn0Jtm@ASc#NzcI=Ul#S!w8x^k}>W(fQb$kp7jo&2t=-7xVw7GkxxHr^IEL12N<;KIP*dT7APsvHY-mvC!5`_cY#sF}aQ}yho zqd{AY&eY4#^3Rxsst*U8CLwK|#7_KU^s{9EfPJ;MPs7>7ZkeP5&;sCS{U_FXV~;_f#88g@QY999w#w9K9HaV5p9DCcf3E0;A{ik zh(io&Uz_?{9A`=tIyH(vr^dyZhN82e=Zo!AS=0UzUJ%_UX&~X7bC}%cXvGWlE{?r$ zSjZS`{tb%M?uQ4`o*lcvBPe2aomytdn@bYqZSFF^nNe(RKmMPH7dT1qe7S%Lw*v0j z7eH^5{}srQ93u2B{1we=xJGwe`bI~bA^^lQ1=&X!NdBDA|K<4M9mI}cZnqugHcr6i zz6Its#Tr${*-eK1auZ$?PS){=*#6PuV5vZ*Z$K7l^j8$_KyxVpjM&iT`Y(cY9ru@K zG8<1Z&+!bsQmMP4PNYmGIQ9~B&Cq>ro$pRs=VjiWeZvPIut5Sk;i%7e4D4q&0&t|b+aDS2_TK9@ z$_Byn*7iT-0#UdW@Q4p&!6_QOptMR2qf$*dm_49Qz`nl96ke+h4^KmOup4a)-3&Oju3u zLtetg58QJNnEGd1g^mRK_HP~krsK`orG)s#WefO`9R(98Ax1O|O*-hFbvpnT;jjPv z&>akdnNQ5^NMk!AxF94x@Uh?S`~2p?-zx=SvR@8~U%8(x#$QE%2Hb%9zlsO^MaA#1 zokTilxcd{t>;wY7Z32slkgDlS4T2yb*bV#xH_^>^S5RmJqK3a;@$RcI$=So;8)+T` zme8JOKnWN^85U?aT_iH!-PzdKu;%p!DroCTlUQ@}pkNjePh-USkB8OBR2`6$kn zIM*L$;Ckfgio)4R7(i_*I!lKB3yJSTSL>yza(;dR1{48}=^+-jybKJX=;O!PXUin~ zdT-AM;{*2*C(8FhVU%A?-Y_c0f5tK3)SS_*`Ck3{@7YSnAOm#tViLQ4`&~Amw(>CG zBHNntfk^{EA$NxdV4QK58DxC#oD(K9^A!$Spp7L(q8($nu4=YI0jFtbd$hP`0e!C5cRVyi914!P>=mq9+CAF<7}nC^Vt3^ zD+J&TgdBhM3lGC%@3US2gy3&~tfY5*x_suAK&<3156KOMA>ZJX4lsSK?HlfV2PoQZ z_X*aMJ}d;eKS2{ag&Sut`bi7i!dD)I^bW5)qker2)iI&1#grOIdFnRBvTD`c)U`GX zF6OLc9?(+J0HcmcHVy&o9bg`QQ9{2kK+OKvUZ|d9i)gRDS^n|X?({uD@~TNuA{Oz^ z2UrG_@Ug19@<-x|v%7%#J6=OA5C|7c#^GpM2nu?!)NisYa0mcOGBW1f;B3=@d0=hN z$aVkgvJz`<*}=6fqII(RP7DlqhczC_nUJGGhv)|bdidHM{$IwK%O$ZF9Wj;W zvMv3)A5SWH?k7*rgf(ctneU_?tUh&Cce2B20@Dn$=H8BSZ*&N4*6s;41|*6lfob;g zh2jCuX@~jC1s&D?8ILR0KAeFbK?B%sfQy{t6a!&9pFe?=$K*0>5EJvmUI1je?n3@d zAia&L!Ej0ugmpQ2e!Sh&!T$eI_0~~Q#qAfbAPN!!A|N0kARsB--5}j9B@L28BcVt) zNVjx%OLs~R4BZ2a#1KQwJ-qMxTle1oSu9xVobP$Q&))m9&1S8U@cQA&XM7(t$lSa0 zq-&kl{9B4_2p&yJqLsud4*(5Dulj3^@54@t%ehiR!KqPHyY4t2H1Su9WJXxR%{m}L z&$Me9JSHxr_n35&SklKba8a-WPO86Y zo+Bf-d$W4{p4;_r`HpUjS_mhy=N_06*~+$uc9%)1Eem5w|NFM!k^g6U1yl%u0j7pe zqG%#3K>RTKfmb`2btpj=laP`2JJ3>MB%HQnoSKjvPL16a%V z3nvM^ve7@iRi33J38-R(%+grh2xiM)T%igl2^FAXFd>}0JMWD zw3JoG15tjodW~lsG&0fgkHM4G1!6;%G~NV9BLPLg2Z6dNK#Y#Omi(=h{lVmo+Ei87 z10BFrf5cr3s{qvnjQ%ZoZZ@|PU0Yuu+y9?9_6ADDp% z9Wl89Ae28YO&;}-EDt|r`PKJ$FWW}jDk2`z?#+oX{jct?YI_%FY$?lF${XE)Sk&+{ zVne*zKmgX9q9S@LG+*~A)r>aa;ZD8yv$&ngpcT@$#l5VUyh@yy*BJ0p*POXJHXolr zOOb5KWNTqnvQ0P?1OzmI(PrKM?bpgpkA_n&LpL?{ma}js)GYel&6O6{nVz5dnZC9V zT@9I-w;L*m@uoF}L7z56m8d63^5ca9{fOy|NhAb>0hgxB_MWVf2B5Jpwp{?Hg zLe>mXdcBo9l7drtUsv?wQ6;A?n&nQ!2*&r?p@7N`8OP>_R!_TTm$~z^xy$33QfaIY zcDuRG-#c+HQes3Nq-@u`BZTf=$c>7HS4x$9Zh$mXp?evTi%C^hqdTkv*?HRPJCRj3PIhD=L%H9+2ar#BCIEjV zc8kUlv58|+zO0|CcqZtOm&Yh_rO*I(x@9?D`dm_AuqO2k5Z)g3vBhKjr~i=8pva;l z(Jeh_MwXQM6yX!(ovXUbUUY}wCHwgdEZxat5vgY4wL`f39pVMUn_ir6K8*Y(!pgDQ zf`p~Io((qz43lM3#KSJLem0pEM(ibmwOA8vb&RO=IAr%=mhCv(@BVwz{*iwCNIrcR zZKwJbGmwkwt3e0if!I4Y{4Y6JlWMx$#vA~+s24M=C_Y0)Ok2z(mYm%kHsrCn2M0T{ zfGg~*Y#_DY{ZcplD^WwOvNG2@38C!(b#w+eEM|(u{f5(dRo_-o3|3WnC5Vn+lq5(= zU*z^Ys>Z8+CrvS$Z|cpKHdUwZu0$RzA}(6GvXKHc^o$r?U6X!cQCXFn2YJCVQR%~g zfF2c^7ALdlZ%kF#Q$Fvo9D%tBx4fH^!Mohpjj>}3>j9xim$pYb`J%O<+(4`<>pjfYCFVXLG&$E2ADEiz|F@L~ z!};HPrHCmUv%(sg5q6E<#{ym_EyHci-@X6%WcTVe7oD)aZ!3P1lzeUQij7gJNWS!S8=@-Ymau0Xsu`IQlb32y9 zJ^UNYtDIi%a;`q<^p*a^+~f8A7ixcZSN&F)V!79{9IcYf%hf8*fN>`Uu|O;(2ixrV z@9Gk8W&qo+2f$I%Ilx_UDiRoic5y;fyeJ_}95Va9`W{-fbGo520+@c~tEu2b#moM! z`-Pn9u<&Zu%gLsGkZc9({!+*Xx@S3+GbdSdUh{u_LFA4SSj zjrtL|EW$zs+zzP^H*Dr3OJp=ny0gFh>y6QmN$}q4!D6eQlnncE(Gs0+MK5F}1j|VJ zjM5>J&1X}~tk-I&kiz;NsE_~qpF{rtI}&8k#j3ZB{*8*589C_bp_h$J97nRnM^}-w zj!pyp&xN16Q6H?s@TUah+>0u0H0b)zM#z;=5x`ia2>2Bnl+@Y6M`$#sAkoRpMql1~ z?j*lF28U13WW>gPGnQU--DQcr|5B9ilg1MVttvaARXWc<$t%Em&Lza_pP^Z%&8w>1^q(C#kboy?f>XVmHbt$8w|plwm=fDm2v;!_`$} zQ?4z1Gt02_Z&UHCuR2$~sL?5-FMPW-_dM1KbW|Ooh`2gusrL}xeT_;I{dj6aT}ZJ5 zY4L|J9#3sroG?!sKyn~f$)mj7hscT`@32b#7e>B6a6_qJEtvaikO}6vJALCvMWtf;+KtOBL7-Ql5kLh>_Z-yUM}i!A=8pl?I7V@3jL=y(2XB)D8(+YDz5 zmc5?HHUje8tUGL@OuR-0x|qA;L=?YF$9C4^{&LvBjc-n`7FUhq)yH=FUF2W`{e`NX zg^E6CE1W6#qlq zm>NP!$J$!_3;PImeqCvNjJ!Tqx#{y1P*Y)QjmZ1~E!9b}$|f0F)5o-$ptUKTNyBZ+ZHLozUyzgVA}N3P#*V>>s)=B7mVG%u%= zlQTH)+ED3^B5L~(n|`e(ol+Kn`z*Wi$FZBQOqo`RagTp4biRfykjift?X|$kR`BiN z|ICX0*H&4{hjU2c#56@MH$4;6Lnm_-6ZvkNf#Grjb`kK)}~WM6CQ1d2b?`$Jrx zdbo$vq1;z>-P1*-BRUWI>2pAyKLmi2A2*3k{pyG4O=A1S8B^nRGq7qbHBcChayrQo zl$&#!)xFqX6}JWEeU9#l2t1Qj%|6t)4N1qsuA*&Qmgg z;V_8Ff@)XQn%y^Vx&U)eAn*ps>|25{0=EhN`vz^t#H=S`><5R@-Xow+ok_}>ZQzhe zAz}Sl{y2hJ`6z+1X}l5BNSbe;O{mbnXLWVSg8Pm-My)riDJhJ4>dDM!$7sKuFAOGt z0QYOcr6z~J`rh#UUF8veG%7sP|7eDh-9!K&i71SnGIp}X1TUn_4Ay?zd2#r6t&?ji z<;^{1^Q5PZb>{YwQ>9;#U(FSG)NY5mZl_hy_wpuXe)LYM!S*MQOfcqURoTWl=10>N z>+_9{WI_4G37`jt9*SZ@{A_>oo80#c+8j~W%Z&EZC4aBuzkapw(B3Z*At5Dc->F=b zhVRsi6!1$lT8b?uEF7iyd|PgRjd^bH@;}6=WW>!JZcoJNwemE}Rd#H)hxR4z$|L&l zrz(uKtn*DT$85qBw`+9u1FS~D=;MtFmLrA-HKgj~z;i@i2q^9 zp@h#S$QZhqi*YOey0L3I=@`p9GP>*xcb+paG7IW_J(kA(&|lU1X8E)y?w3&T)*}J7 z)>83~db40);Vd!GJLQx86o;E~Rt5+QxGN+R^R!z>0PQ(NLVyQRP9*?{-NXYE?P8}r zPG!n%uT*hL2PtxHVKp|l_s4R~i*L*eJ8dt&tU4iE)zHyG(CksuJ_WHxM6`w~7!x zTCIdsT#Hu?MtUnamg`BXIDEC|>e!zztaCe>nz*9cx2lG8nBJ#Jc~N5*d42(|{4))* zYzB2e`W31>@0IIgRY|pSdDeM*yY#HzJ1)^nwzz%e>f%akk9N4>HnTncTyVDlcm>yq zusTi;M+d|W9MkQAi#}j}R`ltd_v~TbMVtUR5J`(IgLGP5!|g=k!{WALZoBCbkwYNQ zSZ_gkld@P1ODlJq+&Mct@3A60&hN>1#|p1-i|rL&Oz;repj z(Z^-aq||3DF2spWIWm&`x=t%Ob$&VaDKb7Rm&u%O4n84*u0q_8Tj3nGk&}Vr$L>xEK#1`$+OcbJA4CGd zxz^&G{8T*o^pz{HZYp4Wyq}bShomej*X}PMGLhanyzjbGn8SuRsYdc8vt!+%CW)1q zfrgjCTBk&{=eav?f=dGUn2=4s;Uh!}Vw8;cY`Z&yC8f1`lFHR;e@F;El#7TLe?k!L7bzg4(gzm@c8lw-N`GOPW z9}uOW7NAfDJciFX>`IVD=!YWU)J^T>x+SyJcBRu&rY1?k^=A(SB7d>5J5D#*aCY1z zc!YWijM_tlFnVr`RMWQ~W!?qGY=lS2W&8X0{55XLfAAf)aKq?kPX6hIEJ5 zcqk?pDt)>=4yPhy#H-FUj?Wl2B*5Poye7j}pQUrgVEgIn8 zQfg09;=UC%Q*3&#%f>rb*Xj*@So8#mdj8AV{%GK8Y(r$HU8y;IG&4WLri!7rh+Nae zeQ(ru#T8NH$5|YBPl*Q~$eNA)g-eiDYK(hhL?Xg#NtQBHb-sPA(x1+1g zWX0rv`Ra68^#EFFu`0NMnb7;r^An%i-iw&C);zZlP?Qdh(93sg{3~k#IT-?8Bg{D+ zJ*-b|`Y{BF*o^4XMG=cfXIG(`?FieJCBC=T^93~(t=?ZAF&M>GsX(MtAy?viIYTMT zufvRG@%msd+w$U*QMPJP}wI~Y4o9XnsV`i^8`nRHmy@INh32uFp^_SSx0wQo7Y{JZP_)ZQSo_F}c? z#x-NqGbWukfHvW`as&`Imx_7mWw`3;n(uc|74h-aE9QO!JluCr0;LH4>mHdllIe5% z1WXDk%hA@V$d_s{``P}NE#S_I9h^z4qW3H0CJ7w)CtPAiGw3OlVn-D%>_!6z1JH629#O zeEW4DojUE22)mo(rnQQW3T1f#(zY_+mZ?S`bq8~!>ej`d_{LsvZi655xfa8t(j9|k zEPeo6V86mM*gK@F$dUhif? zk`Dg%kRKz;zQ$}43#6AvlI?ViOz|xig z;4;RB9bYoXi|T8ZZ$+mTvY@yPZRptELjmLq@+~fwJDtYD_c89gC=k>>xaJ?f5)zI`c>5A8ymz8i!-Rg1PxftQ{ zKI}n+v#R}iGomoiYku_zFXi>EKxa#Ev=(_?tLOc3RMjn`xj*Azy4^yC#$n8n;k#@OFr9~&h3-cp>-g-5bh898k*RiJ>QbvX})TA z^zT_j^3d<5H6ETN(fPRO@X9UBldFG>?GNh^C1+{ww7w<(qIiVgmw!1ZD9#C-$Qh&X zW8O@1L5nNU+A^f&B3+)u9C>-NL}NUd8s{@>BtIgExKsRJ@3uB2&ayUtbml2LV#U4Jj!-8IGy>~UAf$S zR(gn(!e{&4Ez6&<3xLHTHQ4_Acq^F&l{lf&xJ`LL==e6e`t!+=XDYv!3n0nN{CJeU z>TMiW3i)7E<=;e3tcQK_g3k4u=IbaA*OsosSB$%A;Ga$rqtF8D>2t~}6;d|$3b0!| z7F)aO7v*$@tpSyIceZwG(Ql2m(_@a)593XO4(}C=jU{R=#;KcDG)M2)4;hv9{jRTL zL&pc#&Ia;V(syyU49}_Env^v1`YbQML)(`$jw7mpb&1h@`7%|T2dt4&d(Ovq@Umhj z>vt#a0M9fMx!7?lNP=9^JP%!Gx`8gT-*S{_g!1eb+8cRYPn(|grx=Wnf;%oH-GTBs z`%;k(?>DgT+)~1#pVBxa=bb!9 zUmMR%i^gCq>N7$`T*}kL;^Oz3s@;QUg%NkgWLjfYJNy)o;J>e#b;V?axqHU4nb3s1 zhdtksIJU?25snyo6^>@l8HPyCwReHl% zs~#uwujgmFQ^)uTZO&$y`2B6_)f>s9wy1TXi$YLCsaE0IAMrexcu+6$!9a;l2T&;byExU&eFlX0B1KimY6q%xxXsSvA*7t%os(BK?oL^6|H~t8sUO=Xu;3}G%=z_ zxq+*Z$bW^b&?=ST58f{dW=~v}@c~+qE%7qEwW_MvN*Jg0=Sc;xF?=~hf0g$9O6T}c zRUDKsLmqiXs^<5V4~X4G@=!t{jje*w{yHt{P~Ktd%t69 z@vHmC*}86RxiG!>Bemv~%X3k_Da zd?>hQtvQA3+{wdzBppI62gzviuH@H!#wo>9<>F!RrKb(0Cf>0>=6tSQ$_e4mli;etmm+0NL|b6K{P! zXV0w1CA!|Syxh#X>4tTrf$-N~TPx)xBeZ}D8qpX3&AYCr+;}xb@BUDDx%t<8a9Ht6 z6Pp#fUeUya+Mc}%HW{fM$lR}TBJnwnKkYiCK7sZ6P5Tp0{`2W|p-iag!?nhVy=O{? zrl8^8cz3QB(>d!yQTDNeQDj9#E%(0QHCV>5isv@?Df%xb(ob+4R>9rt$IUw12L zCF)C86@OMgiYE3yz4zOpTkwicAbykz+aEVpp-o!yljB|~35*o`qAPbbrw?<3?Qcr0 zg*?U+(Vt`UNQ@|bNzxIuft*nUIKXkW8laA~Y?APYyYGreb%3_B;O$`oztaye+}o~{ zf$xM3QZ^Vra8>-!`%9mi@>a|8U#L21Ezx{lkX@IjA>+C3G_|mb)O!}CT={`8u z^kQhkCG?bPBP7bHgXJDw*bMo^Vl-I`lZb(#D|OB?i3usYJCQ++L)P>UiSInK@7zZq zS&)00#&I;PmglYm!(}|uVGHR>i&=%Vx^ZnH9ZzUKh{SwBxnDNG9ctbKk#yg5#d8{b zu`$eVzE5Gh@(R4x+cNUsTQ#q}T`(Vj-o1cEJ&bI66-3^o411PQ*`1f+5jFF`;l^T{e_|JZ2U@wzo_%k2=Ks z4KSoUc^lSw{SZodG~dm5jrkH`s(@n$ZzWhD(hlS~6=#b%=e6{&LZNR`B;0 zAw8c-B`_!M(*lw=lsANLn%}>~{o<8eHy>nXn+-$_T2Iew&QTWdOX zgM#3{Zp7AZ%dNh491#74HiXl=?Knih3|q+n4*Y#4V*gkILO#OBGe6} z(dQy%6JBf$AV$B=Jfpl_S)25(UbuBwuIGuENZ>V59t(Zpw^T-n-NlMW2rHWAq%We$ z=oyAc3?K<-ovuiby)W~)XdqlX6Ze6 z>45Ru;~?+Z<&{*G&&3H1{ey@pC`J8NKX`)PWpBh`a?Z7#GxRIc_^@~V8OkBI#<3kd z3lNfFr5*k=N1pKEWR;4i+#LaN&UB4^v*IlR-I+}fmHa(k_v9mH8Nd01$fK&(u;#1u z2wlV@6S7@DS)b%6Pq`V|dV>C@`@^om$v;519_$NiJFilVz7RjXI)W?+NRT=0)+np8 zAzM9VxA@K(_fl{7S2M+l*PztCsAwFFkCQw2yq*m8^#3OJ;fA(ZNzVm}&i=&KEr5&W z^bdBFfBwdg?ZlY&?(B#A{B*@%SEJUe`Zc@@?}rRMk3lwJ+`(HLqa)uIJ*fjvb{&*1 za8m0NUGjoH!k+gzDie33OAyUt9e36OSNWZ*O1-{vkK*SIemblMnbvNZ`!MVqKJFHva&bZb3}knonCITiVt=%(6I%BereLSYGO*Cx>J=+ zXEC@P(Xx^`oc#$nVmyQY)0KBI*?DfDF*knw=;Y}PT~6gQ}a5GigHov#{^hj_GGYFFTLdVH`J+XWO14asrk}&OM1Bl zGe}>p*_#;ErFT?_nP$vc`#0^xVLHy*E3+Lh!EdSChN+%sUHRm#50dX)ezG6a+TESb zAHNO;(9$!G=6)?;pOhmG|J&JRxgT^~ySM(r94{Q(LBrVA(1<}B@pVo2_C#FoHVam0 z*L?nR0#-@-eY-NPWbOC53+E;`Dx%=3RoMgV-oDV;tn*H>=p}i1sK))1YT(*_%+r!) z=Zi|@r$;^#{QmJL(LYeFyFcoS6cRtT;BgA~Gh~nYVplrLwxHY55iww>8DYX|ij}Bh zYbj_x?YvfHcKRTY4WLWgOcwJgL@q2#2J&zI0>P_u)1G5#*=PEWlfS_eP`lM_HUW|DrrZ9Ga2;*tb9k<( zGqsBCY->NPd*JwBCeu)OVm}W9=deRs7`3(ct9YoFe0JG<@&s~(6#!|d@|nE7llb+S?F{$~?5Z^=Sdx5cS2}!& zPO6z$V3s7jdms4&kaFm^vc14rgIB#QzPJqxRD2be%NxD2W?`e6o|pTvgHQ7Y;c-+{ z>eQ^O$aTqE;PXq_MV@6``o2eigiHYz0KYw%(Y@&duGEgfjsyPl&L!d??Kgl>@xUN3;Wk+PI|`Nz zAilS{s@CKoZ6ts$TcS5b-fYdqejXhH=KMS*`S5)UkUeaOTec@tCg-<+mK1!rb!nF% zseYq1s~_wFiN5_v|38i=Z9?eh86}a78!5Gwzy(c=#V>IUT$; z*aHhxsmG}Rt|uwtZr_G`JHgP-QhPn7lNmmQqgXI27sX;%*DQC0k;F5IgV?QMFQ8vb z4-%rFk|aF}SjS#qoWScB7PM{Lh3AFaV53$!H4wiGkx*%<7h$R0^<4y8p={M{IYCm_ zc)RRYYe!VsALF$vsM@oss?Lj`fK|fEtH1iVkef1Kop_w5+@Z|Syw(fNPI zH`xTd{O47WwyuQ2QJpsxih}tsm`^O;ciq(vVr(zpc8tbQ@b~x&I#ZV$AcK}_7OyA_ zzh7bP-njil_yT&s5o?WiCF~$nat)n<+kYM9ZMulV!tFrboLPHgBLKgj^DyTP(ev zk)TkvE>Y%~2?*<nOQWrzB6uqFN9J^NswwB--`7BENGbsM5x%C_f(a%b%Os?F4#N5{yQ{X*cKg0G zc+WcRC)opl<`wl=w`Jo7(g<*Cx)Mi@=Omb_ zM8>9!H|CK_giX5#=CNwd=47e-z=6N`wz4Ya)_vUr2nLzGrJj<)^L)!GrA3Nx&(zsR zLu-jqQ7t#lE|WbKI<}^6vQ~0H0^-sxF|uvcIlJt$ zoT#-m#wz>|Ia}^|YTfP-K=MS*J5bhcrQ$6VM^a9MeQBow*fQcmgzcq*&1wN){oMSY zpMAxzFCkL$QB$ub&&6Vx^T2tDaDFI>%LgvK_N9~uu7KMYZ`=V{N?vkVK#;~|XWFq0 zbvLx+bB13BeQ@eSrVuMcR`OJ1s?W;I@dwSzO1jva!?6!MO=cklZaO8~As*x>aF}i3 zIYy>RbMO5v^hccPHHQ=DqD}quFexN_ubWG2{@hH zdo=6pB;VJaCki$P1h_?)Q8+^trLWT7v5sI(%Eo zm{T;0l*v@maCNJ$Cd{77{(&7?}; zrboZjwBLSVyG9AbmIH={ryc@IKg6gp9#6ve=Y@&DNw^O0xDu@vJ_DdWE=|+xHd9vx zh)zvVmKN^8p^$X8&08E3M+hUt`=W90u-a;@xag?m0b}9zAU?&!v41-{YM+fNE8**l z{a$XQOX=Z*$ZVSq?zGbkE!nU!)yoJ=izZZG8T|~TrX&`dH;@JT;HHj7y z*?u?tdW&E;hD6YKw(ZtD20Pv-4I!f6`s~7y?M&GSz=!I@pp9w^u{qhVSV-S1^LyW! zjo!|?t`)OyS)9%gUdHr}vRxYowQlk6oxa##^K?xBiav_{y5m_4v|Hxv-+^l+4Ys%o zA`^t{2dxRF%0jQRCjIE>RkD*xEm;%s$*YpbQc(x9!*|;vZ#DfzFI0ju!i|1Oez`28 zX8a8b<+Pd{rg80xXA#7e?_ANdG^;}PooU0Ov*XWkn<$eGpaDwqMDohZ6&w97M&D-* zUdL`@7XC$x{$`A%yxH(^lC@+q@jbBgGV7BwxN64U`pPrL{`*n8YIEV|b0VbY=8~Yw z%n-g<7qgkI7XO;~&CbZd$JHUb=o8unigoihu~w?bb%RU@-+>w5-=7TB>qJw7It~yK z|1;Mu^RQkBo7`M}J7$XHcb;RxaBs$Gyc7thCD=`^2Dr1Qi0{#KuHs%I@~$MlHy#Tv zrK}(bs=rN=H4|K2bQML(-cfjDdAond+G9t{*FnM_+>mz zuA|lcsnDq4&HnUKkLKhYmV7V*^u|=(Pt7SL3)!?%A@DNe)?sXf+qE8YK{DR~gZpXZh1n*;G+L3LMhUDVb3z=r$xe_)28yaNw}^-Xjrq zz8587m?#*u{{u}8AB)cv(m(D6f{uTxn(6OwJKjm@)wm={7a3=}H>Iyv-{i+s7?oa~ z?~XH7b+mWOsKMY3QRF*|sfTct4rD*E+X5xLvRIthVBXEE4m9)VhOB8zCogcI6vbOxE#24ua8vtYrQiAAdCoT5xVCzoCu8XAQZF!d zATDSuni;g~F`~$Y6^x5&2Lm-j#!SHL_}H>-{*<)y>=-5I7gG0bzo3=V z;AP!u%g!rCBa9S=@al54_+V2dSeb~c`mwub6RW!{28E&p9A(sstas#ZYDwNr;SnX? z{`E#>t0iC(4YI{q-$y_jhVVSZ%gfdSOuC$dA4HskvPAsoWR3sk{}|Go9rq-z+a1f7 zj=*_n#$42-?~XxHd2UC2KGqmrE~|CBn+;}`CtIlaZ8035suM?oMU{+3Ng{HRd+$T* z^y+raGvIHvxG;&Y-}yXC>Dp9mn3l80eC4v(7Tu>r2YrIxQ~Oo*=N?6JSsC+H-0e>P zc=1?2qrC~lll`N0cZt{KT=(*{2{#~x=BH@L+SmmDh5PLOX}#IpmE4E(0ZgQ>suY7# zQvW;o98urxH|__kvhmOS8Py{FS>XwvWJNsM#gn!#hH2E<9sW(N+1w9|#~*ci7aqtY zTpE|-@2laKTAULz=NeIyQ)%Knxu)#GC8S03L;cq1;1exL5cmKIWt%kxcez~+`zR!E z(J9$v17!n3Cw{BZ>Gg-}M(L)-I*f!3<*^ecehkx~Eax9$;K@{03MG4>-f;sKBu0d7 z;*wsu=0iDD_7s%rGxCP3I;^A?w64$5fqB4_8}IdS_pGERPC&_aB>qDJ?ehZs!FlZP zWYxSI6+^5&HtjLpAh87T*w5kD8iHI-=;@ty3r)gU4@5N2qV~i0a$Q#OGDN3Cy#)Z*_Nr1~zlaNjpoa9i6SMjEYh-7NLravQK$EP=))!K=+)3DurYdA?r9bwXuNgMK?^f**JS z?z!KHiSLwh4_zH-kl5Hg%*YslfuKQ6HQ$Gt{h1}a@$!k(0FJ=bxA+ee7$GcZi|o8( z^aRd5^RDBdEd&OdRF16o<)K#D$hdehGBrI}Z0%FQKQ!^amG!0|u^c(P1p4Tdhh(n~ z9JAxFBf*dDkvf=^z|tXi$#rkoCDHEH$gz53>Seb#Y%NnhN< z)JsZK5C}9g#kKi(YcY$~h_4vq;jbsyve+dv&(7A>$b;37z0S8t8xE%c1ivLaPg>1J zIrgIa!57tT@zhCSN_pQuH63W)6JGJmDh3*pq%f{HxV^68@o_ux~4{2;* z3-hOBkHkB|dm|XI5rj=NZEQ#JIY-!PlHGqnArYbFSZu?|zxen0f+gA;lw16-N$yW? zk1r?5P4F-AFn;2_ie0mephTG!@Hny6^t-fQ`3Y`tIL7^So#zhaVYckJoc zNB2D!eV+y*)5Y+Tf;xO}dgN+fVh4(Nomsea5u@~Zm&S#=MRvv>yzn{o{s9WorT_3$ z{3%Kc%FMVdGHs?N?gGN0i1ad?2quS+(?)YM1VhR;4uL3FTC}Q`jj5yJZX}-M< zEb#s@rVr({e;rcfa3RrdH^muiCKf%dF9v-%+i>aA@_EDTbk(fWo}*}Jl?QbB$>Drp zOF^rQEKEv2D6ye&t{=QNI0I`dEfaAt5?sG>ozH0DfIeLn+!;=g70Fgj9`s4>mJB6( zY942bD!$_BP-)b)?x(Y3``nbTe7r_xfpnEP@Yi$BN$lShgM*P`o?_qbFaLcU^^J-A zc8?K4-!&HpkI(l{;c=z&qyY+d;$igFFdR{&(yi+63?$5 z+{9GWRVz?X+a4r7a`1ifxjIg{K8RXa2@t)R{BStiFIFu3?|?3sgYttqt=EA<)f^Wf z!Ft(K(4Gu=YdRS+kcm3iZli5@?p?CH23m*TA~?%S=6tdUM~VLOzSEAtR{JQ#shScR zVA!cn_@O5emHP{Hwi^qmhfc{eLhLwOQWrL0kxyvw&M=z8*C@_F zj$oQg_8n2Y{Zmxc`)QtvFKfJa)67PlrM@LbqxqzT%9$gJ->P65`nC3jM|8WIidLQvM98YOcFrnVZ7c$u#p*=0}pNv+b#$+lvoY^M0LOm0n$j^EYz=9b2LcBPimlcxK`fP=#h-b1@@ z*{SLSzezIB59ZtUZnN}N1l|+Ekk)Y;rK?bFY>SQ17?YCY>$1RhaLwo!_>k0ezKmNr z{`j067MD1m8mynLv1Q|#WcI4?ivny{$!&TbRV}dI`T2D;hK~K)LkC;)ttP+F3AFDL z!T_2#H`0ye9?EuB5B?N|PxJhEu{A6Q+jjVn2XA8aSWvwm-S*vO8ue5@8=mGeih?ETeDnUi3sX&L}O{KlI!nscfGco zG6MV4n5#-^3#n~+@xoX!$E4LWFr!GmhkbCjy+!<#;Dd~KGG@8>-0P5}yPGr2e(l~h z_47vC1v@cN|C;%EqV_KQU*?Mo@8Yi{GcZy>`|sPgER(7Q^%jiR{q4V|_FcdU!EnQa!Ck zPAqxgfgC6%mJqwGSJ(CR~C|6Ch2#@YmCXbT&$E#kV@BTWWC zRY}xaN{6#EU7G%{FZsnAU+&+hG>X;}#+sVwli8EKS$ne$0p@#>u;Wsdg2rrqKTJ~< zaGGhyH}|L5C{?2=%<+TXqEsU@MQcuHnqp|)XdDt#&)M+eGBMA~(Ht+pGO6=ARhl(H zLsAfxCdSL58l9JlzdoWkteM?F(i%}J5bsA=J}_*qr=W^3;J!rd@Nlqom^2}K z8wGzeAM(+_e{W8Za<(Z+47LwOff*d~STnC$iT>s;OIAFp7w0FNuS}j489)ohq_0(b zmrjDJUH{u?LGTquxkEw>N#IoYliTB5_g&AGNU?v^|L>^!OWHw|#@FGRm+E70?ES$` zM2d_{J*GFaNeyO)=Z(6{wTJz({dODL)kFQ;#3V}bmgi!;m*riJj?1i^T|rM@1XmfC z>#BXQ53`q+jiyN0*+ug{lD*oWTezNhDj}mw>E&ZWgK@s@J6o*w{))l;2X-ya-B&kZ zTYEXzk2*g=6DG}a$=~uVKKT4w&+SFy`2qh+*%cFO7af-`GYYMT8t>F?36DMq3%*#3 zBYgS1hHj-StHd2uN?3f#${hIEE{FlssfSg5?sc4aDZqYQ378$KOInE(a`~FFxF6=w zt`d6@k;1I#)TcOW8XQ27POoJ#5)&k?Ff4HXMTH>T_cx|fO!jp+ab?oZO-(=~-Zo@r z;PU9*R6fl2yTTBXL@ARi8mJ3or758Mlt#xhnmo!w7`VE8D==5OpgRw@UlD7^K4zZm z!aw*6L#(8XTrezlniInhM0K+St3m1VX~RI)PyU&W{ga;8YN9}AncLlvzaEjMs02Y^ zCnDDTV)x#v)R^m;d-)}VWXkMRTB=Tq!ZT6C@8KcOz{eQIr?CCA-rbX3er!Qd6m+X} zCg7HG);w6`T;Y14^};Q6l0cO7BkW5~6==H$pXG0yLA-S|mub5P;0@q%dXTsgRyi@; zysB%(7HNTQNs{q8#f|*qW0&(%2 z1;^hs-YU?oVMn|Z|7 zUE9-Lhhi8-!y##~x_G;4_x54?rH1sZ-RSITnNF2;9Y&>5gKfhYbBK%mVdbg*JKk_D zVR+b|PZs{DsSl-1O^iIJ8&TiCpeGb57bTUoTYQ)&04o08F`yi#CMTLliBaNt$yGTd ziU%Smdu6w66lqy9GWhLYvJgqp>u&wPDVDr#JX>6)M8V(3w>E_2H`)_zmpg1e3+3j_ z*03VjkAs(r)2M94p7zKb$GyQCfBEl@%LXiwoL1D#T}HaT;Eh8@MyR94U##AjCE_}N z4WQWgDhx;m$kx9#qtg7r;0yUyR!+9)1hOW*+58yvEgA3Co32qJ4wjrAEFX-{KRMnn zorzrkGE(=L2s0$^*3gGx%X-RJeJUfW8W|Iny?a0pq(zbZmLZFyG1K7Xcad)ucp9}~ z@`ql#lL0)pyvGfL4Nd2xHON_V{#N?!>vdjQMVEbvw-16XDqyfS4r+T<T;0g9L*yakuMy+PjQs2&9z9A`kV}YdNle zJ8`Eap9Ak~e+{U#2bZWrBuj+rG}!T2U5U9f4rppuncm& z&Ei%fxoY3?p?aWN z=|F3XL*^U)S5(x>%_hN(d8O7qf9y}M_D@CSwLYtlPOt&H1bnu5ouA`@?e$-?kX%3y z^<{#>QGWy>KBhOpQ`#wW!#SbD7t}`%CB>?Fo@#L(!V%Zmp5_q?`sz7VT21UmEr%>& zC(;pj44MU1Gu+~1{gn2-*opAv2g&E45{aMVi@NOQVXv8(c62E)=4!&V zY=?2);gMouH0&@YF%2n|ePLNvE7zqWAkGWfl)eg-4E;_zLW8fT9sTfzU$c?yzvu5m zao52xmT0LuAHRD^oEkA|mx@WrNKU?)apytS0#oE0c2oxv?Z`ao>9u-_rd`J4yS;ao zuNB~&EDUqd20O(i8yGK_S49kqR&cv=ioRnaZ@x%NU)BVkJGfY$^Wcx^c|(~hLSupw z+zlIwHKi%A#hP^@V)0!QCx*kl+@Ay9Rf68n?U2$@4z%xntb%eZM*x z^zL50cde?qX3eUqo{rnQD3^Hlx^E(vSNt9xtbUNaR@p9J+~8Zg@}9V@eN-~=_ZLJx zL&2uGPK8}-XduB9w63DDDlaXj&|Sr&_e68el8Fc#x>d27AD~2jrW)La=Ibwb*da|9 zJ7RLaLd1u2HQ4z5L@Y0$d?o-VE$FImn32%0aQ7jrJ`g4G9kf0Sl=I-r^Ip-k%2VER z?LZ<};ZVT}3L}vguaoxWd%wv>Z3IKgLSPQK0x$>MC_Rjnk6KA0ZYcHLS9_z8^42|X zptZ(KktehQNAF_f>_B40#v3KUgVBzFJXDinnd{xQ5E;ejUHz#Nm27V>cl8p+0h^Fo z{%K^8(&t@cgIKDcQ7e?A7Jc`#1eY3BgGPt`K>I=E?;rVa1;dXujKw_bd!PaA&ya5) zY?vno*Vn7!!0zXL6ROPWbc@2J1uN$3ixF+6+YijsX1|27*kalmjbzy1n#!!GU?LeR zZ!ps}ayncH*_$e}()rv508&9_@Cxld{N;RrzI{WVT#zqWAlgDHgOSGMMC-xKys9`M znh-XFj=xBM8W$#m4znzgAu7i2WTk>Bpg8)yfMQ0?iHJ0w?+K`vGEz;?(~#nWzxPld zlDnawjYgY1<{7$rvVMcZCTaA}#pGo`Do`2Ia$*g248?4-_2mNkr=A{Kv)@TX+1Bso zeDW9B-c$YYMLuo(kBpy-UikwZwcC|cpsmEpGWU+G)f<*IRvZOKs_IY&2M&@8MA3Ou zBf7cY4{0B!5KZPbk!aRAmvTKtw-a#F!x{@vX?YJMVFXrgxvE+Hsd~s z+lkGntFvjVrSqz%2_vbt&KT~Lj2Up3)31OT@&?+fnM+8kHBJ`I;eM@t%2_k;_w?%I zlYH;?+MblI1~S71)x?2=dnIp^r%kxv3OY#E`y1Ilg~S8nAez6w(62AG7V{~?NE!HX zBejq3SQPcWqA6v}_ZF%GyN6cZ$yU4(W!W9~3%Hwv#)$>d_L`6$iu4#e=`zq~C&WwZLN zlDFt#Bbv*((1@~ksW!wD?LfABjU zPU`418SP4~CspYyRSB!(jaDRv(6(9r>>vhC>TO0NRvOh;bw-skFIKK92=gtlp>PFG};OU7P*gV_|OXX#I%k&!Dx2{%K8yf^zCEXSwW@`*tv ziB&9PoQ#f3Dhq+8WrN9hW0}z+kF=!mAw)Ak_NCCrnkAkL;4NX#40O;UsM63nNq}-K@Mt?pg7_8< zL<8NDU>XMQ0jAC+=k-m-s;y)SLFdM zm>)Z1${}j_E1U*V^rUO>Wh(_1cWs>xfc^X^fNmOd27622>lgGSNIyKgkqgFDq zEKiJK7#DoZ*pVHo2Idb#u0p=rbN-mY9!lvqyfBw9H&ct6r_@_AL|F|9=P&B^d{&bUCt#1WZJB1bo}*tfj`%8e zQlFhugo3Mw&u&)Is-TCp<7bk7`!fGl0FPFuw1~W?Cm6kJ_UI7?Azy*}-h3A5LW5|2 zQI~Q_RN0Ehv4x7cO2~`sb%#ohCaBl@E7S#dtIp(@ACJ?ui2h3j>S_8lDOq`gI@xHd ziRSHjyUy1OX@H7*yH1HVt;byYJAqLnOFFN^jRCQLh#_CePssvnC+1jl@)qWI=V>G&DIaq%iT+{9pvHX3M?DZEvXt;7 z(%?2+eX>T<(7(TcrFo{^d>)rtpc33_g&jC{m%VWrB%#&xE8yTPlL7bp z#giUvC;D@)Dz|S-(-`(Z6x@7W7Fn7|aM8p}q0z~5c}Jphv(}A_$C}lqBricIMi3;c z-ljtc?S@}xexo3j${xY1BH-`fqj8~s_>Ow|#btS$Aa_p_b`tx^i{;ag@VK?j71~{@ zK1h(n3a#vdp_h;OI_&grF~g`5)4AM3eWVNZ{g)bT)`XJoGvDqm4>f7E%X1b|U1T#O zkAiDCr*Y9FI=zABuh~SNpI|G=nxF{HPw>`3+-M{+ZNo{UXPGTG6Q1>#hqVEX!+=%mS zrR5{n>&Q2&$(vf&-^VdW(%8Ygh|r|NT9@Uo!Tojy_f>vGBl&*H=kk`UHpMkmr}aUU z!k)6ABLXFL^^ula4s8zbQ(wFpG=)e_pisTnb@#L&KelfGpRP|YZ12meNw6nRb zTsK5W7q4mY*@?0b-nmbLQrU4bdEH2>6(ZS^vG1{2egI$SVR`}`V&dq{6tECXHJ~2= za**0)iMMg4$WdqZn%~~E?P!^25t#wYJOHUi`J~hQL4yhNEb+4W<}K#V5IVq13(gAr z5;}BwvY=C0^38a1lt&4^N8j-abdQNNq&^|ZdMmTX>d)yYiRLjQ{e-@4u2exgHE36g z($5U8q?1l&$?tVniHSw_IXT;;i+vAO-OVdBsJa1XNnJaz$ zX;~jcHS#NH zzB%|Cqw|Ea68!!IHssCT7?C5yG(L}nXfG-CyE>~P$@MRL-xf3L&~{?rvaAEd&nOVM z-zSrPKEPBjUTS*+v+QeU*RFAOZhd!thwmg3D;Tw1q0o&KTlx8Bxl;dY3@_^Vq(()2 z9CeKM3%*|^wdJI#Q>W`b1MRTU^#pG%_c(Awy)T4eQuWjvG&G3)2ia^ogf$w>*<{{u zh{u)>-t`EFt=&%CN<<-`DZG6|;DIT_7ummK5D@T?j6SP>^^jaSr zws4{MPL}e`(SGy2 z8`aKisxcYi->q;hQ!^jr&OJnZs;D6Km&lGq5P+9j?X3wM{PitRx3t=KZXe&UV&c2H zw2;&fx+#RJL=+&e1OwI%(0g%rk(g<=VGGlziLG{%t?VShMveCg>(M~muA&#s{qWXF zGEH6v7C3ra9nZA{RrZvkAVJ{Hw+q!sPbM{^IiiJ>)mMp;5&d4nWb4MmK@fGzTGcqH zNjbJahhXNlWY-Z}Vb~yprTC$AKE-mYNtcTdKIem4x&T@unNv&ON~KCc@AY%N2*IeT zxH7qrrRKN1y1*>iXbUoy(j0@`x5~~HfbZIdFp8w zHB9OWvQgAGtbthys9h;F1rkpEiBzaJLNyXKFvvSs7_`>_JWM$LAW!_$o;Mdn{kzVq z&<@3QNrhgMeukk40lCO(5)s9ZXa?@np)c2Yk+$km*pf#mFm852wZt(i*aruHZzIP@9Qs&81jU zZXs^A$`JQqDq>$nIsF$9RkY9qSh@^eh&LZqOWP(X&DHVG@D!TRO8kNgfr=>ap~j zTyj2ZJA0@53zoG}>?69P9(RyS@AM$`fWy$|@y& zQyRMuj3)i^a&KN21GiKRZEdb6D)aqC20-YHzC1;Z%##x1Hkd{87Y>=OA-~8IxEHK7 zcaw8M?=vm;E^dMEF;{$zzcCv(m7>rpia%d($=W(9NN5=taFADawU201#4@XDO)Fj+8MwiWz%@Y z#PM^{%lgqOp+6;{FcsAJi4^hJohfP5dk1(5jWsAr6#K%wCZLzobFF%-#fvO=h|-(% z|AwrR(11Jz`d*E;M8qK0;$=Gi~F5G8?`Y>rTH88&F@F~OV z0^G}~!f%~;^;_7GVw+rsADzwd+)6T!yvste0Gs>{C6469u za{DxrER7GhOK2$iDM0+?J~DKAP?eRgt}ZtqDPhoUP$>0k9W@n-^9zm4sI{K)8&1EI z#TZXu)X7r)Jn#%Z-=O`IdhcLvSTHIQX_tQ{GRvUh?!vupAH*f19`Zx)y-45fx@L<; zdZ{|h*8{5kH(XK}@Pi}8$Dm+QJ#v%>zsZ0P^3$a*6+|2Ii(n+f3Y%~gK>z?UAAQ}s z2;e)+58J{=WOR3`F$UpjRhqtKP;Xp#ll6_Sf)8Haquu5KJf#SSYC{R=;H;PhKQBzFi?bqvdWqCb#E4orpS) zOUU2v*@I>FTy?81pV&@@-2O+7?kv;GhalW|j^;Pr1bU6!i{D>KcO{SLKgNoZUO%c}NL z!cQaVRNtl4k%C*PmrQE090F@$plR2BMM(E%_-=a$KD7eh=xli`*3MCFx_X5Mbq*9$ zPG@Wxih~zO+Pyt~q}sRYi?=hvb_Oi(?YJZQ!cQM7e0}SDFWNUHw>;z}dd;;-t3nbi zN(%#pr~U&%|Kqao;&z;6R$7M46KU3ddjsT z^nxFAhp=b7m?4lIny=j-nlElA`@Jr$3&x%{)jP3>`}CS{Zn>-=3@<_WO*iUMi^KKp z3f4~-uen_>5yc|0{0baF-zlNy&_`aCJ{RLo3>d4mfTkkZb5Mvgzbb zQbxeB6X;>SN$t=G7zbgw3Q93YasCu%Vn^2(K|sy3H-1UOJau6UyZTdP7u_dL51iv` zs$1wR(;qkXG=MreMr+Tg!#onWGFt7#(~ym#ob%|__RV>%aeR|@hr5Y$Cf7r?c*Wl^ zi4WF259QYt%d#y?n+k%-5h3j`t!E#qZ@46dhC(OAGW3SiyHp*WZ&GH9}u#mLLEo?i)E1?>vlYiS?ZqoywEyyv8{ zBs@UONz7&}4{q1Nh>Y!biJ8*U27@T@Gp9LU4|4DCOf-}^t8I?#^ik6L$lcx!i7&Pn0=+V)F^dyYI z`w0w+8n<23qTFbs-0p*%#opzfFKYZyx=;2ieJqc{x~S6MTCkIFDqGBsSWafO4mRfk zyweh(Bf5ezeh3J9B`GFhMGV3|94#*sgZ8g0zd3!K9Vy_oMPDW1)on!hLV&;QWT+q1 z%kQtt-(3Id;q(%uAL>>P?jE-`8w8--tzVdK3JDs<#KyVot z0i0{lRBE}3aBqpWqmN3=m!se1Q|Kd`$vzG6CvVwowj|j^>-4C_g@5*@5#-atd_!-G zVo_gS!Ck60<0H9CENe@Y>mp`=&aARl`m&t4Agukp0+c56bxy$US3(_(bXL8~p>I1f zYD@#?r@=Qh))%y4BP8z)!05#ntllP#PP^pGT}5{)nc$f#V&?bW={y!+2K)1ePpQbz zh|mGh%{$D!{Ekdd09yaIjgw^Ts?l7_`N;HGlfZRf z6fMe9$=XlH?n)kIH>d8WAI%46PpfAw^1+S!*ZZ8sfE0=+ZY}$bjdd?U7`ul%9iVZ5p00jihy;`YQCE!!(#%S z>9%_3mKLv%Gmlutft{7UB651~CWFUxvvF+rljGKM}j_mU6V#7c0){NDL>>oAnJ2+BTl&j09kQs9Y9N9OIDU3XbhfEJ0R zNZm2P=+M>`S`92`oBrm+AIV5@rjX$*6>?uGSw>S{uH{R^is2*?`RUXKDCO#Wo56_l z!%~O$DXJ`9`X<5QgMSCUbe$}rt6FYSha#}AH)=?)9mU;YUXyn zl}!==kEV2AyG89sMAICtG7Bj8tK|y%y4~Xa5t+)^OEq)9<%A|;$RR?@=c|p8vU(qw zO3Mt`Q;uqW4w7Yo&L(=!btnS44!I}l^Owxf>?ALqE{?Q`t9IRW(bUHKA;;|6*H2Pl z2i>Mu{&zSH>c|w$z9^z$njALh!OLJ)guJ9z0t80Es{V4ZwDI^9vXs-H6cm}Qht&-0 zV&;7 zq79w+78P+c3hymz{9cm0g@N74Ri|L%Y|ljR8>9hPEa)h zf&lQmUbAE@IW}iwNbR>K$t{)^eVf@I$>LFjQ~`k;VQjj3{-Th%&F8Ei=na&rI!EO) z(TSGbOK~tKdV4DqONPM{8{jVvP6Jk87O=ceff)$YR*f|PF*I z?W5qw6-*^$+N(}^x?lI4s;=P( zAg8H*&c@Lw7G%*Uzf0vXJ|lG1w|=<4rw)??)dt2Z><`lTPYGoKl9NuJXBzE_%J(=E zy^`4PWTk#Cs{e{ktM9n8XfmZo{ca+%9O^*br;jZUjZOIE#ShujHMo0zc=Kb>tIs=f zvLs80udqS)Z|XB$uhz&e^M#@%QY*Dc(VT_{ch6?;gIf0G&-YIWTP@k5=3-T`#{HRH zW##I<$++mFx2tAGp0P`LfbX~K!(nkzL_TR~zDG4p51ONc4*RDBbP$lZUKs168!^2; zlZFf*BX>vOeol#(7Kulqz!_P)l2MA@M$PmQUblcIl132|IzSK>m!{?!tK@mHw2%?uX!mzJ@V z?e0*@qvDw<-+mxCG7O7|!P6X6Rr(nak8}I=ma*jt*M0^@V?)t+UI3jRofqLd;aN)o zu7p)CFM*ANIz^-mAu!7-m7}-KP|hqUhwm4RLI9c1JfOd!Gxo^406M+p^vMOp3Rd8K z>1x#$tsupN3@0CF-1EmOFgVs-6wowm!C+& zcc7AO4^vAZ{R`Gzy4RMA42}H{NQnYK3jWBuWjKIbVTN9XgJSR_xF+ACkCoA5NrO|uX4b7Nyx8|A)U!cSf*B|R<9%X8W2eZZw+gL14e=Be~dh#q3K_o z(niMO1-ilIjM~lZ8p`}Vzz_NOM+-%@&?uRRCMWVsN-~Rst4mk-HFS2eAZixSC|^1N zM>i{IMBx3x1Vxxwt<(?skhBmAME~sGiSCEZSazGo~XxZdnWj4;7P|S7ilji1vf? zULheV1OOHc&{f6u$^rIXGEsO6Z*_TwzeMou-bSPyAbL9h$7NCZ&00%=Ctc9Bs+3l zmLDP8t7J;G_7r=RX#>@3-dV%^K1cMOd|U=~7&|6I$r54MOflLoca{@rRZC3A%Q_MJ zPwuV`Sh)&Yfm|(gt@*@9@7f^2DCDIuEV;ebAs7T+Ix$_$1z7G^A^QXRn4ybR~Iv;ZB26hmTo)!a>gCGd6h;qE!UglBD z=ZxO6eWL~<>H< zSJcfrBgwDh+*S2}W<4*f0p0pDutHIL;#T%P#d*eysnq!t$x#sMKSA`O2HyZxl;T(wgweF~Qs?dH%|O^*~n%1sIl z=ROK(@{AgY0Eey)(8H48w+(8Z4Q~6`Y^EDDgjNn0SJC97vKidzD&=Q60KGak4ovz0 ziY2y0;+GLn3B`f-e(_iSzR=MI^thNi3?yBfKn6tcIRnD`oB`n-@x=MZRp7u1`wK?t z9luI%lmiNO6o&BFHt@QVua5+5qgC9?3Un5m;NMlG1f<*5Va)~a2v4=zCP!_VLjy7wNZ0w#b|A2;| ze?SAg4{B8*7!kq=IueK;Ze6XM^HBp1d0hUa4`ax!jwc70jSts^eCX!q!~#Cbg*4_H z!EvHR@WP7NkWjRd}a}JX;qvc z@iG*h?2D4G1cbcz*!EXkGO$fo8(pgmCgrs<*{$%D<2^I^^MP{G*r*Y{P#h*ktEI}w z9I>!ZycMsQy`waoc2{#HSp3m5pEXtU(+tY-feSmJ z4L{QW9E0H3CUI3hKBh$e5JfC74k)@nfv@sj`{gvLxCsDsN78=5kY=CY3(h+j1V|!! zEfN~+2YT%~|7VUZ43DiymXPO-(v@;L0xHpXlgsg>>g~#vU$p;gc$1~?c4X^Zyr^ng zm4GV|+!1>=xrMjrR1p3yUVFe(00~?&XtgI_>Kv5R6}7PEcL9un)?d3?Nzu1X1m5$< zsr-2X(}MwaPAy5S72(g-Z=fX5et2!avcmF90Df>bz=)$=^N!_86awXf3VTY$MN*qm zu`b;NG7@rI7Pvr~RsN661;S6zLgW*68U=U{vGKl{ATj6`qCBQ$@0@KLetG{{d1@W{ z0rM&MnJ-Go#{;sgt{n{ScOA|bo*uuj>DB5@#BsIj;09U_QQ{{0H%vL6vK zDakoqD|Fiuy1}+0A==4+!ivpwVX`Rg6@}duWH+C*F*B36JA@E`I}%ZZOk~bAuAH0A@Q%PHcsWyOy-SVwp z(NPf%wAU{bT+oIMbxRip)kpfb9h~~$FjkAXK#h7!lD5<*K_p7*pWpWF_&*IMiRZYM zC|R=t3TZrEBhK_T+YVws?};sjKs`9>+pernvFb%c;y4{hOG`65z)7aMSwFyivRK5! zt<*b^+>gkM=;K^chIDf)oiB>+^vfdQxHWYkRU>*R{25(PheWNu>gpV z{p^$1WnQF8iGVN#Ou6>sI-tMnS>*or0Rk_!wEv_67mK5yT!#Pg z49^($?{&*ZYnVU(a|y@}cB##WN$kByv*A(nALF2VrU$HzgGro`(qt(|gWfE3Ovt;E zj3S`A3oY;cwy-fB$0NSQ{wX6eP+?)k(VtG@j8=c&(!iwKIvSflU^cZk1v%UNc1-mX z&{kFzX-xG`774)!mB{MZzY08-_X*DiQY?4_^2FvB={$9bM(2A(wfVx$34#^0&%*9^ zpN6x6%65!2i84R%V!k3me=0{*-R$z);Cp3@$yiZOmn%q(XCJAL6SQI$2y(GMoqc&Y zH`;uI+-EVQK~_ac!yKo$!4gAUX_qguhpZ6-o?|ZoFvMS=OIGgOaOzo%nIbDf@mqA zDQXhPB{9q=O5l`B<(8J!Ea z2o;E;Zr)5Q$0E?VEfI+2VKnY}DVGF@IPsf}=P4L3HfP+viiKR@bLPpo?fwmqf8}`Q*Gn*8+3yTy`%NhIfK$;g2bHpd+OQxqNWLE0pUmn~|up9bIK`a|Z~Jo)}WDnv{>t>fPt=>G$X0P_S+Mz8+b zu2Tjg5#qn@i0?j-N@&_2LW~OjmlN|vXZ~}m&h5C?J@VeiQ2T=eq@ek}?xeqki@fTC zb?z!k^e>_$eL`-1LZ&kfBIYveSIGyfn=g!-1pk^@-fF1A0A8%`Q;y$1~ zJ`0te{izo5GlS#sQ;ir00h;EYhw#{qmrm=NNY-1Mr5=7yS3S%Xw{| zNZ0@ClJGwPHUqIcG3rDn={rK#0|F5g#;e_=0EaOP?NTBr;zd zFV_EF(Hp45zsqe}$nAhJ_|HoJEZCL=NHDu(_pN0A&yV~2iP(P*44o&!rJM#ogR7Y& zvz~4*zA0J1s#V5jr^@nIUt_jg zqE-0v^?Dv|^ClvdeV|SMd}}brQvLg^ErqK7Kg+O%`(qQ(fK9BLZM^;WB7XeGpDAae z5#jvXCg^|w)1hevdlR87ge%t0M8H4MuzQ`j4Di9S0XqrR zTr{A*1(moh7KliYjnAvZ?$Q_nq4$ZLI#{7ZQ)s_2nsY&mWj?o%5P;~CwOjj{tyuV& z|7~gM>`MyAdEPXGX5dlk`q&?!hBo|%Ggt*U0~1vKWyXImQv09X@7z=nLH8He{r!c3 z@17I;CYZzZ1ON64wCK<0wBIXYp;#fXE*W{yE&t0_cv@K;No|jSGA2MPy}p9}0%*_S zWj^0Xv5JcStOWr+TyRI|860Rmg9G00ybAxa1ixrt!#o>q(EWl#)W2;y8v*KaUSve; z^!>jLx|8XToZkbUR5V!dEHgU$=?hHVXvho99g*$k(;}Ft_94xB)4NFRjs1P;-HH5s ze>h}0pnnO>GWb6pZ-w>wAq$R(5)|_JFTDNNJrbk_c!@tt@(=nP)cdcS@qC?)3m)YE zy5t3!-n-{D&GWA4m4Jrt>ur_+>4?9mteo8^WG@i2A3tOWH!A7Bc7gKjq1*jJIPdX0 z{`CdzAT{xSzFjkejKAyu`dj7l@W@}M{68-%xBqbHdEvk}iakXjLWVOU+pk7oZ`%!f zR9XHxzXOlK*8v?U-1HT|v`f1quzxP#{quDGc`1+hTU2wn9Vqe4&HVSzD_YM#nelOf z2e{x`Vr8T6D`d3`0hrK2)g48=>;uPW2ZAcPd^{f!aJ;*;y1hOnqmW67^S(co2qomH z6FEHV{`+AaVbA-Z!y~b-mWa#VDD(tqPHI!iZ-cK_t`>#}Dd(i)rTnjPRKh*GJLgS+ zTJnsKalg19E?T}1RN|VBuFPH7Gjx=3i#2W zR<3pQ|86l~sOO)<$&4Q4Lk8$hBvu~2&xjfz5|9a{MIWbmK6^xe@5S5F#Xt+2uE={zo`Pmd3foiVbnjR2y}0|uBXPM507 z^q*3&V(yp>CJ6xuw*WZ2%MTD$=N>H6Bh&UTSs|v0ML@4*;3* z;dGveZO9g7(Jv;^fZgBUO<1ByKN8$6T{U~%B>?QS=+)68K%WyU4ko>&0SF1DNlnZIiY~HbD7Mu5n&*Z7g*WL+kQ;=ctR#$=7hcE#B;)VTZ9jU%_>E z9}2Rcyh+aKi7-BQ>d+oj?lC@W#Zf&FvmxRuEa2&pFC|aop21V`WZXeJaodr3(%}_^ zb4bYEkQ|58?s`?n`bmebao54m+oCCOA=cQolC3oj9hCYPLRH>)o@zHR0vrjCk2SDK z9ti?_&;iDj2tZw2rGL&d$&#(8;_Vk`Kb-cl%XC^&z_0Mx%$15Mi5WCl-LFqb{076% zEPq$a$I~i*fG18tge)yBIj(lX^{4Z2Nel&R@+r!TtO((wla86p);SWqx5Fj4L2tg1FhQOGFu5 zOiWD0)lnvBBTK+z$?xZ9Aq21ZJu^eVP5%R1OxE+N3o8NtZqnKDm@Ie<_VwiYub1F_ z`E0?X@6db9qQEW$AJIW5=-U%K0x3_@IJalJ?_MA62;HPAJ0e1~oV1|)FtJ3_aO#Q2 z&QnpTC8sB`c+7+s8Hsh9#cYV%M)`s|9rMrBczdWiiJ%R)>kIH*Mif=9geMzL$rBrd zC`WxVkp@Th>qlr(lX3x8%S#}e>-O@HZqdV0WN;JcZZ#N$@k?4ZK z%U_1mLKb^D>Khpli_cAhADp*Ggj@-r?heZZ_7Cct9tpTC0)fYc>~@rh4w?t~X7O6h zu}WGqYSd7#>gK&){?6$?+U&Hi(*KnT^){YJ3$itzha zSC@zLHRe;aO&|y`d!_&m8T)Rx5se!CH*U-PhW4-U)4I=;^Woz?zDhNiDfqPo zy8rd!MLu!Y@LFg)OcK3_h<0&D3@!gzaAK=wU`;d z*q6)`8N((C+Th~>t-x-E8r3B?-nNtv~rl1{4W)_ntI0!CW`NTQ!lUe1MxkOUBHln0fmAh9A-;qXj*C$ZB>W^RoMsCD}%8_J$R)rveL3UZ6;B)A#}!>0MaV<>nV43oXeH{v=X6c1(Qf?X@G4~_!W-(5Mc*I zEZEl<)y4zkfY&-~WIBO)0!Dc^x?j+)J{t5yGE|K&p(@QtC(xVOqwWm=Y$`cr63)sS zpaq2#pyuY)ehxb+tTZh?PZ_|br5d1BE-@l?U0q)n0cO+3{JaA!BeuT2Udhu3vJ*4N z_~Kr#{T`V5N(n3Hc?*#=h`Q}%^+ka#0;WT|siRu`3d4z7SXkJoy_N^2V}}xORi8HT zA%vo!q9EZgS^}*Tfyv{`+)jIPjgGtFo1EY;R-i;6kc6* z95Z;`-pfZC9^6kuHh%L}i)LMUd1vahd41ens7?*{xEeFA(H}wSkLKpMFy)`vAn-xN z|4*Df$0C*e-tz#iLg_7@x6Ad+5p1R!j_?Y%b&F%kMtF93}k4x~f|O>qF` zkR3K0&euyDxB*We@v4l?NtZ#RCVFqbwY8Of5vO--Adx9C?;H@7DF8Fe$#;p3oXY&d zBkC>=9v|*%9CsC36qUYzlIf2of$=jEa467^DtED&2#*1`*v!|-EMfNk{vm5D)k2te z3z)&c+Rj+6bed3J&$ZC#6$BEu;dWSC^70}?FU)l8LwEcnmqn&r7DDR}1XNU1+;-nv z*H@GulE9gttrr{nWUY8yKLW0kUhge~TzDWQSj(oTH{b?G>ix)q=0rc_J1QwSth22UDK#;6DK`ThS#7TxpNMBxH zuf(qcxqX?X(B*!&MPW)B7tbuIH3|NjGWC>1Tb3$97!$2?-PEnd_g&qU)$bt!3Q z8_I+eea$M~a$-Ze{UC3_PR1-H9etN9!s6 zM!OT%n-)*$zVlBx28;W0kRMf!^K@fPK&Oh6km2%Kk*lW`sm!H2`SF8YQ4eDp$%JEk zuf4SqV*c^`A&K1%b(^ym`$YabubiLB3q^Y0Xelq`5e^q;pP0Nn?=r@p(xOPkS*Sa( z4i&=I^4@=t^5g&1`D4;@1on)}E~Na3^W)V=?d+l|gJ6p@1;2!N8vkVf!R2@Cqha&H zaQPNiA<1~Dx6O8fMRlNNPp0LV>-%U1hD~eoBPs`TE_iGtk*S8$hMoE7 zR`1&nCO;kQM0lxwY}@?@(a#q?8Wf{U^b_f%;vu3L^F8KFFFAUsp5_@Q8=msqUwg>K^~&$IFI()4M|}0q zIv@*~W0R`?{(jc`a*gB=tz?Pg=uv`Lz$;6sk43(i*h0Rb=H|o6-Pz_l1~q{|mi#_x z3goH0?}?}rAr3sd(=B0f{K_9`)0&`-N7O`h94@xn91jLnI!!vS`*sqE_vKm-+LCGt zTeC*egAzyMQ`e3s!w}-uf*^4m7BsZ-S&bzI_xCGTk%C#RwuhW8t`|w(@8hH}iluR^ z1gi{wvP~(EQB#?3kZR|nkWX|3v4mU0;M^76}VzQaTo0JJ<;tpGG**D ze4#+AYy(a#4X9J?VXCq_JG%^y+mFU)ZaWM*pzb_~HCNGX#UZsb{gWTfQvJyLv?GC5 zp%_x}0;RrpFmNQrM6zMV!mAF`xR;+(MTkvk0NMw=zUic`}(I1=C+= zG=Q=!^Ns9dD?IM9R?l*AXSQao?@EaSB2<5CZXEdqvb`n`>g&la=VfLv0nBu{Z2q*a z>P>rh2ycl+wABjyq960eu4C%j8%ETT9d?<@su_752(I;H5 zylA~VT9hvQn4SZ`FvSQwws53Fs*h<|z*w2yI{-D;*jc(G02m{tWIGA<9wz%XQ5VId z`K`fE<~1xdKcaJ}*KX9qB8$8#rD|sy3lqSHNWjv2bBVVF%_;bj2p3t9S@j&5?UKu33k@1|uGXS(9&&xUPSLlFeb(yE zeoD~hb*I9Q^8>*M>jgG5(8m*W|KcqxfFCd7 zzhmkV0LWnl&4?FNS|Tu~|EK%UbGeDL37LP}=~-rQ#_gAsL#pI$MLpu~Lq+la&axQK&bD6$1URp?d7SxU$qqwmnUwohB99N+B zaoq`ZM5mCDuW+Ktq=Gs{ECj zyn#4}wXHk-jz|d04_rh@)mKWiJV=hC`Kq(qh@7ww<y%FpW!uLX zg^hFw(!x!6D>y=cYQ@!$JTt&FgW8Z*lGDe2GqQACZxNKbwGTk*n*0fJsXIk$clSf0 z7Po69c!cW~Z;5s+l}aaN+R1{F=UPw$Ef0j@T)O$Fuq;1N9N5Y}l*Auda8O83bPl7T zXgqJYgy*JstLj%XKV*AnlLd{;c|0(##FqWmSWZ!WdoJT^jZLpGIyDKqCtRJKB4x}) z)(Ui*t!{@*R;Ka~28zMx*gblDQD(MN!Jo? z&0Q~xxxxd-oRbj^pZ6YF@Xqee=6)e$YY{mVn+^WuVNJ?fx1`bdQLFs04@wTBP5pUStQ!TFdWYsqX{@T03rcyng%Gq{M8Z zZm#1HT=lrklxe!)U*~wYMDd#)^aw5uXnzvvRAMdavOgW~t6a`MDbjAGdraSI6vUWQ}0zIiK|e}SlbZkA%)}4hPwvU z3G5@{8by9yh{8kw701Q`W$+!$OUt%!W}E(DEpE~8Ew_C*wnzSPQSH#M63y;pEJC0h zp5BN4gAUyxpYlNpQC|^Bjb=Mou_$b5Gzgje@-#BFoj+EIdMZC4xE8Rp#$nJBP}0-& zpPM4KQq(kS(?XbkNowP9p7aI4x`q4IanrzykPQui}}w;!0;i5{Co2l7@E86^{s)0qD7}f|V{VE``966?Xt|uJaDM zrfTRD*^H?94rNeuupr-`HevR6F=P@u0E~x3-3HZOko!$upXyJW8vn||(= zP)M5IW;0H^hQd9dp-i0%DUy&k?1>IMnu_=`(Zg{}al>oeL9b*ki0O8}HRhl}Htctz z<0S4Pbz#W9uBPk(Hs7|<2|=nU`T*|e`ZzczO0{hrBf2k+FM;=ZC7ZnM+p%Ro-Q>|C zk}+GIlfmUDZZ4DWmc$RHRv$wmWA7LLIr+276e?s%cu{X85-s^F;0)39t=C6AN@$8P zzX-k`y83_xdw&cB`jT&U;1}6zfsx|XypwxgA9ZpFgyom86-5aqWmH+Beqx%ZGKX78 zjNn3)En7LX1Y+0nZ-o-pCFH`9&99C#T5~)zUK>S2&w1Uc4H|xpfvCXBOlpmeyvg%* zFei#>0e}2IjJRUigwX7oN*1P&aJwC2_n zOJ05~QF@6C+WP?I;R4trOL^2@8{Li+!veW`OQpo8DJp@a#^z7XYdo@X(j57N0-kH+ zp3gVjh-vDo)Uy)Va$%H`J9e6{H!CtpRW-#RXrc+jRK*QV)B;5+hvB>FsvGOUl+m-v zezQkN9w)t0ax+g<13Q#8TZ50ZKI$CD$@Cbz*fFjLn$y+yA2k^e83B0OoT`6iuQ0Li z8@FXjfn4Q z#e2WtqRn4i>ky-#!d|pnZ{iMa{eHxCsV090raPR9@wPeHn$z*D2>vDPX-u^O0JaN)p^%a``F}ysh zsI4Y`+nLSxn>Q|nqWN+I<|i6>G^b2ecH!*XA2OKm&#Rxn@d(TVQ;M5yYLn1xwG}@Zto$D&qTp z`e}u;3djzW1uM><1y6i8OV0uzU~B8^R+j>XN8T(2#iqj%ELgpULj#4Pxg4MaeT47Y z(?ooO=PbmXcmaW#$UB2QV5YJ3j+4Ek!VDBQq4d#ip5)-)gvygqYbukjH;Bb*v8M3c z1!WKWqZ)Bs?I*i^L1M0Ce9%~CFfzLlvJNdP{0l@6nLQ%SCR^CAmf1{Qcts`8ev-N+!tqOXA;JJ%@)h`QA+u8vOq)4 zIXg$V-q3OBb6H?CGF=^X1m*{iJzI*00r{}n;);JetU?8t)H5h#;`;?*3wG#y2A_b# zWnkKrooe-ZLkr~q;|5F8buax%r)cpmwJLCX)axNOljVt8xx?RN=7>YnZ;c1~5fH zXt{ikJ$5snV?|9z#W7If=PikCt}|a&V<(2$0hC!lGRo2n-@-#-AeSX6PUd1s+6%)c zHdEYG9oNVobBwY*uCuIZ$fGue+%J{xh^R*j1M~SV40_~&B8Z%!ImR}hIh8P_#7`T9 z$U^9DNFmiBsl2rLi1rzO)D)0z36o|Xyng;JA^5F(41$+Nzd@Mm!}iVI4(`?Zy&thz z-RIHOSf<)nY+f+?j6f~`PtFuqc}xVUcH0)TiM-cdxS-Ge(_bh%op(Y#NQTk#+d(L@ zJ)i!RL~7D7m?JVD!wXOP(q}Qb=(hki+|e`DY~(dS?acemKWP_KIlg9>fI8BrJ<}_b zV_uIl-ycFvjS0-=@nAOM4%wWYJewVSYK1ezGPp-V>>aRujBuSuN{ul@dFKsQlGrIC z(|rywNMuIN(m!}455j>daAZF-iuXXCvZc!$;V&42=L&6oII)1OxB-oKl+NDm_}7fR zOS)(Jic|;(LX7l-R?=sVFJQCj{Erje9t6oMp2=11jDJx%69zR&p zc@Yk#EFsaO(7qz?*n-Wx0$UIs1lG|(EL{NHe-L#iO#(u4pyPvyYwPo9dX)%%c6j}z zMyF`zGbxg+{2jI5)bgm#lPVOz&764Vfl6lO=@ilvneR>T@3I>#F$m-90GY<;Do$J} zB83kQuMqhj6*qWSU76faM$|SJXEj3z;tj}cz5;r>8p9zhDN@T-P0zM-AZBXJ7g#(Y zx6!HO#=Rh*sev{L1RRpOkVe|?c(R-*n6XxE6hUqb(2a^|I*oQTD{Y?X!zvpeNX8FW zuz8f706^A%voC`2QB@$L2wC%!!;xgAUXM}_mH?&=Uj?KKCnu-rd_|_^qcq6}s?0lW z0N@>rr%x7`6c{c3E82UUQ3lsq-RT5`3i??6_%pi+X`)G*Xu!!(h`olPWs}&1=3WqR z5hE7_@gq0NCtu;GHPu`=6zS_>t`?8r-5;))kx+%O-atE{cZyk}Q486RyJ9T{Wy`0BO#mD8?tstmq+F;x_>k`z>jwJzjPSLpSx|s_L%D>n9D&% z7p~6?l8s?spu0ctcQ2KKBX73l~Rr;I7ai=+6qo&DzSFXkKKr78bMP%+Yg`!#~rV$5;AI2cQf?L;J)w`GRbL7LRm? zn>3Qx-RFXZvUAGdO(3G_H|k3dF&gdf2viamRj8-%L3&K@fPlUoMX!0R2oaY)6Dl9rZl8*8=>6Sg%M~wk87~T`g113+q)p? z=NorrwGlFJV!G1V-p-soYs@UQZRNFX9cP3Vnfu$&^&1 z1Rl`bzA?A`&EIVlfA~$ZrrAWPSLl-Ir%#1=#!_=3Rj`lAWlD%5bs1kwU$RBRrMDiA zataim=zbJ3&tI6(dA)QuXw}%9es;OONt9fA{`#GzK&7nugjHXrO|$R%pSf0LuO*N@ zFhHiy05W}x{X5f&HGV0mrz*e{XQlO5wr@k7NdljwqX?IQ)+MKdmJ+ z-{!4yvnDU6I_`#gD=VVx7e4>|`7VXmSt*>JZ)tH+I`>9A46jh?7^8eXN(Y>8CKFfk zK236)E&cR9eP`(&|emQqYKEAg610kSI+e^HwF?(xzcs!tmPrDbk9MMs1i)Tcj;6u$K+-u&%JQVwS&*x4H z&G`o6EU$W?IXa{X=May@jO3i8rzQFGxG%0Zl&>imjhPV8yq<1$zgM@z@w`y|8qm1z z$=L{Vl~Y0F!6-;)Mloi(&n8<3Cu=E6P#RptOFj6|5T+5Y=j?$HcHVm|@fdYZ&NpUw zW%$wPEAEIHiV69pFSNUJb?+f*D4nynFp=KuSl7nxb`b7X!odFYv_bQ&tfP>3?YL5i z>frcw>zay_b4|x5G7nIkluSpHj-wX?IG^FKj|VbxQqxiyRGTh#=D*0bm(&&pA6j2+ z2?+0W61rbXAUy_AaNKlI@{}w0r^r|Yo_Xv|s5nkE$OrCfcWB}KZZr3ne`F-ZmK#h5 z1jw(;eq7>3C)s#mLU*@Xx^tFNbF8pc3YMP9BuX%%s;INL0GFhR6}E(Y$UyPBXY5jJ z$fEn+(xk2BC<`n>O6{?>*cc>Ed!WfhL4njU3BiNT)ojtS$}Z zCNfvq%O63O;q>rdLLMDTAY}8;!P2Q#7qKU7WXA9W%#CJGQYB=Z!!>x=$j(6I?ig>i zmp<B_aTuz)fucMsYf!!|x!5&p;H#Oi?Oo6jHbM(f{pkosPqQeDV2OvuTpsY-z9 zDtubWmBFsDTv1=SUX~ne4BQ0>_*mL=ZkilAQD>#H=XD?C8km*fhfh%@^z`(V#(fQ`YZsfDzU%YmrOkQPGZL)0$h}rHrEiM&9eu zq5c8HoS}e2gN%dY%O;=kI5D?TEU(+!!75W??P^aqo8pl#5mRecl;ha9k{A{4z)uuG zhX2yS_xJ-s0owhCV=LnS$kEwob)Yf8?|MW39*v_R>JXGR&2(MG^_66w{=*1{m6&`i9N`T4;P9^W70T=q1t%iUP3&CQ(qbTBQq@pS zJ`}_OhkBUqpKU0Va_wXii>5#aFzDS*>%i@7zw05?Z^o1E4hoCtmRM^e4&37~R?BZ$ zWoY1bl@{hS><ffHX89s`xo=#^8_+L5ev>s(#`O&S-n8pwW zny=*zT(am~ zy>8t?yaEjy;Ug6kD7L>5@wj_Nv+z zE^1&0!=GNSUf$UnonD?7@Uzm*3J8U(-S9&7lDi*Ab@`ihwN@f)1B(9UZYoSb;9M$q zc(_tx^|&X*q!)z)1TiP#3Fz8<%^-ZB8^%z{HhLpPs`=ba(!EVa0DmT6;BEd%nYpbJ25 z6fJ#|V=OOJY2WD!8=di)_Am!ZUes zWz^B0@+COM+;mX{Y))i>S8bLnJW>XyYi$At(}iR)7@y!`1m%5g7GZ=oo^RI@wj^%_ zp0Chhx&Zj|`;zhc-X2xu3_DeU`E2CUa?TSlP8`yLcz>md6OVHg!&T{EQWZM2NYFO9 zQ-a7nHf)#(6Vval;l47N${Q$d)v4)i@_hB+Rs}xy@v)4Nu@ArkfBqr<_R+)DhD`k6 zk{qTF@?n!O2)q`My@G@H_xvjf&6o$uaLxAsRv2sn$8U?Mqu~hCxBG%@L3tx_S((^p zy9?PpMLCWh^l-4~v(IFRgV&a&WS|&0DJyX;-gFCPlGxXUOyMBUugc8vwWm=_0Pcqi6Vz?mM~q`u6~;KX#_^DZ9arFBDGD=fTwAMoE7dSL z7wHYueZ=z!+hqKsx;_6$2cE~{MW=n2!2$Uhv5fq`JJHBf*`vC{hiw~8#WxM5V#c67 zAY&G^vttXaSgkUYY54}2d4!`os?ox(QA6W`?NIk)Vq(q#0dmL`p>n3w^q0YL`{##i zCqQ3`$8PEi=uM3&HvoibzR|vz{4%DBp2n%|cGXp>A%$gfO`5bXW|Md2@!>&lvpa}d zwG23$1=wr_tk>T$Xx6_YJ;fJ}pJO>{gZ8U`se!qgJ_67mYXGFGv05X1s|n))YbcaN zIDuDkGQPGm#U+rNIEG(=Lpy{;Bo0^}*nff(8PrU)i{kU9jJ_6z(-&%_$+KCGn z0O3mcEgeg>zmpKQ?_3l9BCi|?=`xUm?YYnOix4muaJDe(hx{84f_`t= zhTvCVS9)N+-@C#fE9b3GAPY5gpLp2oU0ye6w%LFWY3N||J zJD79b38%|>d1|xFQpwT?!B?ZX+ei%bh)U?)E@_l5b`f1P8}QMfp0hd+LFc|;oYo!u zSZ2oJg;=5622`6H&qfSK$8_-_&d017jpu25VCTZun=37S-5f93F+P8QH|EQ1Yt(KW zVD$1>DP~1==p$-r{eT>?SJgJ}ga`RRg)7V~NPTb!x973uUoBIcxl^AoZu%tCSQQ&s z|Ab;AIrB8y1@B~Z$LlmR>j9eqAG;ig1EdqF5AKf}0^`-sFjtGLEW zP`sR|W4~1`EC*mCO5#>88+?fpXJ$^zqHN-#)GZM_F*La2uG$SFe@fG2blxHez_&{l{?H zKJ(E4K_dj@hSgS1pwxQVKa)iTgsrZq5W_R9;A6z#L{wDNk4vwFgoMwHfaDo?(4kmaSsN6+rhRSUMZKve z`F>9{+M6{n6vbxV{DA~dvi-^l?HdrF|7S9{qNb+1KrwsTHmd&)&|wLyt0yg|MB20Z zD%I_>*EL8vn`o!=yQ|k;E}E4O@GJ2y)H)oiclY&8>1U$un^Z$@%+;7Ee=WZ_>*M=l?|lbK zwtsNitvAQYj2aVapVRJk^|t|mEl0sc&mk{Q(ONCPi4FggG zEbPo*&9M?g{6(5FS7iW=S=fbp=rviZxVjP?RQVlb+@awpO=&2_Itf*0JwJtv5_x*t z(+5!5zBjVYBO>OYS(~Jkba+6nUl?BM-;ub5o^`&nsT%o*m(>V}1j#aqr6 ze=^n_%-&|8?0=wBNyvGWz8O6zsiD!G7`**v`Q7j9X_6 z+49d9QO4%y=Ic$Zrdw|x^_s(7NsL*DABwm1%o}=Ob!B^>-Dh=F(C-l+lISD$2-Kac zLPYi@r@&l!b=;m$72Ps7pR%!)WIs0`A>8yj%B*4JBjxDbY|~f0I-5sVC&GaDUbFxQ zexWBaQFMR4St5CQ>Qv6)>=qccfU8_f!xvusb5v)xm3b(YkY2Q{-=<}chw&Eebm_3b z^lm_jgMWKhVITUlkQ6AH0)WKe4f;)k!oASbBep8w@r0A3An%2-f~oX76s7G4Q2mjO zqXz&SihV{c(61;If1#JMLq5i zfZEynJK*dXZ*g@TF>uxaNR8XT0qnds^gDUDAbp$-eo zA{|fXvH=HiqHdbamcUX`Q90wiv*3V1M2zcx1&rCn1KW0b@50EmOjw11?7O4+2N2TK zjvxl#gMooX+sP;61|pmu90V7s6nm$c90webUc3Oh%V``M!#IyA-ke(FvgKxH<)WG? zu1Tj|inmmVl(Ym0dr@_a()(oza#-YX#(7JW^TR`_G~q~5Fz8}RO9i# z7;nLh+&+1?*#!6Gr#!3dpGN_Iin4S;^SUJDE=QYmy+Ld+Z(dpXR-%J zqDKy=<1kp5=8y|dsFtT(LE_%l%jTbVqq4=sGN_h^RDMgOu9oW3!%Tx2S$6K-sYs!m z))~MRxy&dak9065Y~0+STc26S8SE<=N)I(hFs%ChCXVjo5APSx5cq9O#_-Z2LEGhu zctCnsG*gOuRP+gp0aap}z@<-~_A#H5ysM#sYWY*}d(p~_KdzgN%Oz*aN1TQNGan{x zvr%lQ6Ju}qbd1R!2^<*TId;$I7T zc!m%WfS%9BXpct7tqyD?@oyA{6|>ql3EMzmq0wPKq3WZBdBI9?9DxSV2cozgd!^Wl zj&u47rV}Lp+!83sJurl6bs;inQr1VFf2nisY!MMDbRxpX2v3di6s~vtxoo$wT!KV9 zYbV_8L%cli28!O3X?)=IiX|dNiqmK^BMeVlfyAfNurohrNhzKd))bBf<$11q%A&M$ z76i>VpcWufj4MCxl@q-y{2hXfw#ebE7`cx>h>hwIVYxLt!{=l*O*(CT9nA>S`jrI= zEaEg%s%7-7WI{8#!^$Nc74mMSqLwy7eoecOviPu5|M}#e%k4WvQPZ?YOk?T>r~!%) zO0%zUs0jt1C}Dg>OVVjp;FmlSC0V2AgAEruAI~o#|1}g~jOJAb!!HYi4 zK}7Az!-w{BD0NY(xKV6ldJd*9-OUWE6~r*E@l$e2v6Zg9$I`0?^CLXg z_L5OSW@g|t5YXG~+_`hDXBWbRR-FiV1*@?KckgG1m~i3E z8KHxbvJnY*5CC#zVoYl>=J;`97WX>KmGz?(n(vtzvG`J7fnPDGjGrSa54S$uiUvr8 zNrSp`n93Qc$<3t4q%HHfl+@a+H^5i)#))kvbBOni5P=e!%Fm9}ps%OL&_eF{bN4mC z1xbVd=27Yko9rv0J&ujz_=Hl6!T+#NM(=lfI9EP|h$^a1#II@pqhPq;{170LqDAEm zY(^@}w3=#{%`*x=%l81tO|Ebwi(C!KxAvc486)o=z)^budX`!qOj1@>#fa>D14#ba z?vFPWl>->jibg1$HLzZwqaZuEj>SdwU8~uTR)_r{%C~BQdb!mdAMhE(Lpr$R)dqIU zC9@zA@rMF}d&6XX?hSeS@m<6uXpB7BWXkd@IufB~AJlYKQ{_MSJ<7?`gd3f2HeYF| zzqgm9t5@NdKu-=@`iIThz@;&4pzt2UHrME{`L}mh2;fPu@BUqHJ*XQCh6TV2s5ga()LBL(KpG$zcpPmllxyD^BBF_h!>Qp z>0z!I=npfeB+8{(*cPHxfctrrcmWUY=hBGAGlt^@<^5TwKsx?ag%BNw^5LFBy7^qR)7OUG9JU@8TW9)^}2_ILX!?oGwOKnL~iOZ()zAALQsC^!gMZG zE8pg*X+|u`h1N(0(EHN`AP!vdp-!1>rE2QP|iJ?RF{gTcNV zj!ZP#5b>^=ARN(w;wD6o)?BP%8J4-IgSMo#N$1-xIzb@YqHNJW%LeT z?4y0l3@EGaGqg{$5>+`VXy=NBGpH^&qoy5+G{UzrW{iR{_H^9O$)v8v)T00ti1rlQ z4%*Z5i2#6lt9bB`m4;5&^ue)8oUlb%sO()RG#lIrmXHBcBgyRyW9sn8$8*ic4R7Ap zr1%H^xBqfaLL`xIWcIYS%k$y6HXK2 zKWk;R$6ZMBN833mBmTjew)3M0>;X%tFeu~I>gw`d~muxZHJZ_itbx7VN$QyfJ`qS(;te=Queu(Z_P{ty@#7@1>BIu@hX zKm(;Dy#}<C-6U$CKf;<+A3q|fmT7_&KEzzyC0n(AJz62x`xM6hzADIk z7TehN-L+R)4y8Ok*accaSbx7HP_fJOxVXBya@YN!lc`7TNDoZNnb(dL`h)*l%-t2w zYx)!xTj{#4x;iGGfj7*d0THOmop|2L_ZK^tWLLTp!Sqk!;;FzRCCX!DV%Zo7hpHJ_4UA|%v~jri@k|WEZrI1c4Q3qje@U7Q3&J;J9OyO=xSwW z*qX-^Z}0vOr*S8$cZv9BgL}3HdTSzlIbF!lU~jRY8#^zqEC#%w{DHeR8QA30-b02H z34{Z-wk*J*WxQNpIm#g5c83DwtbpQTI)7xMGFD`Xf1LLJyWs3j$S3-e!FUoy{q6B{ z1|#uL&6K~~gMYP{!Eat#6~IfIKc7d;sfhr?geoDh|G9f20x=)p34gvrxB2|{b+oIs z0y9i4i&z7XN0EUB;ZPzBxa4_jII?eWKn}Nr%kJh#q6J<^KIpGp0oLg3BeC`Ci@W)9 zJ!)-jEe@A8F_5maW-OEn)*5W7qGMv10gj@+smUz&@~uFA8_+Ny1b{~l`hu}?oZrD) zm_tRu%)f!VgSBsE{dIf(^4=?y03!>^ywiiYdUy~U&htZoOT2;@6(#u3wE)dBhIo(# zo60|^UmlOo{K?bXObn0dd8@hqe6h#lJSJ)L>_SrDg~D)xr4ir?#^At#2~*twQ*7u+ zr&8n#he=D!$M*ph6*Vv@C~EtGkk2K<;dpTjj(qvemBb3)v|`e@HL*9w)=6xK53 zDRPYzHm<)ruX$+;1^j*{2FMfM=AXIv{31S!7R-&WM!kjv8eK5Ixc<}KrAwN(3IVHM zM#KP+VftM(p+9Nfz9~&8+LCe6+ifW#>vQ<8D*+aoSLSU8vqbonK%)$m+il4rm0^U| z!i*5K!NKIUYCT*J`R_)C0}Y%2TigrGhv;P{T7diOJO2w#i2n0Tq+I%x{?FMdOh~}2 z2F`GsZ~vu3`D+r4R|*OfRvRA`e!6%PFn;7()Hi>ExK5D`7){Y?Y`vrYFFu&D4~&V> z^kW9_a*r|__sAoVFB+_xB|Ym_7#rp|M8JfK_OJ&Pq$<|Qybq; z=E@AwlE2+E2x(Cvy4?$TY#_)5w3-{(|NV~CynsSZRd+x%r88Ms^KH(;zr!B{2LPP& zEg-ZncCl{%tGWVKC*urY99zyRwv+z%QvLl)y|RSgz|pJ&9qmJkR^IbnncVz*0o@BX zZC%z@Hvl1RW!4S(_cO#+Z%=w>P!N}MGGj`}zd;MUWqskF#eXp$6*yu4pVzhu{Cz$v zE*pk_zss9lSUiL4uNB~8&Zq^PClDQ}0iX;#9CSc_$trj#o!^a6wXKxNm} z!G;P*6HUVLHQoa7pFaa+&!VsGqA5m?Q;|{e!8N{Wu?%M6RkJ&AkO1lf&Y~2p&^rlb zI7|-*!p7#{hn(z2x~tC-{S`+Kj}I@i#TrRJ$O=wlVj?Li=iQKqTXJ8WWVFmhMy7xP z)18|`#X7X7*}#9??a*8BHY+)_(Xs@$OW>g}^xC0}?2jLH5+Mdt_; zjE;kzZ;ULuFV}Q#mM$YwE8`5>R_4WhkDcRzQB^B!;_1&0Q#e$$(f{70U_!6$Zct91 zLHpz)g5iiXw!4qtvarVyHiiLtcV32_NhOl@s^(Bu{k}m1lBNF8Y5DJ;Gdp3cq z1K5=34GS)gQ~%vldTIrWx;e)7=MSEl9sBZ%Esv4lGd+G`$X?aAr+SO8@!38`?r!V` zzUR0FeOaMYxbr_w*am~y0}|A0{O5a`FGW#`Ym&Taz{d2+7lEvOFNV2PpUkzo&U3LV}3XwUrQQxg91$A^G28 zCWU|Q?7F|GE&SRDX?rNzy3f0NdmLNtP!xWbe96H6_&^2Z#*>YHPasE*2{>r}_npLk zTjV<+M>_54+SF+)1#z86xTD_ggPE@!b`)i0YyX-;+d{FqMI-Hz$CWVy6(|zL;nz$h zzkZctQXw6^hrVU9zh>dK3(;|f*DkXZ=MI%YKJGd|m-uG#J^Yx+of77Ck#@q*#7-Aw ze~M+&=6u6FmO%xN-qdB=G%fSqwltEJ37i7bD_XGI86!FL=RWy#iE)^;@C0@KI^%^F zGU}@<`jkRj6=HMIe0y`~c?ck}y#-7ownB$Y>>HYwmRt!amU;dpXwn1yyl(DvQnUS^c(*+lpmc<=+f_ z4lwEVk|Kc12^Hi=K?&}QqADZp`1DSGa#DBa1oAOlR$-C!aZ&qX%bI)@d3s5*0g7F* zR=7j1epZ8?f7VVY_3KAT@hk$AMK|qoa5kfY`GVZi&UhgnoO1bW8{lT*20ZD#x zs>Z)P(}RvJ`3_=uXvw5U;1Ib-tb4)n8V&1o$!iQ60bTTX5eiwvT#uArs}0Q%lH@sA zmyAMaWnDT=*b5)b<5!(6d{V<)=cg9!kCRyNKyNnr#9c|Vv3^Yl6BgtjlgjtPX;P%^ zia1hUM=D2+3iIT`@Q&&^tKw?sp@Io)@@nec3MhzcLdt~e14}PUova&7v##Z_Ty}a%)%8Y;XEeHCOM55vOxu!qS4qWCA#tWI)vgS~c zcXSSb0qk+dVwXyBXWt=m`)eJ&x=j*t+U2UBg|hqzbt@-0S0n+ zjm+`aH=Dhn{n6~ti7eD&a95;1$^UTw|4@uhLaz%iqOs(p;jfkS^^I5XrC>7>8Eq{d zBqyhL=t$SolW1^WvGLAnvvX2%gQO*VL=7LOT!mex#Wu1f?kuu+Xh2D0y-*R_Xn zIq)n@FG8BdWaVjp2eH^+LEM`j@$9fglpSqK@_4);Vd;f}{-0R@@o(>|9)o)Z_R_;X ze`RR~aGL?CJC|D*^88#~ry)7xBXyA!&987v<*8j>ORD2#-?f@)H)*=_$v32bMl{b| zhg<}7i|gy{?J15aCifT93G}^<-_V6w_V5JLD_{1x3|uW>$$%gBb#dEnQ?>S-O|P&k zg)0~M{xk4e8s^Rx-}}p{SX{!VuEsAnV>w#*jK8=+3RRagk|_PiU@MZJ6Tbg_?}_HE z;h{NJy;Wt5u3UQHRWt|5`Z@9FS_LR(_Sce^KE(Nwwv-8yvU?8mAPH1jro|_9K)PrV z$5n9oF*3qlQIuGXfgFDczJ5mJbOdtn-wBag5e5eyHLqT1=l%00q1UEG8#$^Ci&9Kf zJPspROT%|UaC!5&VnL!UNBkFwSD|Os{0vZH)q#*?FhWnqfbfiXi4t$!9n7!=UC~t9 zb@C?Z=11;&XfxpSX>fNEx%QO=#&0J#vQCv>W!&zBJ$+2Wr9KP#D})CJMsyv@VC@tM zS{Zxzgl-H|2DD26*-+aF9DSc%d=v=lLSZQ?5_YW{gZo?wE>f5aZs45Ed0T9c38}kM zD?X_-RpPr1Uq-n=W!UexXM)dz9MUxtT48^w7Q{K((~5+S>VR%{4i?zHwWC;G+yH{+ z6(6P}mpzL(c z_1~I)DyL8aompg`YG7^;2JifV<%0_qG<9NQKz_l&3P&=oaiQ!EP}16KiSQJg{&QIH zIs}@ZwpZ(TeX;V7g?+Rk#Z4CNb$c+Oen zcbwFgQb@1L$Zymz(fjZ?g{q-zsPE7CGU;yGz+($$TdqX zccgBQ=V ztXxZ2JtsXUlmnzhghSeeI6~4BZSS$x8Nj#8s`+WV6($lNmFi&wk(cK;{zPltTai#g z%bocn8>!%Vl1qqp43v=}x2)YJSkUZM;3GAScQ@|#Z(R=hCIq0&1R8#@R7s2>{JN5$*le7{q)9j*B(WcI%}MDQ0Vb;z;8 z!aG=q)ext&P5h?Pq~$TrjVkg`0Qpv!WK@h0-1_)zb4}Yj^o-Yjm~sA7pkLKybB*WJ zAch}w>UEev!N7qPzpqxuFYa(ei;)Tn`FP;H=`Z)VpJ5<-DYsfU zEYHx+``pNED2buAlA78}KGPgE;`N4{NOz#jOW*B)%KU-(`X>{MighLe}?#tmHD1No~bCP!AO3Iu+^7Glf&wdT(n4I-}>4^h&*qn41~ z@SSr*)#F$Cz4wIG5qzR^csJ2Mt&ErC>DvAlG@haU?SD23JCFbNR^~hn%KTw27N*l)2{Fm>9m3Np z1!)4^rF~#9fe0e8D0((Eza^IN(u@^XMCi&d1i{goEK>sHonWFTl{UyPrg+lO%;`&+ z>c6_~C~C{|*M|*drM4GXgK8Ke7FANl`-&6Q5(ZvYw2QyRLz|wqm(tbaZU;!}n#EmU zDilYPxlCIinAyVHpFGzwbje@ktF};9ogP5i02McCxbs>p?McqNa>y!+u@Ra5eMO!5 zz0j%7U9*U%b$nECXnYO3(y4c{#U`Gfs)GkjgpIa{$&BfpfnT)9Uy|ER0&iJ9M=qp4W#ReBzlI!98?U)a%$hp? z%xJ5doI5_BIgNOLjE~!WQ*1weyx&Gm2mBWsr97;#3|E;ex z=J7Qj?XPrZYCk2XkLE#GqP9vV<>N)I?IqBIvDMMne&*b?Gs{zYTZ*kw&_ykLf@Zwz zh;pz&v4bZzb?%W^nDA!Es`HM<-qP-irr(vHi-erE0Z#PfH{w_kiNe-ZB9fH$r& zrTmyeSGbj)i?0OLX5B4;HYq~Xli95HO8KLv_XVQnPz#>@z;JWR_7g$=P0+0dGARX! zf&FMaPgQohAUTnW&VOPfh^uCY1H73G6*~rD?*MasW)PRYvC(Lx$U`nvz9(~}?tJ|_ z^`^d(;IW-sB?m+LS)a*7YbNA_`=KSNcC*0#tGKeF5zZ*l4cK)L09MTK(Pj0mKUC19 z-?v2~G}xarGZ0>jJ!8okFDgD(f51)+E%nqAe-BO9%vBV%Xt_h#cC*GyMBdwt4F;~G z;VLt23BsgrYS?OWyx9&B^gM|~E=z^&*3ZPUQeD7fKU5XJ^|IH^J3kSs)jo5UZ+lM~{%+ge%hgljS$lSo0a?tx z52troA3ROZDpBE%rPYG^@V(T}-Vi&7ifVInOd0jZJvA-Dt zdaXPV`*v4qlA0g)OqA}*5B(||?YGHeB@ah*IJ3wSf>Al|;*BRX=vdM`4z z`SCzd8pq{BS5d65Ss3#jBZDpmpJnU;XT?yPduV&ir*3cslXVSU+0ncu$$?E?lG5pyk%54g{HCLSxHwL zjxUWLZt4!2{Z}!31wC&9U}0Jw7TB#u&;5Qp zJTZJ(%1kr!`iLtBuXlaqISae#MvK?jke!lCQFebXImwAS`N%$&A_5ez5Re_uQ}%Ql z_*mXN-t}?@=dMCt8Fwd$cw191BEQusvmR4Q@vZlJRZLt*io!46C){okC!U_dp2BC< zb9(Sq;EN7Z4S$3L)BdwpZ`m>~A2~pSg6a-*S!!G^1Xw2v{?Hu}iaoX9Npn~267JBX zVg&fhW}|ymR!jLF3`5jygc4$#mtnvJ6D^Nk5c?P%o`A2bNSsc zA2p^+!XCXqOgAMI&D>vjxj*bdN*{)_#eKlIPOy?mkLO$j(*Pd8sTkxBTp1REShFe%bwn z46YZgP@ILCGWd1f!+?~!RoULPd5yuA`QAZ|xAoJBPeBfw6b#jcgMnVsM>ER)-Rj0U z;T7pb#xaZwTRI^SmGDc})Jp{i9T|u6JAtbrpQ4@N$_r@}jx~gioY|R*uaRR!K@LIf z^4GmE-u1crbBv6$el04y@T>?&Malh_qQ3ymfzwO4v1=yrLpr9Y ziCASy|LRp$P$#FbbIRa0#$@+*FWko#imxf=xv~(`45%xqkWn5J@M+Nn^7h+xV@42J zl?vY^Ie6fX2)g0Htj;^V`xQr0EtqrN?Srs?+bna!38Y<#89BOswoK;skLA$qvj`%J z+A}haoS`|6fmr=P)=~AMT~Q5c;ZlXUI_}0vL&HZ1{CH|^Mk0A0w7KKvPtcdBWi~)J zBjhFdFywpiX~avqQx020rtpN2M)KN#GwR%gfjYAu_R8S&bPs~Lv5-}}OyeuFjCLWi z$oru5hvqxKMs<_ktH1GFqqgTGWUHZ5l#w-!jB8Zgx8W&odX>3{?2L1OJb}XDUdG#e z1xQ9B$-n+z;8U6$+~cu4w2>)=2ovnHK54h(ZVfo6DM{eYjPDsjX$KBo8RXr11p1@F zSJDIx4YhS4H730ty!nDzt%<>w+7cF_k}d|`W|8Q{5#PXokW}gEb{&*4la)%(7>@mu zLfM1t8X{y%^tG0$b;SkP8$H?AAMFrQCzRl>+$82gHZhF_v)f9dt%mtXo9Y?m#F+^a z*THe9B(u0dZl{`=q?C?_#bRz_h2;iQ5zuvT!mb76DRX5ryEJ(wA_?VFKC= zx2;7=NQmCGg4rXq|X3xhj+WPc<%eBHnNurq0rT*byPckf2 zC`NWIE7X4U&`Wv)z_p0K{c%9~js=ci#k^70MqWIZ!iraM$$>|wzYQ82wRR{Ex>b1Z9Q=qosic(}ohd#s6Fo&nrI2!xJGq_v z6KGHfMD1CHru$sgi+%pOB0dJIt-svu34~kw1ci81GKYp=40@`|h-b%d}F*|xDuPDr%Lcw+pnx2b?sE8DCe*H7P<;Hm4QuMb|a>=tA48m=6 zSr`5N#Wi%kfZ2R=buXBWoJP54dz5WAjdOml{*A(cKWF9#)Pd8b-_pdmDCPyAhT1*_ z^^ZfU1=`z^E(gGqq|Xvi+A0Od-$0JnF0K$P*q%m?5=0pH(hp5@zWNJ1VH zlBlN)w86jcIr$HX%cY^iPwN9x>B=)>6ux+BQLx+H9fzVw3+okIpivQf$Q+LZZr_?E(Oz|+VMf41uxgQ$ zh*=`C@iU!2Z@+hd?k>)C=0gqoqU~8LRW+rGXa+4^MnK2!t~;AqL2>XZqN?d@McMhE zE5$cObLgPRHB{3iJ+fBqch(u=OWTQHSkC}Hr~m&Eh5;Xm1ioNgZ$4knh!y)A-bNo3 z5?U<(aYrAM{J=il^wP8ebx4j1VE#N8yrWorv#LY1BTSj>2F^`+l+02Wd7$`A+4qK{ zRToR5lTgWnlQ+Xj0@z9Z<%Wy-orXEtjm|#0aIwZ8SA?bda0N^~tr!d9OTl++haTjlx7Pmi6zmN6jQL!6r=N5_m0^RMp>QIfc;K zdqR{KXHJ^P>%l9L;~DalVKR4{#CcYCFOCMDhl?`sj}55RJkNs#;eG1@7mb*Cqg(f+ z)NIAwxdj}9v(IF|W7L+zm}gFq)vHCC0PL%2L{Ftm2lJP{p#Wx%ydk+@O76kG1dj zB(z|#`M~mV?uLVXj1xg7#`BVypXz1Eek0&F;Mn(oR(~sMFrn2?z^Kx|eaa(#+S_#I zUh31%uRR~zm(A;UB6Qx{6|~Q5joy_G>Np|1qmxJu;8hTL|AE2hi(Tv7!?&!Nayp|M zYXzPBDs3JLV^!n=_1{Z+GxZ)B7|sUiEz-&{E?p(@cM(ww|Qbr_#4okE@I^=L;D|c4K**Q z7X9b&@Shps7Wn%s_SI3Veci(LP>kKbnIg0>IP|Ub#t^{N%@>C(r7 zGZ-NKL7I^GK$yg!RHI8Dq)=30ua5ynOuko-W#7D6`76tEGb$~X67+)4#_%-NR=fcb z;QU)EwJ~HS%wj{-x~K}D_!MHB@KSvV%kJhuR!6Tj6rQx87IKD#ZxX^ZeI`7Y8GB_p z`ldz}M*#r0k|i0qI*4~{yIxI+?L<7$!eA;;dmxVCTSv?Ri}l@$-T1JOIbqWW?mgG} zhDqS-pNBc>L8t2Q0=4_n6YZCw`oR@)qvxdqr&k)#@ucSvB+K;8t>%nBVS{*)OQ3W1 z>&<;-sk>eGbL~1F8U(yOs6-wxRHT3U-6K~_H~k_m52$z*r*`J!BzIrlvVqvBzK~&k zrEzs@CO+qo=$dPfqgtvmu)eteE}W+@KIe~W)y!%f-6{P3)Qc1iv}h7s4LN!Szj$Z%TKJ_cGOKwo#@d{IY)d|OOm!KO#%`)AF_^CxL zms<~A<5?QB&MkIC?N9p;~t(;;be}7*ie?Cl`g*?*qC&eubBK}@ zao5lr=a`NOp}Fn2GnrF3T#fwjmm=FM)vcQjXWm?t|HdeeG76niV-G#o56UKYcrkk-2;5c$i#%#3iDJa*}z9-M_2?ui)qDz z$mcQ2;oOh1;p(@pogHHMP~@UL*e;M2C5EBi=^Lc@qZKHA&5C1U&e5El`onqd?m$V5 z-q%Lmt}V)Wr|8eYCpJrs-20+Mb<5`RwCog6v2pYutyGQuZo@*#(_l^P3m9q8`uiM| zmTC3XP*Uyx^Ktz*GeJ?_TvSLy8hs`lr7HEk@@2B3%R^qTB1-NLD%vYjKMVM)RVzWF zfaYOqYIV-q$i4w_4bN0Puzgg6IJMU#ON=qTEqT-?% zDFaEA&~-3r%%z^be_F$F=Lg-UuI`XuHS&7eg(3lB60)ZoJtwC44sMEZiU?}=& z#2k42koQoa$Vl?d!S%O~r25B>Q!sG{) zXb{MD?Z*s$=Kx#yv4O1OyJ8RJ4QT&k!@*krhsz&W&9{cU4~P9G4TYA&@ax%N#$b_u z#NCGcuniCWYwVV)DoZkrL`vL5<+p<8&pjkGd7vAXLO@{1Vte#Q{qK(g#bTGcv$-7Hr8{$k zuB8CSD%3~)gcyVwXb1dNL*yAWT@@QwYnX7N@eGHvyG0ij1OCe*&bYdIZ{|cid|T__ z*vkL*QLK@Y{XaCEyQO{)v0xAiO!yyMtk^y7PluYLP!TPQfnj#)6lJzX(U`Ph1@N13sIC0z zn-O=YyA?m?k0$!1iPM&sS$WO(Vek6#>->G-w6|d`r7~7=dUoe*pE}0` z@X8!0kC@$+4TT^Xq>|$X2>Y~;$!e=+2xCqy`^z;e&J-dj#M381a`GsDf*-ju=gqubG$oJuCXB9tHswR|88owOVBBRn-Xy=plAbDPl-_wfvF6$y&k51#=wC^k{S02@jZZx_VhrJLG z;s(T%WmM-?=A&P#rxFoF;zBd~bYt1bgMNupyFYm3mA(dK-oMK5!|j7b3!gUEc@`!x z9M5yp_0Pnb6DSvxXC=A_d7SCz#MN7!H0$Z@I{gMzZ)vlr#^HA`%r^7L=?-eNn{t>i z5oqEDXY#JgdUdlk$XeN)g;PZkwhXOr+*T8PezQpBbV1X)vLX_^rflDh00nFkYfzCm9L)3BfK*> z3MWKL1vfoJ8b4jiPIvgabQAGQrwU5)(;arqjJklQjc%9f)=_WT7@VyPNULdT4#ivs zYxo9fK)m{_I4TMi?e;DLfR=E||kuPr1WPNwZ9z#q=l#o4U z^&95kZ2j$UW3p`M?K#}B6gMsJz?Ux46e8k=74_&@=pX6K>l#Mg7}z`gOn;})g@Xk> z{i_@pcP5=$HT?3W6hHaC=10oHBul|CR9&O}3G4p~Lv=||(og0$o;J=JxDSpE#8{Hu zEmMq5HJi^n2eT8518d(hO=XET$w94BaE*L5YrloD`>SOJ>L;=LUe_=&y&{T{J$iOH z%C|3}(=v6Y#IZ#e0G8DMz_M|^d06?&o@{&!G#%gcg|IyYZx0ZHIU_+RS271NWVbTA zA%_vT%RNF(dM8WLBxEXRQHVW#+NMgl;r*T$mf{v|HYe|Jx43t3AcB2W%E6y`z@jj9 z^9Q2W+g%{Ru#!63wU(ykfZRd8H)#qprje$54EUrQrX!!#F4)Pll^oY3i$)^QhmE}9V=}T$2~ZDEeBWrI zM&k5RqljS!L~D6i6DlQOgxUq0NcJ=F(NC`fvH4qdyI)Q@%DrG_=d}tyMD*O55uLAa zfkfl+$Y?P;f<+WU<%{ z#|Dj`LKu?UrJYN@M`)a@HNk>APZ*6jQeez#3l9oyjJl85I~ncB2er(5NaSLBw^)2; zxI)ou(xJ&Zy7?y*G^3Xqp*X~Q*B|);ei;NeJobq7_>Qi2RNU`vrd}hdI`5V;q&3^kKl@oSZyUm=O~DxTP^^#!oCh8t zGysbNzn0(mGNaFw5bf{%M_kGs{~PDeZMC-GU3m=V0k$&GB5XP+d^5f5w^UM8(|$jc z=CTTP3|_-E*M3lr)1qh@U`y~?=nx3JrN4_0?5TlJ?ki!~;QhBl8pRZAp!T?K4@5dj zN59B1_D>ZVbs0BmiHi!6`uac1+|4+3Mr$WZ=7n;a;16|e3$dbHjPPHXdpSg@;v1*6 z{~n>^>cArF%%j8jcB7l2xX?BdM(rAJPgRIYt?rERZBR#D5Q%|7X%Ku@vtomLwtfhU z{D#8qg^;s{3t7)%%M0HsiLDjB#c(Fo6U(7NO?JU5Uz27pliG^BTk`GGKQVGRm(mBK zle5dw3G5zMtN7ypM|7^Yc2+E-ri_eSbZf^;JaNYnIo}n>4qhzGx@QWZ_Bc|vEr~0` zDt#;2GlqJu(Wq zMS{~Q#t`^#kNLO@W$F{CMN(pEhJxd_<}z`o(2P*HNb4wR*E0_rSSyE(-P2b8=l#X1 zxG0G2X8T68zrse4?sSpk%UyLMdab4<<9%5IC9EdyI4pe`Y(i0bTEpZ57PIhDvpt=wU>~ zT6qsTh)K<9#C?fBEbY=B2wDtCHB-Y+p|FLe$a5jJMB{H=Xa=ro>@xA6{o{srhin?z za4-DD>Ijf@6x#m;PG8>e*9bC_NTmJBSG|f(n15eHt;V|Me*RpHfI=w{KmC4&I6M`> zftO?q;LKbW8JmLlyPGh&tWi&YmBGmj`u=4^!@_tbKhg>I9`vHH#oV$GMIUN{okUceh6^-}Ql{bA*N8MQK{lUTzT z?Gm!b)WP?Leg*5e&;*|Uy7adQE_8Er3N%LNy2L7BC+;Uxc2+G)=i9o*++A}_MP&iS*P&5lnxUEjip_AhN@LF3RD5L{7( zP{beE0c_rP2tF!}A_z~$!yLj#n``{@?~qx!sY2;>N+L7oe$Y2t*}uO}h5t`;7Fj_tQf{e5jNaFCQ^9#7guJ(j|;Q#M6RUi>b1x zpf}NP<)9t%a!gafNqJ$Z+(X1c*H%O-`H&@?1f0bL4R-q^rP5n+^(%fOu1Ejg`A@-( zNix8K&weBeGV6DuwSXzbPAFoMLX{Zh{En*u?k>G1_Pk9WC=W`pmN?{197YM3&Y3qD zNzy;#n&-DfMFoufq9@pCCO7h`{%ZN@>fV@VGQQ!;W883jibpllgV~l;FgIK#{8n-d z(4X`~cG@4rHcgGn1jvD?(MxNE?5#;Ewg_}zP07^Ac%s`#_z&%CuZT1qbfHEU1bs0; zR=0B99{&MrKYj&Qr8{nL#@Uv)9MD5b7W*^Uzp^kh*FX4H*;RGM-uvgd;#c znX=p!y`ZRCY?0p~l;gsbixP9iH&_%HG**F8kyU|zD>~3T6)(R&?AY3;AhkwA#7nYO zFIT5vId=Hnl;$5?IvEnrN(~hgK91?o9Z&KRRO&HFHi(<;@(|;kwVgswh|q3|791q z8cgK4Rsih3yIEiChuc(t^?~tb6<4(*z@xQk;M0TAFH1VD>iN3)a+D*P7M}n+JZM%x zHFYr4<4hj?i&fA|L8f#O&h}f7g*wMKN92lNGn7*Ss2h` z$a}^sun9&%=Mg2~2sk^3iivG*WqCoC^y8wFpE-ofK?tQZhWAIaD7D_mQmh?KA~`t<-d zZs~W3US55w@1*U7nK|@-@zk4ns%g#g4#@N`&KV`gT86{ILoQI(UnQt6!xxRlwSu@p zuOWcs55S{{IK7m#bf%*okBPzM(&aP#E+SL$zd)-HO06*+&sw3~&FP2|c`buJF;eU(HtPJHq}9A@jwzcNBh+Ph5FX+8}ymrf|Hvxz~= zcNJ{m?{yi@IZY2eTr=~TI6$O7L*4U-^{v&2bsn-kPM6Lgibpq#=n}>=GkfBtVjrgj zK>1Yw^$CGdBz|j|kR)o0(eE|sL)}j6?=Ku*mcA_MDkJZhb&S<+=`L;TOpmEUB@7fI zpg03@2Cvlx3p2JIwc@c(-YG88_)gu(j;r|s;uxW?{XDZG14o0kU3lg_kGqZ@y{~hI zwFurF7TKF3^BM!ikA7m82wR0Hn|x77N}w26Yg+D#w=xK&O`d9PCGbEeJ1fh&p4 z^XTd%V6Kd=>Z`*@q1i8{Guui98*KK_(`I*1M@4GN zl1cS=qdZiof3of-aHt?8j{3#R897{b$W+Q zj=(?g8r8a1SF{CwgRb?hq)?QEd|s@4EQxOhZ;u2S2{5Bc5-&7i2*yBf3liBCdOK0v zV;MZ7LSS(H{2}5D>Dr`?7P>!4dX{}FYk0hd;r98tW|L65f#=5kt)7O*OS2=3B5#`~ z8ya^K_`w8qTb9PYs#m?Y75G@>xikf_zH)~-d>%n9>_-@$`57CTa<0+G(zQ}m;A@Id zj#C4L5|7}E5b;T0z}J^E4>sVS2*U@*$@%#EBc_b2rb$I#Wt!KcgHc_HX(|m;YlB!% zztVHpF{$TGnNGcTbd9D|6^V3J?w`Al!-7K#vwqolNMBK9h|e+HE7|;Nj&PV7rDl3R^anZSNwS``R zipP7nmN^Cc^LgB`Ji$LpBR7@2cAwQwO}eIbcHST}v9I2#e%qGeKAm|NwmL9`qCwbX zKDo&JFKD0P{lo09n=iuct>q#0f|E&dI;FmA5r=xfJJrfxnq{kLGtr^${&H2i)_>3c zF!6ec6HhbbuVS3KYO-4AQ~av(r$qx3e>gXYnDIX?T=Ve1PhMHCe8xn z*P(&}Bas93s+G(2^a|ko$j=;eF2AK(w0yjM9>yAh$-Dy%;hSP_Ey6^fao{{6hQ4Ew zASYcG7RTS_@Fp|kF;696BqcyXB8!BB^=ki&Ib03$M*N4p~3zIMlvxeh7>NorMXwO^Sz^$&V z#g$lBCBS?_NWCr$u`B7s-1WPB_u6ub1s3!^>tjY&;n_0o!31V21IGHXEHLcbWc&Fz zHmngy`Ca?ouxFh)v1WXtOapn3x%#!^f#23>d@jZvfOVB{<1+NcyF#Exl>yIrNig^L zzfts~ax}>%%9ph81+=QO7>M~)P~E|x7HwR_@`J<8P?rnm8<#RuTQ;<-)}heS_yAvd zg;UF>Z-#XKq@yRNr#;-66NS&iR&_fw5$L!Ww_2|w%Je1E+SD=f)z)$j9v`r-l#*o+ zKIhBHR_ET}_C!wms1K|H*zp8M968uU2@@R;cpCJN1TsR%@-x_GHftwF-(NGX&=Qkn zhU(|A{|v-we)oR5oHo6=C*%`lNTdb=Gx+3#N(GI>&IerP;!{d~8e_bx^wKlp+`5vw z`8Vgjx8qt*4RzemgK|ji(Y^^Pr(P}LHp*c1JzOf@64Mlh4<>&o$_b*8MQ;8oXTpjyiS1M|RFEez} zicvA*ujPr9mC9Hu$&}-n!wt(03H34bQj@P%gV~AnnXpT5S3kq|CLcsQLAj}o*uo(r zJ^5aHli95sCtbq!c)yZ~A^9%H)+P&RNTG|zB+9x4tgE@M0-Mu_!%FBu8cr86X;=Hr zrnt6D@DK9S%*-Xwiy5$;Bd#~gmbpCravMHB`2U z2I(SxHlbo}i(+uhn{*3x;(cIeUZ`siZ+}|GPTYIQ*??@`5hP@ooy+N7X~4?=FO|b( z-!vaw{DdpQ)?F(6Sv}&yv$@hZVZsbr^K<7RG)RF&?VZ%!I%}!&ko?jB(F_{|a}01y zzV&5zL!5I8HO&A;ciCS)JTKnZZ5OCvu^Q8-vZci4hi?c0IIpS~iq0p;4WgxYUF5DT zC|@lir16~QH-DwNTmPz{uhOB8jvXV+cDff8KxCEP>e7|Nrpa$wcKX;STVeIV#}=>N`^scz-isR#eweFT z@@S|1l=f$_$PqJmK$+d4Ck$fBw{#}}0z(>KmpnEs>5(_{d z!T-h2;EI2yVh&|*yX9}#Bb>A3HrbjZ1q_EIXE1~T<}`MZ-A_UWVD5de3PUQiP-_?v zm8=P@?THAQa21Ru{?$~DT1Xvj&|On2dowmCYP@8b%Aqm%f$b1~GVPOe_j$vU;2-Q- zRAH-RxBQQrKL#eNZ!tLzfgAmaaf%X;U(bw@k%$Nj3mqK| z?dgQP9Qy7>JCRae_}b@tV$YDtSZ*KmJorgJ6%9CLDg9W2HS-Oly5DfKSag zuNRcX&7rJM{(OZ(x^9jI#stmFUz(yBtTQ$kz8)^jbIZ*Tu9 zy#iSpSigQ9f~#1x=Km-mDujj%u3g&FEYBzu*-t;^B+ReFslkYK%bUr4G_SJc}kRCs5ACD~@A|a@u`F^>1dzz^ydyAT`+It5DgU z>#6AbLj3zpchpl4s_g)exHRBv?gjrfNvVGP?Cv8=6o**yck*iN*>-=~4>(GD z%|4}hGaZ_FvI1T-Qhlwm3(etcnjk)O>sHXd^%Dzzf-E} zxV2il?%!|J1*wuDC5c-pWe0OsBdlzx2!v+*ps7#D(;wj`mh+y6%yhNYA1)+Ai+Ru* z_Yf^~sB3#2WPjUofR(l3&+6~{DM#NDUKTK+^QxtCW`WwtBg5*CqQTmBG=ScgQqOY8 zMl_4gNj*NC`tJLjuA%T}u#QWI-*cU3A%Y+ShF9OKi+_uN=8?X)IIEyPPZ*kyN;!gG z+|kh173lY>=?(KWa!qk(?fB4>kw*QjxGDBclDKQOxOFi)xeItdmp^7M4^&Dx3g=?C zcyX45t(f~=G37z4u?0vggqr?|f;QPK-6h72e$ygka;xkA%%i?q>^pbE)hT-Rzm&fE zI_2Q^RnktL(~vVrf}+7ILhB<#Qohe$N>0+DdhMYs+GoIl*tPF(IM>{k z8Kvm%MBdQM$#)xNhij+P$H6R*FK^#J0tlrFyU;k9(j0=`gW^p)3}q|kA}JHMWvw|` zE==T0WP6~XcxL|XVe-4WHO$=GufHEDy~_NRnL5={bO!q6*Xr?cR2&ee&{0%(ecz0zcy)+-Lv)`$s@DPCwhK*8WGpR7QeK3G1nK~ z8PrJ?^bXi?5N`yXy;Z4nx2~olAYP24(vIABgk0%vb6cf4NrUDL#AiyH9blfQ5Q_J` zH}ZL}YRwDlQcisauRmK-UA;5#IWozmJT> z#xwFur2Q@sza2>u*7N6+#GAiX*QWtWYEl5i1)hmTVS7i`x3RvvQ=||MWU?D;=^#@8 zo^?c9heAO}-~&c7;v8Km^;Gngw|7q82%6>8F7~35ptFVh+xd@cx~-cDe`4{pe!yy< zwSQbC5RDNpBV&6Yz2bTv&DK$0^I?4E*zr;&P$P$=HA}QS9VM=`V${^J@6ck-R7aPx z>Pi2VqK5V~?30|_|2*ZRG-#8PlR^*b+S-6>;7hvEMObDzuqNO(u=ao3zlnLe`{wn;X)WJr{SyHOfv9!i1y^QI$I6yIN6;O$;_TPXA_Kshrd8@%o9qmL*IRW6a@L);T?^+O}bTX2MT z?RBx6ZIkGngv3@cM~bHQ=kC_MIG4v~ zz=a-EJyx&uD^gt0W@`<;^(K8bqaCLud;!(+e);wWu{!?9%psaRSLi)ypr_LjtERSL zw?)vY_6~M!Er$oS@|!VZwaO1xE0gDQ7hMjWIJOF*PP0j(5?$+G=$5_?*j?iUZE_Nd zM?X@lIm~WZ zsi_du59BG|LfC(fF}v|=7>*Ux+J0XZvq6f!src3SzgYkaiO-hEsy|4&TV-g$?LUw6 z4)ZU4@>}a1{m-|Onk8hc=kb>R-3N_E9x*nJU2+nR1Wj#25KDKlP$T4G*;=_!|D9fs z5P9e-RD5m-r)iu0I#f6TX}HnF{m3^tif9LF=@s&sqgkGZsPveOWUY?NbbZcZN3;&| z@9f>s5RvyDHfiCA8(T+)TJ{`HmeGy1u8IwfF+gr68(^-Zwp_=Joh#+TXh`u2s++?J z)d0(>u%kP+JM^cFRTYMLm)l=YH(!5k5_BeHZ4wc~a|!Ow{(SUZ1Ej^KT!V5}Vj>_eLriyh7M0{cZ^O{qq~x)&RH?m|?+C?I3Pbu&5QPwIIXGHd3$u|hmrr|oe>HPM6jmzqAe`EEKl({@C+=iNYKj*-e%jIH3DdR`yW_bG~c8%H<|DLa3am zuvRN5DEcVN(LUfFH*4G%P_72OBybPSsKvK~T^>qXBKA#t#q!^{7PjJ^jUdJ|K;Oo8 zL*x()X=U}-?wrzXn`2A5h420xQWAH$%p^8(DMy{lGVw)S15=@@J^?q^K7Dx}FS{}f zoHS^lNYBKBy{j^34k&WXKkK4nCR1xC5I%RXv}~#Et%DC@xJYs38vc_UNxi@gR{u9^ zBUV#0GG1!hA?>{|9)5qkOhDkKe9|rAm-72o_J5BwI$Bx6r}+i0AuEn|6fl`_oso2& zRZ9tq4Yyl8_dAnH;ef7hpSLS@1;WKi6f8@Qk9g}{=D%c>4GuA$t%@5S3S}HifGaCc zfm>8B+#mRI7bT6b0=@8WjkT1I!)(qs@BC)xvTDHC@!t>3Biok;^T|oA|9s;%)v^LR zTpaucNWr(*Q=OzZmo?*Hz-DVvhFpu7rw*cl*@oz?q0a3LNiy&kB0-6|+no$Bd`qO8 z%7i7$FOMtp0_gBTn^3TTel`v8F>nd>+{1?DL3*all6BLJk0&7@DTs+L+5QOo#l+O& z3;Lcl?Cw$H`)pp5os>|k@H9`|nQ^*9*<5s78`8{k)|c%4x;i>(w?CF}*I+I5^mO_4 z+Y!IwVP6N(LmI8|H?wK}D0`?f9Nx?%9oB%BSc zpkt2ipF#TW5P#g$rVfLkYm;qFN>qYaZ$YlfnQ~zm)hyd%T2eb_o9e*N4_DDMYPMXQ zvPo1?$=5V$Y`w$B0XK`36V9WVQe+r`=6OM|C$bEZv3m5Ko2Rr{=VO(E4n~@F<$0;; ziM4k<)mu8uZ}6jaEspxQxM~p(1#3clnL= zAncVGfviAO0Ig^=tMuWIJ~`Em-_~k@*sGT;=+l6So)Ds{vG03D)>Em&haCiP&v(8vHi*r*mPCB^vF*rZl zsel6UqTo{c<;=vPk-qhWR#?Fz!mFRs2RwdaNP1i~k{L&`kOvqO`1Xff6mnP^9&8EE zRLL$6aVPQ&?zOiXI7T7Gokd9?uhKH#rXOleINfV3ItRxtc^48Aw`A{d5CPOquEsrxay9VbKcox z+#GXstlV9R0~I=@3mw!hTU>jig`hCVG`YFX7*iDZAf2-~HxQZJO@5sZ551fJDj?Uu zJtgeLU{JgEbLM>S3PAW}u*z8mxY{1sDN{f-B5+a}*|2?-gR6SZ%!4Hygs0*<_J-9S z)Fst3*!d`AboJ8ENl(WvU=;$3syArWy*$*br(v`@gsI^^|_qo{be>d|P^cCXqDuyaI z1-xe1H)@Fd4q67bQackr4v^k#Juexpl!*^2oj#Fp6-j^Ar|Z6Bf-|1rM1WHw2aVnH zC~Qx@cMp3w{HQS3*w|-9!=IEp4L=0u6Yv>KXt2~tMe3~_OjYLuO0XI5>|W+=Q--s3 z0wN165YV#18{}F z^vFkFBhe!TGPm4&`53m3unfIIFnYtk5N|cu{Sx_VMr(c1H(6=xR`lt7RF5$4;gOo| z!0f|jN`9FcTB#qzZ#K}OY44cJ-&Qe-(|u0B<>8#`&G?nz<6O_wY+8yqj{t!g zpd)(!Yyj1>5OrNB5E8V+z$<|Yv@tLhLT+;(puA}Fts64@EHgLmvmX(K!=g;LINkbi1wqkM8VjnEpe>!E~~H;kPwxSXX7zr0(+& z#wiHXr*G~59}uHIVU6!h;WBIYy36~bndvIH?>=Pq6SKQ-CQz|A8CUVzt)IHL zoAX1+%Ek~DTw<_1mhfss?)9Yu=|><@BZFujC_W5S zARl?f;|hAtIx!!%YA#u7Xru}7kAx`qr6SM-T=zb{EH5o`#K&v~n-%^#2s$c5rFQ-0(D+;9s;!7~8s9VuGUOV-j-{Rx`+}YE)AB<%b?FlFGbol&) zxQ#>5E+qPounhm{G}N3UUu0iejqn++X(272J*4hY#rn9`1co@#8^gQiP5L1=#`s}` zyikqj$e1?yOLWLIL`X=?;7lcH^kLs}jLy_$8Fxfj&?q$kN4cn~V8UUyS%yT&kO0WrB<^Z$PKtO?iFw zOQdAKa3ky+1mCLg=99lIb0yzG%{bQIlxH>`Ba$S8M^Aef0oU)V_<^4csL-T+?Q216WXj*(&De5t*;3?+#nZMiV!KBk(?v!= zBZ@1Gh_zg_1a9+l^o4(N6Yonez>E4RG^&LkU621F^KkI`Ib=RAey#6tmsau0lJ#K> zsi5;$)-}g5H58POImjd~NFBw7opb&T>5nG9G|5I$mGwG=?HC1=4bw8rMc9kH6;pH5hkKX~`HR zT|;FE#W7Zq5QCoh6VCbtC^{!PrMI8UIhCk`b1u5&2aYiy-2cG&CVWjKXF=f~Vk3h- z1#L@rMBviIP&W5`1Zb>|bNg;t9VibRi`(%2gYmr(`wSP<{ZEtP^!LeeTp{82WwC_O zXcY&8627Wjv#{Yy|4^xQ2J*0{Nv6eC{S?a+k~}G#E&XqG80eiPd*g|_q!rOTk(re2 zBdFZB>eSz$G^JQ~qi%wu37(>#J*@`6ZEQoC_aseCqUuPpUx|t0R8JytNq#%%meJ4{ zgC>;-uZXBf<0MyK?3=eT1xbR^|F-q=K=x1n#tUO^P?AropC`}y)yO_;dww4_DDT{I z`G6OLJyt?SK^dXUe%pA>MxA3)jqx~2k(a2wb!cAkl&?*aQZlRn#@L7Wa%gWo4m&V{ zP&=8vZD>g+)6T1yz$KuV{kj@jkEZIx!n$}yG&(2@ z3HLWvR61ZIk!7{u7JjwTU)O}CY3us=I#lI@(ARkBbW2RdLgI&s&X&kzRL?2fr&vwG z_X}X@C6_Z|_b*pmR8Ytr#$V64%H;;^H)gY{PD*(FSEBj~CG*jd$?+Y{pWYdI>!SWC z#E+Uj5-Oxv#3V;?@;rXudin4*<|MZ3<}2xJKH!n&xybgRT{HHN`z?%+yf58Ly_ND> zH|AQ}RVh7N;TjW%4HHkTs-oPz-is9ozxhM`UG&s9%<=GfA{97lKiH+)6N&LXP=OF% zbRxQ-7Nb+=_t}RL0wv2pbPHPV*%X$v?{>4RLtB0NO; z@Ww~uXa|J0FV*h{cC)j(;Yh7mJXf<7GSL~KLKc$0Ih;u>Q?F_KTDVHogM)iyPR7;N z0Q@M)Y3RZ4y2Omy4$#5b7u7cJRoYA{>~6a`nmi@srXyCt3jU{_VgeLZg)tj)EJiv2 zr)600wX@o%b|@itRzwf6O2L{PWGGLJWT4K`bwHE8{$RC)hi_`M@$$N&6;8BHijPan z64LTb@=*Blp3=3{RQ#ByFJdb&H`=v|Ig%J=!SKA;Q8ieqePd`y#M>AAW^cjB8b)}Y z&=FP~7|rqw=2!{xT@k4Arvgk7N>cTZ}j6HG4n0eZJ$wh7s0d zu2n&m_xf_{=vqHEMp3fw!EWj^}Ld$XgeJw7Waqzor4_eSgX zf*#e#1{B^WAYUp+bHrawxK6E?ysP}p=3UYCE!83b(TV*66FC%jOAlPGH7~un?nh22 ztmg~oHLnFOSSw2+bBE54pk-<{I8qt0wpPJEs0b)a_S#u^SeTk(^|DWRoqDu|!AEh1SzE|(p9ElFPB#EEz-dBt3=9UiYntz>*;Pckn z4u%tcJiDNk1)v{DLZ-dpQC{FTz>~i=E)qbTjW=l)8dvLaSgZYuexG^>%;20Ima_z` zy_@~4BuBeCzqJL>`}pxK(b~P2fs17IH=nU_f_)k;4=|3}yu$W~{UVFKxSY_W7udoj zlafSbb-$5fM1OaJW}Gd380a9!V4L~_djSkcSsFc_``eCtAECs>?*LzR`Enh?Jm%hZ zI5aZ#`>jtiS5a<+_^P|ZGtKXJTAayg=_W#%Zu+6EzsZg#i(6h0%{9!NaU`&Uw;=1Lz0gcE-w~vw`%%& z1tnRddoY)fHpFE-mTfy&$Ew$a@;1FuEqsViiizs~o{banwQNasW32 z;2MewH+gfNzk79RopfJqc+@v|FU^u>Mt*CMBI{z$<|LvO3 z=qInB5mC~qZUr1<##MH_URKUWM)F|&-zgez4E{Nct8}>Tm3~{z$$Ix9zXo&oH6-D` z@|Rq5t`xGG=BrSmWgnyuEHFr8cCY*_a;uuw%NH`=-57trLHT*0$eyoAzH^ePvV}U9)w9ySuxFpuycO3=Tnq6WkpJ3ogOk-3cxOB)9|% zF2M=z?q4U*^WOX3?-y$g)3dry*Qv9sYS*qNIbR`v%-`G0SW%@TvsrSlvj;EYnON?8 zPA6IPu6_FDIDo4F~G|wW228g2_aMr!C~qp9u4EWViMUfd*|7Z!dIo!m~`-UDfiTMeuwbHuYtN zoj<+CeMhCC+}=umAxB8_?zf>!=a*j^R4`uWak9^$;uGlZJD?x!`LFy<;gG291zz98UX~tZiX(tj?sL~%Q z?}@`Z#Ug|+E0(?mFfJ4GR96kEby7ez&jqpwRy^qkB|wn2r!WA$W;Gv zA%ARGBlCmWUHz^W|BxPSsKTn+j8KlzqvH&N-eyGgMYrB0*q&-i>%}KEB2XA}`x8VG zjy_(H))t$3_NF~kv}RdOWmL=v@p4OHH$3BX$qD8Y`%QDG;zZCRXk> zYSBFF_-0g$s-=A`BU>TV#@A0Re<)AeX%PY1S%Elo`oO)WV?^$FIt35A*3q~Vm2?%Q z>%}7}ylAKnbGIhvvinmA%Q_eMK(zBNU{N)~>|j==N!UF2Mbxx50nCVb?MkFuFo!^| zUc)T%l#IW15vl&N2NS4kVR)_vU%1b0Q`Pmd+n?)A+eHJkNmHu(4a)^&W6@7I(+Rbo z4O7lA&?2gR1M122;{{Ym`)~dus)$pbjh+V{b8J`JdO^|oK z`ejd4n4(EA7l&kzyXS1}IuvIgL;G@C6KF5yb%PDFozACsmhD;&bzIA>m6|oYN|F^59$SrM{GRcBqS!0jH!R)YC{(}zcWj1IPZ%p0IIQ`a*9(h-YlKDhV!xiKML6&TagovwS%OtqWgI$lz zOUj+an_-3_Fc+cJg>P#fKPt2usEZ34gobiBwSVB|=ydAnHb{8VQq1cpPvd*@o?}KF z$r_}#I0*;y5{6LhlM~F}q;Z~w{{F08@LsI^S4|pHb9gp)Ihz>~mF!L;Kue0iOtj_z zBAfyRfwBX4e%(a6q%*E-of%11;yqm=covb+{t|~77gZN}SWj2%MpW}#jDU>91v%PI zUaV0uF14f@mDa5tiuAZW=*@T)&YVs}`>s^em}Y1-+TkNV)qZ+^%wxH@vCjDNcwx*e z1g*ue+&;2k9QXXY0`h$jX;jvvd;zxx~bZz zQi*Tm8;$PEA|3P1P~+aYro=eI-kp(pe~rElmo*4W)f&~wvoR!j zGHu+u>M@rz00)L->~zSRj@>JI#(N3Ktq5YjmwkJQ0E$aDpV>5mB7>7!#KHm_<9pt5?at}D7amC zzX!=~>_4l6x`^t{O54m+8-*Bu(nAnLrSxqLauN?#LnI5Ysh7w$H`8oe+r0E#I98*Q z;uFVkMhhJ6R-Jn}XD-$gbaOSD)0w_t5DJ9FbuccdbFNF;Cn2>$C~0pzW@BxyTNOGJ89j`BJ6G1ot{`KR){8oyF%1REaHV za*kdUln7>b-?QFnHo`r0X|ldNuxD45$OO~mmIr-`aTeS1qS~1Pug$vAwc5ETUkf!` zs}`UmM{EvXYaY^kSX8D^rGo-$<`!P+{95tfNa!b~CXcZSI$f3-R}6@G><0vHeSUi6 zi5fzbkq`b){d`&)pg}p6!eGiA=}K=hnb*QYYyF0d%*=UL3c>3azoCx*A2|AlH5DK_ zNtOH0aoljB6F&VqGt=oIL( zMWem&ZcQ~GNFG`pCZZE|y`wGt$$Iv8v`gNY$-~@#5~;_AvntJgqo3SQ=`&*k!$oVy zYMa=wAa}gCxA&(gw?=wwOmUlfWPX2%2dDNsFQKh=N8U;)j@uQ5|cc62cu zhkyIBnnYfbg>rv8nAe=%J$p^$d#}Gh5_Sr-;r-|P00AL{`k!T}Ugwd2dmP>(g#S^p z>A#mMfJz!@*^)xp)wQt$*w8KeS0t;KB!TI83!rr$T*8o?ak9VyR}G&T+cn*&M1nR^ z3JaK@n^^0O4$Q`6EqT|0j%+8YogPknu{vAHGBC2Xe(cl}0^IfA=UYkZQR5J+Oy-7l zJ^Vy468sKP;OR$fT!(c+C?--G37IpVt>l*t7I!Dh4j)71CZ~oZwz+L@eVKu9V|aib z;d0w8z3HS8&qK}g18f)3(qS{2a;o{i>l9EI>mRl>@=RhijG!0vSi3$f zmr9(L>)k=;P}?AuByS!jC;v?iY?$ffIjEntV8Uj@6s~6U&TJ)^yeL+5t8oB9z&jLs zhcjOF<4Pl})eBqoB`G!;0?f**far^wZ*$ZnRqb<z1!sjv|1lEgGn}uHRE& z96d~Tx0_^ih_R+msblQEvv@mPPY>)6YovbMVWt2Bs(!HWC&6MxvK;2@LSYcoWuYJl z#%o)W`&h&Sp-x3JY40;s$9YtQ^?!3su%M{eQpS@hHWG{4@en+HU}ODqkXiJhwclBx zm&A4kRDabjWYw)_`0r)PeM9;0hDkmq0sv{%0@e!BUk2hoYeOxhz72vVmU>=)DrB0tCCmOkfq{e4UIm%g?EH!+V zj4(G@HxU5N&)VAWPzN)n3qrzjqRC{9s7bS0Y8}^#?;3?8x5}*#ODQY>~U~u+t;{But>{pTUQ z`NLR2B4$bf25EvIiuiILyZy7j?_Hh`bE|%FxD3s{@_QFdfI$%bj>w#4P{fHfxW%(W zRNF%`8%%Z(jH%Ed0dh`Ey6DJwqFQU@mUR|wex8o7<%|Me%(EYFb9;W_l zz2$)Q4z%k=$^FmG|Gn8-PWM%%4l`SYzv%`f2{l1p;fYyaHU+qL1yc z4dYcD!%~3!9`u>Ajk($Le%Gy9LcLlhX*eFs7s!i+4iv7PCnyVoaf6iIP!t#h0%@6q zag(B5K5BeG*DUt3RPMlP<8BOu6-bwwQq}W;s$PLapAMGtIE{8!a`3xb!kbX{@YGQg zA^*?T!N$Z0X!X$qPA=>drS*jWYd-$P6#x2XM)%-TiU0eoqQIq&?Q~-ir~ls~h*IQ6 z{8{vJgQPJ578VwDSrBLk5CNi_#7|xmcI(FZg5-Ok$s&q$=~>i$4KSAC8JIyZ_m?Py zVYbW3ka z4TVWrL)K8m3lC)zkKIa4h9)5Bgu7Z;1L;@JG}{gk1-Y+%9-zFv$Yh1;#^%$L40dne z3Ca5W`tS|vM|-nD^&QtVB44D(9;|iz^jBcbxgJ(DNl1x$(hTXZdVVkNX0cDC<)T}6 zk#lxV;D~#z8+ZOhZ!BMm9{b%>Op>;<&3>Hp7T=96#>WOz!shz3<}1q;$OjdWOlwmL z;G2~6+b-Cz50>DOe~<9I!*Q9|pC{z7fe8!@oXGcdwz;*{7i-730`_VEN6epE95zE3 zui2wC-f;>86yYaA*=Uf2R93_U`}Et%PAQ9hO66(i#2=l|sy|Iu;gi(l^_WceIChB+ zasxH@>Wh{DfSV3K{mkYZ4r811!U-fcGh#FOMrQb0C) zL!+y_Q&>K0fIre7m{`H9fn?95Svlyv4RaqAg>nrqnyrG=xwTLv8CW8p10srLOc>bt zV8}%CAJ` z-XZ>|i&4SYt*27uKbfKItk_KBcjJl^&H3`h-(&#PJvD;-{QO+o)D##L1bu&f46D2J z2nx)-z9ARv?U9K+-RM*BgMzZfu$-xby}Z1f4p5rLM1Z{ei)>~B!!Bl*&y91X{6$ZD zl=Tcf$TgN0a-a0BIyZ5}n!6=mV=(+686sG?>q9h@4z3AXd@2vBulbO($4JO~{!F+_ zu=nxwIGf2pFWnUxlT2l|wFz61R8f7nH)BJ_A7Nj{K3F2d)_`DQMf__xFbp>2yK{E; zyeKlfpOyAw%+^ndj~+<g$@b>PB(q@>!wx?OM__IN z&gUn{$vmGal&=Z4YpkZl8G@Fx<1oKOWYSPqwi? zWU$1G<|h7e`W=ZOHM_~T=e-#o!G-qev)eb(SwqV}rCrNz}4HTUM8@M^Wp8ezI~(@{?y}ye_|TUVyr6kE|gv zXXuPn3Z15ElCkf53-V%4{I}GC9Oc>%U)d_6goWC~6(o2dki9k7famv^Z_=;#F;-{x zwW20GA5W>#UpoVemgr>mNSs;=)+R{xxn%3HEk3!Ox?eM zR<^78pCIWN0f&z6$FFn-MznveAtsH%AOo-l1D~0%0O`k78OwDT`OiVk%E{QTl8kZc z?i;Aq#qt{%`(ut+NlG#j|Ij^0B}qZc{yHH0?VS!6*SjDMREa9M)7JyNW;?8AUaVc^I}`CaiOLiAACAIq?BV}- zloako9UEw>&9NZniIgTz&#N2_OapyfFV!4+Trc_k@fTLc3U*mZ_HTt_V6he&&2&44*E{!%nZuIZp+E8>hJj=|($$cDad60>UL3SvCO= zFd<4!O{F~Hd^&+y%>ddo=yYLkbKA>Ag`)Zex;tmSz3&YC`W4gb z<`9v-przbZwl?25p9ul*q=nmc0bnnl6**pFgEI`IYvN~|lP_jB%T zgo`lC-OF@J9a9FR6EwZP{m>>Jp`RP(hRg@Mym{;iWuAE9ZKAt$-7X>eEtN~Jaa%L% zY0RdkK>llhE@L$RhleiC?%eTB&_e}`&xO&u=_WOOaz-Y>a4t}9g#vm323{7DuCxAO zs#aI&4`czT*Ny+eJT2=sN!ox{GP!K*dH&U=n1KHaZ|N}Ik^QwIxm^8LwMYvD9P^Q|0;)VvS0Xea*8x1(-$U_NK=C(a0p<<3Q=sfFGG6HxZ&CuyLbf``;j z6XSk`bG_`u*iKB1=zaY1^_Q@7R|dLjX>%>xz*p+fI$t>OhsF@hBXkK}UE;24{pjDL zobG070#+b5x+wBUM-yQ+cySdO3gzR!KrR9nvt)U7-`HO&Ow>r0OlrP^tq2ZEich9b zEEn84fL;CKk!K}9%tdsBfbDG}={>OV`zO4txz(l6oNKV7hNCgY`>QoRkV<`o@GPjJ zqW{N8#gnG>Qr4Tnv2L zuz$}dM5VUf@*mX7iR?yc@DdX6%8AZ20P88s6o_x%KBxZ#sx$x-X2nhf&>nUh?;_~< z11Z}OXa)wlcZPoHWjP4nu;xoZP2NPF0CM6x>@7$EIRO?Th zFaG;;2NOZtps^WqPgaHsnETb|bVU4`C2fHds7n_ODJ_@);hlEP1jNf~)?+Fveg>I= zpdN$u<Ob^hEG1ulO#%y3Jw_Bt5`4~Kdj)9X_C^$`0 zV>^2J)y!N0Bh7%X$YSvH7x}ouDm7NIG{3e;;|K0WukCw#r=z0>qPdf0Z)|K>yQMN) zYu@Qn%{905711=&7do0vYW|v-nET00O2s(%VN?;q<09L{o>2D)PwxBU;^w8Ec&2=Td7q}9H3=FHT})3; zQ25~zTQQs0K!r~KjeZl9Ol*#SRTT@`(TmLH%WBdiUJVccNd8Ivxd2whd%|JV4xv;| zq=`Kbi7t*RVc03<4db;C23irmd0akqAOpGl8D~K>@6A0l9XvH8+vG%Y z#zTRJwbt=9?a|yppgDBem>pc<9*zOvL)Fs-Ri60|`$kKdUbL{ft%vj>S=`?mcrb7? zLWU{?gQ0v|GtEZH6aRvq_TQ__-4df>xeO*i)2znbc_G7&EZ~CN?E!pqSv%K|4K%@h z;yib|2S9SO^GaXopD=q=hT_{3p;EPBYBa{n#=ZQaXf+YNbO*X^P4+-3>?3@Nq7lvlv*L_4xAldDHvn~Az1#&w52lP>@L7dYT_YW z!lbrI-gw`T2jc89#@z#YEj!86jcK|MKgU=VVD5tpT@;^=C=EjC@k8vm$9Hvty*q!K25V)ya!aeUs8 z67+k7d4B!cp;8wgm5A49w>?OuOm&PN8JIf*0>3Eb@I{6K9zQJ-nW>J{^Jd{3i$Zh0A{K*qPXV!8e0RkPh{e}N3>%a&E_dmMUnG)p^S}z&K_HtnY5}==u5l^ zCc4No_vd8h$T}9vkmLqqMPG{Dki>Jrr>B_J2JImhw@=)U zmSfsn_Zfx@BbVHdjqiWh-!8dYYo3iKs?!<3LfJF_O{+jCHXv0wuh{Jn1Guh*zCV3K zh7m!M5ZA_1EGH6QeRDxWP(Jr~AwBVKQp}GgR<_0x(n%Cnn!+A-6&aM5^x8&8yZe(Q zsrrQKua<{pWS1s|z3m*E~ihj{my$K68q^Pg-~V zk#a}=iYnK|aDi+QkeB!3Uot)PJEiT;mZF#yA)|AXuC+0Z@#?V>V;(U#wsHVhan{Gh z%yOpZQwby~@OAL+jT}uA9=_UFlgBx@H8`~p@r(MH{pxAhi><;WqD{ieDaozOz^~>f z$%1=%VACpB(72}v42K@IVE)J6PWV|ub?j?WTa6#4g(D93GbWaM)_F5p?{Uof#CrMo z&q-Y7m#>z7YD}W*_rmrD4PK%7mgn^)N;8>lAB|3>oXvw7afHR>Cd|Djt(ORM-I<*m z+&I=Ho;FP}pt=(rkL0+bP#Y$eCsTJ04m5k`$OrL_rsgxN=AV#;J=GcMVee(7AstN(MmR+IG(1fO5*ZP-Q)B&V^0V#ZXw7T%Sy&9&CUxd8VY}*}u6l1HlXC z%4AZQe|rW}4tF$wT;wJTz@qqy>F9kt9~KWJiF%LvV!!U@9a3Sep9IO2pnoq@v)7)f z46;a3`nEbl_|@opZEa0BzpX%lW+d7I-0jAS>n-o9)Bt)vbyi$@j=tcSMt@cLob-(& z=a;zb7zKe@F&QoNEE$kSL&|x@^e2Bf{rm2@J2J?zfmCS?J`kv=G}~&aThC|Cpcr41mCTA$d{^sZ2dCM5xQFGh1`ObEE zp*_^Qo82wH;9C!>cZ`_7-QLZo>gGNRbP&)}JENXFe-)CAiIo^Yr>9sRhp(O+V*_XB z#}}bmaeW#+qt0N@6Z7!9LNO5We#Js6R6M%^H~N4k@i~I8=7ZOACrXY?bL)E|G6O|l^6g;^z)8aDjBPi4FfTh@g|#UF2P ztaA%;%4C&prjCLH?m9N-t1BWr+iG^70m8F6H*5JpIuq{5cjEr^-p@O_l=4ooFUX3h4q6u2~HPHeE5=2UHV`>~-Ov+M}edn92j+8pXZl_3?wQ z96dU-i#?2Ze#z7xnoTV@WvAXF-W{3SK%6bW=b;v6hogq~30gbVYVmt9Z0UrACQeIE zm#v`m-1{I>2Pux>+->eX5X9=SQZw?Ajb^qn9ADjdz5gt=&7>?7@Z8^1;BooL@fg^l zO{_azHAw>~9I zpyqkJ>shH%eAk(-QM{PLk*qc=5*!N7bkfY8i*f6pKh+q^paJrn?2bp!)gE0wdM4Fg zIVE4A@QS~(GJ2ztvV5$_FB3lR_@0Y?mO&S8WA|P5jy{I$`#QDESFa;}rP?v84B0+r zut7HSP)1na%hIjD^^wS6Y|G$cw(M#Gz0$WqbeSzni8N7>s|z_+(hCHJk}$gC{V z3o}9ztJxomsD$|~}Uwqiv*}r!Rf!9+s ztVK<4GgaH0Y->FxyGe%DM!*K))Gxy)jmQ?D?}H2$-Moy2394Q_aHzGmBPV>W&|on5 z$I~;shlhtbbIA+3@ck)tbs`xyb^?7sC&OVW(99qIY){N>e9{TM@K}V>Uas-g^h~@{ zj(Pye$}~Pu(3U7yl)rfjU0gzf-Exr4&h}w;)N)PT(8!alC*gO}^mZ)Ca><}pXRhPr zo=BK|x(E1{%}2;2-RWx!9s(rkZ;tN>$nj~eJQoRDQhmmr)a{q8wNFQRbEVR%`I|~# z?68PqW3R8q-s!x~QG;5WhdWe+x{+R{#KD+%p?KQtlde*E%HYC3Nx_+uS2weQ-z!F5 z>!J4bt{`|ty;2JHBC<}Jo!ve4RluFge%c$8+hggnM49t7kd2U)F0$&+%vIXDu*#~) zKeM=UFq!o2g~Uh{?e~XS^g2=<~nB?n@u4C4@DR z3=~TWg?uq%VWc5`Q>LX_n9JQUX2;H(!w~cRwOvH_7$P@Q_J5<-DsS z@?wkU!5l9jY!wK%-TGL9-@Vr59d`d@-HCm|M4vB+yOsFD#EUou_Wh2x!#N)th22Wa z;X^FFn8R7l-z-Nz!}4%|8-`(TQjJVC7b;o&Ejn({dU1af(DStli z(%uYj>2)zTBu9%9JlRV@0RNUCZ({fNN%(nrWk8aH%1MfwfmIsC&HGrl^R8MjX5$8N zVZu1$l>r--P!Y@OtORc4VX(JB>Ikb??+`3OoIVwTkohvh|vU10{rOtWZ6dQaC8diEC%i;KW{-cUC+4Qgh!h zuA^M46+6e;rOeA#F9vv@x@b#RT5zc&9;n_@q2SF}v3G6ov1eE{?Y;=>pb*3@gz5i! zxUm=H^M--8c+)-Vy9A&CWNJ?2`5F6&F>>%2Zud*-Go}k_)alSLX+%>YiPD`Lz|fyu zzFY2M`&w%}-ktVER+6Ei!IjPQ+Ub_l>|6le-64=`yN^v=q{sluORRUMX1CqrQ?P02%k*mVN}M z>S%qhy|c#Ao`Umd20!=j#DyJ&B(Y#z@56>ofC_KPHZ7cp<9j5tMIeW|F*P5OtTF{b zSD#QMTj&o9T`p5R5iE#F+WgLGPb4B8bq4*^*R01#cP}St)SJqN`=2=lir9nJEu&?b zOy;87P(|C3z@-FQ;w~^b7Hy$zwb5uFzIHUepttA=4;@Sa!!_wC zZ;J#DSV?@LKMx~jKVR34zf9OXI6a^R8__>UFk-XHwG4XwjGRE439my0=gvaCA|;hWLO$%YF~99)IfDo_(oC)z|z7t=D$i zO`dP8=J|}qf^8#w_BPtzNnn1zzw?|$OeHXLN(h@i8W_CW3}Nj?E7r@-&K?fm`)*V- zuj7hEpEsV-Ai>eI|E3$e*&0~Sw0p1KArs`sH@=+L?(+-NV+o=l?q*W$R~9$cnKp(~ z)vqs@5mx-(mRji}a}L-jQy!iOG{U7EABJXEzuup-irz|;#Nc*WeP(mzsr+%|u^Ovi z;{XYZRB!iEvkThCG|xeOdTpC=y?u55P*hF2=u(@^SpNX)DJG+vGLF9Bhr2t*rr1-v zV(7%t?gR#Omyr4AOcS^fZRUlL*Hr>+(4ubpBPI~Q+$#>L>Dn9|;9X+u-Gq4T)2BA0 zN54CxVZ%et^b2XSPf@T{tc}<;+xUNJ1H?uf2v(my=dbsL4!=@Lr+B;fmlt}20#=!W70HYW!%&zjB>$|rhwYySPQHSXF9xOu# zy2AoxX5D#H;k%n$0Vv?3{3RK$_qCwxcE-=#0O-?S%L|V>cPShJ27o9&bFQwgZr+7J z)4Ra;&;8t?}>-h^Wy7RCG+W}Uie+D3e*N-6pJ9-5Ch1!zQ~mIhke}U z)do3io5fq*rp|~6QF0ud;8+pTpq3Vo6`9E2qiOM8@Y)|feD9gUwiwrcH4~?5hjb}r zMlS*-^h?Xo=bb+LPT`l;Dr*Zj<3L!!4ug+7WBQM#P!bXnB>c50zI5vD`3jrLJFeHu zC$~e0E-&`K9tFxZhiDR29@I+4KdyrmJ`UXr|ByW&E_tPxDhLZ&2uE!g3dT8fR>+AW z9N>BDa`cmMB<1Z`Rr(pX~XQ*;z^K2xFE_;QpkbU0}W!FEAuY`wqlOFI-mq`pZQhG<|TH>3{ZoG!du zXlJ{ii~k^U^2Q4KpcpUSI+H=k)EgSv*X1z}SJ8TS1`>Ygm$gmL;rbMFq5HilDJegi zi%h~A+LX=7?qKhI>ieb%X2Yqi-Uo$xLX~Bq2yp!@=0?e^FTmmAuwSJ)T7c7S5QF7wnv2wWk0nXkNs4`3W>FJ za}qm=&ceMxzfw~NMsxF!BQr2L>m5NP3If68nUG;E*k^8FO|0rrU!R{%nESiJKk{O( zj;W$A{WguCD$9YyOZMjRcj#sq{cP7*81mG~qVVgQgYsHczYB6P7x-?tz-RWnRFIC@^c*#VdAsBRZ z8U233_fS#2YA%e2n(Fz2xg^iR^at)?dvsdJepK)nUMk;YI>H=xFsB5lV>hL;u&{DBsQ~yCf#-|r7W7S?Ja^Y z)hzkept(*&^y2;`gXd=*12L%w6~YlN3sph#=SQKj>sI%;glS_LXA^I2^!uKuijMG} zR%8mupznS|gLy16@2}_$xUClOa5Zk)WuBpIwQJ4xU;fmrge30Li*9;!9}Ox8B{^1K zjIzXMg!qK<+B-={c*LUgASy=8l>G6`PjjE1Vz#UT4EJaPg;qho$I*mMo17OP9q*fI zPn~_ELi8Wb-jjGq)A<-E)!pTxl8!!@ZD@UjKeu$)n1sRz?_|J=+kxjY)MikFAAgLv zXufvfxSx*IbPD7oJU1)6$%<+qBiPUSBeMHY^{~T*o=<*^IbT zups#@>BtlogBsL+1KsfHXXspkVI)sX2>29P>06Di#B(d1$e>*bUJ&jR1JRa~3WmTI ztF5u+YNg03ff${T51H|33@b3@qlh!6^kV`sz;Sba|5%Kx+kf^D$_?iu=|KSWwC+m~%w2RwQ@+LsUMhztBmxo8gC_^*pBpI}=YpthZ!mXd-nRGPti`jJ>foOT&!DiU0ahbN5)VoW){e^ushp`?Dx3tjZvm2FNs1I$VPkrQ@@dpdXkFq25bAXS+_Ct~7ii96^=_7!Rv2@5R} zuV&4M&?k8Gvr3>cxEZmo%GbdZ%g6)ul;5Y|cs~6}sdM%50dBf9`%Wss#)IAI+G3N= zUx|>S$eRj+PsjKm-kwaSL@+RPkFzBoUYo{u0MMOtRF&p&a08y(?~onVH)vPJQ^$qB z_nvL44{A23Y|Opb_U84-cw_;SQT{o7ZFjXHE_^p;Lm=eI0&gBamZym5$evroggdz4 zI7L42Vv1T(K2+B|Jj~ifnIRPoO(OHqpMsTA7`?@u*pUai`x@8-L=7o+TwX^>Al#|hd#YE86s0jI_it&h;Z$Porj&1d+sp%7)Nvi4{27DQ! z5~wZkHW#Ma1+;%K9Iieeu4iv*;@_Fk&JpwXg@F1amxx=dfR^5is}5B~23;RnGPOv`qz&EENb)D0}TAPjw!MD6gO-7NljdA>DhS0IoElIe4je$6He&wZ#$sfxhKIf*XOuJkiBl)Yl;!+1S$10(^>yA?}ZX(2KosI?l7y9)C!;P)kXCK6>%R z+x>V~z&)k4-D0dkeDrPlwMyq;v(Y@;Gcd`7HS+grlv;%RX=x(<+$RIWVm0s+9qhds zakJD_jNvX#I0EYT;+)R7NOXRXUhXk{YhQx3Z^j9|0%f~WF)u65>yFMo%nW+JP>l6q z(A$eG?i$kPXR|1mwUg7HnxAnw)hdfs384Wnx4(+tYOuyDpEGXU8^0sU&DKT{{eE(X zy`C&W8YOhzDz0hpDvTn!d6AdDAHcTIWsP z(^8mQC%Vw8+nKjfEzwn8-{2!2Vbw(IyiaJ8q=)6`fBzCE37?09{b< z<&(gB@tL~Ojl}eZl)|)(x-Dmzh(4sQsO6%13{gYG{^B_;xB*`;m>*^<*^8#QN9O4+ zxgZ2G!~;1|5G4!pCiZ^8bz=ifuA%!p@roFopq(7ch!L*g}yCodhL%7=;Z6?jTwt7=C&^^rMR{YBh&*~6IiBfi0@q40!#2U!$T}s+~qc{3sn%_`9tbBut%u$nK8rSR5 zbgN+TMcBBHIhpH>yUy%9pZ%Zjqce~A>?p5;YVuWQDrK}PmB4qb`r9_ z%{+7M(lA({uoE*+hi5Y}2H)X*)w^cj8KI@w5%L}b7x(k=ueQM(WeCiR)FYQ82O2qF zgrKL>=83H?3hkx`-4{EA(_X!I({Fm^c$y&}Trck<+D|*yE$X;uNDL9L-!jqTnjrmJ zx*^C9ESNCTbzVP>7X!pq!^}|kfUsw$tge3Xbqs8^JJiWj_>d4CliGq}_ zx)73GN6h+S4RN>(%Q%o{3RoJrS?_L|k$azQ$&b~XF!*ae z7hxAUxhAAINP1x8*!KkPnl+nFYPTvxN!d^f*dO^yRJ$#d?5}!S-Dc0v3)Ge(UT{t={d1)@Il`)8^94mgok~V< z2AK;C!vfAQjcsV6W%n#7g{o5am{ivvDHTunKgGoebj9y&we@*;oE5r!W!35EjOuhY zCLgocM;#EnQ?3|!)%egEFY{1}uqzOg^ZD>?^WBNQ56>$Gxv=nQ^?*9xD)W~MAy}seR-J+XK~T7e zz}r}?{C$T6oUEarc%&6$+-;T2&9Q5-w%E#froKRvn>6IHA7K|d$Z+%XiAm)=2U(oT zy7)Xqw`HNijsI5p^;4{2jYQ{i+eth14!7|`o|L7gOiMSiyVubdlGjh$x}m|R-tjJ} z^d)yK0n1}UeVH&zAC9uNcLwJ9jxg@IF-8wAr4!PHKBj!=;+s4l-E??bkxOJ%(c<6R zaZ_4xpCwmS@{nYJ_zH;%pb4IJGQ%7ueD?1sYs&!c3Y+Kv)!TgIG6_7E60a!ghUSwL*UbSs>~?dGqMw{S?i|w-;{CxeCaAKP#a-DE$|q?lqWG z0`+!9w!b99T(WPB{EB0?TN_LGXxcauoV2PX?dBc5`+=}zVJX<;aXV#rL^778{C9up zSX(~c@D@4NXI3C9udL0rpX}P-Yfyy6GBOX7lkRVjGp}bP4rPYLG_{|>Ngun1unNi& z4q`VM7%8NdL6J#OB2BV6i*X(l@>_Bv?DE_+dlFImdKJ9$V zr+P?(0m2zFzLOgL6kgTAK|(x__qEM@vv&!<;c@~%u>bh@LicT92+`Z^iS))P-hh4< z?@0Xqk{rSP2HxREIxhV6(QGG-8p1CATZ2GUVvQe~k4-S!!=EHI7n5k|a(OiP)GfrJ z?l~>Sji(7_n6BO*yd*}xZCEPHgolI8ld5Ps-%fsiph5xmaZHQn{z`r2`!n$quhXSA zq%Z_ebpj3QeR^}^-q2(hxJ&r#oCy>7p`_zynrji)$1to>8NiPt{_*3NfFEzR4^Thy z_*Ou{_=67q7#%guc|O-v>ug(}TtU=tI}R3hN4LWHv*CkO9*7b|bg*9ei#C6One^HAlK@U>4!(v=q81M`*SN&^=}G)jo&K~iQcZQo_cXbbxOxcozEW9nwZy~LW$(yo$1_|by}H;j&|K(X z{KO~xTk!E0vwIzD(=q&7FFO7g{gz^$Onn9YES|^|+Jx)E);8-Givz zNnZTAg^F83CBhf^(joH(%tQHtkge&Hb^yrHF!gJz?am*XNt0>4UHDG1INc!8~yu98qwfh zChuhY#wc*B$+1H%9pUs?X2Goi{LQqTFv|%RIdGS6kg|dZ+3E%ILQ!-av;$@YqbSBW5#}*SUW66Vh(C`zkBaINahE%aj9PNK z)v^Lt%y#Ccp!vfyywMokz@C^bLDD-;$v+eurSBT?vL;UT75Jt{hn^W!k#vGzfo)>hMh&6v1m^^ z`@UGm&rr9EUHSaZUq`S7WC~l_*0r`WDQ97Oem_W+q0@{m+9vd4#GEGhVC$dW&S(gop}a**u@5sX1LlEL!~>_EE5&c4?>@3VN);E>S#CH;{QEw+ zjy**}27g*HYSf_+?n~=VN|fw|uwu1~&K93TWFtmWTLlnzzFRxLZ;q|(X-h1f_-SZV zWBrcK@F@u9(mA%#%dj)8wAudTjHU$zP1U-1IlSsChcl9+xi;iO0w(l*Vyc_*)bby- zhUJnw^vpVtv`G8KE(bABD{7-tLlKoG9sIt@~K(740FBj&4qZ0{5p}- zsdgnvDOYJd`CX|O*cFUKpSn(hqsnLSh24jNkgK>-$t=n*D`RbtBC9LoeS4m3`PY3% z%!8>PC(`Tdg~J~giz#RoZ?j)~)Wglv+1BjJ&@PrVT4!IMR951^S6bQofM9p`sV4!& zGUj;;x=3ug-PKIT6&mEDyisBfEin${xE13dL?A6W)P0BtEh=OlN{&gdz1}~ z2)1ApK_YQIR$1~&k~aD{pI5v1$9Op-t@wR@h=SO2eQ)3XGjVRHwY>b&7!3INRn)^7 zXdD$bliw!x7CgJkVRh;I;y&|vTxS`}$MdF(+o&Jhpf)ey za*sSNm}?gwln@fDKrV=}8s2DC^FAyw>)7+yS|8eIzlmG}Uz zF1(sfO{B^x8ljE@yAHwK)O!wizSS`o$;E};6nmpfE;_5M+)!2_%}YNwAc=Uq=VIz3 zCkrLGq+D}KVN1uI6SP^xbDesmPTb44FH{{z1*x@gD=2PTQy8t5T>L4NJnJUV`+F$V z$Kgse>`*N@?>$?an<$UD7IVYx9Q1uJe@oR)2AN#*JcT7!cQe_gb+Vp*eOFq#=$E=* zNSgBVX8W(|-MdX?M1*aFEn8|7znhDzpT!37!tW-(N$BvMQZ*uO#3XN<2BW1C|Hhx9 z5{`(3g^?Qv0n>)cId*DL;|60)uZ_h8jxl% z?JA^)UgKodTTiWC#3}-Jlcn>_rl!DC)8^dV_y&*&$A*}(;5lcp87*;2#$^IQ*=O}o zs?)-H>${!g!Y5$N2xkU=bf3m`(8~G*^umk3D(wL9erMl_VTUGp#slV?DJAq zf{*|;{6Rk|471^_>9sD6?oM&TXln5kwu|TPlag}G$t>C-`e$Bx1y|EJPD|2vyKgo6 zC0CN5@2k_b)|dChizgZ@=Sp&FmH3nKBD%q21dJvJtHl!X{w@8%JTR*l{!eev>!3-eMBHYC(j~ z6LKEPMmhP2iiJTK(0Tz1z;&&;J(k{Nd{1Hcq)Z{^pm{Rq%=YznOuI@)oSzUMALcMu zZ10jZ-daK_o0XEEVl^rU#g3)id%Ezp4yZ|~dtBj!P3jjB9g4VWy-3z&AvjfM) zjY@o98SBx^p7(0q@9x7}><%>V6!?Dh8=Ri+?x8dquVwYqyca?kfpg9J?pm?(6P)E7 zK3gGRRFStkY5K(#89H}ter1~r2yO^ipUP422)6UzkH%f6Va>B81c zB^a~z$Jek1a{D#uhPznf+0Auucy^>=sn-ueksH0LL6no(Jnn(|r#am5S}zv(=lLE; z;l%=%MC&w-AsL9iIWxHyIu6yc;yxDNKY8A&C&%-2%mTsPD+?BILJ3C0@s1~A2|`{i z=6ItXP-$GsNhodDqucw`ap)`}&l9?}0P_;`J#blQZ%a$pBo_Fg>=eC>POLY9{ zb3og$smY6wu)xz8bAaSPa*Jgh=}Hip!qj_@l5Vb{-|qrs0m281nj9>f+aW06J#a_C z5Ej<>3&QgfHunsw_f)|3wYK{b-@Kl$Qx%M|oo~Upz_ZfYRriq_U|8mdyE#%dq$DSQ z^Ti0Fob2Mo4&mRbzX_wi!t(kfzEPT>f z-0N1gZvqjuc|xhZu}F-^>qiInNn|>hlJiab)a;j(RYHXYe#7=>Entf1M_f11jzBW*hifUh-`+DyC zx{>JPXeZjC$F&Q4imptbKLe&`lR(VeFJA!anuhl5ZA}b1r2aUI8Ujex)Oe$j zeU1L1NS=E?Q_Af@QNnpMYSTAmk>{~JZk@Ud*!gM4E;kYbx-?m6BnO3ncqCa`C?^kF zAR^{~i>vnGfN@{81x~uYaNPh5X)C?^GHEvY*n&&iX^TN>=r!a2uPp>SZeYfgeF+Ze zLTK5`nb@yLi}ZWi9pXK0@rC*`Y1q0Kt6)Aw4cqg<`)11&gaFLvMu z`ejz_o8M`iD-Vn=Mssy<>SIqNM_C{$O`?o|8NB79do_D)m$>e#_fWRW?!(K0XeNR9 z4SRe=(Bsuk-6O?<=utS)M^KPU{=JMP*ztCjC@4OEw!E}U^cgMaz8%B#(N`?0vfaq@ z;u-Jmq^=>k}R;efZ@k6N&imE)zK?U=~pTs!=ID@N=IP z4i5VdP@&TriS6Bgln99a{ZhLe23Pp8tal{|E%QF9k;utL48iaN1vtwT|9OV+X!HEB z-y>T#^d>F;rfXTr;!+F`(@~PU_BAA2enA8Yai04Lk$L|*e73KLa=`EMci`Kf$d08a zlcqd`GrQlp7v5&W&AHlBkP6)4ZOKn}&n|%{(pjox#JBX@T<}ZllqJ~fSz*xzn(M+-lkBQDvwthgCf=>ohv2PvbE-t_ zD!#AdO-aUuz3e#~pV9t81#@0Q;xPz~`em{R`=`YO4xN6(4y(lFF+6f~Oz5&|gKEsN z8u5&Laou}uOqd`kldk#ZhMRZzxTe-lm~-6aHtxfYG)9iG)bExX%(NY=!76Q}Bn!Px z<4Q{YvN-M_>Ktw2rNN1Z%kca+nWH5>%gf8V)3u>CRsAGYXpU9piW+Y@q-NleUH%{ zDtqHtSR6E;SD+e0eN8*T9bBHyMGk)Q9->L)8RxcHL|((dpfjo+YuBFuc^2Kji zV@t&V0b#fHq*R|53vWWjqsGr4<{0~9e-Lcs&OHi;=fJAn#Kpz&K6UH;Wp}jmk0S^VEa{bn8P=yz=e>g$|y(L6{In-|mKc|*66n2&5idj2TK z2=vcL)~n>nnE}o%LP=pq4#sIJ}!JQg!dwuPJ9lsTZA+dt`9xvE<>lY)lLjA&WGX1f8Q0adle8MkP$ zX1-obczjI6S}?umrMP~8su+Q>BdUf;sfx^KAKB|*INnG2bBY>M#2}D6Vhqw6vy7n# z*ePG5JN7C20YEgCYtD{G^C#F#0EB+d?YwYB!Ba=t^;oVj8Mahd6^B1vuxY)uueoOI1LA91?&(d>sNZ|Olp4R@3!M-QTA+(Q!3v{ zq3K}61}CsoZFGZX+1y5rljwD^0=)=-cS}f08()rS{mk;%h@oy4^4Ti8^Fm}<=J7Kz zViE}#({%Y-2EU%jki+d-R!^$qXNJd>Z#WaPJK>q6v~;;H4tpt7DadXitGxH^x!vw} zFQc$+P}SbX_RefJ3zSn@eG!*Q?dw+b#J1SzXq-}YjnV_PW)uB(2bjt4ep#L#f0oG9 zXIHux26Uw=ndYKKd@woIUO>RGy9VV~?!_QDs*%M<83cU+U3XpnTHm>VhaD>>jHb-H=(Gz50O@0jm`#7Vy4q2SF5|$*erHX z7t(01W&*kFazLK?5R-(zM!SepN~H3LMak42 zGnkR0N{HTchp*P!?0SSit`3`*765zYK8e0!MkcqsNbEp8g2TEqxkMwU!)teUq$!tb(&3LN!3 z-lQr2c$t5O)^vur2~Qi>_ZQ~t2VlM%+#KA`f$(u`T{{V^^JUgS3a(%8T64vG?0`8l zLGlCY%pfa>&il;6vrfx~x=XHe4V<&*sUB$(^aw6}r=cu5}it;hl8u3hVK}2fOgAuB8ky0LP{H z-cEH4C*$9e8GY;~6%uq_Go`ObQC5#Lw#onPQ_z*h_CIO3q`pnn?L!`RA(k(C-U*S| zqu>ukQfJ$o2L%FVmP>Q_Q&nOQL0|s^Q zZRHzVQT!_JH^Fb6e*LG{M^X%22wAqHJPSncX-9~N0p%14;S5?v3MgrPmkMuRkB5B? zc7fZ{qmelC$kdz#5N-OSn@G~Nr>Xi~4dKvD*^S~L#v5^!!XPU=WEvKFY20y8FlLb~ z{Xb}K5}^qTH#7SGg>z5Lvg$z!Njpa&cL0RTM%0#Y4urH>=TVIzHS!`Dq8FVT zCMN~#u~ZJXPrd3UpDdy)5^?>BCzFi@|7-j0%L8!JkeP^ z;dE8RCQSf)A|7MXq1~7_dVo`lhpGMCUV3@m!u*dAnyuMMz+l4rbPRQqo#MNg$L+Pu zhRZRd0fPeSd3dPSb65(5{!B#d1Tojb+yaC-+ItaGul2(?s`?glIVeG#2Dc{(iplm! zh9l1$v5OeFD{{O?K_{@-X}b1DPUnDKqxT-n=b!NfDl?_ebfaC`(zSc8Lx|UBV2OBg zn+!b{;n;@_blh6#kj`!oLZRI8hyus@Mkh@qXJvPlKNnxzY164L{O}E!{5B(UyoiPC zDk}1Wbsh2h3QSAKWOV-8#z8n=@tk z*AEUt^$=Upm_vr{5w{alb{kKso*t<{i7}Q&jYw|F{4NU;G=b-2i*z{=VV6&=EvG(F zcAg^=jq6SPwEYohq#WgK_7-?jTpOsR4-%k@-@b5GNR9i68%?e`LaIM%G$1iITmmr9 z%YrQWiwj{7S{Hh+5oTpO49oSME)RHAw<+o$_qTLA`gAQ2#p8Ybg%VoDaX};aRqEg5 z50%x1@5~lp^c#I(GQXmU0)@iqUQeO+z+}DW&{tJC6vyskhF6>dS<6ZMFyl&4a%`?; z;SV({o{=k1W$YhT5wCRAJ#GY82JgG6#3tIOs5O`u6iR3Z`vCyhxU(tO)I{?m>2GT`prYtf$}i4r|H%jR!F$@$*P4&ki<^P0&3>L zw)oP|UurPf_{x|o*=<*E7LW2QkJM^1zWS$MtX;7=^0Cc zr%McRY+)}zoE`yt2@I!7G9vBPH>8h`!wT=m)`Krp3ms_!U5;-b$VM^*wGCq#y}1G7 z&rAmbIY>*74h{C)zjHXpCKhmp`BQD#-L@o~9z}4dLWze7wM z2Q{oFzZUQ2ntVc|7OI`#GBPt=Di~Rte3p?64V6`!H0U3z9MdqFWApo>def?RHh;ka zeTy6ofug3+p6bcs(Q+~($}y*#$jT-cA?+$ZjG;Vi_l4(gmWC|kRqM+3P4tZfBaj3k zAh}55k5#XOgCkRoyZc5H_Bwr<3wMQWpYVi)cIV!2;j>*fciy&y_6d^P>iUktOd>&L zeJPdt3gr7+F+0BdR5?7He90pri^+~1=fd%}eF_AH{`Hx_}6R)<>B2aMxI z5f`hQc*$>N*#yvp`@JX_DmYq*5#J4zk|#)4nsxmFcPi4avrbdC>t~#v4_OC2NRw;R zSh|y3h1B_FNo-NGjA^}N)VJCU@OfgI49^@2M~)I(`ZN;Ae zmG~I>0HXxrvg~a|&_UiPY2IkCF1AQXmZZs14h9k9r*+a#?A?P7cx%t(=pK^7NL-E> zH))*EWN>RwxM&hEQhD1AaszsT+KXk`JqJ$pfovAhcp1TZ{NcNii1F8@espmQc}4aa zCrbi&o?J7RK672t_IQ+s>#!X8pOS6V4k5uZ;*rZJ2)hW_yt9dckkv8$@w&QwLAf*P zp%cNYtbqsA+Tjn9TkF^B{V)^_&#a>tv+-)TB$Si!9nHG3aC4S*ezU{y#JHbrK$9-k z1$Pc-d!_IIo*{#{J&gNP{7MZqr^V71mO|j1B%s?gjxk;HJL0bF$ntaQ!z4zw{)Tm` zOB|%pof`&)fp5L1R%9*9uEaIx1}R3Wo!~e~o&zN98^)ougxs^`weF)rjr74=Q|68t z9jH7e&35w>=Lgwm^CU4d&B3Q&vml)bFst3>T0*_Uz5?N*8lc8i+$6DZ>}TpHiA1t~ zd`x0%0phcWF%YT(In6)tvASbYqm2^?s00SVUcU1cumht0lI4CN704(kd5Qj&GkA-P zp=Bw6UlgdUzuQapqs%FK();jJEPnqgPS8&D*2`GTa z4G}L%jE9D)1?G3VwA)Ok`eOUCu|3_SHp4{Q!?EK>#^Ag15CxUUBSq#MH1DlwgKA&w zNR%!Sq}Td#N!<&puLN4`F{gzw^AqU|5O(S-`ZBQ|3aEs5d)0-#1%V@7g0aHj5yVPO zunNsyaEN~9#1Qh^D|rAKC#$g+H#%){KnT_T74jJuF3IjU6t8~dfFx!tOIE^$;)W~| z7Kg6x{_vA*PiA`?-oaBE3m&Tm{S@|Q$J6;0mVT=jGMAXSlGsTDw5Oc%oxUTE)F@(k z@68r&!__JisfR&ms{vQsMhl=-BJ>_V=lLVg=Qqmj!`jz;31il4gC%L>oSI8uF2Q7m z5#pB5V%KGjD)mE_ju{LoHh%iQ7+O&fT3U@fa=c38i;cD04vHo7Zg5cR#yD%cjW9-Tj>p zdW5_N%JQ}S_LD)+xspy0M#8_*?R^D+;No?2lE307`g5#R=*0iz4VY?75dspR1n4WZ zFh_kHzKdudXTQ~UGGJ?a>k$zV3wnz{gf86=!?Ri+2E3=fDTo0*d+W(BHPUfgl$wp< ziuU?$*+c41e}n=zXaRQZB`6@~DUdU&qGRB2;%!RL0zw}BZdRDpRa8=(y@} zN!j8dMdWzsJDEk&t@4rklM>r+68CVBSdtPY2IA%8QlSYdeb)0yGbO>vnb_fY%{)?@ zn7MMN*rK$&G`3{tA{PQ8(5VcO;%iK_eoPrGTs1Db^WU0l_y5v<#hcgTu10R@>e#Bi zj{+awnRhfM3VoqdHAM{Dq*YSfihV0D+8Ct?$@Lw%Jt)C0R7dtl_jkQfsK-Il&5igF}4g z^3XEyRkC|3&M#}1ek_$-kvQ1=j1w4@*@fP28l}Yqv}-(lG;AgaT>rfLK)4TskP-Lr^qwLQr0q^)KA81-hTjLuw2)+Rj0DH_;y~2) z7F%XxTP;su|N4>7u=R;GYke|CI2h}ZBrYEUCQt!Ry`DZ?MB@@;tAGJ;t`dFBp=`r; zbl{8??+yJPEKIZJ4Cmblitr~tV}8wf*G0lSXR#auj}PF>6<4^MsHft zv2el-+bb|3oGnCK7hbb8jHcg$qQu5je%d-|Sj|Y*>4iUQ#BLU}nT#m}wW1^#)Kt}B znVu;XYu8fsjUAyYi;&UwoV43P5N^nfQubs;j26o|nraFC-)g(94>U#tR}FS#k-?U9_ryJV7G88-`L@|J}~$+*US8~tYdW+n?{@~ zG-G$sn~ov*_pXKQqHy$5zcwO8Tj)K+IC)U2X|cXyMLDH%Q+fBJ0%wblpjP!iaa z#r65P)R_+{LLWB~chRyJ{76D^hl9~-UQt#V+uZjIZ5(8y{taKLlE8UlzGD23sm~7Z zGu%5%)V8ZSns&-)x+2_U%ASsUZ-Zh~X@lxqk&0H)_)f2lnzNmLa;d=yjk~u!7Go%x z1!zgkE)fGv_=w|{*H&2F;la_A^lTnjSW#rA%YdCg~edj-y%YwrF zt9I?b_kbX`?fn$3nuL<+MuW+(7TCF_&&^R!d;Ja{m%bM&CG^~5laj_9XW#TO8X0Z= z$|M4p8$>g`KXe{qn?X9Niz2-f1O{{|%GD0XqC+2diid3cT549)Z(XabrPImrbH)5v z{orT}M&6y(-tkmXVGHpUhCV(TMkJRK*4~O=_%J?R^ei$MGh|d{6F&dm zuz7J{K<4r>k2Op9MyfCZr^@*Z`e+#cbZgcTd9d@Tz8A@b_-QZefZHBDOW;hbwaJC; zA~oCP{2Ylk&hRgA>l`rml_#%`xiK058ld5ld*J<4seHXNCbv4dYE8Qr`LV4xol(7x zA(?9kU1Mj|1cgx^vY|kEZP^w->9fdOS>EaF@CP=yPX@}(1aHCpcoV}k7Xe|VI zbAg|LtuJPbIw58Z-T?i4YM4lR(2bC<1T}GQMH)^*~&?YNll4rEJY^JWQl*%!Xaice6tp|FLles zu)lcVeM{^i>HxKmEj_8B>*6{oXUO6k2pR~|mGeh^I8*!p$vjJ?63zSMm_2507Fou7 zfv>0ky$IxX^(x!)N0pPGc+BED|0dd7?J#&0sa1KqIsTo1OtUUp=T19AS<7o147vf&FwjA4OHxb;_V$?k*0krn`VICOXqc$iX~FKx zhD_AADisbM)R(%NH%Rnfir8y^D9oQAAS9=yRK%a1T0ifEg&_k3>E48bT<@b3X4suO z$4C;#U~(aP|HXb;g=}GE_On)xA@*-iw{WczB0nl`pW4wm7WP^4$a8PgJMacr@2yNQ zp6iqGt4@PpPMpWK9b9ql3A)>}Rlox?OA|tH8N?U(=6D1+ECxH}!PQ_TP?giqLQS=} zUirjn#{q=w`Q+gezeNIaLs19H;xT2P5VM1q3*d{zO&I2<)V(Sri( z^g$}{oL8l}#_9XB2q~jPKU{2<-v*TpE1krbla7_ zscVxf)xch28x_cyAd~UI;0&cEnwlG}%BVOhUwprLOn$5zy)hHc3Wkp0^lqz?^RV*=jxwdDTO&ZSwgQE|+q$Vv8|V9%eY*zi zTlA~? ztp+o))#$%-ls)~qH0IX1Cv+e>*c$ocq(-M;YKEBB<7&|8 zw1XLNjCcMzbX|r{&}W}IiOw*?a3AI~l+T6~io;`%Y+CG)gR^tb!EEKs-XPHQ)PjUN z?P65~vx&Z3_^_jftcZGG(CpmDZG@(_qp9?y-wws7_2j?%apsGvZoQZ-JPJau6NOk2 znk?c}krKc%=M_z_lwWQ+^*s~AH3O0(`>_#t#~5v~%=ub2k9FlA)l&we#roO2)~K$J z)#@Gy*4EZ*QutJL(hL5Lzg$2AL27(_4w3pl@@*+tX3!&Fu%$8WvR_c zdeqYR*%z1=n!Vf=4tYt%IGgN{!XE~^H>!w1m}9Y6NSyNs4uWvB9WWX%b?@~KF=Aio z0&VEAv9_Mxv{{wy$aT8q5)}dD2|F|U+vce-#BiyFD@9`yizdl}GlmU#;Lg7#s>9|V zz-IIscP*rl4T!Fb*o;}S5ZItze$xSZiKVj#L>#%*f?gr*s>3ki4gD)g)Pa7DBMXTW z&WRr=AQS{gKmC>qbcB6P_9R7~&jx8LrR=g(d0g-H&pAK|rK(H1_JfK1@fv+bzj58c z#eP{lqYRlv~X8UEq91cA4ev zUNZAd`hPaHNaF8|>WO|ya}m!BD7r7<>t>|~594&Jdst@4Y(SOd{!xz)k*jEla;~yySK2afbt`XIUR{)a>*m%KcC4VF>cFUC z({7@%aEU+hldL-_bS8aY0gmW^0{JP1d90}{VJ(BCgJGeupt7L3UZi~3q zl&nLn+NAJlJOEnU-xb!K0apvX*~U~GdY-+5-C9e0Fp-1|6Egs08f6&q00J--$0%ZA z;wXkOy+hMel_>Ra%w7uuDU0nM1@cVv7yfb%z+YCfbw8&4+g|^zMj}7q|7O)Da(ZWk z|NAiki~e$#hZi5g$Kww_;ej@_XtUC7O7Nas@pG@+AVc6=_P^BdFYDfc{qpH;Big)W z2ny0(K0URJLdB0Oal%dy;S4}Ig)>a#lF$jSaf^N#0O7$Z9L`D7D8`l5j0kS?2$=?| zrIH{tK)4Meep0Ut)_?o5K=P{>k?PF~<|%PQa~uU6%-;)s`NM7Hzoo?X=?1Yr?*Cl) z>#i2Tt2w9ty+MJzFaMAeDd=0J$-%OVJ_)45mM97VRS^Y&!ZCc#LINPI6h?Y zlfr9XgoVlEM*y>!rK-AuRlb=!=seaN1gXN>n^seFKzmjbAlkwM%lIJk!eJCOp@<<^ z&RC@W*K)lqwHl}=z3@x8)Aa&~y8g9Gz&jX+yo9$QfnAEGFK+X{Z{+~&M2nzOwb1=A zG69Fk6PDH`1M7Y4GEBudHn2!L#0Yx5H%F(dS0h9>WF9Zff`DXRehcYv%3xKtcBB); zc=D1Or+C=QPnKm3g)bi2`vsn2x1{*h-+t2|^?%prUnBkxbm4kTU`zjhC8DGM1sjhY z%dj^8-}Uvq5X+jYj+9C+34m{=_EA`=ktyeyCqq$FUO@p_ta~{EY;sPPoZkEExE-7( zswZxLTLb^Y?`euq`4|QGi*Ga?SWY;~xa(i8+tw%l6F2w5>1y_nQE3 z@g)U_Bov1e<<0(gIrO40JYM<WWRxnPS2=-Uu<VM@j0F>q0-mapVWMAtbW;T z|DYGKmh3+a<~dn;O~8RqD%bYQ`}S$|CX@pD@vb8q?eWfdcM0nBA#jE4(;dX*_LovT z<6r$wuD&d*JCwdG?`|pjpDQn(4?f*JyR?Dtp0*xlSNpxbf^FX}R%15;SrT^VP%wu- zH5PnqgqXQAwoQ4t$>%is!8@VgpzY zX(N#bG6uVXXq;64l1<2q+Elc=ka2Sbx|4o@g7gCoMDu_!Iu_`Y|0^}8@h>k0I}f}R z*`DvJ)&Fem7eArD_tw=XzuV!jr~WI6B9ed)_Xe@~Pt*KZ-xakHaKl{Tv_ z_tcF=-W=lAP@q7}A9Fs(_2=1(_)dYlOvfBQC)B{%JF$IqVwK;8@xg_TNndUewP6 zQ0bSqOW%QXmj91AVnYMMb?#V)RjmRe=xsVmVdo1%T(bE<1d`@_x!gQ@E}rcI`Ibv&P`4?GCOa`17_;y8YjDi71-5UF0%K6>Rfk!+}qgT~H2HJynf zmEFc8u4SoyTAilATAJUkOrP0C8;ktExcrtkB-7}HWe&C$f}_oOQOxGatZ6<5&pYH3 zNttpf`&Kd~KiEsMUvct(B`SZZuv+NLmM;P>k?)3V_P>E6a50YSKjxqh(pCOHMo@<2 zC5a(9)@Q_0f&Y>p=DpFFnBzLSy9~z}fvf%1>O7zK+MqcRn26zZ-_~VtQZ(ONJeF@} z7Tn-VW>dytGCJEGFBmC@@4a zqO+{-%Z$L27$zXBiz0^&2a&>dx2xc*B^BOE@gYs~^@LcjH{^YOuOSvEXW`CT^W&D~{p2Ko-o0dQfkfpK$S9HA8m#-~kUsEkU*m5^f z?dr2RvR|x}pFszGzs9_PP=A9t=l%4~*4=}WftHX)|2$tWx`{49Z?=36)Ir7=LlPv< z1wyu&R-l*j$!G`T1uTb{XrzxBTYk=u0TbcS2cO}Q33eQ;l5e^>O8t4P*ww0LIsE+D z;ouJzagztiY?eSseS3Dxwt1zatet8b=65{VCL+k*gPZ;3HHv`u9drsVE*%T;bn|ra zYe#kMfXSPaul1ruR<%hVuO_r>93W{}4L9oq(A`}s)Y_!e%&LUUj2c=+s!9pNLFet@ zXVL|htlqoOV|RwW>KrH3cl7E|TxlG#Ce1)Epz=yfpC1kqP4_1QFZ$7xegkb*)8)2I z{H1SHx|(HqPRxR>4G(+@5#L&&DdPqyU!U{pM0!bYaVQWEZ4Cu%OJ+>VPzHDSJdhYI zdg36i{Gh`gunHu+tKKij?WyV-^g6yPqD!1YWsce^7@MZGPOJ$+rmZ@x-x`@kxQ)+Iaf6zaQ1 zcbq|hf@C3GDDeFnBhoAu;xNpO&3jljUiam#6;9Z5qqyb-G?b%f`w`KblU4Wzo;evQ zl3?+sWL&qx_eiV)lIDkA$d&KOvtD^W17~MuxQg8xEdEg%3{W?OQbQLi{uxO;<%71o zdOxhvf*dX(Dou1*5r_Rp6{x1`ze;hKzlv9k+o)+b30=$(;G9)C>%^=p3U}t3ATjzfBhE+DqwkG`rM|M zqJvycP%AS3PlOBTTJei*4Qw1;kbMU}S$?hG=!oL)FKX{N6f>CEPuTJqIP9^?ehsn3 zbE{M-QS`C+;ch2WwfcjmONOJVjVBEm+5hO}ByvW_XuO2Xy2GY^mQEs(JDl?q*c(B?;{cpW`+$mPaDD%yN#%%_^ZZ$lw^Je^< z%MY-oXh9cyd{`b^Ti*=gG(zhKEad4zJR~eC26uhZ?WgTNS+HpI*WuvT=aSIi;w%FG z`t>jNnnf+aj$3aWz?YNK+qw@&Hv{R+%>vr@70M#u-;fGAOQJ4HDsxZEhcf#c50(Wf zu_xI~FL?jv+w^kn~GdNqT&E&B;>>zM}?b8H>|DMwlN%048}+S{#X67Z!pvElmT1oa7)p6cMz^h z4P(}82*W2_H}Bd5(g98ZN+}*`%T0j_ zvb@d^BUZLCj0}Fg`9Y~-n>(1zXNJ?xync|ncfaP9Ul$NR7PEY@f;(B28R0pi54woA zM`w~5ImW@KcsQiQWr-?1poO{mJ^y~(?0|XDOnvsJqd*O7f=EWtNxl`H#AoJCA!Z5k zcp+xKIf{9RSQHVnGdz53yi0ts&!==utN6!UX zmoU3UZ!gloX@Bc4_|)pE;O zx30m2cf3Wz1KE0UayJhf%Ey&L2sClUt*DpRXm3$p zr_o5I?mrw$8i#taDJ5L4$2zxAoDnxz{ z$U}w4Jc17GYGJe3O9`tBzj`%)69{`5>tkfvaUT3pwfNWyW>vkjP`|Fe$B2j>q{e&t21B z5%5{nXZIXatFbaJ-qM$~?4vlk6!{|L`*h~NVrUL)3%5S7yEjg^-({uHU$lHjPdPP3 zX0ifGN?ily0L*N;E zuQm6abFIDhn$KUJ`e1;OKPc+A_Qz)x+&uS$OFN;#!K<)L7}KVd|1p%RRJW4Yb$deU z98NX)0o3LbVlAT!D%Hh%LM5U|ZcfO(RTK5LPuXr+NOKyu+D_6 zu%FkH4j;L;`8Bd9``QMeN_giJ<*gI&(_eUz4$U6KO4xfL-@PV?BI1FL2M`A2+E@vy z7y|$1@TrTjIo_@Ea5$l@k7#sczSXE<1tbR{ckPx5(H6M`XzTauPQ9?0=#C}_g+i38 zUnX>O&AK%@!LFu(h;g(;1$Ml?uoarM9m;U~FB zK18G}!NdFN#=FS}O0g0m_ScTb%;%@aqv+4t;0g1)het#t*eAEIcHF09l62Wuqja7^ z(~f1D@jFr4vU^c7z90Tw6eV?X9gGI`9#_>y$P|7k-@hNBqAjgU zTD5J;M=$)9o+dl!x$Bxf8M^PD1m8Kr8X;m|-lhewt;eWxUd}#DEpZdsa}Gh-Gp)!o z5U%YB>l}GVMqt3~aQCpaUOzz}3ZlYH9g5kTjwn6!qrN>#%qmL0R&$X|sFHmc~7>jW4Pr z?$qPDnKJdPI84fg=Ws6T8t-|#zj`wOBUj|%+A=r9(;I01@Vjx;|A=^HKA^pJw&zXC zW6J%Qh^>$>pU;xByV`OwS?CZ=NVLRn^tnK*N?_*C$Ysm#=pLlOVNWZg8_Q6tnr6F- zg3HlY3AWhWgZ^gOTSO+HpVuw^?YfM{=O1f=Z5Q@yZ1Iz7)oMI96Z+Ls!bK?Th+gbU zO1tEz1f_h|X|STGuKIG!q)QGfjXFd>LX}m=Al0{zBtXxVB5DMmkd?LBDaw{JT-;96 zmP&df-cmeIz&<6+q^!KiNs=Hvxi+2F*E3N(uCt;uT;Sk%_SMEE<`>Z@O{*``_}yaE z7308L4cLfl|HOHx+LAYCMl^}3bSEfCVVh_rp6}*+Dulb-m*hFSjyiEVB~u@KzI%cy zRjppuT6t@X&zRsfZ59q-?e#TEcFinO`$ybuVrH~~D|WW!T#Ah;F7FIn4(69|#%1S# z9stAZ(0Us%*A80GWw4xSM=dkWn@?`gF%`Yh=G;iPH(!QvB#yeO21fEQg&X5+LqV#T zmT;J~?o1nD8ehqx^E$6TfV_OELaPCxr`%zu1z>$JiwzUuEj<-FUgXN7P!zuwR^qIEl0N zMRyYUM}FujNJrRi?7g>V(R2F>`=aKN%nU*x%E`;`sS@LW_r5^6X%KO#`)PW5 z`g~Wy0lO=%MF8-qJm-Z-{<>zLul~8ti{wnSntMlOddJaFKZA4!4@9NVuXErLA{_O3 zX~9hk`p;+rJQ8lj2d{n5buohw;dZ(*9*=+^NUu6S?0vPXKk`DYtt2*;NQ~dDgu+HY z(Yg?6>f-Wp-fzqr#j}kK$f5qe?lc0U2+_xoXg^2AmM4@Q|HG189;TpdSyhZ%>AZY; zH5dNk0ORq+GPN$c<^FJjO;%K-JmI$q86pH$GH z8(4T)cyv!&CBbCO>L=OtSBw7ThCe>uHjfsY5=GhP;NLMTJX6_-E4g+EJ*Q=$qnz2T z%Bse-b1&3KN-1-pY6LD4yc}!={$`%__Sz`%d-dn;#w@1yZKo7Ds;#}-JD*fePQ`hd z$T|E|kjYD4tgT97YU=Hbr2W|M!};!A=5=b)6rJ*$IrjH@n4v3iG5BuyQ_>}hofGNF$%^k8Z?U;cW}{(G|Cko z14Y&ecgK$g+R;c&SEh8#BSG}s@4%w{(EjYTF)E^KY|wri7xKaWfn@>jwni_tAM&}N z2@5ohjBL$EMkLZ7@Dr&1*`J>p}h>p3bXH%9&Lp7$US*?iu)P2xo)| z*rExp1WOi1*zm}h?&$>Y0sS0Eca50-w4BU-3}g8RT`Qp;2T%?_dkthp=V@yl_K|(e zx;5mrY?*15NW!mK6V*kwdZ+e}`fEc-GFBTbAIVR0hRvc-CRp8gcA<&FuVgX|UWn<6 zOmGnQ*(u6~vF`732B&FjH*B@BJ38Vp!z1i3X6l`Ke8kVPbkGlZ{<$FA_a4&ccWc}- z!qH79wI!v>wd&`1S4q)*$`}h&@8bKSo&T%$6{$W{y?D5FXjQhCXrT)qbs$Rt$R}e? z#`en!pz6=0FP#UOC^rXi*vuEHR;K>@LFFHV4;B5O#^E`jI~x;;>Isd6vB-q#x04qw zza<=3pVYmd3yGnc%T~{6;ui32K^*r8hJCj@5V?~%k>LYFGGLZm@GvLe$bv&-r{n+e z0p5B&c-7*XoQLBKpOXXp?%L9)Sq;0<1Z_D&wys!eQu(zaAoc!|*fZ6I%04IIKWmTe z+g(&5rfQ9T#?uq)^!redg;OPsm;gfy(7h;7y(tWQ#%qZ5wF^rb0)ZJ3FEU9k_I^sZ zifpXU<(H{5xKj@jJo)+Z)}SLR5*Nly3`98hGr~-aJ~vTYid`}S|L=$ijW4+M7=mx! zvukyQO;Nq$|7B8j@WcMb_fo9QW((ga}h9=q50 zCGFIx5XvU_Zf~Ob@D>am@0yI(bYG1QZ8qG0ovsbqgu{t(etDO0l>PpYavK^yogsV9 ze`FbwplkJ=OIOe0$J><(6;9yFBKG>WSTN%;PovuhB1Erk!I17*F!~&Dn_6d9 zAqHkI(?TuyIr;wj7XHDpqIY$Qjl};|&P4nyqL;p|bmy_H1TD*^6@B`VHLX-&sX+Du z&H)_*Zx1k9b=d#tr4PdGSNV`NQ{>uefUHyyV3JJvXp1bKrh;X74wKPJpYhUVzAdBr znUX;)Arx*zQl)Q@t%t|_oa9~zXghnbq!n4bJXy2ub)!9*wJlIk&<3`5tKT8$Ewj^V zD3>pO3hmM_RUAyyJMJh>XD$gsn&tq}=8IGX7$>2%|5vaW!+!9bdixDSgsfpIuhJ72 z4Si|a1n))`^#T-JTa0dS-F0LLG0S_>Wdb!yLmr=M5r>p_%}BbE;VOb=x*%zqdO5f-wFt?&v%1#6k;`7^Nw4xD`W3Xsdl-!DG^9XO;v}H0v^cv#5DhC-HQ4Gr(^{I~M z=ahjv)eS*Zlh<4~hmwXdjp3mD*1|NJ$?ND1FUZ#aVKlt<%^mq=7XmsGN zm7g2kS6|3hfLVaUXApV&hMJ3h`~~|6J-Co?0KS7RsuYsi+M<($eR41AhdI}&$^AW@ zgnDP2z4>X(xZtPB;sE0^9&r*9g6c!~F#3rVJlGu#7$gTpylNlrvE4X+!kMAk&l+4G zL}Y09tcNKOk7L0lF>f$m?#G*{z0=o5+oL-Zff`EKrA_%~(Ofcf#V9WZBpB2(*3RTt$ z8wchbe?z1Mfw*!)tKMJvsl&j;8!Ci*yiT9H+YA-!ru?2~v{?4Gc^z}I$QBG5GUyx7 z1>QbMAq}H$6=U0dq3aUUKth}lIx1yQx{&#?V$;z45(A0Q6+*434Lu}Cku=FdW6z}D z4nds`=M{@l{&x7z&s>38p&F-!>JH2hq8OaOgEqgAUDB#HYP{-f4V7&uPkl zkR(CYb}kk&He%5sItq*ap4rU$*E2tr3?qN_x!ZpE3hB-G>57voGklV=eK-!C|-`4%KIt+9$lvl~}J1aEpvOZql3cGfWx_b@{8TPl7{#jva?X5?p<%1?$2lG@6=oS#^dljZbz0@ zHg-SeMFr-0HVT)N_Y*~@_bpd0lDpH$yKa?h*VdF0#&TSc*K24V4CSciC}qhc>k~1d zfekyG*%RqT$CZRy1U2KWk@#t|GA}k16fITmiQc0w2eGrW2f2)EQQ7&a(=z!9)Aa0O zxZV%lJR*i}XPZw}VO^#TY=EXe=m~|j3qM5{Ois>V_J^|A56w67k4Io)==w=e6Ql%O zsbeA3TIl(esRkIw1xQ2)SGM=3O?yhVQRLjpioyf#Rx)-rhk~JeTEWoXGgX~Tt>cr^ zC0&Hp-KEjDf-Un3D`Vi6ht`Fg7PWM~VIq>wyp({eV8&X~?6K z&(>`%1ZFtRuzCCe827GV7bDZ1Z)JDvGR2yx<1lo<&T0?qbeVMSqMva1m7;m8*G`eR zqjig)GZp^t!GRn2jqWX5CWW%=0$}B5#>Yq2FJ3+J^J`1y-{CK_7WTOyP2?~L=P{EH zqcIz>MJdgD(l`tOFn>tWg&m`SYVX5Lqx}@v$5+zXCctSu^I4?!+~?vJljZQJdN+UG z=YnJXxGhoJ`8p{Mo9cBYR-SAMe-ABO1YcyUh4NV31!ly@4;j)jQKID6GgC;BEa1YB zk4>6g_|vc8o#o1goFu&f2N6i(hOh~({>@@M-I4U3h$$06)SP|-re@OaTu6wwd)AA{ zM>hxE&ix{Wx{}-RGcvlj*L+;`-E#`_Yae&&?~OaVVUV8M;sA+fXT(k^p2aqEEu)qc z+AHmR%!j?x;SJ4mRN2(-W3ZU3`0FP>FzI{4#9sI%e19^2fALZ%U)97llY-~l%du3} z66(l?2xE?ckgE3m#h5%ax3!QscQdE`yMCvZd+_}wg^!Wp;lkQTXIJITOH+TE+0aaP zdudp}Q=jAM@%{@f6OA_-QALtI;kkoPY7dWNx+`B}rkTIfKkCBEI5F ze)!v4Spo87ZYiZ?x5UPGIK!db!(r_9gY$uGtXIiLtRH=Ic>h^ft`@8zRqxTrT zVHAN(WNeILR-1aiX{kk#2pUPNsAE8}Y{S)5Pxo=1^xBGFG)W+M$su|z4(F6`RSjYN z81t?*f{gT9t^_SWeh-XbRHG({Mp4X38kxNr8i)9sp`4`DUv3a5zQtaHbd;olbTzOS z7l;rHpm+Cc(`rI`Ne&-9GdD6V%eM@l49l(XatITI@+T3gvX-(U+=7|SDlTn!dqccd z!|fxO&3>L`=sM$VUG~VhjP#uII{v;N8BEWR?0Ac?(+tj8WnCjWEMWolmTbDx?(c{? zxnKHoF)e=|mRNoH_^Q8Tkr|Gy^oDa_BCOf(FXp8FBMZsrCYNiW-LKCEM%#|l z-C9aLvEaGvCe6n-ycv}7JPP=u6fYwC@&6K_eAxhEXv7S+7H!N?ZRgbQRRb#XPrWbS zFIj@%mDpC#^$+t1r+(rC7wz*rfJy8Y0riZJc^^B9Egks%x_blrDaUzJK!^9*ywX#A zk^!8zY0)X=&T;NNbX1)Ebkf^Jc@iX=`C#IyLN+!4!)gP}l z>!U6;YSwEQaVEKXJg{VZzQe}FwsmdzcD>2CI>#v;%$fxoJ`8Ftt$1Z3!d1L5kU{Ae zE}r-D@#f`h64={QPZi49t9d<_`a(&>bJGyyO~lfFSarjyS^TTD40Dx(V-k7Ssmt0t z+nxGFdDo@7s}a{V%iPN0g=X=afuG*r6pWL=Fi+2v3y%%`OxPiB@%tlZ{Us%R7lwb2 zz)>}f(Iz9vV9Iurkq(FzY>m$?D@Bu@7QBUdAG&+vb7Gkd%b+d${3Riz45oq+SI0?P zM>y)d(vlO+@yUJnvAcpoS_^ed7o_v8nKk1)>_uw58q)pNRl>B>86o>laHaA^MnWbU zyl}V&I$xBAd6^fG5m}jed5RX?*Jtw93e5%cF`nt(8XiP;$)nj{?3&kbmr9mGp$lvZ zJ(80dlx(t9y=eCS(Kkcnt8d%BrjAJoxLtS^H&iwAeOphC2W1iD`2AJpmM#oG$I&R9t{oU&h5`pY%O;@M& zuEe+32YxqA&g`}W5kpE!E=M?oEoVk;0g?6DJ!uDxD?H@3Y%HujYs!Vg;|I8c9xaBO$xIRAI0B1VeLBN>*sg(N8PT#$VHm@soJdk8@jEDb*aR z-Q4TD29G(|H-EBaZM5364>cLA6lVnE;_zC!@lBC&e)W$!o$Hyx2Xix@b4qMbJdN7Q zq|}sC@Vyn(mKvFx=_B=ZW*~>!QM%W;y!af+(Xe(G6Gli_y&rD8MG(*-U2bDE>q}3x zMAe9aDIn}#>qO$*Wdzf7iX3fiByi(F@P3W4smkMi-#ECi^aMm*Q=ZY%g zz8&Ta<9ge>177~EKuvg_k$K04O)2uR-lFAGC`}V+{SX9P=IgCry2nZ44C$EdB^PgU z?V00S3W!y`lwlENMjO!c_SU^Q2-{&{W0Nn@rcJ)=wcT87EeWf2UqbLX?*cAXRQIe( z`f1N8R#+cv-&Ag@2G_k0iiNg;?Yw*XCw1>e*bLkKkc^EpNCn>E8vE}t>Xz&-zacK) z00yi>c;7x&{R;^baND>t~Xo=7uB_Q*y+RbkmS_?s)Wdrlifmu*D2>!yMY*zjs z$w@(%b-x^>`uW?pm=THVM*;!@Hk#cFutv;V=MnM17%HNCf!;(NOkE`s0GZgbl6EW> zbBlLGLCwYGkt9QpiCR}FEQ}0TsS)5)pNe)Jm|y?)E*PYA=^sp}uZ=;nI1f%I0kJ2E z4h@_ht z#YzEFdy|Zcr>Tz-ULc>Op}6zio34>lw)?%dmfyCrjkNp}jjsaz){+}==g#!S*!c$C zIY<%#ZaA!|pe~Q(ICpXr2Rs$Rv-2+Q_`vLe9kZZr96#hFx(x0 z53!+5hUFKD&FJZ^b=(CVNVQ8V2iS5R0h0FcXcdH;@HGrW{Cb&)DO=}tstOGntZU$~ zo8QStvy;M%xnrQ{ruuuS*)sw?2QGpV=Z-eJP)r!}*>2>axVKqSDe8=p*1li_VOl&ZRyLZ7|W?#9;>U45}D&EqV2^ z{w6HMG@_;7K91av(Q?LnPV3Dk1d~Z;C51C0Vy>=0L?Z7#s0xBKmy_==4Z@;?UVbbMRfGCN7Q+E zBOw|c_}?PN4W%6myWb%{%D6~#Idc0kE`D3VA%?Qk`9&G%DskAyI>18ZLq-0c*?xAN zu<3)+t|$dUAnpHDR64ig&DO9YCio7hyt&S#0B?w_#r0(p%bez*xrcM@wqI&^)U(+mhn3LW^>^z~XvdevE<3|>E)SzSnkO&dltHVj zrqA>)Fq>TWO1o)y+JQz~k5!%?Gdqisl-NmINF;BW{-dmYtgE_9&tD70nlP5gBHYcSa_t_dSeT*p#A;WWX$&MtVB|$4_yA&H66-nixI0> z@t%a5&tESf0KE{OJ7EMXzsX)uR;u0WRDO517DrV=IDu@yMSXPxG1z@ECA71XZZPwn zy0nz=p4HCI)BK0NSlrG{1oFu@I*%EmN&eQsao4Y>3U#%ieZz`IaeOS6Y{^kx|IgrX%u*=1aMdFvYM?aAe7caEK0bM);&=N^%<(!4GGc zr*C-V+S=IIkA>g2h6}-6-g2&EDVui|ukc$^u3I=eyyQ;bNze?Ob|VXM_j$lgL2)mi zk$m$O64IVdnyz_MY4Wst0O-RQxFDj6Ht$^fz{yio^wT^QQ~&aU*ReuRZpD@(^L&MA zyzdYTdpn2IuuY2XyMf!k_mGE%Y2 zZhDWsjjinYNN@`RTy{k6$ne<+0m&>5rHIB2tYd1(He~v4bS1~*EJWw+kQ>HbpCL!>9XzLcy4Gw z6F#_UXU<}4$sQbq9dIuNGN+bQH{vQ;dTLcRmphV+$WJyE^JvW#IR8(o zzDamgHdpd0W*hIHQ(Wpb?nQ8JZqCx5SpdQ@M!Th^;(t3I`9w*_JKNi*m9dvt2OgD;allk@uYMx*ao39Ud28G5fH*SGfV=@*F z`@op8dPAW3Uw(H5AX8YmIO)H7PW|1nzro?uTs0mcVHlR}9iHc$JG^(tVbJt;JIq|k zkHFJm$O{y9vY#o2vEO!^&wC@d#PMSvGif;lmcMdoS-^U{vkJWY;wqgo`;TM(k5Ok6 z^oV1S=*?@bFii?)V-lXc3i&$oumHcglUp^{OmOkET-{7^Ofe$$i*yc(@eAu`e|J{u zmf#=YtFf1KT1V!~4abew{5g-Vnkg}&3@RP1wPoH27C#yFwaFL0zo|yOKBtl2nd10e zq9b@V0eQE(==%s8`%L-GJ>vX=QG)Abc;&_<8>BPD5T?VB1$%81F;mi9KGKvBbW`J_ zJ8RI%*!>b#_y#gijPiu2wq$ArR!ryasp?Ak?K~QaVARK&M8ApkJ>e z9}53>uiq1Rcpekwt`nR5&c&{HKCV3q##HLe9o)!P^8 zOOrEg7A)2#fz0)HW&Kim5LW747tmrAA&qpkQdy9p+0^cTw^6TK9A#`KIWVGV!9xO$@^hTs8f#i{gLvuL^OE%A7_jg&nP+%deJocRupgO8-ZnQ6K1Y;kPfi1p?5< za)#Anx9j#ruIXh(qEooegO^WfsG2K5!mFJZI*ven0_6Cx`ogG06 z?CIU$_B-^%#KeByoBKyor88kjXfag<<-`#_l9C|h)H4<_@r8tqbznQqTW3Z3UH!}v zy(Vkf_d|;hTI#NP)}~yVuU3of3Artz7_z`aItV>X|4L>lzzS0V|I4onRS^eV-_Z(i2gXGkmX2rGj#wds1MA?0oP&OPc-X_e_xdgV>M@ z^JW|Dlf1mV&tz3n@2QjSPH`k|Z)~5?2p}&syLG@ozU=eMjqGCXq5h}$7OQRE%;<49 zG?@3@O5>SbeB;z4M&HCOCA7gtI9#eY(8TdqC8>+X#xg!54mTCcmNz#$GSjoOtMK3S zDb*^fXEc}!74}_BE_-7yg%$@M@maJ*au~QTM?*lwA1^zU)iUE!!l?{&4V1Y@#*@9_ zM0B^>W%`}hZW*N`u}Cu(0Yx|}>O;k+&HyBfnA-f0tUybGf3%(`7{OQ`pgLu_6OIpIF4C^X^H|g_i=i~iG1{0=l@9XcN2B$IdUXzu z8SYUV{R1=K$p7ykb+>gm;^sR zR#0fHyw#jQ=j#m2Z4#gR1l)pSKPPuDxW~JJ0a-xU3cYU2smNTyg&_UqS?Hq1nA>b;)1jlkepu*)*Vocv&`-)!9{Yi(WScKhX@h^a4l6wh!W zy(lEobWMRo6^N(|dD}=o=X|ph7hsHpZS2n^r&oySdIGxF9sNfd)9h;OR%)4aY-K_ug_1J%YCR7YZ?v z`;E@ns<2_uJGzU4w^x>-38eAU(Q2;i~O($W&naqTpi zBwPUaRDN4)(115%G4K&s+f z3vQ@^z`U`;jt}j`nQz6J$wiXBhA#beG4S-p)?oarb5Hy@MVk!;{F$%SS$uy&^L1!| z;|_QhY6@8SK!^^Fpvaf+@Pl-m(j5u|7D-JSEey0;p~D9^d_BQ|5%&XyHk8Jyljg?G z&zz#|cbO3MV*3h4B+?KW|NKI%bbu_l4rToU&$+sOJNg5O|Acdx6crwK!v5TjRRfVg z*MrjcZAIVo1ox_XJ%TvMMqtO#xC=suJXO<9xBsh&x@Kp|4b|Fh=j0F0%vX76GS9?R(naX2L~0yD&-Pw80#DSxd-u+? z>l=|NU%)dE3E=5MP|R(D{u4`B4`HsB$8})@Slz@!$&tRh+TcZ(_Z4-4*e!8qA)ANX44Wi1n zs9(Zy0U4TEvj3e%48YM0VR#)O_xGRvSyh|RhX}T{bor#Pfqf!|fzYZprUPZO_Am@m z8umIzvYW4w`nvgTv_E zAp#bK2s*g!1Q8JlNp^AA$ort{BM6t;(WqfzN!SOycgb1M*kj9;1I^2LWwUZE`D|Ay z+kW3z{BUkWGR%1rba{EH{NtYQPOtJYN=V3RVtP8q&ONf&{rq#G>A4bO@c$=X18_am z_9M4KUcL-?(6CbYvGWM@SN5oj+nYtJRpw%N5?{d;U*+osKdANUh4sZBO7%?)LGyAR z$onf_)_&JE0Oz%WH%>*XA`HNG&u$ij1L5-EA!}#5o|Y}_|A0yy5$HsqmMnh#|6a`P zkN?1dX=y@j2mdj!n7-E5thw23Rj=|rxHu)KXY_<+o;#SE7ga~b#i<_v!%WJ`%7(AC zeytDEeevtm23BM$73Ctz$!rhDQir4!k26QFu%D=?QE zD?&=o8t>KPs?ovu$53tT+NtxZFEU>VExHCfQ7sbPYNxPKm=Lj2q^uoTBb*{;mcq!M z?L8wuQY+j)Rm}P-tN|lj=cC`SsHm?h>dR0Y|AfzCinw``Eu+iJQxhFxfBgQ18GnKk zP)`LrEDn9+54teXK6%V_bp^+(1u+8O1;!UB{x8##A3Z=SQ(&+V|F!8?z&5R=`-eQ- z?CYX_DJzrx4OB!+)|Xl4tOpge5X9d7u`rqmfzfOAqZS%jF^-#YOe)^D{#DLCuNKVOfyNqX_&7U@qAzZ};xrm5P6U#`{tV zJp8HrFXRW#{t1L@XCplT@BkCnzio+ay-_H~VCUtBI$L8s*CzK=ph&%yC-pZECx{<# zj#w|kwjY^1J|IJM9@d>U5ji)uNKj|IH`z~Q(L*PAM|t$6^R3AmqQQvUhlBkx?N6Lx zQtt_@fYrVrc*(-ST1E&?2_XZ7c5%WlrC$n1J#TX5-gOR-6K7D(FH`OoYUi^W13_WtOx6L0P^dhBVm}a#< z9&jq>XftGiZm#fl&s^eqfc#esBpZlYFXTK~!;syN>6lsK%z;oC ztt|3ikgX8sQYlo^Aqo2~kG8hF@7ZID58GNbnku6TQ+cqz0Oie!G&- zJ?$D!wXu1Icx4s`sW>{J75nWjG&>pePB$EBVb}^@_So}#R6LfuLH*ZQ8{H3mNkfr48<#l3qmXmu zH`TuQqdEjnwV40Pq3!60wcdPf9Hyz=+s0C~p893B4QHwiZ)t**E$iL|`qI#?jwD{H zd^@2X3kU%x374dQVNYs}{_Q;QCTyR>|I(q29X8Qza;F@X?A%lCxIDnJPZ2CN{k&xp z(=0ObWpYy&0Ri}iN1v+xto?IE;BbNPzh`QaxMH0&2wgYJgv$XnC0jzb$BgUUuA;ofoArF>=dOb& zca4ELqm`agIT_7B5_Bb_OYjRCFee3^w8e*Vd8D^g@-IdsjXnoZ2xZ|M)U6ui`&3cw!2*u;`m zMf5_FR_FQe@X`jnA{Z)w0sSZ2Sme##NY4hH*@#AAsYJ_#3!YEx&*^*&XSbqfzG4Eq z$2uYro&IP0_V9rNW$!3&Gyi4QF!G7Xj-xFk^nG@e`COZ3gy`G${qeP`MJiY(MEW@@$2C_;sTFzfU@ie;V8te-&qJvW z8>^yguxJ}p%!)aN@s}b=A=?B3zr-F`QV!*E1lqOp(s!|A98>tCXtVh%TiPThChs}3 zDQhI7RXPYpdv%N&Or!{27o%frrcStwzx|U86RZz#&I;FeC}m;C4re9-!26lAz;IBs z0*jc)0*-e(wf=jO4FOADQPb9TR(VG=qJWgRNt_XLd%l{XydCY)1HVf4J%4CQ<6dIvcF=q z8Z6W&V<_%@;3iE0X)!Sz%<~i_>M8Z&&`47ib(>Kmt1J=)!WPfmLV(c@mtO_n zU67Id0$sxp&O}(GQqi=34E9&tD>b%QF+@0P{yvL4t#Gol_XxtOcxU`dg3f(jW|kki zN|VxKd=C9!yVFV8K34^ay@DdZQK^S0yL`Pp>EDKlBz=sMNxQtHvs-vSddb=Y{QxH% z9$-#am3A}a6q{{+Ed9KXP^bmP$BSitQ`N^=B`ka2qbyQ`g+tv*vuGW&$cmqn6#ku) zmqPHohwe-fu}NZ1dff={gD(iex%mWx+}&|%Qb?aDc`FgZi0FOH2u)*|vbamosi_Qq zU;c8daH@FIyUI)^ScQA$AS5G?_M*^O*#IB^#` z`f@aS_`9y+{I{R#sxx&Z559!E2ki$Ie}_{2RdD!bYbEKGC#a#$X^J9Os2!PtX6Dx- zmlBUjk7Bak*LLQeGGLZmT5Ncy2~%5)iqgmGXz|(08mXnvbqS9*F_j`J_r}N!Uq>O3 z%n7=?y+|VNARX118x35#-(P(xo6kVkUWh(qG(Y-V@DY-JZIB+{`?k>X(3Qb5im;ce zfvM^A`renOE}`&>=k;pzK6Gr9mS%L0{;>!woE#l+c36*VKnWUZ{Z^QnV0qPD4;`y0 z0Y^$?Q+(N~>lAYXmPo}J`zxT@bg^?Gp=+RPY6I@D>8S#U-0Fn%6p6Fgn0Mi9^;#rp zyksmqFtwGQ$z_XU`;f=3(S=;v-k6knR*y^xieopK5e*iOso=?sjh267Lr0J!YQnPP zFL;k@?l~D>RmHp8=0#1injQB!1)GLh=j4eUifu0}woBAjR*I|V;@oX#;A-q;t!M3V zREuFtwI9&5I9d375YzeWo4vM%1|9MVL$*V5ydvV&Ct-&P+dr@)`-nS%YhuTn^MUE` zsR2iOw%&LCxo~;nXylVYq%fGBPG{p;wuE@+6YF`1Ko7roQKit&H=0W8Qk#Vt0`yH`v-kc|CyEKP7dHe9dc8N z*ozfz7_xrbu1skv77$~rA!G<3Mf#6!f-yVzI_o>;XEBC3_GPgnjXt{>eXVp;N5A&D zYC|k|8{#pNe5WYg>~If>%fyd)4xP6*ficb69527B_1kP>;TV*@XphZ$O0SO&+cP); zOd)Y)skHv?+$qzj7UQ46px{E|I@tfQezi}}Sbuh(x{g!|x3bMc-iyzMd3(MzVPWU^ zW9S^N<&0D>%=~EllB#=|Yk1@Jeh(|nfRw3wE1g>BzP%_d zSj^7TBK8~7MRb{ij4u z-13X#Zb|j-ewCV?;sXlu1=K?e1@|i_pa(Y+iU;=vhTT+Vxf7$twB7tPlO-)~=(O`0A@5-e3+nsOc@N$#S6IpO-} zSick-(>{sw#fD)$S}@+UQY7lIyL}-lJL5R8|IWWYA{ql#OQ1pxfk1z0P*IL`PBD*N zSbydRM4I+&gp0GYY_R6>fJkMPME$^aL>`yJ&)Ff>SLdh3|(ZGuZsu9GM+e}m&e zBiV3`;34$2or~Hs`?g2HExed7h0hx^=?DUy4A%9jx6GNAJ&;oE8OP-8S?K=gd;h%} zgquO8cIOZ}NHbH;zREk6>>=^0rmN>LjZC#g3|#Q7q2V49H0Q`jiwd<9yNmKSHlf|I z6)~KizD(1C1ZK2DEgW>Q6(}fL_-vf={E>i%UenV*d2=qozEzTB!QJydiPkYq0wo2( zqOJUnKhCn%8$i=zo?d-=C6o4l1eSt0*hK47wvp2t$Vet&zmTb8`~|_Rvp)=Wky3|P z*nUYnnf|qs)cyeSN$djgsO2}D2}7#v!5KLn zh4$>p&0c~_NM!TsYzrW7cS>-xO5W{=Z^#^$_%E=_E_Il-Tgv!^95e@q!yN3AIlJfI z(-;@HB#uN&_8BhTe{|m`NtSv;yUxMRA+w&Z6X$ze`15@GFt=(UDl9#0@rs!8#+%-1 z5Ry&h$9FlX(M@%Q3Q%P{I!C>Z=?6&5%i$uRAnEYGpq$d?A|B?Zwc2eP`RrtZ(&8n7 zwQoEU=H(;ydv4LB{NVbxO}_?|Zb`!ptbulA& zd#wE%=WZAU-j@+hGrHY6^p1AYZl&SlaTD6vX*RuS>emw7U#+qY6D9;La7SG2Ztz&y zbLCYx^GIuUj^VZ4p^Avjqhb#c!Ef?<@`QxW85#~!VWBFV{zIhfSxLUa&-Zh2bI<+3 z%syUh#uR|htgRq5vV+DWcs)Emi%}`*-n4XjB~7?NjG5#)G7R68Xw4CV*i2Z5k+h|w>B!1rX$IQR>hfsw6 zfTiAxPsq;_xT2W+a!mNt1~yXl;Fdl~MDLXpZyc+_ao@&7BQo*K3(>N7`h7NXUEi4z zevnBfq|s1wJeP9WsOx9ij$7r;>v*%vbIxw-=(q^Pxp)bXX&SILT}Y6VNO%3`32P>4Qf2XKL;1 zxNu(IE>i^sKQ&5RyA4xRk>Yi(r5OpnXi?>TFXsQTaVWF1=8k?eq4DsTA>@b%W7CsN zlBv@_d!KuSVFV1}UAd2`sJ7bJVi&+V5KGa{6#i(WGyENeU|4dbcim@BOHza%Ln#Js zl1e^S%#pR^y+?Ov&Uvj#s_ILDr6|Q^nQua~0mY}y;^N@Yf(uwrciDKg0%~$R)tQ{< zU#%6`EA!(`n;O_5gQ2@p>M${A?5*@CvSMLi1VmoN9vR}oR6!3UM=}*UOkYlbWRcK#Kvb}U{0;or__Z7X( z`E#x@u$@Kr{`tyiHp$064zl9B zB}<2R)X~X&%Y)FOx9}7U=tiAHR%pr4ixQKK3Bxjr*$vl*TJQ2iq}EEGiI3zT;{4vT z+uQqxzGw2#wqq+Sd`vU)`s`g`cAD&2JhCn1?Zwh1t)2Ls)|s#LPvRGLdv&?U=k222&`j|u3=4t(Y7WkT&Xtu3n1Nw=Ada*;`ldkDJ$o&onGyh^^xJzJHSY;rQB>h@cq4FL^_sF58s(mE&Da!iTEG|W_i&( zICWuLNl0@aIGHriI~t$1WPkk@A1$1EZ^3zL@8R4n%arl|5cU>8RefLBu#}W^w{(Mo zbR(UD5|WZ4bwRqjOGKnwy1TpkB3+m6?uPGR{r>OF`^`IZ$GOAc;heMAUhzE7S{sz5 zdU~>F=Er>-;*#>k<;6E>XLvKB0b_-us_&|%c9-HRaHn|s8Aqe{%$peCuR*f`j4Ms` zdIZjw)g5b>npXPDXL#@tN|2lF^Db{iTK+uSKA?T*!p6FNr@gy zMoJ<8Z)$fkC?g}&_anYZ4=wl8c{1#&yba{j*uNSD2B=X)8=Kw(0ubYm0I18+A0Ok`oKQIz7=a!>0P9^{muG32HO%ILD%BSnz(~kM0tC8fbULSL2Sxrl*xN1^rF44 zv`+Aak$ooWX`g7dBEZSSGP=Mn$BnvLE{k7ATp|E4C8h%p){s9)V=)2m)muPsW^iTF zHSRFTg>pCulNMoN-1e~;DFp9_B<1H7S-K&4kx$lwmsn3+uN!*0xW0}n;rXcIck~c_ zbg%X8D333z*9q;DAN_E%ZjLWU4O6E^u7Zd?o>PW z#5Lc=`(1!n2@Xb0&2FC6@CM6u9z6hC43(8ul9GiEDf%;jq?fqy<}Z@Qwc}V7zp-3B zK;Uc`+2gdgtanaszK|rqyiOd;|0(DY*Ic?s=_1pL2peYNMapWg+{Hb}`Aq90(Nll-2|P;+ZNSAPYF+OT0{K1* z@fpn1d{9>|ox*M;Eb;v%wibbpUJARlKohrJ4ezAYVp0 zOUP~So=e;OMA}cbc&^(1+u5D(-XX(6qz{2J>*zXF6i@;VaY8-5Gwc#_!l?=8H%qt7 zj7lAxKGKud0&1)u*0x@^X}if>+?%Wvy09pSz0|vp)Na$$86Vd^PvA?p`FI-9B~W0i z4YUYn@Y&*_UX&GVmjWzF+p_+Edbw21$uQYN2}0bLxbRvzt@~Zi`dH8VEUVcE^W~LY z7j7Hu?R<1 zjow`~Gj8Y7z@DmM863^;hi&3v1_#96G&PAqJG=Q3cDdP{6B^GrA;4ezj~H!dvOKba z;(q@L1-$$LImlx$RQ(kUfi`i)#(1wQ4{`EojZz|y^>-e=Pv1Srj>MDrZHZH+g>4NY zo(28|fZvk;p&pxyxod~iaKLB>HAGMrPcy;}zFhBV!I&mZoS~SM<>dd}d;kmq=-ohV zGDyl_oYv52#s&&?o~zCd{6tl-NHNs6wb1)$)G8Z-Y;`p0^`gAQKP;f`gX%6z?jv7W(Ai0F#oc29?;PmMinyKELR|`3uB`1IMR2yOLSA ztNArm%j7iA$+}=yQ-XH$P9>2x?ReTYsC7hlTUm{w`V0VMqHQ9huqGVBiA=CG>vx$P za%dzYc!Rf$>h8A=^RcCI^o~ul*i%sGjQ1x?p-9)6HeH_r`WY2bGxR~`kt!^2a;E#E*k7Yxv&|#gt%Q;uOjlW@anlH9rIFC>$t3aGN%bwrb@JU zJGCEmAP+t(@$A)u$dMU4^17*}7Luj|QRtOHPMHsixG?Jt0dFn!E4WiFt0BJ)<~TDK zHJnL%dg5Z>G~uI>By%5m@~#H~Xxu8?U4isxE(?NrhLnol{?+JhpNFjHzwPaw!N`3` zzcF5q&aVrzNU5&gC~TKPl*Q9OGd7+%V$Ue+;6FY$`U-u`*G%waN$S$nuN zZn-SbW^I*2>+#A5U#qTDfJRs90m#~4sri{w7vJ5exS^OaeeqS0YzbDKVE%qAD{{}v zuzNd%-G);llvS!#y^GM-<|pr7x_lNUtEn1;NIf@DyTFf03TKz|)b8slek{gQJ%kx! z%cUcRq&ho)!~*5``&s=Do^>t_MX~WtYFNNHmSKsC)5BJ?_U=!RqBp`1YvFb^kzYMO zIIvctsImz&?SK%mfc`ZuMc43JPj`n1cHeWB zfyQ|W|F@*0pUYc4AT>77-s-db(tvwt+X%m`Qx=E5-W;QvC~ntK$+A5Nz%2@R>;(k%hlMjzW^uXp`X zo;uMI?E;{#Ljrs5j+yqJ_%4SuJ;UR{bnQ%~Lc z)avA39j^t2Q>RmIO@kHHK*8=3Hg`#z&a`u-k7P{lvb&z~Gs^-OQor}s0Ktqh$!XAX zV1wZ?^|ihd_LSOh8X${u(4)A>USTP;`Y3(D4FZ_@^cR$b%(TpgXE^ELywhxDC^u#N zWmLBej-vHDT{TPm9s{GIVkabAd(yT~kgg04tX?va0Z7D@KP+uUlLrr0-S>JDwTWcx zPR*&>c8*h0?&}Mqy$XrjC9*HhpEv-Xuk=Rfb>9`v;nocsf4xB7xTH4pb>3l+z#&{s zi#Eq}*&t?9sF!hHI1kJEMC~{NZ0N1}Nfl%#hZzJwRK$lw$Kj5qUJxXd3Sk9Tmh(o3 zXSLE!hx^-5`mj)9mlvte06nXK4E$4jOUs>kVrr z+y+D2b}4q9PF|Ghe+Hk74(HkD?CzJ6&8x%Dd}dX zJuhPR)@#+ux1*M|L6#f0VpZ(zZJUtIwtjWYfHf7<);~fJai*t8RJ*7DFx(3xXXt@Q zE(bm^*1bOnF}}4qH84(jDv z3+E3ujE8~9s4vke0q=M2VIBHYz?oc}i9#%07sudIlNFWyGTkU>=dHHk+0nJ1#j?Vn z9&QzK*G4k|im%JlSzh$|>^|E9+8d9-FLP(u)+=o=Pe0^d?`* zbUSN%YZ^?xefLbplj6CkwzApSFdB%Cp_P@?y?jaQ*!|pYsj2iXR}RcV$hN39_MM<7 znEyHPsgbqILXjwfGg}Zm-im8E#%;oO_ zZ))oW9BL=!>VT&H&{Ee|E2{QMj<m&BzH&ZSmuMp1wQQUZfWNPRJKeEdoFmgS5nw?`;(!IuA-eXnyDUz% zK~Z|Vd(h;5bwIToPMiMVBCs?%`WAV<4!$abyI37zuy#?4Bk6LTINiQG6~L)YUbd(? z0j*(-HD%kpb#i6_su!^ICs)$i+~9l;_0~@RT)q}@h}JX9J#!TidU1Od*i%VCfw0)u zM{Q2?0%MgwFOQ9Z=G7DBV?X3ZMKr}zmumPS=4V)Q2e;uov+$-so5(pMAC0zDXOlyC z@Fapq;k)lOQ}->qD^gA0{3o!yN3Mdh^tf>cfxN*&Gi-gx($QtiO(T#26KpdgfVa>r z=*o{(?_xGS8q*3lIVXN6+(FGfGo=p^c&j&5Q8zcAgxqnkWedM=x*)HMZ{5rbaF&Ds zUDevHsrIt^7X4A50nsYQzY`UGS!zb6&la$5KS^+qXR81;#C1hV9DU*CxfVBV5BdUI;LV0Iig z0ze(m^xRf0I7i*M&1E}NdpFRoz$Fad6b##9fg5dfzDEn9*4q@C( z18P_)ledV(b^qxf#~A*QNE%7Tjje0E01)e}q@v+DaTdLtmhgAm1PXIJ0v?2iaybkR zQ5!=mfbzn}z_Ut9zA|5n+XH|ZrmpNo`h%MDA)-JOZ3|7PsdHts(eRM2`D)YcfUK#e z&I7-Ry-PGo2Js4p-7`(fU!~aVqnVrGAu6O{HCpyiTo3o~IV*%?B5}2y$jcF6A>M#K zUbIGD1gFPnXahcafXD^oY~i7Gh6C1u^fn+`&4U5-t8mJ(0~(S}C*pbnr@Vxlcqj~W zhJG4R` zLY{%~f7MM=Y!}(t`I4C*rvComivrYLq@n`(x)n5lbD%I^n>CZx0@(eb8IC1vSOC7fpG4VTaG@#cc{OH~ksA>Q+vi7NSl;1X*k(oiI!j1Pm zKZi&I?)ZYet%@18lasigV^mOP?Z|0jDOy|SgpP&Q<0D+Ujrw89Y}i%GNNBm%yHFNU zcjnIK1CYAu=1gAje|dxoez|7(UyUNmW#!BLUl3vVU+ARYO5Q?^&-~i)X9GN@+XBm96?}?f2xc(Zy!m|w^Tb^ z3JL?#TDlhn=wa1~mqbjvW6eUW%mZF=(aSey-lue0cRM=jQ#H>|O9y4h?ErbDP3!|9 z{f3$e4_H>o4m`im_OhfI(n>f=Mp{52QV;RB#Cs$J*g(6l>uIZ6=QNsOY|je~YD(|n zmYT@8*BOo2xvHDPPR(W)XCF8mZR7F=hUIIXKqQpxp=~NWVk{_PpYL0fUq>5Zsk^6Z z2sXcuB?!qGET#}TtDCZ2^AsnwIfLc!G**M-hJ?$#mL^CyIZt-?RdZ`S8JG#4HGMIZ!e=} z6qXt+e_mme$FDDciW_2W^3KmM4(a#b6ejavLxdONCwD|iULZmp?C%_MjS_WthSKs* z>bL~<1nU3;rj7cyx4t4|XSfOERDAACea!w@86JQ+tzFJQh$xbuf|r&!5t8PVE0HCV zbhYep^cX3?dJG}#;ts_3*Zlks*p9%VKaOb!1^c^q^|uF;IZug9mKMj9xku z{b}nVc*r!;C~nAI#L-yzsu>Drt-p_bl10Q`M6jvm0&z-RJ=lC`(MR5cQh*B{ZT_WH z%~Jm{JEEJ&EqERf09caj`-QnMep}6^yqU0G?Mit)8!*W>!fI2)!q^$z zOq2Fv<-1k>qrx)yGiD@46ACrlG#&kbcWrbA`eeVtT1Rr*S>x*22ly zz<^fbNtOBBn0U(5i-6ADB2U4ij$$e9^Kz?khtq@Zu21=!WH#@sOYD4-6ls^_SlOnF zzurF6x?WMJ8fhW9cW}?KO;D^ai{zR1cs_SF571{sn4k3(UMk(;s`GGsY&$k#PIJ|m zdOA155g=~S1-s$UI#f@290w!l;Kfy4A*rWY03>x6?f8NX4QX3_WkKeaAU4U{=m7#D zseBDEF2cP_^AN$0y&v)_)KTwUb8?q@xUNDTc2)4zxKw7^2yTt zJd6^b0pSlA0kXNH;vouWPyu=wJSqr^)&t&9%xH`Q{faA%!Y)ah>1;YsxK3?8anY7N#PK5JAeOGedE4-ETk*@k$I>u^dD#Of`ev%zA zvYLJuc-`}4Q=>5MiQx>cB7LAl5$$1tF4G0}c;=h=_#p6&e}!L+5Bp{8r8DRqUU(e)jfi`X?fhtM!@okJL zBtTyjI4p-JXTikW>UXLFjE4SbpgqtvBalj+yvwI$hi|$eX{n{Bh!%+DlnAvz)p})b zflpga1r1@1t}!iDdlm03c!5?U+aT?(L&) zvG_H8L8KDE6XdF~AMTJXtqQX);agLsg<+KB(ES$?GyVYZX0{q(s5FyT%EAld4@^34 zHFJmqo1XF&&_Knr*)dx3Pmu`qs7Qob+nk` zYAm(m4z8z{r@$~JW{K7H$qJzKk(209=-5qS^}V`{ufMBzNNjVbp)~KsPl9&V8G0kw zP>{B@TnwWG;CJ?l`{;@UV3p#~T4>tj%dpz>16D;FCr_H<|aP9<+>)~ja>UOumg z!}hiqy(mpI)_`_q*%!RIG4cfkuBwO~lWm6RfNXI#jHr#cZ-{Ji7YZ&lSC>s#U=OZj zKK{^_?H(B(ex(j2K;{DUTt33%NaZ1x5v2OA^T~SvO4m6`k;jm_0W~_6n0S2z{S#!! zu#%t|NB$v)Y)1YQoF*EPkvkAmoxuGMz9FTxr6@SBn>Vw~vl(dNHp^qh2)2VM6-BSJaL zQOzzj#vnpQzU_GB#)?ru-1sGVEssq($OnM9TE~Y(R!=U0(jUXyY>Q$zvrg;QO}EP@}kalt23 zS|vjSN7+z^9V8F%m8Qe)yo>WF{dr8O3k3G4W<#VHL^xTt#g|*$t`IaGG%O3X)1950 zWNad2hB^?#)vxXrQsZhazArFtZo^_6vjEM^0i|sBU!etgoc`yP{Mgrm`PkQ@q9Q!p z{NZMy^cj^VBWaqX!uHp%_mqehr|jvN|5ZS`P-gj` zN`5cwJDM1p9{BWt@7?_{0DFr&AT1>8OW!MtrLcq8?SU}~NMO2SFXUKpJpfD*IMsd3M!a+ZrIf@R@ywr^4uw=Xu~lCWaKmfwZpS{ zL3Ku&fIPg%_#z3K*cZ8R0L0Xt2z##6s&$rx&o0XsLP&<|Vfw^_!ZSJwhuC%RS$zMt zJ%v6%U)?$*0w`F3>WX(BQTR5Iv?uapRCzhLZJb1P*KGm(tvhjYB~}WOjMyU(S6@GE zL2EX64RoWlNjLdga36oR^ zwL>w`*DUuR4``6F#0~LitQw-czGr`04PK3mXeF|N%r$nNNkA7u{``5L0sD6 zbibP3KD>Zx+R_K+G3dW-*<1){i$}bm|0NtXP59L=>5}N*g$I4DTs14M@smg+vM=r_ z5Q-AI_}{SpYhZ#}K?L?o>%giX_uPHDxAqAzGTKd-pLj@k2W5gBh)u;(|2O>2%U|5n z>I;ds%1oA^J>099R)nY*0(d{O?WJXJ(}sje}! z^B;~WTq%a!^yGn@{WF;;%`G&*;o5!6Z7c{4BngD@VhOtGdVj3$t{`NfALy!V&Q1jy zP4-e|28p+&H}l|v*dDtNW1o0@Zp~~iTI%aVs36AjEE=|dQNx1sE9$-j`bU8&Xh|=} z#I%8AhGa9&u_;vdjcF{Cff;Hw(Z?fX8R z(pw*U@;)@&83CBd8(InfRXq1k*_%vg`@cyz$Ll{`u*U1!te+#K zaRvJo*{N|-6vL2g`#6w_SJXVw5lBWyJpsDLfJhmVSW_;=e~)}TYWe&_hosfL$N3QK zP*z#QijZ+@=)ByyNZZt;n28buy-X$6Fd~P^f@(UZ03-5SrW&1Q;5V6(gkUMLiBRhH zskoQGcIw@y|(sf3~5 z`?Q3WBTm&HK4&3m^6o(9imZ0TCyg6HXm#=~?}9p}lk$Xprr1K~Uf(PdKmP%o;;B?bv{}8=T(z(M&bYHHmi899Y>XaE?)X8vD@^w35%DT=`r$C*o4?UKXI|%8YS?=gRn$Ro)%WQKweD*v zi2ls23zsr0PZozmFys!_lqJVCcdR(aJ!_Tfrn~ktN`;QO*>*G!-+&$-; zEK2OAC_-JJq#EscznsQ+xtKv2DqP%BQwwy7%YQIe`19LTsYFm2T3lezq=mEq+mBn< za!*0g5aERYjLmsJX{g5fj;Z8-i=+a5I;&0HY5(tU{d(g<&=NAF=wJaU)3@kl{(rB^91(b*7-{I;=L3h9y1*O7k%hYRRaAH`sFqVeNcfg; zL}@9HpjrWeIJD7AT2`@7bwo)IpAc*92Ymj&i%1ERK}*zHrY;l>J3E_-UtIy~Ap;(Z zfObB(Eam#Qb_T%Bl9cPA{QbeNwZQlDRJyhII4KDhPWk@*smkI(khSXtutW`9B zE%yZ~VCq?wsldA1C_k<{b*fO^5p36ea!zomiFyUkaHtJ5Hkj{Y2kyUD(M0~+3Yy;Q zsueN_Eik1<@P~ReYSzyr?cGxgLkTFt{n;1Kp>#1)7xo#;DrEmI!mow9eq7t41q>%+ zY1kw*BqTQR)QbUF7Xq&*aHkULaFZKfzJBpFd+1o{%RQ-arqQt<(4~OBk~I~groh1b z-fw4MF8$|vPzs*mwV$BqJ4P87-LC#e}(+rGp1?-I+UnX41_gm%Vz zUW)i-)m?emgekkS8QIqs~YEO-&nxm=kpQ zGQdF4jEpJL7>bZt&T!#b8jL)|Y8LcVPY9fpx#(YeSj75m;rmf*cGEe>Cxg(}!LuW| zuR$?4x+ASviu*0zA5?W=G-M$eU5niD6l$k~-!j+otuuP;~``7#PViJC|Jd8)nGkml> z6~xU2(mG^#B;S60zDk%bk?o5Z^(jCNEdNa zQ}WIW^`#>3K@YAfuyECFtt9`Jhy*apNemkK$iM#em$QyQ!0!ZIj4aiftD3Bf1G$b;M1;%^iR zTZXqf&KobCJD{nw<-tZYwZ{Qz(^C+-(Hei2bAFC8=e>=~Va$@T0v1;~H(*n@Fy}dnYZdRhAqH{Yv6OZP$2m@_9Aphp^wX7=t-Hhh=n;ro9pFj2g*ot|Xko{5=l-?;dw22SN(#6c641vFtVC zkISwZbuO~9xcK2mR#rUhsbGm>RGipzMubhjH}?qw8zrkdSll+`N?)-=FM~6W(+wl} z=?Eg>Gb!3Da!%4TUR@g|hAVdDo*;4;Rtnf|^ZFj@{J_+hEsBs>8krI2@G4>IJJl8C zZ`K3XTS?y?4DD9Hn4S%VxZ{iryF9=!X&jd(p5+aH&oB(a+$Z)@nZ+tC#pm8M(yDR9 zm``?l+URm#lD82KLNLEt*OYU0K?2UOeX$L+?|8I7Z6XL%ef0A!;*akum9E43$k)0w z4s#-BT}Jq!FQ+kZ*vQ#t1RVP_Fq}<1Lv63dP66-*8=+I)e{D(F<4Ry1J4P zUZ|Q&F`3P6T#Cxk?sJMcy3NsS66~N7a7_ z%EweK`jTj(Q;eVNT7n&n)3=A6y7@4Dk$t!}R4yeW9krMLy*Gl3S2809`7Q}{QNqXU z<8mp5aj0eFhD88w)6Q2;hkqRF z;#(l{x&_B{{TH!;B^Abdj2d=4_#~mLsh&FU!S1gxdZ4clE|R-XRRXM6)|pokQABUL zsMi#|4$;|1UNP6*rj1TouMXHTZ@Z?4O6aB;>bx6s|7@zXty1@y5-z_w<-1T&Z!W?x zNhmo9!(uC8WL}VBb#8ApU|wR0LRM|rD`KiFKB#0Uuiu!ID0Y{A)KY}HOKT4pDSil% zl9FnPBAHoPBlp=w>19g-`m$X~vg8uT$yD)q^ts~&t^OG6gFhTyRRI`X7o9EK0avsiBmOh#<&n4Uw413oYa){lET6#OyY zm!ppm?ljo}xKkqcdwBLqVt>a6k=#Ij{5x}%Z}f!5l~1x+sq}vy`*5SX4CZO= zz6#0$AClgBXs)sxXuszfqB?r2fu*+yd3dLE+G|z7j_Jw&%hK?@AaZSu&oWf%xo)PI zo-tE4!6`(>M6zMa#L(0)17V`Y)t(S|6BED)ge9Sum+JN#uNT2-?Iv@Z?s7@|z2*Yc z)NsSLJ6>3{`bwqxa078?sK;ylAB%+SwqHF(b%M)QOzCPo+g_fJV;`^^_~J?ZSkMtZ z=3caaSufNAQ}@pyVh+lyB;GAcGZPoa=D=RG^K z{@5Kl77b2U!I$64mS_&b|m;+sOme!iF2W0uAc|x>J zpPG@GKs#fQH$Buhlhae9oH5|GU6PKYe;d6P(!glzItr`<}{E_X-f~CURXSk4E(>Hgc|g< zeZ1#ABCrM$a!`%4JyTwP;~o$)f#M9d)A)Zq0@b4AuUtK_g}(wQ%rJ~WA~psmFzo%*yJG?3h91<=P>U>CY`@v^W5scG-8ina@(5qEGcxQ7Ux?{#0D zY$)aGi;v~K$0C`j?hGDQ)jvxc5 zHpe$c&8HR`)nx@=uXZLuizjwFrJx6Q;@cfoYybcMxkSacdO?##@bb`g_Cwewsa#9s0xG76$dbCcVE zS^i8A2kvc$b)XIR$MD4CTO70JLvTlg1iG2!*4CSInT(hqNJkGw`&ztaICV`>fT`Hn zRdMreTUj5&gL#C`@vYNlEBuR3ZQS|1`(Het0|lTh>6^fegAI}2$%yXZh?04PY^1ma zkz}ex6oSfNArUNHl{E}*!q>kE|E9$6dcjUpIpP`pW4$n|CJ@YN!Pa!KJR0 zsi}pZ|4*iOND0Ktp(QdQchQ^V+q&~4GjU&&1vFk6Y3W{W*Gsx^k{cpFWW1kEWDSxx zI$tj;NRIS*{wJ~a+<^^UuC5m}Sg3r4gd~b8e@F{=D)Vite#vAS4;ETlSjc5wso?IY zB9yyXID}A_mu-OM>wURn}gl27zeaxc`O-_S&L0Q zJsDBVbuNNMJu&$+Bvh)?W-a8nSlCnp&+jBJ|cd@)qUuVQWe`Z@B zND5FC(uA|VpQf=M%~`N<1HZ@0n?>prU$w!2(&nE#JX;Ujns_s{J^28KVBI7oTC-3l z4xFKQOD%R@+$1Q6%Y5fp`o4sfRO8Zc`g6ghZ6-e+^G!|z`l=#6AMgBYQdU&rP+f=p zuWiTjM1s|PTzapGZR$}KMG(e2VcG+L(+mnM#E+mP<%1x;ez4tPHSddL7+2a%s|fYkQXWVaVm=@aU)Fst*Uz zM4?jW+*i*y058BoLb4vn_GKXe;JYrmR5hxG1!eOUf!7+pfmq;Qt3S|ceY3su`I|?a zqxt71LY}}E!Bqcb17mT`)2;I5yyMtEiAxLqBh;ZA;Bg-`>Wz38B;ax{Y`#lBZL&MX zRdH~MG_&^{+7%d1M`mrkF>of@Kbk2k`O4e`N#%2TRm zPDwvdt@hB`wW5A>Cq2fXi~cqxU#~c*+XycrxvMN&!g+2XAv{TlB&=wDF#NSxbG3+c>X8o?GR<(TflY-#fAWBr_EX)wch~@d~;|CzZ1C3$VN;oHaYNs za8z8OU-^|KeJWnGOdvDt7!e25=cWcHBeGdtvImD;nihB6;QYyA?Jx>)q=baq>m7w> z&Q~f|FFest9bC={#3UtG`l9=%s_s103&o@Gwx)|-DvN>kbBvfaXt)FEmP8FbK#g@d z#o~R6a#G(Sr^Uw{B6;mi0@VDUeRooHe;nL zaa+%mKD~WmcJ9Ka?_y^WM(Vr>^RFT>T*Fc7svDM|g+%MM@90mmXD?3&K|F4k#OGO+PF#pTewvbaVUeJ!da z5i$ok12bPdcYnp}XM9I@@(8$i~cC+1s z&Jy_T>gJM!98gV75}DrW>r=#Yco0`x&VTJ3B)f^gzSXS4xQZ(`Hkb}pT&q3l1=h9_!x2iQ?(@ zsG@4vr0-?KE4Qh+-JhAVn?Ec0I#dh1F6YL^DK0m_q=M7+7idgc73vcHcxY^{hfQ0B z=<7qghD;lr2aVImh0s;Tv~sSRO?SctJHd+gW||g|g@VGho)H5>+dHzR*{^!9P;6+8 z|1ah4Z{FgAv)MCBFeBSOh`!1?g}~(x=>G)K5#U1}!Kg-ESr8N5Ba{pJj6F`s4&7KU zw>4TX8@l(eqR^rSP#D2nYT55AF}m*MdA3xkc%W=jA@_KiWTA-jJpTx2}+CL3}JM6tcEXOmV!-{s$1XkkHdY-z;MQNFLZT0MpbN^3aL#v((U)BH#CO@^`0(+K%2&93gHL^@ z2Grc@=44z!JX|C9`=8fVVW2ug@ZQc>s_kgEk7_4JV$Ar36?0k$o0#O(LXtxF9B~-m zwVdXH63BSbt2j*~=etcXAS#&Ndl>BjA_)5`b?$dhc{^>#Et5NBM*!!)vv!p0Tw00k zFdHJwN{WQOD`tDCFCNmDvk7E_6|cM~CBv5`+EC=@KzA>KF~I?Tej*4{opTUZgPiCZ zEdq^ldlE=8o63H0&mriU+hAT2_xCi=vjFY`F=x=Z+*q!*=6WYEEaWIdkq`KECZ zRvg*G1(uHcnO&gGYXnGVJI;4#r}4|_{YIffj6rQ`@&=E$A%_c9@@9KoV{Um< z4R`UAR$X>)kX?&Qa6(hUM>;`y{+zJ2rU)~x!n^JAN-y4-j;66A)Sf`gF@Iljd( zW1eUb_PUJ~I%u>_kt}^W?{XH2h5%H67-2*lWv}+^+`r*o-S-dbE+xD4tu%+=%~aB+ z_l0a8>qC}r@ls%1dXE!t%&%zCK~f(mpkyDBwpVl=rTmNy!;Ap0CkE9>v2=I-VHZVC z!_>KlnU~?&Yl8wei37>h|DZ}tsXu#Uv{>HwJKbXfrt7xJoyTb&8TYma(*OV~slP|U zi~Iq!{i%R$a$gq6Rg(Q9*4te^fm0G`i3D-;Vt+p3>I5&tdgbZnSZ?EVs-w?UjrHi8 z(q#e1t;~&qwE-s%@=3BRET=SH=xzT-{GqYP6 z+Ogf+95qg{4Lo?%u3|n1X}*^FC8NB<2I;7ET#lZ`&*SjVVr0rWrrO zW}uuKgD6C>(Is6iVn45be9m;dfV>-z;^Qn;dPRuI9)%O-vnoR+%@A8bw{64K5sC{< zn!STpq@YpM7*<`0H*e4(ae^um9o6QP$2J>--OCa3BP}tkclbt?f&1Tkl?TiffxV%H zj`kwZkWr($!hp3H*zhcM%G9oxt^7P{vsKHLJ|rpoOk|>Tss;F}MVB2i{KYdN1{SkR z^nnR7%iMxjj-}&1I&J>ww;?^dHjt#;y52p$a!BVfPKw@XPxT>W&MA=30aZ4JN%ZR2 zAz@a#(HUou3`F^WVSJkEU+W_-fpDu?VM<9hfsW=Fc>i5G@SxiDq|i0N2fbTUA_s3z zjOxaIrZg!mBnmB1{Txr}>PGOv`xTe{snjc%Qw-VB^;=)*s8Y+jr}f<+%T*Bxu|~WU zp`+)0C6l_HSTnC-&%Zj>EcwMe?ac%7*uH9>sj8cm2scM*zHH1&4s2=uO$t)e|Jz2O zf;}w`oe8J{Xo~C*xe}wv5O?Va~0GyhF{Xi@|x5GX^G2-UNpk|=k4eE!UFHki%mOBSd zw2H}@a9{YsA}Mtzp-ZPH-*cxxX#yKzQ+jdvp^DOS<>}l+=sMV)sTwYCT)0<3`;<7m z>d)Q$gIQTa5m2(ghJ6U9I#YUTLRdh4c2_ty>*#cUF!gYuf{lRI3h%R%ZY3D?r5AH$ zELx-EQg>Jb*askY(94FOBfK(In^s2}Y}W%*evDko;2+Mg5`u53U-Kouds^qIBi3&@ zQ&7u)>cXM7Q;!=tXW>*snR)=(dw=RuR9%R8()8Ql3F)n!rKAuzO`%s^olfF!f(N=E zw1atU7Gg$<{e-8L7a@foLAPg<Uv1YF1Mx|dB; z>1x1XoM-Gad@p6yiO>45i zO+~d_SFhQ9PL6Vpa^~0eN@*FHuBCQK>)ok^V;%8KnG2JPHx$HG7e;=^C()knn;r_3 zbwzhAFinY)sk2$0f?%~TC$xWP^%YEj^>(#LMBEHOC1G9=7BK3=U+p1Zt#SBSXP0m| zuC|aS8NrgHIeAk(wop1i4CpUtVZ}PGDA}`&rz%ZCx~(TY4tZ$A>A`5MeE>^cvJXt(y4n8A2MVNNhR&-oonREPxZ!#$eei2?>D z+P}?mYAAR>t&yJYFJCIkZ_boo*nBpZRQfUAF^2)34Y7Q5_Ut2;wzxvpco0U1<+1r4G+z4bUMWOYDF#E9A+<_wryXTSdl^EIK`_l znzNeOkyG5OFe&O911D0HFh{1)$OOG9yTd=};*&*rA1-(-gsiR_ext|o!%2xvzh@)@ zst3`0+L?Z?r-9X!fz?#%S?|j?7Nd+9-~5o$E6}Zwd2LCy_;f0Z;F>U;SWr_yU(% zC#wMwEEeb69U$x8gR3KVZS8GX}&$45J zLpEc4o;H#<3lVc{pV4zh531RX-n-0?9efRi9*MDAl-1N8E&jf5+(eL)S+5 zNs>8$4G!VSp<52y2L*n53*l7UIgMlAJT#=OeK_wpay;~+S?49+bo)6t`2nYKhsCwY zICI)R-`idms=5#eMCy2v6MNL;(AC&9^}NY}-&w?NoKF1BFedjTQx}d-Sn*%2;_wvI zLthV17It@3|9_0V1ymJU(>^W;N=gb!Hz**|AStbsAl*vmp*uyor39oAP^7z+mO6w; zw}jFShx+e>z`gJHeZP18*J8Qux|i!Yd+*sZ&pb2pY+1yePto)lHq-}Heziaq-u%bC z`#3r>+kCQz$@ETEZto2eoTI&+GrgreTo1$w$tDeTPC#|P#Y>~?-Rm{7$yD)a!E8re zt24CBw$SEz=0s6`cgl>^5W5GJT)o4(o(Pni1iurk1gw8`DDaOK-_#-Fe-Sr&hAXlS zE$7Cx7@#`VZLn?CA|)YV+7bDp^I7TH7p>xMP$wA7ZuEt!|{v_6x%676zT)GHkr&*hVO7oh9I@(9o7H1<~n=8HA&Nef8ys5Myf?};`cUODE z{6^!w72ZBR(?Ot~?mOTnmTyySA{vSZ-9sX&AW*=pv+iC8is6eu5qaupQ>Zsl^+}^I z$e^XgUaTi^sC?t5t==iDXQafCalNsz0lzIz8E2X>lw3#_51JZ}n0yQK&+N$&x>}h@ zg5&fYB5Z#x+-&G!-eN+DGW;@3Z!=|>3&uRqH~#pPAr1OG@4G(bWeCxSH%_?=u;KRU z*4W>tmXPUt;}qMLSJ?2rccid!z3ugy&xjXB;tZ@?KpI3|7hEa^k6o!WRWjF?+EM{ccW4Vn@*`1bHJucaN-WbTO- z?7hgE11iHM{mjBV_|1N@%OiOF)O<;Xa~XB+`Q12mTw;s@FHIed&0#9syuB}LYynO2 zP%F}V_JmwOTa*wz#%t}hU1bt-Vr2K)O@jan`I*DzS*a)YS7Jnuf(Eg1UE&8FEiSqX zX|Su15ZAsv1?DZGW|OYlojys^X!*dAy{U!g_An~aXKx*n*+MVvmXVLQBO_yiPhL?j zZ>inq*J-Q@^~^#+**dB4Sh6|Xb@&i0F9uo&a?`}4A&ShnB4UM#iF7UJbMan9);p+L zvQkgg3MfJYKZUw6s-v$ey9rb>*2)V{khkx0{QjZR|HmnFo+`R;$aEfq$bDzc>7HN{ z{?l`2W(_B4X0yu7m{CMnmEBTb#c4Df(LSgm^=BOE+nNcu3Iu>YfDG9c6=(frbCtn* z$2Fq2-fCGsEO&&L>$tjnXR<)#@ur+*e z3qop0h0gQ}gP4+7?Fdxd&4LdbJ=Y$JNj&i!8)I`lGB0ch-M36FD#;_u#y;<^u$W@V z?5;E&Mp)`iZm;3YvmVx_8>g0x__kPBF$37(<>^9}3eWSKEuwe8V_IwvJwOXs?Bo%; zV2dO=VOI$V%<0p1^@cPqT@sUS`%gha(yvm7Is5DTOOwezHW+Ze^%0pG4pu=7D2${s zpSKKTb?0>)&SBmVnMh4|)+2j}^d^UyP=1lk@kGqM*e05QnHr5z@TicUZiGJM0@lw6 z6U^P)!Y{Ai^0J?1W*Tu(mgBr&Y%)T5*1Qu2vpyE{+)JWZ;^WBX$jC%o34(be^aNek z(cWKr@qI|+r2kOf@O*-C%9}tsNP?KsQ8J&CInL5FhftajWJ+Xp-06m>KgQQdr}$!v z@bA&arU!g-`oB7cad2jj-3rZsNx<1`n7UF(vyXEa!r>}kj`H5p@Na?@66c5YWI z+Ctm1m9j#E^J1XO9m_GF!fCA;i_bMbxrv8~?`poM(s0vPL-F>*R?C{IcTIDGlHKVs ztnk*6$vQ{Bxi!8%>3mtd)4eeDK)`pR#rcZ>?PQS^IpyWa8_I_X>9++7DE>*K2hGo*0t7Yly2H+8RZ|1|mm+ZvtHsr<8Yv2bmq zvdGq-p}bjNmyLw+pwPg&{bhaTsQ2#~$j7j;AS2QRD9i9NGP52i@zCPm78gcYCg29dJvdGO+cIaClr@7#a^k$_NtVO4yWlod7VR^#_H&xEi-o} zU_M6ld6Y(26|Fe1xfO?}h~&k%SE1#jMaf&vu!EDEBOb}3@t82!EaOl_qvc3iLkz|v zYeDYBi^^AyvOm7x+I5!k62RsM4I40%`+;W(v5GIPUOnp?j<^?=c3zIh>v{n1w?Y3lAV3a^vNs})Y*E<}>=H_PW z!%c&@?yVCw6Nlto6-JC*JL38Fs0oue{Uj|uDCn~J2jimIyYX9~a1|9R zVtu1(Q`c7M8*d%TL%v4{j9xr`60!@`7-Q(KwHn{qE|yC*WAev`#dLd=qw1 zYDsk zH>BerxyD|6qS_A5j-S;!Q*JNrosM&~h;oFHTRarv?j^H$5q%d+GtebCaIfO`$kx2` z?8m3uv;K*kX2L~AR0wWx5Sn{WInZwJ4<`w)Ek<#1PI*AycBWP}8^cSw(`KPLhIjAY zeLP$oDQ+yk?T?nO_|Xl}jISW09y&&DqSIjHyut=)jB5*8=k7U^z~K#4METiKp9Eay zOgabn+SYN-{ zJ=OKtYqekWI(6VCVA1jaS5Ecj@G#_&!?XT3?*jorZn&)@0+kG=lo#{N@z`0aywfpL zCQ1M)n$3|ODVi_g?^MA=%TR6>Q9lWEalUl@0EoVY5=aGk*6s}1(xvIvyWVU#-M@YG zlVNM0Ay(`~@hp%moQ7oUHNC|_!NgQh7MvcNg5-ElEp_k;d*f7 zRN|_kIQO7`X=oK*+{}E9j*#zMViZZ2Xf>MQhrgy%V~;w0UWCbR+#``e7pMH76lwQ0 zKc(Z@;`oLE&1NdcR^4F<0eJVBNp)yb;vIg>TT#xP~ zb4FG1%qhHg@=ZR&TjC+OcUC>zmC=&|Gqq8e4jgbn!W#eQTTMv5@ zg&ws((yRz2p!CA%oYt08K`XA}Ig=M+Pu9%n1Fr3F)^#hGki{SLpv^*susZDI_|s+> zf}j&C%`$~X`0eLyeZwybtw?u>30rQ`5|>x9S`3ZJrfyeMPeci*c8dRU0kbPv07lOa z#osm_kv*w*wp=D2*&DYS-JcC07a_A`_RIBbhc=o(5Z|M0-H>^-mhLZ3?%{1~*kxoA z5jz?W$01|H3W4y9j7hyj#It}I+Ba5iPC=KQIW)b%s9MmO8w>Axk!|&ph<`-dUo>04 zSkm1&GS!#kt)sggs`iORUW58-FwFYzUD_Q_9;_Xl+x_kJ*z4>yyt%o-NA7To%3%%0 zv0htPP(3?ZYIBdg7}W>Q61;^vuoHSNcAH31p>MHewaE4BGsVK z$f&D@RC1sH&;{vs5(fg+c3q%wVLaBld$yCt2y6wvZ~9d;;L@);=a8u2_(kFsyn&6! zJm#4VSpi;0FrurQq}PH3+9PfGr|ET+4A?0y@=D`LA(& z{jJ%dDsrCtNd%jJXOb7Y(1hR_%5Q3Fs;qnJ{QW@7n{=$a^-TmpCk#&QNi9&lu6{1^ zqt5+zyV>x%3Q8$`y?}Udp0e*70Ojp}D+Ug>j6Zzm32K#sx~KQ4%Jh{V@S!gw-`uM3 z;g$>)S^XZ3hDsHT|8#7P2DlN7u&bS^?hVV8M) zJVi2$crpUegX&zYOX4wy5$Cl(>R4JfcrfW5}SW=v*ny)N>nH_x+t|9Llp)epCL3 zD%K}O(*C|hY?|=CcMYRVR2#>2d>q<%YWWoY<-T;z^#{5H!({=lD=d$ng{Zh7u{81v z?_i>Jeh+DsoaeNW{jl$p?s2SQ(w9coo7(6t9uh|>c=nfkQ_d_u&{?o{mkVi}y+0&+ z5(Z6c3^N=jhDjTxwD=%qSGqiYU4zMlwT7N=@6Fy{sQG5%>Gov( z{^1bSpOfPa7Kxrj<&SzgyLY30aa)XF;Mg>%mj-N#B@qm;G}=%j6gCPzuVr~j((JF! zuXkSPK|5tST5`W7g4ghI!>6#*o#*0l+~YQp3Q>Q+@M>&tY*ekc9fF5cpr?-%i41dE zFjm{t*Q}4($DS9{q0whOZx9@WKQLXf_2!8Cr;J{d>C-7GCtLIMMWDE9*SgWT5&zWO zZ0K_VaynY%!5k)xEU z$==%spOuWzmc^BJRpiTjgM)`Qs;8DgX_d1Tqf)c+9ATq3!JI%auWioslrD_;DT?1l zbxz&GxbanxgkX!|+fY)$TSIARCR`}QP6~4stJwev-S-eyw>)3qt+R`x0L_s}3J1L^ z(drD3I$K?aOOEuNw^|Vtc^-ALJ)d;l|A~|wV1P&Y$LrPCo!^Ne9oy%o{hdtfWNgc{ z{CfuN_G@B3JWErljW@c4PP>`h3ELlK3~#TDdjjsDSV~=P`3HLR4AMD_nTKFcQYWlQ#_xE6%pZB&}IH;^pan5Q)Nc! zX?%u+saa~|OILS(e=h$ARZxJY+kvNw8?{rEm!{#A1Yk_W-~;Mb*WGDS6pvBiGm+c^ zI7q4!eDPphMc{t;1}~W55HMcKOA!$}-wQ;p5ziouT1pCATYI@xM;ii$nr6R#^F_hJ zk{o&%OG)K>+qK4VEs;R|6s16?{-Jmu6>fdt`<@t91RGxovA06Dy9{DT=fQW^i*aR* zCdT(PDAj~qSKpK5fmU?}&Fs)n+l?CJ=`#HOwFna4;G}2Iauw=prdwv_7Du8o+sjx= z&z}b2wkoLLZ37Nj;@KQO4upy4bB}_1Md!3fCFM^jgF8<|D12~yi}fA$(o(9^?0kgk zjL&VNxlcJ`hIQT=YjrN@#gJ(PjQeaioTjzgKA-;jUTVGl%QTeasAhjLyh+8OVWPM` z9gqVJWe=+Q**M$u%E7kI&)kWmZT9p8oKWF*WP(!mz3cM zv!7uy`f%n>p!tIA&K0d22xwgoLeVLH$3{y-Hbkr9ktgXVsslp!w<>^S`1s z9Xwi}Wz!>$bdt*+I%@Jv<442Xp79f>BDzAWj%tR>*|$b^$F$({Zh7fcjPXxHzC)>X zgHQ@ri`cI|F$xZYKHqy5SgQ98cXhNBtJxp3!qxIi=8o}lU;nlxDd1P42nplXe&&zA zEo^|@5pcDCI$2ZBl;@sU`FG!{{@cD$W^_fst2=$e{YbmX>q;j{eAYAAi{0wp@KO(; z6yJT!Q-yr3oIdjg49|-kIaxdME$q|H+4(Z5_YdAbeuX3=%FeRszrEwFivw-mGb#|(VslMD~9A61!lme8D|#` z0gsQ`7cluU8uDtmDQu+CmJ#=nj)QqtQPYt^`MhZEcqZ)|5#Px_U2Rth3IjYTQ|Xiv zA-T)x*%rEma_3H)AA0;RVi788>KA1uSwqBhEuvMR7V86AhxLEt4i6?fYF1jccgC*t zHF%hFSdP(X*ViJ8e3^g##BIN>dUUT5@3H5K?9N$JdTh7Lc&KyD`|q5H@L6bXEsoXR z6~7)oc&ppxk9RomXp?$0jgzExO%ViG4cXb*rsFoMPk+4t$C8zW!CltL3}wD%cR8-`h~N9|Mc zDeZOqzW?3Gs;uZUBnRBcPa$pIAcM79pC*?k8~OBFYo~N6<<@_hWu)*8>Q6HWche8ovxF6282Zjk43PQmcV0eLcs>CNnT7%J#(($aX zTaX@ncG(*u>{(M<9M*Nx-SuT9~UU#wgk{y(7zNk>*t0t2--d zuViOhN1b{@oOBrswZ98oSqnllUPnAj;p~zL?^7ThaPK>$^4I)x&yA_m!x>-NjwEgp zFlG~4Vkk|`x|NOJyD?UNd!bX@6suhjvKKdYm+`0Gctz@qbP<;yg}TH`y{&NQAWaNM z4rE!(zdHMB$`yuhiU{jI92O5Hwr#YkD2=8>TP__m+flo8(G+%Ik{zlwmB5(@tl2p= z_2{MHLWQjuD!*EGhVuRYXb=vSzL@SrRToc_mzFLKru)6|-NyZLQkS|+=9yGex!0V{ z!VecM0X&wRm_{+A?f4&_&&8-{nBsA=zw=(fwrn_nY_huZOUmXoVMvq%2NFHD- zaj*HtgvLZYR7;(P#X*_SN@?u6@8_i7qu=Yv+WD^$e~QN_NMpXfU`!BhH^0>#XTDN? zQ7=U>?)|`-gaT%2qUkXEOBOOw&`Ukjln#e7PWRV@M70uUr9aP(CGk-jB5zP>tDBB02?7=u_C2KKYt63|6;p5JYAUjkXdrhnV7 zx6j|eNM&=bIYIuPfw4X1I@tok9qkh>(+naX@YnKrLHQ$o?not{Omt^4LM@rXZpriQ z=`j{O49&vBQCk>NVBnsJ7`w%YUS7rPrJPjHZv!70Ia%e{kD?2FBG8y1%j0OvgqR`w z{kfxEtCzJIELRg5Q!ziu4x)BiIdL#4%_%} z)5cbDzP-_9sje@jZfMv<*rD}lTkQho>9mv9w{71~0|pm5ACEO$T^rWpK{JdX`~KzJ zi_cf8nUwQ|6@B(j2j8d@$J*}XqoK2c0H>wci^Z@1snwD~sPD=hiM%965xu>g&SIx- zOEpf4XCN%bGlDHh2E&KB=H%LhPJT--|HlS^t+JAMl@=N($db_q9yCo8-!gaz_ERLx zp1D$H0ipdk#qr?5TYavGRaEB!S z&^-R3kS@wHu{eo%s$hii+Rgp`eg)wT>LQM7`IsB9HT4~u7oJs5o=76X=pPPAY}Fwv zvHuGQ0+FZZ7w%M`GN==vxP>^M9>)4_{q1H>A&#p)HxrO~?qZwLkAA1;TA1?L}}Xj zZMHa&Jpr!LSMkTO+iQ|(0U+5^`|ok^P*Hf~VGgBfsRpA1Q?BNSXm=mJRn>8M8X^J- ze#r8OY|6!$ESyXr95}vo5x?4nU!aiR3~x}+mNCU3JK=xh>uXpY$j?I6$89xn1DH1S zM|AJDzaa8%!&AtEw1{xD=Dzm~(->lT|9rzV95D9}%y)m)`88+(04KZ17u>+|7_pvy z2>~eRywE4Vasmjd@Lpj*#hG#i&#J^~aCiNh#J8lXyWP<96Vd-;)j&-E_|>!Sy*0`* z=1-qK(E+TRZOY6JdX-ZX%6JK`?{5km+n5l^2JZB53+eSFVlt?#KH@MPz@D9bb@)3c z8BD4+Ybe3SH0)u$DmmYR@0u4`j>{+JHt_!$E)26#4IFrW)6^z+z2~>!L#AaY68LkgAcm)U?DL9=xkj~&e@^7`xRI|ZxCzI(_ z4MyrK@5P<%2hH6qD`j9P56EqyE*}F?wEnq)_yZJTzU!o;N*Gu&dL&{|5QtJgKxb7f zb#SIQI1vh_&vR_~E$2b6* z%n4J5H>9F;5ySBNR=R!;XdDFH?y1G^OHz0&diB9TQ0K4WfN!fqbJq5b-RdzSVWGBY z>^@6%f0FE7U;+P5gm;1dnl>b(ttjWjnlovxbwg48Kb#|CiI6Va3^h3xd}n>0$~^SutVq(S6cCEGhCymafdi=W@{D)NTF3b3s$3#P+66Px;bskR~I-ZfNt!C0$3}8>jm>)M&7> zb5E>?-~wcCG_Sr30xpUST$UhASRQS8aA3gdLX-g5!sFqv(ra6|0sK@iBIsH_Ljd-YMRl~=a$zZns*m0bXBmif$@O@mH^_Dh62@J(x;RUM zOXqYFI;IIS8~~CSaVM_-8!%8@;6JU9TtOdCM^s|sL*!lje&ScpCQ@%A@2<1!V_koD z{R{f_HoTjjoEaY33RyIz&GGv*2lW0C%8Q5RZ{XPC^HwJ+ws#DXU?WMnQ!6e@$sFC+R6g=Pr>>O9`zehTxXT03A zO=g4z@RkVvY5pK9H!EDR(5`FS|Ex%yu%`_GPOhbdt(320mRoAI0y;!ye{Iwm5<_rp zLsDPwz&!xyHZJ8!Py<4MKuzV~K=#M7C_;VCA%NNc81fBN12Jz;Hwk;hN{}u7L4l7b z{|iW7C*ih>-4AjokavL=%?sqSlzZ(?5kR_Zee3ngR~q4dcfA5)V_Be$MULo0lbxe) z^+b{Z=wp>;SS{W z!`3PoqpOcQp+N5yFVVY!8${9q@M$UU^o6c|*C!N&Q)r;v%+Ch^2ZC4|jP$P6wKRKLg87HnfIv8T(p#c>SoP1S`~kT8BgG}SPF|Mb zVc`xorilo!P_zHjLf0X13+*f0iTn>PuRW!WX6CcW)R1Ev|MT8p_#ND(ocSyT{spYN z(}59u(Jw$u>-2-o$B#8|9*nV!9|8_`X>}y}8gmLX_`=tg;`zknNm6=>p?z1qLy7#*Ag6h$`0oWX zV7(9_2~%Fq{S`sre^#L&u6*W?k23#Km>eaqg}3H81+{{^ghSlEwx#iCfD8P*9zQ`) z^(YS=GgMkCoCn511#zbAT^sK!AR$Sm0~&!`f^7Gujg0>Idrn|&IWMWn*{x-~>R;q% zhhe8}N|1+_U1{mHQD5W?N5ub!(|yEspYuUlrTDLH!BgjWh2pC_dG7N+N%BX$L{W@ zFhpgbm>|!ZXS;qy>39HnmNmOA$fv#+sL7uIHV9gdVqYS)3G&Km>z||)MyTg28~K@w zL=rSL$;|y{%SH#j*)!6J;cvIrhrm{;DJX z=0NOWK*XP7_?L4#eD_zo>)19nc^r(VJrhpNKwVC?n+jrH#Z4oJ5Tc`BGsboBK}~7z zTWes=ob{k>nETzM8MD_UZ*2Y3q^3%2^I$eBL_WWMaD3q<3Yc6f&~V7rpkYa2(8WIz z_08|Bp65fX9DHZv+0IzyAn$qd%Rj1)8z_Gxh11HiB?L~>o_^QN9W9Nhq${=b0;J5^ zz|PmpUHF;cXo<(^1($;TPe{K0<|@_P)sdFUTQ zAWUZsi6)h=ii@Qf%~D^s$TR-&8W$#SoTOektWAl3a+hd=D=>`3O_b zNNrkaT2*f_6oyZh=)@ZC#{so59^i#iT>ii_514JaG4MieN@@~UXeo{RuNNA2Gn_2V zIdPBn76fGNfi3igW)escApcCdL;~s&DE6WZ3TMZFXk@^cCNI?x3OSGs2o!{{#D-aT zAFuy5D-1T|LTcc1K}eEs8IRG};Ci5r5)oh9RTJqZNNH_#yi*msDeqt5Yb%+yd=s6` zqWSsN$pe%L=ck=J}5f6pm}((|HaId^w|gXj3(7!RvOArJd0Ej+ zJ}8RrA}0#v{u5vMvYDzL^}|t(FWrl04rUc%?99(S!YQt&4?su}xk%7m80ur-)C(KQ zG+bjAej|Xkt|ES+QFD#{3iiNDXiL7o#r6?+30hF{Fw;O&Ipdd7=ADRpsD&Q(aL+~!iiYP? z>G>oS_=+5!wh3K?bdA2Jbkn$j$LWNMkTI{)H$=R~Ql9v3&Ou`faaXAE%>@o`R(zra z+Vl>dJM#jA)W7KRoD}npS24FB|bLtjj>?&aO1SpI? zi<|*`UyS{NzK%DNHGj;C!LWI2(sWdu=$-QY0IA@F7mHiYBVktf6)oA*Z)Pk;|MW5I z0SW|w;;-V%A3^B?g>kl(1mq8hWVJ9qZ`|*B--a3%R2EvJ1*x|A|K?WSZb)MA(JURB z-_R|Z0?2#u-m&PPK3Zxz40K=Ek}9X{6#Hrcp`oFm zT_D(Az=Y!WG$m5o93HJMU>qq$bOgL##=T$h>Z2f%A|7=1B%D%8y4q@3FFb7`(1M5b z+t^L23)OBA>9D2X@Qc_l0lyF3Cl>7)Bqj*7rT~aE#VKcDh=KV0UZGLuK6B+mmX;5p z{Do3B7lylz_abUpTs|{+`8SyH8`>24FkadE?AIa9u7g0?R@ApcF7}t)yCB9R%x7u` zL)?+#jIO+V;e~RYfbs?C{g-I)ncpdWr2XELKNVgh6{rW=G{WRl;x8<&oT)!OJRQUR zcRB_H!9z-V-00`F!(M(KTd&ag-L6osBwgzu!NkAs#I%3zfD&J*#VNnBUZRluvYrS~ zg_e&7`Rulg(2Um4h@XARzM;Oq+V~<5ym}Th>|* zqUXM#G%}(-xFQ#DG5lT?KrqTojfg*OAQs?oO|7_M|5dRRP#Lr%?u4omSoraM{-C*p zwA}n3^4RNe7Fw`WczL!PD>sBlw_Y-L?!D8a0fu@C8gTe)wucML9zfihDZXMW*M!7E_&h@eA)P%A$ zoN+Z*$30NS#hUQa^|!3cml@i{<(n^<`Bk2@?SVksWS4H$kJv#PGpISZDt`q8gc0E^ z;l{@oV-El|Dl04g;zeZRQp=VA1;C%T`$k$>kYe<7FbcNgG^13HLj^bDY3JaeUp>Y< zP=w{(=uv5qFyz69%5LLZcYuIlC;Tyhicf!unxQC8v}Mw$SJ%2}-a3D?;Q!yNUksp6 z;SFUxt{fLAJBcNx`l3`e`GlHaeQf9U8NdC=?;lSptb+JmZ22tPZ(SQ8jh@&=cF`k& z>|})G>Q(qQ9^@4pw8D;14#ejqEVq-nI4=>HtQT)O&Hso6HK#o|D!$Sajr9JT1EB5>gr01kmpYNnC@7iV zJCmYJzVU4N849PpDVicnFokTQRBSlIb7h3PILJ?f8Mu^U=osF3r!fQv$C#%-kD3xe zr};tiHWF$5!*_wM{dX1#nA7}Fww#lkdgIa)zT+OD1XY%~fr^d%N5v+SKN-}@vXqJQ zmCi~`8GC7Uig=UbdXmz_iW!m|IWFre@bWn`ZXd7=Q!YjFk3f3JS((7u6;S4^aFzSE zke?}(6v#Wa_7C0@x+KIIdx`UJG}FJ9QsRfdOdy3dz?C`+SEbo85 z(3bBtw>xUbl`{xSUknUpFdGd2m#L(QBWi4o3-q*6AfAGuX-af{0oL%1dm!SG&WoEv zvHJ-7y2|joL+%GbDH5;^CvyK)aX zbDC54QA*_9cW>v7!B;SD`{y-*3c#yY_H+<4e>s{ZAh-V2&B42484n zK)7DJkXyc7`}8!{B%ec(*)_-pr`pGckbk(AVnkD>)bKpgm=m!+hrW5sPTm_4Hc_fC zeSm{a&a)k9YQyD;O6dW70LMbY->C8oX<%@0D2eu?iOxBbe;;AbN=`CIU8@yXXBv36 zVNR1q_3ri?0LrSHmaM5g5b90p^xW&3J&LaNO>Us7QU6_H{@S~hm8?`2$qa9uch)DM zTszAUhf$at5YX{Hs4WT#Q9{zbk!1k^LyH<0hZ_S!ZWe=l{}qAk;+wnM?y>J4SSi~_<* zAn!^VCxje~W0CTM&e%q==g%J^BBD_-G7^i7XcxbB!MHy__|yl%rB-JnP1g$S&F&JB zLADlKC1wpnKAy1TI)Xs>BEl;sIG_`JJ0OE z@)utGz1!#OJnR+ME~p4T9%w01!KCD0&ip?livSD zga|`)O&~CykH;SXVI|Q+uMiyY0T%*g8exX>gAef;E-nQ2yW%@AQ`L8w`YEDdT7Ida z>+eo<4QW8YPGmMexpH(qb+~<<2jH<7m&xySC&ziQ_Po3HmiZ93xxD9(- zf+MnIFEK3kIJRO^E5gU(e?jO0<40H;1 zjEd27-1Y8YV^FO9IQoR(A9@>T+WEs53;lAoo~zjY8Mt_B<%fSZ5+5&M!iyCC+BKC+ z3_=!2$O&8o(2%-A65K*iM{*;+r+R@M9#KTb#bv`>0dTodfLD`+-HwXAR})FNyNA46 zC`jsS>vt4GEFdE%TkzMv4++lezhNy<4SjiRYjj~=A&?kq4dq{Y*a z2!UPwKkdNt)PY^4Kr3Xbz7{T`cWV)yz&&Vf-04+kj4C#nPDfgA?0vVB%^2!&oB3UY ziSNCNIf`G~3BUC#W2<~>V9aT2xMsD$9kYEZ%;@R%D0}?}ou{WKHuvYxh^{&N-Mtgf zL2g?)x2RW5K_R}A@kxdlec5o=1#Q_5;k_*UZhpOk2x1XcR}x--$fMoN-j$rSWx-)O z`)<2Fg}~^Q%E(dw3x{a5Rh6}=Cqk6X%$vV2i%>6(UHbC!)T?qXqZ_mV^J6NDF(4Oc zW)BmzdQ znBvAJ7UM%td)sdoBmY7qk-|EtCan=>!o!5oYD$4_8#YH`4Z9gxOM;!SuV0lY`xFt)3%! zs37lBQKC6JlLNF;4}q?i-&DDDeP5Pk6xt%zOp>LFf_=g=i!cK^JT}TOa;w+ z7kQU95&vshTP*2X@ctd_G?aD(HIC~FvvuI#qjNGD|6s)g&XV*u+-i2`x zo5qa{B3egFm$tssqvMe2pu|7YZYwt*9x?)M$>%m%uDLv+v*^_ZQKtd3D3ficSS!-W@^pqQJWhjiwo<%9W1t?dBPQBuQI$EaJB7U*DgY~ zfWZGVWWyTxQppy0gs#KZ8aC({vSO(lg%Bz%37t7pf&qhDl5)@|`bM3~!h*w>H1#XHA1 z8_%jd)Kaf&Q^K`ofkV@E?ee33C4~C9z-b&*vZ-glF>Op#x|#k2<4V%CJ>_6sCb3{$ z8j}ehTtK(e#Ay^EpmCfYsk*tYaPOt{T6{Zn)=otI=kM(RQ+V*;k`k}o?WPGYz!n*f|oI8ou$KpXC3|I=9B^!Q0oSb1(}ga09e4-D7qaxK+Ytw z6Y<~CVQoMHVjkQ^yJ=R6E*QQ6Ut5tt*9H%0R1oYF3P~5&c@n_3G56rpYF9pqzE3MEk)*W+g@=05bJdo2a+J#%Ot4Qx7U;& zxE6zV^LBhniOqzV$kB;CQ&-R)sXZ&z2xJ6(`KD>jC!TG=w= z$?1XI?s7kJ7?FV9i=|GCQ)|@$!|7|YrkU}CXMK~6w#_(xj?YPe&`F@0p-=jKI}LhZ z@IRdbccS08JQc|@)cNorZRyw-?hYnIfkhrz_pUME&;5$~aOC)Ram}nz&OB|K0k?PR zi#WKE?$Bku8hbZr2C`W@(Z~zZ2yB%%Qa4UtC$a}@ZPBsa!w^Ib>i?F_TBw7LeDYM! z8WhkkXEM4i#GuJV^!qorGN4OIYjFq(L4_#Qzp$_{zu<$4ywC(d=~oIdz*(5Qxkz-H zv0Yu#wV5N6lRX0FT;w5=>+o#nV0C0b0G{oHN+8s`y|)8S6jWB}z^YksEEgWM*sObY zpu-lCb{Cp_SwWPyyKMlubrI#eSniP*O0)luGed`g#PvVVY-uPdy4Y2^WUNSkUdrAA z_qQ|sKl!>aq6Bbd#0cLF_knGymH*-Uz!#sp@)Tt;iNiFI%aTmOy2vfH=(&(qC8o?@ zlmP!zxPc-K9{3*nMdX_-*-`bmxw4T69RQ#T72vC|$AXuNniDeNzv|5iP>W{f{LvFM zn-8_O!70Hf4wqH+2OWH6%(c|xjhc>-?)3~D$&6B8;*g0Iq%zy5sh+jQ(?Q^DO2K19 z|Ih4{_jS~}cflcq(MsRkm6VW9PEJTn*Euv{vvQ)}!{O-g?~gFvgm>FjzzO6$Q+x+&I*FXfy9WbScKN_0HNZ=N^(szu#yTLSoT$FJkmPF^SWpnaCio>3 z!olwH6ud|(F4h|2H)wM*hB;&_wPzB;xH`okAn~$$c+?lw^nJyzvKk4SohHC69J$tO zT}^iWl1B_b(k(Dbi$#YWwL*Y`0J3BRnWRXQI{qUq6dRl3z%b>M0fV+eXh)SFP(GeL z+=^z<$a?&+T~$me#?NHIXpFdldwaj(9mq+*Q}pnh1bO#gD4>tdr|w9#R|oIZkuPX$ zJ4stRwQFU~)?v4q7B%HxE>Z_x=cZkd<4aEVW+j;`|&p?|^zY9L3lety`qKP=@op;(5jN;7c)08Vey= zC?zMp4>ndjoTmvvAP9|c`p&Z&b9D$aQw|TTxpH*1 zMyH!+0&dvc?2rtl*>}!Gb2m*CtE1fLoBFRUVw?Gbf4TKVB2~4(31m3GlRM@?eR~Oq zBB>VsPU0iUZiET3nPUf`nL+3xH7lzEDJvTru0XAjPl`Z_^&kch2I{XA@Qc;POW|>V zDTptrODkkaD^#bxm%7>&6f6gDt4q>~<=8tU5}A&m!@uZNq&uNF^lC%AkyZiOgbtfp z2MY>$nYwzgCEo{Zv8G!Oo7T&+6(bAE2Gl>ehb)BRsHPWT*yC z+GBmR_@g6=d-bY$9p92iG^(t#3;$S!%a5!N$TVqi7hrBFiLo@lCPD(HC(&g6DH+h*D{O6DR3Ibv`TSS3pUFfOLXFeL7e;xRI&Ywa1SA>I?g_dF7EF+$ zf~Z~m-+;%I$y1>Tfm+A4HqN@L)w@~KZ0;KG&z2*GOhX>+@%R>NpN&!xHs1 z4%pb(Ge3?N+j{{C*k9Z=j~($SB9I|CCza?8!MSPpJiQzR7Z<^+^mIpN&DH;pudj@% zvg_Iv1SF(OT2dsWq>(N~N-60O5RjJcZV+ja5JZsf?iQpww{&-J*l^bNQJ?pm?|knV z4*oC($i3IS=A75OqHTI_e4%?Nlh5rmA8O>Ftxc{eGW=p{0*~hsN5s+kd_{eGt`7Tw zLaK__tlV1(H!MD6E|yioN%>5UBB&omgC(y{6=Bnj~`DhJM=WOg+iLUCa_sQ z@^U+uH+*FJ{@5RpUp-B*T)NDFc+I= zYmVRw!f?tb&ug#7Uky}(}UIQD`g^w;f)xx5H|>gMI;b?i~f zi*me$VWEgy<9>mjO2&Z)oLbm7GNn(pRnx zEs4ss_0I+inoy#N7B4oqO!Ii}1amiVJ8*^)Gn>3!9lFW_nT7LrsJ^9XWy!_0#)Nk3 zZKv^)gO}PZq5f5Hrc~*#vOZDAMqt`Hd^ek^5dn5{y^wm4rNqVgqd2uj2!9=n1cgzy z4H>vp5?&H_{I5hDEdF=_{5vBAfnnaak@s@vXqB~fckYti^NJBTuMIQY*e)?p%hl9) zbcRpJx06_mUwNl_LNQ^NXOeTZXH(m~uTbE#|9=~az)oTkTM>5b=0M_;m8p~=XD?;b zcZ<7!a)M-QzGZ&o($eunb1gynQEtAxpxa$5(=SWg2`}HfzOD;#S*R^gDm`NSPmtDG z{){H60q_&%INx7<+#0qkOE!^(ezl~tjyuXzYl9SJ8}BaXa6 zbUnDe@}mmUvT1F<%UPsbK>Cr}!dKkDAh-Iwi^{8Y%wl5nAm|B<0D8P@%t@?#pJrU- zXx8SFK%DpBAdb-Cep}89dP=ug)KwAdb7WrYdC9R{Rd&jFsKM|H`A(+jdP{ z#LQclujb)}P%%o_suy%WnbPkA4LTVb8(^JoA1bT;JU07H!XlRXkztr$>) zE7%Hmx$hN6oS7V+4f{f1?LFzw15|E3s=s?o@9aM(fz$kId+O!SwY6+*uG$PCiaqj- z3>xKn=U6ct6x?oug-q{3uLQO}|D#vB zkMIv-vTR8-QAyPvF&wCv@&VDcB)*wbIhZoN4{}r=aZWGC)Is-!{Jd>giE%oIE3}Hluq2i6Z$X%U3P8*{qII-BA=*I;g9nG~chJ|5ZVlX) z=i;)WIT@DHR%{F=SYe4QMqT01i0+Z&zr5otLuVv2__U?|O^hDWvG`=;X`O?>-L5Dq znOYiYHoZpjcoxlc<^fB?qxb@6+gZ8I2>F31l7M~+Dz5cs5lkX>1fWPndVAk;uXxP? z)R+ePw47WfeQHS4d81U)5VtkU(LxViro`Z2;p-K&&D6V;k8j<+JmWGO@Fhp=Rd@G$ zRR&Sz!VpOd>WO3K_cz%eG%Z%IF{9>TKnWvq|GJyBEqWy%?16KMi>pjvpn7S+j(67T*iBkaC;9am&!<6nvN;jR+k~L_{T|N^wZ-+EUiJ+%PwIkot z6TV)h<+Bu_+GCxVz3Lg}OF{KGv7ymhSB1dqIU1sRp05ukS0ug$v}@wrc3^;q%yZTY z%NHGGs=Unjz2iG#yw#{G<3MyarY7YenW^&3zY9C>GaS%n6cQSOi$>BXo_eE#lcT>) z7?hfsuJ18wRdH_?yPRM?US2o2#ERsV0&#}6aY_}hEMoKKaD<;d2K4@AKw3X&JSJnm zcZcvG6Sy5^cu@~Wg@;Qnf#e?g<;fYR?e?-ZncS%ZBfVxLis@dalTXpnK9v*#G@$v1 zL4`v;OQ4X9hW$u>32FVf*C-Q$M_+7rp?>An?60ICri*k;v*{idUof_Q#3n4F8InIg z^qSzn-}`~R|Ub)kn!EOsh*B}jyuy4L7G>0Ud|`JR(im>c(ad{QcJ_nPqI3X z!GQT#_q9Ruc=v+zE|LRS)f(L?t09OBW`D#QfNNb1K=bq|63jt zg-^~;Ci)n)`N;8gT_}-nqtkF4Ai|Y_p@h9g?ZMf*yXQBsYoN0=n6H=m-A}%K;Z@bl zXMF8J4|6iIFkNYocV5kR(ArvCOX1x=D*Ti!Rgq5-9`E2=kW?1=r^#=YKnf)cE9WS6 zx#^LnO;8=NuEH{Cqp4TJ_)m^Nu{2oFaSNp8zRVm5Lp=`N z-;M0v-&cS)-%gOyGR}hms48Fa#e*-|+Q0`z%wQpBco}!pd?44d3#V`o z_;yTKHEHB(#4=}=bU0j>ZTx)3Vf(bYX;)u0E1ehCax_|w`)@>Upk%Z%B1O~nf^I~V z)lH3JR2L8JPiOhB_S;-fq=PbLqNsB<2o{;ur85ID2h&?_n*EaxA3fYw-wEY}o4N-#;g%#U-q|6smY0g@ zjJW%J+F{*)SuSC)&Jjm%YO2$MEYggK1cvb^Nt0vH~wH`lpQr%vt79F*D*O9Z-ZzYX!inX7dG&hD7e@s77?41t@X26+bC^w7(_g5uDDp5%_-bS1RIK)Y@T^|)&9skVvW5c6~GOX|X_8tI~T_fp9qQ{`nJ zM|0y!yirzuZsY)8Uz}yVLHFDL>Tv$9<`LGU1~vau6Ye*mue0ei1w`D?EkOLYWNYJn zzT}Zqe~b@lrM?bkBKOYOzoioK(0{ce14mibt>2|5i|&fkn&AU}@B)pwE&+k?$D4{ zg&8*WYtMQM;&2jfakZ7avfxsY%M^ZDMPB#OD(%-ZGFnFErKj)Tt1p! z=f?^A@m}mUM|#sq-Bm}-t&bwd!BE*;WL8LXTpmoRAu z!^OddYrh;_rfCzZf06J(qF?Rr*NiaHqF}APVA)QvC`(Pn?;1e%xuQ!SX!N*4=BV4Q zDELT!AS5sJ$dWAB*;%}=y7kSgs>H*bsm1-15=xy>$0(O7z6R?DJdZy4g_D`N>^V?1L1iLR(#_q6t1k7Pw^8z<`+5`D zg6PADmTN@Z1o+}WCW-5@n!MgUE_5A%wX5cbGgy?+yQjNz+0sJ9(F3d$ilG ziF72K6!wB%ZgUJ#}c`;CuaW~mO zLU)&dgX-m#;exwLj?#R(SAp?ijI(a(2?{RAp|`6lcR4kojwA|sW)KlBs@L?CU7_tt zUd8YlJD$lGYlX4Vc1)KOy&7o%pJW2?}Q^Ck~}bH(EoIW3#^AK7GvS90dw>%rY z@O}0es>PT5DU$0cuZ>#C6L$teac8#37onDEK}P#FJBv82u8QR1LQS5#k0-OPonHI) z_-bNOti{o|+9SGBUsG~Jk+o~x(LjEyBt*DjtS%1Fm~Z4u%uh(`xsAR~>$j|>(KOl_ z`*o(&I<{RLg*cZxFhBpKPs;k7RD{>=ai!%N@3&~dTozI~1+G-Z9OUEUB_ z-uYKYiw2LbP|qh*x$~(L8+#5mU*CJ2-jqL{l2CMxQf9Iy>8W>qA*52M{Z!pladErG z-r051zS^1NsRvDS*g(`c+8!KEy}o^CY4+mip2#V((D>(urwIVcZO!3e%Q3tGBZClZ z6lWy_A!UpV(oWjha2(1sXnoS%miYKUHRe>4sXJ=X<-7XUuQIA&cC_6mPR@!K*Gl>@ zooa@QbjDh{<67<;!$^Ua5P6h3i@}S2#5KTP7pJ1azk9Gb=`WJYm%!=vo5t^RB`3;E zeB3Z)tygy{ldpHf43Cw7o?!aD`>pv#)~xbG-I=a<0g+`^_YBhWn)3&SU7r`Zb z{mTF!4z7>%?`_QPMCPYOyts^E?LT-8!#ihAXMM&fw)um?Ef%uK#@yt<=+m&6cn&0|vNkFrVF0A*ZXf*y6=P+jW0|$pL(tbW8!Epp5WmNqydqed^-<@wc5ODg|mJ^ zdyt!j7JiR|fRH)<7=*qNB5b_R$Cmb(NkJ}haO5*6Yk4f`c8Ye?e_b7 zdIViViq^h?O%E&}Fh@DxdB3brv7dP3|)Yxz*U;X_j8*PQ}^lROTc5g&XINf2p zoOa(u6}Cq|d0%%&(>iP7;$w(eu0=)}oZGhnc?`lTm6rha5i!dPtsgFHma}?IsbUrf znd@x@l&ZafcS(a0V3(w$Tc%$J+{u{L3)BK8l09eA^^AE^$Z#8Q2BPk9w7&H!WP1Ko zUaD8ka&-4*q@Luff=k&J=IrhU(OqVPG0hOpIA-=+s76Ff{GSn8~;uHnV7)edJ+p94(k7os=fQ6F;Dc!bnF zN}`WY*TdeuE;?LANm~1dqx&BlWm7rmRS^PZ50_alFUu8%>(^JOGep4Gi7^9eii4}%-aJ1>0#+Bc|Ew;y zE>|hYWDYl!B57a$%L3wpGgN&(V26go_Gv=<1=wpfgf*rb8vJ!0BrNS;L6Mr7UI4SYQoTN7Adls@EoR=3PR z3P**RJv3ME5J&%P{`zSw>dKx_jX^KZ?{`c8IUMymsGm(C{-~y>2l<40yi56Mi9gC4 zK%Y&QTSD_X4#1oHwlR$#`^ImRS8pq@OOE*;h*6a4+|b_LJe}PQs)Vh*w0)R(xHCMU3!PxeK|&v?JHU33DHQ zgoebezGU8L_YJ&f1Z-H;g@le5PY^vE9V=_4?cM0D4DqOXDkFi{dFGzC7>@5YaOE`U z)OsoZF~^~g>P6D&&Ip+zHie-fEyjEzCn2Qh5+R_i?O>_4qI7a|3KzHGLHm*6q1)+p z$NTJ{Z(H}c%>D6p8p1lbbs8Ko3uemIx2X1pNsOeU9vAoJd0!u{FyFg5M7k<7#jbVS z__D2K1nl0lZZ3m%Vf(MsbngkgE}lxBYWM8-JbU6^2K<{ndA4jOcVjXWH7i{glnfdJ0uQY}MrnK1T|CdOEgQ0RMa)HB6U+D+8w zS|7aq(0E6MG6%Pc+5eIW^aJ3mv$Z=-9$k@h8jV+4!9Ot;aV9b~%in~8$mAsL1iXn& zV8qfJ-!z&{j(!Qkp$!MqAK((Mc&2GWFAOOc0h>5T!sTj6PUW1gd0SnLxL z{6=IaPvU&3nI7r2)60 zX3VxcdDA_@=eGL^+P#CE<`ufw@dL5|skOYL4dPfS;us-Iy6*Gai5dGB#9cUu|N$XQoF z81YcH6Wd)k=)<|{F2zsHcc}VR1CBG4ejF5hd--$E>rCALyzU=aAg`2e_;#@@>9GXi`<7W~=(djDRKh$!R$KvoJY+sv331n558XqhkWgeyk z>jtk8dz7fa$OthCv`c7QPo2f_X8^B)p|WL>?lD_Ct{!oFN=Q7G`hCY;CLG9-KNvRaIWzH-9l`wjaB_Oju!jc{30cZ)pWjIy%*!$*dOCS4?du-YIGC3+ZMEN&2AH6wmm zhxwaM0fI&T$BN*;(JA(XA&oH+dyb8{>$ZiO+TC%!I&lIP6Iga1#~HQpB6`#-C!?}L znZ0&p4i&w$ayzdtGu~?mjHWkQc60{8ib9z~;Yph6W+AGW949lWC1@~0V+(W34GwU7K0b#J=C zxXm^I3EBxNy02X$;m}`v`8;E6Yi5n!_mK!_Ld=GsjZW^m-kzn2bZzzmM1=}wSqKG)-W zZ_$qKV0)mro$a>WKAY&OD>MDR>X=dxvi{^@DshZn^*M(7<1>5*iZu6QA62#19Et2l zJ(=Axa2H1R5}U?ph3gn%1lVYm$7m6GMphc-BJ}+H>*jm^|7+L zLQj|?8N2Q9mig-oRS1hZRZL;tl^1*oV(#xoodMrg|M++iUlPJ_Xy{4BZgWBEj1<)ei%fZ5zVL78E**kDM+d~IWi37EEr6- zN?%z0Y~MCE*?Mgt?afvxL{3|_$^7i9;|hW{~lXVUtws3 zF>Q2)*fMsZf{0@C5&6`tPPQkVRW9|#?dZ*2 z*T? zZyzA@6a)OB!EANKv3D~-MY=Q~QerV7Vy?2AtD2V{Pcyzd*q8!+V`>lk?ny&S^A1rT zcj~IQsIe2Qqho{BvN%>wCU7VzBLPNGZl4D-0- zY^kf!yGx+FYWm&s`r25|iuh|bq~-X?M^R4dnZ{L?M~n>hLh~PzI_&Jh%^9}V@fyCj zH0o-~?{^z+M|*P;yu!8%Hl=UIGqPMI;B1ZN*DD^mUdtdFd>*92dbqj@Fvz>BCbhLe z=SaV#ZHwRpVN;2{dq=`@!zA;Z?XI-Q4bAmsSK6j$m`Bw+I2Q)aiK&3rS*wGIZ}`JE znclLvv%}dakq*dAW!_OY8;z^cfjbC+8mf?caQ>znI2z1nT6-k>y$m+OkDfyzE~1il z)a)M#VL@A$GL>HhMyoVqJG%*i{q*|e0lKHV@-$HY-tW#1eG}D*b~*}rPNzo)9Tq!v z84BSWP6R2qf8|cnh+*S;%|(Jm>(VUidba|6VK_wFVV&FV-6`e*&%;`P@GSczP zlgq$#n=Y3tz~%)4n?D#PolOETjH+Wfd>QKL>^UtzQlYIN*X+Z!McsZ(!_?4ZeLsVC zGu`!_7J0+R(1Pr!k37T7A#9q6eM^fc&$i`s=NxZ66}d7SaaL?tlzmYyb-1ri(}7h# zAc!d6!+-SrH>pOG_7|~G+rLop7Y~%{?_PbM0SpWlkp?H@W#;V{F3(2K*K_J33$Jt2 zlFIOd2EJ|*zmivfK{<20urFX@+0T$|)Ga7D8hkc_GOm<4pUnNk*ubPyo`rnwK|;aN zc>dl@*a8(!VwFPdIK17nex4LsT+HqcTOm&5u;B58S!Paok_p)~(L73h&4XUKo$Xv= zcsVs1Wv%8Z`hsgLzG~9xcu)#y(#oV}+o+A2UQ z5Z)WDNS+?cwil6%AdL_B2++Utrt>pBukk0s4Mqu8vLGULFdHQk5)&_CaeZ~f7ET^R zTkN%$Ql$)bFEIKyOsv=aB1va8b}{{WoAvcNzb?{O4FUpAuFvgx8D5AMeG|DI^Ia*} z$d_&cSq7h%dRd{rL(>8gN;syKuu7z`*by3#?-Zxsdp{j&paTX7~kOp|s zMwQ#Cj@Uxv(fJl+kdz8P+_vFvrhtGo-G3ZRHn<?c>Jlzp|!)w6I zS!R`7z^lH#x8}Du|K<5mWKq z6A=-S-J(v_y1vaKEod)x04X@1{p?X&Y~q zQqU8z0eY-D6B4{hQ(Ajavb{x^2-_t;XL2$xdUb+HLla7xJeiWlWg1yq1r`y-brH;% z0kU-dRBD*D}aHj$g0x^I^IFBOgU8et$>5x4$ylNl<1cbLoz@g;Qz7c$3EissQcaOsw$)iOcD1 zdIC5Uq-IA?O$--7Z>iWc_tSFu^lpG)zXYIp9%+<5_yb4#z@;G&wLkb@UQ`}@s?kdMLdJx_$h-QDO{zT}KWrY;&kl7BAKZ%`QQy3%H_TC{c^N}+ zQPJQye<@OM$+9}@Zu|Pj!aE!rbIFQ;v6NU}( zA_k;HwUI+An5Sg<5gvW$5@;6h3khj|%t5p`hz6!6SRrHn1gaRBfY-R#_q~X0J8Z1; z*-CSZM9B@rYpi>XszF31LYi&nOLlhVO*%Es&_ptiSvX)W#j0McUagcC9qfMkl>8)> z1WJ&)J<|KyaFft%wcZ29&9aQ?ztpjh|(C4#UrpScSOCtdoy!fm=& zHxUCH&AC|Idv@jVeQEAYxH&z3a%7eUeoEPH+eM8XJ==; zITMD`?nH1}3pkef3UC@i=i-H{gL$UT3=hK|RY5U`9AXwU@T4SIOkDerJiHvc)cw4# zCefS~(W5MP;IpK&!WUP$CHI2~A{LliSeALbV3Mf5wDE1+hL}22r@v67D^fV^U{Da$ zK$PqV!h7=3h2`5nmG_iL>CV^xR(pFUvDt`l(ICF74kQ<5bJ}4KedIRGo-I&FBFzz& zCu>fr+Z`QCeM-G)kY6fb&9LR_Ry3V>{1q=(I&S#?cfoxmLIyfoObw@#O?ePGERxom zftjuHIObAS#*b+=QwKY-f&z`$dNP9js|LF;m65LO^_owwvvG@XMbVHBQ$`<*erI=r zvdyEZpe>Dgb;fA(T{4Q!WhT^Xdbvi=W8reEs3BBFQ}}J=0AK`s1~Y{DoF69H&z+uYP-)4d2&+<-<%a|@sQd9Z*7b^6{|W{}-W z8S2*nQXFWy?%axjw4ZEEj3vtuc?v6fKz$E1k7&{%nPnqj7O- zsS9=M0?n-Zlq8pvM;j9PJwA#dUD>YFtup4n3BRuC)G_ zVeKy|Qz=f%^T)4Vxl^*}mJ&2ED%Rixf#Mm_adb9O6vk7aWuYd-HVGsujBn1;ftP_B zE!(;l-k?liT$jX{j26Ya*`BO$A?vvJ7EBK_oG{f-|JZ?c!p|8o){)eWOp06I59c5C z%N1~Cg`2J;$DSqF;<+)s`k)AnW+8t9DDaFN({Q_pU;id00GH%RY<=(Qwio^azA0n; zjZ>$x%C-^_N8dw3qBJ76PhVCRi{{5G5P*qx||vu!mbi{%BuoBKW7wcw3YyGvp?g{1rKm#{#qjH7VG)J)s-A*5LB_K#F!=a+uPV`)Xf&vX@V4X6Yi zwF%#cb-fR#I-?O_T!db^2BRLaDJG)Y-VWw#{|Jt6z?Uztxi{+>@p%dB&#&GHP1nYC ze3__9q{PaqR0G*S{z+oT>iF@-ZwG6HH z=@<)z-)2u(kQ?f`sm%TRf;~_CR_tP#d0&)RK|pJhtP(8qd`R}+q6v^0{Mh@#43kqj zWg={J#SVMibUM9{6KiadpgkO7Z$^IZ_RXVol6C`DuQ1GUppgVnwEwHP`j8>4%uBnu z#35VhAgL5(WM1&%{l_A(z%|s0<;kg=;0$uqYyL3hwJ_et5|cGyu7dI;5MmAXjuMD# z>Te}Cn;zW7h@0)GrkT{LDAuG@W5*c_jqCL2$dupK_|taM{p5qs!jS{BHLU)+y7gdJ zQKLOKpUeyLEy=0Psi30`TQ-|Ru*df0EJq^luDClZzfhG95&S8Z^C0#o4PNQa}wu?tZ zI|lK)IY5FYy$yX5QfWy!7Pm6tkCW=fxo4lRU7;EBlZ}!!!TuZy%>b!&@HKBNlNtZ8 zt}gPiVR?hwnL+<0l|E>?Zm3|pPf<66L_YfLZj|AX5-C*jDs0rPeKC{yKpT}yrz(6N9k7REU z)i4CB53C?3w#ReuN9u88_|!VFl7(NWPcTOzsWPOCWQ>&)i~~uBhX;CwVTehN=2yZi z55Lhseo^0l^dt zDBS;ptx*0H+;KG?pCMLl)>Un611g9FKsJP3%2mpWzK(HDAbRu-yIEuuGRw1kS^6$u z=tr~&^^R*`M+MKG21$XNN}0w$C^@2;%)rmip|WlTxgfpIIQ}Z3VoGmqzZkC;V;=C1 zB1V(_zEFWI8~}ZDT)4r&f1+SFGUe!y2cbbDJ{VXR41rvwKF|PmSj8z14(y|$0xnUo zr1cgKeAnm-TT&ahe0~GP;kk}M5`iEla^>?d#bJ%{+G;|3YLMKyjKfEu_Fpo8j}lRV z=1{G|Hz@8OA&A%JQvXbR-m5nNT;ty^SZ3l#unGP%l1IR}&ZB zw|+^qCq?Z}G_)Q@7$)9YGc$w9W`kWwnSueBetxh^oGQV1VXP|$u1jAm-HR84$ zzK0FVuF>jr=)4akQ+^p+(7`KYj|tW}`X_P*5kNM^>Q0J1TSeM|uG;FA8^M0BsLyTW z?qO$=S&D0hNmBi#W+#j@O$1fDCOR<0B7W)3hmDxB&_MsiF-IMxZ;2~p$A@>23w?ia zFPpVc^JioIxwY6fzUEQ+G#WS1I2XHC7b<|hcD%lXBn{;)6BKKF2Ix9fju?f#J+8t!x& z0j=^3PFU9!`}~HAG)HX;f9FD^G7>xN`YfN|PbtpZ6^=Dv3xs zAWkqz&$*tHGMLSLaP`?HYklp$9=U8amflLCDRr^+s|g3aX-+OH!-aKW6nh)>ufLED z8(u_rtF8(=Xm1>QZ~~Aq&qfd zD)*GEU30X*Wl-b|MWX!{M-SEhT4}-2^Q$X&j-%D};^%4J`}PS|Gob}C*bCx4+yrP? zEihx|bgelQ#{4fXf2jZQsM`V`LL=%57td?n(Z82MP>H}0TOwV^A;-_pGW(bk23AuR zbJn~VY*d8LMD!kkjo1GG@Bi}qS&2+5=@v(&>1nDlK{$hB3FFy%1Bt)M!m)^rewRzK z7W8sm;kpiN(x2mFwxDDJY(*;Lr2o3jZcLt5? zOO|h0G3wQ$BVT5EV%_3ZyAddPWy$5k}fzOr501LLXwP#!t(sF1{7_d zCB*yhtqOPn=0D(~VO+sr2bV=6%x(^6-w}RQ4f3b)a2F15H@FFb?tEnAS7YkPLmh;X z?12_w2Z2jU>T5Q_@R0?iBW)#et>J4|+bF9)Xw*91o+88~8=r`)VWtk?9!ENc4y^^~ z>ynSfmC3&3+bc|2PoJJSY04^Rw)?(&1)H8IZqvp>x z{CyLIhIF@WcT$E&vr+BLysDG7^x}gRPe~qs!~GWe({jff2Vy;+98Q=#@Ke(8qq%N| z?0trlg1|%iIsB7@^~%1K9ZvqvKzZW1cq2=%w!=-As;k^y?@nTPWM}iy?NutK4 z46twb(y?`QA^C1&T*ng91c_On$*$5Y6vCCjT?-&R?)}FGLUbM;+u;;gxGmNz{>fp> zGtl|s;)0r;y;)L}?Rqbj(>R_cRZ)Dce`hdV>sa1?c3A54koq490Jf-?i{UzQY*6&AOm@;QK_Z+lPGXGWO7A6Ia zySsg*pVo(Z9PwaG#^!Z;6-)28qbq|A;j8O{o%><z<;Hk+@Vy-6aU0z+Hcqyex_Vz!c9l_Nm{y-6(0eX`s5PsfihHWd;8tq{Hv}SxAI{a`49B8D0e~)~+4kEyM zid36)>IBzjlb(v`oj&_7zCfULjUns5zi?)sbN>95pV?lDyKG;A-1gB4~Zv4Buv&6Pd9b9HU8tz9+y%|mlDUm>|mIopy~ zZVW473j^{G9oC6+V`;T7`QGWQ?Dox>9%vd3H;6HC$?%QzvmpaAdjn!S(`VmK2nNaY z4S%hUgv5c%LX>`$n-VL!ZR{RuB5*o-r2D+pAB`=?tRx-WE2Q}ryR#RQReQIjqbLLI zRtzyWB`?9%W4PE}5)AxPl-6>8?Bu`IR-*YCCZQ+lWu!wCPs~-HsCZDH=ja3bj|X56 zJeYhNiC~UvAI2TQwqDTM>mQ4+|FMCP+nlU@iCReUx|p0gJ<3vl7I9nGXsmA9pW!k}B@zjPWQBS-< zIVcScY#jD|@>~&5L8sn_j(+;8Ab+Nm6dC4@lO>rI#Yz+oxb1^EnqRbN`?nY}61w|) z+NWO?-REDZ*8O%A0|@Seo^&Ch*2`awFgTlc-*1m&&5N|XI!Ak+B!GEC z$(OTAwKt7yBpsnN0=-#314x-L$=6>V9!GGmuPaK8WFMjcK8V*1zUR5YZ2<@Kpdd=~ z(6zL*TOunPyq$>9Uvfwn(t&W2qD)aAM3i3bvei3R$0OG3kfsNl<9#(H{msqDN%Ak7 zL_0jAa-IOSbyW&xThkiGmh=~A-5}q8u|>Q=pt(=f9Kn)HF;TKisRdpDaH`RiNxa+s41Q9VH-dG*~OcO>p