From 225171aa441bc90874f634d6fff6f8f771f42016 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Tue, 14 May 2024 01:28:29 -0700 Subject: [PATCH 01/30] Fix error when plotting oblique trees in colab. PiperOrigin-RevId: 633479472 --- yggdrasil_decision_forests/port/python/CHANGELOG.md | 6 ++++++ .../port/python/ydf/learner/learner_test.py | 13 +++++++++++++ .../port/python/ydf/model/tree/condition.py | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/yggdrasil_decision_forests/port/python/CHANGELOG.md b/yggdrasil_decision_forests/port/python/CHANGELOG.md index 3bb912f3..98ad761a 100644 --- a/yggdrasil_decision_forests/port/python/CHANGELOG.md +++ b/yggdrasil_decision_forests/port/python/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## HEAD + +### Fix + +- Fix error when plotting oblique trees (`model.plot_tree`) in colab. + ## 0.4.3- 2024-05-07 ### Feature diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py b/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py index fd25a94e..5ab0e156 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py @@ -653,6 +653,8 @@ def test_adult_sparse_oblique(self): sparse_oblique_weights="CONTINUOUS", ) model = learner.train(self.adult.train) + assert isinstance(model, decision_forest_model.DecisionForestModel) + model.plot_tree().html() logging.info("Trained model: %s", model) def test_adult_mhld_oblique(self): @@ -799,6 +801,17 @@ def test_ranking_path(self): self.assertGreaterEqual(evaluation.ndcg, 0.70) self.assertLessEqual(evaluation.ndcg, 0.74) + def test_adult_sparse_oblique(self): + learner = specialized_learners.GradientBoostedTreesLearner( + label="income", + num_trees=5, + split_axis="SPARSE_OBLIQUE", + ) + model = learner.train(self.adult.train) + assert isinstance(model, decision_forest_model.DecisionForestModel) + model.plot_tree().html() + logging.info("Trained model: %s", model) + def test_adult_num_threads(self): learner = specialized_learners.GradientBoostedTreesLearner( label="income", num_threads=12, num_trees=50 diff --git a/yggdrasil_decision_forests/port/python/ydf/model/tree/condition.py b/yggdrasil_decision_forests/port/python/ydf/model/tree/condition.py index 709530be..7d3d1e3b 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/tree/condition.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/tree/condition.py @@ -377,7 +377,7 @@ def _to_json_numerical_sparse_oblique( return { "type": "NUMERICAL_SPARSE_OBLIQUE", "attributes": [dataspec.columns[f].name for f in condition.attributes], - "weights": condition.weights, + "weights": list(condition.weights), "threshold": condition.threshold, } From 005177a2798893628b89069151dd5d40737ca761 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Tue, 14 May 2024 02:35:58 -0700 Subject: [PATCH 02/30] Add `max_depth` argument to `model.print_tree`. PiperOrigin-RevId: 633499732 --- yggdrasil_decision_forests/port/python/CHANGELOG.md | 4 ++++ .../model/decision_forest_model/decision_forest_model.py | 9 +++++++-- .../decision_forest_model/decision_forest_model_test.py | 8 ++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/CHANGELOG.md b/yggdrasil_decision_forests/port/python/CHANGELOG.md index 98ad761a..c81bd9b8 100644 --- a/yggdrasil_decision_forests/port/python/CHANGELOG.md +++ b/yggdrasil_decision_forests/port/python/CHANGELOG.md @@ -2,6 +2,10 @@ ## HEAD +### Feature + +- Add `max_depth` argument to `model.print_tree`. + ### Fix - Fix error when plotting oblique trees (`model.plot_tree`) in colab. diff --git a/yggdrasil_decision_forests/port/python/ydf/model/decision_forest_model/decision_forest_model.py b/yggdrasil_decision_forests/port/python/ydf/model/decision_forest_model/decision_forest_model.py index 0d08e37d..294c2c24 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/decision_forest_model/decision_forest_model.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/decision_forest_model/decision_forest_model.py @@ -57,7 +57,9 @@ def iter_trees(self) -> Iterator[tree_lib.Tree]: return (self.get_tree(tree_idx) for tree_idx in range(self.num_trees())) - def print_tree(self, tree_idx: int = 0, file=sys.stdout) -> None: + def print_tree( + self, tree_idx: int = 0, max_depth: Optional[int] = 6, file=sys.stdout + ) -> None: """Prints a tree in the terminal. Usage example: @@ -78,11 +80,14 @@ def print_tree(self, tree_idx: int = 0, file=sys.stdout) -> None: Args: tree_idx: Index of the tree. Should be in [0, self.num_trees()). + max_depth: Maximum tree depth of the plot. Set to None for full depth. file: Where to print the tree. By default, prints on the terminal standard output. """ - file.write(self.get_tree(tree_idx).pretty(self.data_spec())) + file.write( + self.get_tree(tree_idx).pretty(self.data_spec(), max_depth=max_depth) + ) def plot_tree( self, diff --git a/yggdrasil_decision_forests/port/python/ydf/model/decision_forest_model/decision_forest_model_test.py b/yggdrasil_decision_forests/port/python/ydf/model/decision_forest_model/decision_forest_model_test.py index 6d747570..ed51ef22 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/decision_forest_model/decision_forest_model_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/decision_forest_model/decision_forest_model_test.py @@ -69,6 +69,14 @@ def test_predict_leaves(self): ) self.assertTrue(np.all(leaves >= 0)) + def test_print_api(self): + self.adult_binary_class_gbdt.print_tree() + self.adult_binary_class_gbdt.print_tree(tree_idx=0, max_depth=None) + + def test_plot_api(self): + self.adult_binary_class_gbdt.plot_tree().html() + self.adult_binary_class_gbdt.plot_tree(tree_idx=0, max_depth=None).html() + @parameterized.parameters(x for x in generic_model.NodeFormat) def test_node_format(self, node_format: generic_model.NodeFormat): """Test that the node format is saved correctly.""" From 7981aacb22a113e6d82a42a217b8c9e535a7d888 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Tue, 21 May 2024 08:41:37 -0700 Subject: [PATCH 03/30] Documentation of the Distribute tool. PiperOrigin-RevId: 635819063 --- .../utils/distribute/README.md | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 yggdrasil_decision_forests/utils/distribute/README.md diff --git a/yggdrasil_decision_forests/utils/distribute/README.md b/yggdrasil_decision_forests/utils/distribute/README.md new file mode 100644 index 00000000..05ab6502 --- /dev/null +++ b/yggdrasil_decision_forests/utils/distribute/README.md @@ -0,0 +1,175 @@ +# Distribute + +**Distribute** is an open-source C++ library that allows for the implementation +of distributed algorithms in systems such as Borg and Cloud. Distribute provides +a small number of low-level primitives, granting developers maximum flexibility. + +Distribute is used for all the distributed computation of YDF, including +hyper-parameter tuning, benchmarks, and distributed training. + +## Features + +- Logic control in case of worker preemption or failure. +- Extensive testing on pipelines with hundreds of workers over multiple days. +- Cross-worker communication. +- Generic workers can be used in multiple different pipelines at the same + time. +- Multiple available backends (Borg workers, TensorFlow Distribute, + Multi-threads). Multi-threads backend is particularly useful during + development. + +## Computation model + +**Initialization** + +- Multiple machines execute a manager and N worker processes. Each worker is + assigned to an integer id in [0, N). Each worker knows its id. +- The pipeline is initialized by the manager with a "welcome" data blob (e.g. + a proto). The welcome blob cannot be modified. +- The workers are initialized with this welcome blob of data. After a worker + is preempted and restarted, it is re-initialized with this same welcome blob + of data. + +**Computation** + +- Computation is triggered by "queries". +- A query is created by the manager or a worker and contains a blob of data + called the query data (typically a proto). +- The query is executed by a worker who replies with a blob of data (called + the answer data) and an absl::Status. +- Queries can be issued synchronously (blocking) or asynchronously + (non-blocking). +- Queries can be sent globally (i.e., any available worker can execute them) + or to specific workers (identified by the worker id). +- A worker can emit queries while executing a query. This is useful for + cascade execution. +- The number of queries processed by each worker in parallel is configurable + in the manager and can be adjusted during pipeline execution. +- If a query computation fails, the absl::Status is returned to the caller. + +**Failure scenario** + +- After being preempted and restarted, a worker is re-initialized with a + welcome blob of data. The welcome blob of data can be used, for example, to + specify a CNS path to a checkpoint location. +- If the manager is restarted (e.g., preempted, failure), all the workers are + restarted. +- Each query emitter (manager or workers) is responsible for the queries their + emit. +- If a worker is interrupted while executing a global query (i.e. a query that + any worker can execute), the next available worker will execute this query + automatically. +- If a worker is interrupted while executing a worker targeted query (i.e. a + query that can only be executed by a given worker), the query emitter waits + for the worker to be back online and re-send the query automatically. +- If a query emitter (manager or worker) is interrupted while a worker is + executing one of its query, the query answer is discarded. + +**Shutdown** + +- When the user code on the manager stops a pipeline, the shutdown method is + called on all the workers. +- When the manager stops a pipeline, the worker processes can be interrupted + or kept running. +- If the worker processes are not interrupted, a new manager can be created to + start a new pipeline. + +## Minimal example + +**The worker : `worker.cc`** + +```c++ +class ToyWorker final : public AbstractWorker { + +public: + + virtual ~ToyWorker() = default; + + // Initialize the worker with the welcome blob. + // Note: "Blob" is an alias for "std::string". + absl::Status Setup(Blob welcome_blob) override { + YDF_LOG(INFO) << "I am worker #" << WorkerIdx(); + return absl::OkStatus(); + } + + // Stop the worker. + absl::Status Done() override { + YDF_LOG(INFO) << "Bye"; + return absl::OkStatus(); + } + + // Execute a request. + absl::StatusOr RunRequest(Blob blob) override { + if(blob == "ping") return "pong"; + return absl::InvalidArgumentError("Unknown task"); + } +}; + +constexpr char kToyWorkerKey[] = "ToyWorker"; +REGISTER_Distribution_Worker(ToyWorker, kToyWorkerKey); +``` + +**The manager : `manager.cc`** + +```c++ +// Initialize +proto::Config config; +config.set_implementation_key("MULTI_THREAD"); // For debugging +config.MutableExtension(proto::multi_thread)->set_num_workers(5); +auto manager = CreateManager(config, /*worker_name=*/kToyWorkerKey, /*welcome_blob=*/"hello"); + +// Process + +// Blocking request to any worker. +auto result = manager->BlockingRequest("ping").value(); + +// Blocking request to a specific worker. +auto result = manager->BlockingRequest("ping", /*worker_idx=*/ 2).value(); + +// Async request to any worker. +for(int i=0; i<100; i++){ + manager->AsynchronousRequest("ping"); +} +for(int i=0; i<100; i++){ + auto result = manager->NextAsynchronousAnswer().value(); +} + +// Async request to a specific worker. +for(int i=0; i<100; i++){ + manager->AsynchronousRequest("ping", /*worker_idx=*/ i % manager->NumWorkers()); +} +for(int i=0; i<100; i++){ + auto result = manager->NextAsynchronousAnswer().value(); +} + +// Note: Workers can also execute "AsynchronousRequest". + +// Shutdown. This calls "Done" on all the workers and wait until it finishes. +manager->Done(); +``` + +## Examples + +### Beginner + +- [unit tests](https://source.corp.google.com/piper///depot/google3/third_party/yggdrasil_decision_forests/utils/distribute/distribute_test.cc): + Distribute unit tests. Shows all features. + +- [distribute cli](https://source.corp.google.com/piper///depot/google3/third_party/yggdrasil_decision_forests/utils/distribute_cli/): + Distribute the execution of CLI commands. + +### Intermediate + +- [hyperparameter_sweep](https://source.corp.google.com/piper///depot/google3/third_party/yggdrasil_decision_forests/examples/hyperparameter_sweep/README.md): + Trains and save many models with various input features. + +- [benchmark v2](https://source.corp.google.com/piper///depot/google3/learning/lib/ami/simple_ml/benchmark_v2/README.md): + An ML benchmark trainings and evaluating millions of models. + +- [hyperparameters optimizer](https://source.corp.google.com/piper///depot/google3/third_party/yggdrasil_decision_forests/learner/hyperparameters_optimizer/BUILD): + YDF hyper-parameter tuner. + +### Advanced + +- [Distributed GBT](https://source.corp.google.com/piper///depot/google3/third_party/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/BUILD): + Distributed GBT training. From 2300559f86f77edadbab4383f202779efac19f18 Mon Sep 17 00:00:00 2001 From: TensorFlow Decision Forests Team Date: Thu, 23 May 2024 06:44:14 -0700 Subject: [PATCH 04/30] Update example value of `sparse_oblique_num_projections_exponent` in docs PiperOrigin-RevId: 636533987 --- documentation/public/docs/guide_how_to_improve_model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/public/docs/guide_how_to_improve_model.md b/documentation/public/docs/guide_how_to_improve_model.md index 4750955a..8c728f0d 100644 --- a/documentation/public/docs/guide_how_to_improve_model.md +++ b/documentation/public/docs/guide_how_to_improve_model.md @@ -102,7 +102,7 @@ for more details. learner = ydf.RandomForestLearner( split_axis="SPARSE_OBLIQUE", sparse_oblique_normalization="MIN_MAX", - sparse_oblique_num_projections_exponent=1, + sparse_oblique_num_projections_exponent=1.0, ...) ``` From 15f7c06cbfd780d3d8984fe5e073d64311bc4af3 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Thu, 23 May 2024 06:58:05 -0700 Subject: [PATCH 05/30] Add tools for unit testing debugging. - Plotting of ellipse predictions. - mnist dataset PiperOrigin-RevId: 636536865 --- .../port/python/dev_requirements.txt | 3 +- .../port/python/ydf/model/BUILD | 1 + .../port/python/ydf/model/jax_model_test.py | 149 ++++++++++++++++-- 3 files changed, 139 insertions(+), 14 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/dev_requirements.txt b/yggdrasil_decision_forests/port/python/dev_requirements.txt index 86a58190..5642d727 100644 --- a/yggdrasil_decision_forests/port/python/dev_requirements.txt +++ b/yggdrasil_decision_forests/port/python/dev_requirements.txt @@ -6,4 +6,5 @@ matplotlib jax; platform_machine != 'aarch64' and platform_system != 'Windows' jaxlib; platform_machine != 'aarch64' and platform_system != 'Windows' optax; platform_machine != 'aarch64' and platform_system != 'Windows' and python_version >= '3.9' -flatbuffers; platform_machine != 'aarch64' and platform_system != 'Windows' and python_version >= '3.12' \ No newline at end of file +flatbuffers; platform_machine != 'aarch64' and platform_system != 'Windows' and python_version >= '3.12' +tensorflow-datasets; platform_machine != 'aarch64' and platform_system != 'Windows' and python_version >= '3.9' \ No newline at end of file diff --git a/yggdrasil_decision_forests/port/python/ydf/model/BUILD b/yggdrasil_decision_forests/port/python/ydf/model/BUILD index 49412faa..4330a671 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/model/BUILD @@ -341,6 +341,7 @@ py_test( # numpy dep, # optax dep, # tensorflow dep, + # tensorflow_datasets dep, "@ydf_cc//yggdrasil_decision_forests/dataset:data_spec_py_proto", "//ydf/dataset:dataspec", "//ydf/learner:generic_learner", diff --git a/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py b/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py index d3e03c04..11726d64 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py @@ -14,9 +14,10 @@ import array import logging +import os import sys import tempfile -from typing import Any, Dict, List, Optional, Sequence +from typing import Any, Callable, Dict, List, Optional, Sequence from absl.testing import absltest from absl.testing import parameterized @@ -66,23 +67,67 @@ def create_dataset(columns: List[str], n: int = 1000) -> Dict[str, Any]: return {k: data[k] for k in columns} +def create_dataset_mnist( + num_examples: int, begin_example_idx: int = 0 +) -> Dict[str, Any]: + """Creates a binary classification dataset based on MNIST. + + This function cannot be executed in a sandbox test as it downloads a dataset. + + Args: + num_examples: Number of examples to extract from the training dataset. + begin_example_idx: Index of the first example in the training dataset. + + Returns: + The binary mnist dataset. + """ + + import tensorflow_datasets as tfds # pylint: disable=import-error,g-import-not-at-top + + tf_ds = tfds.load("mnist", split="train") + raw_ds = ( + tf_ds.skip(begin_example_idx) + .batch(num_examples) + .take(1) + .as_numpy_iterator() + .next() + ) + raw_image_shape = raw_ds["image"].shape + return { + "label": raw_ds["label"] >= 5, + "image": raw_ds["image"].reshape([raw_image_shape[0], -1]), + } + + def create_dataset_ellipse( num_examples: int = 1000, num_features: int = 3, plot_path: Optional[str] = None, -): - """Create a binary classification dataset classifying ellipses.""" +) -> Dict[str, np.ndarray]: + """Creates a binary classification dataset classifying ellipses. + + Args: + num_examples: Number of generated examples. + num_features: Number of features. + plot_path: If set, saves a plot of the examples to this file. + + Returns: + An ellipse dataset. + """ + features = np.random.uniform(-1, 1, size=[num_examples, num_features]) - scales = np.array([1.0 + i * 0.1 for i in range(num_features)]) + scales = np.array([1.0 + i * 0.4 for i in range(num_features)]) labels = ( np.sqrt(np.sum(np.multiply(np.square(features), scales), axis=1)) <= 0.80 ) if plot_path: colors = ["blue" if l else "red" for l in labels] - plt.scatter(features[:, 0], features[:, 1], color=colors, s=1.5) - plt.axis("off") - plt.savefig(plot_path) + fig, ax = plt.subplots(1, 1) + ax.scatter(features[:, 0], features[:, 1], color=colors, s=1.5) + ax.set_axis_off() + ax.set_aspect("equal") + fig.savefig(plot_path) data = {"label": labels} for i in range(num_features): @@ -90,6 +135,53 @@ def create_dataset_ellipse( return data +def plot_ellipse_predictions( + prediction_fn: Callable[[Dict[str, jax.Array]], jax.Array], + path: str, + resolution: int = 200, + num_features: int = 3, +) -> None: + """Plots the predictions of a model on the ellipse dataset. + + Args: + prediction_fn: A function taking a batch of examples, and returning + predictions. + path: Save to save the plot. + resolution: Plotting resolution, for each axis. + num_features: Number of features. + """ + + # Compute feature values + vs = np.linspace(-1, 1, resolution) + tuple_features = np.meshgrid(*((vs,) * num_features)) + raw_features = np.stack([np.reshape(f, [-1]) for f in tuple_features], axis=1) + dict_features = { + f"f_{i}": jnp.asarray(raw_features[:, i]) for i in range(num_features) + } + + # Generate predictions + predictions = prediction_fn(dict_features) + + disp_predictions = np.reshape(predictions, [resolution] * num_features) + if len(disp_predictions.shape) > 2: + disp_predictions = np.mean( + disp_predictions, + axis=[i for i in range(2, len(disp_predictions.shape))], + ) + + # Plot predictions + fig, ax = plt.subplots(1, 1) + ax.imshow( + disp_predictions, + interpolation="none", + resample=False, + cmap="coolwarm", + ) + ax.set_axis_off() + ax.set_aspect("equal") + fig.savefig(path) + + class JaxModelTest(parameterized.TestCase): @parameterized.parameters( @@ -798,16 +890,41 @@ def test_to_jax_function( atol=1e-5, ) - def test_fine_tune_model(self): + def test_dataset_ellipse(self): + with tempfile.TemporaryDirectory() as tempdir: + _ = create_dataset_ellipse( + 1000, plot_path=os.path.join(tempdir, "ds.png") + ) + + def prediction_fn(features: Dict[str, jax.Array]) -> jax.Array: + return features["f_0"] >= 0.1 + + plot_ellipse_predictions( + prediction_fn, os.path.join(tempdir, "predictions.png") + ) + + @parameterized.named_parameters( + ("leaves_ellipse", "ellipse", True), + # ("leaves_mnist", "mnist", True), # Not compatible with test sandbox + ) + def test_fine_tune_model(self, dataset: str, leaves_as_params: bool): # Note: Optax cannot be imported in python 3.8. - import optax + import optax # pylint: disable=import-error,g-import-not-at-top # Make datasets label = "label" - train_ds = create_dataset_ellipse(1000) - test_ds = create_dataset_ellipse(10000) - finetune_ds = create_dataset_ellipse(10000) + + if dataset == "ellipse": + train_ds = create_dataset_ellipse(1000) + test_ds = create_dataset_ellipse(10000) + finetune_ds = create_dataset_ellipse(10000) + elif dataset == "mnist": + train_ds = create_dataset_mnist(1000) + test_ds = create_dataset_mnist(10000, 1000) + finetune_ds = create_dataset_mnist(10000, 1000 + 10000) + else: + assert False # Train a model with YDF model = specialized_learners.GradientBoostedTreesLearner(label=label).train( @@ -822,9 +939,15 @@ def test_fine_tune_model(self): jax_model = to_jax.to_jax_function( model, apply_activation=False, - leaves_as_params=True, + leaves_as_params=leaves_as_params, ) + # Check parameter values + if leaves_as_params: + self.assertContainsSubset( + ["leaf_values", "initial_predictions"], jax_model.params + ) + def to_jax_array(d): """Converts a numpy array into a jax array.""" return {k: jnp.asarray(v) for k, v in d.items()} From 827748773b44f75f3db8a118d584bc9ddcf73abb Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Fri, 24 May 2024 02:41:47 -0700 Subject: [PATCH 06/30] Support for oblique splits in JAX converter. PiperOrigin-RevId: 636847004 --- .../port/python/ydf/model/BUILD | 1 + .../port/python/ydf/model/export_jax.py | 89 +++++++++++++++++-- .../port/python/ydf/model/jax_model_test.py | 70 +++++++++++---- 3 files changed, 139 insertions(+), 21 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/ydf/model/BUILD b/yggdrasil_decision_forests/port/python/ydf/model/BUILD index 4330a671..37ff4baa 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/model/BUILD @@ -346,6 +346,7 @@ py_test( "//ydf/dataset:dataspec", "//ydf/learner:generic_learner", "//ydf/learner:specialized_learners", + "//ydf/model/gradient_boosted_trees_model", "//ydf/model/tree:all", ], ) diff --git a/yggdrasil_decision_forests/port/python/ydf/model/export_jax.py b/yggdrasil_decision_forests/port/python/ydf/model/export_jax.py index dfc1f3c1..cfb1822d 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/export_jax.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/export_jax.py @@ -18,7 +18,7 @@ import dataclasses import enum import functools -from typing import Any, Sequence, Dict, Optional, List, Set, Tuple, Callable, Union +from typing import Any, Sequence, Dict, Optional, List, Set, Tuple, Callable, Union, MutableSequence from yggdrasil_decision_forests.dataset import data_spec_pb2 as ds_pb from ydf.dataset import dataspec as dataspec_lib @@ -39,9 +39,9 @@ # pytype: enable=import-error # Typehint for arrays -ArrayFloat = array.array -ArrayInt = array.array -ArrayBool = array.array +ArrayFloat = MutableSequence[float] +ArrayInt = MutableSequence[int] +ArrayBool = MutableSequence[int] # Names of the learnable parameters of the model. _PARAM_LEAF_VALUES = "leaf_values" @@ -52,6 +52,7 @@ class ConditionType(enum.IntEnum): GREATER_THAN = 0 IS_IN = 1 + SPARSE_OBLIQUE = 2 def compact_dtype(values: Sequence[int]) -> Any: @@ -425,13 +426,15 @@ class InternalForest: dataspec: Dataspec. leaf_outputs: Prediction values for each leaf node. split_features: Internal idx of the feature being tested for each non-leaf - node. + node. For oblique splits, "split_features" contains the number of weights. split_parameters: Parameter of the condition for each non-leaf nodes. (1) For "greather than" condition (i.e., feature >= threshold), "split_parameter" is the threshold. (2) For "is in" condition (i.e., feature in mask), "split_parameter" is an uint32 offset in the mask "catgorical_mask" wheret the condition is evaluated as - "catgorical_mask[split_parameter + attribute_value]". + "catgorical_mask[split_parameter + attribute_value]". (3) for oblique + splits, "split_parameter" an uint32 offset for the first weight and + attribute in "oblique_weights" and "oblique_attributes" respectively. negative_children: Node offset of the negative children for each non-leaf node in the forest. positive_children: Node offset of the positive children for each non-leaf @@ -442,6 +445,9 @@ class InternalForest: trees. begin_leaf_nodes: Index of the first leaf node for each of the trees. catgorical_mask: Boolean mask used in "is in" conditions. + oblique_weights: Buffer of weights for the oblique splits. + oblique_attributes: Buffer of attributes for the oblique splits. Has the + same size as "oblique_weights". initial_predictions: Initial predictions of the forest (before any tree is applied). max_depth: Maximum depth of the trees. @@ -482,6 +488,12 @@ class InternalForest: catgorical_mask: ArrayBool = dataclasses.field( default_factory=lambda: array.array("b", []) ) + oblique_weights: ArrayFloat = dataclasses.field( + default_factory=lambda: array.array("f", []) + ) + oblique_attributes: ArrayInt = dataclasses.field( + default_factory=lambda: array.array("l", []) + ) initial_predictions: ArrayFloat = dataclasses.field( default_factory=lambda: array.array("f", []) ) @@ -500,6 +512,8 @@ def clear_array_data(self) -> None: self.begin_non_leaf_nodes = array.array("l", []) self.begin_leaf_nodes = array.array("l", []) self.catgorical_mask = array.array("l", []) + self.oblique_weights = array.array("f", []) + self.oblique_attributes = array.array("l", []) # Note: We don't release "initial_predictions". def __post_init__(self, model: generic_model.GenericModel): @@ -614,6 +628,32 @@ def _add_node( self.split_parameters.append(float_offset) self.condition_types.append(ConditionType.IS_IN) self.catgorical_mask.extend(bitmap) + + elif isinstance(node.condition, tree_lib.NumericalSparseObliqueCondition): + offset = len(self.oblique_weights) + num_weights = len(node.condition.weights) + + # Add the weights + self.oblique_weights.extend(node.condition.weights) + self.oblique_attributes.extend([ + self.feature_spec.inv_numerical[attribute] + for attribute in node.condition.attributes + ]) + # Add the bias + self.oblique_weights.append(node.condition.threshold) + self.oblique_attributes.append(0) + + # Encode the offset as a float32 + float_offset = float( + jax.lax.bitcast_convert_type( + jnp.array(offset, dtype=jnp.int32), jnp.float32 + ) + ) + + self.split_features.append(num_weights) + self.split_parameters.append(float_offset) + self.condition_types.append(ConditionType.SPARSE_OBLIQUE) + else: # TODO: Add support for other types of conditions. raise ValueError( @@ -674,6 +714,8 @@ class InternalForestJaxArrays: begin_non_leaf_nodes: jax.Array = dataclasses.field(init=False) begin_leaf_nodes: jax.Array = dataclasses.field(init=False) catgorical_mask: Optional[jax.Array] = dataclasses.field(init=False) + oblique_weights: Optional[jax.Array] = dataclasses.field(init=False) + oblique_attributes: Optional[jax.Array] = dataclasses.field(init=False) initial_predictions: Optional[jax.Array] = dataclasses.field(init=False) def __post_init__(self, forest: InternalForest): @@ -704,6 +746,16 @@ def __post_init__(self, forest: InternalForest): else: self.catgorical_mask = None + if forest.oblique_weights: + self.oblique_weights = asarray(forest.oblique_weights, dtype=jnp.float32) + else: + self.oblique_weights = None + + if forest.oblique_attributes: + self.oblique_attributes = to_compact_jax_array(forest.oblique_attributes) + else: + self.oblique_attributes = None + self.initial_predictions = asarray( forest.initial_predictions, dtype=jnp.float32 ) @@ -937,6 +989,27 @@ def condition_is_in(node_idx): ) return jax_arrays.catgorical_mask[categorical_mask_offset] + def condition_sparse_oblique(node_idx): + """Evaluates a sparse oblique condition.""" + num_weights = jax_arrays.split_features[node_idx] + offset = jax.lax.bitcast_convert_type( + jax_arrays.split_parameters[node_idx], jnp.int32 + ) + bias = jax_arrays.oblique_weights[offset + num_weights] + numerical_features = intern_feature_values["numerical"] + + def sum_iter(i, a): + return ( + a + + numerical_features[jax_arrays.oblique_attributes[i]] + * jax_arrays.oblique_weights[i] + ) + + weighted_sum = jax.lax.fori_loop( + offset, num_weights + offset, sum_iter, -bias + ) + return weighted_sum >= 0 + # Assemble the condition map. condition_fns = [None] * len(jax_arrays.dense_condition_mapping) if ConditionType.GREATER_THAN in jax_arrays.dense_condition_mapping: @@ -947,6 +1020,10 @@ def condition_is_in(node_idx): condition_fns[jax_arrays.dense_condition_mapping[ConditionType.IS_IN]] = ( condition_is_in ) + if ConditionType.SPARSE_OBLIQUE in jax_arrays.dense_condition_mapping: + condition_fns[ + jax_arrays.dense_condition_mapping[ConditionType.SPARSE_OBLIQUE] + ] = condition_sparse_oblique if len(condition_fns) == 1: # Since there is only one type of conditions, there is not need for a diff --git a/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py b/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py index 11726d64..9939e326 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py @@ -35,17 +35,23 @@ from ydf.model import export_jax as to_jax from ydf.model import generic_model from ydf.model import tree as tree_lib - +from ydf.model.gradient_boosted_trees_model import gradient_boosted_trees_model InternalFeatureItem = to_jax.InternalFeatureItem -def create_dataset(columns: List[str], n: int = 1000) -> Dict[str, Any]: +def create_dataset( + columns: List[str], n: int = 1000, seed: Optional[int] = None +) -> Dict[str, Any]: """Creates a dataset with random values.""" + if seed is not None: + np.random.seed(seed) data = { # Single-dim features "f1": np.random.random(size=n), "f2": np.random.random(size=n), + "f3": np.random.random(size=n), + "f4": np.random.random(size=n), "i1": np.random.randint(100, size=n), "i2": np.random.randint(100, size=n), "c1": np.random.choice(["x", "y", "z"], size=n, p=[0.6, 0.3, 0.1]), @@ -810,6 +816,19 @@ def test_densify_conditions( False, specialized_learners.GradientBoostedTreesLearner, ), + ( + "gbt_regression_num_oblique", + ["f1", "f2", "f3", "f4"], + "label_regress", + generic_learner.Task.REGRESSION, + False, + specialized_learners.GradientBoostedTreesLearner, + { + "split_axis": "SPARSE_OBLIQUE", + "sparse_oblique_normalization": "STANDARD_DEVIATION", + }, + False, # TODO: Check conversion when bug solved. + ), ) def test_to_jax_function( self, @@ -818,12 +837,17 @@ def test_to_jax_function( task: generic_learner.Task, has_encoding: bool, learner, + learner_kwargs=None, + test_tf_conversion: bool = True, ): - if learner == specialized_learners.GradientBoostedTreesLearner: - learner_kwargs = {"validation_ratio": 0.0} - else: + if learner_kwargs is None: learner_kwargs = {} + else: + learner_kwargs = learner_kwargs.copy() + + if learner == specialized_learners.GradientBoostedTreesLearner: + learner_kwargs["validation_ratio"] = 0.0 # Create YDF model columns = features + [label] @@ -831,10 +855,13 @@ def test_to_jax_function( label=label, task=task, **learner_kwargs, - ).train(create_dataset(columns, 1000)) + ).train(create_dataset(columns, 1000, seed=1)) + self.assertIsInstance( + model, gradient_boosted_trees_model.GradientBoostedTreesModel + ) # Golden predictions - test_ds = create_dataset(columns, 1000) + test_ds = create_dataset(columns, 1, seed=2) ydf_predictions = model.predict(test_ds) # Convert model to tf function @@ -859,6 +886,9 @@ def test_to_jax_function( atol=1e-5, ) + if not test_tf_conversion: + return + # Convert to a TensorFlow function tf_model = tf.Module() tf_model.my_call = tf.function( @@ -885,7 +915,7 @@ def test_to_jax_function( restored_tf_predictions = restored_tf_model.my_call(input_values) np.testing.assert_allclose( restored_tf_predictions, - ydf_predictions, + jax_predictions, rtol=1e-5, atol=1e-5, ) @@ -904,10 +934,13 @@ def prediction_fn(features: Dict[str, jax.Array]) -> jax.Array: ) @parameterized.named_parameters( - ("leaves_ellipse", "ellipse", True), - # ("leaves_mnist", "mnist", True), # Not compatible with test sandbox + ("leaves_ellipse", "ellipse", True, False), + # ("leaves_mnist", "mnist", True , False), # Skip in sandboxed test + ("leaves_ellipse_oblique", "ellipse", True, True), ) - def test_fine_tune_model(self, dataset: str, leaves_as_params: bool): + def test_fine_tune_model( + self, dataset: str, leaves_as_params: bool, oblique: bool + ): # Note: Optax cannot be imported in python 3.8. import optax # pylint: disable=import-error,g-import-not-at-top @@ -926,10 +959,15 @@ def test_fine_tune_model(self, dataset: str, leaves_as_params: bool): else: assert False + kwargs = {} + if oblique: + kwargs["split_axis"] = "SPARSE_OBLIQUE" + kwargs["sparse_oblique_normalization"] = "STANDARD_DEVIATION" + # Train a model with YDF - model = specialized_learners.GradientBoostedTreesLearner(label=label).train( - train_ds - ) + model = specialized_learners.GradientBoostedTreesLearner( + label=label, **kwargs + ).train(train_ds) # Evaluate the YDF model pre_tuned_ydf_accuracy = model.evaluate(test_ds).accuracy @@ -976,7 +1014,9 @@ def compute_loss(state, batch): compute_accuracy(jax_model.params, jax_test_ds) ) logging.info("pre_tuned_jax_test_accuracy: %s", pre_tuned_jax_test_accuracy) - self.assertAlmostEqual(pre_tuned_jax_test_accuracy, pre_tuned_ydf_accuracy) + self.assertAlmostEqual( + pre_tuned_jax_test_accuracy, pre_tuned_ydf_accuracy, delta=1.0e-5 + ) # Finetune the JAX model assert jax_model.params is not None From 3aec76ea19dff886dc2e9a6656959bb09a02cd5d Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Fri, 24 May 2024 04:31:58 -0700 Subject: [PATCH 07/30] Sort the dictionary of integer labels casted to string following the numerical order. PiperOrigin-RevId: 636870447 --- .../port/python/ydf/dataset/dataset.py | 50 +++++++++++++-- .../port/python/ydf/dataset/dataset_test.py | 61 ++++++++++++++----- .../port/python/ydf/dataset/dataspec.py | 2 +- .../python/ydf/learner/generic_learner.py | 1 + .../port/python/ydf/model/generic_model.py | 2 +- 5 files changed, 93 insertions(+), 23 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py index 1be34d97..8960f095 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py @@ -56,6 +56,7 @@ def _add_column( column_data: Any, inference_args: Optional[dataspec.DataSpecInferenceArgs], column_idx: Optional[int], + is_label: bool, ): """Adds a column to the dataset and computes the column statistics.""" assert (column_idx is None) != (inference_args is None) @@ -112,7 +113,7 @@ def _add_column( elif column.semantic == dataspec.Semantic.CATEGORICAL: - from_boolean = False + force_dictionary = None if not isinstance(column_data, np.ndarray): column_data = np.array(column_data, dtype=np.bytes_) ydf_dtype = dataspec.np_dtype_to_ydf_dtype(column_data.dtype) @@ -121,7 +122,16 @@ def _add_column( bool_column_data = column_data column_data = np.full_like(bool_column_data, b"false", "|S5") column_data[bool_column_data] = b"true" - from_boolean = True + force_dictionary = [dataspec.YDF_OOD_BYTES, b"false", b"true"] + elif ( + is_label + and column_data.dtype.type in dataspec.NP_SUPPORTED_INT_DTYPE + and (dictionary_size := dense_integer_dictionary_size(column_data)) + ): + column_data = column_data.astype(np.bytes_) + force_dictionary = [dataspec.YDF_OOD_BYTES] + [ + str(i).encode("utf-8") for i in range(dictionary_size) + ] elif ( column_data.dtype.type in [ @@ -145,10 +155,8 @@ def _add_column( if column_data.dtype.type == np.bytes_: if inference_args is not None: guide = dataspec.categorical_column_guide(column, inference_args) - if from_boolean: - guide["dictionary"] = np.array( - [b"", b"false", b"true"], dtype=np.bytes_ - ) + if force_dictionary: + guide["dictionary"] = np.array(force_dictionary, dtype=np.bytes_) self._dataset.PopulateColumnCategoricalNPBytes( column.name, column_data, **guide, ydf_dtype=ydf_dtype ) @@ -259,6 +267,7 @@ def create_vertical_dataset( data_spec: Optional[data_spec_pb2.DataSpecification] = None, required_columns: Optional[Sequence[str]] = None, dont_unroll_columns: Optional[Sequence[str]] = None, + label: Optional[str] = None, ) -> VerticalDataset: """Creates a VerticalDataset from various sources of data. @@ -342,6 +351,7 @@ def create_vertical_dataset( mentioned in the data spec or `columns` are required. dont_unroll_columns: List of columns that cannot be unrolled. If one such column needs to be unrolled, raise an error. + label: Name of the label column, if any. Returns: Dataset to be ingested by the learner algorithms. @@ -364,6 +374,7 @@ def create_vertical_dataset( data_spec=data_spec, inference_args=None, dont_unroll_columns=dont_unroll_columns, + label=label, ) else: inference_args = dataspec.DataSpecInferenceArgs( @@ -382,6 +393,7 @@ def create_vertical_dataset( inference_args=inference_args, data_spec=None, dont_unroll_columns=dont_unroll_columns, + label=label, ) @@ -391,6 +403,7 @@ def create_vertical_dataset_with_spec_or_args( inference_args: Optional[dataspec.DataSpecInferenceArgs], data_spec: Optional[data_spec_pb2.DataSpecification], dont_unroll_columns: Optional[Sequence[str]] = None, + label: Optional[str] = None, ) -> VerticalDataset: """Returns a vertical dataset from inference args or data spec (not both!).""" assert (data_spec is None) != (inference_args is None) @@ -416,6 +429,7 @@ def create_vertical_dataset_with_spec_or_args( required_columns, inference_args=inference_args, data_spec=data_spec, + label=label, ) @@ -447,6 +461,7 @@ def create_vertical_dataset_from_dict_of_values( required_columns: Optional[Sequence[str]], inference_args: Optional[dataspec.DataSpecInferenceArgs], data_spec: Optional[data_spec_pb2.DataSpecification], + label: Optional[str] = None, ) -> VerticalDataset: """Specialization of create_vertical_dataset to dictionary of values. @@ -461,6 +476,7 @@ def create_vertical_dataset_from_dict_of_values( is set. data_spec: Data spec of the given data. Must be None if inference_args is set. + label: Name of the label column, if any. Returns: A Vertical dataset with the given properties. @@ -543,6 +559,7 @@ def dataspec_to_normalized_columns( column_data, inference_args=inference_args, # Might be None column_idx=column_idx if data_spec is not None else None, + is_label=label == column.name, ) if data_spec is None: @@ -667,3 +684,24 @@ def _type(value: Any) -> str: return f"numpy's array of '{value.dtype.name}'" else: return str(type(value)) + + +def dense_integer_dictionary_size(values: np.ndarray) -> Optional[int]: + """Gets the number of items in a dense and zero-indexed array of integers. + + If the array is not dense or not zero-indexed, returns None. + + Args: + values: Numpy array of integer values. + + Returns: + Number of unique dense values, or None. + """ + unique_values = np.unique(values).tolist() + if ( + unique_values + and unique_values[0] == 0 + and unique_values[-1] + 1 == len(unique_values) + ): + return len(unique_values) + return None diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py index 69a455bd..a5f9aa3f 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py @@ -571,7 +571,7 @@ def test_order_boolean(self, values, expected_counts, count_nas): count_nas=count_nas, categorical=ds_pb.CategoricalSpec( items={ - "": VocabValue(index=0, count=expected_counts[0]), + "": VocabValue(index=0, count=expected_counts[0]), "false": VocabValue(index=1, count=expected_counts[1]), "true": VocabValue(index=2, count=expected_counts[2]), }, @@ -582,31 +582,38 @@ def test_order_boolean(self, values, expected_counts, count_nas): ) test_utils.assertProto2Equal(self, ds.data_spec(), expected_data_spec) - @parameterized.parameters( - ([True, True, True], (0, 0, 3), 0), - ([False, False, False], (0, 3, 0), 0), - ([True, False, False], (0, 2, 1), 0), - ) - def test_order_boolean(self, values, expected_counts, count_nas): + def test_order_integers(self): ds = dataset.create_vertical_dataset( - {"col": np.array(values)}, + {"col": np.array([0, 1, 4, 3, 1, 2, 3, 4, 12, 11, 10, 9, 8, 7, 6, 5])}, columns=[Column("col", dataspec.Semantic.CATEGORICAL)], + label="col", ) expected_data_spec = ds_pb.DataSpecification( - created_num_rows=3, + created_num_rows=16, columns=( ds_pb.Column( name="col", type=ds_pb.ColumnType.CATEGORICAL, - dtype=ds_pb.DType.DTYPE_BOOL, - count_nas=count_nas, + dtype=ds_pb.DType.DTYPE_INT64, + count_nas=0, categorical=ds_pb.CategoricalSpec( items={ - "": VocabValue(index=0, count=expected_counts[0]), - "false": VocabValue(index=1, count=expected_counts[1]), - "true": VocabValue(index=2, count=expected_counts[2]), + "": VocabValue(index=0, count=0), + "0": VocabValue(index=1, count=1), + "1": VocabValue(index=2, count=2), + "2": VocabValue(index=3, count=1), + "3": VocabValue(index=4, count=2), + "4": VocabValue(index=5, count=2), + "5": VocabValue(index=6, count=1), + "6": VocabValue(index=7, count=1), + "7": VocabValue(index=8, count=1), + "8": VocabValue(index=9, count=1), + "9": VocabValue(index=10, count=1), + "10": VocabValue(index=11, count=1), + "11": VocabValue(index=12, count=1), + "12": VocabValue(index=13, count=1), }, - number_of_unique_values=3, + number_of_unique_values=14, ), ), ), @@ -1885,5 +1892,29 @@ def test_required_columns_file_inference_args_explicit_success( self.assertEqual(ds._dataset.DebugString(), "f1\n1\n2\n3\n") +class DenseDictionaryTest(parameterized.TestCase): + + @parameterized.parameters( + ([0], 1), + ([0, 1], 2), + ([0, 1, 1, 0], 2), + ([1, 0], 2), + ([4, 3, 4, 1, 2, 0, 1, 2, 4], 5), + ) + def test_dense_integer_dictionary_size(self, values, expected): + self.assertEqual( + dataset.dense_integer_dictionary_size(np.array(values)), expected + ) + + @parameterized.parameters( + ([],), + ([-1, 0, 1],), + ([1, 2, 3, 4],), + ([0, 1, 3, 4],), + ) + def test_dense_integer_dictionary_size_is_none(self, values): + self.assertIsNone(dataset.dense_integer_dictionary_size(np.array(values))) + + if __name__ == "__main__": absltest.main() diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/dataspec.py b/yggdrasil_decision_forests/port/python/ydf/dataset/dataspec.py index 0522f5c3..9a0d8b29 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/dataspec.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/dataspec.py @@ -16,7 +16,6 @@ import dataclasses import enum -import logging from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple, Union import numpy as np @@ -35,6 +34,7 @@ # Must match kOutOfDictionaryItemKey in # yggdrasil_decision_forests/dataset/data_spec.h YDF_OOD = "" +YDF_OOD_BYTES = b"" # Mapping between Numpy dtypes and YDF dtypes. _NP_DTYPE_TO_YDF_DTYPE = { diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py index 2a946788..f624a480 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py @@ -339,6 +339,7 @@ def _get_vertical_dataset( inference_args=effective_data_spec_args, required_columns=None, # All columns in the dataspec are required. dont_unroll_columns=dont_unroll_columns, + label=self._label, ) def cross_validation( diff --git a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py index e96df37c..7eee2933 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py @@ -944,7 +944,7 @@ def label(self) -> str: return self.data_spec().columns[self.label_col_idx()].name def label_classes(self) -> List[str]: - """Returns the label classes for classification tasks, None otherwise.""" + """Returns the label classes for a classification model; fails otherwise.""" if self.task() != Task.CLASSIFICATION: raise ValueError( "Label classes are only available for classification models. This" From 2b3e21cbdf2892d3c3aa531ee3caa6376f0e00b1 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 27 May 2024 03:04:06 -0700 Subject: [PATCH 08/30] Add `verbose` argument to `train` method which is equivalent but sometime more convenient than`ydf.verbose`. PiperOrigin-RevId: 637584007 --- .../port/python/CHANGELOG.md | 2 + .../python/ydf/learner/generic_learner.py | 16 +- .../port/python/ydf/learner/learner_test.py | 10 +- .../specialized_learners_pre_generated.py | 493 +++++++----------- .../ydf/learner/wrapper/wrapper_generator.cc | 7 +- .../ydf/learner/wrapper/wrapper_test.cc | 7 +- .../port/python/ydf/utils/log.py | 14 +- 7 files changed, 242 insertions(+), 307 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/CHANGELOG.md b/yggdrasil_decision_forests/port/python/CHANGELOG.md index c81bd9b8..42b6785b 100644 --- a/yggdrasil_decision_forests/port/python/CHANGELOG.md +++ b/yggdrasil_decision_forests/port/python/CHANGELOG.md @@ -5,6 +5,8 @@ ### Feature - Add `max_depth` argument to `model.print_tree`. +- Add `verbose` argument to `train` method which is equivalent but sometime + more convenient than`ydf.verbose`. ### Fix diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py index f624a480..37da48e0 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py @@ -37,6 +37,7 @@ from ydf.model import generic_model from ydf.model import model_lib from ydf.utils import log +from ydf.utils import log from yggdrasil_decision_forests.utils import fold_generator_pb2 from yggdrasil_decision_forests.utils.distribute.implementations.grpc import grpc_pb2 @@ -116,6 +117,7 @@ def train( self, ds: dataset.InputDataset, valid: Optional[dataset.InputDataset] = None, + verbose: Optional[Union[int, bool]] = None, ) -> generic_model.ModelType: """Trains a model on the given dataset. @@ -162,6 +164,10 @@ def train( do not need validation dataset. Some learners, such as GradientBoostedTrees, automatically extract a validation dataset from the training dataset if the validation dataset is not provided. + verbose: Verbose level during training. If None, uses the global verbose + level of `ydf.verbose`. Levels are: 0 of False: No logs, 1 or True: + Print a few logs in a notebook; prints all the logs in a terminal. 2: + Prints all the logs on all surfaces. Returns: A trained model. @@ -187,7 +193,15 @@ def train( "The validation dataset may only be a path if the training dataset is" " a path." ) - return self._train_from_dataset(ds, valid) + + saved_verbose = log.verbose(verbose) if verbose is not None else None + try: + model = self._train_from_dataset(ds, valid) + finally: + if saved_verbose is not None: + log.verbose(saved_verbose) + + return model def __str__(self) -> str: return f"""\ diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py b/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py index 5ab0e156..8f7816c0 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py @@ -994,14 +994,20 @@ def test_model_with_na_conditions_numerical(self): class LoggingTest(parameterized.TestCase): - @parameterized.parameters(0, 1, 2) - def test_logging(self, verbose): + @parameterized.parameters(0, 1, 2, False, True) + def test_logging_function(self, verbose): save_verbose = log.verbose(verbose) learner = specialized_learners.RandomForestLearner(label="label") ds = pd.DataFrame({"feature": [0, 1], "label": [0, 1]}) _ = learner.train(ds) log.verbose(save_verbose) + @parameterized.parameters(0, 1, 2, False, True) + def test_logging_arg(self, verbose): + learner = specialized_learners.RandomForestLearner(label="label") + ds = pd.DataFrame({"feature": [0, 1], "label": [0, 1]}) + _ = learner.train(ds, verbose=verbose) + class UtilityTest(LearnerTest): diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/specialized_learners_pre_generated.py b/yggdrasil_decision_forests/port/python/ydf/learner/specialized_learners_pre_generated.py index c3f0964a..0b19e100 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/specialized_learners_pre_generated.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/specialized_learners_pre_generated.py @@ -239,6 +239,18 @@ class RandomForestLearner(generic_learner.GenericLearner): expressed in seconds. Each learning algorithm is free to use this parameter at it sees fit. Enabling maximum training duration makes the model training non-deterministic. Default: -1.0. + mhld_oblique_max_num_attributes: For MHLD oblique splits i.e. + `split_axis=MHLD_OBLIQUE`. Maximum number of attributes in the projection. + Increasing this value increases the training time. Decreasing this value + acts as a regularization. The value should be in [2, + num_numerical_features]. If the value is above the total number of + numerical features, the value is capped automatically. The value 1 is + allowed but results in ordinary (non-oblique) splits. Default: None. + mhld_oblique_sample_attributes: For MHLD oblique splits i.e. + `split_axis=MHLD_OBLIQUE`. If true, applies the attribute sampling + controlled by the "num_candidate_attributes" or + "num_candidate_attributes_ratio" parameters. If false, all the attributes + are tested. Default: None. min_examples: Minimum number of examples in a node. Default: 5. missing_value_policy: Method used to handle missing attribute values. - `GLOBAL_IMPUTATION`: Missing attribute values are imputed, with the mean @@ -293,6 +305,16 @@ class RandomForestLearner(generic_learner.GenericLearner): IN_NODE. - IN_NODE: The features are sorted just before being used in the node. This solution is slow but consumes little amount of memory. . Default: "PRESORT". + sparse_oblique_max_num_projections: For sparse oblique splits i.e. + `split_axis=SPARSE_OBLIQUE`. Maximum number of projections (applied after + the num_projections_exponent). Oblique splits try out + max(p^num_projections_exponent, max_num_projections) random projections + for choosing a split, where p is the number of numerical features. + Increasing "max_num_projections" increases the training time but not the + inference time. In late stage model development, if every bit of accuracy + if important, increase this value. The paper "Sparse Projection Oblique + Random Forests" (Tomita et al, 2020) does not define this hyperparameter. + Default: None. sparse_oblique_normalization: For sparse oblique splits i.e. `split_axis=SPARSE_OBLIQUE`. Normalization applied on the features, before applying the sparse oblique projections. - `NONE`: No normalization. - @@ -302,12 +324,28 @@ class RandomForestLearner(generic_learner.GenericLearner): max-min) estimated on the entire train dataset. Default: None. sparse_oblique_num_projections_exponent: For sparse oblique splits i.e. `split_axis=SPARSE_OBLIQUE`. Controls of the number of random projections - to test at each node as `num_features^num_projections_exponent`. Default: - None. - sparse_oblique_projection_density_factor: For sparse oblique splits i.e. - `split_axis=SPARSE_OBLIQUE`. Controls of the number of random projections - to test at each node as `num_features^num_projections_exponent`. Default: - None. + to test at each node. Increasing this value very likely improves the + quality of the model, drastically increases the training time, and doe not + impact the inference time. Oblique splits try out + max(p^num_projections_exponent, max_num_projections) random projections + for choosing a split, where p is the number of numerical features. + Therefore, increasing this `num_projections_exponent` and possibly + `max_num_projections` may improve model quality, but will also + significantly increase training time. Note that the complexity of + (classic) Random Forests is roughly proportional to + `num_projections_exponent=0.5`, since it considers sqrt(num_features) for + a split. The complexity of (classic) GBDT is roughly proportional to + `num_projections_exponent=1`, since it considers all features for a split. + The paper "Sparse Projection Oblique Random Forests" (Tomita et al, 2020) + recommends values in [1/4, 2]. Default: None. + sparse_oblique_projection_density_factor: Density of the projections as an + exponent of the number of features. Independently for each projection, + each feature has a probability "projection_density_factor / num_features" + to be considered in the projection. The paper "Sparse Projection Oblique + Random Forests" (Tomita et al, 2020) calls this parameter `lambda` and + recommends values in [1, 5]. Increasing this value increases training and + inference time (on average). This value is best tuned for each dataset. + Default: None. sparse_oblique_weights: For sparse oblique splits i.e. `split_axis=SPARSE_OBLIQUE`. Possible values: - `BINARY`: The oblique weights are sampled in {-1,1} (default). - `CONTINUOUS`: The oblique @@ -315,9 +353,11 @@ class RandomForestLearner(generic_learner.GenericLearner): split_axis: What structure of split to consider for numerical features. - `AXIS_ALIGNED`: Axis aligned splits (i.e. one condition at a time). This is the "classical" way to train a tree. Default value. - `SPARSE_OBLIQUE`: - Sparse oblique splits (i.e. splits one a small number of features) from - "Sparse Projection Oblique Random Forests", Tomita et al., 2020. Default: - "AXIS_ALIGNED". + Sparse oblique splits (i.e. random splits one a small number of features) + from "Sparse Projection Oblique Random Forests", Tomita et al., 2020. - + `MHLD_OBLIQUE`: Multi-class Hellinger Linear Discriminant splits from + "Classification Based on Multivariate Contrast Patterns", Canete-Sifuentes + et al., 2029 Default: "AXIS_ALIGNED". uplift_min_examples_in_treatment: For uplift models only. Minimum number of examples per treatment in a node. Default: 5. uplift_split_score: For uplift models only. Splitter score i.e. score @@ -401,6 +441,8 @@ def __init__( max_num_nodes: Optional[int] = None, maximum_model_size_in_memory_in_bytes: Optional[float] = -1.0, maximum_training_duration_seconds: Optional[float] = -1.0, + mhld_oblique_max_num_attributes: Optional[int] = None, + mhld_oblique_sample_attributes: Optional[bool] = None, min_examples: Optional[int] = 5, missing_value_policy: Optional[str] = "GLOBAL_IMPUTATION", num_candidate_attributes: Optional[int] = 0, @@ -411,6 +453,7 @@ def __init__( random_seed: Optional[int] = 123456, sampling_with_replacement: Optional[bool] = True, sorting_strategy: Optional[str] = "PRESORT", + sparse_oblique_max_num_projections: Optional[int] = None, sparse_oblique_normalization: Optional[str] = None, sparse_oblique_num_projections_exponent: Optional[float] = None, sparse_oblique_projection_density_factor: Optional[float] = None, @@ -458,6 +501,8 @@ def __init__( maximum_model_size_in_memory_in_bytes ), "maximum_training_duration_seconds": maximum_training_duration_seconds, + "mhld_oblique_max_num_attributes": mhld_oblique_max_num_attributes, + "mhld_oblique_sample_attributes": mhld_oblique_sample_attributes, "min_examples": min_examples, "missing_value_policy": missing_value_policy, "num_candidate_attributes": num_candidate_attributes, @@ -470,6 +515,9 @@ def __init__( "random_seed": random_seed, "sampling_with_replacement": sampling_with_replacement, "sorting_strategy": sorting_strategy, + "sparse_oblique_max_num_projections": ( + sparse_oblique_max_num_projections + ), "sparse_oblique_normalization": sparse_oblique_normalization, "sparse_oblique_num_projections_exponent": ( sparse_oblique_num_projections_exponent @@ -521,6 +569,7 @@ def train( self, ds: dataset.InputDataset, valid: Optional[dataset.InputDataset] = None, + verbose: Optional[Union[int, bool]] = None, ) -> random_forest_model.RandomForestModel: """Trains a model on the given dataset. @@ -551,11 +600,15 @@ def train( do not need validation dataset. Some learners, such as GradientBoostedTrees, automatically extract a validation dataset from the training dataset if the validation dataset is not provided. + verbose: Verbose level during training. If None, uses the global verbose + level of `ydf.verbose`. Levels are: 0 of False: No logs, 1 or True: + Print a few logs in a notebook; prints all the logs in a terminal. 2: + Prints all the logs on all surfaces. Returns: A trained model. """ - return super().train(ds, valid) + return super().train(ds=ds, valid=valid, verbose=verbose) @classmethod def capabilities(cls) -> abstract_learner_pb2.LearnerCapabilities: @@ -619,271 +672,6 @@ def hyperparameter_templates( } -class HyperparameterOptimizerLearner(generic_learner.GenericLearner): - r"""Hyperparameter Optimizer learning algorithm. - - Usage example: - - ```python - import ydf - import pandas as pd - - dataset = pd.read_csv("project/dataset.csv") - - model = ydf.HyperparameterOptimizerLearner().train(dataset) - - print(model.summary()) - ``` - - Hyperparameters are configured to give reasonable results for typical - datasets. Hyperparameters can also be modified manually (see descriptions) - below or by applying the hyperparameter templates available with - `HyperparameterOptimizerLearner.hyperparameter_templates()` (see this - function's documentation for - details). - - Attributes: - label: Label of the dataset. The label column should not be identified as a - feature in the `features` parameter. - task: Task to solve (e.g. Task.CLASSIFICATION, Task.REGRESSION, - Task.RANKING, Task.CATEGORICAL_UPLIFT, Task.NUMERICAL_UPLIFT). - weights: Name of a feature that identifies the weight of each example. If - weights are not specified, unit weights are assumed. The weight column - should not be identified as a feature in the `features` parameter. - ranking_group: Only for `task=Task.RANKING`. Name of a feature that - identifies queries in a query/document ranking task. The ranking group - should not be identified as a feature in the `features` parameter. - uplift_treatment: Only for `task=Task.CATEGORICAL_UPLIFT` and `task=Task`. - NUMERICAL_UPLIFT. Name of a numerical feature that identifies the - treatment in an uplift problem. The value 0 is reserved for the control - treatment. Currently, only 0/1 binary treatments are supported. - features: If None, all columns are used as features. The semantic of the - features is determined automatically. Otherwise, if - include_all_columns=False (default) only the column listed in `features` - are imported. If include_all_columns=True, all the columns are imported as - features and only the semantic of the columns NOT in `columns` is - determined automatically. If specified, defines the order of the features - - any non-listed features are appended in-order after the specified - features (if include_all_columns=True). The label, weights, uplift - treatment and ranking_group columns should not be specified as features. - include_all_columns: See `features`. - max_vocab_count: Maximum size of the vocabulary of CATEGORICAL and - CATEGORICAL_SET columns stored as strings. If more unique values exist, - only the most frequent values are kept, and the remaining values are - considered as out-of-vocabulary. - min_vocab_frequency: Minimum number of occurrence of a value for CATEGORICAL - and CATEGORICAL_SET columns. Value observed less than - `min_vocab_frequency` are considered as out-of-vocabulary. - discretize_numerical_columns: If true, discretize all the numerical columns - before training. Discretized numerical columns are faster to train with, - but they can have a negative impact on the model quality. Using - `discretize_numerical_columns=True` is equivalent as setting the column - semantic DISCRETIZED_NUMERICAL in the `column` argument. See the - definition of DISCRETIZED_NUMERICAL for more details. - num_discretized_numerical_bins: Number of bins used when disretizing - numerical columns. - max_num_scanned_rows_to_infer_semantic: Number of rows to scan when - inferring the column's semantic if it is not explicitly specified. Only - used when reading from file, in-memory datasets are always read in full. - Setting this to a lower number will speed up dataset reading, but might - result in incorrect column semantics. Set to -1 to scan the entire - dataset. - max_num_scanned_rows_to_compute_statistics: Number of rows to scan when - computing a column's statistics. Only used when reading from file, - in-memory datasets are always read in full. A column's statistics include - the dictionary for categorical features and the mean / min / max for - numerical features. Setting this to a lower number will speed up dataset - reading, but skew statistics in the dataspec, which can hurt model quality - (e.g. if an important category of a categorical feature is considered - OOV). Set to -1 to scan the entire dataset. - data_spec: Dataspec to be used (advanced). If a data spec is given, - `columns`, `include_all_columns`, `max_vocab_count`, - `min_vocab_frequency`, `discretize_numerical_columns` and - `num_discretized_numerical_bins` will be ignored. - maximum_model_size_in_memory_in_bytes: Limit the size of the model when - stored in ram. Different algorithms can enforce this limit differently. - Note that when models are compiled into an inference, the size of the - inference engine is generally much smaller than the original model. - Default: -1.0. - maximum_training_duration_seconds: Maximum training duration of the model - expressed in seconds. Each learning algorithm is free to use this - parameter at it sees fit. Enabling maximum training duration makes the - model training non-deterministic. Default: -1.0. - pure_serving_model: Clear the model from any information that is not - required for model serving. This includes debugging, model interpretation - and other meta-data. The size of the serialized model can be reduced - significatively (50% model size reduction is common). This parameter has - no impact on the quality, serving speed or RAM usage of model serving. - Default: False. - random_seed: Random seed for the training of the model. Learners are - expected to be deterministic by the random seed. Default: 123456. - num_threads: Number of threads used to train the model. Different learning - algorithms use multi-threading differently and with different degree of - efficiency. If `None`, `num_threads` will be automatically set to the - number of processors (up to a maximum of 32; or set to 6 if the number of - processors is not available). Making `num_threads` significantly larger - than the number of processors can slow-down the training speed. The - default value logic might change in the future. - resume_training: If true, the model training resumes from the checkpoint - stored in the `working_dir` directory. If `working_dir` does not contain - any model checkpoint, the training starts from the beginning. Resuming - training is useful in the following situations: (1) The training was - interrupted by the user (e.g. ctrl+c or "stop" button in a notebook) or - rescheduled, or (2) the hyper-parameter of the learner was changed e.g. - increasing the number of trees. - working_dir: Path to a directory available for the learning algorithm to - store intermediate computation results. Depending on the learning - algorithm and parameters, the working_dir might be optional, required, or - ignored. For instance, distributed training algorithm always need a - "working_dir", and the gradient boosted tree and hyper-parameter tuners - will export artefacts to the "working_dir" if provided. - resume_training_snapshot_interval_seconds: Indicative number of seconds in - between snapshots when `resume_training=True`. Might be ignored by some - learners. - tuner: If set, automatically select the best hyperparameters using the - provided tuner. When using distributed training, the tuning is - distributed. - workers: If set, enable distributed training. "workers" is the list of IP - addresses of the workers. A worker is a process running - `ydf.start_worker(port)`. - """ - - def __init__( - self, - label: str, - task: generic_learner.Task = generic_learner.Task.CLASSIFICATION, - weights: Optional[str] = None, - ranking_group: Optional[str] = None, - uplift_treatment: Optional[str] = None, - features: dataspec.ColumnDefs = None, - include_all_columns: bool = False, - max_vocab_count: int = 2000, - min_vocab_frequency: int = 5, - discretize_numerical_columns: bool = False, - num_discretized_numerical_bins: int = 255, - max_num_scanned_rows_to_infer_semantic: int = 10000, - max_num_scanned_rows_to_compute_statistics: int = 10000, - data_spec: Optional[data_spec_pb2.DataSpecification] = None, - maximum_model_size_in_memory_in_bytes: Optional[float] = -1.0, - maximum_training_duration_seconds: Optional[float] = -1.0, - pure_serving_model: Optional[bool] = False, - random_seed: Optional[int] = 123456, - num_threads: Optional[int] = None, - working_dir: Optional[str] = None, - resume_training: bool = False, - resume_training_snapshot_interval_seconds: int = 1800, - tuner: Optional[tuner_lib.AbstractTuner] = None, - workers: Optional[Sequence[str]] = None, - ): - - hyper_parameters = { - "maximum_model_size_in_memory_in_bytes": ( - maximum_model_size_in_memory_in_bytes - ), - "maximum_training_duration_seconds": maximum_training_duration_seconds, - "pure_serving_model": pure_serving_model, - "random_seed": random_seed, - } - - data_spec_args = dataspec.DataSpecInferenceArgs( - columns=dataspec.normalize_column_defs(features), - include_all_columns=include_all_columns, - max_vocab_count=max_vocab_count, - min_vocab_frequency=min_vocab_frequency, - discretize_numerical_columns=discretize_numerical_columns, - num_discretized_numerical_bins=num_discretized_numerical_bins, - max_num_scanned_rows_to_infer_semantic=max_num_scanned_rows_to_infer_semantic, - max_num_scanned_rows_to_compute_statistics=max_num_scanned_rows_to_compute_statistics, - ) - - deployment_config = self._build_deployment_config( - num_threads=num_threads, - resume_training=resume_training, - resume_training_snapshot_interval_seconds=resume_training_snapshot_interval_seconds, - working_dir=working_dir, - workers=workers, - ) - - super().__init__( - learner_name="HYPERPARAMETER_OPTIMIZER", - task=task, - label=label, - weights=weights, - ranking_group=ranking_group, - uplift_treatment=uplift_treatment, - data_spec_args=data_spec_args, - data_spec=data_spec, - hyper_parameters=hyper_parameters, - deployment_config=deployment_config, - tuner=tuner, - ) - - def train( - self, - ds: dataset.InputDataset, - valid: Optional[dataset.InputDataset] = None, - ) -> generic_model.GenericModel: - """Trains a model on the given dataset. - - Options for dataset reading are given on the learner. Consult the - documentation of the learner or ydf.create_vertical_dataset() for additional - information on dataset reading in YDF. - - Usage example: - - ``` - import ydf - import pandas as pd - - train_ds = pd.read_csv(...) - - learner = ydf.HyperparameterOptimizerLearner(label="label") - model = learner.train(train_ds) - print(model.summary()) - ``` - - If training is interrupted (for example, by interrupting the cell execution - in Colab), the model will be returned to the state it was in at the moment - of interruption. - - Args: - ds: Training dataset. - valid: Optional validation dataset. Some learners, such as Random Forest, - do not need validation dataset. Some learners, such as - GradientBoostedTrees, automatically extract a validation dataset from - the training dataset if the validation dataset is not provided. - - Returns: - A trained model. - """ - return super().train(ds, valid) - - @classmethod - def capabilities(cls) -> abstract_learner_pb2.LearnerCapabilities: - return abstract_learner_pb2.LearnerCapabilities( - support_max_training_duration=True, - resume_training=False, - support_validation_dataset=False, - support_partial_cache_dataset_format=False, - support_max_model_size_in_memory=False, - support_monotonic_constraints=False, - ) - - @classmethod - def hyperparameter_templates( - cls, - ) -> Dict[str, hyperparameters.HyperparameterTemplate]: - r"""Hyperparameter templates for this Learner. - - This learner currently does not provide any hyperparameter templates, this - method is provided for consistency with other learners. - - Returns: - Empty dictionary. - """ - return {} - - class GradientBoostedTreesLearner(generic_learner.GenericLearner): r"""Gradient Boosted Trees learning algorithm. @@ -1135,6 +923,18 @@ class GradientBoostedTreesLearner(generic_learner.GenericLearner): expressed in seconds. Each learning algorithm is free to use this parameter at it sees fit. Enabling maximum training duration makes the model training non-deterministic. Default: -1.0. + mhld_oblique_max_num_attributes: For MHLD oblique splits i.e. + `split_axis=MHLD_OBLIQUE`. Maximum number of attributes in the projection. + Increasing this value increases the training time. Decreasing this value + acts as a regularization. The value should be in [2, + num_numerical_features]. If the value is above the total number of + numerical features, the value is capped automatically. The value 1 is + allowed but results in ordinary (non-oblique) splits. Default: None. + mhld_oblique_sample_attributes: For MHLD oblique splits i.e. + `split_axis=MHLD_OBLIQUE`. If true, applies the attribute sampling + controlled by the "num_candidate_attributes" or + "num_candidate_attributes_ratio" parameters. If false, all the attributes + are tested. Default: None. min_examples: Minimum number of examples in a node. Default: 5. missing_value_policy: Method used to handle missing attribute values. - `GLOBAL_IMPUTATION`: Missing attribute values are imputed, with the mean @@ -1195,6 +995,16 @@ class GradientBoostedTreesLearner(generic_learner.GenericLearner): IN_NODE. - IN_NODE: The features are sorted just before being used in the node. This solution is slow but consumes little amount of memory. . Default: "PRESORT". + sparse_oblique_max_num_projections: For sparse oblique splits i.e. + `split_axis=SPARSE_OBLIQUE`. Maximum number of projections (applied after + the num_projections_exponent). Oblique splits try out + max(p^num_projections_exponent, max_num_projections) random projections + for choosing a split, where p is the number of numerical features. + Increasing "max_num_projections" increases the training time but not the + inference time. In late stage model development, if every bit of accuracy + if important, increase this value. The paper "Sparse Projection Oblique + Random Forests" (Tomita et al, 2020) does not define this hyperparameter. + Default: None. sparse_oblique_normalization: For sparse oblique splits i.e. `split_axis=SPARSE_OBLIQUE`. Normalization applied on the features, before applying the sparse oblique projections. - `NONE`: No normalization. - @@ -1204,12 +1014,28 @@ class GradientBoostedTreesLearner(generic_learner.GenericLearner): max-min) estimated on the entire train dataset. Default: None. sparse_oblique_num_projections_exponent: For sparse oblique splits i.e. `split_axis=SPARSE_OBLIQUE`. Controls of the number of random projections - to test at each node as `num_features^num_projections_exponent`. Default: - None. - sparse_oblique_projection_density_factor: For sparse oblique splits i.e. - `split_axis=SPARSE_OBLIQUE`. Controls of the number of random projections - to test at each node as `num_features^num_projections_exponent`. Default: - None. + to test at each node. Increasing this value very likely improves the + quality of the model, drastically increases the training time, and doe not + impact the inference time. Oblique splits try out + max(p^num_projections_exponent, max_num_projections) random projections + for choosing a split, where p is the number of numerical features. + Therefore, increasing this `num_projections_exponent` and possibly + `max_num_projections` may improve model quality, but will also + significantly increase training time. Note that the complexity of + (classic) Random Forests is roughly proportional to + `num_projections_exponent=0.5`, since it considers sqrt(num_features) for + a split. The complexity of (classic) GBDT is roughly proportional to + `num_projections_exponent=1`, since it considers all features for a split. + The paper "Sparse Projection Oblique Random Forests" (Tomita et al, 2020) + recommends values in [1/4, 2]. Default: None. + sparse_oblique_projection_density_factor: Density of the projections as an + exponent of the number of features. Independently for each projection, + each feature has a probability "projection_density_factor / num_features" + to be considered in the projection. The paper "Sparse Projection Oblique + Random Forests" (Tomita et al, 2020) calls this parameter `lambda` and + recommends values in [1, 5]. Increasing this value increases training and + inference time (on average). This value is best tuned for each dataset. + Default: None. sparse_oblique_weights: For sparse oblique splits i.e. `split_axis=SPARSE_OBLIQUE`. Possible values: - `BINARY`: The oblique weights are sampled in {-1,1} (default). - `CONTINUOUS`: The oblique @@ -1217,9 +1043,11 @@ class GradientBoostedTreesLearner(generic_learner.GenericLearner): split_axis: What structure of split to consider for numerical features. - `AXIS_ALIGNED`: Axis aligned splits (i.e. one condition at a time). This is the "classical" way to train a tree. Default value. - `SPARSE_OBLIQUE`: - Sparse oblique splits (i.e. splits one a small number of features) from - "Sparse Projection Oblique Random Forests", Tomita et al., 2020. Default: - "AXIS_ALIGNED". + Sparse oblique splits (i.e. random splits one a small number of features) + from "Sparse Projection Oblique Random Forests", Tomita et al., 2020. - + `MHLD_OBLIQUE`: Multi-class Hellinger Linear Discriminant splits from + "Classification Based on Multivariate Contrast Patterns", Canete-Sifuentes + et al., 2029 Default: "AXIS_ALIGNED". subsample: Ratio of the dataset (sampling without replacement) used to train individual trees for the random sampling method. If \\"subsample\\" is set and if \\"sampling_method\\" is NOT set or set to \\"NONE\\", then @@ -1330,6 +1158,8 @@ def __init__( max_num_nodes: Optional[int] = None, maximum_model_size_in_memory_in_bytes: Optional[float] = -1.0, maximum_training_duration_seconds: Optional[float] = -1.0, + mhld_oblique_max_num_attributes: Optional[int] = None, + mhld_oblique_sample_attributes: Optional[bool] = None, min_examples: Optional[int] = 5, missing_value_policy: Optional[str] = "GLOBAL_IMPUTATION", num_candidate_attributes: Optional[int] = -1, @@ -1341,6 +1171,7 @@ def __init__( selective_gradient_boosting_ratio: Optional[float] = 0.01, shrinkage: Optional[float] = 0.1, sorting_strategy: Optional[str] = "PRESORT", + sparse_oblique_max_num_projections: Optional[int] = None, sparse_oblique_normalization: Optional[str] = None, sparse_oblique_num_projections_exponent: Optional[float] = None, sparse_oblique_projection_density_factor: Optional[float] = None, @@ -1407,6 +1238,8 @@ def __init__( maximum_model_size_in_memory_in_bytes ), "maximum_training_duration_seconds": maximum_training_duration_seconds, + "mhld_oblique_max_num_attributes": mhld_oblique_max_num_attributes, + "mhld_oblique_sample_attributes": mhld_oblique_sample_attributes, "min_examples": min_examples, "missing_value_policy": missing_value_policy, "num_candidate_attributes": num_candidate_attributes, @@ -1418,6 +1251,9 @@ def __init__( "selective_gradient_boosting_ratio": selective_gradient_boosting_ratio, "shrinkage": shrinkage, "sorting_strategy": sorting_strategy, + "sparse_oblique_max_num_projections": ( + sparse_oblique_max_num_projections + ), "sparse_oblique_normalization": sparse_oblique_normalization, "sparse_oblique_num_projections_exponent": ( sparse_oblique_num_projections_exponent @@ -1472,6 +1308,7 @@ def train( self, ds: dataset.InputDataset, valid: Optional[dataset.InputDataset] = None, + verbose: Optional[Union[int, bool]] = None, ) -> gradient_boosted_trees_model.GradientBoostedTreesModel: """Trains a model on the given dataset. @@ -1502,11 +1339,15 @@ def train( do not need validation dataset. Some learners, such as GradientBoostedTrees, automatically extract a validation dataset from the training dataset if the validation dataset is not provided. + verbose: Verbose level during training. If None, uses the global verbose + level of `ydf.verbose`. Levels are: 0 of False: No logs, 1 or True: + Print a few logs in a notebook; prints all the logs in a terminal. 2: + Prints all the logs on all surfaces. Returns: A trained model. """ - return super().train(ds, valid) + return super().train(ds=ds, valid=valid, verbose=verbose) @classmethod def capabilities(cls) -> abstract_learner_pb2.LearnerCapabilities: @@ -1845,6 +1686,7 @@ def train( self, ds: dataset.InputDataset, valid: Optional[dataset.InputDataset] = None, + verbose: Optional[Union[int, bool]] = None, ) -> gradient_boosted_trees_model.GradientBoostedTreesModel: """Trains a model on the given dataset. @@ -1875,11 +1717,15 @@ def train( do not need validation dataset. Some learners, such as GradientBoostedTrees, automatically extract a validation dataset from the training dataset if the validation dataset is not provided. + verbose: Verbose level during training. If None, uses the global verbose + level of `ydf.verbose`. Levels are: 0 of False: No logs, 1 or True: + Print a few logs in a notebook; prints all the logs in a terminal. 2: + Prints all the logs on all surfaces. Returns: A trained model. """ - return super().train(ds, valid) + return super().train(ds=ds, valid=valid, verbose=verbose) @classmethod def capabilities(cls) -> abstract_learner_pb2.LearnerCapabilities: @@ -2074,6 +1920,18 @@ class CartLearner(generic_learner.GenericLearner): expressed in seconds. Each learning algorithm is free to use this parameter at it sees fit. Enabling maximum training duration makes the model training non-deterministic. Default: -1.0. + mhld_oblique_max_num_attributes: For MHLD oblique splits i.e. + `split_axis=MHLD_OBLIQUE`. Maximum number of attributes in the projection. + Increasing this value increases the training time. Decreasing this value + acts as a regularization. The value should be in [2, + num_numerical_features]. If the value is above the total number of + numerical features, the value is capped automatically. The value 1 is + allowed but results in ordinary (non-oblique) splits. Default: None. + mhld_oblique_sample_attributes: For MHLD oblique splits i.e. + `split_axis=MHLD_OBLIQUE`. If true, applies the attribute sampling + controlled by the "num_candidate_attributes" or + "num_candidate_attributes_ratio" parameters. If false, all the attributes + are tested. Default: None. min_examples: Minimum number of examples in a node. Default: 5. missing_value_policy: Method used to handle missing attribute values. - `GLOBAL_IMPUTATION`: Missing attribute values are imputed, with the mean @@ -2114,6 +1972,16 @@ class CartLearner(generic_learner.GenericLearner): IN_NODE. - IN_NODE: The features are sorted just before being used in the node. This solution is slow but consumes little amount of memory. . Default: "PRESORT". + sparse_oblique_max_num_projections: For sparse oblique splits i.e. + `split_axis=SPARSE_OBLIQUE`. Maximum number of projections (applied after + the num_projections_exponent). Oblique splits try out + max(p^num_projections_exponent, max_num_projections) random projections + for choosing a split, where p is the number of numerical features. + Increasing "max_num_projections" increases the training time but not the + inference time. In late stage model development, if every bit of accuracy + if important, increase this value. The paper "Sparse Projection Oblique + Random Forests" (Tomita et al, 2020) does not define this hyperparameter. + Default: None. sparse_oblique_normalization: For sparse oblique splits i.e. `split_axis=SPARSE_OBLIQUE`. Normalization applied on the features, before applying the sparse oblique projections. - `NONE`: No normalization. - @@ -2123,12 +1991,28 @@ class CartLearner(generic_learner.GenericLearner): max-min) estimated on the entire train dataset. Default: None. sparse_oblique_num_projections_exponent: For sparse oblique splits i.e. `split_axis=SPARSE_OBLIQUE`. Controls of the number of random projections - to test at each node as `num_features^num_projections_exponent`. Default: - None. - sparse_oblique_projection_density_factor: For sparse oblique splits i.e. - `split_axis=SPARSE_OBLIQUE`. Controls of the number of random projections - to test at each node as `num_features^num_projections_exponent`. Default: - None. + to test at each node. Increasing this value very likely improves the + quality of the model, drastically increases the training time, and doe not + impact the inference time. Oblique splits try out + max(p^num_projections_exponent, max_num_projections) random projections + for choosing a split, where p is the number of numerical features. + Therefore, increasing this `num_projections_exponent` and possibly + `max_num_projections` may improve model quality, but will also + significantly increase training time. Note that the complexity of + (classic) Random Forests is roughly proportional to + `num_projections_exponent=0.5`, since it considers sqrt(num_features) for + a split. The complexity of (classic) GBDT is roughly proportional to + `num_projections_exponent=1`, since it considers all features for a split. + The paper "Sparse Projection Oblique Random Forests" (Tomita et al, 2020) + recommends values in [1/4, 2]. Default: None. + sparse_oblique_projection_density_factor: Density of the projections as an + exponent of the number of features. Independently for each projection, + each feature has a probability "projection_density_factor / num_features" + to be considered in the projection. The paper "Sparse Projection Oblique + Random Forests" (Tomita et al, 2020) calls this parameter `lambda` and + recommends values in [1, 5]. Increasing this value increases training and + inference time (on average). This value is best tuned for each dataset. + Default: None. sparse_oblique_weights: For sparse oblique splits i.e. `split_axis=SPARSE_OBLIQUE`. Possible values: - `BINARY`: The oblique weights are sampled in {-1,1} (default). - `CONTINUOUS`: The oblique @@ -2136,9 +2020,11 @@ class CartLearner(generic_learner.GenericLearner): split_axis: What structure of split to consider for numerical features. - `AXIS_ALIGNED`: Axis aligned splits (i.e. one condition at a time). This is the "classical" way to train a tree. Default value. - `SPARSE_OBLIQUE`: - Sparse oblique splits (i.e. splits one a small number of features) from - "Sparse Projection Oblique Random Forests", Tomita et al., 2020. Default: - "AXIS_ALIGNED". + Sparse oblique splits (i.e. random splits one a small number of features) + from "Sparse Projection Oblique Random Forests", Tomita et al., 2020. - + `MHLD_OBLIQUE`: Multi-class Hellinger Linear Discriminant splits from + "Classification Based on Multivariate Contrast Patterns", Canete-Sifuentes + et al., 2029 Default: "AXIS_ALIGNED". uplift_min_examples_in_treatment: For uplift models only. Minimum number of examples per treatment in a node. Default: 5. uplift_split_score: For uplift models only. Splitter score i.e. score @@ -2214,6 +2100,8 @@ def __init__( max_num_nodes: Optional[int] = None, maximum_model_size_in_memory_in_bytes: Optional[float] = -1.0, maximum_training_duration_seconds: Optional[float] = -1.0, + mhld_oblique_max_num_attributes: Optional[int] = None, + mhld_oblique_sample_attributes: Optional[bool] = None, min_examples: Optional[int] = 5, missing_value_policy: Optional[str] = "GLOBAL_IMPUTATION", num_candidate_attributes: Optional[int] = 0, @@ -2221,6 +2109,7 @@ def __init__( pure_serving_model: Optional[bool] = False, random_seed: Optional[int] = 123456, sorting_strategy: Optional[str] = "PRESORT", + sparse_oblique_max_num_projections: Optional[int] = None, sparse_oblique_normalization: Optional[str] = None, sparse_oblique_num_projections_exponent: Optional[float] = None, sparse_oblique_projection_density_factor: Optional[float] = None, @@ -2261,6 +2150,8 @@ def __init__( maximum_model_size_in_memory_in_bytes ), "maximum_training_duration_seconds": maximum_training_duration_seconds, + "mhld_oblique_max_num_attributes": mhld_oblique_max_num_attributes, + "mhld_oblique_sample_attributes": mhld_oblique_sample_attributes, "min_examples": min_examples, "missing_value_policy": missing_value_policy, "num_candidate_attributes": num_candidate_attributes, @@ -2268,6 +2159,9 @@ def __init__( "pure_serving_model": pure_serving_model, "random_seed": random_seed, "sorting_strategy": sorting_strategy, + "sparse_oblique_max_num_projections": ( + sparse_oblique_max_num_projections + ), "sparse_oblique_normalization": sparse_oblique_normalization, "sparse_oblique_num_projections_exponent": ( sparse_oblique_num_projections_exponent @@ -2319,6 +2213,7 @@ def train( self, ds: dataset.InputDataset, valid: Optional[dataset.InputDataset] = None, + verbose: Optional[Union[int, bool]] = None, ) -> random_forest_model.RandomForestModel: """Trains a model on the given dataset. @@ -2349,18 +2244,22 @@ def train( do not need validation dataset. Some learners, such as GradientBoostedTrees, automatically extract a validation dataset from the training dataset if the validation dataset is not provided. + verbose: Verbose level during training. If None, uses the global verbose + level of `ydf.verbose`. Levels are: 0 of False: No logs, 1 or True: + Print a few logs in a notebook; prints all the logs in a terminal. 2: + Prints all the logs on all surfaces. Returns: A trained model. """ - return super().train(ds, valid) + return super().train(ds=ds, valid=valid, verbose=verbose) @classmethod def capabilities(cls) -> abstract_learner_pb2.LearnerCapabilities: return abstract_learner_pb2.LearnerCapabilities( support_max_training_duration=True, resume_training=False, - support_validation_dataset=False, + support_validation_dataset=True, support_partial_cache_dataset_format=False, support_max_model_size_in_memory=False, support_monotonic_constraints=False, diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.cc b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.cc index e748af20..1d4078c4 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.cc +++ b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.cc @@ -618,6 +618,7 @@ class $0(generic_learner.GenericLearner): self, ds: dataset.InputDataset, valid: Optional[dataset.InputDataset] = None, + verbose: Optional[Union[int, bool]] = None, ) -> $7: """Trains a model on the given dataset. @@ -648,11 +649,15 @@ class $0(generic_learner.GenericLearner): do not need validation dataset. Some learners, such as GradientBoostedTrees, automatically extract a validation dataset from the training dataset if the validation dataset is not provided. + verbose: Verbose level during training. If None, uses the global verbose + level of `ydf.verbose`. Levels are: 0 of False: No logs, 1 or True: + Print a few logs in a notebook; prints all the logs in a terminal. 2: + Prints all the logs on all surfaces. Returns: A trained model. """ - return super().train(ds, valid) + return super().train(ds=ds, valid=valid, verbose=verbose) )", /*$0*/ class_name, /*$1*/ learner_key, /*$2*/ fields_documentation, diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_test.cc b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_test.cc index cc58d8f4..687883d6 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_test.cc +++ b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_test.cc @@ -271,6 +271,7 @@ class FakeAlgorithmLearner(generic_learner.GenericLearner): self, ds: dataset.InputDataset, valid: Optional[dataset.InputDataset] = None, + verbose: Optional[Union[int, bool]] = None, ) -> generic_model.GenericModel: """Trains a model on the given dataset. @@ -301,11 +302,15 @@ class FakeAlgorithmLearner(generic_learner.GenericLearner): do not need validation dataset. Some learners, such as GradientBoostedTrees, automatically extract a validation dataset from the training dataset if the validation dataset is not provided. + verbose: Verbose level during training. If None, uses the global verbose + level of `ydf.verbose`. Levels are: 0 of False: No logs, 1 or True: + Print a few logs in a notebook; prints all the logs in a terminal. 2: + Prints all the logs on all surfaces. Returns: A trained model. """ - return super().train(ds, valid) + return super().train(ds=ds, valid=valid, verbose=verbose) @classmethod def capabilities(cls) -> abstract_learner_pb2.LearnerCapabilities: diff --git a/yggdrasil_decision_forests/port/python/ydf/utils/log.py b/yggdrasil_decision_forests/port/python/ydf/utils/log.py index 1b007953..a30f3c93 100644 --- a/yggdrasil_decision_forests/port/python/ydf/utils/log.py +++ b/yggdrasil_decision_forests/port/python/ydf/utils/log.py @@ -32,7 +32,7 @@ import enum import io import sys -from typing import Any, Optional, Set +from typing import Any, Optional, Set, Union from ydf.cc import ydf @@ -59,13 +59,13 @@ class WarningMessage(enum.Enum): _ALREADY_DISPLAYED_WARNING_IDS: Set[WarningMessage] = set() -def verbose(level: int = 2) -> int: +def verbose(level: Union[int, bool] = 2) -> int: """Sets the verbose level of YDF. The verbose levels are: - 0: Print no logs. - 1: Print a few logs in a colab or notebook cell. Print all the logs in the - console. This is the default verbose level. + 0 or False: Print no logs. + 1 or True: Print a few logs in a colab or notebook cell. Print all the logs + in the console. This is the default verbose level. 2: Prints all the logs on all surfaces. Usage example: @@ -85,6 +85,10 @@ def verbose(level: int = 2) -> int: Returns: The previous verbose level. """ + + if isinstance(level, bool): + level = 1 if level else 0 + global _VERBOSE_LEVEL old = _VERBOSE_LEVEL _VERBOSE_LEVEL = level From f7499ec0190b9534a304d57837d5030e41ae78bf Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Thu, 30 May 2024 06:13:49 -0700 Subject: [PATCH 09/30] Add ability to update YDF model with JAX parameters. PiperOrigin-RevId: 638627996 --- .../port/python/ydf/cc/ydf.pyi | 1 + .../port/python/ydf/model/export_jax.py | 59 ++- .../port/python/ydf/model/generic_model.py | 48 ++- .../model/gradient_boosted_trees_model/BUILD | 1 + .../gradient_boosted_trees_model.py | 9 +- .../gradient_boosted_trees_model_test.py | 31 +- .../gradient_boosted_trees_wrapper.cc | 10 + .../gradient_boosted_trees_wrapper.h | 2 + .../port/python/ydf/model/jax_model_test.py | 335 ++++++++++++------ .../port/python/ydf/model/model.cc | 2 + 10 files changed, 376 insertions(+), 122 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/ydf/cc/ydf.pyi b/yggdrasil_decision_forests/port/python/ydf/cc/ydf.pyi index dcf16fbb..b2a2658c 100644 --- a/yggdrasil_decision_forests/port/python/ydf/cc/ydf.pyi +++ b/yggdrasil_decision_forests/port/python/ydf/cc/ydf.pyi @@ -200,6 +200,7 @@ class GradientBoostedTreesCCModel(DecisionForestCCModel): def validation_loss(self) -> float: ... def validation_evaluation(self) -> metric_pb2.EvaluationResults: ... def initial_predictions(self) -> npt.NDArray[float]: ... + def set_initial_predictions(self, values: npt.NDArray[float]): ... def num_trees_per_iter(self) -> int: ... def loss(self) -> gradient_boosted_trees_pb2.Loss: ... diff --git a/yggdrasil_decision_forests/port/python/ydf/model/export_jax.py b/yggdrasil_decision_forests/port/python/ydf/model/export_jax.py index cfb1822d..05a31211 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/export_jax.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/export_jax.py @@ -351,9 +351,9 @@ def stack(features: List[InternalFeatureItem], dtype): ) -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass class BeginNodeIdx: - """Index of the first leaf and non leaf node in a tree.""" + """Index of leaf and non leaf node in a tree.""" leaf_node: int non_leaf_node: int @@ -1052,3 +1052,58 @@ def sum_iter(i, a): new_node_offset_if_non_leaf, # Non-leaf node_offset, # Leaf ) + + +def update_with_jax_params( + model: generic_model.GenericModel, + params: Dict[str, Any], +): + """Updates the model with JAX params as created by `to_jax_function`. + + Args: + model: A YDF model. + params: See "update_with_jax_params" in generic_model.py. + """ + + if not isinstance(model, decision_forest_model.DecisionForestModel): + raise ValueError("The model is not a decision forest") + + if isinstance( + model, gradient_boosted_trees_model.GradientBoostedTreesModel + ) and (initial_predictions := params.get(_PARAM_INITIAL_PREDICTIONS)): + model.set_initial_predictions(initial_predictions) + + leaf_values = params.get(_PARAM_LEAF_VALUES) + + # Only scan the trees if the user updates node parameters. + # Note: Add other node parameters here. + if leaf_values is not None: + cur_node = BeginNodeIdx(leaf_node=0, non_leaf_node=0) + + for tree_idx, tree in enumerate(model.iter_trees()): + _update_node_with_jax_param(tree.root, cur_node, leaf_values) + model.set_tree(tree_idx, tree) + + +def _update_node_with_jax_param( + node: tree_lib.AbstractNode, + cur_node: BeginNodeIdx, + leaf_values: Optional[jax.Array], +): + """Updates recursively the node values.""" + + if node.is_leaf: + assert isinstance(node, tree_lib.Leaf) + # TODO: Add support for other types of leaf nodes. + if not isinstance(node.value, tree_lib.RegressionValue): + raise ValueError( + "The YDF Jax exporter does not support this leaf value:" + f" {node.value!r}" + ) + node.value.value = leaf_values[cur_node.leaf_node] + cur_node.leaf_node += 1 + else: + cur_node.non_leaf_node += 1 + assert isinstance(node, tree_lib.NonLeaf) + _update_node_with_jax_param(node.neg_child, cur_node, leaf_values) + _update_node_with_jax_param(node.pos_child, cur_node, leaf_values) diff --git a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py index 7eee2933..e6bad13f 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py @@ -836,16 +836,16 @@ def to_jax_function( # pytype: disable=name-error import jax.numpy as jnp # Train a model. - model = ydf.RandomForestLearner(label="l").train({ + model = ydf.GradientBoostedTreesLearner(label="l").train({ "f1": np.random.random(size=100), "f2": np.random.random(size=100), "l": np.random.randint(2, size=100), }) # Convert model to a JAX function. - jax_model = model.o_jax_function() + jax_model = model.to_jax_function() - # Make predictions with the TF module. + # Make predictions with the JAX function. jax_predictions = jax_model.predict({ "f1": jnp.array([0, 0.5, 1]), "f2": jnp.array([1, 0, 0.5]), @@ -875,6 +875,48 @@ def to_jax_function( # pytype: disable=name-error leaves_as_params=leaves_as_params, ) + def update_with_jax_params(self, params: Dict[str, Any]): + """Updates the model with JAX params as created by `to_jax_function`. + + Usage example: + + ```python + import ydf + import numpy as np + import jax.numpy as jnp + + # Train a model with YDF + dataset = { + "f1": np.random.random(size=100), + "f2": np.random.random(size=100), + "l": np.random.randint(2, size=100), + } + model = ydf.GradientBoostedTreesLearner(label="l").train(dataset) + + # Convert model to a JAX function with leave values as parameters. + jax_model = model.to_jax_function( + leaves_as_params=True, + apply_activation=True) + # Note: The learnable model parameter are in `jax_model.params`. + + # Finetune the model parameters with your own logic. + jax_model.params = fine_tune_model(jax_model.params, ...) + + # Update the YDF model with the finetuned parameters + model.update_with_jax_params(jax_model.params) + + # Make predictions with the finetuned YDF model + predictions = model.predict(dataset) + + # Save the YDF model + model.save("/tmp/my_ydf_model") + ``` + + Args: + params: Learnable parameter of the model generated with `to_jax_function`. + """ + _get_export_jax().update_with_jax_params(model=self, params=params) + def hyperparameter_optimizer_logs( self, ) -> Optional[optimizer_logs.OptimizerLogs]: diff --git a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/BUILD b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/BUILD index 9eacedb4..3d07e6f6 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/BUILD @@ -53,6 +53,7 @@ py_test( ":gradient_boosted_trees_model", # absl/logging dep, # absl/testing:absltest dep, + # absl/testing:parameterized dep, # numpy dep, # pandas dep, "//ydf/dataset:dataspec", diff --git a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model.py b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model.py index 8812f8a1..f375806b 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model.py @@ -15,8 +15,9 @@ """Definitions for Gradient Boosted Trees models.""" import math -from typing import Optional +from typing import Optional, Sequence +import numpy as np import numpy.typing as npt from yggdrasil_decision_forests.metric import metric_pb2 @@ -41,6 +42,12 @@ def initial_predictions(self) -> npt.NDArray[float]: """Returns the model's initial predictions (i.e. the model bias).""" return self._model.initial_predictions() + def set_initial_predictions(self, initial_predictions: Sequence[float]): + """Sets the model's initial predictions (i.e. the model bias).""" + return self._model.set_initial_predictions( + np.asarray(initial_predictions, np.float32) + ) + def validation_evaluation(self) -> Optional[metric.Evaluation]: """Returns the validation evaluation of the model, if available. diff --git a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model_test.py b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model_test.py index 2c75fdad..c87eeefc 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model_test.py @@ -19,6 +19,7 @@ from absl import logging from absl.testing import absltest +from absl.testing import parameterized import numpy as np import numpy.testing as npt import pandas as pd @@ -42,18 +43,19 @@ Tree = tree_lib.Tree -class GradientBoostedTreesTest(absltest.TestCase): +def load_model( + name: str, + directory: str = "model", +) -> gradient_boosted_trees_model.GradientBoostedTreesModel: + path = os.path.join(test_utils.ydf_test_data_path(), directory, name) + return model_lib.load_model(path) + + +class GradientBoostedTreesTest(parameterized.TestCase): def setUp(self): super().setUp() - def load_model( - name: str, - directory: str = "model", - ) -> gradient_boosted_trees_model.GradientBoostedTreesModel: - path = os.path.join(test_utils.ydf_test_data_path(), directory, name) - return model_lib.load_model(path) - # This model is a classification model for pure serving. self.adult_binary_class_gbdt = load_model("adult_binary_class_gbdt") @@ -140,6 +142,19 @@ def test_initial_predictions(self): initial_predictions = self.adult_binary_class_gbdt.initial_predictions() np.testing.assert_allclose(initial_predictions, [-1.1630996]) + @parameterized.parameters( + "adult_binary_class_gbdt", + "iris_multi_class_gbdt", + "abalone_regression_gbdt", + ) + def test_set_initial_predictions(self, model_name): + model = load_model(model_name) + initial_predictions = model.initial_predictions() + model.set_initial_predictions(initial_predictions * 2.0) + np.testing.assert_allclose( + initial_predictions * 2, model.initial_predictions() + ) + def test_validation_evaluation_empty(self): dataset = { "x1": np.array([0, 0, 0, 1, 1, 1]), diff --git a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_wrapper.cc b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_wrapper.cc index c10a4e23..d5091f00 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_wrapper.cc +++ b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_wrapper.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -56,4 +57,13 @@ py::array_t GradientBoostedTreesCCModel::initial_predictions() const { return initial_predictions; } +void GradientBoostedTreesCCModel::set_initial_predictions( + const py::array_t& values) { + std::vector std_values(values.size(), 0.0f); + for (int i = 0; i < values.size(); i++) { + std_values[i] = values.at(i); + } + gbt_model_->set_initial_predictions(std::move(std_values)); +} + } // namespace yggdrasil_decision_forests::port::python diff --git a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_wrapper.h b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_wrapper.h index 5edc4fb4..e7defd48 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_wrapper.h +++ b/yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_wrapper.h @@ -63,6 +63,8 @@ class GradientBoostedTreesCCModel : public DecisionForestCCModel { py::array_t initial_predictions() const; + void set_initial_predictions(const py::array_t& values); + ::yggdrasil_decision_forests::model::gradient_boosted_trees::proto::Loss loss() const { return gbt_model_->loss(); diff --git a/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py b/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py index 9939e326..b42c5b0e 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/jax_model_test.py @@ -188,6 +188,116 @@ def plot_ellipse_predictions( fig.savefig(path) +def check_toy_model(test_self, model): + test_self.assertEqual( + model.get_tree(0).pretty(model.data_spec()), + """\ +'f1' >= 2 [score=0 missing=False] + ├─(pos)─ 'c1' in ['x', 'y'] [score=0 missing=False] + │ ├─(pos)─ 'c1' in ['x'] [score=0 missing=False] + │ │ ├─(pos)─ value=1 + │ │ └─(neg)─ value=2 + │ └─(neg)─ value=3 + └─(neg)─ 'f1' >= 1 [score=0 missing=False] + ├─(pos)─ value=4 + └─(neg)─ value=5 +""", + ) + + test_self.assertEqual( + model.get_tree(1).pretty(model.data_spec()), + """\ +'f2' >= 1.5 [score=0 missing=False] + ├─(pos)─ value=6 + └─(neg)─ value=7 +""", + ) + + +def create_toy_model(test_self): + columns = ["f1", "c1", "f2", "label_regress"] + model = specialized_learners.GradientBoostedTreesLearner( + label="label_regress", + task=generic_learner.Task.REGRESSION, + num_trees=1, + ).train(create_dataset(columns)) + model.set_initial_predictions([0]) + model.remove_tree(0) + + # pylint: disable=invalid-name + RegressionValue = tree_lib.RegressionValue + Leaf = tree_lib.Leaf + NonLeaf = tree_lib.NonLeaf + NumericalHigherThanCondition = tree_lib.NumericalHigherThanCondition + CategoricalIsInCondition = tree_lib.CategoricalIsInCondition + Tree = tree_lib.Tree + # pylint: enable=invalid-name + + model.add_tree( + Tree( + root=NonLeaf( + condition=NumericalHigherThanCondition( + missing=False, score=0.0, attribute=1, threshold=2.0 + ), + pos_child=NonLeaf( + condition=CategoricalIsInCondition( + missing=False, + score=0.0, + attribute=2, + mask=[1, 2], + ), + pos_child=NonLeaf( + condition=CategoricalIsInCondition( + missing=False, + score=0.0, + attribute=2, + mask=[1], + ), + pos_child=Leaf( + value=RegressionValue(num_examples=0.0, value=1.0) + ), + neg_child=Leaf( + value=RegressionValue(num_examples=0.0, value=2.0) + ), + ), + neg_child=Leaf( + value=RegressionValue(num_examples=0.0, value=3.0) + ), + ), + neg_child=NonLeaf( + condition=NumericalHigherThanCondition( + missing=False, score=0.0, attribute=1, threshold=1.0 + ), + pos_child=Leaf( + value=RegressionValue(num_examples=0.0, value=4.0) + ), + neg_child=Leaf( + value=RegressionValue(num_examples=0.0, value=5.0) + ), + ), + ) + ) + ) + + model.add_tree( + Tree( + root=NonLeaf( + condition=NumericalHigherThanCondition( + missing=False, score=0.0, attribute=3, threshold=1.5 + ), + pos_child=Leaf( + value=RegressionValue(num_examples=0.0, value=6.0) + ), + neg_child=Leaf( + value=RegressionValue(num_examples=0.0, value=7.0) + ), + ) + ) + ) + check_toy_model(test_self, model) + return model + + class JaxModelTest(parameterized.TestCase): @parameterized.parameters( @@ -561,109 +671,7 @@ def test_categorical_list_to_bitmap_invalid(self): ) def test_internal_forest_on_manual(self): - columns = ["f1", "c1", "f2", "label_regress"] - model = specialized_learners.RandomForestLearner( - label="label_regress", - task=generic_learner.Task.REGRESSION, - num_trees=1, - ).train(create_dataset(columns)) - model.remove_tree(0) - - # pylint: disable=invalid-name - RegressionValue = tree_lib.RegressionValue - Leaf = tree_lib.Leaf - NonLeaf = tree_lib.NonLeaf - NumericalHigherThanCondition = tree_lib.NumericalHigherThanCondition - CategoricalIsInCondition = tree_lib.CategoricalIsInCondition - Tree = tree_lib.Tree - # pylint: enable=invalid-name - - model.add_tree( - Tree( - root=NonLeaf( - condition=NumericalHigherThanCondition( - missing=False, score=0.0, attribute=1, threshold=2.0 - ), - pos_child=NonLeaf( - condition=CategoricalIsInCondition( - missing=False, - score=0.0, - attribute=2, - mask=[1, 2], - ), - pos_child=NonLeaf( - condition=CategoricalIsInCondition( - missing=False, - score=0.0, - attribute=2, - mask=[1], - ), - pos_child=Leaf( - value=RegressionValue(num_examples=0.0, value=1.0) - ), - neg_child=Leaf( - value=RegressionValue(num_examples=0.0, value=2.0) - ), - ), - neg_child=Leaf( - value=RegressionValue(num_examples=0.0, value=3.0) - ), - ), - neg_child=NonLeaf( - condition=NumericalHigherThanCondition( - missing=False, score=0.0, attribute=1, threshold=1.0 - ), - pos_child=Leaf( - value=RegressionValue(num_examples=0.0, value=4.0) - ), - neg_child=Leaf( - value=RegressionValue(num_examples=0.0, value=5.0) - ), - ), - ) - ) - ) - - model.add_tree( - Tree( - root=NonLeaf( - condition=NumericalHigherThanCondition( - missing=False, score=0.0, attribute=3, threshold=1.5 - ), - pos_child=Leaf( - value=RegressionValue(num_examples=0.0, value=6.0) - ), - neg_child=Leaf( - value=RegressionValue(num_examples=0.0, value=7.0) - ), - ) - ) - ) - - self.assertEqual( - model.get_tree(0).pretty(model.data_spec()), - """\ -'f1' >= 2 [score=0 missing=False] - ├─(pos)─ 'c1' in ['x', 'y'] [score=0 missing=False] - │ ├─(pos)─ 'c1' in ['x'] [score=0 missing=False] - │ │ ├─(pos)─ value=1 - │ │ └─(neg)─ value=2 - │ └─(neg)─ value=3 - └─(neg)─ 'f1' >= 1 [score=0 missing=False] - ├─(pos)─ value=4 - └─(neg)─ value=5 -""", - ) - - self.assertEqual( - model.get_tree(1).pretty(model.data_spec()), - """\ -'f2' >= 1.5 [score=0 missing=False] - ├─(pos)─ value=6 - └─(neg)─ value=7 -""", - ) - + model = create_toy_model(self) internal_forest = to_jax.InternalForest(model) self.assertEqual( @@ -934,7 +942,7 @@ def prediction_fn(features: Dict[str, jax.Array]) -> jax.Array: ) @parameterized.named_parameters( - ("leaves_ellipse", "ellipse", True, False), + ("leaves_ellipse_axis_aligned", "ellipse", True, False), # ("leaves_mnist", "mnist", True , False), # Skip in sandboxed test ("leaves_ellipse_oblique", "ellipse", True, True), ) @@ -993,20 +1001,24 @@ def to_jax_array(d): jax_test_ds = to_jax_array(test_ds) jax_finetune_ds = to_jax_array(finetune_ds) + @jax.jit + def compute_predictions(state, batch): + batch = batch.copy() + batch.pop(label) + return jax_model.predict(batch, state) + @jax.jit def compute_accuracy(state, batch): batch = batch.copy() labels = batch.pop(label) - features = batch - predictions = jax_model.predict(features, state) + predictions = jax_model.predict(batch, state) return jnp.mean((predictions >= 0.0) == labels) @jax.jit def compute_loss(state, batch): batch = batch.copy() labels = batch.pop(label) - features = batch - logits = jax_model.predict(features, state) + logits = jax_model.predict(batch, state) loss = optax.sigmoid_binary_cross_entropy(logits, labels).mean() return loss @@ -1068,6 +1080,113 @@ def train_step(opt_state, mdl_state, batch): post_tuned_jax_test_accuracy, pre_tuned_jax_test_accuracy + 0.01 ) + # Update the YDF model with the finetuned parameters + model.update_with_jax_params(mdl_state) + + # Check the weights have been updated + new_jax_model = to_jax.to_jax_function( + model, leaves_as_params=leaves_as_params + ) + np.testing.assert_allclose( + mdl_state["initial_predictions"], + new_jax_model.params["initial_predictions"], + rtol=1e-5, + atol=1e-5, + ) + np.testing.assert_allclose( + mdl_state["leaf_values"], + new_jax_model.params["leaf_values"], + rtol=1e-5, + atol=1e-5, + ) + + # Check the predictions of the updated YDF model + finetuned_ydf_predictions = model.predict( + test_ds, + ) + finetuned_jax_predictions = compute_predictions(mdl_state, jax_test_ds) + np.testing.assert_allclose( + jax.nn.sigmoid(finetuned_jax_predictions), + finetuned_ydf_predictions, + rtol=1e-5, + atol=1e-5, + ) + + def test_update_with_jax_params_manual(self): + model = create_toy_model(self) + check_toy_model(self, model) + + jax_model = to_jax.to_jax_function(model, leaves_as_params=True) + to_jax.update_with_jax_params(model, jax_model.params) + + # Nothing have changed yet + check_toy_model(self, model) + + np.testing.assert_allclose( + jax_model.params["leaf_values"], + [5.0, 4.0, 3.0, 2.0, 1.0, 7.0, 6.0], + rtol=1e-5, + atol=1e-5, + ) + + np.testing.assert_allclose( + jax_model.params["initial_predictions"], + [0.0], + rtol=1e-5, + atol=1e-5, + ) + + jax_model.params["leaf_values"] = jnp.asarray( + [1.0, 2.0, 3.0, 4.0, 5, 6.0, 7.0], jnp.float32 + ) + jax_model.params["initial_predictions"] = jnp.asarray([1.0], jnp.float32) + to_jax.update_with_jax_params(model, jax_model.params) + + np.testing.assert_allclose( + model.initial_predictions(), + jax_model.params["initial_predictions"], + rtol=1e-5, + atol=1e-5, + ) + + new_jax_model = to_jax.to_jax_function(model, leaves_as_params=True) + np.testing.assert_allclose( + jax_model.params["initial_predictions"], + new_jax_model.params["initial_predictions"], + rtol=1e-5, + atol=1e-5, + ) + np.testing.assert_allclose( + jax_model.params["leaf_values"], + new_jax_model.params["leaf_values"], + rtol=1e-5, + atol=1e-5, + ) + + self.assertEqual( + model.get_tree(0).pretty(model.data_spec()), + """\ +'f1' >= 2 [score=0 missing=False] + ├─(pos)─ 'c1' in ['x', 'y'] [score=0 missing=False] + │ ├─(pos)─ 'c1' in ['x'] [score=0 missing=False] + │ │ ├─(pos)─ value=5 + │ │ └─(neg)─ value=4 + │ └─(neg)─ value=3 + └─(neg)─ 'f1' >= 1 [score=0 missing=False] + ├─(pos)─ value=2 + └─(neg)─ value=1 +""", + ) + + self.assertEqual( + model.get_tree(1).pretty(model.data_spec()), + """\ +'f2' >= 1.5 [score=0 missing=False] + ├─(pos)─ value=7 + └─(neg)─ value=6 +""", + ) + if __name__ == "__main__": if sys.version_info < (3, 9): diff --git a/yggdrasil_decision_forests/port/python/ydf/model/model.cc b/yggdrasil_decision_forests/port/python/ydf/model/model.cc index 2ddf4334..20406838 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/model.cc +++ b/yggdrasil_decision_forests/port/python/ydf/model/model.cc @@ -209,6 +209,8 @@ void init_model(py::module_& m) { .def("validation_loss", &GradientBoostedTreesCCModel::validation_loss) .def("initial_predictions", &GradientBoostedTreesCCModel::initial_predictions) + .def("set_initial_predictions", + &GradientBoostedTreesCCModel::set_initial_predictions) .def("validation_evaluation", &GradientBoostedTreesCCModel::validation_evaluation) .def("loss", &GradientBoostedTreesCCModel::loss) From 83bead872281fd328e4355498419b9e67263bf94 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Thu, 30 May 2024 07:25:46 -0700 Subject: [PATCH 10/30] No public description PiperOrigin-RevId: 638644757 --- yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py index 8960f095..138ab6ff 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py @@ -491,7 +491,7 @@ def dataspec_to_normalized_columns( for column_spec in columns: if column_spec.name not in data and column_spec.name in required_columns: raise ValueError( - f"The data spec expects columns {column_spec.name} which was not" + f"The data spec expects columns {column_spec.name!r} which was not" f" found in the data. Available columns: {list(data)}. Required" f" columns: {required_columns}" ) From c92df409d516f25720eec92d700819a06986fea9 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Thu, 30 May 2024 07:44:42 -0700 Subject: [PATCH 11/30] Divide "TrainWithStatus" into "TrainWithStatus" and "TrainWithStatusImpl". PiperOrigin-RevId: 638649524 --- yggdrasil_decision_forests/learner/BUILD | 3 + .../learner/abstract_learner.cc | 73 ++++++++++++++++++- .../learner/abstract_learner.h | 43 +++++++++-- .../learner/abstract_learner_test.cc | 4 +- yggdrasil_decision_forests/learner/cart/BUILD | 1 - .../learner/cart/cart.cc | 14 +--- .../learner/cart/cart.h | 4 +- .../distributed_gradient_boosted_trees/BUILD | 1 - .../distributed_gradient_boosted_trees.cc | 19 +---- .../distributed_gradient_boosted_trees.h | 6 +- .../learner/export_doc_test.cc | 10 ++- .../learner/gradient_boosted_trees/BUILD | 1 - .../gradient_boosted_trees.cc | 34 +-------- .../gradient_boosted_trees.h | 8 +- .../learner/hyperparameters_optimizer/BUILD | 1 - .../hyperparameters_optimizer.cc | 23 +----- .../hyperparameters_optimizer.h | 8 +- .../learner/multitasker/BUILD | 1 - .../learner/multitasker/multitasker.cc | 3 +- .../learner/multitasker/multitasker.h | 4 +- .../learner/random_forest/BUILD | 1 - .../learner/random_forest/random_forest.cc | 15 +--- .../learner/random_forest/random_forest.h | 4 +- .../ydf/learner/wrapper/wrapper_test.cc | 2 +- yggdrasil_decision_forests/utils/usage.h | 2 - .../utils/usage_default.cc | 2 - 26 files changed, 150 insertions(+), 137 deletions(-) diff --git a/yggdrasil_decision_forests/learner/BUILD b/yggdrasil_decision_forests/learner/BUILD index 2a788220..1d25f795 100644 --- a/yggdrasil_decision_forests/learner/BUILD +++ b/yggdrasil_decision_forests/learner/BUILD @@ -75,6 +75,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:status_macros", "//yggdrasil_decision_forests/utils:synchronization_primitives", "//yggdrasil_decision_forests/utils:uid", + "//yggdrasil_decision_forests/utils:usage", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -180,8 +181,10 @@ cc_test( ":abstract_learner", ":abstract_learner_cc_proto", ":export_doc", + "//yggdrasil_decision_forests/model:abstract_model", "//yggdrasil_decision_forests/utils:filesystem", "//yggdrasil_decision_forests/utils:test", + "@com_google_absl//absl/status:statusor", "@com_google_googletest//:gtest_main", ], ) diff --git a/yggdrasil_decision_forests/learner/abstract_learner.cc b/yggdrasil_decision_forests/learner/abstract_learner.cc index bbc9ad12..e33e8ebb 100644 --- a/yggdrasil_decision_forests/learner/abstract_learner.cc +++ b/yggdrasil_decision_forests/learner/abstract_learner.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -32,7 +33,8 @@ #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" -#include "absl/time/time.h" +#include "absl/time/clock.h" +#include "absl/types/optional.h" #include "yggdrasil_decision_forests/dataset/data_spec.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" #include "yggdrasil_decision_forests/dataset/types.h" @@ -53,6 +55,7 @@ #include "yggdrasil_decision_forests/utils/status_macros.h" #include "yggdrasil_decision_forests/utils/synchronization_primitives.h" #include "yggdrasil_decision_forests/utils/uid.h" +#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace model { @@ -271,21 +274,85 @@ absl::Status AbstractLearner::LinkTrainingConfig( return absl::OkStatus(); } +// Non status; dataset in memory. std::unique_ptr AbstractLearner::Train( const dataset::VerticalDataset& train_dataset) const { return TrainWithStatus(train_dataset).value(); } +// Non status; dataset on disk. std::unique_ptr AbstractLearner::Train( const absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec) const { return TrainWithStatus(typed_path, data_spec).value(); } +// API; dataset in memory. absl::StatusOr> AbstractLearner::TrainWithStatus( - const absl::string_view typed_path, + const dataset::VerticalDataset& train_dataset, + absl::optional> + valid_dataset) const { + utils::usage::OnTrainingStart(train_dataset.data_spec(), training_config(), + train_dataset.nrow()); + const auto begin_training = absl::Now(); + + ASSIGN_OR_RETURN(auto model, + TrainWithStatusImpl(train_dataset, valid_dataset)); + + utils::usage::OnTrainingEnd(train_dataset.data_spec(), training_config(), + train_dataset.nrow(), *model, + absl::Now() - begin_training); + + if (training_config().pure_serving_model()) { + RETURN_IF_ERROR(model->MakePureServing()); + } + return model; +} + +// Impl; dataset in memory. +absl::StatusOr> +AbstractLearner::TrainWithStatusImpl( + const dataset::VerticalDataset& train_dataset, + absl::optional> + valid_dataset) const { + // This method should always be implemented by learners. + return absl::UnimplementedError( + "The learner does not implement TrainWithStatusImpl (recommended) " + "TrainWithStatus and " + "TrainWithStatusImpl (deprecated)."); +} + +// API; dataset on disk. +absl::StatusOr> AbstractLearner::TrainWithStatus( + absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec, const absl::optional& typed_valid_path) const { + utils::usage::OnTrainingStart(data_spec, training_config(), + /*num_examples=*/-1); + const auto begin_training = absl::Now(); + + ASSIGN_OR_RETURN( + auto model, TrainWithStatusImpl(typed_path, data_spec, typed_valid_path)); + + utils::usage::OnTrainingEnd(data_spec, training_config(), + /*num_examples=*/-1, *model, + absl::Now() - begin_training); + + if (training_config().pure_serving_model()) { + RETURN_IF_ERROR(model->MakePureServing()); + } + return model; +} + +// Impl; dataset on disk. +absl::StatusOr> +AbstractLearner::TrainWithStatusImpl( + absl::string_view typed_path, + const dataset::proto::DataSpecification& data_spec, + const absl::optional& typed_valid_path) const { + // If training on disk is not implemented, we load the dataset and use + // training from memory. + // List the columns used for the training. // Only these columns will be loaded. proto::TrainingConfigLinking link_config; @@ -310,7 +377,7 @@ absl::StatusOr> AbstractLearner::TrainWithStatus( /*required_columns=*/{}, dataset_loading_config)); valid_dataset = valid_dataset_data; } - return TrainWithStatus(train_dataset, valid_dataset); + return TrainWithStatusImpl(train_dataset, valid_dataset); } absl::Status CheckGenericHyperParameterSpecification( diff --git a/yggdrasil_decision_forests/learner/abstract_learner.h b/yggdrasil_decision_forests/learner/abstract_learner.h index 15e907f4..e33a515f 100644 --- a/yggdrasil_decision_forests/learner/abstract_learner.h +++ b/yggdrasil_decision_forests/learner/abstract_learner.h @@ -18,6 +18,7 @@ #ifndef YGGDRASIL_DECISION_FORESTS_LEARNER_ABSTRACT_LEARNER_H_ #define YGGDRASIL_DECISION_FORESTS_LEARNER_ABSTRACT_LEARNER_H_ +#include #include #include @@ -46,6 +47,9 @@ class AbstractLearner { // Trains a model using the dataset stored on disk at the path "typed_path". // + // A learner might use distributed training, or load the dataset in memory and + // fallback to in-memory training. + // // A typed path is a dataset with a format prefix. prefix format. For example, // "csv:/tmp/dataset.csv". The path supports sharding, globbing and comma // separation. See the "Dataset path and format" section of the user manual @@ -56,8 +60,11 @@ class AbstractLearner { // for validation. If "typed_valid_path" is not provided, a validation dataset // will be extracted from the training dataset. If the algorithm does not have // the "use_validation_dataset" capability, "typed_valid_path" is ignored. + // + // This method is virtual for historical reasons with external codebase. + // Internally or in any new code, this method should not be overridden. virtual absl::StatusOr> TrainWithStatus( - const absl::string_view typed_path, + absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec, const absl::optional& typed_valid_path = {}) const; @@ -68,19 +75,22 @@ class AbstractLearner { // for validation. If "valid_dataset" is not provided, a validation dataset // will be extracted from the training dataset. If the algorithm does not have // the "use_validation_dataset" capability, "valid_dataset" is ignored. + // + // This method is virtual for historical reasons with external codebase. + // Internally or in any new code, this method should not be overridden. virtual absl::StatusOr> TrainWithStatus( const dataset::VerticalDataset& train_dataset, absl::optional> - valid_dataset = {}) const = 0; + valid_dataset = {}) const; - // Similar as TrainWithStatus, but crash in case of error. - // [Deprecated] + // [Deprecated] Similar as TrainWithStatus, but fails (CHECK) in case of + // error. virtual std::unique_ptr Train( - const absl::string_view typed_path, + absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec) const; - // Trains and returns a model from a training dataset stored on drive. - // [Deprecated] + // [Deprecated] Similar as TrainWithStatus, but fails (CHECK) in case of + // error. virtual std::unique_ptr Train( const dataset::VerticalDataset& train_dataset) const; @@ -164,6 +174,25 @@ class AbstractLearner { stop_training_trigger_ = trigger; } + // Implementation of the "TrainWithStatus" function. Callers should call + // "TrainWithStatus". Learners should implement "TrainWithStatusImpl" (either + // both versions, to support both distributed and in-memory training) or only + // the in-memory version below. + virtual absl::StatusOr> TrainWithStatusImpl( + absl::string_view typed_path, + const dataset::proto::DataSpecification& data_spec, + const absl::optional& typed_valid_path) const; + + // Implementation of the "TrainWithStatus" function. Callers should call + // "TrainWithStatus". Learners should implement this "TrainWithStatusImpl" + // function. + // + // This method is not a pure virtual function for historical reasons. + virtual absl::StatusOr> TrainWithStatusImpl( + const dataset::VerticalDataset& train_dataset, + absl::optional> + valid_dataset) const; + protected: // Training configuration. Contains the hyper parameters of the learner. proto::TrainingConfig training_config_; diff --git a/yggdrasil_decision_forests/learner/abstract_learner_test.cc b/yggdrasil_decision_forests/learner/abstract_learner_test.cc index a73b4a46..a01fc566 100644 --- a/yggdrasil_decision_forests/learner/abstract_learner_test.cc +++ b/yggdrasil_decision_forests/learner/abstract_learner_test.cc @@ -212,7 +212,7 @@ TEST(AbstractLearner, EvaluateLearner) { explicit FakeLearner(const proto::TrainingConfig& training_config) : AbstractLearner(training_config) {} - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset = {}) const override { @@ -274,7 +274,7 @@ TEST(AbstractLearner, MaximumModelSizeInMemoryInBytes) { explicit FakeLearner(const proto::TrainingConfig& training_config) : AbstractLearner(training_config) {} - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset = {}) const override { diff --git a/yggdrasil_decision_forests/learner/cart/BUILD b/yggdrasil_decision_forests/learner/cart/BUILD index a02ab53e..90d89b53 100644 --- a/yggdrasil_decision_forests/learner/cart/BUILD +++ b/yggdrasil_decision_forests/learner/cart/BUILD @@ -34,7 +34,6 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:logging", "//yggdrasil_decision_forests/utils:random", "//yggdrasil_decision_forests/utils:status_macros", - "//yggdrasil_decision_forests/utils:usage", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", diff --git a/yggdrasil_decision_forests/learner/cart/cart.cc b/yggdrasil_decision_forests/learner/cart/cart.cc index 31af1801..8bf17378 100644 --- a/yggdrasil_decision_forests/learner/cart/cart.cc +++ b/yggdrasil_decision_forests/learner/cart/cart.cc @@ -47,7 +47,6 @@ #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/random.h" #include "yggdrasil_decision_forests/utils/status_macros.h" -#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace model { @@ -136,7 +135,7 @@ CartLearner::GetGenericHyperParameterSpecification() const { return hparam_def; } -absl::StatusOr> CartLearner::TrainWithStatus( +absl::StatusOr> CartLearner::TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset) const { @@ -177,8 +176,6 @@ absl::StatusOr> CartLearner::TrainWithStatus( YDF_LOG(INFO) << "Training CART on " << train_dataset.nrow() << " example(s) and " << config_link.features().size() << " feature(s)."; - utils::usage::OnTrainingStart(train_dataset.data_spec(), config, config_link, - train_dataset.nrow()); std::vector weights; RETURN_IF_ERROR(dataset::GetWeights(train_dataset, config_link, &weights)); @@ -271,20 +268,11 @@ absl::StatusOr> CartLearner::TrainWithStatus( mdl->mutable_out_of_bag_evaluations()->push_back(oob_evaluation); } - utils::usage::OnTrainingEnd(train_dataset.data_spec(), config, config_link, - train_dataset.nrow(), *mdl, - absl::Now() - begin_training); - // Cache the structural variable importance in the model data. RETURN_IF_ERROR(mdl->PrecomputeVariableImportances( mdl->AvailableStructuralVariableImportances())); decision_tree::SetLeafIndices(mdl->mutable_decision_trees()); - - if (config.pure_serving_model()) { - RETURN_IF_ERROR(mdl->MakePureServing()); - } - return std::move(mdl); } diff --git a/yggdrasil_decision_forests/learner/cart/cart.h b/yggdrasil_decision_forests/learner/cart/cart.h index bd27ff4d..d6da448b 100644 --- a/yggdrasil_decision_forests/learner/cart/cart.h +++ b/yggdrasil_decision_forests/learner/cart/cart.h @@ -49,10 +49,10 @@ class CartLearner : public AbstractLearner { // Generic hyper parameter names. static constexpr char kHParamValidationRatio[] = "validation_ratio"; - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> - valid_dataset = {}) const override; + valid_dataset) const override; // Sets the hyper-parameters of the learning algorithm from "generic hparams". absl::Status SetHyperParametersImpl( diff --git a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/BUILD b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/BUILD index 30f373c4..86812a46 100644 --- a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/BUILD +++ b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/BUILD @@ -37,7 +37,6 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:snapshot", "//yggdrasil_decision_forests/utils:status_macros", "//yggdrasil_decision_forests/utils:uid", - "//yggdrasil_decision_forests/utils:usage", "//yggdrasil_decision_forests/utils/distribute:core", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", diff --git a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.cc b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.cc index 8e651fb9..2794aa4f 100644 --- a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.cc +++ b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.cc @@ -32,7 +32,6 @@ #include "yggdrasil_decision_forests/utils/snapshot.h" #include "yggdrasil_decision_forests/utils/status_macros.h" #include "yggdrasil_decision_forests/utils/uid.h" -#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace model { @@ -54,7 +53,7 @@ DistributedGradientBoostedTreesLearner::Capabilities() const { } absl::StatusOr> -DistributedGradientBoostedTreesLearner::TrainWithStatus( +DistributedGradientBoostedTreesLearner::TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset) const { @@ -199,12 +198,10 @@ DistributedGradientBoostedTreesLearner::GetGenericHyperParameterSpecification() } absl::StatusOr> -DistributedGradientBoostedTreesLearner::TrainWithStatus( +DistributedGradientBoostedTreesLearner::TrainWithStatusImpl( const absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec, const absl::optional& typed_valid_path) const { - const auto begin_training = absl::Now(); - internal::Monitoring monitoring; // Extract and check the configuration. @@ -218,9 +215,6 @@ DistributedGradientBoostedTreesLearner::TrainWithStatus( data_spec, &spe_config)); RETURN_IF_ERROR(internal::CheckConfiguration(deployment_)); - utils::usage::OnTrainingStart(data_spec, config, config_link, - /*num_examples=*/-1); - // Working directory. auto work_directory = deployment().cache_path(); if (!deployment().try_resume_training()) { @@ -269,15 +263,6 @@ DistributedGradientBoostedTreesLearner::TrainWithStatus( updated_deployment, config, config_link, spe_config, dataset_cache_path, typed_valid_path, work_directory, data_spec, log_directory(), &monitoring)); - - if (config.pure_serving_model()) { - RETURN_IF_ERROR(model->MakePureServing()); - } - - utils::usage::OnTrainingEnd(data_spec, config, config_link, - /*num_examples=*/-1, *model, - absl::Now() - begin_training); - return std::move(model); } diff --git a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.h b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.h index 0aa808e9..d3b263e6 100644 --- a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.h +++ b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.h @@ -77,12 +77,12 @@ class DistributedGradientBoostedTreesLearner : public AbstractLearner { static constexpr char kHParamForceNumericalDiscretization[] = "force_numerical_discretization"; - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> - valid_dataset = {}) const override; + valid_dataset) const override; - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec, const absl::optional& typed_valid_path) const override; diff --git a/yggdrasil_decision_forests/learner/export_doc_test.cc b/yggdrasil_decision_forests/learner/export_doc_test.cc index b7f030b5..61e9a395 100644 --- a/yggdrasil_decision_forests/learner/export_doc_test.cc +++ b/yggdrasil_decision_forests/learner/export_doc_test.cc @@ -15,10 +15,14 @@ #include "yggdrasil_decision_forests/learner/export_doc.h" +#include + #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/status/statusor.h" #include "yggdrasil_decision_forests/learner/abstract_learner.h" #include "yggdrasil_decision_forests/learner/abstract_learner.pb.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" #include "yggdrasil_decision_forests/utils/filesystem.h" #include "yggdrasil_decision_forests/utils/test.h" @@ -31,7 +35,7 @@ class FakeLearner1 : public AbstractLearner { explicit FakeLearner1(const proto::TrainingConfig& training_config) : AbstractLearner(training_config) {} - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset = {}) const override { @@ -58,7 +62,7 @@ class FakeLearner2 : public AbstractLearner { explicit FakeLearner2(const proto::TrainingConfig& training_config) : AbstractLearner(training_config) {} - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset = {}) const override { @@ -73,7 +77,7 @@ class FakeLearner3 : public AbstractLearner { explicit FakeLearner3(const proto::TrainingConfig& training_config) : AbstractLearner(training_config) {} - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset = {}) const override { diff --git a/yggdrasil_decision_forests/learner/gradient_boosted_trees/BUILD b/yggdrasil_decision_forests/learner/gradient_boosted_trees/BUILD index 15ea2d29..180b4fad 100644 --- a/yggdrasil_decision_forests/learner/gradient_boosted_trees/BUILD +++ b/yggdrasil_decision_forests/learner/gradient_boosted_trees/BUILD @@ -56,7 +56,6 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:random", "//yggdrasil_decision_forests/utils:snapshot", "//yggdrasil_decision_forests/utils:status_macros", - "//yggdrasil_decision_forests/utils:usage", "@com_google_absl//absl/container:fixed_array", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", diff --git a/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.cc b/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.cc index e5ada9e6..f4533521 100644 --- a/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.cc +++ b/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.cc @@ -69,7 +69,6 @@ #include "yggdrasil_decision_forests/utils/random.h" #include "yggdrasil_decision_forests/utils/snapshot.h" #include "yggdrasil_decision_forests/utils/status_macros.h" -#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace model { @@ -608,7 +607,7 @@ GradientBoostedTreesLearner::InitializeModel( } absl::StatusOr> -GradientBoostedTreesLearner::TrainWithStatus( +GradientBoostedTreesLearner::TrainWithStatusImpl( const absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec, const absl::optional& typed_valid_path) const { @@ -616,8 +615,8 @@ GradientBoostedTreesLearner::TrainWithStatus( gradient_boosted_trees::proto::gradient_boosted_trees_config); if (!gbt_config.has_sample_with_shards()) { // Regular training. - return AbstractLearner::TrainWithStatus(typed_path, data_spec, - typed_valid_path); + return AbstractLearner::TrainWithStatusImpl(typed_path, data_spec, + typed_valid_path); } return ShardedSamplingTrain(typed_path, data_spec, typed_valid_path); @@ -644,10 +643,6 @@ GradientBoostedTreesLearner::ShardedSamplingTrain( internal::AllTrainingConfiguration config; RETURN_IF_ERROR(BuildAllTrainingConfiguration(data_spec, &config)); - utils::usage::OnTrainingStart(data_spec, config.train_config, - config.train_config_link, - /*num_examples=*/-1); - // Initialize the model. auto mdl = InitializeModel(config, data_spec); @@ -1124,22 +1119,12 @@ GradientBoostedTreesLearner::ShardedSamplingTrain( config, early_stopping, validation->dataset, deployment().num_threads(), mdl.get())); } - RETURN_IF_ERROR(FinalizeModel(log_directory_, mdl.get())); - - if (config.train_config.pure_serving_model()) { - RETURN_IF_ERROR(mdl->MakePureServing()); - } - - utils::usage::OnTrainingEnd( - data_spec, config.train_config, config.train_config_link, - /*num_examples=*/-1, *mdl, absl::Now() - begin_training); - return mdl; } absl::StatusOr> -GradientBoostedTreesLearner::TrainWithStatus( +GradientBoostedTreesLearner::TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset) const { @@ -1172,9 +1157,6 @@ GradientBoostedTreesLearner::TrainWithStatus( << " example(s) and " << config.train_config_link.features().size() << " feature(s)."; - utils::usage::OnTrainingStart(train_dataset.data_spec(), config.train_config, - config.train_config_link, train_dataset.nrow()); - if (config.gbt_config->has_sample_with_shards()) { return absl::InvalidArgumentError( "\"sample_with_shards\" is not compatible with training " @@ -1658,15 +1640,7 @@ GradientBoostedTreesLearner::TrainWithStatus( RETURN_IF_ERROR(FinalizeModel(log_directory_, mdl.get())); - utils::usage::OnTrainingEnd(train_dataset.data_spec(), training_config(), - config.train_config_link, train_dataset.nrow(), - *mdl, absl::Now() - begin_training); - decision_tree::SetLeafIndices(mdl->mutable_decision_trees()); - - if (config.train_config.pure_serving_model()) { - RETURN_IF_ERROR(mdl->MakePureServing()); - } return std::move(mdl); } diff --git a/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.h b/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.h index fc84142d..38b72263 100644 --- a/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.h +++ b/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.h @@ -132,15 +132,15 @@ class GradientBoostedTreesLearner : public AbstractLearner { static constexpr char kHParamFocalLossGamma[] = "focal_loss_gamma"; static constexpr char kHParamFocalLossAlpha[] = "focal_loss_alpha"; - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> - valid_dataset = {}) const override; + valid_dataset) const override; - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec, - const absl::optional& typed_valid_path = {}) const override; + const absl::optional& typed_valid_path) const override; // Detects configuration errors and warnings. static absl::Status CheckConfiguration( diff --git a/yggdrasil_decision_forests/learner/hyperparameters_optimizer/BUILD b/yggdrasil_decision_forests/learner/hyperparameters_optimizer/BUILD index 0808d662..d2df9c63 100644 --- a/yggdrasil_decision_forests/learner/hyperparameters_optimizer/BUILD +++ b/yggdrasil_decision_forests/learner/hyperparameters_optimizer/BUILD @@ -38,7 +38,6 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:concurrency", "//yggdrasil_decision_forests/utils:filesystem", "//yggdrasil_decision_forests/utils:hyper_parameters", - "//yggdrasil_decision_forests/utils:usage", "//yggdrasil_decision_forests/utils/distribute:distribute_without_implementations", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", diff --git a/yggdrasil_decision_forests/learner/hyperparameters_optimizer/hyperparameters_optimizer.cc b/yggdrasil_decision_forests/learner/hyperparameters_optimizer/hyperparameters_optimizer.cc index 5def4c84..e20ec858 100644 --- a/yggdrasil_decision_forests/learner/hyperparameters_optimizer/hyperparameters_optimizer.cc +++ b/yggdrasil_decision_forests/learner/hyperparameters_optimizer/hyperparameters_optimizer.cc @@ -35,7 +35,6 @@ #include "yggdrasil_decision_forests/utils/concurrency_streamprocessor.h" #include "yggdrasil_decision_forests/utils/distribute/distribute.h" #include "yggdrasil_decision_forests/utils/filesystem.h" -#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace model { @@ -165,7 +164,7 @@ HyperParameterOptimizerLearner::TrainFromFileOnMemoryDataset( } absl::StatusOr> -HyperParameterOptimizerLearner::TrainWithStatus( +HyperParameterOptimizerLearner::TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset) const { @@ -184,8 +183,6 @@ HyperParameterOptimizerLearner::TrainWithStatus( "deployment configs."); } - const auto begin_training = absl::Now(); - // The effective configuration is the user configuration + the default value + // the automatic configuration (if enabled) + the copy of the non-specified // training configuration field from the learner to the sub-learner (e.g. copy @@ -197,9 +194,6 @@ HyperParameterOptimizerLearner::TrainWithStatus( const proto::HyperParametersOptimizerLearnerTrainingConfig& spe_config = effective_config.GetExtension(proto::hyperparameters_optimizer_config); - utils::usage::OnTrainingStart(train_dataset.data_spec(), effective_config, - config_link, train_dataset.nrow()); - // Initialize the learner with the base hyperparameters. ASSIGN_OR_RETURN(auto base_learner, BuildBaseLearner(spe_config, /*for_tuning=*/true)); @@ -231,9 +225,6 @@ HyperParameterOptimizerLearner::TrainWithStatus( RETURN_IF_ERROR(base_learner->SetHyperParameters(best_params)); ASSIGN_OR_RETURN( auto mdl, base_learner->TrainWithStatus(train_dataset, valid_dataset)); - utils::usage::OnTrainingEnd(train_dataset.data_spec(), training_config(), - config_link, train_dataset.nrow(), *mdl, - absl::Now() - begin_training); *mdl->mutable_hyperparameter_optimizer_logs() = logs; return mdl; } else { @@ -265,7 +256,7 @@ absl::Status HyperParameterOptimizerLearner::GetEffectiveConfiguration( } absl::StatusOr> -HyperParameterOptimizerLearner::TrainWithStatus( +HyperParameterOptimizerLearner::TrainWithStatusImpl( const absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec, const absl::optional& typed_valid_path) const { @@ -274,8 +265,8 @@ HyperParameterOptimizerLearner::TrainWithStatus( deployment().execution_case() == model::proto::DeploymentConfig::ExecutionCase::kLocal) { // Load the dataset in memory and run the in-memory training. - return AbstractLearner::TrainWithStatus(typed_path, data_spec, - typed_valid_path); + return AbstractLearner::TrainWithStatusImpl(typed_path, data_spec, + typed_valid_path); } if (!deployment().has_distribute()) { @@ -284,8 +275,6 @@ HyperParameterOptimizerLearner::TrainWithStatus( "deployment configs."); } - const auto begin_training = absl::Now(); - // The effective configuration is the user configuration + the default value + // the automatic configuration (if enabled) + the copy of the non-specified // training configuration field from the learner to the sub-learner (e.g. copy @@ -300,8 +289,6 @@ HyperParameterOptimizerLearner::TrainWithStatus( // Initialize the remote workers. ASSIGN_OR_RETURN(auto manager, CreateDistributeManager(spe_config)); - utils::usage::OnTrainingStart(data_spec, effective_config, config_link, -1); - // Initialize the learner with the base hyperparameters. ASSIGN_OR_RETURN(auto base_learner, BuildBaseLearner(spe_config, /*for_tuning=*/true)); @@ -337,8 +324,6 @@ HyperParameterOptimizerLearner::TrainWithStatus( best_params, typed_path, data_spec, typed_valid_path, manager.get())); - utils::usage::OnTrainingEnd(data_spec, training_config(), config_link, -1, - *model, absl::Now() - begin_training); *model->mutable_hyperparameter_optimizer_logs() = logs; RETURN_IF_ERROR(manager->Done()); diff --git a/yggdrasil_decision_forests/learner/hyperparameters_optimizer/hyperparameters_optimizer.h b/yggdrasil_decision_forests/learner/hyperparameters_optimizer/hyperparameters_optimizer.h index 74c8d822..7919eab6 100644 --- a/yggdrasil_decision_forests/learner/hyperparameters_optimizer/hyperparameters_optimizer.h +++ b/yggdrasil_decision_forests/learner/hyperparameters_optimizer/hyperparameters_optimizer.h @@ -52,15 +52,15 @@ class HyperParameterOptimizerLearner : public AbstractLearner { // Unique identifier of the learning algorithm. static constexpr char kRegisteredName[] = "HYPERPARAMETER_OPTIMIZER"; - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> - valid_dataset = {}) const override; + valid_dataset) const override; - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec, - const absl::optional& typed_valid_path = {}) const override; + const absl::optional& typed_valid_path) const override; // Sets the hyper-parameters of the learning algorithm from "generic hparams". absl::Status SetHyperParametersImpl( diff --git a/yggdrasil_decision_forests/learner/multitasker/BUILD b/yggdrasil_decision_forests/learner/multitasker/BUILD index e2f21770..5b86efd9 100644 --- a/yggdrasil_decision_forests/learner/multitasker/BUILD +++ b/yggdrasil_decision_forests/learner/multitasker/BUILD @@ -27,7 +27,6 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:regex", "//yggdrasil_decision_forests/utils:status_macros", "//yggdrasil_decision_forests/utils:synchronization_primitives", - "//yggdrasil_decision_forests/utils:usage", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/yggdrasil_decision_forests/learner/multitasker/multitasker.cc b/yggdrasil_decision_forests/learner/multitasker/multitasker.cc index 50093e90..e9b04937 100644 --- a/yggdrasil_decision_forests/learner/multitasker/multitasker.cc +++ b/yggdrasil_decision_forests/learner/multitasker/multitasker.cc @@ -30,7 +30,6 @@ #include "yggdrasil_decision_forests/utils/regex.h" #include "yggdrasil_decision_forests/utils/status_macros.h" #include "yggdrasil_decision_forests/utils/synchronization_primitives.h" -#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace model { @@ -112,7 +111,7 @@ MultitaskerLearner::MultitaskerLearner( : AbstractLearner(training_config) {} absl::StatusOr> -MultitaskerLearner::TrainWithStatus( +MultitaskerLearner::TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset) const { diff --git a/yggdrasil_decision_forests/learner/multitasker/multitasker.h b/yggdrasil_decision_forests/learner/multitasker/multitasker.h index b917969c..7f84d93e 100644 --- a/yggdrasil_decision_forests/learner/multitasker/multitasker.h +++ b/yggdrasil_decision_forests/learner/multitasker/multitasker.h @@ -42,10 +42,10 @@ class MultitaskerLearner : public AbstractLearner { static constexpr char kRegisteredName[] = "MULTITASKER"; - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> - valid_dataset = {}) const override; + valid_dataset) const override; absl::Status SetHyperParameters( const proto::GenericHyperParameters& generic_hyper_params) override; diff --git a/yggdrasil_decision_forests/learner/random_forest/BUILD b/yggdrasil_decision_forests/learner/random_forest/BUILD index 833b9daf..0b5ba20d 100644 --- a/yggdrasil_decision_forests/learner/random_forest/BUILD +++ b/yggdrasil_decision_forests/learner/random_forest/BUILD @@ -49,7 +49,6 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:random", "//yggdrasil_decision_forests/utils:status_macros", "//yggdrasil_decision_forests/utils:synchronization_primitives", - "//yggdrasil_decision_forests/utils:usage", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", diff --git a/yggdrasil_decision_forests/learner/random_forest/random_forest.cc b/yggdrasil_decision_forests/learner/random_forest/random_forest.cc index ba6e6150..1596f16f 100644 --- a/yggdrasil_decision_forests/learner/random_forest/random_forest.cc +++ b/yggdrasil_decision_forests/learner/random_forest/random_forest.cc @@ -34,6 +34,7 @@ #include "absl/time/time.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" #include "yggdrasil_decision_forests/dataset/example_writer.h" +#include "yggdrasil_decision_forests/dataset/types.h" #include "yggdrasil_decision_forests/dataset/vertical_dataset.h" #include "yggdrasil_decision_forests/dataset/weight.h" #include "yggdrasil_decision_forests/dataset/weight.pb.h" @@ -43,7 +44,6 @@ #include "yggdrasil_decision_forests/learner/decision_tree/generic_parameters.h" #include "yggdrasil_decision_forests/learner/decision_tree/training.h" #include "yggdrasil_decision_forests/learner/random_forest/random_forest.pb.h" -#include "yggdrasil_decision_forests/dataset/types.h" #include "yggdrasil_decision_forests/metric/metric.h" #include "yggdrasil_decision_forests/metric/metric.pb.h" #include "yggdrasil_decision_forests/model/abstract_model.h" @@ -61,7 +61,6 @@ #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/status_macros.h" #include "yggdrasil_decision_forests/utils/synchronization_primitives.h" -#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace model { @@ -362,7 +361,7 @@ absl::Status RandomForestLearner::CheckConfiguration( } absl::StatusOr> -RandomForestLearner::TrainWithStatus( +RandomForestLearner::TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> valid_dataset) const { @@ -419,8 +418,6 @@ RandomForestLearner::TrainWithStatus( RETURN_IF_ERROR(CheckConfiguration(train_dataset.data_spec(), config_with_default, config_link, rf_config, deployment())); - utils::usage::OnTrainingStart(train_dataset.data_spec(), config_with_default, - config_link, train_dataset.nrow()); std::vector weights; @@ -891,10 +888,6 @@ RandomForestLearner::TrainWithStatus( deployment().num_threads(), mdl.get())); } - utils::usage::OnTrainingEnd(train_dataset.data_spec(), config_with_default, - config_link, train_dataset.nrow(), *mdl, - absl::Now() - begin_training); - if (!rf_config.export_oob_prediction_path().empty()) { RETURN_IF_ERROR(ExportOOBPredictions( config_with_default, config_link, train_dataset.data_spec(), @@ -907,10 +900,6 @@ RandomForestLearner::TrainWithStatus( decision_tree::SetLeafIndices(mdl->mutable_decision_trees()); - if (config_with_default.pure_serving_model()) { - RETURN_IF_ERROR(mdl->MakePureServing()); - } - return std::move(mdl); } diff --git a/yggdrasil_decision_forests/learner/random_forest/random_forest.h b/yggdrasil_decision_forests/learner/random_forest/random_forest.h index 36555194..4897b85f 100644 --- a/yggdrasil_decision_forests/learner/random_forest/random_forest.h +++ b/yggdrasil_decision_forests/learner/random_forest/random_forest.h @@ -74,10 +74,10 @@ class RandomForestLearner : public AbstractLearner { static constexpr char kHParamSamplingWithReplacement[] = "sampling_with_replacement"; - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, absl::optional> - valid_dataset = {}) const override; + valid_dataset) const override; // Detects configuration errors and warnings. static absl::Status CheckConfiguration( diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_test.cc b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_test.cc index 687883d6..a5d9ae7f 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_test.cc +++ b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_test.cc @@ -41,7 +41,7 @@ class FakeLearner1 : public model::AbstractLearner { explicit FakeLearner1(const model::proto::TrainingConfig& training_config) : AbstractLearner(training_config) {} - absl::StatusOr> TrainWithStatus( + absl::StatusOr> TrainWithStatusImpl( const dataset::VerticalDataset& train_dataset, std::optional> valid_dataset = {}) const override { diff --git a/yggdrasil_decision_forests/utils/usage.h b/yggdrasil_decision_forests/utils/usage.h index 12fc2e53..565146b4 100644 --- a/yggdrasil_decision_forests/utils/usage.h +++ b/yggdrasil_decision_forests/utils/usage.h @@ -37,14 +37,12 @@ namespace usage { void OnTrainingStart( const dataset::proto::DataSpecification& data_spec, const model::proto::TrainingConfig& train_config, - const model::proto::TrainingConfigLinking& train_config_link, int64_t num_examples); // Complete a model training. // Should be called at the end of the "Train" methods of learners. void OnTrainingEnd(const dataset::proto::DataSpecification& data_spec, const model::proto::TrainingConfig& train_config, - const model::proto::TrainingConfigLinking& train_config_link, int64_t num_examples, const model::AbstractModel& model, absl::Duration training_duration); diff --git a/yggdrasil_decision_forests/utils/usage_default.cc b/yggdrasil_decision_forests/utils/usage_default.cc index 5d037adf..55b72bab 100644 --- a/yggdrasil_decision_forests/utils/usage_default.cc +++ b/yggdrasil_decision_forests/utils/usage_default.cc @@ -22,14 +22,12 @@ namespace usage { void OnTrainingStart( const dataset::proto::DataSpecification& data_spec, const model::proto::TrainingConfig& train_config, - const model::proto::TrainingConfigLinking& train_config_link, int64_t num_examples) { // Add usage tracking here. } void OnTrainingEnd(const dataset::proto::DataSpecification& data_spec, const model::proto::TrainingConfig& train_config, - const model::proto::TrainingConfigLinking& train_config_link, int64_t num_examples, const model::AbstractModel& model, absl::Duration training_duration) { // Add usage tracking here. From 0f0cd9714d6294c670136a031e34157220c97930 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Thu, 30 May 2024 08:57:03 -0700 Subject: [PATCH 12/30] - Add placeholder for dataset usage statistics. - Break the dependency from AbstractModel to VerticalDatasetIO. PiperOrigin-RevId: 638670083 --- yggdrasil_decision_forests/cli/BUILD | 1 + yggdrasil_decision_forests/cli/evaluate.cc | 5 +- yggdrasil_decision_forests/dataset/BUILD | 2 + .../dataset/example_writer.cc | 3 + .../dataset/vertical_dataset_io.cc | 4 + yggdrasil_decision_forests/learner/BUILD | 1 + .../learner/abstract_learner.cc | 7 + yggdrasil_decision_forests/model/BUILD | 29 +++- .../model/abstract_model.cc | 123 +------------- .../model/abstract_model.h | 27 +-- .../model/abstract_model_test.cc | 12 +- .../model/evaluate_on_disk.cc | 155 ++++++++++++++++++ .../model/evaluate_on_disk.h | 43 +++++ .../model/model_library.cc | 5 + yggdrasil_decision_forests/utils/BUILD | 2 + yggdrasil_decision_forests/utils/usage.h | 22 ++- .../utils/usage_default.cc | 18 +- 17 files changed, 299 insertions(+), 160 deletions(-) create mode 100644 yggdrasil_decision_forests/model/evaluate_on_disk.cc create mode 100644 yggdrasil_decision_forests/model/evaluate_on_disk.h diff --git a/yggdrasil_decision_forests/cli/BUILD b/yggdrasil_decision_forests/cli/BUILD index 83caca09..b197b7e9 100644 --- a/yggdrasil_decision_forests/cli/BUILD +++ b/yggdrasil_decision_forests/cli/BUILD @@ -149,6 +149,7 @@ cc_binary_ydf( "//yggdrasil_decision_forests/metric:report", "//yggdrasil_decision_forests/model:abstract_model", "//yggdrasil_decision_forests/model:all_models", + "//yggdrasil_decision_forests/model:evaluate_on_disk", "//yggdrasil_decision_forests/model:model_library", "//yggdrasil_decision_forests/model:prediction_cc_proto", "//yggdrasil_decision_forests/serving/decision_forest:register_engines", diff --git a/yggdrasil_decision_forests/cli/evaluate.cc b/yggdrasil_decision_forests/cli/evaluate.cc index 2a6a65f7..a998e6f9 100644 --- a/yggdrasil_decision_forests/cli/evaluate.cc +++ b/yggdrasil_decision_forests/cli/evaluate.cc @@ -31,6 +31,7 @@ #include "yggdrasil_decision_forests/metric/metric.pb.h" #include "yggdrasil_decision_forests/metric/report.h" #include "yggdrasil_decision_forests/model/abstract_model.h" +#include "yggdrasil_decision_forests/model/evaluate_on_disk.h" #include "yggdrasil_decision_forests/model/model_library.h" #include "yggdrasil_decision_forests/model/prediction.pb.h" #include "yggdrasil_decision_forests/utils/filesystem.h" @@ -83,7 +84,9 @@ void Evaluate() { options.set_task(model->task()); } // evaluate model. - evaluation = model->Evaluate(absl::GetFlag(FLAGS_dataset), options, &rnd); + evaluation = + model::EvaluateOnDisk(*model, absl::GetFlag(FLAGS_dataset), options, &rnd) + .value(); const auto format = absl::GetFlag(FLAGS_format); if (format == "text") { diff --git a/yggdrasil_decision_forests/dataset/BUILD b/yggdrasil_decision_forests/dataset/BUILD index c607823f..12613711 100644 --- a/yggdrasil_decision_forests/dataset/BUILD +++ b/yggdrasil_decision_forests/dataset/BUILD @@ -188,6 +188,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:logging", "//yggdrasil_decision_forests/utils:sharded_io", "//yggdrasil_decision_forests/utils:status_macros", + "//yggdrasil_decision_forests/utils:usage", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -212,6 +213,7 @@ cc_library_ydf( ":formats_cc_proto", "//yggdrasil_decision_forests/utils:sharded_io", "//yggdrasil_decision_forests/utils:status_macros", + "//yggdrasil_decision_forests/utils:usage", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/yggdrasil_decision_forests/dataset/example_writer.cc b/yggdrasil_decision_forests/dataset/example_writer.cc index 40ee973e..3fe0237d 100644 --- a/yggdrasil_decision_forests/dataset/example_writer.cc +++ b/yggdrasil_decision_forests/dataset/example_writer.cc @@ -33,6 +33,7 @@ #include "yggdrasil_decision_forests/dataset/formats.pb.h" #include "yggdrasil_decision_forests/utils/sharded_io.h" #include "yggdrasil_decision_forests/utils/status_macros.h" +#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace dataset { @@ -45,6 +46,8 @@ absl::StatusOr> CreateExampleWriter( ASSIGN_OR_RETURN(std::tie(sharded_path, format), GetDatasetPathAndTypeOrStatus(typed_path)); + utils::usage::OnSaveDataset(sharded_path); + const std::string& format_name = proto::DatasetFormat_Name(format); ASSIGN_OR_RETURN( auto writer, diff --git a/yggdrasil_decision_forests/dataset/vertical_dataset_io.cc b/yggdrasil_decision_forests/dataset/vertical_dataset_io.cc index cd0bae9c..f7c1f196 100644 --- a/yggdrasil_decision_forests/dataset/vertical_dataset_io.cc +++ b/yggdrasil_decision_forests/dataset/vertical_dataset_io.cc @@ -35,6 +35,7 @@ #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/sharded_io.h" #include "yggdrasil_decision_forests/utils/status_macros.h" +#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace dataset { @@ -125,6 +126,8 @@ absl::Status LoadVerticalDataset( std::vector shards; RETURN_IF_ERROR(utils::ExpandInputShards(path, &shards)); + utils::usage::OnLoadDataset(path); + if (shards.size() <= 1 || config.num_threads <= 1) { // Loading in a single thread. return LoadVerticalDatasetSingleThread(typed_path, data_spec, dataset, @@ -240,6 +243,7 @@ absl::Status SaveVerticalDataset(const VerticalDataset& dataset, ASSIGN_OR_RETURN(auto writer, CreateExampleWriter(typed_path, dataset.data_spec(), num_records_by_shard)); + proto::Example example; for (VerticalDataset::row_t row = 0; row < dataset.nrow(); row++) { dataset.ExtractExample(row, &example); diff --git a/yggdrasil_decision_forests/learner/BUILD b/yggdrasil_decision_forests/learner/BUILD index 1d25f795..b4749b67 100644 --- a/yggdrasil_decision_forests/learner/BUILD +++ b/yggdrasil_decision_forests/learner/BUILD @@ -57,6 +57,7 @@ cc_library_ydf( ":abstract_learner_cc_proto", "//yggdrasil_decision_forests/dataset:data_spec", "//yggdrasil_decision_forests/dataset:data_spec_cc_proto", + "//yggdrasil_decision_forests/dataset:formats", "//yggdrasil_decision_forests/dataset:types", "//yggdrasil_decision_forests/dataset:vertical_dataset", "//yggdrasil_decision_forests/dataset:vertical_dataset_io", diff --git a/yggdrasil_decision_forests/learner/abstract_learner.cc b/yggdrasil_decision_forests/learner/abstract_learner.cc index e33e8ebb..8a4f7b50 100644 --- a/yggdrasil_decision_forests/learner/abstract_learner.cc +++ b/yggdrasil_decision_forests/learner/abstract_learner.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ #include "absl/types/optional.h" #include "yggdrasil_decision_forests/dataset/data_spec.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" +#include "yggdrasil_decision_forests/dataset/formats.h" #include "yggdrasil_decision_forests/dataset/types.h" #include "yggdrasil_decision_forests/dataset/vertical_dataset.h" #include "yggdrasil_decision_forests/dataset/vertical_dataset_io.h" @@ -327,6 +329,11 @@ absl::StatusOr> AbstractLearner::TrainWithStatus( absl::string_view typed_path, const dataset::proto::DataSpecification& data_spec, const absl::optional& typed_valid_path) const { + std::string path; + ASSIGN_OR_RETURN(std::tie(std::ignore, path), + dataset::SplitTypeAndPath(typed_path)); + utils::usage::OnLoadDataset(path); + utils::usage::OnTrainingStart(data_spec, training_config(), /*num_examples=*/-1); const auto begin_training = absl::Now(); diff --git a/yggdrasil_decision_forests/model/BUILD b/yggdrasil_decision_forests/model/BUILD index 1374941a..97b10344 100644 --- a/yggdrasil_decision_forests/model/BUILD +++ b/yggdrasil_decision_forests/model/BUILD @@ -40,9 +40,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/dataset:data_spec", "//yggdrasil_decision_forests/dataset:data_spec_cc_proto", "//yggdrasil_decision_forests/dataset:example_cc_proto", - "//yggdrasil_decision_forests/dataset:formats", "//yggdrasil_decision_forests/dataset:vertical_dataset", - "//yggdrasil_decision_forests/dataset:vertical_dataset_io", "//yggdrasil_decision_forests/dataset:weight", "//yggdrasil_decision_forests/dataset:weight_cc_proto", "//yggdrasil_decision_forests/metric", @@ -50,7 +48,6 @@ cc_library_ydf( "//yggdrasil_decision_forests/metric:report", "//yggdrasil_decision_forests/serving:example_set", "//yggdrasil_decision_forests/serving:fast_engine", - "//yggdrasil_decision_forests/utils:concurrency", "//yggdrasil_decision_forests/utils:distribution", "//yggdrasil_decision_forests/utils:distribution_cc_proto", "//yggdrasil_decision_forests/utils:logging", @@ -58,7 +55,6 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:protobuf", "//yggdrasil_decision_forests/utils:random", "//yggdrasil_decision_forests/utils:registration", - "//yggdrasil_decision_forests/utils:sharded_io", "//yggdrasil_decision_forests/utils:status_macros", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -70,6 +66,29 @@ cc_library_ydf( ], ) +cc_library_ydf( + name = "evaluate_on_disk", + srcs = ["evaluate_on_disk.cc"], + hdrs = ["evaluate_on_disk.h"], + deps = [ + ":abstract_model", + "//yggdrasil_decision_forests/dataset:formats", + "//yggdrasil_decision_forests/dataset:vertical_dataset", + "//yggdrasil_decision_forests/dataset:vertical_dataset_io", + "//yggdrasil_decision_forests/dataset:weight", + "//yggdrasil_decision_forests/metric", + "//yggdrasil_decision_forests/utils:concurrency", + "//yggdrasil_decision_forests/utils:logging", + "//yggdrasil_decision_forests/utils:random", + "//yggdrasil_decision_forests/utils:sharded_io", + "//yggdrasil_decision_forests/utils:status_macros", + "//yggdrasil_decision_forests/utils:synchronization_primitives", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + ], +) + # Note: The hyper parameter optimizer and the model library are in the same cc_library_ydf because they # co-depend on each others. cc_library_ydf( @@ -88,6 +107,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:filesystem", "//yggdrasil_decision_forests/utils:logging", "//yggdrasil_decision_forests/utils:status_macros", + "//yggdrasil_decision_forests/utils:usage", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -196,6 +216,7 @@ cc_test( "//yggdrasil_decision_forests/dataset:vertical_dataset", "//yggdrasil_decision_forests/dataset:vertical_dataset_io", "//yggdrasil_decision_forests/metric", + "//yggdrasil_decision_forests/model:evaluate_on_disk", "//yggdrasil_decision_forests/model/gradient_boosted_trees", "//yggdrasil_decision_forests/model/random_forest", "//yggdrasil_decision_forests/serving:example_set", diff --git a/yggdrasil_decision_forests/model/abstract_model.cc b/yggdrasil_decision_forests/model/abstract_model.cc index b6c72ac1..42e8295b 100644 --- a/yggdrasil_decision_forests/model/abstract_model.cc +++ b/yggdrasil_decision_forests/model/abstract_model.cc @@ -18,9 +18,9 @@ #include #include -#include +#include +#include #include -#include #include #include #include @@ -34,12 +34,11 @@ #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" +#include "absl/types/optional.h" #include "yggdrasil_decision_forests/dataset/data_spec.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" #include "yggdrasil_decision_forests/dataset/example.pb.h" -#include "yggdrasil_decision_forests/dataset/formats.h" #include "yggdrasil_decision_forests/dataset/vertical_dataset.h" -#include "yggdrasil_decision_forests/dataset/vertical_dataset_io.h" #include "yggdrasil_decision_forests/dataset/weight.h" #include "yggdrasil_decision_forests/dataset/weight.pb.h" #include "yggdrasil_decision_forests/metric/metric.h" @@ -51,13 +50,11 @@ #include "yggdrasil_decision_forests/model/prediction.pb.h" #include "yggdrasil_decision_forests/serving/example_set.h" #include "yggdrasil_decision_forests/serving/fast_engine.h" -#include "yggdrasil_decision_forests/utils/concurrency.h" #include "yggdrasil_decision_forests/utils/distribution.h" #include "yggdrasil_decision_forests/utils/distribution.pb.h" #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/protobuf.h" #include "yggdrasil_decision_forests/utils/random.h" -#include "yggdrasil_decision_forests/utils/sharded_io.h" #include "yggdrasil_decision_forests/utils/status_macros.h" namespace yggdrasil_decision_forests { @@ -126,14 +123,6 @@ metric::proto::EvaluationResults AbstractModel::Evaluate( return EvaluateWithStatus(dataset, option, rnd, predictions).value(); } -metric::proto::EvaluationResults AbstractModel::Evaluate( - const absl::string_view typed_path, - const metric::proto::EvaluationOptions& option, - utils::RandomEngine* rnd) const { - // TODO: Fix. - return EvaluateWithStatus(typed_path, option, rnd).value(); -} - absl::StatusOr AbstractModel::EvaluateWithStatus( const dataset::VerticalDataset& dataset, @@ -150,22 +139,6 @@ AbstractModel::EvaluateWithStatus( return eval; } -absl::StatusOr -AbstractModel::EvaluateWithStatus( - const absl::string_view typed_path, - const metric::proto::EvaluationOptions& option, - utils::RandomEngine* rnd) const { - if (option.task() != task()) { - STATUS_FATAL("The evaluation and the model tasks differ."); - } - metric::proto::EvaluationResults eval; - RETURN_IF_ERROR( - metric::InitializeEvaluation(option, LabelColumnSpec(), &eval)); - RETURN_IF_ERROR(AppendEvaluation(typed_path, option, rnd, &eval)); - RETURN_IF_ERROR(metric::FinalizeEvaluation(option, LabelColumnSpec(), &eval)); - return eval; -} - absl::StatusOr AbstractModel::EvaluateWithEngine( const serving::FastEngine& engine, const dataset::VerticalDataset& dataset, @@ -444,96 +417,6 @@ absl::Status AbstractModel::AppendEvaluation( return absl::OkStatus(); } -absl::Status AbstractModel::AppendEvaluation( - const absl::string_view typed_path, - const metric::proto::EvaluationOptions& option, utils::RandomEngine* rnd, - metric::proto::EvaluationResults* eval) const { - dataset::proto::LinkedWeightDefinition weight_links; - if (option.has_weights()) { - RETURN_IF_ERROR(dataset::GetLinkedWeightDefinition( - option.weights(), data_spec_, &weight_links)); - } - - auto engine_or_status = BuildFastEngine(); - if (engine_or_status.ok()) { - const auto engine = std::move(engine_or_status.value()); - // Extract the shards from the dataset path. - std::string path, prefix; - std::tie(prefix, path) = dataset::SplitTypeAndPath(typed_path).value(); - std::vector shards; - RETURN_IF_ERROR(utils::ExpandInputShards(path, &shards)); - - // Evaluate each shard in a separate thread. - utils::concurrency::Mutex - mutex; // Guards "num_evaluated_shards" and "eval". - int num_evaluated_shards = 0; - absl::Status worker_status; - - const auto process_shard = [&option, eval, &mutex, &prefix, &engine, - &weight_links, &num_evaluated_shards, &shards, - this](absl::string_view shard, - int sub_rnd_seed) -> absl::Status { - utils::RandomEngine sub_rnd(sub_rnd_seed); - - dataset::VerticalDataset dataset; - RETURN_IF_ERROR(dataset::LoadVerticalDataset( - absl::StrCat(prefix, ":", shard), data_spec_, &dataset)); - - metric::proto::EvaluationResults sub_evaluation; - RETURN_IF_ERROR(metric::InitializeEvaluation(option, LabelColumnSpec(), - &sub_evaluation)); - - RETURN_IF_ERROR(AppendEvaluationWithEngine(dataset, option, weight_links, - *engine, &sub_rnd, nullptr, - &sub_evaluation)); - - utils::concurrency::MutexLock lock(&mutex); - RETURN_IF_ERROR(metric::MergeEvaluation(option, sub_evaluation, eval)); - num_evaluated_shards++; - LOG_INFO_EVERY_N_SEC(30, _ << num_evaluated_shards << "/" << shards.size() - << " shards evaluated"); - return absl::OkStatus(); - }; - - { - const int num_threads = std::min(shards.size(), 20); - utils::concurrency::ThreadPool thread_pool("evaluation", num_threads); - thread_pool.StartWorkers(); - for (const auto& shard : shards) { - thread_pool.Schedule([&shard, &mutex, &process_shard, &worker_status, - sub_rnd_seed = (*rnd)()]() -> void { - { - utils::concurrency::MutexLock lock(&mutex); - if (!worker_status.ok()) { - return; - } - } - auto sub_status = process_shard(shard, sub_rnd_seed); - { - utils::concurrency::MutexLock lock(&mutex); - worker_status.Update(sub_status); - } - }); - } - } - - RETURN_IF_ERROR(worker_status); - - } else { - // Evaluate using the (slow) generic inference. - YDF_LOG(WARNING) - << "Evaluation with the slow generic engine without distribution"; - dataset::VerticalDataset dataset; - RETURN_IF_ERROR( - dataset::LoadVerticalDataset(typed_path, data_spec_, &dataset)); - RETURN_IF_ERROR(AppendEvaluation(dataset, option, rnd, eval)); - return absl::OkStatus(); - } - - eval->set_num_folds(eval->num_folds() + 1); - return absl::OkStatus(); -} - absl::Status AbstractModel::AppendEvaluationOverrideType( const dataset::VerticalDataset& dataset, const metric::proto::EvaluationOptions& option, diff --git a/yggdrasil_decision_forests/model/abstract_model.h b/yggdrasil_decision_forests/model/abstract_model.h index 5334e78a..1fbaa731 100644 --- a/yggdrasil_decision_forests/model/abstract_model.h +++ b/yggdrasil_decision_forests/model/abstract_model.h @@ -21,8 +21,8 @@ #ifndef YGGDRASIL_DECISION_FORESTS_MODEL_ABSTRACT_MODEL_H_ #define YGGDRASIL_DECISION_FORESTS_MODEL_ABSTRACT_MODEL_H_ +#include #include -#include #include #include @@ -222,25 +222,6 @@ class AbstractModel { const metric::proto::EvaluationOptions& option, utils::RandomEngine* rnd, std::vector* predictions = nullptr) const; - // Evaluates the model on a dataset stored in disk. `typed_path` defines - // the type and the path pattern of the files, as described in - // `yggdrasil_decision_forests/datasets/format.h` file. - // This method is preferable when the number of examples is large since they - // do not have to be all first loaded into memory. - // Returns a finalized EvaluationResults. - // Evaluates the model on a dataset. Returns a finalized EvaluationResults. - // The random generator "rnd" is used bootstrapping of confidence intervals - // and sub-sampling evaluation (if configured in "option"). - absl::StatusOr EvaluateWithStatus( - const absl::string_view typed_path, - const metric::proto::EvaluationOptions& option, - utils::RandomEngine* rnd) const; - - metric::proto::EvaluationResults Evaluate( - const absl::string_view typed_path, - const metric::proto::EvaluationOptions& option, - utils::RandomEngine* rnd) const; - // Similar to "Evaluate", but allow to override the evaluation objective. absl::StatusOr EvaluateOverrideType( const dataset::VerticalDataset& dataset, @@ -472,9 +453,6 @@ class AbstractModel { "SaveModel/LoadModel instead."); } - protected: - explicit AbstractModel(const absl::string_view name) : name_(name) {} - absl::Status AppendEvaluationWithEngine( const dataset::VerticalDataset& dataset, const metric::proto::EvaluationOptions& option, @@ -483,6 +461,9 @@ class AbstractModel { std::vector* predictions, metric::proto::EvaluationResults* eval) const; + protected: + explicit AbstractModel(const absl::string_view name) : name_(name) {} + // Prints information about the hyper-parameter optimizer logs. void AppendHyperparameterOptimizerLogs(std::string* description) const; diff --git a/yggdrasil_decision_forests/model/abstract_model_test.cc b/yggdrasil_decision_forests/model/abstract_model_test.cc index 4f14c2ea..d9620bf0 100644 --- a/yggdrasil_decision_forests/model/abstract_model_test.cc +++ b/yggdrasil_decision_forests/model/abstract_model_test.cc @@ -29,6 +29,7 @@ #include "yggdrasil_decision_forests/dataset/vertical_dataset_io.h" #include "yggdrasil_decision_forests/metric/metric.h" #include "yggdrasil_decision_forests/model/abstract_model.pb.h" +#include "yggdrasil_decision_forests/model/evaluate_on_disk.h" #include "yggdrasil_decision_forests/model/fast_engine_factory.h" #include "yggdrasil_decision_forests/model/model_library.h" #include "yggdrasil_decision_forests/model/model_testing.h" @@ -420,10 +421,13 @@ TEST(Evaluate, FromDisk) { &model)); utils::RandomEngine rnd; - const auto evaluation = model->Evaluate( - absl::StrCat("csv:", - file::JoinPath(TestDataDir(), "dataset", "adult_test.csv")), - {}, &rnd); + const auto evaluation = + EvaluateOnDisk( + *model, + absl::StrCat("csv:", file::JoinPath(TestDataDir(), "dataset", + "adult_test.csv")), + {}, &rnd) + .value(); EXPECT_NEAR(metric::Accuracy(evaluation), 0.8723513, 0.000001); } diff --git a/yggdrasil_decision_forests/model/evaluate_on_disk.cc b/yggdrasil_decision_forests/model/evaluate_on_disk.cc new file mode 100644 index 00000000..fa8bab1f --- /dev/null +++ b/yggdrasil_decision_forests/model/evaluate_on_disk.cc @@ -0,0 +1,155 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "yggdrasil_decision_forests/model/evaluate_on_disk.h" + +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "yggdrasil_decision_forests/dataset/formats.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset_io.h" +#include "yggdrasil_decision_forests/dataset/weight.h" +#include "yggdrasil_decision_forests/metric/metric.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" +#include "yggdrasil_decision_forests/utils/concurrency.h" +#include "yggdrasil_decision_forests/utils/logging.h" +#include "yggdrasil_decision_forests/utils/random.h" +#include "yggdrasil_decision_forests/utils/sharded_io.h" +#include "yggdrasil_decision_forests/utils/status_macros.h" +#include "yggdrasil_decision_forests/utils/synchronization_primitives.h" + +namespace yggdrasil_decision_forests::model { + +namespace { + +// Evaluates a model and add the evaluation to an already initialized evaluation +// proto. +absl::Status AppendEvaluation(const AbstractModel& model, + const absl::string_view typed_path, + const metric::proto::EvaluationOptions& option, + utils::RandomEngine* rnd, + metric::proto::EvaluationResults* eval) { + dataset::proto::LinkedWeightDefinition weight_links; + if (option.has_weights()) { + RETURN_IF_ERROR(dataset::GetLinkedWeightDefinition( + option.weights(), model.data_spec(), &weight_links)); + } + + auto engine_or_status = model.BuildFastEngine(); + if (engine_or_status.ok()) { + const auto engine = std::move(engine_or_status.value()); + // Extract the shards from the dataset path. + std::string path, prefix; + std::tie(prefix, path) = dataset::SplitTypeAndPath(typed_path).value(); + std::vector shards; + RETURN_IF_ERROR(utils::ExpandInputShards(path, &shards)); + + // Evaluate each shard in a separate thread. + utils::concurrency::Mutex + mutex; // Guards "num_evaluated_shards" and "eval". + int num_evaluated_shards = 0; + absl::Status worker_status; + + const auto process_shard = [&option, eval, &mutex, &prefix, &engine, + &weight_links, &num_evaluated_shards, &shards, + &model](absl::string_view shard, + int sub_rnd_seed) -> absl::Status { + utils::RandomEngine sub_rnd(sub_rnd_seed); + + dataset::VerticalDataset dataset; + RETURN_IF_ERROR(dataset::LoadVerticalDataset( + absl::StrCat(prefix, ":", shard), model.data_spec(), &dataset)); + + metric::proto::EvaluationResults sub_evaluation; + RETURN_IF_ERROR(metric::InitializeEvaluation( + option, model.LabelColumnSpec(), &sub_evaluation)); + + RETURN_IF_ERROR(model.AppendEvaluationWithEngine( + dataset, option, weight_links, *engine, &sub_rnd, nullptr, + &sub_evaluation)); + + utils::concurrency::MutexLock lock(&mutex); + RETURN_IF_ERROR(metric::MergeEvaluation(option, sub_evaluation, eval)); + num_evaluated_shards++; + LOG_INFO_EVERY_N_SEC(30, _ << num_evaluated_shards << "/" << shards.size() + << " shards evaluated"); + return absl::OkStatus(); + }; + + { + const int num_threads = std::min(shards.size(), 20); + utils::concurrency::ThreadPool thread_pool("evaluation", num_threads); + thread_pool.StartWorkers(); + for (const auto& shard : shards) { + thread_pool.Schedule([&shard, &mutex, &process_shard, &worker_status, + sub_rnd_seed = (*rnd)()]() -> void { + { + utils::concurrency::MutexLock lock(&mutex); + if (!worker_status.ok()) { + return; + } + } + auto sub_status = process_shard(shard, sub_rnd_seed); + { + utils::concurrency::MutexLock lock(&mutex); + worker_status.Update(sub_status); + } + }); + } + } + + RETURN_IF_ERROR(worker_status); + + } else { + // Evaluate using the (slow) generic inference. + YDF_LOG(WARNING) + << "Evaluation with the slow generic engine without distribution"; + dataset::VerticalDataset dataset; + RETURN_IF_ERROR( + dataset::LoadVerticalDataset(typed_path, model.data_spec(), &dataset)); + RETURN_IF_ERROR(model.AppendEvaluation(dataset, option, rnd, eval)); + return absl::OkStatus(); + } + + eval->set_num_folds(eval->num_folds() + 1); + return absl::OkStatus(); +} + +} // namespace + +absl::StatusOr EvaluateOnDisk( + const AbstractModel& model, const absl::string_view typed_path, + const metric::proto::EvaluationOptions& option, utils::RandomEngine* rnd) { + if (option.task() != model.task()) { + STATUS_FATAL("The evaluation and the model tasks differ."); + } + metric::proto::EvaluationResults eval; + RETURN_IF_ERROR( + metric::InitializeEvaluation(option, model.LabelColumnSpec(), &eval)); + RETURN_IF_ERROR(AppendEvaluation(model, typed_path, option, rnd, &eval)); + RETURN_IF_ERROR( + metric::FinalizeEvaluation(option, model.LabelColumnSpec(), &eval)); + return eval; +} + +} // namespace yggdrasil_decision_forests::model diff --git a/yggdrasil_decision_forests/model/evaluate_on_disk.h b/yggdrasil_decision_forests/model/evaluate_on_disk.h new file mode 100644 index 00000000..05fec549 --- /dev/null +++ b/yggdrasil_decision_forests/model/evaluate_on_disk.h @@ -0,0 +1,43 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Evaluation of a model on a dataset stored in disk. + +#ifndef YGGDRASIL_DECISION_FORESTS_MODEL_EVALUATE_ON_DISK_H_ +#define YGGDRASIL_DECISION_FORESTS_MODEL_EVALUATE_ON_DISK_H_ + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" +#include "yggdrasil_decision_forests/utils/random.h" + +namespace yggdrasil_decision_forests::model { + +// Evaluates the model on a dataset stored in disk. `typed_path` defines +// the type and the path pattern of the files, as described in +// `yggdrasil_decision_forests/datasets/format.h` file. +// This method is preferable when the number of examples is large since they +// do not have to be all first loaded into memory. +// Returns a finalized EvaluationResults. +// Evaluates the model on a dataset. Returns a finalized EvaluationResults. +// The random generator "rnd" is used bootstrapping of confidence intervals +// and sub-sampling evaluation (if configured in "option"). +absl::StatusOr EvaluateOnDisk( + const AbstractModel& model, const absl::string_view typed_path, + const metric::proto::EvaluationOptions& option, utils::RandomEngine* rnd); + +} // namespace yggdrasil_decision_forests::model + +#endif // YGGDRASIL_DECISION_FORESTS_MODEL_EVALUATE_ON_DISK_H_ diff --git a/yggdrasil_decision_forests/model/model_library.cc b/yggdrasil_decision_forests/model/model_library.cc index 79820d35..349d0b5f 100644 --- a/yggdrasil_decision_forests/model/model_library.cc +++ b/yggdrasil_decision_forests/model/model_library.cc @@ -32,6 +32,7 @@ #include "yggdrasil_decision_forests/utils/filesystem.h" #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/status_macros.h" +#include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace model { @@ -83,6 +84,8 @@ absl::Status SaveModel(absl::string_view directory, const AbstractModel& mdl, absl::Status SaveModel(absl::string_view directory, const AbstractModel* const mdl, ModelIOOptions io_options) { + utils::usage::OnSaveModel(directory); + RETURN_IF_ERROR(mdl->Validate()); RETURN_IF_ERROR(file::RecursivelyCreateDir(directory, file::Defaults())); proto::AbstractModel header; @@ -115,6 +118,8 @@ absl::StatusOr> LoadModel( absl::Status LoadModel(absl::string_view directory, std::unique_ptr* model, ModelIOOptions io_options) { + utils::usage::OnLoadModel(directory); + proto::AbstractModel header; std::string effective_directory = ImproveModelReadingPath(directory); diff --git a/yggdrasil_decision_forests/utils/BUILD b/yggdrasil_decision_forests/utils/BUILD index 7eec3406..da5dd41b 100644 --- a/yggdrasil_decision_forests/utils/BUILD +++ b/yggdrasil_decision_forests/utils/BUILD @@ -494,6 +494,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/dataset:data_spec_cc_proto", "//yggdrasil_decision_forests/learner:abstract_learner_cc_proto", "//yggdrasil_decision_forests/model:abstract_model", + "@com_google_absl//absl/strings", "@com_google_absl//absl/time", ], ) @@ -507,6 +508,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/dataset:data_spec_cc_proto", "//yggdrasil_decision_forests/learner:abstract_learner_cc_proto", "//yggdrasil_decision_forests/model:abstract_model", + "@com_google_absl//absl/strings", "@com_google_absl//absl/time", ], ) diff --git a/yggdrasil_decision_forests/utils/usage.h b/yggdrasil_decision_forests/utils/usage.h index 565146b4..a09138de 100644 --- a/yggdrasil_decision_forests/utils/usage.h +++ b/yggdrasil_decision_forests/utils/usage.h @@ -22,6 +22,9 @@ #ifndef YGGDRASIL_DECISION_FORESTS_TOOL_USAGE_H_ #define YGGDRASIL_DECISION_FORESTS_TOOL_USAGE_H_ +#include + +#include "absl/strings/string_view.h" #include "absl/time/time.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" #include "yggdrasil_decision_forests/learner/abstract_learner.pb.h" @@ -34,10 +37,9 @@ namespace usage { // Start a new model training. // Should be called at the start of the "Train" methods of learners. -void OnTrainingStart( - const dataset::proto::DataSpecification& data_spec, - const model::proto::TrainingConfig& train_config, - int64_t num_examples); +void OnTrainingStart(const dataset::proto::DataSpecification& data_spec, + const model::proto::TrainingConfig& train_config, + int64_t num_examples); // Complete a model training. // Should be called at the end of the "Train" methods of learners. @@ -56,6 +58,18 @@ void OnTrainingEnd(const dataset::proto::DataSpecification& data_spec, void OnInference(int64_t num_examples, const model::proto::Metadata& metadata); void OnInference(int64_t num_examples, const model::MetaData& metadata); +// When a dataset is loaded for training, inference, or other operations. +void OnLoadDataset(absl::string_view path); + +// When a dataset is saved. +void OnSaveDataset(absl::string_view path); + +// When a model is loaded for training, inference, or other operations. +void OnLoadModel(absl::string_view path); + +// When a model is saved. +void OnSaveModel(absl::string_view path); + // Enables / disable usage tracking. void EnableUsage(bool usage); diff --git a/yggdrasil_decision_forests/utils/usage_default.cc b/yggdrasil_decision_forests/utils/usage_default.cc index 55b72bab..9fef44d6 100644 --- a/yggdrasil_decision_forests/utils/usage_default.cc +++ b/yggdrasil_decision_forests/utils/usage_default.cc @@ -13,16 +13,18 @@ * limitations under the License. */ +#include + +#include "absl/strings/string_view.h" #include "yggdrasil_decision_forests/utils/usage.h" namespace yggdrasil_decision_forests { namespace utils { namespace usage { -void OnTrainingStart( - const dataset::proto::DataSpecification& data_spec, - const model::proto::TrainingConfig& train_config, - int64_t num_examples) { +void OnTrainingStart(const dataset::proto::DataSpecification& data_spec, + const model::proto::TrainingConfig& train_config, + int64_t num_examples) { // Add usage tracking here. } @@ -42,6 +44,14 @@ void OnInference(int64_t num_examples, const model::MetaData& metadata) { // Add usage tracking here. } +void OnLoadDataset(absl::string_view path) {} + +void OnSaveDataset(absl::string_view path) {} + +void OnLoadModel(absl::string_view path) {} + +void OnSaveModel(absl::string_view path) {} + void EnableUsage(bool usage) {} } // namespace usage From 5d56fb865ffc092f86eda16f718a39f20ebcce49 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Fri, 31 May 2024 06:25:38 -0700 Subject: [PATCH 13/30] Create a npm package for the js port of ydf. PiperOrigin-RevId: 639014915 --- documentation/public/mkdocs.yml | 4 +- .../port/javascript/README.md | 5 +- .../port/javascript/npm/README.md | 164 + .../port/javascript/npm/dist/readme.txt | 3 + .../port/javascript/npm/package-lock.json | 3823 +++++++++++++++++ .../port/javascript/npm/package.json | 36 + .../javascript/npm/test/inference.test.js | 96 + .../port/javascript/npm/test/model_1.zip | Bin 0 -> 7229 bytes .../port/javascript/npm/test/model_2.zip | Bin 0 -> 1476 bytes 9 files changed, 4126 insertions(+), 5 deletions(-) create mode 100644 yggdrasil_decision_forests/port/javascript/npm/README.md create mode 100644 yggdrasil_decision_forests/port/javascript/npm/dist/readme.txt create mode 100644 yggdrasil_decision_forests/port/javascript/npm/package-lock.json create mode 100644 yggdrasil_decision_forests/port/javascript/npm/package.json create mode 100644 yggdrasil_decision_forests/port/javascript/npm/test/inference.test.js create mode 100644 yggdrasil_decision_forests/port/javascript/npm/test/model_1.zip create mode 100644 yggdrasil_decision_forests/port/javascript/npm/test/model_2.zip diff --git a/documentation/public/mkdocs.yml b/documentation/public/mkdocs.yml index c6edbdc2..0b246796 100644 --- a/documentation/public/mkdocs.yml +++ b/documentation/public/mkdocs.yml @@ -91,14 +91,14 @@ nav: # - example distance: tutorial/example_distance.ipynb # TODO: model inspection, manual tree creation, custom loss. - Other APIs: - - TensorFlow Decision Forests: https://www.tensorflow.org/decision_forests - CLI quickstart: cli_quickstart.md + - JavaScript: https://www.npmjs.com/package/yggdrasil-decision-forests - CLI & C++ user manual: cli_user_manual.md - CLI commands: cli_commands.md - CLI examples: https://github.com/google/yggdrasil-decision-forests/tree/main/examples - C++ examples: https://github.com/google/yggdrasil-decision-forests/tree/main/examples/standalone + - TensorFlow Decision Forests: https://www.tensorflow.org/decision_forests - Go: https://github.com/google/yggdrasil-decision-forests/tree/main/yggdrasil_decision_forests/port/go - - JavaScript: https://github.com/google/yggdrasil-decision-forests/tree/main/yggdrasil_decision_forests/port/javascript - Changelog: changelog.md - Long-time support: lts.md - Contact: contact.md diff --git a/yggdrasil_decision_forests/port/javascript/README.md b/yggdrasil_decision_forests/port/javascript/README.md index e6419fde..5b33b0b8 100644 --- a/yggdrasil_decision_forests/port/javascript/README.md +++ b/yggdrasil_decision_forests/port/javascript/README.md @@ -3,7 +3,6 @@ The JavaScript API makes it possible to run an Yggdrasil Decision Forests model or a TensorFlow Decision Forests model in a webpage. -## Documentation - -Check the [documentation](https://ydf.readthedocs.io/en/latest/js_serving.html) +Check the +[yggdrasil-decision-forests npm package](https://www.npmjs.com/package/yggdrasil-decision-forests) for details. diff --git a/yggdrasil_decision_forests/port/javascript/npm/README.md b/yggdrasil_decision_forests/port/javascript/npm/README.md new file mode 100644 index 00000000..858c4980 --- /dev/null +++ b/yggdrasil_decision_forests/port/javascript/npm/README.md @@ -0,0 +1,164 @@ +# YDF in JS + +With this package, you can generate predictions of machine learning models +trained with [YDF](https://ydf.readthedocs.io) in the browser and with NodeJS. + +## Usage example + +First, let's train a machine learning model in python. For more details, read +[YDF's documentation](https://ydf.readthedocs.io). + +In Python in a Colab or in a Jupyter Notebook, run: + +```python +# Install YDF +!pip install ydf pandas + +import ydf +import pandas as pd + +# Download a training dataset +ds_path = "https://raw.githubusercontent.com/google/yggdrasil-decision-forests/main/yggdrasil_decision_forests/test_data/dataset/" +train_ds = pd.read_csv(ds_path + "adult_train.csv") + +# Train a Gradient Boosted Trees model +learner = ydf.GradientBoostedTreesLearner(label="income", pure_serving_model=True) +model = learner.train(train_ds) + +# Save the model +model.save("/tmp/my_model") + +# Zip the model +# Important: Use -j to not include the directory structure. +!zip -rj /tmp/my_model.zip /tmp/my_model +``` + +Then: + +### Run the model with NodeJS and CommonJS + +```js +(async function (){ + // Load the YDF library + const ydf = await require("yggdrasil-decision-forests")(); + + // Load the model + const fs = require("node:fs"); + let model = await ydf.loadModelFromZipBlob(fs.readFileSync("./model.zip")); + + // Create a batch of examples. + let examples = { + "age": [39, 40, 40, 35], + "workclass": ["State-gov", "Private", "Private", "Federal-gov"], + "fnlwgt": [77516, 121772, 193524, 76845], + "education": ["Bachelors", "Assoc-voc", "Doctorate", "9th"], + "education_num": ["13", "11", "16", "5"], + "marital_status": ["Never-married", "Married-civ-spouse", "Married-civ-spouse", "Married-civ-spouse"], + "occupation": ["Adm-clerical", "Craft-repair", "Prof-specialty", "Farming-fishing"], + "relationship": ["Not-in-family", "Husband", "Husband", "Husband"], + "race": ["White", "Asian-Pac-Islander", "White", "Black"], + "sex": ["Male", "Male", "Male", "Male"], + "capital_gain": [2174, 0, 0, 0], + "capital_loss": [0, 0, 0, 0], + "hours_per_week": [40, 40, 60, 40], + "native_country": ["United-States", null, "United-States", "United-States"] + }; + + // Make predictions + let predictions = model.predict(examples); + console.log("predictions:", predictions); + + // Release model + model.unload(); +}()) +``` + +### Run the model with NodeJS and ES6 + +```js +import * as fs from "node:fs"; +import YggdrasilDecisionForests from 'yggdrasil-decision-forests'; + +// Load the YDF library +let ydf = await YggdrasilDecisionForests(); + +// Load the model +let model = await ydf.loadModelFromZipBlob(fs.readFileSync("./model.zip")); + +// Create a batch of examples. +let examples = { + "age": [39, 40, 40, 35], + "workclass": ["State-gov", "Private", "Private", "Federal-gov"], + "fnlwgt": [77516, 121772, 193524, 76845], + "education": ["Bachelors", "Assoc-voc", "Doctorate", "9th"], + "education_num": ["13", "11", "16", "5"], + "marital_status": ["Never-married", "Married-civ-spouse", "Married-civ-spouse", "Married-civ-spouse"], + "occupation": ["Adm-clerical", "Craft-repair", "Prof-specialty", "Farming-fishing"], + "relationship": ["Not-in-family", "Husband", "Husband", "Husband"], + "race": ["White", "Asian-Pac-Islander", "White", "Black"], + "sex": ["Male", "Male", "Male", "Male"], + "capital_gain": [2174, 0, 0, 0], + "capital_loss": [0, 0, 0, 0], + "hours_per_week": [40, 40, 60, 40], + "native_country": ["United-States", null, "United-States", "United-States"] +}; + +// Make predictions +let predictions = model.predict(examples); +console.log("predictions:", predictions); + +// Release model +model.unload(); +``` + +### Run the model with in Browser + +```html + + + +``` + +## For developers + +### Run unit tests + +```sh +npm test +``` + +### Update the binary bundle + +```sh +# Assume the shell is located in a clone of: +# https://github.com/google/yggdrasil-decision-forests.git + +# Compile the YDF with WebAssembly +yggdrasil_decision_forests/port/javascript/tools/build_zipped_library.sh + +# Extract the the content of `dist` in `yggdrasil_decision_forests/port/javascript/npm/dist`. +unzip dist/ydf.zip -d yggdrasil_decision_forests/port/javascript/npm/dist +``` diff --git a/yggdrasil_decision_forests/port/javascript/npm/dist/readme.txt b/yggdrasil_decision_forests/port/javascript/npm/dist/readme.txt new file mode 100644 index 00000000..3e9ae665 --- /dev/null +++ b/yggdrasil_decision_forests/port/javascript/npm/dist/readme.txt @@ -0,0 +1,3 @@ +Before submitting the npm package, populate this directory with inference.js and +inference.wasm generated by +yggdrasil_decision_forests/port/javascript/tools/build_zipped_library.sh. \ No newline at end of file diff --git a/yggdrasil_decision_forests/port/javascript/npm/package-lock.json b/yggdrasil_decision_forests/port/javascript/npm/package-lock.json new file mode 100644 index 00000000..465db7ec --- /dev/null +++ b/yggdrasil_decision_forests/port/javascript/npm/package-lock.json @@ -0,0 +1,3823 @@ +{ + "name": "yggdrasil-decision-forests", + "version": "0.0.2", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "yggdrasil-decision-forests", + "version": "0.0.2", + "license": "Apache-2.0", + "dependencies": { + "jszip": "^3.10.1" + }, + "devDependencies": { + "jest": "^29.7.0" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.24.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.2.tgz", + "integrity": "sha512-y5+tLQyV8pg3fsiln67BVLD1P13Eg4lh5RW9mF0zUuvLrv9uIQ4MCL+CRT+FTsBlBjcIan6PGsLcBN0m3ClUyQ==", + "dev": true, + "dependencies": { + "@babel/highlight": "^7.24.2", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.24.4", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.24.4.tgz", + "integrity": "sha512-vg8Gih2MLK+kOkHJp4gBEIkyaIi00jgWot2D9QOmmfLC8jINSOzmCLta6Bvz/JSBCqnegV0L80jhxkol5GWNfQ==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.24.4", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.4.tgz", + "integrity": "sha512-MBVlMXP+kkl5394RBLSxxk/iLTeVGuXTV3cIDXavPpMMqnSnt6apKgan/U8O3USWZCWZT/TbgfEpKa4uMgN4Dg==", + "dev": true, + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.24.2", + "@babel/generator": "^7.24.4", + "@babel/helper-compilation-targets": "^7.23.6", + "@babel/helper-module-transforms": "^7.23.3", + "@babel/helpers": "^7.24.4", + "@babel/parser": "^7.24.4", + "@babel/template": "^7.24.0", + "@babel/traverse": "^7.24.1", + "@babel/types": "^7.24.0", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.24.4", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.24.4.tgz", + "integrity": "sha512-Xd6+v6SnjWVx/nus+y0l1sxMOTOMBkyL4+BIdbALyatQnAe/SRVjANeDPSCYaX+i1iJmuGSKf3Z+E+V/va1Hvw==", + "dev": true, + "dependencies": { + "@babel/types": "^7.24.0", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^2.5.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.23.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.23.6.tgz", + "integrity": "sha512-9JB548GZoQVmzrFgp8o7KxdgkTGm6xs9DW0o/Pim72UDjzr5ObUQ6ZzYPqA+g9OTS2bBQoctLJrky0RDCAWRgQ==", + "dev": true, + "dependencies": { + "@babel/compat-data": "^7.23.5", + "@babel/helper-validator-option": "^7.23.5", + "browserslist": "^4.22.2", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-environment-visitor": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", + "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-function-name": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", + "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", + "dev": true, + "dependencies": { + "@babel/template": "^7.22.15", + "@babel/types": "^7.23.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-hoist-variables": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", + "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", + "dev": true, + "dependencies": { + "@babel/types": "^7.22.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.24.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.3.tgz", + "integrity": "sha512-viKb0F9f2s0BCS22QSF308z/+1YWKV/76mwt61NBzS5izMzDPwdq1pTrzf+Li3npBWX9KdQbkeCt1jSAM7lZqg==", + "dev": true, + "dependencies": { + "@babel/types": "^7.24.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.23.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz", + "integrity": "sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==", + "dev": true, + "dependencies": { + "@babel/helper-environment-visitor": "^7.22.20", + "@babel/helper-module-imports": "^7.22.15", + "@babel/helper-simple-access": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/helper-validator-identifier": "^7.22.20" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.0.tgz", + "integrity": "sha512-9cUznXMG0+FxRuJfvL82QlTqIzhVW9sL0KjMPHhAOOvpQGL8QtdxnBKILjBqxlHyliz0yCa1G903ZXI/FuHy2w==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-simple-access": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz", + "integrity": "sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==", + "dev": true, + "dependencies": { + "@babel/types": "^7.22.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-split-export-declaration": { + "version": "7.22.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", + "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", + "dev": true, + "dependencies": { + "@babel/types": "^7.22.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.24.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.1.tgz", + "integrity": "sha512-2ofRCjnnA9y+wk8b9IAREroeUP02KHp431N2mhKniy2yKIDKpbrHv9eXwm8cBeWQYcJmzv5qKCu65P47eCF7CQ==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.23.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.23.5.tgz", + "integrity": "sha512-85ttAOMLsr53VgXkTbkx8oA6YTfT4q7/HzXSLEYmjcSTJPMPQtvq1BD79Byep5xMUYbGRzEpDsjUf3dyp54IKw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.24.4", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.24.4.tgz", + "integrity": "sha512-FewdlZbSiwaVGlgT1DPANDuCHaDMiOo+D/IDYRFYjHOuv66xMSJ7fQwwODwRNAPkADIO/z1EoF/l2BCWlWABDw==", + "dev": true, + "dependencies": { + "@babel/template": "^7.24.0", + "@babel/traverse": "^7.24.1", + "@babel/types": "^7.24.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.24.2", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.2.tgz", + "integrity": "sha512-Yac1ao4flkTxTteCDZLEvdxg2fZfz1v8M4QpaGypq/WPDqg3ijHYbDfs+LG5hvzSoqaSZ9/Z9lKSP3CjZjv+pA==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.22.20", + "chalk": "^2.4.2", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/highlight/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/@babel/highlight/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/highlight/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/parser": { + "version": "7.24.4", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.4.tgz", + "integrity": "sha512-zTvEBcghmeBma9QIGunWevvBAp4/Qu9Bdq+2k0Ot4fVMD6v3dsC9WOcRSKk7tRRyBM/53yKMJko9xOatGQAwSg==", + "dev": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.24.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.24.1.tgz", + "integrity": "sha512-2eCtxZXf+kbkMIsXS4poTvT4Yu5rXiRa+9xGVT56raghjmBTKMpFNc9R4IDiB4emao9eO22Ox7CxuJG7BgExqA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.24.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.24.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.24.1.tgz", + "integrity": "sha512-Yhnmvy5HZEnHUty6i++gcfH1/l68AHnItFHnaCv6hn9dNh0hQvvQJsxpi4BMBFN5DLeHBuucT/0DgzXif/OyRw==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.24.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.0.tgz", + "integrity": "sha512-Bkf2q8lMB0AFpX0NFEqSbx1OkTHf0f+0j82mkw+ZpzBnkk7e9Ql0891vlfgi+kHwOk8tQjiQHpqh4LaSa0fKEA==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.23.5", + "@babel/parser": "^7.24.0", + "@babel/types": "^7.24.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.24.1", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.24.1.tgz", + "integrity": "sha512-xuU6o9m68KeqZbQuDt2TcKSxUw/mrsvavlEqQ1leZ/B+C9tk6E4sRWy97WaXgvq5E+nU3cXMxv3WKOCanVMCmQ==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.24.1", + "@babel/generator": "^7.24.1", + "@babel/helper-environment-visitor": "^7.22.20", + "@babel/helper-function-name": "^7.23.0", + "@babel/helper-hoist-variables": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/parser": "^7.24.1", + "@babel/types": "^7.24.0", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.0.tgz", + "integrity": "sha512-+j7a5c253RfKh8iABBhywc8NSfP5LURe7Uh4qpsh6jc+aLJguvmIUBdjSdEMQv2bENrCR5MfRdjGo7vzS/ob7w==", + "dev": true, + "dependencies": { + "@babel/helper-string-parser": "^7.23.4", + "@babel/helper-validator-identifier": "^7.22.20", + "to-fast-properties": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/console": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", + "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", + "dev": true, + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/core": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", + "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", + "dev": true, + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/reporters": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.7.0", + "jest-config": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-resolve-dependencies": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "jest-watcher": "^29.7.0", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/environment": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", + "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", + "dev": true, + "dependencies": { + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", + "dev": true, + "dependencies": { + "expect": "^29.7.0", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", + "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", + "dev": true, + "dependencies": { + "jest-get-type": "^29.6.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/fake-timers": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", + "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", + "dev": true, + "dependencies": { + "@jest/types": "^29.6.3", + "@sinonjs/fake-timers": "^10.0.2", + "@types/node": "*", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/globals": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", + "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", + "dev": true, + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/types": "^29.6.3", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/reporters": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", + "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", + "dev": true, + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "@jest/console": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "@types/node": "*", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-instrument": "^6.0.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^4.0.0", + "istanbul-reports": "^3.1.3", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "slash": "^3.0.0", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", + "v8-to-istanbul": "^9.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/reporters/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@jest/reporters/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@jest/reporters/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "dev": true, + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/source-map": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", + "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-result": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", + "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", + "dev": true, + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-sequencer": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", + "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", + "dev": true, + "dependencies": { + "@jest/test-result": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/transform": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", + "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", + "dev": true, + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "dev": true, + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", + "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "dev": true + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.6.8", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", + "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", + "dev": true, + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.5.tgz", + "integrity": "sha512-WXCyOcRtH37HAUkpXhUduaxdm82b4GSlyTqajXviN4EfiuPgNYR109xMCKvpl6zPIpua0DGlMEDCq+g8EdoheQ==", + "dev": true, + "dependencies": { + "@babel/types": "^7.20.7" + } + }, + "node_modules/@types/graceful-fs": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "dev": true, + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "dev": true, + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/node": { + "version": "20.12.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.7.tgz", + "integrity": "sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", + "dev": true + }, + "node_modules/@types/yargs": { + "version": "17.0.32", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.32.tgz", + "integrity": "sha512-xQ67Yc/laOG5uMfX/093MRlGGCIBzZMarVa+gfNKJxWAIgykYpVGkBdbqEzGDDfCrVUj6Hiff4mTZ5BA6TmAog==", + "dev": true, + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "dev": true + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/babel-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", + "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", + "dev": true, + "dependencies": { + "@jest/transform": "^29.7.0", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^6.1.1", + "babel-preset-jest": "^29.6.3", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.8.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", + "test-exclude": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "dev": true, + "dependencies": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", + "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", + "dev": true, + "dependencies": { + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14", + "@types/babel__traverse": "^7.0.6" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/babel-preset-current-node-syntax": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.0.1.tgz", + "integrity": "sha512-M7LQ0bxarkxQoN+vz5aJPsLBn77n8QgTFmo8WK0/44auK2xlCXrYcUxHFxgU7qW5Yzw/CjmLRK2uJzaCd7LvqQ==", + "dev": true, + "dependencies": { + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-bigint": "^7.8.3", + "@babel/plugin-syntax-class-properties": "^7.8.3", + "@babel/plugin-syntax-import-meta": "^7.8.3", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.8.3", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.8.3", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-top-level-await": "^7.8.3" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/babel-preset-jest": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", + "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", + "dev": true, + "dependencies": { + "babel-plugin-jest-hoist": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz", + "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001587", + "electron-to-chromium": "^1.4.668", + "node-releases": "^2.0.14", + "update-browserslist-db": "^1.0.13" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "dev": true, + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001613", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001613.tgz", + "integrity": "sha512-BNjJULJfOONQERivfxte7alLfeLW4QnwHvNW4wEcLEbXfV6VSCYvr+REbf2Sojv8tC1THpjPXBxWgDbq4NtLWg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ] + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "engines": { + "node": ">=8" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.3.1.tgz", + "integrity": "sha512-a3KdPAANPbNE4ZUv9h6LckSl9zLsYOP4MBmhIPkRaeyybt+r4UghLvq+xw/YwUcC1gqylCkL4rdVs3Lwupjm4Q==", + "dev": true + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/collect-v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", + "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==", + "dev": true + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "node_modules/create-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", + "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", + "dev": true, + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "prompts": "^2.0.1" + }, + "bin": { + "create-jest": "bin/create-jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/debug/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/dedent": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", + "integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==", + "dev": true, + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/detect-newline": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", + "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/diff-sequences": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", + "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", + "dev": true, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.4.750", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.750.tgz", + "integrity": "sha512-9ItEpeu15hW5m8jKdriL+BQrgwDTXEL9pn4SkillWFu73ZNNNQ2BKKLS+ZHv2vC9UkNhosAeyfxOf/5OSeTCPA==", + "dev": true + }, + "node_modules/emittery": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", + "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sindresorhus/emittery?sponsor=1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/escalade": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", + "dev": true, + "dependencies": { + "@jest/expect-utils": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "node_modules/fb-watchman": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", + "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", + "dev": true, + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" + }, + "node_modules/import-local": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", + "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==", + "dev": true, + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true + }, + "node_modules/is-core-module": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "dev": true, + "dependencies": { + "hasown": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", + "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.2.tgz", + "integrity": "sha512-1WUsZ9R1lA0HtBSohTkm39WTPlNKSJ5iFk7UwqXkBLoHQT+hfqPsfsTDVuZdKGaBwn7din9bS7SsnoAr943hvw==", + "dev": true, + "dependencies": { + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", + "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", + "dev": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", + "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", + "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", + "dev": true, + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-changed-files": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", + "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", + "dev": true, + "dependencies": { + "execa": "^5.0.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-circus": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", + "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", + "dev": true, + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "co": "^4.6.0", + "dedent": "^1.0.0", + "is-generator-fn": "^2.0.0", + "jest-each": "^29.7.0", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0", + "pretty-format": "^29.7.0", + "pure-rand": "^6.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-cli": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", + "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", + "dev": true, + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "create-jest": "^29.7.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "yargs": "^17.3.1" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-cli/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/jest-cli/node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/jest-cli/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/jest-config": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", + "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", + "dev": true, + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/test-sequencer": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-jest": "^29.7.0", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "deepmerge": "^4.2.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-circus": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "micromatch": "^4.0.4", + "parse-json": "^5.2.0", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@types/node": "*", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/jest-config/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/jest-config/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-config/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/jest-diff": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", + "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", + "dev": true, + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "^29.6.3", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-docblock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", + "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", + "dev": true, + "dependencies": { + "detect-newline": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-each": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", + "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", + "dev": true, + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "jest-util": "^29.7.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", + "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", + "dev": true, + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-get-type": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", + "dev": true, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-haste-map": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", + "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", + "dev": true, + "dependencies": { + "@jest/types": "^29.6.3", + "@types/graceful-fs": "^4.1.3", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "micromatch": "^4.0.4", + "walker": "^1.0.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/jest-leak-detector": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", + "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", + "dev": true, + "dependencies": { + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-matcher-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", + "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", + "dev": true, + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-mock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", + "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", + "dev": true, + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-pnp-resolver": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", + "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", + "dev": true, + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "jest-resolve": "*" + }, + "peerDependenciesMeta": { + "jest-resolve": { + "optional": true + } + } + }, + "node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", + "dev": true, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", + "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", + "dev": true, + "dependencies": { + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-pnp-resolver": "^1.2.2", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "resolve": "^1.20.0", + "resolve.exports": "^2.0.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve-dependencies": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", + "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", + "dev": true, + "dependencies": { + "jest-regex-util": "^29.6.3", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runner": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", + "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", + "dev": true, + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/environment": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "graceful-fs": "^4.2.9", + "jest-docblock": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-leak-detector": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-resolve": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-util": "^29.7.0", + "jest-watcher": "^29.7.0", + "jest-worker": "^29.7.0", + "p-limit": "^3.1.0", + "source-map-support": "0.5.13" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runtime": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", + "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", + "dev": true, + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/globals": "^29.7.0", + "@jest/source-map": "^29.6.3", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "cjs-module-lexer": "^1.0.0", + "collect-v8-coverage": "^1.0.0", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0", + "strip-bom": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runtime/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/jest-runtime/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-runtime/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/jest-snapshot": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", + "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", + "dev": true, + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "natural-compare": "^1.4.0", + "pretty-format": "^29.7.0", + "semver": "^7.5.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-snapshot/node_modules/semver": { + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", + "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", + "dev": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-snapshot/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "dev": true, + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", + "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", + "dev": true, + "dependencies": { + "@jest/types": "^29.6.3", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "leven": "^3.1.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-watcher": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", + "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", + "dev": true, + "dependencies": { + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "^29.7.0", + "string-length": "^4.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", + "dev": true, + "dependencies": { + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "node_modules/jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true, + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", + "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", + "dev": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-dir/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "node_modules/makeerror": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", + "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", + "dev": true, + "dependencies": { + "tmpl": "1.0.5" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true + }, + "node_modules/micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "dev": true, + "dependencies": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true + }, + "node_modules/node-releases": { + "version": "2.0.14", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", + "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", + "dev": true + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pirates": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", + "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dev": true, + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "dev": true, + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ] + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "dev": true + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "dev": true, + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve.exports": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", + "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "dev": true + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "dev": true, + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/stack-utils/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-length": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", + "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", + "dev": true, + "dependencies": { + "char-regex": "^1.0.2", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-exclude/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/test-exclude/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/test-exclude/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/tmpl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", + "dev": true + }, + "node_modules/to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, + "node_modules/update-browserslist-db": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", + "integrity": "sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "node_modules/v8-to-istanbul": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.2.0.tgz", + "integrity": "sha512-/EH/sDgxU2eGxajKdwLCDmQ4FWq+kpi3uCmBGpw1xJtnAxEjlD8j8PEiGWpCIMIs3ciNAgH0d3TTJiUkYzyZjA==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/walker": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", + "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", + "dev": true, + "dependencies": { + "makeerror": "1.0.12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/yggdrasil_decision_forests/port/javascript/npm/package.json b/yggdrasil_decision_forests/port/javascript/npm/package.json new file mode 100644 index 00000000..21b40d47 --- /dev/null +++ b/yggdrasil_decision_forests/port/javascript/npm/package.json @@ -0,0 +1,36 @@ +{ + "name": "yggdrasil-decision-forests", + "version": "0.0.2", + "description": "With this package, you can generate predictions of machine learning models trained with YDF in browser and with NodeJS.", + "main": "dist/inference.js", + "scripts": { + "test": "jest" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/google/yggdrasil-decision-forests.git" + }, + "keywords": [ + "ydf", + "machine-learning", + "random-forest", + "gradient-boosting", + "tabular-data", + "interpretable", + "decision-forest", + "decision-tree", + "tensorflow-decision-forest" + ], + "author": "Mathieu Guillame-Bert", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/google/yggdrasil-decision-forests/issues" + }, + "homepage": "https://ydf.readthedocs.io", + "dependencies": { + "jszip": "^3.10.1" + }, + "devDependencies": { + "jest": "^29.7.0" + } +} diff --git a/yggdrasil_decision_forests/port/javascript/npm/test/inference.test.js b/yggdrasil_decision_forests/port/javascript/npm/test/inference.test.js new file mode 100644 index 00000000..a931c61b --- /dev/null +++ b/yggdrasil_decision_forests/port/javascript/npm/test/inference.test.js @@ -0,0 +1,96 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fs = require('node:fs'); + +describe('YDF Inference', () => { + let ydf = null; + let model1 = null; + let model2 = null; + + beforeAll(async () => { + ydf = await require('yggdrasil-decision-forests')(); + + model1 = + await ydf.loadModelFromZipBlob(fs.readFileSync('./test/model_1.zip')); + model2 = + await ydf.loadModelFromZipBlob(fs.readFileSync('./test/model_2.zip')); + }); + + it('loadModelFromZipBlob', () => { + expect(model1).not.toBeNull(); + expect(model2).not.toBeNull(); + }); + + it('predict model1', async () => { + let predictions = model1.predict({ + 'age': [39, 40, 40, 35], + 'workclass': ['State-gov', 'Private', 'Private', 'Federal-gov'], + 'fnlwgt': [77516, 121772, 193524, 76845], + 'education': ['Bachelors', 'Assoc-voc', 'Doctorate', '9th'], + 'education_num': [13, 11, 16, 5], + 'marital_status': [ + 'Never-married', 'Married-civ-spouse', 'Married-civ-spouse', + 'Married-civ-spouse' + ], + 'occupation': + ['Adm-clerical', 'Craft-repair', 'Prof-specialty', 'Farming-fishing'], + 'relationship': ['Not-in-family', 'Husband', 'Husband', 'Husband'], + 'race': ['White', 'Asian-Pac-Islander', 'White', 'Black'], + 'sex': ['Male', 'Male', 'Male', 'Male'], + 'capital_gain': [2174, 0, 0, 0], + 'capital_loss': [0, 0, 0, 0], + 'hours_per_week': [40, 40, 60, 40], + 'native_country': + ['United-States', null, 'United-States', 'United-States'] + }); + console.log('Predictions:', predictions); + + expect(predictions).toEqual([ + 0.13323983550071716, + 0.47678571939468384, + 0.818461537361145, + 0.4974619150161743, + ]); + }); + + it('predict model2', async () => { + let predictions = model2.predict({ + 'f1': [0, 0, 0, 0, 0, 0, 0, 0], + 'f2': [ + ['RED', 'BLUE'], ['GREEN'], [], ['RED', 'BLUE', 'GREEN'], ['BLUE'], [], + ['RED'], ['BLUE', 'RED'] + ], + 'f3': + [['X'], ['Y'], [], ['X', 'Y', 'Z'], ['X'], ['Z', 'Y'], ['Y'], ['Z']], + }); + console.log('Predictions:', predictions); + expect(predictions).toEqual([ + 0.4690462052822113, + 0.4563983976840973, + 0.4563983976840973, + 0.5488502383232117, + 0.4563983976840973, + 0.5943315029144287, + 0.4690462052822113, + 0.5488502383232117, + ]); + }); + + afterAll(async () => { + model1.unload(); + model2.unload(); + }); +}); diff --git a/yggdrasil_decision_forests/port/javascript/npm/test/model_1.zip b/yggdrasil_decision_forests/port/javascript/npm/test/model_1.zip new file mode 100644 index 0000000000000000000000000000000000000000..875c79f073f00f19a381c8b7bb6a8b8e691fcd63 GIT binary patch literal 7229 zcma)BWmH^Cx@~AIIKiQDXe7A11PB&9xJ%;>9h^o&kdWXIJh%jR3xtb11VTe_2oM?q zp@TN|xHE69x%aI#^QO+KbLz*bed^S=tM;n>>1Y5k$N&HU4glR;%%H}Oim3|=007|v z0Qdk9z~08s#>&?VV#n)cYw+R`07x3jWA|r;2H*kEFrxIa02umKXaEaCS2uGN^0l+0 z)!{|UM6rfcFqBrz9Z^D)&5gN&*BmCt#s0GCrCwOgP*YP^@k+3x)WsUqkdZ`}iJ3_W zL-|xim)Iq}3PGeA*!-9&ni&O*ljU%EamQ38TalC`ACAtmH*r=r-)rc*yYl;T%D>0N zM}^5^>Q|YxkFUV*!Awio=D^{+Di<^?h&vs1GI1Hz!p;Ik)^Z{rdC?Mg*DxP_%{Ir} zuzGYXb(7b|_W{Nw^3vmMaPXv8m>yN%B#TO?K+5@58IFBpRPT(b z8iI=qCCsRb8oF{$>!??)3!R*QM$i_nkFQC006K*#Nj zn)sA7bhUy&tc2=##+SJ*#J`o4=UbL8h>%WQ&@17yA6ey(=%Wo^h+K>zxoJA?FK6hv zehKR9xV9PCD;r+Vl14VYNwhB?!3arm#aJ^GLZ6N)xV;_w%`GRBg1WxNvx}a-w-3B= z`5C9Nc?Q!?F5WP4*W>A4d7-A5*l?9M?Cf=d$|FEJ=9~2jm5o- zd+$>ENE=xQev|TzCplUm(?P8ifH67~Qb|JQxti`6D8o?;)(Z>KFWU-qjfr!QX?B$- z4!(ukB)kJSH;%u?_M$1pK7`J!oU{8atg4Lpelv=AEN#sWky<-V3-3Y;A`0sSYvPyR z5<5|HD{yO;EUB>xQ~COlXY)+>!1Uqd*yA;RIF!}`SQQ~hh0gEk`}p=`40kLYyK2!b zyfj$b%G&U#BgO!yY_-go!!oCaw8p^=z0N@WAWG8(Gjn&C4_+yi?W`&%@l~kb>9()N z_6gcrYfa|9jXx2!sTkJ=)KadwRxFS-SX>pGIJH#V$!89;pDt}9#(-}5Cqqer5>QFs9~RJJ2XdpyAN2B{bxH@{^;G z8*iWym-3_gXq+~*hh~^uitD7(Lo5dUaq@0(LsT-h!(m%fntWf{Pq>|g!-Uwk2rjP(bxrM){i~cl&i0P@o05@X|9V`p{ngaU}%I z;!PC-b|Rd$nghq@fFM7?^0eFl4H;2TaBtpuaVeKyvrM6sGa8;msXdt&1Af;J42OL8 zKH?4IB618Hv=L|a03k2@sRMKOrL3Q2*zCPIOfcmuuxV_2-j9o%iuKYp^uvB-GGaZ! zRR=DdW#&fH8$R^Qw8u*x8innY(4SNHeq`i5PywrlJ@Fr1!hXP0S8mf3v-5{`58S{OqZv6%wwmn)ZHcg5dr*=c0%=R*MOpdBo$(Q9fm&W;q1kPq1dW=mvPT(}h zC%op4vU>X~=(x7X^ZWCr%daQj-w%_xKH6E#@OtrFa%G^YifVQ5vOSo zJ^n_1H0)p`tdk`IWsm4!FfmHEj8f~JL|@LWzq%J~Po0^ecr15?msVzV{djCK@>TL` z?E7ME$#XWP(xq0&{#ov=|J-hpq!uMzIf61r&k)zon8UQed4gfrQ=VvO8JS7)F!)G^u~m-_p%MN>=@cO@EwA-QG109LBcWJQe|=49{4(01 zkKBjx+Zzk>C?;M3Evo+X`kZs<}_^~0VM$cH{(FrcJ=&=9(RiH7`?v4TPT5Uf zR<0&eVJJVzcxmP__iSbR{8e=TDiyqF_+79>Hqf=Xu8#G|Lg6>%Ex|0whVY8(PJ&#X zw3?9u$yR2h6{pMEIl#o5H*Q;9Uu6Ai7<|b(4p9wlY953umt0{)y{`xq(qC7fH=`H* zbzOaWZ8e$A0!wZ$_-)yAiru9KvQxefb13k`MT`zN_GJ zA@O`%tt@Wko`4pipT;XBPkrLbVY<%yW^4R7V}`wn)l=9(J1#)EQL;=F0__RxND5@O zR?n1{sH}h=rn@g#Vpz>P+-~3jr4C$Vq&(w>O}KcPp)_A7b~iLv zT;!%YFaRX;WZC7c9@PsDZyXJi#O;cuVS-L3F3ZS7R*ZW;5gJg=t|K`kP$JO4mrfNh zO7X8{zOt2--w~bm5$$HhA{!&o#j6cmfCuW{>2HjLplv;6K5|u!n?Nc|iQPCVqXp+r zSh`%Y#}yB%#96MFlM;hA^lWCg_ubb`8t%G;V;Qi6+9{53H4`loy$SJ5^r-h&p;_tvT$f}l_vFYz2FJ^`BJ09_Wzw);CFJYv$!`wm z3US4|FhmpmsCD7_=DYhce!_SR{5bK$g%<)v??}%!-9_R0$4x}q3t2V~)37rp2j6dN zrv$&gw64Q6_Z_=<+Y@mDe)<`Q zf^AfHHL<15=EdU{8xaZ5WRYDl9ISZF$B)Z~F%8aa2Ue(yJYyX6zx0DeL(q z{*$#&uQUUU+FiIJ+D4gUD*_#>LS&NXdqx>Wwdb$F-I%;g-sjZ1@~gN6TdQBvnf=wgLqrccB_!6qC-ZeefG0&$G)9Tl3|h+ z;VvX?AP4T1zew+a{Al}Y$}5w}K=zJBo1xU2@F$kSEZ1Z;>Sr!VL8D=z(q@`OJ?At; zRWr;zuiViXap{Kd9_SM-FEO%t716}`0_$sZyjbKA={<~wvBPtrj<;Hmuih`6~mz5iU#`1;3A_+3BTuIL7PW85@A3D0f! zeMF;#R(3Pux%CMrSnw`H%ea8Z?m;%^gM&l!XbB9(96U_d7zwo-mxm2K-t{?Ul8~F+ zS%o1$KFx>ER=Tkao|i^w$hKK#$|<009oYn(eAF|uI}tt)?dO`ih2{gAi6q~uR8lFa zhDsxw>u5(V>Ln$!^D-p+W~-%TIkMduU0%)q(g0bBrSy4<#Kx)Bxr^ww`7mATRHVlD zkz{i)f2eh`+}h{O&O5VB^|p^qNsi%>)&A%??3T(j^$o-;An~0MhN$Kd1Eypg6j40V z6**Xx)%Ik9f?kwA(LX2WJ%|M6`ti(u-}S_DiaM>-e7?i6_%xfbXbt62Sl9Mj+AU5& zXTe_9`HzOaW*XxsL^Zp{-u=ZUCCBhrb0w5fKc$JxZ0Ap*ko=z1XXSYKwm!{njGB>` z{4yc$>;x)aQtKw({4y&9=<@J2(2CdOVU99F?HV;ctvWJ5jKq!X)ZcB%nsTFCu}nae`iR!C;WqhKSG^zwqQXE~+I8U(p>b`Yi1+V5( zk##0ieA(A{$`lqWMZ|SAjlKDuf72K5q&p^i$vd5qy^+%pbdfUPNpKc)Qu%h~hhWj& zb^3uK_6OZH3h*u--yuxD4N?X?;{_r%pJ#qwY0g`$+jZx7pm1es1B{(!1A_ptwHcOcIe0Q}0h@X#DwB zR5?9mIJ}m4O27^*|TEobm;ebuF!9cHkl(=zFDr#u1OJeDW$RDv+G^0W}jZ?Xr z#{l2g5)7KOjQSw0Vq)9T75IAL=B-Y|`|gSxiLYchmAyUQQ3VJCx#{N`t1q$r5f{4|HZij)pSHUmVybntee7*NjqLE6Br*2M}V z4Uh9v8KqAnj9|VLp#O}5ZNJ1bX<2ClKL_v1=+lc4@21&!_Dg1TN}4Jf7u#PS#$&mH z5TV+9CEuzCKS%9<#7ayQ*gbO{IDMnAboX6Fe1yI#-p8hJJba8nNBVStCK(hV!XZoWsgdu*GD80LcZse%(IxU0bJs)l*$5%3scY|4jj#x4H8V^2sF*P%)nR ztNCFIu^<#qW&SPQ1A9RE?j74J)$9~8)hQuAnzQ)I_<$5|Tt64yN|)+S3l(4FBT-_`MDjF{WA!;Vj-~L%OJ3btJR~rT>DpW z&P#5_#+}K!G6+|zEqBZ7w^m(YG$Er9rH2z7eHC5rDYdKf(#lOIXT?#<;D?n6-ukVt ziNBf(UYf}mw~*ckc+li-h{^qY_I1S!@epHpP=mQtGLKvo)3D&m(D1SFuVQ=w44O{9 zwZ`mna#uyBdi2g8NG!CL&7Tju7d;|6$7p-aaWqF!|A)1kt?A+D^A2y*4Z$KL(>T1P zS)vJ_JE>+8&QGdW!*kG=XdTH8YI+L%wzl(D=RmNi=kg%0hnH2RieSEsoe-;1b2IiT zooePDprCYnZu8R>WkoZZyh`j(og37FIzE0eC*?{kkP7TORkrwbL~mrmAxS1$cq`X+ zm1X3!9L!skLTIr)KK3zUj6LR2W3ayUa$^RNmOduW2ZuBI0RjN{ E9~mYYlK=n! literal 0 HcmV?d00001 diff --git a/yggdrasil_decision_forests/port/javascript/npm/test/model_2.zip b/yggdrasil_decision_forests/port/javascript/npm/test/model_2.zip new file mode 100644 index 0000000000000000000000000000000000000000..7a5f435ebcf78ac4b1e775ee7d7f5e0d977df0e6 GIT binary patch literal 1476 zcmWIWW@Zs#W?$x^!(AZe0pgUzlEnDpg4ATaf~3bm4-y2E`uo2Z zmK1$86cS6AmJslRpSee>hp#6_OQy%kC$lZeXbw++0+&*XV~dhc$LaQF(G{mooM>$h zY;bJ~e8{w)tyxrGe0A^s{oQi~xY?T-nnlgUizlq+p1>TEboB7iM^A)baXm3)taO9l-UQw3P7A*l$es4npYB^l%HQ*lA01{vvad;K%eK8-`A8T#4tsiJ9gv823w!L$Sd5D?+$*NEdM58) zQ&7wunKMm10uLCT7&uzWzj5ShZscm7`?onpkW2K9)Bf9sMbrwqs%}PS78N%8l<@TN z1C3_Xc9meh)NXLO!TZEzeVu#PuU^!@cm1;dy`!cIN0?V7f7KGGV7-#Oft@=1+zmR*!?56Upz@$PZW;8!4b^m-M1oWRD69aWuiddK=K3cXp&FfaAfwj{Q83*BoW&RdlO7-s?k^j9=t!`ucER9DC4seB5 ze(UL%by{j0@|3&UbH~gx0aL&2%Y9}rwcO``R=LZOKO5aY@%ud8=d|NgvRlGbVFxY4 z+B#cPp>>}oGDbXQo#EATex`z5y3@Cv-@CZQ#B>7Yb1V4h?7x`9Yr1i59_J00=Cj53 z4*7^UaC?jS&e)OJlia450r&sdPG#Ud%^dtDzwz*=14gaSBb-hgKE@?A zjd$hq_cxkDrRpp)^*PNa3$FUK$n$gF3T^(QI*hX;KluMLJv5a|2N;HoO!mx(l!BU( zRe<3L0tyWa8bLI2w&nuo0tN;~1__3SJ+47}HE%L9taS(l(J(&5bX>U%+4O@T(}8&o zo2f9hI`KHT-~#}TkoRo> literal 0 HcmV?d00001 From b94e0255c57e2d8bdef201db8518126a673c08de Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 3 Jun 2024 08:27:41 -0700 Subject: [PATCH 14/30] SKLearn to YDF model converter | Unit tests (part 1) PiperOrigin-RevId: 639792213 --- .../port/python/dev_requirements.txt | 1 + .../port/python/ydf/model/BUILD | 21 +++ .../port/python/ydf/model/export_sklearn.py | 21 +++ .../python/ydf/model/sklearn_model_test.py | 153 ++++++++++++++++++ 4 files changed, 196 insertions(+) create mode 100644 yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py create mode 100644 yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py diff --git a/yggdrasil_decision_forests/port/python/dev_requirements.txt b/yggdrasil_decision_forests/port/python/dev_requirements.txt index 5642d727..d9d73891 100644 --- a/yggdrasil_decision_forests/port/python/dev_requirements.txt +++ b/yggdrasil_decision_forests/port/python/dev_requirements.txt @@ -3,6 +3,7 @@ tensorflow_decision_forests; platform_machine != 'aarch64' and python_version >= tensorflow; platform_machine != 'aarch64' portpicker matplotlib +scikit-learn jax; platform_machine != 'aarch64' and platform_system != 'Windows' jaxlib; platform_machine != 'aarch64' and platform_system != 'Windows' optax; platform_machine != 'aarch64' and platform_system != 'Windows' and python_version >= '3.9' diff --git a/yggdrasil_decision_forests/port/python/ydf/model/BUILD b/yggdrasil_decision_forests/port/python/ydf/model/BUILD index 37ff4baa..b5d0804c 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/model/BUILD @@ -136,6 +136,14 @@ py_library( ], ) +# Note: This build rule does not depends on SKLearn. To use its functionalities, SKLearn needs to be +# imported manually by the call i.e. //third_party/py/sklearn. +py_library( + name = "export_sklearn", + srcs = ["export_sklearn.py"], + deps = [":generic_model"], +) + py_library( name = "model_lib", srcs = ["model_lib.py"], @@ -351,6 +359,19 @@ py_test( ], ) +py_test( + name = "sklearn_model_test", + srcs = ["sklearn_model_test.py"], + python_version = "PY3", + deps = [ + ":export_sklearn", + # absl/testing:absltest dep, + # absl/testing:parameterized dep, + # numpy dep, + # sklearn dep, + ], +) + py_test( name = "benchmark_test", srcs = ["benchmark_test.py"], diff --git a/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py b/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py new file mode 100644 index 00000000..2bf82a7f --- /dev/null +++ b/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py @@ -0,0 +1,21 @@ +# Copyright 2022 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Import and export sklearn models.""" + +from ydf.model import generic_model + + +def from_sklearn(sklearn_model) -> generic_model.GenericModel: + raise NotImplementedError("from_sklearn not implemented") diff --git a/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py b/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py new file mode 100644 index 00000000..9678092e --- /dev/null +++ b/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py @@ -0,0 +1,153 @@ +# Copyright 2022 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from absl.testing import absltest +from absl.testing import parameterized +import numpy as np +from sklearn import datasets +from sklearn import ensemble +from sklearn import linear_model +from sklearn import tree +from ydf.model import export_sklearn + + +class ScikitLearnModelConverterTest(parameterized.TestCase): + + @parameterized.parameters( + (tree.DecisionTreeRegressor(random_state=42),), + (tree.ExtraTreeRegressor(random_state=42),), + (ensemble.RandomForestRegressor(random_state=42),), + (ensemble.ExtraTreesRegressor(random_state=42),), + ( + ensemble.GradientBoostingRegressor( + random_state=42, + ), + ), + (ensemble.GradientBoostingRegressor(random_state=42, init="zero"),), + ( + ensemble.GradientBoostingRegressor( + random_state=42, + init=tree.DecisionTreeRegressor(random_state=42), + ), + ), + ) + def DISABLED_test_import_regression_model( + self, + sklearn_model, + ): + features, labels = datasets.make_regression( + n_samples=100, + n_features=10, + random_state=42, + ) + sklearn_model.fit(features, labels) + sklearn_predictions = sklearn_model.predict(features).astype(np.float32) + + ydf_model = export_sklearn.from_sklearn(sklearn_model) + ydf_predictions = ydf_model.predict(features) + + np.testing.assert_allclose(sklearn_predictions, ydf_predictions, rtol=1e-4) + + @parameterized.parameters( + (tree.DecisionTreeClassifier(random_state=42),), + (tree.ExtraTreeClassifier(random_state=42),), + (ensemble.RandomForestClassifier(random_state=42),), + (ensemble.ExtraTreesClassifier(random_state=42),), + ) + def DISABLED_test_import_classification_model( + self, + sklearn_model, + ): + features, labels = datasets.make_classification( + n_samples=100, + n_features=10, + n_classes=4, + n_clusters_per_class=1, + random_state=42, + ) + sklearn_model.fit(features, labels) + sklearn_predictions = sklearn_model.predict_proba(features).astype( + np.float32 + ) + + ydf_model = export_sklearn.from_sklearn(sklearn_model) + ydf_predictions = ydf_model.predict(features) + np.testing.assert_allclose(sklearn_predictions, ydf_predictions, rtol=1e-5) + + def DISABLED_test_import_raises_when_unrecognised_model_provided(self): + features, labels = datasets.make_regression( + n_samples=100, + n_features=10, + random_state=42, + ) + sklearn_model = linear_model.LinearRegression().fit(features, labels) + with self.assertRaises(NotImplementedError): + export_sklearn.from_sklearn(sklearn_model) + + def DISABLED_test_import_raises_when_sklearn_model_is_not_fit(self): + with self.assertRaises( + ValueError, + msg="Scikit-learn model must be fit to data before converting to TF.", + ): + _ = export_sklearn.from_sklearn(tree.DecisionTreeRegressor()) + + def DISABLED_test_import_raises_when_regression_target_is_multivariate(self): + features, labels = datasets.make_regression( + n_samples=100, + n_features=10, + # This produces a two-dimensional target variable. + n_targets=2, + random_state=42, + ) + sklearn_model = tree.DecisionTreeRegressor().fit(features, labels) + with self.assertRaisesRegex( + ValueError, + "Only scalar regression and single-label classification are supported.", + ): + _ = export_sklearn.from_sklearn(sklearn_model) + + def DISABLED_test_import_raises_when_classification_target_is_multilabel( + self, + ): + features, labels = datasets.make_multilabel_classification( + n_samples=100, + n_features=10, + # This assigns two class labels per example. + n_labels=2, + random_state=42, + ) + sklearn_model = tree.DecisionTreeClassifier().fit(features, labels) + with self.assertRaisesRegex( + ValueError, + "Only scalar regression and single-label classification are supported.", + ): + _ = export_sklearn.from_sklearn(sklearn_model) + + def DISABLED_test_convert_raises_when_gbt_initial_estimator_is_not_tree_or_constant( + self, + ): + features, labels = datasets.make_regression( + n_samples=100, + n_features=10, + random_state=42, + ) + init_estimator = linear_model.LinearRegression() + sklearn_model = ensemble.GradientBoostingRegressor(init=init_estimator) + sklearn_model.fit(features, labels) + with self.assertRaises(ValueError): + _ = export_sklearn.from_sklearn(sklearn_model) + + +if __name__ == "__main__": + absltest.main() From 9ecb005a7d61bc44b28a1023e795747d318f756d Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 3 Jun 2024 08:32:50 -0700 Subject: [PATCH 15/30] SKLearn to YDF model converter | Converter (part 2) PiperOrigin-RevId: 639794115 --- .../port/python/ydf/model/BUILD | 11 +- .../port/python/ydf/model/export_sklearn.py | 379 +++++++++++++++++- .../python/ydf/model/sklearn_model_test.py | 41 +- 3 files changed, 416 insertions(+), 15 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/ydf/model/BUILD b/yggdrasil_decision_forests/port/python/ydf/model/BUILD index b5d0804c..0881c414 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/model/BUILD @@ -141,7 +141,15 @@ py_library( py_library( name = "export_sklearn", srcs = ["export_sklearn.py"], - deps = [":generic_model"], + deps = [ + ":generic_model", + # numpy dep, + "//ydf/learner:generic_learner", + "//ydf/learner:specialized_learners", + "//ydf/model/gradient_boosted_trees_model", + "//ydf/model/random_forest_model", + "//ydf/model/tree:all", + ], ) py_library( @@ -369,6 +377,7 @@ py_test( # absl/testing:parameterized dep, # numpy dep, # sklearn dep, + "//ydf/model/decision_forest_model", ], ) diff --git a/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py b/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py index 2bf82a7f..a1055592 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py @@ -12,10 +12,383 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Import and export sklearn models.""" +"""Import and export Scikit-Learn models from/to YDF.""" +import enum +import functools +from typing import Any, Dict, List, Optional, TypeVar, Union + +import numpy as np + +from ydf.learner import generic_learner +from ydf.learner import specialized_learners from ydf.model import generic_model +from ydf.model import tree as tree_lib +from ydf.model.gradient_boosted_trees_model import gradient_boosted_trees_model +from ydf.model.random_forest_model import random_forest_model + +# pytype: disable=import-error +# pylint: disable=g-import-not-at-top +try: + from sklearn import base + from sklearn import dummy + from sklearn import ensemble + from sklearn import tree +except ImportError as exc: + raise ImportError("Cannot import sklearn") from exc +# pylint: enable=g-import-not-at-top +# pytype: enable=import-error + + +# The column idx=0 is reserved for the label in YDF models. +_LABEL_COLUMN_OFFSET = 1 + +# Name of the label/feature columns +_LABEL_KEY = "label" +_FEATURES_KEY = "features" + + +class TaskType(enum.Enum): + """The type of task that a scikit-learn model performs.""" + + UNKNOWN = 1 + SCALAR_REGRESSION = 2 + SINGLE_LABEL_CLASSIFICATION = 3 + + +ScikitLearnModel = TypeVar("ScikitLearnModel", bound=base.BaseEstimator) +ScikitLearnTree = TypeVar("ScikitLearnTree", bound=tree.BaseDecisionTree) + + +def from_sklearn(sklearn_model: ScikitLearnModel) -> generic_model.GenericModel: + """Converts a tree-based scikit-learn model to a YDF model. + + Usage example: + + ```python + import ydf + from sklearn import datasets + from sklearn import tree + + # Train a SKLearn model + X, y = datasets.make_classification() + skl_model = tree.DecisionTreeClassifier().fit(X, y) + + # Convert the SKLearn model to a YDF model + ydf_model = ydf.from_sklearn(skl_model) + + # Make predictions with the YDF model + ydf_predictions = ydf_model.predict(X) + + # Analyse the YDF model + ydf_model.analyze(X) + ``` + + Currently supported models are: + * sklearn.tree.DecisionTreeClassifier + * sklearn.tree.DecisionTreeRegressor + * sklearn.tree.ExtraTreeClassifier + * sklearn.tree.ExtraTreeRegressor + * sklearn.ensemble.RandomForestClassifier + * sklearn.ensemble.RandomForestRegressor + * sklearn.ensemble.ExtraTreesClassifier + * sklearn.ensemble.ExtraTreesRegressor + * sklearn.ensemble.GradientBoostingRegressor + + Additionally, only single-label classification and scalar regression are + supported (e.g. multivariate regression models will not convert). + + Args: + sklearn_model: the scikit-learn tree based model to be converted. + + Returns: + a YDF Model that emulates the provided scikit-learn model. + """ + + if not hasattr(sklearn_model, "n_features_in_"): + raise ValueError( + "Scikit-Learn model must be fit to data before converting." + ) + return _sklearn_to_ydf_model(sklearn_model) + + +def _gen_fake_features(num_features: int, num_examples: int = 2): + return np.zeros(shape=[num_examples, num_features]) + + +@functools.singledispatch +def _sklearn_to_ydf_model( + sklearn_model: ScikitLearnModel, +) -> generic_model.GenericModel: + """Builds a YDF model from the given scikit-learn model.""" + raise NotImplementedError( + f"Can't build a YDF model for {type(sklearn_model)}" + ) + + +@_sklearn_to_ydf_model.register(tree.DecisionTreeRegressor) +@_sklearn_to_ydf_model.register(tree.ExtraTreeRegressor) +def _(sklearn_model: ScikitLearnTree) -> generic_model.GenericModel: + """Converts a single scikit-learn regression tree to a YDF model.""" + ydf_model = specialized_learners.RandomForestLearner( + label=_LABEL_KEY, + task=generic_learner.Task.REGRESSION, + num_trees=0, + ).train( + { + _LABEL_KEY: [0.0, 1.0], + _FEATURES_KEY: _gen_fake_features(sklearn_model.n_features_in_), + }, + verbose=0, + ) + assert isinstance(ydf_model, random_forest_model.RandomForestModel) + ydf_tree = convert_sklearn_tree_to_ydf_tree(sklearn_model) + ydf_model.add_tree(ydf_tree) + return ydf_model + + +@_sklearn_to_ydf_model.register(tree.DecisionTreeClassifier) +@_sklearn_to_ydf_model.register(tree.ExtraTreeClassifier) +def _(sklearn_model: ScikitLearnTree) -> generic_model.GenericModel: + """Converts a single scikit-learn classification tree to a YDF model.""" + ydf_model = specialized_learners.RandomForestLearner( + label=_LABEL_KEY, + task=generic_learner.Task.CLASSIFICATION, + num_trees=0, + ).train( + { + _LABEL_KEY: [str(c) for c in sklearn_model.classes_], + _FEATURES_KEY: _gen_fake_features( + sklearn_model.n_features_in_, len(sklearn_model.classes_) + ), + }, + verbose=0, + ) + assert isinstance(ydf_model, random_forest_model.RandomForestModel) + ydf_tree = convert_sklearn_tree_to_ydf_tree(sklearn_model) + ydf_model.add_tree(ydf_tree) + return ydf_model + + +@_sklearn_to_ydf_model.register(ensemble.ExtraTreesRegressor) +@_sklearn_to_ydf_model.register(ensemble.RandomForestRegressor) +def _( + sklearn_model: Union[ + ensemble.ExtraTreesRegressor, ensemble.RandomForestRegressor + ], +) -> generic_model.GenericModel: + """Converts a forest regression model into a YDF model.""" + + ydf_model = specialized_learners.RandomForestLearner( + label=_LABEL_KEY, + task=generic_learner.Task.REGRESSION, + num_trees=0, + ).train( + { + _LABEL_KEY: [0.0, 1.0], + _FEATURES_KEY: _gen_fake_features(sklearn_model.n_features_in_), + }, + verbose=0, + ) + assert isinstance(ydf_model, random_forest_model.RandomForestModel) + for sklearn_tree in sklearn_model.estimators_: + ydf_tree = convert_sklearn_tree_to_ydf_tree(sklearn_tree) + ydf_model.add_tree(ydf_tree) + return ydf_model + + +@_sklearn_to_ydf_model.register(ensemble.ExtraTreesClassifier) +@_sklearn_to_ydf_model.register(ensemble.RandomForestClassifier) +def _( + sklearn_model: Union[ + ensemble.ExtraTreesClassifier, ensemble.RandomForestClassifier + ], +) -> generic_model.GenericModel: + """Converts a forest classification model into a YDF model.""" + + ydf_model = specialized_learners.RandomForestLearner( + label=_LABEL_KEY, + task=generic_learner.Task.CLASSIFICATION, + num_trees=0, + ).train( + { + _LABEL_KEY: [str(c) for c in sklearn_model.classes_], + _FEATURES_KEY: _gen_fake_features( + sklearn_model.n_features_in_, len(sklearn_model.classes_) + ), + }, + verbose=0, + ) + assert isinstance(ydf_model, random_forest_model.RandomForestModel) + for sklearn_tree in sklearn_model.estimators_: + ydf_tree = convert_sklearn_tree_to_ydf_tree(sklearn_tree) + ydf_model.add_tree(ydf_tree) + return ydf_model + + +@_sklearn_to_ydf_model.register(ensemble.GradientBoostingRegressor) +def _( + sklearn_model: ensemble.GradientBoostingRegressor, +) -> generic_model.GenericModel: + """Converts a gradient boosting regression model into a YDF model.""" + + if isinstance(sklearn_model.init_, dummy.DummyRegressor): + # If the initial estimator is a DummyRegressor, then it predicts a constant + # which can be passed to GradientBoostedTreeBuilder as a bias. + init_pytree = None + bias = sklearn_model.init_.constant_[0][0] + elif isinstance(sklearn_model.init_, tree.DecisionTreeRegressor): + # If the initial estimator is a DecisionTreeRegressor, we add it as the + # first tree in the ensemble and set the bias to zero. We could also support + # other tree-based initial estimators (e.g. RandomForest), but this seems + # like a niche enough use case that we don't for the moment. + init_pytree = convert_sklearn_tree_to_ydf_tree(sklearn_model.init_) + bias = 0.0 + elif sklearn_model.init_ == "zero": + init_pytree = None + bias = 0.0 + else: + raise ValueError( + "The initial estimator must be either a DummyRegressor" + "or a DecisionTreeRegressor, but got" + f"{type(sklearn_model.init_)}." + ) + + ydf_model = specialized_learners.GradientBoostedTreesLearner( + label=_LABEL_KEY, + task=generic_learner.Task.REGRESSION, + num_trees=0, + ).train( + { + _LABEL_KEY: [0.0, 1.0], + _FEATURES_KEY: _gen_fake_features(sklearn_model.n_features_in_), + }, + verbose=0, + ) + assert isinstance( + ydf_model, gradient_boosted_trees_model.GradientBoostedTreesModel + ) + + ydf_model.set_initial_predictions([bias]) + + if init_pytree: + ydf_model.add_tree(init_pytree) + + for weak_learner in sklearn_model.estimators_.ravel(): + ydf_tree = convert_sklearn_tree_to_ydf_tree( + weak_learner, weight=sklearn_model.learning_rate + ) + ydf_model.add_tree(ydf_tree) + return ydf_model + + +def convert_sklearn_tree_to_ydf_tree( + sklearn_tree: ScikitLearnTree, + weight: Optional[float] = None, +) -> tree_lib.Tree: + """Converts a scikit-learn decision tree into a YDF tree. + + Args: + sklearn_tree: a scikit-learn decision tree. + weight: an optional weight to apply to the values of the leaves in the tree. + This is intended for use when converting gradient boosted tree models. + + Returns: + a YDF tree that has the same structure as the scikit-learn tree. + """ + try: + sklearn_tree_data = sklearn_tree.tree_.__getstate__() + except AttributeError as e: + raise ValueError( + "Scikit-Learn model must be fit to data before converting." + ) from e + + field_names = sklearn_tree_data["nodes"].dtype.names + task_type = _get_sklearn_tree_task_type(sklearn_tree) + if weight and task_type is TaskType.SINGLE_LABEL_CLASSIFICATION: + raise ValueError("weight should not be passed for classification trees.") + + nodes = [] + # For each node + for node_properties, node_output in zip( + sklearn_tree_data["nodes"], + sklearn_tree_data["values"], + ): + # Dictionary of node properties (e.g. "left_child", "threshold") except for + # the node output value. + node = { + field_name: field_value + for field_name, field_value in zip(field_names, node_properties) + } + + # Add the node output value to the dictionary of properties. + if task_type is TaskType.SCALAR_REGRESSION: + scaling_factor = weight if weight else 1.0 + node["value"] = tree_lib.RegressionValue( + value=node_output[0][0] * scaling_factor, + num_examples=node["weighted_n_node_samples"], + ) + elif task_type is TaskType.SINGLE_LABEL_CLASSIFICATION: + # Normalise to probabilities if we have a classification tree. + probabilities = list(node_output[0] / node_output[0].sum()) + node["value"] = tree_lib.ProbabilityValue( + probability=probabilities, + num_examples=node["weighted_n_node_samples"], + ) + else: + raise ValueError( + "Only scalar regression and single-label classification are " + "supported." + ) + nodes.append(node) + + root_node = _convert_sklearn_node_to_ydf_node( + # The root node has index zero. + node_index=0, + nodes=nodes, + ) + return tree_lib.Tree(root_node) + + +def _get_sklearn_tree_task_type(sklearn_tree: ScikitLearnTree) -> TaskType: + """Finds the task type of a scikit learn tree.""" + if hasattr(sklearn_tree, "n_classes_") and sklearn_tree.n_outputs_ == 1: + return TaskType.SINGLE_LABEL_CLASSIFICATION + elif sklearn_tree.n_outputs_ == 1: + return TaskType.SCALAR_REGRESSION + else: + return TaskType.UNKNOWN + + +def _convert_sklearn_node_to_ydf_node( + node_index: int, + nodes: List[Dict[str, Any]], +) -> tree_lib.AbstractNode: + """Converts a node within a scikit-learn tree into a YDF node.""" + if node_index == -1: + raise ValueError("Bad node idx") + + node = nodes[node_index] + is_leaf = node["left_child"] == -1 + if is_leaf: + return tree_lib.Leaf(value=node["value"]) -def from_sklearn(sklearn_model) -> generic_model.GenericModel: - raise NotImplementedError("from_sklearn not implemented") + neg_child = _convert_sklearn_node_to_ydf_node( + node_index=node["left_child"], + nodes=nodes, + ) + pos_child = _convert_sklearn_node_to_ydf_node( + node_index=node["right_child"], + nodes=nodes, + ) + return tree_lib.NonLeaf( + condition=tree_lib.NumericalHigherThanCondition( + attribute=node["feature"] + _LABEL_COLUMN_OFFSET, + threshold=node["threshold"], + missing=False, + score=0.0, + ), + pos_child=pos_child, + neg_child=neg_child, + ) diff --git a/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py b/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py index 9678092e..d0a7941f 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py @@ -20,6 +20,7 @@ from sklearn import linear_model from sklearn import tree from ydf.model import export_sklearn +from ydf.model.decision_forest_model import decision_forest_model class ScikitLearnModelConverterTest(parameterized.TestCase): @@ -42,7 +43,7 @@ class ScikitLearnModelConverterTest(parameterized.TestCase): ), ), ) - def DISABLED_test_import_regression_model( + def test_import_regression_model( self, sklearn_model, ): @@ -55,8 +56,25 @@ def DISABLED_test_import_regression_model( sklearn_predictions = sklearn_model.predict(features).astype(np.float32) ydf_model = export_sklearn.from_sklearn(sklearn_model) - ydf_predictions = ydf_model.predict(features) + assert isinstance(ydf_model, decision_forest_model.DecisionForestModel) + self.assertSequenceEqual( + ydf_model.input_feature_names(), + [ + "features.00_of_10", + "features.01_of_10", + "features.02_of_10", + "features.03_of_10", + "features.04_of_10", + "features.05_of_10", + "features.06_of_10", + "features.07_of_10", + "features.08_of_10", + "features.09_of_10", + ], + ) + self.assertEqual(ydf_model.label(), "label") + ydf_predictions = ydf_model.predict({"features": features}) np.testing.assert_allclose(sklearn_predictions, ydf_predictions, rtol=1e-4) @parameterized.parameters( @@ -65,7 +83,7 @@ def DISABLED_test_import_regression_model( (ensemble.RandomForestClassifier(random_state=42),), (ensemble.ExtraTreesClassifier(random_state=42),), ) - def DISABLED_test_import_classification_model( + def test_import_classification_model( self, sklearn_model, ): @@ -82,10 +100,11 @@ def DISABLED_test_import_classification_model( ) ydf_model = export_sklearn.from_sklearn(sklearn_model) - ydf_predictions = ydf_model.predict(features) + ydf_features = {"features": features} + ydf_predictions = ydf_model.predict(ydf_features) np.testing.assert_allclose(sklearn_predictions, ydf_predictions, rtol=1e-5) - def DISABLED_test_import_raises_when_unrecognised_model_provided(self): + def test_import_raises_when_unrecognised_model_provided(self): features, labels = datasets.make_regression( n_samples=100, n_features=10, @@ -95,14 +114,14 @@ def DISABLED_test_import_raises_when_unrecognised_model_provided(self): with self.assertRaises(NotImplementedError): export_sklearn.from_sklearn(sklearn_model) - def DISABLED_test_import_raises_when_sklearn_model_is_not_fit(self): - with self.assertRaises( + def test_import_raises_when_sklearn_model_is_not_fit(self): + with self.assertRaisesRegex( ValueError, - msg="Scikit-learn model must be fit to data before converting to TF.", + "Scikit-Learn model must be fit to data before converting", ): _ = export_sklearn.from_sklearn(tree.DecisionTreeRegressor()) - def DISABLED_test_import_raises_when_regression_target_is_multivariate(self): + def test_import_raises_when_regression_target_is_multivariate(self): features, labels = datasets.make_regression( n_samples=100, n_features=10, @@ -117,7 +136,7 @@ def DISABLED_test_import_raises_when_regression_target_is_multivariate(self): ): _ = export_sklearn.from_sklearn(sklearn_model) - def DISABLED_test_import_raises_when_classification_target_is_multilabel( + def test_import_raises_when_classification_target_is_multilabel( self, ): features, labels = datasets.make_multilabel_classification( @@ -134,7 +153,7 @@ def DISABLED_test_import_raises_when_classification_target_is_multilabel( ): _ = export_sklearn.from_sklearn(sklearn_model) - def DISABLED_test_convert_raises_when_gbt_initial_estimator_is_not_tree_or_constant( + def test_convert_raises_when_gbt_initial_estimator_is_not_tree_or_constant( self, ): features, labels = datasets.make_regression( From 394c82b901573ef4681aa1966c4fbf6c8f44db1f Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Fri, 7 Jun 2024 02:00:33 -0700 Subject: [PATCH 16/30] Surface the SKLearn converter in the API PiperOrigin-RevId: 641176849 --- documentation/public/docs/py_api/index.md | 2 + documentation/public/docs/py_api/utilities.md | 2 + .../port/python/CHANGELOG.md | 1 + .../port/python/ydf/BUILD | 2 + .../port/python/ydf/__init__.py | 2 +- .../port/python/ydf/api_test.py | 8 +++ .../port/python/ydf/model/export_sklearn.py | 45 +------------- .../port/python/ydf/model/generic_model.py | 60 +++++++++++++++++++ 8 files changed, 77 insertions(+), 45 deletions(-) diff --git a/documentation/public/docs/py_api/index.md b/documentation/public/docs/py_api/index.md index 9a5282dd..f81bf83c 100644 --- a/documentation/public/docs/py_api/index.md +++ b/documentation/public/docs/py_api/index.md @@ -65,6 +65,8 @@ and evaluation. e.g. training date, uid. - [from_tensorflow_decision_forests](utilities.md#ydf.from_tensorflow_decision_forests): Load a TensorFlow Decision Forests model from disk. +- [from_sklearn](utilities.md#ydf.from_sklearn): Convert a scikit-learn model + into a YDF model. ## Custom Loss diff --git a/documentation/public/docs/py_api/utilities.md b/documentation/public/docs/py_api/utilities.md index ece421e7..6657d28c 100644 --- a/documentation/public/docs/py_api/utilities.md +++ b/documentation/public/docs/py_api/utilities.md @@ -26,6 +26,8 @@ ::: ydf.from_tensorflow_decision_forests +::: ydf.from_sklearn + ::: ydf.RegressionLoss ::: ydf.BinaryClassificationLoss diff --git a/yggdrasil_decision_forests/port/python/CHANGELOG.md b/yggdrasil_decision_forests/port/python/CHANGELOG.md index 42b6785b..5339e1e0 100644 --- a/yggdrasil_decision_forests/port/python/CHANGELOG.md +++ b/yggdrasil_decision_forests/port/python/CHANGELOG.md @@ -7,6 +7,7 @@ - Add `max_depth` argument to `model.print_tree`. - Add `verbose` argument to `train` method which is equivalent but sometime more convenient than`ydf.verbose`. +- Add SKLearn to YDF model converter: `ydf.from_sklearn`. ### Fix diff --git a/yggdrasil_decision_forests/port/python/ydf/BUILD b/yggdrasil_decision_forests/port/python/ydf/BUILD index aea54afe..1c910007 100644 --- a/yggdrasil_decision_forests/port/python/ydf/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/BUILD @@ -22,6 +22,7 @@ py_library( "//ydf/learner:tuner", "//ydf/learner:worker", "//ydf/model:export_jax", # buildcleaner: keep + "//ydf/model:export_sklearn", # buildcleaner: keep "//ydf/model:export_tf", # buildcleaner: keep "//ydf/model:generic_model", "//ydf/model:model_lib", @@ -98,6 +99,7 @@ py_test( # absl/testing:absltest dep, # jax dep, # buildcleaner: keep # pandas dep, + # sklearn dep, # buildcleaner: keep # tensorflow dep, # buildcleaner: keep # tensorflow_decision_forests dep, # buildcleaner: keep "//ydf/utils:test_utils", diff --git a/yggdrasil_decision_forests/port/python/ydf/__init__.py b/yggdrasil_decision_forests/port/python/ydf/__init__.py index 412c5c74..d87a56b3 100644 --- a/yggdrasil_decision_forests/port/python/ydf/__init__.py +++ b/yggdrasil_decision_forests/port/python/ydf/__init__.py @@ -74,7 +74,7 @@ def _check_install(): from ydf.dataset.dataset import create_vertical_dataset from ydf.model.model_metadata import ModelMetadata from ydf.model.model_lib import from_tensorflow_decision_forests - +from ydf.model.generic_model import from_sklearn # Custom Loss from ydf.learner.custom_loss import RegressionLoss diff --git a/yggdrasil_decision_forests/port/python/ydf/api_test.py b/yggdrasil_decision_forests/port/python/ydf/api_test.py index 30deab15..44024e53 100644 --- a/yggdrasil_decision_forests/port/python/ydf/api_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/api_test.py @@ -21,6 +21,8 @@ from absl import logging from absl.testing import absltest import pandas as pd +from sklearn import ensemble as skl_ensemble +import sklearn.datasets import ydf # In the world, use "import ydf" from ydf.utils import test_utils @@ -224,6 +226,12 @@ def test_export_jax_function(self): model = ydf.load_model(model_path) _ = model.to_jax_function() + def test_import_sklearn_model(self): + X, y = sklearn.datasets.make_classification() + skl_model = skl_ensemble.RandomForestClassifier().fit(X, y) + ydf_model = ydf.from_sklearn(skl_model) + _ = ydf_model.predict({"features": X}) + if __name__ == "__main__": absltest.main() diff --git a/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py b/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py index a1055592..3d6b6f39 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py @@ -61,50 +61,7 @@ class TaskType(enum.Enum): def from_sklearn(sklearn_model: ScikitLearnModel) -> generic_model.GenericModel: - """Converts a tree-based scikit-learn model to a YDF model. - - Usage example: - - ```python - import ydf - from sklearn import datasets - from sklearn import tree - - # Train a SKLearn model - X, y = datasets.make_classification() - skl_model = tree.DecisionTreeClassifier().fit(X, y) - - # Convert the SKLearn model to a YDF model - ydf_model = ydf.from_sklearn(skl_model) - - # Make predictions with the YDF model - ydf_predictions = ydf_model.predict(X) - - # Analyse the YDF model - ydf_model.analyze(X) - ``` - - Currently supported models are: - * sklearn.tree.DecisionTreeClassifier - * sklearn.tree.DecisionTreeRegressor - * sklearn.tree.ExtraTreeClassifier - * sklearn.tree.ExtraTreeRegressor - * sklearn.ensemble.RandomForestClassifier - * sklearn.ensemble.RandomForestRegressor - * sklearn.ensemble.ExtraTreesClassifier - * sklearn.ensemble.ExtraTreesRegressor - * sklearn.ensemble.GradientBoostingRegressor - - Additionally, only single-label classification and scalar regression are - supported (e.g. multivariate regression models will not convert). - - Args: - sklearn_model: the scikit-learn tree based model to be converted. - - Returns: - a YDF Model that emulates the provided scikit-learn model. - """ - + """Converts a tree-based scikit-learn model to a YDF model.""" if not hasattr(sklearn_model, "n_features_in_"): raise ValueError( "Scikit-Learn model must be fit to data before converting." diff --git a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py index e6bad13f..6baf0fb1 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py @@ -1089,6 +1089,53 @@ def force_engine(self, engine_name: Optional[str]) -> None: self._model.ForceEngine(engine_name) +def from_sklearn(sklearn_model: Any) -> GenericModel: + """Converts a tree-based scikit-learn model to a YDF model. + + Usage example: + + ```python + import ydf + from sklearn import datasets + from sklearn import tree + + # Train a SKLearn model + X, y = datasets.make_classification() + skl_model = tree.DecisionTreeClassifier().fit(X, y) + + # Convert the SKLearn model to a YDF model + ydf_model = ydf.from_sklearn(skl_model) + + # Make predictions with the YDF model + ydf_predictions = ydf_model.predict({"features": X}) + + # Analyse the YDF model + ydf_model.analyze({"features": X}) + ``` + + Currently supported models are: + * sklearn.tree.DecisionTreeClassifier + * sklearn.tree.DecisionTreeRegressor + * sklearn.tree.ExtraTreeClassifier + * sklearn.tree.ExtraTreeRegressor + * sklearn.ensemble.RandomForestClassifier + * sklearn.ensemble.RandomForestRegressor + * sklearn.ensemble.ExtraTreesClassifier + * sklearn.ensemble.ExtraTreesRegressor + * sklearn.ensemble.GradientBoostingRegressor + + Additionally, only single-label classification and scalar regression are + supported (e.g. multivariate regression models will not convert). + + Args: + sklearn_model: the scikit-learn tree based model to be converted. + + Returns: + a YDF Model that emulates the provided scikit-learn model. + """ + return _get_export_sklearn().from_sklearn(sklearn_model) + + def _get_export_jax(): try: from ydf.model import export_jax # pylint: disable=g-import-not-at-top,import-outside-toplevel # pytype: disable=import-error @@ -1114,4 +1161,17 @@ def _get_export_tf(): ) from exc +def _get_export_sklearn(): + try: + from ydf.model import export_sklearn # pylint: disable=g-import-not-at-top,import-outside-toplevel # pytype: disable=import-error + + return export_sklearn + except ImportError as exc: + raise ValueError( + '"scikit-learn" is needed by this function. Make sure ' + "it installed and try again. If using pip, run `pip install" + " scikit-learn`." + ) from exc + + ModelType = TypeVar("ModelType", bound=GenericModel) From 33c15add96dfe66b8f845b1c71f29ae922e1ac3d Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Thu, 13 Jun 2024 03:19:13 -0700 Subject: [PATCH 17/30] Anomaly detection; Create the ANOMALY_DETECTION task (part 1) PiperOrigin-RevId: 642918719 --- yggdrasil_decision_forests/learner/BUILD | 1 + .../learner/abstract_learner.cc | 50 ++++++++++----- .../learner/abstract_learner_test.cc | 64 +++++++++++++++---- yggdrasil_decision_forests/metric/metric.cc | 30 +++++++++ .../metric/metric.proto | 13 +++- .../metric/metric_test.cc | 50 +++++++++++++++ yggdrasil_decision_forests/metric/report.cc | 9 +++ yggdrasil_decision_forests/metric/report.h | 2 + .../metric/report_test.cc | 44 +++++++++++++ .../model/abstract_model.cc | 49 +++++++++++++- .../model/abstract_model.proto | 6 ++ .../model/abstract_model_test.cc | 29 +++++++++ .../model/decision_tree/decision_tree.cc | 27 ++++---- .../model/prediction.proto | 8 ++- .../python/ydf/learner/generic_learner.py | 13 +++- .../specialized_learners_pre_generated.py | 3 +- .../port/python/ydf/model/generic_model.py | 6 ++ .../utils/model_analysis.cc | 9 ++- .../utils/test_utils.cc | 14 ++++ 19 files changed, 374 insertions(+), 53 deletions(-) diff --git a/yggdrasil_decision_forests/learner/BUILD b/yggdrasil_decision_forests/learner/BUILD index b4749b67..15d94263 100644 --- a/yggdrasil_decision_forests/learner/BUILD +++ b/yggdrasil_decision_forests/learner/BUILD @@ -154,6 +154,7 @@ cc_test( "//yggdrasil_decision_forests/model:model_testing", "//yggdrasil_decision_forests/model:prediction_cc_proto", "//yggdrasil_decision_forests/utils:logging", + "//yggdrasil_decision_forests/utils:status_macros", "//yggdrasil_decision_forests/utils:test", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", diff --git a/yggdrasil_decision_forests/learner/abstract_learner.cc b/yggdrasil_decision_forests/learner/abstract_learner.cc index 8a4f7b50..cee772f1 100644 --- a/yggdrasil_decision_forests/learner/abstract_learner.cc +++ b/yggdrasil_decision_forests/learner/abstract_learner.cc @@ -67,23 +67,27 @@ absl::Status AbstractLearner::LinkTrainingConfig( const dataset::proto::DataSpecification& data_spec, proto::TrainingConfigLinking* config_link) { // Label. - int32_t label; - if (!training_config.has_label()) { - STATUS_FATAL("No label specified in the training config. Aborting."); + int32_t label = -1; + // Anomaly detection is the only task that can have or not have labels. + if (training_config.task() != proto::ANOMALY_DETECTION && + !training_config.has_label()) { + STATUS_FATAL("No label specified in the training config."); + } + if (training_config.has_label()) { + RETURN_IF_ERROR(dataset::GetSingleColumnIdxFromName( + training_config.label(), data_spec, &label, + "Retrieving label column failed.")); + config_link->set_num_label_classes( + data_spec.columns(label).categorical().number_of_unique_values()); } - RETURN_IF_ERROR(dataset::GetSingleColumnIdxFromName( - training_config.label(), data_spec, &label, - "Retrieving label column failed. ")); config_link->set_label(label); - config_link->set_num_label_classes( - data_spec.columns(label).categorical().number_of_unique_values()); // CV group. int32_t cv_group = -1; if (training_config.has_cv_group()) { RETURN_IF_ERROR(dataset::GetSingleColumnIdxFromName( training_config.cv_group(), data_spec, &cv_group, - "Retrieving cross-validation group column failed. ")); + "Retrieving cross-validation group column failed.")); } config_link->set_cv_group(cv_group); @@ -475,6 +479,15 @@ absl::Status AbstractLearner::CheckConfiguration( const model::proto::TrainingConfig& config, const model::proto::TrainingConfigLinking& config_link, const model::proto::DeploymentConfig& deployment) { + if (deployment.num_threads() < 0) { + return absl::InvalidArgumentError("The number of threads should be >= 0"); + } + + if (config.task() == model::proto::Task::ANOMALY_DETECTION) { + // Note: ANOMALY_DETECTION is the only task that does not need a label. + return absl::OkStatus(); + } + const auto& label_col_spec = data_spec.columns(config_link.label()); // Check the type of the label column. switch (config.task()) { @@ -487,7 +500,8 @@ absl::Status AbstractLearner::CheckConfiguration( return absl::InvalidArgumentError(absl::StrCat( "The label column \"", config.label(), "\" should be CATEGORICAL for a CLASSIFICATION " - "Task. Note: BOOLEAN columns should be set as CATEGORICAL using a " + "Task. Note: BOOLEAN columns should be set as CATEGORICAL using " + "a " "dataspec guide, even for a binary classification task.")); } break; @@ -535,7 +549,8 @@ absl::Status AbstractLearner::CheckConfiguration( return absl::InvalidArgumentError( "The \"ranking_group\" column must have a " "\"max_number_of_unique_values\" " - "of -1 in the dataspec guide. This ensures that rare groups are " + "of -1 in the dataspec guide. This ensures that rare groups " + "are " "not pruned."); } } @@ -543,7 +558,8 @@ absl::Status AbstractLearner::CheckConfiguration( case model::proto::Task::CATEGORICAL_UPLIFT: { if (label_col_spec.type() != dataset::proto::ColumnType::CATEGORICAL) { return absl::InvalidArgumentError( - "The label column should be CATEGORICAL for an CATEGORICAL_UPLIFT " + "The label column should be CATEGORICAL for an " + "CATEGORICAL_UPLIFT " "task."); } if (!config_link.has_uplift_treatment() || @@ -592,6 +608,8 @@ absl::Status AbstractLearner::CheckConfiguration( "Uplift only supports binary treatments."); } } break; + case model::proto::Task::ANOMALY_DETECTION: + return absl::InternalError("ANOMALY_DETECTION has no labels"); } // Check the label don't contains NaN. if (label_col_spec.count_nas() != 0) { @@ -600,9 +618,7 @@ absl::Status AbstractLearner::CheckConfiguration( "missing values. $1 missing values are found.", config.label(), label_col_spec.count_nas())); } - if (deployment.num_threads() < 0) { - return absl::InvalidArgumentError("The number of threads should be >= 0"); - } + return absl::OkStatus(); } @@ -818,7 +834,9 @@ void InitializeModelWithAbstractTrainingConfig( const proto::TrainingConfig& training_config, const proto::TrainingConfigLinking& training_config_linking, AbstractModel* model) { - model->set_label_col_idx(training_config_linking.label()); + if (training_config.task() != proto::Task::ANOMALY_DETECTION) { + model->set_label_col_idx(training_config_linking.label()); + } if (training_config.task() == proto::Task::RANKING) { model->set_ranking_group_col(training_config_linking.ranking_group()); diff --git a/yggdrasil_decision_forests/learner/abstract_learner_test.cc b/yggdrasil_decision_forests/learner/abstract_learner_test.cc index a01fc566..b7a20faa 100644 --- a/yggdrasil_decision_forests/learner/abstract_learner_test.cc +++ b/yggdrasil_decision_forests/learner/abstract_learner_test.cc @@ -32,6 +32,7 @@ #include "yggdrasil_decision_forests/model/model_testing.h" #include "yggdrasil_decision_forests/model/prediction.pb.h" #include "yggdrasil_decision_forests/utils/logging.h" +#include "yggdrasil_decision_forests/utils/status_macros.h" #include "yggdrasil_decision_forests/utils/test.h" #include "yggdrasil_decision_forests/learner/abstract_learner.h" @@ -57,8 +58,8 @@ TEST(AbstractModel, LinkTrainingConfig) { data_spec.add_columns()->set_name("D"); proto::TrainingConfigLinking config_link; - CHECK_OK(AbstractLearner::LinkTrainingConfig(training_config, data_spec, - &config_link)); + ASSERT_OK(AbstractLearner::LinkTrainingConfig(training_config, data_spec, + &config_link)); EXPECT_EQ(config_link.label(), 0); EXPECT_THAT(config_link.features(), ElementsAre(1, 3)); @@ -76,8 +77,8 @@ TEST(AbstractModel, LinkTrainingConfigNoInputFeatures) { data_spec.add_columns()->set_name("D"); proto::TrainingConfigLinking config_link; - CHECK_OK(AbstractLearner::LinkTrainingConfig(training_config, data_spec, - &config_link)); + ASSERT_OK(AbstractLearner::LinkTrainingConfig(training_config, data_spec, + &config_link)); EXPECT_EQ(config_link.label(), 0); EXPECT_THAT(config_link.features(), ElementsAre(1, 2, 3)); @@ -100,8 +101,8 @@ TEST(AbstractModel, LinkTrainingConfigFullyMissingFeatures) { std::numeric_limits::quiet_NaN()); proto::TrainingConfigLinking config_link; - CHECK_OK(AbstractLearner::LinkTrainingConfig(training_config, data_spec, - &config_link)); + ASSERT_OK(AbstractLearner::LinkTrainingConfig(training_config, data_spec, + &config_link)); EXPECT_EQ(config_link.label(), 0); EXPECT_THAT(config_link.features(), ElementsAre(3)); @@ -120,7 +121,44 @@ TEST(AbstractModel, LinkTrainingConfigMissingLabel) { EXPECT_THAT(AbstractLearner::LinkTrainingConfig(training_config, data_spec, &config_link), StatusIs(absl::StatusCode::kInvalidArgument, - "No label specified in the training config. Aborting.")); + "No label specified in the training config.")); +} + +TEST(AbstractModel, LinkTrainingConfigAnomalyDetection) { + proto::TrainingConfig training_config; + training_config.set_task(proto::Task::ANOMALY_DETECTION); + training_config.add_features(".*"); + + dataset::proto::DataSpecification data_spec; + data_spec.add_columns()->set_name("A"); + data_spec.add_columns()->set_name("B"); + data_spec.add_columns()->set_name("C"); + + proto::TrainingConfigLinking config_link; + ASSERT_OK(AbstractLearner::LinkTrainingConfig(training_config, data_spec, + &config_link)); + + EXPECT_EQ(config_link.label(), -1); + EXPECT_THAT(config_link.features(), ElementsAre(0, 1, 2)); +} + +TEST(AbstractModel, LinkTrainingConfigAnomalyDetectionWithLabel) { + proto::TrainingConfig training_config; + training_config.set_task(proto::Task::ANOMALY_DETECTION); + training_config.set_label("A"); + training_config.add_features(".*"); + + dataset::proto::DataSpecification data_spec; + data_spec.add_columns()->set_name("A"); + data_spec.add_columns()->set_name("B"); + data_spec.add_columns()->set_name("C"); + + proto::TrainingConfigLinking config_link; + ASSERT_OK(AbstractLearner::LinkTrainingConfig(training_config, data_spec, + &config_link)); + + EXPECT_EQ(config_link.label(), 0); + EXPECT_THAT(config_link.features(), ElementsAre(1, 2)); } TEST(AbstractLearner, GenericHyperParameters) { @@ -218,7 +256,7 @@ TEST(AbstractLearner, EvaluateLearner) { valid_dataset = {}) const override { auto model = absl::make_unique(); model::proto::TrainingConfigLinking config_link; - CHECK_OK(AbstractLearner::LinkTrainingConfig( + RETURN_IF_ERROR(AbstractLearner::LinkTrainingConfig( training_config(), train_dataset.data_spec(), &config_link)); InitializeModelWithAbstractTrainingConfig(training_config(), config_link, model.get()); @@ -246,12 +284,12 @@ TEST(AbstractLearner, EvaluateLearner) { dataset::VerticalDataset dataset; dataset.set_data_spec(data_spec); - CHECK_OK(dataset.CreateColumnsFromDataspec()); + ASSERT_OK(dataset.CreateColumnsFromDataspec()); for (int i = 0; i < 1000; i++) { - CHECK_OK(dataset.AppendExampleWithStatus({{"a", "1"}})); - CHECK_OK(dataset.AppendExampleWithStatus({{"a", "2"}})); - CHECK_OK(dataset.AppendExampleWithStatus({{"a", "1"}})); - CHECK_OK(dataset.AppendExampleWithStatus({{"a", "2"}})); + ASSERT_OK(dataset.AppendExampleWithStatus({{"a", "1"}})); + ASSERT_OK(dataset.AppendExampleWithStatus({{"a", "2"}})); + ASSERT_OK(dataset.AppendExampleWithStatus({{"a", "1"}})); + ASSERT_OK(dataset.AppendExampleWithStatus({{"a", "2"}})); } const metric::proto::EvaluationOptions evaluation_options = diff --git a/yggdrasil_decision_forests/metric/metric.cc b/yggdrasil_decision_forests/metric/metric.cc index bb0b2f25..879106f4 100644 --- a/yggdrasil_decision_forests/metric/metric.cc +++ b/yggdrasil_decision_forests/metric/metric.cc @@ -395,6 +395,12 @@ void MergeEvaluationUplift(const proto::EvaluationResults::Uplift& src, std::max(dst->num_treatments(), src.num_treatments())); } +void MergeEvaluationAnomalyDetection( + const proto::EvaluationResults::AnomalyDetection& src, + proto::EvaluationResults::AnomalyDetection* dst) { + // No merging to be done. +} + } // namespace float PValueMeanIsGreaterThanZero(const std::vector& sample) { @@ -835,6 +841,9 @@ absl::Status InitializeEvaluation(const proto::EvaluationOptions& option, RETURN_IF_ERROR(uplift::InitializeNumericalUpliftEvaluation( option, label_column, eval)); break; + case model::proto::Task::ANOMALY_DETECTION: + eval->mutable_anomaly_detection(); + break; default: STATUS_FATALS("Non supported task type: ", model::proto::Task_Name(option.task())); @@ -917,6 +926,9 @@ absl::Status AddPrediction(const proto::EvaluationOptions& option, need_prediction_sampling = true; break; + case model::proto::Task::ANOMALY_DETECTION: + break; + default: break; } @@ -970,6 +982,10 @@ absl::Status FinalizeEvaluation(const proto::EvaluationOptions& option, RETURN_IF_ERROR(uplift::FinalizeUpliftMetricsFromSampledPredictions( option, label_column, eval)); break; + + case model::proto::Task::ANOMALY_DETECTION: + break; + default: break; } @@ -1296,6 +1312,10 @@ absl::Status MergeEvaluation(const proto::EvaluationOptions& option, case proto::EvaluationResults::kUplift: MergeEvaluationUplift(src.uplift(), dst->mutable_uplift()); break; + case proto::EvaluationResults::kAnomalyDetection: + MergeEvaluationAnomalyDetection(src.anomaly_detection(), + dst->mutable_anomaly_detection()); + break; case proto::EvaluationResults::TYPE_NOT_SET: return absl::InvalidArgumentError("Non initialized evaluation"); break; @@ -1658,6 +1678,11 @@ absl::StatusOr GetMetricUplift( return absl::InvalidArgumentError("Not implemented"); } } +absl::StatusOr GetMetricAnomalyDetection( + const proto::EvaluationResults& evaluation, + const proto::MetricAccessor::AnomalyDetection& metric) { + return absl::InvalidArgumentError("No AnomalyDetection metric"); +} absl::StatusOr GetUserCustomizedMetrics( const proto::EvaluationResults& evaluation, @@ -1714,6 +1739,11 @@ absl::StatusOr GetMetric(const proto::EvaluationResults& evaluation, return GetMetricFatalMissing("uplift", evaluation, metric); } return GetMetricUplift(evaluation, metric.uplift()); + case proto::MetricAccessor::kAnomalyDetection: + if (!evaluation.has_anomaly_detection()) { + return GetMetricFatalMissing("anomaly_detection", evaluation, metric); + } + return GetMetricAnomalyDetection(evaluation, metric.anomaly_detection()); case proto::MetricAccessor::kUserMetric: return GetUserCustomizedMetrics(evaluation, metric.user_metric()); case proto::MetricAccessor::TASK_NOT_SET: diff --git a/yggdrasil_decision_forests/metric/metric.proto b/yggdrasil_decision_forests/metric/metric.proto index 14c98e3c..b78d4d73 100644 --- a/yggdrasil_decision_forests/metric/metric.proto +++ b/yggdrasil_decision_forests/metric/metric.proto @@ -68,6 +68,8 @@ message EvaluationOptions { message Uplift {} + message AnomalyDetection {} + // Task of the model. optional model.proto.Task task = 1 [default = CLASSIFICATION]; // Evaluation configuration depending on the type of problem. @@ -76,6 +78,7 @@ message EvaluationOptions { Regression regression = 3; Ranking ranking = 7; Uplift uplift = 8; + AnomalyDetection anomaly_detection = 9; } // Percentage of sampled predictions. If no predictions need to be sampled // (i.e. no part of the configuration needs it), this parameter is ignored and @@ -97,7 +100,7 @@ message EvaluationOptions { // however. optional dataset.proto.WeightDefinition weights = 6; - // Next ID: 8 + // Next ID: 10 } // Evaluation results of a model. @@ -204,6 +207,8 @@ message EvaluationResults { optional double cate_calibration = 4; } + message AnomalyDetection {} + // Number of predictions (weighted by example weight). optional double count_predictions = 1 [default = 0]; // Number of predictions (without weights). @@ -220,6 +225,7 @@ message EvaluationResults { Regression regression = 7; Ranking ranking = 12; Uplift uplift = 14; + AnomalyDetection anomaly_detection = 16; } // The dataspec of the label column. This field can contain information such // as: The possible label values, the distribution of the label values, the @@ -245,7 +251,7 @@ message EvaluationResults { // User can use this field to store value for any customized metrics. map user_metrics = 15; - // Next ID: 16 + // Next ID: 17 } // Reference a metric."MetricAccessor" is used as a parameter of the function @@ -262,6 +268,7 @@ message MetricAccessor { Loss loss = 3; Ranking ranking = 4; Uplift uplift = 5; + AnomalyDetection anomaly_detection = 7; UserMetric user_metric = 6; } @@ -335,6 +342,8 @@ message MetricAccessor { message CateCalibration {} } + message AnomalyDetection {} + message UserMetric { optional string metrics_name = 1; } diff --git a/yggdrasil_decision_forests/metric/metric_test.cc b/yggdrasil_decision_forests/metric/metric_test.cc index 8797b32c..aad1db67 100644 --- a/yggdrasil_decision_forests/metric/metric_test.cc +++ b/yggdrasil_decision_forests/metric/metric_test.cc @@ -809,6 +809,20 @@ TEST(Metric, GetMetricRegression) { EXPECT_NEAR(mae, MAE(results_regression), 0.0001); } +TEST(Metric, GetMetricAnomalyDetection) { + const proto::EvaluationResults results = PARSE_TEST_PROTO(R"pb( + task: ANOMALY_DETECTION + label_column { type: CATEGORICAL } + anomaly_detection {} + count_predictions: 10 + )pb"); + + EXPECT_THAT( + GetMetric(results, PARSE_TEST_PROTO(R"pb(anomaly_detection {})pb")) + .status(), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + TEST(Metric, GetMetricClassification) { const proto::EvaluationResults results_classification = PARSE_TEST_PROTO(R"pb( task: CLASSIFICATION @@ -1700,6 +1714,42 @@ TEST(Metric, MergeEvaluationUplifting) { EXPECT_THAT(dst, EqualsProto(expected_dst)); } +TEST(Metric, MergeEvaluationAnomalyDetection) { + const proto::EvaluationResults src = PARSE_TEST_PROTO( + R"pb( + count_predictions: 1 + count_predictions_no_weight: 2 + sampled_predictions { example_key: "a" } + count_sampled_predictions: 3 + training_duration_in_seconds: 4 + num_folds: 5 + anomaly_detection {} + )pb"); + proto::EvaluationResults dst = PARSE_TEST_PROTO( + R"pb( + count_predictions: 10 + count_predictions_no_weight: 20 + sampled_predictions { example_key: "b" } + count_sampled_predictions: 30 + training_duration_in_seconds: 40 + num_folds: 50 + anomaly_detection {} + )pb"); + EXPECT_OK(MergeEvaluation({}, src, &dst)); + proto::EvaluationResults expected_dst = PARSE_TEST_PROTO( + R"pb( + count_predictions: 11 + count_predictions_no_weight: 22 + sampled_predictions { example_key: "b" } + sampled_predictions { example_key: "a" } + count_sampled_predictions: 33 + training_duration_in_seconds: 44 + num_folds: 55 + anomaly_detection {} + )pb"); + EXPECT_THAT(dst, EqualsProto(expected_dst)); +} + TEST(Metric, HigherIsBetter) { { const proto::MetricAccessor accessor = PARSE_TEST_PROTO( diff --git a/yggdrasil_decision_forests/metric/report.cc b/yggdrasil_decision_forests/metric/report.cc index d0159775..22f4c764 100644 --- a/yggdrasil_decision_forests/metric/report.cc +++ b/yggdrasil_decision_forests/metric/report.cc @@ -382,6 +382,9 @@ absl::Status AppendTextReportWithStatus(const proto::EvaluationResults& eval, case proto::EvaluationResults::TypeCase::kUplift: RETURN_IF_ERROR(AppendTextReportUplift(eval, report)); break; + case proto::EvaluationResults::TypeCase::kAnomalyDetection: + RETURN_IF_ERROR(AppendTextReportAnomalyDetection(eval, report)); + break; default: STATUS_FATAL("This model does not support evaluation reports."); } @@ -576,6 +579,12 @@ absl::Status AppendTextReportUplift(const proto::EvaluationResults& eval, return absl::OkStatus(); } +absl::Status AppendTextReportAnomalyDetection( + const proto::EvaluationResults& eval, std::string* report) { + absl::StrAppend(report, "No report for anomaly detection\n"); + return absl::OkStatus(); +} + absl::Status AppendHtmlReport(const proto::EvaluationResults& eval, std::string* html_report, const HtmlReportOptions& options) { diff --git a/yggdrasil_decision_forests/metric/report.h b/yggdrasil_decision_forests/metric/report.h index 25f398d7..3040e903 100644 --- a/yggdrasil_decision_forests/metric/report.h +++ b/yggdrasil_decision_forests/metric/report.h @@ -43,6 +43,8 @@ absl::Status AppendTextReportRanking(const proto::EvaluationResults& eval, std::string* report); absl::Status AppendTextReportUplift(const proto::EvaluationResults& eval, std::string* report); +absl::Status AppendTextReportAnomalyDetection( + const proto::EvaluationResults& eval, std::string* report); // Add the report in a html format. struct HtmlReportOptions { diff --git a/yggdrasil_decision_forests/metric/report_test.cc b/yggdrasil_decision_forests/metric/report_test.cc index 11d79d0c..3e3e784a 100644 --- a/yggdrasil_decision_forests/metric/report_test.cc +++ b/yggdrasil_decision_forests/metric/report_test.cc @@ -140,6 +140,50 @@ TEST(Report, HtmlReportRegression) { CHECK_OK(file::SetContent(path, html_report)); } +TEST(Report, HtmlReportAnomalyDetection) { + // Create a fake column specification. + dataset::proto::Column label_column; + label_column.set_type(dataset::proto::ColumnType::CATEGORICAL); + label_column.set_name("label"); + label_column.mutable_categorical()->set_number_of_unique_values(3); + label_column.mutable_categorical()->set_most_frequent_value(1); + label_column.mutable_categorical()->set_is_already_integerized(false); + auto& vocab = *label_column.mutable_categorical()->mutable_items(); + vocab["a"].set_index(0); + vocab["b"].set_index(1); + vocab["c"].set_index(2); + + // Configure the evaluation. + utils::RandomEngine rnd; + proto::EvaluationOptions option; + option.set_task(model::proto::Task::ANOMALY_DETECTION); + + // Initialize. + proto::EvaluationResults eval; + ASSERT_OK(InitializeEvaluation(option, label_column, &eval)); + model::proto::Prediction pred; + auto* pred_proba = pred.mutable_classification()->mutable_distribution(); + pred_proba->mutable_counts()->Resize(3, 0); + pred_proba->set_sum(1); + + // Add some predictions. + pred.mutable_anomaly_detection()->set_value(0.5); + ASSERT_OK(AddPrediction(option, pred, &rnd, &eval)); + + pred.mutable_anomaly_detection()->set_value(0.1); + ASSERT_OK(AddPrediction(option, pred, &rnd, &eval)); + + // Finalize. + ASSERT_OK(FinalizeEvaluation(option, label_column, &eval)); + + std::string html_report; + ASSERT_OK(AppendHtmlReport(eval, &html_report)); + + const auto path = + file::JoinPath(test::TmpDirectory(), "report_anomaly_detection.html"); + YDF_LOG(INFO) << "path: " << path; + ASSERT_OK(file::SetContent(path, html_report)); +} } // namespace } // namespace metric } // namespace yggdrasil_decision_forests diff --git a/yggdrasil_decision_forests/model/abstract_model.cc b/yggdrasil_decision_forests/model/abstract_model.cc index 42e8295b..919c5bb6 100644 --- a/yggdrasil_decision_forests/model/abstract_model.cc +++ b/yggdrasil_decision_forests/model/abstract_model.cc @@ -131,6 +131,14 @@ AbstractModel::EvaluateWithStatus( if (option.task() != task()) { STATUS_FATAL("The evaluation and the model tasks differ."); } + if (label_col_idx_ == -1) { + if (task() == proto::Task::ANOMALY_DETECTION) { + STATUS_FATAL( + "Cannot evaluate an anomaly detection model without a label."); + } else { + STATUS_FATAL("A model cannot be evaluated without a label."); + } + } metric::proto::EvaluationResults eval; RETURN_IF_ERROR( metric::InitializeEvaluation(option, LabelColumnSpec(), &eval)); @@ -147,6 +155,14 @@ AbstractModel::EvaluateWithEngine( if (option.task() != task()) { STATUS_FATAL("The evaluation and the model tasks differ."); } + if (label_col_idx_ == -1) { + if (task() == proto::Task::ANOMALY_DETECTION) { + STATUS_FATAL( + "Cannot evaluate an anomaly detection model without a label."); + } else { + STATUS_FATAL("A model cannot be evaluated without a label."); + } + } metric::proto::EvaluationResults eval; RETURN_IF_ERROR( metric::InitializeEvaluation(option, LabelColumnSpec(), &eval)); @@ -313,6 +329,12 @@ void FloatToProtoPrediction(const std::vector& src_prediction, src_prediction.begin() + (example_idx + 1) * num_prediction_dimensions}; break; + + case proto::ANOMALY_DETECTION: + DCHECK_EQ(num_prediction_dimensions, 1); + dst_prediction->mutable_anomaly_detection()->set_value( + src_prediction[example_idx]); + break; } } @@ -660,6 +682,9 @@ absl::Status SetGroundTruth(const dataset::VerticalDataset& dataset, ->values(); prediction->mutable_uplift()->set_treatment(treatments[row_idx]); } break; + case proto::Task::ANOMALY_DETECTION: + // No ground truth to set. + break; default: STATUS_FATAL("Non supported task."); @@ -707,6 +732,10 @@ absl::Status SetGroundTruth(const dataset::proto::Example& example, break; } } break; + case proto::Task::ANOMALY_DETECTION: + // No ground truth to set. + break; + default: STATUS_FATAL("Non supported task."); break; @@ -725,8 +754,10 @@ void AbstractModel::AppendDescriptionAndStatistics( const bool full_definition, std::string* description) const { absl::StrAppendFormat(description, "Type: \"%s\"\n", name()); absl::StrAppendFormat(description, "Task: %s\n", proto::Task_Name(task())); - absl::StrAppendFormat(description, "Label: \"%s\"\n", - data_spec().columns(label_col_idx_).name()); + if (label_col_idx_ != -1) { + absl::StrAppendFormat(description, "Label: \"%s\"\n", + data_spec().columns(label_col_idx_).name()); + } if (ranking_group_col_idx_ != -1) { absl::StrAppendFormat(description, "Rank group: \"%s\"\n", data_spec().columns(ranking_group_col_idx_).name()); @@ -1041,6 +1072,11 @@ void PredictionMerger::Add(const proto::Prediction& src, dst_->mutable_ranking()->set_relevance( dst_->ranking().relevance() + src_factor * src.ranking().relevance()); break; + case proto::Prediction::kAnomalyDetection: + dst_->mutable_anomaly_detection()->set_value( + dst_->anomaly_detection().value() + + src_factor * src.anomaly_detection().value()); + break; default: CHECK(false); } @@ -1066,6 +1102,10 @@ void PredictionMerger::ScalePrediction(const float scale, case proto::Prediction::kRanking: dst->mutable_ranking()->set_relevance(dst->ranking().relevance() * scale); break; + case proto::Prediction::kAnomalyDetection: + dst->mutable_anomaly_detection()->set_value( + dst->anomaly_detection().value() * scale); + break; default: break; } @@ -1092,7 +1132,7 @@ void AbstractModel::CopyAbstractModelMetaData(AbstractModel* dst) const { } absl::Status AbstractModel::Validate() const { - if (label_col_idx_ < 0 || label_col_idx_ >= data_spec().columns_size()) { + if (label_col_idx_ < -1 || label_col_idx_ >= data_spec().columns_size()) { return absl::InvalidArgumentError("Invalid label column"); } @@ -1147,6 +1187,9 @@ absl::Status AbstractModel::Validate() const { dataset::proto::ColumnType_Name(label_col_spec().type()))); } break; + case model::proto::Task::ANOMALY_DETECTION: + // Nothing to check + break; default: return absl::InvalidArgumentError("Unknown task"); } diff --git a/yggdrasil_decision_forests/model/abstract_model.proto b/yggdrasil_decision_forests/model/abstract_model.proto index b50c47ef..f4a660f4 100644 --- a/yggdrasil_decision_forests/model/abstract_model.proto +++ b/yggdrasil_decision_forests/model/abstract_model.proto @@ -39,6 +39,12 @@ enum Task { // Predicts the incremental impact of a treatment on a numerical outcome. // See https://en.wikipedia.org/wiki/Uplift_modelling. NUMERICAL_UPLIFT = 5; + + // Predicts if an instance is similar to the majority of the training data or + // anomalous (a.k.a. an outlier). An anomaly detection prediction is a value + // between 0 and 1, where 0 indicates the possible most normal instance and 1 + // indicates the most possible anomalous instance. + ANOMALY_DETECTION = 6; } // Contains the same information as a model::AbstractModel (without the diff --git a/yggdrasil_decision_forests/model/abstract_model_test.cc b/yggdrasil_decision_forests/model/abstract_model_test.cc index d9620bf0..8b534145 100644 --- a/yggdrasil_decision_forests/model/abstract_model_test.cc +++ b/yggdrasil_decision_forests/model/abstract_model_test.cc @@ -254,6 +254,28 @@ TEST(AbstractLearner, MergeAddPredictionsClassification) { .value())); } +TEST(AbstractLearner, MergeAddPredictionsAnomalyDetection) { + proto::Prediction src = + PARSE_TEST_PROTO(R"pb(anomaly_detection { value: 1 })pb"); + proto::Prediction dst; + PredictionMerger merger(&dst); + + merger.Add(src, 0.25f); + EXPECT_THAT(dst, EqualsProto(utils::ParseTextProto( + "anomaly_detection {value:0.25 }") + .value())); + + merger.Add(src, 0.25f); + EXPECT_THAT(dst, EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 0.5 }") + .value())); + + merger.Add(src, 0.50f); + EXPECT_THAT(dst, EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 1.0 }") + .value())); +} + TEST(AbstractModel, BuildFastEngine) { FakeModelWithoutEngine model_without_engine; EXPECT_THAT(model_without_engine.BuildFastEngine().status(), @@ -395,6 +417,13 @@ TEST(FloatToProtoPrediction, Base) { EXPECT_THAT(prediction, EqualsProto(utils::ParseTextProto( R"(uplift { treatment_effect: 0.4 })") .value())); + + FloatToProtoPrediction({0.2, 0.4}, /*example_idx=*/0, + proto::Task::ANOMALY_DETECTION, + /*num_prediction_dimensions=*/1, &prediction); + EXPECT_THAT(prediction, EqualsProto(utils::ParseTextProto( + R"(anomaly_detection { value: 0.2 })") + .value())); } TEST(Evaluate, FromVerticalDataset) { diff --git a/yggdrasil_decision_forests/model/decision_tree/decision_tree.cc b/yggdrasil_decision_forests/model/decision_tree/decision_tree.cc index 78d908af..ec7d87dc 100644 --- a/yggdrasil_decision_forests/model/decision_tree/decision_tree.cc +++ b/yggdrasil_decision_forests/model/decision_tree/decision_tree.cc @@ -1200,19 +1200,20 @@ void AppendModelStructureHeader( const DecisionForest& trees, const dataset::proto::DataSpecification& data_spec, const int label_col_idx, std::string* description) { - const auto& label_col_spec = data_spec.columns(label_col_idx); - - // Print the label values. - if (label_col_spec.type() == dataset::proto::CATEGORICAL && - !label_col_spec.categorical().is_already_integerized()) { - absl::StrAppend(description, "Label values:\n"); - for (int value = 1; - value < label_col_spec.categorical().number_of_unique_values(); - value++) { - absl::StrAppend( - description, "\t", - dataset::CategoricalIdxToRepresentation(label_col_spec, value, true), - "\n"); + if (label_col_idx != -1) { + const auto& label_col_spec = data_spec.columns(label_col_idx); + // Print the label values. + if (label_col_spec.type() == dataset::proto::CATEGORICAL && + !label_col_spec.categorical().is_already_integerized()) { + absl::StrAppend(description, "Label values:\n"); + for (int value = 1; + value < label_col_spec.categorical().number_of_unique_values(); + value++) { + absl::StrAppend(description, "\t", + dataset::CategoricalIdxToRepresentation(label_col_spec, + value, true), + "\n"); + } } } diff --git a/yggdrasil_decision_forests/model/prediction.proto b/yggdrasil_decision_forests/model/prediction.proto index dce365d6..9cfa5b3f 100644 --- a/yggdrasil_decision_forests/model/prediction.proto +++ b/yggdrasil_decision_forests/model/prediction.proto @@ -71,16 +71,22 @@ message Prediction { } } + message AnomalyDetection { + // Value between 0 (normal) and 1 (anomaly). + optional float value = 1; + } + oneof type { Classification classification = 1; Regression regression = 2; Ranking ranking = 5; Uplift uplift = 6; + AnomalyDetection anomaly_detection = 7; } optional float weight = 3 [default = 1]; // Identifier about the example. optional string example_key = 4; - // Next ID: 6 + // Next ID: 8 } diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py index 37da48e0..4687390a 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py @@ -77,7 +77,7 @@ def __init__( self._deployment_config = deployment_config self._tuner = tuner - if not self._label: + if task != Task.ANOMALY_DETECTION and not self._label: raise ValueError("Constructing the learner requires a non-empty label.") if self._ranking_group is not None and task != Task.RANKING: raise ValueError( @@ -276,6 +276,7 @@ def _get_training_config(self) -> abstract_learner_pb2.TrainingConfig: # Apply monotonic constraints. if self._data_spec_args.columns: for feature in self._data_spec_args.columns: + assert feature is not None if not feature.normalized_monotonic: continue @@ -457,7 +458,7 @@ def _build_data_spec_args(self) -> dataspec.DataSpecInferenceArgs: column are specified as features. """ - def create_label_column(name: str, task: Task) -> dataspec.Column: + def create_label_column(name: str, task: Task) -> Optional[dataspec.Column]: if task in [Task.CLASSIFICATION, Task.CATEGORICAL_UPLIFT]: return dataspec.Column( name=name, @@ -467,6 +468,9 @@ def create_label_column(name: str, task: Task) -> dataspec.Column: ) elif task in [Task.REGRESSION, Task.RANKING, Task.NUMERICAL_UPLIFT]: return dataspec.Column(name=name, semantic=dataspec.Semantic.NUMERICAL) + elif task in [Task.ANOMALY_DETECTION]: + # No label column + return None else: raise ValueError(f"Unsupported task {task.name} for label column") @@ -485,7 +489,10 @@ def create_label_column(name: str, task: Task) -> dataspec.Column: f"Label column {self._label} is also an input feature. A column" " cannot be both a label and input feature." ) - column_defs.append(create_label_column(self._label, self._task)) + if ( + label_column := create_label_column(self._label, self._task) + ) is not None: + column_defs.append(label_column) if self._weights is not None: if dataspec.column_defs_contains_column(self._weights, column_defs): raise ValueError( diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/specialized_learners_pre_generated.py b/yggdrasil_decision_forests/port/python/ydf/learner/specialized_learners_pre_generated.py index 0b19e100..d4708a2c 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/specialized_learners_pre_generated.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/specialized_learners_pre_generated.py @@ -83,7 +83,8 @@ class RandomForestLearner(generic_learner.GenericLearner): label: Label of the dataset. The label column should not be identified as a feature in the `features` parameter. task: Task to solve (e.g. Task.CLASSIFICATION, Task.REGRESSION, - Task.RANKING, Task.CATEGORICAL_UPLIFT, Task.NUMERICAL_UPLIFT). + Task.RANKING, Task.CATEGORICAL_UPLIFT, Task.NUMERICAL_UPLIFT, + Task.ANOMALY_DETECTION). weights: Name of a feature that identifies the weight of each example. If weights are not specified, unit weights are assumed. The weight column should not be identified as a feature in the `features` parameter. diff --git a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py index 6baf0fb1..a231aca1 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py @@ -64,6 +64,10 @@ class Task(enum.Enum): categorical outcome. NUMERICAL_UPLIFT: Predicts the incremental impact of a treatment on a numerical outcome. + ANOMALY_DETECTION: Predicts if an instance is similar to the majority of the + training data or anomalous (a.k.a. an outlier). An anomaly detection + prediction is a value between 0 and 1, where 0 indicates the possible most + normal instance and 1 indicates the most possible anomalous instance. """ CLASSIFICATION = "CLASSIFICATION" @@ -71,6 +75,7 @@ class Task(enum.Enum): RANKING = "RANKING" CATEGORICAL_UPLIFT = "CATEGORICAL_UPLIFT" NUMERICAL_UPLIFT = "NUMERICAL_UPLIFT" + ANOMALY_DETECTION = "ANOMALY_DETECTION" def _to_proto_type(self) -> abstract_model_pb2.Task: if self in TASK_TO_PROTO: @@ -93,6 +98,7 @@ def _from_proto_type(cls, task: abstract_model_pb2.Task): Task.RANKING: abstract_model_pb2.RANKING, Task.CATEGORICAL_UPLIFT: abstract_model_pb2.CATEGORICAL_UPLIFT, Task.NUMERICAL_UPLIFT: abstract_model_pb2.NUMERICAL_UPLIFT, + Task.ANOMALY_DETECTION: abstract_model_pb2.ANOMALY_DETECTION, } PROTO_TO_TASK = {v: k for k, v in TASK_TO_PROTO.items()} diff --git a/yggdrasil_decision_forests/utils/model_analysis.cc b/yggdrasil_decision_forests/utils/model_analysis.cc index 92928dfa..b111e3f1 100644 --- a/yggdrasil_decision_forests/utils/model_analysis.cc +++ b/yggdrasil_decision_forests/utils/model_analysis.cc @@ -953,7 +953,6 @@ absl::StatusOr FeatureVariationNumerical( return item; } - absl::StatusOr FeatureVariationBoolean( const model::AbstractModel& model, const int column_idx, const dataset::proto::Example& example, @@ -1085,6 +1084,14 @@ absl::StatusOr> ListOutputs( return prediction.uplift().treatment_effect(0); }}); break; + case model::proto::Task::ANOMALY_DETECTION: + outputs.push_back( + {.label = "output", + .compute = [](const model::proto::Prediction& prediction) -> float { + return prediction.anomaly_detection().value(); + }}); + break; + default: return absl::InvalidArgumentError( "Non supported model task for feature variation"); diff --git a/yggdrasil_decision_forests/utils/test_utils.cc b/yggdrasil_decision_forests/utils/test_utils.cc index 2d20399a..241ba7cb 100644 --- a/yggdrasil_decision_forests/utils/test_utils.cc +++ b/yggdrasil_decision_forests/utils/test_utils.cc @@ -308,6 +308,9 @@ void TrainAndTestTester::TrainAndEvaluateModel( EXPECT_NEAR(metric::AUUC(e1), metric::AUUC(e2), 0.001); EXPECT_NEAR(metric::Qini(e1), metric::Qini(e2), 0.001); break; + case model::proto::Task::ANOMALY_DETECTION: + // No metrics + break; default: YDF_LOG(FATAL) << "Not implemented"; } @@ -644,6 +647,11 @@ void ExpectEqualPredictions(const model::proto::Task task, } } break; + case model::proto::Task::ANOMALY_DETECTION: + EXPECT_NEAR(a.anomaly_detection().value(), b.anomaly_detection().value(), + epsilon); + break; + default: YDF_LOG(FATAL) << "Not supported task"; } @@ -743,6 +751,12 @@ void ExpectEqualPredictions( } } break; + case model::proto::Task::ANOMALY_DETECTION: + EXPECT_NEAR(generic_prediction.anomaly_detection().value(), + predictions[prediction_idx], epsilon) + << "Predictions don't match."; + break; + default: YDF_LOG(FATAL) << "Not supported task"; } From f3195892994f22d495fe52a968fe51ecf7c6af3b Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Thu, 13 Jun 2024 03:22:53 -0700 Subject: [PATCH 18/30] Anomaly detection; ccleaner + decision tree utilities (part 2) PiperOrigin-RevId: 642919627 --- .../distributed_gradient_boosted_trees/BUILD | 1 + .../distributed_gradient_boosted_trees.cc | 1 + .../learner/gradient_boosted_trees/BUILD | 6 ++++ .../gradient_boosted_trees.cc | 11 ++++++ .../gradient_boosted_trees_test.cc | 1 + yggdrasil_decision_forests/model/BUILD | 3 +- .../model/abstract_model.cc | 14 ++++++++ .../model/abstract_model_test.cc | 36 ++++++++++++++----- .../model/decision_tree/BUILD | 3 ++ .../model/decision_tree/builder.cc | 9 +++++ .../model/decision_tree/builder.h | 3 ++ .../model/decision_tree/builder_test.cc | 18 ++++++++++ .../model/decision_tree/decision_tree.cc | 20 ++++++++--- .../model/decision_tree/decision_tree.h | 14 +++++++- .../model/decision_tree/decision_tree.proto | 16 +++++++-- .../model/decision_tree/decision_tree_test.cc | 21 +++++++++++ .../build_decision_tree_anomaly.txt.expected | 3 ++ yggdrasil_decision_forests/utils/BUILD | 11 +++++- .../utils/feature_importance.cc | 3 +- .../utils/test_utils.cc | 14 ++++++-- 20 files changed, 185 insertions(+), 23 deletions(-) create mode 100644 yggdrasil_decision_forests/test_data/golden/build_decision_tree_anomaly.txt.expected diff --git a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/BUILD b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/BUILD index 86812a46..a173a6b0 100644 --- a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/BUILD +++ b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/BUILD @@ -34,6 +34,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/model/gradient_boosted_trees", "//yggdrasil_decision_forests/utils:filesystem", "//yggdrasil_decision_forests/utils:logging", + "//yggdrasil_decision_forests/utils:sharded_io", "//yggdrasil_decision_forests/utils:snapshot", "//yggdrasil_decision_forests/utils:status_macros", "//yggdrasil_decision_forests/utils:uid", diff --git a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.cc b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.cc index 2794aa4f..2bfb37e4 100644 --- a/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.cc +++ b/yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees/distributed_gradient_boosted_trees.cc @@ -29,6 +29,7 @@ #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.h" #include "yggdrasil_decision_forests/utils/filesystem.h" #include "yggdrasil_decision_forests/utils/logging.h" +#include "yggdrasil_decision_forests/utils/sharded_io.h" #include "yggdrasil_decision_forests/utils/snapshot.h" #include "yggdrasil_decision_forests/utils/status_macros.h" #include "yggdrasil_decision_forests/utils/uid.h" diff --git a/yggdrasil_decision_forests/learner/gradient_boosted_trees/BUILD b/yggdrasil_decision_forests/learner/gradient_boosted_trees/BUILD index 180b4fad..3b2d8b94 100644 --- a/yggdrasil_decision_forests/learner/gradient_boosted_trees/BUILD +++ b/yggdrasil_decision_forests/learner/gradient_boosted_trees/BUILD @@ -45,17 +45,22 @@ cc_library_ydf( "//yggdrasil_decision_forests/model/decision_tree:decision_tree_cc_proto", "//yggdrasil_decision_forests/model/gradient_boosted_trees", "//yggdrasil_decision_forests/model/gradient_boosted_trees:gradient_boosted_trees_cc_proto", + "//yggdrasil_decision_forests/serving:example_set", + "//yggdrasil_decision_forests/serving:fast_engine", "//yggdrasil_decision_forests/serving/decision_forest:register_engines", "//yggdrasil_decision_forests/utils:adaptive_work", "//yggdrasil_decision_forests/utils:compatibility", + "//yggdrasil_decision_forests/utils:concurrency", "//yggdrasil_decision_forests/utils:csv", "//yggdrasil_decision_forests/utils:feature_importance", "//yggdrasil_decision_forests/utils:filesystem", "//yggdrasil_decision_forests/utils:hyper_parameters", "//yggdrasil_decision_forests/utils:logging", "//yggdrasil_decision_forests/utils:random", + "//yggdrasil_decision_forests/utils:sharded_io", "//yggdrasil_decision_forests/utils:snapshot", "//yggdrasil_decision_forests/utils:status_macros", + "//yggdrasil_decision_forests/utils:synchronization_primitives", "@com_google_absl//absl/container:fixed_array", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -134,6 +139,7 @@ cc_test( "//yggdrasil_decision_forests/utils:logging", "//yggdrasil_decision_forests/utils:model_analysis", "//yggdrasil_decision_forests/utils:random", + "//yggdrasil_decision_forests/utils:sharded_io", "//yggdrasil_decision_forests/utils:snapshot", "//yggdrasil_decision_forests/utils:status_macros", "//yggdrasil_decision_forests/utils:test", diff --git a/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.cc b/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.cc index f4533521..43212498 100644 --- a/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.cc +++ b/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees.cc @@ -18,12 +18,15 @@ #include #include #include +#include #include +#include #include #include #include #include #include +#include #include #include @@ -34,9 +37,12 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" #include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" #include "absl/time/clock.h" #include "absl/time/time.h" +#include "absl/types/optional.h" #include "absl/types/span.h" #include "yggdrasil_decision_forests/dataset/data_spec.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" @@ -60,15 +66,20 @@ #include "yggdrasil_decision_forests/model/decision_tree/decision_tree.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.pb.h" +#include "yggdrasil_decision_forests/serving/example_set.h" +#include "yggdrasil_decision_forests/serving/fast_engine.h" #include "yggdrasil_decision_forests/utils/adaptive_work.h" +#include "yggdrasil_decision_forests/utils/concurrency.h" #include "yggdrasil_decision_forests/utils/csv.h" #include "yggdrasil_decision_forests/utils/feature_importance.h" #include "yggdrasil_decision_forests/utils/filesystem.h" #include "yggdrasil_decision_forests/utils/hyper_parameters.h" #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/random.h" +#include "yggdrasil_decision_forests/utils/sharded_io.h" #include "yggdrasil_decision_forests/utils/snapshot.h" #include "yggdrasil_decision_forests/utils/status_macros.h" +#include "yggdrasil_decision_forests/utils/synchronization_primitives.h" namespace yggdrasil_decision_forests { namespace model { diff --git a/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees_test.cc b/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees_test.cc index f0a8d548..206364b7 100644 --- a/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees_test.cc +++ b/yggdrasil_decision_forests/learner/gradient_boosted_trees/gradient_boosted_trees_test.cc @@ -69,6 +69,7 @@ #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/model_analysis.h" #include "yggdrasil_decision_forests/utils/random.h" +#include "yggdrasil_decision_forests/utils/sharded_io.h" #include "yggdrasil_decision_forests/utils/snapshot.h" #include "yggdrasil_decision_forests/utils/status_macros.h" #include "yggdrasil_decision_forests/utils/test.h" diff --git a/yggdrasil_decision_forests/model/BUILD b/yggdrasil_decision_forests/model/BUILD index 97b10344..a0e70fe9 100644 --- a/yggdrasil_decision_forests/model/BUILD +++ b/yggdrasil_decision_forests/model/BUILD @@ -151,7 +151,6 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:html_content", "//yggdrasil_decision_forests/utils:plot", "//yggdrasil_decision_forests/utils:protobuf", - "//yggdrasil_decision_forests/utils:uid", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -208,6 +207,7 @@ cc_test( deps = [ ":abstract_model", ":abstract_model_cc_proto", + ":evaluate_on_disk", ":model_library", ":model_testing", ":prediction_cc_proto", @@ -216,7 +216,6 @@ cc_test( "//yggdrasil_decision_forests/dataset:vertical_dataset", "//yggdrasil_decision_forests/dataset:vertical_dataset_io", "//yggdrasil_decision_forests/metric", - "//yggdrasil_decision_forests/model:evaluate_on_disk", "//yggdrasil_decision_forests/model/gradient_boosted_trees", "//yggdrasil_decision_forests/model/random_forest", "//yggdrasil_decision_forests/serving:example_set", diff --git a/yggdrasil_decision_forests/model/abstract_model.cc b/yggdrasil_decision_forests/model/abstract_model.cc index 919c5bb6..c21d99ab 100644 --- a/yggdrasil_decision_forests/model/abstract_model.cc +++ b/yggdrasil_decision_forests/model/abstract_model.cc @@ -484,15 +484,18 @@ absl::Status AbstractModel::AppendEvaluationOverrideType( sub_example_idx++) { FloatToProtoPrediction(batch_of_predictions, sub_example_idx, task(), num_prediction_dimensions, &original_prediction); + RETURN_IF_ERROR(ChangePredictionType(task(), override_task, original_prediction, &overridden_prediction)); + RETURN_IF_ERROR(model::SetGroundTruth( dataset, begin_example_idx + sub_example_idx, model::GroundTruthColumnIndices(override_label_col_idx, override_group_col_idx, uplift_treatment_col_idx_), override_task, &overridden_prediction)); + if (option.has_weights()) { ASSIGN_OR_RETURN( const float weight, @@ -558,6 +561,17 @@ absl::Status ChangePredictionType(proto::Task src_task, proto::Task dst_task, } else if (src_task == proto::Task::RANKING && dst_task == proto::Task::REGRESSION) { dst_pred->mutable_regression()->set_value(src_pred.ranking().relevance()); + } else if (src_task == proto::Task::ANOMALY_DETECTION && + dst_task == proto::Task::CLASSIFICATION) { + const float value = src_pred.anomaly_detection().value(); + auto* dst_clas = dst_pred->mutable_classification(); + // Assume the positive class is the abnormal one. + dst_clas->set_value(value >= 0.5f ? 2 : 1); + dst_clas->mutable_distribution()->clear_counts(); + dst_clas->mutable_distribution()->set_sum(1.f); + dst_clas->mutable_distribution()->add_counts(0.f); + dst_clas->mutable_distribution()->add_counts(1.f - value); + dst_clas->mutable_distribution()->add_counts(value); } else { STATUS_FATALS("Non supported override of task from ", proto::Task_Name(src_task), " to ", diff --git a/yggdrasil_decision_forests/model/abstract_model_test.cc b/yggdrasil_decision_forests/model/abstract_model_test.cc index 8b534145..230c9049 100644 --- a/yggdrasil_decision_forests/model/abstract_model_test.cc +++ b/yggdrasil_decision_forests/model/abstract_model_test.cc @@ -45,6 +45,7 @@ namespace yggdrasil_decision_forests { namespace model { namespace { +using test::ApproximatelyEqualsProto; using test::EqualsProto; using test::StatusIs; @@ -336,20 +337,39 @@ TEST(ChangePredictionType, ClassificationToRanking) { } } +TEST(ChangePredictionType, AnomalyDetectionToClassification) { + const proto::Prediction src_pred = + PARSE_TEST_PROTO(R"pb(anomaly_detection { value: 0.8 })pb"); + proto::Prediction dst_pred; + ASSERT_OK(ChangePredictionType(proto::Task::ANOMALY_DETECTION, + proto::Task::CLASSIFICATION, src_pred, + &dst_pred)); + EXPECT_THAT(dst_pred, + ApproximatelyEqualsProto(PARSE_TEST_PROTO_WITH_TYPE( + proto::Prediction, + R"pb( + classification { + value: 2 + distribution { counts: 0 counts: 0.2 counts: 0.8 sum: 1 } + } + )pb"))); +} + TEST(FloatToProtoPrediction, Base) { proto::Prediction prediction; FloatToProtoPrediction({0, 0.5, 1}, /*example_idx=*/0, proto::Task::CLASSIFICATION, /*num_prediction_dimensions=*/1, &prediction); - EXPECT_THAT(prediction, - EqualsProto(utils::ParseTextProto(R"( - classification { - value: 1 - distribution { counts: 0 counts: 1 counts: 0 sum: 1 } - } - )") - .value())); + EXPECT_THAT( + prediction, + EqualsProto(utils::ParseTextProto(R"pb( + classification { + value: 1 + distribution { counts: 0 counts: 1 counts: 0 sum: 1 } + } + )pb") + .value())); FloatToProtoPrediction({0, 0.5, 1}, /*example_idx=*/1, proto::Task::CLASSIFICATION, diff --git a/yggdrasil_decision_forests/model/decision_tree/BUILD b/yggdrasil_decision_forests/model/decision_tree/BUILD index f602e057..febad499 100644 --- a/yggdrasil_decision_forests/model/decision_tree/BUILD +++ b/yggdrasil_decision_forests/model/decision_tree/BUILD @@ -40,8 +40,10 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:protobuf", "//yggdrasil_decision_forests/utils:sharded_io", "//yggdrasil_decision_forests/utils:status_macros", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:check", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -116,6 +118,7 @@ cc_library_ydf( hdrs = ["builder.h"], deps = [ ":decision_tree", + "@com_google_absl//absl/log:check", "@com_google_absl//absl/types:span", ], ) diff --git a/yggdrasil_decision_forests/model/decision_tree/builder.cc b/yggdrasil_decision_forests/model/decision_tree/builder.cc index 1cb89106..b8f69b68 100644 --- a/yggdrasil_decision_forests/model/decision_tree/builder.cc +++ b/yggdrasil_decision_forests/model/decision_tree/builder.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/check.h" #include "absl/types/span.h" namespace yggdrasil_decision_forests::model::decision_tree { @@ -72,4 +73,12 @@ void TreeBuilder::LeafRegression(const float value) { node_->mutable_node()->mutable_regressor()->set_top_value(value); } +void TreeBuilder::LeafAnomalyDetection(const int num_examples_without_weight) { + node_->mutable_node() + ->mutable_anomaly_detection() + ->set_num_examples_without_weight(num_examples_without_weight); + node_->mutable_node()->set_num_pos_training_examples_without_weight( + num_examples_without_weight); +} + } // namespace yggdrasil_decision_forests::model::decision_tree diff --git a/yggdrasil_decision_forests/model/decision_tree/builder.h b/yggdrasil_decision_forests/model/decision_tree/builder.h index fa1f001d..550b9880 100644 --- a/yggdrasil_decision_forests/model/decision_tree/builder.h +++ b/yggdrasil_decision_forests/model/decision_tree/builder.h @@ -53,6 +53,9 @@ class TreeBuilder { // Creates a regression leaf. void LeafRegression(float value); + // Creates an anomaly detection leaf. + void LeafAnomalyDetection(int num_examples_without_weight); + private: NodeWithChildren* node_; }; diff --git a/yggdrasil_decision_forests/model/decision_tree/builder_test.cc b/yggdrasil_decision_forests/model/decision_tree/builder_test.cc index fe03ee07..078a4c0b 100644 --- a/yggdrasil_decision_forests/model/decision_tree/builder_test.cc +++ b/yggdrasil_decision_forests/model/decision_tree/builder_test.cc @@ -53,6 +53,24 @@ TEST(TreeBuilder, Base) { "golden/build_decision_tree.txt.expected"); } +TEST(TreeBuilder, AnomalyDetection) { + DecisionTree tree; + TreeBuilder builder(&tree); + + dataset::proto::DataSpecification dataspec; + dataset::AddColumn("f1", dataset::proto::ColumnType::NUMERICAL, &dataspec); + + auto [l1, l2] = builder.ConditionIsGreater(/*attribute=*/0, /*threshold=*/1); + l1.LeafAnomalyDetection(2); + l2.LeafAnomalyDetection(3); + + std::string description; + tree.AppendModelStructure(dataspec, 0, &description); + test::ExpectEqualGolden(description, + "yggdrasil_decision_forests/test_data/" + "golden/build_decision_tree_anomaly.txt.expected"); +} + } // namespace } // namespace decision_tree } // namespace model diff --git a/yggdrasil_decision_forests/model/decision_tree/decision_tree.cc b/yggdrasil_decision_forests/model/decision_tree/decision_tree.cc index ec7d87dc..66cdd8aa 100644 --- a/yggdrasil_decision_forests/model/decision_tree/decision_tree.cc +++ b/yggdrasil_decision_forests/model/decision_tree/decision_tree.cc @@ -21,13 +21,13 @@ #include #include #include -#include #include #include #include #include #include +#include "absl/base/optimization.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/memory/memory.h" @@ -48,7 +48,6 @@ #include "yggdrasil_decision_forests/utils/distribution.pb.h" #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/protobuf.h" -#include "yggdrasil_decision_forests/utils/sharded_io.h" #include "yggdrasil_decision_forests/utils/status_macros.h" namespace yggdrasil_decision_forests { @@ -150,6 +149,11 @@ void AppendValueDescription(const dataset::proto::DataSpecification& data_spec, "] examples_per_treatment_and_outcome:[", sum_weights_per_treatment_and_outcome_str, "]"); } break; + + case proto::Node::OutputCase::kAnomalyDetection: + absl::StrAppend(description, "count:", + node.anomaly_detection().num_examples_without_weight()); + break; } } @@ -543,6 +547,8 @@ void NodeWithChildren::ClearLabelDistributionDetails() { break; case proto::Node::OutputCase::kUplift: break; + case proto::Node::OutputCase::kAnomalyDetection: + break; } } @@ -895,7 +901,7 @@ const proto::Node& DecisionTree::GetLeafWithSwappedAttribute( return current_node->node(); } -const proto::Node& DecisionTree::GetLeaf( +const NodeWithChildren& DecisionTree::GetLeafAlt( const dataset::proto::Example& example) const { // Go down the tree according to an observation attribute values. CHECK(root_ != nullptr); @@ -906,7 +912,12 @@ const proto::Node& DecisionTree::GetLeaf( current_node = condition_result ? current_node->pos_child() : current_node->neg_child(); } - return current_node->node(); + return *current_node; +} + +const proto::Node& DecisionTree::GetLeaf( + const dataset::proto::Example& example) const { + return GetLeafAlt(example).node(); } void DecisionTree::GetPath(const dataset::VerticalDataset& dataset, @@ -1247,6 +1258,7 @@ void DecisionTree::SetLeafIndices() { if (node->IsLeaf()) { node->set_leaf_idx(next_leaf_idx++); } + node->set_depth(depth); }, /*neg_before_pos_child=*/true); } diff --git a/yggdrasil_decision_forests/model/decision_tree/decision_tree.h b/yggdrasil_decision_forests/model/decision_tree/decision_tree.h index 6d1ec72e..432cb5ef 100644 --- a/yggdrasil_decision_forests/model/decision_tree/decision_tree.h +++ b/yggdrasil_decision_forests/model/decision_tree/decision_tree.h @@ -23,12 +23,14 @@ #include +#include #include #include #include #include #include +#include "absl/log/check.h" #include "absl/status/status.h" #include "absl/types/optional.h" #include "absl/types/span.h" @@ -37,7 +39,7 @@ #include "yggdrasil_decision_forests/dataset/vertical_dataset.h" #include "yggdrasil_decision_forests/model/abstract_model.pb.h" #include "yggdrasil_decision_forests/model/decision_tree/decision_tree.pb.h" -#include "yggdrasil_decision_forests/utils/sharded_io.h" +#include "yggdrasil_decision_forests/utils/protobuf.h" namespace yggdrasil_decision_forests { namespace model { @@ -203,6 +205,9 @@ class NodeWithChildren { int32_t leaf_idx() const { return leaf_idx_; } void set_leaf_idx(const int32_t v) { leaf_idx_ = v; } + int32_t depth() const { return depth_; } + void set_depth(const int32_t v) { depth_ = v; } + // Compare a tree to another tree. If they are equal, return an empty string. // If they are different, returns an explanation of the differences. std::string DebugCompare(const dataset::proto::DataSpecification& dataspec, @@ -219,6 +224,10 @@ class NodeWithChildren { // Index of the leaf (if the node is a leaf) in the tree in a depth first // exploration. It is set by calling "SetLeafIndices()". int32_t leaf_idx_ = -1; + + // Depth of the node. Assuming that the root node has depth 0. It is set by + // calling "SetLeafIndices()". + int32_t depth_ = -1; }; // A generic decision tree. This class is designed for cheap modification (by @@ -270,6 +279,9 @@ class DecisionTree { const dataset::VerticalDataset& dataset, dataset::VerticalDataset::row_t row_idx) const; + const NodeWithChildren& GetLeafAlt( + const dataset::proto::Example& example) const; + // Apply the decision tree on an example and returns the path. void GetPath(const dataset::VerticalDataset& dataset, dataset::VerticalDataset::row_t row_idx, diff --git a/yggdrasil_decision_forests/model/decision_tree/decision_tree.proto b/yggdrasil_decision_forests/model/decision_tree/decision_tree.proto index 86765cf1..03065494 100644 --- a/yggdrasil_decision_forests/model/decision_tree/decision_tree.proto +++ b/yggdrasil_decision_forests/model/decision_tree/decision_tree.proto @@ -73,6 +73,14 @@ message NodeUpliftOutput { repeated int64 num_examples_per_treatment = 5 [packed = true]; } +// Output of a node in an anomaliy detection tree. +message NodeAnomalyDetectionOutput { + // Next ID: 2 + + // Number of examples that reached this node. + optional int64 num_examples_without_weight = 1; +} + // The sub-messages of "ConditionParams" are the different types of condition // that can be attached to a node. message Condition { @@ -161,17 +169,19 @@ message NodeCondition { // Node in a decision tree (without the information about the children). message Node { - // Next ID: 6 + // Next ID: 7 // Label value. Might be unspecified for non-leaf nodes. oneof output { NodeClassifierOutput classifier = 1; NodeRegressorOutput regressor = 2; NodeUpliftOutput uplift = 5; + NodeAnomalyDetectionOutput anomaly_detection = 6; } // Branching condition to the children. If not specified, this node is a leaf. optional NodeCondition condition = 3; - // Number of positive examples (non-weighted) that reached this node during - // training. + // Number of examples (non-weighted) that reached this node during + // training. Warning: Contrary to what the name suggest, this is not the count + // of examples branched to the positive child. optional int64 num_pos_training_examples_without_weight = 4; } diff --git a/yggdrasil_decision_forests/model/decision_tree/decision_tree_test.cc b/yggdrasil_decision_forests/model/decision_tree/decision_tree_test.cc index 47c51187..0b5c1d86 100644 --- a/yggdrasil_decision_forests/model/decision_tree/decision_tree_test.cc +++ b/yggdrasil_decision_forests/model/decision_tree/decision_tree_test.cc @@ -811,6 +811,27 @@ TEST(DecisionTree, DebugCompare) { EXPECT_TRUE(tree1.DebugCompare(dataspec, 0, tree2).empty()); } +TEST(DecisionTree, Depth) { + dataset::proto::DataSpecification dataspec; + dataset::AddColumn("f", dataset::proto::ColumnType::NUMERICAL, &dataspec); + + DecisionTree tree; + TreeBuilder builder(&tree); + + auto [pos, l1] = builder.ConditionIsGreater(1, 1); + auto [l2, l3] = pos.ConditionIsGreater(1, 2); + l1.LeafRegression(1); + l2.LeafRegression(2); + l3.LeafRegression(3); + tree.SetLeafIndices(); + + EXPECT_EQ(tree.root().depth(), 0); + EXPECT_EQ(tree.root().pos_child()->depth(), 1); + EXPECT_EQ(tree.root().neg_child()->depth(), 1); + EXPECT_EQ(tree.root().pos_child()->pos_child()->depth(), 2); + EXPECT_EQ(tree.root().pos_child()->neg_child()->depth(), 2); +} + } // namespace } // namespace decision_tree } // namespace model diff --git a/yggdrasil_decision_forests/test_data/golden/build_decision_tree_anomaly.txt.expected b/yggdrasil_decision_forests/test_data/golden/build_decision_tree_anomaly.txt.expected new file mode 100644 index 00000000..d16dbb03 --- /dev/null +++ b/yggdrasil_decision_forests/test_data/golden/build_decision_tree_anomaly.txt.expected @@ -0,0 +1,3 @@ + "f1">=1 [s:0 n:0 np:0 miss:0] + ├─(pos)─ count:2 + └─(neg)─ count:3 diff --git a/yggdrasil_decision_forests/utils/BUILD b/yggdrasil_decision_forests/utils/BUILD index da5dd41b..5f42c7b5 100644 --- a/yggdrasil_decision_forests/utils/BUILD +++ b/yggdrasil_decision_forests/utils/BUILD @@ -513,6 +513,13 @@ cc_library_ydf( ], ) +cc_library( + name = "documentation_cc", + srcs = ["documentation.cc"], + hdrs = ["documentation.h"], + deps = ["@com_google_absl//absl/strings"], +) + cc_library_ydf( name = "feature_importance", srcs = ["feature_importance.cc"], @@ -550,6 +557,7 @@ cc_library_ydf( ":filesystem", ":logging", ":random", + ":sharded_io", ":test", ":testing_macros", ":uid", @@ -579,12 +587,13 @@ cc_library_ydf( "//yggdrasil_decision_forests/serving:example_set", "//yggdrasil_decision_forests/serving:fast_engine", "//yggdrasil_decision_forests/serving/decision_forest", + "@com_google_absl//absl/memory", "@com_google_absl//absl/random", - "@com_google_absl//absl/random:distributions", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", + "@com_google_absl//absl/types:optional", "@com_google_googletest//:gtest", ], ) diff --git a/yggdrasil_decision_forests/utils/feature_importance.cc b/yggdrasil_decision_forests/utils/feature_importance.cc index 408fbed7..b8a2e8de 100644 --- a/yggdrasil_decision_forests/utils/feature_importance.cc +++ b/yggdrasil_decision_forests/utils/feature_importance.cc @@ -235,7 +235,8 @@ absl::Status ComputePermutationFeatureImportance( utils::RandomEngine rng; utils::concurrency::Mutex rng_mutex; - const auto base_evaluation = model->Evaluate(dataset, eval_options, &rng); + ASSIGN_OR_RETURN(const auto base_evaluation, + model->EvaluateWithStatus(dataset, eval_options, &rng)); const auto permutation_evaluation = [&dataset, &eval_options, &rng, &rng_mutex, model](const int feature_idx) diff --git a/yggdrasil_decision_forests/utils/test_utils.cc b/yggdrasil_decision_forests/utils/test_utils.cc index 241ba7cb..d05a0052 100644 --- a/yggdrasil_decision_forests/utils/test_utils.cc +++ b/yggdrasil_decision_forests/utils/test_utils.cc @@ -15,19 +15,25 @@ #include "yggdrasil_decision_forests/utils/test_utils.h" -#include - #include +#include +#include +#include +#include #include +#include +#include #include +#include #include #include +#include #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "absl/random/distributions.h" +#include "absl/memory/memory.h" #include "absl/random/random.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -35,6 +41,7 @@ #include "absl/strings/string_view.h" #include "absl/time/clock.h" #include "absl/time/time.h" +#include "absl/types/optional.h" #include "yggdrasil_decision_forests/dataset/data_spec.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" #include "yggdrasil_decision_forests/dataset/data_spec_inference.h" @@ -63,6 +70,7 @@ #include "yggdrasil_decision_forests/utils/filesystem.h" #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/random.h" +#include "yggdrasil_decision_forests/utils/sharded_io.h" #include "yggdrasil_decision_forests/utils/test.h" #include "yggdrasil_decision_forests/utils/testing_macros.h" #include "yggdrasil_decision_forests/utils/uid.h" From f63eecc9bdf480afe94a5ff13366d4b579746a4c Mon Sep 17 00:00:00 2001 From: TensorFlow Decision Forests Team Date: Thu, 13 Jun 2024 05:13:17 -0700 Subject: [PATCH 19/30] Automated Code Change PiperOrigin-RevId: 642948182 --- yggdrasil_decision_forests/learner/BUILD | 1 + yggdrasil_decision_forests/learner/export_doc_main.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/yggdrasil_decision_forests/learner/BUILD b/yggdrasil_decision_forests/learner/BUILD index 15d94263..40907240 100644 --- a/yggdrasil_decision_forests/learner/BUILD +++ b/yggdrasil_decision_forests/learner/BUILD @@ -16,6 +16,7 @@ cc_binary_ydf( ":export_doc", ":learner_library", "//yggdrasil_decision_forests/utils:logging", + "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/strings", ], ) diff --git a/yggdrasil_decision_forests/learner/export_doc_main.cc b/yggdrasil_decision_forests/learner/export_doc_main.cc index 51d36a2c..8a474dd6 100644 --- a/yggdrasil_decision_forests/learner/export_doc_main.cc +++ b/yggdrasil_decision_forests/learner/export_doc_main.cc @@ -18,6 +18,7 @@ // #include +#include "absl/flags/flag.h" #include "absl/strings/substitute.h" #include "yggdrasil_decision_forests/learner/export_doc.h" #include "yggdrasil_decision_forests/learner/learner_library.h" From 225fd58505e92665eeedf3e7581d4f0faa75398f Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Fri, 14 Jun 2024 05:57:12 -0700 Subject: [PATCH 20/30] Add YDF + JAX tutorial PiperOrigin-RevId: 643324787 --- .../docs/tutorial/compose_with_jax.ipynb | 4658 +++++++++++++++++ documentation/public/mkdocs.yml | 1 + 2 files changed, 4659 insertions(+) create mode 100644 documentation/public/docs/tutorial/compose_with_jax.ipynb diff --git a/documentation/public/docs/tutorial/compose_with_jax.ipynb b/documentation/public/docs/tutorial/compose_with_jax.ipynb new file mode 100644 index 00000000..83037021 --- /dev/null +++ b/documentation/public/docs/tutorial/compose_with_jax.ipynb @@ -0,0 +1,4658 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "OM2KIwYEd0pZ" + }, + "source": [ + "# With JAX\n", + "\n", + "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/google/yggdrasil-decision-forests/blob/main/documentation/public/docs/tutorial/compose_with_jax.ipynb)\n", + "\n", + "## About this tutorial\n", + "\n", + "JAX is a machine learning library to train neural network models. While decision forests trained by YDF are different from neural networks, YDF and JAX can be combined to create powerful hybrid models.\n", + "\n", + "This tutorial is divided into two parts. First, we show how to convert a YDF model into a JAX model, and how to save the resulting model as a SavedModel using `jax2tf`.\n", + "\n", + "Second, we show how YDF and JAX can be combined to solve the distribution shift problem: We train a YDF model, convert it to a JAX model, finetune it using JAX, and convert it back to a YDF model." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ySnymYVDiS7d" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qcIsaJiTiSk2" + }, + "outputs": [], + "source": [ + "# Install dependencies\n", + "!pip install ydf -U -q\n", + "!pip install tensorflow -U -q\n", + "!pip install optax pandas numpy -U -q\n", + "\n", + "!pip install jax[cpu] -U\n", + "# OR\n", + "# !pip install jax[cuda12] -U -q\n", + "# See https://jax.readthedocs.io/en/latest/installation.html for JAX variations." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "id": "8Pm9OxVztyCU" + }, + "outputs": [], + "source": [ + "import tempfile\n", + "\n", + "import jax\n", + "from jax.experimental import jax2tf # To export JAX model to SavedModel\n", + "import optax # To finetune YDF+JAX models\n", + "import pandas as pd # We use Pandas to load small datasets\n", + "import tensorflow as tf # To create SavedModels\n", + "import ydf # Yggdrasil Decision Forests" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OLZoiM72Ek5Q" + }, + "source": [ + "## Convert YDF model into a JAX function\n", + "\n", + "In this section, we train a YDF model on the Adult dataset, convert it into a JAX function, and demonstrate various operations.\n", + "\n", + "First let's download a binary classification dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "id": "jKvcOllNx8_u" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ageworkclassfnlwgteducationeducation_nummarital_statusoccupationrelationshipracesexcapital_gaincapital_losshours_per_weeknative_countryincome
044Private2280577th-8th4Married-civ-spouseMachine-op-inspctWifeWhiteFemale0040Dominican-Republic<=50K
120Private299047Some-college10Never-marriedOther-serviceNot-in-familyWhiteFemale0020United-States<=50K
240Private342164HS-grad9SeparatedAdm-clericalUnmarriedWhiteFemale0037United-States<=50K
330Private361742Some-college10Married-civ-spouseExec-managerialHusbandWhiteMale0050United-States<=50K
467Self-emp-inc171564HS-grad9Married-civ-spouseProf-specialtyWifeWhiteFemale20051030England>50K
\n", + "
" + ], + "text/plain": [ + " age workclass fnlwgt education education_num marital_status \\\n", + "0 44 Private 228057 7th-8th 4 Married-civ-spouse \n", + "1 20 Private 299047 Some-college 10 Never-married \n", + "2 40 Private 342164 HS-grad 9 Separated \n", + "3 30 Private 361742 Some-college 10 Married-civ-spouse \n", + "4 67 Self-emp-inc 171564 HS-grad 9 Married-civ-spouse \n", + "\n", + " occupation relationship race sex capital_gain \\\n", + "0 Machine-op-inspct Wife White Female 0 \n", + "1 Other-service Not-in-family White Female 0 \n", + "2 Adm-clerical Unmarried White Female 0 \n", + "3 Exec-managerial Husband White Male 0 \n", + "4 Prof-specialty Wife White Female 20051 \n", + "\n", + " capital_loss hours_per_week native_country income \n", + "0 0 40 Dominican-Republic <=50K \n", + "1 0 20 United-States <=50K \n", + "2 0 37 United-States <=50K \n", + "3 0 50 United-States <=50K \n", + "4 0 30 England >50K " + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_path = \"https://raw.githubusercontent.com/google/yggdrasil-decision-forests/main/yggdrasil_decision_forests/test_data/dataset\"\n", + "\n", + "# Download and load the dataset as Pandas DataFrames\n", + "train_ds = pd.read_csv(f\"{ds_path}/adult_train.csv\")\n", + "test_ds = pd.read_csv(f\"{ds_path}/adult_test.csv\")\n", + "label = \"income\"\n", + "\n", + "# Print the first 5 training examples\n", + "train_ds.head(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BTLbq0PxjdLT" + }, + "source": [ + "First, we train a YDF model on the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "id": "xmEVpzNpzRlF" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train model on 22792 examples\n", + "Model trained in 0:00:02.277830\n" + ] + } + ], + "source": [ + "learner = ydf.GradientBoostedTreesLearner(label=label)\n", + "model = learner.train(train_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_toMixu3jhhY" + }, + "source": [ + "We convert the YDF model into a JAX function." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "id": "AiUkvYK7zdXs" + }, + "outputs": [], + "source": [ + "jax_model = model.to_jax_function()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "g__NNgkZjvjc" + }, + "source": [ + "The `jax_model` object contains three fields.\n", + "\n", + "- `predict`: A JAX function making predictions.\n", + "- `encoder`: An optional callable class to prepare examples for `predict`. Since JAX does not support string values, categorical string input features have to be prepared before calling `predict`.\n", + "- `params`: A optional dictionary of Jax Arrays defining the differentiable parameters of the model. By default, `params` is None and `predict` does not except any parameters. We show how to use `params` in the second section.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NWXE9CEE2Js8" + }, + "source": [ + "We generate predictions for the first 5 examples in the test set.\n", + "\n", + "First, we select some examples and encode them." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "id": "OChSApgzA52B" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'age': Array([39, 40, 40, 35, 23], dtype=int32),\n", + " 'workclass': Array([4, 1, 1, 6, 3], dtype=int32),\n", + " 'fnlwgt': Array([ 77516, 121772, 193524, 76845, 190709], dtype=int32),\n", + " 'education': Array([ 3, 5, 13, 11, 7], dtype=int32),\n", + " 'education_num': Array([13, 11, 16, 5, 12], dtype=int32),\n", + " 'marital_status': Array([2, 1, 1, 1, 2], dtype=int32),\n", + " 'occupation': Array([ 4, 3, 1, 10, 12], dtype=int32),\n", + " 'relationship': Array([2, 1, 1, 1, 2], dtype=int32),\n", + " 'race': Array([1, 3, 1, 2, 1], dtype=int32),\n", + " 'sex': Array([1, 1, 1, 1, 1], dtype=int32),\n", + " 'capital_gain': Array([2174, 0, 0, 0, 0], dtype=int32),\n", + " 'capital_loss': Array([0, 0, 0, 0, 0], dtype=int32),\n", + " 'hours_per_week': Array([40, 40, 60, 40, 52], dtype=int32),\n", + " 'native_country': Array([1, 0, 1, 1, 1], dtype=int32)}" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Select the first 5 examples from the Pandas Dataframe and remove the labels.\n", + "selected_examples = test_ds[:5].drop(model.label(), axis=1)\n", + "\n", + "# Encode the examples into a dictionary of JAX arrays.\n", + "jax_selected_examples = jax_model.encoder(selected_examples)\n", + "jax_selected_examples" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NWmi2IRD2ml4" + }, + "source": [ + "Then, we generate the predictions." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "id": "_B1YSaLNCQNH" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Array([0.01860434, 0.36130956, 0.83858865, 0.04385566, 0.02917648], dtype=float32)" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "jax_predictions = jax_model.predict(jax_selected_examples)\n", + "jax_predictions" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mEFdGjyO2qGF" + }, + "source": [ + "Note that the predictions of the JAX function are equal to the predictions of the YDF model (modulo float rouding errors)." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "id": "hBHw49HYCfKR" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.01860435, 0.36130956, 0.83858865, 0.04385567, 0.02917649],\n", + " dtype=float32)" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.predict(selected_examples)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wtJAEFtu2yHx" + }, + "source": [ + "JAX does not define a model serialization format e.g. a way to save a model on disk. Instead, to save a JAX model for serving, it is common to export it as a SavedModel." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "id": "xt3lvsvPC0oi" + }, + "outputs": [], + "source": [ + "# Create a TF module with the model.\n", + "tf_model = tf.Module()\n", + "tf_model.predict = tf.function(\n", + " jax2tf.convert(jax_model.predict, with_gradient=False),\n", + " jit_compile=True,\n", + " autograph=False,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "id": "kwPIhMp5DgCO" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check the predictions of the TF module.\n", + "tf_selected_examples = {\n", + " k: tf.constant(v) for k, v in jax_selected_examples.items()\n", + "}\n", + "tf_predictions = tf_model.predict(tf_selected_examples)\n", + "tf_predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "id": "oOw_7ZD5EHbK" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /tmp/tmp90flesgr/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /tmp/tmp90flesgr/assets\n" + ] + } + ], + "source": [ + "# Save the TF module to file.\n", + "with tempfile.TemporaryDirectory() as tempdir:\n", + " tf.saved_model.save(tf_model, tempdir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1qZj2RfgNdfJ" + }, + "source": [ + "**INFO:** YDF's `to_tensorflow_saved_model` function allows to directly create a SavedModel model. This approach results in faster models, but it requires the installation of TensorFlow Decision Forests." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "id": "0_P6KZnoEYHD" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[INFO 24-06-14 14:31:56.6553 CEST kernel.cc:1233] Loading model from path /tmp/tmp71lnhoy9/tmp83xu8mjt/ with prefix e57777e0_\n", + "[INFO 24-06-14 14:31:56.6795 CEST quick_scorer_extended.cc:911] The binary was compiled without AVX2 support, but your CPU supports it. Enable it for faster model inference.\n", + "[INFO 24-06-14 14:31:56.6803 CEST abstract_model.cc:1362] Engine \"GradientBoostedTreesQuickScorerExtended\" built\n", + "[INFO 24-06-14 14:31:56.6803 CEST kernel.cc:1061] Use fast generic engine\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /tmp/tmpi0fp69xz/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /tmp/tmpi0fp69xz/assets\n" + ] + } + ], + "source": [ + "try:\n", + " with tempfile.TemporaryDirectory() as tempdir:\n", + " # Save the YDF model to a SavedModel directly.\n", + " model.to_tensorflow_saved_model(tempdir, mode=\"tf\")\n", + "except Exception as e:\n", + " print(\"Could not save YDF model to SavedModel with to_tensorflow_saved_model\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QU1mZh5EEjn4" + }, + "source": [ + "## Fine tune a YDF model with JAX\n", + "\n", + "A distribution shift problem occurs when the examples of interest (serving examples) follow a different distribution than the training dataset. As an example, distribution shift occurs in hospitals when training a model on data acquired by different devices. Although datasets from different devices should be compatible, subtle differences between them cause a model trained on one dataset to perform poorly on another. For instance, a machine learning model trained to detect tumors on images captured by a device might not work effectively on images captured by a device from another brand. Distribution shifts are also common in dynamic systems that change overtime (e.g., user behaviors).\n", + "\n", + "In this section, we solve a distribution shift issue using finetuning. For that, we use the Adult dataset with a twist. We assume that only people with \"relationship=Wife,\" are of interest. However, only 5% of the people are in this category so we have few training examples.\n", + "\n", + "We will first observe that training only on `relationship=Wife` examples or training on all available examples does not produce the best model. Instead, we will train a YDF model on all examples, finetuned it with JAX on the `relationship=Wife` examples, and observe that this finetune model perform better. Finally, the finetuned JAX model will be converted back into a YDF model and analyzed using YDF tools.\n", + "\n", + "**INFO:** This section assumes you are familiar with [JAX](https://jax.readthedocs.io/) and [Orbax](https://orbax.readthedocs.io/)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SfBKEcCH-6Xk" + }, + "source": [ + "First, let's print the distribution of `relationship` in the test examples. Our objective is optimize the quality of the model on the 483 `relationship == Wife` examples." + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "id": "FyNuf5G2EmWl" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "relationship\n", + "Husband 4002\n", + "Not-in-family 2505\n", + "Own-child 1521\n", + "Unmarried 948\n", + "Wife 483\n", + "Other-relative 310\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_ds[\"relationship\"].value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ba8nUY5Y_KAv" + }, + "source": [ + "We divide the dataset in two groups: Group A contains the `relationship != Wife` examples and group B contains the `relationship == Wife` examples." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "id": "D0wQv8wOFR7f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of examples per group\n", + "\tTrain Group A: 21707\n", + "\tTest Group A: 9286\n", + "\tTrain Group B: 1085\n", + "\tTest Group B: 483\n" + ] + } + ], + "source": [ + "def is_group_B(ds):\n", + " return ds[\"relationship\"] == \"Wife\"\n", + "\n", + "\n", + "train_ds_group_A = train_ds[~is_group_B(train_ds)]\n", + "test_ds_group_A = test_ds[~is_group_B(test_ds)]\n", + "\n", + "train_ds_group_B = train_ds[is_group_B(train_ds)]\n", + "test_ds_group_B = test_ds[is_group_B(test_ds)]\n", + "\n", + "print(\"Number of examples per group\")\n", + "print(\"\\tTrain Group A:\", len(train_ds_group_A))\n", + "print(\"\\tTest Group A:\", len(test_ds_group_A))\n", + "print(\"\\tTrain Group B:\", len(train_ds_group_B))\n", + "print(\"\\tTest Group B:\", len(test_ds_group_B))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XWDLNTKT_gkM" + }, + "source": [ + "Note that group A contains more examples than group B, but what we care are the test examples in group B.\n", + "\n", + "Let's train and evaluate three models on different combinations of group A and B. Those will be our baselines." + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "id": "ZlHNFJApGNq0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy on B, model trained on A: 0.7204968944099379\n", + "Accuracy on B, model trained on B: 0.7329192546583851\n", + "Accuracy on B, model trained on A+B: 0.7556935817805382\n" + ] + } + ], + "source": [ + "# Train model on group A\n", + "model_group_A = ydf.GradientBoostedTreesLearner(label=label).train(\n", + " train_ds_group_A, verbose=0\n", + ")\n", + "# Train model on group B\n", + "model_group_B = ydf.GradientBoostedTreesLearner(label=label).train(\n", + " train_ds_group_B, verbose=0\n", + ")\n", + "\n", + "# Train model on group A + B\n", + "model_group_AB = ydf.GradientBoostedTreesLearner(label=label).train(\n", + " train_ds, verbose=0\n", + ")\n", + "\n", + "# Evaluate the models on group B\n", + "accuracy_test_B_model_A = model_group_A.evaluate(test_ds_group_B).accuracy\n", + "accuracy_test_B_model_B = model_group_B.evaluate(test_ds_group_B).accuracy\n", + "accuracy_test_B_model_AB = model_group_AB.evaluate(test_ds_group_B).accuracy\n", + "\n", + "print(\"Accuracy on B, model trained on A:\", accuracy_test_B_model_A)\n", + "print(\"Accuracy on B, model trained on B:\", accuracy_test_B_model_B)\n", + "print(\"Accuracy on B, model trained on A+B:\", accuracy_test_B_model_AB)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JqJGrlvl_-Rk" + }, + "source": [ + "The model trained on both group A and B is the one performing best on group B. Can we do better?\n", + "\n", + "Let's convert the model trained on A+B into a JAX function." + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "id": "oJZjdugAH3G9" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'leaf_values': Array([-0.1233467 , -0.0927111 , 0.2927755 , ..., 0.05464426,\n", + " 0.12556875, -0.11374608], dtype=float32),\n", + " 'initial_predictions': Array([-1.1630996], dtype=float32)}" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "jax_model_group_AB = model_group_AB.to_jax_function(\n", + " apply_activation=False,\n", + " leaves_as_params=True,\n", + ")\n", + "\n", + "jax_model_group_AB.params" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PCJqBODMA5rK" + }, + "source": [ + "Note that:\n", + "\n", + "- `apply_activation=True` removes the activation function from the model. This allows for the model loss to be computed on logits rather than probabilities which make finetuning more stable numerically.\n", + "- `leaves_as_params=True` specifies that the leave values are exported as model parameters in`params`. This is necessary to finetune the model." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "03Aeofv3BKrZ" + }, + "source": [ + "To finetune the model, we need to generate batches of examples. The following block generate such batches." + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "id": "5zam6V5dL0lU" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'age': Array([44, 67, 26, 30], dtype=int32), 'workclass': Array([1, 5, 0, 1], dtype=int32), 'fnlwgt': Array([228057, 171564, 167835, 118551], dtype=int32), 'education': Array([9, 1, 3, 3], dtype=int32), 'education_num': Array([ 4, 9, 13, 13], dtype=int32), 'marital_status': Array([1, 1, 1, 1], dtype=int32), 'occupation': Array([ 7, 1, 0, 11], dtype=int32), 'relationship': Array([5, 5, 5, 5], dtype=int32), 'race': Array([1, 1, 1, 1], dtype=int32), 'sex': Array([2, 2, 2, 2], dtype=int32), 'capital_gain': Array([ 0, 20051, 0, 0], dtype=int32), 'capital_loss': Array([0, 0, 0, 0], dtype=int32), 'hours_per_week': Array([40, 30, 20, 16], dtype=int32), 'native_country': Array([12, 10, 1, 1], dtype=int32), 'income': Array([False, True, False, True], dtype=bool)}\n" + ] + } + ], + "source": [ + "def get_num_examples(ds):\n", + " return len(next(iter(ds.values())))\n", + "\n", + "\n", + "def prepare_dataset(ds, jax_model, batch=100):\n", + " ds = ds.copy()\n", + "\n", + " # Make the label boolean\n", + " ds[label] = ds[label] == \">50K\"\n", + "\n", + " # Encode the input features\n", + " encoded_ds = jax_model.encoder(ds)\n", + "\n", + " # Yield batches of examples\n", + " n = get_num_examples(encoded_ds)\n", + " i = 0\n", + " while i < n:\n", + " begin_idx = i\n", + " end_idx = min(i + batch, n)\n", + " yield {k: v[begin_idx:end_idx] for k, v in encoded_ds.items()}\n", + " i += batch\n", + "\n", + "\n", + "# Example of utilisation of \"prepare_dataset\".\n", + "for examples in prepare_dataset(train_ds_group_B, jax_model_group_AB, batch=4):\n", + " print(examples)\n", + " break # We only print the first batch" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iGEt9d5bBxAf" + }, + "source": [ + "Let's define utilities to compute and print the loss and accuracy of the model." + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "id": "Ps697bccJU2K" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stage:initial test-accuracy:0.75569 test-loss:0.47798 train-accuracy:0.83963 train-loss:0.37099\n" + ] + } + ], + "source": [ + "@jax.jit\n", + "def compute_accuracy(params, examples, logit=True):\n", + " examples = examples.copy()\n", + " labels = examples.pop(model.label())\n", + " predictions = jax_model_group_AB.predict(examples, params)\n", + " return ((predictions >= 0.0) == labels).mean()\n", + "\n", + "\n", + "@jax.jit\n", + "def compute_loss(params, examples):\n", + " examples = examples.copy()\n", + " labels = examples.pop(model.label())\n", + " logits = jax_model_group_AB.predict(examples, params)\n", + " return optax.sigmoid_binary_cross_entropy(logits, labels).mean()\n", + "\n", + "\n", + "def compute_metric(metric_fn, ds):\n", + " sum_metrics = 0\n", + " num_examples = 0\n", + " for examples in prepare_dataset(ds, jax_model_group_AB):\n", + " n = get_num_examples(examples)\n", + " sum_metrics += n * metric_fn(jax_model_group_AB.params, examples)\n", + " num_examples += n\n", + " return float(sum_metrics / num_examples)\n", + "\n", + "\n", + "def print_logs(stage):\n", + " train_accuracy = compute_metric(compute_accuracy, train_ds_group_B)\n", + " train_loss = compute_metric(compute_loss, train_ds_group_B)\n", + " test_accuracy = compute_metric(compute_accuracy, test_ds_group_B)\n", + " test_loss = compute_metric(compute_loss, test_ds_group_B)\n", + " print(\n", + " f\"stage:{stage:10} \"\n", + " f\"test-accuracy:{test_accuracy:.5f} test-loss:{test_loss:.5f} \"\n", + " f\"train-accuracy:{train_accuracy:.5f} train-loss:{train_loss:.5f}\"\n", + " )\n", + "\n", + "\n", + "# Metrics of the model before training.\n", + "print_logs(\"initial\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tba0RsT_CC0o" + }, + "source": [ + "Following is the train training loop." + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "id": "_trp941IZQu2" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stage:epoch_0 test-accuracy:0.75569 test-loss:0.47798 train-accuracy:0.83963 train-loss:0.37099\n", + "stage:epoch_1 test-accuracy:0.75155 test-loss:0.48035 train-accuracy:0.84424 train-loss:0.36520\n", + "stage:epoch_2 test-accuracy:0.75776 test-loss:0.47823 train-accuracy:0.84240 train-loss:0.35878\n", + "stage:epoch_3 test-accuracy:0.75983 test-loss:0.48016 train-accuracy:0.84608 train-loss:0.35352\n", + "stage:epoch_4 test-accuracy:0.75776 test-loss:0.48063 train-accuracy:0.84793 train-loss:0.34862\n", + "stage:epoch_5 test-accuracy:0.75569 test-loss:0.48173 train-accuracy:0.85069 train-loss:0.34419\n", + "stage:epoch_6 test-accuracy:0.75776 test-loss:0.48283 train-accuracy:0.85346 train-loss:0.34008\n", + "stage:epoch_7 test-accuracy:0.75776 test-loss:0.48381 train-accuracy:0.85806 train-loss:0.33622\n", + "stage:epoch_8 test-accuracy:0.75983 test-loss:0.48495 train-accuracy:0.86175 train-loss:0.33260\n", + "stage:epoch_9 test-accuracy:0.75983 test-loss:0.48595 train-accuracy:0.86267 train-loss:0.32917\n", + "stage:final test-accuracy:0.75983 test-loss:0.48703 train-accuracy:0.86359 train-loss:0.32592\n" + ] + } + ], + "source": [ + "optimizer = optax.adam(0.001)\n", + "\n", + "\n", + "@jax.jit\n", + "def train_step(opt_state, mdl_state, examples):\n", + " loss, grads = jax.value_and_grad(compute_loss)(mdl_state, examples)\n", + " updates, opt_state = optimizer.update(grads, opt_state)\n", + " mdl_state = optax.apply_updates(mdl_state, updates)\n", + " return opt_state, mdl_state, loss\n", + "\n", + "\n", + "opt_state = optimizer.init(jax_model_group_AB.params)\n", + "for epoch_idx in range(10):\n", + " print_logs(f\"epoch_{epoch_idx}\")\n", + " for examples in prepare_dataset(train_ds_group_B, jax_model_group_AB):\n", + " opt_state, jax_model_group_AB.params, _ = train_step(\n", + " opt_state, jax_model_group_AB.params, examples\n", + " )\n", + "\n", + "print_logs(\"final\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "huoeeQAzDj4n" + }, + "source": [ + "Notice both the test and training accuracy improving during training.\n", + "\n", + "We can now update the YDF model with the finetuned weights." + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "id": "G3LZFKM6Dvgl" + }, + "outputs": [], + "source": [ + "model_group_AB.update_with_jax_params(jax_model_group_AB.params)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qiZVYQDuD3q7" + }, + "source": [ + "`model_group_AB` is the finetuned model. Let's evaluate and compare it to the other models:" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "id": "GTEZwCGFbfjQ" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy on B, model trained on A: 0.7204968944099379\n", + "Accuracy on B, model trained on B: 0.7329192546583851\n", + "Accuracy on B, model trained on A+B: 0.7556935817805382\n", + "==================================\n", + "Accuracy on B, model trained on A+B, finetuned on B: 0.7598343685300207\n" + ] + } + ], + "source": [ + "accuracy_test_B_model_AB_finetuned_B = model_group_AB.evaluate(\n", + " test_ds_group_B\n", + ").accuracy\n", + "\n", + "print(\"Accuracy on B, model trained on A:\", accuracy_test_B_model_A)\n", + "print(\"Accuracy on B, model trained on B:\", accuracy_test_B_model_B)\n", + "print(\"Accuracy on B, model trained on A+B:\", accuracy_test_B_model_AB)\n", + "print(\"==================================\")\n", + "print(\n", + " \"Accuracy on B, model trained on A+B, finetuned on B:\",\n", + " accuracy_test_B_model_AB_finetuned_B,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-YwFSyW7EE0N" + }, + "source": [ + "Notice that the new model \"Accuracy on B, model trained on A+B\" shows the best test accuracy.\n", + "\n", + "`model_group_AB` is a YDF model like anyother. For instance, you can save it and analyse it.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "id": "-42P_PvwOjHZ" + }, + "outputs": [], + "source": [ + "# Save the model\n", + "with tempfile.TemporaryDirectory() as tempdir:\n", + " model_group_AB.save(tempdir)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "id": "GSFq5OP1ELAw" + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
Partial Dependence PlotConditional Expectation PlotVariable Importances
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "

Variable importances measure the importance of an input feature for a model.

    1.   "capital_gain"  0.049689 ################\n",
+       "    2.     "occupation"  0.045549 ##############\n",
+       "    3.      "education"  0.026915 ########\n",
+       "    4.  "education_num"  0.026915 ########\n",
+       "    5.            "age"  0.018634 ######\n",
+       "    6.   "capital_loss"  0.018634 ######\n",
+       "    7.      "workclass"  0.014493 #####\n",
+       "    8.         "fnlwgt"  0.002070 #\n",
+       "    9. "native_country"  0.002070 #\n",
+       "   10.   "relationship"  0.000000 \n",
+       "   11.           "race"  0.000000 \n",
+       "   12.            "sex"  0.000000 \n",
+       "   13. "hours_per_week"  0.000000 \n",
+       "   14. "marital_status" -0.002070 \n",
+       "
    1.   "capital_gain"  0.164288 ################\n",
+       "    2.   "capital_loss"  0.048263 #####\n",
+       "    3.     "occupation"  0.033196 ###\n",
+       "    4.      "education"  0.023903 ##\n",
+       "    5.  "education_num"  0.015137 ##\n",
+       "    6.            "age"  0.013872 #\n",
+       "    7.      "workclass"  0.006274 #\n",
+       "    8.           "race"  0.002477 \n",
+       "    9.            "sex"  0.001453 \n",
+       "   10.         "fnlwgt"  0.000984 \n",
+       "   11. "marital_status"  0.000722 \n",
+       "   12.   "relationship"  0.000000 \n",
+       "   13. "native_country" -0.000019 \n",
+       "   14. "hours_per_week" -0.007143 \n",
+       "
    1.   "capital_gain"  0.083385 ################\n",
+       "    2.     "occupation"  0.040765 ########\n",
+       "    3.   "capital_loss"  0.030647 ######\n",
+       "    4.      "education"  0.026051 #####\n",
+       "    5.            "age"  0.024419 #####\n",
+       "    6.  "education_num"  0.016887 ####\n",
+       "    7.      "workclass"  0.010427 ##\n",
+       "    8.           "race"  0.003161 #\n",
+       "    9. "marital_status"  0.000790 #\n",
+       "   10.            "sex"  0.000704 #\n",
+       "   11.   "relationship"  0.000000 #\n",
+       "   12. "native_country" -0.000361 #\n",
+       "   13.         "fnlwgt" -0.001022 \n",
+       "   14. "hours_per_week" -0.006107 \n",
+       "
    1.   "capital_gain"  0.162868 ################\n",
+       "    2.   "capital_loss"  0.048043 #####\n",
+       "    3.     "occupation"  0.033135 ###\n",
+       "    4.      "education"  0.023881 ##\n",
+       "    5.  "education_num"  0.015116 ##\n",
+       "    6.            "age"  0.013875 #\n",
+       "    7.      "workclass"  0.006275 #\n",
+       "    8.           "race"  0.002472 \n",
+       "    9.            "sex"  0.001448 \n",
+       "   10.         "fnlwgt"  0.000990 \n",
+       "   11. "marital_status"  0.000721 \n",
+       "   12.   "relationship"  0.000000 \n",
+       "   13. "native_country" -0.000014 \n",
+       "   14. "hours_per_week" -0.007106 \n",
+       "
    1.            "age"  0.226642 ################\n",
+       "    2.     "occupation"  0.219727 #############\n",
+       "    3.   "capital_gain"  0.214876 ############\n",
+       "    4.      "education"  0.213746 ###########\n",
+       "    5. "marital_status"  0.212739 ###########\n",
+       "    6.   "relationship"  0.206040 #########\n",
+       "    7.         "fnlwgt"  0.203843 ########\n",
+       "    8. "hours_per_week"  0.203735 ########\n",
+       "    9.   "capital_loss"  0.196549 ######\n",
+       "   10. "native_country"  0.190548 ####\n",
+       "   11.      "workclass"  0.187795 ###\n",
+       "   12.  "education_num"  0.184215 ##\n",
+       "   13.           "race"  0.180495 \n",
+       "   14.            "sex"  0.177647 \n",
+       "
    1.            "age" 26.000000 ################\n",
+       "    2.   "capital_gain" 26.000000 ################\n",
+       "    3. "marital_status" 20.000000 ############\n",
+       "    4.   "relationship" 17.000000 ##########\n",
+       "    5.   "capital_loss" 14.000000 ########\n",
+       "    6. "hours_per_week" 14.000000 ########\n",
+       "    7.      "education" 12.000000 #######\n",
+       "    8.         "fnlwgt" 10.000000 #####\n",
+       "    9.           "race"  9.000000 #####\n",
+       "   10.  "education_num"  7.000000 ###\n",
+       "   11.            "sex"  4.000000 #\n",
+       "   12.     "occupation"  2.000000 \n",
+       "   13.      "workclass"  1.000000 \n",
+       "   14. "native_country"  1.000000 \n",
+       "
    1.     "occupation" 724.000000 ################\n",
+       "    2.         "fnlwgt" 513.000000 ###########\n",
+       "    3.            "age" 483.000000 ##########\n",
+       "    4.      "education" 464.000000 ##########\n",
+       "    5. "hours_per_week" 339.000000 #######\n",
+       "    6.   "capital_gain" 326.000000 ######\n",
+       "    7. "native_country" 306.000000 ######\n",
+       "    8.   "capital_loss" 297.000000 ######\n",
+       "    9.   "relationship" 262.000000 #####\n",
+       "   10.      "workclass" 244.000000 #####\n",
+       "   11. "marital_status" 210.000000 ####\n",
+       "   12.  "education_num" 82.000000 #\n",
+       "   13.            "sex" 42.000000 \n",
+       "   14.           "race" 21.000000 \n",
+       "
    1.   "relationship" 3014.690076 ################\n",
+       "    2.   "capital_gain" 2065.521668 ##########\n",
+       "    3.      "education" 1144.490954 ######\n",
+       "    4. "marital_status" 1111.389695 #####\n",
+       "    5.     "occupation" 1094.619502 #####\n",
+       "    6.  "education_num" 796.666823 ####\n",
+       "    7.   "capital_loss" 584.055066 ###\n",
+       "    8.            "age" 582.288569 ###\n",
+       "    9. "hours_per_week" 366.856509 #\n",
+       "   10. "native_country" 263.872689 #\n",
+       "   11.         "fnlwgt" 216.537764 #\n",
+       "   12.      "workclass" 196.085503 #\n",
+       "   13.            "sex" 47.217730 \n",
+       "   14.           "race"  5.428727 \n",
+       "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Analyse the model\n", + "model_group_AB.analyze(test_ds_group_B)" + ] + } + ], + "metadata": { + "colab": { + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "private_outputs": true, + "provenance": [ + { + "file_id": "1LjF5dfXxeLAzb2epxADkd34MHoUN4aHt", + "timestamp": 1716900422680 + } + ] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/documentation/public/mkdocs.yml b/documentation/public/mkdocs.yml index 0b246796..18128006 100644 --- a/documentation/public/mkdocs.yml +++ b/documentation/public/mkdocs.yml @@ -66,6 +66,7 @@ nav: # TODO: boolean, and text - Deep learning: - with TensorFlow: tutorial/compose_with_tf.ipynb + - with Jax: tutorial/compose_with_jax.ipynb - Dataset: - Pandas Dataframe: tutorial/pandas.ipynb - TensorFlow Dataset: tutorial/tf_dataset.ipynb From 933e38def952688ce4ecc6f25966cc33ae0b30fc Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 17 Jun 2024 06:20:43 -0700 Subject: [PATCH 21/30] Anomaly detection; Isolation forest c++ model (part 3) PiperOrigin-RevId: 643984185 --- yggdrasil_decision_forests/model/BUILD | 1 + .../model/abstract_model.h | 5 +- yggdrasil_decision_forests/model/describe.cc | 4 +- .../model/isolation_forest/BUILD | 77 +++++ .../isolation_forest/isolation_forest.cc | 319 ++++++++++++++++++ .../model/isolation_forest/isolation_forest.h | 163 +++++++++ .../isolation_forest/isolation_forest.proto | 54 +++ .../isolation_forest/isolation_forest_test.cc | 301 +++++++++++++++++ .../test_data/dataset/README.md | 26 +- .../test_data/dataset/gaussians_test.csv | 281 +++++++++++++++ .../test_data/dataset/gaussians_train.csv | 281 +++++++++++++++ .../model/gaussians_anomaly_if/data_spec.pb | Bin 0 -> 134 bytes .../test_data/model/gaussians_anomaly_if/done | 0 .../model/gaussians_anomaly_if/header.pb | Bin 0 -> 67 bytes .../isolation_forest_header.pb | Bin 0 -> 23 bytes .../gaussians_anomaly_if/nodes-00000-of-00001 | Bin 0 -> 103016 bytes .../prediction/gaussians_anomaly_if_skl.csv | 280 +++++++++++++++ 17 files changed, 1788 insertions(+), 4 deletions(-) create mode 100644 yggdrasil_decision_forests/model/isolation_forest/BUILD create mode 100644 yggdrasil_decision_forests/model/isolation_forest/isolation_forest.cc create mode 100644 yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h create mode 100644 yggdrasil_decision_forests/model/isolation_forest/isolation_forest.proto create mode 100644 yggdrasil_decision_forests/model/isolation_forest/isolation_forest_test.cc create mode 100644 yggdrasil_decision_forests/test_data/dataset/gaussians_test.csv create mode 100644 yggdrasil_decision_forests/test_data/dataset/gaussians_train.csv create mode 100644 yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/data_spec.pb create mode 100644 yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/done create mode 100644 yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/header.pb create mode 100644 yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/isolation_forest_header.pb create mode 100644 yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/nodes-00000-of-00001 create mode 100644 yggdrasil_decision_forests/test_data/prediction/gaussians_anomaly_if_skl.csv diff --git a/yggdrasil_decision_forests/model/BUILD b/yggdrasil_decision_forests/model/BUILD index a0e70fe9..f48020fb 100644 --- a/yggdrasil_decision_forests/model/BUILD +++ b/yggdrasil_decision_forests/model/BUILD @@ -13,6 +13,7 @@ cc_library_ydf( name = "all_models", deps = [ "//yggdrasil_decision_forests/model/gradient_boosted_trees", + "//yggdrasil_decision_forests/model/isolation_forest", "//yggdrasil_decision_forests/model/random_forest", "//yggdrasil_decision_forests/serving/decision_forest:register_engines", ], diff --git a/yggdrasil_decision_forests/model/abstract_model.h b/yggdrasil_decision_forests/model/abstract_model.h index 1fbaa731..6f88dc42 100644 --- a/yggdrasil_decision_forests/model/abstract_model.h +++ b/yggdrasil_decision_forests/model/abstract_model.h @@ -150,7 +150,10 @@ class AbstractModel { // Get the model target column. int label_col_idx() const { return label_col_idx_; } - // Name of the label column. + // Tests if the model has a label. + bool has_label() const { return label_col_idx_ != -1; } + + // Name of the label column. Should only be called if "has_label()" is true. std::string label() const { DCHECK_GE(label_col_idx_, 0); DCHECK_LT(label_col_idx_, data_spec_.columns_size()); diff --git a/yggdrasil_decision_forests/model/describe.cc b/yggdrasil_decision_forests/model/describe.cc index c6a192d3..9a47caa7 100644 --- a/yggdrasil_decision_forests/model/describe.cc +++ b/yggdrasil_decision_forests/model/describe.cc @@ -68,7 +68,9 @@ utils::html::Html Model(const model::AbstractModel& model) { h::Html content; AddKeyValue(&content, "Name", model.name()); AddKeyValue(&content, "Task", proto::Task_Name(model.task())); - AddKeyValue(&content, "Label", model.label()); + if (model.has_label()) { + AddKeyValue(&content, "Label", model.label()); + } if (model.ranking_group_col_idx() != -1) { AddKeyValue( diff --git a/yggdrasil_decision_forests/model/isolation_forest/BUILD b/yggdrasil_decision_forests/model/isolation_forest/BUILD new file mode 100644 index 00000000..686f7545 --- /dev/null +++ b/yggdrasil_decision_forests/model/isolation_forest/BUILD @@ -0,0 +1,77 @@ +load("//yggdrasil_decision_forests/utils:compile.bzl", "all_proto_library", "cc_library_ydf") + +package( + default_visibility = ["//visibility:public"], + licenses = ["notice"], +) + +# Library +# ======= + +cc_library_ydf( + name = "isolation_forest", + srcs = ["isolation_forest.cc"], + hdrs = ["isolation_forest.h"], + deps = [ + ":isolation_forest_cc_proto", + "//yggdrasil_decision_forests/dataset:example_cc_proto", + "//yggdrasil_decision_forests/dataset:types", + "//yggdrasil_decision_forests/dataset:vertical_dataset", + "//yggdrasil_decision_forests/metric:metric_cc_proto", + "//yggdrasil_decision_forests/model:abstract_model", + "//yggdrasil_decision_forests/model:abstract_model_cc_proto", + "//yggdrasil_decision_forests/model:prediction_cc_proto", + "//yggdrasil_decision_forests/model/decision_tree", + "//yggdrasil_decision_forests/model/decision_tree:decision_forest_interface", + "//yggdrasil_decision_forests/model/decision_tree:decision_tree_cc_proto", + "//yggdrasil_decision_forests/utils:filesystem", + "//yggdrasil_decision_forests/utils:logging", + "//yggdrasil_decision_forests/utils:status_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + ], + alwayslink = 1, +) + +# Proto +# ======== + +all_proto_library( + name = "isolation_forest_proto", + srcs = ["isolation_forest.proto"], + deps = [ + "//yggdrasil_decision_forests/metric:metric_proto", + "//yggdrasil_decision_forests/model:abstract_model_proto", + ], +) + +# Test +# ======== + +cc_test( + name = "isolation_forest_test", + srcs = ["isolation_forest_test.cc"], + data = ["//yggdrasil_decision_forests/test_data"], + deps = [ + ":isolation_forest", + "//yggdrasil_decision_forests/dataset:csv_example_reader", + "//yggdrasil_decision_forests/dataset:data_spec", + "//yggdrasil_decision_forests/dataset:data_spec_cc_proto", + "//yggdrasil_decision_forests/dataset:vertical_dataset", + "//yggdrasil_decision_forests/dataset:vertical_dataset_io", + "//yggdrasil_decision_forests/model:model_library", + "//yggdrasil_decision_forests/model:prediction_cc_proto", + "//yggdrasil_decision_forests/model/decision_tree", + "//yggdrasil_decision_forests/model/decision_tree:builder", + "//yggdrasil_decision_forests/utils:filesystem", + "//yggdrasil_decision_forests/utils:logging", + "//yggdrasil_decision_forests/utils:protobuf", + "//yggdrasil_decision_forests/utils:test", + "//yggdrasil_decision_forests/utils:testing_macros", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/yggdrasil_decision_forests/model/isolation_forest/isolation_forest.cc b/yggdrasil_decision_forests/model/isolation_forest/isolation_forest.cc new file mode 100644 index 00000000..d8415d78 --- /dev/null +++ b/yggdrasil_decision_forests/model/isolation_forest/isolation_forest.cc @@ -0,0 +1,319 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "yggdrasil_decision_forests/dataset/example.pb.h" +#include "yggdrasil_decision_forests/dataset/types.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset.h" +#include "yggdrasil_decision_forests/metric/metric.pb.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" +#include "yggdrasil_decision_forests/model/abstract_model.pb.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree.pb.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree_io.h" +#include "yggdrasil_decision_forests/model/decision_tree/structure_analysis.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.pb.h" +#include "yggdrasil_decision_forests/model/prediction.pb.h" +#include "yggdrasil_decision_forests/utils/filesystem.h" +#include "yggdrasil_decision_forests/utils/logging.h" +#include "yggdrasil_decision_forests/utils/status_macros.h" + +namespace yggdrasil_decision_forests::model::isolation_forest { + +namespace { +// Basename for the shards containing the nodes. +constexpr char kNodeBaseFilename[] = "nodes"; +// Filename containing the isolation forest header. +constexpr char kHeaderBaseFilename[] = "isolation_forest_header.pb"; + +} // namespace + +float PreissAveragePathLength(UnsignedExampleIdx num_examples) { + DCHECK_GT(num_examples, 0); + const float num_examples_float = static_cast(num_examples); + + // Harmonic number + // This is the approximation proposed in "Isolation Forest" by Liu et al. + const auto H = [](const float x) { + constexpr float euler_constant = 0.5772156649f; + return std::log(x) + euler_constant; + }; + + if (num_examples > 2) { + return 2.f * H(num_examples_float - 1.f) - + 2.f * (num_examples_float - 1.f) / num_examples_float; + } else if (num_examples == 2) { + return 1.f; + } else { + return 0.f; // To be safe. + } +} + +float IsolationForestPredictionFromDenominator(const float average_h, + const float denominator) { + if (denominator == 0.f) { + return 0.f; + } + const float term = -average_h / denominator; + return std::pow(2.f, term); +} + +float IsolationForestPrediction(const float average_h, + const UnsignedExampleIdx num_examples) { + return IsolationForestPredictionFromDenominator( + average_h, PreissAveragePathLength(num_examples)); +} + +proto::Header IsolationForestModel::BuildHeaderProto() const { + proto::Header header; + header.set_num_trees(decision_trees_.size()); + header.mutable_isolation_forest(); + header.set_num_examples_per_trees(num_examples_per_trees_); + return header; +} + +void IsolationForestModel::ApplyHeaderProto(const proto::Header& header) { + num_examples_per_trees_ = header.num_examples_per_trees(); +} + +absl::Status IsolationForestModel::Save( + absl::string_view directory, const ModelIOOptions& io_options) const { + RETURN_IF_ERROR(file::RecursivelyCreateDir(directory, file::Defaults())); + RETURN_IF_ERROR(ValidateModelIOOptions(io_options)); + + // Format used to store the nodes. + std::string format; + if (node_format_.has_value()) { + format = node_format_.value(); + } else { + ASSIGN_OR_RETURN(format, decision_tree::RecommendedSerializationFormat()); + } + + int num_shards; + const auto node_base_filename = + absl::StrCat(io_options.file_prefix.value(), kNodeBaseFilename); + RETURN_IF_ERROR(decision_tree::SaveTreesToDisk( + directory, node_base_filename, decision_trees_, format, &num_shards)); + + auto header = BuildHeaderProto(); + header.set_node_format(format); + header.set_num_node_shards(num_shards); + + const auto header_filename = + absl::StrCat(io_options.file_prefix.value(), kHeaderBaseFilename); + RETURN_IF_ERROR(file::SetBinaryProto( + file::JoinPath(directory, header_filename), header, file::Defaults())); + return absl::OkStatus(); +} + +absl::Status IsolationForestModel::Load(absl::string_view directory, + const ModelIOOptions& io_options) { + RETURN_IF_ERROR(ValidateModelIOOptions(io_options)); + + proto::Header header; + decision_trees_.clear(); + const auto header_filename = + absl::StrCat(io_options.file_prefix.value(), kHeaderBaseFilename); + RETURN_IF_ERROR(file::GetBinaryProto( + file::JoinPath(directory, header_filename), &header, file::Defaults())); + const auto node_base_filename = + absl::StrCat(io_options.file_prefix.value(), kNodeBaseFilename); + RETURN_IF_ERROR(decision_tree::LoadTreesFromDisk( + directory, node_base_filename, header.num_node_shards(), + header.num_trees(), header.node_format(), &decision_trees_)); + node_format_ = header.node_format(); + ApplyHeaderProto(header); + return absl::OkStatus(); +} + +absl::Status IsolationForestModel::SerializeModelImpl( + model::proto::SerializedModel* dst_proto, std::string* dst_raw) const { + const auto& specialized_proto = dst_proto->MutableExtension( + isolation_forest::proto::isolation_forest_serialized_model); + *specialized_proto->mutable_header() = BuildHeaderProto(); + if (node_format_.has_value()) { + specialized_proto->mutable_header()->set_node_format(node_format_.value()); + } + ASSIGN_OR_RETURN(*dst_raw, decision_tree::SerializeTrees(decision_trees_)); + return absl::OkStatus(); +} + +absl::Status IsolationForestModel::DeserializeModelImpl( + const model::proto::SerializedModel& src_proto, absl::string_view src_raw) { + const auto& specialized_proto = src_proto.GetExtension( + isolation_forest::proto::isolation_forest_serialized_model); + ApplyHeaderProto(specialized_proto.header()); + if (specialized_proto.header().has_node_format()) { + node_format_ = specialized_proto.header().node_format(); + } + return decision_tree::DeserializeTrees( + src_raw, specialized_proto.header().num_trees(), &decision_trees_); +} + +absl::Status IsolationForestModel::Validate() const { + RETURN_IF_ERROR(AbstractModel::Validate()); + if (decision_trees_.empty()) { + return absl::InvalidArgumentError("Empty isolation forest"); + } + if (task_ != model::proto::Task::ANOMALY_DETECTION) { + return absl::InvalidArgumentError("Wrong task"); + } + return absl::OkStatus(); +} + +absl::optional IsolationForestModel::ModelSizeInBytes() const { + return AbstractAttributesSizeInBytes() + + decision_tree::EstimateSizeInByte(decision_trees_); +} + +void IsolationForestModel::PredictLambda( + std::function + get_leaf, + model::proto::Prediction* prediction) const { + float sum_h = 0.0; + for (const auto& tree : decision_trees_) { + const auto& leaf = get_leaf(*tree); + const auto num_examples = + leaf.node().anomaly_detection().num_examples_without_weight(); + sum_h += leaf.depth() + PreissAveragePathLength(num_examples); + } + + if (!decision_trees_.empty()) { + sum_h /= decision_trees_.size(); + } + DCHECK_GT(num_examples_per_trees_, 0); + const float p = IsolationForestPrediction( + /*average_h=*/sum_h, + /*num_examples=*/num_examples_per_trees_); + prediction->mutable_anomaly_detection()->set_value(p); +} + +void IsolationForestModel::Predict(const dataset::VerticalDataset& dataset, + dataset::VerticalDataset::row_t row_idx, + model::proto::Prediction* prediction) const { + PredictLambda( + [&](const decision_tree::DecisionTree& tree) + -> const decision_tree::NodeWithChildren& { + return tree.GetLeafAlt(dataset, row_idx); + }, + prediction); +} + +void IsolationForestModel::Predict(const dataset::proto::Example& example, + model::proto::Prediction* prediction) const { + PredictLambda( + [&](const decision_tree::DecisionTree& tree) + -> const decision_tree::NodeWithChildren& { + return tree.GetLeafAlt(example); + }, + prediction); +} + +absl::Status IsolationForestModel::PredictGetLeaves( + const dataset::VerticalDataset& dataset, + dataset::VerticalDataset::row_t row_idx, absl::Span leaves) const { + if (leaves.size() != num_trees()) { + return absl::InvalidArgumentError("Wrong number of trees"); + } + for (int tree_idx = 0; tree_idx < decision_trees_.size(); tree_idx++) { + auto& leaf = decision_trees_[tree_idx]->GetLeafAlt(dataset, row_idx); + if (leaf.leaf_idx() < 0) { + return absl::InvalidArgumentError("Leaf idx not set"); + } + leaves[tree_idx] = leaf.leaf_idx(); + } + return absl::OkStatus(); +} + +bool IsolationForestModel::CheckStructure( + const decision_tree::CheckStructureOptions& options) const { + return decision_tree::CheckStructure(options, data_spec(), decision_trees_); +} + +void IsolationForestModel::AppendDescriptionAndStatistics( + bool full_definition, std::string* description) const { + AbstractModel::AppendDescriptionAndStatistics(full_definition, description); + absl::StrAppend(description, "\n"); + StrAppendForestStructureStatistics(data_spec(), decision_trees(), + description); + absl::StrAppend(description, + "Node format: ", node_format_.value_or("NOT_SET"), "\n"); + + absl::StrAppend(description, + "Number of examples per tree: ", num_examples_per_trees_, + "\n"); + + if (full_definition) { + absl::StrAppend(description, "\nModel Structure:\n"); + decision_tree::AppendModelStructure(decision_trees_, data_spec(), + label_col_idx_, description); + } +} + +absl::Status IsolationForestModel::MakePureServing() { + for (auto& tree : decision_trees_) { + tree->IterateOnMutableNodes( + [](decision_tree::NodeWithChildren* node, const int depth) { + if (!node->IsLeaf()) { + // Remove the label information from the non-leaf nodes. + node->mutable_node()->clear_output(); + } + }); + } + return AbstractModel::MakePureServing(); +} + +absl::Status IsolationForestModel::Distance( + const dataset::VerticalDataset& dataset1, + const dataset::VerticalDataset& dataset2, + absl::Span distances) const { + return decision_tree::Distance(decision_trees(), dataset1, dataset2, + distances); +} + +std::string IsolationForestModel::DebugCompare( + const AbstractModel& other) const { + if (const auto parent_compare = AbstractModel::DebugCompare(other); + !parent_compare.empty()) { + return parent_compare; + } + const auto* other_cast = dynamic_cast(&other); + if (!other_cast) { + return "Non matching types"; + } + return decision_tree::DebugCompare( + data_spec_, label_col_idx_, decision_trees_, other_cast->decision_trees_); +} + +REGISTER_AbstractModel(IsolationForestModel, + IsolationForestModel::kRegisteredName); + +} // namespace yggdrasil_decision_forests::model::isolation_forest diff --git a/yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h b/yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h new file mode 100644 index 00000000..002a0b1d --- /dev/null +++ b/yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h @@ -0,0 +1,163 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef YGGDRASIL_DECISION_FORESTS_MODEL_ISOLATION_FOREST_ISOLATION_FOREST_H_ +#define YGGDRASIL_DECISION_FORESTS_MODEL_ISOLATION_FOREST_ISOLATION_FOREST_H_ + +#include + +#include +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "yggdrasil_decision_forests/dataset/example.pb.h" +#include "yggdrasil_decision_forests/dataset/types.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset.h" +#include "yggdrasil_decision_forests/metric/metric.pb.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" +#include "yggdrasil_decision_forests/model/abstract_model.pb.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_forest_interface.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.pb.h" +#include "yggdrasil_decision_forests/model/prediction.pb.h" + +namespace yggdrasil_decision_forests::model::isolation_forest { + +class IsolationForestModel : public AbstractModel, + public DecisionForestInterface { + public: + inline static constexpr char kRegisteredName[] = "ISOLATION_FOREST"; + + IsolationForestModel() : AbstractModel(kRegisteredName) {} + + void Predict(const dataset::VerticalDataset& dataset, + dataset::VerticalDataset::row_t row_idx, + model::proto::Prediction* prediction) const override; + + void Predict(const dataset::proto::Example& example, + model::proto::Prediction* prediction) const override; + + absl::Status PredictGetLeaves(const dataset::VerticalDataset& dataset, + dataset::VerticalDataset::row_t row_idx, + absl::Span leaves) const override; + + bool CheckStructure( + const decision_tree::CheckStructureOptions& options) const override; + + absl::optional ModelSizeInBytes() const override; + + void AppendDescriptionAndStatistics(bool full_definition, + std::string* description) const override; + + absl::Status MakePureServing() override; + + absl::Status Distance(const dataset::VerticalDataset& dataset1, + const dataset::VerticalDataset& dataset2, + absl::Span distances) const override; + + int num_trees() const override { return decision_trees_.size(); } + + // For the serving engines. + // TODO: Move in DecisionForestInterface. + size_t NumTrees() const { return num_trees(); } + int64_t NumNodes() const { + return decision_tree::NumberOfNodes(decision_trees_); + } + void CountFeatureUsage( + std::unordered_map* feature_usage) const { + for (const auto& tree : decision_trees_) { + tree->CountFeatureUsage(feature_usage); + } + } + + const std::vector>& + decision_trees() const override { + return decision_trees_; + } + + std::vector>* + mutable_decision_trees() override { + return &decision_trees_; + } + + void set_node_format(const absl::optional& format) override { + node_format_ = format; + } + + void set_num_examples_per_trees(int64_t value) { + num_examples_per_trees_ = value; + } + + int64_t num_examples_per_trees() const { return num_examples_per_trees_; } + + std::string DebugCompare(const AbstractModel& other) const override; + + absl::Status Save(absl::string_view directory, + const ModelIOOptions& io_options) const override; + + absl::Status Load(absl::string_view directory, + const ModelIOOptions& io_options) override; + + absl::Status Validate() const override; + + private: + void PredictLambda(std::function + get_leaf, + model::proto::Prediction* prediction) const; + + // The decision trees. + std::vector> decision_trees_; + + // Node storage format. + absl::optional node_format_; + + absl::Status SerializeModelImpl(model::proto::SerializedModel* dst_proto, + std::string* dst_raw) const override; + + absl::Status DeserializeModelImpl( + const model::proto::SerializedModel& src_proto, + absl::string_view src_raw) override; + + proto::Header BuildHeaderProto() const; + void ApplyHeaderProto(const proto::Header& header); + + // Number of examples used to grow each tree. + int64_t num_examples_per_trees_ = -1; +}; + +// Analytical expected number of examples in a binary tree trained with +// "num_examples" examples. Called "c" in "Isolation-Based Anomaly Detection" by +// Liu et al. +float PreissAveragePathLength(UnsignedExampleIdx num_examples); + +// Isolation forest prediction. +float IsolationForestPrediction(float average_h, + UnsignedExampleIdx num_examples); + +// Isolation forest prediction, from the pre-computed denominator. +float IsolationForestPredictionFromDenominator(float average_h, + float denominator); + +} // namespace yggdrasil_decision_forests::model::isolation_forest + +#endif // YGGDRASIL_DECISION_FORESTS_MODEL_ISOLATION_FOREST_ISOLATION_FOREST_H_ diff --git a/yggdrasil_decision_forests/model/isolation_forest/isolation_forest.proto b/yggdrasil_decision_forests/model/isolation_forest/isolation_forest.proto new file mode 100644 index 00000000..f3cb8d26 --- /dev/null +++ b/yggdrasil_decision_forests/model/isolation_forest/isolation_forest.proto @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package yggdrasil_decision_forests.model.isolation_forest.proto; + +import "yggdrasil_decision_forests/model/abstract_model.proto"; + +// An isolation forest as defined in "Isolation-Based Anomaly Detection" by Liu +// et al. (2012). In this case, the prediction value is the node depth. +message IsolationForestAnomalityScore {} + +// Header for the isolation forest model. +message Header { + // Next ID: 6 + + // Number of shards used to store the nodes. + optional int32 num_node_shards = 1; + + // Number of trees. + optional int64 num_trees = 2; + + // Container used to store the trees' nodes. + optional string node_format = 3 [default = "TFE_RECORDIO"]; + + // Number of examples used to grow each tree. + optional int64 num_examples_per_trees = 4; + + oneof anomality_score { + IsolationForestAnomalityScore isolation_forest = 5; + } +} + +message IsolationForestSerializedModel { + optional Header header = 1; +} + +extend model.proto.SerializedModel { + optional IsolationForestSerializedModel isolation_forest_serialized_model = + 1003; +} diff --git a/yggdrasil_decision_forests/model/isolation_forest/isolation_forest_test.cc b/yggdrasil_decision_forests/model/isolation_forest/isolation_forest_test.cc new file mode 100644 index 00000000..26839376 --- /dev/null +++ b/yggdrasil_decision_forests/model/isolation_forest/isolation_forest_test.cc @@ -0,0 +1,301 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" + +#include +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/str_cat.h" +#include "absl/types/span.h" +#include "yggdrasil_decision_forests/dataset/data_spec.h" +#include "yggdrasil_decision_forests/dataset/data_spec.pb.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset_io.h" +#include "yggdrasil_decision_forests/model/decision_tree/builder.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree.h" +#include "yggdrasil_decision_forests/model/model_library.h" +#include "yggdrasil_decision_forests/model/prediction.pb.h" +#include "yggdrasil_decision_forests/utils/filesystem.h" +#include "yggdrasil_decision_forests/utils/logging.h" +#include "yggdrasil_decision_forests/utils/protobuf.h" +#include "yggdrasil_decision_forests/utils/test.h" +#include "yggdrasil_decision_forests/utils/testing_macros.h" + +namespace yggdrasil_decision_forests::model::isolation_forest { +namespace { + +const float kEpsilon = 0.00001f; + +using ::testing::ElementsAre; +using ::yggdrasil_decision_forests::test::EqualsProto; + +std::string TestDataDir() { + return file::JoinPath(test::DataRootDirectory(), + "yggdrasil_decision_forests/test_data"); +} + +std::unique_ptr CreateToyModel() { + auto model = std::make_unique(); + model->set_task(model::proto::Task::ANOMALY_DETECTION); + model->set_num_examples_per_trees(10); + + dataset::AddNumericalColumn("a", model->mutable_data_spec()); + dataset::AddNumericalColumn("b", model->mutable_data_spec()); + dataset::AddNumericalColumn("c", model->mutable_data_spec()); + + model->mutable_input_features()->push_back(0); + model->mutable_input_features()->push_back(1); + model->mutable_input_features()->push_back(2); + + // Tree 1 + { + auto tree = std::make_unique(); + decision_tree::TreeBuilder root(tree.get()); + auto [nl1, l1] = root.ConditionIsGreater(0, 0); + auto [l2, nl2] = nl1.ConditionIsGreater(1, 1); + auto [l3, l4] = nl2.ConditionIsGreater(0, 1); + l1.LeafAnomalyDetection(2); + l2.LeafAnomalyDetection(4); + l3.LeafAnomalyDetection(2); + l4.LeafAnomalyDetection(2); + tree->SetLeafIndices(); + model->mutable_decision_trees()->push_back(std::move(tree)); + } + + // Tree 2 + { + auto tree = std::make_unique(); + decision_tree::TreeBuilder root(tree.get()); + auto [l1, l2] = root.ConditionIsGreater(1, 1); + l1.LeafAnomalyDetection(5); + l2.LeafAnomalyDetection(5); + tree->SetLeafIndices(); + model->mutable_decision_trees()->push_back(std::move(tree)); + } + + return model; +} + +dataset::VerticalDataset CreateToyDataset( + const dataset::proto::DataSpecification& dataspec) { + dataset::VerticalDataset dataset; + dataset.set_data_spec(dataspec); + EXPECT_OK(dataset.CreateColumnsFromDataspec()); + dataset.AppendExample({{"a", "-1"}, {"b", "0"}, {"c", "0"}}); + dataset.AppendExample({{"a", "0.5"}, {"b", "0"}, {"c", "0"}}); + dataset.AppendExample({{"a", "1.5"}, {"b", "0"}, {"c", "0"}}); + dataset.AppendExample({{"a", "1.5"}, {"b", "2"}, {"c", "1"}}); + return dataset; +} + +TEST(IsolationForest, PreissAveragePathLength) { + EXPECT_EQ(PreissAveragePathLength(1), 0.f); + EXPECT_EQ(PreissAveragePathLength(2), 1.f); + /* + H = lambda x: math.log(x) + 0.5772156649 + 2 * H(3 - 1) - 2 * (3 - 1) / 3 + >> 1.207392357586557 + */ + EXPECT_NEAR(PreissAveragePathLength(3), 1.20739233f, kEpsilon); +} + +TEST(IsolationForest, IsolationForestPrediction) { + EXPECT_NEAR(IsolationForestPrediction(0, 4), 1.f, kEpsilon); + + /* + 2**( - 1 / (2 * H(4 - 1) - 2 * (4 - 1) / 4)) + >> 0.6877436677784063 + 2**( - 2 / (2 * H(4 - 1) - 2 * (4 - 1) / 4)) + >> 0.472991352569295 + 2**( - 2 / (2 * H(8 - 1) - 2 * (8 - 1) / 8)) + >> 0.6566744390877336 + */ + EXPECT_NEAR(IsolationForestPrediction(1, 4), 0.687743664f, kEpsilon); + EXPECT_NEAR(IsolationForestPrediction(2, 4), 0.472991377f, kEpsilon); + EXPECT_NEAR(IsolationForestPrediction(2, 8), 0.656674445f, kEpsilon); +} + +TEST(IsolationForest, Description) { + const auto model = CreateToyModel(); + const std::string description = model->DescriptionAndStatistics(true); + EXPECT_THAT(description, testing::HasSubstr(R"( +Tree #0: + "a">=0 [s:0 n:0 np:0 miss:0] + ├─(pos)─ "b">=1 [s:0 n:0 np:0 miss:0] + | ├─(pos)─ count:4 + | └─(neg)─ "a">=1 [s:0 n:0 np:0 miss:0] + | ├─(pos)─ count:2 + | └─(neg)─ count:2 + └─(neg)─ count:2 + +Tree #1: + "b">=1 [s:0 n:0 np:0 miss:0] + ├─(pos)─ count:5 + └─(neg)─ count:5 +)")); +} + +TEST(IsolationForest, Serialize) { + const auto original_model = CreateToyModel(); + ASSERT_OK_AND_ASSIGN(std::string serialized_model, + SerializeModel(*original_model)); + ASSERT_OK_AND_ASSIGN(const auto loaded_model, + DeserializeModel(serialized_model)); + EXPECT_EQ(original_model->DebugCompare(*loaded_model), ""); +} + +TEST(IsolationForest, PredictGetLeaves) { + const auto model = CreateToyModel(); + const auto dataset = CreateToyDataset(model->data_spec()); + std::vector leaves(model->num_trees()); + EXPECT_OK(model->PredictGetLeaves(dataset, 0, absl::MakeSpan(leaves))); + EXPECT_THAT(leaves, ElementsAre(0, 0)); + + EXPECT_OK(model->PredictGetLeaves(dataset, 1, absl::MakeSpan(leaves))); + EXPECT_THAT(leaves, ElementsAre(1, 0)); + + EXPECT_OK(model->PredictGetLeaves(dataset, 2, absl::MakeSpan(leaves))); + EXPECT_THAT(leaves, ElementsAre(2, 0)); + + EXPECT_OK(model->PredictGetLeaves(dataset, 3, absl::MakeSpan(leaves))); + EXPECT_THAT(leaves, ElementsAre(3, 1)); +} + +TEST(IsolationForest, PredictVerticalDataset) { + const auto model = CreateToyModel(); + const auto dataset = CreateToyDataset(model->data_spec()); + model::proto::Prediction prediction; + + model->Predict(dataset, 0, &prediction); + EXPECT_THAT(prediction, + EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 0.6111162 }") + .value())); + + model->Predict(dataset, 1, &prediction); + EXPECT_THAT(prediction, + EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 0.5079549 }") + .value())); + + model->Predict(dataset, 2, &prediction); + EXPECT_THAT(prediction, + EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 0.5079549 }") + .value())); + + model->Predict(dataset, 3, &prediction); + EXPECT_THAT(prediction, + EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 0.51496893 }") + .value())); +} + +TEST(IsolationForest, PredictExampleProto) { + const auto model = CreateToyModel(); + const auto dataset = CreateToyDataset(model->data_spec()); + + dataset::proto::Example example; + model::proto::Prediction prediction; + + dataset.ExtractExample(0, &example); + model->Predict(example, &prediction); + EXPECT_THAT(prediction, + EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 0.6111162 }") + .value())); + + dataset.ExtractExample(1, &example); + model->Predict(example, &prediction); + EXPECT_THAT(prediction, + EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 0.5079549 }") + .value())); + + dataset.ExtractExample(2, &example); + model->Predict(example, &prediction); + EXPECT_THAT(prediction, + EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 0.5079549 }") + .value())); + + dataset.ExtractExample(3, &example); + model->Predict(example, &prediction); + EXPECT_THAT(prediction, + EqualsProto(utils::ParseTextProto( + "anomaly_detection { value: 0.51496893 }") + .value())); +} + +TEST(IsolationForest, Distance) { + const auto model = CreateToyModel(); + const auto dataset = CreateToyDataset(model->data_spec()); + ASSERT_OK_AND_ASSIGN(const auto dataset_extract, + dataset.Extract(std::vector{0, 1})); + + std::vector distances(4); + ASSERT_OK(model->Distance(dataset_extract, dataset_extract, + absl::MakeSpan(distances))); + EXPECT_THAT(distances, ElementsAre(0, 0.5, 0.5, 0)); +} + +TEST(IsolationForest, PredictGolden) { + // The model, dataset, and golden predictions have been generated in the test + // "test_import_anomaly_detection_model" in + // ydf/model/sklearn_model_test.py + ASSERT_OK_AND_ASSIGN(const auto model, + model::LoadModel(file::JoinPath( + TestDataDir(), "model", "gaussians_anomaly_if"))); + dataset::VerticalDataset dataset; + ASSERT_OK(dataset::LoadVerticalDataset( + absl::StrCat("csv:", file::JoinPath(TestDataDir(), "dataset", + "gaussians_test.csv")), + model->data_spec(), &dataset)); + + YDF_LOG(INFO) << "Model:\n" << model->DescriptionAndStatistics(true); + + // Those predictions have been checked with sklearn implementation. + model::proto::Prediction prediction; + + model->Predict(dataset, 0, &prediction); + EXPECT_NEAR(prediction.anomaly_detection().value(), 4.192874686491115943e-01, + kEpsilon); + + model->Predict(dataset, 1, &prediction); + EXPECT_NEAR(prediction.anomaly_detection().value(), 4.414360433426349206e-01, + kEpsilon); + + model->Predict(dataset, 2, &prediction); + EXPECT_NEAR(prediction.anomaly_detection().value(), 5.071637878193088200e-01, + kEpsilon); + + model->Predict(dataset, 3, &prediction); + EXPECT_NEAR(prediction.anomaly_detection().value(), 4.252762996248650729e-01, + kEpsilon); + + model->Predict(dataset, 4, &prediction); + EXPECT_NEAR(prediction.anomaly_detection().value(), 3.864382268322048009e-01, + kEpsilon); +} + +} // namespace +} // namespace yggdrasil_decision_forests::model::isolation_forest diff --git a/yggdrasil_decision_forests/test_data/dataset/README.md b/yggdrasil_decision_forests/test_data/dataset/README.md index efd50686..0561f158 100644 --- a/yggdrasil_decision_forests/test_data/dataset/README.md +++ b/yggdrasil_decision_forests/test_data/dataset/README.md @@ -12,7 +12,8 @@ Donors: Ronny Kohavi and Barry Becker Full name: Molecular Biology (Splice-junction Gene Sequences) Data Set -Url: https://archive.ics.uci.edu/ml/datasets/Molecular+Biology+(Splice-junction+Gene+Sequences) +Url: +https://archive.ics.uci.edu/ml/datasets/Molecular+Biology+(Splice-junction+Gene+Sequences) Donors: G. Towell, M. Noordewier, and J. Shavlik @@ -74,7 +75,6 @@ bazel run -c opt --copt=-mavx2 //third_party/yggdrasil_decision_forests/cli/util --ratio_test=0.2 ``` - ## Sim PTE Full name: Simulations for Personalized Treatment Effects @@ -106,3 +106,25 @@ test$ts = NULL write.csv(train,"yggdrasil_decision_forests/test_data/dataset/sim_pte_train.csv", row.names=F, quote=F) write.csv(test,"yggdrasil_decision_forests/test_data/dataset/sim_pte_test.csv", row.names=F, quote=F) ``` + +## Gaussians + +Generate two gaussians for anomaly detection similarly as: +https://scikit-learn.org/stable/auto_examples/ensemble/plot_isolation_forest.html + +```python +def gen_ds(n_samples: int = 120, n_outliers: int = 40, seed: int = 0): + np.random.seed(seed) + covariance = np.array([[0.5, -0.1], [0.7, 0.4]]) + cluster_1 = 0.4 * np.random.randn(n_samples, 2) @ covariance + np.array( + [2, 2] + ) + cluster_2 = 0.3 * np.random.randn(n_samples, 2) + np.array([-2, -2]) + outliers = np.random.uniform(low=-4, high=4, size=(n_outliers, 2)) + features = np.concatenate([cluster_1, cluster_2, outliers]) + labels = np.concatenate([ + np.zeros((2 * n_samples), dtype=bool), + np.ones((n_outliers), dtype=bool), + ]) + return features, labels +``` diff --git a/yggdrasil_decision_forests/test_data/dataset/gaussians_test.csv b/yggdrasil_decision_forests/test_data/dataset/gaussians_test.csv new file mode 100644 index 00000000..81ffc35f --- /dev/null +++ b/yggdrasil_decision_forests/test_data/dataset/gaussians_test.csv @@ -0,0 +1,281 @@ +features.0_of_2,features.1_of_2,label +2.153577276910627,1.8371451592694583,False +1.5939344353435811,1.8494518905455508,False +1.5286506907384565,1.5971375033261677,False +2.1358244205926673,1.8084144252881242,False +1.9939841140777448,1.9473391760813303,False +1.7155821887496516,1.6118931689985763,False +1.827981339890142,1.9514479914135938,False +1.9187843336191588,1.7786666195363376,False +1.7197140014719288,1.8664397814345979,False +2.1716310091835487,2.091561884325908,False +2.100398803312508,2.227180560662855,False +2.321016559011082,2.044335465400587,False +1.9887273892840691,1.8545693045615301,False +1.713406513303731,1.855192499539295,False +2.094921914761489,2.09557239786315,False +1.7505768621352644,1.964185865772056,False +1.6259078803564386,1.8922540053649888,False +1.8622046861353438,2.0248235094064486,False +1.8421743256617302,2.082198925596146,False +2.539732800383628,2.0523349786079788,False +1.713096779583923,1.8656527878408908,False +2.3244556295376015,2.300679067914473,False +1.8318027699157868,1.8960483863578623,False +2.6262545351475692,2.3284042024499163,False +2.1968486612144034,2.093946139453922,False +1.9614041069729802,1.931633211762804,False +1.6736803981199664,1.9898058923348252,False +2.1224756468560595,2.102215479924143,False +2.428505265539897,2.115416996453589,False +2.3049569910466556,2.1301990932731463,False +2.19992349526599,2.2306348224771924,False +2.0191199702548412,1.9317879535668452,False +2.076543549661604,1.9683678000551312,False +2.651874586088501,2.1979055151294946,False +2.0460961073699537,1.6891375700605715,False +1.5699267972890758,1.9770500141457514,False +2.277334711802109,2.133785544600683,False +1.4969106490176003,1.6638224075784953,False +2.1705920974043544,2.1447241033222944,False +2.2593820775602436,2.1127179994353487,False +1.8993221121775288,1.9767718346756147,False +2.1521267394159684,2.0581458079137898,False +2.0729823648514425,2.0111093945241225,False +1.9715854029120514,2.087236697262908,False +2.3406197484136246,2.17584457442625,False +2.291627372875806,1.9816683116013707,False +1.746098495894795,1.912814532810929,False +2.106354089950419,1.995434636773205,False +1.943436384799374,2.0207296441163227,False +2.0714488007505953,2.1364851592094802,False +2.25343644438834,2.2138063753603197,False +2.2469003148645386,2.078832898046299,False +1.8284447121082188,2.0708976631190286,False +1.8810751217130333,1.8177856456668795,False +1.955888371873416,2.015867067711082,False +1.8136211657224905,2.1053501955366367,False +1.9285676660840505,1.8286310235634808,False +1.7026698405863405,1.7760128150554466,False +1.5398446388298672,1.743024243618115,False +2.3387756922694445,2.020567377740466,False +1.7780313554574796,1.8769588191679656,False +2.805539675823081,2.2637860426744068,False +1.9745295556374391,2.2721055194503212,False +2.4201734256706344,1.98897584132596,False +2.0018830824452887,2.1861059721004508,False +1.7947181638022578,1.910609511636205,False +1.908138871739948,2.137288324782873,False +1.9839727688735278,1.8685228086144594,False +1.7836996783201942,1.7960823241773622,False +2.17341104993303,1.9753763264730095,False +1.9341972013103697,1.9911834512716307,False +2.383892490563376,2.085310416098167,False +2.14444940366981,2.0008535806254732,False +2.188710698942128,2.095827996803256,False +2.2376133057578036,2.0999084427311683,False +1.2562220713463892,1.62283062868358,False +2.820119221524675,2.3083637594415047,False +2.0602294233943366,1.9663205849318968,False +1.9393758786438466,1.9864091197118718,False +1.6893166376108721,1.8197806270684977,False +1.6174135966738807,1.8611594858927671,False +1.9667003096137126,1.942585449165151,False +2.050125380067692,1.9522390414820814,False +2.257056529938172,1.994712016700821,False +1.9071588873844407,1.6089542631786358,False +2.1230329920106574,2.17011457019399,False +2.4360474036009507,1.8590854072932705,False +1.9453658107318683,1.9620429614706876,False +2.185845220744233,1.9007724573403333,False +2.046469520283993,1.9216289505810884,False +1.833281610680562,2.101069985483989,False +2.4627402097960984,2.186788748667749,False +1.8050492168770758,1.9056399508627149,False +2.1799586051503703,2.016007490893276,False +2.03566145837529,1.9770753643155299,False +2.3355256745839563,2.012705291820365,False +2.6920276172651376,2.1015827234360414,False +1.67612722384749,1.7132478736573904,False +2.2381428730803026,2.0431523813717045,False +2.4546341210787435,2.134672668241019,False +2.1505459348083287,2.1478760266236225,False +2.434904774798712,2.335272709242209,False +1.240616869784891,1.7715678969968844,False +1.4205001832695263,1.9235399571668763,False +2.323879013325074,2.3577372405349717,False +1.558750879358681,1.7982318536227375,False +2.0586498924859127,1.8615808507173535,False +1.9978642244335243,2.1896083634855437,False +2.0028845640881427,1.9054724436640422,False +2.317656442367249,1.964354652744067,False +2.4641573667314334,2.0157605571656925,False +2.3423938631230916,1.955152209803274,False +2.441874342341385,2.440688759728985,False +1.8351705647945953,1.9900169086570791,False +1.8029698533056726,1.995472828362521,False +2.1050463383010802,2.0225860509765687,False +2.6340083560328975,2.260304501932143,False +1.630633279717299,1.807519265200112,False +1.5771251096023178,1.940772002148525,False +1.8961691893191295,2.198972604214995,False +-2.1494640690893236,-2.093295493490855,False +-2.0005674448514013,-2.4189861273786297,False +-2.2583949082328125,-1.7975865422936084,False +-1.814438260764112,-2.132951579210191,False +-1.456839525762363,-2.3917180767673214,False +-2.103496163046494,-2.0692519229406408,False +-2.837925500043962,-1.4187413559151762,False +-1.8901003956379825,-2.3133768145723375,False +-1.3846479671427667,-1.8243013999482853,False +-1.8711421579934107,-2.1820995194600137,False +-1.9681331827894346,-2.457704094868807,False +-1.7614921716725467,-2.1123314956529664,False +-1.9597855410336131,-1.6393835413400883,False +-1.9145755667452826,-1.9212597663610194,False +-1.917050208553345,-2.219981481168594,False +-1.7491985841697193,-1.5369922667586549,False +-1.7723583019706206,-1.734527355660535,False +-2.2631844556754563,-2.2603361668618778,False +-2.432262807287552,-1.6303240787514692,False +-2.0762539602822105,-1.5800468172557043,False +-2.2345735048060402,-2.131252694848574,False +-1.9713724738426226,-1.7235649794021466,False +-1.981774941260148,-1.936662573497685,False +-1.9950417298083154,-1.9468436839172119,False +-2.3349410053654234,-1.9757218697080163,False +-2.0559736980534398,-2.0170473442657544,False +-1.8522990332190052,-2.2042034423026657,False +-2.025352408221389,-2.089208564832051,False +-1.8748093985075411,-1.7645688046953232,False +-2.286627578712107,-1.8242268706692153,False +-1.3802650039343498,-2.4413470777497874,False +-2.2490515685945343,-2.2641732799532512,False +-2.083729316462987,-1.5131452742137999,False +-1.995994197095847,-2.2084080785861677,False +-1.8134589487083284,-2.179941359321254,False +-1.6629763513934195,-1.9084198879267968,False +-1.5833661810889195,-2.1984032729459058,False +-1.090742866288391,-1.7526246124899627,False +-1.8036259542239899,-2.0153565342822994,False +-2.2176791357403283,-2.260330603287077,False +-2.040793197830177,-2.239180935647939,False +-1.915197286325474,-2.247829229554196,False +-1.8136751897482974,-1.713163488762591,False +-2.211752152220685,-1.642194179673592,False +-2.071382580725655,-1.6534136341735324,False +-1.8685500958126289,-1.6633015035028724,False +-2.299105938658905,-2.0320381960337675,False +-1.5645712218227195,-2.1854110543044736,False +-2.6111603677042385,-2.5827767544429365,False +-2.7519321958028184,-2.6342491765750475,False +-2.123491748956545,-1.6164415751474834,False +-2.1326687838539518,-1.9029417939195703,False +-2.032997447049082,-1.9974353163691927,False +-2.050459651923415,-2.0522541032923973,False +-1.8616507700068947,-2.3527948014323945,False +-1.6969618467995826,-1.723994620025671,False +-2.0585172022627702,-1.7583819727303456,False +-2.210403327877153,-2.1611669071626016,False +-1.953120844918975,-2.0570663075254583,False +-2.1346214098014866,-2.2017344116359787,False +-2.167248416535813,-1.7182493767410536,False +-2.582997021700506,-1.8942516906892002,False +-2.070931085543896,-1.7816559500001055,False +-1.8454779159081902,-2.834760340295877,False +-1.8246060168567722,-1.9027177269654736,False +-1.9934411490120343,-2.1406021448833674,False +-1.7440156334133134,-2.12390879291331,False +-1.449584712005094,-1.830685143351706,False +-1.3586515797681553,-2.2356601990760705,False +-2.5267776920698557,-1.7855631207542553,False +-1.7441887814824142,-1.9893919708835728,False +-2.461637973723393,-2.1343685554148357,False +-1.8146043398238996,-2.0552528976961235,False +-2.0347955556417188,-2.0526376905985257,False +-2.2801743966879506,-2.15990609782508,False +-2.4279666261561594,-1.469612015506692,False +-2.142611862541395,-1.8567169454547328,False +-2.3065657833923927,-1.7616415281196753,False +-2.5619482932905906,-1.723815464583513,False +-2.0106103774636135,-1.3668184839197872,False +-2.3919602218532057,-1.9770858559521225,False +-1.8898304558348407,-1.6301302422871289,False +-2.1268570884172324,-1.9740606780427137,False +-2.6427400187232104,-2.2490506592068122,False +-1.8645152148334274,-1.668747702109036,False +-2.0845208807196838,-1.3830933430405121,False +-1.471925232065272,-2.018195747532443,False +-2.724050900352136,-2.533269912741796,False +-2.233357647988224,-1.6652476676227495,False +-1.9069183136648697,-2.6282743448668437,False +-2.0686297486610488,-1.5159915876310655,False +-2.1124414061907957,-2.2249908851827,False +-1.3836127692445652,-1.9839771389604957,False +-2.143747129635824,-1.8949498523515163,False +-1.9948505820877733,-2.128742683470528,False +-1.6374631014344112,-1.665289459164654,False +-1.7477415325576688,-2.0308661652720614,False +-1.655929887080155,-2.014910773747603,False +-1.8600070198313479,-1.6898939396181496,False +-1.7573466919202954,-1.4630735950381186,False +-1.8646147951879488,-2.505217995760471,False +-2.3480510314946827,-1.5949679543954822,False +-2.0993849509797884,-1.8840382564600726,False +-2.255436696959248,-1.6997355728959098,False +-2.1154496746498372,-1.562567528417144,False +-2.1596702062694533,-1.6645599809846914,False +-1.7976811685537393,-2.2167175716242453,False +-1.6703011001758463,-2.2704903471427995,False +-2.2467401566712755,-1.7834866123661923,False +-2.1876026004397966,-2.1781529201763528,False +-2.1031702127631178,-2.3000507569617548,False +-1.6865016770978307,-1.817445590454545,False +-2.020798609007145,-2.032517620152062,False +-1.8649533461698464,-1.4703994698442786,False +-1.7387090592403274,-2.1525371402826203,False +-1.766774238425341,-2.0356313516309266,False +-2.0596994551411174,-1.4400585874543625,False +-2.125681369303437,-2.1437554745382217,False +-2.5856315861735695,-2.420698743635941,False +-1.8646631183796205,-2.208476270355572,False +-2.4900394649825124,1.1511651544933024,True +2.0344476046103894,-2.314140863333434,True +0.8076339848611198,1.9914270019666915,True +1.1057496849626673,0.7770184235731925,True +-1.6361417147560235,1.8528517699320703,True +3.562467519999598,-0.5955088779076245,True +2.2574545346512425,-3.5508716827046776,True +2.6821728199197272,-2.4619998654228183,True +-0.8392250488802144,-1.5993516325971484,True +-3.3591708495516057,3.237048023489943,True +-1.038766579570094,0.24557950780911586,True +-0.04706987282939945,-2.9427108644564948,True +-2.348367528884536,-3.3904895277414306,True +0.06337359956229971,-1.907603585336103,True +-1.1435071289376646,-3.1354773704221053,True +2.3004147172200575,-3.147328985689591,True +3.8856705897055024,-2.5827106844926613,True +0.5792408994771945,-3.641237323287295,True +2.296930317246673,-2.4831524174536277,True +0.22323182726501312,1.9206203452725727,True +-2.8005481206619525,0.40869739349577383,True +-2.267062333369765,2.0735683945371575,True +1.7833215484497096,-2.587607725783336,True +2.8957324613365873,-3.841799203869246,True +2.8818959793849217,0.47123048928750677,True +-0.7742362231532063,2.0699754481786927,True +1.735432013539759,3.8986093984674124,True +-1.7753196110961378,-3.9696506207252593,True +3.4712208665191024,2.8631768363031505,True +1.8308070201540474,0.13351026337230731,True +1.655649961990088,2.244236455013942,True +-1.000992451705125,2.1625802026396608,True +2.004994553401403,0.9056896923540778,True +-0.785072608257301,1.5784641541147275,True +-3.975097137398463,2.1991731737967823,True +3.1713328196547756,-2.085474340499963,True +-3.033862525526162,-2.237728097650179,True +-1.5832261496792137,3.0642280685106735,True +0.3453314419047562,-1.7063068031485553,True +-2.893162481656428,-1.6788442951636666,True diff --git a/yggdrasil_decision_forests/test_data/dataset/gaussians_train.csv b/yggdrasil_decision_forests/test_data/dataset/gaussians_train.csv new file mode 100644 index 00000000..2d54282d --- /dev/null +++ b/yggdrasil_decision_forests/test_data/dataset/gaussians_train.csv @@ -0,0 +1,281 @@ +features.0_of_2,features.1_of_2,label +2.464854487536355,1.993463059500049,False +2.823197692597556,2.3193933925080037,False +2.0998737916645984,1.7689332196137755,False +2.1476376651817626,1.9377793099713447,False +2.0943238101840325,2.069824514381882,False +2.4360052961818086,2.226922018267641,False +2.1862765496473906,1.989026493632973,False +2.18220145821388,2.035633363070066,False +2.241371500777097,1.9074115148711677,False +1.8234667333256973,1.850821973645688,False +1.6724152435564852,2.2066985679038207,False +1.9650810340580973,1.8466761487805892,False +2.0467285359098675,1.6765113071046933,False +1.9567402254530557,1.9682202433038087,False +2.717976298443771,2.1737862346097074,False +2.136874990627992,2.054308106108471,False +1.2678198393712778,1.7185839949893762,False +1.974195281483884,2.038932321029683,False +2.582724493805179,2.143169148576397,False +1.8378898663573155,1.9671246326042644,False +1.3926843845764685,1.8147392486540477,False +2.2049630725398988,2.380374870862087,False +1.775408759198537,1.9502941990122764,False +1.9671382276229497,2.1745102713351026,False +1.6176531520284985,2.030517469068083,False +1.9292393871618583,2.0977230621052287,False +1.56726196093195,1.831531056043169,False +2.114296478080786,2.069660388418413,False +2.0979955758437723,2.0457348147430383,False +1.7715680547874086,1.9673342971892964,False +1.7648330252134583,1.9693699120645514,False +1.3540116149382397,1.75632063490871,False +1.9229865663124373,1.928618004516528,False +1.803539362154008,2.1392530947627657,False +1.8330850379462704,2.044603197902712,False +2.1819333274475823,1.9914736432340843,False +1.8821289072096374,1.7568518413616836,False +1.8887215027722222,1.8743367198024479,False +1.6637626640295873,1.9422159396049687,False +1.9534157893988542,2.0214485560418542,False +2.019201448190501,2.190738231544004,False +1.6629842557884693,1.7355745126064257,False +2.828499408047683,2.2438121804130695,False +2.1853769602044717,1.924060843423638,False +2.08109595923861,2.2115423811694037,False +2.2616492303124436,2.2117182891401157,False +2.3151139258308118,2.1479312467143195,False +2.269113766528626,2.0987970510237357,False +2.5021437424377977,2.2853192781961007,False +2.1379394403052405,2.0592418144430074,False +1.999257602291366,1.7090325223349585,False +2.0173340785870963,2.2059228732447114,False +2.30958925095897,2.357904325908453,False +1.7079888566446384,1.8969519893998688,False +2.7991325468977157,2.1599646855702925,False +2.6272042964024216,2.070264786906999,False +2.3625730498567887,2.3400594198980333,False +2.1710871166327146,2.1391131581654785,False +2.1460475674893162,1.9373083063945167,False +2.3810337421078436,2.122989892636801,False +1.7674528848675515,1.8090388522603036,False +2.4310356859135798,2.2002922165016825,False +1.819188756761979,2.0038411879368274,False +2.4307631336298883,2.31328833862556,False +2.248548265549998,2.03830210351808,False +1.9970065586726455,2.1170765135845033,False +1.8740460241853432,2.032066195750191,False +2.0622321069092435,2.1336631703265954,False +2.0569945117611494,1.9436085664429232,False +1.7731441200877875,1.8092698900966533,False +1.8247781578129492,2.12995297591055,False +2.2111435014042957,2.094938090087836,False +2.7410832113301042,2.0557909269239087,False +2.130200115578007,2.215235495113103,False +1.6075748285495772,1.9787827596501073,False +2.4660876409968964,2.276864499676884,False +1.619646244765788,1.8975600266964954,False +1.7945355749335234,1.89778157515908,False +1.9229463620795428,1.782145521777592,False +1.647916656989338,1.9758475389373347,False +2.440662484930295,2.32864642663841,False +2.2143985089730056,1.9760313663446,False +1.9913345296263996,2.184115496977454,False +1.3674210234243558,1.7928452384111646,False +2.326349889809418,2.0031896262138837,False +2.27341550758021,2.0141620715196504,False +1.9890789562564972,1.861562680595869,False +1.9839978967619627,2.150424836596439,False +1.6462441293951757,1.9218084221269216,False +1.9137876638235471,2.021017965584703,False +1.544214855592258,1.7941675495032563,False +1.2487234368105824,1.6700002317574074,False +1.6764701466355483,1.718661517061833,False +1.7937295543073828,2.052521746258894,False +2.2841314874356233,2.2764648551207305,False +1.8162028614730814,2.0944424154848074,False +1.6650772569869656,1.8146763530905083,False +2.056622359364121,1.9516215205831702,False +2.3849392733525785,2.10088904258565,False +2.8068750156983278,2.1273150339385953,False +1.859137462690252,1.9764666051055495,False +2.40340576377975,2.0608558130801313,False +1.6752786128124848,1.7156817718663628,False +1.7884861205426201,1.8828880995030473,False +2.0285028107086194,1.9730989536953971,False +2.270856881872083,2.0143477581069047,False +2.026668245335731,1.89391982603789,False +1.6962968347196363,1.9721698993243222,False +2.102758403183135,2.0613450942432707,False +2.4400297876731907,1.9028664977066632,False +1.7119361027056164,1.9828807159078092,False +2.0420956177273504,2.0955808747895976,False +1.709553955487168,2.071753799649838,False +2.096311997729152,2.030888704273429,False +1.8139187018808456,1.9858251660008535,False +1.5770582508620794,1.9780312550213501,False +2.0079217177498556,2.0882824664424966,False +1.9874989821144156,2.1712389935464755,False +1.7193011018944846,1.60902301421818,False +2.274785995964089,2.091254936373715,False +-2.1912311076656685,-2.1191815442986393,False +-2.0398641732760865,-2.0893372638205183,False +-2.092703890714137,-2.502801141898993,False +-1.654300530565064,-1.6761144223889537,False +-2.244009277761261,-2.4399272983407543,False +-1.8436805370641725,-2.17273639094392,False +-1.957414051003766,-2.0957985251435285,False +-1.792538374678944,-1.7915752569031982,False +-2.2176792135390753,-2.4150091866185166,False +-2.4748815192005247,-1.8168861862678385,False +-2.356657777335209,-2.152044906289606,False +-2.1788942115351526,-2.0157701888808637,False +-2.580883941753952,-1.9433664209618515,False +-1.8428326928497383,-1.9734733738866015,False +-2.0932658515095417,-1.970779950119365,False +-1.880286096307961,-2.831777826927995,False +-1.4132263075247917,-1.8829720031936221,False +-2.195722574716106,-2.1172860125562805,False +-1.8518774667952436,-2.03483118171031,False +-2.6092053403344484,-1.3806521415922042,False +-2.033162197169742,-1.6939481864852601,False +-2.2076149543353174,-1.5390868837262608,False +-1.9140968933323161,-1.8173468496573648,False +-2.3135760098440863,-1.6366564130951897,False +-1.7930545506395634,-1.6094461311305004,False +-2.1884262678924737,-2.1443081355382363,False +-1.3088249906948175,-2.318004746816464,False +-2.040784910203496,-1.6589325912191915,False +-1.9706825096855434,-1.8251138960740119,False +-2.1198347087788627,-1.8889832336457444,False +-2.391958055520595,-1.5025607961145435,False +-2.035449213538571,-2.204053461199055,False +-1.8000850753904256,-2.138215936216566,False +-2.400277541420826,-2.4040152517392666,False +-1.7918680541929604,-2.04787203144388,False +-2.040110467900532,-1.6766768582071212,False +-2.338047742627023,-2.2192033258594472,False +-2.1154639427543827,-1.9716945232048777,False +-2.012651435387174,-2.086066157716972,False +-2.018487920628694,-2.0321915828873522,False +-2.2158813165655378,-2.243897896566223,False +-1.9176450926828181,-2.2672745248986583,False +-2.347206577757256,-2.093687675337708,False +-2.047300104849145,-1.322982950810537,False +-2.21141008275687,-1.7170217825091516,False +-1.7758434997386106,-2.3566834865611206,False +-1.7680241067792202,-2.3551641920579955,False +-2.7977516713990225,-1.8181041426921858,False +-2.5267671750313156,-1.8647196614582255,False +-2.205203269321165,-1.5021347611430385,False +-1.6794471802051973,-2.136015741155416,False +-2.2063512833086047,-2.364223220928236,False +-2.1322767896877775,-2.0841066485553528,False +-2.1094080633175056,-1.9529888434182907,False +-1.8264435506813366,-1.895103662902048,False +-2.229243177171933,-2.4313374421404736,False +-1.5906404455692587,-2.2068347553649814,False +-2.195688079980506,-2.156356793690333,False +-2.5529208650469943,-2.143392201212146,False +-2.1438967442023844,-1.8138925104969463,False +-1.7904628552677992,-1.9988687332741193,False +-1.720445487765709,-1.8980105048596214,False +-2.0047046334807663,-1.9517215495105331,False +-2.05719604807442,-2.118454854210035,False +-2.08032006106819,-2.338403399441002,False +-1.9158674884051112,-2.297937083278874,False +-1.7475106207779092,-2.0748375740482845,False +-1.9851515055049729,-1.851848967115713,False +-1.8070056604811215,-2.4711870225900356,False +-2.0620711028491914,-1.7359463263757653,False +-2.5094317458296764,-1.883815857381481,False +-2.676669268820657,-2.306752053090681,False +-1.9884108344479436,-2.4970145306965863,False +-2.295653221305245,-2.441550502239076,False +-1.505559520337732,-1.9507316733537998,False +-1.8298129166441992,-2.0668025301545465,False +-2.10602952462716,-2.4849422565953097,False +-2.0875512088243586,-2.228447663543487,False +-1.742622822712299,-1.6576694400027279,False +-1.5600263853277467,-1.7442344181616305,False +-2.179596181076896,-2.3347690957881184,False +-1.7700010455064743,-1.8931121547583134,False +-2.530561535203109,-1.8933554621768693,False +-1.7556440532536401,-1.982322323245511,False +-2.0555161013028025,-2.242294546284907,False +-2.4339604098690164,-1.7599106151979917,False +-2.0927343334315127,-2.0700399984631077,False +-1.48018364392426,-1.7946496679422428,False +-1.8887524996156695,-1.9573814584438294,False +-1.5440015417702682,-1.4841232077751416,False +-1.7211484665561416,-1.8253326225806228,False +-2.6283809213618436,-1.9628834257299481,False +-2.039032086258111,-1.9718140311843293,False +-1.7170861738032448,-2.8219031501568668,False +-2.1707936160410557,-1.9190286935177716,False +-2.140053663815829,-2.425071833937878,False +-1.7393109539309615,-1.9169384282461617,False +-2.2913313711333454,-1.9055548386452528,False +-1.7535242863850613,-1.9984122061101917,False +-1.759830558970701,-1.9765219474515017,False +-2.1185686947963065,-2.3478261549199737,False +-2.0257792300914836,-1.9417121185862685,False +-1.7372501715238007,-2.034532240546168,False +-1.8627753181337028,-2.2893836041201183,False +-2.2347887467482574,-2.033116789708066,False +-2.316388539195504,-1.7539256488025956,False +-1.861060901204418,-1.916271270682264,False +-1.8983287624352165,-1.3936869315545608,False +-2.140659256390039,-2.6604323856501675,False +-1.9402099409310605,-2.0151810622885,False +-2.155255712753121,-2.293648957807631,False +-2.1317568565406444,-1.9455984712346537,False +-2.1508450101927616,-1.2762638961368755,False +-2.2881513144899444,-2.2379352088123015,False +-2.6865860120043585,-1.924554675493539,False +-2.604921988339928,-2.1618363900123505,False +-2.082701160368167,-2.2129183897540665,False +-1.4783381967636466,-1.7016816826053502,False +-1.6042589371095273,-2.2647256455649756,False +-1.6614217806456295,-1.8511997160968114,False +3.80980530763056,3.1183492515643216,True +2.1164957948616685,1.5859878225463246,True +-1.316014642592803,-2.818515374346341,True +-3.498911975521522,-2.0647863663881214,True +-0.5417481505496111,0.17597018903985973,True +2.184668432438973,3.669927384452744,True +-3.061436156921512,-3.1439668784496675,True +0.717557784108406,1.9631845915783437,True +2.785203042775879,3.4866566417343083,True +3.867409938085136,-0.8015864622037929,True +-0.9573185317794151,-2.817530586421821,True +1.4794755094684753,1.2540956675266965,True +2.896500766809658,-3.221936041698875,True +-0.017784737397265715,0.6486554373765046,True +-2.067543679680653,-2.6477967509667097,True +2.8766466913569717,-3.5317206221155297,True +-0.235032768655417,-3.0733279895929178,True +-0.34352990934906114,3.8396986107384743,True +-0.6103491723562176,2.8569993400365385,True +-3.061475486534449,-1.829983385905087,True +-0.769658074661324,-0.8015028799253541,True +1.371067829361225,-1.2422549809959387,True +1.7101349472801308,1.11349519380314,True +-0.8067108379618153,-0.5459189787654459,True +0.9162215984825659,-3.439662478842843,True +2.5792539068455227,1.2273692889090952,True +1.8107397153426819,0.2953840086591235,True +-3.116183112066042,-0.7597150936244006,True +-0.7570113372115514,-1.4316560796542648,True +-3.760397400762005,1.8980339407718185,True +-3.1217243354999944,0.850465064360681,True +1.625739971737726,1.078290583469558,True +3.6731380158238,-3.173614759318891,True +2.937337272841593,-3.766478121208694,True +0.2793348394166717,-0.7660510564859297,True +0.19347088315006555,-1.0792009835199217,True +-2.4754646804794556,-3.8470168204104818,True +0.14519851032939446,2.7422149014787385,True +-1.0142723540416467,-2.217089455880158,True +-3.3557439722252473,-3.317512615050373,True diff --git a/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/data_spec.pb b/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/data_spec.pb new file mode 100644 index 0000000000000000000000000000000000000000..7fd0b4a15918ef8a73e078d195ea4bf7b04c9f73 GIT binary patch literal 134 zcmd-w=U^1#PfJZKDJ@DZ)-#CDPm4FwlHp{40#Ohx3nCOj1PE9#L@*|BbD?T9BwC{Y Rlav4#2h2tR1_>qwMgT`e6A%CZ literal 0 HcmV?d00001 diff --git a/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/done b/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/done new file mode 100644 index 00000000..e69de29b diff --git a/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/header.pb b/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/header.pb new file mode 100644 index 0000000000000000000000000000000000000000..f33587675c6e6ba1db768ba8071888abce2eeceb GIT binary patch literal 67 zcmd-Q@C^3%aSZYF_ltM)4{{9-5nz+}4+D$_FxG!Y2SyJBCy0ZKL4ZMmL5U%O0RVG5 BA>04} literal 0 HcmV?d00001 diff --git a/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/isolation_forest_header.pb b/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/isolation_forest_header.pb new file mode 100644 index 0000000000000000000000000000000000000000..e306b7d72590a9f91126e647ebf1d0b0ac74b34d GIT binary patch literal 23 ecmd;J6iAWcb@K6diVt=T40ZK$c2!8xVgLX+Mg=MW literal 0 HcmV?d00001 diff --git a/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/nodes-00000-of-00001 b/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/nodes-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..ccce3bf29b64965a3c072b8a6eecabc02d22e52a GIT binary patch literal 103016 zcmb5Xb$nOX(mhO&Ai*WLySo%iaCdi#6?bTXQc8j1?ykk%izT?bySux?`+K`)_PH}> zACl+(ardXQXV$ElEoa*|6t5BHzyHPkfB&B-cBC*deG)~C9U)rS5ygV@{{Iz9e)m%? z$ix4?x?Eu+`I!1J|JfBS{x;x0JFe$a>D{Sy5_(1szrzYHD=hs2*MD@=Ae+3$&R?za zT+z>2_D((r=bph+D(aKO(uaTfZu$}O>5r5bvh>EOXP|VeiHVIE&TY9UL?a8dJ$7c z50vcyd7>NZgdfe%Z{9t{54`K8C~Ebq?Q6{_X7zjV#<-$3K2uw#g`DEx`N+Wq?^ogf z3HZ;R^Fn1R{%;lkzCF1W55$rBPI#+)#C#vf8IMba`|YlM9vR;xo)tG$+o++!&EDJJ zuE#vMRD6zJIcUx6_Qv^xZTY^QzZ2vhcuV_N`eFUUQKe+P2hNl8{#O5ex+J&C8~J7k z74E^Zi9^ssJ4c_W`m4zs>WhrqrM_qAXU7vx*>20*kcZ{OxJmC3_k*~9ivG;w)9KIf z|F~VzKwE!y^uVWarTyslioahpT8}9C!u3n6qkUei zie}Yw+6Q_?n?zP|!vFFK36-4mClfF3KU^yRRWtgA;2(ZUrpaaXQ}SyZVbA$=_XFU3 zT-8wFKz@GhKGj~3HyRbJ#t-6~P^5-+J{pv~o3)=EoJ`+i75}~m*@HtfKEe*z09?vr zcFw0R4L=t~OChJ}J;F0V0DQg@xXNR%2_l`+N?YJKJtB0Ml;y2!K*^Xo40Kaijsh=wOobrC6 z|0W>+&Nzkr`0lBN9?u8PIE7xsfF456YaC3zy)8)2|7efLSDFWl{;2pyYLE=|Mh?4+ z&D8fw)u-P55&k^#8-8|P>DNPV*~%cr8*z8|?T%eOErG-}ykPfiZyZt2^G8yp*Z9DG zD}KkJv`;0z;hDEC72c|t$JG3RJdWQK!J6MKmua2hi3|Iuz4xj43UMwO9IlA1z3;D_ zE!)KtA2!-mKyt!0*g*#@2 zAFk(8>Gi#x!pf&_`P$a}Mw*{4Nb*O#&o7r$`jD4ud)TVKco>am!w33#&euf$X1-i1 zz1SPl3O!!Gq)>W=UsbpIpEl%Q$QhqX9^dYH5&WKf%8EGIPWijjxqJwD2hOUhQ3QWI zPoE#~8h#8u`bmnu7p(G;6_;7%IsTd>`Fijkc>Au<<9zxm|MRQdQGO6l@Qa7S-Yb6v zm&m7gC zPfxrXYW+_2YD?=;j8;U{m~A&M{Jcpd+ZuxCG}H2#27@<~^P19`gaoBmHYVJ_0X(cptm z#YRg5#P`(H!_VY^Wv-bz3-oucHF;=9`L&qKOZW+lrf?N%KisGhx?Vn zdcuWI{2JN!ows6rwep^odyz3teYxS$o4+bBieTCrz-pi-b{K#>!A1;kP z_@68Bi}2?j55QeIqP?}<%ARQx{G)!nAlWKq2fv*<#(e((7WsVP3Up|k8JRe$IP3LEB~+`wYR-ESK#%8Ugr$W_)v^*w;u3v4VL3k;~cmc zDEztm7jXWnaVx;qF1s7s=aJ=&c0k{wo#*DHab)rWKe215eZQ)|_3f&}^}w&yRG!nB zxIFZi@1yf`;}7jrb>L@f+BCQCOP4jU4gq-1A1-dGHB4K;Bon&n1BOZ*P14 zquF+aKa6 za7ymyxc@uzKlIMNPOAJve)}q&uX5Z}I_nkajlSx!o@6_d2k6g!kO=dXks}YYw-tok z!){T(N>;n!J+=gi`P*d={Pc0`^JcNp*%U5#c356O@^RrpFY$1>U-W8^)B8(@fCrvc zV&1}hqw>>o$YR84@TI8oZyw)%7Cap+A=F;Dt4eo+6@@qE^J zF5I?{+ffOsNj&1mW8Iyg?Oc4 zent;|?^n|6Fyy0qQ<@*ij@J`TJHkG55s&vNWapKK{9B8o-3%Z2jd!etlEY7jm(i{H z_QLmVGEkit0{_oB(G@=Ai&oJ&r{M#?0n@LmegXaRpHl;e<5YI5I{V6c*5gv?cl$u+ zxke5@bBEa1lWXJ2^JDIJTzchYNDaNegrCIGFInTf|92#7zLq46EaKyN!s+kG-@nbi zh<1xb@kcjtLT`Az?pFWdM$tNy>|F7=@`-rTx2_*x8)rK^^cQ@*^7Qk{=T`gpF=%~a z_`_eJ;%lt>s}>!AoOyC->=A#_Pq&4hiBsu?ZJs937FXnobl;@%H+n=Lq1Ng2o(Lx=~u8@8^hj@*eAfjze$mqrhMHXi=Fh#VKN4-+%L8h?<#&2c)& zroX0H7>rAWzkb|nr4PUP9;f#h$EfdjJ)EeIPW<(}66TeC{ktdM1^6X9vk&yhL*``k z-B$cJ$15Uu==o+`pQpl)*L)Tx>{pcsHTV`G^>+0P(hu#6As^FJL3ZUUTKJxK&C*4;n`Rt>WtbFU&)#v5Fsg-0yknESp%kav2 zzJUIpS@k;==vO(W-rIQa9@zQ~{Bb>%=KIWY6BqE4Tw1Akf=|2GdOr#MQpxGNrf}9L zQ}^@a&*{g&jdei2L-!glk8_<;yvToo=ORx2u10gk2Y+FT`zRd9XaC$u&a*0>d z-z6l96C3jz=Z8w?ya#@Mjq*X@xxYiudp}RVAAo$*m-NCRrvcE!py`=pd`}x?mKO8T^bHv6vL-FZ4cc$_K zp7k16@xTL*{tM{6zm9|dTlu*aaYzX52fv4N>vJE-@8nnK;szgl*7glnaX>zH^#`lG z>2O*v6JMU^oPGztiTlM8c`)O^BhEK1k6Qh;`N{r{qt%SZR{cLV$a__u7aV?wZ&vuY zR(_5i;>hMl-!Bi|nY!Kt531+&lJk84PV`pvy8_1HQnkm~vvt(^4t$?Zt|a_8>oBU% zT-KV87CDri%P;uVj2WoxAwOD$-pjI`E)9R^x4hgSK#o7icSap*jjQvZknRr=-@xqe zfE$AU_iJeV#d?MxaQ;e@GtkB__MnVb`QeK6{UbcSC+G8d-3}2?RsatB=Th<5R=B0g z6Zlme=x3d87GKrpe9({GfX27Nue-RG=--;>xZf+wdJmoj@9Ezs!EenYX+mY6u)zXr zzY05y)?4H^A(b&y9K+gCzh-_^n)$@&qn~B27t*~Q^b#ka@077eT#FCtbu#oPE}bF! zpRq&zyPy2f?n>TfA)RX(Is9D6v{K?xT+TZ*=yg}`pn1NHr|?$Jkmt!>-`!RB!+|H~ zQ{ha#VLuOc`VZp0J0OQO-jP-9?+(5#jb+sz{xiv4eBfu$pQ_$N?{Z4IcjJ04O?<#R zctW4YBhRHe{`g%tbV>XAeHt>K9UZfqk@^`o0bPFAw_hyQBqU zTL%9(zp+0qO&);1IPwYW{PT573afm>&Iv(s-ULqGuJ(Oj`lRs$5B?78{(U_0LEsWk zK6laI;pc1C3q76h@q&KKCbFLcJAC1< z#r(9^cHdMkqLM>@eeZ{gALO^|oPnJAP#MMWf?mg;69bp(W3s-{L%db)*w^(*ZxS!I zqta4VqDfe)?npR&q%nEC2{b~c+lC-$13f(Oz(rt-b*v;DoE zSAR=&>YZ#)gNJx-6}T(wJ=*!(zJa6d=e3br()^&}8Jk1S8}4=nuFpiZ zA9KYCzOmlR`A+lm>8SUuz`qnm&c8f=>GieWRQZ08`8s;QdzW6fANUFEseE%HF zu}(1hke7})U+DwCdn0{r1pV*5)jE`U8eG`#{mWm(!}~{m$EN%Lr6E;?zOHXvse-I` z`31jpsrCIJ__T!;^Q00a-{`&iFL(YDvpSrCemXB#eB3+`*Z0Z)1jv2{`GvuiA=fyD7x+#%tM{?6pI!Exv^RFBKl-Ifp!5s8 z4L#(2BIhv)^DsEjpZraq=K_Cga#{}>eaPSZ`BCz6;X*I?Jk0}C&)<2czrla9j=2#B z<8Z0`wp{QN`lXS>zR*qDA1nEqrv>DCopHHT`Wt?IU~R_@M`i`dJR`n2XCEjz_&n6~A{PoYBgbEfMj(^~EIJ|Ny4dB4uz zd4~`DjM}cot;%P&UG{zaz~qTT;DLCnUA&6?Qk=XW(mWR2{qcEo@vQyPuADc;GuMqn zs5f%N_shcP!j9X;>A%oxw_dJ~SkHOqgm^N1(C5Ig?_KsPk&1(aXk9Ns2&4VUcqDdgiAqk9)SFOix}w7s(l8PX$85<4|q4< zcG8NMV2oZ5K>vDmdWXt*TpGXNGeP}+!Yd!KHXnw6#^F-o9Y`9%8sFl2_WfY5F!~-E z{QIP!d5Qe+cZts#oOVSVosUP6`E&6DZnK~4>yU(x?ECGvooU`?JT6V#z@73zp35^| z&GURc`+DQRM%u3vKCjE1eJ=c5St0e3BF8{=?k@&&w{rDIxoOlmn@$)O)D?qca*;f?7zSk{wY&X2oa12=ZXwt=>B zy5z9^J^GsTqXPbS9&_3k@lPD5_lxjT<}SUzQoLKFq*@$P^wSO`UVgF zUyP=GxvL&L-hY$pEO%aj^Ew9I4|w3*EKTb$<^Op9$||40o9q(>a>k?5yM17HeSet{ z#x>$faG|}_Q@k#Y*roPaz)x4S`48>--=Xs|g9m?~^Cy6vkps_nt=wO^^9%bqfr$cT zzk_|zq`4&@%2QO%oAC#nqVcj>d7Rx$_a$t{rLzB{_;)qlz~kZ56=FQGe`h@f|D}V2 zg&x1Jbowv!s>ITNuM2(-6lpH^oDi53Qbh5Tb zqyyRycpvVf))5LXFh)Ayu|F=A-rC5Sm0!e>dv`Kn?;0l_c=>O~MtdkfrCR3=mi-NW zuBQ)a|BiT%G##aIA%D>`n!<&AX%hQ7=kua;N*_FYo_+#8$Hk?JV?m|mLXY2ZI`c5_ zN=7|k<-0tU`p&T*E{#9LF(TJBg#-McVuzF*`g`}&`iJm%9}>sl62BRA?!kV!Gz}C>H$T>_mSh` zQu&SJ`a4g+sj#@N>L-xbJg?Tt9=LmYdGu?SKH8(tt?eov$n&?J2RYkuY2tw1fq&Ad zJVD-mqCVe59LdtY6uA7pF}}ftUfX)Ko?yRJ>iM<$HQM(m`{WNpcTQX}JoO9Z$LVk2 z^QlQq#ObMrxF2r_7bN==?9Y`?vCSr5n3wvo;SZd_ahF+fBG0j(Thxt6e}~NA!0(na z^mkJ|c|-5(wm9f_q{rjQ`R+llcz^o78b9Fs`)og9&-)d9?vgdyS;R$t{HgTX7w6q{ zPvrsrvcH$_fpvS;i1Cl`W99|VJOKB@=Wf>i{98`@xqQdCVFP5l0;g^Z{q7(6%U#^- zy#efYCz%F3&6EFI<-MryTIC;p8m^(=l_~vf8A2=^c()!il5PtY~0*BjMzat#G zs<0S0+^=+dHA|k|n)l${*{t%e^OFl))4ql$@;Pjn$GIuzlS^ecvhyI^PZ~MmE>P9p zuDy`H&xzKj=ree-ru&SNDhf=j8bV_b+!lgKw^GIqbOX z$MAsuixc`B4f3mXo2YoeJIxdQt`hQBg@3a5Ll3`2$K46AwR@xdU#;@Q|7?Ps@wim+ z3>o|5d+U5T>AsomximP4Cs7*vd&bjQDu#;3%FFbQ$KWEbMMlW)KJQPQ{sg@V;pKNb z2Jwt-@W6B9KKXvut6!9CHdy5qem?$g-xus(n^lZ2r(eQvn_KpM%e5_;|4;WQeVVpL zzcF!v*U7Z{9S!6e^U%G7($DIjL-O@#*R?@qrJllBbwjS-dEcwoWxk8%Sm(vyhi!uX z>(?IsbAJ9P>q(F2AFuZ7RwAx7Pg|cnQSFNM+1q*Z!ABA4CsP=n`ze>sX z{(R4&$7hGK@;uJ$&s7{X*W25z{W*P)i+tRgA#r(~p!wGwPU{+qkNc12c`|k^)o8g3Wwz7>72vp1Alk?kme=m6`3o~IZVG%c=gNabsYSkiqiqSxIgGp z;WSv3QpEut>0|#9Wcwb4d$`joe^e<|u+#^i-PI;3eaKtv@cMUx;J?(R%)kqQf93@t zorl8Tszr~jJf^g0BmJ2C!0ysOIsfr_p8h*k1=7*|H1kzB`+FUSf2!woi_$}uzxR5KWDpq*TM6Q^LO~cC;fVw*O`a%|9emQ4vp6x&b$JgwuNb4Q1-`j z(EAR?=hEoGzTdKZ7++k^rO`uN*9ON_d53+A=AV%drQbBO{XImu`ESuqMjtp4O3^); zk;89{i1v38rhgfI;0&6RUGW6YyPW#pVS)bLV{}f+aeI}<5AZ6*PGgND6e{0{$aUynqNMfV(R$ED&`sI&cj zN$rGkJ>u1WW)4XjLjOTLXMQWD+7bC0k*|f_-o+O<+q2O5mx%-NM5FaNA><=FR0#gj zeEcBxbDVD*Jn(9~i_S5ahc1=AAHNqca`^uiMW2TN|AimD^I%+M_xG--;BDlvOI|FA zjEC*HRQh3~q_^T6I+sV{7(4Jz9-Yo<8JA0=2R=P-Qr@|qOQlzOsy_FHpS6AL?`?#n{E+aWSEn)UgH0Te z*PJNNuXx|-ykCUg=5F$wjn|v{dpqXDE6LZD7sUHIZLy9``@J|D1hM-rqR+1&%vsRe$kI zPKjsg5y!c?DMY>75A@n5K7@QK`)E_b2g^Kw*Q|~{N)DW(v*`ZE;6Oh?w1`&y^~LF4 zSLs)}mKORPHJ{RaG_EelBxc!hsy7$oyT_Wn9f=x2?1 z$EqK%XKcZX=WS;^0QYrub&jI);`QCz9JQL&Z_G7ct@4Ziez(eZTqJoY{eEn}K;a+{ ze&y}&3C7K!dvwC*zr)V+wpV=3AKXyi8TcvNsuKJ&U(>&k&$sn+$$Hk~(&z(kWWS{W zw)W27pmd;3p5fwo=rf)!75>`M*{wJcE6}{6^jj~r-_vEdxkdU_c5hpr74cehMZAn0_JcC~EXJ`D5A2e4p!+Rj2YI4mou!?J-+HH) zSo`_auC3Mj1$=ItXl;GZQ=vG$10X-4{r9p9HD2Y>UWlXQfE%b+@l}jyzrS+kaq#{0 z;1uHH_^EW(rO-?FyS|@+JX86n9{f<>c0aAJ4L``|?4#eo89Czlurr=@JigreUiwjd zopuL)`9ZYr2thAp+wjUCcue}N_ff#T)lQBNo*&J=R>jk7V;n2~o?&z@rugx^pyL_& z&3-OcF$LY*DI9H|tsJd?3_rvZ6hq&KLEhFUhT;u*tG@C+j^lTg%1_en_I>o4@N^ES z`SX7p9JEW%j47@Cy~gWNLALKf%;?2~|C77$5YNh8U9EN{H#}2v_&+@9q>=+C?=*VH zOMbW=^}ml4VctI2GaoKhod1k2;K3U_#_q}>>j}sGcYL9LBhzW?d^Mt}pA|>~cbIoDK(;UB^ViR_%9m7_4mtDX(&QPuzPw!! zD!pRq;>tWJoNO`b2HN-~czw;v@6{mn9VULx-`@qlY~e3i`SIT`Gvm(iL0+Q9(C1~4 z=YN|PaVY&?_}@8D^0tYqdc+O91Uc#cRoQ(UKyuDImqriqP5v#l#CPezPwe@bL!~z- zrQRRG{^1t;`F7McsY1X7&(hCs1liiR;OS_vH*vw=ip>cGF3$(fdpzjnSuf9Hd0(#k z!B6Y;c`kTW-F_atIB#4UKF}NKx67)h*Ppiwzp}=8>f~Fiy`Cq|m6Y-5`kQUvhYF`r zRlWa39Ib|AwdQ+oc)i|*{-NsXoZcfpYfp9%?d2X0r}ahA=$(KoODUxpZg7$CLh2%f2DD#^rFYhiM%QM zEnDdQr;$^C@F^1f-}rO+rFKZI_koaa2&`uH^Yq3N^b6w${Hvu=@65E}35AXIs&yv6?ak9s$*sQ7|MllBK>ztueWzZIW3CEr=^;pfLm7b>1t`;~^C z!tEVE=N~S=$nWsnx}6}Oe4Ey-3hziadWTDRJbv|e;Cw!*$Mc21!t42*-fu8JE{z`e zeArVxgghYsZ(Gs0F!qqwyVp3_-d}+qz1?Efo{-n$( z9z`2<06(thQpJ&=Vtw%SsQ;qr~e|CvL=X zAahogXV}l0F-hW+pJa+(#@woZk)E^efuG{NYFhnt`MIq*!oK8RGpI&amu zmm)~EKlo;urPlk3&(BeYDm(D>PrMp(;+Kg^XC4Ip%JUxUALVCh93O#a^6L@5|8Mot z-UbK$GVI(OV2d;MOkb53=pX8p$)0ZyzwLfGr0|iClmm8HAoIIp~%!{dq-JyrKtag7UsA82DinYxuueAHI zRqo%4&cBE^zaw<^#mLvh_{4+s_&cl537Y;%=aU|JsO%%hE#sJesN(nAM(1`OcHvYjrKpLt_rtNwueH|e1JA@Fxyy<=Uk_r9;!Wx#1SaE&$p^UA)o@^r>G z?Ca*Ibp^%2?;Z5K7{z~@6;JC8TVwzGdGi-N39yap6SwJo9{G2*8|PQ$wMQoVdDNjU zkKu>yxK#F+`rGff_D$IQ!|!nSyz(4}|J@+x{FeH4IjQIFhtPYyKCJ2=kcVqe_nL}# zjdS!qisRCy(jWgMyTV7iCoP#$$>I0TD0xn1c(Wga5B*W9aVpQjeUq!`2Tzq__^bir4Xjyc{i`InZW*CEXe|?@%6Z z&9km2=09tr=3nTKEl^j^3!Z!scdx?szx(iFHm&2eJ^#1KH*hj{p?920-tt#EclWSc zn$>ZLWQv5^+T~)y|Z^aw&kX^0zTUN9kA9kC>X%YS(+6$N7q@9l-a^j$zig3s<7^ z6yq2A%kRXn`c1Yrq2x>cobNf}Ez(2oUwIt+89&e~yhHz97xE^_Z-Ou3$DxvOoc;s- z64#rm_z=gy4D(fcKwfxm9f9xN-oTG@H>$#i+wABhvGXn zq4oowhfC1AEVk37!GZqoCtIRFu%4+0?-!-oA}>Y`yXBwhzRJj<-(hGP;3@gMhpj#Q z0PoH)uXhx{y&uhEe{1{z=fg|=-T?MVdn83ZmHw>zbWTR`^SfSdAEzGxr}Gk;R}@a4 zYxJ&yaCjY{=l{ofV~IR)KT4?Z*5sl4B3HX0&SCY{{gkplF@w&Lj2!$=9BC;0n|a6R z0YBlxPJyETdGII{{U2-ps5oyFTPMa|FuvLmX4)Bvf+v z&FL%88#LeW{a%T<`0xGcb!X#4r-Yuy*}Ew<{8L%J*bm{Uv`?xxeIj+DreQZ zS^51E_#Se~Ujizfaf0@rI{Tr}WB+>o2(!DD@&o^I=QI-brd>VSZOHh7q1r#5`*%iw ztskGsUlMZ0=Tfypw}VZr{k?6OhSoT`S5f;jg}1L_Nbd-ck1p+KUN`p0bHuo6UFX4j z=QLj{?^!8UTjSRE?8_U=^B;bv$n&2bm&I$=vHJTfPXNY?i39O98Fkja?^b%RXV-?k zD^BENNzp>q_6>Ya=cEd6L;WaLK2f{U-&Y|&ypGX-x3p}VBWQQhvF@Ly-wTT##Ze0}b7^Md{F-E=HU=LafI z@A(CMho+=+M#Bg4-c{=&Z%Q6^8m)tk9Q?P15Ax8yU8`44*U6DP6iIlQP}r_=YHu#Y=K_8*Q*`#Co? zBt7_vQ^{B55%SEf=v|(PAMzS!GFbU*|7nVB5#xmWpA#qIi8R&zd(FG!t_gd7m!kdb zEWJF0P>*Slda?Q;hcr|zNq-K7$>t6gthuPU5~yqYChBZ z#F$IzoW;`*;+b|Yjy1nEtF)2%Rr!g#POWFzkJ9@+Y;kKGF%HnbkKtjLqcz?87(C=} zYRvcw5Ax)rV?wU-qWf>ZNhIR zPk%&L`r!4YV)S6y|A6yo7Oe*j4)g~_&uZ0ga3Q4k6|gT}NZl7xeERQiWKT)&f{cIg z`n}E^F<+Q*sq}^)miO0gJ@Dz3iPqoB{?liD9|Zr+SJU@~c=Ni-nXjOiq>Nm5^ZAXl z?uOpc>thj*@ef|l((Dj%vb`Rc;qPV?dL~ZpH(V*xG&hNVz{Vc*Y*Qf18mG z{a(p)oiC>P8RhL$TRw>dst;ReE{+Iz>SwbE^xVCmkMupKl|T@*^pP?&w}6f$)(iz zgnZ7rPLQ)dtLsnbv_G{ujQG=SG~zWUSAn;RS!irViO5 z=R?SAPW%P!uJmWN?jiew3m5SQehsqTTMz0-{n_9`zhLwDR(~Ul$iIK0^Hnp_CG;!e zVe;z1t6#%O&^P`O?}NIjWL&B~K?~YfFfNrTm0r%2Z>;$HyXKYnWUtp%t+rMa@q{!E zoLIz-LtveLe)9&yQWR#`)VMzo7r}5(4gG z^>=-B{>uys?e`LVjt}>TUlS+#!L>SztmC^zcsg%T`XiHj><5)z<)_b-Kk%rUkKW@b z{Sh~5UCVhf^{{Jp-r399Q{){h>VaJp<3vy8ZKXI%fCdk-wM)i>mQ}d`;>@_vI?yttmH( z@2=w?cA1{hzE{}|&Z@tY(BD^jV}PySJHHQy-u3VDeE^@2IC{v_W#4d!$0OdmW99zc zs~^-Zo=4>c_@`Zerw#mkHtBUZ^ph5&a|GhU{ljT@=tZyTBY1QBI`P3z_oyNKJrR$% zhrAr1`W5hMEzcJ$=VQn}-6&_3=l@h6a`vN3)$V0C()pi~KkP;C4V64lpSPKM{dSgGIbl#BzH+p9} zA7tKCI{gKD%eEi3`stiio+ENUa>gm*x;6I;>=~c^nRuW#ekGj?D0$X|G>^ON!0+x+ z^>+Z+kHLXnav%HpZQAs+f}eYwf?w<+dxaj~&pY{o-ivnn-8THcYu_ip*8h)H&4KTq zM|>;hw}2hjbE)FX|Llaq2cK%4XGwcwhx)duOj-s92{(#%gSNj8xcw4gt%W(qz7V#cf{~p|d-s*iwNcsOq z?+f6!MDc1t(l6vq+x>Xob@xKB)voRMg^&}U^vuh&Bk&96lLDbo#$Qm_=SFivb4Wf{>zTC-!JT6P(t$Z$kXHN z?+`E5^Zh=*4|Dn>csxEp>r};K(!ae_-jL7h{g+$i%_?jceoea?T=+|o%6|W9^j&em zF3R|%fwp<$-G?0kHhI2FepdO$@gwDa$K?-kMocmS_O5#HIW_6LtS3FaPi-{fqv8Sk zXg$5w3-Euu_EUk+=lS|O-sx#^YhK0{mis2|kG!AO{h*+4Ti|fKT&ni_CvHRK5Apo+ zZ=qn>4v-(O(HU~a<5JT$^-<^okpYa2`cZuk`$2_=H zdj1zTTm6pD_y}^ci$|riuLJML8R)#9@u}49ckx02Yd#Mz$!hhVx$sEg-*wK({!BdJ zv$#*e0GUU~XP=J-Iook*?2)%8RrU$Ly8jftK>K-P2mJeC_gV21DZLdBvLFxSpZlr# z4jFy$dH?4E6%XW(=9fnM8-4JoUM+%0zd(J}l9v@8)tH7m)WU ztjDF|*?a~4zR6V&ywcgOseD6zCk*W~3?Ag&7jCuc|J8H_@~_{OAE@yGc~E-J zct_s1lx`yYnst@Q6ZOM#Ui&T7Cz%jl;wbxZ-RK>Gk;8w!@Z+rQRd?Vf!JqTt@Q2?z zk2Y`ruivORy}koKAITPI>u29T#tINPh9~?CUK9Ak`r|S6_W*Xg<7|hW!9&~)X5Nu_ ziocFCi%&Ym8#o;&%5@u$_t=c5{Fcow@5vlLz)QR=2K=x+mj(xM{o>m$Nahjt--2rX zupfMxLhWksV3#}EK`Y+aNVHE<`bX!7bbbq-1^&FKa3QZWR`yrEPl#sxKrh0KQ?Tbe zQ0a_I=9G6$A_@8)6`w}G&%KF&Kzf0yQ7*|Fg{&&u`pKt2@{LhKs z3ZA?#clLk4D;u0ej$h?3V{Lg}#NRjVZ*hSif2W5oi0a`7`EOr&k>G3kq2U9)Ik($y~V{h!2nt7m#K3y zz`hPZ+yftu`C)ymg!e=}pXWK}|G=4-Gm7E`|LIoP|K3KMV{*U2?XSm0!RTp$OZnh; zb$T9JdpVZ!4_;e}6EDsumCoM<0?$|j>EF9hdhY&0alN8*chx>EO40eFlK=9wV6bhT zkMKmDx3Yg_zjlNDymoYS`#DgtS@fMZILJquQgWTZ@wxL4{a7*e`X2HsooK#ezFZoA z$W!5la$ezcIQ{$EVK43boRN#dsk{L<{w;Z)!RJ7F{;2mG*)u;Xb^AZv6c&Dzp6-w3 zciMjk8h+JI%rh$f6o=nf<#Q^_dDQWP{N4<2ZzrwS_|!{<5Bmy#`HOhCef2y0!8zo4 z9`^@Rs(3o(k@wSXJ@{Y0jn;3*9`Y~G8(Q1**N7q9izu90r*nz;oV=ia)UTC1P>wgq zKfRu-)(4RH`>n_pn||un^Y76h zf7hDHfB%&8ulxPplGRoGuuphZ-jD0N#oj~vvJm3^UcWo=6<>{)_<){=kcW{0A637F zymo~O9_@w{5kr-owOsnX73sV{#;D0D`&WiJHz5RZ# z-UDhc&L5Ws7yLFalK0HKUU2h({+{^yyc6>FDZ>ZY#$(e;a};mrA6Y}+1H%V#PQ1JX z`plP0rT606_7L>or}uJytN+o#^v=GrQ9IP1%1l~}Kf!LwzW zh{BH-euY}$3|5QA*lfglLFHPAPB=Z8k1IFg`n6FU(+k9H* zDZIeC&jdcl zqxVH`GI;&o5Pp6wl1BLPnnyZ(q! zz+c5(lT`ac-uZgmAlV-v-^=SHsQ@kR^l<&w)9$YwxZ`MHjdC-gEVGxJGhyJHt_V?#mt3{G=yZj-}oMq)XJ+IH4 zc@}z)s>pS(iNo-L-ttYfPgU~j#Zp@NuS+ZE8Ixya->kX)+*JR5x=#T*$2a!K)65;~ zM0@i6k&bgq*6H?n#o!^%@%|H_r{o25(YXxgTbD*3yox8a&lj6Z+viFDG4E7A2cNyQ z?eD%;+}Wt~DIYiPL$3Mpe;b~_Z=CIx6{qLQ*8+#XXL_7R$p6x+r}OVSXnP=6zR8=2& zp#7f3YrLiCpx=!k-WA0&s(e9S@vO%>h~qVUpx-y4ece4QiGI%v{VWUU`(bd9uZbJK zi1z1k?(Q$tu9;KGJTgBn75>eaKfYf$KV`JFU$u@)`$2<;IP#yY8vI{>^YFK^TYA(R zJMgYHEVHz8*#kex-6KJ=KSN%=Ih|Xu9hK>npPDNk!k+3mpH4po-qbenRovj&y+l@F z@7STfLz)5hd60P-KJe3PODUBP$S-BD9jbgp$X69SUHIVB_D|YJD!-M>(6}~o=-m*R5t;3w5&TAwLD^B&7}8=t%B{ylEPQG5SZcn>qs?+RV{1>aMnvLg>% zuSl2Qe=y_4j$>47onw@11V<1F^Eob~FB{JxXOz{TMvnMx(!rrS#98r}YN& zQ}v(jeG~cN-`{EeF>>3_t(u&d-&;N}_L_(0&MXi9E>e^8aF&h&RdYTYe(Zz>1PWlsm3|trJbzr+vZI-lol zeO@;%4(%rxpGy7iVC938DlYhmwJxUO4|&R*aRh(%@AO0HjcI1zU$H*-UuWJSzdq}% z?K1076~zZ_KN>nP5HevcS0h1#bnKaZx-J6mcm&V#$%5clwa z`RI=Z2l8^)==Uf}zCUX*kMV{$I((Ju7$;8PemwT$eMsIVYJZ^o&mSb`HI7TqKj*j5 z`rp`tU*(%LFBv&_#m;d)NRBt?7Yxkg5ijfx){^fa_}@En&g+2_{jNQq6|U3oIe2_> zY52kZuUye350@VJe9LA(2dml5t6hL!I9BN1ef+!by2=y$P8oa`a^l73;ZD0iFWRPS zLXZ2o)*Eupeh>4oy!?ABPX9t2^~%xrjPt{#%ID_hK`IW|Kgdk{j6Qhx4%{txbH8K$ zn#Y1}O%*7c#?;Ms5atN__xp`Y#Flu94+QT2~P-^0(BhID?# z^<1hv%$`oallQ2<``P0>SK;&<+r)zp;>&%*{$09}=(siRsg>xQka_7+`7a!c&asRf zeh0<-D0uO`n*KY|K?!mp&*Uejr$6BOt!Sj?S;(_BeH~?axD~fr*@J9Q7 z2wMtrPae>lx`xhWSkK^qPmhQCoeAVEGy4gAJ_mQcPtZG6!T#>S*$;q6xwQ84!Nzgh z;P)Jc7xd@!vcF^aw_j?(i^rw*mw#wq$YYBBG_R`odA@e$G3Y06e7UIXpWw5w z-%RO8;W;?ak5x|p{T0XyN8J)&!!LX*Kje&S;(%SipuRt>Pcm|(tasrczGqqNzoWah zV}rCe{$aQ0`$NBSc97+!TnsMy@n=IDz8uWX- z@!(~6fPdK`bg#ns;?n2=FU)wIH`u?Z;{(2A=kCw2-(Ew$=iqa0E}i~E{#wyHJH?}L zQ~U4ArdOtO1*N~MES=YLT#BdmJ0%sJTd=;u^N*NJ#LMlh*E63F=TYs5e7$X7P1u|M z;qnLljaBBU-{&Ad!8s}f+4|4Yk?Df}lY8LJ&O_&SDy~ECz1HdAk=x^4M1>{tzA_E@ z)$8H=z2$l;q_~4J=zCo72-wq9#^;I?`5AxCzFz9Fs6OU5w$r7`%Wog%AcQu6 zTsogp@(Q<~TIbO^_3PRF8+-6?Tg%VdfBvbtM$JRef4MFb3Ez_TarYR#p`!)FZwn6Fv4X%HQ7`Ygyy_`K|uG!0(MYxusv?$?=$W@M!0QS0@C>aRuDInU<)2 zgE&@2_79f&kWb5%7IK##;H@1_`wJCs+D>$S%XVBEJ>XZ}WB*<3`Q`GQ*E?SDXglVu z>VJ?2jLl$;t3^<72;%_uDYM)6RW};Qa}(Z2n)Xp~R%#o@%A?B#oj>Gdz^zy!?_Rh| zKFRYRo}YC;Jvf@a+lCk7yEOB*$OGH!^Vcf*76?6V5B8(7Rc!AsWyPh{853@Xmv{(6fJ5Aqr!7kZI{ry5amG4P%557J%Y5zp^ zUf<34lOF}h`~d&2L3EzN_>7}?cg*(|_EgXJ!_L18fjD+wPl$G4Jc9#&U+&Yq%Jp0- zy>$_bs`iBa-xnf#;KP5)683qG`;*g-z`OnZtjY^;{0h_krr`zohpG#dK5*8(OBW>b z3;m9(O9V?f^e;zK^Oy01`nzZ89D(!0rP4bxsJGQ`-eu~1n(VWB;54cpSAN%BdWa`X z*BywT>vd`PK|gV}=7^v5Og-Y9GF#t^!0rbByKP1v^5F$9sr-}t+mH9<(ZZ{9PU53~ zzxa3W%Q8OYKl3Ab-|qgdBd%W~eNa3h-pk>Z^wjq} z;9oz3$9moPp*ZKzc`)Jg-;3k-^iKbP{=o*c&r>)V{L`rQ8R9sx-~h(I(r^7;e!uzs zg}YzE@4E%#fzNs1QsIVKo>H|N?DOCKY8^Mbezl)dIsYyU;&|VH?g7{@l?nO36_0bt znt}({bE)+5OjYk1J@m3vI;i-ASF|>nRlJZFE2qCd$V0h(Q&l@bp6ARr)qjEWt@uGD zhyKqA%`yTKqZnk&k75LkhJfwR|;7vXlr1Av&hL^HJ z&iVB!Rr~z$jD9zy^_5=RQMBJOa_|Y-YJZnossPRRMjt#kWYy=^z=^TtxizlGQEB}} z{LFX5lP~Z-m)GaNkRRmVWf@+OulsgS+}c2c_ll2PmCYPhX&Gl zatQnfPmLYI_yX?(!S=t)+U}CvM?3srw=lQ-j=8@d;G6!U+DEH+YOmDaQ{ex0CYzPN zvoC>O^|CadseEX?CC5uy`$wa>HPrk7-U&`-xB546vOg8i%wr?eI0oLwDRj?g^uhn_ z2s#&MJT8qM?5BQOA0)q%u#Zu%Lhyfb54#*+-zhtaFHH>C89%`L`jYl}j7Ozjug%&` z^B&dnK3VsZT(9-M5&lnQrQd;*J&zY>9R$6i*Xi7j^vw8Gyt_A}b8aJt{hOmJ5I@;_ zw;$pU9`s7xcLJx{I2tF6qx|gI+ri#nj2`eS9@O_oz`s*Lug4+Zkci&98XU;`CZ=__ zl6PMs&&f=@Mh`riA3u!oK=JVR$+;JR-h-L;^+KkS--SK5zw`Gh!E;Y6HU2&F*SaXp zSB59-SB2N}FyzlVM78FdrGabY$V0bAbe_j{x-|OWH?m2f z6=!U|@oIhmexHli)HsFQXJ@`(+qjSOoc6b#JYb(Evp)ZWyxette*^j5Np!BJaNX-p z=vS)xOz}qi7Y@jKZ(cV$;~Vx{d>_MJ`P086o$-Y3$yJ^U_oR7D**onIJ~8{~^MA+7Mp{4ib&KJ-%z>W1~E2VUK=v0=ycTq>NVb@o{C zn|7IRmG3%PHdMIn%g}um`{h#k-@l*6qmmD*LgzVT$A8z+`FEhv4iB$H6yurqSI#;Y z`TFx3?TcMJfYW0!y+cquQbjH(@Llcb!h^q{_;fF2@ZhIOhD||oTtYr`P((FeAy4(r zV;{qQxm14so^RjRbdQu)`2pWVPckYw{O>su6LR*$rSV7hIcc53^;{Y~;KgX2QO#qJ zw;x3JP(~lP@6+jX1jx@-r29JZlZi_Ad;|O_3)FeF@q_wmqw}L(JaOQ+=*eMfT)_U9 zgz@a-!j%W$EE=Qtk&xG4K<9OaALJ8`F17ki;&<-<^(X z;^*`~;O1LH=LRMZke}OLQ@zuI{PxDu9_^3%Rn5Lw`*Y3AXO$fK%eN&0p7D=%`!z;% z>4)3R=)t~wZuxsIKBsrqOTa5QEUfYm`Kt6gA!mFl8As=9OSgv#2Rz#S{=zDcv;B?0 zH}in2J;0}Zvcy)LS7GeuhCDB5p4IBx$IW5kVA55) zzS27rNAKSe|Gnj3gZ|^kew1D0WDgZj*qs}7C{%vm=A!#9!sqXxKA-QDT#pO*8Pihl zyCE-GR-Q}oyzh)p>L+<=eL((s9^%qzN8oO1Q&XLNXB-WFhCtaGnLBCFhMshr< z`hNrUxCP%@u{(I!p}y|IAI}H(YwF+Iz;1eSde@o}Tvezx{J*Jwkr?JEQw$*GO{SG;w(F+cd9&b^p_N$QaQMjvuts zjo&KBdJp^oO-lya=Cu`biwB7LgzPh`_RXKu{*GCX-&9TP=i>EN&^rl*TkdW-w4;)j zf7?*W(O%E%d+fW7J@n3R-ENJ`$shF6=cMnCvLAg>{tjE)2 zaL#GrH-?}6d~E#Fz2HOkydHG&i@aQjC(rSC{&D&n{B)RvVL-JF0b4tbjL;Em)Avcm0nu)@B8xi zTAz2Otw#G_=Fg?V>)tAbH9t=(>GL?m9qFun-LBiS#=1DbWxre+|G-PyK)nld=^;ON zC$>X=l>PCYpOJ4Rk8-_-N56*O+Yxf0UfDfohs*>OT}mOQu+IBo<~BOC$6k$9%!2E&|~7oWTM8$ICGt&kA@h#&V`(@sVoxbfE0zKHXs zOQR3`Z2wmGXcyo;4Xe*BVD~u*-Qy^{^j$LpPsxkimG5($@eMy2yFL%Ht+O0G_>Gi1 zod;jk|1me5@(+Cf5p*x7+Bv4;d1T{z!Grf<&i(;-(e7`Cp0aOpVU^l%K|k#nnm@Tc zl-;Gphpc|}{czE}bneACTq?X1Hwy;X#%JoX6NDd~-}K%6z@Krc)boztsvw1r{I4Fg z3Gxu^8?AXM^mzZ}_yJz(K_hU^$ap4B_wPFXB%272{c&mh zf>)NP4-`J+?UH%?-bdMQ?=Ii7d(AibXVZP2vTuF+mGY0ccNUcAeC~MyJRj{*d2q!E zyqrU5yi`u`!aMeR`(OT+_tEZrA|I7E%JT$nPiNf^y|pDZFYx^{sywgceS=<~ z1iq2?1MJ7iC;VK=lEZp_aOz!4l^6K$uxmKv+z+TU@v_|H4R(Y5cFFm|)PqO!2BodI z4qxDu&S-xZ)-djUYy0NNq0R}7U*ILpR6z7Qcb=ekGn(GlfcN)@dVE7(fAZac|LxCS zxPucN4M7ih^M>Alp5otTD2-1ehkmjXa^KG5&Wp#FUHUx};!fR5-hVQV^PPjwx2J|en`_-k<2ai~7YOD5# zzxwy9SmoDZ*w<&h;=d93H}M;Pz|D}?er^=jH?`=m?)C%UFnu#w^;qA_Z@{_$6%X(> zjhtz1=lby~W1Xt_x1SpvB-bIpDRV1Yu#_X-uo16V^`HHf5BjQoc;0sAVc-m{XTL8k z_dL14;XI^fT*II8H$R-3pCLcK+E3s({!!m`Z%FGD=tbXD26S5>epK&jRDI8m zIRa%~$xl^UuPXgYz3t}}(~s=7_D|0oJ9x0$ zK0KAogDXDh<;|=8BJUG>%W>w#yQ9BcFPL{5CXeuw+kcR1cgQbo`0@EfEdG0e#Glt0 zrrnGm@E%%c1N^w+MSZ|l`@6Yek>+AN89eYy*JCu~EL1d+8+zq-(R$m316~tS zsr3ZuIp;9I+4?kVplzP%`HOy^2>n%s=>CcQa%tj(|8G;$0+;GjQt57Y;5Hp6zi;mL zfZpUVjjVZnwukPs3=ilJKi43Heha>3%Fua|v4=eKRrS46^{>9$`+Jd__W7jOYuZ0> z-ndk_!y3zfXO{OV`tKM1TvGd|xLduJ{no?Jfq->FF9GXusd!ax*;eM$)Wh$9f%Lvn z$+y+k>pSFkfWM#Wf52(tYdXXwg1@=>XYzR!ex_`xJ+J1V)qxW^If<_w|l znrSD<%l}LF@e0q08+`U;x4)Y>((fwzlfq4Xh1NSp4xFC5lc6_FD8zYCGHhZ!Jgr>dj z=9A|&4qxDpOP3|^e{uNx@94qb{wI3e!oE(B{e8sen|eNhez-otA@GL3c`?EZUZ#Cb z9)Nddle`bmyh~J~a|db%UVrHG(GT78z@NhRIZnTa;CAHF=z;IStz^&jTq?abX|tpM z7&+otzLL(NJnWvlDhxZW=hEPSN9{GVo-=ab|2@op-<)tHt&f#{^EJMq^0zPttuu{1 zc>LUk*1aA){oB)if$Di4;Qfhi$KwapdffO!e0Qtw31K{eSDWc`)xH$=BcJ3|?GJtb zulju(^xI|7>j3C~I7agZ#l_zReXe#s%L~Dm@9A~FOn7I%(9?Md^KTbX&;Go|%ldTJ zt>+Cj%VeGtPFNYW@j0%sR!Sh6X|yqN}eH#{(Ba{ zyPSv4f!H6F&N(k|R~Gl+Y5V~1Ox5QBV*D_!(p#RrHpYRH|GY`>2jG8h@OR)EeaNFN z86^Fgdhkiv>!U|K?4NFz_gcJuaOOMUG;V2s&oZWsJ>SKv(LBNNb7}a&f69C@#C*Wx z$ys;8|MqD1dFR2%=)#`+ziz+I3*|W)>#;xnZ#@rnp3o=gzxGtTA0OzT+5APneBswq5B?HW_#|-^&g2Vff3NEo>hCE0 zR|q;T{Tn;@4H_WN1NiskI)BR#+Rwv&zb((FxSibVL+}a6Nd6R$zTc;*{s8^){q?#R zywk;xAlqHx&F=UCc${}r58S2ELbs3Z;Co#0g8y5y?pWoU52|r!{K775jZcCX|2xEb z9&Xpv2k{wu@E+Jco{ZZozRDkkA09`}yaBwHvv&zS-R^ZtEKqU6?rHM%(vEp>X?P>< z;o0=Q4e}m&?EBRuxx-ui4vw{1$$?XQNDeC>j@yX~`d{~C_ZV-eA6SdlPZSTIlkndQ zbMTOl-y(ahKY=&sY&Fa$oF6Vto`6#|ZWib%dEcV?oD2H*GTYAyJM%j+g99FU-`jr& zo^!F>$D8k^v4_7J8Ks_kzJ;G=G3~rsO_ul8+&)}7c?Hg`p7eKu3=hZ?)z|otZ%Zxt zdEs^{ZGVsN8XxaSPx;SvJh{C+UF`~9;lIYT>nq&8=UaR9OXUB2`ZA%ysoq@QXTjcg zg4enQJYQC_pF74bN9P9~@jWb_EBMFjVc_Sun+$p??j%i9TjhDvrd4vpk?>DBkD7ky z!7t@jb#CLKx8O$T_Gt&^*9w&D1Mn)FV~ko)K)zr)&9f$M$lDjy^E~oYcBekifc*J@ zhhm=N{ifIVwsQ2FYQBWu7E!(k9NrJ;e`j-Y(=$TPJ)VFUJM~oM2Ym0&)8|>x51771 z;BxyJd{_Ix?#%k!0ivI&e12(NT(t{$l)03#=#T6oC;cpNc^#CRd2ng`0e4NaFR(Lm z@NC~v%b}n6`ZlExxzCz}N)DWclj!_O;V-$GALGJ7DI zUWo_MJ4(XmeyG0(H=D|Rw9Z46va}BO;CZg!`w;SkcJKNvo3$SoZ9>1>ci|&nXDaJ= zdcgY{+Xr|qJJer^B;S?ueB#6p{MCNyoWTRHeayebRXlU1mFp+n|01VPB5^tITpB#&vt{3M*8G2(UqHzb|G1>8A2BYM3cp&`c6L5W zukWJ@R)33&+ULvt4GUm?GC1J(M{IqL0(sdFxr62R8u8bxPv298^KT)#2j;lARC=G= z(K!g$b7}P8r^Yo}KN&fAWv(FC8$9luc7@)e?DltkjvoB%{YdX9l%Fe?+F9rM%74EJ z65mDU&!xiIxTFsBmHekK^qnWWR8*RAp!A~`q;n!A@AzvQv=jU1Qt3^pS=eJdBM&ED zs`q`$zU8%%(%!?*q84pbzXX20c3WlM3_j|=PKqeLpSqo*wx1&X8+){OndS0ahQHtZ z@5eiM&_CT+;kn`guMLUp>(GcHww*sr;6DR{=PldceKCetHOc;QORRA&fImJm^iTF#Lz(YV3lw0k(O5-K{i` zGroxjJpBf#b(M!+|9G3xKCXK3+juePhknBN$iLr_k#L-K25<@%e~Y{so{&%ZO~2=Z zJom0L7+1`TOJ(0>54~4a@@$#y`|K{WX#c15du~o=_s4#@GOjj2WT#+u*h1P!#LDwC$4pywzzh*l+wNvOOOvj`s)c=c$wP%6&b* ztK)glnU~;iklsDBKQ2|C8oyr&d#>lw=>30#xX7C=JW`q9>HAK2vK)3Ja0d8`&b0(UU)kl!~GC@|J7}^ zMGb$5dCPv_=s&^T4N+H=a2L)05b+&nHlm3`?Av&h&^bL2{WddhLG(l24a;Zf?I+du a*ykIrE2FNXfQ5rRMmRw13okFjo{j)wn{G=0 literal 0 HcmV?d00001 diff --git a/yggdrasil_decision_forests/test_data/prediction/gaussians_anomaly_if_skl.csv b/yggdrasil_decision_forests/test_data/prediction/gaussians_anomaly_if_skl.csv new file mode 100644 index 00000000..14436e18 --- /dev/null +++ b/yggdrasil_decision_forests/test_data/prediction/gaussians_anomaly_if_skl.csv @@ -0,0 +1,280 @@ +4.192874686491115943e-01 +4.414360433426349206e-01 +5.071637878193088200e-01 +4.252762996248650729e-01 +3.864382268322048009e-01 +4.572857985150768356e-01 +3.828214071190141898e-01 +4.229257548829531421e-01 +4.062915153623989362e-01 +3.924027382415538612e-01 +4.204809538715528761e-01 +4.120245158211452985e-01 +4.021714702533157881e-01 +4.116859482405123560e-01 +3.856297713263709404e-01 +3.879418151101039491e-01 +4.254604004573970255e-01 +3.829543100175958337e-01 +3.894370012297819206e-01 +4.767564669226527219e-01 +4.105111972875190030e-01 +4.531210549381821107e-01 +3.874123385442900802e-01 +5.070949713343407828e-01 +4.004866507650391427e-01 +3.887316633629200879e-01 +4.045394382558925028e-01 +3.907274096454862455e-01 +4.443613684156251242e-01 +4.225904764508747746e-01 +4.323633888377847456e-01 +3.872707488263947839e-01 +3.817428594100403805e-01 +4.999718182628443697e-01 +4.541545263864004700e-01 +4.411337054894758913e-01 +4.173526637361775204e-01 +5.025084696934063455e-01 +4.006944572465925836e-01 +4.094431791281276656e-01 +3.820838149099484715e-01 +3.893680505002969361e-01 +3.810040914235358067e-01 +3.855961731361299272e-01 +4.232722352430269286e-01 +4.110354525550297122e-01 +3.918776416664728202e-01 +3.834264091236884164e-01 +3.837920468095222248e-01 +3.933842928691425600e-01 +4.326440460645166741e-01 +4.056551811690080056e-01 +3.895330258052301375e-01 +4.060813389953127350e-01 +3.851454738878740125e-01 +3.950579767455482116e-01 +4.068190790541990620e-01 +4.313255306243298781e-01 +4.776874716748540362e-01 +4.156832554965157467e-01 +3.950591826845297749e-01 +5.301922036262658455e-01 +4.308485980804026561e-01 +4.393885347009985343e-01 +4.116120753344015837e-01 +3.869103658296884629e-01 +3.979117800684048301e-01 +4.020911435926875499e-01 +4.141215529982648635e-01 +3.942074750986535592e-01 +3.818074385912471058e-01 +4.219578719165194558e-01 +3.868267522953542370e-01 +3.987328486697296337e-01 +4.055211053693148493e-01 +5.500677248550085441e-01 +5.420195167261064872e-01 +3.828485420051598864e-01 +3.817567080948217062e-01 +4.183393305586847188e-01 +4.323695489158039251e-01 +3.842818759027541442e-01 +3.844818053615791098e-01 +4.068890989165111494e-01 +4.618555038832681259e-01 +4.018989074713745224e-01 +4.648227216919613713e-01 +3.819814560184485730e-01 +4.103791772911288271e-01 +3.904823673762904956e-01 +3.899555383140163034e-01 +4.574802701165338603e-01 +3.865763499090780209e-01 +3.951694611570931159e-01 +3.855885909007265577e-01 +4.156832554965157467e-01 +4.985406549658428221e-01 +4.402427532823418033e-01 +4.036571176114390203e-01 +4.544473624936938294e-01 +3.975366726865089406e-01 +4.716395580368268337e-01 +5.287268511798952630e-01 +4.842557205309823143e-01 +4.679571513883666323e-01 +4.559195029152853240e-01 +4.034531926450437567e-01 +4.120864592485077860e-01 +3.904682127823561610e-01 +4.172182514027760192e-01 +4.519915549851231118e-01 +4.205610367050501286e-01 +4.871252088471250685e-01 +3.791240419514353421e-01 +3.832566735533416424e-01 +3.839636789324888944e-01 +5.022452297590203063e-01 +4.359862728110341923e-01 +4.420147071002538675e-01 +4.148646563422070388e-01 +3.826592321605687963e-01 +4.266047739387540672e-01 +4.097065722991519165e-01 +3.905689585798612362e-01 +4.875926043809953869e-01 +3.789379435786642270e-01 +5.500418969423366278e-01 +4.096627265516010197e-01 +4.618205485619392703e-01 +3.938384755246451818e-01 +4.345132144143419306e-01 +3.974581163061812195e-01 +4.302496481849614196e-01 +3.882766841015732329e-01 +3.997871035161660314e-01 +4.586997505551287935e-01 +4.118826061235560942e-01 +4.065925103426547005e-01 +4.656338978518183569e-01 +4.423437630145455879e-01 +3.895478346750532728e-01 +4.103814948857281708e-01 +3.817683297876992565e-01 +3.793772698872787119e-01 +3.993730701909981029e-01 +3.773695170984343594e-01 +3.984209767946685909e-01 +3.821866507638494448e-01 +4.008893660492762745e-01 +4.097956829221550690e-01 +5.046304127250391680e-01 +4.048730213871686301e-01 +4.512826335180712412e-01 +3.920315658275624338e-01 +3.940099922559582857e-01 +4.098016210252675706e-01 +4.423806966764076143e-01 +4.946810305106644212e-01 +3.880441037630216750e-01 +3.988136213344566916e-01 +3.928119252560551833e-01 +4.047339195637830178e-01 +4.135616388584226755e-01 +4.297218879483765686e-01 +4.203077591790538858e-01 +4.265634462984130848e-01 +3.948208310202510063e-01 +4.459889025894496206e-01 +5.206044029282200780e-01 +5.579405715391486664e-01 +4.358864550595425991e-01 +3.833844043711691629e-01 +3.786230316520938777e-01 +3.788119550597274232e-01 +4.236605785765598098e-01 +4.226165349745940047e-01 +3.950578738595934003e-01 +3.882292203069177550e-01 +3.817579965315230917e-01 +3.871879442211887401e-01 +4.141477749441478950e-01 +4.429423350433662754e-01 +3.960791336888356828e-01 +4.984501366941139766e-01 +3.883437150295604434e-01 +3.872289451323157605e-01 +3.997820439178182972e-01 +4.487556562092934742e-01 +4.810655203676491021e-01 +4.466108918716661380e-01 +3.966013053412562606e-01 +4.249603405034386716e-01 +3.861806487610026650e-01 +3.788789309952160789e-01 +4.002886743478978326e-01 +4.877364558262433647e-01 +3.885950090060952822e-01 +4.173414050232285843e-01 +4.660352010052796201e-01 +4.719319980905274936e-01 +4.069173143546231941e-01 +4.345190394359730313e-01 +3.780582739823815097e-01 +4.740919913608709901e-01 +4.226080513284103390e-01 +4.677747720413846233e-01 +4.512101286667756228e-01 +5.286989180949901446e-01 +4.275707741626420533e-01 +4.762117730989295916e-01 +4.455416064365435580e-01 +3.889253486766672263e-01 +4.578419212637818148e-01 +3.834804343711281649e-01 +3.858922224640186083e-01 +4.499084362588428587e-01 +3.972063543467886926e-01 +4.129127707066883035e-01 +4.160268102467659657e-01 +4.668341997467896909e-01 +4.473061848655025541e-01 +4.577191352620764708e-01 +3.844943281007760505e-01 +4.252057879779328475e-01 +4.494716604068641486e-01 +4.245835883611878137e-01 +4.023278468500455785e-01 +4.354858123839681072e-01 +4.103481467812403749e-01 +3.866402448320220286e-01 +3.987291363628146512e-01 +4.118935925884376625e-01 +3.779766179680454918e-01 +4.612377027605725210e-01 +4.039052165777324288e-01 +3.946717258624949376e-01 +4.569480237849761206e-01 +3.829179280218348302e-01 +4.894804677721761865e-01 +3.972207981766076035e-01 +6.593193717487798589e-01 +6.192209945328789322e-01 +5.477797224059469672e-01 +6.173453627703995306e-01 +6.275236972956231840e-01 +6.912391238952994010e-01 +6.763233649866716712e-01 +6.404186030149033870e-01 +5.392811212791098763e-01 +7.286598308044932581e-01 +6.388780842505601409e-01 +6.215996766626598058e-01 +5.696388297677966728e-01 +5.491739082858497767e-01 +5.992520118257051998e-01 +6.619900211379498023e-01 +7.164889333567598939e-01 +6.609778150579318501e-01 +6.278282329056331657e-01 +5.797573451130116906e-01 +6.775016449975370669e-01 +6.334292246222487099e-01 +6.332737346764194530e-01 +6.987914858039537824e-01 +6.331578114099671861e-01 +6.204798967766358420e-01 +5.881668787726149761e-01 +6.115757410551400097e-01 +6.433056106218002501e-01 +5.907053136999610432e-01 +4.613760056477577143e-01 +6.222050253446579360e-01 +5.384364226794347008e-01 +6.394062132296666201e-01 +7.105713663527076784e-01 +6.656745655947218232e-01 +5.287150025616496052e-01 +6.780186910018997093e-01 +5.716047211136439099e-01 +5.259055469274873662e-01 From 887b122a3ba0c5cf5acc818cbb3da6ea156b780b Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 17 Jun 2024 06:24:16 -0700 Subject: [PATCH 22/30] Anomaly detection; Isolation forest c++ learner (part 4) PiperOrigin-RevId: 643984861 --- yggdrasil_decision_forests/learner/BUILD | 2 + .../learner/abstract_learner.cc | 10 +- .../learner/abstract_learner.proto | 4 + .../learner/isolation_forest/BUILD | 91 ++ .../isolation_forest/isolation_forest.cc | 459 +++++++++ .../isolation_forest/isolation_forest.h | 115 +++ .../isolation_forest/isolation_forest.proto | 48 + .../isolation_forest/isolation_forest_test.cc | 245 +++++ .../test_data/dataset/mammographic_masses.csv | 962 ++++++++++++++++++ 9 files changed, 1932 insertions(+), 4 deletions(-) create mode 100644 yggdrasil_decision_forests/learner/isolation_forest/BUILD create mode 100644 yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.cc create mode 100644 yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.h create mode 100644 yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.proto create mode 100644 yggdrasil_decision_forests/learner/isolation_forest/isolation_forest_test.cc create mode 100644 yggdrasil_decision_forests/test_data/dataset/mammographic_masses.csv diff --git a/yggdrasil_decision_forests/learner/BUILD b/yggdrasil_decision_forests/learner/BUILD index 40907240..6b8daa6f 100644 --- a/yggdrasil_decision_forests/learner/BUILD +++ b/yggdrasil_decision_forests/learner/BUILD @@ -31,6 +31,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/learner/cart", "//yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees:dgbt", "//yggdrasil_decision_forests/learner/gradient_boosted_trees", + "//yggdrasil_decision_forests/learner/isolation_forest", "//yggdrasil_decision_forests/learner/random_forest", ], ) @@ -74,6 +75,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/utils:fold_generator_cc_proto", "//yggdrasil_decision_forests/utils:hyper_parameters", "//yggdrasil_decision_forests/utils:logging", + "//yggdrasil_decision_forests/utils:random", "//yggdrasil_decision_forests/utils:status_macros", "//yggdrasil_decision_forests/utils:synchronization_primitives", "//yggdrasil_decision_forests/utils:uid", diff --git a/yggdrasil_decision_forests/learner/abstract_learner.cc b/yggdrasil_decision_forests/learner/abstract_learner.cc index cee772f1..41f89b5f 100644 --- a/yggdrasil_decision_forests/learner/abstract_learner.cc +++ b/yggdrasil_decision_forests/learner/abstract_learner.cc @@ -35,6 +35,7 @@ #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "absl/time/clock.h" +#include "absl/time/time.h" #include "absl/types/optional.h" #include "yggdrasil_decision_forests/dataset/data_spec.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" @@ -54,6 +55,7 @@ #include "yggdrasil_decision_forests/utils/fold_generator.h" #include "yggdrasil_decision_forests/utils/hyper_parameters.h" #include "yggdrasil_decision_forests/utils/logging.h" +#include "yggdrasil_decision_forests/utils/random.h" #include "yggdrasil_decision_forests/utils/status_macros.h" #include "yggdrasil_decision_forests/utils/synchronization_primitives.h" #include "yggdrasil_decision_forests/utils/uid.h" @@ -834,7 +836,7 @@ void InitializeModelWithAbstractTrainingConfig( const proto::TrainingConfig& training_config, const proto::TrainingConfigLinking& training_config_linking, AbstractModel* model) { - if (training_config.task() != proto::Task::ANOMALY_DETECTION) { + if (training_config_linking.has_label()) { model->set_label_col_idx(training_config_linking.label()); } @@ -888,11 +890,11 @@ void InitializeModelMetadataWithAbstractTrainingConfig( } absl::Status AbstractLearner::CheckCapabilities() const { - // All the learners require a label. - if (training_config().label().empty()) { + const auto capabilities = Capabilities(); + + if (capabilities.require_label() && training_config().label().empty()) { return absl::InvalidArgumentError("\"label\" field required."); } - const auto capabilities = Capabilities(); // Maximum training duration. if (!capabilities.support_max_training_duration() && diff --git a/yggdrasil_decision_forests/learner/abstract_learner.proto b/yggdrasil_decision_forests/learner/abstract_learner.proto index 44b758ee..a51d008b 100644 --- a/yggdrasil_decision_forests/learner/abstract_learner.proto +++ b/yggdrasil_decision_forests/learner/abstract_learner.proto @@ -263,6 +263,10 @@ message LearnerCapabilities { // If true, the algorithm supports monotonic constraints over numerical // features. optional bool support_monotonic_constraints = 6 [default = false]; + + // If true, the learner requires a label. If false, the learner does not + // require a label. + optional bool require_label = 7 [default = true]; } // Monotonic constraints between model's output and numerical input features. diff --git a/yggdrasil_decision_forests/learner/isolation_forest/BUILD b/yggdrasil_decision_forests/learner/isolation_forest/BUILD new file mode 100644 index 00000000..a100656f --- /dev/null +++ b/yggdrasil_decision_forests/learner/isolation_forest/BUILD @@ -0,0 +1,91 @@ +load("//yggdrasil_decision_forests/utils:compile.bzl", "all_proto_library", "cc_library_ydf") + +package( + default_visibility = ["//visibility:public"], + licenses = ["notice"], +) + +# Library +# ======= + +cc_library_ydf( + name = "isolation_forest", + srcs = ["isolation_forest.cc"], + hdrs = ["isolation_forest.h"], + deps = [ + ":isolation_forest_cc_proto", + "//yggdrasil_decision_forests/dataset:data_spec_cc_proto", + "//yggdrasil_decision_forests/dataset:types", + "//yggdrasil_decision_forests/dataset:vertical_dataset", + "//yggdrasil_decision_forests/learner:abstract_learner", + "//yggdrasil_decision_forests/learner:abstract_learner_cc_proto", + "//yggdrasil_decision_forests/learner/decision_tree:generic_parameters", + "//yggdrasil_decision_forests/learner/decision_tree:training", + "//yggdrasil_decision_forests/metric:metric_cc_proto", + "//yggdrasil_decision_forests/model:abstract_model", + "//yggdrasil_decision_forests/model:abstract_model_cc_proto", + "//yggdrasil_decision_forests/model/decision_tree", + "//yggdrasil_decision_forests/model/decision_tree:decision_tree_cc_proto", + "//yggdrasil_decision_forests/model/isolation_forest", + "//yggdrasil_decision_forests/serving/decision_forest:register_engines", + "//yggdrasil_decision_forests/utils:concurrency", + "//yggdrasil_decision_forests/utils:hyper_parameters", + "//yggdrasil_decision_forests/utils:logging", + "//yggdrasil_decision_forests/utils:random", + "//yggdrasil_decision_forests/utils:status_macros", + "//yggdrasil_decision_forests/utils:synchronization_primitives", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + ], + alwayslink = 1, +) + +# Proto +# ======== + +all_proto_library( + name = "isolation_forest_proto", + srcs = ["isolation_forest.proto"], + deps = [ + "//yggdrasil_decision_forests/learner:abstract_learner_proto", + "//yggdrasil_decision_forests/learner/decision_tree:decision_tree_proto", + ], +) + +# Test +# ======== + +cc_test( + name = "isolation_forest_test", + srcs = ["isolation_forest_test.cc"], + data = ["//yggdrasil_decision_forests/test_data"], + deps = [ + ":isolation_forest", + ":isolation_forest_cc_proto", + "//yggdrasil_decision_forests/dataset:data_spec", + "//yggdrasil_decision_forests/dataset:data_spec_inference", + "//yggdrasil_decision_forests/dataset:vertical_dataset", + "//yggdrasil_decision_forests/learner:abstract_learner_cc_proto", + "//yggdrasil_decision_forests/learner:learner_library", + "//yggdrasil_decision_forests/learner/decision_tree:training", + "//yggdrasil_decision_forests/learner/hyperparameters_optimizer", + "//yggdrasil_decision_forests/metric:metric_cc_proto", + "//yggdrasil_decision_forests/metric:report", + "//yggdrasil_decision_forests/model:abstract_model_cc_proto", + "//yggdrasil_decision_forests/model/decision_tree", + "//yggdrasil_decision_forests/model/isolation_forest", + "//yggdrasil_decision_forests/utils:filesystem", + "//yggdrasil_decision_forests/utils:logging", + "//yggdrasil_decision_forests/utils:random", + "//yggdrasil_decision_forests/utils:test", + "//yggdrasil_decision_forests/utils:test_utils", + "//yggdrasil_decision_forests/utils:testing_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.cc b/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.cc new file mode 100644 index 00000000..9302dd0e --- /dev/null +++ b/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.cc @@ -0,0 +1,459 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/types/optional.h" +#include "yggdrasil_decision_forests/dataset/data_spec.pb.h" +#include "yggdrasil_decision_forests/dataset/types.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset.h" +#include "yggdrasil_decision_forests/learner/abstract_learner.h" +#include "yggdrasil_decision_forests/learner/abstract_learner.pb.h" +#include "yggdrasil_decision_forests/learner/decision_tree/generic_parameters.h" +#include "yggdrasil_decision_forests/learner/decision_tree/training.h" +#include "yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.pb.h" +#include "yggdrasil_decision_forests/metric/metric.pb.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" +#include "yggdrasil_decision_forests/model/abstract_model.pb.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree.pb.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" +#include "yggdrasil_decision_forests/utils/concurrency.h" +#include "yggdrasil_decision_forests/utils/hyper_parameters.h" +#include "yggdrasil_decision_forests/utils/logging.h" +#include "yggdrasil_decision_forests/utils/random.h" +#include "yggdrasil_decision_forests/utils/status_macros.h" +#include "yggdrasil_decision_forests/utils/synchronization_primitives.h" + +namespace yggdrasil_decision_forests::model::isolation_forest { + +namespace { + +using ::yggdrasil_decision_forests::model::isolation_forest::internal:: + Configuration; +using ::yggdrasil_decision_forests::model::isolation_forest::internal:: + DefaultMaximumDepth; +using ::yggdrasil_decision_forests::model::isolation_forest::internal:: + GetNumExamplesPerTrees; + +// Assembles and checks the configuration. +absl::StatusOr BuildConfig( + const IsolationForestLearner& learner, + const dataset::proto::DataSpecification& data_spec, + const UnsignedExampleIdx num_training_examples) { + internal::Configuration config; + + config.training_config = learner.training_config(); + config.if_config = config.training_config.MutableExtension( + isolation_forest::proto::isolation_forest_config); + + RETURN_IF_ERROR(AbstractLearner::LinkTrainingConfig( + config.training_config, data_spec, &config.config_link)); + + if (config.training_config.task() != model::proto::Task::ANOMALY_DETECTION) { + return absl::InvalidArgumentError(absl::StrCat( + "The ISOLATION_FOREST learner does not support the task ", + model::proto::Task_Name(config.training_config.task()), ".")); + } + + decision_tree::SetDefaultHyperParameters( + config.if_config->mutable_decision_tree()); + + if (!config.if_config->decision_tree().has_max_depth()) { + const auto num_examples_per_trees = + GetNumExamplesPerTrees(*config.if_config, num_training_examples); + config.if_config->mutable_decision_tree()->set_max_depth( + DefaultMaximumDepth(num_examples_per_trees)); + } + + if (!config.if_config->decision_tree().has_min_examples()) { + config.if_config->mutable_decision_tree()->set_min_examples(1); + } + + RETURN_IF_ERROR(learner.CheckConfiguration(data_spec, config.training_config, + config.config_link, + learner.deployment())); + + if (config.config_link.has_weight_definition()) { + return absl::InvalidArgumentError( + "Isolation forest does not support weights"); + } + return config; +} + +} // namespace + +namespace internal { + +absl::StatusOr FindSplit( + const Configuration& config, const dataset::VerticalDataset& train_dataset, + const std::vector& selected_examples, + decision_tree::NodeWithChildren* node, utils::RandomEngine* rnd) { + DCHECK_GT(selected_examples.size(), 0); + + // Sample the order in which features are tested. + // TODO: Use cache. + std::vector feature_order = {config.config_link.features().begin(), + config.config_link.features().end()}; + std::shuffle(feature_order.begin(), feature_order.end(), *rnd); + + // Test features one after another. + for (const auto& attribute_idx : feature_order) { + const auto& col_spec = train_dataset.data_spec().columns(attribute_idx); + if (col_spec.type() != dataset::proto::ColumnType::NUMERICAL) { + // TODO: Add support for other types of features. + continue; + } + + const auto na_replacement = col_spec.numerical().mean(); + ASSIGN_OR_RETURN( + const dataset::VerticalDataset::NumericalColumn* value_container, + train_dataset.ColumnWithCastWithStatus< + dataset::VerticalDataset::NumericalColumn>(attribute_idx)); + const auto& values = value_container->values(); + + // Find minimum and maximum value. + float min_value; + float max_value; + UnsignedExampleIdx num_valid_examples = 0; + for (const auto example_idx : selected_examples) { + const auto value = values[example_idx]; + if (std::isnan(value)) { + continue; + } + if (num_valid_examples == 0 || value < min_value) { + min_value = value; + } + if (num_valid_examples == 0 || value > max_value) { + max_value = value; + } + num_valid_examples++; + } + + if (num_valid_examples == 0 || max_value == min_value) { + // Cannot split. + continue; + } + + // Randomly select a threshold in (min_value, max_value). + const float threshold = std::uniform_real_distribution( + std::nextafter(min_value, std::numeric_limits::max()), + max_value)(*rnd); + DCHECK_GT(threshold, min_value); + DCHECK_LE(threshold, max_value); + + // Count the number of positive examples. + UnsignedExampleIdx num_pos_examples = 0; + for (const auto example_idx : selected_examples) { + auto value = values[example_idx]; + if (std::isnan(value)) { + value = na_replacement; + } + if (value >= threshold) { + num_pos_examples++; + } + } + + DCHECK_GT(num_pos_examples, 0); + DCHECK_LT(num_pos_examples, selected_examples.size()); + + // Set split. + auto* condition = node->mutable_node()->mutable_condition(); + condition->set_attribute(attribute_idx); + condition->mutable_condition()->mutable_higher_condition()->set_threshold( + threshold); + condition->set_na_value(na_replacement >= threshold); + condition->set_num_training_examples_without_weight( + selected_examples.size()); + condition->set_num_pos_training_examples_without_weight(num_pos_examples); + + return true; + } + + return false; // No split found +} + +// Grows recursively a node. +absl::Status GrowNode(const Configuration& config, + const dataset::VerticalDataset& train_dataset, + const std::vector& selected_examples, + const int depth, decision_tree::NodeWithChildren* node, + utils::RandomEngine* rnd) { + if (selected_examples.empty()) { + return absl::InternalError("No examples fed to the node trainer"); + } + + const auto& dt_config = config.if_config->decision_tree(); + + // Set node value + node->mutable_node()->set_num_pos_training_examples_without_weight( + selected_examples.size()); + node->mutable_node() + ->mutable_anomaly_detection() + ->set_num_examples_without_weight(selected_examples.size()); + + // Stop growth + if (selected_examples.size() < dt_config.min_examples() || + (dt_config.max_depth() >= 0 && depth >= dt_config.max_depth())) { + node->FinalizeAsLeaf(dt_config.store_detailed_label_distribution()); + return absl::OkStatus(); + } + + // Look for a split + ASSIGN_OR_RETURN( + const bool found_condition, + FindSplit(config, train_dataset, selected_examples, node, rnd)); + + if (!found_condition) { + // No split found + node->FinalizeAsLeaf(dt_config.store_detailed_label_distribution()); + return absl::OkStatus(); + } + + // Turn the node into a non-leaf node + STATUS_CHECK_EQ( + selected_examples.size(), + node->node().condition().num_training_examples_without_weight()); + node->CreateChildren(); + node->FinalizeAsNonLeaf(dt_config.keep_non_leaf_label_distribution(), + dt_config.store_detailed_label_distribution()); + + // Branch examples to children + // TODO: Use cache to avoid re-allocating selected example + // buffers. + std::vector positive_examples; + std::vector negative_examples; + RETURN_IF_ERROR(decision_tree::internal::SplitExamples( + train_dataset, selected_examples, node->node().condition(), false, + dt_config.internal_error_on_wrong_splitter_statistics(), + &positive_examples, &negative_examples)); + + // Split children + RETURN_IF_ERROR(GrowNode(config, train_dataset, positive_examples, depth + 1, + node->mutable_pos_child(), rnd)); + positive_examples = {}; // Release memory of "positive_examples". + RETURN_IF_ERROR(GrowNode(config, train_dataset, negative_examples, depth + 1, + node->mutable_neg_child(), rnd)); + return absl::OkStatus(); +} + +// Grows and return a tree. +absl::StatusOr> GrowTree( + const Configuration& config, const dataset::VerticalDataset& train_dataset, + const std::vector& selected_examples, + utils::RandomEngine* rnd) { + auto tree = std::make_unique(); + tree->CreateRoot(); + RETURN_IF_ERROR(GrowNode(config, train_dataset, selected_examples, + /*depth=*/0, tree->mutable_root(), rnd)); + return std::move(tree); +} + +int DefaultMaximumDepth(UnsignedExampleIdx num_examples_per_trees) { + return std::ceil(std::log2(num_examples_per_trees)); +} + +std::vector SampleExamples( + const UnsignedExampleIdx num_examples, + const UnsignedExampleIdx num_examples_to_sample, utils::RandomEngine* rnd) { + std::vector examples(num_examples); + std::iota(examples.begin(), examples.end(), 0); + std::shuffle(examples.begin(), examples.end(), *rnd); + examples.resize(num_examples_to_sample); + examples.shrink_to_fit(); + std::sort(examples.begin(), examples.end()); + return examples; +} + +SignedExampleIdx GetNumExamplesPerTrees( + const proto::IsolationForestTrainingConfig& if_config, + const SignedExampleIdx num_training_examples) { + switch (if_config.sampling_method_case()) { + case proto::IsolationForestTrainingConfig::kSubsampleRatio: + return static_cast( + std::ceil(static_cast(if_config.subsample_ratio()) * + num_training_examples)); + default: + return if_config.subsample_count(); + } +} + +} // namespace internal + +IsolationForestLearner::IsolationForestLearner( + const model::proto::TrainingConfig& training_config) + : AbstractLearner(training_config) {} + +absl::Status IsolationForestLearner::SetHyperParametersImpl( + utils::GenericHyperParameterConsumer* generic_hyper_params) { + RETURN_IF_ERROR( + AbstractLearner::SetHyperParametersImpl(generic_hyper_params)); + const auto& if_config = training_config_.MutableExtension( + isolation_forest::proto::isolation_forest_config); + + // Decision tree specific hyper-parameters. + absl::flat_hash_set consumed_hparams; + RETURN_IF_ERROR(decision_tree::SetHyperParameters( + &consumed_hparams, if_config->mutable_decision_tree(), + generic_hyper_params)); + + { + const auto hparam = generic_hyper_params->Get(kHParamNumTrees); + if (hparam.has_value()) { + if_config->set_num_trees(hparam.value().value().integer()); + } + } + + { + const auto hparam = generic_hyper_params->Get(kHParamSubsampleRatio); + if (hparam.has_value()) { + if_config->set_subsample_ratio(hparam.value().value().real()); + } + } + + { + const auto hparam = generic_hyper_params->Get(kHParamSubsampleCount); + if (hparam.has_value()) { + if_config->set_subsample_count(hparam.value().value().integer()); + } + } + + return absl::OkStatus(); +} + +absl::StatusOr +IsolationForestLearner::GetGenericHyperParameterSpecification() const { + ASSIGN_OR_RETURN(auto hparam_def, + AbstractLearner::GetGenericHyperParameterSpecification()); + model::proto::TrainingConfig config; + const auto proto_path = "learner/isolation_forest/isolation_forest.proto"; + + hparam_def.mutable_documentation()->set_description( + R"(An Isolation Forest (https://ieeexplore.ieee.org/abstract/document/4781136) is a collection of decision trees trained without labels and independently to partition the feature space. The Isolation Forest prediction is an anomaly score that indicates whether an example originates from a same distribution to the training examples. We refer to Isolation Forest as both the original algorithm by Liu et al. and its extensions.)"); + + const auto& if_config = + config.GetExtension(isolation_forest::proto::isolation_forest_config); + + { + auto& param = hparam_def.mutable_fields()->operator[](kHParamNumTrees); + param.mutable_integer()->set_minimum(0); + param.mutable_integer()->set_default_value(if_config.num_trees()); + param.mutable_documentation()->set_proto_path(proto_path); + param.mutable_documentation()->set_description( + R"(Number of individual decision trees. Increasing the number of trees can increase the quality of the model at the expense of size, training speed, and inference latency.)"); + } + + { + auto& param = + hparam_def.mutable_fields()->operator[](kHParamSubsampleCount); + param.mutable_integer()->set_minimum(0); + param.mutable_integer()->set_default_value(if_config.num_trees()); + param.mutable_documentation()->set_proto_path(proto_path); + param.mutable_documentation()->set_description( + R"(Number of examples used to grow each tree. Only one of "subsample_ratio" and "subsample_count" can be set. If neither is set, "subsample_count" is assumed to be equal to 256. This is the default value recommended in the isolation forest paper.)"); + } + + { + auto& param = + hparam_def.mutable_fields()->operator[](kHParamSubsampleRatio); + param.mutable_integer()->set_minimum(0); + param.mutable_integer()->set_default_value(if_config.num_trees()); + param.mutable_documentation()->set_proto_path(proto_path); + param.mutable_documentation()->set_description( + R"(Ratio of number of training examples used to grow each tree. Only one of "subsample_ratio" and "subsample_count" can be set. If neither is set, "subsample_count" is assumed to be equal to 256. This is the default value recommended in the isolation forest paper.)"); + } + + RETURN_IF_ERROR(decision_tree::GetGenericHyperParameterSpecification( + if_config.decision_tree(), &hparam_def)); + return hparam_def; +} + +absl::StatusOr> +IsolationForestLearner::TrainWithStatusImpl( + const dataset::VerticalDataset& train_dataset, + absl::optional> + valid_dataset) const { + RETURN_IF_ERROR(dataset::CheckNumExamples(train_dataset.nrow())); + + ASSIGN_OR_RETURN( + const internal::Configuration config, + BuildConfig(*this, train_dataset.data_spec(), train_dataset.nrow())); + + auto model = absl::make_unique(); + InitializeModelWithAbstractTrainingConfig(config.training_config, + config.config_link, model.get()); + model->set_data_spec(train_dataset.data_spec()); + model->set_num_examples_per_trees( + GetNumExamplesPerTrees(*config.if_config, train_dataset.nrow())); + + YDF_LOG(INFO) << "Training isolation forest on " << train_dataset.nrow() + << " example(s) and " << config.config_link.features_size() + << " feature(s)."; + + utils::RandomEngine global_random(config.training_config.random_seed()); + + absl::Status global_status; + utils::concurrency::Mutex global_mutex; + { + yggdrasil_decision_forests::utils::concurrency::ThreadPool pool( + "TrainIF", deployment().num_threads()); + pool.StartWorkers(); + const auto num_trees = config.if_config->num_trees(); + model->mutable_decision_trees()->resize(num_trees); + for (int tree_idx = 0; tree_idx < num_trees; tree_idx++) { + pool.Schedule([&train_dataset, &model, &config, tree_idx, &global_status, + &global_mutex, seed = global_random()]() { + { + utils::concurrency::MutexLock lock(&global_mutex); + if (!global_status.ok()) { + return; + } + } + utils::RandomEngine local_random(seed); + const auto selected_examples = internal::SampleExamples( + train_dataset.nrow(), model->num_examples_per_trees(), + &local_random); + auto tree_or = + GrowTree(config, train_dataset, selected_examples, &local_random); + if (!tree_or.ok()) { + utils::concurrency::MutexLock lock(&global_mutex); + global_status.Update(tree_or.status()); + return; + } + (*model->mutable_decision_trees())[tree_idx] = std::move(*tree_or); + }); + } + } + decision_tree::SetLeafIndices(model->mutable_decision_trees()); + return std::move(model); +} + +} // namespace yggdrasil_decision_forests::model::isolation_forest diff --git a/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.h b/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.h new file mode 100644 index 00000000..7d1b3450 --- /dev/null +++ b/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.h @@ -0,0 +1,115 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Isolation Forest learner. +#ifndef YGGDRASIL_DECISION_FORESTS_LEARNER_ISOLATION_FOREST_H_ +#define YGGDRASIL_DECISION_FORESTS_LEARNER_ISOLATION_FOREST_H_ + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/types/optional.h" +#include "yggdrasil_decision_forests/dataset/types.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset.h" +#include "yggdrasil_decision_forests/learner/abstract_learner.h" +#include "yggdrasil_decision_forests/learner/abstract_learner.pb.h" +#include "yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.pb.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree.h" +#include "yggdrasil_decision_forests/utils/hyper_parameters.h" +#include "yggdrasil_decision_forests/utils/random.h" + +namespace yggdrasil_decision_forests::model::isolation_forest { + +class IsolationForestLearner : public AbstractLearner { + public: + explicit IsolationForestLearner( + const model::proto::TrainingConfig& training_config); + + inline static constexpr char kRegisteredName[] = "ISOLATION_FOREST"; + inline static constexpr char kHParamNumTrees[] = "num_trees"; + inline static constexpr char kHParamSubsampleRatio[] = "subsample_ratio"; + inline static constexpr char kHParamSubsampleCount[] = "subsample_count"; + // TODO: Add all hyper-parameters. + + absl::StatusOr> TrainWithStatusImpl( + const dataset::VerticalDataset& train_dataset, + absl::optional> + valid_dataset) const override; + + absl::Status SetHyperParametersImpl( + utils::GenericHyperParameterConsumer* generic_hyper_params) override; + + absl::StatusOr + GetGenericHyperParameterSpecification() const override; + + model::proto::LearnerCapabilities Capabilities() const override { + model::proto::LearnerCapabilities capabilities; + capabilities.set_require_label(false); + return capabilities; + } +}; + +REGISTER_AbstractLearner(IsolationForestLearner, + IsolationForestLearner::kRegisteredName); + +namespace internal { + +struct Configuration { + model::proto::TrainingConfig training_config; + model::proto::TrainingConfigLinking config_link; + // "if_config" is a non-owning pointer to a sub-component of + // "training_config". + proto::IsolationForestTrainingConfig* if_config = nullptr; +}; + +// Gets the number of examples used to grow each tree. +SignedExampleIdx GetNumExamplesPerTrees( + const proto::IsolationForestTrainingConfig& if_config, + SignedExampleIdx num_training_examples); + +// Sample examples to grow a tree. +std::vector SampleExamples( + UnsignedExampleIdx num_examples, UnsignedExampleIdx num_examples_to_sample, + utils::RandomEngine* rnd); + +// Default maximum depth hyper-parameter according to the number of examples +// used to grow each tree. +int DefaultMaximumDepth(UnsignedExampleIdx num_examples_per_trees); + +// Finds a split (i.e. condition) for a node. +// +// A split is randomly sampled and returned. +// A valid split always branches one training examples in each branch. If not +// valid split can be generated, "FindSplit" returns false and not split is set. +// If a valid split is sampled, the condition of "node" is set and the function +// returns true. +// +// This function currently only implement the original isolation forest +// algorithm: Only split of the form "X >= threshold" are generated. The +// threshold is uniformly sampled between the minimum and maximum values +// observed in the training examples reaching this node +absl::StatusOr FindSplit( + const Configuration& config, const dataset::VerticalDataset& train_dataset, + const std::vector& selected_examples, + decision_tree::NodeWithChildren* node, utils::RandomEngine* rnd); + +} // namespace internal + +} // namespace yggdrasil_decision_forests::model::isolation_forest +#endif // YGGDRASIL_DECISION_FORESTS_LEARNER_ISOLATION_FOREST_H_ diff --git a/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.proto b/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.proto new file mode 100644 index 00000000..3cfafc78 --- /dev/null +++ b/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.proto @@ -0,0 +1,48 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package yggdrasil_decision_forests.model.isolation_forest.proto; + +import "yggdrasil_decision_forests/learner/abstract_learner.proto"; +import "yggdrasil_decision_forests/learner/decision_tree/decision_tree.proto"; + +option java_outer_classname = "IsolationForestLearner"; + +// Training configuration for the Isolation Forest algorithm. +message IsolationForestTrainingConfig { + // Next ID: 5 + + // Decision tree specific parameters. + optional decision_tree.proto.DecisionTreeTrainingConfig decision_tree = 1; + + // Number of trees in the forest. + optional int32 num_trees = 2 [default = 300]; + + // Number of examples used to grow each tree. Only one of "subsample_ratio" + // and "subsample_count" can be set. If neither is set, "subsample_count" is + // assumed to be equal to 256. This is the default value recommended in the + // isolation forest paper + // (https://ieeexplore.ieee.org/abstract/document/4781136). + oneof sampling_method { + float subsample_ratio = 3; + int32 subsample_count = 4 [default = 256]; + } +} + +extend model.proto.TrainingConfig { + optional IsolationForestTrainingConfig isolation_forest_config = 1007; +} diff --git a/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest_test.cc b/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest_test.cc new file mode 100644 index 00000000..db5380ed --- /dev/null +++ b/yggdrasil_decision_forests/learner/isolation_forest/isolation_forest_test.cc @@ -0,0 +1,245 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.h" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "yggdrasil_decision_forests/dataset/data_spec.h" +#include "yggdrasil_decision_forests/dataset/data_spec_inference.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset.h" +#include "yggdrasil_decision_forests/learner/abstract_learner.pb.h" +#include "yggdrasil_decision_forests/learner/isolation_forest/isolation_forest.pb.h" +#include "yggdrasil_decision_forests/learner/learner_library.h" +#include "yggdrasil_decision_forests/metric/metric.pb.h" +#include "yggdrasil_decision_forests/metric/report.h" +#include "yggdrasil_decision_forests/model/abstract_model.pb.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" +#include "yggdrasil_decision_forests/utils/filesystem.h" +#include "yggdrasil_decision_forests/utils/logging.h" +#include "yggdrasil_decision_forests/utils/random.h" +#include "yggdrasil_decision_forests/utils/test.h" +#include "yggdrasil_decision_forests/utils/test_utils.h" +#include "yggdrasil_decision_forests/utils/testing_macros.h" + +namespace yggdrasil_decision_forests::model::isolation_forest { +namespace { + +using test::StatusIs; +using ::testing::ElementsAre; + +class IsolationForestOnGaussians : public utils::TrainAndTestTester { + proto::IsolationForestTrainingConfig* if_config() { + return train_config_.MutableExtension( + isolation_forest::proto::isolation_forest_config); + } + + void SetUp() override { + train_config_.set_learner(IsolationForestLearner::kRegisteredName); + train_config_.set_task(model::proto::Task::ANOMALY_DETECTION); + train_config_.add_features("f.*"); + train_config_.set_label("label"); + dataset_filename_ = "gaussians_train.csv"; + eval_options_.set_task(model::proto::Task::ANOMALY_DETECTION); + + if_config()->set_subsample_count(100); + } +}; + +TEST_F(IsolationForestOnGaussians, DefaultHyperParameters) { + TrainAndEvaluateModel(); + YDF_LOG(INFO) << "Model:\n" << model_->DescriptionAndStatistics(true); + + utils::RandomEngine rnd; + metric::proto::EvaluationOptions options; + options.set_task(model::proto::Task::CLASSIFICATION); + ASSERT_OK_AND_ASSIGN( + const auto evaluation, + model_->EvaluateOverrideType(test_dataset_, options, + model::proto::Task::CLASSIFICATION, + model_->label_col_idx(), -1, &rnd)); + + YDF_LOG(INFO) << "Evaluation:\n" << metric::TextReport(evaluation).value(); + EXPECT_NEAR(evaluation.classification().rocs(1).auc(), 0.99, 0.005f); + + EXPECT_EQ(model_->task(), model::proto::Task::ANOMALY_DETECTION); + EXPECT_EQ(model_->label_col_idx(), 2); + EXPECT_THAT(model_->input_features(), ElementsAre(0, 1)); + + auto if_model = dynamic_cast(model_.get()); + EXPECT_EQ(if_model->num_trees(), 300); + EXPECT_GT(if_model->NumNodes(), if_model->num_trees() * 32); +} + +class IsolationForestOnAdult : public utils::TrainAndTestTester { + proto::IsolationForestTrainingConfig* if_config() { + return train_config_.MutableExtension( + isolation_forest::proto::isolation_forest_config); + } + + void SetUp() override { + train_config_.set_learner(IsolationForestLearner::kRegisteredName); + train_config_.set_task(model::proto::Task::ANOMALY_DETECTION); + train_config_.set_label("income"); + dataset_filename_ = "adult_train.csv"; + eval_options_.set_task(model::proto::Task::ANOMALY_DETECTION); + + if_config()->set_subsample_count(100); + } +}; + +TEST_F(IsolationForestOnAdult, DefaultHyperParameters) { + TrainAndEvaluateModel(); + YDF_LOG(INFO) << "Model:\n" << model_->DescriptionAndStatistics(true); +} + +class IsolationForestOnMammographicMasses : public utils::TrainAndTestTester { + proto::IsolationForestTrainingConfig* if_config() { + return train_config_.MutableExtension( + isolation_forest::proto::isolation_forest_config); + } + + void SetUp() override { + train_config_.set_learner(IsolationForestLearner::kRegisteredName); + train_config_.set_task(model::proto::Task::ANOMALY_DETECTION); + train_config_.set_label("Severity"); + dataset_filename_ = "mammographic_masses.csv"; + eval_options_.set_task(model::proto::Task::ANOMALY_DETECTION); + + if_config()->set_subsample_count(100); + } +}; + +TEST_F(IsolationForestOnMammographicMasses, DefaultHyperParameters) { + TrainAndEvaluateModel(); + YDF_LOG(INFO) << "Model:\n" << model_->DescriptionAndStatistics(true); +} + +TEST(IsolationForest, BadTask) { + std::string dataset_path = absl::StrCat( + "csv:", file::JoinPath(test::DataRootDirectory(), + "yggdrasil_decision_forests/" + "test_data/dataset/gaussians_train.csv")); + + ASSERT_OK_AND_ASSIGN(auto dataspec, dataset::CreateDataSpec(dataset_path)); + + model::proto::TrainingConfig train_config; + train_config.set_learner(IsolationForestLearner::kRegisteredName); + train_config.set_task(model::proto::Task::CLASSIFICATION); + train_config.add_features("f.*"); + train_config.set_label("label"); + ASSERT_OK_AND_ASSIGN(auto learner, model::GetLearner(train_config)); + + EXPECT_THAT(learner->TrainWithStatus(dataset_path, dataspec).status(), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST(DefaultMaximumDepth, Base) { + EXPECT_EQ(internal::DefaultMaximumDepth(254), 8); + EXPECT_EQ(internal::DefaultMaximumDepth(255), 8); + EXPECT_EQ(internal::DefaultMaximumDepth(256), 8); + EXPECT_EQ(internal::DefaultMaximumDepth(257), 9); +} + +TEST(SampleExamples, Base) { + utils::RandomEngine rnd; + const auto samples = internal::SampleExamples(100, 10, &rnd); + EXPECT_EQ(samples.size(), 10); + EXPECT_TRUE(std::is_sorted(samples.begin(), samples.end())); + + // Look for duplicates + for (int i = 1; i < samples.size(); i++) { + EXPECT_LT(samples[i - 1], samples[i]); + } +} + +TEST(GetNumExamplesPerTrees, Default) { + proto::IsolationForestTrainingConfig if_config; + EXPECT_EQ(internal::GetNumExamplesPerTrees(if_config, 1000), 256); +} + +TEST(GetNumExamplesPerTrees, Count) { + proto::IsolationForestTrainingConfig if_config; + if_config.set_subsample_count(5); + EXPECT_EQ(internal::GetNumExamplesPerTrees(if_config, 100), 5); +} + +TEST(GetNumExamplesPerTrees, Rate) { + proto::IsolationForestTrainingConfig if_config; + if_config.set_subsample_ratio(0.5f); + EXPECT_EQ(internal::GetNumExamplesPerTrees(if_config, 100), 50); +} + +TEST(FindSplit, Numerical) { + for (int seed = 0; seed < 10; seed++) { + // This is a stochastic test. + utils::RandomEngine rnd(seed); + + internal::Configuration config; + config.config_link.add_features(0); // Only select "f1". + + decision_tree::NodeWithChildren node; + + dataset::VerticalDataset dataset; + dataset::AddNumericalColumn("f1", dataset.mutable_data_spec()); + dataset::AddNumericalColumn("f2", dataset.mutable_data_spec()); + ASSERT_OK(dataset.CreateColumnsFromDataspec()); + ASSERT_OK_AND_ASSIGN(auto* column, + dataset.MutableColumnWithCastWithStatus< + dataset::VerticalDataset::NumericalColumn>(0)); + *column->mutable_values() = {1, 2, 4, 100}; + + ASSERT_OK_AND_ASSIGN( + const bool found_condition, + FindSplit(config, dataset, + {0, 1, 2}, // Don't select the example with value "100". + &node, &rnd)); + EXPECT_TRUE(found_condition); + EXPECT_EQ(node.node().condition().attribute(), 0); // Always "f1". + EXPECT_TRUE(node.node().condition().condition().has_higher_condition()); + const float threshold = + node.node().condition().condition().higher_condition().threshold(); + EXPECT_GE(threshold, 1.0f); + EXPECT_LE(threshold, 4.0f); // The value 100 is clearly ignored. + } +} + +TEST(GetGenericHyperParameterSpecification, Base) { + model::proto::TrainingConfig train_config; + train_config.set_learner(IsolationForestLearner::kRegisteredName); + train_config.set_task(model::proto::Task::ANOMALY_DETECTION); + ASSERT_OK_AND_ASSIGN(auto learner, model::GetLearner(train_config)); + + ASSERT_OK_AND_ASSIGN(const auto hp_specs, + learner->GetGenericHyperParameterSpecification()); + + for (absl::string_view field : { + IsolationForestLearner::kHParamNumTrees, + IsolationForestLearner::kHParamSubsampleRatio, + IsolationForestLearner::kHParamSubsampleCount, + }) { + EXPECT_TRUE(hp_specs.fields().contains(field)); + } +} + +} // namespace +} // namespace yggdrasil_decision_forests::model::isolation_forest diff --git a/yggdrasil_decision_forests/test_data/dataset/mammographic_masses.csv b/yggdrasil_decision_forests/test_data/dataset/mammographic_masses.csv new file mode 100644 index 00000000..3a09f575 --- /dev/null +++ b/yggdrasil_decision_forests/test_data/dataset/mammographic_masses.csv @@ -0,0 +1,962 @@ +BI-RADS,Age,Shape,Margin,Density,Severity +5,67,3,5,3,1 +4,43,1,1,,1 +5,58,4,5,3,1 +4,28,1,1,3,0 +5,74,1,5,,1 +4,65,1,,3,0 +4,70,,,3,0 +5,42,1,,3,0 +5,57,1,5,3,1 +5,60,,5,1,1 +5,76,1,4,3,1 +3,42,2,1,3,1 +4,64,1,,3,0 +4,36,3,1,2,0 +4,60,2,1,2,0 +4,54,1,1,3,0 +3,52,3,4,3,0 +4,59,2,1,3,1 +4,54,1,1,3,1 +4,40,1,,,0 +,66,,,1,1 +5,56,4,3,1,1 +4,43,1,,,0 +5,42,4,4,3,1 +4,59,2,4,3,1 +5,75,4,5,3,1 +2,66,1,1,,0 +5,63,3,,3,0 +5,45,4,5,3,1 +5,55,4,4,3,0 +4,46,1,5,2,0 +5,54,4,4,3,1 +5,57,4,4,3,1 +4,39,1,1,2,0 +4,81,1,1,3,0 +4,77,3,,,0 +4,60,2,1,3,0 +5,67,3,4,2,1 +4,48,4,5,,1 +4,55,3,4,2,0 +4,59,2,1,,0 +4,78,1,1,1,0 +4,50,1,1,3,0 +4,61,2,1,,0 +5,62,3,5,2,1 +5,44,2,4,,1 +5,64,4,5,3,1 +4,23,1,1,,0 +2,42,,,4,0 +5,67,4,5,3,1 +4,74,2,1,2,0 +5,80,3,5,3,1 +4,23,1,1,,0 +4,63,2,1,,0 +4,53,,5,3,1 +4,43,3,4,,0 +4,49,2,1,1,0 +5,51,2,4,,0 +4,45,2,1,,0 +5,59,2,,,1 +5,52,4,3,3,1 +5,60,4,3,3,1 +4,57,2,5,3,0 +3,57,2,1,,0 +5,74,4,4,3,1 +4,25,2,1,,0 +4,49,1,1,3,0 +5,72,4,3,,1 +4,45,2,1,3,0 +4,64,2,1,3,0 +4,73,2,1,2,0 +5,68,4,3,3,1 +5,52,4,5,3,0 +5,66,4,4,3,1 +5,70,,4,,1 +4,25,1,1,3,0 +5,74,1,1,2,1 +4,64,1,1,3,0 +5,60,4,3,2,1 +5,67,2,4,1,0 +4,67,4,5,3,0 +5,44,4,4,2,1 +3,68,1,1,3,1 +4,57,,4,1,0 +5,51,4,,,1 +4,33,1,,,0 +5,58,4,4,3,1 +5,36,1,,,0 +4,63,1,1,,0 +5,62,1,5,3,1 +4,73,3,4,3,1 +4,80,4,4,3,1 +4,67,1,1,,0 +5,59,2,1,3,1 +5,60,1,,3,0 +5,54,4,4,3,1 +4,40,1,1,,0 +4,47,2,1,,0 +5,62,4,4,3,0 +4,33,2,1,3,0 +5,59,2,,,0 +4,65,2,,,0 +4,58,4,4,,0 +4,29,2,,,0 +4,58,1,1,,0 +4,54,1,1,,0 +4,44,1,1,,1 +3,34,2,1,,0 +4,57,1,1,3,0 +5,33,4,4,,1 +4,45,4,4,3,0 +5,71,4,4,3,1 +5,59,4,4,2,0 +4,56,2,1,,0 +4,40,3,4,,0 +4,56,1,1,3,0 +4,45,2,1,,0 +4,57,2,1,2,0 +5,55,3,4,3,1 +5,84,4,5,3,0 +5,51,4,4,3,1 +4,43,1,1,,0 +4,24,2,1,2,0 +4,66,1,1,3,0 +5,33,4,4,3,0 +4,59,4,3,2,0 +4,76,2,3,,0 +4,40,1,1,,0 +4,52,,4,,0 +5,40,4,5,3,1 +5,67,4,4,3,1 +5,75,4,3,3,1 +5,86,4,4,3,0 +4,60,2,,,0 +5,66,4,4,3,1 +5,46,4,5,3,1 +4,59,4,4,3,1 +5,65,4,4,3,1 +4,53,1,1,3,0 +5,67,3,5,3,1 +5,80,4,5,3,1 +4,55,2,1,3,0 +4,48,1,1,,0 +4,47,1,1,2,0 +4,50,2,1,,0 +5,62,4,5,3,1 +5,63,4,4,3,1 +4,63,4,,3,1 +4,71,4,4,3,1 +4,41,1,1,3,0 +5,57,4,4,4,1 +5,71,4,4,4,1 +4,66,1,1,3,0 +4,47,2,4,2,0 +3,34,4,4,3,0 +4,59,3,4,3,0 +5,55,2,,,1 +4,51,,,3,0 +4,62,2,1,,0 +4,58,4,,3,1 +5,67,4,4,3,1 +4,41,2,1,3,0 +4,23,3,1,3,0 +4,53,,4,3,0 +4,42,2,1,3,0 +5,87,4,5,3,1 +4,68,1,1,3,1 +4,64,1,1,3,0 +5,54,3,5,3,1 +5,86,4,5,3,1 +4,21,2,1,3,0 +4,39,1,1,,0 +4,53,4,4,3,0 +4,44,4,4,3,0 +4,54,1,1,3,0 +5,63,4,5,3,1 +4,62,2,1,,0 +4,45,2,1,2,0 +5,71,4,5,3,0 +5,49,4,4,3,1 +4,49,4,4,3,0 +5,66,4,4,4,0 +4,19,1,1,3,0 +4,35,1,1,2,0 +4,71,3,3,,1 +5,74,4,5,3,1 +5,37,4,4,3,1 +4,67,1,,3,0 +5,81,3,4,3,1 +5,59,4,4,3,1 +4,34,1,1,3,0 +5,79,4,3,3,1 +5,60,3,1,3,0 +4,41,1,1,3,1 +4,50,1,1,3,0 +5,85,4,4,3,1 +4,46,1,1,3,0 +5,66,4,4,3,1 +4,73,3,1,2,0 +4,55,1,1,3,0 +4,49,2,1,3,0 +3,49,4,4,3,0 +4,51,4,5,3,1 +2,48,4,4,3,0 +4,58,4,5,3,0 +5,72,4,5,3,1 +4,46,2,3,3,0 +4,43,4,3,3,1 +,52,4,4,3,0 +4,66,2,1,,0 +4,46,1,1,1,0 +4,69,3,1,3,0 +2,59,1,1,,1 +5,43,2,1,3,1 +5,76,4,5,3,1 +4,46,1,1,3,0 +4,59,2,4,3,0 +4,57,1,1,3,0 +5,43,4,5,,0 +3,45,2,1,3,0 +3,43,2,1,3,0 +4,45,2,1,3,0 +5,57,4,5,3,1 +5,79,4,4,3,1 +5,54,2,1,3,1 +4,40,3,4,3,0 +5,63,4,4,3,1 +2,55,1,,1,0 +4,52,2,1,3,0 +4,38,1,1,3,0 +3,72,4,3,3,0 +5,80,4,3,3,1 +5,76,4,3,3,1 +4,62,3,1,3,0 +5,64,4,5,3,1 +5,42,4,5,3,0 +3,60,,3,1,0 +4,64,4,5,3,0 +4,63,4,4,3,1 +4,24,2,1,2,0 +5,72,4,4,3,1 +4,63,2,1,3,0 +4,46,1,1,3,0 +3,33,1,1,3,0 +5,76,4,4,3,1 +4,36,2,3,3,0 +4,40,2,1,3,0 +5,58,1,5,3,1 +4,43,2,1,3,0 +3,42,1,1,3,0 +4,32,1,1,3,0 +5,57,4,4,2,1 +4,37,1,1,3,0 +4,70,4,4,3,1 +5,56,4,2,3,1 +3,76,,3,2,0 +5,73,4,4,3,1 +5,77,4,5,3,1 +5,67,4,4,1,1 +5,71,4,3,3,1 +5,65,4,4,3,1 +4,43,1,1,3,0 +4,40,2,1,,0 +4,49,2,1,3,0 +5,76,4,2,3,1 +4,55,4,4,3,0 +5,72,4,5,3,1 +3,53,4,3,3,0 +5,75,4,4,3,1 +5,61,4,5,3,1 +5,67,4,4,3,1 +5,55,4,2,3,1 +5,66,4,4,3,1 +2,76,1,1,2,0 +4,57,4,4,3,1 +5,71,3,1,3,0 +5,70,4,5,3,1 +4,35,4,2,,0 +5,79,1,,3,1 +4,63,2,1,3,0 +5,40,1,4,3,1 +4,41,1,1,3,0 +4,47,2,1,2,0 +4,68,1,1,3,1 +4,64,4,3,3,1 +4,65,4,4,,1 +4,73,4,3,3,0 +4,39,4,3,3,0 +5,55,4,5,4,1 +5,53,3,4,4,0 +5,66,4,4,3,1 +4,43,3,1,2,0 +5,44,4,5,3,1 +4,77,4,4,3,1 +4,62,2,4,3,0 +5,80,4,4,3,1 +4,33,4,4,3,0 +4,50,4,5,3,1 +4,71,1,,3,0 +5,46,4,4,3,1 +5,49,4,5,3,1 +4,53,1,1,3,0 +3,46,2,1,2,0 +4,57,1,1,3,0 +4,54,3,1,3,0 +4,54,1,,,0 +2,49,2,1,2,0 +4,47,3,1,3,0 +4,40,1,1,3,0 +4,45,1,1,3,0 +4,50,4,5,3,1 +5,54,4,4,3,1 +4,67,4,1,3,1 +4,77,4,4,3,1 +4,66,4,3,3,0 +4,71,2,,3,1 +4,36,2,3,3,0 +4,69,4,4,3,0 +4,48,1,1,3,0 +4,64,4,4,3,1 +4,71,4,2,3,1 +5,60,4,3,3,1 +4,24,1,1,3,0 +5,34,4,5,2,1 +4,79,1,1,2,0 +4,45,1,1,3,0 +4,37,2,1,2,0 +4,42,1,1,2,0 +4,72,4,4,3,1 +5,60,4,5,3,1 +5,85,3,5,3,1 +4,51,1,1,3,0 +5,54,4,5,3,1 +5,55,4,3,3,1 +4,64,4,4,3,0 +5,67,4,5,3,1 +5,75,4,3,3,1 +5,87,4,4,3,1 +4,46,4,4,3,1 +4,59,2,1,,0 +55,46,4,3,3,1 +5,61,1,1,3,1 +4,44,1,4,3,0 +4,32,1,1,3,0 +4,62,1,1,3,0 +5,59,4,5,3,1 +4,61,4,1,3,0 +5,78,4,4,3,1 +5,42,4,5,3,0 +4,45,1,2,3,0 +5,34,2,1,3,1 +5,39,4,3,,1 +4,27,3,1,3,0 +4,43,1,1,3,0 +5,83,4,4,3,1 +4,36,2,1,3,0 +4,37,2,1,3,0 +4,56,3,1,3,1 +5,55,4,4,3,1 +5,46,3,,3,0 +4,88,4,4,3,1 +5,71,4,4,3,1 +4,41,2,1,3,0 +5,49,4,4,3,1 +3,51,1,1,4,0 +4,39,1,3,3,0 +4,46,2,1,3,0 +5,52,4,4,3,1 +5,58,4,4,3,1 +4,67,4,5,3,1 +5,80,4,4,3,1 +3,46,1,,,0 +3,43,1,,,0 +4,45,1,1,3,0 +5,68,4,4,3,1 +4,54,4,4,,1 +4,44,2,3,3,0 +5,74,4,3,3,1 +5,55,4,5,3,0 +4,49,4,4,3,1 +4,49,1,1,3,0 +5,50,4,3,3,1 +5,52,3,5,3,1 +4,45,1,1,3,0 +4,66,1,1,3,0 +4,68,4,4,3,1 +4,72,2,1,3,0 +5,64,,,3,0 +2,49,,3,3,0 +3,44,,4,3,0 +5,74,4,4,3,1 +5,58,4,4,3,1 +4,77,2,3,3,0 +4,49,3,1,3,0 +4,34,,,4,0 +5,60,4,3,3,1 +5,69,4,3,3,1 +4,53,2,1,3,0 +3,46,3,4,3,0 +5,74,4,4,3,1 +4,58,1,1,3,0 +5,68,4,4,3,1 +5,46,4,3,3,0 +5,61,2,4,3,1 +5,70,4,3,3,1 +5,37,4,4,3,1 +3,65,4,5,3,1 +4,67,4,4,3,0 +5,69,3,4,3,0 +5,76,4,4,3,1 +4,65,4,3,3,0 +5,72,4,2,3,1 +4,62,4,2,3,0 +5,42,4,4,3,1 +5,66,4,3,3,1 +5,48,4,4,3,1 +4,35,1,1,3,0 +5,60,4,4,3,1 +5,67,4,2,3,1 +5,78,4,4,3,1 +4,66,1,1,3,1 +4,26,1,1,,0 +4,48,1,1,3,0 +4,31,1,1,3,0 +5,43,4,3,3,1 +5,72,2,4,3,0 +5,66,1,1,3,1 +4,56,4,4,3,0 +5,58,4,5,3,1 +5,33,2,4,3,1 +4,37,1,1,3,0 +5,36,4,3,3,1 +4,39,2,3,3,0 +4,39,4,4,3,1 +5,83,4,4,3,1 +4,68,4,5,3,1 +5,63,3,4,3,1 +5,78,4,4,3,1 +4,38,2,3,3,0 +5,46,4,3,3,1 +5,60,4,4,3,1 +5,56,2,3,3,1 +4,33,1,1,3,0 +4,,4,5,3,1 +4,69,1,5,3,1 +5,66,1,4,3,1 +4,72,1,3,3,0 +4,29,1,1,3,0 +5,54,4,5,3,1 +5,80,4,4,3,1 +5,68,4,3,3,1 +4,35,2,1,3,0 +4,57,3,,3,0 +5,,4,4,3,1 +4,50,1,1,3,0 +4,32,4,3,3,0 +0,69,4,5,3,1 +4,71,4,5,3,1 +5,87,4,5,3,1 +3,40,2,,3,0 +4,31,1,1,,0 +4,64,1,1,3,0 +5,55,4,5,3,1 +4,18,1,1,3,0 +3,50,2,1,,0 +4,53,1,1,3,0 +5,84,4,5,3,1 +5,80,4,3,3,1 +4,32,1,1,3,0 +5,77,3,4,3,1 +4,38,1,1,3,0 +5,54,4,5,3,1 +4,63,1,1,3,0 +4,61,1,1,3,0 +4,52,1,1,3,0 +4,36,1,1,3,0 +4,41,,,3,0 +4,59,1,1,3,0 +5,51,4,4,2,1 +4,36,1,1,3,0 +5,40,4,3,3,1 +4,49,1,1,3,0 +4,37,2,3,3,0 +4,46,1,1,3,0 +4,63,1,1,3,0 +4,28,2,1,3,0 +4,47,2,1,3,0 +4,42,2,1,3,1 +5,44,4,5,3,1 +4,49,4,4,3,0 +5,47,4,5,3,1 +5,52,4,5,3,1 +4,53,1,1,3,1 +5,83,3,3,3,1 +4,50,4,4,,1 +5,63,4,4,3,1 +4,82,,5,3,1 +4,54,1,1,3,0 +4,50,4,4,3,0 +5,80,4,5,3,1 +5,45,2,4,3,0 +5,59,4,4,,1 +4,28,2,1,3,0 +4,31,1,1,3,0 +4,41,2,1,3,0 +4,21,3,1,3,0 +5,44,3,4,3,1 +5,49,4,4,3,1 +5,71,4,5,3,1 +5,75,4,5,3,1 +4,38,2,1,3,0 +4,60,1,3,3,0 +5,87,4,5,3,1 +4,70,4,4,3,1 +5,55,4,5,3,1 +3,21,1,1,3,0 +4,50,1,1,3,0 +5,76,4,5,3,1 +4,23,1,1,3,0 +3,68,,,3,0 +4,62,4,,3,1 +5,65,1,,3,1 +5,73,4,5,3,1 +4,38,2,3,3,0 +2,57,1,1,3,0 +5,65,4,5,3,1 +5,67,2,4,3,1 +5,61,2,4,3,1 +5,56,4,4,3,0 +5,71,2,4,3,1 +4,49,2,2,3,0 +4,55,,,3,0 +4,44,2,1,3,0 +0,58,4,4,3,0 +4,27,2,1,3,0 +5,73,4,5,3,1 +4,34,2,1,3,0 +5,63,,4,3,1 +4,50,2,1,3,1 +4,62,2,1,3,0 +3,21,3,1,3,0 +4,49,2,,3,0 +4,36,3,1,3,0 +4,45,2,1,3,1 +5,67,4,5,3,1 +4,21,1,1,3,0 +4,57,2,1,3,0 +5,66,4,5,3,1 +4,71,4,4,3,1 +5,69,3,4,3,1 +6,80,4,5,3,1 +3,27,2,1,3,0 +4,38,2,1,3,0 +4,23,2,1,3,0 +5,70,,5,3,1 +4,46,4,3,3,0 +4,61,2,3,3,0 +5,65,4,5,3,1 +4,60,4,3,3,0 +5,83,4,5,3,1 +5,40,4,4,3,1 +2,59,,4,3,0 +4,53,3,4,3,0 +4,76,4,4,3,0 +5,79,1,4,3,1 +5,38,2,4,3,1 +4,61,3,4,3,0 +4,56,2,1,3,0 +4,44,2,1,3,0 +4,64,3,4,,1 +4,66,3,3,3,0 +4,50,3,3,3,0 +4,46,1,1,3,0 +4,39,1,1,3,0 +4,60,3,,,0 +5,55,4,5,3,1 +4,40,2,1,3,0 +4,26,1,1,3,0 +5,84,3,2,3,1 +4,41,2,2,3,0 +4,63,1,1,3,0 +2,65,,1,2,0 +4,49,1,1,3,0 +4,56,2,2,3,1 +5,65,4,4,3,0 +4,54,1,1,3,0 +4,36,1,1,3,0 +5,49,4,4,3,0 +4,59,4,4,3,1 +5,75,4,4,3,1 +5,59,4,2,3,0 +5,59,4,4,3,1 +4,28,4,4,3,1 +5,53,4,5,3,0 +5,57,4,4,3,0 +5,77,4,3,4,0 +5,85,4,3,3,1 +4,59,4,4,3,0 +5,59,1,5,3,1 +4,65,3,3,3,1 +4,54,2,1,3,0 +5,46,4,5,3,1 +4,63,4,4,3,1 +4,53,1,1,3,1 +4,56,1,1,3,0 +5,66,4,4,3,1 +5,66,4,5,3,1 +4,55,1,1,3,0 +4,44,1,1,3,0 +5,86,3,4,3,1 +5,47,4,5,3,1 +5,59,4,5,3,1 +5,66,4,5,3,0 +5,61,4,3,3,1 +3,46,,5,,1 +4,69,1,1,3,0 +5,93,1,5,3,1 +4,39,1,3,3,0 +5,44,4,5,3,1 +4,45,2,2,3,0 +4,51,3,4,3,0 +4,56,2,4,3,0 +4,66,4,4,3,0 +5,61,4,5,3,1 +4,64,3,3,3,1 +5,57,2,4,3,0 +5,79,4,4,3,1 +4,57,2,1,,0 +4,44,4,1,1,0 +4,31,2,1,3,0 +4,63,4,4,3,0 +4,64,1,1,3,0 +5,47,4,5,3,0 +5,68,4,5,3,1 +4,30,1,1,3,0 +5,43,4,5,3,1 +4,56,1,1,3,0 +4,46,2,1,3,0 +4,67,2,1,3,0 +5,52,4,5,3,1 +4,67,4,4,3,1 +4,47,2,1,3,0 +5,58,4,5,3,1 +4,28,2,1,3,0 +4,43,1,1,3,0 +4,57,2,4,3,0 +5,68,4,5,3,1 +4,64,2,4,3,0 +4,64,2,4,3,0 +5,62,4,4,3,1 +4,38,4,1,3,0 +5,68,4,4,3,1 +4,41,2,1,3,0 +4,35,2,1,3,1 +4,68,2,1,3,0 +5,55,4,4,3,1 +5,67,4,4,3,1 +4,51,4,3,3,0 +2,40,1,1,3,0 +5,73,4,4,3,1 +4,58,,4,3,1 +4,51,,4,3,0 +3,50,,,3,1 +5,59,4,3,3,1 +6,60,3,5,3,1 +4,27,2,1,,0 +5,54,4,3,3,0 +4,56,1,1,3,0 +5,53,4,5,3,1 +4,54,2,4,3,0 +5,79,1,4,3,1 +5,67,4,3,3,1 +5,64,3,3,3,1 +4,70,1,2,3,1 +5,55,4,3,3,1 +5,65,3,3,3,1 +5,45,4,2,3,1 +4,57,4,4,,1 +5,49,1,1,3,1 +4,24,2,1,3,0 +4,52,1,1,3,0 +4,50,2,1,3,0 +4,35,1,1,3,0 +5,,3,3,3,1 +5,64,4,3,3,1 +5,40,4,1,1,1 +5,66,4,4,3,1 +4,64,4,4,3,1 +5,52,4,3,3,1 +5,43,1,4,3,1 +4,56,4,4,3,0 +4,72,3,,3,0 +6,51,4,4,3,1 +4,79,4,4,3,1 +4,22,2,1,3,0 +4,73,2,1,3,0 +4,53,3,4,3,0 +4,59,2,1,3,1 +4,46,4,4,2,0 +5,66,4,4,3,1 +4,50,4,3,3,1 +4,58,1,1,3,1 +4,55,1,1,3,0 +4,62,2,4,3,1 +4,60,1,1,3,0 +5,57,4,3,3,1 +4,57,1,1,3,0 +6,41,2,1,3,0 +4,71,2,1,3,1 +4,32,2,1,3,0 +4,57,2,1,3,0 +4,19,1,1,3,0 +4,62,2,4,3,1 +5,67,4,5,3,1 +4,50,4,5,3,0 +4,65,2,3,2,0 +4,40,2,4,2,0 +6,71,4,4,3,1 +6,68,4,3,3,1 +4,68,1,1,3,0 +4,29,1,1,3,0 +4,53,2,1,3,0 +5,66,4,4,3,1 +4,60,3,,4,0 +5,76,4,4,3,1 +4,58,2,1,2,0 +5,96,3,4,3,1 +5,70,4,4,3,1 +4,34,2,1,3,0 +4,59,2,1,3,0 +4,45,3,1,3,1 +5,65,4,4,3,1 +4,59,1,1,3,0 +4,21,2,1,3,0 +3,43,2,1,3,0 +4,53,1,1,3,0 +4,65,2,1,3,0 +4,64,2,4,3,1 +4,53,4,4,3,0 +4,51,1,1,3,0 +4,59,2,4,3,0 +4,56,2,1,3,0 +4,60,2,1,3,0 +4,22,1,1,3,0 +4,25,2,1,3,0 +6,76,3,,3,0 +5,69,4,4,3,1 +4,58,2,1,3,0 +5,62,4,3,3,1 +4,56,4,4,3,0 +4,64,1,1,3,0 +4,32,2,1,3,0 +5,48,,4,,1 +5,59,4,4,2,1 +4,52,1,1,3,0 +4,63,4,4,3,0 +5,67,4,4,3,1 +5,61,4,4,3,1 +5,59,4,5,3,1 +5,52,4,3,3,1 +4,35,4,4,3,0 +5,77,3,3,3,1 +5,71,4,3,3,1 +5,63,4,3,3,1 +4,38,2,1,2,0 +5,72,4,3,3,1 +4,76,4,3,3,1 +4,53,3,3,3,0 +4,67,4,5,3,0 +5,69,2,4,3,1 +4,54,1,1,3,0 +2,35,2,1,2,0 +5,68,4,3,3,1 +4,68,4,4,3,0 +4,67,2,4,3,1 +3,39,1,1,3,0 +4,44,2,1,3,0 +4,33,1,1,3,0 +4,60,,4,3,0 +4,58,1,1,3,0 +4,31,1,1,3,0 +3,23,1,1,3,0 +5,56,4,5,3,1 +4,69,2,1,3,1 +6,63,1,1,3,0 +4,65,1,1,3,1 +4,44,2,1,2,0 +4,62,3,3,3,1 +4,67,4,4,3,1 +4,56,2,1,3,0 +4,52,3,4,3,0 +4,43,1,1,3,1 +4,41,4,3,2,1 +4,42,3,4,2,0 +3,46,1,1,3,0 +5,55,4,4,3,1 +5,58,4,4,2,1 +5,87,4,4,3,1 +4,66,2,1,3,0 +0,72,4,3,3,1 +5,60,4,3,3,1 +5,83,4,4,2,1 +4,31,2,1,3,0 +4,53,2,1,3,0 +4,64,2,3,3,0 +5,31,4,4,2,1 +5,62,4,4,2,1 +4,56,2,1,3,0 +5,58,4,4,3,1 +4,67,1,4,3,0 +5,75,4,5,3,1 +5,65,3,4,3,1 +5,74,3,2,3,1 +4,59,2,1,3,0 +4,57,4,4,4,1 +4,76,3,2,3,0 +4,63,1,4,3,0 +4,44,1,1,3,0 +4,42,3,1,2,0 +4,35,3,,2,0 +5,65,4,3,3,1 +4,70,2,1,3,0 +4,48,1,1,3,0 +4,74,1,1,1,1 +6,40,,3,4,1 +4,63,1,1,3,0 +5,60,4,4,3,1 +5,86,4,3,3,1 +4,27,1,1,3,0 +4,71,4,5,2,1 +5,85,4,4,3,1 +4,51,3,3,3,0 +6,72,4,3,3,1 +5,52,4,4,3,1 +4,66,2,1,3,0 +5,71,4,5,3,1 +4,42,2,1,3,0 +4,64,4,4,2,1 +4,41,2,2,3,0 +4,50,2,1,3,0 +4,30,1,1,3,0 +4,67,1,1,3,0 +5,62,4,4,3,1 +4,46,2,1,2,0 +4,35,1,1,3,0 +4,53,1,1,2,0 +4,59,2,1,3,0 +4,19,3,1,3,0 +5,86,2,1,3,1 +4,72,2,1,3,0 +4,37,2,1,2,0 +4,46,3,1,3,1 +4,45,1,1,3,0 +4,48,4,5,3,0 +4,58,4,4,3,1 +4,42,1,1,3,0 +4,56,2,4,3,1 +4,47,2,1,3,0 +4,49,4,4,3,1 +5,76,2,5,3,1 +5,62,4,5,3,1 +5,64,4,4,3,1 +5,53,4,3,3,1 +4,70,4,2,2,1 +5,55,4,4,3,1 +4,34,4,4,3,0 +5,76,4,4,3,1 +4,39,1,1,3,0 +2,23,1,1,3,0 +4,19,1,1,3,0 +5,65,4,5,3,1 +4,57,2,1,3,0 +5,41,4,4,3,1 +4,36,4,5,3,1 +4,62,3,3,3,0 +4,69,2,1,3,0 +4,41,3,1,3,0 +3,51,2,4,3,0 +5,50,3,2,3,1 +4,47,4,4,3,0 +4,54,4,5,3,1 +5,52,4,4,3,1 +4,30,1,1,3,0 +3,48,4,4,3,1 +5,,4,4,3,1 +4,65,2,4,3,1 +4,50,1,1,3,0 +5,65,4,5,3,1 +5,66,4,3,3,1 +6,41,3,3,2,1 +5,72,3,2,3,1 +4,42,1,1,1,1 +4,80,4,4,3,1 +0,45,2,4,3,0 +4,41,1,1,3,0 +4,72,3,3,3,1 +4,60,4,5,3,0 +5,67,4,3,3,1 +4,55,2,1,3,0 +4,61,3,4,3,1 +4,55,3,4,3,1 +4,52,4,4,3,1 +4,42,1,1,3,0 +5,63,4,4,3,1 +4,62,4,5,3,1 +4,46,1,1,3,0 +4,65,2,1,3,0 +4,57,3,3,3,1 +4,66,4,5,3,1 +4,45,1,1,3,0 +4,77,4,5,3,1 +4,35,1,1,3,0 +4,50,4,5,3,1 +4,57,4,4,3,0 +4,74,3,1,3,1 +4,59,4,5,3,0 +4,51,1,1,3,0 +4,42,3,4,3,1 +4,35,2,4,3,0 +4,42,1,1,3,0 +4,43,2,1,3,0 +4,62,4,4,3,1 +4,27,2,1,3,0 +5,,4,3,3,1 +4,57,4,4,3,1 +4,59,2,1,3,0 +5,40,3,2,3,1 +4,20,1,1,3,0 +5,74,4,3,3,1 +4,22,1,1,3,0 +4,57,4,3,3,0 +4,57,4,3,3,1 +4,55,2,1,2,0 +4,62,2,1,3,0 +4,54,1,1,3,0 +4,71,1,1,3,1 +4,65,3,3,3,0 +4,68,4,4,3,0 +4,64,1,1,3,0 +4,54,2,4,3,0 +4,48,4,4,3,1 +4,58,4,3,3,0 +5,58,3,4,3,1 +4,70,1,1,1,0 +5,70,1,4,3,1 +4,59,2,1,3,0 +4,57,2,4,3,0 +4,53,4,5,3,0 +4,54,4,4,3,1 +4,53,2,1,3,0 +0,71,4,4,3,1 +5,67,4,5,3,1 +4,68,4,4,3,1 +4,56,2,4,3,0 +4,35,2,1,3,0 +4,52,4,4,3,1 +4,47,2,1,3,0 +4,56,4,5,3,1 +4,64,4,5,3,0 +5,66,4,5,3,1 +4,62,3,3,3,0 From 7bc4d1609eac82af823228f7a010ce302fcc259a Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 17 Jun 2024 06:27:52 -0700 Subject: [PATCH 23/30] Anomaly detection; Fast c++ inference of Isolation forest models (part 5) PiperOrigin-RevId: 643985602 --- .../serving/decision_forest/BUILD | 20 ++++- .../decision_forest/decision_forest.cc | 47 +++++++++++ .../serving/decision_forest/decision_forest.h | 8 +- .../decision_forest_serving.cc | 34 ++++++++ .../decision_forest/decision_forest_serving.h | 12 +++ .../decision_forest/decision_forest_test.cc | 21 +++++ .../decision_forest/register_engines.cc | 80 ++++++++++++++++++- .../decision_forest/register_engines.h | 6 +- .../serving/decision_forest/utils.cc | 16 +++- 9 files changed, 237 insertions(+), 7 deletions(-) diff --git a/yggdrasil_decision_forests/serving/decision_forest/BUILD b/yggdrasil_decision_forests/serving/decision_forest/BUILD index 6770a078..0743c5cb 100644 --- a/yggdrasil_decision_forests/serving/decision_forest/BUILD +++ b/yggdrasil_decision_forests/serving/decision_forest/BUILD @@ -62,14 +62,22 @@ cc_library_ydf( ], deps = [ ":decision_forest", + ":decision_forest_serving", ":quick_scorer_extended", "//yggdrasil_decision_forests/dataset:data_spec_cc_proto", "//yggdrasil_decision_forests/model:abstract_model", + "//yggdrasil_decision_forests/model:abstract_model_cc_proto", + "//yggdrasil_decision_forests/model/decision_tree", "//yggdrasil_decision_forests/model/gradient_boosted_trees", "//yggdrasil_decision_forests/model/gradient_boosted_trees:gradient_boosted_trees_cc_proto", + "//yggdrasil_decision_forests/model/isolation_forest", + "//yggdrasil_decision_forests/model/random_forest", "//yggdrasil_decision_forests/serving:example_set_model_wrapper", "//yggdrasil_decision_forests/serving:fast_engine", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", ], alwayslink = 1, ) @@ -91,6 +99,7 @@ cc_library_ydf( "//yggdrasil_decision_forests/model/decision_tree", "//yggdrasil_decision_forests/model/decision_tree:decision_tree_cc_proto", "//yggdrasil_decision_forests/model/gradient_boosted_trees", + "//yggdrasil_decision_forests/model/isolation_forest", "//yggdrasil_decision_forests/model/random_forest", "//yggdrasil_decision_forests/serving:example_set", "//yggdrasil_decision_forests/utils:bitmap", @@ -103,6 +112,7 @@ cc_library_ydf( "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", ], + alwayslink = 1, ) cc_library_ydf( @@ -115,10 +125,13 @@ cc_library_ydf( ], deps = [ "//yggdrasil_decision_forests/model:abstract_model_cc_proto", + "//yggdrasil_decision_forests/model/isolation_forest", "//yggdrasil_decision_forests/serving:example_set", + "//yggdrasil_decision_forests/utils:compatibility", "//yggdrasil_decision_forests/utils:logging", "//yggdrasil_decision_forests/utils:usage", ], + alwayslink = 1, ) cc_library_ydf( @@ -157,10 +170,12 @@ cc_library_ydf( "//yggdrasil_decision_forests/dataset:data_spec_cc_proto", "//yggdrasil_decision_forests/model:abstract_model", "//yggdrasil_decision_forests/model/gradient_boosted_trees", + "//yggdrasil_decision_forests/model/isolation_forest", "//yggdrasil_decision_forests/model/random_forest", + "//yggdrasil_decision_forests/serving:example_set", + "//yggdrasil_decision_forests/utils:logging", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:str_format", ], ) @@ -209,6 +224,9 @@ cc_test( "//yggdrasil_decision_forests/model/decision_tree:decision_tree_cc_proto", "//yggdrasil_decision_forests/model/gradient_boosted_trees", "//yggdrasil_decision_forests/model/gradient_boosted_trees:gradient_boosted_trees_cc_proto", + "//yggdrasil_decision_forests/model/isolation_forest", + "//yggdrasil_decision_forests/model/random_forest", + "//yggdrasil_decision_forests/serving:example_set", "//yggdrasil_decision_forests/utils:concurrency", "//yggdrasil_decision_forests/utils:csv", "//yggdrasil_decision_forests/utils:filesystem", diff --git a/yggdrasil_decision_forests/serving/decision_forest/decision_forest.cc b/yggdrasil_decision_forests/serving/decision_forest/decision_forest.cc index 76a318fb..7218b77b 100644 --- a/yggdrasil_decision_forests/serving/decision_forest/decision_forest.cc +++ b/yggdrasil_decision_forests/serving/decision_forest/decision_forest.cc @@ -37,8 +37,10 @@ #include "yggdrasil_decision_forests/model/decision_tree/decision_tree.h" #include "yggdrasil_decision_forests/model/decision_tree/decision_tree.pb.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" #include "yggdrasil_decision_forests/model/random_forest/random_forest.h" #include "yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.h" +#include "yggdrasil_decision_forests/serving/decision_forest/utils.h" #include "yggdrasil_decision_forests/serving/example_set.h" #include "yggdrasil_decision_forests/utils/bitmap.h" #include "yggdrasil_decision_forests/utils/compatibility.h" @@ -54,6 +56,7 @@ using dataset::proto::ColumnType; using model::decision_tree::NodeWithChildren; using model::gradient_boosted_trees::GradientBoostedTreesModel; using model::gradient_boosted_trees::proto::Loss; +using model::isolation_forest::IsolationForestModel; using model::random_forest::RandomForestModel; using ConditionType = model::decision_tree::proto::Condition::TypeCase; typedef absl::flat_hash_map FeatureDefMap; @@ -634,6 +637,30 @@ absl::Status SetLeafNodeRandomForestNumericalUplift( return absl::OkStatus(); } +template +absl::Status SetLeafNodeIsolationForest( + const IsolationForestModel& src_model, const NodeWithChildren& src_node, + SpecializedModel* dst_model, + typename SpecializedModel::NodeType* dst_node) { + using Node = typename SpecializedModel::NodeType; + static_assert(std::is_same>::value || + std::is_same>::value, + "Non supported node type."); + + const float value = + (src_node.depth() + + model::isolation_forest::PreissAveragePathLength( + src_node.node().anomaly_detection().num_examples_without_weight())) / + src_model.NumTrees(); + + *dst_node = Node::Leaf( + /*.right_idx =*/0, + /*.feature_idx =*/0, + /*.label =*/ + value); + return absl::OkStatus(); +} + // Set the leaf of a binary classification Gradient Boosted Trees. template absl::Status SetLeafGradientBoostedTreesClassification( @@ -1051,6 +1078,26 @@ absl::Status GenericToSpecializedModel( SetLeafNodeRandomForestNumericalUplift, src, dst); } +template <> +absl::Status GenericToSpecializedModel(const IsolationForestModel& src, + IsolationForest* dst) { + using DstType = std::remove_pointer::type; + dst->denominator = model::isolation_forest::PreissAveragePathLength( + src.num_examples_per_trees()); + return GenericToSpecializedGenericModelHelper( + SetLeafNodeIsolationForest, src, dst); +} + +template <> +absl::Status GenericToSpecializedModel(const IsolationForestModel& src, + GenericIsolationForest* dst) { + using DstType = std::remove_pointer::type; + dst->denominator = model::isolation_forest::PreissAveragePathLength( + src.num_examples_per_trees()); + return GenericToSpecializedGenericModelHelper( + SetLeafNodeIsolationForest, src, dst); +} + template <> absl::Status GenericToSpecializedModel( const GradientBoostedTreesModel& src, diff --git a/yggdrasil_decision_forests/serving/decision_forest/decision_forest.h b/yggdrasil_decision_forests/serving/decision_forest/decision_forest.h index b03ed309..4167463c 100644 --- a/yggdrasil_decision_forests/serving/decision_forest/decision_forest.h +++ b/yggdrasil_decision_forests/serving/decision_forest/decision_forest.h @@ -47,11 +47,17 @@ #ifndef YGGDRASIL_DECISION_FORESTS_SERVING_DECISION_FOREST_H_ #define YGGDRASIL_DECISION_FORESTS_SERVING_DECISION_FOREST_H_ +#include +#include +#include + #include "absl/status/status.h" +#include "absl/types/optional.h" +#include "yggdrasil_decision_forests/dataset/vertical_dataset.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.h" #include "yggdrasil_decision_forests/model/random_forest/random_forest.h" #include "yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.h" -#include "yggdrasil_decision_forests/serving/decision_forest/utils.h" +#include "yggdrasil_decision_forests/serving/example_set.h" namespace yggdrasil_decision_forests { namespace serving { diff --git a/yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.cc b/yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.cc index 8b65a1f4..12f2e33c 100644 --- a/yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.cc +++ b/yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.cc @@ -15,6 +15,15 @@ #include "yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.h" +#include +#include +#include +#include +#include + +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" +#include "yggdrasil_decision_forests/serving/example_set.h" +#include "yggdrasil_decision_forests/utils/compatibility.h" #include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/usage.h" @@ -61,6 +70,12 @@ void ActivationGradientBoostedTreesMultinomialLogLikelihood( } } +template +float IsolationForestActivation(const Model& model, const float value) { + return model::isolation_forest::IsolationForestPredictionFromDenominator( + value, model.denominator); +} + // Identity transformation for the output of a decision forest model. // Default value for the "FinalTransform" argument in "PredictHelper". // @@ -647,6 +662,25 @@ void Predict( model, examples, num_examples, predictions); } +template <> +void Predict( + const GenericIsolationForest& model, + const typename GenericIsolationForest::ExampleSet& examples, + int num_examples, std::vector* predictions) { + PredictHelper::type, + IsolationForestActivation>(model, examples, num_examples, + predictions); +} + +template <> +void Predict(const IsolationForest& model, + const typename IsolationForest::ExampleSet& examples, + int num_examples, std::vector* predictions) { + PredictHelper::type, + IsolationForestActivation>(model, examples, num_examples, + predictions); +} + template <> void Predict( const GradientBoostedTreesBinaryClassification& model, diff --git a/yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.h b/yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.h index b73aceaa..2b197c76 100644 --- a/yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.h +++ b/yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.h @@ -16,6 +16,7 @@ #ifndef YGGDRASIL_DECISION_FORESTS_SERVING_DECISION_FOREST_DECISION_FOREST_SERVING_H_ #define YGGDRASIL_DECISION_FORESTS_SERVING_DECISION_FOREST_DECISION_FOREST_SERVING_H_ +#include #include #include "yggdrasil_decision_forests/model/abstract_model.pb.h" @@ -386,6 +387,17 @@ struct GenericRandomForestNumericalUplift : ExampleSetModel { }; using RandomForestNumericalUplift = GenericRandomForestNumericalUplift<>; +// Isolation Forest model. +template +struct GenericIsolationForest : ExampleSetModel { + static constexpr model::proto::Task kTask = + model::proto::Task::ANOMALY_DETECTION; + + // Denominator / normalizer of the prediction output. + float denominator; +}; +using IsolationForest = GenericIsolationForest<>; + // GBDT model for binary classification. template struct GenericGradientBoostedTreesBinaryClassification diff --git a/yggdrasil_decision_forests/serving/decision_forest/decision_forest_test.cc b/yggdrasil_decision_forests/serving/decision_forest/decision_forest_test.cc index 02de472f..db3fb9b8 100644 --- a/yggdrasil_decision_forests/serving/decision_forest/decision_forest_test.cc +++ b/yggdrasil_decision_forests/serving/decision_forest/decision_forest_test.cc @@ -15,8 +15,11 @@ #include "yggdrasil_decision_forests/serving/decision_forest/decision_forest.h" +#include +#include #include #include +#include #include #include @@ -39,11 +42,15 @@ #include "yggdrasil_decision_forests/model/fast_engine_factory.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.pb.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" #include "yggdrasil_decision_forests/model/model_library.h" +#include "yggdrasil_decision_forests/model/random_forest/random_forest.h" #include "yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.h" #include "yggdrasil_decision_forests/serving/decision_forest/quick_scorer_extended.h" #include "yggdrasil_decision_forests/serving/decision_forest/register_engines.h" +#include "yggdrasil_decision_forests/serving/example_set.h" #include "yggdrasil_decision_forests/utils/concurrency.h" // IWYU pragma: keep +#include "yggdrasil_decision_forests/utils/concurrency_streamprocessor.h" #include "yggdrasil_decision_forests/utils/csv.h" #include "yggdrasil_decision_forests/utils/filesystem.h" #include "yggdrasil_decision_forests/utils/logging.h" @@ -57,6 +64,7 @@ namespace decision_forest { namespace { using model::gradient_boosted_trees::GradientBoostedTreesModel; +using model::isolation_forest::IsolationForestModel; using model::random_forest::RandomForestModel; using testing::ElementsAre; @@ -488,6 +496,19 @@ TEST(SimPTECategoricalupliftRF, ManualGeneric) { dataset, *model, engine); } +TEST(GaussiansIF, ManualGeneric) { + const auto model = LoadModel("gaussians_anomaly_if"); + const auto dataset = + LoadDataset(model->data_spec(), "gaussians_test.csv", "csv"); + + auto* if_model = dynamic_cast(model.get()); + IsolationForest engine; + CHECK_OK(GenericToSpecializedModel(*if_model, &engine)); + + utils::ExpectEqualPredictionsTemplate( + dataset, *model, engine); +} + void BuildFullTree(const int d, model::decision_tree::NodeWithChildren* node) { if (d <= 0) { node->mutable_node()->mutable_classifier()->set_top_value(1.f); diff --git a/yggdrasil_decision_forests/serving/decision_forest/register_engines.cc b/yggdrasil_decision_forests/serving/decision_forest/register_engines.cc index aab9da8e..29f71339 100644 --- a/yggdrasil_decision_forests/serving/decision_forest/register_engines.cc +++ b/yggdrasil_decision_forests/serving/decision_forest/register_engines.cc @@ -17,15 +17,33 @@ // #include "yggdrasil_decision_forests/serving/decision_forest/register_engines.h" +#include +#include +#include +#include +#include +#include +#include + +#include "absl/memory/memory.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" #include "yggdrasil_decision_forests/model/abstract_model.h" +#include "yggdrasil_decision_forests/model/abstract_model.pb.h" +#include "yggdrasil_decision_forests/model/decision_tree/decision_tree.h" #include "yggdrasil_decision_forests/model/fast_engine_factory.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.pb.h" -#include "yggdrasil_decision_forests/serving/decision_forest/decision_forest.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" +#include "yggdrasil_decision_forests/model/random_forest/random_forest.h" +#include "yggdrasil_decision_forests/serving/decision_forest/decision_forest.h" // IWYU pragma: keep +#include "yggdrasil_decision_forests/serving/decision_forest/decision_forest_serving.h" #include "yggdrasil_decision_forests/serving/decision_forest/quick_scorer_extended.h" #include "yggdrasil_decision_forests/serving/example_set_model_wrapper.h" +#include "yggdrasil_decision_forests/serving/fast_engine.h" namespace yggdrasil_decision_forests { namespace model { @@ -609,6 +627,66 @@ class RandomForestGenericFastEngineFactory : public model::FastEngineFactory { REGISTER_FastEngineFactory(RandomForestGenericFastEngineFactory, serving::random_forest::kGeneric); +class IsolationForestGenericFastEngineFactory + : public model::FastEngineFactory { + public: + using SourceModel = isolation_forest::IsolationForestModel; + + std::string name() const override { + return serving::isolation_forest::kGeneric; + } + + bool IsCompatible(const AbstractModel* const model) const override { + auto* if_model = dynamic_cast(model); + // This implementation is the most generic and least efficient engine. + if (if_model == nullptr) { + return false; + } + return if_model->CheckStructure({/*.global_imputation_is_higher =*/false}); + } + + std::vector IsBetterThan() const override { return {}; } + + absl::StatusOr> CreateEngine( + const AbstractModel* const model) const override { + auto* if_model = dynamic_cast(model); + if (!if_model) { + return absl::InvalidArgumentError("The model is not an IF."); + } + + if (!if_model->CheckStructure({/*.global_imputation_is_higher =*/false})) { + return NoGlobalImputationError("IsolationForestGenericFastEngineFactory"); + } + + const bool need_uint32_node_index = + MaxNumberOfNodesPerTree(if_model->decision_trees()) >= + std::numeric_limits::max(); + + switch (if_model->task()) { + case model::proto::ANOMALY_DETECTION: { + if (need_uint32_node_index) { + auto engine = absl::make_unique, + serving::decision_forest::Predict>>(); + RETURN_IF_ERROR(engine->LoadModel(*if_model)); + return engine; + } else { + auto engine = absl::make_unique, + serving::decision_forest::Predict>>(); + RETURN_IF_ERROR(engine->LoadModel(*if_model)); + return engine; + } + } + default: + return absl::InvalidArgumentError("Non supported RF model"); + } + } +}; + +REGISTER_FastEngineFactory(IsolationForestGenericFastEngineFactory, + serving::isolation_forest::kGeneric); + class RandomForestOptPredFastEngineFactory : public model::FastEngineFactory { public: using SourceModel = random_forest::RandomForestModel; diff --git a/yggdrasil_decision_forests/serving/decision_forest/register_engines.h b/yggdrasil_decision_forests/serving/decision_forest/register_engines.h index 50edccc9..b7dc89d5 100644 --- a/yggdrasil_decision_forests/serving/decision_forest/register_engines.h +++ b/yggdrasil_decision_forests/serving/decision_forest/register_engines.h @@ -18,8 +18,6 @@ #ifndef YGGDRASIL_DECISION_FORESTS_SERVING_REGISTER_ENGINE_DECISION_FOREST_H_ #define YGGDRASIL_DECISION_FORESTS_SERVING_REGISTER_ENGINE_DECISION_FOREST_H_ -#include "yggdrasil_decision_forests/model/abstract_model.h" -#include "yggdrasil_decision_forests/serving/fast_engine.h" namespace yggdrasil_decision_forests { namespace serving { @@ -36,6 +34,10 @@ constexpr char kGeneric[] = "RandomForestGeneric"; constexpr char kOptPred[] = "RandomForestOptPred"; } // namespace random_forest +namespace isolation_forest { +constexpr char kGeneric[] = "IsolationForestGeneric"; +} // namespace isolation_forest + } // namespace serving } // namespace yggdrasil_decision_forests #endif // YGGDRASIL_DECISION_FORESTS_SERVING_REGISTER_ENGINE_DECISION_FOREST_H_ diff --git a/yggdrasil_decision_forests/serving/decision_forest/utils.cc b/yggdrasil_decision_forests/serving/decision_forest/utils.cc index a459f54b..710989ec 100644 --- a/yggdrasil_decision_forests/serving/decision_forest/utils.cc +++ b/yggdrasil_decision_forests/serving/decision_forest/utils.cc @@ -15,17 +15,24 @@ #include "yggdrasil_decision_forests/serving/decision_forest/utils.h" +#include +#include +#include +#include + #include "absl/status/status.h" -#include "absl/strings/str_format.h" #include "yggdrasil_decision_forests/dataset/data_spec.pb.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" #include "yggdrasil_decision_forests/model/random_forest/random_forest.h" +#include "yggdrasil_decision_forests/serving/example_set.h" +#include "yggdrasil_decision_forests/utils/logging.h" namespace yggdrasil_decision_forests { namespace serving { namespace decision_forest { - // Get the list of input features used by the model. // // The order of the input feature is deterministic. @@ -46,10 +53,15 @@ absl::Status GetInputFeatures( const auto* gbt_model = dynamic_cast< const model::gradient_boosted_trees::GradientBoostedTreesModel*>( &src_model); + const auto* if_model = + dynamic_cast( + &src_model); if (rf_model) { rf_model->CountFeatureUsage(&feature_usage); } else if (gbt_model) { gbt_model->CountFeatureUsage(&feature_usage); + } else if (if_model) { + if_model->CountFeatureUsage(&feature_usage); } else { return absl::InvalidArgumentError("Unsupported decision forest model type"); } From 86f7edf8e7224d08c76794340da342c3cdae6a33 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 17 Jun 2024 09:46:57 -0700 Subject: [PATCH 24/30] Anomaly detection; Enable model and prediction analysis of isolation forest models (part 6) PiperOrigin-RevId: 644041112 --- .../golden/analyze_model_if.html.expected | 676 ++++++++++++++++++ .../analyze_prediction_if.html.expected | 231 ++++++ .../model/gaussians_anomaly_if/header.pb | Bin 67 -> 61 bytes yggdrasil_decision_forests/utils/BUILD | 1 + .../utils/model_analysis.cc | 31 +- .../utils/model_analysis.proto | 3 + .../utils/model_analysis_test.cc | 59 +- .../utils/partial_dependence_plot.cc | 14 + .../utils/partial_dependence_plot.proto | 6 +- 9 files changed, 1012 insertions(+), 9 deletions(-) create mode 100644 yggdrasil_decision_forests/test_data/golden/analyze_model_if.html.expected create mode 100644 yggdrasil_decision_forests/test_data/golden/analyze_prediction_if.html.expected diff --git a/yggdrasil_decision_forests/test_data/golden/analyze_model_if.html.expected b/yggdrasil_decision_forests/test_data/golden/analyze_model_if.html.expected new file mode 100644 index 00000000..eba95d49 --- /dev/null +++ b/yggdrasil_decision_forests/test_data/golden/analyze_model_if.html.expected @@ -0,0 +1,676 @@ + + + + +
SetupDatasetPartial Dependence PlotConditional Expectation PlotModel Description

Analyse dataset: MODEL_PATH

Model: DATASET_PATH

Number of records: 2
+Number of columns: 2
+
+Number of columns by type:
+	NUMERICAL: 2 (100%)
+
+Columns:
+
+NUMERICAL: 2 (100%)
+	0: "features.0_of_2" NUMERICAL mean:0 min:0 max:0 sd:0 dtype:DTYPE_FLOAT64
+	1: "features.1_of_2" NUMERICAL mean:0 min:0 max:0 sd:0 dtype:DTYPE_FLOAT64
+
+Terminology:
+	nas: Number of non-available (i.e. missing) values.
+	ood: Out of dictionary.
+	manually-defined: Attribute whose type is manually defined by the user, i.e., the type was not automatically inferred.
+	tokenized: The attribute value is obtained through tokenization.
+	has-dict: The attribute is attached to a string dictionary e.g. a categorical attribute stored as a string.
+	vocab-size: Number of unique values.
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
Type: "ISOLATION_FOREST"
+Task: ANOMALY_DETECTION
+
+Input Features (2):
+	features.0_of_2
+	features.1_of_2
+
+No weights
+
+Variable Importance disabled i.e. compute_oob_variable_importances=false.
+Cannot compute model self evaluation:This model does not support evaluation reports.
+
+Number of trees: 100
+Total number of nodes: 6488
+
+Number of nodes by tree:
+Count: 100 Average: 64.88 StdDev: 15.1758
+Min: 33 Max: 101 Ignored: 0
+----------------------------------------------
+[  33,  36)  1   1.00%   1.00% #
+[  36,  39)  1   1.00%   2.00% #
+[  39,  43)  4   4.00%   6.00% ###
+[  43,  46)  6   6.00%  12.00% ####
+[  46,  50)  4   4.00%  16.00% ###
+[  50,  53)  2   2.00%  18.00% #
+[  53,  57) 11  11.00%  29.00% ########
+[  57,  60)  9   9.00%  38.00% ######
+[  60,  64) 14  14.00%  52.00% ##########
+[  64,  67)  8   8.00%  60.00% ######
+[  67,  70)  5   5.00%  65.00% ####
+[  70,  74) 11  11.00%  76.00% ########
+[  74,  77)  4   4.00%  80.00% ###
+[  77,  81)  3   3.00%  83.00% ##
+[  81,  84)  4   4.00%  87.00% ###
+[  84,  88)  4   4.00%  91.00% ###
+[  88,  91)  2   2.00%  93.00% #
+[  91,  95)  3   3.00%  96.00% ##
+[  95,  98)  0   0.00%  96.00%
+[  98, 101]  4   4.00% 100.00% ###
+
+Depth by leafs:
+Count: 3294 Average: 5.92077 StdDev: 1.37529
+Min: 1 Max: 7 Ignored: 0
+----------------------------------------------
+[ 1, 2)   13   0.39%   0.39%
+[ 2, 3)   56   1.70%   2.09%
+[ 3, 4)  189   5.74%   7.83% #
+[ 4, 5)  308   9.35%  17.18% ##
+[ 5, 6)  443  13.45%  30.63% ###
+[ 6, 7)  631  19.16%  49.79% ####
+[ 7, 7] 1654  50.21% 100.00% ##########
+
+Number of training obs by leaf:
+Count: 3294 Average: 0 StdDev: 0
+Min: 0 Max: 0 Ignored: 0
+----------------------------------------------
+[ 0, 0] 3294 100.00% 100.00% ##########
+
+Attribute in nodes:
+	1617 : features.0_of_2 [NUMERICAL]
+	1577 : features.1_of_2 [NUMERICAL]
+
+Attribute in nodes with depth <= 0:
+	61 : features.0_of_2 [NUMERICAL]
+	39 : features.1_of_2 [NUMERICAL]
+
+Attribute in nodes with depth <= 1:
+	148 : features.0_of_2 [NUMERICAL]
+	139 : features.1_of_2 [NUMERICAL]
+
+Attribute in nodes with depth <= 2:
+	311 : features.0_of_2 [NUMERICAL]
+	294 : features.1_of_2 [NUMERICAL]
+
+Attribute in nodes with depth <= 3:
+	534 : features.0_of_2 [NUMERICAL]
+	518 : features.1_of_2 [NUMERICAL]
+
+Attribute in nodes with depth <= 5:
+	1206 : features.0_of_2 [NUMERICAL]
+	1161 : features.1_of_2 [NUMERICAL]
+
+Condition type in nodes:
+	3194 : HigherCondition
+Condition type in nodes with depth <= 0:
+	100 : HigherCondition
+Condition type in nodes with depth <= 1:
+	287 : HigherCondition
+Condition type in nodes with depth <= 2:
+	605 : HigherCondition
+Condition type in nodes with depth <= 3:
+	1052 : HigherCondition
+Condition type in nodes with depth <= 5:
+	2367 : HigherCondition
+Node format: BLOB_SEQUENCE
+Number of examples per tree: 100
+
\ No newline at end of file diff --git a/yggdrasil_decision_forests/test_data/golden/analyze_prediction_if.html.expected b/yggdrasil_decision_forests/test_data/golden/analyze_prediction_if.html.expected new file mode 100644 index 00000000..367dbae2 --- /dev/null +++ b/yggdrasil_decision_forests/test_data/golden/analyze_prediction_if.html.expected @@ -0,0 +1,231 @@ + + + + +
Feature Variation
+
+ + +
+ +
\ No newline at end of file diff --git a/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/header.pb b/yggdrasil_decision_forests/test_data/model/gaussians_anomaly_if/header.pb index f33587675c6e6ba1db768ba8071888abce2eeceb..7904759d1ef5bf2117fd4044a85730adfe3a1011 100644 GIT binary patch delta 14 VcmZ>^ogmMw!JsiwPLe5s0RR@N0<8c5 delta 41 jcmcC@o*=Jc@E-;k9T+`e?Ej2G99#?n3=#}V3<(SX1}qf| diff --git a/yggdrasil_decision_forests/utils/BUILD b/yggdrasil_decision_forests/utils/BUILD index 5f42c7b5..68dcb435 100644 --- a/yggdrasil_decision_forests/utils/BUILD +++ b/yggdrasil_decision_forests/utils/BUILD @@ -1219,6 +1219,7 @@ cc_test( "//yggdrasil_decision_forests/model/decision_tree", "//yggdrasil_decision_forests/model/decision_tree:builder", "//yggdrasil_decision_forests/model/gradient_boosted_trees", + "//yggdrasil_decision_forests/model/isolation_forest", "//yggdrasil_decision_forests/model/random_forest", "//yggdrasil_decision_forests/utils:testing_macros", "@com_google_absl//absl/memory", diff --git a/yggdrasil_decision_forests/utils/model_analysis.cc b/yggdrasil_decision_forests/utils/model_analysis.cc index b111e3f1..1f347b80 100644 --- a/yggdrasil_decision_forests/utils/model_analysis.cc +++ b/yggdrasil_decision_forests/utils/model_analysis.cc @@ -188,6 +188,19 @@ absl::Status Set1DCurveData( } break; + case model::proto::Task::ANOMALY_DETECTION: + STATUS_CHECK_EQ(label_value_idx, -1); + switch (target_type) { + case CurveTargetType::kPrediction: + target_dst->push_back( + bin.prediction().sum_of_anomaly_detection_predictions() / + pdp.num_observations()); + break; + default: + return absl::InvalidArgumentError("Not implemented."); + } + break; + default: return absl::InvalidArgumentError("Not implemented."); } @@ -207,7 +220,6 @@ absl::Status PlotPartialDependencePlot1DNumerical( pdp.pdp_bins_size()); const auto& attr_spec = data_spec.columns(attribute_idx); - const auto& label_spec = data_spec.columns(label_col_idx); STATUS_CHECK_EQ(attr_spec.type(), dataset::proto::ColumnType::NUMERICAL); @@ -226,6 +238,7 @@ absl::Status PlotPartialDependencePlot1DNumerical( // PDP switch (task) { case model::proto::Task::CLASSIFICATION: { + const auto& label_spec = data_spec.columns(label_col_idx); for (int label_value_idx = FirstCategoricalLabelValueForPdpPlot(label_spec); label_value_idx < label_spec.categorical().number_of_unique_values(); @@ -308,6 +321,15 @@ absl::Status PlotPartialDependencePlot1DNumerical( -1, dataset::proto::ColumnType::NUMERICAL, prediction_curve)); } break; + case model::proto::Task::ANOMALY_DETECTION: { + auto* prediction_curve = AddCurve(pdp_plot); + prediction_curve->style = plot::LineStyle::SOLID; + RETURN_IF_ERROR(Set1DCurveData(pdp, CurveTargetType::kPrediction, false, + model::proto::Task::ANOMALY_DETECTION, -1, + dataset::proto::ColumnType::NUMERICAL, + prediction_curve)); + } break; + default: return absl::InvalidArgumentError("Not implemented"); } @@ -735,7 +757,8 @@ absl::StatusOr Analyse( options.cep().example_sampling())); } - if (options.permuted_variable_importance().enabled()) { + if (options.permuted_variable_importance().enabled() && + model.label_col_idx() != -1) { RETURN_IF_ERROR(ComputePermutationFeatureImportance( dataset, &model, analysis.mutable_variable_importances(), {options.num_threads(), @@ -1041,10 +1064,10 @@ absl::StatusOr> ListOutputs( const proto::PredictionAnalysisResult& analysis, const proto::PredictionAnalysisOptions& options) { std::vector outputs; - const auto& label_column = - analysis.data_spec().columns(analysis.label_col_idx()); switch (analysis.task()) { case model::proto::Task::CLASSIFICATION: { + const auto& label_column = + analysis.data_spec().columns(analysis.label_col_idx()); const int first_class_idx = (label_column.categorical().number_of_unique_values() == 3) ? 2 : 1; for (int class_idx = first_class_idx; diff --git a/yggdrasil_decision_forests/utils/model_analysis.proto b/yggdrasil_decision_forests/utils/model_analysis.proto index e5ea294d..422e63ad 100644 --- a/yggdrasil_decision_forests/utils/model_analysis.proto +++ b/yggdrasil_decision_forests/utils/model_analysis.proto @@ -48,6 +48,9 @@ message Options { optional PermutedVariableImportance permuted_variable_importance = 7; message PermutedVariableImportance { + // If the model does not have labels (e.g., anomaly detection without + // labels), permutation variable importances are not computed, even if + // enabled=True. optional bool enabled = 1 [default = true]; // Number of repetitions of the estimation. More repetitions increase the diff --git a/yggdrasil_decision_forests/utils/model_analysis_test.cc b/yggdrasil_decision_forests/utils/model_analysis_test.cc index e124f299..db9e6f6c 100644 --- a/yggdrasil_decision_forests/utils/model_analysis_test.cc +++ b/yggdrasil_decision_forests/utils/model_analysis_test.cc @@ -56,7 +56,7 @@ std::string ModelDir() { "yggdrasil_decision_forests/test_data/model"); } -TEST(ModelAnalysis, Basic) { +TEST(ModelAnalysis, Classification) { const std::string dataset_path = absl::StrCat("csv:", file::JoinPath(DatasetDir(), "adult_test.csv")); const std::string model_path = @@ -70,7 +70,7 @@ TEST(ModelAnalysis, Basic) { dataset::VerticalDataset dataset; CHECK_OK(dataset::LoadVerticalDataset( dataset_path, model->data_spec(), &dataset, - /*ensure_non_missing=*/model->input_features())); + /*required_columns=*/model->input_features())); proto::Options options; options.mutable_pdp()->set_example_sampling(0.01f); @@ -87,6 +87,35 @@ TEST(ModelAnalysis, Basic) { "DATASET_PATH", analysis, options)); } +TEST(ModelAnalysis, AnomalyDetection) { + const std::string dataset_path = + absl::StrCat("csv:", file::JoinPath(DatasetDir(), "gaussians_test.csv")); + const std::string model_path = + file::JoinPath(ModelDir(), "gaussians_anomaly_if"); + + ASSERT_OK_AND_ASSIGN(const auto model, model::LoadModel(model_path)); + + dataset::VerticalDataset dataset; + ASSERT_OK(dataset::LoadVerticalDataset( + dataset_path, model->data_spec(), &dataset, + /*required_columns=*/model->input_features())); + + proto::Options options; + options.mutable_pdp()->set_example_sampling(0.01f); + options.mutable_cep()->set_example_sampling(0.1f); + options.set_num_threads(1); + options.set_html_id_prefix("my_report"); + options.mutable_report_header()->set_enabled(false); + const auto report_path = file::JoinPath(test::TmpDirectory(), "analysis"); + + ASSERT_OK_AND_ASSIGN(const auto analysis, + Analyse(*model.get(), dataset, options)); + + ASSERT_OK_AND_ASSIGN(const auto report, + CreateHtmlReport(*model.get(), dataset, "MODEL_PATH", + "DATASET_PATH", analysis, options)); +} + TEST(ModelAnalysis, FailsWithEmptyDataset) { const std::string dataset_path = absl::StrCat("csv:", file::JoinPath(DatasetDir(), "adult_test.csv")); @@ -202,7 +231,7 @@ TEST(ModelAnalysis, PDPPlot) { // TODO: Add a more extensive unit test with a golden report. } -TEST(PredictionAnalysis, Basic) { +TEST(PredictionAnalysis, Classification) { const std::string dataset_path = absl::StrCat("csv:", file::JoinPath(DatasetDir(), "adult_test.csv")); const std::string model_path = @@ -216,7 +245,29 @@ TEST(PredictionAnalysis, Basic) { dataset::VerticalDataset dataset; ASSERT_OK(dataset::LoadVerticalDataset( dataset_path, model->data_spec(), &dataset, - /*ensure_non_missing=*/model->input_features())); + /*required_columns=*/model->input_features())); + + proto::PredictionAnalysisOptions options; + options.set_html_id_prefix("my_prefix"); + dataset::proto::Example example; + dataset.ExtractExample(0, &example); + ASSERT_OK_AND_ASSIGN(const auto analysis, + AnalyzePrediction(*model, example, options)); + ASSERT_OK_AND_ASSIGN(const auto report, CreateHtmlReport(analysis, options)); +} + +TEST(PredictionAnalysis, AnomalyDetection) { + const std::string dataset_path = + absl::StrCat("csv:", file::JoinPath(DatasetDir(), "gaussians_test.csv")); + const std::string model_path = + file::JoinPath(ModelDir(), "gaussians_anomaly_if"); + + ASSERT_OK_AND_ASSIGN(const auto model, model::LoadModel(model_path)); + + dataset::VerticalDataset dataset; + ASSERT_OK(dataset::LoadVerticalDataset( + dataset_path, model->data_spec(), &dataset, + /*required_columns=*/model->input_features())); proto::PredictionAnalysisOptions options; options.set_html_id_prefix("my_prefix"); diff --git a/yggdrasil_decision_forests/utils/partial_dependence_plot.cc b/yggdrasil_decision_forests/utils/partial_dependence_plot.cc index 8bf3c922..7cbf972c 100644 --- a/yggdrasil_decision_forests/utils/partial_dependence_plot.cc +++ b/yggdrasil_decision_forests/utils/partial_dependence_plot.cc @@ -216,6 +216,15 @@ absl::Status UpdateBin( // truth does not have the same scale/range as the predictions. } break; + case model::proto::Task::ANOMALY_DETECTION: { + STATUS_CHECK( + bin->prediction().has_sum_of_anomaly_detection_predictions()); + // Prediction. + bin->mutable_prediction()->set_sum_of_anomaly_detection_predictions( + bin->prediction().sum_of_anomaly_detection_predictions() + + prediction.anomaly_detection().value() * prediction.weight()); + } break; + default: return absl::InvalidArgumentError("Invalid model task"); } @@ -322,6 +331,11 @@ absl::Status InitializePartialDependence( bin->mutable_prediction()->set_sum_of_ranking_predictions(0.0); break; + case model::proto::Task::ANOMALY_DETECTION: + bin->mutable_prediction()->set_sum_of_anomaly_detection_predictions( + 0.0); + break; + default: return absl::InvalidArgumentError("Invalid task"); } diff --git a/yggdrasil_decision_forests/utils/partial_dependence_plot.proto b/yggdrasil_decision_forests/utils/partial_dependence_plot.proto index 752f3657..a85e8575 100644 --- a/yggdrasil_decision_forests/utils/partial_dependence_plot.proto +++ b/yggdrasil_decision_forests/utils/partial_dependence_plot.proto @@ -38,9 +38,13 @@ message PartialDependencePlotSet { // num_observations to obtain the mean prediction. double sum_of_regression_predictions = 2 [default = 0]; - // sum_of_regression_predictions should be normalized with + // sum_of_ranking_predictions should be normalized with // num_observations to obtain the mean prediction. double sum_of_ranking_predictions = 3 [default = 0]; + + // sum_of_anomaly_detection_predictions should be normalized with + // num_observations to obtain the mean prediction. + double sum_of_anomaly_detection_predictions = 4 [default = 0]; } } // Represent the accumulation of evaluation metrics. From fe99922fad740f3eac9a5e41aa63219e7ddc6905 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 17 Jun 2024 09:50:32 -0700 Subject: [PATCH 25/30] Anomaly detection; Isolation forest learner and model is available in python (part 7) PiperOrigin-RevId: 644042237 --- .../port/python/ydf/cc/ydf.pyi | 10 +- .../port/python/ydf/learner/BUILD | 6 +- .../python/ydf/learner/generic_learner.py | 2 +- .../port/python/ydf/learner/learner_test.py | 38 ++++++++ .../port/python/ydf/learner/wrapper/BUILD | 3 + .../ydf/learner/wrapper/wrapper_generator.bzl | 5 +- .../ydf/learner/wrapper/wrapper_generator.cc | 91 ++++++++++++++----- .../port/python/ydf/model/BUILD | 7 ++ .../ydf/model/isolation_forest_model/BUILD | 53 +++++++++++ .../model/isolation_forest_model/__init__.py | 14 +++ .../isolation_forest_model.py | 24 +++++ .../isolation_forest_model_test.py | 76 ++++++++++++++++ .../isolation_forest_wrapper.cc | 43 +++++++++ .../isolation_forest_wrapper.h | 57 ++++++++++++ .../port/python/ydf/model/model.cc | 21 +++++ .../port/python/ydf/model/model_lib.py | 3 + .../port/python/ydf/model/tree/__init__.py | 1 + .../port/python/ydf/model/tree/value.py | 40 +++++++- 18 files changed, 462 insertions(+), 32 deletions(-) create mode 100644 yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/BUILD create mode 100644 yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/__init__.py create mode 100644 yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_model.py create mode 100644 yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_model_test.py create mode 100644 yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_wrapper.cc create mode 100644 yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_wrapper.h diff --git a/yggdrasil_decision_forests/port/python/ydf/cc/ydf.pyi b/yggdrasil_decision_forests/port/python/ydf/cc/ydf.pyi index b2a2658c..40fe3061 100644 --- a/yggdrasil_decision_forests/port/python/ydf/cc/ydf.pyi +++ b/yggdrasil_decision_forests/port/python/ydf/cc/ydf.pyi @@ -10,6 +10,7 @@ from google3.third_party.yggdrasil_decision_forests.model import abstract_model_ from google3.third_party.yggdrasil_decision_forests.model import hyperparameter_pb2 from google3.third_party.yggdrasil_decision_forests.model.decision_tree import decision_tree_pb2 from google3.third_party.yggdrasil_decision_forests.model.gradient_boosted_trees import gradient_boosted_trees_pb2 +from google3.third_party.yggdrasil_decision_forests.model.isolation_forest import isolation_forest_pb2 from google3.third_party.yggdrasil_decision_forests.model.random_forest import random_forest_pb2 from google3.third_party.yggdrasil_decision_forests.utils import fold_generator_pb2 from google3.third_party.yggdrasil_decision_forests.utils import fold_generator_pb2 @@ -84,8 +85,9 @@ class VerticalDataset: required_columns: Optional[Sequence[str]] = None, ) -> None: ... def SetMultiDimDataspec( - self, unrolling: Dict[str, List[str]], - ) -> None: ... + self, + unrolling: Dict[str, List[str]], + ) -> None: ... # Model bindings @@ -194,6 +196,10 @@ class RandomForestCCModel(DecisionForestCCModel): ) -> List[random_forest_pb2.OutOfBagTrainingEvaluations]: ... def winner_takes_all(self) -> bool: ... +class IsolationForestCCModel(DecisionForestCCModel): + @property + def kRegisteredName(self): ... + class GradientBoostedTreesCCModel(DecisionForestCCModel): @property def kRegisteredName(self): ... diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/BUILD b/yggdrasil_decision_forests/port/python/ydf/learner/BUILD index d1957c8b..eca6bedb 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/learner/BUILD @@ -33,6 +33,7 @@ cc_library_ydf( "@ydf_cc//yggdrasil_decision_forests/learner/cart", "@ydf_cc//yggdrasil_decision_forests/learner/distributed_gradient_boosted_trees:dgbt", "@ydf_cc//yggdrasil_decision_forests/learner/gradient_boosted_trees", + "@ydf_cc//yggdrasil_decision_forests/learner/isolation_forest", "@ydf_cc//yggdrasil_decision_forests/learner/random_forest", ], alwayslink = 1, @@ -225,26 +226,23 @@ py_test( tags = [ ], deps = [ - ":custom_loss_py", ":generic_learner", ":specialized_learners", ":tuner", # absl/logging dep, # absl/testing:absltest dep, # absl/testing:parameterized dep, - # jax dep, # numpy dep, # pandas dep, + # sklearn dep, "@ydf_cc//yggdrasil_decision_forests/dataset:data_spec_py_proto", "@ydf_cc//yggdrasil_decision_forests/learner:abstract_learner_py_proto", "@ydf_cc//yggdrasil_decision_forests/model:abstract_model_py_proto", - "//ydf/dataset", "//ydf/dataset:dataspec", "//ydf/metric", "//ydf/model:generic_model", "//ydf/model:model_lib", "//ydf/model/decision_forest_model", - "//ydf/model/gradient_boosted_trees_model", "//ydf/utils:log", "//ydf/utils:test_utils", ], diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py index 4687390a..11c8e3be 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py @@ -53,7 +53,7 @@ def __init__( self, learner_name: str, task: Task, - label: str, + label: Optional[str], weights: Optional[str], ranking_group: Optional[str], uplift_treatment: Optional[str], diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py b/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py index 8f7816c0..5c0b64fc 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py @@ -24,6 +24,7 @@ import numpy as np import numpy.testing as npt import pandas as pd +from sklearn import metrics from yggdrasil_decision_forests.dataset import data_spec_pb2 as ds_pb from yggdrasil_decision_forests.learner import abstract_learner_pb2 @@ -66,6 +67,7 @@ def setUp(self): Column("treat", semantic=dataspec.Semantic.CATEGORICAL), ], ) + self.gaussians = test_utils.load_datasets("gaussians") def _check_adult_model( self, @@ -1009,6 +1011,42 @@ def test_logging_arg(self, verbose): _ = learner.train(ds, verbose=verbose) +class IsolationForestLearnerTest(LearnerTest): + + @parameterized.parameters(False, True) + def test_gaussians(self, with_labels: bool): + if with_labels: + learner = specialized_learners.IsolationForestLearner(label="label") + else: + learner = specialized_learners.IsolationForestLearner( + features=["f1", "f2"] + ) + model = learner.train(self.gaussians.train) + predictions = model.predict(self.gaussians.test) + + auc = metrics.roc_auc_score(self.gaussians.test_pd["label"], predictions) + logging.info("auc:%s", auc) + self.assertGreaterEqual(auc, 0.99) + + _ = model.describe("text") + _ = model.describe("html") + _ = model.analyze_prediction(self.gaussians.test_pd.iloc[:1]) + _ = model.analyze(self.gaussians.test) + + if with_labels: + evaluation = model.evaluate(self.gaussians.test) + self.assertDictEqual( + evaluation.to_dict(), + {"num_examples": 280, "num_examples_weighted": 280.0}, + ) + else: + with self.assertRaisesRegex( + ValueError, + "Cannot evaluate an anomaly detection model without a label", + ): + _ = model.evaluate(self.gaussians.test) + + class UtilityTest(LearnerTest): def test_feature_name_to_regex(self): diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/BUILD b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/BUILD index 1c754532..08334d00 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/BUILD @@ -27,6 +27,7 @@ cc_library_ydf( srcs = ["wrapper_generator.cc"], hdrs = ["wrapper_generator.h"], deps = [ + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -36,6 +37,7 @@ cc_library_ydf( "@ydf_cc//yggdrasil_decision_forests/learner:learner_library", "@ydf_cc//yggdrasil_decision_forests/model:hyperparameter_cc_proto", "@ydf_cc//yggdrasil_decision_forests/utils:hyper_parameters", + "@ydf_cc//yggdrasil_decision_forests/utils:logging", "@ydf_cc//yggdrasil_decision_forests/utils:status_macros", ], ) @@ -55,6 +57,7 @@ cc_test( "@ydf_cc//yggdrasil_decision_forests/learner:abstract_learner", "@ydf_cc//yggdrasil_decision_forests/learner:abstract_learner_cc_proto", "@ydf_cc//yggdrasil_decision_forests/learner/gradient_boosted_trees", + "@ydf_cc//yggdrasil_decision_forests/learner/isolation_forest", "@ydf_cc//yggdrasil_decision_forests/learner/random_forest", "@ydf_cc//yggdrasil_decision_forests/model:abstract_model", "@ydf_cc//yggdrasil_decision_forests/model:hyperparameter_cc_proto", diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.bzl b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.bzl index a473d4d3..43db5a8c 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.bzl +++ b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.bzl @@ -73,8 +73,9 @@ def py_wrap_yggdrasil_learners( "//ydf/dataset:dataset", "//ydf/dataset:dataspec", "//ydf/model:generic_model", - "//ydf/model/gradient_boosted_trees_model:gradient_boosted_trees_model", - "//ydf/model/random_forest_model:random_forest_model", + "//ydf/model/gradient_boosted_trees_model", + "//ydf/model/random_forest_model", + "//ydf/model/isolation_forest_model", "//ydf/learner:generic_learner", "//ydf/learner:hyperparameters", "//ydf/learner:custom_loss_py", diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.cc b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.cc index 1d4078c4..28b71127 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.cc +++ b/yggdrasil_decision_forests/port/python/ydf/learner/wrapper/wrapper_generator.cc @@ -24,6 +24,7 @@ #include #include +#include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/ascii.h" @@ -40,10 +41,53 @@ #include "yggdrasil_decision_forests/learner/learner_library.h" #include "yggdrasil_decision_forests/model/hyperparameter.pb.h" #include "yggdrasil_decision_forests/utils/hyper_parameters.h" +#include "yggdrasil_decision_forests/utils/logging.h" #include "yggdrasil_decision_forests/utils/status_macros.h" namespace yggdrasil_decision_forests { +// Configuration data for each individual learner. +struct LearnerConfig { + // Name of the python class of the model. + std::string model_class_name = "generic_model.GenericModel"; + + // Default value of the "task" learner constructor argument. + std::string default_task = "CLASSIFICATION"; + + // If true, the learner requires a "label" constructor argument. + bool require_label = true; +}; + +absl::flat_hash_map LearnerConfigs() { + absl::flat_hash_map configs; + + configs["RANDOM_FOREST"] = { + .model_class_name = "random_forest_model.RandomForestModel", + }; + + configs["CART"] = { + .model_class_name = "random_forest_model.RandomForestModel", + }; + + configs["GRADIENT_BOOSTED_TREES"] = { + .model_class_name = + "gradient_boosted_trees_model.GradientBoostedTreesModel", + }; + + configs["DISTRIBUTED_GRADIENT_BOOSTED_TREES"] = { + .model_class_name = + "gradient_boosted_trees_model.GradientBoostedTreesModel", + }; + + configs["ISOLATION_FOREST"] = { + .model_class_name = "isolation_forest_model.IsolationForestModel", + .default_task = "ANOMALY_DETECTION", + .require_label = false, + }; + + return configs; +} // namespace yggdrasil_decision_forests + // Gets the number of prefix spaces. int NumLeadingSpaces(const absl::string_view text) { auto char_it = text.begin(); @@ -67,18 +111,6 @@ std::string LearnerKeyToClassName(const absl::string_view key) { return absl::StrCat(absl::StrReplaceAll(value, {{"_", ""}}), "Learner"); } -// Converts a learner name into the model class associated with it. -std::string LearnerKeyToModelClassName(const absl::string_view key) { - if (key == "RANDOM_FOREST" || key == "CART") { - return "random_forest_model.RandomForestModel"; - } else if (key == "GRADIENT_BOOSTED_TREES" || - key == "DISTRIBUTED_GRADIENT_BOOSTED_TREES") { - return "gradient_boosted_trees_model.GradientBoostedTreesModel"; - } else { - return "generic_model.GenericModel"; - } -} - // Converts a learner name into a nice name. // e.g. "RANDOM_FOREST" -> "Random Forest" std::string LearnerKeyToNiceLearnerName(absl::string_view key) { @@ -264,6 +296,7 @@ from $1learner import tuner as tuner_lib from $1model import generic_model from $1model.gradient_boosted_trees_model import gradient_boosted_trees_model from $1model.random_forest_model import random_forest_model +from $1model.isolation_forest_model import isolation_forest_model )", prefix, pydf_prefix); @@ -292,9 +325,20 @@ from typing import Dict, Optional, Sequence, Union )", imports); + const auto learner_configs = LearnerConfigs(); + for (const auto& learner_key : model::AllRegisteredLearners()) { + // Get the learner configuration. + LearnerConfig learner_config; + const auto learner_config_it = learner_configs.find(learner_key); + if (learner_config_it != learner_configs.end()) { + learner_config = learner_config_it->second; + } else { + YDF_LOG(INFO) << "No learner config for " << learner_key + << ". Using default config."; + } + const auto class_name = LearnerKeyToClassName(learner_key); - const auto model_class_name = LearnerKeyToModelClassName(learner_key); // Get a learner instance. std::unique_ptr learner; @@ -439,7 +483,8 @@ from typing import Dict, Optional, Sequence, Union FixGBTDefinition(&fields_documentation, &fields_constructor); } // TODO: Add support for hyperparameter templates. - absl::SubstituteAndAppend(&wrapper, R"( + absl::SubstituteAndAppend( + &wrapper, R"( class $0(generic_learner.GenericLearner): r"""$6 learning algorithm. @@ -555,8 +600,8 @@ class $0(generic_learner.GenericLearner): """ def __init__(self, - label: str, - task: generic_learner.Task = generic_learner.Task.CLASSIFICATION, + label: $9, + task: generic_learner.Task = generic_learner.Task.$8, weights: Optional[str] = None, ranking_group: Optional[str] = None, uplift_treatment: Optional[str] = None, @@ -659,12 +704,14 @@ class $0(generic_learner.GenericLearner): """ return super().train(ds=ds, valid=valid, verbose=verbose) )", - /*$0*/ class_name, /*$1*/ learner_key, - /*$2*/ fields_documentation, - /*$3*/ fields_constructor, /*$4*/ fields_dict, - /*$5*/ free_text_documentation, - /*$6*/ nice_learner_name, - /*$7*/ model_class_name); + /*$0*/ class_name, /*$1*/ learner_key, + /*$2*/ fields_documentation, + /*$3*/ fields_constructor, /*$4*/ fields_dict, + /*$5*/ free_text_documentation, + /*$6*/ nice_learner_name, + /*$7*/ learner_config.model_class_name, + /*$8*/ learner_config.default_task, + /*$9*/ learner_config.require_label ? "str" : "Optional[str] = None"); const auto bool_rep = [](const bool value) -> std::string { return value ? "True" : "False"; diff --git a/yggdrasil_decision_forests/port/python/ydf/model/BUILD b/yggdrasil_decision_forests/port/python/ydf/model/BUILD index 0881c414..99a8b8d7 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/model/BUILD @@ -17,6 +17,7 @@ cc_library_ydf( deps = [ "@ydf_cc//yggdrasil_decision_forests/model/decision_tree:decision_forest_interface", "@ydf_cc//yggdrasil_decision_forests/model/gradient_boosted_trees", + "@ydf_cc//yggdrasil_decision_forests/model/isolation_forest", "@ydf_cc//yggdrasil_decision_forests/model/random_forest", ], ) @@ -65,6 +66,7 @@ pybind_library( ":model_wrapper", "//ydf/model/decision_forest_model:decision_forest_wrapper", "//ydf/model/gradient_boosted_trees_model:gradient_boosted_trees_wrapper", + "//ydf/model/isolation_forest_model:isolation_forest_wrapper", "//ydf/model/random_forest_model:random_forest_wrapper", "//ydf/utils:custom_casters", "//ydf/utils:status_casters", @@ -74,6 +76,7 @@ pybind_library( "@ydf_cc//yggdrasil_decision_forests/model:abstract_model", "@ydf_cc//yggdrasil_decision_forests/model:model_library", "@ydf_cc//yggdrasil_decision_forests/model/gradient_boosted_trees", + "@ydf_cc//yggdrasil_decision_forests/model/isolation_forest", "@ydf_cc//yggdrasil_decision_forests/model/random_forest", "@ydf_cc//yggdrasil_decision_forests/utils:logging", "@ydf_cc//yggdrasil_decision_forests/utils:model_analysis", @@ -144,9 +147,11 @@ py_library( deps = [ ":generic_model", # numpy dep, + # sklearn dep, "//ydf/learner:generic_learner", "//ydf/learner:specialized_learners", "//ydf/model/gradient_boosted_trees_model", + "//ydf/model/isolation_forest_model", "//ydf/model/random_forest_model", "//ydf/model/tree:all", ], @@ -162,6 +167,7 @@ py_library( "//ydf/cc:ydf", "//ydf/dataset:dataspec", "//ydf/model/gradient_boosted_trees_model", + "//ydf/model/isolation_forest_model", "//ydf/model/random_forest_model", "//ydf/utils:log", ], @@ -246,6 +252,7 @@ py_test( # pandas dep, "//ydf/dataset", "//ydf/model/gradient_boosted_trees_model", + "//ydf/model/isolation_forest_model", "//ydf/model/random_forest_model", "//ydf/utils:test_utils", ], diff --git a/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/BUILD b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/BUILD new file mode 100644 index 00000000..d2bb104d --- /dev/null +++ b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/BUILD @@ -0,0 +1,53 @@ +# pytype test and library +load("@pybind11_bazel//:build_defs.bzl", "pybind_library") + +package( + default_visibility = ["//visibility:public"], + licenses = ["notice"], +) + +# Libraries +# ========= + +pybind_library( + name = "isolation_forest_wrapper", + srcs = ["isolation_forest_wrapper.cc"], + hdrs = ["isolation_forest_wrapper.h"], + deps = [ + "//ydf/model/decision_forest_model:decision_forest_wrapper", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@ydf_cc//yggdrasil_decision_forests/model:abstract_model", + "@ydf_cc//yggdrasil_decision_forests/model/isolation_forest", + "@ydf_cc//yggdrasil_decision_forests/utils:logging", + ], +) + +py_library( + name = "isolation_forest_model", + srcs = ["isolation_forest_model.py"], + deps = [ + "//ydf/cc:ydf", + "//ydf/model/decision_forest_model", + ], +) + +# Tests +# ===== + +py_test( + name = "isolation_forest_model_test", + srcs = ["isolation_forest_model_test.py"], + data = [ + "//test_data", + "@ydf_cc//yggdrasil_decision_forests/test_data", + ], + python_version = "PY3", + deps = [ + # absl/testing:absltest dep, + # numpy dep, + # pandas dep, + "//ydf/model:model_lib", + "//ydf/utils:test_utils", + ], +) diff --git a/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/__init__.py b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/__init__.py new file mode 100644 index 00000000..4446e915 --- /dev/null +++ b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_model.py b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_model.py new file mode 100644 index 00000000..ba5100db --- /dev/null +++ b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_model.py @@ -0,0 +1,24 @@ +# Copyright 2022 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Definitions for Isolation Forest models.""" + +from ydf.cc import ydf +from ydf.model.decision_forest_model import decision_forest_model + + +class IsolationForestModel(decision_forest_model.DecisionForestModel): + """An Isolation Forest model for prediction and inspection.""" + + _model: ydf.IsolationForestCCModel diff --git a/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_model_test.py b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_model_test.py new file mode 100644 index 00000000..a8968c01 --- /dev/null +++ b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_model_test.py @@ -0,0 +1,76 @@ +# Copyright 2022 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the isolation forest models.""" + +import logging +import os + +from absl.testing import absltest +import numpy as np +import pandas as pd + +from ydf.model import model_lib +from ydf.utils import test_utils + + +class IsolationForestModelTest(absltest.TestCase): + + def setUp(self): + super().setUp() + + def build_path(*args): + return os.path.join(test_utils.ydf_test_data_path(), *args) + + self.model_gaussians = model_lib.load_model( + build_path("model", "gaussians_anomaly_if") + ) + self.dataset_gaussians_train = pd.read_csv( + build_path("dataset", "gaussians_train.csv") + ) + self.dataset_gaussians_test = pd.read_csv( + build_path("dataset", "gaussians_test.csv") + ) + + def test_predict(self): + predictions = self.model_gaussians.predict(self.dataset_gaussians_test) + np.testing.assert_allclose( + predictions[:5], + [0.419287, 0.441436, 0.507164, 0.425276, 0.386438], + atol=0.0001, + ) + + def test_distance(self): + distances = self.model_gaussians.distance(self.dataset_gaussians_test) + logging.info("distances:\n%s", distances) + self.assertEqual( + distances.shape, + ( + self.dataset_gaussians_test.shape[0], + self.dataset_gaussians_test.shape[0], + ), + ) + + # Find the example most similar to "self.dataset_gaussians_test[0]". + most_similar_example_idx = np.argmin(distances[0, :]) + logging.info("most_similar_example_idx: %s", most_similar_example_idx) + logging.info("Seed example:\n%s", self.dataset_gaussians_test.iloc[0]) + logging.info( + "Most similar example:\n%s", + self.dataset_gaussians_test.iloc[most_similar_example_idx], + ) + + +if __name__ == "__main__": + absltest.main() diff --git a/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_wrapper.cc b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_wrapper.cc new file mode 100644 index 00000000..5d0e9559 --- /dev/null +++ b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_wrapper.cc @@ -0,0 +1,43 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ydf/model/isolation_forest_model/isolation_forest_wrapper.h" + +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" + +namespace yggdrasil_decision_forests::port::python { + +absl::StatusOr> +IsolationForestCCModel::Create( + std::unique_ptr& model_ptr) { + auto* if_model = dynamic_cast(model_ptr.get()); + if (if_model == nullptr) { + return absl::InvalidArgumentError( + "This model is not an isolation forest model."); + } + // Both release and the unique_ptr constructor are noexcept. + model_ptr.release(); + std::unique_ptr new_model_ptr(if_model); + + return std::make_unique(std::move(new_model_ptr), + if_model); +} + +} // namespace yggdrasil_decision_forests::port::python diff --git a/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_wrapper.h b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_wrapper.h new file mode 100644 index 00000000..868b7854 --- /dev/null +++ b/yggdrasil_decision_forests/port/python/ydf/model/isolation_forest_model/isolation_forest_wrapper.h @@ -0,0 +1,57 @@ +/* + * Copyright 2022 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef YGGDRASIL_DECISION_FORESTS_PORT_PYTHON_YDF_MODEL_ISOLATION_FOREST_MODEL_RANDOM_FOREST_WRAPPER_H_ +#define YGGDRASIL_DECISION_FORESTS_PORT_PYTHON_YDF_MODEL_ISOLATION_FOREST_MODEL_RANDOM_FOREST_WRAPPER_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "yggdrasil_decision_forests/model/abstract_model.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" +#include "ydf/model/decision_forest_model/decision_forest_wrapper.h" +#include "yggdrasil_decision_forests/utils/logging.h" + +namespace yggdrasil_decision_forests::port::python { + +class IsolationForestCCModel : public DecisionForestCCModel { + using YDFModel = ::yggdrasil_decision_forests::model::isolation_forest:: + IsolationForestModel; + + public: + // Creates a IsolationForestCCModel if `model_ptr` refers to a + // IsolationForestModel. + // + // If this method returns an invalid status, "model_ptr" is not modified. + // If this method returns an ok status, the content of "model_ptr" is moved + // (and "model_ptr" becomes empty). + static absl::StatusOr> Create( + std::unique_ptr& model_ptr); + + // `model` and `if_model` must point to the same object. Prefer using + // IsolationForestCCModel::Compute for construction. + IsolationForestCCModel(std::unique_ptr model, YDFModel* if_model) + : DecisionForestCCModel(std::move(model), if_model), if_model_(if_model) { + DCHECK_EQ(model_.get(), if_model_); + } + + private: + // This is a non-owning pointer to the model held by `model_`. + YDFModel* if_model_; +}; + +} // namespace yggdrasil_decision_forests::port::python +#endif // YGGDRASIL_DECISION_FORESTS_PORT_PYTHON_YDF_MODEL_ISOLATION_FOREST_MODEL_RANDOM_FOREST_WRAPPER_H_ diff --git a/yggdrasil_decision_forests/port/python/ydf/model/model.cc b/yggdrasil_decision_forests/port/python/ydf/model/model.cc index 20406838..fb61c621 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/model.cc +++ b/yggdrasil_decision_forests/port/python/ydf/model/model.cc @@ -30,10 +30,12 @@ #include "pybind11_protobuf/native_proto_caster.h" #include "yggdrasil_decision_forests/model/abstract_model.h" #include "yggdrasil_decision_forests/model/gradient_boosted_trees/gradient_boosted_trees.h" +#include "yggdrasil_decision_forests/model/isolation_forest/isolation_forest.h" #include "yggdrasil_decision_forests/model/model_library.h" #include "yggdrasil_decision_forests/model/random_forest/random_forest.h" #include "ydf/model/decision_forest_model/decision_forest_wrapper.h" #include "ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_wrapper.h" +#include "ydf/model/isolation_forest_model/isolation_forest_wrapper.h" #include "ydf/model/model_wrapper.h" #include "ydf/model/random_forest_model/random_forest_wrapper.h" #include "ydf/utils/custom_casters.h" @@ -84,6 +86,11 @@ std::unique_ptr CreateCCModel( // `model_ptr` is now invalid. return std::move(gbt_model.value()); } + auto if_model = IsolationForestCCModel::Create(model_ptr); + if (if_model.ok()) { + // `model_ptr` is now invalid. + return std::move(if_model.value()); + } // `model_ptr` is still valid. return std::make_unique(std::move(model_ptr)); } @@ -198,6 +205,20 @@ void init_model(py::module_& m) { return model::random_forest::RandomForestModel::kRegisteredName; }); + py::class_(m, + "IsolationForestCCModel") + .def("__repr__", + [](const GenericCCModel& a) { + return absl::Substitute( + "( m, "GradientBoostedTreesCCModel") diff --git a/yggdrasil_decision_forests/port/python/ydf/model/model_lib.py b/yggdrasil_decision_forests/port/python/ydf/model/model_lib.py index 8f6672a7..4cd1763d 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/model_lib.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/model_lib.py @@ -23,6 +23,7 @@ from ydf.dataset import dataspec from ydf.model import generic_model from ydf.model.gradient_boosted_trees_model import gradient_boosted_trees_model +from ydf.model.isolation_forest_model import isolation_forest_model from ydf.model.random_forest_model import random_forest_model from ydf.utils import log @@ -156,6 +157,8 @@ def load_cc_model(cc_model: ydf.GenericCCModel) -> generic_model.ModelType: return random_forest_model.RandomForestModel(cc_model) if model_name == ydf.GradientBoostedTreesCCModel.kRegisteredName: return gradient_boosted_trees_model.GradientBoostedTreesModel(cc_model) + if model_name == ydf.IsolationForestCCModel.kRegisteredName: + return isolation_forest_model.IsolationForestModel(cc_model) logging.info( "This model has type %s, which is not fully supported. Only generic model" " tasks (e.g. inference) are possible", diff --git a/yggdrasil_decision_forests/port/python/ydf/model/tree/__init__.py b/yggdrasil_decision_forests/port/python/ydf/model/tree/__init__.py index a17c4468..4392f572 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/tree/__init__.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/tree/__init__.py @@ -39,6 +39,7 @@ from ydf.model.tree.value import RegressionValue from ydf.model.tree.value import ProbabilityValue from ydf.model.tree.value import UpliftValue +from ydf.model.tree.value import AnomalyDetectionValue # Plotting from ydf.model.tree.plot import PlotOptions diff --git a/yggdrasil_decision_forests/port/python/ydf/model/tree/value.py b/yggdrasil_decision_forests/port/python/ydf/model/tree/value.py index 9f57b649..b16b3605 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/tree/value.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/tree/value.py @@ -29,7 +29,7 @@ class AbstractValue(metaclass=abc.ABCMeta): """A generic value/prediction/output. Attrs: - num_examples: Number of example in the node. + num_examples: Number of examples in the node with weight. """ num_examples: float @@ -100,6 +100,20 @@ def pretty(self) -> str: return f"value={self.treatment_effect}" +@dataclasses.dataclass +class AnomalyDetectionValue(AbstractValue): + """The value of an anomaly detection tree. + + Attrs: + num_examples_without_weight: Number of examples reaching this node. + """ + + num_examples_without_weight: int + + def pretty(self) -> str: + return f"count={self.num_examples_without_weight}" + + def to_value(proto_node: decision_tree_pb2.Node) -> AbstractValue: """Extracts the "value" part of a proto node.""" @@ -130,6 +144,12 @@ def to_value(proto_node: decision_tree_pb2.Node) -> AbstractValue: num_examples=proto_node.uplift.sum_weights, ) + if proto_node.HasField("anomaly_detection"): + return AnomalyDetectionValue( + num_examples_without_weight=proto_node.anomaly_detection.num_examples_without_weight, + num_examples=-1.0, # The number of weighted examples is not tracked. + ) + raise ValueError("Unsupported value") @@ -179,6 +199,15 @@ def _to_json_uplift(value: UpliftValue) -> Dict[str, Any]: } +@to_json.register +def _to_json_uplift(value: AnomalyDetectionValue) -> Dict[str, Any]: + return { + "type": "ANOMALY_DETECTION", + "num_examples_without_weight": value.num_examples_without_weight, + "num_examples": value.num_examples, + } + + @functools.singledispatch def set_proto_node(value: AbstractValue, proto_node: decision_tree_pb2.Node): """Sets the "value" part in a proto node. @@ -227,3 +256,12 @@ def _set_proto_node_from_uplift( ): proto_node.uplift.treatment_effect[:] = value.treatment_effect proto_node.uplift.sum_weights = value.num_examples + + +@set_proto_node.register +def _set_proto_node_from_anomaly_detection( + value: AnomalyDetectionValue, proto_node: decision_tree_pb2.Node +): + proto_node.anomaly_detection.num_examples_without_weight = ( + value.num_examples_without_weight + ) From 8d691607f5a300a337341ed0d280fc27a45d577f Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 17 Jun 2024 12:08:18 -0700 Subject: [PATCH 26/30] Anomaly detection; Add support for sklearn isolation forests in the model importer (part 8) PiperOrigin-RevId: 644091208 --- .../port/python/ydf/model/export_sklearn.py | 213 +++++++++++++----- .../port/python/ydf/model/generic_model.py | 19 +- .../python/ydf/model/sklearn_model_test.py | 77 ++++++- 3 files changed, 250 insertions(+), 59 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py b/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py index 3d6b6f39..902d2519 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/export_sklearn.py @@ -14,10 +14,10 @@ """Import and export Scikit-Learn models from/to YDF.""" +import dataclasses import enum import functools -from typing import Any, Dict, List, Optional, TypeVar, Union - +from typing import Any, Dict, List, Optional, TypeVar, Union, Sequence import numpy as np from ydf.learner import generic_learner @@ -25,6 +25,7 @@ from ydf.model import generic_model from ydf.model import tree as tree_lib from ydf.model.gradient_boosted_trees_model import gradient_boosted_trees_model +from ydf.model.isolation_forest_model import isolation_forest_model from ydf.model.random_forest_model import random_forest_model # pytype: disable=import-error @@ -43,9 +44,18 @@ # The column idx=0 is reserved for the label in YDF models. _LABEL_COLUMN_OFFSET = 1 -# Name of the label/feature columns -_LABEL_KEY = "label" -_FEATURES_KEY = "features" + +@dataclasses.dataclass(frozen=True) +class InternalOptions: + """Internal options for the conversion. + + Attributes: + label_name: Column name of the created label. + feature_name: Column name of the created feature. + """ + + label_name: str + feature_name: str class TaskType(enum.Enum): @@ -54,19 +64,27 @@ class TaskType(enum.Enum): UNKNOWN = 1 SCALAR_REGRESSION = 2 SINGLE_LABEL_CLASSIFICATION = 3 + ANOMALY_DETECTION = 4 ScikitLearnModel = TypeVar("ScikitLearnModel", bound=base.BaseEstimator) ScikitLearnTree = TypeVar("ScikitLearnTree", bound=tree.BaseDecisionTree) -def from_sklearn(sklearn_model: ScikitLearnModel) -> generic_model.GenericModel: +def from_sklearn( + sklearn_model: ScikitLearnModel, + label_name: str = "label", + feature_name: str = "features", +) -> generic_model.GenericModel: """Converts a tree-based scikit-learn model to a YDF model.""" if not hasattr(sklearn_model, "n_features_in_"): raise ValueError( "Scikit-Learn model must be fit to data before converting." ) - return _sklearn_to_ydf_model(sklearn_model) + return _sklearn_to_ydf_model( + sklearn_model, + InternalOptions(label_name=label_name, feature_name=feature_name), + ) def _gen_fake_features(num_features: int, num_examples: int = 2): @@ -75,9 +93,10 @@ def _gen_fake_features(num_features: int, num_examples: int = 2): @functools.singledispatch def _sklearn_to_ydf_model( - sklearn_model: ScikitLearnModel, + sklearn_model: ScikitLearnModel, options: InternalOptions ) -> generic_model.GenericModel: """Builds a YDF model from the given scikit-learn model.""" + del options raise NotImplementedError( f"Can't build a YDF model for {type(sklearn_model)}" ) @@ -85,44 +104,54 @@ def _sklearn_to_ydf_model( @_sklearn_to_ydf_model.register(tree.DecisionTreeRegressor) @_sklearn_to_ydf_model.register(tree.ExtraTreeRegressor) -def _(sklearn_model: ScikitLearnTree) -> generic_model.GenericModel: +def _( + sklearn_model: ScikitLearnTree, options: InternalOptions +) -> generic_model.GenericModel: """Converts a single scikit-learn regression tree to a YDF model.""" ydf_model = specialized_learners.RandomForestLearner( - label=_LABEL_KEY, + label=options.label_name, task=generic_learner.Task.REGRESSION, num_trees=0, ).train( { - _LABEL_KEY: [0.0, 1.0], - _FEATURES_KEY: _gen_fake_features(sklearn_model.n_features_in_), + options.label_name: [0.0, 1.0], + options.feature_name: _gen_fake_features( + sklearn_model.n_features_in_ + ), }, verbose=0, ) assert isinstance(ydf_model, random_forest_model.RandomForestModel) - ydf_tree = convert_sklearn_tree_to_ydf_tree(sklearn_model) + ydf_tree = convert_sklearn_tree_to_ydf_tree( + sklearn_model, task=generic_learner.Task.REGRESSION + ) ydf_model.add_tree(ydf_tree) return ydf_model @_sklearn_to_ydf_model.register(tree.DecisionTreeClassifier) @_sklearn_to_ydf_model.register(tree.ExtraTreeClassifier) -def _(sklearn_model: ScikitLearnTree) -> generic_model.GenericModel: +def _( + sklearn_model: ScikitLearnTree, options: InternalOptions +) -> generic_model.GenericModel: """Converts a single scikit-learn classification tree to a YDF model.""" ydf_model = specialized_learners.RandomForestLearner( - label=_LABEL_KEY, + label=options.label_name, task=generic_learner.Task.CLASSIFICATION, num_trees=0, ).train( { - _LABEL_KEY: [str(c) for c in sklearn_model.classes_], - _FEATURES_KEY: _gen_fake_features( + options.label_name: [str(c) for c in sklearn_model.classes_], + options.feature_name: _gen_fake_features( sklearn_model.n_features_in_, len(sklearn_model.classes_) ), }, verbose=0, ) assert isinstance(ydf_model, random_forest_model.RandomForestModel) - ydf_tree = convert_sklearn_tree_to_ydf_tree(sklearn_model) + ydf_tree = convert_sklearn_tree_to_ydf_tree( + sklearn_model, task=generic_learner.Task.CLASSIFICATION + ) ydf_model.add_tree(ydf_tree) return ydf_model @@ -131,25 +160,31 @@ def _(sklearn_model: ScikitLearnTree) -> generic_model.GenericModel: @_sklearn_to_ydf_model.register(ensemble.RandomForestRegressor) def _( sklearn_model: Union[ - ensemble.ExtraTreesRegressor, ensemble.RandomForestRegressor + ensemble.ExtraTreesRegressor, + ensemble.RandomForestRegressor, ], + options: InternalOptions, ) -> generic_model.GenericModel: """Converts a forest regression model into a YDF model.""" ydf_model = specialized_learners.RandomForestLearner( - label=_LABEL_KEY, + label=options.label_name, task=generic_learner.Task.REGRESSION, num_trees=0, ).train( { - _LABEL_KEY: [0.0, 1.0], - _FEATURES_KEY: _gen_fake_features(sklearn_model.n_features_in_), + options.label_name: [0.0, 1.0], + options.feature_name: _gen_fake_features( + sklearn_model.n_features_in_ + ), }, verbose=0, ) assert isinstance(ydf_model, random_forest_model.RandomForestModel) for sklearn_tree in sklearn_model.estimators_: - ydf_tree = convert_sklearn_tree_to_ydf_tree(sklearn_tree) + ydf_tree = convert_sklearn_tree_to_ydf_tree( + sklearn_tree, task=generic_learner.Task.REGRESSION + ) ydf_model.add_tree(ydf_tree) return ydf_model @@ -158,34 +193,72 @@ def _( @_sklearn_to_ydf_model.register(ensemble.RandomForestClassifier) def _( sklearn_model: Union[ - ensemble.ExtraTreesClassifier, ensemble.RandomForestClassifier + ensemble.ExtraTreesClassifier, + ensemble.RandomForestClassifier, ], + options: InternalOptions, ) -> generic_model.GenericModel: """Converts a forest classification model into a YDF model.""" ydf_model = specialized_learners.RandomForestLearner( - label=_LABEL_KEY, + label=options.label_name, task=generic_learner.Task.CLASSIFICATION, num_trees=0, ).train( { - _LABEL_KEY: [str(c) for c in sklearn_model.classes_], - _FEATURES_KEY: _gen_fake_features( + options.label_name: [str(c) for c in sklearn_model.classes_], + options.feature_name: _gen_fake_features( sklearn_model.n_features_in_, len(sklearn_model.classes_) ), }, verbose=0, ) - assert isinstance(ydf_model, random_forest_model.RandomForestModel) + assert isinstance( + ydf_model, + random_forest_model.RandomForestModel, + ) for sklearn_tree in sklearn_model.estimators_: - ydf_tree = convert_sklearn_tree_to_ydf_tree(sklearn_tree) + ydf_tree = convert_sklearn_tree_to_ydf_tree( + sklearn_tree, task=generic_learner.Task.CLASSIFICATION + ) + ydf_model.add_tree(ydf_tree) + return ydf_model + + +@_sklearn_to_ydf_model.register(ensemble.IsolationForest) +def _( + sklearn_model: ensemble.IsolationForest, options: InternalOptions +) -> generic_model.GenericModel: + """Converts a single scikit-learn iso-forest to a YDF model.""" + ydf_model = specialized_learners.IsolationForestLearner( + task=generic_learner.Task.ANOMALY_DETECTION, + num_trees=0, + subsample_count=sklearn_model._max_samples, # pylint: disable=protected-access + ).train( + { + options.feature_name: _gen_fake_features( + sklearn_model.n_features_in_ + ), + }, + verbose=0, + ) + assert isinstance(ydf_model, isolation_forest_model.IsolationForestModel) + + for sklearn_tree, attribute_mapping in zip( + sklearn_model.estimators_, sklearn_model.estimators_features_ + ): + ydf_tree = convert_sklearn_tree_to_ydf_tree( + sklearn_tree, + attribute_mapping=attribute_mapping.tolist(), + task=generic_learner.Task.ANOMALY_DETECTION, + ) ydf_model.add_tree(ydf_tree) return ydf_model @_sklearn_to_ydf_model.register(ensemble.GradientBoostingRegressor) def _( - sklearn_model: ensemble.GradientBoostingRegressor, + sklearn_model: ensemble.GradientBoostingRegressor, options: InternalOptions ) -> generic_model.GenericModel: """Converts a gradient boosting regression model into a YDF model.""" @@ -199,7 +272,9 @@ def _( # first tree in the ensemble and set the bias to zero. We could also support # other tree-based initial estimators (e.g. RandomForest), but this seems # like a niche enough use case that we don't for the moment. - init_pytree = convert_sklearn_tree_to_ydf_tree(sklearn_model.init_) + init_pytree = convert_sklearn_tree_to_ydf_tree( + sklearn_model.init_, task=generic_learner.Task.REGRESSION + ) bias = 0.0 elif sklearn_model.init_ == "zero": init_pytree = None @@ -212,13 +287,15 @@ def _( ) ydf_model = specialized_learners.GradientBoostedTreesLearner( - label=_LABEL_KEY, + label=options.label_name, task=generic_learner.Task.REGRESSION, num_trees=0, ).train( { - _LABEL_KEY: [0.0, 1.0], - _FEATURES_KEY: _gen_fake_features(sklearn_model.n_features_in_), + options.label_name: [0.0, 1.0], + options.feature_name: _gen_fake_features( + sklearn_model.n_features_in_ + ), }, verbose=0, ) @@ -233,7 +310,9 @@ def _( for weak_learner in sklearn_model.estimators_.ravel(): ydf_tree = convert_sklearn_tree_to_ydf_tree( - weak_learner, weight=sklearn_model.learning_rate + weak_learner, + weight=sklearn_model.learning_rate, + task=generic_learner.Task.REGRESSION, ) ydf_model.add_tree(ydf_tree) return ydf_model @@ -241,14 +320,19 @@ def _( def convert_sklearn_tree_to_ydf_tree( sklearn_tree: ScikitLearnTree, + task: generic_learner.Task, weight: Optional[float] = None, + attribute_mapping: Optional[Sequence[int]] = None, ) -> tree_lib.Tree: """Converts a scikit-learn decision tree into a YDF tree. Args: sklearn_tree: a scikit-learn decision tree. + task: The task of the model. weight: an optional weight to apply to the values of the leaves in the tree. This is intended for use when converting gradient boosted tree models. + attribute_mapping: Index of the attributes used as input features for this + sktree. Returns: a YDF tree that has the same structure as the scikit-learn tree. @@ -260,10 +344,17 @@ def convert_sklearn_tree_to_ydf_tree( "Scikit-Learn model must be fit to data before converting." ) from e + if hasattr(sklearn_tree, "n_classes_") and sklearn_tree.n_outputs_ == 1: + pass # A classification model + elif sklearn_tree.n_outputs_ == 1: + pass # A regression model + else: + raise ValueError( + "This model type if not supported. `ydf.from_sklearn` only support" + " scalar regression, single-label classification and isolation forests." + ) + field_names = sklearn_tree_data["nodes"].dtype.names - task_type = _get_sklearn_tree_task_type(sklearn_tree) - if weight and task_type is TaskType.SINGLE_LABEL_CLASSIFICATION: - raise ValueError("weight should not be passed for classification trees.") nodes = [] # For each node @@ -278,19 +369,26 @@ def convert_sklearn_tree_to_ydf_tree( for field_name, field_value in zip(field_names, node_properties) } + common_kwargs = {"num_examples": node["weighted_n_node_samples"]} + # Add the node output value to the dictionary of properties. - if task_type is TaskType.SCALAR_REGRESSION: + if task == generic_learner.Task.REGRESSION: scaling_factor = weight if weight else 1.0 node["value"] = tree_lib.RegressionValue( - value=node_output[0][0] * scaling_factor, - num_examples=node["weighted_n_node_samples"], + value=node_output[0][0] * scaling_factor, **common_kwargs ) - elif task_type is TaskType.SINGLE_LABEL_CLASSIFICATION: + elif task == generic_learner.Task.CLASSIFICATION: # Normalise to probabilities if we have a classification tree. + assert weight is None probabilities = list(node_output[0] / node_output[0].sum()) node["value"] = tree_lib.ProbabilityValue( - probability=probabilities, - num_examples=node["weighted_n_node_samples"], + probability=probabilities, **common_kwargs + ) + + elif task == generic_learner.Task.ANOMALY_DETECTION: + assert weight is None + node["value"] = tree_lib.AnomalyDetectionValue( + num_examples_without_weight=node["n_node_samples"], **common_kwargs ) else: raise ValueError( @@ -303,23 +401,17 @@ def convert_sklearn_tree_to_ydf_tree( # The root node has index zero. node_index=0, nodes=nodes, + attribute_mapping=attribute_mapping, + task=task, ) return tree_lib.Tree(root_node) -def _get_sklearn_tree_task_type(sklearn_tree: ScikitLearnTree) -> TaskType: - """Finds the task type of a scikit learn tree.""" - if hasattr(sklearn_tree, "n_classes_") and sklearn_tree.n_outputs_ == 1: - return TaskType.SINGLE_LABEL_CLASSIFICATION - elif sklearn_tree.n_outputs_ == 1: - return TaskType.SCALAR_REGRESSION - else: - return TaskType.UNKNOWN - - def _convert_sklearn_node_to_ydf_node( node_index: int, + task: generic_learner.Task, nodes: List[Dict[str, Any]], + attribute_mapping: Optional[Sequence[int]], ) -> tree_lib.AbstractNode: """Converts a node within a scikit-learn tree into a YDF node.""" if node_index == -1: @@ -333,15 +425,26 @@ def _convert_sklearn_node_to_ydf_node( neg_child = _convert_sklearn_node_to_ydf_node( node_index=node["left_child"], + task=task, nodes=nodes, + attribute_mapping=attribute_mapping, ) pos_child = _convert_sklearn_node_to_ydf_node( node_index=node["right_child"], + task=task, nodes=nodes, + attribute_mapping=attribute_mapping, ) + + attribute = node["feature"] + if attribute_mapping: + attribute = attribute_mapping[attribute] + else: + attribute += _LABEL_COLUMN_OFFSET + return tree_lib.NonLeaf( condition=tree_lib.NumericalHigherThanCondition( - attribute=node["feature"] + _LABEL_COLUMN_OFFSET, + attribute=attribute, threshold=node["threshold"], missing=False, score=0.0, diff --git a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py index a231aca1..90e86166 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/generic_model.py @@ -1095,7 +1095,11 @@ def force_engine(self, engine_name: Optional[str]) -> None: self._model.ForceEngine(engine_name) -def from_sklearn(sklearn_model: Any) -> GenericModel: +def from_sklearn( + sklearn_model: Any, + label_name: str = "label", + feature_name: str = "features", +) -> GenericModel: """Converts a tree-based scikit-learn model to a YDF model. Usage example: @@ -1129,17 +1133,28 @@ def from_sklearn(sklearn_model: Any) -> GenericModel: * sklearn.ensemble.ExtraTreesClassifier * sklearn.ensemble.ExtraTreesRegressor * sklearn.ensemble.GradientBoostingRegressor + * sklearn.ensemble.IsolationForest + + Unlike YDF, Scikit-learn does not name features and labels. Use the fields + `label_name` and `feature_name` to specify the name of the columns in the YDF + model. Additionally, only single-label classification and scalar regression are supported (e.g. multivariate regression models will not convert). Args: sklearn_model: the scikit-learn tree based model to be converted. + label_name: Name of the multi-dimensional feature in the output YDF model. + feature_name: Name of the label in the output YDF model. Returns: a YDF Model that emulates the provided scikit-learn model. """ - return _get_export_sklearn().from_sklearn(sklearn_model) + return _get_export_sklearn().from_sklearn( + sklearn_model=sklearn_model, + label_name=label_name, + feature_name=feature_name, + ) def _get_export_jax(): diff --git a/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py b/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py index d0a7941f..47416b6b 100644 --- a/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/model/sklearn_model_test.py @@ -12,17 +12,52 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Tuple from absl.testing import absltest from absl.testing import parameterized import numpy as np from sklearn import datasets from sklearn import ensemble from sklearn import linear_model +from sklearn import metrics from sklearn import tree from ydf.model import export_sklearn from ydf.model.decision_forest_model import decision_forest_model +def gen_anomaly_detection_dataset( + n_samples: int = 120, + n_outliers: int = 40, + seed: int = 0, +) -> Tuple[np.ndarray, np.ndarray]: + """Generates a two-gaussians anomaly detection dataset. + + This function is similar to the example in: + https://scikit-learn.org/stable/auto_examples/ensemble/plot_isolation_forest.html + + Args: + n_samples: Number of samples to generate in each gaussian. + n_outliers: Number of outliers to generate. + seed: Seed to use for random number generation. + + Returns: + The features and labels for the dataset. + """ + np.random.seed(seed) + covariance = np.array([[0.5, -0.1], [0.7, 0.4]]) + cluster_1 = 0.4 * np.random.randn(n_samples, 2) @ covariance + np.array( + [2, 2] + ) + cluster_2 = 0.3 * np.random.randn(n_samples, 2) + np.array([-2, -2]) + outliers = np.random.uniform(low=-4, high=4, size=(n_outliers, 2)) + features = np.concatenate([cluster_1, cluster_2, outliers]) + labels = np.concatenate([ + np.zeros((2 * n_samples), dtype=bool), + np.ones((n_outliers), dtype=bool), + ]) + return features, labels + + class ScikitLearnModelConverterTest(parameterized.TestCase): @parameterized.parameters( @@ -104,6 +139,44 @@ def test_import_classification_model( ydf_predictions = ydf_model.predict(ydf_features) np.testing.assert_allclose(sklearn_predictions, ydf_predictions, rtol=1e-5) + def test_import_anomaly_detection_model( + self, + ): + train_features, _ = gen_anomaly_detection_dataset(seed=0) + test_features, test_labels = gen_anomaly_detection_dataset(seed=1) + + # Train isolation forest + sklearn_model = ensemble.IsolationForest(max_samples=100, random_state=0) + sklearn_model.fit(train_features) + + # Generate golden predictions + sklearn_predictions = -sklearn_model.score_samples(test_features) + # Note: This is different from "sklearn_model.predict" and + # "sklearn_model.decision_function". + + # Test quality of model + auc = metrics.roc_auc_score(test_labels, sklearn_predictions) + self.assertAlmostEqual(auc, 0.99333, delta=0.0001) + + ydf_model = export_sklearn.from_sklearn(sklearn_model) + self.assertSequenceEqual( + ydf_model.input_feature_names(), + [ + "features.0_of_2", + "features.1_of_2", + ], + ) + ydf_features = {"features": test_features} + ydf_predictions = ydf_model.predict(ydf_features) + + _ = ydf_model.describe("text") + _ = ydf_model.describe("html") + _ = ydf_model.analyze_prediction({"features": test_features[:1]}) + _ = ydf_model.analyze(ydf_features) + + # YDF Predictions match SKLearn predictions + np.testing.assert_allclose(sklearn_predictions, ydf_predictions, rtol=1e-5) + def test_import_raises_when_unrecognised_model_provided(self): features, labels = datasets.make_regression( n_samples=100, @@ -132,7 +205,7 @@ def test_import_raises_when_regression_target_is_multivariate(self): sklearn_model = tree.DecisionTreeRegressor().fit(features, labels) with self.assertRaisesRegex( ValueError, - "Only scalar regression and single-label classification are supported.", + "This model type if not supported", ): _ = export_sklearn.from_sklearn(sklearn_model) @@ -149,7 +222,7 @@ def test_import_raises_when_classification_target_is_multilabel( sklearn_model = tree.DecisionTreeClassifier().fit(features, labels) with self.assertRaisesRegex( ValueError, - "Only scalar regression and single-label classification are supported.", + "This model type if not supported", ): _ = export_sklearn.from_sklearn(sklearn_model) From aa9fcdd7084ff77ad98132218a94ee6234471760 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 17 Jun 2024 12:12:11 -0700 Subject: [PATCH 27/30] Improve error messages when feeding the wrong type of datasets. PiperOrigin-RevId: 644092333 --- .../port/python/CHANGELOG.md | 1 + .../port/python/ydf/dataset/dataset_test.py | 12 +++++++++ .../port/python/ydf/dataset/io/dataset_io.py | 10 ++++++-- .../python/ydf/dataset/io/dataset_io_types.py | 25 ++++++++++++++----- .../port/python/ydf/dataset/io/pandas_io.py | 4 +++ .../python/ydf/learner/generic_learner.py | 3 +++ .../port/python/ydf/learner/learner_test.py | 4 +++ 7 files changed, 51 insertions(+), 8 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/CHANGELOG.md b/yggdrasil_decision_forests/port/python/CHANGELOG.md index 5339e1e0..b41d1211 100644 --- a/yggdrasil_decision_forests/port/python/CHANGELOG.md +++ b/yggdrasil_decision_forests/port/python/CHANGELOG.md @@ -8,6 +8,7 @@ - Add `verbose` argument to `train` method which is equivalent but sometime more convenient than`ydf.verbose`. - Add SKLearn to YDF model converter: `ydf.from_sklearn`. +- Improve error messages when calling the model with non supported data. ### Fix diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py index a5f9aa3f..b7f8e131 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py @@ -1172,6 +1172,18 @@ def test_look_numerical(self, value: str): def test_does_not_look_numerical(self, value: str): self.assertFalse(dataset.look_numerical(value)) + def test_from_numpy(self): + with self.assertRaisesRegex( + ValueError, "Numpy arrays cannot be fed directly" + ): + dataset.create_vertical_dataset(np.array([1, 2, 3])) + + def test_from_column_less_pandas(self): + with self.assertRaisesRegex( + ValueError, "The pandas DataFrame must have string column names" + ): + dataset.create_vertical_dataset(pd.DataFrame([[1, 2, 3], [4, 5, 6]])) + class CategoricalSetTest(absltest.TestCase): diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/io/dataset_io.py b/yggdrasil_decision_forests/port/python/ydf/dataset/io/dataset_io.py index 92e3babf..76f6994b 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/io/dataset_io.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/io/dataset_io.py @@ -166,7 +166,13 @@ def cast_input_dataset_to_dict( return _unroll_dict(data, **unroll_dict_kwargs) # TODO: Maybe this error should be raised at a layer above this one? + raise ValueError( - "Cannot import dataset from" - f" {type(data)}.\n{dataset_io_types.SUPPORTED_INPUT_DATA_DESCRIPTION}" + "Non supported dataset type: " + f"{type(data)}\n\n{dataset_io_types.SUPPORTED_INPUT_DATA_DESCRIPTION}" + + ( + dataset_io_types.HOW_TO_FEED_NUMPY + if isinstance(data, np.ndarray) + else "" + ) ) diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/io/dataset_io_types.py b/yggdrasil_decision_forests/port/python/ydf/dataset/io/dataset_io_types.py index 35a6ed6e..7234be50 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/io/dataset_io_types.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/io/dataset_io_types.py @@ -37,13 +37,26 @@ # Supported types of datasets. IODataset = Union[Dict[str, InputValues], "pd.DataFrame", str, Sequence[str]] +HOW_TO_FEED_NUMPY = """ +Numpy arrays cannot be fed directly. Instead, feed them in a dictionary i.e. +Instead of: + model.predict(np.array([[1,2],[3,4]])) +Do: + model.predict({"features":np.array([[1,2],[3,4]])}) +""" + SUPPORTED_INPUT_DATA_DESCRIPTION = """\ A dataset can be one of the following: -- A Pandas DataFrame. -- A dictionary of column names (str) to values. Values can be lists of int, float, bool, str or bytes. Values can also be Numpy arrays. -- A YDF VerticalDataset -- A TensorFlow Batched Dataset. -- A typed (possibly sharded) path to a CSV file (e.g. csv:mydata). -- A list of typed paths (e.g. ["csv:mydata1", "csv:mydata2"]). + 1. A dictionary of string (column names) to column values. The values of a column can be a list of int, float, bool, str, bytes, or a numpy array. A 2D numpy array is treated as a multi-dimensional column. + 2. A Pandas DataFrame. + 3. A YDF VerticalDataset created with `ydf.create_vertical_dataset`. This option is the most efficient when the same dataset is used multiple times. + 4. A batched TensorFlow Dataset. + 5. A typed path to a csv file e.g. "csv:/tmp/dataset.csv". See supported types below. The path can be sharded (e.g. "csv:/tmp/dataset@10") or globbed ("csv:/tmp/dataset*"). + 6. A list of typed paths e.g. ["csv:/tmp/data1.csv", "csv:/tmp/data2.csv"]. See supported types below. + +The supported file formats and corresponding prefixes are: + - CSV file. prefix 'csv:' + - Non-compressed TFRecord of Tensorflow Examples. prefix 'tfrecordv2+tfe:' + - Compressed TFRecord of Tensorflow Examples. prefix 'tfrecord+tfe:'; not available in default public build. """ diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/io/pandas_io.py b/yggdrasil_decision_forests/port/python/ydf/dataset/io/pandas_io.py index 80f43143..7e6ea031 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/io/pandas_io.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/io/pandas_io.py @@ -53,6 +53,10 @@ def to_dict( raise ValueError("The pandas DataFrame must be two-dimensional.") data_dict = data.to_dict("series") + for k in data_dict: + if not isinstance(k, str): + raise ValueError("The pandas DataFrame must have string column names.") + def clean(values): if values.dtype == "object": return values.to_numpy(copy=False, na_value="") diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py index 11c8e3be..b6cff606 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/generic_learner.py @@ -77,8 +77,11 @@ def __init__( self._deployment_config = deployment_config self._tuner = tuner + if self._label is not None and not isinstance(label, str): + raise ValueError("The 'label' should be a string") if task != Task.ANOMALY_DETECTION and not self._label: raise ValueError("Constructing the learner requires a non-empty label.") + if self._ranking_group is not None and task != Task.RANKING: raise ValueError( "The ranking group should only be specified for ranking tasks." diff --git a/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py b/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py index 5c0b64fc..87678d1f 100644 --- a/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/learner/learner_test.py @@ -676,6 +676,10 @@ def test_warning_categorical_numerical(self): ) _ = learner.train(ds) + def test_label_is_dataset(self): + with self.assertRaisesRegex(ValueError, "should be a string"): + _ = specialized_learners.RandomForestLearner(label=np.array([1, 0])) # pytype: disable=wrong-arg-types + class CARTLearnerTest(LearnerTest): From 78190b2681e760743d266f70a8edb4aa0fb205e0 Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 17 Jun 2024 12:15:27 -0700 Subject: [PATCH 28/30] Anomaly detection; Surface the isolation forest learner and model in the python API (part 9) PiperOrigin-RevId: 644093348 --- documentation/public/docs/py_api/IsolationForestLearner | 3 +++ documentation/public/docs/py_api/IsolationForestModel.md | 3 +++ documentation/public/docs/py_api/index.md | 2 ++ yggdrasil_decision_forests/port/python/CHANGELOG.md | 1 + yggdrasil_decision_forests/port/python/ydf/BUILD | 3 +++ yggdrasil_decision_forests/port/python/ydf/__init__.py | 2 ++ 6 files changed, 14 insertions(+) create mode 100644 documentation/public/docs/py_api/IsolationForestLearner create mode 100644 documentation/public/docs/py_api/IsolationForestModel.md diff --git a/documentation/public/docs/py_api/IsolationForestLearner b/documentation/public/docs/py_api/IsolationForestLearner new file mode 100644 index 00000000..86d7359f --- /dev/null +++ b/documentation/public/docs/py_api/IsolationForestLearner @@ -0,0 +1,3 @@ +[TOC] + +::: ydf.IsolationForestLearner diff --git a/documentation/public/docs/py_api/IsolationForestModel.md b/documentation/public/docs/py_api/IsolationForestModel.md new file mode 100644 index 00000000..817c5097 --- /dev/null +++ b/documentation/public/docs/py_api/IsolationForestModel.md @@ -0,0 +1,3 @@ +[TOC] + +::: ydf.IsolationForestModel diff --git a/documentation/public/docs/py_api/index.md b/documentation/public/docs/py_api/index.md index f81bf83c..9366ea66 100644 --- a/documentation/public/docs/py_api/index.md +++ b/documentation/public/docs/py_api/index.md @@ -13,6 +13,7 @@ A **Learner** trains models and can be cross-validated. - [DecisionTreeLearner](CartLearner.md): Alias to [CartLearner](CartLearner.md). - [DistributedGradientBoostedTreesLearner](DistributedGradientBoostedTreesLearner.md) +- [IsolationForestLearner](IsolationForestLearner.md) All learners derive from [GenericLearner](GenericLearner.md). @@ -29,6 +30,7 @@ arguments of learner classes. - [RandomForestModel](RandomForestModel.md) - [CARTModel](RandomForestModel.md): Alias to [RandomForestModel](RandomForestModel.md). +- [IsolationForestModel](IsolationForestModel.md) All models derive from [GenericModel](GenericModel.md). diff --git a/yggdrasil_decision_forests/port/python/CHANGELOG.md b/yggdrasil_decision_forests/port/python/CHANGELOG.md index b41d1211..77889aeb 100644 --- a/yggdrasil_decision_forests/port/python/CHANGELOG.md +++ b/yggdrasil_decision_forests/port/python/CHANGELOG.md @@ -4,6 +4,7 @@ ### Feature +- Add support for Isolation Forests model. - Add `max_depth` argument to `model.print_tree`. - Add `verbose` argument to `train` method which is equivalent but sometime more convenient than`ydf.verbose`. diff --git a/yggdrasil_decision_forests/port/python/ydf/BUILD b/yggdrasil_decision_forests/port/python/ydf/BUILD index 1c910007..f53318e5 100644 --- a/yggdrasil_decision_forests/port/python/ydf/BUILD +++ b/yggdrasil_decision_forests/port/python/ydf/BUILD @@ -28,6 +28,7 @@ py_library( "//ydf/model:model_lib", "//ydf/model:model_metadata", "//ydf/model/gradient_boosted_trees_model", + "//ydf/model/isolation_forest_model", "//ydf/model/random_forest_model", "//ydf/model/tree:all", "//ydf/utils:log", @@ -56,6 +57,7 @@ py_library( "@ydf_cc//yggdrasil_decision_forests/learner/gradient_boosted_trees/early_stopping:early_stopping_snapshot_py_proto", "@ydf_cc//yggdrasil_decision_forests/learner/hyperparameters_optimizer:hyperparameters_optimizer_py_proto", "@ydf_cc//yggdrasil_decision_forests/learner/hyperparameters_optimizer/optimizers:random_py_proto", + "@ydf_cc//yggdrasil_decision_forests/learner/isolation_forest:isolation_forest_py_proto", "@ydf_cc//yggdrasil_decision_forests/learner/multitasker:multitasker_py_proto", "@ydf_cc//yggdrasil_decision_forests/learner/random_forest:random_forest_py_proto", "@ydf_cc//yggdrasil_decision_forests/metric:metric_py_proto", @@ -64,6 +66,7 @@ py_library( "@ydf_cc//yggdrasil_decision_forests/model:prediction_py_proto", "@ydf_cc//yggdrasil_decision_forests/model/decision_tree:decision_tree_py_proto", "@ydf_cc//yggdrasil_decision_forests/model/gradient_boosted_trees:gradient_boosted_trees_py_proto", + "@ydf_cc//yggdrasil_decision_forests/model/isolation_forest:isolation_forest_py_proto", "@ydf_cc//yggdrasil_decision_forests/model/multitasker:multitasker_py_proto", "@ydf_cc//yggdrasil_decision_forests/model/random_forest:random_forest_py_proto", "@ydf_cc//yggdrasil_decision_forests/serving:serving_py_proto", diff --git a/yggdrasil_decision_forests/port/python/ydf/__init__.py b/yggdrasil_decision_forests/port/python/ydf/__init__.py index d87a56b3..1ef30a05 100644 --- a/yggdrasil_decision_forests/port/python/ydf/__init__.py +++ b/yggdrasil_decision_forests/port/python/ydf/__init__.py @@ -43,6 +43,7 @@ def _check_install(): from ydf.learner.specialized_learners import RandomForestLearner from ydf.learner.specialized_learners import GradientBoostedTreesLearner from ydf.learner.specialized_learners import DistributedGradientBoostedTreesLearner +from ydf.learner.specialized_learners import IsolationForestLearner DecisionTreeLearner = CartLearner @@ -50,6 +51,7 @@ def _check_install(): from ydf.model.generic_model import GenericModel from ydf.model.random_forest_model.random_forest_model import RandomForestModel from ydf.model.gradient_boosted_trees_model.gradient_boosted_trees_model import GradientBoostedTreesModel +from ydf.model.isolation_forest_model.isolation_forest_model import IsolationForestModel # A CART model is a Random Forest with a single tree CARTModel = RandomForestModel From a1f70810215f33f09ceaff05585f0896d63b7def Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Mon, 17 Jun 2024 12:19:16 -0700 Subject: [PATCH 29/30] Anomaly detection; Anomaly detection tutorial (part 10) PiperOrigin-RevId: 644094544 --- .../docs/tutorial/anomaly_detection.ipynb | 3858 +++++++++++++++++ documentation/public/mkdocs.yml | 1 + 2 files changed, 3859 insertions(+) create mode 100644 documentation/public/docs/tutorial/anomaly_detection.ipynb diff --git a/documentation/public/docs/tutorial/anomaly_detection.ipynb b/documentation/public/docs/tutorial/anomaly_detection.ipynb new file mode 100644 index 00000000..337ccfb4 --- /dev/null +++ b/documentation/public/docs/tutorial/anomaly_detection.ipynb @@ -0,0 +1,3858 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Anomaly detection\n", + "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/google/yggdrasil-decision-forests/blob/main/documentation/public/docs/tutorial/anomaly_detection.ipynb)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install ydf ucimlrepo scikit-learn umap-learn plotly -U -q" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## What is anomaly detection?\n", + "\n", + "**Anomaly detection** techniques are non-supervised learning algorithms for identifying rare and unusual patterns in data that deviate significantly from the norm.\n", + "For example, anomaly detection can be used for fraud detection, network intrusion detection, and fault diagnosis, without the need for defining of abnormal instances.\n", + "\n", + "Anomaly detection with decision forests is a straightforward but effective technique for tabular data. The model assigns an anomaly score to each data point, ranging from 0 (normal) to 1 (abnormal). Decision forests also offer interpretability tools and properties, making it easier to understand and characterize detected anomalies.\n", + "\n", + "In anomaly detection, labeled examples are used not for training but for evaluating the model. These labels ensure that the model can detect known anomalies.\n", + "\n", + "\n", + "We train and evaluate two anomaly detection models on the UCI Covertype dataset, which describes forest cover types and other geographic attributes of land cells. The first model is trained on pine and willow data. Given that willow is rarer than pine, the model differentiates between them without labels. This first model will then be interpreted and characterize what constitute a pine cover type." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/gbm/my_venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "2024-06-17 13:06:13.648825: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2024-06-17 13:06:14.292005: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + ] + } + ], + "source": [ + "# Load libraries\n", + "import ydf # For learning the anomaly detection model\n", + "import pandas as pd # We use Pandas to load small datasets\n", + "from sklearn import metrics # Use sklearn to compute AUC\n", + "from ucimlrepo import fetch_ucirepo # To download the dataset\n", + "import matplotlib.pyplot as plt # For plotting\n", + "import seaborn as sns # For plotting\n", + "import umap # For projecting distances in 2d\n", + "\n", + "# For interactive plots\n", + "import plotly.graph_objs as go\n", + "from plotly.offline import iplot\n", + "import plotly.io as pio\n", + "pio.renderers.default=\"colab\"\n", + "\n", + "# Disable Pandas warnings\n", + "pd.options.mode.chained_assignment = None" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We download the Covertype dataset from UCI." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prepare dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# https://archive.ics.uci.edu/dataset/31/covertype\n", + "covertype_repo = fetch_ucirepo(id=31)\n", + "raw_dataset = pd.concat([covertype_repo.data.features, covertype_repo.data.targets], axis=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Select the columns of interest and clean the labels." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ElevationAspectSlopeHorizontal_Distance_To_HydrologyVertical_Distance_To_HydrologyHorizontal_Distance_To_RoadwaysHillshade_9amHillshade_NoonHillshade_3pmHorizontal_Distance_To_Fire_PointsCover_Type
0259651325805102212321486279Aspen
12590562212-63902202351516225Aspen
2280413992686531802342381356121Lodgepole Pine
327851551824211830902382381226211Lodgepole Pine
42595452153-13912202341506172Aspen
\n", + "
" + ], + "text/plain": [ + " Elevation Aspect Slope Horizontal_Distance_To_Hydrology \\\n", + "0 2596 51 3 258 \n", + "1 2590 56 2 212 \n", + "2 2804 139 9 268 \n", + "3 2785 155 18 242 \n", + "4 2595 45 2 153 \n", + "\n", + " Vertical_Distance_To_Hydrology Horizontal_Distance_To_Roadways \\\n", + "0 0 510 \n", + "1 -6 390 \n", + "2 65 3180 \n", + "3 118 3090 \n", + "4 -1 391 \n", + "\n", + " Hillshade_9am Hillshade_Noon Hillshade_3pm \\\n", + "0 221 232 148 \n", + "1 220 235 151 \n", + "2 234 238 135 \n", + "3 238 238 122 \n", + "4 220 234 150 \n", + "\n", + " Horizontal_Distance_To_Fire_Points Cover_Type \n", + "0 6279 Aspen \n", + "1 6225 Aspen \n", + "2 6121 Lodgepole Pine \n", + "3 6211 Lodgepole Pine \n", + "4 6172 Aspen " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = raw_dataset.copy()\n", + "\n", + "# Features of interest\n", + "features = [\"Elevation\", \"Aspect\", \"Slope\", \"Horizontal_Distance_To_Hydrology\",\n", + " \"Vertical_Distance_To_Hydrology\", \"Horizontal_Distance_To_Roadways\",\n", + " \"Hillshade_9am\", \"Hillshade_Noon\", \"Hillshade_3pm\",\n", + " \"Horizontal_Distance_To_Fire_Points\"]\n", + "dataset = dataset[features + [\"Cover_Type\"]]\n", + "\n", + "# Covert type as text\n", + "dataset[\"Cover_Type\"] = dataset[\"Cover_Type\"].map({\n", + " 1: \"Spruce/Fir\",\n", + " 2: \"Lodgepole Pine\",\n", + " 3: \"Ponderosa Pine\",\n", + " 4: \"Cottonwood/Willow\",\n", + " 5: \"Aspen\",\n", + " 6: \"Douglas-fir\",\n", + " 7: \"Krummholz\"\n", + "})\n", + "\n", + "dataset.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first model is trained on the \"filtered dataset\" than only contain spruce/fir and cottonwood/willow examples." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "filtered_dataset = dataset[dataset[\"Cover_Type\"].isin([\"Spruce/Fir\", \"Cottonwood/Willow\"])]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see, the spruce/fir cover is much more common than the cottonwood/willow cover:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Cover_Type\n", + "Spruce/Fir 211840\n", + "Cottonwood/Willow 2747\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "filtered_dataset[\"Cover_Type\"].value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We train a popular anomaly detection decision forest algorithm called isolation forest." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Anomaly detection model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train model on 214587 examples\n", + "Model trained in 0:00:00.074241\n" + ] + } + ], + "source": [ + "model = ydf.IsolationForestLearner(features=features).train(filtered_dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can then generate \"predictions\" i.e. anomaly scores." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.57844853, 0.609949 , 0.5433627 , 0.6099571 , 0.48067462],\n", + " dtype=float32)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "predictions = model.predict(filtered_dataset)\n", + "predictions[:5]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we plot the model anomaly score's distribution for spruce/fir and cottonwood/willow cover. We se than both distributions are \"separated\", indicating the model's ability to differentiate between the two covers.\n", + "\n", + "**Note:** It's important to note that since cottonwood/willow cover is less frequent, the two distributions are normalized separately. Otherwise, the cottonwood/willow distribution would appear flat." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGyCAYAAAD+lC4cAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACGhklEQVR4nO3dd3iT9frH8XeS7t1CF1DassveyAZBGbLEAYoKLjyKR5xHPf5wH8GBoCi4wY2KgqiIAymyd9lQRqEttAW698r398fTBiqrI+2TpPfrunLlaebnIYXcfKdBKaUQQgghhLBDRr0DCCGEEEJUlxQyQgghhLBbUsgIIYQQwm5JISOEEEIIuyWFjBBCCCHslhQyQgghhLBbUsgIIYQQwm5JISOEEEIIuyWFjBBCCCHslpPeAWqb2Wzm1KlTeHt7YzAY9I4jhBBCiEpQSpGdnU2jRo0wGi/T7qJ0tGbNGjVq1CgVGhqqALV06VLLfUVFReo///mPat++vfLw8FChoaHq9ttvVydPnqzSeyQkJChALnKRi1zkIhe52OElISHhst/zurbI5Obm0qlTJ+666y7Gjx9f4b68vDx27NjBjBkz6NSpE+np6UyfPp0xY8awbdu2Sr+Ht7c3AAkJCfj4+Fg1vxBCCCFqR1ZWFmFhYZbv8UsxKGUbm0YaDAaWLl3KuHHjLvmYrVu30rNnT06cOEHTpk0r9bpZWVn4+vqSmZkphYwQQghhJyr7/W1XY2QyMzMxGAz4+fld8jGFhYUUFhZafs7KyqqDZEIIIYTQg93MWiooKODJJ5/klltuuWxlNnPmTHx9fS2XsLCwOkwphBBCiLpkF4VMcXExN998M0opFixYcNnHPv3002RmZlouCQkJdZRSCCGEEHXN5ruWyouYEydO8Ndff11xnIurqyuurq51lE4IIeqO2WymqKhI7xhCWIWzszMmk6nGr2PThUx5EXP48GFWr15NgwYN9I4khBC6KCoqIi4uDrPZrHcUIazGz8+PkJCQGq3zpmshk5OTw5EjRyw/x8XFERMTQ0BAAKGhodx4443s2LGDn3/+mdLSUpKTkwEICAjAxcVFr9hCCFGnlFIkJSVhMpkICwu7/OJgQtgBpRR5eXmcPn0agNDQ0Gq/lq7Tr6Ojoxk8ePAFt0+ePJnnn3+eyMjIiz5v9erVDBo0qFLvIdOvhRD2rri4mCNHjtCoUSN8fX31jiOE1aSmpnL69GlatWp1QTeTXUy/HjRoEJero2xkiRshhNBVaWkpgLREC4fj4eEBaMV6dcfLSPukEELYCdkvTjgaa/xOSyEjhBBCCLslhYwQQghhBREREcydO1fvGPWOFDJCCCFqzZkzZ7j//vtp2rQprq6uhISEMGzYMNavX693tEr79NNP6devH6CN7TQYDBdcSkpK2Lp1K1OnTtU5bf1j0+vICCGEsG833HADRUVFfPrppzRr1oyUlBRWrVpFampqtV+zuLgYZ2dnK6a8vB9//JExY8ZYfr733nt58cUXKzzGycmJwMDAy75OXeeuL6RFRghRf2Unw7E1kJ+hdxKHlJGRwdq1a3n11VcZPHgw4eHh9OzZk6efftpSGBgMBhYsWMCIESNwd3enWbNmLFmyxPIax48fx2Aw8M033zBw4EDc3Nz48ssvef755+ncuXOF95s7dy4REREVbvvkk09o164drq6uhIaG8uCDD1bId8899xAYGIiPjw9XX301u3btqvD8goICfv/99wqFjIeHByEhIRUucGHXUvm5jRkzBk9PT/73v//V5I9TXIIUMkKI+ic/Az4dA7Nbw2djYE57WPUSmEv1TlYpSinyikp0uVRlWQwvLy+8vLxYtmwZhYWFl3zcjBkzuOGGG9i1axeTJk1i4sSJHDhwoMJjnnrqKaZPn86BAwcYNmxYpd5/wYIFTJs2jalTp7Jnzx6WL19OixYtLPffdNNNnD59ml9//ZXt27fTtWtXhgwZQlpamuUxq1atonHjxrRp06bS532+559/nuuvv549e/Zw1113Ves1xOVJ15IQon7JS4PPx0HSLsAAXkGQkwJr34D8NLjuTbDxac75xaW0ffY3Xd57/4vD8HCp3FeHk5MTixYt4t577+W9996ja9euDBw4kIkTJ9KxY0fL42666SbuueceAF566SX++OMP5s2bx/z58y2Pefjhhxk/fnyVsr788ss89thjTJ8+3XJbjx49AFi3bh1btmzh9OnTlv353njjDZYtW8aSJUssY13+2a0EMH/+fD766CPLz/fddx+zZ8++aIZbb72VO++8s0q5RdVIi4wQon5Z+bRWxHg0hH+thccOwbgFgAG2fQIb5umd0KHccMMNnDp1iuXLlzN8+HCio6Pp2rUrixYtsjymd+/eFZ7Tu3fvC1pkunfvXqX3PX36NKdOnWLIkCEXvX/Xrl3k5OTQoEEDS8uRl5cXcXFxHD16FNBavn766acLCplJkyYRExNjuTz99NOXzFHV3KLqpEVGCFF/JG6D3Yu141u/gZAO2nHnW6EwB359Alb/D9qOBf9w/XJegbuzif0vVq57pTbeu6rc3Ny45ppruOaaa5gxYwb33HMPzz33HFOmTKn0a3h6elb42Wg0XtDNVVxcfC6nu/tlXy8nJ4fQ0FCio6MvuM/Pzw+ALVu2UFJSQp8+fSrc7+vrW6GLqiq5hfVJi4wQon5QClY+pR13uhWa/ON/yj3vhcgBUFIAv/237vNVgcFgwMPFSZeLNVZibdu2Lbm5uZafN23aVOH+TZs2ERUVddnXCAwMJDk5uUIxExMTYzn29vYmIiKCVatWXfT5Xbt2JTk5GScnJ1q0aFHh0rBhQ0DrVrruuuuqvXS+qBtSyAgh6oeEzZC4FZzcYcizF95vMMCI18BggoM/w3H7WefEVqWmpnL11VfzxRdfsHv3buLi4vjuu+947bXXGDt2rOVx3333HZ988gmxsbE899xzbNmypcLsoosZNGgQZ86c4bXXXuPo0aO8++67/PrrrxUe8/zzzzN79mzefvttDh8+zI4dO5g3T+s6HDp0KL1792bcuHH8/vvvHD9+nA0bNvDMM8+wbds2AJYvX35Bt5KwPVLICCHqh+2fatftx4NP6MUfExQFXW/Xjte/VTe5HJiXlxe9evVizpw5DBgwgPbt2zNjxgzuvfde3nnnHcvjXnjhBRYvXkzHjh357LPP+Prrr2nbtu1lXzsqKor58+fz7rvv0qlTJ7Zs2cLjjz9e4TGTJ09m7ty5zJ8/n3bt2jFq1CgOHz4MaK1aK1asYMCAAdx55520atWKiRMncuLECYKDgzl69ChHjhyp9AwpoR+DcvAtpiu7DbgQwoHlZ8DsNlCSD3f/AWE9L/3Y1KMwrxug4IHNEFS9abfWVFBQQFxcHJGRkbi5uekdx6oMBgNLly5l3Lhxekep4M033+TPP/9kxYoVekdxaJf73a7s97e0yAghHN/eJVoRExgFTXpc/rENmkPUKO1YZjDVW02aNLnsbCRhO6SQEUI4vn3LtOsukyq3Rkzvf2vXe7+HgsxaiyVs180330z//v31jiEqQaZfCyEcW34GxG/UjttcV7nnhPWEwDZw5iDs/QG6y4JmtcXBRzeIOiAtMkIIx3bkTzCXQMPWENCscs8xGKDLbdrxzs9rL5sQosakkBFCOLbYldp16+FVe17HiWB0gpPbIWW/9XMJIaxCChkhhOMqLYHDf2jHrUZU7blegdCybOrt3iWXf6wQQjdSyAghHFdSDBRkgJvf5adcX0r7sk0K9/6grQwshLA5UsgIIRzX8XXadXhfMFZjmfnWI8DZA9LjtKJICGFzpJARQjiuExu06/A+l3/cpbh4Qqvy7qUfrJNJCGFVUsgIIRyTufTctOuIvtV/nXZl3Uv7l0n3khA2SAoZIYRjStkLhVng4g3BHar/Oi2GgpMbZMTDaZm9JOpOREQEc+fOrfXXNRgMLFu2DIDjx49jMBgq7CRu66SQEUI4pvLdq5teBaYarP3p4gHNBmvHh2TfnepITk7m3//+N82aNcPV1ZWwsDBGjx7NqlWrKvX8559/ns6dO19w+/lfwPVJZGQkP//8M87OzixevLjCfRMnTsRgMHD8+PEKt0dERDBjxgwAtm7dytSpU+sqbq2TQkYI4ZjKu5WqOz7mfK3Lpm4f+rXmr1XPHD9+nG7duvHXX3/x+uuvs2fPHlauXMngwYOZNm2a3vHszu7du0lPT2fYsGF0796d6OjoCvdHR0cTFhZW4fa4uDhOnDjB1VdfDUBgYCAeHh51mLp2SSEjhHBMp3Zq10261/y1WpUtpndyO2Qn1/z1akopKMrV51LFcUIPPPAABoOBLVu2cMMNN9CqVSvatWvHo48+yqZNmwCIj49n7NixeHl54ePjw80330xKSgoAixYt4oUXXmDXrl0YDAYMBgOLFi0iIiICgOuvvx6DwWD5GWDBggU0b94cFxcXWrduzeefV1yd2WAw8NFHH3H99dfj4eFBy5YtWb58ueX+7t2788Ybb1h+HjduHM7OzuTk5ACQmJiIwWDgyJEjAKSnp3PHHXfg7++Ph4cHI0aM4PDhwxXe8/vvv6ddu3a4uroSERHB7NmzK9x/+vRpRo8ejbu7O5GRkXz55ZcX/fP88ccfGT58OM7OzgwePLhCwXLgwAEKCgq4//77K9weHR2Nq6srvXv3BqreZbVmzRp69uyJq6sroaGhPPXUU5SUlADw888/4+fnR2lpKQAxMTEYDAaeeuopy/Pvuecebrvttkq/X1XJXktCCMeTcwYyEwADhHau+et5B0Pj7nBym7ZScLcpNX/NmijOg1ca6fPe/z2lzeaqhLS0NFauXMn//vc/PD0vfI6fnx9ms9lSxKxZs4aSkhKmTZvGhAkTiI6OZsKECezdu5eVK1fy559/AuDr68t1111HUFAQCxcuZPjw4ZhM2vT6pUuXMn36dObOncvQoUP5+eefufPOO2nSpAmDBw+2vPcLL7zAa6+9xuuvv868efOYNGkSJ06cICAggIEDBxIdHc3jjz+OUoq1a9fi5+fHunXrGD58OGvWrKFx48a0aNECgClTpnD48GGWL1+Oj48PTz75JCNHjmT//v04Ozuzfft2br75Zp5//nkmTJjAhg0beOCBB2jQoAFTpkyxvMapU6dYvXo1zs7OPPTQQ5w+ffqCP7Ply5fz6KOPAjB48GBmzpxJUlISoaGhrF69mn79+nH11Vfz/vvvW56zevVqevfujZubW6U+t/OdPHmSkSNHMmXKFD777DMOHjzIvffei5ubG88//zz9+/cnOzubnTt30r17d9asWUPDhg0rFFJr1qzhySefrPJ7V5a0yAghHM+pHdp1w5bg5mOd15TupSo7cuQISinatGlzycesWrWKPXv28NVXX9GtWzd69erFZ599xpo1a9i6dSvu7u54eXnh5ORESEgIISEhuLu7ExgYCGjFUEhIiOXnN954gylTpvDAAw/QqlUrHn30UcaPH1+hhQW0wuGWW26hRYsWvPLKK+Tk5LBlyxYABg0axLp16ygtLWX37t24uLgwadIky5dzdHQ0AwcOBLAUMB999BH9+/enU6dOfPnll5w8edIyfufNN99kyJAhzJgxg1atWjFlyhQefPBBXn/9dQBiY2P59ddf+fDDD7nqqqvo1q0bH3/8Mfn5+RUynzx5kt27dzNihPa72LdvX1xcXC7I1a1bN86ePUtcXBygFRLnF3FVMX/+fMLCwnjnnXdo06YN48aN44UXXmD27NmYzWZ8fX3p3LlzhQyPPPIIO3fuJCcnh5MnT3LkyBHLn1dtkBYZIYTjOVlWyDTqar3XbD0S/noJjkVrXSyVbJWoFc4eWsuIXu9dSZXZ2frAgQOEhYURFhZmua1t27b4+flx4MABevToUaV4Bw4cuGAga9++fXnrrbcq3NaxY0fLsaenJz4+PpYWkPNbGTZs2MDAgQMZNGgQs2bNArTC4IknnrC8n5OTE7169bK8XoMGDWjdujUHDhywPGbs2LEXZJo7dy6lpaWW1+jWrZvl/jZt2uDn51fhOcuXL6dfv36W2z08POjRowfR0dHccsstllxOTk706dOH6OholFLEx8dXu5A5cOAAvXv3xmAwVMiek5NDYmIiTZs2tbRgPfbYY6xdu5aZM2fy7bffsm7dOtLS0mjUqBEtW7as1vtXhhQyQgjHU94i09iKhUxQFPiFQ8YJOLoaokZZ77WrymDQt5CqpJYtW2IwGDh48KDeUS7g7Oxc4WeDwYDZbAa0Vp5OnToRHR3Nxo0bueaaaxgwYAATJkwgNjaWw4cP12oLw6UsX76cMWPGVLht8ODBfPPNN+zbt4/8/Hy6dtV+5wcOHMjq1asxm814eHhUKLSsbdCgQXzyySfs2rULZ2dn2rRpw6BBg4iOjiY9Pb3W/6yka0kI4ViUOjfQ15otMgaD1ioD0r1USQEBAQwbNox3332X3NzcC+7PyMggKiqKhIQEEhISLLfv37+fjIwM2rZtC4CLi4tlMOn5nJ2dL7g9KiqK9evXV7ht/fr1lteqrPJC4O+//2bQoEEEBAQQFRXF//73P0JDQ2nVqpXl/UpKSti8ebPluampqRw6dMjynpfK1KpVK0wmE23atKGkpITt27db7j906BAZGRmWn3Nycli9evUFLTuDBw/m8OHDfPXVV/Tr188yVmjAgAGsWbOG6OhoSxdUdURFRbFx48YKrWvr16/H29ubJk2aAOdasObMmWMpWsoLmejoaAYNGlSt96405eAyMzMVoDIzM/WOIoSoC+nxSj3no9QLAUoV5Vv3tY+t0V771WZKlZZa97UvIz8/X+3fv1/l51v5fOrA0aNHVUhIiGrbtq1asmSJio2NVfv371dvvfWWatOmjTKbzapz586qf//+avv27Wrz5s2qW7duauDAgZbX+PLLL5Wnp6fauXOnOnPmjCooKFBKKdWyZUt1//33q6SkJJWWlqaUUmrp0qXK2dlZzZ8/X8XGxqrZs2crk8mkVq9ebXk9QC1durRCTl9fX7Vw4ULLz8uWLVMmk0mFhIRYbps+fboymUxq4sSJFZ47duxY1bZtW7V27VoVExOjhg8frlq0aKGKioqUUkpt375dGY1G9eKLL6pDhw6pRYsWKXd39wrvN3z4cNWlSxe1adMmtW3bNtWvXz/l7u6u5syZo5RS6rvvvlMdOnS44M83Pz9fubq6Km9vbzVr1izL7QUFBcrNzU15e3urV155pcJzwsPDLa/7zz+PuLg4BaidO3cqpZRKTExUHh4eatq0aerAgQNq2bJlqmHDhuq5556r8JqdO3dWJpNJLViwQCmlVGpqqnJ2dlaAOnjw4AW5z89/qd/tyn5/SyEjhHAsB1doxcb8PtZ/7ZIipV5por1+4jbrv/4l2HMho5RSp06dUtOmTVPh4eHKxcVFNW7cWI0ZM8ZSXJw4cUKNGTNGeXp6Km9vb3XTTTep5ORky/MLCgrUDTfcoPz8/BRgKQCWL1+uWrRooZycnFR4eLjl8fPnz1fNmjVTzs7OqlWrVuqzzz6rkKcyhUxqaqoyGAxqwoQJltuWLl2qAPXee+9VeG5aWpq6/fbbla+vr3J3d1fDhg1TsbGxFR6zZMkS1bZtW+Xs7KyaNm2qXn/99Qr3JyUlqeuuu065urqqpk2bqs8++6xCwXHbbbepZ5555qJ/vgMHDlSA2rRpU4XbBw0apAC1cePGCrdXpZBRSqno6GjVo0cP5eLiokJCQtSTTz6piouLK7zm9OnTFaAOHDhgua1Tp04VCsGLsUYhYyg7CYeVlZWFr68vmZmZ+PhYafaCEMJ2/f06/PUydJwI49+/8uOr6ts7YP+PMOhpGPTUlR9vBQUFBcTFxREZGVmtKbTCvpWUlBAcHMyvv/5Kz5499Y5jVZf73a7s97eMkRFCOJbkvdp1cLvaef2W12rXsb/VzusL8Q9paWk88sgjVZ7BVV9IISOEcCwp+7Tr2ipkWgzVrk/tgJwLFywTwtqCgoL4v//7vwpToMU5UsgIIRxHUR6kHdWOg9vXznt4h0BoJ+34SOU2PRRC1B4pZIQQjuPMQVBm8AzUthWoLeXdS4d/r733uAgHH9Io6iFr/E5LISOEcBy13a1UrryQOboKSktq973AsjZIUVFRrb+XEHUpLy8PuHCBwqqQlX2FEI7DUsjUUrdSucbdwN0f8tMhcSuE967Vt3NycsLDw4MzZ87g7OyM0Sj/BxX2TSlFXl4ep0+fxs/Pz1KsV4cUMkIIx3G6rJAJqtoqrlVmNGmDfvd8p3Uv1XIhYzAYCA0NJS4ujhMnTtTqewlRl8o3/awJKWSEEI7jzCHtOvDSuy1bTcth5wqZoc/V+tu5uLjQsmVL6V4SDsPZ2blGLTHlpJARQjiG/AzISdGOG9beTrsWLYYABkjZC5knwbdxrb+l0WiUBfGE+AfpaBVCOIazsdq1T2Nwq4NVvD0CoEnZAmVH/qj99xNCXJQUMkIIx1DerdSwVd29p2UathQyQuhFChkhhGM4Wz4+pnXdvWfLa7TrY9FQUlh37yuEsJBCRgjhGPRokQnpCF7BUJQD8Rvr7n2FEBZSyAghHMMZHVpkjEZoUdYqI91LQuhC10Lm77//ZvTo0TRq1AiDwcCyZcsq3K+U4tlnnyU0NBR3d3eGDh3K4cOH9QkrhLBdxfmQEa8dN6zDQgbOdS/JbthC6ELXQiY3N5dOnTrx7rvvXvT+1157jbfffpv33nuPzZs34+npybBhwygoKKjjpEIIm3b2MKC01XY9G9btezcfDEYnSD0Macfq9r2FEPquIzNixAhGjBhx0fuUUsydO5f/+7//Y+zYsQB89tlnBAcHs2zZMiZOnFiXUYUQtiy1rKW2YSswGOr2vd18oWlvOL4WDv8JvabW7fsLUc/Z7BiZuLg4kpOTGTp0qOU2X19fevXqxcaNlx5UV1hYSFZWVoWLEMLBpZa1hDRooc/7l3cv1fFu2EIIGy5kkpOTAQgODq5we3BwsOW+i5k5cya+vr6WS1hYWK3mFELYgNQj2nVAM33ev3w9meNroShPnwxC1FM2W8hU19NPP01mZqblkpCQoHckIURtSzuqXevVIhPYBnzDoKQAjq/TJ4MQ9ZTNFjLlu2GmpKRUuD0lJeWyO2W6urri4+NT4SKEcHDlLTINmuvz/gaDdC8JoRObLWQiIyMJCQlh1apVltuysrLYvHkzvXv31jGZEMKm5KVBfrp2rFfXEpy3XcFvoJR+OYSoZ3SdtZSTk8ORI0csP8fFxRETE0NAQABNmzbl4Ycf5uWXX6Zly5ZERkYyY8YMGjVqxLhx4/QLLYSwLall3UrejcDFU78ckQPA5KqtZ3M2tm4X5hOiHtO1kNm2bRuDBw+2/Pzoo48CMHnyZBYtWsR//vMfcnNzmTp1KhkZGfTr14+VK1fKNvZCiHMs42N06lYq5+IJEf3g6Cqte0kKGSHqhK6FzKBBg1CXaYI1GAy8+OKLvPjii3WYSghhV/QeH3O+lteeK2T6/FvvNELUCzY7RkYIISqlvGspwBYKmbIBvyc2QoGsYSVEXZBCRghh32yla6k8Q0BzMBdD3Bq90whRL0ghI4Swb+nHtWv/SF1jWFhmL8k0bCHqghQyQgj7lZ8OBZnasV9TfbOUs6wn84dMwxaiDkghI4SwX+kntGvPQHD10jdLufC+4OwB2UmQvEfvNEI4PClkhBD2y9KtFKFnioqc3SByoHYs3UtC1DopZIQQ9iujrEXGL1zfHP/UqnyczB/65hCiHpBCRghhv2yxRQagRdk4mcQt2hYKQohaI4WMEMJ+lY+RsbVCxi8MgtqCMsPRv/ROI4RDk0JGCGG/LC0yNta1BOdmL8Wu1DeHEA5OChkhhH0yl0JmgnZsay0yAK1Hatexv0FJob5ZhHBguu61JERdyswv5s/9KRSUlGIyGLiqWQMiGuq4W7KomewkKC0CoxP4NNY7zYWa9ASvEMhJhmPR0GqY3omEcEhSyAiHV1BcyqINx1kQfZTM/GLL7SajgVt6hjF9SCsCvV11TCiqpXx8jG8YGE36ZrkYoxHajoEtH8D+5VLICFFLpGtJOLS03CImfLCJWb8eJDO/mGaBnlzbNpgeEf6UmhVfbIpnxFtrOZScrXdUUVW2OmPpfG3HatcHf4bS4ss/VghRLdIiIxzWyYx87vh4M0fP5OLn4cyM69oyrktjTEYDAJuOpfLsj3uJTclh4gcb+fzuXrRv7KtzalFptjzQt1zT3tqqw7lnIO5vaDFE70RCOBxpkREO6WxOIbd8sImjZ3IJ9XVjyb96c0O3JpYiBuCqZg347r4+dArzIz2vmFs/3MSxMzk6phZVkmGjU6/PZzRB1GjteP+P+mYRwkFJISMcTl5RCXcv2kp8Wh5hAe4sub8PLYK8L/pYXw9nvri7J12a+pFVUMLUz7eTXSBdAHbBHrqW4B/dSyX6ZhHCAUkhIxxKqVnx0Nc72ZWYib+HM5/e2ZPGfu6XfY63mzPv396NEB83jpzO4bFvd2E2y67FNi/dRrcn+KfwfuAeAHmpcGK93mmEcDhSyAiHMvfPWP48cBpXJyMfTe5Os8DK7Ygc5O3Ggtu64mIy8vv+FN5dfaSWk4oaKc7XpjWD7bfImJwgapR2vH+ZrlGEcERSyAiH8du+ZOb9pRUgs27oQLfwgCo9v0tTf14a1w6AN/+M5a+DKVbPKKwkI167dvUBd399s1RG23Ha9f4foaRI1yhCOBopZIRDiDuby2Pf7gLgrr6RXN+lSbVeZ0KPptx2VVOUgulfx8jgX1t1/owlg+GyD7UJkQPBK1jrXjryp95phHAoUsgIu1dUYmb64p3kFJbQMzKAp0e2qdHrPTuqHd3D/ckuLOGez7aRJYN/bY+9DPQtZ3KCDjdpx7sX65tFCAcjhYywe2/+EcvuxEx83Z15a2JnnE01+7V2cTIy/7auhPq6cexMLtO/3kmpDP61LfYy0Pd8HSdo14dWQn66vlmEcCBSyAi7tulYKu//fRSAV2/oQKjv5WcoVVaQtxsf3N4dVycjqw+d4bXfDlrldYWV2FuLDEBIBwhqB6WFsG+Z3mmEcBhSyAi7VVhSyn+X7kEpmNA9jOHtQ636+h2a+PL6TZ0AeH/NMZbuTLTq64sasIfF8P7JYIBOZa0yu7/RN4sQDkQKGWG33l9zjGNncmno5cp/r4uqlfcY06kR0wY3B+DJ7/ewKyGjVt5HVFFGgnbt11TfHFXV4WYwGCF+I6TF6Z1GCIcghYywS3Fnc3mnbK2XGaOi8HV3rrX3euya1gyNCqKoxMw9n20jIS2v1t5LVEJhNhRmasc+jfXNUlU+odoMJoDd3+qbRQgHIYWMsEszVxygqMRM/5YNGdOpUa2+l9FoYM6EzrQJ8eZMdiFTFm4hI0/WAtFN5knt2s0XXCu34KFN6TRRu475Esyl+mYRwgFIISPsTkxCBr/vT8FogOdGt8VQB+uIeLs5s/DOHoT6unH0TC73fraNgmL5EtJFVtlYJZ/qrRWku6gx4OanjfORNWWEqDEpZITdeeO3QwCM79rkkptB1oZQX3cW3dkTbzcnth5P59FvY2RPJj2Ut8j42lm3UjkXD+hym3a85UN9swjhAKSQEXZlw5GzrDtyFmeTgelDWtb5+7cO8eaD27vjYjKyYk8yL/9yoM4z1HtZZYWMvY2POV/3u7TrI39C2jF9swhh56SQEXZl7p+HAbi1Z1PCAjx0ydC7eQNev6kjAJ+sj+OrzfG65Ki37L1FBqBBc2gxFFCw9WO90whh16SQEXZjd2IGW46n4WwycP+gFrpmGdu5MU8Maw3Ac8v3sv2ErNRaZ+x9jEy5Hvdq1zu/gCKZCSdEdUkhI+zGx+u0dTdGdWxEiK+bzmnggUHNGdE+hOJSxf1fbOd0doHekeqHzLJCxp5bZABaXqOtg1OQAXu/1zuNEHZLChlhF5IzC/hldxIAd/eL1DmNxmAw8PpNnWgZ5MXp7EKeXLIbpWTwb61S6lzXkj2PkQEwmqD73drxlve1cxNCVJkUMsIufLrxOCVmRc/IANo39tU7joWXqxPvTuqKS9meTF9vSdA7kmPLT4eSfO3Y3gsZgK53gLMHJO+BY6v1TiOEXZJCRti8wpJSvt6iDai1ldaY87UK9uY/ZeNlXv5lPydSc3VO5MDKu5U8A8FZ/+7FGvMIgK6TteO1b+qbRQg7JYWMsHl/7j9NRl4xob5uDI0K1jvORd3VN5JekQHkFZVvZCndBLXCEaZe/1PvaWB0guNrIWGr3mmEsDtSyAibt2S71l0zvmtjTMbaX8W3OoxGA6/d2BEXJyPrj6SyYk+y3pEck2Wgr53PWDqfXxh0LNu2IHqmvlmEsENSyAiblpJVwJrYMwDc0NW2v7zCG3hy/0Btp+yXf9lPbmGJzokckCO2yAAMeFxrlTm6Ck5s1DuNEHZFChlh05buPIlZQfdwf5oF2v4GgfcPak5YgDtJmQWW3bmFFTnCYngXExB5btuCv16SGUxCVIEUMsJmKaX4bpvWrXRTd9tujSnn5mzi2VHtAG3dm1MZ+ToncjCO2iIDMOAJMLnCifVw4Ce90whhN6SQETZr36ksjp7Jxc3ZyMgOoXrHqbShUUH0jAygqMTM3D9j9Y7jWBxxjEw53ybQ59/a8e/PQLEUwUJUhhQywmb9tk8bMDu4dRDebs46p6k8g8HAUyPaALBkeyJHTmfrnMhBmM2QdUo7dsQWGYD+j4J3I8iIhw3v6J1GCLsghYywWb/u1QqZ4e1DdE5SdV2b+nNt22DMCl7/7ZDecRxD7mkwF4PBCN7200JXJS6ecO1L2vG6N8+NCRJCXJIUMsImHTmdzZHTOTibDAxuE6R3nGp5YlhrjAb4bV8KO+JlU8kaK/9S9woBk5O+WWpT+xugaW8ozoM/ntU7jRA2TwoZYZN+25cCQN8WDfGxo26l87UM9rZMGX/114OySF5NZTnIZpFXYjDAiFcBA+xdAkdW6Z1ICJsmhYywSSvLu5Xa2V+30vkevqYVLk5GNselEV22Ho6oJsvUawcc6PtPoZ2g133a8U8PQ2GOrnGEsGVSyAibk5iex56TmRgNMLStbW5JUFmN/dy546pwAF5beQizWVplqs2Rp15fzNUzwLcpZMbDXy/rnUYImyWFjLA5qw+eBqB7eAANvVx1TlNz0wa3wNvViQNJWfyyJ0nvOPbLkadeX4yrF4yeox1vfg8StuibRwgbZdOFTGlpKTNmzCAyMhJ3d3eaN2/OSy+9JGMNHNzaw2cBGNg6UOck1uHv6cI9/ZsBMOfPWEpKzTonslP1rUUGoMVQ6HQLoGD5v6GkUO9EQtgcmy5kXn31VRYsWMA777zDgQMHePXVV3nttdeYN2+e3tFELSkpNbPxaCoA/Vo01DmN9dzVLwI/D2eOncnlx5hTesexT466PcGVDHsFPAPhzEFYO1vvNELYHJsuZDZs2MDYsWO57rrriIiI4MYbb+Taa69lyxZpYnVUuxIzyC4swc/DmfaNffWOYzXebs7cN0DbUHLuqliKpVWmakqLIadsR3GfetK1VM4jAEa8ph2vfRNS9umbRwgbY9OFTJ8+fVi1ahWxsdoy77t27WLdunWMGDHiks8pLCwkKyurwkXYj79jtW6lvs0bYjIadE5jXZP7hNPQy4WEtHy+25aodxz7kp0EygxGZ611or5pdz20vk5bEHD5v8FcqnciIWyGTRcyTz31FBMnTqRNmzY4OzvTpUsXHn74YSZNmnTJ58ycORNfX1/LJSwsrA4Ti5pad0QrZPq3dJxupXIeLk7cP6gFAPP+OkxBsXwZVVp5t5JPIzDa9D9btcNggOtmg6svnNyuDf4VQgA2Xsh8++23fPnll3z11Vfs2LGDTz/9lDfeeINPP/30ks95+umnyczMtFwSEhLqMLGoiayCYmISMgDo54CFDMCkXk0J8XEjKbOAxVvi9Y5jP7Lq0Royl+ITCte+qB2vegnS4vTNI4SNsOlC5oknnrC0ynTo0IHbb7+dRx55hJkzZ17yOa6urvj4+FS4CPuw8WgqpWZFs4aeNPH30DtOrXBzNvHg1VqrzDurj5JfJK0ylVI+9bo+zVi6mK6TIaI/lOTDT9NBZnAKYduFTF5eHsZ/NCObTCbMZhko6YjKZyv1daDZShdzc/cwmvi7czankM83Hdc7jn3Iqqczlv7JYIDRb4GTG8StgZ1f6J1ICN3ZdCEzevRo/ve///HLL79w/Phxli5dyptvvsn111+vdzRRC7af0DZW7BEZoHOS2uXiZOShIS0BWBB9lJzCEp0T2YH6tD3BlTRoDoP/qx2vegEKs/XNI4TObLqQmTdvHjfeeCMPPPAAUVFRPP7449x333289NJLekcTVpZbWML+JG2GWfdwf53T1L7xXRoT2dCT9LxiFq6TsQ5XVL5hZH2ben0pve6HgOaQewbWv6V3GiF0ZdOFjLe3N3PnzuXEiRPk5+dz9OhRXn75ZVxcXPSOJqxsV0IGpWZFI183Gvm56x2n1jmZjDw8VGuV+WDtMTLzinVOZOPq62J4l+LkAte8oB1veAeyZOsLUX/ZdCEj6o9tZd1K3SIcu1vpfKM7NqJVsBfZBSV8tO6Y3nFsV3EB5GnT8uv9YN/ztRkFYVdpA3/XzdE7jRC6kUJG2ITyQqY+dCuVMxoNPDK0FQAL1x+XVplLKR/o6+wB7vXn9+OKDAYY/LR2vH0RZMnWF6J+kkJG6K7UrNhZ3iJTjwoZgGHtQmgd7E1OYQkLN8hYmYs6f7NIg2Ot9lxjkQOhaW8oLYR1c/VOI4QupJARujuUnE12YQlerk60CfHWO06dMhoNlnVlPlkXR3aBtMpcoHwNGRkfcyGDAQY+qR3v+Azy0vTNI4QOpJARutt+QvvHt0tTP5xM9e9XcmSHUJoHepJVUMJnG0/oHcf2WLYnkBlLF9VsEIR01MbKbP1Y7zRC1Ln6960hbM72etqtVM50XqvMx+viZLXff8qSFpnLMhigz7+14y0faIOjhahHpJARutt9MhOAzmF++gbR0eiOjWji705abhFLdsjO2BVknjdGRlxcu+u1P5/c07DvB73TCFGnpJARusouKObYmVwAOjT21TmNfpxMRu7uFwnAR2uPUWqWPXQsZHuCKzM5Q/e7tGPpXhL1jBQyQlf7Tmmr+Tb2c6eBl6vOafR1c/cwfN2dOZGax+/7kvWOYzssi+GF6ZvD1nW9A4zOcHIbJO3SO40QdUYKGaGrPYlat1J9bo0p5+nqxO1XhQPw/t/HULKzsbaPUKH2OyJdS1fgFQRRo7VjaZUR9YgUMkJXe8rGx3RoIoUMwOQ+Ebg4GYlJyCAmIUPvOPorb41x8wVXL32z2IMed2vXe7+Hojx9swhRR6SQEbqyFDLSIgNAoLcrozqGAshUbJDNIquqaR/wC4eiHDj4i95phKgTUsgI3WQVFBN3Vgb6/tPk3hEA/LI7ibM5hfqG0ZtsFlk1RiN0nKAd716sbxYh6ogUMkI3e8taY5r4u+PvKTual+sU5kenJr4UlZr5ZmuC3nH0Vb6qr4yPqbxOE7Xro39Bdoq+WYSoA1LICN3IQN9Lu72sVearzfH1eyq2TL2uugbNoUkPUGbYu0TvNELUuhoVMkVFRSQmJhIfH1/hIkRlyEDfSxvVMRR/D2dOZuQTfei03nH0kyljZKqlvHtpl3QvCcdXrULm8OHD9O/fH3d3d8LDw4mMjCQyMpKIiAgiIyOtnVE4qL0y0PeS3JxNjO+qfXl/t60er/QrLTLV0/4GbU2Z5N2Qsl/vNELUKqfqPGnKlCk4OTnx888/ExoaisFgsHYu4eByC0s4kaZND20b6qNzGtt0c/cwPl4Xx58HUkjNKax/CwYqJdsTVJdHALQaBgd/1gb9XvOi3omEqDXVKmRiYmLYvn07bdq0sXYeUU8cTM5GKQjydq1/X9CV1DrEm45NfNmdmMmymFOWLQzqjfx0bUdnkEKmOjpO0AqZPUtgyPPajCYhHFC1frPbtm3L2bNnrZ1F1CMHk7WtCdpIa8xl3dStvHspof6t9Fs+PsYzEJzd9M1ij1peCy7eWvfcye16pxGi1lSrkHn11Vf5z3/+Q3R0NKmpqWRlZVW4CHElB5K035OoUG+dk9i2MZ0a4+Jk5GByNntP1rO/W1nSrVQjzm5a9xLA/mW6RhGiNlWrkBk6dCibNm1iyJAhBAUF4e/vj7+/P35+fvj7+1s7o3BAB5KyARkfcyW+Hs5c2zYYgB921rNBv+UtMr4yY6na2o7Vrg8s18YcCeGAqjVGZvXq1dbOIeoRs1lxKFkrZKKkkLmisZ0b8/PuJH7ZncT/XdcWk7GeDK6XFpmaazEUnD0gIx6SYqBRF70TCWF11SpkBg4caO0coh5JTM8np7AEF5ORyIaeesexeQNaNcTHzYnT2YVsjkulT/OGekeqG5YWGSlkqs3FA1peA/t/hP3LpZARDqnaw9gzMjKYPXs299xzD/fccw9z5swhMzPTmtmEg9pfNj6mZbAXziaZSXElrk4mRrTXNpL8adcpndPUIZl6bR3l3Uv7f5TuJeGQqvUtsm3bNpo3b86cOXNIS0sjLS2NN998k+bNm7Njxw5rZxQOpnzGknQrVd6Yzo0AWLEnmaISs85p6kiWjJGxipbXgskV0o7CaVkcTzieahUyjzzyCGPGjOH48eP88MMP/PDDD8TFxTFq1CgefvhhK0cUjqZ8xlKbEJmxVFlXNWtAoLcrmfnFrD18Ru84tc9shqwk7VhaZGrG1VsbKwNaq4wQDqbaLTJPPvkkTk7nhtg4OTnxn//8h23btlktnHBMMmOp6kxGA9d1qEfdS7mnwVwMBiN4h+qdxv61HaNdSyEjHFC1ChkfH5+Lbg6ZkJCAt7f8L1tcWm5hCfFlWxPIYnhVM7qT9oW+6sBpCktKdU5Ty8rHx3iFgKlacxLE+VoNB6MTnDkIqUf1TiOEVVWrkJkwYQJ3330333zzDQkJCSQkJLB48WLuuecebrnlFmtnFA7k8OkcAAK9XQnwdNE5jX3pEuZPsI8r2YUlbDiSqnec2pUlM5asyt0PIvppx4d+1TWKENZWrf/qvPHGGxgMBu644w5KSkoAcHZ25v7772fWrFlWDSgcS2zZ+jGtgr10TmJ/jEYDw9uF8OnGE6zYk8TgNkF6R6o95S0yMtDXelqPhGPRcGgF9HlQ7zRCWE21WmRcXFx46623SE9PJyYmhpiYGNLS0pgzZw6urrIBoLi02JTyQka6IKtjRNk4md/3p1Bc6sCzl2QxPOtrNVy7jt8IeWn6ZhHCimq0iIeHhwcdOnSgQ4cOeHh4WCuTcGCHpJCpkR4RATT0ciEzv5iNRx24e0m2J7A+/3AIbg/KDId/1zuNEFZT6a6l8ePHs2jRInx8fBg/fvxlH/vDDz/UOJhwTIdTtDEyUshUj8loYFi7EL7cHM+ve5MY0CpQ70i1o7yQkRYZ62o9ElL2at1LnSbqnUYIq6h0i4yvry8Gg7bHi4+PD76+vpe8CHExmfnFJGcVANqqvqJ6ylf5/X1fCiWO2r1U3rUkg32tq/UI7frIKigp1DeLEFZS6RaZhQsXWo4XLVpUG1mEgztc1q3UyNcNHzdnndPYr17NAvD3cCY1t4gtx9Mcb++l0mLITtaOfaRryapCO2vr8mQnQdxaaDlU70RC1Fi1xshcffXVZGRkXHB7VlYWV199dU0zCQcVW9at1FK6lWrE2WTk2rYhAPy6J1nnNLUgOwlQYHQGTwftOtOL0Xhu0O+hFfpmEcJKqlXIREdHU1RUdMHtBQUFrF27tsahhGMqn7HUWrYmqLERHbRCZuW+ZMxmB9sI0LJZZCPti1dYV5vrtOtDv8omksIhVGkdmd27d1uO9+/fT3Lyuf8NlpaWsnLlSho3lj5tcXHlhUzLIBkfU1N9mjfE282JM9mFbI9Pp0dEgN6RrCdL1pCpVRH9wdkTsk9BUgw06qJ3IiFqpEqFTOfOnTEYDBgMhot2Ibm7uzNv3jyrhROORVpkrMfFycg1bYP5YcdJVuxJcqxCRmYs1S5nN2hxNRz4SWuVkUJG2LkqtdvGxcVx9OhRlFJs2bKFuLg4y+XkyZNkZWVx11131VZWYcdScwo5m6N1R7aQFhmrGFk2e2nlXgfrXpIZS7Wv9UjtWsbJCAdQpRaZ8PBwAMxmB53yKWpN+UDfsAB3PFxkE0Br6NeyIV6uTiRlFhCTmEHXpv56R7IO2Z6g9rUcpu0snrwHMhLAL0zvREJUW7W+UT777LPL3n/HHXdUK4xwXIdPl3UryYwlq3FzNnF1myCW7zrFr3uSHKeQKd8wUqZe1x7PBhB2FcRv0LqXek3VO5EQ1VatQmb69OkVfi4uLiYvLw8XFxc8PDykkBEXOFS2WaRMvbaukR1CtEJmbzL/HRllWbTSrmXKztd1ovWIskJmhRQywq5Va25jenp6hUtOTg6HDh2iX79+fP3119bOKBxA+dYE0iJjXQNbBeHubCIxPZ+9J7P0jlNzxfmQV7aHlAz2rV3l42SOr4OCTH2zCFEDVlukoWXLlsyaNeuC1hohlFLEni5vkZGBvtbk7qJ1LwGs2JukcxoryDqlXTt7gLuDdJXZqoYtoGErMBfDkT/1TiNEtVl1tSknJydOnTplzZcUDuBMdiEZecUYDdA8UAoZaxvevnyV3ySUvS9wdv7Ua0foJrN15XsvHfpV3xxC1EC1xsgsX768ws9KKZKSknjnnXfo27evVYIJx1E+YymigSduziad0ziewW2CcHUycjw1jwNJ2bRt5KN3pOqTqdd1q/VIWP8WHP5d2+PKJHugCftTrUJm3LhxFX42GAwEBgZy9dVXM3v2bGvkEg7kUNlCeK1kfEyt8HJ1YmCrQH7fn8LKvUn2XchYtieQGUt1okkP8GgIeWfhxAZoNlDvREJUWbW6lsxmc4VLaWkpycnJfPXVV4SGhlo7o7Bzhy2FjHQr1ZaRHbS/dyv22vkmklkyY6lOGU3nbSIp3UvCPtV4jIxSqlb75U+ePMltt91GgwYNcHd3p0OHDmzbtq3W3k9Yn6VFRrYmqDVXRwXhYjJy5HSOpXC0S5YWGSlk6oxlnMwvsomksEvVLmQ+/vhj2rdvj5ubG25ubrRv356PPvrImtlIT0+nb9++ODs78+uvv7J//35mz56Nv7/MZrAXSinL1GvpWqo9Pm7O9GvZEIAVe+y4VUbGyNS95oPByQ0y4uH0fr3TCFFl1Roj8+yzz/Lmm2/y73//m969ewOwceNGHnnkEeLj43nxxRetEu7VV18lLCyMhQsXWm6LjIy0ymuLunEqs4CcwhKcjAYiGnjqHcehjWgfwl8HT/Pr3iSmD22pd5zqsWxPIEvm1xkXT2g2CGJXwsEVENxO70RCVEm1WmQWLFjAhx9+yMyZMxkzZgxjxoxh5syZfPDBB8yfP99q4ZYvX0737t256aabCAoKokuXLnz44YeXfU5hYSFZWVkVLkI/5TteNwv0xMXJqrP9xT9c0zYYJ6OBg8nZHDuTo3ecqivIgsKyhdmka6luySaSwo5V65uluLiY7t27X3B7t27dKCkpqXGocseOHWPBggW0bNmS3377jfvvv5+HHnqITz/99JLPmTlzJr6+vpZLWJj8z05P5eM1ZGuC2ufn4ULfFlr30rIYO1zPqbxbyc0XXGVgeJ1qNRwwwKkdkOUACyuKeqVahcztt9/OggULLrj9gw8+YNKkSTUOVc5sNtO1a1deeeUVunTpwtSpU7n33nt57733Lvmcp59+mszMTMslISHBanlE1R1Klq0J6tL4rlpLxvfbEzGb7Wzgpky91o93MDQp+89prMxeEval0mNkHn30UcuxwWDgo48+4vfff+eqq64CYPPmzcTHx1t1w8jQ0FDatm1b4baoqCi+//77Sz7H1dUVV1dXq2UQNVO+67VMva4bw9qF4O3mxMmMfDYeS7W00NgFmXqtr9YjIHGrNg27+116pxGi0ipdyOzcubPCz926dQPg6NGjADRs2JCGDRuyb98+q4Xr27cvhw4dqnBbbGws4eHhVnsPUXvMZpmxVNfcnE2M6dSILzfH8922BPsqZGTqtb5aXwerXoRja6AwR7r3hN2odCGzevXq2sxxUY888gh9+vThlVde4eabb2bLli188MEHfPDBB3WeRVRdYno++cWluDgZCZcZS3Xmxm5N+HJzPL/uTebFgmJ83Oxk2XmZeq2vwNbgHwnpcXD0L2g7Ru9EQlSKTU8j6dGjB0uXLuXrr7+mffv2vPTSS8ydO9eq43BE7SlfCK9FoBcmo2wAWFc6h/nRIsiLwhIzP++yo4GbmWXj2Xyb6pujvjIYoM112rHMXhJ2pNItMuPHj2fRokX4+Pgwfvz4yz72hx9+qHGwcqNGjWLUqFFWez1Rd2JlawJdGAwGburWhJm/HuS77Qnc2stOCoPyna99ZbCvblqPgI3vQOxvUFoCpmotNSZEnap0i4yvry8Gg8FyfLmLEHBeISNbE9S567s2xmQ0sDM+gyOn7WDLArP5vMXwpJDRTdhV4O4P+WmQsFnvNEJUSqXL7fLVdZVSvPDCCwQGBuLu7l5rwYT9iy0f6BskhUxdC/J2Y1CrQFYdPM132xN5ekSU3pEuL+8slBYCBvBppHea+svkBC2Hwe7FWvdSRF+9EwlxRVUeI6OUokWLFiQmJtZGHuEgSkrNHD0jM5b0dFN3rWXjhx0nKSk165zmCsrHx3iHgslOBic7qjbnrfIrm0gKO1DlQsZoNNKyZUtSU1NrI49wECfS8igqMePubKKJv7Tc6eHqNsEEeLpwJruQtYfP6h3n8mR8jO1ofjWYXCDtGJyN1TuNEFdUrVlLs2bN4oknnmDv3r3WziMcxLmtCbwwyowlXbg4GRnbWeum+Warja9wLYWM7XD1hsiB2vHBX/TNIkQlVKuQueOOO9iyZQudOnXC3d2dgICAChchyrcmkG4lfU3ooe019seBFFKyCnROcxlSyNiW1iO060OyXYGwfdWaWzdnzhzLDCYhLkamXtuGNiE+dA/3Z9uJdL7dmsC/h7TUO9LFWdaQkU1ebULrEfDLo9qWBTmnwStI70RCXFK1CpkpU6ZYOYZwNAeSswDti1Toa9JVTdl2Ip2vt8TzwOAWtrk4obTI2BafRtCoC5zaCbEroav19tATwtqq1bVkMpk4ffr0BbenpqZiMplqHErYt4LiUo6fzQWgTah0LeltRPtQ/DycOZVZQPShC//e2gQpZGxP67LZSwdllV9h26pVyKhLTMkrLCzExcWlRoGE/TuckoNZQYCnC4FeshO53tycTdzUTSsQvtwcr3OaiyjOh9wz2rEUMrajvJA5thqK8vTNIsRlVKlr6e233wa0JdA/+ugjvLzOjX8oLS3l77//pk2bNtZNKOzOuW4lbxlLZSNu6dmUD9fGsfrQaRLT82ji76F3pHOyTmnXzp7aqrLCNgS30/a9yozXipnyfZiEsDFVKmTmzJkDaC0y7733XoVuJBcXFyIiInjvvfesm1DYnUPJ2kDf1rI1gc1oFuhF3xYNWH8klcVbEnh8WGu9I51jGejbRNu4UNgGg0FbHG/ze9rieFLICBtVpUImLi4OgMGDB/PDDz/g7y//exIXOljWIhMlA31tyqRe4VohszWB6UNb4myqVs+y9cn4GNvVekRZIbMSzKVglDGQwvZU61+y1atXVyhiSktLiYmJIT093WrBhP2SFhnbdE3bYAK9XTmbU8gf+1P0jnOOFDK2K7wvuPpqe2ElbtM7jRAXVa1C5uGHH+bjjz8GtCJmwIABdO3albCwMKKjo62ZT9iZM9mFnM0pwmCQxfBsjbPJyITu2jotX24+oXOa85zftSRsi8kZWl6jHR+SVX6FbapWIfPdd9/RqVMnAH766SeOHz/OwYMHeeSRR3jmmWesGlDYl/LWmMgGnri7SDO0rZnYMwyDAdYfSeVY2aaeupMWGdtm2URSVvkVtqlahUxqaiohISEArFixgptuuolWrVpx1113sWfPHqsGFPalfHyMdCvZpib+HlzdWlul9estNjIVWwoZ29ZiKBidtQ0kzx7RO40QF6hWIRMcHMz+/fspLS1l5cqVXHON1vSYl5cnC+LVcwfLWmRkRV/bNemqpgB8tz2RguJSfcMoJYWMrXPzhYh+2vHBn/XNIsRFVKuQufPOO7n55ptp3749BoOBoUOHArB582ZZR6aekxYZ2zewVRCN/dzJyCvm171J+obJS4WSss0sfRrrm0VcWtRo7frAT/rmEOIiqlXIPP/883z00UdMnTqV9evX4+qqrd5qMpl46qmnrBpQ2I+SUjOHU7RxF1GyNYHNMhkN3NKzbNDvJp27l8oH+noFg5OsAm2z2lwHGODktnMLGAphI6q1aSTAjTfeeMFtkydPrlEYYd+Op+ZRWGLGw8VEmC2tHCsucHP3MOb+eZhtJ9I5kJRFVKhOXYHSrWQfvEMgrCckbIaDv0DPe/VOJIRFpQuZt99+m6lTp+Lm5mbZquBSHnrooRoHE/anfMZSq2BvjLa4w7KwCPJxY1j7EH7ZncSi9cd59caO+gSRQsZ+RI3WCpkDy6WQETal0oXMnDlzmDRpEm5ubpatCi7GYDBIIVNPHTxvjyVh++7sE8Evu5NYFnOSJ0e0IcBThw1fLYVMWN2/t6iaNqPg9/+D4+shLw08AvROJARQhUKmfHuCfx4LUe5AUvmMJSlk7EG3cH86NPZlz8lMvt4Sz7TBLeo+REbZGB1pkbF9AZEQ3AFS9mhrynSZpHciIYAqFDKPPvpopR5nMBiYPXt2tQMJ+3UopXzGkky9tgcGg4E7+0bw6Le7+HzjCaYOaFb3+y+VFzJ+Tev2fUX1RI3WCpkDP0khI2xGpQuZnTt3Vvh5x44dlJSU0Lq1totubGwsJpOJbt26WTehsAs5hSUkpOUD0iJjT67rGMorKw6SnFXAr3uTGdOpUd0GyCjbKsEvvG7fV1RP1CiIfgWO/gWF2eAqf9eF/ir936/Vq1dbLqNHj2bgwIEkJiayY8cOduzYQUJCAoMHD+a662Sr9/qofKBvsI8r/nqMtRDV4upk4rayBfIWrq/jLuOCLMgv22hWWmTsQ1BbCGgGpYVw5E+90wgBVHMdmdmzZzNz5swKO2D7+/vz8ssvS7dSPXVuoK90K9mbSb3CcTEZ2RmfQUxCRt29cXm3krs/uMnvjV0wGGRxPGFzqlXIZGVlcebMmQtuP3PmDNnZ2TUOJexPeYtMG1kIz+4EersyqlMoUMetMpbxMdKtZFfalBUysb9DSaG+WYSgmoXM9ddfz5133skPP/xAYmIiiYmJfP/999x9992MHz/e2hmFHTgoM5bs2l19IwH4ZXcSKVkFdfOmlvEx0q1kVxp3A+9QKMqGY2v0TiNE9QqZ9957jxEjRnDrrbcSHh5OeHg4t956K8OHD2f+/PnWzihsnFLq3B5LwdJFYI/aN/alR4Q/JWbF5xtP1M2bppe9j7+0yNgVo1FbUwa0xfGE0Fm1ChkPDw/mz59PamoqO3fuZOfOnaSlpTF//nw8PT2tnVHYuMT0fLIKSnAxGWkR5KV3HFFNd/fTWmU+33SC3MKS2n9D6VqyX1FlhczBX6C0Dn5XhLiMGi0a4enpSceOHenYsaMUMPXYvlOZALQK8cLFqY7XIRFWc03bECIaeJCZX8y32xJq/w1l6rX9Cu8HHg0gPw2O/613GlHPybeOqLG9J7VupXahvjonETVhMhq4p38zAD5eF0dJqbn23kypcy0y0rVkf0xOEDVGO963VN8sot6TQkbUWHmLTPvGMj7G3t3YrQkNPF1ITM9nxd7k2nuj/HQo1Apg2WfJTrW7Xrs+8BOUFuubRdRrUsiIGtt7SvtCattIWmTsnZuzicl9IgB4f81RlFK180bl3UqeQeDiUTvvIWpXeF/wDNSKUpm9JHQkhYyokdNZBZzJLtTWyZI1ZBzC7VeF4+5sYt+pLDYcTa2dN5E9luyfdC8JGyGFjKiRfWWtMc0DvfBwqfTWXcKG+Xu6cHN3bTfq9/8+VjtvIlOvHUP7snXDDv4EJUX6ZhH1lhQyokbKx8e0ayTjYxzJPf2bYTTA37Fn2F9WrFqVTL12DE17g1cwFGTCsWi904h6SgoZUSPlLTLtZXyMQwkL8GBkB23bgg/X1kKrjKzq6xiMJmg7Vjve94O+WUS9JYWMqJG90iLjsO4b0ByA5btOkZCWZ90Xl64lx1E+e+ngL1BcR9tbCHEeKWREtWXmF5OQlg9AWylkHE6HJr70b9mQUrPivTVHrffC568hI11L9i/sKvBpok2nj12pdxpRD0khI6ptT6LWGhMW4I6fh4vOaURteHBwCwC+25Zovc0kc89AST5gAN8m1nlNoR+jETrcqB3v/lbfLKJekkJGVNuuxAwAOjXx0zWHqD29mjWgZ0QARaVmPrDWDKby1hifRuDkap3XFPrqNFG7Pvw75KXpm0XUO1LIiGrblZABSCHj6KZdrbXKfLU5ntScwpq/YPpx7VoG+jqOoCgI6QDmYhn0K+qcFDKi2naXdS11CvPTN4ioVQNaNqRjE1/yi0v5ZH1czV9QNot0TB3LWmWke0nUMSlkRLUkZxaQnFWA0SB7LDk6g8HAtLKxMp9tOEFmfg331ZHNIh1T+xvAYISEzZBmhYJXiEqSQkZUS/n4mFbB3rKibz1wTVQwrYO9yS4s4bMNx2v2YumyhoxD8gmFyIHasbTKiDokhYyolt0y0LdeMRoNPDBYW1fm4/Vx5BaWVP/F0soGDftHWiGZsCnlg353f6NNsxeiDkghI6plV4I2PqZjmKzoW1+M6tiIyIaeZOQV8+nG49V7kZIiyEzQjhs0t1o2YSPajAJnD0g7qnUxCVEH7KqQmTVrFgaDgYcffljvKPWa2aykRaYeMhkN/LtsBtP7a46RVVCNsTLpx0GZwdlT26NHOBZXL2hXtpHk9k/1zSLqDbspZLZu3cr7779Px44d9Y5S7x1PzSWroARXJyOtQ7z1jiPq0NjOjWke6ElmfjGfrKvGgM7ybqWAZmAwWDecsA3dJmvX+5ZCfoauUUT9YBeFTE5ODpMmTeLDDz/E399f7zj1Xvm063aNfHA22cWvkLASk9HAo9e0BuDjtXGk5xZV7QXSyrY6aNDMysmEzWjSAwKjtNWb9y7RO42oB+ziW2jatGlcd911DB069IqPLSwsJCsrq8JFWFdM2UJ4HaVbqV4a0T6EqFAfsgtL+KCqO2OnlhUyATI+xmEZDND1Du1YupdEHbD5Qmbx4sXs2LGDmTNnVurxM2fOxNfX13IJCwur5YT1T/nU686yEF69ZDQaeOyaVgAsWn+cM9lVWO23vGtJBvo6tk4TweQCybvh1E690wgHZ9OFTEJCAtOnT+fLL7/Ezc2tUs95+umnyczMtFwSEhJqOWX9UlxqZt8prZVLVvStv4ZEBdEpzI/84lIWRFdhZ+zyrqUA6VpyaB4BEDVGO5ZWGVHLbLqQ2b59O6dPn6Zr1644OTnh5OTEmjVrePvtt3FycqK0tPSC57i6uuLj41PhIqznUHI2RSVmfNyciGjgoXccoRODwcDj12qtMl9sPkFSZv6Vn1RSCJmJ2rF0LTm+8u6lPUugMEffLMKh2XQhM2TIEPbs2UNMTIzl0r17dyZNmkRMTAwmk0nviPWOZcfrMD8MMuukXuvXoiG9IgMoKjHz9qrDV35C+glt6rWLF3gF1X5Aoa+I/lrBWpQNu77WO41wYDZdyHh7e9O+ffsKF09PTxo0aED79u31jlcv7bIM9JWF8Oo7g8HAE8O0GUzfbE3gQNIVBtZbupUiZep1fWA0Qs+p2vHm98Fs1jePcFg2XcgI21O+oq8shCcAukcEcF2HUMwKXv5lP+pyy9Jb1pCRbqV6o8skcPWB1MNw9C+90wgHZXeFTHR0NHPnztU7Rr2UW1jC4dPZgAz0Fec8NaINLk5G1h9J5c8Dpy/9wPKp1zJjqf5w9YYut2nHmxfom0U4LLsrZIR+9p7MxKwgxMeNYJ/KzSITji8swIN7+mkbQP7vl/0UlVyiC0FmLNVPPacCBjjyJ5yJ1TuNcEBSyIhKK1/RV8bHiH96YHALGnq5cjw1j88utaGkdC3VTwGR0HqEdrz5PX2zCIckhYyotB3x6QB0buqnbxBhc7xcnfhP2cDft1YdJjXnH4vknT/1WrqW6p9e/9Kud30NeWn6ZhEORwoZUWk74zMA6NpU9rsSF7qhWxPaNfIhu6CEOX/+owuhfNdrFy/wDNQln9BR5AAIbg/FebDlQ73TCAcjhYyolFMZ+SRnFWAyGqRrSVyUyWhgxqi2AHy1OZ69JzPP3Sm7XtdvBgP0f1Q73jQfCrP1zSMcihQyolLKu5XahHjj4eKkcxphq65q1oBRHbXp2P9duodSc9l0bJmxJNqOgwYtoCADtn6sdxrhQKSQEZWy40QGIN1K4sqeHdUWb1cndidm8sWmE9qNabLrdb1nNEH/x7Tjje9AUZ6+eYTDkEJGVMrOBK1Fpmu4n75BhM0L8nHjP8O1gb+v/3aIlKyCil1Lov7qcBP4NYXcM7DjM73TCAchhYy4osKSUvad1JaflxYZURm39gqnU5gfOYUlvPjTfjh7RLtDupbqN5Mz9HtEO17/ljabTYgakkJGXNHek1kUlZoJ8HShaYDseC2uzGQ08Mr17TEZDazeEwdZZVOvG7bSN5jQX+dJ4N0Isk/Btk/0TiMcgBQy4op2lg307dpUdrwWldeukS939omgueEUAMojEDwCdE4ldOfkCgP/ox2veQ0KMi//eCGuQAoZcUXl68d0kW4lUUWPXNOKHp7a/ksJpjCd0wib0eV2rXUuPw3WzdU7jbBzUsiIKyqfet1FVvQVVeTp6sRtLbRxEGszAjiULOuHCMDkBEOf1443zYfMk7rGEfZNChlxWUmZ+SRlFmA0QKcmfnrHEXaoGdqX1BFzKM8s3YO5fG0ZUb+1HglNe0NJAUS/oncaYcekkBGXVd6t1CbEB09XWQhPVMNZbbuCRFMY206k8822BJ0DCZtgMMA1L2nHMV9B8l598wi7JYWMuKwdJ2T9GFEDpcWWNWSu7t8fgFdWHNDWlhEirIe24q8ywy+PgdmsdyJhh6SQEZe1MyEDgC5hMtBXVEPaMTCXgLMnN199FZ3C/MguKOGZpXtRSrqYBDDsf+DsCQmbIOYLvdMIOySFjLikohIze8o2/usaLoWMqIYzh7Trhi0xmYy8dkNHnE0G/jyQws+7k/TNJmyDbxMY/F/t+I9nIfesvnmE3ZFCRlzSvlOZFJWY8fdwJqKBLIQnquHMQe06KAqA1iHeTBvcAoDnl+8jLbdIr2TClvT6FwR3gPx0rZgRogqkkBGXdP76MbIQnqiW0/u167JCBuCBQS1oE+JNam4RL/y0T6dgwqaYnGDUHMAAMV9C3N96JxJ2RAoZcUk7zlvRV4hqOX1Auw5qa7nJxcnIqzd0xGiAH2NOsepAik7hhE0J6wHd79SOlz0gK/6KSpNCRlxSeYuMbBQpqqWkEFLLNos8r0UGoFOYH/f013bC/u/SPWTkSReTAK55EfzCITMBfn1K7zTCTkghIy4qJauAkxn5GA3QMcxP7zjCHqUe0WYsufqAT+ML7n70mlY0a+hJSlYh/7dMZjEJwNUbrn8fMMCur+DAT3onEnZAChlxUeUbRbYK9sZLFsIT1WHpVorSFj/7BzdnE3MmdMbJaODn3Un8GHOqjgMKmxTeG/pO145/mg7Z0vUoLk8KGXFRO8q7lWTataguy0Dftpd8SKcwPx4a0hKAGT/u5WRGfl0kE7Zu8H8huD3kpcLSqWAu1TuRsGFSyIiLsqzoK+NjRHVdZKDvxTwwqDldmmoL5T32bYzsxSTAyRVu+BicPeBYNPz9ut6JhA2TQkZc4PyF8GTHa1FtKWVTq4PaXPZhTiYjcyd0xsPFxKZjaXy07lgdhBM2L6hN2ZRsIHoWHP1L3zzCZkkhIy5wICmLwhIzfh7ONGvoqXccYY8KMiHjhHYc3P6KDw9v4Mmzo7SWmzd+i2XvSZl6K4BOE6HrZEDB9/dCloyjEheSQkZcoHz9mC5hfrIQnqie8tYYnybgEVCpp0zoEcY1bYMpKjXz4Fc7yC4orsWAwm6MeBVCOkDeWVhyF5SW6J1I2BgpZMQFZP0YUWPJe7XrkA6VforBYOD1GzvS2M+d46l5PPX9HpmSLcDZHW76FFy8IX4j/PWi3omEjZFCRlzA0iIjhYyoruTd2nUVChkAPw8X5t3aBSejgV/2JPHxurhaCCfsToPmMO5d7Xj9W3Bwhb55hE2RQkZUcDqrgMT0fAwG6BTmq3ccYa+S92jXIVceH/NPXZv689+R2krAr6w4wOpDp62ZTNirtmOh1/3a8bJ/QfpxXeMI2yGFjKhgW9m06zYhPni7OeucRtil0pJzU6+r2CJT7s6+EdzcvQlmBf/+aiexKdlWDCjs1jUvQpMe2mDybydDcYHeiYQNkEJGVLD1eBoAPSKkW0lUU+phKC3UxjT4RVTrJQwGAy+P60DPyAByCku4+9OtpOYUWjensD9OLnDjQnAPgKQY+O2/eicSNkAKGVHB9rIWmW6yoq+orvO7lYzV/yfGxcnIe7d1o2mABwlp+fzri+0UlsgKr/WeXxiM/xAwwLaPYfd3eicSOpNCRljkFpaw71QWAD0iKjdlVogLnIrRrkM61vilAjxd+GRKd7zdnNh6PJ2nf5CZTAJoORQGPKEd//QQnD6obx6hKylkhMWuhAxKzYpGvm408nPXO46wV6d2ateNuljl5VoEefPurV0xGQ38sOMk764+YpXXFXZu0FMQORCK82DJnVAs+3TVV1LICIutx7Vupe7SGiOqy1wKSbu0YysVMgADWgXy/Jh2ALzxeyw/7ZIVXus9owlu+Ag8A7UNSv94Tu9EQidSyAiLbSe0gb7dZaCvqK6zh6E4F5w9oWFLq7707VeFc3e/SAAe+24X28t+X0U95hUE4xZox1veh9jf9c0jdCGFjACg1KwsK/p2D5cWGVFN5d1KoZ20/zFb2X9HRjE0KpiiEjP3frad+NQ8q7+HsDMtr4Fe/9KOf3wAcmTdofpGChkBwMHkLHIKS/B2daJ1iLfecYS9svL4mH8yGQ28fUtn2jf2IS23iDsXbSFL9mQSQ1+AoHaQewZ+nAYyILxekUJGAOemXXcJ98dklI0iRTWVFzKNu9baW3i4OPHx5B6E+rpx9Ewuj36zC7NZvrjqNWc3bbyMyRUO/w5bPtA7kahDUsgI4LyBvrJ+jKiukqJzeyyFdq7Vtwr2ceP927vh4mTkzwMpLFhztFbfT9iB4LZw7cva8e8zIGW/vnlEnZFCRgCw/bgM9BU1lLIXSgrA3V/b5K+WdWzix0tjy2cyHeLv2DO1/p7CxvW8F1oO01aW/mEqlMhq0PWBFDKCkxn5nMoswGQ00DnMT+84wl4lbtWum/QAQ910T07o0ZSJPcJQCqYv3kliugz+rdcMBhgzDzwaQMoeWP2K3olEHZBCRrCtrDWmfSMfPFycdE4j7FbCFu26SY86fdvnx7SjQ2Nf0vOKeeDLHRQUyzYG9Zp3MIx+Szte/xac2KBvHlHrpJARbDtevr+STLsWNXB+i0wdcnM2seC2rvh7OLM7MZPnl++r0/cXNihqNHS+DVCw9D4oyNI7kahFUsgItpXNWJIdr0W15ZyGjBOAARp3q/O3b+Lvwdu3dMFggMVbE1i8Jb7OMwgbM3wm+DWFjHhY+bTeaUQtkkKmnssqKOZgsva/lW5SyIjqKu9WCooCNx9dIvRvGcjj17YG4Nnl+9idmKFLDmEj3Hzg+vcBA8R8AQd+0juRqCVSyNRzO+MzUArCG3gQ5O2mdxxhrxI2a9d13K30T/cPbH7eyr/bOJUhGwnWa+F9oO907fin6ZCdom8eUStsupCZOXMmPXr0wNvbm6CgIMaNG8ehQ4f0juVQNh9LBWRbAlFD8Ru16/A+usYwGg28OaETLYO8SMkqZMrCLWTmy8q/9drgZyC4A+SlwvJ/y6q/DsimC5k1a9Ywbdo0Nm3axB9//EFxcTHXXnstubm5ekdzGJvKCpmrmkkhI6qpKPfcir46FzIAPm7OLLyzB0HersSm5DD1s23kF8lMpnrLyQXGf1C26u9vsH2R3omEldl0IbNy5UqmTJlCu3bt6NSpE4sWLSI+Pp7t27frHc0h5BaWsDsxE4CrmjXQOY2wW4lbwVwCvmHa4Eob0MTfg4V39sDL1YnNcWnctWgreUUlescSegluC0Of045/+y+kykrQjsSmC5l/yszUvnQDAi7delBYWEhWVlaFi7i47SfSKTErGvu5ExbgoXccYa/K1+lo2lvfHP/QrpEvn96lFTMbj6Vy58KtssFkfdbrfojoD8V52pTsUilsHYXdFDJms5mHH36Yvn370r59+0s+bubMmfj6+louYWFhdZjSvmws61bq3VxaY0QNlBcyNtCt9E/dwgP49K6elpaZGxdsICFNVv+tl4xGGLcAXH21VsR1c/ROJKzEbgqZadOmsXfvXhYvXnzZxz399NNkZmZaLgkJCXWU0P6cGx8jhYyoppLCcwvhhffVN8sldAv3Z/HUqwj20cbMXD9/PRuOntU7ltCDXxhc94Z2vGYWnJRhCo7ALgqZBx98kJ9//pnVq1fTpEmTyz7W1dUVHx+fChdxofPHx/SKlIG+opoStmgbRXoGQcOWeqe5pPaNfVk2rS9RoT6czSli0kebeeO3QxSXmvWOJupah5ug3fXauK5vp0Bemt6JRA3ZdCGjlOLBBx9k6dKl/PXXX0RGRuodyWFsO5FOqVnRxF/Gx4gaiFujXTcbWGcbRVZXqK8739/fmwndtU0m31l9hDHvrGdXQobe0URdMhhg1Fzwj4TMePj+HjDLrDZ7ZtOFzLRp0/jiiy/46quv8Pb2Jjk5meTkZPLzZZGrmtp4VLqVhBUcKytkIgfqm6OSPFycePXGjsy7pQu+7s4cSMri+vnrefbHvWTkFekdT9QVdz+Y8AU4ucPRVbDmVb0TiRqw6UJmwYIFZGZmMmjQIEJDQy2Xb775Ru9odm/dkTMA9JGBvqK6CrLOjTFoZh+FTLnRnRqx6rGBjOvcCLOCzzaeYNAb0Xy+8Tgl0t1UP4S0P7dL9ppX4eAKffOIarPpQkYpddHLlClT9I5m19Jyi9h3SpuW3q9FQ53TCLt1YgOoUq2J3kbWj6mKhl6uzJ3Yha/u6UXrYG8y8oqZ8eM+Rs1bJ4OB64tOE6DHvdrx93fDyR365hHVYtOFjKgd64+cRSloE+JNkI/srySq6dhq7drOWmP+qU+LhvzyUD9eGtsOPw9nDiZnc+uHm7n/i+0yVbs+GD4Tml+trS/z1QRIP6F3IlFFUsjUQ2sPa91K/VtKa4yogcN/aNfNh+ibwwqcTEZu7x1B9OODmNw7HJPRwK97kxn65ho+XheH2Sz78zgskzPc9Km2H1PuafjyRshP1zuVqAIpZOoZpRTrDmvN5v1aBuqcRtittGOQdhSMTtBskN5prMbPw4UXxrZnxUP96d2sAYUlZl76eT+3fLhJdtJ2ZG4+MOlb8GkMZ2Ph8/FSzNgRKWTqmWNnczmVWYCLyUjPCFk/RlTT4T+166a9tS8BB9M6xJuv7u3F/65vj4eLic1xaYx8ey2rD57WO5qoLT6NYNIScA+AUzvgs7GyxoydkEKmnlkbq3Ur9Yj0x93FpHMaYbeOlHUrtRiqb45aZDAYmNQrnF+n96dDY18y8oq5c9FWZv16UBbSc1TBbWHKz+DREJJ2wadjIFcGfts6KWTqmXVHyrqVWki3kqim4nyIW6sdt7xG3yx1ILyBJ0vu783k3uEAvLfmKLd8sImkTOlqckjB7WDKL+AVDCl74KMhkLJP71TiMqSQqUcKiksthcyAVjLQV1TTsWgoyQefJhDUVu80dcLVycQLY9szf1JXvF2d2HYinZFvrWX1IelqckhBbWDKCm1ZgfTj8NE1sG+Z3qnEJUghU49sPJpKQbGZRr5utA11vHENoo4c/EW7bjPS5rclsLaRHUL5+aF+tG/sQ3peMXcu3MrMXw9QWCJL3Duchi1g6hpt1eriXPhuMvz2DBQX6J1M/IMUMvXInwdSALg6KghDPfsCElZiLoXYldpx65H6ZtFJeANPlvyrD7dfpXU1vb/mGNe9vY4d8TLLxeF4BMBtP0DvB7WfN74D7/WDI6v0zSUqkEKmnlBK8VfZjIshbYJ1TiPsVuJWyD0Drr4Q0U/vNLpxczbx0rj2vHdbNxp6uXLkdA43LNjA49/tIiVL/sfuUExOMOx/MPFrbdxM6mH4Yrw2qynub1CyxpDepJCpJ/YnZZGUWYC7s4nesr+SqK6DP2vXra7VFhKr54a3D+GPRwZwQ9cmKAVLticy6PVo3vrzMPlF0t3kUNqMhGmb4aoHtPWTjkXDp6PhnR6w+hU4vh5KZONRPRiUcuxyMisrC19fXzIzM/Hxqb/jQuatOszsP2IZGhXMR5O76x1H2COl4K2OkBGvrYTabpzeiWzKzvh0Xvp5PzviMwAI9XXjqRFtGNOpkXTlOpr047BhHsR8pW1tUM7ZA0I7a+NrGrSAgObgH6FdXL30yWrHKvv9LYVMPTH23fXsSshg1vgOTOxpfxv8CRuQuB0+ulr7x/qJo+DioXcim6OU4ufdScz69SAny1YC7trUj+dGt6NTmJ++4YT1FWbD/h+1MTNxf0PeZdac8QqB0E4Q0RdaXgtBUXWX005JIVNGChk4lZFPn1l/YTDA5qeHyEaRonp+e0Yb7NhuPNy0UO80Nq2guJSP18Xx7uoj5JV1Md3QtQlPDm8tf/8clVJw+oC25kzqYTh7GNLjtNabi2130KgLXDUN2t8ARhnlcTFSyJSRQgY+WnuMl385QM+IAL79V2+94wh7pBTMaQ9ZiXDz59B2jN6J7EJKVgGvrjzIDztOAuDl6sR/hrdmUi9tY0pRT+RnwJlDcHI7HP1LG19jLtbuC+4AY96Gxl31TGiTKvv9LWVgPfDz7iQARnUK1TmJsFsJm7UixsWrXqzmay3BPm68eXNnlk3rS6cwP3IKS3j2x32MX7CB/aey9I4n6oq7HzTtBb0fgNuWwKMHYPAz4OpTtnrwUFg7W2ZAVZMUMg4uIS2PmIQMDAZthoUQ1bJrsXYdNRqc3fXNYoc6h/nxw/19eGlce7xdndiVkMHod9Yxc8UB8opK9I4n6ppXIAz8DzwUA+2uB1UKq16Epf+SmU/VIIWMg1uxR2uN6RUZQJC39M2LaigphH1LteNOE/XNYsdMRgO3XxXOn48N5LoOoZSaFe//fYxr3vxbdtWurzwbwE2LYNQcMJhg92JYcieUFuudzK5IIePgfikrZEZ1bKRzEmG3Yn+DggzwbgQR/fVOY/eCfdx4d1JXPpnSncZ+7pzMyOfORVuZ9uUOWUyvvup+F9z6DZhctLWalt4HZtlhvbKkkHFgcWdz2Z2YiVG6lURN7Ppau+54MxhN+mZxIFe3CeaPRwcwdUAzTEYDv+xJYujsNXy+8TilZhkrUe+0vAYmfAlGZ9j7PUTP1DuR3ZBCxoF9uy0BgAGtAmno5apzGmGXspK0FhmAzrfqm8UBebg48d+RUSx/UBsMnF1Ywowf93HDgg3sPZmpdzxR11pdC6Pnasd/v3auS1dclhQyDqqk1MyS7YkATOgepnMaYbdivtAGIjbtDYGt9U7jsNo18uWH+/vw4th2eLk6EVM2GPixb3eRlJmvdzxRl7rcBn3+rR3/+G9IPapvHjsghYyDWn3oDGeyC2ng6cKQKNkkUlSD2Qw7PtOOu07WN0s9YDIauKN3BH8+OpDRnRqhFHy/Q9u76bWVB8kqkAGg9caQ57X/PBRly+DfSpBCxkF9s1XrVhrftTEuTvIxi2o4ukrbV8nVF9qO1TtNvRHi68a8W7qwbFpfekYGUFhiZn70UQa9Hs3C9XEUlshmlA7P5AQ3fgLu/pC0C9bN0TuRTZNvOAd0OquA1Ye06ZwTeki3kqimze9r110myb5KOugc5sc3U6/iwzu60zzQk7TcIl74aT9D31zDsp0nMcuAYMfm0whGvK4dr3kNkvfqm8eGSSHjgL7YHE+pWdE93J8WQd56xxH26OwROPIHYICe9+qdpt4yGAxc0zaY3x4ewMzxHQjydiUhLZ+Hv4lh1Lx1rIk9g4PvMlO/dbgRWl+nbWfw4wPSxXQJUsg4mPyiUj7feByAO/tG6htG2K8tH2jXrYZDQDN9swicTEZu6dmUNU8M5olhrfF2dWJ/UhaTP9nCpI82syshQ++IojYYDDDqTXDz07qY1s/VO5FNkkLGwSzZkUh6XjFhAe4MayeDfEU15KXBzs+141736ZtFVODuYmLa4Bb8/Z/B3NMvEheTkQ1HUxn77nqmfbmD42dz9Y4orM07BEa8ph1HvwpnYvXNY4OkkHEgpWbFx2uPAXBX30icTPLximrY8gEU50FIR2g2SO804iL8PV34v1Ft+evxgYzv2hiDQVvF+5o5a5j160FyCmX/JofS8WZoea3WxfTTdFn19x/km86B/LE/heOpefi4OXGzrB0jqqMo99wg334Pa03bwmY18ffgzZs7s+Kh/gxoFUhxqeK9NUe5+o1olu5MlPEzjsJggJFvgLMHxG8412IqAClkHIbZrJj7p9bkeNtV4Xi6OumcSNilbZ9Afhr4R0CUTLm2F1GhPnx6Zw8+ntyd8AYenM4u5JFvdnHDgg3sSZQVgh2CfzgMfkY7/mMG5MhGo+WkkHEQy3ed4mByNt5uTkwdIIMzRTUU5pxbr6L/49paFsJuGAwGhkQF8/sjA/jP8NZ4uJjYEZ/BmHfX8dT3uzktG1Lav17/gtBOUJAJK5/WO43NkELGARSXmnnzD6015r4BzfDzcNE5kbBLWz6AvFRtllKnW/ROI6rJ1cnEA4Na8NdjgxjXWVshePHWBPq/tpqXft4vO2zbM5MTjH4LDEbYuwQO/6l3IpsghYwD+GZrAvFpeTT0cpEp16J68tLOTe0c+JS0xjiAEF835k7swpJ/9aZbuD+FJWY+XhdH31l/8eBXO1h3+CzFpTJo1O406gK97teOf3lEG9dWz0khY+cy8oqY/fshAB4c3ELGxojqWfOq1lwd3F5bhEs4jO4RASz5V28+vasnPSL8KTErft6dxG0fb6bH//7kie928dfBFNn6wJ4M/i/4hmlbiETP0juN7gzKwYe1Z2Vl4evrS2ZmJj4+PnrHsbr/Lt3DV5vjaR3szc8P9cNZplyLqjp7BOb3AnMJ3L4Mmg/WO5GoRftOZfLl5nh+25tMam6R5XYvVycGtg7k2rbBDGodhK+7s44pxRUdWglfTwCDCaZGQ2hHvRNZXWW/v6WQsWO7EjIYN389SsE3U6+iV7MGekcS9kYp+HwcHIvW1qmY9J3eiUQdKTUrtsSlsXJvEiv3JZOSVWi5z8lo4KpmDRjRIYTh7UJo4OWqY1JxSd/eAft/hEZd4Z4/wWjSO5FVSSFTxlELmeJSM9fPX8/ek1mM79KYNyd01juSsEd7lsD3d4PJFaZtku0I6imzWbH7ZCa/70vmj/0pHD6dY7nPZDTQp3kDrusQyrB2Ifh7ymQCm5GdDO/0hMJMGP4qXPUvvRNZlRQyZRy1kJnzRyxvrTqMr7szfzw6gCBvN70jCXuTexbmXwW5Z2Dw/8HAJ/ROJGzEsTM5/LYvhRV7kthz8tw6NCajgb4tGjKqQyjXtguWGZK2YOvH8Muj4OIF0zaDbxO9E1mNFDJlHLGQ2ZOYybj56yk1K96+pQtjOjXSO5KwN0rBN7fBwZ8hqK3Wx+4k3QfiQsfP5vLLniR+2Z3E/qQsy+1ORgN9WjRkcOtABrYKJLKhJwZZCbrumc2wcDgkbIbmQ2DSEjA6xlhJKWTKOFohk1dUwph31nPkdA7XdQzl3Vu76h1J2KOdX8CP08DoDPf+5ZADBYX1HTuTw4o9Sfy8O4mDydkV7gsLcGdgq0AGtAykT4uGeMkMyrpz+iB8MAhK8mHIs9D/Mb0TWYUUMmUcqZBRSvHot7tYuvMkgd6u/PbwAAKkv1pUVfIe+GgolBQ41D96om4dOZ3DqgMp/H34DFvi0iguPfdV4mwy0DnMj16RDejVLIBu4f54uEhhU6t2fA7LH9QWy5vyC4T30TtRjUkhU8aRCpkvN5/gmaV7MRkNfHVPL5mlJKouNxU+GgLpcdDiGrj1W4dphhb6yS0sYdOxVNbEnmFN7BlOpOZVuN/JaKBDE196RgbQKzKAjk38aCgzoaxLKVj6L9i9GLxD4V/rwLOh3qlqRAqZMo5SyGw7nsatH26mqNTM0yPacN/A5npHEvamOB8+G6v1pfs1halrwCNA71TCAZ1IzWXTsVQ2H0tjc1waJzPyL3hMI183OjTxpUNjXzo08aNDY19pYa6pwhz4cDCcjYWI/nDbD+Bkv3+mUsiUcYRC5uiZHG5YsIGMvGKGtQvmvdu6yaA6UTUlhfDN7XD4N3Dzhbv/gMDWeqcS9URiel5ZUZPKthPpxJ3N5WLfPI393OkU5kunJn50bOJHhya+MtamqlL2w8fXQFEOdJwI178Hdvp9IYVMGXsvZE5nFXDDextISMunU5gfi++9CncXx1r0SNSy4nxYchccWgFObtqshsj+eqcS9Vh2QTH7TmWx92QmuxMz2XMyk7izF+4ZZDBA80AvOjXxo3NTP7qE+dE6xFtWML+SI3/ClzeDKoUe98DIN+yymJFCpow9FzKnMvKZ9NFm4s7m0jTAgx8e6CP9yqJqclNh8a2QsElb9O7WxdD8ar1TCXGBrIJi9iZmsisxk10JGexOzOBU5oU7dbs5G2nfyJf2jX1pHeJNq2BvWod4S8vNP8V8DcvuBxR0mwIjZ9vdZrBSyJSx10Lm+Nlcbvt4M4np+TT2c+fre6+iaQMPvWMJe5KwFb6bAlmJWnfSxK8gop/eqYSotDPZhexOzGBXQgY7EzKIScggu6Dkoo9t5OtGRENP7dLAg4gG2nHTAA/cnOtpK/bOL7VlFlDQchjc8KH2b4GdkEKmjD0WMn/sT+HRb2PILighooEHX957FY393PWOJexFcT6seQ3Wv6U1LQc004qYoCi9kwlRI2az4tjZXHYlZHAwOYuDydnEpmRX2CfqnwwGCPXRipzGfu6E+LoR5ONGiI8bwT6uhPi40cDLFZPR/rpeKmX/cvjhXm25Bb9wuOEjCOupd6pKkUKmjD0VMpn5xcz5I5ZFG44D0LWpH+/d1o0gH9l+QFRCaTHs/hZWv6K1wgC0vwFGzQU32/7dF6Im0nOLOHY2h7izeZxIzSXubC4nUvM4fjaX7MKLt+Ccz2Q0EOjlSrCPK8E+bmUX7TjU153G/u408nPD1clOW3ZObtdaZzPiAQN0mwyD/gvewXonuyyHKmTeffddXn/9dZKTk+nUqRPz5s2jZ8/KVZT2UMjkFpbww45E3lp1mLM5RQDc2TeCp0dE4eIkg9rEFaTFwZ7vYPsiyDqp3ebTGEa8ClGjdY0mhJ6UUqTlFnG8rKhJyswnJauQ5KwCTmcVkJxVwJnsQsyV/BYM9HalsZ9W2DQpu2583rW3m3PtnlBN5GfAyqdg19faz05u0OU2bfxMcHubHAzsMIXMN998wx133MF7771Hr169mDt3Lt999x2HDh0iKCjois+31UImp7CETUdTWX3oNMtjTln+19As0JPnR7djQKtAnRMKm1RSBGnH4PR+OL4Ojq3Wfi7nGQR9HoSeU8FZuiOFuJJSs+JsTiEpWQUVi5xMrdBJyizgZHo++cWlV3wtX3fnCoVNqK8b3m7OeLs5lV2c8Sm7dncx4epkxNXJWLfLaRxfD38+B4lbz93m11Tbp6n5YAjpqP1s1L/1yWEKmV69etGjRw/eeecdAMxmM2FhYfz73//mqaeeuuLza6uQ2Xsyk4S0PMwKzEphLvtjNCuF2axdKwWFJaVk5heTkVdMZn4xablFHDmTQ3xaXoV1FCIaeDClTwS39gqXVpj6oiALjvyhdQmVFmlrvZQflxZDUTbkpZVdUiH3tNY0bP5HU7nBpA3i7XwrtLteNn8UwsqUUqTnFXMyPZ+TGXkkpueTmJ7PyYz8stvyycwvrvbru5i0gsbV2Yirk1bguDgZcXU+V+y4OpnK7j/3mPMfX34xGg0YDQaMBjAYzh0bDQYMZddGoOHZzTSNW0zwyVUY1T+ym1yhQXPwDgHPQPBoCC6e4OymteSUX4wmbUsEgxEad9XG41lRZb+/bXouVlFREdu3b+fpp5+23GY0Ghk6dCgbN2686HMKCwspLDw38CszU9uCPisr66KPr65PVu9nyfbEGr1GY383+jZvyJCoYHo3a4DRaKAgL4cLJxwKh5R2DL68s+rPc/aEBi2gSTdt9c6wXufGwOQVApce+CiEqB4nINzHQLiPJzT1vOD+nMISkjLyOZWZT1JGPicztG6rnMJicgpKyS4sIaewmNyCEnIKSyk5rz+roOxS927EnVFMDDnJYy2S4cRGSD0KhQWQtw/YV/mXGjYTut5u1XTl39tXbG9RNuzkyZMKUBs2bKhw+xNPPKF69ux50ec899xzCpCLXOQiF7nIRS4OcElISLhsrWDTLTLV8fTTT/Poo49afjabzaSlpdGgQQObWdY/KyuLsLAwEhISbGrcTm2qj+cM9fO86+M5Q/087/p4zlA/z1uPc1ZKkZ2dTaNGjS77OJsuZBo2bIjJZCIlJaXC7SkpKYSEhFz0Oa6urri6Vhwj4OfnV1sRa8THx6fe/CUoVx/PGernedfHc4b6ed718Zyhfp53XZ+zr6/vFR9j06NKXVxc6NatG6tWrbLcZjabWbVqFb1799YxmRBCCCFsgU23yAA8+uijTJ48me7du9OzZ0/mzp1Lbm4ud95ZjUGSQgghhHAoNl/ITJgwgTNnzvDss8+SnJxM586dWblyJcHBtr0i4eW4urry3HPPXdAF5sjq4zlD/Tzv+njOUD/Puz6eM9TP87blc7b5dWSEEEIIIS7FpsfICCGEEEJcjhQyQgghhLBbUsgIIYQQwm5JISOEEEIIuyWFjBW8++67RERE4ObmRq9evdiyZcslH/vDDz/QvXt3/Pz88PT0pHPnznz++ecVHjNlyhQMBkOFy/Dhw2v7NKqsKud9vsWLF2MwGBg3blyF25VSPPvss4SGhuLu7s7QoUM5fPhwLSSvPmufsyN+1osWLbrgnNzc3Co8xtE+68qcsyN+1gAZGRlMmzaN0NBQXF1dadWqFStWrKjRa9Y1a5/z888/f8Fn3aZNm9o+jSqrynkPGjTognMyGAxcd911lsfo9vfaClsi1WuLFy9WLi4u6pNPPlH79u1T9957r/Lz81MpKSkXffzq1avVDz/8oPbv36+OHDmi5s6dq0wmk1q5cqXlMZMnT1bDhw9XSUlJlktaWlpdnVKlVPW8y8XFxanGjRur/v37q7Fjx1a4b9asWcrX11ctW7ZM7dq1S40ZM0ZFRkaq/Pz8WjyTyquNc3bEz3rhwoXKx8enwjklJydXeIyjfdaVOWdH/KwLCwtV9+7d1ciRI9W6detUXFycio6OVjExMdV+zbpWG+f83HPPqXbt2lX4rM+cOVNXp1QpVT3v1NTUCuezd+9eZTKZ1MKFCy2P0evvtRQyNdSzZ081bdo0y8+lpaWqUaNGaubMmZV+jS5duqj/+7//s/w8efLkC77wbE11zrukpET16dNHffTRRxeco9lsViEhIer111+33JaRkaFcXV3V119/XSvnUFXWPmelHPOzXrhwofL19b3k6zniZ32lc1bKMT/rBQsWqGbNmqmioiKrvWZdq41zfu6551SnTp2sHdWqavq5zJkzR3l7e6ucnByllL5/r6VrqQaKiorYvn07Q4cOtdxmNBoZOnQoGzduvOLzlVKsWrWKQ4cOMWDAgAr3RUdHExQUROvWrbn//vtJTU21ev7qqu55v/jiiwQFBXH33XdfcF9cXBzJyckVXtPX15devXpV6s+yttXGOZdzxM86JyeH8PBwwsLCGDt2LPv27bPc56if9eXOuZyjfdbLly+nd+/eTJs2jeDgYNq3b88rr7xCaWlptV+zLtXGOZc7fPgwjRo1olmzZkyaNIn4+PhaPZeqsMbn8vHHHzNx4kQ8PT0Bff9eSyFTA2fPnqW0tPSCVYaDg4NJTk6+5PMyMzPx8vLCxcWF6667jnnz5nHNNddY7h8+fDifffYZq1at4tVXX2XNmjWMGDHigr8oeqnOea9bt46PP/6YDz/88KL3lz+vqn+WdaU2zhkc87Nu3bo1n3zyCT/++CNffPEFZrOZPn36kJiYCDjmZ32lcwbH/KyPHTvGkiVLKC0tZcWKFcyYMYPZs2fz8ssvV/s161JtnDNAr169WLRoEStXrmTBggXExcXRv39/srOza/V8Kqumn8uWLVvYu3cv99xzj+U2Pf9e2/wWBY7I29ubmJgYcnJyWLVqFY8++ijNmjVj0KBBAEycONHy2A4dOtCxY0eaN29OdHQ0Q4YM0Sl19WVnZ3P77bfz4Ycf0rBhQ73j1InKnrOjfdYAvXv3rrCpa58+fYiKiuL999/npZde0jFZ7anMOTviZ202mwkKCuKDDz7AZDLRrVs3Tp48yeuvv85zzz2nd7xaUZlzHjFihOXxHTt2pFevXoSHh/Ptt99etnXWXnz88cd06NCBnj176h0FkEKmRho2bIjJZCIlJaXC7SkpKYSEhFzyeUajkRYtWgDQuXNnDhw4wMyZMy2FzD81a9aMhg0bcuTIEZv4B6+q53306FGOHz/O6NGjLbeZzWYAnJycOHTokOV5KSkphIaGVnjNzp0718JZVE1tnHPz5s0veJ69f9YX4+zsTJcuXThy5AiAw33WF/PPc74YR/isQ0NDcXZ2xmQyWW6LiooiOTmZoqIiq/xZ1qbaOGcXF5cLnuPn50erVq0u+/tQl2ryueTm5rJ48WJefPHFCrfr+fdaupZqwMXFhW7durFq1SrLbWazmVWrVlX439mVmM1mCgsLL3l/YmIiqampFX459FTV827Tpg179uwhJibGchkzZgyDBw8mJiaGsLAwIiMjCQkJqfCaWVlZbN68uUp/lrWlNs75Yuz9s76Y0tJS9uzZYzknR/usL+af53wxjvBZ9+3blyNHjliKdIDY2FhCQ0NxcXGx2r+RtaU2zvlicnJyOHr0qF1/1uW+++47CgsLue222yrcruvf61odSlwPLF68WLm6uqpFixap/fv3q6lTpyo/Pz/L1Mvbb79dPfXUU5bHv/LKK+r3339XR48eVfv371dvvPGGcnJyUh9++KFSSqns7Gz1+OOPq40bN6q4uDj1559/qq5du6qWLVuqgoICXc7xYqp63v90sRkcs2bNUn5+furHH39Uu3fvVmPHjrW5KbnWPGdH/axfeOEF9dtvv6mjR4+q7du3q4kTJyo3Nze1b98+y2Mc7bO+0jk76mcdHx+vvL291YMPPqgOHTqkfv75ZxUUFKRefvnlSr+m3mrjnB977DEVHR2t4uLi1Pr169XQoUNVw4YN1enTp+v8/C6luv+e9evXT02YMOGir6nX32spZKxg3rx5qmnTpsrFxUX17NlTbdq0yXLfwIED1eTJky0/P/PMM6pFixbKzc1N+fv7q969e6vFixdb7s/Ly1PXXnutCgwMVM7Ozio8PFzde++9NvOX/nxVOe9/ulghYzab1YwZM1RwcLBydXVVQ4YMUYcOHaql9NVjzXN21M/64Ycftjw2ODhYjRw5Uu3YsaPC6znaZ32lc3bUz1oppTZs2KB69eqlXF1dVbNmzdT//vc/VVJSUunXtAXWPucJEyao0NBQ5eLioho3bqwmTJigjhw5UlenU2lVPe+DBw8qQP3+++8XfT29/l4blFKqdtt8hBBCCCFqh4yREUIIIYTdkkJGCCGEEHZLChkhhBBC2C0pZIQQQghht6SQEUIIIYTdkkJGCCGEEHZLChkhhBBC2C0pZIQQQghht6SQEcKBRUREMHfuXMvPBoOBZcuW1XmO559/3iY2hKwt0dHRGAwGMjIy9I4iRL0jhYwQ9UhSUhIjRoyo1GMdvfgQQjgGKWSEsHFFRUVWe62QkBBcXV2t9nrCPlnzd0oIvUkhI0QdGjRoEA8++CAPPvggvr6+NGzYkBkzZnD+lmcRERG89NJL3HHHHfj4+DB16lQA1q1bR//+/XF3dycsLIyHHnqI3Nxcy/NOnz7N6NGjcXd3JzIyki+//PKC9/9n11JiYiK33HILAQEBeHp60r17dzZv3syiRYt44YUX2LVrFwaDAYPBwKJFiwDIyMjgnnvuITAwEB8fH66++mp27dpV4X1mzZpFcHAw3t7e3H333RQUFFz2z6W0tJS7776byMhI3N3dad26NW+99VaFx0yZMoVx48bxxhtvEBoaSoMGDZg2bRrFxcWWx6Snp3PHHXfg7++Ph4cHI0aM4PDhw5b7Fy1ahJ+fHz///DOtW7fGw8ODG2+8kby8PD799FMiIiLw9/fnoYceorS01PK8zz//nO7du+Pt7U1ISAi33norp0+fvui55Obm4uPjw5IlSyrcvmzZMjw9PcnOzr7o85YsWUKHDh1wd3enQYMGDB06tMLn+8knn9CuXTtcXV0JDQ3lwQcftNwXHx/P2LFj8fLywsfHh5tvvpmUlBTL/eWtax999BGRkZG4ubkBlfsshbB5tb4tpRDCYuDAgcrLy0tNnz5dHTx4UH3xxRfKw8NDffDBB5bHhIeHKx8fH/XGG2+oI0eOWC6enp5qzpw5KjY2Vq1fv1516dJFTZkyxfK8ESNGqE6dOqmNGzeqbdu2qT59+ih3d3c1Z84cy2MAtXTpUqWUUtnZ2apZs2aqf//+au3aterw4cPqm2++URs2bFB5eXnqscceU+3atVNJSUkqKSlJ5eXlKaWUGjp0qBo9erTaunWrio2NVY899phq0KCBSk1NVUop9c033yhXV1f10UcfqYMHD6pnnnlGeXt7q06dOl3yz6WoqEg9++yzauvWrerYsWOWP5dvvvnG8pjJkycrHx8f9a9//UsdOHBA/fTTTxf82Y0ZM0ZFRUWpv//+W8XExKhhw4apFi1aqKKiIqWUUgsXLlTOzs7qmmuuUTt27FBr1qxRDRo0UNdee626+eab1b59+9RPP/2kXFxcKuxK//HHH6sVK1aoo0ePqo0bN6revXurESNGWO5fvXq1AlR6erpSSql7771XjRw5ssI5jhkzRt1xxx0XPf9Tp04pJycn9eabb6q4uDi1e/du9e6776rs7GyllFLz589Xbm5uau7cuerQoUNqy5Ytls+1tLRUde7cWfXr109t27ZNbdq0SXXr1k0NHDjQ8vrPPfec8vT0VMOHD1c7duxQu3btqtRnKYQ9kEJGiDo0cOBAFRUVpcxms+W2J598UkVFRVl+Dg8PV+PGjavwvLvvvltNnTq1wm1r165VRqNR5efnq0OHDilAbdmyxXL/gQMHFHDJQub9999X3t7el/zSeu655y4oPtauXat8fHxUQUFBhdubN2+u3n//faWUUr1791YPPPBAhft79ep12ULmYqZNm6ZuuOEGy8+TJ09W4eHhqqSkxHLbTTfdpCZMmKCUUio2NlYBav369Zb7z549q9zd3dW3336rlNIKGUAdOXLE8pj77rtPeXh4WIoGpZQaNmyYuu+++y6ZbevWrQqwPOefhczmzZuVyWRSp06dUkoplZKSopycnFR0dPRFX2/79u0KUMePH7/o/Y0aNVLPPPPMRe/7/ffflclkUvHx8Zbb9u3bV+H34bnnnlPOzs7q9OnTlsdU5rMUwh5I15IQdeyqq67CYDBYfu7duzeHDx+u0JXRvXv3Cs/ZtWsXixYtwsvLy3IZNmwYZrOZuLg4Dhw4gJOTE926dbM8p02bNvj5+V0yR0xMDF26dCEgIKDS2Xft2kVOTg4NGjSokCUuLo6jR48CcODAAXr16lXheb17977ia7/77rt069aNwMBAvLy8+OCDD4iPj6/wmHbt2mEymSw/h4aGWrp4yv8Mzn/vBg0a0Lp1aw4cOGC5zcPDg+bNm1t+Dg4OJiIiAi8vrwq3nd91tH37dkaPHk3Tpk3x9vZm4MCBABfkK9ezZ0/atWvHp59+CsAXX3xBeHg4AwYMuOjjO3XqxJAhQ+jQoQM33XQTH374Ienp6YDWZXjq1CmGDBly0eceOHCAsLAwwsLCLLe1bdsWPz+/CucdHh5OYGCg5efKfJZC2AMnvQMIIS7k6elZ4eecnBzuu+8+HnrooQse27RpU2JjY6v8Hu7u7lV+Tk5ODqGhoURHR19w3+WKpitZvHgxjz/+OLNnz6Z37954e3vz+uuvs3nz5gqPc3Z2rvCzwWDAbDZX6b0u9hqXe93c3FyGDRvGsGHD+PLLLwkMDCQ+Pp5hw4ZddtDsPffcw7vvvstTTz3FwoULufPOOysUsOczmUz88ccfbNiwgd9//5158+bxzDPPsHnzZho2bFil87uUi/1O1cZnKURdkxYZIerYP7+cN23aRMuWLSu0NPxT165d2b9/Py1atLjg4uLiQps2bSgpKWH79u2W5xw6dOiy65p07NiRmJgY0tLSLnq/i4tLhVai8hzJyck4OTldkKP8CzcqKuqi53g569evp0+fPjzwwAN06dKFFi1aVLlVICoqipKSkgrvnZqayqFDh2jbtm2VXut8Bw8eJDU1lVmzZtG/f3/atGlzyYG+57vttts4ceIEb7/9Nvv372fy5MmXfbzBYKBv37688MIL7Ny5ExcXF5YuXYq3tzcRERGsWrXqos+LiooiISGBhIQEy2379+8nIyPjsuddmc9SCHsghYwQdSw+Pp5HH32UQ4cO8fXXXzNv3jymT59+2ec8+eSTbNiwgQcffJCYmBgOHz7Mjz/+aJm50rp1a4YPH859993H5s2b2b59O/fcc89lW11uueUWQkJCGDduHOvXr+fYsWN8//33bNy4EdBmT8XFxRETE8PZs2cpLCxk6NCh9O7dm3HjxvH7779z/PhxNmzYwDPPPMO2bdsAmD59Op988gkLFy4kNjaW5557jn379l32/Fq2bMm2bdv47bffiI2NZcaMGWzdurUqf6y0bNmSsWPHcu+997Ju3Tp27drFbbfdRuPGjRk7dmyVXut8TZs2xcXFhXnz5nHs2DGWL1/OSy+9dMXn+fv7M378eJ544gmuvfZamjRpcsnHbt68mVdeeYVt27YRHx/PDz/8wJkzZ4iKigK0WUezZ8/m7bff5vDhw+zYsYN58+YBMHToUDp06MCkSZPYsWMHW7Zs4Y477mDgwIEXdFGerzKfpRD2QAoZIerYHXfcQX5+Pj179mTatGlMnz7dMsX6Ujp27MiaNWuIjY2lf//+dOnShWeffZZGjRpZHrNw4UIaNWrEwIEDGT9+PFOnTiUoKOiSr+ni4sLvv/9OUFAQI0eOpEOHDsyaNcvSMnTDDTcwfPhwBg8eTGBgIF9//TUGg4EVK1YwYMAA7rzzTlq1asXEiRM5ceIEwcHBAEyYMIEZM2bwn//8h27dunHixAnuv//+y57ffffdx/jx45kwYQK9evUiNTWVBx54oLJ/pBX+DLp168aoUaPo3bs3SilWrFhxQddRVQQGBrJo0SK+++472rZty6xZs3jjjTcq9dy7776boqIi7rrrrss+zsfHh7///puRI0fSqlUr/u///o/Zs2dbFi+cPHkyc+fOZf78+bRr145Ro0ZZppUbDAZ+/PFH/P39GTBgAEOHDqVZs2Z88803l33PynyWQtgDg1LnLWAhhKhVgwYNonPnzhW2DRCO6/PPP+eRRx7h1KlTuLi46B1HCIckg32FEMLK8vLySEpKYtasWdx3331SxAhRi6RrSQghrOy1116jTZs2hISE8PTTT+sdRwiHJl1LQgghhLBb0iIjhBBCCLslhYwQQggh7JYUMkIIIYSwW1LICCGEEMJuSSEjhBBCCLslhYwQQggh7JYUMkIIIYSwW1LICCGEEMJu/T+FnNCGBBJoDwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sns.kdeplot(predictions[filtered_dataset[\"Cover_Type\"] == \"Spruce/Fir\"], label=\"Spruce/Fir\")\n", + "sns.kdeplot(predictions[filtered_dataset[\"Cover_Type\"] == \"Cottonwood/Willow\"], label=\"Cottonwood/Willow\")\n", + "plt.xlabel(\"predicted anomaly score\")\n", + "plt.ylabel(\"distribution\")\n", + "plt.legend()\n", + "None" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The AUC is a metric used to evaluate classification models. It can also be used to quantify the discriminative power of any signal in separating two distinct classes. In the context of anomaly detection, we can use the AUC to quantify how much our anomaly detection model is able to isolate the minority class.\n", + "\n", + "The cover type information are not used to train the model and the dataset is considered static (i.e., the type of coverage does not change overtime). Therefore, we do not need to split the dataset between training and testing, and use all the data both for training the model and evaluate it with the AUC." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.9427246186652949" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metrics.roc_auc_score(filtered_dataset[\"Cover_Type\"] == \"Cottonwood/Willow\", predictions)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This high AUC confirms that the model is well able to separate the two cover types.\n", + "\n", + "We can also analyse the model to understand it: For instance, we see on the partial dependency plot of the elevation that the \"normal\" coverage is around 2900 and 3300 meters of altitude. Other similar conclusions can be taken by looking at other attributes." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
Partial Dependence PlotConditional Expectation Plot
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.analyze(filtered_dataset, sampling=0.001) # Use larger sampling for better results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also interpret individual model predictions. For example, let's select the first Cottonwood/Willow example and generate a prediction:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ElevationAspectSlopeHorizontal_Distance_To_HydrologyVertical_Distance_To_HydrologyHorizontal_Distance_To_RoadwaysHillshade_9amHillshade_NoonHillshade_3pmHorizontal_Distance_To_Fire_PointsCover_Type
198820003187304108201234172268Cottonwood/Willow
\n", + "
" + ], + "text/plain": [ + " Elevation Aspect Slope Horizontal_Distance_To_Hydrology \\\n", + "1988 2000 318 7 30 \n", + "\n", + " Vertical_Distance_To_Hydrology Horizontal_Distance_To_Roadways \\\n", + "1988 4 108 \n", + "\n", + " Hillshade_9am Hillshade_Noon Hillshade_3pm \\\n", + "1988 201 234 172 \n", + "\n", + " Horizontal_Distance_To_Fire_Points Cover_Type \n", + "1988 268 Cottonwood/Willow " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "first_willow_example = filtered_dataset[filtered_dataset[\"Cover_Type\"] == \"Cottonwood/Willow\"][:1]\n", + "first_willow_example" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.5474113], dtype=float32)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.predict(first_willow_example)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, let's see how the model prediction would change with the feature values of this example:\n", + "\n", + "We see than the example elevation of 2000 is uncommon and explain some of the high prediction value. On the other hand, the example \"aspect\" and \"slope\" are relatively normal." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
Feature Variation
\n", + "
\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "
\n", + "\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.analyze_prediction(first_willow_example)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "List all the decision forest algorithms, our isolation forest model define an implicit distance between examples. This distance can be use to cluster the examples or interpretable mapping.\n", + "\n", + "Let's compute the distance between each pair of examples. To make the code run fast, we only select the first 10000 examples." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0. , 0.86 , 0.6766667, 0.85 ],\n", + " [0.86 , 0. , 0.9066667, 0.31 ],\n", + " [0.6766667, 0.9066667, 0. , 0.8833333],\n", + " [0.85 , 0.31 , 0.8833333, 0. ]], dtype=float32)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "distances = model.distance(filtered_dataset[:10000]) # Use more examples for better results\n", + "distances[:4, :4]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can then use UMAP (or any other manifold learning algorithm such as T-SNE) to project the examples in a 2D plot.\n", + "\n", + "Note that the cover types are well separated despite the model having never seen them." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/gbm/my_venv/lib/python3.11/site-packages/umap/umap_.py:1858: UserWarning:\n", + "\n", + "using precomputed metric; inverse_transform will be unavailable\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAGdCAYAAAAvwBgXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOyddXgUd9eG71nJbtw9wS24u7sU10JLKVBq1P2r21v3FuqFtlhxh1IKFC3uwQMkISHuyWbt++NkkyzZ0JaWUmDu68oFmZ2dnZ1N8jtzznOeo9jtdjsqKioqKioqKtcAzbU+ARUVFRUVFZWbFzUQUVFRUVFRUblmqIGIioqKioqKyjVDDURUVFRUVFRUrhlqIKKioqKioqJyzVADERUVFRUVFZVrhhqIqKioqKioqFwz1EBERUVFRUVF5Zqhu9YncDlsNhsXLlzA29sbRVGu9emoqKioqKio/Ansdju5ublERESg0Vw+5/GfDkQuXLhAdHT0tT4NFRUVFRUVlSsgPj6eqKioy+7znw5EvL29AXkjPj4+1/hsVFRUVFRUVP4MOTk5REdHl67jl+M/HYg4yjE+Pj5qIKKioqKionKd8WdkFapYVUVFRUVFReWaoQYiKioqKioqKtcMNRBRUVFRUVFRuWb8pzUiKioqKjcKdrsdi8WC1Wq91qeiovK30Wq16HS6f8Ra44oDkd9++4133nmHPXv2kJSUxOLFixkyZEjp4xMmTGDmzJlOz+nTpw9r1qy54pNVUVFRuR4pLi4mKSmJgoKCa30qKir/GB4eHoSHh+Pm5va3jnPFgUh+fj5NmjRh4sSJDBs2zOU+ffv25bvvviv93mAwXOnLqaioqFyX2Gw24uLi0Gq1RERE4Obmpho0qlzX2O12iouLSU1NJS4ujtq1a/+hadnluOJApF+/fvTr1++y+xgMBsLCwq70JVRUVFSue4qLi7HZbERHR+Ph4XGtT0dF5R/B3d0dvV7PuXPnKC4uxmg0XvGxrqpYdePGjYSEhFC3bl3uvfde0tPTL7u/yWQiJyfH6UtFRUXlRuDv3DGqqPwX+ad+pq/ab0bfvn35/vvvWb9+PW+99RabNm2iX79+lxVqvfHGG/j6+pZ+qfbuKioqKioqNzZXLRAZM2YMgwYNolGjRgwZMoQVK1awa9cuNm7cWOlznnnmGbKzs0u/4uPjr9bpqfwXMRdDbgrkJkNWAmQnQVHutT4rFRWVm4Bq1arx4YcfXuvTuCn513KFNWrUICgoiFOnTlW6j8FgKLVzV23dbyLyMyD9NKQehW0fweqn4NgKKEiBzDjIPH+tz1BF5aYkNTWVe++9lypVqpRq/vr06cPWrVuv9an9aWbOnEnHjh0B6Nq1K4qiVPiyWCzs2rWLKVOmXOOzvTn513xEEhISSE9PJzw8/N96SZXrgYyzcGIN2Mzw83Nl248uAe9wGDMbzv4GSlfwu/wERxWVG5nsgmLS8orJKTLj464nyNMNX4+/1zb5RwwfPpzi4mJmzpxJjRo1uHjxIuvXr/9Dvd/lMJvN6PX6f/AsL8/SpUsZNGhQ6fd33XUXr7zyitM+Op2O4ODgyx7n3z7vm4krzojk5eWxf/9+9u/fD0BcXBz79+/n/Pnz5OXl8cQTT7Bjxw7Onj3L+vXrGTx4MLVq1aJPnz7/1LmrXM8U5UimY9tHEFgD1r1QcZ/cJNj0JgTVAVP2v3+OKir/ES5kFTJ1zj56vL+JodO20eO9TTwwZx8Xsgqv2mtmZWWxefNm3nrrLbp160bVqlVp3bo1zzzzTOnCrigK06dPp1+/fri7u1OjRg0WLFhQeoyzZ8+iKArz5s2jS5cuGI1GZs2axUsvvUTTpk2dXu/DDz+kWrVqTtu+/fZbGjRogMFgIDw8nKlTpzqd3+TJkwkODsbHx4fu3btz4MABp+cXFRXx888/OwUiHh4ehIWFOX1BxdKM470NGjQIT09PXn/99b9zOVUuwxUHIrt376ZZs2Y0a9YMgEcffZRmzZrxwgsvoNVqOXjwIIMGDaJOnTpMmjSJFi1asHnzZtVLRAUyz8LRpRJcnN8OKbFgt7ne9+Q60Bng3DbISQFL8b96qioq15rsgmKeWniQzSfTnLb/djKNpxceJLvg6vxOeHl54eXlxZIlSzCZTJXu9/zzzzN8+HAOHDjAuHHjGDNmDLGxsU77PP300zz00EPExsb+6ZvR6dOnc//99zNlyhQOHTrEsmXLqFWrVunjI0eOJCUlhdWrV7Nnzx6aN29Ojx49yMjIKN1n/fr1REZGUq9evb/47oWXXnqJoUOHcujQISZOnHhFx1D5Y664NNO1a1fsdnulj69du/ZKD61yI5ObBHmpEFofigtA0UBxfuX7223ypXeHwlQozgGfcHDz/PfOWUXlGpKWV1whCHHw28k00vKKr0qJRqfTMWPGDO666y4+//xzmjdvTpcuXRgzZgyNGzcu3W/kyJFMnjwZgFdffZV169bxySefMG3atNJ9Hn744UqNLyvjtdde47HHHuOhhx4q3daqVSsAtmzZws6dO0lJSSm9uX333XdZsmQJCxYsKNV6XFqWAZg2bRpff/116fd333037733nstzGDt2LHfeeedfOm+Vv47a2K7y75FxDooLwegHWiMYfaHp7VCja+XPCW0AliLwrwafd4RpbWDZgyJuvUwgrKJyo5BTZL7s47l/8PjfYfjw4Vy4cIFly5bRt29fNm7cSPPmzZkxY0bpPu3atXN6Trt27SpkRFq2bPmXXjclJYULFy7Qo0cPl48fOHCAvLw8AgMDSzM3Xl5exMXFcfr0aUDcP5cvX14hEBk3blyprGD//v0888wzlZ7HXz1vlStDHXqncvUpyIC8i1CUBUcWQ8wgOLIEjq8ErRs0HgN3rIB5t8k+DhQN9H4dsuJF0OrIjhxeIALWO1ZCYSZ4BoNPJOjVsp/KjYeP8fICSe8/ePzvYjQa6dWrF7169eL5559n8uTJvPjii0yYMOFPH8PT0zmDqdFoKmTUzeaygMrd3f2yx8vLyyM8PNylHYSfnx8AO3fuxGKx0L59e6fHfX19nUo8f+W8Va4OakZE5epSlAumPEjYBTMGQO0+MH8C7PoKci6IXmTTm9KyO+4nyXy4eUKNbjDpZ/AIgvjf4fBC5+PmpUiL76Z3YN5YOLNBAh4VlRuMIC83OtcOcvlY59pBBHld3c6ZS6lfvz75+WXl1B07djg9vmPHDmJiYi57jODgYJKTk52CEUfjA4C3tzfVqlVj/fr1Lp/fvHlzkpOT0el01KpVy+krKEiu1dKlSxkwYABarfavvkWVfxk1I6JydSnOBasJVj0uQcjxVZCfWnG/lCOQEQdj54OigKKV7MeCOyD5sOtjn1wLre+G9FOADdJOSGbEOwy0apudyo2Br4cbbw5vzNMLD/JbOa1I59pBvDW88VVr4U1PT2fkyJFMnDiRxo0b4+3tze7du3n77bcZPHhw6X7z58+nZcuWdOzYkVmzZrFz506++eabyx67a9eupKam8vbbbzNixAjWrFnD6tWrnbyjXnrpJe655x5CQkLo168fubm5bN26lQceeICePXvSrl07hgwZwttvv02dOnW4cOECK1euZOjQobRs2ZJly5ZVaNNV+W+iBiIqVw9LMZjyxZTMYoJqHeD3L0DRYKnZi7S6Y7Fr3fBN+BWPQ7Pg0E/SRbNnhjy/0+NSuqkMoy+ENwWbBQy+Uv7Z/hlUaQdNxoBHwL/xLlVUrjoRfu58cmsz0vKKyS0y423UE+R1dX1EvLy8aNOmDR988AGnT5/GbDYTHR3NXXfdxf/93/+V7vfyyy8zd+5c7rvvPsLDw5kzZw7169e/7LFjYmKYNm0a//vf/3j11VcZPnw4jz/+OF9++WXpPnfccQdFRUV88MEHPP744wQFBTFixAhAWmtXrVrFs88+y5133klqaiphYWF07tyZ0NBQTp8+zalTp1S7iOsExX651pdrTE5ODr6+vmRnZ6suq9cj2QmQfEh0HEvuhZ4vw9ElJHV+izkntcw+kElhsZWedfx4qI0vVU/9gNaUBftnyfNDGkDHR2DRZNfHv3uLZE+OLIascxKURLWUoEfRQGAtwA7ufqBXp56qXBuKioqIi4ujevXqf2tC6X8RRVFYvHgxQ4YMudan4sT777/PL7/8wqpVq671qdzQXO5n+6+s32pGROXqkJ8GF/bBLy+K4FTrBidWk9z7c+5cnMyxi2U15qWH0/nlRBbL736IGvN7lR2jyWhp8607QISt5en/DmSfh59uB1vJIMVD88EzCIZ/J6WZGf3lPOrdAp0fB//qoE5AVVG54YmKirpsN4zKfws1EFG5OmQngMEH2j8IOiNMWAlHFnEw2+gUhDjIL7by6eYEXm94K+4p+6HNPRC7HHIvQs8Xof0DIk5VNFC3vwQcX3QqC0JKD5QGG16Hfm9Bp8fAK0TM077qBpPXQ1Dtf+f9q6ioXDNGjRp1rU9B5S+gBiIq/zzFBVCUDQYvmRdjEsGqrXZ/Fm2vfEbFL8fTeKLrnbjnHRGtR8JOaD0Fkg6I2DWsMditkvno/y6YK7G3jt8B2fGw9H7QaOUYHR6CrR9B12ekK8fd7+q8dxWVm4j/cGVf5TpCzVOr/PNYisHoA0kHATtYCgEFxc0DX0PlrXQeeh2ahN9FfHp+GwyZJsPwlt4vgc3ZzWL1DmDKufw5ODIlNivsmA7uAZB6XFqGC7Nkzk3RHxxDRUVFReWqo2ZEVP55ijKgMBv2zhSdSAlKrZ7c2nE68/Ykunza7S2CCKpeE3ISYdjX8m9BBlgvmaWRdxH8oit/fZ8IaRsuz+5voek4SD8h2ZKwRhKIFGaCOR/QgNEffEKv8E2rqKioqFwJakZE5Z+nMBPWv+wUhABw6heqpm9hcoeqFZ7SKNKb4c0j0f40FrDCqsekI8bVkDu7DU6shWa3u379Lk/Brm+dt2XHi/D1zCZIPQbz7xB/k9PrYflDsPRe2P8DpBxTB+upqKio/IuoGRGVfx5rMVzY6/Ih/+V3cv/9sQxqGMS8PQnkFisMraOnviaB0JV3QP3B8MtL0OJO8I2ufJ7M9k8laxLRTLQfOYkQ3gTa3Q+nf634+qENZHCeZzAk7IZ2U6WlOP73sn0S98KBOTD2Jwis+c9cCxUVFRWVy6IGIir/PMUFl33YvzAe/4t7aWzehF3rhrJ5j3TZgHTLbPifdLzkpUBeMtTuBSfXOR/EEaCc2ybGZ75R4B0C8+8Uh9VL6fwErHoCuj8Hx1fLxN/yQYiD9FMihm1zjypoVVFRUfkXUEszKv88XmGVP6ZowOANO6bB8dUosUvLgpDyaA2QdRY2vQWNR0PHR8EjUB4LbwKjf5RW3HZTJfA4shgKc2Do5xBSbs6FVygM+kQm/97yIdhsMOA98Ksq82xccXihHDPjDGQnqqUaFRUVlauImhFR+edx94PqXSBuU8XHGgyTzpWs8xUfq9YJLh6R5+rdxSnVaoaFk6F6Z3FmNfqWlH72g6ITG/eOD0FeKhycB3X6wqgfsBekybwagzdKdqIEPxv/J5mVKu0gqI4YplVpCxvfuORE7GKIdm47hNSHPd9B09vEu0SrB5065VdFRUXln0LNiKj88/hEwMCPJChQFNmmaKDhCLFsd7TglsfoK2WTpEPQ+1XY94MEAb5R8njcb7BsqjipLpwkx82ME08Qz2ARovZ6Gaq2k0xJdFsy/RuTZ3fHbi6A5Q+Kn8nYn8QGPj9FhKm1e0OrSyzkG4+RGTlHFkmHTv2hgA1+ew/m3QZ7ZsqAvqKsq3kVVVRUrmOqVavGhx9+eNWPqygKS5YsAeDs2bMoiuI0yfh6QA1EVP55FAUCqsOA9+GebeJoet8O6P48fNcP7BYYOQNiBkpGov0DMOJbyE2GPv+TY1TvDPkZMOp7KcU40LpBm7uhZnfwryqTdl2egkKAlxGdwShZE41WJvXOGyfi1vjfxRwt+RA0v0MmA4OIVKt1ktk1PV6QTMreGbD5fajaFjo8DGuehhkDxPU19QTkpbk8BxWVf5TCTCkZJuyGtJPy/b9AcnIyDzzwADVq1MBgMBAdHc3AgQNZv379n3r+Sy+9RNOmTStsL7+A3kxUr16dFStWoNfrmTt3rtNjY8aMQVEUzp4967S9WrVqPP/88wDs2rWLKVOm/Fun+6+glmZUrh6+kfLlICseGg6TQMDoC/UGSMYj+RDs/FoCFqyw5hk4t1VmxPR+DTo8CgHVxEnV4C1mZnabOK16Brl+basF8lNxtxbB+R3STbPmKRmI12ikdOfs/BL2/SjdOR0egg4PigFa2nHskS1Rdn8Du74uO+a+H6SsM/kXMBeJ8ZrdBqZsMBeAZwC4eV3NK6pys5KdCEunwplfy7bV7CH6p/K/Y/8wZ8+epUOHDvj5+fHOO+/QqFEjzGYza9eu5f777+fYsWNX7bVvRA4ePEhmZiZ9+vShZcuWbNy4kTFjxpQ+vnHjRqKjo9m4cSMTJkwAIC4ujnPnztG9e3cAgoODr8WpX1XUjIjKv4feSzIKVdqLU+r+2eJ6Gr8ThnwGZzaAzh36vC4ByrEV8GU3+UNrs0qWoiAdvCMgqF7lpmbWYtk38yzs+VaOZfSVjEtEM6jRRUo8cb/JXWXyQSn3nPwF8tMpDG4kOpbyQYiD89vh1HrIOC06lwUTYe5YEdYWZIgYVkXln6Qws2IQAuKBs+yBq5oZue+++1AUhZ07dzJ8+HDq1KlDgwYNePTRR9mxYwcA58+fZ/DgwXh5eeHj48OoUaO4ePEiADNmzODll1/mwIEDKIqCoijMmDGDatWqATB06FAURSn9HmD69OnUrFkTNzc36tatyw8//OB0Toqi8PXXXzN06FA8PDyoXbs2y5YtK328ZcuWvPvuu6XfDxkyBL1eT15eHgAJCQkoisKpU6cAyMzMZPz48fj7++Ph4UG/fv04efKk02suXLiQBg0aYDAYqFatGu+9957T4ykpKQwcOBB3d3eqV6/OrFmzXF7PpUuX0rdvX/R6Pd26dWPjxo2lj8XGxlJUVMS9997rtH3jxo0YDAbatWsH/PWSz6ZNm2jdujUGg4Hw8HCefvppLBYLACtWrMDPzw+rVZyo9+/fj6IoPP3006XPnzx5Mrfddtuffr0rQQ1EVP49PP2l7bb3qzBuPvR4Uco3w7+CEz9LBsQnXIKFSb/A3VvgtgXgVaIBaTJGgoiAauBVSSYEJIjY9gl811eyJ2GNygKEVpPg19dd+5Ns+wj8o7F5hKIcmlf58Xd/K5qXpfdJKakwA34cJoFIVvzfukQqKhXIT60YhDg4vV4evwpkZGSwZs0a7r//fjw9PSs87ufnh81mY/DgwWRkZLBp0ybWrVvHmTNnGD16NACjR4/mscceo0GDBiQlJZGUlMTo0aPZtWsXAN999x1JSUml3y9evJiHHnqIxx57jMOHD3P33Xdz5513smHDBqfXfvnllxk1ahQHDx6kf//+jBs3joyMDAC6dOlSupDb7XY2b96Mn58fW7ZsAWRhjoyMpFatWgBMmDCB3bt3s2zZMrZv347dbqd///6YzWYA9uzZw6hRoxgzZgyHDh3ipZde4vnnn2fGjBml5zNhwgTi4+PZsGEDCxYsYNq0aaSkpFS4ZsuWLWPw4MEAdOvWjePHj5OUlATAhg0b6NixI927d3cKRDZs2EC7du0wGo1//sMrITExkf79+9OqVSsOHDjA9OnT+eabb3jttdcA6NSpE7m5uezbt6/02gQFBTm9/qZNm+jatetffu2/ghqIqPy7BFQHzxAJLty8pHVXo4cez0uA4cAnHMIbibDUv5p00fxZss6XdLqMlecuuVdeV+8uE4Fzk1w/z27Dmp1ErskqwtbKMOWIVqUgA357W8zXrGbY+qHoX4oLIPMcnN0q83Yy4tS5NipXzh/97Fyln61Tp05ht9upV69epfusX7+eQ4cOMXv2bFq0aEGbNm34/vvv2bRpE7t27cLd3R0vLy90Oh1hYWGEhYXh7u5eWl7w8/MjLCys9Pt3332XCRMmcN9991GnTh0effRRhg0b5pThAFn4b731VmrVqsX//vc/8vLy2LlzJwBdu3Zly5YtWK1WDh48iJubG+PGjStdXDdu3EiXLl0AOHnyJMuWLePrr7+mU6dONGnShFmzZpGYmFiqX3n//ffp0aMHzz//PHXq1GHChAlMnTqVd955B4ATJ06wevVqvvrqK9q2bUuLFi345ptvKCx0HsqZmJjIwYMH6devHwAdOnTAzc2twnm1aNGCtLQ04uLiAAkEunWrxGrgD5g2bRrR0dF8+umn1KtXjyFDhvDyyy/z3nvvYbPZ8PX1pWnTpk7n8Mgjj7Bv3z7y8vJITEzk1KlTpdfraqEGIir/LooC/lUgvJks4D1egDp9yrpjLofNBpnx0u1yar2016adlCxEbsndR2G2dLUoGmg4XFxakw7A3u+h23Oy/XK4ebH2RB72Wr0q36dWLynrgJSVwhvL/xN2iwYlIw62fSplpG2fiKA165zsm3pchu6pqPxZjD5/7/Er5M9M1o2NjSU6Opro6LIyaf369fHz8yM2NvYvv2ZsbCwdOnRw2tahQ4cKx2rcuHHp/z09PfHx8SnNQJS/y9+0aRNdunSha9eupYtt+Tv82NhYdDodbdq0KT1eYGAgdevWLX3Nys7p5MmTWK3W0mO0aNGi9PF69erh5+fn9Jxly5bRsWPH0u0eHh60atWqwnnpdDrat2/Pxo0bOXPmDOfPn7/iQCQ2NpZ27dqhOLoXS849Ly+PhATxb3JkkBzZo2HDhhETE8OWLVvYtGkTERER1K5d+4pe/8+iilVVrh06fcVtNpvc4RWkySyYrPOAHUIaivZj9zciMrVJjROvUOm4MfhD5hnxBzHliAdJYjmb951fij4lujWENoSLhyu+tt4Dm28U89dcZMyozhgCaoipWXncPKVrZ0b/cudcMunXO0y6c4pyRL9y6CcpP616QvQvUBIgDZMOIo9AEd+qqFwOz2ARpp520aVSs4c8fhWoXbs2iqL8JwWper3z3w5FUbCVlF/9/Pxo0qQJGzduZPv27fTq1YvOnTszevRoTpw4wcmTJ6/6Hb4rli1bxqBBg5y2devWjXnz5nHkyBEKCwtp3rw5IMHBhg0bsNlseHh4OAVK/zRdu3bl22+/5cCBA+j1eurVq1cauGVmZv4r10rNiKj8dyjOl8zB/h9h3ljx7Ij7DXRGyTSc2SCOrI4gBMTnY/kj4GaUckpxvpim6QzSyVKerR/Cz89B/3ekRFMeRQO3fIj++EqmtPDhXL4W+63zoOVkKSFpdFB3AExYCauelA4eEIfWAqlN0+YeKdEoChyYC52fhKX3lwUhIF02hxbAr69BSmzlZSIVFQfu/tIdU7OH83ZH14y7/1V52YCAAPr06cNnn31Gfn5+hcezsrKIiYkhPj6e+PgybdTRo0fJysqifv36ALi5uZWKIcuj1+srbI+JiWHr1q1O27Zu3Vp6rD+LYyH/7bff6Nq1KwEBAcTExPD6668THh5OnTp1Sl/PYrHw++9l4x7S09M5fvx46WtWdk516tRBq9VSr149LBYLe/bsKX38+PHjZGVllX6fl5fHhg0bSvUhDrp168bJkyeZPXs2HTt2RKvVAtC5c2c2bdrExo0bS0s4V0JMTEyp7qX8uXt7exMVJVloRwbpgw8+KA06HIHIxo0br7o+BNSMiMp/BbNJRKYL7pR2Xr+qYox2bCUcXQrjl8K2j52foyjS3mvwkQm61mKIaiW+ICfXQbWO0rrbfDz4VwdLkQzE+/kFGDMbEnaJn4hftPiIFGXDgdl06VwHt/iTKFokuGg5oWTi788wc2CZfkRRoOvTsP0zaDlJRLZ2ZLpvi/GS0XE1zwbELK3DQ3B4kQROXiGSTVFRcYVvJIz4RoSpRTlSjvEMvmpBiIPPPvuMDh060Lp1a1555RUaN26MxWJh3bp1TJ8+naNHj9KoUSPGjRvHhx9+iMVi4b777qNLly60bNkSkC6PuLg49u/fT1RUFN7e3qXdJ+vXr6dDhw4YDAb8/f154oknGDVqFM2aNaNnz54sX76cRYsW8csvv/yl8+7atSuffPIJwcHBpRqXrl278umnnzJy5MjS/WrXrs3gwYO56667+OKLL/D29ubpp58mMjKyNGh47LHHaNWqFa+++iqjR49m+/btfPrpp0ybNg2AunXr0rdvX+6++26mT5+OTqfj4Ycfxt29TNe2Zs0a6tSp49QdBNC+fXsMBgOffPIJzz77bOn21q1bk5KSwtKlS3nmmWf+0nsvz3333ceHH37IAw88wNSpUzl+/Dgvvvgijz76KBqN5CH8/f1p3Lgxs2bN4tNPPwUkEBo1ahRms1nNiKjcROQlS5BgNcscmfZTIaK5dNiMmQ1u3hJ0jPpeSiweATIYL+2klGLa3gtNbhWDtHNbodOj4g9Sp6/oRL7rC+tegKjWMOhj8AyVEkmfN6HFJAmAjq2E6l3wNWhwt+bLvmd+hS86SUDj5gHe4RL4VO8M45dJaWXgRzLzxs1bykFuXuBbBfLTK3+/Nqu0XR76SVxeM+L+rSutcr3i7i+lx6iW8u9VDkIAatSowd69e+nWrRuPPfYYDRs2pFevXqxfv57p06ejKApLly7F39+fzp0707NnT2rUqMG8eWVdZ8OHD6dv375069aN4OBg5syZA8B7773HunXriI6OplmzZoC02n700Ue8++67NGjQgC+++ILvvvvuL9+Vd+rUCZvN5rSIdu3aFavVWuFY3333HS1atOCWW26hXbt22O12Vq1aVVr+ad68OT/99BNz586lYcOGvPDCC7zyyiulPh+OY0RERNClSxeGDRvGlClTCAkJKX186dKlFcoyAEajkbZt25Kbm+t0XgaDoXT7lepDACIjI1m1ahU7d+6kSZMm3HPPPUyaNInnnnvOab8uXbo4XZuAgADq169PWFgYdevWveLX/7Mo9j+jSLpG5OTk4OvrS3Z2Nj4+V0eQpfIfwGqF4yvg9AZoPEpKMp0eg+g2kLRfFviIZlKeyY6XMorRT5xRzQWw4XW4sK/seMF1YdjXsPs78REpj5uXBDqBtUQ8evEwRLeS7hqLSe42d38j5Zb6g+V1FpVYwIeUtBB7BEBRHuiNMjhPbxBB6v7Zcm5NRoPBVzpovu7p+j0rihi4zR0ngVZyyXm4eYJXuGv9jMp1SVFREXFxcVSvXv2KWjBVrm8sFguhoaGsXr2a1q1bX+vT+Ue53M/2X1m/1dKMyrWnOEfKMq2niDnYwA8hdiWs/T8RgNYdIAFDZHPJWiQfgv7vQWANWPGIcxDi5iXD7n55CWpdUlNv/6CUbg7MlmxE7T5Qt5/oRjLPyus0vVUG75lyxHW118sSXNhtkHJUsioOQhvI8VY/ASd/Ltt+YDYM/xp8IsX9Nflgxfdcpx+gSPnGZpEAK7qVZEZsNgl2rlI3hIqKyr9HRkYGjzzyCK1atbrWp/KfRQ1EVK49dmTBzo4XB9T8dClZ9H5N0s9HFklWJGYwdHkKbGYoyoV9s6QcE9lC2mS7PyddNClHpX5epR3U6A7V2kPVDuLvgU2yH7ErREfiEwGDPi0ZZvctxC6BO1ZA4j5xT7VaoOdLzgEIiIC275sStJQPQhzs+1GmBQ/7EhbfI5kdBzW6irZk83vQcqIET1EtIS9F2pjPbZWW46rt5LoY/cU+XkVF5bojJCSkQilExRm1NKPy72O1SLdIQTpo9ZJxiN8J7n4iKN30FjQaJfqPc1tEK9LtGYjfBYcXyD61+0gGY9fXYCkUjcjKxyCxTLmO3h1GzxKr+N3lSjQ1uooIdcFEKe3UHyIB0N6Z8niru2Sy7qH58n3nJ6UEc26blFsMvjIAz+ALS+6Wc9e6yQybRiNFQ2K3i6ZE7yFZj8IsERp6BMh57f5Wgpl290NBplyHlKOSGYpuJcfQaAFFOoEM3vJlt0tJyCPw3/ikVP4B1NKMyo2KWppR+WexWqEgVRZmm00WUTSSWbAWSTZB7yGdJXariC21BtDq5Hv3wMvrGmxWCT5MeRIsrH1GjtV6CniFSfmiyRjJDhTlysIf2UJ8P6LbwLoXRczqIHGvaDLGzpN5Nbu/cw5CQFps546Fkd85ByJnNsoi3+Zu2PKBBCrDvykLRI4tF+v5Q/MlY1KtA7i5Q/5FyZJEt5Ugoiir7LoM/wqOLoPZI6Vc0+dNOd+tH8j04PpDIaoFKDrpqqk3AI6vgYMLoHYP+H4QdH1GzmvZAzIXJ6wxtL1Pzuf0BjF/i2gOx7ZJpsi/iszvMXhJu7JGJ9kTrdp9o6Kicv2gBiI3I0U5UgYozJLptXabBAV7vpWpso1HySKZFQcRLaW7JDdJMheKBnZ9Je2w1TpIa21OElRtDw2GgNYoHTB6d/AIBpsJzMWyaJuySzw5NCIojd8J/lUh6ZCcx9Gl0Hg0DJkGhxfCri+lcyUkRrQi3uHOvhtZ56QE0uZu+KoSZbmlSEzR/KrK/g5OrZcgCESjUc55EJ1RSjwegTBiJqTGwoqH5To5nrv1I7htEXR6HLLPi5Pq+e0ibu3xIpzfBj8/W7b/qRIzqqDa0vXz0/3y/fhlsHCyBGGJu8V/xMH57RC/Q0pHKbESoIz6Qa512nEJ7ixFkqU5s0nal2v1kGxKcZ5kbBzvTWeQczOo04GvFf/h5LOKyhXxT/1Mq4HIzYTNCrkXZVJsdoL4bKSflseOrYCAmtLyOnuU86K7/RNZMPNTYf546SYxeMGssn58Tq+H7Z/CsK9g8RTpWtEZpR3XWgwrHxHdA4hnRpenIThG7vzzLkLDoTB7NNTuXWIINhuGTJcyzraPZR5N9+ckY7P2/+S4AEcWQ7PbpOOlMvJSpexTPhCBsudU7SAzYRw0HC7ZlNZTwJwHa54qux4OivNgxUPSTqx3L9OQNB0L+WkVPU8cpJ2U6+LuL4JZvbu4t1bvItf9Uux22PgmdH4Clj8oupLBn4mo1b+amKPtmFa2/6+vQosJ0GpKSbnLF06slRbh6LZQt68Ekapnyb+Gow20oKDAyVtCReV6p6BATCMvdbr9q6iByI1OUa4EAlazZCT2zYIqbSSzkXpC9BXmIlmkM06L78aQaWJL7jDuMhdAyhGxSbeaJWsxe3TF1yrMlAX47q1S5kk/JZ0j3w9yHlWelwIrHxVr9q0fSTfL0aUiTk0/BbHLZbHd/L6zyPP4Krnj7/MGrHq8ZKNdghW/KiV28C4IrS9ZnPI4sgQ6A3R4UMasg0zqrdZJMhqRzSVIcrioXkpKrAQd5Rf1qJaShci5UNknIsGIXxW5JuZCyfRknK58/+x4CaQALh6S9xwzUGbulA9CHOyZIaUtjaYsuPGJkPeWelzEwQYfyZDor8yxUeXPo9Vq8fPzK52F4uHh4TT7Q0XlesNut1NQUEBKSgp+fn6ljrBXihqI3IgUZsto+twk2D9HgoI6fWWBbXknnPxFNBntp0p24VQ518KT68SzY9AnMH9C2XZ3f+kQCaghC+ClGQIHnqGQlwRrnpYAp8EQ5yCkPNs/k/O6sFeyMaknoP0DYh6WEuschDg4tV70Fj4Rstg3HA5xm2R2y6K7Ku4f1khKUUXZzttr95HunDvXwLFVEkDU6iWlI0uRDM4LbQSW4stcaOTx8pOBLUWg9xTdyKUW8w58wsts4e02+dJ7XP51HMGOTxSgSHbrwKzK9//9c9HXgOhRmtwqepgN/5P32Ox2aVV28wLfqqBT/xRcTcLCwgBcjoZXUblecUxP/ruof31uJPJSpNRhypb0//HVcHCOZDGOr5bFe9xCqNIaIlvJnXn5IMTBhX2SmYhoLkEClE2t1eqlC8QVWjdpSz2/XZxGFa2UByrj4mEpfyy+G2IGSTChNUDDEbDto8qfF7tUgobT62WB1RokozDgfdjyvpSdtHqxTu/4CKx/rey5iiIeHn1eB7sGji2TAXvuAXD+d2h5h5SrzAXSJuwTIUGAY7BdeXwipPSRlyIal/w0EazW7Suaj/ICWQduXiIAzi6ZzXFgjrwHrxDJzrgqMUW3LisdtZ4i7zf1mLxeZRSkizmabxQ0HgM/jS8LHovz4PfpkvVpc69kcILqOOtkVP5RFEUhPDyckJAQzGbztT4dFZW/jV6v/9uZEAdqIHKjkHYCfrpDWkAdxAyE25dAVrwsfhqt6DyMvuDhCxteqfx4hxdJNsMRiKSdFF+OhF2SZXBg8BZbdUUruoTjK2D7NClp+FcTh9QuT0lL7qX4RpW1tMbcIvqGXq9KKcV6mUyEpRjq9peOEkUrOpHGIyWrUqWtLKxavbTbftcfuv0fdH9WsiJGH8nuuHmAzS7BhE+ELMJJB2He7dLl0vcN0bCkxspivf1T53NQFOj9uhxr6VQRlC69TzxFWtwJdaNE+3FmY9lzjH4wcib8+nLJ974i2o1oLhmSfm+LQVv5bJNHQMnwvPug6W1iwLb7GwhtLJ/HpZ1CDmp0kwxW8zskE+Iqg5V0QMo3Wz+W6+MTUfk1V/lH0Gq1/9gfbxWVGwXVR+R6oCBD9BoGH/C4ZL5EYS4UXBThaPmR9W6eYqZ1brvcfTcYKlboaSfl8R4vSUbh7GbXrxlYC5qOg/Uvy6I7apYIVH8aL+WdnAsQWBOC68GpX0FByh1Z50S4WT6Q6PqMBDQn1jq/xqBPIaSeuJrumCYtubcvleee2yrTcl0xZDoE1ZXgK/cihDWUksOomZLNMXjB2S0ivI1qLdkGr0pGpaceh+ntXGc87togLq7+1SSY2/ON/BvWCLo8CZ4h0o7s5iEdSAYvyE6UrqHIlqK/KcqFjFNSUvEJl8AhtIFoQwoypGX49K8SVLW+S7YfXihZmeqdJbDKTiwLllY8Kp004+ZLELZwUsXSl5un2Md/3lF0OD+Nd/3eQV4z9TgMeE+yIioqKir/AH9l/VYDkf8yOcmi9TgwRxYfr1C5O/erIhNj89MkRV+QAT/d7vzcWz6Efd+L7mLkdzBnjPNiG9pASgI/V+L41/5BWQCLMqHV5JIgKEjS+ifWSEbi1C8SSHiFSSCCRvQaPuGw5L6yY7l5wdDPxb0UZEFtdZcEMVXbw7xxZd07niFwx3Lp8lh0l5SayhNcT7pn5t9R9n56vSLeHcdXiT6meicptRh8RL9hM0vmwitEFunymAsl6Fk0RcoZIJmKbs+KtqRWD7GLb3MvBFSV9+jwRAmsA+7eFYeP5SbD3lkQXFs+M2uxZCR8wqXEpdFLGas4TwJMnVFKQfmpIlwtyinJWvlLIPTLi5BxVt6Hg5jBYp7m5i4ZqDO/SodNtU5yPZL2SzYmZpC0/VamV+n8uASSQz+X7IyKiorKP4BqaHYjkJcq2YVZI2TuiYPY5dD1/yRNX5wDyx6EVpOcn+sRIHfniXulTLBnZsU7/otHpDQSXFfuiMvjEyFeIhq9uHpqtbKwb3pHAqNaPWXRi24t4sxtH4umJKiOOJbqDDIgzlEmKs4TT45xC2ThDYkRXYVPhLShDvtaMgUH58lzCzJkoR76hbQVH18jpZYmt0qQMedW5/ez7WPo+5ZkEHwiJTvh5ileG+tekMBAoxM79c5PSEDiQO8uZYy7N8t+NrNsU7Ryzl4hMl1334+w7UPJsrS+W8zWLs1OOfAOg7ZT5D2acuU4nkFlnS9/RM4FEfpaLVIeihksxmil5+wh3i7hjaG4UHxUOj0m520xSft0+mm45QO5no1Glpm1lUdRxCzu3HbQqW2lKioq1wY1I/JfJf00LL1fhJ+Xoihw73YRRm55X0ow5dPv1TrKYr/zK+jzPyl7ZCdUPI67v/hgnN0iLqI2iyx6LcaLu+rZLRKsmHIk++LuJ4tgUB0RrCbsLtdGW+7chn8rAtnFU8q237kKNG6yIMdtFt8Qg7fzcwuzRMfisDLPiCuxgdfJc/d9L90gl6LRwdRd8MvLcHRJ2fbgejDiG1j3EmAD7whZeBuNcO50+TNYreIpojPK4v5vUJAG+ZlIvy0iyNXoSlxvFSmfOYKqlONyfls+kBKY3kO6YlpMLPks7i5p/S1BUaRdOumABI/BMSXHVVFRUfn7qBmRG4GibNdBCMgifX6HzEgJritZgBYTxD8CpNxgKPngCzOkPOAqECnMlDt9/xqSOQlrLCWEuePkedW7QvPx8vzcZMku5KdKd0pwPVjvQuxqt8Oqx2D88rJtoQ0lCNjxBez6HCb9UjEIAeeMgaLIdF2NVgbahdZ3HYSAlHm2feochICUreaOk0zMnhKH1JRY0dJ4BjtnRv4IrRa0vn9+/38CjyD5chBYU8o2Wn1Fh1R3P/j5/2RAX48XRVQcu0zKWy0mwJDPJENz6hf52ajSVq5P+weltKYGISoqKtcINRD5z/IHiSqbBX57W4bAGf1E7DjqB3E+vbBPBKIgzpsdH628u6L+YLEY1xtlQNy+H6UF1+grwszYlRDWANY9X/acgowSfUOO62MWZJQ95hkEw76Q8snpDdI+7F8d0s9Awu+SeYluLdoQdxcLvX9VmWKblyQlDoeOozzNbodfXxF30vjfpVzkIPOs6Cz2fCtlplO/wO6vYfSPULXj9bUAa7SXKQeFivg3cTd4RUi2K3GPBB8rHpbgo/+7Yi9vt4M5XwS/7v5g8HR9TBUVFZV/AbU0818l/RQsvKusffZS7tkKX3V39vRoOVE6XbITpDRzdrNMpO35kgQVu78p21ejFd+OzLOQkwjdXxRRZewyCR58IiU7EtYIZt4iC5qDGt1kQfthcOXnf+caMU3zDpfXqD+4rKyx9wf45XlZEB10fETMzC43VTYl1lnY6uYlpSWNDo4sEl1JjS7iXbK1nA9Jn9dlMF75rJBvlNjWB9as/PWuR+x2+axRwFIgmS078plqDTKgUKMrmadTSSeRioqKyt9ELc3cCBh8oPer8ONw5zt8kJp+bnJFY7G9M8WpNP2U+GX4RkubZ9xvIq5scYdkB7QGiGgqHRx2myzoplyI2yAdOTu/KBOfDvzYOQgB6Yzp87pkTS51LAUJJsz5svhnxkkZpuFwuWtP2A3rXHTqbPlAtC21elZ+TUJiYMJq0U6YC6VsseIhZ6+OnV9Cy0kiSv3tHdnmG10xk5KdULnj6/WMoshUXgcFmSJ81RtExKyioqLyH0MNRP6reIWAxSx+ENs/kzKGZ4jU9H2j4MvOFZ9js0oAU3+IBBwKMna++XgZD28thjp9RB9RHlOuaE4Ks+HnciWYlFhpU9W6OfuC2G3iGjrgA1g0yTmzoWjEmGvjGxKEgIglCzOli8U7Ah7YJ8ZfRxfLcUMbyTns+FwcX12VaBx4h8oXwK5vnIMQB7u/gZEz5Fp4h0kQ4mpeTGU29TcSlZVyVFRUVP4jqIHIfxm/SCASer4IpnzRcVhN8HGzyp9jyoXf3oUB74qoUdHI/JfLuTnmp8mCtf2Tio+lnZT2z/2XzDXZ9TW4ecNdG6UrJ/W4tOw2GSMD5hJ2y34RzcTXYvsnohExeIswtt4tULObbDu5VoKjTo9CUQbkXpB2Us9A16JWkCyNq4FvDmKXS1trlXbO3TsOjL4VAzIVFRUVlX8dNRC5HvAOA2/ERdRqlhJFSmzF/QJryVC6/m+Lj8bFIxIQBFQXwy+fSGmFhRLBaUGJ+2c2uAdCzR5iClaeoDpiXpZ6zFnwavCRQXF6T4hoIWLZg/Nk2qujlGT0FfOxOWNEKApi/f7zszJcr+EwWPOUbPevLhmUurfIv3tnyrC23q+LYPVSbNay6cCuMOWI10rSftczWfq+KSUbFRUVFZVriipWvd7ISxdL9x+GSdnEgVeoeGasfkaCjU6PSYaiKFsWYodluEZbEowoogPJOFNiApYpWQa90dlt9bbF4l9hs8h+6Sclk2D0Ew+TsMZiNOZXRUodPz8Hx5ZLoND5Sbh4UAzJXDH8G2mrbXuPmLcVpItbaVgjmDMass5LYHXbYtGdKEiGx81TjLxWPQ77fyw7XnhTsSz3CBRbdlM+FOfKua55UnxJgutKR1FYY9HRqKioqKj846hi1RsZr0Apo0xYJYFE6nHJeFhN4rKakyjTa+02yEkSp1CPQJl0m31e9ByBtaQ7xjtM9tvygWRGer8uBlc1e8gcGpDSyKl1JSZZ7hKMFGWLnwjI60W3Ft2KXzQM/gx6vSwZEGsxbHmv8veSdgo6PCgtxg2HiWjWapIAa9QP8OMwMR8rzpPyjVYPUW1kUJuiQJcn5HwvHpIBf3X7iyupwUfm1JhypQTkGQQDPpSSj3818c0wVlLyUVFRUVH5V1EDkesRjUaCD1uxlE7m3ip26gYfWcAP/STD0KxmCRxGzpByyU+3w7CvJCA5t1XaXD2DodltMgdl09tQr7/oPByBCHYJEEAEn+mnnM/FzUtMthwj5A1e8pWXBkn7ROhqcyEUBajWAc5sEOOyuWMl6wIScHR/UTxH9v0An3eAoNpipPbz/5UJVMObwIB3IPuCeGGc3SoB1a+vioNq6yny+tkJEnxpdBLkWIvBGioBioqKiorKNUUNRK5XFAXQynA4x0yXni/JhN2kA2X7eQbLlNpNb0k2wD0AfhxaNqulOE8W7jp9ZcBbUB3RdjQeLUJTg694kxxe6Po8moyRhd3jkkXdzQP8qsnU3/2zKz5PZwQUKZHMv8P5MatZDMh8I8u8T/q9LfsVZJTtl3QAZg4Uz5Kz28QtdO5YyaJU6yylG0eLrmewZGt8qsnrWoplkJzRR9pabVZpibYUScDiV6UsuHJgt8vxNDq1rKOioqLyD6G51ieg8jfQG2VRBMmGuPs7ByEAfd+AzSXlkZaTYO0zrkfen1gjE3mtZtFhdHxUts+7VUoxLe6s+JyoVpKVaDhczqU8bh6iR2n/IPi5EJsOeFfacF0FKSDH3FbSxRPdGi7sdw5CHFjNUoap1R1O/1riWTJC/EXK+4Tkp0rmqLhADNZyL8j2I8skoxL/OyybCl90hu8HSStx+hmZfwOQHS9t1D8OF/1Kwm45TtpJyDwnx1VRUVFR+cuoGZHrGUUnws2gOlKCSDni/LhnsDzu8PPwryqdNJVxYZ9oKnISReBqKYJOT8icl4hmIko9MFsW3br9REjqGQK+EZWcnyLD1oZ9JV03x1ZIJqHlRNGHHFksZSJXuPtJhw2I6Pbs5srPO2FXibmbBhoME5t6VxpsmxV2fi7ZkCOLpGw17GsRys68RR7XaKF2HwhvBDkJEoQZfSUo2vOtBFXDv5YS1fwJEqBo9fK6XZ+Rqb1Wk/zrEyFBnc6t8nNXUVFRuclRA5HrGd9wyLZJyWHp/RIUlKdqe7lrD6wt5RmUMjdVV7gHyDF2z4C935Vt1+rFvCxhl9jCewXLjBjNHyTUPENEm/Ftbwkmhn8D2z8VTcnyB8WfJLSh61bk1OMQ1VpMz4qyxSr+cq9jt4vmJO1E5cMCQR6v2lH+n5MIs4bBrfMkCFE0MHiaOMfOvKUsmAmqDUO/hIjmEFIXUo7B8gfKjmk1S+vyxSPQ5SnR4rh5yQyc1neJ9sU9QLVUV1FRUXGBWpq53vGNlDv70T+KjXv58fZ2u5QQ2t4r35/bUrmFuqKBGl1h7dMyPt6nXJbDapZSR8NhZcPs/igIARGQdv0/KRulHJU5J4oGDv4kj8cuF42JxkU8fHQ5dHxYgqATa0VQWxnNbhPNiXcYaPSSqamMgBplZRkQAW7cb1L+qTdAHGn3z3bOqKSdlBk3wbXBYpK2ZVdcPCzZEHd/0d78Ph1+Gg/mAkg7VpbhUVFRUVEpRQ1EbgS8QmWxD6gBt84FfclE2XNbxZMjJ1Em6h5ZInNqXGUX+r8r5RivUOmYGfW9CFYd2KxiaKYzVnzu5QioAVM2isYkJ1lEsVnn5DFLEWyfJqUO/+plzwmsCaNmSulp/DI5xtmt0Pu1igLSRiMlY5F+Cs5sggZDxFzNFYoCjUbBsZXO29NPirlZg2Gw5zvXz825INklo29ZqcsVKcec38vFw6JB2T9HtCQqKioqKk6opZkbAa1OsgG5F6UkMPwrGXZmypHSyJHFst+A9yQrMWEVFGWKEFPvLmWD2OWwcmDZMTe9JV04TceWCUpzk/+63kGjkcCi75tyPqZcEbg6XFpPrxctRvsHRNNi8JaswrGVkunIT4PuL4DdAj5RMnX4zEbxKYlqKUGX3lNKRSlHSD+zD03VdvgO+QLNqsckMwESQPR6BQ7MrTh3JqgunP1Nsi8OB1hXZJ0XF1m9u+vZNSAzgoqynLed+lmEwKYcuebufn/tGqqoqKjcwKiByI2CoshCqveQOS2HPpeWV6sF2t4vi/3vX0KrSbBnhtz51+4tOgaNHpqMhsiWsOpRcVsF+OUlGPuTlFJsFpnbYrzCCa56Y0mXj15ec98PUvIB0YOsfFSEomPnyzlHt5ZOntyLYMoWoezqx0Xk2ud/Iqq12yUTYimCoDqc7Pg+D80/ytEFe+laK5rHhqylnmc+OmsRitEH1v4fnNvmfF46o3TZ7PtB2pAN3pVbx/tXBcVetv+l6AyiywmuK+/NWlyiD/GHrATZZnfRsaSioqJyE6OWZm4kPAOl7OJXBfq/J4v+mY2SYciIgxYTIWEn7PwChk4X3w2bGX6fJp4bCTthzGzo8mTZMeM2iTmYf3XRoFxueN6foShThvLdOs+5rdcnEoZ8DlqDZAzObobjqyH/IsQMktk0nZ8E7LDkHulWMWWL9kTvQabVCLnJfN0DttwRxISmXjy8Oo06n6dxxh4hAUetXs6lJe8wGL9UdCuDPhEr+Db3uj5vvypS/lL0Ug6KauX8uM4oouEd0yUA2fA6rHiEnPQUUordyAtvAyGNJPukoqKiolKKOmvmRiUvVRZEUw4svgfq9oU6/aSDpcvT8nhYfVj+kLOviJsX3LlKfDvST0rWwd0f6g+BoFrOr1GUCwWpUqYw+MjCrtVf/ryOLIFfXpS2YFuxlGM0esk2bH4Pmt8Bs0aUDc5z0PdN6VpJOgCrn4CRM2Hd8zBuAVhMWC4cQLfqkbIsi08kyf2+4Z71ZvKL7cxpe44gH28IqCaCU717yfwaDWTGg04PXuFyTvG/w8Y3ZcYOiOla79ckyJt/h4h/wxqLY2tmnGRRPIOlI6jTo7D0ATI7Pk9sQC8+2XiG8+kF1A314sEetagd7IGn+1/U2aioqKhcZ/yV9fuKA5HffvuNd955hz179pCUlMTixYsZMmRI6eN2u50XX3yRr776iqysLDp06MD06dOpXbv2VXkjKi7IT5NFtyhLFnDPEJg7Bkb9KEHKknvLNBQgnTJDvxBh5ukNElg0GCr6CnOhZE/c/WVWS84FWP+ytNfa7bIYd35KdB0aHZjzwVRybIO3mJcBHF0Gi+4Sgeq8cp0wk9aJBmXDa5IV8Q4rMQw7IY9rdCLEzTwr7bw1u0tZ5vAiKMyQkkjd/pIp0ejEwOzgfI53eI8+M86x7XYfIhYNFRGumxf41YTMMzIjR6ORuTzx2+DEz+LJ0nKi+LOYciXbofeAxXeJ1wpAu/uhWhdpoZ5zq2R0Oj8BdjsFxmDmxvvxyvKjTh+HosC0sc3oExOCRqdWRVVUVG5c/pWhd/n5+TRp0oSJEycybNiwCo+//fbbfPzxx8ycOZPq1avz/PPP06dPH44ePYrRqN4R/is4ZqnojFDDX6bbeoVKF43BuywIURQpS4yYAQsnyp2+g20fSZnn9K9iSFbvFuj4iCz0tXuKtiR2hQhHg+tLhsRigt8/h0Pz5f8BNaDvW1IKCmskAU12gmQ4LuyV19nwumQder0qQVPWOZkWHFAd1r8qhmjZCbDpTcnonNsuc2dAshrNboNTv4glfH6alFLa3U8VXSZd6gSjyz9fotswQ3aidMnYrRC7RIb8LbvfeY5O3CbZ3utV2PGZ+IQ4ZuEoiuhaGo0UPUvT2+Q8F0wEUw7J95zlrdU7Knwcdjs8t+QIjSK8iApUA2sVFRUV+BuBSL9+/ejXr5/Lx+x2Ox9++CHPPfccgwcPBuD7778nNDSUJUuWMGbMmCt9WZUrwTNQ/rVZpOvDIxCKc6WjpuPDkuXwCBb30PJBCMjqufoJGDNHgpeIZmL25ega0bpJd43WTfQfacdg51eQfLDsGBlnYPZIEaJW6ygBx4bXJSuybxYcXynBQ1G2lD7Kd6S4+8OwL2HFIyVGbApsfEMEqw5a3yWtygfmlm3LOg+rn8LY8yUe7zKCkPVPyXadUTIds0dJtqVGV7GOv3SYH0hHT8OhMpgvrBG0uRuM/hLkHF0MP90h17TBUAn6jL7g5klCjhmTxbVpXHp+Men5FqIC/8TnpqKionITcFXEqnFxcSQnJ9OzZ5l5lq+vL23atGH79spdL00mEzk5OU5fKv8gflUkaNAbwb8m9H4dNvwP5o4TLcnxla6fZ7NKBqDJrbD6SedAwVos3Sg+EaLZcA9wDkLKs/ZpCTaajIXbFsGBnyC8MYxfLq6rlwYhIPNifn5evEH8oiUTU5DuLDqt1VMyFi5QNr9HjHuWZF5qdJU22m2flJV8avWE2GWVX7NjK6HdVJm9k50o7bknf4Y9MyVrk5Mo2pClU+GWD8DghUap/HBQ0QpFRUVF5WbmqgQiycnJAISGhjptDw0NLX3MFW+88Qa+vr6lX9HR0Vfj9G5u0k5h9woHryBYPEW0Hg4qs34HCThO/1r541s+FBHnxUOV75N+WgSwBenSKtz8dvHt+P1z2Z6f5vp5qcckE+NwZNXopNUXRO+Rn+p6tgyAKRdd3gW4f6e0MxdlyTThEd9JyUhRLmN57w+1esvsGf9qst/csXByrQwTbP+QZFcMPpCbJJmTwFqEeRvwMrhONkb5u+Nn1Mm8GxUVFRWV/1b77jPPPEN2dnbpV3x8/LU+pRuO7OCm5Gr94Ow2yU44MOWI/0VlRLeGxN2VP551DoLrgZt35fvojBJsfNlVOlJO/SLZCeufWJQtxaI5Aag/GE6tl/9bTWJoVhleIaLfOLwIFk6GZQ/ITJjA2jI/J/mwDPC7lCrtpCQUuxw+ayndRjlJcPtimY8Dck2ajoVb3pfW3YTdENUK/+yjvDSofoXMiJtWw/+GNCTKLR9OrIZjq6St2iHqVVFRUbkJuSrS/bCwMAAuXrxIeHiZnfjFixdp2rRppc8zGAwYDIarcUoqJWTZPUgr1NI8J5HSdVLRgHeEdL0smlQxu1CrpxiLBdaGs1tcHzisMaARu3WtvqyNtjxNxoihmClHdB4dHhJB6qlfylppXWUnDN7iGQIQ3lRcWLd/Jt9bzRKMeAZLZsSBRicBRqcn4NBCmZyblyKPJe6VjEzfN8XlNbCWlGAc+hiDN3R+XDQgtXtJCctaLEHJ6idF4zJrpPPsmIAaMPBjSNxD4IJhdJy0h7lT2jJ/TwLxGQXUCfVmVItIIozFaD5rWSYU1mihx0uSHXL3d31tVVRUVG5grkpGpHr16oSFhbF+/frSbTk5Ofz++++0a9fuarykyp8kNbeY11bGYg9rVLaxZnc4sghil8KoHyQboNGJb0bPl8UJddkDEDNQ3EMvRdFIO2txnhh6Dfy4op9IeFNoOUn0FSDdNKnHYc5o2PmlZCla3eX6pLs/Jy3DY+bAwI/g+yFisjb0CzEWO7RAshduJZmRqJYwbr60Ky+9V3xBer0KXZ8pO2bibmkFDinxUhnwPnR8TLI6HR6G42vLWn23fyY293X6QK/XYOGUigPsMs7Ar6+BwQvMBYTN6k5jJY7Hukbx9uA6PNA5ivq6RAI+q+fcMm2zirYm+fAff3gqKioqNyBXnBHJy8vj1KmyToO4uDj2799PQEAAVapU4eGHH+a1116jdu3ape27ERERTl4jKv8+NrudffHZpHvHEOzIIgTWlKmz57bB+R1l4+uLsuHcDvCrQsHYxRiMnmjGzkdZ/lDZ4DefSJnh4u4v1vKmbCmhjP5RptYWpEvHSVE25KeUDbxrfgfMG1uWfclPkQAoqBZs/Vj8QIJqi1A0K15m37S9V2a1FGVJJ8+da6HnK9IVpPeC0bOgIE08SGaNkqm3ABwR/Uabu6HVZNj1tWw+MAe6Py8+JCfXiSC14TDJyhSkwfw7nefGbPgfnNoAbe+BFQ9XvLjxO6DfmxLAWEwYN71KWM1uEuiZ9dJhM/J72PGpZF0ajynTu1zYL9kZo9rWq6KicnNxxYHI7t276datW+n3jz76KAB33HEHM2bM4MknnyQ/P58pU6aQlZVFx44dWbNmjeohco3xcJOPfPS8RNbfvhRl8V2i2/CJlB3yLsLmd52eY89J5NvQVxhQS0P1X56W0ohXiAQRpmzY+pF4gnR+XHQXMweKsDWwpgg5d38LnR6Dvd/LAX0iIS/ZuXxjKZbMRtu7YeQMCQTSTko2wtHhcm4L3Lla2mRbTpQsSvJBaD5eOnccHUFrny0XhJTj9y8kU7L7Wwk2FMQ/Zcpv4OYhFveWIrGy3z+34vA6gPjt0OL2kiGDLoTXmeek+8evirQn//4F/PqqeJbU6C6Zo75vwYm1Ml+nKFuCkZhBUH+gGoioqKjcdFxxINK1a1cuZ8qqKAqvvPIKr7zyypW+hMpVIMBTT81gT06n5jP1F08+GTEDJScRxc2jTAx6KR0fZqTRF5/EzTLbZaUEnRj9ZOHPS5Gul+a3iTPphJXSWRK3Wdp6Oz8hC+/RpSUnUbOiFsQ7TAKArHOAAkvvr3geFhNsekcG8Z1cJ3btWp0ED01uB4OvlIkck31dkXxINCFpJ6DZePE+0buLS2zsCikpNR4tRm1xGyuWYECCrOg2Ze/Hgc5Q1s3T/134abxzsHJ6vQQ8Ec1gYzkfFJtFSmM5STDmB9G7qKioqNwkqD7TNxkRfh68M7IJY7/awcrDKdzapjoRWg3V3e0og6fBykdkwQdZVNs/hBK/g9Azb2Kv1lm2R7eW4MJSLBbo/tUg7jcJSBqNEL1Dq0miCTHlSmah/mB5zJQnC+2lGQu/KnB+O0S1ubyvx5kNkg2p1VOyDEF14PASaHUnLJos2ZrLUiLRrdJePEUKs+DbvmXZDzOiWTmxFga852xD70BrgGIXGZfWd8ORxRJQpRx1nTFpMBSWP+z61OK3Q8ZZCfD+aGaPioqKyg2CGojchDSO9GXNQ51Zuj+RtDwTt807zut9IhhVoxa6sT+hmPJkYfYMhsMLxSzMIwil23Mi8mw3tcTOPLfsoA1HQL0B4vmx8lGZ4FutMyy5W0ooIItrizuhTl9xPm0wTDIBIFmBbZ9Ar2DwuMyEWoO3DMjLS5PsydnfoNPj8ONw+f5idxGwJuyS8w+oLqWnjDPy/KjWYmzmESjHODTfdQkm65y4rYY1rmDQZm92O8rx1ZJJMRdK4NBqkkz3Lc6TQKwyzxVFK11DlZF+UszX0k+Kh0loA/CNrHx/FRUVlescNRC5CdFpNVQL8mRqt1r8HpcBwKJjhWTag7EW5DCpsR9eyx8Q63MH3Z+TMkvPl+HbPhW9Pw4vgND6kmkw58v/Z48Uoy8HVrNkG9z9ILod1OgCEU1h70zJbrj7iyB0xLdicuaKziXOrno3CTICash5OkSwu7+FoZ+LSNZaDBePynkHVCdbG0iKoTa/HU8CcxojGnjhe2JN5Rfq9AbJ/pQPRJqMxeoZQnFwEzxGfS/lIIMXdp07JPyOcmajZGyS9rs+plZXeZsyiF7k11clyNv5lXT1jP1JXGVVVFRUbkD+U4ZmKv8uWq2GKoEeeLhpMeo1JGcX8fWuNPJSE5yDkKDacieffloyDZUZkO2YJhqIcQvk+eWDkPLs/Eq0JSlHJePScqJ0vXR7VvQeZ7c4t9o6aHa7mK7NGQNf94Tv+sGPw+ScHCUZSxFo9NJKu+Q+8SixWTC7h3Cy2J8J3+/n1bVnefXXZFYczy9r+XWFwVuyHDV7SOvybQuhRlfOXMzi7j3hbM4J40COB/OPFdPn+3g20xJbZCtpYa4/xPUxT2+AegNdP+bQhpTPNKUchS3vy/v6L2O1ih2/w5ytIEM0L4UVsz8W22UcfG92ioukrTzjLOQkQ0GWBN4F2fLvlQ1LV1H5T6NmRG5yQnwMfDSmGc8sOsiIFtHM3nkes4ezNT8xg0Sb0e4+SDte+cHy02T43a5vJHipjMJMyY4E15P5NRf2SRdN9c4wcqYMsPOvBrfOldc15cmwvJD6ko0pvyibC+DnZyUL4ldFAoDN78kf854vS2C062v0G9+gZXAMi/s9w8yzwXy2PZXZh/Pp22wSgesfc32eMQNh28cibrWaREA75HNOpejYfCqZzacynXZ/a2Myjcb0xi/rPErGGWg9RTJA5clNlvPKiRdjNQeeweLO+vNzFc9j/2zo9KhMDP4nsVokE+XKG8aB3S4BZW6yOOMqivy/KBs8giRTY7NKSUpR5LPSu0sQZzFBQSr284kkG2pwvtiLlHwbVYO98TLoMGg1RPi7o9zsw3dykwENdnMhHF2M4pibVH8w1OohmcLja0vmJXWRKdaewaoBnsoNgxqI3OS4abV0rBXInCltSck20TDSl11peiIjWqC5sAfcPEmtMYSzUb5sjTXh5zGY9lPuwCf3FGFr7y7TXkCJRbwiBmK1e1X+oka/ktk1GyD7ArSfKuUamxXcg6SrZvEU0WhUaSsL4LEVYoBWWWZg55ditx7WCLZ/IvNkcpOcSzwX9hKyeCST+nzK/ho12Homi2Od29Gmahd05zY5H6/RSFkgzm2VrxLsaSfxDRoMJODhpuXxblXoWVXBagOdfxSKQYvS9l65Llo36cA5sVbeb52+skCvfFS2d3xULN5D6olN/YqHy1qVy2MpAosLp1pXWMyQlwRpp6S1OjhGWqg1GgkqNBrJXhTlwJb3ZJ9Go6FqOylhlcdmk9lBs0eLu6zRB7Z8ABcPSwt2y4mQfV50N/tnw66vROsDIiIe8R3s/JLjde/n9lnxpOaaSg/dupo/H4xqRGJ6HlFBJWMB8lKlZOXud/ng6Dqh2GLDbLXirtehcTUJMT9V3nNRNhRlo3iHQbX2Ivw+vV6yYUcWQY8X5Wdk0xtSAvUOk3KdXQEPv3/9famo/NMo9sv14F5jcnJy8PX1JTs7Gx8f1V/hamOz2bmYU8TCvQn0i7ZQbcuTpLZ9mvs3wJ7zWaX7aRR4aVAD2kdoqbliJEpqrDww9Asw+sOcUbIIrX7S2XbdQafHxbE0qqXc2bl5gneoLNwgQtYl98HZzWXPaTxKBuQdq2RCsMEHBrwrQctP4+UP9dxbJbi5FHd/dvVdzsg55zHoNHw9vArt3c+jPbYcdG5Qo5uIbje+WVHL0fY+zI3GcMEWQKTRhPb4ChRTDlTtgP3ULyj5qVC3v2hLFK0EIHoPEera7bJQW0yiaSnIkMDk4DysDUegnTVMOnLaTRXhr0YrepLci7I4+0Y6Bws2m2QhHBkFSzGc3ybTlMu7tzYcAR0fkQGHO6aVmMe1F1v5n5+HuE2SnbptkbMwNus8TO8gXUAt7oT54ytey9ZTRPw7d2zFx7xCSLpjB4O+2EdqnqnCw0OahPJ6d38Mej26hJ1gM4tHTGGGZNS8wuTzNHhLZus6IbvQzJnUPGZsjeNiroludQLpXz+YKLc8FL2H/JwXZohR3/zxomcC+bybT5BRCFs/kuAboN9bENUGq9VC8eEluB9bLAFm/3cgsMY1e58qKpfjr6zfakZEpRSNRiHcz517utQkPa+Y/KEzmbk9mT3n45z2s9nhxWVHmDGhFe7d3idy5Xhp5w2OgeMlfzw3vCblklVPlGVNFA20mADNbpOywIbX4ehisX8fVlLCsFklqGh3SSCSnSiBS2UE1gLfKrJg+UbLH3dXQQhAYSbB2nwATBYbm88W0NZjB9oGQ2VR//V1SYO7IiQGffJ+qtboCpkJ8roe/pB6HCV2mcyrOTRfSkt3LJMgpCBdylEGT1B0kHtByjI1uwMK1B2Axb8GmtsWobj5yH5b3ofjq0Grx95gKLS7H2Xvj5L1sVslW/P7NBky2GKCiHaLsmUGzqUaHnc/6Xza9nHZtotHYP+PMH4ZLDwjgdfubyXLoWglYDIVyPyhWt1Ei+OKXV9LpscV5kLOZxS4DEIAVhxK4dF2fkQf+UEcbRdMlCyAg8BaEtxq9ZB5Xjqvw5tINsbh1/JvYLOIiPhy5KdBQTq5Nj2zj1h46+eTpQ/tOJPB9N/OsnBSE2rqUmH9i3KdfxrvnOGzWWH3N/Kz034qHF8lgfDBeZCbjOb872R2/4g9UeOpazpEcHGuBKMaVeqncn2jBiIqFdBpNYT6GjmcaOKHHa4nINvtsOtcJvpq1YkcO1/uZLPjJZuw6W0Rtq54RAbb+URIFsAnUvw39n4vM1m6Pwet7wF3HxGwnt4AHR4Uy3avcJk9s+srecHz2+Su/vfPy3xOytN6spRz/KuJFbze/bLv0V6ysHgZdIxr7A0eo7FrFJTMOAmCFk6u+KRmd0g7r9UsGY0Dc0RsazWLi+zAj2DlY6JPyTwLv/5PuoLWPF12jOjWIsoNqCGBQXRb7NU7Y5g3RkpR4+bDnJESuJSg7P5W2oGHTIekvXJtjq8Wm/w1T8O+H2QQn9ZQMQhRNDIjZ9bIiu/HXCjP7/6ieLCkn5IM1u5vxHLer6osiOYi154oIAtl1jlph3bc2Tsw+pKc5cJvpQSLzU5hXjZK3b7ys1I+CAE5n+UPSfBls0pwuOQ+GL9UgleNBgqzwVIogUl+OvYL+7Fp3bCFN0XxCkFn9Kr09SulIEsEw7aS8uGZDeAbJefhFS6BkUYHRm9pATfliCj64DxSe33jFIQ4yCow88rq03zSXY+P0Ve6qiorM+76Stq2q7YX4baiBZsV5fw2wtbcxe8NP2LG6aq8Wj2acHO+ZIxUVK5j1EBExSXpeSZScorIM1kq3Sczv5ik7GKs2my0Ojdw94Vz2yWVvPopSe2vLBGCRraETo+IQdjYn0R3cHaL2J2vf1kEq4M+gVWPSwZFUaD9g3D7Ykg6JItOQA0YtxAWTipzPNW7i9Yi9YS8XtZ5KSVUaStaFFceIUF1OJipo2PNAJ7tW4u03EwOZHsw6ND9Ejhlx8ud+Ka3RTtRq5d4lxRmw/IHIemApNcbjYDh35Ys4qdlMGCPF2Hx3fI6RxZC3UuyBfE7Yd2LUmpqOhaKclDmjJHXjBkER5Y4BSFlF/usPPfwQlnARn0Pc24V07WfbhddgX/1is9zzBGqjMQ9Mt8nuK6YrX3Ts8x6P/kgHFsOk36u/PkgWR9X05bzU6kZUnkg4G3Q4WXPBW0Q1O5dNkvowBwxtwPRowTWlPc6dp4Yxs0aLrb8Wedh05vQ9n7Jnm3/FAXQAlqNlsJeb2OqNxBPe74EMnYbxG2RjFLNblL6MV6yiGfFw6l18vP641DJdIB83qZcmcGk95AAL+WIBCQ6g/yc2a1sT6ykowz47XQmWb0b4NPhYchJgKodnPRHZedwXoKd0IYiwK43ULQ5gCZ5Py27FPP4qjQOtIjC26jH6/qX06jc5KiBiAoAVpud9DwTNrsdPw83zFYbx5JzaRzly8GEbJfPaRrth7e7DktOCtr8i2AqhJRY6YQZNx/O/y5iyOg28kd80RRJpSQdFA3A6V/FjTU7Hro8VZJNKCkD2e1SJ98xTRxLm08QjYjRH3q8IB0DflUBuwQlFpO0/B5fJc+zFsusl7m3Oi+SBh8KB32JX2EkbWtmMXnWQex2WHBXC1i7X4KJfu9KeWTUTBnAt2hyWZdP58clG3F4oQg08y7KtN5Nb4kGQ1HKAiCrmVIn1/Ik7YeuT8Ghn6DBCHn/INmSnV9V/iGd/lX22fW1zOC5dbboR0bPkqxT0gHY+YWLJ/5RV4oi1/iXl1wHFPG/iwjYEdBENCezxkBQNPidX4cSWNO1SZvBhzCDhabRvuyPr/gzdH/7EELyd4GxWESZyYfAK1RKTc1LJj7brJJFUxQpJwXVhpo95TNY/YR0lVhMsP1TCVTr9pegIX4H7msfwxZWDxbcIQFFVEv5rPbPkuf2fAWajS1rm865IEFih4clU+QIQtz95RqfWidjC0w5ZT9jIMHCwI+hVi+KT1d+le12sBflwHed5TWbjZf3uvQ+5+seVEda0MObSunyyCIJ0tK6w8Y30eVfxNPNjYV7E2kQUZ8ryPmoqPynUAMRFZKyC1m4J4FZv5+nyGyld/1Q7utWi11nM3ige22m/LC7gn1BrRAvDDoNdUM80V1A7t62fiwZhcwzMO92KUvoPWSYndNdvp3SxfHMBqjdV/7YO4KQ8ljNEow0HC7ZlPFLJWDp97YskDu/khZejU5ad2/5QO6qDV6ieRgzRzIvWeewhzXCWm8Qb2/JZ8bOneg1GoY0CuCB5kYirEnQ61X49WU5XnBdmDNaygMOUo/BortEiBu/UwKIU+tFsKkostIU5UhXQ1EWRDYXh1RXFKRDUa7cGTuwFF1elGnwknIKSPdE67tlwF5RttxBj18G3uHO/i3ppyGsYeXHjGolwknvMNfXH2DH5zKIcMEkknt/xvoUL+Yczsduh1HN+tPbGEx4uwdFs+LomgmsBf3eItCawrThtXh93XlWH03FZpdy2H3tQhgZegG93d25bJR3UYK6+oMlINj8nrxvm1UC0V6vSCbju36yf/uHYOMbUhazmmXRNheK/qbDQ2iOr5QOqq0fQcJu0WWM+l5KT7+8IBkvS7EExFo9dHlCAoxzWyXr0elRyYid/lUCDq0brHve+fpknZes1MSfaV8zEHD9mTeL9sHnwmb5OclLkeGStXpAl6fFxM5Bt2cle7PknrJth+ZLKa7T41g9Q8gvTsdstXEoMZvogOtHyKui4go1ELnJSc4uYtKMXRxNKjPRmrc7gYMJ2TzRty4L9iTw7YRWvLPmOEeTcjDoNAxoHM7w5lGE+xqIDPCkyL0/7pYsNG3vlYWj6zOyCCfsdv2i4U1k8QCpbxdly7TdyrDbZHEoSJM5NOOXy0Jc/u7fZpFt+SniR1JcIHfXs0dClXbgFYoSuxxdcAyPGndx17juYLcQcOJ7jD/Nl/ZU3yowcZ200Wq0zkFI6bmUZGpa3CGmaSCLisFXgg/vcCkVrHxYSkarn3L9njwC5V/fKmXbYldAo1FSqnJFzCDxTAFZbM35kgW4sE8W4UVTYMws6YRxpPw9g0Q70vFREcCWx80T+rwhZY3qXSq//jmJkJdM8u0bmTjrMEeTygKdFy/k8MOuJH644zHCG4+Q0oZGK0GVZyiKVzBhRfm81TeMJ1vrKSy24U0eIYfeQR81HNa+7vo1jy6FsT9hjRmCNqWkK6tGNwkIGgyV0llOglzHJmPh6BJpeXVwYZ9okoZ9Je9/60ey3W4r+/zSTsr3yYckkPQIhDO/QVRz2bfvG2KW920f+b7/O7DlQ9fna8qFU+sIVXy4tXkD5ux17hYz6DS82j0Q/1WfOD/v1HqZyaQzyHl2eEg6hfTuEjCBBCGxy8WBeOw8DqS7Y7XZ6V4vhNikHPo3Cq/8s1NRuQ5QA5GbnIMJWU5BiIPY5FyyC8x0qBnEW6uPMbV7LaoFeqIARr0GH3c9wd5GAHReXoCXtL52ekRKB33ehO8HVRxu12pyidupVf741+4l2ZM+r1V+kjqDBCwDP4L4XSJm3TvD9b5xv0G7+0VTMOI76cA5t00eC2+C3TcSb78zeCeukVk47e+G9iUlo9xkKU9odJJFqYwL++Ru3YHRT4KCwJqS+dEbYMJqaYvNSaz4/PCmkqloNELuuv2rSzYiab+ce9X2ZefsoOFwyRY4ygXRbSQYKe/Cmh0vYs4B70lQZLNKycYjWFpCo1rCnu/EuyK6lbTkoki5J/WYlLocVvnl0bqBX3U2n0jhaFJehYdPpeSxbn8ct8coKCENoDhHrqfeHUy5aGb2x6tWL7zaTZVrnH4Kmo8Fz5DK3XcBe3YCCW1fpsqu11HqDgCvEAlyfhgs1yGoDkS3x270RSkfhDjISZTSRoNhImLOPCvbE3ZJ9stugxkDyrI4WjfZ7uYN1buKSPTQgrLjeYXK9YlqJT4wHgFy3fb+IO8j/RR+ORd4vJaBLlUaMn13Dun5xbSt7sc9zdypuvlx1+836zxMXi+vB1Cci7UwB/PZnRhTDkpmsUR3ZTv1K6syR9MgwgdPg46qgWo2ROX6Rw1EbnDsdjspOUUUFFvJM1m4mGsi1MdIqI+BAE+pM1fGKyuOsvbhznStG0xGfjEaRSHQy41QH6PTfjabncSsQn49VsDBBAvDG95Naz9PtLcvRtk/W/7we4XIYpidUJZJ6Pu23P0N/wbid8iCGL+z4ok0u13EenGbYNCn4r/hqnPGQe5FCUCOLIa6/eTfgR9DXgrK4rvlzvOWD+D3L0RQajVDRHO5Azb6QG5iWcbCFUZf6dQA6aYwZUtw0fUZSadHNJdMRdX24gdyfFWZNXeVttLqnHRQSgzuATBmNiy5VwKRZQ/IebS5V56n0cmk4fSTEiSBbHMEQpdmbVKPySI9/w75vs8bcHiRaFJ8o8XV1WaVElBeqmQufn0Nuj0DA96XcpRjYXbQ9WlyNN7M2X+h0ksy70geA/Wx+FdvCt+UmNlptHDrPAm29s+Sr8ajpXX1wj5575ch1xDGkhOFPFR/oARJG96Qz8tB2gnY+QVKZW3aUPIzUBLEZJ6VgKTlRPl83QOkOyXpgOxrLYY1T8GkX6D7s/L/8uSliO4oIw62fihBVUQz6PsmHF8p5aj4nQSuuZe+ATVo0+0tzH418NbbcP+mS8Wg3IHRVzQ/ZzaS23Iq8dWGM+vARS7mDWF0s8nUDjLgU5REQIsJWE0F9GkQhk6r4evNZ5h+W4vLXkMVlesBNRC5gUnLLWL32Uw8jTreWXvcSXRaK8SLb+5oSZhP5ZJ7d70WOxDp70Gkf+V3XrHJOYz+Ykdph83CvYk0jvRlcfvTaAszJQCp0k7aMy8ekVR0vVtkMbRZRYAa0kDsq9e/BidKFm6dQYKQiGawbKps+2l8yd2jUvncDY8AyVDkp4o+otersH8WFsVASsf/URAQg7vdRHGzJzDVvQcvCgk58i1uMwbAnatllk1068qH0zUdK2JVr1BJn5typetj0WQJkqxmWXQWTIQ2d4tRmN5D3o/NAlqjmIAVpEs5yTdaOoZyEuS5WjfJyFjNEpCcWAs7p8trV+8spR8UmVZ8Kd5hUuoq/aB7yAK58Q3ALh0YdfuLn0lIDHa7DdPA6djTTmIIqoZy9yaUbZ9KucLRspqwGwqzcWUO6kBREBO18tfLZpXsWKNRZR4mB+dJKaj5eNDosVfrhFLeL8aB3oPzuqr0qxcAPi0kw1M+CHFgNZVlElxht0sQUm+gBHT+1WTRX/+ylMVaTZL3WL6E9vvn0kaef0k7ss0Kx1aVTYwGCbLn3yGt1X7RsO4F2Z5xBv/fnpegx2aG6p3kc7wUjU5+Xg/MoSBmJCvpxNPT99I02o9He9Vh8b5Edp3NwNddz90dHqZVhJ5Vv1wgzM+Dz8Y2r3BToKJyPaIGIjcoOYXFZBeaaBTly1MLD1bofDmVksfdP+zh41ubMXO7i3Q8MLZNFYL+oDcwNdfEg3P2VWjzPZiYzTnf1tQ495LoOkA6HkIbSnrblCNZEM9Q0ZOkxsKvCyRb0ON5ueu0W+W5jiAE5A4+LwVq9oJTLtpK/auVBQNV2kJOEnafCDJqDWdxQRO+WpHOa0M0LNybzrqjF7Ha7Bj1Gia2upeJ/fsTtOF/4vy693spcax63NkYrVpH0SRknZPSyMLJzjb3IAtb8gHx7whvKoHKxregdg/Y+gkMniYdGmnHpEzVcpJ0Trhqsz2yECb+DHV6g7kAq9WK2TMc48ZXnDURDlpPkcwDiAjSO1yEt1EtS8SwnnIH7sBu5+nl5zmc6IW/RzaT2oTRu/tzaPJSJFNwbjvEDMRHyWdcIw92n3PRWgyMbeiF/8llUKOFc5B48mfJFMUuKyuN5FwQ19qUWJTer0oZrXzJQqMjbcA3pNu8aOmliNtrZXqjc1slI3F4gevH6w8RsW3yARH3FmZJW7LdLuWwn5+TzpVWkyVoAvlsNW5QrRPsP1d6TgRUq5glcbDpLeh+iYjVO0x+NvbPkhlKF4+WdUiBBLr93xFRNZDa9H6enZlIgKcbD/Wozd0/7KHQLD97CZmFPDj/KAMbh3NX51oEeroR7nd5rxwVlesFNRC5QckqMGOywMWcAracSne5z7HkXCxWO2NbRzN7p7NxWf1wb4a3iEJ7udtgICPfxOnUfJeP3bU0hXXjV6BZPlWcRNNOyqLc5WnxiajdW+78wxuKCPGW9yXdbimGBRMqll+8wyVIsRZBzxehIFVS/A58oyV4WP4wuPuTHTOWPKuOYLJYkFSTN9bH8VTfunz522l2nS1bUIvMNqZtS8bevioPe4ZjiG4Da5+R7MWtc+W8i/MlCPEKlUDHzUvuoi/VgIQ3lUxCQYYsZr++Jgtex0cgcZ84yOYmiwOtRg+dHpPW2F9elPbgoNqSKTm/Q7ILdrsIFVOPQ3E+pzu+z/2zEpg54EEiLh4qC4K0ellofaKkA2nCKgiJkVITyF23CxRFYXTLKJbsk/ex62wm4b5GHuoQzJDG/TH4VZVyVq9XaOcXQNNoH/bHO7fq1g/3ontILmw5IiWP8pkqu02yD6NmwomfpQVW71EyndguWaNb52BLOwXntlHsVx1zjT7YDMG00lnw8C4574BqknUozpNyi0NDk5ci16tWLzl2eXwipDQ2e5R8lkvvkxlCnZ6QOTuOAHPvTBFAB9aW6681Somr7b0SCFqK5LNJuczAx8yz8hmUp/UU6eoKayzZuXE/SXv76Q3yOcXcIgHMibXgV4WDadJGP6plFF9uPlMahJRn+cEkhrWIonrgZaZGq6hcZ6iByA2GzWbnfEYB++Oz+GzDKR7pVeey++cUmXm8T11GtIhm1u/nyDNZGN48isZRfoT5/nHa12ytfFTR6dQ8Dpga0mz4d9LNUpAuJYusBAlCDs6FNvfJneiJNXKn7hkCmedEiFg+Q+AZJNNpVz4qFvHmIjE8c/eXP/J6DyjKhBWPYg+sRVHvdzBnpxK2402SWj7Bp5vy0WoUaod689Ya1wvKtztTGDtuItFuXtJxcXyVtBd3fEwMyLIvlHiZ+EoAMPAjefzEmhKDs1GSBfm2r7Ovhk+EtJNGtRSdR+qxchdpPUxYAbfOEQFr0n4x0+r4qNwpH18lAdjZzRREtOeL3VmcTMlnzFIbK+5Ygk/BOdFieIbKHbjVLAuc259fqGqFeNO1bjAbj0unR1J2EU+viueDLSlsuq8xxtE/gs1MWHEhn/f2YkdmGD/ukVbcsQ3d6eCfRdjyO0S34sqgKz9NgsusczIgz1osHTxZ56VbJ/kQmi0fQHAMxuJcjPtn4t3t/2Q8wPilgCIZMqOfXPvuL0iQOmeU/Bu7QjQ/ZwfBgVmSqardS15r+YMShCiKZJ7q9JHu8REz5Gdx32zRz5hypGyUdRaC64uvjdVE5l17ScvJ52KuiQA/f4L6hRGy+fkyQ73yKCVW64oiwXZUGwhtJJm9jDOw/lXwjoA290iXVV5SWbZQo8NkkbJW4yg/Pt90puLxSzgQn0W3uiF/+vNVUfmvowYiNxgpOUXM/v0sQ5pG8twtMYT7GhnUJJxlB1x3JwR7GQjwlK+m0X7YsKP7C7Mr/D3dqBXiyYBGEdQJ9SbPZGHp/kS2nU5Hp1EI9HKHgED5w+seIC24ikY6Kto/JNoBx9hzswkaDYdzO2Twm8OhFEQX8euroi3JuVAmeNW6yYybxiNlsR/2JYqbF8acONznjgK7nbzmL5BrysbfQ8/F7EpstZG5M3luwWDOEhfPzHMQXEfukE+uFQOzhiOgdk8p+1iKpLuidm/JlLh5i2ah5Z3ynux2ufNvMUHec9xm5yAEJIOgaGHZg84DAnd8Brd8JItllbYU5aaxM3AYi7aew89Dz3P966EzuoF34789gTXY28DbIxqz80wG326No6DYyoBG4QxtHonRzwP8QuTcrDbCinMYYsykez9PSPgdn6MlC3+/t8Vnw+HvUZ6298r7a3e/dD8dmCMLdJcnJUBZ9bgEFOVLXPlpJXqXHMmibXi9bDKxR4CUQcYtkJJNw+GyPaqlCIR1BrnWhxeUdRkN+AASd4ujr0PH4hUC4xZJ+ejXV2RbSIwEMTkJJEX05ollp8plFM9RKySMrwcvotrSYc7BSI2u2D0CUIZ+IV01niHOrq0B1aUsufl9MdlrMlZ+1n2jRMCddY6mYW6lu2s1Clab6yDfqPsX5+yoqPwLqNN3bwDyTRaSc4pIySki0s+djAIze89l4mnQEu7rjtlqI8zXyOSZu0kqtxB3qxvMB6Ob4ufhdpmjX57cQjMHE7N57+fj7IvPIsDDjVEto2kQ4cPp1Hwmd6qOp8FFvGu1yPC3tBNStw9tKIvU5nflsfYPSLZh/SuSjr91rphGjf5RFhNX1OoBQXUlm+K44wbOjfqFbj+koNdq+GB0U+6b5XqgnaLAhofbU02bCceWid9JYE1ZYO12Od6G/0kAUn+IBEZ9XpeUvtUsmQmNBo6vkVZmlLKMyZi5sPZpyXqUp/PjUma4tF0XpKNo4hrsNhs5eBJfoMNuLSbQQ0dIRDV0+iv/3Coju6AYi03cdS9blrNapSxlKwYUaVtWNNL94wi2jH7y/nISJevV8REJ6jJOlTiKTi0LLi7l1rliOlatg2hIyk8T1uqh/7tSHtv9jQRCtXpKALDmaZks3HC4CIobDIEzG+Vny+FdU57J6+G7vvL59X1D3sueGeTVHsyTF3uy6kjF6dE1g72Y0ymFkFUl84i8w7CN/IEL2kiiIiMr7O9EVjwc/AkOzxe31uxEmD0CLCay2z3Juzk9ses8SMk18fNRF1kXYN0jnakdqs6XUflvo07fvYkoMlvYfS6DMyn5dI8J4dklh9l8sswczF2v5a3hjVh54AJfjW/JwE+3oFEU+jcK59n+MX8rCAE4fCGb2775vVQWkJ5fzPRNp+lUO4i3hjd2HYQAaHWywPtVcd7e6XG5m930ltxJ37lKFjlNyewNV+29Dk7/Ku6TUBqEAASeXU7vev1ZE5tObpGFKH93EjILKzy9d0wwgdpC+K63dMMc/1nuZEG0CL+8JBqCerfAiXUw+FPRfZgLpL04+ZCUkNreJ22hZ7eAR5C0Eqcddz0NOKIZ/Pau6/djKYLUEyjRbfD9uAm+Gp0MAuz0KFyFIATA98/+PGi14F+l4vbbF0sJzmqW8lB2omQw7tooweHhks6XwkwxaNvs4r1X7yIdVtU6iC9M+SAEZMBf7DIxA3PgMDAb9LEEqmnHxfhszhhpj3YYmpWn5STR0Ez8WYKohF2SnQHSuk9jzQbXg/5Op+aREtSW4Ca3Yo9uhzmqLQuOm0gszOXJP4hD8IuGjg+LoZrdDoFGmLIJDszFN2kXD7ftwa4iP3w93dl3PqvC5OKp3WoRcplONxWV6xE1ELlOuZhTRFxaPhezC/H1cKNqoAc/7jjnFIQAFJqtPLHgINPGNScjv5j5U9phw06dUO+/HYSk5Zl4YekRl120m0+mkZFfTMRfVfb7RkLTMbIIWYtBX6J1yEmUYMRVO60Dx4kozqUlr31f8cKIwSRke/PJryd5fUhDnl1y2CkYaRbtw0s9I/BeebeUIX59VTQgphIhblEWhDcWx8/sRGnHXPOMtGcumFj2YvlpksWJGSSttilHSmznSxZeRxurV6iUcLzCLn897FZZ0O/aIHoYr9DL28Bfa3wi5MuB0U88VbQ6CSwzz0mJ5PSv0prd+zVx4y3MlJJKw+HSXrxwsljWO1xVHXgGyXUoH4Q4yEkUvUjd/hKodHhIsicrH5NurPL6lYGfSFC56knR5XiHS+lowiqYOYB8i4ZKKiMAnM3TstL4EHGx+axffBqz1c6MO1v9uWuk0cr7KCVSbN0tJgL1HvTVaskqMDH/3nZsOJbCz0cvEuTlxoT21akZ7Imv+9UJQlVUrhVqIHIdkphZwIQZu8guMPPy4AbkFJrxcdcz+/fzLvc3WWycSsmj0GylTbUAIgPcrzgIKSy2UmwRV9ScQjMnUyo6bTrYcSadhpG+lT5+Wfyiy31jl66biKbiKFoZNbrKNFk3L5nAm7hHtpsLiFg8lO+6vcMFzwYkFhTy9bhG5BZDemoyVXwUQoriCNLnS3ZjyDQ5hrlQshk5yVJycByvw0Mynr7Xq/Db267PJXYZNBsn7bnHV4t4sfEo0S14hUHXp+G3d0S86NAJuCK8iehOIpv/xQv4H8E7tOz/vlHSOWLKkxKWR5CYv9XoKpONC9Ph2EpYcKdkVM7vwB5Q03lkX9WOrv04HMQuk6GIscuk7OMTKaZvjsF2IGJnz0CY0a8seC3MhKX3y3DFwdPx1prRa5VKxdhVAz35fONpDl/IKfneg3phf6NconMrKecJfh4G/DwMTGhfjVEto9FrNbjp/rx2669it9vJzC/GarNjw45GUUqdk1VUrjZqIHKdkW+y8MbqY5y8mMeDPWqx43Q6Hm46GkX5kl9cucNkSq6JcB8D1YM9CfP96/4DOYXFJGYWgqKQb7KQU2Qhwtd4WVGdn7ve5fa/jE+43OVGtpTOiyZj4MBc533cvEoErlMkczL4MxlQ5xi2V5RN8Jq7Cb7lA5rE74KfV4mja8JcWQj17uI02niMZDRyS9LyNbpB71fLvgcRlxZlg7tf5QEESHmo6zMiXCxIk46P8UtFgPn9IOnAsRZLkLJsasXnNxkn/hoBNf/O1ftv4RHo7Fqrd5fPNz9N/D2y4iU7Ed5MPhedmwh3L3V7rZRyP4seQWXmbu7+ZdtbTYb5E1wb4u2dAa0mErxzJmOa3coPu1Mq7NK8ij8Hzmfx+rBGjJi+jc51gnluQMwV/V79EYqiVF7e/AOsNjupuUVYbHZsNjs+Rh02FLwMWnQaDTa7HZ1WQ1aBiawCC/GZBeyMyyDIy0D7moGYzFY0GgW9RsHDoL/i81BR+SPUn6zrjPT8YlYfTmZI0wj6NQznUGI2b64+RqivkaqBHpxLd20jXT/ch+gA9z80KHNFgcnC+mMp1Anx5skFBzhSchc4sX01+jQIZdWhirV0rUahdfXLW3j/JbxDS77CILS+lDm2fwYF6dirtkeJGSgajoIM2f/n5+COlWK9fWGflAtq95ZuFodOQWeA1pNh5ePigGrOK9UIlHJmg2gOer8md+pQYiOKlIAqc3htNEK8Jza9JdoR73A5d/dAOLlOghAoMQ3bKtNtt38q33tHSNmm3i1iR6+/Ce5MPYOk6ymiuWhjDN7g5oktLx1Gz0a7+C4JKs5tlc/C8RleSr2BUvYxeJe1dvtVBZsFe48XUDa/L1odh7maK5IO4p6wjQc734JVE838PQmYrXbpyq0dzPj21Xhwzj5ubR3N7LvaEp+R/6da3f9NkrIKOXwhGwWFebvjOZOaT50QLyZ3rs6plDxqh3iz5kgSjcJ9aVLFjwfn7OVgYlnLuVaj8N7IxjSv6k+eyUKhWUqiajDy5yi22MgvNmPQafFwU6/ZH6FeoesMu93OvCltKTRbSc010TDCh061g5jz+3nu6VKTZxZVdOesEeRJsLc4Meq0fz29m5pnIsLXnTu+20laXnHp9rm745k+rjmHErOJzyjTWygKvDeyCVqNQkGx5Z/9RfQKBoJFd1CtI+ZiE9gsaDe/g8bRrRFYSzo0CjOlG8W7ZFjZvHGS8gfRb+QmQU6S2KhXaQNzx7p+zZxEOZZfFclyWEyyyMXvlPbdMxuc9w9rDDV7SNbDEaQUZcPie8rm6pTnwBwRZTa7TTxL3Nyx56WhGHzB+ybzi3D3c/pW6xVITng7LLdvQJN3ASMm3Pyj0NTuJQFdebzDof5g0ewMniYCYq0eer9GTrENz4aj0AbWwq4zOpd7LsXNE0w5BB/9jsZVX6FbveaYrXbcdBp2xmUwdfZeCoqtLNybSJvqAbSpEYS7/r/xp9RitRGfUcDWU+nYsPPC0iOlj51Jy0OrVXi6b10KzTbGtamKBvh661mnIAQkm/LoTweYd3c7/D30ZBWYcXfTqoHIH1BssZKWa8Jis1NsleBNr9Hg4aYlyNuAolz2J++mRf2puo7IyDeRlF3E/bP2kp4vAYFeqzCxQ3Vqh3hzPDmX14Y05NNfT5GcU4RGgR4xITzeux5eBu1l58VcjvMZBaTmmpyCEICCYiuPLzjI60MaYrLY2HY6nTAfAy2q+jN3VzxPLTzAE33q0SMmBKtN0sCeei1GNy3exn+gbGP0LUsWdPs/MYqymsRbxGaR7Ea7+yUAKN954V8N+r0jokGtG7l2Ax5mE9pLPT7Kk3xISiRZ52HnF9D7dfj5WQks0k44O6y2niy+F64yJee3i+D0UnISJXtSszv2dlNRqrS9+YKQSvDx8cHm5c3FrBDSi834aW0E9H0bpfE+2PUlFBdIhqx2LwkuR/8I+37AFtYY28BP2ZXuRo7VnT6KDU6tx96qBkpUS9e28Vo37CExKJHNsTa9nR37sliy/6jL87JYbXi46Sgw/dmy0dXnXHoBp9PyCPJ244n5B0u367UKi+5tj8VmZ97uBLQahbY1Agn2MvDTrniXx7LZ4WBCNkadhkZRvhRepvSrIiRnF1JktpFfbKGw2EZ2oRmr3U7tEC8sNhs6jUKwj2rNfymqj8h1QLHVyvn0AlJyTNw5Y1epA2N5XhncgO+3n8PLoOPW1lWI9DMS4OmGn4ceo06DxQ4hVyg+2302gxUHk5iz8zwNInzRKOCu19CvUTi+7m7EpeXTu34IduD5pYfZey6TYC8D/xvWiG2n05mzM548k4VgbwNTu9WiTfUAfNz1f72j5q9gs0n6Pf0MdndfyDiNkhkHIfVFO7DxLTi7CfyqcvqW+VTzd0P7TQ9pPXVFjxfh6JKySa0xA6X989R6qNFZ9CWJe8C/OlRtB1/3cH0cz2CZIDynEi+U2xaLTbznP1jWupEpzILiPKwaAzabjUKbhqzcAhKyTXh4+TJv70W2nE5j3pR28vOWFQ/JB7D7VkH5flCZhggklTdoGuYa3bHnp5HlWYP9CTlM+WGPy5ce3jwSmx2GNoukc51gl/v8mxQWW3hh6RF6NQjFarPz1MKD1AnxptBs5YVb6vPjjnMsP+hsbPhQj9oUma188ZtrJ9d7u9Qgu9DMLY0jiPZ3J1q1lq+U9DwTaXkmCkxWzqbn83+LDxMT7s1zA2LwMuqx2+14uGnx99DjZbzxO59UH5EbCJPZSkpuEUcv5HA2vcBlEALw/fZzTGhfjeeWHOZgQhY96oVyV+caLNiTQFpeMfviM3midz061Q7C3/Ov/RJE+Bnp2zCMrnWD2R+fRcdaQeyLz+LDX06SkmuiQYQPjaN8ifA18vqQRmTkF6Mo4G3UU2S2lg7ES8018eKyIzzYoxa3NArnl6PZFJptNIr0Jdjb8M+mfTUaCKwBPhEopnz2WKpQ1a8x/tveQHt8GegMFDUaR0KjqaRb3Knp7gXtHoD1L1U8llYvM0vc/eHQfJmmGtFcsiymbCmrNBhRNuukeqfKzys/VTpx2twLv093fqzjo1JyUoOQP4+7H7j7oQXsVhtpGQWczrOyL7GQpfsP07VOCHPualsW9PpFg85IoVXBOOkXlOOrUM5vx+4bDc1uI0/nx++JEJtsoFFkHpkFxbSq5u80mwjA30PP5E41GPLZViZ1rP6vv21X5Jks9GkQRrC3GxqNwutDGrHvfCa+Hno83HQuywIfrT/JV+NbMOt3XYXBlQDtagYxc1scOq2C8gdzp252CoqtmK12TFYbjy84yFvDG9E0yo8L2YVYbaK70SoKGflmjHrdFZXJb1TUjMh/GKvNzpnUPH7YcQ6tRiE9r5hlBy643NdNq2HRfZJ6tdhsHE7M5t21J7DZ7bw6pCGP/SR38i8Nqs+4NlXR/4VfgovZhby15jiL9iVyf7danEnNY/XhMoGqt0GHHTtf39GStFwTiZmFbD6VzsGELF4d0pC8IgvPLjlcur+XQccHo5uQlF3EC0uPoFHgiT51ubV1lb/tbVIZKTlFvLL8CA0CoF2UHotdYUFsIWjdeLxXTYJ8vcWye/XTzmPe9R6SwQioIdqBg/Ok/BP3W1l2pOFwcXN1OHf2eEH8LC64cHDVGcX06/BiqNMLkg+LaLZGVxFYXmrwpvKXyS0yk1tkQQECvNwwVGKJXlSQL34hNgvFdg12FPw93MjIL+bkxVxMVhuPzNvPm8Mak5RdyNL9FygottK5dhC9G4Rhtdv438pjzJrchhCfy2cb0/NMFFttGHXav3wjcDmsNhtajQaTxUp8RgG3f72THya35umFh5wmJSsKPNMvhqMXcliy33lQ4+CmEdjtVPjbUj/ch/dGNeGXo8n0bxxBiLcb3jfBnfyVkpiZT5HFzoLdkgHuEROKj1FHdpEFk9mGooiI1dddR/VAzxs+u6RmRG4QTl7MZdj0bRQUW2kU6UvP+i60BSXUCvFi2+k02lQPZMneRM5nFJBnsmDQadCWuxN6b+0JesWE/iW9yJ7zWSzal4hOo9Ciqj+fbTgFwNjWVehVP5ScIjPVgzxRFFi2/wJn0wtoXsWPJ/rU5bWVR3imf31qBHtypmRKb57JgtlqJ6zkj7fNDm+tOU6Lqv60rh5Y6Xn8HUJ8jLw4qCFnUvOYsy8RvQZGt6tN1QAPAhydRF6hMr23y1PSSuruD8F1ZQT8ybUQ2UZahw8vkG01u0srqHsAzBpe9mL758DQz2HWiEtS/xro+5a0F7eaCMV52GMGorh5y539zdAd8y/gbdT/KQ2S0cMTV1fc06CjoNjC2fQCisw2Hp63nwYRPvRvFI6bTsPec5ncOWMXX49vyZfjW1w2CMnIL2ZnXAYfrDvB+YwCaod68WTfujSO9MPnT7S3Zxeasdvs+HroURSFhIwCTqXmseLABdzdtPRrGE6wt4GjSTks2JNA3TBvFu1NdApCQORK/1sVy3cTWrH84AWnlvu0PBNP963HiYu5HEvOxajXMLhJJCNbRnEwIYtbGkfgplXUIOQPMOh02O1WzmcU8kD3WuQUmjmalMvba4+RUyjZpmAvA8/dEoOX0USEv8cfTje/WVADkf8oWQXFvLjsCAUlArFDidk80qsOXgbXKdRJnapTJ9SbD34+zqhWVXhk3n4AetcPZevpMrfVXJOFrEIzBr2JC5mFWGw2Ajzd8Dbo8DTqMeqd7x6zCor5erPUj0N9jJxOFdHn87fEEJeaz6SZu+gZE0q/hmE8Ov9AqT5z2+l0vt4Sx/cTW/PzkWQe61WH+2fvKz2uXqtQbHFOxn2x6QwNInyvmjI/2NtAsLeBNjUuE+x4BMhXSD3n7c0nlMxWsUPLKdBsvGQyNG6SSZm4VoIOmxUMXuARAhNWikFX0n7JdDQahd3NG6xm7Bod+YZwjH5h6HXqr+F/DX8PN44m5ZSOAzhyIae0bR3A111PtSAPqgd5VXqMgmILs3ac4711ZfN0MguKOZOaj1GnRa/VEOjlRrCXAcMlv3cpOUVsP5PO99vPYbbauKtTdRpG+hKXlo/FaifEx8jsnef5Ycd5xrSKZkzraLacSuPdEU14ecWRS0+llO1n0mlexc+p1NSqWgBebjqGNIukSoAHVpudlNwiZm4/y4R21fA2aAlSBZZ/SJC3geTsQnrEhJCQWYCHQcdz5TLBIB2Ij/10gO8ntiYtz0ToH2TSbhbUv4D/UcxWG1kFZqdt7687zgejm/LayqOlfiFeBh1P9KlDkyhfZmw7i5+nG0eTsik0W/Hz0DOyZTRTfnDuDtBpFLafSqNOmA+bY1OJSyugTfUAWlT1x89dR4BX2S+HxWovPQ9/Tz3hPkZiwr1x12spMFsZ0SKa29tWZeQX2yo0iRSZbTyz6DCvDWngVA9tXT2AoxdyaBrt57R/UnYRJov1v9kiqCjiDOoK95K0o91e5jECYC4SG/MGw7ArGuw6I4XGYAqKrbjptPj+U4ZvKv84IT5GovzceW1IQ+6btZfb2lShe0woWo3CkYRsAn0MfPbrKZ7pH0NgJd48aXnFfPzrydLv64Z681S/ery64ihxaZIdNOg0PNyzNqNbVSGgpGRzMaeIB+fs4/c48cS5s0M1MgvMjPx8e2nnWtsaAXw8phlPLzzI3F3xDGoSgZebDqNeS25R5V08WQVmPMu10wd5udG9XghfbDnD3J1l3TMfjm5Kel4xQd4GNQj5CxjcFNpUD+BiThEf/3rK5T4Wm53lBy/wSM/a//LZ/Xf5D/7Fvzmx2+0kZUuGQq9o0Gs1fDy6CWa7nftm7eN8RgGHE3N4beVRJnWsTt1Qb8w2O+G+Rk6n5PL5xtMMaBxBck4RX/92hvFtq9KzfigvLz9KkblM4No4yherzYaHQceAjzdjKUnRLtiTQJCXG7Mmt0GnUfDxkD+uPu46OtUOQlHg8d51UYCXBzYgKaeIfJMFRZHUruM1qgd5EuTlxvmMAi7mmDidmoenQUdqjkz9jQ5wZ2q3WhxPysHXXe9ko92qmj9e/8Ug5M9yqRhQbwR9uDxU8uUJeBrUAOR6oGawF7kmM6se7MjaIxd5fP4B0vJMtKjqz6O96lBktpKcU1RpIJKUXehkEf94nzo8PG9faZoeZPzCW2uOE+7rTpc6wWQWmDiWnFcahMSEedMo0pdHSzReDnacyeBc+kGe7R/D1Dn7mL8ngSHNIohNzqFFFf8KpRkHbWsE8Paa42gU6FYvhCf71OXUxTw61QyiX4MwTBYbCZkFGHQKrwxuQOTV7Gy7AfF3N2Iy5eNt1HPqMuMvTqfmU8n0gJuS6/iv/o2DxWrjfEY+Oo2G+Mwitp1OI8jLQIdaQfjptSy8py1L9yXy2urjnEsv4IWlR1jxQEf8PfRE+nsQ5mOkS50QDHotmfnF9IwJIbOgmGHTtpFnKuv9D/Y28N7IJigKPDBnX2kQ4iAtr5j/W3SYD8Y04fezmRQWW6kb5s2dHaszvn01Rn2+nRduqc9PsfGsPChi1VBvA4ObRNIs2o/7u9ciLi2fpKxChreIwtugY/rG09jtYLXbeW9kEzwNWjIKisksNDN7Zzy96oszq0Gn4Y721XCrRFioonItsNnhtZWxrI8ts3rfeiqd7ae3M3Nia7IvyVqWp7xItmawJ2fTCygstuLhpi0tuTqYuf0sDSJ8OJWSx/zdCTSJ8uXxPnUJ8zFyz48uRM9IBjG70Eyknzu5RWZGt4rigTn7+d/Qhuz9YU+FoX01gz2J8nfnxUH1Adh+Oh13vZaaIV6czyjgdGo+Uf7uhPm4UzvEi+2n0nB30930wUhSVgEFxTayCotxd9PibdAR4OGGZyU6JE+jG0UWEzWCPF1O+QaoHeJF4FUS5l+PqIHIf4CLOUVoNRru+XEvR5OkDt0gwofCYittawQS7mugb+MIIvzduW/2fiZ3qo6/p55IPxGcli9lOBT5/h5urH6oMzvjMjiVkkezKn40jPRFUSD2Qk6FP4QO9pzPJN9k5dlFhxnRMgpPg46TF/Mw6jV8MLopGg2lQQhASp6J2qFeTOlSgwdm76PQXHbcCF8jX45viVajUDfMh1MXczmZms9Pu+I5k5ZP7RAvbmkcTlxaPm8Ma0z0FRquqaj801isNix2G8nZRU5BiAObHV5fGctnY5tVeowQbwN+Ja6kbWsE0CDChw9HN6PQbCHQ08D62IvM2imDKh/sXptJM3czqVM1Hu5Zi9Op+Twx/yAvDKxfqstyRWxSDlUDPehQK4hfY1N5cWB9dpzJYObE1ryz9jgHE7Ix6DQMaBzOoCYR3P3DHjJLgqcPxzShwGxl59kMEjILqRXiRaHZys9HL+JhiKJdzSDiUvNv6kDkQlYB2QVmMgrMpZYEsRdyqBPmgw27SwGvt1Hapad2r8Vvl0xDB2njvb1tVdz06k2XAzUQ+Q9gt8P0jac5mpSDQafh7eGN8TBoURSFzIJizFYb4X5GWlQNZPnUDkT5e/xhC6BOqyE6wIPoAOfFPSPfVEHsGuXvzsgWUYT7uZOUXURuoZlv72zF15vjmLbRWV/yUI9aTOpYnW+2xJWee7HFxtMLDzkFIQAXsot4ZcVRWlcP4HBiNs8PqM+UH/eWKvaDvAwMaRbJbW2rVpreVlH5t0nLNbFoXwL9G4az40xGpfsdS87FZhdBd2quiZMX8/Dz1FM10JNQbwOhPkamj2vO++tO0LVOCFNn7yOjxBFZo8DIltG8PKgB59LycXfT8tX4lvx67CKHE3NK293zTRb8PfSlwcOlhPoaSc4pIsTbwK6zGVjtNmoFe+HlpuOTW5uRb7Kg02hYuCeBKd/vKbUdH9YsnCoBnoycvp3ccn8PInyNvDm8MdtOpzGgcTgWm2vfopuBC5kFHEzM5rWVsSRkFmLQabi1dTTDmkVhtljJKqDSTiKtRiEm3Ie3RzTmpXJNB77uet4f1YRqQTd26+5fRQ1ErjEWq40ii5XF+6S3/7WhDQn3c2fmtjjcdBoGNolk+YEkknIKaV8zkO71QvDzuHKNQYCngXphZT3dUzrXoG6oNzO2neVMah7Vgz1pHu1HSl5+Bb8BgI/Wn2LauObM2yW98p5uWk6l5pNd6PoP5c64DO7qVINPfz1FjaBzjG9Xhe+2ngPgvq41qXqD99KrXF8UFlv4/LfTbDqeSp8GoXgaKr9r1SigURSeWXyI1eUGP/q46/h6fEuqBHgQE+7NW8Mb0//jzU46qqHNIgj1MdI02hewczGniBeXHmF8u6pM33S69FiL9yVya+sqTNt4+tKXx02roX3NIPo0CCO7QAasxaUVUCfUG293HblFFtGcaTWMa1uFrvWCScwqJMDTQICnnrtm7nEKQkBuHj785QQ96oVittioGnhzZintdjvJuSamzpYSdq/6oYxrU4VlBy7w/NLD1ArxYkL7ahh1Rfh66F2WlL2NeoY0jaB9zUBSc01oFIVgbwMh3gbVzOwS1EDkGmOyWDFZbJgsNvw99DQI92HijN3Uj/ChVfVAps7ey8AmEQxrFom3Uc+20+lYrHb0Wg1mqw1vo55Qn782TMnfQ8/w5pFkFZrxMep5bH6ZEO5wYg67zmWw8Xhqpc9ffSiJHjEhLN0vXgbl6+TRAe5UDfAkLc/EseRcQDqAAObuimfW5DZ8t/UckztWp1aotD7mmcyk5xVjMlvRajS46xU8DDr0Gk2ldVgVlX+CIrOV5Owifom9yPmMAtrWCKRZFT9OXcxFo2hoXT0AjUIFvQVAj3qhnEzJdQpCAHIKLTw8dz/TbmtOoIeew4k5FJUYWr04sD71w30oMttIzikiJbeYdjWC0CjQMNIHf083J13BttPpDG8RRZ8Goaw7epG2NQIJ8jJwMbuQ+7rXJtBDz+xd8XxZYtFu0GkY1TKa99edYM3hZGx2GNAojHA/d77dEodRr8Vqs/Pp2Gak5plcXpO957O4t2tN9DrtTetzkZRdyMytZ7HY7DSJ8qVvwzAmzdxdms09kJDNon2JTB/XnAbhPigaBTeNgg2FAM+ywMRNpyXK34Motex8WdRA5Cpgt9vJyDdht4HRTYvXZRZTRVHILTJTM9iLjrUD2XEmg+ScIl4b2pA3Vh3j89ta8NPueB6aux+b3U73eiE0ifJDr1UY/NlWfIx6XhhYn061gyo1ccrMLyYpu5ANx1JRFOgVE8I9XWqSZ7Iw9qvfK+wf5GWgZpAXHm5a9p7LKk3nOkjLL+a2NlVoEOFLXGoeVQI9qBLgwTP965GSY+J4ci7tagbyZF8fZm6Lw1Ly/IJiK+56LUvua09mQTHvrj3O1G61eHPNMdYdvcj4dtUY2iySnWdzsFht1I/wwaDT4GnQi5fBFc7KUVFxRbHFytZTaUz5YU/pAvP99nNUD/Tky/EtGPPlDj4a04TnBsTwyopYp+eG+Rh5om9d7vi27PfHcS/wRO+6RPi58/nG0+SaLPSpH8Y3d7TkaFIODcJ9effn46VdMSAttO+MaEIVf3ei/N1pXzOQvg3CiAn3wWqzk1VoZmq3WjzVtx4L9iRwJi2fPg3D8XPXk11kLg1CQFp9f9odz6/HynQt/RtF8Pj8A9js8jvoY9RVWupxYLeDt0F7fXex/Q1MFlupXm9Ch+q8tuKokwkcyDV6cuFBFt3bnrwCC2aLjXt+3MOgppFMaF+NUB9jBV8mFdfcnD9lV5ELmQWYrWIIVGi2EuHnjrHQTICHGx4ufqk93HT4GvW8NqQhhxKz+O1kKsHeBpKzi3h2QAyP/XTA6c7ll9gUdsZl8MOkNv/P3nmGR1mmff83vWZm0ntvkIRQEiBUEVGKBRRBbIBiWXuvq659de29oCKKiNgBu2KhE1oCCZDee5ve534/3GEgJrC77z7r7vPo/zj8IDOZfl/XeZ3nv/DaRQVc/NZOrn5vD29fMpbs6BCMGgX+gIBeLWZLdNvc/O2bw2yq7GRyRgQ9Dg/vba/nnWXjsbt9A3gdaoWUP8/JIdqgIjVSx5wRMfzlTA0dFjdXrdqDo/++UzIi2F7dzY8VneTGGTBq5Lx84RguWVE84LWq5FJeX1zIt2Vi0FZUiAqlXMqNH+xlRIKJi4qSufCNHbSYXdw+M5sum5u5L20J/r1UAtdOzyBCryLWqCY1Qk9G1PENpP7AH/hn0G5xc/V7ewZtMKOSTKze2cDNp2bRZnFTlBbOp1dPZN2+FjqsbqZkRjA81kBzrxOPT+DPpw8nLUKHAKSEaXlrSx1/++Zw8PG2VHWTEKrhnUvH8dKP1QOKEBDVard8WMKLF4xGo5AxOy+G7BgDj399kLEp4YxPDaXb5qGkoY/MaD2v/lzN1wfaMGjkvLl4LLFGNa1mUR4/LjWcV38eGGAnkYDL52daViSjk0Nxe/3EGY9f1OuUMuJNGgwaBWG63yd3SyWXEm/SUNlhQ6OQBdPOfw2L09fvzSKj3exiUkYkK7bU8f3Bdp5aMJJwnQqDRkpkyB8dkRPhj0HV/xAcHh/1XXa6HV42VXVS1+0gIMDT34ot0i67mx770U3a5vJS0W7lb18f4rVNtbRbXJwyLIqUcC0KqYRQjYLddT1Dtk8tLh+f7GnCpFVw95xhaBQy/vb1YcwuLzvretiwv5X9zWZazU6qO22cV5jIUwtGMSrRxJVT0nhn2Xi8fj9RhoGLzFMLRrG+tIUehwe9Ss5zGyu58t09fFvezrrrJnHTjAxMWgXDYw28u6OBpl4n35S1M/+VbTT2ODD+irvi9gW4/v29FKVFAHDjjEx+PNTOkwtGMXdUHA6PnxtPzeKVi8aQEaXnrS11A/4+IMDzP1SJrWiLizs/LqWh2/4/9I39gd87ylssQ4ZIzsyJ5uTsKJ79oYJrV+9lzvObueuT/WRG6/nz6cPYXNXJ3Je20GVz8/z5o/lsbzPLVu7ihvf3UtlpY3W/EuZYNPU6WbG1Fq9/sFotM0rPe8vGIpdKOPPFzagVMn442M69/Z3OHoeXNouLMSmh5MebuHxqGiBugvevL2PJhJTgYzmHUMOFqOW8fclY0iJ1rC9pYUtVFwEB5o6MG/JzuebkDGKM6t9tEQIQopRzxUlp/9B9BQEUcikyqZSiNDGwsrHHyZ76Pu7+dD99Dj8V7VZaeh3YXCfuRP1e8UdH5H8A3VYXPU4vFW1Wvtjfyrdl7fgCAnKphEfOzuNQq5X6bgfDYnTsqutBq5Tj9Pq5bvUeWvpPMp/tbSYpTMtbSwv5oLiR9Cg9b/ZvzMnhWnJiDUglkBNnJCs6BEEQkMukzB4RQ16ckR8OdVDTaeevXx3kzlnDefyrQ1xYlMTwGAPP/VDJWaPiqe1y8LdvDuP0+FkyIZlzCxPZcN1kHv2ynLQIPRUdVqYPi+KbA+38ePhoa/e9HQ1srerinWXjOGtkPK1mF1Myw9lU2Q2I4XyPf32IZZPTuPfzgZbGR0isD87NxeH1U9lhJzVCT0SIine21fNteRuXTExla/VgmdsRrC9pISs6hHGpYfx0uJPFE/8guP6Bfx19zqFPuQlhGu75rIzGnqNcjUNtVu7+9ABJYVpuOS2L0UmhFCSFcsWq3UHjqrEpYWyqOD636pPdzTw0L491JWKHUC6V8OjZeYxLDedgm4WvSlt57JwRJIfpiApR0dbn4u5PDww4jS8am8iVJ6XxcX+qdlmLhRtOOerQqVYMPFtqFTLCdEqWvLUz6MoKcMnbxSxfXEBSuJaV2+qwOH1EG1TcOCOLU4ZF/e5VbCFaJUmhGu49Yzgevz8ow/419Co5erWcgAABQRjgzbSpqpOxqWGs3dXI3FHxNDu9uH0BLE4PcaF/rGHH4o9C5F+A3x+guc/J9wc7eG9HPU6Pn6lZkby1dCwPrC+nutPGnZ/s57OrJ7GnoYf0CB3nvroNEF0b/3rOCP6yroy6frv2hh4Hr/xUzXOLRmF2ekgMVXPppNH0ODwU1/YQGaJiRLyRrw+0sXpnA0qZlIsnJHPR+CT0atFkp7HHyU0f7OPNpaJ/x9bqbk7KjuKxLw9S0WHDpFXw3KLRHG63cvPafQAsLEhkfFoY167ew40zsnns8KEB7/PyKSmcMyaRFVvq2F7Tg1GrYHFRMjfOyGLBa9sIBKCu20FEyNBSNqVcSpvZycs/1fDMeaMI0Si4ZEVxcIFVK6R0WAd3ftIidFw8IZmUcB0GjYIeu4dP9zZz3rjE4yaq/oE/8I9iZIJp0L/FGtWAhH2NfUP+zZkj41ArZGwoaSUrOmSAe6ZUKhmS1HoEvoCA4Rge112zh5GfYGLVjjrOHZNIcrgOf0DAoFVgdnrpsHp4eF4eXTYPT3xzCIvLx5riRnLjDLx32Xg+29vC+8UNA5K0D7dZGZlgpKTJjEIm4e1Lx7JiS92AIgTEw8OylbtYe+UE5o2KwxsQkCChrstGh9VNZMg/R4D/v4hQrYrTcqLxB+ChuXlcv2bvoBiLB87KpcvqIq5/lLV6x9FuWIhKTn68EQGxU3Xt6j2cnh/LgsJE1DbXgCiN3zv+GM38f8Dj81PXZWdvYx/VnXZCdUp0KjktZhdrihu5fs1e/nJmDiq5FEEQJXixBg1+gWC+SHWnjVs+LOHW07IHPPa6khbSIvWEapVcc3ImT3x7mPs+L2N9aStvbalj8Vs7iTaqWVCYgMcf4M3Ntazd1cRpw6MxaRSMTDCiVkoxqBXEGNQYtXIEQaCif8F87Jx8nvz2MM98V8GBZgsHmi3ct66Myg4bKeE6ttV0D3g9hcmhnDkynvNe28ZbW+oob7Wwrbqbq97bw9riRj7+00TkMnHBkg6xcEkkkGDS8MrPNYxJMhGilvH1gbYBp7zKDtug3JmZuTHcOCOTVdsbuOTtYs59dSvvba9n2eRU5L/zBfIP/M8gyqBmZu7AROvLpqTS0je0G+acETFIJXDlu7uZnBkx6IRsdXk59QQJ2acMiyLWJHYajBoFIxNNtFtcmDRKXvm5ill50eTFGbC5fOjVcmJNatQKKSqFlBfOF/kjAK9vqsHlDbCvqY8Xzh9NnEnNvWeIPJWPdjfx2Px8cmINLJ6Qgs3t55sDbUO+HkGA97Y38Mx3lXTb3EglAo98eVDkbR3nM/g9QaeWo1PJ0SqkFKWGse6aSczOiyE9Us+M4VG8f8V45DIJaqUcnz+ATCrh58qjHbGZebHc+cl+rnx3Nw9uKOOdZeORSSWs2FI7wPH6D/xRiPzTaOtz8vGeJk5/fhPnvrqNS94u5uEN5Vw6KZVp2ZGAGCy1obSFF84fTWaUng6ri42HO3B7/aQco8vvsnmwun1EhRxtg3r9Amanj8YeO8s31QTD7Y7FM99VMCcvliPKulXb6+mxe6jusvHg3DxeumAMXTY3D39xkF11vXx/sB2AUYkmKtqtQ2YgbK/uZnisAfmv5Ho3nZrJ099WYBkiSOuDXU3IpRI2XDuZd5eNw+sPkJ9gHBC5srAggV8qOrnqpHQum5JGZbuNnw4PdKr8orSVc0bHo+w/2YXrlMwfE88NH+wLukoKAvxU0cnV7+2mfYjuyR/4A/8swnRKHp6Xx12zhxHebxCYERWCRELwt3gszh6dwOu/1KBTypgxfLCfz5FDxoQh0p0NajmLxiUhk0q5bWY2J2VG4AsIOL1+cuIMXDIxFbVChtsvEBDEsW6sUU1CqIYOsxOlXMqCgngAWvtcyGUStlV3c/PaEgKCwOE2K0+cm8+yyalYXV7OLUjgnNHxon+FVBwhDKXElUrB4w9wy9pSfAGBx+fn89TCkfQ4hh5b/d4QplMhk4Ld40chk3JRURKPzx/BvacPZ19DLzqFjHiTGm8ArnhnV7BjMm9UPG1mZ9DAbn+zhae/qyDepOH0EbGDzB9/7/ijEPkVBEGgodvO/uY+Shr7qO60UddlY1ddD4daLbRZXNy/rhz7MaSwbruHWz8sYenElOBG/nVZG06vn4snJHNxUQoljWZ67J5BG3p9t31AFHR+gpHKdivJEXo+39dy3Ne5q76XEfFGAKxuH1a3j8gQFX6/QJvFxbKVu9h4qIPmPhd6lYIJ6WHce/pwOq0uYo1igm7kMQXQp3ubmZwZwcT0gYuoSascUOX/Gr9UdNLS58Ts9FLaZOaUYVG8tWQsCwoSuOW0LC6dnIpBo2B3fS9Xv7eHlAgdGuXAsYrbF+DVn2t4+cLR5MQaWFCYwFtbage1QUFUOmyt7sbuOX7C6B/4A/8oIkPUXDYljY+umsirF45BI5fyRWkb549PGnA/qQTcXtHz54z8OFrNLqo6bOTEHjUH7LC6KWuxsHBsInfMyiY7OoSEUA0LCxN58YIxLN9UzfaaHvY39XLH7GEIgrh2aJVyDjT34fT46bS62V7TzR0fiyfpFVvrmJUXS0OPnXmjE1DIJCwsTMDl9ROpV9FpFa+HPQ199Dq9vPhjFfubzTy4oZzmPidapYzXLi7kvjNzeO3iAh44K5fIY/gfJ2dHsaNWtAzosXtp7HXQ1OPgzU211HX9QQwHCA/REK5ToFFK8QcEHB4/jb1OtEoFPkHA7Qvg8fmZPiyKC8Yn8cpFY0iN1LGpsotzCxKYkB6ORALfH2wn1qThwQ0HEQTotBwtVH7v+IMjcgw6zE5quh3c9lFJkKgWplPy4Fm5uH1+bv2olOnZUbxw/miue3/vAMa9LyDwfXk7U7Mi2XioA5VMjOO+7/Myrj05Q/TEUMiC8d9HkByu45M9ooOpTCrhmpMz+LK0lTHJJp5dNIoPdzUNII4egcPjQ9XfqlUrpOiUMhQyKWE6BUtWlAfvd7jNwtuXjKOhx8HGQ+3MGRHLjOHRVHbYiDdpSAzT4PMLtJpdKKQSUiN0nFeYyAe7xEjwEw1BlDIpU7Iiue3DUg63W4P/LpXA387Np7rT1m+NLOG2mdko5VKkErhtZjZXvLN7wOe3raabxl4Hr140BoVMyopfKWiOxU+HO4gxqEgI1SKTSsT2tVyGIIgnzJY+Fx/vacIfEDhnTDxp/eTYP/AHhoJMKkElk6JVyfEFBL460Mq9Z+Rw9bR0PtjVgFYhZ3SikdRIHdOyIylINvHh7iZ21/fy3HmjeHFjFcX1vZQ2mblxRhZXrdpNRpSeM0bGolbI2NfQx6VvF3P7rGEcaDKzdGIqPQ4PUSEqXv+liqyTM5mUEUlps5n3dzYMsJVfvaORDaWtrL1yAgqphJWXjmNbdTcf7W7iztnD8AYC7KnvJT1SR3WHjZRwLWE6FeE6JRlReu74uJTiuqNJvCMTjKxYWojF7SPGoMbl8XHX7GFMSAvDL0CMQYVUKmFWXjS9di/tFteAg9LvFSEaJSEaJVKJlBvW7GVPQ1/wNq1SxltLx3L37OGUt1lYsbmOZVNSSAnXYnZ6yY0zcOXUNF77uQZ/QKC600anzY1Jq2BbZSfTsqMwaH7fxo0SQRjq3PnfAYvFgtFoxGw2YzAY/v4f/H/C4fbSafXQbfdw/vLtQ0r6PvrTBGIMKl78sRqzw0NiuG6AkRDAxPRwUiN0vLejgaUTU6jvdvDj4Q7kUglvXzKWdot7gItphF7JI2eP4NrVexibEsalk1N5b3sDp+ZE89KPVfQ5PPxpWjpOj3+QxfOLF4zm7k/2Y3H5WDwhmfwEA7lxJiTArOc2Be/31Q1TePyrQ+xr6uP5RaO557MDNPQcHfeEaRWsuGQsSCTIJBIEBPQqOS19Lt7eWsvSiSks/6WWn4ZQA5w1Mg69Sj6kXFEqgbeWjiVCr2JbdTcv/liF2ekVJYoj41hQmMBlK3cNCN87LSeaK6amUdlh46Ufq46bXLl0YgrVHVaunZ6JSi7F7PLi8QVIi9DzzPcVbChtHXD/GcOjePTsEUT1L6i9dvG7trq8GDUKwnRKTH8kYf6u0dBt5+sDbehUcryBALvqerm4KJkwnQIBCZ/va2Z/k5nMaD3zxyTw6d5mlm+qJUyr4O1Lx2Fz+ZBJJRg0crpsHq5/f2/QNEwqgYuKkpmUHoFeLWdfo1i0XHtyBqVNZpLCNJi0Sqo67dz0wb4hX9+8UXH86aT0Adc2iOGYD8/L47GvD3GwxcJzi0aDRCBcp+K57yv54Rhjs5EJRh6cm4tJo8AbEO3stSo5HRYXXTY3Bo2Cb8va0SqljEsNJy1Cj0krJ0z3B3EVRBfe+9eVsaa4cdBtGoWML66fjMPlxStAXZed9aWtWF1exqeFMyYpFBAQBFi2che3z8zmtLwYlFIJvoBAWuT/PX+kf2b//t13RBxOLy1WFw6vny/2tw5ZhAA8+30Fp+VEs3RSCm6vn3CdinmjYilrsfLQhjIsLj/DYw1kRespazYyNSuSy1YWA/RLuiS8sLEi+HjpkToem59Pa5+TJ84dycFWC7d/VMqIeCOCINDcTxZ76tsKHj17BOmR+iBfYnxqGJ1WN1a3jzkjYpiVF4PHF8Dl9SEIRxeM605Op6zZzE8VndxwSibP/1A5oAiRSODO2cOxuf18daCNj3Y34vIGUMmlLJmYzANn5hIQBO6cPYw9jb1YnAPHIeePS2TpiuIhP6+AAFUdNuRSeOTLg8ilEqYPiyLOqEYihfX7Wnj/svGs2FqHVCLh3MIENAoZKrmUtcWNQZvqX0MigalZkazaXs/sETZWba+nvNVKrFHNjTOyBhUhIJrAnZnfzdzR8bT0Obn9o1I2Vx2VC0/LiuSv80cQa/z9poz+XuFw+6jutPHCxiqUcikXFSVzoFlMrC1rsTApI5w7Pi5hX6MZgF8qu1i5tZ5XLhpDcV0vrWYnu+t7eWC92IXMjNKTF2/goXl5ROpUIIE4k5rqThvxoVru+WQ/V0/PpLXPxeE2Kza3j1ijGovLz+YTjEC/Lmvjkkmpg/69rMXCN2Vt6BRyLC4fN6zZy/OLRuHxBdh4TCdVo5DxyNl5GNVyms1ublyzjycW5PPW5hpOy43BH4C3t9Ry26zhtPY56bZ7aLO4aOjxkx6pJ+mPTCjaLa5g9/rXcHr9lLVYmJIexiNfHebD3U3B24rreokMUfHMeaNw93NDog1qOixOEkxaSpv6/k8WIv8MfveFiMXtpazFgkYhpeKY8cKvcbjdxoycGK58dzdvLhlLl90NgkBWdAifXD0JpUyKFIG7Pj3Aw2eP4J7PDvxKyidwzxm5eH0BYoxqDjSbaTOLBdDGgx3IpBL+fPpwbC7RpOhYvLm5lgvGJ7L8l1ouKkri1JwYDrdZeGvJWLZWd6OSSdGpZPz1i0M8es4IQrUKpBIJZ4yM4+a1YgcmL97Icz9UDnjc03JiUMqlfLq3mY+OuXDcvgCv/1JLY4+T88YmEmtU8fGfJvLR7iY2V3Vh0ipYOjEFvUp+3MINwOL08tovtcHAKIfbR5RBTXOvg4gQFTqVnD/PGYbZ6WXtrmZe31TDTadmERAE4kxq5uTF8OUxjH+ZVMI9pw/n833N+PpntXfPySEgBNAq5Tz/q/d3LFZsrWNcahi3fljC1uqjyiCJBGKMKlr7nHi8AcL1yhNa8v+B/zvwBwQ2V3Vx5ardZEWFcNOpWSxdsTMYTvfxnmb0KjnPLhrFQxvKg8RxX0Dg9o9KeWrhSK5/f9+Ablplh41Yo5pog5pddb0YNDLUCimZUSG4fQEenJuHxx/gvLGJ6JRy/vzZAU4eFolRIz9hEJpCKg1mNh3BnBExLJ6Qglou5ZwxciZnhvPEN4dx+wT0MIBjdd30dJr7XCjDdSxdsROdUo7HF2B+QRK9DjdfH2hlycRUFr22Lchjk0jg3DEJTM6IQCKRDEry/r3B7QsMirs4FnXddiakhfHh7iZy4wwsGptEuF6J0+vn873NfLCzgUsnpaBRyMiJM9DS58Tl9ZMc9keR97suRNrMTly+AJsqOjkpO4rEEwQTJYVq6bK6eHhuHh1WF102D3Vdduq67MweEYNeJceoUXDnnOHolDLeWFzAZ3tbEBBICtdh1Cj4uaSFM/LjuPez/ZQ2izkGcUY1SyeloFXKeeyrQ3QOoQip7rQxPTuKrOgQXtxYxZPfVqDol8w+ODcPgOvf38dZI+P4eHcj95yeQ22Xnf1NZmz96ZqeIQqGM/Nj0apkweTfX+PrsjYWFCbS2OuiucfO4qJkzh+XxK66Hu5fV8bFE1IYHhvCwdahC7jxaeGUNpmZmRtNbZedNrOLc0bryIs34fT68AYCmF0+DGoFE9JCeX0TfLCzgbtPH87tH5Wy9soirjslE5vbF1yEV26t45syUQWUHK6jtsuG3e3D5QucUBJnc/twevxMzozgobm5oqeDSo7HL/DZvmZu+bAUgDPyYzl7dDyxRjUa5e/68vg/j06rizc21WDSKLhiqmjGd6QIOQKb28dDG8q57Fdmfb0OL0aNkunDopBLxSBJp9fP/WfmkhWtx+LyMSUzAhDQKGS0W1yoFXJ21HYRa9Rg1Cp47ZcaRiaakCDB7PQyY3g0HwzR9gc4pyABg0bOo2fncaDZwmm50ZS3WLjinV1YXD5UcinzRsXz+bWTcXv8WN0+5P1tf4CTsqMI1yrptnv48MoJgEhyD1HJsXt8LJmYwpK3igdstIIAH+5uIjVCh04t/90XIkqZhBiDmjaLa8jbM6P02Dw+7pw9DLVCxmu/VPeTm2PFwlMlQ6WQ8+pFY9hW3UW0QU1Dj5PCFNNv+0b+C/G7XWlb+px4fH78AYF2qxuDWs6pOdGsKW4Y0pTosimp/Hi4A68/QFmLjfJWCxqljNw4A1aXj63V3awvaeHmU7PRKmU4PD7So/T87ZtDHGy1opRJOXNkHFqljKunZXDj2n24vAFazC4+39fCmflxQxYhICba6tRyRiaYeHheHgdazCikUmKNat7eWse9nx3g/HFJzMmLYdX2emKMKkYnmXhnWx0T08Op73YglYrt2WNlYxKJBJc3MChr4wgEQSTFbq3qot3qJjPGQHqkluFxBhJCtaze0cDtM7O5bgijn1GJRuJNGq6dnsE9n+3nxhlZTMuKpKZLlCW39DmZOzKOadlRSCWQHWPgvtNzePCLcvocHj69ehIv/1TFl/tb8foFcmINXHNyBkfoszNzo/H5A5S1WFhT3EhBciiTMsLZ03CUmBdv0rBoXCKJoVp0KhlymYT8BCPPfi+OqPLijSwam8S41HB21PawtbqbFzZWsaG0ldcuLiArOuSf+1H9gf81aO1z0uPwcP0pWRg1crwB4bjXX323gxjjYLKzy+vntlnZ1HRYWX35eLw+gcPtFtw+gY92N/H1gTZ8AQGtUsZlk1MpTAklxqjB7vFzoMVCQBB49edqlm+qZuPNJ6FTyTl7dPygg0FmlJ5zx8Tj8Qbw+AJcMTWFz/a18uz3RzuAbl+AD3Y10mp28uDcPN7b2cDcUXF83D9KMKhFn6OHvygPEi0XFsRz5bQMDGoFext6j3vaX7W9nrvmDMfq8h43WPP3gDCdkhtmZHLXJ/sH3ZYbZ0Apk6KWy0gM1XDd+3u5Y9YwIkJUbChp5UCzmZOyIjk9X4dRIyc9So9KLqPb5qa82UJhalgwsff3iN9tIeLyikVIc5+T0/Nj+aasnREJRh6bn89D68ux9ncSlDIpN5+aSVWHjfH9/gDZMSG8+GMVTy8cyc8VnbhaLXy6t5nXLi7ghY1VdFrd3HdmDpeuLA5u0B5/gI/3NLGvsZe3LxnLNzdOpapDPM3nxBmRSeHZHyoGncgAbjwli6j+5FmDRkFmdAhNPQ66HR4uLErm8qlp7Gvopc/pBYmEOz/Zj0GtYH5BPONTw/lifyvr9rWwZGLygEAslUJ6QlUMgFoho8PqxqRRUNNpJyVMi0El48lz83F4/QgCvH9ZEQ9/Wc6BZgs6pYwFhYksmZiMgEC4XslbS8exv6mXLdXd3PPZAZZOTOHSSal8U9bGCz9WMTUrkpOzIjljZCy7G3oYEW/iT6t2D1AYlbdauO79Pbx2cQET08NQK+RoFDLW9qt7dtf3ctW09OCJ5eZTs8iODuHVn6tp6HGQGa3nphlZ7K7rZX0/j6SkyczaXY08c94oLhyXRLhOyfrSVmq77Px0uIMIvfJ3nbfxW8LrD9BhddPTn62kVsqo67SjV8tJDtcRZ/qf4+/Uddv565cH+ba8HUEAuRReuajg77y+gZW2WiGOQ5VSMGlVrN7RyM8VnehVcs4tSODk7Ci+LWsHxBHi8xuruGZaOg6Pj3mjE3h/Rz1XnpTOl/vb8Afgz58f4JG5eSybnMrZo+NZtaMeh9vH9adkEqKW8+LGarbXdDMzN5pxqWG89qtguyP4pbKLPoeX3fU9XDtdtH7vtbvpc/gGEPFPzo5i4dgkvi9v5+TsKOpOkOHUYnZh1MiR/N3V4v82bG4f41LC+MuZObz6czXtFjcKmYTZeWIXNTJEid3pZX1pCzeckklZi4V1JUctGIrrelm1vYHVl49HKZXxt28Pceec4Xxf1kZKpP5/9Df+vw2/y0JE9APws6OmB6fXx8ycWD7eLYbIFaWF8cpFY7B7/EgQrdjXFDewfFMtL184GrdPXDBn58Xw6d5mzi1I4PJ3dnFSVhRbqropbTJz06lZvPZzzZA+GNWddspbrSSFahiZaCKiX9Pv8wdYfVkRV7y7K2jHLJdKuHJqWtAo7QgsTi/3fX6AjYdFcptBI+eFRaO5bOWuY/xNnJS1WLi4KIl3Lx3P679UExWi4qZTs1i5tY7s6BBCNQq6bB5GJZqGtLQeFqNH3i8p7rJ5iDao6LS5KW+1UNLYR4hawYLCRPyBAPecPhyPN0CUUU2oRmxTV3XYWbW9nttnZZMWGcLNa7ewsDCRELWcK97dHXyenw538kqIivcuG8/ts7LZVt0zSOYMIgH21Z9qOGtUHA+uL+OBubkDulf3fXaAv52bT2O3A4vby5Wrjj5Hd00P572+nWcWjmRhYQJrd4mcGK9f4K9fHuLa6RmcPSaBb8vbcfsCfLm/jdNyYv4oRH4DWJ1evi1v5/51ZcEDQKxRzZMLRrKnoZeb1pTw9rKxDIv555RzgiDQ2OPA7PQikUgIUctRSCU8uL6cjceoSXwBsTuokEkGFRwgyjN/7Rp87cmZ+P0B6nqcXP7OrgFE7gc3lDM5I4K75gwLklgB3txSywdXTGBnbQ9LJqbg8QlIJGLncXNlN+0WN35BQCqBopQwClPCsHt8zH1xa7BbMTEjApvbh0Ejx6CR024Z3MWp6rRxen4cX5S2snRSCvEmDc9+XxksQowaBRcVJdFmcfH0dxWMSTJRkBwavCZ+jWExIcFMld8z9CoFEsHDyAQjd88ZjlwqRSqFfQ29gEC4Thx9Fdf2MHdUPM98P5iz1mZx8fJP1dxyahZXTUsnEBD4pbKLs8ck/PZv6L8Iv0tDM6fXj1Im48lvDzMtO5oXf6zgb/NHkh6p591tDZQ29WFQy8mNM/D2lhqWb6oFYF+jWSQzquRE6FU09Tpx+wIEBDg9P4ZP9ooXckak/rhZFSD6YDz61SEuXVEcVMfIZVJGJZpYf91kPr9mEh9cWcTGW07impMzBgVQddvcwSIERELZii11A0zWjuDd7Q20WVwkhGoZnxbOaTlRPHveSP5yZg4XvbkTgHvPyCEtYiBhKjlcw/Pnj2FdSQvzX9nK5e/s4qwXt3D/unIKksKIDlFT321n9nObeHBDOV6fQLfdg04pw+Ly8eD6g1z2zi68gQD7W8xYXF6cXj+z82J4YWPVoNfZaXXz1LeHkUul7D3BZ7ensZcIvQqvX0CtGNjKbDG7WLaymPwkE898NzRx9cEN5Sw+Jq0UoLnPSZhWyRelLZwyPAoAhUyCbCgryj/wP46DbRZu+bAkWIQAtJpdXPp2MeNTw/nzGcO5ZEUxbebj244LgoDN7cXi9NDW52R3fTfflbdzwRs7OPPFLZzxwmZRUuv00tgz2K14fUkLSycOVqUA3DQji5LGPiL1KgqTQ3n2vFF4/QGiDGre3lI3SE0GsLmqi3CdcoD76pExaKfNTbxRS0q4lqUTkzkpK4K8eAO9/eZmD28oZ3RSKN12N7vqe4N8MIDkMC3hOhWvXFjAW0vH8v3NU7l8ysDXHaFXMm9UHJlRei5fuYseu4fd9UdHlucWJPDOtnokSPD6A5y/fAcj4o2E6YaWsP/pJLHT+HuHQaNAkEiJNqjITzASZ1ITFaLivLFJ5MQa+HJ/K/4A5MWb2Fx5/BDPDaUtOL1+og1qnF4/mdEhRIb8vj/f32WJq1XK6RTcmLRKtlR1cWFRCk98c5iTsyO4bWYWUokEmQRkEgkLxyWTHKFHKZOSHRNCiFpBlCHA5spOhsWEBAOn1ArRwAzA5fNjUMuHtEUHCNUqOdRqpbTZzO0flvDg3DwEBMJ0SmKNGpQyKb0OManR1q/1PxbHFhyZUXrOGhnH9Wv2Hvf97qzpxubx8dcvD3L3nBzWFjdh7CfXXbN6D9dOz+DViwrocXgob7EQGaLCpFXwzra6QXK1vY19XL16D4+dM4LceCM3nppNbaeNVrOTCenh7K7vIyAIJIRpWL64gHC9irs+LuXxc/MZFmOgpKnvuK/zu/J27pg1jBjD8bsQRo0Ch8eHxy8u6qFaRdCvAUCChB67+7jz7l6HF7vbN+jvIg0qxqeGc6hNJN6eNzYRk+Z3eXn8pjA7PGyt7ubFC0ZjVCtw+QKs3FrH5qou3L4A35S1YVArODUnmuY+FzG/kli7PD6azS6+KWvjl4pOTBoFF09IISpExbmvbh/QlSxpMnPB8h08c94oLn17oOz8830t3D4zm6cXjuTFjVXUddvJiNJz3fRMVHLxN3XN9Az0KhnJ4VoSwzT02L1My46kot1KdefgDt6mqi7GJIUGuy8SCWiUMuaOiuNwm5WUCB1jksIIBERid1q0Hn8gwF/n5/PW5loq2m2kRup4dtFoNpS2MDUzkvpuB498eZDW/tTuienh3HtGDlOzIrj4zWJiDGqGxYTwQXEjL/V7D0kkYnFyRLqf3X97RpSeqZmR/FTRybWrRdnvY18f4kA/kT5Uq+C2mcPITzD+YWrWjziThh67G7PZhUomQaOQ09jrQIqErBgDEglBsvLxEAiIyiu9Us67W+u54dRMlPLfZU8giN/lSquUS1ErpFxclMzDXxzk2pMzuGJqGm6fn267F4VMQrhOSYxJQ4xJw4h4I2anh26bh0BAIDlMy8y8GDRKGdUdNobHhlDeYmFcShg7anvYUNLK/IKEId1BJRIxLvyVn8VFYkt1N26fny8PtFHa2Mddc4bzt68P8WN/x2NkookHz8phWIwh6KRqUMspSDJx9ckZ1HbZ2VDaymWT04g0qHjsq0OD8mlkMilzR8YTplfSaXFxyaQU7lsnSoR9AYFnv6/k2e8ryY0N4dLJqaSEa1ErZHx4nFZtVYcNQYC0SB1LVxTT5/Dy8VUTcHj87G3oYdHYZKJC1FhdXnrsHuQyKXaXjwi9EvcxHBiZVMLYlFD0KjmH2qw09ToJCALTh0Xx/MaqIUdbSyakEBmiwqCR8/ovNTw8bwQ3r90XbDsHBAGF9MQXtUQCerU8WIhEG1RUd9j4uqyVpRNTKWvuozA5jBDNHyZn/25YXD6MagV6lZzdDb1YXV4un5rKtSenc+nKXVR22Dh7dDzD4wx02QaOIdxeP7XdDha/tXMA0fTrsnaWTkzhhumZPPsrSbfZ6WV/U18wofZY/O2bw/x4y0ncf1Yu4TolIWo5/oDAL5WdjEkOJUKvoriuh3s+2x7kcsWbNNx/Vg4vbqwa9HgKqXQAEXx6dhRuj5+lb+/kuUWjuea9PcGCAsQMqZcuGMMbm2vZWSu6qx5ut/JNWRsvnj+aELWCxW/tHPAcW6u7WbpiJy9eMIY7ZmdxVn4cHVb3ALPFg61WlkxMYU/DPkAM5wvXK/lwdyMvLBrNzroearrs3Ly2hCUTU7jltGy0ChmhOgUauRyNUorsBNLi3xvCdKrgyNbnD5AcIXqAOD1+ttV0MX1YFD12z3EVUKcMj0IukeALBLhhRiapEb9vDxH4nRYiAGqFnIwoPeePS+Tln6pYvqmGwpRQjBoF80bFDzoBGDVKjP0bk9XhZmSCEbcvgEIq5Z7Tc3jsq4PcMCOL3fW9/FzRwXljCyhpNA9QcUglcN+ZuXy4u3HAJlvX7WB7dTdXTE1j+S81XDg+iTkjYhmfGkZ9j4PmPicmrRKZFDw+kYX/5MKRLHlzJ419zuBjReiVPL1wFLd+WELHMQvz2JRQLnm7mNxYA08uHEm7xUmcSUNZi2XAeyxrtXLLh6WcnB3FzadlntAjpKnXwYbSPpp6nUSGqOize+lzelk0LpkXfqhkc1UXkSEqpmRGcMG4JB796hB3zMoOLuBzR8Uxb1Q8m6u66HV4+NNJ6cSZ1NR22kmN1PL4/Hzu/Lh0AAdkXGooUzIjWLevmQ+vnEhdtx2DWs4nV0/kp8OdNPU6GJUYSpRBddyOVFKYFq1SRkf/bF0qgdtnDeOtzbWUtVjITzDx8Nl5RJ2gK/MH/v/g8/lp7HNid/vpdXiwOL3EmTREGVRc8nYxObEGrpiahtXlo8cf4JOrJlLZYcXp8RNjVGP8lWKj0+rihY2VQ6pd3t5ax5orilD9XD3od3ygxUJapH5Q4VCUFka71RXc7E/NiaIoLZxDrVaSw3U09zl59MtDA/6muc/JDWv28eIFYwZ1WaZkRXBLv4/PiHgD956RwxkvbOLC8cm8vqlmQBECIl/phjX7eHbRqGAhAiKHJNqg4i/ryvk1xiSamJgZgU4p48z8OBp6nMhlUqINamr6eVbP/1DJO5eOY9HYRNYUN/L5vhYWFibyxDeHefLbCl69qIBP9zazrbqbdftaCNMpmZgeLtreK2UY/nAdPi6O9X7RqeXMyImh0+LCbvQzPTtywAgdxI7ukgkpCAj8UtFFUrgOlUL2uyaqwu+4EIkMUZEZredAs5k3l4yl0+ZGq5AhIJ6Wo06QTRKiVRGiFb05jGo5Dl+ApxaOwuby8t5l43nqu8Pc9ME+7p4zjOumZ1Da1IdMKiE3zsia4ka+KRto0pUcpmX2iBge/aKc1y4uxOr28W15O+0WJ9Oyo2nocbC+pIWTh0XRaXXz9YE2QtQKnjt/NFKJhA6ri0/3NvPl/jYe//oQl01J5WCrlenDoogMUWFz+zBpFeTEG3n0y4NMzYrkkokpfFfePuT7u2JqGl1WD0qZ9LgjjvhQTdBXISfWwLbaLmYMj+G61XtZPDGZM0bGUd1pIypETV68gb0NvSzfVMu9Zwznjlli7syxqqJP9jSTFKbl7UvGopBKmJAWxs+3TuPzkhYsLh8jE0yEahXc+3kZdV12JmdEYnV7kQDdNg+FyaFMy4pkQ2kL+5v7eGx+Pte9v3fAiVStkHL/WTm0W9wkhGrJjtGzoCCRNcUNwaLs7a11nDM6Hu0fHiL/MnrsbnrsHlzeAD5/gN0NvbzyUzVdNg/xJg2XThZjENIidSwen0RRRgR3f7I/2KmSSGBRYSLzRsdj0iiJDBm4IVpcvn5lytDYXNnFyETTgE0dIDF08KI/JimUh+flcdGbO4L/tquul7NGxvPQhoOkR+n5ePfQHUKHx8/+ZjMj4o3sbxaLm0VjE0kK0/LQ3DzSInWE65SsK2nB5vZTkBzKa78MrXpxev302D1B4uMRGDXKAQeHOJOady4Zh7yfYKtRSPH1T2w9vgBPLMhHLpXy8BcHKa7r4er39vDGkkLOGhXH9ppuxqeGMTM3hm/K2rj8nV3MzovliqlpZEXrSQjVEBWixOz04fQFkHu8qGSyP7oi/yAiDWoigb+clcv0yi4+KG7E5vYxKT2c0/NjiQxRip3oHypZMjEFjUJ63ELE4fFidYo+SXKpBH9AIM6owenzY3P7UMlkKOUSAoK4vv1vlQD/rlfbpDAdCwoTqemy0WFxEa5XMiEtglijOjgGOR7sbh/dNjdrdzXx5uZaZuREMX90AumROmbmxrBkQipZ0XoCgQBufwibKrp48tvBluVn5sfxfnEjO2q6ef6CMby9tY73ixuZlh1JYUoY817agi8g8OL5o/nL52XsOoZ0tnxTDRcXJROqVZARpef6UzJYvaOBU4ZFc6jNyi1rS/D4A8FFNlynZMFr2/n+YAc3n5rJPacP569fHQpu1jKphFtPy+Knwx102dzMHRU3wKr4CFLCtWiU8qCBmMPjQ69ScKDZzJ9PH86T3x4esGgaNHLeWjIWp9fHhn0tzB2TQEWblacWjOSzvc380k/sauhx8Oz3FczMjcGgUdBudnHWyDh21vXw2b5m5uTFEKKW4wsE8AYEbu03IZNJISVcx3OLRuHw+Fm1vYGRiSY+/tMEPtnbTEO3g8zoEGblRROuU7K3oY+zRsXR1OPgmtV7BuTd9Dm8/CfDl7w+P15/ALlUivLv/Ab/W9Flc9Nlc9Pn8KKSS9l4qIMeu4f3dhzNJGruc/LQhoNcOz2DDSWt3HhqJme8sHmAfF0Q4P3iRuJDtcweER1sh/sDIkHcL4D/BFFZXn8A2a/ULhIJzB4Ry96GXt5cUojZ6cWkVSKXwse7m2gzH+2u9Dq8+AICcUY18SZNsMMwFOq6bEzLjiTaoGL+mARSI3Rsqepk2rAolr61k5cuHENFuxjR4DuOb88RWF0+MaH6mKfz+gNEhohpuwqZhA8uL6LL7uHZ7ysxOzz85awcGnudPPVtRXA0OyLeyEPzcnnim8Nsqermjo9LeGTeCKZmRrKjppv5Y+K58qQ0dtX1olHIGJcahlYlxecXuPXD/Wyt7iJMp2TR2CRGJhqJClGTEKo5oQPsHziK5HAdCpmEgqRQrG4vWqUMpUyGLyDwl8/L6LJ58AcEGnocjE0dmHpe12XnYJuFnw51EmVQMSkjApfHT1yohu8PtmNz+wjTKQnXq/iitJU2i4tJGeGMTQlDAv/ruiz/1kLk/vvv54EHHhjwb9nZ2Rw6dOg4f/HbI86kIc6kYXJG5N+/cz/MTi/bqrqo7bZTXNfDaxcXUNdtp6nPQZhOyeTMSP721SHquu08sSCf9Ag94ToVvQ4P35S14w8IKGQSzhmdwKSMcG5eW4JeLafV7OL9/rnikgkpXP7OLnwBgYLkUCo6rAOKkCN4d3s9L184hts+LOG+M3N44tx8lq0spu4Ynsiehl6uWrWHd5eNJ0yrIMqgIjM6hOoOG+uvnURZi0XszITr+HRPE6t2NCCXSnj2vFE4+zN4jqz3w2JCuHvOcHbUdAfn7Hsa+rj3jBx21fWydlfjoJGPxelj2cpdfHr1RA6qrSx8dRvddg9GjUIcQ+XHctcn+xEE+HJ/G5dPSePCN3bw8oVj+L68nTPy4yhKDaPF7OTyKWlc/s4u/IEAYToF54xJYFK6KGms73Zw3thE9jb2cduHpRQmh3Lt9HRC1ArcPj+V7TY+qO9hTEoYzwyRY3Pk/elUv119LgiikZbFKRZANZ12PtzdiFQi4fxxSeTFG/4tjHqzw4XZGcDt8yOXStCrZbh9AUJUcoza/7+xlNcXoKzFzG0flVLZIW66GVF6nl80ijNe2Dzk37y5qZaXLxrDturuIT10QHTTHZsSSojSSW2Pg/e2NyAIAjedmsVJ/WTLoXDK8ChWba8P/r9KLuWRs/OI0qsoTA7jQLMZpVxCVIiKFVtqg+Zfx+KJrw+xfEkhB5rNpEboqOp/X79GWqSec0bH8/JP1UQb1LT0OXn6+0qmZkXxxIJROD0BClNC+7siPqINqiGltyDmULX9amzj9PhZOlEk1T9wZg59Lh+LXhd9QV67qACz08dNH5QM6ADubzZz0Rs7WX35eM56cQs1nQ58AQFBEDg9Pw76Ay7FfCuRO7e/ycz8V47KhXsdXh758iDTh0UxPjWMCenh5CeYhnzdf2AwVHIZa3fVcFJWFHaPn43l7XyytzkoOBgRbyRCf7TT5/H7aeh2cNnKXQPW8Bc2VvHAWbnsaehlZm4MrWYXPXYPy97ZFVyb15W0EBWi4skFI1m5tZYbT80mQqcQLf/V8qBVxH8j/u0rbm5uLt9///3RJ5T/72/CVLRZCdEo2FLdxZUnpbFyay0XT0hhR00PO2trGZ1k4r4zc/D5xWwCi8OLSi7h1lOzueGUTOxuP06vn8/2NXPTWnHxmJoZyfp+85vMKD0HWy3Bk9OcETG8vbXuuK/nu/J2pmZFUtrUR1FaBHXdDqZmRjBvdDwquQxfIMC6khZe/bmK968owucPIJdJ0SllKGRSeuwetlR3EQjAWaPiuCdCx6NfHuSmtfv47OpJXHNyBtWdNkwaJV6/n1vWliCTSnhwbi5Xv7cHX0Bgb0MvOXEGHvny4JCvcVp2JGt3NQ4wVDM7vbz8UzULChK4cFwSq3Y04AsISCQSnlwwEq1Sjl4jp6zVQmKYhgSTlqQwuGJKCt+Wt7Hmigm89GMVlxwzmzdo5Cy/qAC1QopaKeeXwx088tVhsqNDuH1WNh/taeaSyWkkhGqGTPe9a86w3+SC9foDtJqdWJw+nvmugnmj41m5tY5d9b0kmDTMzItmU2UnW6s7WTY5bZBa5B9Fp9VFq9lFm9lFSoSObpuHhh47yWFalHIZh9stFCaHUdsl+m2EapWE6bwY1XLC/8kCqLHXwXmvD0yvbuh2UNFuG9KtGMRRRCAgUDOE6iT4Hmxu+pxeDrZZWdIfshimU7JsipdrpmdQXNczSLp+Wk40Pr/A64sLaelzYtQoSI3QUdFuRQAeWF8WjD+478ycIYsQEBVqcqnoQXLDKZlc9/5gdZpGIaMgKRS/IHDWyDh21HYTolbw5uJC2iwuLllRzE0zMpk1IpZ4k4b3ttdz1+xh3PhByaDHmp0XQ0lT34CuiURCUI1xRn4skzIjeeSLg7h9ATQKGdFGFS9urBrSIdnm9vHT4U5WLRtHVIiaFrODpSt2oVZI+eqGqUQc8x33OTz8Zf2BQeNYlVxKu8VFXryRP3+ynxWXjCPiBKPrvwen20Ob1YPD7Ucuk6BVyjBplIRo/u+5tobrVZyWG8OFb+wYdA2MTQkVXXuP4SO2md08/0PVgCLkCO5fX8aKpWNp7HUQZ1Rz1Xu7BxH6O6xulm+qYViMgfNf385n10ykw+JmU1Unc0fGkRCq+6/0g/m3vyK5XE5MTMy/+2n+7fAHAsikUswO8Yu+fEoaV0/L4KWNVcwvECPtjywe60paeP6HSt6/vAi7y0+sSY3bF0AQRIfVGz7YS2PPwE1QKRcLAoAQtXzAfFgtl2E7gRzM5vYRb9Jw9ph43txUy8Pz8mjpc/LA+nLMTi8hKjkLxyaSE2vA5Q0gk8C3Ze2cPCyKRa9vH/Bcm6u6OGtkHDeckskz31ciAPubzNzxSSnTsqKI0Cvp7FcvrNpez+uLC3l7Sy3P/1DFm0vHHtcu/oz8WK55b2iJ8cd7mnhjyVhW7WhgTJIJg1pOhduHAOiUCqwuL09808Alk1JJCtVwcVEKSMTX+vm+o86FWqWMj/80AbVC9DLpc3iZnBXJlvw4emweLnxrB9OHRbG7vpdHzx7BW1tq+bmiE0EgaEfv8QkEAgLSf6OHSEufE7PTS3FtD5/ubcaglmN1iWTfNZcX4fL5CNep0ChlSCQSAgEBq9PzT6t4mnsdXP7OLlrNLl67uICnvz3EVdMyGJcahs8fQCqRUJQaxqE2C1ev3hdc1AqTQ3ls/ghkUgmmf9DQzePz8862ukHEUG8gII4ZTgCpRMLIRNNxb08O19LS5yTaoEarlHHP6cNRK2R8vreZy6ak8cGVE3hnWx3barqDZMDIEBWL39rJw/PyaOi2My4tnCe+OcSDc/OIMWpYcclYeu1efIEAOpWcaVmR/FzZyehEEyFqBZXtVlrMLm6YkYlBI2dMUigCcPvMbJ774ag5WIxBzcPz8kgM0/D414cHpD/LpRIeOTuPU3Oiefr7SorSwlm+uIBP9jShU8p5auFIXv+5hsPtVsJ1Ss4fl8T4tDB8foEz82Op6bKTHqln7qg4PD5xNDMlIwKfXwgGNx7x/jgiuR0KO2t7OHlYFH1OD8tW7mZ0komH5uYN4spYXD721PcF/18mlXDv6cMZkxyKyxtAKZPw+Ln52N3eIQsRq8tLl83DvsZeJEgYlWgi4pgQyU6bE69XoMvu4fuD7Xy1v43KDhsT0sP5yxk5RPoDgzyT/i8gOkTNKxcV8PovNeyu78WkVbCgIJExySZuXLOPx+fnkxtvxOsP0Nbn5KsDgxPEQRxVHmqz0mV1c86Y+CHN90BcFy8qSmb5phre39HIVdPSCNUp+bmiE5PWTGFyKEat8r+qQ/JvL0QqKyuJi4tDrVYzYcIE/vrXv5KUlDTkfd1uN2730XalxXL8i+u3gNvrp7nPyad7m6lot1KQHMqM4dHoVTK0KhlNvR4umpDMrWtLBs19ex1ebvuolIfm5mLvZ/3LJBIaeuzEGzWDCpG9Db0snZjCxkMd1HTZWTIxJXhbSZOZiekRfLF/6B/o5IxwxiSF0WZ2Mjkjgv3NZt49piVtdft4c3Mt54yOJ96kRqeSE6ZX8MLGygFFyBGsK2npt1IPRyWXopBJEAQobzVz86nZQd7IL5VdlDSZObcggbmj4kXJ36/8OY7A4xOOS3wNCGIcu1wKj80fQWOvk5d+rKa6U2yDm7QKbjglk58PdzItO5JYo5pum5ukMC23zczmiW8OA/DZVROQSaXc+3lZsMAwqOUsm5zKWSPj+OiKIrodXh776hDVnXYWFCZw/rgk/AGBPoc32MYvTA4l9DjmTv8s3F4fUqkEq8sfzDbaWi2abS0oTGBvYx/rrp3ECz9U8tDcXF7+qZJ7zsjljU01fL6vBbcvwKT0cO6YPYxwnY/4EwQzHguz08MdH5dS3mrl/cvHsbmyiz/PycHu8WN3+wgIommbViEjL87E1SelB30ndtX3cvtH+3l64Ug0Svnf5UuB2Dkorhs8OhQEcYM63ihiWEwINrePxDANEXpl0FX4WFw+JY2fKzq56qS0YEcvEAC9Sk6nzY3P5+fyKamcNzaRPoeX5ZtqKK7r5ZJJKbRb3Zw8LJrIECVPLhiFof/UfawEE+BvC/LpsLjZeKidVrOL2SNiyIoKYXd9D4vfLObx+fkcbrdi0ipYf+1k+pweJEgwauWY1ArWl7YOKEJA5ILc9cl+3lwylm/K2vh0XzNpETpGJph4Y3MtbRYX549LIilMS0aUHpvLi93jx+0NcOnkFGo67Wyp7uba1Xtx+/zcMiOLWSNikUrE925z+yhIMqGWy4gxHj+MLSFUQ3KYml6Hjx9uPgmjRjHk71sqEYsPpUxKRpSeO2eLKrfbPyrlUJsViQSmZERw68xsbC7vgJTqXruHldvqeO6HymBBK5GIhdv545KwuURbBLvXT2OPg7w4I2fmx+H1+7l29T4uenMH719eRJhOiUTy7zsE/CcglUr4y7oDLCxM5JJJKTg8ftaXtPDGZtF9O7x/NOP0+HF4/cctMEDkJqoVIo/neBAEggfC7bXdnDc2gc/2NnP+uER8AQGFTEprnxOHR7RU0Cr/852of2shMn78eN5++22ys7NpbW3lgQceYMqUKRw4cICQkMGBYn/9618HcUr+U/D5/Gyr6R7Q6fimrJ3nvq9k1WXjaelxYHf7QcKQjqYgzmhVcimVHVbMDg9ZMQY0ChlXTUtn+6+Y/NWddkK1StIj9VR32nB4/OTEGihvtbCupJnXLy5k46GOAaF1IJ4Ww/Uq1u1rxun1c2FRMn/+bHAoE8Bn+5pZUJhAc5+TgqQw7vm07Ljvf0dNN7eclo0vIJ7EZFIJ7RY3Mqm4eRwx/jI7vby5uRa5VMKXN0zhxhlZ/GXd4Mf9e+3AELWclZeORyqRcOW7u4NtcxAJpA+sL+e5RaOwuX34/AJyqRSv30ufw8PCwkSK63pQKmRc8e7uICkQxFPeM99XopRLOXt0PLvq+1DKpdjcPlZsqRvk9TI60cT/RDOkudeBze2npc+B2elj9Y56IvtdGHUqOfsae5mSGcm41DDsbh9zR8dzy9oS3lw6lmUriwcUqluqu1nw6jZWXTYemURCzD9AQuuwuNlc1c3tp2URqlNx3thEbG4fbRYXfQ4vOpU4lgvTiU7BZ4+O480ttUGexp6GXmxuH3sbeslLMKH/O7wZlVxK/BCScIBXfqrhyQUjuWrVngHfa7hOySNnj6DT6uK2D0t5euEonvjmcFB5YlDLuWFGJi19TuaNiuehLw5S3WlDI5cxJz+WkQkmrl29lyunppMaIaBRipLts0cncMXUdL7e30piqJbs2BAMJwhr67A42VrTw80f7Au2z9/f2UhSmJa/nZtPZnQIl7xdjNl5tMA2qOW8uXQsaRF6um0eXvxxsFswiEX2ztoeRiWGsmp7Ax//aQJGjQKFTEp9t4PHvhL5chlRep5eOBKb281V7+0Z8rEkUgnvbKvj2ukZXDg+iae+q0CllFHWYmbxhOTjujlfOD4Jg0aFQXPiE3CoVskHVxQhCNDQY8ekUVLS3cf1p2Qik0o41Gph1fYGLn27mA+uKBpQiBxqswwI4QNxQ3z868PkxhlFMzeFjPs+P0B5f1q3Si7loXl5vL64gPOX72BHTTehWsWAcdFvCY/PH3RsPuKq7PMHgr9ZZb/bskrxz22bYTolWdEGnv9h8G9Er5IHXa3VChl2t4/cOMOQ1xGIhySpRIJwAkp9Srg26LkTrlOiV8s5tyAej1+MD/jhUAfVHTZGJppIi9ARa9QQrlf+QweOfxf+rfTn2bNns2DBAvLz85k5cyZffvklfX19rF27dsj733XXXZjN5uB/jY1DG8L8FmjodXL9+3sHdTrsHpEjkRyhIzs2BOdxipAj8AYEYo0aPP4Ajb123t/RQI/dw62nZQ2wEFfJpbj9AV65cAxXTUvj1Z+ruX1mNgsKEhAEeOrbw7yxpJAZw6ORScW56nljE3ngrFzuX1fG1ppu0qP0dFhdx62oAwJ02z0IAjT1Ok9IypRJJexv6qW2y4FKLuWWU7MA0SL9tpnZXDg+CV1/y31iejirLx/Ps99V0Of0csesbML7T1wKmYQLxiWSEq5lRPzQWSGJYRpiTRq213Sz8VDHgM3qWKzYUodWKSOAWPEfbLVS1WFjdl4Mi4uSaTW7BhQhx2L5plosLh+TMyNYvriAJ8/NJ86kZkJ6ONnHpOwunZiC8V/wTRAEgZY+JxXtNl7+sZLSJjNRBhUPzcvj1tOyeW+H2HUxO32E65WMSTKxu76HMK2S9CgdpU19g7plIKarvrGplhazk27b0CffI+i1e2g1u0gJ1zIjNxohAD6/QG2XnW6bhxc2VnH5O7u55r09rC1uxO0LYNAoGJsSNuBxzE4vHxQ3UncCtcgRaJVyrjwpbcjbqjtt+AMCzy8axYNzc7lyahrPLxrNykvH8f6OBq56bw+13XY6rC4WT0hm+eICXr5wDG8uGcuEtDAyo/RoFeKY4PWLC3j6vJFMzoigqdfOnBGx3Pv5AZr7XHxb1ka4VgmCOP6aNzqeM/NjBxUhNpeXynYrz35XwQ8H26jrdnDnR6UsLEwMPvfyxQVMzYqkx+7m8a8PDShCQCxwL11RTLtVzIcZqgt4BD0OD4Z+l97nfqikw+bivLGJA+5T1WHj0reLiTWqWTIhedBjTEgLpzAllHMLE3C4fZw5Mo4xSSYONJvptHlICNWwbHIqxzYTFDIJj8zL+4e7ez12saBa8No2ksN11HTZePmnaq5+bw9Xvrubnw538sSCfExaJT9VdBLoXxttbh+v9HfThsLKbXV8ub+VZSt3cdOpWQyPFa83ty/AHR+XYnH5uOGUDLbWdA/K9Pkt0Nhj50BjH619LnodHg63WXhrSy3vba+nptOO2elhQ0kL1Z129jT08fR3Fby9pY5DbRbqu+209DpweY4/OjdoFMHx3bFQyaW8ubQw6Fnl8vpICtNyzckZQx6GitLC8PgDGDRy9Co5M3OjB91HIoHrT8nk3W3iOnPp5FQ6rW7WlbTSZnZx89oSEkI1GDQKXvmpmls/KuWd7fW09DnpdZx4Xfl34jdlrZhMJrKysqiqGvr0oFKpUKn+83OrTouLdovruDa9NV12PH6BELWc9Eh9MLjq14jsn8F9X95OfoKRlHA9Jq2Sm9aW8OV1k5idF0t5qwW3z49Ro2RThWgbv2BMAmOSQpFKYWFhAldMTaPH7kGtkJKfYGDeqDi8AYHvytuCHZsum4dnFkbQ0DuY5HQswnRKPtjZwIycGE7NieKj3UOT9GYMjyZMr6Ctz4VeJeekrAgKkkN5f2cjb22uZUFBAksnpiCXSajtsnPVqj102z18eaCNorQw7p4znNQILaE6Jd02D/sa+njgrDwq2q389aujC3uoVsEzC0fRbXOLJN02C1MyI6jusNHyK+XA4TYrRo0CEHB6faLXiVxGq9nFKcMjCQjw460nIZdKsDi93L++PDguEP0s/Cx7u5jliwsZkWDk3UvHU9Vhpa5f3vtzRQdF6eG//ij+YXRbXbh9ASwuH7d9VMLj5+STHKHlnW31fFDcyAXjkrC6fBxoNjMq0YTF5UWtkLG7rpe8eBNFaeH8cLDjuI+/raaLeaPiKGuxMHtE7HFnvC19TqRSCQsLE8XWr06Jxe3F7PRxx8elwfvZPX5W7WjgYL+UWv4rR9ownZKLi5Kxub1029x/d36fGRXCvWcM59EvB0rCb5uZTVKYBrPTh83tI1ynJC1Czys/V1PZYWVWbgznjElgQ2nLAM7PaTnR3DVnGAq5lCiDhkPtVvrsHqwuH/mJRrKiDWRE6WnqdfDKT1UsmZjC4hU7idSrEBC4qCiZ8akDiyuby8une5sprutl/ph4pFJx7Ljh+sk0dDu469P9/Z0/CbNyY0iPDDkukdbq9tHYIxbqxwuOBDgpK5IwrQKFVILLF8Dm8pMXZ+CkrEh+Pkbx02Xz8OC6cv62IJ/ZI2L5/mA7Lq+fOXmxxJrUqORSAgEBr19AIYVnzxtFQ48DpVzGzWv3cf64JNZdM4nD7TbkUojQq4gzaQZlMg2FbpuLdSXN/HS4k3CdEgG4fs2+Aeva3sY+bvxgH08tGMma4kYuHJeMxSVGJhxvLARidy4rKgSz08utH5byyLw8ru0n/QoCrC1u5PIpadR3O37zQqSh28735e1My45CKhHN344N/5NI4M5Zw5iVG8N17++h9BgujkwqFnpZ0SEU1/cyPDYEo0ZBtEEsOI4o4YwaBUlhWj68cgKH263sruslOVzH2NQwYg3qoBy6uc+FPyAgl8KrFxWwfFMNu+p7CdUquXB8EmePjkcmkdBr93Dh8h3cMjOb3DgjHxQ30ml1k59gZNmUVL4sbaWyw8alk1KINahZu7uJynYrXn+Aq6alB0fTR/Dqz9V8sb+Fdy4Zh9vr/P8mxv8r+E0LEZvNRnV1NRdffPFv+bT/NPwIwdyY48HjC+D3C8Sa1EHFx69x3SkZrNxaR3qkHo9foLrTyjlj4mi1uFj4+g4ePGs4U7Iicbj9+AKQFqHF6fUHGf52t5+qDiv+gECsUY3TG+Dp44S5qRVSFHIJ0SGq47b2UsK1xBk1fFXWxuVT0zFq4vn+YAd9vzrNzcyNwe3143DLuPitYnLjDNwxaxiRIUrOyI9hX6OZD3Y3UflFOasuK+LGNfsGFG1ub4B4kxokEi5+c+cAdUpunIEPrizip0OdhOmVxBnVWJxeoowqIkNEHsHE9Ajunj0Mg0bB1e/tpqRJfC+JYRrkUilahQy3TyAjWs/XZW3cNTsboT9jptvuwe7ykRCm4fH5+Tg8Xu75rJyDrRZUchkrLxmH1e2jocdBdL+MOS/eyMNfHOSiouRgJ+efRYfFSa/DS4haTrvFRa/DS2qEluWba3l/p9jZ21nXw4KCRF7YWMmTC0by4IYynj1vNNEhapHvYHEfN3gMxAXN5vbRanZR12U/biGyobSV1EgdI/rdf31CALc3wIs/Dv3b2V3fS5fNzeG2o7+ZorQwVHIJt31UxnmFSf3fqYakcA1Wlx+/IOb8KGRHNzmDRsH5Y5M4OTuKg60WvH6B+FANtZ02arsc7KrvZXxqGA9tOMj9Zw5nYWEC1R02DrRYuHHN3kEjznEpobi9fjIi9RxoMbOhpBWr28ekjHAEASwuLw6vjz+dlMZbm+uJ6m/pd9rcGNRyzhmTMMiEq80i/kYSQjViKGN/99CgkfPo2SN46YIxPPbVIXbV9/LF/lbcPj/LJqfy8nFO/C19Tl75qZqbTs3imtV7Bh1IksK05MUZUMrFTcsbELhlbQkHWiy8enEBFxclsXZXE76AwHmFiYxKMqFTyZFLpRSlhSMIAuF6JUaNYlASdJ/DQ3a0HpCw8pJxvPpzNS9srCRMq2RcahjLJqcSopb9Q6TETquHNf3WAfMLEnjmu4ohD1d9Di9lLRZGJ5nw+Hxc/MYOMqJDGJlgOm43Mj/BSFU/38vs9OLy+QdwyVr6nAgInD06HtP/EDfrH0Gv3cPBNgtjU8MIEGBPo3VQArEgwF+/OsSYpFAyokMGFCL+gMCfPzvAiqVjcfsCbKnqJkKvJD/ehMvrxaBR4fD4aelzolXKkEvFw+n545MQBFFGbXZ66LZ7sDh9OLw+ShvN/deejLtnD0etlKGSS9GrZHRZ3Vy/piT4Wd72USmjE01cMTWNienhyKQStlR3kZ9o4qpp6TjcPnwBgTc313LP6cPZVt2NQa0YMhepscfJ+tIWTsuNQa2QYfqN3XT/rYXIrbfeyplnnklycjItLS385S9/QSaTcf755/87n/ZfhhAQiDGqkUslQ5oPmbQKQtRyAoKAUiJlRk40SeE6Vm2vp9XsZHisgcsmp7K9poeP9zTz8oVjWLW9nskZEcSZNDT3OXn5wjHc/el+XvqphgWFiSSYNGRG6zGoFexu6OWKd/fwxuICdtb2cP0pmQQEAafHz7xRcXx2zKnxCK6Ymo7fL+D1B/jb/Hwuf2fXgI5CpF5M7Hxwg0ia2lrdxdpdTTy/aDRbqrr46XAnBo2ceaPi8QsCTb3OoJyurMXC4rd2kh2t5+F5eZxbkMCM4VGEqBV02dwDipDkcC1Xn5xOTZeD13+pDlrA3zYzm9GJJgTA7PAyPi2M0qY+2i1ucuIM+PwBfq7oZHJmODKJlENtFvyCwJMLRmF3+5j38lauPTkDnUqGw+NjfUkzEzMimZQRDkho6XNyoMVCSriWyk479T0OxiSHEm/SsHhCEqkRetRyCQ9sKGfjoaOn0MQwDa9dVMD1p2Tg9QXotLqINf1jhNAj8PkDeP0CvXYPaoWsP5FZwA8DFjaLS3zfb2yW8cLGSv42fyRflLayaFwiO2p6sLv9LBqfGNwQfo3FRcnEGsXET7vHhz8gDJkQ7PUH2FbVzaVTUpAgYePBDkYlhQ458jmCfY19GDQKWswupmSE89C8Eeys6+Gi8cm8+GMVXTY3zywcxS9VXazZ2YDT62dWbgwXjk9CKQOPXyTbqfrDH1dsqaOhxzEgauDW07L5/mA7t83MxuULoJBJCderhjTNGxlvpCAljIAAr/xcxfqSo0TQ3fW9RBtUPHnuSBwuUQp/3thEOqzi731KRgR/OStnSAfV7dVdxBo1AzpDIHrd3LhmH28uKeT6UzK58t3dOL1+vj/YwcVFyUMWIlKJyKuo7LDx1f5Wnj1vFM//UEV1pw2ZVMLsvBjumDWMxLCBv6fHz81nzc4G7vy4lHiTmitPymBEvJHIY5QoepWclF8lYv8ax24WvoCTG2ZksmxKKgiigixU+49LYu0eHz39ROH0SD1rdg4+WB1BWYuZu2YPw+r28/qSQq5bvYf7zsxlXUnLINWURiFjZm7MAIl9p9WNQXO0EMmLN6KWSwn5Df17QCRR99g8hGqV6FRylm8a2u0WYE1xAxeNTxoUAuoPCEFX3Ts+LuXe03PwBgJ4AxLu/nQ/P/WT5iP1Km6dmcX41HC6rG5cHh9J4To+KG5kxda6IEl7ckYEOXEG3t1ezanDo9EoRb6KXCohN9YQLEKOYG9jH019TvLiDPgDASamhWN1eZFJpWzY30pmlDgGk0jEgvCHQ8fvuH51oI2UcB3a/2uFSFNTE+effz7d3d1ERkYyefJktm/fTmTkP24e9ltDEATMLnGRv2RS6pA/zttnihblcqmEHw93YHX52FDaytKJKYTrldR1O3j868M09zmJ1KuwuLz0Ojy4fX6kEgl2t49bPyzh4Xl5XPHublZtr+e+M3JYVyJmQKzYWk9KuJYWs4td9b0o5TL0KhmNPU5mDI8mIVTD21vrg+56fzopjSmZkcE2KUh44YLRtJldNPY6yYzSkx6p5+v9LczKiyNcp+Syd3YRFaKi3eJiVm4MoToF3TYvr/5STY/Nw6sXFwRJgyDOm6dkRZIaqSdCr6KmU2DJW2LYlkQCSpkUty/AssmpPP7VIW6fNYy6bge3nprJxIxI9jb08ktlJy/9WE2P3UN+gpHbZ2bz7PeVQaO2O2Zls7O2l4c2lAcLQKkErpuewbc3TaGxx8FJT/zEw/PyuGB8Mpet3MX6ayfT0OOg0+ZmX0Mfj3wx0MfkuukZzM4TdfyrLy/i9pnDuOXUbDYe7uD78nYmpIdT0tjHlMxI+gJe3L5/zlfV7w9Q1y2OBn481MHqy8cTGaJCI5dhdfqCI4qUcC0PnJXLnR+X8uSCkWwobeGKd3exdGIKAjA+LYzClFDaLS5umiFKp4/FVdPSyU8w8fR3FRTX9RCmU3LJxBTmjY4fpKSZMyKWxW/t5KxRcUTqlagVMjQK2XELaxAXyr+dm98vFffybXkbPr/A3/oVSY+ePYI3N9cGv6vRiSZGJYZS2mRGIZOiV8uxuXwMiwlhfWnzkOZ7T357mLeWjqWxx0GsUc2ylbv46zl53D1nGE99WxHcxCL0Sv62IJ/Fb+7khQvGDChCjqDd4ubTfc0sGptIU68DjVJObqyBVy8qICNKR0bUYDI8QLRRzXM/DN0Z8gUEfqnswqCRc9aouGBomUQiQSphkA/EheOT+P6gaDG/vrSV8lYLF45PJtqgDgZrDqUUSw7XcfNp2VwyKRWpREK4/l9XisT+iy6acqmEwpQwfq7opNfuIcqgxuIausORYNIiAdqtLpJMGh4+O49wnYrXLi7giW+OuiqPiDdy/SmZPPNdxQBZf2qEnvb+UY5WKWPu6Hj0Shkm/W9LUvUHBGJNGjz9EQRd1sGKrSNot7gHpaAfQY9d9ETx+ALEhWo42GrlqW8PD/AC6bS5uePj/Tx73igSTBp6nT6Wf16GTCrhrjnDsTi9PPrlQTZXdSGVwIgEE3/+TIzQGB4bwgNn5iKXSbl7znCe/6EyyKMbEW/krtnDeOLbw9w9Zzi/VIid5swoA18daGP0nFAAtlR1s7AggW013cd9j3KplMZeBxlRv30I37+1EFmzZs2/8+H/LXB5fPxS0cXuelH+lxqh5d3t9TT2OMmKDuGyKankxhnw+wMoZXIKU8L6yaQVPLhhcCjVonGJfLKnmbEpYbi8frZUdXHR+GTe2FSDRCKw4bpJWF0+Vu9oYGFhIg63j23V3Ty1cCSrd9Tz8oUFGDVydtf3olFKKS7robnPyXuXjUfeL6utbLfw2d5mzhmTQIhagdnlxezwIpdKGdsvRa3ssBKiUTI+JZS6bgfrr52MTinjlZ+rue2jo6dDvUrOM+eNIjlMi1ou4+OrJiAIIl8gxiguFH12N2aHh9cvLiDKoGbjzSfh8PhRKaQ4PX5SzsxBrZBxXmE8EzIiuOvT/VwyKYU7PxbVPKKVfDZXvLs7aK8eopKTGqHnT6t2D/j8AgI890MVo5NCkUmloiTy0/18cEUR719ehDcQCJqyDSVvfmFjFdOyI3n/8vEEBHHBFSQB5o6MY3KG6MjqCwhY3T5CdQpkiBK5f9Rdtb7HwbmvbqXP4UUiAX8AJEhYOikFnero2OKGGVnc9lEpHVY3F7+5g9NyYnjk7BGo5FL2N/UhQfQbyI4OEUmmOdHBUcT07EiUcinnL98e3Ay7bB6e+LaCbTXdPD5/JPHHnP6TwrVMSBc3laK0MHLjDCAEmD0iZshNXdmvnumwuKnrsvFteQf3z81l/stbAYgKUaGUS4LFxYT0cC4Yl8Rdn5QGu2ESCZwzOh6H109RWgRvbKob8vPaXd+Lob+bGKlXcftH+5mZG80L548WX4tcis8f4FCblWGxBjaUDu7+HcGX+1u5dFJqvx+IwGu/1PDJ3ma+vXHKcf8mNVxHQ8/xeVQNPQ5iDCpyYo8SqyNDVLxz6Tge+/oQh1qtJIRquHZ6JjqlbIDCpbrTHlwD0iK03HNGDq/+XM0DZ+YO2sQUMilRhv+MOmQoRBrUXDIphS1VXXyyt4kLxiUNuZ5JJHBuYTxOrx+tQo7LJ7CnvpfceCMWp5cn+2XQOpWc7TU93Pf5gQHhfiMTjLSanbi8AbKi9Tw+P58IneI3L0JALIIUMilSiYBMKmVMsokv97cNed/xaWEIxyni8xOMqJWiEs/vF92KhzIkO/Kcj351kD0NfcF/++pAGydnR3HP6Tn8ZV0Zv1R2DbBv0ClFV1R/QGBCWhiTM4owO70DlGy3nZZNS5+T9aUiR29EnJGzRsYjk0owahT8dLiDB+fmsnRiCg6Pj0OttkFF8ukjYtlZ18NpOb+979d/n8XafxjeQACry8s3ZW1IJaLV+u0zs5FLpagUUjqtbrw+P/d+Xs7D8/JIi9DRZXXz2sUFXLN6zwCb6pm50aRE6Fi5tY675wxDJZfRY/cgk8CTC0dyqNVKu8WNBJHdHGNQ0uf0sfGWk1AppBQmhRIVoqSlz8Wne5rZVNnJ60vGcufHpcx9aQuRehUz82IIUckZmxJGu8XJ9poeVu9sYGpmJCq5lD0NvQNmgj/dOo3RSaEEBNG46+pp6ZxXmEhZi4VwvZLM6BDUcil13XZSwnXi6V4pp8PqoqnHQYfVg1YpRaOU8/WBNpr7HIxODCVMp6Shx0F2TAgeX4BV2+q554wcbl5bwsLCBJYfE/I1fVgU35S1Dch4mZkXw+f7hibOgpir8/DcPGbmxvDT4Q4q263MyovF6vKilEv5cNfg9v4RrNregM8f4KsDbcwZEcttM7No6HHyyBcHKW8VT29apYw/nZTOnLwYPH43MokE9d8x4nK6vby9tS7IsREEaOpzEqpVsKAwAYVUSlFaGOUtFiQQHFN4/QJf7G8NFk5KmZQvrp/Mt+Xt7Gs2Mz7FRIxBw9JJKTT2OFDKpdz58f4h3Uk3V3XT3OcYUIhE6FU8Mm8EW6q7KG0yE2NQo1cpuOXUbA61WoP26yB2up5YkM8rP1ezdGIKHTY3s0fE0GZ2BaXihSmh/HxMiugVU9K48t3dAxYyQYCP9zSTFqnH359vdGzy9BEckSd+uqeJP58+nOve38s3Ze18U9YeJH1fOTUNpVzaX9gdv0MVCIjfm0mrJCAIVHbYGJsSOkBW+mtolDKyokKG7NiA6Grc0u+xAPRb7KsYHmvgnTgDbm8Aeb+M88f+EV9RWhgzhkczLCaE4bF6eh2iV4tGKeOqk9LptnuOe5r+b0FMiBqry8fLF43hhR+qkEhgQWHCgOtKJZdy7xk5vLmplgi9ikXjkgCB3Dgjbp9ASoSO5j4Xl63cxS2nZSFBgqv/NySXSpiVF8PV0zJo6nPw7rJxJIVpSQ4/8fjp3wmtSk5ymJamPgdSqcDSial8X94xaIMO1So4LSead7bVD3qM3DgDSpmUg60WThkexb7GXlqPY90/It5IRbttQBFyBD8e7mBWXgwxBtEP5th95NLJqaiVUjz948xZz20K3nakkfb0gpGs3tmA3e0jPVJPp83DtOxI1AopH15ZhEQiobXPhcPj567ZwwlRK/jxcAfPfFdBQBCLqTiTmhHxRsL0v+1YBv4oRAbB6xPt1l/YWMVXB9r4uqyNEfFGdCo5Ve02EkI1PLtoFNWdNhp6HOjVcuJCtUQYVHx9wxQq2m20mJ0khmrZ29jH+n3NrL68iA93NfLO9nokwOrLi2judfFdv7ugxx8gKkTF1dPSGZ1k4trV+7hkUjJFaeG88GM122t6iDKouO+sXL4sbWHJhBT0ahm76/sI1SoYnxbOe9sb2FXfw7LJqfQ5vKwrGXyS1Cpl/V0UMd9kX2MfepWMUYkm8uKNA3TkSccsEGaHh26bGNsuk4pE3StX7QzGr3+0u7k/aTSX93Y2cGZ+HGlROnwBgV31vVwyKXVAMZQdE8LGX6lDwnRKqo+T4wHQZnbhCwjcMSubq6el83VZGy//VM1pOdFEGVSYnUfbqgqZhPljEpg+LApfQMCoUdDU6+SrA2102tx0Wj3csGbvAPMsh8fPrroepg+Lwu3zo5RLkDilyKSglEuG9GDosHn48fDA9/HQhnJWLB1LRZuVDrObx87J58H15SdUFXj8ARweP6EaOaE6Ff6AhDarm4p2G6MTTZid3mDBNBQ2V3WTE2sc4NUSZVAzJTOSMYmmfpOkABqFnPvPzMHs9FHa3Ee4TkV6lJ4VW2rZWdvDvafn8OdPD7B8ccEAozt/AFRKkfSZn2BkX2Pfcc3p3ttezx2zh1GUFjZkIVKYEopGIeWUnBh6bG7eWjqWJ785THmrhXCdksUTUpidF0N5i4V3ttVz3tjEAWF5x2JWXjTdVjd6tRy9Wo7Z6eWx+WOINR7/dC2TSrj+lEwWv7Vz0G1Hgt8OtVpZX9pCdnQIL19YECR7HksW7XN4mJgexi+3TePL/a18sqeJZ84bxbPfV/NBcSMefwCJRFSf3T1nGI3ddhL/g5vu34NUKiEzUo9JreCJBfnIJBKmZ0dx+eQ09jb24g8IDI81oFfJyU8wUtLYx/M/VHDTqdno1Qq8/aq0g60W8hOMPPVtBRPSw7n3jByiQlREGdT4AgLNvQ4SQ7WYtApi/wPqjGMRolbg9viIN2lFt1i9kpcvHMNzP1Syv9mMRAKT0iO44ZRM1AoZp+VEU1zXQ0W7DbVCypkj41hYmIjN5ePbsjYem5/PbR+WcsbIuCGfb0ZO9Ak7fF/tb2X6sChW72xArRAL8WcXjiJELef+deVY3T5m5Ubz1Q1TaDU7WV/SyrqSFvwBgfvXl/P8olE4vQHu+/xA8NCTGKrl1YvHcNWq3QM6gblxBh6cm8uw6AJazKIy8v3iRh47ZwTh/6Cb8v8k/ihEfgWtUiamHaaHs6W6G0GA0iaRKyGXSnh20SgEIcANp2TS2GPn833NPHp2Hha3n/JWKz8d6mB2XgzJETqSwrQsKEjggfVlQYLkVdPS8foDPP3dYUqajnIwOqxu7l9fzl/PGcH07HDiTFrmvrhlgJLg6wNt3HBKJttquvmlopPcOAP3nZlDl9WDw+ujqddJQqgGrVI2oNtwBPMLEviytJXhcQY2HuoImnnJpWK2y6k50cGRhCAItFtcmJ1efH6Bb8vbeWtzLVa3j9GJJp48N58HNxwMup829Dj4YFcjYVolaoWU0/sdILVKGXaPyGU5YmFvcXoHBD0B1HTayI0zsPc4EsgR8UZMGjkbD3fy/s5GarpsCAF4c3MtJw+L4q/n5LP4rZ0oZVKeXTSKb8ragjk4CpmEuaPieOa8UajlEg62WQYUIRIJvL10LAFBQK2Q4vCIrVWL08eWqi6mZEaSGa0nOkQ9oEsiCAI65cBLqKnXicvrRyqVcN+GMp7W5PPAWTnHPSWBOA7z+gOs2FpPZYeNN5YUcv37e3l4Xi69Tg+e/gjw4/E79CoxT+gIvP4A3TY3XVYX6v7OVVmLhbQIPWeOjOW9HQ10WN2Ynd5gd+TKqWk4vT56HR6sbh89dk/QuG57TTcPz8vj073NROhVNPcdn/TaYnZh0iiQSCAtQofV7QsWrAXJoWRFhyBBQCmXsquuF7XVJdr/n5yB1e3l0z3NvLm5ltcuLkCnlJERoWXT7Sfj8Ykbu1Ej48p391DVaee66Zk09TpIjdDSbfPwt/n5xBrVg/gWbq8Pq9NHAAG720dyuJYnF+Tz0IaDQRl5criWP88ZzubKTuYXJDI5M4KoEBUKmSgF1yhltPQ58QVEtZxKIUMQ4KpVuznYZuXtS8byztY63tt5lGgsCGIOVJ/Dwz2nD0enlg9Svvw3QSqVEG1Uo1ZKuXVtKX85Yxgun5+dtaK5ocXpo7HXiUwi+sYsnpiK1+/H7fVj0ir6pbhNPDwvlytX7WZbdTfbqo9yEm44JYNFY5PQqeUnNJj7LRFh0PQfsMR2XGa0nrtmD8PjD6CQSTBqlJg0cgQkZMeE8MbiQty+ABLEz6ux24FSLuHa6WKGWEqEjrRIHSq5FH9AwC8IpEXoWTY5hZw4A5/tPX7X1+0LoJBLOSk7EpvLx4ZrJ/Pu9voB5PVt1d3EmzT89ZwRaBUynl44kpvXlmB2eokKUXP2K1sGdFOumpbGdav3DhpHlrVYePq7Cq6fnklVhw2tUsbDc/P+Yx0qiSCcIEf7PwyLxYLRaMRsNmMwDG2G9f8Dq9NNr8OLSi4jVKNAecxmYnV5MTs8+ATYUNLC+zsb6XV4GJsSxtUnpxNvVPNNeTvjUsLwBwJolHIUUik3rd2HVCJhQno4/oDA+LQwdtf1MiEjgove2AGIBL9Hz8mjocfJle/uHvK1RYaoWHtFES//WMWHQwRxSSXw5pKxQRb6ReOTWDoxhd0NvUSGqJEiOvRd8e6uAWqWyRkRXDA+iZVba5mUEUl+gpHSxj6e7idFSiTwzY1TSQjV0Gl1saehj8e/OkybxUVapI6rp6UzKtGETCIhIAi4fAEEQQjyMzqtbn442M6sEbEcarVyyvAownVKlm+q5XC/Pf6LG0X/mEi9irvmDOPmtUdDv2RSCSuWjuXyd3YNYt4Pj9Xzwvlj2Fnbw56GXrKiQ5iaFYkUkUjYY/fQ5/CQGKYV00h/rubrsvZBn90Z+bHceloWK7bWsXJrffB9r792EhanD29A4M6PS4Mz7cgQFXfMyuaHgx3IpRJunZk94EItru3mYKuV+45xkg3XKVl7ZRGf72thX2Mfh9qs3DFrGOE6JY9/fYiD/Y60x+KmGZmkR+n7E4ElLCxM4K9fHeK5RSMZkxRGSWMvXx5oG3J+LZXAx1dNZES8EY8/QFOPky/3tzB9eDS9dg9Xrto9YGFSyCQsX1zIj4c6+LqsjXiTlvPHJdLU66S5z0mYVsm+xj4um5KCSi7j5rUldNs93DlrGKXNfRxus3L+uCQe/hUp+Aiyo0N4bP4IDEopBp0Si9NPQBDQq2X4fALNfU66bG5uXFvCradmUdlhGzJwblyKiZcvLKDN4ubp7ypE102dksUTkpmdF4tKJuD1g08QsLv9hPRv8sdyexq67bSaXTT2OshPMLGhpIX1pS3celo2w2IM+AIBbG4/Pn8AXyAgqieUMrRKGR6/QHFdD029Ds4fm4BGqaDT5qHL5sbu9tNpcYFEwi0flmDUKHjvsvGc8/LW43aKNlw3GZVCGlQx/DejptPG9Kd+ZkFBApdMSkGnkvPW5lpW72wIyp0Lk0O5bnoGbl+A4TEGkAjYXD7e29FAeauV22dls76kheK6HiL0Ki6bkkZmpO6/uit0BD6fjx6HD2//dxmuVx3Xi8Xi9NLj8ODyiEIEiQS+P9jG1KwoajpthOtUCMAdH5eSGRVCSoR2kJvzEdw5exjNvU7OHh2PTiXB5g4w/5VtQ953QWECzv7nNGkVlDT2cXp+LI9+eTTZXiqBlZeO4+I3B3f/juDdS8fRZXPz6d5mHj17BAlh/5xi8ET4Z/bv31VHxOZ002330WIWQ8e21/Rg1MiZnRdLiFrG1uoePtrdxOgkExeOS+KsUXGcmhMdzOXQKGU09Tro6y9iNAoFPkHMDrnm5AxazU5xhOAX6La5WVCYMGAeeNmUNOwu0RvkeOi0ujG7fJw5Kp7PSloGuaQGBNGp8owRsQQQWFCYQJRBjcPjx+H2ISBQXN/D4/PzsXt89Dq85MYZ6LV7UMkljE0NZ0NpC+tLWphfkMBXN0yhqceBze1DJZPwzYE2fAGB2z4qRSKB+87IQS6TsGJLHZ1WN2OSTVw6KY2Nh9opTAnj9V9q2FnbE2xVpkXo6bSIsfYGtZxF4xK56YN9pIbrmDMihi/3i+ORxh4HSyemBFOF/QGB536o5M0lhTy4oTzoSVCUFsZds4dzzitbsTiPFlZPfVvBM+eN4o1N1USFqLl2eiY+v+hwOVQRAiK58drpGWREHt0MXr+oAK8vgEmr4KwXtwzoOnRa3dz+USnLF4sdivPHJxGmlRPSP6bpdXiZkhVJUVoY22t6+l9vODVddqIMKpZNTuWB9eV02tzsbezljtnDeGNTLZurugDR+2XZpFSGxxpQyqXUdNkpSg0P+q5UtNtJDteRG2ck1qThQLNlwMlGIoH7z8wlTKskgMDmyi6uWb2HN5eMpaLdyjPfVQ4oQkDkp9y8toSXLhhDcriOUYkmrl29Jyj1/vqGKWTF6EkI1fJNWTsPn51Hh8VNTaeNxUUpuH1+FHIp4TrlkDlFV5+cTqJJTZ/Lx12flLHxUHtwBn3/mbmUNvdxoNnKc+eNZlNlZ9D741jMHx3HTadmU9vl4II3tgevAbvHyaNfHmJLVRcPzcsjKWzwpubx+emwunD7BC5buYvaLjvLFxdy2cpdNPQ4eOCsXPRqObd9VMqehl5RFhlnQCGTcsaIWMakhLKupJk5I+LJiTMSZVDTavFi9zh5YF055a0WpBK4e87w4PeYEaWnz+k9bhEColdG/H94FPGP4gjxetnkFBQykdd23thEzu3v7u6q72NXfS8PfXGQK6am4fb7WbaymNXLiji3IIEN+1v506rdTEwP59yCBJLCtCSFaoj7B3OS/tOQy+VEGf6xrdGgUQTziwBazU7sbj9PfnOoP85BDOC8aUYWz/9QwdKJyWwobQ12CY8gNULHtOxI1u1r4cI3djA+NSwo+56QFs45Y+LRKsV8oY/3NPFlaSsPzsvjtg9LeH1xIXa3j4OtA/cVlVyGxXl8x18Ah1eMKbn/rFzi/kXl1b+C31Uh0mn3Ud1pC26eR/DcD1XcMSubNoubHbU9hOuVPP7NYZJD1VxQlIxUIuGnQx3c9Zl48lUrxFCopDAtW6q7GJcShsXpQ0BCY68Tk1ZJmE7F+tJWTh8RS0Kohl67B48/QHWXnYQT+FQoZBJ0Shlrixs5NSd6yFNwiFrOldPSONRqpbHXiV6t4M3NtYNi7Y0aBXqVnKRwLbfPzObWD0sGcDUe++oQH+5q5OULxqBTyeiwieY6Rwy4lk1O5WCrZYDPw9cH2vm+vINVy8bz4BdllLeIP36XN8CHu5rYU9/LqxcV8NKP1cwfE4fd7eOx+fl0mEWZ8KWTUoOx7Ilh4ml8W003/oBAUVo4oVoFyy8uwObxIwjie71s5a4BRQiIbcy7P93PA2flcsOaffxc0cWqy8YhO4EKMiBAl9VNUVoYOqWMkYkm9Go5Jq2S1zfVDjn6CAjw4e4mTs+PY/2+FobHZFPZbsXu9pEaocPr83P7zGGYnSIvZ0xSKOv2tXDJ5FQq261cf0omABmROkJUchZPSObGGZlIJRIMGjm1nXbKWi009jgQBGjsdTBnRCwA726r44z8WFy+AB5fgBWXFHKg2cLmyi6iDWpOHhZJmE5JRIiSToubm9eWcHJ2FN+WtTElK/K4I5QeuweLy8vXZW102z3BIiRSr8KoUXBuQb8ktl/2G21QkRWlR6UQyc+TM8JYffl4bvuoNDi2DNUquHvOcMYlh2L1BDh/+Y4Bi21pk5nzXt/GZ9dM4vkfqimu6+Gu2cPIig5h+aaa4GevV8m4fkYWbRYXj355cMi4gp8ruuiwuAnTKgcQU33+ABXtVuRSKXd+sp/aLjtjkkI50GymocdBZIiKxFAtB1usQf6KLyBQ0mTmzPxYnD4/d35cyssXjuGFjVV8srcZo0bBkwtGcuW7u4PE2YAA5a0WVHKRN+Pw+NAqZMd1WAaRQKzX/O9YbvVqOV/fMIWfKzp54YcqrP1S0ZEJRh4/N58Xfqjki/1tVHXY0Cnl2Fxe6ruddNrdpEfqOXdMAlMyIvD4xOTlOJOGGIN6SM+b/2s43GZlSmYkw2IMVHfakUgkNPY4+GRPM08uHMVPh9t5asFIvj7QxncH25FLJZxbkMDYlDAuWL4jOL6WSiX4AwEemptHt93NY18dotvuIVKvYvHEZE4fEdPfnRYNJOeOiqe2y86nx4x+nF7/CeMqxNGTgtxYA1ql5N+aOP738L/jyvgfQLfNRWljL5Wd9gFFyBE8/vVhVl8+njU7G7C7/SSGanjp51pe+rl20H1DtUoi9Ep+PtTBjNxo1AoZj355kH2NRzkfq7bXc9vMbOq77Tx33iguf3c3DrePFzZWsuqyomB65q9xRn4cpU19mLQKoo3qQYWIRCJaaZ/90haOTDAunZRCeqRuUCFidnoxO72clBVBY49jSEe96n7zr5QwDSu31jEzNyZoyDQxPYJLjzEiOgJfQOC+dQd4ZF4eC17bPujxKjtsxJvU2Dx+MqJCeOa7CrrtHs4tSKC2y87Wqi4MajmXTE7jiW8OMzsvmvGpEbj9Aby+AL0OLw09ot1zUph2gMrjWPQ5vChkYjqw0+vnjU21XDs9Y8j7HvvanR4fKy8dR3OfE4vTR1SIwKETkEEr263MGx1PY7cDvyCaP313sJ0fD3fywJm54lgOuKAomYJkEzqVjPmvbOXaaenMyY8jEBDQq+Vo5FLiQzU4vX7qu+2ipX+sgT6HNzieaOp1Eq5XEhmiotPq5qNdjczJj0UuldBt82DUyLlqWjpqhRRBgFCdEq1SzsFWKza3j9FJJtYUNzIxI+KEn4PXH+DyKWnc/YkoqY7QK3ln2bigRDshVMvc0XE8/e1httV0E65XUZgcytTMSKyuAO9sq+HJBfkIgthlidArcHkDWF0+ttR0Dzrxic8p8MIPlbx32TjmvrSV+FAtkXoVqy8fz5IVO3F6Ajxxbj6H26xIpZLj8oVAVBlkRQ/0O2i3ikTkaIMqaLdelBbGxn4Tp0npEfQ4PKwfgjB4zpgElq0s5rOrJ7GmuDH4fcwfk8DKrXWD1DubKrq4c84wvilr52CrFYVUwinDovh+CIv+lHCtGOb2XxS7fiJE6JV8W9Y+oM0PYgr44jd3svryIr460EZAgKZeB8n96cFxJg0hGgXDNAriQzW4vH60Svk/LIX/344uqwuXL8DV7+0ZwNEbmxLKjTMyuXRFMR9cWcR9n5cREaLi1tOySA7T8vm+Fjpt7mAR4O00AwAA7NJJREFUAmJi8jljEvhwV+MAsnanzc1T31awbHIqaRFiIZwUpkGjlKFSSNEpZQN4hcW1PYOiBI5g/pgEIvRK1AoJYfr/bLfu3xp6998Eq0vsWHw8hIvjEXxzoI2J6RFsq+5m2rCoIe+TFa3noXl5xJu0hOmVlDSa2dsgJlTeNjN7wH2f+OYwGoWMbruHk7MiiA/V0GZxc7DVzBtLCoOhcUcwIt7AZVNS+cu6MorSwnAMUahcOTWNT/Y2cyyN4tO9zZw/bnBQFohE1EXjkihrMXPHrGxuOCVzQMgbiN4NaqWc1n7J5pFshBNtzhXtNhQyaX/2y0B8X97OeWMTeXB9GetLW7ioKJlrTs6gvMVCeauFM0bGcWFRMg3dNrKjQ4gz6ajosHK4Vcx9+cu6MsJ1KuKMGly+E4cKOr3+YEbKLxWdwVb7UChMDiVCp2JvoxmH20dWdAjNfU46rW6Sw4/fpUoM09JhcXN6fixen581xQ1kRxs4a2QcvkCAGcOiaeh18thXRzkgRxRPs5/bxOkvbOakJ37ijBe34PQEUMqkxJu0nDEyljC9gk2VRxeJlHAtWoWMF88fTX6CkTe31PHaz9Uo5VJkUkmQQLmjuhu3L4C2n990pKPg8QXQKkUi5fGcKlVyKdnRISSHa/jzGcNZc3kR666dzLCYkAFEz1ijhgfm5vH+5RN47Jx8JqZHsHxTLRe+uYOP9jTz8BcHqeqwERWiZEtVD+e9vp2qThubKruO+1nuqO3BpFGwatk4ZEBzn4OoEDU/3HwSe+6ZQX6CCV9AwOX1ozhBe0uvkiNh4O29dnEEduwm4PWLPCYAqVScm/+ag3Sk2DVoFKgU0mBgGIgKrz1DSH07bW7MDm8weOy1TTXcdGoWhcmhA+6XGqFj+eJCTNr/PRuy2ek9bppwh9VNRbuFs0aJypDkcB0BAqxYOnbAmC1ErSAyRP2/5j3/K7C6vDT3ObG6ffx0qH1Q0Vpc18vGQx1MzRJHLzefmsnXB9r4eE8zPx7u5P3iRjIixQ776EQTTy0cydmj41HKpMd1WX53Wz0xRjUjE4wIwBXv7ObhDQd5dtHoAeF672yr445Z2Zw9Jh55f8dDJZeyZEIKl01JI0QlIyLkPz8y/L//KzkCiQSN4sQzsy6bG51KhscfYOPBDm4+NYunv6sARLnTTadm0djt4KdDHXRYXIxIMPHyj1XUdDk4e3QcE9MjuObkdF768agddHF9LxE6FZdMTiUQEEPsrl+zjwfPyuWTqydS0W6jw+oiL86IQiblgte3Y3P7MagVLChIoNvuYXd9L1EhYksuI1LPzGc3DXjdvQ4vPx7u4JGz83jsy0PBVmq4TskT5+ZjUCuwe/x8XdaIXiVnfkECYToFd39yAI8/QCAgxkMfGSssKEzg6wNtKOUnrlM9foHcOANbqwe69enVcp7+roL7z8qjvtvBnoZeitLCWTQ2EYVcSrfNTQBYua2BzGg9HxQ3sL/ZTHWnnfcuG09lhw2Xz0eMUYXXL2BQy4cMIJRJJYRplUG/C61KjsPt467Zw3lgfdmATsrw2BAen5/Pyz9Vcs6YRBJDNfgCAt8dbCNELeOSSal8XtIyZGt9QUEiXx1oJTlcx6xnN7Fy2Tiae1288EMliWFaXjh/NGt2NeDyBnh4QznPLRrFNSels7gomTU7G2i3ujkpK5KitHD+9N4udAoZj5wzgme/r+SGUzJ5c0khdo8fu9uPSi5BrZBx45p9nJ4fx3XTM/D4xFDDtEgd57++HYlEwl/PGcG+xl7SIsWuQLxJg0ou5dvyds4aGcea4gaun5E5yGkW4NaZ2SSGaVErZGRFn5hEplXK0YbJ+Wh3Ew9tKOe6k9N5aG4uKrlo4GRx+ajvdjIqycRpOVF4AwHCT+BDEK5TIZFK2F3fy5kj/x97Zx0Y1Zm2/d+4z2Ti7p4QQoJ7ocW1SGkL1N1dt93Kbre63bor2pYqdWgLFLcQICSEuHvGfeb744RAmtDt7rvvt9t3c/0FM5OZc0bOcz/3fUk0ZocHp8dHq8WDy+tHpxBylt7ZXs207MgBTeoAJqSF9SMQOnsKDL1S1qsy+rGslQtGxnOgtovdlZ3MyYtickYYJ37hpeL2+tEppXh8gT47SovTQ4hWgW0AE7Q/f3WMV1cUcuGoeNbuqWNLWSt/WTSEbruHxm4HUUEqwrQKooOUyKX/voj1fxRur79fd/V0FNWZyIkysLeqi9RwjTDS+/9sCf7vgM8foLbDRm2nnQ6bm/QIHUFqKX6/H7FIMHNcOSaRlWMSuf3D4j6S+0+LGnhy8VC+PtyETikj1qjigpHx/PmrU7/PC0bFY3Z4qGqz0tBpZ3RyyBl9dNw+Pz5/gD8tHILJ4aayJyH7z18d4+qJKYTphGunSi5GLZdw89Q0rpqQjMvrR6OQoFVI0Sgk6JT/GZ/bf00holNIUcsljEwK6ef9cBIT08N646zX7KllSWEsn18/jgO1XQyNCaLR5GBYfBD58UHY3T6e++EEF41J4KEvSnjqu+OkhzfywoUFvLqlss8Otb7LzoT0EMwOLxeNSSQtXMcLP56g+0sP54+MY9aQKLaUtfFiz2uHauWo5BJuWHuQzCg9iwtj6bZ78PsE0ttAWL+3DoVUzNuXjMDu9qFRSGjocmBQy5j/4vY+MeZHG82MSw3hnlmZPPRFCTqVjIO13SwcFs2qXdUsLoylvMVKSph2QGtrgJFJRkx2QVr6S0xIC+O61Qf4tEjwYsiI1LG/pps9VZ28sqKAdouLLKWUyyck8e6OKnZWdKJRSHlyUR7pEVq+vGE8B+q6mfbsNl5fMZzrp6T2axMDXDAynm+OnhpdCTk5cPP6g9w4NY0IvZI2i4usKB1KqQS/P4BIJOb6NQeIMqh4dEEOZ2WEU9lmJyFYwzNLhnLfp0d6d9QKqZg7pmcQopFx54wM7vzgECankPrb0LPQlDZb2Hyshc+uG8cDnx2ltNlCY7eTmCAVHTY3F41LJEKnZPWuai55Zy+5MXoun5hCSaMFjULKrOd+Jtao4ooJSUxOD0csFqGQChb99V0OGrsFR98QjZyqNitPLRlKs9nJg58f5aIxib3nHqpTcM+sLP74+VGu6LngtFmcvHvpCP626QQn2iwkhmhYOSYBEYIPxm9N2XS4vQyNMQguuwgdhe9KmumwupmeG0mX3c2mYy1cNDZJyNAI1pzRYO7OGRl8e6SZeUOjKWsxE6ZTcfP6Ikp7OkkRegXPnpeP3e3lvBFxHKrv7rco/mFOFlqFBNkvCuUwrYL8OCOddjdLR8SxZnctJ1qtGNQyRiQa2VvdRZvFzdlZEXxxqJGWHkl1TYed7Gg9rWYXcokYvUray0n6vEiIXXjqu7J+5+L1B1DIxISo5Tw8PxuvL4A/AIYQKZmRWrT/IRf5fxQSsbiP3P6XSAnT0GRy8vYlI1BIRP8VRYjD7aW43sSV7+/vcy2dmBbKnxbmYnJ4UcjEVLfZ0Kpk/G1ZPsvf3N37HQtSyUkK1TAvP4Y9VZ3cPzuLcJ0StVxCfLAajULCF4caOdpo5q2Lh1PXae/3/f4lIg1KlDIxXx85tRGsarf1WsOfxEsXFmBQSUkL16KUSVDKJP9xhfF/TSESolWQEKLm8glJbD/R3o/hnhCiRiOX9rHmPd5iQS4Rc1Z6GJ12Dy/9VNGboxAXrOKheblUtFq4YUoqt35wiOOtVg7WdjM9J4Ive7gdY1NCqO20c+sHh1g5OoGcaAML8qMoTAjC5PBS1mLhzo+K+7zudWel8ofPjnKs2dLb6o/UK5mfH8V3Z1CEhGjkLCmMQSIWU9YsRD4XJhpp6HJgcfbvAm0/0cHS4XGMTw3B5hKkq0qphDcuGsGT35QyOTMcrULK/bOzeHhj3121QSXj3plZuLy+ftHnd0zP4KeyU+6EzWYns4ZEkhNj4LwRsYRqFUTplZS2mIkzarhrZjZurw+JWERFq4291V0kh2rYfqIdnz/APZ8IqpVnlg7lta2VHG+xEGtUCwuqCB7pObacaD3z8qN58Ydy2q1uHvjsKDKJCJ1SxmPn5hJrVHOgtouPekZzXXYPV7y3n9dXDuf7khZe3lLB9ZNT+eL68bRYnAQCwg9dJRPT0JNxc7Jb0mETeAgnL0hfFDcxNjWU0cnBPDw/h1vWF1FyGoPdqJbxziUjWVAQw95qIQ+n0eTgzukZvHvpCNbtruO7khYUMgkT08Lw+WFfTRcHa7uIC1JjD/Lwc3k7q3fXYHZ6e0cL40/jgahkEhYOiyEzQsdLW06wdHgsqeFaXvjhBGNTBfVCq8XJs5vKqe9ysCA/mgfmZBP8K7yFFrOTDqsLl9eP2enl9a0VuLx+xqeGMj4tjE6bi+Vv7uHxc4cwIS2Mp749zu3T0/lkdy13zcjgiW/L+nSYpmVHkBaupdvuxuTwkhCiZckrO/twpVrMLi5/dx8fXzuOuz8u5t5ZWbSYneyp6sSolrN4eCwhGhmxxv6KmRCNnCMNJnZWtHPlhCR0Cinv7azhvk8O8/D8XC4clcA3R5u5eGwC7106ivV7a/nycBMSkUAMXJAfTW2XjSvGJ/N0Tyf0UL2JFWMSmJoVzubT+B9iETw4N0cI1guIkIrFfULrfs8wqmWsGJ0wYCaPSiZheEIwYjFo5BK6Hb+eUv5/BQ3dDi57d18/Xt/W8nbe3F7NVROS2F/TTahOTn2nHZvLyx/n5nDN6gOEaOT8bVk+V6/a36eoDtMqeHJJHnKpCI1CysvLC3h3exXpEVqSQ7XsquwgPlg9YCRBSpgWnVKKTCwi+lc2FBq5hFCtnAi9knD9v38Ecyb8V/mIuNxuWq1eWsxOnvqujF2VnSikYhYOi+HqSSlsPtbCmj1CZ2H56ASmZIYTplNQ1W5j/gs/94spP+nhIBHDvBeEXI6RScEsHxXPjeuKmJAWynkj4ojUK9EpBS3+B/vrWXXZSHZVdjIzN5KNhxp5b1cNVpePUK2cm6amkRCi4ZpV+3tfLz8uiD8tzEUugWazm0ve3ttH4aFTSHn/spHsq+nisa9LiQlSsXx0AtlROvQqGVKRiIUv7+g3G182Io55+dFc8e4+FhXGMjYlhKwoPTaXIPv1BwJE6BVYnT5W766hzeJiWHwQs4ZEc7i+i7xYI21WFz+UtqBXyjgnOwKby8eSVwXte7BGzl+XDuW5H07QanEyOimEAAGMajkzciJZs7eWeXnRgqzyi1NBdxKxiKsmJgPw0k8VQibJjHQyIwWZq0Iqxunxs2ZPLRanh7OzIkgK07JmVw2rBnDhXHPFKBJD1Cx6eWef3AuA11cWEmlQ0mp2IRGLMKrliBCCD3dXdpAUquW2Dw/1+Zuvb5yAXCKi2+lBhIiiui4iDUr2VndR22nvXbDOyghn2cg4JCIRWqWEKIMKs93Dtop2wnUKhsQE4Q/4SQhWg0jUy/c4CbfXT7PJwYIXd9Bp77s7XTkmgVvPSR8wJdPs8NBqcdJpc7P0F2Tik5CIRXx5w3gyo/r/rjxeH0X1Jm5ZX9R74TSoZDwwJxu9UsrBum5GJ4egU0r5sayVd7ZX8/z5w9hR0cH8/BjOe20H07IjuXB0AvuqO7G6vExICyNMK8fnD2B1ebE4vRyq7+bJb48PeHwzciK5c0YG+2u6qO9ykBOt75HZin71gtppc1HVbqPV7CLOqEIll+Lw+FDKxBhVMqQSMSa7m6oOO8dbLUTqVSSGqAnXynH5AhTVdZMTrWft3jre3VGNxxdAKhbx0LwcsqP17K3uRKeQMSLJSLRB9R9v3f7PoqbdxrObyvn0UENvMRmskfPK8kJigxXgF1HVYSM7So/xP9ik7V8Bp8fH50WN3PmLxOaTUMkkfHnjeOa+sI0nF+eTFqFle3k7mVE6Ln93P/fNymTt3rpehdnpiAtW8eL5BSx5dSfDE4zcPycbv9+PyeHlro+LeXheLrd+UNSbVAxCwf32JSNQSsXUdzmI0Cs5d4DrO8DlE5K4emIyoQPI5P+3MegjcgYo5HLiguWEaKQ8tXioEJgmFhOqEwxrLh6byPz8GMRiEcEa4QIfCAT4srixXxECwsjihR9O8Ic5Wb0jDIlYhFYh5Y7pGZyTHcHOinZuXX8IpVzMmxcNZ+7QaBxuH2dnRbCrshO9Ws6qy0YhFoloNjlJj9Ly0b563rx4BGKRCJlERJfdwxPflvLgnBz0ShmvLC/kz18d650L3nx2Gja3j0e/PMb1U1LJjw3C5vbi9PixuZ3EBqn47uYJVLbbsLp8OD0+vj7SjKLH3dLm9rF2Ty3hOjlxwWqC1TLCdQoCgQAefwBRQMSl45MQI5B+nR4vmVF6JGJIDFbRanbRZXPjDwTodng4b0Qc6/fWccvZAsdm5ZhEnB4fm0sFY7DRySFYXF40MgkquYQ/fHa0z/vq8wd46acK/rYsn1CtnDarizs+OszCYTHMz4+mrNmC0+PjkrGJyKQiPF4/D3x2lG0n+idLFsQbCdMqqGyzDjhvDdbIabO4aOiy88DnfUO+FFIxf1mU1/v/aIOSV1cUIhaLWL2nlqK6bsJ0SpYMjyXaoCQ9XEuH3cN1k1PxBwJUtluJN6qQiMXY3EInI0QrZ2JaCM1mNyqZmEiD+oxtUrlUTHSQig3XjuXdHVVsK+8gRCukLQ+NDTpjVLdeJaO43kSz+cxzfp8/QKPJQYxRhe40CWwgEKC6w87yN3b3ubCZHB5u+/AQqy8fxdbjbbz0UwVXTEhiYX4Mz/9wAofHR2KIGrvbi14pZ8OBBr492syXN05ALhbj8vo43mIhQq9EIhKjkUsHzNw4iZ2VHTjcPiwOD8MTgogPVuP2+nF74WiDiXari8RQDaHaviZmwRoFwRoFHVZXjyGViIQQNX4CNJucdHU7e9RSHgrjgxGJ4JJ39jIpPYzzR8aRFq5BLpVw5fgklo2Iw+TwopKJUcklyMRizhsRh0H1f38UkRCq4bbp6Vw+MYnqdjtGtYwogxKDUoLHD10uD5mR//eLEBDGmPXdZw5KdHiE5N0HZuXw6pYK7p2dhVQswuHykR6hZWickXs+OTLg39Z1OrC5vbh9frZXdFDf5SAxRIXd7aOu08GfvjrGn88dQrvFTXWHjeRQDUaNHLvLi9cnIT5EzXOby3l1RSFX/8K8cFxKCBeNSfi3FCH/KP6rCpGTUCvkqBX9LyYSiVCUnA6728fe6oEDsgCONJrwBwTzGIfHx/z8aOKChSwFmUjE2ZnhRBtUtNtciBFUD1aXj/ouB+E6BR/sq+MvX5cyM1fYAd60rohDdSYyI/WkRmhpt7oI08q5cUoa/kCA61bvZ97QGP56Xj6CGECERiHh1S2VzMyNJFynQCkT09gtpDNaHB4aTUK+TLhOwZXv70Qtl7JwWAwXjornRKtV2OFLRdR2Oiiq6yY+WE2b2cWh+i7OLYijsdtBariWdXtqqWy3s6OivScJMoSH5udw2/QMdlV0IBKJ2Hq8jQtGxjMzNwIQccWEZF7fVtnHzv67khYmpYfxyPwc/vJN/9n7SXy4r545edG8s6OaJcNjSQrRcPHbp+TEf91UzqgkIw/Pz+XeWVnc/9kR9td0994/NNbAXxYNYevxVspbbWRG6nozGEDoJIlFIv7ydSm3npPO8lHxfToqUzPDKW8RRizD4gw8umAIHr+fc1/qa73/7dFmbpiSyszcCB787Ajz8qOZkRNJqDaEP391jE3HBFMvrUKQ3s7Ji0JM3zyfM0EqEZMUquHeWVmYnd4eDsPft8c2qGW9JN6BIBaBWCTInk8vRDqsLr4obhxwdwXw/A/l3D8nm2Wv7eL1bVVMTA8jNkiFPwAmhxeDSkZTTwFkdfkwOTy0mJxkR+u5atUBwnRy1l4+Gp1YQlLomc8/JkiQOT/85TGuPysFmVjMsjd2c05WBIsLY5FJRdyw5iDTciJYPjqhX1EW8ouRU7PJgcXppbHbgbUn4fqZ74+THxfEqstGMfO5bSwujKOo3sxzm8tpMjmRiEWcnRXOvTOzSPiVY/2/ilijmlgj5EQbem+zu72oRRDxOzFn+5+i3eqiqdtBZuSZd/QRegUKmZiYYBUrxyQi6+HYRBqUvHDhMKrbzlzEgLCBlYmFTq9CIkRMxAWrEYvgRKuV9XtqWToijlhjKJ8famDr8XbWXDEKmUTM/ppOhsYFsWpXDeuuGE19t4MOq5ucaD16lZSYX/Gs+k/Cf2Uh8o/A6/MRazzzjy5Sr0QsFi7oo5KCGRJjoL7Lgcvr46P9DVw8NoEQrYKPDzYwNC6IuzccRquUMjE9DJlEzKKCWIbEBvHijycYnRTMlRNS+Ms3x0iP1FLVbudIg4lPDjZgc3mZkBbKSxcWIhGLcHqEoLQjjWbGpYTSZnWxbEQcLq8ffwC+KG7sU0BlRel47NwhzM+P4aP99byzo5q91Z08Oj8Xu8fL5e8V9THAGplo5NGFQ2jqcpAWoeN4i4VxqaHMzpNzzaRkxGIR2ys62FjcxDnZEQzv4aMcbTSxfm8d8/KjmJ4TRXWHvU8RchJbjrcJZlq/klvSZHKwuDAGmUTEzNyoAT1Ndld18d3RFtIjtPzl3DxcXiFnJUSrQCuX8Pb2at7dVcPVk5L7+ZHcOi2dvdWdROiVvU6tJwsRlUzClZOSaTI5+ey6ceiUUqrarTy3+cSA3bEXfzzBOdkR/HnhEHRKGc0mF3/66lgvpwjA6vLy5LdlgqR6ROwZz3sgyKUSQrW/nWAWoVdwtMFETrS+zzGcxPScSGo67AxPCO5ze7fDM2AL+STKW6wY1acKl1W7apiUHioUrxYnnTYXJ2NvJGIRgYAggV30yk4CAWg1u6lss5IRqWdBfgxv/Vw1IBn6iglJvL29iiiDkqlZEb2kv++PtVCQEMSPpa3cPj2Da1btY1RyCCMS+56Hye7B7fXh9glqmC6bGwgQE6Sk2exixZgEVo4V0o2/K2nm0rGJ/Fjais3t5capaeiVMiRiEdFByn/I9rrV7KTd6iLgDyCTigXbeKUUt8ePRilBJvnPIgn+o/jl+PA/EXaXl3abG7dX8DGJ+CfN1DqsTiwOL69tq+Lms9NICdMM6MV089npaOUSOiwurE4PIRo5Hq8fiUREu8mFSCRCLhEP6LwrEgniBLfPj14lxeH1oQ9IUUhF3HR2GpPTw9lW3saLP1YglwgduevOSqO6w0LALyIrSk9Fq5VlI+J58tsy7B4vQ2IMZEZq0Stk/1aTsn8E//nfqn8jumwutp9o54JR8WfUc18yLgn8Af68UCBE7qjo4PWtlbx4YQGBQIBms4sog5Lbp6Vz47oibj0nnR0VHb3W5tOyI1hUGMuhum7e3VnDqysKeWPlcJpNTraUtbHhQH3v7vTzQ018X9LK25eM4LWtlYxMCiYrUodaLmZYTw5MuE7Biz9W9OviHGuycO/HR3js3CF8tL+eMckhDIsPQqOUcvFbe3rdNU9iT3UXf/3+OIsLY3ljW2Xv+Z90mnx7e1WvbPeZ74/z+fXjuGHtQd6/fBSPfFHCxuImLhiZ0C9tUiQSMneC1HI8Xh/D4oL6EV5PIifaQKxRzfScSH4u72/IcxLv7axhzRWj+KGsldxoA61mJ9etOYDVdapgmD0kmpImM7FGFSmhGi4Zn8Q3R5oZFm8kWCNnW3k7Hp+fobEG4oPV3DA1jTCtnJQwLRq5lLe2VzE8IXjAogp6xnIiEdVdDqRiF4gYsAAAePGnE+RGG2g0tZIfF4RYLMKolv1LA9HCdUpGJws20U98U9rnuKdkhnNuQQxhWkWfxF4QnEHTw3X8VDbw+50QosbjC1CYYGR/TRcdVjdLh8fiDwQYnxrGzOe29j52Vm4kkXoFC1/eQfNp36+//XCCF5YNw6CW8uIFBdz6waHe7o1YJEQhBGnkhOuVPLk4j+J6E0NiDLy+spCmbieZkTpm5EQSADbeOAGHWxj7+PwBmk0OglRyjBo5nVYH4QZhdOjy+fH7BefWxBDBRKrZ5OSC0QkEa2SoZBI2l7aSEqZlw/56siJ1rBybSFyw+jcvYh0WB15/AJVMgsnhoa7NRpDag93jZfWuWuJD1KwYnUCcUf13FRGD+OfQ2G2nvMVCfZeDVbtraTG7uPWcNGYPicao+cdGana3n8MNZhweH98dbeG584fx5DdlbClvIxCAILWMG6ekMT41BH8gQGa0nspWK34ChOkVFNV1UdVuJzVcy/mj4nm355p/OoRwUOH71WFzo5SK8fiFImbOkGhWvrWnzwbxYF03o5KCeWheDluPtxGmV+Dzw7o9tTw0L4eDdd2MTApGKZUQ8Ssp1P9pGCxEfgUN3U6uX1vEj7dN4s8Lh/DHz4/2VrUiEVw8NpGsKD1Or08I1Spu5NWtlQQC8PzmcsEnwelBq5SilIp5cG42t39Y3IcF/db2ar4raeGV5YVc+MZuxGIRSrGY3ZWd6JRSXrhgGAdru3l5SwWBgDCPfH1bJbFGFU98U8qMnEhuPSedKRnhmF1CiuRALnog2FJLxCLeungEe6s62V3Vwajk4H5FyEl8V9LCwmExTMuJ5KP99Xj9AUwOD9etPsBrKwvZWSmkE4tEArHS7PSy4o3drLp8FE6PD61S2kfeOzcviovHJbLjRAcNJgc1nQ6WjYxjzZ7aPqMArULK0uGxLB4eR6vZya3npPPh/oELQYAuuxu728eUjHDu/eRwnyJMJIJ7Zmaxdk8NKWFaxqWE0mIW2u5fH2lmRm4kc4dGMyktFJVMwksXFmBUy/uQEL1+P3urO8mPCzrjMYRrFSjlEvZUdeLxBX517GB2eDE5PdzxUTEhGjkvLy+g3eIkVOtBLBaKvX9FUZIYKsj1Hl+Uh8Pjw+H2oZJL2FnRjtPjIzVc2+9v5DIx5xbG8Nb2U5b3YhFMSg8nJVzD1Mxw3F4/mZE69td0MSkjjOxoA26vj2l/3YbbK/zNiEQhFG3ikz/1G/Mkh2pptTiJNKgYnRLMNzdPoLrdhsPjIyVci0IixuX1M2dIFFKJmGiDguL6LoxqGRPSQmmxuChpsqCQibG5vELx/UM584fF0tDtID8uiO9LmllYEEtpk5U/fnGU+i4HUQYl98/OQiQSIZUIAZUEAhiUcuQyoeuWHa1jyfBYlDLxP9S9aDHZsboF9VeHzY1aLiHSoMTjD9DR5eay8YncueEwa3bX8sFVYxj6K9+lQfxzqOu002V3UdfpxOrycv1ZqaRF6Fj+xi5EiDh/ZPxv7hB4vH4aTXYmpYcyPDGIilYbDrePP52bi9Ptx+Pzo1EIHiJyqQir08+a3bVcOCoBl0e4/9YPirn+rBQyI3VYnV6uOyuV93dVY3Z40cglLBkex7kFMRzr8RsJBITNy1mZ4YiB9fvqBoxp2F3VSXmrleRwDU63n4ZuB3PzozlY102EXoG0J0X594TBQuQMsDjcfHm4kftmZ9LY7SQ1XMPn14+jrEUgSmZHG3C4vXTaXEToFGwqEaLWTzLMt51o59qzUvmyuIm5Q6P58FAdMUGnpFg6hZRQnYJ2i4v6Lgfbytu4YEQcZrsbjVLGT2VtHG0y8+pWwS/jjmmCHBJgS1kbn1w7liWFsZidXkqazEQYlET2+GacCTKJQH695+NiWswulhTG0m4Z2CsABEKjzx9gb3UHw+KDehd4t8/P7spOCuON7KvpIhAAlVxCuE5Bq8VFm8XFs5vKSQ5TM2tIJC/8WMFdM4Tk2qWv7uolja7bU8dXN4zn9ZXDeWRjCeWtViL0Cp5Zms/b26uZ89w2/AHBbfTeWVnMGxrN54f623OPTArmSIOJiemh/PncITR1O9lR0Y5UIqYw3sgnBxv6/V1BvJH3LhlBfbeD6g47o5KC2VfTyXdHW7jurFSyovS9XAypWExymJaKNkElUDKA4+wbFw+nqK4bg1rOvupORiYH93vM6Z/D6bugJ74p46klefj8ASRigbTs8wUEntH/UO8faVARoVfSanHSbnXj9weYPyyWCJ0CqaT/rtyglCFCxHPLhnHPJ4dJDtVwyznpbC5t4UBNl9BFGBlPbrQeo1rGtOwIVry5h4lpoXx8zVj21XSRFq4l2qhi9nPb+hUhYpFg3KRRSok0CEZfcokYfyBAeYuVdXtq6bJ7aDY5mJ4ThUou4UiDieWjEzAoJGiVMh77upQfSlvx+gNMSg/llnPSuX16Jote2cHaK0ZjdnqZkx+D1x9gwwHhYm5Uy3h8UR53byjuU3hr5BLWXzWaYLUMpUyGTPKPj0/sLi9NJsGiW6eU8ZdvStlf04VeKZgHTs4IRyQSces56dz6wSHu+OgQa64Y/buxfP89oN3igAC0mFxsKW9DLZcQG6xmW3k7668aw9JXdjIlM5yo3xDsZnd7abO4cHn8vL+rhoQQDdnRekwOD3uqOlm9q5ZOm5vRycFcPC6JykYLccFqPjpQz/IxCchFYsxOL0a1jJm5Uby9vZopmeFsPd7GQ/NyUUjF+PwBLE4hGDQ5TEuYToHD7WNoXBAXvrGbdy4ewRcDXOtOYmNxIzeclYrV5SMnxoDb6xcKEJ0C4+/Q12WwEDkDzE4vIxND+L6khWaTsLguKojB6fbh8fkpbzEjEYt5ZGMJb188gje2V/HAnOxee+tAQLjIbT3eyvkj41BKpXxxqJFog5LbpmVg1Mjo7Nk52d0+Pj/UyB3TMpBLxdyyvoj752Rz/uuC9HLNnlpeWV5AqFZOt93Du5eOYF9NFx/tq2diRiizh0ShV8opqu0kL86IVCLCe1pYWKxRhUgEeTFBvL+zpnfePj0nckCL9pNQySSEaBVYnB5Uv3CxrOuy9/FN+PZoM/fNzuLm9UWUNJkI0cj5/FAT7106krJmC+NTQ1n8ys5+ypUuh4djTWbOGxFHrFFNfLCaK97b12cnUN1h56pV+1lz+WhWjk5gc1kLa3bXYXIIHaDLxifxY1krI5KMlLdYqeu0c25BLLd/eIiXfjwxIAch0qDkro+LKW0+xRspiDdyzeQUlr+5mz8tHMKCYdHIexalRQUxXPrOPh5flMfFb+/ps8DGGVWYHR5sTi8nWq0kh2mQiUVnTKidmRvF5lLBD2ZqZhgPzBXMsDqtLlQKGV6fn06bG5fHj1QqQiUTo1XK/+nQMJFIRIReRcTf8RGwu7xYnR7Kmiwkhmp4/9KR+AIBLnh9d+/o5EBtNxuLm7hrRgbrrhxNeYuFE61WEkLU6FRS8uMNmO0eiuu6efzcPO7/7AjdPdJDo1rGH+ZkExOkJDro1MgjEICGLgcen592q9DdWjAsFp1SSnGdCYfbx9znf+aV5YWYHW7umZXF/bMzAREOrx9RADw+H1/eMIFdlR38dVM5JodA1j5vRBxPLR5KXaed5zaX9ylCUsK03DUjg/013VhdHuKDNQyNCyLaoBywSBsIVqeHynYbN6w9yNNLh3Leazt7v29mp5e3t1dzsLabS8YlkhquRSYRcbzFSrfdM1iI/A9hc7ppsQimihKxiE6bixijijumZbDird18VtTIBSPjiQlSsnJMIm1WF1KJiLBfUZG0mhxY3T4ueWcvNad5O+kUUt69dCQ+n599PZb/le02PjnYyAdXjcbp9mJz+XC6feiUUsxON385dwh/+bqUbSfaabe6uGFKGt0ONzaXj5QwDQqZhOoOG1Vtdv60IBe9Ssrb26vptrvRKaUY1bJ+dgOnIEIuE9PeIbi8xgeriQvWYlBKf5eS8sFB5QCwuT1Uttm49N29bC1vIz5YzeeHGrnknb18uL+eTaWtPP1dOTetKyIQEIiE3XZPH8vplDANZS1WrpqUwicHG/D4/YTpFLx4QQHhegWlTRbKmq14fQFEIhH3zcoCoLTZwsVjE7E4Pb0SYhDadLOGRHLbtHTe31nLhv31PL44j8KEYN7aXs1jXx/DFxC6GA/PzQFgfn40b188gkvHJbFidCLXT0nli2LBMlsjl2BxepBLxWccOVwwKh69UsKEtNB+EdPpETrquk79UJ/bfIJYo4rVl43iaIOZi8YmEgjA3RsOc+eMTIrrTQMqMeq7HGRF6QWZ2lfH2FfTOWA7MhCAJ74ppdPuZmRiCKsvH8XigmhevrCA93fWMG9oNK9trSQtXMvT3x9n+4l2bC7vgEVIiEZOQ7ejTxECcKC2i2+ONDF7SDQPf1HSx3wo1qjmD7OzeG1rBeuuHM3S4bGkhGkYnRzMy8sLePPnKvJiDWw+1sLM3Che/qmCJ5cMJewXi82IRCNzh0bzeVEj2VF6/jA7G5PDS6fNjUIupa7TTrfDg8vnZ/3+Om7/sJgtx9up67TTanJgdpy5g/U/QV2njcMNJrrtHrRKGXurO1HIxNz3yZEB1TdPfXccu9vH9WuLACHHR6uQkhWpp9Pu4cZ1RRys7WLtFaNYe8UoVl8+ivcvG0VSiJpIwynyoNnhod3q5oviJq5bc5DPihr5vqSFOz8q5pnvjjM9N4KZQ6Jwef08+W0ZTWYXN687iM3tp9Xiwufz89zm4zR0C+345DAN+p6UW6vLy5s/V7GtvJ1JGWG9CwhAlEHJH2ZnUdtpJ8qgxO0N0GRyUtFqpbzFiv8M1ton0WJyUNlmpa7Lgdfn56ULC3j0y2MDft+K6roRIXyHfw9kz/9EdFhdHG7opsPioMXsoNXswOT0opNLkIgDvPjjCcqarTSZnDSaHKy6fBQgbOKkYjHTcyPx+gLc8/Fh2q0Dd407rS7sHh/3fXqkTxECYHF5ufy9fRQkBJMZeSqry+Hx8dAXJehVcgrig/AFAjg8PpJDtUJH5oSwMd10rJX5L27nprVFPPltGQtf2sGXxU0QEBETrCIpVINWISMhWMO7l47kUL2Jqyal8PrK4X2MC09iTl4kMrGYzw81EqyRE6aTE6GXE6b/fY1kTmLwVzEAOq0eHvmyhEBAWCgjDUoMKhkmh6fPxQzg6skp+AN+glSn+BAikcCkPinBClILuu/zhsdRVNfNPR8f7mNIlhuj5+6ZmRxrNLNqdy13z8jEFwgQZVD22iyb7B6mZUcSa1Tx7KZyPr5mLG/vqO51CgX4+kgz2VF6XrqwgCcX59HY7eCyd/f2Xhz/smhIn2OXisWUNpu5aWoaq3fX8ENpK/4AKGViLhgZT3KohoZuJ0a1nBFJxt4kYJ1CSm6MoTeHB4TCxusLEKyRcc+sTEQIpmSvbq2k0eTAPIC7Kwgkq2smp/Dg3Bx+Lm/j0K8krhbVd+P2+rny/f2cnRXOfbOzWP7GHmbmRiKXiJmbF43Z6SVCr+TNn6t4eH4uN6872CenRiEV8+yyfB4bwDIeYGNxE88szefTogZOtFhxefyEaWXY3X5SwjT8YU429V12FhfGctXEFORSYUGt7bCjkkuYmhXB3zaXc+u0DNbvqeHe2T0eMz0JvBVtNm5cexCX189TS/L46EA9LRYXUzLCufOj4t6cILlEzHVnpZAdrefGdUUUJhh5dEEuGq+fQIB/ma12p02wZ3/2++M0mpw8uTiPlDAtB2s7yYrSDTiGAqHgLa43EaEXWsrTsiNRyyR0WN0MjQ3ijmnpvLq1kje3V5McqkGjkGJxelh12aje0YfF6WHb8VZSI3RMSg9jZFIwnx5s6O0qlrda+epwM3PzIpFJRJQ0mblxahqH6k0caTAhFon4qayVKyamsPCl7bx36UhKGi1cPTGlj83154cauGJiUp/jv2x8Em6fn5/L2/npNE6VRCziwbnZPSGDwm0ysbiXj9Lt8KBXyfD7Bc+V2g4b3x5t5qpJKb+qNjpY101yqAary0NquJYg9d+XYA9CQIvJyd0fH+LRBbm027wU1XXh9wfIiTH0RHfIuOXsNF7fVoXL56fF7GDe0Bhm5Uby1ZFmfiht5fopqfxc3samY63UdtoH7EZVtNvQKaTsrOjvRwTCb6Wx28HdMzK4+J19vbfvq+nC6fVzz8wsglRy/AEhu0s+QFet2+Ghu8eR2ecP0GV3E6QWVDMKqZg2q4sVb+7pfbxCKuaP83LQKKR82xNnMSYlhMxIvTAir+rk1nPSiQlS/sfkxvwzGOyIDACnx8fxllO75eL6bt6+ZEQfGa9ELOLScYmclRGGTilj9eWjUMnEfHDVaLbcMZkhMXq+PtzEopd3cO3qA7z0UwX+QIC7f1GEABxpMPPFoSYKEozUdNi5ZvUBfH4h7O0kxqWGMizOwKG6bhYVxtBsdvYpQk6ipMnMxsNNxBpV/HVTeZ8d2rbj7UzrSQu1uX0o5RLCdAquXX2AuGA1r64o5MULCnhmaT4tZhePfnkMl9fPhW/uZkZOJEmhGlLCNLx32Ui+Pnxqfjk6OZgPrx6Nx+fnre3VlDVbufy9fbi8ft69ZARqmYS0XyT+nsTBum5Km80kh6q5YUoqscYzSyVDNYreomLTsVZ2nOhg9eUjGZsSQgA43GAiyqDkignJ1Hc5eOKbUp45L5+7ZmSyqCCGe2dl8tbFI6jrtJ9xgXV5/ZwMoA0At6wvorHbxdFGMy9tqaSu0873R1v4vKiRj/bXUd1uQyWTMDolhH3VnSwujOGsjHBe2VJBYWIwwWoZyaEaksM0fF/Swl++LsXh8RGpV9JidvHa1iqWjYjjujUHeosQEHg4f91UTlaUnnCdgv01XRxrMrOlrLVP1sVAcHi8NHTbqWi1sOV4K+v31lJU29WPP+Rwe3l3RzV3fnSKN/Hq1ko6bS6uPSutj5vjQBABWZF61l4xGr1SygOfH2XyUz8x5emfOFDbzbuXjmRqZjh2t4/RycG8f9moPlLYDqubVqubi97ay7WrD/DIxhJyow08tSSPk1Oojw/Uo5JL+/1mPjnYgEYhYVJGGCVNZlLCtLy8pYLRKSGE6/suMv4AdFrdBJ02hhwSY6C4wdSnCMmM1DElM4zyFgtyqYgWs4vKdhtlrVZaLC6kEjESkYhdFR1c+s5eznl2K3/bfILZedGYHB4Uv6KE0SmlSKViJCIxjy/KGxzL/Ea4vX5e3VrBIwuG8ElRIzP/tpW7Nhzmnk+OsODF7Ww40NCb1LxgWAzvbK9i/tAY6rscLCoUJPIur+Cs+3gPx+6rAYIUvT4/a3bX4PwV7x0QOF1apbTfZx0AQrRyxCJhI9fZY/A47FdIyXmxBoxqOfVddqQiEfuqO/nkYEOfx7i8fu775DArxyQwKjmYJxYN4dH5uWjkUp79/jivrywkwiBH9zs32RvsiAwAqUTUm96ZFKohNVzHLeuLuHaykGroDwjt/SaTQJDqtLoRi0XY3D5+KGsgzqhiUnoYc/MiuWhsAl02DxaXl6ZuB1KxEJHd/YuL/GdFDawcndD7/ye/LePScUnsrOwkWCNnZm4UTq8PuVTMOVkRfH5o4FRSgHaLizUDWJ1/V9LMmxeNYOvxNlrMLtbvreWSsYkUJhh5e3s1b2+v7vP4qyYm80VPIu1zP5zgpQsLUEjFvPlzFRPTw7lsQjJeX4AdFe14/XDHR8U8du4QLn93H15/gOMtVt7fVcOz5+XTYnZydlY4m471DxyMD1azu6qTkYkhzBoSyfM/lA/Y4j5vZByfFZ36oa7ZU8uIRCO1XXZCdQo6bW4e2XiM80bEsagghg0HGthS1sbZWeGYnR5kYjGN3Y5f5cVE6BWYnR7CdAqsTq8QxiYChVTElMxwLus5t5N4eUslN5yVyiXjEln8yg7ev3QU+6u7mJsXRYhWcKr99GAD7+6s5s7pmaSEaXl/Vw2zh0TxypYKJmeE8enBhgHPF2DtnlrOLYjhlS2ChHpiWigurx+Tw9PvPHw+P90OD18WNxIXrOH2Dw/14ahkR+l4feVwYnqKvW67h8xIHWuvGI1YFGDbiQ5W7aphzZ467pieQWO3g5QwLRVtfUdYIHT9xqSEMC0nggAw/8XtffgXm0tb2VnZwefXj++dd5/uIOv2+viiuJGnvzvVVeu2e3h5SwXzhkZz8dhE3tpe3atAEItEZETqqO4QfBwC0COVlfJzeTOp4Vq2HG9DKhbh8gjFZJ/wChFcOj6pt4unkktY3yNJn5Aayl0zM7G7vXTbPaSFaymqM/HwxpLe4k2vknLfrCzGJIeQHKbl2WX5iEUinttczmXv7uP9y0aycFjMgDJ/kQjGpoQK5oE3TSAu5L/DDOxfgTari7hgBfWdDp76RRyAPwAv/HiCIbF6UkJ16FVSJmeEEwgIHeQYozCmmJcfw7rdwncJQDFAp0LoTnjQKWXoFNI+m4LTkRqmpcXkJC1CS6hWQVdPYSITi3C6vcQGq2kyOTjWZCZcp+S26elc9s6+fmPpRQUxVLXbmJAWSqhWjlYp4ZWtlQO+pj8AOys6+MPsLEw9SetjU0K4c2YmQSopRvXvcxxzOgY7IgNAr5QxIzcSEHgSL/14gpoOOy/9VIFYJEIrl6KSSciI0AlqEb2S17ac4NYPDrF+bx1PfXecBS/toKrDwY+lwg42RCMED3190wTev3QkP90+mddWFCLu+QScHj+nXzdrOgQy6MzcSF5dUUib2cG5L+0gJ9qAz8+vumaq5ZIBJbmenhnpU0uGcuf0DJrNTkqaLTw0L4fFhbG9rUS9UsqNU1KJClLx9RGhHXii1YpEBPVdduKD1Xx1uInzX9vFdWsO8MWhRvbXdDExPYwP9tb1Wah9/gB/21xOnFHFnJ5Y+5O8idwYPe9eMpK0cB3hWgVby9uo7bDzt2XDkP6CmDk5PYy0cC27qzp7b+uyCQXgmOQQ9lR1Mio5hIUFMXx8sJ5Lxibx+XXjyIjUcdHbe3n5pwoe2ljCmt21xBhVZEcN3KG5cmIyG/bV8+CcbN78uQqAZpMgNX1/Z3WfczOqZawck4CsJ//mvUtH8ecvS0kIVTM8MZjMSD1vbquiuMHEg3NzMPZ8B245O41ZQ6JoMTsJ1yl6rfoHQm2HnfAecp3XF8DrD2BxefvIorvtborqurhzQzF3fFSMTCrB4/f3kyqWNFn405cltJud1HbasLm9aBVSNh9rYfuJDsalhPD2xSM41mTmUF033xxp5rZp6cgk/Umy15+VSqRBSaRBxU+lbQN+3+xuH29uq8SgkvWzsW+1uHqTrn+JL4obmZAWBsDkjDCcHj8yiYibz05j1a4aABYOi+F4s0VwAg5R02lzE6oVYgnUCmmfIiQhRE24TsG07AieXJxHdA9HpcPqYniCkftmZ3HbB4dY+uouNpe20Gp1ccv6oj4dJLPDy10bDtNidvGHz46w8KUd3PbhIa6fksrS4THc9kERl41PIm0ASfSDc7KJM6o4JzuClHBtLwF6EH8fHq+feflxvPHzwIs0wNo9dUgkEK5TEK5ToJBJyIjU8WVxEyMSjUTo5Dzx3Ynex8/Ki+r3HAqZhDlDomjqdnDtWSkDvs7UzHDUcgnHW608tjCPobFBTMoI44E5OSglIsRiMbd/WMwz358gXK8kNULHW9sq+ejqMSwqiCE+WE1BfBB/XpjLpIxwpmaGo1FICNHK8Prp47XzS1R32Hjppwo2l7YSoVdg1MgI0yowan7/RQgMdkQGRIhWwc1np1FcbyLaoKK81YpCKuaZJUMxauRsOFDPhv1CS3BKVjiXjU/i1ukZXDExhatXHaCh24HPH+D2Dw/xwgUFXPDGbgrig3js3CHcsv4QHl+AKVnhTE4P44dbJ3P2M1tIDtP0+SIqpGLBWGtKKia7B7VCykVjE/n2aDPjU0OYnhPBNz1Fwi9hUEl7Dad+iYZuB58ebGBqz+u3mJ08+mUJd07P4LLxSbRZXJgcHj452MAPpX3TRr1+WPnWXkI0cm6blsHsvCjqOu3kxQax6VgLqeFa3vlFVwWEIuat7dU8MDebLpuLRxbkCI6TcimrdtcIYX3n5jIpIwy5VIIoEOCbmydwoLYLs8NLQoiGsmYLt/8ifG5saihquYTyFis/lrZy1aQUVr61h4XDYnD7/fh8Af7ydV8uyMG6bla+uYfnzh/Ghv31fHWkGZ8/QJhWwXVnCR2v66ak8vq2SkqazMglYkQioUP18Pxcpj8rGHb9cV4OOVF62q0ulDIple12IvQKnlqah9cfwOMLYHK4cfn8/FTW1s8gbHFhLMPigqjrcpAeoWP7ADk5AOkR2l5SsLDjrmVaTiRapQSv34/T4+f1rVW8+NOpC+0Ppa3EBat4anEeV7y3H6/fz1mZ4SwpjCU32kBpi4UgtZx7Pi7mcMOpEdXzP57ggTnZPL10KI99VcrKMQms2V3LmxeN4KP99RxpMBGhV7JsRByFiUZ0ShkOtxeNQvBf8fsDfFfSwtdHmvD0qLZ+LGvjZkdfIjcI2TX2ARxqQehkdNhchOsUnDcyHqfHx2srhvP8D0JycF6sgewoPfHB6l5Pmxd+OME9M7NoNbvYVHoqoTpMq+D1lcNJj9Bhd7nRqaSkRRSilInJjxOKkBvWHux13V2YH8OnBxv6jYJO4oWfTnDz2Wlcu/ogJ1qtXPLOXtZdOZoP9zdgcXh4bUUhx1utbCtvI1ynYHqOELvw35DJ8r8BlVyCx+v/FfUItJpdeH3Cb04tk2CyuwnVKUgJ17JgWAwLX9rZ+9iLxyYQcwYJ79jUUM5/bSdvXzJS8BT6qYJWiwuNXMJ5I+K4aGwiF7y2iwaTE51CyluXjMDs8LCnuoPMSB0i4NyCGG5aV8S3R5uZmBbKPbMyWfDSDu6ZkcXKMYn4/AHsbh9dNhcKmRi1TILPH8Af8DMkxtCPg3gSudEGPi1qYG5eNFEGJVGG34d1+2/FYCFyBgSpZLy8vKCH6S4RJLIqGdeuPtDHKvyzokZ+ONbKy8sLUMslPH/BMG5Yc5CGbgcur59mk7DrPVDbzWdFAsP5x7I2SprMfFbUwIsXFPDKigJkYjFPf3cqd2VOXhT7azqFlFOVjAi9kivGJ7LleAeJoVqkYhFZUbp+ahadQsqUzAgcHh+rd9X0syOXS8RcPiGZRz4/yo7Tugs/lrXx5Y3jueSdvQOGw52dFYFEDNOyw7lyYgr7a7poMjnYVdnB9yUtnFsQQ5NJWDzOZMLz3vZqLhmfRHF9N58ebGBzaStxRjU3Tk3FoBTkzO1WF2KxCLlEwnObT7D2itEse21Xv+dUysRcOi4JsQhKGs08MDeHJa/uYPaQKLKj9Zz70g5eurCgX2w3CLLKa1fv59PrxnH9lFRcXj9ur589VZ2s31vHzyfae0clK8bE89XhZn463sZNU9P4+JoxqORSzA4vOqWMtXvr+PRgAwGE92bu0BiGxOhJDdfSYnYyb2j0gFwesQguGp/EhW/s4upJyazZXTugqujC0QnctaGY1HAtyWEaNHIJWrmEd36upqi+mxGJwZyVGUZZs5lNpxWOdZ0Ovihu4rLxiYxJCeXHsla2lbcjkwhBei1mZ2+xcBKBADy8sYTPrhuHXCIiKVTD1Kxwbl1fxMSMMM4tEIy+hiUYiTWqsbq8HKzt4rkfTnCip1ifNzSaV5YXcvO6IiwuL3qVtF93C4Rspl9DqFbBM0uGIpOI8Hr9/HXTcQjAowtyGZ5oxOn28fJPJ3h4fg5vbqtiWnYkU7PCkUvFKOUCJyklTENauI7onoVHrZAjFksQi0T8XN7OPTMz8PkDfX7PUomYygFsvE+iotVKapi2N+Sy2+7hYE03C/JjkMskKGUShscHMT0n8lfPbxC/DRF6Jc0mB4UJxjM6FefG6FHLJdR3OTgnJwKZWIxcJmJBfgzfHm0mPz6IIJWsR0KtO2NYZHSQitWXj+bVLRVMTA9l7RWj8fn9KGQSXB4vC17c3subsri8PPbVMWYPieKt7dUsHx3P1KwIvH54eH4Od204zNbyds5ttnLluET++IUQ7CkRi1g4LJobp6Yhl4rptLmQSCR4fH5un57Bstf6p2WHaOSkhGvxBwIkh2l+lz4hfw+iQCBwhun0vx//SIzw/wYauuwECPDGtmrOyQqnxeLi1g8ODfjYBfkxBGtljE0Jpd3q4u4NhwEEv4W9ddR02NGrpDy9ZChXvLe/9++unpTM+SPjaTE5cHoD3LK+iEiDkqeXDCVUp+gltXl9QlGzsbiRYfFGbv/gEG9cPIKvDjexfm8dDo+Ps7PCWTEmAYVEzA9lrYxNCeWPnx/ttffOiNDxwNxsNDIJWpUQ+PZ9STMfH2ikrMXCndPTidCruP2jQ31a2zFBKt67bCQmuwujRsGeqk7W7qnD4/MzOSOciemhyCVi7tpQzMVjk7j3E+Hc5RIxM3IjGRYfhMsr2KdfvWo/K0YnMj8/Gl8ggEwsxu/302V3E2lQYXZ6+a6kmSExQdzx0SHOyYrgqonJ/HXTcb452oLPH2B8agi3TssgXCfH7xd2Tc98f5y1e+p4++IRvUqhV1cUctX7p97r0/Hg3GyONJg43GDivllZhGgVfFncxOo9NZgdghnRlROT0SikPPDZUUQieGPlcGKNKj450IAfqG638V1JC2nhWv4wJ5tvjzazo6IDrULKRWMTGJEYjEIm5o4Pi9lW3o5IBJePT2L2kGjUcglSCZidPsqazUQZVNz7yeFeyXCwRs4d0zOobrdh1MgpjDfy0BdHeXZZPn/9vpxD9d3cMT0Dt9dPTYedvFgDIpGIR78s6ZUeRvWkBS9/Yzd/nJfD8RYL6/bWoVVIOXdYDKOSgylrtvDwxmN93ptbz0lnQX40VreX9HAdLRYXJrsHhUxMiEbeeyH/ubyN5acx/E8iJ1rPhaMSuPeTwzyxOI+lw+P6PabT5uLit/dS3mIlO0pHl93TO6IK0yl486LhyCUi7G4fCSFqrC4fIoROYYvZRbvNRUyQiopWK3HBaoxqOVFBqt/stVLSaMJk92B2ebjq/QMAhGvlvH/ZKDz+AA3ddvZUdfHR/vo+5OAJqSEsHRHHzopO1uwReFjLR8czJMbAmOQQ5BLhPZLJBscv/yqYHR6azU7mPv9zv2JdIRXz8bVjiQlSYHH60CulGNSnuk+BQACb24tMIv67xe9JuLxCx8Lm8lPbacPq8hJrVNFmcXPbB4coSDBy4ah4rC4vQWohHmDVrlqC1DIyIrSMTgllX3Un935yhLEpITy5OA+Hxy8YmKlkhGsVODw+qjqsgIg/f3mMQ/Umzhsey7jUUP7ydWnvqHN4gpHrp6Sy/UQHiwpi0CmlvRyv/3T8I+v3YCHyd9BsctDdcxF+7KtSvitpGfBxwRo5t56TTkKIGovTyy3ri3B5/Xxw1WgufWdvb+7J6stHceEbu3v/LilUw8sXFmByurn346O8dGEBGoWkn3rkcL2JC9/YxforR1PdYeeJb8uo7rAxIyeC+fkxiEQifiptZWNxEx9cPabXxvxwvYkRScH4/H6sTh9en59nNh3neIuVYI2cy8YnMW9oFIEAeHx+6rschOoUfF7USKvFRX5cECFaOWa7h2EJRu7/9Ei/kU+sUcXz5w+jzeKi0eSgotXK3uou7p2VxWdFDfx8oh21XMry0fFMyYzA6fHyzPfHEQHXnZWKWCSm2ewgNUyLyenl0Y0lXDQmgSazkz9/Vcrk9DDum52JSCTC5w8gk4iR97iTRhvVdNlcPLyxhCMNZhYMi+HJHnb8U0vyeOKbsj6JuyCQY6+elMK9nxzmmkkp7KzsoDA+iIvGJmJ2evH2GGu9vaOqd2QyMS2Ugngj03IiqO20o1FIWfHmHlQyCa+uKOSGtQd7lRMSsbCATkoP44E52Xh8Ppw9Kbd1XQ6kYhFWlxeL00N+nJHjzWZKmgR3xuQwLVKxMB6UiUVYXIJJmkElI9ao4uGNJRxtNPPk4qHc8dGhXnM6EMYQTy7J496PD9NocnL9Wansre6kMMFIs8mJQS1jZm4kdV0OPj5Qj9vrZ9aQKFLCtNy8vqhXKn7JuES0cgmLCuNIPINVfbvFxfmv7+oXJHgSf1uWz/clLTwwJ5vwAbwNTDYXVrevtxtl93hRSCV8tLeO2UOj2VPVwaSMcDIjdEgkYkx2N3aPD6lY3MdI759FWZOZDrsbnULK3Be28+IFw0gO07KppAWb28uweCNWp2A69qevjvWq6NZfOZrr1x7sJWUDPDwvhymZ4YjFEP07STv9vaHN4qSu08E9Hx+mrCcROyNCxyMLckkMUaKVy1Ar/zWSaIfbi8nuQSQGq9PLz+XtBGsVaOQSUiMEntofPj2Cs4f8KpOIuGFKGlaXl/GpoRhUMlxeH8eaLHx+qJHHFuaS/ov03u+PNmNQy1j22q4+RPWcaD3XTEohNliFRi5FLBbh9flRyyVolBKCf0fE1H9k/R4czfwdRBpUBPwBFDIJavmZK2q1XILb68fp8dFhdaNRSJmaGYxMLGL15aPZU93Jd0eae3MFTkIEBAjQYnZS0Wal0+YiOSyYZpMDu8uLXCZBIRXz2tYKzE4vP59oRyIWceeMDG5eV8TXR1r4+sip4ugPs7MobTKTEq6lrNnSm0vz5s9VLBkex/2n+St02tw8+W0ZRxpMnJ0Vzl83lXPL2Wk8u+k4cqkEvUrGW9urkIpF3D49g4O1XQPyTuq7HHxX0kK33U1hgpGFBTGsGC2oSE55eLh4+rvjuL1+JmWE89DcHGRSMWIRuD0BWi1OvjjUyLTsCPbVdHHf7CyazE4empfDyz9VcM5ftxGkkrJibCJLCmIJiCC6p1hzuP1Mz4mkvNWK7bRRzJs/V/HA3GxuXX+oT/LlnLxINhwQxiVSiQiPz8+3JS3MHBLFR/vr+ykfdAopl45PYmdFB60WFxE6JTa38DoLhkWzZk8tGZE6Lh6biNfnx+MLYNTI+byokTaLgwi9ig6bh7ouCxqFlM8PNfFTWStPLx3KCz+Uc/2UNNIj9VjsHu77/DB/mCN4qmRHG7j348MEgJumpnK81ccPpW3cPzuLP315rE8RAoLC4KEvSnh66VCufG8/o5KDeemnE9w9M5NmsxMCgvHczz0mSwB7q7tIDtXwxOK83oV1QloY1e02QrVnbgHb3N5+Rci8vCjOGxmH0yMk3t4/O6u3CPH6hDl/UV039V12hsQGIROLuHHdQRweHxeOjOfcgljOzgnHoJLx+raqHjdYBaFaJQa1HMNAB/JPwun102VzExOk4tXlBZxos3HdmoN9HnN2Vjgzc6O4f3Y216zez32zspCIRYRpFWjkUkK1cuxuH+PSQjGqZWj+RQvhIPojTKdEJZfw9iXDsTgFo0K9Ukp0kAqR6Ld1wX4ruuxumk0u7D1uqYWJwbSYnTy8sYS/LRvGHR8W93m8xxfgme+P87dl+XTYXDg9vl4LeK8vQG2nXQhx7CGdd9vdBGvlPPxFST+13NFGM9evPciz5+UzNDYInVICAQj9nRqV/VYMFiK/AVFGNTaHm2Uj4/m0aGD//3lDo/mprJULRycQY1Ry6bhE5uRF8/6uaprNTs4dFsvji/Not7qYmR1Bg8mJ1+9nwbBYQjVyNLFG9t0/FYvDS2WbtWdnLcbi8HC4086CYTFMzYpgf3Unc/KjeW5TOa+vHM7XR5oobbIQFaRkQX4MWoWULw83UddlRyQSMT0nks8PNXLh6IQ+HJTT8fWRZhYXxtJidmJQyzlY19eY6Q9zsjhcbxKkrGfAF4cauW5yCrd/WMx5I+KwOD19jMTCtAqeXjqUHRXt+Hx+PtxfT3GDiYRgNfPzY4g0KFFKJSjlUrbffRYHa7qYPSSSxm4nz5+fj1wqwe8XFBE+f4DEYA3tVhcen5BUqVVK0cql5MUG9b7msSYLq3fX8sZFw/mxrJXKNhtZUTrm58fw7VGheNtV2cGMnEie/v44jd0OLhgVz6ikYD7cX0+33cPwRCNnZ0Xwwg/l3Dg1HavTg04hxdnTIh6THMJXh5tYUhjLnR8V93JSZg+J5LqzUnF4fJz/+m6hEECYEZ8/Mp5rJqdw9fv7eeviERyq72bDgQYenpfDc8uG0WX3MCophJJGM2kRWg7WdhOuVeLoec2EEE3vrvCXqGq34fEFeP6CYYTpFMwaEsWxJgvPbS7n1mnpfYqQk6hst7GnqpOxKSF02d3EB6soiA9C+ysLq1QiRikT4/T4iTIoee/SkZgcHspaLBhUMrrsbl7dWsnt0zLQyCX4AgGe/raMjYdPEazTwrWsumwUT3xbxstbKjlQ282Dc7PxBwJcPC6RTcdaGJcacsZj+Gfg9fmp7rBhcnh44LOjrL1iJBF6JVetOiBwwfKiiDWqaekZg45MCsHi9LD28tG8vOUEw+KNTM+N5MeyVu6blUVOjAExDBYh/x+gVcjQKv533+eaDhtrdteybq8QIZETreeqicmo5BKeWJzH2zuqzvi36/fWces56VhdXh7/ppTXVg5nztBI9td094nc8AcC6JWyMyZ5A+yv6SI/zoBYJCL4X9AB/E/HYCHyG6FRyUkMUXPByPje2fBJ5MboGRoXhEou4UBNFzNzI5mSGc53Jc2UNlu4dlIKbp+fQEAINbtwTAIRBiU2p48mkwOzUzDkefzrMj4rasTt85MYoub2aRnkROvZV9XJ6JQQPj5Qz3kj4mjocnDFxGTu2lDMkBgDwxODMTvcVLZbcbh9LBwWw9Xv7+eRhbmYHB6Ot1jQyKX9dtCn40SrlcQQDd32/hbiUQYVpc0Wfm3jIRLBsHgjK0YnUJhg5J6PD/e5//45WbyypYIbpqSx9NVdfeTH7+2s4eXlw0gM0VLVbsPr9+P2BSiuN7NuTy3NZhcjk4JZOSaBILWUbpuHjcVNfLi/jk6rG5lUxE1T07lrZiYWp5eJaaFs7XHn3FnRwe7KDsanhnLbtAwO13fz3OZyhicGU9FmY291FzdOTesde/ztvHwSQzUsHx1PbaeD0iYzm4+1cOeMTJ76tozbpqfTYXNj1Mh555IR6JRSlo9J4JMDDTyyIBcQWrs2l4/Gbic3rTvYhzDs8wdYtauGu2ZkkB6ho6rdRlFtF0khGm5cd5C/Ls1n7e5ahsYFMTQuiGsmpVDSZKbd7iYhWE1ejB6Prz+p9XRYnF7e+rmSxxblsWxEHFe+v59ZQ6L4/gxjRYCvjzTx4JwcUsK1xBlVfzdrJUyrYMlwwddlzRWjuPfjw+ysPEV+jjIoeezcIdzzcTFXT0rl+R/KuWN6BuNSQ7nnE6ErV95q5anvjnPDlFTqO+3sruqk3erG6/OzID+aJ1uO/9P5OmdCRZuN+S/+zDlZEbx/2SgkIhEf7a/lvlmZjEwK4fuSZjYeaiQhRM0Ti4dypLGbaIOaI41mChOCeezrY/xc3o5YJPgMpYZreXV54b/0GAfx70FTt4P7Pz3S6+wLQofixnVFPLUkj+RQDTbnwP4iIHSGFTIxW8u76LJ78PkCbClrI0Sr6OOyGqSS02E9lTM2EILUMqRiMbrfYW7MP4P/jrP8FyHSoOLGqamcWxDD+n112F0+JqaHolXIsLu8ROqV5GUH4fb6KGux8N7OGh6dn4tOJcHuFrP01V29BlMSsYgLRsaTGKrm2jUHuGJCEqnh2t4RQnWHnevXHuS5ZfmcNzKu1169y+4mIUTDG1sreeGCYfj8AcxOLzKxiPouO+dkRfLM96W029zEG9XsrepkTEoI0gG8IE6HXiWjy+YekFHu9wfYV93JlRNT2DyAIRkIHaEum4duu5vEUA1yqbi32BCyPPzMzYvino+L+3mgBKllKKQSvj7STGO3gwBgUMl47TSDn7IWCx/sE8ioYToFGZFabpuWjt8vHHtDlwOby4NOKeHuWZmMLWvnnR3VdNqEcdGKMQnsqGjnUL2JTSUtvHXxCDYeasTm9nHbB0Ia6of76vjD50eZmhnOwmExpIXrGBYXxBeHGrlzQzHPLcvH7vbx9HfHezsSd0xLZ2xqKF5/gHt7iq8F+dFcMTGZPVWd/VRLJ/HOjmpumppGWbNFSPZtt3GsyYLXHyAvLog/flHCmstHcev6Q1T1mHgZ1TJeurAAry/Qa7j3S0jEIlRyCftru/H5A0jFYuxuHyIRA6qhTn3GkBquIfI3Br7JpWKunZzC+NQQ/rapvE8RAtBkcnLXhmLunJFJs9mB1eXl4rf38vz5w/oY22061sJ5I2J5YvFQFr28gz1VnZzTo36ZNzQa7b/wQmxxenji21JGJAZzTnYEN649wMPzc5idF8UH++r5sayN/Lgg7p2VxWNfH+PqVft5YE42iSEq9CoZRXXdLCqIZeWYBESIEItFfLy/npd+OsGjC3NRyQYvp79ntFpcfYqQ0/Hc5hM8s3Qo5xbE8P0ZroG5MXqkIjFvbBO6JlqllH01XSwfnUDoaV0NsViEXikEMv7SRBKETd1Jz5L/FtLzIFn1n4TH48Ps8mJze7E5fXj9PoLUcn4sbSVMp+Sa1Qe4fVoGMUFKsqL1LHhxey+56XTcOyuLLw41crjBxJ8W5vLmtqo+BlcpYRpeurCQx74qYclw4Ys7IS2UYfFGPitq4GBtN1OzwsmI0DEyKZidFe1YXD6m5YRzpMGCTCIiXKdkc2kL+2u6BvSrUMkk/G1ZPle+v597Z2WxsbixT27GvKHRKKRiChKMfHqwoY+pGAjkz6eXDuWC13fh8QWYnhNBqFbB6h531/GpoSSFapieEzGgyuKmqWnsqerkknGJXLVqP29dNIJL393LQN/M7Cg95xbEMDIxmEaTgzCdghCNHI8vIOSDiETolFL8gQBdNg/HWwXTqw/31aOUSbjurFTu/eQwuTF6bp+WwevbKtlR0UG4Rs4H14yl2+6hvsuOXikjVCvH5fOjlkt75ZrL39jdS34VieCrG8dT3+XA7Q2gVUqINaoQI2LbiXaq222IRBATpKah2yFwRk4L3DopL04N03DHR8VUtNn46sbxuL0+RCIxepUMiQjabW5+Pt7Guzur8fgCfHLtOL4sbiJUJ8eokaOSSTAoZajkYrRKKQTA7QugV0gwuXzsqmhHp5ARY1Sx7UQ7xfXd7K/pxuz0oFNKcbh9XDkhmZvPSUf2G1NnT+JEq4UZz247o+/Gc8vyKe4Z6/18op0og5Lnzh/GkldOeTu8dGEBOqWUrw43kRyqZXJ6GGanh3C9krjgfx35s8Pq4u4Nxdw9MxOHR+BzHazr5k9f9lUN6VVS/nbeMG7/8BAOj4/PrhuHu8f6XyWX4vX5sbu9SMQCx2n7iTZmDon+1XiCQfzn4+3tVTz0RckZ7//wqjEEqWXMGUC9IxbBhmvGcuX7+2izuBmRYOTxRUOo73aSH2dAP4AFe12njatXHegjSxaJ4JH5uUxODyU2eGCi+O8Fg2TV/w+QySSEyCSEoKC+y87jX59g3tBodlYKHQiAtHANIVoF20+0D1iEALy3s5orJiRzuMHEql01LCyI6WN7XdEmyMcKEoyo5BIcHsHm/dUtFdw9M5Obzxbi4fUqKVqFjNpOO2IEAucNawXy3ZUTklhYEMP41DCq2g72ccGUSUT8+dwhvNHjIvrCj+U8t2wYL/xwotdc5+sjTay6fBSPf1XK+aPimTkkii+LG/H4AszJi2Jcaihv/lzV60ux6Vgrb6wczvYT7VR32DE7PUQalPjOUPPmROs5UNvF1uNtxASpONFmHbAIASFL50ZjGn/8ooQJaaHcsv4Qd8/MoCDeyPEWCyFawV0xRCMjOVzLjop2XtlyqrMil4oYGmvgUL2Jm9cXsXR4HMtHJ2BQyfB4/ZQ2m8iI1KOQCiTht3dU8+lBYVw2NiWExxfl8drWCuq6HFw7OYUdFR08891xbG4fj8zPJRCAp74t4/yRcZw3Io5jTWb8AeEcF+RHs7emi0c2lhATpMLs8DAyKZjqDjv3zMoiMUSNGBGdNjc7KloxqGSMTArGqJLh9Ph4/vwCqtptmHpIwXd/UkxdpyD3TQ7V8Md5OdR22kkN1+HzByhtsaJXStlW3sFFYxNR9pgnJYVquWx8MsEaGYfrzQSpZaRHaP/hIgQER+AzFSEg7DIzI7W93JQmkxPZaeOWcJ0Cq8uLRCTirIxwgjVyNAoJSrn4X1KEeHw+mk0u3D4/DV127pqZyfs7a/i4qIHnlg3jsa+O9fsbs8PLK1sqOG9EHC/9VMHxFisf7a9j6fA40iN0dNrcbD/RTmGCkT1VnSweHofP/+vjskH8Z8Pj9f9q9INYJBSopc1mXlleyB+/ONork4/UK3lkQQ5vbqukzeImQq/gz+cOwRsIMCw+CN0Z+ENxwRreWDmcijYrW463EayRMyUzAoNSQuR/mfpqsBD5FyDWqObxxXmY7G7C9Uq67G5EIsG0yevzU94ysMQRhLniSa+Qxm4nIZq+lbNSJhYstgMB2q1u5g2NZsnwWC4clYB+gB/O3upO5uRF8UVxc28r/uUtlazZU8erKwpYdfkoSpst7KnqJCFEzejkEJ76tpQ9PV0Os8PLjWsP8trK4SikYrrtHvyBALsqOnhicR7NZieHG8zcOCWNYI0cCDDn+Z+5a0Zmr2TT7fXTaXPzwgUFlDZb+PJQI+NSQmgyOTGqZf3C1HwBYYTg6hlL/T1WgEgEB2q7uGZyMs+fP4zXt1Xy4OendjJRBiUvXDCMaIOf0SkhbLp1Ij4/bD7Wwgs/nOCvS/PZdqKdjw/U83lRIyaHh9lDorjr42JumCKYq/kCAVa+tafX1wNgR0UH+2u62HDNWBweD2aHj8t6lCap4VokYsEUbO7QaKwuH7Of+7l3kZZLxNwzK5Nog5L1V46m0yqMsK5etZ8TrVYWFcYyLTsSnUJKqE7B1MxwLnl3L09+6+PxRUNYVBhLbaedoXEG5FIJm0sb6LKdeh8r221c8d4+NlwzlpvWFVHTYSMqSMXSwlhun57OB/vqeW1rJeE6BddMSiFcJ0ctkzIuNQSpGDx+qGy14PEFkEqE8Y5WIetJmg3QaXNhdno51mTulXXHGlVoFRI0cskZR1BxRsFe/XSi8+kFz41T09hZ0U5mpJ5h8UbCtArBr0T7P1cJVLcLsfAmhwejWo5eKWNfdRdapYwnFuURZVBy3vA4jjaZ+yXn7q7q5NLxQmJvsEZOapiWh744SqvFxaikYO6ckcnW463kxhr4aF8d54+M/x8f7yD+fRCLBQM/uUTcR2F3EmdnRaCWSciNNnDtmgNcNj6JcJ0SERATpMTi9BCmU/LiBcPIjTEQrJWhU/x947GoIBVRQSrG90Qa/LdisBD5F0Etl6KWS9Gr5DR2O1g5OoEdFR2cnRXOkBjDgGFYAClhWhpNwmKXG62nqt3e5/7FhbFoFVLiQjRsPNTEwwtyCf4Vu+hRSSFolVKafuFEanJ4WPbabuRSMX+cm83Vk5LpsLpQyyTcNi2DtHAdRfXdxBnVLBkey77qTlLCdVy3+gBef4Dbp6WzZk8t9d12LhmTxIYD9ZyVGcHhhm7GJofw56+OEayRMy4lBI1Cyl83HRd8VK4cxYhEI/4ea+P7Z2dz2y+s2tssLjpsLubnCy6kqeHa/qFlPciN0XOiRzYarJHz1s9V/UZFTSYn160+yF8WDeGaVQeQSUXMHRLNpeOTGJ1sRCIRVC2JoWo6rG72VXdxpKGbPy0cQlWbDbfPj8vr564ZmWw+1srXR5p6W7Eur5+XfzrB8tEJvQFqIIyvbC4vB2o7GZ4YwkVv9R1BuX1+HvqihNdWFHLFu/t4emk+z/1QTnqEjmeW5tNhc+Px+imq6+bN7VUkhahZd+VoLnprL3KJhMp2G4EAbCtvJ1ijYHRKCJPSw7j8vX296iSX18+Bmi4emZ9Lm9WJWCQiVKfgaIMJt9fPU0vyiDGqCNMo8AYC1HbZUcoE2XmoVo5cKsYX8GFxeVmzp5bqdjsrxySQFKrhRJuV61Yf7MPvyY8z8Pz5w7h6YgpPb+obSHbys0oIVfP4aTb7Y5JDONFmJdao4vZpGZS3Wpg1JIrHvirl7KwIpBL+x0WIzx+gqK6b69ccoMnkZFJ6GHPyorj3k8N4fAGWFMYwNjWEVbtqabO6BL7I1DQe++oYFae5qvoDAXKj9RhUMgoTg8mM0hOskbO1vI2lr+5k3ZWjWb+3joJ4I3a3j06b61d/m4P4z4VELEIlFfPkkjxu/eBQHz5VfLCau2dmopFL6HZ6eX3FcJp7fJrC9UrCtQpkUjFjUv+7i4n/CQY5Iv9LaOy2U1xnIjVCh8fnZ9lruwaMb//zwiG8tb2KijYrr60o5O4Nh3sJrflxBp5ako/N5aHL5iE1Qvt359AtZicOj4fdlV3cteHwgI/JizXw0LxsRIhQy6WEaGXIJEL3o67LjlwipqTJwp+/Oka4XsHVE1PIiNSxuGeuf1ZGGFdOTEYhlVDeaiHWqOaBz472SWkNUst49rx8Htl4jLOzwkkOVTMhLQy3L0CjycGzm8opa7YQF6zi1rPTkUrE1HTY2FjcRHywmjCdgpd+EYqmlIl5/vxhPPjZUXRKGX9ZNITFr+w8Iwnz+fPzuWvD4V5menywmrcvHkGrxcnB2m4i9Eqyo/X4/AEUUjFddg9iEVhdXl76qYJDdd3MGhLFgvxobv+ouDcEzaCS8c4lIzj/9V29I7c/zsshzqjC4/Ozdk8tW44PTHqbPSQKpUwsGM9dNYZvjjTz1vaq3kInJ1rPndMzuO/TI6jlEh6ck41CJuGJb8t6u1YAWoWUv56Xj83p4eYet9/LxycRHaTisa+P9Y7JFFIxD8zNZkJqKLd9WMQTi4fy569K+yhoRiQYuXd2Fi6vwH14dUslkzPCcHn9PLupnHVXjubit/cMOF48b0QsN01NY+2eOt7YVoXDIxBjp2SGc/+sLLaVt/PA54K9dbhOwerLR9Fpc2NQyyiq7SY6SMWzm46TFxvEVROTUcol/5CFtdXlwebyIZeIMfZ0E2s6bMz827bez/2ti0dw5XtCavINU1KJCVJx9y9UXcEaOX9dOpRbPjhEp81NdpSei8cmMCTGQJPZicPtR9rj9HqixUJCqIYP9tbx4LxsGruFGIdYo4ow3f9tv4f/y+i0uthyXFC5lDSaabY4GZ0UTKReSZhOgUYmQaeW/8uVXP9XMcgR+Q9AdJCa6CA17RYnVpeX9y8bya0fHOrdzWsVUq6alEx9lx2by8urywsxquVcPTkFk93N6OQQYo1qpGIIUqnIjtYj+w2pnRF6JU3dAYYnBBNtUA6YinrXjEyONpixuX1Mz4lAp5T1GJjJ0SulQsEkErH2itGo5BKK67v7ZCDUdNgpquvm8W/KmJAWykVjEnl1RSE1HTYON5iINCjRKWT8+atjSMUiRiQFc+X7+/j4mnFc/f5+HpiTxdOL8/D4A/gDAfx+PwFEKKRirp2s5nCDCalYxNsXj2DVrhqazU6GxgYxIzeSZzcdp83q4v452bRbXb+qBPH5YdnwON7aUQ1AbaednZXt7KrsZGNxEwAzciJYPDyOuz4q7i0ADSoZd07PINao4uMDDeyr6eSPc3O4bs2B3vvtbh8Jwaf8PI43W0gMUROuU1HfdeaAroZuByOTgnH1WLN/uL+uD/HtaKOZOzcUc//sbG5Ye5AQrYK3d1T3KUJAKJZuWneQDdeMRSoWCLoT0kK56O29fR7n8vq575MjfHDVaG47J5MnvzneT8a7t6aLP35+lFvOSWfr8TYuHZ/EVe/v5+4ZmczKjaS02XJGjtOnBxu5eGwSRrWML28cj8XpRSkTo1VIcXv9mJ0ezhsex6jkYPLjgyAQoNPm5ofSVo63WNl+oo0Fw2K4aGwCepUUzW/0iXB5fdR22Hny2zKK6rqJMii5YWoaBfFB/FTW1luEpIZrKWs24/UHSI/QMiM3knkvbAeEDI/5+dFEGlQ0dDt4f1cN668czZfFTYxKCibSoOS7khb+uul47/mHaOT8cV4OFqcQ5KeSSSAQQC2XnDHDZBC/DwRrFUzKCKPV7KJFJydUKyc6SIVRLSNMK0c+qIr6X8PgO/u/jFCdklAdtJqdvL6yELvLh8vnR6+UIgbsHj/nj4zrsf32MSUjHL1KSohG0S/G/bciKkhFm9nJWxeP4LGvS9la3kYgAHHBKu6blY3VJZiNzc6LIlyv7BPRHqRREKRRoJYLBVRNp71fZ6Wqw0ZWlB6RSBgVnMxRuWOaQBrdfKyFFouLi8YkolZIufWDIialhfFjaSvNZifX9jhYhmjkLB0eR26MnsxIHXFGFU6vn1ijGovTg8Xp4c4ZGXRa3fxY1sofPjtCXoyBG6em8ebPVZw3Iu5XtfgGlYwZQyJpsbj48rBQeHxzpIUZORFsLG5CIRWzbGQ8l727r09BY3J4uP+zI7y2opBNJS3UdTpotThJCdNS0WZlUUEMq3fVcN2UFG5cWwTAZ0UNLBsZR323nSEx+j7dodORFaXrJbm1Wpz9UmkBWswu7G4fYToFbp+fz4oaBnwuu9vHsSYz6RFaRiWF8E5PwTUQ3t5ezd0zM/nmaFO/+xRSMSq5hJggFWOSQ2k2OYk1qnjj50oemZ/L0aaBw8ZAKHS67G62lXdwvMXK+SPj0CqkHG+2Eh+iYkZuBC5vgHCNHLPLx1eHm/AHYERSMPlxQVx7VgpGlYwglfzvFiFmh4f6bgcf7auj1eJiRGIwc4dGc6C2i0P1Ji5/dx/XTEohIeRUuqpOIe0tMK+bnMreqk58/gDLRycwOjmYzcdaSQvXcsGIOGQ970NmlI7YYBUdVje5MQbWXTGazcdaeWVrBR02IW/kzYuGs6hQiFYIUsvQK6X/FNl3EP9ZCNYoCNYoyIz6fXXgf+8YLET+P2GgvI3/TYTplShlYh6Zn4Pd48PrC6BRCDs4p8fP5LQwFPIzf/wGtQyry0uEXsHwBGOfeOpAAL463MRdMzL5S8/8PxCAJ74tIzpIyesrh7NqZw2vbK2grtNBfpyB++dkM/f5n/u8RofNzctbhPHLvbOyiA5ScLDWxJs9Cp4xySHkxwWhUUg4OzuCszIjkEtE7KzoYG91J2q5hMsnJPHc5hP9jn9onAGjRlCbLCmM5asjTQQCgra/u2dENjUrnI3FTQN2VQIBwSlx7tBoVu+uZVdlB7kxeqIMCpJCNfx1UzlLR8Rxw5RUXttaic3t44UfTnD9lFQuHJ3AFwM8r1wiZmZuFJe+I3QtksO0NJ8h3ryyzUq0QYlEJDpjNwKg2exEJhGTFKph2wCuqSfR2O3A7vb1s5S+amIywxON7KzoYN3eWqZmRiCXqYnSK9lb04VIJCIzQnfG5402KOm2e4gxqli/t5bLJySxdk8tDo+Pkf4QQtRywvRyFHIJlfUmjjaaWToiDr1ShscXYPOxFpaPTugdq5wJZqeH9Xvr+NNpKpeNxU0khqh7gyTdPj+vbK3g02vH9XKMKtttXDwuEQCjRk5Dt4PJGWHEB6v5+nAzF41NwOnx88ymcspbhLyfayenAKBWSNAqpQQCAc7KDGNSRhjnvbYLt8/Pj2WtLC6IRSUT1D2DI5lBDOKfx2Ah8n8YOpUc3QD69d8CuVRCpEFJu8XFQ/Ny2FjcxEcH6nslpxeMikcjl5IbreeDffW0WpyMSw3lrIxwLA4PkzLCyY42EGlQEGVQUd1uO2PnAiApRM2QWAMHa7p7b9MrpZydHc5fvy/npZ8qCNHKeWheDuNSQ8iLC8Lh8ZESpsXm8rFmt7D4iUVwVmY4F45K4Po1B7lrRiYSEeRGGzjcYOLCUfHcsr4IEGK/91f3z845ico2G/lxRgAMKjkrx8Tj8QW4db3Aybj348M8vXQoebFBuLw+ZBIxlp6gtFeWF/DQFyW9qpukUA13TM/g5Z8q8PoDzB4SxdFG0xmlr3HBavZWd6KQiokyKGk6Q8EyNCaI+GAV+XFB7K/t6h39/RIZkXqUPdk+J1/y5rPTaLO4+qRBv/lzNWdlhLFiTAJF9d0ECNDt8JAbo+dIQ//OyDWTUznRaqXV7MTjC9BudXF2diTv76zG4faBBppNTlweP7kxehJDM9hd1cnG2iZCNArGpoTwU2kbC4ZF0+3w0GZxCXJ0pQyL00Oz2UVymAafP9CnCDmJ6g47nx1qZOaQSD4rauwpPqwkhaipbLdjcggckuxoPXa3l8woPSnhWp74poxHFghy5zs/Ku59TyrbbWw53sYj84W04m+OtnDNpGTGpYYiBl5fUcil7+6jusNOkFqGRCwiWK34TSZwgxjEIAbGYCEyiDNCJZcSFyLF4fayfHQCC4fFIBYLCqEglRSRSESkXvDtONpoJjtaT2WrlWijmm+OtvBdSTNeX4Abp6ZR22lnXGrIGQ3VVAoJZz+zlWeX5dNmdZMUqmZITBBLX93V21mYGhvEwdpunv/hVAfktRWFdNlcbLhmDNUddmQSETsqOrhu9QEcHh83rjvI2itGE6yRcd7wWLrtbtqtQqu+2eQkKUzDwbruAc8/IUTTmxGzbGQcPn+APVWdPDQ/h/s/PUJtp52PDzRQkGDk+c3lvXycWbmR/GFuFu9dOhK724fPH6Cy3cpT35ZhcXq5d1Yms4ZE8XFP8N4vYVDJiDYo+cuiPHRKKbdPy+inNALIiNAilYgYGhuEx+vnyonJA3Z45BIxiwpi8PoDTMuJ5JsjzRhUMlLDtDy7qbzf8/5Y1sbo5BAuG5/M5mOtNHU7eG7ZMP62uZyvDjfh8QWINii5enIK1R02xqeG8vwP5cgkIkK1Cm5ZV8QjC4ewYX8dQeoINh9rpbLdSkqYlgtGxTMhNYRnNx0nPy6I1HAtIxKNfH6okUc2HutV5RjVMv4wJ5uvDjexYFhMHzXLL/FlcRNPLRnKZz05UB5fgD/OzeWeTw7T0O3gsa+O8cHVY2i3uDhY101ujIFzC2IwOzw8/nVZvy4RwKNfHmPDNWN5f1ctD288xs1npzE8IQilTMKl4xJRySVoFJI+kfODGMQg/jkMqmYG8S9FVc+COyEtjKRQDXa3F5VcypXv7+OF8wu452NhcTgJmUTEk4uHsmZPLWnhWhYXxvLezhompIXy+rZKjjWd8p9486LhXL1qf68iBGDF6ASSwzTsqerk6yPNDIQF+dFcd1YqLq+PuS9s75UFK6RiXl5eyOXv7h1wMXp1RSF3fHSIS8clsSA/hhOtVjKjdIgQuBEmh4cuu5sYg4runn+LRSKK6rrZWdHBixcOIzpITbPJwScHG0gJ0xKskXOorovHvinlyxsm8NbPVXywv773mKINSl5ZXojT6+WmdYdoMjn5/paJHKjt4slvy2i3upGIRUzLjuDumZlEGVSIRfDc5nLcXj8jkoJ54LOjve9xfLCae2ZlkhOlp93qRq2Q8NS3ZeiUwtjqTO9ZZqSO55YNw+b2opQJfjguj492mxufX+DRfHGogYUFseyq7KCu08452RHEGgUX2cpWKzOHRNJkcqJTynB7/WwsbkSjkHD1pFS8fj9Ot5/tFe2kR2i5dvXBfscgFYv4+NqxfH24CavLx/u7as74vXvxggKuW3MAuUTMixcW8Jevj/HiBQVYXF5EQKRBgQg4VG/GqJbRafMgFsM1qw6c8Tnfv2wkl7+7D5fXj04h5YnFeYTrFbRaXGSE60gM1fzTPK5BDOL/OgZVM4P4tyEpVMsjC3LpsLpxef3EGFVIRCK0cil3flTM/bOzMDk8lDSZSQrVMCYlhOL6bq4Yn0SLxcnCl3YAcE52RJ8iRCWT0O3w9ClCAD7YV8eHV49h3Z6BfVpACFf78nATBAK8dEEBf/rqGPVdDrz+AKVNZl68sIC7NxzulVdrFVLum5WJx+dn/ZVjiAkSskYSQwe2XLY6PYjEIjaXtlLbaWdKhjAaig4SSJNBajnRBhVXrdrPSxcW8KevSvEHYNZz27h/Vjbf3jSRJrMDlUyCye7hpvUHe/1kEkPUSCUilhTGMTEtDKvLi0ImJlgt75OOe8GoeC55Zy9mp4fXVhZidXrxBwLolDK67W6aTA7u/eQof1qYw32zsnD7/Dy88cx21iaHB4fby/clLXTZ3Ww/0cH8/GjOHRZDk9mJXCrmorGJBAIwKT0Mj9dPUpgGnUJKRqSOkYnB7K7q4I1tVbRaXOiVUl5ZXsiPZW0seHE7Lq8fsQim50QyMim4H+k4M1LHosJYbC4fGZE6glRyPj5Yj1oudOisrlOPHZUUzNFGwZDsrpkZrN9bS0WbjU6bm3V76/j8UCOvLC+gMMGIz+8nSC2EjolFvz5OESPqLRAtLi8BwOsL8NKPFTx//jBcXj8q+X9HFsggBvG/icFCZBD/cpxknp+Ody4dyco393D92oPEBKlIDFVjd3uZkxfFksI4SpstXLP61O5UJKIPn8EfCPSxBj8Jl9fP+r11gkyzxdLvfhDsz8N0CsanhnLPx8X8cW4ORrWMNquLrcfb2VYucAL0Khl6pYwIvYIwvQL535FLW5we6rscrNtbR4vZyfkj4rhyQhKhvyAuOj0+hsYb+OSasTg8pwijPj88tLGElWMSCARg1e6aPiZuErGIB+fmYFTJEYtFRAWpOBMiDSrevngERxrNvPxTBRkROmblRaGWSShpMvHSjxVUd9gJBGB7RRt5sUamZUcOOCpLCtXw1OI8NpW2squykyC1jJvPTqOh28Hr2yq5fHwSdoUftULCIxtLuGRcEp8fauTJb8u45Zw0hsYFsaOince/KSNCr+Dms9MYmRiMWCyixewUHE1HxJEQoiEQgKLabm6emsafe4jP98/OQioW8f6uGsamhDBrSBQ2t4/XVwxHr5Lh8frQq+Vs2F/PG9uquGFKKluOt/HmRcP56nATm461MiopmHarm0AggEouIT5Yjc8fIEyn5NGNR7l+Sho+f+CMEne1XEKQRtbHZVMmEREAypotwvfx7wRJDmIQg/htGCxEBvH/BekROj65biwNXQ6aTU4SQ4Wk11CtgqZuBwdqu/oQN3dXdjI5I5wfSoWkS5fXj1wqGVCu++nBBl64oKBXGXM6RCK4clIyaeE6/IEAC4fFcvl7+zg7K5wLRgljnUiDkqRQDTFGdY9t/d+HzeXl80ON3NcTaQ/wzZFmwnUKPrhqTG/3pL7LjsXpxePz4/EFCFLLeXJxHo9sLOl1RH1vZw33zMzk0fm5rN1bS4vZxdBYA9eelUqwSkbQbzymSIOKSIOKqZnh+P0BTE4PIpGIrEg91R120iO06FUygjXKHh5IcD8irFYh5aF5OVz+3r4+Vvw/lLaybEQcQSoZHn+AILWM93ZWs3JMIrd9eAizw8NHV4/hoS9K0CplvPhjBZPTw7hgVDyvbKng2U3lKKRi5udH8+qK4Tz4+RF2VZYhFYuYnhPJjVPTeGtHNdOzI2jocvD2jmrunZVFXaedC9/Y3Vu86RRSHpqfww8/V7FybBLLRsTx3dFmzsmOYMWbe3B5/SSEqHhoXg7+QACXN4xlI+OxuLw8+mUJd8/MIiVcx3s7q7l3VjYPzBX8Wk7vtIlE8OiCXD472Nh72+jkYI42mhkaG0R0kBKxiEGC6iAG8S/CYCEyiP9viDKoiDL039XbXMJCfTo27K/nxQsLONpoosUsOJq+u7Oa+2dncd+nR/oUHF5/gEiDkpcuKOCuj4sxO4QF3qCScd/sLERAAIFkOzM3krxYA2v31LFmdw1z8qIZkxxMxADH9Wtos7i4/9Mj/W5vtbh49MsSnj1vGJ02F1UdNrYeb2f17ppeGe7YlBBeXzmca1YfoLPH4+Kxr0tZNiKOp5cMpcPmRioWEawWRi91nXZCtXJUv5Bb21xeOm1uvH4/GoXgPdPc7cDp89PQ5aC6XcibiTUK53Z2VgR+f4A3tlVx9aRkvj7czPPnD+O9nTV8c6QZr9/PbdPSWb27pl8eEMC6vXW8vnI4lW02Nhyo58E52VS22QgEAjx7Xj5HGkycnRWBQSVDIRWzfEwCV72/v5c86/L6+WBfPftrurl+Siq7Kjvx+gN8ebiJ4oZuHpyTTVywmnkv/ExGhA6ZRNSPF2JxeblrQzGvrxzOle/t44OrxtBpd2N3+bjurFSyo/TEBat4Y1slHx1o6P2eGNUyHjs3j9e3VnLD1DTqOu088U0pN05N5dPrxrFmdy3HmswkhmhYOSaB0mYLr20TwhLjg9VcOzkVu9vLiz+e4KpJKb+5YB3EIAbx9zFYiAzi3w6JRExyaN98GYvLyz0fH+ahebk0djvYV9PZY7gVwlc3TmDdnlrKW60UxAexuDCOWKOKIJWUx8/NQyYVo1NI0atkuD0+xCIRDV1CKq1WKSMjUsaDc7Px+gP/tAnVrsqOMyYE/1DaSrfDTVmLuY8vyknsqOig1eLikfk5ve6pV0xIYsGwGMJ1Srpswsjo8neFHJmTqpebzk4n0iCMfeo77fzlm1K+PiKEG07PieD+WdlY3V6uWbWf6o5TmUUPzcthaKwBqUSESARHG014fAEaux18VtSITCLimaVDEYlEhOnkPPIr3JG91Z2ckx3BpmOtzMiJZFRyCB9eNQaPP4DL6+Ozg42IRSKWDo/jzW1VA3q0VLRZcXl9fcYidZ0OHG4ftZ12/AFYVBjLqjOQUz2+ADsrOkiP1FHVbuXisQl4fPD10SaK67vJiTbw4f6+JnBddg+3rC/i5eUFbCxuZOHQaK6bkkqnzUNzz1hN31NAubx+tEopN01NIzlMQ7BGjk4p5aP9dWRF6xmfGjJgrPsgBjGIfw6Dhcgg/u0I0cj5sbSVlWMSefc0d9CGbgfXrznAR1eP5eKxiX0UCn+Yk43L60cpk/RmP/x0vJ1Vu2r449zs3lRYsUjomCikEppNDiINKgKBAC1mF502F/6AkDMSrvvHvCBOD3/7JfwBgRfi88P7OwdeTE+0WgnWyPno6jGE65VEG1SIxSLcXh+rd9fy1HenQuTcPj9r99bR0O3g2WXDcHv9LH9zd59i46apqdR123no85I+twO8+OMJnlk6lLd+rmJiWhjhegU+v5/iehMvXJDM4ld2suGAsHC/dGEBvyajCwQEl2CAt7ZX02x20m5xcVZmOCVNZoYnGpGKRYxMDubVrRVnfJ69VZ1kRun78DP21nQyJTMcgFCtvE/y8S8hpFbLKWuxkhWlp7TZwsoxiRxvsfCnL/v7jYDwmTV0OajtsHPOs9sYlRTMdVNSSQ3Xsre6E7PDw5CYIEQEyIzQkhWpQy4R0W5zU9/p4JrJqeiVUmL+Tt7TIAYxiH8Mg4XIIP7t0KtkTMuJ4KfSNh5flMf6vXW0Wpzkxhi4/ixhofilTFIqEfcpHLx+Pz+WtvLIvBwMahluX4AdFR18VtSI1eVlXGoIM3Kj8PgCeLw+uh1eOqwu3t5RxdFGC39akMvkzHC0it/2kxiVHHLG+3Ki9QQCQkFicXnP+LiKVit7qru4b1ZW7/m1Wlz9wv5OYmt5O102N/Vd9j7FRrBGjlwqRSr2DUjYbbW4+NNXx/jbecOQSUVcOi6JdXvruGRcIvVdDm6YktrrzbK/posJqaFsLR/YpXVGbkSvoZvZ6UEulfDOzhqONpmZmRtFQ7eDVouLyRnh3DUjk7hgNV6fn2+PtvDt0eZeHlCQWt5Hxn3yPKIMSrQKKbWddjIjdRyqNw14HJlROr492sz0nEjKmi388fOjZEXruXdWVq9PzEA41myhxeLC6w+wvaKD7T0J2bkxBp7dVE6wRs7fluUjloh5Z3s1H+6rZ1FhDLdMTSNk0D11EIP4X8Eg22oQ/1ZYnMLCKhGLOCc7gpwoHbdPS+PlCwt4cG42OdH6AfNYfgmpWExenAGNUgoiEX/9/jh3f3yYnZUdHG4w8cqWSq54dx9mpwepRExlu5UfSlu5+ewMNlw9BqVMQtNpC6PD7cN5BidYj8eHRi7h+1sm8t0tE7l7RgZS8cnjEPHw/FwAlDIJCumZf2KxwWocHm8fCajF6f1VB1pBStu3SBCKpwDWXyl6jjVZqOm0YXF4mJoVQaRBSadNkFhPy47gvUtHcvWkZOQSMbdPzxiwIJufH83B2m5qOoUiaHRyCEcahEJhb3UXUUFKVu2qIS5IRbPJgcvro8XspK7LQXKYhldXFKJXCs87LjWUfb9wtZ2UHsYnBxp4cnEeG/bXc+n4pAHPRa+Skh2lx+H24/D42FzaSnqkjs3HWmnsdpAXazjj+5ARoaWu5/hlEhFz86JYkB/D2VkR/HFeNsEaOat31eL2+CiMM+IPBJiUHo5G+dvC+AYxiEH84xjsiAzi/ztazU4cHh8eX4CP9tez4UA9dpeXyRnh3HJOOplR+n9KkbB4WCw2t4eGLgff/SJhFqDN6uLNn6u4bFwSXTYP49JCkUlEdDsEwudtHx7iqSV5SMRi7D2+ETKJCI1cil4hweH14fXD4QYzT3xTSnWHHZ1Cyvkj4/nulkk8t/k4V01KJSlUg9Xlpd3iYl5+NB/u6++gGqZT4PT4uGpiCnrVqUVOJZP04cr8EhpF/9FAi9mJWCT61W6ORCwiTKfEHwgQH6zmvllZtFlc1HbakYhExAerCFJHUtZspb7LzodXj2HVrhp2VXZgUMlZVBiD1xfgoS+OCschlzBvaDSXv7uv9zXKmi1E6lWE6BR02tyUNlupareilEqYlx+N2enhD3Oy6bK7+fpIU293RCSCe2ZmsW5vHR8faKCizcajC3KxuX38eWEuT313vJfUmxGh466ZmWwqaeaBOVnc8sEhlg6PpbzH2n7tnlrumJbOo18eY2ZuFDkxeixOL9+VtFBU08241FAe/7YUjVzC384fxrdHm7ntw0O4vH5SwjRcOzmVvdWdSCVihicFseX2yQRr5X2CIQcxiEH8azHorDqI/29otzipbrfxbUkL+XFBvPRTBUcb++aXqGQSvrhhHKnhZw5aOxOcbh9dDjd/21TOur0DG5ypZBJeXl7A098dZ2isgZlDoogzqjjSaEIhlZAapqWi3cajG0uobLchEsHk9DDum52FRi5hV2Unt3zQ3259Qmoojy0aQuxpRUJdp51uu5unvzvOT8fbem+PNih5aslQ7G4vo5JD0J2227Y4Pdy0rqhXtnw6wnUKPr9+HE6Pn6nPbOlDBP3LuTkUJoTw103H+epwf7fUcwtiWDFaMFmLGCCA0e3x0WJxYnJ42Hq8DbFIxPi0MCCARi5l1e4a3ttZg9cfYFJaGJeMS+TJ78r65M88NC8HjVxCmE5BRZuVWKOaRpOTUK0ch9vH0UYz5w6LocPuQi2X8lNpG+F6BcmhGj7aX88XxaeSgUUimPP/2rvv8Kiq9IHj33unt2SSSe89lECA0EFQxIaKvSKLDduqa9l1dXdddXWta1n72nvvXRREVHpvIZCQ3vv0fn9/DARiApafGMr5PI/PIzd37pzMJLnvnPOe9x2ewqWHZaPXquhwBdCqJCRJwu7x8+22Nt5YXovLH+TTqyYDkSaK8SYtWo1MMARr67poc/oYkWYlzqIlSqemptOD1aBleVUHb62sZf1uSz85cUZuP7mIVKthR0Co0GL3E2fRkfQLd1UJwqFOVFYV9jt1HW7c/iB6rYoZRcm0On19ghCIJBTeP28r/zmjGNPPzNfYSa9VEXYpSHupM6Wg0OkOsKG+mw313by3pp4XLhjDmMwYlm7voKrdxUUvruiZkVCUSO+VzY123rhkPHftKLr1Y9+Vt9Hm8PUKRNJjjUQZInUvnL4gjV1eYkwaog0aLHoNNpO2z8yPRa/h9pOG0tTtYfNulWXjzFpeunAsSdEGvIEQ/zuvhMtf3VXu/raPt/DJlRP5y9GFRBs0vLuqHn8ojE4tc/aYDM4bn4FRq+o3CAHQalSkx5qwegJMzocfyttw+gLYTDq0KpkLJmVzysg07J4A87e0cNUba3q2SUOkXH6K1UBGrIF2p5+P1jWydrcePqlWA3eeUkRIUdDKMmtqOllR1c410wv442ur6fb0XlaSJQm3P4hKJfPNlhZy4s089/12fqjo2O0c+OcJQ3oC2sfOHUlZs4NASOGm9zf0vDZpMQaumZ7PiPQYbCYdZr2KYwYnsL62C7UkkZtoZs74TNrdfh78ahvVHW7yEsxcPS2PZKsehyeALEskiBwRQdgnxIyIsE/ZPQHquzysre3i43UNBEJhLp6cw7zNTT07NX5Mp5ZZ+OfD91pJdE8auzxUd7g5+6ml/X79pBEpqGSJ93Z77uK0aP41cyhIEnd8upkVe+jI+8CZxTz2TfkeG7DdcuIQLpjUf17DL9Xq8FHf5WZbs3NHJVpTT8l4AF8gRLPDx9raLjpcPkZnxvYkena6/bh8QTyBMEatCqNWxqLX/uzAzh8K0dTlZcn2drLjzMx5bjnDU6O59/ThdHkC/PntdT1LIRBpqnfv6cPZ2uzgjNFp3PFpKfNL+87opMUYeOisEaRZ9VR3uLHoNfiCIcLhyJKTyxdApZKxGjSElUi+jSwpeAIKLQ4fWpVMrElLdZuLshYHg5KieGNFLV9uamJSno2JuTaGJEcz96WVPcs+xw9L5pSRqTz49VY2NdgxaFScVpLKBZOyUUnQ7vQRZ9Yzf0tLvyXv7z5tGJNz4/CHwuTEm3/p2ygIhywxIyLsN1ocXh77ZhsZsSZOHZVKXaeHj9bVYzXuuQ6DUatC2tu0xl4kWw0EwmGOHZrEF5t6L1HEmbWcNiqNuS+t7HV8XV03Go2M0xtkXW3/uzQAFm1tpTDRssdAxPYbFrmKt+iIt+gYkR7T79d1mkjZ8ozYvltJ/7/LCFqVigybiQybCX8wzNlj0nl+cRXTH/yWW04cwv9ml1Df5WFNTRdJ0XqKUqKQJYlBSRYCIaXfZSWIbLn1BcPUdnmINenwBcMsr4wEfYcXxmPRa6jv8vLG8lq6PQHGZscyKiOGBVua+e/8yK4eq1HDvacNZ0KOjTnPr+jpD3R6STqvLK1Go5K54ohcitOsSEjoNDLnPbusZ4bLEwjxytIa1tR08e+Ti3B4Q9hMCvd80f9M192fb+HNS8ZjFD1lBGGfEYGIsM90unyEwwqXTcml0xNgXW0XNpOWY6blEw4rvLqspt/HnTsugzjzr7up2z0BdGoVN58wmJnFKTy/uBKXL8S0QfGMzorl5g834guG+zzO6w/hC4aJt+j6bCvdKT3WSLi/Nr1EgqehKXverXGg0qplLp2aw5raLtbWdvGPDzZxeGE8WTYTW5rsTMiNxWrQEAwrLNrayszilD0m2mpUElq1hEWnoc3pZ0uTnbdW1lHR6mRsViwLylp6thEDfL6xifRYA0/NHs3H6xrZ3uaiyx3gj6+t5oULxvLPE4Zw/duRfB29WmZoShRTC+K545NSHp5fzssXjeXfn5aiKJFZtkybEW8gTE2Hm00Ndlz+EPkJpsjsi0qiv01HXe4AzXYfhYliNkQQ9hURiAj7jNsfwu4L8vSiSr7cbXZCLUvcdeownjhvVJ827IMSLcwen7nHXTPeQJCwEinXvjtPIER5s5N7v9zCyqpO4ixarj+qkEfOGUlYUQiGFKb+Z2G/N8lhqdEsq+ygJDOGM8ek8+BXW/ucI0kwY1gyWpXMpxsae9Xx0KllHjlnJPG/Mnja3yVFG3j6DyVUtbv5obyNTJuJihYHS7d3sHR7R69zpw1KQK+Re8rZ7xRr0vLgmSP4cE0Db6+qwxcMkxNn4o9H5FHb6SYMvYKQnWo7PDy1qII/H1PIFTuaIgZCkaBnXI6NlGg9Z43NIN6iI8ao4bMNTVwwOZszRqehVclsa3Hw56MLKUg0RwquZcaQEWvEFwzjD4b5cH0jxw5NYt41hzHpnoX9fv+yLLHXxCNBEP5fRI6IsE+4fEEq25x8W9bGffPK+nxdkuCTqybjC4R5d3Ud3Z4AJxanUJxm7SljvrtWh5ctjQ6iDGrSYoyRAjiSQowpcu7yynbOfmppT3O0CydlceHkbAKhMG5fCK1K4sN1jTz6Te+bnU4t8+Yl43l7VS0mnYapBfG8uLiq1/ZftSxx7+nDGZMVQ2O3hziznopWJ8srO0iNMTApN444k46YgzQQ2V2Hy8ejC8qZMSyZ059c0ufrJwxPJjnawNM7+rRkxhq49/RI4nFDlwe1SmZJRTsvLK7sSSZ97NyRtDp83Ppx/6XltSqZd6+YyImPfN9z7LiiJCbnxWEza/lwbQOfb+y9DHfF4bkcOTiBuk4PH6xpYEVVO59cNZnNjQ6e/b6SNqePkekxXDI1m7VVXUzIt/HI/G28t1ujO4jktfxvdgkpUXpizL07SguCsGciR0T4XTi9AdqcfsqaHGhUEvmJFuItOvQaFW1OH4GQwusrIssvOrWMzaSlyxPA7Q+hKPDxugbmTs7mhmMKsejVyHL/syCtDi9rqjvJT7SwudHOi0uqsZl0nDwiBZcvhEGr5u/vbySsQEmGlXtPL2bJ9jbOfXoZDV0eBidHccmUHIpSo3ju/NG88EMVzXYfY7Ji+MPELEob7EwtSOC15TXEGDVcMiWHP0zIZH1dNzEmLaMzY9BrZByeABKRT8Yj0qKZkGNDLYNee+gUu4o16Zh7WA5rajq5/ugC7p/Xe/aoss3F9UcVIMuwoLSFR84ZyfVvr+vZISVJcPSQJB46ayR/emMNwbDCf+Zt5daZQ/b4nP5QmMCPltMKEi0YtSranP4+QQjA4wsrGJcdS7PdyzdlLcy/firPfl/Vq4ledbubzzY08urccbQ5vPzlmMJegYhOLXPriUOxGjQiCBGEfUgEIsKv0uny88LiKh5ZsK1nFkKjkrjr1OEcW5SEyxdCrZIIBMP866ShxFt0NHR5SLDo8QVDPPT1Nuo6PT+rbHZjl4f8JAsXv7iKilYnWpXMNdPzCYTCeIJhVHKIf59cBFIkmfHp77b3qiOyob6bq15fwx0nF7GgtIUbjxuESaumodtDh9PLn95ci04tc9HkbAqTovAGQ6RY9WTHmVDJEhWtTm56dwM1O3qf3HLiEM4Ynf6zy8EfbJKtBlqcPhZta+PjKycxb3MzHS4/03aUyK/pcHNcURLnjMng4pdWUr7bDhtFgS83NWHSqTh1VBpvraylss1F/F5u9OOzY1lds2snk0mrYkS6lfRYA9e8uXaPj3t7ZR1xFi0lmdH4AqE+nXwhEuTc+tEmHjp7BBqVzJwJWVS2OclPtHDSiBRijdpeu5UEQfjtHZp/SYVfpa7TTSisoJYldCqZ0I7utUNTohmfE0tYUXhqUQVDU6Iw61V0ukI8fM5IbvloE1uadtXESLUaeOSckYTCYdocXsx6zR7LuPuCIWwmHY8uLKei1cnM4mSunV5Adbub2z8pZXubk9x4M3+ans+q6g7GZtl4c2X/xcz++/U2bpwxiDs+LWXWuAye/b6Km44bxPPnj+GT9Q20OnwYtZHdKDtvPnWdkS202fFmJuTZmD0+i4xY4yEbhOwUb9biCYS48/Mt3HrCEPyhMNuanczb3ExTl4cjBycyNCWqVxCyu4/XNfDouaN4a2UtOrVMWIHJeTa+L2/vdZ5WJXP90QVc8eoaALLjTPxtxmC6PD4Sgjq63IE9jrHd5cdm1nLlEfksq+zY43mbGuyEQgqyBFdNy8UbDKOWJWKMWnQ/o72AIAj/P4f2X1PhZ2mxe5m3uZknFlbQ0O2hMNHC9UcXcNaYdE4vSaOmw8UT31bg9Yc5Y3Q6nW4/8RYLte1uHltY0SsIgUhX3WvfWsttM4dS3eEm2qDGqNX0+8lTURQCYYV4i47nzx9DWoyBxm4vf3lnPa1OHwArqzuZ/exy/n1yEf5QaI+7NlqdPvRqFdtbXZh1aqL0arrcforTozli0Ih+H5MWY2TOxCzOHpuOWpZ/Ven5g1GK1cgNxwxiRXUH76yqZebIVOy+IG+uqOXDP07C7Q/ucfcRRBJOgztyRI4fnsyirS3ccfIwvtrczItLquhyB5iYa+PSqTmYdWqe+kMJ3kCIsBIpV//J+ga+LWtnXHYsdZ3916OZUhAX6SuzoybJ3kiShC8YxqhVEycKlwnC72qfByKPPfYY9913H01NTRQXF/PII48wduzYff20wm/E4QnwyILyXtPaW5oczH1pFfeeNoyJuXFE6TXcPrMIfzDMW6tqWVHVzo3HDibGpGVxRXu/161ud0dqVDy1lAfPGoFWJaGSpT6VP7UqFS5/kGXbO3h4Ry2J0Zkx3HP6cB76emuvEt13flbK+3+ctNfvR6OSyI43EWPUcmJxMiMzrD9545EkCb1GxOw/lhpjIDUmlan5cTR2e3l9WQ0FiWYkIruY9lbPRKuSUckSRSlRXD0tn2A4TDCscHiBjSMHJxAIKYTDCsFwmIpmJ2+sqMUXijS521jfTViB4rQo7j9jBJ9uaOyzS8dm0nL0kCRaHD6e/2E71x1duMcePqMzYzDrVCApRO+lvo0gCPvGPv149+abb3Lddddxyy23sHr1aoqLiznmmGNoaem/4JGw/2l2eHllWd+1dYC7Pt9CXZeHv7yznpve38C6+m6mDUrkL0cP4qb3NlDbuedPxBBpI69Rydz8wUaSogy0OX10e3q3cK/rdHPW/5b2mlpfWd3Jla+t5s9HF/b6pOvyh7B7Apj2UHxqeFo0W5udzD0sUv102qAE8en3NxBj0qHTqGh2eJlZnIpBq+LRBeV0ewIU7KH+xqmjUsm0GXjugjGkWPXoNSp8wRCyKtKxuKrdxWMLt/HemnpyEs0sKm9jWWUHG+q7uXBSFt9cP4XHzi2hzenjpQvHMj4nFoiUfT9maCJP/aEEBWjudnN6STo6tcx1RxX0GYdFp+bWmUMJhhQSLX2LwwmCsO/t0495DzzwAHPnzuWCCy4A4Mknn+TTTz/lueee48Ybb9yXTy38St2eAA5vAAmIMWrZ3ura41JHpztAh8vfU+57RVUnJwxPpig1mssPz0UlS2hVMv5Q3wJiADaTDrc/hNsfoqHby2WvrOLCSVn88Yg8bGYdwVCYN1fU9tve3u0P8fnGRo4cnNirRokkwe0nF3HDO+t7ynwDxBg1/G3G4Ejia7yZZKvhd19mCYXCdHsC+EJhUCLVUw+WpR6VBEUpUVj0akJhhYpWF++truGJWSX8+e11rNnRd0aW4KQRqVx5RB5pu1WFTYsxkrZbEdnUGCNHDU5EliWq211oVBJXHJ7LzOJUPlxbz/Vvr8dq1DKzOIVmu5dRGTFcOCmb9Fgj9Z1urAYNF724gkun5vLIN1u49cQhnFScwtisWF5aUkWLw8fY7FhOHZmGVa8i2qiN1AsRBOF3t88CEb/fz6pVq7jpppt6jsmyzPTp01mypG/9AQCfz4fP5+v5t93etymasG8EQ2HKW53c8Ukp35e3oZYlThmRysmjUvf6OI2q9x/vT9Y3MrM4he/L28iJM3HOuHReXNx3RuXwgnjW1O7aCRHeEe0890MVY7NjGZkejTcQ5tttrX0eu9Pq6i6OLUrqCUQSLDr0ahUfrm3gpYvGsryyg+2tLsZmxzIh14ZeLTMs2YLJ8PtPvzd2ufH4w7gDISQJwmEFlz+IUasixXrgfxJ3+kJcNS2fj9Y1MDHXxpjMGOZMzOHcp5dy3oRM/jgtD38w0oSv2xOgyxMgORRGtZdAbGdgEK1X8+nVkwmH4bQnF/dKUF2wpYXzxmWg16q45OVVFCSaOWtMOhmxJpAgSq+hvsvDE99u5+FzRpIWo+fm44cQUhTMOhUGrRqtWiSkCsJA2meBSFtbG6FQiMTExF7HExMT2bKl/74Od911F7fddtu+GpKwF9Udbk5+7IeetfZgWOHt1XVMH5pIlF6N3dt3VqIoNYqtzX13RSyr7KBxx1bdc8amMz7Hhl6twu0PYdSqcHgDGLRqrnwtUinTrFP3zLqkWQ0UJFp4bUe/kTjTnrd12sxaHN7ITUmrkrn7tGF8sr6R9XVdRBs0nDIiskyAohAMK6TEDMwNv9Pli2xxliIBnz8UjvRBUcu4/SFaHV7iD/AlIqtRw7Pfb2fG0GSCoRB/mp7PPz7YSLPD16fWCMB7l09kbW0nWTYzNsue3+NWh5cnFlYgSxLlrc5+d8m8sqyGZ+eM5sXFVWxtdpIRa4p00I030eLwEm3QcPSQRGwmLZIk6oEIwv5mv8rAu+mmm7juuut6/m2320lPTx/AER0aPIEgj39T3ifhD+CJhRU8eNYILn9lda8llliTlj8fXcgN76zv8xhZklCrJCQU2p1+3lhey7dbd81sTMy1MfewHNSyhI9IXQ5fMMgTs0aRYjWwqrqD00elodeq2NrsZOHW/mdFzhufwVebW7h4cjYnFqfgDYRIiNLx+iXj+dt7G7j4sBxSrQYKE83otQP3o+7yB6lq83DnZ6Vsbox0gJ05IoXjhyWTGmMg0M/rfqBJsOgYnRXLgq0tGDUqzhyTzrrdEol3MulUPHDGCAwamUBIosPtxxeMJLbunAEJBCN9f7rcAZ5cVMErS2t4fNYonvuhco/Pv6q6kyHJUayr6yYQChOlV1PW5OS0knTUssT0IYm/upGiIAj71j776xwXF4dKpaK5ubnX8ebmZpKSkvp9jE6nQ6cTn1h+b3ZPkO+2tfX7tbW1XSzb3s6HV05i3qYmajrcjEiPITfBxD/e30iLw9fnMWOzYylrtlOSGcsjC7b1CkIAFle0E1YUbps5lOx4E4lReiQF3IEgallmZEYM29siHW4zbSYuPzyHJxZu73WNiyZnMzzVSlFqNBpZQpYk6ro8lDXZue3jzSRF6XH5goQV5XcLQpzeABqVhE6jJhQOEwqDhEJ1u4fZz/XuAPvmilo21ndzzfR8hiQd+O0LtGoVJwxPoTgtmleX1dDm9Pc5R6+ReeeyiXyyvgGdRuajdQ1sa3aSEWtg7pRcEiw6Grs9bKzvJs6soyDRwpu7FabbQ79BAEJhBVmKzDLp1DKJUZEEWJUs8dKF40gboNkwQRB+2j77C63VaikpKWH+/PmcfPLJAITDYebPn8+VV165r55W+BVUsoTVqOk3qABosvtYXN5GqtXA+vpu3l5Vy5+PLqTT3fdmc0ZJGh5/kCMHJeL2B/l2aytDU6LQqmS2Njtw+UMAbG6wM/rUGLyBMK8urWZ7m4s5E7LY2NDNfV+W9fQh0WtkXrpgLCcVp7JkeySAmZBjw6yLVEaNMqi5+YNNrKjalW8iSXD90QVUt7soTrf+9i/Yj7Q7PTi8YWo73ayr7SI91siw1Gg63T5MWk1PB9gf29Rgx+ULRZJXDwLRBg3RqVZOHhHim7IWJuXa+GG37dv/OH4wH6yuY0hqNBe/uLInmXhDfTefbmjiwbOKeW91fU9Q/MSsUT0/B+tqu5iYa9vjdvDRWbG8sLiKWeMysBo1fL25mSfPKyHKoD7gl70E4WC3Tz8qXnfddcyZM4fRo0czduxYHnroIVwuV88uGmH/EGfWcdHkHP76bt9lFoBji5LYUN9NVpyJyXlxfLCmnme/r+TZOWP4vryVBVtasRo0nD46jWybCV8wzEUvrOCpP5TwwgVj2dJkx+ULcu1R+QRCSs+n14e+3kaXO8D4XBtnlMTg8Aa487Pe+UPRO1rMN9u9hMMK35W38cIPVUwblMBxw5JZvr2T22YO5YGvtrK2tovByVHMHp/J0u0dHD00kdh9XBeizenB4Q31lJ/fyaRV8fwFY0g0a3n47BH4giG+2drK04sq6fbsynNYV9fFyN8hWPo95cabeWlJNVcckcfGBnvP9zsiPQadWsWdn5X22tG009/f38h/zx7RE4jsvqPorZW1PHjWCNbVdvUEszvNLE6hvMXBdUcVcPSQRFSyxEkjUw6KJGBBOBTs00DkrLPOorW1lX/+8580NTUxYsQIvvjiiz4JrMLAmzYonumDE/i6tHeNl4smZ7Ohrpsx2bFc/OIKsuNMHFuUTEGCmc2N3Rw7NIlpgxKw6DUYNTLeoMLFL67knycMYVV1F7d8tAmbScudpxRh0qmpanOxvq6bF5fs2kmzcGsrqVbDjsJmvbf7Xn54Hi5/iHu/2NKzTRjgxSXVfL6xiftOH86FL6zgtbnjaej20GL3IUtw5ug0wopCu8tPWFFIiPrtPxWHwwoef5jbPyntFYRApKbJxS+u5N3LJ7K9LVLJ9fCCeKbkxXHRi6t6qsLazDq0moNjC+9OcRYdt5w4hKXb23n/iol8ur6R78vb8AXDmPVqmu39z7y5/SHCYQWNSiIQUlhT08mkPBs/lLfT6Q7wn3llPHFeCZ9taGRFVQdWo5aLJ2czKNmCLEnEmzUYdaIgmSAcaPb54vmVV14plmIOAPEWPf86qYjZExx8W9aKVi0zJiuWxRVtpMcaeHNFLWEFKlpdPPZNORNybFx8WBZqWabJ7kOnjvSeaez2UtnmIi3WyGWvribeouOp80pYW9fF1W+s5dFzR/YKQnaq7/Lw5ooaji1K4qN1uzqgFiSa2dxg7xWE7NTi8LFwayuDk6N57vtKrj+6kPyEENXtbi57ZRVV7W4AMm1G7jt9OCPSrXvdqtnu9NHh9hMIhok2akn8iTofHW4/Hn+Yb8r6L9Bn9wZ3dPDV8Je31zEkJYprjszn7lOHcdFLK5ElmFYYj1W/X+WM/yYSovR0eQIEQmEWV7Rz68yhqGWJVsfeHxfYbabkpSXVPHruSOyeIBvqu9lYb+fCF1Zwzth0/jd7NHq1jEGrItqgOWjqsQjCoejg+wso/GopVgMaSSLLZqKqzU2n28f4HBsvLanuk8w6KS8Oo1bNtAe+BeDdyybwxspazhuXyZyJmby3ug6I5Gp0ewPc/fkWRmXE7HGNHyI1SO47fXhPIKJRSWhkaY83eojUkThnbAaPLijnj9PycPlCzHpmGYEdTczOKEnjrDEZO6p1uokxavrNGShvcfCnN9ayqcHOiPRozh6dzuisGKxGLb5gpA19tzdAjFFLeqwxsgNDUfAG99zbBiLByqcbGpk1PpP7viwjJ97M2WPSiTFquGVHi3mDTrPnCxzARqRb0alVNHZ7mPXMMj7640Qseg3xFh2t/eQjGTQqtCq5Jy/E6Qty9RtreH3ueIIhhep2Fxk2E8nR+j6tAARBOHCJjxFCL/HRenRqiXSrnqLUaK5+fW2fICTOrGVKQRznPrMMgCybEVmW+HhHAHHKiFSa7ZFOtjGGSBKsLxhGrZLwB/ecmBkIhVHL8m7/VtBpVGj28mlXq5IJhhQURUFR4MUfKgmEFFSyxFOzR3PmmHQe+KqMOc8v58O1DWyst1PWZKep202bw8P2Vge1HS7+9PpasmKNzL9uCv85fQTD0qyEFOjyBGhz+mh3+7HoNXj8QUobI4X2DFo1Jq2aWNOelwOybCa+29ZKdpwJgDdW1OAPhnl97njG58QOWG2T30NajAGHx89Nxw2mw+Xn2P9+R1ackVtPHEJ/RUz/ckwhry2v6XUs1qjFpFVj0auZWhDPiHSrCEIE4SAjAhGhj6RoIzqtCoNW5tW545iYawMiu2uOK0ri1YvHc9Xra3pmApKjDfiDYbyBMF0eP7EmLZPz4og1aWmy+wjtmG5fV9fFuOzYPT7vEYUJ2Mxakna70ayp6eTkEXuu7nrC8GTmlzYzc0QKeo3M6h2lxP8wIZNog4a5L63CFwxzz2nDWVHVwZ2fldLq8NHhDrC12cWWJidPLKxg7pQcrpiWhy8Y5tq31lDX5eGpRduZ8d/vOemxxVz84krmlzajUanodPtpsXvpcPnZ1NDFn47M63dsE3NtbGt2oCi7mq3ZPcEdr5l+r03hDgaxJh3JViNFKRaeml1CYpSeqfd+Q1FKFB9eOZmTRqRQkGjmqCGJvHPZBAYlmdlUHwnyJCmSt/Tc+WOw6FXkxJtFQzpBOEiJpRmhX6kxRpq63cSZNNx+UhEuf6QmR3KUnq3NTqp35F9AJL8jrCgMS41me6uLDqeP4vRoDBoVNpMWo06NWpawe4K0u/xMyLWx5EdLNEatiquOzCNKr+bpHQ3LIFJ1Va2SOHJQAvO39F6iKUqNIj/RwitLa/jvOSMwaFWkxxrY1GDn+GHJLN3ejjcQ4soj8rjk5VXMmZDJCcUp/O39DWzcccNLjNJxzfQCvtvWyg/lbeTEm5gxLJnnf6hk6fZIo72CRDOzxmUSZ9bR5fGTG2/G6Q3g8of505vr+PJPU7jv9OHcP28rTXYvBo2KU0amMqUgnj+9sYajhiTy3Y5S9Zk2I3qNfMjcVOMsOkBHtFHDM3NG4w2EcQXCdDm8/H3GIDyBSNn3QDiETiXz9mUTcPmCaNUyMSYNcWYx+yEIBztJUfa2wj2w7HY70dHRdHd3ExV14Bd9OhAFgmHanF66PQECYYVovZo2Z4BTn1jc67wXLxyDxx/irs9KuXVmETnxJryBEPVdHr7b1oZRq+axb8rRqWXuPm04DV0e3ltdj8MbYFJeHJcfnku0QY1Zp0azo0W8hNRTbbO520Npk4PXltUQCCkcW5RItEFDZauLE4tTehqora3p5OTHF/PuZRN4eEE5xWnRJFsNNHZ5OHZoEmc/s7RnVmJ3j88axS0fbeLOU4qQJYmLXlwJwGmjUpmQa+PRBeVUtbtRyxLHFiVx/dEFBIJhjn7oOwAW//UIGru9NDt8hMMKn21s5MtNzSRH67nzlGFc+vIqPIEQD589khOLkw/ZKp/eQJBOdwBZkojSq1CQCIbDoEhEGQ7OXBlBOBT9kvu3CESEn8XhDVDe7ODil1by7uWTuP3TzczfbatvXoKZe04dRrvLz+cbG7lkSg4WrRpXIIQkSSza2orNrOOtFbU0dEeCgtNL0vCHwthM2p+9TOHxB3H5Q4TCYSRJwmbU9jRO8wZCtNq9uAMhZClSh2JTvZ2vS5uJN+uYOTKVpRXt/Puz0j7XHZMVw5isWAKhMCPSrfzxtTVkxBq5/ugC/vTGWiCy1HLu2AxkOVLBMz/BzLVvrmNVTScTc238fcZgWh0+FpS10On2Mz7bhtWo5V+fbCIYUrjh2EFMH5KAbS/9cwRBEA4GIhAR9ommbjdvraznuKGJKEjc/1UZ8zY3oyiRNf0zS9K4cloene4AKkkizqxBliVc3jDeUBBFkZAAtUrCpFNhNWoxaH6b1cFOl5/SRjvJVj1lTQ6GJEdxw7vrWVbZ0WtXy80nDEZR4I5PewcjJq2KW2cOZWuzg8Py4/nDc8v567GFfLSugdJGB+eNyyAn3sxDX2/taQA4JDmKh84ewSUvraSq3c3hBfHccGwhIUUhuGMLcCCk4A+FiNJrSI8xilbzgiAcEn7J/VvkiAg/m06t5pSRqXyyvoHpgxP424zBXD0tH4c3SIxJQyAU5sRHv6fLHblR//vkIiblxaLXqpGCalSShCwpaNQy8ZbfNlGzvsuDLIHLGyTFamDh1lamDUrk0im5fLahkbdXRbYT3/5JKZ9cNRm1DLtv4Em2Gmh3+hmXY6Pb48dm0pJpM1Ha6CDBomN8jo0rX1/T6zk3N9q58PkVvHzxWNqcfspbIt1hM21GJAAJog0qYk3aQ3YpRhAE4aeIQET42WJMWsKKwonDUwiEwnR7IjMf0UY1X2xs4uWl1XS5g2THmbj+qAKGp0eTZNGj1ey5iNiv5fH5CYQhEFRQqaC2w01ugol3V9Xz1Hfbe2ZBJAkum5LLZVNzePLbSOO8hWUtnF6Szhu7NVSbNS6DLpefzFgj//pkE/eePhyXP0iUXs1po9J4aWnfImwAdV0ePlzbwJ+OzGdM1p53BAmCIAj9E4GI8IvYzJH8BqcngFmnxhcKIwOnlqQxY1gyigIatURDpxe1LP+mQYjDG8DpDeD0hdjUYEclSwxOtqBVyxSlRuELhpg+OIFpgxIoa7Lz4PxtdLoCPPFtBY+cMxKrUUOXO4DdG+yp/SFJcO6YDMZmx7K6upOlFW3cfeowAiHwh8K8dOE4XL4AH69v2OO41tV24w+F0e2laqsgCILQPxGICL+K2aDBvGOXQ32nh7dW1LC8sh1FgTHZkaTOFOtvt/zi8gXpdvt5a1U9jyzY1jPjkRZj4IlZo1hd04lKlpmcH4tJoyLTZuS4YcnUd3o495llfLC2nuOKknh9eS1TC+IJhsIUJFoYkhyFRa8iIcpAnFlHOBymyxOkosWJWiXj8QfZUNfFQ2eN4IpXV/fboXhwsgWtKDEuCILwq4hARPh/S40xcPWR+XS4MgGINWn3Wg311/AFQpS3unh4/raeY1qVzF2nDMPuCTAlPx5ZglAY1tbZ2dLkYGS6lbQYI9/dcDhPLdqOQaNmbHYs6bFG9CoZtVoiSr+rT0kwGOaFJVU890NVTxG2OLOWO04extPfbedfJw3lsldW9xqXWpY4vSRN5IAIgiD8SiIQEX4TGpW8T0tv+0Nhnvy2otexY4uSiDVpCIQV5m9pZkpBAle8upqpBfGMzorh9eW1tDp9TMq1ceboDGKNas4bn0HMblt+d2qxe/muvI2nv6vsdbzN6eeaN9fw+KwS3L4gufEmKlpdAFiNGh45ZyRpMQd3hVRBEIR9SQQiwgEhEArT1O3tdeykESmoJJn1jV1kx5m594stFKdFY9KpuHy3mQuHN8C47FhMOlOkA6zTh9sXIjFKj2lH51uXL8ij35T3+9zeQJhNDd2gwH/OKKbN4SPeoiMhKtJ8TSW25AqCIPxqYmFbOCAYNCqK0629jiVH67H7Ary6rJrEKD3flLVwYnEKjyzYFVD86ch8npldQlK0gc0Ndh78ehuvLavBFwrTbPfQ6fIDoEhQ1+nZ4/PXdLjJsBm5f95WsuNNjMiIIcVqEEGIIAjC/5OYEREODBJcOiWHzzY09rSJ9wXCKEQCCH8wTHqMkS1Njp5E1om5NmYOTyKgwGUvr2JLs6Pncg8vKOeWE4cwrTCBGJMWGXotu/xYXryZoSlRjDiliPSDuGOuIAjC703MiAgHhDiznlijhucvGEtuvBmAeaXNmLRqChMtKCjYTFqCoV1lVC+YlIVBq+L57yt7BSE73fbxZlz+IHZPAJ1axXVHFfT73NEGTaSbsFFDps0kqqMKgiD8hsSMiHDASLIaMenUPDOnBLsniCSBRadm7pQcXlhcxWmj0kjabcuwzaTDF1J4a2XdHq/5+cYmLpyURVK0nuJ0K7ecOIT/fFmGyx8CIj10HjprBDazhljRCVYQBOE3JwIR4YBiMWixGLTUdrj4bmsbOXFG8uNNTC1IQFHCGDUyZ4+JVE2VZQgr4AmE9ni9Lk9gR68ciVSrgeOLkji8MJ4udwCtWsaiU2PUqoiziCBEEARhXxCBiHBASo81cWqJnk63H5UkceSgeDrdAUJhhcun5nLEoAS2NTkYn2ujJDOGVdWd/V5nUq6tZ+eMJEkk7OgC7A+ECIUVdBqVWIoRBEHYh0QgIhyw9BoVydG7lmJiTLqe/0+26ml3elHLEjcdN4iznlraU6Rsp6LUKPITLP2WZt8X/XEEQRCEvkSyqnBQ0qpVJFtNxEcZSYnS8+Yl4xmXHduTV3L+xCwePHMEqaIYmSAIwoASMyLCQS8l1kiS1cB9ZwzHFwgjSRCl1xBv0YnS7IIgCANMBCLCIUGWJTJiTQM9DEEQBOFHxNKMIAiCIAgDRgQigiAIgiAMGBGICIIgCIIwYEQgIgiCIAjCgBGBiCAIgiAIA0YEIoIgCIIgDBgRiAiCIAiCMGBEICIIgiAIwoARgYggCIIgCANGBCKCIAiCIAwYEYgIgiAIgjBgRCAiCIIgCMKAEYGIIAiCIAgDRgQigiAIgiAMGBGICIIgCIIwYEQgIgiCIAjCgBGBiCAIgiAIA0YEIoIgCIIgDBgRiAiCIAiCMGBEICIIgiAIwoARgYggCIIgCANGBCKCIAiCIAwYEYgIgiAIgjBgRCAiCIIgCMKAUQ/0AAShP4FQGLUsUd/pJhBWqOnwsLKqnZEZMWTZTKhVEtF6DSadmvouD3ZPAJ1aRq9RYdapkCUZq0k70N+GIAiC8BNEICL87py+AB0uP75AGItejUWnxukL4g2G8ATCfFvWyoqqDrJsJk4emcpbK2p5f009D5w1gm+2tGLQdDAxz4bbFyIhSocnEKKixUF+YhRWgwZZAl8wQJvTR5RBQ0KUfqC/ZUEQBGEPJEVRlIEexJ7Y7Xaio6Pp7u4mKipqoIdzSOpyebF7Q8iShFErE2uO3NRD4TAquffKntMXwB9UsOhVaFSqfq9X3+nm35+W8sWmJvITLNx8whA0KonF5W0UZ8RwzRtrsHuDPefLEvznjGLy4s3c9P4G/jAhk1VVnby9uo6dP7kJFh2PnDOSOz8tpc3l56wx6YzMsGLRqVlX18X0wYmkxhj3zQskCIIg9PFL7t8iEBH65fb6aHUFqW53saq6ky1NDk4vSSM/0YJBLfH68jrs3gBnjkkn1qilvMXJ/xZV0Ob0MzkvjnPHZZAWY0QlSz3XbLZ7mf3sMrY2O1HLEs+dP4b/fVvOsUXJuPwhPlrbwOZGe5+xpFr13H5yEc98V8mEXBv3z9va5xyLTs3Tc0Zz9lNLAZg+OIEx2bFMzLXh9AbJTzATZxEzI4IgCL+HX3L/FsmqQh+NXW5+qOjkz2+v47aPN9PQ5WXWuEwe/6aCRxeU4wmEObE4GVmWOPah7/hoXQNvrKjl261tbGqw879F25nx3+/Y1uLodd2qdhdbm50AHD00kU83NHJaSTqPfVNBdpyp3yAEIDvOzLdlbcwen8lz31f2e47DF6Shy0NuvAmAr0tbyIkzM39zCx+tbcDuDVLd7mRbs4NFW1tZU9NJm8PHfhyHC4IgHBJEICL08AdDVLe7eOLbCua+vIoVVZ1UtLp4e1Udl768in+cMJjGLg+tTj9qWWJaQTxpMQb+/Vkpp4xMRdo1+YHLH+KWDzfR5fb3HFtf293z/8VpVhZXtGEzaWmyewmF9xwQeAIhYkwaEqN0dLoDezxva5ODIwYl9Px7c6OdqnYXRp2atbVdOLxBLnl5Je+trqfLHeCbshZ+qGijw+XDEwgQ3ssYBEEQhH1DBCICAKGwQlW7m7pODy8tqenzdU8gxF2fbeH6YwrZUNeF1ahFo1Fx5uh0FAU2NdjJizf3esyyyg7snl2BQ4p119KINxjGotNQ2+kmy2bE6QuSGKXrd2w78zyQJJL2knial2gmEAr3/NusU6FRyXgDId5eWUc4DNccWcCUgjjMOjXDUqORJYkHvtrKPz/YzLraTqranFS1OanrdOPy+ff4XIIgCMJvQwQiAgAdLh9rajpZVd25x3NWVnfSYvdRnG5FUSAQDJMaYwAigYpW3ffHafc5huHpVgyaSBLrFxsbOWlECi8tqea6owp4eUk11x1V2GtWZadzxmSwvq4LSVG4dGpOv2NLsOjIjTezsioyfrUskRNv5oThyayu7kRBQQEKkyzEm3UYNDIvLq7i3KeX8dqyGkqyYvi+op1znl7GEfd/y2WvrGJldRd1ne6f9wIKgiAIv4oIRAQAut0Blm/v+MmcCQWFhWWtdLn9qGSJ5m4vAMVp0ZS3OHudOy47BqtB0/Pv5Cg9L100FrNOTWmjgzizjniLjvouD6eOSmV1dQdPzCphQo6NKIOawckWbj+piLQYA397fyMuX4hJeXH88Yg8jNpdu3KGpkTxwJnFmHVqNjXY0aolXr5oHCatirpOD+eNz+TvMwbT7QngC4Zo6PJQ3+3l9RW1AFwyJYcftrVx/7ytNHZ7URTYWG9nznMrWLa9g3an97d6mQVBEIQfEXVEBADCKDQ5vBxfnLLHcybm2lhb08W2FidHDU4kpCi8s6qOowYnsrXZiS+4a1nEqFVx20lFRBt3FRVTq2RGplv58prD2NbipN3l418zh9Lm9FPR5uTCydn4gmFuP2koKlkiDFz75lrW13WjliVsZi1fb25icq6NkkwrHn8YrVrC7gmgkiRQ4LIpOZw7LpPaTheEJQqSzJQ22rnq9TX84/ghqFQSBUkWnv+hCgCVLDE228aF367o93u+98stDEsbh80sdtwIgiDsCyIQEQCw6DUcV5TMyqoO/nv2CH4ob2fp9nZqOiJLE1EGNZdMyeGhr7by4FkjiDJoUJQwT543ErNeS1WbiykFcbQ5/EzOs3Hu+EzS+6ndoVbJpMYYe9X1yE2IzGo8+PU2ajvc6NQyW5udTBsUzwNnFvPGilq2NTtRq2Tu/2obmbFGZk/IJDFKT1iBbS1Obv+0lIsmZXNsURLbW52EFIWl2zsIhsNMyLXx6Lmj+Os763jw7JH4A0Fc/kitkgSLjqo21x5fl2a7jy53AKfXj1kvKrUKgiD81kQgIgBgM2kZlx3L5gY7n6xvJBhS+OMRecQYNayp6WJ8Tix6tcSDZ42gw+2n2eElGII4i5ZgKMzQlCgenzUKfzBSLXVPBc32xBMIM29zE7Udnp5j29ucHDU0iSn5cUAk3yQUVtje5uK2jzf3uYZBq0KnkfGFwnS6/WhUEh+ubeT5H6o4cnAC/zmzmMXlbYzPsXFYXhzzS1vwBkKYdHv/NTDpVDh9IcSkiCAIwm9PBCIHEI8vgMcfotsXRFFAo5JIsfYuGvZrdXsC3P35FuZvaek59k1ZC8NTo/nzMYXUdbiwmnRc+cZSWh0+AIYkR/HPE4cQ1CqRyqs6FTFGLVJ/Gac/waJXMzk3jtc7anuOBUIKV7++hifPG8UZJWloZIlJeTZ+KG/v9xpHD0lkQ303jy+soLbDTX6ihX+eMITFFe28tryG6YMT0WtUVHe4mJQfR5bNSFW7G6tRg14j4w2E+1xzTFYMalnm842NHD88hQRRFE0QBOE3JSqrHiCa7R58wTDfbGnlhcVVtDl8jMywctW0fFKs+v93CfMFpc1c+OLKXscyYo1cfFg2g5OjQIEOt5/3Vtfx5aZmAG49cQg58WZSovV0ewM4vUE0KhmrUUuMUUOy1fCLxrC91cmMh7/rExDEmbQ8PWc0MUYNjd0+LnxhBZ5AqNc5D589gu1tLh76eluf6946cyifb2jE6Qty/xnFbGtxYjVqyLaZaOz24vQFiTNreXtVHW+uqO3JdYm36Lj/jGJSovW4AyHUssSQlOhf9D0JgiAcin7J/XufzYhkZWVRXV3d69hdd93FjTfeuK+e8qBld/vxBUIs3NLKiIwYHjizGLUsoQAPz9/GrHGZ6NUytp/xad0TCOLwBNnW4sThDTA4OdIo7sUlu94rq1HN32cMpijVynPfb2dNTReN3R7W13X3BCbtTh8atYReI3P+Cyuo64wsqcQYNVx3VCEGrczY7FiiDRqiDT8vtyIj1sh7l0/ito83sqyyE0mCIwrjuWZ6ATFGbSR/RCPz3PmjeXtVHSurOjgsP56jhiSSEWvk8YUVJEXpabL33uXy36+3cuvModz3ZRlatUSn28/80mamFsRzw7vrCYQUVLLE8cOSePOS8by+oobCxChSrAbmbWriiEEJ3PTeBl68cOzPf9MEQRCEn2WfzYhkZWVx0UUXMXfu3J5jFosFk8n0s68hZkQiqtpcKIpCaZOD2z7eRLM9sjSSHWfi3tOH4/EHSYsxkh5rQFHA5YvU9NCpZVocPhzeABa9Gl8gzIKyVkob7eTGmyhItHD/vK1cf3Q+b6+qJ9qg4YrDs/EFFL4rb8PrDzMp34ZBo2LepmaGpUXz5LcVHD8shaGpURg0Kk5+7Ideu2V2euK8UcQYNWTZTOg1KqzGn5/o2e320+0NICFhNWpweAM9BdeueWMtfz9+EDlxZnQaFS12DylWIy5fkLACDm8QfyjEU4u2s6JqV02Ux2eN4rttrZw2KpX7vtzK+BwbZr2af39a2uu5p+bHcUJxCk8srECjkrj7tOHMemYZbn+IPx6Ry2VTc7HoNT8esiAIgrCb/WJGBCKBR1JS0r58ioOaoig0dHnY0tBFaqyZpxZV4NytM21lm4vZzy7j/Ssmopagss3Nq0urWVndSVKUnjkTszBqZJqdPqL0Gi59eRVu/64lDYtOzdN/KMGs13BmSRpV7W6q273EmrS0Ony8tKSa++aVcdzQJG6aMYgb3lnH7AnZzNvUSFGqmZp2L6eVpFHT7qasyUGr09dz7VeWVnPOmHQyYo04vAE63X5SrcZ+i579WLRR22vbr0WvoabDhVGrJiPWyANfbeO2mUMB+KG8jVeX1eALhlHLEjOGJXPC8GTmTMhEliSWVXYAkQJns8Zl8t+vt+EPhkmw6ChrdvR57m+3tfGnowq4/uhCHN4AF7ywouc1+2xDE2eUpIlARBAE4Te0TwORu+++m9tvv52MjAzOPfdcrr32WtTqPT+lz+fD59t1M7Pb+2+Cdqho7PYQDCnkJkWxoLSFCTk2/jZjMIoCpY12viptZnFFOwu3tjE+O5azn1qKLxhGp5YZmhKN2x9Cp5YpTLDwxLflvYIQgGFp0Zj0GuY8t5x2165y5mkxBh46awRTC+L52/sb+XxTE1MK4/n3qcPYWNfN9UcX4gmEsOgVZhQlkxStQ6OSaXf5eGpRJV9sbGJbs5OsOBNbmhzoVDLlLQ7qunxcfFg2iXsp074nGlki3qLl+qML2NrsYG1tF50uP68s21WOPhhW+GhdA3ZvgOK0aOZOyWFZZQfZcSZy402sr+/mq9Jmrj+6gLwEM/d8uaXf52ro8nDLhxtpd/Xua6P7GUGUIAiC8Mvss0Dk6quvZtSoUcTGxrJ48WJuuukmGhsbeeCBB/b4mLvuuovbbrttXw3pgOP0Bllb182N764nNcbAv08exvtr6rF7AhxemMCVR+Rx64lDaXV4+fv7G/EFwxg0Kh4+ZyQfr2vgytdWY9SqOL0kjZkjUjlheApXvrYGlz+ERiVx8wlDmPvSyl5BCEBdp4c7Pi1lxrAkbjlxCOc/v4JnvqtkUm4sBUkW1td1848PN9K1owGdRiVx5RF5HDM0ieOGJhFr1FLW7ODbra2UZMZS2+7iiEGJTLlvIb5giBuPG4RR+8t+9Cx6DXaPn0ybEasxMmMz96WV/Z67sKyVOROyqGh1MjjJwr9PGcbXpc3c9XkZWTYj0wcn8sTCCuyeYL+P12tUfYIQgFNGpmLQiI1mgiAIv6Vf9Ff1xhtv5J577tnrOaWlpQwaNIjrrruu59jw4cPRarVceuml3HXXXeh0/Tc3u+mmm3o9zm63k56e/kuGeNBQFAVPIMxf312PBNxxUhH/+GADfz9+CKurO/nPvDLcO0qeX31kPvefMZzKdjexJi33zytjRVUnZ4xOY/rgRN5eWcvtn5SSn2jmf7NLeHNFLZIEnS5/T5Lpj62t7eKPR+SxbHsHE3JtbG1yEAxDm9PPtW+tJRDalVoUCCk8+PU2MmKNFKdbyU800+0JcO2b63j2+yoeO3ckgVAYlSzx+vIaLpqcTabt5/3ohcMKTXYv29tcNHV7GJcdiyxBIBQmuJduuZ1uP25/iP/NLuHdVbW8t7aBPx2Zz7FFSWys7+q1jLS7QUkWvD/akQMwIt1KeqwRf6hvPowgCILw6/2iQOT666/n/PPP3+s5OTn9NyUbN24cwWCQqqoqCgsL+z1Hp9PtMUg51HR7/Mzb3ISiwLTBCczf0sIVR+Tx6IJyNtR395z3xaYmvt3ayvMXjCFar8btD7KiqpPJeXEMSY7i0pdXAZEbbJvDx7VvreX6owuJNWjpcO29u2wgFGbJ9nbGZseSaNHh9odYtLWtVxCyu6e+285Nxw3iiYUVDEuzctW0PP7+wUYc3mBP19xASKHLHSDT9tOvQTissKnRzuxnl/XMvgC8del49Jq9F0yz6DWUZOp54YdKjh2WzEkj0+hy+3D5AhSlRFOcHsOf31rH+t1ey9x4M3eeOoyyJjv3n1nMV5ubCYUVjihMQKOSeGDeVl6+SOycEQRB+C39okAkPj6e+Pj4X/VEa9euRZZlEhISftXjDzUhBVp27I4ZmxXLF5uaGJYa3SsI2ckTCPHc95WcMjKVDnckuJg1PoM/v7WOqQXxnD8xi00NdtqcPs4ek4FaJZFs1dPl6bv8sJNOLaOWJaINGnyBEHMPy8HhC/aUfO9PTbubYEghzqzjqUXbufiwbI4ZmkhVu4thqVGEdsxg/FQl050a7d4+QQjAzR9s4oEzi5mQa2NJRd/iZnkJZuyeADaThucWV/Pc4mpmjkjhmiPzAajv8kTKwp9cRFhRaOz2EqVXYzNpaXH4uem9jcSbdUzKi0Mlw+MLy6nr9PC3GYOJ1oulGUEQhN/SPvmrumTJEpYtW8YRRxyBxWJhyZIlXHvttZx33nnExMTsi6c86ChhOHxQPG+vqiMYVhiWGs2S7f1XFAX4dmsrp45KJSXagCSBosDIjBhOLE5m7ksrey1j5MabefK8Ubi7PcwsTuGjdQ19rjdrXAafrG/k1FGpZMQaMevVaFQS+YlmvtzU/xgKEi1IEjTu6Mj7+rIa7jl9OEaNmmA4sqQxPieWOPPP28pb0eLsE4QAuPyRXjG3nzSUq19fy+bGXUnNmTYjd586jIQoHec+tazn+EdrGzh9VBqdLj9/enNtz3GrQcMT541CJUk02r0sLGvlqml5PLKgnA/W1vecNyHXxnFFSRjFjhlBEITf1D4JRHQ6HW+88Qa33norPp+P7Oxsrr322l75H8LeWY0aBidFkWkz8vXmZuZMzOxVF+PHDFoV3kCYxCg98SYdkgQXTMri8ldW9wpCdlZff291HbMnZIICqVY9Ly2pxuUPYTVqmD0+E5tZh1mnZmS6FQXQyhKtniCTcm08931lnx04AJdOzcGoU7OyOjJOlz+EViWTYTPQ7PBTmGjhntOG/+yaIs0/Kky200nFKdz9eRl2j59nzx9DeEf/GYteg82sRSODyxegrqt3/sumhm6GpkSjUUk9y0tdngCXvrKKZ/4wmmybiQuXrOTsMek8d/4YVlZ14AmEGJ0ZS068ScyGCIIg7AP75C/rqFGjWLp06b649CFDrZIxamWemzOGRxZsQ6dRMX1wIq/utl11dzOLU3B4A7yzqpa7Tx+OWpYob3HiD4XRa2TGZ9s4rCCOIwcnopElZElCkmBMVixmvZpTRqURCIVxeIPUtLsoyYxlfX0XV70e2WVz1JBEThuVilqW+N/sEm7+YCNV7bs689547GASLXru+mLXlli1LJFhM1LaaGdkupUXLxhD0i8o+16YZOn3eE68GY1aZkhyVGT3izfAtEEJWI0aHvp6Ky0OH/8+uajP4wxaNR0uHyadutdMi90TpNsdICVaz8WHZfPMd5W8vaqOYanRaNUyUXoNIzOsRP2ComyCIAjCzyM+4u3HkqKNNHa5ueGYQjzBMGpJ4pIpOTy1aHuv8/ISzBw1JJFQWOGWjzaxrLKTZ+eMZkuTnUun5FCSGcN321rZVG8nI9ZItCHSUTcvwUx6rJG8BDMfr2vgoa+3MSzVyjlj07nlo018X97W8xzlLU7eWlHLG3PHkxJt4Ok/jMYTCBEKK1j0auo7PTz6TTl58WZy4kx8t62NcTmxGNQq7viklJcvHkuW7Zf1nkmONjAiPZq1tb3zYmwmDVXtCpfsSMSFSLGx7DgTt5w4hKXb20GJBEI7Z4M0Kon0GENP9dUf6/JEZlBOGJbMxNw4NtZ3E2VQMyHHRnK0gSiDWJIRBEHYF0Qgsp9Ltkaa2bU7vYTCcP7ETGYUJfHWyjq6PH6mFsSTE2/G5Qvy13fXE1agotWJLxhiSn48b6yo7XXDfm9NPUWpUdx64lCufG01apXMKSNTOXlkKlajjnu/2EJYoVcQslO7y88z328nyqBhWEo0RWnRuP1Bmrq9uPxBphTEM7+0GVmWuPaoAopSoiht7CYhSodKln9xl+B4i47HZ5Vwz+db+GRDI6GwQpxZS4xJx+MLKxieFo1altnSZMftD1HZ5uKLjU38YUImLn+wJwiRJPjbjMEsrWgnzqLrSZrdXWGShbdW1qJTq1hb08kdpxQRZ9QSH/3LgidBEAThlxHddwdIIBSi1eEnEAojEdkh4vIFyY03E2fW7XFniaIoBMMKbn8QXyBMU7cXo1ZmdW03i7a2khxtYNqgeAoSzdR2eDj58cX9Xufiw7Ipa3Lw3bZIwJEbb+a2mUNIizHy6IJtvLO6vt/HmXVq3r18Asc89B0nDE/mT0fmU93h5r9fb2VDfe9KuBNybdx8/GBW13QxPDWa4enWX/VauXxB2p0+vMEwsUYty6s60GtUrKruxBcMUZIZQ1Wbi/vnbUWjknn/ion4giEeX7idxCgdhxcmUNZkZ3JePOc8vRSnr/eMyKQ8G7fPHIokSwSCCiadGqNWIsb0yyvACoIgCPtRrxmhf03dXl5YXMnLOxJEBydbuHxqLgu2tPLx+gb+fFQB54zLwKhV7WhaF8SoVRFn1mLSadCopEhH2x0f1re3Omns8nDaqDQKEs1sbrTjC4Z5dw/BBMCHaxq4Znp+TyBS0eqkrNmJ1ahB3svMhSyBRiUzoyiJT9Y3cvKIZKraPX2CEIAlFe2UtzgjCbK/bDKkF5NO3ROYdbl8lDbaeWRBec/Xn/mukiMKE7jr1GHc8O569BoV2XFG/nXyUFzeIC5/iLz4ZJy+INdMz+epRdtpcfjQa2TOKElj7pRcrHoVKpWESSfyQARBEH5PIhD5nbU5fFz12mpWVO/aAVPa6ODqN9by4FkjWF/XxT1fljE0NZotTQ4e/GornkAIWYLjipL5x/GDSd6R8NnY5aG0yU55i4NJ+XGkWQ1UtbnIjTfT5Qpg30udEKcv2Kco2IItzdhMGk4emcpbK+v6fdyxRcl8uLaea6YXcEJxCnmJUdz/1bY9Ps+bK2u5bGouZr2asiY7GpWMPximyx0gNcZAnEWH4SeKk+2ust3dKwjZ6ZuyFsblxHLO2HT0ahmdRk2iRg0WqO9088GaeorTo5mSH8ek3DiCShijRo3NrP1FnYEFQRCE35YIRP4fQqEw3R4/IOEPhfEEgkjIJEXr91j5s7bT3SsI2d1j35Qza3wGt39SymPflFOcbsWzo9x4WIFPNzTSbPfyv9klOLxBznl6aU/NDgCjVsUbl4wny2ai1eHjuGFJfNhPjRCAKQVxrKnpPQ6dWoUvGCbRouf4Ycl8uqGx19eTo/UcPyyJi19ayVFDkihrcuDxhwiEFCQJjihI4NRRqeg1MlajFl8wjDcQIilKj9MT5Nq31lLX6eH0kjTGZMVywQsruOHYQk4dlUb0z0gGDYXCvLa8/11DAO+squO/Z4/oM/uSGmPkkqm5dLj8qCSJGKMGlUo0sBMEQdgfiEDkJwRDYWo73LS7/Di8QZKtekxaFVpZBknBG1LodPpx+oJEG7QYtRJfbW5iVEYMqTHGPtfb2Za+P+UtTpJ3JEfWd3mYNqhvFdqV1Z00dHv59yebewUhAG5/iDnPLefTqw8jxWpgSCiKwUkWSpt6t7s3aFTMGpfJ5a+s6nX8uKIksm0mFEVh7pRsphTE88n6Bly+IIflxzMsLZob39tAIKRQ0+FmbW0nxxQlMbM4mUl5cYTCCnUdHlKsBuo63dz+aSmtDh86tcwFk7J49NxRnPzYD7y6rAaHN8iZY9K57ePNFCRamJQX99PvRVih1dF/jxiALrefUFhB108QqFHJv6rrryAIgrBviUBkL/yBEFuaHVzx6uqe5nCSFOnCeu2R+bS7/Hy4roHXltUgSxJWo4aceDP/mjmU77a1cHhhIkk/2nVhM+15GUCrktmZOjwkOYqqdle/51W2Oqlo6/9rne4A9V2RYCDNauTRWaN4b3Udry2rwe0PccSgeOZMyOKeL8pw7VaUbEZREnkJZqINGiQJ1LLMQ19vZfrgRHQame+2tfHmilom5NrQqCAnzoReo+KbLc2cNCKVv7y9juW7FVzLSzDz+LmjuPbNtdR1eXjy2+2YdGqumZ7PPV+U8cn6Bp6ZM4YXF1fxyIJtDE2J+sklEp1GxeEFCSwsa+3362OzY4k2aIgS1U8FQRAOGCIQ2Yu6Lg9znltO527FrxQFPlhTz6WHZTNvczOfbWjktpOGYtFpaLZ7SYrWs7nRzhGFkboeTp8f824JkGOzY3vVt9jdsUVJLNjSjCzB2WMzuOq11f2Oy2bW4fDuOf9jZ26ILEtE6VScMyaDyXlxxJt1aNWRYmaXTc3hnVV1qFUys8ZmEGPS8tg35Syr7OCIwnjOG5fJsNRoXl5ajU4t888ThqBRy8wvbQGgocvDjccNJhAM8e9PS3sFIRCZ3bnxvQ08ObuEN1bU8srSap79vpIXLxgLlBFWwLMjEKrt8NDtCfysXI2phXEkfquj2d57ZkSnlrlqWj561S/fJiwIgiAMHBGI7EVpk6NXELLTiPQY7N4Qn29s5L7Ti/nnh7uqjMKO2YBZoyL5E2EFnz+MzRJZFkiM0vPYrFFc8erqXvUs8hLMnDIylds/2cwLF4xlY313rxmLnZKj9UTpNWhVMt5A/y3ps+JMPf+vVkmUt7rYUG9n0dYWzp+YhV6jIivOxD+OH4wvEKLDHeCUx3/oKXs+b1Mzl03N4c9HF5AYpWNEegzvrKrr1evmy01NTMq1ccvMoXxV2tzvOCpanTR0eYi3aDm9JI13VtX1ChJ0mkieRmGSGb068v8tDi+SAhq1hNXYtxNzqtXAixeM5ZEF5Xy5qYlgWGFcdiw3HjeIKL2axF9QuVUQBEEYeCIQ2YuKFke/x6MMavyhMOeNz+Tfn5b2CkJgx2zAu+s5YXgy2XFmTFoZRYI4cySJdWpBPAuun8rCra00d3sZn2MjPcZAMKzw+iXjSYzSk59gZsGWlp6+LRAJQh6fNYoXFlcyd0oO98/b2mdsJ49IIc686wYeDkNNhxtvIESb048CXPryqp4g59qjCvhmS0tPEAJQkhVDtzfIxS+s5LaZQwkpSr8N936oaKfLHWBvlWg63QGeWVTJw+eM5MuNjQRCYcZkxeD2h6jYsbV3zsRswgpUtTlx+kJUd7gwaFTkxpvRyBIpu+XaaNUqcuJN/G3GIC4/PBdJApNWjcWgxmbqG7gIgiAI+zcRiOzFkJTofo+3OrxYDRqybCbKmvsGK5IEMUYtRw5OpNPlx6RX0+H0s6Kyk+J0KylWA5k2E3MmmPq5ekSy1cD/ZpfQ7PBR3eYiIUpPqtWA2x/g3dX1XDUtj1tOHMKz31dS1+kh2qBh1rgMZo3L6LUDRZEgy2bixcXVnDE6jUcWlPPgWSO48b0NdLj8FCSaefCr3gGNhIRakujyBNjUaGflXprt+YKhXk3kfsxm1uLyB1lb28U1RxWwuLyNOROziNJruOPTzdx16jASzBo8gRDr67rY0uRgTFYsyys7WFzRzuEFCYSUSBCm3rHTRatWkRpjJFU0chYEQTjgiUBkL/LiTSRG9c1HqO3wYNL1vz1XJUu8eMEYdBoVz3xXSYvDy/A0K1MK4vEGQpz82A+8Pnc8uQnmPo/1BkJ0uPyEFQWzTo3NrMNm1jEkeVdVui63zKkjU3lkQTlFqVFcNjWXGKMWbzDEkOSoPjt14sx6nN4QvmCYVKuBaIOGB7/eyi0nDsGiVxNv1iFLke3BO62u6cSsV5MdZ0ItS/iCfZeIdlpZ1cmZo9P7bcY3OjOGrU0OwgoEQmEKEsx0uAMUJprx+ENcNjWXN5bXMDl3BPXdHqra3Th9IS56cSU6tczxw5Mpa7bT4fKhzY4lUZRbFwRBOOiIYgp7kRFr4uWLxjE0ZVcgoFPLzBqfgSRBWkzfG+Mzfygh2qDB6Qtyekkql07NJRwOc8Urq8iJN3PDMQWUNtrZWN9NWZODFkdkC259p5vbPtrEEf9ZyOR7vuGiF1ayvq4LX6B3EGA1avnb8YP5zxnD8fjD3PdlGe+tqWNQkoXsuP5nWJKidPxvdgnPfV/JjccN4orDc/mhvB2LXs1H6xo44kfbhLs9ATrdAe46dRhrazr7fL0XSeHiydmcOzYDjSqS/yFJMG1QApcfnstj30SKj03MtfHH19bw57fXIUkSt3y0mRvf3cBlh+ehoLBsezs5cSZeWVrNyHQrT/9hNJNy4xiXHcvQlGgcvhCVrU7sHj/dbh++QIBQqP8cGUEQBOHAIXrN/Ax1HW7s3gCeQIgovYaaDjcfr6vn0im5PPJNOZ9taAIi+RayBBa9GgmJN1fU0uLwMjI9hj9Oy0W7ozR7k93LHZ+Ukh5rZHRmDIcVxHHhCyup/NGWXLUs8eGVkxi6xyUiH6GwgkmnwvIztqzWdrjwBsK0O328vLSa00enc9Vra3h81iiuf3tdrxodBo3MgusPp9XpIxBSuPr1NdR3eXpdLy3GwG0zhxIMKeQnmvGHwlS2ulCrZJZub+e1ZTV4AiGm5McxMTeOu7/YAsBtM4cyPicWg1ZNokVHs8PLoq1tLNraysrqTh48s5gYkxaXN4hOq+LuzyK7cqL0ah6fNQpvMMxHa+sxatWcVpJGokVHSrQBtVrE1YIgCPsD0WvmN5YWa8TpDeDyRfqWpMYYuO6oQkwaib8eOwirQcPq6i4SLTrKW53UtLt5e9WuEulGrYxKkgiGI1trLXoN/zhhMCadmnAoxOYGR58gBCIFvO75fAuPnjuq3zb08ZZflpyZHhuZMcmOM2Ez6wiGFQKhMH97fwN3nlLE1mYnq6o7iTFqOGdsBmurOylKt6IoYZ48bxQfrG3g0/WNSBIcMzSJwwvj+WZLK25/EIcvQLxZS3qskf9+vY1lle0kROk4bVQaaTEG/vru+p5xNHZ7MGnVJFsNqGQJCUiPMdDlCXDW6HQ0apkutx+DRs05Ty3FF4zMfPz37JHc+2UZ6+u6e671xopazh6TzmVTc8m0GZEksXVXEAThQCI+Qv5MZr2GxGgD2XEm9GoVrQ4fdl+Ixk4PZ43J4InZkd0sk3LjegUhF07KYu6UXP71SSk17W4e/aaCWU8vZV1dNy5fELVazdd72P4KsGR7O51uPxvru3csTey5fsjPpVbJaNUyHn+QP0zIpK7Tw9yXVvHZhkZiTVqcvhAfrq0n1Wakqt2FXiOjVsk4vQEuOzyXS6fmUt3u5vkfKpmYZ+ODtfUkRxuw7Ohae+nUHO44uYg/TMhkfmkz1721rlcy67DUaC58cSWfrm/A4Q1g0alJjzUyLiuWSflxPUXfXl5a3ROETMixsbG+u1cQstMbK2rZ3hZp/CcIgiAcWMSMyC8kSRJZcaaeWh3Z8RbqOt34g2FCYSjbrZy6Wafm1FFp/OWddQxPs7KgrIUJuTYumJTFvE1N5MSZqOnw7LXPitWgpa7Tw6xnlgFweGE8d54yjJR+6mWEwgrNdg+drgAGrQqVLKGSIRBUqOv04PAGSI81kmI1EG/WsbqmkxOKkwkpCq8urWFTg53NjXYOL0jg8MIEZj+7nFEZMRw9NIEp+fFMyo9Dp1Zh9wSYPSGTdbVdXP36Gsw6NfEWHR5fkLASmZEIhBRu/6S0zxgzbUZkSWJrc6TR36sXj2NEWhRmnYqTR6YiSZFCZ+1OPyuqdpXDnzEsiSe/3b7H1+mjtY3kH9M3AVgQBEHYv4kZkd9AWoyRaIOGVKsB7W55CjOGJdHh8lPa6ODIQQkMSrRQ3uLk9CeXgCTxv0Xb2dzQzfTBe04GPXN0Ghvqd80CLCxr5crXVlPV7mJTQzf1nR4CoRCNXW62NNm587MtaDUy766qwx8MU9nm5uYPN/KPDzfywdoGWhw+Xl9ejdMfZGxWLBISRw1K5OOrJvPErFE8NXs0OfEmrnxtDd2eAN+UtfDxukbeW11PmtXIze9v5NaPNnHRiyv47/xtSBI8eV4JZU12ZEnCotPsmEGBO08ZRuqOgEmW4KghCTw+axQ3f7ix5/t56KutVHV4CIfh43UNmHQqZFmi2xMgdrdy+CadGrc/uMfXyRMIEthDgTdBEARh/yVmRH4jNrOOP0zMBCRUskQorJBqNdCwY7lAUWBQchR3fh5J2MyymXhrZS3bW50cXhjP9UcX9ClQVpIZw4nFKZz51JJex1fXdNHm9PHgV1tZX9fNp1dNJhhW+NMba3hiVgn3fr6FvxxbyJebmvnPvLKex1W3u/m6tJn7Ti+mssWJUafmue8rmT44EYtejVolc/krq3qVn0+K0nPZ1Fyuen0N35e38fLFY1lT08XmBjv5iRYm5sYiSxLpMQaSrQYkSSIxWk+CRYfTGyQnbjiSJGHWq6ntcHP2/5bg8O3aCbSlyUFoR/derVqmxe7Foo/sOjptVBqbGjYDsLyyg8MHJfD+6vp+X//JeXGoVCI/RBAE4UAjApHf0KiMGFZUdfDXYwq58/MtbG12MjIjUnUrMUrHy0ure85tc/pIiTawudHO6ppIouvz549hRVUHLn+IMZkxWI0aPl7XgN3TdyagtsPNTccNxh8MEQwrrKvt4tyxmVS0Orl0ai51nV4enr+tz+OMWjVqlYQiQUOXl7PGZGDWq2iye8lNMPHWpRP4pqyFNqefkkwrNpOOpxZVcN64DGYMS+YPzy3HoFGRHmtkeVVHz+zGvGumUNPuJj3WiLwjENvcYGdIajTflrVy9xdbevI9dpceayTKoMbuDXJYvo11tXZGZ1k5emgSa2o6OWZoEl9uauL9NfW8c/lEvtrUjNPX+/XITzAzKCkKs040uxMEQTjQiKWZ35DVqGVqQTwzhiXz0R8nYTNpSYzSkZdgpqzJgd276wb6/pp6zh2XAcD987ayod6OJEFuvImSDCuDkix8tK6BxxZW9Ptc6TFGHv+mnO/K21FJsKG+m9FZMWxrdrK+rotOlx9/P3U27j+zmGBYYUO9nX9/VsqsZ5Zywzvr8frDeHxhQopCjFFLm8PHFxubduSRpHDKqDRmPbuMZruPqnY3321rY8uOfBiDRkWzw8t/5pVR2+nG5Q3gDYb5rqKdmnY3Q1OjevXV2d2scRnoNSqeWrSdzY0OhqREcdvHm5FQGJ5m5eghiTw1u4TzJ2ZR1tjNe1dM5IThyZi0KuLMWi6YlMV9ZxSTYtVjM4sS74IgCAcaMSPyG9OqVaTFGkmLNTI0NRqHx89j547iv19vZcawJL7aHNkhU9nmwuENMGdiFi8vqeLlpdW8uqyawkQLfzmmkBaHF5ev/5yI3Hgz5a1OjhqaRLRejSJJFCZFoZIlMm0GvtjYzMwRKX0eNyHHhj8QZk11J68sq0GSIoXGThieQlK0HgVosXuZkm9jYq4Ntz9S6bWyzYU3EMLh7X88Z45JY/7mZj5e38iUgngOL4jjzZW1HFGYwAuLq/jbjME8dPYIbnp3A44d35NKljh/YhbBsILTF2RxRTsrqjr479kjMOrUnPXUMi6dksO0QfGoZJlBSRZ8wTCypHDDMYVcMz2fsAImjYpooxqz/qc79wqCIAj7HxGI7EMqWcJq0hFt1HLjjEEEQwo5cSa276gZcs8XZZw9Jp3nzh+DNxBCp1aRbNVz/5dbOHdcJtcdXUinO8Diil0N5/ITzNx8whCuf3sd/zljOKuqOvnb+xt5be44ylvsDEuzsra2m5x4M3pN7w69Rw9NJNqo4bXlNeg1Mg+cOYJ1tV1E6TX8b9H2niBJluCFC8byzqpaThmZxpebmsi0Gbn39OHc8M76XrMbxenRnD8hC6cvSG2nh1eWVjMmK5ZtzU6GpURz5OAEVlR3ICHx0kVj6XQH6Hb7iTZq+WxDI2lWA75gmCiDmtJGB1uaHJw/MYv0mBYeWVDOXZ9vISVaz1XT8plaGEeKtXcJe0EQBOHAJiqr/o5aHS4cXoVXl9Xw9qpafIEwRxTGc/WRBUQbVKiQOPaR7+lyBzhjdBpnlqSxuKKdYWlWOl1+ogwa6rs8PLagnFanjxcuGMMVr65i9vgszhuXgScQRKtRY3cHUKtgS5OT699e19Md997ThiPLEn9+ex23zRzKR+sasOjV5CWYeea7yl5jTYrSc9epw6hscwIS2fEmMmOMhBSFH8rbaHf5mZhrI86so7bDQ22ni0m5cdz1eSk3nziUDqcPrVqFNxDEpNMgSxJvLK/h/bX1+Hf0vblpxiCq2lxMzo/n262t/PvTUrQqmb8fP5jceBMKIEsSVqMGm0lLkug1IwiCcED4JfdvEYj8zlrtHoLhSIO7nba1Osm2mYgxaOj2Balqc1HZ5mJqQTynPr4Yhy+IQaPCGwz1BBUqWeLdyyZQ2uRgSUU7X2xs4qtrD2Nbi5P8Hc3lovRqKtvdfLyugfpOD2eNTiPGrOP6t9Zx+0lDufqNtTxwZjG3frSpV/7K7p75w2i8gRClTQ6mFsSxsKyVyjYXalli0bZWvIEw/zxhCKMyrbj9IUxaNW5/iHanD6cvSGaciUAgRFacgVBYoqHbS6fbT7xFzz1fbGFVVSef/WkyZU0O3l5Vx8KyVgDizFrG59iYnGdjcl48kkSfhn6CIAjC/kmUeN+PxUcZcPsCBMJh/EGFUDhMjs1ElFFDnEWPTVGw6NVk2Iy8taKGP03P545PS/H8qPndueMysHsCdLj8fLSuAYAnF22nvtNDcrSeyw7PBQVeW1qDWa9meFo0C8pauGZ6IaMzY1hV0wWAXqPaYxACsHR7O0cNSWRVdSedLj+P95M8+48PN/LcnDHc+VkpZ45OY+KOrbTBsII/GMbtD9FkD1DR6uTeL8rwBELccuIQluxYcpIliVSrgaMGJ3Lm6HSWbm9Ho5I5LD+OLJsJSYIYk8gBEQRBOBiJQGQAGHUaMvaw1VSSJBIsehIsei6dkofdGyA1xsAD87ayrcVJWoyBq4/Mx6JTE2vW8tqymp7HbmqwM31wIg98tZV1dd08cu4I/nrcIP7xwYaeYEWrlrlqWh7zduSDdLj8pMUYqOvsvzx6SVYM17+1jn+dNJRHFpT3e46iwJebmrjj5CKq2l08uqCcU0am8vH6BhZtbes5LyfOxL2nD+d/31ZQ0+HuOX7162t4cnYJeQlmvIEQRw5KIM6iRUYCFGRJwqgVP6qCIAgHI/HXfT8WZ9ERZ9GRE29mTFYs/mAYtSyREKWnw+Wjyx2gsXtXALG+rpurpuUTb9GxpclBY5eX7DgTd54yjDannzanF5tZR4xRw8wRKTy+sII3VtRw0eRsbvt4c5/nT4sxkGY1EGVQE2fW0dC9514utZ1u5m1uYniaFatRw6cbGnsFIQDb21z8/YMNXHdUAcHdes9sbLDzl7fXcf+ZI3D5gsgSO7oKq5EkKG+JbH0uSLSIpnaCIAgHGVFH5AARZ9aRYjWQEKUHINakw2rUUpQa3eu8f32yifvPKOboIYnc/fkWGru9ePwBsmwGhqVGkxYT2aUiKfCP4wezsd5Om9PPDccU9up5Mz4nljtPGcbLS6p5fFYJ321rpSil93PtriglGrNOzYItLRxemMAn6xv7Pa+2w4NWpUKvUfXqHrxkewcT717Avz7ZTJcngCyB2x/inx9uwqTX0u3273HWRhAEQThwiUDkABZr0vK3GYN7Havt8HD5K6vIjjPx6LmjSLMasRp1NHR58fhDaFUySys7OOHR7ylvcfLRlZNw+YKEFYUXLxjDO5dN4Lk5oxmVEcNVr6+h2xtAJUdySS6dmovcz4SEUatiUl4cFS2uyNZehT0WMANodfp47odK/nP6cDJtuxJQJSmSpBpj1OIPKSjA1dPyufiFFRh0GlbXdOzxmoIgCMKBSSzNHOCGpkTxv9kl3PLhJprsXgCKUqM4a0w62XEmmu1evP4wqTEG9BqZKL2WwwvieSXOxBsralFLEsPSo5lf2sIbK2r7zDqcWJzC8z9U8d22NlodPh4+eyR3flZKQ3fkuQoTLdxxShH/+WILle1ubji2EFmW0Knlfku6AyRH69lUb+fvH2zkhmMKSYs10u0OYDNrMWpVnPzYD9i9QUakW7nhmEL+N3s0L/5QyQWTsvAGQug1qn37ogqCIAi/GxGIHOAseg1HD0mkOC0auzeIRiUTY9SgliW+3NTEp+sbe27galnCawwTbdDw3PljWFnVyYfr6rk4O4dHFpT3CUIm5tooSLBwzRtrCYYVtrU4GZIcxasXj6Pd5cflC5Iea2RVdScFSVEsq+rEFwizsaOb88Zn8Oz3VX3GW5QSKfc+KS+OIwcnIEsSKgnKW51sbVYizet25IGsre1iW4sTbzCEWiWh16jQqMQkniAIwsFEBCIHAUmSSIo2kLRbCsfiijY+WNPAZYfnsKXBzthcG+0OP2trWxiUZCHaoGViro3xOTF0ugI8NbuEHyra+XhdAzq1zOzxmWTHmTj1icW9uvFWtDoJhBXUssQ1b64lI9bIv08pYmSGlY0N3fzrk83cdeowBiVZCAQVXl9RQ2BHYuqkPBtXHJ5HUpSWS6dmgwL13V7UsswHa+qxewPUdXp4es5obnxvA+UtTr7c1MTgJAtHDUkipETqpwiCIAgHD1HQ7CDU4fQx5/nl3HFyEWtruphamMD1b6+jrtPNfacX8+qyGuaXNnPltFxmDEsmHIZgWOHpRRUMTbVy5KB4HvxqG59s6Jtwev7ELLQqmYJEE5lxZm58N1Ly/W8zBpMbb6bF4WPZ9nbyEkyMyIjB5QvS2O1FJUs4vEEKEs0EgmHum7eVldUd2D1Bzp+YRU2HmwVbWgCINmh45JyRXPLySkZmxJCfYGZGUTLr67u4ZEru7/1yCoIgCL+QKGh2iPMGI8svvmCYKYXxPDR/G6uqO3ns3FHc9N4G6rsiSzD5CVGsq+1ClmSGJFu49qhCmu1e1td3cf0xBbgCQRaWtaIooFXJnD02nelDEpj97HJijVr+e/YIHp81Ck8gRCCkIEnw/bZWFle08/nGJsqaHZi0Ko4emkSMUYPLF6S+00OsScvG+m7snkghtW0tDrJspp7xd3sCvLWylpnFqQxOtvD68hrOKEmjtNGOyxfAtIcaLIIgCMKBRwQiB6FIF14jEuAPhvl0fQM5cSZaHN6eIGSnv7yzgb/NGARSFOvru0i1GhiTGYssw79mDqXbE8TjD6HTyHS7A7h9IZ4/fwypVgNhFMoaHXy0roH5W1pQSRIvXDCGx7+t6ClF7/KHeH9NPQA3HTeID9bW888Th9Di8PWMoTDRQmWbu9e4vi5t5uGzR+LyhxieZuXT9Y0cOzQJuycoAhFBEISDiAhEDkIJFh1jsmxEG7R4g5HZisIkC6urO3ud5/AFSIzScednW1DJWzh+WDJxFh0VLS5UEvx1xiCMWhV3fVbKDzvKsRs0Ki46LJsog4ZQWOHqN9b2XC+oKLy5opb7zyjmr++u78kNAZhZnIJZp+a4oiTeXFHbc1yjkpg2KJE5zy/vNTaNLJObYOb+L7Ywe2IWZp0arUrGEwzh9QfRi0qrgiAIBwXx1/wgJEkSE3Nt2D0B9GoV0QYNTl+Q7DhTr/PeWF7LZVNzue3jzYTC8NG6XTkhQ1Oi8PhDPLO0kltOHIICOH1B9BoVXe4Ad3++hZnFyZxYnMzHuz3u4/WNqGT4/E9TKG200+7ykRtvpqbdTVacCXdDiPdWR2ZI4s067j5tGE8tquhTd+TUUanoVBKjsmK594syJubFodfITCuMx+kPiUBEEAThICH+mh+kEqL0SEgEwyH+eEQu931ZxgWTsnlpSXXPOWtruxifE8s/TxjCk99W0OLwoVFJHDM0iVNHpbG5oZsoQ6SDr0WvIjXaSJsr0lX35uMHY9KpmZIfT2asiVeWVdPlDpARa2R0lo13V9WSYtVz1JBEWuxe8hPNxO8oL//63PEYNDJWo5a3V9WyaFvvUvBpMQbOn5TFPz7YxMKtrZw3LoOtzQ6+2tzMuKxY2H/zqwVBEIRfSOyaOciFwgr1nW4+39hETYeb9Fgj93yxpde9/PyJmZw7LpNAMIxGJePyBej2BvAFwqRaDcgqiW53gFiTFplIBVSPP4SskpEl0KlUrK3rwqxTY9KpdlwjxJKKNibk2rjghZVIEnx+9WGsqOpg3uZmDi+I57iiJAJhhco2F68tq8HhCzKjKIlJeXHc8tFGvtvWjlGr4snzSrjghRWEwgrXTM9n1rgM4i36AXtNBUEQhL0Tu2aEHipZIsNm4ozRaXS7AwTDCkcUJrC8sh27N8jQlCi2NDk4/uHviDfruO7oAuyeAIlRero9Aa56fS1XT8vlxBGpKGEFXzDMxoZuEsx6kGD2s8t5+JwR2Ew6/jOvjA313UgSTM2P58LJ2fzzw41AZBKjyx1gRLqVZKuBGKMGg1aFJqwwv7SRv88YhEqW+aasmWMeWkQgpFCSGcMVh+fyn3llPUs3Hn8IjUrUEhEEQThYiBmRQ4zDG8DrDxNSQkhI+ILhHdtvw5h1alodPj5Y20BYUThqSBKVbU7CChyWF4ckKejVarQaCYcnCJLEA/O28nVpM/85YzhFqVa2tTiQkFhe2c47q+qweyNbdDUqiY+vnMz35S10uoOcNz6T5GgDoVCYxxZW8Mqyap7/w2jcwRAefwi3P8zmxm5eX15L6247bN66dDxjs20D9fIJgiAIP4OYERH2yKLXsHNVo9Plxx8Ms662C6tRy9raLo4eksiZo9P5YmMjy7a3Mzk/jrImBzMf+567Tx3G4QXx1Hd5ufL11dR3ern9pKHMHJHMK0trsJl1vPBDFcsq+zanO2VkKgatiqkFiSRF67HoI1twVSqZ00vSeHd1HZIsoSgQDClc/foa/KHevWom59lIjzX2ubYgCIJw4BIzIgLNXR48wRAOX4jPNzZy6sgUDBo1mxrsfLSugTizjjNK0kix6okx6Wjs9lDd7qaixcna2i7yEszMGJaELEEoDP/6ZDPzt7SgKJGZkFNGpnHltFwyYk17HENDl4fFFW2MzYyh2xvEEwjxv0XbWba9A6tRw/kTs5hZnEJClMgNEQRB2N/9kvu3CESEXryBEA5vgHBYQaeRUcsyBo0K1Y+azXkDIbo8AVQS6NQqogyRGQ6XL4jTG6DbE8TlD2LWqYk1abGZdT/53IqiYPcE8AfDeIMhQmEIKwp6jUxytAFJErkhgiAIBwKxNCP8anqNCr1G9bPOS+rnPJNOjUmnJjG6nwf9BEmSiDZqf/kDBUEQhAOW6KkuCIIgCMKAEYGIIAiCIAgDRgQigiAIgiAMGBGICIIgCIIwYEQgIgiCIAjCgBGBiCAIgiAIA0YEIoIgCIIgDBgRiAiCIAiCMGBEICIIgiAIwoARgYggCIIgCANmvy7xvrMNjt1uH+CRCIIgCILwc+28b/+cdnb7dSDicDgASE9PH+CRCIIgCILwSzkcDqKj9958bL/uvhsOh2loaMBisfyqzqt2u5309HRqa2tF9979iHhf9k/ifdk/ifdl/yPek5+mKAoOh4OUlBRkee9ZIPv1jIgsy6Slpf2/rxMVFSV+WPZD4n3ZP4n3Zf8k3pf9j3hP9u6nZkJ2EsmqgiAIgiAMGBGICIIgCIIwYA7qQESn03HLLbeg0+kGeijCbsT7sn8S78v+Sbwv+x/xnvy29utkVUEQBEEQDm4H9YyIIAiCIAj7NxGICIIgCIIwYEQgIgiCIAjCgBGBiCAIgiAIA+aQCUSysrKQJKnXf3ffffdAD+uQ89hjj5GVlYVer2fcuHEsX758oId0SLv11lv7/F4MGjRooId1yFm0aBEnnngiKSkpSJLEBx980OvriqLwz3/+k+TkZAwGA9OnT2fbtm0DM9hDyE+9L+eff36f359jjz12YAZ7ADtkAhGAf/3rXzQ2Nvb8d9VVVw30kA4pb775Jtdddx233HILq1evpri4mGOOOYaWlpaBHtohbejQob1+L77//vuBHtIhx+VyUVxczGOPPdbv1++9914efvhhnnzySZYtW4bJZOKYY47B6/X+ziM9tPzU+wJw7LHH9vr9ef3113/HER4c9usS7781i8VCUlLSQA/jkPXAAw8wd+5cLrjgAgCefPJJPv30U5577jluvPHGAR7doUutVovfiwF23HHHcdxxx/X7NUVReOihh/jHP/7BSSedBMBLL71EYmIiH3zwAWefffbvOdRDyt7el510Op34/fl/OqRmRO6++25sNhsjR47kvvvuIxgMDvSQDhl+v59Vq1Yxffr0nmOyLDN9+nSWLFkygCMTtm3bRkpKCjk5OcyaNYuampqBHpKwm8rKSpqamnr97kRHRzNu3Djxu7MfWLhwIQkJCRQWFnL55ZfT3t4+0EM64BwyMyJXX301o0aNIjY2lsWLF3PTTTfR2NjIAw88MNBDOyS0tbURCoVITEzsdTwxMZEtW7YM0KiEcePG8cILL1BYWEhjYyO33XYbhx12GBs3bsRisQz08ASgqakJoN/fnZ1fEwbGsccey6mnnkp2djYVFRX87W9/47jjjmPJkiWoVKqBHt4B44AORG688UbuueeevZ5TWlrKoEGDuO6663qODR8+HK1Wy6WXXspdd90lyvQKh6zdp52HDx/OuHHjyMzM5K233uKiiy4awJEJwv5v92WxYcOGMXz4cHJzc1m4cCFHHnnkAI7swHJAByLXX389559//l7PycnJ6ff4uHHjCAaDVFVVUVhYuA9GJ+wuLi4OlUpFc3Nzr+PNzc1ifXU/YrVaKSgooLy8fKCHIuyw8/ejubmZ5OTknuPNzc2MGDFigEYl9CcnJ4e4uDjKy8tFIPILHNCBSHx8PPHx8b/qsWvXrkWWZRISEn7jUQn90Wq1lJSUMH/+fE4++WQAwuEw8+fP58orrxzYwQk9nE4nFRUVzJ49e6CHIuyQnZ1NUlIS8+fP7wk87HY7y5Yt4/LLLx/YwQm91NXV0d7e3itgFH7aAR2I/FxLlixh2bJlHHHEEVgsFpYsWcK1117LeeedR0xMzEAP75Bx3XXXMWfOHEaPHs3YsWN56KGHcLlcPbtohN/fn//8Z0488UQyMzNpaGjglltuQaVScc455wz00A4pTqez1yxUZWUla9euJTY2loyMDK655hruuOMO8vPzyc7O5uabbyYlJaUnqBf2jb29L7Gxsdx2222cdtppJCUlUVFRwQ033EBeXh7HHHPMAI76AKQcAlatWqWMGzdOiY6OVvR6vTJ48GDlzjvvVLxe70AP7ZDzyCOPKBkZGYpWq1XGjh2rLF26dKCHdEg766yzlOTkZEWr1SqpqanKWWedpZSXlw/0sA4533zzjQL0+W/OnDmKoihKOBxWbr75ZiUxMVHR6XTKkUceqZSVlQ3soA8Be3tf3G63cvTRRyvx8fGKRqNRMjMzlblz5ypNTU0DPewDjqQoijJQQZAgCIIgCIe2Q6qOiCAIgiAI+xcRiAiCIAiCMGBEICIIgiAIwoARgYggCIIgCANGBCKCIAiCIAwYEYgIgiAIgjBgRCAiCIIgCMKAEYGIIAiCIAgDRgQigiAIgiAMGBGICIIgCIIwYEQgIgiCIAjCgBGBiCAIgiAIA+b/AD0SLfSbSZkkAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "manifold = umap.UMAP(n_components=2, n_neighbors=10, metric=\"precomputed\").fit_transform(distances)\n", + "sns.scatterplot(x=manifold[:, 0],\n", + " y=manifold[:, 1],\n", + " hue=filtered_dataset[\"Cover_Type\"][:manifold.shape[0]])\n", + "plt.legend()" + ] + } + ], + "metadata": { + "colab": { + "private_outputs": true + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/documentation/public/mkdocs.yml b/documentation/public/mkdocs.yml index 18128006..9c1b2144 100644 --- a/documentation/public/mkdocs.yml +++ b/documentation/public/mkdocs.yml @@ -56,6 +56,7 @@ nav: - Regression: tutorial/regression.ipynb - Ranking: tutorial/ranking.ipynb - Uplifting: tutorial/uplifting.ipynb + - Anomaly detection: tutorial/anomaly_detection.ipynb - Input feature: - numerical: tutorial/numerical_feature.ipynb - categorical: tutorial/categorical_feature.ipynb From 74026c8759c2e49bdb0e81d3de2cc8925cec08ab Mon Sep 17 00:00:00 2001 From: Mathieu Guillame-Bert Date: Tue, 18 Jun 2024 00:03:04 -0700 Subject: [PATCH 30/30] Release YDF 0.5.0 PiperOrigin-RevId: 644269898 --- yggdrasil_decision_forests/port/python/CHANGELOG.md | 8 +++++++- yggdrasil_decision_forests/port/python/config/setup.py | 2 +- .../port/python/tools/build_windows_release.bat | 2 +- .../port/python/ydf/dataset/dataset.py | 6 +++--- .../port/python/ydf/dataset/dataset_test.py | 1 - .../port/python/ydf/dataset/dataspec.py | 1 - .../port/python/ydf/utils/test_utils.py | 2 +- yggdrasil_decision_forests/port/python/ydf/version.py | 2 +- 8 files changed, 14 insertions(+), 10 deletions(-) diff --git a/yggdrasil_decision_forests/port/python/CHANGELOG.md b/yggdrasil_decision_forests/port/python/CHANGELOG.md index 77889aeb..195f4008 100644 --- a/yggdrasil_decision_forests/port/python/CHANGELOG.md +++ b/yggdrasil_decision_forests/port/python/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## HEAD +## 0.5.0 - 2024-06-17 ### Feature @@ -10,6 +10,12 @@ more convenient than`ydf.verbose`. - Add SKLearn to YDF model converter: `ydf.from_sklearn`. - Improve error messages when calling the model with non supported data. +- Add support for numpy 2.0. + +### Tutorials + +- Add anomaly detection tutorial. +- Add YDF and JAX model composition tutorial. ### Fix diff --git a/yggdrasil_decision_forests/port/python/config/setup.py b/yggdrasil_decision_forests/port/python/config/setup.py index 8727d908..3a78340f 100644 --- a/yggdrasil_decision_forests/port/python/config/setup.py +++ b/yggdrasil_decision_forests/port/python/config/setup.py @@ -21,7 +21,7 @@ from setuptools.command.install import install from setuptools.dist import Distribution -_VERSION = "0.4.3" +_VERSION = "0.5.0" with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() diff --git a/yggdrasil_decision_forests/port/python/tools/build_windows_release.bat b/yggdrasil_decision_forests/port/python/tools/build_windows_release.bat index 816f382c..6256c9bb 100644 --- a/yggdrasil_decision_forests/port/python/tools/build_windows_release.bat +++ b/yggdrasil_decision_forests/port/python/tools/build_windows_release.bat @@ -34,7 +34,7 @@ cls setlocal -set YDF_VERSION=0.4.3 +set YDF_VERSION=0.5.0 set BAZEL=bazel.exe set BAZEL_SH=C:\msys64\usr\bin\bash.exe set BAZEL_FLAGS=--config=windows_cpp20 --config=windows_avx2 diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py index 138ab6ff..e7ad9aa4 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset.py @@ -136,7 +136,7 @@ def _add_column( column_data.dtype.type in [ np.object_, - np.string_, + np.bytes_, np.str_, ] or column_data.dtype.type in dataspec.NP_SUPPORTED_INT_DTYPE @@ -215,7 +215,7 @@ def _add_column( if column_data.dtype.type in [ np.object_, - np.string_, + np.bytes_, np.bool_, ] or np.issubdtype(column_data.dtype, np.integer): column_data = column_data.astype(np.bytes_) @@ -648,7 +648,7 @@ def infer_semantic(name: str, data: Any) -> dataspec.Semantic: ): return dataspec.Semantic.NUMERICAL - if data.dtype.type in [np.string_, np.bytes_, np.str_]: + if data.dtype.type in [np.bytes_, np.str_]: return dataspec.Semantic.CATEGORICAL if data.dtype.type in [np.object_]: diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py index b7f8e131..6928432a 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/dataset_test.py @@ -48,7 +48,6 @@ class GenericDatasetTest(parameterized.TestCase): (np.array([1], np.float64), Semantic.NUMERICAL), (np.array([1], np.bool_), Semantic.BOOLEAN), (np.array(["a"], np.bytes_), Semantic.CATEGORICAL), - (np.array(["a"], np.string_), Semantic.CATEGORICAL), (np.array(["a", np.nan], np.object_), Semantic.CATEGORICAL), ) def test_infer_semantic(self, value, expected_semantic): diff --git a/yggdrasil_decision_forests/port/python/ydf/dataset/dataspec.py b/yggdrasil_decision_forests/port/python/ydf/dataset/dataspec.py index 9a0d8b29..d33cfa80 100644 --- a/yggdrasil_decision_forests/port/python/ydf/dataset/dataspec.py +++ b/yggdrasil_decision_forests/port/python/ydf/dataset/dataspec.py @@ -50,7 +50,6 @@ np.float32: ds_pb.DType.DTYPE_FLOAT32, np.float64: ds_pb.DType.DTYPE_FLOAT64, np.bool_: ds_pb.DType.DTYPE_BOOL, - np.string_: ds_pb.DType.DTYPE_BYTES, np.str_: ds_pb.DType.DTYPE_BYTES, np.bytes_: ds_pb.DType.DTYPE_BYTES, np.object_: ds_pb.DType.DTYPE_BYTES, diff --git a/yggdrasil_decision_forests/port/python/ydf/utils/test_utils.py b/yggdrasil_decision_forests/port/python/ydf/utils/test_utils.py index 802ba242..e61f79ef 100644 --- a/yggdrasil_decision_forests/port/python/ydf/utils/test_utils.py +++ b/yggdrasil_decision_forests/port/python/ydf/utils/test_utils.py @@ -175,7 +175,7 @@ def test_almost_equal(a, b) -> Optional[str]: if a.dtype != b.dtype: return f"numpy array type mismatch: {a} != {b}" - if a.dtype.type in [np.string_, np.bytes_, np.str_]: + if a.dtype.type in [np.bytes_, np.str_]: if not np.equal(a, b).all(): return f"numpy array mismatch: {a} != {b}" else: diff --git a/yggdrasil_decision_forests/port/python/ydf/version.py b/yggdrasil_decision_forests/port/python/ydf/version.py index d066b205..e782e81c 100644 --- a/yggdrasil_decision_forests/port/python/ydf/version.py +++ b/yggdrasil_decision_forests/port/python/ydf/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -version = "0.4.3" +version = "0.5.0"