diff --git a/src/qiboml/ansatze.py b/src/qiboml/models/ansatze.py similarity index 100% rename from src/qiboml/ansatze.py rename to src/qiboml/models/ansatze.py diff --git a/src/qiboml/differentiation/psr.py b/src/qiboml/operations/differentiation.py similarity index 79% rename from src/qiboml/differentiation/psr.py rename to src/qiboml/operations/differentiation.py index 68c2c49..fcc001a 100644 --- a/src/qiboml/differentiation/psr.py +++ b/src/qiboml/operations/differentiation.py @@ -1,15 +1,12 @@ -from typing import List, Optional, Union - import numpy as np -import qibo from qibo.backends import construct_backend from qibo.config import raise_error from qibo.hamiltonians.abstract import AbstractHamiltonian def parameter_shift( - circuit, hamiltonian, + circuit, parameter_index, initial_state=None, scale_factor=1, @@ -161,52 +158,3 @@ def circuit(nqubits = 1): result = float(generator_eigenval * (forward - backward) * scale_factor) return result - - -def expectation_on_backend( - observable: qibo.hamiltonians.Hamiltonian, - circuit: qibo.Circuit, - initial_state: Optional[Union[List, qibo.Circuit]] = None, - nshots: int = 1000, - backend: str = "qibojit", -): - - params = circuit.get_parameters() - nparams = len(params) - - # read the frontend user choice - frontend = observable.backend - # construct differentiation backend - exec_backend = construct_backend(backend) - - if "tensorflow" in frontend.name: - import tensorflow as tf # pylint: disable=import-error - - @tf.custom_gradient - def _expectation_with_tf(params): - params = tf.Variable(params) - - def grad(upstream): - gradients = [] - for p in range(nparams): - gradients.append( - upstream - * parameter_shift( - circuit=circuit, - hamiltonian=observable, - parameter_index=p, - nshots=nshots, - backend=backend, - ) - ) - return gradients - - expval = exec_backend.execute_circuit( - circuit=circuit, initial_state=initial_state, nshots=nshots - ).expectation_from_samples(observable) - return expval, grad - - return _expectation_with_tf(params) - - else: - raise_error(NotImplementedError, "Only tensorflow supported at this time.") diff --git a/src/qiboml/operations/expectation.py b/src/qiboml/operations/expectation.py new file mode 100644 index 0000000..228e910 --- /dev/null +++ b/src/qiboml/operations/expectation.py @@ -0,0 +1,142 @@ +"""Compute expectation values of target observables with the freedom of setting any qibo's backend.""" + +from typing import List, Optional, Union + +import qibo +from qibo.backends import construct_backend +from qibo.config import raise_error + +from qiboml.backends import TensorflowBackend + + +def expectation( + observable: qibo.hamiltonians.Hamiltonian, + circuit: qibo.Circuit, + initial_state: Optional[Union[List, qibo.Circuit]] = None, + nshots: int = None, + backend: str = "qibojit", + differentiation_rule: Optional[callable] = None, +): + """ + Compute the expectation value of ``observable`` over the state obtained by + executing ``circuit`` starting from ``initial_state``. The final state is + reconstructed from ``nshots`` execution of ``circuit`` on the selected ``backend``. + In addition, a differentiation rule can be set, which is going to be integrated + within the used high-level framework. For example, if TensorFlow is used + in the user code and one parameter shift rule is selected as differentiation + rule, the expectation value is computed informing the TensorFlow graph to + use as gradient the output of the parameter shift rule executed on the selected + backend. + + Args: + observable (qibo.Hamiltonian): the observable whose expectation value has + to be computed. + circuit (qibo.Circuit): quantum circuit returning the final state over which + the expectation value of ``observable`` is computed. + initial_state (Optional[Union[List, qibo.Circuit]]): initial state on which + the quantum circuit is applied. + nshots (int): number of times the quantum circuit is executed. Increasing + the number of shots will reduce the variance of the estimated expectation + value while increasing the computational cost of the operation. + backend (str): backend on which the circuit is executed. This same backend + is used if the chosen differentiation rule makes use of expectation + values. + differentiation_rule (Optional[callable]): the chosen differentiation + rule. It can be selected among the methods implemented in + ``qiboml.differentiation``. + """ + + # read the frontend user choice + frontend = observable.backend + exec_backend = construct_backend(backend) + + kwargs = dict( + observable=observable, + circuit=circuit, + initial_state=initial_state, + nshots=nshots, + backend=backend, + differentiation_rule=differentiation_rule, + exec_backend=exec_backend, + ) + + if differentiation_rule is not None: + if isinstance(frontend, TensorflowBackend): + return _with_tf(**kwargs) + + elif nshots is None: + return _exact(observable, circuit, initial_state, exec_backend) + else: + return _with_shots(observable, circuit, initial_state, nshots, exec_backend) + + raise_error( + NotImplementedError, + "Only tensorflow automatic differentiation is supported at this moment.", + ) + + +def _exact(observable, circuit, initial_state, exec_backend): + """Helper function to compute exact expectation values.""" + return observable.expectation( + exec_backend.execute_circuit( + circuit=circuit, initial_state=initial_state + ).state() + ) + + +def _with_shots(observable, circuit, initial_state, nshots, exec_backend): + """Helper function to compute expectation values from samples.""" + return exec_backend.execute_circuit( + circuit=circuit, initial_state=initial_state, nshots=nshots + ).expectation_from_samples(observable) + + +def _with_tf( + observable, + circuit, + initial_state, + nshots, + backend, + differentiation_rule, +): + """ + Compute expectation sample integrating the custom differentiation rule with + TensorFlow's automatic differentiation. + """ + import tensorflow as tf # pylint: disable=import-error + + params = circuit.get_parameters() + nparams = len(params) + + exec_backend = construct_backend(backend) + + @tf.custom_gradient + def _expectation(params): + params = tf.Variable(params) + + def grad(upstream): + gradients = [] + for p in range(nparams): + gradients.append( + upstream + * differentiation_rule( + circuit=circuit, + hamiltonian=observable, + parameter_index=p, + initial_state=initial_state, + nshots=nshots, + backend=backend, + ) + ) + return gradients + + if nshots is None: + expval = _exact(observable, circuit, initial_state, exec_backend) + else: + expval = _with_shots( + observable, circuit, initial_state, nshots, exec_backend + ) + + return expval, grad + + return _expectation(params) diff --git a/tutorials/custom_differentiation.ipynb b/tutorials/custom_differentiation.ipynb new file mode 100644 index 0000000..68152a5 --- /dev/null +++ b/tutorials/custom_differentiation.ipynb @@ -0,0 +1,593 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "594e8add-362c-40a7-bb46-2d468529d9e9", + "metadata": {}, + "source": [ + "## Custom automatic differentiation\n", + "\n", + "In `Qiboml` we inherit the `backend` mechanism introduced in `Qibo`, extending it to the possibility of executing our quantum circuits on a specific engine indipendently of the choice of the high-level interface. \n", + "\n", + "This means you can decide to work with TensorFlow, or Pytorch, or others, depending on your personal preference, while keeping the possibility to freely set any `Qibo` backend for the circuit's execution.\n", + "\n", + "Moreover, we allow free choice of the differentiation rule to be used, which can be selected among the available differentiation rules implemented in `Qiboml`.\n", + "\n", + "A schematic representation of the pipeline follows, where we use as an example the custom differentiation rule of TensorFlow.\n", + "
\"drawing\"
" + ] + }, + { + "cell_type": "markdown", + "id": "e183c5e9-fac2-4fb0-add6-dad9deffa00e", + "metadata": {}, + "source": [ + "In practice, one defines the problem setup by setting the `Qibo` backend as usual. Let's set `tensorflow`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1cbe98d3-e4c9-4f89-9a27-3918f8877e51", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import time\n", + "from copy import deepcopy\n", + "\n", + "# disabling hardware accelerators warnings\n", + "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"\n", + "\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from qibo import set_backend\n", + "from qibo import Circuit, gates, hamiltonians\n", + "\n", + "from qiboml.operations import differentiation, expectation" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "bbbf2339-efe7-4170-9408-a7b66dff35ae", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Qibo 0.2.7|INFO|2024-04-29 18:33:34]: Using tensorflow backend on /device:CPU:0\n" + ] + } + ], + "source": [ + "set_backend(\"tensorflow\")" + ] + }, + { + "cell_type": "markdown", + "id": "d904f949-3546-4641-b1d4-c16c75cc8d54", + "metadata": {}, + "source": [ + "Now let's setup a simple problem. We build a quantum circuit $U$ composed of some rotations and we compute the gradients of\n", + "$$ \\langle 0 | U^{\\dagger} O U | 0 \\rangle, $$\n", + "where $O$ is an observable.\n", + "\n", + "Let's start with the circuit:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fd9264ff-150c-4be7-89ba-5b5fb37afa3b", + "metadata": {}, + "outputs": [], + "source": [ + "def build_parametric_circuit(nqubits, nlayers):\n", + " \"\"\"Build a Parametric Quantum Circuit with Qibo.\"\"\"\n", + " \n", + " c = Circuit(nqubits)\n", + " for _ in range(nlayers):\n", + " for q in range(nqubits):\n", + " c.add(gates.RY(q=q, theta=0))\n", + " c.add(gates.RZ(q=q, theta=0))\n", + " for q in range(0, nqubits-1, 1):\n", + " c.add(gates.CNOT(q0=q, q1=q+1))\n", + " c.add(gates.CNOT(q0=nqubits-1, q1=0))\n", + " c.add(gates.M(*range(nqubits)))\n", + "\n", + " return c" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "37ba84ff-ecb5-46b1-a060-8a4df87d1c1c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "q0: ─RY─RZ─o───X─RY─RZ─o───X─RY─RZ─o───X─M─\n", + "q1: ─RY─RZ─X─o─|─RY─RZ─X─o─|─RY─RZ─X─o─|─M─\n", + "q2: ─RY─RZ───X─o─RY─RZ───X─o─RY─RZ───X─o─M─\n" + ] + } + ], + "source": [ + "# circuit\n", + "nqubits = 3\n", + "nlayers = 3\n", + "\n", + "c = build_parametric_circuit(nqubits, nlayers)\n", + "print(c.draw())" + ] + }, + { + "cell_type": "markdown", + "id": "398f2f1f-3c91-4437-9a23-8801753c0785", + "metadata": {}, + "source": [ + "We can fill the circuit with a set of random parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "6b4afa80-bf2d-45e9-ba46-c7996a463fba", + "metadata": {}, + "outputs": [], + "source": [ + "# set random parameters\n", + "nparams = len(c.get_parameters())\n", + "np.random.seed(42)\n", + "params = np.random.uniform(0, 2*np.pi, nparams)\n", + "\n", + "c.set_parameters(params)" + ] + }, + { + "cell_type": "markdown", + "id": "26162d5b-45ca-45aa-af62-3367ef0d57a7", + "metadata": {}, + "source": [ + "We can now define a simple hamiltonian, which will be our target observable." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b73dad19-cadc-45d5-8d58-85ee5f4d4247", + "metadata": {}, + "outputs": [], + "source": [ + "# an observable\n", + "obs = hamiltonians.Z(nqubits=nqubits)" + ] + }, + { + "cell_type": "markdown", + "id": "2142eec0-4010-4140-b393-cccf8d32d1fa", + "metadata": {}, + "source": [ + "Once executed the circuit, we can use the final state to compute the expectation value of the target observable using the appropriate`Qibo` function. " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "6ea9d1a0-e908-4940-9759-fa4c92053ffa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tf.Tensor(0.34862167327428123, shape=(), dtype=float64)\n" + ] + } + ], + "source": [ + "# compute the expectation value\n", + "final_state = c(nshots=1000).state()\n", + "\n", + "print(obs.expectation(final_state))" + ] + }, + { + "cell_type": "markdown", + "id": "793479b8-ee33-4ec4-9b31-5b539234e904", + "metadata": {}, + "source": [ + "On the other hand, we developed a customized version of the `expectation` function in `qiboml`, which allows the user to keep the name convention, while integrating the possibility of customize the automatic differentiation provided by the chosen machine learning framework. It can be called from the `qiboml.expectation` module.\n", + "\n", + "This function accepts some more argument than the `qibo`'s one. In particular:\n", + "\n", + "- `observable`: the target observable, whose expectation value we are interested in;\n", + "- `circuit`: the circuit which returns the final state used to compute the expectation value;\n", + "- `inital_state`: the state of the system before applying the circuit;\n", + "- `nshots`: the number of shots to compute the expectation value;\n", + "- `backend`: the `qibo` backend on which we want to execute the circuit. This backend can even be a real quantum computer, when setting the `qibolab` backend;\n", + "- `differentiation_rule`: the actual differentiation rule one wants to apply. " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "0c279101-960f-4ce4-b11f-3297c24e5e72", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tf.Tensor(0.2, shape=(), dtype=float64)\n" + ] + } + ], + "source": [ + "exp = expectation.expectation(\n", + " observable=obs,\n", + " circuit=c,\n", + " backend=\"numpy\",\n", + " differentiation_rule=differentiation.parameter_shift,\n", + " nshots=100,\n", + ")\n", + "\n", + "print(exp)" + ] + }, + { + "cell_type": "markdown", + "id": "a859b18b-c18d-4297-9a08-df433480f6e2", + "metadata": {}, + "source": [ + "To check if we are actually changing backend, we can compute a certain number of times the expectation value, and plot the time of execution of different backend engines, such that `tensorflow` or `numpy`." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "bec419f1-0fca-4d7a-beca-e09947951278", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0/500 exec\n", + "100/500 exec\n", + "200/500 exec\n", + "300/500 exec\n", + "400/500 exec\n" + ] + } + ], + "source": [ + "np_times, tf_times = [0.], [0.]\n", + "nexec = 500\n", + "\n", + "for n in range(nexec):\n", + " # some logging messages\n", + " if (n%100==0):\n", + " print(f\"{n}/{nexec} exec\")\n", + " \n", + " # executing on numpy backend\n", + " it = time.time()\n", + " expectation.expectation(\n", + " observable=obs,\n", + " circuit=c,\n", + " backend=\"numpy\",\n", + " differentiation_rule=differentiation.parameter_shift\n", + " )\n", + " ft = time.time()\n", + " np_times.append((ft-it))\n", + "\n", + " # executing on tensorflow backend\n", + " it = time.time()\n", + " expectation.expectation(\n", + " observable=obs,\n", + " circuit=c,\n", + " backend=\"tensorflow\",\n", + " differentiation_rule=differentiation.parameter_shift\n", + " )\n", + " ft = time.time()\n", + " tf_times.append((ft-it))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "72af880a-092d-4531-85d8-e1809359788d", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgsAAAGZCAYAAAAHGXIbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbRUlEQVR4nO3dd3hTZf8G8DtJ092kbdIJhYLIkL1FFPAHCDIEQQTkhTJEEVAQHCDKUDaCvAryKkiLynAxVLZAQZAtWyhDNnTvmXGe3x+xkdBSUkibpLk/15WL9pyTkyffhp675zzneWRCCAEiIiKie5DbuwFERETk2BgWiIiIqEQMC0RERFQihgUiIiIqEcMCERERlYhhgYiIiErEsEBEREQlcrN3Ax6GJEm4desW/Pz8IJPJ7N0cIiIipyGEQFZWFsLDwyGXl3zuwKnDwq1btxAREWHvZhARETmt69evo3LlyiVu49Rhwc/PD4DpjapUKpvsU6/XY9u2bXjmmWegVCptsk9Xx5raFutpe6ypbbGetlcWNc3MzERERIT5WFoSpw4LhZceVCqVTcOCt7c3VCoVP+Q2wpraFutpe6ypbbGetleWNbXmMj47OBIREVGJGBaIiIioRAwLREREVCKn7rNgLaPRCL1eb9W2er0ebm5uyM/Ph9FoLOOWuQbW1LbuVU+lUgmFQmHHlhFRRVWhw4IQAvHx8UhPTy/Vc0JDQ3H9+nWO3WAjrKltlVRPf39/hIaGss5EZFMVOiwUBoXg4GB4e3tb9QtUkiRkZ2fD19f3voNUkHVYU9sqrp5CCOTm5iIxMREAEBYWZs8mElEFU2HDgtFoNAcFjUZj9fMkSYJOp4OnpycPbDbCmtrWverp5eUFAEhMTERwcDAvSRCRzVTY39yFfRS8vb3t3BKi8lP4ebe2jw4RkTUqbFgoxGu35Er4eSeislDhwwIRERE9HIYFKjdDhgxBz549y/Q1pk6dikaNGpXpaxRn8ODBZf7eiIjshWHBAQ0ePBgymQyzZ8+2WL5+/XqeZiYickFZuRKSMr3s9voMCw7K09MTc+bMQVpamr2bQkREdnTqYj5em5OEn/+sjswcyS5tYFhwUB06dEBoaChmzZp1z22KO+W+cOFCREZGmr8vPD0+c+ZMhISEwN/fHx9++CEMBgPefvttBAYGonLlyoiOjjY/58qVK5DJZFizZg2eeOIJeHp6ol69eti9ezcA0z39NWrUwMcff2zx2sePH4dMJsPFixdLfG/Tpk1DUFAQVCoVRowYAZ1OZ163ZcsWPPnkk/D394dGo0G3bt1w6dIli+ffuHED/fv3R2BgIHx8fNCsWTMcPHiw2Ne6dOkSqlevjtGjR0MIgYKCArz11luoVKkSfHx80LJlS8TGxpq3j4mJgb+/P7Zu3Yo6derA19cXnTt3xu3bt83bGI1GjBs3ztzGd955B0KIEt8zEVFpGY0CKzZm4M1PEpGULkEhF0jLss8ouK4VFoQAcnLK//EABxKFQoGZM2fis88+w40bNx7qbe/cuRO3bt3Cnj17sGDBAkyZMgXdunVDQEAADh48iBEjRuDVV18t8jpvv/02xo8fj2PHjqFVq1bo3r07UlJSIJPJMHToUIuAAQDR0dFo06YNatSocc+27NixA2fPnkVsbCxWr16NtWvXYtq0aeb1OTk5GDduHI4cOYIdO3ZALpfj+eefhySZ0nR2djbatm2Lmzdv4ueff8aJEyfwzjvvmNff6eTJk3jyySfx0ksvYdGiRZDJZBg9ejT279+PNWvW4OTJk+jTpw86d+6MCxcumJ+Xm5uLjz/+GN988w327NmDa9eu4a233jKvnz9/PmJiYrB8+XLs3bsXqampWLduXel+KEREJUhINWDcwkSs2JgBSQAdmnvhpSfOoWqonab8Fk4sIyNDABAZGRlF1uXl5Ym//vpL5OXl/bswO1sI06G7fB/Z2aV6X1FRUaJHjx5CCCEef/xxMXToUCGEEOvWrRN3/simTJkiGjZsaPHcTz75RFStWtViX1WrVhVGo9G8rFatWuKpp54yf28wGISPj49YvXq1EEKIy5cvCwBi9uzZ5m30er2oXLmymDNnjhBCiJs3bwqFQiEOHjwohBBCp9MJrVYrYmJiirwfo9Eo0tLSxKBBg0RgYKDIyckxr1uyZInw9fW1aN+dkpKSBABx6tQpIYQQX3zxhfDz8xMpKSnFbl9Yk3379omAgADx8ccfm9ddvXpVKBQKcfPmTYvntG/fXkycOFEIIUR0dLQAIC5evGhev3jxYhESEmL+PiwsTMydO7dIbQp/ZmWtsJ7F1azYzz3dl06nE+vXrxc6nc7eTakQWM+Hs/vPHNF93DXx9GtXRdc3r4ntB7PLpKYlHUPv5lpnFpzQnDlzsGLFCpw9e/aB91G3bl2Lkf5CQkJQv3598/cKhQIajcY8VHChVq1amb92c3NDs2bNzO0IDw9H165dsXz5cgDAL7/8goKCAvTp06fEtjRs2NBioKxWrVohOzsb169fBwBcuHAB/fv3R/Xq1aFSqcyXVK5duwbAdKmjcePGCAwMvOdrXLt2DR07dsTkyZMxfvx48/JTp07BaDSiZs2a8PX1NT92795tcanD29sbjzzyiPn7sLAwc20yMjJw+/ZttGzZskhtiIgeRr5OwvyVKZi6NBnZeQK1I93xxcRQdGjhY++mVdzhnovl7Q1kZ5e4iSRJyMzMhEqlst3QxA8ximSbNm3QqVMnTJw4EYMHD7ZYJ5fLi1wrL27kPqXS8rSVTCYrdllxp/JL8vLLL2PgwIH45JNPEB0djb59+z70iJndu3dH1apVsXTpUoSHh0OSJNSrV8/cr6FwSOOSBAUFITw8HKtXr8bQoUOhUqkAmC5hKBQKHD16tMhQyL6+vuavi6vN3XUmIrKlSzd0mL48GVfjDZDJgH4dVRjSXQ03hWPcAedaYUEmA3zuk9AkCTAaTds5yDwGs2fPRqNGjVCrVi2L5UFBQYiPj4cQwnxL5fHjx232ugcOHECbNm0AAAaDAUePHsXo0aPN67t06QIfHx8sWbIEW7ZswZ49e+67zxMnTiAvL8980D9w4AB8fX0RERGBlJQUxMXFYenSpXjqqacAAHv37rV4foMGDbBs2TKkpqbe8+yCl5cXfv31V3Tp0gWdOnXCtm3b4Ofnh8aNG8NoNCIxMdG8/9JSq9UICwvDwYMHi9SmSZMmD7RPInJdQgisi83GF+vSoDcAGrUCE6M0aFLb095Ns+AYR0MqUf369TFgwAB8+umnFsvbtWuHpKQkzJ07F5cuXcLixYuxefNmm73u4sWLsW7dOpw7dw6jRo1CWloahg4dal6vUCgwePBgTJw4EY8++qjFZYt70el0GDZsGP766y9s2rQJU6ZMwejRoyGXyxEQEACNRoMvv/wSFy9exM6dOzFu3DiL5/fv3x+hoaHo2bMn9u3bh7///hs//fQT9u/fb7Gdj48PNm7cCDc3Nzz77LPIzs5GzZo1MWDAAAwaNAhr167F5cuXcejQIcyaNQsbN260ui5jxozB7NmzsX79epw7dw4jR44s1TToREQAkJ5lxKQlSVj0gykoPF7PE0vfC3W4oAAwLDiNDz/8sMhlgjp16uDzzz/H4sWL0bBhQxw6dMii1/7Dmj17NmbPno2GDRti7969+Pnnn6HVai22GTZsGHQ6HYYMGWLVPtu3b49HH30Ubdq0Qd++ffHcc89h6tSpAEyXVdasWYOjR4+iXr16ePPNNzFv3jyL57u7u2Pbtm0IDg5Gly5dUL9+fcyePbvYGRZ9fX2xefNmCCHQtWtX5OTkIDo6GoMGDcL48eNRq1Yt9OzZE4cPH0aVKlWsrsv48eMxcOBAREVFoVWrVvDz88Pzzz9v9fOJiP48l4/hM+Nx4HQ+lG7A6D4BmPFaEPz9HHO2WJlw4ouxmZmZUKvVyMjIMF+XLpSfn4/Lly+jWrVq8PS0PqWVSZ8FJ3PlyhVUq1YNx44du+/Qyb///jvat2+P69evIyQkpNhtWFPbKqmeD/q5d3V6vR6bNm1Cly5divRZodJjPe/NYBSI/iUDa7ZnQgigSogbPhimxSOV3Ut8XlnUtKRj6N1cq88C2UxBQQGSkpIwdepU9OnT555BgYiITG4m6TEjOgXnrpg6bHd70hcjX/CHp7vj/xHl+C0kh7R69WpUrVoV6enpmDt3rr2bQ0Tk0H47lINXZ8Xj3BUdfL1kmDpci3EvBTpFUAB4ZoGKERkZed9bBQcPHlzkVk4iIrKUmy/h0+/SsO1gDgCg/iMeeG+IBiGBznX4da7WEhEROYm4qwWYvjwFN5MMkMuAgV3U+E9nFRQOMnZCaTAsEBER2ZAkCXy/IwtfbUiHUQKCAxSYNESD+jWct9MxwwIREZGNpGYYMWtFCo6eywcAtGnshfEDNPDzdo6+CffCsEBERGQDB8/kYc6KFKRnS/BQyjCqTwC6tvYxj7DrzBgWiIiIHoJOL7BsQzp+3JkFAKgersT7w7SIDKs4Y0wwLBARET2gawl6TF+ejIvXTZP4Pd/OF68+HwB3pfOfTbiTc19EIacREBCA9evXl9vrRUZGYuHChWX6Gu3atcPYsWPL9DWKU716dSxZsqTcX5eI/iWEwOb92RgxKx4Xr+uh8pFj+ggtXn8xsMIFBYBhweHIZLISH4XzKDiayMjIIm2tXLmyvZtFRGRz2bkSPvoqBfO+SUW+TqBxLQ8smxSKJxp427tpZYaXIRzM7du3zV9/9913mDx5MuLi4szLfH197dGse9LpdHB3N41p/uGHH2L48OHmdcVN7kRE5MxOXyrAjOhkJKQaoZADQ7qp0fcZFRTyinc24U48s+BgQkNDzQ+1Wg2ZTGaxbM2aNahTpw48PT1Ru3ZtfP755+bnXrlyBTKZDGvXrsXTTz8Nb29vNGzY0GL65qtXr6J79+4ICAiAj48P6tati02bNpnX7969Gy1atICHhwfCwsIwYcIEGAwG8/p27dph9OjRGDt2LLRaLTp16mRe5+fnZ9HWoKCge77PU6dO4f/+7//g5eUFjUaDV155BdnZ2QCA06dPQy6XIykpCQCQmpoKuVyOfv36mZ8/ffp0PPnkkyXWMisrC/3794ePjw8qVaqExYsXW6xfsGAB6tevDx8fH0RERGDkyJHmNhTat28f2rVrB29vbwQEBKBTp05IS0sr9vU2btwItVqNlStXAgCuX7+OF198Ef7+/ggMDESPHj1w5coV8/aDBw9Gz5498fHHHyMsLAwajQajRo2CXq83b5OYmIju3bvDy8sL1apVM++biMqXURL4elMGxi5IQEKqEWFaN3z6Vghe6qyu8EEBcLGwIIRAXoF030e+zrrtrH3YamLPlStXYvLkyZgxYwbOnj2LmTNn4oMPPsCKFSsstps0aRLeeustHD9+HDVr1kT//v3NB/xRo0ahoKAAe/bswalTpzBnzhzz2YqbN2+iS5cuaN68OU6cOIElS5bgq6++wvTp0y32v2LFCri7u2Pfvn343//+V+r3kZOTg06dOiEgIACHDx/GDz/8gN9++w2jR48GANStWxcajQa7d+8GYJrZ8s7vAVOoadeuXYmvM2/ePDRs2BDHjh3DhAkTMGbMGGzfvt28Xi6X49NPP8WZM2ewYsUK7Ny5E++88455/fHjx9G+fXs89thj2L9/P/bu3Yvu3bvDaDQWea1Vq1ahf//+WLlyJQYMGAC9Xo9OnTrBz88Pv//+O/bt2wdfX1907twZOp3O/Lxdu3bh0qVL2LVrF1asWIGYmBjExMSY1w8ePBjXr1/Hrl278OOPP+Lzzz9HYmJiqepNRA8nIdWAcQsTEfNrBiQBdGjhjS8nhqJOpIe9m1ZuXOoyRL5OoOubN6zcOstmr7vxk8rw8nj45DllyhTMnz8fvXr1AgBUq1YNf/31F7744gtERUWZt3vrrbfQtWtXAMC0adNQt25dXLx4EbVr18a1a9fQu3dv1K9fH4Cps1yhzz//HBEREVi0aBFkMhlq166NW7du4d1338XkyZPN0yE/+uijxU4e9e677+L99983fz9z5ky88cYbRbZbtWoV8vPz8fXXX8PHxwcAsGjRInTv3h1z5sxBSEgI2rRpg9jYWLzwwguIjY3FkCFDsGzZMpw7dw6PPPII/vjjD4sDe3Fat26NCRMmAABq1qyJffv24ZNPPkHHjh0BwKJzYmRkJKZPn44RI0aYz9bMnTsXzZo1szh7U7du3SKvs3jxYkyaNAm//PIL2rZtC8B0CUmSJCxbtsx8j3V0dDT8/f0RGxuLZ555BoCp4+eiRYugUChQu3ZtdO3aFTt27MDw4cNx/vx5bN68GYcOHULz5s0BAF999RXq1KlT4vsmItvZcywXH3+bguw8AS8PGcb2C0THlj72bla5c6mw4MxycnJw6dIlDBs2zKJfgMFggFqttti2QYMG5q/DwsIAmE5n165dG2+88QZee+01bNu2DR06dEDv3r3N2589exatWrWyGECkdevWyM7Oxo0bN1ClShUAQNOmTYtt49tvv20xuZRWqy12u7Nnz6Jhw4bmoFD4OpIkIS4uDiEhIWjbti2+/PJLAKazCDNnzsT58+cRGxuL1NRU6PV6tG7dusSatWrVqsj3d94h8dtvv2HWrFk4d+4cMjMzYTAYkJ+fj9zcXHh7e+P48ePo06dPia/x448/IjExEfv27TMf0AHgxIkTuHjxIvz8/Cy2z8/Px6VLl8zf161b16JvR1hYGE6dOmWuk5ubm0W9a9euDX9//xLbREQPL69Awuc/pmHjPtMEULUj3TFpiAaVgirO2Aml4VJhwdNdho2flNxDX5IkZGVlwc/Pz/yXtC1e92EVXktfunQpWrZsabHu7o6ESuW/H+bCA78kSQCAl19+GZ06dcLGjRuxbds2zJo1C/Pnz8frr79udVvuPMjfSavVokaNGlbvpySFtyVeuHABf/31F5588kmcO3cOsbGxSEtLQ7NmzeDt/eA9j69cuYJu3brhtddew4wZMxAYGIi9e/di2LBh0Ol08Pb2hpeX133307hxY/z5559Yvnw5mjVrZq53dnY2mjZtWmwfgzv7ctz5swJMP6/CnxUR2cfF6zpMX56MawkGyGRAv44qDOmuhpsTTgBlKy7VZ0Emk8HLQ37fh6e7ddtZ+7DFUJ8hISEIDw/H33//jRo1alg8qlWrVqp9RUREYMSIEVi7di3Gjx+PpUuXAgDq1KmD/fv3W/Sx2LdvH/z8/Gx6G2SdOnVw4sQJ5OTkWLyOXC5HrVq1AAD169dHQEAApk+fjkaNGsHX1xft2rXD7t27ERsbe9/+CgBw4MCBIt8XnsI/evQoJEnC/Pnz8fjjj6NmzZq4deuWxfYNGjTAjh07SnyNRx55BLt27cKGDRssAleTJk1w4cIFBAcHF/l53X0m6F5q164Ng8GAo0ePmpfFxcUhPT3dqucTUekIIfDjzkyMmhePawkGaNQKzHsjGMN7+rt0UABcLCw4u2nTpmHWrFn49NNPcf78eZw6dQrR0dFYsGCB1fsYO3Ystm7disuXL+PPP//Erl27zAfQkSNH4vr163j99ddx7tw5bNiwAVOmTMG4ceNsdpYFAAYMGABPT09ERUXh9OnT2LVrF15//XUMHDgQISEhAEzBrk2bNli5cqU5GDRo0AAFBQXYsWOHuW9ASfbt24e5c+fi/PnzWLx4MX744QeMGTMGAFCjRg3o9Xp89tln+Pvvv/HNN98U6aw5ceJEHD58GCNHjsTJkydx7tw5LFmyBMnJyRbb1axZE7t27cJPP/1k7gcxYMAAaLVa9OjRA7///jsuX76M2NhYvPHGG7hxw7p+M7Vq1ULnzp3x6quv4uDBgzh69Chefvllq854EFHppGUZ8d7nSfj8x3ToDUCr+l5YNikUTWo570yRtsSw4ERefvllLFu2DNHR0ahfvz7atm2LmJiYUp1ZMBqNGDVqFOrUqYPOnTujZs2a5g58lSpVwqZNm3Do0CE0bNgQI0aMwLBhwyw6LdqCt7c3tm7ditTUVDRv3hwvvPAC2rdvj0WLFlls17ZtWxiNRnNYkMvlaNOmDWQy2X37KwDA+PHjceTIETRu3BjTp0/HggULzLd6NmzYEAsWLMCcOXNQr149rFy5ErNmzbJ4fs2aNbFt2zacOHECLVq0QKtWrbBhwwa4uRW9elerVi3s3LkTq1evxvjx4+Ht7Y09e/agSpUq6NWrF+rUqYNhw4YhPz8fKpXK6lpFR0cjPDwcbdu2Ra9evfDKK68gODjY6ucT0f0d/isPL8+4jYNn8qF0A97oG4DpI7RQ+3KsmEIyYav7+uwgMzMTarUaGRkZRX4B5+fn4/Lly6hWrRo8Pa1PhpIkITMzEyqVyqZ/Tbsy1tS2Sqrng37uXZ1er8emTZvQpUuXIv1IqPScpZ56g2kCqB92mO5+iwxT4v2hGlSv5G7nlhVVFjUt6Rh6N5fq4EhERAQA1/+ZAOrCPxNAPfeUL17r7Q8Pd/5BUxyGBSIichlCCGw5kIPPvk9DfoGAykeOt/8TiNYNK+68DrbAsEBERC4hO1fCJ6tTsetoLgCgUU0PTBysQZA/D4X3wwoREVGFd+cEUHI5MNRFJoCyFbtenDEajfjggw9QrVo1eHl54ZFHHsFHH31ks7kUANh0X0SOjp93IktGSeCbTRkY+8k/E0BpFPhsvOtMAGUrdj2zMGfOHCxZsgQrVqxA3bp1ceTIEQwZMgRqtbrYOQVKo7C3aG5uLu9LJ5eRm2s6verIPdCJyktSmgEzY1Jw4kIBAKB9c2+M7RcIHy92Yiwtu4aFP/74Az169DBPehQZGYnVq1fj0KFDD71vhUIBf39/8wx93t7eVo2kKEkSdDod8vPzeZufjbCmtlVcPYUQyM3NRWJiIvz9/YsMAU7kan4/nouPv01FVq4ELw8ZxvQLxDMuOAGUrdg1LDzxxBP48ssvcf78edSsWRMnTpzA3r177zkiYUFBAQoKCszfZ2ZmAjDdf6rX64tsr9FoYDQakZCQYHWbhBDIz8+Hp6enTYZpJtbU1kqqp0qlgkajKfb/A91bYb1YN9uwZz0LdAJfrM/Exn2ms2w1I5SYEOWPSkFuTv3zLYualmZfdh2USZIkvPfee5g7dy4UCgWMRiNmzJiBiRMnFrv91KlTMW3atCLLV61aVeKkQjKZjH9pUYVnNBrZZ4FcWnKWJ7acjERKtunSc9PIBLR69DYUcv6/KE5ubi5eeuklqwZlsmtYWLNmDd5++23MmzcPdevWxfHjxzF27FgsWLAAUVFRRbYv7sxCREQEkpOTSzWEbkn0ej22b9+Ojh078rqvjbCmtsV62h5ralvlXU8hBH7Zm4sv12dCbwACVXK8NcAfTWt7lPlrl5eyqGlmZia0Wq3jj+D49ttvY8KECejXrx8A00yDV69exaxZs4oNCx4eHvDwKPrDVyqVNv9AlsU+XR1ralusp+2xprZVHvXMyDZi3rep+ONkHgCgZV1PvDNIgwC/ink22ZY1Lc1+7BoWcnNzi3R4UygUkCTJTi0iIiJncSwuHzNjUpCSYYTSDXilpz96Pe3HvlFlwK5hoXv37pgxYwaqVKmCunXr4tixY1iwYAGGDh1qz2YREZEDMxgFVvyagVXbMiEEEBHihg+GalEjwvEmgKoo7BoWPvvsM3zwwQcYOXIkEhMTER4ejldffRWTJ0+2Z7OIiMhB3U42YPryZJy9ogMAdGntg1EvBMDLg7dllyW7hgU/Pz8sXLgQCxcutGcziIjICew4nIOFq1ORky/g4yXD+AEatGvCCaDKA+eGICIih5aXL+HT79Ow9UAOAKBudXdMGqJFqIaHsPLCShMRkcM6f02H6cuTcSPRALkM+M+zKgx8Vg2Fgp0YyxPDAhERORxJEvhxZxaWbUiHwQgE+Svw3hANGj7qae+muSSGBSIiciipGUbM/joFR87mAwCeauSF8QMCofKpmGMnOAOGBSIichiHzuRhztcpSMuS4K6UYdQL/uj2pC/HTrAzhgUiIrI7nV7gq5/T8cOOLABA9XAlJg3VoFo4x05wBAwLRERkV9cS9Ji+PBkXr5tmQezZ1hcjegXAXcmzCY6CYYGIiOxCCIEtB3Lw2XdpyNcJqHzkeHtgIFo34NgJjoZhgYiIyl12roRPVqdi19FcAECjmh6YOFiDIH8elhwRfypERFSuTl8qwIzoZCSkGiGXA0O7qdH3GRUUcl52cFQMC0REVC6MksCqrZlYsTEDkgSEaRR4f6gWdap52LtpdB8MC0REVOaS0gyYGZOCExcKAADtm3tjTL9A+HpxAihnwLBARERlau/xXHy8MhWZORI8PWQY2zcAHVv6cOwEJ8KwQEREZaJAJ7D4x1T8/Hs2AKBmFXe8P1SDysFKO7eMSothgYiIbC45yxOvz0/G1XgDAODFDn4Y9pw/lG48m+CMGBaIiMhmhBD45fccrDlQC0bJgACVHBMGadD8MS97N40eAsMCERHZREa2ER9/m4p9J/MAyNG8jgcmDNYiwI8TQDk7hgUiInpox8/nY2ZMCpLTjXBTAE/UuIGJrzSDhweDQkXAsEBERA/MYBRYsTEDq7ZmQgggIsQNEwb54/zJY5BzkKUKg2GBiIgeyO1kA2ZEJ+OvyzoAQJcnfDCqTwDc5EacP2nnxpFNMSwQEVGp7TySg09WpSInX8DHS4bxLwWiXVMfAIBeb7Rz68jWGBaIiMhqefkSPv0+DVsP5AAAHqvmjveHahGq4eGkIuNPl4iIrHL+mg7TlyfjRqIBchkwoLMKg7qooVCwb0JFx7BAREQlkiSBn3ZlYen6dBiMQJC/Au8N1qBhTU97N43KCcMCERHdU2qmEXO/TsGhv/IBAE829MJb/wmEyoe3RLoShgUiIirW4b/yMHtFCtKyJLgrZRjZ2x/dn/LlBFAuiGGBiIgs6PQCX/2cjh92ZAEAqoUr8f5QDaqFu9u5ZWQvDAtERGR2LUGP6cuTcfG6HgDQs60vXn3eHx7ucju3jOyJYYGIiCCEwOY/crDohzTk6wRUPnK8MzAQTzTwtnfTyAEwLBARubjsXAnzV6Vi95+5AIAmtTwwIUoDrT8PEWTCTwIRkQs7dTEfM2JSkJhqhEIODH3OH307+HFeB7LAsEBE5IKMRoFvt2Tim00ZkAQQHuSGSUM0qBPpYe+mkQNiWCAicjHxKQbMiknBqUsFAIBnWvrgjb4B8PZkJ0YqHsMCEZELif0zF/NXpiAnT8DbU4Y3+weifXMfezeLHBzDAhGRC8grkLD4hzRs+sM0AVTtSNMEUOFaHgbo/vgpISKq4C5cN00AdT3BAJkMeOkZFaK6qeHGCaDISgwLREQVVOEEUMs2pENvADRq0wRQjWtxAigqHYYFIqIK6O4JoFo3ME0ApfblBFBUegwLREQVzKEzeZjz9b8TQL3W2x/PcQIoeggMC0REFYROL7BsQzp+3MkJoMi2GBaIiCoATgBFZYlhgYjIiXECKCoPDAtERE4qK1fCAk4AReWAnygiIifECaCoPDEsEBE5EaNR4JvNGfh2cyYngKJyw7BAROQk4lMMmBmTgtOcAIrKGcMCEZET4ARQZE8MC0REDuzuCaDqRLpjEieAonLGTxsRkYM6f800AdSNRE4ARfbFsEBE5GAKJ4Bauj4dBiOg9VdgYhQngCL7YVggInIgnACKHBHDAhGRg7h7AqiRvf3RnRNAkQNgWCAisrO7J4CqHq7EJE4ARQ6EYYGIyI44ARQ5A4YFIiI74ARQ5EwYFoiIyhkngCJnw08mEVE54gRQ5IwYFoiIykFxE0C9P0SD2pwAipwAwwIRURlLTDVgRkwKTl3kBFDknOz+Sb158yb+85//QKPRwMvLC/Xr18eRI0fs3SwiIpv4/Xguhs+Mx6mLBfD2lOG9wRpMiNIwKJBTseuZhbS0NLRu3RpPP/00Nm/ejKCgIFy4cAEBAQH2bBYR0UMr0ElY8lM6fv49GwBQq6o73h+qQaUgpZ1bRlR6dg0Lc+bMQUREBKKjo83LqlWrds/tCwoKUFBQYP4+MzMTAKDX66HX623SpsL92Gp/xJraGutpe7au6ZVbesxckY6r8QYAQJ/2Pojq4gelm2v83PgZtb2yqGlp9iUTQgibvXIpPfbYY+jUqRNu3LiB3bt3o1KlShg5ciSGDx9e7PZTp07FtGnTiixftWoVvL15bzIR2ZcQwKkbGuw5VxlGSQ5vdz2eqX8VVbVZ9m4aURG5ubl46aWXkJGRAZVKVeK2dg0Lnp6mGdTGjRuHPn364PDhwxgzZgz+97//ISoqqsj2xZ1ZiIiIQHJy8n3fqLX0ej22b9+Ojh07Qqnk6UJbYE1ti/W0PVvUNDNHwsI1Gdh30jQBVLM6Hnh7gBr+fq43ARQ/o7ZXFjXNzMyEVqu1KizY9TKEJElo1qwZZs6cCQBo3LgxTp8+fc+w4OHhAQ+PorcZKZVKm38gy2Kfro41tS3W0/YetKYnL+ZjZnQKEtOMcFMAw3v6o/fTHDuBn1Hbs2VNS7Mfu4aFsLAwPPbYYxbL6tSpg59++slOLSIisp5REvh2cya+2ZQBSQCVgtzwwTAtalbhBFBUsdg1LLRu3RpxcXEWy86fP4+qVavaqUVERNZJSjdgxvIUnOTYCeQC7BoW3nzzTTzxxBOYOXMmXnzxRRw6dAhffvklvvzyS3s2i4ioRAfP5GH2ihRkZEvw8pBhbL9AdGzpY+9mEZUZu4aF5s2bY926dZg4cSI+/PBDVKtWDQsXLsSAAQPs2SwiomIZjALLf07Hmu2muxtqVFZi8staVA7mdXmq2Ow+3HO3bt3QrVs3ezeDiKhEOr3AhMWJOH7edNmhZ1tfjOgVAHela3diJNdg97BAROQMPv0+FcfPF8DHU4a3B2rQpjHHdiHXwbBARHQfpy7mY9O+HMhkwNRXgtC0tqe9m0RUrhgWiIjuQQiBLftz8Nn3aQCAzo/7MCiQS2JYICIqRnauhAWrUhH7Zy4AoHEtD4zozUnuyDUxLBAR3eXUxXzMiElBYqoRCjkw9Dl/9O3AERnJdTEsEBH9w2gUWLk1Hd9uzoQkgPAgN7w/RIPakUWHmSdyJQwLREQAMvOUePuzFJy5bJq2lyMyEv2LYYGIXF7sn3lY+Udt6Ax6+HjKMLZ/INo354iMRIUYFojIZeXlS/j0+zRsPZADwA11IpV4f2gQwrT81Uh0J/6PICKXFHe1ADOiU3Aj0QC5DGhW7Tamjm4MT0/+WiS6G/9XEJFLkSSB73dkYfnP6TAYgSB/Bd4dqMb188egUPBuB6LiMCwQkctIyTBi9ooUHD2XDwBo09gL4wdo4Kk04vp5OzeOyIExLBCRS/jjZC7mfZuKjGwJnu4yjOoTgC5P+EAmk0GvN9q7eUQOjWGBiCq0Ap2EL9alY/3ubACmaaXfH6pFlVBOK01kLYYFIqqwLt/SYfryFFy+ZRo7oU97Pwx7zp/TShOVEsMCEVU4Qgj8vCcbS9amQ6cXCPCT491BGrSo62XvphE5JavCws8//1zqHXfs2BFeXvyPSUTlKyPbiHnfpuKPk3kAgBZ1PfHOQA0CVQo7t4zIeVkVFnr27FmqncpkMly4cAHVq1d/kDYRET2QP8/lY9aKFKRkGKF0A17p6Y/n23ECKKKHZfVliPj4eAQHB1u1rZ+f3wM3iIiotPQGgehfM/Dd9kwIAVQJccMHw7R4pLK7vZtGVCFYFRaioqJKdUnhP//5D1Qq1QM3iojIWjcT9Zi+PAVx13QAgG5P+mLkC/7wdOcEUES2YlVYiI6OLtVOlyxZ8kCNISKylhAC2w7m4NPv0pBXIODnLcf4AYFo09jb3k0jqnAe+m6IzMxM7Ny5E7Vq1UKdOnVs0SYiohJl50n4ZHUqdh3JBQA0fNQDE6M0CA7kDV5EZaHU/7NefPFFtGnTBqNHj0ZeXh6aNWuGK1euQAiBNWvWoHfv3mXRTiIiAMCZvwswIzoZ8SlGyOXAkG5q9HtGBQU7MRKVmVJf1NuzZw+eeuopAMC6desghEB6ejo+/fRTTJ8+3eYNJCICAKMk8M2mDIxZkID4FCPCNAp8Oj4EAzqrGRSIylipw0JGRgYCAwMBAFu2bEHv3r3h7e2Nrl274sKFCzZvIBFRQqoB4xYmIvrXDEgS0KG5N758LwyPVfOwd9OIXEKpL0NERERg//79CAwMxJYtW7BmzRoAQFpaGjw9PW3eQCJybbv/zMX8lSnIzhPw8pBhbL9AdGzpY+9mEbmUUoeFsWPHYsCAAfD19UXVqlXRrl07AKbLE/Xr17d1+4jIReUVSFj8Yxo27csBANSOdMekIRpUCuIEUETlrdRhYeTIkWjZsiWuXbuGjh07Qi43XcmoXr06+ywQkU1cuK7D9OXJuJ5ggEwG9H9GhcHd1HBTsG8CkT080H1GTZs2RdOmTS2Wde3a1SYNIiLXJUkCP+3KwtL16TAYAa2/AhOjNGhci5c4iezJqg6O48aNQ05OjtU7nThxIlJTUx+4UUTkelIzjJj4eRKW/GQKCq0bemHpe6EMCkQOwKqw8N///he5ublW73Tx4sVIT09/0DYRkYs5cDoPw2fexuG/8uGulGFsvwB8+IoWal/OFEnkCKy6DCGEQM2aNSGTWXe9sDRnIYjIden0Al+uT8faXVkAgOqVlHh/qBaRYezESORIymRuCAAICQkp9XOIyHVcva3H9OhkXLqhBwD0aueLV54PgLuSnRiJHI3Vs04SEdmCEAIb9+Vg8Q9pKNAL+PvK8c5ADR6vb/3MtkRUvjjrChGVm4xsI+avTMXeE3kAgKa1PTEhSgONmn0TiBwZwwIRlYvj5/MxMyYFyelGuCmAl3v444X/84Oc8zoQOTyGBSIqUwajwIpfM7BqWyaEACoHu+H9oVrUrOJu76YRkZUYFoiozNxKNmDG8mScvaIDAHR5wgejXgiAl2ep57AjIjt64LBw8eJFXLp0CW3atIGXlxeEEFbfWklEFd/2gzn473epyM0X8PWSYdwADdo18bZ3s4joAZQ6LKSkpKBv377YuXMnZDIZLly4gOrVq2PYsGEICAjA/Pnzy6KdROQkcvIk/Pe7VPx2yDSQW/1HPPDeEA1CAnkik8hZlfpc4Jtvvgk3Nzdcu3YN3t7//pXQt29fbNmyxaaNIyLncvZyAV6ZFY/fDuVCLgMGd1NjwdhgBgUiJ1fq/8Hbtm3D1q1bUblyZYvljz76KK5evWqzhhGR8zBKAmu2ZSL61wxIEhASqMCkIVrUe8TD3k0jIhsodVjIycmxOKNQKDU1FR4e/MVA5GqS0gyYFZOC4xcKAABPN/XGm/0D4evNToxEFUWp/zc/9dRT+Prrr83fy2QySJKEuXPn4umnn7Zp44jIse09novhM+Nx/EIBPD1keGdgIN4fqmFQIKpgSn1mYe7cuWjfvj2OHDkCnU6Hd955B2fOnEFqair27dtXFm0kIgeTr5Ow5Md0/LI3GwBQs4o73h+qQeVgTgBFVBGVOizUq1cP58+fx6JFi+Dn54fs7Gz06tULo0aNQlhYWFm0kYgcyKUbOkxfnoyr8QYAQL+OfhjS3R9KN946TVRRPVAXZbVajUmTJtm6LUTkwIQQWBebjS/WpUFvADRqBSZEadC0tqe9m0ZEZeyBwkJ+fj5OnjyJxMRESJJkse65556zScOIyHFkZBsx95tU7D9lmgDq8XqeeGegBv5+nACKyBWUOixs2bIFgwYNQnJycpF1MpkMRqPRJg0jIsdwLM40AVRKhhFKN+DV5wPwfDtfjthK5EJK3WX59ddfR58+fXD79m1IkmTxYFAgqjiMRoHlv6TjrU8TkZJhRJUQN3z+Tih6Pe3HoEDkYkp9ZiEhIQHjxo1DSEhIWbSHiBxAfIoBM6KTcebvOyaA6hMALw/eEknkikodFl544QXExsbikUceKYv2EJGd7TmWi4+/TUF2noCPpwzjXgrE08187N0sIrKjUoeFRYsWoU+fPvj9999Rv359KJWW91W/8cYbNmscEZWfAp2Ez+8YO6FOpDveH6pFmJbzOhC5ulL/Fli9ejW2bdsGT09PxMbGWly7lMlkDAtETujyLR0++ioFV27rIZMB/TqqMKS7Gm4K9k0gogcIC5MmTcK0adMwYcIEyOW8fknkzIQQ+HVvNhb/mA6dXiBQJcfEwVqOnUBEFkodFnQ6Hfr27cugQOTksnIlzF+Zgj3HTGMntHjME+9GaRDAsROI6C6lPuJHRUXhu+++K4u2EFE5OX2pAMNn3saeY3lwUwCv9fbHzJFBDApEVKxSn1kwGo2YO3cutm7digYNGhTp4LhgwQKbNY6IbMsoCazemomYjRmQJCA8yA0fDNWgVlVOL09E91bqsHDq1Ck0btwYAHD69GmLdQ8zUMvs2bMxceJEjBkzBgsXLnzg/RBR8ZLSDZgVk4Lj5wsAAB2ae2NMv0D4ePGSIhGVrNRhYdeuXTZvxOHDh/HFF1+gQYMGNt83EQH7T+VhztcpyMyR4Okhw9i+AXjmcV97N4uInITd/6TIzs7GgAEDsHTpUgQEBNi7OUQVik4vsOiHNExakoTMHAk1IpT4YkIogwIRlYpVZxZ69eqFmJgYqFQq9OrVq8Rt165dW6oGjBo1Cl27dkWHDh0wffr0ErctKChAQUGB+fvMzEwAgF6vh16vL9Xr3kvhfmy1P2JNbc3aet5INGDWijRcvGEAADzf1gdDn/ODuxt/FnfjZ9S2WE/bK4ualmZfVoUFtVpt7o+gVqsfrFXFWLNmDf78808cPnzYqu1nzZqFadOmFVm+bds2eHt726xdALB9+3ab7o9YU1u7Vz2FAM7eCkTs2crQGxXwUurRsf41VPHMxG/byrmRToafUdtiPW3PljXNzc21eluZEEJYs+GHH36It956y2YH5evXr6NZs2bYvn27ua9Cu3bt0KhRo3t2cCzuzEJERASSk5OhUqls0i69Xo/t27ejY8eORe70oAfDmtpWSfXMzZfw2fcZ2Hk0HwDQ8FF3vDvQHxo1b4ksCT+jtsV62l5Z1DQzMxNarRYZGRn3PYZa3cFx2rRpGDFihM3CwtGjR5GYmIgmTZqYlxmNRuzZsweLFi1CQUEBFArLX3AeHh7w8Ch6i5dSqbT5B7Is9unqWFPburuecVcL8NHyFNxKMkAuBwZ3VaN/JxUUcg7ZbC1+Rm2L9bQ9W9a0NPuxOixYeQLCau3bt8epU6cslg0ZMgS1a9fGu+++WyQoEFHxJEngx51ZWLYhHQYjEByowPtDtKj3CMdOICLbKNWtkw8zjsLd/Pz8UK9ePYtlPj4+0Gg0RZYTUfHSsoyYsyIFh/4yXXZo09gL4wdo4Odt9xudiKgCKVVYqFmz5n0DQ2pq6kM1iIis82dcAeZ9m47UTAnuShlG9wlA19Y+Ng31RERAKcPCtGnTbHo3xN1iY2PLbN9EFYXBKLDvfBiObkuFEEBkmBIfDNOgWri7vZtGRBVUqcJCv379EBwcXFZtIaL7SEg14MNlKTh7JRQA0P0pX4zs7Q8Pd152IKKyY3VY4KlNIvv642Qu5nydiqxcCe5uBrwzUIv/a26bW4aJiEpit7shiMg6eoPAsg3p+GFHFgCgZhUlnqh6Bk81irBzy4jIVVgdFiRJKst2EFEx4lMM+OirZJy9ogMAvPB/fhjc1Qfbt+ns3DIiciWlnnWSiMrHvhO5mPN1CrLzBHy9ZHh3kAatG3pzvH0iKncMC0QORm8Q+HJ9On7aabrsUCfSHR8M0yJUw/+uRGQf/O1D5EBuJxvw4VfJiLtquszQp70fXu7hD6UbOxgTkf0wLBA5iN+P52LuNynIyRPw85bjnUGBaN3AtrOpEhE9CIYFIjvT6QX+tzYN63dnAwAeq+aO94fysgMROQ7+NiKyoxuJenz4VTIuXjd1Wnyxg+myg5uClx2IyHEwLBDZyY7DOViwKhV5BQJqXzkmRGnQsq6XvZtFRFQEwwJROcsrkLDo+zRs3p8DAGhQwwOThmoQ5M//jkTkmPjbiagcXb6lw4dfpeDqbT1kMmDgsyoMfFYNBS87EJEDY1ggKgdCCPy6NxuLf0yHTi+gUSvw3mANGtfytHfTiIjui2GBqIxl5UqYvzIFe47lAQBaPOaJd6M0CPBT2LllRETWYVggKkOnLxVgenQyElONUMiB4T398cL/+UEu52UHInIeDAtEZcAoCazZlonoXzMgSUCY1g0fDNWgdqSHvZtGRFRqDAtENpaWZcTM6BQcPZcPAGjf3Btj+wXCx0tu55YRET0YhgUiGzpxPh/To1OQkmGEp7sMr/cNQOfHfSCT8bIDETkvhgUiG5AkgdXbMhH9SwYkAVQNdcOU4UGIDFPau2lERA+NYYHoIaVnGTFrRQoO/2W67PBMSx+M6RcALw9ediCiioFhgeghnLqYj4+WpyA53Qh3pQxj+gagcytediCiioVhgegBSJLAd79l4auf0yFJQESIG6a8rEX1Su72bhoRkc0xLBCVUlqWEXO/TsHBM6bLDh2ae+PN/oHw8uRlByKqmBgWiErhyNk8zF6RgtRMCUo34PUXA9G1NS87EFHFxrBAZAW9QWD5z+n47rcsAEBkmBLvD9XwsgMRuQSGBaL7uJmox/TlKYi7pgMAdH/KF6/19oenOy87EJFrYFggKsG2gzn475pU5BUI+HnL8dZ/AvFUI297N4uIqFwxLBAVIydPwn/XpOK3w7kAgAY1PPDeYA2CA/lfhohcD3/zEd3l7OUCTI9Owe1kA+RyIKqrGi91UkHBmSKJyEUxLBD9Q5IE1mw3DdlslICQQAUmDdGi3iOcKZKIXBvDAhGA5HQDZq1IwbG4AgDA001NYyf4erMTIxERwwK5vD9O5mLuN6nIzJFMM0W+yCGbiYjuxLBALkunF/jf2jSs350NAKgRocT7Q7WoEsKZIomI7sSwQC7pym09pn+VjL9v6QEAfdr7Ydhz/nBX8mwCEdHdGBbIpQgh8OvebHz+YzoK9AIBfnK8O0iDFnW97N00IiKHxbBALiMzx4j5K1Px+/E8AEDzxzzx7iANAlUKO7eMiMixMSyQSzhxIR8zo1OQlG6EmwJ4uYc/Xvg/P8g5dgIR0X0xLFCFZjQKfL0pAyu3ZEISQOVgN7w/VIuaVTgBFBGRtRgWqMKKTzFgRnQyzvxtmgCqcysfvN4nAF6eHDuBiKg0GBaoQoo9moP5q1KRkyfg4ynDmy8F4v+a+di7WURETolhgSqUvAIJi39Iw6Y/cgAAj1Vzx6QhWoRp+VEnInpQ/A1KFcaF6zpMX56M6wkGyGTAS51UiOqqhpuCnRiJiB4GwwI5PSEE1u7Kwpfr06E3AFp/BSZGadC4lqe9m0ZEVCEwLJBTy8g2Yu43qdh/yjR2QusGXnjrP4FQ+3LsBCIiW2FYIKd16mI+pi83jZ2gdANe6x2AHm18OQEUEZGNMSyQ0zFKAqu3ZiJmYwYkyTR2wuRhWtSI4NgJRERlgWGBnEpqhhEzY5LxZ1wBAKBjC2+M7RfIsROIiMoQwwI5jSNn8zArJgVpWRI83WUY0y8AnR73tXeziIgqPIYFcnhGo0DMrxlYtS0TQgDVKykxeZgWVUKV9m4aEZFLYFggh5aUZsD06BScumi67ND9SV+MfMEfHu687EBEVF4YFshhHTidh9krUpCZI8HbU4bxLwXiaQ7ZTERU7hgWyOEYjALLf07Hmu1ZAIBHI0yXHSoF87IDEZE9MCyQQ0lKN+Cjr1Jw+pLpskPPtr4Y0SsA7kqOnUBEZC8MC+Qwjp7Lx4zlyUjPluDjKcPbAzVo09jb3s0iInJ5DAvkENZsy8TSDekQAqhRWYkpw7WoFMTLDkREjoBhgexuw27TJFAA0LW1D0b3CeDdDkREDoRhgexq8/5sfPp9GgBgcDc1BnVR27lFRER0N7v++TZr1iw0b94cfn5+CA4ORs+ePREXF2fPJlE50ekFFqxKxbxvUiGEafyEgc+q7N0sIiIqhl3Dwu7duzFq1CgcOHAA27dvh16vxzPPPIOcnBx7NovK2O1kA17/OB6/7s2GTAYM6qLCG/0COFskEZGDsutliC1btlh8HxMTg+DgYBw9ehRt2rSxU6uoLP1xMhezV6QgO09A5SPHpCEaNH/My97NIiKiEjhUn4WMjAwAQGBgYLHrCwoKUFBQYP4+MzMTAKDX66HX623ShsL92Gp/ZKqlJAFLN6Tjx515AIDaVZWYNCQAwQEK1rqU+Bm1PdbUtlhP2yuLmpZmXzIhhLDZKz8ESZLw3HPPIT09HXv37i12m6lTp2LatGlFlq9atQre3rwf31HlFLhhy8lI3Ej1AwA0rJKIp2rdgkLuEB89IiKXlJubi5deegkZGRlQqUruM+YwYeG1117D5s2bsXfvXlSuXLnYbYo7sxAREYHk5OT7vlFr6fV6bN++HR07doRSyfv8H9apSzrMjElDaqYET3fgzf7+aNeElx0eBj+jtsea2hbraXtlUdPMzExotVqrwoJDXIYYPXo0fv31V+zZs+eeQQEAPDw84OHhUWS5Uqm0+QeyLPbpSoQQ+GGHafwESQICffIw540IPBLBM0C2ws+o7bGmtsV62p4ta1qa/dg1LAgh8Prrr2PdunWIjY1FtWrV7NkcspHsPAlzv07B3hOm/glPN/VEbf8TqBJa3c4tIyKiB2HXsDBq1CisWrUKGzZsgJ+fH+Lj4wEAarUaXl48Ve2MLt3QYerSZNxMMkDpBox8IQDPPu6BzZslezeNiIgekF3DwpIlSwAA7dq1s1geHR2NwYMHl3+D6KFs3p+N/65Jg04vEByowNSXtagd6cEe0URETs7ulyHI+RXoJHz2fRo2/WEaTKtFXU9MjNJA7auwc8uIiMgWHKKDIzmvm0l6TFuajIs39JDJTPM7DOikglzO0RiJiCoKhgV6YPtO5GL21ynIyRPw95Vj0lAtmtb2tHeziIjIxhgWqNSMRoGvfk7Hmu1ZAIC61d0xeZgWQQH8OBERVUT87U6lkpJhxEdfJePkRdPgWL3/zw+vPu8PNwUvOxARVVQMC2S1E+fz8eHyZKRlSvD2lOHt/2jQtgkHWSIiqugYFui+hBBYsz0LX21IhySAauFKTB2uRUQIR2YjInIFDAtUouxcCbO/TsEfJ02jMXZs4Y2x/QPh5SG3c8uIiKi8MCzQPV24bhqN8XayaTTG118MRNfWPpDJ2D+BiMiVMCxQEUIIbPojB59+lwq9AQjVKDB1eBBqVnG3d9OIiMgOGBbIQl6+hE/WpOK3Q7kAgMfreWJClAYqH47GSETkqhgWyOzyLR2mLU3GtQQD5HJgaHc1+nXkaIxERK6OYYEAANsP5mDBqlQU6AU0agU+GKZBgxocjZGIiBgWXJ7eIPD5T2nYsDsbANCsjifeG6yBvx8vOxARkQnDggtLSjNg2rJk/HVZBwAY1EWFgV3UUPCyAxER3YFhwUUdi8vHR18lIz1bgq+XDO8N1uLx+l72bhYRETkghgUXI4TAd9uzsOyf0RhrVFZi6itBCNfyo0BERMXjEcKFZOdJmPt1CvaeMI3G2OlxH4ztFwAPd47GSERE98aw4CIu39JhypfJuJFoGo1xdJ8AdHvSl6MxEhHRfTEsuIAdh3Mwf2Uq8nUCwQEKTB2uRe1ID3s3i4iInATDQgWmNwh8sTYNa2NNt0U2re2J94dqoPblbZFERE5HCLu9NMNCBXUzUY+Plqfg/DXTbZEDOqswuBtviyQicihCAFlZQEICEB9v+bhjmVt8PLolJkLKyACUynJvJsNCBbTjcA4+WZ2K3HwBlY8c7wwMxBMNvO3dLCIi15Gff98AYH7k5d13dzIACgBSUhJQpUqZN/9uDAsVSF6BhEXfp2Hz/hwAQP0aHnh/iAZBAfwxExE9NIMBSEqyLgBkZJRu335+QEgIEBpa9BESAoNWix1nzuD/goPL5r3dB48iFcTfN3X46KtkXI03QCYDBj6rwsBn1VAoeNmBiOiehABSU+9/8I+PB5KTS9dvwMOjxABg8bWPT8nN1OuRn5AAKOzT54xhwckJIbBxXw4W/ZAG3T+TQL03WIPGtTgJFBG5KCGA7GzrAkBiIqDXW79vuRwIDi754F/4UKuBCnJ7OsOCE9PpBf67JtV82aFFXU9MGMRJoIiogrK2H0BCApCbW7p9BwZaFwA0Grv9dW9PDAtOKjHVgClLkxF3VQe5DBj6nBr9Oqog590ORORMDAbTX/fWhIDS9gPw9f33oB8Wdu9LAsHBgLt72by/CoJhwQmdOJ+PactMk0CpfOR4f6gGzepwEigichCSBPfMTOD0aSAlpfi//G3dD+DuZVb0AyDrMSw4ESEE1sVm4/Of0iBJwCOVlfjwlSCEcRIoIiprheMB3K8fQEIC3BIS8KzBYP2+5XLTwd2aEFCB+gE4Ex5lnES+TsKCVan47ZDpOlyH5t4YNyAQnpwEiogeRl5eyX/537nMivEAANOYAAAgNBrI7nXt/85lLtoPwJkwLDiB+BQDJn+ZhIvX9ZDLgRG9/NH7aT9OAkVExdPrrR8PIDOzdPv287tvJ0B9YCA2Hz2KZ3v0gNIOow2S7TEsOLjDf+VhRnQKMnMk+PvKMfllLRrV5G2RRC5HkoqOB3CvAJCSUvp+APe7C6BwubcVo8Hq9RAnTz74eyWHw7DgoCRJYNXWTET/mgEhgFpV3TF1uBYhgfyREVUYxfUDuFcASEw03TlgLYWi6HgA97ocwH4AdB888jig7FwJs79OwR8nTdcHuz3pi9F9AuCu5H9mIqdQBv0AzDQa6wIA+wGQDTEsOJjLt3SY/EUybiYZoHQDxvYLxLNP+Nq7WUQEmEYFvHEDuH7d9G9xIwImJDzYvADWXALgeABkJwwLDmTH4RzMX5mKfJ1AcKAC04ZrUauqh72bRVSx6fXAzZtQX7wI2aZNpvv+73UmICvL+v3e2Q+gpADA8QDICTAsOACDUeCLden4aafpF1HT2p54f6gGal+eQiR6IJJk6uRX0pwAheuSk6EE0M7afatUQOXKpkd4+L0vCbAfAFUgDAt2lpphxIdfJePkxQIAwIBOKgzuroaCwzYTWbpXZ8Di+gQkJABGo/W7ViiQr1bDs2pVyEoaFjg01BQWiFwMw4Idnfm7AFOXJiMlwwhvTxkmDNLgyUZW3JZEVJFY2xkwPt40kVBpaLUlH/j/WWdQqbBtyxZ06dKF4wIQFYNhwQ6EEPj592ws/iENBiNQNdQN014NQpUQ/pKiCsJgKH5QoOICwcN0BizproDgYMDaA39ppigmckEMC+Xs7mml2zT2wjsDNfD25LDN5ARycoDbt00H+Tv/vXtZYqJ9BwUiIptiWChHSekGTPkyGeeumKaVfrmHP/p25LDNZGeFnQGtCQGluRugpMmB7g4F7AxI5NAYFsrJ6UsFmLI0CWmZEvy8TdNKN3+M00pTGSoouP/B//Zt0+WA0owM6O0NhIWZDvJhYZZf37lMq+WgQEQVBMNCGRNCYP3ubCz5ydQ/oXq4EtNe1aJSEPsn0AMQAkhLg+/165Dt2mUaE+BeISAtrXT71mqtCwG+vjwLQORiGBbKUFauhHnfpGDvCdNwrm0ae+HdgRp4sX8C3c1g+LfDX0lnAuLjoczPR3tr9+vuXvwB/+5lISHWdwYkIpfDsFBGzl4uwEfLkxGfYoSbAnj1eX/04rTSric727q+AElJpeoQqPPxgbJyZcgKBwW6VwgICOBZACJ6aAwLNiZJAj/syMKyDekwSkCY1g2Th2k4bHNFIkn/nv6/XwjIzrZ+vwrFvx3+SjgToNdosHnXLo4JQETlhmHBhjKyjZjzdQoOnDYNHNOuiTfGDQiErxcvOziF/PyiB//iQkApRweEj8+9r//f+bW1swRyTAAiKmcMCzZy8mI+pi9PQXK6EUo3YHSfAHR70peXHeztnw6BVoWA9PTS7TsoyPoOgUREToxh4SEZJYFVWzKxYmMGJAFEhLhh8jAtHqnMaWTLlMFgGvjnzoP+nY87Q0FBgfX7LRwc6H4hoDSjAxIROTmGhYeQnG7AzJgUHD9vOhg909IHY/oG8G6Hh6HXmw7wN2+aHncf/AsfSUmmvgPWCgiwLgT4+7NDIBHRXRgWHtCB03mY83UKMrIleHrIMLZvAJ55nKeb76nwckByMnDr1r9hoPBx44bp34QE6+8KKBwhsLjr/3cuCw0FPD3L9v0REVVgDAulZJQEvlyXjh92mIa9rVFZiQ+GaRHhqpNACQFkZpoCwO3bpn8LH/9873brFrrduAGFTmfdPt3cgEqVgPBw0+NeISAoiCMEEhGVA4aFUvpqw79BoVc7X7zyfADclRX0tHVWVpEDf3FhALm5Je5GBsB8SPf3/zcEVK5sCgWFj8Lvg4JMZw2IiMghMCyUQtzVAqzZbgoKEwYFOu9lh8KBgoo78N/5fWnGCFCrTQEgLOzfMPDP94bgYOyKi0O7/v2hVKnK7n0REVGZYFiwkhACKzZmAAA6NPd2zKCQk2N54L/738JHaWYO9PMrcvAv8n1YmGksgXsQej1yMzIAL06cRUTkjBgWrGCUBL76OQMHTudDLgP+86y6fBuQl/dvR8CSwkBpQoCPj+VBv7gwwDECiIgIDAv3lZ5lxMyYFBw5axqVcVSfAFQJtVFnRqPRNFZAYQi4MxDc+XVpZg/09i76V39hZ8E7g4Cfn23eAxERVXgOERYWL16MefPmIT4+Hg0bNsRnn32GFi1a2LtZOH2pAB9+lYzkdCM8lDK82d/K2yOFMP2Vf6+Df+HX8fHWDxvs5WV5h8CdB/87//Xz4zgBRERkU3YPC9999x3GjRuH//3vf2jZsiUWLlyITp06IS4uDsHBwXZpkxDA2l3ZWPZzFoySaVTGqcO1qBbubuoXcO1a8aMGFl4OuHnTtJ015HLTbYDh4f/eFVDc12o1QwAREdmF3cPCggULMHz4cAwZMgQA8L///Q8bN27E8uXLMWHChHJvj14vYfuhUJxNN13/f1p2FuOPfQXvrhdMISE11fqdFd4hUFwIKPw+JMQ0rgAREZGDsutRSqfT4ejRo5g4caJ5mVwuR4cOHbB///4i2xcUFKDgjnH+MzMzAQB6vR56G8zEpzMIzBz6M84GtoSbUYeR+z5CjzNf4+6/54VKBYSHQ4SFASEhpn9DQyFCQoBKlUzfV6pU4h0C/+5MVPhZBAt/Nrb4GRHrWRZYU9tiPW2vLGpamn3ZNSwkJyfDaDQiJCTEYnlISAjOnTtXZPtZs2Zh2rRpRZZv27YN3t7eD90eSQAy6OFuyMe7xyfj0cBbuNC7N/KCgpCr1SIvKAh5Wi0MJYWAnBzg4kXTgyxs377d3k2oUFhP22NNbYv1tD1b1jT3PgPq3cmpzn9PnDgR48aNM3+fmZmJiIgIPPPMM1DZaLCfDo1u47tdp9Fy4XIoOaugTej1emzfvh0dO3ZkTW2A9bQ91tS2WE/bK4uaFp6dt4Zdw4JWq4VCoUBCQoLF8oSEBISGhhbZ3sPDAx4eHkWWK5VK230gK4UhKPCYbfdJAGz8cyLWswywprbFetqeLWtamv3YdQB+d3d3NG3aFDt27DAvkyQJO3bsQKtWrezYMiIiIipk98sQ48aNQ1RUFJo1a4YWLVpg4cKFyMnJMd8dQURERPZl97DQt29fJCUlYfLkyYiPj0ejRo2wZcuWIp0eiYiIyD7sHhYAYPTo0Rg9erS9m0FERETFsGufBSIiInJ8DAtERERUIoYFIiIiKhHDAhEREZWIYYGIiIhKxLBAREREJWJYICIiohIxLBAREVGJHGJQpgclhABQupmz7kev1yM3NxeZmZmcAMVGWFPbYj1tjzW1LdbT9sqipoXHzsJjaUmcOixkZWUBACIiIuzcEiIiIueUlZUFtVpd4jYyYU2kcFCSJOHWrVvw8/ODTCazyT4zMzMRERGB69evQ6VS2WSfro41tS3W0/ZYU9tiPW2vLGoqhEBWVhbCw8Mhl5fcK8GpzyzI5XJUrly5TPatUqn4Ibcx1tS2WE/bY01ti/W0PVvX9H5nFAqxgyMRERGViGGBiIiISsSwcBcPDw9MmTIFHh4e9m5KhcGa2hbraXusqW2xnrZn75o6dQdHIiIiKns8s0BEREQlYlggIiKiEjEsEBERUYkYFoiIiKhEDAt3Wbx4MSIjI+Hp6YmWLVvi0KFD9m6SQ9qzZw+6d++O8PBwyGQyrF+/3mK9EAKTJ09GWFgYvLy80KFDB1y4cMFim9TUVAwYMAAqlQr+/v4YNmwYsrOzy/FdOI5Zs2ahefPm8PPzQ3BwMHr27Im4uDiLbfLz8zFq1ChoNBr4+vqid+/eSEhIsNjm2rVr6Nq1K7y9vREcHIy3334bBoOhPN+Kw1iyZAkaNGhgHsSmVatW2Lx5s3k96/lwZs+eDZlMhrFjx5qXsaalM3XqVMhkMotH7dq1zesdqp6CzNasWSPc3d3F8uXLxZkzZ8Tw4cOFv7+/SEhIsHfTHM6mTZvEpEmTxNq1awUAsW7dOov1s2fPFmq1Wqxfv16cOHFCPPfcc6JatWoiLy/PvE3nzp1Fw4YNxYEDB8Tvv/8uatSoIfr371/O78QxdOrUSURHR4vTp0+L48ePiy5duogqVaqI7Oxs8zYjRowQERERYseOHeLIkSPi8ccfF0888YR5vcFgEPXq1RMdOnQQx44dE5s2bRJarVZMnDjRHm/J7n7++WexceNGcf78eREXFyfee+89oVQqxenTp4UQrOfDOHTokIiMjBQNGjQQY8aMMS9nTUtnypQpom7duuL27dvmR1JSknm9I9WTYeEOLVq0EKNGjTJ/bzQaRXh4uJg1a5YdW+X47g4LkiSJ0NBQMW/ePPOy9PR04eHhIVavXi2EEOKvv/4SAMThw4fN22zevFnIZDJx8+bNcmu7o0pMTBQAxO7du4UQpvoplUrxww8/mLc5e/asACD2798vhDAFOLlcLuLj483bLFmyRKhUKlFQUFC+b8BBBQQEiGXLlrGeDyErK0s8+uijYvv27aJt27bmsMCalt6UKVNEw4YNi13naPXkZYh/6HQ6HD16FB06dDAvk8vl6NChA/bv32/Hljmfy5cvIz4+3qKWarUaLVu2NNdy//798Pf3R7NmzczbdOjQAXK5HAcPHiz3NjuajIwMAEBgYCAA4OjRo9Dr9RY1rV27NqpUqWJR0/r16yMkJMS8TadOnZCZmYkzZ86UY+sdj9FoxJo1a5CTk4NWrVqxng9h1KhR6Nq1q0XtAH5GH9SFCxcQHh6O6tWrY8CAAbh27RoAx6unU08kZUvJyckwGo0WRQeAkJAQnDt3zk6tck7x8fEAUGwtC9fFx8cjODjYYr2bmxsCAwPN27gqSZIwduxYtG7dGvXq1QNgqpe7uzv8/f0ttr27psXVvHCdKzp16hRatWqF/Px8+Pr6Yt26dXjsscdw/Phx1vMBrFmzBn/++ScOHz5cZB0/o6XXsmVLxMTEoFatWrh9+zamTZuGp556CqdPn3a4ejIsEDmYUaNG4fTp09i7d6+9m+L0atWqhePHjyMjIwM//vgjoqKisHv3bns3yyldv34dY8aMwfbt2+Hp6Wnv5lQIzz77rPnrBg0aoGXLlqhatSq+//57eHl52bFlRfEyxD+0Wi0UCkWRnqYJCQkIDQ21U6ucU2G9SqplaGgoEhMTLdYbDAakpqa6dL1Hjx6NX3/9Fbt27bKYfj00NBQ6nQ7p6ekW299d0+JqXrjOFbm7u6NGjRpo2rQpZs2ahYYNG+K///0v6/kAjh49isTERDRp0gRubm5wc3PD7t278emnn8LNzQ0hISGs6UPy9/dHzZo1cfHiRYf7jDIs/MPd3R1NmzbFjh07zMskScKOHTvQqlUrO7bM+VSrVg2hoaEWtczMzMTBgwfNtWzVqhXS09Nx9OhR8zY7d+6EJElo2bJlubfZ3oQQGD16NNatW4edO3eiWrVqFuubNm0KpVJpUdO4uDhcu3bNoqanTp2yCGHbt2+HSqXCY489Vj5vxMFJkoSCggLW8wG0b98ep06dwvHjx82PZs2aYcCAAeavWdOHk52djUuXLiEsLMzxPqM27S7p5NasWSM8PDxETEyM+Ouvv8Qrr7wi/P39LXqakklWVpY4duyYOHbsmAAgFixYII4dOyauXr0qhDDdOunv7y82bNggTp48KXr06FHsrZONGzcWBw8eFHv37hWPPvqoy946+dprrwm1Wi1iY2MtbqPKzc01bzNixAhRpUoVsXPnTnHkyBHRqlUr0apVK/P6wtuonnnmGXH8+HGxZcsWERQU5LK3pU2YMEHs3r1bXL58WZw8eVJMmDBByGQysW3bNiEE62kLd94NIQRrWlrjx48XsbGx4vLly2Lfvn2iQ4cOQqvVisTERCGEY9WTYeEun332mahSpYpwd3cXLVq0EAcOHLB3kxzSrl27BIAij6ioKCGE6fbJDz74QISEhAgPDw/Rvn17ERcXZ7GPlJQU0b9/f+Hr6ytUKpUYMmSIyMrKssO7sb/iaglAREdHm7fJy8sTI0eOFAEBAcLb21s8//zz4vbt2xb7uXLlinj22WeFl5eX0Gq1Yvz48UKv15fzu3EMQ4cOFVWrVhXu7u4iKChItG/f3hwUhGA9beHusMCalk7fvn1FWFiYcHd3F5UqVRJ9+/YVFy9eNK93pHpyimoiIiIqEfssEBERUYkYFoiIiKhEDAtERERUIoYFIiIiKhHDAhEREZWIYYGIiIhKxLBAREREJWJYICIiohIxLBCRw7ty5QpkMhmOHz9u76YQuSSGBSInlZSUBHd3d+Tk5ECv18PHxwfXrl2zd7Me2uDBg9GzZ0+LZREREbh9+zbq1atnn0YRuTiGBSIntX//fjRs2BA+Pj74888/ERgYiCpVqti7WWVCoVAgNDQUbm5u9m4KkUtiWCByUn/88Qdat24NANi7d6/56/tZtmwZ6tSpA09PT9SuXRuff/65ed3QoUPRoEEDFBQUAAB0Oh0aN26MQYMGmbfZsGEDmjRpAk9PT1SvXh3Tpk2DwWAwr09PT8err76KkJAQeHp6ol69evj1118BAFOnTkWjRo0s2rNw4UJERkaa169YsQIbNmyATCaDTCZDbGxssZchdu/ejRYtWsDDwwNhYWGYMGGCRTvatWuHN954A++88w4CAwMRGhqKqVOnmtcLITB16lRUqVIFHh4eCA8PxxtvvGFVDYlcjs2npiKiMnP16lWhVquFWq0WSqVSeHp6CrVaLdzd3YWHh4dQq9Xitddeu+fzv/32WxEWFiZ++ukn8ffff4uffvpJBAYGipiYGCGEaerx6tWri7FjxwohhHjrrbdEZGSkyMjIEEIIsWfPHqFSqURMTIy4dOmS2LZtm4iMjBRTp04VQghhNBrF448/LurWrSu2bdsmLl26JH755RexadMmIYQQU6ZMEQ0bNrRo0yeffCKqVq1qfv0XX3xRdO7c2TxNd0FBgbh8+bIAII4dOyaEEOLGjRvC29tbjBw5Upw9e1asW7dOaLVaMWXKFPN+27ZtK1QqlZg6dao4f/68WLFihcUU1T/88INQqVRi06ZN4urVq+LgwYPiyy+/fKifD1FFxbBA5ET0er24fPmyOHHihFAqleLEiRPi4sWLwtfXV+zevVtcvnxZJCUl3fP5jzzyiFi1apXFso8++ki0atXK/P0ff/whlEql+OCDD4Sbm5v4/fffzevat28vZs6cafH8b775RoSFhQkhhNi6dauQy+VFpiMvdL+wIIQQUVFRokePHhbb3B0W3nvvPVGrVi0hSZJ5m8WLFwtfX19hNBqFEKaw8OSTT1rsp3nz5uLdd98VQggxf/58UbNmTaHT6YptKxH9i5chiJyIm5sbIiMjce7cOTRv3hwNGjRAfHw8QkJC0KZNG0RGRkKr1Rb73JycHFy6dAnDhg2Dr6+v+TF9+nRcunTJvF2rVq3w1ltv4aOPPsL48ePx5JNPmtedOHECH374ocXzhw8fjtu3byM3NxfHjx9H5cqVUbNmzTKtw9mzZ9GqVSvIZDLzstatWyM7Oxs3btwwL2vQoIHF88LCwpCYmAgA6NOnD/Ly8lC9enUMHz4c69ats7iMQUT/Ym8hIidSt25dXL16FXq9HpIkwdfXFwaDAQaDAb6+vqhatSrOnDlT7HOzs7MBAEuXLkXLli0t1ikUCvPXkiRh3759UCgUuHjxYpF9TJs2Db169Sqyf09PT3h5eZXYfrlcDiGExTK9Xl/icx6GUqm0+F4mk0GSJACmOyzi4uLw22+/Yfv27Rg5ciTmzZuH3bt3F3kekatjWCByIps2bYJer0f79u0xd+5cNG3aFP369cPgwYPRuXPnEg9yISEhCA8Px99//40BAwbcc7t58+bh3Llz2L17Nzp16oTo6GgMGTIEANCkSRPExcWhRo0axT63QYMGuHHjBs6fP1/s2YWgoCDEx8dDCGE+K3D32Anu7u4wGo0l1qFOnTr46aefLPazb98++Pn5oXLlyiU+905eXl7o3r07unfvjlGjRqF27do4deoUmjRpYvU+iFwBwwKRE6latSri4+ORkJCAHj16QCaT4cyZM+jduzfCwsLu+/xp06bhjTfegFqtRufOnVFQUIAjR44gLS0N48aNw7FjxzB58mT8+OOPaN26NRYsWIAxY8agbdu2qF69OiZPnoxu3bqhSpUqeOGFFyCXy3HixAmcPn0a06dPR9u2bdGmTRv07t0bCxYsQI0aNXDu3DnIZDJ07twZ7dq1Q1JSEubOnYsXXngBW7ZswebNm6FSqcxtjIyMxNatWxEXFweNRgO1Wl3kfYwcORILFy7E66+/jtGjRyMuLg5TpkzBuHHjIJdbd3U1JiYGRqMRLVu2hLe3N7799lt4eXmhatWq1v9AiFyFnftMEFEprV692txxb8+ePaJGjRqlev7KlStFo0aNhLu7uwgICBBt2rQRa9euFXl5eeKxxx4Tr7zyisX2zz33nHjiiSeEwWAQQgixZcsW8cQTTwgvLy+hUqlEixYtLO4iSElJEUOGDBEajUZ4enqKevXqiV9//dW8fsmSJSIiIkL4+PiIQYMGiRkzZlh0cExMTBQdO3YUvr6+AoDYtWtXkQ6OQggRGxsrmjdvLtzd3UVoaKh49913hV6vN69v27atGDNmjMV76dGjh4iKihJCCLFu3TrRsmVLoVKphI+Pj3j88cfFb7/9VqpaErkKmRB3XUAkIiIiugPvhiAiIqISMSwQERFRiRgWiIiIqEQMC0RERFQihgUiIiIqEcMCERERlYhhgYiIiErEsEBEREQlYlggIiKiEjEsEBERUYkYFoiIiKhE/w/8n5lh+4oMvAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.figure(figsize=(6, 6*6/8))\n", + "plt.plot(np.add.accumulate(np_times), color=\"red\", label=\"Numpy backend\")\n", + "plt.plot(np.add.accumulate(tf_times), color=\"royalblue\", label=\"TensorFlow backend\")\n", + "plt.legend()\n", + "plt.xlabel(\"# executions\")\n", + "plt.ylabel(\"Time [s]\")\n", + "plt.grid(True)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "3e96cb14-5896-4b60-bd90-138906b2c121", + "metadata": {}, + "source": [ + "This behaviour is expected, since TensorFlow is typically slower than the Numpy backend in executing circuits. It is clear, here, that setting the backend engine to `numpy`, even setting the `frontend` as `tensorflow`, is more convenient." + ] + }, + { + "cell_type": "markdown", + "id": "23608465-be0b-4a78-8733-cfad381b1cf7", + "metadata": {}, + "source": [ + "### Optimization example\n", + "\n", + "In the following, we optimize the parameters of the parametric circuit to minimize a target cost function, which is the expectation value of a target hamiltonian. We will repeat this three times, using different configuation of the `qiboml` custom differentiation. Let us describe each of these scenarios with the triad `(frontend, differentiation rule, execution backend)`:\n", + "\n", + "1. `(tf, psr, tf)`: we are working with the TensorFlow interface, we use the parameter shift rule as differentiation algorithm and we execute the circuits using the Qibo's `tensorflow` backend;\n", + "2. `(tf, psr, np)`: we are working with the TensorFlow interface, we use the parameter shift rule as differentiation algorithm and we execute the circuits using the Qibo's `numpy` backend;\n", + "3. `(tf, tf, tf)`: we are working with the TensorFlow interface, we use the tensorflow automatic differentiation algorithm and we execute the circuits using the Qibo's `tensorflow` backend.\n", + "\n", + "Note that these are only three of the many possible combinations. One has to pay attention to the problem configuration, e.g. considering the parameter shift algorithm when executing circuits on real hardware (`qibolab` backend), or activating the shot-noise. In fact, the automatic differentiation of tensorflow breaks when shot-noise is activated.\n", + "\n", + "We make use of the following function, through which we can customize the training." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "6c4b84d3-aa09-4536-988e-d3ca02a60dcd", + "metadata": {}, + "outputs": [], + "source": [ + "def train_circuit(circuit, hamiltonian, nepochs, exec_backend, differentiation_rule=None, nshots=None):\n", + " \"\"\"\n", + " Perform a simple gradient descent training of ``circuit`` to minimize the expectation \n", + " value of ``hamiltonian``. Gradients are computed via the chosen ``differentiation_rule``\n", + " and expectation values calculated executing circuit on the selected ``exec_backend``.\n", + "\n", + " Returns: \n", + " float: total execution time.\n", + " \"\"\"\n", + " # a couple of hyper-parameters\n", + " learning_rate = 0.01\n", + " random_seed = 42\n", + "\n", + " # random parameters\n", + " np.random.seed(random_seed)\n", + " nparams = len(circuit.get_parameters())\n", + " params = tf.Variable(np.random.uniform(0, 2*np.pi, nparams))\n", + "\n", + " it = time.time()\n", + "\n", + " for epoch in range(nepochs):\n", + " with tf.GradientTape() as tape:\n", + " circuit.set_parameters(params)\n", + " cost = expectation.expectation(\n", + " observable=hamiltonian,\n", + " circuit=circuit,\n", + " backend=exec_backend,\n", + " differentiation_rule=differentiation_rule,\n", + " nshots=nshots,\n", + " )\n", + " if (epoch % 10 == 0):\n", + " print(f\"Cost: {round(cost, 4)} \\t |\\t Epoch: {epoch}\")\n", + " gradients = tape.gradient(cost, params, )\n", + " init_params = params.assign_sub(learning_rate * gradients)\n", + " ft = time.time()\n", + " \n", + " return (ft - it) " + ] + }, + { + "cell_type": "markdown", + "id": "63950898-fd94-422b-a9cf-b9222700c331", + "metadata": {}, + "source": [ + "#### Training scenario 1: `(tf, psr, tf)`" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "26bd1eeb-4479-45c6-859b-21b9dca97c2c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cost: -0.1995 \t |\t Epoch: 0\n", + "Cost: -0.5636 \t |\t Epoch: 10\n", + "Cost: -0.8084 \t |\t Epoch: 20\n", + "Cost: -0.9856 \t |\t Epoch: 30\n", + "Cost: -1.1273 \t |\t Epoch: 40\n", + "Cost: -1.2492 \t |\t Epoch: 50\n", + "Cost: -1.3579 \t |\t Epoch: 60\n", + "Cost: -1.4559 \t |\t Epoch: 70\n", + "Cost: -1.5443 \t |\t Epoch: 80\n", + "Cost: -1.624 \t |\t Epoch: 90\n", + "Execution time with (tf, psr, tf): 170.2510015964508\n" + ] + } + ], + "source": [ + "nqubits = 3\n", + "nlayers = 5\n", + "\n", + "# setup the problem\n", + "circuit = build_parametric_circuit(nqubits, nlayers)\n", + "hamiltonian = hamiltonians.Z(nqubits)\n", + "\n", + "tf_psr_tf_time = train_circuit(\n", + " circuit=circuit,\n", + " hamiltonian=hamiltonian,\n", + " nepochs=100,\n", + " exec_backend=\"tensorflow\",\n", + " differentiation_rule=differentiation.parameter_shift\n", + ")\n", + "\n", + "print(f\"Execution time with (tf, psr, tf): {tf_psr_tf_time}\")" + ] + }, + { + "cell_type": "markdown", + "id": "08aba037-9dac-4b06-beba-0e3177138091", + "metadata": {}, + "source": [ + "#### Training scenario 2: `(tf, psr, np)`" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "775d2bb6-2112-4a47-9c4e-e22938be0194", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cost: -0.1995 \t |\t Epoch: 0\n", + "Cost: -0.5636 \t |\t Epoch: 10\n", + "Cost: -0.8084 \t |\t Epoch: 20\n", + "Cost: -0.9856 \t |\t Epoch: 30\n", + "Cost: -1.1273 \t |\t Epoch: 40\n", + "Cost: -1.2492 \t |\t Epoch: 50\n", + "Cost: -1.3579 \t |\t Epoch: 60\n", + "Cost: -1.4559 \t |\t Epoch: 70\n", + "Cost: -1.5443 \t |\t Epoch: 80\n", + "Cost: -1.624 \t |\t Epoch: 90\n", + "Execution time with (tf, psr, np): 13.80636191368103\n" + ] + } + ], + "source": [ + "tf_psr_np_time = train_circuit(\n", + " circuit=circuit,\n", + " hamiltonian=hamiltonian,\n", + " nepochs=100,\n", + " exec_backend=\"numpy\",\n", + " differentiation_rule=differentiation.parameter_shift\n", + ")\n", + "\n", + "print(f\"Execution time with (tf, psr, np): {tf_psr_np_time}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c24cfb3b-d9e2-4aaf-a590-63712e9d9afb", + "metadata": {}, + "source": [ + "#### Training scenario 3: `(tf, tf, tf)`" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "20e27788-e721-4a66-a606-fd7d8f5eda34", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cost: -0.1995 \t |\t Epoch: 0\n", + "Cost: -0.5636 \t |\t Epoch: 10\n", + "Cost: -0.8084 \t |\t Epoch: 20\n", + "Cost: -0.9856 \t |\t Epoch: 30\n", + "Cost: -1.1273 \t |\t Epoch: 40\n", + "Cost: -1.2492 \t |\t Epoch: 50\n", + "Cost: -1.3579 \t |\t Epoch: 60\n", + "Cost: -1.4559 \t |\t Epoch: 70\n", + "Cost: -1.5443 \t |\t Epoch: 80\n", + "Cost: -1.624 \t |\t Epoch: 90\n", + "Execution time with (tf, tf, tf): 8.758509635925293\n" + ] + } + ], + "source": [ + "tf_tf_tf_time = train_circuit(\n", + " circuit=circuit,\n", + " hamiltonian=hamiltonian,\n", + " nepochs=100,\n", + " exec_backend=\"tensorflow\",\n", + " differentiation_rule=None\n", + ")\n", + "\n", + "print(f\"Execution time with (tf, tf, tf): {tf_tf_tf_time}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "2a905fa8-7b67-444c-b953-5a7ac3703b75", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(tf, psr, tf): 170.2510015964508\n", + "(tf, psr, np): 13.80636191368103\n", + "(tf, tf, tf): 8.758509635925293\n" + ] + } + ], + "source": [ + "print(f\"(tf, psr, tf): {tf_psr_tf_time}\")\n", + "print(f\"(tf, psr, np): {tf_psr_np_time}\")\n", + "print(f\"(tf, tf, tf): {tf_tf_tf_time}\")" + ] + }, + { + "cell_type": "markdown", + "id": "79c62b66-ac75-4cad-97af-d45851cffe1b", + "metadata": {}, + "source": [ + "It is clear in this setup the last configuration is the fastest. This can be explained considering TensorFlow's automatic differentiation routines, which are well optimized to compute the backpropagation algorithm. But if we activate the shot-noise the discussion changes. In fact, the TensorFlow automatic differentiation is not usable in a shot-noise setup, and one of the other options has to be set. In that case, using the `numpy` (or the `) backend" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/figures/customdiff.svg b/tutorials/figures/customdiff.svg new file mode 100644 index 0000000..d8af700 --- /dev/null +++ b/tutorials/figures/customdiff.svg @@ -0,0 +1,12 @@ +
Diff. rule
Backend
Numpy
Qibojit
Pytorch
Tensorflow
Qibolab
Hardware
QML
Fast/lightweight
Frontend
Tensorflow
Pytorch
JAX
@tf.custom_gradient
qiboml.expectation