diff --git a/contributor.md b/contributor.md index 250a1852f..6abc900ff 100644 --- a/contributor.md +++ b/contributor.md @@ -27,5 +27,5 @@ | [AITM](models/multitask/aitm/) | [renmada](https://github.com/renmada) | https://github.com/PaddlePaddle/PaddleRec/pull/756 | 论文复现赛第六期 | | [IPRec](models/rank/iprec/) | [renmada](https://github.com/renmada) | https://github.com/PaddlePaddle/PaddleRec/pull/774 | 论文复现赛第六期 | | [KIM](models/match/kim/) | [renmada](https://github.com/renmada) | https://github.com/PaddlePaddle/PaddleRec/pull/790 | 论文复现赛第六期 | - + | [AutoInt](models/rank/autoint/) | [kafaichan](https://github.com/kafaichan) | https://github.com/PaddlePaddle/PaddleRec/pull/830 | 论文复现赛第七期 | diff --git a/datasets/criteo_autoint/convert2txt.py b/datasets/criteo_autoint/convert2txt.py new file mode 100644 index 000000000..e8108403b --- /dev/null +++ b/datasets/criteo_autoint/convert2txt.py @@ -0,0 +1,57 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import numpy as np +import argparse + +parser = argparse.ArgumentParser('convert npy to txt file') +parser.add_argument( + '--root_dir', + type=str, + default='./Criteo', + required=False, + help='root directory of src data') +args = parser.parse_args() + + +def write_to_file(output_folder, file_path_list): + fmt_str = ['%d'] + ['%d'] * 39 + ['%.7f'] * 39 + for folder in file_path_list: + if not os.path.isdir(folder): continue + print("begin {}".format(folder)) + feature_index = np.load( + os.path.join(folder, 'train_i.npy'), + allow_pickle=True).astype('int64') + feature_value = np.load( + os.path.join(folder, 'train_x2.npy'), + allow_pickle=True).astype('float32') + label = np.load( + os.path.join(folder, 'train_y.npy'), + allow_pickle=True).astype('int64').reshape([-1, 1]) + data = np.concatenate((label, feature_index, feature_value), axis=1) + np.savetxt( + os.path.join(output_folder, os.path.basename(folder)), + data, + fmt=' '.join(fmt_str)) + print("complete {}".format(folder)) + + +if __name__ == '__main__': + train_folders = [ + os.path.join(args.root_dir, 'part{}'.format(i)) for i in range(3, 11) + ] + test_folders = [os.path.join(args.root_dir, 'part1')] + write_to_file('./slot_test_data_full', test_folders) + write_to_file('./slot_train_data_full', train_folders) diff --git a/datasets/criteo_autoint/download.sh b/datasets/criteo_autoint/download.sh new file mode 100644 index 000000000..acb3ad712 --- /dev/null +++ b/datasets/criteo_autoint/download.sh @@ -0,0 +1,11 @@ +wget --no-check-certificate https://fleet.bj.bcebos.com/ctr_data.tar.gz + +tar -zxvf ctr_data.tar.gz + +mkdir ./tmp +mv raw_data tmp +mv test_data tmp + +find ./tmp -type f -name 'part*' -exec cat {} \; > criteo.data +rm -rf ./tmp +echo "Complete data download." diff --git a/datasets/criteo_autoint/preprocess.py b/datasets/criteo_autoint/preprocess.py new file mode 100644 index 000000000..8ff61eeb5 --- /dev/null +++ b/datasets/criteo_autoint/preprocess.py @@ -0,0 +1,125 @@ +#Copyright (c) 2018 Chence Shi + +#Permission is hereby granted, free of charge, to any person obtaining a copy +#of this software and associated documentation files (the "Software"), to deal +#in the Software without restriction, including without limitation the rights +#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the Software is +#furnished to do so, subject to the following conditions: + +#The above copyright notice and this permission notice shall be included in all +#copies or substantial portions of the Software. + +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +#SOFTWARE. + +import argparse +import os + +parser = argparse.ArgumentParser(description='criteo dataset preprocesser') +parser.add_argument( + '--source_data', + type=str, + required=True, + default='./criteo.txt', + help='source path') +parser.add_argument( + '--output_path', + type=str, + required=True, + default='./Criteo', + help='output path') +args = parser.parse_args() + +if not os.path.exists(args.output_path): + os.mkdir(args.output_path) + +train_path = args.source_data +f1 = open(train_path, 'r') +dic = {} +# generate three fold. +# train_x: value +# train_i: index +# train_y: label +f_train_value = open(os.path.join(args.output_path, 'train_x.txt'), 'w') +f_train_index = open(os.path.join(args.output_path, 'train_i.txt'), 'w') +f_train_label = open(os.path.join(args.output_path, 'train_y.txt'), 'w') + +for i in range(39): + dic[i] = {} + +cnt_train = 0 + +#for debug +#limits = 10000 +index = [1] * 26 +for line in f1: + cnt_train += 1 + if cnt_train % 100000 == 0: + print('now train cnt : %d\n' % cnt_train) + #if cnt_train > limits: + # break + split = line.strip('\n').split('\t') + # 0-label, 1-13 numerical, 14-39 category + for i in range(13, 39): + #dic_len = len(dic[i]) + if split[i + 1] not in dic[i]: + # [1, 0] 1 is the index for those whose appear times <= 10 0 indicates the appear times + dic[i][split[i + 1]] = [1, 0] + dic[i][split[i + 1]][1] += 1 + if dic[i][split[i + 1]][0] == 1 and dic[i][split[i + 1]][1] > 10: + index[i - 13] += 1 + dic[i][split[i + 1]][0] = index[i - 13] +f1.close() +print('total entries :%d\n' % (cnt_train - 1)) + +# calculate number of category features of every dimension +kinds = [13] +for i in range(13, 39): + kinds.append(index[i - 13]) +print('number of dimensions : %d' % (len(kinds) - 1)) +print(kinds) + +for i in range(1, len(kinds)): + kinds[i] += kinds[i - 1] +print(kinds) + +# make new data + +f1 = open(train_path, 'r') +cnt_train = 0 +print('remake training data...\n') +for line in f1: + cnt_train += 1 + if cnt_train % 100000 == 0: + print('now train cnt : %d\n' % cnt_train) + #if cnt_train > limits: + # break + entry = ['0'] * 39 + index = [None] * 39 + split = line.strip('\n').split('\t') + label = str(split[0]) + for i in range(13): + if split[i + 1] != '': + entry[i] = (split[i + 1]) + index[i] = (i + 1) + for i in range(13, 39): + if split[i + 1] != '': + entry[i] = '1' + index[i] = (dic[i][split[i + 1]][0]) + for j in range(26): + index[13 + j] += kinds[j] + index = [str(item) for item in index] + f_train_value.write(' '.join(entry) + '\n') + f_train_index.write(' '.join(index) + '\n') + f_train_label.write(label + '\n') +f1.close() + +f_train_value.close() +f_train_index.close() +f_train_label.close() diff --git a/datasets/criteo_autoint/run.sh b/datasets/criteo_autoint/run.sh new file mode 100644 index 000000000..dc6f950c5 --- /dev/null +++ b/datasets/criteo_autoint/run.sh @@ -0,0 +1,8 @@ +sh download.sh +mkdir slot_train_data_full +mkdir slot_test_data_full + +python preprocess.py --source_data ./criteo.data --output_path=./Criteo +python stratifiedKfold.py +python scale.py +python convert2txt.py diff --git a/datasets/criteo_autoint/scale.py b/datasets/criteo_autoint/scale.py new file mode 100644 index 000000000..d63d0b1dd --- /dev/null +++ b/datasets/criteo_autoint/scale.py @@ -0,0 +1,58 @@ +#Copyright (c) 2018 Chence Shi + +#Permission is hereby granted, free of charge, to any person obtaining a copy +#of this software and associated documentation files (the "Software"), to deal +#in the Software without restriction, including without limitation the rights +#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the Software is +#furnished to do so, subject to the following conditions: + +#The above copyright notice and this permission notice shall be included in all +#copies or substantial portions of the Software. + +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +#SOFTWARE. + +import math +import numpy as np +import argparse +import os + +parser = argparse.ArgumentParser(description='criteo dataset scale') +parser.add_argument( + '--src_path', + type=str, + required=False, + default='./Criteo', + help='source path') +args = parser.parse_args() + + +def scale(x): + if x > 2: + x = int(math.log(float(x))**2) + return x + + +def scale_each_fold(): + for i in range(1, 11): + print('now part %d' % i) + data = np.load( + os.path.join(args.src_path, 'part' + str(i), 'train_x.npy'), + allow_pickle=True) + part = data[:, 0:13] + for j in range(part.shape[0]): + if j % 100000 == 0: + print(j) + part[j] = list(map(scale, part[j])) + np.save( + os.path.join(args.src_path, 'part' + str(i), 'train_x2.npy'), data) + + +if __name__ == '__main__': + scale_each_fold() diff --git a/datasets/criteo_autoint/stratifiedKfold.py b/datasets/criteo_autoint/stratifiedKfold.py new file mode 100644 index 000000000..2ab4ccbbb --- /dev/null +++ b/datasets/criteo_autoint/stratifiedKfold.py @@ -0,0 +1,158 @@ +#Copyright (c) 2018 Chence Shi + +#Permission is hereby granted, free of charge, to any person obtaining a copy +#of this software and associated documentation files (the "Software"), to deal +#in the Software without restriction, including without limitation the rights +#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the Software is +#furnished to do so, subject to the following conditions: + +#The above copyright notice and this permission notice shall be included in all +#copies or substantial portions of the Software. + +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +#SOFTWARE. + +#Email of the author: zjduan@pku.edu.cn + +import numpy as np +import os +import pandas as pd +from sklearn.model_selection import StratifiedKFold +from sklearn import preprocessing +import argparse + +parser = argparse.ArgumentParser(description='criteo dataset k-fold') +parser.add_argument( + '--src_path', + type=str, + required=False, + default='./Criteo', + help='source path') +parser.add_argument( + '--num_split', + type=int, + required=False, + default=10, + help='number of K-fold split') +parser.add_argument( + '--random_seed', + type=int, + required=False, + default=2018, + help='random seed') +args = parser.parse_args() + +scale = "" +train_x_name = "train_x.npy" +train_y_name = "train_y.npy" + + +def _load_data(_nrows=None, debug=False): + + train_x = pd.read_csv( + os.path.join(args.src_path, 'train_x.txt'), + header=None, + sep=' ', + nrows=_nrows, + dtype=np.float) + train_y = pd.read_csv( + os.path.join(args.src_path, 'train_y.txt'), + header=None, + sep=' ', + nrows=_nrows, + dtype=np.int32) + + train_x = train_x.values + train_y = train_y.values.reshape([-1]) + + print('data loading done!') + print('training data : %d' % train_y.shape[0]) + + assert train_x.shape[0] == train_y.shape[0] + + return train_x, train_y + + +def save_x_y(fold_index, train_x, train_y): + _get = lambda x, l: [x[i] for i in l] + for i in range(len(fold_index)): + print("now part %d" % (i + 1)) + part_index = fold_index[i] + Xv_train_, y_train_ = _get(train_x, part_index), _get(train_y, + part_index) + save_dir_Xv = os.path.join(args.src_path, 'part{}'.format(i + 1)) + save_dir_y = os.path.join(args.src_path, 'part{}'.format(i + 1)) + if (os.path.exists(save_dir_Xv) == False): + os.makedirs(save_dir_Xv) + if (os.path.exists(save_dir_y) == False): + os.makedirs(save_dir_y) + save_path_Xv = os.path.join(save_dir_Xv, train_x_name) + save_path_y = os.path.join(save_dir_y, train_y_name) + np.save(save_path_Xv, Xv_train_) + np.save(save_path_y, y_train_) + + +def save_i(fold_index): + _get = lambda x, l: [x[i] for i in l] + train_i = pd.read_csv( + os.path.join(args.src_path, 'train_i.txt'), + header=None, + sep=' ', + nrows=None, + dtype=np.int32) + train_i = train_i.values + feature_size = train_i.max() + 1 + print("feature_size = %d" % feature_size) + feature_size = [feature_size] + feature_size = np.array(feature_size) + np.save(os.path.join(args.src_path, "feature_size.npy"), feature_size) + + print("train_i size: %d" % len(train_i)) + + for i in range(len(fold_index)): + print("now part %d" % (i + 1)) + part_index = fold_index[i] + Xi_train_ = _get(train_i, part_index) + save_path_Xi = os.path.join(args.src_path, "part" + str(i + 1), + 'train_i.npy') + np.save(save_path_Xi, Xi_train_) + + +def main(): + train_x, train_y = _load_data() + print('loading data done!') + + folds = list( + StratifiedKFold( + n_splits=10, shuffle=True, random_state=args.random_seed).split( + train_x, train_y)) + + fold_index = [] + for i, (train_id, valid_id) in enumerate(folds): + fold_index.append(valid_id) + + print("fold num: %d" % (len(fold_index))) + + fold_index = np.array(fold_index) + np.save( + os.path.join(args.src_path, 'fold_index.npy'), + fold_index, + allow_pickle=True) + + save_x_y(fold_index, train_x, train_y) + print("save train_x_y done!") + + fold_index = np.load( + os.path.join(args.src_path, "fold_index.npy"), allow_pickle=True) + save_i(fold_index) + print("save index done!") + + +if __name__ == "__main__": + main() diff --git a/models/rank/autoint/__init__.py b/models/rank/autoint/__init__.py new file mode 100644 index 000000000..abf198b97 --- /dev/null +++ b/models/rank/autoint/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/rank/autoint/config.yaml b/models/rank/autoint/config.yaml new file mode 100644 index 000000000..8faf6e5b8 --- /dev/null +++ b/models/rank/autoint/config.yaml @@ -0,0 +1,54 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +runner: + train_data_dir: "data/sample_data/train" + train_reader_path: "criteo_reader" # importlib format + use_gpu: True + use_xpu: False # Enable this option only if you have an xpu device + use_auc: True + train_batch_size: 2 + epochs: 3 + print_interval: 2 + # model_init_path: "output_model_deepfm/2" # init model + model_save_path: "output_model_autoint" + test_data_dir: "data/sample_data/train" + infer_reader_path: "criteo_reader" # importlib format + infer_batch_size: 2 + infer_load_path: "output_model_autoint" + infer_start_epoch: 0 + infer_end_epoch: 3 + # use fleet + use_fleet: False + +# hyper parameters of user-defined network +hyper_parameters: + # optimizer config + optimizer: + class: Adam + learning_rate: 0.001 + strategy: async + # user-defined pairs + feature_number: 1000001 + embedding_dim: 16 + fc_sizes: [] #[256, 128] + use_residual: True + scaling: True + use_wide: True + use_sparse: True + head_num: 2 + num_field: 39 + attn_layer_sizes: [64,64,64] + distributed_embedding: 0 diff --git a/models/rank/autoint/config_bigdata.yaml b/models/rank/autoint/config_bigdata.yaml new file mode 100644 index 000000000..2475b0d20 --- /dev/null +++ b/models/rank/autoint/config_bigdata.yaml @@ -0,0 +1,53 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# global settings + +runner: + train_data_dir: "../../../datasets/criteo_autoint/slot_train_data_full" + train_reader_path: "criteo_reader" # importlib format + use_gpu: True + use_xpu: False # Enable this option only if you have an xpu device + use_auc: False + train_batch_size: 1024 + epochs: 3 + print_interval: 10 + #model_init_path: "output_model_all_autoint/0" # init model + model_save_path: "output_model_all_autoint" + test_data_dir: "../../../datasets/criteo_autoint/slot_test_data_full" + infer_reader_path: "criteo_reader" # importlib format + infer_batch_size: 1024 + infer_load_path: "output_model_all_autoint" + infer_start_epoch: 0 + infer_end_epoch: 3 + +# hyper parameters of user-defined network +hyper_parameters: + # optimizer config + optimizer: + class: Adam + learning_rate: 0.001 + strategy: async + # user-defined pairs + feature_number: 998961 + embedding_dim: 16 + fc_sizes: [400,400,400] + use_residual: True + scaling: True + use_wide: False + use_sparse: True + head_num: 2 + num_field: 39 + attn_layer_sizes: [64,64,64] + distributed_embedding: 0 diff --git a/models/rank/autoint/criteo_reader.py b/models/rank/autoint/criteo_reader.py new file mode 100644 index 000000000..f9eeca114 --- /dev/null +++ b/models/rank/autoint/criteo_reader.py @@ -0,0 +1,60 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import numpy as np +import paddle +import os +from paddle.io import IterableDataset + + +class RecDataset(IterableDataset): + def __init__(self, file_list, config): + super(RecDataset, self).__init__() + self.file_list = file_list + if config: + use_fleet = config.get("runner.use_fleet", False) + self.inference = config.get("runner.inference", False) + else: + use_fleet = False + if use_fleet: + worker_id = paddle.distributed.get_rank() + worker_num = paddle.distributed.get_world_size() + file_num = len(file_list) + if file_num < worker_num: + raise ValueError( + "The number of data files is less than the number of workers" + ) + blocksize = int(file_num / worker_num) + self.file_list = file_list[worker_id * blocksize:(worker_id + 1) * + blocksize] + remainder = file_num - (blocksize * worker_num) + if worker_id < remainder: + self.file_list.append(file_list[-(worker_id + 1)]) + self.file_name = ['train_i.npy', 'train_x2.npy', 'train_y.npy'] + + def __iter__(self): + for file in self.file_list: + with open(file, 'r') as rf: + for l in rf: + line = l.strip().split(' ') + + output_list = [] + output_list.append(np.array([line[0]]).astype('int64')) + output_list.append(np.array(line[1:40]).astype('int64')) + output_list.append(np.array(line[40:]).astype('float32')) + if self.inference: + yield output_list[1:] + else: + yield output_list diff --git a/models/rank/autoint/data/sample_data/train/sample_train.txt b/models/rank/autoint/data/sample_data/train/sample_train.txt new file mode 100644 index 000000000..7ab4949df --- /dev/null +++ b/models/rank/autoint/data/sample_data/train/sample_train.txt @@ -0,0 +1,100 @@ +0 1 2 3 4 5 6 7 8 9 10 11 12 13 41 1466 2018 177793 306302 306607 306816 318556 319185 319189 360699 365579 540646 543583 543880 554870 720072 720201 724686 726703 726714 899031 899049 899064 955520 955606 2.0000000 0.0000000 1.0000000 6.0000000 44.0000000 20.0000000 1.0000000 2.0000000 30.0000000 1.0000000 1.0000000 1.0000000 14.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1484 2322 177971 306302 306607 307702 318556 319186 319189 360550 365722 540553 543584 544151 555028 720075 720466 724687 726705 726856 899030 899049 899157 955523 955673 0.0000000 2.0000000 41.0000000 0.0000000 119.0000000 0.0000000 0.0000000 2.0000000 17.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 27 1458 2011 177805 306302 306610 306647 318558 319185 319400 360429 365572 540429 543583 545048 554880 720070 720137 724686 726703 726723 899030 899048 899068 955520 955606 0.0000000 -1.0000000 0.0000000 0.0000000 53.0000000 0.0000000 8.0000000 0.0000000 1.0000000 0.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1458 4122 179610 306302 306609 307129 318557 319185 319199 360470 367714 540420 543584 545237 556947 720073 720584 724686 726703 728722 899030 899049 899065 955520 955606 0.0000000 0.0000000 6.0000000 3.0000000 78.0000000 26.0000000 2.0000000 2.0000000 3.0000000 0.0000000 1.0000000 0.0000000 3.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1458 2269 178117 306303 306607 306964 318556 319185 327056 360605 365840 540607 543583 544684 555160 720074 720649 724686 726703 726982 899030 899049 899372 955520 955606 0.0000000 8.0000000 2.0000000 2.0000000 78.0000000 26.0000000 2.0000000 1.0000000 10.0000000 0.0000000 2.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 48 1852 2024 177804 306305 306608 311170 318556 319185 319189 363135 365586 541825 543585 545549 554879 720072 721052 724687 726705 726722 899030 899052 899064 955542 956892 0.0000000 3.0000000 2.0000000 1.0000000 62.0000000 0.0000000 0.0000000 1.0000000 6.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 20 1468 4352 180637 306302 306607 306797 318564 319186 319249 360426 367944 540410 543583 543789 557387 720070 720171 724687 726706 729144 899030 899049 901529 955522 957467 0.0000000 1.0000000 2.0000000 0.0000000 146.0000000 0.0000000 0.0000000 1.0000000 2.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1531 4565 180216 306303 306607 307105 318571 319185 319189 361158 368156 540741 543585 543877 556991 720070 720168 724687 726705 728767 899030 899048 901220 955523 956513 13.0000000 4.0000000 13.0000000 12.0000000 10.0000000 12.0000000 13.0000000 10.0000000 12.0000000 1.0000000 1.0000000 0.0000000 12.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 21 1458 2013 177794 306302 306607 306633 318563 319186 319189 360424 365574 540413 543583 544457 554865 720078 720347 724686 726703 726709 899031 899048 899066 955520 955606 0.0000000 24.0000000 2.0000000 2.0000000 0.0000000 0.0000000 0.0000000 2.0000000 2.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 34 1458 2021 177801 306304 306609 306731 318556 319185 319485 360976 365583 540569 543584 543893 554876 720072 720123 724686 726703 726719 899030 899048 899076 955520 955606 2.0000000 0.0000000 0.0000000 3.0000000 37.0000000 13.0000000 3.0000000 12.0000000 31.0000000 1.0000000 1.0000000 3.0000000 3.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 17 1462 2028 177811 306302 306607 306849 318556 319185 321067 360692 365591 540695 543586 543714 554883 720070 720134 724698 726705 726727 899031 899048 899084 955522 955619 6.0000000 6.0000000 19.0000000 12.0000000 35.0000000 22.0000000 7.0000000 13.0000000 16.0000000 1.0000000 2.0000000 0.0000000 14.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 21 1459 2016 177797 306303 306608 307470 318557 319185 319297 360648 365577 540653 543586 543617 554868 720073 720093 724689 726705 726712 899030 899048 899065 955521 955609 0.0000000 1.0000000 18.0000000 9.0000000 75.0000000 27.0000000 2.0000000 9.0000000 17.0000000 0.0000000 1.0000000 0.0000000 9.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1488 2011 177795 306302 306607 307407 318556 319185 320051 361142 365572 541094 543583 543757 554863 720071 720207 724690 726706 726707 899030 899050 899067 955524 955607 0.0000000 3.0000000 18.0000000 1.0000000 85.0000000 27.0000000 1.0000000 10.0000000 16.0000000 0.0000000 1.0000000 0.0000000 7.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1471 2011 301805 306302 306607 306629 318556 319185 319194 360413 365572 540408 543584 543619 554863 720076 720091 724688 726704 726707 899030 899048 899138 955522 962753 0.0000000 61.0000000 1.0000000 0.0000000 75.0000000 5.0000000 9.0000000 0.0000000 5.0000000 0.0000000 2.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1535 2500 178220 306302 306610 306628 318566 319185 319189 360567 366076 540415 543584 543948 555411 720073 720172 724686 726703 727221 899030 899049 899065 955520 955606 0.0000000 -1.0000000 0.0000000 0.0000000 79.0000000 11.0000000 1.0000000 0.0000000 11.0000000 0.0000000 2.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1474 57624 179062 306302 306607 306655 318556 319185 319249 360426 421385 540410 543584 543628 556732 720073 720105 724686 726703 780975 899030 899049 899071 955520 955606 0.0000000 8.0000000 7.0000000 4.0000000 70.0000000 0.0000000 0.0000000 4.0000000 4.0000000 0.0000000 0.0000000 0.0000000 4.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 20 1468 2075 177885 306303 306612 308974 318556 319185 319189 361191 365644 541136 543587 543695 554951 720070 720118 724687 726706 726787 899030 899048 899163 955522 956184 1.0000000 55.0000000 1.0000000 1.0000000 10.0000000 0.0000000 1.0000000 14.0000000 24.0000000 2.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 19 1460 2012 177793 306302 306607 306718 318557 319185 360129 360557 365573 540559 543586 543610 554864 720070 720082 724686 726703 726708 899030 899049 899064 955520 955606 0.0000000 -1.0000000 0.0000000 0.0000000 113.0000000 0.0000000 0.0000000 6.0000000 10.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1547 2193 177900 306304 306607 306902 318556 319185 319199 360466 365766 540422 543584 543872 555082 720071 720301 724686 726703 726907 899030 899049 899111 955520 955606 0.0000000 -1.0000000 0.0000000 0.0000000 56.0000000 32.0000000 1.0000000 0.0000000 9.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1711 2012 177793 306302 306607 306882 318556 319185 319189 360727 365573 540728 543584 545007 554864 720075 720575 724686 726703 726708 899030 899054 899064 955520 955606 0.0000000 1.0000000 1.0000000 3.0000000 112.0000000 0.0000000 0.0000000 4.0000000 31.0000000 0.0000000 0.0000000 0.0000000 4.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1459 2011 181408 306303 306607 310860 318556 319185 332398 360968 365572 540945 543585 544164 554863 720070 720206 724732 726705 726707 899032 899048 899101 955521 955646 0.0000000 2.0000000 15.0000000 6.0000000 77.0000000 21.0000000 6.0000000 5.0000000 20.0000000 0.0000000 1.0000000 0.0000000 6.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1465 2011 177792 306303 306612 307145 318556 319185 319231 360579 365572 540580 543591 543785 554863 720070 720083 724688 726704 726707 899030 899048 943173 955541 990085 0.0000000 -1.0000000 10.0000000 0.0000000 91.0000000 36.0000000 3.0000000 2.0000000 15.0000000 0.0000000 2.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1490 2012 177793 306306 306609 306889 318556 319185 319192 361573 365573 540737 543584 544010 554864 720077 720289 724686 726703 726708 899033 899049 899064 955520 955606 0.0000000 2.0000000 1.0000000 8.0000000 63.0000000 37.0000000 3.0000000 15.0000000 21.0000000 0.0000000 1.0000000 1.0000000 8.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 21 1623 2013 177794 306304 306607 312404 318557 319185 319189 363002 365574 542574 543583 546256 554865 720075 720317 724686 726703 726709 899031 899048 899066 955520 955606 0.0000000 25.0000000 0.0000000 2.0000000 48.0000000 0.0000000 0.0000000 6.0000000 12.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 20 1458 2210 177948 306302 306609 306635 318556 319186 319189 360425 365783 540410 543584 544140 555100 720078 720479 724686 726703 726925 899030 899053 899212 955520 955606 0.0000000 26.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1485 2018 177793 306302 306607 306661 318556 319185 319401 360422 365579 540423 543583 543673 554870 720073 720104 724686 726703 726714 899030 899049 899064 955520 955606 0.0000000 0.0000000 1.0000000 0.0000000 82.0000000 14.0000000 2.0000000 0.0000000 6.0000000 0.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 21 1488 2011 177792 306302 306608 311744 318556 319185 319284 361736 365572 541184 543583 543688 554863 720074 720116 724687 726706 726707 899030 899049 899070 955522 955608 0.0000000 7.0000000 1.0000000 1.0000000 75.0000000 14.0000000 2.0000000 2.0000000 12.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1504 2013 177794 306302 306608 306645 318556 319185 319189 360443 365574 540441 543585 543871 554865 720070 720128 724687 726706 726709 899030 899048 899066 955521 974597 0.0000000 2.0000000 4.0000000 6.0000000 87.0000000 14.0000000 5.0000000 8.0000000 14.0000000 0.0000000 1.0000000 0.0000000 8.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 23 1463 2015 177796 306304 306607 307161 318571 319185 319385 360643 365576 540648 543584 543627 554867 720070 720086 724686 726703 726711 899030 899049 899069 955520 955606 0.0000000 -1.0000000 1.0000000 0.0000000 72.0000000 29.0000000 3.0000000 4.0000000 54.0000000 0.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1475 2120 177807 306303 306610 306755 318556 319185 319843 360598 365605 540598 543585 543925 554886 720076 720338 724686 726703 726743 899030 899048 899079 955520 955606 0.0000000 8.0000000 0.0000000 1.0000000 64.0000000 0.0000000 0.0000000 0.0000000 17.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 17 1460 2012 177793 306303 306607 316356 318557 319185 319189 363878 365573 543143 543585 543683 554864 720070 720164 724686 726703 726708 899032 899049 899064 955520 955606 0.0000000 -1.0000000 0.0000000 0.0000000 82.0000000 34.0000000 4.0000000 0.0000000 21.0000000 0.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1458 2021 177801 306302 306607 308029 318556 319185 319189 360904 365583 540561 543584 543810 554876 720073 720152 724686 726703 726719 899031 899054 899076 955520 955606 0.0000000 3.0000000 6.0000000 0.0000000 57.0000000 43.0000000 1.0000000 10.0000000 25.0000000 0.0000000 1.0000000 0.0000000 7.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1549 2011 177798 306302 306607 306634 318556 319185 319189 360427 365572 540424 543584 544657 554890 720072 720189 724686 726703 726864 899031 899048 899068 955520 955606 1.0000000 14.0000000 1.0000000 1.0000000 1.0000000 2.0000000 5.0000000 6.0000000 14.0000000 1.0000000 1.0000000 5.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 141 1564 2071 177873 306302 306609 307182 318556 319185 319511 360619 365639 540623 543584 543913 554940 720070 720334 724686 726703 726783 899030 899048 899083 955520 955606 0.0000000 1.0000000 2.0000000 4.0000000 98.0000000 25.0000000 12.0000000 8.0000000 21.0000000 0.0000000 2.0000000 0.0000000 4.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 17 1522 6115 182875 306302 306607 307256 318556 319185 320308 360504 369748 540510 543584 543895 559326 720076 720181 724687 726706 730987 899030 899048 899849 955524 956209 0.0000000 20.0000000 9.0000000 1.0000000 72.0000000 0.0000000 0.0000000 8.0000000 26.0000000 0.0000000 0.0000000 0.0000000 8.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1472 2011 177981 306303 306610 306808 318556 319185 319769 360455 365572 540456 543583 543746 554863 720073 720151 724687 726705 726707 899033 899048 899204 955523 955641 0.0000000 1.0000000 8.0000000 2.0000000 78.0000000 26.0000000 1.0000000 2.0000000 19.0000000 0.0000000 1.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1485 2012 177793 306302 306610 306709 318556 319185 319197 360417 365573 540414 543584 543668 554864 720071 720104 724686 726703 726708 899030 899048 899064 955520 955606 0.0000000 11.0000000 0.0000000 0.0000000 112.0000000 44.0000000 0.0000000 2.0000000 19.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 17 1687 2011 180406 306302 306609 306663 318556 319185 321213 360429 365572 540429 543594 545038 554863 720072 720725 724686 726703 726707 899030 899049 899638 955520 955606 1.0000000 6.0000000 5.0000000 2.0000000 10.0000000 2.0000000 14.0000000 3.0000000 53.0000000 1.0000000 8.0000000 1.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 21 1491 2011 177798 306302 306607 311609 318556 319185 319974 361325 365587 541246 543584 543630 554873 720074 720109 724686 726703 726717 899030 899048 899068 955520 955606 0.0000000 0.0000000 5.0000000 5.0000000 82.0000000 31.0000000 2.0000000 5.0000000 44.0000000 0.0000000 1.0000000 0.0000000 5.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 22 1517 2699 178165 306302 306609 307007 318556 319185 319189 360726 366276 540717 543585 543854 555642 720070 720149 724687 726705 727432 899030 899054 899341 955535 955698 1.0000000 0.0000000 52.0000000 3.0000000 7.0000000 1.0000000 1.0000000 5.0000000 4.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1502 2087 177908 306302 306612 308331 318556 319185 319189 361845 365657 541690 543586 544187 554970 720073 720294 724687 726706 726802 899034 899048 899175 955521 956333 0.0000000 2.0000000 6.0000000 0.0000000 110.0000000 25.0000000 1.0000000 0.0000000 25.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1461 2011 178877 306302 306607 311996 318556 319185 335651 362737 365572 542391 543583 543614 560731 720074 720084 724687 726706 726707 899030 899048 900004 955521 956348 3.0000000 -1.0000000 0.0000000 0.0000000 35.0000000 5.0000000 23.0000000 5.0000000 26.0000000 1.0000000 6.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1479 2073 177883 306302 306608 306897 318556 319185 319189 360487 365642 540491 543585 544347 554949 720070 720199 724687 726705 726785 899030 899053 899160 955553 955697 0.0000000 4.0000000 5.0000000 6.0000000 53.0000000 14.0000000 1.0000000 14.0000000 21.0000000 0.0000000 1.0000000 0.0000000 6.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 22 1458 10420 187645 306302 306607 308330 318556 319185 319189 361637 374140 540984 543585 546693 563810 720070 721786 724686 726703 735458 899030 899049 899480 955520 955606 0.0000000 6.0000000 2.0000000 1.0000000 80.0000000 33.0000000 4.0000000 3.0000000 20.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1504 2014 177796 306306 306609 306793 318556 319185 319189 360570 365575 540572 543584 543748 554866 720075 720128 724687 726705 726710 899031 899054 899069 955521 955688 0.0000000 20.0000000 1.0000000 0.0000000 111.0000000 0.0000000 0.0000000 1.0000000 8.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1501 2011 177795 306302 306607 307447 318556 319185 319481 360458 365572 540461 543583 543644 554863 720070 720124 724688 726704 726707 899030 899050 899067 955521 955607 0.0000000 16.0000000 1.0000000 2.0000000 51.0000000 20.0000000 4.0000000 1.0000000 32.0000000 0.0000000 1.0000000 0.0000000 8.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 24 1460 2012 177793 306303 306610 307251 318556 319186 319189 361064 365573 540631 543586 543610 554864 720078 720082 724686 726703 726708 899031 899052 899064 955520 955606 0.0000000 -1.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 4.0000000 8.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 41 1458 2011 177792 306302 306607 307417 318556 319185 319199 360875 365572 540522 543584 543949 554863 720070 720360 724686 726703 726707 899030 899048 899073 955520 955606 0.0000000 7.0000000 1.0000000 12.0000000 54.0000000 25.0000000 11.0000000 9.0000000 51.0000000 0.0000000 4.0000000 0.0000000 18.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 17 1500 2011 177792 306302 306607 307009 318556 319185 325830 360842 365572 540842 543585 543850 554863 720072 720125 724687 726704 726707 899030 899048 899070 955521 955608 0.0000000 2.0000000 10.0000000 0.0000000 64.0000000 16.0000000 1.0000000 4.0000000 7.0000000 0.0000000 1.0000000 0.0000000 3.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 17 1484 6270 183061 306302 306613 306649 318566 319185 320493 360414 369910 540411 543584 544151 559483 720070 720466 724687 726705 731151 899030 899048 900023 955523 955673 5.0000000 1.0000000 5.0000000 6.0000000 23.0000000 7.0000000 6.0000000 8.0000000 30.0000000 1.0000000 2.0000000 0.0000000 7.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1460 2012 177793 306306 306610 311824 318557 319185 321509 362511 365573 542211 543586 543610 554864 720071 720082 724686 726703 726708 899030 899049 899064 955520 955606 0.0000000 0.0000000 3.0000000 2.0000000 95.0000000 0.0000000 0.0000000 2.0000000 0.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 21 1472 9990 187200 306302 306609 307473 318562 319185 319189 361169 373710 541116 543584 543739 563387 720074 720151 724687 726705 735025 899030 899048 905839 955523 955980 1.0000000 1.0000000 13.0000000 4.0000000 33.0000000 13.0000000 1.0000000 4.0000000 5.0000000 1.0000000 1.0000000 1.0000000 5.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1509 2029 177812 306305 306607 306721 318561 319185 319189 360554 365592 540556 543584 543881 554885 720075 720160 724686 726703 726728 899030 899054 899086 955520 955606 0.0000000 0.0000000 1.0000000 2.0000000 79.0000000 0.0000000 0.0000000 1.0000000 5.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 21 1458 2011 177792 306303 306610 306634 318556 319185 319189 360427 365572 540424 543589 544328 554863 720072 720491 724686 726703 726707 899030 899049 899327 955520 955606 0.0000000 25.0000000 1.0000000 1.0000000 60.0000000 8.0000000 3.0000000 7.0000000 16.0000000 0.0000000 1.0000000 3.0000000 6.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1461 2011 177797 306302 306609 307778 318556 319186 320752 360527 365572 540530 543583 543651 554884 720070 720089 724687 726706 726707 899030 899048 899065 955521 955610 0.0000000 0.0000000 20.0000000 3.0000000 187.0000000 0.0000000 0.0000000 3.0000000 20.0000000 0.0000000 0.0000000 0.0000000 3.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 19 1474 2429 177822 306303 306607 309847 318556 319185 319189 361592 366001 541479 543584 543628 554921 720073 720105 724686 726703 726890 899030 899048 899071 955520 955606 0.0000000 10.0000000 2.0000000 2.0000000 84.0000000 0.0000000 0.0000000 9.0000000 0.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1459 2016 177797 306302 306607 306921 318556 319185 320130 360770 365577 540775 543586 543617 554868 720070 720093 724689 726705 726712 899030 899048 899065 955521 955609 1.0000000 8.0000000 8.0000000 5.0000000 35.0000000 10.0000000 11.0000000 9.0000000 33.0000000 1.0000000 3.0000000 0.0000000 5.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1458 2011 178577 306306 306607 311688 318556 319185 319189 363420 365572 542869 543585 544760 554863 720074 720668 724686 726703 726707 899031 899049 899508 955520 955606 0.0000000 0.0000000 1.0000000 2.0000000 51.0000000 22.0000000 10.0000000 6.0000000 26.0000000 0.0000000 2.0000000 1.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 21 1715 19396 178611 306303 306607 307258 318563 319185 319189 361072 366220 541035 543585 550815 555574 720070 720587 724686 726703 727374 899030 899057 899087 955520 955606 0.0000000 -1.0000000 0.0000000 0.0000000 102.0000000 22.0000000 9.0000000 5.0000000 8.0000000 0.0000000 2.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1479 116510 269396 306305 306608 307375 318556 319185 319189 360526 479591 540529 543585 543703 665430 720070 720111 724687 726705 839709 899030 899054 909659 955525 955615 0.0000000 17.0000000 4.0000000 4.0000000 111.0000000 13.0000000 1.0000000 5.0000000 6.0000000 0.0000000 0.0000000 0.0000000 4.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1458 2011 177899 306302 306610 306642 318556 319186 319189 360440 365572 540426 543584 544546 554963 720078 720288 724686 726703 727171 899033 899054 899170 955520 955606 0.0000000 58.0000000 4.0000000 1.0000000 0.0000000 0.0000000 0.0000000 2.0000000 3.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1462 2011 177792 306302 306608 306872 318556 319185 319330 360553 365572 540555 543583 543624 554863 720070 720092 724709 726704 726707 899030 899048 904935 955522 960794 1.0000000 14.0000000 2.0000000 0.0000000 5.0000000 0.0000000 5.0000000 1.0000000 3.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 22 1489 2022 177802 306307 306608 307620 318556 319185 326773 360759 365584 540761 543583 543633 554877 720070 720106 724687 726705 726720 899030 899048 899065 955521 955611 1.0000000 -1.0000000 0.0000000 0.0000000 40.0000000 4.0000000 6.0000000 6.0000000 17.0000000 1.0000000 2.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1465 2686 178681 306302 306609 308537 318557 319185 323497 360635 366263 540642 543584 543623 555629 720073 720083 724687 726706 727418 899030 899053 899729 955528 956077 0.0000000 0.0000000 1.0000000 1.0000000 60.0000000 8.0000000 2.0000000 6.0000000 6.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1463 2012 177793 306302 306611 307302 318557 319185 330649 360764 365573 540769 543584 543658 554864 720076 720126 724686 726703 726708 899030 899054 899064 955520 955606 0.0000000 5.0000000 3.0000000 5.0000000 106.0000000 0.0000000 0.0000000 6.0000000 18.0000000 0.0000000 0.0000000 0.0000000 5.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1461 2020 177797 306302 306607 306630 318556 319185 319192 360483 365582 540421 543584 543647 554874 720070 720089 724687 726705 726718 899030 899049 899065 955521 955612 8.0000000 -1.0000000 2.0000000 1.0000000 36.0000000 11.0000000 8.0000000 13.0000000 19.0000000 2.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1560 2564 177860 306302 306609 306673 318556 319185 320209 360434 366140 540435 543583 544088 555482 720070 720218 724686 726703 727290 899030 899052 899136 955520 955606 7.0000000 1.0000000 5.0000000 4.0000000 52.0000000 8.0000000 13.0000000 8.0000000 24.0000000 0.0000000 1.0000000 0.0000000 7.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 17 1499 2195 178032 306302 306610 307370 318556 319185 319191 360875 365768 540522 543583 543715 555084 720070 720131 724691 726704 726909 899032 899049 899070 955531 955608 0.0000000 1.0000000 14.0000000 7.0000000 53.0000000 15.0000000 1.0000000 14.0000000 17.0000000 0.0000000 2.0000000 0.0000000 7.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1475 39456 178050 306302 306610 310130 318556 319185 319211 361330 366918 541251 543583 544137 555827 720074 720231 724686 726703 728002 899031 899049 899108 955520 955606 6.0000000 5.0000000 2.0000000 4.0000000 49.0000000 5.0000000 7.0000000 5.0000000 11.0000000 0.0000000 2.0000000 1.0000000 4.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 57 1458 2021 177801 306305 306607 307472 318557 319185 319189 360520 365583 540525 543584 544771 554876 720071 720152 724686 726703 726719 899031 899054 899076 955520 955606 1.0000000 62.0000000 1.0000000 0.0000000 2.0000000 0.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1498 2016 177797 306304 306607 306628 318568 319185 319189 360815 365577 540415 543583 543783 554868 720071 720122 724687 726705 726712 899030 899048 899065 955521 955648 1.0000000 2.0000000 1.0000000 3.0000000 46.0000000 11.0000000 1.0000000 8.0000000 21.0000000 1.0000000 2.0000000 0.0000000 5.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1478 2011 178579 306302 306608 306906 318556 319185 319189 360531 365572 540533 543584 543648 560114 720070 720097 724687 726704 726707 899030 899048 899489 955525 955908 0.0000000 16.0000000 5.0000000 7.0000000 67.0000000 10.0000000 2.0000000 4.0000000 10.0000000 0.0000000 1.0000000 0.0000000 7.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1466 2015 177796 306304 306607 306738 318556 319185 319191 360522 365576 540422 543584 543615 554867 720070 720087 724686 726703 726711 899030 899048 899069 955520 955606 0.0000000 1.0000000 2.0000000 2.0000000 10.0000000 0.0000000 0.0000000 2.0000000 2.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1477 2011 177792 306302 306612 309059 318556 319185 320934 361179 365572 541126 543583 543721 554863 720073 720096 724687 726706 726707 899030 899048 902078 955526 958026 0.0000000 5.0000000 1.0000000 0.0000000 91.0000000 22.0000000 1.0000000 0.0000000 8.0000000 0.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1498 2011 177792 306304 306607 307171 318556 319185 319334 360498 365572 540502 543585 543766 554863 720074 720122 724687 726705 726707 899030 899048 908053 955521 955615 0.0000000 0.0000000 4.0000000 2.0000000 43.0000000 0.0000000 0.0000000 4.0000000 5.0000000 0.0000000 0.0000000 1.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1522 3738 179976 306304 306610 313141 318557 319185 335898 363257 367333 542765 543583 543820 556769 720070 720165 724687 726706 728545 899032 899052 901020 955524 957079 3.0000000 17.0000000 14.0000000 7.0000000 3.0000000 5.0000000 5.0000000 10.0000000 15.0000000 1.0000000 1.0000000 0.0000000 5.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1482 2889 178946 306306 306612 307117 318561 319185 320761 360701 366470 540701 543589 543667 555843 720070 720098 724706 726704 727635 899030 899048 899467 955530 955890 1.0000000 0.0000000 0.0000000 0.0000000 27.0000000 4.0000000 2.0000000 2.0000000 7.0000000 1.0000000 2.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1568 2011 204128 306302 306607 308431 318556 319185 319189 360873 365572 540804 543583 543885 554863 720075 720211 724687 726705 726707 899030 899054 900320 955521 960990 2.0000000 1.0000000 20.0000000 11.0000000 31.0000000 11.0000000 2.0000000 11.0000000 11.0000000 1.0000000 1.0000000 0.0000000 11.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1538 12919 190274 306320 306609 307539 318560 319185 321693 360538 376655 540542 543583 543845 566409 720071 720185 724687 726706 738004 899030 899051 899070 955546 958409 0.0000000 23.0000000 24.0000000 0.0000000 77.0000000 31.0000000 1.0000000 0.0000000 21.0000000 0.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1476 2011 177799 306302 306609 309264 318557 319185 324476 362243 365580 542019 543584 543642 554871 720070 720095 724686 726703 726715 899031 899048 899068 955520 955606 0.0000000 -1.0000000 0.0000000 0.0000000 112.0000000 0.0000000 0.0000000 8.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1648 2012 177793 306302 306607 306800 318559 319185 319189 360417 365573 540414 543583 544080 554864 720075 720446 724686 726703 726708 899030 899049 899064 955520 955606 0.0000000 20.0000000 0.0000000 0.0000000 53.0000000 0.0000000 0.0000000 3.0000000 5.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 17 1603 2011 178122 306303 306607 306798 318556 319185 319189 360607 365572 540609 543584 544082 555245 720070 720276 724687 726706 726707 899030 899048 899388 955534 955800 0.0000000 1.0000000 2.0000000 2.0000000 112.0000000 56.0000000 0.0000000 2.0000000 47.0000000 0.0000000 0.0000000 0.0000000 3.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 36 1459 2011 177792 306302 306607 306630 318569 319185 319192 360691 365572 540421 543584 543818 554863 720072 720206 725713 726705 726707 899032 899049 899101 955521 955646 0.0000000 0.0000000 6.0000000 6.0000000 53.0000000 6.0000000 34.0000000 3.0000000 36.0000000 0.0000000 5.0000000 6.0000000 6.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 23 1713 3976 178192 306303 306610 310651 318563 319185 319986 361580 367422 541467 543597 544803 555404 720074 720645 724686 726703 727759 899030 899049 899449 955520 955606 0.0000000 0.0000000 0.0000000 3.0000000 95.0000000 26.0000000 2.0000000 1.0000000 28.0000000 0.0000000 1.0000000 0.0000000 3.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1476 2011 177799 306302 306609 306636 318556 319185 319433 360417 365580 540414 543584 543625 554871 720070 720095 724686 726703 726715 899030 899049 899068 955520 955606 0.0000000 7.0000000 4.0000000 4.0000000 51.0000000 40.0000000 2.0000000 14.0000000 40.0000000 0.0000000 1.0000000 0.0000000 4.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 17 1479 49480 222329 306302 306608 306669 318556 319185 319729 360462 413220 540464 543583 546440 602571 720070 720553 724687 726706 774427 899030 899048 923843 955525 955631 0.0000000 0.0000000 2.0000000 2.0000000 112.0000000 11.0000000 1.0000000 2.0000000 10.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1463 2015 177796 306302 306607 312175 318577 319185 319189 363098 365576 542653 543583 543639 554867 720075 720085 724686 726703 726711 899030 899051 899069 955520 955606 0.0000000 1.0000000 0.0000000 6.0000000 39.0000000 0.0000000 0.0000000 6.0000000 6.0000000 0.0000000 0.0000000 0.0000000 6.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1481 3639 179855 306302 306609 306673 318556 319185 319404 360434 367237 540435 543588 544528 556671 720070 720099 724724 726704 728446 899030 899050 899575 955526 955664 0.0000000 -1.0000000 12.0000000 9.0000000 53.0000000 18.0000000 1.0000000 14.0000000 26.0000000 0.0000000 2.0000000 0.0000000 12.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1465 2016 177797 306305 306607 306653 318556 319185 323350 360448 365577 540451 543583 543616 554868 720070 720083 724689 726705 726712 899030 899049 899065 955521 955609 1.0000000 0.0000000 14.0000000 7.0000000 7.0000000 5.0000000 2.0000000 6.0000000 30.0000000 1.0000000 1.0000000 0.0000000 5.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1478 2011 177792 306302 306607 306769 318562 319185 320412 360428 365572 540428 543583 543649 554863 720077 720097 724866 726706 726707 899030 899051 899063 955525 997441 0.0000000 1.0000000 7.0000000 5.0000000 107.0000000 16.0000000 0.0000000 9.0000000 12.0000000 0.0000000 0.0000000 0.0000000 9.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1460 2012 177793 306302 306610 306651 318558 319185 319189 360418 365573 540417 543586 543610 554864 720075 720082 724686 726703 726708 899032 899052 899064 955520 955606 0.0000000 8.0000000 6.0000000 2.0000000 87.0000000 0.0000000 0.0000000 2.0000000 10.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1463 2017 177796 306304 306607 311177 318557 319185 319189 363219 365578 542738 543585 543694 554869 720076 720085 724686 726703 726713 899030 899048 899069 955520 955606 0.0000000 6.0000000 0.0000000 2.0000000 88.0000000 0.0000000 0.0000000 6.0000000 1.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1458 2011 182385 306309 306607 306641 318556 319185 319275 360436 365572 540437 543588 546877 554863 720070 720985 724686 726703 726707 899031 899048 900103 955520 955606 0.0000000 13.0000000 1.0000000 2.0000000 79.0000000 30.0000000 4.0000000 4.0000000 19.0000000 0.0000000 1.0000000 0.0000000 4.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1458 2011 177792 306302 306613 306857 318557 319185 319327 360442 365572 540443 543584 543883 554863 720070 720315 724686 726703 726707 899031 899053 899073 955520 955606 0.0000000 48.0000000 0.0000000 1.0000000 45.0000000 5.0000000 23.0000000 5.0000000 15.0000000 0.0000000 3.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1773 2013 177794 306303 306607 306656 318557 319185 319204 360418 365574 540417 543587 545686 554865 720074 720764 724686 726703 726709 899032 899048 899066 955520 955606 1.0000000 24.0000000 0.0000000 2.0000000 1.0000000 0.0000000 8.0000000 2.0000000 16.0000000 1.0000000 3.0000000 0.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 16 1511 2011 180513 306303 306607 306638 318556 319185 320669 360419 365572 540418 543587 543906 554863 720070 720325 724686 726703 726707 899030 899050 899323 955520 955606 0.0000000 18.0000000 0.0000000 0.0000000 106.0000000 35.0000000 0.0000000 1.0000000 31.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1482 2011 177792 306302 306611 306653 318556 319185 327115 360448 365572 540451 543583 543666 554863 720070 720098 724688 726704 726707 899030 899050 914284 955521 955747 2.0000000 1.0000000 3.0000000 5.0000000 5.0000000 5.0000000 8.0000000 13.0000000 27.0000000 1.0000000 4.0000000 0.0000000 5.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 +0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1475 2151 177983 306302 306609 306755 318557 319185 319644 360598 365725 540598 543584 544636 555039 720070 720760 724686 726703 726867 899030 899048 899079 955520 955606 0.0000000 11.0000000 0.0000000 0.0000000 82.0000000 30.0000000 1.0000000 0.0000000 1.0000000 0.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1458 2013 177794 306302 306612 306679 318556 319185 319214 360435 365574 540436 543585 546747 554865 720070 721194 724686 726703 726709 899030 899048 899066 955520 955606 0.0000000 -1.0000000 0.0000000 0.0000000 105.0000000 14.0000000 2.0000000 0.0000000 1.0000000 0.0000000 2.0000000 0.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 0.0000000 0.0000000 1.0000000 0.0000000 0.0000000 0.0000000 +1 1 2 3 4 5 6 7 8 9 10 11 12 13 15 1483 2017 177796 306303 306609 306880 318557 319185 319370 360453 365578 540455 543584 543637 554869 720070 720113 724686 726703 726713 899030 899053 899069 955520 955606 0.0000000 2.0000000 0.0000000 0.0000000 112.0000000 0.0000000 0.0000000 2.0000000 15.0000000 0.0000000 0.0000000 0.0000000 2.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 1.0000000 0.0000000 0.0000000 1.0000000 0.0000000 1.0000000 1.0000000 0.0000000 0.0000000 diff --git a/models/rank/autoint/dygraph_model.py b/models/rank/autoint/dygraph_model.py new file mode 100644 index 000000000..5ec977fc6 --- /dev/null +++ b/models/rank/autoint/dygraph_model.py @@ -0,0 +1,93 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import paddle.nn as nn +import paddle.nn.functional as F +import math + +import net + + +class DygraphModel(): + # define model + def create_model(self, config): + feature_number = config.get("hyper_parameters.feature_number") + embedding_dim = config.get("hyper_parameters.embedding_dim") + fc_sizes = config.get("hyper_parameters.fc_sizes") + use_residual = config.get("hyper_parameters.use_residual") + scaling = config.get("hyper_parameters.scaling") + use_wide = config.get("hyper_parameters.use_wide") + use_sparse = config.get("hyper_parameters.use_sparse") + head_num = config.get("hyper_parameters.head_num") + num_field = config.get("hyper_parameters.num_field") + attn_layer_sizes = config.get("hyper_parameters.attn_layer_sizes") + + autoint_model = net.AutoInt( + feature_number, embedding_dim, fc_sizes, use_residual, scaling, + use_wide, use_sparse, head_num, num_field, attn_layer_sizes) + return autoint_model + + # define feeds which convert numpy of batch data to paddle.tensor + def create_feeds(self, batch_data, config): + label = paddle.to_tensor(batch_data[0], dtype='int64') + feat_index = paddle.to_tensor(batch_data[1], dtype='int64') + feat_value = paddle.to_tensor(batch_data[2], dtype='float32') + return label, feat_index, feat_value + + # define loss function by predicts and label + def create_loss(self, pred, label): + cost = paddle.nn.functional.log_loss( + input=pred, label=paddle.cast( + label, dtype="float32")) + avg_cost = paddle.mean(x=cost) + return avg_cost + + # define optimizer + def create_optimizer(self, dy_model, config): + lr = config.get("hyper_parameters.optimizer.learning_rate", 0.001) + optimizer = paddle.optimizer.Adam( + learning_rate=lr, parameters=dy_model.parameters()) + return optimizer + + # define metrics such as auc/acc + # multi-task need to define multi metric + def create_metrics(self): + metrics_list_name = ["auc"] + auc_metric = paddle.metric.Auc("ROC") + metrics_list = [auc_metric] + return metrics_list, metrics_list_name + + # construct train forward phase + def train_forward(self, dy_model, metrics_list, batch_data, config): + label, feat_index, feat_value = self.create_feeds(batch_data, config) + + pred = dy_model.forward(feat_index, feat_value) + loss = self.create_loss(pred, label) + # update metrics + predict_2d = paddle.concat(x=[1 - pred, pred], axis=1) + metrics_list[0].update(preds=predict_2d.numpy(), labels=label.numpy()) + + print_dict = {'loss': loss} + # print_dict = None + return loss, metrics_list, print_dict + + def infer_forward(self, dy_model, metrics_list, batch_data, config): + label, feat_index, feat_value = self.create_feeds(batch_data, config) + + pred = dy_model.forward(feat_index, feat_value) + # update metrics + predict_2d = paddle.concat(x=[1 - pred, pred], axis=1) + metrics_list[0].update(preds=predict_2d.numpy(), labels=label.numpy()) + return metrics_list, None diff --git a/models/rank/autoint/net.py b/models/rank/autoint/net.py new file mode 100644 index 000000000..187b7a574 --- /dev/null +++ b/models/rank/autoint/net.py @@ -0,0 +1,212 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import paddle.nn as nn +import paddle.nn.functional as F +import math + + +class InteractingLayer(nn.Layer): + def __init__(self, embedding_size, interact_layer_size, head_num, + use_residual, scaling): + super(InteractingLayer, self).__init__() + self.attn_emb_size = interact_layer_size // head_num + self.head_num = head_num + self.use_residual = use_residual + self.scaling = scaling + self.W_Query = paddle.create_parameter( + shape=[embedding_size, interact_layer_size], + dtype='float32', + default_initializer=paddle.nn.initializer.XavierUniform()) + self.W_Key = paddle.create_parameter( + shape=[embedding_size, interact_layer_size], + dtype='float32', + default_initializer=paddle.nn.initializer.XavierUniform()) + self.W_Value = paddle.create_parameter( + shape=[embedding_size, interact_layer_size], + dtype='float32', + default_initializer=paddle.nn.initializer.XavierUniform()) + + if self.use_residual: + self.W_Res = paddle.create_parameter( + shape=[embedding_size, interact_layer_size], + dtype='float32', + default_initializer=paddle.nn.initializer.XavierUniform()) + self.layer_norm = paddle.nn.LayerNorm( + interact_layer_size, epsilon=1e-08) + + def forward(self, inputs): + querys = F.relu(paddle.einsum('bnk,kj->bnj', inputs, self.W_Query)) + keys = F.relu(paddle.einsum('bnk,kj->bnj', inputs, self.W_Key)) + values = F.relu(paddle.einsum('bnk,kj->bnj', inputs, self.W_Value)) + + q = paddle.stack(paddle.split(querys, self.head_num, axis=2)) + k = paddle.stack(paddle.split(keys, self.head_num, axis=2)) + v = paddle.stack(paddle.split(values, self.head_num, axis=2)) + + inner_prod = paddle.einsum('bnik,bnjk->bnij', q, k) + if self.scaling: + inner_prod /= self.attn_emb_size**0.5 + self.normalized_attn_scores = F.softmax(inner_prod, axis=-1) + result = paddle.matmul(self.normalized_attn_scores, v) + + result = paddle.concat( + paddle.split(result, self.head_num), axis=-1).squeeze(axis=0) + if self.use_residual: + result += F.relu(paddle.einsum('bnk,kj->bnj', inputs, self.W_Res)) + result = F.relu(result) + result = self.layer_norm(result) + return result + + +class EmbeddingLayer(nn.Layer): + def __init__(self, feature_number, embedding_dim, num_field, fc_sizes, + use_wide, use_sparse): + super(EmbeddingLayer, self).__init__() + self.feature_number = feature_number + self.embedding_dim = embedding_dim + self.num_field = num_field + self.use_wide = use_wide + self.fc_sizes = fc_sizes + + self.feature_embeddings = paddle.nn.Embedding( + self.feature_number, + self.embedding_dim, + sparse=use_sparse, + weight_attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Normal(0, 0.01))) + if self.use_wide: + self.feature_bias = paddle.nn.Embedding( + self.feature_number, + 1, + sparse=use_sparse, + weight_attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Normal(0, 0.001))) + if len(self.fc_sizes) > 0: + self.dnn_layers = [] + linear = paddle.nn.Linear( + in_features=num_field * embedding_dim, + out_features=fc_sizes[0], + weight_attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Normal( + 0, + math.sqrt(2. / + (num_field * embedding_dim + fc_sizes[0])))), + bias_attr=paddle.nn.initializer.Normal( + 0, + math.sqrt(2. / (num_field * embedding_dim + fc_sizes[0])))) + self.add_sublayer('linear_0', linear) + self.add_sublayer('relu_0', paddle.nn.ReLU()) + self.dnn_layers.append(linear) + for i in range(1, len(fc_sizes)): + linear = paddle.nn.Linear( + in_features=fc_sizes[i - 1], + out_features=fc_sizes[i], + weight_attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Normal( + 0, math.sqrt(2. / + (fc_sizes[i - 1] + fc_sizes[i])))), + bias_attr=paddle.nn.initializer.Normal( + 0, math.sqrt(2. / (fc_sizes[i - 1] + fc_sizes[i])))) + self.add_sublayer('linear_%d' % i, linear) + self.dnn_layers.append(linear) + norm = paddle.nn.BatchNorm( + fc_sizes[i], + is_test=self.training, + momentum=0.99, + epsilon=0.001) + self.add_sublayer('norm_%d' % i, norm) + self.dnn_layers.append(norm) + act = paddle.nn.ReLU() + self.add_sublayer('relu_%d' % i, act) + self.dnn_layers.append(act) + self.add_sublayer('dropout_%d' % i, paddle.nn.Dropout()) + linear = paddle.nn.Linear( + in_features=fc_sizes[-1], + out_features=1, + weight_attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Normal( + 0, math.sqrt(2. / (fc_sizes[-1] + 1)))), + bias_attr=paddle.nn.initializer.Normal()) + self.add_sublayer('pred_dense', linear) + self.dnn_layers.append(linear) + + def forward(self, feat_index, feat_value): + emb = self.feature_embeddings(feat_index) + x = feat_value.reshape(shape=[-1, self.num_field, 1]) + emb = paddle.multiply(emb, x) + + if self.use_wide: + y_first_order = paddle.sum( + paddle.multiply(self.feature_bias(feat_index), x), axis=1) + else: + y_first_order = None + + if len(self.fc_sizes) > 0: + y_dense = emb.reshape( + shape=[-1, self.num_field * self.embedding_dim]) + for layer in self.dnn_layers: + y_dense = layer(y_dense) + else: + y_dense = None + return emb, y_first_order, y_dense + + +class AutoInt(nn.Layer): + def __init__(self, feature_number, embedding_dim, fc_sizes, use_residual, + scaling, use_wide, use_sparse, head_num, num_field, + attn_layer_sizes): + super(AutoInt, self).__init__() + self.embedding_dim = embedding_dim + self.num_field = num_field + self.output_size = attn_layer_sizes[-1] + self.use_wide = use_wide + self.fc_sizes = fc_sizes + self.emb_layer = EmbeddingLayer(feature_number, embedding_dim, + num_field, fc_sizes, use_wide, + use_sparse) + + self.attn_layer_sizes = [embedding_dim] + attn_layer_sizes + self.iteraction_layers = nn.Sequential(* [ + InteractingLayer(self.attn_layer_sizes[i], self.attn_layer_sizes[ + i + 1], head_num, use_residual, scaling) for i in range( + len(self.attn_layer_sizes) - 1) + ]) + + self.linear = nn.Linear( + self.output_size * self.num_field, + 1, + weight_attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Normal( + 0, math.sqrt(2. / + (self.output_size * self.num_field + 1)))), + bias_attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Normal())) + + def forward(self, feat_index, feat_value): + out, y_first_order, y_dense = self.emb_layer(feat_index, feat_value) + + for layer in self.iteraction_layers: + out = layer(out) + + out = paddle.flatten(out, start_axis=1) + out = self.linear(out) + + if self.use_wide: + out += y_first_order + + if len(self.fc_sizes) > 0: + out += y_dense + return F.sigmoid(out) diff --git a/models/rank/autoint/picture/1.JPG b/models/rank/autoint/picture/1.JPG new file mode 100644 index 000000000..868a44eee Binary files /dev/null and b/models/rank/autoint/picture/1.JPG differ diff --git a/models/rank/autoint/picture/10.JPG b/models/rank/autoint/picture/10.JPG new file mode 100644 index 000000000..b2fd0b149 Binary files /dev/null and b/models/rank/autoint/picture/10.JPG differ diff --git a/models/rank/autoint/picture/2.JPG b/models/rank/autoint/picture/2.JPG new file mode 100644 index 000000000..4f56c4e3a Binary files /dev/null and b/models/rank/autoint/picture/2.JPG differ diff --git a/models/rank/autoint/picture/3.JPG b/models/rank/autoint/picture/3.JPG new file mode 100644 index 000000000..8c76d0b34 Binary files /dev/null and b/models/rank/autoint/picture/3.JPG differ diff --git a/models/rank/autoint/picture/4.JPG b/models/rank/autoint/picture/4.JPG new file mode 100644 index 000000000..ac3dba5b1 Binary files /dev/null and b/models/rank/autoint/picture/4.JPG differ diff --git a/models/rank/autoint/picture/5.JPG b/models/rank/autoint/picture/5.JPG new file mode 100644 index 000000000..1bb06a7a4 Binary files /dev/null and b/models/rank/autoint/picture/5.JPG differ diff --git a/models/rank/autoint/picture/6.JPG b/models/rank/autoint/picture/6.JPG new file mode 100644 index 000000000..701b45b72 Binary files /dev/null and b/models/rank/autoint/picture/6.JPG differ diff --git a/models/rank/autoint/picture/7.JPG b/models/rank/autoint/picture/7.JPG new file mode 100644 index 000000000..840875c14 Binary files /dev/null and b/models/rank/autoint/picture/7.JPG differ diff --git a/models/rank/autoint/picture/8.JPG b/models/rank/autoint/picture/8.JPG new file mode 100644 index 000000000..63bdf00cd Binary files /dev/null and b/models/rank/autoint/picture/8.JPG differ diff --git a/models/rank/autoint/picture/9.JPG b/models/rank/autoint/picture/9.JPG new file mode 100644 index 000000000..1f23d3bb8 Binary files /dev/null and b/models/rank/autoint/picture/9.JPG differ diff --git a/models/rank/autoint/readme.md b/models/rank/autoint/readme.md new file mode 100644 index 000000000..be2588ea6 --- /dev/null +++ b/models/rank/autoint/readme.md @@ -0,0 +1,162 @@ +# AutoInt模型 + +**[AI Studio在线运行环境](https://aistudio.baidu.com/aistudio/projectdetail/4434725)** + + +以下是本例的简要目录结构及说明： + +``` +├── data #样例数据 + ├── sample_data #样例数据 + ├── train + ├── sample_train.txt #训练数据样例 +├── __init__.py +├── README.md #文档 +├── config.yaml # sample数据配置 +├── config_bigdata.yaml # 全量数据配置 +├── net.py # 模型核心组网（动静统一） +├── criteo_reader.py #数据读取程序 +├── static_model.py # 构建静态图 +├── dygraph_model.py # 构建动态图 +``` + +注：在阅读该示例前，建议您先了解以下内容： + +[paddlerec入门教程](https://github.com/PaddlePaddle/PaddleRec/blob/master/README.md) + +## 内容 + +- [模型简介](#模型简介) +- [数据准备](#数据准备) +- [运行环境](#运行环境) +- [快速开始](#快速开始) +- [模型组网](#模型组网) +- [效果复现](#效果复现) +- [进阶使用](#进阶使用) +- [FAQ](#FAQ) + +## 模型简介 +`CTR(Click Through Rate)`，即点击率，是“推荐系统/计算广告”等领域的重要指标，对其进行预估是商品推送/广告投放等决策的基础。简单来说，CTR预估对每次广告的点击情况做出预测，预测用户是点击还是不点击。CTR预估模型综合考虑各种因素、特征，在大量历史数据上训练，最终对商业决策提供帮助。本模型实现了下述论文中的AutoInt模型： + +```text +@article{weiping2018autoint, + title={AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks}, + author={Weiping, Song and Chence, Shi and Zhiping, Xiao and Zhijian, Duan and Yewen, Xu and Ming, Zhang and Jian, Tang}, + journal={arXiv preprint arXiv:1810.11921}, + year={2018} +} +``` + +## 数据准备 + +训练及测试数据集选用[Display Advertising Challenge](https://www.kaggle.com/c/criteo-display-ad-challenge/)所用的Criteo数据集。该数据集包括两部分：训练集和测试集。训练集包含一段时间内Criteo的部分流量，测试集则对应训练数据后一天的广告点击流量。 +每一行数据格式如下所示： +``` +