From 8ae02c28781931fb7342f2bfc6ce07c2237b2857 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Fri, 26 Jun 2020 17:19:50 -0400 Subject: [PATCH 001/281] [pygw] ref counting of stream object --- pygenomeworks/genomeworks/cuda/cuda.pxd | 23 +++++++++++++++++++ pygenomeworks/genomeworks/cuda/cuda.pyx | 6 ----- .../genomeworks/cudaaligner/cudaaligner.pyx | 19 ++++++++++----- pygenomeworks/genomeworks/cudapoa/cudapoa.pyx | 18 ++++++++++++--- .../test/test_cudaaligner_bindings.py | 11 ++------- 5 files changed, 53 insertions(+), 24 deletions(-) create mode 100644 pygenomeworks/genomeworks/cuda/cuda.pxd diff --git a/pygenomeworks/genomeworks/cuda/cuda.pxd b/pygenomeworks/genomeworks/cuda/cuda.pxd new file mode 100644 index 000000000..d1dd2f48e --- /dev/null +++ b/pygenomeworks/genomeworks/cuda/cuda.pxd @@ -0,0 +1,23 @@ +# +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# NVIDIA CORPORATION and its licensors retain all intellectual property +# and proprietary rights in and to this software, related documentation +# and any modifications thereto. Any use, reproduction, disclosure or +# distribution of this software and related documentation without an express +# license agreement from NVIDIA CORPORATION is strictly prohibited. +# + +# cython: profile=False +# distutils: language = c++ +# cython: embedsignature = True +# cython: language_level = 3 + +# This file declares public cython utility objects for CUDA. + +cdef class CudaStream: + # Using size_t to store stream since underlying + # representation of cudaStream_t is a (void *) + # and python doesn't know how to deal with converting + # (void *) to python objects. + cdef size_t stream diff --git a/pygenomeworks/genomeworks/cuda/cuda.pyx b/pygenomeworks/genomeworks/cuda/cuda.pyx index 3dc80a9df..28cb314cd 100644 --- a/pygenomeworks/genomeworks/cuda/cuda.pyx +++ b/pygenomeworks/genomeworks/cuda/cuda.pyx @@ -32,12 +32,6 @@ cdef class CudaStream: """Class to abstract the usage of CUDA streams which enable easy asynchronous execution of CUDA kernels. """ - # Using size_t to store stream since underlying - # representation of cudaStream_t is a (void *) - # and python doesn't know how to deal with converting - # (void *) to python objects. - cdef size_t stream - def __cinit__(self): """Constructs a CudaStream object to encapsulate a CUDA stream. """ diff --git a/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx b/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx index 9558f5393..5b6b3f894 100644 --- a/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx +++ b/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx @@ -15,7 +15,6 @@ """Bindings for CUDAALIGNER.""" -import genomeworks.cuda as cuda from cython.operator cimport dereference as deref from libcpp.vector cimport vector @@ -23,7 +22,9 @@ from libcpp.memory cimport unique_ptr, shared_ptr from libcpp.string cimport string from libc.stdint cimport uint16_t +from cpython.ref cimport Py_INCREF, Py_DECREF from genomeworks.cuda.cuda_runtime_api cimport _Stream +from genomeworks.cuda.cuda cimport CudaStream cimport genomeworks.cudaaligner.cudaaligner as cudaaligner @@ -122,18 +123,18 @@ cdef class CudaAlignerBatch: """Python API for CUDA-accelerated sequence to sequence alignment. """ cdef unique_ptr[cudaaligner.Aligner] aligner - cdef public object stream + cdef public CudaStream stream def __cinit__( self, max_query_length, max_target_length, max_alignments, + *args, alignment_type="global", stream=None, device_id=0, max_device_memory_allocator_caching_size=-1, - *args, **kwargs): """Construct a CudaAligner object to run CUDA-accelerated sequence to sequence alignment across all pairs in a batch. @@ -153,12 +154,16 @@ cdef class CudaAlignerBatch: cdef _Stream temp_stream if (stream is None): temp_stream = NULL - elif (not isinstance(stream, cuda.CudaStream)): + elif (not isinstance(stream, CudaStream)): raise RuntimeError("Type for stream option must be CudaStream") else: st = stream.stream temp_stream = <_Stream>st - self.stream = stream # keep a reference to the stream, such that it gets destroyed after the aligner. + # keep a reference to the stream, such that it gets destroyed after the aligner. + self.stream = stream + # Increasing ref count of CudaStream object to ensure it doesn't get garbage + # collected before CudaAlignerBatch object. + Py_INCREF(stream) cdef cudaaligner.AlignmentType alignment_type_enum if (alignment_type == "global"): @@ -180,11 +185,11 @@ cdef class CudaAlignerBatch: max_query_length, max_target_length, max_alignments, + *args, alignment_type="global", stream=None, device_id=0, max_device_memory_allocator_caching_size=-1, - *args, **kwargs): """Dummy implementation of __init__ function to allow for Python subclassing. @@ -261,6 +266,8 @@ cdef class CudaAlignerBatch: def __dealloc__(self): self.aligner.reset() + # Decreasing ref count for CudaStream object. + Py_DECREF(self.stream) def reset(self): """Reset the contents of the batch so the same GPU memory can be used to diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx index fd3d137db..9d7a8f6ee 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx @@ -17,8 +17,6 @@ import networkx as nx -import genomeworks.cuda as cuda - from cython.operator cimport dereference as deref from libc.stdint cimport uint16_t, int8_t, int32_t from libcpp.memory cimport unique_ptr, make_unique @@ -26,7 +24,9 @@ from libcpp.pair cimport pair from libcpp.string cimport string from libcpp.vector cimport vector +from cpython.ref cimport Py_INCREF, Py_DECREF from genomeworks.cuda.cuda_runtime_api cimport _Stream +from genomeworks.cuda.cuda cimport CudaStream from genomeworks.cudapoa.graph cimport DirectedGraph cimport genomeworks.cudapoa.cudapoa as cudapoa @@ -61,6 +61,7 @@ cdef class CudaPoaBatch: """Python API for CUDA-accelerated partial order alignment algorithm.""" cdef unique_ptr[cudapoa.Batch] batch cdef unique_ptr[cudapoa.BatchSize] batch_size + cdef CudaStream stream def __cinit__( self, @@ -103,11 +104,16 @@ cdef class CudaPoaBatch: cdef _Stream temp_stream if (stream is None): temp_stream = NULL - elif (not isinstance(stream, cuda.CudaStream)): + elif (not isinstance(stream, CudaStream)): raise RuntimeError("Type for stream option must be CudaStream") else: st = stream.stream temp_stream = <_Stream>st + # keep a reference to the stream, such that it gets destroyed after the batch. + self.stream = stream + # Increasing ref count of CudaStream object to ensure it doesn't get garbage + # collected before CudaPoaBatch object. + Py_INCREF(stream) if (output_type == "consensus"): output_mask = cudapoa.consensus @@ -296,3 +302,9 @@ cdef class CudaPoaBatch: assigned to batch object. """ deref(self.batch).reset() + + def __dealloc__(self): + # Decreasing ref count for CudaStream object. + self.batch.reset() + self.batch_size.reset() + Py_DECREF(self.stream) diff --git a/pygenomeworks/test/test_cudaaligner_bindings.py b/pygenomeworks/test/test_cudaaligner_bindings.py index 6c861cac4..e5ecadde3 100644 --- a/pygenomeworks/test/test_cudaaligner_bindings.py +++ b/pygenomeworks/test/test_cudaaligner_bindings.py @@ -16,13 +16,6 @@ from genomeworks.simulators.readsim import NoisyReadSimulator -@pytest.fixture -def get_stream(): - stream = cuda.CudaStream() - yield stream - del stream - - @pytest.mark.gpu @pytest.mark.parametrize("query, target, cigar", [ ("AAAAAAA", "TTTTTTT", "7M"), @@ -31,11 +24,11 @@ def get_stream(): ("TGCA", "ATACGCT", "1I1M2I3M"), pytest.param("ACGT", "TCGA", "5M", marks=pytest.mark.xfail), ]) -def test_cudaaligner_simple_batch(query, target, cigar, get_stream): +def test_cudaaligner_simple_batch(query, target, cigar): """Test valid calculation of alignments by checking cigar strings. """ device = cuda.cuda_get_device() - stream = get_stream + stream = cuda.CudaStream() batch = CudaAlignerBatch(len(query), len(target), 1, alignment_type="global", stream=stream, device_id=device) batch.add_alignment(query, target) batch.align_all() From 5084a695a787f4fef286818073cd1c46492e27bc Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Sun, 28 Jun 2020 14:25:00 -0400 Subject: [PATCH 002/281] [pygw] updating comments --- pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx | 2 ++ pygenomeworks/genomeworks/cudapoa/cudapoa.pyx | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx b/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx index 5b6b3f894..9f4816e8b 100644 --- a/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx +++ b/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx @@ -163,6 +163,8 @@ cdef class CudaAlignerBatch: self.stream = stream # Increasing ref count of CudaStream object to ensure it doesn't get garbage # collected before CudaAlignerBatch object. + # NOTE: Ideally this is taken care of by just storing the reference + # in the line above, but that doesn't seem to be persistent. Py_INCREF(stream) cdef cudaaligner.AlignmentType alignment_type_enum diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx index 9d7a8f6ee..165d5ef11 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx @@ -113,6 +113,8 @@ cdef class CudaPoaBatch: self.stream = stream # Increasing ref count of CudaStream object to ensure it doesn't get garbage # collected before CudaPoaBatch object. + # NOTE: Ideally this is taken care of by just storing the reference + # in the line above, but that doesn't seem to be persistent. Py_INCREF(stream) if (output_type == "consensus"): From 47c5c8fcad46e9dc3bd94689cba458bdad01aaeb Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Tue, 30 Jun 2020 16:40:41 +0200 Subject: [PATCH 003/281] [cudamapper] Pinning HostIndexCopy arrays while they are being copied from --- cudamapper/src/index_host_copy.cu | 57 ++++++++++++++++++++++++++++++ cudamapper/src/index_host_copy.cuh | 9 +++++ 2 files changed, 66 insertions(+) diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 3e65633b1..1f7322b6e 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -22,6 +22,60 @@ namespace genomeworks namespace cudamapper { +namespace details +{ + +/// IndexHostMemoryPinner - registers all host arrays in given IndexHostCopy as pinned memory and unregisters it at the end +class IndexHostMemoryPinner +{ +public: + /// @brief Constructor - registers pinned memory + /// @param index_host_copy - IndexHostCopy whose arrays should be registered + IndexHostMemoryPinner(IndexHostCopy& index_host_copy) + : index_host_copy_(index_host_copy) + { + GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.representations_.data(), + index_host_copy_.representations_.size() * sizeof(representation_t), + cudaHostRegisterDefault)); + + GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.read_ids_.data(), + index_host_copy_.read_ids_.size() * sizeof(read_id_t), + cudaHostRegisterDefault)); + + GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.positions_in_reads_.data(), + index_host_copy_.positions_in_reads_.size() * sizeof(position_in_read_t), + cudaHostRegisterDefault)); + + GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.directions_of_reads_.data(), + index_host_copy_.directions_of_reads_.size() * sizeof(SketchElement::DirectionOfRepresentation), + cudaHostRegisterDefault)); + + GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.unique_representations_.data(), + index_host_copy_.unique_representations_.size() * sizeof(representation_t), + cudaHostRegisterDefault)); + + GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.first_occurrence_of_representations_.data(), + index_host_copy_.first_occurrence_of_representations_.size() * sizeof(std::uint32_t), + cudaHostRegisterDefault)); + } + + /// @brief Destructor - unregisters the arrays + ~IndexHostMemoryPinner() + { + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.representations_.data())); + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.read_ids_.data())); + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.positions_in_reads_.data())); + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.directions_of_reads_.data())); + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.unique_representations_.data())); + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.first_occurrence_of_representations_.data())); + } + +private: + IndexHostCopy& index_host_copy_; +}; + +} // namespace details + IndexHostCopy::IndexHostCopy(const Index& index, const read_id_t first_read_id, const std::uint64_t kmer_size, @@ -80,6 +134,9 @@ IndexHostCopy::IndexHostCopy(const Index& index, std::unique_ptr IndexHostCopy::copy_index_to_device(DefaultDeviceAllocator allocator, const cudaStream_t cuda_stream) const { + // pin_memory_object registers all host arrays as pinned memory and unregisters them on its destruction (i.e. at the end of this function) + details::IndexHostMemoryPinner pin_memory_object(const_cast(*this)); + return std::make_unique>(allocator, *this, cuda_stream); diff --git a/cudamapper/src/index_host_copy.cuh b/cudamapper/src/index_host_copy.cuh index 8243552db..6f4721877 100644 --- a/cudamapper/src/index_host_copy.cuh +++ b/cudamapper/src/index_host_copy.cuh @@ -20,6 +20,12 @@ namespace genomeworks namespace cudamapper { + +namespace details +{ + class IndexHostMemoryPinner; +} + /// IndexHostCopy - Creates and maintains a copy of computed IndexGPU elements on the host /// /// @@ -92,6 +98,9 @@ public: std::uint64_t window_size() const override; private: + // used to register host arrays as pinned memory when they are copied from + friend class details::IndexHostMemoryPinner; + std::vector representations_; std::vector read_ids_; std::vector positions_in_reads_; From 04210e93c0f0049d5c205acbb05d3ecfb6421fa8 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 1 Jul 2020 17:17:41 +0200 Subject: [PATCH 004/281] [cudamapper] IndexHostCopy uses one underlying array for all internal arrays --- .../genomeworks/cudamapper/index.hpp | 23 +++-- cudamapper/src/index_gpu.cuh | 24 ++--- cudamapper/src/index_host_copy.cu | 95 ++++++++++--------- cudamapper/src/index_host_copy.cuh | 28 +++--- 4 files changed, 92 insertions(+), 78 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index a7920d5df..998b847a8 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -112,6 +112,17 @@ class Index class IndexHostCopyBase { public: + /// ArrayView - helper struct that provides a view of part of underlying array + /// \tparam T type of data in this view + template + struct ArrayView + { + /// pointer to first element in view + T* data; + /// total elements of type T in view + std::ptrdiff_t size; + }; + /// \brief copy cached index vectors from the host and create an object of Index on GPU /// \param allocator asynchronous device allocator used for temporary buffer allocations /// \param cuda_stream H2D copy is done on this stream. Device arrays are also associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor is done @@ -124,27 +135,27 @@ class IndexHostCopyBase /// \brief returns an array of representations of sketch elements (stored on host) /// \return an array of representations of sketch elements - virtual const std::vector& representations() const = 0; + virtual const ArrayView representations() const = 0; /// \brief returns an array of reads ids for sketch elements (stored on host) /// \return an array of reads ids for sketch elements - virtual const std::vector& read_ids() const = 0; + virtual const ArrayView read_ids() const = 0; /// \brief returns an array of starting positions of sketch elements in their reads (stored on host) /// \return an array of starting positions of sketch elements in their reads - virtual const std::vector& positions_in_reads() const = 0; + virtual const ArrayView positions_in_reads() const = 0; /// \brief returns an array of directions in which sketch elements were read (stored on host) /// \return an array of directions in which sketch elements were read - virtual const std::vector& directions_of_reads() const = 0; + virtual const ArrayView directions_of_reads() const = 0; /// \brief returns an array where each representation is recorded only once, sorted by representation (stored on host) /// \return an array where each representation is recorded only once, sorted by representation - virtual const std::vector& unique_representations() const = 0; + virtual const ArrayView unique_representations() const = 0; /// \brief returns first occurrence of corresponding representation from unique_representations(), plus one more element with the total number of sketch elements (stored on host) /// \return first occurrence of corresponding representation from unique_representations(), plus one more element with the total number of sketch elements - virtual const std::vector& first_occurrence_of_representations() const = 0; + virtual const ArrayView first_occurrence_of_representations() const = 0; /// \brief returns number of reads in input data /// \return number of reads in input data diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index dc4db9b18..b696c75a5 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -599,28 +599,28 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, number_of_basepairs_in_longest_read_ = index_host_copy.number_of_basepairs_in_longest_read(); //H2D- representations_d_ = index_host_copy.representations(); - representations_d_.clear_and_resize(index_host_copy.representations().size()); - cudautils::device_copy_n(index_host_copy.representations().data(), index_host_copy.representations().size(), representations_d_.data(), cuda_stream); + representations_d_.clear_and_resize(index_host_copy.representations().size); + cudautils::device_copy_n(index_host_copy.representations().data, index_host_copy.representations().size, representations_d_.data(), cuda_stream); //H2D- read_ids_d_ = index_host_copy.read_ids(); - read_ids_d_.clear_and_resize(index_host_copy.read_ids().size()); - cudautils::device_copy_n(index_host_copy.read_ids().data(), index_host_copy.read_ids().size(), read_ids_d_.data(), cuda_stream); + read_ids_d_.clear_and_resize(index_host_copy.read_ids().size); + cudautils::device_copy_n(index_host_copy.read_ids().data, index_host_copy.read_ids().size, read_ids_d_.data(), cuda_stream); //H2D- positions_in_reads_d_ = index_host_copy.positions_in_reads(); - positions_in_reads_d_.clear_and_resize(index_host_copy.positions_in_reads().size()); - cudautils::device_copy_n(index_host_copy.positions_in_reads().data(), index_host_copy.positions_in_reads().size(), positions_in_reads_d_.data(), cuda_stream); + positions_in_reads_d_.clear_and_resize(index_host_copy.positions_in_reads().size); + cudautils::device_copy_n(index_host_copy.positions_in_reads().data, index_host_copy.positions_in_reads().size, positions_in_reads_d_.data(), cuda_stream); //H2D- directions_of_reads_d_ = index_host_copy.directions_of_reads(); - directions_of_reads_d_.clear_and_resize(index_host_copy.directions_of_reads().size()); - cudautils::device_copy_n(index_host_copy.directions_of_reads().data(), index_host_copy.directions_of_reads().size(), directions_of_reads_d_.data(), cuda_stream); + directions_of_reads_d_.clear_and_resize(index_host_copy.directions_of_reads().size); + cudautils::device_copy_n(index_host_copy.directions_of_reads().data, index_host_copy.directions_of_reads().size, directions_of_reads_d_.data(), cuda_stream); //H2D- unique_representations_d_ = index_host_copy.unique_representations(); - unique_representations_d_.clear_and_resize(index_host_copy.unique_representations().size()); - cudautils::device_copy_n(index_host_copy.unique_representations().data(), index_host_copy.unique_representations().size(), unique_representations_d_.data(), cuda_stream); + unique_representations_d_.clear_and_resize(index_host_copy.unique_representations().size); + cudautils::device_copy_n(index_host_copy.unique_representations().data, index_host_copy.unique_representations().size, unique_representations_d_.data(), cuda_stream); //H2D- first_occurrence_of_representations_d_ = index_host_copy.first_occurrence_of_representations(); - first_occurrence_of_representations_d_.clear_and_resize(index_host_copy.first_occurrence_of_representations().size()); - cudautils::device_copy_n(index_host_copy.first_occurrence_of_representations().data(), index_host_copy.first_occurrence_of_representations().size(), first_occurrence_of_representations_d_.data(), cuda_stream); + first_occurrence_of_representations_d_.clear_and_resize(index_host_copy.first_occurrence_of_representations().size); + cudautils::device_copy_n(index_host_copy.first_occurrence_of_representations().data, index_host_copy.first_occurrence_of_representations().size, first_occurrence_of_representations_d_.data(), cuda_stream); // This is not completely necessary, but if removed one has to make sure that the next step // uses the same stream or that sync is done in caller diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 1f7322b6e..88205dc0b 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -13,6 +13,8 @@ #include "index_gpu.cuh" #include "minimizer.hpp" +#include + namespace claraparabricks { @@ -34,40 +36,15 @@ public: IndexHostMemoryPinner(IndexHostCopy& index_host_copy) : index_host_copy_(index_host_copy) { - GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.representations_.data(), - index_host_copy_.representations_.size() * sizeof(representation_t), - cudaHostRegisterDefault)); - - GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.read_ids_.data(), - index_host_copy_.read_ids_.size() * sizeof(read_id_t), - cudaHostRegisterDefault)); - - GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.positions_in_reads_.data(), - index_host_copy_.positions_in_reads_.size() * sizeof(position_in_read_t), - cudaHostRegisterDefault)); - - GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.directions_of_reads_.data(), - index_host_copy_.directions_of_reads_.size() * sizeof(SketchElement::DirectionOfRepresentation), - cudaHostRegisterDefault)); - - GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.unique_representations_.data(), - index_host_copy_.unique_representations_.size() * sizeof(representation_t), - cudaHostRegisterDefault)); - - GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.first_occurrence_of_representations_.data(), - index_host_copy_.first_occurrence_of_representations_.size() * sizeof(std::uint32_t), + GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.underlying_array_.data(), + index_host_copy_.underlying_array_.size() * sizeof(unsigned char), cudaHostRegisterDefault)); } /// @brief Destructor - unregisters the arrays ~IndexHostMemoryPinner() { - GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.representations_.data())); - GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.read_ids_.data())); - GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.positions_in_reads_.data())); - GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.directions_of_reads_.data())); - GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.unique_representations_.data())); - GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.first_occurrence_of_representations_.data())); + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.underlying_array_.data())); } private: @@ -87,40 +64,64 @@ IndexHostCopy::IndexHostCopy(const Index& index, { GW_NVTX_RANGE(profiler, "cache_index"); - representations_.resize(index.representations().size()); + // Use only one large array to store all arrays in order to reduce fragmentation when using pool allocators + // Align all arrays by 64 bits + const std::size_t representations_bits = claraparabricks::genomeworks::ceiling_divide(index.representations().size() * sizeof(representation_t), sizeof(std::uint64_t)) * sizeof(uint64_t); + const std::size_t read_ids_bits = claraparabricks::genomeworks::ceiling_divide(index.read_ids().size() * sizeof(read_id_t), sizeof(std::uint64_t)) * sizeof(uint64_t); + const std::size_t positions_in_reads_bits = claraparabricks::genomeworks::ceiling_divide(index.positions_in_reads().size() * sizeof(position_in_read_t), sizeof(std::uint64_t)) * sizeof(uint64_t); + const std::size_t directions_of_reads_bits = claraparabricks::genomeworks::ceiling_divide(index.directions_of_reads().size() * sizeof(SketchElement::DirectionOfRepresentation), sizeof(std::uint64_t)) * sizeof(uint64_t); + const std::size_t unique_representations_bits = claraparabricks::genomeworks::ceiling_divide(index.unique_representations().size() * sizeof(representation_t), sizeof(std::uint64_t)) * sizeof(uint64_t); + const std::size_t first_occurrence_of_representations_bits = claraparabricks::genomeworks::ceiling_divide(index.first_occurrence_of_representations().size() * sizeof(std::uint32_t), sizeof(std::uint64_t)) * sizeof(uint64_t); + + const std::size_t total_bits = representations_bits + + read_ids_bits + + positions_in_reads_bits + + directions_of_reads_bits + + unique_representations_bits + + first_occurrence_of_representations_bits; + underlying_array_.resize(total_bits); + + std::size_t current_bit = 0; + representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.representations().size()}; + current_bit += representations_bits; + read_ids_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.read_ids().size()}; + current_bit += read_ids_bits; + positions_in_reads_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.positions_in_reads().size()}; + current_bit += positions_in_reads_bits; + directions_of_reads_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.directions_of_reads().size()}; + current_bit += directions_of_reads_bits; + unique_representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.unique_representations().size()}; + current_bit += unique_representations_bits; + first_occurrence_of_representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.first_occurrence_of_representations().size()}; + cudautils::device_copy_n(index.representations().data(), index.representations().size(), - representations_.data(), + representations_.data, cuda_stream); - read_ids_.resize(index.read_ids().size()); cudautils::device_copy_n(index.read_ids().data(), index.read_ids().size(), - read_ids_.data(), + read_ids_.data, cuda_stream); - positions_in_reads_.resize(index.positions_in_reads().size()); cudautils::device_copy_n(index.positions_in_reads().data(), index.positions_in_reads().size(), - positions_in_reads_.data(), + positions_in_reads_.data, cuda_stream); - directions_of_reads_.resize(index.directions_of_reads().size()); cudautils::device_copy_n(index.directions_of_reads().data(), index.directions_of_reads().size(), - directions_of_reads_.data(), + directions_of_reads_.data, cuda_stream); - unique_representations_.resize(index.unique_representations().size()); cudautils::device_copy_n(index.unique_representations().data(), index.unique_representations().size(), - unique_representations_.data(), + unique_representations_.data, cuda_stream); - - first_occurrence_of_representations_.resize(index.first_occurrence_of_representations().size()); +; cudautils::device_copy_n(index.first_occurrence_of_representations().data(), index.first_occurrence_of_representations().size(), - first_occurrence_of_representations_.data(), + first_occurrence_of_representations_.data, cuda_stream); number_of_reads_ = index.number_of_reads(); @@ -142,32 +143,32 @@ std::unique_ptr IndexHostCopy::copy_index_to_device(DefaultDeviceAllocato cuda_stream); } -const std::vector& IndexHostCopy::representations() const +const IndexHostCopyBase::ArrayView IndexHostCopy::representations() const { return representations_; } -const std::vector& IndexHostCopy::read_ids() const +const IndexHostCopyBase::ArrayView IndexHostCopy::read_ids() const { return read_ids_; } -const std::vector& IndexHostCopy::positions_in_reads() const +const IndexHostCopyBase::ArrayView IndexHostCopy::positions_in_reads() const { return positions_in_reads_; } -const std::vector& IndexHostCopy::directions_of_reads() const +const IndexHostCopyBase::ArrayView IndexHostCopy::directions_of_reads() const { return directions_of_reads_; } -const std::vector& IndexHostCopy::unique_representations() const +const IndexHostCopyBase::ArrayView IndexHostCopy::unique_representations() const { return unique_representations_; } -const std::vector& IndexHostCopy::first_occurrence_of_representations() const +const IndexHostCopyBase::ArrayView IndexHostCopy::first_occurrence_of_representations() const { return first_occurrence_of_representations_; } diff --git a/cudamapper/src/index_host_copy.cuh b/cudamapper/src/index_host_copy.cuh index 6f4721877..2d32f8a70 100644 --- a/cudamapper/src/index_host_copy.cuh +++ b/cudamapper/src/index_host_copy.cuh @@ -23,7 +23,7 @@ namespace cudamapper namespace details { - class IndexHostMemoryPinner; +class IndexHostMemoryPinner; } /// IndexHostCopy - Creates and maintains a copy of computed IndexGPU elements on the host @@ -55,27 +55,27 @@ public: /// \brief returns an array of representations of sketch elements (stored on host) /// \return an array of representations of sketch elements - const std::vector& representations() const override; + const ArrayView representations() const override; /// \brief returns an array of reads ids for sketch elements (stored on host) /// \return an array of reads ids for sketch elements - const std::vector& read_ids() const override; + const ArrayView read_ids() const override; /// \brief returns an array of starting positions of sketch elements in their reads (stored on host) /// \return an array of starting positions of sketch elements in their reads - const std::vector& positions_in_reads() const override; + const ArrayView positions_in_reads() const override; /// \brief returns an array of directions in which sketch elements were read (stored on host) /// \return an array of directions in which sketch elements were read - const std::vector& directions_of_reads() const override; + const ArrayView directions_of_reads() const override; /// \brief returns an array where each representation is recorded only once, sorted by representation (stored on host) /// \return an array where each representation is recorded only once, sorted by representation - const std::vector& unique_representations() const override; + const ArrayView unique_representations() const override; /// \brief returns first occurrence of corresponding representation from unique_representations(), plus one more element with the total number of sketch elements (stored on host) /// \return first occurrence of corresponding representation from unique_representations(), plus one more element with the total number of sketch elements - const std::vector& first_occurrence_of_representations() const override; + const ArrayView first_occurrence_of_representations() const override; /// \brief returns number of reads in input data /// \return number of reads in input data @@ -101,13 +101,15 @@ private: // used to register host arrays as pinned memory when they are copied from friend class details::IndexHostMemoryPinner; - std::vector representations_; - std::vector read_ids_; - std::vector positions_in_reads_; - std::vector directions_of_reads_; + // use a single underlying vector in order to reduce memory fragmentation when using pool allocators + std::vector underlying_array_; - std::vector unique_representations_; - std::vector first_occurrence_of_representations_; + ArrayView representations_; + ArrayView read_ids_; + ArrayView positions_in_reads_; + ArrayView directions_of_reads_; + ArrayView unique_representations_; + ArrayView first_occurrence_of_representations_; read_id_t number_of_reads_; position_in_read_t number_of_basepairs_in_longest_read_; From 66ed19853d217eb3995c51b791f3b7053e3a0873 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 2 Jul 2020 10:53:04 +0200 Subject: [PATCH 005/281] [cudamapper] Using pinned memory when copying indices from device to host as well --- cudamapper/src/index_host_copy.cu | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 88205dc0b..949175a77 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -36,6 +36,7 @@ public: IndexHostMemoryPinner(IndexHostCopy& index_host_copy) : index_host_copy_(index_host_copy) { + GW_NVTX_RANGE(profiler, "register_pinned_memory"); GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.underlying_array_.data(), index_host_copy_.underlying_array_.size() * sizeof(unsigned char), cudaHostRegisterDefault)); @@ -44,6 +45,7 @@ public: /// @brief Destructor - unregisters the arrays ~IndexHostMemoryPinner() { + GW_NVTX_RANGE(profiler, "unregister_pinned_memory"); GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.underlying_array_.data())); } @@ -62,7 +64,7 @@ IndexHostCopy::IndexHostCopy(const Index& index, , kmer_size_(kmer_size) , window_size_(window_size) { - GW_NVTX_RANGE(profiler, "cache_index"); + GW_NVTX_RANGE(profiler, "index_host_copy"); // Use only one large array to store all arrays in order to reduce fragmentation when using pool allocators // Align all arrays by 64 bits @@ -79,10 +81,14 @@ IndexHostCopy::IndexHostCopy(const Index& index, directions_of_reads_bits + unique_representations_bits + first_occurrence_of_representations_bits; - underlying_array_.resize(total_bits); + + { + GW_NVTX_RANGE(profiler, "index_host_copy::allocate_host_memory"); + underlying_array_.resize(total_bits); + } std::size_t current_bit = 0; - representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.representations().size()}; + representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.representations().size()}; current_bit += representations_bits; read_ids_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.read_ids().size()}; current_bit += read_ids_bits; @@ -94,6 +100,9 @@ IndexHostCopy::IndexHostCopy(const Index& index, current_bit += unique_representations_bits; first_occurrence_of_representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.first_occurrence_of_representations().size()}; + // pin_memory_object registers host array as pinned memory and unregisters it on its destruction (i.e. at the end of this function) + details::IndexHostMemoryPinner pin_memory_object(const_cast(*this)); + cudautils::device_copy_n(index.representations().data(), index.representations().size(), representations_.data, @@ -118,7 +127,7 @@ IndexHostCopy::IndexHostCopy(const Index& index, index.unique_representations().size(), unique_representations_.data, cuda_stream); -; + cudautils::device_copy_n(index.first_occurrence_of_representations().data(), index.first_occurrence_of_representations().size(), first_occurrence_of_representations_.data, @@ -135,7 +144,7 @@ IndexHostCopy::IndexHostCopy(const Index& index, std::unique_ptr IndexHostCopy::copy_index_to_device(DefaultDeviceAllocator allocator, const cudaStream_t cuda_stream) const { - // pin_memory_object registers all host arrays as pinned memory and unregisters them on its destruction (i.e. at the end of this function) + // pin_memory_object registers host array as pinned memory and unregisters it on its destruction (i.e. at the end of this function) details::IndexHostMemoryPinner pin_memory_object(const_cast(*this)); return std::make_unique>(allocator, From 2f0d114f546197eb38ca48310112e27037db6fe2 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 2 Jul 2020 11:56:02 +0200 Subject: [PATCH 006/281] [cudamapper] Do not save cuda_stream_ in IndexGPU, just pass it around during construction --- cudamapper/src/index_gpu.cuh | 38 +++++++++++++++++------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index b696c75a5..61207054f 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -131,7 +131,8 @@ private: const read_id_t first_read_id, const read_id_t past_the_last_read_id, const bool hash_representations, - const double filtering_parameter); + const double filtering_parameter, + const cudaStream_t cuda_stream); device_buffer representations_d_; device_buffer read_ids_d_; @@ -149,9 +150,6 @@ private: read_id_t number_of_reads_ = 0; position_in_read_t number_of_basepairs_in_longest_read_ = 0; - // CUDA stream on which the work is to be done and device arrays are to be associated with - cudaStream_t cuda_stream_; - DefaultDeviceAllocator allocator_; }; @@ -566,17 +564,17 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, , directions_of_reads_d_(allocator) , unique_representations_d_(allocator) , first_occurrence_of_representations_d_(allocator) - , cuda_stream_(cuda_stream) { generate_index(parser, first_read_id_, past_the_last_read_id, hash_representations, - filtering_parameter); + filtering_parameter, + cuda_stream); // This is not completely necessary, but if removed one has to make sure that the next step // uses the same stream or that sync is done in caller - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); } template @@ -593,7 +591,6 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, , directions_of_reads_d_(allocator) , unique_representations_d_(allocator) , first_occurrence_of_representations_d_(allocator) - , cuda_stream_(cuda_stream) { number_of_reads_ = index_host_copy.number_of_reads(); number_of_basepairs_in_longest_read_ = index_host_copy.number_of_basepairs_in_longest_read(); @@ -624,7 +621,7 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, // This is not completely necessary, but if removed one has to make sure that the next step // uses the same stream or that sync is done in caller - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); } template @@ -692,7 +689,8 @@ void IndexGPU::generate_index(const io::FastaParser& parser, const read_id_t first_read_id, const read_id_t past_the_last_read_id, const bool hash_representations, - const double filtering_parameter) + const double filtering_parameter, + const cudaStream_t cuda_stream) { // check if there are any reads to process @@ -760,19 +758,19 @@ void IndexGPU::generate_index(const io::FastaParser& parser, // move basepairs to the device GW_LOG_INFO("Allocating {} bytes for read_id_to_basepairs_section_d", read_id_to_basepairs_section_h.size() * sizeof(decltype(read_id_to_basepairs_section_h)::value_type)); - device_buffer read_id_to_basepairs_section_d(read_id_to_basepairs_section_h.size(), allocator_, cuda_stream_); + device_buffer read_id_to_basepairs_section_d(read_id_to_basepairs_section_h.size(), allocator_, cuda_stream); cudautils::device_copy_n(read_id_to_basepairs_section_h.data(), read_id_to_basepairs_section_h.size(), read_id_to_basepairs_section_d.data(), - cuda_stream_); // H2D + cuda_stream); // H2D GW_LOG_INFO("Allocating {} bytes for merged_basepairs_d", merged_basepairs_h.size() * sizeof(decltype(merged_basepairs_h)::value_type)); - device_buffer merged_basepairs_d(merged_basepairs_h.size(), allocator_, cuda_stream_); + device_buffer merged_basepairs_d(merged_basepairs_h.size(), allocator_, cuda_stream); cudautils::device_copy_n(merged_basepairs_h.data(), merged_basepairs_h.size(), merged_basepairs_d.data(), - cuda_stream_); // H2D - cudaStreamSynchronize(cuda_stream_); + cuda_stream); // H2D + cudaStreamSynchronize(cuda_stream); merged_basepairs_h.clear(); merged_basepairs_h.shrink_to_fit(); @@ -786,7 +784,7 @@ void IndexGPU::generate_index(const io::FastaParser& parser, read_id_to_basepairs_section_h, read_id_to_basepairs_section_d, hash_representations, - cuda_stream_); + cuda_stream); device_buffer generated_representations_d = std::move(sketch_elements.representations_d); device_buffer generated_rest_d = std::move(sketch_elements.rest_d); @@ -802,7 +800,7 @@ void IndexGPU::generate_index(const io::FastaParser& parser, // *** sort sketch elements by representation *** // As this is a stable sort and the data was initailly grouper by read_id this means that the sketch elements within each representations are sorted by read_id // TODO: consider using a CUB radix sort based function here - thrust::stable_sort_by_key(thrust::cuda::par(allocator_).on(cuda_stream_), + thrust::stable_sort_by_key(thrust::cuda::par(allocator_).on(cuda_stream), std::begin(generated_representations_d), std::end(generated_representations_d), std::begin(generated_rest_d)); @@ -816,7 +814,7 @@ void IndexGPU::generate_index(const io::FastaParser& parser, const std::uint32_t threads = 256; const std::uint32_t blocks = ceiling_divide(representations_d_.size(), threads); - details::index_gpu::copy_rest_to_separate_arrays<<>>(generated_rest_d.data(), + details::index_gpu::copy_rest_to_separate_arrays<<>>(generated_rest_d.data(), read_ids_d_.data(), positions_in_reads_d_.data(), directions_of_reads_d_.data(), @@ -827,7 +825,7 @@ void IndexGPU::generate_index(const io::FastaParser& parser, unique_representations_d_, first_occurrence_of_representations_d_, representations_d_, - cuda_stream_); + cuda_stream); if (filtering_parameter < 1.0) { @@ -839,7 +837,7 @@ void IndexGPU::generate_index(const io::FastaParser& parser, directions_of_reads_d_, unique_representations_d_, first_occurrence_of_representations_d_, - cuda_stream_); + cuda_stream); } } From 67b206dea8d44a2869fc2d7c9b9d98c58db86e8d Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 2 Jul 2020 12:13:08 +0200 Subject: [PATCH 007/281] [cudamapper] Do not save allocator_ in IndexGPU, just pass it around during construction --- cudamapper/src/index_gpu.cuh | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index 61207054f..93b0e52b8 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -132,6 +132,7 @@ private: const read_id_t past_the_last_read_id, const bool hash_representations, const double filtering_parameter, + DefaultDeviceAllocator allocator, const cudaStream_t cuda_stream); device_buffer representations_d_; @@ -149,8 +150,6 @@ private: const std::uint64_t window_size_ = 0; read_id_t number_of_reads_ = 0; position_in_read_t number_of_basepairs_in_longest_read_ = 0; - - DefaultDeviceAllocator allocator_; }; namespace details @@ -557,7 +556,6 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, , window_size_(window_size) , number_of_reads_(0) , number_of_basepairs_in_longest_read_(0) - , allocator_(allocator) , representations_d_(allocator) , read_ids_d_(allocator) , positions_in_reads_d_(allocator) @@ -570,6 +568,7 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, past_the_last_read_id, hash_representations, filtering_parameter, + allocator, cuda_stream); // This is not completely necessary, but if removed one has to make sure that the next step @@ -584,7 +583,6 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, : first_read_id_(index_host_copy.first_read_id()) , kmer_size_(index_host_copy.kmer_size()) , window_size_(index_host_copy.window_size()) - , allocator_(allocator) , representations_d_(allocator) , read_ids_d_(allocator) , positions_in_reads_d_(allocator) @@ -690,6 +688,7 @@ void IndexGPU::generate_index(const io::FastaParser& parser, const read_id_t past_the_last_read_id, const bool hash_representations, const double filtering_parameter, + DefaultDeviceAllocator allocator, const cudaStream_t cuda_stream) { @@ -758,14 +757,14 @@ void IndexGPU::generate_index(const io::FastaParser& parser, // move basepairs to the device GW_LOG_INFO("Allocating {} bytes for read_id_to_basepairs_section_d", read_id_to_basepairs_section_h.size() * sizeof(decltype(read_id_to_basepairs_section_h)::value_type)); - device_buffer read_id_to_basepairs_section_d(read_id_to_basepairs_section_h.size(), allocator_, cuda_stream); + device_buffer read_id_to_basepairs_section_d(read_id_to_basepairs_section_h.size(), allocator, cuda_stream); cudautils::device_copy_n(read_id_to_basepairs_section_h.data(), read_id_to_basepairs_section_h.size(), read_id_to_basepairs_section_d.data(), cuda_stream); // H2D GW_LOG_INFO("Allocating {} bytes for merged_basepairs_d", merged_basepairs_h.size() * sizeof(decltype(merged_basepairs_h)::value_type)); - device_buffer merged_basepairs_d(merged_basepairs_h.size(), allocator_, cuda_stream); + device_buffer merged_basepairs_d(merged_basepairs_h.size(), allocator, cuda_stream); cudautils::device_copy_n(merged_basepairs_h.data(), merged_basepairs_h.size(), merged_basepairs_d.data(), @@ -775,7 +774,7 @@ void IndexGPU::generate_index(const io::FastaParser& parser, merged_basepairs_h.shrink_to_fit(); // sketch elements get generated here - auto sketch_elements = SketchElementImpl::generate_sketch_elements(allocator_, + auto sketch_elements = SketchElementImpl::generate_sketch_elements(allocator, number_of_reads_, kmer_size_, window_size_, @@ -800,7 +799,7 @@ void IndexGPU::generate_index(const io::FastaParser& parser, // *** sort sketch elements by representation *** // As this is a stable sort and the data was initailly grouper by read_id this means that the sketch elements within each representations are sorted by read_id // TODO: consider using a CUB radix sort based function here - thrust::stable_sort_by_key(thrust::cuda::par(allocator_).on(cuda_stream), + thrust::stable_sort_by_key(thrust::cuda::par(allocator).on(cuda_stream), std::begin(generated_representations_d), std::end(generated_representations_d), std::begin(generated_rest_d)); @@ -815,13 +814,13 @@ void IndexGPU::generate_index(const io::FastaParser& parser, const std::uint32_t blocks = ceiling_divide(representations_d_.size(), threads); details::index_gpu::copy_rest_to_separate_arrays<<>>(generated_rest_d.data(), - read_ids_d_.data(), - positions_in_reads_d_.data(), - directions_of_reads_d_.data(), - representations_d_.size()); + read_ids_d_.data(), + positions_in_reads_d_.data(), + directions_of_reads_d_.data(), + representations_d_.size()); // now generate the index elements - details::index_gpu::find_first_occurrences_of_representations(allocator_, + details::index_gpu::find_first_occurrences_of_representations(allocator, unique_representations_d_, first_occurrence_of_representations_d_, representations_d_, @@ -829,7 +828,7 @@ void IndexGPU::generate_index(const io::FastaParser& parser, if (filtering_parameter < 1.0) { - details::index_gpu::filter_out_most_common_representations(allocator_, + details::index_gpu::filter_out_most_common_representations(allocator, filtering_parameter, representations_d_, read_ids_d_, From 6cc50cb9637e38631ca616909b3ef8816bd4bcf6 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 2 Jul 2020 12:46:46 +0200 Subject: [PATCH 008/281] [cudamapper] Setting sizes of buffers in Index in constructor when copying from IndexHostCopy --- cudamapper/src/index_gpu.cuh | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index 93b0e52b8..0fa275fc0 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -583,38 +583,32 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, : first_read_id_(index_host_copy.first_read_id()) , kmer_size_(index_host_copy.kmer_size()) , window_size_(index_host_copy.window_size()) - , representations_d_(allocator) - , read_ids_d_(allocator) - , positions_in_reads_d_(allocator) - , directions_of_reads_d_(allocator) - , unique_representations_d_(allocator) - , first_occurrence_of_representations_d_(allocator) + , representations_d_(index_host_copy.representations().size, allocator, cuda_stream) + , read_ids_d_(index_host_copy.read_ids().size, allocator, cuda_stream) + , positions_in_reads_d_(index_host_copy.positions_in_reads().size, allocator, cuda_stream) + , directions_of_reads_d_(index_host_copy.directions_of_reads().size, allocator, cuda_stream) + , unique_representations_d_(index_host_copy.unique_representations().size, allocator, cuda_stream) + , first_occurrence_of_representations_d_(index_host_copy.first_occurrence_of_representations().size, allocator, cuda_stream) { number_of_reads_ = index_host_copy.number_of_reads(); number_of_basepairs_in_longest_read_ = index_host_copy.number_of_basepairs_in_longest_read(); //H2D- representations_d_ = index_host_copy.representations(); - representations_d_.clear_and_resize(index_host_copy.representations().size); cudautils::device_copy_n(index_host_copy.representations().data, index_host_copy.representations().size, representations_d_.data(), cuda_stream); //H2D- read_ids_d_ = index_host_copy.read_ids(); - read_ids_d_.clear_and_resize(index_host_copy.read_ids().size); cudautils::device_copy_n(index_host_copy.read_ids().data, index_host_copy.read_ids().size, read_ids_d_.data(), cuda_stream); //H2D- positions_in_reads_d_ = index_host_copy.positions_in_reads(); - positions_in_reads_d_.clear_and_resize(index_host_copy.positions_in_reads().size); cudautils::device_copy_n(index_host_copy.positions_in_reads().data, index_host_copy.positions_in_reads().size, positions_in_reads_d_.data(), cuda_stream); //H2D- directions_of_reads_d_ = index_host_copy.directions_of_reads(); - directions_of_reads_d_.clear_and_resize(index_host_copy.directions_of_reads().size); cudautils::device_copy_n(index_host_copy.directions_of_reads().data, index_host_copy.directions_of_reads().size, directions_of_reads_d_.data(), cuda_stream); //H2D- unique_representations_d_ = index_host_copy.unique_representations(); - unique_representations_d_.clear_and_resize(index_host_copy.unique_representations().size); cudautils::device_copy_n(index_host_copy.unique_representations().data, index_host_copy.unique_representations().size, unique_representations_d_.data(), cuda_stream); //H2D- first_occurrence_of_representations_d_ = index_host_copy.first_occurrence_of_representations(); - first_occurrence_of_representations_d_.clear_and_resize(index_host_copy.first_occurrence_of_representations().size); cudautils::device_copy_n(index_host_copy.first_occurrence_of_representations().data, index_host_copy.first_occurrence_of_representations().size, first_occurrence_of_representations_d_.data(), cuda_stream); // This is not completely necessary, but if removed one has to make sure that the next step From 7dc19608fd5bc424a2f276aff8a75c7134d07eb3 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 2 Jul 2020 14:51:44 +0200 Subject: [PATCH 009/281] [cudmapper] Create streams outside of scopes in which they are used in Test_CudamapperIndexCache.cu --- cudamapper/tests/Test_CudamapperIndexCache.cu | 2073 +++++++++-------- 1 file changed, 1043 insertions(+), 1030 deletions(-) diff --git a/cudamapper/tests/Test_CudamapperIndexCache.cu b/cudamapper/tests/Test_CudamapperIndexCache.cu index 2b8e5de27..e968ae7c7 100644 --- a/cudamapper/tests/Test_CudamapperIndexCache.cu +++ b/cudamapper/tests/Test_CudamapperIndexCache.cu @@ -161,384 +161,386 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) cudaStream_t cuda_stream; GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); - const bool same_query_and_target = true; - std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/catcaag_aagcta.fasta"); - std::shared_ptr target_parser = query_parser; - DefaultDeviceAllocator allocator = create_default_device_allocator(); - const std::uint64_t k = 3; - const std::uint64_t w = 2; - const bool hash_representations = false; - const double filtering_parameter = 1.0; - - // ************* expected indices ************* - - // ** CATCAAG: AAG(4f), ATC(1f), CAA(3f), ATG(0r) - std::vector catcaag_representations; - std::vector catcaag_read_ids; - std::vector catcaag_positions_in_reads; - std::vector catcaag_directions_of_reads; - std::vector catcaag_unique_representations; - std::vector catcaag_first_occurrence_of_representations; - - // AAG(4f) - catcaag_representations.push_back(0b000010); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(4); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b000010); - catcaag_first_occurrence_of_representations.push_back(0); - // ATC(1f) - catcaag_representations.push_back(0b001101); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(1); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b001101); - catcaag_first_occurrence_of_representations.push_back(1); - // ATG(0r) - catcaag_representations.push_back(0b001110); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(0); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - catcaag_unique_representations.push_back(0b001110); - catcaag_first_occurrence_of_representations.push_back(2); - // CAA(3f) - catcaag_representations.push_back(0b010000); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(3); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b010000); - catcaag_first_occurrence_of_representations.push_back(3); - // trailing elements - catcaag_first_occurrence_of_representations.push_back(4); - - const read_id_t catcaag_number_of_reads = 1; - const std::vector catcaag_read_ids_to_read_names({"read_0"}); - const std::vector catcaag_read_ids_to_read_lengths({7}); - const read_id_t catcaag_smallest_read_id = 0; - const read_id_t catcaag_largest_read_id = 0; - const position_in_read_t catcaag_number_of_basepairs_in_longest_read = 7; - const uint64_t catcaag_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) - std::vector aagcta_representations; - std::vector aagcta_read_ids; - std::vector aagcta_positions_in_reads; - std::vector aagcta_directions_of_reads; - std::vector aagcta_unique_representations; - std::vector aagcta_first_occurrence_of_representations; - - // AAG(0f) - aagcta_representations.push_back(0b000010); - aagcta_read_ids.push_back(1); - aagcta_positions_in_reads.push_back(0); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b000010); - aagcta_first_occurrence_of_representations.push_back(0); - // AGC(2r) - aagcta_representations.push_back(0b001001); - aagcta_read_ids.push_back(1); - aagcta_positions_in_reads.push_back(2); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - aagcta_unique_representations.push_back(0b001001); - aagcta_first_occurrence_of_representations.push_back(1); - // CTA(3f) - aagcta_representations.push_back(0b011100); - aagcta_read_ids.push_back(1); - aagcta_positions_in_reads.push_back(3); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b011100); - aagcta_first_occurrence_of_representations.push_back(2); - // trailing elements - aagcta_first_occurrence_of_representations.push_back(3); - - const read_id_t aagcta_number_of_reads = 1; - const read_id_t aagcta_smallest_read_id = 1; - const read_id_t aagcta_largest_read_id = 1; - const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; - const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ** CATCAAG_AAGCTA: AAG(4f0), AAG(0f1), AGC(2r1), ATC(1f0), ATG(0r0), CAA(3f0), CTA(3f1) - std::vector catcaag_aagcta_representations; - std::vector catcaag_aagcta_read_ids; - std::vector catcaag_aagcta_positions_in_reads; - std::vector catcaag_aagcta_directions_of_reads; - std::vector catcaag_aagcta_unique_representations; - std::vector catcaag_aagcta_first_occurrence_of_representations; - - // AAG(4f0) - catcaag_aagcta_representations.push_back(0b000010); - catcaag_aagcta_read_ids.push_back(0); - catcaag_aagcta_positions_in_reads.push_back(4); - catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_aagcta_unique_representations.push_back(0b000010); - catcaag_aagcta_first_occurrence_of_representations.push_back(0); - // AAG(0f1) - catcaag_aagcta_representations.push_back(0b000010); - catcaag_aagcta_read_ids.push_back(1); - catcaag_aagcta_positions_in_reads.push_back(0); - catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - // AGC(2r1) - catcaag_aagcta_representations.push_back(0b001001); - catcaag_aagcta_read_ids.push_back(1); - catcaag_aagcta_positions_in_reads.push_back(2); - catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - catcaag_aagcta_unique_representations.push_back(0b001001); - catcaag_aagcta_first_occurrence_of_representations.push_back(2); - // ATC(1f0) - catcaag_aagcta_representations.push_back(0b001101); - catcaag_aagcta_read_ids.push_back(0); - catcaag_aagcta_positions_in_reads.push_back(1); - catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_aagcta_unique_representations.push_back(0b001101); - catcaag_aagcta_first_occurrence_of_representations.push_back(3); - // ATG(0r0) - catcaag_aagcta_representations.push_back(0b001110); - catcaag_aagcta_read_ids.push_back(0); - catcaag_aagcta_positions_in_reads.push_back(0); - catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - catcaag_aagcta_unique_representations.push_back(0b001110); - catcaag_aagcta_first_occurrence_of_representations.push_back(4); - // CAA(3f0) - catcaag_aagcta_representations.push_back(0b010000); - catcaag_aagcta_read_ids.push_back(0); - catcaag_aagcta_positions_in_reads.push_back(3); - catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_aagcta_unique_representations.push_back(0b010000); - catcaag_aagcta_first_occurrence_of_representations.push_back(5); - // CTA(3f1) - catcaag_aagcta_representations.push_back(0b011100); - catcaag_aagcta_read_ids.push_back(1); - catcaag_aagcta_positions_in_reads.push_back(3); - catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_aagcta_unique_representations.push_back(0b011100); - catcaag_aagcta_first_occurrence_of_representations.push_back(6); - // trailing elements - catcaag_aagcta_first_occurrence_of_representations.push_back(7); - - const read_id_t catcaag_aagcta_number_of_reads = 2; - const read_id_t catcaag_aagcta_smallest_read_id = 0; - const read_id_t catcaag_aagcta_largest_read_id = 1; - const position_in_read_t catcaag_aagcta_number_of_basepairs_in_longest_read = 7; - const uint64_t catcaag_aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ************* IndexCacheHost tests ************* - - IndexDescriptor catcaag_index_descriptor(0, 1); - IndexDescriptor aagcta_index_descriptor(1, 1); - IndexDescriptor catcaag_aagcta_index_descriptor(0, 2); - std::vector catcaag_index_descriptors({catcaag_index_descriptor}); - std::vector aagcta_index_descriptors({aagcta_index_descriptor}); - std::vector catcaag_aagcta_separate_index_descriptors({catcaag_index_descriptor, aagcta_index_descriptor}); - std::vector catcaag_aagcta_one_index_descriptors({catcaag_aagcta_index_descriptor}); - - IndexCacheHost index_host_cache(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream); - - index_host_cache.generate_query_cache_content(catcaag_index_descriptors); - - auto index_query_catcaag = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); - check_if_index_is_correct(index_query_catcaag, - catcaag_representations, - catcaag_read_ids, - catcaag_positions_in_reads, - catcaag_directions_of_reads, - catcaag_unique_representations, - catcaag_first_occurrence_of_representations, - catcaag_number_of_reads, - catcaag_smallest_read_id, - catcaag_largest_read_id, - catcaag_number_of_basepairs_in_longest_read, - catcaag_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_1"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - - index_host_cache.generate_target_cache_content(aagcta_index_descriptors); - - index_query_catcaag = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); - check_if_index_is_correct(index_query_catcaag, - catcaag_representations, - catcaag_read_ids, - catcaag_positions_in_reads, - catcaag_directions_of_reads, - catcaag_unique_representations, - catcaag_first_occurrence_of_representations, - catcaag_number_of_reads, - catcaag_smallest_read_id, - catcaag_largest_read_id, - catcaag_number_of_basepairs_in_longest_read, - catcaag_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_2"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - auto index_target_aagcta = index_host_cache.get_index_from_target_cache(aagcta_index_descriptor); - check_if_index_is_correct(index_target_aagcta, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_3"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - - index_host_cache.generate_query_cache_content(aagcta_index_descriptors); - - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_index_descriptor)); - auto index_query_aagcta = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); - check_if_index_is_correct(index_query_aagcta, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_4"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - index_target_aagcta = index_host_cache.get_index_from_target_cache(aagcta_index_descriptor); - check_if_index_is_correct(index_target_aagcta, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_5"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - - index_host_cache.generate_query_cache_content(catcaag_aagcta_separate_index_descriptors); - - auto index_query_catcaag_separate = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); - check_if_index_is_correct(index_query_catcaag_separate, - catcaag_representations, - catcaag_read_ids, - catcaag_positions_in_reads, - catcaag_directions_of_reads, - catcaag_unique_representations, - catcaag_first_occurrence_of_representations, - catcaag_number_of_reads, - catcaag_smallest_read_id, - catcaag_largest_read_id, - catcaag_number_of_basepairs_in_longest_read, - catcaag_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_6"); - auto index_query_aagcta_separate = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); - check_if_index_is_correct(index_query_aagcta_separate, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_7"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - index_target_aagcta = index_host_cache.get_index_from_target_cache(aagcta_index_descriptor); - check_if_index_is_correct(index_target_aagcta, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_8"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - - index_host_cache.generate_target_cache_content(catcaag_aagcta_one_index_descriptors); - - index_query_catcaag_separate = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); - check_if_index_is_correct(index_query_catcaag_separate, - catcaag_representations, - catcaag_read_ids, - catcaag_positions_in_reads, - catcaag_directions_of_reads, - catcaag_unique_representations, - catcaag_first_occurrence_of_representations, - catcaag_number_of_reads, - catcaag_smallest_read_id, - catcaag_largest_read_id, - catcaag_number_of_basepairs_in_longest_read, - catcaag_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_9"); - index_query_aagcta_separate = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); - check_if_index_is_correct(index_query_aagcta_separate, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_10"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(aagcta_index_descriptor)); - auto catcaag_aagcta_target_aagcta = index_host_cache.get_index_from_target_cache(catcaag_aagcta_index_descriptor); - check_if_index_is_correct(catcaag_aagcta_target_aagcta, - catcaag_aagcta_representations, - catcaag_aagcta_read_ids, - catcaag_aagcta_positions_in_reads, - catcaag_aagcta_directions_of_reads, - catcaag_aagcta_unique_representations, - catcaag_aagcta_first_occurrence_of_representations, - catcaag_aagcta_number_of_reads, - catcaag_aagcta_smallest_read_id, - catcaag_aagcta_largest_read_id, - catcaag_aagcta_number_of_basepairs_in_longest_read, - catcaag_aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_same_query_and_target_11"); + { // create and destroy stream outside of current scope + const bool same_query_and_target = true; + std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/catcaag_aagcta.fasta"); + std::shared_ptr target_parser = query_parser; + DefaultDeviceAllocator allocator = create_default_device_allocator(); + const std::uint64_t k = 3; + const std::uint64_t w = 2; + const bool hash_representations = false; + const double filtering_parameter = 1.0; + + // ************* expected indices ************* + + // ** CATCAAG: AAG(4f), ATC(1f), CAA(3f), ATG(0r) + std::vector catcaag_representations; + std::vector catcaag_read_ids; + std::vector catcaag_positions_in_reads; + std::vector catcaag_directions_of_reads; + std::vector catcaag_unique_representations; + std::vector catcaag_first_occurrence_of_representations; + + // AAG(4f) + catcaag_representations.push_back(0b000010); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(4); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b000010); + catcaag_first_occurrence_of_representations.push_back(0); + // ATC(1f) + catcaag_representations.push_back(0b001101); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(1); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b001101); + catcaag_first_occurrence_of_representations.push_back(1); + // ATG(0r) + catcaag_representations.push_back(0b001110); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(0); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + catcaag_unique_representations.push_back(0b001110); + catcaag_first_occurrence_of_representations.push_back(2); + // CAA(3f) + catcaag_representations.push_back(0b010000); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(3); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b010000); + catcaag_first_occurrence_of_representations.push_back(3); + // trailing elements + catcaag_first_occurrence_of_representations.push_back(4); + + const read_id_t catcaag_number_of_reads = 1; + const std::vector catcaag_read_ids_to_read_names({"read_0"}); + const std::vector catcaag_read_ids_to_read_lengths({7}); + const read_id_t catcaag_smallest_read_id = 0; + const read_id_t catcaag_largest_read_id = 0; + const position_in_read_t catcaag_number_of_basepairs_in_longest_read = 7; + const uint64_t catcaag_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) + std::vector aagcta_representations; + std::vector aagcta_read_ids; + std::vector aagcta_positions_in_reads; + std::vector aagcta_directions_of_reads; + std::vector aagcta_unique_representations; + std::vector aagcta_first_occurrence_of_representations; + + // AAG(0f) + aagcta_representations.push_back(0b000010); + aagcta_read_ids.push_back(1); + aagcta_positions_in_reads.push_back(0); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b000010); + aagcta_first_occurrence_of_representations.push_back(0); + // AGC(2r) + aagcta_representations.push_back(0b001001); + aagcta_read_ids.push_back(1); + aagcta_positions_in_reads.push_back(2); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + aagcta_unique_representations.push_back(0b001001); + aagcta_first_occurrence_of_representations.push_back(1); + // CTA(3f) + aagcta_representations.push_back(0b011100); + aagcta_read_ids.push_back(1); + aagcta_positions_in_reads.push_back(3); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b011100); + aagcta_first_occurrence_of_representations.push_back(2); + // trailing elements + aagcta_first_occurrence_of_representations.push_back(3); + + const read_id_t aagcta_number_of_reads = 1; + const read_id_t aagcta_smallest_read_id = 1; + const read_id_t aagcta_largest_read_id = 1; + const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; + const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ** CATCAAG_AAGCTA: AAG(4f0), AAG(0f1), AGC(2r1), ATC(1f0), ATG(0r0), CAA(3f0), CTA(3f1) + std::vector catcaag_aagcta_representations; + std::vector catcaag_aagcta_read_ids; + std::vector catcaag_aagcta_positions_in_reads; + std::vector catcaag_aagcta_directions_of_reads; + std::vector catcaag_aagcta_unique_representations; + std::vector catcaag_aagcta_first_occurrence_of_representations; + + // AAG(4f0) + catcaag_aagcta_representations.push_back(0b000010); + catcaag_aagcta_read_ids.push_back(0); + catcaag_aagcta_positions_in_reads.push_back(4); + catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_aagcta_unique_representations.push_back(0b000010); + catcaag_aagcta_first_occurrence_of_representations.push_back(0); + // AAG(0f1) + catcaag_aagcta_representations.push_back(0b000010); + catcaag_aagcta_read_ids.push_back(1); + catcaag_aagcta_positions_in_reads.push_back(0); + catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + // AGC(2r1) + catcaag_aagcta_representations.push_back(0b001001); + catcaag_aagcta_read_ids.push_back(1); + catcaag_aagcta_positions_in_reads.push_back(2); + catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + catcaag_aagcta_unique_representations.push_back(0b001001); + catcaag_aagcta_first_occurrence_of_representations.push_back(2); + // ATC(1f0) + catcaag_aagcta_representations.push_back(0b001101); + catcaag_aagcta_read_ids.push_back(0); + catcaag_aagcta_positions_in_reads.push_back(1); + catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_aagcta_unique_representations.push_back(0b001101); + catcaag_aagcta_first_occurrence_of_representations.push_back(3); + // ATG(0r0) + catcaag_aagcta_representations.push_back(0b001110); + catcaag_aagcta_read_ids.push_back(0); + catcaag_aagcta_positions_in_reads.push_back(0); + catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + catcaag_aagcta_unique_representations.push_back(0b001110); + catcaag_aagcta_first_occurrence_of_representations.push_back(4); + // CAA(3f0) + catcaag_aagcta_representations.push_back(0b010000); + catcaag_aagcta_read_ids.push_back(0); + catcaag_aagcta_positions_in_reads.push_back(3); + catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_aagcta_unique_representations.push_back(0b010000); + catcaag_aagcta_first_occurrence_of_representations.push_back(5); + // CTA(3f1) + catcaag_aagcta_representations.push_back(0b011100); + catcaag_aagcta_read_ids.push_back(1); + catcaag_aagcta_positions_in_reads.push_back(3); + catcaag_aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_aagcta_unique_representations.push_back(0b011100); + catcaag_aagcta_first_occurrence_of_representations.push_back(6); + // trailing elements + catcaag_aagcta_first_occurrence_of_representations.push_back(7); + + const read_id_t catcaag_aagcta_number_of_reads = 2; + const read_id_t catcaag_aagcta_smallest_read_id = 0; + const read_id_t catcaag_aagcta_largest_read_id = 1; + const position_in_read_t catcaag_aagcta_number_of_basepairs_in_longest_read = 7; + const uint64_t catcaag_aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ************* IndexCacheHost tests ************* + + IndexDescriptor catcaag_index_descriptor(0, 1); + IndexDescriptor aagcta_index_descriptor(1, 1); + IndexDescriptor catcaag_aagcta_index_descriptor(0, 2); + std::vector catcaag_index_descriptors({catcaag_index_descriptor}); + std::vector aagcta_index_descriptors({aagcta_index_descriptor}); + std::vector catcaag_aagcta_separate_index_descriptors({catcaag_index_descriptor, aagcta_index_descriptor}); + std::vector catcaag_aagcta_one_index_descriptors({catcaag_aagcta_index_descriptor}); + + IndexCacheHost index_host_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream); + + index_host_cache.generate_query_cache_content(catcaag_index_descriptors); + + auto index_query_catcaag = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); + check_if_index_is_correct(index_query_catcaag, + catcaag_representations, + catcaag_read_ids, + catcaag_positions_in_reads, + catcaag_directions_of_reads, + catcaag_unique_representations, + catcaag_first_occurrence_of_representations, + catcaag_number_of_reads, + catcaag_smallest_read_id, + catcaag_largest_read_id, + catcaag_number_of_basepairs_in_longest_read, + catcaag_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_1"); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + + index_host_cache.generate_target_cache_content(aagcta_index_descriptors); + + index_query_catcaag = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); + check_if_index_is_correct(index_query_catcaag, + catcaag_representations, + catcaag_read_ids, + catcaag_positions_in_reads, + catcaag_directions_of_reads, + catcaag_unique_representations, + catcaag_first_occurrence_of_representations, + catcaag_number_of_reads, + catcaag_smallest_read_id, + catcaag_largest_read_id, + catcaag_number_of_basepairs_in_longest_read, + catcaag_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_2"); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); + auto index_target_aagcta = index_host_cache.get_index_from_target_cache(aagcta_index_descriptor); + check_if_index_is_correct(index_target_aagcta, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_3"); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + + index_host_cache.generate_query_cache_content(aagcta_index_descriptors); + + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_index_descriptor)); + auto index_query_aagcta = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); + check_if_index_is_correct(index_query_aagcta, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_4"); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); + index_target_aagcta = index_host_cache.get_index_from_target_cache(aagcta_index_descriptor); + check_if_index_is_correct(index_target_aagcta, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_5"); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + + index_host_cache.generate_query_cache_content(catcaag_aagcta_separate_index_descriptors); + + auto index_query_catcaag_separate = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); + check_if_index_is_correct(index_query_catcaag_separate, + catcaag_representations, + catcaag_read_ids, + catcaag_positions_in_reads, + catcaag_directions_of_reads, + catcaag_unique_representations, + catcaag_first_occurrence_of_representations, + catcaag_number_of_reads, + catcaag_smallest_read_id, + catcaag_largest_read_id, + catcaag_number_of_basepairs_in_longest_read, + catcaag_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_6"); + auto index_query_aagcta_separate = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); + check_if_index_is_correct(index_query_aagcta_separate, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_7"); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); + index_target_aagcta = index_host_cache.get_index_from_target_cache(aagcta_index_descriptor); + check_if_index_is_correct(index_target_aagcta, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_8"); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + + index_host_cache.generate_target_cache_content(catcaag_aagcta_one_index_descriptors); + + index_query_catcaag_separate = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); + check_if_index_is_correct(index_query_catcaag_separate, + catcaag_representations, + catcaag_read_ids, + catcaag_positions_in_reads, + catcaag_directions_of_reads, + catcaag_unique_representations, + catcaag_first_occurrence_of_representations, + catcaag_number_of_reads, + catcaag_smallest_read_id, + catcaag_largest_read_id, + catcaag_number_of_basepairs_in_longest_read, + catcaag_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_9"); + index_query_aagcta_separate = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); + check_if_index_is_correct(index_query_aagcta_separate, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_10"); + ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(aagcta_index_descriptor)); + auto catcaag_aagcta_target_aagcta = index_host_cache.get_index_from_target_cache(catcaag_aagcta_index_descriptor); + check_if_index_is_correct(catcaag_aagcta_target_aagcta, + catcaag_aagcta_representations, + catcaag_aagcta_read_ids, + catcaag_aagcta_positions_in_reads, + catcaag_aagcta_directions_of_reads, + catcaag_aagcta_unique_representations, + catcaag_aagcta_first_occurrence_of_representations, + catcaag_aagcta_number_of_reads, + catcaag_aagcta_smallest_read_id, + catcaag_aagcta_largest_read_id, + catcaag_aagcta_number_of_basepairs_in_longest_read, + catcaag_aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_same_query_and_target_11"); + } GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); @@ -595,167 +597,169 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta cudaStream_t cuda_stream; GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); - const bool same_query_and_target = false; - std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/aagcta.fasta"); - std::shared_ptr target_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/catcaag.fasta"); - DefaultDeviceAllocator allocator = create_default_device_allocator(); - const std::uint64_t k = 3; - const std::uint64_t w = 2; - const bool hash_representations = false; - const double filtering_parameter = 1.0; - - // ************* expected indices ************* - - // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) - std::vector aagcta_representations; - std::vector aagcta_read_ids; - std::vector aagcta_positions_in_reads; - std::vector aagcta_directions_of_reads; - std::vector aagcta_unique_representations; - std::vector aagcta_first_occurrence_of_representations; - - // AAG(0f) - aagcta_representations.push_back(0b000010); - aagcta_read_ids.push_back(0); - aagcta_positions_in_reads.push_back(0); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b000010); - aagcta_first_occurrence_of_representations.push_back(0); - // AGC(2r) - aagcta_representations.push_back(0b001001); - aagcta_read_ids.push_back(0); - aagcta_positions_in_reads.push_back(2); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - aagcta_unique_representations.push_back(0b001001); - aagcta_first_occurrence_of_representations.push_back(1); - // CTA(3f) - aagcta_representations.push_back(0b011100); - aagcta_read_ids.push_back(0); - aagcta_positions_in_reads.push_back(3); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b011100); - aagcta_first_occurrence_of_representations.push_back(2); - // trailing elements - aagcta_first_occurrence_of_representations.push_back(3); - - const read_id_t aagcta_number_of_reads = 1; - const read_id_t aagcta_smallest_read_id = 0; - const read_id_t aagcta_largest_read_id = 0; - const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; - const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ** CATCAAG: AAG(4f), ATC(1f), CAA(3f), ATG(0r) - std::vector catcaag_representations; - std::vector catcaag_read_ids; - std::vector catcaag_positions_in_reads; - std::vector catcaag_directions_of_reads; - std::vector catcaag_unique_representations; - std::vector catcaag_first_occurrence_of_representations; - - // AAG(4f) - catcaag_representations.push_back(0b000010); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(4); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b000010); - catcaag_first_occurrence_of_representations.push_back(0); - // ATC(1f) - catcaag_representations.push_back(0b001101); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(1); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b001101); - catcaag_first_occurrence_of_representations.push_back(1); - // ATG(0r) - catcaag_representations.push_back(0b001110); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(0); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - catcaag_unique_representations.push_back(0b001110); - catcaag_first_occurrence_of_representations.push_back(2); - // CAA(3f) - catcaag_representations.push_back(0b010000); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(3); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b010000); - catcaag_first_occurrence_of_representations.push_back(3); - // trailing elements - catcaag_first_occurrence_of_representations.push_back(4); - - const read_id_t catcaag_number_of_reads = 1; - const read_id_t catcaag_smallest_read_id = 0; - const read_id_t catcaag_largest_read_id = 0; - const position_in_read_t catcaag_number_of_basepairs_in_longest_read = 7; - const uint64_t catcaag_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ************* IndexCacheHost tests ************* - - // both descriptors are the same, but they are going to be used with different parsers - IndexDescriptor index_descriptor(0, 1); - std::vector index_descriptors({index_descriptor}); - - IndexCacheHost index_host_cache(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream); - - index_host_cache.generate_query_cache_content(index_descriptors); - - auto index_query_aagcta = index_host_cache.get_index_from_query_cache(index_descriptor); - check_if_index_is_correct(index_query_aagcta, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_not_the_same_query_and_target_1"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(index_descriptor)); - - index_host_cache.generate_target_cache_content(index_descriptors); - - index_query_aagcta = index_host_cache.get_index_from_query_cache(index_descriptor); - check_if_index_is_correct(index_query_aagcta, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_not_the_same_query_and_target_2"); - auto index_target_catcaag = index_host_cache.get_index_from_target_cache(index_descriptor); - check_if_index_is_correct(index_target_catcaag, - catcaag_representations, - catcaag_read_ids, - catcaag_positions_in_reads, - catcaag_directions_of_reads, - catcaag_unique_representations, - catcaag_first_occurrence_of_representations, - catcaag_number_of_reads, - catcaag_smallest_read_id, - catcaag_largest_read_id, - catcaag_number_of_basepairs_in_longest_read, - catcaag_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_not_the_same_query_and_target_3"); + { // create and destroy stream outside of current scope + const bool same_query_and_target = false; + std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/aagcta.fasta"); + std::shared_ptr target_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/catcaag.fasta"); + DefaultDeviceAllocator allocator = create_default_device_allocator(); + const std::uint64_t k = 3; + const std::uint64_t w = 2; + const bool hash_representations = false; + const double filtering_parameter = 1.0; + + // ************* expected indices ************* + + // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) + std::vector aagcta_representations; + std::vector aagcta_read_ids; + std::vector aagcta_positions_in_reads; + std::vector aagcta_directions_of_reads; + std::vector aagcta_unique_representations; + std::vector aagcta_first_occurrence_of_representations; + + // AAG(0f) + aagcta_representations.push_back(0b000010); + aagcta_read_ids.push_back(0); + aagcta_positions_in_reads.push_back(0); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b000010); + aagcta_first_occurrence_of_representations.push_back(0); + // AGC(2r) + aagcta_representations.push_back(0b001001); + aagcta_read_ids.push_back(0); + aagcta_positions_in_reads.push_back(2); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + aagcta_unique_representations.push_back(0b001001); + aagcta_first_occurrence_of_representations.push_back(1); + // CTA(3f) + aagcta_representations.push_back(0b011100); + aagcta_read_ids.push_back(0); + aagcta_positions_in_reads.push_back(3); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b011100); + aagcta_first_occurrence_of_representations.push_back(2); + // trailing elements + aagcta_first_occurrence_of_representations.push_back(3); + + const read_id_t aagcta_number_of_reads = 1; + const read_id_t aagcta_smallest_read_id = 0; + const read_id_t aagcta_largest_read_id = 0; + const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; + const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ** CATCAAG: AAG(4f), ATC(1f), CAA(3f), ATG(0r) + std::vector catcaag_representations; + std::vector catcaag_read_ids; + std::vector catcaag_positions_in_reads; + std::vector catcaag_directions_of_reads; + std::vector catcaag_unique_representations; + std::vector catcaag_first_occurrence_of_representations; + + // AAG(4f) + catcaag_representations.push_back(0b000010); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(4); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b000010); + catcaag_first_occurrence_of_representations.push_back(0); + // ATC(1f) + catcaag_representations.push_back(0b001101); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(1); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b001101); + catcaag_first_occurrence_of_representations.push_back(1); + // ATG(0r) + catcaag_representations.push_back(0b001110); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(0); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + catcaag_unique_representations.push_back(0b001110); + catcaag_first_occurrence_of_representations.push_back(2); + // CAA(3f) + catcaag_representations.push_back(0b010000); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(3); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b010000); + catcaag_first_occurrence_of_representations.push_back(3); + // trailing elements + catcaag_first_occurrence_of_representations.push_back(4); + + const read_id_t catcaag_number_of_reads = 1; + const read_id_t catcaag_smallest_read_id = 0; + const read_id_t catcaag_largest_read_id = 0; + const position_in_read_t catcaag_number_of_basepairs_in_longest_read = 7; + const uint64_t catcaag_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ************* IndexCacheHost tests ************* + + // both descriptors are the same, but they are going to be used with different parsers + IndexDescriptor index_descriptor(0, 1); + std::vector index_descriptors({index_descriptor}); + + IndexCacheHost index_host_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream); + + index_host_cache.generate_query_cache_content(index_descriptors); + + auto index_query_aagcta = index_host_cache.get_index_from_query_cache(index_descriptor); + check_if_index_is_correct(index_query_aagcta, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_not_the_same_query_and_target_1"); + ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(index_descriptor)); + + index_host_cache.generate_target_cache_content(index_descriptors); + + index_query_aagcta = index_host_cache.get_index_from_query_cache(index_descriptor); + check_if_index_is_correct(index_query_aagcta, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_not_the_same_query_and_target_2"); + auto index_target_catcaag = index_host_cache.get_index_from_target_cache(index_descriptor); + check_if_index_is_correct(index_target_catcaag, + catcaag_representations, + catcaag_read_ids, + catcaag_positions_in_reads, + catcaag_directions_of_reads, + catcaag_unique_representations, + catcaag_first_occurrence_of_representations, + catcaag_number_of_reads, + catcaag_smallest_read_id, + catcaag_largest_read_id, + catcaag_number_of_basepairs_in_longest_read, + catcaag_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_not_the_same_query_and_target_3"); + } GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); @@ -768,140 +772,145 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) cudaStream_t cuda_stream; GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); - const bool same_query_and_target = true; - std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/aagcta.fasta"); - std::shared_ptr target_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/aagcta.fasta"); - DefaultDeviceAllocator allocator = create_default_device_allocator(); - const std::uint64_t k = 3; - const std::uint64_t w = 2; - const bool hash_representations = false; - const double filtering_parameter = 1.0; - - // ************* expected indices ************* - - // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) - std::vector aagcta_representations; - std::vector aagcta_read_ids; - std::vector aagcta_positions_in_reads; - std::vector aagcta_directions_of_reads; - std::vector aagcta_unique_representations; - std::vector aagcta_first_occurrence_of_representations; - - // AAG(0f) - aagcta_representations.push_back(0b000010); - aagcta_read_ids.push_back(0); - aagcta_positions_in_reads.push_back(0); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b000010); - aagcta_first_occurrence_of_representations.push_back(0); - // AGC(2r) - aagcta_representations.push_back(0b001001); - aagcta_read_ids.push_back(0); - aagcta_positions_in_reads.push_back(2); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - aagcta_unique_representations.push_back(0b001001); - aagcta_first_occurrence_of_representations.push_back(1); - // CTA(3f) - aagcta_representations.push_back(0b011100); - aagcta_read_ids.push_back(0); - aagcta_positions_in_reads.push_back(3); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b011100); - aagcta_first_occurrence_of_representations.push_back(2); - // trailing elements - aagcta_first_occurrence_of_representations.push_back(3); - - const read_id_t aagcta_number_of_reads = 1; - const read_id_t aagcta_smallest_read_id = 0; - const read_id_t aagcta_largest_read_id = 0; - const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; - const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ************* IndexCacheDevice tests ************* - - IndexDescriptor index_descriptor(0, 1); - std::vector index_descriptors({index_descriptor}); - - auto index_cache_host = std::make_shared(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream); - - index_cache_host->generate_query_cache_content(index_descriptors, - index_descriptors); - index_cache_host->generate_target_cache_content(index_descriptors, - index_descriptors); - - auto index_query_temp_device_cache = index_cache_host->get_index_from_query_cache(index_descriptor); - auto index_query_copy_from_host = index_cache_host->get_index_from_query_cache(index_descriptor); - auto index_target_temp_device_cache = index_cache_host->get_index_from_target_cache(index_descriptor); - auto index_target_copy_from_host = index_cache_host->get_index_from_target_cache(index_descriptor); - - ASSERT_EQ(index_query_temp_device_cache, index_target_temp_device_cache); - ASSERT_NE(index_query_temp_device_cache, index_query_copy_from_host); - ASSERT_NE(index_target_temp_device_cache, index_target_copy_from_host); - - check_if_index_is_correct(index_query_temp_device_cache, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_keep_on_device_1"); - check_if_index_is_correct(index_query_copy_from_host, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_keep_on_device_2"); - check_if_index_is_correct(index_target_temp_device_cache, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_keep_on_device_3"); - check_if_index_is_correct(index_target_copy_from_host, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_host_keep_on_device_4"); + { // create and destroy stream outside of current scope + const bool same_query_and_target = true; + std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/aagcta.fasta"); + std::shared_ptr target_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/aagcta.fasta"); + DefaultDeviceAllocator allocator = create_default_device_allocator(); + const std::uint64_t k = 3; + const std::uint64_t w = 2; + const bool hash_representations = false; + const double filtering_parameter = 1.0; + + // ************* expected indices ************* + + // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) + std::vector aagcta_representations; + std::vector aagcta_read_ids; + std::vector aagcta_positions_in_reads; + std::vector aagcta_directions_of_reads; + std::vector aagcta_unique_representations; + std::vector aagcta_first_occurrence_of_representations; + + // AAG(0f) + aagcta_representations.push_back(0b000010); + aagcta_read_ids.push_back(0); + aagcta_positions_in_reads.push_back(0); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b000010); + aagcta_first_occurrence_of_representations.push_back(0); + // AGC(2r) + aagcta_representations.push_back(0b001001); + aagcta_read_ids.push_back(0); + aagcta_positions_in_reads.push_back(2); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + aagcta_unique_representations.push_back(0b001001); + aagcta_first_occurrence_of_representations.push_back(1); + // CTA(3f) + aagcta_representations.push_back(0b011100); + aagcta_read_ids.push_back(0); + aagcta_positions_in_reads.push_back(3); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b011100); + aagcta_first_occurrence_of_representations.push_back(2); + // trailing elements + aagcta_first_occurrence_of_representations.push_back(3); + + const read_id_t aagcta_number_of_reads = 1; + const read_id_t aagcta_smallest_read_id = 0; + const read_id_t aagcta_largest_read_id = 0; + const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; + const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ************* IndexCacheDevice tests ************* + + IndexDescriptor index_descriptor(0, 1); + std::vector index_descriptors({index_descriptor}); + + auto index_cache_host = std::make_shared(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream); + + index_cache_host->generate_query_cache_content(index_descriptors, + index_descriptors); + index_cache_host->generate_target_cache_content(index_descriptors, + index_descriptors); + + auto index_query_temp_device_cache = index_cache_host->get_index_from_query_cache(index_descriptor); + auto index_query_copy_from_host = index_cache_host->get_index_from_query_cache(index_descriptor); + auto index_target_temp_device_cache = index_cache_host->get_index_from_target_cache(index_descriptor); + auto index_target_copy_from_host = index_cache_host->get_index_from_target_cache(index_descriptor); + + ASSERT_EQ(index_query_temp_device_cache, index_target_temp_device_cache); + ASSERT_NE(index_query_temp_device_cache, index_query_copy_from_host); + ASSERT_NE(index_target_temp_device_cache, index_target_copy_from_host); + + check_if_index_is_correct(index_query_temp_device_cache, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_keep_on_device_1"); + check_if_index_is_correct(index_query_copy_from_host, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_keep_on_device_2"); + check_if_index_is_correct(index_target_temp_device_cache, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_keep_on_device_3"); + check_if_index_is_correct(index_target_copy_from_host, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_host_keep_on_device_4"); + } + + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); + GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); } // *** Test IndexCacheDevice *** @@ -919,186 +928,188 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) cudaStream_t cuda_stream; GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); - const bool same_query_and_target = true; - std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/catcaag_aagcta.fasta"); - std::shared_ptr target_parser = query_parser; - DefaultDeviceAllocator allocator = create_default_device_allocator(); - const std::uint64_t k = 3; - const std::uint64_t w = 2; - const bool hash_representations = false; - const double filtering_parameter = 1.0; - - // ************* expected indices ************* - - // ** CATCAAG: AAG(4f), ATC(1f), CAA(3f), ATG(0r) - std::vector catcaag_representations; - std::vector catcaag_read_ids; - std::vector catcaag_positions_in_reads; - std::vector catcaag_directions_of_reads; - std::vector catcaag_unique_representations; - std::vector catcaag_first_occurrence_of_representations; - - // AAG(4f) - catcaag_representations.push_back(0b000010); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(4); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b000010); - catcaag_first_occurrence_of_representations.push_back(0); - // ATC(1f) - catcaag_representations.push_back(0b001101); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(1); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b001101); - catcaag_first_occurrence_of_representations.push_back(1); - // ATG(0r) - catcaag_representations.push_back(0b001110); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(0); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - catcaag_unique_representations.push_back(0b001110); - catcaag_first_occurrence_of_representations.push_back(2); - // CAA(3f) - catcaag_representations.push_back(0b010000); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(3); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b010000); - catcaag_first_occurrence_of_representations.push_back(3); - // trailing elements - catcaag_first_occurrence_of_representations.push_back(4); - - const read_id_t catcaag_number_of_reads = 1; - const read_id_t catcaag_smallest_read_id = 0; - const read_id_t catcaag_largest_read_id = 0; - const position_in_read_t catcaag_number_of_basepairs_in_longest_read = 7; - const uint64_t catcaag_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) - std::vector aagcta_representations; - std::vector aagcta_read_ids; - std::vector aagcta_positions_in_reads; - std::vector aagcta_directions_of_reads; - std::vector aagcta_unique_representations; - std::vector aagcta_first_occurrence_of_representations; - - // AAG(0f) - aagcta_representations.push_back(0b000010); - aagcta_read_ids.push_back(1); - aagcta_positions_in_reads.push_back(0); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b000010); - aagcta_first_occurrence_of_representations.push_back(0); - // AGC(2r) - aagcta_representations.push_back(0b001001); - aagcta_read_ids.push_back(1); - aagcta_positions_in_reads.push_back(2); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - aagcta_unique_representations.push_back(0b001001); - aagcta_first_occurrence_of_representations.push_back(1); - // CTA(3f) - aagcta_representations.push_back(0b011100); - aagcta_read_ids.push_back(1); - aagcta_positions_in_reads.push_back(3); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b011100); - aagcta_first_occurrence_of_representations.push_back(2); - // trailing elements - aagcta_first_occurrence_of_representations.push_back(3); - - const read_id_t aagcta_number_of_reads = 1; - const read_id_t aagcta_smallest_read_id = 1; - const read_id_t aagcta_largest_read_id = 1; - const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; - const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ************* IndexCacheDevice tests ************* - - IndexDescriptor catcaag_index_descriptor(0, 1); - IndexDescriptor aagcta_index_descriptor(1, 1); - std::vector catcaag_index_descriptors({catcaag_index_descriptor}); - std::vector aagcta_index_descriptors({aagcta_index_descriptor}); - std::vector catcaag_aagcta_index_descriptors({catcaag_index_descriptor, aagcta_index_descriptor}); - - auto index_cache_host = std::make_shared(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream); - - IndexCacheDevice index_cache_device(same_query_and_target, - index_cache_host); - - index_cache_host->generate_query_cache_content(catcaag_index_descriptors); - ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(catcaag_index_descriptor)); - index_cache_device.generate_query_cache_content(catcaag_index_descriptors); - auto index_query_catcaag = index_cache_device.get_index_from_query_cache(catcaag_index_descriptor); - check_if_index_is_correct(index_query_catcaag, - catcaag_representations, - catcaag_read_ids, - catcaag_positions_in_reads, - catcaag_directions_of_reads, - catcaag_unique_representations, - catcaag_first_occurrence_of_representations, - catcaag_number_of_reads, - catcaag_smallest_read_id, - catcaag_largest_read_id, - catcaag_number_of_basepairs_in_longest_read, - catcaag_maximum_kmer_size, - cuda_stream, - "test_index_cache_device_same_query_and_target_1"); - ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(aagcta_index_descriptor)); - - index_cache_host->generate_target_cache_content(catcaag_aagcta_index_descriptors); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(aagcta_index_descriptor)); - index_cache_device.generate_target_cache_content(catcaag_aagcta_index_descriptors); - - auto index_target_catcaag = index_cache_device.get_index_from_target_cache(catcaag_index_descriptor); - ASSERT_EQ(index_query_catcaag, index_target_catcaag); // check same object is used because same_query_and_target == true - check_if_index_is_correct(index_target_catcaag, - catcaag_representations, - catcaag_read_ids, - catcaag_positions_in_reads, - catcaag_directions_of_reads, - catcaag_unique_representations, - catcaag_first_occurrence_of_representations, - catcaag_number_of_reads, - catcaag_smallest_read_id, - catcaag_largest_read_id, - catcaag_number_of_basepairs_in_longest_read, - catcaag_maximum_kmer_size, - cuda_stream, - "test_index_cache_device_same_query_and_target_2"); - - auto index_target_aagcta = index_cache_device.get_index_from_target_cache(aagcta_index_descriptor); - check_if_index_is_correct(index_target_aagcta, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_device_same_query_and_target_3"); - - // get the same query and target indices again and make sure they point to the same objects as the last time - auto index_query_catcaag_1 = index_cache_device.get_index_from_query_cache(catcaag_index_descriptor); - auto index_target_aagcta_1 = index_cache_device.get_index_from_target_cache(aagcta_index_descriptor); - ASSERT_EQ(index_query_catcaag, index_query_catcaag_1); - ASSERT_EQ(index_target_aagcta, index_target_aagcta_1); + { // create and destroy stream outside of current scope + const bool same_query_and_target = true; + std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/catcaag_aagcta.fasta"); + std::shared_ptr target_parser = query_parser; + DefaultDeviceAllocator allocator = create_default_device_allocator(); + const std::uint64_t k = 3; + const std::uint64_t w = 2; + const bool hash_representations = false; + const double filtering_parameter = 1.0; + + // ************* expected indices ************* + + // ** CATCAAG: AAG(4f), ATC(1f), CAA(3f), ATG(0r) + std::vector catcaag_representations; + std::vector catcaag_read_ids; + std::vector catcaag_positions_in_reads; + std::vector catcaag_directions_of_reads; + std::vector catcaag_unique_representations; + std::vector catcaag_first_occurrence_of_representations; + + // AAG(4f) + catcaag_representations.push_back(0b000010); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(4); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b000010); + catcaag_first_occurrence_of_representations.push_back(0); + // ATC(1f) + catcaag_representations.push_back(0b001101); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(1); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b001101); + catcaag_first_occurrence_of_representations.push_back(1); + // ATG(0r) + catcaag_representations.push_back(0b001110); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(0); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + catcaag_unique_representations.push_back(0b001110); + catcaag_first_occurrence_of_representations.push_back(2); + // CAA(3f) + catcaag_representations.push_back(0b010000); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(3); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b010000); + catcaag_first_occurrence_of_representations.push_back(3); + // trailing elements + catcaag_first_occurrence_of_representations.push_back(4); + + const read_id_t catcaag_number_of_reads = 1; + const read_id_t catcaag_smallest_read_id = 0; + const read_id_t catcaag_largest_read_id = 0; + const position_in_read_t catcaag_number_of_basepairs_in_longest_read = 7; + const uint64_t catcaag_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) + std::vector aagcta_representations; + std::vector aagcta_read_ids; + std::vector aagcta_positions_in_reads; + std::vector aagcta_directions_of_reads; + std::vector aagcta_unique_representations; + std::vector aagcta_first_occurrence_of_representations; + + // AAG(0f) + aagcta_representations.push_back(0b000010); + aagcta_read_ids.push_back(1); + aagcta_positions_in_reads.push_back(0); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b000010); + aagcta_first_occurrence_of_representations.push_back(0); + // AGC(2r) + aagcta_representations.push_back(0b001001); + aagcta_read_ids.push_back(1); + aagcta_positions_in_reads.push_back(2); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + aagcta_unique_representations.push_back(0b001001); + aagcta_first_occurrence_of_representations.push_back(1); + // CTA(3f) + aagcta_representations.push_back(0b011100); + aagcta_read_ids.push_back(1); + aagcta_positions_in_reads.push_back(3); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b011100); + aagcta_first_occurrence_of_representations.push_back(2); + // trailing elements + aagcta_first_occurrence_of_representations.push_back(3); + + const read_id_t aagcta_number_of_reads = 1; + const read_id_t aagcta_smallest_read_id = 1; + const read_id_t aagcta_largest_read_id = 1; + const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; + const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ************* IndexCacheDevice tests ************* + + IndexDescriptor catcaag_index_descriptor(0, 1); + IndexDescriptor aagcta_index_descriptor(1, 1); + std::vector catcaag_index_descriptors({catcaag_index_descriptor}); + std::vector aagcta_index_descriptors({aagcta_index_descriptor}); + std::vector catcaag_aagcta_index_descriptors({catcaag_index_descriptor, aagcta_index_descriptor}); + + auto index_cache_host = std::make_shared(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream); + + IndexCacheDevice index_cache_device(same_query_and_target, + index_cache_host); + + index_cache_host->generate_query_cache_content(catcaag_index_descriptors); + ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(catcaag_index_descriptor)); + index_cache_device.generate_query_cache_content(catcaag_index_descriptors); + auto index_query_catcaag = index_cache_device.get_index_from_query_cache(catcaag_index_descriptor); + check_if_index_is_correct(index_query_catcaag, + catcaag_representations, + catcaag_read_ids, + catcaag_positions_in_reads, + catcaag_directions_of_reads, + catcaag_unique_representations, + catcaag_first_occurrence_of_representations, + catcaag_number_of_reads, + catcaag_smallest_read_id, + catcaag_largest_read_id, + catcaag_number_of_basepairs_in_longest_read, + catcaag_maximum_kmer_size, + cuda_stream, + "test_index_cache_device_same_query_and_target_1"); + ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(aagcta_index_descriptor)); + + index_cache_host->generate_target_cache_content(catcaag_aagcta_index_descriptors); + ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(aagcta_index_descriptor)); + index_cache_device.generate_target_cache_content(catcaag_aagcta_index_descriptors); + + auto index_target_catcaag = index_cache_device.get_index_from_target_cache(catcaag_index_descriptor); + ASSERT_EQ(index_query_catcaag, index_target_catcaag); // check same object is used because same_query_and_target == true + check_if_index_is_correct(index_target_catcaag, + catcaag_representations, + catcaag_read_ids, + catcaag_positions_in_reads, + catcaag_directions_of_reads, + catcaag_unique_representations, + catcaag_first_occurrence_of_representations, + catcaag_number_of_reads, + catcaag_smallest_read_id, + catcaag_largest_read_id, + catcaag_number_of_basepairs_in_longest_read, + catcaag_maximum_kmer_size, + cuda_stream, + "test_index_cache_device_same_query_and_target_2"); + + auto index_target_aagcta = index_cache_device.get_index_from_target_cache(aagcta_index_descriptor); + check_if_index_is_correct(index_target_aagcta, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_device_same_query_and_target_3"); + + // get the same query and target indices again and make sure they point to the same objects as the last time + auto index_query_catcaag_1 = index_cache_device.get_index_from_query_cache(catcaag_index_descriptor); + auto index_target_aagcta_1 = index_cache_device.get_index_from_target_cache(aagcta_index_descriptor); + ASSERT_EQ(index_query_catcaag, index_query_catcaag_1); + ASSERT_EQ(index_target_aagcta, index_target_aagcta_1); + } GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); @@ -1112,183 +1123,185 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ cudaStream_t cuda_stream; GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); - const bool same_query_and_target = false; - std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/aagcta.fasta"); - std::shared_ptr target_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/catcaag.fasta"); - DefaultDeviceAllocator allocator = create_default_device_allocator(); - const std::uint64_t k = 3; - const std::uint64_t w = 2; - const bool hash_representations = false; - const double filtering_parameter = 1.0; - - // ************* expected indices ************* - - // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) - std::vector aagcta_representations; - std::vector aagcta_read_ids; - std::vector aagcta_positions_in_reads; - std::vector aagcta_directions_of_reads; - std::vector aagcta_unique_representations; - std::vector aagcta_first_occurrence_of_representations; - - // AAG(0f) - aagcta_representations.push_back(0b000010); - aagcta_read_ids.push_back(0); - aagcta_positions_in_reads.push_back(0); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b000010); - aagcta_first_occurrence_of_representations.push_back(0); - // AGC(2r) - aagcta_representations.push_back(0b001001); - aagcta_read_ids.push_back(0); - aagcta_positions_in_reads.push_back(2); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - aagcta_unique_representations.push_back(0b001001); - aagcta_first_occurrence_of_representations.push_back(1); - // CTA(3f) - aagcta_representations.push_back(0b011100); - aagcta_read_ids.push_back(0); - aagcta_positions_in_reads.push_back(3); - aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - aagcta_unique_representations.push_back(0b011100); - aagcta_first_occurrence_of_representations.push_back(2); - // trailing elements - aagcta_first_occurrence_of_representations.push_back(3); - - const read_id_t aagcta_number_of_reads = 1; - const read_id_t aagcta_smallest_read_id = 0; - const read_id_t aagcta_largest_read_id = 0; - const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; - const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ** CATCAAG: AAG(4f), ATC(1f), CAA(3f), ATG(0r) - std::vector catcaag_representations; - std::vector catcaag_read_ids; - std::vector catcaag_positions_in_reads; - std::vector catcaag_directions_of_reads; - std::vector catcaag_unique_representations; - std::vector catcaag_first_occurrence_of_representations; - - // AAG(4f) - catcaag_representations.push_back(0b000010); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(4); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b000010); - catcaag_first_occurrence_of_representations.push_back(0); - // ATC(1f) - catcaag_representations.push_back(0b001101); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(1); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b001101); - catcaag_first_occurrence_of_representations.push_back(1); - // ATG(0r) - catcaag_representations.push_back(0b001110); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(0); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); - catcaag_unique_representations.push_back(0b001110); - catcaag_first_occurrence_of_representations.push_back(2); - // CAA(3f) - catcaag_representations.push_back(0b010000); - catcaag_read_ids.push_back(0); - catcaag_positions_in_reads.push_back(3); - catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); - catcaag_unique_representations.push_back(0b010000); - catcaag_first_occurrence_of_representations.push_back(3); - // trailing elements - catcaag_first_occurrence_of_representations.push_back(4); - - const read_id_t catcaag_number_of_reads = 1; - const read_id_t catcaag_smallest_read_id = 0; - const read_id_t catcaag_largest_read_id = 0; - const position_in_read_t catcaag_number_of_basepairs_in_longest_read = 7; - const uint64_t catcaag_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; - - // ************* IndexCacheDevice tests ************* - - // both descriptors are the same, but they are going to be used with different parsers - IndexDescriptor index_descriptor(0, 1); - std::vector index_descriptors({index_descriptor}); - - auto index_cache_host = std::make_shared(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream); - - IndexCacheDevice index_cache_device(same_query_and_target, - index_cache_host); - - index_cache_host->generate_query_cache_content(index_descriptors); - ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(index_descriptor)); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(index_descriptor)); - - index_cache_device.generate_query_cache_content(index_descriptors); - auto index_query = index_cache_device.get_index_from_query_cache(index_descriptor); - check_if_index_is_correct(index_query, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_device_not_the_same_query_and_target_1"); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(index_descriptor)); - - ASSERT_ANY_THROW(index_cache_device.generate_target_cache_content(index_descriptors)); - - index_cache_host->generate_target_cache_content(index_descriptors); - index_cache_device.generate_target_cache_content(index_descriptors); - - index_query = index_cache_device.get_index_from_query_cache(index_descriptor); - auto index_target = index_cache_device.get_index_from_target_cache(index_descriptor); - ASSERT_NE(index_query, index_target); - check_if_index_is_correct(index_query, - aagcta_representations, - aagcta_read_ids, - aagcta_positions_in_reads, - aagcta_directions_of_reads, - aagcta_unique_representations, - aagcta_first_occurrence_of_representations, - aagcta_number_of_reads, - aagcta_smallest_read_id, - aagcta_largest_read_id, - aagcta_number_of_basepairs_in_longest_read, - aagcta_maximum_kmer_size, - cuda_stream, - "test_index_cache_device_not_the_same_query_and_target_2"); - check_if_index_is_correct(index_target, - catcaag_representations, - catcaag_read_ids, - catcaag_positions_in_reads, - catcaag_directions_of_reads, - catcaag_unique_representations, - catcaag_first_occurrence_of_representations, - catcaag_number_of_reads, - catcaag_smallest_read_id, - catcaag_largest_read_id, - catcaag_number_of_basepairs_in_longest_read, - catcaag_maximum_kmer_size, - cuda_stream, - "test_index_cache_device_not_the_same_query_and_target_3"); - - // get the same query and target indices again and make sure they point to the same objects as the last time - auto index_query_1 = index_cache_device.get_index_from_query_cache(index_descriptor); - auto index_target_1 = index_cache_device.get_index_from_target_cache(index_descriptor); - ASSERT_EQ(index_query, index_query_1); - ASSERT_EQ(index_target, index_target_1); + { // create and destroy stream outside of current scope + const bool same_query_and_target = false; + std::shared_ptr query_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/aagcta.fasta"); + std::shared_ptr target_parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/catcaag.fasta"); + DefaultDeviceAllocator allocator = create_default_device_allocator(); + const std::uint64_t k = 3; + const std::uint64_t w = 2; + const bool hash_representations = false; + const double filtering_parameter = 1.0; + + // ************* expected indices ************* + + // ** AAGCTA: AAG(0f), AGC(2f), CTA(3f) + std::vector aagcta_representations; + std::vector aagcta_read_ids; + std::vector aagcta_positions_in_reads; + std::vector aagcta_directions_of_reads; + std::vector aagcta_unique_representations; + std::vector aagcta_first_occurrence_of_representations; + + // AAG(0f) + aagcta_representations.push_back(0b000010); + aagcta_read_ids.push_back(0); + aagcta_positions_in_reads.push_back(0); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b000010); + aagcta_first_occurrence_of_representations.push_back(0); + // AGC(2r) + aagcta_representations.push_back(0b001001); + aagcta_read_ids.push_back(0); + aagcta_positions_in_reads.push_back(2); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + aagcta_unique_representations.push_back(0b001001); + aagcta_first_occurrence_of_representations.push_back(1); + // CTA(3f) + aagcta_representations.push_back(0b011100); + aagcta_read_ids.push_back(0); + aagcta_positions_in_reads.push_back(3); + aagcta_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + aagcta_unique_representations.push_back(0b011100); + aagcta_first_occurrence_of_representations.push_back(2); + // trailing elements + aagcta_first_occurrence_of_representations.push_back(3); + + const read_id_t aagcta_number_of_reads = 1; + const read_id_t aagcta_smallest_read_id = 0; + const read_id_t aagcta_largest_read_id = 0; + const position_in_read_t aagcta_number_of_basepairs_in_longest_read = 6; + const uint64_t aagcta_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ** CATCAAG: AAG(4f), ATC(1f), CAA(3f), ATG(0r) + std::vector catcaag_representations; + std::vector catcaag_read_ids; + std::vector catcaag_positions_in_reads; + std::vector catcaag_directions_of_reads; + std::vector catcaag_unique_representations; + std::vector catcaag_first_occurrence_of_representations; + + // AAG(4f) + catcaag_representations.push_back(0b000010); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(4); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b000010); + catcaag_first_occurrence_of_representations.push_back(0); + // ATC(1f) + catcaag_representations.push_back(0b001101); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(1); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b001101); + catcaag_first_occurrence_of_representations.push_back(1); + // ATG(0r) + catcaag_representations.push_back(0b001110); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(0); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::REVERSE); + catcaag_unique_representations.push_back(0b001110); + catcaag_first_occurrence_of_representations.push_back(2); + // CAA(3f) + catcaag_representations.push_back(0b010000); + catcaag_read_ids.push_back(0); + catcaag_positions_in_reads.push_back(3); + catcaag_directions_of_reads.push_back(SketchElement::DirectionOfRepresentation::FORWARD); + catcaag_unique_representations.push_back(0b010000); + catcaag_first_occurrence_of_representations.push_back(3); + // trailing elements + catcaag_first_occurrence_of_representations.push_back(4); + + const read_id_t catcaag_number_of_reads = 1; + const read_id_t catcaag_smallest_read_id = 0; + const read_id_t catcaag_largest_read_id = 0; + const position_in_read_t catcaag_number_of_basepairs_in_longest_read = 7; + const uint64_t catcaag_maximum_kmer_size = sizeof(representation_t) * CHAR_BIT / 2; + + // ************* IndexCacheDevice tests ************* + + // both descriptors are the same, but they are going to be used with different parsers + IndexDescriptor index_descriptor(0, 1); + std::vector index_descriptors({index_descriptor}); + + auto index_cache_host = std::make_shared(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream); + + IndexCacheDevice index_cache_device(same_query_and_target, + index_cache_host); + + index_cache_host->generate_query_cache_content(index_descriptors); + ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(index_descriptor)); + ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(index_descriptor)); + + index_cache_device.generate_query_cache_content(index_descriptors); + auto index_query = index_cache_device.get_index_from_query_cache(index_descriptor); + check_if_index_is_correct(index_query, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_device_not_the_same_query_and_target_1"); + ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(index_descriptor)); + + ASSERT_ANY_THROW(index_cache_device.generate_target_cache_content(index_descriptors)); + + index_cache_host->generate_target_cache_content(index_descriptors); + index_cache_device.generate_target_cache_content(index_descriptors); + + index_query = index_cache_device.get_index_from_query_cache(index_descriptor); + auto index_target = index_cache_device.get_index_from_target_cache(index_descriptor); + ASSERT_NE(index_query, index_target); + check_if_index_is_correct(index_query, + aagcta_representations, + aagcta_read_ids, + aagcta_positions_in_reads, + aagcta_directions_of_reads, + aagcta_unique_representations, + aagcta_first_occurrence_of_representations, + aagcta_number_of_reads, + aagcta_smallest_read_id, + aagcta_largest_read_id, + aagcta_number_of_basepairs_in_longest_read, + aagcta_maximum_kmer_size, + cuda_stream, + "test_index_cache_device_not_the_same_query_and_target_2"); + check_if_index_is_correct(index_target, + catcaag_representations, + catcaag_read_ids, + catcaag_positions_in_reads, + catcaag_directions_of_reads, + catcaag_unique_representations, + catcaag_first_occurrence_of_representations, + catcaag_number_of_reads, + catcaag_smallest_read_id, + catcaag_largest_read_id, + catcaag_number_of_basepairs_in_longest_read, + catcaag_maximum_kmer_size, + cuda_stream, + "test_index_cache_device_not_the_same_query_and_target_3"); + + // get the same query and target indices again and make sure they point to the same objects as the last time + auto index_query_1 = index_cache_device.get_index_from_query_cache(index_descriptor); + auto index_target_1 = index_cache_device.get_index_from_target_cache(index_descriptor); + ASSERT_EQ(index_query, index_query_1); + ASSERT_EQ(index_target, index_target_1); + } GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); From 3f11b6de05cfdfd03172b665b3f0ab8b1bf0fc60 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 9 Jul 2020 14:19:28 +0200 Subject: [PATCH 010/281] [cudamapper] Overlapping copying of new indices to host with generation of other indices --- .../genomeworks/cudamapper/index.hpp | 17 +- cudamapper/src/index.cu | 6 +- cudamapper/src/index_cache.cu | 101 +++++++---- cudamapper/src/index_cache.cuh | 75 ++++++--- cudamapper/src/index_gpu.cuh | 17 +- cudamapper/src/index_host_copy.cu | 96 ++++++----- cudamapper/src/index_host_copy.cuh | 42 ++++- cudamapper/src/main.cu | 43 +++-- cudamapper/tests/Test_CudamapperIndexCache.cu | 159 +++++++++++------- 9 files changed, 354 insertions(+), 202 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index 998b847a8..c7faa02ce 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -92,7 +92,8 @@ class Index /// \param window_size w - the length of the sliding window used to find sketch elements (i.e. the number of adjacent kmers in a window, adjacent = shifted by one basepair) /// \param hash_representations if true, hash kmer representations /// \param filtering_parameter filter out all representations for which number_of_sketch_elements_with_that_representation/total_skech_elements >= filtering_parameter, filtering_parameter == 1.0 disables filtering - /// \param cuda_stream CUDA stream on which the work is to be done. Device arrays are also associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor is done + /// \param cuda_stream_generation CUDA stream on which index is generated. Device arrays are associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor has been done + /// \param cuda_stream_copy CUDA stream on which the index is copied to host if needed. Device arrays are associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor has been done /// \return instance of Index static std::unique_ptr create_index(DefaultDeviceAllocator allocator, @@ -101,9 +102,10 @@ class Index const read_id_t past_the_last_read_id, const std::uint64_t kmer_size, const std::uint64_t window_size, - const bool hash_representations = true, - const double filtering_parameter = 1.0, - const cudaStream_t cuda_stream = 0); + const bool hash_representations = true, + const double filtering_parameter = 1.0, + const cudaStream_t cuda_stream_generation = 0, + const cudaStream_t cuda_stream_copy = 0); }; /// IndexHostCopyBase - Creates and maintains a copy of computed IndexGPU elements on the host, then allows to retrieve target @@ -133,6 +135,9 @@ class IndexHostCopyBase /// \brief virtual destructor virtual ~IndexHostCopyBase() = default; + /// \brief waits for copy to host to be done + virtual void finish_copying_to_host() const = 0; + /// \brief returns an array of representations of sketch elements (stored on host) /// \return an array of representations of sketch elements virtual const ArrayView representations() const = 0; @@ -177,8 +182,8 @@ class IndexHostCopyBase /// \return window_size_ virtual std::uint64_t window_size() const = 0; - /// \brief Constructor - /// \brief creates a copy of input processed index on the host + /// \brief Starts creating a copy of index on the host + /// Copy is done asynchronously and one should wait for it to finish with finish_copying_to_host() /// \param index - pointer to computed index parameters (vectors of sketch elements) on GPU /// \param first_read_id - representing smallest read_id in index /// \param kmer_size - number of basepairs in a k-mer diff --git a/cudamapper/src/index.cu b/cudamapper/src/index.cu index 9ce28c487..a8ae8e845 100644 --- a/cudamapper/src/index.cu +++ b/cudamapper/src/index.cu @@ -30,7 +30,8 @@ std::unique_ptr Index::create_index(DefaultDeviceAllocator allocator, const std::uint64_t window_size, const bool hash_representations, const double filtering_parameter, - const cudaStream_t cuda_stream) + const cudaStream_t cuda_stream_generation, + const cudaStream_t cuda_stream_copy) { GW_NVTX_RANGE(profiler, "create_index"); return std::make_unique>(allocator, @@ -41,7 +42,8 @@ std::unique_ptr Index::create_index(DefaultDeviceAllocator allocator, window_size, hash_representations, filtering_parameter, - cuda_stream); + cuda_stream_generation, + cuda_stream_copy); } std::unique_ptr IndexHostCopyBase::create_cache(const Index& index, diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index bba3d1a76..cd622766b 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -34,7 +34,8 @@ IndexCacheHost::IndexCacheHost(const bool same_query_and_target, const std::uint64_t window_size, const bool hash_representations, const double filtering_parameter, - const cudaStream_t cuda_stream) + const cudaStream_t cuda_stream_generation, + const cudaStream_t cuda_stream_copy) : same_query_and_target_(same_query_and_target) , allocator_(allocator) , query_parser_(query_parser) @@ -43,28 +44,41 @@ IndexCacheHost::IndexCacheHost(const bool same_query_and_target, , window_size_(window_size) , hash_representations_(hash_representations) , filtering_parameter_(filtering_parameter) - , cuda_stream_(cuda_stream) + , cuda_stream_generation_(cuda_stream_generation) + , cuda_stream_copy_(cuda_stream_copy) { } -void IndexCacheHost::generate_query_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - const bool skip_copy_to_host) +void IndexCacheHost::start_generating_query_cache_content(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device, + const bool skip_copy_to_host) { - generate_cache_content(descriptors_of_indices_to_cache, - descriptors_of_indices_to_keep_on_device, - skip_copy_to_host, - CacheSelector::query_cache); + start_generating_cache_content(descriptors_of_indices_to_cache, + descriptors_of_indices_to_keep_on_device, + skip_copy_to_host, + CacheSelector::query_cache); } -void IndexCacheHost::generate_target_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - const bool skip_copy_to_host) +void IndexCacheHost::finish_generating_query_cache_content() { - generate_cache_content(descriptors_of_indices_to_cache, - descriptors_of_indices_to_keep_on_device, - skip_copy_to_host, - CacheSelector::target_cache); + finish_generating_cache_content(CacheSelector::query_cache); + return; +} + +void IndexCacheHost::start_generating_target_cache_content(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device, + const bool skip_copy_to_host) +{ + start_generating_cache_content(descriptors_of_indices_to_cache, + descriptors_of_indices_to_keep_on_device, + skip_copy_to_host, + CacheSelector::target_cache); +} + +void IndexCacheHost::finish_generating_target_cache_content() +{ + finish_generating_cache_content(CacheSelector::target_cache); + return; } std::shared_ptr IndexCacheHost::get_index_from_query_cache(const IndexDescriptor& descriptor_of_index_to_cache) @@ -79,20 +93,21 @@ std::shared_ptr IndexCacheHost::get_index_from_target_cache(const IndexDe CacheSelector::target_cache); } -void IndexCacheHost::generate_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - const bool skip_copy_to_host, - const CacheSelector which_cache) +void IndexCacheHost::start_generating_cache_content(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device, + const bool skip_copy_to_host, + const CacheSelector which_cache) { // skip_copy_to_host only makes sense if descriptors_of_indices_to_cache and descriptors_of_indices_to_keep_on_device are the same // otherwise some indices would be created and not saved on either host or device assert(!skip_copy_to_host || (descriptors_of_indices_to_cache == descriptors_of_indices_to_keep_on_device)); - cache_type_t& cache_to_edit = (CacheSelector::query_cache == which_cache) ? query_cache_ : target_cache_; - const cache_type_t& cache_to_check = (CacheSelector::query_cache == which_cache) ? target_cache_ : query_cache_; - device_cache_type_t& temp_device_cache_to_edit = (CacheSelector::query_cache == which_cache) ? query_temp_device_cache_ : target_temp_device_cache_; - const device_cache_type_t& temp_device_cache_to_check = (CacheSelector::query_cache == which_cache) ? target_temp_device_cache_ : query_temp_device_cache_; - const genomeworks::io::FastaParser* parser = (CacheSelector::query_cache == which_cache) ? query_parser_.get() : target_parser_.get(); + cache_type_t& cache_to_edit = (CacheSelector::query_cache == which_cache) ? query_cache_ : target_cache_; + const cache_type_t& cache_to_check = (CacheSelector::query_cache == which_cache) ? target_cache_ : query_cache_; + device_cache_type_t& temp_device_cache_to_edit = (CacheSelector::query_cache == which_cache) ? query_temp_device_cache_ : target_temp_device_cache_; + const device_cache_type_t& temp_device_cache_to_check = (CacheSelector::query_cache == which_cache) ? target_temp_device_cache_ : query_temp_device_cache_; + const genomeworks::io::FastaParser* parser = (CacheSelector::query_cache == which_cache) ? query_parser_.get() : target_parser_.get(); + std::vector>& indices_in_progress = (CacheSelector::query_cache == which_cache) ? query_indices_in_progress : target_indices_in_progress; // convert descriptors_of_indices_to_keep_on_device into set for faster search std::unordered_set descriptors_of_indices_to_keep_on_device_set(begin(descriptors_of_indices_to_keep_on_device), @@ -109,9 +124,9 @@ void IndexCacheHost::generate_cache_content(const std::vector& std::shared_ptr index_on_host = nullptr; std::shared_ptr index_on_device = nullptr; + // if query and target are the same check if this index is already cached in the other cache if (same_query_and_target_) { - // check if the same index already exists in the other cache auto existing_cache = cache_to_check.find(descriptor_of_index_to_cache); if (existing_cache != cache_to_check.end()) { @@ -125,15 +140,15 @@ void IndexCacheHost::generate_cache_content(const std::vector& } else { - index_on_device = index_on_host->copy_index_to_device(allocator_, cuda_stream_); + index_on_device = index_on_host->copy_index_to_device(allocator_, cuda_stream_generation_); } } } } + // if index was not found in the other cache check current cache and generate index if needed if (nullptr == index_on_host) { - // check if this index is already cached in this cache auto existing_cache = cache_to_edit.find(descriptor_of_index_to_cache); if (existing_cache != cache_to_edit.end()) { @@ -141,7 +156,7 @@ void IndexCacheHost::generate_cache_content(const std::vector& index_on_host = existing_cache->second; if (keep_on_device) { - index_on_device = index_on_host->copy_index_to_device(allocator_, cuda_stream_); + index_on_device = index_on_host->copy_index_to_device(allocator_, cuda_stream_generation_); } } else @@ -155,15 +170,22 @@ void IndexCacheHost::generate_cache_content(const std::vector& window_size_, hash_representations_, filtering_parameter_, - cuda_stream_); + cuda_stream_generation_); + // copy it to host memory if (!skip_copy_to_host) { + // wait for index to be generated on cuda_stream_generation_ before copying it on cuda_stream_copy_ + // TODO: do this sync using an event + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation_)); + index_on_host = IndexHostCopy::create_cache(*index_on_device, descriptor_of_index_to_cache.first_read(), kmer_size_, window_size_, - cuda_stream_); + cuda_stream_copy_); + // index is being copied from device to host which is done asynchronously + indices_in_progress.push_back(index_on_host); } } } @@ -184,6 +206,21 @@ void IndexCacheHost::generate_cache_content(const std::vector& std::swap(new_cache, cache_to_edit); } +void IndexCacheHost::finish_generating_cache_content(CacheSelector which_cache) +{ + std::vector>& indices_in_progress = (CacheSelector::query_cache == which_cache) ? query_indices_in_progress : target_indices_in_progress; + + // wait for all indices to be generated + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation_)); + + // wait for all indices to finish being copied to host memory + for (auto& index_in_progress : indices_in_progress) + { + index_in_progress->finish_copying_to_host(); + } + indices_in_progress.clear(); +} + std::shared_ptr IndexCacheHost::get_index_from_cache(const IndexDescriptor& descriptor_of_index_to_cache, const CacheSelector which_cache) { @@ -203,7 +240,7 @@ std::shared_ptr IndexCacheHost::get_index_from_cache(const IndexDescripto else { // TODO: throw custom exception if index not found - index = host_cache.at(descriptor_of_index_to_cache)->copy_index_to_device(allocator_, cuda_stream_); + index = host_cache.at(descriptor_of_index_to_cache)->copy_index_to_device(allocator_, cuda_stream_generation_); } return index; diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index aaec92d67..2f501f742 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -48,20 +48,22 @@ public: /// \param allocator allocator to use for device arrays /// \param query_parser /// \param target_parser - /// \param kmer_size // see Index - /// \param window_size // see Index - /// \param hash_representations // see Index - /// \param filtering_parameter // see Index - /// \param cuda_stream // device memory used for Index copy will only we freed up once all previously scheduled work on this stream has finished + /// \param kmer_size see Index + /// \param window_size see Index + /// \param hash_representations see Index + /// \param filtering_parameter see Index + /// \param cuda_stream_generate index generation is done one this stream, device memory in resulting device copies of index will only we freed once all previously scheduled work on this stream has finished + /// \param cuda_stream_copy D2H and H2D copies of indices will be done on this stra, device memory in resulting device copies of index will only we freed once all previously scheduled work on this stream has finished IndexCacheHost(bool same_query_and_target, genomeworks::DefaultDeviceAllocator allocator, std::shared_ptr query_parser, std::shared_ptr target_parser, std::uint64_t kmer_size, std::uint64_t window_size, - bool hash_representations = true, - double filtering_parameter = 1.0, - cudaStream_t cuda_stream = 0); + bool hash_representations = true, + double filtering_parameter = 1.0, + cudaStream_t cuda_stream_generate = 0, + cudaStream_t cuda_stream_copy = 0); IndexCacheHost(const IndexCacheHost&) = delete; IndexCacheHost& operator=(const IndexCacheHost&) = delete; @@ -69,31 +71,41 @@ public: IndexCacheHost& operator=(IndexCacheHost&&) = delete; ~IndexCacheHost() = default; - /// \brief Discards previously cached query Indices, creates new Indices and copies them to host memory + /// \brief Discards previously cached query Indices, creates new Indices and starts copying them to host memory + /// + /// Copy is done asynchronously and one should wait for it to finish with finish_generating_query_cache_content() /// - /// Expected usage pattern is to call generate_query_cache_content() with certain index descriptors and then immediately retrieve some of them. + /// Expected usage pattern is to immediately after creation retrieve some of generated indices. /// To avoid immediately copying back those indices from host to device it is possible to specify descriptors_of_indices_to_keep_on_device // which will be copied to host, but also kept on device until retrieved for the first time using get_index_from_query_cache /// /// \param descriptors_of_indices_to_cache descriptors on indices to keep in host memory /// \param descriptors_of_indices_to_keep_on_device descriptors of indices to keep in device memory in addition to host memory until retrieved for the first time /// \param skip_copy_to_host this option should be used if descriptors_of_indices_to_cache == descriptors_of_indices_to_keep_on_device and if each index will be queried only once, it's usefull for small cases where host cache isn't actually needed - void generate_query_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device = {}, - bool skip_copy_to_host = false); + void start_generating_query_cache_content(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device = {}, + bool skip_copy_to_host = false); - /// \brief Discards previously cached target Indices, creates new Indices and copies them to host memory + /// \brief waits for copies started in start_generating_query_cache_content() to finish + void finish_generating_query_cache_content(); + + /// \brief Discards previously cached target Indices, creates new Indices and starts copying them to host memory + /// + /// Copy is done asynchronously and one should wait for it to finish with finish_generating_target_cache_content() /// - /// Expected usage pattern is to call generate_target_cache_content() with certain index descriptors and then immediately retrieve some of them. + /// Expected usage pattern is to immediately after creation retrieve some of generated indices. /// To avoid immediately copying back those indices from host to device it is possible to specify descriptors_of_indices_to_keep_on_device /// which will be copied to host, but also kept on device until retrieved for the first time using get_index_from_taget_cache /// /// \param descriptors_of_indices_to_cache descriptors on indices to keep in host memory /// \param descriptors_of_indices_to_keep_on_device descriptors of indices to keep in device memory in addition to host memory until retrieved for the first time /// \param skip_copy_to_host this option should be used if descriptors_of_indices_to_cache == descriptors_of_indices_to_keep_on_device and if each index will be queried only once, it's usefull for small cases where host cache isn't actually needed - void generate_target_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device = {}, - bool skip_copy_to_host = false); + void start_generating_target_cache_content(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device = {}, + bool skip_copy_to_host = false); + + /// \brief waits for copies started in start_generating_target_cache_content() to finish + void finish_generating_target_cache_content(); /// \brief Copies request Index to device memory /// throws if that index is currently not in cache @@ -118,14 +130,20 @@ private: target_cache }; - /// \brief Discards previously cached Indices, creates new Indices and copies them to host memory + /// \brief Discards previously cached query Indices, creates new Indices and starts copying them to host memory + /// + /// Copy is done asynchronously and one should wait for it to finish with finish_generating_cache_content() + /// /// Uses which_cache to determine if it should be working on query of target indices /// /// If same_query_and_target_ is true function checks the other cache to see if that index is already in cache - void generate_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - bool skip_copy_to_host, - CacheSelector which_cache); + void start_generating_cache_content(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device, + bool skip_copy_to_host, + CacheSelector which_cache); + + /// \brief waits for copies started in start_generating_cache_content() to finish + void finish_generating_cache_content(CacheSelector which_cache); /// \brief Fetches requested index /// Copies index from host to device memory, unless index is saved in temp device cache @@ -133,13 +151,17 @@ private: std::shared_ptr get_index_from_cache(const IndexDescriptor& descriptor_of_index_to_cache, CacheSelector which_cache); - /// Host copies of indices + // Host copies of indices cache_type_t query_cache_; cache_type_t target_cache_; - /// User can instruct cache to also keep certain indices in device memory until retrieved for the first time + // User can instruct cache to also keep certain indices in device memory until retrieved for the first time device_cache_type_t query_temp_device_cache_; device_cache_type_t target_temp_device_cache_; + // list of all indices whose generation or movement between host and device was started in start_generating_cache_content() + std::vector> query_indices_in_progress; + std::vector> target_indices_in_progress; + const bool same_query_and_target_; genomeworks::DefaultDeviceAllocator allocator_; std::shared_ptr query_parser_; @@ -148,7 +170,8 @@ private: const std::uint64_t window_size_; const bool hash_representations_; const double filtering_parameter_; - const cudaStream_t cuda_stream_; + const cudaStream_t cuda_stream_generation_; + const cudaStream_t cuda_stream_copy_; }; /// IndexCacheDevice - Keeps copies of Indices in device memory diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index 0fa275fc0..6b3191d42 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -65,16 +65,18 @@ public: /// \param window_size w - the length of the sliding window used to find sketch elements (i.e. the number of adjacent k-mers in a window, adjacent = shifted by one basepair) /// \param hash_representations - if true, hash kmer representations /// \param filtering_parameter - filter out all representations for which number_of_sketch_elements_with_that_representation/total_skech_elements >= filtering_parameter, filtering_parameter == 1.0 disables filtering - /// \param cuda_stream CUDA stream on which the work is to be done. Device arrays are also associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor is done + /// \param cuda_stream_generation CUDA stream on which index is generated. Device arrays are associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor has been done + /// \param cuda_stream_copy CUDA stream on which the index is copied to host if needed. Device arrays are associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor has been done IndexGPU(DefaultDeviceAllocator allocator, const io::FastaParser& parser, const read_id_t first_read_id, const read_id_t past_the_last_read_id, const std::uint64_t kmer_size, const std::uint64_t window_size, - const bool hash_representations = true, - const double filtering_parameter = 1.0, - const cudaStream_t cuda_stream = 0); + const bool hash_representations = true, + const double filtering_parameter = 1.0, + const cudaStream_t cuda_stream_generation = 0, + const cudaStream_t cuda_stream_copy = 0); /// \brief Constructor which copies the index from host copy /// @@ -550,7 +552,8 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, const std::uint64_t window_size, const bool hash_representations, const double filtering_parameter, - const cudaStream_t cuda_stream) + const cudaStream_t cuda_stream_generation, + const cudaStream_t cuda_stream_copy) : first_read_id_(first_read_id) , kmer_size_(kmer_size) , window_size_(window_size) @@ -569,11 +572,11 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, hash_representations, filtering_parameter, allocator, - cuda_stream); + cuda_stream_generation); // This is not completely necessary, but if removed one has to make sure that the next step // uses the same stream or that sync is done in caller - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation)); } template diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 949175a77..92284133d 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -8,7 +8,6 @@ * license agreement from NVIDIA CORPORATION is strictly prohibited. */ -#include #include "index_host_copy.cuh" #include "index_gpu.cuh" #include "minimizer.hpp" @@ -24,37 +23,6 @@ namespace genomeworks namespace cudamapper { -namespace details -{ - -/// IndexHostMemoryPinner - registers all host arrays in given IndexHostCopy as pinned memory and unregisters it at the end -class IndexHostMemoryPinner -{ -public: - /// @brief Constructor - registers pinned memory - /// @param index_host_copy - IndexHostCopy whose arrays should be registered - IndexHostMemoryPinner(IndexHostCopy& index_host_copy) - : index_host_copy_(index_host_copy) - { - GW_NVTX_RANGE(profiler, "register_pinned_memory"); - GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.underlying_array_.data(), - index_host_copy_.underlying_array_.size() * sizeof(unsigned char), - cudaHostRegisterDefault)); - } - - /// @brief Destructor - unregisters the arrays - ~IndexHostMemoryPinner() - { - GW_NVTX_RANGE(profiler, "unregister_pinned_memory"); - GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.underlying_array_.data())); - } - -private: - IndexHostCopy& index_host_copy_; -}; - -} // namespace details - IndexHostCopy::IndexHostCopy(const Index& index, const read_id_t first_read_id, const std::uint64_t kmer_size, @@ -63,6 +31,8 @@ IndexHostCopy::IndexHostCopy(const Index& index, : first_read_id_(first_read_id) , kmer_size_(kmer_size) , window_size_(window_size) + , memory_pinner_(*this) + , cuda_stream_(cuda_stream) { GW_NVTX_RANGE(profiler, "index_host_copy"); @@ -100,52 +70,56 @@ IndexHostCopy::IndexHostCopy(const Index& index, current_bit += unique_representations_bits; first_occurrence_of_representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.first_occurrence_of_representations().size()}; - // pin_memory_object registers host array as pinned memory and unregisters it on its destruction (i.e. at the end of this function) - details::IndexHostMemoryPinner pin_memory_object(const_cast(*this)); + // register pinned memory, memory gets unpinned in finish_copying_to_host() + memory_pinner_.register_pinned_memory(); cudautils::device_copy_n(index.representations().data(), index.representations().size(), representations_.data, - cuda_stream); + cuda_stream_); cudautils::device_copy_n(index.read_ids().data(), index.read_ids().size(), read_ids_.data, - cuda_stream); + cuda_stream_); cudautils::device_copy_n(index.positions_in_reads().data(), index.positions_in_reads().size(), positions_in_reads_.data, - cuda_stream); + cuda_stream_); cudautils::device_copy_n(index.directions_of_reads().data(), index.directions_of_reads().size(), directions_of_reads_.data, - cuda_stream); + cuda_stream_); cudautils::device_copy_n(index.unique_representations().data(), index.unique_representations().size(), unique_representations_.data, - cuda_stream); + cuda_stream_); cudautils::device_copy_n(index.first_occurrence_of_representations().data(), index.first_occurrence_of_representations().size(), first_occurrence_of_representations_.data, - cuda_stream); + cuda_stream_); number_of_reads_ = index.number_of_reads(); number_of_basepairs_in_longest_read_ = index.number_of_basepairs_in_longest_read(); - // This is not completely necessary, but if removed one has to make sure that the next step - // uses the same stream or that sync is done in caller - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); + // no stream synchronization, synchronization done in finish_copying_to_host() +} + +void IndexHostCopy::finish_copying_to_host() const +{ + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_)); + memory_pinner_.unregister_pinned_memory(); } std::unique_ptr IndexHostCopy::copy_index_to_device(DefaultDeviceAllocator allocator, const cudaStream_t cuda_stream) const { // pin_memory_object registers host array as pinned memory and unregisters it on its destruction (i.e. at the end of this function) - details::IndexHostMemoryPinner pin_memory_object(const_cast(*this)); + IndexHostMemoryPinner pin_memory_object(const_cast(*this)); return std::make_unique>(allocator, *this, @@ -207,6 +181,40 @@ std::uint64_t IndexHostCopy::window_size() const return window_size_; } +IndexHostCopy::IndexHostMemoryPinner::IndexHostMemoryPinner(IndexHostCopy& index_host_copy) + : index_host_copy_(index_host_copy) + , memory_pinned_(false) +{ +} + +IndexHostCopy::IndexHostMemoryPinner::~IndexHostMemoryPinner() +{ + // if memory was not unregistered (due to either a bug or an expection) unregister it + if (memory_pinned_) + { + assert(!"memory should always be unregistered by unregister_pinned_memory()"); + GW_NVTX_RANGE(profiler, "unregister_pinned_memory"); + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.underlying_array_.data())); + } +} + +void IndexHostCopy::IndexHostMemoryPinner::register_pinned_memory() +{ + GW_NVTX_RANGE(profiler, "register_pinned_memory"); + GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.underlying_array_.data(), + index_host_copy_.underlying_array_.size() * sizeof(unsigned char), + cudaHostRegisterDefault)); + memory_pinned_ = true; +} + +void IndexHostCopy::IndexHostMemoryPinner::unregister_pinned_memory() +{ + assert(memory_pinned_); + GW_NVTX_RANGE(profiler, "unregister_pinned_memory"); + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.underlying_array_.data())); + memory_pinned_ = false; +} + } // namespace cudamapper } // namespace genomeworks diff --git a/cudamapper/src/index_host_copy.cuh b/cudamapper/src/index_host_copy.cuh index 2d32f8a70..207f7c715 100644 --- a/cudamapper/src/index_host_copy.cuh +++ b/cudamapper/src/index_host_copy.cuh @@ -24,7 +24,7 @@ namespace cudamapper namespace details { class IndexHostMemoryPinner; -} +} // namespace details /// IndexHostCopy - Creates and maintains a copy of computed IndexGPU elements on the host /// @@ -32,8 +32,8 @@ class IndexHostMemoryPinner; class IndexHostCopy : public IndexHostCopyBase { public: - /// \brief Constructor - /// \brief cache the computed index to host + /// \brief Constructor - Starts creating a copy of index on the host + /// Copy is done asynchronously and one should wait for it to finish with finish_copying_to_host() /// \param index - pointer to computed index parameters (vectors of sketch elements) on GPU /// \param first_read_id - representing smallest read_id in index /// \param kmer_size - number of basepairs in a k-mer @@ -53,6 +53,9 @@ public: std::unique_ptr copy_index_to_device(DefaultDeviceAllocator allocator, const cudaStream_t cuda_stream = 0) const override; + /// \brief waits for copy to host to be done + void finish_copying_to_host() const override; + /// \brief returns an array of representations of sketch elements (stored on host) /// \return an array of representations of sketch elements const ArrayView representations() const override; @@ -98,8 +101,33 @@ public: std::uint64_t window_size() const override; private: - // used to register host arrays as pinned memory when they are copied from - friend class details::IndexHostMemoryPinner; + /// IndexHostMemoryPinner - registers and unregisters host array in given IndexHostCopy as pinned memory + class IndexHostMemoryPinner + { + public: + /// \brief Constructor - registers pinned memory + /// \param index_host_copy - IndexHostCopy whose arrays should be registered + IndexHostMemoryPinner(IndexHostCopy& index_host_copy); + + IndexHostMemoryPinner(const IndexHostMemoryPinner&) = delete; + IndexHostMemoryPinner& operator=(const IndexHostMemoryPinner&) = delete; + IndexHostMemoryPinner(IndexHostMemoryPinner&&) = delete; + IndexHostMemoryPinner& operator=(IndexHostMemoryPinner&&) = delete; + + /// \brief Destructor - unregisters pinned memory if it hasn't been unregistered yet + ~IndexHostMemoryPinner(); + + /// \brief registers pinned memory + void register_pinned_memory(); + + /// \brief unregisteres pinned memory + void unregister_pinned_memory(); + + private: + IndexHostCopy& index_host_copy_; + // is memory currently pinned + bool memory_pinned_; + }; // use a single underlying vector in order to reduce memory fragmentation when using pool allocators std::vector underlying_array_; @@ -111,6 +139,10 @@ private: ArrayView unique_representations_; ArrayView first_occurrence_of_representations_; + mutable IndexHostMemoryPinner memory_pinner_; + + cudaStream_t cuda_stream_; + read_id_t number_of_reads_; position_in_read_t number_of_basepairs_in_longest_read_; diff --git a/cudamapper/src/main.cu b/cudamapper/src/main.cu index d8f2e2bc6..8d1cd2a08 100644 --- a/cudamapper/src/main.cu +++ b/cudamapper/src/main.cu @@ -292,12 +292,14 @@ void process_one_batch(const BatchOfIndices& batch, assert(!host_batch.query_indices.empty() && !host_batch.target_indices.empty() && !device_batches.empty()); GW_NVTX_RANGE(profiler, "main::process_one_batch::host_indices"); - host_cache.generate_query_cache_content(host_batch.query_indices, - device_batches.front().query_indices, - skip_copy_to_host); - host_cache.generate_target_cache_content(host_batch.target_indices, - device_batches.front().target_indices, - skip_copy_to_host); + host_cache.start_generating_query_cache_content(host_batch.query_indices, + device_batches.front().query_indices, + skip_copy_to_host); + host_cache.start_generating_target_cache_content(host_batch.target_indices, + device_batches.front().target_indices, + skip_copy_to_host); + host_cache.finish_generating_query_cache_content(); + host_cache.finish_generating_target_cache_content(); } // process device batches one by one @@ -375,12 +377,16 @@ void postprocess_and_write_thread_function(const int32_t device_id, /// \param batches_of_indices /// \param application_parameters /// \param output_mutex -/// \param cuda_stream +/// \param cuda_stream_computation +/// \param cuda_stream_copy +/// \param number_of_total_batches +/// \param number_of_processed_batches void worker_thread_function(const int32_t device_id, ThreadsafeDataProvider& batches_of_indices, const ApplicationParameters& application_parameters, std::mutex& output_mutex, - cudaStream_t cuda_stream, + cudaStream_t cuda_stream_computation, + cudaStream_t cuda_stream_copy, const int64_t number_of_total_batches, std::atomic& number_of_processed_batches) { @@ -400,7 +406,8 @@ void worker_thread_function(const int32_t device_id, application_parameters.windows_size, true, // hash_representations application_parameters.filtering_parameter, - cuda_stream); + cuda_stream_computation, + cuda_stream_copy); // create host_cache, data is not loaded at this point but later as each batch gets processed IndexCacheDevice device_cache(application_parameters.all_to_all, @@ -441,7 +448,7 @@ void worker_thread_function(const int32_t device_id, *host_cache, device_cache, overlaps_and_cigars_to_process, - cuda_stream); + cuda_stream_computation); } // tell writer thread that there will be no more overlaps and it can finish once it has written all overlaps @@ -453,7 +460,7 @@ void worker_thread_function(const int32_t device_id, } // by this point all GPU work should anyway be done as postprocess_and_write_thread also finished and all GPU work had to be done before last values could be written - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_computation)); } } // namespace @@ -490,8 +497,9 @@ int main(int argc, char* argv[]) std::atomic number_of_processed_batches(0); ThreadsafeDataProvider batches_of_indices(std::move(batches_of_indices_vect)); - // explicitly assign one stream to each GPU - std::vector cuda_streams(parameters.num_devices); + // explicitly assign one stream for computations and one for D2H and H2D copies of indices to each GPU + std::vector cuda_streams_computation(parameters.num_devices); + std::vector cuda_streams_copy(parameters.num_devices); // create worker threads (one thread per device) // these thread process batches_of_indices one by one @@ -499,13 +507,15 @@ int main(int argc, char* argv[]) for (int32_t device_id = 0; device_id < parameters.num_devices; ++device_id) { GW_CU_CHECK_ERR(cudaSetDevice(device_id)); - GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_streams[device_id])); + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_streams_computation[device_id])); + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_streams_copy[device_id])); worker_threads.emplace_back(worker_thread_function, device_id, std::ref(batches_of_indices), std::ref(parameters), std::ref(output_mutex), - cuda_streams[device_id], + cuda_streams_computation[device_id], + cuda_streams_copy[device_id], number_of_total_batches, std::ref(number_of_processed_batches)); } @@ -515,7 +525,8 @@ int main(int argc, char* argv[]) { GW_CU_CHECK_ERR(cudaSetDevice(device_id)); worker_threads[device_id].join(); - GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_streams[device_id])); // no need to sync, it should be done at the end of worker_threads + GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_streams_computation[device_id])); // no need to sync, it should be done at the end of worker_threads + GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_streams_copy[device_id])); } return 0; diff --git a/cudamapper/tests/Test_CudamapperIndexCache.cu b/cudamapper/tests/Test_CudamapperIndexCache.cu index e968ae7c7..59de3f340 100644 --- a/cudamapper/tests/Test_CudamapperIndexCache.cu +++ b/cudamapper/tests/Test_CudamapperIndexCache.cu @@ -158,8 +158,10 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) // CATCAAG_AAGCTA // All minimizers: AAG(4f0), AAG(0f1), AGC(2r1), ATC(1f0), ATG(0r0), CAA(3f0), CTA(3f1) - cudaStream_t cuda_stream; - GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); + cudaStream_t cuda_stream_generate; + cudaStream_t cuda_stream_copy; + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_copy)); { // create and destroy stream outside of current scope const bool same_query_and_target = true; @@ -340,9 +342,11 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) w, hash_representations, filtering_parameter, - cuda_stream); + cuda_stream_generate, + cuda_stream_copy); - index_host_cache.generate_query_cache_content(catcaag_index_descriptors); + index_host_cache.start_generating_query_cache_content(catcaag_index_descriptors); + index_host_cache.finish_generating_query_cache_content(); auto index_query_catcaag = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag, @@ -357,7 +361,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_largest_read_id, catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_1"); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(aagcta_index_descriptor)); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); @@ -365,7 +369,8 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(aagcta_index_descriptor)); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - index_host_cache.generate_target_cache_content(aagcta_index_descriptors); + index_host_cache.start_generating_target_cache_content(aagcta_index_descriptors); + index_host_cache.finish_generating_target_cache_content(); index_query_catcaag = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag, @@ -380,7 +385,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_largest_read_id, catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_2"); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(aagcta_index_descriptor)); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); @@ -398,11 +403,12 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_3"); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - index_host_cache.generate_query_cache_content(aagcta_index_descriptors); + index_host_cache.start_generating_query_cache_content(aagcta_index_descriptors); + index_host_cache.finish_generating_target_cache_content(); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_index_descriptor)); auto index_query_aagcta = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); @@ -418,7 +424,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_4"); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); @@ -435,11 +441,12 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_5"); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - index_host_cache.generate_query_cache_content(catcaag_aagcta_separate_index_descriptors); + index_host_cache.start_generating_query_cache_content(catcaag_aagcta_separate_index_descriptors); + index_host_cache.finish_generating_target_cache_content(); auto index_query_catcaag_separate = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag_separate, @@ -454,7 +461,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_largest_read_id, catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_6"); auto index_query_aagcta_separate = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); check_if_index_is_correct(index_query_aagcta_separate, @@ -469,7 +476,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_7"); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); @@ -486,11 +493,12 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_8"); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - index_host_cache.generate_target_cache_content(catcaag_aagcta_one_index_descriptors); + index_host_cache.start_generating_target_cache_content(catcaag_aagcta_one_index_descriptors); + index_host_cache.finish_generating_target_cache_content(); index_query_catcaag_separate = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag_separate, @@ -505,7 +513,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_largest_read_id, catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_9"); index_query_aagcta_separate = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); check_if_index_is_correct(index_query_aagcta_separate, @@ -520,7 +528,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_10"); ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); @@ -538,12 +546,13 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_aagcta_largest_read_id, catcaag_aagcta_number_of_basepairs_in_longest_read, catcaag_aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_same_query_and_target_11"); } - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); - GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy)); + GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream_generate)); } TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_target) @@ -594,8 +603,10 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta // All minimizers: AAG(4f), ATC(1f), ATG(0r), CAA(3f) - cudaStream_t cuda_stream; - GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); + cudaStream_t cuda_stream_generate; + cudaStream_t cuda_stream_copy; + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_copy)); { // create and destroy stream outside of current scope const bool same_query_and_target = false; @@ -706,9 +717,11 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta w, hash_representations, filtering_parameter, - cuda_stream); + cuda_stream_generate, + cuda_stream_copy); - index_host_cache.generate_query_cache_content(index_descriptors); + index_host_cache.start_generating_query_cache_content(index_descriptors); + index_host_cache.finish_generating_query_cache_content(); auto index_query_aagcta = index_host_cache.get_index_from_query_cache(index_descriptor); check_if_index_is_correct(index_query_aagcta, @@ -723,11 +736,12 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_not_the_same_query_and_target_1"); ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(index_descriptor)); - index_host_cache.generate_target_cache_content(index_descriptors); + index_host_cache.start_generating_target_cache_content(index_descriptors); + index_host_cache.finish_generating_target_cache_content(); index_query_aagcta = index_host_cache.get_index_from_query_cache(index_descriptor); check_if_index_is_correct(index_query_aagcta, @@ -742,7 +756,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_not_the_same_query_and_target_2"); auto index_target_catcaag = index_host_cache.get_index_from_target_cache(index_descriptor); check_if_index_is_correct(index_target_catcaag, @@ -757,20 +771,23 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta catcaag_largest_read_id, catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_not_the_same_query_and_target_3"); } - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); - GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy)); + GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream_generate)); } TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) { // AAGCTA: AAG(0f), AGC(2r), CTA(3f) - cudaStream_t cuda_stream; - GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); + cudaStream_t cuda_stream_generate; + cudaStream_t cuda_stream_copy; + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_copy)); { // create and destroy stream outside of current scope const bool same_query_and_target = true; @@ -835,12 +852,13 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) w, hash_representations, filtering_parameter, - cuda_stream); + cuda_stream_generate, + cuda_stream_copy); - index_cache_host->generate_query_cache_content(index_descriptors, - index_descriptors); - index_cache_host->generate_target_cache_content(index_descriptors, - index_descriptors); + index_cache_host->start_generating_query_cache_content(index_descriptors, index_descriptors); + index_cache_host->start_generating_target_cache_content(index_descriptors, index_descriptors); + index_cache_host->finish_generating_query_cache_content(); + index_cache_host->finish_generating_target_cache_content(); auto index_query_temp_device_cache = index_cache_host->get_index_from_query_cache(index_descriptor); auto index_query_copy_from_host = index_cache_host->get_index_from_query_cache(index_descriptor); @@ -863,7 +881,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_keep_on_device_1"); check_if_index_is_correct(index_query_copy_from_host, aagcta_representations, @@ -877,7 +895,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_keep_on_device_2"); check_if_index_is_correct(index_target_temp_device_cache, aagcta_representations, @@ -891,7 +909,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_keep_on_device_3"); check_if_index_is_correct(index_target_copy_from_host, aagcta_representations, @@ -905,12 +923,13 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_host_keep_on_device_4"); } - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); - GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy)); + GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream_generate)); } // *** Test IndexCacheDevice *** @@ -925,8 +944,10 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) // CATCAAG minimizers: AAG(4f), ATC(1f), ATG(0r), CAA(3f) // AAGCTA minimizers: AAG(0f), AGC(2r), CTA(3f) - cudaStream_t cuda_stream; - GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); + cudaStream_t cuda_stream_generate; + cudaStream_t cuda_stream_copy; + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_copy)); { // create and destroy stream outside of current scope const bool same_query_and_target = true; @@ -1039,12 +1060,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) w, hash_representations, filtering_parameter, - cuda_stream); + cuda_stream_generate, + cuda_stream_copy); IndexCacheDevice index_cache_device(same_query_and_target, index_cache_host); - index_cache_host->generate_query_cache_content(catcaag_index_descriptors); + index_cache_host->start_generating_query_cache_content(catcaag_index_descriptors); + index_cache_host->finish_generating_query_cache_content(); ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(catcaag_index_descriptor)); index_cache_device.generate_query_cache_content(catcaag_index_descriptors); auto index_query_catcaag = index_cache_device.get_index_from_query_cache(catcaag_index_descriptor); @@ -1060,13 +1083,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) catcaag_largest_read_id, catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_device_same_query_and_target_1"); ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(aagcta_index_descriptor)); ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(catcaag_index_descriptor)); ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(aagcta_index_descriptor)); - index_cache_host->generate_target_cache_content(catcaag_aagcta_index_descriptors); + index_cache_host->start_generating_target_cache_content(catcaag_aagcta_index_descriptors); + index_cache_host->finish_generating_target_cache_content(); ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(catcaag_index_descriptor)); ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(aagcta_index_descriptor)); index_cache_device.generate_target_cache_content(catcaag_aagcta_index_descriptors); @@ -1085,7 +1109,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) catcaag_largest_read_id, catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_device_same_query_and_target_2"); auto index_target_aagcta = index_cache_device.get_index_from_target_cache(aagcta_index_descriptor); @@ -1101,7 +1125,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_device_same_query_and_target_3"); // get the same query and target indices again and make sure they point to the same objects as the last time @@ -1111,8 +1135,9 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) ASSERT_EQ(index_target_aagcta, index_target_aagcta_1); } - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); - GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy)); + GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream_generate)); } TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_target) @@ -1120,8 +1145,10 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ // AAGCTA: AAG(0f), AGC(2r), CTA(3f) // CATCAAG: AAG(4f), ATC(1f), ATG(0r), CAA(3f) - cudaStream_t cuda_stream; - GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream)); + cudaStream_t cuda_stream_generate; + cudaStream_t cuda_stream_copy; + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamCreate(&cuda_stream_copy)); { // create and destroy stream outside of current scope const bool same_query_and_target = false; @@ -1232,12 +1259,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ w, hash_representations, filtering_parameter, - cuda_stream); + cuda_stream_generate, + cuda_stream_copy); IndexCacheDevice index_cache_device(same_query_and_target, index_cache_host); - index_cache_host->generate_query_cache_content(index_descriptors); + index_cache_host->start_generating_query_cache_content(index_descriptors); + index_cache_host->finish_generating_query_cache_content(); ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(index_descriptor)); ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(index_descriptor)); @@ -1255,13 +1284,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_device_not_the_same_query_and_target_1"); ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(index_descriptor)); ASSERT_ANY_THROW(index_cache_device.generate_target_cache_content(index_descriptors)); - index_cache_host->generate_target_cache_content(index_descriptors); + index_cache_host->start_generating_target_cache_content(index_descriptors); + index_cache_host->finish_generating_target_cache_content(); index_cache_device.generate_target_cache_content(index_descriptors); index_query = index_cache_device.get_index_from_query_cache(index_descriptor); @@ -1279,7 +1309,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ aagcta_largest_read_id, aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_device_not_the_same_query_and_target_2"); check_if_index_is_correct(index_target, catcaag_representations, @@ -1293,7 +1323,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ catcaag_largest_read_id, catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, - cuda_stream, + cuda_stream_generate, "test_index_cache_device_not_the_same_query_and_target_3"); // get the same query and target indices again and make sure they point to the same objects as the last time @@ -1303,8 +1333,9 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ ASSERT_EQ(index_target, index_target_1); } - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); - GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generate)); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy)); + GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream_generate)); } } // namespace cudamapper From 7046cf40aa7817c473b6459c0d1327c45f11e4d6 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 10 Jul 2020 15:29:58 +0200 Subject: [PATCH 011/281] [cudamapper] Renamed finish_copying_to_host() into finish_copying() --- .../claraparabricks/genomeworks/cudamapper/index.hpp | 4 ++-- cudamapper/src/index_cache.cu | 2 +- cudamapper/src/index_host_copy.cu | 6 +++--- cudamapper/src/index_host_copy.cuh | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index c7faa02ce..b1dcfec30 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -136,7 +136,7 @@ class IndexHostCopyBase virtual ~IndexHostCopyBase() = default; /// \brief waits for copy to host to be done - virtual void finish_copying_to_host() const = 0; + virtual void finish_copying() const = 0; /// \brief returns an array of representations of sketch elements (stored on host) /// \return an array of representations of sketch elements @@ -183,7 +183,7 @@ class IndexHostCopyBase virtual std::uint64_t window_size() const = 0; /// \brief Starts creating a copy of index on the host - /// Copy is done asynchronously and one should wait for it to finish with finish_copying_to_host() + /// Copy is done asynchronously and one should wait for it to finish with finish_copying() /// \param index - pointer to computed index parameters (vectors of sketch elements) on GPU /// \param first_read_id - representing smallest read_id in index /// \param kmer_size - number of basepairs in a k-mer diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index cd622766b..b158e16c9 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -216,7 +216,7 @@ void IndexCacheHost::finish_generating_cache_content(CacheSelector which_cache) // wait for all indices to finish being copied to host memory for (auto& index_in_progress : indices_in_progress) { - index_in_progress->finish_copying_to_host(); + index_in_progress->finish_copying(); } indices_in_progress.clear(); } diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 92284133d..8ca7a183c 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -70,7 +70,7 @@ IndexHostCopy::IndexHostCopy(const Index& index, current_bit += unique_representations_bits; first_occurrence_of_representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.first_occurrence_of_representations().size()}; - // register pinned memory, memory gets unpinned in finish_copying_to_host() + // register pinned memory, memory gets unpinned in finish_copying() memory_pinner_.register_pinned_memory(); cudautils::device_copy_n(index.representations().data(), @@ -106,10 +106,10 @@ IndexHostCopy::IndexHostCopy(const Index& index, number_of_reads_ = index.number_of_reads(); number_of_basepairs_in_longest_read_ = index.number_of_basepairs_in_longest_read(); - // no stream synchronization, synchronization done in finish_copying_to_host() + // no stream synchronization, synchronization done in finish_copying() } -void IndexHostCopy::finish_copying_to_host() const +void IndexHostCopy::finish_copying() const { GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_)); memory_pinner_.unregister_pinned_memory(); diff --git a/cudamapper/src/index_host_copy.cuh b/cudamapper/src/index_host_copy.cuh index 207f7c715..5f928ad86 100644 --- a/cudamapper/src/index_host_copy.cuh +++ b/cudamapper/src/index_host_copy.cuh @@ -33,7 +33,7 @@ class IndexHostCopy : public IndexHostCopyBase { public: /// \brief Constructor - Starts creating a copy of index on the host - /// Copy is done asynchronously and one should wait for it to finish with finish_copying_to_host() + /// Copy is done asynchronously and one should wait for it to finish with finish_copying() /// \param index - pointer to computed index parameters (vectors of sketch elements) on GPU /// \param first_read_id - representing smallest read_id in index /// \param kmer_size - number of basepairs in a k-mer @@ -53,8 +53,8 @@ public: std::unique_ptr copy_index_to_device(DefaultDeviceAllocator allocator, const cudaStream_t cuda_stream = 0) const override; - /// \brief waits for copy to host to be done - void finish_copying_to_host() const override; + /// \brief waits for copy to be done + void finish_copying() const override; /// \brief returns an array of representations of sketch elements (stored on host) /// \return an array of representations of sketch elements From 4474b10680a2f0ea6716f5eb6c31f5e3eda46fc3 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 29 Jul 2020 15:21:59 +0200 Subject: [PATCH 012/281] [cudamapper] Overlapping fetching indices from host with overlap generation --- .../genomeworks/cudamapper/index.hpp | 7 + cudamapper/src/index_cache.cu | 411 +++++++++--------- cudamapper/src/index_cache.cuh | 300 ++++++------- cudamapper/src/index_gpu.cuh | 92 ++-- cudamapper/src/index_host_copy.cu | 38 +- cudamapper/src/index_host_copy.cuh | 6 +- cudamapper/src/main.cu | 112 ++--- cudamapper/tests/Test_CudamapperIndexCache.cu | 391 +++++++++-------- 8 files changed, 700 insertions(+), 657 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index b1dcfec30..dd06c879a 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -83,6 +83,13 @@ class Index return sizeof(representation_t) * CHAR_BIT / 2; } + /// \brief checks if index is ready to be used, index might not be ready if its creation is asynchronous + /// \return whether the index is ready to be used + virtual bool is_ready() const = 0; + + /// \brief if is_ready() is true returns immediately, blocks until it becomes ready otherwise + virtual void wait_to_be_ready() const = 0; + /// \brief generates a mapping of (k,w)-kmer-representation to all of its occurrences for one or more sequences /// \param allocator The device memory allocator to use for temporary buffer allocations /// \param parser parser for the whole input file (part that goes into this index is determined by first_read_id and past_the_last_read_id) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index b158e16c9..c2a9acf9e 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -26,16 +26,16 @@ namespace genomeworks namespace cudamapper { -IndexCacheHost::IndexCacheHost(const bool same_query_and_target, - genomeworks::DefaultDeviceAllocator allocator, - std::shared_ptr query_parser, - std::shared_ptr target_parser, - const std::uint64_t kmer_size, - const std::uint64_t window_size, - const bool hash_representations, - const double filtering_parameter, - const cudaStream_t cuda_stream_generation, - const cudaStream_t cuda_stream_copy) +IndexCache::IndexCache(bool same_query_and_target, + genomeworks::DefaultDeviceAllocator allocator, + std::shared_ptr query_parser, + std::shared_ptr target_parser, + std::uint64_t kmer_size, + std::uint64_t window_size, + bool hash_representations, + double filtering_parameter, + cudaStream_t cuda_stream_generation, + cudaStream_t cuda_stream_copy) : same_query_and_target_(same_query_and_target) , allocator_(allocator) , query_parser_(query_parser) @@ -49,285 +49,278 @@ IndexCacheHost::IndexCacheHost(const bool same_query_and_target, { } -void IndexCacheHost::start_generating_query_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - const bool skip_copy_to_host) +void IndexCache::generate_content_query_host(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device, + const bool skip_copy_to_host) { - start_generating_cache_content(descriptors_of_indices_to_cache, - descriptors_of_indices_to_keep_on_device, - skip_copy_to_host, - CacheSelector::query_cache); + generate_content_host(descriptors_of_indices_to_cache, + descriptors_of_indices_to_keep_on_device, + skip_copy_to_host, + CacheSelector::query_cache); } -void IndexCacheHost::finish_generating_query_cache_content() +void IndexCache::generate_content_target_host(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device, + const bool skip_copy_to_host) { - finish_generating_cache_content(CacheSelector::query_cache); - return; + generate_content_host(descriptors_of_indices_to_cache, + descriptors_of_indices_to_keep_on_device, + skip_copy_to_host, + CacheSelector::target_cache); } -void IndexCacheHost::start_generating_target_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - const bool skip_copy_to_host) +void IndexCache::start_generating_content_query_device(const std::vector& descriptors_of_indices_to_cache) { - start_generating_cache_content(descriptors_of_indices_to_cache, - descriptors_of_indices_to_keep_on_device, - skip_copy_to_host, - CacheSelector::target_cache); + start_generating_content_device(descriptors_of_indices_to_cache, + CacheSelector::query_cache); } -void IndexCacheHost::finish_generating_target_cache_content() +void IndexCache::finish_generating_content_query_device() { - finish_generating_cache_content(CacheSelector::target_cache); - return; + finish_generating_content_device(CacheSelector::query_cache); } -std::shared_ptr IndexCacheHost::get_index_from_query_cache(const IndexDescriptor& descriptor_of_index_to_cache) +void IndexCache::start_generating_content_target_device(const std::vector& descriptors_of_indices_to_cache) { - return get_index_from_cache(descriptor_of_index_to_cache, + start_generating_content_device(descriptors_of_indices_to_cache, + CacheSelector::target_cache); +} + +void IndexCache::finish_generating_content_target_device() +{ + finish_generating_content_device(CacheSelector::target_cache); +} + +std::shared_ptr IndexCache::get_index_from_query_cache(const IndexDescriptor& index_descriptor) const +{ + return get_index_from_cache(index_descriptor, CacheSelector::query_cache); } -std::shared_ptr IndexCacheHost::get_index_from_target_cache(const IndexDescriptor& descriptor_of_index_to_cache) +std::shared_ptr IndexCache::get_index_from_target_cache(const IndexDescriptor& index_descriptor) const { - return get_index_from_cache(descriptor_of_index_to_cache, + return get_index_from_cache(index_descriptor, CacheSelector::target_cache); } -void IndexCacheHost::start_generating_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - const bool skip_copy_to_host, - const CacheSelector which_cache) +void IndexCache::generate_content_host(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device, + const bool skip_copy_to_host, + const CacheSelector which_cache) { // skip_copy_to_host only makes sense if descriptors_of_indices_to_cache and descriptors_of_indices_to_keep_on_device are the same // otherwise some indices would be created and not saved on either host or device assert(!skip_copy_to_host || (descriptors_of_indices_to_cache == descriptors_of_indices_to_keep_on_device)); + assert(!descriptors_of_indices_to_cache.empty()); - cache_type_t& cache_to_edit = (CacheSelector::query_cache == which_cache) ? query_cache_ : target_cache_; - const cache_type_t& cache_to_check = (CacheSelector::query_cache == which_cache) ? target_cache_ : query_cache_; - device_cache_type_t& temp_device_cache_to_edit = (CacheSelector::query_cache == which_cache) ? query_temp_device_cache_ : target_temp_device_cache_; - const device_cache_type_t& temp_device_cache_to_check = (CacheSelector::query_cache == which_cache) ? target_temp_device_cache_ : query_temp_device_cache_; - const genomeworks::io::FastaParser* parser = (CacheSelector::query_cache == which_cache) ? query_parser_.get() : target_parser_.get(); - std::vector>& indices_in_progress = (CacheSelector::query_cache == which_cache) ? query_indices_in_progress : target_indices_in_progress; + host_cache_t& this_cache = (CacheSelector::query_cache == which_cache) ? query_host_cache_ : target_host_cache_; + const host_cache_t& other_cache = (CacheSelector::query_cache == which_cache) ? target_host_cache_ : query_host_cache_; + device_cache_t& indices_kept_on_device = (CacheSelector::query_cache == which_cache) ? query_indices_kept_on_device_ : target_indices_kept_on_device_; + const genomeworks::io::FastaParser* parser = (CacheSelector::query_cache == which_cache) ? query_parser_.get() : target_parser_.get(); // convert descriptors_of_indices_to_keep_on_device into set for faster search std::unordered_set descriptors_of_indices_to_keep_on_device_set(begin(descriptors_of_indices_to_keep_on_device), end(descriptors_of_indices_to_keep_on_device)); - cache_type_t new_cache; - temp_device_cache_to_edit.clear(); // this should be empty by now anyway + host_cache_t new_cache; + indices_kept_on_device.empty(); // normally this should be empty anyway + + // In most cases index is generated on device and then moved to host. These two operations can be overlapped, i.e. while one index is being copied + // to host the next index can be generated. + // Index cache is expected to be larger than the available device memory, meaning it is not possible to keep all indices on device while they are + // being copied to host. In this implementation only two copies of index are kept on device: the one currently being generated and the one currently + // being copied to host (from the previous step). + + std::shared_ptr index_on_host = nullptr; + std::shared_ptr index_on_host_from_previous_step = nullptr; + std::shared_ptr index_on_device = nullptr; + std::shared_ptr index_on_device_from_previous_step = nullptr; + bool started_copy = false; // if index is found on host copy is not needed + bool started_copy_from_previous_step = false; for (const IndexDescriptor& descriptor_of_index_to_cache : descriptors_of_indices_to_cache) { - // check if this index should be kept on device in addition to copying it to host - const bool keep_on_device = descriptors_of_indices_to_keep_on_device_set.count(descriptor_of_index_to_cache) != 0; + const bool host_copy_needed = !skip_copy_to_host; + const bool device_copy_needed = descriptors_of_indices_to_keep_on_device_set.count(descriptor_of_index_to_cache) != 0; - std::shared_ptr index_on_host = nullptr; - std::shared_ptr index_on_device = nullptr; + // check if host copy already exists - // if query and target are the same check if this index is already cached in the other cache - if (same_query_and_target_) + // check if index is already in this cache + auto index_in_this_cache = this_cache.find(descriptor_of_index_to_cache); + if (index_in_this_cache != this_cache.end()) + { + // index already cached + index_on_host = index_in_this_cache->second; + } + // if index not found in this cache and query and target input files are the same check the other cache as well + if (!index_on_host && same_query_and_target_) { - auto existing_cache = cache_to_check.find(descriptor_of_index_to_cache); - if (existing_cache != cache_to_check.end()) + auto index_in_other_cache = other_cache.find(descriptor_of_index_to_cache); + if (index_in_other_cache != other_cache.end()) { - index_on_host = existing_cache->second; - if (keep_on_device) - { - auto existing_device_cache = temp_device_cache_to_check.find(descriptor_of_index_to_cache); - if (existing_device_cache != temp_device_cache_to_check.end()) - { - index_on_device = existing_device_cache->second; - } - else - { - index_on_device = index_on_host->copy_index_to_device(allocator_, cuda_stream_generation_); - } - } + index_on_host = index_in_other_cache->second; } } - // if index was not found in the other cache check current cache and generate index if needed - if (nullptr == index_on_host) + if (!index_on_host) { - auto existing_cache = cache_to_edit.find(descriptor_of_index_to_cache); - if (existing_cache != cache_to_edit.end()) + // create index + index_on_device = Index::create_index(allocator_, + *parser, + descriptor_of_index_to_cache.first_read(), + descriptor_of_index_to_cache.first_read() + descriptor_of_index_to_cache.number_of_reads(), + kmer_size_, + window_size_, + hash_representations_, + filtering_parameter_, + cuda_stream_generation_); + + // wait for index to be generated on cuda_stream_generation_ before copying it on cuda_stream_copy_ + // TODO: do this sync using an event + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation_)); + + if (host_copy_needed) { - // index already cached - index_on_host = existing_cache->second; - if (keep_on_device) + // if a D2H copy has been been started in the previous step wait for it to finish + if (started_copy_from_previous_step) { - index_on_device = index_on_host->copy_index_to_device(allocator_, cuda_stream_generation_); - } - } - else - { - // create index - index_on_device = Index::create_index(allocator_, - *parser, - descriptor_of_index_to_cache.first_read(), - descriptor_of_index_to_cache.first_read() + descriptor_of_index_to_cache.number_of_reads(), - kmer_size_, - window_size_, - hash_representations_, - filtering_parameter_, - cuda_stream_generation_); - - // copy it to host memory - if (!skip_copy_to_host) - { - // wait for index to be generated on cuda_stream_generation_ before copying it on cuda_stream_copy_ - // TODO: do this sync using an event - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation_)); - - index_on_host = IndexHostCopy::create_cache(*index_on_device, - descriptor_of_index_to_cache.first_read(), - kmer_size_, - window_size_, - cuda_stream_copy_); - // index is being copied from device to host which is done asynchronously - indices_in_progress.push_back(index_on_host); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy_)); + // copying is done, these pointer are not needed anymore + index_on_host_from_previous_step = nullptr; + index_on_device_from_previous_step = nullptr; } + + index_on_host = IndexHostCopy::create_cache(*index_on_device, + descriptor_of_index_to_cache.first_read(), + kmer_size_, + window_size_, + cuda_stream_copy_); + started_copy = true; // index is being copied to host memory, a sync will be needed } } - // save pointer to cached index - if (!skip_copy_to_host) + if (host_copy_needed) { - assert(nullptr != index_on_host); + assert(index_on_host); new_cache[descriptor_of_index_to_cache] = index_on_host; } - if (keep_on_device) + // Device copy of index is only saved if is already exists, i.e. if the index has been generated + // If the index has been found on host it won't be copied back to device at this point + // TODO: check device caches from this index in that case, this is not expected to happen frequently so performance gains are going to be small + if (device_copy_needed && index_on_device) { - temp_device_cache_to_edit[descriptor_of_index_to_cache] = index_on_device; + indices_kept_on_device[descriptor_of_index_to_cache] = index_on_device; } - } - std::swap(new_cache, cache_to_edit); -} - -void IndexCacheHost::finish_generating_cache_content(CacheSelector which_cache) -{ - std::vector>& indices_in_progress = (CacheSelector::query_cache == which_cache) ? query_indices_in_progress : target_indices_in_progress; - - // wait for all indices to be generated - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation_)); + // if a D2H copy has been been started in the previous step and it has not been waited for yet wait for it to finish + if (started_copy_from_previous_step && index_on_host_from_previous_step && index_on_device_from_previous_step) + { + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy_)); + // copying is done, these pointer are not needed anymore + index_on_host_from_previous_step = nullptr; + index_on_device_from_previous_step = nullptr; + } - // wait for all indices to finish being copied to host memory - for (auto& index_in_progress : indices_in_progress) - { - index_in_progress->finish_copying(); + // prepare for next step + started_copy_from_previous_step = started_copy; + if (started_copy) + { + index_on_host_from_previous_step = index_on_host; + index_on_device_from_previous_step = index_on_device; + } + else + { + index_on_host_from_previous_step = nullptr; + index_on_device_from_previous_step = nullptr; + } + index_on_host = nullptr; + index_on_device = nullptr; } - indices_in_progress.clear(); -} - -std::shared_ptr IndexCacheHost::get_index_from_cache(const IndexDescriptor& descriptor_of_index_to_cache, - const CacheSelector which_cache) -{ - std::shared_ptr index; - const cache_type_t& host_cache = (CacheSelector::query_cache == which_cache) ? query_cache_ : target_cache_; - device_cache_type_t& temp_device_index_cache = (CacheSelector::query_cache == which_cache) ? query_temp_device_cache_ : target_temp_device_cache_; - - auto temp_device_index_cache_iter = temp_device_index_cache.find(descriptor_of_index_to_cache); - // check if index is present in device memory, copy from host if not - if (temp_device_index_cache_iter != temp_device_index_cache.end()) - { - index = temp_device_index_cache_iter->second; - // indices are removed from device cache after they have been used for the first time - temp_device_index_cache.erase(temp_device_index_cache_iter); - } - else + // wait for the last copy to finish + if (started_copy_from_previous_step) { - // TODO: throw custom exception if index not found - index = host_cache.at(descriptor_of_index_to_cache)->copy_index_to_device(allocator_, cuda_stream_generation_); + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy_)); } - return index; -} - -IndexCacheDevice::IndexCacheDevice(const bool same_query_and_target, - std::shared_ptr index_cache_host) - : same_query_and_target_(same_query_and_target) - , index_cache_host_(index_cache_host) -{ -} - -void IndexCacheDevice::generate_query_cache_content(const std::vector& descriptors_of_indices_to_cache) -{ - generate_cache_content(descriptors_of_indices_to_cache, CacheSelector::query_cache); -} - -void IndexCacheDevice::generate_target_cache_content(const std::vector& descriptors_of_indices_to_cache) -{ - generate_cache_content(descriptors_of_indices_to_cache, CacheSelector::target_cache); -} - -std::shared_ptr IndexCacheDevice::get_index_from_query_cache(const IndexDescriptor& descriptor_of_index_to_cache) -{ - // TODO: throw custom exception if index not found - return query_cache_.at(descriptor_of_index_to_cache); + std::swap(new_cache, this_cache); } -std::shared_ptr IndexCacheDevice::get_index_from_target_cache(const IndexDescriptor& descriptor_of_index_to_cache) +void IndexCache::start_generating_content_device(const std::vector& descriptors_of_indices_to_cache, + const CacheSelector which_cache) { - // TODO: throw custom exception if index not found - return target_cache_.at(descriptor_of_index_to_cache); -} + const host_cache_t& host_cache = (CacheSelector::query_cache == which_cache) ? query_host_cache_ : target_host_cache_; + device_cache_t& this_device_cache = (CacheSelector::query_cache == which_cache) ? query_device_cache_ : target_device_cache_; + const device_cache_t& other_device_cache = (CacheSelector::query_cache == which_cache) ? target_device_cache_ : query_device_cache_; + device_cache_t& indices_kept_on_device = (CacheSelector::query_cache == which_cache) ? query_indices_kept_on_device_ : target_indices_kept_on_device_; + device_cache_t& new_cache = (CacheSelector::query_cache == which_cache) ? next_query_device_cache_ : next_target_device_cache_; -void IndexCacheDevice::generate_cache_content(const std::vector& descriptors_of_indices_to_cache, - const CacheSelector which_cache) -{ - cache_type_t& cache_to_edit = (CacheSelector::query_cache == which_cache) ? query_cache_ : target_cache_; - const cache_type_t& cache_to_check = (CacheSelector::query_cache == which_cache) ? target_cache_ : query_cache_; - - cache_type_t new_cache; - - for (const IndexDescriptor& descriptor_of_index_to_cache : descriptors_of_indices_to_cache) + for (const IndexDescriptor& index_descriptor : descriptors_of_indices_to_cache) { + std::shared_ptr device_index = nullptr; - std::shared_ptr index = nullptr; + // check if index was kept on device after creation + auto index_kept_on_device = indices_kept_on_device.find(index_descriptor); + if (index_kept_on_device != indices_kept_on_device.end()) + { + device_index = index_kept_on_device->second; + indices_kept_on_device.erase(index_descriptor); + } - if (same_query_and_target_) + // check if value is already in this cache + if (!device_index) { - // check if the same index already exists in the other cache - auto existing_cache = cache_to_check.find(descriptor_of_index_to_cache); - if (existing_cache != cache_to_check.end()) + auto index_in_this_cache = this_device_cache.find(index_descriptor); + if (index_in_this_cache != this_device_cache.end()) { - index = existing_cache->second; + device_index = index_in_this_cache->second; } } - if (nullptr == index) + // if query and target files are the same check the other index as well + if (!device_index && same_query_and_target_) { - // check if this index is already cached in this cache - auto existing_cache = cache_to_edit.find(descriptor_of_index_to_cache); - if (existing_cache != cache_to_edit.end()) + auto index_in_other_cache = other_device_cache.find(index_descriptor); + if (index_in_other_cache != other_device_cache.end()) { - // index already cached - index = existing_cache->second; - } - else - { - // index not already cached -> fetch it from index_cache_host_ - if (CacheSelector::query_cache == which_cache) - { - index = index_cache_host_->get_index_from_query_cache(descriptor_of_index_to_cache); - } - else - { - index = index_cache_host_->get_index_from_target_cache(descriptor_of_index_to_cache); - } + device_index = index_in_other_cache->second; } } - assert(nullptr != index); + // if index has not been found on device copy it from host + if (!device_index) + { + // TODO: Throw a custom exception if index not found instead of std::out_of_range + std::shared_ptr index_on_host = host_cache.at(index_descriptor); + device_index = index_on_host->copy_index_to_device(allocator_, cuda_stream_copy_); + } + + assert(device_index); + new_cache[index_descriptor] = device_index; + } +} + +void IndexCache::finish_generating_content_device(const CacheSelector which_cache) +{ + device_cache_t& this_device_cache = (CacheSelector::query_cache == which_cache) ? query_device_cache_ : target_device_cache_; + device_cache_t& new_cache = (CacheSelector::query_cache == which_cache) ? next_query_device_cache_ : next_target_device_cache_; - // save pointer to cached index - new_cache[descriptor_of_index_to_cache] = index; + for (const auto device_index : new_cache) + { + device_index.second->wait_to_be_ready(); } - std::swap(new_cache, cache_to_edit); + this_device_cache.clear(); + std::swap(this_device_cache, new_cache); +} + +std::shared_ptr IndexCache::get_index_from_cache(const IndexDescriptor& index_descriptor, + const CacheSelector which_cache) const +{ + const device_cache_t& this_device_cache = (CacheSelector::query_cache == which_cache) ? query_device_cache_ : target_device_cache_; + // TODO: Throw a custom exception if index not found instead of std::out_of_range + return this_device_cache.at(index_descriptor); } } // namespace cudamapper diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index 2f501f742..5490150ea 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -35,16 +35,21 @@ namespace cudamapper class Index; class IndexHostCopyBase; -/// IndexCacheHost - Creates Indices, stores them in host memory and on demand copies them back to device memory +/// IndexCache - Creates indices, stores them in host memory and on demands moves them to device memory /// -/// The user tells cache which Indices to keep in cache using generate_query_cache_content() and generate_target_cache_content() and -/// retrieves indices using get_index_from_query_cache() and get_index_from_target_cache(). Trying to retrieve an -/// Index which was not previously stored in cache results in an exception -class IndexCacheHost +/// Class contains separate caches for query and target. In this example query cache is going to be used, target cache is equivalent. +/// The user generates indices and stores them in host memory using generate_content_query_host(). The user then copies some of those indices +/// to device memory using start_generating_content_query_device() and finish_generating_content_query_device(). It is user's responsibility +/// to make sure that indices requested by start_generating_content_query_device() were generated by generate_content_query_host(). +/// Memory copy to device is done asynchronously, the user should make sure that every call to start_generating_content_query_device() is +/// followed by a call finish_generating_content_query_device(). +/// The class tries to minimize the number of index creation and movemens, e.g. by reusing already existing indices, but not guarantees are given. +class IndexCache { public: - /// \brief Constructor - /// \param same_query_and_target true means that both query and target are the same, meaning that if requested index exists in query cache it can also be used by target cache directly + /// \brief Constructor only initializes cache, no index is generated at this point + /// + /// \param same_query_and_target true means that both query and target files are the same, meaning that if some index exists in query cache it can also be used by target cache directly /// \param allocator allocator to use for device arrays /// \param query_parser /// \param target_parser @@ -52,77 +57,93 @@ public: /// \param window_size see Index /// \param hash_representations see Index /// \param filtering_parameter see Index - /// \param cuda_stream_generate index generation is done one this stream, device memory in resulting device copies of index will only we freed once all previously scheduled work on this stream has finished - /// \param cuda_stream_copy D2H and H2D copies of indices will be done on this stra, device memory in resulting device copies of index will only we freed once all previously scheduled work on this stream has finished - IndexCacheHost(bool same_query_and_target, - genomeworks::DefaultDeviceAllocator allocator, - std::shared_ptr query_parser, - std::shared_ptr target_parser, - std::uint64_t kmer_size, - std::uint64_t window_size, - bool hash_representations = true, - double filtering_parameter = 1.0, - cudaStream_t cuda_stream_generate = 0, - cudaStream_t cuda_stream_copy = 0); - - IndexCacheHost(const IndexCacheHost&) = delete; - IndexCacheHost& operator=(const IndexCacheHost&) = delete; - IndexCacheHost(IndexCacheHost&&) = delete; - IndexCacheHost& operator=(IndexCacheHost&&) = delete; - ~IndexCacheHost() = default; - - /// \brief Discards previously cached query Indices, creates new Indices and starts copying them to host memory - /// - /// Copy is done asynchronously and one should wait for it to finish with finish_generating_query_cache_content() - /// - /// Expected usage pattern is to immediately after creation retrieve some of generated indices. - /// To avoid immediately copying back those indices from host to device it is possible to specify descriptors_of_indices_to_keep_on_device - // which will be copied to host, but also kept on device until retrieved for the first time using get_index_from_query_cache - /// - /// \param descriptors_of_indices_to_cache descriptors on indices to keep in host memory - /// \param descriptors_of_indices_to_keep_on_device descriptors of indices to keep in device memory in addition to host memory until retrieved for the first time - /// \param skip_copy_to_host this option should be used if descriptors_of_indices_to_cache == descriptors_of_indices_to_keep_on_device and if each index will be queried only once, it's usefull for small cases where host cache isn't actually needed - void start_generating_query_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device = {}, - bool skip_copy_to_host = false); - - /// \brief waits for copies started in start_generating_query_cache_content() to finish - void finish_generating_query_cache_content(); - - /// \brief Discards previously cached target Indices, creates new Indices and starts copying them to host memory - /// - /// Copy is done asynchronously and one should wait for it to finish with finish_generating_target_cache_content() - /// - /// Expected usage pattern is to immediately after creation retrieve some of generated indices. - /// To avoid immediately copying back those indices from host to device it is possible to specify descriptors_of_indices_to_keep_on_device - /// which will be copied to host, but also kept on device until retrieved for the first time using get_index_from_taget_cache - /// - /// \param descriptors_of_indices_to_cache descriptors on indices to keep in host memory - /// \param descriptors_of_indices_to_keep_on_device descriptors of indices to keep in device memory in addition to host memory until retrieved for the first time - /// \param skip_copy_to_host this option should be used if descriptors_of_indices_to_cache == descriptors_of_indices_to_keep_on_device and if each index will be queried only once, it's usefull for small cases where host cache isn't actually needed - void start_generating_target_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device = {}, - bool skip_copy_to_host = false); - - /// \brief waits for copies started in start_generating_target_cache_content() to finish - void finish_generating_target_cache_content(); - - /// \brief Copies request Index to device memory - /// throws if that index is currently not in cache - std::shared_ptr get_index_from_query_cache(const IndexDescriptor& descriptor_of_index_to_cache); - - /// \brief Copies request Index to device memory - /// throws if that index is currently not in cache - std::shared_ptr get_index_from_target_cache(const IndexDescriptor& descriptor_of_index_to_cache); + /// \param cuda_stream_generation index generation is done one this stream, device memory in resulting device copies of index will only we freed once all previously scheduled work on this stream has finished + /// \param cuda_stream_copy D2H and H2D copies of indices will be done on this stream, device memory in resulting device copies of index will only we freed once all previously scheduled work on this stream has finished + IndexCache(bool same_query_and_target, + genomeworks::DefaultDeviceAllocator allocator, + std::shared_ptr query_parser, + std::shared_ptr target_parser, + std::uint64_t kmer_size, + std::uint64_t window_size, + bool hash_representations = true, + double filtering_parameter = 1.0, + cudaStream_t cuda_stream_generation = 0, + cudaStream_t cuda_stream_copy = 0); + + /// \brief Generates indices on device and copies them to host memory + /// + /// If index already exists on host is may be reused. + /// Indices from descriptors_of_indices_to_keep_on_device will be kept on device in addition to being to host. This is useful if the same indices + /// are going to be requested by start_generating_content_query_device() immediately after this call. + /// If skip_copy_to_host is ture indices are going to be kept on device and not copied to host. In that case descriptors_of_indices_to_cache must + /// be equal to descriptors_of_indices_to_keep_on_device and there must be only one call to start_generating_content_query_device() + /// + /// \param descriptors_of_indices_to_cache + /// \param descriptors_of_indices_to_keep_on_device + /// \param skip_copy_to_host + void generate_content_query_host(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device = {}, + bool skip_copy_to_host = false); + + /// \brief Generates indices on device and copies them to host memory + /// + /// If index already exists it is not going to be regenerated. + /// Indices from descriptors_of_indices_to_keep_on_device will be kept on device in addition to being to host. This is useful if the same indices + /// are going to be requested by start_generating_content_target_device() immediately after this call. + /// If skip_copy_to_host is ture indices are going to be kept on device and not copied to host. In that case descriptors_of_indices_to_cache must + /// be equal to descriptors_of_indices_to_keep_on_device and there must be only one call to start_generating_content_target_device() + /// + /// \param descriptors_of_indices_to_cache + /// \param descriptors_of_indices_to_keep_on_device + /// \param skip_copy_to_host + void generate_content_target_host(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device = {}, + bool skip_copy_to_host = false); + + /// \brief Begins copying indices to device + /// + /// If index already exists on device it may be reused. + /// This copy is done asynchronously, copy is finised by calling finish_generating_content_query_device(). + /// The user should make sure that every call to start_generating_content_query_device() is followed by a call to finish_generating_content_query_device() + /// + /// \param descriptors_of_indices_to_cache + void start_generating_content_query_device(const std::vector& descriptors_of_indices_to_cache); + + /// \brief Waits for copy started in start_generating_content_query_device() + /// + /// The user should make sure that there was a call to start_generating_content_query_device() before this call + void finish_generating_content_query_device(); + + /// \brief Begins copying indices to device + /// + /// If index already exists on device it may be reused. + /// This copy is done asynchronously, copy is finised by calling finish_generating_content_target_device(). + /// The user should make sure that every call to start_generating_content_target_device() is followed by a call to finish_generating_content_target_device() + /// + /// \param descriptors_of_indices_to_cache + void start_generating_content_target_device(const std::vector& descriptors_of_indices_to_cache); + + /// \brief Waits for copy started in start_generating_content_target_device() + /// + /// The user should make sure that there was a call to start_generating_content_target_device() before this call + void finish_generating_content_target_device(); + + /// \brief Returns a pointer to requested index + /// \return a pointer to requested index + std::shared_ptr get_index_from_query_cache(const IndexDescriptor& index_descriptor) const; + + /// \brief Returns a pointer to requested index + /// \return a pointer to requested index + std::shared_ptr get_index_from_target_cache(const IndexDescriptor& index_descriptor) const; private: - using cache_type_t = std::unordered_map, IndexDescriptorHash>; - using device_cache_type_t = std::unordered_map, - IndexDescriptorHash>; + using device_cache_t = std::unordered_map, + IndexDescriptorHash>; enum class CacheSelector { @@ -130,37 +151,61 @@ private: target_cache }; - /// \brief Discards previously cached query Indices, creates new Indices and starts copying them to host memory + /// \brief Generates indices on device and copies them to host memory + /// + /// If index already exists on host is may be reused. + /// Indices from descriptors_of_indices_to_keep_on_device will be kept on device in addition to being to host. This is useful if the same indices + /// are going to be requested by start_generating_content_device() immediately after this call. + /// If skip_copy_to_host is ture indices are going to be kept on device and not copied to host. In that case descriptors_of_indices_to_cache must + /// be equal to descriptors_of_indices_to_keep_on_device and there must be only one call to start_generating_content_device() /// - /// Copy is done asynchronously and one should wait for it to finish with finish_generating_cache_content() + /// \param descriptors_of_indices_to_cache + /// \param descriptors_of_indices_to_keep_on_device + /// \param skip_copy_to_host + /// \param which_cache + void generate_content_host(const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device, + bool skip_copy_to_host, + CacheSelector which_cache); + + /// \brief Begins copying indices to device /// - /// Uses which_cache to determine if it should be working on query of target indices + /// If index already exists on device it may be reused. + /// This copy is done asynchronously, copy is finised by calling finish_generating_content_device(). + /// The user should make sure that every call to start_generating_content_device() is followed by a call to finish_generating_content_device() /// - /// If same_query_and_target_ is true function checks the other cache to see if that index is already in cache - void start_generating_cache_content(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - bool skip_copy_to_host, - CacheSelector which_cache); + /// \param descriptors_of_indices_to_cache + /// \param which_cache + void start_generating_content_device(const std::vector& descriptors_of_indices_to_cache, + CacheSelector which_cache); - /// \brief waits for copies started in start_generating_cache_content() to finish - void finish_generating_cache_content(CacheSelector which_cache); + /// \brief Waits for copy started in start_generating_content_device() + /// + /// The user should make sure that there was a call to start_generating_content_device() before this call + /// \param which_cache + void finish_generating_content_device(CacheSelector which_cache); - /// \brief Fetches requested index - /// Copies index from host to device memory, unless index is saved in temp device cache - /// If that is the case it returs that device copy are removes it from temp device cache - std::shared_ptr get_index_from_cache(const IndexDescriptor& descriptor_of_index_to_cache, - CacheSelector which_cache); + /// \brief Returns a pointer to requested index + /// \return a pointer to requested index + std::shared_ptr get_index_from_cache(const IndexDescriptor& index_descriptor, + CacheSelector which_cache) const; - // Host copies of indices - cache_type_t query_cache_; - cache_type_t target_cache_; - // User can instruct cache to also keep certain indices in device memory until retrieved for the first time - device_cache_type_t query_temp_device_cache_; - device_cache_type_t target_temp_device_cache_; + // Indices kept on host + host_cache_t query_host_cache_; + host_cache_t target_host_cache_; - // list of all indices whose generation or movement between host and device was started in start_generating_cache_content() - std::vector> query_indices_in_progress; - std::vector> target_indices_in_progress; + // Indices kept of device because of descriptors_of_indices_to_keep_on_device + device_cache_t query_indices_kept_on_device_; + device_cache_t target_indices_kept_on_device_; + + // Indices on device ready to be used + device_cache_t query_device_cache_; + device_cache_t target_device_cache_; + + // Indices currently being copied to device, they are going to replace the content of query_device_cache_ + // and target_device_cache_ after the next call to finish_generating_content_device( + device_cache_t next_query_device_cache_; + device_cache_t next_target_device_cache_; const bool same_query_and_target_; genomeworks::DefaultDeviceAllocator allocator_; @@ -174,67 +219,6 @@ private: const cudaStream_t cuda_stream_copy_; }; -/// IndexCacheDevice - Keeps copies of Indices in device memory -/// -/// The user tells cache which Indices to keep in cache using generate_query_cache_content() and generate_target_cache_content() and -/// retrieves indices using get_index_from_query_cache() and get_index_from_target_cache(). Trying to retrieve an -/// Index which was not previously stored in cache results in an exception. -/// -/// IndexCacheDevice relies on IndexCacheHost to provide actual indices for caching -class IndexCacheDevice -{ -public: - /// \brief Constructor - /// \param same_query_and_target true means that both query and target are the same, meaning that if requested index exists in query cache it can also be used by target cache directly - /// \param index_cache_host underlying host cache to get the indices from - IndexCacheDevice(bool same_query_and_target, - std::shared_ptr index_cache_host); - - IndexCacheDevice(const IndexCacheDevice&) = delete; - IndexCacheDevice& operator=(const IndexCacheDevice&) = delete; - IndexCacheDevice(IndexCacheDevice&&) = delete; - IndexCacheDevice& operator=(IndexCacheDevice&&) = delete; - ~IndexCacheDevice() = default; - - /// \brief Discards previously cached query Indices, creates new Indices and copies them to host memory - void generate_query_cache_content(const std::vector& descriptors_of_indices_to_cache); - - /// \brief Discards previously cached target Indices, creates new Indices and copies them to host memory - void generate_target_cache_content(const std::vector& descriptors_of_indices_to_cache); - - /// \brief Copies request Index to device memory - /// throws if that index is currently not in cache - std::shared_ptr get_index_from_query_cache(const IndexDescriptor& descriptor_of_index_to_cache); - - /// \brief Copies request Index to device memory - /// throws if that index is currently not in cache - std::shared_ptr get_index_from_target_cache(const IndexDescriptor& descriptor_of_index_to_cache); - -private: - using cache_type_t = std::unordered_map, - IndexDescriptorHash>; - - enum class CacheSelector - { - query_cache, - target_cache - }; - - /// \brief Discards previously cached Indices and caches new ones - /// Uses which_cache to determine if it should be working on query of target indices - /// - /// If same_query_and_target_ is true function checks the other cache to see if that index is already in cache - void generate_cache_content(const std::vector& descriptors_of_indices_to_cache, - CacheSelector which_cache); - - cache_type_t query_cache_; - cache_type_t target_cache_; - - const bool same_query_and_target_; - std::shared_ptr index_cache_host_; -}; - } // namespace cudamapper } // namespace genomeworks diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index 6b3191d42..c3d22d305 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -84,7 +84,7 @@ public: /// \param index_host_copy is a copy of index for a set of reads which has been previously computed and stored on the host. /// \param cuda_stream CUDA stream on which the work is to be done. Device arrays are also associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor is done IndexGPU(DefaultDeviceAllocator allocator, - const IndexHostCopy& index_host_copy, + const IndexHostCopy* const index_host_copy, const cudaStream_t cuda_stream = 0); /// \brief returns an array of representations of sketch elements @@ -127,6 +127,13 @@ public: /// \return length of the longest read in this index position_in_read_t number_of_basepairs_in_longest_read() const override; + /// \brief checks if index is ready to be used, index might not be ready if its creation is asynchronous + /// \return whether the index is ready to be used + bool is_ready() const override; + + /// \brief if is_ready() is true returns immediately, blocks until it becomes ready otherwise + void wait_to_be_ready() const override; + private: /// \brief generates the index void generate_index(const io::FastaParser& query_parser, @@ -152,6 +159,11 @@ private: const std::uint64_t window_size_ = 0; read_id_t number_of_reads_ = 0; position_in_read_t number_of_basepairs_in_longest_read_ = 0; + + // has index generation finished + mutable bool is_ready_; + // if index is not generated but copied from the host a pointer to it is saved here + const IndexHostCopy* const index_host_copy_source_; }; namespace details @@ -565,6 +577,8 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, , directions_of_reads_d_(allocator) , unique_representations_d_(allocator) , first_occurrence_of_representations_d_(allocator) + , is_ready_(false) // set to true at the end of generate_index() + , index_host_copy_source_(nullptr) { generate_index(parser, first_read_id_, @@ -581,42 +595,40 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, template IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, - const IndexHostCopy& index_host_copy, + const IndexHostCopy* const index_host_copy, const cudaStream_t cuda_stream) - : first_read_id_(index_host_copy.first_read_id()) - , kmer_size_(index_host_copy.kmer_size()) - , window_size_(index_host_copy.window_size()) - , representations_d_(index_host_copy.representations().size, allocator, cuda_stream) - , read_ids_d_(index_host_copy.read_ids().size, allocator, cuda_stream) - , positions_in_reads_d_(index_host_copy.positions_in_reads().size, allocator, cuda_stream) - , directions_of_reads_d_(index_host_copy.directions_of_reads().size, allocator, cuda_stream) - , unique_representations_d_(index_host_copy.unique_representations().size, allocator, cuda_stream) - , first_occurrence_of_representations_d_(index_host_copy.first_occurrence_of_representations().size, allocator, cuda_stream) + : first_read_id_(index_host_copy->first_read_id()) + , kmer_size_(index_host_copy->kmer_size()) + , window_size_(index_host_copy->window_size()) + , representations_d_(index_host_copy->representations().size, allocator, cuda_stream) + , read_ids_d_(index_host_copy->read_ids().size, allocator, cuda_stream) + , positions_in_reads_d_(index_host_copy->positions_in_reads().size, allocator, cuda_stream) + , directions_of_reads_d_(index_host_copy->directions_of_reads().size, allocator, cuda_stream) + , unique_representations_d_(index_host_copy->unique_representations().size, allocator, cuda_stream) + , first_occurrence_of_representations_d_(index_host_copy->first_occurrence_of_representations().size, allocator, cuda_stream) + , is_ready_(false) // set to true in wait_to_be_ready() + , index_host_copy_source_(index_host_copy) { - number_of_reads_ = index_host_copy.number_of_reads(); - number_of_basepairs_in_longest_read_ = index_host_copy.number_of_basepairs_in_longest_read(); + number_of_reads_ = index_host_copy->number_of_reads(); + number_of_basepairs_in_longest_read_ = index_host_copy->number_of_basepairs_in_longest_read(); - //H2D- representations_d_ = index_host_copy.representations(); - cudautils::device_copy_n(index_host_copy.representations().data, index_host_copy.representations().size, representations_d_.data(), cuda_stream); + //H2D- representations_d_ = index_host_copy->representations(); + cudautils::device_copy_n(index_host_copy->representations().data, index_host_copy->representations().size, representations_d_.data(), cuda_stream); - //H2D- read_ids_d_ = index_host_copy.read_ids(); - cudautils::device_copy_n(index_host_copy.read_ids().data, index_host_copy.read_ids().size, read_ids_d_.data(), cuda_stream); + //H2D- read_ids_d_ = index_host_copy->read_ids(); + cudautils::device_copy_n(index_host_copy->read_ids().data, index_host_copy->read_ids().size, read_ids_d_.data(), cuda_stream); - //H2D- positions_in_reads_d_ = index_host_copy.positions_in_reads(); - cudautils::device_copy_n(index_host_copy.positions_in_reads().data, index_host_copy.positions_in_reads().size, positions_in_reads_d_.data(), cuda_stream); + //H2D- positions_in_reads_d_ = index_host_copy->positions_in_reads(); + cudautils::device_copy_n(index_host_copy->positions_in_reads().data, index_host_copy->positions_in_reads().size, positions_in_reads_d_.data(), cuda_stream); - //H2D- directions_of_reads_d_ = index_host_copy.directions_of_reads(); - cudautils::device_copy_n(index_host_copy.directions_of_reads().data, index_host_copy.directions_of_reads().size, directions_of_reads_d_.data(), cuda_stream); + //H2D- directions_of_reads_d_ = index_host_copy->directions_of_reads(); + cudautils::device_copy_n(index_host_copy->directions_of_reads().data, index_host_copy->directions_of_reads().size, directions_of_reads_d_.data(), cuda_stream); - //H2D- unique_representations_d_ = index_host_copy.unique_representations(); - cudautils::device_copy_n(index_host_copy.unique_representations().data, index_host_copy.unique_representations().size, unique_representations_d_.data(), cuda_stream); + //H2D- unique_representations_d_ = index_host_copy->unique_representations(); + cudautils::device_copy_n(index_host_copy->unique_representations().data, index_host_copy->unique_representations().size, unique_representations_d_.data(), cuda_stream); - //H2D- first_occurrence_of_representations_d_ = index_host_copy.first_occurrence_of_representations(); - cudautils::device_copy_n(index_host_copy.first_occurrence_of_representations().data, index_host_copy.first_occurrence_of_representations().size, first_occurrence_of_representations_d_.data(), cuda_stream); - - // This is not completely necessary, but if removed one has to make sure that the next step - // uses the same stream or that sync is done in caller - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); + //H2D- first_occurrence_of_representations_d_ = index_host_copy->first_occurrence_of_representations(); + cudautils::device_copy_n(index_host_copy->first_occurrence_of_representations().data, index_host_copy->first_occurrence_of_representations().size, first_occurrence_of_representations_d_.data(), cuda_stream); } template @@ -679,6 +691,24 @@ position_in_read_t IndexGPU::number_of_basepairs_in_longest_r return number_of_basepairs_in_longest_read_; } +template +bool IndexGPU::is_ready() const +{ + return is_ready_; +} + +template +void IndexGPU::wait_to_be_ready() const +{ + // if index is not ready for usage wait for it to become ready + if (!is_ready()) + { + assert(index_host_copy_source_); + index_host_copy_source_->finish_copying(); + is_ready_ = true; + } +} + template void IndexGPU::generate_index(const io::FastaParser& parser, const read_id_t first_read_id, @@ -835,6 +865,10 @@ void IndexGPU::generate_index(const io::FastaParser& parser, first_occurrence_of_representations_d_, cuda_stream); } + + // TODO: replace with stream callback + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); + is_ready_ = true; } } // namespace cudamapper diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 8ca7a183c..800ff9de2 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -118,12 +118,14 @@ void IndexHostCopy::finish_copying() const std::unique_ptr IndexHostCopy::copy_index_to_device(DefaultDeviceAllocator allocator, const cudaStream_t cuda_stream) const { - // pin_memory_object registers host array as pinned memory and unregisters it on its destruction (i.e. at the end of this function) - IndexHostMemoryPinner pin_memory_object(const_cast(*this)); + // register pinned memory, memory gets unpinned in finish_copying() + memory_pinner_.register_pinned_memory(); return std::make_unique>(allocator, - *this, + this, cuda_stream); + + // no stream synchronization, synchronization done in finish_copying() } const IndexHostCopyBase::ArrayView IndexHostCopy::representations() const @@ -183,14 +185,14 @@ std::uint64_t IndexHostCopy::window_size() const IndexHostCopy::IndexHostMemoryPinner::IndexHostMemoryPinner(IndexHostCopy& index_host_copy) : index_host_copy_(index_host_copy) - , memory_pinned_(false) + , times_memory_pinned_(0) { } IndexHostCopy::IndexHostMemoryPinner::~IndexHostMemoryPinner() { // if memory was not unregistered (due to either a bug or an expection) unregister it - if (memory_pinned_) + if (times_memory_pinned_ != 0) { assert(!"memory should always be unregistered by unregister_pinned_memory()"); GW_NVTX_RANGE(profiler, "unregister_pinned_memory"); @@ -200,19 +202,27 @@ IndexHostCopy::IndexHostMemoryPinner::~IndexHostMemoryPinner() void IndexHostCopy::IndexHostMemoryPinner::register_pinned_memory() { - GW_NVTX_RANGE(profiler, "register_pinned_memory"); - GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.underlying_array_.data(), - index_host_copy_.underlying_array_.size() * sizeof(unsigned char), - cudaHostRegisterDefault)); - memory_pinned_ = true; + // only pin memory if it hasn't been pinned yet + if (0 == times_memory_pinned_) + { + GW_NVTX_RANGE(profiler, "register_pinned_memory"); + GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.underlying_array_.data(), + index_host_copy_.underlying_array_.size() * sizeof(unsigned char), + cudaHostRegisterDefault)); + } + ++times_memory_pinned_; } void IndexHostCopy::IndexHostMemoryPinner::unregister_pinned_memory() { - assert(memory_pinned_); - GW_NVTX_RANGE(profiler, "unregister_pinned_memory"); - GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.underlying_array_.data())); - memory_pinned_ = false; + assert(times_memory_pinned_ > 0); + // only unpin memory if this is the last unpinning + if (1 == times_memory_pinned_) + { + GW_NVTX_RANGE(profiler, "unregister_pinned_memory"); + GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.underlying_array_.data())); + } + --times_memory_pinned_; } } // namespace cudamapper diff --git a/cudamapper/src/index_host_copy.cuh b/cudamapper/src/index_host_copy.cuh index 5f928ad86..1bc1eb2f3 100644 --- a/cudamapper/src/index_host_copy.cuh +++ b/cudamapper/src/index_host_copy.cuh @@ -125,8 +125,10 @@ private: private: IndexHostCopy& index_host_copy_; - // is memory currently pinned - bool memory_pinned_; + // Theoretically speaking it could happen that the same IndexHostCopy gets copied multiple time in a row. In that case + // memory should be unpinned only once the last copy has finished (i.e. the caller waits for it to be finished). + // This counter is increased every time new copy have started and decreased every time the user waits for the copy to finish. + int32_t times_memory_pinned_; }; // use a single underlying vector in order to reduce memory fragmentation when using pool allocators diff --git a/cudamapper/src/main.cu b/cudamapper/src/main.cu index 8d1cd2a08..75195755c 100644 --- a/cudamapper/src/main.cu +++ b/cudamapper/src/main.cu @@ -191,13 +191,13 @@ struct OverlapsAndCigars }; /// \brief does overlapping and matching for pairs of query and target indices from device_batch -/// \param device_batch -/// \param device_cache data will be loaded into cache within the function +/// \param device_batch list of query and target indices to be processed +/// \param index_cache indices should be loaded into cache beforehand /// \param application_parameters -/// \param overlaps_and_cigars_to_process overlaps and cigars are output here and the then consumed by another thread -/// \param cuda_stream +/// \param overlaps_and_cigars_to_process overlaps and cigars are output here and then consumed by another thread +/// \param cuda_stream cuda stream on which the computation should be done void process_one_device_batch(const IndexBatch& device_batch, - IndexCacheDevice& device_cache, + IndexCache& index_cache, const ApplicationParameters& application_parameters, DefaultDeviceAllocator device_allocator, ThreadsafeProducerConsumer& overlaps_and_cigars_to_process, @@ -206,11 +206,7 @@ void process_one_device_batch(const IndexBatch& device_batch, GW_NVTX_RANGE(profiler, "main::process_one_device_batch"); const std::vector& query_index_descriptors = device_batch.query_indices; const std::vector& target_index_descriptors = device_batch.target_indices; - - // fetch indices for this batch from host memory assert(!query_index_descriptors.empty() && !target_index_descriptors.empty()); - device_cache.generate_query_cache_content(query_index_descriptors); - device_cache.generate_target_cache_content(target_index_descriptors); // process pairs of query and target indices for (const IndexDescriptor& query_index_descriptor : query_index_descriptors) @@ -220,8 +216,8 @@ void process_one_device_batch(const IndexBatch& device_batch, // if doing all-to-all skip pairs in which target batch has smaller id than query batch as it will be covered by symmetry if (!application_parameters.all_to_all || target_index_descriptor.first_read() >= query_index_descriptor.first_read()) { - std::shared_ptr query_index = device_cache.get_index_from_query_cache(query_index_descriptor); - std::shared_ptr target_index = device_cache.get_index_from_target_cache(target_index_descriptor); + std::shared_ptr query_index = index_cache.get_index_from_query_cache(query_index_descriptor); + std::shared_ptr target_index = index_cache.get_index_from_target_cache(target_index_descriptor); // find anchors and overlaps auto matcher = Matcher::create_matcher(device_allocator, @@ -264,17 +260,15 @@ void process_one_device_batch(const IndexBatch& device_batch, } /// \brief loads one batch into host memory and then processes its device batches one by one -/// \param batch +/// \param batch list of all query and target indices that belong to this batch, as well as subgroups of those indices that belong to device batches /// \param application_parameters -/// \param host_cache data will be loaded into cache within the function -/// \param device_cache data will be loaded into cache within the function +/// \param index_cache data will be loaded into cache within the function /// \param overlaps_and_cigars_to_process overlaps and cigars are output to this structure and the then consumed by another thread /// \param cuda_stream void process_one_batch(const BatchOfIndices& batch, const ApplicationParameters& application_parameters, DefaultDeviceAllocator device_allocator, - IndexCacheHost& host_cache, - IndexCacheDevice& device_cache, + IndexCache& index_cache, ThreadsafeProducerConsumer& overlaps_and_cigars_to_process, cudaStream_t cuda_stream) { @@ -289,28 +283,53 @@ void process_one_batch(const BatchOfIndices& batch, // load indices into host memory { + GW_NVTX_RANGE(profiler, "main::process_one_batch::generte_host_indices"); assert(!host_batch.query_indices.empty() && !host_batch.target_indices.empty() && !device_batches.empty()); - GW_NVTX_RANGE(profiler, "main::process_one_batch::host_indices"); - host_cache.start_generating_query_cache_content(host_batch.query_indices, - device_batches.front().query_indices, - skip_copy_to_host); - host_cache.start_generating_target_cache_content(host_batch.target_indices, - device_batches.front().target_indices, - skip_copy_to_host); - host_cache.finish_generating_query_cache_content(); - host_cache.finish_generating_target_cache_content(); + index_cache.generate_content_query_host(host_batch.query_indices, + device_batches.front().query_indices, + skip_copy_to_host); + index_cache.generate_content_target_host(host_batch.target_indices, + device_batches.front().target_indices, + skip_copy_to_host); } // process device batches one by one - for (const IndexBatch& device_batch : batch.device_batches) + // Processing one device batch is overlapped with fetching indices from host to device for the next device batch. + // Loop uses copy_device_batch_index as its index, compute_device_batch_index is equal to copy_device_batch_index - 1 + // (compute_device_batch_index does not actually exist). + // copy_device_batch_index loops over one more element because for such element compute_device_batch_index would have been equal to the last batch + for (int32_t copy_device_batch_index = 0; copy_device_batch_index < get_size(batch.device_batches) + 1; ++copy_device_batch_index) { - process_one_device_batch(device_batch, - device_cache, - application_parameters, - device_allocator, - overlaps_and_cigars_to_process, - cuda_stream); + if (copy_device_batch_index > 0) + { + // if not the first batch wait for previous batch to finish copying + GW_NVTX_RANGE(profiler, "main::process_one_batch::finish_generating_device_indices"); + index_cache.finish_generating_content_query_device(); + index_cache.finish_generating_content_target_device(); + } + + if (copy_device_batch_index < get_size(batch.device_batches)) + { + // if not pass-the-last batch start copying the batch + GW_NVTX_RANGE(profiler, "main::process_one_batch::start_generating_device_indices"); + const std::vector& query_index_batch_to_start_copying = batch.device_batches[copy_device_batch_index].query_indices; + const std::vector& target_index_batch_to_start_copying = batch.device_batches[copy_device_batch_index].target_indices; + index_cache.start_generating_content_query_device(query_index_batch_to_start_copying); + index_cache.start_generating_content_target_device(target_index_batch_to_start_copying); + } + + if (copy_device_batch_index > 0) + { + // when copy_device_batch_index == 0 then compute_device_batch_index == -1, so there is no batch to process + GW_NVTX_RANGE(profiler, "main::process_one_batch::process_device_batch"); + process_one_device_batch(batch.device_batches[copy_device_batch_index - 1], + index_cache, + application_parameters, + device_allocator, + overlaps_and_cigars_to_process, + cuda_stream); + } } } @@ -397,21 +416,17 @@ void worker_thread_function(const int32_t device_id, DefaultDeviceAllocator device_allocator = create_default_device_allocator(application_parameters.max_cached_memory_bytes); - // create host_cache, data is not loaded at this point but later as each batch gets processed - auto host_cache = std::make_shared(application_parameters.all_to_all, - device_allocator, - application_parameters.query_parser, - application_parameters.target_parser, - application_parameters.kmer_size, - application_parameters.windows_size, - true, // hash_representations - application_parameters.filtering_parameter, - cuda_stream_computation, - cuda_stream_copy); - - // create host_cache, data is not loaded at this point but later as each batch gets processed - IndexCacheDevice device_cache(application_parameters.all_to_all, - host_cache); + // create index_cache, indices are not created at this point but later as each batch gets processed + IndexCache index_cache(application_parameters.all_to_all, + device_allocator, + application_parameters.query_parser, + application_parameters.target_parser, + application_parameters.kmer_size, + application_parameters.windows_size, + true, // hash_representations + application_parameters.filtering_parameter, + cuda_stream_computation, + cuda_stream_copy); // data structure used to exchnage data with postprocess_and_write_thread ThreadsafeProducerConsumer overlaps_and_cigars_to_process; @@ -445,8 +460,7 @@ void worker_thread_function(const int32_t device_id, process_one_batch(batch_of_indices.value(), application_parameters, device_allocator, - *host_cache, - device_cache, + index_cache, overlaps_and_cigars_to_process, cuda_stream_computation); } diff --git a/cudamapper/tests/Test_CudamapperIndexCache.cu b/cudamapper/tests/Test_CudamapperIndexCache.cu index 59de3f340..210452b0f 100644 --- a/cudamapper/tests/Test_CudamapperIndexCache.cu +++ b/cudamapper/tests/Test_CudamapperIndexCache.cu @@ -30,9 +30,7 @@ namespace genomeworks namespace cudamapper { -// *** Test IndexCacheHost *** - -void check_if_index_is_correct(const std::shared_ptr& index, +void check_if_index_is_correct(const std::shared_ptr& index, const std::vector& expected_representations, const std::vector& expected_read_ids, const std::vector& expected_positions_in_reads, @@ -102,7 +100,7 @@ void check_if_index_is_correct(const std::shared_ptr& index, ASSERT_EQ(index_maximum_kmer_size, expected_maximum_kmer_size) << " test_uid: " << test_uid; } -TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) +TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) { // catcaag_aagcta.fasta k = 3 w = 2 @@ -334,21 +332,22 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) std::vector catcaag_aagcta_separate_index_descriptors({catcaag_index_descriptor, aagcta_index_descriptor}); std::vector catcaag_aagcta_one_index_descriptors({catcaag_aagcta_index_descriptor}); - IndexCacheHost index_host_cache(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate, - cuda_stream_copy); - - index_host_cache.start_generating_query_cache_content(catcaag_index_descriptors); - index_host_cache.finish_generating_query_cache_content(); - - auto index_query_catcaag = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); + IndexCache index_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate, + cuda_stream_copy); + + index_cache.generate_content_query_host(catcaag_index_descriptors); + index_cache.start_generating_content_query_device(catcaag_index_descriptors); + index_cache.finish_generating_content_query_device(); + + auto index_query_catcaag = index_cache.get_index_from_query_cache(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag, catcaag_representations, catcaag_read_ids, @@ -362,17 +361,18 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_1"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - - index_host_cache.start_generating_target_cache_content(aagcta_index_descriptors); - index_host_cache.finish_generating_target_cache_content(); - - index_query_catcaag = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); + "test_index_cache_same_query_and_target_1"); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + + index_cache.generate_content_target_host(aagcta_index_descriptors); + index_cache.start_generating_content_target_device(aagcta_index_descriptors); + index_cache.finish_generating_content_target_device(); + + index_query_catcaag = index_cache.get_index_from_query_cache(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag, catcaag_representations, catcaag_read_ids, @@ -386,11 +386,11 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_2"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - auto index_target_aagcta = index_host_cache.get_index_from_target_cache(aagcta_index_descriptor); + "test_index_cache_same_query_and_target_2"); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + auto index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, aagcta_read_ids, @@ -404,14 +404,15 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_3"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + "test_index_cache_same_query_and_target_3"); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - index_host_cache.start_generating_query_cache_content(aagcta_index_descriptors); - index_host_cache.finish_generating_target_cache_content(); + index_cache.generate_content_query_host(aagcta_index_descriptors); + index_cache.start_generating_content_query_device(aagcta_index_descriptors); + index_cache.finish_generating_content_query_device(); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_index_descriptor)); - auto index_query_aagcta = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_index_descriptor)); + auto index_query_aagcta = index_cache.get_index_from_query_cache(aagcta_index_descriptor); check_if_index_is_correct(index_query_aagcta, aagcta_representations, aagcta_read_ids, @@ -425,10 +426,10 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_4"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - index_target_aagcta = index_host_cache.get_index_from_target_cache(aagcta_index_descriptor); + "test_index_cache_same_query_and_target_4"); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, aagcta_read_ids, @@ -442,13 +443,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_5"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + "test_index_cache_same_query_and_target_5"); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - index_host_cache.start_generating_query_cache_content(catcaag_aagcta_separate_index_descriptors); - index_host_cache.finish_generating_target_cache_content(); + index_cache.generate_content_query_host(catcaag_aagcta_separate_index_descriptors); + index_cache.start_generating_content_query_device(catcaag_aagcta_separate_index_descriptors); + index_cache.finish_generating_content_query_device(); - auto index_query_catcaag_separate = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); + auto index_query_catcaag_separate = index_cache.get_index_from_query_cache(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag_separate, catcaag_representations, catcaag_read_ids, @@ -462,8 +464,8 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_6"); - auto index_query_aagcta_separate = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); + "test_index_cache_same_query_and_target_6"); + auto index_query_aagcta_separate = index_cache.get_index_from_query_cache(aagcta_index_descriptor); check_if_index_is_correct(index_query_aagcta_separate, aagcta_representations, aagcta_read_ids, @@ -477,10 +479,10 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_7"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - index_target_aagcta = index_host_cache.get_index_from_target_cache(aagcta_index_descriptor); + "test_index_cache_same_query_and_target_7"); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, aagcta_read_ids, @@ -494,13 +496,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_8"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + "test_index_cache_same_query_and_target_8"); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - index_host_cache.start_generating_target_cache_content(catcaag_aagcta_one_index_descriptors); - index_host_cache.finish_generating_target_cache_content(); + index_cache.generate_content_target_host(catcaag_aagcta_one_index_descriptors); + index_cache.start_generating_content_target_device(catcaag_aagcta_one_index_descriptors); + index_cache.finish_generating_content_target_device(); - index_query_catcaag_separate = index_host_cache.get_index_from_query_cache(catcaag_index_descriptor); + index_query_catcaag_separate = index_cache.get_index_from_query_cache(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag_separate, catcaag_representations, catcaag_read_ids, @@ -514,8 +517,8 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_9"); - index_query_aagcta_separate = index_host_cache.get_index_from_query_cache(aagcta_index_descriptor); + "test_index_cache_same_query_and_target_9"); + index_query_aagcta_separate = index_cache.get_index_from_query_cache(aagcta_index_descriptor); check_if_index_is_correct(index_query_aagcta_separate, aagcta_representations, aagcta_read_ids, @@ -529,11 +532,11 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_10"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(aagcta_index_descriptor)); - auto catcaag_aagcta_target_aagcta = index_host_cache.get_index_from_target_cache(catcaag_aagcta_index_descriptor); + "test_index_cache_same_query_and_target_10"); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor)); + auto catcaag_aagcta_target_aagcta = index_cache.get_index_from_target_cache(catcaag_aagcta_index_descriptor); check_if_index_is_correct(catcaag_aagcta_target_aagcta, catcaag_aagcta_representations, catcaag_aagcta_read_ids, @@ -547,7 +550,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) catcaag_aagcta_number_of_basepairs_in_longest_read, catcaag_aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_same_query_and_target_11"); + "test_index_cache_same_query_and_target_11"); } GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generate)); @@ -555,7 +558,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_same_query_and_target) GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream_generate)); } -TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_target) +TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target) { // aagcta.fasta ctacaag.fasta k = 3 w = 2 @@ -709,21 +712,22 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta IndexDescriptor index_descriptor(0, 1); std::vector index_descriptors({index_descriptor}); - IndexCacheHost index_host_cache(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate, - cuda_stream_copy); - - index_host_cache.start_generating_query_cache_content(index_descriptors); - index_host_cache.finish_generating_query_cache_content(); - - auto index_query_aagcta = index_host_cache.get_index_from_query_cache(index_descriptor); + IndexCache index_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate, + cuda_stream_copy); + + index_cache.generate_content_query_host(index_descriptors); + index_cache.start_generating_content_query_device(index_descriptors); + index_cache.finish_generating_content_query_device(); + + auto index_query_aagcta = index_cache.get_index_from_query_cache(index_descriptor); check_if_index_is_correct(index_query_aagcta, aagcta_representations, aagcta_read_ids, @@ -737,13 +741,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_not_the_same_query_and_target_1"); - ASSERT_ANY_THROW(index_host_cache.get_index_from_target_cache(index_descriptor)); + "test_index_cache_not_the_same_query_and_target_1"); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(index_descriptor)); - index_host_cache.start_generating_target_cache_content(index_descriptors); - index_host_cache.finish_generating_target_cache_content(); + index_cache.generate_content_target_host(index_descriptors); + index_cache.start_generating_content_target_device(index_descriptors); + index_cache.finish_generating_content_target_device(); - index_query_aagcta = index_host_cache.get_index_from_query_cache(index_descriptor); + index_query_aagcta = index_cache.get_index_from_query_cache(index_descriptor); check_if_index_is_correct(index_query_aagcta, aagcta_representations, aagcta_read_ids, @@ -757,8 +762,8 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_not_the_same_query_and_target_2"); - auto index_target_catcaag = index_host_cache.get_index_from_target_cache(index_descriptor); + "test_index_cache_not_the_same_query_and_target_2"); + auto index_target_catcaag = index_cache.get_index_from_target_cache(index_descriptor); check_if_index_is_correct(index_target_catcaag, catcaag_representations, catcaag_read_ids, @@ -772,7 +777,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_not_the_same_query_and_target_3"); + "test_index_cache_not_the_same_query_and_target_3"); } GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generate)); @@ -780,7 +785,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_not_the_same_query_and_ta GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream_generate)); } -TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) +TEST(TestCudamapperIndexCaching, test_index_cache_keep_on_device) { // AAGCTA: AAG(0f), AGC(2r), CTA(3f) @@ -844,30 +849,34 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) IndexDescriptor index_descriptor(0, 1); std::vector index_descriptors({index_descriptor}); - auto index_cache_host = std::make_shared(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate, - cuda_stream_copy); - - index_cache_host->start_generating_query_cache_content(index_descriptors, index_descriptors); - index_cache_host->start_generating_target_cache_content(index_descriptors, index_descriptors); - index_cache_host->finish_generating_query_cache_content(); - index_cache_host->finish_generating_target_cache_content(); - - auto index_query_temp_device_cache = index_cache_host->get_index_from_query_cache(index_descriptor); - auto index_query_copy_from_host = index_cache_host->get_index_from_query_cache(index_descriptor); - auto index_target_temp_device_cache = index_cache_host->get_index_from_target_cache(index_descriptor); - auto index_target_copy_from_host = index_cache_host->get_index_from_target_cache(index_descriptor); - - ASSERT_EQ(index_query_temp_device_cache, index_target_temp_device_cache); - ASSERT_NE(index_query_temp_device_cache, index_query_copy_from_host); - ASSERT_NE(index_target_temp_device_cache, index_target_copy_from_host); + IndexCache index_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate, + cuda_stream_copy); + + index_cache.generate_content_query_host(index_descriptors, index_descriptors); + index_cache.start_generating_content_query_device(index_descriptors); // 1st copy, kept on device + index_cache.generate_content_target_host(index_descriptors, index_descriptors); + index_cache.start_generating_content_target_device(index_descriptors); + index_cache.finish_generating_content_query_device(); + index_cache.finish_generating_content_target_device(); + + auto index_query_temp_device_cache = index_cache.get_index_from_query_cache(index_descriptor); + auto index_target_temp_device_cache = index_cache.get_index_from_target_cache(index_descriptor); + + index_cache.start_generating_content_query_device(index_descriptors); + index_cache.start_generating_content_target_device(index_descriptors); + index_cache.finish_generating_content_query_device(); + index_cache.finish_generating_content_target_device(); + + auto index_query_copy_from_host = index_cache.get_index_from_query_cache(index_descriptor); // 2nd copy, copied from host + auto index_target_copy_from_host = index_cache.get_index_from_target_cache(index_descriptor); check_if_index_is_correct(index_query_temp_device_cache, aagcta_representations, @@ -882,7 +891,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_keep_on_device_1"); + "test_index_cache_keep_on_device_1"); check_if_index_is_correct(index_query_copy_from_host, aagcta_representations, aagcta_read_ids, @@ -896,7 +905,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_keep_on_device_2"); + "test_index_cache_keep_on_device_2"); check_if_index_is_correct(index_target_temp_device_cache, aagcta_representations, aagcta_read_ids, @@ -910,7 +919,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_keep_on_device_3"); + "test_index_cache_keep_on_device_3"); check_if_index_is_correct(index_target_copy_from_host, aagcta_representations, aagcta_read_ids, @@ -924,7 +933,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_host_keep_on_device_4"); + "test_index_cache_keep_on_device_4"); } GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generate)); @@ -932,9 +941,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_host_keep_on_device) GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream_generate)); } -// *** Test IndexCacheDevice *** - -TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) +TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target_2) { // >read_0 // CATCAAG @@ -1052,25 +1059,22 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) std::vector aagcta_index_descriptors({aagcta_index_descriptor}); std::vector catcaag_aagcta_index_descriptors({catcaag_index_descriptor, aagcta_index_descriptor}); - auto index_cache_host = std::make_shared(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate, - cuda_stream_copy); - - IndexCacheDevice index_cache_device(same_query_and_target, - index_cache_host); - - index_cache_host->start_generating_query_cache_content(catcaag_index_descriptors); - index_cache_host->finish_generating_query_cache_content(); - ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(catcaag_index_descriptor)); - index_cache_device.generate_query_cache_content(catcaag_index_descriptors); - auto index_query_catcaag = index_cache_device.get_index_from_query_cache(catcaag_index_descriptor); + IndexCache index_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate, + cuda_stream_copy); + + index_cache.generate_content_query_host(catcaag_index_descriptors); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_index_descriptor)); + index_cache.start_generating_content_query_device(catcaag_index_descriptors); + index_cache.finish_generating_content_query_device(); + auto index_query_catcaag = index_cache.get_index_from_query_cache(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag, catcaag_representations, catcaag_read_ids, @@ -1084,18 +1088,18 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_device_same_query_and_target_1"); - ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(aagcta_index_descriptor)); - - index_cache_host->start_generating_target_cache_content(catcaag_aagcta_index_descriptors); - index_cache_host->finish_generating_target_cache_content(); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(aagcta_index_descriptor)); - index_cache_device.generate_target_cache_content(catcaag_aagcta_index_descriptors); - - auto index_target_catcaag = index_cache_device.get_index_from_target_cache(catcaag_index_descriptor); + "test_index_cache_same_query_and_target_2_1"); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor)); + + index_cache.generate_content_target_host(catcaag_aagcta_index_descriptors); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor)); + index_cache.start_generating_content_target_device(catcaag_aagcta_index_descriptors); + index_cache.finish_generating_content_target_device(); + + auto index_target_catcaag = index_cache.get_index_from_target_cache(catcaag_index_descriptor); ASSERT_EQ(index_query_catcaag, index_target_catcaag); // check same object is used because same_query_and_target == true check_if_index_is_correct(index_target_catcaag, catcaag_representations, @@ -1110,9 +1114,9 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_device_same_query_and_target_2"); + "test_index_cache_same_query_and_target_2_2"); - auto index_target_aagcta = index_cache_device.get_index_from_target_cache(aagcta_index_descriptor); + auto index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, aagcta_read_ids, @@ -1126,11 +1130,11 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_device_same_query_and_target_3"); + "test_index_cache_same_query_and_target_2_3"); // get the same query and target indices again and make sure they point to the same objects as the last time - auto index_query_catcaag_1 = index_cache_device.get_index_from_query_cache(catcaag_index_descriptor); - auto index_target_aagcta_1 = index_cache_device.get_index_from_target_cache(aagcta_index_descriptor); + auto index_query_catcaag_1 = index_cache.get_index_from_query_cache(catcaag_index_descriptor); + auto index_target_aagcta_1 = index_cache.get_index_from_target_cache(aagcta_index_descriptor); ASSERT_EQ(index_query_catcaag, index_query_catcaag_1); ASSERT_EQ(index_target_aagcta, index_target_aagcta_1); } @@ -1140,7 +1144,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_same_query_and_target) GW_CU_CHECK_ERR(cudaStreamDestroy(cuda_stream_generate)); } -TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_target) +TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target_2) { // AAGCTA: AAG(0f), AGC(2r), CTA(3f) // CATCAAG: AAG(4f), ATC(1f), ATG(0r), CAA(3f) @@ -1251,27 +1255,24 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ IndexDescriptor index_descriptor(0, 1); std::vector index_descriptors({index_descriptor}); - auto index_cache_host = std::make_shared(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate, - cuda_stream_copy); - - IndexCacheDevice index_cache_device(same_query_and_target, - index_cache_host); - - index_cache_host->start_generating_query_cache_content(index_descriptors); - index_cache_host->finish_generating_query_cache_content(); - ASSERT_ANY_THROW(index_cache_device.get_index_from_query_cache(index_descriptor)); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(index_descriptor)); - - index_cache_device.generate_query_cache_content(index_descriptors); - auto index_query = index_cache_device.get_index_from_query_cache(index_descriptor); + IndexCache index_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate, + cuda_stream_copy); + + index_cache.generate_content_query_host(index_descriptors); + ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(index_descriptor)); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(index_descriptor)); + + index_cache.start_generating_content_query_device(index_descriptors); + index_cache.finish_generating_content_query_device(); + auto index_query = index_cache.get_index_from_query_cache(index_descriptor); check_if_index_is_correct(index_query, aagcta_representations, aagcta_read_ids, @@ -1285,17 +1286,15 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_device_not_the_same_query_and_target_1"); - ASSERT_ANY_THROW(index_cache_device.get_index_from_target_cache(index_descriptor)); - - ASSERT_ANY_THROW(index_cache_device.generate_target_cache_content(index_descriptors)); + "test_index_cache_not_the_same_query_and_target_2_1"); + ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(index_descriptor)); - index_cache_host->start_generating_target_cache_content(index_descriptors); - index_cache_host->finish_generating_target_cache_content(); - index_cache_device.generate_target_cache_content(index_descriptors); + index_cache.generate_content_target_host(index_descriptors); + index_cache.start_generating_content_target_device(index_descriptors); + index_cache.finish_generating_content_target_device(); - index_query = index_cache_device.get_index_from_query_cache(index_descriptor); - auto index_target = index_cache_device.get_index_from_target_cache(index_descriptor); + index_query = index_cache.get_index_from_query_cache(index_descriptor); + auto index_target = index_cache.get_index_from_target_cache(index_descriptor); ASSERT_NE(index_query, index_target); check_if_index_is_correct(index_query, aagcta_representations, @@ -1310,7 +1309,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ aagcta_number_of_basepairs_in_longest_read, aagcta_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_device_not_the_same_query_and_target_2"); + "test_index_cache_not_the_same_query_and_target_2_2"); check_if_index_is_correct(index_target, catcaag_representations, catcaag_read_ids, @@ -1324,11 +1323,11 @@ TEST(TestCudamapperIndexCaching, test_index_cache_device_not_the_same_query_and_ catcaag_number_of_basepairs_in_longest_read, catcaag_maximum_kmer_size, cuda_stream_generate, - "test_index_cache_device_not_the_same_query_and_target_3"); + "test_index_cache_not_the_same_query_and_target_2_3"); // get the same query and target indices again and make sure they point to the same objects as the last time - auto index_query_1 = index_cache_device.get_index_from_query_cache(index_descriptor); - auto index_target_1 = index_cache_device.get_index_from_target_cache(index_descriptor); + auto index_query_1 = index_cache.get_index_from_query_cache(index_descriptor); + auto index_target_1 = index_cache.get_index_from_target_cache(index_descriptor); ASSERT_EQ(index_query, index_query_1); ASSERT_EQ(index_target, index_target_1); } From 31b70df240e324d9e39ced6906a216ea3c04c5a8 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 31 Jul 2020 14:25:17 +0200 Subject: [PATCH 013/281] [cudamapper] Allocator supports using default stream other than stream 0 --- .../genomeworks/utils/allocator.hpp | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp index f9f4919b2..5e227b4e8 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp @@ -38,7 +38,11 @@ class CudaMallocAllocator using pointer = T*; /// \brief default constructor - CudaMallocAllocator() = default; + /// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead, ignored in this allocator + CudaMallocAllocator(const std::vector& default_streams = {{0}}) + { + static_cast(default_streams); + } /// \brief copy constructor /// \param rhs input allocator @@ -108,7 +112,7 @@ class CudaMallocAllocator /// \return pointer to allocated memory /// \throw device_memory_allocation_exception if allocation was not successful pointer allocate(std::size_t n, - const std::vector& streams = {{0}}) + const std::vector& streams = {}) { static_cast(streams); void* ptr = nullptr; @@ -159,8 +163,11 @@ class CachingDeviceAllocator /// \brief Constructor /// \param max_cached_bytes max bytes used by memory resource - CachingDeviceAllocator(size_t max_cached_bytes) + /// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead + CachingDeviceAllocator(size_t max_cached_bytes, + const std::vector& default_streams = {{0}}) : memory_resource_(std::make_shared(max_cached_bytes)) + , default_streams_(default_streams) { } @@ -231,13 +238,13 @@ class CachingDeviceAllocator /// \brief asynchronously allocates a device array with enough space for n elements of value_type /// \param n number of elements to allocate the array for - /// \param streams on deallocation this memory block is guaranteed to live at least until all previously scheduled work in these streams has finished, if no stream is specified default stream (0) is used + /// \param streams on deallocation this memory block is guaranteed to live at least until all previously scheduled work in these streams has finished, if no streams are default_streams from constructor are used, if no default_streams were specified in constructor default stream is used /// \return pointer to allocated memory /// \throw device_memory_allocation_exception if allocation was not successful pointer allocate(std::size_t n, - const std::vector& streams = {{0}}) + const std::vector& streams = {}) { - assert(!streams.empty()); + assert(!streams.empty() || !default_streams_.empty()); if (!memory_resource_) { @@ -246,7 +253,9 @@ class CachingDeviceAllocator std::abort(); } void* ptr = nullptr; - cudaError_t err = memory_resource_->DeviceAllocate(&ptr, n * sizeof(T), streams); + cudaError_t err = memory_resource_->DeviceAllocate(&ptr, + n * sizeof(T), + streams.empty() ? default_streams_ : streams); // if no streams have been specified use default_streams_ if (err == cudaErrorMemoryAllocation) { throw device_memory_allocation_exception(); @@ -277,6 +286,7 @@ class CachingDeviceAllocator private: std::shared_ptr memory_resource_; + std::vector default_streams_; }; #ifdef GW_ENABLE_CACHING_ALLOCATOR @@ -295,12 +305,16 @@ using DefaultDeviceAllocator = CudaMallocAllocator; /// Default constuction of CachingDeviceAllocator yields an dummy object /// which cannot allocate memory. /// \param max_cached_bytes max bytes used by memory resource used by CachingDeviceAllocator (default: 2GiB, unused for CudaMallocAllocator) -inline DefaultDeviceAllocator create_default_device_allocator(std::size_t max_caching_size = 2ull * 1024 * 1024 * 1024) +/// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead (unused for CudaMallocAllocator) +inline DefaultDeviceAllocator create_default_device_allocator(std::size_t max_caching_size = 2ull * 1024 * 1024 * 1024, + const std::vector& default_streams = {}) { #ifdef GW_ENABLE_CACHING_ALLOCATOR - return DefaultDeviceAllocator(max_caching_size); + return DefaultDeviceAllocator(max_caching_size, + default_streams); #else static_cast(max_caching_size); + static_cast(default_streams); return DefaultDeviceAllocator(); #endif } From 358b78929f8712a9d241e79f0436fcc1c9d36b56 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 31 Jul 2020 15:05:26 +0200 Subject: [PATCH 014/281] [cudamapper] DefaultDeviceAllocator in main.cu uses compute stream as the default stream --- cudamapper/src/main.cu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cudamapper/src/main.cu b/cudamapper/src/main.cu index 75195755c..3d0eb80f9 100644 --- a/cudamapper/src/main.cu +++ b/cudamapper/src/main.cu @@ -414,7 +414,9 @@ void worker_thread_function(const int32_t device_id, // This function is expected to run in a separate thread so set current device in order to avoid problems GW_CU_CHECK_ERR(cudaSetDevice(device_id)); - DefaultDeviceAllocator device_allocator = create_default_device_allocator(application_parameters.max_cached_memory_bytes); + // Whenever device_allocator is used directly (e.g. in Thrust library) it will be associated with cuda_stream_computation + DefaultDeviceAllocator device_allocator = create_default_device_allocator(application_parameters.max_cached_memory_bytes, + {cuda_stream_computation}); // create index_cache, indices are not created at this point but later as each batch gets processed IndexCache index_cache(application_parameters.all_to_all, From 85e4e02eda30f0def76992090f8d441bbf054f91 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 31 Jul 2020 15:07:39 +0200 Subject: [PATCH 015/281] [cudamapper] Setting streams for device_buffer in places where it hasn't been done already --- .../claraparabricks/genomeworks/utils/cudasort.cuh | 2 +- cudamapper/src/index_gpu.cuh | 12 ++++++------ cudamapper/src/matcher_gpu.cu | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/cudasort.cuh b/common/base/include/claraparabricks/genomeworks/utils/cudasort.cuh index c9c4d7bfd..5ba0e67c1 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/cudasort.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/cudasort.cuh @@ -170,7 +170,7 @@ void sort_by_two_keys(device_buffer& more_significant_keys, device_buffer less_significant_key_sorted(number_of_elements, allocator, cuda_stream); device_buffer move_to_index_sorted(number_of_elements, allocator, cuda_stream); - device_buffer temp_storage_vect(0, allocator); + device_buffer temp_storage_vect(0, allocator, cuda_stream); details::perform_radix_sort(temp_storage_vect, less_significant_keys.data(), diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index c3d22d305..1232e130d 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -571,12 +571,12 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, , window_size_(window_size) , number_of_reads_(0) , number_of_basepairs_in_longest_read_(0) - , representations_d_(allocator) - , read_ids_d_(allocator) - , positions_in_reads_d_(allocator) - , directions_of_reads_d_(allocator) - , unique_representations_d_(allocator) - , first_occurrence_of_representations_d_(allocator) + , representations_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) + , read_ids_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) + , positions_in_reads_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) + , directions_of_reads_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) + , unique_representations_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) + , first_occurrence_of_representations_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) , is_ready_(false) // set to true at the end of generate_index() , index_host_copy_source_(nullptr) { diff --git a/cudamapper/src/matcher_gpu.cu b/cudamapper/src/matcher_gpu.cu index 9b6d7c365..440d373cc 100644 --- a/cudamapper/src/matcher_gpu.cu +++ b/cudamapper/src/matcher_gpu.cu @@ -34,7 +34,7 @@ MatcherGPU::MatcherGPU(DefaultDeviceAllocator allocator, const Index& query_index, const Index& target_index, const cudaStream_t cuda_stream) - : anchors_d_(allocator) + : anchors_d_(0, allocator, cuda_stream) { GW_NVTX_RANGE(profile, "matcherGPU"); if (query_index.unique_representations().size() == 0 || target_index.unique_representations().size() == 0) From 89f9f2fe1549e9e78854ce19a6eded85d1ecca08 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 31 Jul 2020 16:36:02 +0200 Subject: [PATCH 016/281] [cudamapper] Formatting fix in allocator --- .../include/claraparabricks/genomeworks/utils/allocator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp index 5e227b4e8..9cedfedaa 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp @@ -306,7 +306,7 @@ using DefaultDeviceAllocator = CudaMallocAllocator; /// which cannot allocate memory. /// \param max_cached_bytes max bytes used by memory resource used by CachingDeviceAllocator (default: 2GiB, unused for CudaMallocAllocator) /// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead (unused for CudaMallocAllocator) -inline DefaultDeviceAllocator create_default_device_allocator(std::size_t max_caching_size = 2ull * 1024 * 1024 * 1024, +inline DefaultDeviceAllocator create_default_device_allocator(std::size_t max_caching_size = 2ull * 1024 * 1024 * 1024, const std::vector& default_streams = {}) { #ifdef GW_ENABLE_CACHING_ALLOCATOR From 4b4101f29ff72bacc8cb2f20007f85decd2b99d2 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 31 Jul 2020 16:41:55 +0200 Subject: [PATCH 017/281] [cudamapper] Bettwe NVTX ranges for IndexHostCopy Signed-off-by: Milos Maric --- cudamapper/src/index_host_copy.cu | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 800ff9de2..6a7552668 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -34,7 +34,7 @@ IndexHostCopy::IndexHostCopy(const Index& index, , memory_pinner_(*this) , cuda_stream_(cuda_stream) { - GW_NVTX_RANGE(profiler, "index_host_copy"); + GW_NVTX_RANGE(profiler, "index_host_copy::constructor"); // Use only one large array to store all arrays in order to reduce fragmentation when using pool allocators // Align all arrays by 64 bits @@ -53,7 +53,7 @@ IndexHostCopy::IndexHostCopy(const Index& index, first_occurrence_of_representations_bits; { - GW_NVTX_RANGE(profiler, "index_host_copy::allocate_host_memory"); + GW_NVTX_RANGE(profiler, "index_host_copy::constructor::allocate_host_memory"); underlying_array_.resize(total_bits); } @@ -118,6 +118,7 @@ void IndexHostCopy::finish_copying() const std::unique_ptr IndexHostCopy::copy_index_to_device(DefaultDeviceAllocator allocator, const cudaStream_t cuda_stream) const { + GW_NVTX_RANGE(profiler, "index_host_copy::copy_index_to_device"); // register pinned memory, memory gets unpinned in finish_copying() memory_pinner_.register_pinned_memory(); @@ -195,7 +196,7 @@ IndexHostCopy::IndexHostMemoryPinner::~IndexHostMemoryPinner() if (times_memory_pinned_ != 0) { assert(!"memory should always be unregistered by unregister_pinned_memory()"); - GW_NVTX_RANGE(profiler, "unregister_pinned_memory"); + GW_NVTX_RANGE(profiler, "index_host_memory_pinner::unregister_pinned_memory"); GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.underlying_array_.data())); } } @@ -205,7 +206,7 @@ void IndexHostCopy::IndexHostMemoryPinner::register_pinned_memory() // only pin memory if it hasn't been pinned yet if (0 == times_memory_pinned_) { - GW_NVTX_RANGE(profiler, "register_pinned_memory"); + GW_NVTX_RANGE(profiler, "index_host_memory_pinner::register_pinned_memory"); GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.underlying_array_.data(), index_host_copy_.underlying_array_.size() * sizeof(unsigned char), cudaHostRegisterDefault)); @@ -219,7 +220,7 @@ void IndexHostCopy::IndexHostMemoryPinner::unregister_pinned_memory() // only unpin memory if this is the last unpinning if (1 == times_memory_pinned_) { - GW_NVTX_RANGE(profiler, "unregister_pinned_memory"); + GW_NVTX_RANGE(profiler, "index_host_memory_pinner::unregister_pinned_memory"); GW_CU_CHECK_ERR(cudaHostUnregister(index_host_copy_.underlying_array_.data())); } --times_memory_pinned_; From f99ad3a58d586f2aa10dda032c041ab47766dad8 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Tue, 4 Aug 2020 17:34:51 +0200 Subject: [PATCH 018/281] [cudamapper] Use custom exception in IndexCache --- cudamapper/src/index_cache.cu | 30 ++++++++-- cudamapper/src/index_cache.cuh | 32 ++++++++++ cudamapper/tests/Test_CudamapperIndexCache.cu | 58 +++++++++---------- 3 files changed, 86 insertions(+), 34 deletions(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 7ec9f553f..6a13001be 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -297,9 +297,13 @@ void IndexCache::start_generating_content_device(const std::vector index_on_host = host_cache.at(index_descriptor); - device_index = index_on_host->copy_index_to_device(allocator_, cuda_stream_copy_); + const auto index_on_host_iter = host_cache.find(index_descriptor); + if (index_on_host_iter == host_cache.end()) + { + throw IndexNotFoundException(index_descriptor, + IndexNotFoundException::IndexType::host_cache); + } + device_index = index_on_host_iter->second->copy_index_to_device(allocator_, cuda_stream_copy_); } assert(device_index); @@ -325,8 +329,24 @@ std::shared_ptr IndexCache::get_index_from_cache(const IndexDescrip const CacheSelector which_cache) const { const device_cache_t& this_device_cache = (CacheSelector::query_cache == which_cache) ? query_device_cache_ : target_device_cache_; - // TODO: Throw a custom exception if index not found instead of std::out_of_range - return this_device_cache.at(index_descriptor); + const auto index_iter = this_device_cache.find(index_descriptor); + if (index_iter == this_device_cache.end()) + { + throw IndexNotFoundException(index_descriptor, + IndexNotFoundException::IndexType::device_cache); + } + return index_iter->second; +} + +IndexNotFoundException::IndexNotFoundException(IndexDescriptor index_descriptor, + IndexType index_type) + : error_message_("Index not found in " + std::string(index_type == IndexType::host_cache ? "host" : "device") + " cache. First read: " + std::to_string(index_descriptor.first_read()) + ", number of reads: " + std::to_string(index_descriptor.number_of_reads())) +{ +} + +const char* IndexNotFoundException::what() const noexcept +{ + return error_message_.c_str(); } } // namespace cudamapper diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index 3cdad0067..2172c85ff 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -16,7 +16,9 @@ #pragma once +#include #include +#include #include #include @@ -225,6 +227,36 @@ private: const cudaStream_t cuda_stream_copy_; }; +/// IndexNotFoundException - Exception to be thrown if Index is reuqsted, but not found +class IndexNotFoundException : public std::exception +{ +public: + /// IndexType - Was the Index requested from host or device cache + enum class IndexType + { + host_cache, + device_cache + }; + + /// \brief constructor + /// \param index_descriptor + /// \param index_type was Index equested from host or device cache + IndexNotFoundException(IndexDescriptor index_descriptor, + IndexType index_type); + + IndexNotFoundException(const IndexNotFoundException&) = default; + IndexNotFoundException& operator=(const IndexNotFoundException&) = default; + IndexNotFoundException(IndexNotFoundException&&) = default; + IndexNotFoundException& operator=(IndexNotFoundException&&) = default; + virtual ~IndexNotFoundException() = default; + + /// Returns the error message of the exception + virtual const char* what() const noexcept; + +private: + const std::string error_message_; +}; + } // namespace cudamapper } // namespace genomeworks diff --git a/cudamapper/tests/Test_CudamapperIndexCache.cu b/cudamapper/tests/Test_CudamapperIndexCache.cu index dd83d4f88..59a14be6a 100644 --- a/cudamapper/tests/Test_CudamapperIndexCache.cu +++ b/cudamapper/tests/Test_CudamapperIndexCache.cu @@ -368,11 +368,11 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) catcaag_maximum_kmer_size, cuda_stream_generate, "test_index_cache_same_query_and_target_1"); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); index_cache.generate_content_target_host(aagcta_index_descriptors); index_cache.start_generating_content_target_device(aagcta_index_descriptors); @@ -393,9 +393,9 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) catcaag_maximum_kmer_size, cuda_stream_generate, "test_index_cache_same_query_and_target_2"); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); auto index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, @@ -411,13 +411,13 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate, "test_index_cache_same_query_and_target_3"); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); index_cache.generate_content_query_host(aagcta_index_descriptors); index_cache.start_generating_content_query_device(aagcta_index_descriptors); index_cache.finish_generating_content_query_device(); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_index_descriptor), IndexNotFoundException); auto index_query_aagcta = index_cache.get_index_from_query_cache(aagcta_index_descriptor); check_if_index_is_correct(index_query_aagcta, aagcta_representations, @@ -433,8 +433,8 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate, "test_index_cache_same_query_and_target_4"); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, @@ -450,7 +450,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate, "test_index_cache_same_query_and_target_5"); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); index_cache.generate_content_query_host(catcaag_aagcta_separate_index_descriptors); index_cache.start_generating_content_query_device(catcaag_aagcta_separate_index_descriptors); @@ -486,8 +486,8 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate, "test_index_cache_same_query_and_target_7"); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, @@ -503,7 +503,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate, "test_index_cache_same_query_and_target_8"); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); index_cache.generate_content_target_host(catcaag_aagcta_one_index_descriptors); index_cache.start_generating_content_target_device(catcaag_aagcta_one_index_descriptors); @@ -539,9 +539,9 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate, "test_index_cache_same_query_and_target_10"); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor), IndexNotFoundException); auto catcaag_aagcta_target_aagcta = index_cache.get_index_from_target_cache(catcaag_aagcta_index_descriptor); check_if_index_is_correct(catcaag_aagcta_target_aagcta, catcaag_aagcta_representations, @@ -748,7 +748,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate, "test_index_cache_not_the_same_query_and_target_1"); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_target_cache(index_descriptor), IndexNotFoundException); index_cache.generate_content_target_host(index_descriptors); index_cache.start_generating_content_target_device(index_descriptors); @@ -1077,7 +1077,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target_2) cuda_stream_copy); index_cache.generate_content_query_host(catcaag_index_descriptors); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(catcaag_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_index_descriptor), IndexNotFoundException); index_cache.start_generating_content_query_device(catcaag_index_descriptors); index_cache.finish_generating_content_query_device(); auto index_query_catcaag = index_cache.get_index_from_query_cache(catcaag_index_descriptor); @@ -1095,13 +1095,13 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target_2) catcaag_maximum_kmer_size, cuda_stream_generate, "test_index_cache_same_query_and_target_2_1"); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor), IndexNotFoundException); index_cache.generate_content_target_host(catcaag_aagcta_index_descriptors); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor), IndexNotFoundException); index_cache.start_generating_content_target_device(catcaag_aagcta_index_descriptors); index_cache.finish_generating_content_target_device(); @@ -1273,8 +1273,8 @@ TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target_ cuda_stream_copy); index_cache.generate_content_query_host(index_descriptors); - ASSERT_ANY_THROW(index_cache.get_index_from_query_cache(index_descriptor)); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_query_cache(index_descriptor), IndexNotFoundException); + ASSERT_THROW(index_cache.get_index_from_target_cache(index_descriptor), IndexNotFoundException); index_cache.start_generating_content_query_device(index_descriptors); index_cache.finish_generating_content_query_device(); @@ -1293,7 +1293,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target_ aagcta_maximum_kmer_size, cuda_stream_generate, "test_index_cache_not_the_same_query_and_target_2_1"); - ASSERT_ANY_THROW(index_cache.get_index_from_target_cache(index_descriptor)); + ASSERT_THROW(index_cache.get_index_from_target_cache(index_descriptor), IndexNotFoundException); index_cache.generate_content_target_host(index_descriptors); index_cache.start_generating_content_target_device(index_descriptors); From ba7a09c77a3e33b3e01e29d41f59d92fcec5d0d0 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 21 Aug 2020 14:21:17 -0700 Subject: [PATCH 019/281] [cudaungappedextender] Add skeleton for API and sample usage in segalign --- .../cudaungappedextender.hpp | 103 +++++ .../samples/segalign_seedfilter.cu | 359 ++++++++++++++++++ 2 files changed, 462 insertions(+) create mode 100644 cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp create mode 100644 cudaungappedextender/samples/segalign_seedfilter.cu diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp new file mode 100644 index 000000000..96222670f --- /dev/null +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -0,0 +1,103 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#pragma once + +namespace claraparabricks +{ +namespace genomeworks +{ +namespace cudaungappedextender +{ + +struct Anchor +{ + /// position of first sketch element in query_read_id_ + position_in_read_t query_position_in_read_; + /// position of second sketch element in target_read_id_ + position_in_read_t target_position_in_read_; +}; + +struct ScoredSegment { + /// Anchor for the segment + Anchor anchor; + /// length of the segment + int32_t len; + /// score of the segment + int32_t score; +}; + +/// CUDA Ungapped Extension object +class UngappedExtender +{ + public: + /// \brief Destructor for UngappedExtender. + ~UngappedExtender() = default; + /// \brief Constructor Prototype + UngappedExtender(int gpu_id=0, + int* h_sub_mat, + int input_xdrop, + bool input_noentropy, + cudaStream_t stream=0); + + + + /// \brief Host pointer prototype for ungapped extension + /// + /// Takes values from host data structures, + /// copies them over to device, + /// launches async extension kernels on specified stream. Filters + /// segments on device based on input_hspthresh. + StatusType ungapped_extend(const char* h_query, + int32_t query_length, + const char* h_target, + int32_t target_length, + int32_t input_hspthresh, + std::vector& h_seedHits); + + /// \brief Device pointer prototype for ungapped extension + /// + /// Memcopies to device memory are assumed to be done before this + /// function. Output array d_hsp_out must be pre-allocated on device. + /// Launches async extension kernel. Filters segments on device + /// based on input_hspthresh. + StatusType ungapped_extend(const char* d_query, + int32_t query_length, + const char* d_target, + int32_t target_length, + int32_t input_hspthresh, + Anchor* d_seed_hits, + int32_t num_hits, + ScoredSegment* d_hsp_out, + int32_t* num_hsps); + + /// \brief Waits for CUDA accelerated extension to finish + /// + /// Blocking call that waits for all the extensions scheduled + /// on the GPU to come to completion. + StatusType sync_extensions(); + + + /// \brief Return the computed segments + /// + /// \return Vector of Scored Segments + const std::vector>& get_scored_segments(); + + /// \brief Reset UngappedExtender object. + void reset(); +}; +} // namespace cudaungappedextender +} // namespace genomeworks +} // namespace claraparabricks diff --git a/cudaungappedextender/samples/segalign_seedfilter.cu b/cudaungappedextender/samples/segalign_seedfilter.cu new file mode 100644 index 000000000..4f741656c --- /dev/null +++ b/cudaungappedextender/samples/segalign_seedfilter.cu @@ -0,0 +1,359 @@ +#include +#include +#include +#include +#include +#include +#include +#include "parameters.h" +#include "seed_filter.h" + +#define MAX_SEED_HITS_PER_GB 8388608 +#define MAX_UNGAPPED_PER_GB 4194304 + +using namespace claraparabricks::genomeworks; +using namespace claraparabricks::genomeworks::cudapoa; + + +// Control Variables +std::mutex mu; +std::condition_variable cv; +std::vector available_gpus; +std::vector g_cuda_ungapped_extenders; // indexed by gpu_id +std::vector g_cuda_streams; // indexed by gpu_id + + +int NUM_DEVICES; + +// Seed Variables +uint32_t MAX_SEEDS; +uint32_t MAX_SEED_HITS; + +char** d_ref_seq; +uint32_t ref_len; + +char** d_query_seq; +char** d_query_rc_seq; +uint32_t query_length[BUFFER_DEPTH]; + +uint32_t seed_size; +uint32_t** d_index_table; +uint32_t** d_pos_table; + +uint64_t** d_seed_offsets; + +uint32_t** d_hit_num_array; +std::vector > d_hit_num_vec; + +Anchor** d_hit; +std::vector > d_hit_vec; + +ScoredSegment** d_hsp; +std::vector > d_hsp_vec; + +//UngappedExtend Variables (ideally not visible to the user in the API) +uint32_t MAX_UNGAPPED; //maximum extensions per iteration in the UngappedExtension function + +int **d_sub_mat; // substitution score matrix +int xdrop; // xdrop parameter for the UngappedExtension function +int hspthresh; // score threshold for qualifying as an HSP +bool noentropy; // whether or not to adjust scores of segments as a factor of the Shannon entropy + +// convert input sequence from alphabet to integers +__global__ +void compress_string (char* dst_seq, char* src_seq, uint32_t len){ +... +} + +// convert input sequence to its reverse complement and convert from alphabet to integers +__global__ +void compress_rev_comp_string (char* dst_seq, char* src_seq, uint32_t len){ +... +} + + +///////////////////// End Ungapped Extension related functions executed on the GPU /////////////// + +__global__ +void find_num_hits (int num_seeds, const uint32_t* __restrict__ d_index_table, uint64_t* seed_offsets, uint32_t* seed_hit_num){ + ... +} + +__global__ +void find_hits (const uint32_t* __restrict__ d_index_table, const uint32_t* __restrict__ d_pos_table, uint64_t* d_seed_offsets, uint32_t seed_size, uint32_t* seed_hit_num, int num_hits, Anchor* d_hit, uint32_t start_seed_index, uint32_t start_hit_index){ + ... +} + +int InitializeProcessor (int num_gpu, bool transition, uint32_t WGA_CHUNK, uint32_t input_seed_size, int* sub_mat, int input_xdrop, int input_hspthresh, bool input_noentropy){ + + int nDevices; + + cudaError_t err = cudaGetDeviceCount(&nDevices); + if (err != cudaSuccess) { + fprintf(stderr, "Error: No GPU device found!\n"); + exit(1); + } + + if(num_gpu == -1){ + NUM_DEVICES = nDevices; + } + else{ + if(num_gpu <= nDevices){ + NUM_DEVICES = num_gpu; + } + else{ + fprintf(stderr, "Requested GPUs greater than available GPUs\n"); + exit(10); + } + } + + fprintf(stderr, "Using %d GPU(s)\n", NUM_DEVICES); + + seed_size = input_seed_size; + + if(transition) + MAX_SEEDS = 13*WGA_CHUNK; + else + MAX_SEEDS = WGA_CHUNK; + + cudaDeviceProp deviceProp; + cudaGetDeviceProperties(&deviceProp, 0); + float global_mem_gb = static_cast(deviceProp.totalGlobalMem / 1073741824.0f); + MAX_SEED_HITS = global_mem_gb*MAX_SEED_HITS_PER_GB; + + Anchor zeroHit; + zeroHit.query_position_in_read_ = 0; + zeroHit.target_position_in_read_ = 0; + + ScoredSegment zeroHsp; + zeroHsp.anchor.query_position_in_read_ = 0; + zeroHsp.anchor.target_position_in_read_ = 0; + zeroHsp.len = 0; + zeroHsp.score = 0; + + d_ref_seq = (char**) malloc(NUM_DEVICES*sizeof(char*)); + d_query_seq = (char**) malloc(BUFFER_DEPTH*NUM_DEVICES*sizeof(char*)); + d_query_rc_seq = (char**) malloc(BUFFER_DEPTH*NUM_DEVICES*sizeof(char*)); + + d_index_table = (uint32_t**) malloc(NUM_DEVICES*sizeof(uint32_t*)); + d_pos_table = (uint32_t**) malloc(NUM_DEVICES*sizeof(uint32_t*)); + + d_seed_offsets = (uint64_t**) malloc(NUM_DEVICES*sizeof(uint64_t*)); + + d_hit_num_array = (uint32_t**) malloc(NUM_DEVICES*sizeof(int32_t*)); + d_hit_num_vec.reserve(NUM_DEVICES); + + d_hit = (seedHit**) malloc(NUM_DEVICES*sizeof(Anchor*)); + d_hit_vec.reserve(NUM_DEVICES); + + d_hsp = (segment**) malloc(NUM_DEVICES*sizeof(ScoredSegment*)); + d_hsp_vec.reserve(NUM_DEVICES); + + for(int g = 0; g < NUM_DEVICES; g++){ + + check_cuda_setDevice(g, "InitializeProcessor"); + + check_cuda_malloc((void**)&d_seed_offsets[g], MAX_SEEDS*sizeof(uint64_t), "seed_offsets"); + + d_hit_num_vec.emplace_back(MAX_SEEDS, 0); + d_hit_num_array[g] = thrust::raw_pointer_cast(d_hit_num_vec.at(g).data()); + + d_hit_vec.emplace_back(MAX_SEED_HITS, zeroHit); + d_hit[g] = thrust::raw_pointer_cast(d_hit_vec.at(g).data()); + + d_hsp_vec.emplace_back(MAX_SEED_HITS, zeroHsp); + d_hsp[g] = thrust::raw_pointer_cast(d_hsp_vec.at(g).data()); + cudaStream_t stream; + cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking); + g_cuda_streams.push_back(stream); + g_cuda_ungapped_extenders.emplace_back(g, sub_mat, input_xdrop, input_noentropy, stream); + available_gpus.push_back(g); + } + return NUM_DEVICES; +} + +void InclusivePrefixScan (uint32_t* data, uint32_t len) { +... +} + +void SendSeedPosTable (uint32_t* index_table, uint32_t index_table_size, uint32_t* pos_table, uint32_t num_index, uint32_t max_pos_index){ +... +} + +void SendRefWriteRequest (size_t start_addr, uint32_t len){ +... +} + +void SendQueryWriteRequest (size_t start_addr, uint32_t len, uint32_t buffer){ +... +} + +std::vector SeedAndFilter (std::vector seed_offset_vector, bool rev, uint32_t buffer, int input_hspthresh){ + + uint32_t num_hits = 0; + uint32_t total_anchors = 0; + + uint32_t num_seeds = seed_offset_vector.size(); + + uint64_t* tmp_offset = (uint64_t*) malloc(num_seeds*sizeof(uint64_t)); + for (uint32_t i = 0; i < num_seeds; i++) { + tmp_offset[i] = seed_offset_vector[i]; + } + + int g; + std::unique_lock locker(mu); + if (available_gpus.empty()) { + cv.wait(locker, [](){return !available_gpus.empty();}); + } + g = available_gpus.back(); + available_gpus.pop_back(); + locker.unlock(); + + check_cuda_setDevice(g, "SeedAndFilter"); + + check_cuda_memcpy((void*)d_seed_offsets[g], (void*)tmp_offset, num_seeds*sizeof(uint64_t), cudaMemcpyHostToDevice, "seed_offsets"); + + find_num_hits <<>> (num_seeds, d_index_table[g], d_seed_offsets[g], d_hit_num_array[g]); + + thrust::inclusive_scan(d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, d_hit_num_vec[g].begin()); + + check_cuda_memcpy((void*)&num_hits, (void*)(d_hit_num_array[g]+num_seeds-1), sizeof(uint32_t), cudaMemcpyDeviceToHost, "num_hits"); + + int num_iter = num_hits/MAX_UNGAPPED+1; + uint32_t iter_hit_limit = MAX_UNGAPPED; + thrust::device_vector limit_pos (num_iter); + + for(int i = 0; i < num_iter-1; i++){ + thrust::device_vector::iterator result_end = thrust::lower_bound(d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin()+num_seeds, iter_hit_limit); + uint32_t pos = thrust::distance(d_hit_num_vec[g].begin(), result_end)-1; + iter_hit_limit = d_hit_num_vec[g][pos]+MAX_UNGAPPED; + limit_pos[i] = pos; + } + + limit_pos[num_iter-1] = num_seeds-1; + + segment** h_hsp = (segment**) malloc(num_iter*sizeof(segment*)); + uint32_t* num_anchors = (uint32_t*) calloc(num_iter, sizeof(uint32_t)); + + uint32_t start_seed_index = 0; + uint32_t start_hit_val = 0; + uint32_t iter_num_seeds, iter_num_hits; + int32_t* d_num_anchors; + GW_CU_CHECK_ERR(cudaMalloc(&d_num_anchors, sizeof(int32_t))); + + if(num_hits > 0){ + + for(int i = 0; i < num_iter; i++){ + iter_num_seeds = limit_pos[i] + 1 - start_seed_index; + iter_num_hits = d_hit_num_vec[g][limit_pos[i]] - start_hit_val; + + find_hits <<>> (d_index_table[g], d_pos_table[g], d_seed_offsets[g], seed_size, d_hit_num_array[g], iter_num_hits, d_hit[g], start_seed_index, start_hit_val); + + if(rev){ + if(!g_cuda_ungapped_extenders[g].ungapped_extend(d_query_rc_seq[buffer*NUM_DEVICES+g], query_length[buffer], d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, d_hsp[g], d_num_anchors)); + { + //err... + } + } + else{ + if(!g_cuda_ungapped_extenders[g].ungapped_extend(d_query_seq[buffer*NUM_DEVICES+g], query_length[buffer], d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, d_hsp[g], d_num_anchors)); + { + //err... + } + } + GW_CU_CHECK_ERR(cudaMemcpyAsync(&num_anchors[i], d_num_anchors, sizeof(int32_t), cudaMemcpyDeviceToHost, g_cuda_streams[g])); + cudaStreamSynchronize(g_cuda_streams[g]); + + + total_anchors += num_anchors[i]; + + if(num_anchors[i] > 0){ + h_hsp[i] = (segment*) calloc(num_anchors[i], sizeof(segment)); + + check_cuda_memcpy((void*)h_hsp[i], (void*)d_hsp[g], num_anchors[i]*sizeof(segment), cudaMemcpyDeviceToHost, "hsp_output"); + } + + start_seed_index = limit_pos[i] + 1; + start_hit_val = d_hit_num_vec[g][limit_pos[i]]; + } + } + + limit_pos.clear(); + + { + std::unique_lock locker(mu); + available_gpus.push_back(g); + locker.unlock(); + cv.notify_one(); + } + + std::vector gpu_filter_output; + + segment first_el; + first_el.len = total_anchors; + first_el.score = num_hits; + gpu_filter_output.push_back(first_el); + + if(total_anchors > 0){ + for(int it = 0; it < num_iter; it++){ + + for(int i = 0; i < num_anchors[it]; i++){ + gpu_filter_output.push_back(h_hsp[it][i]); + } + } + free(h_hsp); + } + + free(tmp_offset); + return gpu_filter_output; +} + +void clearRef(){ +... +} + +void clearQuery(uint32_t buffer){ +... +} + +void ShutdownProcessor(){ + + d_hit_num_vec.clear(); + d_hit_vec.clear(); + d_hsp_vec.clear(); + + g_cuda_ungapped_extenders.clear(); + for(auto & cudaStream: g_cuda_streams) + { + cudaStreamDestry(cudaStream); + } + g_cuda_streams.clear(); + cudaDeviceReset(); +} + +///// Start Ungapped Extension related functions ///// + +void CompressSeq(char* input_seq, char* output_seq, uint32_t len){ +... +} + +void CompressRevCompSeq(char* input_seq, char* output_seq, uint32_t len){ +... +} + + + +CompressSeq_ptr g_CompressSeq = CompressSeq; +CompressRevCompSeq_ptr g_CompressRevCompSeq = CompressRevCompSeq; + +///// End Ungapped Extension related functions ///// + +InitializeProcessor_ptr g_InitializeProcessor = InitializeProcessor; +InclusivePrefixScan_ptr g_InclusivePrefixScan = InclusivePrefixScan; +SendSeedPosTable_ptr g_SendSeedPosTable = SendSeedPosTable; +SendRefWriteRequest_ptr g_SendRefWriteRequest = SendRefWriteRequest; +SendQueryWriteRequest_ptr g_SendQueryWriteRequest = SendQueryWriteRequest; +SeedAndFilter_ptr g_SeedAndFilter = SeedAndFilter; +clearRef_ptr g_clearRef = clearRef; +clearQuery_ptr g_clearQuery = clearQuery; +ShutdownProcessor_ptr g_ShutdownProcessor = ShutdownProcessor; From 950e30537e618b8a915ec7fc358adda1608f92cd Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 21 Aug 2020 14:40:44 -0700 Subject: [PATCH 020/281] [cudaungappedextender] fix copy-pasta --- cudaungappedextender/samples/segalign_seedfilter.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaungappedextender/samples/segalign_seedfilter.cu b/cudaungappedextender/samples/segalign_seedfilter.cu index 4f741656c..2b522b983 100644 --- a/cudaungappedextender/samples/segalign_seedfilter.cu +++ b/cudaungappedextender/samples/segalign_seedfilter.cu @@ -12,7 +12,7 @@ #define MAX_UNGAPPED_PER_GB 4194304 using namespace claraparabricks::genomeworks; -using namespace claraparabricks::genomeworks::cudapoa; +using namespace claraparabricks::genomeworks::cudaungappedextender; // Control Variables From a8fd9e412985eb7117f1680f6beef17210a1d1ac Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 21 Aug 2020 18:05:07 -0700 Subject: [PATCH 021/281] Add basic samples --- .../cudaungappedextender.hpp | 6 +- cudaungappedextender/samples/device_sample.cu | 127 ++++++++++++++++++ cudaungappedextender/samples/host_sample.cu | 95 +++++++++++++ .../samples/segalign_seedfilter.cu | 6 +- 4 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 cudaungappedextender/samples/device_sample.cu create mode 100644 cudaungappedextender/samples/host_sample.cu diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index 96222670f..34cb44f49 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -46,10 +46,10 @@ class UngappedExtender /// \brief Destructor for UngappedExtender. ~UngappedExtender() = default; /// \brief Constructor Prototype - UngappedExtender(int gpu_id=0, - int* h_sub_mat, + UngappedExtender(int* h_sub_mat, int input_xdrop, bool input_noentropy, + int gpu_id=0, cudaStream_t stream=0); @@ -81,7 +81,7 @@ class UngappedExtender Anchor* d_seed_hits, int32_t num_hits, ScoredSegment* d_hsp_out, - int32_t* num_hsps); + int32_t* d_num_hsps); /// \brief Waits for CUDA accelerated extension to finish /// diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu new file mode 100644 index 000000000..cf31b2ff0 --- /dev/null +++ b/cudaungappedextender/samples/device_sample.cu @@ -0,0 +1,127 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#include +#include +#include +#include + +#include +#include +#include + + +namespace claraparabricks +{ +namespace genomeworks +{ +namespace cudaungappedextender +{ + int main(int argc, char* argv[]) + { + const int32_t input_xdrop = 10; + const int32_t input_no_entropy = 0; + const int32_t hsp_threshold = 20000; + // Fasta query and target files + std::string target_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); + // Assumes that only one sequence is present per file + std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); + + magic_sequence query_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); + // Assumes that only one sequence is present per file + magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); + + // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ + std::string anchors_file_path = "../data/anchors.csv"; + + std::vector h_hits; + // Following function loops through all hits in the anchors.csv and returns results in + // the passed array + parse_anchors(anchors_file_path, h_hits); + + // Following sections TBD based on encoding + ScoreMatrix = magic_number_matrix; + std::string h_encoded_target = magic_encode(magic_base, target_sequence); + std::string h_encoded_query = magic_encode(magic_base, query_sequence); + + // Create a stream for async use + cudaStream_t stream0; + cudaStreamCreateWithFlags(&stream0, cudaStreamNonBlocking); + // Allocate space on device for target and query sequences, hits, + // high scoring segment pairs (hsps) and num_hsps. + char* d_query; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_query, sizeof(char)*h_encoded_query.size())); + char* d_target; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_target, sizeof(char)*h_target_query.size())); + Anchor* d_hits; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_hits, sizeof(Anchor)*h_hits.size())); + // Allocate a minimum of num_hits as all hits could be hsps in the worst case + int32_t h_num_hsps = 0; + ScoredSegment* d_hsps; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_hsps, sizeof(ScoredSegment)*h_hits.size())); + int32_t* d_num_hsps; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_num_hsps, sizeof(int32_t)); + + // Async Memcopy all the input values to device + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query.c_str(), sizeof(char)*h_encoded_query.size(), cudaMemcpyHostToDevice, stream0)); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target.c_str(), sizeof(char)*h_encoded_target.size(), cudaMemcpyHostToDevice, stream0)); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_hits, &h_hits[0], sizeof(Anchor)*h_hits.size(), cudaMemcpyHostToDevice, stream0)); + + // Create an ungapped extender object + std::unique_ptr ungapped_extender = std::make_unique(0, + magic_number_matrix, + input_xdrop, + input_no_entropy, + stream0); + // Launch the ungapped extender device function + ungapped_extender->ungapped_extend(d_query, // Type TBD based on encoding + encoded_query.size(), + d_target.c_str(), + encoded_target.size(), + hsp_threshold, + d_hits, + h_hits.size(), + d_hsps, + d_num_hsps); + // Copy back the number of hsps to host + GW_CU_CHECK_ERR(cudaMemcpyAsync(&h_num_hsps, d_num_hsps, sizeof(int32_t), cudaMemcpyDeviceToHost, stream0)); + + // Wait for ungapped extender to finish + GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0)); + + //Get results + if(h_num_hsps > 0) + { + std::vector h_hsps(h_num_hsps); + // Don't care about asynchronous copies here + GW_CU_CHECK_ERR(cudaMemcpy(&h_hsps[0], d_hsps, sizeof(ScoredSegment)*h_num_hsps, cudaMemcpyDeviceToHost)); + + int32_t i = 0; + for (const auto& segment : h_hsps) + { + std::cout << "Segment: " << i << "Length: " << segment.len << "Score: " << segment.score << std::endl; + std::cout << "Position in query: " << segment.anchor.query_position_in_read_< +#include +#include +#include +#include +#include + + +namespace claraparabricks +{ +namespace genomeworks +{ +namespace cudaungappedextender +{ + int main(int argc, char* argv[]) + { + const int32_t input_xdrop = 10; + const int32_t input_no_entropy = 0; + const int32_t hsp_threshold = 20000; + // Fasta query and target files + std::string target_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); + // Assumes that only one sequence is present per file + std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); + + magic_sequence query_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); + // Assumes that only one sequence is present per file + magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); + + // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ + std::string anchors_file_path = "../data/anchors.csv"; + + std::vector h_hits; + // Following function loops through all hits in the anchors.csv and returns results in + // the passed array + parse_anchors(anchors_file_path, h_hits); + + // Following sections TBD based on encoding + ScoreMatrix = magic_number_matrix; + std::string encoded_target = magic_encode(magic_base, target_sequence); + std::string encoded_query = magic_encode(magic_base, query_sequence); + + // Create a stream for async use + cudaStream_t stream0; + cudaStreamCreateWithFlags(&stream0, cudaStreamNonBlocking); + // Create an ungapped extender object + std::unique_ptr ungapped_extender = std::make_unique(0, + magic_number_matrix, + input_xdrop, + input_no_entropy, + stream0); + // Launch the ungapped extender host function + ungapped_extender->ungapped_extend(encoded_query.c_str(), // Type TBD based on encoding + encoded_query.size(), + encoded_target.c_str(), + encoded_target.size(), + hsp_threshold, + h_hits); + + // Wait for ungapped extender to finish + ungapped_extender->sync_extensions(); + + //Get results + const std::vector>& segments = ungapped_extender->get_scored_segments(); + int32_t i = 0; + for (const auto& segment : segments) + { + std::cout << "Segment: " << i << "Length: " << segment->len << "Score: " << segment->score << std::endl; + std::cout << "Position in query: " << segment->anchor.query_position_in_read_<anchor.target_position_in_read_< SeedAndFilter (std::vector seed_offset_vector, bo uint32_t start_hit_val = 0; uint32_t iter_num_seeds, iter_num_hits; int32_t* d_num_anchors; - GW_CU_CHECK_ERR(cudaMalloc(&d_num_anchors, sizeof(int32_t))); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_anchors, sizeof(int32_t))); if(num_hits > 0){ @@ -323,9 +323,9 @@ void ShutdownProcessor(){ d_hsp_vec.clear(); g_cuda_ungapped_extenders.clear(); - for(auto & cudaStream: g_cuda_streams) + for(auto& cudaStream: g_cuda_streams) { - cudaStreamDestry(cudaStream); + cudaStreamDestroy(cudaStream); } g_cuda_streams.clear(); cudaDeviceReset(); From 75ae4baed668548df9020d2131357d523a6cd98b Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 21 Aug 2020 18:08:20 -0700 Subject: [PATCH 022/281] [cudaungappedextender] Add sample data --- cudaungappedextender/data/example.fa | 5001 + cudaungappedextender/data/example_hits.csv | 143670 +++++++++++++++ cudaungappedextender/samples/device_sample.cu | 2 +- cudaungappedextender/samples/host_sample.cu | 2 +- 4 files changed, 148673 insertions(+), 2 deletions(-) create mode 100644 cudaungappedextender/data/example.fa create mode 100644 cudaungappedextender/data/example_hits.csv diff --git a/cudaungappedextender/data/example.fa b/cudaungappedextender/data/example.fa new file mode 100644 index 000000000..b44bb7128 --- /dev/null +++ b/cudaungappedextender/data/example.fa @@ -0,0 +1,5001 @@ +>chr1 +CACAGACTTTTCTGGATATGAACATACAGTAGAAAAAGAATTGAACAACT +ATCCCATCACCCTAGCTGAATGTAGAGAAATGATTAATAGTAAGAAATGT +GGATACGGAACTTTATCAAAAGGTAGTTCTAACGTATGGAGTACAAATAA +TAAAATCGAAGTTGATTACCCAAATAGATTTACAAGCTTCTTCTCACCTA +AAAGATTCCACAAAGAAAATTGTATAGTTATTGAGACGAAAGTTTATTCC +CATTTTAATCAATCTAAGCCAACCAATATTTTAGCTGATATGAACAATTG +CAAGTATCAAGACGGTTTCTGCGagatcaagaaaaatgaaattataacAT +GGGATGTCAACAGAGATCAGAAATGTCAATTTATCTCTATTGGAATACTT +GATGGGATGTACAATAACAAACTCTGGGTTAACAATAAGAACCAAATTGc +gttaaattttgaaacacagAAGATTGTGAATGATTGTAACATAGACCTAA +TGATTAGTGATGAGGGTTTTGCTGTaaaaagaattgaaagatCTCAATAT +TCTCCAAGACACATTCCTATTCCAATCCCATCATATTCTCAACAAGATAT +TCAACGTCAAAGACAACAACAGGAGGACGACAGAAGAAAACGTGAACAAG +AGGAACAACAGAGAAAGCGGGAGCAAGAAGATAGTAGGAGACGGGATCaa +gaagatatcagaaaacgGGATCAAGAAGATGCTAGAAGACGTGATCaaga +aagacaaaaacagaatgaagctgattttgaaaagaggaaacaagaaaagc +tgaagaaggaagaagaacttgagaaaaagaaacaggagaaaactaagaaa +gctcaggaagaggctgaaagaaaagaaatcgaagCATTGAGGAAGCTACG +GAAAGATGAAGACTGGACAGTTCATGATGAGAAATCGAAAGGTGTAGACA +AAAAATCCAACAGAACTAAAAGAGAAATCTTTCAACCATTACCATTACca +aattatgatgaatatttaAAGATCAAACAAGACTGTGATAACTTACCTAG +TTATGAAGCATTCAAGAATAGTAGAGAAGGATGTGATTCACTTACAACTT +ATACAGATTTTATAAACannnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnGTCTTATAAGAGAA +AATCCAAGAAGATACAtccaaaaacaattgaatcaTTCAGCCATCAAAGT +GAGATTCATAGATTCTCAGGATCCAATAGTACAAGTTACATTTTGTAAAG +AAATTGATGAAGAAGAGATTGACTTTATAGACGTTACTCATTTTGATGGA +TACAAGTTAAGTATTGCTCTCACACTCGGACAAATACCAATCAAGATAAA +AAGTCTAGGAGATAAAATTTGGTATTACAAGAATGATAATTTTGGAGGAG +TGATATCAAGTAAACCTCAGCTTCATCCTATGATCGATTATAGAACTAAC +ATTACTAatatgaaaagtttcaaagagtTTGATTTAAAATTCTTAGATGA +CAAAGTAGTATTCCATGAGCACATTCTATTAGATTTGAAGACAAATATTG +AAGAAGAACTCATTgaagaaatgagagaaaagtCTGATGTTTCAATTGAA +GCTCATGCTAGTAATCAAGATTCTACTTTACCTCATGAAATATTAGACGA +TGTAGAAGGATTCTTTGGAAAGTGGTGGCTTAGAATTTGGAGAGTTGGTG +TTACTTTGCCAGCAATGTTCGTATACCTATTTATAGCTAGATCATTTTGG +ATGATACtttctccaaaaacttttattgatagaaaaaatagaaaaaggaa +aagggagTATGAGGCTGTAGAACTAAGAGAAATGCCGTCTATTCAATCTC +TTACTGTTGATTAATTTTTAACTAAATAAATGAGATTCTTTAATGCTGGA +AATGCTTCTAGTTATTTAATTATGGGTTCAGAAGAAAAACCTAAAGTAGA +ACTTGTAGAAAATTGTAGAAAGCCTGATATGAACCTATAGATAGTCTTAA +TAAGTCAATTAGAGAGTTGGTATTTGATGGTATGAAAGAAATAGAACTTC +CTGATGTTGAAGATGTAGTTGGGGTCTTAGCTACAAAATATTTAGACTTT +ATGAACGCCCATTTATCACTGGAATCTGCATTATATCTTTTGGATTCTTT +ATCAAAACCTTGGTACAGAGCATTGGAgcataataatattttttggtaat +ttgaaGAATCTTCAGTTGTAACCTTGTTTAGATCAGGAGACTTTTTGAGT +AGTAATTCCATCAGACCCTCAGTTGCTGTGTATTCTTCATTATTAACTTT +GATAATATTCTGATCGAATGTAATGTCTTTGTTACCAATAGCATACTTCT +TTTCTGCTTTGTTGAACTTGATTGAGTATCCAGCATTACTTCTATCTGAA +TTATCTTGAAGATAACTTACAATTAGATTACTTACAACTTGAGGTGGAGA +AGAACTAGGTAGAGAGGGAGGTAGGATTACTGAGGGTTGCTTAATGAGAC +TTTTACGATTTGTCTTTGGTGTCGnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnGAAGGAGATTTAAAGTTGTTGAGAGTTTCTACTATTTTATTCTT +TTCATCAATTATTTCTTCTTGTAGTTTTTTGTGGATTCTAAAATAGGTTT +AAATACTTCAATTCTATAATCTTCTACATGTTCTAAATCTGATCGCTCTT +GtaaattcttgattttatgctctttccgatttttcaaaaagtcttgAACA +ATGAAATTTCGCTTTTTAGGGTCATATATCTCAACTACAGAATGCATTTA +TTACAGGAAATATCATCGATTTTAGGTTGATTtattcaatatttctgaaa +atttattatagGTTTTAAATGATACTTTTCGTGCTTTATAAATGGAACAA +GttgattacaaaaaattggataggGATCAACTACGAAAACTTGTAAGAAA +CAGAACTAACACAAGTAGAGATGATGTTAGAAATATGAGCGTGCCTGAAA +TGCGTAAAATACTCAAACAGCTGGATATAGATGAACTTAGCGAAAAGTCA +AATCTTCAAATGATGAATCTATTTCCCTTCGATGAAACTGTCTTTAGTGA +TTACAAAGAAGTTGGTGATACGTTGAGAGATACTTTTCACagtggaaatt +ctaaaacattAGAATCCTTTTTAGATAAGAAAGGGCCTgatattgtgaaa +aaaatttatgaaaatagtGTGAAAATATATGTATCCTTAGAATGTGAAAT +GATAAAAGAAGGTACAGAAGATACCCAAGTAGCTCATTTGACTACAAAAA +GTGTATTATTGAATCCAGGTGATAGTAGAATTGActtcatgaaaaatatt +aaatctGATTTAGTAAATAGATTAGAACATTATCAAAGCAGAGGATCAGG +ATTTAGATTGAATAGAATTATTGGCTTGAGAATGTCTCAAACTAAGATTA +TGCCATTATCAGGATCAAAATATATTGAATTACCAGATtggataaaaaat +aaaaaagcagTAGTCAATGTTCAAAAcaaagatgaaaaatgttttatgtg +GTGTATTCTAGCACACTTATATCCAGTTGAAGAACATCCTGAACGAATAA +GCAAATATAAAGAGCATGCTTCTAAAATTAATTTCGAAGGTTTCGAATTT +CCTTTCCAAGTTAAAGATGTAGATAAGTTTGAGAAGAGAAATAACTTGGC +TGTTAATATTGTAACTCATGATGTCTCAGGACCAGCAATAATTGAGTCAT +ATAAAGTATCTAAAAACATCAACGTAGATTTAAGTAGAGTGATAAACCTT +TTGTTAGTAACTGATGAATCTGGAGAAGGTCATTACTGTTtgattaaaaa +tattgatagaTTGATGAACACCCAAAATAGTAGAAGCAATAAATTTTGTA +TTAGATGTCATAATAACTTTTACAATGcagagaagtttgaaaatcattta +ggTGACTGTATGAGTAATGCTCCAATGCAAATGATTAAACCTTCCAATGA +TtatattgaattttctaatattcaGAAAACACAGAAACATAGATTTGTAT +GTTACGCCGATTTTGAATCTGTTATCTATAAAATTGATTCTGTTAACAAT +TCTCCTAACAAATCTTGGAGTGAGAATGTTGGTAACATAAGGCCTCTGCT +TCTGTGTGATGTTGGTTGATTCTTTCACTCAGAGCATTTACGAAATGAAG +AGCTATGTTGGATATAATACTATTTTAAAAGTCAATGAGTATATCTTAGA +TGTATGTGCAAGACTATTGAATAAAGgtgatgagaaaatgaaaaatttaa +ccAATGAAGAGTTAGAATGTTATAATAACTGTAATTCATGTCCTCAGTGT +GAGAATGTGTTTGATGGGGATAAAGTTAGAGATCACGATCATTGGACTGG +ATTATACAGAGGGCCTCTATGTAATGCATGTAACCTATTGAAATGTAGAA +ATAATTTTATACCAGTATTTTTCCATAATCTTAAAGGCTATGACTCACAT +CATATAATTTGTGATGAGGAATCAAGCAAactattgaagaaaaaaggtgT +TGAAATTAAAAGCATCTCAGCAAATATAGAAAAGTTTATTAGTTTCAGTT +ATATATACAAAAATCCTAAACGATGTGAAATAAGATTTTTAGATAGTTTT +GGTTTTATGCCTTCCAGTCTGGACTATCTCAGCAGTAACCTACAAGATGA +AGAATGTGTGATTACTAATAAGTATCACAAGAATGAAAAGATGTTTAATC +TATTAAGACGCAAAGGTGTGATCCATATATTTTATGGATTCCTTCAAAAA +GTATTCTAACGAAGAACTTCCTCCAATTGAAAACTTCTACAATGCTCTAT +CTGGTGAAAATATTAGTGAAGAGAGTTATAAATATGCTCAGAAAGTATGG +AAGGAAACAAATTGTAAAACTTTAGAAGACTACACTAGAATTTATATGAT +TAATGATGTGCTACTATTAGctgatgtttttgaaactttcagaaatgtgt +ctcTCAAAGAATATAAGTTAGATCCATGTTGGTACTATACATCTCCCGGA +TTAGCTTGGGATGCTATGTTAAAGAAAACTGGAGTTAAGCTACAGACAAT +CAAGGATATTGAAATGtataattttattgaaaaaggtATTCGAGGAGGTA +TGTGTAATACCATGCTAAGATATTCGAAAGCTAATAACAAGTACATGTCT +AATTATAATCCCGAAGAGGAATCAAAATACTTATTATATCTAGATGCAAA +TAACTTATATGGATGGGCAATGAGTCAGAAATTACCATATGATGAATTCG +AATTCGTTGAAGACTTTACCTTAGAAATGATAGATGATCttgtttcaaaa +gaaaaaggttGCATACTAGAAGTAGATTTAGATTATCCAGAAGAATTACA +TGATAAACATAATGATTTACCTTTCTGTCCAGAGAATAAACGAGTTGGAT +CTACCAATAAACTAATAAATGATTTTAGTCCAAAGAGAAATTATGTAATA +CATTATAAAGCCCTTTTGCAAGTTTTAGATCATGgattaattttgaagaa +aattcatAGAGTTATTACTTTTAAAGAAAGCAATTGGTTATCTTCTTACA +TAGAACTGAATACtaacttgagaaaaaatgccaagaatgattttgagaaa +gacTTTTTCAAGCTCATGAACAATTCTGTCTTTGGTAAAACTATGGAGAA +CGTTAGAAATAGAGTGGATGTCAAAGTGGTGTCAAATATGGATAAAGTTA +TGAAACTAGCTGCTAGCAATAATTTTAAACAGAGACATATTATAAATGAG +AATATGATTTTGTAGAGATGACTCAGAAGAATATTAAATTAGATAAGCCA +ATTTATGTGGGGATGTCTATTTTAGATTTGAGCAAGTATCTAATGTATGA +GTTCCACTATGATGTGATGCTTCCAAAATATTCTTCGAATTTGAGACTGT +GCTATCAAGATACTGATTCTTATATTTATGAAATTAAAACAGACGATGTT +TATGAAGATATGTTAGCTATGAAAGaacactttgatttttcggattatCC +AAAAGAAAATAAACTTCATAGCattgagaacaaaaaagttattggtAAAT +TTAAAGATGAGTTGAATGGTAAAATAATGACTGAGATTGTAGCCTTTAGA +CCAAAACAATATGCTTTTAAAACTGATGAGGGAATGGAGTCTAAAAGAAA +TAAAGGTGTGAAGAAATCTGTTGTTAAAAAGGAAATGACTTTTAATGACT +ATAAGAACTGCTTATTAAATCATACTATTGAAAGAAGACAACAAATGTTA +ATTAACTCTAGCAGACATAATCTCCACAGTGTAAAACAAAATAAAGTTGT +GTTAAATAATCTAGTTGGTAAAGATAAAGAATCAAAAAGGTATACAGTAG +ATAATATAGACACTCTAGCTTTCGAGCATTATAGAATTCCTAAATCAGAG +TTAGAATAGAATTTTTATAATCatgtttttctaataattttaaTAAGAGT +AGACAGTAGTATCCACAATGATgagtatcaaatttttgtatttgaaacGT +TGAAGTAATTATTTCGTTTGAAGTATTATGTTTAAgatattttattattt +cttgAGGTGGGTCCAAACCAAAAGAgtcaaaataataatattcaCTTTTA +TTTACTTTATAACAGACCCAATGAGTACCTTTATTATCTGAACTATCAAG +ATTAATGATGCCTACTTCTACTTCTTTACGTTTTTCTGGTAAAGTATCCT +TCATAAATACACCTCTGAAATCTTTGATTTGTAACTCTTTTGCTATTTCT +AATAAATCTACATTTGTTAATGGTTTCAAATCTTTATTCAGCATTTAATA +AAACGTAATAATGGTaaagaattgaatgaaaagaGTGTTGGTTaacactt +gatttttttgtcgTTTCTTCATGGTTCTGAcggttgatttttttcatgtt +tgctcgacgagaatttttggctgatttttctgattttttcagtttttttc +atgattttttttcgttttttttcgattttttttcaatttttcttgcatat +ttttCGGATCTCATTAAGTGGATAATCCTGAGCTTTTGAGTGGGGTTCGA +ACTTTCTACCATTCAAACACAAGACAGACGCTTTACCATTCTGACATGAG +AACAAATTGATAAGAGAGTTGAAAACGCATTTAAGTGTAGAgaatttctt +atcagttgGGGTGAAAAGTGACTGTTTTGAGAGgttgaaaaaagtgggac +aAAAGTGGGAAAGTGAGTGGGGGTGAACTCAGTTCACCAAAATCTGAGAG +TGTGAAAATCGTGGATTTTGCTAAGAAtattatatgaaaaaaataattag +ggcaaaaaacttgaaaaaacagcaaaataaaaatttgaaaaaagtgggac +aaaaGCTGTCCCAGAAAGGACAATTCCCTACTACTCAAACCTCATTTGAC +AACTCCTGATTGAAGTACAAACGAAGCTGAGCGATGGTGTCGCTGAATGA +GGTTGGTGGAAAAGATGGAACCGATTCGCCTTGAGCGACGGTGCCGATTG +GATCAACGGGCAAcatctgaaagaaatttttgaatctcatcTAATCTGAC +TTCAAGTCAGATATGCAAATCTCATTTGACAACTCCTGATTGAAGTACAA +ACGAAGCTGAGCGATGGTGTCGCTGAATGAGGTTGGTGGAAAAGATGGAA +CCGATTCGCCTTGAGCGACTGTGCCGATTGGATCAACGGGCAGcatctga +aagaaatttttgaatctcatcTAATCTGACTTCAAGTCAGATATGCAAAC +CTCTTCTGCTATCTCCTCAGTTATGGACGACCGAGATGCCTCTTGCTCAT +TCGGTAGGGTCAATTGAATCGATTCGTCTGGATCGATTGGACTGATTGAA +ACTCccgacattttctgaaaaaaaaattggtgattagACAATGAAATGTG +ATGGGGATGGTGAGATCCAAATCTGACTTCAGGTCAGATCCAAATCTGAC +TTCAAgtcagattctgaagaattgACTTGAAGCCAATTCATTGAAATCTG +ACTTCAAGTCAAACCTCTTAATATATACAAACCTCATCTGACATGTCAGT +CTTCGATGAATGCGAGATTTCTTGAGCGGACTGGTCGATGGAATCATCCG +ACAACTTTTCTTCATCGAAGGGGGTGATCGATGTGATCGTTTCCTCGTGA +TGGGAGGTCTCTTGAGAGAGTTGAACGGTGGGAGTATCCGccatttgctg +aaaaaaaatttttgattagaCAATGAAATGGGACTGCATGAAACGTAAAG +AAATAGGATGTGAAGAGGATAGCGTGGAGGAGAATCTGACTTCAAGTCAG +ATTTTGAAGATATGAATGGTGTGAAGGTGAATGAATGGTGCggaggagca +aaaaaaaagaaacgaccTCCTCCCACCAAGACAAATGGCACGAATGACTG +GAAGGTAGCAGAAATGAcgagactccgcccctttacCGTATGatgggtct +caccacgaaaatGCGTATctaacttttaaaatattaCATGTTTCGAATTG +AAAGGCATCCAAATCTGACTTCAAGTCAGATTTCCGATATTTGACTTGAA +GccaattcttttaaaaaatttgaattcaagttAGTTAGGCACAAGttatt +actctcgtccgaaatcaatgcacgttgcggccgaaaatttactcaaaatc +gcagaTTTCTCAGCCGCCACGTGCATTGActtcggacaagagtaatagtT +AGGGCAAGTGAAATAAGTGACCTCTTGTTCCGCTGTTTCTTGATGATTTC +TGGCGGAAAGACGATGCAATGTGAGTAGCGCTTCCCGTTCTTCTACTTTG +GTAATGTCCACAACTTCCAGCTCCGTTTCTTCGTTGAAGAAGGTCACGTG +ATAACAGGAGAGCAGCTTCACATCCGACTTCCACAACACTATTTCATCGG +CGTTTTTGAGTCTTCTCTCAGcaatcaacattttctttcCGTTCAGTACA +ATCTGAAAACATCACAATTAAATACCAATTCGGGGATCCTCTAGannnnn +nnnnnnnnnnnnnnnnnnnnnnaaTGACTGAAGCAGAGCAGAAATGGCAA +GTCTCCGCCCCTTTACCGTATGatgggtctcaccacgaaaacGCGTATCT +AACTAATAACTGACGTGGCGCCTAAAATCAGattcaaactacaaattttg +agttgagctgaaatttcgagttgAACTGAAATATCGAGTTGACTTGTTAT +ATTCATTGAAAAGCAATCATAGCCGTTGCTAGCGCCTCTTGAAATCTGAC +TTCAAGTCAgaattgaaaacttaaaatcatATCCTCATCTGGCAAATCAT +CAGTCGTCGACGAACGcgagttctttttttcataaacaGGTTGAGAACAG +AACacaagaattaaaaaatataatttattgaaatggGGTTAGTAAAGTTC +TACAATtaagttgaaaaacaaaaacaaaaacagttccatacagaaaagtt +gggttagaaaaaacgataaagaagaaactgaaattcgtGGTTGTTTTCTT +GCGTGGTTGTTTTCCTAAGCTCATCGCTCATCAATTTCCGCTGGATTTTC +GGTCCTTGTAGAGGATGTAGGCTTCGATGGCGGATTTCAGAAGATGACGT +GGCTTGATGGGGTGTTTGATGGTTGTTATGCgggattcctgaaaatgaga +aatttgagatttaaggattactgtaaatgtCTTTATTTcaaaggttactg +tatttttttttgaatcaaaagaaaaaatcaatattttttcatgaaaaatg +atcaatgatgatgatttaaaaactttcatgagaaaaattaaagatttttt +ttccaattttttttcagaattttcaattatttatgAATCTTCTTACCATG +ATCCATTTGAcacactttttgtttttgagaaagttctTTGTTAAATCACT +CGCCAATGGATTTGCCAACTCGAATTCCAGAACGGCTTCGTATCgttttt +tgtctggaatttttttttggaaaaaaaacagctaCGCTTAGGCTCAGGCG +CAGGCGTAGCCTGCGCTATTTGGGCTGAGGTCGGAGGggtgaaattggaa +ttttcaagtttttttttcaccattgGATAGagcatcgaaaaaaaaaggga +cATACTATCCTTAGTTTTTCACCTAAGGGCACCGTTGCCGCGCTATGACG +GTccaaagtcaaaatttcaaaattttctttttaatatAACTTTTGAACGG +ATTAATTTTATGAATTCGGAGTTTGACCATTCGATGCGTCTTTGGAAGGC +GCAtttttttAAcaagacaaaattttttttttggaccccccgacccccGA +AAAGTcgattttaaactttcaaattgaCCGTTTAGCTACACCTTCTATac +gaatttcctaatttttttactgaagGTCCGCCTTGGGCTTGCGCGTTTTT +TAAGTATAGTATGTGAACACATATAATTGTGGGAAAGtccccaaattttg +aaaaaaccggtttttggtttcttttctCACTCTTCTATAATTTCTATGAA +AGTTAACATACTCCACTTCTATTTTAGTCAGATTTTTCGGACTTAGTTTC +ATTACTGAAACTAAAACATGATTTGGGCTGCTGGATCGACCTATGAAAGT +TAATAGACCGCTCGCTTCGCTCGCAAGATAATCCTATGAAACCACCTGTT +ATTAAGTAATAAGCAACAATTTAGAAAATGGtcgttaaaaaaattttcaa +aaaaaaaattttcaaaaaaaaaaattttcaaaaaaaaaattttcaataaa +aaatttttttttcaaaatttcaaaaaccctaaaaatacttaaaaaCACTT +AATTTACACTCAGGGACCCCTACAGTactcgaaaaaatggaaaaaaaaat +tagtgcCACAGGCGGGTTCCTCGGACCGAAGGTGAAATTTTGGACACACT +TCCCAACATTTCCTATCCATGATGCAAAAATCAGAACATGTTTATCTCTT +TTCTCAACTGAGTTATTACACGTCAAACAGTGGTGCGCGCTCGTCAAACA +GTGCAAAGCCACAGTACATTAAAACTTAAAACGACCGTAACTTGCGTCAA +AATGGTTGTAGAGGGATGGTaaaaaaaCGAATCGATCAATGAACTCTACA +TTTTTGTTCCCCTTCATCTATACCCTATCTTTACGTTTAAGACctcaaaa +ttgccaaaaacctcgaaaaaatGACGTTTTTTGTGTTTGACCGACTGTAG +CTCGAAAACGAACCGTTTAACATATTGTTTTAAAACCTTAAATCgagcta +tttttcaatttccaatcgATTAAAAGTGGTCCGAAAAGGATTCTATAGAC +AGTTTTTTagcagtttttttcaaacggCCATATCACACCCGGGTGGTcac +aaaagtgaaaattttttttgcccaTGATTTAAACCTCATTATGTACTACA +ACTTTCCTTCTGTCGGTACCATGCCGCCTCTGACCGTTAACGAGATATGG +ACCCCAAAAaccggttttttcaaaatttggggaCTTTCCCACAATTATAT +GTGTTCACATACTATACTTAAAAAACGCGCAAGCCCAAGGCGGACcttca +gtaaaaaaattaggaaattcgtATAGAAGGTGTAGCTAAACGGtcaattt +gaaagtttaaaatcgACTTTTCgggggtcggggggtccaaaaaaaaaatt +ttgtcttgTTaaaaaaaTGCGCCTTCCAAAGACGCATCGAATGGTCAAAC +TCCGAATTCATAAAATTAATCCGTTCAAAAGTTatattaaaaagaaaatt +ttgaaattttgactttggACCGTCATAGCGACGGTGCCCTTAGGTGAAAA +ACTAAGGATAGTATGTCCCtttttttCGATGCTCTATCcaatggtgaaaa +aaaaacttgaaaattccaatttcaccCCTCCGACCTCAGCCCAAATAGCG +CAGGCTACGCCTGCGCCTGAGCCTAAGCgtagctgtttttttttccaaaa +aaaaattccagacaaaaatCGATACGAAGCCGTTCTGGAATTCGAGTTGG +CAAATCCATTGGCGAGTGATTTAACAAagaactttctcaaaaacaaaaag +tgtgTCAAATGGATCATGGTAAGAAGATTcataaataattgaaaattctg +aaaaaaaattgaaaaataatctttaatttttctcatgaaagtttttaaat +catcatcattgatcatttttcatgaaaaaatattgattttttcttttgat +tcaaaaaaaaatacagtaacctttgAAATAAAGacatttacagtaatcct +taaatctcaaatttctcattttcaggaatcccGCATAACAACCATCAAAC +ACCCCATCAAGCCACGTCATCTTCTGAAATCCGCCATCGAAGCCTACATC +CTCTACAAGGACCGAAAATCCagcggaaatttcaaatttcccgccaaact +CCACCCGCTCATCGCCACCACGCCTTCAGGCATCATTCAGGAGacgttat +caaaaattttggatccagaaaattctgaatatttatCACTGAAGAAGGCA +AAAAAGAGAGTGGAAAACGCGTCGGGAATGAAGATTCTAGTGGAATTGAC +GTCGAATCAGAAGGCGATTCAGatgaaggattctgaagacgtggaggatt +cagaagacgaagaagacgttgacatcgaacgtcttctaaaatattctaga +aaaatttcacgTGAAATAGacgaaaagaggaaaatagaAATGGATGAGAT +ACTGGAAAAAGCAGATCCCGAGAAAATTTGgttaaaaatgtctgaaaatt +cggaaaacttgGAGATGAACGAAGacatgatgaagaagaagacgtcgctg +aagaaaatcgccaaaaagcagaagaaaaatgtgagaaatcgatcgaaaaa +attggtggAGGCGTCAGATGATTCAGATCAAGATGGCGTGATCTGTTTAG +AAGacacttcagaagacgtgaagaATAAAGATCCCGCGAagtccaaaaat +aaaaaaaggaaggatCCAGAAGAGTTCCCAGAAGacatttcaaattctga +atccaaaaaatctgaaaaaagaaaacgtcGAAGTACAAAAGACGTGACGG +ATTCTGAATCCAAATTTActaaaacgaagaagaagaagaagacttcTGAT +GACGTGCCAACTTCGGAAGACTTAACCGGGAGACGTGGCGGATTCTGAAT +GTGAAAAGTGGATAAAAGAGGAGTCAGTTGACGTGGCGGATTCTGAATGG +ACgaaagtgacaaaaaagaagaaggagaattctgaagacgtgcCAACTTC +AGGAGACTTTCNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN +NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN +NNNNNNNNNNNGTCATTTATCTAGTAATGAACACCTTTCGTATAATAGTA +TTTATAGTGCAATGGCCtatgattccaaaaatgaagaCCTAGGATCAGAC +AAATTGGAATCCTACAACTGGAAAACGTGGCGCTATGTATCCGACAAATC +CGGTATATTAGAGTTATACTGTGTGACGGCCTAGAGTCCAAAACCGATGG +CCTAGGATTCAAGACCTCACTTTCTAAACTGAAATTCATCAGGGTACTGT +ACCCGAAATTCAGAGACAGAGAGTTTTAATTGAAttgttccgaaaaaaat +gtcCGAATTTCCCATGCTGGGACACTTCAAATTTCCTTTTCAGCAGAGGG +GGGCTCTGGAGAACTTGGGtgaaattggggaaaaaatattaaaaataggT +CCCCAAATTCCCATTTTGAAGGGGAAgcgaaaattgggaaattccgCTTT +TGAAAGAGGGAAACTtgcaattccaaaaaaaaatttttagactaaaaaac +atttattacCATTTCAGAtgatttgagctcaatttgtGGTCCGttttgaa +aaaggaatttcagatttttttttggaaatttttgatattcctggaatttg +aaacaacttaaaaaaccgaaaattcaaaaaaaaattaaaaactcgaaaca +tatagtttgtagtctttgtagtcccttcaaaaatcttatagttagtctaa +aaaaaatacaaatttccttctttttggtCCCATTTTCGGCCCCTAAATTT +ATCCATTTTCCACCCAATTTTACGACCTTCGGAAGTCAATTCTCAACCat +tttctcttctatttcacaaccttttattttcgatttttcgcttttttttt +tgttttttgaaaattgttcacatggacaattttgcttttttcccccattt +tttcccaataaaaatttccattttaaGTACATaatttgagtgaaaaattg +agatttttcaggattttagaattttaggaatttctgaatttcagaaaaaa +atgctgaaaaagaaaTGTAGAGTTTTCAAGAATCCTCAAAAAGTCTCAAA +TCCTAGGCCATCTCTTTTGGGCTTCTAGGCCATCATAATAATAGGCCATC +ATAATTAACAAAGTAAATCTAGACTTCCGGTTTTATGGATCCTAGGCCac +gttttccaattttaggtCTCTTGGATTCTAGGTCATGTACATCCCAGTAC +CAATCCTAGGCCATTGCTTTTAAGATTCTAGACCATTGTCGGATCCGAGG +CCATGACGATTGAACTGAAATATAAAACTCTTTCTAGGATTAGAAATTAC +CGTAATCCATAACCggaactaaaattttttgaaaatccgtggttttcgaa +aaattccaatttctcagAAAAGTCATAAAGTTCTATGGGTAATAAATCTT +TATTTTCTTGGGGCAATTTTTGAGGTAAACATAGTAATTAAGTAATAAAT +GTGGGTgaaagattacggtagatctCCAATtagaggattacggtagaccG +GAGTTCTATAGAAACCCTGTAAAACCAGCAATTCTTCAAATCAATTTATT +ttggttcttgtgaaagatatcaaaatgtgaccaactaataaaatattgcg +ttttttatgatgaacattttattagttgattacttttttataacttttac +attaaccgagataaatcctGTTGAAGTTTACGCTATGAGGACCCTTGGAG +ACCGCCATTGGAACCGTTTACTTCAAGCCAatatatctcgcttcctgtaa +gagatatcaaaaagtggtcaactatcaaaatattgatccgttcttgataa +acattttgtcagttattaactttttgatttctcttacagtaaccgagata +aatttagTTAAAGTTGACGCAATATAGGACTTTTCTATCGCAGCTTTTac +catgaaattttttctaccCTCATGACAATATTTCAATCCCATTTGAATGC +ATATAATATTACTTTTTGTCCCTTTTCTTtatttggaccaaaaaaatttt +gtctgaTGAGACCACaagaaaaataacaagaatcatcattttcagacaat +tcaagttctgaaaattattttttcactaaaaatttcaaaattttcgatga +cgtgcattccagaattttttttctagaattcacAGTAATTAGATCCTGCT +GACTTTCCCATCGAACCAACGTCAATCAGGCcttctgacattttcagaag +gaataattttcggatttcaaaaaatttgtttttttatttttatttttgtt +ctcTTGTACCCTAAATAAAAACTATTGTAAtccctttgaaaatttgaatt +tttaagtaccgaaacccatttttctaattttttagaacatttcCTATTAC +CCcgaaattctccaaattttcccCCAGTGAcctctcctcctcctcgactc +ccaccgtgccccctttcctgtacgacctccctccctttctcctTCGGGGC +CTCCTTAGCTCAACAGAAAACGaaannnnnnnnnnGAAGGTGGTACATAA +CTTGTTGCATTGTTTGCAAATCCTCATCTGAtcttatcaaattttttcat +aatctcTTCCTAAAACCtttttgctaatattggtattttttgctcttatt +aatatttttccttttttttcgatatatcAAATGTCGTTGattgacgtcat +ctgaatgaCATAggttgacccattttttggTTCATGTAAActacttttcc +aaaaaaggcttaaaccatcgaaaaactccctgaaaactaatgaaaactaa +tgaaaacacaaaaacaaaacttaaaaaaaatcaagccACCTAGGGGACTC +GAACCCCTGACCTTAAGATTAAAAGTCTCACGCTCTACCAACTGAGCTAA +GGAGGACACGTAGCACAATGTGGGAGTGCGTACAAGAGGGCGCGCCCTCT +CGTTACTTTTAAGTTCCCCGTGAAGGTATTTGGGtggattcaaaaatcaa +aaaaatccaatcaaaataaaaaaaggaattgtCTCAttcaaaagagagaa +aattggtttttccaGGGAAATATATGATGAAAATgtatcagaatttgaaa +tgtATCAGAACGGAGAAAATGTATCAGAATGGAGAATTGTGTAGCGTTTA +GTACCCAGACAGAGGtataaaaatttatattctagacatttcttttttgt +tgaaattcgcagaaaattggatttttcagaatggagAAGAAACCACTTTG +aggtttgaaaattaaaattttcaaaaatttagaaatttcaaaaatttaga +attttcgagaactctaaatttcgagaaatttctgaaattcagaaattcag +aaattcagaaattcagaaattcagaaattcagaaattcagaaattcagta +attcagaaattcagaaattcagaaattcagaaattcagaaattcagaaat +tcagaaattcagaaattcagaaattcagaaattcagaaattcagaaattc +agaaattcagaaattcagaaattcagaaattcagaaattcagaaattcag +aaattcagaaattcagaaattcagaaattcagaaattaagaaattcagaa +attcagaaattcataaattcagaaattcggaaattcagaagttcagaaat +tcagaaattcagaaattcagaaattcagaaattcagaaattcagaaattc +agaaattcagaaattcagaaattcagccattcagaaatgcggaaatccag +aaattcagaaattcagaaattcggaaattcagaaatccagaattccagaa +attcagaannnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnttcagaaattcggaaattcaaaaattcagaaact +ctaaaactcctaggtcaaaaaGTCCAActaaatcaaatttttggaatctt +ctagaaactccaaaaaaattttgtacccaaaaacattttttcttctgatt +tcctTGTATTCTAGAATCTCTGtctagaattccaaattctcatttttccc +acGTTTCGTCTCTGATTGCTCATTACTCATTACTCATTCACAGTTCTCAG +ATTTcttagctccgcccacttttctctctctctctctctctctctctctc +tctgtctctctctctctctctctctctctctctctctctctctctctctc +tctctctctctctctctctctctctctctctctctctctctctctctctc +tctctctctctctctctctctctctctctctctctctctctctctctctc +tctgtctctctctctctctctctcacacaatcaatctctctcactctgcg +tctccctctctctAACTTTGATTTATAGACTCCTTCCAAGTCTCTtaaga +ctccgcccacttcgcCGCGAAACTTTtcggctccgcccactttcgaGAGC +AGCTCCACTTCTTCGACGGCGAGCTCTCATTCCATTTCTCCTCCACACAG +AGTCTTGACGTTCTTCCCAAGTACTCTGAAAGCAGCAAATATGCTTCACC +GAACTGTCTATTTGGCTCTTTTGGCCCTCATCGTCGTCTCGGCCACGGCC +TCGGCCATCAGTCTTGAGCAAGCCGgtcagttttgaaaagttttgttaTG +AGAAGGCTATTTAATCTAGAAGggcattttctcaaaataccgTTGTGTTC +AATAGAGCGCTCTTGCCGAAATTCGCTGTATTGTCAGCAGGGCGCGGTTG +CGCGCGGCGCCAATAGTATTGCTTGGAAAAATCAggtttttatcgatttt +ttttgccattttgacATGATTTTCTTGTCTAGAAAATGATTTAAAGGCca +aaacacacattttgagtacaaaagactgaaaaattacagaaaaatcgaat +ttttgaattttttgaaaacaattttttatttgaaaatttcgattttttgc +catttcgacatgatttcctcatatattttgattgCTAACACATCCATGAC +ATGAaagctgaaaactttgaattttttgaaaaaaaattttttgaaatttt +aaaattttttaatttaatatttttaagatttcccaaatatttcgaatttc +aggatttttgtagTCTCTAGGAACCTggaattttcactgaaatttttaag +ctccgtgagaaaaaaaaatcacaacttttcaaaaaaaataattatttaat +taaaacaaagggctttcaaaaaGGTTGTGGGAGTCGTTTTCATTTTGCCC +TTTTTGACGAAAACTCCTATACTGAATTATGTGTTAATTGCCTCCATTCt +agaggaaaaaaatttcaatgggTACATAATCCATTTGagataaattcaaa +atgttgagTTCAAAGGGAGTTATGGAGCTTGAAGTGAGCTAAGCGCTAAT +CTACCAACATGGAATTATACCTAGATTGTGTTTTGATTCGCCTGCAAGCT +AGCATTGATCAAAATTAATAATGTATAATATTAATCTGTAGCTCTTGTCC +AGGGCTACATTTAACATTTtgattcatcaaaaaatattaggtTCGACGAC +ATATATGagcaaaaaaccaaaagtagCAGCGATCTAGACTTTCGCACCTT +CTGCTTGAAATAGGGCTAGATTCACATAGTTAATGCTTAGTATTATTTTA +CAGCTCTATCCAAGAGCTACAAGATACAAGTTAAATCAATAAAAACTATT +GAGTTTGAAACGagttataatcaaaaaacctaGGGTGTAGTGGTGGTGGT +AGTATAGCGAGCTAGAGATCTTGTAGTTTGTACCCTCCTGCTTAGTACAA +GAACGgaaattttgtaaataatATTTGATGGTTCTGTAGAACTTTTCAAG +CGCTACACGAcacactttgaatttttcaaaaattctaattttgaagcgag +ttataagCATAAATGCTAGAGTGTAGTGGTGGGGTAGTCTAGCTAGCTAG +AGAACTTGTAGTTTTGACTGTCCTGCTGGGCATTAGGGATGAGTTGGCAT +AGTTGATATTTATACCAATTTGTAGCTCTTTTTAAGGGCTACAAGATACA +AATTGAGCTACTGCCAAAATCTGATTGCGAAGCTAGTTTTAAGCAATTCA +CTAAGCCAGCCCCTCAATATCAGGCATCCCACTGACTGCGGCATCATAGC +TCCCAGACTTCGCTGTTGACATAGTTGATATTAATATCAATTTGTAGCTC +TTTTTAAGGGCTACAAGATacattttgaattgtttcaaaattatattttt +gaagcaagttataaGAGATTTTCTGAGCTtaaccttctttttttaaattc +aaatttcccgccaaatccTCTTCTTCCCATATTTCCTCCAAGTCAACAAG +GAATTCATATGAATAGTATAGATCCATGCTTCTCAGAACTTTCTagagac +aaaaagaaaagttttatGGCTCTTCCCATGGCTCCGAACCAATAAAAACA +AGACTCTCCGACTTTTTGATGACGTGGCGAAgggcgaaaaaaagagacaa +ggAACCCAAAAATATGTGGaaaaaaaTAAGCATGAgcgatctggaaattt +gagaaaaaacggcCTAGGAATGATGTCATATAAGGAATtataagaaaatg +tttttctttaaaaaaaatagaaagttccagaaaaatggggaaaaatggca +aaaGGGAGGCGCTCAGGCTCTTTCGtgtcccaaaaaattaaaaattctga +acaaaataaaaGTTCCCATAGCAACTATTGTGCTCTTCTAAGAGCCAGAA +GAAACACACAAGAAaggaaaaacctgaaaaattaatattttcagacggCC +GAATGAAGATGTGCCCTCCCGGGGGCTCTACCTTCACCATGGCCTGGAGC +ATGTCGTGCTCGATGCGCCGGAAACGCGACGTCTCCGCCGATGACTACTA +CGGAGGCATTGGTGAGTataattttgcatttttaatATGCTCAGGAGATC +AGGCATAGTCGAAAACCCATtgtcacaaaaaaaaccagagaGAAAAACCT +ATCGTACCGCCCAGGGGAGTCGAACCAAGAACCCTCTTCGGCTCCAAAGT +CACCACCACTAGCCACTACACCACACTAGTGTGGGCGTCCAACCACTGTT +GCGTGGCGCCTCGCCACCCACCGCGCCCTTATAAACCAAACCAATTGTCG +GCATAGCCCGCCCCCCATAGCCTAGTGGTTAGTGACGCCGCCCTTGAAGC +AGGGACTTGTTGGTTCGACTCCCCTGGGCGACccgttgatttttaatttt +ttgctttattcTACAAGGTGAATCAAGATCGCTTTATTCGATAGAAGCAA +TataatttgtgattttttttcagccaaacgCGCCCTGATCGCTCCATCGA +TCCGTCAGCTCCAAACAATCTGCTGCCAGGTCGGCTGTAATGTGGAGGAC +CTCCTCGCCTACTGTGCACCAATCTAgtaggccacgcccccttgTTTCCA +TCTTCACTTCTTCAATTCCACTGTATTTGTACTTTTCCACACCAACATGA +GCCATCATCCCACCCCCAGACACcgtcaatttccaaaaaaaaaaccaaac +atcTTCATCTTCCCTTTCTTCGTAATTTATCTTTTTGTCCTGTCGTCATC +AATTGTTTCCACGAAAATTCCACCTGAATGAATCAAGCTGTTAAAGAAag +tattttttagaattttcagtttttcagggagaccagaaaaatgatttcaa +aaatatttaggattaatttttaaaaaaaatatatattacAGAAAAACGCA +CAAATAGgggaaaaatgcaaatgcgctctattggacaCGAAGTTCaaaat +ggagcgcattttcaacgaaatttccaaaaataacgTAAAGAGAGATATct +agaagaaaatgagaattttcggaaacttttttgaaaaattaaaaaaattt +tttttgggaatttcgaaaaattaaaggGACACACATATTATGCAGATTTG +TGAGCAGATGAGATTCGGGGTGGTGGTCagaaaattgtacaaaaagaGGA +AGGTtaggcgggaaattcgaattttggggATAgtaaatttagagaaaaaa +aagaaaacatcaagtttttggataaaaaaatttcgaataaaaaggcggga +aatgcaaatttctgaaaatttgaaaattaataaaaaattagtaaaaaaac +ggagaaattTAGAGTAGAAGCGAAGCGTCGTCACGGGGCAGCAGCTTGAA +TGGGTCGCCGTCGAaccactgaaaataaaaatgtttattgaGAGAATCTG +GATAAACAGACACACAGctggatttattttttgtgggaacaaatttgaga +atttgttttctagcgaagtggtggcctagaaaagcaacattgcaaaagtt +aggccaccacttcaaaatggcctaaaaatgacctgacatgaccctggtgt +gttttctaggccattttgaagtggtggcctaacttttgcaatgttgcttt +tctaggccaccacttcgctagaaaacaaattctcaaatttgttcccacaa +aaaataaatccagCTGTGTGTCTGTTTATCCAGATTCTnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnGTACGAGAAATAGGCGTGAAG +GAGCTTGACGTTGCTCTTCAGCATCTGTGAGCATCCCGCGATTTTTGGGG +CTGTTAGTctgaagaaaagagagaatttAGAAAGTTTTTCGGCTAATTAA +ATTTTCCTCACCTCTCAGCGACTTCATAATAATCCgccattttcaactga +ttcaAATACGTCcatgccaaattcaaattgatgaaAGCAATCTTATTATG +CGACATGGTGTCGTCTGGATTAATGAAATTCAGCGCCGAGACGAAATGCT +TCTCCGCGTAGTCGTATTGATGGAAATAACAACATTGGAGaccctggaaa +tttgaaatttttacagttaaattgaaaaaaaaattaggtaaaaGTTAGCT +TGAAAAAGTGCCATTGAAATATACCCTTTTTTAACGGACGGAAGTTTCCT +AGTCCACGGCCACCATAGTCTTCTTCAGCTCCAATATTTTGGAACCAACT +GAATTGAGTATGGTAGAAAAACTCAAGAGCtccaaaatgatactatttgg +attactgtaccttgtcAATGAACAAAGTCAGCAAGCCAATAAGCAGTGAT +CGCTTAGCTCGTTTGGTTTTTGATTTGTAACTtcgttcatagacaagata +taGTATCCATAAATAGCTTCGTTATGTAGCTCATGAGtggaaaattgtca +aaaaattattgaaaaatgtccataattcataggaaaatgatccaaaattc +atgaaaaatggtccaaaatttacggaaaatggctcaaaattcatgagaat +ggcttaaaatttgtgaaaaatggtcaaaaattgatatatagtccaaaagt +catcaaaaactgccaaaatttatgaaaaatgactcaaaatccatcaaaat +tgtccaaaattcatgaaaaatggtccaaatttcttggaaaacggtaaaaa +atacattttttaaaaacccaccaaaatATACTGAATATGTGGTGCATAGA +ATCGATACAACAGCGGTCCACTGGATCGATTACACATCTCCAGCATATCC +CGAACATTCGACAGTGTCTGTTCTGGTCGACACGCCATCACATTACAGCA +AGCCATCATCTCCAACGTCGTCATTTTGATCATATTGATACATCGAACGA +CACCCGGCTCCTGNGGTGTTCGTGTGTACTTGTAGATGGCGTCATCCGCA +TGTTTCACGActaaattgaaatatttctcCAGTCGACCCGGATTCGATTG +AACCAACGCTGAGGTGATGTTCATTACGCATGCGAACAGGGTCATCGACG +CTTCGCCCAGCCATCTGnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnaTTACGCCAGCTGGCGAAAGGGGGATGTG +CTGCAAGGCGATTAAGTTGGGTAACGCCAGGGTTTTCCCAGTCACGACGT +TGTAAAACGACGGCCAGTGCCAAGCTTGCATGCCTGCAGGTCGACTCTAG +AGGATCCCCTCAAATCCGTCAAAAAATCTTGTTCTGAAGCCGAGTTATCT +GCCGGCCCGCCATCTGCCCGGGTCGGCCCGTTTGCAAATATGGTCTAAAA +TAATTATAAAGGGTCAAAAACTATCCGAGAATggtcaaaaagtgatcaaa +aatccatcgaaaattgtgaaaaggaactcaaaaatggacgaaaagtgctc +aaaatcagtcaaaaataggcccaaaatgatccaaaaatttttcaaaaacg +ggtaaaaaataggttttttggctgatttttgaccatttttgattgctttt +cgagcatttctcacaatttcagactattttgtctgattttttgcaatttt +gacaacttttaaaactgattttagaccattttgactgatttttaaccgtt +tttcataccatttttgactgaattttcgtcgttttatgaccgattttttt +tcgtttttcagctgatttttgatcatttttgactgattttagACTAATTC +TAAGTCAAAAAGGGCcgaaaaatggtccgaaataaGTCAAATTGACCCAA +AGTGGGTCACAAATTAGtctaaaatggtctagaaatggccgaaaatccaT +CAAATATCGTGTaaaaggaggtcaaaaatggacgaaaagtgctcaaaatc +agtcaaaaataggttcaaaatgatccaaaaatgtttcgcaaaagggtaaa +aactagttttttggctgatttttgaccatttttgattgattttggagcat +ttttctcaatttccgactattttgtctgattttttgcaattttgacggac +tttttAACATTACTAGGACCAATTTCGGCTGATTTTAGACagtttttaag +ccaaaaaggGTCCGAAAGTGgtccaaaatcagtcaaattaacccaaaatt +cgtcaaaaattagtcaaaaatggtccaaaattggtcGAATTTGGCCGCga +atagctttaaaaattaGTCAGAATATTCGGgttaaaatggtcaaaaattc +attgaaattgattttttaatgggTCTAAAATGACTCAACAGTGGTCAAAA +ATAGGTCAGAAATTGTCTAAAATCACTATTGGACCAAATTTTGCGCCtta +ttttggaccaatttcaatcaattttgacggaattttgaccactttttaac +CAATTTCTCACAATTCCACCCTACCTGATCCCCTGAATTGTCGTCTCTCC +ACTCGTCTGTTGCTTTTGAATTTGTCTCAAAAGTTGTCTGGACGTTCGCG +ATTTGCCTTGATAAAACATGTAGGCCAGCTTTGTACAAACGAAAAAATCT +TTGATCACTTCCAGATGAGCCTTGTCTGCTGGATCCGCTTCCAGTGCTAA +AAACTGGTGATTTAGATGGATTTTAGTGACGAAAAAggtcaattttaggc +caaaattagctcatttttcgtcggattttcgaccattttcaccAATTTCG +GATCATTTTGACTGTTTTCTGACTCGTTTTAGACCCAGTAAtgaattttt +gaccattttaacCCGAATTCAgactattttttaaagctatACGCGACCAA +ATTCAACTAGttttggtgtattttttgacctatttttacgatttccaacc +atttttgcgcaaaattagcactatttttgaaacttttcgacTAATTTTTT +GGTTCATGTTGGGCTAGTTTCtgactcattttcagccattttcaattttc +agccattttttaaggattaatttcagcagttgattttcTGTCAAAATTCA +TCGATTCTGGAgatttttaaactcaattttcagcatgttGAAGCCATTTT +CCcgtattttcagccaattttcagccatttcacggtatttttttgctatt +tcaaATGTCGTTTTGGCCGTTTTTAGATTGATTTTAACCAATTTCAGCTT +CATTTCCTATCAGTTAGCTTAGCTTAAAACTGACTGATTCCGGTGATTTT +TAGCactttttagctcaatttctagcgtttttcagctatttcgagcccat +tttcaatgattttcctGCTCACCCTTAATCAATTGTCCAATTTTGAGCAC +ATCCTCGTTGGGCGGCTCCTCGTGCATCAATTGATATGTCACCAACGTTT +TCGTGATTCGAAAGTATAATTCCATTTTGGCGTTGTTCGCAGCGGCGCTT +TGTGCGATGCCATAGTGACACATATCGAATGCGCCGTCCGAGTCGTTGTC +gattttgttgatttcctGGAAGAGAGAAATtgggagttttctaggccagc +aagagttatggcctagttttgaaggtttttaGGCCATTGGGGAGActtgt +ggcctaacttttttctaggccatcttaTAGGACCCTACCTGGAAGAGGTA +TGCCTAGGTTGaaggtttctaggccagctaAGAACTGTGGCCCAACTTTT +GCAATCatcattttctaggccatttttgaggaatttcgattttctaagTC +AGCTAAGTTTTCATGAGaaccattttgaagtggtggcctagaaaaccaaa +ttgtcgaaagttaggccaccacttgaAAATGACTTAggaaacacaccagg +gtcatgAGCATTGTGGATTAATTAAGATCTCTAGGCCAACTTCTtgtgtg +gcctaacttttccattttgctattttctaggccacccctTTTCTAAATGG +cctaaattttgagatttctgaatttctaggccatttccgacatttctagg +ccatcaggtATCTTTCGGAGGCCATTTTTGAGGTTTCCATGCCAGAaatt +ttggtggcctaactttcttatttttcgattttctaggccatctccATTTT +ACTGGAAGagggtttctaggccaatttttttggatgacctaacttttgag +atttatgagtttctaggccatttaagacatttctaggccatcaggcTTCT +TTCTAAGGCAATTTTTATGTGGTTTCTATATCAAAACTTCACGTGGCCTA +ACTGTTTTATTTgcgattttctaggccacaccATTtctgtggcctaactt +ttgagatttatggatttctaggcagtttctcgattttctaaGCCACCCTT +ACAATCAATTGAAACAGGAGCTTATTGGTGAGCGTCTGGAATCCACGTGT +ATGCAGAATCTCGTGTTTTAGCGCGTCTTTAATCGGTCGAGCCGGCCagt +tttggtctgaaaattgaaatttttgtaaatctgTGCATCCGGATGTCCTT +ACAGTGAATATAAAGATCGGCAATCATCGAAATAGCGTGCATTCGCTCCC +GAGCAGCGAAATCGCCCATTTGTGTCATTTTGTCGtactggaaattgaaa +attttgaatattatgGTGTTCTGGaccaattttagaccatttttaaaCCG +TTTTAAACCTATTTAACTCACCGCCAACTGCAAATGATTGCGGGCTAGGT +CGAAATTCTCTGTAtagaaaaagagcaattttCCGAGCTGCAAATTGCAT +CGAGCCCTCGTTTCGTCGTTTGTGTCCAGTTTTACAGcgctctggaattt +ttttccgggtcatttttgtgtatttttggaattttttgaccgattttgac +attttttgagcaaaatcggaaaaaaaattttttccgaattttcaaaattt +tccagattttcagctctcttttctcatttttctttaattttcgacaattt +ttcatttcaatttcagctcaCTCTTGCACATTTAATAgccattttgagtt +ttggcgGGTCCTGAGTTCTCAGCGCTTCCGCCATGCCCAACAGCGCCTTC +GCCACTTCATCCTGGttcattttcctgaaatttttgatacaaaattttaa +ttttttcaagtttttagcgGAAATTGCACTCAAAACAGactaaatttcaa +atctggCGGTGGCTGAATCGACTTGGAAACCGCGGTGGAAATGTCAGcga +tgcaagcgcgctccgtTGAGAAAGCGAAATTATTTTATTGGAAAACAGGG +AAAAGATAGATTTTCTTCGGTAATTTGTGATTTATGCGATTCGAAAGGTT +GAAAATCGTGTTtagcaaaaaaatccaaaccaAATGACATGTAAAACTTT +GCAAAATAATCAACTAGGCgctgagaaattgaaaaaattgcatttgaaaa +atgatctttttcggattttcttttatttgtATAGGATTTCAGctatgaaa +attggaaattttcaattttcaggatacACATAAACActaaaatatcgatt +ttcctGCTGAAAAGTTAACAAAATAAGTTTCCCATCACATTGGAAATCAC +ATAAATCTTGCTTAAACTTTACTTTCGCACtagcgaaaaaattgaggaaa +aaaagatggaaaattgtctatttttcactgcatttccgcttttttaatct +tctcgatcccatggattttttcgtccttttttccttttttgagctTGTGT +CCCAAGCCTTGGGGATGGGAGAAAAGGGGAataaaataggtttttttgat +gaaaaaaaaaacaaaaacagtgATTAATTTTACAATGTTTGAATTTATTA +ATATGGAATAAAAGTTTTCCAATAAGATTCAAAGCATGTTTCCAgcaaag +aaacgaaaaaaagcaaGTTATACTCAAAAACAGGCCTTCAAACttcgacc +caatttatctcggttcctgtgaaaggaagcagaaagtggtaaactaacaa +aatgtttttgataatatgataaacattttattagttgaccacttttttgt +gcctcttacagggaccgagaaaagtcggctcaaagttgagggtaAGGTTT +TTCAGTGTTTCGGCTGAACATCCTTTGTTATGTAAATAAAGCTCTTTAGA +ACAATTGCCTTTGAagatgttttatttttcaaaaaactagaaaaattgat +tagaATACAAGATGAGAAGACAGTAGAAATTCAACATTCTTATAGTTTGA +GCACTTCCATTTTCAGTGACCATTGATTTCTGTTAATAGATTGATGGAAT +ATTGGATCACCATAGGAGACGAATAGTATGGAGGATGGACCAATTGACAG +TTTGatagatctgaaaatatcggTTGTTGGAAAACTGGCAATatcaaacg +cgctctaatggAAGAAGCATATCTCGCctcaaacttggaatttctcagcg +aatcaaaatttgaactgtACCTACGATCCGATTCGATAACTTGAGCATTT +CCTCTTAGAATATCTATATACCGCAACGGTTCCTCTCCTTGAAAGGGAAG +CCCTTGAACTTCAAACGACCAACTCGTCCCAACTGGTCTTCCGTTTTCCA +GCCAAAGGCTCATTAAGCGAGCAAAGCGGTTTTCAGGAATCCGAAACATG +cgaacttttaaatttgaaactctCATAAGTTGATCTATATGACAAATATA +CCCTCCCGAATATAGATGCAGTTTTTCAGTATTCACCACAATTTCGTGat +ccaaatttctgagatttacTGCACAGCAGAGGCTTTTCAAGGGGAAACTG +GTTGGATGACATATCGATCGAAGACCATCCAAATTGACAGAATCATAACC +CATCACTTGTAAATTTCGAGCTAATGGCTTTACTCCGGACGCAATCCAAC +GAAGGTTGCCAGCCGATATAAACATGGATTCTACATTCCAAATCACTGAT +CGATTTCCAAATATCACACCAATCAGCATTTTCATTCCTTCAtaaatttt +aatgatttttgaacttttgtaAATCCTTGGCTCTCCATTTTCAGGTTTGA +TTGTCAATTTCAAAGTGTGCCTGGATTCAGGACGCTTGTTCTCTTTCATC +GTCTCCATCAGGCCCAACTTCTCATGTCTCAAACGAATTCTTCTTACTGC +TTCTACCTCATCATCACTGTCATTTTCCATACGAACATCTCCATTCATCA +TTATACCGTTTGTGTTTCTTCTGAATCCAAACTCATCGACATCATGAAAC +ACGGACATGCTGATTTCATATTGTCGATCGATGCTGATTGTGAGTGCATA +TCTAGTTTTATTGACAGAGATTCGGGCGTCTGTGAGAGCCAGTTcgttaa +ttttcagatgagttCCTTTCTCGATGGAACGGATCGAAGGCAGGTGTAGG +GAGTGATGGAATCTGAGAAAATATgtttaagaattttttaaatccaaaaa +atgtctGTTAACTCACCGAAAGTTAGAATCCATGTATTCAAAAACAGTCT +GGCTCAGTTCATACAGTAGTTGTTTGCTCGGCATTTGTTCTGAAATGGGA +AAGTTAAAACAAATGAGACACGAAAAAGATGTACTACTCGACGATTTTTT +CCATCTCCATAAATCTGTTATGAATAGCGGCATTTTGAGAGGAAATTcca +agggggactccgaccaaaaatttctaTTGCATTAAACTGTTCGAAAAGTA +gtctattttcccaatatggtagcgaaaaaattccaataacatccgaaatc +cccgaaaaatgccgcacaaagcgaaaaattctcaaaaaaaaagttgttgg +gAAGGACGAAGATCTGAATCCCTAAGGATGATGAATGTCTGGTTAattct +tcttttgaaaatttttgtaatccatggcttttttcttggtttttcttttc +tgaaaatgctttgaacctttcagaaaaacctttagttgaaatgaatgaat +ttttCGTAAAGATGCTTCGCTCTCACATAGTGCAATAAATGATCGTAGTT +TGCTTGAAAAAAGCCTTCAAACCTTGCGTAGCTGATTGCTGAATTACTCA +GAAGGACAACCAGGAAtcttttttaagaaatttgattcatttttaatCGA +GCTGTAGCTCCGCCGCCAACACATACCATATCCCAGnnnnnnnnnnaTCC +GCAAAAATTACCATGTAAGAAAGGCGCTAcgatgccgtggcctaggtttc +tcactgcgcgggaattcaaattctatttttttttcaacgccgtttttcgg +ggttttaggattttttttttgaattttttcgcaataatattcggaaaata +gataactttaaaggtgggtggacatgaaaaaaataatgctctcgaaattc +cgttcaaagatgcccctcgaggagttgaacttgtgaggacgATGATTGAA +AGTGCGGTGCCgcaaccgactgagctacgcgggcaaagagaaagcgcacg +cccgattgcatacataaaggcgaggacgacagctgtacgcagcgtgacat +taagcgttgtacaccgtgacggtagaatgcatactctagctacaaaaaaa +ctacgaaaaatctcctgtccccccacctttaaacactttttaatgcaata +aaaatttttggtctgagtccccctttaattgaAAGAAATAACTTTTGACT +CAACGTATGTCGTTTCCTGTTAGTCTATTAAAATAAGGAAAATTAACACA +CCGAATGTATCGATTGGTTGGACCTCATCTGTGTGTGACCTGTTTGTATA +CCATCTGTCAATGTGTTAGTGCGAGACAGTGGGACCACCAGAGACGAGAT +GTGCAAATATAAGAATGGTCCGTATCAGGCATGAGCCGAATTCCTTATTC +CGTATTctcgattctcaaaatttttattccgctTTTGTCTGGTCCGTATA +GATCACATAGGTTCTgcttaaaatttattttttatcgcttgaaaaaaatt +gggaaacattttttagcgatggggaatggaaaaaaagggggaataaattc +gatttttatactgaaaaacaaaacaagggattggtttttttttaaatcag +ttGTGTTGAATTCAAAtgaagcatttttcaaacaaagaaacaaaaaactt +aaTGAAAGAGCCTGACAACTTTAATCCAacatatctcgttttctgtgaga +gGTAACAAAtacttgtcaactaataagaGTAGTCCGTATAGATCATATTG +GTGCCTGACACAATTTGAGGACTGGGATATGGTATGTGTTGGCGGCGGAG +CTACAGCTCGattaaaaatgaatcaaatttcttaaaaaagaTTCCTGGTT +GTCCTTCTGAGTAATTCAGCAATCAGCTACGCAAGGTTTGAAGGCTTTTT +TCAAGCAAACTACGATCATTTATTGCACTATGTGAGAGCGAAGCATCTTT +ACGaaaaattcattcatttcaactaaaggtttttctgaaaggttcaaagc +attttcagaaaagaaaaaccaagaaaaaaaccatggattacaaaaatttt +caaaaaaaaaattaaccaGACATTCATCATCCTTATCGGCGGAGCACTTC +CATTTTCACAATCCAAATCTCTCGTCGGCTACCAGCATCATGGAGTATTG +AATTGCCATAGGAGACGGATAGAATAGATGATTGAcccattgaaattttg +atggatctgaaaaCATGAGTTGTTGGGGGAAGGATAATTGGAAACGCGCT +CAAAGTACAATGAGAAACTTCATTTACCTACGATCCGACTCGATTACTTG +AGCATTCCTCCTTAGAAATTTTATATATCGAGACGGTTTCtctcttccaa +attttcgaatttcgaatagCCAACTCGTCCCAACTGGCCTTCCATGTTCC +AACCAAAATTCTATTAAACCAGCGACAGTATTCAAAGAAACCCTATTGAT +TAGaactttttggtttgaaaTCTTCATTAGATTCAAAGAGTCCATATCTT +GTCTGATGATTTCCGAGCGAACAATCAAATATTCCGCAGTTGCCAGAGTT +TGGTGATCCAACATTTGATGactcatttcgaaatttagaagcTTCAAGGG +GAAACTTGTACTGTCACATATCAACTTGACACCATCCAAGTTGACGGAAT +TATAGCTTCCCAGTTTTACATTCCGAACAAATGGTTTCTCTCCAGGCTGA +ATCCAGCGAACGTTTCGACTTGATATACTCATCGAATTCACAGTCCAAAT +CATTTTTCGATTACTGAATCAACACTTTTATTCCTTCATAAATTGTTGTA +ATTGTTGAACTTGTGTAAATCCTTATCTCTCCATATCCAGTAATGTATGT +CAATTTTAAAGTGGATTTGCATTCAGGTGGCTCCTTCTCTTTCACAGTCT +CCATCAGTCTTTCATATGCTCTCGTTGACCTGCTATTTTCGTTTTCCATA +CAAACATCTCCATTCATCATTATACAGTTTGTGTTTGCTGGAAATCcaaa +ttcattttcatcatGAATCTTGTTGATTCTGAACGGGCGATCTTCAATTG +CATATTTTcgatcaattttgagctcatatttCGTTTTGTTCTCGATGAAC +AATTGATTGTCTTCAAAAACTAGTTGGTTGATATTTAGACGAGCTTCTTT +TTCGACTGAACGGAGAGCAGGCAGGTGTAAGGAGAGatggaatctggaaa +aatactTATTGAAAAGTTATCTGGAAAGTCCAGAAggcgtaaaacctgta +atagtagtacatctcggttcctgtcaaagatatcaaaaatttgtaaactg +ataaactgttccTGATTTCATGATAAACGATGGAGCTaaaaaaaaTAAGC +TCGCACTTTACGGACCTgcttttacaggttttacggtattttcagattta +gttttattttttaattctaaggACACcagaaacatctgaaaacaaCCAAC +TGAATAGCAATTTTGCCGTTTCCGAATTCATACTCACCGAAAACTAGGGT +TTTATGTACTTCAACAATTATTTAAAGGCACATTCTCCTAcagcaaaatt +caaatttaaaagttccaAGGTTTTCGGAAATAGTTGGTGATTCATTTTcg +aattaaaggtgggggacatgaaaaaaataatgttctcgaaatttcgttca +aagatgcccctcgaggagttgaacttgtgagggcgatgatttaGGGTGCG +Gcaccgctaccgactgagccacgcgggcaaagagaaagcggacgcccgat +tgcatacataaaggcgaggacgacagctgtacgcagagtgacattaagcg +ttgtacaccgtgacggtagaatgcatactctagctacaaaaaaactacga +aaaatctcctgtccccccacctttaaactcACCGAAAACTAGGGTCCATG +TATTCGAAGACATTCCGGCTCAACTCCTTTTGATGTTCTGAGAgcatttt +ttatctgaaataGAGATAATAGGACAGATGGGATATGGGAACACGAGCAA +TAAAACTAATACTAAGTGTAAGCCAGGCGCATTTTCCCTGCGTCtcaact +tatttttttctgacagCACGGTGTTCCGATCGGCGAAACCGGTTTAGTAC +ACGTGGCATCGCAGTGGAGCGCAGAAGCCCAAgcgaaaattcagtttttt +gttagtttttcaggtttaaTGAAGGGCCCGGCCCCGGATaggatttttac +aataaacaAGTTTTGCAGCaattgaaataccgtaaaatctgtatttgaAG +Ggccccctttaatagaagggccctccaagaggggtttggaaaaatagaag +ggccggcccttcttaatacaggttttacagtatgtttttttaaaaaaaat +gtttgaacacATAGAAAAATTATAACAGACGAACGAACGAGAGGGAAAAG +TatacaaaagaaaaatgaaaagatccCAAAGGAGGAACAAACTGAAACAA +CCATTCTGTTCGCCGCGAGTCTCTCTTTTATATGTTTCAGAAATGTGTGG +GAACCGTATGCGGGAGAGTTGTATATGAGAAAAGCGAATAAAATATCAAA +GGTGACAATTTAGAATATTAcaagaattgaaaagaaaGTCGTTCTTTTAT +AAAACAAGATGTTGAGGTTAATAAATGGACAAATGTGCAATGAACAATAA +ACTTTTATATCTTTCTCTTTCGCGTTCTTCTGTATCGAAATGATGTTCGA +CCACATCTACTGTGACAGTCAAAACGCATTTCcgaagaatttttcagatt +ttgagattttctggaaggttctcggaacatttttgatttctggagTTTAT +CCAATACTAATTTTTCTAACGCaaagatttattgaaaaaaaaaacatgac +tTTCAGCAAAGTTATTGGTCAGTTACAATGCAAAGGtaatttttcaagca +aaacgttttcaaaaagaaaaaaaaacaatgaaaaagaCAAATTAGAGAAC +ATATATCCAGAAACTCAACATCGTTATCGTTGGAGCACTTCCATTTTCAC +TGTCCAGATCATCTCTTGTCTGTTAATAGCACCATGGAGTATCGGATCAT +CATAGGAGACGGATAGTATGGAGGATGGACTCATTGGCATTTtgatgaat +ctgaaaatatcgttTTTAGAGAACAGGTAATTGCAAACACGCTCTAATCA +CAATGAGAAACTTCACTTACCTACGATCTGACTCTATCGCTTGAGCAATC +TCTCTTAGAGTAGATATGTATTCAGATAGTTCTGATCCTTTAACTCGAAA +AGACCAATTCGTCCCAACTGGCCTTCCATGTTCCAGCCAAAGATCCATTA +AATGAGCAAAAGTGTTTAGAGAAATCGGAGGGAGTCGAACTTTTAAGTTT +GAAATCTTCATGAGTAGATCCATGGAGTCATAATTATAAAGTTGCGAGCC +GACAACGAGATACTCCGCGTTTACAAGAATTTCGTGACCCAAGATTATAT +CGCTCGCTTCGAAACATAGCCAATTTAAGGGAAAACTTGTTTCGTGACAT +ATCGATCGAAGACCATCCAAATTGGCAAAACGATAGCAGCCaagcttcat +ttttcgaacaaaCGGCTTCACTCCAGACACAATCCAACGAAGGTTCTTAG +TTGATATAATCATGAAATCCACATCCCAAATAACAGTTCGATTCCCGAAC +ATCGTGGCTATTAGCATTTTCATTCCCtcataaatttttgtaatttttga +gcaTGTATAAATCCATGGCTCGCCATTTCCAGAGGTGATTGTCAATTTTA +AGGTAGACTTGGATTTAGGTATCCCCTTCTTTTCCAACGTCTTCATCGGA +ACCGATTCCTCATCTCTCAAACTCATTCTTCTTAATGTATTTACATCGTA +ATTACCGTGATTTTCCATACGAACATCTCCATTCATTATTATATCCTTTG +TAAATGTTTTAAATCCGAATTCATCCACATCATGATCTACGGGACCATTT +CTTCTCAACGGAGGTTTCTcgatttcatattttcgatCGATGCTGAGTTT +ATATGTAGTCTTGTTGACGGTGATTTCGTTATCATTCAAAATCagtttgt +tgattttcagatgagcTCTTTTTTCGACAGAACGGAGAGAAGGCAGGTGT +AGGGAGAGgtggaatctggaaaaaattaaataatttctattcagattttc +aagagattttcggaaatctGAGAACCCCAGAAACTTATAAGTTCCAAGGT +TTTCGGAAATAGTTGGTGACTCATTTTCGAATTgatttatctcgctttct +ctaaaagatataaaaaacttgaaaacaagtaaaatattggtgataatatg +ataaacattcttgcagttgacaattttttgatatctctcacaggagccga +gataattaaactcaaaATGCAAGCGTTTTTAGAGTGTAAATGCTGTGTTG +GAACACTACTTCtatgagcccaattatctcggttactgtaagagatatca +aacagctgtcaattgacaaaatattggtgataatataaccaacattttat +cagttagcccATTGTCAACATCTCTttcaagaaccgagataaatttggtc +aaaaatcggaaaatttctaacatttgagtgaaatttaaagattttatGTA +CTTCAACAATTATTTAAAGGCACATTCTCCTAtagcaaaattcaaattta +aaagttccaatcaaaaagttgttaattagtaaaatattggtgataatatg +ataagcatttttgtagttaataactttttgatatctcttacagtaaccga +gataaccAAGCTCAACGTACTTGCTCTTTCTAGAGTGTAAGTCACCGTAA +ATGCTTTAACGGAACACCACTTttatgagctcaattaactcggtttctgt +aagagatatcaaaaagctgtcaactgacaaaatattggtgataatataac +aaaaattttatcagttggctatttttttgatatctcttccaagaaccgag +ataaatttggtcaaaaatcgagaaattgcTAACATTTgagtgaaaattga +agattttatGTACTTCAACAATTATTTGAAGGCACATTCTCCTaaagcaa +aattcaaatttaaaagttccaAGGTTTTCGGAAATAGTTGGTGATTCATT +TTCGAATTAAACTCACCGAAAACTAGGGTCCATGTATTCGAAGACAGTCT +CCCTCAATTCCTTTAGTAGTAGTGGTTGTGACATTtatctgaaatgaaaa +taataaaatatgaTGAGAcatagaagaagagaaactaAGAATCGGAAGCT +CAACATTCACGAAAACTACTCAAAAGTGAAACAGTTAAAACTTGACAGAA +CTCACATCGATCACAGAATCTTCAGTTAAAAAATGGCGAAACCCATGAAA +TTGACATTGAGCCAATTGATCGgtgttcctgtgaaaggtatcaagaACTT +GTCATTTGGTCAAATGTTTTTGCTatcatgatgaacaatttatcagttga +gcacttttcGACACCTTTCACAAgtgccgagataaattggctcaaagtcc +TGGGTCCCAACAgatttcccagttttttctccttctttgtCCCTAGCCAA +AAGGTGATGACTGataagaaataattttttatcatcaGCCGTTTTTGTTT +CTCTGATGGACATTGTTATACGTGTCACCTTGTGATGCTCATGAAGGTGT +CAATTTAGAGAGCCTTGCAAAACTtgtgactgaaaatgaaactgGATGAG +CACcttcattttctgattttcttttctctagTGACTTCAACCTTtaatat +ctcggctcctgtgagagatatcaaaacattgtcaactgataaaatgttaa +tcacaatatcagaaatattttatcagttgacaaattgttgatatctctca +taggaactgagataattcagATCAAAGTGCAAGAATATCCTTAAATGTGC +AAGGCTCGCCTACAAGAAGAAATACAGTATCTTtgaccctaattatctcg +cttcttgAAAGAGtgacaaaaaagttatcaactaataaaatgtgagTTAT +AAAAATGcgcacattttattagttgattactttttgatatttttgctaga +aacagaaaacaataGAATTGAAGGAAGAGTTAGTTAGAGGGTTCCTACAC +CCTGTTaaatttaagccaatttttctcattttctgtaaaagatatcaaaa +agttgtaaattagtaaaatattggtgataatatgataaacatttttgtag +ttgacaactttttgatatctcttacagtaaccgagataaccAAGCTCAAC +GTACTTGCGCTTTCTAGAGTGTAAGTCACCGTAAATGCTTTAACGGAACA +CCACTTTTACGAGCTCAATTAAgtcggtttctgtaagagatatcaaaaat +cttaaaaaaaaatcaaaaatcggaaaatttctaacatttgagtgaaaatt +aaaagattttataaacttcaaaaaataattaaaggCATATTCTCCTACag +caaaattcttaaaattccatgcgcctttaaagttccttaaaagccaattt +ttccatttttttgcagcttcattaaaaaaaaagatttttatagAATCTac +gaatcgagaaaaatgggcTTCAAGaaactttaaaggctcatGGAACAGGA +GGAGAAAAACAATATTAAAACTTCAATGAATATCTCTCTCGCCTAAACAT +TCAATTCGAAATTCCCGCGCTTTTCTGTCCCCGTCTATCCATCCCTCCCC +acgaaaaatgtgtcaatatagagggaagagacgcagacacagtGCCTAGT +GTCTAACATAAATGATCGATAGAGAAAAGGCTATGTACGCGTTCTCATCG +CTTTCGAATCGCGACACTCTTTCCCTCTCACGCCTGCGTCCCTCACTCCG +CCACACTCTCTCTCCCTTTCTCTACTATAGAAAAGGAAGACGTGCCCGGC +CCGTCGTTTGCATCGCGGAGCCTTTAGCTCCACGGGGtcttctccttccg +ttcttttcctcctctctattccctacactctctcgctctctcactctctc +tctcttttcaattctttcccatcgtcatcatcaatttttcgttcGCTACT +CCATTTTTGGCGGATGAAAATGAGCCCATTTATTTTAAATTGCTCCGAGA +GCCTGAACTGCATCGTTTTCATTTAATTACGGTTGTGATGTTTGATTGGC +CCGAGATGGCTTAATTTTTGCGcgggagatttgaaaaaagaaaacgttcT +AATATGTAGGGTTTTTAGTGtctaatgtttcaaaatattggttgaatttt +tgaatttttaaaataatttctaaattctcaaatctctgtgaaataatatt +ttagaaaattgaaatttcggcaatttttaaaaacataattcaaaataaaa +actttgagaaaatttggaatctcGAGGTTCTGAACTTCTTGAAACCTtaa +attcttggattcttagttttttgaatgtttggatttttaaaattctgaaa +ttcccagtttctcaaattcttagacttgaagattctgagactctaggatt +ctaaaattttgaaattctgaaatctttagattaggaatttttcgaatttt +gcggGTTTTGGATActtggattttgagattctgacactttcgaattccga +aattctgaagtcctgGGAATCggaaattatgagattctgaactcCCTGAA +GCCAATTTTCATTAGCTTTGTCCTGTTTAGCTTaaaaacagatttcaaaa +atagatttttatcaaaaattcttaatttagaaaaccacaaaaatggaACC +AGATTTTCTACGAATTAagccctctctctctctctctctctctctctctc +tctctctctctttccaaTTAAACTATTTAGAAAAGGTCCTCTTTTGATCC +TAGAAGACATCTCACGTCATctcatctctctctctctctcatctGTTATG +TATTGGAtaagcgagagagagagagagagagagagagagagagagaggaa +ggtCAAATCGATAAGagaagataagaaaaaaggaggcAGGAAGATTTATG +TATGCATAATAGGACCTTTTCGTTATCAGCATTTTTATATTATTACTTAA +TTAGTGTAATTAAGCTAGCATTATCTATAGAGAAAACGTATACTAATTTC +TTcatgtggcctagttttccgtcagctggcctaactttgaaatttctcgg +TCATCCAAACAAATTCGTTAAAATCCGTTTCAGCTAAATAATTCCTTATC +TCTTTTTAATCTTCCCTCAACATGACTCATCttgttttcccttttttaaa +gcattttcccatttttagtattacttcttttttctgtttcgtcTGAATCA +TCGGAAAAAATTTACTGGCAAAAAATCCCtaaatttttatcactttattc +tttaagcttaattatcttgattccccTAAGAGATATCGAGCatctgtcaa +cagataaaatataggaaatattatcacaaactttttatcagttgactgaa +ttttgatatctctcataagaactgtgataactgagctcaaggTGGAAGCG +TTTTTAAGTTACCTTAATTGCTCATATCGTTTGTGCTTtaagcctaatta +tctcggttcctgtaatagatatcaaaaagttgtcaactgataaaatgttc +aggaaccgagatatctGGGATTCAAGAAGCGAGTACTAattaaaagttcc +aaaattcagaaaaatctaaacATCAAAACTCCCTTAAATCCATTTTGCTA +GTGTGTGTCCTCCTGAATAAATGATATCCTTTCCCTGCCCCTTTCTGTGA +GGGGGGTCggtttcattttctttcataTTCTTCCCCTTTTGCTTACCCTT +ATTCACAATCTCCATCTCTCCTATTCCTATTTCTTGATGTTCTTGTTTTA +TATATTCCGTTTCCCATTGAATGctcctctctccctctctctcttttcgt +TTTCTACGTCCCTTCTCTTACTCTTGATACTGACACCAACTAGTAGAAGA +CTTCTGATTCAtggacacacacacataccCAAAGACATTCGGGTGTTCCC +ACGGGAACACCCGGGACCGAATACTTTTGGCTTCTGGAGGGTCTTGACTT +ATTTTTTCCCTCGTTTTGttcttttaaaggcgcatagtattcttagattc +ttatattcttagattcttagattcttagattcttagattcttagattctt +agattcttagattcttaaattcttagattcttagattcttagattcttag +attcttagattcttagattcttagattcttagattcttagattcttagat +tcttagattcttagattcttagattcttagatccttagattcttagattc +ttagattcttagattcttagattcttagattcttagattcttagattctt +agattcttagattcttagattcttagattcttagattcttagattcttag +attcttagattcttagattcttagattcttagattcttagattcttagat +tcttagattcttagattcttagattctttttcttgtctcattattaCACA +AGAAATATTTAGTGCTGAGCAATCATCTTCTTAATTTCCCTCCCCCTGAA +CTACACGAAAATCCATTCATCACTTTGCGCTACTCAACAGCACCACACAT +TTCTTATTACAcaattaaaattgttttctctgtttctctaattctctcac +tctctctctatctttctTATTAGCTTCATATATCCGCTTTTCGTTATCCT +AGGCTCATGATGCCGCCGGCCGACTAGATTTGGTTACCTAGGCCGCTTGG +TTTTTTGgctataactttgttcaaaactGGTCTACGAAGATGGTTAATAT +ATCTATCGAtaggaaatcaaattttacacCGATTCAATAGAAGGTATATA +GATTTGAATGATTTCTAATTGAGTTATGTTCCATTATCCTAAAATCCTTT +CTAGACGCTTCTAGAGCCTAGAAGCACAAGTTTTGTGCCGTCTTTTTAAT +TTTGGCCTGTCtagacaattaaaaaaaacttcaaataactCATAgtcagt +tttcttttcttttcccaaTTCCCAgaacaaatcaatttttgccCCTAAGC +GAAACGCTAGGCAAACATGATTTATGGTAGTTTGTCTCTACTTTCTACCT +GTCAACCAATCTCACTTCCCTCTCTAGGTTTTACCCTTTTACGTTTCTTT +TTGAGATCTAAAAGCAACCTTATATGTTCCcacctagaaaaattcaatcc +aatttttaaatgaaaaagctaaaaaaaatgCGGTCGGCGGCACTGTTATC +ATTGTCCTCTGTAGCGAAGAAGAGTGTGCGGCAGCGTGACCGAGGGGTTA +ACTCTCCTGTTTTTGGATCTGTCACTGTCAAGTTCGATTCCACCGAGCGC +CGAACtctttccttccttttttgaattatttcttGAAGTAGTGTATTGAT +TCTAaagtcatttttaagcttGGAAAATTCCATTCCTTCTGTTTTTGTCT +CTAAAATCtcatgaattttagaaatttctatatttcccTCCCAGGCTGGG +AGTCCCATCCCGAAATTTCGCCCCCTCTGAAAGCATATACTATTGacatt +tttgcaaaaacgACGGGAAATATATAGATGAGGGGGTCCAAAAACGCATA +AAAATCTTAATATTGGCCTGAGTACGCCCATTTACGAGCAGCCTTCTCTC +AAGATATGGGTGGGGTAggcatttccattttttttgggacctcctTTCCC +TTTATTATATATGCCTCTACCCCCCgaaaatatatgaatatttagaaaat +tgcagaaaatattttttgaattttgaaatttttaagttttttcaaaagtt +tattttcaTTAGCGCGCCctttcagtttcaaaaccCCTTCCCTCCTGTAC +GCCTTTCATTCCCTTCTCTCTATGCGACCACGTGGCCTAGTGGCTAACAC +TGggacacaggtgattcgcgaacaatccgttacggattcTCCGCCATTCG +CGCGCGGATGAAATAGGAATCGTATACGGACCGTACGCGaatccgtaacg +gatccgtttgcaACTTATTTCTTGAAAACCCTCGCCgttccttttttcag +tttctattattttttgctgAGTTTTCATGATAGTTGTAGACTTAGTATTC +GGAACTTTATTAAGAAAACAtgtaaaagaaagaaaaaacatcagAAGTAC +AACATtaagttcgaaaatttatcaaaatccTGCTTTTTTGGGTTGTTGAA +CGATCATCCAAAACGAAAAGAGAcctgcaaaaaatttgggaatctgTCAG +AACATGTTGCATACTGATCACCTATTACTatccttttttgcttttgggtt +ccaatgtttttctcgtttttaacGTTGGAGTCGGCTTGCTCTATAGATGT +CCATTGCTGAAAAGTGTAAAAgtcattgtttcgaaaattagcGTATTTTA +TACTTACATTCGTCAAGCATCTTTCCTGGAGCAAACGCGAGCTTCGAATC +AAtcattttcggtcattttgctCACGAAAAGGTTAAGTGATACCTATAAC +AAACCCTATGTTTCGAAGAATTACCTTTGTGTTACAAACCTTTTTTCCTT +GTAGAAtcgtctttcgaaaaaaatcattgtcaaaaatcaacatatttgcc +aggaaaagtgagagaaaacggcaaaaaataggtttaaaatgcaaataaat +gatgcaaaaatATGATGGAAGAGGTAAGAGAAAGCTAGTCCCTCACAACT +CGGCCATCcggaacggatccgttcgcgaatgGGCATGGGAGCTATCTCTC +GACCTCCAGCTGCATCCGTATCcggtccgttcgcgaactccgtatcctgt +ccgattgcgaatgttgaaaaaacggatccgttcgcgaactcagaTTCGCG +AAACTGcccgtaacggacccgttaCGGATAGAAAACCTAGTGGACAccga +aacggatccgttcgcattttacctgtggaGTTTTTGATCCGACCCTCACG +AGTTCGACTCCTTGTTGCtcaaaatcttcttttttccttttttgattctt +tatCCCCAAAAGGGCGCATTTGGTCCCTTCCCTAATTTATAGCCTTGcac +tttttgggattttaattgaatttttctggagAGAACTCATCTAGTGAAAG +TGCTCGCGCTGATGAAGTAATCAGTGAAATGCGATAGGAAAACAGGGGCT +TCTCAAAAAACGTTCTTGTGTAGGTTTACGAGCCGCTCGTAAATCTACTA +GAGATTTTGTAAAGGTAAAGGGAATCATTCTGAGGAACAAGATATACCTT +ATGAGGTCCACCGTATACatgaagtttgagctcaattatctcggttcttt +tgaaagatatcataaagtaactaactgataaaatgtttatcagtttatca +ccaatattttatcagttggtcacttttttatatcttttactggagccgag +ataattgagttcaaagtgaaCTCAGCAGAACAATgggttctctgcgtctc +tcactgaTCCATCTCTCTACTCGAAATCCTACTAGTTTTGTATTATAATG +TGTCCAGTCGTCCGTTTCAAATCCATTCCGCTGACACCCATTTGTTTCCC +TTTTGGTCACACATTTTGTCCTTTTTCGGATAGACAGACATACAGACAGA +CatagaactttccagaaaacaCCACAATTAGCATAGCTAAGGGACCATGG +TTTTCCCCTTTTATGTAGATTCTCACACATTTCCTAGTCTTAATAATAAT +AAGCTCAGTCTCTCGCCTAGGagcttcctctctctctctctctctctctc +tctctctctctctctctctctctctctctctctctcactttctCCCTGGA +GAAAAGAGAgcttattattattttttcgtcgcttcttccatcatttattt +tcttattttcatttacgcttattactcctctcattttatttttactggag +aaatttatGTGAAAGGAtctagaacttgaaaaaaactgaaaaatcgttAG +TGGCACTTAAGGGACAAATGGGACAGtctcattatttttaaatgttatAG +TCTattttgtcttgaaatttccaacattctGAGCCCCATATCGACTACAT +TCTGgcagttttgattttcgaccataATTAGGCCGCGGCCCGGCCGGCTG +gcccaaaatcaaattttaaggaTTATAGCCAATATTatactcaaaatttc +acaaatttcataagaaaaaatataacgATATTTGTTTTCTCTGAACTAgg +ccacttttcaaaatcacagaattttagaattctaaaaattcggatttttt +gatttttgattttttttttctccaaaaagattaatatttttcttctctct +tgtctaacacaaaaagtgaaaaaatttcttcgaATGATATTGTTTTTTCT +GCCTGGGATTAGTGTCTTTACTTCTTTTTCCCCTGCCGTTTGACAAGAAG +AAACATGACAAATGAATTGGCattcagaagaagacgtgCGACAAGAAGAT +TCTAGTAGTAGAAGCATAAGAATTTGTTTCATTTGTAGAGTTTTGAGAAG +ACTAGGCTCTTGGTAAACTTCTCAAGCGCGCCCTATTGACAAAAGACACT +TAGGGGACACATGGGACAGTCTAAAGATTTTTGAGAGCtacattctattt +tttcataaaatttccaacatgtTGAGCCCCATATCGACTATATTCcaatc +aatttgaattttcagaaaattttggtgcaaatgcgctccaatgcgaaaag +tgaaaatttttgaaacccagccatgattatacttattttgtcAGCCTTAA +TTAGCTAAATTCAAATCTTCTAATTTTATTGCTCTAGGGGCAAAATAAAT +CGAGTTATGACAGTTTTAGTGGTGCTTaatttttgctaaaaaccGTCGCT +GACACTTAGGGGACAAATGGGACAGCGTAAAATTGTTCGAGAACTATAAT +CCAATTTGACCttaaatttccaacattttgagccccatattgaaTATATT +CCggatatttttaattttcaggtgcAAATGCGCCCTAatgataaaaaaat +caaactttaaGGAATATAATACTAAAATATaatactcaaaatttcagaaa +ttttgtgagaaaaacAAATCTGTTGTTTGCTTTTTCAGATAACTCGCTCT +ATTgcactttttttctcaatttccgttTTAATCTAACTAACGTTTCCCTG +CCTTATTCATTCGACACACATCAAGTTGGATGGAGCAAGTAAATTAATTC +TCTCCCTCCCTTCGCTTTCCCACGTTTCCCTATTtctctaagtctcgctt +ctctttctatactctctcgatgtctctgcgtctctcctccctctttctct +ctctctcttcctaaTTAATTCGACACGAGACACGAAGCACCCAATAATCC +ATGTCCTTTTTCCGTCCCGTTTCTATTGGTCCCCTCCCCATTTTCATCAT +TGAACAggcagaattttcaggatgtCATCTGAAGCGGCTCCAGAAGTCGT +CTCGGCCACGGCCTCCACAACGCCGGCGGCCACGCCCCCTTtgaaagggg +gcggagttCCGGGTGGTGGTGGGACAACTACCGCCGAGCAACAACAAGTC +CCATTTCGTAtgtttcatttctttctttttttaaaacaaaattttcaata +gtttatcgATAAAATGGACGGTTTCACTTCAAGCGCGCTCcaattgcaaa +ttttgagaaaaatgaaaatttttgaaacctagtcatgattatactcattt +tatcTGTCTTATTTTGCAGATTCCGAAACTAAAACTTTCATTGTTCCAAG +TGCAAAATTAACCGAGTTATTGtaattttaatgaatttaGTCAAATGCGC +CACATTGctaaaaattaacatttttggaactaaaccatgtttatactcat +tttatcTGACTTGTTTTGCTGATGACACAACTGTTTTCATTGCTCTAGCT +GCAAAATGAACCGAGTTATAACAGTTTTAATAAATATGACCAAGTGCGCC +CCATTGAGAAAAACTAGAATACCAGAACCCAGTCTTGATTATACTCTTTT +TATCTGGCTTGTTTTGATTATTCCAAAACTAACAAGTAAAATTTTCTAGG +TGTAAAATGAACCGAGTTATGGCAGTTTTAAGAAACTTGACAAtatgcgc +tccattgacaaCAGACACTTAGGGGACAAATGGGACAATTTCTGATAACA +TATTCAACTTTGTcatgaaatttccggaaatttgaGCTCCATATCGACCT +TATTCcgattattttgattttcgatcaAATTTACCAGTAGGCCGCGGCCT +CGGCTGcggtcgaaattcaaatttcaaggaaTATAGGCAATATTatactc +aaaatttcacaaattttatgGAGAATCCGAATCTAGCACTTATTTTTGAT +AGATATGCgtctatagaaaaaaaaattgcggggttttccaaaaatccgca +aaaaaatttcgtgaCTGTAAACAAGAGATACATATTTTCAGGAGGCTTCT +CACTGGAATATTAAAAATTCCTCGTTTTTGGTCTGTaaaaannnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnTCTCTC +TCTTTCCAATTAAACTATTTAGAAAAGGTCCTCTTTTGATCCTAGAAGAC +ATCTCACGTCATctcatctctctctctctctcatctGTTATGTATTGGAt +aagcgagagagagagagagagagagagagagagagagagagaaggtcAAA +TCGATAAGagaagataagaaaaaaggaggcAGGAAGATTTATGTATGCAT +AATAGGACCTTTTCGTTATCAGCATTTTTATATTATTACTTAATTAGTGT +AATTAAGCTAGCATTATCTATAGAGAAAACGTATACTAATTTCTTcatgt +ggcctagttttccgtcagctggcctaactttgaaatttctcggTCATCCA +AACAAATTCGTTAAAATCCGTTTCAGCTAAATAATTCCTTATCTCTTTTT +AATCTTCCCTCAACATGACTCATCttgttttcccttttttaaagcatttt +cccatttttagtattacttcttttttctgtttcgtcTGAATCATCGGAAA +AAATTTACTGGCAAAAAATCCCtaaatttttatcactttattctttaagc +ttaattatcttgattccccTAAGAGATATCGAGCatctgtcaacagataa +aatataggaaatattatcacaaactttttatcagttgactgaattttgat +atctctcataagaactgtgataactgagctcaaggTGGAAGCGTTTTTAA +GTTACCTTAATTGCTCATATCGTTTGTGCTTtaagcctaattatctcggt +tcctgtaatagatatcaaaaagttgtcaactgataaaatgttcaggaacc +gagatatctGGGATTCAAGAAGCGAGTACTAattaaaagttccaaaattc +agaaaaatctaaacATCAAAACTCCCTTAAATCCATTTTGCTAGTGTGTG +TCCTCCTGAATAAATGATATCCTTTCCCTGCCCCTTTCTGTGAGGGGGGT +CggtttcattttctttcataTTCTTCCCCTTTTGCTTACCCTTATTCACA +ATCTCCATCTCTCCTATTCCTATTTCTTGATGTTTTGTTTTATATATTCC +GTTTCCGATTGAATGctcctctctccctctctctcttttcgtTTTCTACG +TCCCTTCTCTTACTCTTGATACTGACACCAACTAGTAGAAGACTTCTGAT +TCAtggacacacacacataccCAAAGACATTCGGGTGTTCCCACGGGAAC +ACCCGGGACCGAATACTTTTGGCTTCTGGAGGGTCTTGACTTATTTTTTC +CCTCGTTTTGttcttttaaaggcgcatagtattcttagattcttagattc +ttagattcttagattcttagattcttagattcttagattcttagattctt +agattcttagattcttagattcttagattcttagattcttagattcttag +attcttagattcttagattcttagattcttagattcttagattcttagat +tcttagattcttagattcttagattcttagattcttagattcttagattc +ttagattcttagattcgtagattcttagattcttagattcttagattctt +agattcttagattcttagattcttagattctttttcttgtctcattatta +CACAAGAAGTATTTGGTGCTGAGCAATCATCTTCTTAATTTCCCTCCCCC +TGAACTACACGAAAATCCATTCATCACTTTGCGCTACTCAACAGCACCAC +ACATTTCTTATTACAcaattaaaattgttttctctgtttctctaattctc +tcactctctctctatctttctTATTAGCTTCATATATCCGCTTTTCGTTA +TCCTAGGCTCATGATGCCGCCGGCCGACTAGATTTGGTTACCTAGGCCGC +TTGGTTTTTTGgctataactttgttcaaaactGGTCTACGAAGATGGTTA +ATATATCTATCGAtaggaaatcaaattttacacCGATTCAATAGAAGGTA +TATAGATTTGAATGATTTCTAATTGAGTTATGTTCCATTATCCTAAAATC +CTTTCTAGACGCTTCTAGAGCCTAGAAGCACAAGTTTTGTGCCGTCTTTT +TAATTTTGGCCTGTCtagacaattaaaaaaaacttcaaataactCATAgt +cagttttcttttcttttcccaaTTCCCAgaacaaatcaatttttgccCCT +AAGCGAAACGCTAGGCAAACATGATTTATGGTAGTTTGTCTCTACTTTCT +ACCTGTCAACCAATCTCACTTCCCTCTCTAGGTTTTACCCTTTTACGTTT +CTTTTTGAGATCTAAAAGCAACCTTATATGTTCCcacctagaaaaattca +atccaatttttaaatgaaaaagctaaaaaaaatgCGGTCGGCGGCACTGT +TATCATTGTCCTCTGTAGCGAAGAAGAGTGTGCGGCAGCGTGACCGAGGG +GTTAACTCTCCTGTTTTTGGATCTGTCACTGTCAAGTTCGATTCCACCGA +GCGCCGAACtctttccttccttttttgaattatttcttGAAGTAGTGTAT +TGATTCTAaagtcatttttaagcttGGAAAATTCCATTCCTTCTGTTTTT +GTCTCTAAAATCtcatgaattttagaaatttctatatttcccTCCCAGGC +TGGGAGTCCCATCCCGAAATTTCGCCCCCTCTGAAAGCATATACTATTGa +catttttgcaaaaacgACGGGAAATATATAGATGAGGGGGTCCAAAAACG +CATAAAAATCTTAATATTGGCCTGAGTACGCCCATTTACGAGCAGCCTTC +TCTCAAGATATGGGTGGGGTAggcatttccattttttttgggacctcctT +TCCCTTTATTATATATGCCTCTACCCCCCgaaaatatatgaatatttaga +aaattgcagaaaatattttttgaattttgaaatttttaagttttttcaaa +agtttattttcaTTAGCGCGCCctttcagtttcaaaaccCCTTCCCTCCT +GTACGCCTTTCATTCCCTTCTCTCTATGCGACCACGTGGCCTAGTGGCTA +ACACTGggacacaggtgattcgcgaacaatccgttacggattcTCCGCCA +TTCGCGCGCGGATGAAATAGGAATCGTATACGGACCGTACGCGaatccgt +aacggatccgtttgcaACTTATTTCTTGAAAACCCTCGCCgttccttttt +tcagtttctattattttttgctgAGTTTTCATGATAGTTGTAGACTTAGT +ATTCGGAACTTTATTAAGAAAACAtgtaaaagaaagaaaaaacatcagAA +GTACAACATtaagttcgaaaatttatcaaaatccTGCTTTTTTGGGTTGT +TGAACGATCATCCAAAACGAAAAGAGAcctgcaaaaaatttgggaatctg +TCAGAACATGTTGCATACTGATCACCTATTACTatccttttttgcttttg +ggttccaatgtttttctcgtttttaacGTTGGAGTCGGCTTGCTCTATAG +ATGTCCATTGCTGAAAAGTGTAAAAgtcattgtttcgaaaattagcGTAT +TTTATACTTACATTCGTCAAGCATCTTTCCTGGAGCAAACGCGAGCTTCG +AATCAAtcattttcggtcattttgctCACGAAAAGGTTAAGTGATACCTA +TAACAAACCCTATGTTTCGAAGAATTACCTTTGTGTTACAAACCTTTTTT +CCTTGTAGAAtcgtctttcgaaaaaaatcattgtcaaaaatcaacatatt +tgccaggaaaagtgagagaaaacggcaaaaaataggtttaaaatgcaaat +aaatgatgcaaaaatATGATGGAAGAGGTAAGAGAAAGCTAGTCCCTCAC +AACTCGGCCATCcggaacggatccgttcgcgaatgGGCATGGGAGCTATC +TCTCGACCTCCAGCTGCATCCGTATCcggtccgttcgcgaactccgtatc +ctgtccgattgcgaatgttgaaaaaacggatccgttcgcgaactcagaTT +CGCGAAACTGcccgtaacggacccgttaCGGATAGAAAACCTAGTGGACA +ccgaaacggatccgttcgcattttacctgtggaGTTTTTGATCCGACCCT +CACGAGTTCGACTCCTTGTTGCtcaaaatcttcttttttccttttttgat +tctttatCCCCAAAAGGGCGCATTTGGTCCCTTCCCTAATTTATAGCCTT +GcactttttgggattttaattgaatttttctggagAGAACTCATCTAGTG +AAAGTGCTCGCGCTGATGAAGTAATCAGTGAAATGCGATAGGAAAACAGG +GGCTTCTCAAAAAACGTTCTTGTGTAGGTTTACGAGCCGCTCGTAAATCT +ACTAGAGATTTTGTAAAGGTAAAGGGAATCATTCTGAGGAACAAGATATA +CCTTATGAGGTCCACCGTATACatgaagtttgagctcaattatctcggtt +cttttgaaagatatcataaagtaactaactgataaaatgtttatcagttt +atcaccaatattttatcagttggtcacttttttatatcttttactggagc +cgagataattgagttcaaagtgaaCTCAGCAGAACAATgggttctctgcg +tctctcactgaTCCATCTCTCTACTCGAAATCCTACTAGTTTTGTATTAT +AATGTGTCCAGTCGTCCGTTTCAAATCCATTCCGCTGACACCCATTTGTT +TCCCTTTTGGTCACACATTTTGTCCTTTTTCGGATAGACAGACATACAGA +CAGACatagaactttccagaaaacaCCACAATTAGCATAGCTAAGGGACC +ATGGTTTTCCCCTTTTATGTAGATTCTCACACATTTCCTAGTCTTAATAA +TAATAAGCTCAGTCTCTCGCCTAGGagcttcctctctctctctctctctc +tctctctctctctctctctctctctctctctctctctctcactttctCCC +TGGAGAAAAGAGAgcttattattattttttcgtcgcttcttccatcattt +attttcttattttcatttacgcttattactcctctcattttatttttact +ggagaaatttatGTGAAAGGAtctagaacttgaaaaaaactgaaaaatcg +ttAGTGGCACTTAAGGGACAAATGGGACAGtctcattatttttaaatgtt +atAGTCTattttgtcttgaaatttccaacattctGAGCCCCATATCGACT +ACATTCTGgcagttttgattttcgaccataATTAGGCCGCGGCCCGGCCG +GCTGgcccaaaatcaaattttaaggaTTATAGCCAATATTatactcaaaa +tttcacaaatttcataagaaaaaatataacgATATTTGTTTTCTCTGAAC +TAggccacttttcaaaatcacagaattttagaattctaaaaattcggatt +ttttgatttttgattttttttttctccaaaaagattaatatttttcttct +ctcttgtctaacacaaaaagtgaaaaaatttcttcgaATGATATTGTTTT +TTCTGCCTGGGATTAGTGTCTTTACTTCTTTTTCCCCTGCCGTTTGACAA +GAAGAAACATGACAAATGAATTGGCattcagaagaagacgtgCGACAAGA +AGATTCTAGTAGTAGAAGCATAAGAATTTGTTTCATTTGTAGAGTTTTGA +GAAGACTAGGCTCTTGGTAAACTTCTCAAGCGCGCCCTATTGACAAAAGA +CACTTAGGGGACACATGGGACAGTCTAAAGATTTTTGAGAGCtacattct +attttttcataaaatttccaacatgtTGAGCCCCATATCGACTATATTCc +aatcaatttgaattttcagaaaattttggtgcaaatgcgctccaatgcga +aaagtgaaaatttttgaaacccagccatgattatacttattttgtcAGCC +TTAATTAGCTAAATTCAAATCTTCTAATTTTATTGCTCTAGGGGCAAAAT +AAATCGAGTTATGACAGTTTTAGTGGTGCTTaatttttgctaaaaaccGT +CGCTGACACTTAGGGGACAAATGGGACAGCGTAAAATTGTTCGAGAACTA +TAATCCAATTTGACCttaaatttccaacattttgagccccatattgaaTA +TATTCCggatatttttaattttcaggtgcAAATGCGCCCTAatgataaaa +aaatcaaactttaaGGAATATAATACTAAAATATaatactcaaaatttca +gaaattttgtgagaaaaacAAATCTGTTGTTTGCTTTTTCAGATAACTCG +CTCTATTgcactttttttctcaatttccgttTTAATCTAACTAACGTTTC +CCTGCCTTATTCATTCGACACACATCAAGTTGGATGGAGCAAGTAAATTA +ATTCTCTCCCTCCCTTCGCTTTCCCACGTTTCCCTATTtctctaagtctc +gcttctctttctatactctctcgatgtctctgcgtctctcctccctcttt +ctctctctctcttcctaaTTAATTCGACACGAGACACGAAGCACCCAATA +ATCCATGTCCTTTTTCCGTCCCGTTTCTATTGGTCCCCTCCCCATTTTCA +TCATTGAACAggcagaattttcaggatgtCATCTGAAGCGGCTCCAGAAG +TCGTCTCGGCCACGGCCTCCACAACGCCGGCGGCCACGCCCCCTTtgaaa +gggggcggagttCCGGGTGGTGGTGGGACAACTACCGCCGAGCAACAACA +AGTCCCATTTCGTAtgtttcatttctttctttttttaaaacaaaattttc +aatagtttatcgATAAAATGGACGGTTTCACTTCAAGCGCGCTCcaattg +caaattttgagaaaaatgaaaatttttgaaacctagtcatgattatactc +attttatcTGTCTTATTTTGCAGATTCCGAAACTAAAACTTTCATTGTTC +CAAGTGCAAAATTAACCGAGTTATTGtaattttaatgaatttaGTCAAAT +GCGCCACATTGctaaaaattaacatttttggaactaaaccatgtttatac +tcattttatcTGACTTGTTTTGCTGATGACACAACTGTTTTCATTGCTCT +AGCTGCAAAATGAACCGAGTTATAACAGTTTTAATAAATATGACCAAGTG +CGCCCCATTGAGAAAAACTAGAATACCAGAACCCAGTCTTGATTATACTC +TTTTTATCTGGCTTGTTTTGATTATTCCAAAACTAACAAGTAAAATTTTC +TAGGTGTAAAATGAACCGAGTTATGGCAGTTTTAAGAAACTTGACAAtat +gcgctccattgacaaCAGACACTTAGGGGACAAATGGGACAATTTCTGAT +AACATATTCAACTTTGTcatgaaatttccggaaatttgaGCTCCATATCG +ACCTTATTCcgattattttgattttcgatcaAATTTACCAGTAGGCCGCG +GCCTCGGCTGcggtcgaaattcaaatttcaaggaaTATAGGCAATATTat +actcaaaatttcacaaattttatgGAGAATCCGAATCTAGCACTTATTTT +TGATAGATATGCgtctatagaaaaaaaaattgcggggttttccaaaaatc +cgcaaaaaaatttcgtgaCTGTAAACAAGAGATACATATTTTCAGGAGGC +TTCTCACTGGAATATTAAAAATTCCTCGTTTTTGGTCTGTAAAAAAGCCA +AGAAAGACGATGTCCTCTATTATTTTCTTGTCTTGTTTTCTCCTCCCATC +GTCCCCAACAACAAAGAAATAGGCTATTATTTATGTCGCCTAACCTAGTC +GCCTCCTGTCACCACGGAAGAGAAAAtagacatttttgtttttcagggGA +AAAATTATGGAATCTTCTGagtcatcaattttttctctgttttgaaattg +taattGCCTATTTTCTTGATTAAATACAAtaattttctataaattttgtg +aaatttagaGTATAATCTTGCCTATATTCctcgaaattcgattttaggCC +TGGCAGGCCAAAAATCCGCGGCTTAAAGAGACAAAAAGTGCAATAGAGCG +AGTTTCGGCTAGCTCCGGGAAAAACAAACACCAGATTCgtttttctcaca +aaatttctgaagttttgagtaTAATAATGTCCATATTCCTTGAACTTTGA +TGTTTTCATCATTAGGGCGCATTTgcacctgaaaattcaaactaatCGGA +ATATagtcaatatggggctcaaaatgtgGGGAATTTTATGACAAATTGGC +TTATAgaattcaaacaatttcaCGCTGTCCCATTTGTCCCCTAAGTGTCA +GCGACGATTTTTACCAAAAATTAATCGCCATTAAAACTGTCATAACTCGA +TTTATTTTGCCCCTAgagcaataaaatttgaagatttgaattcAGCTAAT +CAAGACTGACAGAATAGATATAATCATAGCtgggtttcaaaaaactttca +tttttcgcattggagcgcatttgcaccAAAATTTTCTGCAAATTCAAACT +GATCGGAATATAGGCAATATGGAgctaaaaactttgaaaattccatgaCA +AAGTTGAATATGGTATCAGTAACTGTCCCATTGTCCCCGAAGTGTCTGTT +GTCAATGGGGCGCACTTGCtcatttttattaaaactGTCGTAACTTGGTT +CATTTGGCAGCTAAAGCAATAAATCTTTCAGTTTTGGAATTAGCAAAACA +AGCCAGTTAGAATAAATATAATCATGACTGGGTTCTAGGGATTTTAGTTT +TTCGCAATGGGGCGCACTTGTTAATATTTAGTAAAATAACTCGGTTCATT +TTTCACCTAGAGCCATAAAATTTGCAGTTTGGAAATCAGCAGAACAATTC +AATTCAGATGAATTTAATTATGACAAGCCATTGAAAAGTTTAGTTTTTAt +cagtagagcgcacttactcaaaatttttttaacttaaCTCGGTTTATTTT +GCATCTAGATCAATGAAATTGTCAGTCTTGAAATCAGCAAACTAAGCCGG +TTAAAATGATCATAATAATGACTAGGTTGCAAACGTTGAGATTTTCGCAA +TGGGGCGCACTTGCTCAATTTTATTAAAACTGtcataactcggttcattT +TGCACCTAGAGCATTAAAATTTGCAGTTTTGGAATCAGTAAAACAAGCCA +ATGCATAATATGCTCAATAGAGGTTTTTCACTACttcaacttcaattttt +tcagaaatgattCCACCCGGTCACTTCTTCGCGAATCCATTCATGAATCC +ATATGTGCCATCAGGTGTCCCACAGCAACAAGAAACAGGAGGAGAACCAA +GTGGCGTGCCTCAACCGAATCAAATGGTTTTCTCGCCGGCACAATATCAA +GAAGTCATGCATCActattttcaacaaatgatGGCCGCCAGTGGAGCGCA +GTTTCCCATTCCCTTCCCTATGCAATTTCAACCGGCGTTCCAACAGCCGC +GACCTAGTTCTCAGGTTTGTAGCGCAAACGTTAGAGGagctatgaaaatt +ttgtaaattgtaaactgtgtaaaattttatgctctttaatttgttagttg +acaaaaatttgattggagctttagtttttgagatgcaAGCTTTTGAAGAG +GACTAGCTACAAGCTCTTGTAGGAAACATTTCTTTGAACACCGATATCTC +AATAACGAAAACAGTTACCAAGATTTTGTAAATTCAAGAAATGTgttaaa +ttttaaaagctaTATTTTACTAGTTTAAATTTTTGCCTAGCTCTATTGGT +TTTTGATATAAACTCGTTTAAAGTCAAGTGACATTATGGTGCCTTGGTTG +AGCCACCTGAACGGCGCTCTACTTTCAAACGcctttatctcaaaaaccaa +catAGTTAGGTAAAATTTactaactaaaaaaatgtgtaaaatgttGTCCT +Ctccattttattagttgacaaatttttgctagaaattttagtttttgaga +tacaagcTTTTGAAGAGTAttagttccaaaaattggtGTATTTGCGGTGA +AGCTTCTTTGAACACCGATATCTCAATAACAAAAACAGttaccaaaattt +tgtaaattaaaaaaatgtatgaaattttataagctaTATTTTACTAGTTT +AAATTTTTGCCTAGCTCTATTGGTTTTTGATATAAACTCGTTTAAAGTCA +AGTGACATTATGGTGCCTTGGTTGAGCCACCTGAACGGCGCTCTACTTTC +AAACGcctttatctcaaaaaccaacatAGTTAGGTAAAATTTACTAACca +agaaaatgtgtaaaatgttGTCCTctccattttatcagttgacaaatctt +tgctagaaattttagtttttgagatacatgCTTTTGAAGAGCACTAGTGT +TACAAAATTTGATGTCTAGAATTATTTGGTTCCTCTTAGTCATCCTTTTT +CGCAGTTTCAACATGATCTATTTCGAAATGTCTATTTCTTACGACGTGCA +ACaaaaaaaGGGACATAAAACATTTCGCAACATGCTCCAAGAGTCCTTGC +TCCCAGACATCAGGACAAACCGACAAACAGGCAGAAGAGTATACGAAACC +ACTTGATGATCTCAAGAGACATGTGTCCCTGTcccattttctattttgtc +AGTCATCCCCTCAAATGACATTCATCTGTGCTCTCCTCTTTTCATAAATA +CATATAAGTATCCTCCTATTCTTCAGGCATCATCATCTCATCGCTCAGAA +GACGATAATGGAAGGCAGACAGCTGGCAGTATGGGTTGTGTCTCGTCGAA +TGTATCACCAAATCATCGAGAGGTTCGTTTAGAAACACTtgagaatgaat +aaaaaagaagaaaaaacaatattgcctctagtggggttcgaaccccagtG +GGCTAAATTGAGGGCGGCAGCGTTAGCCAGTAAGCCACGAAGGCGCGCAG +GAATTATAGGTGTGCCGGGCGATTAGAAGGGAGCGGGTCGCCAAACTTTT +TGAGGAGCGTCGGGGGGCCATCGTGAGGTGTAAGATAGTTCGCTCCAATG +AGAACAACTGATTTTTAAGGGAATTTAGTGTCTTAAACCGTTCAAAGACG +AATTTAAATCGAAATTAAgttattttcagacgaaacCAGAGGGATCTAAG +GAAGCATCAACGTCGCCAGAAACTCAGGAGCCTGTTGTTTCAAATTCCGC +TGGTGagtattttttctagtttttgatatatgttCCCCCTTCTCTAGTAC +TCCTCTAGTGTCCCAAAAGAAGGCATCCTGAAACACAAGATCATCTCCCT +CTCTCGCCTTTTGGGCGGCTCAACGAGagattttctctgcgtctcttgtc +atttcactctctctctctcgcacATCTCGTTTGTCGATTCTCCACTAATT +GTTCGATGTGTGTATGTATTTGTGGAACGAGAGAAGATGTTGAGTTGTAT +CTGAGCAGTCAGACTACTTTTTGAGGGACCTGAAAAGCCAACTTTAGAGC +TAATTTGCActaaaaaagtattaaaatttgagaaaaatgagataaaaacC +CGAAATTCCTTCATAAACcactatgaaaaaaaaacgaagacaAAAAACCG +CCGGTGGGTGTCGAACCCGAGACTTCAAAACTAAAGAGCAGCAGTTTTAG +CCACTAGGCTACGCGCGCACAGTTTAACTGTGAGAGCGAAGGTAATTAGG +AAGGTGGTGGTCGGCGCAACTTTTCAAAGGAAAGCGAGcggccaaatttt +ttttgtaatgtaAAATTAGGAAATATGAAAACGCGCTCTGCTGACAATTA +CTAAAAATCAAGCGctaaattagaaatttttcagaagacacCCTGGTTCC +AACCTCCTCTGACGTTGCCGTTGGCGACGTCATTTCGGGAGAAGCGTCTC +CAgaatccacgtcatctggagaagagaaatcggaggagaagagaaagaat +tCCGGCGATCGAACGGATTCGTTGATTCGGAAACAGATGAGCGAAATGGA +AAAGGAGATTACAAGGAGGTCGCAAAACAAGAATATTAAGAAGGTAAGTG +TAGAATTAGTGTGAGGAAGAGGGGCTCTACAACTTAAAATAGATGtacct +caaaaactagaggagctagcaaaattttgctaacaaacaaaataaaaagc +gtaaaattttatacatttttgcagttgaacacttttttctatctatttta +gtttttcagatacaaGCGTTTGAAGTTAAAAGATGGTGTCTTAACTTTAA +GCGgatgtatctcaaaaactgtaattgcgagcaaatttttgtcaactaac +aaaataaagagcgtaaaattttacacattttcttagttgacaactttttt +ttaaaagttttagtttttgagatacgccGCTTGGAAGTTTAAGCGCTCAC +TTTCCACTCTCAATACGCCAAAAagcgagacactctagcttctctgcgtc +tctttttccTGGAAGCTGAGCTAGAGATTTCAAGAACAATCCACTATTTC +CTCCCCTAACCCCGTCCCCTTTCTCTCCCATAATCTAATAACGGtcccat +tgctcattttctctggtccctcttcaaaaattctgctctctctctcttct +tcagATGGTCCCTTTCCATTCCcaaatccacaaaaaaaagCACACACttt +ttcactcacttctttttttttcccccgattttctcctattttttcactac +tttttcaaaaaaaaatgggaattaaaaaaaaatttttgagttcaattttt +agctattaGCACTAACCATCACTTTAGAGGACTAATTGAGGAAGGTTTAA +AAGTACCATAACTTGGATAAACTCAGTCGGAAACTCATCAtacttatatt +tttcaattcagcaagtcaatagcttcaaaatgagtataaacaagccttTG +TTTAGAAAagtgggtcccaccacgaaaactacagtagccctGAAACTCTT +CCGAAACtagccgtgtttatactcattttgaagcgtaTGAACAagccaat +tcaaaaaatttaaattcattCGAATTGAGCCAAAACCAACGGAGATATTG +AACAAATTTGAATGTTcgctgattttcaaaattcaaatgacCATAACTCG +CTTGGTTTTGACTCAATTTGAGTGAAACTTATTTTGTTTGATTTGGCTGa +ttgtgagcttcaaaatgagtataaacacgcctaTGTTTAGAAAAATGGGT +CCCACcgcgaaaactacagtaacccgctAACTCTTTCAAAGCTAGctttg +tttatactcattttgaagcgcttgacaagacaaatttaaaaatatcagTT +TCAAGCGAAAACGATCAAAATGGTTTGAGTTATGgacattaaaaaatgtt +taaggTCTAACAAATCGCTTATGggtttttcgggaaaattttcggtttct +taACCCCTTCTACACTACCCCTATCTATGCAGATCGACGACGACAGTTTG +GCGGAGCTCATCGGAGCATCGAAACCCATCACTCAACTAACCGCCGACGA +TTTTTCACCGTTCTCCGAGAAGGTACCACATCTCGTTGTGATTGTCTCAA +TTGTGCTCCATCGCGAATGTTTATCAATGGAGCACAAGTTCCCGCATGTA +CTAACCCGTTGTGCACCTTTTAACTTATTATCTATGAAGCGAGTTTTCCT +ATTCTATTGCTGTTGTTGTgcgattttcgaaacaaaaattgaattcccgC +CTGATCAACACTTTTCCAGAGCGGTCTACCGTATACCGCACCTGCTGTGG +TTGAGAAGCCAGCTCCAAAGGAGTCTCTAAACCAACTACGCTCATCTTTC +AATCTTCCCGAAGATTCTCCGACGGTTGGACCCGTCGGACCCGCTCAGAA +TACGGTACCTGCGGCCCAATTCGCTAATAACTCGATGTTCGCCGGGAATT +TTGCGCAGAACGCAGCATTTGTACAGGTAtttaaaaatatcgatttttga +agaattggcgagaaatttgaatttttcagatgactcCACAAGCTCCATTC +GGCACCACGCCAGGCTTCCAAATGATGCAACCACACCAACATACTCTTTT +CATGCAACAACCTAATCGtaagtgatttttcaaatgcgctctattgcgAA +TTTGTCAAGTGGAGCGCGGTTGCTCGAAGtgtttttgtgaaattcttact +aaaatcgatttttgtcgattttcgcTTATTTTTGCTGATAAAAGTAGTTC +TTTTCATGCTGAAATACACATTTtaagtggaaaaaaaaatcaaaaaatcg +ataattttttgaaaatttccgattttctctctcatttttctaccGAACAA +ACTGTTTTTGCactcaaaatgtttgtcttagCCTGATGAACACCAGTTTG +Tagcaaaaagttcgaaaaaacgacgaaaatttctgaaaattggctgaaaa +ccaagtttccaaacaaaaaaattttgtcgcCGCCGCAACCGCGCTCCACT +GAACTGGCGAGAGTGAGTGGTGTTCTATACGCCTTGTAGACATAGTTCCT +TCGTGTGCGTCGTGGTGTAGTGGTTAGCGGCATTGCTTTAGGAACAGTGG +CTCGCGGGTTCGAATCCAACCagggaaaaaatgttttttttgctcttttt +gaagGCGTTAGGAAAGTTTAAAGcattcagaaatgttttttttttcaaaa +ttctaaggtctcgagaaaattttgtcagGGGACCAAAACAAATGCTTTTA +TTCATTAAAtaaaagtttattttcagctaCCTTCGTGAACAACGGGACCA +ATCCATTCCTCCAACAACAACCACCCCTTCCCAACTTCGCCAACGGCACC +ACTCCACTTGTACCTACAGTAACCGCTCAGCAATTCACTCCTGAGCAACT +GGCCGCCGCATTTGCTCAACAACAAATTGCTCAGCAGCACATGCAAGCAG +CCCCAACACCATTCGATGCTCCGCCCCCttcgacgtcatctggaaCCACC +ACGTCATCAACTGGTCTGGcaccacctccacctccaTCTCATCCGATTCA +TCGAAAGAACTCGGGTTCTGGATGGCCTGAAGAGAATAAGGAGAACGGGA +CGTCGAATACTAGCACGACGTCTAACGGAACACCTAATGGAGCACCatcc +acgtcatctggaaCCGAGGATCCAGTATGGGTTCTTCGGGATTCGTACCT +CAAAAAGATGCAGAGGGAGCAAAGGACTTCCGAGGAGGAGGTATGTTGTA +TATGCCTTGCGGCATAGGGAGAAAGTGTGTAGCGCGTGGTGTGGTGACTA +GCGACTTTGGTTTTGAAGCCATGGCtcgcgggttcgaaccccatcGAGGA +CCATAAggtccaaactttttttttgcttctttttagAATAGTTATCCATA +GTAGAGTTCTTTCTACAAACCAAATCTCaagtttttacaatttttcgagt +ttggaactctttttaaaaaagttccaaacTTATGATAATGCGATATAAAA +ATGAGTGTAGTATATaaagaaaagcagaaaaaggaaataatgACACCCTT +GAGAAAAAGATGTATAGCTGGAGAAAAATCTGAGGAAAGCCGGAGAAATA +TTTCCggaacttttttccttttttggattttttgaaaatttggaatcttt +atttttaaaggttctcaatttcagatttcagattttctagagaaatattg +atctttttttttaatagaaataatgacttttcaacatttttggatttttt +agtttcccAGCAGATGGCCCGCCAATTTTCACAAACTCACATAAACCTAA +ATAATCTTTATTTCCCTAAAATTGATTAAAACCTTATTCCCATGtcctga +aaaagaagaaaaaagaaaaaagctcGGATGGGTTCGAACCCTTGGCCTCT +ACCTCAAAACGCAAGCGCGCTACCACTGCGCCACACCACCACCAACGAGG +AAAGAGAGTGGAAAGCATAGAGAAAGGCGTGCCGTTTTGCCGCTTTTCAA +GATAGATTGATGCGCTTGAGCATTTTCTCCTTCCAAAAAAGTAATGAAAT +TCTCTTAATTTCCGGTCCCCAATTTATCTTTTAAAACGGATTTTTCCATT +AAAACGGTTTTCTCTGCAGGAAATGAATTGGCATGAAGCGGCGACAGCTG +CACAGGAAGCCGCAGAAAATGGGGCTGGTGATGAGCAAGAAGAGCAAGAA +ACCGATAGGCTATTGAATGGAGCCGGTGGGGGACCATCAAGCGGAATTAA +AGGCGCACCGGGTACGGAGAATCGGCGAGGAAGTatggataagaagaaga +atagcaaagagagtgagttttttttcaataccgTTAACACAAACCCCCtt +aactgtcaactttgagccaagttatctcggttcctgtgagagatatcaaa +aagtgatcaaaatgttgctgatattatgataaacattttgtcagttgacc +gttttttgatacctttcacagcaaatgagatatttgagctcaaagttgaa +gggtTTACGGTACgtttaaaaaaagttgaaaagttggAACTGACTAAAAC +TGTGTTTAATAAGTATTAGTGTggttttctaaaactttcaatgcagtgga +ttttcagtttcaaacgaaaaattcaaaaaaaaagtcaaaaaattcaaatt +gaccAAGCGGACCTTCTGCACTTGTGTTGTCATCTCCTATTTTCTCTTAC +ACACACCTCTAGTTATTATTTAGTTTtattagaaaattattttcaaaaaa +aaagattcttgaaactcaaaaaaccccCTATTACATCGGCACCCACATAC +ATACGTCCTATGTCTACACACAGTAGTTGTTTAAAATAATTTGGTGAGTT +ATCGTGATCTACAGTACTCCCTGCTaggagtacagtaatccttcaaaACT +ATGGAATGCGCTGCAAAGTCCCTCTCCCTTGATGCACTGTGGCTTTTTCT +GGCAATTCCTAAGTATTCAATATATGCCTTTTTAGTTggttttctttcat +tttttctgcatGCCAAATATCACTATTTGTTTATAGTTTCCGCAGCAAAA +CGAGATTCCATACTTTAAAAGAggatatctcaagaaccatcAGAGCTaga +caaaagttttcaattaacaaaataaagagcataaaaaatgacatacttta +tcagttgagctCTTCTTTctatcttttctattttttaagaTATAACCGTT +CCAAGCAAGCGCGCTCCACTGCAAGAGAGATCTTACCGTAAACAAGGTAT +ATCTCGATAACTATTAACGctaaacaaaacaattcaacaacaaaaatata +gctcatcaaattttacatattttgttagttaatacttttgttctatcttt +tttagtttttgagatacgacCATTCCAAGCAAGCGCGCTCCGCTGCAAGA +GAAATTACGCGCCTCACTTTAAACAACGTATATCTAAAGAGCTATTAAAG +ATAGACagaaattgtcaactacaggAATAAAGCTcattaaattttacata +tttcagcaattgatacttttttttctagcttttttagtttttgagataga +aCTGCTCTAAAGAGGCATAAAACGCGCTCCACTGGTTTTATGCGTCTTAC +CTTTAACCGGGTATaaatcaaaaaccaaaacagtTAACAAAATATTATGC +ATCACAAAAATATAGCTCATAAATTGTtatacattttgttagttgaaact +ttttttctatcgtttttggtttttgagatacgacCATTCCAAGCAAGCGC +GCTCCGCTGCAAGAGAAATTATGCGCCTCACCTTAAACAtggaatatctc +aaaaactatcaaCGCTATACAAAAATTGCgaactacaaaaaaatagctca +taaaattttacaaattttgttagttagcACCTTTCTTCTAGCTTTCTTGG +TTTCAGTGATATTACCTTTTAAAATATcatcaaatattcaaattttagtt +tttgagatataaatCACTCTAACACTGTGTGTTTCTTACACCTCTATCCT +ATTTGGCTCTATTggccccgccccttttcaAAACTCGGCTTATTGATCGA +TTTTATAATTTCTCTTTCTATGGTGGTGGCTTAtttttttGACGAGTGAA +CCGGACCCTCTTCCCCCCTCTCAGCTTATAAGgcatccaaaaaattcaaa +ggtttcggcgccaaattcaaatttttgttaaaatttcagcaatgGTTCAT +GAACCTGCAGTTCTCATCGAGGGTGTCCTGTTCCGTGCTCGATACTTGGG +CTCAACTCAAATGCTCTGCGAATCGCGTGGCTCGAAAGCGGCTCGAATGG +CTCAGGCGCAGGAGGCGGTGGCGAGAGTCAAGGCGCCAGATGGAGATGTG +CAACCGTCGACGGAAATCGATTTGTTCATTTCGACCGAGAAGATTATGGT +GTTGAATACGGACCTGCAGGTTGGTCgggaatttttggttttctagctTT +TGATCTATAATGCCATAAAAACTATGCAAAATACCTCCCTcagaaccgtc +aactttgaacccatttatctcgatttctgtgaaagatatcaaagaattgt +caactgataaaatatgtatCACATTATAAtacacattttgtcaattaacc +gttttttggtatctttgaCTAGAGCCGAGATAAACTTAATCAAAGTTAAG +GGTACCTTTTATTGGTATAGACATTTGATGTTGGGCCAACTTTGGGTGGT +TATATCTATAAAACGAATTTAGCTAGCTAAactttgtaaactacaaaaat +gtagctcatgaaattttgcatatttttagtgttgacaaagtttttatagc +ttttttagtttttgagatacagtCCTTCAAAGCACAGCGCTATACTGCAA +GCGCACCAAGATGCCTTACCTTTTGCGGAgcttatctcaaaaactgaatt +agctaacaaaattttgttaactgacaaaataaaaaacactaatttttgca +tattttattggttattaacacttttgtcttaactttttcattttttgaga +tacaaatgttcaaaatttcagagaatatcCGACACCGATGTTCGACAGGA +CATTCTAATGGACCACGCTCTACGGACAATCAGCTATATTGCCGATATCG +GCGATTTGGTAGTTCTGATGGCGAGGAGAATGTCCACAAGTCATTCGGAT +GAAAGTTGTTCGGATGGGTAAGGATTACTGTGTGTTATCATTGTGACCAT +AAcaacccaattttcagcgacaGCAGCAGTGGAGGTGTCCGGAAGACGCC +GAAAGTCATTTGCCACGTATTCGAATCCGACGAAGCCTCATTCATTGCTC +AATCAATTGGACAAGCGTTCCAAGTAGCGTATGTGGAGTTTTTAAGAGCC +AATGGCATCGATGATCCGAGTTATTTGAGGCAGATTGATTATCAGGTAAG +CCcagttgaaaatcgaaaactttgaagccCTACAAAAgtgtttcacaatt +tttgaacttgtttaaaaatttcattgtacactaaacaaaaatagagcttc +atttttgtagttggcaagtttttcgtaaaatttgacctagaaaagatatg +ccacgtggaagagaaGATCCTCAGAAGGCGCGTCAGCGCACTAGCGGTGG +TGGCGCCCGAAGGCACCCacacttccacgtggcatatctcttacaggtca +aattttacaaaaaagttgtcaactactaaattgaagctctatctTTGTTT +AGTGTGCTCTAAAACTTTTGGATTGATCTGACAATTATTAGTTTTACTAG +CACAGGTAAAACGcgaacgggtccgttacggccgcttcgcgttcgcgcgc +ggttgaaataggaaccgcatacggacggcacgcgaaccgttttcagtaaa +tagcagtacattattcggattgatccgtattctgtccgttcgcgagctcc +gtatccgatccgattgcgaatcatgaccaaacagatccgtttgcgaactc +aataaaacatttgattnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnGAttc +gcaaacggatccgtttggtcatgattcgcaatcggatcggatacggagct +cgcgaacggacagaatacggatcaatccgaataatgtACTGCTATTTACT +AAAAACGGTTCGcgcaaccgcgcgcgaacgcggagcGACCGTaccggatt +gttcgcgaatcacctgtgaggaaacgtttgaaaattatagctgttttctt +caaaaaataaagatgTGGTTATTTTTCGACGGCTTCAAAAAGCGGCTCGC +ATTTGATAGattgataaaaaaatttcagagtatactaaccAAAAacagag +ctttaattttgtagttgacactttttttgtgaaatttaagactgaagaga +tatgccacgtggaagaagGGGATCCTCAGAAGGCGCGTCGGCGCACTAGC +GGTGGTGGGCCCGAAGGAACTCACACTTCCACGTTTCATATCTCTTTTAG +gtcaaatttcacaaaaaagttgtcaactacaaaattgaagctctatctTT +GTTTAGtagactctgaaattttcagacaaattcaaagaaattggACCACT +TTTGTAGGGCTTCAAAGttggcgattttcagctaaaaatagttaaaatca +aaaagatactgtatctcaaaaacgagaATAGCTATCATAAAACggttaac +tagaaaaatgaagctcataaaattttctacaattaACTcttgtttttaaa +gttttttgacacaattttttgtattttcaggaAGTGCTAAACTCCCAAGA +GCTACTTGGTGACGAGTTGGAAATGTTTGCCAAAAAGGAAACCCAAAAAG +AAGTGGTGGTCCCGAAAAAAGCCGGTGAACCCCTGGGAATTGTTGTCGTG +GAGTCTGGATGGGGGTCCATGCTACCTACCGTAGTCCTTGCACACATGAA +TCCCATCGGACCGGCCGCCCACTCGAATAAATTGAATATTGGAGATCAAA +TCATCAATATCAATGGAATCTCGCTCGTTGGATTGCCATTATCAGCTGCT +CAGACGCAAATCAAGAATATGAAAACTGCGACGGCGGTCCGGATGACGGT +AGTGTCTACGCCGCCCGTGGTGGAGGTTAGGATACGACGACCGGATACCA +AATACCAGTTGGGATTCTCTGTGCAAAACGGTGTCATTTGCTCTTTATTG +AGAGGTaaggaaactttgaaatttttaaatttgaatttggcgcgctaaga +gcttcaaaataagtataaacacaCCTAGACTTATAAAATTGGGTCCCACC +GCGAAAACTACAATAATcctcaaactttttggaaagtagctatttgaaaa +atactcattttgaagctcgtaacaagcgaaatctgaaaaaattaattttc +ttcaaactGTGCCAAAAATTACtgagatattttgaaaatttgaatttttc +acaaatttcaaaatgaaatagatcataactcgcttggttttgactcaatt +tgagtaaaactttttttgttcaattttgttcattgtgaacttcaaaatga +gtataaacacgcctgaaattacaaaattgggtcccaccacgaaaactaca +gtaatcctcaaaatcttctaaaactagtcgtgtttatactcatctTGAAG +CGCCTGGCaaggcaaattcaaaaatatatgtttCAAACAACTCAGATCCA +AATTTAATGAGTTGTGATCTTAGCAAAGTTAAGAAAACGTTGCCTTTAAC +ttacggattactgtaattaatTACTGTACAAAGTCTTAATCAAACATTTT +AGGCGGTATCGCTGAACGCGGAGGAATTCGCGTCGGACACCGAATCATCG +AAATCAACGGCACGTCGGTGGTAGCGGTGGCACATGATAGGTGAggctaa +aaaatcgataattctgaaaaatcgataatttttcagaattgtgaATATGC +TCGCCACGGCGGTTGGAGAAATTCATATGAAGACGATGCCGACGTCAATG +TTCCGATTGTTAACGGGTCAAGAACAACCACAGTACATTTGAgaaaatcg +attatcgattttttttctttttttctttttttctctcttccctcTCCCAA +CACCGCACTTTCTCTGTGATAATcatgttttctttctttcttctttttct +cttttttttaaagtgaaaatgcgctctattgagggttttgaaattctcaa +aagagcgcattttcttcttgaatttcagtttctaGGTGATTCTCTGCGAG +TCCACATTGCCatgtttttttgctctttttttcgaaattttcagtttttc +caaaaaaaattcattcattttttttcaattggtcacaatcgggatttctc +tttcttttttcgtctttttctcaattttttttgaatcttcggtaatttaa +taaaatagatttttttttcaaaaaattggtttaaacattttcaaaattta +agaatcctgagaatcctatgaatccaaagaatccaaagaatactgagaat +cctaGTAATAGTGGGATCCCTAACTTTGATCCAGTTTATCTCGGGtcctc +tgaaagatatcaaaaagtggttaactgataaaatgtttatcataatatca +gcaatatttcatcagttgatcactttttgatatattgtacagtaaccgag +ataattaggttcaGAAGAAGCAATCTTCTTAAGGAGATTCGCACctttca +aaagtgaaaattcaataattattcaaagttgattttaaaaaattcgaaaa +attaagaatatcgaatttcaacaaaacgaaaacgactacgtgattttttt +tttcagaaacccacGACCCTAGATGCCCCATTGACCTCAGGAGGAATACt +ataaaaaaagaggaaagtgtagggaagaatcagttctccctGCTCTGGGC +TCCAGAGGAGGTGACCCGCGGGATCCGGGAACCAGACCCAAGGGGTCCTG +GGGTCAgtcggcggctacaatctTCTGGCTAGACTATGTAaaatctaggt +cgtgacccaaaaaatctaggtcatgttccgaaaaatctaggtcacgaccc +aaaaaatctaggtcacgacccaaaaaaatctagatcaggacccaaaaaaa +tctaggttacACCCAGGACCCAGGTCCCGGTTCCCGGCCACAAGACCCCA +GCATCCCCCGGCTCCCCCTCTCTGGATCCCAAAAAAGGGAGAACTGATTG +TTTTCCCCGCTTCCCTCTTCCTTTATAGCACTCTTTCCAAGGTCAATGGC +CTAGGGtcaaggatttttgaattcttttgaaaattggtttttgatcacgt +ttaaaaaattattgaatttccacttttgaaatctgtgaaactccTTCCAA +AAGGTCGCtcctttgagcccaattatcttgattcctgtgagagatatcga +aaatttgtcaactaataaaatgttggtgatattatgttaaacgttttatc +agttgacaggtttttgatatgttttgaatgagccgagataatcaagctca +aagcgCATACATTCTTTTAGAGTATAAGTTACCGTAACTTCTCTAATGGA +ATTTCGtgcccaattatctcgattcctgttagAGATACCACAAAGTAgtg +aactgataaaaggtATTTCATAGTAtcgccaacattttatcagttgccag +aTGTTTGATTTCGTTTCTCGAAACTGAGATATTCAAGCTCAAAATGGAGG +CTGCAAGGTGTGAAGCTCCTTTAAAAGATTGCTTCTTTtgaacctaatta +tctcggttactgtaaaagatatcaaaaagtagtcaactgataaaatgatg +tttattatgataaacattttattagtttactactttttgatatttttcac +aggagccgagataaactggctcaaagttaggaaCCCTAATATGACGggtt +ttcttaaaatttggtatcatctgaaaatggcaattttttgatgtaaaaaa +aatcaaaatttcaaaaaaaaatgacatttttcttaaaatcagaacctcag +aattccagaattttcaaaaattagatttttcagattttcgattttttttt +tagaatttcagaattctctataaaattgaaattttcgattttgcgaacaa +tttttgaattattcggcattatttttctagatttttgggGTTCGGagact +ttttagaatttctcagaaatcttaaaatctcagaatttttaattttcgat +cGATTTTCTTAAATACCAATTATCAATTAAAcgaatttcattttgatgAG +CACACAGCGAGAGAAAAGAGCAAGAGAcgaaaatatttagaaaagagaca +cagagggggggggggggaatttTCCGGGAATTGGGAATTTAATAGTTAAg +aataaaaatcgataaaaagagaggaaaacaTGACATATTACTACTTTACA +AACACACACATATTTGATGGAcggggggcggagcttattgGGGGGATGTT +ACAGTAAGAAAAGTGGGGCGGagcggaaattctaaaaaacccgttgtaat +agaaaaaagtatgaaaaattaTAGGGTAATTAAGGGATTATTTTAATtat +aaacgaaaaaaagagaaaaataaaatttgaaaaaaaatgaaaatgagaga +ttCAGGGTcaaaattacgattttcagatgatttttcagagcattttcaTC +ACGAAATCTCTCATTTTCGCCACACAAAAAAGGGGGAGACCCGAAGACAC +ACATATGATAAAATGGAATAATCAATCACAATATTAAAATCAttcgaaaa +agagaatttAAACCGCCACGACGGTGGTTCTTGGCGGCATAATGCCCATG +CTCAAAAACGAGATCGTCTCCTCGGCATTCGGCTCATTGAGGTGCTCAAC +AGTCATGTGAAATGCCAGCTGGGAGCCTTCGTGCGAGACGTATTCACACA +ATGGGCACTTGATGATGGAGCCAGATCGTTTGACAATCTCCTCGGGCGTT +GGAGCCGCATCGgcgttctgaaattcaaaaaaaagttaggaaatgcttag +gctccgcccactattccctgaatttttttgggcaatttcaaaggcttttt +aggtttaaaaaaatgcatttttttctgctgaaaattgctgaaaatttatt +ctgaaatctaaaaaaatataagcgtttaggctccgcccacttttaccAAA +ACTTTTTCGGGCAATTGTAAAGGcgtttttagttcaaaaaatgcattttt +tcggctgaaaattgtcacCTCGCGGGttcaaaattagcaaaaaGAGTCTC +TAATTTTCactgtcaaaaatattctgaaaagcaacaaaatgaatttctga +gcatttaggctccgcccactttcagaTTCTCGACACATAATTTAAGAGAT +TTTAAACTATATATCACGGAATCGCACAGtgtgcaagtgcgctccactga +tagccaagaaaaattccgcatttttaaatttttcagcagagcgtattcgc +ttttttgaaggcaattttaaggcttttttCGTcggaaaaatcttaaattt +cagacaatttctatgagttgaaaattctcaatttccacaGGCGCCAAAAA +GTTCACCTTTTCAAGCCTTCAGACGTTTATTGTCTGTGTGTCCTACCTCC +GGATTCTGAATGTGTCCGTGCTCGATGCGAATCTTCGTCGCTTGAATAAT +GCCGCGAAGCGTTTGATCCTTGACCGAGTGATGGTGCATTTGGTGGAAGT +CGACGACCGACTTGCACGCGTCCGAATAGTCGCAAATCTTGCATTTGTAC +AGTTGATTCCATCCGTGACGATCCAAATGCTCGATGAAGACGCTCTGAGT +GACACACTTGAACGTGCACTTTTTGCAGTGCTTCGCATTCTTCGCCACCA +TCAAATGCTTCATTTTCGGCCGTTGATACTCGGCTTTCGGCGGTTGTCGC +TTCATCGAGACGCGAATTCGATTGATCAAATGAACGCGATGAATCGCCGT +GTGCTCTTTCAGCTTGTGCTCCGTGGTCGTCTTGAACGAGCAACTCGTAC +AGCTGTACCCTTTCACACCGTGCGGAATAGTGTCTCCCTTCCGTTGATGC +TTCTGAAGACGTGGATTCTTCGCGTGCCGCATGTCCTCGTCATTTCCGTC +GTTGTGCACTTTCAAGTGTTGAATTAGAGCCGTTGGCGTGATCGTGTTAA +ATGTGCACTCCGAGCAAACGAATGGGCGTGTCGGTCCGTTCGGCGAGTGG +AAGAATCGGTGGTTTTCGAGAATCTTCGCGTCCGGCGATTCGAATGGACA +GTCGGGGCATTTGATTggtgctgaaaatgagaattcggggttatttttca +tgtttgactccgccccttttagCAACTATGGAGCATAGTTTCATCTATTC +TACAATTTCAGAGTGTACTAAAATTAATCTAAAAAAGGCCTTAGAAGTca +tttggcctaaaaattagACTAAACCGCCAAAAACTGATCAGGTACAGCCA +AAATTGGACCGAACATCGGTCTAGATCGGGCCAGAATGTCTTAAGCTTGG +CCTAAAATAGCCTGACATGGCTAAAAATTCTAGGCCGTTACTAGGCCCTC +TCTGAAGTCATCAAAAATCCATTCAGGTAAGCCCAGGATGGCTTtaaata +ggcctaaaatatCCTAAATTTGGCCCGAAATTCCTCTAAATCGGCTCGAA +ATGTCTTGGAATTGGCCTTAAAATTTAGTTTATTTCTAGGCCCGAAATTC +GGTTTCGTTTCCAAAGTCCAAAATCAGTCTAGAAAAGCCTTGACTAGGTT +AATAGTAGGCTTTAAACGGTCTagaattggcctaaaatttaCGCTGtttt +agaatctcaatttGGTCTCCTAAGCCCAAAATCAATCCATATAGGcccaa +aatgttttaaaataggcttaaaacgACTTAAATTTGGCccaaaatagccg +aaaaaaCTCACCATCAACACCAAATTCGTCTCTGAGCAAATTGCGAATCT +TCCAGTGTCCAATATCCATCGGATCCTCGTCCTCCTCCATCGGCTTCACA +ACGTTCGTCGCTTCTCCATCCTCCTTTCCCTCGTCATCCTCCTCCATCTC +AGCCAACCTCATCAGATTCGACGAGCTCGACGACGGCGTCGCCTTTCCAG +ACCCGTTCTTTGAGAGCAGCGAATAGTTGATCTTATTCCTGGAGACTCGT +TCTGAGAGCTGCCGCGTCGGCTCGGGATCCGGCGATGGTTCAACCGACGT +ACGACGTCTTTTTTGAGCTGGAGAGTTGTAGATTCTGAACTCCTTATCCT +CATTCTTAACCTTGTTATATCTTGATGGTCGTTTGGAGGAGGATCTTGGC +ACTTCATCCTCGTCAtcctcttcatcttcatcttccaACGATTCTGGAtc +ctcttctggaaccttcacctcttgcttctcttgcttctccgccttctcca +acttcctcttctgcacctttctctccCCAGCCTTGTTATCCGGCCCCCCG +TTCAAATCAGTTAGCGACGATACCGGCCGCCCATGATGGTCCACTCGCTT +CACAAATGGAATCTGTTCCGTTGAGCCACCCATAGCCAGATGGACCAGAA +TGTGCTCCTCGATTTTCTTGGTATCCAGCGAGCTATACGTGCAGTTCGAG +CAGATGCTGTGCTTCGGCACACGACGATGATAGCGAGCATGACGCCAGAG +GCATCCAATGGATACTGTGGTGTAAGGGCAACGAGAGCAGTTGTAGATGC +GGTCCTTGCCGAATCCACGACGCGTTCCAATTGGGATTCCTTGATTGACA +ACGATAGGATGGGTTGCTCCTGATCCATTAGCCAACGAGTTGTTAACCAC +TAGATGCTGTTTCATATGATCCTTAAGACTAACAGGACGGTTGGTGGTGT +ATGTGCATTGAACACATCTAAACGATTGCTCTATATCGTGTCGCTTCTTA +TGACTTCTCATATCACCACGATACTTCGATGTGTACGGACAATCAGTGCA +CGCGAATCCTTTAATCGAGTCAATCATTTTGCGAGTGAACGGCTCGCCGA +GCTCCGGTCGCTTCGCCTTCTCCGCCAGACACCATTTATGAATGGATTTG +AAGAGATCCGTCTGCTTGAACTCTTCATGTTCCACATGCTTCGCCTTCTC +TTCGTCTGTCTTCATCGCGTGCTGCTCGGCCGCCAACTTTGCcgcttttt +cagccaaatccaCGCGATGCATCTCTTCATGCGCCTTCATCACCTCCTCT +CCTCGCGCCGTCCATGAGCACATTTTGCATTTGATCGGGTCGTTGGTGAA +GTGCTTCGCGTAGTGTCGCTGGCAACGGTTCATCGATGGTGCACGGTAGG +GGCAAACGTTGCAGCAGAATTCCACCATTTTCAGTGCGGCAAACTCTTCT +GGGGTTAGATTTGCTAGGGCTTCGGGGcggaaactggaaaattaagGCTT +GTTTAGGCTTGGTTTTTAGGCTGTTTCAGGCCACAATTCTAAACTATTTT +AGGCcgtttttagaccattttatACTTACTTTCTGACCATTCTAGTCAAT +TATTGAACTattttagatcatttctagaccattttagaccattttccgaa +tatttagATCATTTCTGGAtcattctagatttttttaaagccaTTCGAGT +CTAATCTAGTCCATTTTTGGATCATTCTAGACTATTTTAAGACCATTCTA +GATCATTTTTGGGCTATTTCCGACCATTTTCCGAATATTCTAGACTATTT +TGAGCCTTTCTAGACTGATCTAAACCATTTTCTGcctattttggaccatt +ttagatCTTTTTTCTGACCACACTAGactatttttcaatcattcCAGACC +ATTTTCGGGTCattctagaccatttttcaatcattctagaccatttttag +accattctagtctaattttagaccatttttcaatcatttcagactatttt +caaccattttagatcatttttcgaccattAGATTGCCTCCAgctttatat +tttttctagtctagttttcaatgattttcgactattttagacctttttta +gCGAGTTCTGAACCATTTTTAGCAGTTTCTAGACCTATTCTCGATCATTT +TAGacactttttggaccattctAGCCCATTTTAGAAAACTCTGGACTatt +ctagaccatttttggctcatttcggaccatttttgagccattcgAGTCTA +ATCTAGTCCATTATAGCCCATTCTAGACcattttcgagcatttttcgACC +ATTCTAGATAACTTTTAGCCCATCGAAAAATGACCTTAAGTTTAGGGCCT +ATTCTAGACCCCTTTTCAACCATTCTACGCCTCTTTTCGACGATTTCTAA +ACTCACATCGAATCAAACGGCTCATCGTCGACGTCTTCGTCCTGCTGAGC +GAAGAACAGCATCTGGCGCATCGTCGGTGCCACGACTGGCActgaaaatc +tcttttttggtTGTGCTGTGAATTCAGAGGCACTCTTACCGTGAGGAAAT +TTCATTGGCGCTGTCACAGCCACAGCGTTTGACGATGACGGTTGGGAGTC +GTCGAGAACAAGCAAcggaggaggaggtggagTCGCACCGATTCCCGTGA +AATACTTGATGTAGAACTTATCGCCAATGGCTTCTTTGAGATAGAACGTT +CGATGGAATCGATTTGGAGCCGTGAGGTGGTTGGTCATTTTGTGAGCACG +AAGAGCACCGAAATTGGTGGTACGGTAGCCGCAAGTGGAGCAAAGGAAAC +GATGGAATGGGAAGGCCTTTTTCTGAAATGCTGTCGTCTAGAGAAAAGGC +CTAAAAAGCCTAAAAGCCTATAACTTACCGCTCCATGCTTATTGGTCTTA +TGGAAAATCAACTGAGTGATCGACGTCgtcaaaaaatcgcattctCCAAT +GTTGCACTGCTTGTGTGGGACACCGTCAATGATCCGGGTCTTGTGAGATT +TGATTTGAGCCTGGAAATAAGGAatttaaggatgagcaatcctAAAGGTG +CTCCTGGAGCTCTAGAATCTTCGTCTAGGCCAAAAGCGGTCTCTAGAGCC +CTAAAGAAtataaggatgagcaattctAGCTGATACTTGTGCTCTAGAAT +CCAAGTCTTGGCCTGAAAATGCTCTACTAGAGCTCTGGAGCCATCCTGTA +GCTGAGAAACTCCTAGACCTTCTTCTAGAGCTCCAGTAGAgcattttcag +ttcattttcagactcgGTCATTCCTCTTACCTTAGCGATCAACCCAACTC +TCGCTTTCGACGTCAGTGGAATCAAAGGCATCTTCTCTTCCggcacgtct +tctgattcctcCTCAACCTTCTCGATTTGAAGCTCCGGCAtcttctccga +cgccttcatcttattcaacttcttcaactccttctgctccttcttctcct +tctccagacgcttctgctccggtgacttcttctcgtacttcctcttcttT +GGCTTCTCCTCCACCTCGAGCTTCGGTGGCGCCTCTTCCTCCTCCATCGT +CACGTCTTCAACCGATGTGTCCAGATCCTCCGGCTCTTCACCAGTACTCG +GATGATTTGGATCCTCCACCTCTTCAATCGTCACCTTTCCATTCTTTTTA +TTGATTACTTTCCGACAGAACTTCGGTTGTCCTAGCAAAGGTTTCGAGTG +AAGAATGTAGTGCTTAGCAATGAATCCCCTCTTTCGACACATGAAATGAC +ATTCGGGCTCTGGACAAATGTATCCCTCATCTAGAGTGTGTCCACGGAAG +TGTCTCTGAAGACGTGTGACCGATGAGCAAGTGTACGGGCATTTGGAGCA +TGGGACGCCGCGCTTCGATGTCTCCTTCTGAACGACCGGAGGTGTCTGCT +TCTGAACTACTGGTGGTGGTGTCACCGACGATGGTGCAGATGTCGTTGGA +AGAGATCCACGTGGTGATGGTGTGTCTTCTTGAAGCCTTCCGGAAGCTGT +GGTGTCATCCAAAACCGGCGGCAACGCATCCAGAATCGGCAACGTCTTCT +GCTGCTGCTTCGTATACTTGCGTGGCTTCTTCTCCTCCAGCTTCTTTTGT +GTTGGCTTTGAGGTTCTTCCCGATTTGCGAACTGCAAGATGGTCTGGAGA +CTCGGAGACCTCTCTCTCCTGATCCACTTCCCTCTGCTCATCTTCCCGCT +CCAAAACCGGCGGCCCATCGTCTTCATCCTCctcgtcttctggattctca +gCATCCAGAAGTGGTGCTGCCTCATCACCATCATCATCGTGACCTTCAGG +TGCTTCCGGCAGTGGCTCCACAGACAAATCTGGAGCCACCAGAGACGCTG +TGGACGTTGACGAGTCCATCGAGGTGTTCTGGGCCTCAGGTGCTGGCGCC +GAATCCGAAGAATCTGAAGCCGGAGAGCCGGCAGACGCAGCGTCAGGGGC +CGGAATTGCTCCAGCAGTCGTCACTGGTGAAATCTGGGGACTCAGCACAC +CGAACTGCTTATGAACGTTGTTGTGCCACGTCAGCGATTTCAGTGTTGCG +ACGAGAAGCAGCACAGCTCACACTTAAACTCCTTCCGCTCTTTGGTGTGC +ATCTTGGCGTGGTTGGCTAGGCTGGACTTGGATTCGTAGCGCGATGGACA +CTTGTGACACGCGAATGGACCGCTCAtctggaatttcttttttcggggtt +ttagactccgcccatttttgatgatttttgggtttctaggccgaTCAATC +TTGTGGCCtacgaattcgaaattttgggatttttaccGCGTCACAAGCTC +AAGCCTTTTCCGGTCTcatgtggcctagaaacccaagatttctaggccac +cacaaTATAGCCGAAGCTTCAATTTCCTGTGGCCTAGAactttggatttc +taggcggcctagaaacccaGGACTTCTAGGCCGTGAAAAAGGAATCTTGT +GGCCTAGAATCActaaatttctaggccaccagaatAACCAGCTTAGAGAc +gttgtggcctagaaacccaatttttcctacagtaatctcaccTCCACCCA +ATTATCATAAGTATCCTGTTGATATCCGTGAAACACGGCCGTACTCCTTC +TGCTACGCCGTCTACCATCCGATCCTGCTCCTCCACCAGTTTCCACATTC +AACTGCTCCGCATATGCTCCGCCTCCCGCTTCTGAATCATCCAAATCGTC +GTAGGCATCGTCGTCGGAATCTGCGACTTCGCCGTAGTCGCGCTTCCGAC +GCTTCTGATTGTTTCCATGGGCGGCGGCGACGACGTCAAGTGTGAACTTT +CGACGTTTCGATCGTCGGAGGCTATCGTCAGCCGATGCTGCTGCTGCTGG +AGCTGAAGGCGTCACTGGAGCCGATGAAGTCGTCGGAACAGATGACatcg +attctgaaagaaggGGGCGGAGTTTGGTGTGCTGTAGgagaggagagaga +aaagcgatgaaaaaagagaaaaaattcggtnnnnnnnnnnCGAGTAGTAG +GATAAGGGTTACGGTACTTCCAATCCTAACAAGAATTCCattgtctacag +taatccttttagCATTTGATGACCTAGAATTCCACCTAGGGGGATTTCTA +GGCCCCGCCCATATTTAGAAGACAAagggactgaaaaaaaacagaaacag +aagaaaaataggggaaatggctagaaggagaaggcgaaaaccgataaaaa +CGAGTAGAAGAGAGATACGgcagtgtctgcgtctcttcgtctcgCACACA +CTCTCGCTCTCTCTACACACGCGGCGCGCAAAGCGTGTGCGTTGCGTGCG +CGtttttttcactctctctgtgttctttttttctctgcgctctctcacac +tctctcactgtctcttctgcccaaaaaaaagttttctagaagaagaagag +gaatatATAAATTTGATTTATTGATACTTAATTAGAAAATGATAAaggca +gagagagagagagagagagggtgagagagtgtggggagatgagagagcgc +agagaaaatGAAGCCTACAGTAGGACCGAAagaaggggcggagcctaaac +tTTCCAAAGGAAATTGAGCAGAAGCAGAAAAAGCGAGGAGCCAAAATTAT +GGGTTGTTCtgtgtttgtatgtgtgtgtgtgtgtgtcaaaataggctccg +cccttTTTGTGGGGTACGGTAGAAAAATGCATGACCGGgatgacctagtt +tttctaGCTCCcggattctaggccactgtATCATTGCGACCTGGCCGtgg +gaattctaggccatcggcTTGTGGCCTAGGTTTTCGCCTGAGCAAAATTT +AGCTAGCTAAAGCTAGATCAGCTAAAAGGCGGTGGCCTAGTTTGGGATTC +TGGGCCAGCAGCAACTTTCACAGCCTCAAGACCATGGCataaaactaccg +tagtcGTCGTCTCGCTCCTCCAgcggagagaaagaagaaagtgagagatt +tagagaaaaatttagGTTAGAGGGAAAAATGTGTGCCTTCTTGGCTCCTT +CCTTTGCAAATGAGACCCCAGAAAATttccgagaagaagaagaaggaaaa +agggggcggggccaaaaaaaaagaacgacgGAGAGTGCAAAAAGGCACAG +AGGAAAGGAGGTTCATCTAGGGAAAGTGGGTCCcagactttcagattttc +aaattttcctaaaatccgaaaatcgcAAAAGTTGTGAAATTCCGCAGAGT +TTCCTAGGAATCCAGAatcttcaaattccaaaattcaggaTCCTAAAAAT +CCAGGATCCCCAGGTTCTAGAATGCTGGAAATCCTGAAACTCAAGGTTCT +AGAATCCAAGGATCTAGAGATCCAGAAtccaggttttcagaatttcagag +atcaaGAGTCCAAGAATCCAGGAAGCCGCAGTTCTAGGTTCCAGAATCTC +GAAAgcccagaaatccagaatctagaagctccgcccactaaACCCAAAAA +ATGGCCTCTTCTCGAAGACTTTCCTAGACCTAAGGCCTAAGACCTCCCAG +AACATCCTACCCGGGTGGTCCGGCCCACCCAGCACCAAAGGTCTAAAACC +CCAAACTGAACCAAGAATTCTAACTAAAAAAGacagagaagagaaaaatg +cgAGAGTACGTTCTGGGGCGCGGAGGGTGCGAGTGAATCAATAGAAGAAC +CCAGGACCGCCCCGGCGAACAACGAATGTGGCGAGTGATCATAtgagacg +aagagacgcagggaGGCGGGAGAGAGACGTAGTGGAGATGCGATGCGCCC +GTCCGCACCGACCGCTCGCCGCTCTCTCAGGTTTTCGTGTTGAATGGGAG +GAAGATTGgagaaaagggaagaagaagatgaagagggGCAGATGATATTG +GCTCCTGGAATCAGGTCAACCTGTCAGAATCTCTTCGAACGCCTTCTGCT +CTTGGAGACCTTCAGATTCTTTGGGTCTAGAACTCGGAGCAGTTTAGCCA +GCTCCAGCCTCTGATCTGGACATTTGAAGCTTCCTTGCCAGATTCTGACA +AGCTAGGAGAGAAACGCACGTCTTCTGATCTCCAGTCTGAGCTGAGATAA +GCAGGTCCCAGTAAAACAAACGATCTAGCTAGGTCTCAGAAGCAActgga +agcttctggaaacttGAGGAACGATCTTACGCCTTCCAGAACCATTTTGG +CTCATCTAGTCGCGTCCATTGACCATTTAAAGACAACCGAAACCTACTGT +AGAGTCCGTGCCAAGACTTCTGACAAGCTAGGAAAGCGTCGCATACCTTC +CTATCAGCTCTCAGAGTTCTGAGATCCAGATTTAGGCTTGGATAAGCCAT +TCTCTTGGTAAACTTGGTCATCGAATCATTTGGTAGCTTCTGGAAGATTT +TGGGAGCTCAAGACACCTTCCAGAACccattttgaccaatttcgGCTCGT +CTAGAAGACTATGGGCGAAATTAATGACAACAGAAACTTCTTGTAGATAT +ACAGTAGACCCAGAATTTACAGCTACGTCTAGAACCTTTAAATCCAGAGA +ATTGATCTTTCGCCTTCGTACACCATTCCTAGGTTCTCGATTTTCTGCCC +AAGTCGACAGTTCATCGATaacatcttcatttttttctccatgACGTCTT +CATACCGATCTCTACTTCTTACTTTACACAGCTACTGTCAtcctcgaaaa +aaaacgacttcTCGAAGAATtcttgagaagaagaaatattGTCGACAAAG +AtacctttttttctagaaaaaaaaatgagaaagcgAACTGGCTTGGAGGC +CCATATAAGACACTTTCTATATAGTGCCTTCAGGAAGTACATCTCCTAAA +CCATTAATCAATCGCGGGTGCCAGGTGCGCGACCTTTTCGGAGTGATTGA +AAGGCCAAGATCGGTGCATTCGACATTATTTACCTCTGGACCATAGATAT +CATCGTTGCAAACGGGCCGGCAGATAACTCGGCTTCAGAACAAGATGTTT +TGACGGATTTGATATCAAAAGATAGATCTCGACGGGAGNNAGNGAGTNCA +TCTATTCCAGGTGTAGGGCCGCCCGGAGAGGTGGAACCGTAGAGGTGGTT +GGAGCTAGATTTCTCATTCTCTTCTTCAAGGTTCGTACCTTGGGCACAGC +CgaaagtttttagattttccgactcccagaatctcaggaaGCTTTAGAAG +CCCAAATCCTTCTGAAGTTCCCTCAAAAACCCTAGTTTTAGATGAGTTTG +Gnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnaTGAAGTATTTTTGTAGAATTCCAAACTT +GACTTTATTCAAACCTGTCATAGTAGACTTCATAAGTGCCAtataagctt +aattatctcggtttccattaaagatatgaaaattttgtgaactgataaaa +tgttggtgataatgtGTTAAGCAGTTTATCAGtccccaattttttggttt +atttaagtggagctgagataatttgactcaaattcaaaagttaagGGACT +GTTATATTATAGATAGACCGGAATTTTTCCATATCTGCCATTCTAGACAG +GCCATAGGCATGTTAACGGACCAGAAACCGGCCCGTGTGGCATGTTAACG +GACCCGAAACCGGCCCGTGGGCTAAGATGGACCtcgtcgagatctatcat +ttgatAAAATATCTCGTTCTGAAGCCGAGTTATCCACAACTATCTCGAGT +TAGCCCGTTTGTAACTATGATAGTAATATCCTATGGTCAGGAAGGACCAT +CCCTTGGAGGATAAAAAGTCAGTTAAAGGCGCAGCGGCCAACAAACAGAG +ACCAGGCGCCTTTAAACgattttcggcgattttttcaatcagaattttga +gaaaaacatgGTTTTTAAAGGCACATGTTCTCAAGATCAGAATCGCAATA +TTAGTCTCATGGCATTTGAAATTCGGCATCGAGTCAAGTTCAGACGCGCC +GCGTGTCGGCGAGGCGGTCGCCAGATGAGCAGTTCAGCTCAACACCCCAT +TAAGATgctttaaacttcaaaatttcaggtttcctACCGGAATACACCTA +ACCTCAAACTGGGTAAAATTTGCCATGGCAATTACGGTCAACTTGAAGCG +GCCAAACGCCACGTGGCGTCTTCTGGGTCGGCCGAAAAGACTCGGAAACT +AcaagaattttcggaaactccGATGACCGTAGGCGTTGGTCTCGACACGA +AAGCGTTGAATTGAACTTTCGAATCTTACTGTTTtgtgaaaactcgaaaa +tttctaaaaaatccgagaattttcagaaaaaacggAGCAATTTCGAAcat +cgatctgaaaaattccaaaaatcgaaattcgaataagccggaaaaacgag +aaaaatgcGGCTCGTTCCATGCCACGTGGCAAGCGCCggctgaaattccg +aaaattaccgcatttttggctgaaattacAGTCCTGCTGATTTCCTGAAC +CTTCTGCTCTGAACCTAGTCGCCACGTGGCAAAATTTCCAGGCCCACCGC +CGTTCCAACAGGCGAATTTCAGTCAGACCGGAATTTCTGAGACTTCCGAT +GAGATTTTGACGGTTATCAaccatttttcgatgaaaaaacccaaaaattg +agtCGCCACGTGTCCCGGAAAAGGTCGGAAAAGGTGGGAAAATCGGAAAG +CCTGTAAGTATCCAGTGGTCCCGCCGAGCCatcggaaattggaaaaaagg +cTGTTCTTAAGGTCGGAAATAGGCGATTCAGAGCGATTTTGGCGGAGAAA +AAAGTCGAGATTGGATGGAAGATTTCATGGCAGCTGAAAATCTTGGTATG +CCAAACTCATCTAAAACTAGGGTTTTTGAGGGAACTTCAGAAGGATTTGG +GCTTCTAAAGCttcctgagattctgggagtcggaaaatctaaaaactttc +GGCTGTGCCCAAGGTACGAACCTTGAAGAAGAGAATGAGAAATCTAGCTC +CAACCACCTCTACGGTTCCACCTCTCCGGGCGGCCCTACACCTGGAATAG +ATGAACTCACTTACTCCAccaatccagaagacgttacGATTTCCAGAAGG +GGGCTTATCAGAAGGGGTGGGGaaagaagaatgagaagaCTATCCCATTC +TCCTAGATTCTTCGATGACATactcttcttcgttttctctAACACATAGG +AAgaagatagagagagagagacagaaagACACCACGACCGCCGCTACACA +CATTTCTCTAAATCTCATACACTCTTACACTATTTCCCAGCTGTCTATCT +CCTCTCTTGCTTCCATTCGGGTTGTtcttttctgcgtctctagccgcAGA +GCGCCAGAGTGTGGTGTAGAGAGTGTGGGAGAAAAGGAGGGCAGAGACGA +CGAGACAAAGataggagaagaaaaaggggcggagtctgaGGTGGAAAGGT +TTGCATGGTAcgagaaactgaaaatgggATGAAATTGGGATAATTGAGGG +ATCAGGGGGTATTCTGGAACAATTGAGAAGGACCAGAGGATCGGGAAACA +TCGGTGGActctttttttagatttccgaGACCCAGAAAGATTCCAAACTG +CTTCAACgtcgatttttccagacttttcTAGGTTTCTTCTGGATCTTTCA +CTGATTTCTACTGATTTCCATCGATTCTTATCAATTTCTCACGAAAATTC +TCGAGAAACCCAGCGAAAAAACTCCGAGAATCAAGTTCGCCGGAAATGAA +AACTTTATCCGATTTTTGTCGgggttttttggagaaattgaaattttagc +taGTTTCCAGCaatttcttcaactttttcgcTGAATTCTGACctagaatt +cttgaaaaagtgaggcgaaaactcgaatttcgaaGCCAGCGGCGGTGCTG +AGCAATGGCGACGGATTCAGATGAAATTCTAGGTTTTTCTAAGCGGTTAG +TcggtttttcagaacttctggtAATTTTAACtagaaaatattccgaaaaa +atcgaaagaatcAAAGAAAACCCAACAGTGACGTCATACATTGTGCAAGC +GAGCTTCTGAAGCTCGCCACGTGGATTCGGGGTGTTTTTTGGtggaaatt +tgtgaaaaccGAATActtctgttgattttcagaatagtttagcagaaaat +tcaagaaaatttggtGATAGAAATTTTAGCGAATTTCATCCAAATCGCTG +TCGCAAGTGCAGAAAAATTGGGCGATTTTTCTAGGGGGAGTATTCAGCGA +AAAATCCAGATAATCGACTCAATTATCAATGATTATATGTTTAAAAGTGA +CGAAACAACTGAAAATCCGAATGAAAATCTCGAAGCGGAGGGGGCGCGAA +AAAATCCAAGGAAGATCAATTTTGGCCAATTTGGTATGGAAAACGTCTAT +CGGAGTCAGAAAAAGAGacgaaaacaaagaaaattcatatgaatttcagc +aaaatcagcattttcagcaagAAACACCGGAGaaaatagtggaaaaaacg +agaaaaatggctcaaaaaccaTAGGAAAGCCGAGGAGTCGTCGCCACGTG +GATTACGGTTTTTCGGGAGAGATTCAGGcgaatttgttgattttcagcgt +ttttcgctgaaatttttcttaaatttcactTAAAAAACCACGAAATTCGT +CTGGATTAATGCATAAGCGCCATTTAAAGCAACAATTTtcgcaaaacaaa +aaaagaactatcggaattccaattttcgagtttttgaagcGTTTTTGACC +TACAAATTGGTAGAAGCAGAAAAGCATGAAAACTAGAAATACGGCTTAGT +TAAGCCTAAATTTACTCTCGTCCGCGAAATCGTGAGCGGGCCAGGCCGGC +CAGCCAAGCTTGGTGTCAAACGAAGCTTGGCGGCTACCGCATTGGAATTC +TGCAAAttcgaaaactggaaatttccaaaacttccagcaaaaaaaatcca +cagaatttaaaattcaaaggtCTCTGACAAGGGTCTCACCGCGAAAGCCA +ACTTTTTCGAAAGCTaattttcgtggtgagacctGTTGGATAGAGTTTCT +ACCGtagaagcttccagaagcTTCGATGCCTTTATTCGTCTCTCCGAGTT +CTTTCTCTTATCAAGTAGTATTTAGTGGTAGTAGTACAAAATAATAGATA +TTTCATCAagtcattcttcttcttcttctgtttactcttttttgttttct +ctgcgtctctcatcaCTAGGcgattctttcaaaaatctattcaGATACTG +GAcgatgaaaaaattaattaaagcTTCTGAAAAGTGCATAGAGTGGAAGC +TGGAAACGGCGAAAAGCGGGAGATTGATGAAAAAACAAGTGAAACAGAGG +GGAAGAATTCTCGGCGGAAACGATCCATTGGGATaaaatggatggaagca +aaaaagaaaagggcgggaaatttaaattctccaaaaaaattatgttttga +CACGATTTTTCCCGGGATTCCTGTATTACTCTAGTGctgtttttttgtaa +aaagttggctttcgtggtgagaccctTATCTGATACCTGTGAATTTAAAC +GACTTaattctttggatttttgccAGAATTTAGCGGAAATATCAAGTTTT +CGAATGTGCAGAATCCCAATGCGATAGCCGCCAAGCTTCATTTGACACCA +AGATTGGTGGCCAAGGATGGCCGGCCTGGCCCGCTCACGATTTCGCGGAC +GAGAGTAAATTTAGGCTTAACTAAGCCGTATTTCTAGTTTTCATGCTTTT +CTGCTTCTACCAATTTGTAGGTCAAAAAcgcttcaaaaactcaaaaattg +gaattccgatagttctttttttgttttgcggAAATTGTTTCTTTAAATGG +CGCTTATGCATTAACCAGACgaatttcttggtttttttaatgtgaaattt +cagtaaaatttcagcgaaaaacgctgaaaatcaacaaattcgCCTGAATC +TCTCCCGAAAAACCGTAATCCACGTGGCGACGACGCCTCGGCTTTCCTAT +GGTTTTTGAGCCGGTTTTTcttgctgaaaatgctgaatttgctgaaattc +atacaaattttctttgttttcgtCTCTTTTTCTGACTCCGATAGACGTTT +TCCATACCAAATTGGCCAAAATTGAtctttcttggattttttcgcGCCGT +CGCTTCGAGTTTTTtatccgaattttcagcgGTTTCgttacatttttgaa +gatattATCATTTATAATTGAGTcgattatttgaatttttcgctgaaTAC +TCCCCCTAAACGCGAGAAAAATCGCCCAATTTTTCTGCACTTGCGACAGC +GATTTGGCTGCTCACTTGCACGGCGAGTGACGTCACTGTCGGGTTTTCTT +AgattctttcgattttttcggaatttttccagTTAAAGTTTTAGGTCCTA +GTCTAATTTCAATGCGCATTGCGGTCGAGAATTCGGAATGagcggagctt +aggtccggtcgcgaaaactttaaattcgcatttaaaggagcatgtgtttt +ttttctatttcccgACATATTTGAtggattttgaactttttttgaggaCA +AAAATTCAGACAATCGCAAGAAATTgcataaaaacttttttcgacgaact +tttgaaattcgtcatgcatgcgcctttaaagctgTCGCTACGGTAAAATA +GCGAAAGTttgagttacagtaatcaattCTGCCTTGAAAAATCATGCGGA +CATTCCTAACCCATCATTAAATTTGCTCATTCGAAACCCCACCCGTCTAG +GAGCAGGTGCAGATGGAGGTTCTGAGCGGCACCGAGACACCAAAGAGGGG +GATCAAGTCTGGTTCATGAGCCAGCAATTAGTCAGGCTCGGAAGAAATGG +TTGACAAATTGAGGTGGGCGGGGGGACCATGTGAAAAGGTGTAAAAATTA +AAGaagactttgagctcagttatctcggttacagtaaccgagataattga +gattaaAATGGAAggtatcgatttttcaaaatttgtagtGTTAACAgtca +taaaaatgaaattcgtttaataaaaaaagaaaattcaaacaacgataaca +atgataacaaattcaacaaaaaatcagtaaaaatcctaGTCTTCATCATT +GAATAAAATGCGCTTTCCAGCTGGTTTCGCCGCCGCCAACTCCTTCTTTT +TGGCTTGAGAAGCGAGCCACGATGGATGCATTTGgccttctgaagacgtc +gaatTCTTCGATGTGcccttcttcttgttttcggTGGTAGTAGCTGGGGg +ttggaaatctggaaaattggggaatttaggtttgaaaaaatctgattact +tggaatcctaaaatttcaaagaaccAGAATTgtagaatctttgaatttca +ggatctcagaatttcagaattccggagtttcaaagattcagaaattcaga +atcttagaatttctgaatttctgaattccggagtttcagagattcagatt +ttcagaaattcgaaaattcagaaattcagaaactcagaattttagaaatt +cagaattctagaatctgagaatttcagaattctagaatcttggaatatta +gaatttcgaaaattcagaatttcagaaatttcagaattcggatGAATCCC +ATAATTCCCGAATCCCGGAGTTTCAGAGAATCAGAGATTCAgatcttcag +aaaattcagaaattcagaaacttagaatattaaaatttccgaatttcaga +acctagaattcagaatctcaattttcgggGAAATTAAGAATCccagcatt +tcagaatttcagaatctcccaattttcaaaactttcagaattcaaaattt +ctgaattttgggaaattttagaatccaatATGCTTCCAgaatacaaattt +cagaattctcaaaattccagaaatcaagGCTTCagaatccgaattttcag +atttctctgattttcagaattcagaatctttggaatttcaaaaattcgaa +ttccctcCAAAAAACGCACtcttctttcccactggtcccttcttcttcgt +attcttattcttcttcatcttcagaTCCTCCTCAACTCccaatttctcca +ttttcttcgCGATTCCTCCAGATTCCGGCACTGCTTTCGATTTAGAAGCT +GAAGACGTTCCTTTAGAGGACGCTGGAAGGAAGAATCCAGAGCTCtcgtc +ttcagaattctcctcctccacgtcttctgaagcctTCTTTGGAGCATGAG +CAGCCATCGTCGATTTCGGAGCCGCCGTCGATTTGGCTATTTTTCCACCT +TGTGATAGGTCGATTTTCATGACCAGAGTTTGCTGgaagaattagaaaat +ttctgaaagttctgaaaaaaaatcaaaattttgaaaaaaaaaatcgaaat +ttcgaaaaaatcaaaaattcaaaattttgcaaaaatcgaaattctgaaaa +attcaaattttttgaaaaatccgaaaaattcaaaatttcaagtctttgaa +aattaataattttcaaagtctgaaaaattcgaaaaatcgatattttgcaa +aattcaacattttttgactaaaaatgtttttttcaaaattttcttttcca +gaattttctggaattttcagacaaacctGATTTGGATCCTCTTCCTCCTC +GACGTcatctgcttcttcttccacgtcttcagaTTCTACTTCAGAAGACA +CTTCCACCTTCTTTGGAACCAtttttttCGGCTTTTCCACCTGCAAAAAT +GTCcaattttccaggattttcgTCCTCCGTCCATCTGTGTATCCAGATGT +CCTCTCTTCATACCTTTTCAGCCTTCAACGCCTTCTCCAACACTGCCTTC +ATCGCTTCTTTCGTTTTCTGCTTCTTCGATGGTTCAGATGACGTCGTCTC +CGTGACGTCTTGTGAATCGTCGTCAacgatttttcgcttttttggaTTGA +GAGCCGGACGGGACGCGTCTTCCTTGACACCGATTAGATTGAGGAgctgg +aaatttaaatttaagaatctgtgaacttcagaatcccagaaattcaaatt +ttcagattttcataacttcagaatcctagaatctgaacatcccaaaatct +caaaatttcgaaatttcagaatcctagaatcccagattttcggattttca +gaaattcggaatccctgaatcctggaatctcaaaatctcagaatcctttt +tttttttgaaaaatccgaaaaattcaaaatttcaagtctttgaaaataaa +taattttcaaagtctgaatcccagaatttcagaagccTCAGGAGTTCaga +atcttaaattttcagaatctaagaatctaagaattgcagatttctgaact +tcaggGTCTCAAGGACTTCAGGATCCTGGCGttccagattctcagaattt +cggaattcaagaatttcaaattttcagaattccatgAATCTAGAATGCCA +GAATCGTAGAATCCtgacatttcagaatctgagaattccagaatctgtga +atttcagaatcccagaaactcagattttcagattctcggaatttcgaaat +ttcagaattctagaatacTGGAATCTGAGAATCGTAGAATCTGCGAATTC +AAGGATTTCaggttttccgaatttccgaaattcagaatctaagaattcta +gaatctcagaatcgattttagaatttcggaatactagaacttcagaatcc +cagaatttcgaaatttcagaatcctagagCTCTAGAATCCTGGAATCTTA +GAGtatcagatttccagaatcctccaaattcagaagttcagatttCCAAA +GTTTCCGAACTCCAGattctcaatatttcagattttgagaatctcggaat +ttcaaaaatccagaatctgagaatccgaGAATCGTAAgaactgaaaatct +caaaacttcaaaatttcagaaatgcagaatcttagaatctgaaactttca +gaatcccagaatctcaaaatccccTTCTCACCAtatttctcctcttcttc +gcCGCCTCCACCCCGACGTCTTCATCATCACTATCCACCATTTCGACGTC +CGACTGATCGTAGTCATCTGGAGCGTCCtccccctcttcttcttcttttt +ccacgtcttctgaatcatcCAACGCATCTTCCTCCACATCGCtctcttct +ccagaatccACGACGTTTTCTGAATCCGTTTTCGCCTTCTGAATTCCCTT +CGTTTCATCAGAATGCTTcgcttttttctcctttttctctggTTTCGTGG +AGTATTGCCTTCCGAGCCGTTGCATGAAAAACGCCGCCGTCTTGTTCCAT +TCGATAtacttttgtctgaaaatttcagaattttcagtcatttttcagct +attttcagccattttcagttattttccgctattctgttttttttcagcca +aaaaaaacctcgaattttcggattttggattattttcagtcaattttcag +ccgattttcagctattttcagccattttcagtcaatttttagccggtttt +tagcttcaaatagccagaaaaccctaaaaattccctaaaaattgaatttt +cagctttgaaaatgtcatttttcaagccatttttcaccaatttttcagtt +accaatttttccaatttttcagccgaaaatgtcgaatttttgaatttttc +agccaattgtTAGCCGATTTTCAGTTACTTTCAACCAATTCTCAGGCAAA +AAAcgccgaatttttgtatttttctgacaattttcagatatatttccagc +caattttcaacttcaaataGCTTTTTTGAGACACTataaattggattttt +agcttcgaaaatgtcatttttcagactttttttcagccaaaaaacgtcga +attttcgaatttttgcatttcggatgattttcagccaatctcagctattt +tcagcatttgcagtcattctgaacctattttcagccaattttcagttatt +ttcagctattttaagtccaatttcagccgtttttcaaccaaaaatgtcga +atttttgaatttttcagcctattttcagccatttttcagcgattttcagc +caatttttatctattttcagctattttcagccaattttctcccatttttc +agccaaaaaaactttaaagtttTACTTGATCGCCTGAATTCTGGAAACGA +CGACTTCATGGCACGCCAACTTGTACATGACTTGCTCGGCCATCGGCGTG +GATCCAtcgattttcagttcattCAACGTTTTGGTGTTCAGCGAAGCGAA +TTTGGCGATGGTGTCCCGATCgattgtctggaaaaaaaatttcagtgaaa +attttcgaaattctcgaattttctagatttttctatgGCTAAAAACCCCG +ATTTTCTAGTTAATTTTAGtgagaaaaaactgatttttcggtgattttca +ggggttttaactttaaaatttggatttttcgcctttttttggctgaaaat +ctcagcattttttcagaaaatttttggggtccaaaaaactttttggagcT +CAAAGAAGgctaaaaatcgctgaaatgcctgctggaaaaattcgaaaatt +cgaaaaaaaaatgttgtttcaaACCAAACTTTCAGccagaaaaatcgaag +ttttagagccaaaaatccaatttctagagtcatttttaagccatttttag +cgattttcagcgattttcagctatttttagtgaatttttagtcaatttcc +gcttattttcaactattttcttacaatttccatttttcagctactttcag +ccttttattcagccaattttagccaattttcagctatttccagccatttt +tcagttattttccactattttcggccaatttcccgctaatttacagccat +tttttcagccaattttagccaatttacagtcgttattcccgctttcattt +tcagccattttcagcgattttcaagacattttcaagccgattttcaagcc +aatttcagctactttcagccttttattcagccaattttagccaatttcca +gctattttcagttatttttcagttattttcagctattttcagcccttatt +tcgatttttaaccCCCAAAAATGCTTTAATTTCTCACCTTTAACGCCAAA +ATCTCCTCCATCATCCTGTTCAATTTCGCAtccaccacctcctcttttcc +accttttccctcctttctcttcttttccaattcctttttcttcctgATCA +CTTTGCTGATCCATTGATCTTTAGCCTTCGTTACCGTCGCGCGAAGTCCC +ACCAGCtcattgttcaatttttctgctGTGAGCTGGATTCGGCCAGTTTC +AGCatctttttcagccaatttttgagcttctggagcctttttcgcttctt +ttctgagtgatttttgagcattttccgGTCTTCTAGCGTCTGAACCTTCC +TCCCCTTCAGAatccgatttttgagcttctggagcaattttcactgattt +tttcggttttttcatcgattttcgatcattttcagccttttccatggttt +ttggcgcatttttcGCTACCAAATTCGGCAACTCGAATTCGATGACCGGC +GCGGCGATGTTGGACATATCGAACACTTCATAGTcctctgccacgtcatc +agcatCGTCGATTTGGGCCATTGGCATTTCGATCGAGGCAGAATCGTCGA +TTTTCTTGACTTTTTTGACCATCTGAAGCTTATTTTCgtaaaaaatgggg +aaaatggaaaagaaacggaaatttgccccaaaagtaggaggaaaactgaa +gaaattcaatgaaaaatatgtcgaaaaaccggaaaatttgcaccgaaaaa +tggaattgaattttttggaaaaattggcgaaaagtgaaatttcggagaaa +tttCGCCCAGTGAAATCTATGAGAATTATGTTgaaaacccggaaatttcg +ctcaaaaaatagggaaaaaatatcaaaatttggaaaaaaaatttagttgt +TTTGGATCATAAAAccctttgaaaaatcgataaaagcacaaatttttaca +aaagttttctaacacattcgatttttctcacttttccttcaatttttgtt +ctttttcgtgtagaaaaacatcaaaaatctcaattttctacccgtggcct +agaaaacagaGGAACTTTGGgaaaaagctaggccaccggcgaaattcgaa +ttttctgcctcattttttggtcatttttcatccaaaaatgcgtttttcgt +gctcaaaatcgagaattttgggattttttttgtaattagtcaattttaaa +tagaaaaatcaaagaaattttcgagatttttcagaccaattttcgaaaaa +tttcctgaaaaacctaaaaaatcagaaagcaATAATTTATATATAAATAT +CGTATAAGCTCAGCGATattctatatatttttcaactgGGAAAATGGAGG +GGATGGGCGGAggggcaaatttttgaaaatttacaaaaattaaaaattta +ggaaaaattcagaaaattgaggggaaattaagcaccaaattgataaacaa +acaaagaaaaagcgggcaaaaattcagaattttcaaattttcaaaaatta +gaaaattagaaaaaaaatcacgggaTTCTGGAGGGAAAAAAGTAGGTGTG +GCGCTGGGAAGTTTTAAGAAATTCTAGAGAGAATTATcacatttttctgt +tgatttcgGGATGAAACAAAGGCACTTAACTGGGAATAATTATGGTATAT +AAATTagggaaaaatcaataaaaatgaatattgGAAGGATAGgagaagaa +tacaaaaatgtacaaaatttgagagagaaaaaagggcattttaagagaaa +tttgagcgcagattgctcagctttaattAAAGTGGCGGCTGGAACGAGTC +GCCGGCCGCGTCGTGATGCATTGGCGGTGGTTGATTGCTCATTGGTGGCG +GGTTGAACATTCCTTCGGTCAGATAACCAGCGACTCCCGTTGGGGCGCCT +GGAAATAGTgatttttggttgaaattccaggtttttagattatgtttttt +ttcttcgaatttcagatattcagaatctcagaaatcgTAGAAAATTCAAC +TAACCAGTAACCGCCAGCGTCCTCGCCGCGGGCAAGAACAAATCGTGTCG +TTGATCCTTGAATTGTGCCCAAATGGGGCTCTTCTCAAGTATCTCCTGAA +TCTTCGTTCCCACTTTCAAATCCAAAATCGCACTTTTCAGTGCATCAACT +ATCTCGGCACGGGCGGCTGCGCCATTCGAGACGCCATTCATCGAACTGTC +CAGCACCTCCAAAAGGTACGGTACCATTCCACAATTGAGCATTTGTTGAG +CCAATTCCCCAGTGTTTCGCTTCATTAGACACTTCAACGCGTGCGCGGAT +TCTCGCATCAACGACGGCtgatttttcatcgatttcatGATTCCGTCGAT +GCAGGGCAGTTGGGACAGGGCGTCACAACAGAACTGGTCTCGCTGAGCtc +ctggaaaaatgacaatttttttgaaaatttccattttttggagaattcgc +TGAAACACTAAACTCATAGATTTCAAGCCATAATTTGTTACTTTGggcca +gattttttttgaaatttcgaagcaaattttaatttttaaccagtttccag +ctattttcagccatatataagctattttcagctattttcagctaattgtt +agctattctcagctaattttaaccattttcagtcttttttaatttatttt +cagccaatttttagtcattttcaggcagtttccaactatttcagccattt +ttagccaatttccagccatttaagccaattttcggctaagatttagctat +tttcagcctatttcgatctattttcagccaattttcagccaatgttcagt +caattttcagctgctttccgccaatttcagtcgtatttcagcctaattag +agccattttcagccaatttttagcttttttcagCCGATATCCAGCTTCAA +ATAGACTTTTTGAGACtctaaaattggattttttagcttttcttTGCCCA +TTTTCTGTCGAAAATGCCGtttttttgtcagacaattctgattttctcgt +tttttaaattttttcatcttttctagaccataaacatgaatttttagctt +taaaaattgcattttccagctattttcaatcattttgagccaattttcag +ttatttttagctaatttccagccaattttaagctattttcagccaatttt +cagttattttcaactattttcagccaatttcccgctcattttcagctaat +tcaagccaatttcgagccattttcaagccaattttcagttattttcagcc +attttcaagccaattttaagccagatttcagccaattgtcagccaaattt +ttgtattttcagacaatctcaagccatttttttttcaaatttaaggcTAA +AAAGCCTCACCTGAAGGATCAAAATGGCCACTTTGGACGCTTCTGGCTTC +TGACTGCCCATATTCGTACAGAACTGTGGCAGATATCCTTGAGCCGGTAG +CTGATCAGCAGTGTTCGGATGATTACGGACCAGTGCCACGAAAGGCGGTG +GTGACCATTTCGATTTGAGGTGGCGGGCGGGCCATTAGTTCGAGGAccct +ggaaaaatgggaattgggTTATTGGTTAGGAaatggcgggaaattcgaat +tttaaattttcaatttttagaatttaaaggaattttaaaggtaattaGGG +CGCTAATTAgagaatctcaatttttcagaatctcacaatccCAGTATAGA +ATTTCGGATTCCAAAGGAAATTAGGTACTGAGAacctcagaattccagaa +tctgtggatttcaaaatcataGAGCTACAgtatttcggattttcagaatc +acggAATCTAAGAAGCCTACTAATCTTAGAAtcctcgaatttcaaaattt +ctgaattccagagtcagaaaacctcagaatctcagaatttccagactcag +aatctcagaatttccagaaggagaatctcagaatttccagaatttcagat +attcgtaatctcatttttcagaatctcagaatcccagaataccagattct +cagaatctcagaatttcagaatctcagaataccagaacaccagattctca +gaatctcagaataccagaatttcagaatctcagtttttcaaaatttcaaa +attcgtgaatttcagaatcttaaaaatttcaggttttcagatttttagat +tttcatgatcgagaatcttagaattccagtttctcagaattccgtaaatt +cagattctcagaatctcgaaatttcagaatttaagaatttcagtacacca +gattgtcagaattccagtttctcagaatcttagaatttcagaatccctga +atttcaaattttcaaaaagtttgattttcagaatctcagaatttcagaat +ctgcgagtttcagaagttctaaacctcaaaaattcaaattctcagaatct +caatactccagaaatccagaattccagaattccataTATTCAGATTcttt +gaatctaaaaattccagaatttccgaattccagaGACacgaaatttaaga +atctgcGAATCTCGgaacttccaaatttcagattcccagaatttcaaaac +tccagaatttcagaataccagattctcaaaattccagattcgcagaatct +caaaatttaggaatcccataatttcagaatttcagattctcagaatatca +gaatcttcgaatttcaaaatttctgatttccctCTCAAACTCACTTCTCA +ATCAACTCCCTGGCAAACTCCTTGGGATGTCTGACAGGCCATCCGGGATT +CTCGACAAACAGCCGAATGAAGACGCCTCCAACGACGAGTTCTCCACTGA +TACTATCACCATAAGCACAGTAATCGTCTGATAACGTTGTGGTATTCCAT +TTGGCCGTCGGGTCCTTGATATGCTTCTGATACAGCGTTTCCACCTCTTT +ATCAATGatttctttcacttttttccggGTGTTTTCGTTCCAAATCAACT +CGGGATGCTCGTTTGATGAATCGAACATTTGTAGTCTACAAAAGGTTGGA +CTACGGTCATTCAGatttggctccgcccatttttgaaaaactattgCTAG +TTTCCGTAGCTTTTGTCTCATGCTTCCCAGAAAATTCAGttccaaagtgg +gcggagctacaGACCTTCAGtacatttttaagtttctcatcaaaatgggc +ggggcCTAATATACCTTTGCTCCGCCCCTTTCCATTTTAGATGTCATTTC +GAGCtcctcccaaaaaattttaaaagtgggcggagctacaGATCCTTAAT +ctgttttttcatgaaaatgggcggagcttgaaaTTTCTATGAGCATTAAT +GACTTATATTTGCCgattgttttgattttcagaattttaagctccgccca +ctttcgaCAGCAATAGCTGTTAAAAATTTGGtcgaaaagtgggcggggct +agATATACTTATAttacaaatttcagtttttcatgaaaatgggcggagcc +aatttcagaattttttagaaattgggttcaaaagtgggcggagctgaAGA +TCCTtagtatattttttttccgatttttcgtgAAAATGGGAGGAGCCTGA +AATACGgaagcaaaaaatgaaagaaaattcgttcaaaagtgggcggagcc +tgtcACCTAATCGagatttttcaatcttttgaaaaatgtgggcggagcct +aaaatccgaagttttagatttttctaaaagtgggcggagtctgaaCCCTA +ATCGAGAAATTTTGATCTcttgaaaaagtgggcggagcctaaaatccgaa +tttcggattggtctaaaagtgggcggagcttgaaatccgaaattttggat +ttgtcTAAACGTGGTCGGAGcttaaaatccgaaattttggatttgtcTAA +ACGTGGTAGGAGCTTAGGATCACTAGCACTAGCACTAAATTTCACCCAAA +TATTTAGGACCAAAATTAAActgtttcgatttttctcaaaagtgggcgga +gcctaaaaacCAACTCACGCTGTATTCGGCGAGTCCCTCAACGCATCAGC +AAAGATCGCCGGCAGGAATTTGTTGATAAATCGTGTCCACCGTGGTCCCG +TCAGCTTATCAGCCTGCAGTTTCGCCAGAAGCTCCGCCGATTGAAGACGT +TGTCCCTGATCGGAATTCGTGAGGCAGAGGATGGATAGAATGTACATCAG +CCCTCCGTGCTCTAGCATCTCCTTGACTATTTGACCGTTCGAAGAGAGAG +CGATAAGGACATCCAGGACGCGGGCAACGATTTTAGGATGGTCTTTGAGC +AGTGTGAAGAGGGTCGTTGTGATATTACACGTGGCAAGATCAGTGACACA +CTCCTTATTGGCTGCGGCGAGGAGGATGACTTGGAGAGCGGCGATTTGAA +GTTGAGGGTGTTTTCGGCCGCGGAGATAGGTGATGAGCAATGGGAAGTTG +CCGATGAGCAGGATTTCGACGCCTGGAATATGCAtattattgctcatttt +ttgggcattttggCCATAAGGTCTGATAAGGGGGAGGAGAGAAGCATGTG +GAAAGGTTTGGGGGTGTTTTTAAGTGGAAATTCGAAGCTAAACGAAGGAA +TCGGGGGTCAAGACAGTCAAGGAATCAGGcgcaaaaactacaaatttttt +gatagattcACCAAAAACCTTACAACATTTATTCTCACCCTTATTGTTGG +CAAATACGGCACCTGGAATGGTATTCTTGTAATTTTGATAATTAGAACAG +TACCTTAGCGTCTGTACTAGAAGGCggccccctctaatagaaggacccca +tCGAAAAGATCATGAAGAAATAAGAGGACCCTCTTCAATAGTTTAAGCAT +CTACGACATATCAGAACTGTCtcattttagaactgtgacatttcaaaact +gtgacatttcagaactgtgaTATCTTAGAACTGActttttagaactgcga +catatcagaactgcgacatttcagaactgtgacattttagaactgcgaca +tttcagaactgtgacatattagaactgcgacacttcaaaactgtgacatt +tcagaactataacatttttaaactgtgacgtttcagaactgtgaTATCTT +AGAACTGActttttagaactgcgacatatcagaactgcgacattttagaa +ctgtgacatttcaaaactgtgacatttcagaactatgacattttttaact +gtgacatttcagaactgtgaTATCTTAGAACtatgacattttagaactgc +gacatttcagaactgtgacatttcaaaattgtgacatttttaaactgtga +cattttagaactgactttttagaactgcgacatatCAGAACTGCgatatt +tcagaactttgacatattagaactgcgacatttcaaAACTATGATATTct +agaactgtgacattttagaactatcacatttcaaaactgcgacatttcag +aactctgacattttagaactgcgacatttcagaactgagACATTTTAGTA +GGGCGCTCTAAAGGTCCCGcccttctgatacaggttttacggtatatttg +gCTCAATTTCAGACGGTTTTCAGCCATCCCATGCTCACCAGGATTCGCGG +AAACCAGATTCGACAACGCTGTCATCACCATCAACACCTTGGTCTCTGTG +TTCAATGTGTTTCCATTCTCCTGTGATGACGATGGTCCATTTCCCCAATC +GATTTCAATCAAATCGTCTGGATTCTTTTTGATTGGCTTCGGCACACCGT +TCAGCTCATCCGAGCTCTTTCGGATATAGTCCAACAGATCGACGGCCACT +TTTTTCGGTTCCTGGAAAATTCGGGGTGAAAATCGTAGTTTTGGAGTCGT +TTGGGCTCAAAAATCTAGTATTTTCGAGCCGAATTTCGCAAATTTCGCCT +CTGAAACgtgattttgagctcaaaatcgagatttttgggtacaaaaatcg +tcaaaattaGGCAAAAATCCACCTTTTGGGGttttttcttgcaatttttt +gcagatttcgcctctgaaaatcgtaattttgagctcaaaaatcgagattt +tcagcttcaaataAGTAAAAATTAGGGGCCAGAAATGATGTTATGTTGtt +ctaaatcaaaatttgggtGTTTTCCAGCCATATTTCGCAAATTTCTCCAC +GAAAGCTCGAGATTTCGAGCTCAAAATCTAGagttttggcttaaaattca +tcaaaattagACAAAAATCCAACTTTTGGGGTTTTTCCTACAATATTTTG +CAAATTTCGCCtctgaaaatcgcaattttgagctcaaaatcgagattttt +ggattcaaatcAGTCCAAATTAGGCAAAAATCCAACTTTTGGGGGTTTTC +ctgtaattttttgcaaatttagcctctgaaaatcgtgattttgagcttca +aattcgtcaaaatccgggcaaatttttgaagtttccggATCTGAATTGAT +CAGAAATAGGGGtcaaaattcaatgtttcGTTCAAAATCGACTCACATGA +AGCGCAAACGTCGGCTGCTCGTTGTATATCCGAATGAAGACGTCGCCAAC +AATCAGCTCGTTCTTGTGCACGGACAACTCGAACTCCGCTCCCATCAGTG +TGCTCTGAAGCGGAATTTGAGATGAGCAATAGGTCAGAAATAGccaattt +tgttcaaaaatgctcaaactTTGTGCGGCTCAAATTAAgccgaaaattct +aattttccagtctaaatttcaagatattttagtctgaattttaaaatttt +aagttgagcaatgagtcaaaaatcggctcggctcaaaattcgagaaaaaa +tgcattttccaAACTCTAAACACCTACCGATGGCTCATTCGACGTTCGAT +GTTTCTCCACAAACTCCAGCACCTCAGCTCTCGTCCCGTTATCCCAAATC +ATATACGGATTCTCCAGATTACTATTCAACGTCTTCAGCACCATATCGTT +CGTCTCCACACGCATACATCTACAGATATATGGTGTGAGCAACGCGCGAA +GGCTCTTCTGGACGCCTTCATTATCCGGCGTGTTCTCACGATAGCCGGCG +AGTGCCGCAAGGGCTTCGGAGGAGGACCGAGCCAGGGAATTCGCCAGCGA +TTGCTTGTTGGAGTCTTCTGAATGCTGAACGCCGCCTTCGTCGAGGGTGT +AGTCGTAGTGGAAGAGGTGGGGGACTAGTTGCCAGAGAACGCCAGTTTGG +AATAGTTGGAATTGGAGGAGCTCGTCGACGGACATTGCACAAATGCATTG +GGCAGCGGCTGTGGACAGGCGGGGGAGGTGctggaaattcaagattttca +gacctAAATTTTAAGATATAGGATTCTGGGGCCTGTAATTTACAGGCtgt +aaattccagaatccgGGCCAGAAATCGTGTAATTTTAggttcaaaattga +agttttcagcATCAGAAACTTCCCATTTTTACAGtcaaaaatctataatt +ttcagggcaaattttggaattttagaccGATTTCCGGGCCCAAtgcagac +attttttgaagctgaaaatcgataattttcagaccaaaaattgcaatttt +taagcggaattttaaagccaaaaatctATAACTTTCAGAactaattttct +cgatttttcagggttttttttcatgaaaaatctataattttcagaactaa +ttttcgacatttttagacaggatttttttcgatttttcagggtttttttt +cacgaaaaatctataattttcagaccagaatttggaatttttagacggta +ttttcttggttttaaaagccaaaatttcgtatttttcaaatcgaaaaaaa +tgctCACCGAGAACTGCAAAAGTCTTGTGAGTGCACCGAAAATCGAAACC +ATCTCAGCCAACTTCTGTCGACACATCTCGAATGTTGACGCCGTCGCGAA +GCAGTCGCAAACGTGGATACAAACTTGGACAGGCATGTCGTCTGGCGTCG +AACTCATCGTCACCATAGGGACGCAACGATCGAATGCGGTGACCAGGGCC +TGTGGAAATTAgagatttgagatttttaactATTCCTGACTAtttttttG +GGTGTTCGAGATTACTGCACAACCTGCATTAGAAGGCGCTTtcattgttc +caaggtacctctgGAGtgagagcccttctaatacatgctttacggtattt +ccattttagccgatttttagccgtttatgacgagattttttggattattt +tcagatttatttcaggccaattttaggcgttttgaggctatttcagtttg +attttttgagtagaAATTCATGcgtttatcatttttcagcatcGAAAGGC +TAAATTTCTCGAATATTAGCTATTCCAAATTTCCTCTCCTaagattttct +aggtcaaataCCTCCAATCCACCATCTCGTCTCAATTGCTCGGCATTCAG +TGCACTGCTAATCAGTGTGTAGTTGGCCAACTCGATCGCAGCTGACAGCA +AATCTCCGCCTCCTTTGATGAACAACGCGTCGTCCTTGGCCTCCAGGTTG +ATGGTTTTGATGAGTTGCGAGTAGCCGGCGTACTTGTATTCAGAGAGTTC +TGGaaattgggttactgtaggccaGATATTAGCCaatttcggtcaatttt +tAGGGGCatcattcaaaagtttcatccTATTTCCTCAGAAAacctcaaat +ttcaatttttcagctcacagctcagaaagcaaaaaatgaaaaacctgTTC +GAACCCGGATCGTGAGATTGGTAATAATGCATCGCACAGAGCCATTTGAG +CAgatggctcaaaaatggtcgaaattcggttttttttttggaaattagtC +ACTatatggttactgtagactatTTTTGGCCCCGCAGAGCTCTATTTTGG +GCTCCTACCTTGACTATACCGACTGTATATGATGGACTGTGCTTGTAGAC +ACAACACAATCCTATGCGAATCAGGCATCGTCGAGTTGTTGGCACTCTCA +CTGCTGAGCAATTCATAAGCCGCATTGATCCTCTCGAACATCTCTCTGCC +CTCTGGATTCTTATCCGGATGGTATTTGGCAGCCAGCTTGTAGTACTGAC +GACGTATGAATGCCGGTTTCCGATGCTCCTCATTGCTCAGATCCACTGAT +AGAATCTCGCGTGCCAAGTCGGCAGACATCGTCGATGGCTTCTTCTCCAA +CTCGTTGTACCATGTAGCCAGACAGGAGCGGAGGAAAGGGATCGGATCTC +GAATCGGCCAATCTGGGAACCTCTGTCGATTACACAGATGCCTCAAATAG +TAGACGTGACAGAAGAGCTCCTGAGTCAATTCTGGATAATCAATCAGTGG +AATCGGACAGAATTGATACAACGCTCGGACGTTGGAAGTGAGTCGATGTG +AGAAATCTGCCACGTGCACAGCGATCCTCTCAATCAGATGACGTCGCATC +GCAGTGTTCCAAATGATTTCCGGATTATCAAACTCTCCGAGGAACACCTC +CGCATACTTCTCCGGTCCATACTGCTCCAAATAGAACGTGGCGGCTTCGG +GAAGCAACGTGGCGAGAATCGATTGTCGTGTGCTCTCCAGATGAGGTAAC +GTCGAGCGATAGGCCTGCTTCATGTGGGTGTAATGAAGGAAACGGGCGAT +CGGGAGCACGTTGGATCCATTGTACATTAGAATGAAATAGAAGACTCCAG +AGAGGTAGAGACGGGGGAGGAATGGGTTGTCTTGCATGACTAGGTAGAGC +AGGGAAGCCACTCTTTGAACGATTTGAGGTTCGTAGGTGAGAAGAAGTTG +GACGAGTTGGTAGAGGAGGACGGGCTCCGTGAGCTGCTTCTTGACGGATG +GCATCGGGCGGACGACACAGTCGTACGAGTCCCTGGAACGATATGGAATT +GTAAGATTCAAAgattaaggatctaaggatatacggatctagggatctag +ggatctagggatctaaggatctaaggatctaagaatcggaggatcaaagg +atcggaggattttcagagctaatgatctagagatttaaggattgatctag +ggttcaaagaatctgaggatctagggatctggagatctagggatctatga +atctgaagatctgaagatctagggatctaatgatctagggatctaatgat +ctagggatctggggatctgggaatctgggaatctggggatctaGGCCGAC +ATATCAAAATGAACTAGAAATGACCAAATATTCGCCTAGTGTGCCTAGCC +GAGAAATCCCgagttgagaaagttaggccaccatgttgatATGGCTTAGA +AATGGTCTAATATTCATCTAGTgtgtttctaggtcatttcaacatggtgg +cctagaaaactcatgttgagaaagttaggccatcatgctaaaatggccta +gaattggTCTACATTTCCCCTAGTGGGTTATCTAGGTCATTTCaacttga +tggcctagaaaacccaagtTAGGCCattatgtcatttttcaaacagttgt +ctctgaaaaatttgaaattcaaggaTCTAAAGGATCTAAGAATTCAGGGA +TCTAGGGGTCTATTCGAGAATCCATAATCCTAGGGcttcagaatctccag +aatcttGAATTCAAGAATCTAAGGATTTgaggattctggaaatctaggat +tctgagattcgagAAGCCAGAATCCTTGGTCTTCAGAATCTCCATTCTCA +CCTCGATGGAAAGAACTCGCACATCTGCAGAATCGTATCCAAACAAAGCA +CACTCAAATCCGTGAAATTCATCTGTGTCGCTTGGAATGGTCCCTTCCCA +ACGTTGATCTCCTGCGGGATGTCTTTTGCCATCTGACACACCGTCCATCG +GAACTGTGGCACCGCCGCGAGACTCATCCATTTATCCATTCCAGCTGCCC +AGATCTGGGTCTTCTCAAACACCGTCTTCTCCACGAACAACGTCTTCATC +TTCTCAAATGACAATGGTCCCACTTGTTTAGCCTCCTTATCATGATAATA +CCACTCCTCGGATCCTCCTTCAGCCATTTGCTCAGCGCTCGCCTCGATCA +CATTCGTctgattttgaactttggcTCTTTGAACGTGCAAGTGTGCTAAA +ACGCATAAATCCACTAGGAGGGGCAGGACATTGGCAATGATCAGCTCTCG +AACGTTATCCTTGTTTAGAGCCAACTTGGAGAGCAGCAGGACTAGATGGT +CTCGTTCCAAGGGGTTGGCGCATTTTTGGAGCATCTCCACGAAATATTTG +GCGTCCTGGAATGGTCCGATTGTCATGTGATGGCGGGAATAGGTGATGGA +CATCGCACGAAGGCACAGGCACTTCATATCGGTTTTTGTGGAGAGCAGGA +AGCGGTGATAGACGTTGTTGAAGAATTCCAGGCTGGAAATGATAGGAGAA +TTGGGGATTTTTCGGAGAAAATTCtagcattttcagatttaagtCTCTGG +AAAGAGTTGGAATTGTTGGGGTTAAGGATCTAAAAATCTGACGATTCTGG +AACTCCTGGATCTCGAGAATTGCGAATCCATCTCCAAACCtctgggatct +gggatctaaggatctacggatctacggatctacggatctacggatctagg +gatctacggatctacggatctacggatctagggatctagggatctaggga +tctagggatctagggatctatggttctatggatctagggatctaaggatc +tagagatctagggatatagggatctagagatctacggatctagcgatcCA +GACATTCGAGAACCTAGAATTTTGGGAATCCAGAATATGAAGACTCCAGG +ATCCAAGGATTCAAAGATCGGAGGATCGGAGGAAACAAGAATCCTGCGAT +CTACAGATTCAAAGATCccgaaaaaagctgaaaatttcaaaatatggaTC +AAATCATACaatttctggctgaaaatacgcactattttcagccgaaattc +actattttcacagtcgaaaattcataatttttttgtctcaatTTGACAAA +TTCTAGATCGAAActcatgattttcagatctaCATGTGGAATTTTCCGCG +CTAAAAATTCACTATTTTTAGAGTCGATTCTGAgtcgaaaattcataatt +ttgggTCTGAACTCCTGATTTTTCTGGTCAAAATTAATGATTTTCCGATC +GAAATGTGGGATTTTCAGCGCCGAAATTCgctatttttaggctgaaaact +CATAATTTTCGGTCTTAATTTCACGAATTCGAGATCGAAActcatgattt +tcagatcaaaatttggctttttGAGCGCCGTCATGTTAAGCCGAAATTCG +CTATTTTCGGGACTGAAAACCGTCATTTTCAGTCCAAAAAGTGTTTAAAA +CGGTGAAAATTCGcacttttttcagccaaaattcaCCATttttaaagtcg +aaaaaaactcatatttttgggtctcaatttcacaaattcgagatcgattt +tatgattttcagagcgaactttggaattttcagcgccaaaaagtgttttt +gaaaagGTGAAAATTCGCACTGTTTCCAGCCAAAATTGACTATTTCTAGG +TTGAAAAATCATAGTTTTTAGTCTTAATTTCACAAATCAAGATCGGAATT +CCTGACTGTTCAGGTCGAAATTAGGCATTTTCAGCGCCACAAatgccatt +ttcagacaaaattcacaattttaaagtcgaaaattcaataattttggttc +ttaattcctgatttttcagatcaaaatttcaaattttcagggccaGAAAA +CTGTCAATTTTTATCCTGGTTTTAAGTATTTTCTGGATGAAAAttagcca +ttttttctcaGCTAAAATCATTAGTTGCAGCTTTTTAGGACTAAAAACTC +ACGGATTATGAATAGGTGTCGCATTCTCATCGGCCTCAATCAACAACAGT +CTCAAGTAGTAGTCTCCAATCTTAATCTCCTCCAACAGCGAAGGATATCG +AATCTGGAATTCCGTATGATTCCAAGCGATCGGCAGTCCAGTAGGCGCTT +GTTCCTTCTCACTAGCCAGAGTTCGCTCCTCGTTCTCCATCGAATGCCTA +AACTCCTCTCGCGTCTTCTCATTCCAAATCAGATCCGCCTGCGAGTGATC +TTTCGAGAACTGATAAGCGAAAAGCCTCCAATTAACATTAGGGCGCACTC +GATTCCTTCGTTTCCGAAGGACCACCGGTTGCTGTCGTTGTTGTTGCTTC +TCGTCGATAACTCGTTTCGGTAGGAAGTTCAGCTTCTGCTCCAGATCCCA +GTGCTGCACAATCTCTCGAATCCCGCCTCCGCTGTCACTCTCAGGTGATC +CAGTTTCTCTTTAACCGTGTTCTGCTTCGATTCATTGGTAGCGGCACTCA +AGTTGTTCCTGGGGATCAGCAGATCCTTCTCACTAATCGGCACCTTCTCA +TCCGACTCCATAAAGTTTAGCAGGCCACGTGGCAAGCATCTAACCAGTAG +ATCATTGGCTTGTTGGTTATCAGCAATCCACAAACTTATCAAATGCCCTG +ACAATTGCTTATTCGTCATTCTCCGTAAATCCTTTCCTACTGTAAGCAGC +GCCGTGTGTAGATGAGTCAGAAATGCTCCTTCTGACAGTGCCAACATCTG +CATCGATTTGGAAGTCTCGACGTCCGCCTCCTCGATGATAGCTCTCATCA +CCATCCCTGCGCCTTTGACAATTGTCATACTGGGACACTGGAAGAGACGG +AAGAAGCTACGACCACGTAACGCGACGAGCTTCAGGATATGATCAAAAGT +GTCGCCAGAAGTGGTCTCCGAGTATGGGGAGCAGACGGAAAAGGTCAGGA +AGTTGAGCATAGAGGCGATGACAAGCCAACCGGTGGAGCGATCCACGTGG +AGGACGATAAGATCCAAAAGGTTCTCGACAAACTTCGGACTGCTCATCAG +ACTCTGCTTATTCAATTGCTCAATCCTCAATTCATATTGATCATGCATCG +GATACATCAGTGCACACAGAGCCTCCACAGTAGAATGATCGATAGATTCG +GATTTCCACGAGAGCACACGAACTACAAGACTTCCAAGCTTCTCGCGGAC +TCCCGTGACTTCTGTGAAAGCCTGAAATCCGCTTTTCGATGCAAACAGCC +TTCTGAGACACTGTAGCTGTGCTTCTGTCTTATGCTTGTACTCCTTCTCC +TCCTTTGTATAATTCTCGCCTAGCACTGCTTCAATAGCGTTGACAATCAC +TTTTCCCTTATTTTCCGAGAAGAATCCTTCGTGAGATTTCGAGAATCGAA +GGCCTGAATACGGGACGTTGGCGTTGAAACGGCGGATGGTGTCGCAGCGA +CGAAGGCCTGGGGGCGGGGCGATGATGTGTTTCATGCATTGAGACTCGGC +ATCTTCGTCCAGTGTCGTTGAGAATGGAATCACTCGGAGATTCCTCTCGA +AGCGGTGACCGCACACGAAGACTTCTTGATTTCCAGAAGCACGGATTCCG +TCGAGAAGAGAGGCTAGGATGAGGTCTCGTTCAGCTGCCACGTACGCTCT +TCCGTCGCCGTTCTCATATTCGACGATGAATTGTTGAGGGTCTTTCTCGC +TTCGCACGAGGCAAACGATCTGGAAATATgggttttaggatttttagagt +ttttggaatttcatatttttagaatcttggaatttcagaatctctgaatt +tcagattcctggAATATCAGGACTTGAGAATACCAGAATTCAAGCATCTT +AAATTCTCGGATTCTTAGAATTTacgttttcgaaattttcagattctcga +aatttcaaaatttccgattcttgtttttgagctttttggaaatttcagat +tttcaagtttttgaaatttcagaatttcagattttcagatttttggaatt +tcagaatttaataattccagaactccagaatcttagaattttagattctc +ggattctctgattctagaaatttcagattttcagaatttttcgaatttca +gattttgagatgaaattcttagaattacgacatcgaaattttcagaattt +ttggaagtttagaatttcagattatttaaaatttcagattttcagattct +cggaATATCagaacctaaaattttcagaattacagattctcagaatttca +gattttcaaattcttggaatttcagattttcagaatcttggtgtttcagc +tttttcaaatttcggattcttggaattttctgaaattccacatttttttt +tcaaaaatctgaaattagaagcatcggaatttctgaaattccgaggatat +gtaattttgaaaatgctagaaaaatattttcaaaaatctaaaaattctaa +aattctgaaaatttgaaaactccaaaatttcgtaattaccgattcttggt +ttttgaggttttcgaaatttcagatttttagattctcggAATATTAGACT +TTCAAATTCctggaatatcagaatctaacattttcagaattacagatcct +cagaatttcaaattcttggaatttcaggttctctgaaattctatttttca +ggcgtttttctagaattttcagcccaaaactAACATGTTTCAGCGGGGTA +GCACAAATGACTGCATACGTCGCCAAGTCCCTCTCAATGATACACGTCTC +ACTCAAACACAACAGTCTCCGCACAGGCAATTCGGCTCGTCTCGTGATTT +TTGACACTTTGAATTCTGAATACGACGTCAACTCTTCGTCACGACTACAT +AATCCGAGGCGGGTCTTGAGGAAATCGTCGAGTGAGATGAGTTCTTTGGC +CATTGGAATCGTATAACCAATATTCTCAATTGCCAGACGTCGACATTCTT +TGATTAGCTCTTCTTGATTCGACGAGGCGAACATGTGCTGAAAATAaggg +attttcagacaaaaaatatagcgaaaaatccgattttcagggtctaaaat +tgccaaaaaaaaccCTGATTTTTGGCGAAtttttcgtctagaaaaggtga +taggtttttcagccatttttcagattttaggtttttcagccatttttcag +aattttttgtctgaaattctgatggAACGGCGTTCTGGATTACtgcaggt +tactgtagaaaatcccatattttgtagtttttttctcctgcaaacacaat +cacgtcaaaggttttctacagtaatctacagtaatcttcaatATCATCTC +CCGTCGATTTTCAggcgaaaaattcggaaaaaaaaagtcagattCGGGtc +taaaatgggtcaaaaaccGCCGCAAATTTTGaggaaatctaggtcatgac +ctagaaaaatttaggccatgacctagaaaaatctgggtcatgacctagaa +aaatctgggtcatgacctagaaaaatcttgtttttttcggtttttttttt +catttttttttcagaaaatcttcaaaattctgttttcgGTTTCGTTTTAG +CATTAAAACGTCAATTTATTCAAagaatttgttttaaaaatctgattttt +ggcTCCTTTTTTAGGTCATAAGTCAGTTAAAATGTGACATTTCAGGACGA +GTTTTGactctaaaaatcattttttccagccaaaaatgCTCCTTACCCGT +CTCCTATGCTCGCCGACATCGACCACAAACGCTCCGGGACAATCGGAAAT +CTTTCCGATACTTCGTATATTCTTATACGGATACATCTGCACGACGACAC +CGCGATTATCGATTTGTTCCAATCCGAAGGCGTTGGCACGGAGAGTGACT +TGGATGCGACGATCACTCCAGCTCTGTTTGAATGCTGTcactgtctgaaa +ttttggggaaaataTGAGGATTTCCatgtgtccagatgtccgaAAAGCAC +AGAAAatattacagattttcagctattttcagcttttaccgaacattttt +tgaatttttagggaCATCCAAacggcgccaaattcaaattttgagacgTC +AAAAGATCTCAGGGCCTCCAAATTTATCGTTAAAAACAAAGCTCACCACT +GGATCAAAGACCTTATCAGCGAATTTGGTATGAAATTGAAGACAGTGCGT +CAGAATGTCGGGTGTGTACTCCGATGAGAACTTCATATCGTCCTTTTTGC +CACGTTGTCGATACCAAATCTTGAAGCCAGTCTCCTCTTGACGAGTATCA +TTGGGTTGGAACGGTTTCACTGATATGAAGTCTTCGTAGAGCCACTGGAA +ACATgcgaattcagatttttagaatttttggaatttcaggtttttgagat +tctaagaattctaaaatcaaaatttttggaatttttggattttcagattt +tcagggtcttaggaatttcaggtttgcgaaatttcagaatcatcagaatt +tcagaatttcagattttcaagttttttgagcttcagattctcagattctc +gaaatttcagattttcagattgttctaattcagatttttggaatttcaga +atctcagaatttcagattcctggAACATCAGATTCTCGGatcctcagaat +ttcagaatctgagattttcagaattacaggttctcgaaatttcagatttt +cagattctaagaattccaaaattaaaattttcagaattttggaatttcag +attttcaatttttcgattttttgcaaattttctagaccttttttccggtt +atttttgtcaattcttcttccattttcccgatttttgtctctgaaaatct +taaaagattactgtagaaaacctggtattttggagtttttttaaatctgc +aTTTCGACGTGATTGGGTTTGCAgagaaaaactacaaaactttgtacagt +aatctacagtaacccaggaCACCGTttcatcagaatttcagacgaaaaaa +accagaaaaatcgaaaaatatgttttgaaaatagctgaaaaacctggaaa +tgtcaaaaattcttgatttttgcccattttcagTGGAatttcggtccgca +ttttgccatttttgatgattttccagctcatttcaccgatttttcagatt +tcaaaatgaaaatttagtGATTTCTGACGATTTTCCAGCTAGtttgaagc +caaaaatcagCATTTTGAGACATAAAATCTCAACTTTTGCCTATTTTCAG +TGTAATTCAAGccagagatttcaaaatgagaatttagatcaatttagacc +aatttttaccaatttttatCAGCCTTCTCCCAtattttcttcagatttca +aagttttcagtcaaAAGTGCTACCTGGTTAGTGATTTCCAACGAGTTGGG +ATTATAGGTTGTGATGCCAGCGTTCCGATCGAGAAAACCCGTTTGTATCT +GGAAATTTATCATTTCTACgggtagttttttttcaattttttgtcatttt +tgctcttttttaaaacctaaatttggctaaaaatcgcgaaaattcccatt +tttgactaaaaaatctgacttttttttcgaatttctgccagaaaaagctc +tttttttctgacattttcgagcccaaaaatcgctattttcgGCCATCTTT +TGCTCATTTCTGAGCccaaaaagtcggaaaatagacaattttcgactttt +ttttcgaaaattttgacatttttgactgcgaaaaccgcaattttcagcta +ttttttgctcatttctgACCCCAGAAAGTCGAAAAATAGGCAACTTTCTA +CTTTTTTGAACGTTTTAAATGCATTTGTTACATTTTTGACTGCAAAAACT +GCTCttttcggctattttttgctcatttctgagcccaaaaagtcggaaaa +taggggatttttgacacttttgacTGCAGAAATCGctattttcggtcatt +ttttgctcatttttcctctattttcctATTAAAATCCTCACTTTCCCTTC +CATGAGTGTTTGGTGACCAAGTAGCACGCGACATCGCGATTCTCCGTACA +AAACTGATTGGCCATTTCGTTCAGTTTGACATTTACActgaaaaatagcg +aaaaatccggtgaaaaatccccaaaaaacggcataaataacggaaatttc +gcaaaaaaaatgcaatttggAACTAATTAACGGCGATAATTGCGGTGGGA +AAATTggcacaaaaaagttatctCTCGAGAGAAAAATCGAGTGCGGCGGA +GCGCGTTTTCCATTCCCACGACGTTGACTCATTTGAGTAAATAGACCGGT +TTTTGTCATgtgaaacatgaaaattgataaaaatcctggtttttgagctg +aaaaatcatgaaaattgggtgatttttagaggaaaattatcgattttttg +ggggtttttgttttgaatgaaattgtATACAAAAAGTGTCGAAGGGGGAC +AGAGAGGAAGAGGAGAGACGGACCAGATTACGAAACGAAACACGTTATAC +AATTGATGGAGATCGCGAACGGAGAGGAGGAATTCTGAGAACtttttttg +cgggaaatttgaaattttttgaaaagttttgagagcgaaaatttgaaaaa +ttgcgagtctacaattggttttttgaataaaaattttaattttaaatttt +tttttggattttttcgagaatattcagttattgttttttttctgtcctga +aaaaaattcagccgctggctaaattttgaagtgtgaaaatctgaaaaaat +ttgaatttttaattttttttttcgaaaattttaaatttgcttgaaactcg +attttttaccagatttttatgacaaatttgcgtattttaaaaattttttc +gatgttttgtttcggaaaatcgaCCATCGCATCGGCACAACCTTACAGAT +TTctcattagaaaaaaagggtGTTACAACCTAAgcctcaaatttcaaaca +aaaaggcaaaataatctgaaaactccaatcaaaaagtctttaaattccca +aaTattagcaaattttacactttttttagcaaattttacactttttatta +gcaaattttacacttttttcactgcaaaattttcagaatattccgttttt +tgttctatattttAGGCCTGAGGCTAAAACACCTTTTTTACGACAAGAAG +TTTCATTCTAAGCCTTTGTATGAACAAAAAACACGGAACATCCTGAtcat +tttcagggaaaaaagtgaaaaaatagaggtgggcGGGATTGCGCTCCACC +GAAATGAGCAGTTTCGTTTGTGAAATTCCGTCGctgttctggaaaattcg +aatggaaattcgaaaaataccctttctcttttttcgaggtttttctaaat +tttcagatttttctcgcACTTCTGTCGCTgaaaaaaagcactttataacg +aaaaatctgaaaaaaaggaaaatccgggaatttgaaaaacctcaattttc +agattttcagacgcATTATCTTTCACTTCAgacttattttcagaaaaata +ctaaaattatTAATTCACTCTATGaagaaaaacctgaaaattccatCTAA +AATCTAGAATAACCCACTCAAATTGATGAGAcacaattctcattttttga +aaaaaaaaacgttgctCGTCAATTAAAGTGCAAAAAATTGCGAACACCAC +CACGCAAAATCACACACACcggagagaaaaaatggagggagatttgaatt +tttcaactttttcagagaaaagtgattttcaggCAATAATTCCAAACAAA +ATGGACACCCGGACACACAAAcagacctgaaaaaaacaaattttacacat +ctagaatttccaatttttgtgacCATTTCCTGTCATTTTCTGTccttatt +ttccaatttcctgcTTTCCCACACATTCCAAATTAGGCACATAGGCATAT +AATACGCCTCCTTTTCCCCCCTAgtttattccaattttccatgGCAACCA +GAAGCAGACACCTGCAGAGGTGAAACTAGCgagagggagaaagtgtggga +gatgaagagacgcagagaaagagagagggaggaaatagggagaaCGTGCT +CTCATTATGCGCTCTTTCAAGAGCCCAAGAAGCTGTTGTGCACTTAGATT +TCTGCAAAATAcctttattttcattttgcaagaattttcttagattttta +acctttataaaatttaaattcaaaatttgataaaatgtaaATTCGCTCTA +CTgttaatctaattttttttcaaactttcttttttaaaaacggattaaac +ttgcgtttttttGTGAATGaaaaataaggcaaaaagtagtttttccttca +aaaaaatttttaacataaaaaaaagatttttagcaaatgcgctctaatga +aaaaaggtttttgaaaattatcattagagcgcatttccTCAGTgaatttg +tttgtttttaaaaacttcccATTTCCCTATAATCGTAGAAACAAAATAAA +CAAAGCCtgaatatcaaatttccaattttcatattctaaattttctagaa +ttttccagaaaaatacaaaattttggggacaatttttttcagaaaaataa +ccatggcatctgaaaataaggaccaacgaaaatcgtattttctcCGGGTG +GCGTCTTATTTGCTCGGATTGAATATTGTTgaggaaaagttgaaaaatac +agaGCCATTGGAGACTTTTCTAGACTCAAACACCAATTTATTGGTTTTCT +CGCGGTCTGAACAGGTTGGTTTTGTTAGAATTGGATTCAGCTTgttgaaa +gcttcaaaatgagcctaaacacgactatattCCTTTAAAagttgggtccc +accacgaaaggtacagtaatccccaaaaaaaaccgaaattgaCCTAGATA +TGTCGCGgttaagctcattttgaagctcatactgtgctgaaaaataataa +aaaaatttcatgaagaaatatccaaaaataagcaaattataagaaattta +agatttcctaaaattttcaaaaaatcaatttgtgtcataatttttgtatt +tttgattttctttgaaCGAACTCATCGTATTTCAATTCAGTTTGAtgagc +gcttcaaaatgagcctaaacacgactatattCCTTTGAAAGTTGGGTacc +accacgaaaactacagtaatctcgaaaatttccaaattgacCTAGGTATG +TCgagtttaggctcattttgaagctcttgacacGCCTAAAACGATTTTAG +TAATTTAATGTTTAAATTTAGAAACATAAGGGAGATataagaatttttag +attttctttaaaatttttttgaaaatcaatttttcgccacaattcggtta +tttttgaacttttcagactgaaacctattttttctaattgggCTCAataa +atgcttcaaaatgagcctaaacacgactatattCCTTTGAAagttgggtc +ccaccacgaaaagtaccgtaatcccacgaaaaatcaaaatgcagctagac +tagtcgtgtttaagctcattttgaagctctgaacaaGTTGAAAAGAAATA +GACTATTGTTGAAGagcaaatctcaaaattaaagaagttttcgaagtttg +aaaatttacgaaaaaaaaactcccaaaattaaatgttttcagaaactgGA +ACTCTCCAACAAAATGAAGTCTTCTGCTCCATCGGCCAATGTCCTCCGAG +TGGTTTTCTACAAAACGCAGTCTGTCCCTTTAAACAATGAGAACTTCAAA +AGTGTGGTCAATGTGATATCAGCCAATGGAACCCTGAATCATGTCTTTTT +GAAATCCGTTCAGAATGTAACAAACattctcattagagcgaattttcata +attttgaaattctaggttTTCGGAAAAGAACTCACCGAGGGTAACAACAT +GCAACTGATTGCGGCGGTCAATGAACTTGAGGAGAGTTTGCTGGCTACTG +TGGAAATGAGTGGaggtaagtgcgctctattgataaattTGTATTCAgct +cgttgagagcttcaaaacgagctcCAACACGACATATCTAggccaatttt +gaatttttcgggattactgtagttttcgtggtgggatcCTATTCAACGGA +TTAcggtcgtgtttgggctcattttgcagcattttttaactttaatgaaa +actttttaatttcGTTAAGAGAAAATCCAAACTATCGAAATAATGACAGA +AAAACGAtagtcaaaactttttcagaaaatctgaaatttcttaaatttcg +cctgtttttgaaatattgataTAAAATTATTGTGTTCTGATTCAGCgtgt +caagagcttcaaaatgagcccaaacacgactagtctagctacattttgat +attttgtaggattactgtacttttcgtGCTGAGACCCTATTCAACGGAAT +ATGGTaatgtttaggcttattttgaagggTTTAAAACACCAAACTGGAAT +AAAAAACTTCGATTTAGAAcattccaaaataaaataattcaggcgaaaaa +ttgattttccaaaaattttcagaaaatcataaatGTCTTTAAATTTGCTT +ATTGCTTTTAAATTCaaccaaaaattattttgattttttcagctcgttga +gagcttcaaaatgagcccaaacacgactagtctagcacacatttgatatt +ttgtagggttactgtacttttcgtggtgggacccaattTTCTTATATTCA +AAAGGAATgtggtcgtgtttgggctcgttttgaagtttcaacttcaaaat +ttcaacatttcacctcaaatttcaattttctgactttAGGAGGTTCTCTA +CACGATGAGATTCGAAGTTGGAAAGGACAATCGGGACGGGCGGCCCAGGA +CTATAACGATGCGTTCAAACAACTACAATTACTTGTGGAGACGATGGAGG +AAAGACGAATTGATGAGTTATCAGAACTTGTAGAAACATTTGAAGACACT +TGTGATGAATTATGGAATTGCGGAGTACCGTATCCCCAGCCAAGGATGAA +GATGCTGATCGAGTACGGATCGAGTTATTTGTGTGAGGCCATTACTTTCA +AAGTATGTGCCTATTTTAAGattttgttttggaatttctaggaaTATCCA +CTTAACAGAGCGTTAAGgctcatttttgtatttttcaacttttttctagc +ttttctagttcttgagatataaTCATATGAAGGTTTTCACAGTCTGTTAA +GGGTGTCttaattttgacctaatttatctcacttcttttgaaagatatca +aaaagtggtcaactgataaaatgtttatcataatatcagcaacattttat +caattgattacTTTCcaatatctttcacgggaagctagataaattgaatc +aaacTTTTAATATAAATGTTTGGTtatgaaaaccaattttctgtCCAAAC +TCTGctatttctgaactttctgttttgaaattgtaatATTCTAGTTTGTC +TAGTTAagctcttcaaaatgaatctAAACACAACTATATCTCTTCtgaat +tgaggaaaattggggTCCCAATacgaaaagtacagtaatcccacaAAAAC +TCCATAATTTTTCTATACTaggcgtgtttaggctcattttgaagctctta +atgTGATCAAGAATATGAAATTGGCTGAATGtcagaattttgggaattaa +GGCAATTATAAAGAGTTTATAAAAATTAGCTAGATGTTCTAGTGTTGGAG +ATATGTGTTTGCGCTCAAAGACGACATTCGAGTTAAAAGAAGGAGAATGT +CATCTTTAAGcgcgtatatctcaaaaaccaataaatttagaaaaacgttg +tcaactagagaaatgtacaaaattttatgggctacacttttgtagtttac +aaaatgAAGCTAGCTatcttagtttttgagatatttacCTCCAAAGTTAC +AACACTcttcttttggaatttgaagtAGATAAACGCTATTTTTGAGcgcg +tatatctcaaaaactaatgaatctataaaaaagttgtcaattgacaaaat +gtgtgaaatgttATGTGctatgtttttgttgtttgccAAATTTAGCTagt +tattttggtttttgagatataaccCTTCAAAACTAGAATACACTTTTATC +CCGCGAGTCGAGTCATGAGTCAGAGATgccttaaaattgaaaacctaTAA +CAAATTGCAGATCGATGACTCAGCGATCTGGCGAGACGAAAAAGTGTCGG +ATCAGTTGCGATCAGCAATCGATGTGTGTGATCAAATGCTGATCGTAATT +CGTCTACTAACATCTCAAACCTGGAAACGAAACGTCGAGCACACATGGGA +AGGCGATCCGATGGAAATGAAGTTTCTCAATGGTTTCAAAGAACGACTCG +ATGAAATTCTGTCGCTTCGAAGTCTGGGAGGCCAGTTGGAAGGACTTTTG +GAAGAAAGGGGCATTCGAGAGGAGACCGAAAAAACGATAGAGACGGCAAT +GAGAGGAATGGCACCAATGGCATATAATCCGTTTACGGAACCCAATTGGA +AGAGTCGACTGCTGGTGGCGGAGAGGGCGATTGAGGGGACAATTGATAGG +ACGCTGCCGATTTTAAAGCAGAGACTTGCACCGAGTAATGGAGATAGTCA +GAGTGTAAGTAGCCTGGAATTTCACAAGGTCATCGGGGACCAATAAGGCG +TTTTAAGCTCGGAAGACTCAAAAACCCCTAGAGTCACTCTCCGGCgcata +actcggttcaaaattgaaattcgtgtttgaattaaaaagtgaaacgTAGA +GTTTGACAATATCTACAAAATAATATTGAATTCGACTACTTTGGGATGTC +aagttctgaaataaaaaattttaaatagcAAAGTCCGCCAGAACGCGGGC +CCGGCTACTCTGCGGCCTCTGGTGAAACACTCGACATTTCAATATAGTAG +GTttgaatatcattttgtagatcttgtcAAGTGCTAAATTCTAAGTTtag +ttttgttcaaaaattccatttttgcgCCAAGTTATGAGCCTCTAAAGTTT +GATCTCCCTTGAAGCGCGCGCtgtcgcttctctgcgtctccccttTTCCT +CTactatctcaaaaactattgtcggaataaaaaaagtgagactcggtgtt +tgcgtactttgattcccgcctccccccaccgacgctccgcccatcgcttt +gcaagtgcggcgtggtgttttggCGCTACTTGGGAGATCATCGGatttaa +tgagattttcatagaaaatatgaTCCCCAAGTGGCGCGccaaaacaccac +gccgcacttgcaaagcgatgggcggagcgtcggtggggggaggcgggaat +caaagtacgcaaacaccgagtctcactttttttattccgacaataaaata +gctatgaaaaagttgtcaactagtaaaatgtgtaaaattttaaaatctct +atttttctagCTTACAAAAGTTTTGTAGCGTGTTTAGTTTAAGAGATACT +AGCCCCTGAAGTTGAGGCGCCATTCAAGTTGTACAGACAatattgaaact +tttgaatctcATAGAATAATAGCGCTAAActaaaattttaacaaaatgtA +GATCAAAATGTTTTACATATTTTGCTAATTGGCAAAATTTAGTTAGCGTT +TTTAGTTTTAGATGTAGACACTTTTAAATTGTTATCTACTTTGAAGCGCG +TTGTTATCTACTTTGAAGCTCTGCCGCCTCTCTGCGCCAAAACGCATATC +TCAACAACTAAAAACgctattaaaaagttgtcaactaacaaaatatagcc +cacaaaattttacacattttactaatttttaactttttgctAGTTATTTC +ATTTCTCGAGTTATAGGCGTTTAAAGTTTGACCTACTTTGTTGCGCGCCC +TATCGCACCTCTGCGTCTTTCTTCGACTTCAAAAGCGCGTATCTCAAACA +CTAGAATAGCtagtcaaattttaaaaactacaaaagtgtaGCCCATAAAA +TTatatacatttttctagtttacaacttttttctagctattttaaCTCTT +GAGTTATAGGCTTTTAAATTTTGACCTACTTCACTGCGCGCGCTCTCACA +TCagttttctgcgtctctctttccgtttaaaggcgcatatctcaaaaact +agaaaagctatcgaaatgttgtcaactacaaaaactttcaaaattttatc +agctacttttttatagtttacaaatttcagcTAGCTATTCTAGTTTTCAA +GATAATCGTTTCCAGATCGTTCTAAGCctggaaaagctgaaaagcTTCCT +GTGCCGAGCCAACATCAAGGAGAAACTTCAACACGAGCGGGAAATGTTCC +TGAATCGTTTGATATCAATGCTATCAcagaaaaatcaggaattcagCGAG +AAATCTCTTCAAGTCGAcgctaaaaattttcaatttctcacaGAAGTCGC +CGCGAGAATCGTCTGGATTCGGCAGCAGACAAGTCAGGTAAAGGACATat +ggacaaacagacacacagcttttttttttaaatttctttagATGGAAAGT +ATCAGATCGCTGTCGAAAATGATGTTGAACGATATCTCGAACTACAATCA +ATTCGCACAAAAGCTGgatgaatttattgaaaaactacagtatgccGAGA +AGGAGTGTTTCGATGATTGGTGTCGAGAAACCGTCGGGTTGATTGATAAT +AAAAATGAGACGATAAATCTAGAAACGACTGGAAAGATTATGTATTTGGA +AGCGTCGAATCGAGAGTTGAATGTGAACTATTCAGATCGTCTTTTGAGAT +TGTTGAAggcatggcctaactttttttctaggccatgtaatctctttttt +ccaggaagTCCGCCAACTGAGCAGTTTGGGATTCAACATTCCATCGAAGA +TCCTGTCATGCGCTAACAATGGAGAGAAGTACTATCGATTTGGTGTTATC +CTCAAAcagattgctcatttttataaTACCATCGATCAACAGATGATTCC +GTCGCAACAGGTAAGCATGATTAGCTTTAGAAGCGCATTTCTCTAAAACT +AAAATAGTTAACAAAAAGTTATCAGCTAttaaaatgtataaaatttttat +agactATATTGCATTTGCTaaaaaaaTTCAGCTAACTATTTTAGTTCATG +AAGTATTAGGCTTCGAAGTTAGGGCACATTTTTGTCCCGCGAGTCCGTTT +AAGAAAAGTTTaccttttggaaatttttggattttcggctgtttttttca +aaaatatggaaaaattggatcatgacctaaaaaaatctaggtcatgacct +gggaaatttaggtcatgacctgggaaatctaggtcgtgacctagaaaaat +taggtcatgacctaaaaaaatctaggctatgacctaaaaaaatctaggct +ATGACCTgtaaaatctaggtcatgacctaaaaaaattgagccatgatcta +gaaaaattgggtcaagtttttagtttttgagttatacGCTTTCAAAGTTT +AAGCATCTCTGACTCAGGACAAACGTGCATCCTAGTTTCGGAAgtttata +tctcaaaaactacaatagctagcaaaaatttgtcaactacaaaatttttt +gaaattttaagagctatATTTTAAGACTTAACACATTTTAACTAGTTATA +TTAGCTTCAGAGATATTCGAGTTAGAAAGCGCGCGCTCTGGATTCTCTTT +GGTTCCAACTTTCTTTTACTTTGTAGACgcatatcttaaaaactaagaca +gctatcaaaaagtcgctaactagtaaaatgtttaaaattttatgggcttt +atttttgtagtttacaaaatttaccTAGCTatcctagtttttgagatatt +catGTTGAAAGACAACccacttgaaaattttatatctcaaaaactagaaa +agctatttcaatttgattttcagtctcTCATGCTGGAAGAAGCAATTGCA +TTTGAAAAACTGGTTATTCCAAGGAAGGATGCCTCCAACTCGGCTTCGAA +AGTAACATGGAACGATCCAAAACAACTGGAAGAGTTCATTGTCCAACTGC +AAACCGCCGAACAAAAATTGTCGAATCGAAACCGTCGGTTGCGAAATGTT +CATATGGAATTGATAGAAATGGTTGAGAAATTGATGGATTTGAATATTGT +GAAACAGAATAATGAGTGGAAAGAGATTATTCTGAAGATTCGATCGAAAA +TGAAGGAGGAAGAAGTTGTACATGGAGCGGCGAGGAATAATATGAAGCCG +TGGTTGATTCATTGGGATTTTCAATTGTATAAGGTGATTTGGCTCTACTG +TAGAATTTGGGcctcaaaaataaatatggtCCTTGAAGACAATTTTGGTc +cgaggtttttgaaaaattcggaacttcagaaattcagaaattcagaaatt +cagaaattcagaaattcagaaattcagaaattcagaaattcagaaattca +gaaattcagaaattcagaaattcagaaattcagaaattcagaaattcaga +aattcagaaattcagaaattcagaaattcagaaattcagaaattcagaaa +ttcagaaattcagaaattcagaaattcagaaattcagaaattcagaaatt +cagaaactcagaaattcaggaaatcaggaaatcagaaattcggaaattcg +gaaattcagaattttagaagttcagagaatcagaatattagaagttcaga +aattcagaaattctaaagttctagaattctacaaattcaaaattttgaaa +attcaaagttccagcaattcagaaaattaaaaattcgggaattcaaaagt +ttccagaatctcaaaatcttagaaatcaaatattctgatatttttttaaa +aattccagaaaagtcTAAATGtacagaaattctaaaattctaagattcta +aatttctagaattctgaaaattcaaaaatccagcaattcggaaattccta +tttttcttagggttttcagaatttcaaaagttctttaaaaatttcaaatt +gttcaaATCTTCAGGCTCTTCTAATCCAATACGAATGGGGAATCGAAAGT +ATCCAATCTCAATTATCCACAATTTCCGTCAGTCTAGTTTTTGTGTaagt +gtcgatttacgggctCTTTGATTCCtcgattttcggtttttcagcGATCA +AAAAATCCAACTGCGTCCGGCAATCGAAGAAATCCGTTCCAAATATTACA +AAGAGCTCTGTCGATTTTTAAGAATTCCAGACAAATTTCGAGGAGTTCAG +GAGGATGAGACAgtaagtgcgctccattgataatttctgtttttctgaat +ttctagaatttcaaaatttagaatttctgaatttctgaatttagaaaaaa +ttgtgttttagTCAACCAAATTCTACGCCCAAATGATCGAACGAAGTATG +CATTTGTTGCCATCAGTCTACGAGAAAGCTGAGCAATTGATGCAAAAAGT +GGAGACTTGTGATGCCATTTTCGTGGTATGCTTATcggtttgtccggatg +tccgcaattgaaaaaattttcaggattggCTAGTCATCTCCCAAGTGGAC +CTCGAGGAGCTTattgaggaaaatttgaaaaccgcAGCGGATTGGGAATC +GCAgtttaagattttgaaagcCAAAGCAAGAGAGGCCGAACGTTTGCCAC +AGTAagattggattactgtactttagaaaattcgaactGCAAGTCATTAA +GACTTTGGTTCAATTAGATTACCGTACATTTGTTCTAGCAAAGCATTTTG +TTCATCACAGAGCGCACAGGGTTTGTTTTTCTATCTTTGCAAACTCGTTC +TATTGATAAGGGGCGGGGCGAagctttttctaattttcaaaaatgaaaaa +attctcccCTAAAACGAGAGATCTTGTTTCGGTGAAGAGACTCAGAACAT +TTCTGTGAAGAGACGAAATTTcggtgaagagacgcagacaacgaaacgaa +acgtttgatcctcagccatcGATTTGGCTGCGTTCTCATTAAAGCGCATT +CACAAAGATAGTAAAACAAACGCTGTGAGAGCGTTTGCGAGGCTTGCAGT +TCGGTTTTGTTAGCTGCAGAGATTACTGTATATTTCTTTCAGTGAACTCA +AATTCGAGTGCATATTGGTGAGTACTGCTGGTGTTAAAACTGCAATCGAG +GATGCGATTCAGAGATTGTACGATGCGCTGACATGGACTTTAAGGTAAGT +AGGACATCTGGACACGCAAACACACagaaaacccaaatttcagacactCC +ATCTCCACGACGTCCACCTCTATAAGCACATTTCTCTCTCAAGCTATCGA +AGTCCTGAATACAGTACCCGGATCCTTGGATGAAGTCGCCGAGGCCAATG +CGAAACACGTCATCTTTGCTGAAACGAATCgtcagctgaaaatggaatgG +AAAGTCATGGAGGAGCAATTGACACTGTTGAGATCAGTTGCTGGTCAAGG +AATGGAGCAGATTGATAACTTGGAACAAACATGGGATCGATTTGAGCTGA +TGTTAGATGCGCATCAAGGAGTTATTAAGGAACAAGTCGAGGCGTTGAAG +GTAGTTGACCcctaaaagaagaaaagtttgacccaatttatctgggttct +tgtaagagatatcataaacgtgtcaactaataaaatgttggagatattat +gataaacattttatcagttgacaaatttttcatagcttttaagggaacag +agataaattaaTTAGGTCAAGTTTGGGTCAAAATTGGGTGTAGGTAACTT +CAAGCttgcatatctcaaaaactaaaatttctagacaaaatttgaaaact +aacaaaatgtagctcatgaaattttatacattttactagtttttaacttt +ttgatatctcttctagcaacaatcaaaaatttgacccAAAACCGCAATTT +CCTTCCAGACCAACGTTGAAACATCAATCAAGGGAATGAAGGATGAGGCG +GAGAAGCTGAAAGCGAGATGGGACCAATTCAAGCCAAGAAGTGATGCATT +ACAAGGCGATCGAGAGGAAATGCTAAAAGCGATTCAGTTTATTAAAGAAA +AACGAGTTCAGTGGCAAGAGCTGAGTGAtggaagagagaaaattgaGTGA +GATGTTTTGGTTGTTTTACTGTGAATGTTCTTGAGGCACATTATCTCCTA +TAAGcgatgttaaaaatttttcaactaaaaaatgtttctgataacttgat +aaacattttattagttgatcattttttgatatctcttaaagaaattgaga +taaatgatCTTAAAGTCGGCAAGCACAGTTGTGGCTCAGTCAACGGCGAC +GGCCGCGTCGCAGACCATAACGGCACGtattttgagcctaattatctcgg +ttcctataagcgatattaaaaatttttcaactaaaaaatgtttctgatat +cgtgataaacattttattagttgatcactttttgatatctctaatagaca +ttgagataattaagcttaaagtaGGTAAGCCCAGTTGTGGCGCAGTCAAA +GGCGCCGACCGCGTCGCCAGTGGCGgcttttttgagcaaaattatcttgg +ttgctttaagagatatcgaaaatctgtcaactgataaaatgttgctgata +atgtgataaacattttattaattgaccactttttgatgtctgttatagaa +attgagatagtCAAGCTCAAAGTCAGCAAGCACAGTTGTGGCTCAGTCAA +CGGGGCCGACCGCGTCGCCAGTTGCGGcgcattttttgagctaaattatc +ccagttcccataagagatatcaaaagttggttaactaaaaaaatgttgct +aataatgtggtgaacattttattagttgataactttttgatatctcttat +agaaattgagataatgcAGCTCAAATGACAAGTAAAAAGTAGCGGTCTTT +CGTACATGCTTTGCTTCGCTATGATACATATCTCGTTAGGGCGCATTTGC +ACAAAAATGTTGTCAGGTACAAAAATGGAGCTCTTTTCCTGTTAAATATa +atccaaacttttcaaaacaataGAACAAGTACTAAGTACGTTTGCAGCCG +CTCAAAATTAGCACATGGGTGAAACGTAAGGATCGAGGCCGCTACCTAAA +TGAACCTCGATAGCTACACTTCAAGCCTAATAAACTCAgttaatataccg +taaaacctctaatagaacgacacccgaatcgtccactttgagcctatata +tctattttctgtgaaagatannnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnaaaaTTTAGTTAGCGTTTTTAGTTTTAGAT +GTAGACACTTTTAAATTGTTATCTACTTTGAAGCGCGTTGTTATCTACTT +TGAAGCTCTGCCGCCTCTCTGCGCCAAAACGCATATCTCAACAACTAAAA +ACgctattaaaaagttgtcaactaacaaaatatagcccacaaaattttac +acattttactaatttttaactttttgctAGTTATTTCATTTCTCGAGTTA +TAGGCGTTTAAAGTTTGACCTACTTTGTTGCGCGCCCTATCGCACCTCTG +CGTCTTTCTTCGACTTCAAAAGCGCGTATCTCAAACACTAGAATAGCtag +tcaaattttaaaaactacaaaagtgtaGCCCATAAAATTatatacatttt +tctagtttacaacttttttctagctattttaaCTCTTGAGTTATAGGCTT +TTAAATTTTGACCTACTTCACTGCGCGCGCTCTCACATCagttttctgcg +tctctctttccgtttaaaggcgcatatctcaaaaactagaaaagctatcg +aaatgttgtcaactacaaaaactttcaaaattttatcagctactttttta +tagtttacaaatttcagcTAGCTATTCTAGTTTTCAAGATAATCGTTTCC +AGATCGTTCTAAGCctggaaaagctgaaaagcTTCCTGTGCCGAGCCAAC +ATCAAGGAGAAACTTCAACACGAGCGGGAAATGTTCCTGAATCGTTTGAT +ATCAATGCTATCAcagaaaaatcaggaattcagCGAGAAATCTCTTCAAG +TCGAcgctaaaaattttcaatttctcacaGAAGTCGCCGCGAGAATCGTC +TGGATTCGGCAGCAGACAAGTCAGGTAAAGGACATatggacaaacagaca +cacagcttttttttttaaatttctttagATGGAAAGTATCAGATCGCTGT +CGAAAATGATGTTGAACGATATCTCGAACTACAATCAATTCGCACAAAAG +CTGgatgaatttattgaaaaactacagtatgccGAGAAGGAGTGTTTCGA +TGATTGGTGTCGAGAAACCGTCGGGTTGATTGATAATAAAAATGAGACGA +TAAATCTAGAAACGACTGGAAAGATTATGTATTTGGAAGCGTCGAATCGA +GAGTTGAATGTGAACTATTCAGATCGTCTTTTGAGATTGTTGAAggcatg +gcctaactttttttctaggccatgtaatctcttttttccaggaagTCCGC +CAACTGAGCAGTTTGGGATTCAACATTCCATCGAAGATCCTGTCATGCGC +TAACAATGGAGAGAAGTACTATCGATTTGGTGTTATCCTCAAAcagattg +ctcatttttataaTACCATCGATCAACAGATGATTCCGTCGCAACAGGTA +AGCATGATTAGCTTTAGAAGCGCATTTCTCTAAAACTAAAATAGTTAACA +AAAAGTTATCAGCTAttaaaatgtataaaatttttatagactATATTGCA +TTTGCTaaaaaaaTTCAGCTAACTATTTTAGTTCATGAAGTATTAGGCTT +CGAAGTTAGGGCACATTTTTGTCCCGCGAGTCCGTTTAAGAAAAGTTTac +cttttggaaatttttggattttcggctgtttttttcaaaaatatggaaaa +attggatcatgacctaaaaaaatctaggtcatgacctgggaaatttaggt +catgacctgggaaatctaggtcgtgacctagaaaaattaggtcatgacct +aaaaaaatctaggctatgacctaaaaaaatctaggctATGACCTgtaaaa +tctaggtcatgacctaaaaaaattgagccatgatctagaaaaattgggtc +aagtttttagtttttgagttatacGCTTTCAAAGTTTAAGCATCTCTGAC +TCAGGACAAACGTGCATCCTAGTTTCGGAAgtttatatctcaaaaactac +aatagctagcaaaaatttgtcaactacaaaattttttgaaattttaagag +ctatATTTTAAGACTTAACACATTTTAACTAGTTATATTAGCTTCAGAGA +TATTCGAGTTAGAAAGCGCGCGCTCTGGATTCTCTTTGGTTCCAACTTTC +TTTTACTTTGTAGACgcatatcttaaaaactaagacagctatcaaaaagt +cgctaactagtaaaatgtttaaaattttatgggctttatttttgtagttt +acaaaatttaccTAGCTatcctagtttttgagatattcatGTTGAAAGAC +AACccacttgaaaattttatatctcaaaaactagaaaagctatttcaatt +tgattttcagtctcTCATGCTGGAAGAAGCAATTGCATTTGAAAAACTGG +TTATTCCAAGGAAGGATGCCTCCAACTCGGCTTCGAAAGTAACATGGAAC +GATCCAAAACAACTGGAAGAGTTCATTGTCCAACTGCAAACCGCCGAACA +AAAATTGTCGAATCGAAACCGTCGGTTGCGAAATGTTCATATGGAATTGA +TAGAAATGGTTGAGAAATTGATGGATTTGAATATTGTGAAACAGAATAAT +GAGTGGAAAGAGATTATTCTGAAGATTCGATCGAAAATGAAGGAGGAAGA +AGTTGTACATGGAGCGGCGAGGAATAATATGAAGCCGTGGTTGATTCATT +GGGATTTTCAATTGTATAAGGTGATTTGGCTCTACTGTAGAATTTGGGcc +tcaaaaataaatatggtCCTTGAAGACAATTTTGGTccgaggtttttgaa +aaattcggaacttcagaaattcagaaattcagaaattcagaaattcagaa +attcagaaattcagaaattcagaaattcagaaattcagaaattcagaaat +tcagaaattcagaaattcagaaattcagaaattcagaaattcagaaattc +agaaattcagaaattcagaaattcagaaattcagaaattcagaaattcag +aaattcagaaattcagaaattcagaaattcagaaattcagaaactcagaa +attcaggaaatcaggaaatcagaaattcggaaattcggaaattcagaatt +ttagaagttcagagaatcagaatattagaagttcagaaattcagaaattc +taaagttctagaattctacaaattcaaaattttgaaaattcaaagttcca +gcaattcagaaaattaaaaattcgggaattcaaaagtttccagaatctca +aaatcttagaaatcaaatattctgatatttttttaaaaattccagaaaag +tcTAAATGtacagaaattctaaaattctaagattctaaatttctagaatt +ctgaaaattcaaaaatccagcaattcggaaattcctatttttcttagggt +tttcagaatttcaaaagttctttaaaaatttcaaattgttcaaATCTTCA +GGCTCTTCTAATCCAATACGAATGGGGAATCGAAAGTATCCAATCTCAAT +TATCCACAATTTCCGTCAGTCTAGTTTTTGTGTaagtgtcgatttacggg +ctCTTTGATTCCtcgattttcggtttttcagcGATCAAAAAATCCAACTG +CGTCCGGCAATCGAAGAAATCCGTTCCAAATATTACAAAGAGCTCTGTCG +ATTTTTAAGAATTCCAGACAAATTTCGAGGAGTTCAGGAGGATGAGACAg +taagtgcgctccattgataatttctgtttttctgaatttctagaatttca +aaatttagaatttctgaatttctgaatttagaaaaaattgtgttttagTC +AACCAAATTCTACGCCCAAATGATCGAACGAAGTATGCATTTGTTGCCAT +CAGTCTACGAGAAAGCTGAGCAATTGATGCAAAAAGTGGAGACTTGTGAT +GCCATTTTCGTGGTATGCTTATcggtttgtccggatgtccgcaattgaaa +aaattttcaggattggCTAGTCATCTCCCAAGTGGACCTCGAGGAGCTTa +ttgaggaaaatttgaaaaccgcAGCGGATTGGGAATCGCAgtttaagatt +ttgaaagcCAAAGCAAGAGAGGCCGAACGTTTGCCACAGTAagattggat +tactgtactttagaaaattcgaactGCAAGTCATTAAGACTTTGGTTCAA +TTAGATTACCGTACATTTGTTCTAGCAAAGCATTTTGTTCATCACAGAGC +GCACAGGGTTTGTTTTTCTATCTTTGCAAACTCGTTCTATTGATAAGGGG +CGGGGCGAagctttttctaattttcaaaaatgaaaaaattctcccCTAAA +ACGAGAGATCTTGTTTCGGTGAAGAGACTCAGAACATTTCTGTGAAGAGA +CGAAATTTcggtgaagagacgcagacaacgaaacgaaacgtttgatcctc +agccatcGATTTGGCTGCGTTCTCATTAAAGCGCATTCACAAAGATAGTA +AAACAAACGCTGTGAGAGCGTTTGCGAGGCTTGCAGTTCGGTTTTGTTAG +CTGCAGAGATTACTGTATATTTCTTTCAGTGAACTCAAATTCGAGTGCAT +ATTGGTGAGTACTGCTGGTGTTAAAACTGCAATCGAGGATGCGATTCAGA +GATTGTACGATGCGCTGACATGGACTTTAAGGTAAGTAGGACATCTGGAC +ACGCAAACACACagaaaacccaaatttcagacactCCATCTCCACGACGT +CCACCTCTATAAGCACATTTCTCTCTCAAGCTATCGAAGTCCTGAATACA +GTACCCGGATCCTTGGATGAAGTCGCCGAGGCCAATGCGAAACACGTCAT +CTTTGCTGAAACGAATCgtcagctgaaaatggaatgGAAAGTCATGGAGG +AGCAATTGACACTGTTGAGATCAGTTGCTGGTCAAGGAATGGAGCAGATT +GATAACTTGGAACAAACATGGGATCGATTTGAGCTGATGTTAGATGCGCA +TCAAGGAGTTATTAAGGAACAAGTCGAGGCGTTGAAGGTAGTTGACCcct +aaaagaagaaaagtttgacccaatttatctgggttcttgtaagagatatc +ataaacgtgtcaactaataaaatgttggagatattatgataaacatttta +tcagttgacaaatttttcatagcttttaagggaacagagataaattaaTT +AGGTCAAGTTTGGGTCAAAATTGGGTGTAGGTAACTTCAAGCttgcatat +ctcaaaaactaaaatttctagacaaaatttgaaaactaacaaaatgtagc +tcatgaaattttatacattttactagtttttaactttttgatatctcttc +tagcaacaatcaaaaatttgacccAAAACCGCAATTTCCTTCCAGACCAA +CGTTGAAACATCAATCAAGGGAATGAAGGATGAGGCGGAGAAGCTGAAAG +CGAGATGGGACCAATTCAAGCCAAGAAGTGATGCATTACAAGGCGATCGA +GAGGAAATGCTAAAAGCGATTCAGTTTATTAAAGAAAAACGAGTTCAGTG +GCAAGAGCTGAGTGAtggaagagagaaaattgaGTGAGATGTTTTGGTTG +TTTTACTGTGAATGTTCTTGAGGCACATTATCTCCTATAAGcgatgttaa +aaatttttcaactaaaaaatgtttctgataacttgataaacattttatta +gttgatcattttttgatatctcttaaagaaattgagataaatgatCTTAA +AGTCGGCAAGCACAGTTGTGGCTCAGTCAACGGCGACGGCCGCGTCGCAG +ACCATAACGGCACGtattttgagcctaattatctcggttcctataagcga +tattaaaaatttttcaactaaaaaatgtttctgatatcgtgataaacatt +ttattagttgatcactttttgatatctctaatagacattgagataattaa +gcttaaagtaGGTAAGCCCAGTTGTGGCGCAGTCAAAGGCGCCGACCGCG +TCGCCAGTGGCGgcttttttgagcaaaattatcttggttgctttaagaga +tatcgaaaatctgtcaactgataaaatgttgctgataatgtgataaacat +tttattaattgaccactttttgatgtctgttatagaaattgagatagtCA +AGCTCAAAGTCAGCAAGCACAGTTGTGGCTCAGTCAACGGGGCCGACCGC +GTCGCCAGTTGCGGcgcattttttgagctaaattatcccagttcccataa +gagatatcaaaagttggttaactaaaaaaatgttgctaataatgtggtga +acattttattagttgataactttttgatatctcttatagaaattgagata +atgcAGCTCAAATGACAAGTAAAAAGTAGCGGTCTTTCGTACATGCTTTG +CTTCGCTATGATACATATCTCGTTAGGGCGCATTTGCACAAAAATGTTGT +CAGGTACAAAAATGGAGCTCTTTTCCTGTTAAATATaatccaaacttttc +aaaacaataGAACAAGTACTAAGTACGTTTGCAGCCGCTCAAAATTAGCA +CATGGGTGAAACGTAAGGATCGAGGCCGCTACCTAAATGAACCTCGATAG +CTACACTTCAAGCCTAATAAACTCAgttaatataccgtaaaacctctaat +agaacgacacccgaatcgtccactttgagcctatatatctcacttctttt +aaaagatatcaaaaagtggttaactagtaaaatgtttatcaggatatcag +caacattttactagttagccactttttgatatctttcacagaaaatgaga +tatataggctcaaagtggacgatccgggtgtcgttctattagaggtttta +cggtaagagatatcacaaatttttcaactaatgaaatCTTTCCCATAtca +tcacaaacattttcttagttgacaggtttttgatatcttttacaggaact +gagataactGAGATCGATGTTGACCCTTTCGTAAAATTCTGTTAGAGTAT +acgacaattgagctcaaaattagcgcctaaatttgaaaatgagcaGCTTA +ACCCctagaaccttccagaaaAGAATGCGGTCAATTTGGACTGGAACCTC +CAAAACTGGATCTAATCGACGAAATCGACGATGACATCAAACAATTTGAA +GTCAATTGGCGATTTATGAAATGTTCAACAATGACCTAGACACACTTTCC +CAGGAGGAATGGATCGTTTTTCGATCGAAAACCTAtctttttgatgaatt +tttgggaaaatggatggaaaaattgaagggaGGCTCGCAGACGCATATGA +GTGTTCGGTTGATGAAGGATGTCGAGCATTTTAAAGAAGTGAGCAGTGCG +CTGAAGTTCTGCAGAGGTGATGTGCTGAGTGCGGATCATTGGCACGAGAT +GTTTCGGTTTTTGGGATTACCAAGGtattgatttttggggttctgggttt +ggggggggggggtcttaATGTTATTTGGTTGCACAGATAGCAGTAACCTA +ACAGGAGGGTACAGAGAAACGAGGCTGCctattttctctgcgtctcctcc +tGGCTTGTTGGGCCCTCCTGATGCAACCACCTGTAACTCAAAAGCTAGAA +TAGCTAGAATAAAATTGTTAACTAACGAAATGTGTATAATATTATGagct +atatttttgtaatttacaaaatttagcTAGCTAAtttttttATAGAAGTC +CTCAAAGTTTGAACTTATTTGTGGCGCGCGCATTTGTTTCTCTGAGCTTC +AAACGCatgtaactcaaaaactaaaataccgtaaaacctctaataaaacg +acacccggatcgtccactttgagcctatatatctcacttgttttgaaaga +tgtcaaaaagtggttaacaagtaaaatgtttatcaggatatcagcaacat +tttactaattagccactttttgatatctttcacagaaaatgagataaatg +ggctcaaagtggacgatccgggtgtcgttctattagaggttttacggtag +ctagcaaaaaactgtcaactagtaaaatgcaTAAAACTTTATGAACTATA +TTTTGcctgttgacaaattttagccagctttcttagtttttgagatacat +gTCCTTAAAGTAAAGGCACACCAGTCTGAAGGTGGAAGCCGCACGCTCTT +GTTTCTCTGCGCTTCGAGCGCGTATGCCGCAAAAGCTAACCGACTTAGCA +TAAAGTTTTTTACTAATAAAATTTGTAGAATTTCATGagctatattttat +tagtttacaaaatttagcTAGCTTCTTTAGTTTTAGAGATACAAGCgctc +aaaaattgtgtttccAGAGGAACCACCATCGAAAAGCTAAAATTCGCGGA +TCTCCTCTCCGTTTCCAAAGCTATCATTGAAAATGCGGACCAGTTGAAAC +AGTTGAACTCTCGTGCTCAAGGAGAAGTTGCCATACGAGATGCGATTCAA +GAATTGACACTATGGGCTGCTCAAACCGAATTCACATTGGCTGATTATAA +ACATTCGAATGGGCAGAacttgaaaatcatcaaagaATGGAAGGAGTCGA +TTAATTCGGTTAGTTTAGTTTGGGGTTCGAACAGCCATAGGATAGTTTCT +TGTTAGCATCCGGAATGAGTGATGACATTGTtcggtttctgtgaatttcc +aaaaattaagaCGCTGTTCAAACAGTTTGACGCTAGAGACACAGCTGAGT +TGTTGAGATTCAAATGTGTGAGAGAATGAGACACAAATGATGGATAGAGT +GAACAGAAGAATGGTCCATTGACCACACAGTCAGAGcaccagaaaaatcg +aaaagaggCATTAGTAAGGGAAAATCAATAGGTATTGTCTGTAGGCACAA +AAGAAGATGTGAGAGCGAGAATCAATCAGTGGTCAAGACGAGCAGAAGAA +TGTGTCCTTGAGAGGTCAAATGTGCTCTAGGGAAACTACTAATATGATGA +CAAAATCTACAAACAAGTTGTTCTAGccaaaaattcttttcttcgaattt +gcggttttaaatttttttttcattttggattAAAGTTGAATCCTTTGAGT +TGGACTTTCCGAGTTAATAACAAGAAAGTTCGTGTTTTAAGCAGCTAGAG +GGGAAATAATAACTTTAAAGACACACGAGAACCTTTTCTAATCGAAAACA +ATGCGGATTGCAAGAAagcgatttttggaatcaactGACTAGAGAAGTGA +CAAGTACaacattagaaaaaaagaaggaaatagGAATTGTATCATTTATT +TCTTTATCACAATACTCGCCATGAAAGTAGAAGATGATAATACTATAGAA +GCTAATCAGAGCTAATTAGCATTAAttagttttgaaataaaacaaaaatt +aaaaatgacttTGAACCACTTCCATTTTGATTGTGTACGgaggaattctt +gaattctgatcTTGTTCGCAGGTAACTTTAAGTTGCAGTTGATGGGGGAG +TGGAATGATGATGAGGCTGGAAAACAagagaaatctcaaaaaccagacGA +CGTGAGCAATTTCTaactgttttcattttcaagttACCAGCAgggttgca +aaaaaaatcacttaaaGAGAACGTTTAGCCTTACCTCTCCTCTACAGCAG +CATTCTCGAAGCGAGCAGCAATAGCTCTCATCGGATCTCCCTCTGTATCA +TACCATCTCACTTGGTTGAATTCATAACATGTCCCGAGTGTCCCCTTTGC +GTTGATCAAGTTCTCTACAATCATTGCATATTCTCCTGGCTGTATCACGT +AGCCAATGGAGATCACAAATTTCTTATTCGGTACTTCACGACACAATGCC +ATAACATCGACTTCCCAGTTGTTGACCAACACCAAAACTCCAGCATCTCG +AACTTTCGGATTTCTACCGTCTCCCAATTTCAAGCCATCCGTCTCGAGAC +GAGTGAACGGGCGGTTTGGATGCCCCAGAATTGTTTCGACTCGTTGTAAA +ACTTCTGACAGGTTTCCGGAAGTTCCAAACCCTTGAACATCAAACTTGAT +GCCTTCTGGAAGACCAATAACTAGACCGTTACTTTCACGCGCCCAGAAGC +CGAGCGATCTGATTTTCGTAACCAATTGACGATTTCCAAGGAATTTGTTG +ATCAGATACTTCTGTGCCTCCCTCAAAGTTCTATCGTGTTTAAAACGTTC +GATGTAGACAGTACCATCTGAAGACTTCCTCGTGAACTGGAAATAGGGTA +CTTTAGAAAACGCATGAAGTTAATACAATTACCTGAATGAACATGTCATA +TGGTAATGGACGGTCATCTCGTTTACATTGAAAACGTAGCACATATAATT +CGTCCTTCTCCAATGCCATTTTCATATGTCGAATTTCTTCATTGATTGCT +TGGAGTCGCACTTCTCGACTAACCCCCtggaatcttcttcttcttctgtc +aaGCTCTTCTGGTTCTAATTCCAGTGGATACTGTGGGAGATTTTCAACCT +CTTCTGGACCATATTGAAGATCCCTTCTCTCTCTTTCCATATCAGCTAGC +ACTTTCTGATCTCTCACCACATTTGCCTCTGCCAATTCAAGATTCATGTC +TAAGTCTGCGGCAGGATCGTAATCTCGAATAAGAATATCCCCAGGAGTTA +GTTCTGGAAGGGAGCGCTTCTCGAATCCAAACCTATCGAAATCGCATTTA +CATCCACCTCTTCGCTGTTCCTCTTTGATTATTTCGGGAGTGGGACCCTC +ACGGGGTTTGCAAATAATACCAAGACGGTATTTGGTGTCGTTTATTTTGA +CGCTcccttcagaaattgagagTTTAGAGATATAAAGTGGAACAGCTTTC +TCGGCAAGACTTATTTTCGGAAGACGTTCGGCAAgtcgaaatctgaaagt +tctcAACGAAATCACTTAATTCTTTTTCGTAAAAGTTACTCACCTAAAAT +TCACCTCTAACTTTTGAAGTACACATTTCAAACAATCATAGGATAGTTTC +TTGTTAGCGTCCGGTATAGCTGATGACATTGTCTGTAAATAGGTTTAATT +CATTTGTAGAGGCTTTCATAATCTACATTTTATTACTAGAAATTGTTCGA +GTTGCAGGAAGGTTATGATACATAAAGACCCTTTACCTCAGATTCGAGGT +ACTCTAGTGAGTTTCGGCTTTTTAACAAAACAATTATTAAAAATGAGTAT +GTTGGgttagaaaaagagaatgagaCACAAATGTTAGTTAAGGCAGACAA +AACAATGGACCATTTACCATAACGCGAGAGTCAGGGCATCGGTGAAAATC +GAAGAGGGTGACTAGCAacggaaaatcaataattgtcTCATGTAGAGACG +TCAGCACAAAGAAGATGTGCGAATGAGAGTCATACAGTGGTCGCTGAAGA +ATGAGCAATGTGTCCTTGAGAGGTTCACATCGGCGGGTTAAACATCTATG +GCCGAAAGGTGGGATAACACTAAAACGTGGGCGTTTTGAACTATCGAATG +CTGAAACTCCTCCAATTTCATAGAGACCGTCTCCAGACGAGCCAAAGGGC +GGTTGGGATGCTCCAGAAGATTTTCGGCTCGTTCCAAAACGTTTGAaaga +tttccaaaagttccaaaCGCTTGAACATCCAACTTGATGCCTTCTGGAAG +GTGAATAACTGGCAATCTCAATTGATTCGGCCAGAATCCGAGCGTTTGAA +CTATAGAGACTAATTGGCGGTTCCCAAGGAACTTGTTGATTAGATATTTT +TATGCCTCGTTCAGACTTTTGTTGTAGGTGACACGTTCGATGTAGACAGT +ACCATTCGGAGACGTCTTCGTGAGCTAAAAACAAACTCTAAAATCACTAA +TTGGTACCGTATTACCTTAAtctcaggtaaacgattctgaattagatctg +acttcattcggaattagatctgtattcgttcgtaaatagaagtggcatga +tctaatacagaatttagctgatctaattccgaacgaacaccgaatgaact +acgaacgaattatatctaattccgaacaggacggcgggatttcggagcat +cgtagcgcgcgcttttcaatttttgttttcttttcaacgccattcacttt +atttataataaaatttattaaaaaagaaaacagaacaagtatgagaaaaa +gaaaaacatagattaaaaTGTAGTTATAGGAAATCGGAACGAAAATGAGG +GGTGTGATACAAATGAATGAATGggatcagcaaaaaagaaaacgatgaga +tcaGCGAGAATAATGGGAACTatgagataagaaaaagtgacaaaatttat +gaaaaaaaatctcaattcgaattcttccgctcgtccatattttcgatttt +gaaaattctcttgtCATAGTTTCCGGTTGTAGATAAGATGGATATTCTTT +ATGAAATGAATGTtgatttaactgaaaaaagaagattttggCTTTGTGAT +GTCGCTTTGGCTTTGTGATATAATCGCTTTGGCTTAGTGATATAATATTT +TGGCTTTGTGATATAATAAACTTACTTGATCCATTTCAATATAATTTCCG +TACTCAAATCTTTGCAAATCCAAATAAATTCATTAGAGCAGTTGTTTTAT +ACaacaaatcccaaatttttgaatcgtttTAATCGGGTCATGACACATTC +GGTTGAGAGCCTGCAATTTCACGCATGTTTCTTTGTCCACCAGACTTTCG +AAAACTAACCTGCTCATGACAAGACACGGCTTCCGAAAGTAAGGCCTTCT +CGACTTGTGTAAATTGAGCTCACTATCAGAGCAATttatcatctcttttt +tcttgtttttttcggtGATCCCTTGCATCGTTTTTACGTAATCCTTCAAg +ctgaaaaaggcaaaaattatCTTACCTGCAGAGTTTTCTCCgaatttaaa +atcttcaaaatcgtcGTATTTTTGTCGATTCATAGATTACTGTCTTCTAT +AGAAGTCGTATAtataaattgaactttgaatattcagctgttcctaaaaa +aattcacctaaaattttgcaaaaacgtgcacaacacattaaaatatgttc +cagaaaatatagtttaaataattgagcgaaaaaaaaggcaaaagttGAGA +GAAATACCCATTTTcctgacgaaaaaaatttaggacaacaaaaaaacaaa +acaattcatcCGCGGCAGCTGAGTCATTGatggcgcatttctcgtggcAA +CTTCGGTATTAGATCTAGTCCatatctaattccgaatgaactcagatcta +attccgaatttcgagttattcgatctaattccgaaagaagccagatctaa +ttccgaatcgtttacctgagaTGAACATTTCGAATGGTGATGAAAGATTA +TCCCTTTTGCACTGATAAAATTTCACTTGTAATTGACTATGTTCTAGGTA +GATTCTGATGTTTCGAAGCTGTGTATCTAGATCGTTTAACTGCTCTTCGC +GGTTCTTCCGGTGACCGGCTCTATTGACCACAACGGATTCGGTGATCTCC +acctcttccttcttcttcttcagttcATCCAAATCGGATTGTAATCTTTT +CACCCATCTCTCCGCACCCTCCAAAGTTTGCCGACGTCTCTCCTTATCCT +CTCGGACTAAAATATCGCCAGAAGTGATTTCCAATCGCTAAGCGTGATCC +TTTAAACCGAATCTATCGATATCAAATTCAACTCCACCTTTTTGGTTCTT +GAGTTTGATTGAAGATGGAGTGAGACCTTCACGAGCTTGACGAATAACGC +CAAGACGGTATTTGGTGTCATTCAATTGGAAGCcggattccaaaaataaa +agtttggAGATGTGAAGTGGGACAGCTTTTTCAGCGAGACGAATAGTTGG +AAGACGTAGGGCCAGTTGGAATCTGAGAAGCAATTTTTTGGAGGGCGGGG +GAGGGGTGGAGGGGGTGGACTTCATCTGAAAAGACTCACCTGAAATTTGG +CTCCAGCTGTTGAAGTACAGATTTCAAGCTATCATAGGACAGTTGTTTAT +CAGCGCTCGGGTAGATGGTGGTGGTGTTCAAAGACATATTTTGGTtctga +aaactaaaagtttcaaaaaaaatatcagcTTAAATGATAGATTCTTTGAA +GTTGATGGATAGAGAGGTTATAGTTAACTGTAGTTCTCTCTAACCcaaat +taataaaatttaaacatttggaattttatatGGTTTTGTAGCTATTATTA +AGTTCCACATTTTGCAGTTCGAATTGACTGTAAATCTTGAATTTGCAAGA +AGCTAGCCAGGAGAGGAAATGAAAAGTTGGGATTCGTCATGCGTTATGGT +GTGGGCAGCAAACTACTCGGACTAGCCATTGTACAAATATCATGTTGTTT +gctgtgaaaatttgggaaaaagagagagagagagagagagagagagagag +agagagagagagagagagagagagagagagagagagagagagagagagag +agagagagaggaaaacgttcagccaatttatctcagtttctgtaaaagat +gtcaaaaaatggtgaactggtaaaatattattgctattatgataaacatt +ttagcagttgacacatttttgatatctttcactggaGACGAGATAACTGG +TCACAAAGTGAAAGCATAGTGGTGGCGCAGCTATAGGCGCCTACCCCGTC +GCCAGTAGCAGCGCacgttttgagctcaattatctcggttactgtaagag +atagcaaaaaatgtttaactaaaaaaatgtggaacattttatgacaaaca +ttttatcagttttctaatttcagttGTCTCTTATAAGagcctagataatt +gagctgttAAAGCAATTATACTATACCAAACAAATTATCTGTCAGTGTAT +CCATATGTTCATGTTTCAGCTCAAAGACAGTCAAGCTCTGCTTCAATCGC +TCAAATCCTCACCCTACTATTCTCAATTCTCGGATAAGACAGCAGTATGG +GAGACGCGTCTTGCCGATTTGGATGTGTTCCTGGCACAAATGAACGAGAT +TCAAAGGAAATGGATTTATCTAGAGCCAATTTTCGGGAGAGGAGCGTTGC +CATCCGAAGCTTCGCGATTCTCGAGAGTGGACTCTGAGTATCGAGCAATT +TTGAATGGTAAGAATTAGAGCCGAACTGCaaaaaagcttcaattttttca +gacgtATCCAAAGATGCTAGATTAGTATCGCTGTGCAGCAGACAGTCGCT +GAAAAAGTCGTTGGAACAAATCGTGGATCAATTGAATCGATGTCAGAAGG +CCTTGAATCAGTTCTTGGAGCAAAAACGAACCGCTTTCCCGCGATTCTAC +TTCATTGGAGACGATGATCTTCTAGAGATCCTTGGCCAATCAACCAATCC +GCAAGTGATTCAAACTCATATGAAGAAGCTGTTCCAGGGGATCAATCGAG +TCGTTTTCTCATCAACCGGAGAGACGATAACATCAATGGTGTCAGCTGAA +GGAGAGACTGTACCTCTGTCGAAAAGTGTTCGAATCGTTCCACAGGTGGA +AACATGGCTCCAAGAGTTGTCGGACGAAATGAGGAGAACGTTGAAGACGT +TGACGGCTCAGGCTGTAGCAGATGGACAAATTTCATTGGGAAAATACCCG +TCGCAAGTTTTGTGTCTCGCTGAGGAGGTCAAGTTTAGTGCCAGTGTTGA +GAATGTGAGTATCTCGAGTAGagtttgaactttgagccagattatatcgg +ttactgtaagagatatccaaaacgtgtcaactgataaaagtcttgtgatt +ttatgataaacattttatcagttgagcactttttgataacttttacaaAA +AGCAAGATAAATTGAATCGAAATTGGGGAGCCTTTAGGTGTGGTGTAGAA +GCATCGtcccccatttttcactttgatcCTGTTTTCAGcgcctgtggaag +atatcaaaaaaatgtcaactaacaaaatttttatcattatatcagtaaca +ttttgctagttgataactttttgatatctcttacagtaaccgaaataACA +TGGCTTAAAGTGGGTGAACATGGTGCAACTGTATTAGACAGTTTGAACGG +TAATTGCGAACGACCGTATCTTCTGATAGCCTTAATACCTTTTGACAAAt +ttaaactaaaaattgagaactaCAAGATGATAATATTTAGAGAGCTGTAG +TTTGTCTAATTCTCTAGCTACATAATCGAGCTACACGCTAGCATCGAATA +TGAGGAAcctccactttgagctcaattatctcggattttataagagatat +caaaaagtaatcaaatgataaagtttttttgatgaactgataaacatttt +attagttgacaaatgtttggtatttctttaaaaaatcgagaaaattgagc +tcaaagtggttGCGCAGCCAACGGAGCTGTTAGCGGCACCAGTATCGGCT +CGTGTTTTGAgattaattatctcaatcactgtaatagatatcaaacatct +gtcaagtgataaaatgttattgataaaatgtttatcatataatcacctaa +attttatcagtagactcatttttgatatctcttatagaaattgaaataat +tggcCCCGAAAATAGTAaactaaatatttttcagatcctcAACTCCTCAT +CCGACCTGTCTTCATTAAAATCGCAACTTCAAGAGAAACTGAAATCTCAT +ACGAACATGAAAGTCGATGATAAAGTGTCGGATCTCAAATTGAAATCACT +GATTCTCGATCTCATCCATCATATCGATGTGGTCGATCAGTTAATGGACA +ATAAGGCGAAAGGAATAAATTGTTGGACTTGGCAAAGGCAATTGAGGTTC +TATTTGGTGAATGGAGGAATCGTGTTGAGACAGGTGAGGGATGGACCTGT +CTGTGTTTAACCTTGTCTGTCTGGCAAAAGATGGGCAAGTGTTGGGCAAA +TACAAAGCTTCAAATTTCAGGTGTCCAGTGAGTTCGAATACACCTACGAG +TATCAAGGAAACTATGCGAAACTTGTGCATACACCGCTGACGGATAAGTG +CTATTTGACTTTGACACAAGGTAtgtggacatctggacatatagataaat +aaataaatttggtttttgggaTTGTCTACCGAAccaaaaattcgttttcc +CGCGAAGCCTGTCATAGTGGGCGCTCTGTTGGTACTGCCAACTTTGACAT +CATTTATCTCGACTCTCAAAAatggtatcaaaaatctgtcaactgataaa +atgttaatgatgacatgataaacattttgttagttgaccactgtttgata +cctcttacgagaacagagataattaggctcacGCGCCGCTACTGGCGCCG +TTGGCTACATCACCACTGTCGATCGAGTTcgagctcatttttctcgattt +ctttaagagatataaaaaaaatagacaactgataaaatgtgtaagataaa +atgatgaaaatttggttagttggccactttttcatatcttttacagtaac +cgagataaattttgACAAAGTTGAGAGCCAATGCTTGCGTCGGCGTGCCG +ACGTCTTAGCTCACAAGCTCAATCTCCTGAGGCTCTTCAATAACGAAACA +AAAGCAATTGAAAGCTTTTTGCTTATGTCCCACTAACTGTCCCAATTCAT +GTCCATTGTGTCCCACCAGCTTTTACATCGAATTCAGACAAATCATATTG +GCTTcctttctaaaaatcttacaGTTTTAAGAATTAGCTGTTTCCAGCTA +TGTACATGGGACTTGGAGGTAATCCATACGGACCTGCTGGTACCGGAAAG +ACAGAATCCGTGAAGGCGTTGGCGGCACTAATGGGTCGTCAGGTGTTGGT +ATTCAATTGTGACGAAGGAATCGATGTGACGTCGATGGGACGAATATTCA +CTGGAATTGTCGAGTGCGGAGCGTGGGGATGTTTCGATGAGTTCAATCGA +TTGGATTCGGTGAGTGTCTATTTCGCAGCCAGACGTTTCGCAACTACTTG +GTTTTTCggcctatttatctcggtccctgtcaaagatatcaaaaagttgt +caactaatgaaatgtttgcgatattatgatgaagtttttatcagtttact +acattttgatatcttgtgtaggagccaagaaaaattagcttacACTTGAC +AGGGTTCTTTTTCAGACTGTCCTTTCCGCGGTTTCAATGCAAATTCAAAC +CATTCAAGGTGCAATAAAATCGCGAGCTGGCTCATGTACCTTTGGAGGCA +AAAATGTTCAAGTCAACCCgaattcggcaattttcgtAACCCTGAACCCG +GCTGGAAAGGGTTATGGAGGACGGCAAAAAATGCCGGATAACTTGAAACA +GTTGTTCCGAGCGGTGGTTATGGGAAAACCAGATAACGAGCTGATTTCGT +CTACGATTTTGTATTCGGAAGGATTTGTGGATGCTACGTCGTTGGCAAGG +AAGATAGTGTCCGGTGaggattttgagctcatatatctcatttcctgtga +aaaatattcaaaagtcgtaaactgacaaaatgtttagcaTATTATTAGAA +Accttttattagttgacaactttttgatatctcttatgggaagtgagata +aattgagctcaaagtgcaggcATAACCAACAGCGACGACCGCGCCGCCAG +TAGCGGCGCATGCTTTGAgattaattatctcagttcctgtgagagatact +aaagatgtgtcaactgaaaaaatgtttataacattattataaacatttta +ccagttaacaactttttgatatctcttataggaagtgagataaatttagc +tcaaaatgtaaGCATAGCCAACATCGCCCACCGCATCACCAGTAACGGTG +CAtgctttgagctccattatctcggttgctttcggagatattgaaaatgt +tcagtctgataaaatgtgtatcacATTATCatttatattttatcagttga +caactttcaaatatctcttacagtaatcaagataaattggtACAAAGTTA +GGGTATCTGCATGATTTTATGCTTTGTAGGTTTCGTTGAGCGTTGAGCTC +TAGACTTtggtttttgatttaaaaagaaattccaaGCTTACAGTATTCCA +ACTATCCCGTCAAATGCTCTCCAAACAACAACACTACGATTGGGGACTCC +GAGCTCTAAAAGTGGTCCTAGGTGGATGTGGAGCTCTTCGACGAGCTCAG +CCGAACAAGAATGAGACGGATTTGGTGGTCCAGGCGCTTCTACTGAACAC +ACTCTCAAAACTGACATTCTCTGATTCGGAAAGATTCAATTCTCTGATAG +ACGACATTTTCGCAAATGTGACGAAGGAGATGACAAAGTTTGAGGAGCTC +GTAGAGCCATTGGGAGTGGCCGCAAAGGAAATAGGAATTAGTTTGGGGGA +AAAACAGATGGAGAAGGTTTTCCAGGTGAGATGGGAATATGGAGATCAAa +ctttgagcgcaattatctagattcctataaaagttatcaaaaaacagtca +actaaaaaaatgtttctaacattatgacaaacatttctTTAGCCGACaga +tctttgatatcttttataggaaccgagataatagagctcaaagttagcgg +tctcaaaaaaaaaaagttaaagttaaagctcaaagttgatgtcAGACAAT +CCCAAGAAAAAGGCGAGGAAGCCCACTAATCCATACGCCTTCTCACTCTA +ACCCTTCAAGCTTCCGGGTTGTGTAGTGGCTAGCGCGCCACACTTTTGCC +CACGAGACTCGAGTTCgagtcccccccccccccccccccccccccgaggC +CAACTACtggtatttttctttttgctacattttataattttcagctctac +gAGCAAATGCGTCAACGTATCGGAGTGGTGGTTGTCGGTGCCGCTGGATC +TGGAAAATCCACAATCTGGAAGGTTCTTCAACGAGCACTGATCCTAACGA +AAAAGCCCCTGAGAGTGACTCAATTCAATCCAAAAGCTGTGAATCGAAAC +AAGTTGTTGGGTAATATGGATATGGACACAAGAGAATGGACAGATGGAAT +TATAACAATGGCAGCGAGAGAAGTGCAAAAAGATACTACAGTACACCATT +GGATTGTGTGTGATGGGGATATTGATCCGGAATGGGTTGAAGCGTTGAAC +TCGGTGTTGGATGATAATCGATTGGTGAGTTGGGGCtagattatctcggt +tactgtaagagatattaaaaagtaatcaattgataaaatgtttatcatga +aatgacctacattttattaattgacaactttttgatatcttttacagtaa +ccgagataattaagctcaaatcAAAAGCATAGTGGTGGCGCAGCCAACGG +CGCCTATTGCGTGCTTTCAcgctaattatctcggttactgtgagagaaat +caaaaagtggtaaactgatagaatgtttatcttaaaggaggcgagacagc +aaaaatttcgggggacagttcgggggacgaaatcccgcctcgcttatgta +gaccatcggtcgtcagccgccttcgcgcccccgtagctcagccggtagtg +gcgctgtacccctgagctgcctgtcatgggttcaacccctcatgggggtt +ctctatgcaaaatatcaggaaaactttttttttgatgtctggcctccttt +aaaattcctaacattttatcagttgacaactttttgatatttattacagg +aaccgagataatcaatatcttctttttcagctcaCAATGCCGTCTGGAGA +GCGAATTCAATTCGGGTCCAACGTGAATTTCCTCTTCGAAACCGACTCTC +TGCAGTTTGCCTCaccagctacagtatcccgaaTGGGAATGATTTACATA +AGGTtagtgaaaataaaaaaaaatctaggtcatgacctagaaaaatctag +gtcacggctcaaaaaaatctaaatcatgacctaaaaaatctaggccataa +cctaaaaaatctaggtcatgacctaacaaaaaaatctagatcatgaccta +gaaaaatctaggtcacggcctggaaaaatctaggtcatgacctaaaaaaa +tctaggtcatgacctaaaaaatctaggccataacctaaaaaatctaggtc +atgacctaacaaaaaatctagatcatgacctagaaaaatctaggtcatga +cctaaagaaatctaggtcatgacctagaaaaattcaggtagtgacctaaa +aaaatctaggtcatggcctagaaaaatatgaaatctaggtcatgacctaa +aaaaatctaggtcgtgacctagatttttaaaaaaatttctcaatttttag +cgaaGAAGACGTGAAACCTCAAGGCATCGTTGCGAGTTGGCTCgcaaaga +attcagaagacgtgcacGCTGAAACAGCCACGTGGCTTGAATCTCACTTC +TGGAGATGTTTCAAATGGGTTCAAACGAAGCAAATACCAGGAATCACCAG +TTTCGCTTTATTGAAAAACGGGCTGACGCATTTGAAATCAGCCAAAACCA +AAACGCATTTTTTGGTGCTCCTATTCAACGGATTCATGCCCTACGTCGTG +CCAGAATCTCGAAATGAATTCGCGAAAAGTGTAGTTTTccaagggattac +tgtagctgaccCCAAGAATATTTGCTACGACGAGCGGGTCGATGGAGTCA +TGACGTATTCAGATGACGTGACACAGAATGTGACGAAGGAAGAAGTCGAG +AGAGAAGATTTGAGACCATTTGTACAAACTGCCGATATTCAGAGATATTC +GGATATtgtgagttttcttttttttggaattttcaaaattttcaaaaatc +aattttttgaaaaaatttctgagattcatCTTTAGCGCATTACATATTCT +GCTTCAGCTCGTTGAGAGCGTTAAAATGAGCCCCACCACGACCTATCTgg +gtcaattttgattttttgttagattactgtagttttcgtggtgggaccca +actTTCAAAGGAatatagtcgtgtttgggctcatttttaaGTCCTTAGCG +AGCTGAatgaaaacaattaaaaattgcTGACGAATATCacaaatttcatg +aatataatgaaaaactgattaaaaaacagaaaaacagaaaaaaaattttt +ctgaaaatcttcaatttttcaaatttagttttttctgaaaattccaattt +caaattatttcaatCTTGCTCAACTTATCGAGAGCTTCAAATTAAGCCTA +AACACCACCTATCTGGgtcgattttgatttttgtggaattactgtagatt +tcgtggtgggacccgaTGTGAACCGAAcctaggcttgtttatactttttt +tgaagctcaaaacaagagaaattcaaaaatatatgtttctttcaaaatga +ggaaatttatactgagttactgtagtttcaaaaaaatattccagatcGGC +TCGTGGCTCCGCCCCGGGAACCGGGAAAGCTTCCTAATCACGGGAACAAC +GGGATGTGGAAAACAACAACTTCTCAAACATTGCTTTCAAAACGATCCAG +AATCTCAGTTGGCCAGTTTATATTGCTCAGCGCAGTCTAGCTCCGCACAC +TTGCTGCAATTGATTCAACAGAACTGTGTGCAGGCTAGTAATCCGACGGG +ACGCGTGTGGAGGCCAAAAGATCGGCCGAATATGATTCTTTTCTTGAAGG +GCATCAATTTACCAGCACCGGATAAGGTATGGTTTTTGTGgtgatttgga +aaaaaaacaagttttcaaaatttcgatgaaaagAAGCTAACTTTGGAGTC +CTGAAAAAGTAGCCCACATTATTtcgatttatctgaaaattacagagtat +actaaacagaaatagagcttcatttttgtaattgacaacttttttgtgca +aacgCGCCCTAGCGAGATACGCCTGCGTTCCTAAGTATCCTCTTCCGCGC +GCCGTCGAAGATGATAGCTACACTttcacgtggcatatctcgtTAGGGCG +CATTTGcacaaaaagttgttaactacaaaaataaagcactatttttgttt +actatacactgaaattttcagataaatcgaAACATTCTGAAGTACTTTTG +TAGCTGGCGCCGCAACCACTGACGCCGCCACCATCTAGCGATGGCTGTGC +TGCGTAGCTCAGCTGGCAGCAACGCGGAACCTCCAACGGCGAGCTCACAA +ACTCAACTTTTTGAGGGGCTTATagaacaagtttttgaaaaaaagtctaT +TAAAATTCCATGTTCCtttaataaaaaagttttcagtacGGCACCAACGA +ACTCCTGGCACTTCTCCAACAACTTCTAACCTATCAAGGATTCTTCGATC +ACAATTTAGAATGGGTTTCAATCGAAAACATTCAATTCGTTGGGTCCATG +AACCCACTCGGCGACGGTGCATCCACAAATATCTCCAATCGCCTCTTCTC +CCTTCTTCGATGTATCTCTCTGAACAACACGGACACTCCCCAGCTGACTT +CTATCTACCGTACCTATCTAACTCCAATTCTTGAAGACGTTGGAgaacga +aattcagaaacactCGCTTCTCGAATGGTTGATGTCTATACCAAGGTTCA +ACAAAGCTTCAAACCCTCTGACAGTATCGTATTCCAATTTTCGCCCAGAG +ACCTTACCAATTGGGTAGTCTCACTTTTGAGACATGAGCTGGATCAAGGC +AAACTGGAAGGAGTGATTTGTTTCGAGGGAAAACGAATATTTGCGGATCG +ACTGCCAACCGAAAACGATAAACAGAAATTTGAGGATATTCTGAGAAATG +TGATTCCAGTGTCGCAAAGTGCAGGTGAGGATTCTACTCTAGAAGAATAG +CGcccaacttcgagctcaattatctcaatccttgtgaaagatatcgaaca +GTGATAAACatataaaatgtttgtgatattatgataaacattttatcagt +tgacacatttttgatatctttcacaggagacGAGATAACTGATCACAAAG +TGCAAGCATCGTATTGGCGCAGCCTACGGCGCCGACCACATTGCCAGAAG +CGGTGTGtgatttgagctcaattatctcagtccTTGTCAACTCaaacagt +tgtcaactgataaaatatttttgcttttatgataaacattttgttagttg +actactttctgatatctttcacaaaaaatgagttaTGTAGGCTCAAAATA +GACGAGAGTATTTACGGGCTGTCTTAAAACCCTAGCCATCAACTTtgacc +aatttatctcggtttctgggaaagttatcaaaaaatagtctattgatgaa +acgtttttgattttatgataaacattttagtagttaaccacattttgata +tctctcacgataaccaagataattaagctcaaagttgacggtcagAGATT +CTACAGTAagatcaaaacatttttcagacaaagtcTACGTCACAACTGGA +TCCGTTGTACCTGGAGAGAGTAACACAGGTCTTCCACTAACTCCGATCAA +CATGAAGGAATACAATCAACTCTTGTTGAAATCAATTAATCGATTCAGTG +AGTTAACTTTAAcacaatttatctcaatttctgtaaaaaatattgaaaag +ttgtcaattaagaaaatgtttgtgatattatgatacacatttcattagtt +gactacatttttgtaacttttacaggaaccaaatTAAACGAACTTAAAGC +CTGCATTGAGgtcaactatctcggttcttataagagatattgaaaatgtg +taaacttataaaatttttaccaTATTATATGCAACATTTTAgcagtggac +aactttttgataacttttacagaaaccgagataaatggttTTCTTTGCAG +CCTTCGAAGTGGCCAACTTCAATTCGCCCCTTACCAGTCAATTATCCTAT +TTCTGTGCATGCCTTGATCGTGTTCTAACCACCCCCGGTGGTCATTTATT +CCTACCCGGACGGACTGGATTTGGACGAAGAGATGGTGTTCGACTAGTTG +CTCATATGCACAATATCCAAGTGTTCTCGCCGGCGGTGACAGCGAATTTC +TCATCGAAACAGTTTGATAATGAAATGAAAGCGGTAAGTGGGTTTTCAGG +TCCCACCAAAGttgttcaaacttttttggattggtctaaaaatttcagag +aatactAACCAGAAATATGGCTTAAATTGTGTAGTTAGCatctttttttg +ggaaaacgcGCTCTAGCGAGATATGTCTGTTTTTCTTAAGACGATCTCCG +CCGTGGATGATAGCTGCAATTCCACTTGCCATAATTCGCTATAGCGCATT +TGCGCAAAAAcgttgttaactacaaaaatgaagctctgtGTTTGTTTaaa +atactctgaaattttcagagtaatcGAGCTGTTGTAAATGTGTACCTGAT +AGACAGGCTTATGTCGTTTTTCACTCTGTCTTCGAGTATCGTAAatactt +gagctcaattatctcggttcctttgagtgatatcaaaaatttgtcaacta +ataaaatgtttatcacaatatccttaatatttttgtagtttgccaaattt +tgatatctcttataggagccgagataattaagctcaaagtgtaagATTAA +GTTTCCAGGCCATCACCCAAGCTATCACGAACAACGAACACGTCGTTCTG +ATTCTTGAAGATCATCAACTTCGGAAAAATGTCTTCCTTCAAGCCATAAA +TTCTTTATTGGCTAGCGGAAATGTGCCCGGATTGTTCACTCAACAAGAAT +TGGATGGGCTAGTGGCGCTGGTGTCTGAAGCAGCCAATCAGGCGTCGTTC +ACAGGGGCATTGCAACAGTTTTTGGCGCATCGtgagtttcagaattgatA +TTTGTTCACCAATTAAAATGGatttaatagaaaaaatgaagagctgcaaa +atgaaaatatttagattactgtatcttgtctgtaaacaaagttacagaca +agTTACAGTCGAACACAGCGACGGCCATGGCGGGGACAGAATTACTTTTT +AATTTGtttctataactttgttcacagaTTACATACAGCTTTgcaaataa +tatcattttgtagcttttgagtttttctataACATCCTTTttatttgatc +aaaaaataacataCCTGAAGAAAGTTTCGAATGGAAAATTACCGTTAACC +ATATAATGGTCCAACCCCTTAACAAACTTTGAACCAGATTATCTCTGTTC +ccttaaaagatatcaaaaagtaatcgactgataaaatgttgctaatatta +tgataaacattttattagttgatcactttttgatatctctcccagaaaat +gagatagaaaggctcaaagttgacggtttggaggGGATCTATTAGAGTTT +CTACGATATGTTATAGTCTGAAACTTCAGGAATCCGAAGCCTGGTCCATG +TCGTGTTGATTCTTGAAGTGGAAGCCAATGATTTCAAACTGAACATCACT +GAAAATCCAGCGATCCTGAAACATTGTAACGTAATCTTTGCTGATCGATT +CGATAAATCTAGCTTGAACGAGGTAagggacatccggacacacagatccc +aaaaacttgaatttcagattcccaaGCTATTAATGGAATCAAAAGGCATC +ACAGCAACCGACTCGATTCTAACTGGATTCCCGGATATTCTCATCAACCT +GCCTGAAAATTTATCCATTCAAACAATTAAATACAGGCAGTTTGTAGAAA +ATTGTTGTCAGCTGTTGGGCTATAAACGGAGTACACTTTCAGCTAGACTT +GATAGGTTGAAGGTATGCATGgtcgctctattgataattaaCAAAAAGTA +AATGTTTAGGGCGGAGTTTCAAAACTGAACGAAGCTCGTGAAGAAGTGgc +gaaaatgcagaaaaaagcTGGCaaaaaaaGTAAGCTACTCGCTGAAAAAC +AGGCGGAAGCTGATGAGGCGTTGAAGGCGATTACGGAGAGTATGAGTgta +agttttgagattttttgattttagattAGGgacttcaaacttttttaggG +AGCAGAAGACCAAAAACTGTCCATGGAGCAGTTGAAAGAGGCGACCGAGA +AGGAAAATGTGGTTATTGAGGAGAAGAAAGCCAAGATAGATGAGCAATTG +AAAGAAGTGCAGCCGTTAATTGatgtaagttttcaaaaaaagttgaaaaa +agaaataaaaagaattGACTTCAAAGGGGTCAAACGTAAGAGTGCCAGAT +TAAAAAGAAAGAGCGTTAGCAAATCAGCCACGCGCACACCAACAGCTCTC +GGCTGCGGCTGTCGTCATTTGGGCGAAAGTTGGCCCCCTTTTGGGCGCTG +TTTGGCCAGATTACGCTTAAGATTTTCCCAGAAGGTGCCCAAACGCCGCT +CAAAAGCTGGCAAAACTTCGCCCAGCTTTCGCCTAAAACCTTCCACCGTT +GCCTTGGGTTCgctttaaatttgtttttctttctttcttggAAGACTAGt +tcaatgagaaaattttatttattcaaaCAATTATGaaaactaatacaaaa +aactgttgaaaaaaagaacatacaaaaaaagaagcttttaaaatttttcg +catCCCCGGCTTGATGTTGCCATCTTTACTTTATCCTTGTCCGTGTCcaa +gtcgatttttcgatcGGGAAGTTGGCTGTAGTtgttatttctgaaaaaat +ttattgattttcaaaaaattgtatcatttatttttcccactgaaaatttG +CTAAGTCTGACACTTAGCAGACTAAAAGGCTTGTGCAGAAttgaatttcg +atatttttcaaactatttctAACGACATTTCGCCTACCATTCCactagaa +atctcaaaagaATTTTAAATGGACTCGATGAGGTCGGATTTTCATTGCCG +ATATACTTAGGGAAATATATCTTCACAAcattacaaacattttaaaagca +ttttctcaatatttttggacgtTTTCGAAAGCTTCAAACCCAGAATATAC +GAACCTTTTGAAGCTTCTTCTGACTTCGCGGATTCTCTGACTTGTCCCTT +CCAAGCGAAGGATTTGCGGCATTCACGAAATTCCGAATCTTGAGTCTAAA +AACGTTATTAATGACTGAATAACATAATTGTTGTTTACCTTACATGCTAT +CAGTGGAAAACTTCTGGAGATGTGCTCAAAgggggggTAGACGAGATAGA +actcagaaaaatgattttctcaGTTCGACGCTGAATGAAAAAGGatttaa +agggggggggggggacatgaaaaaaaaatgttctcgaaattccgtttaaa +gatgcccctcgaggagttgaacttgtgagggcgatgattgagggtgcggc +gccgctaccgactgagccacgcgggcaaagagagagcgcacgcccgattg +catacataaaggcaaggacgacagctgtacgcagcgtgacattaagcgtt +gtacaccgtgacggtagaatgcatactctagctacaaaaaaactacgaaa +aaccTCCTGTccccccctttaattgaaattttctattccACTCACGGTAT +TTGTTCTTCGTcgttttcggtttcaaaaatctaatcatttttggataaaa +atatttagaagtTTGAGCAGGAATTCAGataaaacactgaaaataggaaa +aaataggtgatgcaaaattcgaaatgcaCGCGTCGCTTGCACGCAACGCA +TTCAAAACTCGCCCAGAGTTCGCCGAAGTTCGCCAAACCTGCGTCCGACC +TTCTCCTACACCACCGACTTCCTATTTCCGTCGGGGTTCCCCAAATTTCG +CCAGAGTTCACTTTTATTCTCTACCTgagtgagctcaattatttcgattc +ctatgaaagatttcaaaaagttgtcaactgatggaAAATAGGTGACGtta +tgataaattatttatcagttgacaaatttttgatatctttaaaaagagcc +aagataattgagctcaaaattgacggtttgggagGTCTTCTATTAGGGGA +AATACGGTAGTCCGTTTGGGCTTTGGTCTAGAGATTGTATATCAAATACG +AGTTTTATGTCTTTTTACAGACACCAACAGTTGACTTTTTAGAGGGCATA +CATTTTATGATGTTTTCTCGCAATCAgcatcacttttttcaaccGAGAAA +TTGTCcacaacttttgaaactttaaatttcaagTTCTGAGTATAAATTAT +CAATTATGAATTAAAACTGAGAAAGCAaaatatcttttaaaaaatacatt +caAAATATAATGTCATGTTTTTATTGGCAACTTAACAATTCATTCAAACT +CACACACAATTCATTACATGTAATATTTTTATCAACAGTTTGCCAACAGA +GATTACCATCAAATTTCGCCCAAAAGGCGCCCAAAGTCTGCTCAAAAGAC +GGTGGCGCCCCAACTTCGCCCAAAAGGTGCTCATAGTTCGCTCAAAAGAC +GGCAAGCCCTTAACTATTGCCCAAAAGACAACAGCCGCAGCCGAGCTAGC +ACACCAAAAGGCGTAGATAAGCGCGGGCGTTTCCGCTACTTTTCAGGTGT +GCTTGGCTTCACGAGCTAGGGTAGTCTTTTGGctggaattccgaaatctc +AAATTAAATTTCCAGGAAGCCCGTCGAGCTGTTGGATCGATCAAATCCGA +GTCGCTCAGTGAGATCAGAAGTCTTCGTGCACCTCCTGAAGCCGTGAGAG +ACATTCTACAAGCCGTTTTAGTATTCATGGGTATTCTGGATACGTCTTGG +GAGGCTATGAGAAAGTAAGGAGGACATCCGGGATGTTCAGATACACAGAt +agatttcagattcttgtCGAAAAGCGGGGTGAAGGATGATATAATGAATT +TCGATGCGAATCGAATCACAAGCGAGATTCACAAGAAGGTGACGGCTATT +GTCAAACAAAAGTCTGCATCTTTTGAGGAAGCGGTAGGTGGTagacatct +ggacaaacagattttaattcaaagaaaaatttatagaaCGCTAAACGAGC +ATCGGCAGCGGCTGCACCATTGGCCGCATGGGTAAAGGCGAATTTGGAGT +ACTcaaagattttggaaaagatTTCTcctttggaaaatgaaaagaataaG +CTGATAAAGTGggtttttttaagaaaaaaaggaagaaaaagggtTTTGAG +CCAAGCGCGAAGTCGAACCCGCGACTCTCCGATTGCAAAACAGGCGCGCT +AGCCACTGCGCCACGCGGCAGACAAAGGTTGGAGCACCGCAAGGTGTAGA +GGAGGGTGGCGGATTTTGCGTTTTTCCAGGAGTGGGAGAGGGTCGCCTCa +tatgattaaaaaaaattccagaaatctcaaaaaagctgaaaaacaaatgg +aaaatctatccaaaGGCTTGCAAAGTGTTGACGAAGTGGTTGGGGAGTTG +AAACGGAAATTCGAAGTGCTGATGAAAGAAGCGACGCAGATAAAAGTGGA +TTTGGATCGAGAACAAGACACGATTCGGATAGCTGGCACACTTGTTGAGA +GTCTTTCCGGCGAGTTTGAGAGATGGAAAGTGCAAATTGAGACATTGGTA +GAGGAGCAGTCAAAGGTAAGGGTCCTGTAGGCAGAGAGATACGAATTAAA +ACAACTATATGTGGCTTTCTAAACAGTCAACTTCGaccttatttatctcg +gttcctgtgaaagatatcaaaaattggtcaattgtTAGAATACTTTCCTT +GTTATgctgaacattttattagttgaccactttttgatatcttttacagt +aaccgagataaatcagattttccaaaaggTTCCAATAGTCCGTCGGCCGT +CCGTCAACGACCCGTCCTTTGGTTTTCAGGCCATAAAAGAtccaaaatca +atttcaaacgTATTTTCAGATAGAAATGTGCTCTCTTATCACCTCTGCGT +TCCTCACCTATCTTGGTGGTTGCTcagaaaaagatagaaaatcACTACTA +AAATCCATGTGTAAAACATTCAATATGCCCTCTGGCTTCAACCCTCTCTC +CTTTGCCAGCCTGGAAACCGAGCAACTAAACTGGAAAACCAAAGGACTAC +CAGCAGATCAACTCTCCCTAGAAAACGGATCGATAATGTTCAATTCACAG +CATGTGCCTTTAATCATCGATCCATCTGGACAAGTGTCAACGTTCCTGGC +CAAGTTTCTGGATAAATCTGAAACATTCAAAGCTGCTCAACCGGACCTTA +TGACTCAAATCGAGTTGGCGATTCGATTTGGAAAAACTATAATCGTAGAA +GATGTGGTGGAGTTCGATGCAGCACTGATTCCGATTCTAAGAAGGGATTT +ATCAAGTCAGGGACCACGTCAAGTCATCTCCTTTGGTGGAAAATCGATTG +ACTATAACtccgatttcaaaatctttatttGCACCAGAGACGATAAAGTT +GAAATAAGACCCAATGCATCTGCCCAATTGAATATTGTGAACTTTACAAC +TACGATAAGTGCTCTCTCTGCTCAGCTTCTCGACGTCGCTATCCATCTTG +AAAAGCCGGAGCTAGAAGAGCGATCATCGAGTCTTCTCCGAGACgctgag +ctcaaaaaattggagcttgAGGGTTTGGAGCAGTTGTTGCTCCAACAGTT +GGCTACGTCACAGGGAAATCTTCTGGAGAACACTGCACTTTTGGATTCGT +TGAACAAGTCGAAAGAGAGTGCCGAGATTATTAGTAAGAGTATTGTGGAG +AGTGAGCAATTGCACAAGGAATTGACTACGGTAGGTCttgatatcgaaaa +gttgtcaactgataaagaaCTTGTCGTAGTTTGTGTCACATTTTAGTAGT +Tggtaaatttttgatatctcttacagtaaccgagataattgagctcaatt +atctcggttcctataagagatattaaaaagatctcaactaataaaatatt +tctcatgttatgataaatattttatcagttgacagatttttgatatctct +tacagtaaccgttTTACAGCAAAAAGAAATCTATGTGCCGTTGAGTCAAT +TCACTTCAAGCCTTTTCTTCTCGTTCTCCAATCTACAGTACTACAATTCA +ATGTACAATTATAGTGTCAACACGATAATGaagctttttgaaaagacTAT +CAAGTCGTGTGAGGTAAGAATTCTGTGCACTTTTGCGTGATGGTGTACAA +CGACTACGGTATCTCGATCGGCCGCCAAAGAGTCGACTGCTACCCCAACT +TTGAAGCCCCACGAAAGTAATCCAAAAAgcttcgatttgtctgaaaattt +cagtgtatactaaacaaaaatcgagcttcattttcgtagttgacaacttt +tttgtgcaaacgCGCCCTAGCGAGATATGACGCAAAGAAGCGTCGCGGTG +GCCCACGGCGCCGCCGCCACCTCTACctcccttcaattagaggttAGCCG +CCTTCTTTGCGCCGTATCTCGCTAGGGCGcgtttgcacaaaaaagttgtc +aactacaaaattaaagttcTATTTTCGTTCAGTATACTTACCAAATTTCA +gagcaattgaaaaatgttggactacttttgtagggcttcaaagttggcaa +ttttcaaatttttgaaaaaaaagaccaataaaaataaggaatttttGTTC +TAGGACAAAAGCTCAACTCGTGTAGAAAGCCTTGCTCGTCAAATGCAGCT +TACCGTATTCTATCATATCTCTCGTGGAATCTTCCGTCAGGACCGTTTGA +TGTTCGCTGTCGCCTTCATAAATGCAACACAACCGAAAATGTTTCAACCA +AAAGAGTGGGAACTGTTCACAGGTATGTTATGGTATGAAATAATGTGGAT +AACTAAGTTCGGCTTGaaataagcttcaaaatgagcccaaacaagCCTAG +GTTTGCTTCGAAtcgaattactgtagttttcgtggtgggacacaattttg +agcaaaaatattcgtgtttaggctcattttggagCTCGAAATGAGACAAG +CTTAAATTAAAAcctatattttaaaatttagctTGTTTagtgtttcaaaa +tgagcctaaacaagCCTAGGTTTAGTTCATATTGGCTCCCACCAcaaaga +ctacagtaatcaaactaAAAAGCAGAGTTGACCTAGATAAGTCTAGATAA +GAAGCTCTCAGCAAGCTAAATTTTGTTGATATGGTCAAACTGATGTTCCA +GGAGTTGTAAAATATCTTTAAACAGAATTAGGTTTGCTTGAAAtcagact +actgtagttttcgtggtgggacccgaaCCAAGGCTTGCTTCAGGCTCATT +TTGTAGCTTTAAACAAGCTTTTACTTACCAAACTAAGCTTCCAGGGCTTC +TTAAGAGCTTAAAAAGGGGGTATCTTGAaattgaattactgtagtacAAG +GTTAACTTaaaattggattactgtagtttaggcTACCCAAAGTCTCCTTA +CTTCTAAAGTCCAACCTTATTATTACAGGGGTTCTCGTAGACGAATCAAC +GGATCTTAGTGGCATCCGAGTGTCCTGGATTGCTCCAGATCGTCTTCCAT +CTCTAGCCAGAATCCGAAACCATCTTCCGAGTCTCTTCAATAATTTCCAA +ATTGGCGACGACGCGACGTGGAGCGAATTCGCAAAGATGCTCCAGTGCGA +AACGGCGTTTCCGAGGAATGTTGAAGTGAAAATGACACATTTCCAGAAGG +TTCTTTTCATACAGGCCGCAAGACCAGAACGGTTGTATAATTGTTTGATG +GATTTTGTGCTCAAGACGTTGAGTGGGTGATTTTGGAGAatcagattgct +cagcttccagattttcagtaatttctgattctttaaatttccagacatcC +CCTCAATCAATCCACCAGCTTTTGAACTCCGTCACATCTTCTCAGAATCC +GAATCCTCCGAACCCATCCTCTTTATCCTTGCCGATGGAGCAGACCCATC +CCAGGAGCTCGCGGAGCTCGCCAGCGCTCTTAAAGTACCGTACCACTCGA +TTTCCATGGGACAGGGACAGGAGCAAGCCGCGTATGAAGCAATTCGGAAC +TCGGCCGCTAATGGGGAATGGTTGTGTTTGAACAATTTACATTTGATGCT +TCAAGCGGTCCCGTCGATTTTTAAGGTAggctttttgaaaacgttttagG +AGCCCGTTTAGCTTGCTGAAGCCTTcaaaacatatatttttgaatttagc +tcgttaagagcttcaaaatgagcctaaagaCGACTTAtctaggtcaattt +tgaatttttgttggattactgtagttttcgtggagGGGCCCGATGGGAAC +CAAAACTAGTCTTGtttaaactcattttgaagctccaaacaagagaaact +caaaaatatacatttcTTTCCAAATGGTGTAGTTTCCactgagttacggt +agtttcaaactaaaaaaaaaagattttgagtttcaaactACTGTCACTCG +GGGAGAAATTTACTATTCTCAAagaaacaaacatttttgaatttctcatg +cttggagcttcaaaatgaacctaaACAAGCCTAGGTTTGGCTCACAtcgg +gtcccaccacgaaaactactgTGATCcagaaattacagaaatttaagatt +tttttgaaaaatattcgaaaacgattttttcgtcataattcaggtatttt +caattttcttcatttgaattttaacaattcttattcaGCTTGATAAATGC +TTTGAAACAAGCCTAAACACgtccatatttttttttaaattgcaaAATAa +atgggtcccaccacgaaacctacagtaatccaaaaaaaaatcaaaactag +tCTAGAcaggtcgtgtttaggctcatttagAAGCTCATCCcaagccaaat +tcaaaaatacaattttcttctgaaataatagaattttcactgttacggta +gtttcaaattgaacgacagtttttttgaatggaaaactaccgtaactcac +tGAAAATTCTGCCAATTTGAAGGaaacatatatttttgaatttggcttgt +ttagagcttcaaaatgagtataaacaagcctttgttcgggtcccaccacg +aaaactacagtaatcccaaatcAAGCAATCCtagccgtgtttaggctcat +tttgaagctcattcCATGCCCAATCCAACTCATAACATAATTTTCCAGCA +CCTTTCCCTCACAACGCCCCACGAGAACTTCCGCCTATGGTTGACTACAG +AAGCAGACGCCCGTTTCCCCTCAATGATGCTCCAGCAATCgctaaaaatc +acatttgaACCGCCGCCAGGCGTCCGGAACAACCTGCTGCGGACCTATAT +ACAAATCGATCGAAATGGGCGGAGTGTTGTGACGTGTCAATCGATATTTG +TGTTAGCTTGGCTTCATGCCCTTCTTCAGGAGAGACGAACTTTCATTCCA +CAGGGATGGACCAAATTTTATGAGTTCTCAGCTAGTGATGTGAGAGTTGC +CAAGTCGTTTGTAGAGCAATTGACTGAGAATAGTGAGTGGCtttcaattt +gaagaatttcagaaatttttttttgaatttcgaatttctgaatttctgaa +tttctgaattttagaattttagaattttagaatttctgaattttagaatt +ttagaattttagaattttagaattttagaattttagaattttagaatttt +agaattttagaattttagaattttagaattttagaattttagaattttag +aattttagaattttagaattttagaattttagaattttagaattttagaa +ttttagaattttagaattttagaattttagaattttagaattttagaatt +ttagaattttagaattttagaattttagaattttagaatttctgaatttt +aaaatttttgaatttcagagtttctgaatttcagagtttaataggaaact +gaaaatcctaaaaatcgtcaaaaatctaaaaatatgaaaagactaaaaaa +ctgaaatctcaaaaatcttaaactctagaaatctgaaaaagtttcttttt +ttcttaatttcaggtttttcagattttctagaaaatttgaaaatctggat +gaaaatcctaaaaattctgaaaaatgtgttttttttcgaattttgaaatt +ttcaggttttctcagaatctcaaaatctgagaatctttAAATAtccaaga +atctgaaatcaaTTCTATtgtcaaattcgaaaatcttaagaaccctaaaa +atctgaaaatcttttaaaatttcagaatttcagaatctaaaattctaaaa +ttccaaataatcagataccgtaatccctgtaatagaagggctctctaact +ttcaactttgagtcaaattATCTCGCTTACTgtcgaagatataaaaaagt +tgtcaactgataaattgtttatcacaatatcacaaacattttatcagttg +acaaattctcgatatcttttaaaggagccgagataattaagctcaaactt +gacaagtgggggcccttctattacaggtttcacgatattttgaaattacc +tcaaaaattttttttcaaaattcaaaattctcatggcctaacttttcact +aaaaccccattttctcttccagaatCCGATTGGGAATTCGTCCgtggaat +tctgaagttcgtGATCTACGGCGGTCGAATCGAGAATGATTTCGATGCGA +AAGTTCTCGAATCCtatttaaatattttgttttccgATGACAAAATTAAT +GGGCGCACTGGCCAGCAATTGGTGAAAGGAATCGATATTCCAGCGAGTAC +TAATGTTCAGGTACgatctcaatggagcgcacatgctaaattttttgaaa +attattcaaaaaattatccacGGAGCGCgctttcattcttttttctgtcg +agaaaaaatcgatgaaaattttgCCGCACCGCTCAACCGCGCTCCACCGA +ACAAACAACGAATGGAGCGCGTTAattgtcaatagagcgcactttctcaa +taaaaaattcaaatttcttcaggAATACATCTCGCACATCTCAAAGACGG +TCCCATCCGTCGACGAACCATTCCTCTTTGGTCTTCCcgaaaatatcaaa +tattCATGGCAAATCGTAGAAGCCGATCGAACGATTTCATCGATACGAAC +ACTAGGttcgtttgaaaatcaatatggtggcctagaaaatctctaggcca +ccaatccCAAAACTATTAcacgtggcctagaaacccaagtTTAAAGTTTC +TTGATGTAGAGTATCcataaatggcctagaaacccaaatttgaaCCTTTT +GgactagaaatccaaaactaggaaatagtggcctagaaacccaaactTGG +GAGatgctggcctagaaacccgagTTTGGTAGTGGCCTAGAACCATGCAA +AGTTGCATTCTATGCCGCTATTTCCATAGTTCgggtttttaggccaccat +atcctaatttgggtttctaggccccattttatattaggttggcgggaaag +tctttgtcactttttcaatagtttatttctcaaagaaaaaataaccgaat +aattaattatattgtccatcattatcaacaacatgtctccaacgttccct +caatttatggattccattgcggtagaatccagccgggcgtgaggcaaagt +attgtttgagtgccgcttcgaccgctccacgggaattgaagcttcttcct +tcgaaggcacgtgtcatatcgccaaataaccaatagtcggaaggagcaag +atcaggagaatacggggggtgagggagaacattccatccatatcccatca +gctcctgcttcacttgtttcgaaacatgtgggcgagcgttatcatgttga +tagtacactttcttgtctttcagggccgaacggtcgactgctttcttgag +gtttcgtagttgagtagtgtagaggtctccagttattgttttgccctcgg +gcagaagttcccagtagatggggccgtccactccccaccacacagagagc +ataacttttttcggatgaaggtcgggtttggggacatcctgtgggctctc +tccttctccaacccactgggatttcctgtggtgattatcgtagaagaccc +acttctcatctccggttattagccggtccagccatcgatcgcccgcgtgg +agagtgagaagagtaatcgaatcgtcgactcggcggtcaaggttggcttg +agtcaggtgatgaggtatgaatcgaccaaaaagttttctcattccaagag +acttcaaaccttcaaggactgtgctatggtgaactccgaatgtgacagct +agttcacggctagattgaaatggatccgattgcaatgcgtgcttcagctt +gtccaaatccaactcacggggtctcccagagcgatccttttcatcgagcg +agaagtcatctttttcgaaacgatggaaccaattgcgaatagtattgtag +tgaggggcatgtgtgttgtacacattcttcaagaaagtttccgaatcctt +catcgtggtgtgagataggaagaggtaaatcaaaacatgtctgatgtgaa +tagcagtgggaacaaacatatcgtttcaattacagatgaggagcagggaa +ggagatggggacttgtttatatggaaggttcttatcaccttgttattcac +taaagtcgatatcatacacagcctccatatacgagaacagtgattaaaca +aacagagtgacaaagactttcccgccaacctaataatataagatttctag +gccaccaaatctCAGCTTTCCAGAATTCTAGGTCATCGATCAAAAACCGT +Gtcctagaattccaaatttcgaatcgaatgacctagaaacacaattcaaa +aattctgaaccagaattctggatttctaggccaccatgttaaaatctggg +attcctaggccaccatttcctatttttcgatttctgggCCACCATCttca +aatttggttttctaggtcataataTAAgtcttctaggccaccaaatctCA +ATTTTGGAGATAGAAACCCAAATGTTGGAGGTAGTGCCCTAGGAAACCAA +TATTTTGGGTTCCTAGGCCATCGATCaaaaaaaCGTGccctagattttca +aatttcgaatcgcatggcctagaaacacaattctaaaattctaaaccaga +attttggatttctaggccacgcccactaAATCTTTTCAGCACTGGGCGAC +TCGAAAAATGCGCTCTCTTCGAATAACACCTCCGAATCGATATCCCAAAT +AATTTCGCTATGGAGAAAATTGTGTCAATCGGATGATCTCCCaaaaaaaG +AGCTGCCAAAAAATACATCAAAATCGTCGGATCCAATCAATGAGGTCTTG +TGTTTGGAGACGATAAACGCACTGATTTTGATCAGAAACCTCCATAAAAG +CATTGGAGACGTCGCCAAATCCATGAAGAATCCATCGTTGGCATCTGGAG +TCACTCAGAAGACGATCCAATCGTTGATCTTTCAACAGACACCTGATGAA +TGGGATCGATTGTGGGCGGGGCCAACGGACCCCGCGGATTTTTTGAACGC +GGTGGTGAAGAAGACGCGGGGAACGATGCAGGTggggggttttgggattt +tctaggggaaattttgaatttttgaaaatttgaaaaaaataagaaaaatt +aatttttggctcaaaaatttgaaaatttaaaaactcaaaaaaaattgaaa +aaatttttttcaagtcttttaatttttgattttcagacaaatttttcaaa +ttcaaaaaaacaatttttttggaaattttgaaatttcgattttaggatca +aaattggattttccggtttttcggctcgaaaatttgaatttttcgaatta +aaatcgatttttttatattaaaaaaagggcttttttcaaaaaaaaaaata +tttctttgaaaatttgaatttttggctccaaatttgaaaattttaaactc +aaaaaaattgaaaaaaattgtgattttaagaactttttttagattttttt +ttgacaagattcgaaaaatattttttgcctcgaaaattatttttttcaaa +atttcgaaaattttgaattttttgctattttccgactttcaaaaatttcg +aattttttaaaatttttcagatttttttgaacttttaaaatttcagattt +tttttgaattttttgaaattttgatttttttgcagcTCTACGAAGCCTCC +AAATCCTCGTCGATCCTCTCCTCTCccatcgatttttcggatctCTTCTA +TCCGAACATCTTCTTGAATGCATTGAGACAAACCACGTCTCGACAACTCC +AAATTCCGTTGGATCAACTGATTCTCTCTTCCGCCTGGACACCATCCCAA +CTTCCACCGAAACAGTGTGTCCAGGTGCAGGGTCTTCTACTCCAAGGTGC +CACTTTCGATTCGTTCCTCCGTGAGACTACCGTATCCAGTGCTGCCTACG +TACAGGCGCCGATCCTCTACTTGGCATGGACTTCTGACAGCTCTTCGACG +ATCACTGGCGAGCAAATCCAGGTGCCGCTGTACAATTCCAGTGAGCGTTC +CGACTTGATCACCGCTGTCAACATGCCGTGTCGTGGTGCCGATCAATGGA +ACATCGCAGCTGTTGCCCTATTTCTgcgataacaaaaaaaaagatttttt +aacaactaaaaatacagcttaaaattaaataaaatgtattgtgcgtttaa +aagtgaataaaaaatcatgaaaaaagatCACTAGTTATCATTACCAGCTA +gcagatcgaaaaaaaaggaaaaaaattattgtacATGAAAGCGATAATAA +ATGAagcgaaaaataaaacaatttagtTTTCAGTGTCTTCCTCCTCATCG +ATCGGCGCTTCTCTCCGCTTCTGAATCGCGTTTGGACTTGGGAAGAAGTT +CGTCGAGTCCAGGAGGCTGTCCCTTCTCACGAACGCCGTCACACGACGCT +TCTTTTTCGCGAGCTCTCCGTCGCTTCCGTCCTCAGCGTCCTCCTCCATT +CCGAGCAATGTCTCCGGCTCCACGGGTTTCACGAGCTTCTCACTGGCTGG +GTACTCGGGAATGATTCTCGGCGGTACATTGTTCGATGGGTCGACGAATG +TCATTTTGACTGCATTGATGGCTCCGGTGGCCTCTTCAGTGACGATGTGC +GTGGCTTCAGCGAACATTGTGAAGAGTGACATCTggaaattatcattttt +aagTGTATACAGTTTgatagagcgcatttacatATTTTTAGCTGGGAAAT +ACCGCTACACATGACAGTTCTTTCGCAAACGGTGTCAGTTCAATTCCCCG +TGCACATCCCCAAAACTTCTTTTAACTTACCGTTCTTTCTAACGACTCCT +GGACCnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnaaaaaagttgtcaactgataaattgtttatcacaatatcacaaac +attttatcagttgacaaattctcgatatcttttaaaggagccgagataat +taagctcaaacttgacaatgtgggggcccttctattacaggtttcacgat +attttgaaattacctcaaaaattttttttcaaaattcaaaattctcatgg +cctaacttttcactaaaaccccattttctcttccagaatCCGATTGGGAA +TTCGTCCgtggaattctgaagttcgtGATCTACGGCGGTCGAATCGAGAA +TGATTTCGATGCGAAAGTTCTCGAATCCtatttaaatattttgttttccg +ATGACAAAATTAATGGGCGCACTGGCCAGCAATTGGTGAAAGGAATCGAT +ATTCCAGCGAGTACTAATGTTCAGGTACgatctcaatggagcgcacatgc +taaattttttgaaaattattcaaaaaattatccacGGAGCGCgctttcat +tcttttttctgtcgagaaaaaatcgatgaaaattttgCCGCACCGCTCAA +CCGCGCTCCACCGAACAAACAACGAATGGAGCGCGTTAattgtcaataga +gcgcactttctcaataaaaaattcaaatttcttcaggAATACATCTCGCA +CATCTCAAAGACGGTCCCATCCGTCGACGAACCATTCCTCTTTGGTCTTC +CcgaaaatatcaaatattCATGGCAAATCGTAGAAGCCGATCGAACGATT +TCATCGATACGAACACTAGGttcgtttgaaaatcaatatggtggcctaga +aaatctctaggccaccaatccCAAAACTATTAcacgtggcctagaaaccc +aagtTTAAAGTTTCTTGATGTAGAGTATCcataaatggcctagaaaccca +aatttgaaCCTTTTGgactagaaatccaaaactaggaaatagtggcctag +aaacccaaactTGGGAGatgctggcctagaaacccgagTTTGGTAGTGGC +CTAGAACCATGCAAAGTTGCATTCTATGCCGCTATTTCCATAGTTCgggt +ttttaggccaccatatcctaatttgggtttctaggccccattttatatta +ggttggcgggaaagtctttgtcactttttcaatagtttatttctcaaaga +aaaaataaccgaataattaattatattgtccatcattatcaacaacatgt +ctccaacgttccctcaatttatggattccattgcggtagaatccagccgg +gcgtgaggcaaagtattgtttgagtgccgcttcgaccgctccacgggaat +tgaagcttcttccttcgaaggcacgtgtcatatcgccaaataaccaatag +tcggaaggagcaagatcaggagaatacggggggtgagggagaacattcca +tccatatcccatcagctcctgcttcacttgtttcgaaacatgtgggcgag +cgttatcatgttgatagtacactttcttgtctttcagggccgaacggtcg +actgctttcttgaggtttcgtagttgagtagtgtagaggtctccagttat +tgttttgccctcgggcagaagttcccagtagatggggccgtccactcccc +accacacagagagcataacttttttcggatgaaggtcgggtttggggaca +tcctgtgggctctctccttctccaacccactgggatttcctgtggtgatt +atcgtagaagacccacttctcatctccggttattagccggtccagccatc +gatcgcccgcgtggagagtgagaagagtaatcgaatcgtcgactcggcgg +tcaaggttggcttgagtcaggtgatgaggtatgaatcgaccaaaaagttt +tctcattccaagagacttcaaaccttcaaggactgtgctatggtgaactc +cgaatgtgacagctagttcacggctagattgaaatggatccgattgcaat +gcgtgcttcagcttgtccaaatccaactcacggggtctcccagagcgatc +cttttcatcgagcgagaagtcatctttttcgaaacgatggaaccaattgc +gaatagtattgtagtgaggggcatgtgtgttgtacacattcttcaagaaa +gtttccgaatccttcatcgtggtgtgagataggaagaggtaaatcaaaac +atgtctgatgtgaatagcagtgggaacaaacatatcgtttcaattacaga +tgaggagcagggaaggagatggggacttgtttatatggaaggttcttatc +accttgttattcactaaagtcgatatcatacacagcctccatatacgaga +acagtgattaaacaaacagagtgacaaagactttcccgccaacctaataa +tataagatttctaggccaccaaatctCAGCTTTCCAGAATTCTAGGTCAT +CGATCAAAAACCGTGtcctagaattccaaatttcgaatcgaatgacctag +aaacacaattcaaaaattctgaaccagaattctggatttctaggccacca +tgttaaaatctgggattcctaggccaccatttcctatttttcgatttctg +ggCCACCATCttcaaatttggttttctaggtcataataTAAgtcttctag +gccaccaaatctCAATTTTGGAGATAGAAACCCAAATGTTGGAGGTAGTG +CCCTAGGAAACCAATATTTTGGGTTCCTAGGCCATCGATCaaaaaaaCGT +Gccctagattttcaaatttcgaatcgcatggcctagaaacacaattctaa +aattctaaaccagaattttggatttctaggccacgcccactaAATCTTTT +CAGCACTGGGCGACTCGAAAAATGCGCTCTCTTCGAATAACACCTCCGAA +TCGATATCCCAAATAATTTCGCTATGGAGAAAATTGTGTCAATCGGATGA +TCTCCCaaaaaaaGAGCTGCCAAAAAATACATCAAAATCGTCGGATCCAA +TCAATGAGGTCTTGTGTTTGGAGACGATAAACGCACTGATTTTGATCAGA +AACCTCCATAAAAGCATTGGAGACGTCGCCAAATCCATGAAGAATCCATC +GTTGGCATCTGGAGTCACTCAGAAGACGATCCAATCGTTGATCTTTCAAC +AGACACCTGATGAATGGGATCGATTGTGGGCGGGGCCAACGGACCCCGCG +GATTTTTTGAACGCGGTGGTGAAGAAGACGCGGGGAACGATGCAGGTggg +gggttttgggattttctaggggaaattttgaatttttgaaaatttgaaaa +aaataagaaaaattaatttttggctcaaaaatttgaaaatttaaaaactc +aaaaaaaattgaaaaaatttttttcaagtcttttaatttttgattttcag +acaaatttttcaaattcaaaaaaacaatttttttggaaattttgaaattt +cgattttaggatcaaaattggattttccggtttttcggctcgaaaatttg +aatttttcgaattaaaatcgatttttttatattaaaaaaagggctttttt +caaaaaaaaaaatatttctttgaaaatttgaatttttggctccaaatttg +aaaattttaaactcaaaaaaattgaaaaaaattgtgattttaagaacttt +ttttagatttttttttgacaagattcgaaaaatattttttgcctcgaaaa +ttatttttttcaaaatttcgaaaattttgaattttttgctattttccgac +tttcaaaaatttcgaattttttaaaatttttcagatttttttgaactttt +aaaatttcagattttttttgaattttttgaaattttgatttttttgcagc +TCTACGAAGCCTCCAAATCCTCGTCGATCCTCTCCTCTCccatcgatttt +tcggatctCTTCTATCCGAACATCTTCTTGAATGCATTGAGACAAACCAC +GTCTCGACAACTCCAAATTCCGTTGGATCAACTGATTCTCTCTTCCGCCT +GGACACCATCCCAACTTCCACCGAAACAGTGTGTCCAGGTGCAGGGTCTT +CTACTCCAAGGTGCCACTTTCGATTCGTTCCTCCGTGAGACTACCGTATC +CAGTGCTGCCTACGTACAGGCGCCGATCCTCTACTTGGCATGGACTTCTG +ACAGCTCTTCGACGATCACTGGCGAGCAAATCCAGGTGCCGCTGTACAAT +TCCAGTGAGCGTTCCGACTTGATCACCGCTGTCAACATGCCGTGTCGTGG +TGCCGATCAATGGAACATCGCAGCTGTTGCCCTATTTCTgcgataacaaa +aaaaaagattttttaacaactaaaaatacagcttaaaattaaataaaatg +tattgtgcgtttaaaagtgaataaaaaatcatgaaaaaagatCACTAGTT +ATCATTACCAGCTAgcagatcgaaaaaaaaggaaaaaaattattgtacAT +GAAAGCGATAATAAATGAagcgaaaaataaaacaatttagtTTTCAGTGT +CTTCCTCCTCATCGATCGGCGCTTCTCTCCGCTTCTGAATCGCGTTTGGA +CTTGGGAAGAAGTTCGTCGAGTCCAGGAGGCTGTCCCTTCTCACGAACGC +CGTCACACGACGCTTCTTTTTCGCGAGCTCTCCGTCGCTTCCGTCCTCAG +CGTCCTCCTCCATTCCGAGCAATGTCTCCGGCTCCACGGGTTTCACGAGC +TTCTCACTGGCTGGGTACTCGGGAATGATTCTCGGCGGTACATTGTTCGA +TGGGTCGACGAATGTCATTTTGACTGCATTGATGGCTCCGGTGGCCTCTT +CAGTGACGATGTGCGTGGCTTCAGCGAACATTGTGAAGAGTGACATCTgg +aaattatcatttttaagTGTATACAGTTTgatagagcgcatttacatATT +TTTAGCTGGGAAATACCGCTACACATGACAGTTCTTTCGCAAACGGTGTC +AGTTCAATTCCCCGTGCACATCCCCAAAACTTCTTTTAACTTACCGTTCT +TTCTAACGACTCCTGGACCTTCGTCTGGAGGCTTTCGCCAGTCGTTCGGC +ATGTTTCACTCGTTGTCGAGTATCTGCCAAGTGCAGCCTCAACTTGAGCA +GTCGTCTCCTCACGAATCGCATTGGCCTTCTCCAGAAGCGTGCTTGTCAC +CTTCTTCACCTCTCGGTCCTTCTCCACAGCTCTTTCTCTTGCCTCCTTAA +CCGCGTTCTCCATATCCATATCGATATCATCGAGATCCCCACGTGCACTG +TCTTCAACTTCCTTGACATCGCAACACTCCTCGATGACGTCTTCAACCGA +TTGGATGACATTTTCAGCAGCGTTGATTGTCGAGAGAATGGATTCTTGAG +TGTTCTGATCACCAATGAGTTGTCTGAAAACCGGGGAACTTCGAATGAGA +TTTTCTAATTGTTACAACTTACCTCTCATGATTCTCCTCTGCTTTCACTG +ACAACTGTTCCACTTGCTGCATCAACATCTTCAGAGTCTCCGCCATCCCC +GAGCACGTCTTCTTGACTTCAGCGACGTTTTCGTCGCTCTGCTGAACTCT +CTTCTCGAAGCTATCCGAAGTGGTGTTCCGAATTTCAGTGAGCAACGGTC +TCACATCATTGGCAAATGCAGCGACTTGTCGTTTGATCTCTGCAAGCTTC +TCCGTCAGCTGCTTACTCAATTCCTCTGATCTCGTTGCGTGAGCCTTGTA +GTACTCAACGCACGTCTTCAGAACAGCATCCCATTGCTCACGATTGTCGT +TAGACAGCGAATTGATGGTGTTGGCCGCATTCTCCTCGATCTCGGTAGTC +TGTTGGAATACATTCATGAGTGATGTCTGGATTTCCTTAGAACTTTGGTC +GATGCTGTCCGCAACAGATCCAAACGTCTTCACATGTGGCGTCACTTCAT +GCACGAACTCTTCTGATGCCATTCTCCCATCATTGGCGAACGCTTCTGCT +TGGAGTGAAGTCTTTTCGATGAAGGAGGTCATCTGGAATGATAGAAGCTT +TAGTTTTCCAAATTACCAACAAGcttcttcaaaaactcaccttcCGCCTG +AATGCTTCGAAAATATCCTTATTCCGATCGTCAGCAGCTCCCATGTCGTC +CACTTTTCTCCAAAACGCCGCAATGTCGGTCGTCAACTCATCGGCAGCTT +CTCGAGATTGCAATAGCTGATCAAATGACTTCAACGCGTTGGTGTGCATC +ACTCCGATCGCTTCAATGTTCTTATTGAGAACGTCGTTCGTTTCCGCGAG +CTCCACCACTTTCTCAGCATACTCGCGATGACTTTCTCTTAGACGGCTCT +CCAGCTCTCCCTTCCGCTCATACAGCTGCTTGTACTGATCATCCATGTAC +AACAGATCCTCTGTGTATCGTCGAAGTCTGTCCACTGCGCTATCCAACTT +CTGTTCGAGCTCATTCACCTTTTCAGTGTTGCGTTGGAATTCCTCGTGCG +ATTCTTCACTGATGATGACTCCGTTCTTCTCGCGAGCCGCTCTCAAGTCC +CGACGCAACTTCTCAATCTCATCCGAGTACTCTTTGAGAATCGTCTTCTT +TGACACCTTCGTATTGCACACTGGCTTATTCTTGATGTTCGCTGCTCTCA +TGGCGTACTCCAGTGTGCTCTGACTCtcctcaaaatttgagctcgATGGT +GAGAGAGTTGCGATGAGTGAAGTGATGGTGGATCCTCCCAAACTGTCCTG +TAGAAGTCTTGTGAGCTTCGACTCTCTGTATGGGACGTGTTGAGCATTGG +TGGTGAGCGATCGGATGACTCTTCCAAGTGTCAAAAGTGACTGATTGATC +GATCCAGCTTCCTTGGCACGATTCCCTTGAGCTCCTGATCGTCCGATGTT +TTCAGATCCAGCCAAATCGACGAGGTTCAGCTTTCCTTGCTTCACCAACT +CCTCTCCAGTGTTCGTGTTTTCTCTTACAACCACATTGACCATGAACAGA +CTGTGAGATCTGCTGCTGTTCATGTTCATTAGTGTGGCTGCCGTCTTCCG +CTTCTCGGCTCCAGTTTTCAGCAATTTGAAGACGTCTGAACGACTTCTGA +CCGGCACTTCTTCGACTCCCGACACAATCACTCCCTTTTTATGGGGATCG +TCAAAAATGCGAAGACGTTCACGGTCTTCGTCGTCCGAAGATGCCAGAAG +ATCCAAGAGCTCTTCGTTGTACAATTCCACGTAGCTGACTCGCAACGAGT +ATTCCTCACATTTTGATGATtccaacttttcgaaaatgtccTCAACAGCT +CGTGGAATAATCCCAGTGGTCGGATCGTCTTGTGAGCTCTTGGCATCCGT +ACGCCCTCCTTCCATCGTGAACGTCTTTCCCGTTCCAGTTTGTCCGTAAC +TGAAAGAATGGATATTACTggttgaattctgaaaaaatcttacGCGAAAA +CTGTGCAGTTGAATCCAGCGATCACTTTCTTGATTTGAGGAGACACCAAG +TCGGCGTAGATCTCGGCTTGCGTCGTATCCGTGTCAAACACTCGAGAAAA +AGGTCCAAACGGCTTTGCCTTGAGTTCGACAGTGTTCTTGATCTTGTCGC +TTCTCGCGATGCTCGTGCACTTGTCGCTCCGCTCGGCTCCATTGAGtggt +ctggaaaattgaattttgaagtttccaaactgtttcaaagaattttcctG +ATCCTCACAGCCACTCGAATATGTGACGTCGCTTCAGAGACCTTCTTCCG +TGTAGTCATCGTTAAATTGGATGTCGTATCCTGAAAAATACGATAAATTG +AGTATTTGAGCAATAAATGGGTGTGCTGGTGCATTCAAAAAGTGTTGCTT +CGCGGCGCTCTCATCAGAGCGnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnTCTATAGT +TCAAAAGAACCGCAATTATTTACTCTTCATTttgtatggatgaatttttg +attcgtctgatataattttttttatgattaaaaaacaacgaaaaaatgtg +attttgtgcaatttcgtgaggttttttgattataaaaaaaagttttatca +gacaaatcaaaaattcatcaataCAAAATGAAGAGTAAATCATTGCGGTt +cttttgagctatagaactcctagaaatgtcaaaaactgaaagaggagata +catttttaCTGTAAATAACTTTcgaaaatgaccatttttgaaatgttttc +ttccggaaaaggaccatacaggattctataaccactaattagttttttaa +gaatcatttgaatcatcttcaaTCTCcgcaaaatcagataaaaaaatctg +aaattatggttttagacctgcaattgtaaTCGcgccaatatttttttctg +tagatcaaaatttaatttgcagttcggaaatgttctgtgggagaccgTGA +AATGaggaataataataatatatttacttgcaatgcatgggggaaataga +tagataatttggcaagaaatgggacctgtacaaataaaagaagtaaaatg +accagacggaccgactagagtcctgtgaaaaatgaaagaaaattcaaaaa +ctcccgctcaaaaattcgaaaaccatCTAGACCTTTCTGCGGGTTTCGCC +ACGAAAGTTCAATTTTaagcatgcgcctttaagcaGGATTACGGTATTTA +GAAGGTCTCGTCACGGAGAGCTTGCgcctttgaaaatatttcctTTTCGA +AGTTGGTCTGAATTTAAAATGAGTGTTCACccgttttcagcgatttttca +acgttttttcagctttttttctacttttttgattaaaaaattcaaaaagt +ttcagaatatcatGGTACTTGCCCTCGATATACCGCTAAGACCGCCGGAG +CGAGCCGTACTAGAGTTTTTATtgtaggattttttgaaaaaaaaacccgt +tttccgaagaaaattttttcagaggcGTAGTCTTCGGACCGCAAAGAAAT +CTGAGCACCTACACAAGCCGCAGAAGTTGTCAGCGACACGATGTGGCAGC +AGCCAGTCACATTATTCGCCTCTATCTTGATAGGGGTCGAAGAAAGAGCG +GTTTCACCGATGCTTGCCATCTCGTCTATGGCAACACGATTGTGTTGAGT +ACACAGGTCGCCAGACTTTTGCGTAAGCAGGATTTTTGCCGGAATTTTGA +CTTTTAAACGGGTTTTTCTTCCAGGTGATGCGATTCGAGCCAAAGAACTC +CTGTCCTATTCTTCCTATCTCgcagaggaggaggagagaagACGTAGAAC +TCGCGAACAGATGGAGAACGGGCTCATTGACAACTCAAAGAcaccgaaga +agaagaaaaggtcGAATGGAGCAGATGAGGACGAATTCGCTATGAAAAGT +CCAGTGGTCAGTACCaaaaaaaCTTGTCTGGactttttaaaatcgaaatt +ttcagagtgtcGCCTTGAAGAAGGACATCACAATGCGCGACGTCTTGGGA +ACGCCTCAAATCCAAAACATCGAGATTGACGATGATTTGGCACCGGCCAC +AGTCGATGATCTTGCTCTGCTCTATGGATTCCTTGGATCAGAAGCCACAC +AAGTTGATAAGTCAGTTTTTCTGccgaaaaattcgtttttgagcccaaaa +tcatcattttcagaacatttatCGATCAACAATCCTCATCAGCCGCGAAA +AACCGTGAGGAACTGATCCAAGACTTCGATTTCGAcgctccaccaccacc +gccacGACAATCATTCGCACCGCCCATCGACGACGATTTCGAGTTCCAGC +GTCTAAGCGACGAGATGATCCAGCAGTCGCATCAGACGCCAGTGAAAAAG +AGACGATCGGGAGAACGAACATTCGAGCCGGGACCTCATCTCGCCGAAGT +GCTTCTTTATGGACGATTGCCACTCGTCGATGAACGTAGGAGTATGGAGG +ATGAGATGGAAGACGCCAGGAATCACGGAATTGTAAGTAATTGGTCCTAA +AGTTGtcgaaaattaggtcaaatttCGAAGAGAAGCGGTCTAAAGATCAT +TTTTGACCCGAAACCAACATAATCTTTTCAGAACAACGACAACCTGGACG +TTCCTTCTGCCCTGGTCGATGTCCCACGACTGCCATCGCAAATCTCCCTG +GACGATATTCCATTTCCTTCGAGACCACCCCAGCGAACACCCTTCAGAAC +ACCGCAGAGCACACGGAAAGCCAAGTTCGATGTGTACAATGGAGAAGTGG +ACCCGACGCAGGCCGATAACAATCGGTTCAGAATTCCATTGGTAGGGTTT +TGACTGTCTGAAGACCctataaatttcgaatttccagcgAGAGATGAATC +GAATGATGGAGGACTTCTCTAGTCTCCACTACGAGGATGGCAAGATGCTC +AGAGTGAAGCCAGCAGCCAAGAGCGCAGTCAAAGAGCTGTTCAGATCTGT +TCCCTACTTCATAAAAGATCGtgagtagttttttttctggatttttgact +tgaaaatcgttcatttttgtctaaaatcgatatttttgacctgaaaatgg +tttttttcgacctaaaattggtcattttcgaactaaaaaaaatagttttt +gaccCTGATCATCTAGAATTTCTGCTTTCAGGACGCAACGCGGATACATG +GGACGTGTTGGATGTGGGTTATAAGAAGCCGAATTCAAGCATATCCGGCA +ACACTGTGTCTGAAGATGAAGACGAAATCGAATCGTCTAGTGGTTTTCGT +CGTCcaaaacaatacaaaaatttggagCCTATCGATTTGTCGAACCTGAA +GGTCTCCATGATGCCGTTGAAGGAGACGTCGCCAGTGTAAGCATTTTAGc +ctaaaaaaaccaatttttgactcaaaaatcaactgtccagcaaattgatt +tttagggtATCGGGAGAACAAAATTCAACTGTTTTCCCTCTTTCAGAATA +TCTCCACTCCGCCCAGTCGTCGATGGTATCGATATGGAGGATTTGGAACG +TCCAAGAGACCATAACGACGTCCCTCCAGAGTTCGATTTTCCAAGACCAG +AAGACGTTCATCGTGAGCCAGGAGAAGTTCCAGAGGAGATTGAATTCCCT +AGAAGAGAGGATAATGGATCCAATATCACTAAGATCTACGACGACGCCAT +GCTCGATGAAGACCCTTTCCCGActtccaaaactttcgaaCCACCATCAG +CTGGTGATAAGTCGGCTGCCACCATTCGCGAGGAGCTTTTGAGGTGAGTT +TTGGCTGAAATGCTTCATTTTCCACTgcaaaacgctgaaaattttagaca +gTGTGAGGAGGAGAGCCCGTACCCGATCCAGTTGGACAGCATGACTCCAA +TCGAAACAACATCTCGACGAGAAGCTGCGAGGACGTTCTATGTGGTTCTT +GGTtagtttaattttgaaaatctgagacttcttctgaaaattcttcttcT +AGAACTCATCAAAAACCGGCGCATCGAAGTCACTCAGGTGTCGCCATTTG +GACAAATCGACGTGCAGCTCTCGGCAGAAGACTTCGAAGATTTGGCGGTG +TTGGCAATGGAGGAGGAGATGGACGATGGCGATTTTTAATTGGACCCCAT +TAATTCtgtcatttttgtctgtataatgtgtttttacgacatttttcgtc +gatttttagtttgtcttttttctaaGCCAGAAAGTCTTGTTAAATTTAGT +CTACTGTAGAattttctagttagttttcattttctagtatttaaaaatta +tttttctttttaatttttttttgtcgggttctcaaaaaatcacaccCAGT +GCATGTATCATCCTAATCATgtttcatctcttttttttgttttaataaaa +ttctatttttgttgtttataaatttgcaaactttgttaaattCTTgagat +ttttcttcattccccGACTCTCTCCCCGACAATCGATTCTTCTGCCCGGC +GCGAAGGTAAACGCGCTCCACggctaattcaaaattttcgcatGCTAGCG +ATTGGCTCCATTGGTTTGCATGGAGTTTAATCCGGATAAGTTCATATAAT +TGCTCTTATTGCTTATCTTTATCGATCAttttttagggaaacttgttttt +gtctccttttttccgttcttttttgtgtttgcctttAAAAGtatcgaaaa +attagaaattttgttgtgaaaaattgtggaaaaaccACTAAATTATAAGA +ATTGCAGGtttttggtctaaaaattgctgaaaatctgaaaaattccgcat +tttttaacgattttccagtcttttttcgttgtttttttttcgagctgtaa +aattttaaaaataaaagaattttcgctaaaaaaaaGTGCTGAAACATGTC +GAAAATTGccttattttcagcctaaaaaacaataaaaattggaatttcgc +ctcaaataggctccgccccctttccACTCGATATTCTTTTCATCTTTCCC +GATTTGCCAGgttgtaatttcaaaaattttcagcactttccggcctgaaa +aatctctaaatttcctttttttttgcttaaaataggctccgcccactcgt +TTATTCGATTTCCCTGACTTTCCaagttttttcccaatttgcTCCAAAGT +CCCAGATCCATCTgacaaactgaaaatttcaagcgATGCCACCAAAGAGG +CGAGTCCGTGGTCCCCCGCTTCCAGCACTTCCAGAATCCCCAGATCGTCA +AGAAGACCCAATCGATGAGGTGGATCCAATGGAATCGCCAGATCGAGACG +ACGATCGAGATAATCAGAGACATCGAAATATGCCAGACTATATGGATGAC +GACTTTGATAACGATGAAGAGGAGACATCAGACGGCGAGGATGTCCAACT +TATTGAACAGGAGCATGATACGAATGTTTATGATGATCGAACCATTCAGG +GATTGGAAATTCGAATGAGATACACTAAAGTTACGCCAGATATGACTGGA +GATCAGAAGATGTTGGTTCAGACTGGCAGATTGGTTAAGTGAGTCAGGAA +ATTGGCCTTTTTCAAGCTCAAAAGCGCATTTATCTGCTCAAAAAGTACCT +AAAAATGATGCTTTTATTAGGAAATCATGGTTTCTTCGGTCAATTATTGC +TAAAACTTGTAATTTTCTACTGGAAATTGAAATGgctctgaaattcttta +ttttcagcttcgaaatcgttttatttttatttttttcctgccaaaaattc +attgaaactGCCTGTAAAATTGTctctttttcaacttcaaaatcacattt +tttcaacaaaaatagtgattttccAACAAACAATACTTTTTCTgtcagaa +atttctaaaaatttcaatttttgaccgaaaatttgctgaaaatggcttcA +AAAAGTCCTgttttttcgacaaaattgTCCATTTTTGACAAGAAAATCAT +GTTTCTGGGCATATTTCAGTCGTTTTCTAGTCgaaatttcctcaaaaatg +gcatttttttttgtcaatttcattaggaaattaaaaaatttcgggatttt +tagcTCCGAAAATCGCTATGTGAAGCcagaaaatcactgaaatttgaata +atttttttcaaaatttcagaaaaggagTCATCGATTCGGTTCGTCAGGCG +TTTCAGGCTCGAGACAATATCGGGGAAGAGCTGATTATCGCATCTCAAAA +GATGATTGTCGCTTTTAAAAGGGTAAGAATTCGAAAGCATTCGAGAAGTC +TGGcaaaaattccatgaaatttGGTCCACAATGACCGGAAACAGTCaatt +ttagcttatttttatcgattcttatagattttttcgatgaaaattgccaa +aaatcgagaaaattatcCCCAAAATGCACACCTTTCTCTCTTACAGGCCA +ACAACACGCGTCACAAGAAATGGAATCTGGTCAAGGAGTTCTTCATGGAG +CTCGACGCTCGTCTAGTGATGATGTTCGaggaggttactgtagccgaaaA +CCGTGCGAGAGTCTACAAAATGGTTGCGATGGTGGCGATTGAGCTGCTGG +TTCTGGGCTACAATCAAGAGTTCTTCCACTTTGTATTGGGATTGGTGGAG +AGTGTGAGTGACaagatttttgtgatttttgaagctatttttcatcaatt +tctagCCAAGAATTGAATCCAAATATGCCTGAAAATAAGCTATTTTTTGA +CATGTTTGTGTTTTGTTATTGACTAAAAATTCGTCAGAAATCGCAAAAGG +AAAATGCTTCGaaaattgaacgaaaattAGACTTTCCAATCGAATAACtg +attttttgtgtcattttttgttggattttgagccagttttgttatttttg +aacctatttttttgactaaaactTGATCAAAATTGGCTTTAAAATTCACC +GATTTCTAGTGgaaaatcgtcaactttggaTGCCTTAAAATTCACCCAGT +tagttaactttgagctcaattatctcgattcctttgagagatatcaaaat +tttgtcaactgataaaatgtttaccatgatatcagcaatattctatcagt +tgacaaatttttgacatctctcacatgaattgagataattgagctcaaag +ttgacgattaagtggggcttctattagattccacattctccatttttcag +tgGTCCGAAAGCGAGAGTGTCGCGTCACGAATCAACGCCTCGTGCTGCGT +CAGCTACATATTCGAGACTGGAAAAACGCACAATCGCCAACTGACGGGCG +AGAACTTCTCGGGATTCCCACTCGAAGGAGCCTATCGTCTGTTTGCGGTC +CTGAAGAGGGCGTTGTTGGATAAGGAGGCGAGCGCCAGAATCCCAGCGAT +CAAAGGAATGGGAGTGATCCAGAATTGTGCCATTCCATCGGCATGGCCAG +TTGAGGGACGGGAGTCCAGTGCCAGGgagtgtgagttttttttgtggatt +ttgaaccatttttcatgatttttggacttttcatagattttagaccattt +tttcatgcattttgggtcattttccattatctttggtctatttttcatgg +attttgaaccatttttgggcatttttatttcaaacttttcagtgCTGACC +CGCTCTTGCCGTGACATGTCTTGGGAGTGCCGCCTGGTCGCTATACAATC +CTTCATGCCACGTGACGAAGACCTTCAGCTTGTCTCCGACATCGCCTACT +ACGACAAATCGACAATCGTCCGAGTAGCTGCACTCGAGCAGTTGGGCAAA +ATGAGACCACATCGGGATTGTAGGGAGAAGGTCGTGCTCGTCGACATGTG +TTTCAAGGATCATGAACGTAAGATatttagagaaagagagatacCGTAAT +CCCCTTCTAGTGATCTAGATGGCCCTGGTGACAGTCCTGATGGCCTCTAA +CGACCCTTCCAACAGCTAGATAGCCCTAGACAGTCCTAGTGATCACCTTG +ACGGCCCTAGATGGCCTTAGACGACTAAGCCGTTCCTACTGTCCCCTAGA +CGACCCTAGACTGCCCTAGATGGATCTAGTATGGATCTAGTCACACCTGA +CGGCCCTCGAGACCCCTAACGGCTTTTTTAACAGTCCTAGTAACATCTAA +CGTCCCTAATGACAGCCTAGATGGCCCTAGATGGACCTAGTGACACCTAA +AGGCCCTTCTAACGACCCTAAAGTACTCTAGTGACCTCCAACTGCCCTGG +TGACTCCTTAGACGACTTCAGATGACCCTAGTCATCATTCACGATCATTC +TAACAGCCTTATGGACCCTAATAGCCTAGACAACCCTACTGACTGCCTTA +GAAGGCCCTAGACGATCCTAATGACCCTTATATGCTCTAGTGACGACCCT +AAATGTCTCTAGTCATCTCTAATGTCTTTGCTGACGGCTTTGACAGCTCT +AGACGCCCCTAGATGGCCCTAGACGCTCTGAGACGGCCGGAGACGGTCCC +AGTGACCCCAAAAAACAACATTCCCATTTCCAGCGGCCGTTCGAGATGCC +GCAAAGATGGTGCTCCGTGAGTGGGTGGCTCGTCTTGCCCGCCGATACGT +CAACAAGGCCAAACAACAAGATGTGGTCCCGATGCGCGGTCCAGACGATG +AGGAAGCCATCCCACCGCCAACAGGAAAAGGATATATTCTTGCTGGACAG +GGACTCTGCCTTCTATGGCTTGTTCATATGGTGGATAGTATGGAGAGTCA +CCTAAATCTTCGCAGAATGATGACCCATTTGTTGGATGTTATTCGTCATA +TGTACCTGTGTCATGCGGAGCCAATTAGCACGTTTGCTGAGCAAATCGTT +GTGGATCTGAAGGAGACCCAACCGAATTGTCCGGTTCCGTTGGTGACGAA +ATCGACGATTGATACGTTGTTGGTTGATAATCCGTTGAACGCTgataagt +gagtttttaaaaatccttttttagcCAAACTGAACTGCTTTTTTCAGCCA +AGACACCAACACGACCCGCGCGATGGTGATGTTCTGGCGCTGTCTCGTCG +AGTACATGTCCGATCGAGCCAGAAACGAAGGAGACAAATTCCACGCGATG +AGTCGTTTCACCCACCCGTTGGGTACCATGTGCGAGAGTTTGGAGACGTT +TTTGAATCGAGTCAAAGAGGAGGAGGTCCCACCGTCTGGCGGAGGATATC +ATGAAGGATTTGAAGACGACGTCTATGTAGTGCAAATGTCGATTATGGAG +AATTACCTCCCGGTGTTGAAGTACTGTCAACAGGATCAGTCTGGTCTCGA +CGCCTACAAGCAGTTGCTCATCTCAATGTTGATGAATCCGTTTTATCAGA +AGAAGGTGATGGATATCATCGTCCAGGAGCTGGGGCAATTCTATAAAGAC +GATCCGCATGAGTTGTTTTCACTGATTTGCGCGCGGACCAGTGAGATGAG +AGACAAATACGATGGTTATCATTTTCCGGTGCTCGAGGGAACGGTTTTGA +TTGGAGAGCAGAAGTTTAGGAGGGATTGGAGAAATGAGATTGCCGCCGGG +GGGACGccgaagaagaaggagaaggatgTGAGTGTGGAGGGagattctgg +aagattctagaTGGTTCTGGAAGGCCCTGGAAGGCAGAAGACTCCAGAAG +CTTTTGAAAGGTTCTGGAAGGTGtttagaagattctggaagactcgggaa +catttcgaatgattctggaagattctagaagataCTAGAACGCTCTAGTA +GTAGCTTGTGGAAGACGCTGGAAGGTTTTTGGAGATTCCCGAAGTTTCTG +GATAATTCTAGAAGCTGTCGGAAGACTCTGAAATACTCTGAAAGATTCTA +GACGATTCTGAAACATTCTAGTTGGTTTttggaacattctagaacGATCT +GGAAGGCTCTAGAAGATTCGagagaattctagaatcccttggaagtttct +ggaagttttttgaaggttATAGAAGTTGTCGGAGATTTCTGGAATATAAT +GGATCATTCTGGAACATTATAACAGGCTCTAGAAGCTTTTGGAAGATTCT +GGAGCACTATGAAAGATTCTAGACGATTCTGGAGCATTTTAGAAGTCTCT +AAATGCTTCTGAAAGGTCGCGGAAGATTCTAGAAGGCTCCAGAAGCTGTt +ggaagtttagaaatttcatatttgCAGCTCGTCAACCGTCACATCGATCA +ATACGAGATAAAAGTGCTCAATTGCTGTCTGAAGTTGGCACTGCTGAAAG +AATGGATTCCAGAATATGATGATCGGTATCGGGAGAAGGTTTGTAGCTGa +aattcgactgaaaattatcaaaatgaccatttttcatgccttttttttca +gctccgcGCCCAAACCAAAGCCAAAGACGACTCCACCAAAATCTGCGCAA +TCGAGTGTCTCGGAATCATTGGCTGCTACGATTTCGACCTTGTCGCCGAC +GATTTCCGTGAAATGGCTGACTTCCACTTCCAAAAACTGGCTCGCCACGA +TCCGGTGACCCCAGAAGAGCACATGGTTCAAGCATCGGTGGTCCTCGCAC +TGGCCGATATGTTCGTTGAGAATCCCGATAAATACACGGATAAACTGTAC +GAGCCGCCGTTGGAGCACTGGAAGAACGCGGAGAACACACCGACGTTCTT +CGCTGATATCATCTGTTCGGATCAACATAAATCCGGATCGGAAATTTTGG +TTCGAACTGTTGAAGCACTGTGTCGAACacttttgaattcgaaaatcgat +cCGACTTGTCCCGCTTGGCAGAGATGTATTGTTACGTTGATGTACAGAGC +TTCGTTCAAAATTACGGATCACTATAGTGCTAGACTTCGATCCACGATCC +TCGTTACACTCCAATTCTTCTGCTCCATGaatcagtgagtttttgagcta +ttttcaactgatttctAGTGgaaaatggtggcctaactttgagttttcta +ggccaccatgactTAGAAATGTCCTGATATTCACCTGatgcgttttctag +gctattttactatgatggcctagaaaaaacaacttgcaaaagttaggcca +ccgtgtTAAACGACCTAAAAATAGCATGATATTCgcctgatgtgttttct +agactatttatcatggtggcctaacttttgcaagttgagttttctaggca +gCCATTCTCAAAAATGGCCTGATATTCCCTtggtatgttttctaggtcat +ttaccatagcggcctagaaaactcaatttttgaaagttagtccactgtta +aaatggcctagaaatggcttGACTTTCATTTTATGTGCTTTCTGAGTCAT +TTTAACAAGGTGGCCTAACGTATGCAAGttgagtttcctaggccatcata +gaaaatgacctagaaaacaccaGGTGCATTTTAGACTATTTCTATACCAT +TTCCCATGGTGTCCTAGAAAActaaacttgcaaaagttaggccaccatgt +cctTAAAATCCtataattcgaaatttctgatttcagaCGCAACCAACTCG +TCGTCGTCCGATCGTTCCGTCGATTCTTCTCCCTCTGGACCAACACCGCC +GACGTTGGtcttctcactgaaaatcgcaACGAGCTATTCCCGAAGCTGAA +GCGATGTGCTCACGCGTTTGCAGCTCTGACACGTCATACGGTGCTTCCGG +CTGACATGCAGAAAAATGTATGTGGAAAAGTGGGCAGAGTCAAAGGGGCG +GAGCTGAAATGcaagaatttaaatttttgaaaaagtggacGGAgcctttg +aaaaatatgaaaatctctaaaagtCCTTTAGAGATCAATGGATTTTTGTA +CTTCGAAATTTGTAGGATTTCGAATTCCTgagttttcggaaaattgggcg +gagccaaaagactaaaagggggcggagtttgatgataaaaatcgatattt +gcAGTTCCAACCGTCGCACATCGAGCTATTCGACGACATTTTGGACGAAA +TCCATGGACGCAGTGCGACGAATCTCGTCGATTTCTATCTCGCCGCCCTG +CCACACATTGAATTCGAGAGCTTCAGTCGACAGGTGCTCACCAAATATCA +TACGGATCTTCAGTGCTCAATGGAGGTGAGTTTTCAAGAATCAGGAAATT +cgaacgttgagctcaattatcttgattcctgtgaaagatatcaaaaattt +gtcaactgataaaatgttgctgagaATATTagagacattttatcagttga +ccaatttttgatatcttttacaggaaatgagataaatgggctccaAATTA +CCGGGCTTTTGCTTAACTTGATCAAGTCaaattctctaatttttgaccaa +aaattaCGCAaacaggattactgtatcctcGATTCTCGGACATAAAAACC +AACAGACTTTTCCAACTTTGCAGCTCAACGGCGAGGACATGGAACGTTCG +CATCGTAACGAAATTCGTCGTGTTCAGCGAAAAATCGCCAAGCTCCTCGG +CCTAAATATGGACGATGGTGATATCGAGGAGATGCCATCTACAGTATCAG +AACTCCGCCCAACACGACGTACGGCCAAAGCGACAGCAACGTTGGATGGG +ATTCAGGAGGAGGACAACGAGGATTCAGATGACGATAATCAACCGGGACC +ATCAACTGTCAAACCAAAACGCACACGGGCGGCGCCGACGCCAAGAAAAG +CGCCAGCGTCTGCACAGAAGAAGAGCAGAAAAGATGCGGAAGAGCTAATG +GCCAGTCCACCGGTGAAAGCAGGCAAAAGTGAGTggaaaggggcggggct +tagctggacattttcgaaattgtggCTCTAAAACAGTTCTTGACACtaga +aaaggggcggggcttgattgggaaaagaggagaagcgtaaataagaaaaa +agttgcGGAGCTtacattttttacattttttaaagcatAGGGAAAAGGGa +taagggggcggagcttaactagaaaaaagaaaaaaatggcgGGACTAGGC +CGAAAAATTGGTATATTTAGTAAGAAAACGAGGCGGAGTTTAGATAGGAC +AAAGGGGTTGAATCTATCCATTGAAGGGGGCGGAGATTACCtagaataaa +acatttttcagtacacattgtagtttgtagtcctTGTAGCCTAGCTTTTA +TAATTTGAAgccaattaaaattttgttttttccagaaccatCTCGTCCAA +TGACGGCGACTCGTCCATCATCACGTACTGCATCAGCCACACCGATGCGT +ACAATGCCGCCAAGAAGTGCGCGAAGACTtaattaattgattttttcaac +tttttttaaacccCATTTATTCGTCCCAAAATCTCAAGTTTCCCCcagtt +ttctttaaaaatctcgaatttctaCCTAATTTTTCGTCTTAAAATCTCCC +CAAAAGTCCCCCCTCCTCTTCCAGTCGACACCCCACAGTACTGTAATTTC +TGCGATCTCTCCTTTTGTTTTCCATGCATAATATCTGTATTCTATTTATT +TTTACCAGTTTAtatttcagtacaaaaaaggCCTGgaagaatctcaaatt +taaatttcccgcccattttttaccaaaaaatctCTCTCCACGTCGGATTT +TCCCACCTGaataccaaaaattcaaatttcccgctcgtttttttttgtca +ttttagtttcaaaaaaatgcaaaaatctcCTCTTCCTTCTCTTATCACCa +ttccccttcttcttctcattatCACCGATTCGACAAACGCATtcaatgta +atttttttttgttgggtcccaccacgaaaactacagtaatcctggatCTT +TTGTGATTTGATCCAATTTTCTTCTGTGCAATCGTTTATTGATTTATTTC +CAATGATAATGATGAACTATTAATTTTggattcgaatttttgaaaaaaat +caaaaatttccaaatttgaaaattcaaaaaaaaagtttcagcaacatttt +tttcaaaaaattccagattacGGTGGCGAAGCCAGATGGTGGGACCTACT +ATCGTTTCTGGTTTGTTTATGGGtgtttttcgattggaactacagtaatc +ctcattgTGTTCACTTCGATTTCAATGTGCCGAAGATACAAATTGGTGGC +CAGTCAGGACGAAGAGgtgaaaagggggcggagctttggaaaaaaagggc +GGGGCCTCACTTTTTGAATCTGAATTAGaacaaaaaaggggcggagccta +attttctgaaaatttcagatttccgaaattcaatgATGTGGAGAAtttta +agaagaagaagacgtgaatACTTTCGAAGTGatctatttgattttttttc +gtattttctgaatgaatatggaatctgaaagtttagatttttttgatttt +ttttgaatttttcgaatttttcaaaattttgggatttttttcgaattttt +gggatttttgtgtttttgaatttttgaattttttgagttttttgaagttt +ttgggatttcagatttttttggattgtttttggttttttgaatttcggaa +ttttgggatttttgaatttttcaaattttcagaattaattcCGCGACCCA +ATGAACTACTTTCTATGTGACCTTCCGACTTGTCACGTCTATTTTCTGAC +TGTTGAGTCCTATTCAGCTAATTGGCACCGTTACAtttttatgcagtttt +tttttcaattttcatattttttctttgaaagaaaaattttaaaaattcca +aaaaactgagtatttttGGATGTTCAGCCCAaaagttgacttttttcatc +ttttttcatcCGTCAAAATTCTCCAGCGACTTCCCCGCGCGCCTTCTCCT +TTCCTCCCCTTTCTTATACGGCTTCCGCGCCCTTCTCTCTAGTGTGAGCA +CTAGCTCAATGGCTAACTCTTTTGCCTGTCGATCTAGAGTTCGCGGGTTC +GAACCTTCTCGGGAAATTtgtctttttatttttgtaatttttccaggaac +aACCGACATGGCTCGTACAAAGCAAACCGCGCGTAAGTCGACCGGAGGAA +AGGCTCCAAGAGCGCAGCTGGCGAGCAAGGCTGCTCGCAAAGCAGCTTCA +GCTGGAGGAGGAgtgaagagaaaagagaaaaaacatttccatAACCTACA +AATTTGGAATCGGCTCAtcgtgagcttcaaaatgagtataatcatgactc +tggacttcaaaaatttcaaatttcccgccataacttgtccacgaggagtc +CAAAACTAAAAACGTTCAGTAGAGCGCGCTTACGTTGTactcaggtaaac +gattctgaattagatctgacttcattcggaattagatcgaataactcgaa +attctgaattagatctgagttcattcggaattagatatGGACTAGATCTA +ATACCGAAGTTgccacgagaaatgcgccatCAATGACTCAGCTGCCGCGG +ATGaatcgttttgtttttttgttgtcctaaatttttttcgtcaggAAAAT +GGGTATTTCTCTCaacttttgccttttttttcgctcaattatttaaacta +tttttctggaacatattttaatgtgttgtgcacgtttttgcaaaatttta +ggtgaatttttttaggaacagctgaatattcaaagttcaatttataTATA +CGACTTCTATAGAAGACAGTAATTCTATGAATCGACAAAAATACgacgat +tttgaagattttaaattcGGAGAAAACTCTGCAGGTAAGataatttttgc +ctttttcagcTTGAAGGATTACGTAAAAACGATGCAAGGGATCaccgaaa +aaaacaagaaaaaagagatgataaATTGCTCTGATAGTGAGCTCAATTTA +CACAAGTCGAGAAGGCCTTACTTTCGGAAGCCGTGTCTTGTCATGAGCAG +GTTAGTTTTCGAAAGTCTGGTGGACAAAGAAACATGCGTGAAATTGCAGG +CTCTCAACCGAATGTGTCATGACCCGATTAaaacgattcaaaaatttggg +atttgttGTATAAAACAACTGCTCTGATATGAATTTATTTGGATTTGCAA +AGATTTGAGTACGGAAATTATATTGAAATGGCTCAAGTAAGTTTATTATC +AACATCACAAAGccaaaatcttcttttttcagttaaatcaACATTCATTT +CATAAAGAATATCCATCTTATCTACAACCGAAAACTATGacaagagaatt +ttcaaaatcgaaaatatggaCGAGCGGATGAATTCGAATtgagatgtttt +tttcataaatttcgtcactttttttttcttatctcaaTGTTCCCATTATT +CTCGCtgatctcatcgttttcttttttgctgatccCATTCATTCATTTGT +ATCACACCCCTCATTTTCGTTCCGATTTCCTATAACTACAttttaatcta +tgtttttctttttctcatacttgttctgttttcttttttaataaatttta +ttataaataaagtgaatggcgttgaaaagaaaacaaaaattgaaaagtgc +gcgctacgatgctccgaaatcccgctggtctgttcggaattagatctaat +tcgttcgtagttcattcggtgttcgttcggaattagataagctaaattct +gtattagatcatgccacttctatttacgaacgaacccagatctaattccg +aataaAGTCAGATCTAGttcagaatcgtttacctgagtAACTAAACCcag +aaaataggtttttttttcagtttccggTGTTTTCGACTCTTCGAATCTTT +GGGGAAATCGACGGGAATCGAAAACAAACAAATCGGGGAAATTTTCTGAC +ACTTTCCAAACATTGAGCCAAAAGAgccgaaaatttttcttgtcCCAACa +atatctaaattttctaatttttccaaaaaatgggaattcgaagcattttc +tggaaagttttgGCTGGGAGAGGTAACATCTCtttttgaggattactgta +gaatttctaaaaatgttcaaaaacatatttttagctCCGGAAAATGGCGG +CTGGGTGGATAGTATGCAAGTGGAACACTTTGATCCATATACACCGGGAC +AGCTCACCAAAAACTTCACGGCTACTGTAGGTTCTCATATTTGCACATAG +CTCACAGCTTGctcatttcttgaaattttcgtcATAAAACTTATTAAATC +TTGCTCAGCTTGCTGAGAGCTTCgaaataagcccaaacacgactagtcta +gcgCTATTTGGAGTTTTGACCtacgcgttgcggtcgcgtcgcggtttgTT +AGATTTAAACTCAAAGCTAAGcgaatctagtcgtgtttaggctcatttga +aagctctcgGCTAGCTGAGCAATATAATGAAGgtttgattttgaaagttt +gagaaatAAGCGAGTCATGAgtaatttgagattttcagaaaaccttaaat +tgctcaaaagttgctcttttttgtactcctcgtggacaatcaaaaatttt +ggagcttATCCATGACTATGTTTATTTTAAGGCTATTAAatatgaaaata +gaaaacgcgctccattCCAGGTAGCCGATGAGATTAGTGTGACACGTGGC +ACGACAGTAAAAGCGCTCTACCGAGACGATCAATGGATTTATGTGCAAGT +TTCGGATGGAAGGAAAGGATTCGTTCCACAGACATACTGTAAATTGTTGC +TGAATgcgcgaaattcgaagaaaGAAGTTGTGGACAAGGTGAGGGCGCTC +TATTAgacattttgagttttgtactcctcgtggacaagctgtggcgggaa +atttgaaactttttggagttgggtcatgattatactcattttgaagctca +cgaTGAGccgattctaaatttttggttttggaaacgtttttttcaaattt +cagaagaaagttGGAACTGCTGGAACAACATTGGATCGAAAATGGGAGAA +GTCGAACTTGCAGCGTTTCATCGACAGTTTGCCAGTACAAAAAGAAGAGG +GGGAGGTTTTCAAAATggtgggaaatttgaatttcgaaaaaagcgcGCTC +TTTTAgacattttgagttttggactcctcgtggacaagtcGTTGGCGGCA +AATTTGAAAGTATTCGTAGATAGGCCATGGTGATacttgttttgaagctc +ttgagaAGCTGAATTCGAATATTGcaaccaaattcaaaattttttgttcc +aggACGAAATCTGTGAAGCTCAAATTCTGTCGAGCTTCGATGCGTCCGCT +CCAGATGACATCACCGTCAAAGAAGGGGAAATTGTcacaattctgaattt +ctcagatCCCGAATGGATTtatattcgaaattctgagaaccAATCGGGAT +TTGTGCCGGtggcaaatttgaaaattccagatgagCATcgcgaaattcag +aatcgtgTGCCAAAGGAGAAGTGGTGAggaattattgatttttcagcagt +tctgaaagttctggaagCTGTGGAATcgccaaaaattcgaaatttcaaat +gattttttcagtggaaaccccaaaaaattcaaatttctcgctgaaacttg +tccacgaggagtccaaaatccaaaatctctaatagagcgcactttctttA +TTCGCAAcgtcagatattcagaattagCTCtctgagagcttcaaaatgag +tataatcatgacccaactttcaaaaatttcaaatttcccgccaacgactt +gtccacgaggagtacaaaaCTAGAATTTGTCCGAATTGgttgtttttgca +tttaaAATGCGCTCcgctgaacaaaaattttgtggctcgaatttttcaaa +tttcccggggcttcagaacttcaagaattttaaatttcctacCTTTTCAG +GGAAAACGAAAATCTCCTCGTGGTCGAACCATTCGCCGGTCGATCACCTC +TCGACCTGACCGTCCGTCCGGGTGAATGGATTCGTTGCACCGGCAAGCCA +GTCGACGATTGGCTATGGGCGGTCCGGATAGCCGATGAGAAGCAGGGATT +CATACCGAACAAAGTCGTCATTCTAGCCACCGATCTTTAaccttttttcg +ttgttttttctccttctgaTCAAAAGGGGCGTggtctaaaaatgaaattt +tcgatttcgatTACACCATTGTTGCTTTATTATTTCAGTATTATATAGGC +CCGGCTTAGATCTGAGAATGATGGAGAGACAGTAATTGATTGGGGGGAAG +GGGAAAATAAATTactaccaaaaaaaagaaattaattgGGGGATGGGAAT +AGATTCTGACGGTAGAGAATAGATCATTTGAAGTGAAAGAACCAgatttg +acagatttttcaaaaaatttgattttttttgagaattaaaaaaaaaggac +agTAGGATGAGAGAGATACTGTGGAGATACAACACAAAATTGAACTGGGA +ATTTTATTTTAGAGAGATAAGGGGATATGGAGAGGGTGTCAGAGAGTGAT +GCCAATATTACAGATTTTAGAGATTTAGGGATTTAGAGTGTCGGCCGTTC +CAAGACCATTCCACATGTGCTCTTCAGAAGTTCATCGCGTAGCCGCCACA +GTGCTTTGGTGACGTCTTCTTCGCGGGCAACTGGACGACCGGCTTcagtt +ctctgaaaattgcaaaatttgagttttgaactcctcgtggacaagtcatt +ggcgcgaaatttgaaatttttcagagtagggtcatgattatactcatttt +gaagctctcagaGAGCTAGCTTTGAATATCTGGTCAGTAAACACACTCTA +CtgacaattttccagattgttGAGTATAAACGCGCTCTATCGACATTAGT +CAAGATTTGAACGAAAGAAACTGTAAAATACACAGAAAACACCCATCGCA +AAATGTTATTGAATGGGAAAAATGCGAAAAGAGGCTCCTTACCCTAATCC +AATTGGTCGGTGGTCTCGCATTGGGATTTGGTTGCGTGTTTCGGATGGTT +GATCCCAGATTCTTCTCCCAGCGAGACACggatttttggcgattttgaGC +CAGCGATTCAATGTATCTGGAAAACAATTGATaattaaggatgagcaatt +gaTTGCCAAAATTCACTCAAAACTTCAGTATTTTTTAGCATGAACAATCC +CGCCTAAAAGgtcaaagtttggaaaaaaaaacttcgttTTCAAATAGAGA +ATAGATATTCTGGAGCGAAATTGGCTTGAAATCCAATtcaccgtaaaact +tgtaatagaagggcccccacattgagctcatttatctcggctcccttaaa +agatatcaaaaatgtgtcaactgataaaatgtttgtgatatcctgataaa +caatttatcagttgacaacttcttgatatctttcaaaggaggcgagataa +attgactcaaagtcaaaagttagagggcccttctattacaggtttttttt +ctggccaAACTCACTTCAAATACGCTTCCGAGTGCAGCACTCTCTTGGTG +TGGATCTGATTGGGCGGCGCGATGAACGCGGGCACCACACATCGGACCAC +GGTCCTTCCGACGTCGGCGATTTGTGGGACttgttgcatttgctgctgct +ggggttgttgtccctgttgcatttgcatcatttgctgttgttggggtggt +agtggtggtggtccttgatggtattgttggtgttgtgggtgctgctgctg +tggatgctgtggatggtggtgttgctggGAGGATGATGGCCCCTGTTGGT +ACATTCCGTTCTGTTGGGCTGAATGGCCAGGctgtaaaaaatcaataaat +cgataTTGGAGGATTCTAGGAATATAAGATCCAAGGCTGTAGGGAACAGG +GGCTTACAGCCTTGATAAGAACCCAATTCCTCCAGTAGGACCTACCTGAT +AatgctgctgctgctgatgGTGTGAATGATCTACATGCTGTTGCTGATGC +AACCCTCCACCATTCATCAACTGTGCCTGAAGATGTTGTCCAACGGTTCC +ATTCTGATGGTGTCCATTGACGTGGTGTACCTGCTGTGGTGGCGCCTGCA +TCGTCTGTGGCGGTCCCGTCGGTGCGTTTCCATACGGTTGATGACTGACA +CGTGTCGGCTCGCCCGGAGCACTCATTACTCGATAGAAGTTCTTGCCGAG +CTGCGTTTGATGAATCGTCTTGGCCGACGTTGGCATGTGCTTCTCCTTGA +TGTGACGATGAAGACGTGGCAGTGAGGGGAACGGTTTTCCGTCTTTTCGA +TTCCGAAGACACGTCATCCACATGCACACGAACTGTTGGTCGCTGTCCAC +TGAAAGAAGAGGAGGGTTTTATTAAAATGTCAAGGAAATGAGCAATTACG +CTCCAATGAtagaaggaggagaaggaagATAAAAATACCACATACATCtc +taattcttcaaaaatttcaaattttgcgcCACAACTTGTCCATGAGGAGT +TCAAAACTCGTTATTGAGAAAATACGAGTCgtgtactcctcgtggacaag +cTAAAGcggtaaatttgaaaacttttgaagattGAAGCTTGCTAAGAGCC +taattttaattgaaaatccaCAAACATCTctaattcttcaaaatgagtat +aacaAGCTTTGgactcctcgtggacaagttgtggcgggaaatttggaatt +tttgaacgaGTTAGGTCACAActatactcatttcgaagctcttcaaattt +ttcaaatttatccgCCATaacttgtccacgaggagttCAAAACTCGTTAT +CGAGAATTGGgctcctcgtggacaagttaaaacgggaaatttgaaatttt +tgaagtccaaagtcatgattatactcattttaaagctctctcttcaaaag +aaaaaaaaactactcaCTGATAATCTGACTAGTATGGTGTTGAATCACAT +GCTCCAGAAGCGCCGTCTCCGTATCGTATTGAGTGTCACAGGCGGCCCAT +TTACACGCATAGATCCGCACTTGTCCGGGAAGTAGTGTCATCGCCTGATG +GGCTTTAATGGCATCCTGTTTCGCCTTCTCGACGGCCACCACTTCCGCAC +GCAGCTCGTAGTGCTTCTTTTGCTCTTCCGACAgttttttccactgaaaa +tatGGCAAATTTAGAAGTTGCTCTCTCTCTACAAGATGCTCCTCCCTAGA +GAGCCTCTTACCTCTACACCAACTATTCTCGACACGTCTCCAAACCCAGA +ATCCGGATTCTCGTGCATAATTCGCTTCCTGACTTCTGCGGAGAATAGAA +TGTAGCCACTTTTGCTTTTGACCGCTTTCGGTGTCACTGGCGTTGATGCG +GTTTGATTGGCGACAGCGGCTACTGTGGCTGCCACGTCTCCTCCAGATTC +CGACGGCGTCTCAGGCGGCGATGAAGATGTCGCCGCTGTGACCGGCGCCG +GGGTCTCCTCCTTGACGTGACGTGGAGGATGGCTACTGCTCGTAACCGAC +TCACTGCCATCCGTGTCGTCGTCCATCATTTCATCGAGATGATCCAGTGG +CATGACGCCGTCGCCGTTCACACGTGGTGAGAGttccttttccattttta +ttggCTGCTTGTAGAACAGAATCTCTTCATTCGGGATCGACACGTGCAGT +TTGTAGGTCTGTAAAAATTCCAAGGATGAGGGGGGAGGGATTCTGGAGAg +ggcggaaattcaaaaatcaaaatttcggaagaCGTGGGAAAGCTCTCCTA +GAAGACGTCAGGCGATTCTAGAAACCAATCAAACCAAAGcttctggattt +tcaggtgagtggtcagatgacgtcatctggattCAATGGATTCAGGAAAG +CGAGAGCCATGAAAAAGAGTGGGaatcggattttcagattttgagtgTCG +CAGGCTACGGTCAGCTGACGTCATCTTAAAGTGCTGGATTCAGGAAAACG +AGAGTCCACGATTCTATAGGGACTTCTAGATCCCATCACTGTTATTCTAT +TCTACTTGTAGCCTACATCGGGGCTCCATGGCAACGCTAATCCCCCGGGG +GCCCCTGATTCTCATAGCCCTCCGCTCCTACTAGAATGCCCTATGCTTCT +AGAATCTAGGATCAAAGGGGGCGCCCTACAGTACCACCACCTCCTCTATG +AAGATAGACGACGGCGGCGGAgacagagaaagagaaactCCGAGTTGGAG +CGGCTAGGGCGGCGGTGGCGGCCAAGCGGAGGTAAGGAGAAGAAGGGAGA +GCTGGCTGCAGGCAGCAGCAGCAAGCCTGCAGCCGGCatcagaagagaag +aagaagattctaCTGCTGCTGCCTGTACGCACACACACAGGCTCTTGTCG +TCCGCGCCAGGCAAGCAGGCAGGCCGATAGACCAAGATGCAAGTGAGGGG +GGTCTCCGCTATAGACTGGGCAGGCACGTCAGCATAGCGcacacagcaca +cacacacacacaccacCATCTGcctccccccccccttcttcttttatttc +aCGACCTCCTTACTGTCTACCCCACTCTCCTCGTTCGGTGTGCTCTCTCC +CTTGGCAAGCCACTCCGCTCTCCTAATCCGTCCCACTCTCTCGTCGAGTG +TCTGTTCGTCTGTTTCCTCTTGGGGCCAATCGATGCGCGCGCAGGGACGC +ACAGAGAGAGGAGGAGGCCAGAGCGAGGGTAGTGGCCATGTAGAAGAAAC +GCAGAGTGACGTTTCTGTCTCTCCTCGGGGGGTTCTCGAATTGAAGGGTG +GCGAGGGGAGGAGGGTAGGCTGACTGTGTGAGAAGACAATTAGTGTTGGC +TGCTGCTGCTGGTGGTCCACTAGGAGGAAACTAGGAGAAGGCagctgctt +cttcttctgcgGCGTCGGCGCCTTCTGAATCTCAGCTCAGctcaaaaaga +aggaaattccTAATCTTCTGCTGATAATGACGCTGACGACTACAAAGACG +TGCTCAATGAGCAATGCTGGGTGGAGCTTAAGAAAACAGGGGATGAAGCC +TAAAAGGGATGAAAACGgatagaaaaaaggagaaaatgagTGATTATATA +TGGCAGAGTTCTCTTTTATCAACGGGACGAAAAACTGACAGCTGAAACCT +CCGAAACCTCCGAGGCGCGAAAAAATAGATCACAAAtcgatctacagtaa +tcctaacgATTTGTTGGAAGGCTTACTGTAGGTCTTGATCTGGAGAGATT +CTAAAGTTTAGTttgggggtactgtagttgagACATAGGGGGTAACctac +aaaaatccaaaatgctCAGgacgggttactgtagttctgtATCTGTTTAT +CTAGATGTATTGATCTACAAATTTGATCAATGAACGAACCTTCAtctttt +tcatggattttgcaTAGTCCAGATTCATCGTTTGCACATCTGTAAACTCG +TTCGCCGGATCCGGATAGCCAATCAATGAGCAATCGTCTGGAGCATCGGC +AGTGCCGTTGACCATTGTTTCGCACACGAAGACGTCGCATTCCGATATTT +CGGTTTgacgttctgaaaaaaaagagattttcgaGGAACCAAGACTTACT +GTAGGTTTTTATCTGGAGAGATTTTAAAGTTTAGTTTGGGGGTATTGTAG +CTGGAGACCTAGCTGATAAGGAGGGAGTAATcgacagtaatccaaaatgc +TCAGGAGGgttattactgtagtttatgTATCTGTTTATTCAGATGTCCGA +AACACAGATAAAAGTATTCTGatgaaagattactgtagctacagCCTGTG +AACGCAGTTACCGTAATCTACAGTAGCCTCGTCGACTGACCTCATTTTGG +GCGTTTttgaggtacagtaatccggaaaaTGTGTGTACAGTGAACcccaa +aaaatcaggattttgTTCTCCGCCCCTTTTAGATAAAGATTACGGAAGCA +CTCGCCAAAAATCCGGATTTCAAGCCCCTCCCTTTTTTCGGTAACATGGG +GGCGTTTCCAGGAGTACTGTAACTGCTACAATTCtaaagattacggtagc +tacaGTACCTGGAAAACCCGGAATTTCAAGCCCCTCAAAAACTCACGCTT +CGTATAGAATTTCGCTGGCATCACATCGCAAGTTCTCTGTATTTCGCACA +ACTTGCGCGTCTCATCATTCCGATAAACACCAATGACCTCATTCTTAAAG +AACAGTCGACCCTCGTCATGAATCGTTTCCGATGGACGAGCAAACCAGGA +TCCGGAGAACCATTCCGATCCATCCGCCTCCTCTCTCCACAGCTTATTGA +TCCGCATCACATCACAGAGGGGTTTCAGGGGATTGAACACCAGAACGCAT +TGTCCGAGCCAAAAGAATTTGCCAGTTGAAGAGCGGATCTGTTGGTAGTA +GATCTTTCCGTCCTCCTTGGTCGACTCATACTCGAATCGTTCCATATCCA +AAGAGACTTCTAGACGTGCACGATCCTCCTCATCATCGTCTTCCACCTCC +ACAATTTCCTtctcttctgaagacgtcggaTCTACCACCGTTAGAATCCT +CTTTGGCGTCACTGGCTTTCCCCGTTTCGTGCATTCCAACTCCTCATCTT +CTGAACTGTACGGCCAGGAtctcaatttcgaaaagtaCTTTGGTTTTCCA +TGATACTTGTATTCACACAGATAGACGTCTTCTTCAGCGAAATCGCTCAG +AATCTTGGTAGAATACGTGGCAAGCGACACAACTGCACACTGTCCACGCA +GTCGTTCGGCAAGAATCGTGCCTCGGAACGGAGTCAGAAAGACTTCTTGT +TTGATGAATTTGCGGCTAGCCAGATGAAGTGTCTCGTCGGGACGATAGAC +CCAGAATCCTTGAAGAGCCTTCTCGCCAGTTTCGTCTTTGAACGTACGCT +CGATTCGAAATATGTGGAGGGGAGTCTTCTTCTCGTCGGCACGCGCGATG +TAGGCGTAGCACGGCGCCACATACTTGACACCGTCGACTTCGACGTCCTC +CATTTCCACTTCGGATTCCTGGAAAATGAGAGGGGGATAGATCATAAAAA +GGCGTAGGTGTGTCCAGAAGGCACTGGGGAAATTCGAAAGGATCCCAAAG +AACTCTAGAGGCTGCTAGAAGCTTTCGAAGGATCTTAGAGATCTTTAGAA +GCTTCCGAACGACTTTGGATCCTTCAAAAGGATCCGAGAAGCGTCATAAG +gattctagaatcttccaAAGGTTCCCAAAGGCCTCTAGAGGATCCTAAAG +AGCTCTTGACGGTTCTAGAAGCTTCCGAAGAATCctagaagtttgaaaac +gAGCCTAGAGAGTTCTAGACAGTTCTAGAAGCTACCGAAGGATCCTATAT +AGCTTTAGAGGATTCGAGAGGATCCAAGATGCTTCAGAACGATCCTAGAG +AGCTCTAGACAGTTCTAGAAGCTTCAGAAGGATTCTAGACTGATCTAGCA +GATTCTAGAAGCTAAGCTTTGGAGGGTTCTAGAAGCTTCTTTAAcacaaa +aactcacatttctcTTAATAACCGCTGCTCTTCCACCCATCGCCGGGCTC +TCTCCACCCTCATCCGACTCCTCCTTCGCCTTGACCACCTTCTCCTTCTC +AATCGCCTCATCGATATCCTTCTCGATGACACTGTACGCTTGCGAGTGAA +TCACGTCCTTGCACTTCGCATCTCTCTCCACAATGAACGCCTTCTGCAAC +TGACACGCCGCTTCATAGATCGGTGATTCGAGCTTCGCCAACTCTCTTGC +TGTTTCGAACAAGTCGAAAAAGTCCTTTTGAAGTTTGTCGAGCCGTCTAT +ACCGACACTTATCGATATTGATTTTAATTTGGTCCATTGTGAATGGCCAT +TCAGCTGGTGGCACTCCAGCAGCCTTCATTAGGTCCGGAAACTCAATGAA +ATAGTCTGCCCAACACTTTCCATCCTCATCCTTCTTACTGTATAAGGCGG +CGAAGATCGACGTGAAGATCGTACGAAGCTCGACTTGTACCCTCGGAGCA +TCCTCTCCTGAATCCAACGATCGTTTCATTTCGAGAAGATACTTCTGAAG +AGACACAGCCTCCTTATAGATCACAGAATCAGTTTCATTGAATTTGCAGG +CGTTGGAGAGCATGAGCATGAAGTCAGAGACCACATCAAGAAGGGTTACA +TACTGACGATTCTCCAATTTCGCCTTGATTCTCATAAAATCCATTGGCTT +CTTAATCACATCATAGTAGCCTGGAAACTCTTCTTTCGTCGGAAGACGCA +TGAAATTCACAGCCAGCGGGCGACTCGAATCGTCGTCGCTTCTCACGTCT +TTCATCGATTTGAACAGTCTCCACATCTTCTGTTCTTCGATGGGCAGCTG +GAGAATGGCTTGGTCAATTTGACGTTGCTGTTGAATCATTTGTTGACGAG +CCAGCTCTTCTTCATTGTCCTCTGGACCTCCATACTTCTTCTTGCGGCCA +CGTGTCTccgtttttggggttttccTGTaaaaacccatagaaaaaagtcc +aaaaatcatgaaaaatggtcctaaatccagaaaaaaaaaagtagtttaaa +atccattgaaaatggtcaaaaatctaTGAAGAATGGTCTaaaatccatgg +aaactagtccaaaaatcatggaaaaaaggtcctaaattaatgaaaaatag +tcctaaatccattaaaaatggtctaaaatccatagaaaaagtccaaaaat +cataaaaaatagctcaacatccattgaaaagagactaaaaatcatggaaa +atggtctaaaatccATGTAAATGGGTCGAAATCCATggaaaatgatctaa +aatccatgaaaaaaaaacagaaatttcggaccatttttcatgaattttgg +ataatttttaatggattttggATCAAATCTATGGAAAGTGTGTCCCAAAA +CTTACATATTCGTCAAATTCGACGCTGATCCATGTGGTGTTGCTGGCATG +GACCCGGCTGGCATCATATACGGCTGTGGTGTTGGTTGCCCATATGGTAC +CATCATGGGGTACTGCTGTTGACTCCTCATTCCATCATACGCACGAAGAA +CCGCCTTTTCCAATTGGATCGCATCCTGATGAATATTGGAACCGGGCTCG +TTGAACTGTCTCGCGTTGGCGAACATCCTACGACAATCCGCAATCATCGC +GTCGACCATCGGATACTGATGAGTGTCGATTCGGGTTCGGATGATTTTCA +TATCAATTGGGTTCTGGATGACTTGATAGTACTCGGGATACACCTTTGCC +GATGGGAGGTCTATGAAAGCGCCCGCAGGCCAGTAGGCCAAGTTTCCAGG +AGCTCTAAACTGATGAATAGCATTCCACAACCCCTGCATCATCGTCTTTT +GTTTCATATAGGCAGCGTTCGGGTCCACAGGAGGCAtggatttccgattt +ttatagCCCGGCGTTGGCGTCCCGTTCTCCCTGTTCTTCTTCGGCGATTT +TCGTTTATGCGACGGTGGAGGCGTCCGACTATCATCTGAATCCGTATCCG +GCATCTCGATCTTCACCTGTCGGGCGCGCGCTTTCGGCTTCTGTGGGGTG +TTGGGTGCCGATGGTGCCGCTGGCTCAAACTGGCTCACATCGAACGAGGG +AACCAATTCCTTACACGTGGCAATCGTGAGATTCCGAAGCTTCTGAGCCG +CAATACAGACATCCGACGACTCGAGATTGTATTCGAACGCGTTGGAGTAC +ATTGTCATGAGGTCGGCGATGAGGGTCTTGAAGTCGTACTGCCCGTTTTT +GAGACGTTTGTTGATCATGAACAAGGAGACAGGTTGCTTTATCTCGTCGT +AGTAGTCAGGATAGGATCTGGAAGATTTTTTGGTACAGTCTGAGTCATTT +TGAGGGTATTTAGGACCCCTTGAAGGCCCTTAAAATAACTTTGAAAGCCT +TAAGCATTCCTTTGAGGTCGCTAaaactgatattttgataaaaattgata +ttttgataattcgGTATCACTGTGGTCTAATTTACggccgaaattttgag +agtgggcggagcttaggtacGGTAGCGAaggatactgtatttttttattt +ttcataattttgaaggtatttttctttgattttacgatgaatttggaacc +tatttttcacaaaaattcttACAGTAACACCAAAATGACgagctggaatt +ttgaaaaaaaaaataccctaaaacctgtaatagaagggcgcccACATTgt +catcaaatttgagcttatttatctcggctgctttaaaagatatcaaaaat +gtgttaactgataaaatgtttgtgatattatgataaacaatttataagtt +gacaacttcttgatatctttgacaagaaccgagataaattgggtcaaagt +tagaggcctattctatcacaggttttaagACTTTTTAgtgatataccgta +aaacctgtaatagaagggcctcctaacttttaacttcaagccaatatatc +tcggtttctgtcaaagatattgaaaagttgtcaactgataaattgtttat +caagatatcagaaatattttatcatttgcttcttttttgatatctctctt +aagaatcgagataattgagctcaaacttgacagtggggggcccttctaat +acaggttataAGCCCCAAAACTCACTCCTTGCAAGGCAACTCCAAAAAGT +TATCCGCCAACGGCACATCCTTATCCATCGCATTCCGAATCGTCCAGTAC +ACCTTCCACATCGGTTCGTCAGCCTGTTCCGTCTCCTCATCCTCTTCAGA +ATCCTCCGAAAAGTTGTCATTCACTGCGACAGCGGTCAACAGAGCGTCAA +CAAGTCGTGTAGCTTTGGCCCTTCTCATCGGATGAACTCCCTTCTCTTCC +AGCTTCGCAATTTTCTCTCTTACTACTGTCTTCAATTGCTCAGCATCTTT +CCAGATGTCACTCCCCTTTCCACTGAATTGTTGAGCGTTTTTGAACAGGA +GAAACAGGTCGTCCTTCAAGTGCTGCATTGTTTCATACTTCTTGTCCTTG +CCGTTCTGAGCAATCGTTTTCAAATCGATCGGCTTTGCAATCTTCTGGTA +GTACTCTGGAAAATCCTCTTTCGAGTGTAAAACTCTAAACGGAGGACAGA +GAAGTCGTCCAGAGCTGTCTGCGGCTTCTAGAACAGCTCCTAAAAGATCA +CACAACATGTAGTCGTCCAGCTCCATTGGTGAAACATCACGTGACGATGA +TTCTCCCGTCTCCTGCGCGACTTCTGAATCCTCTTCTTCTGGCGTTACGT +CTTCCacatcgtcgtcgtcgtcgtcttcctcttcctcctgCTTATCATTC +AAATACTCTCCGGAATCGACCTTCTCACATGCCGCATTGTAGAGCTCCTG +GATCTTCAGCATATCCTTGTGCTCATCagaatctttctgaaaatcgattt +ttggctattttcagctatttttcgcCATGAATTTACCTGATAATAGGCCA +GATTATTACTTATAAACGTGTCGAAATCGCTTTGAAACTGATCAAAAGTC +TGATAATCCGGATTTTTCAGCTTGTGTTGAATCGTAGTAATATCGATTGG +CTCCTTGACAGCTTCATAATATTCTGGCTCTAGACGACGTGACGGTACTC +GGAGAAAACTCTCGAATATCGGCTCATTGGTGCCGGCTGTGCTGCGTTGA +CGGCGGATTTTCGACATGACATAGTTGCCTTTGACGAGctgaaattgaca +aaaatgagtcaactttattaattttggggcgttttttcccgaattttcga +ccaaaaagGCCAGAAATaacctaatttttaaattttccccACCTCAATTT +CCCGTTTCTCCCTCTCGGCAAGAGTCATTCGTCCTTTCCGCTTCTTGGCG +GCGTTCGATGTGGACGGCGTATCCTTCTCGGGCTCCAGCTCGGCTGCACG +CTTTCGTTTTgtggtcatttttaggcctgAAATGAGGGGAAATTTAGAAG +tattcagatttgaaaaaaaaaaagaggtttcaagaagaaaaatggaccga +aattcatgaaaattggttcaaaattcatgaaaaatggtcagaaaataatg +taaataggccaaaaaacatggaaaaccggaccaaaagccatgaaaaatag +cccaaaactcatgaaaaatCGTCCAAGttacatgaaaaatggccaaaatc +tatgaaaaacgatccaaaacaactaaaaaactattttttttaaagcaaat +aATAggcaaaaattggattttcttcCCAAAACGGGTTgattttcaagcca +tttcgtttttcaaacatgctgaaaaaaaagaggtctCGAGAAGAAAAGTG +GACCGAAATCCAtggaaaatggctcaaaattcatgaaaaatagtcagaaa +tccatgaaaaatagcccaaaattcatgaaaaacggtccaaattatatgaa +aaaataggcaaaaattggattttctggCCAAAAACGGGTTGATTTCAAAG +CCATTTCGTGATAAaagatgttgaaaaaaaaacaatatttttggcctttt +tggaccaaaaacccgtttacagagataaaaaatacaaaaaattcaatttt +tcgggcaaaaaaacctaatttaaaggcatttttcaaaatagaaaccCCGT +CAAAATAGGTATTTAGctggatttcgaaaatctatgAGAACCCCATTCAA +AATTACTAAAATCAAActtattttctatgaaaatccCCTAAAATGGCact +tttcactaattttctccccttttctccttctcaaagtcctcaattttcag +cattttcagctctaaaacGTTCAATTATATatttctttgtatttttcctt +gaaaaatccCTCAATTTCCCATATCCTAATCTCATTTCCCCATCGGTAAT +CCGTGTCTAAACCCTGAAAGGCGGCGCGACGTGCGCgcgaaactcgaaaa +tttgaaaattggagaggAGGAGGCGGGGCGATTATTGGAGTTGGAGACga +ccgaaatttcgaataaaatgAGTTCCAGGGGGTAAAAAATGATGGGGAAT +CgatggaaaatgagaaaatcgagataatgtGTGATTTTAAACcgaaaaat +cggggaaaattgatgaaaaattgacgaaaatcgGTTCCAGACGAATTTTC +GCGCATTTTTACGATAGAACAGAACCCAAAAATAGGGGCGGGAAAAGCAA +CGAGTGTGTCCCAACACCCAGGCCTTCAAAATAACGAAAACGCGCGCGAA +TTGTGTTGGCTGGGTGCCAGACACAGCGTTGTATTGGTTGGTTGAGGAAA +GAGAGAGAAGGGCGAGTGAGCGCGCAGAATCAAGAGACGCAGCGCGAAAC +GGGTCGGGGCCGATGAGTGAGAGAGAGCGTGCACGCGAAACGCGCCAACG +CCGCCGCCGCTGTTTTCGACTCGTTGTGTACGCTTCTTCGCTGGTGTATT +GATCCAAACAGGTTAGTACTAGGTTAACaacatgctgaaaattgttttta +agcCATGAAAGCGATTAAAATGCTGCTGAAAACACCAAGAAAGTGTTATA +ATTGGCAAGAGTGTTCAGCGAAGggaaattcaatatttttctttgaaaat +atgatttaaaaagtttaaaaccAAGAGAAACTAATAGAAAATGTGTGTAA +ATCATGTTTTTAGCCCGAGAATCCAAAAAGAATCAGCGGAAAAcgctcga +aattcaaattttaaatttgcgCGCCAAGCGAAAAGCTTGACGCGTTTCTC +GTGCAGTGACTCTCTGCGCGACCTCGAATGGGTAGAGAAATGGAGTGACT +CTCCGACTGTGCGCGCGCAACAAGGGCAGATGGTACACAGAGAAAATACA +ACACAATGACACATTATATAAAAGAGAGGAGAaatctttcagaaaaccaT +GAGTGAACGAGGAGGACAGCCACCACCGAAAAAGCGGCGGACCAACGACC +GGCAACGATCGAAAAGTGCAGATTTGACGGTGAATAATGAGTATGAGGAT +GTGGTTGAAAAACTAAAGGGAGCAGACAAAGAATCGGCTGTGAAAGCGGC +GAAAAAGCTGAAGCAGATGCTGTCGAACGCCGCGAAAATTGGCGGATTCG +TGGCGCATCGCCGGAATCTGGAAGATTTAGTTTCAGTGTTTTCAAGGGAA +ACTGCATTGCTAGGCACTTCGATGTGAGTGTgggaagctgaaaataatga +aaattcgCTAAACCTCAGGTACCACGAAGACATCCATAGGTACTCTATAA +GTATCCTTGCAAATTGCTGTTATATCGATAAAACGACGTCAACTGGGAAT +CAAATTCGACGAGCTTCTCGAAGTTTCCTAGATCAAGCCGGTTAGCGAgc +acttttttatttgaattctaGTGTTTCCCGCCCGAAAACGATTGTGCATT +GGAGCGTAATTGCTCTCGAGGTTTCGAGGAGcggagaccccgcaaattcc +ggggaaataaaaaactggagaaaagaatcggggaaaagtgggcggagcct +aaaatgcaaccctggcacggttttgccgcagggtttttctgcacatttgg +acACGGTCATGAGGGTGCGGAGCATNAATTGCatccctggcacggttttt +NTNgactccgcccactttccccggagtttttttttcccggaatttgcggg +gtctcgaGGAGCGagagcttttttaatgtttgaagtcgttaagagctttt +ttaatgtttgaagtcgttaaaaattgaaaaaaataaagtcgggaaataaa +aaaataataatgtttgcgcgtgggattgcgtgctttaggtgcgaacaccg +cgacggcaaaatacacattccctgaaattttccccgaaatttttgctgtc +tgcccTCCTTTAAAGCAtaatgttcctttaaaagtgCTACCGTAGTCTTT +agtgggaaatttgaatttttcagtccGAATCTTCGAAAGCACCACTTCCA +ACATAGAAATCCGCGTCGCGACGTGTCGATTAATCGGAAATTTGTGCAAT +AATAAGGAGCTGGGACCAGTTGGCTGTCAAGCAACACGATCCTTTTCGAT +CGAATCGCTCTTCTTCTAGAGTCCACAAACGAGAATATCGTCACACAATG +TCTTCGGATTTTTCTCTTCCTATCGAATCATTCGTTTACCCGGGTGAGTA +ATTCTagttggagcgcatttgcattatTTACATTTTTATTATTAAACCCA +AAAAGGAATGTTTAAATACAGAGAGATTACTATAGTGGTACACAGTTTAC +AATAAGCTAGacagaggaaaaaagaaattatatgttattcttgtgcaaaa +aaattggagttgtggccgaattttcacacaagttaggccaccgtatttca +aggagaattttcgaaaacacagcctccaatttttttgcacaagaataata +caggaaaaattttcaaaaaaaaaactttttttgggggTCCGACATCATGg +gagttttagaattctgagatttcagCAGTTTAATGGGTCTGTTTGCGTGG +CTGGAATTGGTGCTGTCCGGACTTGTCGGCACGGAACACGTGGTTTGGTT +GAATGGCCGGTTGCGGCTTCTCGTGGACCCGTTTGACAGCTTCAGTTGGA +TACGAGCGGTCCATCTGGAAAAAGTTAGATTTGCAGGTTTTGGTAGTTTA +GTTGCTAGAGGACTCTGATCGTATGGTTGTGGTTGTCTAGGCTTACTAAG +GAGTCTATTACTGTTTCTCTTGCGTCCAAGCTAGCGGCTAGAAATGCTCT +GGAAGGTTTAAAGAAGCCCCAGCAGCACTGAGAGCAGGTTCCAGCCCCTG +CTGGGACGTTTCAAAAAGCTCTGAAAGGATCCTAGGCTTATCCTAGAAAA +CCTCAGAAGGACAGATGTCTTCTGGAGGCTTGGTAAGTTTCTAGACGATT +CTAAAagcttctagaaggttctggcCACTTCTAGAAGGATCCTGATGGAC +CTAGAACCCCCAGATCGGTTCCTAGAGTTCTCGAAAGATCGGGAACCTTT +TGAAATCCTTCTAGAAAGTTCGAGACGGTCTAGACGACCTTAGAAAGAca +gacgtcttctggaggcTTGGAGGCGTATCTGACGGCTTCTAAATAATTCT +GACCGCTGCTTGACGGTCCTAGAAGTCTCAGGACCCTCTGGATGCTCCTG +GAGCTTTCAGATCGATTCCTATAGTCCTTAAAAAAGTCAGAAACCTTCTA +GAAGGTCCCTAGCATACCCCAGGCTATGGCCCATATCTCACCTTTGCCGG +CAAATCCGCATCCAAAATCTCGCGAACGACTTCATCGGAGCTCTCCGAGT +TGGTGTGCTCATTGGAATCCGACGCCTTGCGATCCTTGCGATTCACGATG +CGACGTCCGCCAACCATGGTTGCTGGTGGATGTCCGACCTTCATCTCAGT +CGTCTCGTTGTTATCCATAGCGCTcattttcggaactttctgGAAGATTC +GATAAAGGCTTTGAATATAAAATCAAtgtggagaaaaaaaacgagtggTG +ACCTCCTCCGTCCGAGGTAGGCAGAGCGAATGGCGCTGAGGGAAGGGACA +TAAACAGAGTGCGCACTGAGAGACGCAGGGTAGGGGCGGAGTCACATGAC +AGCGGGGCGCGGGGCTCCCTCTTTCTCACAAAAACACATTCAacgaaaac +aagaagaagagagtaTGTGTGGTTAGAAGACGACGCCGATCTATGGCTCA +CTTAAATAGAAAGGaagaaaagggggcggagcttgtttACTGCCGCCGAA +CATACACAACACATTGCGAAATAGAATGTGGAAGTGatcggtttttgaaa +ttgagagaatGGTCAGTTGAATGAGATTGGCAGAAGATGGATTGGTTATA +TGTTGATAATGATAGCGGAGGAGTGTGACTAACGTGATAGTTGCTagcca +tcaactttgagtcaattcatttcggttactgtaacagatatcgaaaagct +gtcaactgataaaacattgatcGTATTACGAtgaacatttcatcagttta +tcactttttaatatctttcacaggaaccgagataaattggctcaaaaata +gctcTTCTGGTTTGGACCCCTCATTCGACTTCACATTCAAGATAATGTga +ttttgagagacgcagagaaaaatgtaaaaaaaaacagtacaAAAATAGCT +GTTTTCGTTTCCAAAATACATTAAATTACACATGTTTATGATGGGCATGA +CATGtatttttgtatgtttttgaATGGGAAAGGTGTCActttaaaggggg +gggggggggaagtgACGCTGTAACACTTGTATAAAACCACCGAGATGGAC +AGAGTACTCGGTGTGTCAGGAGTACTGTAGTCTGCTTGTCAATGTACAGT +ACTTCTCGGCAAgctttcggttttttttaaatagattTTTGGCTAGCTAG +TACTGTATATGTCGAATTAAACAGCAAAAGGTAGACTAGAAGTACTGTAC +TTTGGGAAACAGGAAtcgattttcgatattttgatCTTTCTAAAATTATC +ACCAGCTGACACCTTTGGTCTTTGAAAGTCCTTCTTTCTTATCATTTGTT +CTCTGATCAGACCCTTCAAGTTGTTTCAGCCAACCAACCTAATCTAGGCC +AACAAACGAATCCACGTCATGCAGCTTACTTGCCGAAATGGTCGTTTCGT +GTGATCGCTTGTTTACAGGCACGACGATCACtaatttctctgcgtctcta +gaatgttctagatagggagagagagagagaagagggggaaaagaaaaatg +gcaaatatttaaaaaaaactgatttccaGGTGATTATGCTGTCGAATGGA +GGCTTCTATTTGGGCAAATTGCTGGCTAAAGAGGAGCGAAGAGATGAGGT +tgtgaagattttgaaagtgCTCCTCAAACGGCATCGAAATGATTTGGATA +GACAGTTGGCGAGAAGGTAGGAGAAGAGgagacatctggacacacagaca +gacagagATGTCATTTTATTCAGTGAATGCAGCGAACACGTGATTGGAAT +GTTCTTTGAAAGGAACAACAACAAGCATCGACGGCAATTTGTGATTGATT +TGTTCTCGAATTCGATTTGTATTAGAGAACAGTTAGTTGGGCGAAAAAAT +ACGGGGGGAGAAAATGGGGTTTTGGGACACtgttttgaatgattttggat +catttttcatggattctggacgttttttttttgacaatttggggattttt +ggagcatttcctttggattttgagccatttttgacgATCTTGGGACAAGT +TTTCAAGGATTTCGGCCATTATTCAggtttttttggcattttccaATGGA +TCTTCAGCCAattttacatgatttttagatcagttttgatcgattttaga +catttgttgatgattttgagacaatttttcaaggtttttttggacaattt +ttaaaggatcattttccattaattttgactcatttttcatgatttctagg +acaaattttcGTTATTTTGGGGCCATctttcctaaatttttagagcacga +tttcaaagattttagatcatttttcataattgttggaccatttttcacaa +aattagGGATACtttttcatcgatttgaacaatttttcatgaattttaat +gatttttccaaggatttttggccatttttcataaattttgaatcattttt +tcctaaatttttggcatttttcatgaatttgagatctttttttcatgaat +ttcgacccatttttcGAGGACGTTTGGTCAatttcactgattttaggact +atattttttcatggatttcgactaaattttcattattttttagacaatta +tccatggttttttggaaaagtccatgaaaatttaaaaatcgaaaatctat +AATTATAAACGAACATGtaaaataagtttttgaagttttatagttgtagg +attttcgatttttgaattttcattaaattttttgcctatttttttgcttc +ttttggttcattttcgcaaaatttaaTCTTAAATCCATGAAAATTACCCA +AAATCCTTAACAATTGGTTCGaaaatctatggaaaatgcttcaaaaacta +tgaaaaatgagccaaaaaaaattggaacatggttcaaaaaatcatgaaaa +atgggtcgaaatttatgaaaaattgttttaaaattttttaaaatcctaaG +TTCATGAGaattggggaaaaaaatccataaaaattttacctgaaaatcat +tctgaaaaaagactaaacaccgcaaatggaaaaccaaaaattaaggaaaa +tggtccaaaaccccatgaaaatcgaaaaatctgcTTACTAGATTTGGGTT +TCTAagtcacaaaaatttgaatttcaaaaagcgatagtttcaagttcaaa +ttcgtttcaaaaaatcaaaagatcagGGAAACCCCCCTCGTGTTTGTTGT +TTACGAGTCTCTTTTAAAAGTCCACACATCCCCAGACAGACGACAataaa +agagacgcagagaaacagacAACCCCAaccccattttttcagaatggagA +CCGAAAAGTTCATCGAACAACTCCTCGCTTGCTGGCTTCCCGAAAAACAT +CGAATGCTCGCCGCGTTCTCCCAAACAGCGTGGGGTCGTGCGAAACTTCG +AACTTCTGGAGCACTCGACGTCGTCGTTGACGTCTTCTCAACTTCAGATT +CTATTGAGGAGAAGCTCGCCTCCGTTGCAGCATTCCGACATTTCGTGCAC +GATAACGCCGGAATGCGTCATTTGagtgaaaacttgatttttgtgaaTAC +CGTAGTCCGAGATGTGAACGCCTACGTGGAAAAGAATCGTGCCGAGTGTG +AGCCTACGGTAGACCTGGAGTTTGTGCAGAATTTGAGAGCAAGGggcgag +ggggcggagccaggAGGGGAGAGACAGGATGTGGAAGAGCCGGCGGAGAA +TAGGTTGCATAAGGATTTCTATTCCATGTGGAGCTACGATACACCGCAGA +CGTCACCGAAGACGTCGCCAAGGAGAGCTTCAATGTCGCCACAGTACAAT +GCACCGTCGCCCAGCTATAGTTTCCCATCGTCGGTGGCGTCGAGTGCACC +GTCTTCGCCGGAACCCAGGAGGTATACTACGAcacctaccgtaacctaga +aatatttttattttccatatttgtactttttttgaaatccaaagaGCGAT +AAAAATTGACGGAAATCCGGATCTTTTAAATATTCTGTGTACTTTGTCGT +CAATTTCGACGCATTAATGCTAGACACCATATTTTACACCATATTtcacg +acaaaatgcacagaagTTCAGACAAtgccaactaccgtaacctataTTTG +TATTTCCcgttttttggacatttttacTCGAAAAAAGCGATAAAAATAGA +cggaaatccaaatttcgcgctcaaaaaaaccgaaatttcgaatttctgtg +cattttgtcgtaagcTTTGATGCTGCATCGAATCTGACGACAAAGTACac +agatatttgaattttcgctgGTTTTTCATCGAGTTTCCTGTTAAAAGCCa +gctaaaattcaaatttctgtgtACATTGTCGTCAAGTTCGATGCATCAAT +GCTAAACACCGAAtcttacgacaaaatgcacagaaattaaaaatttcggt +aTCCAGAGCGCGAAATTTGGATTCGTGCCTATTTTTATCGCtttttttCC +gattaaaaatgttcaaaaaacaacgGGAAATACAAAtataggttacggta +gttgtcgttgtgagaccaaagctccgattttttgtactagTATAATAGAA +ACTGGTGTTTCAGACCCAGTGAAGGAATTCTAGACGATTTGAGTGATGTC +GAATCGGATAATggagaggagaaggtggagaaggagaacgaagacgagaa +gaagagacggcaggagaagagTGTAAGTGTGGACTACGGTAGATGGCTta +gatttggatttctaggccaccatttgctcgaaaaatgcattgTCGCATTG +CTGACattgttttctcgtttttcgtcattttgagcatgatggcctaactt +ttgcggGTTGGGTTCCCTAGGCCACCACcccaaaatggcctagaattgaT +CTCATATTCACCTAGTgtgtttcctaggccattttgaggtggtggcctag +aaaactcagtttgcaaaagttcggccatcaCGTCAAAATAGTCTAGGAAT +CACACTAAATGAATatcaggccatttctaggtcattttgaagtggtggcc +tagaaaacccaacttgcaaaaaaattaggccTCCATGGTGGACTTCTCGG +TCATCATGAAACTTGCAGATAATCGAAGGAGAGCTCTGGCTGCTGACATG +GCTCTCCCAAGACGATCACAACCTGCAGTTCCTAATCCGGGCCGACGTGG +TCGATGCCGTCGTCGCCTACCTCCGATCCACTGCCGACCCGGACTTCCGA +ACGTATCGAGTGCTCCGTCGAATGGCCACCCATCGAACCTACGTCTATCG +TCTTCTGGACCTTCAATTTCACGTGCGCATACTCACTGGTCTATGCGCCT +CGCCGTGTCGAATGCTGAAATACGCGAAGACATGCAAACACTGTGATAAG +TGTTCAGAGCATGGACGGGAGATTCTGAGAGAGTTCGCGAGCCACGTGGA +TAACAGCTATGGAGACGCCCACATCAATTCGCAGTTCACAAAAGAAGACT +TTGTGCATCGGACGAAGGCGGCGATTGCGAAAATCGTGTTGGTCAAGGAG +AGGATTCGTCTGGCGAAGGTGCCGGCGCTCACCGAGTTGTTCACCGCGTT +GGCACATGTGATGAGCTCGGTGGAGAACTTCGAAGAGTTGGGAAAGATGA +CGACGTATGAGAATGGACCATCGTTTGTGTCGCAGATCATTGGAGCGCTG +TCGATTTTGATTAGTGGACAGAAGATCAAGTgtgttttggagaaaaaatg +aatttctgtgcattttgtcgtaagaTTCGATGCCTAGCGTTAATGCATCG +AAATCGGCGACAAAGTACACagatattcgaattttcgctgGTTTTTCATC +GAGTTTTCTGTTAAAAGCCagctaaaattcaaatatctctGTACTTTGTC +GTCAAGTTCGATCCATCAACGCTAAACACCGAAtcttacgacaaaatgca +cagaagTTCAGACAACGACAACTannnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnGGCCTAGAATCCAAATCTcgtgta +gtcctctcggagaagagcatttttgaacaatttaaACGAGTTAACAGTGA +TAAAGAACAGATAAACAGGTGGttgtaccgtaaaacctgtattagaaggg +ccctctgtgatAAATGACCTATattaaatcgaatttttttttacagagAA +CTCTGCGAAAACGACATGTGGTATTTTCCGGTGATCGACGGCGGTGAATG +TGtacttcagaagacgtcttcagTTGAAGAACTTTTACACTTTACGGATG +AGAAAAACGAGGAAATCGCGGCGGCACCAATGCGAGCCATTTGTGAGAAC +AGTCACTATTTCGAAGGAATGTTCTCGTCGAATTTCGTCGAAAAAACCGA +GgcgattcgaaaattccagttgCAATCCGAGTCGTGTCCGCCTGGGGATT +TCCGACTGTTCATCCATCTCCTTGCCACGTGTACCGGGCCATGTACGGTA +GTCGCGTCGGCCGAGCAATGTGCGAATCTGTTGGCGGTGAGGGTTTCGCC +TTGGAAAATATggatgatttttgaattttttgcagcTCTCGGATCAATTC +CTCTGTCCTACGATATCCAAACAGCTGTGCGCGGATGATGGACCACTACG +AACGTTGctaaatgggcggagcttgcaCACATTGCTACCAGTGGCTCTAG +CAACGAATGCACATCCAACGTAAgttttaggtcatggccttGATAGGTCA +tcatattgaattttcagccttctCGACACCGTGTACCTGACTCTGGTCCG +CTTCTCAtcatcagaagacgtgacgaaGGCGCTAGAAGCGATTTGTCAGA +ATTCTACAGTAGTCAACACGTTTGTCACAAGTTTAAGAACGTTTTTGACC +ACTAAATGTTGATTGTTTGTTGTGCTAATATCGGGTATCTCGATTTTCTT +GTCCTAAAaataaacgatttttttcaaaagaattttagATCAATGTCTTC +TTTTAGGcgaattttatcgatttttgaatagttCTCATCGTATAACCAAT +GAAATTCATCGCAAAGTCGACATAAATAGACCAGTTCtgatttagaaatt +ctgaaaaaatgccttcaaattttttattcggGTCACGACTGGAGgagaat +taattttttgtttttttcaatttttatattttttgcaatatAAAACTTGT +TTATCTGTTCTTTATCACTGTTTATCACTgttaaaattgttcaaaaatgc +tcttctccgagaggactacacgAGGTTTnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn +nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnaGTCCTCTCGGATAAGGCt +agagaaaatatgaaaatcaagcgggattttcatattttttgcgaatttcg +ctgattttcaggcttttcagaatgtcaaaaCTCCTTGAAGAACGTCTAGA +TGCTTGTGCGAGGTTCGACGACGATAAATATTACACAGAGTTTCAAAGCT +ATTACAATATTGTCGCTCAAGTGCACTCAGTGGTTTTGACCGCCAGAAAT +GTCTATAAaataaacgatttttttcttcaacagattttttttggcaacaa +aaaatcaaatttcaaatcaattgTCCTTCTTTCCCTCATCTGATCCTTTT +TTCGGCGATTCTGGTGCCACGTCATCACCATCACTGTCATCCATTTCGGC +GTGCCGTTTGTCCATATGATCGATTTCGTTTTTAACGTCTTCAAcgtcct +gaaaattttcgaaaaattgaaaattgcaagtttgtagtttgtagtctctc +cTCACCTTAATTCGATGATCATTCGCAATATCATCATCGTCGGGTGCAAT +TTTGATCGCTCCTCCAGATACACAGAACGCTCCCATGACCAATAGAAGAC +cggtcatctgaaaataaagctagtaaaaagaataaaaaaggaaaatcgcC +TAGATGGAATCGAACTTGCGAGCTGTCGGATTTAAAACAGACGCGTTGCC +GGCTGCGCCACGGCGGAACACAGAATTGATGGGCGTCTAGGGCGTATAGA +AAAGGTGAAGGTGGTGGCGCCGAGAGCCAaagttgggggggggggggggg +aggtgGATGAGTCGAAACGAAATGAAGAATTTCGATTTCGATagttttag +gaaattttagaTCCTTTTGGGCCTCTCAAAAGTCACAGTTTCAGactcaa +tggagcgcacttacCTCCACCACCGAAATCACTGAAAGCCCCAACCATAA +GCCAGCTTGTCCTCCCAAATCGTTGATCACATTGCTCACCTGTAGAAATT +CCGAGATCTTCTCAATAGAGCACACTTACCGTATATCCAGCAGTCTCTGT +ATACAACTCATAACTCAGCGTCTCCAAGAAAACTTGTAAGATCATTCCGT +TATTATTATACCATTTTTGACACGCTGCCTTATTggcaaaattggaattt +gatgAGTAACAACTGCCAACCTGAAGACATCGGGGACATCGGGACACTAA +gatttacagtaacctactCCGGcggatgatgacgtggcaacaTAGTAGCT +GGTTGATGGGAACTGAGCAAGGGATACGGTAGGTGTGTAGGTGGACTCGT +CACATGGCGGATTACAAGAGCATTCGACTAGAAGATCGATGTTCGGGTCG +GAATTGGTTTGGTTTCCTTTGAGACCTTGGAGGCAGtccactgaaatttt +ttttggattttctaccTTTCGTAGTTCTCATAGAACTCCGAAAGTATGTT +AAAACTAacttagaaatttgaaactttcagaatatatagaacaaaaatag +agtttcatttttgtagtttacaacttttctgaaaaattaggtcctaggga +gatatgccacgtggaagaggcTTATCATCCAGCCAGTGACCCACACTTCC +ATGTGTCATATCTTCTCAGGAATTGATTTTACaacaaagttgtcaactac +aaaaaggaaaccctatttttgtttcatatactctgaaattttcagatttg +tatAGTAGTGGAAAAATGCTTTTTAGCGCCCCAAAGTAAgcatgttgaaa +aattgagaatatgGTTTTTGATGCACTGAAAAGCATTTTCAGCATTGTCg +gatttattcgaaaaaattcaaaaaacactaAACGAAAccagaacttcatt +tttgtagttgacaatttttctgaaaaatcaggtcCTAGGGAGATATGCCA +TGTGGAAGAGGCTCTCCAGTCCAGCGCGCACTAGAGGCGGCGGCGCCAAG +AGATGTCTTTACTTCCACGTGGTATATCTCTCTAgaacctaatttttcaa +caaaattgtcaactacaaaaagaaagctctatttttgtttcatataCTCT +GAAATACAagttcgctctattgataatcgCAGTCCGTTTACTCACAATCC +GCCTTTATCGGTTGACATGCCGTATCTGCTGGTCCCAATTTCAACCTAGG +ATTCGCACAGTTACACTTGGCTACCGTATCCCGTTGTTTGCATCCGTTGA +AGCATGTTTCCAAGGTGTAGGTGAAGTTCTGGAATAtgggacatctggac +acaaCAGATACACAGACATAGTACCTTGTAATAGTCAGAAGTGTCAGGCT +CCCGACTGACACACTTCCCGTAAGGACGTTTGGCACGGACATGTTGGGTC +TagaaattgtagtttgtagtttcagAGCCTTCTGGAACCTTCTAGAGTGT +TACCAATGTCACTGAAACAGCACTTTCGTATCCTACGCCTGCATCGATTC +CATTTGAATCTAATCCTGGCTCATTTCCTCTTGAATTGACTGCAACTCGA +GCTCCAGCCAGTTTGTGGTTGGCAAAAAGTCTGTAGTTCCACTGGTTTTG +GTTTGCTGGAAGAGAAAACGATTTCTTAGACTTCCTAGGCCACCCTACCG +TAACCGTCAACATCAGTTTTAAACCGAATTGAATACCTTCACGGCTTACA +GAGTAGTTTAGTGAGGCGTCCTCGTTGAACGAGTAGCAAGCGCCGTAGAC +AGGATcaatgaactttttgaaatcagatcTGAAGGAGAATATAGgataga +ctacaaactacaacttACTCAGCACACGGGATTCCAGCAAAACTACAATC +CTGGACCAGATCGGAATAGGTGTATAGTGCTGGCGCTTTATTAGTGTCGG +ATATCAAATTAGCTTCCAAAACCAATGCATCAGCAGCGCGAGCATCCAAA +TCGTATTCTTCAGTTTGTTCTTCCTAAAaatcttcttgttcttcctaAAA +ATCGGCAAATCCTCAACTTcaaacccatatatctcattttctgtaaaaga +tatcaaaaacctgtaaactaacaaagtgtttattataatattaaaaatat +tttgtcagtttatcactttttgatatctcttgcaggaaccgagataaatt +ggcttaaagttaagGAGAGGTCATTTATACggtagctcaattatctcaat +ttatGTAAGagaaacaaacattttgtcaactaataaaatgtttggcaTAA +AATCattgacattttatcagttgacaatttttcgatatcttcaataggaa +ctgagataattgagctcaaagtatatATTAGCGGACATGTTCTTGAGTTC +AAGGGTGATGGGCAAGCCCACCGGCTAGGGACCAGAAAGCCCACTAGCCT +GTACGCCCTCTCCACCATCTCAGCCTGTGGTCGCCGTGGTGTGGTGGCTA +GCGCGCTAGACTTTTGCACACAACATGCGAGTTCGAATCCAGTAGAGTGC +AAATCcttatttggtttttgaaaacttgattttagCAAACGCGCTCCCCT +GGACAGTTTTAGAAACTTCAAACTACCATTTTGACCTACATAAAGACCCC +ATTTATCAGTTCCATAGGTGCCAGCGACAGCATTCGAATAAGCAGTCATC +AGACTATTGACTCCTTGAAATGCAGTGTTCGATTTGACTACCGTATACTT +GTAGGGATTCATATTGCATACAGTAACAActggaaaaatctggaaaactt +tttttctgatttccaaaaatctctgaaaattactttACCTGTTTTTCAAA +GAGAATCTCGGTGTTGACCGTCGCTGGataggagaaaaatttggCGATTA +AAATGTAGAGTTCATAGGCAAACATGCCAAGGACAAACACAAAAATGACA +ATCCAGAAGATTCgggaaatctggaattttcagaaatccagaaattcaaa +aattcagaaattcagaaattcagaaattcagaaattcaaaaattcagaat +tttcagaaatctggaaattcaaaaattcagaaattcagaaattcagaaat +tcagaaattcagaaattcagaaattcaaaagctcagaaattcagaaattc +aaaaattcagaattttcagaaatccagaaattcaaaaattcagaaattca +gaaattccaaaattcagaaattcagaaattcaaaaattcagaaattcaaa +aattcagaaattcagaaattcaaaaattcagaattttagaaattcagaaa +ttcagaaattcagaaattcagaaattcagaaattcaaaaattccaaaatt +cagaaattcagaaattcagaaattcagaaattcagaaattccaaaattca +gaaattcagaaattccaaaattcagaaattcagaaattccaaaattcaga +aattcagaaattccaaaattcagaaattcagaaattcagaaattcagaaa +ttcagaaattcagaaattcagaaattcagaaattcagacggcctaacttt +cccatttttggggttttctaggccaccaccgtATTCCCTCAATACGGTCC +CCTTCGTTCCTCAATTTCTCACCTTAGAAGAACAACTTGTTTATCTCTGT +TTCGGAATGACAtcattccgaattttcttatcttctctctctctctctct +ctctctctctctctctctctcacctTATTATTTGCATTTGCCACGTGCGG +AACGAcagctacagtactccaattggaaaaatctttaaaaatctgaacgA +ATCCGCCGCCTctcatttttggaagaaagtgaaaaaaatggaagatttCG +AAGACTTTTTCTGGTAACCATGGATACCAAGGATTGgaggaattttaaat +tttttggagaagagcatttcattgtttcagaaacatgttttttgctcttt +tgcttcttttctttcggggaattttttgaaaaggagcAATATTGTGACTT +GGGGACTAGAAATGAGGGAATcggggttttctaggtcaccgaATCTCCAG +AATTCGAAGATTTTTAATTTGGAGATCAATACgctaaaaatctgaagaaa +tgTTGCTTTTTTGTCTGAACAACTTTAggatattccagaaaaattgcata +atctgaaattaaaaaattggtgaaatatacaaaatttctagaattcagaa +tttcagagccagaatttctgaaattctgaaattctgagattctaagattc +tgaaattctgaaattctaaaattctggaattctgaaattctgaatttctg +agattctaagattctgagattttgaaatcctgaaattctgatgattatgg +aactagaaattctgagattatgaaattctgaaacctctgaaattctgaaa +ttctaaaaattccgatcctttttcgaaattttaagcttCTGGAAGATTTT +AGGATAGTTTTCCTATTTTATTGCTCTTTGTTTCCCGAACTTCcataacc +tttttttcagaacttgcttcatttttttttctggtcaTTTTTAGATCACC +TTGCCTCCCCATCctaaaaatatacaaaatatCACATCACCAAAAAACGG +ATCCCTTTTTCGTCTATATTTTGTTCagacaacaacaaaaaactttgaCG +CTGACAACGGTGAGCAGCTGAATCTCTCTTAGGTGTTAAGGATTAGATGG +GGGATTTGAAATCCCCggattttccccttttttcttttggctTATACGTA +TACTGAAAATCGTATAGTTGAGGAAatgaaggaaaagagagagagagaga +gagagagagagagagagagagagagagagagagagagagaaatagaaATA +TCCCAGAAATAGGTAGCTGGAACACAACGAAATTTTCTATGAGTTAGAAG +TACCGTACCAaacaaaatgtgaaaaaatacaGCCCACCCGCACCTacaac +tttgatctcaattatctcgattcctgtaaaagttattaaaaatgtgtcaa +ctgataaaatatttttgatattatgatgaatctTCTAAgagttgacaaaa +tgttgacaactttcataggaatcgagataatcaagctcaaagtgggcggt +GTGGGTggggttctattagaggaattGCTGTACCCAAAGTTGACTATCTT +CCACTAAAATagcaatttcgaaatttctttgtTGGTTATTACTGtactag +attttttcagaagttttaaaacGGTTACTGTAATCATTCCCTAAATTTTG +GCTAATTTTAAGTGGCTAACAAAGTCTGAAAAAGGATAAAACCAAAAGAA +ATAtatggaaaatttggtttttgaacaaattgtattttttatttctcata +ataatacaaatattttatcagtagacaaaattttagtatctttcacagga +accgagataattgagcttcaaGTTGACGGTGTGGGTGCGGTTCTGTTAGA +GCAAGTACAGTAGTCAAACTTGCAGTCCTACAAAAgttctccagaatttt +tcggcttgcttgaaaattttaaggaatactaaacaaaagcttcatttttg +tagttgacaacttttttgccaattttcgtTCTGATGAGATATACCATACC +ATGCCGTTTGTaataaaaaaagtgtcaaagtCTCATATCTCGCTTTGCTA +agtttgttcaaaaaagaaacaacttacaaaaatgaagatctaTTCTTGTT +TATTGtgctctggaattttctgaaaaatcggaaaattcaatttctggaCT +ACTTTTGTAGAACTTTGAATTTGACTAATTCCAattgaaaatgccaaaaa +tgtaGTTAACCCTCAAAATCGAGGGGGCCTATCCTTATCGAACGGGCTCC +GTGTCACTTCaaaaccgcgacgtttcgaaaccggtcatttcgaaaccgcg +aaaTTTGCACgctgcaattttttctttaatttaagacttcaaaatttcct +agtttcaatttcatttttacgtGACCGTGTTTGTTTttcggtttcgaagt +gtcgcggtttcgaaatgaccggtttcgaagtgtcgcagaGCCATTGAACT +CACTgaagtgaaatttttgagagtttcaaCCTCCTGAAACATTCTTGAAT +TTAGTAGTATCCACTACTTCAATCCACTTCATTATTACATCGAATTAATG +CTAGAGCGCACTTTCATGTCCCTCAACCAGTCGTTTTTCCTTACCTCTCC +CCTTCCTTTTTTCCCCCTCCCAGATGACGTCACCCATCTGTCCACCCTTC +TAACGGTcccccccACCATCTGCATGGTGTCCTCGGGGGTGAACAGCTGC +ACATTTATTGTTCCCTTCTATTCCCCCCTCCTCGGTCATCGCGGTTTTAT +CCCCGCCGTCCATTTGACCATTCTTTGCGTCtcttccctctctctctctc +tctctctctctctctctctctctctctctctctctctttctctctctctc +tctcttctttttcttctaggCGAAAGAGGTCACATGGAAGAGaagaggat +gatgatgatgatgatggtgggGTGAGAATAGAGTGGTGGTCTAGAGGAAG +AGGTCTACAATAACGGacaggaaaaatggaattgatGGGTTTTGCGGTGG +TGCGCGAGTAACCATGATAACTTATCAATGGAGCGAAGGTGCTACAGAAC +CTCTGGAGCTTTAATCCTGGAAGGGAGTGTGTTGTCAACGGAGCGCACTT +GCTGTGGACCCAAAAACAGCTGGAAGTGTTTCATCGGGAAATGAATTATT +CTTTAAGGTCAACTTCCTAAAATTATTATAGTTCTAgtgactctgaaaat +ctgaaacccTGTCATATATGACATTTTCGGAATCCTTCAAGAATTTAGTA +GGATTTTTCTCAGATCTGCACAGAGAAGATTTTGGAGGAACTAATTTAGG +AAATATCTAAATCTGCAGAATGAGGAGATTTAATGAGATTTATGAGCCGC +GTTATTATTTATgagttattttgaaaatttagaggATTCAAATTTTAACC +TTTTGGGCTTTAAATAAGAGGGGACCTGAAAAACGGGAGATGCCGTGTCG +CAACCagacaccccccccccccccctcaacTTTCATTCCATTTATAaatg +ggtcaaagttaaagggCTTCTTGGTCCTTCTGATTCtgtgaatctcagaa +tttccaaaccTCATAAActaattcctagatttttgcAACTTCAGAGatac +agattttcagatttttcagaaattaagaatttccaacaaaaattgaaatt +cacaaGTTCGATCTTTCCTGTggctttttcattttttttttgtttttttt +tctgaaaatttttttaaaaaatcttaaattttcaaactttttattaataa +attaagtgaattatttcgaaaaactgaatagacacaaagagaaaaaaaaa +cgaaaactaaaaacggaaaaaagaaCATGTACTGTATATACAAATATGAA +TGGGTTTtgggcgggaatttgaatttttttttttcgaaattcatatTCTT +ccgaaattaacaaaaataaCTGGGCGACTGGGACATACAGGGACTGACTG +AGAatggaaagttctggaaatttccagaaaaatgaaaaaaaaagaaagag +ataaTGATAAAATGAAAGAGATTTTAGGAGAAAAGTGGTGGGTTCCcgta +gtttttttaaaggaaaaataaaGTATTTTTTGGATGATGGGATTGGGGGC +Ggagcctttttttttgagacaattttttttcgaaaatttttttctaccag +taatttctttttaaaagtgTGAGTGGGGTGTTTGAAGTATTATATGAATC +GATAAGGAAAGTGAAAGGAAACATAGGAGCCTAGTAATTAAGTCAAAATA +GTGCCTAAAAAGACGACAGAAAATTGTGGAGATTTTCGcctcgattttca +gatcttgATACGCTTCTGGAACGCTACATTGAAGTCGTCGAAGAGTTCGG +TGGAGCGACGTTTCTTTTTGTTAGCATGACATTCTCCAGTTGCCACGTCG +AGATCTGTCAGCTCATGGGTGTCATGGAGTGGTTCCGGAGAAGTGGatgt +ctgaaattatgatctttgaaattttttagaattcaaaattttctgccTCC +CATCCTCTGGTTCAGGCGCGATCTTATCTATGACGTAGACGCCTTCCGTG +CCTCACATGACATCTGCACGCGTGGCCTACTAGCTAGCAGCGCTGGTTTC +GATGAGAGGGTTCGCTGGTTCAAGCCCCCTGGCTGCCTATTCCTTTTTTG +CTTCTATTTTAGAATGCCCAACTCACAAAGACTGGCTCCACGGATGCTCT +GTTGGCCAGGAAGTAGTCGTCTCCCGGGAAATAGGTCGCCGGTCCTCCTG +GATTCGCCGACGTGTACGCATCGTAGCGAGACGGCTCACGAACGCATCCG +GAAATGCTTGCCAGCGGATCGGAAGCCATTGGAAGATCTCCTTCGGTCGC +CGACTCGGACATCGAGCTACCACCACCATACATATCAAACGATCCATAGC +CGGAAGAGTAGGATGGTGGAGACATGTCGAATGGATTGTAGGATACTGTA +GACATGGATCCGAAGCCTCCGCTCAGCTCGTTGACCGGATACGGTTGATG +CAGCCAAATATCATTGTGCATCATTTGATGGTGATGGTGGTCGTAGTTGT +TGGTGAGGTCTTCCAGACGGAACTCGTCAATGTCTTCCTCGAAAAATGCC +ACTTCAGACATATCGTAATCGTCGTACTCCTCGTCAGACGATGAGGCCAT +GTCTTGTTCTCGAGGCGACGTTGgcttttttggcattttgaaGAGGTCCG +ATACAAAGTTTTTGATGAGTACTCTGGAAAGATTCCAAGGTTCTAGAGGG +GCTGAGGGAAAATACTCTAACACCATTCCTTCTAAGTTTGAGTTAATTTA +TCTAGACtcctgtcaaaggtatcaaaaagtgatcaactgataaaatgttt +accataattcAAGAAAccttttattagttgacaattttctgatatctcta +acaggaaacaagataaattagctcaaactaaACCATCCTAGAACCCCCAA +TACCAACTACAGTATCACGAACCTCTTCCTCATGGCTCTTCCTCCACACC +CGGCTCGCTTCTTCTCCGACTTATCACTGCATTTGTGAAGTGCCACGTTG +CACAAATGGCCGAAGATCTCCTCTGTCAGATCGTTCAGCTTGTACGTCTT +GTTCATCATTATATCTGGAAGGATTCGAGAAATTAGAGAtagaaacaaga +agaagaagaaggacataagttcagaagacgttatgATTTTTGATACTCTT +CAAGGGGAAGTGTACTTCGGAAGGCAGTAGTTACAGCAAGGGCGCGTCAG +AGAAGGTGACTTATCGCTACAGTACCCCACGCCAAAATTGAGAGAACGCG +CGCGCGCTTATCACGGGACTTTCACACCTTGTAAAAAGTGTGCGCGCGTG +ACCGACTGGCTAGAAGGCTCGTTTTGGTGCGAGAGGCCAGGGGTTCGATT +CCCAGCTAGGGTAAAAGcctttttgcttttgttttATAGCTTCTAGTGGG +GTGTAGGATGTCGTAGGAATGGATCCAATGTCTAGGGGCCCCGTTTTAGG +CCTAAAGATTCTGTTTTTCGCTTTTCTAGGCCTAATGGATCTAGTTTGAG +ACACTAGCCTGTATGTCTAAAGCTAGCCTTTATGTCTAAAGCTAGGCTGG +TCCTTAACGATAGATTTTCTAAACCTTATGGCCTAAGCTAGCTTTCGATC +TTAAGACCAAGACCTAGGCTACCGTATTATTTATCTAGGACTAATGCATC +TAGGTAGAAGGCTAGGCCGAATCTAGCCTTCAATGTTAAGATCAAGACCT +AGGCTACCGTAGCTGCTCCCCATACATTCCTGGTCTTCGAAAATAACTTT +TGTGTCTATTTCCGTTCTTCGTTCCCACACTCTGACTGGACCTAGACTCT +AAAGAACACGCACACAATTTGTATCATTTCCTCTCgggtttattttttct +cctaaCCAATTGGTCATTTTGTGATGTTTGCATGTGCTGTCTCGCCACAC +TCACACAAAAAATAACTATGTGTCTGGAGAGATAAATGAATGCACTTTGC +TCTCTCATTCTACCCgtgtgtctctctctctctctcaacaATGAGATGGT +GATGATGATGTGTTTTTTCGCGTATCGCGCCCCTATAGGAGGCCCCCGTG +AGgcagcaagaagaagaagaagaagaaagaattgaaagaagCGGCGCACA +CAAACCCAACACAGCCCAAACTGCGCCCCATAGTGAAAAagcgaaagaag +aagaagaagggagggGGCTGTCCTATACGCTGTGCCCCGCGGGCTGCTGT +GGCTGCTATAGCGAGCGAGCGCGCGCGCACAAGGCCAcccctctccccct +tctttctctcgtcttgtcctctctcgcttctctctgcattctctaattca +ctatctccttaccccctcttctctggcctctcctctcCGTCACATGATGT +CTTTGTAATGACGAGTGGTGCTCTCCTCGTTCCCAACACTATTTCGCATC +CATACATACACATCGGGCGAAGCGGCAACCCGACGGCGGCCCAACCAACC +AACCAACACAGCCTTCTTCCCTTCTCTTCCCTTCAATTTGATtcgaaaga +aggagaagaagaaggcaatGGAAAGAAGGGTTTTGGCTGGGCTGGTGGGG +GAAACAAAGTCTAAAAGGTTCGCCGATGTATGTTCATCTTCTTGTTCAGG +ACCAAATGGCCGAGGATTTGCAATCGGCGGCCGGGAGACGGGTTGAACGA +AAAAAGTGTATCTTTTTactcgaaaattgagagaacACCCCGGCATGTTT +ATCATCATCGATGGGGGAAAACCCCCCGTGTCCTATTAATAGACGACAGA +cctggaaatgagaaaaaaacggacGGGAGGTCAAAATGTCTACTATCAGT +TCGGGCGGTACCTGAAATgtctgtttaaaaaaaagcggaACACAAAAGAA +AAGTAACACTTTATTGGATCGACGAAGAAATGGAATCTGGAACTTGACGG +TTATGACCTactgtactttgagctcaattatctgaatatctatgagagat +atcaaaaattgctaaactacaaaaaattagtGGTATTAtgaagaacattt +tattagttgaaagatttttgatatctcttaaaggaactgatatgaattgg +ctcaaagttggaagtcCATTTTATTAAGGCAATCTATGAATTTCAGGATT +ACAAAGActcagaatttggaattcttagattctgcaGTCTTTTAGAATGT +TTCTGCAATTACGGTATGTGAAAGGGCCAAATCTTGatagccattttcag +caaaacaaTCGTTTTCTAACATTTTGACCAAGTCTAAAACGGACCCATGT +CTTAAAGGGTCTTTCAAAAGTTGGCAGGTCAAATTTTAGGCGTATAGTTC +AACTCAGTTTGAATCTcctttttgttgaaaatttaaaatttaaaattttt +gggatttcatgGCCTAGTTTATGTCATCTAAACAAGAGGTTTTAGGTTAG +AACTTCCTTtaggaactttcaaatttctcataaaTCTGTCTGTGTGTTGG +TTTGTACGGATTTCCTCTttcttattttaaaaattatttgctTCTAGTTG +GTTTGAAGCTTAACcaacttttaaatttcttcttctgttaAAGTCTGTCG +GTGAGTCTGTGTATCCGGATGTCCAAGTTTTtcgtttggaatttttgaag +aatgccTTTGGGTTCCTGTCTGCATTTCGACGTGATGGTGTTTGCGGAAA +ttagagctacagtaattcacAGGTCAGAATACACACGAAAACATCGCAAT +GAGAAAGGAAACTTGCTCCtacagaattcagaaattttccaaattttagg +atttttaggatttctagaatttctagggattttggatttctcttggagtt +ttagattctagattttcaaatttttcgaaaatcttcagaatctcaaagtc +tgtttgtccagatgtcccaTGTTCTcgactctgaaatttttgattttaga +aaactcaaaaattcgcctattcttccatttttgaattttcctatcaacct +gaaaattttgaatttttcgaattaccgtagaacctgtattagaagggccc +ctctagtagaagggcccctctcatagaagggccctctgtgaggggcttgg +aaaaatagaagggccccctgtaatagaagggccggcccttctaatacagg +ttttacggtaagtgcTTTCTAAACCAcccattccattttttttcagctcc +gcccaaaaaaacaattttttcgatttcataaattttttcacttttcggCA +GAGGACGCAGGTGGTGTCTAGATTGCCACAGCCTCTCTCCGCTAGTACTT +GTTGTTGACTcaccaaaagaaaaaaagaatttcgcAATAtctattttaca +aaaaaaatattcggtgtgtccggatgtccggaTTGCGTTTCTTCACAGCT +ATAGTAGTCTGGAGCatgcctgaaaatatttttccatcGCGAAGATTGCC +TCAAAACATtcgtacagtaatcccagagCTCTTTTGGATGAGTCATACGG +TAgatggtgatgatgatgataagaggaaaatggaagaaagagagtatgga +aaaatgagagagtctagagaaacagagagtgtattggagcagaagaagaa +gaacgtcATCAGATTGtcttatcaaaaatgaaaaaaaaacgcgggggaag +aaaaaaatgtggaggaaaattttgaaatttagagaactaggaggaaaagg +gaaaatgcaaaactatgaacggaaaaaaaGTAGTGTCCTTGGAAAAACGC +CTTTTGGTGGGATTGTGAAATTGGATATGTTATTTTTGGCAAAATAATAA +ATGTGTCTAATAACGATTTTTGGtcattctaaattttgaaaaaaagtgga +atttgAAACCAGACGGAGTCCTAACTTTGACCACATttatctaggttact +gtaaaagatatcaaaaagttgtcaactaataaaatgtttgtgatactatg +ataaacattttatcagttgatcactttttgatatcttttatgaaaatcga +gaaaaatcagCTTAAATTTTGGGACGGACTTCTACGCTAAGATAGGGTAG +AAAGAACTCCTAGCTTTGGCCCAATTTATCCAGGTCCctgaaagagatat +caaaaagtgatgaactaacaaaatgtttgtgaaaaaatgataaacatttt +atcagttgactacttttttatatctatcacagaaagagagataaattggc +ttaaagtaaAAGGACTCTTAAACAGACTAACTTTTaggatttactttttt +tttttgaaaacggcAAATTTTACATGATCTACAAAAGCagttaaaactat +ttttatttttctgaaatttttagagtatactaaacaaaaatacatAGTCC +TTCAAGTTtatagttgaccacttttttgtgttatttcaTCTAAATGAGAT +ATGGCGCAGAGAAGATGTCTGACGCCTCTacttggcaccgtgccaagaAG +GAACAAGAATGTTGGCACGGTGCCAGGacctcaaaaaaatgttgctctat +ttttattttcttttagaaaggaaaatctgaacttttatgaggaatagaaa +tttcagaaaaaatgaaaaattgaaatttccaaaaaaaaaactataggAGG +ATTTTCCTAAAATCTGTCCGTTTGTATTTCCTAATATCCGGTTGTCccta +aactaaaaaaattgttctcaaaaattctgatttgtcaaaaaaaaggagaa +gaatgtTCTATTTAGGACGGAAACAAGAAATCAgagagacacaaaaaaaa +gcaaaaacgtgcaataagaaaattagaagatgaaaaaagagaaataacga +aaaagagaaacaaaacatagcgtttttttttcgacgggaatgaaaaatga +agaaaatggagaaaacgaaaaaaagtgtgaaactttttttttgattggtg +ggaatttttttgtcttgaGTGGGAGgaaaaggaagaggaaaatGATGAAC +TGGAGGTTTTGAAAAGGAATCAcacgaatttgaaaatttgagttcccGCC +AAGATTCCACGGGActctaatttgaaaattttcgaaaatttgaaaattca +aaatatgaaatattttctgtgtcaaatgacagaaaaattAATCAATCTTC +GGCTCCGaatgttgttgttgttgtaaAAGATACGCAGCGAAATTTGAAGC +GGAGCCATTTGATGCTCAGTTaacggttttttttatttgtgaaGAAATAT +TATGACGTATAGATGAAAGCAGCCctggaaatctagaatttcatgaaatc +gaaaaaaaaaattaccaaaaaagcTTCCTGTCTATTCTATTCCTGGAATA +AAAGGGTTTCCCAATTTTGACCCAACTTATCTCGATTCGCAttggagata +tcaaaaagttatcagctcataaaatgtttatcatgatatcacaaacattt +tatcagttgacaactttttgatatctcttacatgaaccgagataaattgg +gccAAAGCTAAAAGGTCTTCTTACCCTATCTTTGCGTAGAAGTCATCCCC +CAATTtaagctgaattatctcaatgctcataagagatatgaaaaaattgt +caactaacaaaatgtttaccatattATCATAAATATATTGTaagtttatc +attttttgacaactcttacaggaacctagataaattgggtcaaaactAGG +AGGTTCTTCTACCCTACCTTAGCATAGAAGCCTCttcctaactttgagcc +aacttatctcgattctcataagagatatcaaaaagttgtcaactaacaaa +atgtttgtgatattatgataaacaatttatctgttgacacatttttgata +tctttcacaaaaagcaagataatttggctcaaagttgagggaCCTTTCTG +TTACATCGGGTTAAAATGAACCTTCTAAATTTAAATatgaatctcaggat +ttcaaaatctcagaattttgaaattttgtattctCACTTCcctgaaaaat +atggatttaGCCAAGAGAATTGGAAAGAAAAGTGAAATGACTAGaagaaa +gaaacgaaaagCCAATAGCTATGGAGACCCgtcagaaaaccaaatacgCA +AACTTTTCACATCATCTCAAACAAAACACACAAATACACAAAAAGGGAAA +GAACTAGAGGaaatttcatgagaaaaatacaatttgttcaaaaaccaaat +tttccataTATTTCTTTTGGCttagacattttcagactttgttAGCCACT +GAAATTAGCCAAAATTTAGGGAATGATTACAGTAACCgttttaaaacttc +tgaaaaaatctagtaCAGTAATAACCAacaaagaaatttcgaaattgctA +TTTTAGTGGAAGATAGTCAACTTTGGGTACAGCaattcctctaatagaac +cccACCCACaccgcccactttgagcttgattatctcgattcctatgaaag +ttgtcaacattttgtcaactttTAGAagattcatcataatatcaaaaata +ttttatcagttgacacatttttaataacttttacagaaaccgagatagtt +gaactcaaagttgatggtgtGGGTGtgggttctattagagggaatacggt +agttaAACCTGAAGTCCTACAAAAGTGctccagaatttttagatttgtcg +gaaaatttcaggttacactaaacaaaaatagagcttcatttttgtattaa +acatatttttgtgtAGTCACTGGTTACATTGGATGAAAGACATGGCATCC +AAAAATGCCGGAATTCCATATGAAATTGTTTCAGACTGTCTTTTCGGGGG +AGGCGGGGAGATCCTTTCTACGAGTGcatgtacaaaaaagttgctaatTA +CAAACATAGAGTACTGTAATTGGTTAGTGATATCTGCAACTTTCAATAGA +GCAATTACTAAGAAACCCCAACCTCAATTGTCCTCTAGATGCATTCTCAG +GTTCGGCTCCTTTTTGCATAATTgcgcctttgccgggccttcgcggaatc +tgccatagtttcttagaaaatcaatatttttaaattaatttaattagata +tgatagaacttgtCGAATGTTAcgaaatgatactatttggagtactgtgt +cttgtctatgaacaaagttacaaacgAGAAACAGAGGCCCGGCCACTTta +tgacttgtaactttgttcatagacaagatacaatattcaaaacaatatca +ttttgtagcactcGTTGAGTTTTATCATATCTATTTTAGTCATGTCAAAA +ATATTGCTTCtcaaagaaactatggcaaggCCCGCGAAGGGCCCGGCAAA +TGCGCGGTCACAAgtatgcatttttgaaatcttcatgCTTTTCTGTGTGA +ATAAAAGAACGAAAGTTGATTGATGTTATAATAaaatatataggctcaaa +ttaACCTAAACAAATACAAATTTCCCGGGACGAACTTCGAAATAATATGA +TAcaaatttgaagaaacaaAGCTATTTTAGGAATCCAGACGAGTTTAAAT +ATAATGAAAATTGGGTGGTAGtttagagccaaaaatcgctaaaacATTCA +AAAGATAATATGATCTGGACcttaaaattggaaattccaaaacagtGCCA +AACTAAAAAGAAATGTGCTAACCTGAACACACTGACTGACACGTACATAT +CAATACACCATGTTATAAAAATGTGGTGGTTTATTGATGGGGGGTTTTAG +AGGGAATAGAGAACGAGTGAAttggagagagaaagagaaaaagaaaaggt +tcAATAATGGCCGATTGTGCAGTTTTATTACTTTGGgtgaaatggagaaa +aagggaatgaaactcgaaaaaaaaaacaaaaaaaaggaaaaaagaactaa +aaactacgaaatcaggggaaaaTACTTTTGAGATCATGCTCCTTTAAATA +CCTACTGTAGAATCAAAGGAACATGATTAGAAGTGATAAGGTAGCGGCAC +ATCCCGTGAGATACCCGAAAATGCGAGATAAAAATTCGCTTAACTATCTC +GCTTCCcattaaagatatcaaaaatttgtcaactgataaaatgttttttc +ataaaatcattaagattttatcagttgacaattttttcatatctcttggA +GGAACCAAGGAGCTTAAAGTAAAGTTGACATTTTGCAGGTCATTTAAACA +GTATCAGAAGACAGAAAATTTCGTATTCAACAAAAGTGGTTGTTCAACAG +AGCGCGTTTTCTTAGAAGATTTAGTCATTTTTTAGATTaggtttttgaag +aatttgaacaaaatacAATCGCTTGCGAGTGAGCTTGGCgttctctggaa +aatttcagatcaagaattcgaaaaaaaaatctgactaAGCAAGCCTAGaa +atttcctgtaaaagtcTGACATGTACAGAAAAAAGAGTGAGTGAGTGAca +ccgaaaaaaagtgaacacGAAAAAGAAGTGGTAGTTCGGTTGGAAAACAT +AAATATATACATAGCGAAGAGGACAGACAACCCTTTCTTCCATCCCATAC +TGACACATAGACATATAACATAATACATATGAACAcacaaaagaaaataa +atttaggcagacagaagaagaagaataaatgTGGAAAAGAATGGAGGAAA +GGGCAGAACAAATGAGAAATTGGAGGAGGAGAATAAGAATTGCTAGTTTT +GGctatttttgggaaaaggaGGAGATGGTGTACTTTGTGTTGTTGATTGA +GAAGATTCTCGAATCGGCTGCAAAATTGTCTAAATGACCTAACACTTTGA +ACAAGTTTTGGGAGTGCTGATTTGAGACAATTTAAGACAGCTAAAAATAG +TAACAACGAGTTTTCCTTCCATAGAAATCAGCTTCTAGATATTACAGTAA +ACGTATATAGagggcaaataattcggagggtacaaaaatttaggaatcat +tggggtacaaagcaatttaggggtacaaaatcttcGATTCGGAggtacaa +aatacctgaaaaaggCTTAGTTTTGTACTCCCGAATTGTGAATTTCGTAT +ccctaaatcactttgtaccccattgtacccccgaattatTGTACCATCAG +AATTATGTGCCTATAGAGTCATGTTCGATGTATATTCgctgagtagcgac +tcaacgACCCAACTAGAGTGATACGAAACACACATTTCTAGTAGTTTAGC +ACATTTTCAAGAGAATTCTGAATACATAATTTTCCCCGAATTGGAGAAAA +TAAATGCGTAGGCTGGTGACGAGAACAAGAACCGCTCGTTAGCGATTTGA +ACAGAAAGACACTACCTGCATATTTTGTGTACTGGAGATTAGATTATGAA +TGCAGAGGGTATTCAAAATAAATGTAGATTGAGAAGGAGTACATAGGAGG +AGAGGAtcgaagagaagaagaggaaatgaACAGGTAACAATTACAATTTA +CTTATCTTCGGACCCCAAAGAAGGCTCTAATAGAACACAACTCTTATGAA +cacaattatcttgattcctgtgacagatatcagatttttgtctactaata +aaattttgggtgataatatgataaacattttattagttggcaaatTCGAA +TTATCTTTCAAGgaaagcgagataattaagatttttgaaaaagctgaATT +CTCactttgacgtcttctgaagacatCTAGAAACAAAAAGTCACCGCCAA +CAGAAAGGTTTACGTCATCAAAGATGACATATGGACAATTTGAGTAGTGA +GAAAGGGGAGTATGAAAATTGGTTCCATTCGGCAtcttttctgaactttt +agcTGAGCAAACCAAACTGCAAACTTGACATTAGCACACAAGAATATCAA +AATCCATTAATCCAGAAAAACACAACTCCTCTaatggattttcagagatt +tcggCTAAATGTATCTTGTCATACAACACTAGAATTAATCTATCTTGGTC +AAGATTAAGGATTGGGGAGGTACGCAGCGAAATCAAACTCTACGTTTAAT +ACTCAGTTCCTTATTTTGTGAATGTCGCTtagtgaatctgaaaatgcagT +TAAAGGCGCAGCATGGAAAAACCATGGATACAATAATTATTGCATACAGT +AACAGGAAGATagttaatttttagaaagttttgaaagtagTCCAGAATGT +TTcgattattctgaaaatgttgaaattgactaaaaatacGGTAACTCGCT +CTTGCGCCTTTAAGTGTTCCTTTAATGTTACTCTCGCTCTCAGCGTATAT +TGATTTCGAACTAgtcagaaaaatctagaacatcacatcatttttctcaa +caacaaaacacacaaaatgaaagaaattttctgggtgaagaagaagaaaa +aagaagaagtgagtgttcttttttcattgtgccattttccaaattggtgg +aaaaaagaagaaaaagaggaaacaaaaTGAAGTCCACACACATGTGATAC +atcaattagaaaaatgagcattcggggaaggaggagaaggaaaAGTAATG +TTTGATTTTATaaataaaggaaagaagagagaaaagagcagttgaaaaga +gagaggaaaaaagaagaagaagatgagcaaCACCTCATTACCACCCGATG +CTCCGAGGATTTTATATAAGAAGGATAATGGGATTTGAGCAATGGGAGGG +GCATTAAGGGGATCTgtttattgaaaaagtttggagTTAGATGTGCAGTT +GAAACTCATAAAACTCACTAGTCCACTAGCTAACAGTTTTTAGCTAgctg +ttttggtttttgagatatgcgcctatAGATTTTGCGCGCCCTTCTAGGGA +GTGTACACCTTTAAAGACGCATGTCTCGAAAACTATGGTagttaaaaaca +aaattctaaataagaaatgtgtgaaattttaggCTGAGCAATTTAATAGT +AAGCAGCTTTTAGCTAGCTTTtatggttcttgagatatgcgcttttaaag +taGGCACATCCCTAGAGGAGAAGGAGGTGTGTCTCCTTCTTAGCACTata +cttcaaaggcgcatatctcaaaaactactgtagcagACAACAAATCGTaa +attatcaaaatgtgtaaaattctATGCTGAGCAATTCATTAGTGGACAAC +TTAATGATAACTGTCttggttttggagatatgcgcctttaaagtttgtgC +GCCCTCCTAGttatgctaactttatgctaacattatgttatgctaacatt +atgctaactttaaaagcgcatatctccacAACTGCTGTAGCTATCTAAAA +ACttttaactagtaaaatatgtgaaatttaATATTGATTAAATTTGTAAT +TAACTAATTTTAGCTTGGAGTTAGATGTGCAGATTTTTGGGTTGTTAAAA +TGGAAGGCATGATGTACCAGCCCGCTCCGTATCACTCTAGTTGtccttta +agctcaattatctcggttcctgtcaaagctatcaaacatttgtcaactga +taaaatgcttttCATAAGATaatttacattttatcagttggctcattttt +gatatctctcccaggaaccgaggtaattgagttcaaattaGAAGTATGGG +AATTCCAggcgaatttgaaaaaaatggaaattggcGGGGTTAGTTTAGAA +CTAATAATCGATcttcaaaatggaaattccaaaacagtGCCAAACTAAAA +CGAAATGTGCTAACCCGAGCACACTGACTGACATGTACATATCAATACAc +cattttataaaaatgtgGTGGTTTATTGATGGGGGGTTTTAGAGGGAATA +GAGAACGAGTGAAttggagagagaaagagaaaaagaaaaggttcAATAAT +GGCCGATTGTGCAGTTTTATTACTTTGGgtgaaatggagaaaaagggaat +gaaaaactcgaaaaaaaaggaaaaaagaactaCAAACTACCGTATTATTG +GAGTATCATGCTTAGAGAAAATAAGGTAACGACACAATCATCAATCGAAT +GAgataccgttaaacctgtattagaagggccccttcaatagaagggcccc +tgtaatagaagggccctccgagaggagcttagaaaaataaaagggtccct +gtaatagaagggccttctgTGAAAAAGGACCCAACTTCAAAACGTGtgtt +gaaatcaaaaaatttccgtactttttcagtggaaaacatcccgggaactc +cacaatttaatagaaatttgctgaaattctacttttcgataaaaatcttg +ctaaattttggggaaattggcagattttcatgaattttgaacgaaaattc +gtCGAATGTTTCGATGAAAAACTTCCGAAGAAATGGCCgcaaaaatagaa +gggccccttttaattgaagggcccctgtaatagaagggccctccggaggt +accttggaacaatcgAAGGGTCGAccattctaatacaggttttacggtat +ttcaaattgcgagaaaagtgaaatccgaaaaaagtgttttctggCTCAGC +AAAATAGTTATCTACCTATTTGTATCACAGGAAATATATATCTTACAACC +CAACCAATGATTTGTCTGGACGAGAAGATATGAAAGCGGTTCGTAGGAAA +ATCTCTATTCTTCCTGGAAACACGGGGAATTGGAAACTATATATAATGCC +TTCAAACCGTTGACTTTGAGCTCTTATGTCCTATGAAATATTCTTAACAT +AGTTTCctacactctaatagaactaAACTaaatacttgcactttgagcat +aattatctcgattctcataaaagatatcaaaaattggtaaacgtaaaaaa +tgtttttcataatatgaggaacttttttttttaatatctttaacaggaac +ctagataaattaaGCCAAAGTTGGGAAACCTTTTCAACTAAAATAATATT +GTAAAAATCAGTTCAATGATCGCCTGTGGTGGGCCAGTCTGATCTCTGCG +ATTGAATCCAAACTCGTGTGGTCCAGCGTGTAACAAGGCCagattcaaaa +ttgttttttttcgagagattcaaaatgagtataatcatgacccaactccc +acaatttttcaaatttcccgccacaACTTGTCCGCGAGGAGTCCAAAACT +AGCCGTcgagaaagtaggtcatgattatattcCTTTTGAAGTGTTCGAAG +AGCTCATTCTGAATATTTAAAACTGGAAACGTGGGAATCGaaaattacca +tttttgatcattttcaaatctccCGCCAAACCATGCCTATCGAGTTGATC +AATTGTGGCGGCTGCAAGAAACGAATTACCGAGGATGAAGTAACCAAGGT +ATGTGGCCTAGAACTTTCTAGCCcattcattttccaaaaacttacaGGAT +CAAGTGGTGTCGTTGTTGAATCGTATCTGGCACAAAGACCATATTTCGTG +TGTTTATTGCAAAGTAagcaatctacagtaatctacagtaatccaaaaaa +caattccagCTCACCATCTCGGACGGTCGCTTTTTCAAGTCCGCAGTCGA +CCCAATGCGACCGTCGTGCTACGCGTGTCATATTCAAACAACTCATCCAG +CGTGTGTTGGTAagtgtggcctagaaaaattaggccattgAAATAGGCTc +ttcaagagcttcaaaatgagtataatcatgacccatcttttgaaaaattt +aaatttcgcGCCACCtcttgtccacgaggagtcCAAAACTCTTTATcgag +aaagtaggtcatgattatattcaTATTAAATGTCTTGAAAAGCAGAATCT +GAGTATCAAAGCCCAAATACATTGCAATTTTCAGGCTGTTCACTGCCCGT +AATCGAACGAGGACTGGTTGCATTCGGTCGTCTGTTTCATATTGACTGCT +TCCGTTGTGCCATCTGTAATCGAACGATTCCACAGAGAAAAGGATTCTAT +GAGAGAGATTTGATGTTGTACGATGATGTCTGTTATATGATGTACATTAA +AGATTCGCCGAATCCACCAATGAGAAGTCCGATGAGCAATAATTGAGAAA +CTAAATTTGAATGAatagtttttggaataaatgaTATGATTAGAAATTGA +TTGATGGCTAGTTAGATTCAAAATCTGATATTTTCGGGTCCAAAATTTAC +ATTTCTAGATAAAAATTACAacttttttaactttttccgTCCACCACACA +ATAGAATAGCGTAAGACCtgttgagccaatttatctcggtggttgtaaaa +gatatctgacatcggtcaactgacaaaatgtttatcgtaatgtcagaaac +attttatcagttgattactttttgtatcttttacaagaaatgagatatat +ggaacTGAAGCAAACAGTATGGGAgtccagattttttcgagaagatTCTA +ATAGATTCATCCttgtagatttttgaattttgtaaaaaattggatttcta +gattttgaatttaataCTGTAAAGTCTGtttttttGGTCCGCAATGTAGG +Gtccccaactttgagcttgattatcttggttcatgtgaaagatatcaaaa +agttgtcaactgataaaatgtttatcataatatcagcaatattttatcag +tttactactttttgatatcttttacagtaaccgagataaactgaaTCAAA +GTGGGCTTCTTCCATTACAGGCGTTTCCATAAATTGCAGCGATCTTTTTC +ACAGCCTCATAATGACTCAACAATAAGAGGAAAAATTTACCCTTCCACAA +AAACAGCACAGAATCTATTTTCGCCCTCCTTTCTCCTTATCAAAACTCCT +CTTGTGAATTGTgaatttctttcatttctctaatccccctctctctctct +ctctctctctctctctctctctctttccagTTGCTCACCTTGCTCAGCTG +ATCCCTCCCTTCCCTTCTCCGCCAAAATCGGATttagtttttgtttcttt +tgcCCCTCCCtcttcttcatcaccaccatattctttatctgtgcttcttc +ttctacttctcccttcttcttcaactgctcttgttgctcttTAACATAAA +ACACACTTCTCCGCCTTTTTCCATATAAAGTTTTGTATATATAATATGCA +CACACACTAGAGGTTGTAACATATGATATCCTGggatttcttctcatttt +tatatcttcctcctgatctcttttttttccattttggtTTGTATGTACCA +CTTTTCAAGTTAAATGCTAGGAAAAATAACGACGCAATTTTTAGCGCATT +TACTTTGGCTCAAcgtatctcggtccctgtgaaagatatcagaaagttgt +taattaaaaaaatgttaccataatatcacaaacaatttatcagttgacca +ctttttaatatctcttacagaaagtGAGATAAATAGGATCAAAGTAAGCT +AGGTAGTTTGTCCATCTCATGACCCTCTCTAGCCCTCCCCGTTTATCTCA +Gttagagaaatcaaaaagttaacaactaatgaaatgtttctgatattatg +ataaacattttatcagttgattactttttgatatctctcacaggaagcGA +AATAAATGGGGTCATAGTTATGGACCCCACTATTATAATTTTTACGGCAT +GCTACGGACAAAAACATGCAcatttttcatctaaaaatgCGCATTTCAGA +TTTGGAAATGACGAatttagaatcaaaaaatttagatataaaaaccaaaa +atgtttttccaaattccgttttccaaaaaatctggagGGATCCCGAAAAG +TCAACGCCACTTCTAGGAGCGcgattatctcgtttcctattagaggtatc +aaaaatgtgtcaattagCAAAATGTTGAGCAATATATCACacatgtttta +tcagttgaccactttttgatatctttcatagaaaccgagataattgagcc +ggAAGGGAAGGTATACGGTGGGGCTCGGAAGGcgccgcgtagctcagccg +gtagcggcacGACACCTCTAATATTTTGCCCGCAAGTTCAACTTCTCCAG diff --git a/cudaungappedextender/data/example_hits.csv b/cudaungappedextender/data/example_hits.csv new file mode 100644 index 000000000..8be1518b1 --- /dev/null +++ b/cudaungappedextender/data/example_hits.csv @@ -0,0 +1,143670 @@ +18,18 +19,19 +20,20 +21,21 +22,22 +23,23 +24,24 +25,25 +26,26 +27,27 +28,28 +29,29 +30,30 +31,31 +32,32 +370,32 +249110,32 +33,33 +107969,34 +34,34 +214409,34 +35,35 +36,36 +37,37 +38,38 +39,39 +214243,39 +40,40 +143910,40 +41,41 +42,42 +43,43 +44,44 +45,45 +46,46 +961,46 +47,47 +48,48 +49,49 +5820,49 +50,50 +51,51 +52,52 +53,53 +54,54 +55,55 +56,56 +28277,56 +57,57 +58,58 +59,59 +60,60 +61,61 +62,62 +63,63 +64,64 +65,65 +66,66 +67,67 +68,68 +69,69 +70,70 +71,71 +72,72 +73,73 +74,74 +75,75 +76,76 +77,77 +78,78 +29424,78 +79,79 +80,80 +30970,80 +5513,80 +81,81 +82,82 +83,83 +84,84 +85,85 +86,86 +87,87 +140104,87 +88,88 +89,89 +90,90 +91,91 +92,92 +93,93 +94,94 +95,95 +96,96 +97,97 +181288,97 +98,98 +99,99 +100,100 +101,101 +102,102 +103,103 +140707,103 +104,104 +239414,104 +105,105 +106,106 +107,107 +108,108 +58132,108 +109,109 +110,110 +111,111 +112,112 +113,113 +20650,113 +19944,113 +114,114 +115,115 +116,116 +117,117 +118,118 +187364,118 +119,119 +111242,119 +120,120 +11626,120 +5129,120 +121,121 +122,122 +123,123 +124,124 +125,125 +126,126 +127,127 +128,128 +129,129 +130,130 +131,131 +132,132 +133,133 +134,134 +135,135 +136,136 +137,137 +138,138 +139,139 +140,140 +141,141 +142,142 +143,143 +144,144 +145,145 +146,146 +147,147 +148,148 +149,149 +150,150 +151,151 +152,152 +153,153 +154,154 +155,155 +156,156 +157,157 +158,158 +159,159 +160,160 +161,161 +162,162 +163,163 +164,164 +165,165 +140855,165 +241519,165 +166,166 +167,167 +168,168 +123288,168 +169,169 +170,170 +171,171 +172,172 +173,173 +174,174 +175,175 +36731,175 +176,176 +177,177 +178,178 +179,179 +180,180 +181,181 +182,182 +248168,182 +183,183 +184,184 +185,185 +186,186 +187,187 +188,188 +189,189 +190,190 +191,191 +192,192 +193,193 +53888,193 +45784,193 +194,194 +216796,194 +195,195 +196,196 +197,197 +198,198 +199,199 +200,200 +201,201 +202,202 +140697,202 +203,203 +204,204 +205,205 +4567,205 +206,206 +207,207 +208,208 +209,209 +210,210 +211,211 +212,212 +213,213 +214,214 +215,215 +97432,216 +216,216 +217,217 +218,218 +219,219 +220,220 +151500,220 +221,221 +222,222 +223,223 +224,224 +225,225 +226,226 +227,227 +228,228 +229,229 +187418,229 +230,230 +231,231 +232,232 +233,233 +234,234 +235,235 +236,236 +237,237 +238,238 +239,239 +240,240 +241,241 +242,242 +243,243 +184353,243 +244,244 +245,245 +246,246 +114169,247 +247,247 +248,248 +249,249 +250,250 +251,251 +252,252 +253,253 +93939,253 +254,254 +87614,254 +255,255 +27127,255 +256,256 +121567,256 +257,257 +5655,257 +258,258 +259,259 +236564,259 +260,260 +77847,260 +53762,260 +45658,260 +261,261 +262,262 +263,263 +226664,263 +264,264 +265,265 +266,266 +267,267 +268,268 +269,269 +76403,269 +270,270 +271,271 +88042,271 +272,272 +273,273 +274,274 +275,275 +88121,275 +276,276 +157863,276 +277,277 +278,278 +279,279 +280,280 +281,281 +282,282 +283,283 +284,284 +285,285 +286,286 +287,287 +288,288 +289,289 +290,290 +291,291 +292,292 +293,293 +294,294 +295,295 +296,296 +297,297 +298,298 +299,299 +300,300 +301,301 +302,302 +303,303 +304,304 +305,305 +306,306 +307,307 +308,308 +143170,309 +309,309 +310,310 +24973,310 +311,311 +312,312 +313,313 +314,314 +315,315 +316,316 +317,317 +318,318 +319,319 +320,320 +321,321 +322,322 +366,366 +367,367 +368,368 +369,369 +370,370 +32,370 +371,371 +372,372 +373,373 +141319,373 +374,374 +375,375 +376,376 +74215,376 +377,377 +378,378 +379,379 +20139,379 +380,380 +381,381 +382,382 +383,383 +384,384 +385,385 +386,386 +156636,386 +210244,386 +387,387 +388,388 +15585,388 +389,389 +390,390 +70548,390 +391,391 +392,392 +19023,392 +393,393 +394,394 +395,395 +396,396 +397,397 +3145,397 +398,398 +399,399 +3042,399 +400,400 +401,401 +402,402 +403,403 +404,404 +64039,404 +405,405 +406,406 +407,407 +408,408 +111448,408 +409,409 +410,410 +60398,410 +411,411 +412,412 +413,413 +414,414 +415,415 +416,416 +198034,416 +417,417 +418,418 +419,419 +420,420 +421,421 +422,422 +423,423 +123076,423 +424,424 +425,425 +426,426 +427,427 +428,428 +429,429 +430,430 +431,431 +432,432 +433,433 +434,434 +435,435 +436,436 +437,437 +438,438 +439,439 +440,440 +441,441 +442,442 +443,443 +444,444 +445,445 +446,446 +447,447 +448,448 +487,487 +488,488 +5034,488 +489,489 +490,490 +491,491 +492,492 +493,493 +494,494 +495,495 +496,496 +497,497 +5943,497 +498,498 +499,499 +500,500 +501,501 +502,502 +503,503 +504,504 +505,505 +506,506 +507,507 +508,508 +35117,508 +509,509 +510,510 +511,511 +512,512 +513,513 +514,514 +515,515 +516,516 +517,517 +518,518 +519,519 +520,520 +521,521 +522,522 +523,523 +245015,523 +524,524 +560,560 +561,561 +562,562 +563,563 +564,564 +565,565 +566,566 +567,567 +568,568 +569,569 +570,570 +571,571 +36743,571 +572,572 +185286,572 +573,573 +574,574 +179560,574 +575,575 +33360,575 +106721,576 +576,576 +577,577 +578,578 +87136,578 +579,579 +580,580 +581,581 +582,582 +583,583 +584,584 +585,585 +50132,585 +41939,585 +586,586 +587,587 +588,588 +589,589 +590,590 +146910,590 +591,591 +592,592 +593,593 +594,594 +595,595 +103391,595 +596,596 +597,597 +598,598 +599,599 +142281,600 +600,600 +145261,600 +601,601 +602,602 +603,603 +604,604 +605,605 +606,606 +607,607 +608,608 +609,609 +610,610 +611,611 +612,612 +613,613 +614,614 +615,615 +616,616 +617,617 +618,618 +114327,618 +619,619 +620,620 +621,621 +622,622 +623,623 +624,624 +625,625 +626,626 +627,627 +628,628 +184707,628 +629,629 +630,630 +631,631 +632,632 +633,633 +634,634 +635,635 +636,636 +637,637 +638,638 +639,639 +640,640 +641,641 +642,642 +643,643 +644,644 +645,645 +60231,645 +646,646 +647,647 +648,648 +649,649 +650,650 +651,651 +652,652 +243499,652 +653,653 +654,654 +655,655 +202892,655 +656,656 +657,657 +658,658 +63701,658 +659,659 +660,660 +661,661 +662,662 +663,663 +664,664 +665,665 +666,666 +667,667 +668,668 +669,669 +36161,669 +670,670 +671,671 +672,672 +673,673 +674,674 +675,675 +676,676 +677,677 +678,678 +679,679 +680,680 +162868,680 +681,681 +682,682 +683,683 +684,684 +136048,684 +128461,684 +685,685 +686,686 +687,687 +688,688 +689,689 +690,690 +739,691 +691,691 +692,692 +693,693 +694,694 +695,695 +696,696 +697,697 +735,735 +736,736 +737,737 +738,738 +739,739 +691,739 +740,740 +741,741 +742,742 +743,743 +744,744 +745,745 +902,902 +903,903 +904,904 +905,905 +906,906 +907,907 +908,908 +139071,908 +909,909 +910,910 +911,911 +912,912 +913,913 +914,914 +915,915 +916,916 +917,917 +918,918 +116341,918 +919,919 +920,920 +921,921 +922,922 +923,923 +924,924 +925,925 +926,926 +927,927 +928,928 +929,929 +930,930 +931,931 +932,932 +103631,932 +164452,932 +933,933 +146112,933 +934,934 +142592,934 +935,935 +936,936 +937,937 +938,938 +939,939 +940,940 +941,941 +242090,941 +942,942 +140382,942 +943,943 +944,944 +945,945 +2453,945 +946,946 +947,947 +948,948 +949,949 +950,950 +951,951 +187091,951 +952,952 +953,953 +954,954 +7099,954 +955,955 +956,956 +957,957 +958,958 +959,959 +960,960 +961,961 +46,961 +13250,961 +962,962 +963,963 +194105,963 +964,964 +965,965 +108603,965 +966,966 +967,967 +968,968 +194447,968 +969,969 +970,970 +971,971 +972,972 +973,973 +974,974 +975,975 +976,976 +977,977 +978,978 +979,979 +980,980 +981,981 +982,982 +983,983 +984,984 +985,985 +986,986 +241772,986 +987,987 +988,988 +989,989 +990,990 +5654,990 +991,991 +992,992 +33232,992 +993,993 +994,994 +36093,994 +995,995 +996,996 +997,997 +1036,1036 +1037,1037 +1038,1038 +1039,1039 +1040,1040 +1041,1041 +1042,1042 +1043,1043 +1044,1044 +1045,1045 +1046,1046 +1047,1047 +1048,1048 +1049,1049 +1050,1050 +1051,1051 +30956,1051 +1052,1052 +1053,1053 +3312,1053 +1054,1054 +1055,1055 +1056,1056 +1057,1057 +1058,1058 +1059,1059 +1060,1060 +1061,1061 +1062,1062 +1063,1063 +1064,1064 +137535,1064 +129948,1064 +1065,1065 +1066,1066 +1067,1067 +207933,1067 +1068,1068 +1069,1069 +1070,1070 +1071,1071 +1072,1072 +1073,1073 +1074,1074 +1075,1075 +1076,1076 +87377,1076 +1077,1077 +1078,1078 +1079,1079 +1080,1080 +1081,1081 +1082,1082 +1083,1083 +1084,1084 +1085,1085 +1086,1086 +1087,1087 +1088,1088 +1089,1089 +1090,1090 +1091,1091 +1092,1092 +1093,1093 +1094,1094 +1095,1095 +1096,1096 +1097,1097 +1098,1098 +1099,1099 +1100,1100 +1101,1101 +1102,1102 +1103,1103 +234896,1103 +1104,1104 +1105,1105 +1106,1106 +1107,1107 +1108,1108 +201075,1109 +1109,1109 +1110,1110 +1111,1111 +1112,1112 +1113,1113 +1114,1114 +1115,1115 +1116,1116 +1954,1954 +1955,1955 +1956,1956 +1957,1957 +1958,1958 +1959,1959 +1960,1960 +151120,1960 +1961,1961 +230398,1961 +70200,1961 +1962,1962 +1963,1963 +1964,1964 +200979,1964 +1965,1965 +2003,2003 +2004,2004 +2005,2005 +2006,2006 +2007,2007 +2008,2008 +2009,2009 +2010,2010 +2011,2011 +2012,2012 +2013,2013 +2014,2014 +2015,2015 +183422,2015 +2016,2016 +2017,2017 +2018,2018 +68294,2018 +2019,2019 +2020,2020 +2021,2021 +2022,2022 +165199,2023 +2023,2023 +248014,2023 +2024,2024 +2025,2025 +2026,2026 +2027,2027 +2028,2028 +2029,2029 +2030,2030 +27115,2030 +2031,2031 +2032,2032 +2033,2033 +2034,2034 +2035,2035 +2036,2036 +166725,2036 +2037,2037 +2038,2038 +2039,2039 +2040,2040 +2041,2041 +57799,2041 +2042,2042 +2043,2043 +14639,2043 +2044,2044 +2045,2045 +2046,2046 +2047,2047 +2048,2048 +2049,2049 +2050,2050 +85785,2050 +2051,2051 +2052,2052 +2053,2053 +135272,2053 +127685,2053 +2054,2054 +2055,2055 +2056,2056 +2057,2057 +2058,2058 +238002,2058 +2059,2059 +2060,2060 +2061,2061 +2062,2062 +2063,2063 +2064,2064 +156474,2064 +2065,2065 +2066,2066 +2067,2067 +2068,2068 +2069,2069 +2070,2070 +2071,2071 +2072,2072 +2073,2073 +2074,2074 +2075,2075 +2076,2076 +2077,2077 +2078,2078 +160458,2078 +4799,2078 +2079,2079 +2080,2080 +2081,2081 +2082,2082 +2083,2083 +2084,2084 +2085,2085 +2086,2086 +2087,2087 +2088,2088 +2089,2089 +2090,2090 +2091,2091 +2092,2092 +2093,2093 +2094,2094 +20156,2094 +2095,2095 +2096,2096 +2097,2097 +2098,2098 +2099,2099 +2100,2100 +2101,2101 +2102,2102 +2103,2103 +2104,2104 +2105,2105 +2106,2106 +2107,2107 +2108,2108 +2109,2109 +2110,2110 +2111,2111 +2112,2112 +2113,2113 +2114,2114 +2115,2115 +2116,2116 +2117,2117 +2118,2118 +2119,2119 +55406,2119 +47302,2119 +2120,2120 +2121,2121 +2122,2122 +2123,2123 +2124,2124 +2125,2125 +2126,2126 +2127,2127 +2128,2128 +2129,2129 +2130,2130 +2131,2131 +2132,2132 +2133,2133 +147079,2133 +39022,2134 +2134,2134 +2135,2135 +2136,2136 +2137,2137 +2138,2138 +2139,2139 +2140,2140 +2141,2141 +2142,2142 +2143,2143 +2144,2144 +2145,2145 +2146,2146 +2147,2147 +2148,2148 +2149,2149 +2150,2150 +2151,2151 +2152,2152 +2153,2153 +2154,2154 +2155,2155 +2156,2156 +2157,2157 +2158,2158 +2159,2159 +14506,2159 +2160,2160 +2161,2161 +247953,2161 +2162,2162 +2163,2163 +2164,2164 +202890,2164 +2165,2165 +2166,2166 +2167,2167 +2168,2168 +2169,2169 +2170,2170 +2171,2171 +2172,2172 +2173,2173 +2174,2174 +2175,2175 +2176,2176 +2177,2177 +2178,2178 +2179,2179 +2180,2180 +114245,2180 +93402,2180 +2181,2181 +2182,2182 +2183,2183 +2184,2184 +2185,2185 +195180,2185 +2186,2186 +2187,2187 +2188,2188 +2189,2189 +2190,2190 +2191,2191 +2192,2192 +2193,2193 +2194,2194 +2195,2195 +2196,2196 +2197,2197 +2198,2198 +2199,2199 +13494,2199 +2200,2200 +2201,2201 +2202,2202 +2203,2203 +2204,2204 +2205,2205 +2206,2206 +85583,2206 +25048,2206 +2207,2207 +206577,2207 +6431,2207 +2208,2208 +132712,2208 +125125,2208 +2209,2209 +2210,2210 +2211,2211 +2212,2212 +2213,2213 +2214,2214 +52322,2214 +44218,2214 +2215,2215 +2216,2216 +148444,2216 +2217,2217 +2218,2218 +5468,2218 +2219,2219 +2220,2220 +2221,2221 +2222,2222 +219929,2222 +2223,2223 +2224,2224 +2225,2225 +2226,2226 +2227,2227 +2228,2228 +2229,2229 +2230,2230 +2231,2231 +2232,2232 +2233,2233 +2234,2234 +2235,2235 +2236,2236 +2237,2237 +2238,2238 +2239,2239 +2240,2240 +2241,2241 +2242,2242 +106526,2242 +2243,2243 +2244,2244 +2245,2245 +2246,2246 +2247,2247 +2248,2248 +2249,2249 +2250,2250 +2251,2251 +2252,2252 +2253,2253 +2254,2254 +2255,2255 +2256,2256 +2296,2296 +2297,2297 +2298,2298 +2299,2299 +2300,2300 +2301,2301 +2302,2302 +2303,2303 +59151,2303 +58777,2303 +2304,2304 +31822,2304 +83331,2304 +2305,2305 +2306,2306 +146443,2306 +2307,2307 +2308,2308 +2309,2309 +2310,2310 +2311,2311 +2312,2312 +145784,2312 +2313,2313 +2314,2314 +2315,2315 +2316,2316 +2317,2317 +9008,2317 +2318,2318 +2319,2319 +2320,2320 +2321,2321 +65157,2321 +2322,2322 +2323,2323 +2324,2324 +2325,2325 +2326,2326 +2327,2327 +2328,2328 +2329,2329 +2330,2330 +2331,2331 +2332,2332 +149404,2332 +2333,2333 +2334,2334 +2335,2335 +2336,2336 +2337,2337 +2338,2338 +2339,2339 +2340,2340 +2341,2341 +2342,2342 +2343,2343 +2344,2344 +89625,2344 +2345,2345 +59136,2345 +58762,2345 +2346,2346 +2347,2347 +2348,2348 +248131,2348 +2349,2349 +2350,2350 +2351,2351 +11514,2351 +2352,2352 +2353,2353 +2354,2354 +2355,2355 +2356,2356 +2357,2357 +2358,2358 +2359,2359 +2360,2360 +2361,2361 +2362,2362 +2363,2363 +36184,2363 +2401,2401 +2402,2402 +2403,2403 +2404,2404 +2405,2405 +2406,2406 +2407,2407 +89708,2407 +2408,2408 +2409,2409 +2410,2410 +2411,2411 +2412,2412 +2413,2413 +201339,2413 +2414,2414 +2415,2415 +152894,2415 +2416,2416 +2417,2417 +2418,2418 +2419,2419 +210260,2419 +2420,2420 +2421,2421 +2422,2422 +2423,2423 +2424,2424 +2425,2425 +2426,2426 +2427,2427 +2428,2428 +2429,2429 +2430,2430 +2431,2431 +2432,2432 +77544,2432 +2433,2433 +2434,2434 +2435,2435 +2436,2436 +2437,2437 +2438,2438 +2439,2439 +2440,2440 +2441,2441 +2442,2442 +2443,2443 +133193,2443 +125606,2443 +59375,2443 +2444,2444 +2445,2445 +133195,2445 +125608,2445 +2446,2446 +2447,2447 +2448,2448 +2449,2449 +211604,2449 +135070,2449 +127483,2449 +2450,2450 +2451,2451 +2452,2452 +2453,2453 +945,2453 +2454,2454 +2455,2455 +2456,2456 +2457,2457 +2458,2458 +2459,2459 +2460,2460 +6295,2460 +2461,2461 +109126,2461 +109127,2462 +2462,2462 +2463,2463 +2464,2464 +2465,2465 +2466,2466 +2467,2467 +2468,2468 +2469,2469 +2470,2470 +2471,2471 +2472,2472 +2473,2473 +2474,2474 +60656,2474 +2475,2475 +2476,2476 +2477,2477 +2478,2478 +2479,2479 +2480,2480 +2481,2481 +197539,2481 +2482,2482 +2483,2483 +70251,2483 +2484,2484 +2485,2485 +179538,2485 +2486,2486 +2487,2487 +2488,2488 +150116,2488 +2489,2489 +2490,2490 +2491,2491 +2492,2492 +2493,2493 +2494,2494 +2495,2495 +2496,2496 +2497,2497 +2498,2498 +2499,2499 +2500,2500 +2501,2501 +2502,2502 +2503,2503 +2504,2504 +2505,2505 +2506,2506 +83735,2506 +2507,2507 +2508,2508 +2509,2509 +2510,2510 +2511,2511 +2512,2512 +2513,2513 +2514,2514 +2515,2515 +2516,2516 +2517,2517 +2518,2518 +2519,2519 +2520,2520 +2521,2521 +2522,2522 +2523,2523 +2524,2524 +2525,2525 +2526,2526 +2527,2527 +2528,2528 +2529,2529 +2530,2530 +2531,2531 +2532,2532 +2533,2533 +2534,2534 +2535,2535 +2536,2536 +2537,2537 +2538,2538 +2539,2539 +2540,2540 +2541,2541 +2542,2542 +166610,2542 +2543,2543 +2544,2544 +2545,2545 +2546,2546 +2547,2547 +2548,2548 +2549,2549 +2550,2550 +2551,2551 +2552,2552 +2553,2553 +2554,2554 +2555,2555 +2556,2556 +2625,2625 +2626,2626 +2627,2627 +2628,2628 +2629,2629 +2630,2630 +2631,2631 +2632,2632 +2633,2633 +2634,2634 +2635,2635 +2636,2636 +2637,2637 +2638,2638 +2639,2639 +2640,2640 +2641,2641 +2642,2642 +2643,2643 +2644,2644 +2645,2645 +2646,2646 +2647,2647 +2648,2648 +2649,2649 +2650,2650 +2651,2651 +2652,2652 +2653,2653 +2654,2654 +2655,2655 +2656,2656 +2657,2657 +2658,2658 +2659,2659 +2660,2660 +2661,2661 +2662,2662 +2663,2663 +2664,2664 +2665,2665 +244355,2666 +2666,2666 +2667,2667 +2668,2668 +2669,2669 +2670,2670 +2671,2671 +24030,2671 +2672,2672 +2673,2673 +2674,2674 +2675,2675 +201340,2675 +2676,2676 +60064,2676 +2677,2677 +39731,2677 +2678,2678 +57788,2678 +2679,2679 +2680,2680 +5811,2680 +2681,2681 +143911,2681 +2682,2682 +2683,2683 +65747,2683 +2684,2684 +2685,2685 +7601,2685 +2686,2686 +2687,2687 +2688,2688 +2689,2689 +2690,2690 +2691,2691 +182321,2691 +155613,2691 +2692,2692 +2693,2693 +2694,2694 +220950,2694 +220618,2694 +2695,2695 +2696,2696 +2697,2697 +2698,2698 +2699,2699 +2700,2700 +2701,2701 +2702,2702 +83031,2702 +2703,2703 +9755,2703 +2704,2704 +52641,2704 +44537,2704 +2705,2705 +2706,2706 +2707,2707 +2708,2708 +2709,2709 +108524,2709 +2710,2710 +2711,2711 +2712,2712 +142602,2712 +53690,2712 +45586,2712 +2713,2713 +2714,2714 +2715,2715 +2716,2716 +191177,2716 +2717,2717 +2718,2718 +2719,2719 +2720,2720 +2721,2721 +2722,2722 +2723,2723 +2724,2724 +2725,2725 +2726,2726 +2727,2727 +2728,2728 +2729,2729 +2730,2730 +2731,2731 +2732,2732 +120314,2732 +2733,2733 +2734,2734 +2735,2735 +2736,2736 +2737,2737 +2738,2738 +2739,2739 +2740,2740 +2741,2741 +2742,2742 +2743,2743 +2744,2744 +242602,2744 +2745,2745 +4664,2745 +217829,2745 +2746,2746 +2747,2747 +2748,2748 +2749,2749 +2750,2750 +2751,2751 +79432,2751 +2752,2752 +78917,2752 +2753,2753 +56544,2753 +48440,2753 +2754,2754 +2755,2755 +2756,2756 +2757,2757 +2758,2758 +2759,2759 +120689,2760 +2760,2760 +2761,2761 +5637,2761 +2762,2762 +2763,2763 +142486,2763 +243316,2763 +2764,2764 +2765,2765 +2766,2766 +2767,2767 +2768,2768 +2769,2769 +191873,2769 +2770,2770 +2771,2771 +2772,2772 +2773,2773 +2774,2774 +2775,2775 +2776,2776 +2777,2777 +2778,2778 +2779,2779 +2780,2780 +2781,2781 +2782,2782 +2783,2783 +216261,2783 +2784,2784 +2785,2785 +2786,2786 +2787,2787 +2788,2788 +2789,2789 +2790,2790 +2791,2791 +2792,2792 +2793,2793 +156385,2793 +2794,2794 +2795,2795 +216587,2795 +2796,2796 +2797,2797 +2798,2798 +2799,2799 +2800,2800 +2801,2801 +2802,2802 +2803,2803 +2804,2804 +2805,2805 +2806,2806 +2807,2807 +2808,2808 +2809,2809 +2810,2810 +2811,2811 +2812,2812 +2813,2813 +2814,2814 +2815,2815 +2816,2816 +77335,2816 +2817,2817 +2818,2818 +8070,2818 +2819,2819 +2820,2820 +2821,2821 +2822,2822 +201758,2823 +2823,2823 +2824,2824 +2825,2825 +2826,2826 +2827,2827 +2828,2828 +2829,2829 +2830,2830 +2831,2831 +2832,2832 +2833,2833 +115962,2833 +9263,2833 +2834,2834 +2835,2835 +2836,2836 +199265,2836 +2837,2837 +2838,2838 +2839,2839 +2840,2840 +2841,2841 +2842,2842 +2843,2843 +2844,2844 +147537,2844 +2845,2845 +2846,2846 +2847,2847 +2848,2848 +2849,2849 +2850,2850 +2851,2851 +2852,2852 +121384,2852 +2853,2853 +2854,2854 +2855,2855 +161773,2855 +2856,2856 +2857,2857 +2858,2858 +2859,2859 +2860,2860 +2861,2861 +2862,2862 +2863,2863 +2864,2864 +2865,2865 +2866,2866 +2867,2867 +2868,2868 +2869,2869 +2870,2870 +2871,2871 +225556,2871 +161743,2872 +2872,2872 +2873,2873 +229802,2873 +2874,2874 +2875,2875 +2876,2876 +2877,2877 +2878,2878 +2879,2879 +2880,2880 +2881,2881 +2882,2882 +2883,2883 +2884,2884 +2885,2885 +2886,2886 +2887,2887 +2888,2888 +2889,2889 +2890,2890 +2891,2891 +2892,2892 +2893,2893 +2894,2894 +2895,2895 +2896,2896 +90686,2896 +2897,2897 +2898,2898 +2899,2899 +2900,2900 +2901,2901 +218671,2901 +2902,2902 +2903,2903 +2904,2904 +2905,2905 +2906,2906 +2907,2907 +2908,2908 +96037,2908 +2909,2909 +160971,2909 +2910,2910 +2911,2911 +2912,2912 +2913,2913 +2914,2914 +2915,2915 +2916,2916 +2917,2917 +2918,2918 +2919,2919 +2920,2920 +2921,2921 +2922,2922 +185943,2922 +2923,2923 +2924,2924 +2925,2925 +2926,2926 +2927,2927 +2928,2928 +2929,2929 +2930,2930 +2931,2931 +2932,2932 +2933,2933 +2934,2934 +2935,2935 +2936,2936 +2937,2937 +2938,2938 +2939,2939 +2940,2940 +2941,2941 +2942,2942 +2943,2943 +2944,2944 +2945,2945 +2946,2946 +2947,2947 +2948,2948 +2949,2949 +2950,2950 +2951,2951 +2952,2952 +2953,2953 +2954,2954 +49512,2954 +41319,2954 +2955,2955 +2956,2956 +233363,2956 +26231,2956 +2957,2957 +2958,2958 +2959,2959 +2960,2960 +2961,2961 +2962,2962 +2963,2963 +2964,2964 +2965,2965 +106959,2965 +2966,2966 +2967,2967 +2968,2968 +2969,2969 +2970,2970 +2971,2971 +2972,2972 +2973,2973 +2974,2974 +2975,2975 +2976,2976 +3023,3023 +3024,3024 +3025,3025 +3026,3026 +3027,3027 +3028,3028 +3029,3029 +3030,3030 +3031,3031 +3032,3032 +3033,3033 +3034,3034 +3035,3035 +3036,3036 +3037,3037 +3038,3038 +3039,3039 +3040,3040 +3041,3041 +135928,3041 +128341,3041 +3042,3042 +399,3042 +3043,3043 +3044,3044 +3045,3045 +3046,3046 +3047,3047 +3048,3048 +3049,3049 +3050,3050 +3051,3051 +202052,3051 +3052,3052 +3053,3053 +3054,3054 +3055,3055 +3056,3056 +3057,3057 +3058,3058 +3059,3059 +3060,3060 +3061,3061 +3062,3062 +3063,3063 +3064,3064 +3065,3065 +3066,3066 +3067,3067 +3068,3068 +3069,3069 +3070,3070 +201677,3071 +3071,3071 +3072,3072 +3073,3073 +3074,3074 +3075,3075 +3076,3076 +3077,3077 +3078,3078 +3079,3079 +3080,3080 +3081,3081 +3082,3082 +3083,3083 +3084,3084 +3085,3085 +3086,3086 +3087,3087 +3088,3088 +3089,3089 +3090,3090 +10156,3090 +3091,3091 +3092,3092 +3093,3093 +3094,3094 +3095,3095 +3096,3096 +3097,3097 +3098,3098 +3099,3099 +207261,3100 +3100,3100 +142235,3100 +145206,3100 +3101,3101 +3102,3102 +143137,3102 +141549,3102 +3103,3103 +26262,3103 +3104,3104 +3105,3105 +3106,3106 +3107,3107 +3108,3108 +3109,3109 +3110,3110 +3111,3111 +3112,3112 +3113,3113 +3114,3114 +3115,3115 +3116,3116 +3117,3117 +3118,3118 +3119,3119 +3120,3120 +3121,3121 +3122,3122 +123111,3122 +3123,3123 +3124,3124 +3125,3125 +3126,3126 +3127,3127 +3128,3128 +3129,3129 +79068,3129 +3130,3130 +3131,3131 +3132,3132 +3133,3133 +3134,3134 +137155,3134 +129568,3134 +3135,3135 +3136,3136 +3137,3137 +3138,3138 +189884,3138 +3139,3139 +3140,3140 +3141,3141 +3142,3142 +3143,3143 +3144,3144 +3145,3145 +397,3145 +3146,3146 +3147,3147 +3148,3148 +64309,3149 +3149,3149 +3150,3150 +3151,3151 +3152,3152 +3153,3153 +3154,3154 +3155,3155 +3156,3156 +3157,3157 +3158,3158 +3159,3159 +3160,3160 +3161,3161 +3162,3162 +3163,3163 +213851,3164 +3164,3164 +3165,3165 +3166,3166 +3167,3167 +3168,3168 +3169,3169 +3170,3170 +3171,3171 +3172,3172 +3173,3173 +19032,3173 +3174,3174 +3175,3175 +3176,3176 +194026,3176 +3177,3177 +3178,3178 +3179,3179 +3180,3180 +3181,3181 +3182,3182 +3183,3183 +3184,3184 +3185,3185 +3186,3186 +58036,3186 +3187,3187 +3188,3188 +3189,3189 +3190,3190 +3191,3191 +3192,3192 +3193,3193 +3194,3194 +3195,3195 +3196,3196 +3197,3197 +3198,3198 +184446,3198 +3199,3199 +3200,3200 +3201,3201 +3202,3202 +3203,3203 +3204,3204 +3205,3205 +3206,3206 +3207,3207 +3208,3208 +210213,3208 +3209,3209 +3210,3210 +3211,3211 +3212,3212 +3213,3213 +3214,3214 +3215,3215 +3216,3216 +3217,3217 +3218,3218 +3219,3219 +3220,3220 +3221,3221 +3222,3222 +3223,3223 +3224,3224 +3225,3225 +3226,3226 +3227,3227 +3228,3228 +3229,3229 +3230,3230 +3231,3231 +3232,3232 +3233,3233 +3234,3234 +28370,3234 +3235,3235 +3236,3236 +3237,3237 +3238,3238 +3239,3239 +3240,3240 +3241,3241 +3242,3242 +3243,3243 +3244,3244 +3245,3245 +3246,3246 +3247,3247 +3248,3248 +3249,3249 +3250,3250 +3251,3251 +3252,3252 +3253,3253 +3254,3254 +3255,3255 +3256,3256 +3257,3257 +3258,3258 +3259,3259 +3260,3260 +3261,3261 +3262,3262 +80199,3262 +3263,3263 +3264,3264 +3265,3265 +3266,3266 +3267,3267 +3268,3268 +3269,3269 +3270,3270 +3271,3271 +3272,3272 +3273,3273 +3274,3274 +163084,3274 +3275,3275 +3276,3276 +141470,3276 +3277,3277 +3278,3278 +3279,3279 +3280,3280 +3281,3281 +205275,3281 +3282,3282 +3283,3283 +3284,3284 +3285,3285 +3286,3286 +3287,3287 +3288,3288 +3289,3289 +3290,3290 +3291,3291 +217736,3291 +3292,3292 +3293,3293 +3294,3294 +3295,3295 +3296,3296 +3297,3297 +3298,3298 +3299,3299 +3300,3300 +3301,3301 +3302,3302 +3303,3303 +3304,3304 +3305,3305 +3306,3306 +3307,3307 +3308,3308 +3309,3309 +3310,3310 +3311,3311 +3312,3312 +1053,3312 +3313,3313 +3314,3314 +85806,3314 +3315,3315 +3316,3316 +3317,3317 +3318,3318 +3319,3319 +3320,3320 +3321,3321 +3322,3322 +3323,3323 +3474,3474 +3475,3475 +3476,3476 +3477,3477 +3478,3478 +3479,3479 +64872,3479 +3480,3480 +3481,3481 +3482,3482 +3483,3483 +3484,3484 +3485,3485 +3486,3486 +3487,3487 +3488,3488 +3489,3489 +3490,3490 +3491,3491 +195518,3492 +3492,3492 +3493,3493 +3494,3494 +3495,3495 +3496,3496 +3497,3497 +3498,3498 +3499,3499 +3500,3500 +180100,3500 +3501,3501 +23889,3501 +3502,3502 +3503,3503 +3504,3504 +3505,3505 +3506,3506 +3507,3507 +6573,3508 +3508,3508 +3509,3509 +3510,3510 +158078,3510 +3511,3511 +3512,3512 +3513,3513 +10807,3513 +142365,3513 +3514,3514 +3515,3515 +3516,3516 +3517,3517 +3518,3518 +3519,3519 +92666,3519 +3520,3520 +3521,3521 +3522,3522 +3523,3523 +3524,3524 +3525,3525 +3526,3526 +20097,3526 +56737,3526 +49824,3526 +41631,3526 +3527,3527 +3528,3528 +3529,3529 +3530,3530 +17231,3530 +3531,3531 +3532,3532 +3533,3533 +3534,3534 +191511,3534 +3535,3535 +3536,3536 +3537,3537 +135263,3537 +127676,3537 +3538,3538 +3539,3539 +3540,3540 +3541,3541 +3542,3542 +3543,3543 +3544,3544 +3545,3545 +3546,3546 +3547,3547 +3548,3548 +3549,3549 +3550,3550 +3551,3551 +3552,3552 +3553,3553 +3554,3554 +3555,3555 +3556,3556 +3557,3557 +3558,3558 +26310,3558 +3559,3559 +3560,3560 +3561,3561 +3562,3562 +3563,3563 +3564,3564 +11469,3564 +3565,3565 +3566,3566 +3567,3567 +3568,3568 +3569,3569 +3570,3570 +3571,3571 +3572,3572 +3573,3573 +3574,3574 +3575,3575 +66820,3575 +3576,3576 +3577,3577 +3578,3578 +3579,3579 +3580,3580 +3581,3581 +3582,3582 +141699,3582 +3583,3583 +3584,3584 +3585,3585 +3586,3586 +3587,3587 +164901,3587 +118432,3587 +3588,3588 +81446,3588 +37014,3588 +3589,3589 +3590,3590 +3591,3591 +3592,3592 +3593,3593 +3594,3594 +3595,3595 +3596,3596 +3597,3597 +3598,3598 +3599,3599 +3600,3600 +3664,3664 +3665,3665 +3666,3666 +3667,3667 +142673,3667 +3668,3668 +3669,3669 +3670,3670 +3671,3671 +3672,3672 +3673,3673 +3674,3674 +3675,3675 +3676,3676 +223480,3676 +3677,3677 +3678,3678 +3679,3679 +3680,3680 +3681,3681 +3682,3682 +3683,3683 +3684,3684 +3685,3685 +3686,3686 +3687,3687 +3688,3688 +3689,3689 +3690,3690 +3691,3691 +231932,3692 +3692,3692 +3693,3693 +3694,3694 +3695,3695 +3696,3696 +3697,3697 +3698,3698 +50805,3698 +42701,3698 +3699,3699 +3700,3700 +3701,3701 +3702,3702 +3703,3703 +3704,3704 +3705,3705 +3706,3706 +3707,3707 +3708,3708 +3709,3709 +3710,3710 +3711,3711 +3712,3712 +3713,3713 +160676,3713 +3714,3714 +3715,3715 +3716,3716 +3717,3717 +3718,3718 +3719,3719 +3720,3720 +3721,3721 +96453,3722 +3722,3722 +3723,3723 +229569,3723 +3724,3724 +3725,3725 +3726,3726 +3727,3727 +3728,3728 +3729,3729 +3730,3730 +3731,3731 +10496,3731 +50154,3779 +41962,3779 +3779,3779 +3780,3780 +9004,3780 +3781,3781 +3782,3782 +3783,3783 +3784,3784 +3785,3785 +3786,3786 +3787,3787 +3788,3788 +3789,3789 +3790,3790 +3791,3791 +3792,3792 +3793,3793 +3794,3794 +3795,3795 +3796,3796 +3797,3797 +3798,3798 +3799,3799 +3800,3800 +121429,3842 +3842,3842 +3843,3843 +3844,3844 +3845,3845 +3846,3846 +3847,3847 +3848,3848 +3849,3849 +3850,3850 +3851,3851 +3852,3852 +3853,3853 +3854,3854 +3855,3855 +3856,3856 +3857,3857 +3858,3858 +3859,3859 +3860,3860 +3861,3861 +3862,3862 +3863,3863 +3864,3864 +3865,3865 +3866,3866 +3867,3867 +3868,3868 +3869,3869 +3870,3870 +3871,3871 +3872,3872 +3873,3873 +3874,3874 +3875,3875 +3876,3876 +3877,3877 +3878,3878 +3879,3879 +25837,3879 +136542,3879 +128955,3879 +3880,3880 +221292,3880 +3881,3881 +3882,3882 +3883,3883 +3884,3884 +3885,3885 +3886,3886 +3887,3887 +3888,3888 +3889,3889 +3890,3890 +3891,3891 +3892,3892 +3893,3893 +3894,3894 +3895,3895 +3896,3896 +3897,3897 +3898,3898 +3899,3899 +3900,3900 +3901,3901 +3902,3902 +3903,3903 +3904,3904 +3905,3905 +3906,3906 +3907,3907 +3908,3908 +3909,3909 +3910,3910 +3911,3911 +3912,3912 +3913,3913 +3914,3914 +3915,3915 +3916,3916 +3917,3917 +89194,3917 +3918,3918 +3919,3919 +3920,3920 +3921,3921 +3922,3922 +3923,3923 +3924,3924 +3925,3925 +3926,3926 +3927,3927 +3928,3928 +3929,3929 +3930,3930 +3931,3931 +3932,3932 +3933,3933 +3934,3934 +3935,3935 +3936,3936 +3937,3937 +3938,3938 +3939,3939 +161396,3939 +3940,3940 +3941,3941 +3942,3942 +242616,3942 +3943,3943 +3944,3944 +3945,3945 +3946,3946 +3947,3947 +3948,3948 +3949,3949 +3950,3950 +3951,3951 +3952,3952 +3953,3953 +146848,3953 +3954,3954 +3955,3955 +3956,3956 +3957,3957 +3958,3958 +3959,3959 +3960,3960 +3961,3961 +3962,3962 +3963,3963 +3964,3964 +3965,3965 +3966,3966 +3967,3967 +99218,3967 +6196,3967 +3968,3968 +3969,3969 +3970,3970 +3971,3971 +3972,3972 +3973,3973 +121565,3973 +3974,3974 +235011,3974 +3975,3975 +3976,3976 +3977,3977 +3978,3978 +3979,3979 +3980,3980 +3981,3981 +3982,3982 +3983,3983 +3984,3984 +3985,3985 +3986,3986 +3987,3987 +3988,3988 +3989,3989 +3990,3990 +3991,3991 +3992,3992 +3993,3993 +3994,3994 +3995,3995 +3996,3996 +146713,3996 +3997,3997 +3998,3998 +134188,3999 +126601,3999 +3999,3999 +4000,4000 +243396,4001 +4001,4001 +4002,4002 +26077,4002 +4003,4003 +4004,4004 +4005,4005 +4006,4006 +4007,4007 +4008,4008 +4009,4009 +4010,4010 +4011,4011 +4012,4012 +4013,4013 +4014,4014 +4015,4015 +4016,4016 +4017,4017 +4018,4018 +4019,4019 +4020,4020 +4021,4021 +4022,4022 +4023,4023 +4024,4024 +4025,4025 +4026,4026 +4027,4027 +4028,4028 +4029,4029 +4030,4030 +4031,4031 +4032,4032 +4033,4033 +55029,4033 +46925,4033 +4034,4034 +247960,4034 +136226,4034 +128639,4034 +4035,4035 +4036,4036 +4037,4037 +4038,4038 +4039,4039 +4078,4078 +68299,4078 +4079,4079 +4080,4080 +4081,4081 +4082,4082 +4083,4083 +4084,4084 +4085,4085 +4086,4086 +4087,4087 +4136,4136 +4137,4137 +4138,4138 +4139,4139 +4140,4140 +4141,4141 +4142,4142 +4143,4143 +4144,4144 +4145,4145 +4146,4146 +4147,4147 +4148,4148 +4149,4149 +4150,4150 +4151,4151 +4152,4152 +4153,4153 +4154,4154 +4155,4155 +4156,4156 +12664,4156 +4157,4157 +4158,4158 +4159,4159 +4160,4160 +4161,4161 +4162,4162 +4163,4163 +4164,4164 +4165,4165 +4166,4166 +4167,4167 +4168,4168 +102342,4168 +4169,4169 +159362,4169 +4170,4170 +4171,4171 +4172,4172 +4173,4173 +4174,4174 +4175,4175 +4176,4176 +4177,4177 +4178,4178 +4179,4179 +4180,4180 +4181,4181 +4182,4182 +4183,4183 +111457,4183 +4184,4184 +4185,4185 +4186,4186 +4187,4187 +4188,4188 +35660,4188 +4189,4189 +4190,4190 +4191,4191 +4192,4192 +4193,4193 +4194,4194 +4195,4195 +4196,4196 +220842,4196 +4197,4197 +6383,4197 +4198,4198 +4199,4199 +4200,4200 +4201,4201 +4202,4202 +4203,4203 +4204,4204 +197721,4204 +4205,4205 +4206,4206 +4207,4207 +4208,4208 +11865,4208 +4209,4209 +4210,4210 +4211,4211 +4212,4212 +4213,4213 +7256,4213 +4214,4214 +4215,4215 +4216,4216 +4217,4217 +4218,4218 +4219,4219 +4220,4220 +4221,4221 +148468,4221 +4222,4222 +4223,4223 +29950,4223 +4224,4224 +4225,4225 +4226,4226 +4227,4227 +4228,4228 +4229,4229 +4230,4230 +4231,4231 +4232,4232 +4233,4233 +4274,4274 +4275,4275 +4276,4276 +4277,4277 +4278,4278 +232949,4278 +4279,4279 +4280,4280 +4281,4281 +4282,4282 +4283,4283 +4284,4284 +4285,4285 +4286,4286 +4287,4287 +4288,4288 +143192,4288 +4289,4289 +4290,4290 +4291,4291 +4292,4292 +4293,4293 +4294,4294 +4295,4295 +4296,4296 +202380,4296 +4297,4297 +4298,4298 +4299,4299 +4300,4300 +4301,4301 +229772,4301 +4302,4302 +4303,4303 +4304,4304 +4305,4305 +4306,4306 +4307,4307 +4308,4308 +4309,4309 +4310,4310 +4311,4311 +4312,4312 +7361,4312 +4313,4313 +4314,4314 +4315,4315 +4316,4316 +200868,4316 +4317,4317 +197693,4317 +4318,4318 +4319,4319 +4320,4320 +4321,4321 +4322,4322 +4323,4323 +4324,4324 +4325,4325 +4326,4326 +4327,4327 +4328,4328 +4329,4329 +4330,4330 +4331,4331 +4332,4332 +4333,4333 +4334,4334 +4335,4335 +4336,4336 +183057,4336 +4337,4337 +4338,4338 +4339,4339 +4340,4340 +4341,4341 +4342,4342 +4343,4343 +4344,4344 +4345,4345 +4346,4346 +4347,4347 +4348,4348 +4349,4349 +4350,4350 +4351,4351 +4352,4352 +4353,4353 +4354,4354 +4355,4355 +4356,4356 +4357,4357 +4358,4358 +4359,4359 +4360,4360 +225304,4360 +4361,4361 +4362,4362 +4363,4363 +4364,4364 +4365,4365 +4366,4366 +141296,4366 +4367,4367 +4368,4368 +187372,4369 +4369,4369 +33217,4370 +4370,4370 +4371,4371 +53448,4372 +45344,4372 +4372,4372 +4373,4373 +4374,4374 +4375,4375 +4376,4376 +4377,4377 +4378,4378 +4379,4379 +4380,4380 +4381,4381 +4382,4382 +4383,4383 +4384,4384 +4385,4385 +4386,4386 +4387,4387 +104718,4387 +4468,4468 +4469,4469 +4470,4470 +4471,4471 +4472,4472 +4473,4473 +51100,4473 +42996,4473 +4474,4474 +4475,4475 +4476,4476 +4477,4477 +4478,4478 +4479,4479 +4480,4480 +4481,4481 +4482,4482 +4483,4483 +78579,4483 +4484,4484 +4485,4485 +4486,4486 +4487,4487 +4488,4488 +4489,4489 +4490,4490 +4491,4491 +4492,4492 +4493,4493 +4494,4494 +4495,4495 +195811,4495 +4496,4496 +4497,4497 +97433,4497 +4498,4498 +4499,4499 +4500,4500 +4501,4501 +4502,4502 +190579,4502 +138894,4502 +4503,4503 +4504,4504 +4505,4505 +4506,4506 +4507,4507 +4508,4508 +4509,4509 +4510,4510 +4511,4511 +4512,4512 +4513,4513 +4514,4514 +4515,4515 +4516,4516 +4517,4517 +4518,4518 +4519,4519 +4520,4520 +4521,4521 +4522,4522 +4523,4523 +4524,4524 +4525,4525 +4526,4526 +7071,4526 +4527,4527 +4528,4528 +4529,4529 +4530,4530 +4531,4531 +4532,4532 +4533,4533 +4534,4534 +16265,4534 +4535,4535 +4536,4536 +4537,4537 +4538,4538 +4539,4539 +4540,4540 +4541,4541 +4542,4542 +6424,4542 +4543,4543 +4544,4544 +4545,4545 +4546,4546 +4547,4547 +193574,4547 +4548,4548 +4549,4549 +4550,4550 +4551,4551 +4552,4552 +4553,4553 +4554,4554 +4555,4555 +4556,4556 +4557,4557 +4558,4558 +4559,4559 +142103,4559 +4560,4560 +4561,4561 +4562,4562 +4563,4563 +4564,4564 +4565,4565 +4566,4566 +4567,4567 +205,4567 +4568,4568 +4569,4569 +4570,4570 +4571,4571 +4572,4572 +4573,4573 +4574,4574 +8972,4574 +4575,4575 +4576,4576 +4577,4577 +132966,4577 +125379,4577 +4578,4578 +4579,4579 +4580,4580 +4581,4581 +185442,4581 +4582,4582 +4583,4583 +4584,4584 +4585,4585 +4586,4586 +4587,4587 +6700,4588 +4588,4588 +185438,4588 +4589,4589 +4590,4590 +4591,4591 +4592,4592 +11375,4593 +4593,4593 +4594,4594 +4595,4595 +4596,4596 +4597,4597 +4598,4598 +4599,4599 +4600,4600 +4601,4601 +4602,4602 +4603,4603 +4604,4604 +4605,4605 +4606,4606 +151464,4606 +23546,4606 +4607,4607 +4608,4608 +4609,4609 +4610,4610 +4611,4611 +4612,4612 +4613,4613 +4614,4614 +4615,4615 +4616,4616 +4617,4617 +4618,4618 +4619,4619 +4620,4620 +4621,4621 +4622,4622 +4623,4623 +4624,4624 +4625,4625 +4626,4626 +4627,4627 +4628,4628 +4629,4629 +4630,4630 +4631,4631 +4632,4632 +4633,4633 +4634,4634 +4635,4635 +4636,4636 +4637,4637 +4638,4638 +4639,4639 +4640,4640 +4641,4641 +4642,4642 +145077,4642 +4643,4643 +4644,4644 +4645,4645 +4646,4646 +4647,4647 +4648,4648 +4649,4649 +4650,4650 +4651,4651 +4652,4652 +4653,4653 +65738,4653 +4654,4654 +4655,4655 +4656,4656 +4657,4657 +4658,4658 +4659,4659 +4660,4660 +4661,4661 +4662,4662 +4663,4663 +164521,4663 +4664,4664 +2745,4664 +4665,4665 +4666,4666 +4667,4667 +4668,4668 +249106,4668 +4669,4669 +25124,4669 +83255,4669 +4670,4670 +4671,4671 +4672,4672 +4673,4673 +4674,4674 +4675,4675 +4676,4676 +4677,4677 +4678,4678 +4679,4679 +4680,4680 +39734,4680 +4681,4681 +4682,4682 +4683,4683 +4684,4684 +4685,4685 +4686,4686 +223629,4687 +4687,4687 +4688,4688 +4689,4689 +4690,4690 +63498,4690 +4691,4691 +4692,4692 +4693,4693 +20385,4693 +4694,4694 +4695,4695 +4696,4696 +4697,4697 +4698,4698 +4699,4699 +4700,4700 +4701,4701 +4702,4702 +4703,4703 +4704,4704 +4705,4705 +4706,4706 +4707,4707 +4708,4708 +242049,4708 +4709,4709 +4710,4710 +4711,4711 +4712,4712 +4713,4713 +4714,4714 +4715,4715 +4716,4716 +4717,4717 +4718,4718 +4719,4719 +4720,4720 +4721,4721 +4722,4722 +4723,4723 +4724,4724 +4725,4725 +4726,4726 +4727,4727 +4728,4728 +191105,4728 +4729,4729 +4730,4730 +4731,4731 +4732,4732 +4733,4733 +4734,4734 +5490,4734 +4735,4735 +4736,4736 +4737,4737 +4738,4738 +4739,4739 +4778,4778 +4779,4779 +4780,4780 +165609,4780 +4781,4781 +4782,4782 +4783,4783 +4784,4784 +4785,4785 +4786,4786 +4787,4787 +4788,4788 +4789,4789 +4790,4790 +148266,4790 +4791,4791 +4792,4792 +4793,4793 +4794,4794 +248056,4794 +4795,4795 +4796,4796 +54679,4797 +46575,4797 +4797,4797 +4798,4798 +4799,4799 +2078,4799 +4800,4800 +4801,4801 +4802,4802 +4803,4803 +217174,4804 +4804,4804 +4805,4805 +4806,4806 +4807,4807 +4808,4808 +4809,4809 +4810,4810 +4811,4811 +162184,4811 +4812,4812 +7704,4812 +4813,4813 +4814,4814 +4815,4815 +4816,4816 +4817,4817 +4818,4818 +4819,4819 +36848,4820 +4820,4820 +87786,4820 +4821,4821 +4822,4822 +4823,4823 +4824,4824 +4825,4825 +4826,4826 +213921,4826 +4870,4870 +4871,4871 +4872,4872 +4873,4873 +4874,4874 +4875,4875 +4876,4876 +4877,4877 +4878,4878 +4879,4879 +4880,4880 +4881,4881 +4882,4882 +4883,4883 +4884,4884 +4885,4885 +4886,4886 +4887,4887 +4888,4888 +4889,4889 +4890,4890 +4891,4891 +4892,4892 +4893,4893 +147572,4893 +4894,4894 +4895,4895 +4896,4896 +4897,4897 +4898,4898 +4899,4899 +4900,4900 +4942,4942 +162598,4942 +4943,4943 +216265,4943 +4944,4944 +4945,4945 +4946,4946 +161803,4946 +4947,4947 +4948,4948 +4949,4949 +4950,4950 +4951,4951 +12765,4951 +4952,4952 +4953,4953 +4954,4954 +188193,4954 +4955,4955 +4956,4956 +4957,4957 +4958,4958 +4959,4959 +4960,4960 +4961,4961 +4962,4962 +4963,4963 +4964,4964 +4965,4965 +4966,4966 +4967,4967 +4968,4968 +4969,4969 +4970,4970 +4971,4971 +4972,4972 +4973,4973 +4974,4974 +4975,4975 +4976,4976 +4977,4977 +4978,4978 +4979,4979 +4980,4980 +57571,4980 +4981,4981 +4982,4982 +56261,4983 +48157,4983 +4983,4983 +4984,4984 +4985,4985 +4986,4986 +4987,4987 +4988,4988 +4989,4989 +4990,4990 +109830,4990 +4991,4991 +4992,4992 +4993,4993 +4994,4994 +4995,4995 +7719,4995 +156638,4995 +4996,4996 +4997,4997 +4998,4998 +88129,4999 +4999,4999 +5000,5000 +5001,5001 +5002,5002 +5003,5003 +5004,5004 +5005,5005 +5006,5006 +5007,5007 +5008,5008 +5009,5009 +190054,5009 +5010,5010 +5011,5011 +5012,5012 +5013,5013 +5014,5014 +162518,5014 +5015,5015 +5016,5016 +5017,5017 +5018,5018 +5019,5019 +5020,5020 +5021,5021 +5022,5022 +5023,5023 +5024,5024 +5025,5025 +5026,5026 +5027,5027 +79420,5027 +5028,5028 +5029,5029 +5030,5030 +5031,5031 +5032,5032 +5033,5033 +5034,5034 +488,5034 +5035,5035 +5036,5036 +5037,5037 +5038,5038 +5039,5039 +5040,5040 +5041,5041 +5042,5042 +5043,5043 +5044,5044 +5045,5045 +5046,5046 +5047,5047 +5048,5048 +5049,5049 +5050,5050 +5051,5051 +5052,5052 +5053,5053 +5054,5054 +5055,5055 +5056,5056 +5057,5057 +5058,5058 +5059,5059 +5060,5060 +5061,5061 +5062,5062 +5063,5063 +5064,5064 +5065,5065 +5066,5066 +5067,5067 +5068,5068 +5069,5069 +5070,5070 +5071,5071 +59990,5071 +5072,5072 +5073,5073 +5074,5074 +5075,5075 +5076,5076 +5077,5077 +5078,5078 +5079,5079 +5080,5080 +5081,5081 +5082,5082 +5083,5083 +5084,5084 +5085,5085 +76467,5085 +5086,5086 +5087,5087 +5088,5088 +5089,5089 +5090,5090 +5091,5091 +5092,5092 +5093,5093 +5094,5094 +5095,5095 +5096,5096 +5097,5097 +212118,5097 +5098,5098 +5099,5099 +5100,5100 +5101,5101 +5102,5102 +5103,5103 +5104,5104 +5105,5105 +5106,5106 +5107,5107 +5108,5108 +67275,5108 +5109,5109 +5110,5110 +5111,5111 +5112,5112 +5113,5113 +5114,5114 +64152,5114 +5115,5115 +5116,5116 +132608,5116 +125021,5116 +5117,5117 +5118,5118 +5119,5119 +5120,5120 +5121,5121 +5122,5122 +5123,5123 +5124,5124 +5125,5125 +5126,5126 +5127,5127 +157401,5127 +5128,5128 +5129,5129 +241820,5129 +120,5129 +5130,5130 +154098,5130 +219819,5130 +73076,5130 +5131,5131 +5132,5132 +5133,5133 +5134,5134 +5135,5135 +5136,5136 +5137,5137 +5138,5138 +5139,5139 +5140,5140 +5141,5141 +161083,5141 +5142,5142 +5143,5143 +5144,5144 +249837,5144 +5145,5145 +5146,5146 +5147,5147 +5148,5148 +5149,5149 +5150,5150 +5151,5151 +5152,5152 +5153,5153 +5154,5154 +5155,5155 +5156,5156 +5157,5157 +5158,5158 +5159,5159 +5160,5160 +5161,5161 +5162,5162 +5163,5163 +5164,5164 +5165,5165 +5166,5166 +5167,5167 +5168,5168 +5169,5169 +5170,5170 +166613,5170 +23902,5170 +5171,5171 +5172,5172 +5173,5173 +5174,5174 +5175,5175 +5220,5220 +5221,5221 +5222,5222 +134186,5222 +126599,5222 +5223,5223 +5224,5224 +5225,5225 +5226,5226 +5227,5227 +5228,5228 +5229,5229 +5230,5230 +5231,5231 +5232,5232 +178327,5232 +172363,5232 +5233,5233 +5234,5234 +5235,5235 +5236,5236 +5237,5237 +5238,5238 +5239,5239 +5240,5240 +5241,5241 +5242,5242 +5243,5243 +5244,5244 +5245,5245 +5246,5246 +5247,5247 +5248,5248 +5249,5249 +5250,5250 +5251,5251 +5252,5252 +5253,5253 +5254,5254 +5255,5255 +5256,5256 +5257,5257 +5258,5258 +5259,5259 +5260,5260 +5261,5261 +5262,5262 +5263,5263 +214407,5263 +5264,5264 +5265,5265 +246967,5265 +122185,5265 +5266,5266 +5267,5267 +5268,5268 +5269,5269 +151568,5269 +5270,5270 +5271,5271 +7275,5271 +5272,5272 +5273,5273 +5274,5274 +6646,5274 +216023,5274 +5275,5275 +5276,5276 +5277,5277 +5278,5278 +5279,5279 +5280,5280 +5281,5281 +5282,5282 +201064,5282 +5283,5283 +5284,5284 +5285,5285 +5286,5286 +5287,5287 +5288,5288 +5289,5289 +5290,5290 +5291,5291 +5292,5292 +96376,5292 +5293,5293 +124080,5293 +5294,5294 +5295,5295 +78956,5295 +5296,5296 +5297,5297 +5298,5298 +5299,5299 +5300,5300 +5301,5301 +5302,5302 +5303,5303 +5304,5304 +176726,5304 +170762,5304 +5305,5305 +5306,5306 +5307,5307 +5308,5308 +5309,5309 +5310,5310 +5311,5311 +5312,5312 +5313,5313 +99228,5314 +5314,5314 +5315,5315 +5316,5316 +5317,5317 +5318,5318 +5319,5319 +5320,5320 +5321,5321 +5322,5322 +5323,5323 +5324,5324 +5325,5325 +5326,5326 +5327,5327 +5328,5328 +5329,5329 +5330,5330 +5331,5331 +5332,5332 +5333,5333 +5334,5334 +5335,5335 +5336,5336 +5337,5337 +5338,5338 +5339,5339 +176725,5339 +170761,5339 +5340,5340 +5341,5341 +162781,5341 +5342,5342 +5343,5343 +5344,5344 +5345,5345 +5346,5346 +139183,5346 +5347,5347 +5348,5348 +5349,5349 +247796,5349 +5350,5350 +5351,5351 +5352,5352 +5353,5353 +5354,5354 +5355,5355 +5356,5356 +5357,5357 +133219,5357 +125632,5357 +5358,5358 +5359,5359 +5360,5360 +5361,5361 +5362,5362 +5363,5363 +5364,5364 +5365,5365 +5366,5366 +5367,5367 +5368,5368 +5369,5369 +5370,5370 +231959,5370 +5371,5371 +5372,5372 +35770,5372 +5373,5373 +5374,5374 +5375,5375 +5376,5376 +65934,5376 +5377,5377 +5378,5378 +5379,5379 +5380,5380 +5381,5381 +5382,5382 +110673,5382 +5383,5383 +5384,5384 +12191,5384 +122840,5385 +5385,5385 +5386,5386 +234404,5386 +5387,5387 +5388,5388 +5389,5389 +5390,5390 +198724,5390 +5391,5391 +5392,5392 +5393,5393 +5394,5394 +5395,5395 +5396,5396 +5397,5397 +5398,5398 +5399,5399 +5400,5400 +5401,5401 +5402,5402 +5403,5403 +5404,5404 +5405,5405 +5406,5406 +5407,5407 +5408,5408 +5409,5409 +5410,5410 +5411,5411 +5412,5412 +5413,5413 +5414,5414 +5415,5415 +59429,5415 +5416,5416 +5417,5417 +5418,5418 +5419,5419 +5420,5420 +5421,5421 +5422,5422 +5423,5423 +5424,5424 +5425,5425 +5426,5426 +5427,5427 +5428,5428 +5467,5467 +5468,5468 +2218,5468 +5469,5469 +105656,5469 +5470,5470 +105657,5470 +5471,5471 +5472,5472 +5473,5473 +5474,5474 +232973,5474 +5475,5475 +5476,5476 +5477,5477 +5478,5478 +5479,5479 +5480,5480 +5481,5481 +5482,5482 +5483,5483 +83985,5483 +5484,5484 +5485,5485 +5486,5486 +5487,5487 +5488,5488 +62081,5488 +5489,5489 +5490,5490 +4734,5490 +5491,5491 +243847,5492 +5492,5492 +5493,5493 +5494,5494 +5495,5495 +88036,5495 +5496,5496 +5497,5497 +5498,5498 +5499,5499 +5500,5500 +5501,5501 +5502,5502 +5503,5503 +5504,5504 +5505,5505 +220582,5505 +5506,5506 +5507,5507 +5508,5508 +5509,5509 +5510,5510 +5511,5511 +5512,5512 +5513,5513 +80,5513 +5514,5514 +5515,5515 +34058,5515 +5516,5516 +5517,5517 +5518,5518 +5519,5519 +5520,5520 +5521,5521 +5522,5522 +5523,5523 +5524,5524 +5525,5525 +185370,5525 +5526,5526 +5527,5527 +5528,5528 +5529,5529 +221928,5529 +5530,5530 +5531,5531 +5532,5532 +5533,5533 +5534,5534 +5535,5535 +5536,5536 +5537,5537 +5538,5538 +122826,5538 +5539,5539 +29517,5539 +5540,5540 +5541,5541 +5542,5542 +29799,5542 +5543,5543 +5544,5544 +5545,5545 +5546,5546 +5547,5547 +224391,5547 +5548,5548 +137152,5548 +129565,5548 +5549,5549 +5550,5550 +5551,5551 +5552,5552 +5553,5553 +5554,5554 +5555,5555 +135128,5555 +127541,5555 +5556,5556 +5557,5557 +5558,5558 +5559,5559 +5560,5560 +5561,5561 +5562,5562 +180475,5562 +69635,5563 +5563,5563 +5564,5564 +5565,5565 +5566,5566 +5567,5567 +5568,5568 +5569,5569 +5570,5570 +5571,5571 +5572,5572 +5573,5573 +5574,5574 +5575,5575 +5576,5576 +5577,5577 +5578,5578 +5579,5579 +5580,5580 +5581,5581 +5582,5582 +5583,5583 +5584,5584 +5585,5585 +5586,5586 +5587,5587 +5588,5588 +5589,5589 +5590,5590 +148658,5590 +5591,5591 +5592,5592 +5593,5593 +5594,5594 +5595,5595 +5596,5596 +5597,5597 +5598,5598 +5599,5599 +5600,5600 +5601,5601 +5602,5602 +5603,5603 +5604,5604 +109528,5604 +5605,5605 +5606,5606 +5607,5607 +5608,5608 +5609,5609 +193421,5609 +5610,5610 +5611,5611 +133549,5611 +125962,5611 +165081,5612 +5612,5612 +174349,5612 +168385,5612 +5613,5613 +31494,5613 +5614,5614 +5615,5615 +5616,5616 +5617,5617 +5618,5618 +5619,5619 +5620,5620 +5621,5621 +5622,5622 +5623,5623 +5624,5624 +5625,5625 +80531,5625 +5626,5626 +5627,5627 +5628,5628 +5629,5629 +5630,5630 +5631,5631 +5632,5632 +5633,5633 +5634,5634 +5635,5635 +5636,5636 +5637,5637 +2761,5637 +5638,5638 +5639,5639 +5640,5640 +5641,5641 +5642,5642 +5643,5643 +5644,5644 +5645,5645 +29516,5645 +5646,5646 +5647,5647 +5648,5648 +5649,5649 +5650,5650 +5651,5651 +5652,5652 +5653,5653 +137079,5653 +129492,5653 +5654,5654 +990,5654 +5655,5655 +257,5655 +5656,5656 +5657,5657 +5658,5658 +112094,5658 +5659,5659 +5660,5660 +5661,5661 +6148,5662 +5662,5662 +5663,5663 +5664,5664 +5665,5665 +5666,5666 +5667,5667 +5668,5668 +5669,5669 +5670,5670 +5671,5671 +5672,5672 +5673,5673 +5674,5674 +5675,5675 +5676,5676 +5677,5677 +5678,5678 +5679,5679 +5680,5680 +31137,5680 +5681,5681 +5682,5682 +5683,5683 +5684,5684 +5685,5685 +5686,5686 +5687,5687 +5688,5688 +5689,5689 +5690,5690 +5691,5691 +5692,5692 +5693,5693 +5694,5694 +5695,5695 +5696,5696 +5697,5697 +5698,5698 +69639,5698 +5699,5699 +5700,5700 +5701,5701 +5702,5702 +5703,5703 +5704,5704 +5705,5705 +6198,5705 +5706,5706 +5707,5707 +5708,5708 +5709,5709 +5710,5710 +5711,5711 +5712,5712 +5713,5713 +5714,5714 +5715,5715 +5716,5716 +5717,5717 +5718,5718 +5719,5719 +5720,5720 +5721,5721 +5722,5722 +5723,5723 +5724,5724 +33974,5724 +5725,5725 +5726,5726 +117052,5726 +5727,5727 +5728,5728 +5729,5729 +5730,5730 +5731,5731 +28943,5731 +5732,5732 +140697,5732 +5733,5733 +5734,5734 +5735,5735 +5736,5736 +5737,5737 +5738,5738 +5739,5739 +5740,5740 +5741,5741 +215399,5742 +5742,5742 +114358,5742 +5743,5743 +5744,5744 +5745,5745 +162536,5745 +5746,5746 +5747,5747 +5748,5748 +5749,5749 +5750,5750 +5751,5751 +33927,5751 +5752,5752 +5753,5753 +5754,5754 +152965,5754 +5755,5755 +5756,5756 +5757,5757 +5758,5758 +217858,5758 +5759,5759 +5760,5760 +5761,5761 +5762,5762 +5763,5763 +5764,5764 +5765,5765 +5766,5766 +5767,5767 +5768,5768 +9284,5769 +5769,5769 +5770,5770 +5771,5771 +5772,5772 +5773,5773 +135949,5773 +128362,5773 +208502,5773 +5774,5774 +5775,5775 +5776,5776 +5777,5777 +5778,5778 +5779,5779 +5780,5780 +5781,5781 +5782,5782 +5783,5783 +5784,5784 +5785,5785 +5786,5786 +206918,5786 +5787,5787 +5788,5788 +5789,5789 +5790,5790 +5791,5791 +82947,5791 +5792,5792 +5793,5793 +5794,5794 +5795,5795 +5796,5796 +5797,5797 +5798,5798 +5799,5799 +5800,5800 +161089,5800 +5801,5801 +5802,5802 +5803,5803 +5804,5804 +5805,5805 +5806,5806 +5807,5807 +5808,5808 +9359,5808 +5809,5809 +5810,5810 +5811,5811 +2680,5811 +5812,5812 +5813,5813 +5814,5814 +151456,5814 +5815,5815 +5816,5816 +5817,5817 +5818,5818 +5819,5819 +5820,5820 +49,5820 +5821,5821 +5822,5822 +210350,5822 +207595,5822 +5823,5823 +5824,5824 +5825,5825 +5826,5826 +5827,5827 +5828,5828 +185403,5828 +29969,5829 +5829,5829 +33265,5829 +5830,5830 +5831,5831 +5832,5832 +5833,5833 +5834,5834 +5835,5835 +5836,5836 +5837,5837 +5838,5838 +89558,5838 +5839,5839 +5840,5840 +5841,5841 +5842,5842 +5843,5843 +5844,5844 +5845,5845 +5846,5846 +5847,5847 +5848,5848 +5849,5849 +5850,5850 +5851,5851 +5852,5852 +5853,5853 +5854,5854 +148312,5854 +5855,5855 +7607,5855 +5856,5856 +5857,5857 +5858,5858 +5859,5859 +37496,5859 +5860,5860 +5861,5861 +5862,5862 +5863,5863 +5864,5864 +5865,5865 +5866,5866 +5867,5867 +5868,5868 +5869,5869 +5921,5921 +5922,5922 +5923,5923 +5924,5924 +5925,5925 +5926,5926 +5927,5927 +5928,5928 +5929,5929 +5930,5930 +5931,5931 +5932,5932 +5933,5933 +5934,5934 +5935,5935 +5936,5936 +147682,5936 +5937,5937 +5938,5938 +5939,5939 +5940,5940 +5941,5941 +5942,5942 +5943,5943 +497,5943 +5944,5944 +5945,5945 +5946,5946 +5947,5947 +5948,5948 +5949,5949 +5950,5950 +5951,5951 +5952,5952 +5953,5953 +5954,5954 +5955,5955 +5956,5956 +5957,5957 +5958,5958 +5959,5959 +5960,5960 +5961,5961 +5962,5962 +5963,5963 +5964,5964 +5965,5965 +5966,5966 +5967,5967 +5968,5968 +5969,5969 +5970,5970 +5971,5971 +5972,5972 +5973,5973 +5974,5974 +5975,5975 +5976,5976 +5977,5977 +5978,5978 +5979,5979 +5980,5980 +5981,5981 +5982,5982 +5983,5983 +5984,5984 +5985,5985 +5986,5986 +5987,5987 +5988,5988 +5989,5989 +5990,5990 +5991,5991 +5992,5992 +5993,5993 +5994,5994 +5995,5995 +5996,5996 +5997,5997 +5998,5998 +5999,5999 +6000,6000 +6001,6001 +6002,6002 +6003,6003 +6004,6004 +6005,6005 +6006,6006 +6007,6007 +66151,6008 +6008,6008 +6009,6009 +6010,6010 +6011,6011 +6012,6012 +6013,6013 +6014,6014 +6015,6015 +187129,6015 +6055,6055 +6056,6056 +6057,6057 +6058,6058 +6059,6059 +6060,6060 +6061,6061 +6062,6062 +6063,6063 +6064,6064 +6065,6065 +6066,6066 +234798,6066 +6067,6067 +6068,6068 +6069,6069 +6070,6070 +6071,6071 +6072,6072 +6073,6073 +6074,6074 +6075,6075 +11070,6075 +6076,6076 +6077,6077 +6078,6078 +6079,6079 +6080,6080 +6081,6081 +6082,6082 +6083,6083 +6084,6084 +6085,6085 +6086,6086 +6087,6087 +6088,6088 +6089,6089 +6090,6090 +6091,6091 +6092,6092 +6093,6093 +6094,6094 +6095,6095 +6096,6096 +6097,6097 +6098,6098 +6099,6099 +6100,6100 +6101,6101 +6102,6102 +6103,6103 +6104,6104 +6105,6105 +6106,6106 +6107,6107 +6108,6108 +6109,6109 +6110,6110 +6111,6111 +6112,6112 +6113,6113 +6114,6114 +6115,6115 +6116,6116 +6117,6117 +6118,6118 +6119,6119 +6120,6120 +6121,6121 +6122,6122 +6123,6123 +6124,6124 +6125,6125 +6126,6126 +6127,6127 +6128,6128 +6129,6129 +6130,6130 +6131,6131 +6132,6132 +6133,6133 +6134,6134 +242326,6134 +6135,6135 +6136,6136 +6137,6137 +6138,6138 +6139,6139 +6140,6140 +184797,6140 +6141,6141 +235007,6141 +51008,6141 +42904,6141 +6142,6142 +6143,6143 +6144,6144 +6145,6145 +6146,6146 +6147,6147 +6148,6148 +5662,6148 +6149,6149 +6150,6150 +6151,6151 +6152,6152 +6153,6153 +6154,6154 +6155,6155 +6156,6156 +6157,6157 +6158,6158 +6159,6159 +6160,6160 +6161,6161 +6162,6162 +6163,6163 +6164,6164 +6165,6165 +6166,6166 +6167,6167 +6168,6168 +6169,6169 +6170,6170 +6171,6171 +6172,6172 +6173,6173 +6174,6174 +6175,6175 +6176,6176 +6177,6177 +6178,6178 +6179,6179 +248048,6179 +104439,6180 +6180,6180 +6181,6181 +6182,6182 +161760,6182 +110710,6182 +6183,6183 +20504,6183 +6184,6184 +6185,6185 +6186,6186 +107975,6186 +6187,6187 +6188,6188 +6189,6189 +6190,6190 +6191,6191 +6192,6192 +6193,6193 +213443,6193 +6194,6194 +6195,6195 +6196,6196 +110612,6196 +3967,6196 +6197,6197 +6198,6198 +5705,6198 +6199,6199 +244357,6199 +6200,6200 +6201,6201 +6202,6202 +6203,6203 +6204,6204 +205991,6204 +6205,6205 +6206,6206 +6207,6207 +6208,6208 +6209,6209 +6210,6210 +6211,6211 +6212,6212 +6213,6213 +6214,6214 +6215,6215 +6216,6216 +6217,6217 +231946,6217 +6218,6218 +6219,6219 +6220,6220 +6221,6221 +6222,6222 +6223,6223 +91965,6223 +6224,6224 +6225,6225 +223105,6225 +6226,6226 +6227,6227 +6228,6228 +6229,6229 +6230,6230 +6231,6231 +6232,6232 +6233,6233 +6234,6234 +6235,6235 +6236,6236 +6237,6237 +6238,6238 +6278,6278 +6279,6279 +6280,6280 +6281,6281 +6282,6282 +6283,6283 +6284,6284 +6285,6285 +6286,6286 +6287,6287 +216242,6287 +6288,6288 +6289,6289 +6290,6290 +6291,6291 +83757,6291 +6292,6292 +6293,6293 +6294,6294 +6295,6295 +2460,6295 +6296,6296 +6297,6297 +142149,6297 +6298,6298 +6299,6299 +6300,6300 +6301,6301 +6302,6302 +6303,6303 +6304,6304 +6305,6305 +7022,6306 +6306,6306 +6307,6307 +6308,6308 +6309,6309 +57420,6309 +6310,6310 +6311,6311 +6312,6312 +6313,6313 +6314,6314 +6315,6315 +6316,6316 +6317,6317 +176591,6317 +170627,6317 +6318,6318 +6319,6319 +6320,6320 +6321,6321 +6322,6322 +6323,6323 +6324,6324 +6325,6325 +6326,6326 +6327,6327 +6328,6328 +6329,6329 +6330,6330 +6331,6331 +6332,6332 +6333,6333 +6334,6334 +6335,6335 +6336,6336 +6337,6337 +6338,6338 +6339,6339 +6340,6340 +6341,6341 +6342,6342 +6343,6343 +6344,6344 +6345,6345 +6346,6346 +6347,6347 +6348,6348 +6349,6349 +6350,6350 +6351,6351 +6352,6352 +6353,6353 +6354,6354 +6355,6355 +6356,6356 +6357,6357 +6358,6358 +6359,6359 +6360,6360 +6361,6361 +6362,6362 +6363,6363 +6364,6364 +6365,6365 +6366,6366 +6367,6367 +6368,6368 +6369,6369 +6370,6370 +174177,6370 +168213,6370 +6371,6371 +6372,6372 +6373,6373 +6374,6374 +6375,6375 +6376,6376 +6377,6377 +6378,6378 +6379,6379 +6380,6380 +6381,6381 +6382,6382 +6383,6383 +37018,6383 +4197,6383 +6384,6384 +6385,6385 +6386,6386 +6387,6387 +6388,6388 +6389,6389 +6390,6390 +6391,6391 +6392,6392 +6393,6393 +6394,6394 +6395,6395 +6396,6396 +6397,6397 +6398,6398 +6399,6399 +6400,6400 +6401,6401 +6402,6402 +6403,6403 +6404,6404 +6405,6405 +6406,6406 +6407,6407 +6408,6408 +6409,6409 +6410,6410 +6411,6411 +6412,6412 +6413,6413 +190086,6413 +6414,6414 +6415,6415 +6416,6416 +6417,6417 +6418,6418 +6419,6419 +154501,6419 +6420,6420 +6421,6421 +6422,6422 +6423,6423 +6424,6424 +4542,6424 +6425,6425 +6426,6426 +6427,6427 +6428,6428 +6429,6429 +6430,6430 +6431,6431 +2207,6431 +6432,6432 +6433,6433 +6475,6475 +6476,6476 +140670,6476 +6477,6477 +6478,6478 +6479,6479 +6480,6480 +6481,6481 +6482,6482 +6483,6483 +6484,6484 +6485,6485 +6486,6486 +6487,6487 +6488,6488 +58027,6488 +6489,6489 +6490,6490 +6491,6491 +6492,6492 +6493,6493 +6494,6494 +6495,6495 +6496,6496 +140150,6496 +6497,6497 +6498,6498 +6499,6499 +6500,6500 +6501,6501 +6502,6502 +6503,6503 +6504,6504 +6505,6505 +6506,6506 +6507,6507 +6508,6508 +176709,6508 +170745,6508 +6509,6509 +6510,6510 +6511,6511 +6512,6512 +6571,6571 +6572,6572 +6573,6573 +3508,6573 +6574,6574 +160660,6574 +6575,6575 +6576,6576 +6577,6577 +162519,6577 +191830,6577 +191609,6577 +6578,6578 +6579,6579 +6580,6580 +6581,6581 +6582,6582 +6583,6583 +6584,6584 +113505,6584 +6585,6585 +6586,6586 +6587,6587 +231206,6587 +6588,6588 +6589,6589 +6590,6590 +6591,6591 +206602,6591 +6592,6592 +6593,6593 +6594,6594 +6595,6595 +6596,6596 +6597,6597 +6598,6598 +97429,6598 +6599,6599 +6600,6600 +6601,6601 +6602,6602 +6603,6603 +6604,6604 +6605,6605 +6606,6606 +6607,6607 +6608,6608 +6609,6609 +6610,6610 +6611,6611 +6612,6612 +6613,6613 +6614,6614 +142888,6615 +6615,6615 +13461,6615 +6616,6616 +6617,6617 +6618,6618 +6619,6619 +6620,6620 +6621,6621 +6622,6622 +6623,6623 +6624,6624 +6625,6625 +6626,6626 +6627,6627 +6628,6628 +6629,6629 +215638,6629 +104968,6630 +6630,6630 +6631,6631 +6632,6632 +6633,6633 +6634,6634 +154770,6634 +6635,6635 +6636,6636 +6637,6637 +6638,6638 +6639,6639 +6640,6640 +6641,6641 +6642,6642 +6643,6643 +6644,6644 +6645,6645 +6646,6646 +5274,6646 +6647,6647 +6648,6648 +6649,6649 +6650,6650 +62081,6650 +249570,6650 +6651,6651 +6652,6652 +6653,6653 +6654,6654 +6655,6655 +6656,6656 +6657,6657 +6658,6658 +6659,6659 +6660,6660 +6661,6661 +6662,6662 +6663,6663 +6664,6664 +6665,6665 +6666,6666 +6667,6667 +6668,6668 +6669,6669 +6670,6670 +6671,6671 +6672,6672 +6673,6673 +6674,6674 +6675,6675 +6676,6676 +6677,6677 +6678,6678 +6679,6679 +6680,6680 +6681,6681 +6682,6682 +57317,6682 +6683,6683 +6684,6684 +6685,6685 +6686,6686 +6687,6687 +6688,6688 +6689,6689 +6690,6690 +225333,6690 +6691,6691 +243413,6691 +6692,6692 +6693,6693 +6694,6694 +6695,6695 +83952,6695 +6696,6696 +6697,6697 +6698,6698 +158492,6698 +33280,6698 +142695,6698 +31215,6698 +6699,6699 +6700,6700 +4588,6700 +185438,6700 +6701,6701 +6702,6702 +6703,6703 +6704,6704 +6705,6705 +6706,6706 +249660,6706 +6707,6707 +6708,6708 +6709,6709 +6710,6710 +6711,6711 +6712,6712 +6713,6713 +6714,6714 +6715,6715 +6716,6716 +80676,6716 +6717,6717 +6718,6718 +201017,6718 +6719,6719 +6720,6720 +6721,6721 +6722,6722 +6723,6723 +6724,6724 +6725,6725 +231998,6725 +6726,6726 +6727,6727 +6728,6728 +6729,6729 +6730,6730 +80661,6730 +6731,6731 +6732,6732 +6733,6733 +77014,6733 +6734,6734 +6735,6735 +6736,6736 +6737,6737 +6738,6738 +6739,6739 +6740,6740 +6741,6741 +6742,6742 +6743,6743 +6744,6744 +6745,6745 +36708,6745 +6746,6746 +6747,6747 +6748,6748 +6749,6749 +6750,6750 +6751,6751 +6752,6752 +6753,6753 +6754,6754 +6755,6755 +6756,6756 +6757,6757 +6758,6758 +6759,6759 +6760,6760 +6761,6761 +6762,6762 +6763,6763 +6764,6764 +6765,6765 +6766,6766 +6767,6767 +6768,6768 +6769,6769 +6770,6770 +6771,6771 +6772,6772 +6773,6773 +158982,6773 +6774,6774 +230548,6774 +6775,6775 +6776,6776 +6777,6777 +6778,6778 +6779,6779 +6780,6780 +6781,6781 +6782,6782 +6783,6783 +6784,6784 +6785,6785 +6786,6786 +6787,6787 +6788,6788 +6789,6789 +108063,6789 +6790,6790 +6791,6791 +6792,6792 +6793,6793 +6794,6794 +6795,6795 +6796,6796 +6797,6797 +6798,6798 +6799,6799 +6800,6800 +6801,6801 +6802,6802 +6803,6803 +6804,6804 +6805,6805 +6806,6806 +6807,6807 +6808,6808 +6809,6809 +6810,6810 +6811,6811 +6812,6812 +6813,6813 +6814,6814 +6815,6815 +6816,6816 +6817,6817 +6818,6818 +6819,6819 +6820,6820 +6821,6821 +6822,6822 +6823,6823 +6824,6824 +6825,6825 +6826,6826 +6827,6827 +6828,6828 +6829,6829 +6830,6830 +6831,6831 +6832,6832 +6833,6833 +180248,6833 +6834,6834 +6835,6835 +6836,6836 +6837,6837 +62771,6837 +6838,6838 +6839,6839 +6840,6840 +104055,6840 +231955,6841 +6841,6841 +6842,6842 +6843,6843 +211768,6843 +6844,6844 +29900,6844 +6845,6845 +6846,6846 +6847,6847 +6848,6848 +57500,6848 +6849,6849 +6850,6850 +6851,6851 +6852,6852 +108846,6852 +6853,6853 +233568,6853 +6854,6854 +6855,6855 +6856,6856 +6857,6857 +6858,6858 +6859,6859 +6860,6860 +6861,6861 +6862,6862 +6863,6863 +6864,6864 +232645,6864 +6865,6865 +6866,6866 +6867,6867 +6868,6868 +6869,6869 +6870,6870 +6871,6871 +6872,6872 +6873,6873 +6874,6874 +6875,6875 +6876,6876 +6877,6877 +6878,6878 +6879,6879 +6880,6880 +6881,6881 +120302,6881 +6882,6882 +6883,6883 +6884,6884 +6885,6885 +6886,6886 +6887,6887 +15416,6887 +6888,6888 +6889,6889 +6890,6890 +6891,6891 +6892,6892 +6893,6893 +6894,6894 +6895,6895 +6896,6896 +6897,6897 +6898,6898 +6899,6899 +6900,6900 +177503,6900 +171539,6900 +6901,6901 +6902,6902 +6903,6903 +6904,6904 +110610,6904 +6905,6905 +6906,6906 +6907,6907 +6908,6908 +6909,6909 +187165,6909 +6910,6910 +6911,6911 +6912,6912 +235263,6913 +6913,6913 +6914,6914 +6915,6915 +6916,6916 +6917,6917 +6918,6918 +6919,6919 +6920,6920 +80075,6921 +6921,6921 +6922,6922 +6923,6923 +6924,6924 +6966,6966 +6967,6967 +6968,6968 +6969,6969 +6970,6970 +6971,6971 +7014,7014 +11525,7014 +7015,7015 +7016,7016 +7017,7017 +7018,7018 +7019,7019 +7020,7020 +7021,7021 +7022,7022 +6306,7022 +7023,7023 +7024,7024 +7025,7025 +7026,7026 +7027,7027 +7028,7028 +7029,7029 +7030,7030 +7031,7031 +7032,7032 +7033,7033 +7034,7034 +7035,7035 +7036,7036 +7037,7037 +7038,7038 +7039,7039 +7040,7040 +7041,7041 +7042,7042 +7043,7043 +7044,7044 +82538,7044 +7045,7045 +7046,7046 +7047,7047 +7048,7048 +7049,7049 +7050,7050 +11927,7050 +7051,7051 +7052,7052 +7053,7053 +7054,7054 +70349,7054 +7055,7055 +222943,7055 +7056,7056 +7057,7057 +7058,7058 +7059,7059 +7060,7060 +246947,7060 +7061,7061 +7062,7062 +7063,7063 +7064,7064 +234714,7064 +7065,7065 +7066,7066 +7067,7067 +7068,7068 +7069,7069 +7070,7070 +7071,7071 +4526,7071 +7072,7072 +7073,7073 +7074,7074 +7075,7075 +7076,7076 +7077,7077 +7078,7078 +7079,7079 +7080,7080 +7081,7081 +7082,7082 +7083,7083 +7084,7084 +7085,7085 +7086,7086 +7087,7087 +7088,7088 +7089,7089 +7090,7090 +7091,7091 +7092,7092 +7093,7093 +7094,7094 +7095,7095 +7096,7096 +7097,7097 +7098,7098 +7099,7099 +954,7099 +246649,7099 +146509,7099 +7100,7100 +7101,7101 +7102,7102 +7103,7103 +7104,7104 +225026,7104 +7105,7105 +7106,7106 +7107,7107 +7108,7108 +7109,7109 +7110,7110 +7111,7111 +7112,7112 +82339,7112 +7113,7113 +7114,7114 +7115,7115 +7116,7116 +7117,7117 +7118,7118 +7119,7119 +14005,7119 +7120,7120 +7121,7121 +7122,7122 +7123,7123 +7124,7124 +7125,7125 +7126,7126 +7127,7127 +7128,7128 +7129,7129 +7130,7130 +7131,7131 +7132,7132 +7133,7133 +7134,7134 +7135,7135 +7136,7136 +7137,7137 +7138,7138 +7139,7139 +7140,7140 +7141,7141 +7142,7142 +7143,7143 +7144,7144 +7200,7144 +7145,7145 +7146,7146 +7147,7147 +7148,7148 +7149,7149 +7150,7150 +7151,7151 +7152,7152 +7153,7153 +7154,7154 +7155,7155 +7156,7156 +7157,7157 +75284,7157 +7158,7158 +7159,7159 +7160,7160 +7161,7161 +7162,7162 +7163,7163 +7164,7164 +7165,7165 +7166,7166 +7167,7167 +7168,7168 +7169,7169 +7170,7170 +7171,7171 +38198,7172 +37323,7172 +34694,7172 +7172,7172 +101883,7172 +7173,7173 +7174,7174 +7175,7175 +7176,7176 +7177,7177 +7178,7178 +7179,7179 +7180,7180 +7181,7181 +7182,7182 +7183,7183 +7184,7184 +7185,7185 +49125,7185 +7186,7186 +7187,7187 +7188,7188 +7189,7189 +7190,7190 +7191,7191 +7192,7192 +7193,7193 +7194,7194 +7195,7195 +7196,7196 +7197,7197 +7198,7198 +7199,7199 +7200,7200 +39717,7200 +7144,7200 +7201,7201 +7202,7202 +227760,7202 +7203,7203 +7204,7204 +7205,7205 +7206,7206 +7207,7207 +7208,7208 +7209,7209 +7210,7210 +7211,7211 +7212,7212 +7213,7213 +7214,7214 +7215,7215 +7216,7216 +7217,7217 +7218,7218 +7219,7219 +7220,7220 +7221,7221 +7222,7222 +7223,7223 +110652,7223 +7224,7224 +7225,7225 +7226,7226 +7227,7227 +7228,7228 +7229,7229 +7230,7230 +7231,7231 +7232,7232 +249117,7232 +7233,7233 +7234,7234 +7235,7235 +7236,7236 +7237,7237 +7238,7238 +7239,7239 +7240,7240 +7241,7241 +176537,7241 +170573,7241 +7242,7242 +7243,7243 +7244,7244 +7245,7245 +7246,7246 +7247,7247 +7248,7248 +7249,7249 +7250,7250 +7251,7251 +7252,7252 +7253,7253 +7254,7254 +7255,7255 +7256,7256 +4213,7256 +7257,7257 +7258,7258 +7259,7259 +7260,7260 +29060,7260 +7261,7261 +7262,7262 +7263,7263 +7264,7264 +7265,7265 +7266,7266 +7267,7267 +7268,7268 +7269,7269 +7270,7270 +7271,7271 +7272,7272 +7273,7273 +7274,7274 +7275,7275 +5271,7275 +7276,7276 +7277,7277 +7278,7278 +7279,7279 +7280,7280 +7281,7281 +7282,7282 +7283,7283 +7284,7284 +7285,7285 +7286,7286 +7287,7287 +82345,7287 +7288,7288 +7289,7289 +7290,7290 +7291,7291 +7292,7292 +7293,7293 +59542,7293 +7294,7294 +7295,7295 +7296,7296 +7297,7297 +7298,7298 +7299,7299 +7300,7300 +7301,7301 +111201,7301 +7302,7302 +7303,7303 +7304,7304 +7305,7305 +7306,7306 +7307,7307 +7308,7308 +7309,7309 +7310,7310 +7311,7311 +7312,7312 +7313,7313 +7314,7314 +7315,7315 +7316,7316 +7317,7317 +7318,7318 +7319,7319 +7320,7320 +7321,7321 +7322,7322 +7323,7323 +7324,7324 +7325,7325 +7326,7326 +7327,7327 +7328,7328 +7329,7329 +7330,7330 +7331,7331 +7332,7332 +7333,7333 +7334,7334 +207932,7334 +7335,7335 +7336,7336 +7337,7337 +207113,7338 +7338,7338 +7339,7339 +7340,7340 +7341,7341 +7342,7342 +7343,7343 +7344,7344 +7345,7345 +109261,7345 +7346,7346 +193148,7346 +7347,7347 +7348,7348 +7349,7349 +7350,7350 +7351,7351 +7352,7352 +7353,7353 +7354,7354 +7355,7355 +7356,7356 +7357,7357 +7358,7358 +7359,7359 +7360,7360 +7361,7361 +4312,7361 +7362,7362 +7363,7363 +7364,7364 +7365,7365 +7366,7366 +7367,7367 +7368,7368 +7369,7369 +7370,7370 +7371,7371 +246958,7371 +7411,7411 +7412,7412 +7413,7413 +7414,7414 +7415,7415 +7416,7416 +7417,7417 +7418,7418 +7419,7419 +7420,7420 +7421,7421 +7466,7466 +7467,7467 +7468,7468 +7469,7469 +7470,7470 +7471,7471 +7472,7472 +7473,7473 +7474,7474 +7475,7475 +7476,7476 +7477,7477 +7478,7478 +7479,7479 +7480,7480 +77262,7480 +7481,7481 +7482,7482 +7483,7483 +7484,7484 +7604,7484 +7522,7522 +21480,7522 +7523,7523 +7524,7524 +7562,7562 +7563,7563 +7564,7564 +92107,7564 +7565,7565 +7566,7566 +7567,7567 +7568,7568 +7569,7569 +7570,7570 +7571,7571 +7572,7572 +7573,7573 +7574,7574 +7575,7575 +7576,7576 +7577,7577 +7578,7578 +7579,7579 +7580,7580 +7581,7581 +7582,7582 +7583,7583 +7584,7584 +7585,7585 +7586,7586 +7587,7587 +7588,7588 +7589,7589 +86355,7589 +184973,7589 +7590,7590 +7591,7591 +7592,7592 +7593,7593 +7594,7594 +7595,7595 +7596,7596 +7597,7597 +7598,7598 +7599,7599 +7600,7600 +7601,7601 +2685,7601 +7602,7602 +7603,7603 +7604,7604 +7484,7604 +7605,7605 +143213,7605 +7606,7606 +7607,7607 +5855,7607 +7608,7608 +7609,7609 +7610,7610 +90546,7610 +7611,7611 +7612,7612 +7613,7613 +7614,7614 +7615,7615 +7616,7616 +7617,7617 +7618,7618 +50111,7618 +41918,7618 +7619,7619 +7620,7620 +7621,7621 +7622,7622 +7623,7623 +7624,7624 +7625,7625 +7626,7626 +7627,7627 +7628,7628 +7629,7629 +7630,7630 +7631,7631 +7632,7632 +7633,7633 +7634,7634 +7635,7635 +7636,7636 +7637,7637 +7638,7638 +7639,7639 +7640,7640 +7641,7641 +7642,7642 +7643,7643 +7644,7644 +7645,7645 +7646,7646 +7647,7647 +7648,7648 +7649,7649 +156297,7649 +7650,7650 +7651,7651 +7652,7652 +164459,7652 +7653,7653 +7654,7654 +7655,7655 +7656,7656 +7657,7657 +236520,7657 +7658,7658 +7659,7659 +7660,7660 +7661,7661 +7662,7662 +7663,7663 +7664,7664 +231429,7664 +7665,7665 +111394,7665 +7666,7666 +7667,7667 +7668,7668 +7669,7669 +7670,7670 +7671,7671 +7672,7672 +7673,7673 +7674,7674 +7675,7675 +7676,7676 +7677,7677 +7678,7678 +7679,7679 +7680,7680 +7681,7681 +7682,7682 +7683,7683 +7684,7684 +196270,7684 +7685,7685 +7686,7686 +7687,7687 +7688,7688 +7689,7689 +203827,7689 +7690,7690 +7691,7691 +7692,7692 +7693,7693 +7694,7694 +7695,7695 +7696,7696 +7697,7697 +7698,7698 +7699,7699 +59117,7699 +58743,7699 +7700,7700 +37881,7700 +7701,7701 +7702,7702 +7703,7703 +208210,7703 +7704,7704 +4812,7704 +7705,7705 +7706,7706 +103619,7706 +7707,7707 +7708,7708 +7709,7709 +7710,7710 +7711,7711 +7712,7712 +7713,7713 +9019,7714 +7714,7714 +7715,7715 +165394,7715 +7716,7716 +7717,7717 +7718,7718 +7719,7719 +4995,7719 +7720,7720 +7721,7721 +7722,7722 +7723,7723 +7724,7724 +40233,7724 +7725,7725 +7726,7726 +7727,7727 +7728,7728 +7729,7729 +68223,7729 +7730,7730 +7731,7731 +7732,7732 +7733,7733 +7734,7734 +7735,7735 +7736,7736 +7737,7737 +160624,7737 +7738,7738 +185619,7738 +217475,7738 +7739,7739 +33002,7739 +7740,7740 +7741,7741 +7742,7742 +7743,7743 +7744,7744 +111269,7744 +7745,7745 +7746,7746 +7747,7747 +7748,7748 +7749,7749 +142570,7750 +7750,7750 +7751,7751 +7752,7752 +7753,7753 +145225,7753 +160566,7753 +7754,7754 +249662,7754 +7755,7755 +7756,7756 +7757,7757 +7758,7758 +74069,7759 +7759,7759 +11610,7759 +7760,7760 +7761,7761 +7762,7762 +19922,7762 +7763,7763 +7764,7764 +7972,7972 +7973,7973 +143245,7973 +7974,7974 +7975,7975 +7976,7976 +7977,7977 +7978,7978 +7979,7979 +7980,7980 +7981,7981 +7982,7982 +7983,7983 +7984,7984 +240523,7985 +7985,7985 +7986,7986 +113400,7986 +7987,7987 +7988,7988 +7989,7989 +7990,7990 +7991,7991 +7992,7992 +7993,7993 +7994,7994 +7995,7995 +7996,7996 +7997,7997 +7998,7998 +7999,7999 +8000,8000 +140131,8001 +8001,8001 +8002,8002 +8003,8003 +8004,8004 +8005,8005 +179385,8005 +8006,8006 +8007,8007 +8008,8008 +8009,8009 +8010,8010 +8011,8011 +8012,8012 +8013,8013 +8014,8014 +8015,8015 +8016,8016 +8017,8017 +8018,8018 +8019,8019 +8020,8020 +8021,8021 +8022,8022 +8023,8023 +8024,8024 +8025,8025 +8026,8026 +8027,8027 +8028,8028 +8029,8029 +8030,8030 +8031,8031 +8032,8032 +8033,8033 +8034,8034 +8035,8035 +8036,8036 +8037,8037 +8038,8038 +8039,8039 +8040,8040 +8041,8041 +8042,8042 +8043,8043 +8044,8044 +8045,8045 +8046,8046 +8047,8047 +8048,8048 +8049,8049 +8050,8050 +8051,8051 +8052,8052 +8053,8053 +8054,8054 +8055,8055 +8056,8056 +8057,8057 +8058,8058 +8059,8059 +8060,8060 +8061,8061 +8062,8062 +248725,8062 +8063,8063 +8064,8064 +8065,8065 +8066,8066 +8067,8067 +8068,8068 +8069,8069 +8070,8070 +2818,8070 +8071,8071 +8072,8072 +8073,8073 +8074,8074 +8075,8075 +8076,8076 +8077,8077 +8078,8078 +8079,8079 +8080,8080 +8081,8081 +8082,8082 +122597,8082 +8083,8083 +60067,8083 +8084,8084 +8085,8085 +8086,8086 +138852,8086 +8087,8087 +8088,8088 +8089,8089 +8090,8090 +8126,8126 +8127,8127 +8128,8128 +8129,8129 +8130,8130 +8131,8131 +109822,8132 +8132,8132 +8169,8169 +8170,8170 +8171,8171 +8172,8172 +8173,8173 +8174,8174 +8175,8175 +8176,8176 +8177,8177 +8178,8178 +8179,8179 +8180,8180 +8181,8181 +8182,8182 +8183,8183 +8184,8184 +8185,8185 +8186,8186 +8187,8187 +8188,8188 +8189,8189 +8190,8190 +8191,8191 +8192,8192 +8193,8193 +8194,8194 +8195,8195 +8196,8196 +8197,8197 +8198,8198 +8199,8199 +8200,8200 +8201,8201 +8202,8202 +8203,8203 +8204,8204 +8205,8205 +8206,8206 +8207,8207 +8208,8208 +8209,8209 +29006,8209 +8210,8210 +8211,8211 +8212,8212 +8213,8213 +8214,8214 +123117,8214 +8215,8215 +8216,8216 +8217,8217 +8218,8218 +8219,8219 +8220,8220 +8221,8221 +8222,8222 +8223,8223 +8224,8224 +8225,8225 +8226,8226 +8322,8322 +8323,8323 +8324,8324 +17387,8324 +8325,8325 +8326,8326 +8327,8327 +8328,8328 +8329,8329 +8330,8330 +8331,8331 +8332,8332 +8333,8333 +8334,8334 +8335,8335 +8336,8336 +8337,8337 +8338,8338 +8339,8339 +8340,8340 +8341,8341 +8342,8342 +8343,8343 +8344,8344 +8345,8345 +8346,8346 +8347,8347 +8348,8348 +8349,8349 +8991,8349 +8350,8350 +8351,8351 +8352,8352 +8353,8353 +8533,8353 +8534,8354 +8354,8354 +8355,8355 +8535,8355 +79483,8355 +8356,8356 +8536,8356 +8357,8357 +8537,8357 +8538,8358 +8358,8358 +8539,8359 +8359,8359 +8540,8360 +8360,8360 +8541,8361 +8361,8361 +8542,8362 +8362,8362 +8543,8363 +8363,8363 +8544,8364 +8364,8364 +8545,8365 +8365,8365 +8546,8366 +8366,8366 +8547,8367 +8367,8367 +8548,8368 +8368,8368 +8549,8369 +8369,8369 +8550,8370 +8370,8370 +8551,8371 +8371,8371 +8552,8372 +8372,8372 +8553,8373 +8373,8373 +8554,8374 +8374,8374 +8555,8375 +8375,8375 +8556,8376 +8376,8376 +8557,8377 +8377,8377 +8558,8378 +8378,8378 +8559,8379 +8379,8379 +8560,8380 +8380,8380 +8561,8381 +8381,8381 +8562,8382 +8382,8382 +8563,8383 +8383,8383 +8564,8384 +8384,8384 +8565,8385 +8385,8385 +8566,8386 +8386,8386 +217095,8386 +8567,8387 +8387,8387 +8568,8388 +8388,8388 +8569,8389 +8389,8389 +8570,8390 +8390,8390 +8571,8391 +8391,8391 +8572,8392 +8392,8392 +8573,8393 +8393,8393 +8574,8394 +8394,8394 +8575,8395 +8395,8395 +8576,8396 +8396,8396 +8577,8397 +8397,8397 +8578,8398 +8398,8398 +8579,8399 +8399,8399 +85877,8399 +8580,8400 +8400,8400 +8581,8401 +8401,8401 +185175,8401 +8582,8402 +8402,8402 +8583,8403 +8403,8403 +108965,8403 +8584,8404 +8404,8404 +8585,8405 +8405,8405 +8586,8406 +8406,8406 +8587,8407 +8407,8407 +8588,8408 +8408,8408 +8589,8409 +8409,8409 +8590,8410 +8410,8410 +8591,8411 +8411,8411 +8592,8412 +8412,8412 +193428,8412 +223615,8413 +8593,8413 +8413,8413 +8594,8414 +8414,8414 +8595,8415 +8415,8415 +8596,8416 +8416,8416 +8597,8417 +8417,8417 +8598,8418 +8418,8418 +8599,8419 +8419,8419 +8600,8420 +8420,8420 +8601,8421 +8421,8421 +8602,8422 +8422,8422 +8603,8423 +8423,8423 +8604,8424 +8424,8424 +202784,8424 +8605,8425 +8425,8425 +8606,8426 +8426,8426 +246641,8426 +8607,8427 +8427,8427 +8608,8428 +8428,8428 +8609,8429 +8429,8429 +235823,8429 +8610,8430 +8430,8430 +8611,8431 +8431,8431 +8612,8432 +8432,8432 +8613,8433 +8433,8433 +8614,8434 +8434,8434 +8615,8435 +8435,8435 +8616,8436 +8436,8436 +8617,8437 +8437,8437 +190772,8437 +8618,8438 +8438,8438 +8439,8439 +8440,8440 +8441,8441 +8442,8442 +8623,8443 +8443,8443 +8444,8444 +8625,8445 +8445,8445 +8446,8446 +8627,8447 +8447,8447 +9077,8447 +8628,8448 +8448,8448 +8449,8449 +8450,8450 +8631,8451 +8451,8451 +79951,8451 +8632,8452 +8452,8452 +8453,8453 +8634,8454 +8454,8454 +8455,8455 +8456,8456 +8457,8457 +8638,8458 +8458,8458 +8639,8459 +8459,8459 +8640,8460 +8460,8460 +8641,8461 +8461,8461 +8462,8462 +8642,8462 +8689,8509 +8509,8509 +78665,8509 +9479,8510 +9250,8510 +8887,8510 +8690,8510 +8510,8510 +8962,8510 +9480,8511 +9251,8511 +8691,8511 +8511,8511 +8888,8511 +8692,8512 +8512,8512 +8693,8513 +8513,8513 +8694,8514 +8514,8514 +8695,8515 +8515,8515 +8696,8516 +8516,8516 +8697,8517 +8517,8517 +8698,8518 +8518,8518 +8519,8519 +8699,8519 +8520,8520 +8700,8520 +52653,8520 +44549,8520 +8521,8521 +8701,8521 +8522,8522 +8702,8522 +8523,8523 +8524,8524 +8525,8525 +62411,8525 +8526,8526 +8527,8527 +8707,8527 +8528,8528 +144028,8528 +8529,8529 +8991,8529 +8530,8530 +8531,8531 +8532,8532 +8533,8533 +8353,8533 +8534,8534 +8354,8534 +8535,8535 +8355,8535 +8536,8536 +8356,8536 +8537,8537 +8357,8537 +8538,8538 +8358,8538 +8539,8539 +8359,8539 +8540,8540 +8360,8540 +8541,8541 +8361,8541 +8542,8542 +8362,8542 +8543,8543 +8363,8543 +8544,8544 +8364,8544 +8545,8545 +8365,8545 +8546,8546 +8366,8546 +8547,8547 +8367,8547 +8548,8548 +8368,8548 +8549,8549 +8369,8549 +8550,8550 +8370,8550 +8551,8551 +8371,8551 +8552,8552 +8372,8552 +8553,8553 +8373,8553 +8554,8554 +8374,8554 +8555,8555 +8375,8555 +8556,8556 +8376,8556 +8557,8557 +8377,8557 +8558,8558 +8378,8558 +8559,8559 +8379,8559 +8560,8560 +8380,8560 +8561,8561 +8381,8561 +8562,8562 +8382,8562 +8563,8563 +8383,8563 +8564,8564 +8384,8564 +8565,8565 +8385,8565 +8566,8566 +8386,8566 +217095,8566 +8567,8567 +8387,8567 +8568,8568 +8388,8568 +8569,8569 +8389,8569 +8570,8570 +8390,8570 +8571,8571 +8391,8571 +8572,8572 +8392,8572 +8573,8573 +8393,8573 +8574,8574 +8394,8574 +8575,8575 +8395,8575 +8576,8576 +8396,8576 +8577,8577 +8397,8577 +8578,8578 +8398,8578 +8579,8579 +8399,8579 +85877,8579 +8580,8580 +8400,8580 +8581,8581 +8401,8581 +185175,8581 +8582,8582 +8402,8582 +8583,8583 +8403,8583 +108965,8583 +8584,8584 +8404,8584 +8585,8585 +8405,8585 +8586,8586 +8406,8586 +8587,8587 +8407,8587 +8588,8588 +8408,8588 +8589,8589 +8409,8589 +8590,8590 +8410,8590 +8591,8591 +8411,8591 +8592,8592 +8412,8592 +193428,8592 +223615,8593 +8593,8593 +8413,8593 +8594,8594 +8414,8594 +8595,8595 +8415,8595 +8596,8596 +8416,8596 +8597,8597 +8417,8597 +8598,8598 +8418,8598 +8599,8599 +8419,8599 +8600,8600 +8420,8600 +8601,8601 +8421,8601 +8602,8602 +8422,8602 +8603,8603 +8423,8603 +8604,8604 +8424,8604 +202784,8604 +8605,8605 +8425,8605 +8606,8606 +8426,8606 +246641,8606 +8607,8607 +8427,8607 +8608,8608 +8428,8608 +8609,8609 +8429,8609 +235823,8609 +8610,8610 +8430,8610 +8611,8611 +8431,8611 +8612,8612 +8432,8612 +8613,8613 +8433,8613 +8614,8614 +8434,8614 +8615,8615 +8435,8615 +8616,8616 +8436,8616 +8617,8617 +8437,8617 +190772,8617 +8618,8618 +8438,8618 +8619,8619 +8620,8620 +8788,8620 +8621,8621 +8622,8622 +8623,8623 +8443,8623 +8624,8624 +8625,8625 +8445,8625 +8626,8626 +8627,8627 +8447,8627 +9077,8627 +8628,8628 +8448,8628 +8629,8629 +8630,8630 +141468,8630 +8631,8631 +8451,8631 +79951,8631 +8632,8632 +8452,8632 +8633,8633 +8634,8634 +8454,8634 +8635,8635 +8636,8636 +8637,8637 +8638,8638 +8458,8638 +8639,8639 +8459,8639 +8640,8640 +8460,8640 +8641,8641 +8461,8641 +8642,8642 +8462,8642 +8689,8689 +8509,8689 +78665,8689 +9479,8690 +9250,8690 +8887,8690 +8690,8690 +8510,8690 +8962,8690 +9480,8691 +9251,8691 +8691,8691 +8511,8691 +8888,8691 +8692,8692 +8512,8692 +8693,8693 +8513,8693 +8694,8694 +8514,8694 +8695,8695 +8515,8695 +8696,8696 +8516,8696 +8697,8697 +8517,8697 +8698,8698 +8518,8698 +8699,8699 +8519,8699 +8700,8700 +8520,8700 +8701,8701 +8521,8701 +8702,8702 +8522,8702 +8703,8703 +8704,8704 +8705,8705 +8706,8706 +83775,8706 +8707,8707 +8527,8707 +8708,8708 +8709,8709 +8710,8710 +8711,8711 +8712,8712 +8713,8713 +8714,8714 +8715,8715 +8716,8716 +8717,8717 +8718,8718 +8719,8719 +8720,8720 +84204,8720 +8721,8721 +8722,8722 +8723,8723 +18529,8723 +8724,8724 +8725,8725 +8726,8726 +8727,8727 +8728,8728 +8729,8729 +8730,8730 +8731,8731 +8732,8732 +8733,8733 +8734,8734 +8735,8735 +8736,8736 +8737,8737 +8738,8738 +8739,8739 +8740,8740 +8741,8741 +8742,8742 +8743,8743 +8744,8744 +8745,8745 +8746,8746 +8747,8747 +8748,8748 +8749,8749 +8750,8750 +8751,8751 +8752,8752 +8753,8753 +8754,8754 +8755,8755 +8756,8756 +8757,8757 +8758,8758 +8759,8759 +8760,8760 +8761,8761 +8762,8762 +8763,8763 +8764,8764 +8765,8765 +8766,8766 +193779,8766 +8767,8767 +8768,8768 +8769,8769 +8770,8770 +187086,8770 +8771,8771 +8772,8772 +8773,8773 +8774,8774 +8775,8775 +8776,8776 +8777,8777 +8778,8778 +8779,8779 +8780,8780 +8781,8781 +8782,8782 +8783,8783 +8784,8784 +8785,8785 +8786,8786 +8787,8787 +8788,8788 +8620,8788 +8789,8789 +108534,8789 +8790,8790 +8791,8791 +8792,8792 +8793,8793 +8794,8794 +8795,8795 +8796,8796 +8797,8797 +8798,8798 +8799,8799 +8800,8800 +8801,8801 +8802,8802 +8803,8803 +8855,8855 +8856,8856 +8857,8857 +9219,8857 +8858,8858 +8859,8859 +8860,8860 +8861,8861 +60655,8861 +8862,8862 +205429,8862 +8863,8863 +136997,8863 +129410,8863 +8864,8864 +8865,8865 +8866,8866 +8867,8867 +8868,8868 +8869,8869 +8870,8870 +8871,8871 +8872,8872 +8873,8873 +8874,8874 +8875,8875 +8899,8876 +8876,8876 +8877,8877 +8878,8878 +142462,8878 +8879,8879 +8903,8880 +8880,8880 +8881,8881 +8904,8881 +58263,8881 +8882,8882 +9474,8882 +8883,8883 +9475,8883 +8884,8884 +9476,8884 +9477,8885 +8885,8885 +8886,8886 +9478,8886 +9479,8887 +9250,8887 +8887,8887 +8690,8887 +8510,8887 +8962,8887 +8888,8888 +9480,8888 +9251,8888 +8691,8888 +8511,8888 +8889,8889 +9481,8889 +9252,8889 +8890,8890 +8891,8891 +8892,8892 +8893,8893 +8894,8894 +8895,8895 +8896,8896 +8897,8897 +8898,8898 +8899,8899 +8876,8899 +8900,8900 +8901,8901 +8902,8902 +8903,8903 +8880,8903 +8904,8904 +8881,8904 +8942,8942 +8943,8943 +8944,8944 +57966,8944 +8945,8945 +8946,8946 +8947,8947 +228968,8947 +8948,8948 +8949,8949 +8950,8950 +8951,8951 +8952,8952 +8953,8953 +8954,8954 +8955,8955 +8956,8956 +8957,8957 +9474,8957 +10206,8958 +8958,8958 +10207,8959 +8959,8959 +10208,8960 +8960,8960 +8961,8961 +9478,8961 +8962,8962 +9479,8962 +9250,8962 +8887,8962 +8690,8962 +8510,8962 +8963,8963 +8964,8964 +8965,8965 +8966,8966 +8967,8967 +8968,8968 +8969,8969 +8970,8970 +8971,8971 +8972,8972 +4574,8972 +8973,8973 +8974,8974 +8975,8975 +8976,8976 +8977,8977 +8978,8978 +8979,8979 +8980,8980 +8981,8981 +8982,8982 +8983,8983 +8984,8984 +8985,8985 +8986,8986 +8987,8987 +239530,8987 +8988,8988 +8989,8989 +8990,8990 +144028,8990 +8991,8991 +8529,8991 +8349,8991 +8992,8992 +8993,8993 +8994,8994 +8995,8995 +8996,8996 +8997,8997 +8998,8998 +8999,8999 +9000,9000 +9001,9001 +9002,9002 +9003,9003 +209963,9003 +9004,9004 +3780,9004 +9005,9005 +9006,9006 +9007,9007 +9008,9008 +2317,9008 +9009,9009 +9010,9010 +9011,9011 +9012,9012 +9013,9013 +9014,9014 +9015,9015 +9016,9016 +9017,9017 +9018,9018 +9019,9019 +7714,9019 +9020,9020 +9021,9021 +9022,9022 +9023,9023 +233509,9023 +9024,9024 +9025,9025 +149263,9025 +9026,9026 +9027,9027 +9028,9028 +9029,9029 +9030,9030 +9031,9031 +9032,9032 +76510,9032 +9033,9033 +9034,9034 +9035,9035 +9036,9036 +9037,9037 +9038,9038 +9039,9039 +9040,9040 +154816,9040 +9041,9041 +9042,9042 +9043,9043 +9044,9044 +9045,9045 +9046,9046 +9047,9047 +24595,9047 +9048,9048 +9049,9049 +9050,9050 +9051,9051 +9052,9052 +9053,9053 +9054,9054 +9055,9055 +9056,9056 +9057,9057 +9058,9058 +9059,9059 +9060,9060 +113525,9060 +9061,9061 +9062,9062 +201124,9062 +9063,9063 +9064,9064 +9065,9065 +9066,9066 +9067,9067 +24545,9067 +9068,9068 +228729,9069 +9069,9069 +9070,9070 +83234,9070 +9071,9071 +9072,9072 +235965,9072 +9073,9073 +9074,9074 +9075,9075 +9076,9076 +9077,9077 +8627,9077 +8447,9077 +9078,9078 +9079,9079 +9080,9080 +9081,9081 +174357,9081 +168393,9081 +9082,9082 +9083,9083 +9084,9084 +9085,9085 +120265,9085 +9086,9086 +141612,9086 +9087,9087 +244763,9087 +9088,9088 +9089,9089 +9090,9090 +191122,9090 +9091,9091 +9092,9092 +179201,9092 +37050,9092 +9093,9093 +217561,9093 +9094,9094 +9095,9095 +108545,9095 +9096,9096 +9097,9097 +184811,9097 +9098,9098 +9099,9099 +9100,9100 +9101,9101 +9102,9102 +9103,9103 +9104,9104 +9105,9105 +9106,9106 +112212,9106 +9107,9107 +9108,9108 +9109,9109 +9110,9110 +9111,9111 +9112,9112 +9113,9113 +9114,9114 +9115,9115 +107074,9115 +9116,9116 +181238,9117 +9117,9117 +9118,9118 +9119,9119 +9120,9120 +9121,9121 +9122,9122 +87375,9123 +9123,9123 +9124,9124 +9125,9125 +9126,9126 +9127,9127 +9128,9128 +9129,9129 +9130,9130 +9131,9131 +9132,9132 +9133,9133 +9134,9134 +57052,9134 +9135,9135 +9136,9136 +60364,9136 +9137,9137 +9138,9138 +16822,9138 +9139,9139 +9188,9188 +9189,9189 +9190,9190 +9191,9191 +9192,9192 +9193,9193 +55086,9193 +46982,9193 +9194,9194 +9195,9195 +9196,9196 +9197,9197 +9198,9198 +9199,9199 +9200,9200 +9201,9201 +9202,9202 +9203,9203 +249614,9203 +9204,9204 +9205,9205 +9206,9206 +9207,9207 +9208,9208 +9209,9209 +9210,9210 +9211,9211 +9212,9212 +9213,9213 +9214,9214 +9215,9215 +9216,9216 +242782,9216 +9217,9217 +9218,9218 +9219,9219 +8857,9219 +151569,9219 +9220,9220 +9221,9221 +9222,9222 +9223,9223 +9224,9224 +9225,9225 +202767,9225 +9226,9226 +9227,9227 +9228,9228 +9229,9229 +9230,9230 +9231,9231 +9232,9232 +9233,9233 +9234,9234 +9235,9235 +9236,9236 +9237,9237 +9238,9238 +9239,9239 +9240,9240 +9241,9241 +9242,9242 +9243,9243 +9244,9244 +9245,9245 +9246,9246 +9247,9247 +9248,9248 +9249,9249 +9478,9249 +9479,9250 +9250,9250 +8887,9250 +8690,9250 +8510,9250 +8962,9250 +9480,9251 +9251,9251 +8691,9251 +8511,9251 +8888,9251 +9481,9252 +9252,9252 +8889,9252 +9482,9253 +9253,9253 +9254,9254 +9483,9254 +9255,9255 +98336,9255 +9256,9256 +86459,9256 +9257,9257 +207780,9258 +9258,9258 +135279,9258 +127692,9258 +11197,9258 +207870,9258 +9259,9259 +9260,9260 +9261,9261 +9262,9262 +9263,9263 +2833,9263 +9264,9264 +9265,9265 +9266,9266 +9267,9267 +9268,9268 +9269,9269 +9270,9270 +9271,9271 +9272,9272 +9273,9273 +9274,9274 +9275,9275 +9276,9276 +9277,9277 +9278,9278 +231740,9278 +9279,9279 +9280,9280 +9281,9281 +9282,9282 +9283,9283 +9284,9284 +5769,9284 +9285,9285 +9286,9286 +9287,9287 +9288,9288 +9289,9289 +9290,9290 +9335,9335 +9336,9336 +9337,9337 +9338,9338 +206636,9338 +208647,9338 +9339,9339 +9340,9340 +9341,9341 +9342,9342 +9343,9343 +9344,9344 +248735,9344 +9345,9345 +9346,9346 +180219,9346 +9347,9347 +9348,9348 +180342,9349 +9349,9349 +9350,9350 +9351,9351 +9352,9352 +9353,9353 +50240,9353 +42048,9353 +9354,9354 +9355,9355 +9356,9356 +9357,9357 +9358,9358 +9359,9359 +5808,9359 +9360,9360 +9361,9361 +9362,9362 +23917,9362 +199549,9362 +9363,9363 +9364,9364 +9365,9365 +9366,9366 +9454,9454 +9455,9455 +78780,9455 +9456,9456 +9457,9457 +15383,9458 +9458,9458 +176285,9458 +170321,9458 +9459,9459 +9460,9460 +9461,9461 +9462,9462 +111787,9462 +206538,9462 +9463,9463 +139180,9463 +9464,9464 +9465,9465 +211804,9465 +208777,9465 +9466,9466 +9467,9467 +9468,9468 +9469,9469 +9470,9470 +9471,9471 +9472,9472 +9473,9473 +9474,9474 +33848,9474 +8882,9474 +8957,9474 +9475,9475 +8883,9475 +205098,9475 +9476,9476 +145135,9476 +8884,9476 +9477,9477 +8885,9477 +9478,9478 +8961,9478 +8886,9478 +9249,9478 +9479,9479 +9250,9479 +8887,9479 +8690,9479 +8510,9479 +8962,9479 +9480,9480 +9251,9480 +8691,9480 +8511,9480 +8888,9480 +9481,9481 +9252,9481 +8889,9481 +9482,9482 +9253,9482 +9483,9483 +9254,9483 +9484,9484 +9485,9485 +9486,9486 +9487,9487 +15075,9487 +9488,9488 +9489,9489 +9490,9490 +9491,9491 +9492,9492 +9493,9493 +9494,9494 +9495,9495 +9496,9496 +9497,9497 +9498,9498 +9499,9499 +9500,9500 +9623,9623 +9624,9624 +226642,9624 +9625,9625 +9626,9626 +9627,9627 +9628,9628 +9667,9667 +9668,9668 +9669,9669 +9670,9670 +9671,9671 +9672,9672 +9673,9673 +9674,9674 +9675,9675 +9676,9676 +84192,9676 +9677,9677 +9678,9678 +9679,9679 +9680,9680 +9681,9681 +86572,9681 +9682,9682 +9683,9683 +9684,9684 +205310,9684 +9685,9685 +9686,9686 +9687,9687 +9688,9688 +9689,9689 +9690,9690 +9691,9691 +9692,9692 +9693,9693 +9694,9694 +9695,9695 +9696,9696 +9697,9697 +9698,9698 +9699,9699 +9700,9700 +9701,9701 +9702,9702 +9703,9703 +9704,9704 +9705,9705 +9706,9706 +9707,9707 +9708,9708 +9709,9709 +9710,9710 +9711,9711 +9712,9712 +9713,9713 +9714,9714 +9715,9715 +9716,9716 +70271,9716 +9717,9717 +9718,9718 +9719,9719 +9720,9720 +9721,9721 +163403,9721 +9722,9722 +9723,9723 +9724,9724 +20158,9724 +9725,9725 +9726,9726 +9727,9727 +9728,9728 +9729,9729 +9730,9730 +9731,9731 +9732,9732 +9733,9733 +9734,9734 +9735,9735 +9736,9736 +9737,9737 +9738,9738 +9739,9739 +9740,9740 +9741,9741 +210199,9741 +9742,9742 +235019,9742 +9743,9743 +9744,9744 +9745,9745 +9746,9746 +9747,9747 +193040,9748 +9748,9748 +9749,9749 +9750,9750 +9751,9751 +9752,9752 +9753,9753 +9754,9754 +9755,9755 +2703,9755 +9756,9756 +9757,9757 +9758,9758 +9759,9759 +9760,9760 +9761,9761 +9762,9762 +9763,9763 +9764,9764 +9765,9765 +9766,9766 +9767,9767 +9768,9768 +9769,9769 +9770,9770 +9771,9771 +9772,9772 +9773,9773 +9774,9774 +206753,9774 +9775,9775 +9776,9776 +9777,9777 +9778,9778 +9779,9779 +9780,9780 +9781,9781 +9782,9782 +9783,9783 +9784,9784 +9785,9785 +9786,9786 +9787,9787 +9788,9788 +9789,9789 +9790,9790 +9791,9791 +9792,9792 +9793,9793 +9794,9794 +9795,9795 +179530,9795 +9796,9796 +9797,9797 +9798,9798 +9799,9799 +9800,9800 +9801,9801 +9802,9802 +9803,9803 +9804,9804 +9805,9805 +9806,9806 +9807,9807 +9808,9808 +9809,9809 +9810,9810 +9811,9811 +9812,9812 +9813,9813 +9814,9814 +9815,9815 +9816,9816 +9817,9817 +9818,9818 +9819,9819 +9820,9820 +9821,9821 +9822,9822 +9823,9823 +9824,9824 +9825,9825 +9826,9826 +9827,9827 +75585,9827 +9828,9828 +9829,9829 +55759,9829 +47655,9829 +9830,9830 +9831,9831 +9832,9832 +9833,9833 +9834,9834 +9835,9835 +9836,9836 +9837,9837 +9838,9838 +9839,9839 +9840,9840 +9841,9841 +9842,9842 +9843,9843 +9844,9844 +9845,9845 +9846,9846 +231240,9847 +9847,9847 +9848,9848 +9849,9849 +9850,9850 +9851,9851 +9852,9852 +9853,9853 +9854,9854 +9855,9855 +9856,9856 +9857,9857 +9858,9858 +9859,9859 +9860,9860 +9861,9861 +9862,9862 +9863,9863 +9864,9864 +9865,9865 +9866,9866 +9867,9867 +154342,9867 +9868,9868 +9869,9869 +9870,9870 +9907,9907 +9908,9908 +243311,9908 +9909,9909 +9910,9910 +9911,9911 +143314,9911 +9912,9912 +9913,9913 +9914,9914 +9915,9915 +9916,9916 +9917,9917 +9918,9918 +9919,9919 +9920,9920 +9921,9921 +9922,9922 +9923,9923 +9924,9924 +9925,9925 +9926,9926 +9927,9927 +9928,9928 +9929,9929 +9930,9930 +9931,9931 +9932,9932 +9933,9933 +9934,9934 +9935,9935 +9936,9936 +109600,9937 +9937,9937 +9938,9938 +9939,9939 +9940,9940 +108766,9940 +9941,9941 +9942,9942 +9943,9943 +9992,9992 +9993,9993 +9994,9994 +9995,9995 +9996,9996 +9997,9997 +9998,9998 +9999,9999 +10000,10000 +111574,10000 +10001,10001 +10002,10002 +10003,10003 +10004,10004 +10005,10005 +10006,10006 +10007,10007 +10008,10008 +10009,10009 +10010,10010 +10011,10011 +10012,10012 +10013,10013 +10014,10014 +10015,10015 +10016,10016 +10017,10017 +10018,10018 +30243,10018 +10019,10019 +10020,10020 +10021,10021 +10060,10060 +10061,10061 +10062,10062 +10063,10063 +10064,10064 +10065,10065 +60112,10065 +10066,10066 +10067,10067 +10068,10068 +10069,10069 +10070,10070 +10071,10071 +222018,10071 +10072,10072 +10073,10073 +10074,10074 +10075,10075 +10076,10076 +10077,10077 +10078,10078 +10079,10079 +10080,10080 +10141,10141 +10142,10142 +10143,10143 +10144,10144 +10145,10145 +10146,10146 +10147,10147 +234694,10147 +10148,10148 +10149,10149 +53867,10149 +45763,10149 +10150,10150 +10151,10151 +10152,10152 +10153,10153 +10154,10154 +10155,10155 +10156,10156 +3090,10156 +10157,10157 +10158,10158 +10159,10159 +33835,10159 +10160,10160 +10161,10161 +194725,10161 +10162,10162 +10163,10163 +10164,10164 +10165,10165 +10166,10166 +184436,10166 +10167,10167 +10168,10168 +10169,10169 +14354,10169 +10170,10170 +10171,10171 +10172,10172 +10173,10173 +10174,10174 +10175,10175 +10176,10176 +10177,10177 +10178,10178 +10179,10179 +10180,10180 +10181,10181 +10182,10182 +10183,10183 +10184,10184 +10185,10185 +10186,10186 +10187,10187 +10188,10188 +10189,10189 +10190,10190 +179881,10190 +10191,10191 +10192,10192 +73022,10193 +10193,10193 +10194,10194 +10195,10195 +10196,10196 +10197,10197 +10198,10198 +10199,10199 +10200,10200 +10201,10201 +10202,10202 +108678,10202 +92649,10202 +10203,10203 +10204,10204 +10205,10205 +10206,10206 +8958,10206 +10207,10207 +8959,10207 +10208,10208 +8960,10208 +10248,10248 +10249,10249 +10250,10250 +10251,10251 +10252,10252 +10253,10253 +10254,10254 +10255,10255 +10256,10256 +10257,10257 +10258,10258 +10259,10259 +10260,10260 +10261,10261 +10262,10262 +10263,10263 +10264,10264 +10265,10265 +10354,10354 +10355,10355 +246937,10355 +10456,10456 +10457,10457 +206072,10457 +10458,10458 +10459,10459 +10460,10460 +10461,10461 +10462,10462 +10463,10463 +84293,10463 +10464,10464 +10465,10465 +10466,10466 +10467,10467 +10468,10468 +10469,10469 +10470,10470 +10471,10471 +10472,10472 +10473,10473 +10474,10474 +10475,10475 +10476,10476 +10477,10477 +10478,10478 +10479,10479 +10480,10480 +10481,10481 +110814,10481 +10482,10482 +10483,10483 +10484,10484 +10485,10485 +27081,10485 +10486,10486 +10487,10487 +17216,10487 +30516,10487 +10488,10488 +135113,10488 +127526,10488 +10489,10489 +10490,10490 +10491,10491 +10492,10492 +10493,10493 +10494,10494 +10495,10495 +10496,10496 +3731,10496 +10497,10497 +10498,10498 +10499,10499 +180061,10499 +10500,10500 +10501,10501 +10502,10502 +10503,10503 +10504,10504 +10505,10505 +10506,10506 +10507,10507 +10508,10508 +10509,10509 +10510,10510 +10511,10511 +10512,10512 +10513,10513 +10514,10514 +10515,10515 +10516,10516 +10517,10517 +10518,10518 +10519,10519 +10520,10520 +80632,10520 +10521,10521 +10522,10522 +50340,10522 +42148,10522 +10523,10523 +10524,10524 +10525,10525 +10526,10526 +10527,10527 +162617,10527 +10528,10528 +10529,10529 +10530,10530 +36187,10530 +10531,10531 +10532,10532 +70436,10532 +10533,10533 +10534,10534 +10535,10535 +229806,10536 +10536,10536 +10537,10537 +10538,10538 +10539,10539 +10540,10540 +10541,10541 +10542,10542 +10543,10543 +10544,10544 +10545,10545 +10546,10546 +10547,10547 +10548,10548 +10549,10549 +10550,10550 +185500,10550 +10551,10551 +10552,10552 +202723,10552 +10553,10553 +10554,10554 +10555,10555 +10556,10556 +10557,10557 +10558,10558 +10559,10559 +10560,10560 +207290,10560 +10561,10561 +10562,10562 +10563,10563 +10564,10564 +75426,10564 +10565,10565 +10566,10566 +10567,10567 +10568,10568 +10569,10569 +10570,10570 +10571,10571 +10572,10572 +161291,10572 +10573,10573 +10574,10574 +10575,10575 +10576,10576 +10577,10577 +10578,10578 +10579,10579 +10580,10580 +165059,10580 +10803,10803 +10804,10804 +10805,10805 +67986,10805 +10806,10806 +10807,10807 +3513,10807 +10808,10808 +10809,10809 +10855,10855 +10856,10856 +10857,10857 +10858,10858 +10859,10859 +10860,10860 +10861,10861 +10862,10862 +10863,10863 +10864,10864 +10865,10865 +10866,10866 +10867,10867 +10868,10868 +10869,10869 +10870,10870 +10871,10871 +162575,10872 +10872,10872 +10873,10873 +10874,10874 +10875,10875 +118123,10875 +10876,10876 +10877,10877 +10878,10878 +10879,10879 +10880,10880 +10881,10881 +10882,10882 +10883,10883 +10884,10884 +10885,10885 +10886,10886 +10887,10887 +189874,10887 +10888,10888 +104866,10888 +10889,10889 +10890,10890 +10891,10891 +10892,10892 +10893,10893 +10952,10952 +10953,10953 +10954,10954 +10955,10955 +10956,10956 +10957,10957 +10958,10958 +10959,10959 +10960,10960 +92054,10960 +10961,10961 +10962,10962 +10963,10963 +10964,10964 +10965,10965 +10966,10966 +10967,10967 +10968,10968 +10969,10969 +10970,10970 +10971,10971 +10972,10972 +10973,10973 +23498,10973 +10974,10974 +133211,10974 +125624,10974 +10975,10975 +10976,10976 +10977,10977 +10978,10978 +10979,10979 +10980,10980 +10981,10981 +10982,10982 +10983,10983 +10984,10984 +10985,10985 +10986,10986 +11069,11069 +11070,11070 +6075,11070 +11071,11071 +11072,11072 +11073,11073 +11074,11074 +11075,11075 +11076,11076 +11077,11077 +11078,11078 +11079,11079 +11080,11080 +11081,11081 +11082,11082 +11083,11083 +11084,11084 +11085,11085 +11086,11086 +11087,11087 +11088,11088 +11089,11089 +11090,11090 +11091,11091 +11092,11092 +11093,11093 +11094,11094 +11095,11095 +11096,11096 +11097,11097 +11098,11098 +11099,11099 +11100,11100 +11101,11101 +200551,11101 +11155,11155 +11156,11156 +11157,11157 +11158,11158 +11159,11159 +154110,11159 +11160,11160 +11161,11161 +11162,11162 +11163,11163 +11164,11164 +11165,11165 +11166,11166 +11167,11167 +11168,11168 +11169,11169 +11170,11170 +11171,11171 +11172,11172 +11173,11173 +11174,11174 +11175,11175 +11176,11176 +11177,11177 +11178,11178 +11179,11179 +11180,11180 +11181,11181 +11182,11182 +11183,11183 +11184,11184 +11185,11185 +230703,11186 +11186,11186 +11187,11187 +11188,11188 +11189,11189 +11190,11190 +183564,11190 +11191,11191 +208778,11191 +11192,11192 +207004,11193 +11193,11193 +11194,11194 +11195,11195 +11196,11196 +11197,11197 +207780,11197 +9258,11197 +11198,11198 +11199,11199 +11200,11200 +11201,11201 +11202,11202 +11295,11295 +11296,11296 +11297,11297 +11343,11343 +11344,11344 +11345,11345 +11346,11346 +11347,11347 +11348,11348 +11349,11349 +11350,11350 +11351,11351 +11352,11352 +11353,11353 +11354,11354 +11355,11355 +11356,11356 +11357,11357 +11358,11358 +11359,11359 +11360,11360 +11361,11361 +11362,11362 +11363,11363 +11364,11364 +11365,11365 +11366,11366 +102098,11366 +11367,11367 +11368,11368 +181311,11368 +11369,11369 +11370,11370 +11371,11371 +11372,11372 +11373,11373 +97149,11373 +11374,11374 +11375,11375 +4593,11375 +11376,11376 +11377,11377 +11378,11378 +233715,11378 +11379,11379 +11380,11380 +11381,11381 +11382,11382 +109525,11382 +11383,11383 +11384,11384 +11385,11385 +11386,11386 +11445,11445 +11446,11446 +11447,11447 +11448,11448 +11449,11449 +179118,11449 +11450,11450 +11451,11451 +11452,11452 +11453,11453 +59148,11453 +58774,11453 +11454,11454 +11455,11455 +11456,11456 +11457,11457 +11458,11458 +11459,11459 +11460,11460 +11461,11461 +11462,11462 +11463,11463 +11464,11464 +11465,11465 +186320,11465 +11466,11466 +11467,11467 +154608,11467 +11468,11468 +21187,11468 +11469,11469 +3564,11469 +11470,11470 +11471,11471 +11472,11472 +11473,11473 +11474,11474 +11475,11475 +11476,11476 +11477,11477 +11478,11478 +11479,11479 +11480,11480 +11481,11481 +11482,11482 +11483,11483 +11484,11484 +193569,11484 +11485,11485 +11486,11486 +11487,11487 +11488,11488 +11489,11489 +11490,11490 +11491,11491 +11492,11492 +11493,11493 +11494,11494 +11495,11495 +11496,11496 +11497,11497 +11498,11498 +11499,11499 +119288,11500 +11500,11500 +11501,11501 +11502,11502 +11503,11503 +198037,11503 +11504,11504 +11505,11505 +11506,11506 +11507,11507 +11508,11508 +11509,11509 +11510,11510 +68661,11510 +11511,11511 +11512,11512 +11513,11513 +11514,11514 +2351,11514 +11515,11515 +11516,11516 +11517,11517 +11518,11518 +11519,11519 +11520,11520 +11521,11521 +11522,11522 +11523,11523 +11524,11524 +11525,11525 +7014,11525 +11526,11526 +11527,11527 +11528,11528 +11529,11529 +11530,11530 +11531,11531 +11532,11532 +11533,11533 +11534,11534 +11535,11535 +138966,11535 +11536,11536 +11537,11537 +11538,11538 +11539,11539 +11540,11540 +11541,11541 +11542,11542 +11543,11543 +11544,11544 +11545,11545 +11546,11546 +11547,11547 +11548,11548 +11549,11549 +11550,11550 +11551,11551 +11552,11552 +11553,11553 +11554,11554 +11555,11555 +93862,11555 +11556,11556 +11557,11557 +11558,11558 +11559,11559 +11560,11560 +11561,11561 +132597,11561 +125010,11561 +11562,11562 +28459,11563 +11563,11563 +107126,11563 +11564,11564 +11565,11565 +11566,11566 +11567,11567 +11568,11568 +11569,11569 +11570,11570 +11571,11571 +11572,11572 +11573,11573 +11574,11574 +11575,11575 +11576,11576 +11577,11577 +11578,11578 +11579,11579 +11580,11580 +11581,11581 +11582,11582 +11583,11583 +11584,11584 +11585,11585 +11586,11586 +11587,11587 +11588,11588 +11589,11589 +11590,11590 +11591,11591 +11592,11592 +11593,11593 +11594,11594 +11595,11595 +217690,11595 +11596,11596 +11597,11597 +11598,11598 +11599,11599 +11600,11600 +11601,11601 +11602,11602 +11603,11603 +233715,11603 +11604,11604 +11605,11605 +11606,11606 +11607,11607 +11608,11608 +11609,11609 +11610,11610 +74069,11610 +7759,11610 +11611,11611 +11612,11612 +11613,11613 +11614,11614 +11615,11615 +11616,11616 +11617,11617 +11618,11618 +11619,11619 +11620,11620 +11621,11621 +11622,11622 +11623,11623 +11624,11624 +11625,11625 +248048,11625 +11626,11626 +120,11626 +11627,11627 +11628,11628 +54616,11628 +46512,11628 +232003,11628 +11629,11629 +240424,11629 +11630,11630 +11763,11763 +11764,11764 +11765,11765 +11766,11766 +11767,11767 +11768,11768 +11769,11769 +11770,11770 +11771,11771 +11772,11772 +11773,11773 +11774,11774 +11775,11775 +11824,11824 +11825,11825 +11826,11826 +11827,11827 +11828,11828 +11829,11829 +11830,11830 +11831,11831 +205246,11831 +11832,11832 +11833,11833 +11834,11834 +11835,11835 +11836,11836 +11837,11837 +11838,11838 +11839,11839 +11840,11840 +145386,11840 +11841,11841 +11842,11842 +11843,11843 +11844,11844 +11845,11845 +11846,11846 +11847,11847 +11848,11848 +11849,11849 +11850,11850 +56291,11850 +48187,11850 +11851,11851 +11852,11852 +11853,11853 +11854,11854 +11855,11855 +11856,11856 +11857,11857 +11858,11858 +11859,11859 +11860,11860 +11861,11861 +235079,11861 +11862,11862 +11863,11863 +11864,11864 +11865,11865 +4208,11865 +11866,11866 +11867,11867 +36684,11867 +11868,11868 +11869,11869 +11870,11870 +11871,11871 +11872,11872 +11873,11873 +11874,11874 +11875,11875 +11876,11876 +11877,11877 +76071,11877 +11878,11878 +11879,11879 +11880,11880 +108328,11881 +11881,11881 +11882,11882 +11883,11883 +11884,11884 +11885,11885 +11886,11886 +11887,11887 +11888,11888 +11889,11889 +11890,11890 +11891,11891 +11892,11892 +11893,11893 +11894,11894 +11895,11895 +11896,11896 +11897,11897 +11898,11898 +11899,11899 +11900,11900 +11901,11901 +11902,11902 +144036,11902 +11903,11903 +11904,11904 +11905,11905 +11906,11906 +11907,11907 +11908,11908 +11909,11909 +11910,11910 +11911,11911 +11912,11912 +11913,11913 +11914,11914 +11915,11915 +11916,11916 +73032,11916 +11917,11917 +11918,11918 +11919,11919 +11920,11920 +11921,11921 +11922,11922 +11923,11923 +11924,11924 +11925,11925 +11926,11926 +11927,11927 +7050,11927 +11928,11928 +11929,11929 +11930,11930 +11931,11931 +11932,11932 +11933,11933 +11934,11934 +11935,11935 +220191,11935 +11936,11936 +11937,11937 +11938,11938 +11939,11939 +11940,11940 +11941,11941 +11942,11942 +11943,11943 +11944,11944 +11945,11945 +11946,11946 +11947,11947 +11948,11948 +11949,11949 +11950,11950 +11951,11951 +11952,11952 +11953,11953 +11954,11954 +11955,11955 +11956,11956 +11957,11957 +11958,11958 +11959,11959 +11960,11960 +11961,11961 +11962,11962 +11963,11963 +11964,11964 +11965,11965 +11966,11966 +11967,11967 +151539,11967 +214249,11967 +11968,11968 +11969,11969 +11970,11970 +11971,11971 +11972,11972 +11973,11973 +11974,11974 +11975,11975 +11976,11976 +11977,11977 +11978,11978 +11979,11979 +201693,11979 +11980,11980 +11981,11981 +11982,11982 +11983,11983 +11984,11984 +11985,11985 +11986,11986 +11987,11987 +11988,11988 +11989,11989 +86556,11989 +11990,11990 +11991,11991 +84187,11991 +11992,11992 +11993,11993 +164676,11993 +11994,11994 +11995,11995 +31677,11995 +11996,11996 +213655,11996 +11997,11997 +11998,11998 +11999,11999 +12000,12000 +12001,12001 +12002,12002 +12003,12003 +12004,12004 +12005,12005 +12006,12006 +12007,12007 +12008,12008 +12009,12009 +12010,12010 +12011,12011 +12012,12012 +12013,12013 +189607,12013 +12014,12014 +12015,12015 +12016,12016 +12240,12016 +12017,12017 +12018,12018 +12019,12019 +12045,12045 +12046,12046 +36916,12046 +12047,12047 +12048,12048 +12049,12049 +12050,12050 +12051,12051 +12052,12052 +12053,12053 +12054,12054 +12055,12055 +12056,12056 +12057,12057 +12058,12058 +12059,12059 +12060,12060 +12061,12061 +12062,12062 +12063,12063 +123509,12063 +12064,12064 +52069,12064 +43965,12064 +12065,12065 +12066,12066 +12067,12067 +12068,12068 +12069,12069 +12070,12070 +12071,12071 +12072,12072 +12073,12073 +12074,12074 +12075,12075 +12076,12076 +12077,12077 +12078,12078 +12079,12079 +36278,12079 +12080,12080 +12081,12081 +12082,12082 +12083,12083 +12084,12084 +12085,12085 +12086,12086 +12087,12087 +12088,12088 +12089,12089 +12090,12090 +64984,12090 +12091,12091 +12092,12092 +12140,12140 +32388,12140 +12141,12141 +12142,12142 +12143,12143 +12144,12144 +12145,12145 +144195,12145 +12146,12146 +12147,12147 +12148,12148 +12149,12149 +12150,12150 +245413,12150 +12151,12151 +12152,12152 +12153,12153 +12154,12154 +109760,12154 +12155,12155 +12156,12156 +12157,12157 +12158,12158 +81872,12158 +12159,12159 +12160,12160 +12161,12161 +162428,12161 +12162,12162 +12163,12163 +12164,12164 +243383,12164 +12165,12165 +12166,12166 +12167,12167 +12168,12168 +12169,12169 +12170,12170 +12171,12171 +12172,12172 +12173,12173 +12174,12174 +12175,12175 +76486,12175 +12176,12176 +12177,12177 +12178,12178 +12179,12179 +12180,12180 +12181,12181 +12182,12182 +12183,12183 +57818,12183 +12184,12184 +12185,12185 +12186,12186 +12187,12187 +12188,12188 +12189,12189 +12190,12190 +12191,12191 +5384,12191 +12192,12192 +12193,12193 +12236,12236 +12237,12237 +12238,12238 +12239,12239 +12240,12240 +150626,12240 +12016,12240 +12241,12241 +12242,12242 +12243,12243 +12244,12244 +12245,12245 +12246,12246 +12247,12247 +12248,12248 +12249,12249 +12250,12250 +218304,12251 +12251,12251 +12252,12252 +12253,12253 +246753,12253 +12254,12254 +12255,12255 +12256,12256 +12257,12257 +12295,12295 +12296,12296 +12342,12342 +12343,12343 +12344,12344 +12345,12345 +12346,12346 +12347,12347 +12348,12348 +187469,12348 +12349,12349 +12350,12350 +12351,12351 +12352,12352 +12353,12353 +12354,12354 +12355,12355 +12356,12356 +12357,12357 +12358,12358 +12359,12359 +12360,12360 +12361,12361 +12362,12362 +12363,12363 +12364,12364 +12365,12365 +12366,12366 +12367,12367 +12368,12368 +12369,12369 +12370,12370 +83893,12370 +12371,12371 +12372,12372 +12373,12373 +12374,12374 +12375,12375 +12376,12376 +12377,12377 +12378,12378 +12379,12379 +12380,12380 +12381,12381 +12382,12382 +12383,12383 +12384,12384 +12385,12385 +12386,12386 +12387,12387 +12388,12388 +12389,12389 +12390,12390 +12391,12391 +12392,12392 +12393,12393 +12394,12394 +12395,12395 +12396,12396 +12397,12397 +12398,12398 +12399,12399 +12400,12400 +12401,12401 +12402,12402 +12403,12403 +12404,12404 +12405,12405 +12406,12406 +12407,12407 +12408,12408 +12451,12451 +12452,12452 +12453,12453 +12454,12454 +12455,12455 +12456,12456 +85210,12456 +12457,12457 +12458,12458 +12459,12459 +12460,12460 +12461,12461 +12462,12462 +12463,12463 +12464,12464 +12465,12465 +12466,12466 +12467,12467 +12468,12468 +12469,12469 +12470,12470 +12471,12471 +12472,12472 +12473,12473 +12474,12474 +79459,12475 +12475,12475 +12476,12476 +12477,12477 +12478,12478 +12479,12479 +12480,12480 +12481,12481 +12482,12482 +12483,12483 +12484,12484 +12485,12485 +12486,12486 +12487,12487 +12488,12488 +12489,12489 +12490,12490 +12491,12491 +12492,12492 +12493,12493 +12494,12494 +12540,12540 +12541,12541 +12542,12542 +12635,12635 +12636,12636 +12637,12637 +12638,12638 +12639,12639 +12640,12640 +12641,12641 +216865,12641 +12642,12642 +246608,12642 +12643,12643 +12644,12644 +12645,12645 +12646,12646 +12647,12647 +12648,12648 +12649,12649 +12650,12650 +12651,12651 +12652,12652 +110805,12652 +12653,12653 +12654,12654 +12655,12655 +114427,12655 +12656,12656 +12657,12657 +37021,12658 +12658,12658 +12659,12659 +12660,12660 +12661,12661 +12662,12662 +12663,12663 +12664,12664 +4156,12664 +12665,12665 +12666,12666 +12667,12667 +12668,12668 +12669,12669 +12670,12670 +12671,12671 +12672,12672 +12673,12673 +12674,12674 +12675,12675 +12676,12676 +12677,12677 +12678,12678 +12679,12679 +12680,12680 +12681,12681 +12682,12682 +12736,12736 +12737,12737 +12738,12738 +12739,12739 +12740,12740 +12741,12741 +12742,12742 +12743,12743 +12744,12744 +12745,12745 +12746,12746 +12747,12747 +12748,12748 +12749,12749 +12750,12750 +12751,12751 +12752,12752 +12753,12753 +12754,12754 +12755,12755 +12756,12756 +12757,12757 +12758,12758 +12759,12759 +12760,12760 +12761,12761 +12762,12762 +12763,12763 +12764,12764 +12765,12765 +4951,12765 +12766,12766 +12767,12767 +12768,12768 +12844,12844 +12845,12845 +12846,12846 +12847,12847 +12848,12848 +12849,12849 +12850,12850 +12851,12851 +12852,12852 +12853,12853 +12854,12854 +12855,12855 +12856,12856 +12857,12857 +12858,12858 +12859,12859 +12860,12860 +12861,12861 +12862,12862 +12863,12863 +12864,12864 +12865,12865 +12866,12866 +12867,12867 +12868,12868 +12869,12869 +12870,12870 +12871,12871 +12872,12872 +12873,12873 +12874,12874 +190029,12874 +12875,12875 +12876,12876 +12877,12877 +12937,12937 +12938,12938 +12939,12939 +12940,12940 +59904,12941 +12941,12941 +12942,12942 +12943,12943 +12944,12944 +12945,12945 +12946,12946 +12947,12947 +12948,12948 +12949,12949 +65743,12949 +34226,12949 +12950,12950 +12951,12951 +12952,12952 +12953,12953 +12954,12954 +12955,12955 +12956,12956 +12957,12957 +12958,12958 +162589,12958 +33831,12958 +12959,12959 +12960,12960 +12961,12961 +12962,12962 +12963,12963 +12964,12964 +12965,12965 +12966,12966 +12967,12967 +12968,12968 +12969,12969 +12970,12970 +12971,12971 +12972,12972 +12973,12973 +12974,12974 +12975,12975 +12976,12976 +13022,13022 +13023,13023 +13024,13024 +13025,13025 +13026,13026 +13027,13027 +13028,13028 +13250,13250 +961,13250 +13251,13251 +13252,13252 +13253,13253 +13254,13254 +13255,13255 +13256,13256 +13257,13257 +13258,13258 +13259,13259 +13260,13260 +24689,13260 +13261,13261 +13262,13262 +13263,13263 +13264,13264 +13265,13265 +13266,13266 +13267,13267 +13268,13268 +13269,13269 +13270,13270 +13271,13271 +13272,13272 +13273,13273 +13274,13274 +13275,13275 +13276,13276 +13277,13277 +13278,13278 +13279,13279 +13280,13280 +13281,13281 +13282,13282 +13283,13283 +13284,13284 +13285,13285 +13286,13286 +13287,13287 +13288,13288 +13289,13289 +13290,13290 +247005,13290 +13291,13291 +13292,13292 +13293,13293 +13294,13294 +13295,13295 +13296,13296 +13297,13297 +67029,13297 +13298,13298 +148809,13298 +13299,13299 +13300,13300 +84908,13300 +13301,13301 +235720,13301 +13302,13302 +13303,13303 +13304,13304 +13305,13305 +13306,13306 +13307,13307 +13308,13308 +13309,13309 +13310,13310 +13311,13311 +207882,13311 +13312,13312 +13313,13313 +13314,13314 +13315,13315 +13316,13316 +13317,13317 +13318,13318 +13319,13319 +13320,13320 +13368,13368 +13369,13369 +13370,13370 +13371,13371 +13372,13372 +13373,13373 +13374,13374 +13375,13375 +13376,13376 +13377,13377 +13378,13378 +13379,13379 +13380,13380 +13381,13381 +13382,13382 +13383,13383 +13384,13384 +13385,13385 +13386,13386 +83895,13386 +13387,13387 +13388,13388 +13389,13389 +13390,13390 +13391,13391 +13392,13392 +13454,13454 +13455,13455 +13456,13456 +13457,13457 +13458,13458 +13459,13459 +13460,13460 +13461,13461 +142888,13461 +6615,13461 +148176,13461 +13462,13462 +13463,13463 +13464,13464 +13465,13465 +13466,13466 +13467,13467 +120705,13468 +13468,13468 +13469,13469 +13470,13470 +13471,13471 +13472,13472 +13473,13473 +13474,13474 +13475,13475 +13476,13476 +13477,13477 +13478,13478 +13479,13479 +28892,13479 +13480,13480 +13481,13481 +13482,13482 +13483,13483 +13484,13484 +176778,13484 +170814,13484 +13485,13485 +234709,13485 +13486,13486 +13487,13487 +13488,13488 +13489,13489 +13490,13490 +13491,13491 +13492,13492 +13493,13493 +13494,13494 +2199,13494 +13495,13495 +13496,13496 +13497,13497 +13498,13498 +13499,13499 +13500,13500 +13501,13501 +13502,13502 +13503,13503 +107353,13503 +13504,13504 +13505,13505 +13506,13506 +13507,13507 +13508,13508 +13509,13509 +13510,13510 +13511,13511 +243484,13511 +13512,13512 +13513,13513 +13514,13514 +13515,13515 +13516,13516 +13517,13517 +13518,13518 +13519,13519 +13520,13520 +154769,13520 +13521,13521 +13656,13656 +13657,13657 +13658,13658 +20896,13658 +13659,13659 +13660,13660 +13661,13661 +13662,13662 +214423,13662 +13663,13663 +13664,13664 +13665,13665 +13666,13666 +13667,13667 +13668,13668 +13669,13669 +13670,13670 +13671,13671 +13672,13672 +13673,13673 +56180,13673 +48076,13673 +13674,13674 +13675,13675 +13676,13676 +97433,13676 +13677,13677 +13678,13678 +13679,13679 +13826,13826 +13827,13827 +13828,13828 +13829,13829 +13830,13830 +13831,13831 +13832,13832 +13833,13833 +13834,13834 +13835,13835 +164954,13835 +73046,13835 +13836,13836 +13837,13837 +13838,13838 +13839,13839 +13840,13840 +13841,13841 +13842,13842 +13843,13843 +13844,13844 +13845,13845 +13846,13846 +13847,13847 +13848,13848 +13849,13849 +13850,13850 +179390,13850 +13851,13851 +13852,13852 +13933,13933 +13997,13997 +13998,13998 +13999,13999 +14000,14000 +14001,14001 +14002,14002 +14003,14003 +14004,14004 +14005,14005 +7119,14005 +14006,14006 +14007,14007 +14098,14007 +14147,14008 +14099,14008 +14008,14008 +14009,14009 +14010,14010 +14011,14011 +14012,14012 +109555,14012 +14013,14013 +14014,14014 +14015,14015 +14016,14016 +14017,14017 +14064,14064 +14065,14065 +14066,14066 +14067,14067 +14068,14068 +14069,14069 +14070,14070 +14071,14071 +14072,14072 +14073,14073 +14074,14074 +14075,14075 +14076,14076 +14077,14077 +14078,14078 +14079,14079 +14080,14080 +14081,14081 +14082,14082 +14083,14083 +14084,14084 +14085,14085 +236943,14085 +14086,14086 +14087,14087 +14088,14088 +14089,14089 +14090,14090 +14091,14091 +14092,14092 +225386,14092 +14093,14093 +14094,14094 +14095,14095 +14096,14096 +14097,14097 +14098,14098 +14007,14098 +14147,14099 +14099,14099 +14008,14099 +14148,14100 +14100,14100 +14101,14101 +14102,14102 +14103,14103 +14104,14104 +14105,14105 +14106,14106 +159451,14106 +14107,14107 +14108,14108 +14109,14109 +14110,14110 +14111,14111 +200872,14111 +14112,14112 +109150,14112 +14113,14113 +14114,14114 +14115,14115 +35642,14116 +14116,14116 +14117,14117 +14118,14118 +14119,14119 +14120,14120 +14121,14121 +14122,14122 +14123,14123 +14124,14124 +184779,14124 +14125,14125 +14126,14126 +14127,14127 +14128,14128 +151630,14128 +14129,14129 +14130,14130 +14131,14131 +14132,14132 +14133,14133 +14134,14134 +14135,14135 +14136,14136 +14137,14137 +14138,14138 +14139,14139 +14140,14140 +14141,14141 +14142,14142 +14143,14143 +14144,14144 +14145,14145 +14146,14146 +14147,14147 +14099,14147 +14008,14147 +14148,14148 +14100,14148 +14149,14149 +70261,14149 +14150,14150 +14151,14151 +14329,14329 +14330,14330 +14331,14331 +14332,14332 +14333,14333 +14334,14334 +14335,14335 +14336,14336 +14337,14337 +14338,14338 +248782,14338 +14339,14339 +14340,14340 +207249,14340 +14341,14341 +14342,14342 +14343,14343 +14344,14344 +14345,14345 +14346,14346 +14347,14347 +14348,14348 +14349,14349 +14350,14350 +14351,14351 +14352,14352 +14353,14353 +14354,14354 +10169,14354 +89706,14354 +14355,14355 +14356,14356 +14357,14357 +14358,14358 +14359,14359 +14360,14360 +14361,14361 +14362,14362 +14363,14363 +14364,14364 +14365,14365 +14366,14366 +14405,14405 +14406,14406 +14407,14407 +14408,14408 +14409,14409 +23908,14409 +14410,14410 +14411,14411 +14412,14412 +14413,14413 +14414,14414 +14415,14415 +14416,14416 +14417,14417 +14418,14418 +56182,14418 +48078,14418 +14419,14419 +14420,14420 +14421,14421 +14422,14422 +14423,14423 +14424,14424 +14425,14425 +14426,14426 +14427,14427 +185406,14427 +14428,14428 +14429,14429 +14430,14430 +14431,14431 +14432,14432 +14433,14433 +30522,14434 +14434,14434 +14435,14435 +14436,14436 +14437,14437 +215500,14437 +14438,14438 +38236,14439 +35002,14439 +14439,14439 +14440,14440 +14441,14441 +14442,14442 +14443,14443 +14444,14444 +14445,14445 +14446,14446 +14447,14447 +14448,14448 +14449,14449 +14450,14450 +14451,14451 +14452,14452 +14453,14453 +14454,14454 +14455,14455 +225251,14455 +14456,14456 +14457,14457 +14458,14458 +14459,14459 +14460,14460 +14461,14461 +14462,14462 +236423,14462 +14463,14463 +14464,14464 +244837,14464 +14465,14465 +106737,14465 +14466,14466 +14467,14467 +14468,14468 +14469,14469 +14470,14470 +14471,14471 +14472,14472 +14473,14473 +14474,14474 +14475,14475 +14476,14476 +14477,14477 +14478,14478 +14479,14479 +14480,14480 +14481,14481 +14482,14482 +14483,14483 +14484,14484 +14485,14485 +14486,14486 +14487,14487 +14488,14488 +14489,14489 +14490,14490 +14491,14491 +14492,14492 +14493,14493 +14494,14494 +14495,14495 +14496,14496 +14497,14497 +14498,14498 +14499,14499 +14500,14500 +14501,14501 +14502,14502 +14503,14503 +14504,14504 +72603,14504 +14505,14505 +14506,14506 +89516,14506 +2159,14506 +14507,14507 +14508,14508 +14509,14509 +14510,14510 +14511,14511 +14512,14512 +14513,14513 +14514,14514 +14515,14515 +14516,14516 +14517,14517 +14518,14518 +14519,14519 +14520,14520 +14521,14521 +14522,14522 +14523,14523 +184082,14523 +14524,14524 +14525,14525 +14526,14526 +14527,14527 +14528,14528 +14529,14529 +14530,14530 +14531,14531 +14532,14532 +14533,14533 +14534,14534 +14535,14535 +150718,14536 +14536,14536 +14537,14537 +14538,14538 +14539,14539 +14540,14540 +73574,14540 +14541,14541 +14542,14542 +14543,14543 +119781,14543 +14544,14544 +14545,14545 +14546,14546 +14547,14547 +14548,14548 +173973,14548 +168009,14548 +14549,14549 +14550,14550 +14551,14551 +14552,14552 +14553,14553 +14554,14554 +231454,14554 +14555,14555 +122994,14555 +14556,14556 +14557,14557 +14558,14558 +14559,14559 +14560,14560 +14561,14561 +14562,14562 +14563,14563 +14564,14564 +14565,14565 +14566,14566 +14567,14567 +14568,14568 +14569,14569 +14570,14570 +14571,14571 +14572,14572 +14573,14573 +14574,14574 +14575,14575 +245838,14575 +241846,14575 +14576,14576 +55942,14576 +47838,14576 +109522,14576 +14577,14577 +14578,14578 +14579,14579 +14580,14580 +14581,14581 +14582,14582 +14583,14583 +14621,14621 +14622,14622 +14623,14623 +14624,14624 +14625,14625 +14626,14626 +14627,14627 +14628,14628 +216212,14628 +14629,14629 +14630,14630 +14631,14631 +14632,14632 +143016,14632 +14633,14633 +165393,14633 +14634,14634 +14635,14635 +14636,14636 +14637,14637 +14638,14638 +14639,14639 +75171,14639 +2043,14639 +14640,14640 +14641,14641 +56359,14641 +48255,14641 +14642,14642 +14643,14643 +14644,14644 +14645,14645 +14646,14646 +242350,14646 +14647,14647 +14648,14648 +14649,14649 +189109,14649 +14650,14650 +14651,14651 +14652,14652 +14653,14653 +14654,14654 +176414,14654 +170450,14654 +14655,14655 +14656,14656 +14657,14657 +14658,14658 +14659,14659 +14660,14660 +14661,14661 +14662,14662 +14663,14663 +14664,14664 +14665,14665 +14666,14666 +14667,14667 +14668,14668 +14717,14717 +14718,14718 +14719,14719 +14720,14720 +14721,14721 +14722,14722 +14723,14723 +14724,14724 +14725,14725 +14764,14764 +15046,15046 +15047,15047 +15048,15048 +15049,15049 +109928,15049 +15050,15050 +15051,15051 +64977,15052 +15052,15052 +15053,15053 +15054,15054 +15055,15055 +15056,15056 +15057,15057 +15058,15058 +15059,15059 +15060,15060 +15061,15061 +15062,15062 +15063,15063 +15064,15064 +15065,15065 +15066,15066 +15067,15067 +15068,15068 +226829,15068 +15069,15069 +15070,15070 +15071,15071 +15072,15072 +15073,15073 +15074,15074 +15075,15075 +9487,15075 +15076,15076 +15077,15077 +15078,15078 +15079,15079 +15080,15080 +15081,15081 +15082,15082 +15083,15083 +15084,15084 +15085,15085 +15086,15086 +15087,15087 +15088,15088 +15089,15089 +15090,15090 +15091,15091 +15092,15092 +15093,15093 +15094,15094 +15095,15095 +15096,15096 +208133,15096 +15097,15097 +15333,15333 +15334,15334 +15335,15335 +97713,15335 +15336,15336 +15337,15337 +15338,15338 +15339,15339 +15340,15340 +15341,15341 +15342,15342 +15343,15343 +15344,15344 +15345,15345 +15346,15346 +15347,15347 +15348,15348 +15349,15349 +15350,15350 +15351,15351 +15352,15352 +15353,15353 +15354,15354 +15355,15355 +15356,15356 +15357,15357 +15358,15358 +15359,15359 +15360,15360 +15361,15361 +15362,15362 +15363,15363 +179048,15363 +77449,15363 +15364,15364 +15365,15365 +15366,15366 +15519,15366 +15367,15367 +30690,15367 +15368,15368 +15369,15369 +15370,15370 +15371,15371 +67585,15371 +15372,15372 +15373,15373 +15374,15374 +15375,15375 +15376,15376 +15377,15377 +15530,15377 +15378,15378 +15379,15379 +15380,15380 +15381,15381 +15382,15382 +176284,15382 +170320,15382 +15383,15383 +9458,15383 +176285,15383 +170321,15383 +15384,15384 +157014,15384 +221656,15385 +15385,15385 +15386,15386 +15387,15387 +15388,15388 +15389,15389 +15390,15390 +15391,15391 +15392,15392 +15393,15393 +15394,15394 +15395,15395 +15396,15396 +15397,15397 +15398,15398 +15399,15399 +15400,15400 +15401,15401 +21957,15401 +15402,15402 +15403,15403 +15404,15404 +143171,15404 +159477,15404 +15405,15405 +15406,15406 +15407,15407 +15408,15408 +15409,15409 +15410,15410 +15411,15411 +57317,15411 +15412,15412 +15413,15413 +15414,15414 +15415,15415 +15416,15416 +6887,15416 +15417,15417 +15418,15418 +144030,15418 +15419,15419 +15420,15420 +15421,15421 +54756,15421 +46652,15421 +15422,15422 +15423,15423 +15424,15424 +15425,15425 +15426,15426 +15427,15427 +15428,15428 +15429,15429 +15430,15430 +15431,15431 +15432,15432 +88646,15432 +135915,15432 +128328,15432 +15433,15433 +15434,15434 +15435,15435 +15436,15436 +15437,15437 +15438,15438 +15439,15439 +15440,15440 +15441,15441 +15442,15442 +15443,15443 +15444,15444 +15445,15445 +15446,15446 +15447,15447 +15485,15485 +15486,15486 +15487,15487 +15488,15488 +15489,15489 +15490,15490 +15491,15491 +15492,15492 +15493,15493 +15494,15494 +15495,15495 +15496,15496 +15497,15497 +15498,15498 +15499,15499 +15500,15500 +15501,15501 +15502,15502 +15503,15503 +15504,15504 +15505,15505 +15506,15506 +15507,15507 +15508,15508 +15509,15509 +15510,15510 +15511,15511 +15512,15512 +15513,15513 +15514,15514 +15515,15515 +15516,15516 +15517,15517 +15518,15518 +15519,15519 +15366,15519 +15520,15520 +15521,15521 +15522,15522 +15523,15523 +15524,15524 +15525,15525 +15526,15526 +15527,15527 +15528,15528 +15529,15529 +15530,15530 +15377,15530 +15531,15531 +15532,15532 +15533,15533 +15534,15534 +15535,15535 +15536,15536 +15537,15537 +15538,15538 +15539,15539 +15540,15540 +15541,15541 +15542,15542 +15543,15543 +15544,15544 +15545,15545 +15546,15546 +15547,15547 +15548,15548 +15549,15549 +15550,15550 +15551,15551 +15552,15552 +15553,15553 +15554,15554 +15555,15555 +15556,15556 +15557,15557 +15558,15558 +15559,15559 +15560,15560 +15561,15561 +149107,15561 +15562,15562 +15563,15563 +15564,15564 +15565,15565 +15566,15566 +15567,15567 +15568,15568 +15569,15569 +15570,15570 +15571,15571 +15572,15572 +15573,15573 +15574,15574 +15575,15575 +15576,15576 +15577,15577 +15578,15578 +15579,15579 +15580,15580 +15581,15581 +15582,15582 +15583,15583 +15584,15584 +15585,15585 +388,15585 +15586,15586 +15587,15587 +15588,15588 +15589,15589 +15590,15590 +15591,15591 +15592,15592 +15593,15593 +15594,15594 +110306,15594 +15595,15595 +15596,15596 +15597,15597 +15598,15598 +15599,15599 +15600,15600 +179566,15600 +15601,15601 +15602,15602 +15603,15603 +15604,15604 +72886,15604 +15605,15605 +15606,15606 +15607,15607 +15608,15608 +15609,15609 +142762,15609 +15610,15610 +15611,15611 +38447,15611 +15612,15612 +15613,15613 +15685,15685 +15686,15686 +15687,15687 +15688,15688 +15689,15689 +15690,15690 +15691,15691 +15692,15692 +28973,15692 +15693,15693 +15694,15694 +15695,15695 +15696,15696 +15697,15697 +15698,15698 +15699,15699 +206855,15699 +15700,15700 +117685,15700 +15701,15701 +15702,15702 +15703,15703 +15704,15704 +15705,15705 +15706,15706 +220606,15706 +15707,15707 +15708,15708 +15709,15709 +235843,15709 +15710,15710 +15711,15711 +15712,15712 +15713,15713 +15714,15714 +15715,15715 +15716,15716 +15717,15717 +15718,15718 +15719,15719 +15720,15720 +15721,15721 +15722,15722 +15723,15723 +15724,15724 +15725,15725 +15726,15726 +135514,15726 +127927,15726 +15727,15727 +15728,15728 +15729,15729 +15730,15730 +15731,15731 +142475,15731 +15732,15732 +15733,15733 +15734,15734 +187430,15734 +15735,15735 +15736,15736 +15737,15737 +15738,15738 +15739,15739 +15740,15740 +244085,15740 +15741,15741 +15742,15742 +15743,15743 +140870,15743 +15744,15744 +15745,15745 +15746,15746 +15747,15747 +15748,15748 +143909,15748 +15749,15749 +15750,15750 +15751,15751 +15752,15752 +15753,15753 +35654,15753 +15754,15754 +15755,15755 +15817,15817 +15818,15818 +15819,15819 +15820,15820 +79449,15820 +15821,15821 +199291,15821 +15822,15822 +15823,15823 +246955,15823 +15824,15824 +15825,15825 +15826,15826 +15827,15827 +15828,15828 +15829,15829 +15830,15830 +15831,15831 +246607,15831 +15832,15832 +15833,15833 +15834,15834 +15835,15835 +15836,15836 +15837,15837 +15838,15838 +15839,15839 +153927,15839 +15840,15840 +15841,15841 +15842,15842 +155933,15842 +15843,15843 +15844,15844 +204278,15844 +15845,15845 +40227,15845 +15846,15846 +15847,15847 +15848,15848 +15849,15849 +15987,15987 +58052,15987 +15988,15988 +15989,15989 +15990,15990 +15991,15991 +15992,15992 +15993,15993 +15994,15994 +15995,15995 +15996,15996 +15997,15997 +15998,15998 +15999,15999 +16000,16000 +16001,16001 +16002,16002 +16003,16003 +16004,16004 +16005,16005 +16006,16006 +16007,16007 +16008,16008 +16009,16009 +16010,16010 +16011,16011 +16012,16012 +16013,16013 +16014,16014 +16015,16015 +16016,16016 +16017,16017 +16018,16018 +16019,16019 +148655,16019 +16020,16020 +16021,16021 +16022,16022 +16023,16023 +16024,16024 +16025,16025 +16026,16026 +16027,16027 +16028,16028 +16029,16029 +16175,16175 +16176,16176 +16177,16177 +16178,16178 +110590,16179 +16179,16179 +16180,16180 +16181,16181 +16182,16182 +16183,16183 +16184,16184 +16185,16185 +16186,16186 +193143,16186 +16187,16187 +16188,16188 +16189,16189 +16190,16190 +16191,16191 +16192,16192 +16193,16193 +16194,16194 +16195,16195 +16196,16196 +16197,16197 +16236,16236 +16237,16237 +16238,16238 +16239,16239 +16240,16240 +16241,16241 +16242,16242 +16243,16243 +16244,16244 +16245,16245 +236561,16245 +16246,16246 +16247,16247 +86433,16247 +16248,16248 +16249,16249 +227649,16249 +16250,16250 +16251,16251 +16252,16252 +16253,16253 +16254,16254 +16255,16255 +16256,16256 +16257,16257 +16258,16258 +209652,16258 +16259,16259 +51747,16259 +43643,16259 +16260,16260 +59159,16260 +58785,16260 +16261,16261 +16262,16262 +16263,16263 +54979,16263 +46875,16263 +16264,16264 +16265,16265 +4534,16265 +16266,16266 +16267,16267 +16268,16268 +16269,16269 +16270,16270 +16271,16271 +16272,16272 +16273,16273 +16274,16274 +20652,16274 +16275,16275 +16276,16276 +50358,16276 +42166,16276 +16277,16277 +16278,16278 +16451,16451 +16452,16452 +16453,16453 +16454,16454 +16455,16455 +36374,16455 +16456,16456 +132108,16456 +124521,16456 +16457,16457 +16458,16458 +16459,16459 +16460,16460 +16461,16461 +16462,16462 +16463,16463 +16464,16464 +16465,16465 +16466,16466 +16467,16467 +16468,16468 +16469,16469 +16470,16470 +16471,16471 +136952,16471 +129365,16471 +16472,16472 +16473,16473 +16474,16474 +16475,16475 +16476,16476 +16477,16477 +16478,16478 +16479,16479 +16480,16480 +16571,16571 +16572,16572 +16573,16573 +16574,16574 +16575,16575 +16576,16576 +16577,16577 +16578,16578 +16579,16579 +16761,16761 +16762,16762 +16763,16763 +16764,16764 +16765,16765 +16766,16766 +16767,16767 +16768,16768 +16769,16769 +16770,16770 +16771,16771 +16803,16803 +16804,16804 +16805,16805 +16806,16806 +16807,16807 +16808,16808 +16809,16809 +16810,16810 +16811,16811 +52755,16812 +44651,16812 +16812,16812 +16813,16813 +16814,16814 +16815,16815 +16816,16816 +16817,16817 +16818,16818 +16819,16819 +16820,16820 +16821,16821 +16822,16822 +9138,16822 +16823,16823 +16824,16824 +16825,16825 +16826,16826 +27057,16826 +16827,16827 +216661,16827 +16828,16828 +16829,16829 +16830,16830 +17105,17105 +17106,17106 +17107,17107 +17108,17108 +17109,17109 +17110,17110 +17111,17111 +17112,17112 +17113,17113 +17114,17114 +17115,17115 +17116,17116 +17117,17117 +17118,17118 +17119,17119 +17120,17120 +17121,17121 +17122,17122 +17123,17123 +17124,17124 +17125,17125 +17126,17126 +17127,17127 +17128,17128 +17129,17129 +17130,17130 +17131,17131 +17132,17132 +17133,17133 +17134,17134 +17135,17135 +17136,17136 +17137,17137 +17138,17138 +17139,17139 +118201,17139 +17140,17140 +17141,17141 +17142,17142 +17143,17143 +17144,17144 +17145,17145 +17146,17146 +17147,17147 +17148,17148 +17149,17149 +17150,17150 +17151,17151 +143047,17151 +17152,17152 +17153,17153 +17154,17154 +17155,17155 +17156,17156 +17157,17157 +17158,17158 +17159,17159 +17160,17160 +17161,17161 +17162,17162 +17163,17163 +17164,17164 +17165,17165 +17166,17166 +17167,17167 +17168,17168 +17169,17169 +17170,17170 +17171,17171 +17172,17172 +17173,17173 +17174,17174 +17175,17175 +17176,17176 +17177,17177 +17178,17178 +17179,17179 +17180,17180 +234548,17180 +17181,17181 +17182,17182 +17183,17183 +17184,17184 +17185,17185 +17186,17186 +88265,17186 +17187,17187 +17188,17188 +17189,17189 +17190,17190 +17191,17191 +17192,17192 +17193,17193 +17194,17194 +17195,17195 +17196,17196 +17197,17197 +55410,17197 +47306,17197 +17198,17198 +17199,17199 +17200,17200 +17201,17201 +17202,17202 +62485,17202 +179181,17202 +17203,17203 +234507,17203 +17204,17204 +17205,17205 +17206,17206 +17207,17207 +17208,17208 +139707,17208 +133734,17208 +126147,17208 +17209,17209 +51156,17209 +43052,17209 +17210,17210 +17211,17211 +17212,17212 +17213,17213 +17214,17214 +17215,17215 +17216,17216 +10487,17216 +17217,17217 +17218,17218 +17219,17219 +17220,17220 +17221,17221 +27944,17221 +17222,17222 +17223,17223 +17224,17224 +17225,17225 +17226,17226 +17227,17227 +17228,17228 +17229,17229 +17230,17230 +185195,17230 +17231,17231 +3530,17231 +17232,17232 +17233,17233 +100575,17233 +17332,17332 +17333,17333 +17371,17371 +17372,17372 +17373,17373 +17374,17374 +17375,17375 +17376,17376 +17377,17377 +17378,17378 +17379,17379 +65146,17379 +17380,17380 +166781,17380 +17381,17381 +17382,17382 +242773,17382 +17383,17383 +17384,17384 +17385,17385 +17386,17386 +17387,17387 +8324,17387 +17388,17388 +17389,17389 +17390,17390 +17391,17391 +17392,17392 +17393,17393 +17394,17394 +17395,17395 +149372,17395 +17396,17396 +17397,17397 +138724,17397 +17398,17398 +17399,17399 +17400,17400 +17401,17401 +17402,17402 +17403,17403 +17404,17404 +17405,17405 +17406,17406 +17407,17407 +17408,17408 +17409,17409 +17410,17410 +155022,17410 +17411,17411 +17412,17412 +140648,17412 +17413,17413 +17414,17414 +18520,18520 +18521,18521 +18522,18522 +231438,18522 +18523,18523 +18524,18524 +18525,18525 +18526,18526 +18527,18527 +18528,18528 +18529,18529 +8723,18529 +18530,18530 +18531,18531 +18532,18532 +18533,18533 +18534,18534 +18535,18535 +18536,18536 +18537,18537 +111687,18537 +18538,18538 +18539,18539 +56359,18539 +48255,18539 +18540,18540 +18541,18541 +18542,18542 +18543,18543 +18544,18544 +202690,18544 +65937,18544 +18545,18545 +18546,18546 +18547,18547 +18548,18548 +18549,18549 +18550,18550 +18551,18551 +18552,18552 +18553,18553 +36065,18831 +18831,18831 +18832,18832 +18833,18833 +217931,18833 +18834,18834 +18835,18835 +18836,18836 +18837,18837 +18838,18838 +18839,18839 +18840,18840 +18841,18841 +18842,18842 +18843,18843 +18844,18844 +18914,18914 +18915,18915 +18916,18916 +18917,18917 +18918,18918 +18919,18919 +18920,18920 +18921,18921 +18922,18922 +18923,18923 +18924,18924 +18925,18925 +18926,18926 +18927,18927 +100643,18927 +18928,18928 +18929,18929 +18930,18930 +18931,18931 +18932,18932 +18933,18933 +18934,18934 +18935,18935 +18936,18936 +18937,18937 +18938,18938 +18939,18939 +18940,18940 +18941,18941 +231314,18941 +18942,18942 +18943,18943 +20788,18943 +18944,18944 +18945,18945 +18946,18946 +18947,18947 +18948,18948 +235028,18948 +18949,18949 +18950,18950 +18951,18951 +18952,18952 +18953,18953 +18954,18954 +18955,18955 +18956,18956 +18957,18957 +18958,18958 +18959,18959 +162834,18959 +18960,18960 +18961,18961 +18962,18962 +18963,18963 +18964,18964 +18965,18965 +18966,18966 +18967,18967 +18968,18968 +18969,18969 +18970,18970 +18971,18971 +18972,18972 +18973,18973 +18974,18974 +18975,18975 +18976,18976 +18977,18977 +18978,18978 +18979,18979 +18980,18980 +18981,18981 +18982,18982 +18983,18983 +18984,18984 +18985,18985 +18986,18986 +18987,18987 +18988,18988 +18989,18989 +18990,18990 +18991,18991 +18992,18992 +18993,18993 +80597,18993 +18994,18994 +18995,18995 +18996,18996 +116549,18996 +18997,18997 +18998,18998 +18999,18999 +19000,19000 +19001,19001 +19002,19002 +19003,19003 +174165,19003 +168201,19003 +63403,19003 +19004,19004 +19005,19005 +19006,19006 +19007,19007 +19008,19008 +19009,19009 +19010,19010 +19011,19011 +19012,19012 +19013,19013 +19014,19014 +19015,19015 +19016,19016 +19017,19017 +19018,19018 +19019,19019 +19020,19020 +19021,19021 +92092,19021 +19022,19022 +19023,19023 +392,19023 +19024,19024 +19025,19025 +19026,19026 +19027,19027 +19028,19028 +19029,19029 +19030,19030 +19031,19031 +208013,19031 +19032,19032 +3173,19032 +19033,19033 +19034,19034 +19035,19035 +19036,19036 +19037,19037 +19038,19038 +19039,19039 +19040,19040 +180557,19041 +19041,19041 +19042,19042 +19043,19043 +19044,19044 +19045,19045 +19046,19046 +19047,19047 +19048,19048 +224913,19048 +19049,19049 +55667,19050 +47563,19050 +19050,19050 +55668,19051 +47564,19051 +19051,19051 +19052,19052 +19053,19053 +19054,19054 +19055,19055 +19056,19056 +19057,19057 +19058,19058 +19059,19059 +19060,19060 +19061,19061 +19062,19062 +19063,19063 +19064,19064 +19065,19065 +19066,19066 +19067,19067 +19068,19068 +19069,19069 +19070,19070 +19071,19071 +19072,19072 +19073,19073 +24503,19073 +19074,19074 +19116,19116 +19117,19117 +19118,19118 +19119,19119 +110707,19119 +19120,19120 +19121,19121 +19160,19160 +19161,19161 +19162,19162 +19163,19163 +19164,19164 +196931,19164 +19165,19165 +19166,19166 +225229,19166 +19167,19167 +19168,19168 +19169,19169 +19170,19170 +19171,19171 +19172,19172 +19173,19173 +19174,19174 +19175,19175 +19176,19176 +19177,19177 +19178,19178 +19179,19179 +19180,19180 +19181,19181 +19182,19182 +19183,19183 +19184,19184 +19185,19185 +162240,19185 +19186,19186 +19187,19187 +19188,19188 +19189,19189 +19190,19190 +19191,19191 +19192,19192 +19193,19193 +231795,19194 +19194,19194 +19195,19195 +19196,19196 +19197,19197 +19198,19198 +19199,19199 +19200,19200 +19201,19201 +19202,19202 +19203,19203 +19204,19204 +19205,19205 +19206,19206 +19207,19207 +19208,19208 +19209,19209 +19210,19210 +163711,19210 +19211,19211 +19212,19212 +19213,19213 +19214,19214 +19215,19215 +19216,19216 +19217,19217 +19218,19218 +19219,19219 +19220,19220 +19221,19221 +19222,19222 +19223,19223 +19224,19224 +19225,19225 +19226,19226 +19227,19227 +19228,19228 +19229,19229 +19230,19230 +201470,19230 +19231,19231 +19232,19232 +19233,19233 +19283,19283 +19284,19284 +19285,19285 +19286,19286 +19287,19287 +19288,19288 +19289,19289 +19290,19290 +19291,19291 +19292,19292 +79425,19292 +19293,19293 +19294,19294 +19295,19295 +19296,19296 +19297,19297 +19452,19452 +19453,19453 +19690,19690 +145422,19690 +19691,19691 +19692,19692 +19693,19693 +19694,19694 +19695,19695 +19696,19696 +19697,19697 +19698,19698 +19699,19699 +19700,19700 +19701,19701 +19702,19702 +19703,19703 +225316,19703 +19704,19704 +19705,19705 +19706,19706 +19707,19707 +19708,19708 +19709,19709 +19710,19710 +19711,19711 +19712,19712 +19713,19713 +19714,19714 +19715,19715 +19716,19716 +19717,19717 +19718,19718 +19719,19719 +19720,19720 +19721,19721 +19722,19722 +19723,19723 +19724,19724 +19725,19725 +19726,19726 +19727,19727 +19728,19728 +19729,19729 +19730,19730 +19731,19731 +19732,19732 +19733,19733 +19734,19734 +19735,19735 +19736,19736 +19737,19737 +19738,19738 +19739,19739 +19740,19740 +19741,19741 +19742,19742 +19743,19743 +19744,19744 +19745,19745 +19746,19746 +19747,19747 +19748,19748 +19826,19826 +19827,19827 +19828,19828 +249407,19828 +19829,19829 +19830,19830 +19831,19831 +19832,19832 +19833,19833 +19834,19834 +19835,19835 +19836,19836 +19837,19837 +19838,19838 +33038,19838 +19839,19839 +19840,19840 +19841,19841 +19842,19842 +19843,19843 +19844,19844 +19845,19845 +19846,19846 +19847,19847 +19848,19848 +19849,19849 +19850,19850 +19851,19851 +19852,19852 +19853,19853 +19854,19854 +19855,19855 +19856,19856 +19857,19857 +19858,19858 +19859,19859 +19860,19860 +19861,19861 +19862,19862 +19863,19863 +19864,19864 +19865,19865 +19866,19866 +19867,19867 +19868,19868 +19869,19869 +19870,19870 +19871,19871 +19872,19872 +19873,19873 +19874,19874 +19875,19875 +186386,19875 +19876,19876 +19877,19877 +19878,19878 +19879,19879 +19880,19880 +19881,19881 +19882,19882 +19883,19883 +19884,19884 +19885,19885 +19886,19886 +19887,19887 +19888,19888 +19889,19889 +19890,19890 +19891,19891 +19892,19892 +19893,19893 +19894,19894 +19895,19895 +19896,19896 +19897,19897 +19898,19898 +51737,19898 +43633,19898 +19899,19899 +19900,19900 +19901,19901 +19902,19902 +19903,19903 +19904,19904 +19905,19905 +19906,19906 +19907,19907 +19908,19908 +19909,19909 +19910,19910 +19911,19911 +19912,19912 +19913,19913 +191999,19913 +19914,19914 +19915,19915 +19916,19916 +19917,19917 +19918,19918 +19919,19919 +19920,19920 +19921,19921 +19922,19922 +183120,19922 +7762,19922 +19923,19923 +19924,19924 +19925,19925 +19926,19926 +19927,19927 +19928,19928 +19929,19929 +19930,19930 +19931,19931 +19932,19932 +19933,19933 +19934,19934 +19935,19935 +19936,19936 +19937,19937 +19938,19938 +19939,19939 +19940,19940 +19941,19941 +19942,19942 +20649,19943 +19943,19943 +20650,19944 +19944,19944 +113,19944 +164457,19944 +19945,19945 +19946,19946 +19947,19947 +19948,19948 +19949,19949 +19950,19950 +19951,19951 +19952,19952 +19953,19953 +67122,19953 +19954,19954 +19955,19955 +20661,19955 +20489,19955 +19956,19956 +19957,19957 +19958,19958 +20664,19958 +20492,19958 +20120,19958 +19959,19959 +113449,19959 +19960,19960 +19961,19961 +19962,19962 +19963,19963 +19964,19964 +19965,19965 +19966,19966 +19967,19967 +51456,19967 +43352,19967 +19968,19968 +20044,20044 +20045,20045 +20046,20046 +20047,20047 +20048,20048 +20049,20049 +248780,20049 +20050,20050 +20051,20051 +20052,20052 +20053,20053 +20054,20054 +20055,20055 +20056,20056 +108912,20056 +20057,20057 +20058,20058 +20059,20059 +20060,20060 +20061,20061 +20062,20062 +20063,20063 +20064,20064 +20065,20065 +20066,20066 +20067,20067 +20068,20068 +20069,20069 +20070,20070 +20071,20071 +20072,20072 +20073,20073 +20074,20074 +20075,20075 +20076,20076 +20077,20077 +20078,20078 +20079,20079 +20080,20080 +20081,20081 +20082,20082 +20083,20083 +20084,20084 +20085,20085 +20086,20086 +20087,20087 +20088,20088 +20089,20089 +20090,20090 +20091,20091 +20092,20092 +20093,20093 +20094,20094 +20095,20095 +133221,20095 +125634,20095 +20096,20096 +20097,20097 +3526,20097 +20098,20098 +20099,20099 +85206,20099 +20100,20100 +20101,20101 +20102,20102 +20103,20103 +23471,20103 +20104,20104 +20105,20105 +20106,20106 +20107,20107 +20108,20108 +20109,20109 +114386,20109 +20110,20110 +111675,20110 +103948,20111 +20111,20111 +20112,20112 +33004,20113 +20113,20113 +20114,20114 +20115,20115 +20116,20116 +20117,20117 +20118,20118 +20119,20119 +20120,20120 +19958,20120 +20121,20121 +20666,20122 +20494,20122 +20122,20122 +20123,20123 +20124,20124 +20125,20125 +146928,20125 +20126,20126 +20127,20127 +20128,20128 +20129,20129 +20501,20129 +24906,20129 +20130,20130 +20131,20131 +161887,20131 +20132,20132 +20133,20133 +20505,20133 +20134,20134 +20135,20135 +20136,20136 +20137,20137 +20138,20138 +20139,20139 +379,20139 +20140,20140 +20141,20141 +20142,20142 +20143,20143 +20144,20144 +20145,20145 +20146,20146 +96676,20146 +20147,20147 +20148,20148 +235168,20148 +20149,20149 +20150,20150 +20151,20151 +20152,20152 +20153,20153 +20154,20154 +20155,20155 +20156,20156 +2094,20156 +20157,20157 +20158,20158 +9724,20158 +20159,20159 +20160,20160 +20161,20161 +20200,20200 +20201,20201 +20202,20202 +20203,20203 +20204,20204 +20205,20205 +20206,20206 +20207,20207 +20208,20208 +20209,20209 +20210,20210 +20211,20211 +20212,20212 +20213,20213 +20214,20214 +20215,20215 +20401,20216 +20216,20216 +123102,20216 +20217,20217 +187097,20217 +20218,20218 +20219,20219 +20220,20220 +64480,20220 +20221,20221 +20222,20222 +20223,20223 +20224,20224 +20225,20225 +51283,20225 +43179,20225 +20226,20226 +20412,20227 +20227,20227 +166747,20227 +57828,20227 +20228,20228 +20229,20229 +20230,20230 +20231,20231 +74712,20231 +20232,20232 +20233,20233 +20234,20234 +20235,20235 +20236,20236 +20237,20237 +20238,20238 +20423,20238 +20239,20239 +20240,20240 +20241,20241 +20242,20242 +20243,20243 +20244,20244 +20245,20245 +20246,20246 +20247,20247 +20248,20248 +20249,20249 +20250,20250 +20251,20251 +20252,20252 +20253,20253 +20254,20254 +20290,20290 +20291,20291 +20292,20292 +20293,20293 +20294,20294 +20295,20295 +20296,20296 +20297,20297 +20298,20298 +20299,20299 +20300,20300 +20301,20301 +20302,20302 +20303,20303 +20304,20304 +207913,20304 +20305,20305 +20306,20306 +20307,20307 +20308,20308 +20309,20309 +20375,20375 +20376,20376 +20377,20377 +20378,20378 +20379,20379 +20380,20380 +20381,20381 +20382,20382 +20383,20383 +20384,20384 +20385,20385 +4693,20385 +20386,20386 +20387,20387 +20388,20388 +20389,20389 +20390,20390 +85194,20390 +20391,20391 +57061,20391 +20392,20392 +20393,20393 +20394,20394 +20395,20395 +20396,20396 +20397,20397 +83702,20397 +20398,20398 +20399,20399 +20400,20400 +20401,20401 +20216,20401 +123102,20401 +20402,20402 +20403,20403 +20404,20404 +20405,20405 +20406,20406 +20407,20407 +20408,20408 +20409,20409 +20410,20410 +20411,20411 +20412,20412 +20227,20412 +166747,20412 +57828,20412 +20413,20413 +20414,20414 +20415,20415 +20416,20416 +20417,20417 +20418,20418 +20419,20419 +20420,20420 +20421,20421 +20422,20422 +20423,20423 +20238,20423 +20424,20424 +20425,20425 +20426,20426 +20427,20427 +20428,20428 +235798,20429 +20429,20429 +20430,20430 +20431,20431 +20432,20432 +20433,20433 +20434,20434 +20435,20435 +20436,20436 +20437,20437 +20438,20438 +20439,20439 +20440,20440 +20441,20441 +20442,20442 +20443,20443 +20444,20444 +20445,20445 +20446,20446 +20447,20447 +20448,20448 +20449,20449 +20450,20450 +20451,20451 +20452,20452 +20453,20453 +20454,20454 +20455,20455 +229808,20455 +139955,20455 +20456,20456 +150678,20456 +20457,20457 +20458,20458 +20459,20459 +20460,20460 +165393,20460 +20461,20461 +20462,20462 +20463,20463 +20464,20464 +20465,20465 +20466,20466 +20638,20466 +20467,20467 +20640,20468 +20468,20468 +20469,20469 +20641,20469 +20470,20470 +20471,20471 +240524,20471 +20644,20472 +20472,20472 +20645,20473 +20473,20473 +20474,20474 +20475,20475 +20647,20475 +20476,20476 +20477,20477 +20478,20478 +20651,20479 +20479,20479 +20480,20480 +20652,20480 +20481,20481 +20653,20481 +20482,20482 +20654,20482 +20655,20483 +20483,20483 +160416,20483 +20656,20484 +20484,20484 +20657,20485 +20485,20485 +20658,20486 +20486,20486 +20659,20487 +20487,20487 +20660,20488 +20488,20488 +20661,20489 +20489,20489 +19955,20489 +20662,20490 +20490,20490 +20663,20491 +20491,20491 +20664,20492 +20492,20492 +19958,20492 +20665,20493 +20493,20493 +215477,20493 +20666,20494 +20494,20494 +20122,20494 +20667,20495 +20495,20495 +20668,20496 +20496,20496 +204480,20496 +20497,20497 +20498,20498 +20499,20499 +20500,20500 +20501,20501 +20129,20501 +20502,20502 +20503,20503 +161887,20503 +20504,20504 +6183,20504 +89175,20504 +20505,20505 +20133,20505 +20506,20506 +20507,20507 +20508,20508 +20509,20509 +20510,20510 +20511,20511 +20512,20512 +20513,20513 +20514,20514 +20515,20515 +20516,20516 +20517,20517 +20518,20518 +20519,20519 +20520,20520 +20521,20521 +20522,20522 +191800,20522 +20523,20523 +20524,20524 +20525,20525 +20526,20526 +20527,20527 +20528,20528 +20529,20529 +20530,20530 +20531,20531 +20532,20532 +20533,20533 +20534,20534 +20535,20535 +20536,20536 +20537,20537 +20538,20538 +20539,20539 +20540,20540 +20541,20541 +20542,20542 +20543,20543 +86716,20543 +20544,20544 +20545,20545 +20546,20546 +20547,20547 +20548,20548 +94490,20548 +20549,20549 +20550,20550 +204003,20550 +20551,20551 +20552,20552 +20553,20553 +20554,20554 +20555,20555 +20556,20556 +20557,20557 +20558,20558 +20559,20559 +20560,20560 +20561,20561 +20562,20562 +20563,20563 +20564,20564 +20565,20565 +20566,20566 +20567,20567 +20568,20568 +20569,20569 +20570,20570 +20571,20571 +20572,20572 +20573,20573 +20574,20574 +20575,20575 +20576,20576 +20577,20577 +20578,20578 +20579,20579 +20580,20580 +20581,20581 +179012,20581 +20582,20582 +20583,20583 +20584,20584 +20585,20585 +20586,20586 +20587,20587 +20588,20588 +20589,20589 +20590,20590 +20591,20591 +20592,20592 +20593,20593 +20594,20594 +20595,20595 +20596,20596 +20597,20597 +20598,20598 +20599,20599 +20600,20600 +20601,20601 +20602,20602 +20603,20603 +20604,20604 +20605,20605 +20606,20606 +20607,20607 +20608,20608 +20609,20609 +20610,20610 +20611,20611 +211959,20611 +20612,20612 +20613,20613 +20614,20614 +20615,20615 +20616,20616 +20617,20617 +20618,20618 +184084,20618 +20619,20619 +20620,20620 +20621,20621 +20622,20622 +20623,20623 +20624,20624 +20625,20625 +20626,20626 +20627,20627 +20628,20628 +20629,20629 +20630,20630 +20631,20631 +20632,20632 +20633,20633 +20634,20634 +20635,20635 +20636,20636 +20637,20637 +20638,20638 +20466,20638 +20639,20639 +20640,20640 +20468,20640 +20641,20641 +20469,20641 +20642,20642 +224625,20642 +20643,20643 +20644,20644 +20472,20644 +20645,20645 +20473,20645 +20646,20646 +20647,20647 +20475,20647 +20648,20648 +20649,20649 +19943,20649 +20650,20650 +19944,20650 +113,20650 +164457,20650 +20651,20651 +20479,20651 +20652,20652 +16274,20652 +20480,20652 +20653,20653 +20481,20653 +20654,20654 +145712,20654 +20482,20654 +20655,20655 +20483,20655 +160416,20655 +20656,20656 +20484,20656 +20657,20657 +20485,20657 +20658,20658 +20486,20658 +20659,20659 +20487,20659 +20660,20660 +20488,20660 +20661,20661 +20489,20661 +19955,20661 +20662,20662 +20490,20662 +20663,20663 +20491,20663 +20664,20664 +20492,20664 +19958,20664 +20665,20665 +20493,20665 +215477,20665 +20666,20666 +20494,20666 +20122,20666 +20667,20667 +20495,20667 +20668,20668 +20496,20668 +204480,20668 +20786,20786 +20787,20787 +20788,20788 +217953,20788 +18943,20788 +20789,20789 +20790,20790 +20791,20791 +20792,20792 +20793,20793 +20794,20794 +20795,20795 +20796,20796 +20797,20797 +119174,20797 +20798,20798 +20799,20799 +236635,20799 +20800,20800 +20801,20801 +20802,20802 +20803,20803 +20804,20804 +166441,20804 +20805,20805 +20806,20806 +20807,20807 +20808,20808 +20809,20809 +20810,20810 +20811,20811 +20812,20812 +20813,20813 +20814,20814 +20815,20815 +20816,20816 +20817,20817 +20818,20818 +20819,20819 +20820,20820 +218300,20820 +20821,20821 +20822,20822 +20823,20823 +20824,20824 +20825,20825 +20826,20826 +106544,20826 +20827,20827 +20828,20828 +20829,20829 +20830,20830 +20831,20831 +20832,20832 +20833,20833 +20834,20834 +20835,20835 +20836,20836 +20837,20837 +20838,20838 +20839,20839 +20840,20840 +20841,20841 +20842,20842 +59408,20842 +20843,20843 +20885,20885 +20886,20886 +20887,20887 +20888,20888 +20889,20889 +20890,20890 +20891,20891 +20892,20892 +20893,20893 +20894,20894 +20895,20895 +50931,20895 +42827,20895 +20896,20896 +13658,20896 +20897,20897 +20898,20898 +20899,20899 +20900,20900 +240282,20900 +20901,20901 +20902,20902 +83397,20902 +20903,20903 +20904,20904 +20905,20905 +20906,20906 +20907,20907 +148147,20907 +20908,20908 +20909,20909 +20910,20910 +20911,20911 +20912,20912 +20913,20913 +190278,20913 +20914,20914 +20915,20915 +20916,20916 +55563,20916 +47459,20916 +20917,20917 +83672,20917 +20918,20918 +187205,20918 +20919,20919 +20920,20920 +20921,20921 +20922,20922 +20923,20923 +20924,20924 +20925,20925 +20926,20926 +20927,20927 +59171,20927 +58797,20927 +20928,20928 +20929,20929 +20930,20930 +21031,21031 +21032,21032 +21033,21033 +21034,21034 +122909,21034 +21035,21035 +21036,21036 +21037,21037 +21121,21121 +21122,21122 +21123,21123 +21124,21124 +21179,21179 +21180,21180 +21181,21181 +21182,21182 +147114,21182 +21183,21183 +21184,21184 +21185,21185 +21186,21186 +21187,21187 +11468,21187 +21188,21188 +21189,21189 +21190,21190 +21191,21191 +21192,21192 +102044,21192 +21193,21193 +21194,21194 +21195,21195 +21196,21196 +21197,21197 +21198,21198 +21199,21199 +21200,21200 +180336,21200 +23843,21200 +21201,21201 +21202,21202 +21203,21203 +21204,21204 +21205,21205 +21206,21206 +201441,21206 +21207,21207 +21208,21208 +21209,21209 +189031,21209 +69824,21209 +21210,21210 +21211,21211 +21212,21212 +21213,21213 +21266,21266 +21267,21267 +21268,21268 +21269,21269 +21270,21270 +21271,21271 +21272,21272 +21273,21273 +21274,21274 +21275,21275 +21276,21276 +21277,21277 +21278,21278 +21279,21279 +21280,21280 +21281,21281 +21282,21282 +21283,21283 +21284,21284 +21285,21285 +21286,21286 +21287,21287 +21288,21288 +21289,21289 +21290,21290 +21291,21291 +21292,21292 +21293,21293 +21294,21294 +21295,21295 +21296,21296 +21297,21297 +21298,21298 +21299,21299 +21300,21300 +21301,21301 +21302,21302 +21303,21303 +21304,21304 +21305,21305 +21306,21306 +21307,21307 +21308,21308 +21309,21309 +21310,21310 +21311,21311 +21312,21312 +21313,21313 +21314,21314 +21315,21315 +21316,21316 +21317,21317 +21318,21318 +21319,21319 +21320,21320 +21321,21321 +21322,21322 +21323,21323 +21324,21324 +21325,21325 +21326,21326 +21327,21327 +207219,21327 +21328,21328 +21329,21329 +21330,21330 +21331,21331 +21332,21332 +21333,21333 +21334,21334 +21335,21335 +21336,21336 +21337,21337 +21338,21338 +21339,21339 +21340,21340 +21341,21341 +21342,21342 +21343,21343 +21344,21344 +21345,21345 +21346,21346 +21347,21347 +21348,21348 +21349,21349 +201182,21349 +81366,21349 +189500,21349 +21350,21350 +21351,21351 +21352,21352 +21353,21353 +21354,21354 +21355,21355 +21356,21356 +21357,21357 +21358,21358 +21359,21359 +21360,21360 +21361,21361 +21362,21362 +21363,21363 +21364,21364 +21365,21365 +21366,21366 +21404,21404 +161837,21404 +21405,21405 +21406,21406 +21407,21407 +21408,21408 +21409,21409 +21410,21410 +21411,21411 +21412,21412 +21413,21413 +21414,21414 +21415,21415 +21416,21416 +21417,21417 +21418,21418 +21419,21419 +21459,21459 +122603,21459 +21460,21460 +21461,21461 +21462,21462 +21463,21463 +21464,21464 +21465,21465 +21466,21466 +21467,21467 +21468,21468 +21469,21469 +21470,21470 +21471,21471 +21472,21472 +21473,21473 +21474,21474 +21475,21475 +21476,21476 +21477,21477 +21478,21478 +21479,21479 +21480,21480 +7522,21480 +21481,21481 +21482,21482 +21483,21483 +21484,21484 +21485,21485 +21486,21486 +21487,21487 +21488,21488 +21489,21489 +21490,21490 +21491,21491 +21492,21492 +21493,21493 +21494,21494 +189988,21494 +21495,21495 +21496,21496 +21497,21497 +21498,21498 +21499,21499 +21500,21500 +21501,21501 +21502,21502 +21503,21503 +21504,21504 +21505,21505 +21506,21506 +21507,21507 +21508,21508 +21509,21509 +21510,21510 +21511,21511 +21512,21512 +21513,21513 +21514,21514 +21515,21515 +21516,21516 +21517,21517 +93839,21517 +21518,21518 +21519,21519 +21520,21520 +21521,21521 +36266,21521 +21522,21522 +21523,21523 +21524,21524 +21525,21525 +141343,21525 +21526,21526 +21527,21527 +21528,21528 +21529,21529 +21530,21530 +21531,21531 +21532,21532 +21533,21533 +21534,21534 +21535,21535 +21536,21536 +21537,21537 +21538,21538 +21539,21539 +21540,21540 +189698,21540 +21541,21541 +89513,21541 +21542,21542 +21543,21543 +21544,21544 +21545,21545 +21546,21546 +21547,21547 +21548,21548 +21549,21549 +21550,21550 +21551,21551 +21552,21552 +21553,21553 +21554,21554 +21555,21555 +21556,21556 +91003,21556 +21557,21557 +21558,21558 +21559,21559 +21560,21560 +21561,21561 +183472,21561 +21562,21562 +21563,21563 +21564,21564 +21565,21565 +21566,21566 +21567,21567 +21568,21568 +21569,21569 +21570,21570 +21571,21571 +21572,21572 +21573,21573 +21574,21574 +21575,21575 +21576,21576 +21577,21577 +21578,21578 +21579,21579 +21580,21580 +21581,21581 +21582,21582 +21583,21583 +21584,21584 +21585,21585 +21586,21586 +21587,21587 +233586,21587 +21588,21588 +21589,21589 +21590,21590 +21591,21591 +21592,21592 +21593,21593 +21594,21594 +21595,21595 +21596,21596 +21597,21597 +21598,21598 +21599,21599 +21600,21600 +21601,21601 +21602,21602 +21603,21603 +21604,21604 +21605,21605 +21606,21606 +21607,21607 +21608,21608 +21609,21609 +179327,21610 +21610,21610 +89533,21610 +21611,21611 +21612,21612 +21613,21613 +21614,21614 +21615,21615 +21616,21616 +21617,21617 +21618,21618 +21619,21619 +21620,21620 +21621,21621 +21622,21622 +21623,21623 +21624,21624 +21625,21625 +21626,21626 +21627,21627 +21628,21628 +21629,21629 +21630,21630 +21631,21631 +21632,21632 +209624,21633 +21633,21633 +21634,21634 +21635,21635 +21636,21636 +21637,21637 +21638,21638 +21639,21639 +21640,21640 +21641,21641 +21642,21642 +21643,21643 +21644,21644 +21645,21645 +21646,21646 +21647,21647 +21648,21648 +21649,21649 +21650,21650 +21651,21651 +21652,21652 +21653,21653 +21654,21654 +21655,21655 +21656,21656 +21657,21657 +21658,21658 +21659,21659 +21660,21660 +21661,21661 +21662,21662 +28077,21662 +21663,21663 +21664,21664 +21665,21665 +21666,21666 +21667,21667 +21668,21668 +21669,21669 +21670,21670 +21671,21671 +21672,21672 +21673,21673 +21674,21674 +21675,21675 +21676,21676 +114872,21676 +21677,21677 +21678,21678 +86880,21678 +21679,21679 +202118,21679 +21680,21680 +21681,21681 +21729,21729 +21730,21730 +21731,21731 +21732,21732 +21733,21733 +21734,21734 +21735,21735 +21736,21736 +21737,21737 +21738,21738 +21739,21739 +21740,21740 +21741,21741 +21742,21742 +21743,21743 +21744,21744 +21745,21745 +21746,21746 +21747,21747 +21748,21748 +21749,21749 +21750,21750 +154107,21750 +21798,21798 +21799,21799 +21800,21800 +21801,21801 +21802,21802 +21803,21803 +21804,21804 +21805,21805 +21806,21806 +21807,21807 +21808,21808 +21809,21809 +21810,21810 +21811,21811 +21812,21812 +21813,21813 +21814,21814 +199854,21814 +222476,21815 +21815,21815 +21816,21816 +21817,21817 +21818,21818 +21819,21819 +21820,21820 +21821,21821 +21822,21822 +21823,21823 +21824,21824 +21825,21825 +21826,21826 +21827,21827 +21828,21828 +21829,21829 +21830,21830 +213873,21830 +21831,21831 +21832,21832 +21833,21833 +21834,21834 +21835,21835 +21836,21836 +21837,21837 +21838,21838 +21839,21839 +21840,21840 +21841,21841 +21842,21842 +21843,21843 +21844,21844 +21845,21845 +21846,21846 +21847,21847 +21848,21848 +21849,21849 +21850,21850 +21851,21851 +21852,21852 +21853,21853 +21854,21854 +21855,21855 +27056,21856 +21856,21856 +21857,21857 +21858,21858 +21859,21859 +21860,21860 +21861,21861 +21862,21862 +21863,21863 +21864,21864 +21865,21865 +21866,21866 +21867,21867 +21868,21868 +21869,21869 +21870,21870 +21871,21871 +21872,21872 +21873,21873 +21874,21874 +21875,21875 +21912,21912 +21913,21913 +21914,21914 +235735,21914 +21915,21915 +79811,21915 +21916,21916 +21917,21917 +21918,21918 +21919,21919 +21920,21920 +21921,21921 +21922,21922 +21923,21923 +21924,21924 +21925,21925 +21926,21926 +21927,21927 +21928,21928 +21929,21929 +208546,21929 +21930,21930 +21931,21931 +21932,21932 +21933,21933 +21934,21934 +21935,21935 +21936,21936 +207027,21936 +21937,21937 +21938,21938 +21939,21939 +21940,21940 +21941,21941 +21942,21942 +21943,21943 +21944,21944 +21945,21945 +33724,21945 +21946,21946 +106789,21946 +21947,21947 +21948,21948 +21949,21949 +21950,21950 +21951,21951 +21952,21952 +21953,21953 +21954,21954 +21955,21955 +21956,21956 +21957,21957 +15401,21957 +21958,21958 +21959,21959 +21960,21960 +21961,21961 +21962,21962 +21963,21963 +21964,21964 +21965,21965 +21966,21966 +21967,21967 +21968,21968 +21969,21969 +21970,21970 +21971,21971 +132954,21971 +125367,21971 +21972,21972 +22021,22021 +92677,22021 +22022,22022 +76383,22022 +22023,22023 +22024,22024 +22025,22025 +22026,22026 +22027,22027 +22028,22028 +22029,22029 +22030,22030 +36287,22030 +22031,22031 +22032,22032 +50210,22032 +42018,22032 +22033,22033 +22034,22034 +22035,22035 +144000,22035 +22036,22036 +211449,22036 +22037,22037 +22038,22038 +22039,22039 +206862,22039 +22040,22040 +64715,22040 +22041,22041 +22042,22042 +186056,22042 +51164,22043 +43060,22043 +22043,22043 +22044,22044 +96038,22044 +22045,22045 +22046,22046 +22047,22047 +22048,22048 +22049,22049 +22050,22050 +22051,22051 +22052,22052 +22053,22053 +22054,22054 +22055,22055 +22056,22056 +22057,22057 +22058,22058 +22059,22059 +22060,22060 +22061,22061 +22062,22062 +22063,22063 +22064,22064 +22065,22065 +22066,22066 +233794,22066 +22067,22067 +22068,22068 +143187,22068 +22069,22069 +22070,22070 +22071,22071 +22072,22072 +22073,22073 +22074,22074 +22075,22075 +22076,22076 +22077,22077 +22078,22078 +22079,22079 +22080,22080 +22081,22081 +236117,22081 +221676,22081 +138109,22081 +130522,22081 +22082,22082 +22083,22083 +22084,22084 +22085,22085 +217982,22085 +22086,22086 +22087,22087 +22088,22088 +22089,22089 +22090,22090 +22091,22091 +22092,22092 +22093,22093 +22094,22094 +22095,22095 +22096,22096 +22097,22097 +22395,22395 +22396,22396 +22397,22397 +22398,22398 +22399,22399 +22400,22400 +22401,22401 +22402,22402 +22403,22403 +22404,22404 +22405,22405 +102100,22405 +22406,22406 +22407,22407 +22408,22408 +22409,22409 +22410,22410 +22411,22411 +22412,22412 +22413,22413 +22414,22414 +22415,22415 +22416,22416 +22417,22417 +22418,22418 +22419,22419 +22420,22420 +163699,22421 +22421,22421 +139683,22421 +22422,22422 +22423,22423 +22424,22424 +22425,22425 +22426,22426 +22427,22427 +22627,22627 +83497,22627 +22628,22628 +22629,22629 +22630,22630 +22631,22631 +22632,22632 +22633,22633 +22634,22634 +22635,22635 +22636,22636 +22637,22637 +22638,22638 +22639,22639 +22640,22640 +22641,22641 +22642,22642 +22643,22643 +22644,22644 +22645,22645 +22646,22646 +22647,22647 +22648,22648 +22649,22649 +22650,22650 +22651,22651 +138138,22651 +130551,22651 +22652,22652 +22653,22653 +22654,22654 +22655,22655 +22656,22656 +22657,22657 +22658,22658 +97652,22658 +22659,22659 +22660,22660 +22661,22661 +22662,22662 +22663,22663 +30206,22663 +22709,22709 +22710,22710 +22711,22711 +22712,22712 +22713,22713 +22714,22714 +22715,22715 +22716,22716 +53798,22716 +45694,22716 +22981,22981 +22982,22982 +22983,22983 +236818,22983 +22984,22984 +22985,22985 +236711,22985 +22986,22986 +22987,22987 +23447,23447 +23448,23448 +23449,23449 +23450,23450 +23451,23451 +23452,23452 +23453,23453 +23454,23454 +201485,23454 +23455,23455 +23456,23456 +23457,23457 +23458,23458 +23459,23459 +23460,23460 +23461,23461 +23462,23462 +23463,23463 +23464,23464 +202491,23464 +23465,23465 +23466,23466 +23467,23467 +23468,23468 +23469,23469 +23470,23470 +23471,23471 +20103,23471 +23472,23472 +23473,23473 +23474,23474 +23475,23475 +23476,23476 +23477,23477 +23478,23478 +23479,23479 +23480,23480 +23481,23481 +23482,23482 +23483,23483 +57171,23483 +23484,23484 +23485,23485 +23486,23486 +23487,23487 +23488,23488 +23489,23489 +23490,23490 +23491,23491 +23492,23492 +23493,23493 +23494,23494 +23495,23495 +23496,23496 +23497,23497 +23498,23498 +10973,23498 +23499,23499 +23500,23500 +23501,23501 +23502,23502 +23503,23503 +23504,23504 +23505,23505 +23506,23506 +55022,23506 +46918,23506 +23507,23507 +23546,23546 +4606,23546 +23547,23547 +23548,23548 +137080,23548 +129493,23548 +23549,23549 +23550,23550 +98128,23550 +23551,23551 +113508,23551 +23552,23552 +23553,23553 +139773,23553 +23554,23554 +23555,23555 +23556,23556 +23557,23557 +220585,23557 +23558,23558 +23559,23559 +23560,23560 +23561,23561 +23562,23562 +23563,23563 +23564,23564 +23565,23565 +23566,23566 +23567,23567 +23568,23568 +23569,23569 +23570,23570 +23571,23571 +23572,23572 +23573,23573 +23574,23574 +23575,23575 +23576,23576 +23577,23577 +23578,23578 +23579,23579 +23580,23580 +23581,23581 +23582,23582 +23583,23583 +142799,23583 +23584,23584 +23653,23653 +23654,23654 +23655,23655 +23656,23656 +23657,23657 +23658,23658 +23659,23659 +23660,23660 +23661,23661 +23662,23662 +23663,23663 +23664,23664 +23665,23665 +23666,23666 +86024,23666 +23667,23667 +23668,23668 +23669,23669 +23670,23670 +23671,23671 +23672,23672 +23673,23673 +134277,23673 +126690,23673 +23674,23674 +75502,23674 +23675,23675 +178358,23675 +172394,23675 +23676,23676 +23677,23677 +23678,23678 +23679,23679 +57969,23679 +23680,23680 +23681,23681 +23682,23682 +23683,23683 +23684,23684 +23685,23685 +23686,23686 +23687,23687 +23688,23688 +23689,23689 +23690,23690 +23691,23691 +23692,23692 +23693,23693 +23694,23694 +23695,23695 +23696,23696 +23697,23697 +23698,23698 +23699,23699 +23700,23700 +23701,23701 +90995,23701 +23702,23702 +23703,23703 +23704,23704 +23705,23705 +23706,23706 +23707,23707 +235271,23707 +23708,23708 +23709,23709 +23710,23710 +23711,23711 +23712,23712 +23713,23713 +23714,23714 +23715,23715 +23716,23716 +23717,23717 +23718,23718 +23719,23719 +23720,23720 +23721,23721 +23722,23722 +23723,23723 +23724,23724 +107390,23724 +23725,23725 +23726,23726 +23727,23727 +23728,23728 +23729,23729 +139046,23729 +23730,23730 +23731,23731 +23732,23732 +23733,23733 +23734,23734 +23735,23735 +23736,23736 +23737,23737 +23738,23738 +23739,23739 +23811,23811 +23812,23812 +23813,23813 +23814,23814 +23815,23815 +23816,23816 +23817,23817 +23818,23818 +23819,23819 +23820,23820 +23821,23821 +23822,23822 +23823,23823 +23824,23824 +23825,23825 +23826,23826 +23827,23827 +23828,23828 +23829,23829 +23830,23830 +36880,23830 +23831,23831 +23832,23832 +23833,23833 +23834,23834 +23835,23835 +23836,23836 +23837,23837 +23838,23838 +23839,23839 +23840,23840 +23841,23841 +23842,23842 +23843,23843 +21200,23843 +23844,23844 +23845,23845 +23846,23846 +23847,23847 +23848,23848 +23849,23849 +23850,23850 +23851,23851 +23852,23852 +23853,23853 +23854,23854 +23855,23855 +23856,23856 +23857,23857 +23858,23858 +55664,23858 +47560,23858 +23859,23859 +23860,23860 +23861,23861 +23862,23862 +23863,23863 +23864,23864 +23865,23865 +23866,23866 +23867,23867 +23868,23868 +23869,23869 +23870,23870 +23871,23871 +23872,23872 +23873,23873 +23874,23874 +23875,23875 +23876,23876 +23877,23877 +23878,23878 +23879,23879 +23880,23880 +23881,23881 +23882,23882 +23883,23883 +23884,23884 +23885,23885 +108709,23885 +23886,23886 +23887,23887 +23888,23888 +23889,23889 +3501,23889 +23890,23890 +23891,23891 +23892,23892 +23893,23893 +23894,23894 +199834,23894 +23895,23895 +23896,23896 +23897,23897 +23898,23898 +23899,23899 +23900,23900 +23901,23901 +23902,23902 +5170,23902 +23903,23903 +23904,23904 +23905,23905 +23906,23906 +23907,23907 +23908,23908 +14409,23908 +23909,23909 +23910,23910 +23911,23911 +23912,23912 +23913,23913 +23914,23914 +23915,23915 +23916,23916 +23917,23917 +9362,23917 +164542,23918 +23918,23918 +23919,23919 +23920,23920 +23921,23921 +23922,23922 +23923,23923 +23924,23924 +23925,23925 +23926,23926 +23927,23927 +23928,23928 +23929,23929 +23984,23984 +23985,23985 +23986,23986 +23987,23987 +23988,23988 +23989,23989 +23990,23990 +23991,23991 +23992,23992 +23993,23993 +23994,23994 +23995,23995 +23996,23996 +23997,23997 +23998,23998 +23999,23999 +24000,24000 +24001,24001 +24002,24002 +24003,24003 +24004,24004 +24005,24005 +24006,24006 +236649,24006 +24007,24007 +24008,24008 +24009,24009 +24010,24010 +24011,24011 +24012,24012 +24013,24013 +24014,24014 +142074,24014 +24015,24015 +24016,24016 +24017,24017 +24018,24018 +24019,24019 +59126,24019 +58752,24019 +24020,24020 +50958,24020 +42854,24020 +24021,24021 +24022,24022 +203820,24022 +59130,24023 +58756,24023 +24023,24023 +77584,24023 +24024,24024 +80883,24024 +24025,24025 +24026,24026 +24027,24027 +24028,24028 +24029,24029 +24030,24030 +2671,24030 +24031,24031 +24070,24070 +24071,24071 +24072,24072 +24073,24073 +24074,24074 +24075,24075 +24076,24076 +24077,24077 +24078,24078 +24079,24079 +24080,24080 +24081,24081 +24082,24082 +24083,24083 +24084,24084 +24085,24085 +24086,24086 +24395,24395 +24396,24396 +24397,24397 +24398,24398 +24399,24399 +24400,24400 +24401,24401 +93389,24401 +24402,24402 +24403,24403 +24404,24404 +24405,24405 +24406,24406 +24407,24407 +24408,24408 +24409,24409 +134061,24409 +126474,24409 +24410,24410 +24411,24411 +24412,24412 +24413,24413 +24414,24414 +24415,24415 +24416,24416 +24417,24417 +24418,24418 +24419,24419 +24420,24420 +24421,24421 +24422,24422 +24423,24423 +24424,24424 +24425,24425 +24426,24426 +24427,24427 +24428,24428 +24429,24429 +24430,24430 +24431,24431 +24432,24432 +24433,24433 +24434,24434 +24435,24435 +24436,24436 +24437,24437 +24438,24438 +24439,24439 +24440,24440 +215992,24440 +24441,24441 +24442,24442 +24443,24443 +24444,24444 +24445,24445 +24446,24446 +24447,24447 +24448,24448 +24449,24449 +24450,24450 +249071,24450 +226381,24450 +24451,24451 +24452,24452 +24453,24453 +24454,24454 +24455,24455 +24456,24456 +24457,24457 +24458,24458 +155190,24458 +24459,24459 +24460,24460 +24461,24461 +24462,24462 +24463,24463 +24464,24464 +24465,24465 +24466,24466 +24467,24467 +24468,24468 +24469,24469 +24470,24470 +24471,24471 +24472,24472 +24473,24473 +24474,24474 +24475,24475 +24476,24476 +24477,24477 +24478,24478 +24479,24479 +24480,24480 +24481,24481 +24482,24482 +215932,24483 +24483,24483 +24484,24484 +24485,24485 +24486,24486 +24487,24487 +24488,24488 +24489,24489 +24490,24490 +24491,24491 +24492,24492 +24493,24493 +24494,24494 +24495,24495 +24496,24496 +24497,24497 +24498,24498 +24499,24499 +24500,24500 +24501,24501 +24502,24502 +24503,24503 +19073,24503 +24504,24504 +24505,24505 +24506,24506 +24507,24507 +24508,24508 +220629,24508 +24509,24509 +24510,24510 +24511,24511 +24512,24512 +24513,24513 +77682,24513 +24514,24514 +154585,24514 +24515,24515 +24516,24516 +24517,24517 +24518,24518 +24519,24519 +24520,24520 +24521,24521 +24522,24522 +24523,24523 +24524,24524 +24525,24525 +24526,24526 +24527,24527 +24528,24528 +24529,24529 +24530,24530 +24531,24531 +24532,24532 +24533,24533 +24534,24534 +24535,24535 +24536,24536 +24537,24537 +161038,24537 +189750,24537 +24538,24538 +178332,24538 +172368,24538 +24539,24539 +24540,24540 +24541,24541 +24542,24542 +24543,24543 +24544,24544 +24545,24545 +9067,24545 +24546,24546 +24547,24547 +24548,24548 +24549,24549 +24550,24550 +24551,24551 +24552,24552 +24553,24553 +24554,24554 +24555,24555 +24556,24556 +24557,24557 +24558,24558 +24559,24559 +24560,24560 +24561,24561 +24562,24562 +24582,24582 +60372,24582 +24583,24583 +24584,24584 +24585,24585 +24586,24586 +24587,24587 +24588,24588 +24589,24589 +24590,24590 +24591,24591 +24592,24592 +24593,24593 +24594,24594 +24595,24595 +9047,24595 +24596,24596 +24597,24597 +24598,24598 +24599,24599 +24600,24600 +24601,24601 +24602,24602 +24603,24603 +24604,24604 +24605,24605 +24606,24606 +24607,24607 +24608,24608 +24609,24609 +24647,24647 +24648,24648 +24649,24649 +24650,24650 +24651,24651 +24652,24652 +24653,24653 +33519,24653 +24654,24654 +24655,24655 +164100,24655 +24656,24656 +24657,24657 +24658,24658 +24659,24659 +24660,24660 +24661,24661 +24662,24662 +24663,24663 +24664,24664 +24665,24665 +24666,24666 +24667,24667 +24668,24668 +24669,24669 +24670,24670 +24671,24671 +24672,24672 +24673,24673 +24674,24674 +24675,24675 +24676,24676 +24677,24677 +24678,24678 +24679,24679 +24680,24680 +24681,24681 +24682,24682 +24683,24683 +24684,24684 +24685,24685 +144201,24685 +24686,24686 +24687,24687 +24688,24688 +77341,24688 +24689,24689 +13260,24689 +24690,24690 +24691,24691 +24692,24692 +24693,24693 +24694,24694 +24695,24695 +24696,24696 +24697,24697 +24698,24698 +24699,24699 +24700,24700 +24701,24701 +24702,24702 +183464,24702 +24703,24703 +24704,24704 +24705,24705 +112304,24705 +24706,24706 +24707,24707 +24708,24708 +24709,24709 +24710,24710 +24711,24711 +24712,24712 +24713,24713 +24714,24714 +24715,24715 +24716,24716 +24890,24890 +24891,24891 +24892,24892 +24893,24893 +24894,24894 +24895,24895 +24896,24896 +24897,24897 +24898,24898 +24899,24899 +24900,24900 +24901,24901 +24902,24902 +24903,24903 +24904,24904 +24905,24905 +24906,24906 +20129,24906 +24907,24907 +24908,24908 +24909,24909 +98221,24909 +24910,24910 +24911,24911 +24912,24912 +24913,24913 +24914,24914 +24915,24915 +24916,24916 +24917,24917 +24918,24918 +24919,24919 +24920,24920 +24921,24921 +24922,24922 +24923,24923 +24924,24924 +24925,24925 +24926,24926 +24927,24927 +24928,24928 +24929,24929 +24930,24930 +24931,24931 +24932,24932 +24933,24933 +24934,24934 +24935,24935 +24936,24936 +24937,24937 +24938,24938 +24939,24939 +24940,24940 +24941,24941 +24942,24942 +24943,24943 +24944,24944 +24945,24945 +24946,24946 +24947,24947 +24948,24948 +24949,24949 +24950,24950 +24951,24951 +24952,24952 +24953,24953 +24954,24954 +24955,24955 +24956,24956 +24957,24957 +24958,24958 +24959,24959 +153115,24959 +24960,24960 +24961,24961 +24962,24962 +24963,24963 +24964,24964 +24965,24965 +24966,24966 +24967,24967 +24968,24968 +24969,24969 +24970,24970 +24971,24971 +24972,24972 +24973,24973 +310,24973 +24974,24974 +24975,24975 +24976,24976 +24977,24977 +24978,24978 +24979,24979 +24980,24980 +24981,24981 +24982,24982 +24983,24983 +24984,24984 +24985,24985 +24986,24986 +24987,24987 +24988,24988 +24989,24989 +24990,24990 +24991,24991 +24992,24992 +24993,24993 +24994,24994 +24995,24995 +225496,24995 +24996,24996 +24997,24997 +24998,24998 +24999,24999 +25000,25000 +25001,25001 +25002,25002 +25003,25003 +25004,25004 +25005,25005 +25006,25006 +25007,25007 +25008,25008 +25009,25009 +25010,25010 +25011,25011 +25012,25012 +25013,25013 +25014,25014 +25015,25015 +25016,25016 +25017,25017 +151132,25017 +25018,25018 +25019,25019 +25020,25020 +25021,25021 +25022,25022 +25023,25023 +210334,25023 +25024,25024 +25025,25025 +25026,25026 +25027,25027 +25028,25028 +25029,25029 +25030,25030 +25031,25031 +25032,25032 +25033,25033 +25034,25034 +25035,25035 +25036,25036 +208460,25036 +25037,25037 +25038,25038 +120063,25038 +25039,25039 +85574,25039 +25040,25040 +25041,25041 +25042,25042 +25043,25043 +85578,25043 +85579,25044 +25044,25044 +25045,25045 +85580,25045 +25046,25046 +85581,25046 +25047,25047 +85582,25047 +85583,25048 +25048,25048 +2206,25048 +25049,25049 +85584,25049 +25050,25050 +25051,25051 +25052,25052 +25053,25053 +25054,25054 +25055,25055 +25056,25056 +25057,25057 +25058,25058 +25059,25059 +25060,25060 +25061,25061 +25062,25062 +25063,25063 +25064,25064 +25065,25065 +25066,25066 +25067,25067 +25068,25068 +25069,25069 +25070,25070 +25071,25071 +25072,25072 +25073,25073 +25074,25074 +25075,25075 +85886,25075 +132915,25075 +125328,25075 +25076,25076 +25077,25077 +25078,25078 +25079,25079 +25080,25080 +25081,25081 +25082,25082 +25083,25083 +25084,25084 +25085,25085 +25086,25086 +25087,25087 +25088,25088 +25089,25089 +25090,25090 +25091,25091 +25092,25092 +25093,25093 +85620,25093 +25094,25094 +25095,25095 +25096,25096 +25097,25097 +25098,25098 +29532,25098 +25099,25099 +25100,25100 +38995,25100 +25101,25101 +25102,25102 +25103,25103 +25104,25104 +164528,25104 +25105,25105 +25106,25106 +25107,25107 +25108,25108 +84179,25108 +25109,25109 +25110,25110 +138888,25110 +25111,25111 +25112,25112 +25113,25113 +49129,25114 +40937,25114 +25114,25114 +25115,25115 +25116,25116 +25117,25117 +25118,25118 +25119,25119 +25120,25120 +25121,25121 +25122,25122 +25123,25123 +25124,25124 +4669,25124 +25125,25125 +25126,25126 +25127,25127 +25128,25128 +25129,25129 +25130,25130 +155448,25130 +25131,25131 +25510,25510 +94501,25510 +25511,25511 +25512,25512 +25513,25513 +25514,25514 +25553,25553 +25554,25554 +25555,25555 +25556,25556 +25557,25557 +25558,25558 +25559,25559 +210037,25559 +25560,25560 +25561,25561 +25562,25562 +25563,25563 +25564,25564 +25565,25565 +25566,25566 +25823,25823 +25824,25824 +25825,25825 +25826,25826 +25827,25827 +25828,25828 +25829,25829 +25830,25830 +25831,25831 +25832,25832 +25833,25833 +25834,25834 +25835,25835 +25836,25836 +25837,25837 +3879,25837 +25838,25838 +25839,25839 +26041,26041 +26042,26042 +26043,26043 +26044,26044 +26045,26045 +26046,26046 +26047,26047 +152873,26047 +26048,26048 +26049,26049 +26050,26050 +26051,26051 +75917,26051 +26052,26052 +26053,26053 +26054,26054 +26055,26055 +26056,26056 +26057,26057 +26058,26058 +26059,26059 +26060,26060 +104438,26060 +26061,26061 +26062,26062 +26063,26063 +26064,26064 +26065,26065 +26066,26066 +26067,26067 +26068,26068 +26069,26069 +108059,26069 +26070,26070 +26071,26071 +26072,26072 +26073,26073 +26074,26074 +26075,26075 +26076,26076 +26077,26077 +4002,26077 +26078,26078 +26079,26079 +26080,26080 +26081,26081 +65468,26081 +26082,26082 +26083,26083 +26084,26084 +26085,26085 +26086,26086 +26087,26087 +26088,26088 +26089,26089 +26090,26090 +26091,26091 +26092,26092 +193692,26092 +26093,26093 +26094,26094 +26095,26095 +26096,26096 +26168,26168 +26169,26169 +26170,26170 +26171,26171 +26172,26172 +26173,26173 +26174,26174 +26175,26175 +26176,26176 +26177,26177 +26178,26178 +26179,26179 +26180,26180 +26181,26181 +26182,26182 +26183,26183 +26184,26184 +26185,26185 +26186,26186 +26187,26187 +26188,26188 +26189,26189 +26190,26190 +26191,26191 +26192,26192 +26193,26193 +26194,26194 +26195,26195 +26196,26196 +26197,26197 +209798,26197 +26198,26198 +26199,26199 +26200,26200 +26201,26201 +26202,26202 +26203,26203 +26204,26204 +26205,26205 +26206,26206 +26207,26207 +26208,26208 +26209,26209 +26210,26210 +26211,26211 +26212,26212 +26213,26213 +75400,26213 +26214,26214 +26215,26215 +26216,26216 +26217,26217 +26218,26218 +26219,26219 +211677,26219 +26220,26220 +31195,26220 +26221,26221 +26222,26222 +26223,26223 +26224,26224 +164507,26224 +156829,26225 +26225,26225 +26226,26226 +26227,26227 +26228,26228 +26229,26229 +26230,26230 +26231,26231 +114157,26231 +2956,26231 +26232,26232 +26233,26233 +26234,26234 +26235,26235 +26236,26236 +26237,26237 +26238,26238 +26239,26239 +26240,26240 +26241,26241 +215507,26241 +26242,26242 +26243,26243 +241090,26243 +26244,26244 +26245,26245 +26246,26246 +26247,26247 +26248,26248 +26249,26249 +26250,26250 +26251,26251 +26252,26252 +26253,26253 +26254,26254 +26255,26255 +26256,26256 +26257,26257 +26258,26258 +26259,26259 +26260,26260 +26261,26261 +26262,26262 +3103,26262 +26263,26263 +26264,26264 +26265,26265 +26266,26266 +26267,26267 +26268,26268 +26269,26269 +26270,26270 +238003,26270 +26271,26271 +26272,26272 +26273,26273 +26274,26274 +26275,26275 +26276,26276 +209731,26276 +26277,26277 +26278,26278 +26279,26279 +26280,26280 +26281,26281 +26282,26282 +26283,26283 +26284,26284 +26285,26285 +26286,26286 +26287,26287 +26288,26288 +26289,26289 +26290,26290 +26291,26291 +94949,26291 +26292,26292 +26293,26293 +26294,26294 +110596,26294 +26295,26295 +26296,26296 +26297,26297 +26298,26298 +26299,26299 +26300,26300 +26301,26301 +26302,26302 +26303,26303 +26304,26304 +26305,26305 +26306,26306 +26307,26307 +35803,26307 +26308,26308 +26309,26309 +26310,26310 +3558,26310 +26311,26311 +26312,26312 +26313,26313 +26314,26314 +26315,26315 +26316,26316 +26317,26317 +26318,26318 +26319,26319 +26320,26320 +112235,26320 +26321,26321 +26322,26322 +26323,26323 +26324,26324 +26325,26325 +26326,26326 +26327,26327 +26328,26328 +110841,26328 +26329,26329 +26330,26330 +26331,26331 +26332,26332 +26333,26333 +26334,26334 +208464,26335 +26335,26335 +26336,26336 +26337,26337 +74307,26337 +26338,26338 +26339,26339 +26340,26340 +26341,26341 +26342,26342 +26343,26343 +26344,26344 +26345,26345 +26346,26346 +26347,26347 +26348,26348 +26349,26349 +26350,26350 +26351,26351 +26352,26352 +26353,26353 +26354,26354 +26355,26355 +26356,26356 +26357,26357 +26358,26358 +26359,26359 +26360,26360 +26361,26361 +26362,26362 +26363,26363 +26364,26364 +26365,26365 +58132,26365 +26366,26366 +26367,26367 +26368,26368 +151468,26368 +84802,26369 +26369,26369 +26370,26370 +26371,26371 +145209,26371 +26372,26372 +26373,26373 +26374,26374 +26375,26375 +26376,26376 +26377,26377 +26378,26378 +26379,26379 +26380,26380 +26381,26381 +26382,26382 +143905,26383 +26383,26383 +26384,26384 +26385,26385 +26461,26461 +26462,26462 +26463,26463 +26464,26464 +26465,26465 +26466,26466 +26467,26467 +26468,26468 +26469,26469 +26470,26470 +26471,26471 +26472,26472 +26473,26473 +26474,26474 +26475,26475 +26476,26476 +26477,26477 +26478,26478 +26479,26479 +26480,26480 +26481,26481 +26482,26482 +26483,26483 +26484,26484 +26485,26485 +26486,26486 +26487,26487 +26488,26488 +26489,26489 +26490,26490 +26558,26558 +26559,26559 +26659,26659 +26660,26660 +26661,26661 +59388,26661 +26662,26662 +26663,26663 +26664,26664 +26665,26665 +26666,26666 +26667,26667 +26668,26668 +26669,26669 +26670,26670 +26756,26756 +187246,26756 +26757,26757 +26758,26758 +26759,26759 +26760,26760 +26872,26872 +26873,26873 +26874,26874 +210200,26874 +26875,26875 +26876,26876 +39367,26876 +37883,26876 +26877,26877 +26878,26878 +26879,26879 +110303,26880 +26880,26880 +26881,26881 +78068,26881 +26882,26882 +26883,26883 +26884,26884 +26885,26885 +26886,26886 +26887,26887 +26888,26888 +26889,26889 +26890,26890 +26891,26891 +26892,26892 +26893,26893 +165387,26893 +26894,26894 +26895,26895 +26896,26896 +26897,26897 +26898,26898 +26899,26899 +26900,26900 +26901,26901 +26902,26902 +26903,26903 +26904,26904 +26905,26905 +51327,26905 +43223,26905 +26906,26906 +26907,26907 +195162,26907 +26908,26908 +26909,26909 +26910,26910 +26911,26911 +26912,26912 +26913,26913 +26914,26914 +26915,26915 +26916,26916 +26917,26917 +26918,26918 +26919,26919 +26920,26920 +26921,26921 +26922,26922 +26923,26923 +26924,26924 +26925,26925 +26926,26926 +26927,26927 +26928,26928 +26929,26929 +26930,26930 +26931,26931 +26932,26932 +26933,26933 +26934,26934 +26935,26935 +26936,26936 +152925,26936 +26937,26937 +26938,26938 +26939,26939 +26940,26940 +26941,26941 +26942,26942 +26943,26943 +26944,26944 +26945,26945 +149412,26945 +26946,26946 +26947,26947 +26948,26948 +26949,26949 +26950,26950 +26951,26951 +26952,26952 +26953,26953 +27034,27034 +27035,27035 +27036,27036 +27037,27037 +27038,27038 +27039,27039 +27040,27040 +27041,27041 +27042,27042 +27043,27043 +27044,27044 +27045,27045 +27046,27046 +27047,27047 +27048,27048 +27049,27049 +27050,27050 +27051,27051 +27052,27052 +27053,27053 +27054,27054 +27055,27055 +138649,27055 +27056,27056 +21856,27056 +27057,27057 +16826,27057 +27058,27058 +27059,27059 +27060,27060 +27061,27061 +27062,27062 +27063,27063 +27064,27064 +27065,27065 +27066,27066 +27067,27067 +27068,27068 +27069,27069 +27070,27070 +27071,27071 +27072,27072 +27073,27073 +27074,27074 +27075,27075 +27076,27076 +27077,27077 +27078,27078 +27079,27079 +27080,27080 +27081,27081 +10485,27081 +27082,27082 +27083,27083 +27084,27084 +27085,27085 +27086,27086 +27087,27087 +27088,27088 +27089,27089 +27090,27090 +141797,27090 +27091,27091 +27092,27092 +27093,27093 +27094,27094 +27095,27095 +27096,27096 +27097,27097 +27098,27098 +27099,27099 +27100,27100 +27101,27101 +29518,27101 +27102,27102 +27103,27103 +27104,27104 +27105,27105 +27106,27106 +27107,27107 +27108,27108 +239537,27108 +27109,27109 +27110,27110 +27111,27111 +27112,27112 +27113,27113 +27114,27114 +106321,27114 +27115,27115 +2030,27115 +27116,27116 +27117,27117 +27118,27118 +27119,27119 +27120,27120 +27121,27121 +27122,27122 +27123,27123 +185698,27123 +27124,27124 +27125,27125 +27126,27126 +27127,27127 +255,27127 +27128,27128 +27129,27129 +27130,27130 +27131,27131 +27132,27132 +27133,27133 +27134,27134 +27135,27135 +27136,27136 +27137,27137 +27138,27138 +27139,27139 +157201,27139 +27140,27140 +27141,27141 +27142,27142 +27143,27143 +27144,27144 +27145,27145 +27146,27146 +157208,27146 +27147,27147 +27148,27148 +27149,27149 +27150,27150 +27151,27151 +27152,27152 +27153,27153 +27154,27154 +27155,27155 +27156,27156 +27157,27157 +27158,27158 +27159,27159 +27160,27160 +27161,27161 +27162,27162 +27163,27163 +27164,27164 +27165,27165 +27166,27166 +27167,27167 +27168,27168 +27169,27169 +27170,27170 +64834,27171 +27171,27171 +27172,27172 +27173,27173 +27174,27174 +27175,27175 +27176,27176 +27177,27177 +208264,27177 +27178,27178 +27179,27179 +27180,27180 +27181,27181 +27182,27182 +27183,27183 +27184,27184 +27185,27185 +27186,27186 +27187,27187 +27188,27188 +27189,27189 +27190,27190 +140028,27190 +27191,27191 +27192,27192 +222365,27192 +27193,27193 +27194,27194 +137552,27194 +129965,27194 +27195,27195 +27196,27196 +27197,27197 +163001,27197 +27198,27198 +27199,27199 +27345,27345 +27346,27346 +27347,27347 +27348,27348 +27349,27349 +27350,27350 +27351,27351 +64144,27351 +27352,27352 +27353,27353 +27354,27354 +27355,27355 +27356,27356 +27357,27357 +27358,27358 +27359,27359 +27360,27360 +27398,27398 +27399,27399 +27400,27400 +27401,27401 +27402,27402 +27403,27403 +27404,27404 +27405,27405 +27466,27466 +27574,27574 +27575,27575 +136565,27575 +128978,27575 +27576,27576 +142804,27576 +27577,27577 +27578,27578 +27579,27579 +27580,27580 +27581,27581 +27582,27582 +27583,27583 +27584,27584 +27585,27585 +27586,27586 +27587,27587 +27588,27588 +27589,27589 +27590,27590 +135273,27590 +127686,27590 +27591,27591 +27592,27592 +27593,27593 +37051,27593 +27727,27727 +27728,27728 +27915,27728 +27729,27729 +27730,27730 +27731,27731 +27732,27732 +27733,27733 +27734,27734 +27735,27735 +27736,27736 +27737,27737 +99243,27737 +27738,27738 +27739,27739 +27740,27740 +27741,27741 +27742,27742 +27743,27743 +27744,27744 +83740,27744 +27745,27745 +27914,27914 +27915,27915 +27728,27915 +35913,27915 +27916,27916 +27917,27917 +27918,27918 +27919,27919 +27920,27920 +27921,27921 +27922,27922 +84524,27922 +27923,27923 +27924,27924 +27925,27925 +27926,27926 +104712,27926 +27927,27927 +27928,27928 +27929,27929 +27930,27930 +27931,27931 +27932,27932 +27933,27933 +27934,27934 +27935,27935 +27936,27936 +27937,27937 +27938,27938 +27939,27939 +27940,27940 +27941,27941 +27942,27942 +27943,27943 +27944,27944 +17221,27944 +27945,27945 +27946,27946 +27947,27947 +27948,27948 +27949,27949 +27950,27950 +27951,27951 +27952,27952 +27953,27953 +27954,27954 +27955,27955 +27956,27956 +27957,27957 +27958,27958 +27959,27959 +27960,27960 +27961,27961 +28059,28059 +28060,28060 +28061,28061 +28062,28062 +28063,28063 +28064,28064 +64731,28064 +160648,28064 +28065,28065 +28066,28066 +28067,28067 +28068,28068 +28069,28069 +28070,28070 +28071,28071 +28072,28072 +28073,28073 +102556,28073 +28074,28074 +28075,28075 +28076,28076 +28077,28077 +21662,28077 +28078,28078 +28079,28079 +28080,28080 +28081,28081 +28082,28082 +28083,28083 +28084,28084 +28085,28085 +28086,28086 +28087,28087 +28088,28088 +28089,28089 +28090,28090 +85422,28090 +28091,28091 +242863,28091 +28092,28092 +28093,28093 +28094,28094 +28095,28095 +28096,28096 +132126,28096 +124539,28096 +28097,28097 +221996,28097 +28098,28098 +28099,28099 +28100,28100 +28101,28101 +28102,28102 +28103,28103 +28104,28104 +28105,28105 +28106,28106 +28107,28107 +28108,28108 +28109,28109 +28110,28110 +28111,28111 +28112,28112 +28113,28113 +28114,28114 +28115,28115 +28116,28116 +28117,28117 +28118,28118 +28119,28119 +28120,28120 +28121,28121 +28122,28122 +28123,28123 +28124,28124 +65755,28124 +28125,28125 +28126,28126 +28127,28127 +28128,28128 +28129,28129 +236513,28129 +28130,28130 +111062,28130 +28131,28131 +28132,28132 +28133,28133 +28134,28134 +28135,28135 +185564,28135 +28136,28136 +28137,28137 +28138,28138 +28139,28139 +28140,28140 +28141,28141 +28142,28142 +28143,28143 +28144,28144 +28145,28145 +28146,28146 +28201,28201 +28202,28202 +28203,28203 +28204,28204 +28205,28205 +28206,28206 +28207,28207 +28208,28208 +220300,28208 +28209,28209 +28210,28210 +28211,28211 +28212,28212 +28213,28213 +28214,28214 +28215,28215 +28216,28216 +28217,28217 +28218,28218 +28219,28219 +28220,28220 +28221,28221 +28222,28222 +28223,28223 +28224,28224 +58135,28224 +57836,28224 +28225,28225 +28226,28226 +209596,28226 +28227,28227 +28228,28228 +28229,28229 +178943,28229 +28230,28230 +28231,28231 +28232,28232 +36621,28232 +30483,28232 +139180,28232 +28233,28233 +28234,28234 +28235,28235 +28236,28236 +28237,28237 +28238,28238 +28239,28239 +28240,28240 +28241,28241 +28242,28242 +28243,28243 +28244,28244 +28245,28245 +28246,28246 +28247,28247 +28248,28248 +28249,28249 +28250,28250 +28251,28251 +28252,28252 +28253,28253 +28254,28254 +28255,28255 +28256,28256 +28257,28257 +28258,28258 +28259,28259 +28260,28260 +28261,28261 +243952,28261 +28262,28262 +28263,28263 +28264,28264 +28265,28265 +28266,28266 +28267,28267 +179642,28267 +28268,28268 +28269,28269 +28270,28270 +28271,28271 +190530,28271 +28272,28272 +28273,28273 +28274,28274 +28275,28275 +28276,28276 +28277,28277 +56,28277 +28278,28278 +28279,28279 +28280,28280 +28281,28281 +28282,28282 +28283,28283 +28284,28284 +216809,28284 +28365,28365 +28366,28366 +28367,28367 +28368,28368 +28369,28369 +138174,28369 +130587,28369 +28370,28370 +3234,28370 +28371,28371 +28372,28372 +28373,28373 +28374,28374 +28375,28375 +28376,28376 +28377,28377 +28378,28378 +28379,28379 +28380,28380 +28381,28381 +28382,28382 +28383,28383 +28384,28384 +28385,28385 +28386,28386 +28387,28387 +28388,28388 +28389,28389 +28390,28390 +28391,28391 +28392,28392 +28393,28393 +28394,28394 +28395,28395 +28396,28396 +28397,28397 +28398,28398 +28399,28399 +28400,28400 +28401,28401 +28402,28402 +28403,28403 +234920,28403 +28404,28404 +28405,28405 +28406,28406 +28407,28407 +28408,28408 +28409,28409 +28410,28410 +28411,28411 +28412,28412 +28413,28413 +142557,28413 +28450,28450 +28451,28451 +28452,28452 +56118,28453 +48014,28453 +28453,28453 +151452,28453 +28454,28454 +28455,28455 +28456,28456 +28457,28457 +28458,28458 +28459,28459 +11563,28459 +107126,28459 +28460,28460 +28461,28461 +28462,28462 +28463,28463 +28464,28464 +28465,28465 +28466,28466 +28467,28467 +28468,28468 +28469,28469 +28470,28470 +28471,28471 +28472,28472 +28473,28473 +28474,28474 +28475,28475 +28476,28476 +28477,28477 +28478,28478 +234589,28478 +28479,28479 +28480,28480 +28481,28481 +28482,28482 +28483,28483 +28484,28484 +28485,28485 +28486,28486 +28724,28724 +28725,28725 +28726,28726 +28727,28727 +28728,28728 +28729,28729 +28730,28730 +28731,28731 +28732,28732 +28733,28733 +28734,28734 +28735,28735 +224604,28736 +28736,28736 +28737,28737 +28738,28738 +28739,28739 +28740,28740 +28741,28741 +28742,28742 +28743,28743 +28744,28744 +28745,28745 +28746,28746 +28747,28747 +28748,28748 +28749,28749 +28750,28750 +104939,28750 +28751,28751 +28752,28752 +227556,28752 +151138,28752 +28753,28753 +100646,28754 +28754,28754 +28755,28755 +28756,28756 +63736,28756 +28757,28757 +28758,28758 +28759,28759 +28760,28760 +28761,28761 +28762,28762 +28763,28763 +227576,28763 +28764,28764 +28837,28837 +28874,28874 +28875,28875 +162580,28875 +28876,28876 +28877,28877 +61781,28877 +28878,28878 +28879,28879 +28880,28880 +28881,28881 +28882,28882 +28883,28883 +28884,28884 +28885,28885 +28886,28886 +28887,28887 +28888,28888 +28889,28889 +28890,28890 +28891,28891 +28892,28892 +13479,28892 +28893,28893 +28894,28894 +28895,28895 +28896,28896 +28934,28934 +28935,28935 +28936,28936 +28937,28937 +28938,28938 +144083,28938 +28939,28939 +28940,28940 +28941,28941 +28942,28942 +28943,28943 +5731,28943 +140696,28943 +133993,28943 +126406,28943 +28944,28944 +203021,28944 +246650,28944 +28945,28945 +100567,28945 +28946,28946 +28947,28947 +28948,28948 +28949,28949 +28950,28950 +100597,28950 +28951,28951 +135954,28951 +128367,28951 +28952,28952 +28953,28953 +28954,28954 +28955,28955 +28956,28956 +28957,28957 +28958,28958 +28959,28959 +28960,28960 +28961,28961 +28962,28962 +190438,28962 +28963,28963 +28964,28964 +28965,28965 +28966,28966 +28967,28967 +28968,28968 +28969,28969 +28970,28970 +28971,28971 +28972,28972 +28973,28973 +15692,28973 +28974,28974 +28975,28975 +208819,28975 +57800,28975 +28976,28976 +28977,28977 +28978,28978 +28979,28979 +28980,28980 +28981,28981 +28982,28982 +28983,28983 +28984,28984 +28985,28985 +28986,28986 +28987,28987 +28988,28988 +28989,28989 +86449,28989 +28990,28990 +28991,28991 +28992,28992 +28993,28993 +28994,28994 +28995,28995 +28996,28996 +28997,28997 +28998,28998 +28999,28999 +150465,28999 +29000,29000 +123501,29001 +29001,29001 +29002,29002 +29003,29003 +29004,29004 +29005,29005 +29006,29006 +8209,29006 +29007,29007 +29008,29008 +29009,29009 +29010,29010 +29011,29011 +29050,29050 +29051,29051 +29052,29052 +29053,29053 +59479,29053 +29054,29054 +29055,29055 +29056,29056 +29057,29057 +237995,29057 +29058,29058 +29059,29059 +29060,29060 +7260,29060 +29061,29061 +66948,29061 +29062,29062 +29063,29063 +157068,29063 +29064,29064 +29065,29065 +73571,29065 +29066,29066 +111922,29066 +29067,29067 +29068,29068 +29223,29223 +29224,29224 +29225,29225 +29226,29226 +29227,29227 +29228,29228 +29229,29229 +29230,29230 +29231,29231 +29232,29232 +29233,29233 +29234,29234 +29235,29235 +29236,29236 +29237,29237 +29238,29238 +29239,29239 +29240,29240 +29241,29241 +29242,29242 +29243,29243 +29244,29244 +29245,29245 +29246,29246 +29247,29247 +29248,29248 +29249,29249 +29250,29250 +208584,29250 +29251,29251 +29252,29252 +29253,29253 +29254,29254 +29255,29255 +29256,29256 +29257,29257 +29258,29258 +29259,29259 +29260,29260 +29261,29261 +245415,29262 +29262,29262 +159485,29262 +29263,29263 +29264,29264 +29265,29265 +29266,29266 +29267,29267 +29268,29268 +29269,29269 +29270,29270 +29271,29271 +29272,29272 +29273,29273 +29274,29274 +29275,29275 +29276,29276 +29277,29277 +221878,29278 +29278,29278 +29279,29279 +29413,29413 +29414,29414 +29415,29415 +29416,29416 +29417,29417 +29418,29418 +29419,29419 +29420,29420 +29421,29421 +29422,29422 +29423,29423 +29424,29424 +78,29424 +29425,29425 +29426,29426 +29427,29427 +29428,29428 +29429,29429 +29490,29490 +29491,29491 +29492,29492 +29493,29493 +29494,29494 +29495,29495 +29496,29496 +29497,29497 +29498,29498 +29499,29499 +29500,29500 +29501,29501 +29502,29502 +29503,29503 +29504,29504 +29505,29505 +245452,29505 +29506,29506 +29507,29507 +29508,29508 +78916,29508 +29509,29509 +29510,29510 +29511,29511 +29512,29512 +29513,29513 +29514,29514 +29515,29515 +29516,29516 +5645,29516 +29517,29517 +5539,29517 +29518,29518 +27101,29518 +29519,29519 +29520,29520 +29521,29521 +135093,29522 +127506,29522 +29522,29522 +29523,29523 +29524,29524 +29525,29525 +29526,29526 +29527,29527 +29528,29528 +29529,29529 +29530,29530 +29531,29531 +29532,29532 +25098,29532 +242873,29532 +29533,29533 +241506,29533 +29534,29534 +29535,29535 +29536,29536 +29537,29537 +29538,29538 +29539,29539 +29540,29540 +29541,29541 +33001,29541 +29542,29542 +29543,29543 +29584,29584 +29585,29585 +29586,29586 +29587,29587 +29588,29588 +29589,29589 +29590,29590 +29591,29591 +29592,29592 +29762,29762 +29763,29763 +29764,29764 +218666,29764 +29765,29765 +29766,29766 +29767,29767 +29768,29768 +29769,29769 +192426,29769 +29770,29770 +29771,29771 +29772,29772 +29773,29773 +29774,29774 +29775,29775 +29776,29776 +29777,29777 +29778,29778 +29779,29779 +29780,29780 +29781,29781 +29782,29782 +29783,29783 +29784,29784 +29785,29785 +29786,29786 +29787,29787 +29788,29788 +29789,29789 +29790,29790 +29791,29791 +29792,29792 +29793,29793 +29794,29794 +29795,29795 +29796,29796 +29797,29797 +29798,29798 +29799,29799 +5542,29799 +29800,29800 +29801,29801 +29802,29802 +29803,29803 +29804,29804 +29805,29805 +74515,29805 +29806,29806 +29807,29807 +29808,29808 +29809,29809 +29810,29810 +29811,29811 +29812,29812 +29813,29813 +29872,29872 +29873,29873 +183965,29873 +29874,29874 +29875,29875 +29876,29876 +142973,29876 +29877,29877 +140849,29877 +29878,29878 +29879,29879 +153145,29879 +195792,29879 +29880,29880 +29881,29881 +29882,29882 +29883,29883 +29884,29884 +29885,29885 +139052,29885 +212116,29886 +29886,29886 +29887,29887 +29888,29888 +29889,29889 +29890,29890 +29891,29891 +29892,29892 +36076,29892 +29893,29893 +246694,29893 +29894,29894 +29895,29895 +29896,29896 +109824,29896 +29897,29897 +29898,29898 +29899,29899 +29900,29900 +6844,29900 +29901,29901 +36085,29901 +29902,29902 +29903,29903 +29904,29904 +29905,29905 +29906,29906 +111896,29906 +151156,29906 +29907,29907 +29908,29908 +29909,29909 +29910,29910 +29911,29911 +36096,29912 +29912,29912 +29913,29913 +29914,29914 +29915,29915 +29916,29916 +29917,29917 +29918,29918 +29919,29919 +29920,29920 +29921,29921 +29922,29922 +29923,29923 +29924,29924 +36840,29924 +29925,29925 +29926,29926 +29927,29927 +29928,29928 +36591,29928 +29929,29929 +29930,29930 +29931,29931 +29932,29932 +29933,29933 +70548,29933 +29934,29934 +29935,29935 +29936,29936 +29937,29937 +29938,29938 +29939,29939 +29940,29940 +84097,29940 +29941,29941 +29942,29942 +29943,29943 +29944,29944 +29945,29945 +29946,29946 +29947,29947 +226759,29947 +29948,29948 +29949,29949 +29950,29950 +4223,29950 +29951,29951 +29952,29952 +29953,29953 +29954,29954 +29955,29955 +29956,29956 +29957,29957 +29958,29958 +29959,29959 +29960,29960 +29961,29961 +36151,29961 +29962,29962 +29963,29963 +33259,29963 +36153,29963 +29964,29964 +29965,29965 +36155,29965 +29966,29966 +33262,29966 +36157,29967 +29967,29967 +33263,29967 +29968,29968 +36158,29968 +29969,29969 +5829,29969 +33265,29969 +29970,29970 +33266,29970 +29971,29971 +36161,29971 +29972,29972 +36162,29972 +146624,29972 +29973,29973 +29974,29974 +36164,29974 +29975,29975 +36165,29975 +29976,29976 +29977,29977 +36167,29977 +29978,29978 +36168,29978 +36169,29979 +29979,29979 +36170,29980 +29980,29980 +33276,29980 +29981,29981 +36171,29981 +29982,29982 +36172,29982 +36173,29983 +29983,29983 +36174,29984 +29984,29984 +29985,29985 +36175,29985 +36176,29986 +29986,29986 +29987,29987 +36177,29987 +29988,29988 +36178,29988 +29989,29989 +36179,29989 +29990,29990 +36180,29990 +29991,29991 +29992,29992 +29993,29993 +29994,29994 +147081,29994 +29995,29995 +36185,29995 +29996,29996 +29997,29997 +29998,29998 +29999,29999 +30000,30000 +194455,30000 +30001,30001 +30002,30002 +30003,30003 +30041,30041 +30136,30136 +33393,30136 +30137,30137 +30138,30138 +30139,30139 +30140,30140 +30141,30141 +33398,30141 +30142,30142 +33400,30143 +30143,30143 +36293,30143 +30144,30144 +30145,30145 +33402,30145 +33403,30146 +30146,30146 +36296,30146 +30147,30147 +30148,30148 +30149,30149 +30150,30150 +33407,30150 +30151,30151 +30152,30152 +30153,30153 +30154,30154 +30155,30155 +30156,30156 +30157,30157 +33414,30157 +30158,30158 +33415,30158 +30159,30159 +30160,30160 +30161,30161 +30162,30162 +30163,30163 +30164,30164 +30165,30165 +49120,30165 +30166,30166 +30167,30167 +30168,30168 +59020,30168 +30169,30169 +30170,30170 +30171,30171 +193147,30171 +85419,30172 +30172,30172 +30173,30173 +30174,30174 +30175,30175 +30176,30176 +30177,30177 +30178,30178 +30179,30179 +30180,30180 +30181,30181 +30182,30182 +33439,30182 +30183,30183 +30184,30184 +30185,30185 +30186,30186 +30187,30187 +30188,30188 +30189,30189 +30190,30190 +30191,30191 +30192,30192 +30193,30193 +30194,30194 +30195,30195 +30196,30196 +30197,30197 +30198,30198 +30199,30199 +30200,30200 +30201,30201 +30202,30202 +30203,30203 +30204,30204 +30205,30205 +30206,30206 +22663,30206 +30207,30207 +30208,30208 +30209,30209 +30210,30210 +30211,30211 +30212,30212 +30213,30213 +30214,30214 +30215,30215 +141544,30215 +30216,30216 +30217,30217 +30218,30218 +30219,30219 +30220,30220 +30221,30221 +30222,30222 +30223,30223 +30224,30224 +118118,30224 +30225,30225 +30226,30226 +30227,30227 +132943,30227 +125356,30227 +244823,30228 +30228,30228 +30229,30229 +106182,30229 +36365,30230 +30230,30230 +30231,30231 +30232,30232 +30233,30233 +36369,30234 +30234,30234 +36370,30235 +30235,30235 +30236,30236 +33487,30236 +55583,30236 +47479,30236 +30237,30237 +36372,30237 +33488,30237 +30238,30238 +33489,30238 +30239,30239 +33490,30239 +33491,30240 +30240,30240 +36375,30240 +30241,30241 +36376,30241 +33492,30241 +30242,30242 +36377,30242 +33493,30242 +30243,30243 +10018,30243 +30244,30244 +30245,30245 +30246,30246 +75215,30246 +30247,30247 +36383,30248 +33499,30248 +30248,30248 +30249,30249 +30250,30250 +234728,30250 +36385,30250 +30251,30251 +30252,30252 +30253,30253 +30254,30254 +30255,30255 +248925,30255 +30256,30256 +36391,30256 +30257,30257 +36392,30257 +30258,30258 +30259,30259 +30260,30260 +30261,30261 +30262,30262 +30263,30263 +36398,30263 +30264,30264 +30265,30265 +36400,30265 +30266,30266 +30267,30267 +30268,30268 +30269,30269 +30270,30270 +30271,30271 +30272,30272 +30273,30273 +30274,30274 +30275,30275 +30276,30276 +30277,30277 +30278,30278 +30279,30279 +30280,30280 +30281,30281 +30282,30282 +30283,30283 +233299,30283 +30284,30284 +30285,30285 +30286,30286 +30287,30287 +30288,30288 +30289,30289 +30290,30290 +30291,30291 +30292,30292 +30293,30293 +30294,30294 +30295,30295 +30296,30296 +30297,30297 +30298,30298 +30299,30299 +30341,30341 +30342,30342 +92122,30342 +30343,30343 +30344,30344 +30345,30345 +30346,30346 +30347,30347 +30348,30348 +30349,30349 +159254,30349 +30350,30350 +30351,30351 +30352,30352 +30353,30353 +30354,30354 +30355,30355 +30356,30356 +30357,30357 +30358,30358 +30359,30359 +30360,30360 +30361,30361 +30362,30362 +30363,30363 +30364,30364 +30365,30365 +30366,30366 +30367,30367 +189762,30367 +30368,30368 +30369,30369 +30370,30370 +30371,30371 +30372,30372 +30373,30373 +30374,30374 +30375,30375 +30376,30376 +30377,30377 +30378,30378 +30379,30379 +30380,30380 +30381,30381 +30382,30382 +30383,30383 +30384,30384 +30385,30385 +30386,30386 +30387,30387 +30388,30388 +30389,30389 +30390,30390 +30391,30391 +30392,30392 +36530,30392 +30393,30393 +30394,30394 +30395,30395 +30396,30396 +30397,30397 +30437,30437 +30438,30438 +30439,30439 +30440,30440 +225277,30440 +30441,30441 +30442,30442 +30443,30443 +30444,30444 +30445,30445 +30446,30446 +30447,30447 +30448,30448 +30449,30449 +30450,30450 +30451,30451 +30452,30452 +30453,30453 +30454,30454 +30455,30455 +30456,30456 +30457,30457 +30458,30458 +30459,30459 +30460,30460 +30461,30461 +30462,30462 +36600,30462 +30463,30463 +30464,30464 +30465,30465 +30466,30466 +30467,30467 +30468,30468 +30469,30469 +30470,30470 +30471,30471 +30472,30472 +30473,30473 +36611,30473 +36612,30474 +30474,30474 +178493,30474 +172529,30474 +36613,30475 +30475,30475 +36614,30476 +30476,30476 +36615,30477 +30477,30477 +36616,30478 +30478,30478 +36617,30479 +30479,30479 +36618,30480 +30480,30480 +36619,30481 +30481,30481 +36620,30482 +30482,30482 +36621,30483 +30483,30483 +28232,30483 +36622,30484 +30484,30484 +140708,30484 +36623,30485 +30485,30485 +30486,30486 +36624,30486 +30487,30487 +36625,30487 +178393,30487 +172429,30487 +30488,30488 +36626,30488 +30489,30489 +30490,30490 +36628,30490 +30491,30491 +33748,30491 +30492,30492 +33749,30492 +30493,30493 +30494,30494 +33751,30494 +30495,30495 +36712,30495 +33831,30495 +30496,30496 +30497,30497 +33754,30497 +30498,30498 +30499,30499 +146621,30499 +30500,30500 +30501,30501 +70194,30501 +30502,30502 +30503,30503 +30504,30504 +30505,30505 +30506,30506 +30507,30507 +30508,30508 +30509,30509 +30510,30510 +30511,30511 +30512,30512 +30513,30513 +30514,30514 +162996,30514 +119586,30514 +30515,30515 +30516,30516 +228717,30516 +10487,30516 +30517,30517 +30518,30518 +30519,30519 +30520,30520 +30521,30521 +30522,30522 +14434,30522 +30523,30523 +140617,30524 +30524,30524 +30525,30525 +30526,30526 +30527,30527 +30528,30528 +30529,30529 +30530,30530 +30531,30531 +30532,30532 +30533,30533 +30534,30534 +30535,30535 +30536,30536 +30537,30537 +30538,30538 +30539,30539 +30540,30540 +30541,30541 +30542,30542 +30543,30543 +30544,30544 +36682,30544 +30545,30545 +36683,30545 +30546,30546 +30547,30547 +30548,30548 +36686,30548 +36687,30549 +30549,30549 +30550,30550 +30551,30551 +36689,30551 +30552,30552 +36690,30552 +30553,30553 +36691,30553 +30554,30554 +36693,30555 +30555,30555 +30556,30556 +36694,30556 +30557,30557 +30558,30558 +30559,30559 +30560,30560 +160084,30560 +30561,30561 +30562,30562 +30563,30563 +30564,30564 +30565,30565 +30566,30566 +30567,30567 +30568,30568 +30569,30569 +30570,30570 +30571,30571 +30572,30572 +146494,30572 +30573,30573 +177786,30573 +171822,30573 +30574,30574 +30575,30575 +30576,30576 +30577,30577 +30578,30578 +30579,30579 +30580,30580 +192008,30580 +30581,30581 +30582,30582 +30583,30583 +30584,30584 +36722,30584 +140510,30584 +163406,30584 +30585,30585 +30586,30586 +30587,30587 +57087,30587 +30588,30588 +30589,30589 +30590,30590 +36728,30590 +30591,30591 +30592,30592 +30593,30593 +30594,30594 +30595,30595 +30596,30596 +220014,30596 +192572,30596 +30597,30597 +30598,30598 +30599,30599 +30600,30600 +30601,30601 +30602,30602 +30930,30602 +36740,30602 +30603,30603 +30604,30604 +30605,30605 +30606,30606 +30607,30607 +30608,30608 +30609,30609 +30610,30610 +30611,30611 +30612,30612 +30613,30613 +30614,30614 +30615,30615 +30616,30616 +30617,30617 +30618,30618 +30619,30619 +30620,30620 +30621,30621 +149085,30621 +30622,30622 +30623,30623 +30624,30624 +30625,30625 +30626,30626 +30627,30627 +30628,30628 +30629,30629 +30630,30630 +30631,30631 +30632,30632 +30633,30633 +30634,30634 +30635,30635 +30636,30636 +30637,30637 +30638,30638 +33884,30638 +30639,30639 +33886,30640 +30640,30640 +30641,30641 +33887,30641 +36827,30689 +30689,30689 +64974,30689 +30690,30690 +15367,30690 +30691,30691 +36829,30691 +30692,30692 +30693,30693 +30694,30694 +30695,30695 +30696,30696 +30697,30697 +30698,30698 +30699,30699 +30700,30700 +30701,30701 +95924,30701 +30702,30702 +30703,30703 +30704,30704 +30705,30705 +30706,30706 +30707,30707 +30708,30708 +30709,30709 +30710,30710 +30711,30711 +30712,30712 +70394,30712 +30713,30713 +30714,30714 +30715,30715 +30716,30716 +30717,30717 +30718,30718 +30719,30719 +30720,30720 +30721,30721 +30722,30722 +30723,30723 +30724,30724 +30725,30725 +30726,30726 +30727,30727 +30728,30728 +30729,30729 +30730,30730 +30731,30731 +30732,30732 +30733,30733 +224372,30733 +30734,30734 +30735,30735 +30736,30736 +30737,30737 +30738,30738 +30739,30739 +30740,30740 +30741,30741 +30742,30742 +30743,30743 +30744,30744 +30745,30745 +30746,30746 +30747,30747 +30748,30748 +30749,30749 +30750,30750 +30751,30751 +30752,30752 +75330,30752 +30753,30753 +30754,30754 +30755,30755 +30756,30756 +34003,30757 +30757,30757 +36895,30757 +30758,30758 +34004,30758 +30759,30759 +30760,30760 +30761,30761 +30762,30762 +30763,30763 +30764,30764 +30765,30765 +30766,30766 +30767,30767 +30768,30768 +30769,30769 +77386,30769 +30770,30770 +30771,30771 +30772,30772 +30773,30773 +67115,30773 +30774,30774 +30775,30775 +30776,30776 +30777,30777 +30778,30778 +30779,30779 +30780,30780 +30781,30781 +30782,30782 +30783,30783 +30784,30784 +30785,30785 +30786,30786 +30787,30787 +121207,30787 +30788,30788 +30789,30789 +30790,30790 +30791,30791 +30792,30792 +37511,30792 +30793,30793 +30794,30794 +30795,30795 +30796,30796 +30797,30797 +30798,30798 +30799,30799 +30800,30800 +30801,30801 +30802,30802 +30803,30803 +30804,30804 +30805,30805 +30806,30806 +223034,30806 +30807,30807 +234193,30808 +30808,30808 +51654,30808 +43550,30808 +30809,30809 +30810,30810 +30811,30811 +30812,30812 +30813,30813 +209479,30813 +30814,30814 +30815,30815 +30816,30816 +30817,30817 +223778,30817 +30818,30818 +30819,30819 +30820,30820 +30821,30821 +30822,30822 +30823,30823 +30824,30824 +198027,30824 +30825,30825 +30826,30826 +36964,30826 +30827,30827 +30828,30828 +30829,30829 +30830,30830 +30831,30831 +30832,30832 +36970,30832 +30833,30833 +36971,30833 +30834,30834 +30835,30835 +36973,30835 +30836,30836 +30837,30837 +30838,30838 +34057,30838 +36977,30839 +30839,30839 +36978,30840 +34059,30840 +30840,30840 +36979,30841 +34060,30841 +30841,30841 +36980,30842 +30842,30842 +34061,30842 +36981,30843 +30843,30843 +34062,30843 +36982,30844 +34063,30844 +30844,30844 +36983,30845 +34064,30845 +30845,30845 +36984,30846 +30846,30846 +34065,30846 +36985,30847 +34066,30847 +30847,30847 +30848,30848 +36986,30848 +34067,30848 +30849,30849 +36987,30849 +34068,30849 +30850,30850 +36988,30850 +34069,30850 +34070,30851 +30851,30851 +36989,30851 +36990,30852 +34071,30852 +30852,30852 +34072,30853 +30853,30853 +36991,30853 +36992,30854 +34073,30854 +30854,30854 +34074,30855 +30855,30855 +30856,30856 +36994,30856 +30857,30857 +30858,30858 +30859,30859 +34079,30860 +30860,30860 +36999,30861 +30861,30861 +34081,30862 +30862,30862 +30863,30863 +37001,30863 +34083,30864 +30864,30864 +34084,30865 +30865,30865 +30866,30866 +30867,30867 +30868,30868 +30869,30869 +30870,30870 +30871,30871 +30872,30872 +30873,30873 +30874,30874 +30875,30875 +30876,30876 +30877,30877 +30878,30878 +30879,30879 +30880,30880 +30881,30881 +30882,30882 +216131,30882 +30883,30883 +30884,30884 +30885,30885 +30886,30886 +30887,30887 +30888,30888 +30889,30889 +30890,30890 +30891,30891 +37029,30891 +30892,30892 +37031,30893 +30893,30893 +30894,30894 +108698,30894 +30895,30895 +58161,30895 +218020,30895 +30896,30896 +37034,30896 +30897,30897 +30898,30898 +30899,30899 +30900,30900 +30901,30901 +30902,30902 +30903,30903 +30904,30904 +30905,30905 +30906,30906 +136258,30906 +128671,30906 +30907,30907 +30908,30908 +30909,30909 +30910,30910 +30911,30911 +30912,30912 +30913,30913 +30914,30914 +30915,30915 +30916,30916 +191047,30916 +30917,30917 +30918,30918 +67116,30918 +30919,30919 +31162,30919 +30920,30920 +30921,30921 +30922,30922 +30923,30923 +30924,30924 +30925,30925 +30926,30926 +30927,30927 +30928,30928 +30929,30929 +30930,30930 +30602,30930 +30931,30931 +30932,30932 +188256,30932 +212309,30932 +30933,30933 +30934,30934 +30935,30935 +30936,30936 +30937,30937 +30938,30938 +30939,30939 +30940,30940 +30941,30941 +30942,30942 +30943,30943 +30944,30944 +30945,30945 +30946,30946 +30947,30947 +30948,30948 +30949,30949 +30950,30950 +147520,30950 +30951,30951 +30952,30952 +30953,30953 +30954,30954 +30955,30955 +30956,30956 +1051,30956 +30957,30957 +30958,30958 +30959,30959 +30960,30960 +30961,30961 +30962,30962 +30963,30963 +30964,30964 +30965,30965 +30966,30966 +30967,30967 +30968,30968 +30969,30969 +30970,30970 +80,30970 +37124,30971 +30971,30971 +30972,30972 +30973,30973 +30974,30974 +30975,30975 +30976,30976 +30977,30977 +30978,30978 +30979,30979 +71345,30979 +30980,30980 +30981,30981 +30982,30982 +60134,30982 +30983,30983 +30984,30984 +30985,30985 +30986,30986 +30987,30987 +30988,30988 +30989,30989 +30990,30990 +30991,30991 +30992,30992 +30993,30993 +31032,31032 +31033,31033 +31034,31034 +31035,31035 +31036,31036 +31037,31037 +31038,31038 +31039,31039 +31040,31040 +31041,31041 +31042,31042 +31043,31043 +31044,31044 +31045,31045 +37199,31046 +31046,31046 +31047,31047 +31048,31048 +31049,31049 +31050,31050 +31051,31051 +31052,31052 +31053,31053 +31054,31054 +31055,31055 +31056,31056 +31057,31057 +31058,31058 +31059,31059 +31060,31060 +31061,31061 +31062,31062 +31063,31063 +31064,31064 +31065,31065 +31066,31066 +31067,31067 +31068,31068 +31069,31069 +31070,31070 +31071,31071 +31072,31072 +31073,31073 +31124,31124 +31125,31125 +31126,31126 +31127,31127 +31128,31128 +31129,31129 +31130,31130 +38230,31130 +31131,31131 +91988,31131 +31132,31132 +31133,31133 +31134,31134 +38234,31134 +35000,31134 +31135,31135 +38235,31135 +35001,31135 +31136,31136 +31137,31137 +5680,31137 +31138,31138 +31139,31139 +31140,31140 +31141,31141 +31142,31142 +31143,31143 +31144,31144 +74158,31144 +31145,31145 +193687,31145 +31146,31146 +38246,31146 +31147,31147 +38247,31147 +38248,31148 +31148,31148 +31149,31149 +31150,31150 +31151,31151 +31152,31152 +31153,31153 +31154,31154 +35020,31154 +31155,31155 +35021,31155 +31156,31156 +31157,31157 +31158,31158 +31159,31159 +233977,31159 +31160,31160 +31161,31161 +31162,31162 +30919,31162 +31163,31163 +31164,31164 +31165,31165 +31166,31166 +31167,31167 +31168,31168 +31169,31169 +31170,31170 +31171,31171 +31172,31172 +31173,31173 +31174,31174 +31175,31175 +31176,31176 +31177,31177 +31178,31178 +31179,31179 +31180,31180 +31181,31181 +31182,31182 +31183,31183 +31184,31184 +31185,31185 +31186,31186 +31187,31187 +31188,31188 +209802,31188 +31189,31189 +31190,31190 +31191,31191 +31192,31192 +31193,31193 +31194,31194 +31195,31195 +26220,31195 +31196,31196 +99312,31196 +31197,31197 +31198,31198 +31199,31199 +31200,31200 +31201,31201 +31202,31202 +31203,31203 +31204,31204 +31205,31205 +31206,31206 +31207,31207 +31208,31208 +31209,31209 +31210,31210 +31211,31211 +31212,31212 +31213,31213 +31214,31214 +31215,31215 +6698,31215 +31216,31216 +31217,31217 +31218,31218 +31219,31219 +31220,31220 +31221,31221 +31222,31222 +31223,31223 +31224,31224 +31225,31225 +31226,31226 +31227,31227 +82774,31227 +31228,31228 +31229,31229 +191213,31230 +31230,31230 +31231,31231 +31232,31232 +31233,31233 +31234,31234 +31235,31235 +31236,31236 +31237,31237 +31238,31238 +219959,31239 +31239,31239 +31240,31240 +31241,31241 +31242,31242 +38396,31242 +31243,31243 +31244,31244 +31245,31245 +31246,31246 +31247,31247 +31248,31248 +31249,31249 +31250,31250 +31251,31251 +31252,31252 +31253,31253 +85975,31253 +31254,31254 +31255,31255 +88941,31256 +31256,31256 +31257,31257 +31258,31258 +31259,31259 +31260,31260 +31261,31261 +31262,31262 +31263,31263 +31264,31264 +31265,31265 +31266,31266 +31267,31267 +31268,31268 +31269,31269 +31270,31270 +31271,31271 +31272,31272 +31273,31273 +31274,31274 +77392,31274 +31275,31275 +31276,31276 +31277,31277 +31278,31278 +31279,31279 +31280,31280 +31281,31281 +31282,31282 +31283,31283 +31284,31284 +31285,31285 +31286,31286 +31287,31287 +31288,31288 +31289,31289 +153911,31289 +31290,31290 +31291,31291 +31292,31292 +31293,31293 +31294,31294 +31295,31295 +31296,31296 +31344,31344 +31345,31345 +31346,31346 +31347,31347 +187395,31348 +31348,31348 +31349,31349 +147551,31349 +31469,31469 +31470,31470 +31471,31471 +31472,31472 +31473,31473 +31474,31474 +31475,31475 +31476,31476 +123496,31476 +31477,31477 +31478,31478 +31479,31479 +31480,31480 +31481,31481 +31482,31482 +31483,31483 +31484,31484 +31485,31485 +31486,31486 +31487,31487 +31488,31488 +31489,31489 +31490,31490 +31491,31491 +31492,31492 +31493,31493 +31494,31494 +5613,31494 +31622,31622 +31623,31623 +31624,31624 +31625,31625 +31626,31626 +31627,31627 +31628,31628 +31629,31629 +31630,31630 +31631,31631 +31632,31632 +31633,31633 +31634,31634 +31635,31635 +31636,31636 +31637,31637 +61490,31637 +31638,31638 +31639,31639 +31640,31640 +31641,31641 +31642,31642 +31643,31643 +133034,31643 +125447,31643 +31644,31644 +31645,31645 +31646,31646 +31647,31647 +31648,31648 +31649,31649 +31650,31650 +160788,31650 +31651,31651 +235198,31651 +31652,31652 +31653,31653 +31654,31654 +31655,31655 +31656,31656 +132398,31656 +124811,31656 +31657,31657 +31658,31658 +31659,31659 +31660,31660 +31661,31661 +31662,31662 +31663,31663 +31664,31664 +31665,31665 +31666,31666 +31667,31667 +31668,31668 +31669,31669 +179733,31669 +31670,31670 +31671,31671 +31672,31672 +31673,31673 +31674,31674 +31675,31675 +149794,31675 +31676,31676 +31677,31677 +11995,31677 +31678,31678 +31679,31679 +31680,31680 +31681,31681 +31682,31682 +31683,31683 +31684,31684 +31685,31685 +31686,31686 +31687,31687 +31688,31688 +31689,31689 +31690,31690 +31691,31691 +31692,31692 +31693,31693 +31694,31694 +31695,31695 +31696,31696 +31697,31697 +31698,31698 +31699,31699 +111671,31699 +31700,31700 +31701,31701 +31702,31702 +87378,31702 +31703,31703 +31704,31704 +31705,31705 +31706,31706 +31707,31707 +31708,31708 +53443,31708 +45339,31708 +31709,31709 +31710,31710 +31711,31711 +31712,31712 +31713,31713 +31714,31714 +31715,31715 +31765,31765 +31766,31766 +31767,31767 +31768,31768 +31769,31769 +31770,31770 +31771,31771 +31772,31772 +31773,31773 +31774,31774 +31775,31775 +31776,31776 +31777,31777 +31778,31778 +31779,31779 +31780,31780 +31781,31781 +31782,31782 +31783,31783 +31784,31784 +118115,31784 +31785,31785 +31815,31815 +31816,31816 +31817,31817 +31818,31818 +31819,31819 +31820,31820 +31821,31821 +31822,31822 +2304,31822 +158606,31822 +31823,31823 +31824,31824 +200749,31824 +31825,31825 +31826,31826 +31827,31827 +31828,31828 +32059,32059 +32060,32060 +32300,32300 +32301,32301 +32302,32302 +100554,32302 +32303,32303 +35023,32303 +32304,32304 +32305,32305 +32306,32306 +32307,32307 +32308,32308 +32309,32309 +32310,32310 +32311,32311 +32312,32312 +32313,32313 +32314,32314 +32315,32315 +32316,32316 +32317,32317 +211186,32317 +32318,32318 +32319,32319 +32320,32320 +138029,32320 +130442,32320 +32321,32321 +32322,32322 +32323,32323 +32324,32324 +32325,32325 +32326,32326 +32327,32327 +32328,32328 +32329,32329 +32330,32330 +32331,32331 +32332,32332 +32333,32333 +136139,32333 +128552,32333 +32334,32334 +32335,32335 +32336,32336 +32337,32337 +32338,32338 +32339,32339 +32340,32340 +120434,32340 +32341,32341 +32342,32342 +32343,32343 +32344,32344 +32345,32345 +32346,32346 +32347,32347 +32348,32348 +32349,32349 +32350,32350 +32351,32351 +32352,32352 +32353,32353 +32354,32354 +32355,32355 +32356,32356 +32357,32357 +32358,32358 +32359,32359 +32360,32360 +32361,32361 +32362,32362 +32363,32363 +32364,32364 +32365,32365 +32366,32366 +32367,32367 +32368,32368 +32369,32369 +200536,32369 +32370,32370 +32371,32371 +32372,32372 +32373,32373 +32374,32374 +32375,32375 +32376,32376 +32377,32377 +32378,32378 +165217,32378 +32379,32379 +77125,32379 +32380,32380 +32381,32381 +32382,32382 +32383,32383 +32384,32384 +32385,32385 +32386,32386 +32387,32387 +32388,32388 +12140,32388 +32389,32389 +32390,32390 +32391,32391 +32392,32392 +32393,32393 +32394,32394 +32395,32395 +32396,32396 +32397,32397 +32398,32398 +32399,32399 +32400,32400 +32401,32401 +32402,32402 +32403,32403 +77434,32403 +32404,32404 +32405,32405 +32406,32406 +32407,32407 +32408,32408 +32409,32409 +32410,32410 +160695,32410 +32411,32411 +32412,32412 +32413,32413 +32414,32414 +32415,32415 +32416,32416 +32417,32417 +32418,32418 +32419,32419 +113301,32419 +32420,32420 +32421,32421 +32422,32422 +32423,32423 +32424,32424 +32425,32425 +32426,32426 +220624,32426 +32427,32427 +32428,32428 +32429,32429 +32430,32430 +32431,32431 +32432,32432 +32433,32433 +32434,32434 +32435,32435 +32436,32436 +32437,32437 +32438,32438 +32439,32439 +32440,32440 +32441,32441 +32442,32442 +32443,32443 +32444,32444 +32445,32445 +32446,32446 +155604,32446 +32447,32447 +32448,32448 +32449,32449 +32450,32450 +32451,32451 +74962,32451 +32452,32452 +32453,32453 +32454,32454 +32455,32455 +32456,32456 +32457,32457 +32458,32458 +32459,32459 +32460,32460 +32461,32461 +32462,32462 +32463,32463 +32464,32464 +32465,32465 +32466,32466 +32467,32467 +32468,32468 +32469,32469 +32470,32470 +32471,32471 +32472,32472 +32473,32473 +32474,32474 +32475,32475 +32476,32476 +32477,32477 +32478,32478 +32479,32479 +32480,32480 +32481,32481 +32482,32482 +32483,32483 +32484,32484 +32485,32485 +32486,32486 +32487,32487 +32488,32488 +32489,32489 +32490,32490 +32491,32491 +32492,32492 +32493,32493 +32494,32494 +32495,32495 +32496,32496 +32497,32497 +32498,32498 +32499,32499 +32500,32500 +32501,32501 +32502,32502 +154910,32502 +32503,32503 +32504,32504 +32505,32505 +32551,32551 +32552,32552 +32553,32553 +32554,32554 +32555,32555 +32556,32556 +32557,32557 +32558,32558 +32559,32559 +32560,32560 +32561,32561 +32562,32562 +32563,32563 +32564,32564 +32770,32770 +32771,32771 +32772,32772 +32773,32773 +122258,32773 +32774,32774 +32775,32775 +32776,32776 +32777,32777 +32846,32846 +32847,32847 +32848,32848 +32849,32849 +32850,32850 +32851,32851 +32852,32852 +32853,32853 +32854,32854 +105133,32855 +32855,32855 +32856,32856 +32857,32857 +32858,32858 +32859,32859 +32860,32860 +32861,32861 +32862,32862 +32863,32863 +32864,32864 +32865,32865 +53241,32865 +45137,32865 +32866,32866 +32867,32867 +32868,32868 +230379,32868 +32869,32869 +32870,32870 +32871,32871 +32872,32872 +32873,32873 +32874,32874 +32875,32875 +32876,32876 +32877,32877 +32878,32878 +32879,32879 +32880,32880 +32881,32881 +32882,32882 +32883,32883 +32884,32884 +32885,32885 +32886,32886 +237977,32886 +32887,32887 +32888,32888 +32889,32889 +32890,32890 +32891,32891 +32892,32892 +32893,32893 +32894,32894 +32895,32895 +32896,32896 +233866,32896 +32897,32897 +32898,32898 +32899,32899 +32900,32900 +32901,32901 +32902,32902 +32903,32903 +32904,32904 +32905,32905 +32906,32906 +32907,32907 +32908,32908 +32909,32909 +32959,32959 +32960,32960 +32961,32961 +32962,32962 +32963,32963 +32964,32964 +32965,32965 +32966,32966 +32967,32967 +201133,32967 +32968,32968 +32969,32969 +32970,32970 +32971,32971 +32972,32972 +32973,32973 +32974,32974 +32975,32975 +32976,32976 +32977,32977 +32978,32978 +32979,32979 +32980,32980 +32981,32981 +32982,32982 +32983,32983 +32984,32984 +32985,32985 +32986,32986 +32987,32987 +32988,32988 +32989,32989 +32990,32990 +32991,32991 +32992,32992 +32993,32993 +32994,32994 +32995,32995 +32996,32996 +32997,32997 +32998,32998 +32999,32999 +33000,33000 +33001,33001 +117688,33001 +29541,33001 +33002,33002 +7739,33002 +33003,33003 +33004,33004 +20113,33004 +33005,33005 +33006,33006 +33007,33007 +33008,33008 +33009,33009 +33010,33010 +33011,33011 +33012,33012 +33013,33013 +33014,33014 +33015,33015 +33016,33016 +33017,33017 +33018,33018 +33019,33019 +33020,33020 +33021,33021 +33022,33022 +33023,33023 +33024,33024 +33025,33025 +33026,33026 +33027,33027 +33028,33028 +33029,33029 +33030,33030 +33031,33031 +33032,33032 +33033,33033 +33034,33034 +33035,33035 +33036,33036 +33037,33037 +75712,33037 +33038,33038 +19838,33038 +33039,33039 +33040,33040 +33041,33041 +33042,33042 +33043,33043 +33044,33044 +33045,33045 +33046,33046 +33047,33047 +33048,33048 +33049,33049 +33050,33050 +33051,33051 +33052,33052 +62874,33052 +33186,33186 +33187,33187 +206937,33187 +33188,33188 +33189,33189 +33190,33190 +33191,33191 +33192,33192 +33193,33193 +33194,33194 +33195,33195 +33196,33196 +36087,33196 +33197,33197 +178827,33197 +33198,33198 +33199,33199 +33200,33200 +36091,33200 +36092,33201 +33201,33201 +33202,33202 +33203,33203 +36094,33203 +33204,33204 +33205,33205 +212359,33205 +33206,33206 +33207,33207 +80475,33207 +36098,33207 +36099,33208 +33208,33208 +33209,33209 +33210,33210 +33211,33211 +33212,33212 +33213,33213 +36104,33213 +33214,33214 +33215,33215 +33216,33216 +134089,33216 +126502,33216 +33217,33217 +4370,33217 +33218,33218 +36109,33218 +33219,33219 +33220,33220 +64327,33220 +33221,33221 +33222,33222 +33223,33223 +33224,33224 +33225,33225 +33226,33226 +33227,33227 +33228,33228 +33229,33229 +33230,33230 +33231,33231 +33232,33232 +158802,33232 +992,33232 +33233,33233 +33234,33234 +33235,33235 +33236,33236 +33237,33237 +33238,33238 +33239,33239 +33240,33240 +33241,33241 +33242,33242 +33243,33243 +33244,33244 +33245,33245 +33246,33246 +33247,33247 +33248,33248 +33249,33249 +33250,33250 +33251,33251 +33252,33252 +36146,33252 +33253,33253 +33254,33254 +33255,33255 +33256,33256 +33257,33257 +33258,33258 +135504,33258 +127917,33258 +33259,33259 +29963,33259 +33260,33260 +33261,33261 +33262,33262 +29966,33262 +33263,33263 +36157,33263 +29967,33263 +33264,33264 +33265,33265 +29969,33265 +5829,33265 +33266,33266 +29970,33266 +33267,33267 +33268,33268 +33269,33269 +33270,33270 +33271,33271 +33272,33272 +33273,33273 +248762,33273 +33274,33274 +33275,33275 +33276,33276 +36170,33276 +29980,33276 +33277,33277 +33278,33278 +33279,33279 +33280,33280 +6698,33280 +33281,33281 +33282,33282 +33283,33283 +33284,33284 +33331,33331 +33332,33332 +33333,33333 +33334,33334 +33335,33335 +33336,33336 +33337,33337 +33338,33338 +33339,33339 +33340,33340 +33341,33341 +33342,33342 +60651,33342 +33343,33343 +33344,33344 +33345,33345 +33346,33346 +33347,33347 +33348,33348 +60736,33349 +33349,33349 +33350,33350 +33351,33351 +33352,33352 +33353,33353 +33354,33354 +33355,33355 +33356,33356 +33357,33357 +33358,33358 +33359,33359 +33360,33360 +36253,33360 +575,33360 +33361,33361 +205465,33361 +33362,33362 +33363,33363 +36256,33363 +33364,33364 +33365,33365 +33366,33366 +36259,33366 +33367,33367 +33368,33368 +120958,33369 +33369,33369 +33370,33370 +193810,33370 +33371,33371 +33372,33372 +33373,33373 +117706,33373 +33374,33374 +36267,33374 +33375,33375 +36268,33375 +36269,33376 +33376,33376 +33377,33377 +36270,33377 +36271,33378 +33378,33378 +33379,33379 +36272,33379 +36273,33380 +33380,33380 +36274,33381 +33381,33381 +33382,33382 +36275,33382 +33383,33383 +36276,33383 +36277,33384 +33384,33384 +33385,33385 +36278,33385 +33386,33386 +33387,33387 +36280,33387 +33388,33388 +33389,33389 +36282,33389 +33390,33390 +194125,33390 +33391,33391 +33392,33392 +33393,33393 +30136,33393 +33394,33394 +33395,33395 +33396,33396 +33397,33397 +33398,33398 +30141,33398 +33399,33399 +33400,33400 +30143,33400 +36293,33400 +33401,33401 +33402,33402 +30145,33402 +33403,33403 +30146,33403 +36296,33403 +249248,33404 +33404,33404 +33405,33405 +33406,33406 +33407,33407 +30150,33407 +33408,33408 +33409,33409 +33410,33410 +33411,33411 +33412,33412 +101029,33412 +33413,33413 +33414,33414 +30157,33414 +33415,33415 +30158,33415 +33416,33416 +33417,33417 +33418,33418 +33419,33419 +33420,33420 +33421,33421 +33422,33422 +34080,33422 +33423,33423 +33424,33424 +33425,33425 +77262,33425 +33426,33426 +33427,33427 +33428,33428 +137968,33428 +130381,33428 +33429,33429 +33430,33430 +33431,33431 +33432,33432 +140670,33432 +109267,33432 +33433,33433 +33434,33434 +33435,33435 +33436,33436 +65008,33437 +33437,33437 +33438,33438 +33439,33439 +30182,33439 +33487,33487 +30236,33487 +36371,33487 +36372,33488 +33488,33488 +30237,33488 +33489,33489 +30238,33489 +36373,33489 +33490,33490 +30239,33490 +36374,33490 +33491,33491 +30240,33491 +36375,33491 +36376,33492 +33492,33492 +30241,33492 +36377,33493 +33493,33493 +30242,33493 +36378,33494 +33494,33494 +36379,33495 +33495,33495 +36380,33496 +33496,33496 +36381,33497 +33497,33497 +36382,33498 +33498,33498 +36383,33499 +33499,33499 +30248,33499 +36384,33500 +33500,33500 +33501,33501 +36385,33501 +33502,33502 +36386,33502 +139126,33502 +33503,33503 +36387,33503 +33504,33504 +36388,33504 +36389,33505 +33505,33505 +33506,33506 +36390,33506 +33507,33507 +36391,33507 +132936,33507 +125349,33507 +33508,33508 +33509,33509 +33510,33510 +33511,33511 +33512,33512 +33513,33513 +33514,33514 +33515,33515 +33516,33516 +36400,33516 +33517,33517 +33518,33518 +33519,33519 +24653,33519 +33520,33520 +33521,33521 +211960,33521 +33522,33522 +33523,33523 +33524,33524 +33525,33525 +141629,33525 +33526,33526 +33527,33527 +218420,33527 +33528,33528 +33529,33529 +33530,33530 +33531,33531 +36415,33531 +33532,33532 +33533,33533 +33534,33534 +33535,33535 +33536,33536 +33537,33537 +36422,33538 +33538,33538 +33539,33539 +33540,33540 +33541,33541 +33542,33542 +33543,33543 +33544,33544 +33545,33545 +33546,33546 +33547,33547 +33548,33548 +33549,33549 +33550,33550 +191251,33550 +33551,33551 +33552,33552 +33588,33588 +33589,33589 +33590,33590 +33591,33591 +33592,33592 +33593,33593 +33594,33594 +33595,33595 +33596,33596 +33597,33597 +33598,33598 +33599,33599 +33600,33600 +33601,33601 +33602,33602 +33603,33603 +191979,33603 +33604,33604 +33605,33605 +33606,33606 +33607,33607 +33608,33608 +33609,33609 +33610,33610 +33611,33611 +33612,33612 +33613,33613 +33614,33614 +33615,33615 +33616,33616 +33617,33617 +33618,33618 +33619,33619 +33620,33620 +33621,33621 +33622,33622 +56237,33622 +48133,33622 +33623,33623 +33624,33624 +33625,33625 +33626,33626 +33627,33627 +33628,33628 +33629,33629 +64949,33629 +33630,33630 +33631,33631 +33632,33632 +33633,33633 +33634,33634 +33635,33635 +33636,33636 +33637,33637 +33638,33638 +33639,33639 +33640,33640 +187281,33640 +33641,33641 +33642,33642 +33643,33643 +33644,33644 +33645,33645 +33646,33646 +33647,33647 +33648,33648 +33649,33649 +33650,33650 +33651,33651 +33652,33652 +33653,33653 +33654,33654 +33655,33655 +33656,33656 +33657,33657 +33658,33658 +33659,33659 +33660,33660 +33661,33661 +33662,33662 +33663,33663 +33664,33664 +33665,33665 +33666,33666 +33667,33667 +162951,33667 +33668,33668 +33669,33669 +33710,33710 +33711,33711 +33712,33712 +33713,33713 +33714,33714 +33715,33715 +33716,33716 +33717,33717 +33718,33718 +33719,33719 +33720,33720 +33721,33721 +33722,33722 +33723,33723 +33724,33724 +21945,33724 +33725,33725 +33726,33726 +33727,33727 +33728,33728 +33729,33729 +33730,33730 +33731,33731 +194264,33731 +33732,33732 +33733,33733 +33734,33734 +33735,33735 +33736,33736 +33737,33737 +33738,33738 +33739,33739 +33740,33740 +33741,33741 +142602,33741 +53690,33741 +45586,33741 +33742,33742 +33743,33743 +33744,33744 +33745,33745 +33746,33746 +33747,33747 +33748,33748 +30491,33748 +33749,33749 +30492,33749 +33750,33750 +33751,33751 +30494,33751 +33752,33752 +33753,33753 +33754,33754 +30497,33754 +33755,33755 +33756,33756 +33757,33757 +33758,33758 +33759,33759 +33760,33760 +103379,33760 +33761,33761 +33762,33762 +33763,33763 +33764,33764 +145479,33764 +33765,33765 +33766,33766 +33767,33767 +33768,33768 +33769,33769 +33770,33770 +33771,33771 +33772,33772 +33773,33773 +33774,33774 +79636,33774 +33775,33775 +33776,33776 +33777,33777 +33778,33778 +33779,33779 +33780,33780 +33781,33781 +33782,33782 +33783,33783 +33784,33784 +33785,33785 +33786,33786 +33787,33787 +33788,33788 +33789,33789 +33790,33790 +58332,33790 +33791,33791 +33792,33792 +33793,33793 +33794,33794 +33795,33795 +33796,33796 +33797,33797 +33798,33798 +33799,33799 +33800,33800 +33801,33801 +96443,33801 +33802,33802 +33803,33803 +33804,33804 +33805,33805 +33806,33806 +33807,33807 +33808,33808 +33809,33809 +33810,33810 +33811,33811 +33812,33812 +33813,33813 +33814,33814 +33815,33815 +208476,33815 +33816,33816 +214720,33817 +33817,33817 +33818,33818 +33819,33819 +33820,33820 +180831,33821 +33821,33821 +177770,33821 +171806,33821 +33822,33822 +177771,33822 +171807,33822 +33823,33823 +33824,33824 +33825,33825 +33826,33826 +33827,33827 +33828,33828 +33829,33829 +150703,33829 +33830,33830 +33831,33831 +12958,33831 +30495,33831 +33832,33832 +33833,33833 +33834,33834 +33835,33835 +10159,33835 +33836,33836 +33837,33837 +33838,33838 +33839,33839 +33840,33840 +33841,33841 +33842,33842 +33843,33843 +33844,33844 +33845,33845 +33846,33846 +33847,33847 +33848,33848 +9474,33848 +33849,33849 +33850,33850 +33851,33851 +33852,33852 +33853,33853 +33854,33854 +33855,33855 +81058,33855 +33856,33856 +33857,33857 +33858,33858 +33859,33859 +33860,33860 +33861,33861 +33862,33862 +33863,33863 +33864,33864 +33865,33865 +33866,33866 +33867,33867 +33868,33868 +33869,33869 +33870,33870 +208530,33870 +33871,33871 +33872,33872 +33873,33873 +33874,33874 +33875,33875 +33876,33876 +33877,33877 +33878,33878 +33879,33879 +33880,33880 +33881,33881 +33882,33882 +33883,33883 +33884,33884 +30638,33884 +33885,33885 +33886,33886 +30640,33886 +33887,33887 +30641,33887 +33888,33888 +33889,33889 +33890,33890 +33891,33891 +230344,33891 +33892,33892 +33893,33893 +33894,33894 +116400,33894 +33895,33895 +33896,33896 +33897,33897 +33898,33898 +33899,33899 +33900,33900 +33901,33901 +33902,33902 +33903,33903 +33904,33904 +33905,33905 +33906,33906 +33907,33907 +33908,33908 +33909,33909 +33910,33910 +33911,33911 +33912,33912 +33913,33913 +33914,33914 +33915,33915 +33916,33916 +33917,33917 +33918,33918 +122172,33918 +164521,33918 +33919,33919 +33920,33920 +33921,33921 +33922,33922 +33923,33923 +33924,33924 +33925,33925 +33926,33926 +33927,33927 +5751,33927 +33928,33928 +177504,33928 +171540,33928 +33929,33929 +33930,33930 +33931,33931 +33932,33932 +33933,33933 +33934,33934 +33935,33935 +33936,33936 +33937,33937 +33938,33938 +33939,33939 +33940,33940 +33941,33941 +33942,33942 +33943,33943 +33944,33944 +33945,33945 +33946,33946 +33947,33947 +33948,33948 +33949,33949 +33950,33950 +33951,33951 +33952,33952 +33953,33953 +33954,33954 +33955,33955 +88077,33955 +33956,33956 +33957,33957 +33958,33958 +33959,33959 +33960,33960 +33961,33961 +249209,33961 +33962,33962 +33963,33963 +33964,33964 +33965,33965 +33966,33966 +33967,33967 +57173,33967 +33968,33968 +36860,33968 +33969,33969 +153022,33969 +33970,33970 +33971,33971 +33972,33972 +33973,33973 +33974,33974 +5724,33974 +33975,33975 +36324,33975 +33976,33976 +33977,33977 +36869,33977 +33978,33978 +33979,33979 +33980,33980 +33981,33981 +33982,33982 +33983,33983 +33984,33984 +33985,33985 +33986,33986 +151378,33986 +33987,33987 +33988,33988 +33989,33989 +33990,33990 +33991,33991 +33992,33992 +33993,33993 +33994,33994 +33995,33995 +33996,33996 +217030,33996 +33997,33997 +33998,33998 +33999,33999 +34000,34000 +34001,34001 +34002,34002 +34003,34003 +30757,34003 +36895,34003 +34004,34004 +30758,34004 +50108,34004 +41915,34004 +34005,34005 +34006,34006 +34007,34007 +34008,34008 +34009,34009 +34010,34010 +34011,34011 +154900,34011 +34012,34012 +34013,34013 +34014,34014 +34015,34015 +34016,34016 +59132,34016 +58758,34016 +34017,34017 +34018,34018 +34019,34019 +34020,34020 +34021,34021 +34022,34022 +34023,34023 +34024,34024 +34025,34025 +34026,34026 +34027,34027 +34028,34028 +34029,34029 +64696,34029 +83229,34029 +34030,34030 +34031,34031 +34032,34032 +34033,34033 +148998,34033 +34034,34034 +34035,34035 +34036,34036 +34037,34037 +184514,34037 +34038,34038 +34039,34039 +34040,34040 +34041,34041 +34042,34042 +34043,34043 +34044,34044 +34045,34045 +85230,34045 +34046,34046 +34047,34047 +206198,34047 +34048,34048 +34049,34049 +34050,34050 +34051,34051 +34052,34052 +132291,34052 +124704,34052 +34053,34053 +34054,34054 +34055,34055 +34056,34056 +34057,34057 +30838,34057 +34058,34058 +5515,34058 +36978,34059 +34059,34059 +30840,34059 +36979,34060 +34060,34060 +30841,34060 +34061,34061 +36980,34061 +30842,34061 +132300,34061 +124713,34061 +34062,34062 +36981,34062 +30843,34062 +36982,34063 +34063,34063 +30844,34063 +36983,34064 +34064,34064 +30845,34064 +34065,34065 +36984,34065 +30846,34065 +36985,34066 +34066,34066 +30847,34066 +34067,34067 +30848,34067 +34068,34068 +30849,34068 +34069,34069 +30850,34069 +34070,34070 +30851,34070 +36989,34070 +36990,34071 +34071,34071 +30852,34071 +34072,34072 +30853,34072 +36991,34072 +36992,34073 +34073,34073 +30854,34073 +34074,34074 +30855,34074 +34075,34075 +34076,34076 +34077,34077 +34078,34078 +34079,34079 +30860,34079 +34080,34080 +33422,34080 +34081,34081 +30862,34081 +34082,34082 +34083,34083 +30864,34083 +34084,34084 +30865,34084 +34085,34085 +34086,34086 +34087,34087 +34088,34088 +34089,34089 +34090,34090 +34091,34091 +34092,34092 +34093,34093 +145734,34094 +34094,34094 +34095,34095 +34132,34132 +34133,34133 +34134,34134 +34135,34135 +34136,34136 +34137,34137 +34138,34138 +247431,34138 +34139,34139 +34140,34140 +34141,34141 +34142,34142 +34143,34143 +34144,34144 +34145,34145 +34146,34146 +34147,34147 +141270,34147 +34148,34148 +34149,34149 +34150,34150 +34151,34151 +34152,34152 +34153,34153 +34154,34154 +34155,34155 +34156,34156 +34157,34157 +106945,34157 +34199,34199 +34200,34200 +34201,34201 +34202,34202 +34203,34203 +34204,34204 +34205,34205 +34206,34206 +34207,34207 +34208,34208 +34209,34209 +34210,34210 +34211,34211 +178924,34211 +34212,34212 +34213,34213 +34214,34214 +63653,34214 +34215,34215 +34216,34216 +34217,34217 +34218,34218 +34219,34219 +247792,34219 +110961,34220 +34220,34220 +34221,34221 +34222,34222 +34223,34223 +34224,34224 +34225,34225 +34226,34226 +12949,34226 +34227,34227 +34228,34228 +34229,34229 +34230,34230 +34231,34231 +34232,34232 +34233,34233 +34234,34234 +141334,34234 +34235,34235 +178862,34235 +34236,34236 +34237,34237 +34238,34238 +34239,34239 +34240,34240 +34241,34241 +34242,34242 +34243,34243 +34244,34244 +199835,34244 +34245,34245 +34246,34246 +34247,34247 +34248,34248 +34249,34249 +34250,34250 +34251,34251 +160455,34251 +34252,34252 +34253,34253 +34254,34254 +34255,34255 +34256,34256 +34257,34257 +75354,34257 +208151,34257 +34258,34258 +34259,34259 +34260,34260 +34261,34261 +34262,34262 +34263,34263 +34264,34264 +37186,34264 +37187,34265 +34265,34265 +34266,34266 +34267,34267 +40201,34267 +34268,34268 +34269,34269 +154032,34269 +34270,34270 +37193,34271 +34271,34271 +34272,34272 +34273,34273 +34274,34274 +37197,34275 +34275,34275 +37198,34276 +34276,34276 +34277,34277 +34278,34278 +34279,34279 +37201,34279 +34280,34280 +37202,34280 +201103,34280 +34281,34281 +34282,34282 +37204,34282 +34283,34283 +34284,34284 +34285,34285 +34324,34324 +34325,34325 +34326,34326 +34327,34327 +34328,34328 +34329,34329 +34330,34330 +34331,34331 +34332,34332 +34333,34333 +34334,34334 +34430,34430 +34431,34431 +247974,34431 +34432,34432 +34433,34433 +34434,34434 +34435,34435 +34436,34436 +34463,34463 +34464,34464 +34465,34465 +34466,34466 +34563,34563 +34564,34564 +34565,34565 +34566,34566 +34567,34567 +34568,34568 +34569,34569 +34570,34570 +34571,34571 +34572,34572 +34573,34573 +34574,34574 +34575,34575 +34576,34576 +34577,34577 +34578,34578 +34579,34579 +34580,34580 +34581,34581 +34582,34582 +34583,34583 +34584,34584 +34585,34585 +34586,34586 +34587,34587 +34588,34588 +34589,34589 +34590,34590 +34591,34591 +34592,34592 +34593,34593 +34594,34594 +34595,34595 +38224,34595 +34596,34596 +34597,34597 +34598,34598 +38228,34599 +34599,34599 +34600,34600 +38229,34600 +86723,34600 +86724,34601 +34601,34601 +34602,34602 +34603,34603 +34604,34604 +80257,34604 +34605,34605 +34606,34606 +34607,34607 +34608,34608 +34609,34609 +34610,34610 +34611,34611 +34612,34612 +34613,34613 +34614,34614 +34615,34615 +34616,34616 +34617,34617 +34618,34618 +34619,34619 +34620,34620 +34621,34621 +34622,34622 +37715,34623 +34623,34623 +38127,34623 +37716,34624 +34624,34624 +38128,34624 +37717,34625 +34625,34625 +38129,34625 +245348,34625 +37718,34626 +34626,34626 +38130,34626 +38131,34627 +37719,34627 +34627,34627 +37720,34628 +34628,34628 +38132,34628 +38133,34629 +37721,34629 +34629,34629 +37722,34630 +34630,34630 +38134,34630 +38135,34631 +37723,34631 +34631,34631 +38136,34632 +37724,34632 +34632,34632 +141793,34632 +37725,34633 +34633,34633 +38137,34633 +37726,34634 +34634,34634 +38138,34634 +38139,34635 +37727,34635 +34635,34635 +38140,34636 +37728,34636 +34636,34636 +37729,34637 +34637,34637 +38141,34637 +38142,34638 +37730,34638 +34638,34638 +37731,34639 +34639,34639 +38189,34685 +37314,34685 +34685,34685 +38190,34686 +37315,34686 +34686,34686 +38191,34687 +37316,34687 +34687,34687 +38192,34688 +37317,34688 +34688,34688 +38193,34689 +37318,34689 +34689,34689 +38194,34690 +34690,34690 +37319,34690 +38195,34691 +34691,34691 +37320,34691 +38196,34692 +34692,34692 +37321,34692 +38197,34693 +34693,34693 +37322,34693 +38198,34694 +37323,34694 +34694,34694 +7172,34694 +101883,34694 +38199,34695 +34695,34695 +241834,34695 +37324,34695 +38200,34696 +37325,34696 +34696,34696 +38201,34697 +34697,34697 +37326,34697 +38232,34998 +34998,34998 +38233,34999 +34999,34999 +38234,35000 +35000,35000 +31134,35000 +38235,35001 +35001,35001 +31135,35001 +38236,35002 +35002,35002 +14439,35002 +38237,35003 +35003,35003 +38238,35004 +35004,35004 +38239,35005 +35005,35005 +38240,35006 +35006,35006 +38241,35007 +35007,35007 +38242,35008 +35008,35008 +38243,35009 +35009,35009 +38244,35010 +35010,35010 +38245,35011 +35011,35011 +193687,35011 +35012,35012 +35013,35013 +35014,35014 +35015,35015 +35016,35016 +35017,35017 +35018,35018 +35019,35019 +35020,35020 +31154,35020 +35021,35021 +31155,35021 +35022,35022 +35023,35023 +32303,35023 +35024,35024 +35025,35025 +35026,35026 +179190,35026 +35027,35027 +35028,35028 +35029,35029 +35030,35030 +241113,35030 +35031,35031 +35032,35032 +35033,35033 +35034,35034 +35035,35035 +35036,35036 +35037,35037 +35038,35038 +35039,35039 +63872,35039 +35040,35040 +35041,35041 +35042,35042 +35080,35080 +35081,35081 +35082,35082 +35083,35083 +35084,35084 +35085,35085 +35086,35086 +35087,35087 +35088,35088 +140480,35088 +35089,35089 +232137,35089 +35090,35090 +35091,35091 +35092,35092 +35093,35093 +35094,35094 +35095,35095 +190968,35095 +35096,35096 +35097,35097 +35098,35098 +35099,35099 +35100,35100 +35101,35101 +35102,35102 +35103,35103 +35104,35104 +35105,35105 +35106,35106 +35107,35107 +35108,35108 +35109,35109 +35110,35110 +35111,35111 +35112,35112 +35113,35113 +35114,35114 +35115,35115 +35116,35116 +35117,35117 +508,35117 +35118,35118 +35119,35119 +35120,35120 +35121,35121 +206054,35121 +35122,35122 +35123,35123 +35124,35124 +35125,35125 +35126,35126 +35127,35127 +35128,35128 +35129,35129 +35130,35130 +35131,35131 +35132,35132 +35133,35133 +35134,35134 +35135,35135 +35136,35136 +35137,35137 +35138,35138 +35139,35139 +35140,35140 +35141,35141 +35142,35142 +35143,35143 +35184,35184 +35185,35185 +35186,35186 +35187,35187 +35188,35188 +35189,35189 +35190,35190 +35191,35191 +35192,35192 +35193,35193 +35194,35194 +35195,35195 +35196,35196 +35197,35197 +35198,35198 +35199,35199 +35200,35200 +35201,35201 +35202,35202 +35203,35203 +35204,35204 +35205,35205 +35206,35206 +35207,35207 +35208,35208 +35209,35209 +35210,35210 +35211,35211 +35212,35212 +35213,35213 +35214,35214 +35215,35215 +35216,35216 +35217,35217 +35218,35218 +35219,35219 +35220,35220 +35221,35221 +35222,35222 +35223,35223 +35224,35224 +35225,35225 +35226,35226 +35227,35227 +35228,35228 +35229,35229 +35230,35230 +35231,35231 +35287,35287 +35288,35288 +35478,35478 +35479,35479 +216349,35479 +35480,35480 +35481,35481 +35482,35482 +35483,35483 +35484,35484 +35485,35485 +120706,35485 +35486,35486 +57884,35486 +35487,35487 +35488,35488 +35489,35489 +35490,35490 +35491,35491 +35492,35492 +35493,35493 +35494,35494 +35495,35495 +35496,35496 +117737,35496 +35497,35497 +35498,35498 +35499,35499 +39878,35499 +35500,35500 +35544,35544 +35545,35545 +147044,35545 +140302,35545 +35546,35546 +35547,35547 +35548,35548 +35549,35549 +35550,35550 +35551,35551 +35552,35552 +35553,35553 +35554,35554 +35555,35555 +35556,35556 +35557,35557 +35558,35558 +189791,35558 +35559,35559 +35560,35560 +35561,35561 +35562,35562 +35563,35563 +35564,35564 +35565,35565 +35566,35566 +35567,35567 +35568,35568 +35569,35569 +35570,35570 +35571,35571 +35572,35572 +35573,35573 +35574,35574 +35575,35575 +35576,35576 +35577,35577 +35578,35578 +35579,35579 +35580,35580 +35581,35581 +75098,35581 +35582,35582 +35583,35583 +35584,35584 +35585,35585 +35586,35586 +35587,35587 +103462,35587 +69837,35588 +35588,35588 +35589,35589 +35590,35590 +35591,35591 +35592,35592 +35593,35593 +35594,35594 +35595,35595 +58693,35595 +35596,35596 +35597,35597 +35598,35598 +35599,35599 +35600,35600 +35601,35601 +35602,35602 +242913,35602 +35603,35603 +35604,35604 +35605,35605 +35606,35606 +35607,35607 +35608,35608 +35609,35609 +35610,35610 +35611,35611 +35612,35612 +35613,35613 +35614,35614 +35615,35615 +88254,35615 +35616,35616 +35617,35617 +35618,35618 +35619,35619 +35620,35620 +35621,35621 +35622,35622 +35623,35623 +35624,35624 +35625,35625 +35626,35626 +35627,35627 +35628,35628 +35629,35629 +122924,35629 +35630,35630 +35631,35631 +164980,35631 +35632,35632 +83331,35632 +35633,35633 +82875,35633 +153028,35633 +35634,35634 +35635,35635 +35636,35636 +148290,35636 +35637,35637 +35638,35638 +35639,35639 +35640,35640 +35641,35641 +35642,35642 +14116,35642 +35643,35643 +35644,35644 +35645,35645 +135232,35645 +127645,35645 +35646,35646 +35647,35647 +35648,35648 +35649,35649 +35650,35650 +35651,35651 +35652,35652 +35653,35653 +35654,35654 +15753,35654 +35655,35655 +35656,35656 +35657,35657 +35658,35658 +35659,35659 +35660,35660 +4188,35660 +35661,35661 +35662,35662 +35663,35663 +85446,35663 +35664,35664 +35665,35665 +35666,35666 +35667,35667 +35668,35668 +35669,35669 +35705,35705 +35706,35706 +141345,35706 +35707,35707 +35708,35708 +35709,35709 +35710,35710 +35711,35711 +35712,35712 +35713,35713 +35714,35714 +35715,35715 +35716,35716 +159362,35716 +35717,35717 +35718,35718 +35719,35719 +35720,35720 +35721,35721 +35722,35722 +35723,35723 +35724,35724 +202875,35724 +35725,35725 +109615,35725 +35726,35726 +35727,35727 +35728,35728 +222267,35728 +35729,35729 +35730,35730 +35731,35731 +89187,35731 +218217,35732 +35732,35732 +35733,35733 +35734,35734 +35735,35735 +35736,35736 +35737,35737 +35738,35738 +35739,35739 +35740,35740 +35741,35741 +35742,35742 +35743,35743 +35744,35744 +35745,35745 +35746,35746 +228035,35746 +35747,35747 +35748,35748 +35749,35749 +35750,35750 +35751,35751 +35752,35752 +35753,35753 +35754,35754 +35755,35755 +35756,35756 +35757,35757 +35758,35758 +35759,35759 +35760,35760 +35761,35761 +35762,35762 +35763,35763 +35764,35764 +35765,35765 +35766,35766 +35767,35767 +35768,35768 +35769,35769 +35770,35770 +5372,35770 +35771,35771 +35772,35772 +35773,35773 +35774,35774 +35775,35775 +35776,35776 +211134,35776 +35777,35777 +50038,35777 +41845,35777 +35778,35778 +35779,35779 +89598,35779 +35780,35780 +35781,35781 +35782,35782 +40775,35782 +35783,35783 +35784,35784 +35785,35785 +35786,35786 +35787,35787 +35788,35788 +35789,35789 +35790,35790 +35791,35791 +35792,35792 +35793,35793 +213852,35793 +35794,35794 +35795,35795 +35796,35796 +35797,35797 +35798,35798 +35799,35799 +35800,35800 +35801,35801 +35802,35802 +35803,35803 +26307,35803 +35804,35804 +35805,35805 +35806,35806 +35807,35807 +35808,35808 +35809,35809 +35810,35810 +35811,35811 +35812,35812 +35813,35813 +35814,35814 +192210,35814 +35815,35815 +35816,35816 +138685,35816 +35817,35817 +35818,35818 +35819,35819 +35820,35820 +35821,35821 +35822,35822 +35823,35823 +35824,35824 +35825,35825 +35826,35826 +227627,35826 +35827,35827 +35828,35828 +35829,35829 +35830,35830 +35831,35831 +35913,35913 +146532,35913 +27915,35913 +35914,35914 +35915,35915 +35916,35916 +35917,35917 +35918,35918 +35919,35919 +35920,35920 +35921,35921 +35969,35969 +89622,35969 +35970,35970 +35971,35971 +35972,35972 +35973,35973 +35974,35974 +35975,35975 +35976,35976 +35977,35977 +35978,35978 +35979,35979 +35980,35980 +35981,35981 +35982,35982 +35983,35983 +55836,35983 +47732,35983 +35984,35984 +35985,35985 +36055,36055 +36056,36056 +36057,36057 +36058,36058 +36059,36059 +36060,36060 +36061,36061 +36062,36062 +36063,36063 +36064,36064 +36065,36065 +18831,36065 +36066,36066 +135285,36066 +127698,36066 +36067,36067 +36068,36068 +36069,36069 +36070,36070 +36071,36071 +224737,36071 +36072,36072 +36073,36073 +36074,36074 +207434,36074 +36075,36075 +36076,36076 +29892,36076 +36077,36077 +36078,36078 +36079,36079 +36080,36080 +36081,36081 +245492,36081 +36082,36082 +36083,36083 +36084,36084 +36085,36085 +29901,36085 +36086,36086 +36087,36087 +33196,36087 +36088,36088 +110925,36088 +36089,36089 +36090,36090 +36091,36091 +33200,36091 +36092,36092 +33201,36092 +36093,36093 +994,36093 +36094,36094 +33203,36094 +36095,36095 +36096,36096 +29912,36096 +36097,36097 +36098,36098 +33207,36098 +36099,36099 +33208,36099 +36100,36100 +36101,36101 +36102,36102 +36103,36103 +36104,36104 +33213,36104 +36105,36105 +36106,36106 +36107,36107 +36108,36108 +36109,36109 +33218,36109 +36110,36110 +36111,36111 +36112,36112 +36113,36113 +36114,36114 +36115,36115 +36116,36116 +132831,36116 +125244,36116 +36117,36117 +36118,36118 +145847,36118 +36119,36119 +36120,36120 +36121,36121 +36122,36122 +36123,36123 +36124,36124 +86465,36124 +36125,36125 +36126,36126 +36127,36127 +36128,36128 +36129,36129 +36130,36130 +36131,36131 +36132,36132 +36133,36133 +36134,36134 +36135,36135 +36136,36136 +36137,36137 +36138,36138 +222174,36138 +36139,36139 +36140,36140 +36141,36141 +36142,36142 +36143,36143 +36144,36144 +36145,36145 +36146,36146 +33252,36146 +141410,36146 +36147,36147 +36148,36148 +36149,36149 +36150,36150 +36151,36151 +29961,36151 +36152,36152 +36153,36153 +29963,36153 +36154,36154 +36155,36155 +29965,36155 +36156,36156 +36157,36157 +29967,36157 +33263,36157 +36158,36158 +29968,36158 +36159,36159 +36160,36160 +36161,36161 +29971,36161 +669,36161 +36162,36162 +29972,36162 +36163,36163 +36164,36164 +29974,36164 +36165,36165 +29975,36165 +36166,36166 +36167,36167 +29977,36167 +36168,36168 +29978,36168 +36169,36169 +29979,36169 +36170,36170 +29980,36170 +33276,36170 +36171,36171 +29981,36171 +204196,36171 +36172,36172 +29982,36172 +36173,36173 +29983,36173 +36174,36174 +29984,36174 +36175,36175 +29985,36175 +36176,36176 +29986,36176 +36177,36177 +29987,36177 +36178,36178 +29988,36178 +36179,36179 +29989,36179 +36180,36180 +29990,36180 +36181,36181 +36182,36182 +36183,36183 +36184,36184 +2363,36184 +36185,36185 +29995,36185 +36186,36186 +36187,36187 +10530,36187 +36188,36188 +36189,36189 +36190,36190 +36191,36191 +134306,36191 +126719,36191 +36232,36232 +36233,36233 +36234,36234 +36235,36235 +36236,36236 +161885,36236 +36237,36237 +36238,36238 +36239,36239 +36240,36240 +36241,36241 +36242,36242 +36243,36243 +36244,36244 +36245,36245 +36246,36246 +36247,36247 +36248,36248 +36249,36249 +36250,36250 +36251,36251 +36252,36252 +36253,36253 +33360,36253 +36254,36254 +36255,36255 +36256,36256 +33363,36256 +36257,36257 +36258,36258 +36259,36259 +33366,36259 +36260,36260 +36261,36261 +36262,36262 +36263,36263 +36264,36264 +36265,36265 +36266,36266 +21521,36266 +36267,36267 +33374,36267 +71875,36267 +36268,36268 +33375,36268 +36269,36269 +33376,36269 +36270,36270 +33377,36270 +36271,36271 +33378,36271 +36272,36272 +33379,36272 +36273,36273 +33380,36273 +36274,36274 +33381,36274 +36275,36275 +33382,36275 +36276,36276 +33383,36276 +36277,36277 +33384,36277 +36278,36278 +33385,36278 +12079,36278 +36279,36279 +36280,36280 +33387,36280 +36281,36281 +36282,36282 +33389,36282 +36283,36283 +36284,36284 +36285,36285 +36286,36286 +36287,36287 +22030,36287 +36288,36288 +36289,36289 +36290,36290 +36291,36291 +36292,36292 +36293,36293 +33400,36293 +30143,36293 +226314,36293 +36294,36294 +36295,36295 +36296,36296 +33403,36296 +30146,36296 +36297,36297 +36298,36298 +36299,36299 +66131,36299 +36300,36300 +36301,36301 +155071,36301 +36302,36302 +36303,36303 +36304,36304 +36305,36305 +64930,36305 +36306,36306 +36307,36307 +36308,36308 +36309,36309 +36310,36310 +36311,36311 +36312,36312 +36313,36313 +36314,36314 +36315,36315 +36316,36316 +36317,36317 +36318,36318 +36319,36319 +36320,36320 +36321,36321 +36322,36322 +36323,36323 +36324,36324 +33975,36324 +36325,36325 +36326,36326 +36327,36327 +36328,36328 +36329,36329 +36330,36330 +36331,36331 +36332,36332 +36333,36333 +36334,36334 +106352,36334 +36335,36335 +36336,36336 +104719,36336 +36337,36337 +36338,36338 +151394,36338 +36339,36339 +36340,36340 +36341,36341 +36342,36342 +89713,36342 +36343,36343 +36344,36344 +36345,36345 +36346,36346 +36347,36347 +36348,36348 +36349,36349 +36350,36350 +135484,36350 +127897,36350 +36351,36351 +52424,36352 +44320,36352 +36352,36352 +185403,36352 +36353,36353 +36354,36354 +36355,36355 +36356,36356 +36357,36357 +36358,36358 +36359,36359 +36360,36360 +96340,36360 +36361,36361 +36362,36362 +36363,36363 +36364,36364 +36365,36365 +30230,36365 +36366,36366 +36367,36367 +36368,36368 +36369,36369 +30234,36369 +36370,36370 +30235,36370 +36371,36371 +33487,36371 +36372,36372 +33488,36372 +30237,36372 +36373,36373 +33489,36373 +36374,36374 +16455,36374 +33490,36374 +36375,36375 +33491,36375 +30240,36375 +36376,36376 +33492,36376 +30241,36376 +36377,36377 +33493,36377 +30242,36377 +36378,36378 +33494,36378 +36379,36379 +33495,36379 +36380,36380 +33496,36380 +36381,36381 +33497,36381 +36382,36382 +33498,36382 +36383,36383 +33499,36383 +30248,36383 +36384,36384 +33500,36384 +36385,36385 +33501,36385 +30250,36385 +36386,36386 +33502,36386 +36387,36387 +33503,36387 +36388,36388 +33504,36388 +36389,36389 +33505,36389 +36390,36390 +33506,36390 +36391,36391 +33507,36391 +30256,36391 +36392,36392 +30257,36392 +36393,36393 +36394,36394 +36395,36395 +36396,36396 +36397,36397 +36398,36398 +30263,36398 +36399,36399 +36400,36400 +33516,36400 +30265,36400 +36401,36401 +36402,36402 +36403,36403 +36404,36404 +68051,36404 +36405,36405 +36406,36406 +36407,36407 +36408,36408 +36409,36409 +36410,36410 +36411,36411 +36412,36412 +36413,36413 +36414,36414 +36415,36415 +33531,36415 +36416,36416 +36417,36417 +36418,36418 +36419,36419 +36420,36420 +36421,36421 +36422,36422 +33538,36422 +36423,36423 +36424,36424 +36425,36425 +36426,36426 +36427,36427 +36428,36428 +36429,36429 +36430,36430 +36431,36431 +36432,36432 +198415,36432 +36433,36433 +36434,36434 +36435,36435 +36436,36436 +36437,36437 +36438,36438 +36439,36439 +36440,36440 +36441,36441 +36442,36442 +36443,36443 +36444,36444 +36445,36445 +36446,36446 +36447,36447 +36448,36448 +36449,36449 +36450,36450 +36451,36451 +36452,36452 +36453,36453 +36454,36454 +196810,36455 +36455,36455 +36456,36456 +178397,36456 +172433,36456 +36457,36457 +36458,36458 +36459,36459 +36460,36460 +36461,36461 +36462,36462 +36463,36463 +36464,36464 +36465,36465 +36466,36466 +36467,36467 +36468,36468 +36469,36469 +75788,36469 +138661,36469 +36470,36470 +36471,36471 +36472,36472 +36473,36473 +36474,36474 +36475,36475 +36476,36476 +36477,36477 +36478,36478 +36479,36479 +153099,36479 +36480,36480 +36481,36481 +36482,36482 +36483,36483 +36484,36484 +36485,36485 +82183,36485 +36486,36486 +36487,36487 +36488,36488 +36489,36489 +36490,36490 +36491,36491 +36492,36492 +36493,36493 +36494,36494 +36495,36495 +36496,36496 +36497,36497 +36498,36498 +147330,36498 +36499,36499 +36500,36500 +36501,36501 +36502,36502 +36503,36503 +36504,36504 +36505,36505 +36506,36506 +36507,36507 +36508,36508 +36509,36509 +36510,36510 +36511,36511 +36512,36512 +36513,36513 +36514,36514 +36515,36515 +36516,36516 +36517,36517 +36518,36518 +36519,36519 +36520,36520 +70378,36520 +36521,36521 +36522,36522 +36523,36523 +36524,36524 +36525,36525 +36526,36526 +36527,36527 +36528,36528 +36529,36529 +36530,36530 +30392,36530 +36531,36531 +36532,36532 +36533,36533 +36534,36534 +36535,36535 +36536,36536 +36537,36537 +96468,36537 +36538,36538 +36539,36539 +36540,36540 +36541,36541 +36542,36542 +36543,36543 +243823,36543 +36544,36544 +36545,36545 +36546,36546 +36547,36547 +36548,36548 +36549,36549 +36550,36550 +36551,36551 +36552,36552 +36553,36553 +80923,36553 +36554,36554 +36555,36555 +36556,36556 +36557,36557 +36558,36558 +36559,36559 +36560,36560 +36561,36561 +36562,36562 +36563,36563 +36564,36564 +36565,36565 +36566,36566 +36567,36567 +36568,36568 +36569,36569 +36570,36570 +36571,36571 +36572,36572 +36573,36573 +36574,36574 +36575,36575 +36576,36576 +36577,36577 +36578,36578 +36579,36579 +36580,36580 +36581,36581 +36582,36582 +36583,36583 +36584,36584 +36585,36585 +36586,36586 +36587,36587 +36588,36588 +36589,36589 +36590,36590 +36591,36591 +29928,36591 +36592,36592 +36593,36593 +36594,36594 +36595,36595 +36596,36596 +36597,36597 +36598,36598 +36599,36599 +36600,36600 +30462,36600 +36601,36601 +36602,36602 +36603,36603 +36604,36604 +36605,36605 +36606,36606 +36607,36607 +36608,36608 +36609,36609 +36610,36610 +36611,36611 +30473,36611 +36612,36612 +30474,36612 +178493,36612 +172529,36612 +36613,36613 +30475,36613 +36614,36614 +30476,36614 +36615,36615 +30477,36615 +36616,36616 +30478,36616 +36617,36617 +30479,36617 +36618,36618 +30480,36618 +36619,36619 +30481,36619 +36620,36620 +30482,36620 +36621,36621 +30483,36621 +28232,36621 +36622,36622 +30484,36622 +140708,36622 +36623,36623 +30485,36623 +36624,36624 +30486,36624 +36625,36625 +30487,36625 +36626,36626 +148622,36626 +30488,36626 +36627,36627 +36628,36628 +30490,36628 +36629,36629 +116335,36629 +36630,36630 +36631,36631 +36632,36632 +36633,36633 +36634,36634 +36635,36635 +36636,36636 +145756,36636 +36637,36637 +36638,36638 +160834,36639 +36639,36639 +36640,36640 +36680,36680 +36681,36681 +36682,36682 +30544,36682 +36683,36683 +30545,36683 +36684,36684 +11867,36684 +36685,36685 +36686,36686 +176683,36686 +170719,36686 +30548,36686 +36687,36687 +30549,36687 +36688,36688 +36689,36689 +30551,36689 +36690,36690 +30552,36690 +36691,36691 +30553,36691 +36692,36692 +36693,36693 +30555,36693 +36694,36694 +30556,36694 +36695,36695 +36696,36696 +36697,36697 +36698,36698 +36699,36699 +36700,36700 +36701,36701 +36702,36702 +36703,36703 +36704,36704 +36705,36705 +36706,36706 +36707,36707 +36708,36708 +6745,36708 +36709,36709 +36710,36710 +36711,36711 +36712,36712 +30495,36712 +36713,36713 +202736,36713 +36714,36714 +36715,36715 +36716,36716 +36717,36717 +36718,36718 +36719,36719 +36720,36720 +36721,36721 +36722,36722 +30584,36722 +36723,36723 +36724,36724 +36725,36725 +36726,36726 +36727,36727 +36728,36728 +141367,36728 +30590,36728 +36729,36729 +36730,36730 +36731,36731 +175,36731 +36732,36732 +36733,36733 +36734,36734 +36735,36735 +36736,36736 +70508,36736 +36737,36737 +36738,36738 +36739,36739 +36740,36740 +30602,36740 +36741,36741 +36742,36742 +36743,36743 +571,36743 +36744,36744 +36745,36745 +36746,36746 +36747,36747 +36748,36748 +36749,36749 +36750,36750 +136258,36750 +128671,36750 +36751,36751 +135765,36751 +128178,36751 +36752,36752 +36753,36753 +36754,36754 +36755,36755 +36756,36756 +36757,36757 +36758,36758 +36759,36759 +36760,36760 +36761,36761 +36762,36762 +36763,36763 +36764,36764 +36765,36765 +36766,36766 +36767,36767 +36768,36768 +36769,36769 +36770,36770 +146562,36770 +36771,36771 +36772,36772 +36773,36773 +36774,36774 +36775,36775 +36776,36776 +36821,36821 +36822,36822 +36823,36823 +36824,36824 +36825,36825 +36826,36826 +36827,36827 +30689,36827 +64974,36827 +36828,36828 +36829,36829 +30691,36829 +36830,36830 +36831,36831 +36832,36832 +36833,36833 +36834,36834 +36835,36835 +36836,36836 +36837,36837 +36838,36838 +36839,36839 +36840,36840 +29924,36840 +36841,36841 +36842,36842 +36843,36843 +36844,36844 +36845,36845 +36846,36846 +36847,36847 +36848,36848 +4820,36848 +87786,36848 +36849,36849 +36850,36850 +122258,36850 +36851,36851 +36852,36852 +36853,36853 +36854,36854 +36855,36855 +36856,36856 +36857,36857 +85417,36857 +36858,36858 +36859,36859 +36860,36860 +33968,36860 +36861,36861 +36862,36862 +36863,36863 +36864,36864 +36865,36865 +36866,36866 +36867,36867 +36868,36868 +36869,36869 +33977,36869 +36870,36870 +36871,36871 +36872,36872 +36873,36873 +36874,36874 +36875,36875 +108442,36875 +36876,36876 +36877,36877 +36878,36878 +36879,36879 +36880,36880 +23830,36880 +36881,36881 +36882,36882 +36883,36883 +36884,36884 +36885,36885 +36886,36886 +36887,36887 +197827,36887 +36888,36888 +36889,36889 +36890,36890 +36891,36891 +36892,36892 +36893,36893 +36894,36894 +36895,36895 +34003,36895 +30757,36895 +92651,36895 +36896,36896 +36897,36897 +36898,36898 +36899,36899 +36900,36900 +36901,36901 +36902,36902 +36903,36903 +36904,36904 +36905,36905 +36906,36906 +36907,36907 +36908,36908 +36909,36909 +226666,36909 +36910,36910 +36911,36911 +62434,36911 +36912,36912 +36913,36913 +142203,36913 +36914,36914 +36915,36915 +36916,36916 +12046,36916 +36917,36917 +36918,36918 +36919,36919 +36920,36920 +36921,36921 +36922,36922 +36923,36923 +36924,36924 +36925,36925 +36926,36926 +36927,36927 +190049,36927 +36928,36928 +187777,36928 +36929,36929 +36930,36930 +217936,36930 +36931,36931 +36932,36932 +111895,36932 +36933,36933 +36934,36934 +36935,36935 +36936,36936 +36937,36937 +36938,36938 +36939,36939 +36940,36940 +36941,36941 +36942,36942 +36943,36943 +36944,36944 +36945,36945 +36946,36946 +111698,36946 +36947,36947 +36948,36948 +36949,36949 +36950,36950 +36951,36951 +36952,36952 +60065,36952 +36953,36953 +36954,36954 +36955,36955 +36956,36956 +36957,36957 +36958,36958 +36959,36959 +36960,36960 +36961,36961 +36962,36962 +36963,36963 +36964,36964 +30826,36964 +36965,36965 +36966,36966 +36967,36967 +36968,36968 +36969,36969 +65766,36969 +36970,36970 +30832,36970 +36971,36971 +30833,36971 +36972,36972 +36973,36973 +30835,36973 +36974,36974 +36975,36975 +36976,36976 +36977,36977 +30839,36977 +36978,36978 +34059,36978 +30840,36978 +36979,36979 +34060,36979 +30841,36979 +36980,36980 +30842,36980 +34061,36980 +36981,36981 +30843,36981 +34062,36981 +36982,36982 +34063,36982 +30844,36982 +36983,36983 +34064,36983 +30845,36983 +36984,36984 +30846,36984 +34065,36984 +36985,36985 +34066,36985 +30847,36985 +36986,36986 +30848,36986 +36987,36987 +30849,36987 +36988,36988 +30850,36988 +36989,36989 +34070,36989 +30851,36989 +36990,36990 +34071,36990 +30852,36990 +36991,36991 +34072,36991 +30853,36991 +36992,36992 +34073,36992 +30854,36992 +36993,36993 +36994,36994 +30856,36994 +36995,36995 +36996,36996 +36997,36997 +36998,36998 +36999,36999 +30861,36999 +37000,37000 +37001,37001 +30863,37001 +37002,37002 +37003,37003 +37004,37004 +37005,37005 +37006,37006 +37007,37007 +37008,37008 +37009,37009 +37010,37010 +37011,37011 +37012,37012 +37013,37013 +37014,37014 +3588,37014 +37015,37015 +37016,37016 +37017,37017 +37018,37018 +6383,37018 +37019,37019 +164527,37019 +37020,37020 +37021,37021 +12658,37021 +37022,37022 +37023,37023 +37024,37024 +37025,37025 +37026,37026 +37027,37027 +37028,37028 +183364,37028 +37029,37029 +30891,37029 +37030,37030 +37031,37031 +30893,37031 +37032,37032 +165225,37032 +37033,37033 +37034,37034 +30896,37034 +37035,37035 +37036,37036 +37037,37037 +37038,37038 +37039,37039 +37040,37040 +37041,37041 +37042,37042 +83264,37042 +37043,37043 +37044,37044 +37045,37045 +37046,37046 +37047,37047 +37048,37048 +37049,37049 +37050,37050 +9092,37050 +37051,37051 +27593,37051 +37052,37052 +178078,37052 +172114,37052 +37053,37053 +111979,37053 +37054,37054 +37055,37055 +37056,37056 +37057,37057 +37058,37058 +37059,37059 +37060,37060 +37061,37061 +37062,37062 +37063,37063 +37064,37064 +37065,37065 +37066,37066 +37067,37067 +37068,37068 +37105,37105 +37106,37106 +37107,37107 +37108,37108 +37109,37109 +37110,37110 +37111,37111 +37112,37112 +37113,37113 +37114,37114 +37115,37115 +37116,37116 +37117,37117 +37118,37118 +37119,37119 +37120,37120 +143220,37120 +37121,37121 +37122,37122 +37123,37123 +37124,37124 +30971,37124 +37125,37125 +37126,37126 +37127,37127 +37128,37128 +37129,37129 +37130,37130 +37131,37131 +37132,37132 +37133,37133 +37134,37134 +37135,37135 +37136,37136 +37137,37137 +37138,37138 +37139,37139 +37140,37140 +37141,37141 +37142,37142 +37184,37184 +205579,37184 +37185,37185 +37186,37186 +34264,37186 +37187,37187 +34265,37187 +37188,37188 +37189,37189 +37190,37190 +37191,37191 +37192,37192 +37193,37193 +34271,37193 +37194,37194 +37195,37195 +37196,37196 +37197,37197 +34275,37197 +37198,37198 +34276,37198 +37199,37199 +31046,37199 +37200,37200 +37201,37201 +34279,37201 +37202,37202 +34280,37202 +37203,37203 +37204,37204 +34282,37204 +37205,37205 +37206,37206 +153152,37206 +37207,37207 +37287,37287 +37288,37288 +37289,37289 +37290,37290 +37291,37291 +183961,37291 +37292,37292 +37293,37293 +37294,37294 +164754,37294 +37295,37295 +37296,37296 +165771,37296 +37297,37297 +37298,37298 +37299,37299 +37300,37300 +37301,37301 +37302,37302 +37303,37303 +37304,37304 +37305,37305 +37306,37306 +37307,37307 +37308,37308 +37309,37309 +37310,37310 +37311,37311 +37312,37312 +37313,37313 +38189,37314 +37314,37314 +34685,37314 +38190,37315 +37315,37315 +34686,37315 +38191,37316 +37316,37316 +34687,37316 +38192,37317 +37317,37317 +34688,37317 +38193,37318 +37318,37318 +34689,37318 +37319,37319 +38194,37319 +34690,37319 +37320,37320 +38195,37320 +34691,37320 +37321,37321 +38196,37321 +34692,37321 +37322,37322 +38197,37322 +34693,37322 +38198,37323 +37323,37323 +34694,37323 +7172,37323 +101883,37323 +37324,37324 +38199,37324 +34695,37324 +38200,37325 +37325,37325 +34696,37325 +37326,37326 +244116,37326 +38201,37326 +34697,37326 +163143,37326 +38202,37327 +37327,37327 +38203,37328 +37328,37328 +37329,37329 +38204,37329 +37330,37330 +38205,37330 +38206,37331 +37331,37331 +38207,37332 +37332,37332 +37484,37484 +37485,37485 +37486,37486 +37487,37487 +37488,37488 +37489,37489 +37490,37490 +37491,37491 +37492,37492 +37493,37493 +37494,37494 +37495,37495 +37496,37496 +5859,37496 +37497,37497 +37498,37498 +37499,37499 +37500,37500 +37501,37501 +37502,37502 +37503,37503 +37504,37504 +218529,37505 +37505,37505 +37506,37506 +37507,37507 +37508,37508 +37509,37509 +37510,37510 +37511,37511 +30792,37511 +37715,37715 +34623,37715 +38127,37715 +37716,37716 +34624,37716 +38128,37716 +37717,37717 +34625,37717 +38129,37717 +245348,37717 +37718,37718 +34626,37718 +38130,37718 +38131,37719 +37719,37719 +34627,37719 +37720,37720 +34628,37720 +38132,37720 +38133,37721 +37721,37721 +34629,37721 +37722,37722 +34630,37722 +38134,37722 +38135,37723 +37723,37723 +34631,37723 +38136,37724 +37724,37724 +34632,37724 +141793,37724 +37725,37725 +34633,37725 +38137,37725 +37726,37726 +34634,37726 +38138,37726 +38139,37727 +37727,37727 +34635,37727 +38140,37728 +37728,37728 +34636,37728 +37729,37729 +34637,37729 +38141,37729 +38142,37730 +37730,37730 +34638,37730 +37731,37731 +34639,37731 +37875,37875 +37876,37876 +37877,37877 +159907,37877 +39362,37878 +37878,37878 +37879,37879 +39364,37880 +37880,37880 +37881,37881 +7700,37881 +39366,37882 +37882,37882 +39367,37883 +37883,37883 +26876,37883 +37884,37884 +37885,37885 +39370,37886 +37886,37886 +39371,37887 +37887,37887 +37888,37888 +39373,37889 +37889,37889 +37890,37890 +37891,37891 +37892,37892 +39377,37893 +37893,37893 +39378,37894 +37894,37894 +39379,37895 +37895,37895 +39380,37896 +37896,37896 +39381,37897 +37897,37897 +72885,37898 +39382,37898 +37898,37898 +39383,37899 +37899,37899 +72886,37899 +39384,37900 +37900,37900 +39385,37901 +37901,37901 +39386,37902 +37902,37902 +39387,37903 +37903,37903 +39388,37904 +37904,37904 +39389,37905 +37905,37905 +39390,37906 +37906,37906 +39391,37907 +37907,37907 +39392,37908 +37908,37908 +39393,37909 +37909,37909 +39394,37910 +37910,37910 +39395,37911 +37911,37911 +39396,37912 +37912,37912 +39397,37913 +37913,37913 +39398,37914 +37914,37914 +39399,37915 +37915,37915 +39400,37916 +37916,37916 +39401,37917 +37917,37917 +39402,37918 +37918,37918 +39403,37919 +37919,37919 +39404,37920 +37920,37920 +39405,37921 +37921,37921 +59197,37921 +58823,37921 +38127,38127 +37715,38127 +34623,38127 +38128,38128 +37716,38128 +34624,38128 +38129,38129 +37717,38129 +34625,38129 +38130,38130 +37718,38130 +34626,38130 +38131,38131 +37719,38131 +34627,38131 +38132,38132 +37720,38132 +34628,38132 +38133,38133 +37721,38133 +34629,38133 +38134,38134 +37722,38134 +34630,38134 +38135,38135 +37723,38135 +34631,38135 +38136,38136 +37724,38136 +34632,38136 +141793,38136 +38137,38137 +107199,38137 +37725,38137 +34633,38137 +38138,38138 +37726,38138 +34634,38138 +38139,38139 +37727,38139 +34635,38139 +38140,38140 +37728,38140 +34636,38140 +38141,38141 +37729,38141 +34637,38141 +38142,38142 +37730,38142 +34638,38142 +38189,38189 +37314,38189 +34685,38189 +38190,38190 +37315,38190 +34686,38190 +38191,38191 +37316,38191 +34687,38191 +38192,38192 +37317,38192 +34688,38192 +38193,38193 +37318,38193 +34689,38193 +38194,38194 +34690,38194 +37319,38194 +38195,38195 +34691,38195 +37320,38195 +38196,38196 +34692,38196 +37321,38196 +38197,38197 +34693,38197 +37322,38197 +38198,38198 +37323,38198 +34694,38198 +7172,38198 +101883,38198 +38199,38199 +34695,38199 +241834,38199 +37324,38199 +38200,38200 +37325,38200 +34696,38200 +38201,38201 +34697,38201 +37326,38201 +38202,38202 +37327,38202 +38203,38203 +37328,38203 +38204,38204 +37329,38204 +38205,38205 +229588,38205 +37330,38205 +38206,38206 +37331,38206 +38207,38207 +37332,38207 +38208,38208 +121469,38208 +38209,38209 +38210,38210 +38211,38211 +38212,38212 +38213,38213 +38214,38214 +38215,38215 +38216,38216 +38217,38217 +38218,38218 +38219,38219 +38220,38220 +38221,38221 +38222,38222 +38223,38223 +38224,38224 +34595,38224 +38225,38225 +38226,38226 +86720,38226 +211896,38226 +38227,38227 +38228,38228 +34599,38228 +38229,38229 +34600,38229 +38230,38230 +31130,38230 +38231,38231 +38232,38232 +34998,38232 +38233,38233 +34999,38233 +38234,38234 +35000,38234 +31134,38234 +38235,38235 +35001,38235 +31135,38235 +38236,38236 +35002,38236 +14439,38236 +38237,38237 +35003,38237 +38238,38238 +35004,38238 +38239,38239 +35005,38239 +38240,38240 +35006,38240 +38241,38241 +35007,38241 +38242,38242 +35008,38242 +38243,38243 +35009,38243 +38244,38244 +35010,38244 +38245,38245 +35011,38245 +193687,38245 +38246,38246 +31146,38246 +60935,38246 +38247,38247 +31147,38247 +38248,38248 +31148,38248 +38249,38249 +38250,38250 +38251,38251 +38252,38252 +38253,38253 +38254,38254 +38255,38255 +38256,38256 +38257,38257 +38258,38258 +191163,38258 +38259,38259 +233977,38259 +38260,38260 +38261,38261 +38262,38262 +38263,38263 +38264,38264 +38265,38265 +38266,38266 +38267,38267 +38268,38268 +38269,38269 +38270,38270 +38271,38271 +38272,38272 +38273,38273 +38274,38274 +63499,38274 +38275,38275 +38276,38276 +38277,38277 +38278,38278 +38279,38279 +38280,38280 +38281,38281 +84703,38281 +38282,38282 +38283,38283 +38284,38284 +38355,38355 +38356,38356 +38357,38357 +38358,38358 +38359,38359 +38360,38360 +38361,38361 +38362,38362 +188482,38362 +38363,38363 +38364,38364 +38365,38365 +38366,38366 +38367,38367 +38368,38368 +38369,38369 +38370,38370 +38371,38371 +38372,38372 +158742,38373 +38373,38373 +38374,38374 +38375,38375 +38376,38376 +38377,38377 +38378,38378 +38379,38379 +38380,38380 +38381,38381 +38382,38382 +38383,38383 +38384,38384 +38385,38385 +38386,38386 +38387,38387 +38388,38388 +38389,38389 +188035,38389 +38390,38390 +38391,38391 +38392,38392 +38393,38393 +236435,38393 +38394,38394 +38395,38395 +38396,38396 +31242,38396 +38397,38397 +38398,38398 +38399,38399 +38400,38400 +38401,38401 +38402,38402 +38403,38403 +38404,38404 +38405,38405 +38406,38406 +38407,38407 +38408,38408 +38409,38409 +38410,38410 +38411,38411 +38412,38412 +38413,38413 +38414,38414 +38415,38415 +38416,38416 +38417,38417 +38418,38418 +71194,38418 +38419,38419 +38420,38420 +38421,38421 +38422,38422 +38423,38423 +38424,38424 +38425,38425 +38426,38426 +179554,38426 +38427,38427 +38428,38428 +38429,38429 +38430,38430 +38431,38431 +38432,38432 +38433,38433 +38434,38434 +38435,38435 +38436,38436 +38437,38437 +38438,38438 +38439,38439 +38440,38440 +38441,38441 +38442,38442 +38443,38443 +39201,38443 +38444,38444 +38445,38445 +38446,38446 +38447,38447 +15611,38447 +38448,38448 +38449,38449 +38450,38450 +146757,38450 +38451,38451 +38452,38452 +38453,38453 +180961,38453 +38454,38454 +38455,38455 +38456,38456 +38457,38457 +38458,38458 +38459,38459 +38460,38460 +38461,38461 +38462,38462 +38463,38463 +137012,38463 +129425,38463 +38464,38464 +38465,38465 +38466,38466 +38467,38467 +38468,38468 +38469,38469 +157831,38469 +38470,38470 +38471,38471 +38514,38514 +38515,38515 +138723,38516 +38516,38516 +38517,38517 +38518,38518 +38519,38519 +160678,38519 +38520,38520 +38521,38521 +38522,38522 +38523,38523 +38658,38658 +38659,38659 +38660,38660 +38661,38661 +38662,38662 +38705,38705 +38706,38706 +38707,38707 +38708,38708 +38709,38709 +38710,38710 +38711,38711 +89946,38711 +38712,38712 +38713,38713 +38714,38714 +38715,38715 +38716,38716 +38717,38717 +38718,38718 +38719,38719 +38720,38720 +38721,38721 +38722,38722 +38723,38723 +38724,38724 +38725,38725 +189591,38725 +38726,38726 +38727,38727 +38728,38728 +38729,38729 +38730,38730 +38731,38731 +38732,38732 +38733,38733 +38734,38734 +38735,38735 +38736,38736 +38737,38737 +38738,38738 +38739,38739 +38740,38740 +38741,38741 +38742,38742 +38743,38743 +38744,38744 +38745,38745 +38746,38746 +38747,38747 +38748,38748 +38749,38749 +38750,38750 +38751,38751 +136560,38751 +128973,38751 +38752,38752 +216559,38752 +38753,38753 +38754,38754 +38755,38755 +38756,38756 +38757,38757 +38758,38758 +38759,38759 +38760,38760 +38761,38761 +38762,38762 +38763,38763 +38764,38764 +38765,38765 +38766,38766 +38767,38767 +38768,38768 +38769,38769 +38770,38770 +38771,38771 +38772,38772 +38773,38773 +38988,38988 +38989,38989 +187195,38989 +38990,38990 +38991,38991 +38992,38992 +38993,38993 +38994,38994 +38995,38995 +25100,38995 +38996,38996 +38997,38997 +141878,38997 +38998,38998 +38999,38999 +158737,38999 +39000,39000 +39001,39001 +39002,39002 +39003,39003 +39004,39004 +39005,39005 +39006,39006 +39007,39007 +39008,39008 +39009,39009 +39010,39010 +39011,39011 +39012,39012 +39013,39013 +39014,39014 +39015,39015 +39016,39016 +39017,39017 +39018,39018 +39019,39019 +39020,39020 +39021,39021 +39022,39022 +2134,39022 +39023,39023 +39024,39024 +39025,39025 +39026,39026 +39027,39027 +39028,39028 +39029,39029 +234165,39030 +39030,39030 +39031,39031 +39032,39032 +39182,39182 +39183,39183 +39184,39184 +39185,39185 +39186,39186 +39187,39187 +39188,39188 +39189,39189 +39190,39190 +39191,39191 +39192,39192 +39193,39193 +207605,39194 +39194,39194 +39195,39195 +39196,39196 +39197,39197 +39198,39198 +39199,39199 +63429,39199 +39200,39200 +39201,39201 +38443,39201 +39202,39202 +39203,39203 +39204,39204 +39205,39205 +39359,39359 +39360,39360 +39361,39361 +39362,39362 +37878,39362 +39363,39363 +39364,39364 +37880,39364 +39365,39365 +39366,39366 +37882,39366 +39367,39367 +37883,39367 +26876,39367 +39368,39368 +39369,39369 +39370,39370 +37886,39370 +39371,39371 +37887,39371 +39372,39372 +39373,39373 +37889,39373 +39374,39374 +39375,39375 +39376,39376 +39377,39377 +37893,39377 +39378,39378 +37894,39378 +39379,39379 +37895,39379 +39380,39380 +37896,39380 +39381,39381 +37897,39381 +72885,39382 +39382,39382 +37898,39382 +39383,39383 +37899,39383 +72886,39383 +39384,39384 +37900,39384 +39385,39385 +37901,39385 +39386,39386 +37902,39386 +39387,39387 +37903,39387 +39388,39388 +37904,39388 +39389,39389 +37905,39389 +39390,39390 +37906,39390 +39391,39391 +37907,39391 +39392,39392 +37908,39392 +39393,39393 +37909,39393 +39394,39394 +37910,39394 +39395,39395 +37911,39395 +39396,39396 +37912,39396 +39397,39397 +37913,39397 +39398,39398 +37914,39398 +39399,39399 +37915,39399 +39400,39400 +37916,39400 +39401,39401 +37917,39401 +39402,39402 +37918,39402 +39403,39403 +37919,39403 +39404,39404 +37920,39404 +39405,39405 +37921,39405 +59197,39405 +58823,39405 +39406,39406 +39407,39407 +54972,39407 +46868,39407 +39408,39408 +39409,39409 +39410,39410 +39411,39411 +39412,39412 +39413,39413 +39414,39414 +39415,39415 +39416,39416 +39417,39417 +39418,39418 +39419,39419 +39420,39420 +39421,39421 +79257,39421 +39709,39709 +39710,39710 +39711,39711 +39712,39712 +39713,39713 +39714,39714 +39715,39715 +39716,39716 +86662,39716 +39717,39717 +7200,39717 +39718,39718 +39719,39719 +39720,39720 +39721,39721 +39722,39722 +39723,39723 +39724,39724 +39725,39725 +39726,39726 +39727,39727 +39728,39728 +39729,39729 +39730,39730 +39731,39731 +2677,39731 +39732,39732 +39733,39733 +39734,39734 +4680,39734 +39735,39735 +142351,39735 +39736,39736 +39737,39737 +39738,39738 +39739,39739 +39740,39740 +39741,39741 +196574,39741 +39742,39742 +39743,39743 +39744,39744 +39745,39745 +39746,39746 +39747,39747 +39748,39748 +39749,39749 +39750,39750 +39751,39751 +39752,39752 +217967,39752 +39753,39753 +39754,39754 +39755,39755 +39756,39756 +39757,39757 +39758,39758 +39759,39759 +39760,39760 +39761,39761 +39762,39762 +39763,39763 +39764,39764 +39765,39765 +39766,39766 +39767,39767 +39768,39768 +39769,39769 +39770,39770 +39771,39771 +39772,39772 +39773,39773 +39774,39774 +39775,39775 +39776,39776 +39777,39777 +39778,39778 +39779,39779 +39780,39780 +39781,39781 +39782,39782 +39783,39783 +39784,39784 +39785,39785 +39786,39786 +39787,39787 +39788,39788 +39789,39789 +39790,39790 +196540,39790 +174639,39790 +168675,39790 +39791,39791 +39792,39792 +39793,39793 +39794,39794 +39795,39795 +39796,39796 +39797,39797 +39798,39798 +39799,39799 +39861,39861 +39862,39862 +39863,39863 +39864,39864 +249125,39864 +180344,39865 +39865,39865 +39866,39866 +39867,39867 +39868,39868 +39869,39869 +39870,39870 +39871,39871 +39872,39872 +39873,39873 +39874,39874 +39875,39875 +39876,39876 +39877,39877 +39878,39878 +35499,39878 +39879,39879 +39880,39880 +39881,39881 +39882,39882 +39883,39883 +39884,39884 +39885,39885 +39886,39886 +39887,39887 +39888,39888 +39889,39889 +244074,39889 +39890,39890 +39891,39891 +39892,39892 +39893,39893 +39894,39894 +39895,39895 +39896,39896 +39897,39897 +39898,39898 +39899,39899 +39900,39900 +39901,39901 +39902,39902 +39903,39903 +39904,39904 +112182,39904 +39905,39905 +39906,39906 +39907,39907 +39908,39908 +39909,39909 +178090,39909 +172126,39909 +68176,39909 +39910,39910 +39911,39911 +39912,39912 +39913,39913 +39914,39914 +39915,39915 +39916,39916 +39917,39917 +39918,39918 +39919,39919 +39920,39920 +77697,39920 +39921,39921 +75362,39921 +39922,39922 +39923,39923 +39924,39924 +39925,39925 +39926,39926 +39927,39927 +39928,39928 +39929,39929 +39930,39930 +39931,39931 +39932,39932 +39933,39933 +39934,39934 +39935,39935 +39936,39936 +179878,39936 +39937,39937 +228848,39937 +39938,39938 +39939,39939 +205094,39939 +39940,39940 +39941,39941 +39942,39942 +39943,39943 +39944,39944 +39945,39945 +39946,39946 +39947,39947 +39948,39948 +39949,39949 +39950,39950 +39951,39951 +39952,39952 +94991,39952 +39953,39953 +39954,39954 +39955,39955 +39956,39956 +39957,39957 +39958,39958 +39959,39959 +39960,39960 +39961,39961 +39962,39962 +39963,39963 +39964,39964 +39965,39965 +39966,39966 +39967,39967 +39968,39968 +39969,39969 +110913,39969 +39970,39970 +39971,39971 +39972,39972 +39973,39973 +39974,39974 +39975,39975 +39976,39976 +39977,39977 +195174,39977 +39978,39978 +39979,39979 +39980,39980 +39981,39981 +39982,39982 +39983,39983 +39984,39984 +39985,39985 +184817,39985 +39986,39986 +39987,39987 +39988,39988 +39989,39989 +39990,39990 +39991,39991 +39992,39992 +39993,39993 +39994,39994 +39995,39995 +39996,39996 +39997,39997 +39998,39998 +39999,39999 +40000,40000 +89530,40000 +40001,40001 +40002,40002 +40003,40003 +40004,40004 +40005,40005 +40006,40006 +40007,40007 +40008,40008 +40009,40009 +40010,40010 +40011,40011 +40012,40012 +40013,40013 +40014,40014 +40015,40015 +40016,40016 +40017,40017 +40018,40018 +40019,40019 +40020,40020 +40021,40021 +40022,40022 +40023,40023 +40024,40024 +40025,40025 +40026,40026 +40027,40027 +40028,40028 +40029,40029 +40030,40030 +40031,40031 +40032,40032 +40033,40033 +40034,40034 +40035,40035 +40036,40036 +40162,40162 +40163,40163 +40164,40164 +40165,40165 +40166,40166 +40167,40167 +40168,40168 +40169,40169 +40170,40170 +40171,40171 +40172,40172 +40173,40173 +222950,40173 +40174,40174 +40175,40175 +40176,40176 +40177,40177 +40178,40178 +40179,40179 +40180,40180 +40181,40181 +40182,40182 +40183,40183 +40184,40184 +40185,40185 +40186,40186 +240082,40186 +40187,40187 +40188,40188 +40189,40189 +40190,40190 +40191,40191 +40192,40192 +40193,40193 +40194,40194 +106190,40194 +40195,40195 +40196,40196 +40197,40197 +40198,40198 +40199,40199 +40200,40200 +40201,40201 +148432,40201 +247202,40201 +34267,40201 +40202,40202 +40203,40203 +40204,40204 +111249,40204 +40205,40205 +40206,40206 +40207,40207 +40208,40208 +40209,40209 +40210,40210 +40211,40211 +40212,40212 +40213,40213 +40214,40214 +40215,40215 +181025,40215 +40216,40216 +40217,40217 +40218,40218 +40219,40219 +40220,40220 +40221,40221 +40222,40222 +40223,40223 +230958,40223 +40224,40224 +40225,40225 +40226,40226 +40227,40227 +15845,40227 +40228,40228 +40229,40229 +40230,40230 +40231,40231 +40232,40232 +40233,40233 +7724,40233 +40234,40234 +40235,40235 +40236,40236 +40237,40237 +40238,40238 +205826,40238 +121456,40239 +40239,40239 +40240,40240 +40241,40241 +40242,40242 +40243,40243 +40244,40244 +40245,40245 +40246,40246 +203857,40246 +40247,40247 +40248,40248 +40249,40249 +40250,40250 +100589,40250 +40251,40251 +40252,40252 +40253,40253 +40254,40254 +40255,40255 +40256,40256 +40257,40257 +40258,40258 +40259,40259 +40260,40260 +40261,40261 +40262,40262 +40263,40263 +40264,40264 +40265,40265 +40266,40266 +105184,40266 +40267,40267 +40268,40268 +40269,40269 +40270,40270 +40271,40271 +67776,40271 +40317,40317 +40318,40318 +40319,40319 +40491,40491 +40492,40492 +40493,40493 +40494,40494 +40495,40495 +40496,40496 +40762,40762 +40763,40763 +40764,40764 +40765,40765 +40766,40766 +40767,40767 +40768,40768 +40769,40769 +40770,40770 +40771,40771 +40772,40772 +40773,40773 +231302,40773 +40774,40774 +40775,40775 +35782,40775 +40776,40776 +40777,40777 +40778,40778 +40779,40779 +40780,40780 +40781,40781 +40782,40782 +40865,40865 +40866,40866 +49127,40935 +40935,40935 +49128,40936 +40936,40936 +49129,40937 +40937,40937 +25114,40937 +49130,40938 +40938,40938 +49131,40939 +40939,40939 +49132,40940 +40940,40940 +49133,40941 +40941,40941 +179911,40941 +49134,40942 +40942,40942 +49135,40943 +40943,40943 +160163,40943 +49136,40944 +40944,40944 +49137,40945 +40945,40945 +49138,40946 +40946,40946 +49139,40947 +40947,40947 +49140,40948 +40948,40948 +176596,40948 +170632,40948 +49141,40949 +40949,40949 +49142,40950 +40950,40950 +49143,40951 +40951,40951 +49144,40952 +40952,40952 +49145,40953 +40953,40953 +49146,40954 +40954,40954 +49147,40955 +40955,40955 +49148,40956 +40956,40956 +49149,40957 +40957,40957 +49150,40958 +40958,40958 +49151,40959 +40959,40959 +49152,40960 +40960,40960 +49153,40961 +40961,40961 +49154,40962 +40962,40962 +49155,40963 +40963,40963 +49156,40964 +40964,40964 +49157,40965 +40965,40965 +49158,40966 +40966,40966 +49159,40967 +40967,40967 +49160,40968 +40968,40968 +49161,40969 +40969,40969 +49298,41105 +41105,41105 +49299,41106 +41106,41106 +49300,41107 +41107,41107 +49301,41108 +41108,41108 +243329,41109 +49302,41109 +41109,41109 +49303,41110 +41110,41110 +49304,41111 +41111,41111 +49305,41112 +41112,41112 +49306,41113 +41113,41113 +49307,41114 +41114,41114 +49308,41115 +41115,41115 +49309,41116 +41116,41116 +49310,41117 +41117,41117 +49311,41118 +41118,41118 +49312,41119 +41119,41119 +49313,41120 +41120,41120 +49314,41121 +41121,41121 +49315,41122 +41122,41122 +49316,41123 +41123,41123 +49317,41124 +41124,41124 +49318,41125 +41125,41125 +49319,41126 +41126,41126 +49320,41127 +41127,41127 +49321,41128 +41128,41128 +49322,41129 +41129,41129 +49323,41130 +41130,41130 +185192,41130 +49324,41131 +41131,41131 +49325,41132 +41132,41132 +49326,41133 +41133,41133 +49327,41134 +41134,41134 +52747,41134 +44643,41134 +49328,41135 +41135,41135 +49329,41136 +41136,41136 +49330,41137 +41137,41137 +49331,41138 +41138,41138 +49332,41139 +41139,41139 +49333,41140 +41140,41140 +49334,41141 +41141,41141 +49335,41142 +41142,41142 +49336,41143 +41143,41143 +49337,41144 +41144,41144 +49338,41145 +41145,41145 +49339,41146 +41146,41146 +49340,41147 +41147,41147 +49341,41148 +41148,41148 +49342,41149 +41149,41149 +82184,41149 +49343,41150 +41150,41150 +49344,41151 +41151,41151 +49345,41152 +41152,41152 +49346,41153 +41153,41153 +49347,41154 +41154,41154 +49348,41155 +41155,41155 +49349,41156 +41156,41156 +49350,41157 +41157,41157 +49351,41158 +41158,41158 +49352,41159 +41159,41159 +49353,41160 +41160,41160 +49354,41161 +41161,41161 +49355,41162 +41162,41162 +49356,41163 +41163,41163 +49357,41164 +41164,41164 +49358,41165 +41165,41165 +49359,41166 +41166,41166 +49360,41167 +41167,41167 +49361,41168 +41168,41168 +49362,41169 +41169,41169 +49363,41170 +41170,41170 +49364,41171 +41171,41171 +49365,41172 +41172,41172 +49366,41173 +41173,41173 +49367,41174 +41174,41174 +76004,41174 +49368,41175 +41175,41175 +49369,41176 +41176,41176 +49370,41177 +41177,41177 +49371,41178 +41178,41178 +49372,41179 +41179,41179 +49373,41180 +41180,41180 +49374,41181 +41181,41181 +49375,41182 +41182,41182 +49376,41183 +41183,41183 +49377,41184 +41184,41184 +49378,41185 +41185,41185 +49379,41186 +41186,41186 +49380,41187 +41187,41187 +49381,41188 +41188,41188 +49382,41189 +41189,41189 +49383,41190 +41190,41190 +49384,41191 +41191,41191 +49385,41192 +41192,41192 +49386,41193 +41193,41193 +49387,41194 +41194,41194 +49388,41195 +41195,41195 +49389,41196 +41196,41196 +49390,41197 +41197,41197 +211475,41197 +49391,41198 +41198,41198 +49392,41199 +41199,41199 +49393,41200 +41200,41200 +49394,41201 +41201,41201 +49461,41268 +41268,41268 +49462,41269 +41269,41269 +49463,41270 +41270,41270 +49464,41271 +41271,41271 +49465,41272 +41272,41272 +49466,41273 +41273,41273 +49467,41274 +41274,41274 +49468,41275 +41275,41275 +49469,41276 +41276,41276 +49470,41277 +41277,41277 +49471,41278 +41278,41278 +49472,41279 +41279,41279 +49473,41280 +41280,41280 +49474,41281 +41281,41281 +49475,41282 +41282,41282 +49476,41283 +41283,41283 +119290,41283 +49477,41284 +41284,41284 +49478,41285 +41285,41285 +49479,41286 +41286,41286 +49480,41287 +41287,41287 +49481,41288 +41288,41288 +246943,41288 +49482,41289 +41289,41289 +49483,41290 +41290,41290 +49484,41291 +41291,41291 +49485,41292 +41292,41292 +49486,41293 +41293,41293 +49487,41294 +41294,41294 +49488,41295 +41295,41295 +49489,41296 +41296,41296 +49490,41297 +41297,41297 +49491,41298 +41298,41298 +49492,41299 +41299,41299 +49493,41300 +41300,41300 +49494,41301 +41301,41301 +49495,41302 +41302,41302 +59117,41302 +58743,41302 +49496,41303 +41303,41303 +156636,41303 +49497,41304 +41304,41304 +49498,41305 +41305,41305 +49499,41306 +41306,41306 +49500,41307 +41307,41307 +49501,41308 +41308,41308 +49502,41309 +41309,41309 +49503,41310 +41310,41310 +49504,41311 +41311,41311 +49505,41312 +41312,41312 +49506,41313 +41313,41313 +49507,41314 +41314,41314 +49508,41315 +41315,41315 +49509,41316 +41316,41316 +54586,41316 +46482,41316 +49510,41317 +41317,41317 +49511,41318 +41318,41318 +49512,41319 +41319,41319 +2954,41319 +49513,41320 +41320,41320 +49514,41321 +41321,41321 +49515,41322 +41322,41322 +49516,41323 +41323,41323 +49517,41324 +41324,41324 +49518,41325 +41325,41325 +49519,41326 +41326,41326 +61732,41326 +49520,41327 +41327,41327 +49521,41328 +41328,41328 +49522,41329 +41329,41329 +49523,41330 +41330,41330 +49604,41411 +41411,41411 +49605,41412 +41412,41412 +49606,41413 +41413,41413 +49607,41414 +41414,41414 +49608,41415 +41415,41415 +49609,41416 +41416,41416 +49610,41417 +41417,41417 +49611,41418 +41418,41418 +116611,41418 +49612,41419 +41419,41419 +49613,41420 +41420,41420 +49614,41421 +41421,41421 +49615,41422 +41422,41422 +49616,41423 +41423,41423 +142957,41423 +49617,41424 +41424,41424 +49618,41425 +41425,41425 +49619,41426 +41426,41426 +49620,41427 +41427,41427 +49621,41428 +41428,41428 +49803,41610 +41610,41610 +49804,41611 +41611,41611 +49805,41612 +41612,41612 +49806,41613 +41613,41613 +49807,41614 +41614,41614 +49808,41615 +41615,41615 +217573,41616 +49809,41616 +41616,41616 +49810,41617 +41617,41617 +49811,41618 +41618,41618 +49812,41619 +41619,41619 +57163,41619 +49813,41620 +41620,41620 +49814,41621 +41621,41621 +49815,41622 +41622,41622 +49816,41623 +41623,41623 +49817,41624 +41624,41624 +49818,41625 +41625,41625 +198748,41625 +198738,41625 +49819,41626 +41626,41626 +49820,41627 +41627,41627 +49821,41628 +41628,41628 +49822,41629 +41629,41629 +49823,41630 +41630,41630 +56737,41631 +49824,41631 +41631,41631 +3526,41631 +49825,41632 +41632,41632 +49826,41633 +41633,41633 +49827,41634 +41634,41634 +49828,41635 +41635,41635 +49829,41636 +41636,41636 +49830,41637 +41637,41637 +49927,41734 +41734,41734 +49928,41735 +41735,41735 +49929,41736 +41736,41736 +49930,41737 +41737,41737 +49982,41789 +41789,41789 +49983,41790 +41790,41790 +49984,41791 +41791,41791 +49985,41792 +41792,41792 +49986,41793 +41793,41793 +49987,41794 +41794,41794 +49988,41795 +41795,41795 +49989,41796 +41796,41796 +49990,41797 +41797,41797 +49991,41798 +41798,41798 +49992,41799 +41799,41799 +49993,41800 +41800,41800 +49994,41801 +41801,41801 +49995,41802 +41802,41802 +49996,41803 +41803,41803 +49997,41804 +41804,41804 +49998,41805 +41805,41805 +49999,41806 +41806,41806 +50000,41807 +41807,41807 +50001,41808 +41808,41808 +50002,41809 +41809,41809 +50003,41810 +41810,41810 +50004,41811 +41811,41811 +50005,41812 +41812,41812 +50006,41813 +41813,41813 +50007,41814 +41814,41814 +50008,41815 +41815,41815 +50009,41816 +41816,41816 +50010,41817 +41817,41817 +64091,41817 +50011,41818 +41818,41818 +50012,41819 +41819,41819 +50013,41820 +41820,41820 +50014,41821 +41821,41821 +50015,41822 +41822,41822 +50016,41823 +41823,41823 +50017,41824 +41824,41824 +50018,41825 +41825,41825 +185462,41825 +50019,41826 +41826,41826 +50020,41827 +41827,41827 +50021,41828 +41828,41828 +50022,41829 +41829,41829 +50023,41830 +41830,41830 +50024,41831 +41831,41831 +50025,41832 +41832,41832 +50026,41833 +41833,41833 +135226,41833 +127639,41833 +50027,41834 +41834,41834 +50028,41835 +41835,41835 +50029,41836 +41836,41836 +50030,41837 +41837,41837 +50031,41838 +41838,41838 +50032,41839 +41839,41839 +50033,41840 +41840,41840 +50034,41841 +41841,41841 +50035,41842 +41842,41842 +50036,41843 +41843,41843 +50037,41844 +41844,41844 +211135,41844 +50038,41845 +41845,41845 +35777,41845 +50039,41846 +41846,41846 +50040,41847 +41847,41847 +50041,41848 +41848,41848 +50042,41849 +41849,41849 +50043,41850 +41850,41850 +76463,41850 +50044,41851 +41851,41851 +50045,41852 +41852,41852 +50046,41853 +41853,41853 +50047,41854 +41854,41854 +50048,41855 +41855,41855 +50049,41856 +41856,41856 +50050,41857 +41857,41857 +50088,41895 +41895,41895 +50089,41896 +41896,41896 +50090,41897 +41897,41897 +50091,41898 +41898,41898 +50092,41899 +41899,41899 +50093,41900 +41900,41900 +50094,41901 +41901,41901 +50095,41902 +41902,41902 +50096,41903 +41903,41903 +50097,41904 +41904,41904 +50098,41905 +41905,41905 +50099,41906 +41906,41906 +50100,41907 +41907,41907 +50101,41908 +41908,41908 +50102,41909 +41909,41909 +50103,41910 +41910,41910 +50104,41911 +41911,41911 +50105,41912 +41912,41912 +154924,41912 +50106,41913 +41913,41913 +50107,41914 +41914,41914 +50108,41915 +41915,41915 +34004,41915 +50109,41916 +41916,41916 +50110,41917 +41917,41917 +50111,41918 +41918,41918 +7618,41918 +50112,41919 +41919,41919 +76379,41919 +50113,41920 +41920,41920 +50114,41921 +41921,41921 +50115,41922 +41922,41922 +50116,41923 +41923,41923 +50117,41924 +41924,41924 +50118,41925 +41925,41925 +50119,41926 +41926,41926 +50120,41927 +41927,41927 +50121,41928 +41928,41928 +50122,41929 +41929,41929 +50123,41930 +41930,41930 +50124,41931 +41931,41931 +50125,41932 +41932,41932 +50126,41933 +41933,41933 +50127,41934 +41934,41934 +69845,41934 +77461,41934 +50128,41935 +41935,41935 +50129,41936 +41936,41936 +50130,41937 +41937,41937 +50131,41938 +41938,41938 +50132,41939 +41939,41939 +585,41939 +50133,41940 +41940,41940 +41941,41941 +50134,41941 +41942,41942 +50135,41942 +41943,41943 +41944,41944 +80319,41944 +41945,41945 +41946,41946 +41947,41947 +165684,41947 +41948,41948 +41949,41949 +41950,41950 +41951,41951 +41952,41952 +41953,41953 +41954,41954 +41955,41955 +41956,41956 +41957,41957 +239492,41957 +41958,41958 +55928,41958 +47824,41958 +50150,41958 +41959,41959 +50151,41959 +50152,41960 +41960,41960 +50153,41961 +41961,41961 +50154,41962 +41962,41962 +3779,41962 +50155,41963 +41963,41963 +41964,41964 +41965,41965 +41966,41966 +41967,41967 +50160,41968 +41968,41968 +41969,41969 +50162,41970 +41970,41970 +41971,41971 +50164,41972 +41972,41972 +50210,42018 +42018,42018 +22032,42018 +50211,42019 +42019,42019 +50212,42020 +42020,42020 +50213,42021 +42021,42021 +50214,42022 +42022,42022 +50215,42023 +42023,42023 +50216,42024 +42024,42024 +50217,42025 +42025,42025 +145103,42025 +50218,42026 +42026,42026 +196305,42026 +50219,42027 +42027,42027 +61483,42027 +50220,42028 +42028,42028 +147587,42029 +50221,42029 +42029,42029 +50222,42030 +42030,42030 +50223,42031 +42031,42031 +50224,42032 +42032,42032 +50225,42033 +42033,42033 +50226,42034 +42034,42034 +50227,42035 +42035,42035 +50228,42036 +42036,42036 +50229,42037 +42037,42037 +234858,42037 +50230,42038 +42038,42038 +211334,42038 +56757,42038 +50231,42039 +42039,42039 +50232,42040 +42040,42040 +50233,42041 +42041,42041 +50234,42042 +42042,42042 +50235,42043 +42043,42043 +50236,42044 +42044,42044 +50237,42045 +42045,42045 +50238,42046 +42046,42046 +50239,42047 +42047,42047 +50240,42048 +42048,42048 +9353,42048 +50241,42049 +42049,42049 +220184,42049 +50242,42050 +42050,42050 +50243,42051 +42051,42051 +50244,42052 +42052,42052 +50245,42053 +42053,42053 +50246,42054 +42054,42054 +166443,42054 +50247,42055 +42055,42055 +118513,42055 +50248,42056 +42056,42056 +50249,42057 +42057,42057 +50250,42058 +42058,42058 +50251,42059 +42059,42059 +50252,42060 +42060,42060 +50289,42097 +42097,42097 +50290,42098 +42098,42098 +50291,42099 +42099,42099 +50292,42100 +42100,42100 +50293,42101 +42101,42101 +50294,42102 +42102,42102 +50295,42103 +42103,42103 +50296,42104 +42104,42104 +50297,42105 +42105,42105 +50298,42106 +42106,42106 +50299,42107 +42107,42107 +50300,42108 +42108,42108 +50301,42109 +42109,42109 +50302,42110 +42110,42110 +50303,42111 +42111,42111 +50304,42112 +42112,42112 +50305,42113 +42113,42113 +50306,42114 +42114,42114 +50307,42115 +42115,42115 +50308,42116 +42116,42116 +50309,42117 +42117,42117 +50310,42118 +42118,42118 +50311,42119 +42119,42119 +50312,42120 +42120,42120 +50313,42121 +42121,42121 +50314,42122 +42122,42122 +50315,42123 +42123,42123 +50316,42124 +42124,42124 +50317,42125 +42125,42125 +75170,42125 +50318,42126 +42126,42126 +50319,42127 +42127,42127 +50320,42128 +42128,42128 +50321,42129 +42129,42129 +50322,42130 +42130,42130 +50323,42131 +42131,42131 +50324,42132 +42132,42132 +50325,42133 +42133,42133 +50326,42134 +42134,42134 +50327,42135 +42135,42135 +50328,42136 +42136,42136 +50329,42137 +42137,42137 +50330,42138 +42138,42138 +50331,42139 +42139,42139 +50332,42140 +42140,42140 +50333,42141 +42141,42141 +50334,42142 +42142,42142 +50335,42143 +42143,42143 +50336,42144 +42144,42144 +50337,42145 +42145,42145 +50338,42146 +42146,42146 +50339,42147 +42147,42147 +50340,42148 +42148,42148 +10522,42148 +50341,42149 +42149,42149 +50342,42150 +42150,42150 +50343,42151 +42151,42151 +50344,42152 +42152,42152 +50345,42153 +42153,42153 +50346,42154 +42154,42154 +202796,42154 +50347,42155 +42155,42155 +50348,42156 +42156,42156 +50349,42157 +42157,42157 +50350,42158 +42158,42158 +54585,42158 +46481,42158 +50351,42159 +42159,42159 +56743,42159 +50352,42160 +42160,42160 +50353,42161 +42161,42161 +50354,42162 +42162,42162 +50355,42163 +42163,42163 +50356,42164 +42164,42164 +50357,42165 +42165,42165 +66802,42165 +50358,42166 +42166,42166 +16276,42166 +50359,42167 +42167,42167 +42614,42614 +50718,42614 +42615,42615 +50720,42616 +42616,42616 +42617,42617 +50721,42617 +42618,42618 +50722,42618 +50723,42619 +42619,42619 +42620,42620 +79968,42620 +42621,42621 +42622,42622 +50726,42622 +50727,42623 +42623,42623 +42624,42624 +50728,42624 +50729,42625 +42625,42625 +42626,42626 +50730,42626 +42627,42627 +50731,42627 +42628,42628 +50732,42628 +50733,42629 +42629,42629 +50734,42630 +42630,42630 +50735,42631 +42631,42631 +50736,42632 +42632,42632 +50737,42633 +42633,42633 +50738,42634 +42634,42634 +50739,42635 +42635,42635 +50740,42636 +42636,42636 +50741,42637 +42637,42637 +50742,42638 +42638,42638 +50743,42639 +42639,42639 +213290,42639 +50744,42640 +42640,42640 +50745,42641 +42641,42641 +50746,42642 +42642,42642 +50747,42643 +42643,42643 +97959,42643 +50748,42644 +42644,42644 +50749,42645 +42645,42645 +50750,42646 +42646,42646 +50751,42647 +42647,42647 +50752,42648 +42648,42648 +50753,42649 +42649,42649 +50754,42650 +42650,42650 +50755,42651 +42651,42651 +50756,42652 +42652,42652 +50757,42653 +42653,42653 +50758,42654 +42654,42654 +50759,42655 +42655,42655 +50760,42656 +42656,42656 +50761,42657 +42657,42657 +50762,42658 +42658,42658 +205239,42658 +50763,42659 +42659,42659 +178528,42660 +172564,42660 +50764,42660 +42660,42660 +50765,42661 +42661,42661 +50766,42662 +42662,42662 +50767,42663 +42663,42663 +50768,42664 +42664,42664 +50769,42665 +42665,42665 +50770,42666 +42666,42666 +50771,42667 +42667,42667 +50772,42668 +42668,42668 +50773,42669 +42669,42669 +50774,42670 +42670,42670 +50775,42671 +42671,42671 +50776,42672 +42672,42672 +50777,42673 +42673,42673 +50778,42674 +42674,42674 +50779,42675 +42675,42675 +50780,42676 +42676,42676 +50781,42677 +42677,42677 +50782,42678 +42678,42678 +50783,42679 +42679,42679 +50784,42680 +42680,42680 +50785,42681 +42681,42681 +50786,42682 +42682,42682 +50787,42683 +42683,42683 +50788,42684 +42684,42684 +50789,42685 +42685,42685 +50790,42686 +42686,42686 +50791,42687 +42687,42687 +50792,42688 +42688,42688 +50793,42689 +42689,42689 +50794,42690 +42690,42690 +50795,42691 +42691,42691 +50796,42692 +42692,42692 +50797,42693 +42693,42693 +50798,42694 +42694,42694 +50799,42695 +42695,42695 +50800,42696 +42696,42696 +50801,42697 +42697,42697 +50802,42698 +42698,42698 +50803,42699 +42699,42699 +50804,42700 +42700,42700 +50805,42701 +42701,42701 +3698,42701 +50806,42702 +42702,42702 +50807,42703 +42703,42703 +50808,42704 +42704,42704 +50809,42705 +42705,42705 +50810,42706 +42706,42706 +50811,42707 +42707,42707 +151807,42707 +50812,42708 +42708,42708 +50813,42709 +42709,42709 +50814,42710 +42710,42710 +50889,42785 +42785,42785 +50890,42786 +42786,42786 +50891,42787 +42787,42787 +50892,42788 +42788,42788 +50893,42789 +42789,42789 +50894,42790 +42790,42790 +178485,42790 +172521,42790 +50895,42791 +42791,42791 +50896,42792 +42792,42792 +50897,42793 +42793,42793 +69838,42793 +50898,42794 +42794,42794 +50899,42795 +42795,42795 +50900,42796 +42796,42796 +50901,42797 +42797,42797 +50902,42798 +42798,42798 +50903,42799 +42799,42799 +50904,42800 +42800,42800 +228725,42800 +50905,42801 +42801,42801 +234728,42801 +50906,42802 +42802,42802 +50907,42803 +42803,42803 +50908,42804 +42804,42804 +50909,42805 +42805,42805 +50910,42806 +42806,42806 +50911,42807 +42807,42807 +50912,42808 +42808,42808 +50913,42809 +42809,42809 +50914,42810 +42810,42810 +50915,42811 +42811,42811 +50916,42812 +42812,42812 +50917,42813 +42813,42813 +81591,42813 +50918,42814 +42814,42814 +50919,42815 +42815,42815 +50920,42816 +42816,42816 +50921,42817 +42817,42817 +50922,42818 +42818,42818 +50923,42819 +42819,42819 +50924,42820 +42820,42820 +50925,42821 +42821,42821 +76513,42821 +50926,42822 +42822,42822 +50927,42823 +42823,42823 +154814,42823 +50928,42824 +42824,42824 +50929,42825 +42825,42825 +50930,42826 +42826,42826 +50931,42827 +42827,42827 +20895,42827 +50932,42828 +42828,42828 +50933,42829 +42829,42829 +50934,42830 +42830,42830 +50935,42831 +42831,42831 +50936,42832 +42832,42832 +50937,42833 +42833,42833 +50938,42834 +42834,42834 +50939,42835 +42835,42835 +50940,42836 +42836,42836 +50941,42837 +42837,42837 +50942,42838 +42838,42838 +50943,42839 +42839,42839 +50944,42840 +42840,42840 +50945,42841 +42841,42841 +50946,42842 +42842,42842 +50947,42843 +42843,42843 +50948,42844 +42844,42844 +50949,42845 +42845,42845 +50950,42846 +42846,42846 +50951,42847 +42847,42847 +50952,42848 +42848,42848 +50953,42849 +42849,42849 +50954,42850 +42850,42850 +50955,42851 +42851,42851 +50956,42852 +42852,42852 +50957,42853 +42853,42853 +50958,42854 +42854,42854 +24020,42854 +50959,42855 +42855,42855 +50960,42856 +42856,42856 +51000,42896 +42896,42896 +51001,42897 +42897,42897 +159461,42897 +51002,42898 +42898,42898 +199234,42898 +51003,42899 +42899,42899 +51004,42900 +42900,42900 +51005,42901 +42901,42901 +51006,42902 +42902,42902 +51007,42903 +42903,42903 +51008,42904 +42904,42904 +6141,42904 +51009,42905 +42905,42905 +51010,42906 +42906,42906 +51011,42907 +42907,42907 +51012,42908 +42908,42908 +51051,42947 +42947,42947 +51052,42948 +42948,42948 +51053,42949 +42949,42949 +51054,42950 +42950,42950 +51055,42951 +42951,42951 +51056,42952 +42952,42952 +51057,42953 +42953,42953 +51058,42954 +42954,42954 +51059,42955 +42955,42955 +51060,42956 +42956,42956 +145779,42956 +51061,42957 +42957,42957 +51062,42958 +42958,42958 +187084,42958 +51063,42959 +42959,42959 +51064,42960 +42960,42960 +51065,42961 +42961,42961 +51066,42962 +42962,42962 +51067,42963 +42963,42963 +51068,42964 +42964,42964 +51069,42965 +42965,42965 +51070,42966 +42966,42966 +51071,42967 +42967,42967 +51072,42968 +42968,42968 +51073,42969 +42969,42969 +51074,42970 +42970,42970 +51075,42971 +42971,42971 +51076,42972 +42972,42972 +51077,42973 +42973,42973 +51078,42974 +42974,42974 +51079,42975 +42975,42975 +51080,42976 +42976,42976 +51081,42977 +42977,42977 +51082,42978 +42978,42978 +51083,42979 +42979,42979 +51084,42980 +42980,42980 +51085,42981 +42981,42981 +51086,42982 +42982,42982 +51087,42983 +42983,42983 +51088,42984 +42984,42984 +51089,42985 +42985,42985 +51090,42986 +42986,42986 +51091,42987 +42987,42987 +51092,42988 +42988,42988 +51093,42989 +42989,42989 +51094,42990 +42990,42990 +51095,42991 +42991,42991 +51096,42992 +42992,42992 +51097,42993 +42993,42993 +51098,42994 +42994,42994 +51099,42995 +42995,42995 +51100,42996 +42996,42996 +4473,42996 +51101,42997 +42997,42997 +51102,42998 +42998,42998 +111283,42998 +51103,42999 +42999,42999 +51104,43000 +43000,43000 +51105,43001 +43001,43001 +51106,43002 +43002,43002 +51107,43003 +43003,43003 +51108,43004 +43004,43004 +51109,43005 +43005,43005 +51110,43006 +43006,43006 +51111,43007 +43007,43007 +51112,43008 +43008,43008 +51113,43009 +43009,43009 +51114,43010 +43010,43010 +51115,43011 +43011,43011 +51116,43012 +43012,43012 +51117,43013 +43013,43013 +51118,43014 +43014,43014 +226595,43014 +51119,43015 +43015,43015 +191834,43016 +51120,43016 +43016,43016 +51121,43017 +43017,43017 +51122,43018 +43018,43018 +51123,43019 +43019,43019 +51124,43020 +43020,43020 +51125,43021 +43021,43021 +51126,43022 +43022,43022 +51127,43023 +43023,43023 +51128,43024 +43024,43024 +51129,43025 +43025,43025 +51130,43026 +43026,43026 +51131,43027 +43027,43027 +51132,43028 +43028,43028 +61438,43028 +51133,43029 +43029,43029 +51134,43030 +43030,43030 +51135,43031 +43031,43031 +51136,43032 +43032,43032 +51137,43033 +43033,43033 +51138,43034 +43034,43034 +51139,43035 +43035,43035 +51140,43036 +43036,43036 +51141,43037 +43037,43037 +147140,43037 +51142,43038 +43038,43038 +51143,43039 +43039,43039 +51144,43040 +43040,43040 +51145,43041 +43041,43041 +51146,43042 +43042,43042 +51147,43043 +43043,43043 +51148,43044 +43044,43044 +51149,43045 +43045,43045 +51150,43046 +43046,43046 +51151,43047 +43047,43047 +51152,43048 +43048,43048 +51153,43049 +43049,43049 +51154,43050 +43050,43050 +51155,43051 +43051,43051 +51156,43052 +43052,43052 +17209,43052 +51157,43053 +43053,43053 +51158,43054 +43054,43054 +51159,43055 +43055,43055 +51160,43056 +43056,43056 +51161,43057 +43057,43057 +186051,43057 +51162,43058 +43058,43058 +51163,43059 +43059,43059 +51164,43060 +43060,43060 +22043,43060 +51265,43161 +43161,43161 +51266,43162 +43162,43162 +51267,43163 +43163,43163 +51268,43164 +43164,43164 +51269,43165 +43165,43165 +51270,43166 +43166,43166 +51271,43167 +43167,43167 +51272,43168 +43168,43168 +51273,43169 +43169,43169 +51274,43170 +43170,43170 +51275,43171 +43171,43171 +51276,43172 +43172,43172 +51277,43173 +43173,43173 +51278,43174 +43174,43174 +51279,43175 +43175,43175 +51280,43176 +43176,43176 +51281,43177 +43177,43177 +51282,43178 +43178,43178 +51283,43179 +43179,43179 +20225,43179 +51284,43180 +43180,43180 +51285,43181 +43181,43181 +51286,43182 +43182,43182 +51287,43183 +43183,43183 +51288,43184 +43184,43184 +51289,43185 +43185,43185 +51290,43186 +43186,43186 +51291,43187 +43187,43187 +51292,43188 +43188,43188 +51293,43189 +43189,43189 +51294,43190 +43190,43190 +51295,43191 +43191,43191 +51296,43192 +43192,43192 +51297,43193 +43193,43193 +51298,43194 +43194,43194 +220606,43194 +51299,43195 +43195,43195 +51300,43196 +43196,43196 +51301,43197 +43197,43197 +51302,43198 +43198,43198 +51303,43199 +43199,43199 +74947,43199 +51304,43200 +43200,43200 +51305,43201 +43201,43201 +158990,43202 +51306,43202 +43202,43202 +51307,43203 +43203,43203 +51308,43204 +43204,43204 +248792,43204 +51309,43205 +43205,43205 +51310,43206 +43206,43206 +51311,43207 +43207,43207 +51312,43208 +43208,43208 +51313,43209 +43209,43209 +51314,43210 +43210,43210 +51315,43211 +43211,43211 +51316,43212 +43212,43212 +51317,43213 +43213,43213 +51318,43214 +43214,43214 +51319,43215 +43215,43215 +51320,43216 +43216,43216 +51321,43217 +43217,43217 +51322,43218 +43218,43218 +51323,43219 +43219,43219 +51324,43220 +43220,43220 +51325,43221 +43221,43221 +51326,43222 +43222,43222 +51327,43223 +43223,43223 +26905,43223 +51328,43224 +43224,43224 +51329,43225 +43225,43225 +51330,43226 +43226,43226 +51331,43227 +43227,43227 +51332,43228 +43228,43228 +51333,43229 +43229,43229 +51334,43230 +43230,43230 +51335,43231 +43231,43231 +51336,43232 +43232,43232 +51337,43233 +43233,43233 +51338,43234 +43234,43234 +51339,43235 +43235,43235 +51340,43236 +43236,43236 +51341,43237 +43237,43237 +51342,43238 +43238,43238 +51343,43239 +43239,43239 +51344,43240 +43240,43240 +85610,43240 +51345,43241 +43241,43241 +51346,43242 +43242,43242 +51347,43243 +43243,43243 +51348,43244 +43244,43244 +51349,43245 +43245,43245 +51350,43246 +43246,43246 +51351,43247 +43247,43247 +51352,43248 +43248,43248 +51353,43249 +43249,43249 +51354,43250 +43250,43250 +78636,43250 +51355,43251 +43251,43251 +51356,43252 +43252,43252 +51357,43253 +43253,43253 +51358,43254 +43254,43254 +51359,43255 +43255,43255 +51360,43256 +43256,43256 +51361,43257 +43257,43257 +51362,43258 +43258,43258 +51363,43259 +43259,43259 +51364,43260 +43260,43260 +51365,43261 +43261,43261 +51366,43262 +43262,43262 +51367,43263 +43263,43263 +51368,43264 +43264,43264 +51369,43265 +43265,43265 +51370,43266 +43266,43266 +51371,43267 +43267,43267 +51372,43268 +43268,43268 +51373,43269 +43269,43269 +51374,43270 +43270,43270 +51375,43271 +43271,43271 +51376,43272 +43272,43272 +51377,43273 +43273,43273 +51378,43274 +43274,43274 +51379,43275 +43275,43275 +51380,43276 +43276,43276 +51381,43277 +43277,43277 +51382,43278 +43278,43278 +51383,43279 +43279,43279 +51452,43348 +43348,43348 +51453,43349 +43349,43349 +51454,43350 +43350,43350 +51455,43351 +43351,43351 +51456,43352 +43352,43352 +19967,43352 +51457,43353 +43353,43353 +51458,43354 +43354,43354 +51459,43355 +43355,43355 +184657,43355 +51460,43356 +43356,43356 +51461,43357 +43357,43357 +51462,43358 +43358,43358 +51463,43359 +43359,43359 +51464,43360 +43360,43360 +51465,43361 +43361,43361 +51466,43362 +43362,43362 +51467,43363 +43363,43363 +51468,43364 +43364,43364 +93874,43365 +51469,43365 +43365,43365 +51470,43366 +43366,43366 +51471,43367 +43367,43367 +51472,43368 +43368,43368 +51473,43369 +43369,43369 +51474,43370 +43370,43370 +51475,43371 +43371,43371 +51476,43372 +43372,43372 +51477,43373 +43373,43373 +51478,43374 +43374,43374 +51479,43375 +43375,43375 +51480,43376 +43376,43376 +51481,43377 +43377,43377 +51482,43378 +43378,43378 +51483,43379 +43379,43379 +51484,43380 +43380,43380 +51485,43381 +43381,43381 +51486,43382 +43382,43382 +51487,43383 +43383,43383 +51488,43384 +43384,43384 +51489,43385 +43385,43385 +51490,43386 +43386,43386 +51491,43387 +43387,43387 +51492,43388 +43388,43388 +51493,43389 +43389,43389 +51494,43390 +43390,43390 +51495,43391 +43391,43391 +51496,43392 +43392,43392 +51497,43393 +43393,43393 +51498,43394 +43394,43394 +51499,43395 +43395,43395 +51500,43396 +43396,43396 +51501,43397 +43397,43397 +51502,43398 +43398,43398 +51503,43399 +43399,43399 +51504,43400 +43400,43400 +51505,43401 +43401,43401 +51506,43402 +43402,43402 +51507,43403 +43403,43403 +51508,43404 +43404,43404 +51509,43405 +43405,43405 +86092,43405 +51510,43406 +43406,43406 +51511,43407 +43407,43407 +51512,43408 +43408,43408 +51513,43409 +43409,43409 +51514,43410 +43410,43410 +51515,43411 +43411,43411 +51516,43412 +43412,43412 +51517,43413 +43413,43413 +51518,43414 +43414,43414 +148372,43414 +51519,43415 +43415,43415 +51520,43416 +43416,43416 +51521,43417 +43417,43417 +123296,43417 +196583,43418 +51522,43418 +43418,43418 +51523,43419 +43419,43419 +51524,43420 +43420,43420 +51525,43421 +43421,43421 +51526,43422 +43422,43422 +51527,43423 +43423,43423 +51528,43424 +43424,43424 +51529,43425 +43425,43425 +51530,43426 +43426,43426 +51531,43427 +43427,43427 +51532,43428 +43428,43428 +51533,43429 +43429,43429 +51534,43430 +43430,43430 +51535,43431 +43431,43431 +51536,43432 +43432,43432 +51537,43433 +43433,43433 +51538,43434 +43434,43434 +51539,43435 +43435,43435 +51540,43436 +43436,43436 +51541,43437 +43437,43437 +221289,43437 +51542,43438 +43438,43438 +51543,43439 +43439,43439 +51544,43440 +43440,43440 +51545,43441 +43441,43441 +51546,43442 +43442,43442 +51547,43443 +43443,43443 +145187,43443 +51548,43444 +43444,43444 +51549,43445 +43445,43445 +51550,43446 +43446,43446 +51551,43447 +43447,43447 +51552,43448 +43448,43448 +154508,43448 +51553,43449 +43449,43449 +51554,43450 +43450,43450 +51555,43451 +43451,43451 +51556,43452 +43452,43452 +51557,43453 +43453,43453 +51558,43454 +43454,43454 +51606,43502 +43502,43502 +51607,43503 +43503,43503 +51643,43539 +43539,43539 +51644,43540 +43540,43540 +51645,43541 +43541,43541 +51646,43542 +43542,43542 +51647,43543 +43543,43543 +140120,43543 +51648,43544 +43544,43544 +51649,43545 +43545,43545 +51650,43546 +43546,43546 +51651,43547 +43547,43547 +51652,43548 +43548,43548 +51653,43549 +43549,43549 +51654,43550 +43550,43550 +234193,43550 +30808,43550 +51655,43551 +43551,43551 +51656,43552 +43552,43552 +51657,43553 +43553,43553 +51658,43554 +43554,43554 +51659,43555 +43555,43555 +217033,43555 +51660,43556 +43556,43556 +51661,43557 +43557,43557 +51710,43606 +43606,43606 +51711,43607 +43607,43607 +158972,43607 +51712,43608 +43608,43608 +51713,43609 +43609,43609 +51714,43610 +43610,43610 +51715,43611 +43611,43611 +51716,43612 +43612,43612 +51717,43613 +43613,43613 +247816,43613 +51718,43614 +43614,43614 +51719,43615 +43615,43615 +51720,43616 +43616,43616 +51721,43617 +43617,43617 +51722,43618 +43618,43618 +51723,43619 +43619,43619 +206001,43619 +51724,43620 +43620,43620 +51725,43621 +43621,43621 +51726,43622 +43622,43622 +51727,43623 +43623,43623 +51728,43624 +43624,43624 +51729,43625 +43625,43625 +51730,43626 +43626,43626 +51731,43627 +43627,43627 +51732,43628 +43628,43628 +51733,43629 +43629,43629 +51734,43630 +43630,43630 +161276,43630 +51735,43631 +43631,43631 +51736,43632 +43632,43632 +51737,43633 +43633,43633 +19898,43633 +51738,43634 +43634,43634 +51739,43635 +43635,43635 +51740,43636 +43636,43636 +51741,43637 +43637,43637 +51742,43638 +43638,43638 +51743,43639 +43639,43639 +51744,43640 +43640,43640 +51745,43641 +43641,43641 +51746,43642 +43642,43642 +51747,43643 +43643,43643 +16259,43643 +51748,43644 +43644,43644 +51784,43680 +43680,43680 +51785,43681 +43681,43681 +51786,43682 +43682,43682 +152302,43682 +51787,43683 +43683,43683 +51788,43684 +43684,43684 +51789,43685 +43685,43685 +51790,43686 +43686,43686 +51791,43687 +43687,43687 +51792,43688 +43688,43688 +51793,43689 +43689,43689 +51794,43690 +43690,43690 +51795,43691 +43691,43691 +51796,43692 +43692,43692 +51797,43693 +43693,43693 +51798,43694 +43694,43694 +51799,43695 +43695,43695 +51800,43696 +43696,43696 +51801,43697 +43697,43697 +196715,43697 +51802,43698 +43698,43698 +51803,43699 +43699,43699 +51804,43700 +43700,43700 +51805,43701 +43701,43701 +51806,43702 +43702,43702 +51807,43703 +43703,43703 +176536,43703 +170572,43703 +51808,43704 +43704,43704 +51809,43705 +43705,43705 +51810,43706 +43706,43706 +51811,43707 +43707,43707 +140684,43707 +51812,43708 +43708,43708 +64735,43708 +51813,43709 +43709,43709 +51814,43710 +43710,43710 +184422,43710 +51815,43711 +43711,43711 +51816,43712 +43712,43712 +51817,43713 +43713,43713 +51818,43714 +43714,43714 +51819,43715 +43715,43715 +57686,43715 +176241,43715 +170277,43715 +104028,43715 +51820,43716 +43716,43716 +51821,43717 +43717,43717 +51822,43718 +43718,43718 +51823,43719 +43719,43719 +51824,43720 +43720,43720 +51825,43721 +43721,43721 +51826,43722 +43722,43722 +51827,43723 +43723,43723 +51828,43724 +43724,43724 +51829,43725 +43725,43725 +51830,43726 +43726,43726 +51831,43727 +43727,43727 +51832,43728 +43728,43728 +51833,43729 +43729,43729 +51834,43730 +43730,43730 +51835,43731 +43731,43731 +51836,43732 +43732,43732 +51837,43733 +43733,43733 +51838,43734 +43734,43734 +51839,43735 +43735,43735 +51840,43736 +43736,43736 +51841,43737 +43737,43737 +51842,43738 +43738,43738 +51843,43739 +43739,43739 +51844,43740 +43740,43740 +51845,43741 +43741,43741 +51846,43742 +43742,43742 +51847,43743 +43743,43743 +51848,43744 +43744,43744 +51849,43745 +43745,43745 +51850,43746 +43746,43746 +51851,43747 +43747,43747 +203676,43747 +51852,43748 +43748,43748 +166458,43748 +51853,43749 +43749,43749 +51854,43750 +43750,43750 +51855,43751 +43751,43751 +51856,43752 +43752,43752 +51857,43753 +43753,43753 +51858,43754 +43754,43754 +51859,43755 +43755,43755 +51860,43756 +43756,43756 +51861,43757 +43757,43757 +51862,43758 +43758,43758 +51863,43759 +43759,43759 +51864,43760 +43760,43760 +51865,43761 +43761,43761 +51866,43762 +43762,43762 +235617,43762 +51867,43763 +43763,43763 +51868,43764 +43764,43764 +51869,43765 +43765,43765 +51870,43766 +43766,43766 +51917,43813 +43813,43813 +51918,43814 +43814,43814 +51919,43815 +43815,43815 +51920,43816 +43816,43816 +51921,43817 +43817,43817 +51922,43818 +43818,43818 +51923,43819 +43819,43819 +51924,43820 +43820,43820 +51925,43821 +43821,43821 +51926,43822 +43822,43822 +51927,43823 +43823,43823 +51928,43824 +43824,43824 +52057,43953 +43953,43953 +180875,43953 +52058,43954 +43954,43954 +52059,43955 +43955,43955 +52060,43956 +43956,43956 +180878,43956 +52061,43957 +43957,43957 +52062,43958 +43958,43958 +52063,43959 +43959,43959 +52064,43960 +43960,43960 +52065,43961 +43961,43961 +52066,43962 +43962,43962 +52067,43963 +43963,43963 +52068,43964 +43964,43964 +52069,43965 +43965,43965 +12064,43965 +52070,43966 +43966,43966 +52071,43967 +43967,43967 +52072,43968 +43968,43968 +52073,43969 +43969,43969 +52074,43970 +43970,43970 +52075,43971 +43971,43971 +52076,43972 +43972,43972 +52077,43973 +43973,43973 +52078,43974 +43974,43974 +52079,43975 +43975,43975 +52080,43976 +43976,43976 +52081,43977 +43977,43977 +52082,43978 +43978,43978 +52083,43979 +43979,43979 +52084,43980 +43980,43980 +52085,43981 +43981,43981 +52086,43982 +43982,43982 +52087,43983 +43983,43983 +52088,43984 +43984,43984 +52089,43985 +43985,43985 +52090,43986 +43986,43986 +52091,43987 +43987,43987 +52092,43988 +43988,43988 +52093,43989 +43989,43989 +52094,43990 +43990,43990 +81541,43991 +52095,43991 +43991,43991 +52096,43992 +43992,43992 +52097,43993 +43993,43993 +52098,43994 +43994,43994 +52099,43995 +43995,43995 +52100,43996 +43996,43996 +52101,43997 +43997,43997 +52102,43998 +43998,43998 +52103,43999 +43999,43999 +52104,44000 +44000,44000 +52105,44001 +44001,44001 +52161,44057 +44057,44057 +52162,44058 +44058,44058 +52163,44059 +44059,44059 +52164,44060 +44060,44060 +52165,44061 +44061,44061 +52166,44062 +44062,44062 +52167,44063 +44063,44063 +52168,44064 +44064,44064 +52169,44065 +44065,44065 +52170,44066 +44066,44066 +52171,44067 +44067,44067 +52172,44068 +44068,44068 +52173,44069 +44069,44069 +52174,44070 +44070,44070 +52175,44071 +44071,44071 +52176,44072 +44072,44072 +52177,44073 +44073,44073 +52178,44074 +44074,44074 +52179,44075 +44075,44075 +52180,44076 +44076,44076 +52181,44077 +44077,44077 +52182,44078 +44078,44078 +52183,44079 +44079,44079 +52184,44080 +44080,44080 +52185,44081 +44081,44081 +52186,44082 +44082,44082 +52187,44083 +44083,44083 +52188,44084 +44084,44084 +52189,44085 +44085,44085 +52190,44086 +44086,44086 +52191,44087 +44087,44087 +52192,44088 +44088,44088 +52234,44130 +44130,44130 +52235,44131 +44131,44131 +52236,44132 +44132,44132 +52237,44133 +44133,44133 +52238,44134 +44134,44134 +52239,44135 +44135,44135 +52291,44187 +44187,44187 +52292,44188 +44188,44188 +52293,44189 +44189,44189 +52294,44190 +44190,44190 +52295,44191 +44191,44191 +52296,44192 +44192,44192 +52297,44193 +44193,44193 +52298,44194 +44194,44194 +52299,44195 +44195,44195 +52300,44196 +44196,44196 +52301,44197 +44197,44197 +52302,44198 +44198,44198 +52303,44199 +44199,44199 +52304,44200 +44200,44200 +60354,44200 +52305,44201 +44201,44201 +52306,44202 +44202,44202 +52307,44203 +44203,44203 +52308,44204 +44204,44204 +52309,44205 +44205,44205 +52310,44206 +44206,44206 +52311,44207 +44207,44207 +52312,44208 +44208,44208 +52313,44209 +44209,44209 +52314,44210 +44210,44210 +52315,44211 +44211,44211 +52316,44212 +44212,44212 +52317,44213 +44213,44213 +52318,44214 +44214,44214 +52319,44215 +44215,44215 +52320,44216 +44216,44216 +89184,44217 +52321,44217 +44217,44217 +52322,44218 +44218,44218 +2214,44218 +52323,44219 +44219,44219 +52402,44298 +44298,44298 +52403,44299 +44299,44299 +52404,44300 +44300,44300 +52405,44301 +44301,44301 +52406,44302 +44302,44302 +52407,44303 +44303,44303 +245711,44303 +52408,44304 +44304,44304 +52409,44305 +44305,44305 +52410,44306 +44306,44306 +52411,44307 +44307,44307 +52412,44308 +44308,44308 +52413,44309 +44309,44309 +52414,44310 +44310,44310 +52415,44311 +44311,44311 +52416,44312 +44312,44312 +52417,44313 +44313,44313 +52418,44314 +44314,44314 +52419,44315 +44315,44315 +52420,44316 +44316,44316 +52421,44317 +44317,44317 +52422,44318 +44318,44318 +52423,44319 +44319,44319 +52424,44320 +44320,44320 +36352,44320 +185403,44320 +52425,44321 +44321,44321 +52426,44322 +44322,44322 +52468,44364 +44364,44364 +52469,44365 +44365,44365 +52470,44366 +44366,44366 +52471,44367 +44367,44367 +52472,44368 +44368,44368 +52473,44369 +44369,44369 +52474,44370 +44370,44370 +52475,44371 +44371,44371 +52476,44372 +44372,44372 +52477,44373 +44373,44373 +52478,44374 +44374,44374 +145056,44374 +52479,44375 +44375,44375 +52480,44376 +44376,44376 +52481,44377 +44377,44377 +52482,44378 +44378,44378 +111926,44378 +52545,44441 +44441,44441 +52546,44442 +44442,44442 +52547,44443 +44443,44443 +204911,44443 +52548,44444 +44444,44444 +52549,44445 +44445,44445 +52550,44446 +44446,44446 +52551,44447 +44447,44447 +52552,44448 +44448,44448 +52553,44449 +44449,44449 +52554,44450 +44450,44450 +52555,44451 +44451,44451 +52556,44452 +44452,44452 +52557,44453 +44453,44453 +52558,44454 +44454,44454 +52559,44455 +44455,44455 +52560,44456 +44456,44456 +52561,44457 +44457,44457 +52562,44458 +44458,44458 +52563,44459 +44459,44459 +52564,44460 +44460,44460 +52565,44461 +44461,44461 +52566,44462 +44462,44462 +52567,44463 +44463,44463 +52568,44464 +44464,44464 +52569,44465 +44465,44465 +52570,44466 +44466,44466 +52571,44467 +44467,44467 +52572,44468 +44468,44468 +52573,44469 +44469,44469 +52574,44470 +44470,44470 +52614,44510 +44510,44510 +52615,44511 +44511,44511 +52616,44512 +44512,44512 +52617,44513 +44513,44513 +52618,44514 +44514,44514 +52619,44515 +44515,44515 +52620,44516 +44516,44516 +52621,44517 +44517,44517 +52622,44518 +44518,44518 +52623,44519 +44519,44519 +111697,44519 +52624,44520 +44520,44520 +52625,44521 +44521,44521 +52626,44522 +44522,44522 +52627,44523 +44523,44523 +166454,44523 +52628,44524 +44524,44524 +52629,44525 +44525,44525 +52630,44526 +44526,44526 +52631,44527 +44527,44527 +52632,44528 +44528,44528 +52633,44529 +44529,44529 +52634,44530 +44530,44530 +52635,44531 +44531,44531 +246720,44531 +52636,44532 +44532,44532 +52637,44533 +44533,44533 +52638,44534 +44534,44534 +52639,44535 +44535,44535 +52640,44536 +44536,44536 +52641,44537 +44537,44537 +2704,44537 +52642,44538 +44538,44538 +52643,44539 +44539,44539 +52644,44540 +44540,44540 +52645,44541 +44541,44541 +52646,44542 +44542,44542 +52647,44543 +44543,44543 +52648,44544 +44544,44544 +52649,44545 +44545,44545 +52650,44546 +44546,44546 +52651,44547 +44547,44547 +52652,44548 +44548,44548 +52653,44549 +44549,44549 +8520,44549 +52654,44550 +44550,44550 +52655,44551 +44551,44551 +52694,44590 +44590,44590 +52695,44591 +44591,44591 +83494,44591 +52696,44592 +44592,44592 +52697,44593 +44593,44593 +52698,44594 +44594,44594 +52699,44595 +44595,44595 +52700,44596 +44596,44596 +52701,44597 +44597,44597 +52702,44598 +44598,44598 +52703,44599 +44599,44599 +52704,44600 +44600,44600 +52705,44601 +44601,44601 +52706,44602 +44602,44602 +52707,44603 +44603,44603 +52708,44604 +44604,44604 +52709,44605 +44605,44605 +52710,44606 +44606,44606 +52711,44607 +44607,44607 +52712,44608 +44608,44608 +52713,44609 +44609,44609 +52714,44610 +44610,44610 +52715,44611 +44611,44611 +52716,44612 +44612,44612 +52717,44613 +44613,44613 +52718,44614 +44614,44614 +52719,44615 +44615,44615 +79986,44615 +52720,44616 +44616,44616 +52721,44617 +44617,44617 +52722,44618 +44618,44618 +52723,44619 +44619,44619 +52724,44620 +44620,44620 +52725,44621 +44621,44621 +52726,44622 +44622,44622 +226363,44623 +52727,44623 +44623,44623 +52728,44624 +44624,44624 +52729,44625 +44625,44625 +52730,44626 +44626,44626 +52731,44627 +44627,44627 +52732,44628 +44628,44628 +202708,44628 +52733,44629 +44629,44629 +52734,44630 +44630,44630 +52735,44631 +44631,44631 +52736,44632 +44632,44632 +52737,44633 +44633,44633 +122234,44633 +52738,44634 +44634,44634 +52739,44635 +44635,44635 +52740,44636 +44636,44636 +52741,44637 +44637,44637 +52742,44638 +44638,44638 +52743,44639 +44639,44639 +52744,44640 +44640,44640 +52745,44641 +44641,44641 +148363,44641 +52746,44642 +44642,44642 +52747,44643 +44643,44643 +143979,44643 +49327,44643 +41134,44643 +52748,44644 +44644,44644 +52749,44645 +44645,44645 +52750,44646 +44646,44646 +145541,44646 +52751,44647 +44647,44647 +52752,44648 +44648,44648 +211447,44648 +52753,44649 +44649,44649 +52754,44650 +44650,44650 +52755,44651 +44651,44651 +16812,44651 +52756,44652 +44652,44652 +52757,44653 +44653,44653 +52758,44654 +44654,44654 +52759,44655 +44655,44655 +52883,44779 +44779,44779 +52884,44780 +44780,44780 +146954,44780 +52885,44781 +44781,44781 +107063,44781 +52886,44782 +44782,44782 +52887,44783 +44783,44783 +52888,44784 +44784,44784 +52889,44785 +44785,44785 +52890,44786 +44786,44786 +52891,44787 +44787,44787 +52892,44788 +44788,44788 +187316,44788 +52893,44789 +44789,44789 +52894,44790 +44790,44790 +52895,44791 +44791,44791 +52896,44792 +44792,44792 +52897,44793 +44793,44793 +52898,44794 +44794,44794 +52899,44795 +44795,44795 +52900,44796 +44796,44796 +52901,44797 +44797,44797 +52902,44798 +44798,44798 +52903,44799 +44799,44799 +52904,44800 +44800,44800 +52905,44801 +44801,44801 +52906,44802 +44802,44802 +52907,44803 +44803,44803 +52908,44804 +44804,44804 +52909,44805 +44805,44805 +52910,44806 +44806,44806 +52911,44807 +44807,44807 +52953,44849 +44849,44849 +52954,44850 +44850,44850 +52955,44851 +44851,44851 +52956,44852 +44852,44852 +52957,44853 +44853,44853 +52958,44854 +44854,44854 +52959,44855 +44855,44855 +217481,44855 +52960,44856 +44856,44856 +52961,44857 +44857,44857 +52962,44858 +44858,44858 +52963,44859 +44859,44859 +52964,44860 +44860,44860 +52965,44861 +44861,44861 +52966,44862 +44862,44862 +52967,44863 +44863,44863 +52968,44864 +44864,44864 +52969,44865 +44865,44865 +52970,44866 +44866,44866 +52971,44867 +44867,44867 +80557,44867 +52972,44868 +44868,44868 +52973,44869 +44869,44869 +52974,44870 +44870,44870 +52975,44871 +44871,44871 +53096,44992 +44992,44992 +53097,44993 +44993,44993 +53098,44994 +44994,44994 +53099,44995 +44995,44995 +53151,45047 +45047,45047 +53152,45048 +45048,45048 +110332,45048 +53153,45049 +45049,45049 +53154,45050 +45050,45050 +53155,45051 +45051,45051 +53156,45052 +45052,45052 +53157,45053 +45053,45053 +53158,45054 +45054,45054 +53159,45055 +45055,45055 +53160,45056 +45056,45056 +53161,45057 +45057,45057 +53162,45058 +45058,45058 +53163,45059 +45059,45059 +53164,45060 +45060,45060 +53165,45061 +45061,45061 +84909,45061 +53166,45062 +45062,45062 +241111,45062 +53167,45063 +45063,45063 +53168,45064 +45064,45064 +53169,45065 +45065,45065 +53170,45066 +45066,45066 +53171,45067 +45067,45067 +53225,45121 +45121,45121 +53226,45122 +45122,45122 +53227,45123 +45123,45123 +53228,45124 +45124,45124 +53229,45125 +45125,45125 +53230,45126 +45126,45126 +53231,45127 +45127,45127 +53232,45128 +45128,45128 +110907,45128 +53233,45129 +45129,45129 +53234,45130 +45130,45130 +53235,45131 +45131,45131 +53236,45132 +45132,45132 +53237,45133 +45133,45133 +53238,45134 +45134,45134 +53239,45135 +45135,45135 +53240,45136 +45136,45136 +53241,45137 +45137,45137 +32865,45137 +53242,45138 +45138,45138 +53243,45139 +45139,45139 +187446,45139 +53244,45140 +45140,45140 +53245,45141 +45141,45141 +53246,45142 +45142,45142 +53247,45143 +45143,45143 +53248,45144 +45144,45144 +53249,45145 +45145,45145 +53250,45146 +45146,45146 +79149,45146 +53303,45199 +45199,45199 +53304,45200 +45200,45200 +53305,45201 +45201,45201 +179506,45201 +111339,45201 +53306,45202 +45202,45202 +53307,45203 +45203,45203 +53308,45204 +45204,45204 +53309,45205 +45205,45205 +53310,45206 +45206,45206 +53311,45207 +45207,45207 +53312,45208 +45208,45208 +53313,45209 +45209,45209 +53314,45210 +45210,45210 +53315,45211 +45211,45211 +53316,45212 +45212,45212 +53317,45213 +45213,45213 +53318,45214 +45214,45214 +58020,45214 +53319,45215 +45215,45215 +53320,45216 +45216,45216 +53321,45217 +45217,45217 +53322,45218 +45218,45218 +53323,45219 +45219,45219 +53324,45220 +45220,45220 +53325,45221 +45221,45221 +53326,45222 +45222,45222 +53327,45223 +45223,45223 +53328,45224 +45224,45224 +53329,45225 +45225,45225 +53330,45226 +45226,45226 +53331,45227 +45227,45227 +53332,45228 +45228,45228 +53333,45229 +45229,45229 +89194,45229 +53334,45230 +45230,45230 +244743,45230 +53335,45231 +45231,45231 +53336,45232 +45232,45232 +53337,45233 +45233,45233 +53338,45234 +45234,45234 +53339,45235 +45235,45235 +53340,45236 +45236,45236 +53341,45237 +45237,45237 +53342,45238 +45238,45238 +53343,45239 +45239,45239 +53344,45240 +45240,45240 +53345,45241 +45241,45241 +53346,45242 +45242,45242 +53347,45243 +45243,45243 +53348,45244 +45244,45244 +242313,45244 +53349,45245 +45245,45245 +53350,45246 +45246,45246 +53351,45247 +45247,45247 +53352,45248 +45248,45248 +53353,45249 +45249,45249 +53354,45250 +45250,45250 +53355,45251 +45251,45251 +53356,45252 +45252,45252 +53357,45253 +45253,45253 +53358,45254 +45254,45254 +53359,45255 +45255,45255 +53360,45256 +45256,45256 +108668,45256 +53361,45257 +45257,45257 +243867,45257 +53362,45258 +45258,45258 +53363,45259 +45259,45259 +53364,45260 +45260,45260 +53365,45261 +45261,45261 +53366,45262 +45262,45262 +53367,45263 +45263,45263 +53368,45264 +45264,45264 +206584,45264 +53369,45265 +45265,45265 +53370,45266 +45266,45266 +53371,45267 +45267,45267 +178695,45267 +53372,45268 +45268,45268 +53373,45269 +45269,45269 +53374,45270 +45270,45270 +53375,45271 +45271,45271 +53376,45272 +45272,45272 +53377,45273 +45273,45273 +53378,45274 +45274,45274 +53379,45275 +45275,45275 +53380,45276 +45276,45276 +53381,45277 +45277,45277 +53382,45278 +45278,45278 +53383,45279 +45279,45279 +53384,45280 +45280,45280 +53385,45281 +45281,45281 +53386,45282 +45282,45282 +53387,45283 +45283,45283 +53388,45284 +45284,45284 +177674,45284 +171710,45284 +53389,45285 +45285,45285 +53390,45286 +45286,45286 +53391,45287 +45287,45287 +53392,45288 +45288,45288 +53393,45289 +45289,45289 +53394,45290 +45290,45290 +53395,45291 +45291,45291 +53396,45292 +45292,45292 +53397,45293 +45293,45293 +53398,45294 +45294,45294 +53399,45295 +45295,45295 +53400,45296 +45296,45296 +53401,45297 +45297,45297 +53402,45298 +45298,45298 +53403,45299 +45299,45299 +53404,45300 +45300,45300 +53405,45301 +45301,45301 +53406,45302 +45302,45302 +53407,45303 +45303,45303 +53408,45304 +45304,45304 +53409,45305 +45305,45305 +53410,45306 +45306,45306 +53411,45307 +45307,45307 +53412,45308 +45308,45308 +53413,45309 +45309,45309 +53414,45310 +45310,45310 +53415,45311 +45311,45311 +92714,45311 +216285,45311 +53416,45312 +45312,45312 +53417,45313 +45313,45313 +53418,45314 +45314,45314 +53419,45315 +45315,45315 +53420,45316 +45316,45316 +53421,45317 +45317,45317 +53422,45318 +45318,45318 +53423,45319 +45319,45319 +53424,45320 +45320,45320 +53425,45321 +45321,45321 +53426,45322 +45322,45322 +198415,45322 +53427,45323 +45323,45323 +53428,45324 +45324,45324 +53429,45325 +45325,45325 +53430,45326 +45326,45326 +53431,45327 +45327,45327 +53432,45328 +45328,45328 +53433,45329 +45329,45329 +53434,45330 +45330,45330 +53435,45331 +45331,45331 +53436,45332 +45332,45332 +53437,45333 +45333,45333 +53438,45334 +45334,45334 +53439,45335 +45335,45335 +107153,45335 +53440,45336 +45336,45336 +53441,45337 +45337,45337 +53442,45338 +45338,45338 +53443,45339 +45339,45339 +31708,45339 +53444,45340 +45340,45340 +53445,45341 +45341,45341 +53446,45342 +45342,45342 +53447,45343 +45343,45343 +219844,45343 +53448,45344 +45344,45344 +4372,45344 +53449,45345 +45345,45345 +53450,45346 +45346,45346 +53451,45347 +45347,45347 +53452,45348 +45348,45348 +53453,45349 +45349,45349 +53454,45350 +45350,45350 +53455,45351 +45351,45351 +53456,45352 +45352,45352 +53457,45353 +45353,45353 +53458,45354 +45354,45354 +53459,45355 +45355,45355 +53460,45356 +45356,45356 +53461,45357 +45357,45357 +53462,45358 +45358,45358 +53463,45359 +45359,45359 +53464,45360 +45360,45360 +53465,45361 +45361,45361 +53466,45362 +45362,45362 +53467,45363 +45363,45363 +164059,45363 +53468,45364 +45364,45364 +53469,45365 +45365,45365 +53470,45366 +45366,45366 +53471,45367 +45367,45367 +53679,45575 +45575,45575 +53680,45576 +45576,45576 +53681,45577 +45577,45577 +53682,45578 +45578,45578 +53683,45579 +45579,45579 +53684,45580 +45580,45580 +53685,45581 +45581,45581 +53686,45582 +45582,45582 +53687,45583 +45583,45583 +53688,45584 +45584,45584 +78062,45584 +53689,45585 +45585,45585 +142602,45586 +53690,45586 +45586,45586 +33741,45586 +2712,45586 +53691,45587 +45587,45587 +53692,45588 +45588,45588 +53693,45589 +45589,45589 +53694,45590 +45590,45590 +53695,45591 +45591,45591 +53696,45592 +45592,45592 +53697,45593 +45593,45593 +53698,45594 +45594,45594 +213870,45594 +53699,45595 +45595,45595 +53700,45596 +45596,45596 +200744,45596 +53701,45597 +45597,45597 +53702,45598 +45598,45598 +53703,45599 +45599,45599 +53704,45600 +45600,45600 +53705,45601 +45601,45601 +53706,45602 +45602,45602 +75468,45602 +53707,45603 +45603,45603 +53708,45604 +45604,45604 +53709,45605 +45605,45605 +53710,45606 +45606,45606 +53711,45607 +45607,45607 +53712,45608 +45608,45608 +53713,45609 +45609,45609 +53714,45610 +45610,45610 +53715,45611 +45611,45611 +53716,45612 +45612,45612 +53717,45613 +45613,45613 +53718,45614 +45614,45614 +53719,45615 +45615,45615 +53720,45616 +45616,45616 +53721,45617 +45617,45617 +149635,45618 +53722,45618 +45618,45618 +53723,45619 +45619,45619 +53724,45620 +45620,45620 +53725,45621 +45621,45621 +53726,45622 +45622,45622 +53727,45623 +45623,45623 +53728,45624 +45624,45624 +53729,45625 +45625,45625 +201108,45625 +53730,45626 +45626,45626 +53731,45627 +45627,45627 +53732,45628 +45628,45628 +53733,45629 +45629,45629 +201112,45629 +53734,45630 +45630,45630 +53735,45631 +45631,45631 +53736,45632 +45632,45632 +53737,45633 +45633,45633 +76462,45633 +53738,45634 +45634,45634 +53739,45635 +45635,45635 +53740,45636 +45636,45636 +53741,45637 +45637,45637 +53742,45638 +45638,45638 +53743,45639 +45639,45639 +53744,45640 +45640,45640 +53745,45641 +45641,45641 +53746,45642 +45642,45642 +53747,45643 +45643,45643 +53748,45644 +45644,45644 +53749,45645 +45645,45645 +53750,45646 +45646,45646 +53751,45647 +45647,45647 +53752,45648 +45648,45648 +53753,45649 +45649,45649 +53754,45650 +45650,45650 +53755,45651 +45651,45651 +53756,45652 +45652,45652 +53779,45652 +45675,45652 +53757,45653 +45653,45653 +53758,45654 +45654,45654 +53759,45655 +45655,45655 +53760,45656 +45656,45656 +53761,45657 +45657,45657 +77847,45658 +53762,45658 +45658,45658 +260,45658 +53763,45659 +45659,45659 +53764,45660 +45660,45660 +53765,45661 +45661,45661 +53766,45662 +45662,45662 +53767,45663 +45663,45663 +53768,45664 +45664,45664 +53769,45665 +45665,45665 +53770,45666 +45666,45666 +59399,45666 +53771,45667 +45667,45667 +53772,45668 +45668,45668 +53773,45669 +45669,45669 +53774,45670 +45670,45670 +53775,45671 +45671,45671 +53776,45672 +45672,45672 +53777,45673 +45673,45673 +53778,45674 +45674,45674 +53779,45675 +45675,45675 +110913,45675 +53756,45675 +45652,45675 +53780,45676 +45676,45676 +53781,45677 +45677,45677 +53782,45678 +45678,45678 +122988,45679 +53783,45679 +45679,45679 +53784,45680 +45680,45680 +53785,45681 +45681,45681 +53786,45682 +45682,45682 +53787,45683 +45683,45683 +53788,45684 +45684,45684 +53789,45685 +45685,45685 +53790,45686 +45686,45686 +53791,45687 +45687,45687 +53792,45688 +45688,45688 +53793,45689 +45689,45689 +53794,45690 +45690,45690 +53795,45691 +45691,45691 +53796,45692 +45692,45692 +53797,45693 +45693,45693 +53798,45694 +45694,45694 +22716,45694 +53799,45695 +45695,45695 +53800,45696 +45696,45696 +53801,45697 +45697,45697 +53802,45698 +45698,45698 +53803,45699 +45699,45699 +53804,45700 +45700,45700 +53843,45739 +45739,45739 +53844,45740 +45740,45740 +53845,45741 +45741,45741 +53846,45742 +45742,45742 +53847,45743 +45743,45743 +53848,45744 +45744,45744 +87351,45744 +53849,45745 +45745,45745 +53850,45746 +45746,45746 +53851,45747 +45747,45747 +53852,45748 +45748,45748 +53853,45749 +45749,45749 +53854,45750 +45750,45750 +53855,45751 +45751,45751 +53856,45752 +45752,45752 +53857,45753 +45753,45753 +53858,45754 +45754,45754 +53859,45755 +45755,45755 +53860,45756 +45756,45756 +53861,45757 +45757,45757 +53862,45758 +45758,45758 +53863,45759 +45759,45759 +53864,45760 +45760,45760 +53865,45761 +45761,45761 +53866,45762 +45762,45762 +53867,45763 +45763,45763 +10149,45763 +53868,45764 +45764,45764 +53869,45765 +45765,45765 +53870,45766 +45766,45766 +53871,45767 +45767,45767 +53872,45768 +45768,45768 +53873,45769 +45769,45769 +53874,45770 +45770,45770 +53875,45771 +45771,45771 +53876,45772 +45772,45772 +53877,45773 +45773,45773 +53878,45774 +45774,45774 +53879,45775 +45775,45775 +53880,45776 +45776,45776 +53881,45777 +45777,45777 +53882,45778 +45778,45778 +53883,45779 +45779,45779 +53884,45780 +45780,45780 +53885,45781 +45781,45781 +53886,45782 +45782,45782 +53887,45783 +45783,45783 +53888,45784 +45784,45784 +193,45784 +53889,45785 +45785,45785 +53890,45786 +45786,45786 +53891,45787 +45787,45787 +53892,45788 +45788,45788 +53893,45789 +45789,45789 +53894,45790 +45790,45790 +53895,45791 +45791,45791 +53896,45792 +45792,45792 +53897,45793 +45793,45793 +53898,45794 +45794,45794 +53899,45795 +45795,45795 +53900,45796 +45796,45796 +53901,45797 +45797,45797 +53902,45798 +45798,45798 +53903,45799 +45799,45799 +53904,45800 +45800,45800 +53905,45801 +45801,45801 +53906,45802 +45802,45802 +53907,45803 +45803,45803 +53908,45804 +45804,45804 +53909,45805 +45805,45805 +53910,45806 +45806,45806 +53911,45807 +45807,45807 +53912,45808 +45808,45808 +53913,45809 +45809,45809 +53914,45810 +45810,45810 +53915,45811 +45811,45811 +53916,45812 +45812,45812 +53917,45813 +45813,45813 +53918,45814 +45814,45814 +53919,45815 +45815,45815 +60256,45815 +53920,45816 +45816,45816 +53921,45817 +45817,45817 +247358,45817 +53922,45818 +45818,45818 +53923,45819 +45819,45819 +53924,45820 +45820,45820 +54170,46066 +46066,46066 +54171,46067 +46067,46067 +54172,46068 +46068,46068 +55071,46068 +46967,46068 +54173,46069 +46069,46069 +54174,46070 +46070,46070 +185921,46070 +181319,46070 +56337,46071 +55074,46071 +54175,46071 +48233,46071 +46970,46071 +46071,46071 +56338,46072 +55075,46072 +54769,46072 +54176,46072 +48234,46072 +46971,46072 +46665,46072 +46072,46072 +54177,46073 +46073,46073 +56339,46073 +55076,46073 +48235,46073 +46972,46073 +56340,46074 +55077,46074 +54771,46074 +54178,46074 +48236,46074 +46973,46074 +46667,46074 +46074,46074 +54179,46075 +46075,46075 +55078,46075 +54772,46075 +46974,46075 +46668,46075 +54252,46148 +46148,46148 +54845,46148 +46741,46148 +54253,46149 +46149,46149 +54846,46149 +46742,46149 +54254,46150 +46150,46150 +54847,46150 +46743,46150 +54848,46151 +54255,46151 +46744,46151 +46151,46151 +54256,46152 +46152,46152 +54257,46153 +46153,46153 +54298,46194 +46194,46194 +54299,46195 +46195,46195 +54300,46196 +46196,46196 +54301,46197 +46197,46197 +54302,46198 +46198,46198 +54303,46199 +46199,46199 +54399,46295 +46295,46295 +54400,46296 +46296,46296 +54401,46297 +46297,46297 +54555,46451 +46451,46451 +54556,46452 +46452,46452 +54557,46453 +46453,46453 +54558,46454 +46454,46454 +54559,46455 +46455,46455 +54560,46456 +46456,46456 +54561,46457 +46457,46457 +67289,46457 +54562,46458 +46458,46458 +54563,46459 +46459,46459 +54564,46460 +46460,46460 +54565,46461 +46461,46461 +243441,46461 +54566,46462 +46462,46462 +54567,46463 +46463,46463 +162810,46463 +54568,46464 +46464,46464 +54569,46465 +46465,46465 +54570,46466 +46466,46466 +54571,46467 +46467,46467 +54572,46468 +46468,46468 +54573,46469 +46469,46469 +54574,46470 +46470,46470 +54575,46471 +46471,46471 +54576,46472 +46472,46472 +54577,46473 +46473,46473 +54578,46474 +46474,46474 +54579,46475 +46475,46475 +235084,46475 +54580,46476 +46476,46476 +54581,46477 +46477,46477 +54582,46478 +46478,46478 +54583,46479 +46479,46479 +54584,46480 +46480,46480 +54585,46481 +46481,46481 +50350,46481 +42158,46481 +54586,46482 +46482,46482 +49509,46482 +41316,46482 +54587,46483 +46483,46483 +54588,46484 +46484,46484 +54589,46485 +46485,46485 +54590,46486 +46486,46486 +54591,46487 +46487,46487 +54592,46488 +46488,46488 +54593,46489 +46489,46489 +54594,46490 +46490,46490 +249443,46490 +54595,46491 +46491,46491 +54596,46492 +46492,46492 +54597,46493 +46493,46493 +54598,46494 +46494,46494 +54599,46495 +46495,46495 +54600,46496 +46496,46496 +54601,46497 +46497,46497 +54602,46498 +46498,46498 +54603,46499 +46499,46499 +54604,46500 +46500,46500 +54605,46501 +46501,46501 +54606,46502 +46502,46502 +54607,46503 +46503,46503 +54608,46504 +46504,46504 +54609,46505 +46505,46505 +54610,46506 +46506,46506 +54611,46507 +46507,46507 +54612,46508 +46508,46508 +54613,46509 +46509,46509 +54614,46510 +46510,46510 +54615,46511 +46511,46511 +54616,46512 +46512,46512 +11628,46512 +88639,46512 +54617,46513 +46513,46513 +54618,46514 +46514,46514 +242764,46514 +54619,46515 +46515,46515 +54620,46516 +46516,46516 +54621,46517 +46517,46517 +214531,46517 +54622,46518 +46518,46518 +54623,46519 +46519,46519 +54624,46520 +46520,46520 +54660,46556 +46556,46556 +54661,46557 +46557,46557 +54662,46558 +46558,46558 +54663,46559 +46559,46559 +54664,46560 +46560,46560 +54665,46561 +46561,46561 +107053,46561 +54666,46562 +46562,46562 +54667,46563 +46563,46563 +54668,46564 +46564,46564 +54669,46565 +46565,46565 +54670,46566 +46566,46566 +54671,46567 +46567,46567 +54672,46568 +46568,46568 +54673,46569 +46569,46569 +100664,46569 +54674,46570 +46570,46570 +54675,46571 +46571,46571 +54676,46572 +46572,46572 +54677,46573 +46573,46573 +54678,46574 +46574,46574 +54679,46575 +46575,46575 +4797,46575 +54680,46576 +46576,46576 +107651,46576 +54681,46577 +46577,46577 +54682,46578 +46578,46578 +54683,46579 +46579,46579 +54684,46580 +46580,46580 +54685,46581 +46581,46581 +54686,46582 +46582,46582 +243878,46582 +54687,46583 +46583,46583 +54688,46584 +46584,46584 +54689,46585 +46585,46585 +54690,46586 +46586,46586 +54691,46587 +46587,46587 +142606,46587 +54692,46588 +46588,46588 +76039,46588 +54693,46589 +46589,46589 +185456,46589 +54694,46590 +46590,46590 +54695,46591 +46591,46591 +54696,46592 +46592,46592 +54697,46593 +46593,46593 +54698,46594 +46594,46594 +54699,46595 +46595,46595 +54700,46596 +46596,46596 +54701,46597 +46597,46597 +54702,46598 +46598,46598 +54703,46599 +46599,46599 +54704,46600 +46600,46600 +54705,46601 +46601,46601 +54706,46602 +46602,46602 +54707,46603 +46603,46603 +54708,46604 +46604,46604 +54709,46605 +46605,46605 +54710,46606 +46606,46606 +54711,46607 +46607,46607 +57250,46607 +54712,46608 +46608,46608 +54713,46609 +46609,46609 +54714,46610 +46610,46610 +141264,46610 +54715,46611 +46611,46611 +54716,46612 +46612,46612 +162993,46612 +54717,46613 +46613,46613 +54718,46614 +46614,46614 +247804,46614 +54719,46615 +46615,46615 +54720,46616 +46616,46616 +54721,46617 +46617,46617 +54722,46618 +46618,46618 +54723,46619 +46619,46619 +54724,46620 +46620,46620 +54725,46621 +46621,46621 +54726,46622 +46622,46622 +54727,46623 +46623,46623 +244104,46623 +54728,46624 +46624,46624 +54729,46625 +46625,46625 +54730,46626 +46626,46626 +54731,46627 +46627,46627 +54732,46628 +46628,46628 +54733,46629 +46629,46629 +54734,46630 +46630,46630 +54735,46631 +46631,46631 +54736,46632 +46632,46632 +54737,46633 +46633,46633 +54738,46634 +46634,46634 +54739,46635 +46635,46635 +54740,46636 +46636,46636 +54741,46637 +46637,46637 +54742,46638 +46638,46638 +54743,46639 +46639,46639 +54744,46640 +46640,46640 +201394,46640 +54745,46641 +46641,46641 +54746,46642 +46642,46642 +54747,46643 +46643,46643 +54748,46644 +46644,46644 +54749,46645 +46645,46645 +54750,46646 +46646,46646 +54751,46647 +46647,46647 +54752,46648 +46648,46648 +54753,46649 +46649,46649 +54754,46650 +46650,46650 +84188,46650 +54755,46651 +46651,46651 +54756,46652 +46652,46652 +15421,46652 +54757,46653 +46653,46653 +54758,46654 +46654,46654 +54759,46655 +46655,46655 +54760,46656 +46656,46656 +54761,46657 +46657,46657 +54762,46658 +46658,46658 +54763,46659 +46659,46659 +54764,46660 +46660,46660 +54765,46661 +46661,46661 +54766,46662 +46662,46662 +56336,46663 +55073,46663 +54767,46663 +48232,46663 +46969,46663 +46663,46663 +54768,46664 +46664,46664 +56338,46665 +55075,46665 +54769,46665 +54176,46665 +48234,46665 +46971,46665 +46665,46665 +46072,46665 +54770,46666 +46666,46666 +56340,46667 +55077,46667 +54771,46667 +54178,46667 +48236,46667 +46973,46667 +46667,46667 +46074,46667 +55078,46668 +54772,46668 +46974,46668 +46668,46668 +56341,46668 +48237,46668 +54179,46668 +46075,46668 +54773,46669 +46669,46669 +54774,46670 +46670,46670 +54775,46671 +46671,46671 +54776,46672 +46672,46672 +54777,46673 +46673,46673 +54778,46674 +46674,46674 +55084,46674 +46980,46674 +54779,46675 +46675,46675 +54780,46676 +46676,46676 +54781,46677 +46677,46677 +54782,46678 +46678,46678 +54783,46679 +46679,46679 +54784,46680 +46680,46680 +54785,46681 +46681,46681 +54786,46682 +46682,46682 +54787,46683 +46683,46683 +54788,46684 +46684,46684 +54789,46685 +46685,46685 +54790,46686 +46686,46686 +54791,46687 +46687,46687 +54844,46740 +46740,46740 +54845,46741 +46741,46741 +54252,46741 +46148,46741 +54846,46742 +46742,46742 +54253,46742 +46149,46742 +54847,46743 +46743,46743 +54254,46743 +46150,46743 +54848,46744 +54255,46744 +46744,46744 +46151,46744 +54964,46860 +46860,46860 +54965,46861 +46861,46861 +54966,46862 +46862,46862 +54967,46863 +46863,46863 +54968,46864 +46864,46864 +55930,46864 +47826,46864 +54969,46865 +46865,46865 +54970,46866 +46866,46866 +54971,46867 +46867,46867 +54972,46868 +46868,46868 +39407,46868 +54973,46869 +46869,46869 +54974,46870 +46870,46870 +54975,46871 +46871,46871 +54976,46872 +46872,46872 +54977,46873 +46873,46873 +54978,46874 +46874,46874 +54979,46875 +46875,46875 +16263,46875 +54980,46876 +46876,46876 +54981,46877 +46877,46877 +54982,46878 +46878,46878 +54983,46879 +46879,46879 +62607,46879 +54984,46880 +46880,46880 +124074,46881 +54985,46881 +46881,46881 +54986,46882 +46882,46882 +132408,46882 +124821,46882 +54987,46883 +46883,46883 +80450,46883 +67182,46883 +229575,46883 +211758,46884 +54988,46884 +46884,46884 +54989,46885 +46885,46885 +54990,46886 +46886,46886 +54991,46887 +46887,46887 +54992,46888 +46888,46888 +54993,46889 +46889,46889 +54994,46890 +46890,46890 +54995,46891 +46891,46891 +54996,46892 +46892,46892 +54997,46893 +46893,46893 +54998,46894 +46894,46894 +54999,46895 +46895,46895 +55000,46896 +46896,46896 +56111,46896 +48007,46896 +55001,46897 +46897,46897 +55002,46898 +46898,46898 +214190,46899 +55003,46899 +46899,46899 +55004,46900 +46900,46900 +56115,46900 +48011,46900 +55005,46901 +46901,46901 +55006,46902 +46902,46902 +55007,46903 +46903,46903 +56269,46903 +48165,46903 +55008,46904 +46904,46904 +55009,46905 +46905,46905 +55010,46906 +46906,46906 +55011,46907 +46907,46907 +56122,46907 +48018,46907 +55012,46908 +46908,46908 +56123,46908 +48019,46908 +56274,46908 +48170,46908 +55013,46909 +46909,46909 +57726,46909 +55014,46910 +46910,46910 +55015,46911 +46911,46911 +56277,46911 +48173,46911 +56126,46911 +48022,46911 +55016,46912 +46912,46912 +56127,46912 +48023,46912 +55017,46913 +46913,46913 +55018,46914 +46914,46914 +56129,46914 +48025,46914 +55019,46915 +46915,46915 +56281,46915 +48177,46915 +55020,46916 +46916,46916 +56131,46916 +48027,46916 +55021,46917 +46917,46917 +56283,46917 +56132,46917 +48179,46917 +48028,46917 +55022,46918 +46918,46918 +56284,46918 +48180,46918 +23506,46918 +55023,46919 +46919,46919 +55024,46920 +46920,46920 +55025,46921 +46921,46921 +55026,46922 +46922,46922 +55027,46923 +46923,46923 +55028,46924 +46924,46924 +55029,46925 +46925,46925 +4033,46925 +162536,46925 +55030,46926 +46926,46926 +55066,46962 +46962,46962 +55067,46963 +46963,46963 +55068,46964 +46964,46964 +55069,46965 +46965,46965 +55070,46966 +46966,46966 +55071,46967 +46967,46967 +54172,46967 +46068,46967 +56335,46968 +55072,46968 +48231,46968 +46968,46968 +56336,46969 +55073,46969 +54767,46969 +48232,46969 +46969,46969 +46663,46969 +56337,46970 +55074,46970 +54175,46970 +48233,46970 +46970,46970 +46071,46970 +56338,46971 +55075,46971 +54769,46971 +54176,46971 +48234,46971 +46971,46971 +46665,46971 +46072,46971 +56339,46972 +55076,46972 +48235,46972 +46972,46972 +54177,46972 +46073,46972 +56340,46973 +55077,46973 +54771,46973 +54178,46973 +48236,46973 +46973,46973 +46667,46973 +46074,46973 +55078,46974 +54772,46974 +46974,46974 +46668,46974 +56341,46974 +48237,46974 +54179,46974 +46075,46974 +55079,46975 +46975,46975 +145325,46975 +55080,46976 +46976,46976 +55081,46977 +46977,46977 +55082,46978 +46978,46978 +55083,46979 +46979,46979 +55084,46980 +46980,46980 +54778,46980 +46674,46980 +55085,46981 +46981,46981 +55086,46982 +46982,46982 +9193,46982 +55087,46983 +46983,46983 +55088,46984 +46984,46984 +55089,46985 +46985,46985 +55090,46986 +46986,46986 +55091,46987 +46987,46987 +55092,46988 +46988,46988 +55093,46989 +46989,46989 +75811,46989 +55094,46990 +46990,46990 +55095,46991 +46991,46991 +55096,46992 +46992,46992 +110858,46992 +55097,46993 +46993,46993 +55098,46994 +46994,46994 +55099,46995 +46995,46995 +55100,46996 +46996,46996 +55101,46997 +46997,46997 +55102,46998 +46998,46998 +55103,46999 +46999,46999 +55104,47000 +47000,47000 +55105,47001 +47001,47001 +55106,47002 +47002,47002 +55107,47003 +47003,47003 +94487,47003 +55108,47004 +47004,47004 +55109,47005 +47005,47005 +55110,47006 +47006,47006 +90131,47006 +55111,47007 +47007,47007 +55112,47008 +47008,47008 +55113,47009 +47009,47009 +55114,47010 +47010,47010 +55232,47128 +47128,47128 +55233,47129 +47129,47129 +55287,47183 +47183,47183 +55288,47184 +47184,47184 +55289,47185 +47185,47185 +104654,47185 +55290,47186 +47186,47186 +84827,47186 +55291,47187 +47187,47187 +55292,47188 +47188,47188 +55293,47189 +47189,47189 +224769,47190 +55294,47190 +47190,47190 +55295,47191 +47191,47191 +55296,47192 +47192,47192 +55297,47193 +47193,47193 +55298,47194 +47194,47194 +55299,47195 +47195,47195 +55300,47196 +47196,47196 +55301,47197 +47197,47197 +55302,47198 +47198,47198 +55303,47199 +47199,47199 +55304,47200 +47200,47200 +55305,47201 +47201,47201 +55306,47202 +47202,47202 +55349,47245 +47245,47245 +55350,47246 +47246,47246 +55351,47247 +47247,47247 +55352,47248 +47248,47248 +55353,47249 +47249,47249 +55354,47250 +47250,47250 +55355,47251 +47251,47251 +55356,47252 +47252,47252 +55357,47253 +47253,47253 +55358,47254 +47254,47254 +55359,47255 +47255,47255 +55360,47256 +47256,47256 +55361,47257 +47257,47257 +55362,47258 +47258,47258 +55363,47259 +47259,47259 +55364,47260 +47260,47260 +55365,47261 +47261,47261 +55366,47262 +47262,47262 +55367,47263 +47263,47263 +55368,47264 +47264,47264 +55369,47265 +47265,47265 +55370,47266 +47266,47266 +55371,47267 +47267,47267 +55372,47268 +47268,47268 +55373,47269 +47269,47269 +55374,47270 +47270,47270 +55375,47271 +47271,47271 +55376,47272 +47272,47272 +55377,47273 +47273,47273 +55378,47274 +47274,47274 +55379,47275 +47275,47275 +55380,47276 +47276,47276 +55381,47277 +47277,47277 +55382,47278 +47278,47278 +55383,47279 +47279,47279 +55384,47280 +47280,47280 +55385,47281 +47281,47281 +55386,47282 +47282,47282 +55387,47283 +47283,47283 +55388,47284 +47284,47284 +55389,47285 +47285,47285 +55390,47286 +47286,47286 +55391,47287 +47287,47287 +55392,47288 +47288,47288 +55393,47289 +47289,47289 +55394,47290 +47290,47290 +55395,47291 +47291,47291 +55396,47292 +47292,47292 +55397,47293 +47293,47293 +55398,47294 +47294,47294 +55399,47295 +47295,47295 +55400,47296 +47296,47296 +55401,47297 +47297,47297 +55402,47298 +47298,47298 +55403,47299 +47299,47299 +55404,47300 +47300,47300 +55405,47301 +47301,47301 +79551,47301 +55406,47302 +47302,47302 +2119,47302 +55407,47303 +47303,47303 +55408,47304 +47304,47304 +55409,47305 +47305,47305 +55410,47306 +47306,47306 +17197,47306 +55411,47307 +47307,47307 +55412,47308 +47308,47308 +55413,47309 +47309,47309 +55414,47310 +47310,47310 +55415,47311 +47311,47311 +55416,47312 +47312,47312 +55417,47313 +47313,47313 +55418,47314 +47314,47314 +55419,47315 +47315,47315 +55420,47316 +47316,47316 +55421,47317 +47317,47317 +55422,47318 +47318,47318 +55423,47319 +47319,47319 +55424,47320 +47320,47320 +55425,47321 +47321,47321 +55426,47322 +47322,47322 +55427,47323 +47323,47323 +55428,47324 +47324,47324 +76353,47324 +55429,47325 +47325,47325 +55430,47326 +47326,47326 +55431,47327 +47327,47327 +55432,47328 +47328,47328 +55433,47329 +47329,47329 +55434,47330 +47330,47330 +55435,47331 +47331,47331 +55436,47332 +47332,47332 +55437,47333 +47333,47333 +55536,47432 +47432,47432 +55537,47433 +47433,47433 +55538,47434 +47434,47434 +55539,47435 +47435,47435 +55540,47436 +47436,47436 +55541,47437 +47437,47437 +55542,47438 +47438,47438 +55543,47439 +47439,47439 +55544,47440 +47440,47440 +55545,47441 +47441,47441 +55546,47442 +47442,47442 +55547,47443 +47443,47443 +55548,47444 +47444,47444 +55549,47445 +47445,47445 +55550,47446 +47446,47446 +55551,47447 +47447,47447 +55552,47448 +47448,47448 +55553,47449 +47449,47449 +55554,47450 +47450,47450 +55555,47451 +47451,47451 +55556,47452 +47452,47452 +55557,47453 +47453,47453 +55558,47454 +47454,47454 +55559,47455 +47455,47455 +55560,47456 +47456,47456 +183606,47456 +55561,47457 +47457,47457 +55562,47458 +47458,47458 +55563,47459 +47459,47459 +20916,47459 +55564,47460 +47460,47460 +55565,47461 +47461,47461 +55566,47462 +47462,47462 +55567,47463 +47463,47463 +55568,47464 +47464,47464 +224606,47464 +55569,47465 +47465,47465 +55570,47466 +47466,47466 +55571,47467 +47467,47467 +141859,47467 +55572,47468 +47468,47468 +55573,47469 +47469,47469 +55574,47470 +47470,47470 +55575,47471 +47471,47471 +55576,47472 +47472,47472 +101031,47472 +55577,47473 +47473,47473 +55578,47474 +47474,47474 +55579,47475 +47475,47475 +55580,47476 +47476,47476 +55581,47477 +47477,47477 +55582,47478 +47478,47478 +55583,47479 +47479,47479 +30236,47479 +55584,47480 +47480,47480 +55585,47481 +47481,47481 +55586,47482 +47482,47482 +55587,47483 +47483,47483 +55588,47484 +47484,47484 +55589,47485 +47485,47485 +55590,47486 +47486,47486 +55591,47487 +47487,47487 +55592,47488 +47488,47488 +55593,47489 +47489,47489 +55594,47490 +47490,47490 +55595,47491 +47491,47491 +55596,47492 +47492,47492 +55597,47493 +47493,47493 +55598,47494 +47494,47494 +55599,47495 +47495,47495 +55600,47496 +47496,47496 +55601,47497 +47497,47497 +55602,47498 +47498,47498 +55603,47499 +47499,47499 +55604,47500 +47500,47500 +55605,47501 +47501,47501 +55606,47502 +47502,47502 +55607,47503 +47503,47503 +55608,47504 +47504,47504 +55609,47505 +47505,47505 +55647,47543 +47543,47543 +240934,47543 +55648,47544 +47544,47544 +55649,47545 +47545,47545 +55650,47546 +47546,47546 +55651,47547 +47547,47547 +55652,47548 +47548,47548 +55653,47549 +47549,47549 +55654,47550 +47550,47550 +55655,47551 +47551,47551 +111516,47551 +55656,47552 +47552,47552 +55657,47553 +47553,47553 +55658,47554 +47554,47554 +55659,47555 +47555,47555 +55660,47556 +47556,47556 +55661,47557 +47557,47557 +55662,47558 +47558,47558 +55663,47559 +47559,47559 +55664,47560 +47560,47560 +23858,47560 +55665,47561 +47561,47561 +55666,47562 +47562,47562 +55667,47563 +47563,47563 +19050,47563 +55668,47564 +47564,47564 +19051,47564 +55669,47565 +47565,47565 +55670,47566 +47566,47566 +55671,47567 +47567,47567 +55672,47568 +47568,47568 +55673,47569 +47569,47569 +55674,47570 +47570,47570 +55675,47571 +47571,47571 +194290,47571 +55676,47572 +47572,47572 +55677,47573 +47573,47573 +55678,47574 +47574,47574 +55679,47575 +47575,47575 +55680,47576 +47576,47576 +55681,47577 +47577,47577 +55682,47578 +47578,47578 +55683,47579 +47579,47579 +55684,47580 +47580,47580 +55685,47581 +47581,47581 +55686,47582 +47582,47582 +55687,47583 +47583,47583 +55688,47584 +47584,47584 +55689,47585 +47585,47585 +55690,47586 +47586,47586 +55691,47587 +47587,47587 +55692,47588 +47588,47588 +55693,47589 +47589,47589 +55694,47590 +47590,47590 +55730,47626 +47626,47626 +55731,47627 +47627,47627 +55732,47628 +47628,47628 +55733,47629 +47629,47629 +55734,47630 +47630,47630 +55735,47631 +47631,47631 +55736,47632 +47632,47632 +55737,47633 +47633,47633 +55738,47634 +47634,47634 +55739,47635 +47635,47635 +55740,47636 +47636,47636 +55741,47637 +47637,47637 +55742,47638 +47638,47638 +55743,47639 +47639,47639 +55744,47640 +47640,47640 +55745,47641 +47641,47641 +55746,47642 +47642,47642 +55747,47643 +47643,47643 +55748,47644 +47644,47644 +55749,47645 +47645,47645 +55750,47646 +47646,47646 +55751,47647 +47647,47647 +55752,47648 +47648,47648 +55753,47649 +47649,47649 +55754,47650 +47650,47650 +55755,47651 +47651,47651 +55756,47652 +47652,47652 +55757,47653 +47653,47653 +55758,47654 +47654,47654 +55759,47655 +47655,47655 +9829,47655 +55760,47656 +47656,47656 +152960,47656 +55761,47657 +47657,47657 +55762,47658 +47658,47658 +55763,47659 +47659,47659 +206553,47659 +55830,47726 +47726,47726 +55831,47727 +47727,47727 +55832,47728 +47728,47728 +55833,47729 +47729,47729 +55834,47730 +47730,47730 +55835,47731 +47731,47731 +55836,47732 +47732,47732 +35983,47732 +55837,47733 +47733,47733 +55838,47734 +47734,47734 +55839,47735 +47735,47735 +55840,47736 +47736,47736 +55841,47737 +47737,47737 +123518,47737 +55842,47738 +47738,47738 +55843,47739 +47739,47739 +55926,47822 +47822,47822 +50148,47822 +55927,47823 +47823,47823 +55928,47824 +47824,47824 +41958,47824 +55929,47825 +47825,47825 +55930,47826 +47826,47826 +54968,47826 +46864,47826 +55931,47827 +47827,47827 +55932,47828 +47828,47828 +135268,47828 +127681,47828 +55933,47829 +47829,47829 +55934,47830 +47830,47830 +55935,47831 +47831,47831 +55936,47832 +47832,47832 +55937,47833 +47833,47833 +55938,47834 +47834,47834 +55939,47835 +47835,47835 +55940,47836 +47836,47836 +248132,47836 +55941,47837 +47837,47837 +55942,47838 +47838,47838 +14576,47838 +55943,47839 +47839,47839 +55944,47840 +47840,47840 +55945,47841 +47841,47841 +55946,47842 +47842,47842 +55947,47843 +47843,47843 +131927,47843 +124340,47843 +55948,47844 +47844,47844 +55949,47845 +47845,47845 +55950,47846 +47846,47846 +187467,47846 +55951,47847 +47847,47847 +55952,47848 +47848,47848 +55953,47849 +47849,47849 +55954,47850 +47850,47850 +55955,47851 +47851,47851 +55956,47852 +47852,47852 +55957,47853 +47853,47853 +116372,47853 +55958,47854 +47854,47854 +55959,47855 +47855,47855 +56108,47855 +48004,47855 +55960,47856 +47856,47856 +55961,47857 +47857,47857 +55962,47858 +47858,47858 +55963,47859 +47859,47859 +55964,47860 +47860,47860 +55965,47861 +47861,47861 +55966,47862 +47862,47862 +55967,47863 +47863,47863 +55968,47864 +47864,47864 +55969,47865 +47865,47865 +55970,47866 +47866,47866 +56271,47867 +55971,47867 +48167,47867 +47867,47867 +55972,47868 +47868,47868 +55973,47869 +47869,47869 +61998,47870 +55974,47870 +47870,47870 +55975,47871 +47871,47871 +79171,47871 +56078,47974 +47974,47974 +133187,47974 +125600,47974 +56079,47975 +47975,47975 +56080,47976 +47976,47976 +143283,47977 +56081,47977 +47977,47977 +56082,47978 +47978,47978 +56083,47979 +47979,47979 +56084,47980 +47980,47980 +56085,47981 +47981,47981 +56086,47982 +47982,47982 +56087,47983 +47983,47983 +56088,47984 +47984,47984 +56089,47985 +47985,47985 +56090,47986 +47986,47986 +56091,47987 +47987,47987 +56092,47988 +47988,47988 +56093,47989 +47989,47989 +56094,47990 +47990,47990 +56095,47991 +47991,47991 +111667,47991 +56096,47992 +47992,47992 +56097,47993 +47993,47993 +56098,47994 +47994,47994 +56099,47995 +47995,47995 +56100,47996 +47996,47996 +56101,47997 +47997,47997 +56102,47998 +47998,47998 +56103,47999 +47999,47999 +56104,48000 +48000,48000 +62098,48000 +56105,48001 +48001,48001 +56106,48002 +48002,48002 +116381,48002 +56107,48003 +48003,48003 +56108,48004 +48004,48004 +55959,48004 +47855,48004 +56109,48005 +48005,48005 +56110,48006 +48006,48006 +56111,48007 +48007,48007 +55000,48007 +46896,48007 +56112,48008 +48008,48008 +56113,48009 +48009,48009 +56114,48010 +48010,48010 +56265,48010 +48161,48010 +56115,48011 +48011,48011 +55004,48011 +46900,48011 +56266,48011 +48162,48011 +56116,48012 +48012,48012 +56268,48013 +56117,48013 +48164,48013 +48013,48013 +56118,48014 +48014,48014 +28453,48014 +151452,48014 +56119,48015 +48015,48015 +56120,48016 +48016,48016 +56121,48017 +48017,48017 +56272,48017 +48168,48017 +56122,48018 +48018,48018 +55011,48018 +46907,48018 +56273,48018 +48169,48018 +227755,48018 +56123,48019 +48019,48019 +55012,48019 +46908,48019 +56124,48020 +48020,48020 +56125,48021 +48021,48021 +56126,48022 +48022,48022 +55015,48022 +46911,48022 +56127,48023 +48023,48023 +56278,48023 +48174,48023 +55016,48023 +46912,48023 +56128,48024 +48024,48024 +56279,48024 +48175,48024 +56129,48025 +48025,48025 +56280,48025 +48176,48025 +55018,48025 +46914,48025 +56130,48026 +48026,48026 +56281,48026 +48177,48026 +56131,48027 +48027,48027 +56282,48027 +48178,48027 +55020,48027 +46916,48027 +56283,48028 +56132,48028 +48179,48028 +48028,48028 +55021,48028 +46917,48028 +56133,48029 +48029,48029 +56284,48029 +48180,48029 +56134,48030 +48030,48030 +56135,48031 +48031,48031 +56136,48032 +48032,48032 +56137,48033 +48033,48033 +56138,48034 +48034,48034 +56139,48035 +48035,48035 +56140,48036 +48036,48036 +56141,48037 +48037,48037 +56142,48038 +48038,48038 +56143,48039 +48039,48039 +56294,48039 +48190,48039 +56144,48040 +48040,48040 +56145,48041 +48041,48041 +56146,48042 +48042,48042 +56147,48043 +48043,48043 +56148,48044 +48044,48044 +56149,48045 +48045,48045 +56150,48046 +48046,48046 +56151,48047 +48047,48047 +56152,48048 +48048,48048 +56153,48049 +48049,48049 +88500,48049 +56154,48050 +48050,48050 +56155,48051 +48051,48051 +56156,48052 +48052,48052 +56157,48053 +48053,48053 +56158,48054 +48054,48054 +56159,48055 +48055,48055 +56160,48056 +48056,48056 +56161,48057 +48057,48057 +56162,48058 +48058,48058 +56163,48059 +48059,48059 +158605,48059 +56164,48060 +48060,48060 +56165,48061 +48061,48061 +56166,48062 +48062,48062 +56167,48063 +48063,48063 +56168,48064 +48064,48064 +56169,48065 +48065,48065 +56170,48066 +48066,48066 +56171,48067 +48067,48067 +56172,48068 +48068,48068 +56173,48069 +48069,48069 +56174,48070 +48070,48070 +56175,48071 +48071,48071 +56176,48072 +48072,48072 +56177,48073 +48073,48073 +56178,48074 +48074,48074 +56179,48075 +48075,48075 +56180,48076 +48076,48076 +13673,48076 +56181,48077 +48077,48077 +135677,48077 +128090,48077 +56182,48078 +48078,48078 +14418,48078 +56183,48079 +48079,48079 +56184,48080 +48080,48080 +56185,48081 +48081,48081 +56186,48082 +48082,48082 +56187,48083 +48083,48083 +56188,48084 +48084,48084 +56189,48085 +48085,48085 +56190,48086 +48086,48086 +56191,48087 +48087,48087 +56192,48088 +48088,48088 +56193,48089 +48089,48089 +56194,48090 +48090,48090 +56195,48091 +48091,48091 +56196,48092 +48092,48092 +56197,48093 +48093,48093 +56198,48094 +48094,48094 +56199,48095 +48095,48095 +56200,48096 +48096,48096 +56201,48097 +48097,48097 +56202,48098 +48098,48098 +56203,48099 +48099,48099 +79897,48099 +56204,48100 +48100,48100 +56205,48101 +48101,48101 +56206,48102 +48102,48102 +56207,48103 +48103,48103 +56208,48104 +48104,48104 +56209,48105 +48105,48105 +56210,48106 +48106,48106 +56211,48107 +48107,48107 +56212,48108 +48108,48108 +56213,48109 +48109,48109 +56214,48110 +48110,48110 +56215,48111 +48111,48111 +180290,48112 +56216,48112 +48112,48112 +56217,48113 +48113,48113 +56218,48114 +48114,48114 +56219,48115 +48115,48115 +115470,48115 +56220,48116 +48116,48116 +210213,48116 +141650,48116 +56221,48117 +48117,48117 +56222,48118 +48118,48118 +59104,48118 +58730,48118 +56223,48119 +48119,48119 +56224,48120 +48120,48120 +56225,48121 +48121,48121 +56226,48122 +48122,48122 +65857,48122 +56227,48123 +48123,48123 +56228,48124 +48124,48124 +56229,48125 +48125,48125 +56230,48126 +48126,48126 +56231,48127 +48127,48127 +56232,48128 +48128,48128 +56233,48129 +48129,48129 +68615,48129 +56234,48130 +48130,48130 +56235,48131 +48131,48131 +56236,48132 +48132,48132 +56237,48133 +48133,48133 +33622,48133 +56238,48134 +48134,48134 +56239,48135 +48135,48135 +56240,48136 +48136,48136 +56241,48137 +48137,48137 +56242,48138 +48138,48138 +56243,48139 +48139,48139 +56244,48140 +48140,48140 +56245,48141 +48141,48141 +56246,48142 +48142,48142 +56247,48143 +48143,48143 +236535,48143 +56248,48144 +48144,48144 +56249,48145 +48145,48145 +56250,48146 +48146,48146 +150569,48146 +56251,48147 +48147,48147 +187468,48147 +56252,48148 +48148,48148 +56253,48149 +48149,48149 +56254,48150 +48150,48150 +56255,48151 +48151,48151 +56256,48152 +48152,48152 +56257,48153 +48153,48153 +56258,48154 +48154,48154 +56259,48155 +48155,48155 +56260,48156 +48156,48156 +56261,48157 +48157,48157 +4983,48157 +56262,48158 +48158,48158 +134200,48158 +126613,48158 +56263,48159 +48159,48159 +56264,48160 +48160,48160 +56265,48161 +48161,48161 +56114,48161 +48010,48161 +56266,48162 +48162,48162 +56115,48162 +48011,48162 +56267,48163 +48163,48163 +56268,48164 +56117,48164 +48164,48164 +48013,48164 +56269,48165 +48165,48165 +55007,48165 +46903,48165 +213603,48165 +56270,48166 +48166,48166 +56271,48167 +55971,48167 +48167,48167 +47867,48167 +56272,48168 +48168,48168 +56121,48168 +48017,48168 +56273,48169 +48169,48169 +56122,48169 +48018,48169 +56274,48170 +48170,48170 +55012,48170 +46908,48170 +56275,48171 +48171,48171 +56276,48172 +48172,48172 +56277,48173 +48173,48173 +55015,48173 +46911,48173 +56278,48174 +48174,48174 +56127,48174 +48023,48174 +56279,48175 +48175,48175 +56128,48175 +48024,48175 +56280,48176 +48176,48176 +56129,48176 +48025,48176 +56281,48177 +48177,48177 +56130,48177 +48026,48177 +55019,48177 +46915,48177 +96098,48177 +56282,48178 +48178,48178 +56131,48178 +48027,48178 +56283,48179 +56132,48179 +48179,48179 +48028,48179 +55021,48179 +46917,48179 +56284,48180 +48180,48180 +55022,48180 +46918,48180 +56133,48180 +48029,48180 +56285,48181 +48181,48181 +56286,48182 +48182,48182 +56287,48183 +48183,48183 +56288,48184 +48184,48184 +56289,48185 +48185,48185 +56290,48186 +48186,48186 +56291,48187 +48187,48187 +11850,48187 +56292,48188 +48188,48188 +56293,48189 +48189,48189 +56294,48190 +48190,48190 +56143,48190 +48039,48190 +56295,48191 +48191,48191 +56296,48192 +48192,48192 +56333,48229 +48229,48229 +56334,48230 +48230,48230 +56335,48231 +55072,48231 +48231,48231 +46968,48231 +56336,48232 +55073,48232 +54767,48232 +48232,48232 +46969,48232 +46663,48232 +56337,48233 +55074,48233 +54175,48233 +48233,48233 +46970,48233 +46071,48233 +56338,48234 +55075,48234 +54769,48234 +54176,48234 +48234,48234 +46971,48234 +46665,48234 +46072,48234 +56339,48235 +55076,48235 +48235,48235 +46972,48235 +54177,48235 +46073,48235 +56340,48236 +55077,48236 +54771,48236 +54178,48236 +48236,48236 +46973,48236 +46667,48236 +46074,48236 +56341,48237 +48237,48237 +55078,48237 +54772,48237 +46974,48237 +46668,48237 +56342,48238 +48238,48238 +56343,48239 +48239,48239 +56344,48240 +48240,48240 +56345,48241 +48241,48241 +56346,48242 +48242,48242 +56347,48243 +48243,48243 +56348,48244 +48244,48244 +56349,48245 +48245,48245 +56350,48246 +48246,48246 +56351,48247 +48247,48247 +56352,48248 +48248,48248 +56353,48249 +48249,48249 +56354,48250 +48250,48250 +56355,48251 +48251,48251 +56356,48252 +48252,48252 +66826,48252 +56357,48253 +48253,48253 +56358,48254 +48254,48254 +56359,48255 +48255,48255 +18539,48255 +14641,48255 +56360,48256 +48256,48256 +215600,48256 +56361,48257 +48257,48257 +56362,48258 +48258,48258 +56363,48259 +48259,48259 +226346,48259 +56364,48260 +48260,48260 +56365,48261 +48261,48261 +56366,48262 +48262,48262 +56407,48303 +48303,48303 +56408,48304 +48304,48304 +56449,48345 +48345,48345 +56450,48346 +48346,48346 +56451,48347 +48347,48347 +56452,48348 +48348,48348 +56453,48349 +48349,48349 +56454,48350 +48350,48350 +56455,48351 +48351,48351 +56456,48352 +48352,48352 +56457,48353 +48353,48353 +56458,48354 +48354,48354 +56459,48355 +48355,48355 +56542,48438 +48438,48438 +56543,48439 +48439,48439 +56544,48440 +48440,48440 +236144,48440 +2753,48440 +56545,48441 +48441,48441 +56546,48442 +48442,48442 +56547,48443 +48443,48443 +56548,48444 +48444,48444 +56549,48445 +48445,48445 +56550,48446 +48446,48446 +180978,48446 +56551,48447 +48447,48447 +56552,48448 +48448,48448 +56553,48449 +48449,48449 +56554,48450 +48450,48450 +56555,48451 +48451,48451 +56556,48452 +48452,48452 +56557,48453 +48453,48453 +56558,48454 +48454,48454 +56559,48455 +48455,48455 +56560,48456 +48456,48456 +56561,48457 +48457,48457 +56636,48532 +48532,48532 +161053,48532 +56637,48533 +48533,48533 +56638,48534 +48534,48534 +56639,48535 +48535,48535 +56640,48536 +48536,48536 +56641,48537 +48537,48537 +56642,48538 +48538,48538 +56643,48539 +48539,48539 +56644,48540 +48540,48540 +56645,48541 +48541,48541 +56646,48542 +48542,48542 +56647,48543 +48543,48543 +56648,48544 +48544,48544 +162644,48544 +56649,48545 +48545,48545 +140395,48545 +56650,48546 +48546,48546 +56651,48547 +48547,48547 +77537,48548 +56652,48548 +48548,48548 +56653,48549 +48549,48549 +56654,48550 +48550,48550 +92605,48550 +56655,48551 +48551,48551 +56656,48552 +48552,48552 +56657,48553 +48553,48553 +56658,48554 +48554,48554 +56659,48555 +48555,48555 +56660,48556 +48556,48556 +56661,48557 +48557,48557 +56662,48558 +48558,48558 +56663,48559 +48559,48559 +56664,48560 +48560,48560 +56665,48561 +48561,48561 +56666,48562 +48562,48562 +56667,48563 +48563,48563 +56668,48564 +48564,48564 +56669,48565 +48565,48565 +56670,48566 +48566,48566 +99223,48566 +56671,48567 +48567,48567 +56672,48568 +48568,48568 +56673,48569 +48569,48569 +56674,48570 +48570,48570 +56675,48571 +48571,48571 +97713,48571 +56676,48572 +48572,48572 +56677,48573 +48573,48573 +56678,48574 +48574,48574 +56679,48575 +48575,48575 +187432,48576 +56680,48576 +48576,48576 +56681,48577 +48577,48577 +56682,48578 +48578,48578 +193545,48578 +56683,48579 +48579,48579 +56684,48580 +48580,48580 +56685,48581 +48581,48581 +56686,48582 +48582,48582 +244049,48582 +56687,48583 +48583,48583 +56688,48584 +48584,48584 +56689,48585 +48585,48585 +49112,49112 +49113,49113 +49114,49114 +49115,49115 +49116,49116 +49117,49117 +49118,49118 +49119,49119 +49120,49120 +30165,49120 +49121,49121 +49122,49122 +49123,49123 +49124,49124 +49125,49125 +7185,49125 +49126,49126 +49127,49127 +40935,49127 +49128,49128 +40936,49128 +49129,49129 +40937,49129 +25114,49129 +49130,49130 +40938,49130 +49131,49131 +40939,49131 +49132,49132 +40940,49132 +49133,49133 +40941,49133 +179911,49133 +49134,49134 +40942,49134 +49135,49135 +40943,49135 +160163,49135 +49136,49136 +40944,49136 +49137,49137 +40945,49137 +49138,49138 +40946,49138 +49139,49139 +40947,49139 +49140,49140 +40948,49140 +176596,49140 +170632,49140 +49141,49141 +40949,49141 +49142,49142 +40950,49142 +49143,49143 +40951,49143 +49144,49144 +40952,49144 +49145,49145 +40953,49145 +49146,49146 +40954,49146 +49147,49147 +40955,49147 +49148,49148 +40956,49148 +49149,49149 +40957,49149 +49150,49150 +40958,49150 +49151,49151 +40959,49151 +49152,49152 +40960,49152 +49153,49153 +40961,49153 +49154,49154 +40962,49154 +49155,49155 +40963,49155 +49156,49156 +40964,49156 +49157,49157 +40965,49157 +49158,49158 +40966,49158 +49159,49159 +40967,49159 +49160,49160 +40968,49160 +49161,49161 +40969,49161 +49298,49298 +41105,49298 +49299,49299 +41106,49299 +49300,49300 +41107,49300 +49301,49301 +41108,49301 +243329,49302 +49302,49302 +41109,49302 +49303,49303 +41110,49303 +49304,49304 +41111,49304 +49305,49305 +41112,49305 +49306,49306 +41113,49306 +49307,49307 +41114,49307 +49308,49308 +41115,49308 +49309,49309 +41116,49309 +49310,49310 +41117,49310 +49311,49311 +41118,49311 +49312,49312 +41119,49312 +49313,49313 +41120,49313 +49314,49314 +41121,49314 +49315,49315 +41122,49315 +49316,49316 +41123,49316 +49317,49317 +41124,49317 +49318,49318 +41125,49318 +49319,49319 +41126,49319 +49320,49320 +41127,49320 +49321,49321 +41128,49321 +49322,49322 +41129,49322 +49323,49323 +41130,49323 +185192,49323 +49324,49324 +41131,49324 +49325,49325 +41132,49325 +49326,49326 +41133,49326 +49327,49327 +41134,49327 +52747,49327 +44643,49327 +49328,49328 +41135,49328 +49329,49329 +41136,49329 +49330,49330 +41137,49330 +49331,49331 +41138,49331 +49332,49332 +41139,49332 +49333,49333 +41140,49333 +49334,49334 +41141,49334 +49335,49335 +41142,49335 +49336,49336 +41143,49336 +49337,49337 +41144,49337 +49338,49338 +41145,49338 +49339,49339 +41146,49339 +49340,49340 +41147,49340 +49341,49341 +41148,49341 +49342,49342 +41149,49342 +82184,49342 +49343,49343 +41150,49343 +49344,49344 +41151,49344 +49345,49345 +41152,49345 +49346,49346 +41153,49346 +49347,49347 +41154,49347 +49348,49348 +41155,49348 +49349,49349 +41156,49349 +49350,49350 +41157,49350 +49351,49351 +41158,49351 +49352,49352 +41159,49352 +49353,49353 +41160,49353 +49354,49354 +41161,49354 +49355,49355 +41162,49355 +49356,49356 +41163,49356 +49357,49357 +41164,49357 +49358,49358 +41165,49358 +49359,49359 +41166,49359 +49360,49360 +41167,49360 +49361,49361 +41168,49361 +49362,49362 +41169,49362 +49363,49363 +41170,49363 +49364,49364 +41171,49364 +49365,49365 +41172,49365 +49366,49366 +41173,49366 +49367,49367 +41174,49367 +76004,49367 +49368,49368 +41175,49368 +49369,49369 +41176,49369 +49370,49370 +41177,49370 +49371,49371 +41178,49371 +49372,49372 +41179,49372 +49373,49373 +41180,49373 +49374,49374 +41181,49374 +49375,49375 +41182,49375 +49376,49376 +41183,49376 +49377,49377 +41184,49377 +49378,49378 +41185,49378 +49379,49379 +41186,49379 +49380,49380 +41187,49380 +49381,49381 +41188,49381 +49382,49382 +41189,49382 +49383,49383 +41190,49383 +49384,49384 +41191,49384 +49385,49385 +41192,49385 +49386,49386 +41193,49386 +49387,49387 +41194,49387 +49388,49388 +41195,49388 +49389,49389 +41196,49389 +49390,49390 +41197,49390 +211475,49390 +49391,49391 +41198,49391 +49392,49392 +41199,49392 +49393,49393 +41200,49393 +49394,49394 +41201,49394 +49461,49461 +41268,49461 +49462,49462 +41269,49462 +49463,49463 +41270,49463 +49464,49464 +41271,49464 +49465,49465 +41272,49465 +49466,49466 +41273,49466 +49467,49467 +41274,49467 +49468,49468 +41275,49468 +49469,49469 +41276,49469 +49470,49470 +41277,49470 +49471,49471 +41278,49471 +49472,49472 +41279,49472 +49473,49473 +41280,49473 +49474,49474 +41281,49474 +49475,49475 +41282,49475 +49476,49476 +41283,49476 +119290,49476 +49477,49477 +41284,49477 +49478,49478 +41285,49478 +49479,49479 +41286,49479 +49480,49480 +41287,49480 +49481,49481 +41288,49481 +246943,49481 +49482,49482 +41289,49482 +49483,49483 +41290,49483 +49484,49484 +41291,49484 +49485,49485 +41292,49485 +49486,49486 +41293,49486 +49487,49487 +41294,49487 +49488,49488 +41295,49488 +49489,49489 +41296,49489 +49490,49490 +41297,49490 +49491,49491 +41298,49491 +49492,49492 +41299,49492 +49493,49493 +41300,49493 +49494,49494 +41301,49494 +49495,49495 +41302,49495 +59117,49495 +58743,49495 +49496,49496 +41303,49496 +156636,49496 +49497,49497 +41304,49497 +49498,49498 +41305,49498 +49499,49499 +41306,49499 +49500,49500 +41307,49500 +49501,49501 +41308,49501 +49502,49502 +41309,49502 +49503,49503 +41310,49503 +49504,49504 +41311,49504 +49505,49505 +41312,49505 +49506,49506 +41313,49506 +49507,49507 +41314,49507 +49508,49508 +41315,49508 +49509,49509 +41316,49509 +54586,49509 +46482,49509 +49510,49510 +41317,49510 +49511,49511 +41318,49511 +49512,49512 +41319,49512 +2954,49512 +49513,49513 +41320,49513 +49514,49514 +41321,49514 +49515,49515 +41322,49515 +49516,49516 +41323,49516 +49517,49517 +41324,49517 +49518,49518 +41325,49518 +49519,49519 +41326,49519 +61732,49519 +49520,49520 +41327,49520 +49521,49521 +41328,49521 +49522,49522 +41329,49522 +49523,49523 +41330,49523 +49604,49604 +41411,49604 +49605,49605 +41412,49605 +49606,49606 +41413,49606 +49607,49607 +41414,49607 +49608,49608 +41415,49608 +49609,49609 +41416,49609 +49610,49610 +41417,49610 +49611,49611 +41418,49611 +116611,49611 +49612,49612 +41419,49612 +49613,49613 +41420,49613 +49614,49614 +41421,49614 +49615,49615 +41422,49615 +49616,49616 +41423,49616 +142957,49616 +49617,49617 +41424,49617 +49618,49618 +41425,49618 +49619,49619 +41426,49619 +49620,49620 +41427,49620 +49621,49621 +41428,49621 +49803,49803 +41610,49803 +49804,49804 +41611,49804 +49805,49805 +41612,49805 +49806,49806 +41613,49806 +49807,49807 +41614,49807 +49808,49808 +41615,49808 +217573,49809 +49809,49809 +41616,49809 +49810,49810 +41617,49810 +49811,49811 +41618,49811 +49812,49812 +41619,49812 +57163,49812 +49813,49813 +41620,49813 +49814,49814 +41621,49814 +49815,49815 +41622,49815 +49816,49816 +41623,49816 +49817,49817 +41624,49817 +49818,49818 +41625,49818 +198748,49818 +198738,49818 +49819,49819 +41626,49819 +49820,49820 +41627,49820 +49821,49821 +41628,49821 +49822,49822 +41629,49822 +49823,49823 +41630,49823 +56737,49824 +49824,49824 +41631,49824 +3526,49824 +49825,49825 +41632,49825 +49826,49826 +41633,49826 +49827,49827 +41634,49827 +49828,49828 +41635,49828 +49829,49829 +41636,49829 +49830,49830 +41637,49830 +49927,49927 +41734,49927 +49928,49928 +41735,49928 +49929,49929 +41736,49929 +49930,49930 +41737,49930 +49982,49982 +41789,49982 +49983,49983 +41790,49983 +49984,49984 +41791,49984 +49985,49985 +41792,49985 +49986,49986 +41793,49986 +49987,49987 +41794,49987 +49988,49988 +41795,49988 +49989,49989 +41796,49989 +49990,49990 +41797,49990 +49991,49991 +41798,49991 +49992,49992 +41799,49992 +49993,49993 +41800,49993 +49994,49994 +41801,49994 +49995,49995 +41802,49995 +49996,49996 +41803,49996 +49997,49997 +41804,49997 +49998,49998 +41805,49998 +49999,49999 +41806,49999 +50000,50000 +41807,50000 +50001,50001 +41808,50001 +50002,50002 +41809,50002 +50003,50003 +41810,50003 +50004,50004 +41811,50004 +50005,50005 +41812,50005 +50006,50006 +41813,50006 +50007,50007 +41814,50007 +50008,50008 +41815,50008 +50009,50009 +41816,50009 +50010,50010 +41817,50010 +64091,50010 +50011,50011 +41818,50011 +50012,50012 +41819,50012 +50013,50013 +41820,50013 +50014,50014 +41821,50014 +50015,50015 +41822,50015 +50016,50016 +41823,50016 +50017,50017 +41824,50017 +50018,50018 +41825,50018 +185462,50018 +50019,50019 +41826,50019 +50020,50020 +41827,50020 +50021,50021 +41828,50021 +50022,50022 +41829,50022 +50023,50023 +41830,50023 +50024,50024 +41831,50024 +50025,50025 +41832,50025 +50026,50026 +41833,50026 +135226,50026 +127639,50026 +50027,50027 +41834,50027 +50028,50028 +41835,50028 +50029,50029 +41836,50029 +50030,50030 +41837,50030 +50031,50031 +41838,50031 +50032,50032 +41839,50032 +50033,50033 +41840,50033 +50034,50034 +41841,50034 +50035,50035 +41842,50035 +50036,50036 +41843,50036 +50037,50037 +41844,50037 +211135,50037 +50038,50038 +41845,50038 +35777,50038 +50039,50039 +41846,50039 +50040,50040 +41847,50040 +50041,50041 +41848,50041 +50042,50042 +41849,50042 +50043,50043 +41850,50043 +76463,50043 +50044,50044 +41851,50044 +50045,50045 +41852,50045 +50046,50046 +41853,50046 +50047,50047 +41854,50047 +50048,50048 +41855,50048 +50049,50049 +41856,50049 +50050,50050 +41857,50050 +50088,50088 +41895,50088 +50089,50089 +41896,50089 +50090,50090 +41897,50090 +50091,50091 +41898,50091 +50092,50092 +41899,50092 +50093,50093 +41900,50093 +50094,50094 +41901,50094 +50095,50095 +41902,50095 +50096,50096 +41903,50096 +50097,50097 +41904,50097 +50098,50098 +41905,50098 +50099,50099 +41906,50099 +50100,50100 +41907,50100 +50101,50101 +41908,50101 +50102,50102 +41909,50102 +50103,50103 +41910,50103 +50104,50104 +41911,50104 +50105,50105 +41912,50105 +154924,50105 +50106,50106 +41913,50106 +50107,50107 +41914,50107 +50108,50108 +41915,50108 +34004,50108 +50109,50109 +41916,50109 +50110,50110 +41917,50110 +50111,50111 +41918,50111 +7618,50111 +50112,50112 +41919,50112 +76379,50112 +50113,50113 +41920,50113 +50114,50114 +41921,50114 +50115,50115 +41922,50115 +50116,50116 +41923,50116 +50117,50117 +41924,50117 +50118,50118 +41925,50118 +50119,50119 +41926,50119 +50120,50120 +41927,50120 +50121,50121 +41928,50121 +50122,50122 +41929,50122 +50123,50123 +41930,50123 +50124,50124 +41931,50124 +50125,50125 +41932,50125 +50126,50126 +41933,50126 +50127,50127 +41934,50127 +69845,50127 +77461,50127 +50128,50128 +41935,50128 +50129,50129 +41936,50129 +50130,50130 +41937,50130 +50131,50131 +41938,50131 +50132,50132 +41939,50132 +585,50132 +50133,50133 +41940,50133 +50134,50134 +41941,50134 +50135,50135 +41942,50135 +50136,50136 +50137,50137 +50138,50138 +50139,50139 +50140,50140 +50141,50141 +50142,50142 +50143,50143 +50144,50144 +50145,50145 +50146,50146 +50147,50147 +50148,50148 +55926,50148 +47822,50148 +50149,50149 +50150,50150 +132929,50150 +125342,50150 +41958,50150 +50151,50151 +41959,50151 +50152,50152 +41960,50152 +50153,50153 +41961,50153 +50154,50154 +41962,50154 +3779,50154 +50155,50155 +41963,50155 +50156,50156 +50157,50157 +50158,50158 +50159,50159 +50160,50160 +41968,50160 +50161,50161 +50162,50162 +41970,50162 +50163,50163 +50164,50164 +41972,50164 +50210,50210 +42018,50210 +22032,50210 +50211,50211 +42019,50211 +50212,50212 +42020,50212 +50213,50213 +42021,50213 +50214,50214 +42022,50214 +50215,50215 +42023,50215 +50216,50216 +42024,50216 +50217,50217 +42025,50217 +145103,50217 +50218,50218 +42026,50218 +196305,50218 +50219,50219 +42027,50219 +61483,50219 +50220,50220 +42028,50220 +147587,50221 +50221,50221 +42029,50221 +50222,50222 +42030,50222 +50223,50223 +42031,50223 +50224,50224 +42032,50224 +50225,50225 +42033,50225 +50226,50226 +42034,50226 +50227,50227 +42035,50227 +50228,50228 +42036,50228 +50229,50229 +42037,50229 +234858,50229 +50230,50230 +42038,50230 +211334,50230 +56757,50230 +50231,50231 +42039,50231 +50232,50232 +42040,50232 +50233,50233 +42041,50233 +50234,50234 +42042,50234 +50235,50235 +42043,50235 +50236,50236 +42044,50236 +50237,50237 +42045,50237 +50238,50238 +42046,50238 +50239,50239 +42047,50239 +50240,50240 +42048,50240 +9353,50240 +50241,50241 +42049,50241 +220184,50241 +50242,50242 +42050,50242 +50243,50243 +42051,50243 +50244,50244 +42052,50244 +50245,50245 +42053,50245 +50246,50246 +42054,50246 +166443,50246 +50247,50247 +42055,50247 +118513,50247 +50248,50248 +42056,50248 +50249,50249 +42057,50249 +50250,50250 +42058,50250 +50251,50251 +42059,50251 +50252,50252 +42060,50252 +50289,50289 +42097,50289 +50290,50290 +42098,50290 +50291,50291 +42099,50291 +50292,50292 +42100,50292 +50293,50293 +42101,50293 +50294,50294 +42102,50294 +50295,50295 +42103,50295 +50296,50296 +42104,50296 +50297,50297 +42105,50297 +50298,50298 +42106,50298 +50299,50299 +42107,50299 +50300,50300 +42108,50300 +50301,50301 +42109,50301 +50302,50302 +42110,50302 +50303,50303 +42111,50303 +50304,50304 +42112,50304 +50305,50305 +42113,50305 +50306,50306 +42114,50306 +50307,50307 +42115,50307 +50308,50308 +42116,50308 +50309,50309 +42117,50309 +50310,50310 +42118,50310 +50311,50311 +42119,50311 +50312,50312 +42120,50312 +50313,50313 +42121,50313 +50314,50314 +42122,50314 +50315,50315 +42123,50315 +50316,50316 +42124,50316 +50317,50317 +42125,50317 +75170,50317 +50318,50318 +42126,50318 +50319,50319 +42127,50319 +50320,50320 +42128,50320 +50321,50321 +42129,50321 +50322,50322 +42130,50322 +50323,50323 +42131,50323 +50324,50324 +42132,50324 +50325,50325 +42133,50325 +50326,50326 +42134,50326 +50327,50327 +42135,50327 +50328,50328 +42136,50328 +50329,50329 +42137,50329 +50330,50330 +42138,50330 +50331,50331 +42139,50331 +50332,50332 +42140,50332 +50333,50333 +42141,50333 +50334,50334 +42142,50334 +50335,50335 +42143,50335 +50336,50336 +42144,50336 +50337,50337 +42145,50337 +50338,50338 +42146,50338 +50339,50339 +42147,50339 +50340,50340 +42148,50340 +10522,50340 +50341,50341 +42149,50341 +50342,50342 +42150,50342 +50343,50343 +42151,50343 +50344,50344 +42152,50344 +50345,50345 +42153,50345 +50346,50346 +42154,50346 +202796,50346 +50347,50347 +42155,50347 +50348,50348 +42156,50348 +50349,50349 +42157,50349 +50350,50350 +42158,50350 +54585,50350 +46481,50350 +50351,50351 +42159,50351 +56743,50351 +50352,50352 +42160,50352 +50353,50353 +42161,50353 +50354,50354 +42162,50354 +50355,50355 +42163,50355 +50356,50356 +42164,50356 +50357,50357 +42165,50357 +66802,50357 +50358,50358 +42166,50358 +16276,50358 +50359,50359 +42167,50359 +50718,50718 +42614,50718 +50719,50719 +50720,50720 +42616,50720 +50721,50721 +42617,50721 +50722,50722 +42618,50722 +50723,50723 +42619,50723 +50724,50724 +79968,50724 +50725,50725 +50726,50726 +42622,50726 +50727,50727 +42623,50727 +50728,50728 +42624,50728 +50729,50729 +42625,50729 +50730,50730 +42626,50730 +50731,50731 +42627,50731 +50732,50732 +42628,50732 +50733,50733 +42629,50733 +50734,50734 +42630,50734 +50735,50735 +42631,50735 +50736,50736 +42632,50736 +50737,50737 +42633,50737 +50738,50738 +42634,50738 +50739,50739 +42635,50739 +50740,50740 +42636,50740 +50741,50741 +42637,50741 +50742,50742 +42638,50742 +50743,50743 +42639,50743 +213290,50743 +50744,50744 +42640,50744 +50745,50745 +42641,50745 +50746,50746 +42642,50746 +50747,50747 +42643,50747 +97959,50747 +50748,50748 +42644,50748 +50749,50749 +42645,50749 +50750,50750 +42646,50750 +50751,50751 +42647,50751 +50752,50752 +42648,50752 +50753,50753 +42649,50753 +50754,50754 +42650,50754 +50755,50755 +42651,50755 +50756,50756 +42652,50756 +50757,50757 +42653,50757 +50758,50758 +42654,50758 +50759,50759 +42655,50759 +50760,50760 +42656,50760 +50761,50761 +42657,50761 +50762,50762 +42658,50762 +205239,50762 +50763,50763 +42659,50763 +178528,50764 +172564,50764 +50764,50764 +42660,50764 +50765,50765 +42661,50765 +50766,50766 +42662,50766 +50767,50767 +42663,50767 +50768,50768 +42664,50768 +50769,50769 +42665,50769 +50770,50770 +42666,50770 +50771,50771 +42667,50771 +50772,50772 +42668,50772 +50773,50773 +42669,50773 +50774,50774 +42670,50774 +50775,50775 +42671,50775 +50776,50776 +42672,50776 +50777,50777 +42673,50777 +50778,50778 +42674,50778 +50779,50779 +42675,50779 +50780,50780 +42676,50780 +50781,50781 +42677,50781 +50782,50782 +42678,50782 +50783,50783 +42679,50783 +50784,50784 +42680,50784 +50785,50785 +42681,50785 +50786,50786 +42682,50786 +50787,50787 +42683,50787 +50788,50788 +42684,50788 +50789,50789 +42685,50789 +50790,50790 +42686,50790 +50791,50791 +42687,50791 +50792,50792 +42688,50792 +50793,50793 +42689,50793 +50794,50794 +42690,50794 +50795,50795 +42691,50795 +50796,50796 +42692,50796 +50797,50797 +42693,50797 +50798,50798 +42694,50798 +50799,50799 +42695,50799 +50800,50800 +42696,50800 +50801,50801 +42697,50801 +50802,50802 +42698,50802 +50803,50803 +42699,50803 +50804,50804 +42700,50804 +50805,50805 +42701,50805 +3698,50805 +50806,50806 +42702,50806 +50807,50807 +42703,50807 +50808,50808 +42704,50808 +50809,50809 +42705,50809 +50810,50810 +42706,50810 +50811,50811 +42707,50811 +151807,50811 +50812,50812 +42708,50812 +50813,50813 +42709,50813 +50814,50814 +42710,50814 +50889,50889 +42785,50889 +50890,50890 +42786,50890 +50891,50891 +42787,50891 +50892,50892 +42788,50892 +50893,50893 +42789,50893 +50894,50894 +42790,50894 +178485,50894 +172521,50894 +50895,50895 +42791,50895 +50896,50896 +42792,50896 +50897,50897 +42793,50897 +69838,50897 +50898,50898 +42794,50898 +50899,50899 +42795,50899 +50900,50900 +42796,50900 +50901,50901 +42797,50901 +50902,50902 +42798,50902 +50903,50903 +42799,50903 +50904,50904 +42800,50904 +228725,50904 +50905,50905 +42801,50905 +234728,50905 +50906,50906 +42802,50906 +50907,50907 +42803,50907 +50908,50908 +42804,50908 +50909,50909 +42805,50909 +50910,50910 +42806,50910 +50911,50911 +42807,50911 +50912,50912 +42808,50912 +50913,50913 +42809,50913 +50914,50914 +42810,50914 +50915,50915 +42811,50915 +50916,50916 +42812,50916 +50917,50917 +42813,50917 +81591,50917 +50918,50918 +42814,50918 +50919,50919 +42815,50919 +50920,50920 +42816,50920 +50921,50921 +42817,50921 +50922,50922 +42818,50922 +50923,50923 +42819,50923 +50924,50924 +42820,50924 +50925,50925 +42821,50925 +76513,50925 +50926,50926 +42822,50926 +50927,50927 +42823,50927 +154814,50927 +50928,50928 +42824,50928 +50929,50929 +42825,50929 +50930,50930 +42826,50930 +50931,50931 +42827,50931 +20895,50931 +50932,50932 +42828,50932 +50933,50933 +42829,50933 +50934,50934 +42830,50934 +50935,50935 +42831,50935 +50936,50936 +42832,50936 +50937,50937 +42833,50937 +50938,50938 +42834,50938 +50939,50939 +42835,50939 +50940,50940 +42836,50940 +50941,50941 +42837,50941 +50942,50942 +42838,50942 +50943,50943 +42839,50943 +50944,50944 +42840,50944 +50945,50945 +42841,50945 +50946,50946 +42842,50946 +50947,50947 +42843,50947 +50948,50948 +42844,50948 +50949,50949 +42845,50949 +50950,50950 +42846,50950 +50951,50951 +42847,50951 +50952,50952 +42848,50952 +50953,50953 +42849,50953 +50954,50954 +42850,50954 +50955,50955 +42851,50955 +50956,50956 +42852,50956 +50957,50957 +42853,50957 +50958,50958 +42854,50958 +24020,50958 +50959,50959 +42855,50959 +50960,50960 +42856,50960 +51000,51000 +42896,51000 +51001,51001 +42897,51001 +159461,51001 +51002,51002 +42898,51002 +199234,51002 +51003,51003 +42899,51003 +51004,51004 +42900,51004 +51005,51005 +42901,51005 +51006,51006 +42902,51006 +51007,51007 +42903,51007 +51008,51008 +42904,51008 +6141,51008 +51009,51009 +42905,51009 +51010,51010 +42906,51010 +51011,51011 +42907,51011 +51012,51012 +42908,51012 +51051,51051 +42947,51051 +51052,51052 +42948,51052 +51053,51053 +42949,51053 +51054,51054 +42950,51054 +51055,51055 +42951,51055 +51056,51056 +42952,51056 +51057,51057 +42953,51057 +51058,51058 +42954,51058 +51059,51059 +42955,51059 +51060,51060 +42956,51060 +145779,51060 +51061,51061 +42957,51061 +51062,51062 +42958,51062 +187084,51062 +51063,51063 +42959,51063 +51064,51064 +42960,51064 +51065,51065 +42961,51065 +51066,51066 +42962,51066 +51067,51067 +42963,51067 +51068,51068 +42964,51068 +51069,51069 +42965,51069 +51070,51070 +42966,51070 +51071,51071 +42967,51071 +51072,51072 +42968,51072 +51073,51073 +42969,51073 +51074,51074 +42970,51074 +51075,51075 +42971,51075 +51076,51076 +42972,51076 +51077,51077 +42973,51077 +51078,51078 +42974,51078 +51079,51079 +42975,51079 +51080,51080 +42976,51080 +51081,51081 +42977,51081 +51082,51082 +42978,51082 +51083,51083 +42979,51083 +51084,51084 +42980,51084 +51085,51085 +42981,51085 +51086,51086 +42982,51086 +51087,51087 +42983,51087 +51088,51088 +42984,51088 +51089,51089 +42985,51089 +51090,51090 +42986,51090 +51091,51091 +42987,51091 +51092,51092 +42988,51092 +51093,51093 +42989,51093 +51094,51094 +42990,51094 +51095,51095 +42991,51095 +51096,51096 +42992,51096 +51097,51097 +42993,51097 +51098,51098 +42994,51098 +51099,51099 +42995,51099 +51100,51100 +42996,51100 +4473,51100 +51101,51101 +42997,51101 +51102,51102 +42998,51102 +111283,51102 +51103,51103 +42999,51103 +51104,51104 +43000,51104 +51105,51105 +43001,51105 +51106,51106 +43002,51106 +51107,51107 +43003,51107 +51108,51108 +43004,51108 +51109,51109 +43005,51109 +51110,51110 +43006,51110 +51111,51111 +43007,51111 +51112,51112 +43008,51112 +51113,51113 +43009,51113 +51114,51114 +43010,51114 +51115,51115 +43011,51115 +51116,51116 +43012,51116 +51117,51117 +43013,51117 +51118,51118 +43014,51118 +226595,51118 +51119,51119 +43015,51119 +191834,51120 +51120,51120 +43016,51120 +51121,51121 +43017,51121 +51122,51122 +43018,51122 +51123,51123 +43019,51123 +51124,51124 +43020,51124 +51125,51125 +43021,51125 +51126,51126 +43022,51126 +51127,51127 +43023,51127 +51128,51128 +43024,51128 +51129,51129 +43025,51129 +51130,51130 +43026,51130 +51131,51131 +43027,51131 +51132,51132 +43028,51132 +61438,51132 +51133,51133 +43029,51133 +51134,51134 +43030,51134 +51135,51135 +43031,51135 +51136,51136 +43032,51136 +51137,51137 +43033,51137 +51138,51138 +43034,51138 +51139,51139 +43035,51139 +51140,51140 +43036,51140 +51141,51141 +43037,51141 +147140,51141 +51142,51142 +43038,51142 +51143,51143 +43039,51143 +51144,51144 +43040,51144 +51145,51145 +43041,51145 +51146,51146 +43042,51146 +51147,51147 +43043,51147 +51148,51148 +43044,51148 +51149,51149 +43045,51149 +51150,51150 +43046,51150 +51151,51151 +43047,51151 +51152,51152 +43048,51152 +51153,51153 +43049,51153 +51154,51154 +43050,51154 +51155,51155 +43051,51155 +51156,51156 +43052,51156 +17209,51156 +51157,51157 +43053,51157 +51158,51158 +43054,51158 +51159,51159 +43055,51159 +51160,51160 +43056,51160 +51161,51161 +43057,51161 +186051,51161 +51162,51162 +43058,51162 +51163,51163 +43059,51163 +51164,51164 +43060,51164 +22043,51164 +51265,51265 +43161,51265 +51266,51266 +43162,51266 +51267,51267 +43163,51267 +51268,51268 +43164,51268 +51269,51269 +43165,51269 +51270,51270 +43166,51270 +51271,51271 +43167,51271 +51272,51272 +43168,51272 +51273,51273 +43169,51273 +51274,51274 +43170,51274 +51275,51275 +43171,51275 +51276,51276 +43172,51276 +51277,51277 +43173,51277 +51278,51278 +43174,51278 +51279,51279 +43175,51279 +51280,51280 +43176,51280 +51281,51281 +43177,51281 +51282,51282 +43178,51282 +51283,51283 +43179,51283 +20225,51283 +51284,51284 +43180,51284 +51285,51285 +43181,51285 +51286,51286 +43182,51286 +51287,51287 +43183,51287 +51288,51288 +43184,51288 +51289,51289 +43185,51289 +51290,51290 +43186,51290 +51291,51291 +43187,51291 +51292,51292 +43188,51292 +51293,51293 +43189,51293 +51294,51294 +43190,51294 +51295,51295 +43191,51295 +51296,51296 +43192,51296 +51297,51297 +43193,51297 +51298,51298 +43194,51298 +220606,51298 +51299,51299 +43195,51299 +51300,51300 +43196,51300 +51301,51301 +43197,51301 +51302,51302 +43198,51302 +51303,51303 +43199,51303 +74947,51303 +51304,51304 +43200,51304 +51305,51305 +43201,51305 +158990,51306 +51306,51306 +43202,51306 +51307,51307 +43203,51307 +51308,51308 +43204,51308 +248792,51308 +51309,51309 +43205,51309 +51310,51310 +43206,51310 +51311,51311 +43207,51311 +51312,51312 +43208,51312 +51313,51313 +43209,51313 +51314,51314 +43210,51314 +51315,51315 +43211,51315 +51316,51316 +43212,51316 +51317,51317 +43213,51317 +51318,51318 +43214,51318 +51319,51319 +43215,51319 +51320,51320 +43216,51320 +51321,51321 +43217,51321 +51322,51322 +43218,51322 +51323,51323 +43219,51323 +51324,51324 +43220,51324 +51325,51325 +43221,51325 +51326,51326 +43222,51326 +51327,51327 +43223,51327 +26905,51327 +51328,51328 +43224,51328 +51329,51329 +43225,51329 +51330,51330 +43226,51330 +51331,51331 +43227,51331 +51332,51332 +43228,51332 +51333,51333 +43229,51333 +51334,51334 +43230,51334 +51335,51335 +43231,51335 +51336,51336 +43232,51336 +51337,51337 +43233,51337 +51338,51338 +43234,51338 +51339,51339 +43235,51339 +51340,51340 +43236,51340 +51341,51341 +43237,51341 +51342,51342 +43238,51342 +51343,51343 +43239,51343 +51344,51344 +43240,51344 +85610,51344 +51345,51345 +43241,51345 +51346,51346 +43242,51346 +51347,51347 +43243,51347 +51348,51348 +43244,51348 +51349,51349 +43245,51349 +51350,51350 +43246,51350 +51351,51351 +43247,51351 +51352,51352 +43248,51352 +51353,51353 +43249,51353 +51354,51354 +43250,51354 +78636,51354 +51355,51355 +43251,51355 +51356,51356 +43252,51356 +51357,51357 +43253,51357 +51358,51358 +43254,51358 +51359,51359 +43255,51359 +51360,51360 +43256,51360 +51361,51361 +43257,51361 +51362,51362 +43258,51362 +51363,51363 +43259,51363 +51364,51364 +43260,51364 +51365,51365 +43261,51365 +51366,51366 +43262,51366 +51367,51367 +43263,51367 +51368,51368 +43264,51368 +51369,51369 +43265,51369 +51370,51370 +43266,51370 +51371,51371 +43267,51371 +51372,51372 +43268,51372 +51373,51373 +43269,51373 +51374,51374 +43270,51374 +51375,51375 +43271,51375 +51376,51376 +43272,51376 +51377,51377 +43273,51377 +51378,51378 +43274,51378 +51379,51379 +43275,51379 +51380,51380 +43276,51380 +51381,51381 +43277,51381 +51382,51382 +43278,51382 +51383,51383 +43279,51383 +51452,51452 +43348,51452 +51453,51453 +43349,51453 +51454,51454 +43350,51454 +51455,51455 +43351,51455 +51456,51456 +43352,51456 +19967,51456 +51457,51457 +43353,51457 +51458,51458 +43354,51458 +51459,51459 +43355,51459 +184657,51459 +51460,51460 +43356,51460 +51461,51461 +43357,51461 +51462,51462 +43358,51462 +51463,51463 +43359,51463 +51464,51464 +43360,51464 +51465,51465 +43361,51465 +51466,51466 +43362,51466 +51467,51467 +43363,51467 +51468,51468 +43364,51468 +93874,51469 +51469,51469 +43365,51469 +51470,51470 +43366,51470 +51471,51471 +43367,51471 +51472,51472 +43368,51472 +51473,51473 +43369,51473 +51474,51474 +43370,51474 +51475,51475 +43371,51475 +51476,51476 +43372,51476 +51477,51477 +43373,51477 +51478,51478 +43374,51478 +51479,51479 +43375,51479 +51480,51480 +43376,51480 +51481,51481 +43377,51481 +51482,51482 +43378,51482 +51483,51483 +43379,51483 +51484,51484 +43380,51484 +51485,51485 +43381,51485 +51486,51486 +43382,51486 +51487,51487 +43383,51487 +51488,51488 +43384,51488 +51489,51489 +43385,51489 +51490,51490 +43386,51490 +51491,51491 +43387,51491 +51492,51492 +43388,51492 +51493,51493 +43389,51493 +51494,51494 +43390,51494 +51495,51495 +43391,51495 +51496,51496 +43392,51496 +51497,51497 +43393,51497 +51498,51498 +43394,51498 +51499,51499 +43395,51499 +51500,51500 +43396,51500 +51501,51501 +43397,51501 +51502,51502 +43398,51502 +51503,51503 +43399,51503 +51504,51504 +43400,51504 +51505,51505 +43401,51505 +51506,51506 +43402,51506 +51507,51507 +43403,51507 +51508,51508 +43404,51508 +51509,51509 +43405,51509 +86092,51509 +51510,51510 +43406,51510 +51511,51511 +43407,51511 +51512,51512 +43408,51512 +51513,51513 +43409,51513 +51514,51514 +43410,51514 +51515,51515 +43411,51515 +51516,51516 +43412,51516 +51517,51517 +43413,51517 +51518,51518 +43414,51518 +148372,51518 +51519,51519 +43415,51519 +51520,51520 +43416,51520 +51521,51521 +43417,51521 +123296,51521 +196583,51522 +51522,51522 +43418,51522 +51523,51523 +43419,51523 +51524,51524 +43420,51524 +51525,51525 +43421,51525 +51526,51526 +43422,51526 +51527,51527 +43423,51527 +51528,51528 +43424,51528 +51529,51529 +43425,51529 +51530,51530 +43426,51530 +51531,51531 +43427,51531 +51532,51532 +43428,51532 +51533,51533 +43429,51533 +51534,51534 +43430,51534 +51535,51535 +43431,51535 +51536,51536 +43432,51536 +51537,51537 +43433,51537 +51538,51538 +43434,51538 +51539,51539 +43435,51539 +51540,51540 +43436,51540 +51541,51541 +43437,51541 +221289,51541 +51542,51542 +43438,51542 +51543,51543 +43439,51543 +51544,51544 +43440,51544 +51545,51545 +43441,51545 +51546,51546 +43442,51546 +51547,51547 +43443,51547 +145187,51547 +51548,51548 +43444,51548 +51549,51549 +43445,51549 +51550,51550 +43446,51550 +51551,51551 +43447,51551 +51552,51552 +43448,51552 +154508,51552 +51553,51553 +43449,51553 +51554,51554 +43450,51554 +51555,51555 +43451,51555 +51556,51556 +43452,51556 +51557,51557 +43453,51557 +51558,51558 +43454,51558 +51606,51606 +43502,51606 +51607,51607 +43503,51607 +51643,51643 +43539,51643 +51644,51644 +43540,51644 +51645,51645 +43541,51645 +51646,51646 +43542,51646 +51647,51647 +43543,51647 +140120,51647 +51648,51648 +43544,51648 +51649,51649 +43545,51649 +51650,51650 +43546,51650 +51651,51651 +43547,51651 +51652,51652 +43548,51652 +51653,51653 +43549,51653 +51654,51654 +43550,51654 +234193,51654 +30808,51654 +51655,51655 +43551,51655 +51656,51656 +43552,51656 +51657,51657 +43553,51657 +51658,51658 +43554,51658 +51659,51659 +43555,51659 +217033,51659 +51660,51660 +43556,51660 +51661,51661 +43557,51661 +51710,51710 +43606,51710 +51711,51711 +43607,51711 +158972,51711 +51712,51712 +43608,51712 +51713,51713 +43609,51713 +51714,51714 +43610,51714 +51715,51715 +43611,51715 +51716,51716 +43612,51716 +51717,51717 +43613,51717 +247816,51717 +51718,51718 +43614,51718 +51719,51719 +43615,51719 +51720,51720 +43616,51720 +51721,51721 +43617,51721 +51722,51722 +43618,51722 +51723,51723 +43619,51723 +206001,51723 +51724,51724 +43620,51724 +51725,51725 +43621,51725 +51726,51726 +43622,51726 +51727,51727 +43623,51727 +51728,51728 +43624,51728 +51729,51729 +43625,51729 +51730,51730 +43626,51730 +51731,51731 +43627,51731 +51732,51732 +43628,51732 +51733,51733 +43629,51733 +51734,51734 +43630,51734 +161276,51734 +51735,51735 +43631,51735 +51736,51736 +43632,51736 +51737,51737 +43633,51737 +19898,51737 +51738,51738 +43634,51738 +51739,51739 +43635,51739 +51740,51740 +43636,51740 +51741,51741 +43637,51741 +51742,51742 +43638,51742 +51743,51743 +43639,51743 +51744,51744 +43640,51744 +51745,51745 +43641,51745 +51746,51746 +43642,51746 +51747,51747 +43643,51747 +16259,51747 +51748,51748 +43644,51748 +51784,51784 +43680,51784 +51785,51785 +43681,51785 +51786,51786 +43682,51786 +152302,51786 +51787,51787 +43683,51787 +51788,51788 +43684,51788 +51789,51789 +43685,51789 +51790,51790 +43686,51790 +51791,51791 +43687,51791 +51792,51792 +43688,51792 +51793,51793 +43689,51793 +51794,51794 +43690,51794 +51795,51795 +43691,51795 +51796,51796 +43692,51796 +51797,51797 +43693,51797 +51798,51798 +43694,51798 +51799,51799 +43695,51799 +51800,51800 +43696,51800 +51801,51801 +43697,51801 +196715,51801 +51802,51802 +43698,51802 +51803,51803 +43699,51803 +51804,51804 +43700,51804 +51805,51805 +43701,51805 +51806,51806 +43702,51806 +51807,51807 +43703,51807 +176536,51807 +170572,51807 +51808,51808 +43704,51808 +51809,51809 +43705,51809 +51810,51810 +43706,51810 +51811,51811 +43707,51811 +140684,51811 +51812,51812 +43708,51812 +64735,51812 +51813,51813 +43709,51813 +51814,51814 +43710,51814 +184422,51814 +51815,51815 +43711,51815 +51816,51816 +43712,51816 +51817,51817 +43713,51817 +51818,51818 +43714,51818 +51819,51819 +43715,51819 +57686,51819 +176241,51819 +170277,51819 +104028,51819 +51820,51820 +43716,51820 +51821,51821 +43717,51821 +51822,51822 +43718,51822 +51823,51823 +43719,51823 +51824,51824 +43720,51824 +51825,51825 +43721,51825 +51826,51826 +43722,51826 +51827,51827 +43723,51827 +51828,51828 +43724,51828 +51829,51829 +43725,51829 +51830,51830 +43726,51830 +51831,51831 +43727,51831 +51832,51832 +43728,51832 +51833,51833 +43729,51833 +51834,51834 +43730,51834 +51835,51835 +43731,51835 +51836,51836 +43732,51836 +51837,51837 +43733,51837 +51838,51838 +43734,51838 +51839,51839 +43735,51839 +51840,51840 +43736,51840 +51841,51841 +43737,51841 +51842,51842 +43738,51842 +51843,51843 +43739,51843 +51844,51844 +43740,51844 +51845,51845 +43741,51845 +51846,51846 +43742,51846 +51847,51847 +43743,51847 +51848,51848 +43744,51848 +51849,51849 +43745,51849 +51850,51850 +43746,51850 +51851,51851 +43747,51851 +203676,51851 +51852,51852 +43748,51852 +166458,51852 +51853,51853 +43749,51853 +51854,51854 +43750,51854 +51855,51855 +43751,51855 +51856,51856 +43752,51856 +51857,51857 +43753,51857 +51858,51858 +43754,51858 +51859,51859 +43755,51859 +51860,51860 +43756,51860 +51861,51861 +43757,51861 +51862,51862 +43758,51862 +51863,51863 +43759,51863 +51864,51864 +43760,51864 +51865,51865 +43761,51865 +51866,51866 +43762,51866 +235617,51866 +51867,51867 +43763,51867 +51868,51868 +43764,51868 +51869,51869 +43765,51869 +51870,51870 +43766,51870 +51917,51917 +43813,51917 +51918,51918 +43814,51918 +51919,51919 +43815,51919 +51920,51920 +43816,51920 +51921,51921 +43817,51921 +51922,51922 +43818,51922 +51923,51923 +43819,51923 +51924,51924 +43820,51924 +51925,51925 +43821,51925 +51926,51926 +43822,51926 +51927,51927 +43823,51927 +51928,51928 +43824,51928 +52057,52057 +43953,52057 +180875,52057 +52058,52058 +43954,52058 +52059,52059 +43955,52059 +52060,52060 +43956,52060 +180878,52060 +52061,52061 +43957,52061 +52062,52062 +43958,52062 +52063,52063 +43959,52063 +52064,52064 +43960,52064 +52065,52065 +43961,52065 +52066,52066 +43962,52066 +52067,52067 +43963,52067 +52068,52068 +43964,52068 +52069,52069 +43965,52069 +12064,52069 +52070,52070 +43966,52070 +52071,52071 +43967,52071 +52072,52072 +43968,52072 +52073,52073 +43969,52073 +52074,52074 +43970,52074 +52075,52075 +43971,52075 +52076,52076 +43972,52076 +52077,52077 +43973,52077 +52078,52078 +43974,52078 +52079,52079 +43975,52079 +52080,52080 +43976,52080 +52081,52081 +43977,52081 +52082,52082 +43978,52082 +52083,52083 +43979,52083 +52084,52084 +43980,52084 +52085,52085 +43981,52085 +52086,52086 +43982,52086 +52087,52087 +43983,52087 +52088,52088 +43984,52088 +52089,52089 +43985,52089 +52090,52090 +43986,52090 +52091,52091 +43987,52091 +52092,52092 +43988,52092 +52093,52093 +43989,52093 +52094,52094 +43990,52094 +81541,52095 +52095,52095 +43991,52095 +52096,52096 +43992,52096 +52097,52097 +43993,52097 +52098,52098 +43994,52098 +52099,52099 +43995,52099 +52100,52100 +43996,52100 +52101,52101 +43997,52101 +52102,52102 +43998,52102 +52103,52103 +43999,52103 +52104,52104 +44000,52104 +52105,52105 +44001,52105 +52161,52161 +44057,52161 +52162,52162 +44058,52162 +52163,52163 +44059,52163 +52164,52164 +44060,52164 +52165,52165 +44061,52165 +52166,52166 +44062,52166 +52167,52167 +44063,52167 +52168,52168 +44064,52168 +52169,52169 +44065,52169 +52170,52170 +44066,52170 +52171,52171 +44067,52171 +52172,52172 +44068,52172 +52173,52173 +44069,52173 +52174,52174 +44070,52174 +52175,52175 +44071,52175 +52176,52176 +44072,52176 +52177,52177 +44073,52177 +52178,52178 +44074,52178 +52179,52179 +44075,52179 +52180,52180 +44076,52180 +52181,52181 +44077,52181 +52182,52182 +44078,52182 +52183,52183 +44079,52183 +52184,52184 +44080,52184 +52185,52185 +44081,52185 +52186,52186 +44082,52186 +52187,52187 +44083,52187 +52188,52188 +44084,52188 +52189,52189 +44085,52189 +52190,52190 +44086,52190 +52191,52191 +44087,52191 +52192,52192 +44088,52192 +52234,52234 +44130,52234 +52235,52235 +44131,52235 +52236,52236 +44132,52236 +52237,52237 +44133,52237 +52238,52238 +44134,52238 +52239,52239 +44135,52239 +52291,52291 +44187,52291 +52292,52292 +44188,52292 +52293,52293 +44189,52293 +52294,52294 +44190,52294 +52295,52295 +44191,52295 +52296,52296 +44192,52296 +52297,52297 +44193,52297 +52298,52298 +44194,52298 +52299,52299 +44195,52299 +52300,52300 +44196,52300 +52301,52301 +44197,52301 +52302,52302 +44198,52302 +52303,52303 +44199,52303 +52304,52304 +44200,52304 +60354,52304 +52305,52305 +44201,52305 +52306,52306 +44202,52306 +52307,52307 +44203,52307 +52308,52308 +44204,52308 +52309,52309 +44205,52309 +52310,52310 +44206,52310 +52311,52311 +44207,52311 +52312,52312 +44208,52312 +52313,52313 +44209,52313 +52314,52314 +44210,52314 +52315,52315 +44211,52315 +52316,52316 +44212,52316 +52317,52317 +44213,52317 +52318,52318 +44214,52318 +52319,52319 +44215,52319 +52320,52320 +44216,52320 +89184,52321 +52321,52321 +44217,52321 +52322,52322 +44218,52322 +2214,52322 +52323,52323 +44219,52323 +52402,52402 +44298,52402 +52403,52403 +44299,52403 +52404,52404 +44300,52404 +52405,52405 +44301,52405 +52406,52406 +44302,52406 +52407,52407 +44303,52407 +245711,52407 +52408,52408 +44304,52408 +52409,52409 +44305,52409 +52410,52410 +44306,52410 +52411,52411 +44307,52411 +52412,52412 +44308,52412 +52413,52413 +44309,52413 +52414,52414 +44310,52414 +52415,52415 +44311,52415 +52416,52416 +44312,52416 +52417,52417 +44313,52417 +52418,52418 +44314,52418 +52419,52419 +44315,52419 +52420,52420 +44316,52420 +52421,52421 +44317,52421 +52422,52422 +44318,52422 +52423,52423 +44319,52423 +52424,52424 +44320,52424 +36352,52424 +185403,52424 +52425,52425 +44321,52425 +52426,52426 +44322,52426 +52468,52468 +44364,52468 +52469,52469 +44365,52469 +52470,52470 +44366,52470 +52471,52471 +44367,52471 +52472,52472 +44368,52472 +52473,52473 +44369,52473 +52474,52474 +44370,52474 +52475,52475 +44371,52475 +52476,52476 +44372,52476 +52477,52477 +44373,52477 +52478,52478 +44374,52478 +145056,52478 +52479,52479 +44375,52479 +52480,52480 +44376,52480 +52481,52481 +44377,52481 +52482,52482 +44378,52482 +111926,52482 +52545,52545 +44441,52545 +52546,52546 +44442,52546 +52547,52547 +44443,52547 +204911,52547 +52548,52548 +44444,52548 +52549,52549 +44445,52549 +52550,52550 +44446,52550 +52551,52551 +44447,52551 +52552,52552 +44448,52552 +52553,52553 +44449,52553 +52554,52554 +44450,52554 +52555,52555 +44451,52555 +52556,52556 +44452,52556 +52557,52557 +44453,52557 +52558,52558 +44454,52558 +52559,52559 +44455,52559 +52560,52560 +44456,52560 +52561,52561 +44457,52561 +52562,52562 +44458,52562 +52563,52563 +44459,52563 +52564,52564 +44460,52564 +52565,52565 +44461,52565 +52566,52566 +44462,52566 +52567,52567 +44463,52567 +52568,52568 +44464,52568 +52569,52569 +44465,52569 +52570,52570 +44466,52570 +52571,52571 +44467,52571 +52572,52572 +44468,52572 +52573,52573 +44469,52573 +52574,52574 +44470,52574 +52614,52614 +44510,52614 +52615,52615 +44511,52615 +52616,52616 +44512,52616 +52617,52617 +44513,52617 +52618,52618 +44514,52618 +52619,52619 +44515,52619 +52620,52620 +44516,52620 +52621,52621 +44517,52621 +52622,52622 +44518,52622 +52623,52623 +44519,52623 +111697,52623 +52624,52624 +44520,52624 +52625,52625 +44521,52625 +52626,52626 +44522,52626 +52627,52627 +44523,52627 +166454,52627 +52628,52628 +44524,52628 +52629,52629 +44525,52629 +52630,52630 +44526,52630 +52631,52631 +44527,52631 +52632,52632 +44528,52632 +52633,52633 +44529,52633 +52634,52634 +44530,52634 +52635,52635 +44531,52635 +246720,52635 +52636,52636 +44532,52636 +52637,52637 +44533,52637 +52638,52638 +44534,52638 +52639,52639 +44535,52639 +52640,52640 +44536,52640 +52641,52641 +44537,52641 +2704,52641 +52642,52642 +44538,52642 +52643,52643 +44539,52643 +52644,52644 +44540,52644 +52645,52645 +44541,52645 +52646,52646 +44542,52646 +52647,52647 +44543,52647 +52648,52648 +44544,52648 +52649,52649 +44545,52649 +52650,52650 +44546,52650 +52651,52651 +44547,52651 +52652,52652 +44548,52652 +52653,52653 +44549,52653 +8520,52653 +52654,52654 +44550,52654 +52655,52655 +44551,52655 +52694,52694 +44590,52694 +52695,52695 +44591,52695 +83494,52695 +52696,52696 +44592,52696 +52697,52697 +44593,52697 +52698,52698 +44594,52698 +52699,52699 +44595,52699 +52700,52700 +44596,52700 +52701,52701 +44597,52701 +52702,52702 +44598,52702 +52703,52703 +44599,52703 +52704,52704 +44600,52704 +52705,52705 +44601,52705 +52706,52706 +44602,52706 +52707,52707 +44603,52707 +52708,52708 +44604,52708 +52709,52709 +44605,52709 +52710,52710 +44606,52710 +52711,52711 +44607,52711 +52712,52712 +44608,52712 +52713,52713 +44609,52713 +52714,52714 +44610,52714 +52715,52715 +44611,52715 +52716,52716 +44612,52716 +52717,52717 +44613,52717 +52718,52718 +44614,52718 +52719,52719 +44615,52719 +79986,52719 +52720,52720 +44616,52720 +52721,52721 +44617,52721 +52722,52722 +44618,52722 +52723,52723 +44619,52723 +52724,52724 +44620,52724 +52725,52725 +44621,52725 +52726,52726 +44622,52726 +226363,52727 +52727,52727 +44623,52727 +52728,52728 +44624,52728 +52729,52729 +44625,52729 +52730,52730 +44626,52730 +52731,52731 +44627,52731 +52732,52732 +44628,52732 +202708,52732 +52733,52733 +44629,52733 +52734,52734 +44630,52734 +52735,52735 +44631,52735 +52736,52736 +44632,52736 +52737,52737 +44633,52737 +122234,52737 +52738,52738 +44634,52738 +52739,52739 +44635,52739 +52740,52740 +44636,52740 +52741,52741 +44637,52741 +52742,52742 +44638,52742 +52743,52743 +44639,52743 +52744,52744 +44640,52744 +52745,52745 +44641,52745 +148363,52745 +52746,52746 +44642,52746 +52747,52747 +44643,52747 +143979,52747 +49327,52747 +41134,52747 +52748,52748 +44644,52748 +52749,52749 +44645,52749 +52750,52750 +44646,52750 +145541,52750 +52751,52751 +44647,52751 +52752,52752 +44648,52752 +211447,52752 +52753,52753 +44649,52753 +52754,52754 +44650,52754 +52755,52755 +44651,52755 +16812,52755 +52756,52756 +44652,52756 +52757,52757 +44653,52757 +52758,52758 +44654,52758 +52759,52759 +44655,52759 +52883,52883 +44779,52883 +52884,52884 +44780,52884 +146954,52884 +52885,52885 +44781,52885 +107063,52885 +52886,52886 +44782,52886 +52887,52887 +44783,52887 +52888,52888 +44784,52888 +52889,52889 +44785,52889 +52890,52890 +44786,52890 +52891,52891 +44787,52891 +52892,52892 +44788,52892 +187316,52892 +52893,52893 +44789,52893 +52894,52894 +44790,52894 +52895,52895 +44791,52895 +52896,52896 +44792,52896 +52897,52897 +44793,52897 +52898,52898 +44794,52898 +52899,52899 +44795,52899 +52900,52900 +44796,52900 +52901,52901 +44797,52901 +52902,52902 +44798,52902 +52903,52903 +44799,52903 +52904,52904 +44800,52904 +52905,52905 +44801,52905 +52906,52906 +44802,52906 +52907,52907 +44803,52907 +52908,52908 +44804,52908 +52909,52909 +44805,52909 +52910,52910 +44806,52910 +52911,52911 +44807,52911 +52953,52953 +44849,52953 +52954,52954 +44850,52954 +52955,52955 +44851,52955 +52956,52956 +44852,52956 +52957,52957 +44853,52957 +52958,52958 +44854,52958 +52959,52959 +44855,52959 +217481,52959 +52960,52960 +44856,52960 +52961,52961 +44857,52961 +52962,52962 +44858,52962 +52963,52963 +44859,52963 +52964,52964 +44860,52964 +52965,52965 +44861,52965 +52966,52966 +44862,52966 +52967,52967 +44863,52967 +52968,52968 +44864,52968 +52969,52969 +44865,52969 +52970,52970 +44866,52970 +52971,52971 +44867,52971 +80557,52971 +52972,52972 +44868,52972 +52973,52973 +44869,52973 +52974,52974 +44870,52974 +52975,52975 +44871,52975 +53096,53096 +44992,53096 +53097,53097 +44993,53097 +53098,53098 +44994,53098 +53099,53099 +44995,53099 +53151,53151 +45047,53151 +53152,53152 +45048,53152 +110332,53152 +53153,53153 +45049,53153 +53154,53154 +45050,53154 +53155,53155 +45051,53155 +53156,53156 +45052,53156 +53157,53157 +45053,53157 +53158,53158 +45054,53158 +53159,53159 +45055,53159 +53160,53160 +45056,53160 +53161,53161 +45057,53161 +53162,53162 +45058,53162 +53163,53163 +45059,53163 +53164,53164 +45060,53164 +53165,53165 +45061,53165 +84909,53165 +53166,53166 +45062,53166 +241111,53166 +53167,53167 +45063,53167 +53168,53168 +45064,53168 +53169,53169 +45065,53169 +53170,53170 +45066,53170 +53171,53171 +45067,53171 +53225,53225 +45121,53225 +53226,53226 +45122,53226 +53227,53227 +45123,53227 +53228,53228 +45124,53228 +53229,53229 +45125,53229 +53230,53230 +45126,53230 +53231,53231 +45127,53231 +53232,53232 +45128,53232 +110907,53232 +53233,53233 +45129,53233 +53234,53234 +45130,53234 +53235,53235 +45131,53235 +53236,53236 +45132,53236 +53237,53237 +45133,53237 +53238,53238 +45134,53238 +53239,53239 +45135,53239 +53240,53240 +45136,53240 +53241,53241 +45137,53241 +32865,53241 +53242,53242 +45138,53242 +53243,53243 +45139,53243 +187446,53243 +53244,53244 +45140,53244 +53245,53245 +45141,53245 +53246,53246 +45142,53246 +53247,53247 +45143,53247 +53248,53248 +45144,53248 +53249,53249 +45145,53249 +53250,53250 +45146,53250 +79149,53250 +53303,53303 +45199,53303 +53304,53304 +45200,53304 +53305,53305 +45201,53305 +179506,53305 +111339,53305 +53306,53306 +45202,53306 +53307,53307 +45203,53307 +53308,53308 +45204,53308 +53309,53309 +45205,53309 +53310,53310 +45206,53310 +53311,53311 +45207,53311 +53312,53312 +45208,53312 +53313,53313 +45209,53313 +53314,53314 +45210,53314 +53315,53315 +45211,53315 +53316,53316 +45212,53316 +53317,53317 +45213,53317 +53318,53318 +45214,53318 +58020,53318 +53319,53319 +45215,53319 +53320,53320 +45216,53320 +53321,53321 +45217,53321 +53322,53322 +45218,53322 +53323,53323 +45219,53323 +53324,53324 +45220,53324 +53325,53325 +45221,53325 +53326,53326 +45222,53326 +53327,53327 +45223,53327 +53328,53328 +45224,53328 +53329,53329 +45225,53329 +53330,53330 +45226,53330 +53331,53331 +45227,53331 +53332,53332 +45228,53332 +53333,53333 +45229,53333 +89194,53333 +53334,53334 +45230,53334 +244743,53334 +53335,53335 +45231,53335 +53336,53336 +45232,53336 +53337,53337 +45233,53337 +53338,53338 +45234,53338 +53339,53339 +45235,53339 +53340,53340 +45236,53340 +53341,53341 +45237,53341 +53342,53342 +45238,53342 +53343,53343 +45239,53343 +53344,53344 +45240,53344 +53345,53345 +45241,53345 +53346,53346 +45242,53346 +53347,53347 +45243,53347 +53348,53348 +45244,53348 +242313,53348 +53349,53349 +45245,53349 +53350,53350 +45246,53350 +53351,53351 +45247,53351 +53352,53352 +45248,53352 +53353,53353 +45249,53353 +53354,53354 +45250,53354 +53355,53355 +45251,53355 +53356,53356 +45252,53356 +53357,53357 +45253,53357 +53358,53358 +45254,53358 +53359,53359 +45255,53359 +53360,53360 +45256,53360 +108668,53360 +53361,53361 +45257,53361 +243867,53361 +53362,53362 +45258,53362 +53363,53363 +45259,53363 +53364,53364 +45260,53364 +53365,53365 +45261,53365 +53366,53366 +45262,53366 +53367,53367 +45263,53367 +53368,53368 +45264,53368 +206584,53368 +53369,53369 +45265,53369 +53370,53370 +45266,53370 +53371,53371 +45267,53371 +178695,53371 +53372,53372 +45268,53372 +53373,53373 +45269,53373 +53374,53374 +45270,53374 +53375,53375 +45271,53375 +53376,53376 +45272,53376 +53377,53377 +45273,53377 +53378,53378 +45274,53378 +53379,53379 +45275,53379 +53380,53380 +45276,53380 +53381,53381 +45277,53381 +53382,53382 +45278,53382 +53383,53383 +45279,53383 +53384,53384 +45280,53384 +53385,53385 +45281,53385 +53386,53386 +45282,53386 +53387,53387 +45283,53387 +53388,53388 +45284,53388 +177674,53388 +171710,53388 +53389,53389 +45285,53389 +53390,53390 +45286,53390 +53391,53391 +45287,53391 +53392,53392 +45288,53392 +53393,53393 +45289,53393 +53394,53394 +45290,53394 +53395,53395 +45291,53395 +53396,53396 +45292,53396 +53397,53397 +45293,53397 +53398,53398 +45294,53398 +53399,53399 +45295,53399 +53400,53400 +45296,53400 +53401,53401 +45297,53401 +53402,53402 +45298,53402 +53403,53403 +45299,53403 +53404,53404 +45300,53404 +53405,53405 +45301,53405 +53406,53406 +45302,53406 +53407,53407 +45303,53407 +53408,53408 +45304,53408 +53409,53409 +45305,53409 +53410,53410 +45306,53410 +53411,53411 +45307,53411 +53412,53412 +45308,53412 +53413,53413 +45309,53413 +53414,53414 +45310,53414 +53415,53415 +45311,53415 +92714,53415 +216285,53415 +53416,53416 +45312,53416 +53417,53417 +45313,53417 +53418,53418 +45314,53418 +53419,53419 +45315,53419 +53420,53420 +45316,53420 +53421,53421 +45317,53421 +53422,53422 +45318,53422 +53423,53423 +45319,53423 +53424,53424 +45320,53424 +53425,53425 +45321,53425 +53426,53426 +45322,53426 +198415,53426 +53427,53427 +45323,53427 +53428,53428 +45324,53428 +53429,53429 +45325,53429 +53430,53430 +45326,53430 +53431,53431 +45327,53431 +53432,53432 +45328,53432 +53433,53433 +45329,53433 +53434,53434 +45330,53434 +53435,53435 +45331,53435 +53436,53436 +45332,53436 +53437,53437 +45333,53437 +53438,53438 +45334,53438 +53439,53439 +45335,53439 +107153,53439 +53440,53440 +45336,53440 +53441,53441 +45337,53441 +53442,53442 +45338,53442 +53443,53443 +45339,53443 +31708,53443 +53444,53444 +45340,53444 +53445,53445 +45341,53445 +53446,53446 +45342,53446 +53447,53447 +45343,53447 +219844,53447 +53448,53448 +45344,53448 +4372,53448 +53449,53449 +45345,53449 +53450,53450 +45346,53450 +53451,53451 +45347,53451 +53452,53452 +45348,53452 +53453,53453 +45349,53453 +53454,53454 +45350,53454 +53455,53455 +45351,53455 +53456,53456 +45352,53456 +53457,53457 +45353,53457 +53458,53458 +45354,53458 +53459,53459 +45355,53459 +53460,53460 +45356,53460 +53461,53461 +45357,53461 +53462,53462 +45358,53462 +53463,53463 +45359,53463 +53464,53464 +45360,53464 +53465,53465 +45361,53465 +53466,53466 +45362,53466 +53467,53467 +45363,53467 +164059,53467 +53468,53468 +45364,53468 +53469,53469 +45365,53469 +53470,53470 +45366,53470 +53471,53471 +45367,53471 +53679,53679 +45575,53679 +53680,53680 +45576,53680 +53681,53681 +45577,53681 +53682,53682 +45578,53682 +53683,53683 +45579,53683 +53684,53684 +45580,53684 +53685,53685 +45581,53685 +53686,53686 +45582,53686 +53687,53687 +45583,53687 +53688,53688 +45584,53688 +78062,53688 +53689,53689 +45585,53689 +142602,53690 +53690,53690 +45586,53690 +33741,53690 +2712,53690 +53691,53691 +45587,53691 +53692,53692 +45588,53692 +53693,53693 +45589,53693 +53694,53694 +45590,53694 +53695,53695 +45591,53695 +53696,53696 +45592,53696 +53697,53697 +45593,53697 +53698,53698 +45594,53698 +213870,53698 +53699,53699 +45595,53699 +53700,53700 +45596,53700 +200744,53700 +53701,53701 +45597,53701 +53702,53702 +45598,53702 +53703,53703 +45599,53703 +53704,53704 +45600,53704 +53705,53705 +45601,53705 +53706,53706 +45602,53706 +75468,53706 +53707,53707 +45603,53707 +53708,53708 +45604,53708 +53709,53709 +45605,53709 +53710,53710 +45606,53710 +53711,53711 +45607,53711 +53712,53712 +45608,53712 +53713,53713 +45609,53713 +53714,53714 +45610,53714 +53715,53715 +45611,53715 +53716,53716 +45612,53716 +53717,53717 +45613,53717 +53718,53718 +45614,53718 +53719,53719 +45615,53719 +53720,53720 +45616,53720 +53721,53721 +45617,53721 +149635,53722 +53722,53722 +45618,53722 +53723,53723 +45619,53723 +53724,53724 +45620,53724 +53725,53725 +45621,53725 +53726,53726 +45622,53726 +53727,53727 +45623,53727 +53728,53728 +45624,53728 +53729,53729 +45625,53729 +201108,53729 +53730,53730 +45626,53730 +53731,53731 +45627,53731 +53732,53732 +45628,53732 +53733,53733 +45629,53733 +201112,53733 +53734,53734 +45630,53734 +53735,53735 +45631,53735 +53736,53736 +45632,53736 +53737,53737 +45633,53737 +76462,53737 +53738,53738 +45634,53738 +53739,53739 +45635,53739 +53740,53740 +45636,53740 +53741,53741 +45637,53741 +53742,53742 +45638,53742 +53743,53743 +45639,53743 +53744,53744 +45640,53744 +53745,53745 +45641,53745 +53746,53746 +45642,53746 +53747,53747 +45643,53747 +53748,53748 +45644,53748 +53749,53749 +45645,53749 +53750,53750 +45646,53750 +53751,53751 +45647,53751 +53752,53752 +45648,53752 +53753,53753 +45649,53753 +53754,53754 +45650,53754 +53755,53755 +45651,53755 +53756,53756 +45652,53756 +53779,53756 +45675,53756 +53757,53757 +45653,53757 +53758,53758 +45654,53758 +53759,53759 +45655,53759 +53760,53760 +45656,53760 +53761,53761 +45657,53761 +77847,53762 +53762,53762 +45658,53762 +260,53762 +53763,53763 +45659,53763 +53764,53764 +45660,53764 +53765,53765 +45661,53765 +53766,53766 +45662,53766 +53767,53767 +45663,53767 +53768,53768 +45664,53768 +53769,53769 +45665,53769 +53770,53770 +45666,53770 +59399,53770 +53771,53771 +45667,53771 +53772,53772 +45668,53772 +53773,53773 +45669,53773 +53774,53774 +45670,53774 +53775,53775 +45671,53775 +53776,53776 +45672,53776 +53777,53777 +45673,53777 +53778,53778 +45674,53778 +53779,53779 +45675,53779 +110913,53779 +53756,53779 +45652,53779 +53780,53780 +45676,53780 +53781,53781 +45677,53781 +53782,53782 +45678,53782 +122988,53783 +53783,53783 +45679,53783 +53784,53784 +45680,53784 +53785,53785 +45681,53785 +53786,53786 +45682,53786 +53787,53787 +45683,53787 +53788,53788 +45684,53788 +53789,53789 +45685,53789 +53790,53790 +45686,53790 +53791,53791 +45687,53791 +53792,53792 +45688,53792 +53793,53793 +45689,53793 +53794,53794 +45690,53794 +53795,53795 +45691,53795 +53796,53796 +45692,53796 +53797,53797 +45693,53797 +53798,53798 +45694,53798 +22716,53798 +53799,53799 +45695,53799 +53800,53800 +45696,53800 +53801,53801 +45697,53801 +53802,53802 +45698,53802 +53803,53803 +45699,53803 +53804,53804 +45700,53804 +53843,53843 +45739,53843 +53844,53844 +45740,53844 +53845,53845 +45741,53845 +53846,53846 +45742,53846 +53847,53847 +45743,53847 +53848,53848 +45744,53848 +87351,53848 +53849,53849 +45745,53849 +53850,53850 +45746,53850 +53851,53851 +45747,53851 +53852,53852 +45748,53852 +53853,53853 +45749,53853 +53854,53854 +45750,53854 +53855,53855 +45751,53855 +53856,53856 +45752,53856 +53857,53857 +45753,53857 +53858,53858 +45754,53858 +53859,53859 +45755,53859 +53860,53860 +45756,53860 +53861,53861 +45757,53861 +53862,53862 +45758,53862 +53863,53863 +45759,53863 +53864,53864 +45760,53864 +53865,53865 +45761,53865 +53866,53866 +45762,53866 +53867,53867 +45763,53867 +10149,53867 +53868,53868 +45764,53868 +53869,53869 +45765,53869 +53870,53870 +45766,53870 +53871,53871 +45767,53871 +53872,53872 +45768,53872 +53873,53873 +45769,53873 +53874,53874 +45770,53874 +53875,53875 +45771,53875 +53876,53876 +45772,53876 +53877,53877 +45773,53877 +53878,53878 +45774,53878 +53879,53879 +45775,53879 +53880,53880 +45776,53880 +53881,53881 +45777,53881 +53882,53882 +45778,53882 +53883,53883 +45779,53883 +53884,53884 +45780,53884 +53885,53885 +45781,53885 +53886,53886 +45782,53886 +53887,53887 +45783,53887 +53888,53888 +45784,53888 +193,53888 +53889,53889 +45785,53889 +53890,53890 +45786,53890 +53891,53891 +45787,53891 +53892,53892 +45788,53892 +53893,53893 +45789,53893 +53894,53894 +45790,53894 +53895,53895 +45791,53895 +53896,53896 +45792,53896 +53897,53897 +45793,53897 +53898,53898 +45794,53898 +53899,53899 +45795,53899 +53900,53900 +45796,53900 +53901,53901 +45797,53901 +53902,53902 +45798,53902 +53903,53903 +45799,53903 +53904,53904 +45800,53904 +53905,53905 +45801,53905 +53906,53906 +45802,53906 +53907,53907 +45803,53907 +53908,53908 +45804,53908 +53909,53909 +45805,53909 +53910,53910 +45806,53910 +53911,53911 +45807,53911 +53912,53912 +45808,53912 +53913,53913 +45809,53913 +53914,53914 +45810,53914 +53915,53915 +45811,53915 +53916,53916 +45812,53916 +53917,53917 +45813,53917 +53918,53918 +45814,53918 +53919,53919 +45815,53919 +60256,53919 +53920,53920 +45816,53920 +53921,53921 +45817,53921 +247358,53921 +53922,53922 +45818,53922 +53923,53923 +45819,53923 +53924,53924 +45820,53924 +54170,54170 +46066,54170 +54171,54171 +46067,54171 +54172,54172 +46068,54172 +55071,54172 +46967,54172 +54173,54173 +46069,54173 +54174,54174 +46070,54174 +185921,54174 +181319,54174 +56337,54175 +55074,54175 +54175,54175 +48233,54175 +46970,54175 +46071,54175 +56338,54176 +55075,54176 +54769,54176 +54176,54176 +48234,54176 +46971,54176 +46665,54176 +46072,54176 +54177,54177 +46073,54177 +56339,54177 +55076,54177 +48235,54177 +46972,54177 +56340,54178 +55077,54178 +54771,54178 +54178,54178 +48236,54178 +46973,54178 +46667,54178 +46074,54178 +54179,54179 +46075,54179 +55078,54179 +54772,54179 +46974,54179 +46668,54179 +54252,54252 +46148,54252 +54845,54252 +46741,54252 +54253,54253 +46149,54253 +54846,54253 +46742,54253 +54254,54254 +46150,54254 +54847,54254 +46743,54254 +54848,54255 +54255,54255 +46744,54255 +46151,54255 +54256,54256 +46152,54256 +54257,54257 +46153,54257 +54298,54298 +46194,54298 +54299,54299 +46195,54299 +54300,54300 +46196,54300 +54301,54301 +46197,54301 +54302,54302 +46198,54302 +54303,54303 +46199,54303 +54399,54399 +46295,54399 +54400,54400 +46296,54400 +54401,54401 +46297,54401 +54555,54555 +46451,54555 +54556,54556 +46452,54556 +54557,54557 +46453,54557 +54558,54558 +46454,54558 +54559,54559 +46455,54559 +54560,54560 +46456,54560 +54561,54561 +46457,54561 +67289,54561 +54562,54562 +46458,54562 +54563,54563 +46459,54563 +54564,54564 +46460,54564 +54565,54565 +46461,54565 +243441,54565 +54566,54566 +46462,54566 +54567,54567 +46463,54567 +162810,54567 +54568,54568 +46464,54568 +54569,54569 +46465,54569 +54570,54570 +46466,54570 +54571,54571 +46467,54571 +54572,54572 +46468,54572 +54573,54573 +46469,54573 +54574,54574 +46470,54574 +54575,54575 +46471,54575 +54576,54576 +46472,54576 +54577,54577 +46473,54577 +54578,54578 +46474,54578 +54579,54579 +46475,54579 +235084,54579 +54580,54580 +46476,54580 +54581,54581 +46477,54581 +54582,54582 +46478,54582 +54583,54583 +46479,54583 +54584,54584 +46480,54584 +54585,54585 +46481,54585 +50350,54585 +42158,54585 +54586,54586 +46482,54586 +49509,54586 +41316,54586 +54587,54587 +46483,54587 +54588,54588 +46484,54588 +54589,54589 +46485,54589 +54590,54590 +46486,54590 +54591,54591 +46487,54591 +54592,54592 +46488,54592 +54593,54593 +46489,54593 +54594,54594 +46490,54594 +249443,54594 +54595,54595 +46491,54595 +54596,54596 +46492,54596 +54597,54597 +46493,54597 +54598,54598 +46494,54598 +54599,54599 +46495,54599 +54600,54600 +46496,54600 +54601,54601 +46497,54601 +54602,54602 +46498,54602 +54603,54603 +46499,54603 +54604,54604 +46500,54604 +54605,54605 +46501,54605 +54606,54606 +46502,54606 +54607,54607 +46503,54607 +54608,54608 +46504,54608 +54609,54609 +46505,54609 +54610,54610 +46506,54610 +54611,54611 +46507,54611 +54612,54612 +46508,54612 +54613,54613 +46509,54613 +54614,54614 +46510,54614 +54615,54615 +46511,54615 +54616,54616 +46512,54616 +11628,54616 +88639,54616 +54617,54617 +46513,54617 +54618,54618 +46514,54618 +242764,54618 +54619,54619 +46515,54619 +54620,54620 +46516,54620 +54621,54621 +46517,54621 +214531,54621 +54622,54622 +46518,54622 +54623,54623 +46519,54623 +54624,54624 +46520,54624 +54660,54660 +46556,54660 +54661,54661 +46557,54661 +54662,54662 +46558,54662 +54663,54663 +46559,54663 +54664,54664 +46560,54664 +54665,54665 +46561,54665 +107053,54665 +54666,54666 +46562,54666 +54667,54667 +46563,54667 +54668,54668 +46564,54668 +54669,54669 +46565,54669 +54670,54670 +46566,54670 +54671,54671 +46567,54671 +54672,54672 +46568,54672 +54673,54673 +46569,54673 +100664,54673 +54674,54674 +46570,54674 +54675,54675 +46571,54675 +54676,54676 +46572,54676 +54677,54677 +46573,54677 +54678,54678 +46574,54678 +54679,54679 +46575,54679 +4797,54679 +54680,54680 +46576,54680 +107651,54680 +54681,54681 +46577,54681 +54682,54682 +46578,54682 +54683,54683 +46579,54683 +54684,54684 +46580,54684 +54685,54685 +46581,54685 +54686,54686 +46582,54686 +243878,54686 +54687,54687 +46583,54687 +54688,54688 +46584,54688 +54689,54689 +46585,54689 +54690,54690 +46586,54690 +54691,54691 +46587,54691 +142606,54691 +54692,54692 +46588,54692 +76039,54692 +54693,54693 +46589,54693 +185456,54693 +54694,54694 +46590,54694 +54695,54695 +46591,54695 +54696,54696 +46592,54696 +54697,54697 +46593,54697 +54698,54698 +46594,54698 +54699,54699 +46595,54699 +54700,54700 +46596,54700 +54701,54701 +46597,54701 +54702,54702 +46598,54702 +54703,54703 +46599,54703 +54704,54704 +46600,54704 +54705,54705 +46601,54705 +54706,54706 +46602,54706 +54707,54707 +46603,54707 +54708,54708 +46604,54708 +54709,54709 +46605,54709 +54710,54710 +46606,54710 +54711,54711 +46607,54711 +57250,54711 +54712,54712 +46608,54712 +54713,54713 +46609,54713 +54714,54714 +46610,54714 +141264,54714 +54715,54715 +46611,54715 +54716,54716 +46612,54716 +162993,54716 +54717,54717 +46613,54717 +54718,54718 +46614,54718 +247804,54718 +54719,54719 +46615,54719 +54720,54720 +46616,54720 +54721,54721 +46617,54721 +54722,54722 +46618,54722 +54723,54723 +46619,54723 +54724,54724 +46620,54724 +54725,54725 +46621,54725 +54726,54726 +46622,54726 +54727,54727 +46623,54727 +244104,54727 +54728,54728 +46624,54728 +54729,54729 +46625,54729 +54730,54730 +46626,54730 +54731,54731 +46627,54731 +54732,54732 +46628,54732 +54733,54733 +46629,54733 +54734,54734 +46630,54734 +54735,54735 +46631,54735 +54736,54736 +46632,54736 +54737,54737 +46633,54737 +54738,54738 +46634,54738 +54739,54739 +46635,54739 +54740,54740 +46636,54740 +54741,54741 +46637,54741 +54742,54742 +46638,54742 +54743,54743 +46639,54743 +54744,54744 +46640,54744 +201394,54744 +54745,54745 +46641,54745 +54746,54746 +46642,54746 +54747,54747 +46643,54747 +54748,54748 +46644,54748 +54749,54749 +46645,54749 +54750,54750 +46646,54750 +54751,54751 +46647,54751 +54752,54752 +46648,54752 +54753,54753 +46649,54753 +54754,54754 +46650,54754 +84188,54754 +54755,54755 +46651,54755 +54756,54756 +46652,54756 +15421,54756 +54757,54757 +46653,54757 +54758,54758 +46654,54758 +54759,54759 +46655,54759 +54760,54760 +46656,54760 +54761,54761 +46657,54761 +54762,54762 +46658,54762 +54763,54763 +46659,54763 +54764,54764 +46660,54764 +54765,54765 +46661,54765 +54766,54766 +46662,54766 +56336,54767 +55073,54767 +54767,54767 +48232,54767 +46969,54767 +46663,54767 +54768,54768 +46664,54768 +56338,54769 +55075,54769 +54769,54769 +54176,54769 +48234,54769 +46971,54769 +46665,54769 +46072,54769 +54770,54770 +46666,54770 +56340,54771 +55077,54771 +54771,54771 +54178,54771 +48236,54771 +46973,54771 +46667,54771 +46074,54771 +55078,54772 +54772,54772 +46974,54772 +46668,54772 +56341,54772 +48237,54772 +54179,54772 +46075,54772 +54773,54773 +46669,54773 +54774,54774 +46670,54774 +54775,54775 +46671,54775 +54776,54776 +46672,54776 +54777,54777 +46673,54777 +54778,54778 +46674,54778 +55084,54778 +46980,54778 +54779,54779 +46675,54779 +54780,54780 +46676,54780 +54781,54781 +46677,54781 +54782,54782 +46678,54782 +54783,54783 +46679,54783 +54784,54784 +46680,54784 +54785,54785 +46681,54785 +54786,54786 +46682,54786 +54787,54787 +46683,54787 +54788,54788 +46684,54788 +54789,54789 +46685,54789 +54790,54790 +46686,54790 +54791,54791 +46687,54791 +54844,54844 +46740,54844 +54845,54845 +46741,54845 +54252,54845 +46148,54845 +54846,54846 +46742,54846 +54253,54846 +46149,54846 +54847,54847 +46743,54847 +54254,54847 +46150,54847 +54848,54848 +54255,54848 +46744,54848 +46151,54848 +54964,54964 +46860,54964 +54965,54965 +46861,54965 +54966,54966 +46862,54966 +54967,54967 +46863,54967 +54968,54968 +46864,54968 +55930,54968 +47826,54968 +54969,54969 +46865,54969 +54970,54970 +46866,54970 +54971,54971 +46867,54971 +54972,54972 +46868,54972 +39407,54972 +54973,54973 +46869,54973 +54974,54974 +46870,54974 +54975,54975 +46871,54975 +54976,54976 +46872,54976 +54977,54977 +46873,54977 +54978,54978 +46874,54978 +54979,54979 +46875,54979 +16263,54979 +54980,54980 +46876,54980 +54981,54981 +46877,54981 +54982,54982 +46878,54982 +54983,54983 +46879,54983 +62607,54983 +54984,54984 +46880,54984 +124074,54985 +54985,54985 +46881,54985 +54986,54986 +46882,54986 +132408,54986 +124821,54986 +54987,54987 +46883,54987 +80450,54987 +67182,54987 +229575,54987 +211758,54988 +54988,54988 +46884,54988 +54989,54989 +46885,54989 +54990,54990 +46886,54990 +54991,54991 +46887,54991 +54992,54992 +46888,54992 +54993,54993 +46889,54993 +54994,54994 +46890,54994 +54995,54995 +46891,54995 +54996,54996 +46892,54996 +54997,54997 +46893,54997 +54998,54998 +46894,54998 +54999,54999 +46895,54999 +55000,55000 +46896,55000 +56111,55000 +48007,55000 +55001,55001 +46897,55001 +55002,55002 +46898,55002 +214190,55003 +55003,55003 +46899,55003 +55004,55004 +46900,55004 +56115,55004 +48011,55004 +55005,55005 +46901,55005 +55006,55006 +46902,55006 +55007,55007 +46903,55007 +56269,55007 +48165,55007 +55008,55008 +46904,55008 +55009,55009 +46905,55009 +55010,55010 +46906,55010 +55011,55011 +46907,55011 +56122,55011 +48018,55011 +55012,55012 +46908,55012 +56123,55012 +48019,55012 +56274,55012 +48170,55012 +55013,55013 +46909,55013 +57726,55013 +55014,55014 +46910,55014 +55015,55015 +46911,55015 +56277,55015 +48173,55015 +56126,55015 +48022,55015 +55016,55016 +46912,55016 +56127,55016 +48023,55016 +55017,55017 +46913,55017 +55018,55018 +46914,55018 +56129,55018 +48025,55018 +55019,55019 +46915,55019 +56281,55019 +48177,55019 +55020,55020 +46916,55020 +56131,55020 +48027,55020 +55021,55021 +46917,55021 +56283,55021 +56132,55021 +48179,55021 +48028,55021 +55022,55022 +46918,55022 +56284,55022 +48180,55022 +23506,55022 +55023,55023 +46919,55023 +55024,55024 +46920,55024 +55025,55025 +46921,55025 +55026,55026 +46922,55026 +55027,55027 +46923,55027 +55028,55028 +46924,55028 +55029,55029 +46925,55029 +4033,55029 +162536,55029 +55030,55030 +46926,55030 +55066,55066 +46962,55066 +55067,55067 +46963,55067 +55068,55068 +46964,55068 +55069,55069 +46965,55069 +55070,55070 +46966,55070 +55071,55071 +46967,55071 +54172,55071 +46068,55071 +56335,55072 +55072,55072 +48231,55072 +46968,55072 +56336,55073 +55073,55073 +54767,55073 +48232,55073 +46969,55073 +46663,55073 +56337,55074 +55074,55074 +54175,55074 +48233,55074 +46970,55074 +46071,55074 +56338,55075 +55075,55075 +54769,55075 +54176,55075 +48234,55075 +46971,55075 +46665,55075 +46072,55075 +56339,55076 +55076,55076 +48235,55076 +46972,55076 +54177,55076 +46073,55076 +56340,55077 +55077,55077 +54771,55077 +54178,55077 +48236,55077 +46973,55077 +46667,55077 +46074,55077 +55078,55078 +54772,55078 +46974,55078 +46668,55078 +56341,55078 +48237,55078 +54179,55078 +46075,55078 +55079,55079 +46975,55079 +145325,55079 +55080,55080 +46976,55080 +55081,55081 +46977,55081 +55082,55082 +46978,55082 +55083,55083 +46979,55083 +55084,55084 +46980,55084 +54778,55084 +46674,55084 +55085,55085 +46981,55085 +55086,55086 +46982,55086 +9193,55086 +55087,55087 +46983,55087 +55088,55088 +46984,55088 +55089,55089 +46985,55089 +55090,55090 +46986,55090 +55091,55091 +46987,55091 +55092,55092 +46988,55092 +55093,55093 +46989,55093 +75811,55093 +55094,55094 +46990,55094 +55095,55095 +46991,55095 +55096,55096 +46992,55096 +110858,55096 +55097,55097 +46993,55097 +55098,55098 +46994,55098 +55099,55099 +46995,55099 +55100,55100 +46996,55100 +55101,55101 +46997,55101 +55102,55102 +46998,55102 +55103,55103 +46999,55103 +55104,55104 +47000,55104 +55105,55105 +47001,55105 +55106,55106 +47002,55106 +55107,55107 +47003,55107 +94487,55107 +55108,55108 +47004,55108 +55109,55109 +47005,55109 +55110,55110 +47006,55110 +90131,55110 +55111,55111 +47007,55111 +55112,55112 +47008,55112 +55113,55113 +47009,55113 +55114,55114 +47010,55114 +55232,55232 +47128,55232 +55233,55233 +47129,55233 +55287,55287 +47183,55287 +55288,55288 +47184,55288 +55289,55289 +47185,55289 +104654,55289 +55290,55290 +47186,55290 +84827,55290 +55291,55291 +47187,55291 +55292,55292 +47188,55292 +55293,55293 +47189,55293 +224769,55294 +55294,55294 +47190,55294 +55295,55295 +47191,55295 +55296,55296 +47192,55296 +55297,55297 +47193,55297 +55298,55298 +47194,55298 +55299,55299 +47195,55299 +55300,55300 +47196,55300 +55301,55301 +47197,55301 +55302,55302 +47198,55302 +55303,55303 +47199,55303 +55304,55304 +47200,55304 +55305,55305 +47201,55305 +55306,55306 +47202,55306 +55349,55349 +47245,55349 +55350,55350 +47246,55350 +55351,55351 +47247,55351 +55352,55352 +47248,55352 +55353,55353 +47249,55353 +55354,55354 +47250,55354 +55355,55355 +47251,55355 +55356,55356 +47252,55356 +55357,55357 +47253,55357 +55358,55358 +47254,55358 +55359,55359 +47255,55359 +55360,55360 +47256,55360 +55361,55361 +47257,55361 +55362,55362 +47258,55362 +55363,55363 +47259,55363 +55364,55364 +47260,55364 +55365,55365 +47261,55365 +55366,55366 +47262,55366 +55367,55367 +47263,55367 +55368,55368 +47264,55368 +55369,55369 +47265,55369 +55370,55370 +47266,55370 +55371,55371 +47267,55371 +55372,55372 +47268,55372 +55373,55373 +47269,55373 +55374,55374 +47270,55374 +55375,55375 +47271,55375 +55376,55376 +47272,55376 +55377,55377 +47273,55377 +55378,55378 +47274,55378 +55379,55379 +47275,55379 +55380,55380 +47276,55380 +55381,55381 +47277,55381 +55382,55382 +47278,55382 +55383,55383 +47279,55383 +55384,55384 +47280,55384 +55385,55385 +47281,55385 +55386,55386 +47282,55386 +55387,55387 +47283,55387 +55388,55388 +47284,55388 +55389,55389 +47285,55389 +55390,55390 +47286,55390 +55391,55391 +47287,55391 +55392,55392 +47288,55392 +55393,55393 +47289,55393 +55394,55394 +47290,55394 +55395,55395 +47291,55395 +55396,55396 +47292,55396 +55397,55397 +47293,55397 +55398,55398 +47294,55398 +55399,55399 +47295,55399 +55400,55400 +47296,55400 +55401,55401 +47297,55401 +55402,55402 +47298,55402 +55403,55403 +47299,55403 +55404,55404 +47300,55404 +55405,55405 +47301,55405 +79551,55405 +55406,55406 +47302,55406 +2119,55406 +55407,55407 +47303,55407 +55408,55408 +47304,55408 +55409,55409 +47305,55409 +55410,55410 +47306,55410 +17197,55410 +55411,55411 +47307,55411 +55412,55412 +47308,55412 +55413,55413 +47309,55413 +55414,55414 +47310,55414 +55415,55415 +47311,55415 +55416,55416 +47312,55416 +55417,55417 +47313,55417 +55418,55418 +47314,55418 +55419,55419 +47315,55419 +55420,55420 +47316,55420 +55421,55421 +47317,55421 +55422,55422 +47318,55422 +55423,55423 +47319,55423 +55424,55424 +47320,55424 +55425,55425 +47321,55425 +55426,55426 +47322,55426 +55427,55427 +47323,55427 +55428,55428 +47324,55428 +76353,55428 +55429,55429 +47325,55429 +55430,55430 +47326,55430 +55431,55431 +47327,55431 +55432,55432 +47328,55432 +55433,55433 +47329,55433 +55434,55434 +47330,55434 +55435,55435 +47331,55435 +55436,55436 +47332,55436 +55437,55437 +47333,55437 +55536,55536 +47432,55536 +55537,55537 +47433,55537 +55538,55538 +47434,55538 +55539,55539 +47435,55539 +55540,55540 +47436,55540 +55541,55541 +47437,55541 +55542,55542 +47438,55542 +55543,55543 +47439,55543 +55544,55544 +47440,55544 +55545,55545 +47441,55545 +55546,55546 +47442,55546 +55547,55547 +47443,55547 +55548,55548 +47444,55548 +55549,55549 +47445,55549 +55550,55550 +47446,55550 +55551,55551 +47447,55551 +55552,55552 +47448,55552 +55553,55553 +47449,55553 +55554,55554 +47450,55554 +55555,55555 +47451,55555 +55556,55556 +47452,55556 +55557,55557 +47453,55557 +55558,55558 +47454,55558 +55559,55559 +47455,55559 +55560,55560 +47456,55560 +183606,55560 +55561,55561 +47457,55561 +55562,55562 +47458,55562 +55563,55563 +47459,55563 +20916,55563 +55564,55564 +47460,55564 +55565,55565 +47461,55565 +55566,55566 +47462,55566 +55567,55567 +47463,55567 +55568,55568 +47464,55568 +224606,55568 +55569,55569 +47465,55569 +55570,55570 +47466,55570 +55571,55571 +47467,55571 +141859,55571 +55572,55572 +47468,55572 +55573,55573 +47469,55573 +55574,55574 +47470,55574 +55575,55575 +47471,55575 +55576,55576 +47472,55576 +101031,55576 +55577,55577 +47473,55577 +55578,55578 +47474,55578 +55579,55579 +47475,55579 +55580,55580 +47476,55580 +55581,55581 +47477,55581 +55582,55582 +47478,55582 +55583,55583 +47479,55583 +30236,55583 +55584,55584 +47480,55584 +55585,55585 +47481,55585 +55586,55586 +47482,55586 +55587,55587 +47483,55587 +55588,55588 +47484,55588 +55589,55589 +47485,55589 +55590,55590 +47486,55590 +55591,55591 +47487,55591 +55592,55592 +47488,55592 +55593,55593 +47489,55593 +55594,55594 +47490,55594 +55595,55595 +47491,55595 +55596,55596 +47492,55596 +55597,55597 +47493,55597 +55598,55598 +47494,55598 +55599,55599 +47495,55599 +55600,55600 +47496,55600 +55601,55601 +47497,55601 +55602,55602 +47498,55602 +55603,55603 +47499,55603 +55604,55604 +47500,55604 +55605,55605 +47501,55605 +55606,55606 +47502,55606 +55607,55607 +47503,55607 +55608,55608 +47504,55608 +55609,55609 +47505,55609 +55647,55647 +47543,55647 +240934,55647 +55648,55648 +47544,55648 +55649,55649 +47545,55649 +55650,55650 +47546,55650 +55651,55651 +47547,55651 +55652,55652 +47548,55652 +55653,55653 +47549,55653 +55654,55654 +47550,55654 +55655,55655 +47551,55655 +111516,55655 +55656,55656 +47552,55656 +55657,55657 +47553,55657 +55658,55658 +47554,55658 +55659,55659 +47555,55659 +55660,55660 +47556,55660 +55661,55661 +47557,55661 +55662,55662 +47558,55662 +55663,55663 +47559,55663 +55664,55664 +47560,55664 +23858,55664 +55665,55665 +47561,55665 +55666,55666 +47562,55666 +55667,55667 +47563,55667 +19050,55667 +55668,55668 +47564,55668 +19051,55668 +55669,55669 +47565,55669 +55670,55670 +47566,55670 +55671,55671 +47567,55671 +55672,55672 +47568,55672 +55673,55673 +47569,55673 +55674,55674 +47570,55674 +55675,55675 +47571,55675 +194290,55675 +55676,55676 +47572,55676 +55677,55677 +47573,55677 +55678,55678 +47574,55678 +55679,55679 +47575,55679 +55680,55680 +47576,55680 +55681,55681 +47577,55681 +55682,55682 +47578,55682 +55683,55683 +47579,55683 +55684,55684 +47580,55684 +55685,55685 +47581,55685 +55686,55686 +47582,55686 +55687,55687 +47583,55687 +55688,55688 +47584,55688 +55689,55689 +47585,55689 +55690,55690 +47586,55690 +55691,55691 +47587,55691 +55692,55692 +47588,55692 +55693,55693 +47589,55693 +55694,55694 +47590,55694 +55730,55730 +47626,55730 +55731,55731 +47627,55731 +55732,55732 +47628,55732 +55733,55733 +47629,55733 +55734,55734 +47630,55734 +55735,55735 +47631,55735 +55736,55736 +47632,55736 +55737,55737 +47633,55737 +55738,55738 +47634,55738 +55739,55739 +47635,55739 +55740,55740 +47636,55740 +55741,55741 +47637,55741 +55742,55742 +47638,55742 +55743,55743 +47639,55743 +55744,55744 +47640,55744 +55745,55745 +47641,55745 +55746,55746 +47642,55746 +55747,55747 +47643,55747 +55748,55748 +47644,55748 +55749,55749 +47645,55749 +55750,55750 +47646,55750 +55751,55751 +47647,55751 +55752,55752 +47648,55752 +55753,55753 +47649,55753 +55754,55754 +47650,55754 +55755,55755 +47651,55755 +55756,55756 +47652,55756 +55757,55757 +47653,55757 +55758,55758 +47654,55758 +55759,55759 +47655,55759 +9829,55759 +55760,55760 +47656,55760 +152960,55760 +55761,55761 +47657,55761 +55762,55762 +47658,55762 +55763,55763 +47659,55763 +206553,55763 +55830,55830 +47726,55830 +55831,55831 +47727,55831 +55832,55832 +47728,55832 +55833,55833 +47729,55833 +55834,55834 +47730,55834 +55835,55835 +47731,55835 +55836,55836 +47732,55836 +35983,55836 +55837,55837 +47733,55837 +55838,55838 +47734,55838 +55839,55839 +47735,55839 +55840,55840 +47736,55840 +55841,55841 +47737,55841 +123518,55841 +55842,55842 +47738,55842 +55843,55843 +47739,55843 +55926,55926 +47822,55926 +50148,55926 +55927,55927 +47823,55927 +55928,55928 +47824,55928 +41958,55928 +55929,55929 +47825,55929 +55930,55930 +47826,55930 +54968,55930 +46864,55930 +55931,55931 +47827,55931 +55932,55932 +47828,55932 +135268,55932 +127681,55932 +55933,55933 +47829,55933 +55934,55934 +47830,55934 +55935,55935 +47831,55935 +55936,55936 +47832,55936 +55937,55937 +47833,55937 +55938,55938 +47834,55938 +55939,55939 +47835,55939 +55940,55940 +47836,55940 +248132,55940 +55941,55941 +47837,55941 +55942,55942 +47838,55942 +14576,55942 +55943,55943 +47839,55943 +55944,55944 +47840,55944 +55945,55945 +47841,55945 +55946,55946 +47842,55946 +55947,55947 +47843,55947 +131927,55947 +124340,55947 +55948,55948 +47844,55948 +55949,55949 +47845,55949 +55950,55950 +47846,55950 +187467,55950 +55951,55951 +47847,55951 +55952,55952 +47848,55952 +55953,55953 +47849,55953 +55954,55954 +47850,55954 +55955,55955 +47851,55955 +55956,55956 +47852,55956 +55957,55957 +47853,55957 +116372,55957 +55958,55958 +47854,55958 +55959,55959 +47855,55959 +56108,55959 +48004,55959 +55960,55960 +47856,55960 +55961,55961 +47857,55961 +55962,55962 +47858,55962 +55963,55963 +47859,55963 +55964,55964 +47860,55964 +55965,55965 +47861,55965 +55966,55966 +47862,55966 +55967,55967 +47863,55967 +55968,55968 +47864,55968 +55969,55969 +47865,55969 +55970,55970 +47866,55970 +56271,55971 +55971,55971 +48167,55971 +47867,55971 +55972,55972 +47868,55972 +55973,55973 +47869,55973 +61998,55974 +55974,55974 +47870,55974 +55975,55975 +47871,55975 +79171,55975 +56078,56078 +47974,56078 +133187,56078 +125600,56078 +56079,56079 +47975,56079 +56080,56080 +47976,56080 +143283,56081 +56081,56081 +47977,56081 +56082,56082 +47978,56082 +56083,56083 +47979,56083 +56084,56084 +47980,56084 +56085,56085 +47981,56085 +56086,56086 +47982,56086 +56087,56087 +47983,56087 +56088,56088 +47984,56088 +56089,56089 +47985,56089 +56090,56090 +47986,56090 +56091,56091 +47987,56091 +56092,56092 +47988,56092 +56093,56093 +47989,56093 +56094,56094 +47990,56094 +56095,56095 +47991,56095 +111667,56095 +56096,56096 +47992,56096 +56097,56097 +47993,56097 +56098,56098 +47994,56098 +56099,56099 +47995,56099 +56100,56100 +47996,56100 +56101,56101 +47997,56101 +56102,56102 +47998,56102 +56103,56103 +47999,56103 +56104,56104 +48000,56104 +62098,56104 +56105,56105 +48001,56105 +56106,56106 +48002,56106 +116381,56106 +56107,56107 +48003,56107 +56108,56108 +48004,56108 +55959,56108 +47855,56108 +56109,56109 +48005,56109 +56110,56110 +48006,56110 +56111,56111 +48007,56111 +55000,56111 +46896,56111 +56112,56112 +48008,56112 +56113,56113 +48009,56113 +56114,56114 +48010,56114 +56265,56114 +48161,56114 +56115,56115 +48011,56115 +55004,56115 +46900,56115 +56266,56115 +48162,56115 +56116,56116 +48012,56116 +56268,56117 +56117,56117 +48164,56117 +48013,56117 +56118,56118 +48014,56118 +28453,56118 +151452,56118 +56119,56119 +48015,56119 +56120,56120 +48016,56120 +56121,56121 +48017,56121 +56272,56121 +48168,56121 +56122,56122 +48018,56122 +55011,56122 +46907,56122 +56273,56122 +48169,56122 +227755,56122 +56123,56123 +48019,56123 +55012,56123 +46908,56123 +56124,56124 +48020,56124 +56125,56125 +48021,56125 +56126,56126 +48022,56126 +55015,56126 +46911,56126 +56127,56127 +48023,56127 +56278,56127 +48174,56127 +55016,56127 +46912,56127 +56128,56128 +48024,56128 +56279,56128 +48175,56128 +56129,56129 +48025,56129 +56280,56129 +48176,56129 +55018,56129 +46914,56129 +56130,56130 +48026,56130 +56281,56130 +48177,56130 +56131,56131 +48027,56131 +56282,56131 +48178,56131 +55020,56131 +46916,56131 +56283,56132 +56132,56132 +48179,56132 +48028,56132 +55021,56132 +46917,56132 +56133,56133 +48029,56133 +56284,56133 +48180,56133 +56134,56134 +48030,56134 +56135,56135 +48031,56135 +56136,56136 +48032,56136 +56137,56137 +48033,56137 +56138,56138 +48034,56138 +56139,56139 +48035,56139 +56140,56140 +48036,56140 +56141,56141 +48037,56141 +56142,56142 +48038,56142 +56143,56143 +48039,56143 +56294,56143 +48190,56143 +56144,56144 +48040,56144 +56145,56145 +48041,56145 +56146,56146 +48042,56146 +56147,56147 +48043,56147 +56148,56148 +48044,56148 +56149,56149 +48045,56149 +56150,56150 +48046,56150 +56151,56151 +48047,56151 +56152,56152 +48048,56152 +56153,56153 +48049,56153 +88500,56153 +56154,56154 +48050,56154 +56155,56155 +48051,56155 +56156,56156 +48052,56156 +56157,56157 +48053,56157 +56158,56158 +48054,56158 +56159,56159 +48055,56159 +56160,56160 +48056,56160 +56161,56161 +48057,56161 +56162,56162 +48058,56162 +56163,56163 +48059,56163 +158605,56163 +56164,56164 +48060,56164 +56165,56165 +48061,56165 +56166,56166 +48062,56166 +56167,56167 +48063,56167 +56168,56168 +48064,56168 +56169,56169 +48065,56169 +56170,56170 +48066,56170 +56171,56171 +48067,56171 +56172,56172 +48068,56172 +56173,56173 +48069,56173 +56174,56174 +48070,56174 +56175,56175 +48071,56175 +56176,56176 +48072,56176 +56177,56177 +48073,56177 +56178,56178 +48074,56178 +56179,56179 +48075,56179 +56180,56180 +48076,56180 +13673,56180 +56181,56181 +48077,56181 +135677,56181 +128090,56181 +56182,56182 +48078,56182 +14418,56182 +56183,56183 +48079,56183 +56184,56184 +48080,56184 +56185,56185 +48081,56185 +56186,56186 +48082,56186 +56187,56187 +48083,56187 +56188,56188 +48084,56188 +56189,56189 +48085,56189 +56190,56190 +48086,56190 +56191,56191 +48087,56191 +56192,56192 +48088,56192 +56193,56193 +48089,56193 +56194,56194 +48090,56194 +56195,56195 +48091,56195 +56196,56196 +48092,56196 +56197,56197 +48093,56197 +56198,56198 +48094,56198 +56199,56199 +48095,56199 +56200,56200 +48096,56200 +56201,56201 +48097,56201 +56202,56202 +48098,56202 +56203,56203 +48099,56203 +79897,56203 +56204,56204 +48100,56204 +56205,56205 +48101,56205 +56206,56206 +48102,56206 +56207,56207 +48103,56207 +56208,56208 +48104,56208 +56209,56209 +48105,56209 +56210,56210 +48106,56210 +56211,56211 +48107,56211 +56212,56212 +48108,56212 +56213,56213 +48109,56213 +56214,56214 +48110,56214 +56215,56215 +48111,56215 +180290,56216 +56216,56216 +48112,56216 +56217,56217 +48113,56217 +56218,56218 +48114,56218 +56219,56219 +48115,56219 +115470,56219 +56220,56220 +48116,56220 +210213,56220 +141650,56220 +56221,56221 +48117,56221 +56222,56222 +48118,56222 +59104,56222 +58730,56222 +56223,56223 +48119,56223 +56224,56224 +48120,56224 +56225,56225 +48121,56225 +56226,56226 +48122,56226 +65857,56226 +56227,56227 +48123,56227 +56228,56228 +48124,56228 +56229,56229 +48125,56229 +56230,56230 +48126,56230 +56231,56231 +48127,56231 +56232,56232 +48128,56232 +56233,56233 +48129,56233 +68615,56233 +56234,56234 +48130,56234 +56235,56235 +48131,56235 +56236,56236 +48132,56236 +56237,56237 +48133,56237 +33622,56237 +56238,56238 +48134,56238 +56239,56239 +48135,56239 +56240,56240 +48136,56240 +56241,56241 +48137,56241 +56242,56242 +48138,56242 +56243,56243 +48139,56243 +56244,56244 +48140,56244 +56245,56245 +48141,56245 +56246,56246 +48142,56246 +56247,56247 +48143,56247 +236535,56247 +56248,56248 +48144,56248 +56249,56249 +48145,56249 +56250,56250 +48146,56250 +150569,56250 +56251,56251 +48147,56251 +187468,56251 +56252,56252 +48148,56252 +56253,56253 +48149,56253 +56254,56254 +48150,56254 +56255,56255 +48151,56255 +56256,56256 +48152,56256 +56257,56257 +48153,56257 +56258,56258 +48154,56258 +56259,56259 +48155,56259 +56260,56260 +48156,56260 +56261,56261 +48157,56261 +4983,56261 +56262,56262 +48158,56262 +134200,56262 +126613,56262 +56263,56263 +48159,56263 +56264,56264 +48160,56264 +56265,56265 +48161,56265 +56114,56265 +48010,56265 +56266,56266 +48162,56266 +56115,56266 +48011,56266 +56267,56267 +48163,56267 +56268,56268 +56117,56268 +48164,56268 +48013,56268 +56269,56269 +48165,56269 +55007,56269 +46903,56269 +213603,56269 +56270,56270 +48166,56270 +56271,56271 +55971,56271 +48167,56271 +47867,56271 +56272,56272 +48168,56272 +56121,56272 +48017,56272 +56273,56273 +48169,56273 +56122,56273 +48018,56273 +56274,56274 +48170,56274 +55012,56274 +46908,56274 +56275,56275 +48171,56275 +56276,56276 +48172,56276 +56277,56277 +48173,56277 +55015,56277 +46911,56277 +56278,56278 +48174,56278 +56127,56278 +48023,56278 +56279,56279 +48175,56279 +56128,56279 +48024,56279 +56280,56280 +48176,56280 +56129,56280 +48025,56280 +56281,56281 +48177,56281 +56130,56281 +48026,56281 +55019,56281 +46915,56281 +96098,56281 +56282,56282 +48178,56282 +56131,56282 +48027,56282 +56283,56283 +56132,56283 +48179,56283 +48028,56283 +55021,56283 +46917,56283 +56284,56284 +48180,56284 +55022,56284 +46918,56284 +56133,56284 +48029,56284 +56285,56285 +48181,56285 +56286,56286 +48182,56286 +56287,56287 +48183,56287 +56288,56288 +48184,56288 +56289,56289 +48185,56289 +56290,56290 +48186,56290 +56291,56291 +48187,56291 +11850,56291 +56292,56292 +48188,56292 +56293,56293 +48189,56293 +56294,56294 +48190,56294 +56143,56294 +48039,56294 +56295,56295 +48191,56295 +56296,56296 +48192,56296 +56333,56333 +48229,56333 +56334,56334 +48230,56334 +56335,56335 +55072,56335 +48231,56335 +46968,56335 +56336,56336 +55073,56336 +54767,56336 +48232,56336 +46969,56336 +46663,56336 +56337,56337 +55074,56337 +54175,56337 +48233,56337 +46970,56337 +46071,56337 +56338,56338 +55075,56338 +54769,56338 +54176,56338 +48234,56338 +46971,56338 +46665,56338 +46072,56338 +56339,56339 +55076,56339 +48235,56339 +46972,56339 +54177,56339 +46073,56339 +56340,56340 +55077,56340 +54771,56340 +54178,56340 +48236,56340 +46973,56340 +46667,56340 +46074,56340 +56341,56341 +48237,56341 +55078,56341 +54772,56341 +46974,56341 +46668,56341 +56342,56342 +48238,56342 +56343,56343 +48239,56343 +56344,56344 +48240,56344 +56345,56345 +48241,56345 +56346,56346 +48242,56346 +56347,56347 +48243,56347 +56348,56348 +48244,56348 +56349,56349 +48245,56349 +56350,56350 +48246,56350 +56351,56351 +48247,56351 +56352,56352 +48248,56352 +56353,56353 +48249,56353 +56354,56354 +48250,56354 +56355,56355 +48251,56355 +56356,56356 +48252,56356 +66826,56356 +56357,56357 +48253,56357 +56358,56358 +48254,56358 +56359,56359 +48255,56359 +18539,56359 +14641,56359 +56360,56360 +48256,56360 +215600,56360 +56361,56361 +48257,56361 +56362,56362 +48258,56362 +56363,56363 +48259,56363 +226346,56363 +56364,56364 +48260,56364 +56365,56365 +48261,56365 +56366,56366 +48262,56366 +56407,56407 +48303,56407 +56408,56408 +48304,56408 +56449,56449 +48345,56449 +56450,56450 +48346,56450 +56451,56451 +48347,56451 +56452,56452 +48348,56452 +56453,56453 +48349,56453 +56454,56454 +48350,56454 +56455,56455 +48351,56455 +56456,56456 +48352,56456 +56457,56457 +48353,56457 +56458,56458 +48354,56458 +56459,56459 +48355,56459 +56542,56542 +48438,56542 +56543,56543 +48439,56543 +56544,56544 +48440,56544 +236144,56544 +2753,56544 +56545,56545 +48441,56545 +56546,56546 +48442,56546 +56547,56547 +48443,56547 +56548,56548 +48444,56548 +56549,56549 +48445,56549 +56550,56550 +48446,56550 +180978,56550 +56551,56551 +48447,56551 +56552,56552 +48448,56552 +56553,56553 +48449,56553 +56554,56554 +48450,56554 +56555,56555 +48451,56555 +56556,56556 +48452,56556 +56557,56557 +48453,56557 +56558,56558 +48454,56558 +56559,56559 +48455,56559 +56560,56560 +48456,56560 +56561,56561 +48457,56561 +56636,56636 +48532,56636 +161053,56636 +56637,56637 +48533,56637 +56638,56638 +48534,56638 +56639,56639 +48535,56639 +56640,56640 +48536,56640 +56641,56641 +48537,56641 +56642,56642 +48538,56642 +56643,56643 +48539,56643 +56644,56644 +48540,56644 +56645,56645 +48541,56645 +56646,56646 +48542,56646 +56647,56647 +48543,56647 +56648,56648 +48544,56648 +162644,56648 +56649,56649 +48545,56649 +140395,56649 +56650,56650 +48546,56650 +56651,56651 +48547,56651 +77537,56652 +56652,56652 +48548,56652 +56653,56653 +48549,56653 +56654,56654 +48550,56654 +92605,56654 +56655,56655 +48551,56655 +56656,56656 +48552,56656 +56657,56657 +48553,56657 +56658,56658 +48554,56658 +56659,56659 +48555,56659 +56660,56660 +48556,56660 +56661,56661 +48557,56661 +56662,56662 +48558,56662 +56663,56663 +48559,56663 +56664,56664 +48560,56664 +56665,56665 +48561,56665 +56666,56666 +48562,56666 +56667,56667 +48563,56667 +56668,56668 +48564,56668 +56669,56669 +48565,56669 +56670,56670 +48566,56670 +99223,56670 +56671,56671 +48567,56671 +56672,56672 +48568,56672 +56673,56673 +48569,56673 +56674,56674 +48570,56674 +56675,56675 +48571,56675 +97713,56675 +56676,56676 +48572,56676 +56677,56677 +48573,56677 +56678,56678 +48574,56678 +56679,56679 +48575,56679 +187432,56680 +56680,56680 +48576,56680 +56681,56681 +48577,56681 +56682,56682 +48578,56682 +193545,56682 +56683,56683 +48579,56683 +56684,56684 +48580,56684 +56685,56685 +48581,56685 +56686,56686 +48582,56686 +244049,56686 +56687,56687 +48583,56687 +56688,56688 +48584,56688 +56689,56689 +48585,56689 +56690,56690 +56691,56691 +56692,56692 +56693,56693 +56694,56694 +224951,56694 +56695,56695 +140401,56695 +56696,56696 +56697,56697 +56698,56698 +56699,56699 +56700,56700 +56701,56701 +56702,56702 +56703,56703 +56704,56704 +56705,56705 +215604,56705 +56706,56706 +56707,56707 +56708,56708 +56709,56709 +56710,56710 +56711,56711 +56712,56712 +56713,56713 +56714,56714 +56715,56715 +56716,56716 +56717,56717 +56718,56718 +56719,56719 +56720,56720 +56721,56721 +56722,56722 +56723,56723 +56724,56724 +56725,56725 +56726,56726 +56727,56727 +56728,56728 +56729,56729 +56730,56730 +56731,56731 +56732,56732 +56733,56733 +113564,56733 +56734,56734 +56735,56735 +56736,56736 +56737,56737 +49824,56737 +41631,56737 +3526,56737 +56738,56738 +56739,56739 +56740,56740 +56741,56741 +56742,56742 +239492,56742 +56743,56743 +50351,56743 +42159,56743 +56744,56744 +56745,56745 +56746,56746 +56747,56747 +56748,56748 +56749,56749 +56750,56750 +56751,56751 +56752,56752 +56753,56753 +56754,56754 +56755,56755 +56756,56756 +56757,56757 +50230,56757 +42038,56757 +56758,56758 +56759,56759 +56760,56760 +56761,56761 +56762,56762 +56763,56763 +56764,56764 +56765,56765 +56766,56766 +56767,56767 +56768,56768 +56769,56769 +56770,56770 +56771,56771 +56772,56772 +56773,56773 +56774,56774 +56775,56775 +56776,56776 +56777,56777 +56778,56778 +140320,56778 +56779,56779 +56780,56780 +120059,56780 +56781,56781 +122254,56781 +56782,56782 +56783,56783 +56784,56784 +56785,56785 +56786,56786 +56787,56787 +56788,56788 +56789,56789 +56790,56790 +110948,56790 +56791,56791 +56792,56792 +56793,56793 +56794,56794 +164765,56794 +56795,56795 +56796,56796 +56797,56797 +56798,56798 +56799,56799 +56800,56800 +56801,56801 +183510,56801 +56802,56802 +56803,56803 +56804,56804 +56805,56805 +56806,56806 +56807,56807 +56808,56808 +56809,56809 +56810,56810 +56811,56811 +56812,56812 +56813,56813 +56814,56814 +56815,56815 +56816,56816 +56817,56817 +56818,56818 +56819,56819 +56820,56820 +192268,56821 +56821,56821 +56822,56822 +56823,56823 +56824,56824 +56825,56825 +56866,56866 +56867,56867 +56868,56868 +56923,56923 +56924,56924 +56925,56925 +56926,56926 +56927,56927 +56928,56928 +56977,56977 +56978,56978 +57016,57016 +57017,57017 +57018,57018 +57019,57019 +57020,57020 +57021,57021 +57022,57022 +57023,57023 +57024,57024 +57025,57025 +57026,57026 +241818,57026 +57027,57027 +57028,57028 +57029,57029 +57030,57030 +57031,57031 +57032,57032 +57033,57033 +57034,57034 +57035,57035 +57036,57036 +57037,57037 +57038,57038 +157907,57038 +161021,57038 +57039,57039 +57040,57040 +57041,57041 +57042,57042 +64121,57042 +57043,57043 +57044,57044 +57045,57045 +57046,57046 +57047,57047 +57048,57048 +57049,57049 +57050,57050 +57051,57051 +57052,57052 +247237,57052 +9134,57052 +57053,57053 +57054,57054 +57055,57055 +57056,57056 +57057,57057 +142666,57057 +57058,57058 +57059,57059 +57060,57060 +57061,57061 +20391,57061 +57062,57062 +57063,57063 +57064,57064 +57065,57065 +57066,57066 +114420,57066 +57067,57067 +57068,57068 +57069,57069 +57070,57070 +57071,57071 +57072,57072 +57073,57073 +57074,57074 +57075,57075 +219948,57075 +57076,57076 +57077,57077 +57078,57078 +57079,57079 +57080,57080 +57081,57081 +57082,57082 +57083,57083 +57084,57084 +57085,57085 +57086,57086 +57087,57087 +30587,57087 +57139,57139 +57140,57140 +57141,57141 +57142,57142 +57143,57143 +246667,57143 +57144,57144 +57145,57145 +57146,57146 +57147,57147 +57148,57148 +57149,57149 +57150,57150 +208282,57150 +57151,57151 +57152,57152 +57153,57153 +57154,57154 +57155,57155 +57156,57156 +57157,57157 +57158,57158 +57159,57159 +57160,57160 +57161,57161 +57162,57162 +57163,57163 +49812,57163 +41619,57163 +57164,57164 +57165,57165 +57166,57166 +57167,57167 +57168,57168 +57169,57169 +179671,57170 +57170,57170 +57171,57171 +23483,57171 +57172,57172 +57173,57173 +33967,57173 +57246,57246 +57247,57247 +57248,57248 +57249,57249 +57250,57250 +54711,57250 +46607,57250 +57251,57251 +57252,57252 +225331,57252 +57253,57253 +57254,57254 +57290,57290 +57291,57291 +106136,57291 +57292,57292 +57293,57293 +57294,57294 +57295,57295 +57296,57296 +57297,57297 +57298,57298 +57299,57299 +57300,57300 +57301,57301 +57302,57302 +57303,57303 +57304,57304 +57305,57305 +57306,57306 +57307,57307 +57308,57308 +57309,57309 +57310,57310 +57311,57311 +57312,57312 +57313,57313 +57314,57314 +57315,57315 +57316,57316 +57317,57317 +6682,57317 +133101,57317 +125514,57317 +15411,57317 +57318,57318 +57319,57319 +57320,57320 +57321,57321 +57322,57322 +57323,57323 +198919,57323 +57324,57324 +57325,57325 +57326,57326 +57327,57327 +57328,57328 +57329,57329 +57330,57330 +57331,57331 +57332,57332 +133059,57332 +125472,57332 +57333,57333 +57334,57334 +57335,57335 +57336,57336 +57337,57337 +57338,57338 +57339,57339 +57340,57340 +57341,57341 +57342,57342 +57343,57343 +57344,57344 +57345,57345 +57346,57346 +57347,57347 +57348,57348 +57349,57349 +57350,57350 +57351,57351 +57352,57352 +57353,57353 +57354,57354 +57355,57355 +57356,57356 +57654,57356 +57357,57357 +57358,57358 +62059,57358 +57359,57359 +57360,57360 +235200,57360 +57361,57361 +57362,57362 +57363,57363 +57808,57364 +57364,57364 +57956,57364 +57411,57411 +57412,57412 +57413,57413 +242069,57413 +57414,57414 +57415,57415 +57416,57416 +57417,57417 +85226,57417 +57418,57418 +57419,57419 +57420,57420 +6309,57420 +57421,57421 +57422,57422 +57423,57423 +57424,57424 +57425,57425 +57426,57426 +57427,57427 +57428,57428 +57429,57429 +57495,57495 +57496,57496 +201785,57496 +57497,57497 +57498,57498 +57499,57499 +57500,57500 +6848,57500 +57501,57501 +57502,57502 +178774,57502 +57503,57503 +57504,57504 +57505,57505 +57506,57506 +57507,57507 +57508,57508 +142191,57508 +57509,57509 +57510,57510 +212321,57510 +57511,57511 +57512,57512 +57513,57513 +240419,57513 +57514,57514 +57515,57515 +57516,57516 +57517,57517 +57518,57518 +57519,57519 +57520,57520 +57521,57521 +57522,57522 +57566,57566 +57567,57567 +57568,57568 +166730,57568 +57569,57569 +57570,57570 +57571,57571 +4980,57571 +57572,57572 +57573,57573 +57574,57574 +57575,57575 +57576,57576 +57577,57577 +57578,57578 +57579,57579 +57580,57580 +57581,57581 +110624,57581 +57582,57582 +161534,57582 +57583,57583 +57584,57584 +57585,57585 +57586,57586 +57587,57587 +57588,57588 +57589,57589 +57590,57590 +57591,57591 +57592,57592 +57593,57593 +57594,57594 +57595,57595 +57596,57596 +57597,57597 +57598,57598 +57599,57599 +57600,57600 +57601,57601 +57602,57602 +190164,57603 +57603,57603 +57604,57604 +179685,57604 +132404,57604 +124817,57604 +57605,57605 +57606,57606 +57607,57607 +57608,57608 +83547,57608 +57609,57609 +57610,57610 +57611,57611 +214450,57611 +57612,57612 +57613,57613 +231797,57614 +57614,57614 +57615,57615 +57616,57616 +231799,57616 +231800,57617 +57617,57617 +57618,57618 +231801,57618 +57654,57654 +57356,57654 +57655,57655 +57656,57656 +57657,57657 +57658,57658 +57659,57659 +57660,57660 +57661,57661 +57662,57662 +57663,57663 +57664,57664 +57665,57665 +57666,57666 +57667,57667 +57668,57668 +57669,57669 +57670,57670 +57671,57671 +57672,57672 +57673,57673 +57674,57674 +57675,57675 +57676,57676 +57677,57677 +57678,57678 +57679,57679 +57680,57680 +57681,57681 +57682,57682 +57683,57683 +57684,57684 +57685,57685 +57686,57686 +51819,57686 +43715,57686 +57687,57687 +176477,57687 +170513,57687 +57688,57688 +57689,57689 +57690,57690 +57691,57691 +57692,57692 +57693,57693 +179143,57694 +57694,57694 +57695,57695 +57989,57695 +57696,57696 +58141,57696 +57697,57697 +58142,57697 +57698,57698 +58143,57698 +58144,57699 +57845,57699 +57699,57699 +57700,57700 +57701,57701 +58146,57701 +58147,57702 +57702,57702 +57703,57703 +58148,57703 +57704,57704 +57705,57705 +58150,57705 +57706,57706 +89447,57707 +57707,57707 +57708,57708 +57709,57709 +57710,57710 +58004,57710 +57711,57711 +57712,57712 +57713,57713 +57714,57714 +57715,57715 +57716,57716 +57717,57717 +57718,57718 +57719,57719 +57720,57720 +191205,57720 +57721,57721 +57722,57722 +57723,57723 +57724,57724 +57725,57725 +57726,57726 +55013,57726 +46909,57726 +57727,57727 +58022,57728 +57728,57728 +57729,57729 +57730,57730 +132768,57730 +125181,57730 +57731,57731 +134206,57731 +126619,57731 +57732,57732 +57733,57733 +232555,57733 +57734,57734 +57735,57735 +57736,57736 +57737,57737 +57738,57738 +57739,57739 +57740,57740 +57741,57741 +57742,57742 +57743,57743 +218661,57743 +57744,57744 +57745,57745 +57746,57746 +57747,57747 +57748,57748 +57749,57749 +57750,57750 +193833,57750 +57751,57751 +57752,57752 +57753,57753 +57754,57754 +57755,57755 +57756,57756 +57757,57757 +111011,57757 +57758,57758 +57759,57759 +57760,57760 +57761,57761 +57762,57762 +57763,57763 +57764,57764 +57765,57765 +58059,57766 +57766,57766 +58060,57767 +57767,57767 +108591,57767 +58061,57768 +57768,57768 +58062,57769 +57769,57769 +57770,57770 +58063,57770 +57771,57771 +58064,57771 +57772,57772 +57773,57773 +57774,57774 +57775,57775 +57776,57776 +57777,57777 +57778,57778 +57779,57779 +57780,57780 +57781,57781 +57782,57782 +57783,57783 +57784,57784 +57785,57785 +57786,57786 +57787,57787 +57788,57788 +2678,57788 +59134,57788 +58760,57788 +57789,57789 +57790,57790 +57791,57791 +57792,57792 +57793,57793 +57794,57794 +57795,57795 +57796,57796 +57797,57797 +57798,57798 +57799,57799 +2041,57799 +57800,57800 +28975,57800 +57801,57801 +57802,57802 +57803,57803 +57804,57804 +57805,57805 +57806,57806 +57807,57807 +57808,57808 +57364,57808 +57956,57808 +57809,57809 +57810,57810 +57811,57811 +57812,57812 +225635,57812 +57813,57813 +57814,57814 +57815,57815 +57816,57816 +57817,57817 +57965,57817 +57818,57818 +12183,57818 +57819,57819 +57820,57820 +57821,57821 +58121,57822 +57822,57822 +57823,57823 +57824,57824 +57825,57825 +58125,57826 +57826,57826 +57827,57827 +57828,57828 +20412,57828 +20227,57828 +57829,57829 +57830,57830 +57831,57831 +57832,57832 +57833,57833 +57834,57834 +58133,57834 +57835,57835 +157538,57835 +58135,57836 +57836,57836 +28224,57836 +57837,57837 +58137,57838 +57838,57838 +57839,57839 +57840,57840 +57841,57841 +57842,57842 +57843,57843 +232019,57843 +57844,57844 +58144,57845 +57845,57845 +57699,57845 +57846,57846 +57847,57847 +57848,57848 +57849,57849 +57850,57850 +57851,57851 +57852,57852 +57853,57853 +57854,57854 +179732,57855 +57855,57855 +57856,57856 +57857,57857 +57858,57858 +57859,57859 +57860,57860 +57861,57861 +242944,57861 +57862,57862 +57863,57863 +57864,57864 +57865,57865 +57866,57866 +57867,57867 +57868,57868 +57869,57869 +57870,57870 +57871,57871 +57872,57872 +153102,57872 +57873,57873 +57874,57874 +57875,57875 +57876,57876 +57877,57877 +57878,57878 +57879,57879 +57880,57880 +121542,57881 +57881,57881 +57882,57882 +57883,57883 +57884,57884 +35486,57884 +57885,57885 +57886,57886 +57887,57887 +57888,57888 +57889,57889 +57890,57890 +57891,57891 +57892,57892 +57893,57893 +57894,57894 +57895,57895 +57896,57896 +57897,57897 +57898,57898 +57955,57955 +57956,57956 +57808,57956 +57364,57956 +57957,57957 +57958,57958 +142559,57958 +57959,57959 +57960,57960 +57961,57961 +57962,57962 +57963,57963 +57964,57964 +57965,57965 +57817,57965 +57966,57966 +8944,57966 +57967,57967 +58119,57968 +57968,57968 +227100,57968 +57969,57969 +23679,57969 +57970,57970 +103515,57970 +57971,57971 +57972,57972 +146338,57972 +122250,57972 +57973,57973 +57974,57974 +57975,57975 +57976,57976 +57977,57977 +57978,57978 +57979,57979 +57980,57980 +57981,57981 +57982,57982 +57983,57983 +57984,57984 +57985,57985 +57986,57986 +57987,57987 +57988,57988 +57989,57989 +57695,57989 +247316,57989 +57990,57990 +57991,57991 +247229,57991 +57992,57992 +57993,57993 +57994,57994 +57995,57995 +58146,57995 +57996,57996 +57997,57997 +57998,57998 +174630,57998 +168666,57998 +57999,57999 +58000,58000 +58001,58001 +58002,58002 +58003,58003 +58004,58004 +57710,58004 +58005,58005 +62269,58005 +58006,58006 +58007,58007 +58008,58008 +58009,58009 +58010,58010 +58011,58011 +58012,58012 +58013,58013 +58014,58014 +58015,58015 +58016,58016 +58017,58017 +58018,58018 +58019,58019 +58020,58020 +53318,58020 +45214,58020 +58021,58021 +58022,58022 +57728,58022 +58023,58023 +58024,58024 +58025,58025 +58026,58026 +58027,58027 +6488,58027 +58028,58028 +58029,58029 +58030,58030 +58031,58031 +58032,58032 +58033,58033 +58034,58034 +108903,58034 +58035,58035 +58036,58036 +3186,58036 +58037,58037 +58038,58038 +58039,58039 +58040,58040 +58041,58041 +58042,58042 +58043,58043 +58044,58044 +58045,58045 +58046,58046 +58047,58047 +58048,58048 +58049,58049 +58050,58050 +58051,58051 +58052,58052 +15987,58052 +58053,58053 +58054,58054 +231732,58054 +58055,58055 +225220,58055 +58056,58056 +58057,58057 +58058,58058 +58059,58059 +57766,58059 +58060,58060 +57767,58060 +108591,58060 +58061,58061 +57768,58061 +58062,58062 +57769,58062 +58063,58063 +57770,58063 +80708,58063 +58064,58064 +57771,58064 +58065,58065 +58066,58066 +58067,58067 +58068,58068 +58069,58069 +58070,58070 +58071,58071 +113401,58071 +58072,58072 +58073,58073 +58074,58074 +58075,58075 +58076,58076 +58077,58077 +58078,58078 +58079,58079 +58080,58080 +198820,58080 +58081,58081 +58117,58117 +58118,58118 +58119,58119 +57968,58119 +227100,58119 +58120,58120 +58121,58121 +57822,58121 +58122,58122 +58123,58123 +58124,58124 +58125,58125 +57826,58125 +58126,58126 +58127,58127 +58128,58128 +58129,58129 +58130,58130 +58131,58131 +122197,58131 +58132,58132 +26365,58132 +108,58132 +58133,58133 +57834,58133 +58134,58134 +58135,58135 +57836,58135 +28224,58135 +58136,58136 +58137,58137 +57838,58137 +58138,58138 +58139,58139 +58140,58140 +58141,58141 +57696,58141 +58142,58142 +57697,58142 +204447,58142 +58143,58143 +57698,58143 +58144,58144 +57845,58144 +57699,58144 +58145,58145 +58146,58146 +57995,58146 +57701,58146 +58147,58147 +57702,58147 +58148,58148 +57703,58148 +58149,58149 +58150,58150 +57705,58150 +58151,58151 +58152,58152 +58153,58153 +58154,58154 +58155,58155 +58156,58156 +58157,58157 +58158,58158 +58159,58159 +149665,58159 +58160,58160 +58161,58161 +30895,58161 +58162,58162 +58163,58163 +58164,58164 +58165,58165 +58166,58166 +58167,58167 +58168,58168 +58169,58169 +58170,58170 +58171,58171 +58172,58172 +58173,58173 +107455,58173 +58174,58174 +58175,58175 +58176,58176 +58177,58177 +58178,58178 +58179,58179 +58180,58180 +58181,58181 +58182,58182 +58183,58183 +58230,58230 +58231,58231 +58232,58232 +58233,58233 +58234,58234 +58235,58235 +58236,58236 +58237,58237 +58238,58238 +58239,58239 +58240,58240 +58241,58241 +58242,58242 +58243,58243 +58244,58244 +58245,58245 +58246,58246 +58247,58247 +176644,58247 +170680,58247 +58248,58248 +58249,58249 +58250,58250 +58251,58251 +58252,58252 +58253,58253 +58254,58254 +58255,58255 +58256,58256 +58257,58257 +58258,58258 +58259,58259 +58260,58260 +58261,58261 +58262,58262 +58263,58263 +8881,58263 +58264,58264 +58265,58265 +58266,58266 +58267,58267 +58268,58268 +58269,58269 +58270,58270 +58271,58271 +58272,58272 +58273,58273 +58274,58274 +58275,58275 +58276,58276 +58277,58277 +58278,58278 +58279,58279 +58280,58280 +58281,58281 +58282,58282 +58283,58283 +58284,58284 +58285,58285 +58286,58286 +58287,58287 +58288,58288 +58289,58289 +58290,58290 +58291,58291 +58292,58292 +58293,58293 +58294,58294 +58295,58295 +58296,58296 +58297,58297 +58298,58298 +58299,58299 +58300,58300 +155097,58300 +58301,58301 +58302,58302 +58303,58303 +58304,58304 +58305,58305 +58306,58306 +58307,58307 +58308,58308 +58309,58309 +58310,58310 +58311,58311 +58312,58312 +58313,58313 +58314,58314 +58315,58315 +58316,58316 +58317,58317 +58318,58318 +58319,58319 +58320,58320 +58321,58321 +58322,58322 +58323,58323 +58324,58324 +58325,58325 +58326,58326 +58327,58327 +58328,58328 +58329,58329 +58330,58330 +58331,58331 +58332,58332 +33790,58332 +58333,58333 +58334,58334 +58335,58335 +58336,58336 +58337,58337 +58338,58338 +58339,58339 +58340,58340 +58341,58341 +58342,58342 +58343,58343 +58344,58344 +58345,58345 +58346,58346 +58347,58347 +58348,58348 +58349,58349 +58350,58350 +58351,58351 +58352,58352 +58353,58353 +58354,58354 +58355,58355 +58356,58356 +58357,58357 +58358,58358 +58359,58359 +58360,58360 +58361,58361 +58362,58362 +58363,58363 +58400,58400 +58401,58401 +98210,58402 +58402,58402 +58403,58403 +58404,58404 +58405,58405 +58406,58406 +111016,58406 +58407,58407 +58408,58408 +58409,58409 +58410,58410 +58411,58411 +58412,58412 +58413,58413 +58414,58414 +58415,58415 +58416,58416 +248912,58416 +58417,58417 +58418,58418 +58419,58419 +58420,58420 +58421,58421 +201116,58421 +58422,58422 +58423,58423 +58424,58424 +202501,58424 +58425,58425 +58426,58426 +141870,58427 +58427,58427 +58428,58428 +58429,58429 +58430,58430 +58431,58431 +58432,58432 +58433,58433 +58434,58434 +163521,58435 +58435,58435 +58436,58436 +58437,58437 +58438,58438 +58439,58439 +58440,58440 +58441,58441 +58442,58442 +58443,58443 +58444,58444 +58445,58445 +58446,58446 +58447,58447 +58448,58448 +58449,58449 +58450,58450 +80688,58450 +58451,58451 +58452,58452 +58453,58453 +58454,58454 +58455,58455 +58456,58456 +58457,58457 +58458,58458 +58459,58459 +58460,58460 +58461,58461 +58462,58462 +58463,58463 +58464,58464 +58465,58465 +58466,58466 +58467,58467 +58468,58468 +58469,58469 +58470,58470 +58471,58471 +58472,58472 +58473,58473 +58474,58474 +58475,58475 +58476,58476 +58477,58477 +58478,58478 +227516,58478 +58479,58479 +58480,58480 +58481,58481 +58482,58482 +58483,58483 +58484,58484 +162865,58484 +217834,58484 +58485,58485 +58486,58486 +58605,58605 +58606,58606 +58607,58607 +58608,58608 +58609,58609 +58610,58610 +58611,58611 +192281,58612 +58612,58612 +58613,58613 +58614,58614 +58615,58615 +58616,58616 +58617,58617 +58618,58618 +58619,58619 +58620,58620 +58621,58621 +58622,58622 +102066,58622 +58623,58623 +58624,58624 +58625,58625 +58626,58626 +58627,58627 +58628,58628 +58629,58629 +58630,58630 +58631,58631 +58632,58632 +58633,58633 +58634,58634 +58635,58635 +122210,58635 +58636,58636 +58637,58637 +58638,58638 +58639,58639 +140157,58639 +58640,58640 +58641,58641 +58642,58642 +58643,58643 +58644,58644 +58645,58645 +58646,58646 +59020,58646 +59021,58647 +58647,58647 +59022,58648 +58648,58648 +59023,58649 +58649,58649 +110600,58650 +59024,58650 +58650,58650 +59025,58651 +58651,58651 +59026,58652 +58652,58652 +59027,58653 +58653,58653 +59028,58654 +58654,58654 +59029,58655 +58655,58655 +58656,58656 +59030,58656 +58657,58657 +59031,58657 +58658,58658 +59032,58658 +58659,58659 +59033,58659 +59034,58660 +58660,58660 +58661,58661 +59035,58661 +59036,58662 +58662,58662 +58663,58663 +59037,58663 +146694,58663 +58664,58664 +133095,58665 +125508,58665 +58665,58665 +58666,58666 +58667,58667 +58668,58668 +58669,58669 +58670,58670 +58671,58671 +58672,58672 +58673,58673 +58674,58674 +58675,58675 +58676,58676 +231956,58676 +58677,58677 +58678,58678 +181096,58678 +58679,58679 +58680,58680 +58681,58681 +58682,58682 +58683,58683 +58684,58684 +58685,58685 +58686,58686 +58687,58687 +84111,58687 +58688,58688 +58689,58689 +207909,58689 +58690,58690 +58691,58691 +238001,58691 +58692,58692 +58693,58693 +35595,58693 +59104,58730 +58730,58730 +56222,58730 +48118,58730 +59105,58731 +58731,58731 +141885,58731 +59106,58732 +58732,58732 +238434,58732 +59107,58733 +58733,58733 +59108,58734 +58734,58734 +59109,58735 +58735,58735 +59110,58736 +58736,58736 +59111,58737 +58737,58737 +59112,58738 +58738,58738 +59113,58739 +58739,58739 +59114,58740 +58740,58740 +59115,58741 +58741,58741 +59116,58742 +58742,58742 +59117,58743 +58743,58743 +7699,58743 +49495,58743 +41302,58743 +59118,58744 +58744,58744 +68629,58744 +59119,58745 +58745,58745 +59120,58746 +58746,58746 +59121,58747 +58747,58747 +59122,58748 +58748,58748 +59123,58749 +58749,58749 +110928,58749 +59124,58750 +58750,58750 +59125,58751 +58751,58751 +59126,58752 +58752,58752 +24019,58752 +59127,58753 +58753,58753 +59128,58754 +58754,58754 +59129,58755 +58755,58755 +59130,58756 +58756,58756 +24023,58756 +77584,58756 +59131,58757 +58757,58757 +59132,58758 +58758,58758 +34016,58758 +59133,58759 +58759,58759 +59134,58760 +58760,58760 +57788,58760 +59135,58761 +58761,58761 +59136,58762 +58762,58762 +2345,58762 +59137,58763 +58763,58763 +59138,58764 +58764,58764 +59139,58765 +58765,58765 +59140,58766 +58766,58766 +59141,58767 +58767,58767 +180486,58767 +59142,58768 +58768,58768 +59143,58769 +58769,58769 +59144,58770 +58770,58770 +162700,58770 +59145,58771 +58771,58771 +59146,58772 +58772,58772 +59147,58773 +58773,58773 +59148,58774 +58774,58774 +11453,58774 +59149,58775 +58775,58775 +59150,58776 +58776,58776 +59151,58777 +58777,58777 +2303,58777 +59152,58778 +58778,58778 +59153,58779 +58779,58779 +59154,58780 +58780,58780 +59155,58781 +58781,58781 +59156,58782 +58782,58782 +59157,58783 +58783,58783 +104297,58783 +59158,58784 +58784,58784 +59159,58785 +58785,58785 +16260,58785 +59160,58786 +58786,58786 +59161,58787 +58787,58787 +59162,58788 +58788,58788 +59163,58789 +58789,58789 +59164,58790 +58790,58790 +59165,58791 +58791,58791 +59166,58792 +58792,58792 +123057,58792 +59167,58793 +58793,58793 +162883,58793 +59168,58794 +58794,58794 +59169,58795 +58795,58795 +59170,58796 +58796,58796 +59171,58797 +58797,58797 +20927,58797 +59172,58798 +58798,58798 +59173,58799 +58799,58799 +59174,58800 +58800,58800 +59175,58801 +58801,58801 +140995,58801 +59176,58802 +58802,58802 +59177,58803 +58803,58803 +59178,58804 +58804,58804 +114893,58804 +59179,58805 +58805,58805 +59180,58806 +58806,58806 +59181,58807 +58807,58807 +59182,58808 +58808,58808 +59183,58809 +58809,58809 +59184,58810 +58810,58810 +59185,58811 +58811,58811 +59186,58812 +58812,58812 +59187,58813 +58813,58813 +59188,58814 +58814,58814 +59189,58815 +58815,58815 +59190,58816 +58816,58816 +59191,58817 +58817,58817 +108789,58818 +59192,58818 +58818,58818 +59193,58819 +58819,58819 +59194,58820 +58820,58820 +59195,58821 +58821,58821 +59196,58822 +58822,58822 +59197,58823 +58823,58823 +39405,58823 +37921,58823 +59198,58824 +58824,58824 +59199,58825 +58825,58825 +59200,58826 +58826,58826 +59201,58827 +58827,58827 +59202,58828 +58828,58828 +59203,58829 +58829,58829 +59204,58830 +58830,58830 +59005,59005 +59006,59006 +59007,59007 +59008,59008 +59009,59009 +59010,59010 +59011,59011 +59012,59012 +59013,59013 +59014,59014 +59015,59015 +59016,59016 +59017,59017 +59018,59018 +59019,59019 +59020,59020 +30168,59020 +58646,59020 +59021,59021 +58647,59021 +59022,59022 +58648,59022 +59023,59023 +58649,59023 +110600,59024 +59024,59024 +58650,59024 +59025,59025 +58651,59025 +59026,59026 +58652,59026 +59027,59027 +58653,59027 +59028,59028 +58654,59028 +59029,59029 +58655,59029 +59030,59030 +58656,59030 +59031,59031 +58657,59031 +59032,59032 +58658,59032 +59033,59033 +58659,59033 +59034,59034 +58660,59034 +59035,59035 +58661,59035 +59036,59036 +58662,59036 +59037,59037 +58663,59037 +59104,59104 +58730,59104 +56222,59104 +48118,59104 +59105,59105 +58731,59105 +141885,59105 +59106,59106 +58732,59106 +238434,59106 +59107,59107 +58733,59107 +59108,59108 +58734,59108 +59109,59109 +58735,59109 +59110,59110 +58736,59110 +59111,59111 +58737,59111 +59112,59112 +58738,59112 +59113,59113 +58739,59113 +59114,59114 +58740,59114 +59115,59115 +58741,59115 +59116,59116 +58742,59116 +59117,59117 +58743,59117 +7699,59117 +49495,59117 +41302,59117 +59118,59118 +58744,59118 +68629,59118 +59119,59119 +58745,59119 +59120,59120 +58746,59120 +59121,59121 +58747,59121 +59122,59122 +58748,59122 +59123,59123 +58749,59123 +110928,59123 +59124,59124 +58750,59124 +59125,59125 +58751,59125 +59126,59126 +58752,59126 +24019,59126 +59127,59127 +58753,59127 +59128,59128 +58754,59128 +59129,59129 +58755,59129 +59130,59130 +58756,59130 +24023,59130 +77584,59130 +59131,59131 +58757,59131 +59132,59132 +58758,59132 +34016,59132 +59133,59133 +58759,59133 +59134,59134 +58760,59134 +57788,59134 +59135,59135 +58761,59135 +59136,59136 +58762,59136 +2345,59136 +59137,59137 +58763,59137 +59138,59138 +58764,59138 +59139,59139 +58765,59139 +59140,59140 +58766,59140 +59141,59141 +58767,59141 +180486,59141 +59142,59142 +58768,59142 +59143,59143 +58769,59143 +59144,59144 +58770,59144 +162700,59144 +59145,59145 +58771,59145 +59146,59146 +58772,59146 +59147,59147 +58773,59147 +59148,59148 +58774,59148 +11453,59148 +59149,59149 +58775,59149 +59150,59150 +58776,59150 +59151,59151 +58777,59151 +2303,59151 +59152,59152 +58778,59152 +59153,59153 +58779,59153 +59154,59154 +58780,59154 +59155,59155 +58781,59155 +59156,59156 +58782,59156 +59157,59157 +58783,59157 +104297,59157 +59158,59158 +58784,59158 +59159,59159 +58785,59159 +16260,59159 +59160,59160 +58786,59160 +59161,59161 +58787,59161 +59162,59162 +58788,59162 +59163,59163 +58789,59163 +59164,59164 +58790,59164 +59165,59165 +58791,59165 +59166,59166 +58792,59166 +123057,59166 +59167,59167 +58793,59167 +162883,59167 +59168,59168 +58794,59168 +59169,59169 +58795,59169 +59170,59170 +58796,59170 +59171,59171 +58797,59171 +20927,59171 +59172,59172 +58798,59172 +59173,59173 +58799,59173 +59174,59174 +58800,59174 +59175,59175 +58801,59175 +140995,59175 +59176,59176 +58802,59176 +59177,59177 +58803,59177 +59178,59178 +58804,59178 +114893,59178 +59179,59179 +58805,59179 +59180,59180 +58806,59180 +59181,59181 +58807,59181 +59182,59182 +58808,59182 +59183,59183 +58809,59183 +59184,59184 +58810,59184 +59185,59185 +58811,59185 +59186,59186 +58812,59186 +59187,59187 +58813,59187 +59188,59188 +58814,59188 +59189,59189 +58815,59189 +59190,59190 +58816,59190 +59191,59191 +58817,59191 +108789,59192 +59192,59192 +58818,59192 +59193,59193 +58819,59193 +59194,59194 +58820,59194 +59195,59195 +58821,59195 +59196,59196 +58822,59196 +59197,59197 +58823,59197 +39405,59197 +37921,59197 +59198,59198 +58824,59198 +59199,59199 +58825,59199 +59200,59200 +58826,59200 +59201,59201 +58827,59201 +59202,59202 +58828,59202 +59203,59203 +58829,59203 +59204,59204 +58830,59204 +59245,59245 +59246,59246 +59247,59247 +59348,59348 +59349,59349 +59350,59350 +59351,59351 +59352,59352 +59353,59353 +59354,59354 +59355,59355 +59356,59356 +59357,59357 +59358,59358 +59359,59359 +59360,59360 +59361,59361 +59362,59362 +59363,59363 +59364,59364 +59365,59365 +59366,59366 +59367,59367 +59368,59368 +59369,59369 +59370,59370 +247362,59370 +59371,59371 +59372,59372 +59373,59373 +59374,59374 +133193,59375 +125606,59375 +59375,59375 +2443,59375 +59376,59376 +59377,59377 +59378,59378 +59379,59379 +59380,59380 +59381,59381 +59382,59382 +59383,59383 +234803,59383 +59384,59384 +59385,59385 +59386,59386 +59387,59387 +59388,59388 +26661,59388 +59389,59389 +59390,59390 +59391,59391 +59392,59392 +59393,59393 +59394,59394 +59395,59395 +59396,59396 +59397,59397 +59398,59398 +59399,59399 +53770,59399 +45666,59399 +59400,59400 +206943,59400 +59401,59401 +59402,59402 +59403,59403 +111276,59403 +59404,59404 +59405,59405 +59406,59406 +59407,59407 +59408,59408 +20842,59408 +59409,59409 +59410,59410 +59411,59411 +59412,59412 +243402,59412 +59413,59413 +59414,59414 +59415,59415 +59416,59416 +59417,59417 +59418,59418 +59419,59419 +59420,59420 +59421,59421 +201390,59421 +59422,59422 +59423,59423 +59424,59424 +156270,59424 +59425,59425 +59426,59426 +59427,59427 +59428,59428 +59429,59429 +5415,59429 +59430,59430 +59431,59431 +59432,59432 +59433,59433 +59434,59434 +59435,59435 +59436,59436 +59437,59437 +66967,59437 +59438,59438 +59439,59439 +59440,59440 +59441,59441 +59442,59442 +59443,59443 +59444,59444 +110685,59444 +59445,59445 +59446,59446 +59447,59447 +59448,59448 +59449,59449 +59450,59450 +201696,59450 +59451,59451 +59477,59477 +59478,59478 +59479,59479 +29053,59479 +59480,59480 +59481,59481 +59482,59482 +59483,59483 +59484,59484 +59485,59485 +59486,59486 +59487,59487 +59488,59488 +59489,59489 +59490,59490 +59491,59491 +59492,59492 +138546,59492 +59493,59493 +59494,59494 +59495,59495 +59496,59496 +59497,59497 +59498,59498 +59499,59499 +59500,59500 +59501,59501 +59502,59502 +59503,59503 +59504,59504 +59505,59505 +209748,59505 +59506,59506 +59507,59507 +59508,59508 +59509,59509 +59510,59510 +59511,59511 +59512,59512 +59513,59513 +59514,59514 +59515,59515 +59516,59516 +59517,59517 +59518,59518 +59519,59519 +59520,59520 +59521,59521 +59522,59522 +194283,59523 +59523,59523 +59524,59524 +59525,59525 +248725,59525 +59526,59526 +83627,59526 +59527,59527 +59528,59528 +59529,59529 +59530,59530 +59531,59531 +59532,59532 +59533,59533 +59534,59534 +59535,59535 +201436,59535 +59536,59536 +231825,59536 +88615,59536 +59537,59537 +193414,59537 +59538,59538 +59539,59539 +59540,59540 +59541,59541 +59542,59542 +7293,59542 +59543,59543 +59544,59544 +193421,59544 +59545,59545 +59546,59546 +59547,59547 +59548,59548 +59549,59549 +59550,59550 +59551,59551 +59552,59552 +59553,59553 +59554,59554 +59555,59555 +122888,59555 +59556,59556 +59557,59557 +59558,59558 +59559,59559 +59560,59560 +59561,59561 +59562,59562 +233539,59562 +59563,59563 +59564,59564 +59565,59565 +59566,59566 +59567,59567 +59568,59568 +59569,59569 +59570,59570 +59571,59571 +59572,59572 +59573,59573 +59574,59574 +59575,59575 +59576,59576 +59577,59577 +59578,59578 +59579,59579 +59580,59580 +59581,59581 +59618,59618 +59619,59619 +59620,59620 +59621,59621 +59622,59622 +59623,59623 +59624,59624 +59625,59625 +59626,59626 +59627,59627 +156628,59627 +59628,59628 +59629,59629 +59630,59630 +59631,59631 +59632,59632 +59633,59633 +59634,59634 +59635,59635 +192611,59635 +59636,59636 +59637,59637 +59638,59638 +59639,59639 +59640,59640 +59641,59641 +59642,59642 +196512,59642 +79888,59642 +101031,59642 +59643,59643 +59644,59644 +59645,59645 +59646,59646 +59647,59647 +230344,59647 +59648,59648 +59649,59649 +59650,59650 +203024,59650 +59651,59651 +59652,59652 +59653,59653 +59654,59654 +205809,59654 +59655,59655 +59656,59656 +59657,59657 +59658,59658 +59659,59659 +59660,59660 +59661,59661 +59662,59662 +59663,59663 +241575,59663 +59664,59664 +59665,59665 +59666,59666 +59667,59667 +59668,59668 +59669,59669 +59670,59670 +108755,59670 +59671,59671 +59672,59672 +59673,59673 +59674,59674 +59675,59675 +59676,59676 +59677,59677 +59678,59678 +59679,59679 +59680,59680 +59681,59681 +59682,59682 +59683,59683 +59684,59684 +59685,59685 +59686,59686 +59687,59687 +59688,59688 +59689,59689 +59690,59690 +69838,59690 +59691,59691 +59692,59692 +59693,59693 +59694,59694 +59695,59695 +59696,59696 +59697,59697 +59698,59698 +71084,59699 +59699,59699 +59700,59700 +59701,59701 +59702,59702 +59703,59703 +59704,59704 +82762,59704 +59705,59705 +59706,59706 +191959,59706 +59707,59707 +59708,59708 +59709,59709 +59710,59710 +59711,59711 +59712,59712 +59713,59713 +59714,59714 +82337,59714 +59715,59715 +59716,59716 +59717,59717 +59718,59718 +59719,59719 +59720,59720 +59721,59721 +59722,59722 +59723,59723 +140245,59723 +59724,59724 +59725,59725 +59726,59726 +59727,59727 +59728,59728 +59729,59729 +59730,59730 +59731,59731 +59732,59732 +59733,59733 +59734,59734 +59735,59735 +59736,59736 +59737,59737 +59738,59738 +59739,59739 +59740,59740 +59741,59741 +59742,59742 +59743,59743 +59744,59744 +59745,59745 +59746,59746 +59747,59747 +59748,59748 +80117,59748 +59749,59749 +59750,59750 +59751,59751 +59752,59752 +59753,59753 +59754,59754 +59755,59755 +59756,59756 +59757,59757 +59758,59758 +59759,59759 +59760,59760 +59761,59761 +59762,59762 +59763,59763 +106253,59763 +59764,59764 +59765,59765 +59766,59766 +59767,59767 +59768,59768 +59769,59769 +59770,59770 +59771,59771 +59772,59772 +59773,59773 +59774,59774 +59775,59775 +59776,59776 +153638,59776 +218663,59776 +59777,59777 +142119,59777 +208421,59777 +59778,59778 +190667,59778 +59779,59779 +59780,59780 +59781,59781 +59782,59782 +206913,59782 +59783,59783 +59784,59784 +59785,59785 +246371,59785 +59786,59786 +59787,59787 +59788,59788 +59867,59867 +59868,59868 +59869,59869 +59870,59870 +59871,59871 +59872,59872 +59873,59873 +59874,59874 +59875,59875 +59876,59876 +59877,59877 +59878,59878 +59879,59879 +59880,59880 +59881,59881 +59882,59882 +59883,59883 +59884,59884 +67751,59884 +59885,59885 +59886,59886 +59887,59887 +59888,59888 +59889,59889 +59890,59890 +59891,59891 +59892,59892 +59893,59893 +204873,59893 +59894,59894 +59895,59895 +59896,59896 +59897,59897 +59898,59898 +59899,59899 +59900,59900 +59901,59901 +59902,59902 +59903,59903 +59904,59904 +12941,59904 +59905,59905 +59906,59906 +59907,59907 +59908,59908 +59909,59909 +59910,59910 +115958,59910 +59911,59911 +59912,59912 +59913,59913 +59914,59914 +59915,59915 +59916,59916 +59917,59917 +59918,59918 +65161,59918 +59919,59919 +59920,59920 +59921,59921 +59922,59922 +59923,59923 +59924,59924 +59925,59925 +59926,59926 +59927,59927 +59928,59928 +59929,59929 +59930,59930 +59931,59931 +59932,59932 +59933,59933 +59934,59934 +59935,59935 +59936,59936 +59937,59937 +59938,59938 +59939,59939 +59940,59940 +59941,59941 +204624,59941 +59942,59942 +59943,59943 +59944,59944 +59945,59945 +59946,59946 +59947,59947 +59948,59948 +59949,59949 +59950,59950 +59951,59951 +59952,59952 +59953,59953 +59954,59954 +59955,59955 +59956,59956 +59957,59957 +59958,59958 +59959,59959 +68217,59959 +59960,59960 +59961,59961 +59962,59962 +59963,59963 +59964,59964 +59965,59965 +59966,59966 +59967,59967 +105065,59967 +59968,59968 +59969,59969 +59970,59970 +59971,59971 +59972,59972 +59973,59973 +59974,59974 +59975,59975 +199732,59975 +59976,59976 +59977,59977 +59978,59978 +59979,59979 +59980,59980 +59981,59981 +59982,59982 +59983,59983 +59984,59984 +59985,59985 +59986,59986 +59987,59987 +136591,59988 +129004,59988 +59988,59988 +59989,59989 +59990,59990 +5071,59990 +59991,59991 +59992,59992 +59993,59993 +59994,59994 +156402,59994 +59995,59995 +59996,59996 +59997,59997 +59998,59998 +176550,59998 +170586,59998 +59999,59999 +60000,60000 +60001,60001 +60002,60002 +60003,60003 +60004,60004 +60005,60005 +60006,60006 +60007,60007 +214692,60007 +60008,60008 +60009,60009 +60010,60010 +60011,60011 +60012,60012 +60013,60013 +60014,60014 +60015,60015 +60016,60016 +60017,60017 +60018,60018 +60019,60019 +158470,60019 +60020,60020 +60021,60021 +60022,60022 +60023,60023 +60024,60024 +60025,60025 +60026,60026 +60027,60027 +60028,60028 +60029,60029 +60030,60030 +60031,60031 +60032,60032 +232955,60032 +60033,60033 +60034,60034 +60035,60035 +140639,60035 +60036,60036 +60037,60037 +60038,60038 +84551,60038 +60039,60039 +60040,60040 +60041,60041 +159455,60041 +60042,60042 +60043,60043 +60044,60044 +60045,60045 +60046,60046 +60047,60047 +60048,60048 +60049,60049 +60050,60050 +60051,60051 +60052,60052 +60053,60053 +60054,60054 +60055,60055 +60056,60056 +60057,60057 +60058,60058 +60059,60059 +100552,60059 +60060,60060 +60061,60061 +60062,60062 +60063,60063 +157536,60063 +60064,60064 +230259,60064 +2676,60064 +60065,60065 +36952,60065 +60066,60066 +60067,60067 +8083,60067 +60104,60104 +60105,60105 +60106,60106 +60107,60107 +60108,60108 +60109,60109 +60110,60110 +60111,60111 +60112,60112 +10065,60112 +60113,60113 +60114,60114 +60115,60115 +60116,60116 +60117,60117 +60118,60118 +60119,60119 +60120,60120 +60121,60121 +60122,60122 +60123,60123 +60124,60124 +60125,60125 +60126,60126 +60127,60127 +60128,60128 +60129,60129 +60130,60130 +60131,60131 +60132,60132 +60133,60133 +60134,60134 +30982,60134 +60135,60135 +60136,60136 +134127,60136 +126540,60136 +60137,60137 +60138,60138 +60139,60139 +60140,60140 +60141,60141 +60142,60142 +205571,60142 +60143,60143 +60144,60144 +60145,60145 +153179,60145 +60146,60146 +60147,60147 +60148,60148 +60149,60149 +60150,60150 +60151,60151 +60152,60152 +60153,60153 +60154,60154 +60202,60202 +60203,60203 +60204,60204 +60205,60205 +60206,60206 +60207,60207 +60208,60208 +60209,60209 +60210,60210 +60211,60211 +60212,60212 +60213,60213 +60214,60214 +60215,60215 +60216,60216 +60217,60217 +60218,60218 +60219,60219 +60220,60220 +60221,60221 +60222,60222 +60223,60223 +60224,60224 +60225,60225 +60226,60226 +60227,60227 +229703,60227 +60228,60228 +60229,60229 +60230,60230 +60231,60231 +645,60231 +60232,60232 +60233,60233 +60234,60234 +60235,60235 +60236,60236 +60237,60237 +60238,60238 +60239,60239 +60240,60240 +60241,60241 +74164,60241 +60242,60242 +60243,60243 +60244,60244 +60245,60245 +60246,60246 +153924,60246 +60247,60247 +60248,60248 +74171,60248 +60249,60249 +60250,60250 +60251,60251 +60252,60252 +60253,60253 +60254,60254 +145091,60254 +189485,60254 +60255,60255 +60256,60256 +53919,60256 +45815,60256 +60257,60257 +60258,60258 +60259,60259 +60260,60260 +60261,60261 +60262,60262 +60263,60263 +60264,60264 +60265,60265 +60266,60266 +60267,60267 +60268,60268 +212301,60268 +60269,60269 +60270,60270 +60271,60271 +60272,60272 +60273,60273 +60274,60274 +60275,60275 +60276,60276 +60341,60341 +60342,60342 +60343,60343 +156418,60343 +60344,60344 +60345,60345 +60346,60346 +60347,60347 +60348,60348 +60349,60349 +60350,60350 +60351,60351 +97915,60351 +60352,60352 +60353,60353 +60354,60354 +52304,60354 +44200,60354 +60355,60355 +60356,60356 +60357,60357 +60358,60358 +60359,60359 +60360,60360 +60361,60361 +60362,60362 +60363,60363 +60364,60364 +9136,60364 +60365,60365 +188509,60365 +235133,60365 +60366,60366 +60367,60367 +60368,60368 +60369,60369 +60370,60370 +60371,60371 +60372,60372 +24582,60372 +60373,60373 +60374,60374 +60375,60375 +236818,60375 +60376,60376 +60377,60377 +60378,60378 +60379,60379 +107640,60379 +60380,60380 +60381,60381 +60382,60382 +60383,60383 +60384,60384 +60385,60385 +60386,60386 +60387,60387 +60388,60388 +60389,60389 +60390,60390 +60391,60391 +60392,60392 +60393,60393 +60394,60394 +60395,60395 +60396,60396 +102203,60396 +60397,60397 +60398,60398 +410,60398 +60399,60399 +60400,60400 +60401,60401 +60402,60402 +60403,60403 +60404,60404 +60405,60405 +60406,60406 +60407,60407 +60408,60408 +60409,60409 +60410,60410 +60411,60411 +60412,60412 +84673,60412 +60413,60413 +60414,60414 +60415,60415 +60416,60416 +60417,60417 +60418,60418 +60419,60419 +60420,60420 +60421,60421 +60422,60422 +60423,60423 +60424,60424 +60425,60425 +60426,60426 +60427,60427 +60428,60428 +60429,60429 +60430,60430 +60431,60431 +60432,60432 +60433,60433 +60434,60434 +180655,60435 +60435,60435 +103617,60435 +60436,60436 +60437,60437 +60438,60438 +150807,60438 +60439,60439 +60440,60440 +60441,60441 +60442,60442 +60443,60443 +60444,60444 +115786,60444 +60445,60445 +60446,60446 +60447,60447 +60448,60448 +60449,60449 +60450,60450 +60451,60451 +60452,60452 +60453,60453 +60454,60454 +60455,60455 +60456,60456 +60457,60457 +60458,60458 +60517,60517 +84929,60517 +60560,60560 +60561,60561 +60562,60562 +60563,60563 +60564,60564 +60565,60565 +60566,60566 +60567,60567 +60568,60568 +60569,60569 +60570,60570 +60571,60571 +60572,60572 +165066,60572 +72495,60572 +60573,60573 +60574,60574 +60575,60575 +60576,60576 +60577,60577 +60578,60578 +105722,60578 +60579,60579 +60580,60580 +60581,60581 +60582,60582 +60583,60583 +60584,60584 +60585,60585 +60586,60586 +60587,60587 +60588,60588 +60589,60589 +60590,60590 +60591,60591 +60592,60592 +60593,60593 +60594,60594 +60595,60595 +60596,60596 +60597,60597 +60598,60598 +60599,60599 +60600,60600 +60601,60601 +60602,60602 +60603,60603 +60604,60604 +60605,60605 +60606,60606 +60607,60607 +60608,60608 +60609,60609 +60610,60610 +60611,60611 +60612,60612 +60613,60613 +60614,60614 +60615,60615 +60616,60616 +60617,60617 +60618,60618 +60619,60619 +60620,60620 +60621,60621 +60622,60622 +60623,60623 +60624,60624 +60625,60625 +60626,60626 +60627,60627 +60628,60628 +60629,60629 +235037,60629 +60630,60630 +60631,60631 +60632,60632 +60633,60633 +60634,60634 +60635,60635 +60636,60636 +60637,60637 +60638,60638 +60639,60639 +60640,60640 +60641,60641 +60642,60642 +60643,60643 +60644,60644 +60645,60645 +60646,60646 +60647,60647 +60648,60648 +60649,60649 +60650,60650 +60651,60651 +33342,60651 +60652,60652 +60653,60653 +60654,60654 +60655,60655 +8861,60655 +60656,60656 +2474,60656 +60657,60657 +60658,60658 +60659,60659 +60660,60660 +60661,60661 +60662,60662 +60663,60663 +60664,60664 +60665,60665 +60666,60666 +60667,60667 +60668,60668 +60669,60669 +60670,60670 +60671,60671 +60672,60672 +60673,60673 +60674,60674 +60675,60675 +60676,60676 +60677,60677 +60678,60678 +60679,60679 +60680,60680 +60681,60681 +213568,60681 +60682,60682 +60683,60683 +60684,60684 +60685,60685 +60686,60686 +60687,60687 +137092,60687 +129505,60687 +60730,60730 +60731,60731 +60732,60732 +60733,60733 +60734,60734 +60735,60735 +60736,60736 +33349,60736 +60737,60737 +60738,60738 +60739,60739 +60740,60740 +60741,60741 +60742,60742 +60743,60743 +79075,60743 +60744,60744 +60745,60745 +60746,60746 +60747,60747 +60748,60748 +60749,60749 +60750,60750 +60751,60751 +60752,60752 +60753,60753 +60754,60754 +60755,60755 +60756,60756 +60757,60757 +192332,60757 +60758,60758 +135262,60758 +127675,60758 +60759,60759 +60760,60760 +60761,60761 +60762,60762 +60763,60763 +60809,60809 +60810,60810 +60811,60811 +188701,60811 +60812,60812 +60813,60813 +60814,60814 +60815,60815 +60816,60816 +60817,60817 +60818,60818 +60819,60819 +60820,60820 +60821,60821 +60822,60822 +60823,60823 +60824,60824 +60825,60825 +60826,60826 +60827,60827 +60828,60828 +60829,60829 +71350,60829 +60830,60830 +60831,60831 +60832,60832 +60833,60833 +60834,60834 +60835,60835 +60836,60836 +60837,60837 +60838,60838 +60839,60839 +60840,60840 +60841,60841 +60842,60842 +60843,60843 +60844,60844 +60845,60845 +60846,60846 +60847,60847 +60848,60848 +60849,60849 +60850,60850 +60851,60851 +60919,60919 +60920,60920 +60921,60921 +222108,60921 +60922,60922 +60923,60923 +60924,60924 +174356,60924 +168392,60924 +60925,60925 +60926,60926 +60927,60927 +60928,60928 +112303,60928 +60929,60929 +60930,60930 +60931,60931 +60932,60932 +60933,60933 +60934,60934 +60935,60935 +38246,60935 +60936,60936 +60937,60937 +60938,60938 +60939,60939 +60940,60940 +60941,60941 +60942,60942 +60943,60943 +60944,60944 +60945,60945 +60946,60946 +162380,60946 +60947,60947 +60948,60948 +60949,60949 +60950,60950 +60951,60951 +60952,60952 +158456,60952 +60953,60953 +60954,60954 +60955,60955 +60956,60956 +60957,60957 +60958,60958 +60959,60959 +158463,60959 +60960,60960 +60961,60961 +60962,60962 +60963,60963 +60964,60964 +60965,60965 +60966,60966 +60967,60967 +60968,60968 +60969,60969 +60970,60970 +60971,60971 +60972,60972 +60973,60973 +60974,60974 +60975,60975 +107046,60975 +60976,60976 +122173,60976 +60977,60977 +60978,60978 +60979,60979 +60980,60980 +60981,60981 +60982,60982 +60983,60983 +60984,60984 +213583,60984 +60985,60985 +242605,60985 +60986,60986 +60987,60987 +60988,60988 +60989,60989 +60990,60990 +60991,60991 +139158,60991 +60992,60992 +60993,60993 +60994,60994 +60995,60995 +60996,60996 +60997,60997 +60998,60998 +60999,60999 +61000,61000 +61001,61001 +61002,61002 +217833,61002 +61003,61003 +61004,61004 +61005,61005 +61006,61006 +61007,61007 +61008,61008 +61009,61009 +61010,61010 +61011,61011 +61012,61012 +61013,61013 +61014,61014 +61015,61015 +214397,61015 +61016,61016 +61017,61017 +61018,61018 +61019,61019 +61020,61020 +61021,61021 +61022,61022 +61023,61023 +61024,61024 +61025,61025 +61026,61026 +61027,61027 +61028,61028 +61029,61029 +61030,61030 +61031,61031 +61032,61032 +61033,61033 +61034,61034 +61035,61035 +61036,61036 +61037,61037 +61038,61038 +61039,61039 +61040,61040 +61041,61041 +61042,61042 +61043,61043 +61044,61044 +61183,61183 +61184,61184 +61185,61185 +61186,61186 +247976,61186 +61187,61187 +61188,61188 +61189,61189 +61190,61190 +61191,61191 +61192,61192 +61193,61193 +61194,61194 +61195,61195 +61196,61196 +61197,61197 +61198,61198 +238078,61198 +61199,61199 +61200,61200 +61201,61201 +80235,61201 +61202,61202 +61346,61346 +61347,61347 +61348,61348 +61349,61349 +61350,61350 +61351,61351 +61352,61352 +61353,61353 +61354,61354 +61355,61355 +61356,61356 +61357,61357 +61358,61358 +156274,61358 +61359,61359 +61360,61360 +61361,61361 +61362,61362 +61363,61363 +61364,61364 +61365,61365 +61366,61366 +113423,61366 +61367,61367 +61368,61368 +61369,61369 +61370,61370 +61371,61371 +77692,61371 +61372,61372 +61427,61427 +225474,61427 +61428,61428 +61429,61429 +61430,61430 +61431,61431 +61432,61432 +145476,61432 +61433,61433 +61434,61434 +61435,61435 +61436,61436 +61437,61437 +61438,61438 +51132,61438 +43028,61438 +61439,61439 +61440,61440 +61441,61441 +61442,61442 +61443,61443 +61444,61444 +61445,61445 +61446,61446 +61447,61447 +61448,61448 +61449,61449 +61450,61450 +61451,61451 +61452,61452 +61453,61453 +61454,61454 +242645,61454 +61455,61455 +61456,61456 +61457,61457 +61458,61458 +61459,61459 +61460,61460 +61461,61461 +61462,61462 +61463,61463 +61464,61464 +61465,61465 +77562,61466 +61466,61466 +61467,61467 +61468,61468 +228701,61468 +61469,61469 +61470,61470 +61471,61471 +61472,61472 +61473,61473 +61474,61474 +61475,61475 +61476,61476 +61477,61477 +61478,61478 +61479,61479 +61480,61480 +61481,61481 +61482,61482 +61483,61483 +50219,61483 +42027,61483 +61484,61484 +248928,61484 +61485,61485 +61486,61486 +61487,61487 +61488,61488 +61489,61489 +61490,61490 +31637,61490 +61491,61491 +61492,61492 +61493,61493 +61726,61726 +61727,61727 +61728,61728 +61729,61729 +61730,61730 +61731,61731 +61732,61732 +49519,61732 +41326,61732 +61733,61733 +61734,61734 +61735,61735 +61736,61736 +61737,61737 +61738,61738 +145726,61738 +61739,61739 +61740,61740 +61741,61741 +61742,61742 +61743,61743 +61744,61744 +61745,61745 +61746,61746 +61747,61747 +61748,61748 +61749,61749 +61750,61750 +61751,61751 +61752,61752 +61753,61753 +61754,61754 +61755,61755 +61756,61756 +61757,61757 +61758,61758 +61759,61759 +61760,61760 +61761,61761 +61762,61762 +141388,61762 +61763,61763 +61764,61764 +61765,61765 +61766,61766 +61767,61767 +61768,61768 +61769,61769 +61770,61770 +61771,61771 +141791,61771 +121436,61771 +61772,61772 +61773,61773 +61774,61774 +61775,61775 +61776,61776 +61777,61777 +61778,61778 +61779,61779 +61780,61780 +61781,61781 +28877,61781 +61782,61782 +61783,61783 +222177,61783 +100567,61783 +61784,61784 +61785,61785 +61786,61786 +61787,61787 +61788,61788 +61789,61789 +61988,61988 +61989,61989 +61990,61990 +61991,61991 +61992,61992 +61993,61993 +61994,61994 +61995,61995 +61996,61996 +61997,61997 +61998,61998 +55974,61998 +47870,61998 +61999,61999 +62000,62000 +62001,62001 +62002,62002 +62003,62003 +62004,62004 +62005,62005 +160712,62005 +133101,62005 +125514,62005 +62006,62006 +62007,62007 +62008,62008 +62009,62009 +62010,62010 +62011,62011 +62012,62012 +146606,62013 +62013,62013 +62014,62014 +62015,62015 +62059,62059 +57358,62059 +62060,62060 +62061,62061 +62062,62062 +62063,62063 +62064,62064 +62065,62065 +62066,62066 +62067,62067 +62068,62068 +238276,62068 +62069,62069 +62070,62070 +62071,62071 +62072,62072 +62073,62073 +62074,62074 +62075,62075 +62076,62076 +62077,62077 +62078,62078 +62079,62079 +85970,62079 +62080,62080 +62081,62081 +5488,62081 +6650,62081 +62082,62082 +214057,62082 +62083,62083 +62084,62084 +62085,62085 +62086,62086 +62087,62087 +62088,62088 +62089,62089 +62090,62090 +62091,62091 +62092,62092 +62093,62093 +62094,62094 +62095,62095 +62096,62096 +62097,62097 +62098,62098 +56104,62098 +48000,62098 +62151,62151 +62152,62152 +62153,62153 +62154,62154 +62266,62266 +62267,62267 +154784,62267 +62268,62268 +62269,62269 +58005,62269 +62270,62270 +62271,62271 +62272,62272 +62273,62273 +62274,62274 +62275,62275 +62276,62276 +62277,62277 +156574,62277 +62278,62278 +62279,62279 +166624,62279 +62280,62280 +62281,62281 +62282,62282 +62283,62283 +62370,62370 +62371,62371 +62372,62372 +62373,62373 +62374,62374 +62375,62375 +62376,62376 +62377,62377 +62378,62378 +62379,62379 +62380,62380 +62381,62381 +62382,62382 +62383,62383 +62384,62384 +62385,62385 +62386,62386 +62387,62387 +62388,62388 +62389,62389 +62390,62390 +62391,62391 +184440,62391 +62392,62392 +204663,62392 +62393,62393 +62394,62394 +62395,62395 +62396,62396 +62397,62397 +62398,62398 +62399,62399 +62400,62400 +62401,62401 +62402,62402 +62403,62403 +62404,62404 +62405,62405 +62406,62406 +62407,62407 +62408,62408 +62409,62409 +62410,62410 +62411,62411 +8525,62411 +62412,62412 +62413,62413 +62414,62414 +62415,62415 +62416,62416 +62417,62417 +62418,62418 +62419,62419 +62420,62420 +62421,62421 +62422,62422 +62423,62423 +62424,62424 +62425,62425 +62426,62426 +62427,62427 +62428,62428 +62429,62429 +62430,62430 +62431,62431 +62432,62432 +62433,62433 +62434,62434 +36911,62434 +62435,62435 +62436,62436 +62437,62437 +62438,62438 +62439,62439 +62440,62440 +62441,62441 +62442,62442 +62443,62443 +62444,62444 +62445,62445 +62446,62446 +62447,62447 +62448,62448 +62449,62449 +62450,62450 +105671,62450 +62451,62451 +178982,62451 +62452,62452 +62453,62453 +62454,62454 +62455,62455 +62456,62456 +62457,62457 +62458,62458 +62459,62459 +62460,62460 +158798,62460 +62461,62461 +62462,62462 +62463,62463 +78060,62463 +191618,62463 +62464,62464 +62465,62465 +76337,62465 +62466,62466 +62467,62467 +62468,62468 +62469,62469 +62470,62470 +62471,62471 +203531,62471 +62472,62472 +62473,62473 +62474,62474 +62475,62475 +62476,62476 +62477,62477 +62478,62478 +62479,62479 +62480,62480 +62481,62481 +62482,62482 +62483,62483 +62484,62484 +62485,62485 +17202,62485 +62486,62486 +62487,62487 +62488,62488 +62489,62489 +62490,62490 +62491,62491 +62492,62492 +62493,62493 +62494,62494 +62495,62495 +62496,62496 +216087,62496 +62497,62497 +62498,62498 +62499,62499 +62500,62500 +62501,62501 +62502,62502 +62503,62503 +62504,62504 +62505,62505 +62506,62506 +62507,62507 +62508,62508 +62509,62509 +62510,62510 +62511,62511 +62512,62512 +62513,62513 +62514,62514 +62515,62515 +62516,62516 +62517,62517 +62518,62518 +62519,62519 +62520,62520 +62521,62521 +62522,62522 +62523,62523 +62524,62524 +62525,62525 +62526,62526 +62527,62527 +62528,62528 +62529,62529 +154543,62529 +62530,62530 +62531,62531 +62532,62532 +62533,62533 +62534,62534 +62535,62535 +62536,62536 +62537,62537 +62538,62538 +62539,62539 +62540,62540 +62541,62541 +62542,62542 +62543,62543 +62544,62544 +62545,62545 +62546,62546 +62547,62547 +62548,62548 +62549,62549 +62550,62550 +62551,62551 +62552,62552 +62553,62553 +62554,62554 +62555,62555 +62556,62556 +62557,62557 +62558,62558 +62559,62559 +62560,62560 +62561,62561 +62562,62562 +62563,62563 +188994,62563 +62564,62564 +62565,62565 +62566,62566 +62567,62567 +62568,62568 +62569,62569 +62570,62570 +62571,62571 +62572,62572 +62573,62573 +62574,62574 +62575,62575 +62576,62576 +62577,62577 +62578,62578 +62579,62579 +62580,62580 +62581,62581 +62582,62582 +62583,62583 +62584,62584 +62585,62585 +62586,62586 +62587,62587 +62588,62588 +62589,62589 +62590,62590 +62591,62591 +62592,62592 +62593,62593 +62594,62594 +62595,62595 +62596,62596 +103686,62596 +160405,62596 +62597,62597 +62598,62598 +62599,62599 +62600,62600 +62601,62601 +62602,62602 +62603,62603 +62604,62604 +62605,62605 +62606,62606 +62607,62607 +54983,62607 +46879,62607 +62608,62608 +229573,62609 +62609,62609 +62610,62610 +229574,62610 +62611,62611 +62612,62612 +211328,62612 +62613,62613 +62614,62614 +62615,62615 +62616,62616 +62617,62617 +62618,62618 +62667,62667 +180457,62667 +62668,62668 +62669,62669 +62670,62670 +62671,62671 +62672,62672 +62673,62673 +62674,62674 +62675,62675 +62676,62676 +62677,62677 +62678,62678 +62679,62679 +62680,62680 +62681,62681 +62682,62682 +62683,62683 +62684,62684 +62685,62685 +62686,62686 +62687,62687 +62688,62688 +62689,62689 +62690,62690 +187027,62690 +62691,62691 +62692,62692 +62693,62693 +62694,62694 +62695,62695 +62696,62696 +62697,62697 +62698,62698 +62699,62699 +62700,62700 +62701,62701 +62702,62702 +62703,62703 +62704,62704 +62705,62705 +62706,62706 +62707,62707 +62708,62708 +62709,62709 +62710,62710 +62711,62711 +62712,62712 +62713,62713 +206113,62713 +62714,62714 +62715,62715 +62716,62716 +62717,62717 +62718,62718 +62719,62719 +62720,62720 +62721,62721 +62722,62722 +62723,62723 +62724,62724 +62725,62725 +62726,62726 +62727,62727 +62728,62728 +62729,62729 +62730,62730 +62731,62731 +62732,62732 +62733,62733 +62734,62734 +62735,62735 +62736,62736 +62737,62737 +62738,62738 +62739,62739 +62740,62740 +62741,62741 +62742,62742 +62743,62743 +62744,62744 +62745,62745 +62746,62746 +62747,62747 +62748,62748 +62749,62749 +62750,62750 +62751,62751 +62752,62752 +62753,62753 +62754,62754 +62755,62755 +62756,62756 +62757,62757 +62758,62758 +62759,62759 +62760,62760 +62761,62761 +62762,62762 +191863,62762 +75407,62762 +62763,62763 +248926,62763 +62764,62764 +62765,62765 +154741,62765 +62766,62766 +62767,62767 +62768,62768 +62769,62769 +193278,62769 +62770,62770 +62771,62771 +6837,62771 +62772,62772 +87015,62772 +62773,62773 +62774,62774 +62775,62775 +62776,62776 +62777,62777 +62778,62778 +62779,62779 +62780,62780 +149340,62780 +62781,62781 +62782,62782 +62783,62783 +62784,62784 +62785,62785 +162958,62785 +62786,62786 +62787,62787 +62788,62788 +62789,62789 +62790,62790 +62791,62791 +62792,62792 +62793,62793 +62794,62794 +62795,62795 +162537,62795 +62796,62796 +62797,62797 +62798,62798 +62799,62799 +62800,62800 +62801,62801 +62802,62802 +62803,62803 +62804,62804 +62805,62805 +62806,62806 +62807,62807 +62808,62808 +62809,62809 +62810,62810 +62811,62811 +62812,62812 +62813,62813 +62814,62814 +62815,62815 +62816,62816 +62817,62817 +62818,62818 +62819,62819 +62820,62820 +62821,62821 +62822,62822 +62823,62823 +62824,62824 +62825,62825 +62826,62826 +62827,62827 +62828,62828 +62829,62829 +62830,62830 +62831,62831 +86717,62831 +62832,62832 +62833,62833 +62834,62834 +62835,62835 +62836,62836 +62837,62837 +62838,62838 +62839,62839 +62840,62840 +62841,62841 +62842,62842 +62843,62843 +62844,62844 +62845,62845 +62846,62846 +62847,62847 +62848,62848 +62849,62849 +62850,62850 +62851,62851 +62852,62852 +141646,62852 +62853,62853 +62854,62854 +62855,62855 +62856,62856 +62857,62857 +62858,62858 +62859,62859 +62860,62860 +62861,62861 +62862,62862 +62863,62863 +62864,62864 +62865,62865 +62866,62866 +62867,62867 +62868,62868 +187245,62868 +62869,62869 +62870,62870 +62871,62871 +62872,62872 +62873,62873 +62874,62874 +33052,62874 +62875,62875 +62876,62876 +62877,62877 +62878,62878 +62954,62954 +62955,62955 +62956,62956 +62957,62957 +62958,62958 +62959,62959 +62960,62960 +64820,62960 +62961,62961 +62962,62962 +62963,62963 +62964,62964 +62965,62965 +62966,62966 +62967,62967 +62968,62968 +62969,62969 +62970,62970 +62971,62971 +62972,62972 +62973,62973 +62974,62974 +62975,62975 +62976,62976 +62977,62977 +62978,62978 +62979,62979 +62980,62980 +62981,62981 +62982,62982 +62983,62983 +62984,62984 +62985,62985 +147685,62985 +62986,62986 +62987,62987 +62988,62988 +62989,62989 +62990,62990 +62991,62991 +62992,62992 +62993,62993 +62994,62994 +62995,62995 +62996,62996 +62997,62997 +62998,62998 +62999,62999 +63000,63000 +63001,63001 +63002,63002 +63003,63003 +63004,63004 +63005,63005 +63006,63006 +63007,63007 +63008,63008 +218416,63008 +63009,63009 +63010,63010 +63011,63011 +63012,63012 +63013,63013 +63014,63014 +63015,63015 +63016,63016 +63017,63017 +63066,63066 +63067,63067 +63068,63068 +63069,63069 +63070,63070 +63071,63071 +63072,63072 +63073,63073 +63074,63074 +63075,63075 +63076,63076 +63077,63077 +63078,63078 +63143,63143 +63144,63144 +63145,63145 +63146,63146 +63147,63147 +63148,63148 +63149,63149 +142597,63149 +63150,63150 +63151,63151 +63152,63152 +63153,63153 +63154,63154 +103492,63154 +63155,63155 +63156,63156 +63157,63157 +63158,63158 +63159,63159 +63160,63160 +63161,63161 +63162,63162 +63163,63163 +63164,63164 +63165,63165 +63166,63166 +63167,63167 +63168,63168 +110333,63169 +63169,63169 +63170,63170 +63171,63171 +63172,63172 +63299,63299 +63300,63300 +63398,63398 +63399,63399 +63400,63400 +63401,63401 +63402,63402 +174165,63403 +168201,63403 +63403,63403 +227703,63403 +19003,63403 +63404,63404 +63405,63405 +63406,63406 +63407,63407 +63408,63408 +63409,63409 +63410,63410 +63411,63411 +63412,63412 +63413,63413 +63414,63414 +63415,63415 +63416,63416 +63417,63417 +63418,63418 +63419,63419 +63420,63420 +63421,63421 +63422,63422 +63423,63423 +63424,63424 +63425,63425 +63426,63426 +63427,63427 +63428,63428 +63429,63429 +39199,63429 +63430,63430 +63431,63431 +63432,63432 +63433,63433 +63434,63434 +63435,63435 +63436,63436 +63437,63437 +63438,63438 +63439,63439 +63440,63440 +63441,63441 +63442,63442 +63443,63443 +63444,63444 +63445,63445 +63446,63446 +63447,63447 +63448,63448 +63449,63449 +63450,63450 +63451,63451 +63452,63452 +63453,63453 +63454,63454 +63455,63455 +63456,63456 +63457,63457 +137980,63457 +130393,63457 +63458,63458 +63459,63459 +63460,63460 +63461,63461 +63462,63462 +63463,63463 +63464,63464 +63465,63465 +63466,63466 +63467,63467 +63468,63468 +63469,63469 +63470,63470 +63471,63471 +63472,63472 +63473,63473 +63474,63474 +63475,63475 +63476,63476 +63477,63477 +227601,63477 +63478,63478 +63479,63479 +63480,63480 +63481,63481 +63482,63482 +63483,63483 +63484,63484 +63485,63485 +63486,63486 +63487,63487 +63488,63488 +63489,63489 +63490,63490 +63491,63491 +63492,63492 +63493,63493 +183543,63493 +63494,63494 +63495,63495 +63496,63496 +63497,63497 +63498,63498 +4690,63498 +63499,63499 +38274,63499 +63500,63500 +63501,63501 +63502,63502 +63503,63503 +63504,63504 +63505,63505 +63506,63506 +63507,63507 +63508,63508 +63509,63509 +63510,63510 +63511,63511 +63512,63512 +63513,63513 +63514,63514 +148438,63514 +63515,63515 +63516,63516 +102525,63516 +63517,63517 +63518,63518 +63519,63519 +63520,63520 +63572,63572 +63646,63646 +63647,63647 +63648,63648 +63649,63649 +63650,63650 +63651,63651 +63652,63652 +63653,63653 +34214,63653 +63654,63654 +63655,63655 +63656,63656 +231877,63656 +65001,63657 +63657,63657 +63658,63658 +63697,63697 +63698,63698 +63699,63699 +63700,63700 +63701,63701 +658,63701 +63702,63702 +63703,63703 +63704,63704 +63705,63705 +134076,63706 +126489,63706 +63706,63706 +63707,63707 +63708,63708 +63709,63709 +63710,63710 +63711,63711 +63712,63712 +63713,63713 +63714,63714 +63715,63715 +63716,63716 +63717,63717 +63718,63718 +63719,63719 +63720,63720 +63721,63721 +63722,63722 +63723,63723 +63724,63724 +63725,63725 +63726,63726 +63727,63727 +63728,63728 +63729,63729 +63730,63730 +63731,63731 +63732,63732 +63733,63733 +63734,63734 +63735,63735 +63736,63736 +28756,63736 +63737,63737 +63738,63738 +63739,63739 +63740,63740 +63741,63741 +63742,63742 +63743,63743 +63744,63744 +63745,63745 +63746,63746 +63747,63747 +63748,63748 +63749,63749 +63750,63750 +150287,63750 +63751,63751 +63752,63752 +63753,63753 +63754,63754 +63755,63755 +63756,63756 +63757,63757 +63758,63758 +63759,63759 +63760,63760 +63761,63761 +63762,63762 +123983,63762 +63763,63763 +63764,63764 +63765,63765 +63766,63766 +63767,63767 +63768,63768 +63769,63769 +63770,63770 +63771,63771 +63772,63772 +235165,63772 +63773,63773 +63774,63774 +63775,63775 +63776,63776 +63777,63777 +63778,63778 +63779,63779 +63780,63780 +63781,63781 +63782,63782 +63783,63783 +63784,63784 +63785,63785 +63786,63786 +63787,63787 +63788,63788 +63789,63789 +63790,63790 +178079,63790 +172115,63790 +63791,63791 +63792,63792 +63793,63793 +63794,63794 +63795,63795 +63796,63796 +63797,63797 +63798,63798 +63799,63799 +140486,63799 +63800,63800 +63801,63801 +63802,63802 +63803,63803 +63804,63804 +63805,63805 +63806,63806 +63807,63807 +63808,63808 +63809,63809 +63810,63810 +63811,63811 +63812,63812 +106266,63813 +63813,63813 +63814,63814 +63815,63815 +63816,63816 +63817,63817 +63818,63818 +63819,63819 +63820,63820 +63821,63821 +106275,63822 +63822,63822 +63823,63823 +63824,63824 +63825,63825 +63826,63826 +63827,63827 +63828,63828 +63829,63829 +63830,63830 +63831,63831 +63832,63832 +63833,63833 +63834,63834 +82623,63834 +63835,63835 +63836,63836 +63837,63837 +63838,63838 +63839,63839 +63840,63840 +63841,63841 +63842,63842 +63843,63843 +63844,63844 +63845,63845 +63846,63846 +63847,63847 +63848,63848 +63849,63849 +63850,63850 +63851,63851 +155423,63851 +63852,63852 +63853,63853 +63854,63854 +63855,63855 +63856,63856 +63857,63857 +63858,63858 +202089,63858 +63859,63859 +63860,63860 +63861,63861 +63862,63862 +63863,63863 +63864,63864 +63865,63865 +63866,63866 +63867,63867 +63868,63868 +63869,63869 +63870,63870 +63871,63871 +77119,63871 +63872,63872 +35039,63872 +63873,63873 +63874,63874 +63875,63875 +63876,63876 +63877,63877 +63913,63913 +63914,63914 +63915,63915 +63916,63916 +63917,63917 +63918,63918 +63954,63954 +63955,63955 +63956,63956 +63957,63957 +63958,63958 +63959,63959 +63960,63960 +63961,63961 +63962,63962 +63963,63963 +63964,63964 +63965,63965 +63966,63966 +63967,63967 +63968,63968 +63969,63969 +63970,63970 +204601,63970 +63971,63971 +63972,63972 +63973,63973 +63974,63974 +63975,63975 +63976,63976 +63977,63977 +63978,63978 +63979,63979 +63980,63980 +63981,63981 +233956,63981 +63982,63982 +63983,63983 +63984,63984 +63985,63985 +63986,63986 +63987,63987 +63988,63988 +63989,63989 +63990,63990 +63991,63991 +63992,63992 +63993,63993 +63994,63994 +63995,63995 +63996,63996 +63997,63997 +149773,63997 +63998,63998 +63999,63999 +64000,64000 +64001,64001 +64002,64002 +64003,64003 +64004,64004 +64005,64005 +64006,64006 +64007,64007 +64008,64008 +64009,64009 +64010,64010 +64011,64011 +64012,64012 +77152,64012 +64013,64013 +64014,64014 +64015,64015 +163064,64015 +64016,64016 +64017,64017 +64018,64018 +64019,64019 +64020,64020 +64021,64021 +64022,64022 +64023,64023 +64024,64024 +64025,64025 +64026,64026 +64027,64027 +64028,64028 +64029,64029 +64030,64030 +64031,64031 +64032,64032 +64033,64033 +64034,64034 +64035,64035 +139738,64036 +64036,64036 +64037,64037 +64038,64038 +64039,64039 +404,64039 +64040,64040 +64041,64041 +64042,64042 +64043,64043 +64044,64044 +64045,64045 +64081,64081 +64082,64082 +64083,64083 +64084,64084 +64085,64085 +165280,64085 +64086,64086 +64087,64087 +64088,64088 +64089,64089 +64090,64090 +64091,64091 +50010,64091 +41817,64091 +64092,64092 +64093,64093 +64094,64094 +64095,64095 +64096,64096 +64097,64097 +64098,64098 +64099,64099 +84839,64099 +64100,64100 +64101,64101 +64102,64102 +64103,64103 +103974,64103 +64104,64104 +64105,64105 +64106,64106 +201015,64106 +83331,64106 +64107,64107 +64108,64108 +64109,64109 +64110,64110 +64111,64111 +64112,64112 +64113,64113 +64114,64114 +64115,64115 +64116,64116 +161641,64116 +64117,64117 +64118,64118 +64119,64119 +136068,64119 +128481,64119 +64120,64120 +64121,64121 +57042,64121 +64122,64122 +64123,64123 +64124,64124 +64125,64125 +64126,64126 +64127,64127 +111781,64127 +64128,64128 +64129,64129 +64130,64130 +64131,64131 +64132,64132 +64133,64133 +64134,64134 +64135,64135 +64136,64136 +64137,64137 +64138,64138 +64139,64139 +64140,64140 +64141,64141 +64142,64142 +64143,64143 +64144,64144 +27351,64144 +138871,64145 +64145,64145 +64146,64146 +64147,64147 +64148,64148 +64149,64149 +64150,64150 +64151,64151 +64152,64152 +5114,64152 +233231,64153 +64153,64153 +64154,64154 +64155,64155 +64156,64156 +64157,64157 +64158,64158 +64159,64159 +64160,64160 +64161,64161 +64162,64162 +64163,64163 +64164,64164 +64165,64165 +64166,64166 +64167,64167 +64168,64168 +64169,64169 +64170,64170 +64171,64171 +64172,64172 +64173,64173 +64174,64174 +64175,64175 +120456,64175 +64176,64176 +64177,64177 +64178,64178 +64179,64179 +64180,64180 +64181,64181 +64182,64182 +64183,64183 +64184,64184 +64185,64185 +64186,64186 +64187,64187 +64188,64188 +145417,64188 +64189,64189 +64190,64190 +64191,64191 +64192,64192 +64193,64193 +64194,64194 +64195,64195 +64196,64196 +64197,64197 +64198,64198 +64199,64199 +64200,64200 +64201,64201 +227602,64202 +64202,64202 +64203,64203 +64204,64204 +64205,64205 +64206,64206 +64207,64207 +64208,64208 +64209,64209 +64210,64210 +64211,64211 +64212,64212 +64213,64213 +64214,64214 +64215,64215 +64216,64216 +64217,64217 +64218,64218 +64219,64219 +64220,64220 +64221,64221 +64222,64222 +162184,64222 +64223,64223 +64224,64224 +64304,64304 +64305,64305 +64306,64306 +64307,64307 +64308,64308 +64309,64309 +3149,64309 +64310,64310 +64311,64311 +64312,64312 +72586,64312 +64313,64313 +64314,64314 +64315,64315 +64316,64316 +64317,64317 +64318,64318 +64319,64319 +64320,64320 +64321,64321 +64322,64322 +64323,64323 +67616,64323 +64324,64324 +64325,64325 +64326,64326 +64327,64327 +33220,64327 +149835,64397 +64397,64397 +64398,64398 +64399,64399 +64400,64400 +64401,64401 +64402,64402 +64403,64403 +64404,64404 +64405,64405 +158507,64405 +64406,64406 +64407,64407 +64408,64408 +64409,64409 +64410,64410 +64411,64411 +100671,64411 +64412,64412 +64413,64413 +64414,64414 +64460,64460 +64461,64461 +64462,64462 +64463,64463 +64464,64464 +64465,64465 +64466,64466 +64467,64467 +64468,64468 +64469,64469 +64470,64470 +64471,64471 +64472,64472 +64473,64473 +243483,64473 +64474,64474 +64475,64475 +64476,64476 +64477,64477 +64478,64478 +64479,64479 +64480,64480 +20220,64480 +64481,64481 +64482,64482 +189849,64482 +64483,64483 +64484,64484 +64485,64485 +64486,64486 +64487,64487 +64488,64488 +64489,64489 +64490,64490 +64491,64491 +64492,64492 +64493,64493 +64494,64494 +64495,64495 +64496,64496 +64497,64497 +64498,64498 +64499,64499 +64500,64500 +64501,64501 +64502,64502 +64503,64503 +64504,64504 +64676,64676 +64677,64677 +64678,64678 +64679,64679 +64680,64680 +64681,64681 +64682,64682 +64683,64683 +64684,64684 +64685,64685 +64686,64686 +64687,64687 +64688,64688 +64689,64689 +64690,64690 +64691,64691 +123275,64691 +64692,64692 +64693,64693 +64694,64694 +64695,64695 +64696,64696 +34029,64696 +64697,64697 +64698,64698 +64699,64699 +64700,64700 +64701,64701 +64702,64702 +64703,64703 +64704,64704 +64705,64705 +64706,64706 +64707,64707 +64708,64708 +64709,64709 +64710,64710 +64711,64711 +64712,64712 +64713,64713 +64714,64714 +64715,64715 +22040,64715 +64716,64716 +64717,64717 +64718,64718 +64719,64719 +64720,64720 +64721,64721 +64722,64722 +64723,64723 +92790,64723 +64724,64724 +64725,64725 +64726,64726 +64727,64727 +64728,64728 +64729,64729 +64730,64730 +64731,64731 +28064,64731 +64732,64732 +64733,64733 +64734,64734 +64735,64735 +51812,64735 +43708,64735 +64736,64736 +64737,64737 +64738,64738 +64739,64739 +164847,64739 +64740,64740 +64741,64741 +64742,64742 +64743,64743 +64793,64793 +64794,64794 +64795,64795 +64796,64796 +64797,64797 +64798,64798 +64799,64799 +64800,64800 +64801,64801 +64802,64802 +64803,64803 +64804,64804 +64805,64805 +64806,64806 +64807,64807 +64808,64808 +76022,64808 +64809,64809 +64810,64810 +64811,64811 +64812,64812 +64813,64813 +64814,64814 +64815,64815 +64816,64816 +64817,64817 +64818,64818 +64819,64819 +64820,64820 +62960,64820 +64821,64821 +64822,64822 +64823,64823 +64824,64824 +64825,64825 +64826,64826 +64827,64827 +64828,64828 +64829,64829 +64830,64830 +64831,64831 +64832,64832 +64833,64833 +64834,64834 +27171,64834 +64835,64835 +64836,64836 +64837,64837 +64838,64838 +64839,64839 +64840,64840 +64841,64841 +64842,64842 +64843,64843 +64844,64844 +64845,64845 +66326,64845 +64846,64846 +64847,64847 +64848,64848 +247293,64848 +64849,64849 +64850,64850 +64851,64851 +64852,64852 +64853,64853 +190429,64853 +64854,64854 +64855,64855 +64856,64856 +64857,64857 +64858,64858 +64859,64859 +64860,64860 +64861,64861 +64862,64862 +64863,64863 +64864,64864 +64865,64865 +64866,64866 +64867,64867 +64868,64868 +64869,64869 +64870,64870 +64871,64871 +64872,64872 +3479,64872 +64873,64873 +64874,64874 +64875,64875 +64876,64876 +64877,64877 +64878,64878 +64879,64879 +64880,64880 +64881,64881 +245429,64881 +64882,64882 +64883,64883 +64884,64884 +64885,64885 +64886,64886 +64887,64887 +64888,64888 +64889,64889 +64890,64890 +64891,64891 +64892,64892 +64893,64893 +64894,64894 +64895,64895 +64896,64896 +64897,64897 +64898,64898 +64899,64899 +64900,64900 +64901,64901 +135259,64901 +127672,64901 +64902,64902 +64903,64903 +64904,64904 +64905,64905 +206389,64906 +64906,64906 +64907,64907 +64908,64908 +64909,64909 +64910,64910 +64911,64911 +64912,64912 +64913,64913 +131931,64913 +124344,64913 +64914,64914 +64915,64915 +64916,64916 +64917,64917 +64918,64918 +64919,64919 +64920,64920 +64921,64921 +64922,64922 +64923,64923 +64924,64924 +64925,64925 +65847,64925 +64926,64926 +64927,64927 +64928,64928 +179837,64928 +64929,64929 +64930,64930 +36305,64930 +64931,64931 +64932,64932 +208047,64932 +64933,64933 +64934,64934 +64935,64935 +64936,64936 +64937,64937 +64938,64938 +64939,64939 +64940,64940 +64941,64941 +64942,64942 +64943,64943 +64944,64944 +64945,64945 +64946,64946 +64947,64947 +64948,64948 +64949,64949 +33629,64949 +64950,64950 +64951,64951 +64952,64952 +64953,64953 +64954,64954 +64955,64955 +64956,64956 +64957,64957 +64958,64958 +64959,64959 +111962,64960 +64960,64960 +64961,64961 +64962,64962 +64963,64963 +64964,64964 +64965,64965 +64966,64966 +64967,64967 +64968,64968 +64969,64969 +64970,64970 +64971,64971 +64972,64972 +64973,64973 +144059,64973 +64974,64974 +36827,64974 +30689,64974 +64975,64975 +64976,64976 +64977,64977 +15052,64977 +64978,64978 +64979,64979 +64980,64980 +64981,64981 +64982,64982 +64983,64983 +64984,64984 +12090,64984 +64985,64985 +64986,64986 +64987,64987 +75141,64987 +64988,64988 +64989,64989 +64990,64990 +64991,64991 +64992,64992 +64993,64993 +64994,64994 +64995,64995 +64996,64996 +243334,64996 +64997,64997 +64998,64998 +64999,64999 +65000,65000 +65001,65001 +63657,65001 +65002,65002 +65003,65003 +65004,65004 +65005,65005 +65006,65006 +65007,65007 +65008,65008 +33437,65008 +65009,65009 +65010,65010 +65011,65011 +65012,65012 +65013,65013 +65014,65014 +65015,65015 +65016,65016 +65017,65017 +65018,65018 +65019,65019 +65020,65020 +142358,65020 +65021,65021 +65022,65022 +65023,65023 +65024,65024 +65025,65025 +65026,65026 +211980,65026 +119172,65026 +65027,65027 +65028,65028 +65029,65029 +65030,65030 +65031,65031 +65032,65032 +65033,65033 +65034,65034 +65035,65035 +65036,65036 +204362,65036 +65037,65037 +156379,65037 +65038,65038 +65039,65039 +65040,65040 +65041,65041 +65042,65042 +65043,65043 +65044,65044 +65045,65045 +65046,65046 +65047,65047 +65048,65048 +65049,65049 +65050,65050 +65051,65051 +65052,65052 +65053,65053 +65054,65054 +65055,65055 +65056,65056 +65057,65057 +65058,65058 +65059,65059 +65060,65060 +65061,65061 +65062,65062 +65063,65063 +65064,65064 +65065,65065 +65066,65066 +65067,65067 +82773,65067 +65068,65068 +65069,65069 +65070,65070 +65071,65071 +65072,65072 +65073,65073 +65074,65074 +65075,65075 +65076,65076 +65077,65077 +65078,65078 +65079,65079 +65080,65080 +65081,65081 +65082,65082 +65083,65083 +65084,65084 +65085,65085 +65086,65086 +65087,65087 +65088,65088 +65089,65089 +65090,65090 +65091,65091 +65092,65092 +65093,65093 +158339,65093 +65094,65094 +65095,65095 +65096,65096 +65097,65097 +65098,65098 +65099,65099 +65100,65100 +65101,65101 +65102,65102 +65103,65103 +65104,65104 +185675,65104 +65105,65105 +65106,65106 +65107,65107 +65108,65108 +65109,65109 +65110,65110 +65111,65111 +65112,65112 +65113,65113 +65114,65114 +65115,65115 +158323,65115 +65116,65116 +65117,65117 +65118,65118 +65119,65119 +65120,65120 +65121,65121 +65122,65122 +136533,65122 +128946,65122 +65123,65123 +65124,65124 +65125,65125 +65126,65126 +65127,65127 +65128,65128 +65129,65129 +65130,65130 +65131,65131 +65132,65132 +65133,65133 +65134,65134 +65135,65135 +65136,65136 +65137,65137 +65138,65138 +65139,65139 +65140,65140 +65141,65141 +65142,65142 +65143,65143 +65144,65144 +65145,65145 +65146,65146 +17379,65146 +65147,65147 +65148,65148 +65149,65149 +65150,65150 +65151,65151 +65152,65152 +142785,65152 +65153,65153 +65154,65154 +65155,65155 +65156,65156 +65157,65157 +2321,65157 +65158,65158 +65159,65159 +212754,65159 +65160,65160 +65161,65161 +59918,65161 +65162,65162 +65163,65163 +65164,65164 +65165,65165 +65166,65166 +149773,65166 +65167,65167 +65168,65168 +65169,65169 +65170,65170 +65171,65171 +65172,65172 +65173,65173 +65174,65174 +65175,65175 +65176,65176 +65177,65177 +65178,65178 +65179,65179 +65180,65180 +65181,65181 +65182,65182 +65183,65183 +65455,65455 +65456,65456 +65457,65457 +65458,65458 +65459,65459 +185939,65459 +65460,65460 +65461,65461 +65462,65462 +65463,65463 +194721,65464 +65464,65464 +65465,65465 +65466,65466 +65467,65467 +65468,65468 +26081,65468 +65469,65469 +65470,65470 +65471,65471 +65572,65572 +65573,65573 +65574,65574 +65575,65575 +65576,65576 +65577,65577 +65578,65578 +65579,65579 +65580,65580 +65581,65581 +65582,65582 +101046,65582 +65583,65583 +65584,65584 +65585,65585 +65586,65586 +65587,65587 +65588,65588 +65860,65588 +65589,65589 +65590,65590 +65591,65591 +65592,65592 +65593,65593 +65594,65594 +65595,65595 +65596,65596 +65597,65597 +65598,65598 +65599,65599 +65600,65600 +179873,65600 +65601,65601 +65602,65602 +65603,65603 +65604,65604 +156411,65604 +65605,65605 +160423,65605 +65606,65606 +65607,65607 +65608,65608 +65609,65609 +65610,65610 +65611,65611 +65612,65612 +65613,65613 +65614,65614 +65615,65615 +65616,65616 +65617,65617 +65618,65618 +145484,65618 +65619,65619 +65620,65620 +65621,65621 +65622,65622 +65623,65623 +65624,65624 +65625,65625 +65695,65695 +220336,65695 +65696,65696 +65697,65697 +65698,65698 +65699,65699 +65700,65700 +65701,65701 +65702,65702 +65703,65703 +65704,65704 +65705,65705 +65706,65706 +65707,65707 +65708,65708 +65709,65709 +65710,65710 +65711,65711 +65712,65712 +65713,65713 +65714,65714 +65715,65715 +65716,65716 +65717,65717 +65718,65718 +65719,65719 +65720,65720 +65721,65721 +104660,65721 +65722,65722 +65723,65723 +65724,65724 +65725,65725 +65726,65726 +65727,65727 +65728,65728 +65729,65729 +65730,65730 +65731,65731 +65732,65732 +65733,65733 +65734,65734 +65735,65735 +65736,65736 +65737,65737 +65738,65738 +4653,65738 +65739,65739 +88994,65739 +65740,65740 +242931,65740 +65741,65741 +65742,65742 +65743,65743 +12949,65743 +134328,65743 +126741,65743 +65744,65744 +65745,65745 +65746,65746 +65747,65747 +2683,65747 +65748,65748 +65749,65749 +65750,65750 +65751,65751 +65752,65752 +65753,65753 +65754,65754 +65755,65755 +28124,65755 +65756,65756 +65757,65757 +65758,65758 +65759,65759 +65760,65760 +65761,65761 +65762,65762 +65763,65763 +206273,65764 +65764,65764 +65765,65765 +65766,65766 +36969,65766 +65767,65767 +65768,65768 +65769,65769 +65770,65770 +65771,65771 +65772,65772 +65773,65773 +65774,65774 +65815,65815 +65816,65816 +65817,65817 +65818,65818 +65819,65819 +65820,65820 +65821,65821 +65822,65822 +65823,65823 +113332,65823 +65824,65824 +65825,65825 +65826,65826 +65827,65827 +181248,65827 +65828,65828 +65829,65829 +65830,65830 +65831,65831 +65832,65832 +65833,65833 +65834,65834 +65835,65835 +65836,65836 +65837,65837 +65838,65838 +65839,65839 +65840,65840 +65841,65841 +65842,65842 +65843,65843 +65844,65844 +65845,65845 +65846,65846 +65847,65847 +64925,65847 +65848,65848 +65849,65849 +65850,65850 +65851,65851 +65852,65852 +65853,65853 +65854,65854 +65855,65855 +65856,65856 +65857,65857 +56226,65857 +48122,65857 +65858,65858 +65859,65859 +178563,65859 +172599,65859 +65860,65860 +65588,65860 +65861,65861 +65862,65862 +65863,65863 +65864,65864 +65865,65865 +65866,65866 +65867,65867 +65868,65868 +65869,65869 +231812,65869 +65870,65870 +65871,65871 +65872,65872 +65873,65873 +65874,65874 +65875,65875 +65876,65876 +65877,65877 +65878,65878 +65879,65879 +65880,65880 +65881,65881 +65882,65882 +248133,65882 +65883,65883 +65884,65884 +65885,65885 +65886,65886 +65929,65929 +65930,65930 +65931,65931 +65932,65932 +65933,65933 +65934,65934 +5376,65934 +65935,65935 +65936,65936 +65937,65937 +18544,65937 +65938,65938 +65939,65939 +65940,65940 +233354,65940 +65941,65941 +65942,65942 +65943,65943 +65944,65944 +65945,65945 +65946,65946 +65947,65947 +154514,65947 +65948,65948 +65949,65949 +65950,65950 +65951,65951 +65952,65952 +65953,65953 +65954,65954 +65955,65955 +65956,65956 +121311,65956 +65957,65957 +65958,65958 +65959,65959 +244091,65959 +65960,65960 +65961,65961 +65962,65962 +65963,65963 +65964,65964 +65965,65965 +65966,65966 +65967,65967 +65968,65968 +65969,65969 +65970,65970 +65971,65971 +66108,66108 +66109,66109 +66110,66110 +66111,66111 +66112,66112 +66113,66113 +66114,66114 +66652,66114 +66288,66114 +66115,66115 +66653,66115 +66289,66115 +66654,66116 +66290,66116 +66116,66116 +66117,66117 +66655,66117 +66291,66117 +66118,66118 +66656,66118 +66292,66118 +66119,66119 +66657,66119 +66293,66119 +66120,66120 +66658,66120 +66294,66120 +66659,66121 +66295,66121 +66121,66121 +66122,66122 +66660,66122 +66296,66122 +66661,66123 +66297,66123 +66123,66123 +66124,66124 +66662,66124 +66298,66124 +66663,66125 +66299,66125 +66125,66125 +66664,66126 +66300,66126 +66126,66126 +66127,66127 +66665,66127 +66301,66127 +66128,66128 +66129,66129 +66667,66129 +66303,66129 +166684,66129 +66130,66130 +66668,66130 +66304,66130 +66131,66131 +36299,66131 +66132,66132 +66133,66133 +66134,66134 +66135,66135 +66136,66136 +146963,66136 +66137,66137 +66138,66138 +66139,66139 +66140,66140 +66141,66141 +66142,66142 +66143,66143 +66144,66144 +66145,66145 +66146,66146 +66147,66147 +66148,66148 +66149,66149 +66330,66149 +66150,66150 +66151,66151 +6008,66151 +66152,66152 +66153,66153 +66154,66154 +66155,66155 +66156,66156 +66157,66157 +66158,66158 +66159,66159 +66160,66160 +66161,66161 +66162,66162 +66163,66163 +66164,66164 +66165,66165 +66166,66166 +66167,66167 +66168,66168 +66652,66288 +66288,66288 +66114,66288 +66653,66289 +66289,66289 +66115,66289 +66654,66290 +66290,66290 +66116,66290 +66655,66291 +66291,66291 +66117,66291 +66656,66292 +66292,66292 +66118,66292 +66657,66293 +66293,66293 +66119,66293 +66658,66294 +66294,66294 +66120,66294 +66659,66295 +66295,66295 +66121,66295 +66660,66296 +66296,66296 +66122,66296 +66661,66297 +66297,66297 +66123,66297 +66662,66298 +66298,66298 +66124,66298 +66663,66299 +66299,66299 +66125,66299 +66664,66300 +66300,66300 +66126,66300 +66665,66301 +66301,66301 +66127,66301 +66666,66302 +66302,66302 +66667,66303 +66303,66303 +66129,66303 +66668,66304 +66304,66304 +66130,66304 +66669,66305 +66305,66305 +66670,66306 +66306,66306 +66307,66307 +66671,66307 +66308,66308 +66672,66308 +66309,66309 +66673,66309 +66310,66310 +66674,66310 +66675,66311 +66311,66311 +66312,66312 +66676,66312 +66677,66313 +66313,66313 +66314,66314 +66678,66314 +66679,66315 +66315,66315 +233254,66315 +66680,66316 +66316,66316 +66317,66317 +66318,66318 +66319,66319 +66683,66319 +66320,66320 +66684,66320 +66321,66321 +66685,66321 +66322,66322 +66686,66322 +66323,66323 +66687,66323 +66324,66324 +66325,66325 +66326,66326 +64845,66326 +66327,66327 +66328,66328 +66329,66329 +66330,66330 +66149,66330 +66331,66331 +66332,66332 +244923,66332 +66333,66333 +66334,66334 +66335,66335 +66336,66336 +66337,66337 +66338,66338 +66339,66339 +66340,66340 +66341,66341 +66342,66342 +66343,66343 +66344,66344 +244935,66344 +66345,66345 +66346,66346 +66347,66347 +74704,66347 +66348,66348 +66349,66349 +206078,66349 +66350,66350 +66351,66351 +66352,66352 +66353,66353 +66354,66354 +66355,66355 +66469,66469 +66470,66470 +66471,66471 +66472,66472 +66473,66473 +66474,66474 +66475,66475 +66476,66476 +66477,66477 +66478,66478 +66479,66479 +66480,66480 +66481,66481 +66482,66482 +66483,66483 +66484,66484 +66485,66485 +66486,66486 +66487,66487 +132396,66487 +124809,66487 +66488,66488 +66489,66489 +66490,66490 +66491,66491 +66492,66492 +66493,66493 +66494,66494 +66495,66495 +66496,66496 +66497,66497 +66498,66498 +66499,66499 +66500,66500 +66501,66501 +66502,66502 +66503,66503 +66504,66504 +66505,66505 +66506,66506 +66507,66507 +66508,66508 +66509,66509 +66510,66510 +66511,66511 +66512,66512 +66513,66513 +66552,66552 +66553,66553 +66554,66554 +66555,66555 +66556,66556 +66557,66557 +66558,66558 +66559,66559 +66560,66560 +66561,66561 +66562,66562 +66563,66563 +66564,66564 +66565,66565 +66566,66566 +66567,66567 +66568,66568 +66569,66569 +66570,66570 +66571,66571 +66572,66572 +66573,66573 +66574,66574 +66575,66575 +66576,66576 +66652,66652 +66288,66652 +66114,66652 +66653,66653 +66289,66653 +66115,66653 +66654,66654 +66290,66654 +66116,66654 +66655,66655 +66291,66655 +66117,66655 +66656,66656 +66292,66656 +66118,66656 +66657,66657 +66293,66657 +66119,66657 +66658,66658 +66294,66658 +66120,66658 +66659,66659 +66295,66659 +66121,66659 +66660,66660 +66296,66660 +66122,66660 +66661,66661 +66297,66661 +66123,66661 +66662,66662 +66298,66662 +66124,66662 +66663,66663 +66299,66663 +66125,66663 +66664,66664 +66300,66664 +66126,66664 +66665,66665 +66301,66665 +66127,66665 +66666,66666 +66302,66666 +66667,66667 +66303,66667 +66129,66667 +66668,66668 +66304,66668 +66130,66668 +66669,66669 +66305,66669 +66670,66670 +66306,66670 +66671,66671 +66307,66671 +66672,66672 +66308,66672 +66673,66673 +66309,66673 +66674,66674 +66310,66674 +66675,66675 +66311,66675 +66676,66676 +66312,66676 +66677,66677 +66313,66677 +66678,66678 +66314,66678 +66679,66679 +66315,66679 +233254,66679 +66680,66680 +66316,66680 +66681,66681 +66682,66682 +66683,66683 +66319,66683 +66684,66684 +66320,66684 +66685,66685 +66321,66685 +66686,66686 +66322,66686 +66687,66687 +66323,66687 +66688,66688 +66796,66796 +78625,66796 +66797,66797 +66798,66798 +66799,66799 +241388,66799 +66800,66800 +66801,66801 +66802,66802 +50357,66802 +42165,66802 +66803,66803 +66804,66804 +66805,66805 +66806,66806 +66807,66807 +66808,66808 +66809,66809 +66810,66810 +66811,66811 +66812,66812 +135757,66813 +128170,66813 +66813,66813 +66814,66814 +66815,66815 +66816,66816 +66817,66817 +66818,66818 +66819,66819 +195165,66819 +66820,66820 +3575,66820 +66821,66821 +66822,66822 +66823,66823 +66824,66824 +66825,66825 +66826,66826 +56356,66826 +48252,66826 +66882,66882 +66883,66883 +66884,66884 +66885,66885 +146570,66885 +66886,66886 +66887,66887 +66888,66888 +66889,66889 +66890,66890 +66891,66891 +66892,66892 +66893,66893 +66894,66894 +66895,66895 +66896,66896 +66897,66897 +66898,66898 +66899,66899 +66900,66900 +66901,66901 +179479,66901 +66902,66902 +66903,66903 +66904,66904 +66905,66905 +66906,66906 +202610,66906 +66907,66907 +66908,66908 +66909,66909 +66910,66910 +66911,66911 +66912,66912 +66948,66948 +29061,66948 +246971,66948 +79171,66948 +66949,66949 +66950,66950 +66951,66951 +66952,66952 +66953,66953 +66954,66954 +66955,66955 +66956,66956 +66957,66957 +66958,66958 +66959,66959 +66960,66960 +66961,66961 +66962,66962 +66963,66963 +66964,66964 +66965,66965 +66966,66966 +66967,66967 +59437,66967 +66968,66968 +66969,66969 +66970,66970 +66971,66971 +66972,66972 +66973,66973 +66974,66974 +66975,66975 +66976,66976 +66977,66977 +66978,66978 +66979,66979 +108968,66979 +66980,66980 +66981,66981 +66982,66982 +67008,67008 +67009,67009 +67010,67010 +67011,67011 +67012,67012 +67013,67013 +67014,67014 +138749,67014 +67015,67015 +67016,67016 +67017,67017 +67018,67018 +67019,67019 +67020,67020 +177542,67020 +171578,67020 +67021,67021 +67022,67022 +67023,67023 +67024,67024 +67025,67025 +67026,67026 +67027,67027 +67028,67028 +67029,67029 +13297,67029 +67030,67030 +67031,67031 +67112,67112 +67113,67113 +67114,67114 +67115,67115 +30773,67115 +67116,67116 +30918,67116 +67117,67117 +67118,67118 +67119,67119 +67120,67120 +67121,67121 +67122,67122 +19953,67122 +67123,67123 +67124,67124 +67125,67125 +67126,67126 +67127,67127 +133232,67127 +125645,67127 +108562,67128 +67128,67128 +67129,67129 +67130,67130 +80349,67130 +67131,67131 +67132,67132 +67133,67133 +67134,67134 +67135,67135 +67136,67136 +67137,67137 +226126,67137 +67138,67138 +67139,67139 +67140,67140 +67141,67141 +67142,67142 +67143,67143 +67144,67144 +67145,67145 +67146,67146 +67147,67147 +67148,67148 +67149,67149 +67150,67150 +67151,67151 +67152,67152 +67153,67153 +67154,67154 +67155,67155 +67156,67156 +67157,67157 +67158,67158 +67159,67159 +67160,67160 +67161,67161 +67162,67162 +67163,67163 +67164,67164 +67165,67165 +67166,67166 +67167,67167 +67168,67168 +67169,67169 +67170,67170 +67171,67171 +67172,67172 +67173,67173 +204541,67173 +67174,67174 +67175,67175 +67176,67176 +67177,67177 +67178,67178 +67179,67179 +67180,67180 +67181,67181 +67182,67182 +54987,67182 +46883,67182 +67183,67183 +67184,67184 +67185,67185 +67186,67186 +67187,67187 +67188,67188 +67189,67189 +67190,67190 +67191,67191 +67192,67192 +67193,67193 +67194,67194 +67195,67195 +67196,67196 +67197,67197 +67198,67198 +67199,67199 +67200,67200 +67201,67201 +67202,67202 +123188,67202 +67203,67203 +67204,67204 +67205,67205 +67206,67206 +67207,67207 +67208,67208 +67209,67209 +67210,67210 +67211,67211 +67212,67212 +67213,67213 +67214,67214 +67215,67215 +67216,67216 +67217,67217 +67218,67218 +67219,67219 +67220,67220 +67221,67221 +204755,67221 +67222,67222 +67223,67223 +67224,67224 +67225,67225 +210080,67225 +67226,67226 +67227,67227 +67228,67228 +67229,67229 +67230,67230 +67231,67231 +67232,67232 +67233,67233 +67234,67234 +67235,67235 +67236,67236 +67237,67237 +67238,67238 +67239,67239 +67240,67240 +67241,67241 +67242,67242 +67243,67243 +67244,67244 +67245,67245 +67246,67246 +67247,67247 +67248,67248 +67249,67249 +67250,67250 +67251,67251 +67252,67252 +67253,67253 +67254,67254 +190933,67255 +67255,67255 +67256,67256 +67257,67257 +67258,67258 +67259,67259 +67260,67260 +67261,67261 +67262,67262 +220295,67262 +67263,67263 +67264,67264 +67265,67265 +67266,67266 +67267,67267 +67268,67268 +67269,67269 +173998,67269 +168034,67269 +67270,67270 +67271,67271 +67272,67272 +67273,67273 +206298,67273 +67274,67274 +67275,67275 +5108,67275 +67276,67276 +67277,67277 +194175,67277 +67278,67278 +67279,67279 +67280,67280 +67281,67281 +142426,67281 +67282,67282 +227116,67282 +67283,67283 +67284,67284 +67285,67285 +67286,67286 +67287,67287 +67288,67288 +76931,67288 +67289,67289 +54561,67289 +46457,67289 +67290,67290 +67291,67291 +67292,67292 +67293,67293 +67294,67294 +67295,67295 +67296,67296 +67297,67297 +67298,67298 +67299,67299 +67300,67300 +67301,67301 +67302,67302 +67303,67303 +67304,67304 +67305,67305 +67306,67306 +67307,67307 +67308,67308 +67309,67309 +67310,67310 +67311,67311 +67312,67312 +67313,67313 +67314,67314 +67315,67315 +67316,67316 +149236,67316 +67317,67317 +67318,67318 +67319,67319 +67320,67320 +67321,67321 +225548,67321 +67322,67322 +67323,67323 +67324,67324 +67325,67325 +67366,67366 +67367,67367 +67368,67368 +67369,67369 +67370,67370 +67371,67371 +67372,67372 +67373,67373 +67374,67374 +67375,67375 +67376,67376 +67377,67377 +67378,67378 +67379,67379 +123986,67379 +67380,67380 +67381,67381 +67382,67382 +67383,67383 +67384,67384 +67385,67385 +67386,67386 +139906,67386 +67387,67387 +67388,67388 +67389,67389 +67536,67536 +158137,67536 +67537,67537 +67538,67538 +67539,67539 +67540,67540 +67541,67541 +67542,67542 +67543,67543 +67544,67544 +67545,67545 +201770,67545 +67546,67546 +67547,67547 +67548,67548 +67549,67549 +67550,67550 +67551,67551 +67552,67552 +67553,67553 +67554,67554 +67555,67555 +67556,67556 +67557,67557 +67558,67558 +67559,67559 +67560,67560 +67561,67561 +67562,67562 +67563,67563 +67564,67564 +67565,67565 +67566,67566 +67567,67567 +67568,67568 +67569,67569 +67570,67570 +67571,67571 +67572,67572 +67573,67573 +67574,67574 +67575,67575 +67576,67576 +67577,67577 +67578,67578 +67579,67579 +67580,67580 +67581,67581 +67582,67582 +67583,67583 +67584,67584 +67585,67585 +15371,67585 +67586,67586 +67587,67587 +67588,67588 +67589,67589 +208854,67589 +67590,67590 +67591,67591 +132117,67591 +131924,67591 +124530,67591 +124337,67591 +67592,67592 +67593,67593 +67594,67594 +67595,67595 +67596,67596 +209460,67596 +67597,67597 +67598,67598 +67599,67599 +67600,67600 +67601,67601 +67602,67602 +67603,67603 +67604,67604 +67605,67605 +67606,67606 +198741,67606 +195096,67606 +67607,67607 +67608,67608 +67609,67609 +67610,67610 +67611,67611 +244914,67611 +67612,67612 +67613,67613 +67614,67614 +67615,67615 +67616,67616 +64323,67616 +67617,67617 +67618,67618 +67619,67619 +67620,67620 +67621,67621 +67622,67622 +67623,67623 +67624,67624 +67625,67625 +214695,67625 +67626,67626 +133098,67626 +125511,67626 +67627,67627 +67628,67628 +67629,67629 +67741,67741 +67742,67742 +67743,67743 +67744,67744 +67745,67745 +67746,67746 +67747,67747 +67748,67748 +67749,67749 +67750,67750 +67751,67751 +59884,67751 +67752,67752 +67753,67753 +67754,67754 +67755,67755 +67756,67756 +67757,67757 +67758,67758 +67759,67759 +67760,67760 +67761,67761 +67762,67762 +67763,67763 +67764,67764 +67765,67765 +67766,67766 +67767,67767 +67768,67768 +67769,67769 +67770,67770 +67771,67771 +67772,67772 +67773,67773 +178140,67773 +172176,67773 +67774,67774 +161274,67774 +67775,67775 +67776,67776 +40271,67776 +67777,67777 +67778,67778 +67947,67947 +67948,67948 +67949,67949 +67950,67950 +67951,67951 +67952,67952 +67953,67953 +67954,67954 +67955,67955 +67956,67956 +67957,67957 +67958,67958 +67959,67959 +67960,67960 +67961,67961 +67962,67962 +67963,67963 +67964,67964 +67965,67965 +67966,67966 +67967,67967 +67968,67968 +67969,67969 +67970,67970 +67971,67971 +67972,67972 +67973,67973 +67974,67974 +67975,67975 +67976,67976 +67977,67977 +67978,67978 +67979,67979 +67980,67980 +67981,67981 +67982,67982 +67983,67983 +67984,67984 +67985,67985 +67986,67986 +10805,67986 +67987,67987 +67988,67988 +67989,67989 +67990,67990 +67991,67991 +67992,67992 +67993,67993 +67994,67994 +67995,67995 +114359,67996 +67996,67996 +67997,67997 +67998,67998 +67999,67999 +68000,68000 +68001,68001 +68002,68002 +68003,68003 +68004,68004 +68005,68005 +201074,68005 +68006,68006 +224625,68006 +68007,68007 +68008,68008 +68009,68009 +68010,68010 +68011,68011 +68012,68012 +68013,68013 +68014,68014 +68015,68015 +68016,68016 +179090,68016 +68017,68017 +68018,68018 +68019,68019 +68020,68020 +234830,68020 +68021,68021 +136168,68021 +128581,68021 +68022,68022 +68023,68023 +68024,68024 +154592,68024 +68025,68025 +68026,68026 +68027,68027 +68028,68028 +68029,68029 +109099,68029 +68030,68030 +68031,68031 +68032,68032 +68033,68033 +68034,68034 +68035,68035 +68036,68036 +68037,68037 +68038,68038 +68039,68039 +68040,68040 +68041,68041 +68042,68042 +68043,68043 +68044,68044 +68045,68045 +68046,68046 +68047,68047 +68048,68048 +68049,68049 +68050,68050 +68051,68051 +36404,68051 +68052,68052 +68053,68053 +143693,68053 +68054,68054 +68055,68055 +122817,68055 +68056,68056 +158681,68057 +68057,68057 +68058,68058 +68059,68059 +68060,68060 +68061,68061 +68062,68062 +211999,68062 +68063,68063 +68064,68064 +68065,68065 +68066,68066 +68067,68067 +68068,68068 +68069,68069 +68070,68070 +68071,68071 +68072,68072 +68073,68073 +68074,68074 +68075,68075 +68076,68076 +68077,68077 +68078,68078 +68079,68079 +68080,68080 +68081,68081 +68082,68082 +68083,68083 +68084,68084 +68085,68085 +68086,68086 +68087,68087 +68088,68088 +68089,68089 +68090,68090 +68091,68091 +68092,68092 +68093,68093 +68094,68094 +68095,68095 +68096,68096 +236477,68096 +68097,68097 +68098,68098 +68099,68099 +68100,68100 +68101,68101 +68140,68140 +68141,68141 +68142,68142 +68143,68143 +68144,68144 +68145,68145 +68146,68146 +68147,68147 +68148,68148 +68149,68149 +68150,68150 +68151,68151 +68152,68152 +68153,68153 +68154,68154 +247147,68154 +68155,68155 +68156,68156 +68157,68157 +68158,68158 +68159,68159 +68160,68160 +68161,68161 +68162,68162 +68163,68163 +68164,68164 +68165,68165 +68166,68166 +68167,68167 +68168,68168 +68169,68169 +68170,68170 +68171,68171 +68172,68172 +68173,68173 +68174,68174 +68175,68175 +68176,68176 +39909,68176 +68177,68177 +68178,68178 +68179,68179 +68180,68180 +68181,68181 +68182,68182 +100634,68182 +68183,68183 +68184,68184 +68185,68185 +235672,68185 +68186,68186 +68187,68187 +68188,68188 +68189,68189 +68190,68190 +68191,68191 +68192,68192 +68193,68193 +108505,68193 +217041,68193 +136038,68193 +128451,68193 +68194,68194 +68195,68195 +68196,68196 +68197,68197 +112257,68197 +68198,68198 +68199,68199 +68200,68200 +68201,68201 +68202,68202 +228714,68202 +68203,68203 +84562,68203 +68204,68204 +68205,68205 +68206,68206 +68207,68207 +68208,68208 +68209,68209 +68210,68210 +68211,68211 +68212,68212 +68213,68213 +68214,68214 +68215,68215 +68216,68216 +68217,68217 +59959,68217 +68218,68218 +68219,68219 +68220,68220 +68221,68221 +68222,68222 +68223,68223 +7729,68223 +68224,68224 +68225,68225 +68226,68226 +68227,68227 +68228,68228 +68229,68229 +68230,68230 +68231,68231 +68232,68232 +68233,68233 +68234,68234 +68235,68235 +68236,68236 +68237,68237 +68238,68238 +68239,68239 +68240,68240 +68241,68241 +68242,68242 +68243,68243 +68244,68244 +68245,68245 +68246,68246 +68247,68247 +68248,68248 +68249,68249 +68250,68250 +68251,68251 +68252,68252 +146529,68252 +68253,68253 +68254,68254 +68255,68255 +68256,68256 +68257,68257 +68258,68258 +68259,68259 +68260,68260 +68261,68261 +68262,68262 +68263,68263 +68264,68264 +68265,68265 +68266,68266 +68267,68267 +68268,68268 +68269,68269 +68270,68270 +68271,68271 +68272,68272 +68273,68273 +68274,68274 +68275,68275 +176468,68275 +170504,68275 +68276,68276 +68277,68277 +68278,68278 +68279,68279 +68280,68280 +68281,68281 +68282,68282 +68283,68283 +68284,68284 +68285,68285 +68286,68286 +68287,68287 +68288,68288 +68289,68289 +68290,68290 +68291,68291 +68292,68292 +68293,68293 +68294,68294 +2018,68294 +68295,68295 +68296,68296 +68297,68297 +68298,68298 +68299,68299 +4078,68299 +68300,68300 +98101,68301 +68301,68301 +69788,68483 +68483,68483 +68484,68484 +69789,68484 +68485,68485 +69790,68485 +68486,68486 +69791,68486 +68487,68487 +69792,68487 +69793,68488 +68488,68488 +68489,68489 +69794,68489 +69795,68490 +68490,68490 +68491,68491 +69796,68491 +69797,68492 +68492,68492 +69798,68493 +68493,68493 +68494,68494 +69799,68494 +68495,68495 +69800,68495 +69801,68496 +68496,68496 +69802,68497 +68497,68497 +68498,68498 +69803,68498 +69804,68499 +68499,68499 +68500,68500 +247610,68500 +69805,68500 +68501,68501 +69806,68501 +68502,68502 +69807,68502 +68503,68503 +226143,68503 +68504,68504 +68505,68505 +226146,68506 +68506,68506 +68507,68507 +68508,68508 +68509,68509 +68510,68510 +68511,68511 +68512,68512 +68513,68513 +68514,68514 +68515,68515 +68516,68516 +68517,68517 +68612,68612 +68613,68613 +68614,68614 +68615,68615 +56233,68615 +48129,68615 +68616,68616 +68617,68617 +68618,68618 +68619,68619 +68620,68620 +68621,68621 +231756,68621 +68622,68622 +68623,68623 +68624,68624 +68625,68625 +68626,68626 +240426,68626 +68627,68627 +68628,68628 +119755,68628 +68629,68629 +59118,68629 +58744,68629 +68630,68630 +68631,68631 +68632,68632 +110334,68633 +68633,68633 +104042,68633 +68634,68634 +68635,68635 +68636,68636 +68637,68637 +68638,68638 +68639,68639 +97491,68639 +68640,68640 +68641,68641 +68642,68642 +68643,68643 +68644,68644 +68645,68645 +68646,68646 +68647,68647 +68648,68648 +68649,68649 +68650,68650 +68651,68651 +68652,68652 +68653,68653 +68654,68654 +68655,68655 +68656,68656 +68657,68657 +68658,68658 +68659,68659 +68660,68660 +68661,68661 +11510,68661 +68662,68662 +69505,69505 +69506,69506 +69507,69507 +69508,69508 +69509,69509 +69510,69510 +69511,69511 +69633,69633 +69634,69634 +69635,69635 +5563,69635 +69636,69636 +69637,69637 +69638,69638 +69639,69639 +5698,69639 +69640,69640 +69641,69641 +69642,69642 +69643,69643 +69644,69644 +69645,69645 +69646,69646 +69647,69647 +69648,69648 +69649,69649 +69650,69650 +69651,69651 +69652,69652 +69653,69653 +69654,69654 +69655,69655 +69656,69656 +69657,69657 +69658,69658 +69786,69786 +69787,69787 +69788,69788 +68483,69788 +69789,69789 +68484,69789 +69790,69790 +68485,69790 +69791,69791 +68486,69791 +69792,69792 +68487,69792 +69793,69793 +68488,69793 +69794,69794 +68489,69794 +69795,69795 +68490,69795 +69796,69796 +68491,69796 +69797,69797 +68492,69797 +69798,69798 +68493,69798 +69799,69799 +68494,69799 +69800,69800 +233493,69800 +68495,69800 +69801,69801 +68496,69801 +69802,69802 +68497,69802 +69803,69803 +68498,69803 +69804,69804 +68499,69804 +69805,69805 +68500,69805 +69806,69806 +68501,69806 +69807,69807 +68502,69807 +69808,69808 +69809,69809 +69810,69810 +69811,69811 +69812,69812 +69813,69813 +69814,69814 +69815,69815 +69816,69816 +69817,69817 +69818,69818 +69819,69819 +69820,69820 +143232,69820 +69821,69821 +69822,69822 +69823,69823 +69824,69824 +21209,69824 +69825,69825 +69826,69826 +69827,69827 +69828,69828 +69829,69829 +69830,69830 +69831,69831 +69832,69832 +69833,69833 +69834,69834 +69835,69835 +69836,69836 +69837,69837 +35588,69837 +69838,69838 +50897,69838 +42793,69838 +59690,69838 +69839,69839 +225810,69840 +69840,69840 +69841,69841 +69842,69842 +225812,69842 +69843,69843 +225813,69843 +226179,69843 +69844,69844 +69845,69845 +50127,69845 +41934,69845 +69846,69846 +69847,69847 +69848,69848 +69849,69849 +69962,69962 +69963,69963 +69964,69964 +69965,69965 +69966,69966 +164664,69966 +70150,70150 +70151,70151 +70152,70152 +70153,70153 +70154,70154 +70155,70155 +70156,70156 +216681,70156 +70157,70157 +70158,70158 +70159,70159 +70160,70160 +70161,70161 +70162,70162 +70163,70163 +70164,70164 +208608,70164 +70165,70165 +70166,70166 +70167,70167 +239370,70167 +70168,70168 +70169,70169 +70170,70170 +70171,70171 +70172,70172 +70173,70173 +70174,70174 +70175,70175 +70176,70176 +70177,70177 +70178,70178 +70179,70179 +70180,70180 +70181,70181 +70182,70182 +70183,70183 +70184,70184 +70185,70185 +70186,70186 +70187,70187 +70188,70188 +70189,70189 +70190,70190 +70191,70191 +70192,70192 +70193,70193 +70194,70194 +30501,70194 +70195,70195 +70196,70196 +70197,70197 +70198,70198 +70199,70199 +230398,70200 +70200,70200 +1961,70200 +70201,70201 +214432,70201 +70202,70202 +70203,70203 +70204,70204 +70205,70205 +134205,70205 +126618,70205 +70206,70206 +70207,70207 +70208,70208 +70209,70209 +70210,70210 +70211,70211 +211413,70211 +70212,70212 +70213,70213 +70214,70214 +70215,70215 +74044,70215 +70216,70216 +70217,70217 +70218,70218 +70219,70219 +70220,70220 +70221,70221 +70222,70222 +70223,70223 +70224,70224 +70225,70225 +70226,70226 +70227,70227 +70228,70228 +70229,70229 +192385,70229 +70230,70230 +70231,70231 +70232,70232 +70233,70233 +70234,70234 +70235,70235 +70236,70236 +70237,70237 +70238,70238 +81637,70238 +70239,70239 +70240,70240 +70241,70241 +70242,70242 +70243,70243 +70244,70244 +70245,70245 +70246,70246 +70247,70247 +70248,70248 +70249,70249 +70250,70250 +70251,70251 +2483,70251 +70252,70252 +70253,70253 +70254,70254 +70255,70255 +70256,70256 +70257,70257 +70258,70258 +70259,70259 +70260,70260 +70261,70261 +14149,70261 +70262,70262 +70263,70263 +243857,70263 +70264,70264 +70265,70265 +70266,70266 +70267,70267 +70268,70268 +70269,70269 +70270,70270 +70271,70271 +9716,70271 +70272,70272 +70273,70273 +70274,70274 +70275,70275 +70276,70276 +70277,70277 +70278,70278 +70279,70279 +70280,70280 +70281,70281 +70282,70282 +70283,70283 +70284,70284 +70285,70285 +70286,70286 +70287,70287 +70288,70288 +70289,70289 +70290,70290 +70291,70291 +70292,70292 +70293,70293 +70294,70294 +70295,70295 +70296,70296 +70297,70297 +70298,70298 +70299,70299 +70300,70300 +70301,70301 +70302,70302 +70303,70303 +70304,70304 +70305,70305 +70306,70306 +70307,70307 +70308,70308 +70309,70309 +70310,70310 +70311,70311 +70312,70312 +70313,70313 +70314,70314 +70315,70315 +70316,70316 +70317,70317 +70318,70318 +70319,70319 +70320,70320 +70321,70321 +70322,70322 +70323,70323 +70324,70324 +70325,70325 +70326,70326 +70327,70327 +70328,70328 +70329,70329 +70330,70330 +70331,70331 +97892,70331 +70332,70332 +70333,70333 +70334,70334 +70335,70335 +70336,70336 +139957,70336 +70337,70337 +70338,70338 +70339,70339 +197749,70339 +70340,70340 +243480,70340 +70341,70341 +70342,70342 +70343,70343 +70344,70344 +70345,70345 +70346,70346 +70347,70347 +70348,70348 +70349,70349 +7054,70349 +70350,70350 +70351,70351 +70352,70352 +70353,70353 +70354,70354 +70355,70355 +70356,70356 +70357,70357 +70358,70358 +70359,70359 +70360,70360 +247216,70360 +70361,70361 +70362,70362 +70363,70363 +70364,70364 +70365,70365 +70366,70366 +70367,70367 +70368,70368 +70369,70369 +70370,70370 +226660,70370 +70371,70371 +70372,70372 +158237,70372 +70373,70373 +70374,70374 +70375,70375 +70376,70376 +70377,70377 +70378,70378 +36520,70378 +70379,70379 +70380,70380 +70381,70381 +70382,70382 +70383,70383 +70384,70384 +70385,70385 +70386,70386 +70387,70387 +70388,70388 +70389,70389 +70390,70390 +70391,70391 +70392,70392 +70393,70393 +70394,70394 +30712,70394 +70395,70395 +70396,70396 +70397,70397 +70398,70398 +70399,70399 +70400,70400 +140021,70400 +116403,70400 +70401,70401 +70402,70402 +70403,70403 +70404,70404 +70405,70405 +70406,70406 +70407,70407 +70408,70408 +70409,70409 +70410,70410 +70411,70411 +70412,70412 +70413,70413 +242934,70413 +70414,70414 +70415,70415 +70416,70416 +70417,70417 +70418,70418 +70419,70419 +70420,70420 +70421,70421 +70422,70422 +70423,70423 +70424,70424 +70425,70425 +70426,70426 +70427,70427 +70428,70428 +70429,70429 +70430,70430 +70431,70431 +70432,70432 +87248,70432 +70433,70433 +70434,70434 +70435,70435 +70436,70436 +10532,70436 +70437,70437 +199207,70437 +70438,70438 +70439,70439 +70440,70440 +70441,70441 +70442,70442 +70443,70443 +70444,70444 +70445,70445 +70446,70446 +70447,70447 +146455,70447 +70448,70448 +70449,70449 +70450,70450 +70451,70451 +70452,70452 +70453,70453 +70454,70454 +70455,70455 +70456,70456 +70457,70457 +70458,70458 +70459,70459 +70460,70460 +70461,70461 +70462,70462 +70463,70463 +70464,70464 +70465,70465 +70466,70466 +70467,70467 +70468,70468 +70469,70469 +70470,70470 +70471,70471 +70472,70472 +70473,70473 +70474,70474 +70475,70475 +70476,70476 +70477,70477 +70478,70478 +70479,70479 +70480,70480 +70481,70481 +70482,70482 +70483,70483 +70484,70484 +70485,70485 +70486,70486 +70487,70487 +70488,70488 +70489,70489 +70490,70490 +94953,70490 +70491,70491 +70492,70492 +70493,70493 +70494,70494 +70495,70495 +70496,70496 +70497,70497 +70498,70498 +70499,70499 +70500,70500 +70501,70501 +70502,70502 +70503,70503 +70504,70504 +70505,70505 +70506,70506 +70507,70507 +70508,70508 +36736,70508 +70509,70509 +70510,70510 +70511,70511 +153961,70511 +70512,70512 +70513,70513 +70514,70514 +70515,70515 +70516,70516 +70517,70517 +70518,70518 +70519,70519 +70520,70520 +70521,70521 +70522,70522 +70523,70523 +74233,70523 +70524,70524 +70525,70525 +109287,70526 +70526,70526 +70527,70527 +70528,70528 +70529,70529 +70530,70530 +70531,70531 +70532,70532 +70533,70533 +70534,70534 +70535,70535 +70536,70536 +70537,70537 +70538,70538 +70539,70539 +70540,70540 +70541,70541 +70542,70542 +70543,70543 +120179,70543 +70544,70544 +70545,70545 +70546,70546 +70547,70547 +70548,70548 +29933,70548 +390,70548 +70549,70549 +70550,70550 +70551,70551 +70552,70552 +70553,70553 +70554,70554 +70555,70555 +70641,70641 +70642,70642 +70643,70643 +70644,70644 +70645,70645 +70646,70646 +70647,70647 +70648,70648 +70649,70649 +70650,70650 +70651,70651 +70652,70652 +70653,70653 +70654,70654 +70655,70655 +70656,70656 +70657,70657 +70658,70658 +70659,70659 +70660,70660 +70661,70661 +70662,70662 +70663,70663 +70664,70664 +70665,70665 +149666,70665 +70666,70666 +71051,71051 +71052,71052 +71053,71053 +71054,71054 +71055,71055 +71056,71056 +71057,71057 +71058,71058 +71059,71059 +71060,71060 +71061,71061 +71062,71062 +71063,71063 +208554,71063 +71064,71064 +71065,71065 +71066,71066 +121387,71066 +71067,71067 +71068,71068 +71069,71069 +71070,71070 +71071,71071 +71072,71072 +71073,71073 +71074,71074 +71075,71075 +71076,71076 +71077,71077 +71078,71078 +71079,71079 +71080,71080 +71081,71081 +71082,71082 +71083,71083 +71084,71084 +59699,71084 +71085,71085 +71086,71086 +71087,71087 +71088,71088 +71089,71089 +71090,71090 +71091,71091 +71092,71092 +71093,71093 +71094,71094 +71095,71095 +71096,71096 +71097,71097 +71098,71098 +71099,71099 +226481,71099 +71139,71139 +71140,71140 +71141,71141 +71142,71142 +192558,71142 +71143,71143 +71144,71144 +71145,71145 +71146,71146 +71147,71147 +71148,71148 +71149,71149 +71150,71150 +71151,71151 +71152,71152 +71153,71153 +71154,71154 +71155,71155 +71156,71156 +71157,71157 +71158,71158 +71159,71159 +71160,71160 +71161,71161 +71162,71162 +71163,71163 +71164,71164 +71165,71165 +71166,71166 +71167,71167 +71168,71168 +71169,71169 +71170,71170 +71171,71171 +71172,71172 +71173,71173 +71174,71174 +71175,71175 +71176,71176 +155910,71176 +71177,71177 +71178,71178 +71179,71179 +71180,71180 +71181,71181 +71182,71182 +71183,71183 +71184,71184 +71185,71185 +71186,71186 +71187,71187 +71188,71188 +71189,71189 +71190,71190 +71191,71191 +71192,71192 +71193,71193 +227057,71193 +71194,71194 +38418,71194 +71195,71195 +71196,71196 +71197,71197 +71198,71198 +71199,71199 +71200,71200 +71201,71201 +71202,71202 +71203,71203 +71204,71204 +71205,71205 +71206,71206 +71207,71207 +71208,71208 +71209,71209 +71210,71210 +71211,71211 +71212,71212 +71213,71213 +71214,71214 +71215,71215 +71216,71216 +71217,71217 +179679,71218 +71218,71218 +71219,71219 +71220,71220 +71221,71221 +71222,71222 +71223,71223 +71224,71224 +71225,71225 +71226,71226 +71227,71227 +71228,71228 +71229,71229 +71230,71230 +71231,71231 +71232,71232 +71233,71233 +71234,71234 +71235,71235 +71236,71236 +71237,71237 +71238,71238 +71239,71239 +71240,71240 +71241,71241 +71242,71242 +71243,71243 +71312,71312 +71313,71313 +71314,71314 +71315,71315 +71316,71316 +71317,71317 +71318,71318 +71319,71319 +71320,71320 +71321,71321 +71322,71322 +71323,71323 +71324,71324 +71325,71325 +71326,71326 +71327,71327 +71328,71328 +71329,71329 +71330,71330 +71331,71331 +71332,71332 +71333,71333 +71334,71334 +71335,71335 +71336,71336 +71337,71337 +71338,71338 +71339,71339 +222122,71339 +71340,71340 +71341,71341 +71342,71342 +71343,71343 +71344,71344 +71345,71345 +30979,71345 +71346,71346 +71347,71347 +71348,71348 +71349,71349 +71350,71350 +60829,71350 +71351,71351 +71352,71352 +71353,71353 +71354,71354 +71355,71355 +71356,71356 +71357,71357 +71358,71358 +71359,71359 +71360,71360 +71361,71361 +71362,71362 +71363,71363 +71364,71364 +71365,71365 +71366,71366 +71367,71367 +71368,71368 +71369,71369 +71370,71370 +71371,71371 +71372,71372 +71373,71373 +157148,71373 +71374,71374 +71375,71375 +71376,71376 +227514,71376 +71377,71377 +71378,71378 +71379,71379 +71380,71380 +71381,71381 +71382,71382 +71383,71383 +71384,71384 +71385,71385 +71386,71386 +71387,71387 +136251,71387 +128664,71387 +71388,71388 +71389,71389 +71390,71390 +71391,71391 +71462,71462 +71463,71463 +71464,71464 +71465,71465 +71466,71466 +71467,71467 +71468,71468 +71469,71469 +71470,71470 +71471,71471 +71472,71472 +71602,71602 +71603,71603 +71604,71604 +71605,71605 +71606,71606 +71607,71607 +71608,71608 +71609,71609 +71610,71610 +71872,71872 +71873,71873 +71874,71874 +71875,71875 +36267,71875 +71876,71876 +71877,71877 +77330,71877 +71878,71878 +71879,71879 +71880,71880 +71881,71881 +71882,71882 +71883,71883 +71884,71884 +180564,71884 +71885,71885 +71886,71886 +238277,71887 +71887,71887 +71888,71888 +71889,71889 +71890,71890 +71891,71891 +239533,71892 +71892,71892 +71893,71893 +71894,71894 +72031,72031 +72032,72032 +72033,72033 +145393,72033 +72034,72034 +72035,72035 +72036,72036 +72037,72037 +72038,72038 +72039,72039 +72040,72040 +72041,72041 +72042,72042 +72043,72043 +72182,72182 +72183,72183 +72184,72184 +190143,72184 +72185,72185 +72186,72186 +72187,72187 +72188,72188 +72189,72189 +72190,72190 +72191,72191 +72192,72192 +72193,72193 +72194,72194 +72195,72195 +72196,72196 +72197,72197 +72198,72198 +72259,72259 +209997,72259 +72260,72260 +72261,72261 +72262,72262 +72263,72263 +72264,72264 +72265,72265 +72266,72266 +72267,72267 +72268,72268 +72269,72269 +72270,72270 +72271,72271 +72272,72272 +72273,72273 +72274,72274 +72275,72275 +72276,72276 +72277,72277 +72278,72278 +72279,72279 +72280,72280 +72281,72281 +72282,72282 +72283,72283 +72284,72284 +72285,72285 +72286,72286 +72287,72287 +72288,72288 +72289,72289 +72290,72290 +72291,72291 +72292,72292 +72293,72293 +72294,72294 +72295,72295 +72296,72296 +72297,72297 +72298,72298 +72299,72299 +72300,72300 +72301,72301 +72302,72302 +72303,72303 +72304,72304 +72305,72305 +72478,72478 +72486,72479 +72479,72479 +72480,72480 +72481,72481 +72482,72482 +72483,72483 +72484,72484 +72485,72485 +72486,72486 +72479,72486 +72487,72487 +72488,72488 +72489,72489 +72490,72490 +72491,72491 +72492,72492 +72493,72493 +72494,72494 +72495,72495 +60572,72495 +72496,72496 +72497,72497 +72498,72498 +72499,72499 +78519,72499 +72500,72500 +72501,72501 +72502,72502 +72503,72503 +72504,72504 +72505,72505 +72506,72506 +72507,72507 +141244,72507 +72508,72508 +72509,72509 +72510,72510 +72511,72511 +72512,72512 +72513,72513 +72514,72514 +72515,72515 +72516,72516 +72517,72517 +72518,72518 +72519,72519 +77452,72519 +72520,72520 +72521,72521 +72522,72522 +72523,72523 +72524,72524 +72525,72525 +72526,72526 +72527,72527 +72528,72528 +72529,72529 +72530,72530 +72531,72531 +72532,72532 +72533,72533 +72534,72534 +72535,72535 +72536,72536 +72537,72537 +72538,72538 +72539,72539 +72540,72540 +232139,72541 +72541,72541 +72542,72542 +72543,72543 +72544,72544 +72545,72545 +72546,72546 +72547,72547 +72548,72548 +72549,72549 +241521,72549 +72550,72550 +72551,72551 +72552,72552 +72553,72553 +72554,72554 +72555,72555 +72556,72556 +72557,72557 +72558,72558 +72559,72559 +72560,72560 +72561,72561 +223772,72561 +72562,72562 +72563,72563 +72564,72564 +72565,72565 +72566,72566 +205031,72566 +72567,72567 +72568,72568 +72569,72569 +72570,72570 +72571,72571 +72572,72572 +72573,72573 +72574,72574 +72575,72575 +72576,72576 +72577,72577 +72578,72578 +72579,72579 +72580,72580 +72581,72581 +72582,72582 +72583,72583 +86876,72583 +72584,72584 +72585,72585 +228724,72585 +72586,72586 +64312,72586 +72587,72587 +72588,72588 +72589,72589 +100929,72589 +72590,72590 +72591,72591 +72592,72592 +72593,72593 +72594,72594 +72595,72595 +72596,72596 +72597,72597 +72598,72598 +72599,72599 +83236,72600 +72600,72600 +72601,72601 +72602,72602 +72603,72603 +14504,72603 +72604,72604 +72605,72605 +72873,72873 +72874,72874 +72875,72875 +72876,72876 +72877,72877 +72878,72878 +72879,72879 +72880,72880 +72881,72881 +72882,72882 +72883,72883 +72884,72884 +72885,72885 +39382,72885 +37898,72885 +72886,72886 +39383,72886 +37899,72886 +15604,72886 +72887,72887 +72888,72888 +72889,72889 +72890,72890 +72891,72891 +72892,72892 +237486,72892 +72893,72893 +72894,72894 +72895,72895 +72896,72896 +139189,72896 +72897,72897 +72898,72898 +72899,72899 +72900,72900 +72901,72901 +72902,72902 +72903,72903 +72904,72904 +72905,72905 +73019,73019 +235020,73019 +73020,73020 +73021,73021 +73022,73022 +10193,73022 +73023,73023 +73024,73024 +76496,73024 +73025,73025 +73026,73026 +73027,73027 +73028,73028 +73029,73029 +73030,73030 +73031,73031 +73032,73032 +11916,73032 +73033,73033 +73034,73034 +73035,73035 +73036,73036 +73037,73037 +73038,73038 +73039,73039 +73040,73040 +73041,73041 +73042,73042 +73043,73043 +133714,73043 +126127,73043 +73044,73044 +73045,73045 +73046,73046 +13835,73046 +73047,73047 +73048,73048 +73049,73049 +73050,73050 +73051,73051 +88734,73051 +107295,73051 +73052,73052 +73053,73053 +73054,73054 +73055,73055 +73056,73056 +73057,73057 +73058,73058 +73059,73059 +161187,73059 +73060,73060 +73061,73061 +73062,73062 +73063,73063 +73064,73064 +73065,73065 +73066,73066 +73067,73067 +73068,73068 +73069,73069 +73070,73070 +73071,73071 +73072,73072 +187381,73072 +73073,73073 +73074,73074 +73075,73075 +73076,73076 +5130,73076 +73077,73077 +73078,73078 +73079,73079 +73080,73080 +73081,73081 +73082,73082 +73083,73083 +73084,73084 +73085,73085 +73086,73086 +73087,73087 +73569,73569 +73570,73570 +73571,73571 +29065,73571 +73572,73572 +73573,73573 +73574,73574 +14540,73574 +73575,73575 +73576,73576 +73577,73577 +73578,73578 +73579,73579 +73580,73580 +73616,73616 +73617,73617 +73618,73618 +73619,73619 +73620,73620 +73621,73621 +73622,73622 +73623,73623 +208589,73623 +73624,73624 +73625,73625 +73626,73626 +73690,73690 +73691,73691 +73692,73692 +73693,73693 +73694,73694 +73695,73695 +73696,73696 +73697,73697 +73698,73698 +73748,73748 +73749,73749 +73750,73750 +177509,73750 +171545,73750 +73751,73751 +73752,73752 +73753,73753 +73754,73754 +224772,73754 +73755,73755 +73756,73756 +73757,73757 +73758,73758 +73759,73759 +73760,73760 +73761,73761 +73762,73762 +73763,73763 +73764,73764 +73765,73765 +73766,73766 +150747,73767 +73767,73767 +73768,73768 +73769,73769 +73770,73770 +73808,73808 +73809,73809 +73810,73810 +73811,73811 +73812,73812 +73813,73813 +73814,73814 +73815,73815 +102133,73815 +73816,73816 +73817,73817 +73818,73818 +73819,73819 +73820,73820 +73889,73889 +73890,73890 +73891,73891 +73892,73892 +200878,73892 +73893,73893 +73894,73894 +73895,73895 +73896,73896 +74016,74016 +74017,74017 +74018,74018 +74019,74019 +74020,74020 +74021,74021 +200628,74021 +74022,74022 +74023,74023 +74024,74024 +74025,74025 +74026,74026 +74027,74027 +74028,74028 +74029,74029 +74030,74030 +74031,74031 +74032,74032 +74033,74033 +74034,74034 +74035,74035 +74036,74036 +74037,74037 +74038,74038 +74039,74039 +74040,74040 +74041,74041 +74042,74042 +74043,74043 +74044,74044 +70215,74044 +74045,74045 +74046,74046 +74047,74047 +74048,74048 +74049,74049 +74050,74050 +74051,74051 +74052,74052 +74053,74053 +74054,74054 +74055,74055 +74056,74056 +74057,74057 +74058,74058 +74059,74059 +74060,74060 +74061,74061 +74062,74062 +74063,74063 +74064,74064 +74065,74065 +74066,74066 +74067,74067 +74068,74068 +74069,74069 +7759,74069 +11610,74069 +74070,74070 +97156,74070 +74071,74071 +74072,74072 +93262,74072 +74073,74073 +74074,74074 +74075,74075 +74076,74076 +74077,74077 +74078,74078 +74079,74079 +74080,74080 +74081,74081 +74082,74082 +74083,74083 +74084,74084 +74085,74085 +74086,74086 +74087,74087 +74088,74088 +74089,74089 +74090,74090 +74091,74091 +74127,74127 +74128,74128 +74129,74129 +74130,74130 +74131,74131 +74132,74132 +74133,74133 +74134,74134 +74135,74135 +74136,74136 +74137,74137 +74138,74138 +74139,74139 +74140,74140 +74141,74141 +74142,74142 +74143,74143 +74144,74144 +74145,74145 +74146,74146 +74147,74147 +74148,74148 +74149,74149 +74150,74150 +74151,74151 +74152,74152 +74153,74153 +166677,74153 +74154,74154 +74155,74155 +74156,74156 +74157,74157 +136044,74157 +128457,74157 +74158,74158 +31144,74158 +74159,74159 +74160,74160 +74161,74161 +74162,74162 +74163,74163 +74164,74164 +60241,74164 +74165,74165 +74166,74166 +74167,74167 +74168,74168 +74169,74169 +74170,74170 +74171,74171 +60248,74171 +74172,74172 +74173,74173 +74174,74174 +74175,74175 +74176,74176 +74177,74177 +74178,74178 +74179,74179 +179331,74179 +74180,74180 +74181,74181 +74182,74182 +74183,74183 +74184,74184 +212340,74184 +74185,74185 +74186,74186 +74187,74187 +74188,74188 +74189,74189 +74190,74190 +74191,74191 +74192,74192 +74193,74193 +120415,74193 +74194,74194 +74195,74195 +74196,74196 +74197,74197 +74198,74198 +74199,74199 +74200,74200 +74201,74201 +74202,74202 +74203,74203 +74204,74204 +74205,74205 +74206,74206 +74207,74207 +74208,74208 +74209,74209 +74210,74210 +74211,74211 +74212,74212 +74213,74213 +74214,74214 +74215,74215 +161769,74215 +376,74215 +74216,74216 +74217,74217 +74218,74218 +74219,74219 +74220,74220 +74221,74221 +74222,74222 +74223,74223 +74224,74224 +74225,74225 +74226,74226 +74227,74227 +74228,74228 +74229,74229 +74230,74230 +74231,74231 +74232,74232 +74233,74233 +70523,74233 +74234,74234 +74235,74235 +74236,74236 +74237,74237 +74238,74238 +74239,74239 +74240,74240 +74241,74241 +74242,74242 +74243,74243 +74244,74244 +74245,74245 +74246,74246 +74247,74247 +227619,74247 +74248,74248 +74249,74249 +74250,74250 +247590,74250 +74251,74251 +74252,74252 +74253,74253 +74254,74254 +74255,74255 +74256,74256 +74257,74257 +234444,74257 +74258,74258 +74259,74259 +74260,74260 +74261,74261 +74262,74262 +74263,74263 +74264,74264 +74265,74265 +74266,74266 +74267,74267 +74268,74268 +74269,74269 +74270,74270 +74271,74271 +74272,74272 +74273,74273 +74274,74274 +74275,74275 +74276,74276 +74277,74277 +74278,74278 +74279,74279 +74280,74280 +74281,74281 +74282,74282 +74283,74283 +74284,74284 +74285,74285 +74286,74286 +74287,74287 +74288,74288 +74289,74289 +74290,74290 +74291,74291 +74292,74292 +74293,74293 +74294,74294 +74295,74295 +74296,74296 +74297,74297 +74298,74298 +74299,74299 +74300,74300 +74301,74301 +74302,74302 +74303,74303 +74304,74304 +74305,74305 +74306,74306 +74307,74307 +26337,74307 +74308,74308 +74309,74309 +74310,74310 +74311,74311 +74515,74515 +29805,74515 +74516,74516 +74517,74517 +74518,74518 +74519,74519 +74520,74520 +74521,74521 +74522,74522 +74694,74694 +74695,74695 +74696,74696 +101854,74696 +74697,74697 +74698,74698 +74699,74699 +74700,74700 +74701,74701 +74702,74702 +74703,74703 +74704,74704 +66347,74704 +74705,74705 +74706,74706 +74707,74707 +165177,74707 +74708,74708 +227724,74708 +74709,74709 +74710,74710 +139599,74710 +74711,74711 +74712,74712 +20231,74712 +74713,74713 +74714,74714 +74715,74715 +157847,74715 +74716,74716 +74717,74717 +74718,74718 +74719,74719 +74720,74720 +74721,74721 +74722,74722 +74723,74723 +74724,74724 +74725,74725 +74726,74726 +74727,74727 +74728,74728 +74907,74907 +74908,74908 +74909,74909 +74910,74910 +74911,74911 +74912,74912 +74913,74913 +74914,74914 +85784,74914 +74915,74915 +74916,74916 +74917,74917 +74918,74918 +74919,74919 +74920,74920 +179071,74921 +74921,74921 +74922,74922 +74923,74923 +74924,74924 +74925,74925 +74926,74926 +86403,74926 +74927,74927 +74928,74928 +74929,74929 +74930,74930 +74931,74931 +74932,74932 +74933,74933 +74934,74934 +74935,74935 +74936,74936 +74937,74937 +74938,74938 +74939,74939 +74940,74940 +74941,74941 +74942,74942 +74943,74943 +74944,74944 +74945,74945 +74946,74946 +74947,74947 +51303,74947 +43199,74947 +74948,74948 +74949,74949 +211185,74949 +74950,74950 +74951,74951 +74952,74952 +74953,74953 +74954,74954 +74955,74955 +74956,74956 +81432,74956 +74957,74957 +74958,74958 +74959,74959 +74960,74960 +74961,74961 +74962,74962 +32451,74962 +74963,74963 +74964,74964 +74965,74965 +74966,74966 +74967,74967 +161627,74967 +74968,74968 +74969,74969 +74970,74970 +74971,74971 +74972,74972 +74973,74973 +74974,74974 +74975,74975 +138774,74975 +74976,74976 +74977,74977 +74978,74978 +74979,74979 +74980,74980 +74981,74981 +74982,74982 +74983,74983 +74984,74984 +74985,74985 +74986,74986 +74987,74987 +74988,74988 +74989,74989 +74990,74990 +74991,74991 +208739,74991 +74992,74992 +74993,74993 +74994,74994 +74995,74995 +74996,74996 +74997,74997 +74998,74998 +74999,74999 +75000,75000 +75001,75001 +75002,75002 +75003,75003 +75004,75004 +166483,75004 +75005,75005 +75006,75006 +75007,75007 +75008,75008 +75009,75009 +75010,75010 +75011,75011 +75012,75012 +75013,75013 +75014,75014 +75015,75015 +75016,75016 +75017,75017 +75018,75018 +75019,75019 +75020,75020 +75021,75021 +75022,75022 +75023,75023 +75024,75024 +75025,75025 +75026,75026 +75027,75027 +75028,75028 +75029,75029 +75030,75030 +75031,75031 +75032,75032 +75033,75033 +75034,75034 +75035,75035 +75036,75036 +75037,75037 +75038,75038 +75039,75039 +75040,75040 +235868,75040 +75041,75041 +75042,75042 +75043,75043 +75044,75044 +75045,75045 +75046,75046 +75047,75047 +75048,75048 +75049,75049 +75050,75050 +75051,75051 +75052,75052 +75053,75053 +75054,75054 +75055,75055 +75056,75056 +75057,75057 +75058,75058 +75059,75059 +75060,75060 +75061,75061 +75062,75062 +75063,75063 +75064,75064 +75065,75065 +75066,75066 +75067,75067 +185579,75067 +75068,75068 +75069,75069 +75070,75070 +75071,75071 +75072,75072 +75073,75073 +75074,75074 +75075,75075 +75076,75076 +75077,75077 +75078,75078 +75079,75079 +75080,75080 +75081,75081 +75082,75082 +75083,75083 +75084,75084 +75085,75085 +75086,75086 +157061,75086 +75087,75087 +75088,75088 +75089,75089 +75090,75090 +75091,75091 +75092,75092 +75093,75093 +75094,75094 +75095,75095 +75096,75096 +75097,75097 +75098,75098 +35581,75098 +75099,75099 +75100,75100 +75101,75101 +75102,75102 +75103,75103 +75104,75104 +75105,75105 +75106,75106 +75107,75107 +75108,75108 +75109,75109 +207468,75109 +75110,75110 +75111,75111 +75112,75112 +75113,75113 +183619,75113 +75114,75114 +75115,75115 +75116,75116 +75117,75117 +75118,75118 +75119,75119 +75120,75120 +218076,75120 +75121,75121 +75122,75122 +75123,75123 +75124,75124 +75125,75125 +75126,75126 +75127,75127 +243952,75127 +75128,75128 +75129,75129 +75130,75130 +75131,75131 +75132,75132 +75133,75133 +75134,75134 +75135,75135 +75136,75136 +75137,75137 +75138,75138 +75139,75139 +75140,75140 +75141,75141 +64987,75141 +75142,75142 +75143,75143 +75144,75144 +75145,75145 +185517,75145 +75146,75146 +75147,75147 +75148,75148 +75149,75149 +75150,75150 +75151,75151 +75152,75152 +75153,75153 +75154,75154 +113342,75154 +75155,75155 +75156,75156 +75157,75157 +75158,75158 +75159,75159 +75160,75160 +75161,75161 +75162,75162 +75163,75163 +75164,75164 +75165,75165 +75166,75166 +75167,75167 +75168,75168 +75169,75169 +75170,75170 +50317,75170 +42125,75170 +75171,75171 +14639,75171 +75172,75172 +75173,75173 +75174,75174 +75175,75175 +75176,75176 +75177,75177 +75178,75178 +75179,75179 +84594,75179 +75180,75180 +75181,75181 +75182,75182 +75183,75183 +75184,75184 +75185,75185 +75186,75186 +75187,75187 +75188,75188 +75189,75189 +75190,75190 +75191,75191 +75192,75192 +75193,75193 +75194,75194 +75195,75195 +75196,75196 +75197,75197 +75198,75198 +75199,75199 +75200,75200 +75201,75201 +75202,75202 +75203,75203 +75204,75204 +230372,75204 +75205,75205 +97970,75205 +75206,75206 +120190,75206 +75207,75207 +75208,75208 +75209,75209 +75210,75210 +75211,75211 +75212,75212 +75213,75213 +75214,75214 +75215,75215 +30246,75215 +75216,75216 +75217,75217 +220456,75217 +75218,75218 +75219,75219 +75220,75220 +75221,75221 +75222,75222 +75223,75223 +75224,75224 +75225,75225 +75226,75226 +75227,75227 +75228,75228 +75229,75229 +75230,75230 +75231,75231 +75232,75232 +75233,75233 +75234,75234 +75235,75235 +75236,75236 +75237,75237 +75238,75238 +75239,75239 +75240,75240 +75241,75241 +75242,75242 +75243,75243 +76119,75243 +75244,75244 +75245,75245 +75246,75246 +75247,75247 +75248,75248 +75249,75249 +75250,75250 +75251,75251 +75252,75252 +75253,75253 +75254,75254 +75255,75255 +75256,75256 +75257,75257 +75258,75258 +75259,75259 +75260,75260 +75261,75261 +75262,75262 +75263,75263 +75264,75264 +75265,75265 +75266,75266 +75267,75267 +75268,75268 +75269,75269 +75270,75270 +75271,75271 +75272,75272 +75273,75273 +75274,75274 +75275,75275 +75276,75276 +75277,75277 +75278,75278 +75279,75279 +75280,75280 +75281,75281 +75282,75282 +75283,75283 +75284,75284 +7157,75284 +75285,75285 +75286,75286 +75287,75287 +75288,75288 +75289,75289 +75290,75290 +75291,75291 +75292,75292 +75293,75293 +75294,75294 +75295,75295 +75296,75296 +75297,75297 +75298,75298 +75299,75299 +75300,75300 +75301,75301 +75302,75302 +75303,75303 +75304,75304 +75305,75305 +75306,75306 +75307,75307 +75308,75308 +97965,75308 +152908,75308 +75309,75309 +75310,75310 +75311,75311 +75312,75312 +75313,75313 +75314,75314 +75315,75315 +75316,75316 +75317,75317 +75318,75318 +75319,75319 +75320,75320 +75321,75321 +75322,75322 +75323,75323 +75324,75324 +75325,75325 +75326,75326 +75327,75327 +75328,75328 +75329,75329 +75330,75330 +30752,75330 +75331,75331 +75332,75332 +75333,75333 +75334,75334 +75335,75335 +75336,75336 +75337,75337 +75338,75338 +75339,75339 +75340,75340 +75341,75341 +75342,75342 +75343,75343 +75344,75344 +75345,75345 +75346,75346 +75347,75347 +75348,75348 +75349,75349 +75350,75350 +75351,75351 +75352,75352 +75353,75353 +75354,75354 +34257,75354 +75355,75355 +75356,75356 +75357,75357 +75358,75358 +75359,75359 +75360,75360 +75361,75361 +75362,75362 +39921,75362 +75363,75363 +75364,75364 +75365,75365 +75366,75366 +75367,75367 +75368,75368 +75369,75369 +75370,75370 +75371,75371 +75372,75372 +75373,75373 +75374,75374 +75375,75375 +75376,75376 +75377,75377 +75378,75378 +75379,75379 +75380,75380 +75381,75381 +75382,75382 +75383,75383 +75384,75384 +75385,75385 +75386,75386 +75387,75387 +75388,75388 +75389,75389 +75390,75390 +75391,75391 +75392,75392 +75393,75393 +75394,75394 +75395,75395 +141254,75396 +75396,75396 +75397,75397 +75398,75398 +75399,75399 +75400,75400 +187450,75400 +26213,75400 +75401,75401 +75402,75402 +75403,75403 +75404,75404 +75405,75405 +75406,75406 +75407,75407 +62762,75407 +75408,75408 +75409,75409 +75410,75410 +75411,75411 +75412,75412 +75413,75413 +75414,75414 +209401,75414 +75415,75415 +75416,75416 +75417,75417 +75418,75418 +75419,75419 +75420,75420 +75421,75421 +75422,75422 +75423,75423 +75424,75424 +75425,75425 +75426,75426 +10564,75426 +75427,75427 +75428,75428 +75429,75429 +75430,75430 +75431,75431 +75432,75432 +75433,75433 +75434,75434 +75435,75435 +75436,75436 +196499,75436 +75437,75437 +75438,75438 +75439,75439 +75440,75440 +75441,75441 +75442,75442 +75443,75443 +75444,75444 +75445,75445 +75446,75446 +75447,75447 +75448,75448 +75449,75449 +75450,75450 +75451,75451 +75452,75452 +75453,75453 +76616,75453 +75454,75454 +75455,75455 +75456,75456 +75457,75457 +75458,75458 +75459,75459 +75460,75460 +75461,75461 +75462,75462 +75463,75463 +75464,75464 +75465,75465 +75466,75466 +75467,75467 +75468,75468 +53706,75468 +45602,75468 +75469,75469 +75470,75470 +75471,75471 +242845,75471 +75472,75472 +75473,75473 +75474,75474 +86906,75474 +75475,75475 +75476,75476 +75477,75477 +75478,75478 +75479,75479 +75480,75480 +75481,75481 +75482,75482 +75483,75483 +75484,75484 +75485,75485 +75486,75486 +75487,75487 +112022,75487 +75488,75488 +75489,75489 +75490,75490 +75491,75491 +75492,75492 +75493,75493 +75494,75494 +75495,75495 +75496,75496 +75497,75497 +75498,75498 +75499,75499 +75500,75500 +75501,75501 +75502,75502 +23674,75502 +75503,75503 +75504,75504 +75505,75505 +75506,75506 +75507,75507 +75508,75508 +75509,75509 +75510,75510 +75511,75511 +75512,75512 +75513,75513 +75514,75514 +75515,75515 +75516,75516 +75517,75517 +75518,75518 +75519,75519 +75520,75520 +75521,75521 +75522,75522 +75523,75523 +75524,75524 +75525,75525 +75526,75526 +75527,75527 +75528,75528 +75529,75529 +75530,75530 +75531,75531 +75532,75532 +75533,75533 +75534,75534 +75535,75535 +75536,75536 +75537,75537 +75538,75538 +75539,75539 +181334,75539 +75540,75540 +75541,75541 +75542,75542 +75543,75543 +75544,75544 +75545,75545 +75546,75546 +150644,75546 +75547,75547 +75548,75548 +75549,75549 +75550,75550 +75551,75551 +75552,75552 +75553,75553 +75554,75554 +75555,75555 +148319,75555 +75556,75556 +75557,75557 +75558,75558 +75559,75559 +75560,75560 +75561,75561 +75562,75562 +75563,75563 +75564,75564 +75565,75565 +109160,75565 +75566,75566 +75567,75567 +75568,75568 +75569,75569 +75570,75570 +109267,75570 +75571,75571 +75572,75572 +75573,75573 +75574,75574 +75575,75575 +75576,75576 +75577,75577 +75578,75578 +75579,75579 +75580,75580 +75581,75581 +75582,75582 +75583,75583 +199847,75583 +75584,75584 +75585,75585 +9827,75585 +75586,75586 +75587,75587 +75588,75588 +75589,75589 +75590,75590 +75591,75591 +75592,75592 +75593,75593 +75594,75594 +75595,75595 +75596,75596 +75597,75597 +75598,75598 +75599,75599 +75600,75600 +75601,75601 +137758,75601 +130171,75601 +75602,75602 +75603,75603 +75604,75604 +75605,75605 +75606,75606 +75607,75607 +75608,75608 +75609,75609 +75610,75610 +75611,75611 +75612,75612 +75613,75613 +75614,75614 +75615,75615 +75690,75690 +75691,75691 +75692,75692 +75693,75693 +75694,75694 +75695,75695 +75696,75696 +75697,75697 +75698,75698 +75699,75699 +75700,75700 +75701,75701 +75702,75702 +75703,75703 +75704,75704 +75705,75705 +75706,75706 +75707,75707 +75708,75708 +75709,75709 +75710,75710 +75711,75711 +75712,75712 +33037,75712 +75713,75713 +75714,75714 +75715,75715 +75716,75716 +75717,75717 +75718,75718 +75719,75719 +75720,75720 +75721,75721 +75722,75722 +75723,75723 +75724,75724 +75725,75725 +75726,75726 +75727,75727 +75728,75728 +75729,75729 +75730,75730 +75731,75731 +75732,75732 +75733,75733 +75734,75734 +75735,75735 +216249,75735 +75736,75736 +75737,75737 +199724,75738 +75738,75738 +75739,75739 +75740,75740 +75741,75741 +75742,75742 +75743,75743 +75744,75744 +75745,75745 +75746,75746 +211595,75746 +75747,75747 +75785,75785 +75786,75786 +75787,75787 +75788,75788 +36469,75788 +75789,75789 +75790,75790 +75791,75791 +75792,75792 +75793,75793 +75794,75794 +75795,75795 +75796,75796 +75797,75797 +75798,75798 +76088,75798 +75799,75799 +75800,75800 +75801,75801 +75802,75802 +75803,75803 +75804,75804 +75805,75805 +75806,75806 +75807,75807 +75808,75808 +75809,75809 +75810,75810 +75811,75811 +55093,75811 +46989,75811 +75812,75812 +75813,75813 +75814,75814 +75815,75815 +75816,75816 +75817,75817 +103634,75817 +75818,75818 +75819,75819 +75820,75820 +75821,75821 +75822,75822 +75823,75823 +75824,75824 +75825,75825 +75826,75826 +75827,75827 +75828,75828 +75829,75829 +75830,75830 +75831,75831 +75832,75832 +75833,75833 +75834,75834 +157389,75834 +75835,75835 +75836,75836 +75837,75837 +75838,75838 +76003,75838 +75839,75839 +75840,75840 +75841,75841 +164845,75841 +75842,75842 +75843,75843 +75844,75844 +75845,75845 +75846,75846 +75847,75847 +75848,75848 +75849,75849 +75850,75850 +75851,75851 +235792,75851 +75852,75852 +75853,75853 +75854,75854 +75855,75855 +75856,75856 +75857,75857 +75858,75858 +75859,75859 +75860,75860 +75861,75861 +75862,75862 +75863,75863 +75864,75864 +75865,75865 +75866,75866 +75867,75867 +75868,75868 +75869,75869 +75870,75870 +75871,75871 +75872,75872 +75873,75873 +75874,75874 +75875,75875 +75876,75876 +75877,75877 +75878,75878 +75879,75879 +75880,75880 +75881,75881 +75882,75882 +75883,75883 +75884,75884 +75885,75885 +75886,75886 +75887,75887 +75888,75888 +75889,75889 +75890,75890 +75891,75891 +75892,75892 +75893,75893 +75894,75894 +75895,75895 +75896,75896 +75897,75897 +75898,75898 +75899,75899 +75900,75900 +75901,75901 +75902,75902 +75903,75903 +75904,75904 +75905,75905 +138719,75905 +75906,75906 +75907,75907 +75908,75908 +75909,75909 +75910,75910 +75911,75911 +75912,75912 +75913,75913 +178845,75913 +75914,75914 +75915,75915 +75916,75916 +75917,75917 +26051,75917 +75918,75918 +75919,75919 +75920,75920 +75921,75921 +114894,75921 +108066,75921 +75922,75922 +75923,75923 +75924,75924 +75925,75925 +75926,75926 +75927,75927 +75928,75928 +75929,75929 +75930,75930 +75931,75931 +75932,75932 +76876,75932 +75933,75933 +75934,75934 +75935,75935 +75936,75936 +75937,75937 +75938,75938 +75939,75939 +75940,75940 +75941,75941 +75942,75942 +75943,75943 +75980,75980 +75981,75981 +75982,75982 +75983,75983 +75984,75984 +75985,75985 +143367,75985 +75986,75986 +75987,75987 +75988,75988 +75989,75989 +75990,75990 +75991,75991 +75992,75992 +75993,75993 +75994,75994 +75995,75995 +75996,75996 +75997,75997 +75998,75998 +162432,75998 +75999,75999 +76000,76000 +76001,76001 +76002,76002 +76003,76003 +75838,76003 +76004,76004 +49367,76004 +41174,76004 +76005,76005 +240941,76006 +76006,76006 +76007,76007 +76008,76008 +189423,76008 +76009,76009 +216152,76009 +76010,76010 +76011,76011 +76012,76012 +76013,76013 +146986,76013 +205242,76013 +76014,76014 +76015,76015 +76016,76016 +76017,76017 +192735,76017 +76018,76018 +76019,76019 +76020,76020 +76021,76021 +76022,76022 +64808,76022 +114157,76022 +76023,76023 +76024,76024 +76025,76025 +76026,76026 +76027,76027 +76028,76028 +76029,76029 +76030,76030 +76031,76031 +76032,76032 +76033,76033 +76034,76034 +230357,76034 +76035,76035 +203678,76035 +76036,76036 +76037,76037 +76038,76038 +76039,76039 +54692,76039 +46588,76039 +76040,76040 +76041,76041 +76042,76042 +76043,76043 +76044,76044 +76045,76045 +76046,76046 +76047,76047 +76048,76048 +147651,76048 +76049,76049 +76050,76050 +76051,76051 +76052,76052 +76053,76053 +76054,76054 +76055,76055 +76056,76056 +76057,76057 +76058,76058 +76059,76059 +76060,76060 +154330,76060 +76061,76061 +76062,76062 +76063,76063 +76064,76064 +76065,76065 +76066,76066 +76067,76067 +76068,76068 +76069,76069 +76070,76070 +76071,76071 +11877,76071 +134089,76071 +126502,76071 +76072,76072 +76073,76073 +76074,76074 +76182,76074 +76075,76075 +76183,76075 +76076,76076 +76077,76077 +76078,76078 +76079,76079 +76080,76080 +76081,76081 +76082,76082 +76083,76083 +76084,76084 +76085,76085 +76086,76086 +76087,76087 +76088,76088 +75798,76088 +76089,76089 +76090,76090 +76091,76091 +76092,76092 +76093,76093 +76094,76094 +76095,76095 +76096,76096 +76097,76097 +76098,76098 +76099,76099 +242868,76099 +76100,76100 +76101,76101 +76102,76102 +76103,76103 +76104,76104 +76105,76105 +76106,76106 +76107,76107 +76108,76108 +76109,76109 +76110,76110 +76111,76111 +76112,76112 +76113,76113 +76114,76114 +76115,76115 +76116,76116 +76117,76117 +76118,76118 +76119,76119 +75243,76119 +226311,76119 +76120,76120 +216429,76120 +76121,76121 +76181,76181 +76182,76182 +76074,76182 +76183,76183 +76075,76183 +76184,76184 +76185,76185 +76186,76186 +76187,76187 +76188,76188 +76189,76189 +153661,76189 +158423,76189 +76190,76190 +76191,76191 +76192,76192 +76193,76193 +76194,76194 +76274,76274 +76275,76275 +76276,76276 +76277,76277 +142645,76277 +76278,76278 +76279,76279 +194933,76279 +194168,76279 +76280,76280 +76281,76281 +76282,76282 +76283,76283 +76284,76284 +76285,76285 +76286,76286 +76287,76287 +76288,76288 +76289,76289 +76290,76290 +76291,76291 +76292,76292 +76293,76293 +76294,76294 +76295,76295 +76296,76296 +76297,76297 +76298,76298 +99674,76298 +76299,76299 +76300,76300 +76301,76301 +76302,76302 +76303,76303 +76304,76304 +76305,76305 +76306,76306 +76307,76307 +76308,76308 +76309,76309 +76310,76310 +76311,76311 +76312,76312 +76313,76313 +76314,76314 +76315,76315 +76316,76316 +76317,76317 +76318,76318 +186960,76318 +76319,76319 +80441,76319 +76320,76320 +76321,76321 +76322,76322 +76323,76323 +76324,76324 +76325,76325 +76326,76326 +76327,76327 +76328,76328 +212006,76328 +76329,76329 +76330,76330 +76331,76331 +76332,76332 +76333,76333 +76334,76334 +76373,76334 +76335,76335 +76336,76336 +92449,76336 +76337,76337 +211179,76337 +62465,76337 +76338,76338 +76339,76339 +76340,76340 +76341,76341 +79879,76341 +76342,76342 +76343,76343 +178069,76343 +172105,76343 +76344,76344 +76345,76345 +76346,76346 +76347,76347 +76348,76348 +76349,76349 +76350,76350 +76351,76351 +76352,76352 +76353,76353 +55428,76353 +47324,76353 +76354,76354 +76355,76355 +76356,76356 +76357,76357 +76358,76358 +152996,76358 +76359,76359 +76360,76360 +76361,76361 +76362,76362 +76363,76363 +76364,76364 +92658,76364 +235725,76365 +76365,76365 +76366,76366 +104657,76366 +76367,76367 +76368,76368 +76369,76369 +76370,76370 +76371,76371 +76372,76372 +76373,76373 +76334,76373 +76374,76374 +76375,76375 +186795,76375 +76376,76376 +76377,76377 +76378,76378 +76379,76379 +50112,76379 +41919,76379 +76380,76380 +76381,76381 +76382,76382 +76383,76383 +22022,76383 +76384,76384 +76385,76385 +76386,76386 +76387,76387 +76388,76388 +76389,76389 +76390,76390 +76391,76391 +208383,76391 +76392,76392 +76393,76393 +98122,76393 +178207,76394 +172243,76394 +76394,76394 +76395,76395 +76396,76396 +76397,76397 +206946,76397 +77465,76397 +76398,76398 +211184,76399 +76399,76399 +76400,76400 +76401,76401 +76402,76402 +76403,76403 +269,76403 +76404,76404 +248923,76404 +76405,76405 +76406,76406 +76407,76407 +76408,76408 +76409,76409 +76410,76410 +141209,76410 +76411,76411 +76412,76412 +76413,76413 +76414,76414 +76415,76415 +76416,76416 +76417,76417 +76418,76418 +76419,76419 +76420,76420 +76421,76421 +76422,76422 +76423,76423 +76424,76424 +76425,76425 +76426,76426 +76427,76427 +76428,76428 +76429,76429 +76430,76430 +76431,76431 +76432,76432 +105028,76433 +76433,76433 +76434,76434 +76435,76435 +76436,76436 +76437,76437 +76438,76438 +76439,76439 +76440,76440 +76441,76441 +76442,76442 +76443,76443 +76444,76444 +90994,76444 +76445,76445 +76446,76446 +76447,76447 +76448,76448 +76449,76449 +76450,76450 +76451,76451 +76452,76452 +76453,76453 +76454,76454 +76455,76455 +76456,76456 +114805,76456 +76457,76457 +76458,76458 +76459,76459 +76460,76460 +76461,76461 +76462,76462 +53737,76462 +45633,76462 +76463,76463 +50043,76463 +41850,76463 +76464,76464 +76465,76465 +76466,76466 +76467,76467 +5085,76467 +76468,76468 +76469,76469 +76470,76470 +76471,76471 +76472,76472 +105639,76472 +76473,76473 +76474,76474 +76475,76475 +76476,76476 +76477,76477 +76478,76478 +76479,76479 +76480,76480 +76481,76481 +76482,76482 +76483,76483 +76484,76484 +76485,76485 +76486,76486 +12175,76486 +76487,76487 +76488,76488 +76489,76489 +76490,76490 +76491,76491 +76492,76492 +76493,76493 +76494,76494 +76495,76495 +76496,76496 +73024,76496 +76497,76497 +76498,76498 +76499,76499 +76500,76500 +76501,76501 +76502,76502 +76503,76503 +76504,76504 +76505,76505 +76506,76506 +76507,76507 +76508,76508 +76509,76509 +76510,76510 +9032,76510 +76511,76511 +76512,76512 +76513,76513 +50925,76513 +42821,76513 +76514,76514 +76515,76515 +76516,76516 +76517,76517 +76518,76518 +76519,76519 +76520,76520 +76521,76521 +76522,76522 +76523,76523 +76524,76524 +76525,76525 +76526,76526 +76527,76527 +76528,76528 +76529,76529 +76530,76530 +76531,76531 +76532,76532 +76533,76533 +76534,76534 +76535,76535 +76536,76536 +76537,76537 +76538,76538 +76539,76539 +76540,76540 +76541,76541 +76542,76542 +76543,76543 +200982,76543 +76544,76544 +76545,76545 +76546,76546 +76547,76547 +190255,76547 +76548,76548 +76549,76549 +76550,76550 +76551,76551 +76552,76552 +76553,76553 +76554,76554 +76555,76555 +76556,76556 +76557,76557 +76558,76558 +76559,76559 +76560,76560 +76561,76561 +76562,76562 +76563,76563 +76564,76564 +136404,76564 +128817,76564 +76565,76565 +76566,76566 +76567,76567 +76568,76568 +76569,76569 +76570,76570 +76571,76571 +76572,76572 +76573,76573 +76574,76574 +76575,76575 +76576,76576 +76577,76577 +76578,76578 +76579,76579 +76580,76580 +76581,76581 +164446,76581 +76582,76582 +76583,76583 +145472,76583 +76584,76584 +76585,76585 +76586,76586 +121695,76586 +153300,76586 +76587,76587 +76588,76588 +76589,76589 +76590,76590 +76591,76591 +76592,76592 +76593,76593 +76594,76594 +76595,76595 +76596,76596 +76597,76597 +76598,76598 +76599,76599 +76600,76600 +76601,76601 +76602,76602 +76603,76603 +76604,76604 +76605,76605 +76606,76606 +76607,76607 +76608,76608 +76609,76609 +76610,76610 +76611,76611 +76612,76612 +76613,76613 +76614,76614 +76615,76615 +76616,76616 +75453,76616 +76617,76617 +76618,76618 +76619,76619 +76620,76620 +76621,76621 +76622,76622 +76623,76623 +76624,76624 +76625,76625 +76626,76626 +76627,76627 +76628,76628 +76629,76629 +76630,76630 +76631,76631 +76632,76632 +76633,76633 +142296,76633 +76634,76634 +76635,76635 +76636,76636 +76637,76637 +76638,76638 +76639,76639 +76640,76640 +76641,76641 +76642,76642 +76643,76643 +76644,76644 +76645,76645 +76646,76646 +76647,76647 +76648,76648 +165085,76648 +76649,76649 +76650,76650 +76651,76651 +76652,76652 +76653,76653 +76654,76654 +76655,76655 +76656,76656 +76657,76657 +189188,76657 +76658,76658 +76659,76659 +76660,76660 +76661,76661 +76662,76662 +76663,76663 +76664,76664 +76794,76794 +84667,76794 +76795,76795 +76796,76796 +76797,76797 +76798,76798 +76799,76799 +76800,76800 +76801,76801 +76802,76802 +76803,76803 +76804,76804 +76805,76805 +76806,76806 +76807,76807 +76808,76808 +76809,76809 +76810,76810 +76811,76811 +76812,76812 +76813,76813 +76814,76814 +76815,76815 +76816,76816 +76817,76817 +76818,76818 +76819,76819 +76820,76820 +76821,76821 +76822,76822 +76823,76823 +76824,76824 +76825,76825 +76826,76826 +76827,76827 +76828,76828 +76829,76829 +76830,76830 +76831,76831 +76832,76832 +76833,76833 +76834,76834 +76835,76835 +76836,76836 +76837,76837 +76838,76838 +76839,76839 +76840,76840 +76841,76841 +76842,76842 +76843,76843 +76844,76844 +76845,76845 +76846,76846 +76847,76847 +76848,76848 +76849,76849 +76850,76850 +76851,76851 +76852,76852 +76853,76853 +76854,76854 +248669,76854 +76855,76855 +76856,76856 +76857,76857 +76858,76858 +76859,76859 +76860,76860 +76861,76861 +76862,76862 +76863,76863 +76864,76864 +76865,76865 +76866,76866 +76867,76867 +76868,76868 +76869,76869 +76870,76870 +76871,76871 +76872,76872 +76873,76873 +76874,76874 +76875,76875 +76876,76876 +75932,76876 +76877,76877 +157095,76877 +76878,76878 +76879,76879 +76880,76880 +76881,76881 +76882,76882 +76883,76883 +76884,76884 +76885,76885 +76886,76886 +76887,76887 +76888,76888 +76889,76889 +76890,76890 +76891,76891 +76892,76892 +76893,76893 +76894,76894 +76895,76895 +76896,76896 +76897,76897 +247025,76897 +76898,76898 +76899,76899 +76900,76900 +76901,76901 +76902,76902 +76903,76903 +76904,76904 +76905,76905 +76906,76906 +76907,76907 +76908,76908 +76909,76909 +76910,76910 +76911,76911 +76912,76912 +76913,76913 +76914,76914 +76915,76915 +222935,76916 +76916,76916 +76917,76917 +76918,76918 +76919,76919 +76920,76920 +76921,76921 +76922,76922 +76923,76923 +76924,76924 +76925,76925 +231211,76925 +76926,76926 +76927,76927 +76928,76928 +76929,76929 +76930,76930 +76931,76931 +67288,76931 +76932,76932 +76933,76933 +76934,76934 +76935,76935 +76936,76936 +76937,76937 +76938,76938 +76939,76939 +76940,76940 +76941,76941 +76942,76942 +76943,76943 +76944,76944 +76945,76945 +76946,76946 +76947,76947 +76948,76948 +76949,76949 +76950,76950 +76951,76951 +76952,76952 +243846,76952 +76953,76953 +76954,76954 +76955,76955 +76956,76956 +76957,76957 +76958,76958 +76959,76959 +76960,76960 +76961,76961 +76962,76962 +233315,76962 +76963,76963 +76964,76964 +76965,76965 +76966,76966 +76967,76967 +76968,76968 +76969,76969 +76970,76970 +76971,76971 +76972,76972 +76973,76973 +76974,76974 +76975,76975 +76976,76976 +76977,76977 +76978,76978 +76979,76979 +76980,76980 +76981,76981 +76982,76982 +76983,76983 +76984,76984 +76985,76985 +76986,76986 +76987,76987 +76988,76988 +76989,76989 +76990,76990 +76991,76991 +76992,76992 +76993,76993 +76994,76994 +76995,76995 +76996,76996 +76997,76997 +76998,76998 +76999,76999 +77000,77000 +77001,77001 +77002,77002 +77003,77003 +77004,77004 +77005,77005 +77006,77006 +77007,77007 +77008,77008 +77009,77009 +77010,77010 +77011,77011 +77012,77012 +77013,77013 +77014,77014 +6733,77014 +77015,77015 +77016,77016 +77017,77017 +77018,77018 +77019,77019 +77020,77020 +77021,77021 +77022,77022 +77023,77023 +77024,77024 +77025,77025 +77026,77026 +77027,77027 +77028,77028 +77029,77029 +77030,77030 +77031,77031 +77032,77032 +77033,77033 +77034,77034 +77035,77035 +77036,77036 +77037,77037 +77038,77038 +77039,77039 +77040,77040 +77041,77041 +77042,77042 +77043,77043 +77044,77044 +77045,77045 +77046,77046 +77047,77047 +77048,77048 +77049,77049 +77050,77050 +77051,77051 +77052,77052 +77053,77053 +77054,77054 +77055,77055 +77056,77056 +77057,77057 +77058,77058 +77059,77059 +233251,77059 +77060,77060 +77061,77061 +77062,77062 +77063,77063 +77064,77064 +77065,77065 +77066,77066 +77067,77067 +77068,77068 +77069,77069 +77070,77070 +77071,77071 +77072,77072 +77073,77073 +77074,77074 +77075,77075 +77076,77076 +77077,77077 +77078,77078 +163704,77078 +77079,77079 +77080,77080 +77081,77081 +77082,77082 +77083,77083 +77084,77084 +77085,77085 +77086,77086 +77087,77087 +77088,77088 +77089,77089 +77090,77090 +77091,77091 +77092,77092 +77093,77093 +77094,77094 +202449,77094 +77095,77095 +77096,77096 +77097,77097 +226314,77097 +77098,77098 +77099,77099 +77100,77100 +77101,77101 +77102,77102 +77103,77103 +164910,77103 +77104,77104 +77105,77105 +77106,77106 +77107,77107 +77108,77108 +77109,77109 +99319,77109 +77110,77110 +77111,77111 +77112,77112 +77113,77113 +77114,77114 +77115,77115 +77116,77116 +77117,77117 +77118,77118 +77119,77119 +63871,77119 +77120,77120 +77121,77121 +77122,77122 +77123,77123 +77124,77124 +77125,77125 +32379,77125 +77126,77126 +77127,77127 +77128,77128 +77129,77129 +77130,77130 +77131,77131 +149086,77131 +77132,77132 +77133,77133 +77134,77134 +77135,77135 +77136,77136 +77137,77137 +178310,77137 +172346,77137 +77138,77138 +77139,77139 +77140,77140 +77141,77141 +77142,77142 +77143,77143 +77144,77144 +77145,77145 +227495,77145 +77146,77146 +77147,77147 +77148,77148 +77149,77149 +77150,77150 +92643,77150 +93831,77150 +77151,77151 +77152,77152 +64012,77152 +77153,77153 +77154,77154 +77155,77155 +77156,77156 +223458,77156 +77157,77157 +77158,77158 +77159,77159 +77160,77160 +77161,77161 +89589,77161 +77162,77162 +77163,77163 +77164,77164 +77165,77165 +77166,77166 +77167,77167 +77168,77168 +249126,77169 +77169,77169 +77170,77170 +77171,77171 +77172,77172 +77173,77173 +77174,77174 +77175,77175 +77176,77176 +77177,77177 +77178,77178 +77179,77179 +77180,77180 +77181,77181 +77182,77182 +77183,77183 +142748,77183 +77184,77184 +77185,77185 +77186,77186 +77187,77187 +77188,77188 +77189,77189 +77190,77190 +77191,77191 +77192,77192 +77193,77193 +77194,77194 +77195,77195 +149294,77195 +77196,77196 +77197,77197 +77198,77198 +77199,77199 +77200,77200 +77201,77201 +77202,77202 +77203,77203 +222800,77203 +77204,77204 +77205,77205 +77206,77206 +77207,77207 +77208,77208 +77209,77209 +77210,77210 +77211,77211 +77212,77212 +77213,77213 +77214,77214 +77215,77215 +77216,77216 +114903,77216 +77217,77217 +77218,77218 +77219,77219 +77220,77220 +77221,77221 +77222,77222 +77223,77223 +77224,77224 +155943,77225 +77225,77225 +77226,77226 +77227,77227 +77228,77228 +77229,77229 +77230,77230 +192242,77230 +77231,77231 +77232,77232 +77233,77233 +77234,77234 +77235,77235 +211607,77235 +77236,77236 +77237,77237 +77238,77238 +77239,77239 +77240,77240 +77241,77241 +77242,77242 +77243,77243 +77244,77244 +77245,77245 +181101,77245 +77246,77246 +77247,77247 +157812,77247 +77248,77248 +77249,77249 +77250,77250 +77251,77251 +77252,77252 +77253,77253 +77254,77254 +77255,77255 +77256,77256 +77257,77257 +77258,77258 +77259,77259 +77260,77260 +77261,77261 +77262,77262 +7480,77262 +33425,77262 +77263,77263 +77264,77264 +77265,77265 +77266,77266 +77267,77267 +77268,77268 +77269,77269 +77270,77270 +77271,77271 +77272,77272 +213565,77272 +77273,77273 +77274,77274 +77275,77275 +77276,77276 +184254,77276 +77277,77277 +77278,77278 +77279,77279 +77280,77280 +77281,77281 +77282,77282 +77283,77283 +77284,77284 +77285,77285 +77286,77286 +77287,77287 +153202,77288 +77288,77288 +77289,77289 +77290,77290 +180623,77290 +77291,77291 +77292,77292 +77293,77293 +77294,77294 +77295,77295 +77296,77296 +77297,77297 +77298,77298 +77299,77299 +77300,77300 +77301,77301 +77302,77302 +77303,77303 +77304,77304 +77305,77305 +77306,77306 +77307,77307 +77308,77308 +77309,77309 +77310,77310 +77311,77311 +77312,77312 +77313,77313 +77314,77314 +77315,77315 +77316,77316 +77317,77317 +77318,77318 +77319,77319 +77320,77320 +77321,77321 +77322,77322 +77323,77323 +77324,77324 +77325,77325 +77326,77326 +77327,77327 +174366,77327 +168402,77327 +77328,77328 +77329,77329 +77330,77330 +227116,77330 +71877,77330 +77331,77331 +77332,77332 +77333,77333 +77334,77334 +77335,77335 +2816,77335 +77336,77336 +77337,77337 +77338,77338 +77339,77339 +77340,77340 +77341,77341 +24688,77341 +77342,77342 +77343,77343 +77344,77344 +77345,77345 +77346,77346 +77347,77347 +77348,77348 +77349,77349 +77350,77350 +77351,77351 +77352,77352 +77353,77353 +77354,77354 +77355,77355 +77356,77356 +77357,77357 +192255,77357 +77358,77358 +77359,77359 +122863,77359 +77360,77360 +77361,77361 +77362,77362 +77363,77363 +77364,77364 +77365,77365 +157045,77365 +77366,77366 +77367,77367 +77368,77368 +77369,77369 +77370,77370 +77371,77371 +77372,77372 +77373,77373 +77374,77374 +77375,77375 +77376,77376 +77377,77377 +77378,77378 +77379,77379 +77380,77380 +77381,77381 +77382,77382 +77383,77383 +77384,77384 +77385,77385 +84276,77385 +77386,77386 +30769,77386 +77387,77387 +77388,77388 +77389,77389 +77390,77390 +77391,77391 +77392,77392 +31274,77392 +77393,77393 +77394,77394 +77395,77395 +77396,77396 +77397,77397 +77398,77398 +77399,77399 +77400,77400 +77401,77401 +216864,77401 +77402,77402 +190916,77402 +77403,77403 +77404,77404 +77405,77405 +232037,77405 +77406,77406 +77407,77407 +77408,77408 +77409,77409 +77410,77410 +77411,77411 +77412,77412 +141443,77412 +77413,77413 +77414,77414 +77415,77415 +77416,77416 +77417,77417 +77418,77418 +77419,77419 +77420,77420 +77421,77421 +77422,77422 +77423,77423 +77424,77424 +77425,77425 +77426,77426 +77427,77427 +77428,77428 +77429,77429 +77430,77430 +77431,77431 +77432,77432 +77433,77433 +77434,77434 +32403,77434 +77435,77435 +77436,77436 +77437,77437 +77438,77438 +77439,77439 +77440,77440 +77441,77441 +77442,77442 +77443,77443 +77444,77444 +77445,77445 +77446,77446 +77447,77447 +77448,77448 +77449,77449 +15363,77449 +77450,77450 +77451,77451 +77452,77452 +72519,77452 +201124,77452 +77453,77453 +77454,77454 +77455,77455 +77456,77456 +77457,77457 +77458,77458 +77459,77459 +77460,77460 +77461,77461 +50127,77461 +41934,77461 +77462,77462 +80343,77462 +77463,77463 +77464,77464 +77539,77464 +77465,77465 +76397,77465 +77466,77466 +77467,77467 +77468,77468 +77469,77469 +77470,77470 +77471,77471 +77472,77472 +77473,77473 +77474,77474 +77475,77475 +77476,77476 +77477,77477 +77478,77478 +77479,77479 +77480,77480 +77481,77481 +77482,77482 +77483,77483 +77484,77484 +77485,77485 +77486,77486 +77487,77487 +77488,77488 +77489,77489 +77490,77490 +77491,77491 +77530,77530 +189456,77530 +77531,77531 +77532,77532 +77533,77533 +77534,77534 +77535,77535 +77536,77536 +77537,77537 +56652,77537 +48548,77537 +77538,77538 +77539,77539 +77464,77539 +77540,77540 +111698,77540 +77541,77541 +77542,77542 +77543,77543 +208038,77543 +77544,77544 +2432,77544 +77545,77545 +77546,77546 +77547,77547 +211192,77547 +77548,77548 +77549,77549 +77550,77550 +77551,77551 +77552,77552 +77553,77553 +77554,77554 +77555,77555 +77556,77556 +77557,77557 +77558,77558 +108792,77558 +77559,77559 +77560,77560 +77561,77561 +77562,77562 +61466,77562 +77563,77563 +77564,77564 +77565,77565 +77566,77566 +77567,77567 +77568,77568 +77569,77569 +77570,77570 +77571,77571 +77572,77572 +77573,77573 +77574,77574 +83724,77574 +77575,77575 +77576,77576 +77577,77577 +77578,77578 +77579,77579 +77580,77580 +77581,77581 +77582,77582 +77583,77583 +135476,77583 +127889,77583 +77584,77584 +59130,77584 +58756,77584 +24023,77584 +77585,77585 +77586,77586 +77587,77587 +77588,77588 +77589,77589 +77590,77590 +77591,77591 +109825,77591 +77592,77592 +77593,77593 +77594,77594 +213693,77594 +77595,77595 +77596,77596 +77597,77597 +77598,77598 +77599,77599 +77600,77600 +77601,77601 +77602,77602 +77603,77603 +77604,77604 +77605,77605 +77606,77606 +77607,77607 +77608,77608 +77609,77609 +77610,77610 +77611,77611 +77612,77612 +77613,77613 +77614,77614 +77615,77615 +77616,77616 +77617,77617 +77618,77618 +77619,77619 +77620,77620 +77621,77621 +77622,77622 +77623,77623 +77624,77624 +77625,77625 +77626,77626 +155428,77627 +77627,77627 +151799,77627 +77628,77628 +77629,77629 +77630,77630 +77631,77631 +77632,77632 +77633,77633 +77634,77634 +77635,77635 +77636,77636 +77637,77637 +77638,77638 +77639,77639 +77640,77640 +77641,77641 +77642,77642 +77643,77643 +77644,77644 +77645,77645 +77646,77646 +77647,77647 +77648,77648 +77649,77649 +77650,77650 +77651,77651 +77652,77652 +77653,77653 +77654,77654 +77655,77655 +77656,77656 +77657,77657 +77658,77658 +77659,77659 +77660,77660 +77661,77661 +77662,77662 +77663,77663 +77664,77664 +77665,77665 +77666,77666 +77667,77667 +77668,77668 +77669,77669 +77670,77670 +77671,77671 +77672,77672 +77673,77673 +77674,77674 +77675,77675 +77676,77676 +77677,77677 +77678,77678 +77679,77679 +77680,77680 +77681,77681 +77682,77682 +24513,77682 +77683,77683 +77684,77684 +152255,77684 +77685,77685 +77686,77686 +77687,77687 +77688,77688 +77689,77689 +77690,77690 +77691,77691 +77692,77692 +61371,77692 +77693,77693 +77694,77694 +77695,77695 +77696,77696 +77697,77697 +39920,77697 +77698,77698 +77699,77699 +77700,77700 +77701,77701 +77702,77702 +77703,77703 +77704,77704 +77705,77705 +77706,77706 +77707,77707 +77708,77708 +77709,77709 +97723,77709 +77710,77710 +77711,77711 +77712,77712 +77713,77713 +114389,77713 +77714,77714 +77715,77715 +77716,77716 +77717,77717 +77718,77718 +77719,77719 +77720,77720 +77721,77721 +77722,77722 +77723,77723 +140128,77723 +77724,77724 +77725,77725 +77726,77726 +77764,77764 +77765,77765 +225829,77765 +77766,77766 +77767,77767 +77768,77768 +77769,77769 +77770,77770 +77771,77771 +77772,77772 +77773,77773 +77774,77774 +119179,77774 +77775,77775 +77776,77776 +77777,77777 +77778,77778 +77779,77779 +77780,77780 +77781,77781 +226670,77781 +77782,77782 +77783,77783 +77784,77784 +77785,77785 +77786,77786 +77787,77787 +77788,77788 +77789,77789 +163473,77789 +77790,77790 +77791,77791 +77792,77792 +77793,77793 +84125,77793 +77794,77794 +77795,77795 +77796,77796 +77797,77797 +77798,77798 +77799,77799 +77800,77800 +77801,77801 +77802,77802 +77803,77803 +77841,77841 +77842,77842 +77843,77843 +77844,77844 +77845,77845 +77846,77846 +77847,77847 +53762,77847 +45658,77847 +260,77847 +77848,77848 +77849,77849 +77850,77850 +77851,77851 +77852,77852 +77853,77853 +77854,77854 +77855,77855 +77856,77856 +77857,77857 +77858,77858 +77961,77961 +77962,77962 +77963,77963 +208286,77963 +77964,77964 +77965,77965 +241385,77965 +77966,77966 +77967,77967 +77968,77968 +78010,77968 +77969,77969 +77970,77970 +77971,77971 +77972,77972 +77973,77973 +77974,77974 +77975,77975 +77976,77976 +157811,77976 +77977,77977 +133750,77977 +126163,77977 +77978,77978 +77979,77979 +78043,77980 +78001,77980 +77980,77980 +77981,77981 +77982,77982 +78065,77982 +77983,77983 +77984,77984 +77985,77985 +77986,77986 +77987,77987 +77988,77988 +77989,77989 +77990,77990 +77991,77991 +118621,77991 +77992,77992 +77993,77993 +77994,77994 +77995,77995 +193150,77996 +77996,77996 +77997,77997 +77998,77998 +77999,77999 +78000,78000 +78043,78001 +78001,78001 +77980,78001 +78002,78002 +78003,78003 +78004,78004 +78005,78005 +78006,78006 +78007,78007 +191839,78008 +78008,78008 +191618,78008 +78080,78008 +78565,78009 +78009,78009 +78010,78010 +77968,78010 +78011,78011 +78567,78011 +78012,78012 +78013,78013 +78014,78014 +78015,78015 +78016,78016 +78017,78017 +88231,78018 +78018,78018 +78019,78019 +78020,78020 +78021,78021 +78022,78022 +78023,78023 +78024,78024 +78025,78025 +149084,78025 +94486,78025 +78026,78026 +78027,78027 +78028,78028 +78029,78029 +78030,78030 +78031,78031 +78032,78032 +78033,78033 +78034,78034 +78035,78035 +78036,78036 +78037,78037 +78038,78038 +78039,78039 +78040,78040 +78041,78041 +78042,78042 +78043,78043 +78001,78043 +77980,78043 +78044,78044 +78045,78045 +78046,78046 +78047,78047 +78048,78048 +196293,78049 +78049,78049 +78050,78050 +78051,78051 +78052,78052 +98098,78052 +78053,78053 +78054,78054 +78055,78055 +78075,78055 +78056,78056 +78057,78057 +78058,78058 +78059,78059 +78060,78060 +62463,78060 +78061,78061 +78062,78062 +53688,78062 +45584,78062 +78063,78063 +78064,78064 +78065,78065 +77982,78065 +78066,78066 +78067,78067 +78068,78068 +26881,78068 +78069,78069 +78070,78070 +78071,78071 +78072,78072 +78073,78073 +116368,78073 +78074,78074 +78075,78075 +78055,78075 +78076,78076 +78077,78077 +78078,78078 +209773,78078 +78079,78079 +78080,78080 +191839,78080 +78008,78080 +78081,78081 +78082,78082 +78083,78083 +78124,78124 +78125,78125 +78369,78369 +78370,78370 +190903,78370 +78371,78371 +78372,78372 +78373,78373 +78374,78374 +78375,78375 +78376,78376 +78377,78377 +78378,78378 +78379,78379 +212258,78379 +78380,78380 +78381,78381 +78382,78382 +78383,78383 +198460,78383 +78384,78384 +78385,78385 +78386,78386 +78387,78387 +78388,78388 +78389,78389 +78390,78390 +78391,78391 +78392,78392 +78393,78393 +78394,78394 +212911,78394 +78395,78395 +78396,78396 +78397,78397 +78623,78398 +78398,78398 +78399,78399 +78400,78400 +78401,78401 +78402,78402 +78440,78440 +78441,78441 +78442,78442 +78443,78443 +78444,78444 +78445,78445 +78446,78446 +78512,78512 +78513,78513 +78514,78514 +78515,78515 +78516,78516 +78517,78517 +78518,78518 +78519,78519 +72499,78519 +78520,78520 +78521,78521 +179363,78521 +78522,78522 +78523,78523 +78524,78524 +78525,78525 +226133,78525 +119222,78525 +78526,78526 +78527,78527 +78565,78565 +78009,78565 +78566,78566 +78567,78567 +78011,78567 +78568,78568 +78569,78569 +78570,78570 +78571,78571 +78572,78572 +78573,78573 +78574,78574 +78575,78575 +212007,78575 +78576,78576 +78577,78577 +78578,78578 +78579,78579 +4483,78579 +78580,78580 +78581,78581 +78582,78582 +78583,78583 +78584,78584 +78585,78585 +78586,78586 +78587,78587 +78588,78588 +78589,78589 +78590,78590 +78591,78591 +78592,78592 +78593,78593 +78594,78594 +78595,78595 +78596,78596 +78597,78597 +78598,78598 +78599,78599 +78600,78600 +78601,78601 +78602,78602 +78603,78603 +78604,78604 +78605,78605 +98080,78605 +78606,78606 +78607,78607 +78608,78608 +78609,78609 +78610,78610 +78611,78611 +78612,78612 +78613,78613 +78614,78614 +78615,78615 +78616,78616 +78617,78617 +203741,78617 +78618,78618 +240269,78618 +78619,78619 +78640,78619 +78620,78620 +78621,78621 +78622,78622 +78623,78623 +78398,78623 +78624,78624 +78625,78625 +66796,78625 +78626,78626 +78627,78627 +78628,78628 +78629,78629 +78630,78630 +131774,78630 +124187,78630 +78631,78631 +78632,78632 +78633,78633 +78634,78634 +78635,78635 +78636,78636 +51354,78636 +43250,78636 +78637,78637 +78638,78638 +78639,78639 +78640,78640 +78619,78640 +78641,78641 +78642,78642 +78643,78643 +78644,78644 +78645,78645 +78646,78646 +83535,78646 +78647,78647 +228718,78647 +78648,78648 +78649,78649 +78650,78650 +78651,78651 +78652,78652 +230870,78652 +85198,78652 +78653,78653 +78654,78654 +78655,78655 +78656,78656 +78657,78657 +78658,78658 +78659,78659 +78660,78660 +78661,78661 +78662,78662 +78663,78663 +78664,78664 +78665,78665 +8689,78665 +8509,78665 +78666,78666 +78667,78667 +78668,78668 +78669,78669 +78670,78670 +184766,78670 +78671,78671 +78672,78672 +78673,78673 +78674,78674 +78675,78675 +78676,78676 +78677,78677 +78678,78678 +78679,78679 +78680,78680 +78681,78681 +150754,78681 +78682,78682 +78683,78683 +207198,78683 +78684,78684 +78685,78685 +78686,78686 +78687,78687 +78688,78688 +92677,78688 +78689,78689 +78690,78690 +78691,78691 +78692,78692 +78693,78693 +78694,78694 +78695,78695 +78696,78696 +78697,78697 +78698,78698 +78699,78699 +78700,78700 +78701,78701 +211669,78701 +78702,78702 +78703,78703 +78704,78704 +78705,78705 +78706,78706 +143331,78706 +78707,78707 +78708,78708 +78709,78709 +78710,78710 +78711,78711 +78712,78712 +78713,78713 +78714,78714 +78715,78715 +78716,78716 +78717,78717 +78718,78718 +78719,78719 +78720,78720 +78721,78721 +78722,78722 +157231,78722 +78723,78723 +78724,78724 +78725,78725 +78726,78726 +78727,78727 +78728,78728 +205954,78729 +78729,78729 +78730,78730 +78731,78731 +205956,78731 +78732,78732 +78733,78733 +78734,78734 +78735,78735 +78736,78736 +78737,78737 +78738,78738 +78739,78739 +78740,78740 +78779,78779 +78780,78780 +9455,78780 +78781,78781 +78782,78782 +78783,78783 +78784,78784 +78785,78785 +78786,78786 +78787,78787 +78788,78788 +78789,78789 +78790,78790 +78791,78791 +78792,78792 +78793,78793 +78794,78794 +78795,78795 +78796,78796 +226525,78796 +78797,78797 +78798,78798 +78799,78799 +78800,78800 +110978,78800 +78801,78801 +78802,78802 +78803,78803 +78804,78804 +78805,78805 +78806,78806 +78807,78807 +78808,78808 +78809,78809 +78810,78810 +78811,78811 +78812,78812 +78813,78813 +78814,78814 +140388,78814 +78815,78815 +78816,78816 +78817,78817 +103545,78817 +78818,78818 +78819,78819 +78820,78820 +78821,78821 +78822,78822 +78823,78823 +78824,78824 +78825,78825 +78826,78826 +78827,78827 +78828,78828 +78829,78829 +78830,78830 +78831,78831 +78832,78832 +78833,78833 +78834,78834 +78835,78835 +78836,78836 +78837,78837 +78838,78838 +78839,78839 +78840,78840 +78841,78841 +78842,78842 +78843,78843 +78844,78844 +78845,78845 +78846,78846 +78847,78847 +78848,78848 +78849,78849 +78850,78850 +78851,78851 +78852,78852 +78853,78853 +78854,78854 +78855,78855 +78856,78856 +78857,78857 +218314,78857 +78858,78858 +78859,78859 +78860,78860 +78861,78861 +78862,78862 +78863,78863 +78864,78864 +78900,78900 +78901,78901 +78902,78902 +78903,78903 +78904,78904 +78905,78905 +78906,78906 +78907,78907 +78908,78908 +78909,78909 +78910,78910 +78911,78911 +78912,78912 +78913,78913 +78914,78914 +78915,78915 +234408,78915 +78916,78916 +29508,78916 +78917,78917 +2752,78917 +78918,78918 +78919,78919 +78920,78920 +78921,78921 +78922,78922 +78923,78923 +78924,78924 +78925,78925 +78926,78926 +78927,78927 +78928,78928 +78929,78929 +78930,78930 +78931,78931 +78932,78932 +78933,78933 +78934,78934 +78935,78935 +78936,78936 +78937,78937 +141329,78937 +78938,78938 +233791,78938 +78939,78939 +78940,78940 +78941,78941 +78942,78942 +78943,78943 +78944,78944 +78945,78945 +78946,78946 +78947,78947 +78948,78948 +78949,78949 +78950,78950 +78951,78951 +78952,78952 +78953,78953 +78954,78954 +78955,78955 +78956,78956 +5295,78956 +78957,78957 +78958,78958 +78959,78959 +78960,78960 +78961,78961 +78962,78962 +78963,78963 +78964,78964 +78965,78965 +78966,78966 +78967,78967 +78968,78968 +78969,78969 +78970,78970 +78971,78971 +78972,78972 +78973,78973 +78974,78974 +78975,78975 +78976,78976 +78977,78977 +78978,78978 +78979,78979 +78980,78980 +78981,78981 +78982,78982 +78983,78983 +78984,78984 +78985,78985 +78986,78986 +78987,78987 +78988,78988 +78989,78989 +78990,78990 +78991,78991 +78992,78992 +78993,78993 +78994,78994 +78995,78995 +78996,78996 +78997,78997 +78998,78998 +78999,78999 +79000,79000 +79001,79001 +79002,79002 +79003,79003 +79004,79004 +79005,79005 +79006,79006 +79007,79007 +79008,79008 +79009,79009 +79010,79010 +79011,79011 +79012,79012 +79013,79013 +79014,79014 +100674,79014 +79015,79015 +79016,79016 +79017,79017 +79018,79018 +79019,79019 +79020,79020 +79021,79021 +79022,79022 +79023,79023 +79024,79024 +79025,79025 +79026,79026 +79027,79027 +79028,79028 +79029,79029 +79030,79030 +79031,79031 +79032,79032 +79033,79033 +79034,79034 +79035,79035 +79036,79036 +79037,79037 +79038,79038 +79039,79039 +79040,79040 +79041,79041 +79042,79042 +79043,79043 +79044,79044 +89165,79044 +79045,79045 +79046,79046 +79047,79047 +79048,79048 +79049,79049 +79050,79050 +79051,79051 +146740,79052 +79052,79052 +79053,79053 +79054,79054 +79055,79055 +79056,79056 +79057,79057 +79058,79058 +79059,79059 +79060,79060 +79061,79061 +204747,79061 +79062,79062 +79063,79063 +79064,79064 +79065,79065 +79066,79066 +79067,79067 +79068,79068 +3129,79068 +79069,79069 +79070,79070 +79071,79071 +207247,79071 +79072,79072 +79073,79073 +79074,79074 +79075,79075 +60743,79075 +79076,79076 +107518,79076 +79077,79077 +79078,79078 +79079,79079 +79080,79080 +79081,79081 +79082,79082 +79083,79083 +79084,79084 +79085,79085 +79086,79086 +79087,79087 +79088,79088 +79089,79089 +79090,79090 +82589,79090 +79091,79091 +79092,79092 +79093,79093 +79094,79094 +79095,79095 +79096,79096 +79097,79097 +79098,79098 +79099,79099 +79100,79100 +79101,79101 +79102,79102 +79103,79103 +79104,79104 +79105,79105 +79106,79106 +222352,79106 +79107,79107 +79108,79108 +79109,79109 +79110,79110 +79111,79111 +79112,79112 +79113,79113 +79114,79114 +79115,79115 +79116,79116 +79117,79117 +79118,79118 +79119,79119 +79120,79120 +79121,79121 +79122,79122 +79123,79123 +79124,79124 +79125,79125 +79126,79126 +79127,79127 +79128,79128 +79129,79129 +79130,79130 +79131,79131 +79132,79132 +79133,79133 +79134,79134 +79135,79135 +79136,79136 +79137,79137 +79138,79138 +79139,79139 +79140,79140 +79141,79141 +79142,79142 +79143,79143 +79144,79144 +79145,79145 +79146,79146 +79147,79147 +79148,79148 +145199,79148 +79149,79149 +53250,79149 +45146,79149 +79150,79150 +79151,79151 +79152,79152 +180152,79152 +79153,79153 +79154,79154 +233128,79154 +79155,79155 +79156,79156 +79157,79157 +79158,79158 +79159,79159 +79160,79160 +79161,79161 +79162,79162 +79163,79163 +79164,79164 +79165,79165 +79166,79166 +79167,79167 +79168,79168 +79169,79169 +79170,79170 +79171,79171 +55975,79171 +47871,79171 +66948,79171 +79172,79172 +79173,79173 +79174,79174 +79175,79175 +79176,79176 +79213,79213 +79214,79214 +79215,79215 +79216,79216 +79217,79217 +202066,79217 +79218,79218 +79219,79219 +145405,79219 +79220,79220 +79221,79221 +79222,79222 +79223,79223 +79224,79224 +79225,79225 +79226,79226 +79227,79227 +79228,79228 +79229,79229 +79230,79230 +79231,79231 +79232,79232 +79233,79233 +79234,79234 +79235,79235 +79236,79236 +79237,79237 +79238,79238 +79239,79239 +79240,79240 +79241,79241 +79242,79242 +210179,79242 +79243,79243 +216813,79243 +79244,79244 +79245,79245 +150677,79245 +79246,79246 +79247,79247 +79248,79248 +79249,79249 +79250,79250 +79251,79251 +79252,79252 +79253,79253 +79254,79254 +79255,79255 +79256,79256 +119273,79256 +79257,79257 +39421,79257 +79258,79258 +79259,79259 +79260,79260 +79261,79261 +79262,79262 +79263,79263 +224939,79263 +79264,79264 +79265,79265 +79266,79266 +79267,79267 +79268,79268 +79269,79269 +79270,79270 +79271,79271 +79272,79272 +79434,79311 +79311,79311 +79312,79312 +79313,79313 +79314,79314 +79315,79315 +79316,79316 +79317,79317 +111020,79317 +79318,79318 +79319,79319 +79320,79320 +108441,79320 +79321,79321 +178941,79321 +79322,79322 +79323,79323 +79324,79324 +79325,79325 +79326,79326 +79327,79327 +79328,79328 +79329,79329 +79330,79330 +79331,79331 +79332,79332 +79333,79333 +79334,79334 +241400,79334 +79335,79335 +79336,79336 +79337,79337 +187375,79337 +79338,79338 +79339,79339 +79340,79340 +79341,79341 +79342,79342 +79343,79343 +79344,79344 +237469,79344 +79345,79345 +79346,79346 +79347,79347 +79348,79348 +79349,79349 +79350,79350 +79351,79351 +79352,79352 +79353,79353 +79354,79354 +79355,79355 +162188,79355 +79356,79356 +176642,79356 +170678,79356 +79357,79357 +79395,79395 +79396,79396 +79397,79397 +79398,79398 +79399,79399 +221203,79399 +79400,79400 +79401,79401 +79402,79402 +151360,79402 +79403,79403 +79404,79404 +79405,79405 +79406,79406 +79407,79407 +79408,79408 +79409,79409 +79410,79410 +79411,79411 +79412,79412 +79413,79413 +79414,79414 +79415,79415 +79416,79416 +79417,79417 +79418,79418 +79419,79419 +79420,79420 +5027,79420 +79421,79421 +79422,79422 +79423,79423 +79424,79424 +79425,79425 +19292,79425 +79426,79426 +79427,79427 +79428,79428 +79429,79429 +79430,79430 +79431,79431 +79432,79432 +2751,79432 +103956,79433 +79433,79433 +79434,79434 +79311,79434 +79435,79435 +79436,79436 +79437,79437 +79438,79438 +79439,79439 +79440,79440 +79441,79441 +79442,79442 +79443,79443 +79444,79444 +79445,79445 +79446,79446 +79447,79447 +79448,79448 +79449,79449 +15820,79449 +79450,79450 +79451,79451 +79452,79452 +79453,79453 +79454,79454 +79455,79455 +79456,79456 +79457,79457 +79458,79458 +79459,79459 +12475,79459 +79460,79460 +79461,79461 +79462,79462 +79463,79463 +79464,79464 +79465,79465 +79466,79466 +79467,79467 +79468,79468 +79469,79469 +79470,79470 +79471,79471 +79472,79472 +225211,79472 +79473,79473 +159368,79473 +79474,79474 +79475,79475 +79476,79476 +178576,79476 +79477,79477 +87139,79477 +79478,79478 +180253,79478 +79479,79479 +79480,79480 +79481,79481 +79482,79482 +79483,79483 +109128,79483 +8355,79483 +79484,79484 +79485,79485 +79486,79486 +79487,79487 +79488,79488 +79489,79489 +234738,79489 +79534,79534 +79535,79535 +79536,79536 +79537,79537 +79538,79538 +79539,79539 +79540,79540 +79541,79541 +79542,79542 +79543,79543 +79544,79544 +79545,79545 +79546,79546 +79547,79547 +79548,79548 +144021,79548 +79549,79549 +79550,79550 +79551,79551 +55405,79551 +47301,79551 +79552,79552 +79553,79553 +79554,79554 +79555,79555 +79556,79556 +79557,79557 +79558,79558 +79559,79559 +79560,79560 +79561,79561 +79562,79562 +79563,79563 +79564,79564 +79565,79565 +79566,79566 +79567,79567 +79568,79568 +79569,79569 +79570,79570 +79571,79571 +79572,79572 +79573,79573 +177529,79573 +171565,79573 +79574,79574 +79575,79575 +79576,79576 +79577,79577 +79578,79578 +79579,79579 +79580,79580 +79581,79581 +79582,79582 +79583,79583 +79584,79584 +79585,79585 +79586,79586 +79587,79587 +79588,79588 +79589,79589 +79590,79590 +79629,79629 +79630,79630 +79631,79631 +79632,79632 +79633,79633 +79634,79634 +79635,79635 +79636,79636 +33774,79636 +79637,79637 +79638,79638 +79639,79639 +79767,79767 +106953,79767 +79768,79768 +79769,79769 +79770,79770 +79771,79771 +79772,79772 +79773,79773 +160120,79773 +79774,79774 +79775,79775 +79776,79776 +79777,79777 +79778,79778 +79779,79779 +79780,79780 +79781,79781 +79782,79782 +79783,79783 +79784,79784 +79785,79785 +79786,79786 +79787,79787 +79788,79788 +79789,79789 +79790,79790 +79791,79791 +79792,79792 +79793,79793 +79794,79794 +79795,79795 +79796,79796 +79797,79797 +79798,79798 +79799,79799 +79800,79800 +79801,79801 +79802,79802 +79803,79803 +79804,79804 +92454,79804 +79805,79805 +211169,79805 +79806,79806 +79807,79807 +79808,79808 +79809,79809 +79810,79810 +79811,79811 +21915,79811 +79812,79812 +79813,79813 +79814,79814 +79815,79815 +79816,79816 +79817,79817 +79818,79818 +79819,79819 +79820,79820 +79821,79821 +79822,79822 +79823,79823 +79824,79824 +79825,79825 +79826,79826 +79827,79827 +79828,79828 +79829,79829 +79830,79830 +79831,79831 +79832,79832 +79833,79833 +79834,79834 +79835,79835 +79836,79836 +79837,79837 +209887,79837 +79838,79838 +79839,79839 +79840,79840 +79841,79841 +79842,79842 +79843,79843 +79844,79844 +79845,79845 +79846,79846 +79847,79847 +79848,79848 +79849,79849 +79850,79850 +79851,79851 +79852,79852 +79853,79853 +79854,79854 +79855,79855 +99322,79855 +79856,79856 +79857,79857 +79858,79858 +107525,79858 +79859,79859 +79860,79860 +150631,79860 +79861,79861 +79862,79862 +79863,79863 +79864,79864 +79865,79865 +216398,79865 +79866,79866 +185353,79867 +79867,79867 +79868,79868 +79869,79869 +79870,79870 +79871,79871 +92446,79871 +79872,79872 +88951,79872 +79873,79873 +79874,79874 +79875,79875 +79876,79876 +79877,79877 +79878,79878 +79879,79879 +76341,79879 +79880,79880 +208055,79880 +81439,79880 +79881,79881 +79882,79882 +79883,79883 +79884,79884 +79885,79885 +162258,79885 +79886,79886 +79887,79887 +196512,79888 +79888,79888 +154849,79888 +59642,79888 +79889,79889 +79890,79890 +79891,79891 +79892,79892 +79893,79893 +79894,79894 +79895,79895 +79896,79896 +79897,79897 +56203,79897 +48099,79897 +79898,79898 +92666,79898 +79899,79899 +79900,79900 +79901,79901 +79902,79902 +79903,79903 +79904,79904 +79905,79905 +79906,79906 +79907,79907 +79908,79908 +79909,79909 +79910,79910 +79911,79911 +79912,79912 +79913,79913 +79914,79914 +196277,79914 +211137,79914 +79915,79915 +79916,79916 +79917,79917 +79918,79918 +79919,79919 +79920,79920 +79921,79921 +79922,79922 +79923,79923 +79924,79924 +79925,79925 +79926,79926 +79927,79927 +79928,79928 +79929,79929 +79930,79930 +79931,79931 +79932,79932 +79933,79933 +79934,79934 +79935,79935 +79936,79936 +79937,79937 +79938,79938 +79939,79939 +79940,79940 +79941,79941 +79942,79942 +79943,79943 +79944,79944 +79945,79945 +79946,79946 +79947,79947 +79948,79948 +79949,79949 +79950,79950 +79951,79951 +8631,79951 +8451,79951 +79952,79952 +79953,79953 +79954,79954 +79955,79955 +110612,79955 +79956,79956 +79957,79957 +79958,79958 +79959,79959 +79960,79960 +79961,79961 +79962,79962 +79963,79963 +79964,79964 +79965,79965 +79966,79966 +79967,79967 +79968,79968 +50724,79968 +42620,79968 +79969,79969 +79970,79970 +79971,79971 +79972,79972 +138733,79972 +79973,79973 +79974,79974 +79975,79975 +79976,79976 +79977,79977 +79978,79978 +79979,79979 +79980,79980 +79981,79981 +79982,79982 +79983,79983 +107515,79983 +79984,79984 +79985,79985 +79986,79986 +52719,79986 +44615,79986 +79987,79987 +79988,79988 +79989,79989 +180105,79989 +79990,79990 +79991,79991 +79992,79992 +79993,79993 +79994,79994 +79995,79995 +79996,79996 +79997,79997 +79998,79998 +79999,79999 +80000,80000 +145490,80000 +80001,80001 +80002,80002 +80003,80003 +80004,80004 +80005,80005 +80006,80006 +80007,80007 +80008,80008 +80009,80009 +80010,80010 +80011,80011 +80012,80012 +80013,80013 +80014,80014 +80015,80015 +80016,80016 +80017,80017 +80018,80018 +80019,80019 +80020,80020 +80021,80021 +105046,80021 +80022,80022 +80023,80023 +80024,80024 +80025,80025 +80026,80026 +80027,80027 +80028,80028 +80029,80029 +80030,80030 +80031,80031 +80032,80032 +80033,80033 +80034,80034 +80035,80035 +80036,80036 +80037,80037 +80038,80038 +80039,80039 +80040,80040 +80041,80041 +80042,80042 +185305,80043 +80043,80043 +80044,80044 +80045,80045 +80046,80046 +160698,80046 +80047,80047 +80048,80048 +80049,80049 +80050,80050 +80051,80051 +80052,80052 +80053,80053 +80054,80054 +80055,80055 +80056,80056 +80057,80057 +80058,80058 +80059,80059 +80060,80060 +80061,80061 +80062,80062 +80063,80063 +80064,80064 +80065,80065 +80066,80066 +80067,80067 +80068,80068 +80069,80069 +80070,80070 +80071,80071 +80072,80072 +80073,80073 +80074,80074 +80075,80075 +6921,80075 +80076,80076 +80077,80077 +80078,80078 +80079,80079 +80080,80080 +80081,80081 +80082,80082 +80083,80083 +80084,80084 +80085,80085 +80086,80086 +80087,80087 +80088,80088 +80089,80089 +80090,80090 +80091,80091 +80092,80092 +80093,80093 +80094,80094 +80095,80095 +163074,80095 +80096,80096 +80097,80097 +80098,80098 +80099,80099 +80100,80100 +80101,80101 +80102,80102 +80103,80103 +80104,80104 +232137,80104 +80105,80105 +80106,80106 +80107,80107 +80108,80108 +80109,80109 +80110,80110 +80111,80111 +80112,80112 +80113,80113 +80114,80114 +80115,80115 +205072,80115 +80116,80116 +80117,80117 +59748,80117 +80118,80118 +80119,80119 +80120,80120 +80121,80121 +80122,80122 +80123,80123 +80124,80124 +80125,80125 +80126,80126 +80127,80127 +80128,80128 +80129,80129 +80130,80130 +154872,80130 +80131,80131 +80132,80132 +80133,80133 +154875,80133 +80134,80134 +80135,80135 +80136,80136 +80137,80137 +146917,80137 +80138,80138 +80139,80139 +80140,80140 +80141,80141 +207638,80141 +80142,80142 +80143,80143 +80144,80144 +80145,80145 +232143,80145 +80146,80146 +80147,80147 +80148,80148 +80149,80149 +80150,80150 +80151,80151 +80152,80152 +80153,80153 +80154,80154 +80155,80155 +80156,80156 +80157,80157 +80158,80158 +80159,80159 +80160,80160 +80161,80161 +247897,80161 +80162,80162 +80163,80163 +80164,80164 +80165,80165 +80166,80166 +80167,80167 +80168,80168 +80169,80169 +80170,80170 +80171,80171 +80172,80172 +80173,80173 +80174,80174 +80175,80175 +80176,80176 +80177,80177 +80178,80178 +80179,80179 +80180,80180 +80181,80181 +80182,80182 +80183,80183 +80184,80184 +80185,80185 +80186,80186 +80187,80187 +80188,80188 +80189,80189 +80190,80190 +80191,80191 +80192,80192 +80193,80193 +80194,80194 +80195,80195 +80196,80196 +80197,80197 +80198,80198 +80199,80199 +3262,80199 +80200,80200 +80201,80201 +80202,80202 +80203,80203 +80204,80204 +80205,80205 +80206,80206 +80207,80207 +80208,80208 +80209,80209 +80210,80210 +80211,80211 +80212,80212 +80213,80213 +80214,80214 +80215,80215 +80216,80216 +203940,80216 +80217,80217 +80218,80218 +80219,80219 +80220,80220 +80221,80221 +80222,80222 +80223,80223 +80224,80224 +80225,80225 +80226,80226 +80227,80227 +80228,80228 +80229,80229 +80230,80230 +80231,80231 +80232,80232 +80233,80233 +80234,80234 +80235,80235 +61201,80235 +80236,80236 +80237,80237 +80238,80238 +80239,80239 +80240,80240 +80241,80241 +80242,80242 +80243,80243 +80244,80244 +80245,80245 +80246,80246 +80247,80247 +80248,80248 +104926,80248 +80249,80249 +80250,80250 +80251,80251 +80252,80252 +80253,80253 +80254,80254 +80255,80255 +80256,80256 +80257,80257 +34604,80257 +80258,80258 +80259,80259 +80260,80260 +80261,80261 +80262,80262 +80263,80263 +80264,80264 +80265,80265 +80266,80266 +80267,80267 +80268,80268 +80269,80269 +80270,80270 +80271,80271 +80272,80272 +80273,80273 +80274,80274 +80275,80275 +80276,80276 +80277,80277 +80278,80278 +80279,80279 +80280,80280 +80281,80281 +80282,80282 +80283,80283 +80284,80284 +205379,80285 +80285,80285 +80286,80286 +80287,80287 +80288,80288 +80289,80289 +80290,80290 +80291,80291 +80292,80292 +80293,80293 +80294,80294 +80295,80295 +80296,80296 +80297,80297 +80298,80298 +80299,80299 +110785,80299 +80300,80300 +80301,80301 +80302,80302 +80303,80303 +80304,80304 +80305,80305 +80306,80306 +80307,80307 +80308,80308 +80309,80309 +80310,80310 +80311,80311 +80312,80312 +80313,80313 +80314,80314 +80315,80315 +80316,80316 +111704,80316 +80317,80317 +80318,80318 +80319,80319 +223023,80319 +41944,80319 +80320,80320 +198743,80320 +80321,80321 +80322,80322 +80323,80323 +80324,80324 +80325,80325 +80326,80326 +80327,80327 +80328,80328 +80329,80329 +80330,80330 +80331,80331 +80332,80332 +201169,80332 +80333,80333 +80334,80334 +80335,80335 +80336,80336 +80337,80337 +80338,80338 +80339,80339 +80340,80340 +80341,80341 +80342,80342 +80343,80343 +77462,80343 +80344,80344 +80345,80345 +80346,80346 +80347,80347 +80348,80348 +80349,80349 +67130,80349 +80350,80350 +80351,80351 +80352,80352 +80353,80353 +203844,80353 +80354,80354 +80355,80355 +80356,80356 +80357,80357 +80358,80358 +80359,80359 +190192,80360 +80360,80360 +80361,80361 +80362,80362 +80363,80363 +80364,80364 +80365,80365 +80366,80366 +80367,80367 +80368,80368 +80369,80369 +80370,80370 +80371,80371 +80372,80372 +80373,80373 +80374,80374 +80375,80375 +80376,80376 +80377,80377 +134086,80377 +126499,80377 +80378,80378 +80379,80379 +80380,80380 +80381,80381 +80382,80382 +80383,80383 +80384,80384 +80385,80385 +80386,80386 +80387,80387 +80388,80388 +80389,80389 +80390,80390 +80391,80391 +80392,80392 +80393,80393 +80394,80394 +80395,80395 +80396,80396 +80397,80397 +80398,80398 +80399,80399 +80400,80400 +80401,80401 +80402,80402 +80403,80403 +80404,80404 +80405,80405 +80406,80406 +80407,80407 +221979,80407 +80408,80408 +80409,80409 +80410,80410 +80411,80411 +205609,80411 +80412,80412 +80413,80413 +80414,80414 +80415,80415 +80416,80416 +80417,80417 +80418,80418 +80419,80419 +80420,80420 +205053,80420 +80421,80421 +80422,80422 +80423,80423 +80424,80424 +80425,80425 +227763,80425 +80426,80426 +80427,80427 +80428,80428 +80429,80429 +226516,80429 +80430,80430 +80431,80431 +80432,80432 +80433,80433 +80434,80434 +80435,80435 +80436,80436 +80437,80437 +80438,80438 +80439,80439 +80440,80440 +80441,80441 +76319,80441 +178521,80441 +172557,80441 +80442,80442 +248903,80442 +92659,80443 +80443,80443 +105678,80443 +80444,80444 +80445,80445 +80446,80446 +80447,80447 +80448,80448 +80449,80449 +80450,80450 +54987,80450 +46883,80450 +80451,80451 +80452,80452 +80453,80453 +80454,80454 +80455,80455 +158058,80455 +80456,80456 +80457,80457 +80458,80458 +80459,80459 +80460,80460 +80461,80461 +80462,80462 +80463,80463 +80464,80464 +80465,80465 +80466,80466 +80467,80467 +80468,80468 +80469,80469 +80470,80470 +80471,80471 +80472,80472 +80473,80473 +80474,80474 +80475,80475 +217027,80475 +33207,80475 +80476,80476 +80477,80477 +80478,80478 +80479,80479 +80480,80480 +80519,80519 +80520,80520 +80521,80521 +80522,80522 +80523,80523 +80524,80524 +80525,80525 +80526,80526 +80527,80527 +80528,80528 +80529,80529 +224579,80529 +80530,80530 +80531,80531 +5625,80531 +80532,80532 +224582,80532 +80533,80533 +80536,80533 +80534,80534 +80535,80535 +206934,80535 +80536,80536 +224583,80536 +80533,80536 +80537,80537 +80538,80538 +80539,80539 +80540,80540 +80541,80541 +80542,80542 +80543,80543 +80544,80544 +80545,80545 +231383,80545 +80546,80546 +221864,80547 +80547,80547 +80548,80548 +80549,80549 +80550,80550 +80551,80551 +80552,80552 +80553,80553 +80554,80554 +100725,80554 +80555,80555 +210084,80556 +80556,80556 +80557,80557 +52971,80557 +44867,80557 +80558,80558 +80559,80559 +80560,80560 +80561,80561 +80562,80562 +80563,80563 +80564,80564 +80565,80565 +80566,80566 +80567,80567 +80568,80568 +80569,80569 +80570,80570 +80571,80571 +80572,80572 +80573,80573 +80574,80574 +80575,80575 +80576,80576 +80577,80577 +80578,80578 +80579,80579 +80580,80580 +80581,80581 +80582,80582 +80583,80583 +80584,80584 +80585,80585 +80586,80586 +80587,80587 +80588,80588 +80589,80589 +80590,80590 +80591,80591 +80592,80592 +80593,80593 +80594,80594 +80595,80595 +80596,80596 +216207,80596 +80597,80597 +18993,80597 +80598,80598 +80599,80599 +80600,80600 +80601,80601 +80602,80602 +80603,80603 +80604,80604 +80605,80605 +80606,80606 +80607,80607 +80608,80608 +80609,80609 +80610,80610 +80611,80611 +80612,80612 +80613,80613 +80614,80614 +80615,80615 +113328,80615 +80616,80616 +80617,80617 +80618,80618 +80619,80619 +80620,80620 +80621,80621 +80622,80622 +80623,80623 +80624,80624 +80625,80625 +80626,80626 +80627,80627 +80628,80628 +80629,80629 +80630,80630 +80631,80631 +80632,80632 +10520,80632 +80633,80633 +80634,80634 +80635,80635 +98057,80635 +80636,80636 +80637,80637 +80638,80638 +80639,80639 +80640,80640 +80641,80641 +80642,80642 +80643,80643 +80644,80644 +80645,80645 +231769,80645 +80646,80646 +80647,80647 +80648,80648 +80649,80649 +163717,80649 +80650,80650 +80651,80651 +136378,80651 +128791,80651 +80652,80652 +80653,80653 +80654,80654 +80655,80655 +80656,80656 +80657,80657 +80658,80658 +80659,80659 +80660,80660 +80661,80661 +6730,80661 +80662,80662 +80663,80663 +80664,80664 +80665,80665 +80666,80666 +80667,80667 +80668,80668 +80669,80669 +80670,80670 +235679,80670 +80671,80671 +80672,80672 +80673,80673 +80674,80674 +80675,80675 +80676,80676 +6716,80676 +80677,80677 +80678,80678 +80679,80679 +80680,80680 +80681,80681 +80682,80682 +80683,80683 +80684,80684 +80685,80685 +80686,80686 +80687,80687 +80688,80688 +58450,80688 +80689,80689 +80690,80690 +80691,80691 +80692,80692 +80693,80693 +80694,80694 +80695,80695 +80696,80696 +80697,80697 +80698,80698 +80699,80699 +80700,80700 +203942,80700 +80701,80701 +80702,80702 +80703,80703 +80704,80704 +80705,80705 +80706,80706 +80707,80707 +80708,80708 +58063,80708 +80709,80709 +191994,80709 +80710,80710 +80711,80711 +80712,80712 +80713,80713 +80714,80714 +80715,80715 +80716,80716 +80717,80717 +80718,80718 +80719,80719 +80720,80720 +80721,80721 +80722,80722 +80723,80723 +80724,80724 +181127,80724 +80725,80725 +80726,80726 +80727,80727 +80728,80728 +80729,80729 +80730,80730 +80731,80731 +80732,80732 +80733,80733 +80734,80734 +208749,80734 +80735,80735 +80736,80736 +80737,80737 +80738,80738 +80739,80739 +140257,80739 +80740,80740 +80741,80741 +80742,80742 +80743,80743 +80744,80744 +80745,80745 +80746,80746 +80747,80747 +80748,80748 +240430,80748 +80749,80749 +80750,80750 +80751,80751 +80752,80752 +80753,80753 +80754,80754 +80755,80755 +80756,80756 +80757,80757 +80758,80758 +80759,80759 +80760,80760 +80761,80761 +80762,80762 +80763,80763 +80764,80764 +80765,80765 +80766,80766 +80767,80767 +80768,80768 +80769,80769 +80770,80770 +80771,80771 +80772,80772 +80773,80773 +80774,80774 +80775,80775 +80776,80776 +80777,80777 +80778,80778 +80779,80779 +80780,80780 +80781,80781 +80782,80782 +80783,80783 +80784,80784 +80785,80785 +80786,80786 +80787,80787 +80788,80788 +80789,80789 +80790,80790 +80791,80791 +80792,80792 +80793,80793 +80794,80794 +80795,80795 +80796,80796 +80797,80797 +80798,80798 +80799,80799 +80800,80800 +80801,80801 +80802,80802 +80803,80803 +80804,80804 +135953,80804 +128366,80804 +80805,80805 +80806,80806 +80807,80807 +80808,80808 +80809,80809 +80810,80810 +80811,80811 +80812,80812 +80813,80813 +80814,80814 +80815,80815 +80816,80816 +80817,80817 +80818,80818 +80819,80819 +80820,80820 +80821,80821 +80822,80822 +80823,80823 +80824,80824 +80825,80825 +80826,80826 +80827,80827 +80828,80828 +80829,80829 +80830,80830 +80831,80831 +80832,80832 +80833,80833 +80834,80834 +80835,80835 +80836,80836 +80837,80837 +80838,80838 +80839,80839 +80840,80840 +80841,80841 +80842,80842 +80843,80843 +100582,80843 +80844,80844 +80845,80845 +226718,80845 +80846,80846 +80847,80847 +80848,80848 +80849,80849 +80850,80850 +80851,80851 +80852,80852 +80853,80853 +178624,80853 +80854,80854 +80855,80855 +80856,80856 +80857,80857 +80858,80858 +80859,80859 +80860,80860 +80861,80861 +80862,80862 +80863,80863 +80864,80864 +80865,80865 +80866,80866 +80867,80867 +80868,80868 +80869,80869 +80870,80870 +80871,80871 +80872,80872 +80873,80873 +80874,80874 +80875,80875 +80876,80876 +80877,80877 +164163,80877 +80878,80878 +80879,80879 +80880,80880 +80881,80881 +80882,80882 +80883,80883 +24024,80883 +80884,80884 +80885,80885 +80886,80886 +80887,80887 +80888,80888 +80889,80889 +80890,80890 +80891,80891 +80892,80892 +80893,80893 +80894,80894 +80895,80895 +80896,80896 +80897,80897 +80898,80898 +80899,80899 +97708,80899 +80900,80900 +80901,80901 +80902,80902 +80903,80903 +80904,80904 +80905,80905 +246977,80905 +80906,80906 +80907,80907 +80908,80908 +80909,80909 +80910,80910 +80911,80911 +80912,80912 +80913,80913 +80914,80914 +80915,80915 +80916,80916 +80917,80917 +80918,80918 +80919,80919 +80920,80920 +80921,80921 +80922,80922 +80923,80923 +36553,80923 +80924,80924 +80925,80925 +81055,81055 +81056,81056 +81057,81057 +81058,81058 +33855,81058 +81059,81059 +81060,81060 +81061,81061 +81062,81062 +81063,81063 +81064,81064 +81065,81065 +81123,81123 +97913,81123 +81124,81124 +81125,81125 +81126,81126 +81127,81127 +81128,81128 +81129,81129 +81130,81130 +81131,81131 +81132,81132 +147595,81132 +81133,81133 +81134,81134 +81135,81135 +81136,81136 +81137,81137 +81187,81187 +81188,81188 +81189,81189 +222856,81189 +81190,81190 +81191,81191 +81192,81192 +81193,81193 +149901,81193 +81194,81194 +81195,81195 +238931,81195 +81196,81196 +81197,81197 +81198,81198 +81199,81199 +81200,81200 +81201,81201 +81202,81202 +81203,81203 +81204,81204 +81205,81205 +81206,81206 +81207,81207 +81208,81208 +81209,81209 +81210,81210 +81211,81211 +81310,81310 +81311,81311 +209623,81311 +81312,81312 +81313,81313 +81314,81314 +81315,81315 +81316,81316 +81317,81317 +81318,81318 +81319,81319 +141796,81319 +81320,81320 +81321,81321 +247894,81321 +81322,81322 +81323,81323 +218299,81323 +81324,81324 +81325,81325 +81326,81326 +81327,81327 +81328,81328 +81329,81329 +156310,81329 +81330,81330 +81331,81331 +81332,81332 +81333,81333 +81334,81334 +81335,81335 +116577,81336 +81336,81336 +81337,81337 +81338,81338 +207414,81338 +81339,81339 +81340,81340 +81341,81341 +81342,81342 +81343,81343 +81344,81344 +81345,81345 +81346,81346 +81347,81347 +81348,81348 +81349,81349 +81350,81350 +81351,81351 +111876,81351 +81352,81352 +81353,81353 +81354,81354 +81355,81355 +184476,81355 +81356,81356 +81357,81357 +81358,81358 +81359,81359 +81360,81360 +81361,81361 +81362,81362 +81363,81363 +81364,81364 +81365,81365 +81366,81366 +21349,81366 +81367,81367 +81368,81368 +81369,81369 +81370,81370 +81371,81371 +81372,81372 +81373,81373 +81374,81374 +81375,81375 +81376,81376 +81377,81377 +81378,81378 +81379,81379 +81380,81380 +81381,81381 +81382,81382 +81383,81383 +81384,81384 +111153,81384 +81385,81385 +192627,81385 +81386,81386 +81387,81387 +81388,81388 +81389,81389 +81390,81390 +81391,81391 +81392,81392 +81393,81393 +81432,81393 +81394,81394 +81395,81395 +206740,81396 +81396,81396 +81397,81397 +81398,81398 +81399,81399 +81400,81400 +81401,81401 +81402,81402 +81403,81403 +81404,81404 +81405,81405 +81406,81406 +81407,81407 +211484,81407 +81408,81408 +81409,81409 +81410,81410 +81411,81411 +81412,81412 +81413,81413 +81414,81414 +81415,81415 +81416,81416 +81417,81417 +81418,81418 +81419,81419 +81420,81420 +81421,81421 +81422,81422 +81423,81423 +81424,81424 +81425,81425 +81426,81426 +81427,81427 +81428,81428 +81429,81429 +81430,81430 +81431,81431 +81432,81432 +81393,81432 +74956,81432 +81433,81433 +81434,81434 +81435,81435 +81436,81436 +81437,81437 +81438,81438 +81439,81439 +79880,81439 +81440,81440 +81441,81441 +81442,81442 +81443,81443 +81444,81444 +88755,81444 +81445,81445 +242884,81445 +81446,81446 +3588,81446 +81447,81447 +180418,81447 +81448,81448 +81449,81449 +84653,81449 +81450,81450 +177423,81451 +171459,81451 +81451,81451 +81452,81452 +118448,81452 +81453,81453 +81454,81454 +81455,81455 +81456,81456 +81457,81457 +81458,81458 +81459,81459 +81460,81460 +81461,81461 +81462,81462 +233835,81462 +81463,81463 +81464,81464 +81465,81465 +81466,81466 +81467,81467 +81468,81468 +81469,81469 +81470,81470 +81471,81471 +81472,81472 +81473,81473 +81474,81474 +81475,81475 +156444,81475 +214451,81475 +81476,81476 +81477,81477 +81478,81478 +81479,81479 +146562,81479 +81480,81480 +81481,81481 +81482,81482 +81483,81483 +81484,81484 +81485,81485 +81486,81486 +81487,81487 +81488,81488 +81489,81489 +192194,81489 +224741,81489 +81490,81490 +81491,81491 +81492,81492 +81493,81493 +81494,81494 +81495,81495 +81496,81496 +81497,81497 +81498,81498 +81499,81499 +81500,81500 +81501,81501 +226637,81501 +81502,81502 +81503,81503 +81504,81504 +81505,81505 +81506,81506 +81507,81507 +81508,81508 +81509,81509 +81510,81510 +81511,81511 +81512,81512 +106886,81512 +81513,81513 +81514,81514 +81515,81515 +81516,81516 +81517,81517 +81518,81518 +81519,81519 +81520,81520 +81521,81521 +81522,81522 +81523,81523 +81524,81524 +81525,81525 +81526,81526 +81527,81527 +81528,81528 +81529,81529 +81530,81530 +81531,81531 +81532,81532 +81533,81533 +81534,81534 +185734,81534 +81535,81535 +81536,81536 +81537,81537 +81538,81538 +81539,81539 +81540,81540 +81541,81541 +52095,81541 +43991,81541 +81542,81542 +81543,81543 +203876,81543 +81544,81544 +81545,81545 +81546,81546 +81547,81547 +81548,81548 +81549,81549 +115786,81549 +81550,81550 +81551,81551 +81552,81552 +81553,81553 +81554,81554 +111371,81554 +81555,81555 +81556,81556 +81557,81557 +81558,81558 +81559,81559 +81560,81560 +81561,81561 +81562,81562 +81563,81563 +81564,81564 +81565,81565 +81566,81566 +81567,81567 +81568,81568 +81569,81569 +81570,81570 +81571,81571 +81572,81572 +81573,81573 +81574,81574 +121563,81574 +81575,81575 +81576,81576 +81577,81577 +81578,81578 +81579,81579 +81580,81580 +81581,81581 +81582,81582 +81583,81583 +81584,81584 +81585,81585 +200493,81585 +81586,81586 +81587,81587 +81588,81588 +193787,81588 +81589,81589 +81590,81590 +81591,81591 +50917,81591 +42813,81591 +81592,81592 +81593,81593 +81594,81594 +81595,81595 +81596,81596 +81597,81597 +81598,81598 +81599,81599 +81600,81600 +196755,81600 +81601,81601 +81602,81602 +187241,81602 +81603,81603 +81604,81604 +81605,81605 +81606,81606 +81607,81607 +81608,81608 +81609,81609 +81610,81610 +81611,81611 +81612,81612 +81613,81613 +81614,81614 +81615,81615 +81616,81616 +110357,81616 +81617,81617 +81618,81618 +81619,81619 +81620,81620 +81621,81621 +81622,81622 +185165,81622 +158010,81623 +81623,81623 +81624,81624 +81625,81625 +81626,81626 +81627,81627 +81628,81628 +81629,81629 +81630,81630 +81631,81631 +81632,81632 +81633,81633 +81634,81634 +81635,81635 +81636,81636 +81637,81637 +70238,81637 +146715,81637 +81638,81638 +81639,81639 +81640,81640 +81641,81641 +81642,81642 +81643,81643 +81644,81644 +81645,81645 +81682,81682 +81683,81683 +81684,81684 +81685,81685 +81686,81686 +233650,81686 +81758,81758 +81759,81759 +81760,81760 +81761,81761 +81762,81762 +81763,81763 +81764,81764 +81765,81765 +81766,81766 +81767,81767 +81768,81768 +81769,81769 +81770,81770 +81771,81771 +141527,81772 +81772,81772 +81773,81773 +81774,81774 +81775,81775 +81776,81776 +81777,81777 +81778,81778 +81779,81779 +81780,81780 +81781,81781 +81782,81782 +81783,81783 +81784,81784 +81785,81785 +81786,81786 +103394,81786 +81787,81787 +81788,81788 +81830,81830 +81831,81831 +81832,81832 +81833,81833 +81834,81834 +207458,81834 +81835,81835 +81836,81836 +81837,81837 +81838,81838 +81839,81839 +81840,81840 +81841,81841 +81842,81842 +81843,81843 +81844,81844 +81845,81845 +81846,81846 +81847,81847 +81848,81848 +81849,81849 +81850,81850 +81851,81851 +81852,81852 +81853,81853 +81854,81854 +81855,81855 +187015,81855 +81856,81856 +141476,81856 +103318,81856 +81857,81857 +81858,81858 +81859,81859 +81860,81860 +224784,81860 +81861,81861 +81862,81862 +81863,81863 +81864,81864 +81865,81865 +81866,81866 +81867,81867 +81868,81868 +81869,81869 +81870,81870 +227714,81870 +81871,81871 +81872,81872 +12158,81872 +81873,81873 +81874,81874 +81968,81968 +82012,82012 +82013,82013 +82014,82014 +82015,82015 +82016,82016 +82017,82017 +178727,82018 +82018,82018 +82019,82019 +108671,82019 +82020,82020 +82021,82021 +82022,82022 +82023,82023 +139937,82023 +82024,82024 +82025,82025 +82026,82026 +82027,82027 +82028,82028 +82029,82029 +82030,82030 +82031,82031 +82032,82032 +82033,82033 +82034,82034 +82035,82035 +82036,82036 +82037,82037 +82038,82038 +82039,82039 +82040,82040 +82041,82041 +82042,82042 +82043,82043 +82044,82044 +82045,82045 +82046,82046 +82047,82047 +82048,82048 +82049,82049 +82050,82050 +82051,82051 +82174,82174 +82175,82175 +136250,82175 +128663,82175 +82176,82176 +217580,82176 +82177,82177 +82178,82178 +82179,82179 +82180,82180 +82181,82181 +82182,82182 +82183,82183 +36485,82183 +82184,82184 +49342,82184 +41149,82184 +82185,82185 +82186,82186 +82187,82187 +82188,82188 +82189,82189 +82190,82190 +82191,82191 +82192,82192 +82193,82193 +82194,82194 +82277,82277 +82278,82278 +82279,82279 +82319,82319 +82320,82320 +82321,82321 +82322,82322 +82323,82323 +82324,82324 +82325,82325 +82326,82326 +82327,82327 +82328,82328 +82329,82329 +82330,82330 +82331,82331 +122932,82331 +82332,82332 +82333,82333 +136993,82333 +129406,82333 +82334,82334 +162871,82334 +82335,82335 +82336,82336 +82337,82337 +59714,82337 +82338,82338 +82339,82339 +7112,82339 +82340,82340 +82341,82341 +82342,82342 +82343,82343 +82344,82344 +82345,82345 +7287,82345 +82346,82346 +82347,82347 +82348,82348 +82349,82349 +82350,82350 +82351,82351 +82352,82352 +109525,82352 +82353,82353 +82354,82354 +187317,82354 +82355,82355 +82356,82356 +82357,82357 +153198,82357 +82358,82358 +82423,82423 +82424,82424 +82425,82425 +82426,82426 +82427,82427 +82428,82428 +82429,82429 +82430,82430 +82431,82431 +82432,82432 +82433,82433 +82434,82434 +82435,82435 +82436,82436 +82437,82437 +200753,82437 +82536,82536 +82537,82537 +82538,82538 +7044,82538 +82539,82539 +82540,82540 +82541,82541 +82542,82542 +82543,82543 +82544,82544 +82545,82545 +82546,82546 +82547,82547 +82548,82548 +111913,82548 +82549,82549 +82550,82550 +82551,82551 +82552,82552 +82553,82553 +82554,82554 +82555,82555 +82556,82556 +82557,82557 +82558,82558 +82559,82559 +82560,82560 +82561,82561 +157788,82561 +82562,82562 +82563,82563 +82564,82564 +82565,82565 +82566,82566 +82567,82567 +82568,82568 +179187,82568 +82569,82569 +82570,82570 +82571,82571 +123952,82571 +82572,82572 +82573,82573 +82574,82574 +82575,82575 +82576,82576 +139158,82576 +82577,82577 +249393,82577 +82578,82578 +161238,82578 +82579,82579 +82580,82580 +143050,82580 +82581,82581 +82582,82582 +82583,82583 +82584,82584 +82585,82585 +82586,82586 +82587,82587 +82588,82588 +82589,82589 +79090,82589 +82590,82590 +82591,82591 +104979,82591 +82592,82592 +82593,82593 +82594,82594 +191253,82594 +82595,82595 +82596,82596 +82597,82597 +82598,82598 +82599,82599 +82600,82600 +82601,82601 +82602,82602 +82603,82603 +82604,82604 +82605,82605 +82606,82606 +82607,82607 +82608,82608 +82609,82609 +82610,82610 +82611,82611 +82612,82612 +82613,82613 +82614,82614 +82615,82615 +82616,82616 +82617,82617 +82618,82618 +82619,82619 +82620,82620 +82621,82621 +82622,82622 +82623,82623 +63834,82623 +82624,82624 +82625,82625 +82626,82626 +82627,82627 +82628,82628 +82629,82629 +82630,82630 +82631,82631 +82632,82632 +82633,82633 +82634,82634 +82635,82635 +82636,82636 +82637,82637 +82733,82733 +82734,82734 +82735,82735 +82736,82736 +82737,82737 +82738,82738 +82739,82739 +82740,82740 +82741,82741 +82742,82742 +82743,82743 +158693,82743 +82744,82744 +82745,82745 +232108,82745 +82746,82746 +82747,82747 +82748,82748 +82749,82749 +82750,82750 +82751,82751 +82752,82752 +82753,82753 +84840,82753 +82754,82754 +82755,82755 +82756,82756 +82757,82757 +82758,82758 +82759,82759 +82760,82760 +82761,82761 +82762,82762 +59704,82762 +82763,82763 +82764,82764 +82765,82765 +82766,82766 +82767,82767 +82768,82768 +82769,82769 +82770,82770 +82771,82771 +82772,82772 +82773,82773 +65067,82773 +82774,82774 +31227,82774 +82775,82775 +82848,82848 +82849,82849 +189032,82849 +82850,82850 +82851,82851 +82852,82852 +82853,82853 +82854,82854 +82855,82855 +82856,82856 +82857,82857 +82858,82858 +82859,82859 +82860,82860 +82861,82861 +82862,82862 +82863,82863 +82864,82864 +82865,82865 +82866,82866 +82867,82867 +82868,82868 +82869,82869 +82870,82870 +82871,82871 +82872,82872 +82873,82873 +82874,82874 +82875,82875 +35633,82875 +82876,82876 +82877,82877 +82878,82878 +82879,82879 +82880,82880 +82881,82881 +209344,82882 +82882,82882 +82944,82944 +82945,82945 +82946,82946 +82947,82947 +5791,82947 +82948,82948 +82949,82949 +82950,82950 +82951,82951 +82952,82952 +176469,82952 +170505,82952 +82953,82953 +82954,82954 +82955,82955 +82956,82956 +82957,82957 +82958,82958 +82959,82959 +82960,82960 +82961,82961 +82962,82962 +82963,82963 +82964,82964 +82965,82965 +83008,83008 +83009,83009 +83010,83010 +139904,83010 +83011,83011 +83012,83012 +189039,83012 +83013,83013 +228017,83013 +83014,83014 +83046,83014 +83015,83015 +83016,83016 +93850,83016 +83017,83017 +83018,83018 +83019,83019 +209748,83019 +83136,83019 +83020,83020 +83021,83021 +83022,83022 +173925,83022 +167961,83022 +83023,83023 +83024,83024 +83025,83025 +83026,83026 +83027,83027 +83028,83028 +83029,83029 +83030,83030 +83031,83031 +2702,83031 +83032,83032 +83033,83033 +83034,83034 +83035,83035 +83036,83036 +83037,83037 +149796,83037 +83038,83038 +91478,83038 +83039,83039 +83040,83040 +83041,83041 +83042,83042 +109264,83042 +83043,83043 +83044,83044 +132373,83044 +124786,83044 +83045,83045 +83046,83046 +83014,83046 +83047,83047 +236151,83047 +83048,83048 +114693,83048 +83049,83049 +83050,83050 +83051,83051 +83052,83052 +83053,83053 +83054,83054 +141402,83054 +83055,83055 +83056,83056 +83057,83057 +83058,83058 +83059,83059 +83060,83060 +83061,83061 +109558,83061 +83062,83062 +234593,83062 +83063,83063 +83064,83064 +108377,83064 +102651,83064 +83065,83065 +83066,83066 +83067,83067 +83068,83068 +83069,83069 +83070,83070 +83071,83071 +83072,83072 +83073,83073 +83074,83074 +83075,83075 +83076,83076 +83123,83123 +83124,83124 +83125,83125 +83126,83126 +83127,83127 +83128,83128 +83129,83129 +83130,83130 +83131,83131 +83132,83132 +83133,83133 +83134,83134 +83135,83135 +83136,83136 +83019,83136 +83137,83137 +83138,83138 +83139,83139 +83140,83140 +83141,83141 +83142,83142 +83143,83143 +83144,83144 +83145,83145 +83146,83146 +83147,83147 +83148,83148 +83149,83149 +83150,83150 +83151,83151 +83152,83152 +83153,83153 +83209,83209 +83210,83210 +83211,83211 +83212,83212 +83213,83213 +83214,83214 +83215,83215 +248750,83215 +83216,83216 +120063,83216 +83217,83217 +83218,83218 +83219,83219 +83220,83220 +83221,83221 +83222,83222 +83223,83223 +83224,83224 +83225,83225 +83226,83226 +83227,83227 +83228,83228 +83229,83229 +34029,83229 +83230,83230 +234825,83230 +83231,83231 +83232,83232 +83233,83233 +83234,83234 +9070,83234 +83235,83235 +83236,83236 +72600,83236 +83237,83237 +83238,83238 +83239,83239 +83240,83240 +83241,83241 +83242,83242 +83243,83243 +83244,83244 +83245,83245 +83246,83246 +83247,83247 +83248,83248 +83249,83249 +83250,83250 +83251,83251 +83252,83252 +83253,83253 +83254,83254 +83255,83255 +4669,83255 +83256,83256 +83257,83257 +83258,83258 +83259,83259 +83260,83260 +83261,83261 +83262,83262 +83263,83263 +83264,83264 +37042,83264 +83265,83265 +83266,83266 +83267,83267 +83268,83268 +83269,83269 +83270,83270 +83271,83271 +83272,83272 +83273,83273 +83274,83274 +83275,83275 +83276,83276 +83277,83277 +83278,83278 +83279,83279 +83280,83280 +83281,83281 +83282,83282 +83283,83283 +83284,83284 +83285,83285 +83286,83286 +83287,83287 +83288,83288 +83289,83289 +83290,83290 +83291,83291 +83292,83292 +83293,83293 +83294,83294 +83295,83295 +83296,83296 +83297,83297 +83298,83298 +83299,83299 +83300,83300 +83301,83301 +83302,83302 +83303,83303 +83304,83304 +83305,83305 +83306,83306 +83307,83307 +83308,83308 +83309,83309 +83310,83310 +83311,83311 +83312,83312 +83313,83313 +83314,83314 +83315,83315 +83316,83316 +83317,83317 +83318,83318 +83319,83319 +83320,83320 +83321,83321 +83322,83322 +83323,83323 +83324,83324 +83325,83325 +83326,83326 +83327,83327 +83328,83328 +83329,83329 +83330,83330 +83331,83331 +35632,83331 +158606,83331 +2304,83331 +64106,83331 +83370,83370 +83371,83371 +83372,83372 +83373,83373 +83374,83374 +83375,83375 +83376,83376 +83377,83377 +83378,83378 +83379,83379 +83380,83380 +83381,83381 +83382,83382 +83383,83383 +83384,83384 +83385,83385 +83386,83386 +83387,83387 +83388,83388 +83389,83389 +83390,83390 +83391,83391 +83392,83392 +83393,83393 +83394,83394 +154022,83395 +83395,83395 +83396,83396 +83397,83397 +20902,83397 +83398,83398 +83399,83399 +83400,83400 +83401,83401 +83402,83402 +83403,83403 +215418,83403 +83404,83404 +83405,83405 +83406,83406 +83407,83407 +83408,83408 +83409,83409 +83410,83410 +83411,83411 +83412,83412 +83413,83413 +83414,83414 +83415,83415 +83416,83416 +83417,83417 +83418,83418 +83419,83419 +83420,83420 +83421,83421 +83422,83422 +83423,83423 +83424,83424 +83425,83425 +83426,83426 +83427,83427 +83428,83428 +83429,83429 +83430,83430 +83431,83431 +83432,83432 +83433,83433 +83434,83434 +83435,83435 +83436,83436 +83437,83437 +83438,83438 +83439,83439 +83440,83440 +83441,83441 +83442,83442 +83482,83482 +190252,83482 +83483,83483 +83484,83484 +83485,83485 +83486,83486 +83487,83487 +83488,83488 +83489,83489 +83490,83490 +83491,83491 +83492,83492 +83493,83493 +83494,83494 +52695,83494 +44591,83494 +83495,83495 +83496,83496 +83497,83497 +22627,83497 +83498,83498 +83499,83499 +83500,83500 +83501,83501 +83502,83502 +83503,83503 +83504,83504 +83505,83505 +83506,83506 +83507,83507 +83508,83508 +83509,83509 +83510,83510 +83511,83511 +164955,83511 +83512,83512 +83513,83513 +83514,83514 +83515,83515 +83516,83516 +83517,83517 +83518,83518 +83519,83519 +83520,83520 +83521,83521 +83522,83522 +83523,83523 +208002,83524 +83524,83524 +83525,83525 +83526,83526 +206733,83526 +83527,83527 +83528,83528 +83529,83529 +83530,83530 +83531,83531 +83532,83532 +83533,83533 +83534,83534 +83535,83535 +78646,83535 +83536,83536 +83537,83537 +83538,83538 +83539,83539 +83540,83540 +83541,83541 +83542,83542 +83543,83543 +83544,83544 +83545,83545 +83546,83546 +83547,83547 +57608,83547 +83548,83548 +83549,83549 +83550,83550 +83551,83551 +83552,83552 +83553,83553 +83554,83554 +83555,83555 +83556,83556 +83557,83557 +194788,83557 +83605,83605 +83606,83606 +83607,83607 +83608,83608 +83609,83609 +83610,83610 +83611,83611 +83612,83612 +192423,83612 +83613,83613 +83614,83614 +83615,83615 +83616,83616 +83617,83617 +83618,83618 +83619,83619 +194909,83619 +83620,83620 +83621,83621 +83622,83622 +83623,83623 +83624,83624 +83625,83625 +83626,83626 +83627,83627 +59526,83627 +83628,83628 +83629,83629 +83630,83630 +83631,83631 +83632,83632 +83633,83633 +83634,83634 +83635,83635 +83636,83636 +83637,83637 +146983,83638 +83638,83638 +83639,83639 +83640,83640 +83641,83641 +83642,83642 +83643,83643 +210349,83643 +83644,83644 +159368,83644 +83645,83645 +83646,83646 +83647,83647 +83648,83648 +83649,83649 +83650,83650 +83651,83651 +83652,83652 +83653,83653 +83654,83654 +83655,83655 +83656,83656 +83657,83657 +83658,83658 +83659,83659 +83660,83660 +83661,83661 +178206,83661 +172242,83661 +83662,83662 +83663,83663 +83664,83664 +83665,83665 +83666,83666 +83667,83667 +83668,83668 +83669,83669 +83670,83670 +83671,83671 +198454,83671 +83672,83672 +20917,83672 +83673,83673 +83674,83674 +83675,83675 +83676,83676 +83677,83677 +83678,83678 +83679,83679 +83680,83680 +139836,83680 +83681,83681 +83682,83682 +83683,83683 +83684,83684 +83685,83685 +83686,83686 +83687,83687 +83688,83688 +83689,83689 +83690,83690 +83691,83691 +83692,83692 +83693,83693 +83694,83694 +83695,83695 +83696,83696 +83697,83697 +83698,83698 +83699,83699 +83700,83700 +83701,83701 +83702,83702 +20397,83702 +83703,83703 +83704,83704 +83705,83705 +83706,83706 +83707,83707 +83708,83708 +83709,83709 +83710,83710 +83711,83711 +83712,83712 +83713,83713 +83714,83714 +83715,83715 +83716,83716 +83717,83717 +83718,83718 +83719,83719 +83720,83720 +83721,83721 +83722,83722 +83723,83723 +83724,83724 +77574,83724 +83725,83725 +83726,83726 +83727,83727 +83728,83728 +83729,83729 +83730,83730 +83731,83731 +83732,83732 +83733,83733 +83734,83734 +83735,83735 +2506,83735 +83736,83736 +83737,83737 +83738,83738 +83739,83739 +83740,83740 +27744,83740 +83741,83741 +83742,83742 +83743,83743 +83744,83744 +83745,83745 +83746,83746 +83747,83747 +235035,83748 +83748,83748 +83749,83749 +83750,83750 +83751,83751 +83752,83752 +83753,83753 +83754,83754 +83755,83755 +83756,83756 +218320,83756 +83757,83757 +6291,83757 +83758,83758 +83759,83759 +83760,83760 +83985,83760 +83761,83761 +83762,83762 +83763,83763 +83764,83764 +83765,83765 +83766,83766 +83767,83767 +83992,83767 +83768,83768 +83769,83769 +83770,83770 +83771,83771 +83772,83772 +83773,83773 +155883,83774 +83774,83774 +190510,83774 +83775,83775 +8706,83775 +83776,83776 +83777,83777 +83778,83778 +216732,83778 +83779,83779 +83780,83780 +83781,83781 +83782,83782 +83783,83783 +83784,83784 +83785,83785 +83786,83786 +83787,83787 +83788,83788 +83789,83789 +83790,83790 +83791,83791 +83792,83792 +83793,83793 +83794,83794 +83795,83795 +83796,83796 +83797,83797 +146439,83797 +83798,83798 +83799,83799 +83800,83800 +83801,83801 +83802,83802 +83803,83803 +83804,83804 +83805,83805 +83806,83806 +83807,83807 +83808,83808 +83809,83809 +83810,83810 +83811,83811 +83812,83812 +83813,83813 +164463,83813 +83814,83814 +83815,83815 +83816,83816 +83817,83817 +83818,83818 +83819,83819 +83820,83820 +83821,83821 +83822,83822 +83823,83823 +83824,83824 +83825,83825 +88537,83825 +83826,83826 +83827,83827 +83828,83828 +83829,83829 +83830,83830 +83831,83831 +83832,83832 +83833,83833 +83834,83834 +83835,83835 +83836,83836 +83837,83837 +83838,83838 +224878,83838 +83839,83839 +83840,83840 +83841,83841 +105640,83841 +83842,83842 +83843,83843 +83844,83844 +83883,83883 +83884,83884 +83885,83885 +83886,83886 +83887,83887 +83888,83888 +83889,83889 +83890,83890 +207909,83890 +83891,83891 +83892,83892 +83893,83893 +12370,83893 +83894,83894 +83895,83895 +13386,83895 +83896,83896 +83897,83897 +83898,83898 +83899,83899 +83900,83900 +83901,83901 +83902,83902 +83903,83903 +83904,83904 +83905,83905 +206723,83905 +83906,83906 +83907,83907 +83908,83908 +83909,83909 +83910,83910 +83911,83911 +83912,83912 +83913,83913 +83914,83914 +83915,83915 +161095,83915 +83916,83916 +83917,83917 +83918,83918 +83919,83919 +83920,83920 +83921,83921 +83922,83922 +83923,83923 +83924,83924 +83925,83925 +83926,83926 +83927,83927 +83928,83928 +83929,83929 +83930,83930 +83931,83931 +83932,83932 +83933,83933 +83934,83934 +83935,83935 +83936,83936 +83937,83937 +83938,83938 +83939,83939 +83940,83940 +83941,83941 +83942,83942 +120681,83942 +83943,83943 +184443,83943 +83944,83944 +83945,83945 +83946,83946 +83947,83947 +83948,83948 +83949,83949 +84193,83949 +83950,83950 +83951,83951 +83952,83952 +6695,83952 +83953,83953 +83954,83954 +83955,83955 +83956,83956 +83957,83957 +83958,83958 +83959,83959 +83960,83960 +83961,83961 +83962,83962 +83963,83963 +83964,83964 +83965,83965 +83966,83966 +83967,83967 +83968,83968 +209352,83968 +83969,83969 +83970,83970 +83971,83971 +83972,83972 +83973,83973 +83974,83974 +83975,83975 +83976,83976 +83977,83977 +83978,83978 +83979,83979 +83980,83980 +83981,83981 +83982,83982 +109518,83982 +83983,83983 +83984,83984 +83985,83985 +83760,83985 +178387,83985 +172423,83985 +5483,83985 +83986,83986 +83987,83987 +83988,83988 +83989,83989 +83990,83990 +83991,83991 +83992,83992 +83767,83992 +83993,83993 +83994,83994 +83995,83995 +83996,83996 +207424,83996 +83997,83997 +83998,83998 +83999,83999 +84000,84000 +136942,84000 +129355,84000 +84001,84001 +84002,84002 +84003,84003 +84004,84004 +84005,84005 +84006,84006 +84007,84007 +84008,84008 +84009,84009 +84010,84010 +84011,84011 +84012,84012 +84013,84013 +84014,84014 +84015,84015 +84016,84016 +165266,84016 +84017,84017 +84018,84018 +84019,84019 +84020,84020 +102013,84020 +84021,84021 +84022,84022 +84023,84023 +84024,84024 +216561,84024 +84025,84025 +84026,84026 +84027,84027 +84028,84028 +84029,84029 +84030,84030 +84031,84031 +84032,84032 +84033,84033 +84034,84034 +84035,84035 +84036,84036 +84037,84037 +84038,84038 +84039,84039 +84040,84040 +84041,84041 +84042,84042 +84043,84043 +84044,84044 +84045,84045 +84046,84046 +84047,84047 +89576,84047 +84048,84048 +84049,84049 +84050,84050 +84051,84051 +206714,84051 +84052,84052 +84053,84053 +84054,84054 +84055,84055 +84056,84056 +84057,84057 +84058,84058 +84059,84059 +84060,84060 +84061,84061 +84062,84062 +84063,84063 +84064,84064 +179823,84064 +84065,84065 +84066,84066 +84067,84067 +84068,84068 +84069,84069 +84070,84070 +84071,84071 +84072,84072 +84073,84073 +84074,84074 +84075,84075 +84076,84076 +84077,84077 +84078,84078 +84079,84079 +84080,84080 +197689,84080 +84081,84081 +84082,84082 +84083,84083 +84084,84084 +84085,84085 +84086,84086 +84087,84087 +84088,84088 +84089,84089 +141504,84089 +84090,84090 +84091,84091 +84092,84092 +216550,84092 +84093,84093 +84094,84094 +84095,84095 +191664,84095 +84096,84096 +84097,84097 +141563,84097 +143151,84097 +29940,84097 +84098,84098 +207883,84098 +191797,84098 +84099,84099 +210277,84099 +84100,84100 +84101,84101 +163144,84101 +84102,84102 +191801,84102 +84103,84103 +84104,84104 +198404,84104 +84105,84105 +84106,84106 +221991,84106 +191565,84107 +84107,84107 +154504,84107 +84108,84108 +84109,84109 +84110,84110 +84111,84111 +58687,84111 +84112,84112 +84113,84113 +84114,84114 +84115,84115 +84116,84116 +84117,84117 +84118,84118 +84119,84119 +84120,84120 +84121,84121 +84122,84122 +84123,84123 +84124,84124 +84125,84125 +77793,84125 +84162,84162 +84163,84163 +84164,84164 +84165,84165 +84166,84166 +84167,84167 +84168,84168 +84169,84169 +84170,84170 +84171,84171 +84172,84172 +84173,84173 +84174,84174 +84175,84175 +84176,84176 +159463,84176 +84177,84177 +84178,84178 +84179,84179 +25108,84179 +84180,84180 +84181,84181 +84182,84182 +84183,84183 +84184,84184 +84185,84185 +88621,84185 +84186,84186 +84187,84187 +11991,84187 +84188,84188 +54754,84188 +46650,84188 +84189,84189 +84190,84190 +84191,84191 +84192,84192 +9676,84192 +84193,84193 +83949,84193 +84194,84194 +206590,84194 +84195,84195 +84196,84196 +84197,84197 +208520,84197 +84198,84198 +84199,84199 +84200,84200 +84201,84201 +84202,84202 +84203,84203 +84204,84204 +8720,84204 +84205,84205 +84206,84206 +84207,84207 +84208,84208 +84209,84209 +84210,84210 +84211,84211 +84212,84212 +88635,84212 +84213,84213 +84214,84214 +84215,84215 +84216,84216 +84217,84217 +84218,84218 +84219,84219 +84220,84220 +84221,84221 +84222,84222 +84223,84223 +84224,84224 +84225,84225 +84226,84226 +84227,84227 +84228,84228 +84229,84229 +84230,84230 +142205,84230 +84231,84231 +84232,84232 +84233,84233 +84234,84234 +84235,84235 +84236,84236 +84237,84237 +84238,84238 +149692,84238 +84239,84239 +84240,84240 +84241,84241 +84242,84242 +84243,84243 +84244,84244 +84245,84245 +84246,84246 +84247,84247 +84248,84248 +84249,84249 +84250,84250 +84251,84251 +84252,84252 +84253,84253 +84254,84254 +84255,84255 +84256,84256 +84257,84257 +84258,84258 +84259,84259 +84260,84260 +84261,84261 +84262,84262 +84263,84263 +84264,84264 +84265,84265 +84266,84266 +84267,84267 +96030,84268 +84268,84268 +84269,84269 +84270,84270 +84271,84271 +84272,84272 +84273,84273 +84274,84274 +84275,84275 +84276,84276 +77385,84276 +84277,84277 +84278,84278 +84279,84279 +84280,84280 +84281,84281 +84282,84282 +84283,84283 +84284,84284 +84285,84285 +84286,84286 +84287,84287 +84288,84288 +84289,84289 +84290,84290 +84291,84291 +84292,84292 +84293,84293 +10463,84293 +84294,84294 +211314,84294 +84295,84295 +84296,84296 +84297,84297 +84298,84298 +84299,84299 +84300,84300 +84301,84301 +84302,84302 +84303,84303 +84304,84304 +84305,84305 +84306,84306 +163011,84306 +84307,84307 +84308,84308 +84309,84309 +84310,84310 +84311,84311 +84312,84312 +84313,84313 +84314,84314 +84315,84315 +84316,84316 +84317,84317 +84318,84318 +84361,84361 +84362,84362 +84363,84363 +84364,84364 +84365,84365 +84366,84366 +203593,84366 +84367,84367 +234417,84367 +84368,84368 +84369,84369 +84370,84370 +193818,84370 +84371,84371 +84372,84372 +84373,84373 +84374,84374 +84375,84375 +84376,84376 +84377,84377 +84378,84378 +178818,84378 +84379,84379 +156411,84379 +84380,84380 +84381,84381 +84382,84382 +84383,84383 +84384,84384 +84385,84385 +84386,84386 +84387,84387 +84388,84388 +84389,84389 +84503,84503 +84504,84504 +84505,84505 +84506,84506 +84507,84507 +84508,84508 +84509,84509 +84510,84510 +84511,84511 +84512,84512 +84513,84513 +84514,84514 +84515,84515 +84516,84516 +84517,84517 +84518,84518 +84519,84519 +84520,84520 +84522,84520 +84521,84521 +84522,84522 +84520,84522 +84523,84523 +84524,84524 +27922,84524 +84525,84525 +84526,84526 +84527,84527 +84528,84528 +84529,84529 +223022,84529 +84530,84530 +84531,84531 +84532,84532 +84533,84533 +84534,84534 +84535,84535 +84536,84536 +84537,84537 +84538,84538 +84539,84539 +84540,84540 +84541,84541 +84542,84542 +84543,84543 +84544,84544 +84545,84545 +84546,84546 +84547,84547 +84548,84548 +84549,84549 +84550,84550 +84551,84551 +60038,84551 +84552,84552 +84553,84553 +84554,84554 +84555,84555 +84556,84556 +84557,84557 +84558,84558 +91463,84558 +84559,84559 +84560,84560 +84561,84561 +84562,84562 +90572,84562 +68203,84562 +84563,84563 +84564,84564 +84565,84565 +84566,84566 +137931,84566 +130344,84566 +84567,84567 +84568,84568 +84569,84569 +84570,84570 +84571,84571 +84572,84572 +84573,84573 +84574,84574 +84575,84575 +84576,84576 +84577,84577 +84578,84578 +84579,84579 +84580,84580 +84581,84581 +84582,84582 +84583,84583 +84584,84584 +84585,84585 +84586,84586 +84587,84587 +84588,84588 +84589,84589 +84590,84590 +84591,84591 +84592,84592 +84593,84593 +84594,84594 +75179,84594 +84595,84595 +84596,84596 +84597,84597 +84598,84598 +84599,84599 +84600,84600 +84601,84601 +111826,84601 +84602,84602 +84603,84603 +84604,84604 +84605,84605 +84606,84606 +84607,84607 +84608,84608 +84609,84609 +84610,84610 +84611,84611 +114463,84611 +84612,84612 +84613,84613 +84614,84614 +84615,84615 +84616,84616 +84617,84617 +84618,84618 +84619,84619 +84620,84620 +84621,84621 +84622,84622 +84623,84623 +84624,84624 +84625,84625 +84626,84626 +84627,84627 +84628,84628 +84629,84629 +199713,84629 +84630,84630 +84631,84631 +84632,84632 +84633,84633 +84634,84634 +84635,84635 +84636,84636 +208194,84636 +84637,84637 +84638,84638 +84639,84639 +84640,84640 +84641,84641 +84642,84642 +84643,84643 +84644,84644 +84645,84645 +84646,84646 +84647,84647 +179906,84647 +84648,84648 +84649,84649 +84650,84650 +84651,84651 +84652,84652 +84653,84653 +81449,84653 +84654,84654 +84655,84655 +84656,84656 +84657,84657 +84658,84658 +84659,84659 +84660,84660 +215370,84660 +84661,84661 +84662,84662 +84663,84663 +174364,84663 +168400,84663 +84664,84664 +84665,84665 +84666,84666 +84667,84667 +76794,84667 +84668,84668 +84669,84669 +84670,84670 +84671,84671 +84672,84672 +84673,84673 +60412,84673 +84674,84674 +84675,84675 +204878,84675 +84676,84676 +188487,84676 +84677,84677 +84678,84678 +84679,84679 +84680,84680 +84681,84681 +84682,84682 +84683,84683 +84684,84684 +84685,84685 +84686,84686 +84687,84687 +84688,84688 +84689,84689 +84690,84690 +84691,84691 +84692,84692 +84693,84693 +84694,84694 +84695,84695 +84696,84696 +84697,84697 +84698,84698 +84699,84699 +84700,84700 +84701,84701 +84702,84702 +84703,84703 +38281,84703 +84704,84704 +84705,84705 +84706,84706 +84707,84707 +84708,84708 +84709,84709 +84710,84710 +241104,84710 +190962,84711 +84711,84711 +84712,84712 +180213,84712 +84713,84713 +84714,84714 +84715,84715 +84716,84716 +84717,84717 +84718,84718 +84719,84719 +84720,84720 +84721,84721 +84722,84722 +84723,84723 +84724,84724 +84725,84725 +84726,84726 +84727,84727 +84728,84728 +84729,84729 +84730,84730 +84731,84731 +84732,84732 +84733,84733 +84734,84734 +84735,84735 +84736,84736 +84737,84737 +84766,84766 +226420,84766 +84767,84767 +84768,84768 +84769,84769 +84770,84770 +84771,84771 +84772,84772 +84773,84773 +84774,84774 +84775,84775 +84776,84776 +84777,84777 +84778,84778 +84779,84779 +84780,84780 +84781,84781 +84782,84782 +84783,84783 +84784,84784 +84785,84785 +84786,84786 +84787,84787 +84788,84788 +84789,84789 +84790,84790 +84791,84791 +84792,84792 +84793,84793 +84794,84794 +84795,84795 +84796,84796 +84797,84797 +233145,84797 +84798,84798 +84799,84799 +84800,84800 +84801,84801 +191278,84801 +84802,84802 +26369,84802 +84803,84803 +84804,84804 +84805,84805 +84806,84806 +84807,84807 +84808,84808 +84809,84809 +84810,84810 +84811,84811 +84812,84812 +84813,84813 +84814,84814 +84815,84815 +84816,84816 +84817,84817 +84818,84818 +84819,84819 +84820,84820 +84821,84821 +84822,84822 +84823,84823 +84824,84824 +84825,84825 +84826,84826 +84827,84827 +224642,84827 +55290,84827 +47186,84827 +84828,84828 +84829,84829 +84830,84830 +84831,84831 +84832,84832 +179294,84832 +84833,84833 +84834,84834 +149439,84834 +84835,84835 +135834,84835 +128247,84835 +84836,84836 +84837,84837 +84838,84838 +84839,84839 +64099,84839 +84840,84840 +82753,84840 +84841,84841 +84842,84842 +84843,84843 +84844,84844 +84845,84845 +84846,84846 +84847,84847 +84848,84848 +84849,84849 +84850,84850 +84908,84908 +13300,84908 +84909,84909 +53165,84909 +45061,84909 +111851,84910 +84910,84910 +84911,84911 +84912,84912 +84913,84913 +84914,84914 +84915,84915 +84916,84916 +84917,84917 +84918,84918 +84919,84919 +84920,84920 +84921,84921 +84922,84922 +84923,84923 +84924,84924 +84925,84925 +84926,84926 +84927,84927 +84928,84928 +84929,84929 +60517,84929 +84930,84930 +84931,84931 +84932,84932 +84933,84933 +84934,84934 +104668,84934 +84935,84935 +84936,84936 +84937,84937 +84938,84938 +84939,84939 +84940,84940 +84941,84941 +241818,84941 +84942,84942 +84943,84943 +84944,84944 +84945,84945 +84946,84946 +84947,84947 +84948,84948 +131908,84948 +124321,84948 +84949,84949 +84950,84950 +85190,85190 +85191,85191 +85192,85192 +85193,85193 +85194,85194 +20390,85194 +85195,85195 +85196,85196 +85197,85197 +85198,85198 +78652,85198 +85199,85199 +85200,85200 +85201,85201 +85202,85202 +85203,85203 +85204,85204 +85205,85205 +85206,85206 +20099,85206 +85207,85207 +85208,85208 +85209,85209 +208630,85209 +85210,85210 +108666,85210 +12456,85210 +85211,85211 +85212,85212 +85213,85213 +119230,85213 +85214,85214 +85215,85215 +85216,85216 +85217,85217 +85218,85218 +85219,85219 +85220,85220 +85221,85221 +85222,85222 +85223,85223 +85224,85224 +85225,85225 +85226,85226 +57417,85226 +85227,85227 +85228,85228 +85229,85229 +85230,85230 +34045,85230 +85231,85231 +85232,85232 +85233,85233 +85234,85234 +85235,85235 +85236,85236 +85237,85237 +85238,85238 +85239,85239 +85240,85240 +85413,85413 +85414,85414 +85415,85415 +85416,85416 +85417,85417 +36857,85417 +224891,85417 +85418,85418 +85419,85419 +30172,85419 +85420,85420 +85421,85421 +85422,85422 +28090,85422 +85423,85423 +85424,85424 +85425,85425 +85426,85426 +232724,85426 +85427,85427 +85428,85428 +85429,85429 +85430,85430 +85431,85431 +85432,85432 +158424,85432 +179433,85432 +85433,85433 +85434,85434 +85435,85435 +85436,85436 +85437,85437 +231968,85437 +85438,85438 +85439,85439 +85440,85440 +85441,85441 +85442,85442 +85443,85443 +85444,85444 +85445,85445 +85446,85446 +35663,85446 +85447,85447 +85448,85448 +85449,85449 +85450,85450 +85451,85451 +209812,85451 +85452,85452 +85453,85453 +85454,85454 +85455,85455 +85456,85456 +85457,85457 +85458,85458 +85459,85459 +85460,85460 +85461,85461 +85462,85462 +85463,85463 +85464,85464 +85465,85465 +85466,85466 +85467,85467 +85468,85468 +85469,85469 +85470,85470 +85471,85471 +85472,85472 +85473,85473 +85474,85474 +85475,85475 +85508,85476 +85476,85476 +85477,85477 +85510,85478 +85478,85478 +154516,85478 +85479,85479 +85512,85480 +85480,85480 +85513,85481 +85481,85481 +85482,85482 +85483,85483 +85516,85484 +85484,85484 +85517,85485 +85485,85485 +85486,85486 +85487,85487 +85488,85488 +179437,85488 +85489,85489 +85490,85490 +85491,85491 +85492,85492 +85493,85493 +85494,85494 +85495,85495 +85496,85496 +85497,85497 +85498,85498 +85499,85499 +85500,85500 +85501,85501 +85502,85502 +85503,85503 +85504,85504 +85505,85505 +85506,85506 +85507,85507 +85508,85508 +85476,85508 +85509,85509 +85510,85510 +85478,85510 +154516,85510 +85511,85511 +85512,85512 +85480,85512 +85513,85513 +85481,85513 +85514,85514 +85515,85515 +85516,85516 +85484,85516 +85517,85517 +85485,85517 +85518,85518 +85519,85519 +85520,85520 +85521,85521 +85522,85522 +85523,85523 +85524,85524 +85525,85525 +85526,85526 +85527,85527 +85528,85528 +85529,85529 +85530,85530 +85531,85531 +85532,85532 +85573,85573 +85574,85574 +25039,85574 +85575,85575 +85576,85576 +85577,85577 +85578,85578 +25043,85578 +85579,85579 +25044,85579 +85580,85580 +25045,85580 +85581,85581 +25046,85581 +85582,85582 +25047,85582 +85583,85583 +25048,85583 +2206,85583 +85584,85584 +25049,85584 +85585,85585 +85586,85586 +85587,85587 +85588,85588 +85589,85589 +179258,85589 +85590,85590 +85591,85591 +85592,85592 +85593,85593 +85594,85594 +85595,85595 +85596,85596 +85597,85597 +148661,85597 +85598,85598 +184143,85599 +85599,85599 +110656,85599 +85600,85600 +85601,85601 +85602,85602 +85603,85603 +85604,85604 +85605,85605 +85606,85606 +85607,85607 +85608,85608 +85609,85609 +85610,85610 +51344,85610 +43240,85610 +85611,85611 +85612,85612 +115996,85612 +85613,85613 +85614,85614 +85615,85615 +85616,85616 +85617,85617 +85618,85618 +85619,85619 +85620,85620 +25093,85620 +85621,85621 +85622,85622 +85623,85623 +85624,85624 +85625,85625 +85626,85626 +85627,85627 +85628,85628 +85629,85629 +85630,85630 +85631,85631 +85632,85632 +85633,85633 +85634,85634 +85635,85635 +85636,85636 +85637,85637 +85638,85638 +85639,85639 +85640,85640 +85641,85641 +85642,85642 +85643,85643 +85644,85644 +85645,85645 +85646,85646 +85647,85647 +85648,85648 +85649,85649 +85650,85650 +85651,85651 +85652,85652 +144853,85652 +85653,85653 +85654,85654 +85655,85655 +85656,85656 +85657,85657 +86895,85657 +85658,85658 +85659,85659 +85660,85660 +85661,85661 +85662,85662 +85663,85663 +85664,85664 +85665,85665 +85666,85666 +86689,85666 +85667,85667 +85668,85668 +85669,85669 +85670,85670 +85671,85671 +85672,85672 +85673,85673 +85674,85674 +85675,85675 +85676,85676 +85677,85677 +85678,85678 +85679,85679 +85680,85680 +85681,85681 +85682,85682 +85683,85683 +85684,85684 +85685,85685 +85686,85686 +85687,85687 +85688,85688 +201147,85689 +85689,85689 +85690,85690 +85691,85691 +85692,85692 +85693,85693 +85694,85694 +85695,85695 +85696,85696 +85697,85697 +85698,85698 +85699,85699 +85700,85700 +85701,85701 +85702,85702 +85703,85703 +85704,85704 +85705,85705 +85706,85706 +85707,85707 +85708,85708 +85709,85709 +85710,85710 +85711,85711 +85712,85712 +85713,85713 +85714,85714 +85715,85715 +85716,85716 +242048,85716 +85778,85778 +85779,85779 +85780,85780 +85781,85781 +173921,85781 +167957,85781 +85782,85782 +85783,85783 +85784,85784 +74914,85784 +85785,85785 +2050,85785 +85786,85786 +85787,85787 +85788,85788 +85789,85789 +85790,85790 +85791,85791 +85792,85792 +85793,85793 +85794,85794 +85795,85795 +85796,85796 +85797,85797 +85798,85798 +85799,85799 +85800,85800 +85801,85801 +85802,85802 +85803,85803 +85804,85804 +85805,85805 +85806,85806 +3314,85806 +85807,85807 +85808,85808 +85809,85809 +85810,85810 +85811,85811 +85812,85812 +85813,85813 +85814,85814 +85815,85815 +85816,85816 +197695,85816 +85817,85817 +203787,85817 +85818,85818 +85819,85819 +85820,85820 +85821,85821 +85822,85822 +85823,85823 +110835,85823 +85824,85824 +85825,85825 +85826,85826 +85827,85827 +85828,85828 +85829,85829 +85830,85830 +85831,85831 +85832,85832 +91015,85832 +85833,85833 +85834,85834 +85835,85835 +85836,85836 +85837,85837 +85838,85838 +85839,85839 +85840,85840 +85841,85841 +85842,85842 +85843,85843 +85844,85844 +85845,85845 +85846,85846 +85847,85847 +161547,85847 +85848,85848 +85849,85849 +85850,85850 +85851,85851 +85852,85852 +85853,85853 +85854,85854 +85855,85855 +85856,85856 +85857,85857 +85858,85858 +85859,85859 +85860,85860 +85861,85861 +85862,85862 +85863,85863 +85864,85864 +85865,85865 +85866,85866 +85867,85867 +215379,85867 +85868,85868 +85869,85869 +85870,85870 +85871,85871 +85872,85872 +85873,85873 +85874,85874 +85875,85875 +85876,85876 +85877,85877 +8579,85877 +8399,85877 +85878,85878 +143040,85878 +85879,85879 +85880,85880 +85881,85881 +85882,85882 +85883,85883 +85884,85884 +85885,85885 +85886,85886 +25075,85886 +85887,85887 +85888,85888 +85889,85889 +85890,85890 +85891,85891 +85892,85892 +85893,85893 +85894,85894 +85895,85895 +85896,85896 +85897,85897 +85898,85898 +85899,85899 +179284,85900 +85900,85900 +85901,85901 +85902,85902 +85903,85903 +85904,85904 +85905,85905 +85953,85953 +85954,85954 +85955,85955 +85956,85956 +85957,85957 +85958,85958 +85959,85959 +85960,85960 +85961,85961 +85962,85962 +85963,85963 +85964,85964 +85965,85965 +85966,85966 +85967,85967 +85968,85968 +85969,85969 +85970,85970 +62079,85970 +85971,85971 +85972,85972 +85973,85973 +85974,85974 +85975,85975 +31253,85975 +85976,85976 +85977,85977 +86679,85977 +85978,85978 +85979,85979 +85980,85980 +85981,85981 +85982,85982 +85983,85983 +85984,85984 +85985,85985 +85986,85986 +85987,85987 +85988,85988 +180675,85988 +85989,85989 +85990,85990 +85991,85991 +85992,85992 +85993,85993 +85994,85994 +189481,85994 +85995,85995 +85996,85996 +85997,85997 +85998,85998 +85999,85999 +86000,86000 +86001,86001 +86002,86002 +86003,86003 +86004,86004 +86005,86005 +86006,86006 +86007,86007 +86008,86008 +86009,86009 +86010,86010 +86011,86011 +86012,86012 +86013,86013 +86014,86014 +86015,86015 +86016,86016 +86017,86017 +86018,86018 +86019,86019 +86020,86020 +86021,86021 +86022,86022 +86023,86023 +86024,86024 +23666,86024 +86025,86025 +86026,86026 +86027,86027 +86028,86028 +86029,86029 +86030,86030 +86031,86031 +86032,86032 +86033,86033 +203883,86033 +86034,86034 +86035,86035 +86036,86036 +86037,86037 +148379,86037 +105641,86037 +211212,86038 +86038,86038 +86039,86039 +86040,86040 +86041,86041 +86042,86042 +86043,86043 +86044,86044 +86045,86045 +86046,86046 +86047,86047 +86048,86048 +86049,86049 +86050,86050 +86089,86089 +86090,86090 +86091,86091 +86092,86092 +51509,86092 +43405,86092 +86093,86093 +86094,86094 +86095,86095 +86096,86096 +86097,86097 +86098,86098 +86099,86099 +86100,86100 +86101,86101 +86102,86102 +86103,86103 +156481,86103 +86104,86104 +86105,86105 +86106,86106 +86107,86107 +178921,86107 +86108,86108 +86109,86109 +86110,86110 +86111,86111 +86112,86112 +86113,86113 +86114,86114 +86115,86115 +86116,86116 +86117,86117 +86118,86118 +86119,86119 +86120,86120 +86121,86121 +86122,86122 +86123,86123 +121467,86123 +86124,86124 +86125,86125 +86126,86126 +86127,86127 +86128,86128 +86129,86129 +86130,86130 +86131,86131 +86132,86132 +249217,86132 +86133,86133 +86276,86276 +86277,86277 +86278,86278 +86279,86279 +86280,86280 +86281,86281 +86282,86282 +86283,86283 +86284,86284 +86285,86285 +86286,86286 +86347,86347 +86348,86348 +86349,86349 +86350,86350 +86351,86351 +86352,86352 +86353,86353 +86354,86354 +86355,86355 +7589,86355 +96474,86355 +86356,86356 +86357,86357 +86358,86358 +86359,86359 +86360,86360 +86361,86361 +86362,86362 +243449,86362 +86363,86363 +86364,86364 +86365,86365 +86366,86366 +86367,86367 +86368,86368 +86369,86369 +144719,86369 +86370,86370 +86371,86371 +86372,86372 +86373,86373 +86374,86374 +86375,86375 +86376,86376 +86377,86377 +86378,86378 +86379,86379 +86380,86380 +86381,86381 +86382,86382 +86383,86383 +86384,86384 +86385,86385 +86386,86386 +86387,86387 +86388,86388 +86389,86389 +86390,86390 +86391,86391 +86392,86392 +86393,86393 +213623,86393 +86394,86394 +86395,86395 +86396,86396 +86397,86397 +86398,86398 +86399,86399 +86400,86400 +86401,86401 +86402,86402 +86403,86403 +74926,86403 +86404,86404 +86405,86405 +86406,86406 +86407,86407 +86408,86408 +86409,86409 +150576,86409 +86410,86410 +86411,86411 +86412,86412 +86413,86413 +86414,86414 +86415,86415 +86416,86416 +86417,86417 +86418,86418 +86419,86419 +86420,86420 +86421,86421 +86422,86422 +86423,86423 +142105,86423 +86424,86424 +86425,86425 +86426,86426 +86427,86427 +86428,86428 +86429,86429 +86430,86430 +86431,86431 +86432,86432 +86433,86433 +16247,86433 +96443,86433 +86434,86434 +86435,86435 +86436,86436 +226626,86436 +86437,86437 +86438,86438 +86439,86439 +86440,86440 +86441,86441 +86442,86442 +86443,86443 +86444,86444 +86445,86445 +86446,86446 +86447,86447 +86448,86448 +86449,86449 +28989,86449 +86450,86450 +86451,86451 +86452,86452 +86453,86453 +86454,86454 +86455,86455 +86456,86456 +86457,86457 +86458,86458 +86459,86459 +9256,86459 +86460,86460 +86461,86461 +86462,86462 +86463,86463 +86464,86464 +86465,86465 +36124,86465 +86466,86466 +86467,86467 +86468,86468 +86521,86521 +86522,86522 +86523,86523 +86524,86524 +86525,86525 +86526,86526 +86527,86527 +86528,86528 +220111,86528 +86529,86529 +86530,86530 +86531,86531 +86532,86532 +86533,86533 +86534,86534 +86535,86535 +86536,86536 +86537,86537 +86538,86538 +86539,86539 +86540,86540 +86541,86541 +86542,86542 +86543,86543 +86544,86544 +86545,86545 +86546,86546 +86547,86547 +86548,86548 +86549,86549 +86550,86550 +86551,86551 +86552,86552 +86553,86553 +86554,86554 +86555,86555 +86556,86556 +11989,86556 +86557,86557 +86558,86558 +86559,86559 +86560,86560 +86561,86561 +160817,86561 +86562,86562 +86563,86563 +86564,86564 +86565,86565 +86566,86566 +86567,86567 +86568,86568 +86569,86569 +86570,86570 +86571,86571 +86572,86572 +9681,86572 +86573,86573 +86574,86574 +86575,86575 +86576,86576 +86577,86577 +86578,86578 +86579,86579 +86619,86619 +86620,86620 +86621,86621 +86622,86622 +86623,86623 +86624,86624 +86625,86625 +107398,86625 +86626,86626 +86627,86627 +86628,86628 +86629,86629 +86630,86630 +86631,86631 +86632,86632 +86633,86633 +86634,86634 +86635,86635 +86636,86636 +86637,86637 +86638,86638 +86639,86639 +86640,86640 +86641,86641 +86642,86642 +86643,86643 +86644,86644 +86645,86645 +86646,86646 +86647,86647 +86648,86648 +86649,86649 +86650,86650 +86651,86651 +86652,86652 +86653,86653 +86654,86654 +86655,86655 +86656,86656 +86657,86657 +86658,86658 +86659,86659 +153150,86659 +86660,86660 +86661,86661 +86662,86662 +39716,86662 +86663,86663 +86664,86664 +89146,86664 +86665,86665 +86666,86666 +158314,86666 +86667,86667 +86668,86668 +86669,86669 +86670,86670 +86671,86671 +86672,86672 +86673,86673 +86674,86674 +86675,86675 +86676,86676 +86677,86677 +86678,86678 +86679,86679 +85977,86679 +86680,86680 +86681,86681 +86682,86682 +86683,86683 +86684,86684 +216691,86684 +86685,86685 +86686,86686 +86687,86687 +86688,86688 +149248,86688 +86689,86689 +85666,86689 +86690,86690 +86691,86691 +210074,86691 +86692,86692 +200464,86692 +86693,86693 +86694,86694 +86695,86695 +86696,86696 +86697,86697 +86698,86698 +86699,86699 +86700,86700 +86701,86701 +86702,86702 +86703,86703 +86704,86704 +86705,86705 +86706,86706 +86707,86707 +86708,86708 +86709,86709 +86710,86710 +86711,86711 +86712,86712 +86713,86713 +86714,86714 +86715,86715 +86716,86716 +20543,86716 +217686,86716 +86717,86717 +62831,86717 +86718,86718 +86719,86719 +86720,86720 +38226,86720 +86721,86721 +86722,86722 +86723,86723 +34600,86723 +86724,86724 +34601,86724 +86725,86725 +86726,86726 +86727,86727 +86728,86728 +86729,86729 +86730,86730 +86731,86731 +86732,86732 +86733,86733 +86734,86734 +86735,86735 +86736,86736 +86737,86737 +86738,86738 +86739,86739 +86740,86740 +86741,86741 +86742,86742 +86743,86743 +86744,86744 +86745,86745 +86746,86746 +86747,86747 +86748,86748 +86749,86749 +191278,86749 +86750,86750 +136532,86750 +128945,86750 +86751,86751 +86752,86752 +143078,86752 +86753,86753 +86754,86754 +86755,86755 +86756,86756 +86757,86757 +86758,86758 +86759,86759 +86760,86760 +86818,86818 +86819,86819 +86820,86820 +86821,86821 +86822,86822 +86823,86823 +86824,86824 +86825,86825 +86826,86826 +86827,86827 +86828,86828 +142627,86828 +86829,86829 +190405,86830 +86830,86830 +86831,86831 +86832,86832 +86833,86833 +86834,86834 +86835,86835 +242783,86835 +86836,86836 +86837,86837 +86838,86838 +86839,86839 +86840,86840 +86841,86841 +86842,86842 +86843,86843 +86844,86844 +86845,86845 +86846,86846 +86847,86847 +86848,86848 +149841,86848 +86849,86849 +86850,86850 +86851,86851 +86852,86852 +86853,86853 +86854,86854 +86855,86855 +86856,86856 +86857,86857 +86858,86858 +86859,86859 +86860,86860 +86861,86861 +86862,86862 +86863,86863 +86864,86864 +86865,86865 +86866,86866 +86867,86867 +86868,86868 +214727,86868 +86869,86869 +120180,86869 +178235,86869 +172271,86869 +86870,86870 +86871,86871 +86872,86872 +86873,86873 +86874,86874 +86875,86875 +86876,86876 +72583,86876 +86877,86877 +86878,86878 +86879,86879 +86880,86880 +21678,86880 +86881,86881 +86882,86882 +86883,86883 +86884,86884 +86885,86885 +86886,86886 +86887,86887 +86888,86888 +86889,86889 +86890,86890 +86891,86891 +86892,86892 +86893,86893 +86894,86894 +86895,86895 +85657,86895 +86896,86896 +86897,86897 +86898,86898 +86899,86899 +86900,86900 +86901,86901 +86902,86902 +86903,86903 +86904,86904 +86905,86905 +86906,86906 +75474,86906 +86907,86907 +86908,86908 +86909,86909 +86910,86910 +176586,86910 +170622,86910 +86911,86911 +86912,86912 +86913,86913 +86914,86914 +86915,86915 +86916,86916 +86917,86917 +86955,86955 +86956,86956 +86957,86957 +86958,86958 +86959,86959 +86960,86960 +86961,86961 +86962,86962 +222212,86962 +117090,86962 +86963,86963 +86964,86964 +86965,86965 +86966,86966 +86967,86967 +86968,86968 +86969,86969 +86970,86970 +86971,86971 +87007,87007 +87008,87008 +87009,87009 +87010,87010 +87011,87011 +87012,87012 +87013,87013 +87014,87014 +87015,87015 +62772,87015 +87016,87016 +87017,87017 +87018,87018 +87019,87019 +87020,87020 +87096,87096 +87097,87097 +87098,87098 +87099,87099 +87100,87100 +87101,87101 +87102,87102 +87103,87103 +87104,87104 +87105,87105 +87106,87106 +87107,87107 +87108,87108 +87109,87109 +87110,87110 +87111,87111 +87112,87112 +87113,87113 +87114,87114 +87115,87115 +87116,87116 +87117,87117 +87118,87118 +87119,87119 +87120,87120 +87121,87121 +160656,87121 +87122,87122 +87123,87123 +87124,87124 +87125,87125 +87126,87126 +87127,87127 +198803,87127 +87128,87128 +87129,87129 +87130,87130 +87131,87131 +87132,87132 +87133,87133 +87134,87134 +87135,87135 +87136,87136 +578,87136 +87137,87137 +87138,87138 +87139,87139 +79477,87139 +199584,87139 +87140,87140 +87141,87141 +87142,87142 +87143,87143 +87144,87144 +87145,87145 +87146,87146 +87147,87147 +87148,87148 +87149,87149 +87150,87150 +87151,87151 +87152,87152 +87153,87153 +87154,87154 +87155,87155 +87156,87156 +87157,87157 +87158,87158 +87159,87159 +87160,87160 +87161,87161 +87162,87162 +87163,87163 +87164,87164 +87165,87165 +87166,87166 +87167,87167 +87168,87168 +87169,87169 +87170,87170 +87171,87171 +87172,87172 +87173,87173 +87174,87174 +87215,87215 +87216,87216 +87217,87217 +87218,87218 +87219,87219 +87220,87220 +87221,87221 +87222,87222 +87223,87223 +87224,87224 +87225,87225 +87226,87226 +87227,87227 +149375,87227 +87228,87228 +87229,87229 +87230,87230 +87231,87231 +87232,87232 +87233,87233 +87234,87234 +87235,87235 +87236,87236 +87237,87237 +87238,87238 +87239,87239 +87240,87240 +87241,87241 +87242,87242 +87243,87243 +87244,87244 +87245,87245 +87246,87246 +87247,87247 +117018,87247 +87248,87248 +70432,87248 +87249,87249 +87250,87250 +87251,87251 +194435,87251 +87252,87252 +87253,87253 +87254,87254 +87255,87255 +87256,87256 +225475,87256 +87257,87257 +87258,87258 +87259,87259 +87307,87307 +87308,87308 +87309,87309 +87346,87346 +87347,87347 +87348,87348 +87349,87349 +87350,87350 +87351,87351 +53848,87351 +45744,87351 +87352,87352 +176501,87352 +170537,87352 +87353,87353 +87354,87354 +87355,87355 +87356,87356 +87357,87357 +87358,87358 +87359,87359 +87360,87360 +87361,87361 +87362,87362 +87363,87363 +87364,87364 +87365,87365 +87366,87366 +87367,87367 +87368,87368 +87369,87369 +87370,87370 +87371,87371 +87372,87372 +87373,87373 +87374,87374 +87375,87375 +9123,87375 +87376,87376 +87377,87377 +1076,87377 +87378,87378 +31702,87378 +87379,87379 +87380,87380 +87381,87381 +87382,87382 +87383,87383 +87384,87384 +87385,87385 +87386,87386 +87387,87387 +87388,87388 +87389,87389 +87390,87390 +87391,87391 +87392,87392 +222867,87392 +87393,87393 +87394,87394 +87395,87395 +87396,87396 +87397,87397 +87398,87398 +87399,87399 +87400,87400 +87401,87401 +87402,87402 +87403,87403 +87404,87404 +87405,87405 +87406,87406 +87407,87407 +87446,87446 +87447,87447 +87448,87448 +87449,87449 +207891,87449 +87450,87450 +221725,87450 +87451,87451 +87452,87452 +87453,87453 +87454,87454 +87455,87455 +87456,87456 +87496,87496 +164609,87497 +87497,87497 +87498,87498 +87499,87499 +87500,87500 +87501,87501 +87502,87502 +87503,87503 +208874,87503 +87504,87504 +87505,87505 +87506,87506 +87507,87507 +87508,87508 +87509,87509 +87510,87510 +87511,87511 +87512,87512 +87513,87513 +87514,87514 +87515,87515 +87516,87516 +87517,87517 +179232,87517 +87518,87518 +87519,87519 +87520,87520 +87521,87521 +87522,87522 +87523,87523 +87524,87524 +87525,87525 +87526,87526 +150556,87526 +87527,87527 +87528,87528 +87529,87529 +87530,87530 +87531,87531 +186416,87531 +87532,87532 +87533,87533 +87534,87534 +87535,87535 +106186,87535 +87536,87536 +87537,87537 +87538,87538 +87539,87539 +87540,87540 +87541,87541 +87542,87542 +87543,87543 +87544,87544 +87545,87545 +87546,87546 +87547,87547 +87548,87548 +87549,87549 +87550,87550 +87551,87551 +87552,87552 +87553,87553 +241589,87553 +87554,87554 +87555,87555 +153632,87555 +87556,87556 +87557,87557 +87558,87558 +87559,87559 +87560,87560 +87561,87561 +87562,87562 +87563,87563 +87564,87564 +87565,87565 +87566,87566 +87567,87567 +87568,87568 +87569,87569 +87570,87570 +87571,87571 +87572,87572 +87573,87573 +87574,87574 +87575,87575 +87576,87576 +87577,87577 +87578,87578 +87579,87579 +216602,87579 +87580,87580 +87581,87581 +87582,87582 +87583,87583 +87584,87584 +87585,87585 +87586,87586 +87587,87587 +87588,87588 +87589,87589 +87590,87590 +87591,87591 +87592,87592 +87593,87593 +87594,87594 +87595,87595 +87596,87596 +217237,87596 +142539,87596 +87597,87597 +87598,87598 +87599,87599 +87600,87600 +87601,87601 +105647,87601 +87602,87602 +226768,87602 +87603,87603 +87604,87604 +87605,87605 +87606,87606 +87607,87607 +87608,87608 +87609,87609 +87610,87610 +87611,87611 +87612,87612 +87613,87613 +87614,87614 +254,87614 +87615,87615 +87616,87616 +87617,87617 +87618,87618 +87619,87619 +87620,87620 +87752,87752 +87753,87753 +87754,87754 +87755,87755 +87756,87756 +87757,87757 +87758,87758 +87759,87759 +87760,87760 +87761,87761 +87762,87762 +87763,87763 +87764,87764 +87765,87765 +87766,87766 +87767,87767 +87768,87768 +87769,87769 +87770,87770 +87771,87771 +87772,87772 +87773,87773 +216998,87773 +87774,87774 +87775,87775 +158609,87775 +87776,87776 +87777,87777 +87778,87778 +87779,87779 +87780,87780 +87781,87781 +87782,87782 +87783,87783 +87784,87784 +87785,87785 +87786,87786 +36848,87786 +4820,87786 +87787,87787 +87788,87788 +87789,87789 +87790,87790 +238008,87790 +87791,87791 +155193,87791 +87792,87792 +87793,87793 +87794,87794 +87795,87795 +87796,87796 +87797,87797 +224364,87797 +87798,87798 +87799,87799 +87800,87800 +87880,87880 +87881,87881 +87882,87882 +87883,87883 +87884,87884 +87885,87885 +143345,87885 +87886,87886 +87887,87887 +87888,87888 +87889,87889 +87890,87890 +87891,87891 +87892,87892 +87893,87893 +87894,87894 +87895,87895 +87896,87896 +87897,87897 +87898,87898 +87899,87899 +87900,87900 +87901,87901 +87902,87902 +87903,87903 +87904,87904 +87905,87905 +87906,87906 +87907,87907 +87908,87908 +87909,87909 +87910,87910 +87911,87911 +87912,87912 +87913,87913 +87914,87914 +87915,87915 +87916,87916 +87917,87917 +87918,87918 +165272,87918 +87919,87919 +87920,87920 +220020,87921 +87921,87921 +87922,87922 +87923,87923 +87924,87924 +87925,87925 +87926,87926 +87927,87927 +87928,87928 +87929,87929 +87930,87930 +87931,87931 +87932,87932 +87933,87933 +87934,87934 +87935,87935 +87936,87936 +87937,87937 +87938,87938 +87939,87939 +87940,87940 +87941,87941 +185010,87941 +88033,88033 +88034,88034 +88035,88035 +88036,88036 +5495,88036 +88037,88037 +88038,88038 +88039,88039 +88040,88040 +88041,88041 +88042,88042 +271,88042 +88043,88043 +88044,88044 +88045,88045 +88046,88046 +88047,88047 +132642,88047 +125055,88047 +88048,88048 +88049,88049 +88050,88050 +88051,88051 +88052,88052 +88053,88053 +88054,88054 +88055,88055 +88056,88056 +88057,88057 +88058,88058 +88059,88059 +88060,88060 +88061,88061 +88062,88062 +88063,88063 +88064,88064 +92548,88064 +88065,88065 +88066,88066 +88067,88067 +88068,88068 +88069,88069 +88070,88070 +88071,88071 +88072,88072 +88073,88073 +88074,88074 +88075,88075 +193731,88075 +88076,88076 +242770,88076 +88077,88077 +33955,88077 +88078,88078 +88079,88079 +90133,88079 +88080,88080 +88081,88081 +88082,88082 +110277,88082 +88083,88083 +88084,88084 +88085,88085 +88086,88086 +88087,88087 +88088,88088 +88089,88089 +88090,88090 +88091,88091 +88092,88092 +88093,88093 +88094,88094 +88095,88095 +88096,88096 +88097,88097 +88098,88098 +88099,88099 +88100,88100 +88101,88101 +244081,88101 +88102,88102 +88103,88103 +88104,88104 +88105,88105 +88106,88106 +88107,88107 +88108,88108 +88109,88109 +88110,88110 +88111,88111 +88112,88112 +214255,88112 +88113,88113 +88114,88114 +88115,88115 +88116,88116 +88117,88117 +88118,88118 +88119,88119 +88120,88120 +88121,88121 +275,88121 +88122,88122 +149095,88122 +88123,88123 +88124,88124 +88125,88125 +88126,88126 +88127,88127 +88128,88128 +88129,88129 +4999,88129 +88130,88130 +88131,88131 +138683,88131 +88132,88132 +88133,88133 +88134,88134 +97491,88134 +160665,88134 +88135,88135 +88136,88136 +88137,88137 +88138,88138 +88139,88139 +88140,88140 +88141,88141 +88142,88142 +88143,88143 +119764,88143 +88144,88144 +88145,88145 +88146,88146 +88147,88147 +88148,88148 +88149,88149 +88150,88150 +88151,88151 +88152,88152 +88153,88153 +211924,88153 +88154,88154 +88155,88155 +88156,88156 +88157,88157 +88158,88158 +88159,88159 +88160,88160 +88161,88161 +214460,88161 +88162,88162 +88163,88163 +88164,88164 +88165,88165 +88166,88166 +88167,88167 +88168,88168 +88169,88169 +88170,88170 +88171,88171 +88172,88172 +88173,88173 +88174,88174 +88175,88175 +88176,88176 +88177,88177 +88178,88178 +88179,88179 +132756,88179 +125169,88179 +88180,88180 +88181,88181 +88182,88182 +111369,88182 +88183,88183 +88184,88184 +88185,88185 +211539,88185 +88186,88186 +88187,88187 +88188,88188 +88189,88189 +234408,88189 +242780,88189 +88190,88190 +88191,88191 +88192,88192 +88193,88193 +88194,88194 +88195,88195 +88196,88196 +88197,88197 +88198,88198 +88199,88199 +88200,88200 +88201,88201 +88202,88202 +88203,88203 +88204,88204 +88205,88205 +88206,88206 +88207,88207 +88208,88208 +88209,88209 +88210,88210 +88211,88211 +88212,88212 +88213,88213 +88214,88214 +88215,88215 +88216,88216 +88217,88217 +88218,88218 +88219,88219 +88220,88220 +88221,88221 +88222,88222 +164473,88222 +88223,88223 +88224,88224 +88225,88225 +88226,88226 +88227,88227 +88228,88228 +88229,88229 +143165,88229 +88230,88230 +88231,88231 +78018,88231 +88232,88232 +88233,88233 +88234,88234 +88235,88235 +88236,88236 +88237,88237 +88238,88238 +88239,88239 +88240,88240 +88241,88241 +88242,88242 +137923,88242 +130336,88242 +88243,88243 +187259,88243 +88244,88244 +88245,88245 +88246,88246 +88247,88247 +88248,88248 +88249,88249 +88250,88250 +222222,88251 +88251,88251 +88252,88252 +88253,88253 +88254,88254 +35615,88254 +88255,88255 +88256,88256 +88257,88257 +88258,88258 +88259,88259 +88260,88260 +173899,88260 +167935,88260 +88261,88261 +88262,88262 +88263,88263 +88264,88264 +88265,88265 +17186,88265 +88266,88266 +88389,88389 +88390,88390 +88391,88391 +88392,88392 +88393,88393 +88394,88394 +88395,88395 +88396,88396 +88397,88397 +88398,88398 +88399,88399 +88400,88400 +88401,88401 +88402,88402 +88403,88403 +88404,88404 +88405,88405 +88406,88406 +88407,88407 +88408,88408 +88409,88409 +88410,88410 +88411,88411 +88412,88412 +88413,88413 +88414,88414 +88415,88415 +88416,88416 +88417,88417 +88418,88418 +88419,88419 +88420,88420 +88421,88421 +88422,88422 +88423,88423 +88424,88424 +88425,88425 +88426,88426 +88427,88427 +88498,88498 +176540,88499 +170576,88499 +88499,88499 +88500,88500 +56153,88500 +48049,88500 +162769,88500 +88501,88501 +88502,88502 +88503,88503 +88504,88504 +88505,88505 +88506,88506 +88507,88507 +88508,88508 +222095,88508 +88509,88509 +88510,88510 +88511,88511 +88512,88512 +88513,88513 +88514,88514 +88515,88515 +88516,88516 +88517,88517 +88518,88518 +88519,88519 +88520,88520 +88521,88521 +88522,88522 +88523,88523 +88524,88524 +88525,88525 +88526,88526 +88527,88527 +88528,88528 +88529,88529 +104410,88529 +88530,88530 +227781,88530 +88531,88531 +88532,88532 +88533,88533 +88534,88534 +88535,88535 +88536,88536 +244035,88536 +88537,88537 +83825,88537 +88608,88608 +88609,88609 +88610,88610 +88611,88611 +88612,88612 +88613,88613 +88614,88614 +231825,88615 +88615,88615 +59536,88615 +88616,88616 +88617,88617 +88618,88618 +88619,88619 +88620,88620 +88621,88621 +84185,88621 +88622,88622 +88623,88623 +88624,88624 +88625,88625 +88626,88626 +88627,88627 +242074,88627 +88628,88628 +88629,88629 +88630,88630 +88631,88631 +88632,88632 +88633,88633 +88634,88634 +88635,88635 +84212,88635 +88636,88636 +88637,88637 +88638,88638 +88639,88639 +54616,88639 +46512,88639 +88640,88640 +88641,88641 +88642,88642 +88643,88643 +88644,88644 +88645,88645 +88646,88646 +15432,88646 +88647,88647 +88648,88648 +88649,88649 +88650,88650 +140250,88650 +88651,88651 +88652,88652 +88653,88653 +88654,88654 +88655,88655 +88656,88656 +88657,88657 +88658,88658 +88659,88659 +88660,88660 +88661,88661 +88662,88662 +88663,88663 +88664,88664 +88665,88665 +88666,88666 +88667,88667 +88668,88668 +88669,88669 +88670,88670 +88671,88671 +88672,88672 +231883,88672 +88673,88673 +88674,88674 +88675,88675 +88676,88676 +88677,88677 +88678,88678 +88679,88679 +88680,88680 +88681,88681 +88682,88682 +88683,88683 +88684,88684 +88685,88685 +88686,88686 +88687,88687 +88688,88688 +88689,88689 +88690,88690 +88691,88691 +88692,88692 +221993,88692 +88693,88693 +88694,88694 +88695,88695 +88696,88696 +88697,88697 +88698,88698 +88699,88699 +88700,88700 +88701,88701 +88702,88702 +88703,88703 +88704,88704 +88705,88705 +88706,88706 +88707,88707 +88708,88708 +88709,88709 +88710,88710 +88711,88711 +88712,88712 +88713,88713 +88714,88714 +88715,88715 +88716,88716 +88717,88717 +88718,88718 +88719,88719 +88720,88720 +88721,88721 +88722,88722 +88723,88723 +88724,88724 +88725,88725 +88726,88726 +88727,88727 +88728,88728 +88729,88729 +88730,88730 +88731,88731 +88732,88732 +88733,88733 +88734,88734 +73051,88734 +88735,88735 +88736,88736 +88737,88737 +88738,88738 +88739,88739 +88740,88740 +88741,88741 +88742,88742 +88743,88743 +141482,88743 +88744,88744 +88745,88745 +88746,88746 +88747,88747 +88748,88748 +88749,88749 +88750,88750 +88751,88751 +88752,88752 +88753,88753 +88754,88754 +88755,88755 +81444,88755 +88839,88839 +88840,88840 +88841,88841 +88842,88842 +88843,88843 +88844,88844 +88845,88845 +88846,88846 +88847,88847 +88848,88848 +88849,88849 +88850,88850 +88851,88851 +88852,88852 +88853,88853 +88854,88854 +88855,88855 +88856,88856 +88857,88857 +88858,88858 +88859,88859 +88860,88860 +88861,88861 +88862,88862 +88863,88863 +88864,88864 +153635,88864 +88902,88902 +88903,88903 +88939,88939 +88940,88940 +88941,88941 +31256,88941 +88942,88942 +88943,88943 +88944,88944 +88945,88945 +88946,88946 +88947,88947 +88948,88948 +88949,88949 +88950,88950 +88951,88951 +79872,88951 +88952,88952 +88953,88953 +88954,88954 +88955,88955 +88956,88956 +88957,88957 +88958,88958 +88959,88959 +88960,88960 +88961,88961 +88962,88962 +88963,88963 +88964,88964 +88965,88965 +230362,88965 +88966,88966 +88967,88967 +88968,88968 +88969,88969 +88970,88970 +231209,88970 +88971,88971 +88972,88972 +88973,88973 +88974,88974 +88975,88975 +88976,88976 +88977,88977 +88978,88978 +88979,88979 +88980,88980 +88981,88981 +88982,88982 +88983,88983 +88984,88984 +88985,88985 +88986,88986 +88987,88987 +88988,88988 +88989,88989 +88990,88990 +88991,88991 +88992,88992 +88995,88992 +88993,88993 +88994,88994 +65739,88994 +88995,88995 +88992,88995 +88996,88996 +88997,88997 +88998,88998 +88999,88999 +89000,89000 +89001,89001 +89002,89002 +89003,89003 +89004,89004 +89005,89005 +89006,89006 +89007,89007 +89142,89142 +89143,89143 +89144,89144 +89145,89145 +89146,89146 +86664,89146 +89147,89147 +89148,89148 +89149,89149 +89150,89150 +89151,89151 +89152,89152 +89153,89153 +89154,89154 +89155,89155 +89156,89156 +89157,89157 +89158,89158 +89159,89159 +89160,89160 +89161,89161 +89162,89162 +89163,89163 +89164,89164 +89165,89165 +79044,89165 +89166,89166 +89167,89167 +89168,89168 +89169,89169 +89170,89170 +89171,89171 +184212,89171 +89172,89172 +89173,89173 +89174,89174 +89175,89175 +20504,89175 +89176,89176 +89177,89177 +89178,89178 +89179,89179 +89180,89180 +89181,89181 +204446,89181 +89182,89182 +89183,89183 +89184,89184 +52321,89184 +44217,89184 +89185,89185 +89186,89186 +89187,89187 +35731,89187 +89188,89188 +89189,89189 +89190,89190 +89191,89191 +89192,89192 +89193,89193 +89194,89194 +53333,89194 +45229,89194 +3917,89194 +89195,89195 +89196,89196 +214425,89197 +89197,89197 +89198,89198 +89199,89199 +89200,89200 +89201,89201 +89202,89202 +89203,89203 +89204,89204 +117009,89204 +89205,89205 +89206,89206 +176786,89206 +170822,89206 +89207,89207 +149344,89207 +89208,89208 +89209,89209 +89210,89210 +89211,89211 +89212,89212 +89213,89213 +205298,89213 +89214,89214 +89215,89215 +89216,89216 +89217,89217 +89218,89218 +89219,89219 +109137,89219 +89220,89220 +89221,89221 +89222,89222 +89223,89223 +135424,89224 +127837,89224 +89224,89224 +89225,89225 +89226,89226 +89227,89227 +89228,89228 +185618,89228 +89229,89229 +89230,89230 +201158,89230 +89231,89231 +89232,89232 +89233,89233 +89318,89318 +89319,89319 +89320,89320 +89321,89321 +89322,89322 +89323,89323 +89324,89324 +89325,89325 +89326,89326 +207112,89326 +89327,89327 +89328,89328 +89329,89329 +164833,89329 +89330,89330 +89331,89331 +89332,89332 +89333,89333 +89334,89334 +179398,89334 +89335,89335 +89394,89394 +89395,89395 +89396,89396 +89397,89397 +89398,89398 +89399,89399 +89400,89400 +89401,89401 +89402,89402 +89403,89403 +89404,89404 +89442,89442 +89443,89443 +89444,89444 +89445,89445 +89446,89446 +89447,89447 +57707,89447 +89448,89448 +89449,89449 +89450,89450 +89451,89451 +89452,89452 +89453,89453 +89454,89454 +89470,89455 +89455,89455 +89456,89456 +89457,89457 +89458,89458 +89459,89459 +89460,89460 +89461,89461 +89462,89462 +89463,89463 +89464,89464 +124088,89464 +89465,89465 +89466,89466 +89467,89467 +89468,89468 +89469,89469 +89470,89470 +89455,89470 +89471,89471 +89472,89472 +89473,89473 +89474,89474 +89475,89475 +89476,89476 +89477,89477 +89478,89478 +89479,89479 +89480,89480 +89481,89481 +89482,89482 +89483,89483 +89484,89484 +89485,89485 +89486,89486 +89487,89487 +89488,89488 +89489,89489 +89490,89490 +89491,89491 +89492,89492 +110677,89492 +89493,89493 +208229,89493 +89494,89494 +89495,89495 +89496,89496 +89497,89497 +89498,89498 +89499,89499 +89500,89500 +89501,89501 +89502,89502 +89503,89503 +89504,89504 +146818,89504 +89505,89505 +89506,89506 +89507,89507 +89508,89508 +89509,89509 +89510,89510 +89511,89511 +89512,89512 +198708,89512 +89513,89513 +21541,89513 +89514,89514 +89515,89515 +89516,89516 +14506,89516 +89517,89517 +89518,89518 +89519,89519 +89520,89520 +89521,89521 +89522,89522 +89523,89523 +89524,89524 +89525,89525 +89526,89526 +89527,89527 +89528,89528 +89529,89529 +89530,89530 +40000,89530 +89531,89531 +89532,89532 +89533,89533 +179327,89533 +21610,89533 +89534,89534 +89535,89535 +89536,89536 +89537,89537 +89538,89538 +89539,89539 +89540,89540 +89541,89541 +89542,89542 +89543,89543 +209984,89543 +89544,89544 +89545,89545 +89546,89546 +89547,89547 +89548,89548 +89549,89549 +205437,89550 +89550,89550 +89551,89551 +89552,89552 +89553,89553 +89554,89554 +89555,89555 +89556,89556 +89557,89557 +89558,89558 +5838,89558 +89559,89559 +89560,89560 +89561,89561 +106254,89561 +89562,89562 +89563,89563 +89564,89564 +89565,89565 +89566,89566 +155619,89566 +89567,89567 +89568,89568 +89569,89569 +89570,89570 +89571,89571 +89572,89572 +89573,89573 +89574,89574 +89575,89575 +89576,89576 +84047,89576 +89577,89577 +89578,89578 +89579,89579 +89580,89580 +89581,89581 +89582,89582 +89583,89583 +89584,89584 +89585,89585 +89586,89586 +89587,89587 +89588,89588 +89589,89589 +77161,89589 +89590,89590 +89591,89591 +89592,89592 +89593,89593 +89594,89594 +89595,89595 +89596,89596 +89597,89597 +89598,89598 +35779,89598 +89599,89599 +104655,89599 +89600,89600 +89601,89601 +89602,89602 +89603,89603 +89604,89604 +89605,89605 +89606,89606 +89607,89607 +89608,89608 +89609,89609 +89610,89610 +89611,89611 +89612,89612 +89613,89613 +89614,89614 +89615,89615 +89616,89616 +89617,89617 +89618,89618 +89619,89619 +111090,89619 +89620,89620 +89621,89621 +89622,89622 +35969,89622 +89623,89623 +89624,89624 +89625,89625 +249213,89625 +2344,89625 +89626,89626 +89627,89627 +89699,89699 +89700,89700 +89701,89701 +212038,89701 +89702,89702 +89703,89703 +89704,89704 +89705,89705 +89706,89706 +14354,89706 +89707,89707 +89708,89708 +2407,89708 +89709,89709 +89710,89710 +89711,89711 +89712,89712 +89713,89713 +36342,89713 +89714,89714 +89715,89715 +89716,89716 +89717,89717 +89718,89718 +89719,89719 +89810,89810 +89811,89811 +89812,89812 +89813,89813 +89814,89814 +222948,89814 +89815,89815 +89816,89816 +89817,89817 +89818,89818 +89819,89819 +89820,89820 +89821,89821 +89822,89822 +89823,89823 +89824,89824 +202822,89824 +89825,89825 +89826,89826 +89827,89827 +89828,89828 +89829,89829 +89830,89830 +89831,89831 +89832,89832 +89833,89833 +89834,89834 +89835,89835 +89836,89836 +89837,89837 +89838,89838 +89839,89839 +89840,89840 +202094,89840 +89841,89841 +89842,89842 +89843,89843 +89844,89844 +89845,89845 +89846,89846 +89847,89847 +89848,89848 +89849,89849 +89850,89850 +89851,89851 +89852,89852 +89853,89853 +89854,89854 +89855,89855 +89856,89856 +89857,89857 +89858,89858 +89859,89859 +89860,89860 +89861,89861 +89862,89862 +89863,89863 +221739,89863 +89864,89864 +89865,89865 +89866,89866 +89867,89867 +89868,89868 +89941,89941 +89942,89942 +89943,89943 +89944,89944 +89945,89945 +89946,89946 +38711,89946 +89947,89947 +89948,89948 +89949,89949 +89950,89950 +89951,89951 +89952,89952 +89953,89953 +89954,89954 +89955,89955 +89956,89956 +89957,89957 +89958,89958 +89959,89959 +89960,89960 +89961,89961 +89962,89962 +89963,89963 +89964,89964 +89965,89965 +89966,89966 +89967,89967 +89968,89968 +89969,89969 +89970,89970 +89971,89971 +89972,89972 +89973,89973 +89974,89974 +90013,90013 +90115,90115 +90116,90116 +90117,90117 +90118,90118 +90119,90119 +90120,90120 +90121,90121 +90122,90122 +90123,90123 +90124,90124 +90125,90125 +90126,90126 +90127,90127 +90128,90128 +90129,90129 +90130,90130 +223635,90130 +90131,90131 +55110,90131 +47006,90131 +90132,90132 +90133,90133 +88079,90133 +90134,90134 +90135,90135 +90136,90136 +90137,90137 +90138,90138 +90139,90139 +90140,90140 +90141,90141 +90142,90142 +90143,90143 +90144,90144 +90145,90145 +90146,90146 +90147,90147 +90148,90148 +90149,90149 +90150,90150 +90151,90151 +90152,90152 +90153,90153 +90154,90154 +90155,90155 +90156,90156 +90157,90157 +90158,90158 +90159,90159 +90160,90160 +90161,90161 +90162,90162 +202660,90162 +90163,90163 +90164,90164 +90165,90165 +90166,90166 +102181,90166 +90167,90167 +90168,90168 +90169,90169 +90170,90170 +116576,90170 +90171,90171 +90172,90172 +90173,90173 +90174,90174 +90175,90175 +90176,90176 +90177,90177 +90178,90178 +90179,90179 +90180,90180 +90181,90181 +90182,90182 +90183,90183 +90184,90184 +90185,90185 +90186,90186 +90187,90187 +90188,90188 +90189,90189 +90190,90190 +90191,90191 +90192,90192 +90193,90193 +90194,90194 +90195,90195 +90196,90196 +90197,90197 +90198,90198 +90199,90199 +90200,90200 +90250,90250 +115969,90250 +90251,90251 +90252,90252 +110322,90252 +90253,90253 +90254,90254 +90255,90255 +90256,90256 +90257,90257 +90258,90258 +90259,90259 +90260,90260 +90261,90261 +90262,90262 +90263,90263 +90264,90264 +90265,90265 +90266,90266 +90267,90267 +90268,90268 +90269,90269 +90270,90270 +90271,90271 +90272,90272 +90273,90273 +90274,90274 +90275,90275 +90276,90276 +90277,90277 +90278,90278 +90279,90279 +90280,90280 +90281,90281 +90282,90282 +90283,90283 +90284,90284 +90285,90285 +90286,90286 +90287,90287 +90288,90288 +90289,90289 +90416,90416 +90417,90417 +90418,90418 +90419,90419 +161592,90419 +90420,90420 +90421,90421 +90422,90422 +90423,90423 +187405,90423 +90424,90424 +90502,90502 +90503,90503 +90504,90504 +231243,90504 +90505,90505 +90506,90506 +90507,90507 +90546,90546 +7610,90546 +90547,90547 +90548,90548 +90549,90549 +90550,90550 +90551,90551 +233653,90551 +90552,90552 +148157,90552 +90553,90553 +90554,90554 +90555,90555 +90556,90556 +90557,90557 +90558,90558 +90559,90559 +90560,90560 +90561,90561 +90562,90562 +90563,90563 +90564,90564 +90565,90565 +90566,90566 +90567,90567 +90568,90568 +90569,90569 +90570,90570 +90571,90571 +90572,90572 +84562,90572 +90573,90573 +90574,90574 +90575,90575 +90576,90576 +90577,90577 +90578,90578 +90579,90579 +90580,90580 +90581,90581 +90582,90582 +90583,90583 +90584,90584 +90585,90585 +90586,90586 +90587,90587 +90588,90588 +90589,90589 +90590,90590 +90591,90591 +90592,90592 +90593,90593 +90594,90594 +90595,90595 +90596,90596 +90597,90597 +90598,90598 +90599,90599 +90600,90600 +90601,90601 +90602,90602 +90603,90603 +90604,90604 +90605,90605 +90606,90606 +90607,90607 +90608,90608 +90609,90609 +90610,90610 +90611,90611 +90612,90612 +90613,90613 +90614,90614 +90615,90615 +90616,90616 +145369,90616 +90617,90617 +90618,90618 +90619,90619 +90620,90620 +90621,90621 +90622,90622 +90623,90623 +90624,90624 +90625,90625 +90626,90626 +90627,90627 +90628,90628 +90629,90629 +90630,90630 +90631,90631 +90632,90632 +90633,90633 +90634,90634 +90635,90635 +90636,90636 +90637,90637 +161818,90637 +90638,90638 +90639,90639 +90640,90640 +90641,90641 +90642,90642 +90643,90643 +90644,90644 +90645,90645 +90646,90646 +90647,90647 +90648,90648 +90649,90649 +90650,90650 +90651,90651 +122957,90651 +90652,90652 +90653,90653 +90654,90654 +90655,90655 +90656,90656 +90657,90657 +90658,90658 +90659,90659 +160411,90659 +90660,90660 +90661,90661 +90662,90662 +90663,90663 +90664,90664 +90665,90665 +90666,90666 +90667,90667 +90668,90668 +90669,90669 +90670,90670 +90671,90671 +90672,90672 +90673,90673 +90674,90674 +90675,90675 +90676,90676 +90677,90677 +90678,90678 +90679,90679 +90680,90680 +90681,90681 +90682,90682 +90683,90683 +90684,90684 +90685,90685 +90686,90686 +2896,90686 +90687,90687 +90688,90688 +90689,90689 +90690,90690 +90691,90691 +90692,90692 +90693,90693 +164998,90693 +90694,90694 +90695,90695 +90696,90696 +90697,90697 +90698,90698 +90699,90699 +90700,90700 +90701,90701 +90702,90702 +90703,90703 +90704,90704 +90705,90705 +90706,90706 +90707,90707 +90708,90708 +90709,90709 +134169,90710 +126582,90710 +90710,90710 +90711,90711 +140279,90712 +90712,90712 +90713,90713 +90714,90714 +90715,90715 +90716,90716 +90717,90717 +90718,90718 +90719,90719 +90720,90720 +90721,90721 +90722,90722 +90723,90723 +90724,90724 +90725,90725 +90726,90726 +90727,90727 +90728,90728 +90729,90729 +90730,90730 +90731,90731 +90732,90732 +90733,90733 +90734,90734 +90735,90735 +90736,90736 +90737,90737 +90738,90738 +90739,90739 +102356,90739 +90740,90740 +90741,90741 +90742,90742 +90743,90743 +90744,90744 +90745,90745 +90746,90746 +90747,90747 +90748,90748 +90749,90749 +90750,90750 +194719,90750 +90751,90751 +90752,90752 +230397,90752 +90956,90956 +90957,90957 +90958,90958 +160648,90958 +90959,90959 +90960,90960 +90961,90961 +90962,90962 +90963,90963 +90964,90964 +90965,90965 +90966,90966 +183454,90966 +90967,90967 +90968,90968 +90969,90969 +90970,90970 +90971,90971 +90972,90972 +90973,90973 +90974,90974 +201513,90974 +90975,90975 +90976,90976 +90977,90977 +90978,90978 +211167,90978 +90979,90979 +90980,90980 +90981,90981 +90982,90982 +90983,90983 +90984,90984 +90985,90985 +90986,90986 +90987,90987 +90988,90988 +90989,90989 +90990,90990 +90991,90991 +90992,90992 +90993,90993 +90994,90994 +76444,90994 +90995,90995 +23701,90995 +90996,90996 +90997,90997 +90998,90998 +90999,90999 +91000,91000 +91001,91001 +91002,91002 +91003,91003 +21556,91003 +91004,91004 +91005,91005 +91006,91006 +91007,91007 +91008,91008 +114221,91008 +91009,91009 +91010,91010 +91011,91011 +91012,91012 +91013,91013 +91014,91014 +91015,91015 +85832,91015 +91016,91016 +91017,91017 +91018,91018 +91019,91019 +91020,91020 +91021,91021 +91022,91022 +91023,91023 +91024,91024 +91025,91025 +158489,91025 +91026,91026 +150627,91027 +91027,91027 +158491,91027 +91028,91028 +91029,91029 +91030,91030 +91031,91031 +91032,91032 +91033,91033 +91461,91461 +91462,91462 +91463,91463 +225145,91463 +84558,91463 +91464,91464 +240915,91464 +91465,91465 +91466,91466 +141399,91466 +91467,91467 +91468,91468 +91469,91469 +91470,91470 +91471,91471 +234431,91471 +91472,91472 +141406,91473 +91473,91473 +91474,91474 +91475,91475 +91476,91476 +93552,91477 +91477,91477 +91478,91478 +83038,91478 +91479,91479 +91480,91480 +91481,91481 +91482,91482 +91483,91483 +91484,91484 +201087,91485 +91485,91485 +91486,91486 +91487,91487 +91488,91488 +201082,91488 +91489,91489 +91490,91490 +91927,91927 +91928,91928 +91929,91929 +91930,91930 +91931,91931 +91932,91932 +91933,91933 +91934,91934 +91935,91935 +91936,91936 +91937,91937 +91938,91938 +91939,91939 +91940,91940 +91941,91941 +91942,91942 +91943,91943 +91944,91944 +91945,91945 +91946,91946 +91947,91947 +91948,91948 +91949,91949 +91950,91950 +91951,91951 +91952,91952 +91953,91953 +91954,91954 +91955,91955 +91956,91956 +91957,91957 +91958,91958 +91959,91959 +91960,91960 +91961,91961 +91962,91962 +91963,91963 +91964,91964 +91965,91965 +6223,91965 +91966,91966 +91967,91967 +91968,91968 +91969,91969 +91970,91970 +91971,91971 +91972,91972 +91973,91973 +91974,91974 +91975,91975 +91976,91976 +91977,91977 +91978,91978 +207570,91978 +91979,91979 +91980,91980 +91981,91981 +91982,91982 +91983,91983 +91984,91984 +191388,91984 +91985,91985 +91986,91986 +91987,91987 +91988,91988 +31131,91988 +91989,91989 +91990,91990 +91991,91991 +91992,91992 +91993,91993 +91994,91994 +92053,92053 +92054,92054 +10960,92054 +92055,92055 +92056,92056 +92057,92057 +92058,92058 +162875,92058 +92059,92059 +92060,92060 +92061,92061 +92062,92062 +92063,92063 +92064,92064 +92065,92065 +92066,92066 +92067,92067 +92068,92068 +185331,92068 +92069,92069 +92070,92070 +92071,92071 +92072,92072 +92073,92073 +92074,92074 +92075,92075 +92076,92076 +92077,92077 +92078,92078 +92079,92079 +107027,92079 +92080,92080 +92081,92081 +92082,92082 +152228,92082 +92083,92083 +92084,92084 +92085,92085 +92086,92086 +185895,92086 +92087,92087 +92088,92088 +92089,92089 +92090,92090 +92091,92091 +231472,92091 +92092,92092 +19021,92092 +92093,92093 +92094,92094 +92095,92095 +92096,92096 +92097,92097 +92098,92098 +92099,92099 +92100,92100 +92101,92101 +106953,92101 +92102,92102 +92103,92103 +92104,92104 +92105,92105 +92106,92106 +92107,92107 +7564,92107 +92108,92108 +92109,92109 +92110,92110 +92111,92111 +92112,92112 +193260,92112 +92113,92113 +92114,92114 +92115,92115 +92116,92116 +92117,92117 +92118,92118 +92119,92119 +92120,92120 +92121,92121 +166722,92121 +92122,92122 +30342,92122 +92123,92123 +92124,92124 +92125,92125 +107492,92125 +92126,92126 +187210,92126 +92127,92127 +92128,92128 +92129,92129 +92130,92130 +92131,92131 +92132,92132 +92133,92133 +92134,92134 +92446,92446 +79871,92446 +92447,92447 +92448,92448 +92449,92449 +76336,92449 +92450,92450 +92451,92451 +92452,92452 +92453,92453 +92454,92454 +79804,92454 +92501,92501 +92502,92502 +92503,92503 +92504,92504 +92505,92505 +92506,92506 +92507,92507 +92508,92508 +92509,92509 +92510,92510 +92511,92511 +92512,92512 +92513,92513 +92514,92514 +92515,92515 +92516,92516 +92517,92517 +92518,92518 +92519,92519 +92520,92520 +92546,92546 +92547,92547 +92548,92548 +88064,92548 +92549,92549 +144263,92549 +92550,92550 +92551,92551 +92552,92552 +92590,92590 +92591,92591 +92592,92592 +92593,92593 +92594,92594 +92595,92595 +92596,92596 +92597,92597 +92598,92598 +216777,92598 +92599,92599 +206196,92600 +92600,92600 +236822,92600 +206197,92601 +92601,92601 +236823,92601 +92602,92602 +92603,92603 +92604,92604 +92605,92605 +56654,92605 +48550,92605 +92606,92606 +92607,92607 +92608,92608 +231349,92608 +92609,92609 +92610,92610 +92611,92611 +92612,92612 +92613,92613 +92614,92614 +92615,92615 +92616,92616 +92617,92617 +92618,92618 +92619,92619 +92620,92620 +92621,92621 +92622,92622 +92623,92623 +92624,92624 +92625,92625 +92626,92626 +92627,92627 +92628,92628 +92629,92629 +92630,92630 +92631,92631 +92632,92632 +92633,92633 +92634,92634 +224641,92634 +92635,92635 +111147,92635 +104363,92635 +92636,92636 +92637,92637 +96202,92637 +92638,92638 +92639,92639 +92640,92640 +92641,92641 +92642,92642 +92643,92643 +77150,92643 +92644,92644 +92645,92645 +92646,92646 +92647,92647 +92648,92648 +108678,92649 +92649,92649 +10202,92649 +153938,92650 +92650,92650 +92651,92651 +36895,92651 +92652,92652 +92653,92653 +92654,92654 +92655,92655 +92656,92656 +92657,92657 +92658,92658 +76364,92658 +92659,92659 +80443,92659 +105678,92659 +92660,92660 +233399,92660 +92661,92661 +92662,92662 +92663,92663 +92664,92664 +92665,92665 +92666,92666 +3519,92666 +79898,92666 +92667,92667 +92668,92668 +92669,92669 +92670,92670 +92671,92671 +92672,92672 +92673,92673 +137953,92673 +130366,92673 +92674,92674 +92675,92675 +92676,92676 +92677,92677 +22021,92677 +78688,92677 +92678,92678 +92679,92679 +92680,92680 +92681,92681 +92682,92682 +92683,92683 +92684,92684 +92685,92685 +92686,92686 +111672,92686 +92687,92687 +92688,92688 +92689,92689 +92690,92690 +233259,92690 +92691,92691 +92692,92692 +184161,92692 +92693,92693 +92694,92694 +92695,92695 +92696,92696 +92697,92697 +92698,92698 +92699,92699 +92700,92700 +92701,92701 +92702,92702 +92703,92703 +92704,92704 +92705,92705 +92706,92706 +92707,92707 +92708,92708 +92709,92709 +92710,92710 +92711,92711 +92712,92712 +92713,92713 +92714,92714 +53415,92714 +45311,92714 +92715,92715 +92716,92716 +92717,92717 +92718,92718 +111997,92718 +92719,92719 +92720,92720 +92721,92721 +199298,92721 +92722,92722 +92723,92723 +92724,92724 +92764,92764 +92765,92765 +92766,92766 +92767,92767 +92768,92768 +92769,92769 +92770,92770 +92771,92771 +92772,92772 +92773,92773 +92774,92774 +92775,92775 +92776,92776 +92777,92777 +92778,92778 +92779,92779 +92780,92780 +92781,92781 +92782,92782 +92783,92783 +92784,92784 +92785,92785 +92786,92786 +92787,92787 +92788,92788 +92789,92789 +92790,92790 +64723,92790 +92791,92791 +92792,92792 +92793,92793 +92794,92794 +93173,93173 +93174,93174 +93175,93175 +93176,93176 +93177,93177 +187757,93178 +93178,93178 +93179,93179 +93180,93180 +93254,93254 +93255,93255 +93256,93256 +93257,93257 +93258,93258 +191503,93258 +93259,93259 +93260,93260 +93261,93261 +93389,93261 +93262,93262 +74072,93262 +93263,93263 +173927,93263 +167963,93263 +93386,93386 +93387,93387 +93388,93388 +93389,93389 +24401,93389 +93261,93389 +113413,93389 +93390,93390 +93391,93391 +93392,93392 +93393,93393 +93394,93394 +93395,93395 +93396,93396 +93397,93397 +93398,93398 +93399,93399 +93400,93400 +93401,93401 +93402,93402 +2180,93402 +93403,93403 +164966,93404 +93404,93404 +93405,93405 +108300,93405 +93406,93406 +93407,93407 +93408,93408 +93547,93547 +93548,93548 +93549,93549 +93550,93550 +110674,93550 +93551,93551 +93552,93552 +91477,93552 +93613,93613 +109531,93613 +93614,93614 +93820,93820 +93821,93821 +93822,93822 +93823,93823 +93824,93824 +93825,93825 +93826,93826 +93827,93827 +93828,93828 +235955,93828 +93829,93829 +209965,93830 +93830,93830 +93831,93831 +77150,93831 +93832,93832 +93833,93833 +224586,93833 +93834,93834 +93835,93835 +93836,93836 +93837,93837 +93838,93838 +93839,93839 +21517,93839 +93840,93840 +93841,93841 +93842,93842 +93843,93843 +224597,93844 +93844,93844 +93845,93845 +93846,93846 +93847,93847 +93848,93848 +93849,93849 +93850,93850 +83016,93850 +93851,93851 +93852,93852 +93853,93853 +93854,93854 +93855,93855 +93856,93856 +93857,93857 +93858,93858 +93859,93859 +135425,93859 +127838,93859 +93860,93860 +93861,93861 +93862,93862 +11555,93862 +93863,93863 +93864,93864 +93865,93865 +93866,93866 +93867,93867 +93868,93868 +93869,93869 +93870,93870 +93871,93871 +93872,93872 +93873,93873 +93874,93874 +51469,93874 +43365,93874 +93875,93875 +93876,93876 +93877,93877 +93937,93937 +93938,93938 +93939,93939 +253,93939 +93940,93940 +93941,93941 +93977,93977 +93978,93978 +93979,93979 +93980,93980 +93981,93981 +93982,93982 +93983,93983 +93984,93984 +93985,93985 +93986,93986 +93987,93987 +93988,93988 +93989,93989 +93990,93990 +93991,93991 +93992,93992 +93993,93993 +93994,93994 +93995,93995 +93996,93996 +202803,93996 +93997,93997 +93998,93998 +93999,93999 +94000,94000 +94001,94001 +94002,94002 +94003,94003 +94004,94004 +94005,94005 +94006,94006 +94007,94007 +94008,94008 +94009,94009 +94010,94010 +94011,94011 +94012,94012 +94013,94013 +94014,94014 +94015,94015 +94016,94016 +94017,94017 +94060,94060 +94061,94061 +180097,94061 +94062,94062 +94063,94063 +94064,94064 +94065,94065 +94066,94066 +94067,94067 +94068,94068 +94069,94069 +94070,94070 +94071,94071 +94072,94072 +94073,94073 +94074,94074 +94075,94075 +94076,94076 +94077,94077 +94078,94078 +94079,94079 +94080,94080 +246370,94080 +94081,94081 +94082,94082 +162871,94082 +94083,94083 +94084,94084 +94085,94085 +94086,94086 +94087,94087 +94088,94088 +94089,94089 +94090,94090 +94091,94091 +94092,94092 +94093,94093 +94094,94094 +94095,94095 +94096,94096 +94097,94097 +94098,94098 +94099,94099 +94100,94100 +94101,94101 +94102,94102 +94103,94103 +94104,94104 +94105,94105 +94478,94478 +94479,94479 +94480,94480 +94481,94481 +94482,94482 +94483,94483 +94484,94484 +94485,94485 +94486,94486 +78025,94486 +94487,94487 +55107,94487 +47003,94487 +94488,94488 +94489,94489 +94490,94490 +20548,94490 +94491,94491 +94492,94492 +94493,94493 +94494,94494 +94495,94495 +94496,94496 +94497,94497 +100564,94497 +94498,94498 +94499,94499 +100566,94499 +94500,94500 +94501,94501 +25510,94501 +94502,94502 +94940,94940 +94941,94941 +94942,94942 +94943,94943 +94944,94944 +94945,94945 +94946,94946 +94947,94947 +94948,94948 +94949,94949 +26291,94949 +94950,94950 +94951,94951 +113426,94951 +94952,94952 +94953,94953 +70490,94953 +94954,94954 +94955,94955 +94956,94956 +144248,94956 +94957,94957 +94958,94958 +94959,94959 +94960,94960 +94961,94961 +94962,94962 +94963,94963 +94964,94964 +94965,94965 +94966,94966 +94967,94967 +94968,94968 +94969,94969 +94970,94970 +94971,94971 +94972,94972 +94973,94973 +94974,94974 +94975,94975 +94976,94976 +94977,94977 +94978,94978 +94979,94979 +94980,94980 +94981,94981 +94982,94982 +94983,94983 +94984,94984 +94985,94985 +94986,94986 +94987,94987 +233790,94987 +208074,94987 +94988,94988 +94989,94989 +94990,94990 +94991,94991 +39952,94991 +94992,94992 +94993,94993 +94994,94994 +94995,94995 +94996,94996 +94997,94997 +94998,94998 +94999,94999 +95000,95000 +95001,95001 +95002,95002 +95003,95003 +233453,95003 +206673,95003 +111280,95003 +95004,95004 +95005,95005 +95041,95041 +95042,95042 +95043,95043 +95044,95044 +95045,95045 +95046,95046 +95047,95047 +95048,95048 +95049,95049 +95050,95050 +95051,95051 +190575,95051 +95052,95052 +114782,95052 +95053,95053 +95054,95054 +95055,95055 +95056,95056 +95057,95057 +95058,95058 +95059,95059 +95060,95060 +95061,95061 +95062,95062 +95063,95063 +95064,95064 +95065,95065 +95066,95066 +95067,95067 +95068,95068 +95069,95069 +95070,95070 +95155,95155 +95156,95156 +158040,95156 +95157,95157 +95158,95158 +95159,95159 +190980,95159 +95160,95160 +95161,95161 +95162,95162 +95163,95163 +191555,95163 +95164,95164 +95165,95165 +95166,95166 +95167,95167 +95168,95168 +204478,95168 +95881,95881 +195148,95881 +208480,95881 +95882,95882 +95883,95883 +95884,95884 +95885,95885 +95886,95886 +95887,95887 +95888,95888 +95889,95889 +186419,95890 +95890,95890 +95891,95891 +158977,95891 +95892,95892 +95893,95893 +95894,95894 +95895,95895 +95896,95896 +95897,95897 +95898,95898 +95899,95899 +95900,95900 +95901,95901 +97978,95902 +95902,95902 +95903,95903 +95904,95904 +95905,95905 +95906,95906 +95907,95907 +95908,95908 +95909,95909 +95910,95910 +95911,95911 +95912,95912 +95913,95913 +95914,95914 +95915,95915 +95916,95916 +95917,95917 +95918,95918 +95919,95919 +95920,95920 +95921,95921 +95922,95922 +95923,95923 +95924,95924 +30701,95924 +95925,95925 +95926,95926 +95927,95927 +95928,95928 +95929,95929 +96014,96014 +96015,96015 +96016,96016 +96017,96017 +96018,96018 +96019,96019 +96020,96020 +96021,96021 +96022,96022 +96023,96023 +96024,96024 +96025,96025 +96026,96026 +96027,96027 +174299,96028 +168335,96028 +96028,96028 +96029,96029 +96030,96030 +84268,96030 +96031,96031 +96032,96032 +96033,96033 +96034,96034 +178752,96035 +96035,96035 +96036,96036 +96037,96037 +2908,96037 +96038,96038 +22044,96038 +96039,96039 +96040,96040 +96041,96041 +96042,96042 +96043,96043 +96044,96044 +96045,96045 +96046,96046 +96047,96047 +96048,96048 +96049,96049 +96050,96050 +96051,96051 +96052,96052 +96053,96053 +96054,96054 +96055,96055 +96095,96095 +96096,96096 +96097,96097 +96098,96098 +56281,96098 +48177,96098 +96099,96099 +96100,96100 +96101,96101 +96102,96102 +96195,96195 +96196,96196 +96197,96197 +96198,96198 +96199,96199 +96200,96200 +96201,96201 +96202,96202 +92637,96202 +96203,96203 +96204,96204 +96205,96205 +96206,96206 +111895,96206 +96207,96207 +96208,96208 +96209,96209 +96210,96210 +96332,96332 +96333,96333 +96334,96334 +96335,96335 +96336,96336 +96337,96337 +96338,96338 +96339,96339 +96340,96340 +36360,96340 +96341,96341 +96342,96342 +96343,96343 +96344,96344 +96345,96345 +96346,96346 +96347,96347 +96348,96348 +96349,96349 +96350,96350 +96351,96351 +96352,96352 +96353,96353 +96354,96354 +240895,96354 +96355,96355 +96356,96356 +96357,96357 +96358,96358 +96359,96359 +96360,96360 +96361,96361 +96362,96362 +96363,96363 +96364,96364 +96365,96365 +96366,96366 +96367,96367 +96368,96368 +96369,96369 +96370,96370 +99362,96371 +96371,96371 +96372,96372 +96373,96373 +187207,96373 +96374,96374 +96375,96375 +96376,96376 +5292,96376 +96377,96377 +96378,96378 +96379,96379 +96380,96380 +96381,96381 +96382,96382 +96383,96383 +233599,96383 +96423,96423 +96424,96424 +204050,96424 +96425,96425 +96426,96426 +96427,96427 +96428,96428 +96429,96429 +96430,96430 +96431,96431 +96432,96432 +96433,96433 +96434,96434 +96435,96435 +96436,96436 +96437,96437 +96438,96438 +96439,96439 +96440,96440 +96441,96441 +96442,96442 +96443,96443 +86433,96443 +33801,96443 +96444,96444 +96445,96445 +214558,96445 +96446,96446 +96447,96447 +96448,96448 +96449,96449 +96450,96450 +96451,96451 +96452,96452 +96453,96453 +3722,96453 +96454,96454 +96455,96455 +96456,96456 +96457,96457 +96458,96458 +96459,96459 +96460,96460 +96461,96461 +96462,96462 +96463,96463 +109894,96463 +96464,96464 +96465,96465 +96466,96466 +96467,96467 +96468,96468 +36537,96468 +96469,96469 +96470,96470 +96471,96471 +176284,96471 +170320,96471 +96472,96472 +178819,96472 +96473,96473 +96474,96474 +86355,96474 +96475,96475 +108585,96475 +96476,96476 +96477,96477 +96478,96478 +96479,96479 +96480,96480 +96481,96481 +96482,96482 +96483,96483 +96484,96484 +96485,96485 +159370,96485 +96486,96486 +96671,96671 +96672,96672 +96673,96673 +96674,96674 +96675,96675 +96676,96676 +20146,96676 +96677,96677 +96678,96678 +96679,96679 +96680,96680 +96681,96681 +218453,96681 +97149,97149 +11373,97149 +232546,97149 +97150,97150 +97151,97151 +97152,97152 +97153,97153 +97154,97154 +97155,97155 +97156,97156 +74070,97156 +97157,97157 +97158,97158 +97159,97159 +97160,97160 +97161,97161 +97162,97162 +97163,97163 +97164,97164 +97165,97165 +97166,97166 +97206,97206 +97207,97207 +97208,97208 +97209,97209 +216988,97209 +97210,97210 +97393,97393 +97394,97394 +97395,97395 +97396,97396 +97397,97397 +151626,97397 +97398,97398 +97399,97399 +97400,97400 +97401,97401 +97402,97402 +97403,97403 +97404,97404 +97405,97405 +97406,97406 +97407,97407 +97408,97408 +233429,97408 +97409,97409 +97410,97410 +97411,97411 +97412,97412 +97413,97413 +97414,97414 +97415,97415 +97416,97416 +97417,97417 +97418,97418 +97419,97419 +97420,97420 +97421,97421 +97422,97422 +97423,97423 +97424,97424 +97425,97425 +97426,97426 +97427,97427 +97428,97428 +97429,97429 +6598,97429 +97430,97430 +97431,97431 +97432,97432 +216,97432 +97433,97433 +13676,97433 +4497,97433 +97434,97434 +97435,97435 +97436,97436 +97476,97476 +97477,97477 +97478,97478 +97479,97479 +162381,97480 +97480,97480 +97481,97481 +97482,97482 +97483,97483 +97484,97484 +97485,97485 +97486,97486 +97487,97487 +97488,97488 +97489,97489 +97490,97490 +97491,97491 +88134,97491 +68639,97491 +97492,97492 +97493,97493 +97494,97494 +97495,97495 +97496,97496 +97497,97497 +97498,97498 +97499,97499 +97500,97500 +97501,97501 +141847,97501 +97502,97502 +97503,97503 +97504,97504 +97646,97646 +97647,97647 +97648,97648 +97649,97649 +97650,97650 +202656,97650 +97651,97651 +97652,97652 +22658,97652 +97653,97653 +97654,97654 +97655,97655 +97656,97656 +97657,97657 +97658,97658 +97659,97659 +97660,97660 +97661,97661 +97662,97662 +97663,97663 +97664,97664 +97665,97665 +97666,97666 +97667,97667 +97668,97668 +97669,97669 +97670,97670 +97671,97671 +97672,97672 +97673,97673 +97674,97674 +97675,97675 +97676,97676 +97677,97677 +97678,97678 +97679,97679 +97680,97680 +97681,97681 +97682,97682 +97683,97683 +97684,97684 +97685,97685 +97686,97686 +97687,97687 +97688,97688 +97689,97689 +97690,97690 +97691,97691 +97692,97692 +97693,97693 +97694,97694 +97695,97695 +103536,97695 +97696,97696 +97697,97697 +97698,97698 +97699,97699 +97700,97700 +97701,97701 +97702,97702 +97703,97703 +97704,97704 +97705,97705 +97706,97706 +97707,97707 +97708,97708 +80899,97708 +97709,97709 +97710,97710 +97711,97711 +97712,97712 +97713,97713 +56675,97713 +48571,97713 +15335,97713 +97714,97714 +97715,97715 +97716,97716 +97717,97717 +97718,97718 +97719,97719 +97720,97720 +97721,97721 +97722,97722 +234279,97722 +97723,97723 +77709,97723 +97724,97724 +97725,97725 +97726,97726 +97727,97727 +97728,97728 +97729,97729 +97730,97730 +97731,97731 +97732,97732 +97733,97733 +97734,97734 +97735,97735 +97736,97736 +97737,97737 +97738,97738 +97739,97739 +133038,97739 +125451,97739 +97740,97740 +97741,97741 +97742,97742 +97743,97743 +97744,97744 +97745,97745 +97746,97746 +97747,97747 +97748,97748 +216756,97748 +97749,97749 +97750,97750 +97751,97751 +97752,97752 +97753,97753 +97754,97754 +97755,97755 +97756,97756 +97757,97757 +97758,97758 +97855,97855 +227727,97855 +97856,97856 +97857,97857 +97858,97858 +97859,97859 +97860,97860 +97861,97861 +97862,97862 +120386,97862 +97863,97863 +97864,97864 +97865,97865 +97866,97866 +97867,97867 +97868,97868 +97869,97869 +97870,97870 +97871,97871 +97872,97872 +164875,97872 +97873,97873 +97874,97874 +97875,97875 +97876,97876 +97877,97877 +97878,97878 +97879,97879 +97880,97880 +97881,97881 +97882,97882 +97883,97883 +97884,97884 +97885,97885 +97886,97886 +97887,97887 +97888,97888 +97889,97889 +97890,97890 +97891,97891 +156445,97891 +97892,97892 +70331,97892 +97893,97893 +97894,97894 +97895,97895 +97896,97896 +97897,97897 +97898,97898 +97899,97899 +97900,97900 +97901,97901 +97902,97902 +136584,97903 +128997,97903 +97903,97903 +97904,97904 +97905,97905 +97906,97906 +97907,97907 +97908,97908 +97909,97909 +97910,97910 +97911,97911 +97912,97912 +97913,97913 +81123,97913 +97914,97914 +97915,97915 +60351,97915 +97916,97916 +97917,97917 +97918,97918 +97919,97919 +97920,97920 +97921,97921 +97922,97922 +97923,97923 +97924,97924 +97925,97925 +97926,97926 +97927,97927 +97928,97928 +97929,97929 +97930,97930 +97931,97931 +97932,97932 +97933,97933 +97934,97934 +97935,97935 +97936,97936 +97937,97937 +108667,97937 +97938,97938 +97939,97939 +97940,97940 +97941,97941 +97942,97942 +97943,97943 +97944,97944 +97945,97945 +97946,97946 +97947,97947 +159372,97948 +97948,97948 +97949,97949 +97950,97950 +97951,97951 +97952,97952 +97953,97953 +97954,97954 +97955,97955 +97956,97956 +97957,97957 +97958,97958 +97959,97959 +108447,97959 +50747,97959 +42643,97959 +97960,97960 +111395,97960 +97961,97961 +211776,97961 +97962,97962 +97963,97963 +97964,97964 +180359,97964 +97965,97965 +75308,97965 +97966,97966 +97967,97967 +97968,97968 +97969,97969 +97970,97970 +75205,97970 +97971,97971 +97972,97972 +97973,97973 +97974,97974 +97975,97975 +97976,97976 +97977,97977 +97978,97978 +95902,97978 +97979,97979 +97980,97980 +97981,97981 +97982,97982 +242655,97982 +97983,97983 +97984,97984 +97985,97985 +97986,97986 +97987,97987 +97988,97988 +97989,97989 +97990,97990 +97991,97991 +97992,97992 +97993,97993 +97994,97994 +97995,97995 +97996,97996 +97997,97997 +97998,97998 +97999,97999 +98000,98000 +98001,98001 +98002,98002 +98003,98003 +98004,98004 +98005,98005 +98006,98006 +98007,98007 +98008,98008 +98009,98009 +98010,98010 +98011,98011 +98012,98012 +98013,98013 +98014,98014 +98015,98015 +98016,98016 +98017,98017 +98018,98018 +98019,98019 +98020,98020 +98021,98021 +98022,98022 +98023,98023 +98024,98024 +98025,98025 +98026,98026 +98027,98027 +98028,98028 +98029,98029 +98030,98030 +98031,98031 +98032,98032 +98033,98033 +98034,98034 +98035,98035 +98036,98036 +98037,98037 +98038,98038 +139997,98038 +108446,98038 +98039,98039 +98040,98040 +166432,98040 +98041,98041 +108598,98041 +98042,98042 +98043,98043 +98044,98044 +98045,98045 +98046,98046 +98047,98047 +98048,98048 +98049,98049 +98050,98050 +98051,98051 +98052,98052 +98053,98053 +178867,98053 +98054,98054 +98055,98055 +98056,98056 +98057,98057 +80635,98057 +98058,98058 +98059,98059 +98060,98060 +98061,98061 +98062,98062 +98063,98063 +214487,98063 +98064,98064 +98065,98065 +98066,98066 +98067,98067 +98068,98068 +98069,98069 +98070,98070 +98071,98071 +98072,98072 +98073,98073 +98074,98074 +98075,98075 +98076,98076 +98077,98077 +98078,98078 +98079,98079 +98080,98080 +78605,98080 +98081,98081 +98082,98082 +98083,98083 +98084,98084 +98085,98085 +98086,98086 +98087,98087 +140006,98087 +98088,98088 +98089,98089 +98090,98090 +98091,98091 +98092,98092 +98093,98093 +243144,98093 +98094,98094 +98095,98095 +98096,98096 +98097,98097 +98098,98098 +78052,98098 +98099,98099 +98100,98100 +98101,98101 +68301,98101 +98102,98102 +98103,98103 +98104,98104 +98105,98105 +98106,98106 +98107,98107 +98108,98108 +98109,98109 +98110,98110 +98111,98111 +98112,98112 +98113,98113 +98114,98114 +98115,98115 +98116,98116 +98117,98117 +98118,98118 +98119,98119 +206596,98119 +98120,98120 +98121,98121 +98122,98122 +76393,98122 +98123,98123 +98124,98124 +98125,98125 +98126,98126 +98127,98127 +98128,98128 +23550,98128 +98129,98129 +98130,98130 +98131,98131 +98132,98132 +98133,98133 +98134,98134 +98135,98135 +98136,98136 +98137,98137 +98138,98138 +98139,98139 +98140,98140 +98141,98141 +194949,98141 +98142,98142 +98143,98143 +98144,98144 +98145,98145 +98146,98146 +98147,98147 +98148,98148 +98149,98149 +98150,98150 +98151,98151 +98152,98152 +98153,98153 +98154,98154 +98155,98155 +98156,98156 +98157,98157 +98158,98158 +98159,98159 +98160,98160 +98161,98161 +98162,98162 +98163,98163 +98164,98164 +98165,98165 +98166,98166 +98167,98167 +98206,98206 +98207,98207 +98208,98208 +98209,98209 +98210,98210 +58402,98210 +98211,98211 +98212,98212 +98213,98213 +98214,98214 +98215,98215 +98216,98216 +98217,98217 +139951,98217 +98218,98218 +98219,98219 +98220,98220 +98221,98221 +24909,98221 +98222,98222 +98223,98223 +98224,98224 +98225,98225 +98226,98226 +98227,98227 +183832,98227 +98228,98228 +98229,98229 +98230,98230 +98231,98231 +98232,98232 +98233,98233 +158229,98233 +98234,98234 +98235,98235 +98236,98236 +98237,98237 +98238,98238 +98239,98239 +132500,98239 +124913,98239 +98240,98240 +98241,98241 +98242,98242 +98243,98243 +98244,98244 +98245,98245 +98246,98246 +233305,98246 +98247,98247 +98318,98318 +98319,98319 +98320,98320 +140671,98320 +98321,98321 +98322,98322 +98323,98323 +119248,98323 +98324,98324 +98325,98325 +98326,98326 +98327,98327 +98328,98328 +98329,98329 +98330,98330 +98331,98331 +98332,98332 +98333,98333 +98334,98334 +98335,98335 +110077,98335 +98336,98336 +9255,98336 +98337,98337 +190855,98337 +98338,98338 +98339,98339 +98340,98340 +98341,98341 +98342,98342 +98343,98343 +98344,98344 +98751,98751 +98752,98752 +98753,98753 +98754,98754 +98755,98755 +98756,98756 +98757,98757 +98758,98758 +98759,98759 +98760,98760 +98761,98761 +98762,98762 +98763,98763 +98764,98764 +98765,98765 +98812,98812 +158787,98813 +98813,98813 +98814,98814 +98815,98815 +98816,98816 +98817,98817 +98818,98818 +108592,98818 +99215,99215 +99216,99216 +99217,99217 +99218,99218 +3967,99218 +99219,99219 +99220,99220 +99221,99221 +99222,99222 +99223,99223 +56670,99223 +48566,99223 +99224,99224 +99225,99225 +99226,99226 +99227,99227 +99228,99228 +5314,99228 +99229,99229 +99230,99230 +99231,99231 +99232,99232 +99233,99233 +99234,99234 +99235,99235 +99236,99236 +99237,99237 +99238,99238 +99239,99239 +99240,99240 +99241,99241 +99242,99242 +99243,99243 +27737,99243 +99244,99244 +99245,99245 +99246,99246 +99247,99247 +99248,99248 +99249,99249 +99250,99250 +99251,99251 +99252,99252 +221695,99252 +99253,99253 +99254,99254 +99255,99255 +99256,99256 +99257,99257 +99258,99258 +99259,99259 +99260,99260 +99261,99261 +99262,99262 +99263,99263 +99264,99264 +99265,99265 +99266,99266 +99267,99267 +99268,99268 +99269,99269 +99270,99270 +99271,99271 +99272,99272 +99273,99273 +99274,99274 +99275,99275 +99276,99276 +99277,99277 +99278,99278 +99279,99279 +99280,99280 +99281,99281 +99282,99282 +99283,99283 +99284,99284 +99285,99285 +99286,99286 +99287,99287 +99288,99288 +202412,99289 +99289,99289 +99290,99290 +99291,99291 +99292,99292 +99293,99293 +99294,99294 +99295,99295 +99296,99296 +99297,99297 +99298,99298 +99299,99299 +99300,99300 +99301,99301 +99302,99302 +99303,99303 +99304,99304 +99305,99305 +99306,99306 +99307,99307 +99308,99308 +99309,99309 +99310,99310 +99311,99311 +99312,99312 +31196,99312 +99313,99313 +99314,99314 +99315,99315 +99316,99316 +99317,99317 +99318,99318 +99319,99319 +77109,99319 +99320,99320 +194330,99320 +99321,99321 +99322,99322 +79855,99322 +99323,99323 +99324,99324 +99325,99325 +99326,99326 +99327,99327 +99328,99328 +99329,99329 +99330,99330 +99331,99331 +99332,99332 +99333,99333 +99334,99334 +99335,99335 +99336,99336 +99337,99337 +99338,99338 +99339,99339 +99340,99340 +99341,99341 +99342,99342 +99343,99343 +99344,99344 +99345,99345 +99346,99346 +99347,99347 +99348,99348 +99349,99349 +99350,99350 +99351,99351 +99352,99352 +99353,99353 +99354,99354 +99355,99355 +99356,99356 +99357,99357 +205188,99357 +99358,99358 +99359,99359 +99360,99360 +99361,99361 +99362,99362 +96371,99362 +99363,99363 +99364,99364 +99365,99365 +99366,99366 +99367,99367 +99368,99368 +99369,99369 +99370,99370 +99371,99371 +99372,99372 +99373,99373 +99374,99374 +99375,99375 +99376,99376 +99377,99377 +99378,99378 +99379,99379 +99380,99380 +99381,99381 +99382,99382 +99383,99383 +99384,99384 +99385,99385 +99386,99386 +99387,99387 +99388,99388 +99389,99389 +99390,99390 +99391,99391 +181077,99391 +99392,99392 +99393,99393 +99394,99394 +99395,99395 +99559,99559 +99560,99560 +99561,99561 +99562,99562 +99563,99563 +99564,99564 +99565,99565 +99566,99566 +99567,99567 +99568,99568 +99569,99569 +99570,99570 +99571,99571 +99572,99572 +99573,99573 +99574,99574 +99575,99575 +99576,99576 +99577,99577 +99578,99578 +99579,99579 +195568,99579 +99580,99580 +99581,99581 +99582,99582 +99672,99672 +99673,99673 +99674,99674 +76298,99674 +99675,99675 +99676,99676 +99677,99677 +99678,99678 +99679,99679 +99680,99680 +100551,100551 +100552,100552 +60059,100552 +100553,100553 +100554,100554 +32302,100554 +100555,100555 +100556,100556 +100557,100557 +100558,100558 +100559,100559 +100560,100560 +100561,100561 +100562,100562 +100563,100563 +100564,100564 +94497,100564 +100565,100565 +100566,100566 +94499,100566 +100567,100567 +28945,100567 +61783,100567 +100568,100568 +100569,100569 +112248,100569 +100570,100570 +100571,100571 +100572,100572 +100573,100573 +100574,100574 +100575,100575 +17233,100575 +100576,100576 +100577,100577 +159521,100578 +100578,100578 +100579,100579 +100580,100580 +100581,100581 +100582,100582 +80843,100582 +100583,100583 +159390,100584 +100584,100584 +100585,100585 +100586,100586 +100587,100587 +100588,100588 +100589,100589 +40250,100589 +100590,100590 +174338,100590 +168374,100590 +185361,100590 +100591,100591 +100592,100592 +100593,100593 +223014,100593 +100594,100594 +100595,100595 +100596,100596 +100597,100597 +28950,100597 +100598,100598 +100599,100599 +100600,100600 +140032,100600 +100601,100601 +100602,100602 +100603,100603 +111794,100603 +100604,100604 +100605,100605 +100606,100606 +100607,100607 +100608,100608 +100609,100609 +100610,100610 +100611,100611 +100612,100612 +100613,100613 +100614,100614 +100615,100615 +100616,100616 +100617,100617 +100618,100618 +100619,100619 +100620,100620 +100621,100621 +100622,100622 +100623,100623 +100624,100624 +100625,100625 +100626,100626 +100627,100627 +100628,100628 +100629,100629 +100630,100630 +104348,100631 +100631,100631 +180998,100631 +100632,100632 +100633,100633 +104350,100633 +100634,100634 +68182,100634 +100635,100635 +100636,100636 +100637,100637 +100638,100638 +100639,100639 +100640,100640 +100641,100641 +100642,100642 +100643,100643 +18927,100643 +100644,100644 +100645,100645 +202819,100645 +100646,100646 +28754,100646 +100647,100647 +100648,100648 +100649,100649 +100650,100650 +100651,100651 +100652,100652 +100653,100653 +100654,100654 +100655,100655 +100656,100656 +195184,100656 +100657,100657 +100658,100658 +141874,100658 +100659,100659 +104672,100659 +100660,100660 +100661,100661 +100662,100662 +100663,100663 +100664,100664 +54673,100664 +46569,100664 +100665,100665 +100666,100666 +100667,100667 +100668,100668 +100669,100669 +100670,100670 +100671,100671 +64411,100671 +100672,100672 +100673,100673 +100674,100674 +79014,100674 +100675,100675 +100676,100676 +100677,100677 +100678,100678 +100679,100679 +203547,100680 +100680,100680 +241775,100680 +100681,100681 +100682,100682 +100683,100683 +100684,100684 +100685,100685 +100686,100686 +100687,100687 +100688,100688 +100689,100689 +100690,100690 +100691,100691 +100692,100692 +100693,100693 +100694,100694 +100695,100695 +100696,100696 +100697,100697 +100698,100698 +100699,100699 +100700,100700 +147534,100700 +100701,100701 +100702,100702 +100703,100703 +100704,100704 +102134,100704 +100705,100705 +100706,100706 +100707,100707 +100708,100708 +100709,100709 +100710,100710 +100711,100711 +100712,100712 +100713,100713 +100714,100714 +100715,100715 +100716,100716 +100717,100717 +100718,100718 +100719,100719 +100720,100720 +100721,100721 +100722,100722 +100723,100723 +100724,100724 +100725,100725 +80554,100725 +100726,100726 +100727,100727 +100728,100728 +100729,100729 +100730,100730 +100731,100731 +100732,100732 +100733,100733 +100734,100734 +100735,100735 +100736,100736 +100737,100737 +100738,100738 +100739,100739 +100740,100740 +100741,100741 +100742,100742 +100743,100743 +100744,100744 +179968,100744 +100745,100745 +100746,100746 +100747,100747 +100748,100748 +100749,100749 +100750,100750 +100751,100751 +100752,100752 +100753,100753 +100754,100754 +100755,100755 +100756,100756 +114483,100756 +100796,100796 +100797,100797 +100798,100798 +100799,100799 +100800,100800 +100801,100801 +100802,100802 +100803,100803 +100804,100804 +100805,100805 +100806,100806 +100807,100807 +100808,100808 +100809,100809 +100810,100810 +100811,100811 +100812,100812 +100813,100813 +100814,100814 +100815,100815 +100816,100816 +100817,100817 +100818,100818 +100819,100819 +100820,100820 +100821,100821 +100822,100822 +100823,100823 +100824,100824 +100825,100825 +100826,100826 +218387,100826 +100827,100827 +100828,100828 +100829,100829 +100830,100830 +100831,100831 +100832,100832 +100833,100833 +100834,100834 +100835,100835 +100836,100836 +100837,100837 +100838,100838 +100839,100839 +100840,100840 +100841,100841 +100842,100842 +100843,100843 +100844,100844 +222044,100844 +100845,100845 +100846,100846 +100847,100847 +100848,100848 +100849,100849 +100850,100850 +100851,100851 +100852,100852 +100853,100853 +100854,100854 +100855,100855 +100856,100856 +100857,100857 +100858,100858 +100859,100859 +185410,100859 +100860,100860 +100861,100861 +100862,100862 +100863,100863 +100914,100914 +100915,100915 +100916,100916 +100917,100917 +100918,100918 +100919,100919 +100920,100920 +100921,100921 +100922,100922 +100923,100923 +100924,100924 +100925,100925 +100926,100926 +100927,100927 +100928,100928 +100929,100929 +72589,100929 +100930,100930 +100931,100931 +100932,100932 +100933,100933 +100934,100934 +216654,100934 +111279,100934 +206560,100935 +100935,100935 +100936,100936 +100937,100937 +100938,100938 +101023,101023 +101024,101024 +101025,101025 +101026,101026 +101027,101027 +101028,101028 +101029,101029 +33412,101029 +101030,101030 +101031,101031 +55576,101031 +47472,101031 +59642,101031 +101032,101032 +101033,101033 +101034,101034 +101035,101035 +101036,101036 +101037,101037 +165223,101037 +101038,101038 +101039,101039 +160441,101039 +101040,101040 +101041,101041 +101042,101042 +101043,101043 +101044,101044 +101045,101045 +101046,101046 +65582,101046 +101047,101047 +101048,101048 +101049,101049 +101050,101050 +101051,101051 +101052,101052 +102385,101052 +101053,101053 +101151,101151 +101152,101152 +101153,101153 +101154,101154 +101155,101155 +101156,101156 +101157,101157 +101158,101158 +101159,101159 +101160,101160 +101161,101161 +101162,101162 +101163,101163 +101225,101225 +101226,101226 +101227,101227 +101228,101228 +101229,101229 +101230,101230 +101400,101400 +101401,101401 +103884,101401 +101402,101402 +101403,101403 +101404,101404 +101405,101405 +101563,101563 +101564,101564 +101565,101565 +101566,101566 +101567,101567 +101715,101715 +101716,101716 +101717,101717 +101718,101718 +101719,101719 +101720,101720 +101721,101721 +101722,101722 +101723,101723 +101724,101724 +101725,101725 +101726,101726 +101727,101727 +101728,101728 +101729,101729 +101730,101730 +101731,101731 +101732,101732 +101733,101733 +101734,101734 +101735,101735 +101736,101736 +101737,101737 +101738,101738 +101739,101739 +101740,101740 +101741,101741 +101742,101742 +101743,101743 +101744,101744 +101745,101745 +101746,101746 +101747,101747 +101748,101748 +101749,101749 +101750,101750 +101751,101751 +101752,101752 +101753,101753 +101754,101754 +101755,101755 +157137,101755 +101756,101756 +101757,101757 +101758,101758 +101759,101759 +101760,101760 +101761,101761 +101762,101762 +101763,101763 +101764,101764 +110595,101764 +101765,101765 +163397,101766 +101766,101766 +101767,101767 +201772,101767 +101768,101768 +101769,101769 +101828,101828 +101829,101829 +101830,101830 +101831,101831 +101832,101832 +101833,101833 +101834,101834 +101835,101835 +101836,101836 +101837,101837 +101838,101838 +101839,101839 +101840,101840 +226646,101840 +101841,101841 +101842,101842 +177713,101842 +171749,101842 +101843,101843 +101844,101844 +101845,101845 +101846,101846 +101847,101847 +101848,101848 +101849,101849 +101850,101850 +101851,101851 +101852,101852 +101853,101853 +101854,101854 +74696,101854 +101855,101855 +101856,101856 +101857,101857 +101858,101858 +101859,101859 +101860,101860 +101861,101861 +101862,101862 +101863,101863 +101864,101864 +101865,101865 +101866,101866 +196698,101866 +101867,101867 +101868,101868 +101869,101869 +101870,101870 +101871,101871 +101872,101872 +101873,101873 +101874,101874 +101875,101875 +101876,101876 +101877,101877 +101878,101878 +101879,101879 +101880,101880 +101881,101881 +101882,101882 +101883,101883 +38198,101883 +37323,101883 +34694,101883 +7172,101883 +101884,101884 +101885,101885 +101886,101886 +101887,101887 +101888,101888 +101889,101889 +101890,101890 +101891,101891 +101892,101892 +101893,101893 +101894,101894 +101895,101895 +101896,101896 +101897,101897 +101898,101898 +101899,101899 +101900,101900 +101901,101901 +101902,101902 +101903,101903 +101904,101904 +101905,101905 +101906,101906 +101907,101907 +101908,101908 +101909,101909 +101910,101910 +101911,101911 +101912,101912 +101913,101913 +101914,101914 +101915,101915 +101916,101916 +101917,101917 +101918,101918 +101919,101919 +106200,101919 +101920,101920 +101921,101921 +101922,101922 +101923,101923 +101924,101924 +101925,101925 +101926,101926 +101927,101927 +101928,101928 +101929,101929 +101930,101930 +101931,101931 +101932,101932 +101933,101933 +101934,101934 +101935,101935 +101936,101936 +101937,101937 +101938,101938 +101939,101939 +101940,101940 +101941,101941 +101942,101942 +101943,101943 +101944,101944 +101945,101945 +161795,101945 +101946,101946 +101947,101947 +101948,101948 +101949,101949 +101950,101950 +101951,101951 +101952,101952 +110940,101952 +101953,101953 +101954,101954 +101955,101955 +101956,101956 +101957,101957 +101958,101958 +101959,101959 +101960,101960 +101961,101961 +101962,101962 +101963,101963 +101964,101964 +101965,101965 +101966,101966 +101967,101967 +102265,101967 +101968,101968 +101969,101969 +101970,101970 +101971,101971 +101972,101972 +101973,101973 +101974,101974 +101975,101975 +101976,101976 +101977,101977 +101978,101978 +101979,101979 +101980,101980 +101981,101981 +108375,101981 +101982,101982 +101983,101983 +101984,101984 +101985,101985 +101986,101986 +101987,101987 +101988,101988 +194747,101988 +101989,101989 +101990,101990 +101991,101991 +101992,101992 +101993,101993 +101994,101994 +101995,101995 +101996,101996 +226541,101996 +101997,101997 +101998,101998 +101999,101999 +102000,102000 +102001,102001 +102002,102002 +102003,102003 +102004,102004 +102005,102005 +102006,102006 +102007,102007 +102008,102008 +102009,102009 +102010,102010 +102011,102011 +102012,102012 +102013,102013 +84020,102013 +102014,102014 +102015,102015 +102016,102016 +102017,102017 +102018,102018 +102019,102019 +102020,102020 +162347,102020 +102021,102021 +102022,102022 +102023,102023 +102024,102024 +102025,102025 +102026,102026 +102027,102027 +102028,102028 +102029,102029 +102030,102030 +102031,102031 +102032,102032 +102033,102033 +102034,102034 +102035,102035 +102036,102036 +102037,102037 +102038,102038 +102039,102039 +102040,102040 +102041,102041 +102042,102042 +102043,102043 +102044,102044 +21192,102044 +102045,102045 +102046,102046 +102047,102047 +102048,102048 +102049,102049 +102050,102050 +102051,102051 +102052,102052 +102053,102053 +102054,102054 +102055,102055 +102056,102056 +102057,102057 +102058,102058 +102059,102059 +102060,102060 +102061,102061 +102062,102062 +161127,102062 +190410,102062 +122253,102062 +102063,102063 +102064,102064 +102065,102065 +102066,102066 +58622,102066 +102067,102067 +102068,102068 +102069,102069 +102070,102070 +102071,102071 +102072,102072 +102073,102073 +102074,102074 +102075,102075 +102076,102076 +102077,102077 +207437,102077 +102078,102078 +102079,102079 +102080,102080 +102081,102081 +102082,102082 +102083,102083 +102084,102084 +102085,102085 +102086,102086 +102087,102087 +102088,102088 +102089,102089 +141012,102089 +102090,102090 +102091,102091 +102092,102092 +102093,102093 +102094,102094 +102095,102095 +102096,102096 +102097,102097 +102098,102098 +11366,102098 +102099,102099 +102100,102100 +22405,102100 +102101,102101 +102102,102102 +102103,102103 +102104,102104 +102105,102105 +102106,102106 +102107,102107 +102108,102108 +102109,102109 +102110,102110 +102111,102111 +102112,102112 +243857,102112 +102113,102113 +102114,102114 +102115,102115 +102116,102116 +102117,102117 +102118,102118 +158106,102118 +102119,102119 +102120,102120 +102121,102121 +102122,102122 +102123,102123 +102124,102124 +102125,102125 +102126,102126 +102127,102127 +102128,102128 +102129,102129 +102130,102130 +102131,102131 +102132,102132 +102133,102133 +73815,102133 +102134,102134 +100704,102134 +102135,102135 +102136,102136 +102137,102137 +102138,102138 +102139,102139 +102140,102140 +102141,102141 +102142,102142 +102143,102143 +102144,102144 +102145,102145 +102146,102146 +102147,102147 +102148,102148 +102149,102149 +102150,102150 +102151,102151 +102152,102152 +102153,102153 +102154,102154 +102155,102155 +102156,102156 +102157,102157 +102158,102158 +102159,102159 +102160,102160 +102161,102161 +102162,102162 +102163,102163 +102164,102164 +102165,102165 +102166,102166 +226142,102166 +102167,102167 +102168,102168 +102169,102169 +102170,102170 +102171,102171 +102172,102172 +102173,102173 +102174,102174 +102175,102175 +102176,102176 +102177,102177 +102178,102178 +102179,102179 +102180,102180 +102181,102181 +234461,102181 +90166,102181 +102182,102182 +102183,102183 +102184,102184 +102185,102185 +102186,102186 +142920,102186 +102187,102187 +102188,102188 +102189,102189 +102190,102190 +102191,102191 +102192,102192 +102193,102193 +102194,102194 +188338,102194 +102195,102195 +102196,102196 +149422,102196 +102197,102197 +102198,102198 +102199,102199 +102200,102200 +102201,102201 +102202,102202 +102203,102203 +60396,102203 +102204,102204 +102205,102205 +102206,102206 +102207,102207 +102208,102208 +102209,102209 +102210,102210 +102211,102211 +102212,102212 +102213,102213 +102214,102214 +102215,102215 +102216,102216 +102217,102217 +102218,102218 +102219,102219 +102220,102220 +102221,102221 +102222,102222 +102223,102223 +102224,102224 +102225,102225 +208406,102226 +102226,102226 +102227,102227 +102228,102228 +102229,102229 +102230,102230 +102231,102231 +102232,102232 +102233,102233 +206116,102233 +102234,102234 +102235,102235 +102236,102236 +102237,102237 +102238,102238 +102239,102239 +102240,102240 +102241,102241 +102242,102242 +102243,102243 +102244,102244 +102245,102245 +102246,102246 +218325,102246 +102247,102247 +102248,102248 +102249,102249 +102250,102250 +121430,102250 +102251,102251 +102252,102252 +102253,102253 +102254,102254 +102255,102255 +102256,102256 +102257,102257 +102258,102258 +102259,102259 +102260,102260 +102261,102261 +102262,102262 +102263,102263 +102264,102264 +102265,102265 +101967,102265 +217753,102265 +102266,102266 +116809,102266 +102267,102267 +102268,102268 +102269,102269 +102270,102270 +102271,102271 +162619,102271 +102272,102272 +102273,102273 +102274,102274 +102275,102275 +102276,102276 +102277,102277 +102278,102278 +102279,102279 +102280,102280 +102281,102281 +102282,102282 +102283,102283 +102331,102331 +102332,102332 +102333,102333 +102334,102334 +102335,102335 +102336,102336 +102337,102337 +102338,102338 +102339,102339 +102340,102340 +102341,102341 +102342,102342 +4168,102342 +102343,102343 +102344,102344 +102345,102345 +102346,102346 +102347,102347 +102348,102348 +102349,102349 +102350,102350 +102351,102351 +102352,102352 +102353,102353 +102354,102354 +102355,102355 +102356,102356 +90739,102356 +102357,102357 +102358,102358 +102359,102359 +150791,102359 +102360,102360 +102361,102361 +102362,102362 +102363,102363 +151324,102363 +102364,102364 +102365,102365 +102366,102366 +102367,102367 +102368,102368 +102369,102369 +102370,102370 +102371,102371 +102372,102372 +102373,102373 +102374,102374 +102375,102375 +102376,102376 +102377,102377 +102378,102378 +102379,102379 +102380,102380 +102381,102381 +102382,102382 +102383,102383 +103936,102383 +103687,102383 +102384,102384 +102385,102385 +101052,102385 +102386,102386 +191263,102386 +102387,102387 +192549,102387 +224398,102387 +102388,102388 +102389,102389 +102390,102390 +102391,102391 +214383,102391 +102392,102392 +102393,102393 +102394,102394 +102395,102395 +102396,102396 +102397,102397 +102398,102398 +102399,102399 +102400,102400 +102401,102401 +102402,102402 +102403,102403 +102404,102404 +102405,102405 +102406,102406 +102407,102407 +102408,102408 +102409,102409 +102410,102410 +102411,102411 +102412,102412 +102413,102413 +102414,102414 +102415,102415 +102416,102416 +102417,102417 +102418,102418 +102419,102419 +102420,102420 +102421,102421 +102422,102422 +102423,102423 +102424,102424 +221549,102424 +102425,102425 +221551,102426 +102426,102426 +102427,102427 +102428,102428 +102477,102477 +102478,102478 +102479,102479 +163847,102479 +102480,102480 +102481,102481 +102482,102482 +102483,102483 +102484,102484 +102485,102485 +102486,102486 +102487,102487 +102488,102488 +102489,102489 +102490,102490 +102491,102491 +102492,102492 +102493,102493 +102494,102494 +102495,102495 +102496,102496 +102497,102497 +102498,102498 +102499,102499 +102500,102500 +102501,102501 +102502,102502 +102503,102503 +102504,102504 +102505,102505 +102506,102506 +102507,102507 +102508,102508 +102509,102509 +102510,102510 +102511,102511 +102512,102512 +102513,102513 +102514,102514 +102515,102515 +102516,102516 +102517,102517 +102518,102518 +102519,102519 +102520,102520 +102521,102521 +223049,102522 +102522,102522 +102523,102523 +102524,102524 +102525,102525 +63516,102525 +102526,102526 +102527,102527 +102528,102528 +102529,102529 +102530,102530 +102531,102531 +102532,102532 +102533,102533 +102534,102534 +102535,102535 +102536,102536 +102537,102537 +102538,102538 +102539,102539 +102540,102540 +102541,102541 +102542,102542 +102543,102543 +102544,102544 +102545,102545 +102546,102546 +102547,102547 +102548,102548 +102549,102549 +102550,102550 +102551,102551 +102552,102552 +102553,102553 +102554,102554 +102555,102555 +102556,102556 +28073,102556 +102557,102557 +102558,102558 +102559,102559 +102560,102560 +102561,102561 +235006,102562 +102562,102562 +102563,102563 +102564,102564 +102565,102565 +102566,102566 +102567,102567 +102568,102568 +102569,102569 +102570,102570 +102571,102571 +102572,102572 +102573,102573 +102574,102574 +102619,102619 +102620,102620 +246383,102620 +102621,102621 +102622,102622 +102623,102623 +102624,102624 +102625,102625 +248048,102625 +102626,102626 +102627,102627 +102628,102628 +102629,102629 +102630,102630 +102631,102631 +102632,102632 +102633,102633 +102634,102634 +102635,102635 +102636,102636 +102637,102637 +102638,102638 +102639,102639 +102640,102640 +102641,102641 +102642,102642 +102643,102643 +102644,102644 +102645,102645 +102646,102646 +102647,102647 +102648,102648 +102649,102649 +102650,102650 +102651,102651 +83064,102651 +133544,102651 +125957,102651 +102652,102652 +102653,102653 +102654,102654 +102655,102655 +102656,102656 +102657,102657 +102658,102658 +102659,102659 +102660,102660 +102661,102661 +102662,102662 +152895,102662 +102663,102663 +102664,102664 +102665,102665 +102666,102666 +102667,102667 +102668,102668 +102669,102669 +161957,102669 +102670,102670 +103307,103307 +103308,103308 +103309,103309 +236335,103309 +103310,103310 +103311,103311 +103312,103312 +103313,103313 +103314,103314 +103315,103315 +103316,103316 +103317,103317 +103318,103318 +81856,103318 +103369,103369 +103370,103370 +198028,103370 +103371,103371 +103372,103372 +103373,103373 +103374,103374 +103375,103375 +103376,103376 +103377,103377 +103378,103378 +207844,103378 +194768,103378 +103379,103379 +33760,103379 +103380,103380 +103381,103381 +103382,103382 +103383,103383 +103384,103384 +103385,103385 +103386,103386 +103387,103387 +103388,103388 +103389,103389 +103390,103390 +103391,103391 +595,103391 +103392,103392 +103393,103393 +103394,103394 +81786,103394 +154741,103394 +103395,103395 +103396,103396 +103397,103397 +103398,103398 +103399,103399 +103400,103400 +103401,103401 +214830,103401 +103402,103402 +103403,103403 +103404,103404 +103405,103405 +103406,103406 +103407,103407 +103408,103408 +103409,103409 +103410,103410 +103411,103411 +103412,103412 +185277,103412 +103413,103413 +103414,103414 +103415,103415 +103416,103416 +103417,103417 +103418,103418 +103419,103419 +103420,103420 +103421,103421 +103422,103422 +103423,103423 +103424,103424 +103425,103425 +103426,103426 +103427,103427 +103428,103428 +103429,103429 +103430,103430 +103431,103431 +103432,103432 +103433,103433 +103434,103434 +103435,103435 +103436,103436 +103437,103437 +103438,103438 +103439,103439 +103440,103440 +103441,103441 +103442,103442 +103443,103443 +103444,103444 +103445,103445 +103446,103446 +103447,103447 +103448,103448 +103449,103449 +103450,103450 +103451,103451 +103452,103452 +103453,103453 +103454,103454 +103455,103455 +103456,103456 +103457,103457 +103458,103458 +103459,103459 +103460,103460 +103461,103461 +103462,103462 +195201,103462 +35587,103462 +103463,103463 +103464,103464 +103465,103465 +103466,103466 +103467,103467 +103468,103468 +103469,103469 +103470,103470 +103471,103471 +103472,103472 +103473,103473 +103474,103474 +103475,103475 +103476,103476 +103477,103477 +166707,103477 +103478,103478 +103479,103479 +103480,103480 +103481,103481 +103482,103482 +103483,103483 +103484,103484 +103485,103485 +103486,103486 +103487,103487 +103488,103488 +103489,103489 +103490,103490 +103491,103491 +138714,103491 +103492,103492 +63154,103492 +103493,103493 +103494,103494 +103495,103495 +103496,103496 +103497,103497 +103498,103498 +103499,103499 +103500,103500 +103501,103501 +103502,103502 +103503,103503 +103504,103504 +103505,103505 +103506,103506 +231230,103506 +103507,103507 +103508,103508 +103509,103509 +103510,103510 +103511,103511 +201402,103511 +103512,103512 +103513,103513 +103514,103514 +211942,103514 +103515,103515 +57970,103515 +103516,103516 +103517,103517 +103518,103518 +103519,103519 +103520,103520 +103521,103521 +103522,103522 +103523,103523 +103524,103524 +103525,103525 +103526,103526 +103527,103527 +103528,103528 +103529,103529 +103530,103530 +103531,103531 +103532,103532 +103533,103533 +103534,103534 +103535,103535 +103536,103536 +97695,103536 +103537,103537 +103538,103538 +103539,103539 +103540,103540 +103541,103541 +103542,103542 +103543,103543 +103544,103544 +103545,103545 +78817,103545 +103546,103546 +103547,103547 +103548,103548 +103549,103549 +103550,103550 +183236,103550 +103551,103551 +103552,103552 +103553,103553 +103554,103554 +103555,103555 +103556,103556 +103557,103557 +103558,103558 +103559,103559 +103560,103560 +103561,103561 +103562,103562 +103563,103563 +103564,103564 +103565,103565 +103566,103566 +103567,103567 +103568,103568 +103569,103569 +103570,103570 +103571,103571 +103572,103572 +103573,103573 +103574,103574 +103575,103575 +103576,103576 +103577,103577 +103578,103578 +103579,103579 +103580,103580 +103581,103581 +103582,103582 +103583,103583 +103584,103584 +103585,103585 +103586,103586 +103587,103587 +103588,103588 +103589,103589 +103590,103590 +103591,103591 +103592,103592 +103593,103593 +103594,103594 +157373,103594 +103595,103595 +103596,103596 +103597,103597 +103598,103598 +103599,103599 +103600,103600 +103601,103601 +103602,103602 +103603,103603 +103604,103604 +103605,103605 +103606,103606 +103607,103607 +103608,103608 +103609,103609 +103610,103610 +103611,103611 +103612,103612 +103613,103613 +176278,103613 +170314,103613 +103614,103614 +103615,103615 +146433,103615 +103616,103616 +103617,103617 +180655,103617 +60435,103617 +103618,103618 +180906,103618 +103619,103619 +7706,103619 +103620,103620 +103621,103621 +103622,103622 +144151,103623 +103623,103623 +103624,103624 +103625,103625 +103626,103626 +103627,103627 +103628,103628 +103629,103629 +103630,103630 +194726,103630 +103631,103631 +932,103631 +103632,103632 +103633,103633 +103634,103634 +75817,103634 +103635,103635 +103636,103636 +103637,103637 +103638,103638 +103639,103639 +103640,103640 +103641,103641 +103642,103642 +103643,103643 +103644,103644 +103645,103645 +103646,103646 +103647,103647 +103648,103648 +103649,103649 +103650,103650 +185901,103650 +103651,103651 +103652,103652 +103653,103653 +103654,103654 +103655,103655 +103656,103656 +103657,103657 +103658,103658 +103659,103659 +103660,103660 +103661,103661 +103662,103662 +103663,103663 +103664,103664 +103665,103665 +103666,103666 +103667,103667 +103668,103668 +103669,103669 +103670,103670 +103671,103671 +103672,103672 +103673,103673 +103674,103674 +103675,103675 +103676,103676 +103677,103677 +103678,103678 +103679,103679 +103680,103680 +103681,103681 +103682,103682 +103683,103683 +103684,103684 +103685,103685 +103686,103686 +62596,103686 +103687,103687 +102383,103687 +103688,103688 +103689,103689 +103690,103690 +103939,103690 +103691,103691 +103692,103692 +103693,103693 +103694,103694 +103943,103694 +110057,103695 +103695,103695 +103696,103696 +103697,103697 +103698,103698 +103699,103699 +103700,103700 +103701,103701 +103702,103702 +103703,103703 +103704,103704 +207341,103704 +103705,103705 +103706,103706 +104140,103777 +103777,103777 +103778,103778 +104142,103779 +104029,103779 +103779,103779 +104143,103780 +104030,103780 +103780,103780 +103881,103881 +103882,103882 +103883,103883 +103884,103884 +101401,103884 +212757,103884 +103885,103885 +103886,103886 +103887,103887 +103888,103888 +103889,103889 +103890,103890 +103891,103891 +103892,103892 +103893,103893 +103894,103894 +103895,103895 +103896,103896 +103897,103897 +103936,103936 +102383,103936 +103937,103937 +235144,103937 +103938,103938 +103939,103939 +103690,103939 +103940,103940 +103941,103941 +103942,103942 +103943,103943 +103694,103943 +103944,103944 +103945,103945 +229164,103945 +103946,103946 +103947,103947 +103948,103948 +20111,103948 +103949,103949 +103950,103950 +103951,103951 +103952,103952 +103953,103953 +103954,103954 +103955,103955 +103956,103956 +79433,103956 +103957,103957 +103958,103958 +103959,103959 +207352,103959 +103960,103960 +103961,103961 +216424,103961 +103962,103962 +103963,103963 +103964,103964 +103965,103965 +103966,103966 +103967,103967 +103968,103968 +103969,103969 +103970,103970 +103971,103971 +103972,103972 +103973,103973 +103974,103974 +64103,103974 +103975,103975 +103976,103976 +103977,103977 +103978,103978 +234864,103978 +103979,103979 +103980,103980 +104028,104028 +51819,104028 +43715,104028 +104141,104028 +104142,104029 +104029,104029 +103779,104029 +104143,104030 +104030,104030 +103780,104030 +104031,104031 +232107,104031 +105123,104031 +104032,104032 +104033,104033 +184649,104033 +104034,104034 +104148,104035 +104035,104035 +104036,104036 +208205,104036 +104037,104037 +104038,104038 +104039,104039 +104040,104040 +104041,104041 +104042,104042 +110334,104042 +68633,104042 +104043,104043 +104044,104044 +104045,104045 +104046,104046 +104047,104047 +104048,104048 +139774,104049 +104049,104049 +104050,104050 +104051,104051 +104052,104052 +104053,104053 +104054,104054 +104055,104055 +6840,104055 +104056,104056 +104057,104057 +104058,104058 +104059,104059 +104060,104060 +104129,104129 +104130,104130 +162516,104130 +104131,104131 +104132,104132 +104133,104133 +104134,104134 +104135,104135 +104136,104136 +104137,104137 +104138,104138 +104139,104139 +104140,104140 +103777,104140 +104141,104141 +104028,104141 +104142,104142 +104029,104142 +103779,104142 +104143,104143 +104030,104143 +103780,104143 +104144,104144 +104145,104145 +104146,104146 +104147,104147 +104148,104148 +104035,104148 +104149,104149 +104256,104256 +104257,104257 +104258,104258 +104259,104259 +104260,104260 +104261,104261 +104297,104297 +59157,104297 +58783,104297 +104298,104298 +104299,104299 +104300,104300 +104301,104301 +104302,104302 +157093,104302 +104303,104303 +152881,104303 +104304,104304 +104305,104305 +104306,104306 +104307,104307 +104308,104308 +104309,104309 +104310,104310 +104311,104311 +104312,104312 +104313,104313 +104314,104314 +104315,104315 +104316,104316 +134127,104316 +126540,104316 +104317,104317 +104318,104318 +104319,104319 +104320,104320 +104321,104321 +104322,104322 +104323,104323 +223471,104323 +104324,104324 +104325,104325 +104326,104326 +104327,104327 +104328,104328 +104329,104329 +104330,104330 +104331,104331 +104332,104332 +104333,104333 +104334,104334 +104335,104335 +104336,104336 +104337,104337 +104338,104338 +104339,104339 +104340,104340 +104341,104341 +104342,104342 +247231,104342 +104343,104343 +104344,104344 +104345,104345 +104346,104346 +104347,104347 +104348,104348 +100631,104348 +180998,104348 +104349,104349 +104350,104350 +100633,104350 +153929,104350 +104351,104351 +104352,104352 +104353,104353 +104354,104354 +104355,104355 +104356,104356 +104357,104357 +104358,104358 +108896,104358 +104359,104359 +104360,104360 +104361,104361 +104362,104362 +104363,104363 +92635,104363 +104364,104364 +104365,104365 +104366,104366 +104367,104367 +104368,104368 +104369,104369 +104370,104370 +104371,104371 +104372,104372 +104373,104373 +104374,104374 +104375,104375 +104376,104376 +104377,104377 +104378,104378 +104379,104379 +104380,104380 +104381,104381 +104382,104382 +104383,104383 +104384,104384 +104385,104385 +104386,104386 +104387,104387 +104388,104388 +104389,104389 +104390,104390 +104391,104391 +104392,104392 +104393,104393 +104394,104394 +104395,104395 +164850,104395 +104396,104396 +104397,104397 +104398,104398 +104399,104399 +104400,104400 +177581,104400 +171617,104400 +104401,104401 +104402,104402 +104403,104403 +104404,104404 +104405,104405 +104406,104406 +104407,104407 +104408,104408 +104409,104409 +104410,104410 +88529,104410 +104411,104411 +104412,104412 +104413,104413 +104414,104414 +104415,104415 +214810,104416 +104416,104416 +104417,104417 +104418,104418 +104419,104419 +104420,104420 +104421,104421 +104422,104422 +104423,104423 +104424,104424 +104425,104425 +104426,104426 +104427,104427 +104428,104428 +104429,104429 +104430,104430 +104431,104431 +104432,104432 +104433,104433 +104434,104434 +104435,104435 +104436,104436 +104437,104437 +104438,104438 +248048,104438 +26060,104438 +104439,104439 +6180,104439 +104440,104440 +248050,104440 +104441,104441 +231661,104441 +104442,104442 +104629,104629 +104630,104630 +104631,104631 +104632,104632 +104633,104633 +190216,104633 +104634,104634 +104635,104635 +104636,104636 +104637,104637 +179290,104637 +104638,104638 +104639,104639 +104640,104640 +104641,104641 +104642,104642 +104643,104643 +104644,104644 +104645,104645 +104646,104646 +104647,104647 +104648,104648 +104649,104649 +104650,104650 +104651,104651 +104652,104652 +104653,104653 +104654,104654 +55289,104654 +47185,104654 +104655,104655 +89599,104655 +104656,104656 +104657,104657 +76366,104657 +104658,104658 +104659,104659 +104660,104660 +65721,104660 +104661,104661 +104662,104662 +104663,104663 +104664,104664 +104665,104665 +104666,104666 +104667,104667 +104668,104668 +84934,104668 +104669,104669 +104670,104670 +104671,104671 +104672,104672 +100659,104672 +104673,104673 +104674,104674 +104675,104675 +208344,104675 +104676,104676 +104677,104677 +189146,104677 +104678,104678 +104679,104679 +104680,104680 +104681,104681 +104682,104682 +104683,104683 +104684,104684 +104685,104685 +104686,104686 +104687,104687 +104688,104688 +104689,104689 +104690,104690 +104691,104691 +104692,104692 +104693,104693 +104694,104694 +104695,104695 +104696,104696 +104697,104697 +104698,104698 +219975,104698 +104699,104699 +104700,104700 +104701,104701 +104702,104702 +104703,104703 +104704,104704 +104705,104705 +104706,104706 +104707,104707 +104708,104708 +104709,104709 +104710,104710 +104711,104711 +108294,104711 +104712,104712 +110522,104712 +27926,104712 +104713,104713 +104714,104714 +104715,104715 +104716,104716 +104717,104717 +104718,104718 +4387,104718 +104719,104719 +36336,104719 +104720,104720 +104721,104721 +104722,104722 +104723,104723 +104724,104724 +104725,104725 +104726,104726 +104727,104727 +104728,104728 +110831,104728 +104729,104729 +104730,104730 +104731,104731 +104732,104732 +104733,104733 +104734,104734 +104735,104735 +104736,104736 +104737,104737 +104738,104738 +104739,104739 +104740,104740 +104741,104741 +142010,104742 +104742,104742 +104743,104743 +104744,104744 +104745,104745 +104746,104746 +108488,104746 +104747,104747 +104748,104748 +104749,104749 +104750,104750 +104751,104751 +104752,104752 +104753,104753 +104754,104754 +104755,104755 +104756,104756 +104757,104757 +104758,104758 +104759,104759 +104760,104760 +104761,104761 +104762,104762 +104763,104763 +104764,104764 +136315,104764 +128728,104764 +104765,104765 +104766,104766 +104767,104767 +104768,104768 +104769,104769 +104770,104770 +104771,104771 +104772,104772 +104773,104773 +104774,104774 +104775,104775 +104776,104776 +104777,104777 +104778,104778 +235662,104778 +104779,104779 +104780,104780 +104781,104781 +104782,104782 +104783,104783 +104784,104784 +104785,104785 +104786,104786 +104787,104787 +104788,104788 +104789,104789 +104790,104790 +104791,104791 +104792,104792 +104793,104793 +104794,104794 +104795,104795 +104796,104796 +104797,104797 +104798,104798 +104799,104799 +104800,104800 +104801,104801 +104802,104802 +104803,104803 +104804,104804 +104805,104805 +104806,104806 +104807,104807 +104808,104808 +104809,104809 +104810,104810 +104811,104811 +104812,104812 +104813,104813 +104814,104814 +104815,104815 +104816,104816 +104817,104817 +210320,104817 +104818,104818 +104819,104819 +104820,104820 +104821,104821 +104822,104822 +104823,104823 +104824,104824 +104825,104825 +104826,104826 +104827,104827 +104828,104828 +104829,104829 +154539,104829 +104830,104830 +104831,104831 +104832,104832 +104833,104833 +104834,104834 +104835,104835 +104836,104836 +104837,104837 +104838,104838 +104839,104839 +104840,104840 +104841,104841 +104842,104842 +104843,104843 +104844,104844 +104845,104845 +104846,104846 +104847,104847 +104848,104848 +104849,104849 +104850,104850 +104851,104851 +104852,104852 +104853,104853 +104854,104854 +104855,104855 +104856,104856 +104857,104857 +104858,104858 +104859,104859 +104860,104860 +104861,104861 +104862,104862 +104863,104863 +104864,104864 +104865,104865 +104866,104866 +10888,104866 +104867,104867 +104868,104868 +104869,104869 +104870,104870 +104871,104871 +104872,104872 +104873,104873 +104874,104874 +104875,104875 +104876,104876 +116999,104876 +104877,104877 +104878,104878 +104879,104879 +104880,104880 +104881,104881 +104882,104882 +104883,104883 +104884,104884 +104885,104885 +104886,104886 +120390,104886 +104887,104887 +104888,104888 +104889,104889 +104890,104890 +104891,104891 +104892,104892 +104893,104893 +104894,104894 +104895,104895 +104896,104896 +104897,104897 +183383,104897 +104898,104898 +194407,104898 +104899,104899 +104900,104900 +104901,104901 +104902,104902 +104903,104903 +104904,104904 +104905,104905 +104906,104906 +104907,104907 +104908,104908 +104909,104909 +104910,104910 +104911,104911 +104912,104912 +104913,104913 +104914,104914 +104915,104915 +104916,104916 +104917,104917 +104918,104918 +104919,104919 +104920,104920 +139120,104920 +104921,104921 +104922,104922 +104923,104923 +104924,104924 +104925,104925 +104926,104926 +80248,104926 +104927,104927 +104928,104928 +104929,104929 +104930,104930 +104931,104931 +104932,104932 +104933,104933 +104934,104934 +104935,104935 +104936,104936 +104937,104937 +104938,104938 +104939,104939 +28750,104939 +104940,104940 +104941,104941 +104942,104942 +104943,104943 +104944,104944 +104945,104945 +104946,104946 +104947,104947 +104948,104948 +104949,104949 +104950,104950 +104951,104951 +104952,104952 +104953,104953 +104954,104954 +104955,104955 +233419,104955 +104956,104956 +104957,104957 +104958,104958 +104959,104959 +104960,104960 +104961,104961 +104962,104962 +104963,104963 +145281,104963 +104964,104964 +104965,104965 +104966,104966 +104967,104967 +104968,104968 +6630,104968 +104969,104969 +104970,104970 +104971,104971 +104972,104972 +104973,104973 +104974,104974 +104975,104975 +104976,104976 +104977,104977 +104978,104978 +122705,104978 +104979,104979 +82591,104979 +104980,104980 +104981,104981 +104982,104982 +104983,104983 +104984,104984 +104985,104985 +104986,104986 +104987,104987 +104988,104988 +104989,104989 +104990,104990 +104991,104991 +104992,104992 +104993,104993 +104994,104994 +104995,104995 +104996,104996 +104997,104997 +104998,104998 +104999,104999 +105000,105000 +105001,105001 +105002,105002 +105003,105003 +105004,105004 +105005,105005 +105006,105006 +105007,105007 +233039,105007 +213825,105007 +105008,105008 +105009,105009 +105010,105010 +105011,105011 +105012,105012 +105013,105013 +105014,105014 +105015,105015 +185910,105015 +105016,105016 +105017,105017 +105018,105018 +105019,105019 +105020,105020 +158613,105020 +105021,105021 +105022,105022 +105023,105023 +105024,105024 +105025,105025 +105026,105026 +105027,105027 +105028,105028 +76433,105028 +105029,105029 +105030,105030 +105031,105031 +105032,105032 +105033,105033 +105034,105034 +105035,105035 +105036,105036 +105037,105037 +105038,105038 +105039,105039 +105040,105040 +105041,105041 +105042,105042 +105043,105043 +105044,105044 +105045,105045 +105046,105046 +80021,105046 +105047,105047 +105048,105048 +105049,105049 +105050,105050 +105051,105051 +105052,105052 +105053,105053 +105054,105054 +105055,105055 +105056,105056 +105057,105057 +105058,105058 +105059,105059 +105060,105060 +105061,105061 +105062,105062 +105063,105063 +105064,105064 +105065,105065 +59967,105065 +105066,105066 +106700,105066 +105067,105067 +105068,105068 +105069,105069 +105070,105070 +105071,105071 +105072,105072 +105073,105073 +105074,105074 +105075,105075 +105076,105076 +105077,105077 +105078,105078 +105079,105079 +105123,105123 +104031,105123 +105124,105124 +105125,105125 +105126,105126 +105127,105127 +105128,105128 +105129,105129 +105130,105130 +105131,105131 +105132,105132 +105133,105133 +32855,105133 +105134,105134 +105135,105135 +105136,105136 +105137,105137 +105138,105138 +105139,105139 +105140,105140 +105141,105141 +105142,105142 +105143,105143 +105144,105144 +105145,105145 +105146,105146 +105183,105183 +105184,105184 +40266,105184 +105185,105185 +229164,105185 +105186,105186 +105228,105228 +105229,105229 +105230,105230 +105231,105231 +105232,105232 +105233,105233 +105622,105622 +105623,105623 +105624,105624 +105625,105625 +105626,105626 +105627,105627 +105628,105628 +105629,105629 +105630,105630 +105631,105631 +180265,105631 +242914,105631 +105632,105632 +105633,105633 +105634,105634 +105635,105635 +105636,105636 +105637,105637 +105638,105638 +105639,105639 +76472,105639 +105640,105640 +83841,105640 +105641,105641 +86037,105641 +105642,105642 +105643,105643 +105644,105644 +105645,105645 +105646,105646 +105647,105647 +87601,105647 +105648,105648 +105649,105649 +105650,105650 +105651,105651 +105652,105652 +105653,105653 +105654,105654 +105655,105655 +246188,105655 +105656,105656 +5469,105656 +105657,105657 +5470,105657 +105658,105658 +105659,105659 +105660,105660 +105661,105661 +105662,105662 +105663,105663 +105664,105664 +105665,105665 +165304,105665 +105666,105666 +105667,105667 +105668,105668 +105669,105669 +105670,105670 +105671,105671 +62450,105671 +105672,105672 +105673,105673 +105674,105674 +105675,105675 +105676,105676 +105677,105677 +105678,105678 +92659,105678 +80443,105678 +203739,105678 +105679,105679 +105680,105680 +105681,105681 +105682,105682 +105683,105683 +105684,105684 +105685,105685 +105686,105686 +105687,105687 +140255,105687 +105688,105688 +105689,105689 +105690,105690 +105691,105691 +105692,105692 +105693,105693 +105694,105694 +105695,105695 +105696,105696 +105697,105697 +105698,105698 +105699,105699 +210028,105699 +105700,105700 +105701,105701 +105702,105702 +105703,105703 +105704,105704 +105705,105705 +105706,105706 +105707,105707 +105708,105708 +105709,105709 +105710,105710 +105711,105711 +105712,105712 +105713,105713 +105714,105714 +105715,105715 +105716,105716 +105717,105717 +105718,105718 +105719,105719 +105720,105720 +105721,105721 +105722,105722 +60578,105722 +105723,105723 +105724,105724 +105725,105725 +105726,105726 +105727,105727 +105728,105728 +105729,105729 +105730,105730 +105731,105731 +105732,105732 +105733,105733 +105734,105734 +105735,105735 +105736,105736 +105737,105737 +105738,105738 +105739,105739 +105740,105740 +105741,105741 +105742,105742 +105743,105743 +105744,105744 +105745,105745 +105746,105746 +105747,105747 +105748,105748 +105749,105749 +105750,105750 +105751,105751 +105752,105752 +105753,105753 +105754,105754 +105755,105755 +105756,105756 +105757,105757 +105758,105758 +105759,105759 +105760,105760 +105761,105761 +105762,105762 +105763,105763 +105764,105764 +105765,105765 +105766,105766 +105767,105767 +105768,105768 +105769,105769 +105770,105770 +105771,105771 +105772,105772 +105773,105773 +105774,105774 +105775,105775 +105776,105776 +105777,105777 +105778,105778 +105779,105779 +187105,105780 +105780,105780 +105781,105781 +105782,105782 +105783,105783 +105784,105784 +105785,105785 +105786,105786 +105787,105787 +105788,105788 +105789,105789 +105790,105790 +105791,105791 +105792,105792 +105793,105793 +105794,105794 +105795,105795 +105796,105796 +105797,105797 +105798,105798 +105799,105799 +105800,105800 +105801,105801 +105802,105802 +105803,105803 +105804,105804 +136248,105805 +128661,105805 +105805,105805 +105806,105806 +105807,105807 +239163,105807 +105808,105808 +105809,105809 +105810,105810 +105867,105867 +105868,105868 +105869,105869 +105870,105870 +142928,105870 +105871,105871 +105872,105872 +105873,105873 +105874,105874 +105875,105875 +105876,105876 +105877,105877 +105878,105878 +105879,105879 +105880,105880 +105881,105881 +105882,105882 +105883,105883 +105884,105884 +105885,105885 +105886,105886 +105887,105887 +105888,105888 +105889,105889 +105890,105890 +106111,106111 +106112,106112 +139899,106112 +106113,106113 +106114,106114 +106115,106115 +106116,106116 +106117,106117 +106118,106118 +106119,106119 +106120,106120 +106121,106121 +106122,106122 +106123,106123 +106124,106124 +106125,106125 +185391,106125 +106126,106126 +144071,106126 +106127,106127 +106128,106128 +106129,106129 +106130,106130 +106131,106131 +106132,106132 +223497,106132 +106133,106133 +106134,106134 +106135,106135 +106136,106136 +57291,106136 +106137,106137 +106138,106138 +106139,106139 +106178,106178 +106179,106179 +106180,106180 +106181,106181 +106182,106182 +30229,106182 +106183,106183 +106184,106184 +106185,106185 +106186,106186 +87535,106186 +106187,106187 +106188,106188 +106189,106189 +106190,106190 +40194,106190 +106191,106191 +106192,106192 +106193,106193 +106194,106194 +106195,106195 +106196,106196 +106197,106197 +106198,106198 +106199,106199 +178271,106199 +172307,106199 +106200,106200 +101919,106200 +106201,106201 +106202,106202 +106203,106203 +106204,106204 +106205,106205 +106206,106206 +106207,106207 +106208,106208 +106209,106209 +106210,106210 +106211,106211 +106212,106212 +211140,106212 +106213,106213 +106214,106214 +106215,106215 +106216,106216 +106217,106217 +106218,106218 +106219,106219 +106220,106220 +106221,106221 +106222,106222 +217749,106222 +106223,106223 +106224,106224 +106225,106225 +106226,106226 +106227,106227 +106228,106228 +106229,106229 +106230,106230 +106231,106231 +106232,106232 +106233,106233 +106234,106234 +106235,106235 +106236,106236 +106237,106237 +106238,106238 +106239,106239 +106240,106240 +106241,106241 +106242,106242 +106243,106243 +106244,106244 +106245,106245 +106246,106246 +106247,106247 +106248,106248 +106249,106249 +106250,106250 +106251,106251 +106252,106252 +106253,106253 +59763,106253 +106254,106254 +89561,106254 +106255,106255 +106256,106256 +106257,106257 +106258,106258 +106259,106259 +106260,106260 +106261,106261 +106262,106262 +106263,106263 +162418,106263 +106264,106264 +106265,106265 +106266,106266 +63813,106266 +106267,106267 +106268,106268 +106269,106269 +106270,106270 +106271,106271 +106272,106272 +106273,106273 +106274,106274 +218073,106274 +106275,106275 +63822,106275 +106276,106276 +106277,106277 +106278,106278 +106279,106279 +106280,106280 +106281,106281 +106282,106282 +106283,106283 +106284,106284 +106285,106285 +106286,106286 +106287,106287 +106288,106288 +106289,106289 +106290,106290 +106291,106291 +106292,106292 +106293,106293 +106294,106294 +179651,106294 +106295,106295 +106296,106296 +106297,106297 +106298,106298 +106299,106299 +106300,106300 +106301,106301 +106302,106302 +106303,106303 +106304,106304 +106305,106305 +106306,106306 +106307,106307 +106308,106308 +106309,106309 +106310,106310 +106311,106311 +106312,106312 +106313,106313 +106314,106314 +106315,106315 +116625,106315 +106316,106316 +112090,106316 +106317,106317 +106318,106318 +106319,106319 +106320,106320 +106321,106321 +27114,106321 +106322,106322 +106323,106323 +106324,106324 +106325,106325 +106326,106326 +106327,106327 +106328,106328 +106329,106329 +106330,106330 +106331,106331 +106332,106332 +106333,106333 +106334,106334 +106335,106335 +106336,106336 +106337,106337 +106338,106338 +106339,106339 +106340,106340 +106341,106341 +106342,106342 +106343,106343 +106344,106344 +106345,106345 +106346,106346 +106347,106347 +106348,106348 +192550,106348 +106349,106349 +106350,106350 +106351,106351 +106352,106352 +36334,106352 +106515,106515 +106516,106516 +106517,106517 +106518,106518 +106519,106519 +106520,106520 +106521,106521 +106522,106522 +106523,106523 +106524,106524 +106525,106525 +106526,106526 +2242,106526 +110865,106526 +106527,106527 +106528,106528 +106529,106529 +142884,106529 +106530,106530 +106531,106531 +106532,106532 +189188,106532 +106533,106533 +106534,106534 +106535,106535 +106536,106536 +106537,106537 +106538,106538 +106539,106539 +106540,106540 +106541,106541 +106542,106542 +106543,106543 +106544,106544 +20826,106544 +106545,106545 +106546,106546 +106547,106547 +106548,106548 +106549,106549 +106550,106550 +106551,106551 +106641,106641 +106642,106642 +106643,106643 +106644,106644 +106645,106645 +106646,106646 +106647,106647 +106648,106648 +106649,106649 +106650,106650 +106651,106651 +106652,106652 +106653,106653 +106654,106654 +106655,106655 +106656,106656 +106657,106657 +106658,106658 +106659,106659 +106660,106660 +183428,106661 +106661,106661 +106662,106662 +106663,106663 +106664,106664 +106665,106665 +106666,106666 +106667,106667 +106668,106668 +106669,106669 +106670,106670 +106671,106671 +106672,106672 +245777,106672 +241785,106672 +106673,106673 +106674,106674 +106675,106675 +106676,106676 +106677,106677 +106678,106678 +106679,106679 +106680,106680 +106681,106681 +106682,106682 +106683,106683 +106684,106684 +106685,106685 +106686,106686 +106687,106687 +106688,106688 +106689,106689 +106690,106690 +106691,106691 +106692,106692 +162328,106692 +106693,106693 +106694,106694 +106695,106695 +106696,106696 +106697,106697 +106698,106698 +106699,106699 +106700,106700 +105066,106700 +106701,106701 +106702,106702 +106703,106703 +106704,106704 +106705,106705 +106706,106706 +106707,106707 +177500,106707 +171536,106707 +106708,106708 +106709,106709 +106710,106710 +106711,106711 +106712,106712 +106713,106713 +106714,106714 +106715,106715 +106716,106716 +106717,106717 +106718,106718 +106719,106719 +106720,106720 +106721,106721 +576,106721 +106722,106722 +106723,106723 +106724,106724 +106725,106725 +106726,106726 +106727,106727 +106728,106728 +106729,106729 +106730,106730 +106731,106731 +106732,106732 +106733,106733 +106734,106734 +106735,106735 +106736,106736 +106737,106737 +14465,106737 +209603,106737 +106738,106738 +106739,106739 +106740,106740 +106741,106741 +106742,106742 +106743,106743 +106744,106744 +106745,106745 +106746,106746 +225637,106746 +106747,106747 +106748,106748 +106749,106749 +106750,106750 +106751,106751 +106752,106752 +106753,106753 +106754,106754 +106755,106755 +106756,106756 +113418,106756 +106757,106757 +106758,106758 +106759,106759 +106760,106760 +106761,106761 +106762,106762 +106763,106763 +106764,106764 +106765,106765 +106766,106766 +106767,106767 +106768,106768 +106769,106769 +106770,106770 +106771,106771 +106772,106772 +106773,106773 +106774,106774 +106775,106775 +106776,106776 +106777,106777 +111689,106777 +106778,106778 +106779,106779 +106780,106780 +106781,106781 +106782,106782 +106783,106783 +106784,106784 +106785,106785 +106786,106786 +106787,106787 +106788,106788 +106789,106789 +21946,106789 +106790,106790 +106791,106791 +106792,106792 +106793,106793 +106794,106794 +106795,106795 +145061,106795 +106796,106796 +106797,106797 +159371,106797 +106798,106798 +106799,106799 +106800,106800 +106801,106801 +106802,106802 +106803,106803 +106804,106804 +106805,106805 +106806,106806 +106807,106807 +106808,106808 +106809,106809 +106810,106810 +106811,106811 +183932,106811 +106812,106812 +106813,106813 +106814,106814 +106815,106815 +106816,106816 +106817,106817 +106818,106818 +106819,106819 +106820,106820 +202498,106820 +106821,106821 +106822,106822 +106823,106823 +106824,106824 +106825,106825 +106826,106826 +106827,106827 +106828,106828 +106829,106829 +106830,106830 +106831,106831 +106832,106832 +106833,106833 +106834,106834 +106835,106835 +106836,106836 +106837,106837 +106838,106838 +106839,106839 +106840,106840 +106841,106841 +106842,106842 +106843,106843 +106844,106844 +106845,106845 +106846,106846 +106847,106847 +106848,106848 +106849,106849 +106850,106850 +106851,106851 +106852,106852 +106853,106853 +106854,106854 +106855,106855 +106856,106856 +106857,106857 +106858,106858 +106859,106859 +106860,106860 +106861,106861 +106862,106862 +106863,106863 +106864,106864 +106865,106865 +106866,106866 +106867,106867 +106868,106868 +106869,106869 +106870,106870 +106871,106871 +106872,106872 +106873,106873 +106874,106874 +106875,106875 +106876,106876 +106877,106877 +106878,106878 +106879,106879 +106880,106880 +106881,106881 +198736,106881 +106882,106882 +106883,106883 +106884,106884 +106885,106885 +106886,106886 +81512,106886 +106887,106887 +106888,106888 +106889,106889 +106890,106890 +106891,106891 +106892,106892 +106893,106893 +106894,106894 +106895,106895 +106896,106896 +106897,106897 +106898,106898 +106899,106899 +106900,106900 +106901,106901 +106902,106902 +106903,106903 +106904,106904 +106905,106905 +106906,106906 +106907,106907 +141994,106907 +106908,106908 +106909,106909 +106910,106910 +106911,106911 +106912,106912 +106913,106913 +106914,106914 +106915,106915 +106916,106916 +106917,106917 +111185,106917 +106918,106918 +106919,106919 +106920,106920 +106921,106921 +106922,106922 +106923,106923 +106924,106924 +106925,106925 +106926,106926 +106927,106927 +106928,106928 +106929,106929 +106930,106930 +106931,106931 +106932,106932 +106933,106933 +106934,106934 +106935,106935 +106936,106936 +106937,106937 +106938,106938 +106939,106939 +106940,106940 +106941,106941 +106942,106942 +106943,106943 +220035,106943 +106944,106944 +106945,106945 +34157,106945 +154848,106945 +106946,106946 +106947,106947 +106948,106948 +106949,106949 +106950,106950 +106951,106951 +106952,106952 +106953,106953 +92101,106953 +79767,106953 +106954,106954 +106955,106955 +106956,106956 +106957,106957 +106958,106958 +107381,106958 +106959,106959 +2965,106959 +106960,106960 +106961,106961 +106962,106962 +106963,106963 +106964,106964 +106965,106965 +106966,106966 +106967,106967 +106968,106968 +106969,106969 +106970,106970 +106971,106971 +106972,106972 +106973,106973 +106974,106974 +106975,106975 +106976,106976 +106977,106977 +106978,106978 +106979,106979 +106980,106980 +106981,106981 +106982,106982 +106983,106983 +106984,106984 +106985,106985 +214090,106985 +106986,106986 +106987,106987 +106988,106988 +106989,106989 +106990,106990 +106991,106991 +106992,106992 +106993,106993 +106994,106994 +106995,106995 +106996,106996 +106997,106997 +106998,106998 +106999,106999 +107000,107000 +107001,107001 +107002,107002 +142175,107003 +107003,107003 +107004,107004 +107005,107005 +107006,107006 +107007,107007 +107008,107008 +107009,107009 +107010,107010 +107011,107011 +107012,107012 +107013,107013 +107014,107014 +107015,107015 +107016,107016 +107017,107017 +107018,107018 +107019,107019 +107020,107020 +107021,107021 +107022,107022 +107023,107023 +107024,107024 +107025,107025 +107026,107026 +107027,107027 +92079,107027 +107028,107028 +107029,107029 +107030,107030 +107031,107031 +107032,107032 +107033,107033 +107034,107034 +107035,107035 +107036,107036 +107037,107037 +107038,107038 +107039,107039 +107040,107040 +164074,107040 +107041,107041 +107042,107042 +233243,107042 +107043,107043 +107044,107044 +107045,107045 +107046,107046 +60975,107046 +107047,107047 +107048,107048 +107049,107049 +107050,107050 +107051,107051 +107052,107052 +107053,107053 +54665,107053 +46561,107053 +107054,107054 +107055,107055 +173968,107055 +168004,107055 +107056,107056 +107057,107057 +107058,107058 +107059,107059 +107060,107060 +107061,107061 +107062,107062 +107063,107063 +52885,107063 +44781,107063 +107064,107064 +107065,107065 +107066,107066 +107067,107067 +107068,107068 +107069,107069 +107070,107070 +107071,107071 +107072,107072 +107073,107073 +107074,107074 +9115,107074 +107075,107075 +107076,107076 +107077,107077 +107078,107078 +107079,107079 +146731,107079 +107080,107080 +107081,107081 +107082,107082 +107083,107083 +107084,107084 +107085,107085 +107086,107086 +107087,107087 +107088,107088 +107089,107089 +107090,107090 +107091,107091 +107229,107091 +107092,107092 +107093,107093 +166538,107093 +107094,107094 +107095,107095 +176005,107095 +170041,107095 +107096,107096 +208303,107096 +107097,107097 +107098,107098 +107099,107099 +107100,107100 +148247,107100 +107101,107101 +107102,107102 +107103,107103 +107104,107104 +107105,107105 +107106,107106 +184185,107106 +107107,107107 +107108,107108 +107109,107109 +107110,107110 +107111,107111 +118186,107111 +107112,107112 +107113,107113 +107114,107114 +107115,107115 +183905,107115 +107116,107116 +107117,107117 +107118,107118 +107119,107119 +107120,107120 +107121,107121 +107122,107122 +107123,107123 +107124,107124 +107125,107125 +107126,107126 +28459,107126 +11563,107126 +107127,107127 +107128,107128 +107129,107129 +107130,107130 +107131,107131 +107132,107132 +107133,107133 +107134,107134 +139685,107135 +107135,107135 +107136,107136 +207227,107136 +107137,107137 +107138,107138 +107139,107139 +107140,107140 +107141,107141 +107142,107142 +107143,107143 +107144,107144 +107145,107145 +107146,107146 +107147,107147 +107148,107148 +107149,107149 +107150,107150 +107151,107151 +107152,107152 +107153,107153 +53439,107153 +45335,107153 +107154,107154 +107155,107155 +107156,107156 +107157,107157 +107158,107158 +107159,107159 +107160,107160 +107161,107161 +107162,107162 +107163,107163 +107164,107164 +107165,107165 +107166,107166 +107167,107167 +107168,107168 +107169,107169 +107170,107170 +107171,107171 +107172,107172 +107173,107173 +107174,107174 +245795,107174 +241803,107174 +107175,107175 +107176,107176 +107177,107177 +107178,107178 +107179,107179 +107180,107180 +107181,107181 +107182,107182 +107183,107183 +107184,107184 +199294,107184 +107185,107185 +107186,107186 +107187,107187 +107188,107188 +107189,107189 +107190,107190 +107191,107191 +107192,107192 +107193,107193 +107194,107194 +107195,107195 +107196,107196 +107197,107197 +107198,107198 +107199,107199 +38137,107199 +107200,107200 +107201,107201 +107202,107202 +107203,107203 +107204,107204 +107205,107205 +107206,107206 +107207,107207 +107208,107208 +107209,107209 +107210,107210 +107211,107211 +107212,107212 +107213,107213 +107214,107214 +107215,107215 +107216,107216 +107217,107217 +107218,107218 +107219,107219 +107220,107220 +107221,107221 +107222,107222 +107223,107223 +107224,107224 +107225,107225 +107226,107226 +107227,107227 +107228,107228 +107229,107229 +107091,107229 +107230,107230 +107231,107231 +107232,107232 +107233,107233 +107234,107234 +107235,107235 +208194,107235 +107236,107236 +107237,107237 +107238,107238 +107239,107239 +107240,107240 +107241,107241 +107242,107242 +107243,107243 +107244,107244 +107245,107245 +107246,107246 +107247,107247 +107248,107248 +107249,107249 +107250,107250 +107251,107251 +107252,107252 +107253,107253 +107254,107254 +107255,107255 +107256,107256 +107257,107257 +107258,107258 +107259,107259 +192400,107259 +107260,107260 +107261,107261 +107262,107262 +107263,107263 +179203,107263 +107264,107264 +107265,107265 +107266,107266 +234984,107266 +107267,107267 +107268,107268 +107269,107269 +107270,107270 +107271,107271 +107272,107272 +107273,107273 +107274,107274 +107275,107275 +107276,107276 +107277,107277 +165208,107277 +107278,107278 +107279,107279 +107280,107280 +107281,107281 +107282,107282 +107283,107283 +107284,107284 +107285,107285 +107286,107286 +107287,107287 +107288,107288 +107289,107289 +107290,107290 +107291,107291 +107292,107292 +107293,107293 +107294,107294 +107295,107295 +73051,107295 +107296,107296 +107297,107297 +107298,107298 +107299,107299 +107300,107300 +107301,107301 +107302,107302 +143148,107302 +141560,107302 +107303,107303 +107304,107304 +107305,107305 +107306,107306 +107307,107307 +107308,107308 +107309,107309 +107310,107310 +107311,107311 +107312,107312 +107313,107313 +107314,107314 +107315,107315 +107316,107316 +107317,107317 +107318,107318 +107319,107319 +107320,107320 +107321,107321 +107322,107322 +107323,107323 +107324,107324 +107325,107325 +107326,107326 +107327,107327 +107328,107328 +112229,107328 +107329,107329 +107330,107330 +107331,107331 +107332,107332 +178902,107332 +107333,107333 +107334,107334 +107335,107335 +107336,107336 +107337,107337 +107338,107338 +107339,107339 +107340,107340 +166764,107340 +107341,107341 +107342,107342 +107343,107343 +107344,107344 +107345,107345 +107346,107346 +107347,107347 +107348,107348 +107349,107349 +107350,107350 +107351,107351 +107352,107352 +107353,107353 +13503,107353 +107354,107354 +162879,107354 +107355,107355 +107356,107356 +107357,107357 +107358,107358 +107359,107359 +107360,107360 +107361,107361 +107362,107362 +107363,107363 +107364,107364 +107365,107365 +107366,107366 +107367,107367 +107368,107368 +107369,107369 +107370,107370 +107371,107371 +107372,107372 +107373,107373 +107374,107374 +107375,107375 +107376,107376 +107377,107377 +107378,107378 +107379,107379 +107380,107380 +107381,107381 +106958,107381 +107382,107382 +107383,107383 +107384,107384 +107385,107385 +107386,107386 +107387,107387 +107388,107388 +107389,107389 +107390,107390 +23724,107390 +107391,107391 +107392,107392 +107393,107393 +107394,107394 +107395,107395 +107396,107396 +107397,107397 +107398,107398 +86625,107398 +107399,107399 +107400,107400 +180509,107400 +107401,107401 +107402,107402 +107403,107403 +107404,107404 +107405,107405 +107406,107406 +107407,107407 +107408,107408 +107409,107409 +107410,107410 +107411,107411 +107412,107412 +107413,107413 +107414,107414 +107415,107415 +107416,107416 +107417,107417 +107418,107418 +107419,107419 +107420,107420 +107421,107421 +107422,107422 +107423,107423 +107424,107424 +107425,107425 +107426,107426 +107427,107427 +107428,107428 +107429,107429 +107430,107430 +107431,107431 +107432,107432 +107433,107433 +107434,107434 +107435,107435 +107436,107436 +107437,107437 +187270,107437 +107438,107438 +107439,107439 +107440,107440 +107441,107441 +107442,107442 +107443,107443 +107444,107444 +107445,107445 +107446,107446 +107447,107447 +107448,107448 +107449,107449 +107450,107450 +107451,107451 +107452,107452 +107453,107453 +107454,107454 +107455,107455 +58173,107455 +107456,107456 +107457,107457 +107458,107458 +107459,107459 +107460,107460 +107461,107461 +107462,107462 +107463,107463 +107464,107464 +107465,107465 +107466,107466 +107467,107467 +107468,107468 +107469,107469 +107470,107470 +107471,107471 +107472,107472 +107473,107473 +107474,107474 +107475,107475 +107476,107476 +107477,107477 +107478,107478 +161580,107478 +107479,107479 +107480,107480 +107481,107481 +107482,107482 +107483,107483 +107484,107484 +107485,107485 +107486,107486 +107487,107487 +107488,107488 +107489,107489 +107490,107490 +178279,107490 +172315,107490 +107491,107491 +107492,107492 +92125,107492 +107493,107493 +107494,107494 +107495,107495 +107496,107496 +107497,107497 +107498,107498 +107499,107499 +107500,107500 +107501,107501 +107502,107502 +107503,107503 +107504,107504 +107505,107505 +107506,107506 +107507,107507 +107508,107508 +107509,107509 +107510,107510 +107511,107511 +107512,107512 +107513,107513 +107514,107514 +107515,107515 +79983,107515 +107516,107516 +107517,107517 +107518,107518 +79076,107518 +107519,107519 +107520,107520 +107521,107521 +107522,107522 +107523,107523 +207927,107523 +107524,107524 +107525,107525 +79858,107525 +107526,107526 +107527,107527 +107528,107528 +107529,107529 +107530,107530 +107531,107531 +136141,107531 +128554,107531 +107532,107532 +107533,107533 +107534,107534 +107535,107535 +107536,107536 +107537,107537 +107538,107538 +107539,107539 +107540,107540 +107541,107541 +107542,107542 +107543,107543 +107544,107544 +244723,107544 +155245,107544 +107545,107545 +107546,107546 +107547,107547 +107548,107548 +178113,107548 +172149,107548 +107549,107549 +107550,107550 +200774,107551 +107551,107551 +107552,107552 +123051,107552 +107553,107553 +107554,107554 +107555,107555 +178120,107555 +172156,107555 +107556,107556 +156398,107557 +107557,107557 +145366,107557 +107558,107558 +222189,107558 +107559,107559 +107560,107560 +204755,107560 +107561,107561 +107562,107562 +107563,107563 +150805,107564 +107564,107564 +107565,107565 +213696,107565 +107566,107566 +107567,107567 +107568,107568 +107569,107569 +107570,107570 +107571,107571 +107572,107572 +107573,107573 +107574,107574 +107575,107575 +107576,107576 +107577,107577 +107578,107578 +107579,107579 +107580,107580 +107581,107581 +107582,107582 +107583,107583 +196712,107583 +107584,107584 +107585,107585 +107586,107586 +107587,107587 +107588,107588 +107589,107589 +107590,107590 +107591,107591 +107592,107592 +107593,107593 +203965,107594 +107594,107594 +107595,107595 +107596,107596 +107597,107597 +107598,107598 +107599,107599 +107600,107600 +107601,107601 +107602,107602 +107603,107603 +107604,107604 +107605,107605 +107606,107606 +107607,107607 +107608,107608 +107609,107609 +107610,107610 +107611,107611 +107612,107612 +107613,107613 +107614,107614 +107615,107615 +107616,107616 +107617,107617 +107618,107618 +107619,107619 +107620,107620 +107621,107621 +107622,107622 +107623,107623 +107624,107624 +107625,107625 +107626,107626 +107627,107627 +107628,107628 +107629,107629 +107630,107630 +164882,107630 +107631,107631 +107632,107632 +107633,107633 +107634,107634 +107635,107635 +107636,107636 +107637,107637 +107638,107638 +107639,107639 +107640,107640 +60379,107640 +138946,107640 +107641,107641 +107642,107642 +107643,107643 +107644,107644 +107645,107645 +107646,107646 +107647,107647 +107648,107648 +231199,107648 +107649,107649 +109214,107650 +107650,107650 +107651,107651 +54680,107651 +46576,107651 +107652,107652 +107653,107653 +107654,107654 +115957,107654 +107655,107655 +107656,107656 +107657,107657 +107658,107658 +107659,107659 +207815,107659 +107660,107660 +162529,107660 +107661,107661 +107961,107961 +107962,107962 +107963,107963 +107964,107964 +107965,107965 +107966,107966 +107967,107967 +107968,107968 +107969,107969 +34,107969 +214409,107969 +107970,107970 +107971,107971 +107972,107972 +107973,107973 +107974,107974 +107975,107975 +6186,107975 +107976,107976 +107977,107977 +107978,107978 +107979,107979 +107980,107980 +107981,107981 +108063,107981 +107982,107982 +107983,107983 +107984,107984 +192733,107984 +107985,107985 +107986,107986 +107987,107987 +107988,107988 +107989,107989 +107990,107990 +107991,107991 +107992,107992 +107993,107993 +107994,107994 +107995,107995 +107996,107996 +107997,107997 +107998,107998 +107999,107999 +108000,108000 +108001,108001 +108002,108002 +108003,108003 +108004,108004 +108005,108005 +108006,108006 +108007,108007 +108008,108008 +108009,108009 +108058,108058 +108059,108059 +26069,108059 +108060,108060 +108061,108061 +108062,108062 +108063,108063 +6789,108063 +107981,108063 +108064,108064 +108065,108065 +108066,108066 +75921,108066 +108067,108067 +192734,108067 +108068,108068 +108069,108069 +108070,108070 +108071,108071 +108175,108175 +108176,108176 +108177,108177 +108178,108178 +108179,108179 +108180,108180 +108181,108181 +108182,108182 +108183,108183 +108184,108184 +108185,108185 +108186,108186 +108187,108187 +226610,108188 +108188,108188 +108189,108189 +108190,108190 +108191,108191 +108192,108192 +108293,108293 +108294,108294 +104711,108294 +108295,108295 +108296,108296 +108297,108297 +108298,108298 +108299,108299 +108300,108300 +93405,108300 +108301,108301 +108302,108302 +108303,108303 +108304,108304 +108305,108305 +108306,108306 +201079,108306 +108307,108307 +108308,108308 +108309,108309 +108310,108310 +108311,108311 +108312,108312 +108313,108313 +148303,108313 +108314,108314 +108315,108315 +108316,108316 +108317,108317 +108318,108318 +108319,108319 +108320,108320 +108321,108321 +108322,108322 +108323,108323 +108324,108324 +108325,108325 +108326,108326 +108327,108327 +108328,108328 +11881,108328 +108329,108329 +108330,108330 +108331,108331 +108332,108332 +108333,108333 +108334,108334 +108374,108374 +108375,108375 +101981,108375 +108376,108376 +236349,108376 +109245,108376 +108377,108377 +83064,108377 +108431,108431 +108432,108432 +108433,108433 +108434,108434 +108435,108435 +112186,108436 +108436,108436 +108437,108437 +108438,108438 +109535,108438 +108439,108439 +108440,108440 +108441,108441 +79320,108441 +111014,108441 +108442,108442 +36875,108442 +108443,108443 +108444,108444 +108445,108445 +108446,108446 +98038,108446 +108447,108447 +97959,108447 +108448,108448 +178060,108449 +172096,108449 +108449,108449 +211776,108449 +108450,108450 +108451,108451 +108452,108452 +108453,108453 +108454,108454 +108455,108455 +108456,108456 +108457,108457 +189821,108457 +108458,108458 +108459,108459 +108460,108460 +108461,108461 +108462,108462 +108463,108463 +108464,108464 +108465,108465 +108466,108466 +108467,108467 +108468,108468 +108469,108469 +108470,108470 +108471,108471 +108472,108472 +108473,108473 +108474,108474 +108475,108475 +108476,108476 +108477,108477 +108478,108478 +108479,108479 +108480,108480 +164996,108480 +108481,108481 +108482,108482 +108483,108483 +108484,108484 +108485,108485 +205900,108485 +108486,108486 +108487,108487 +108488,108488 +104746,108488 +108489,108489 +108490,108490 +108491,108491 +108492,108492 +211473,108492 +108493,108493 +108494,108494 +108495,108495 +108496,108496 +108497,108497 +108498,108498 +108499,108499 +108500,108500 +108501,108501 +108502,108502 +108503,108503 +108504,108504 +108505,108505 +68193,108505 +108506,108506 +108507,108507 +108508,108508 +108509,108509 +246045,108509 +108510,108510 +108511,108511 +108512,108512 +108513,108513 +108514,108514 +108515,108515 +108516,108516 +108517,108517 +108518,108518 +108519,108519 +108520,108520 +108521,108521 +108522,108522 +108523,108523 +108524,108524 +2709,108524 +108525,108525 +108526,108526 +108527,108527 +108528,108528 +108529,108529 +108530,108530 +108531,108531 +108532,108532 +108533,108533 +108534,108534 +8789,108534 +108535,108535 +108536,108536 +108537,108537 +108538,108538 +108539,108539 +108540,108540 +215910,108540 +108541,108541 +108542,108542 +108543,108543 +108544,108544 +108545,108545 +9095,108545 +108546,108546 +108547,108547 +108548,108548 +108549,108549 +108550,108550 +108551,108551 +108552,108552 +215352,108552 +108553,108553 +108554,108554 +108555,108555 +108556,108556 +108557,108557 +108558,108558 +108559,108559 +108560,108560 +108561,108561 +108562,108562 +67128,108562 +108563,108563 +108564,108564 +108565,108565 +108566,108566 +108567,108567 +108568,108568 +108569,108569 +108570,108570 +108571,108571 +108572,108572 +108573,108573 +108574,108574 +108575,108575 +108576,108576 +108577,108577 +108578,108578 +108579,108579 +108580,108580 +108581,108581 +108582,108582 +108583,108583 +108584,108584 +108585,108585 +96475,108585 +108586,108586 +108587,108587 +108588,108588 +108589,108589 +108590,108590 +190755,108590 +108591,108591 +58060,108591 +57767,108591 +108592,108592 +98818,108592 +108593,108593 +108594,108594 +108595,108595 +108596,108596 +108597,108597 +108598,108598 +98041,108598 +108599,108599 +108600,108600 +108601,108601 +108602,108602 +108603,108603 +965,108603 +108604,108604 +108605,108605 +108606,108606 +108607,108607 +108608,108608 +108609,108609 +108610,108610 +108611,108611 +108612,108612 +108613,108613 +108614,108614 +108615,108615 +108616,108616 +108617,108617 +108618,108618 +108619,108619 +108620,108620 +108621,108621 +108622,108622 +108623,108623 +108624,108624 +108625,108625 +108626,108626 +108627,108627 +108628,108628 +108629,108629 +108630,108630 +108631,108631 +108632,108632 +108633,108633 +108634,108634 +108635,108635 +108636,108636 +108637,108637 +108638,108638 +108639,108639 +108640,108640 +108641,108641 +108642,108642 +108643,108643 +108644,108644 +108645,108645 +177962,108645 +171998,108645 +108646,108646 +108647,108647 +108648,108648 +108649,108649 +108650,108650 +108651,108651 +165195,108652 +108652,108652 +108653,108653 +108654,108654 +108655,108655 +108656,108656 +108657,108657 +108658,108658 +108659,108659 +108660,108660 +108661,108661 +108662,108662 +108663,108663 +108664,108664 +108665,108665 +108666,108666 +85210,108666 +108667,108667 +97937,108667 +108668,108668 +53360,108668 +45256,108668 +108669,108669 +134191,108669 +126604,108669 +108670,108670 +108671,108671 +82019,108671 +108672,108672 +108673,108673 +108674,108674 +108675,108675 +108676,108676 +108677,108677 +208142,108677 +108695,108677 +108678,108678 +92649,108678 +10202,108678 +108679,108679 +108680,108680 +108681,108681 +108682,108682 +225275,108682 +108683,108683 +108684,108684 +108685,108685 +108686,108686 +108687,108687 +108688,108688 +110686,108688 +108689,108689 +108690,108690 +108691,108691 +108692,108692 +108693,108693 +108694,108694 +108695,108695 +108677,108695 +177478,108695 +171514,108695 +108696,108696 +108697,108697 +108698,108698 +30894,108698 +108699,108699 +108700,108700 +108701,108701 +108702,108702 +108703,108703 +208780,108703 +108704,108704 +227570,108704 +108705,108705 +108706,108706 +108707,108707 +108708,108708 +108709,108709 +23885,108709 +108710,108710 +108711,108711 +108712,108712 +108713,108713 +108714,108714 +108715,108715 +108716,108716 +160754,108716 +108717,108717 +108718,108718 +108719,108719 +108720,108720 +108721,108721 +108722,108722 +108723,108723 +108724,108724 +108725,108725 +108726,108726 +108727,108727 +179017,108727 +108728,108728 +108729,108729 +141691,108729 +108730,108730 +108731,108731 +108732,108732 +108733,108733 +108734,108734 +108735,108735 +108736,108736 +108737,108737 +108738,108738 +108739,108739 +108740,108740 +108741,108741 +108742,108742 +108743,108743 +108744,108744 +108745,108745 +108746,108746 +108747,108747 +108748,108748 +108749,108749 +108750,108750 +108751,108751 +108752,108752 +108753,108753 +108754,108754 +108755,108755 +59670,108755 +108756,108756 +108757,108757 +108758,108758 +108759,108759 +108760,108760 +108761,108761 +108762,108762 +108763,108763 +108764,108764 +108765,108765 +108766,108766 +9940,108766 +108767,108767 +108768,108768 +180830,108768 +108769,108769 +108770,108770 +108771,108771 +108772,108772 +108773,108773 +108774,108774 +108775,108775 +108776,108776 +108777,108777 +108778,108778 +108779,108779 +108780,108780 +108781,108781 +108782,108782 +108783,108783 +179218,108783 +108784,108784 +108785,108785 +108786,108786 +108787,108787 +108788,108788 +108789,108789 +59192,108789 +58818,108789 +108790,108790 +108791,108791 +108792,108792 +77558,108792 +108793,108793 +246974,108793 +108794,108794 +108795,108795 +108796,108796 +108797,108797 +108798,108798 +108799,108799 +108800,108800 +108801,108801 +108802,108802 +108803,108803 +108804,108804 +108805,108805 +108806,108806 +108843,108843 +108844,108844 +108845,108845 +108846,108846 +6852,108846 +108847,108847 +108848,108848 +108849,108849 +108850,108850 +108851,108851 +108852,108852 +108853,108853 +108854,108854 +108855,108855 +108856,108856 +108857,108857 +108858,108858 +108859,108859 +180908,108859 +108860,108860 +108861,108861 +108862,108862 +108863,108863 +244826,108864 +108864,108864 +108865,108865 +108866,108866 +108867,108867 +108868,108868 +108869,108869 +108870,108870 +108871,108871 +108872,108872 +108873,108873 +108874,108874 +108875,108875 +108876,108876 +108877,108877 +108878,108878 +108879,108879 +108880,108880 +215938,108880 +108881,108881 +108882,108882 +108883,108883 +108884,108884 +108885,108885 +108886,108886 +108887,108887 +108888,108888 +108889,108889 +108890,108890 +108891,108891 +108892,108892 +108893,108893 +108894,108894 +108895,108895 +108896,108896 +104358,108896 +108897,108897 +108898,108898 +108899,108899 +108900,108900 +178665,108901 +108901,108901 +108902,108902 +108903,108903 +58034,108903 +108904,108904 +108905,108905 +108906,108906 +108907,108907 +108908,108908 +108909,108909 +108910,108910 +108911,108911 +108912,108912 +20056,108912 +108913,108913 +108914,108914 +108915,108915 +108916,108916 +108917,108917 +108918,108918 +108919,108919 +108920,108920 +108921,108921 +108922,108922 +108923,108923 +108924,108924 +108925,108925 +108926,108926 +108927,108927 +164768,108927 +108928,108928 +108929,108929 +108930,108930 +108931,108931 +147151,108931 +108932,108932 +108933,108933 +108934,108934 +108935,108935 +108936,108936 +108937,108937 +108938,108938 +108939,108939 +108940,108940 +108941,108941 +108942,108942 +108943,108943 +108944,108944 +108945,108945 +108946,108946 +108947,108947 +108948,108948 +181061,108948 +108949,108949 +108950,108950 +108951,108951 +108952,108952 +108953,108953 +108954,108954 +108955,108955 +108956,108956 +108957,108957 +108958,108958 +108959,108959 +108960,108960 +108961,108961 +108962,108962 +108963,108963 +108964,108964 +108965,108965 +8583,108965 +8403,108965 +108966,108966 +108967,108967 +108968,108968 +66979,108968 +108969,108969 +108970,108970 +108971,108971 +108972,108972 +108973,108973 +108974,108974 +108975,108975 +108976,108976 +108977,108977 +108978,108978 +108979,108979 +108980,108980 +205195,108980 +108981,108981 +108982,108982 +214507,108983 +108983,108983 +108984,108984 +108985,108985 +108986,108986 +108987,108987 +108988,108988 +108989,108989 +108990,108990 +108991,108991 +108992,108992 +108993,108993 +108994,108994 +108995,108995 +108996,108996 +108997,108997 +108998,108998 +108999,108999 +109000,109000 +109001,109001 +146835,109001 +109002,109002 +109003,109003 +109004,109004 +109005,109005 +109006,109006 +109007,109007 +109008,109008 +109009,109009 +109010,109010 +109011,109011 +109012,109012 +109013,109013 +109014,109014 +109015,109015 +109016,109016 +109017,109017 +109018,109018 +109019,109019 +109020,109020 +109021,109021 +109022,109022 +109023,109023 +109024,109024 +109025,109025 +109026,109026 +109027,109027 +109028,109028 +180833,109028 +109029,109029 +109030,109030 +109031,109031 +148382,109031 +109032,109032 +109033,109033 +109034,109034 +109035,109035 +109036,109036 +109037,109037 +109038,109038 +229110,109038 +109039,109039 +109040,109040 +109041,109041 +109042,109042 +109043,109043 +109044,109044 +109045,109045 +109046,109046 +109047,109047 +109048,109048 +109049,109049 +213394,109049 +109050,109050 +109051,109051 +109052,109052 +109053,109053 +109054,109054 +109055,109055 +109056,109056 +109057,109057 +109058,109058 +109059,109059 +109060,109060 +109061,109061 +109062,109062 +109063,109063 +109064,109064 +109065,109065 +109066,109066 +109067,109067 +109068,109068 +109069,109069 +109070,109070 +109071,109071 +109072,109072 +109073,109073 +109074,109074 +109075,109075 +109076,109076 +109077,109077 +109078,109078 +109079,109079 +109080,109080 +109081,109081 +109082,109082 +109083,109083 +109084,109084 +109085,109085 +109086,109086 +109087,109087 +109088,109088 +109089,109089 +109090,109090 +109091,109091 +109092,109092 +109093,109093 +109094,109094 +109095,109095 +109096,109096 +109097,109097 +109098,109098 +109099,109099 +68029,109099 +109100,109100 +109101,109101 +109102,109102 +109103,109103 +109104,109104 +109105,109105 +109106,109106 +109107,109107 +109108,109108 +109109,109109 +109110,109110 +109111,109111 +109112,109112 +109113,109113 +109114,109114 +109115,109115 +109116,109116 +109117,109117 +109118,109118 +109119,109119 +109120,109120 +109121,109121 +109122,109122 +109123,109123 +109124,109124 +109125,109125 +109126,109126 +2461,109126 +109127,109127 +2462,109127 +109128,109128 +79483,109128 +109129,109129 +109130,109130 +109131,109131 +109132,109132 +109133,109133 +109134,109134 +109135,109135 +109136,109136 +109137,109137 +89219,109137 +109138,109138 +109139,109139 +109140,109140 +109141,109141 +109142,109142 +109143,109143 +109144,109144 +109145,109145 +109146,109146 +109147,109147 +109148,109148 +109149,109149 +207162,109149 +109150,109150 +14112,109150 +109151,109151 +109152,109152 +109153,109153 +109154,109154 +109155,109155 +109156,109156 +109157,109157 +109158,109158 +109159,109159 +109160,109160 +75565,109160 +109161,109161 +109162,109162 +109163,109163 +109164,109164 +109165,109165 +109166,109166 +109167,109167 +109168,109168 +109169,109169 +109170,109170 +109171,109171 +109172,109172 +109173,109173 +109212,109212 +109213,109213 +109214,109214 +107650,109214 +109215,109215 +199560,109215 +109216,109216 +109217,109217 +154791,109217 +109218,109218 +109219,109219 +109220,109220 +109221,109221 +109222,109222 +109223,109223 +109224,109224 +109225,109225 +109226,109226 +179524,109226 +109227,109227 +109228,109228 +136980,109228 +129393,109228 +109229,109229 +109230,109230 +109231,109231 +109232,109232 +109233,109233 +109234,109234 +109235,109235 +109236,109236 +109237,109237 +109238,109238 +109239,109239 +109240,109240 +109241,109241 +109242,109242 +109243,109243 +109244,109244 +236349,109245 +109245,109245 +108376,109245 +109246,109246 +109247,109247 +109248,109248 +109249,109249 +109250,109250 +109251,109251 +109252,109252 +109253,109253 +235679,109253 +109254,109254 +109255,109255 +109256,109256 +109257,109257 +109258,109258 +109259,109259 +109260,109260 +109261,109261 +7345,109261 +211803,109261 +109262,109262 +109263,109263 +109264,109264 +83042,109264 +109265,109265 +109266,109266 +109267,109267 +33432,109267 +75570,109267 +176573,109267 +170609,109267 +109268,109268 +109269,109269 +109270,109270 +109271,109271 +109272,109272 +109273,109273 +109274,109274 +109275,109275 +109276,109276 +109277,109277 +109278,109278 +109279,109279 +109280,109280 +109281,109281 +109282,109282 +109283,109283 +205220,109283 +109284,109284 +109285,109285 +109286,109286 +109287,109287 +70526,109287 +109288,109288 +109289,109289 +109516,109516 +109517,109517 +109518,109518 +83982,109518 +109519,109519 +109520,109520 +109521,109521 +109522,109522 +14576,109522 +109523,109523 +109524,109524 +109525,109525 +82352,109525 +11382,109525 +202394,109526 +109526,109526 +109527,109527 +109528,109528 +5604,109528 +109529,109529 +109530,109530 +109531,109531 +93613,109531 +109532,109532 +109533,109533 +109534,109534 +109535,109535 +108438,109535 +109536,109536 +109537,109537 +109538,109538 +109539,109539 +109540,109540 +109541,109541 +109542,109542 +109543,109543 +109544,109544 +109545,109545 +109546,109546 +109547,109547 +109548,109548 +109549,109549 +135287,109549 +127700,109549 +109550,109550 +109551,109551 +109552,109552 +109553,109553 +109554,109554 +109555,109555 +14012,109555 +109556,109556 +109557,109557 +109558,109558 +83061,109558 +109559,109559 +109560,109560 +109561,109561 +109562,109562 +109563,109563 +109564,109564 +109565,109565 +109566,109566 +109567,109567 +109616,109568 +109568,109568 +109569,109569 +109570,109570 +109571,109571 +109572,109572 +109573,109573 +109574,109574 +109575,109575 +109576,109576 +109577,109577 +109578,109578 +109579,109579 +109580,109580 +109581,109581 +109582,109582 +109583,109583 +109584,109584 +109585,109585 +109586,109586 +109587,109587 +109588,109588 +109589,109589 +109590,109590 +109591,109591 +109592,109592 +151365,109592 +109593,109593 +109594,109594 +109595,109595 +109596,109596 +109597,109597 +109598,109598 +109599,109599 +109600,109600 +9937,109600 +109601,109601 +109602,109602 +109603,109603 +109604,109604 +180722,109604 +109605,109605 +109606,109606 +109607,109607 +109608,109608 +109609,109609 +109610,109610 +109611,109611 +109612,109612 +109613,109613 +109614,109614 +109615,109615 +35725,109615 +109616,109616 +109568,109616 +109760,109760 +12154,109760 +109761,109761 +110305,109762 +109762,109762 +109800,109800 +109801,109801 +109802,109802 +109803,109803 +109804,109804 +109805,109805 +109806,109806 +109807,109807 +109808,109808 +109809,109809 +109810,109810 +109811,109811 +110601,109811 +109812,109812 +109813,109813 +109814,109814 +109815,109815 +109816,109816 +183014,109816 +109817,109817 +139760,109817 +230358,109818 +109818,109818 +109819,109819 +114171,109819 +109820,109820 +109821,109821 +110299,109821 +109822,109822 +8132,109822 +109823,109823 +109824,109824 +29896,109824 +109825,109825 +77591,109825 +109826,109826 +109827,109827 +135756,109828 +128169,109828 +109828,109828 +155936,109828 +109829,109829 +109830,109830 +4990,109830 +109831,109831 +109832,109832 +109873,109873 +109874,109874 +158040,109874 +109875,109875 +109876,109876 +109877,109877 +109878,109878 +109879,109879 +109880,109880 +109881,109881 +109882,109882 +110370,109883 +109883,109883 +109884,109884 +109885,109885 +109886,109886 +109887,109887 +109888,109888 +109889,109889 +109890,109890 +221196,109890 +109891,109891 +109892,109892 +109893,109893 +109894,109894 +96463,109894 +109895,109895 +109896,109896 +109897,109897 +109898,109898 +109899,109899 +109900,109900 +109901,109901 +109902,109902 +109903,109903 +109904,109904 +109905,109905 +109906,109906 +162229,109907 +109907,109907 +109908,109908 +109909,109909 +109910,109910 +109911,109911 +109912,109912 +109913,109913 +109914,109914 +109915,109915 +109916,109916 +109917,109917 +110384,109918 +109918,109918 +109919,109919 +109920,109920 +109921,109921 +109922,109922 +109923,109923 +109924,109924 +109925,109925 +109926,109926 +109927,109927 +109928,109928 +15049,109928 +109968,109968 +110327,109968 +109969,109969 +110328,109969 +109970,109970 +109971,109971 +109972,109972 +110331,109972 +109973,109973 +110332,109973 +109974,109974 +109975,109975 +109976,109976 +109977,109977 +109978,109978 +109979,109979 +109980,109980 +109981,109981 +109982,109982 +109983,109983 +109984,109984 +109985,109985 +109986,109986 +109987,109987 +109988,109988 +109989,109989 +110040,110040 +110041,110041 +110042,110042 +110043,110043 +110044,110044 +110045,110045 +110046,110046 +110047,110047 +110048,110048 +110049,110049 +110050,110050 +194187,110050 +110051,110051 +110052,110052 +110053,110053 +110054,110054 +110055,110055 +110056,110056 +110057,110057 +103695,110057 +110058,110058 +110059,110059 +110060,110060 +110061,110061 +110062,110062 +110063,110063 +110064,110064 +110065,110065 +110066,110066 +110067,110067 +110068,110068 +110069,110069 +110070,110070 +194099,110070 +110071,110071 +110072,110072 +110073,110073 +110074,110074 +110075,110075 +110076,110076 +110077,110077 +98335,110077 +110078,110078 +110079,110079 +110080,110080 +110081,110081 +110082,110082 +110083,110083 +110084,110084 +110085,110085 +110086,110086 +110087,110087 +202806,110087 +110088,110088 +222943,110088 +160724,110088 +110089,110089 +110090,110090 +110091,110091 +110092,110092 +110093,110093 +110094,110094 +110095,110095 +187952,110095 +110096,110096 +110097,110097 +110098,110098 +110099,110099 +203005,110099 +110100,110100 +110101,110101 +110102,110102 +110103,110103 +110104,110104 +110105,110105 +110106,110106 +110107,110107 +110108,110108 +110109,110109 +110110,110110 +110111,110111 +188032,110112 +110112,110112 +110274,110274 +110275,110275 +110276,110276 +110277,110277 +88082,110277 +110278,110278 +110279,110279 +110280,110280 +110281,110281 +110282,110282 +110283,110283 +110284,110284 +151402,110284 +110285,110285 +110286,110286 +110287,110287 +110288,110288 +110289,110289 +110290,110290 +110291,110291 +110292,110292 +110293,110293 +110294,110294 +110295,110295 +110296,110296 +110297,110297 +110298,110298 +110299,110299 +109821,110299 +242288,110299 +110300,110300 +110301,110301 +110302,110302 +110303,110303 +26880,110303 +110304,110304 +110305,110305 +109762,110305 +110306,110306 +15594,110306 +146974,110306 +110307,110307 +110308,110308 +110309,110309 +110310,110310 +110311,110311 +110312,110312 +110313,110313 +110314,110314 +110315,110315 +110316,110316 +110317,110317 +110318,110318 +110319,110319 +110320,110320 +110321,110321 +110322,110322 +90252,110322 +110323,110323 +110324,110324 +110325,110325 +110326,110326 +110327,110327 +109968,110327 +110328,110328 +109969,110328 +110329,110329 +110330,110330 +110331,110331 +109972,110331 +110332,110332 +53152,110332 +45048,110332 +109973,110332 +110333,110333 +63169,110333 +110334,110334 +68633,110334 +104042,110334 +110335,110335 +110336,110336 +110337,110337 +110338,110338 +110339,110339 +110340,110340 +110341,110341 +110342,110342 +110343,110343 +110344,110344 +110345,110345 +110346,110346 +110347,110347 +110348,110348 +110349,110349 +232938,110349 +110350,110350 +110351,110351 +110352,110352 +110353,110353 +110354,110354 +110355,110355 +110356,110356 +110357,110357 +81616,110357 +110358,110358 +110359,110359 +110360,110360 +110361,110361 +110362,110362 +110363,110363 +110364,110364 +110365,110365 +110366,110366 +110367,110367 +110368,110368 +110369,110369 +110370,110370 +109883,110370 +110371,110371 +110372,110372 +110373,110373 +110374,110374 +110375,110375 +110376,110376 +110377,110377 +110378,110378 +110379,110379 +110380,110380 +110381,110381 +110382,110382 +110383,110383 +110384,110384 +109918,110384 +110385,110385 +110386,110386 +110387,110387 +110388,110388 +110389,110389 +110390,110390 +110391,110391 +110513,110513 +110514,110514 +238827,110514 +110515,110515 +110516,110516 +110517,110517 +110518,110518 +110519,110519 +110520,110520 +110521,110521 +110522,110522 +104712,110522 +110523,110523 +110524,110524 +110525,110525 +110526,110526 +110527,110527 +110528,110528 +234898,110528 +110529,110529 +110530,110530 +110531,110531 +110532,110532 +110533,110533 +110534,110534 +110535,110535 +110536,110536 +110537,110537 +110538,110538 +110539,110539 +110540,110540 +110541,110541 +157535,110541 +110542,110542 +141694,110542 +110578,110578 +110579,110579 +110580,110580 +110581,110581 +110582,110582 +110583,110583 +110584,110584 +110585,110585 +110586,110586 +110587,110587 +110588,110588 +110589,110589 +110590,110590 +16179,110590 +110591,110591 +110592,110592 +110593,110593 +110594,110594 +110595,110595 +101764,110595 +110596,110596 +26294,110596 +110597,110597 +110598,110598 +110599,110599 +110600,110600 +59024,110600 +58650,110600 +110601,110601 +109811,110601 +110602,110602 +110603,110603 +110604,110604 +236938,110604 +110605,110605 +176590,110606 +170626,110606 +110606,110606 +110607,110607 +110608,110608 +238824,110608 +110609,110609 +110610,110610 +6904,110610 +110611,110611 +110612,110612 +6196,110612 +79955,110612 +188981,110612 +110613,110613 +123393,110613 +110614,110614 +110615,110615 +110616,110616 +110617,110617 +173987,110617 +168023,110617 +110618,110618 +110619,110619 +110620,110620 +110621,110621 +110622,110622 +110623,110623 +110624,110624 +57581,110624 +110625,110625 +110626,110626 +110627,110627 +110628,110628 +110629,110629 +110630,110630 +110631,110631 +110632,110632 +110633,110633 +110634,110634 +110635,110635 +110636,110636 +110637,110637 +110638,110638 +110639,110639 +110640,110640 +110641,110641 +110642,110642 +110643,110643 +110644,110644 +110645,110645 +110646,110646 +110647,110647 +194104,110648 +110648,110648 +110649,110649 +110650,110650 +110651,110651 +110652,110652 +7223,110652 +110653,110653 +179152,110653 +110654,110654 +110655,110655 +110656,110656 +184143,110656 +85599,110656 +110657,110657 +110658,110658 +110659,110659 +110660,110660 +110661,110661 +110662,110662 +110663,110663 +110664,110664 +110665,110665 +110666,110666 +110667,110667 +110668,110668 +110669,110669 +110670,110670 +110671,110671 +122254,110671 +110672,110672 +110673,110673 +5382,110673 +110674,110674 +93550,110674 +110675,110675 +110676,110676 +110677,110677 +89492,110677 +110678,110678 +110679,110679 +110680,110680 +110681,110681 +110682,110682 +110683,110683 +110684,110684 +110685,110685 +59444,110685 +110686,110686 +108688,110686 +110687,110687 +110688,110688 +110689,110689 +110690,110690 +110691,110691 +110692,110692 +110693,110693 +110694,110694 +110695,110695 +110696,110696 +110697,110697 +110698,110698 +110699,110699 +110700,110700 +110701,110701 +110702,110702 +110703,110703 +110704,110704 +110705,110705 +110706,110706 +110707,110707 +176729,110707 +170765,110707 +19119,110707 +110708,110708 +110709,110709 +110710,110710 +6182,110710 +110711,110711 +110712,110712 +110713,110713 +110714,110714 +110715,110715 +110716,110716 +110717,110717 +110718,110718 +110719,110719 +110720,110720 +149874,110720 +110721,110721 +110722,110722 +110723,110723 +110724,110724 +110725,110725 +110726,110726 +110727,110727 +110728,110728 +110729,110729 +110730,110730 +110731,110731 +110732,110732 +110733,110733 +110734,110734 +110735,110735 +110736,110736 +110737,110737 +110738,110738 +110739,110739 +110740,110740 +110741,110741 +110742,110742 +110743,110743 +110744,110744 +110745,110745 +110746,110746 +110747,110747 +110748,110748 +110749,110749 +110750,110750 +110751,110751 +110752,110752 +110753,110753 +110754,110754 +110755,110755 +110756,110756 +110757,110757 +110758,110758 +110759,110759 +249417,110759 +178142,110760 +172178,110760 +110760,110760 +110761,110761 +110762,110762 +110763,110763 +110764,110764 +110765,110765 +110766,110766 +110767,110767 +110768,110768 +110769,110769 +110770,110770 +110771,110771 +110772,110772 +110773,110773 +110774,110774 +110775,110775 +110776,110776 +156689,110776 +110777,110777 +110778,110778 +110779,110779 +110780,110780 +110781,110781 +110782,110782 +110783,110783 +110784,110784 +110785,110785 +80299,110785 +110786,110786 +110787,110787 +110788,110788 +110789,110789 +110790,110790 +110791,110791 +110792,110792 +110793,110793 +110794,110794 +110795,110795 +110796,110796 +184483,110797 +110797,110797 +110798,110798 +110799,110799 +110800,110800 +234734,110800 +110801,110801 +110802,110802 +110803,110803 +110804,110804 +110805,110805 +12652,110805 +143010,110806 +110806,110806 +110807,110807 +143011,110807 +110808,110808 +110809,110809 +110810,110810 +110811,110811 +110812,110812 +110813,110813 +110814,110814 +10481,110814 +110815,110815 +110816,110816 +196482,110817 +110817,110817 +189987,110817 +110818,110818 +110819,110819 +110820,110820 +110821,110821 +110822,110822 +211190,110822 +110823,110823 +110824,110824 +110825,110825 +110826,110826 +110827,110827 +110828,110828 +110829,110829 +110830,110830 +110831,110831 +104728,110831 +110832,110832 +110833,110833 +110834,110834 +110835,110835 +85823,110835 +110836,110836 +110837,110837 +110838,110838 +110839,110839 +110840,110840 +222993,110840 +110841,110841 +26328,110841 +110842,110842 +110843,110843 +110844,110844 +110845,110845 +202033,110846 +110846,110846 +110847,110847 +110848,110848 +110849,110849 +110850,110850 +110851,110851 +110852,110852 +110853,110853 +110854,110854 +110855,110855 +110856,110856 +110857,110857 +110858,110858 +55096,110858 +46992,110858 +110859,110859 +110860,110860 +110861,110861 +110862,110862 +110863,110863 +110864,110864 +110865,110865 +106526,110865 +110866,110866 +110867,110867 +110868,110868 +110869,110869 +110870,110870 +110871,110871 +116751,110871 +110872,110872 +110873,110873 +110874,110874 +110875,110875 +110876,110876 +110877,110877 +110878,110878 +110879,110879 +110880,110880 +110881,110881 +110882,110882 +110883,110883 +110884,110884 +110885,110885 +110886,110886 +110887,110887 +110888,110888 +110889,110889 +110890,110890 +110891,110891 +110892,110892 +110893,110893 +110894,110894 +110895,110895 +110896,110896 +110897,110897 +110898,110898 +110899,110899 +110900,110900 +110901,110901 +110902,110902 +110903,110903 +110904,110904 +110905,110905 +110906,110906 +110907,110907 +53232,110907 +45128,110907 +110908,110908 +110909,110909 +110910,110910 +110911,110911 +110912,110912 +110913,110913 +53779,110913 +45675,110913 +39969,110913 +110914,110914 +110915,110915 +110916,110916 +110917,110917 +110918,110918 +110919,110919 +110920,110920 +110921,110921 +110922,110922 +110923,110923 +110924,110924 +110925,110925 +36088,110925 +110926,110926 +110927,110927 +110928,110928 +59123,110928 +58749,110928 +110929,110929 +110930,110930 +110931,110931 +110932,110932 +110933,110933 +110934,110934 +110935,110935 +110936,110936 +110937,110937 +110938,110938 +110939,110939 +110940,110940 +101952,110940 +110941,110941 +110942,110942 +110943,110943 +110944,110944 +233750,110944 +110945,110945 +110946,110946 +110947,110947 +110948,110948 +56790,110948 +110949,110949 +110950,110950 +110951,110951 +110952,110952 +181105,110952 +110953,110953 +110954,110954 +110955,110955 +110956,110956 +110957,110957 +110958,110958 +110959,110959 +110960,110960 +110961,110961 +34220,110961 +110962,110962 +110963,110963 +110964,110964 +110965,110965 +110966,110966 +110967,110967 +110968,110968 +110969,110969 +110970,110970 +110971,110971 +110972,110972 +123399,110972 +110973,110973 +110974,110974 +110975,110975 +110976,110976 +110977,110977 +110978,110978 +78800,110978 +110979,110979 +110980,110980 +110981,110981 +110982,110982 +110983,110983 +110984,110984 +110985,110985 +110986,110986 +110987,110987 +110988,110988 +110989,110989 +110990,110990 +110991,110991 +110992,110992 +110993,110993 +110994,110994 +110995,110995 +110996,110996 +110997,110997 +110998,110998 +110999,110999 +111000,111000 +111001,111001 +111002,111002 +111003,111003 +111004,111004 +111005,111005 +111006,111006 +111007,111007 +111008,111008 +111009,111009 +111010,111010 +111011,111011 +57757,111011 +111012,111012 +111013,111013 +111014,111014 +108441,111014 +111015,111015 +111016,111016 +58406,111016 +111017,111017 +111018,111018 +132626,111018 +125039,111018 +111019,111019 +111020,111020 +79317,111020 +111021,111021 +111022,111022 +111023,111023 +111024,111024 +111025,111025 +111026,111026 +111027,111027 +111028,111028 +111029,111029 +111030,111030 +111031,111031 +111032,111032 +111033,111033 +212345,111033 +111034,111034 +111035,111035 +111036,111036 +111037,111037 +111038,111038 +111039,111039 +111040,111040 +111041,111041 +111042,111042 +111043,111043 +111044,111044 +111045,111045 +111046,111046 +111047,111047 +111048,111048 +111049,111049 +111050,111050 +111051,111051 +111052,111052 +111053,111053 +111054,111054 +111055,111055 +111056,111056 +111057,111057 +111058,111058 +111059,111059 +111060,111060 +111061,111061 +111062,111062 +28130,111062 +111063,111063 +111064,111064 +111065,111065 +111066,111066 +111067,111067 +111068,111068 +111069,111069 +111070,111070 +111071,111071 +111072,111072 +163367,111072 +111073,111073 +111074,111074 +111075,111075 +111076,111076 +111077,111077 +111078,111078 +111079,111079 +111080,111080 +111081,111081 +111082,111082 +111083,111083 +111084,111084 +111085,111085 +111086,111086 +111087,111087 +111088,111088 +111089,111089 +111090,111090 +89619,111090 +111091,111091 +111092,111092 +111093,111093 +111094,111094 +111095,111095 +111096,111096 +111097,111097 +111098,111098 +111099,111099 +142567,111099 +111100,111100 +111101,111101 +111102,111102 +111103,111103 +226120,111103 +111104,111104 +111105,111105 +111106,111106 +111107,111107 +111108,111108 +111109,111109 +111110,111110 +111111,111111 +111112,111112 +111113,111113 +111114,111114 +111115,111115 +111116,111116 +146893,111117 +111117,111117 +111118,111118 +111119,111119 +111120,111120 +111121,111121 +111122,111122 +111123,111123 +111124,111124 +111125,111125 +111126,111126 +111127,111127 +111128,111128 +111129,111129 +111130,111130 +111131,111131 +111132,111132 +111133,111133 +111134,111134 +111135,111135 +111136,111136 +111137,111137 +225170,111137 +111138,111138 +111139,111139 +111140,111140 +111141,111141 +111142,111142 +111143,111143 +111144,111144 +111145,111145 +111146,111146 +111147,111147 +92635,111147 +111148,111148 +111149,111149 +111150,111150 +111151,111151 +111152,111152 +111153,111153 +81384,111153 +111154,111154 +111155,111155 +111156,111156 +111157,111157 +111158,111158 +111159,111159 +111160,111160 +111161,111161 +111162,111162 +111163,111163 +111164,111164 +111165,111165 +111166,111166 +111167,111167 +111168,111168 +111169,111169 +111170,111170 +247308,111170 +111171,111171 +111172,111172 +111173,111173 +111174,111174 +111175,111175 +111176,111176 +111177,111177 +111178,111178 +111179,111179 +111180,111180 +111181,111181 +111182,111182 +111183,111183 +111184,111184 +111185,111185 +106917,111185 +111186,111186 +111187,111187 +111188,111188 +111189,111189 +111190,111190 +111191,111191 +111192,111192 +111193,111193 +111194,111194 +111195,111195 +111196,111196 +111197,111197 +111198,111198 +111199,111199 +111200,111200 +111201,111201 +134225,111201 +126638,111201 +7301,111201 +111202,111202 +111203,111203 +111204,111204 +111205,111205 +111206,111206 +111207,111207 +111208,111208 +196562,111208 +111209,111209 +111210,111210 +111211,111211 +111212,111212 +151556,111212 +111213,111213 +111214,111214 +111215,111215 +111216,111216 +111217,111217 +111218,111218 +111219,111219 +111220,111220 +111221,111221 +194340,111221 +111222,111222 +111223,111223 +111224,111224 +111225,111225 +111226,111226 +111227,111227 +111228,111228 +111229,111229 +111230,111230 +111231,111231 +111232,111232 +111233,111233 +111234,111234 +111235,111235 +111236,111236 +111237,111237 +111238,111238 +111239,111239 +111240,111240 +111241,111241 +111242,111242 +119,111242 +111243,111243 +111244,111244 +111245,111245 +111246,111246 +111247,111247 +111248,111248 +111249,111249 +40204,111249 +111250,111250 +111251,111251 +111252,111252 +111253,111253 +111254,111254 +111255,111255 +111256,111256 +111257,111257 +235870,111257 +111258,111258 +111259,111259 +111260,111260 +111261,111261 +111262,111262 +111263,111263 +111264,111264 +111265,111265 +111266,111266 +111267,111267 +111268,111268 +111960,111268 +111269,111269 +7744,111269 +111270,111270 +111271,111271 +111272,111272 +111273,111273 +111274,111274 +111275,111275 +111276,111276 +59403,111276 +111277,111277 +111278,111278 +111279,111279 +100934,111279 +206673,111280 +111280,111280 +95003,111280 +111281,111281 +111282,111282 +111283,111283 +51102,111283 +42998,111283 +111284,111284 +111285,111285 +111286,111286 +111287,111287 +111288,111288 +111289,111289 +111290,111290 +111291,111291 +111292,111292 +111293,111293 +111294,111294 +111295,111295 +111296,111296 +111297,111297 +111298,111298 +111299,111299 +111300,111300 +111301,111301 +111302,111302 +111303,111303 +111304,111304 +111305,111305 +111306,111306 +111307,111307 +111308,111308 +111309,111309 +111310,111310 +111311,111311 +111312,111312 +111313,111313 +111314,111314 +111315,111315 +111316,111316 +111317,111317 +111318,111318 +111319,111319 +111320,111320 +143045,111320 +111321,111321 +111322,111322 +111323,111323 +111324,111324 +111325,111325 +111326,111326 +111327,111327 +111328,111328 +111329,111329 +111330,111330 +111331,111331 +111332,111332 +111333,111333 +111334,111334 +111335,111335 +111336,111336 +111337,111337 +111338,111338 +111339,111339 +53305,111339 +45201,111339 +111340,111340 +111341,111341 +111342,111342 +111343,111343 +111344,111344 +111345,111345 +111346,111346 +111347,111347 +111348,111348 +111349,111349 +111350,111350 +111351,111351 +111352,111352 +111353,111353 +111354,111354 +111355,111355 +111356,111356 +111357,111357 +111358,111358 +111359,111359 +111360,111360 +111361,111361 +111362,111362 +111363,111363 +111364,111364 +111365,111365 +111366,111366 +111367,111367 +111368,111368 +111369,111369 +88182,111369 +111370,111370 +111371,111371 +81554,111371 +111372,111372 +111373,111373 +111374,111374 +111375,111375 +111376,111376 +111377,111377 +111378,111378 +111379,111379 +111380,111380 +111381,111381 +111382,111382 +114836,111382 +111383,111383 +191946,111384 +111384,111384 +111385,111385 +111386,111386 +111387,111387 +136311,111387 +128724,111387 +111388,111388 +111389,111389 +111390,111390 +111391,111391 +111392,111392 +111393,111393 +111394,111394 +7665,111394 +111395,111395 +97960,111395 +111396,111396 +111397,111397 +206294,111397 +111398,111398 +111399,111399 +111400,111400 +111401,111401 +111402,111402 +111403,111403 +111404,111404 +111405,111405 +111406,111406 +111407,111407 +111408,111408 +111409,111409 +111410,111410 +111411,111411 +111412,111412 +111413,111413 +111414,111414 +111415,111415 +111416,111416 +111417,111417 +111418,111418 +111419,111419 +111420,111420 +111421,111421 +184469,111421 +111422,111422 +111423,111423 +111424,111424 +111425,111425 +120192,111425 +111426,111426 +111427,111427 +111428,111428 +111429,111429 +111430,111430 +111431,111431 +111432,111432 +111433,111433 +111434,111434 +111435,111435 +111436,111436 +111437,111437 +111438,111438 +111439,111439 +111440,111440 +111441,111441 +111442,111442 +111443,111443 +111444,111444 +111445,111445 +111446,111446 +111447,111447 +111448,111448 +408,111448 +111449,111449 +111450,111450 +111451,111451 +111452,111452 +111453,111453 +111454,111454 +111455,111455 +111456,111456 +111457,111457 +4183,111457 +111458,111458 +111459,111459 +111460,111460 +117031,111460 +111461,111461 +111462,111462 +111463,111463 +111464,111464 +111465,111465 +111466,111466 +111467,111467 +111468,111468 +111469,111469 +111470,111470 +111471,111471 +111472,111472 +111473,111473 +111474,111474 +111475,111475 +111476,111476 +111477,111477 +111478,111478 +111479,111479 +111480,111480 +160979,111480 +111481,111481 +111482,111482 +111483,111483 +111484,111484 +111485,111485 +111486,111486 +111487,111487 +111488,111488 +111489,111489 +111490,111490 +111491,111491 +111492,111492 +111493,111493 +111494,111494 +111495,111495 +111496,111496 +111497,111497 +111498,111498 +111499,111499 +111500,111500 +111501,111501 +111502,111502 +246699,111502 +111503,111503 +111504,111504 +111505,111505 +111506,111506 +111507,111507 +111508,111508 +111509,111509 +111510,111510 +111511,111511 +211254,111511 +111512,111512 +111513,111513 +145126,111513 +111514,111514 +111515,111515 +111516,111516 +55655,111516 +47551,111516 +111517,111517 +111518,111518 +111519,111519 +145132,111519 +111520,111520 +111521,111521 +111522,111522 +111523,111523 +111524,111524 +111525,111525 +111526,111526 +111527,111527 +111528,111528 +111529,111529 +111530,111530 +111531,111531 +111532,111532 +111533,111533 +111534,111534 +111535,111535 +111536,111536 +111537,111537 +192380,111537 +111538,111538 +111539,111539 +111540,111540 +111541,111541 +111542,111542 +111543,111543 +111544,111544 +111545,111545 +111546,111546 +111547,111547 +111548,111548 +111549,111549 +111550,111550 +184321,111550 +111551,111551 +111552,111552 +111553,111553 +111554,111554 +111555,111555 +111556,111556 +111557,111557 +111558,111558 +111559,111559 +111560,111560 +111561,111561 +111562,111562 +111563,111563 +111564,111564 +111565,111565 +111566,111566 +111567,111567 +111568,111568 +111569,111569 +111570,111570 +111571,111571 +111572,111572 +111573,111573 +111574,111574 +10000,111574 +111575,111575 +111576,111576 +111577,111577 +111578,111578 +111579,111579 +111580,111580 +111581,111581 +111582,111582 +111583,111583 +111584,111584 +111585,111585 +111586,111586 +111587,111587 +111588,111588 +111589,111589 +111590,111590 +111591,111591 +111592,111592 +141608,111592 +111593,111593 +111594,111594 +111595,111595 +111596,111596 +111597,111597 +111598,111598 +111599,111599 +111600,111600 +111601,111601 +111602,111602 +111603,111603 +111604,111604 +111605,111605 +111606,111606 +156659,111606 +111607,111607 +111608,111608 +111609,111609 +111610,111610 +111611,111611 +111612,111612 +111613,111613 +111614,111614 +111615,111615 +111616,111616 +111617,111617 +111618,111618 +111619,111619 +246635,111619 +111620,111620 +111621,111621 +111622,111622 +111623,111623 +111624,111624 +111625,111625 +111626,111626 +180715,111626 +111627,111627 +214818,111627 +111628,111628 +111629,111629 +111630,111630 +120212,111630 +111631,111631 +111632,111632 +111633,111633 +111634,111634 +111635,111635 +111636,111636 +111637,111637 +111638,111638 +111639,111639 +111640,111640 +111641,111641 +111642,111642 +111643,111643 +111644,111644 +111645,111645 +111646,111646 +111647,111647 +111648,111648 +111649,111649 +111650,111650 +111651,111651 +111652,111652 +111653,111653 +111654,111654 +111655,111655 +111656,111656 +111657,111657 +111658,111658 +164000,111658 +111659,111659 +111660,111660 +111661,111661 +111662,111662 +111663,111663 +111664,111664 +111665,111665 +111666,111666 +111667,111667 +56095,111667 +47991,111667 +111668,111668 +111669,111669 +111670,111670 +111671,111671 +31699,111671 +111672,111672 +92686,111672 +111673,111673 +111674,111674 +111675,111675 +20110,111675 +111676,111676 +111677,111677 +111678,111678 +111679,111679 +111680,111680 +111681,111681 +164307,111681 +111682,111682 +111683,111683 +111684,111684 +111685,111685 +111686,111686 +111687,111687 +18537,111687 +111688,111688 +111689,111689 +106777,111689 +111690,111690 +111691,111691 +111692,111692 +111693,111693 +111694,111694 +111695,111695 +111696,111696 +111697,111697 +52623,111697 +44519,111697 +111698,111698 +36946,111698 +77540,111698 +111699,111699 +111700,111700 +111701,111701 +111702,111702 +111703,111703 +111704,111704 +80316,111704 +111705,111705 +111706,111706 +111707,111707 +111708,111708 +111709,111709 +111710,111710 +111711,111711 +111712,111712 +111713,111713 +111714,111714 +225396,111714 +111715,111715 +111716,111716 +111717,111717 +111718,111718 +111719,111719 +111720,111720 +111721,111721 +111722,111722 +111723,111723 +111724,111724 +111725,111725 +111726,111726 +111727,111727 +111728,111728 +111729,111729 +111730,111730 +111731,111731 +111732,111732 +111733,111733 +111734,111734 +111735,111735 +111736,111736 +111737,111737 +111738,111738 +111739,111739 +111740,111740 +111741,111741 +111742,111742 +111743,111743 +111744,111744 +142490,111744 +111745,111745 +111746,111746 +111747,111747 +111748,111748 +111749,111749 +111750,111750 +111751,111751 +111752,111752 +134304,111752 +126717,111752 +111753,111753 +223463,111753 +111754,111754 +111755,111755 +111756,111756 +111757,111757 +111758,111758 +111759,111759 +111760,111760 +111761,111761 +145234,111761 +111762,111762 +111763,111763 +111764,111764 +111765,111765 +111766,111766 +111767,111767 +111768,111768 +111769,111769 +111770,111770 +111771,111771 +111772,111772 +111773,111773 +111774,111774 +111775,111775 +111776,111776 +111777,111777 +111778,111778 +111779,111779 +111780,111780 +111781,111781 +64127,111781 +111782,111782 +111783,111783 +111784,111784 +111785,111785 +240900,111785 +111786,111786 +111787,111787 +9462,111787 +188044,111787 +111788,111788 +111789,111789 +111790,111790 +111791,111791 +111792,111792 +111793,111793 +179429,111793 +111794,111794 +100603,111794 +111795,111795 +111796,111796 +111797,111797 +111798,111798 +111799,111799 +111800,111800 +111801,111801 +111802,111802 +111803,111803 +111804,111804 +111805,111805 +111806,111806 +111807,111807 +111808,111808 +215986,111808 +111809,111809 +111810,111810 +111811,111811 +111812,111812 +111813,111813 +111814,111814 +111815,111815 +111816,111816 +111817,111817 +111818,111818 +111819,111819 +111820,111820 +111821,111821 +111822,111822 +111823,111823 +111824,111824 +111825,111825 +111826,111826 +84601,111826 +111827,111827 +111828,111828 +197323,111828 +111829,111829 +111830,111830 +111831,111831 +111832,111832 +111833,111833 +111834,111834 +111835,111835 +111836,111836 +133077,111836 +125490,111836 +111837,111837 +111838,111838 +111839,111839 +111840,111840 +111841,111841 +111842,111842 +111843,111843 +111844,111844 +111845,111845 +111846,111846 +111847,111847 +111848,111848 +111849,111849 +111850,111850 +111851,111851 +84910,111851 +111852,111852 +111853,111853 +111854,111854 +111855,111855 +111856,111856 +111857,111857 +111858,111858 +111859,111859 +111860,111860 +111861,111861 +203837,111861 +111862,111862 +111863,111863 +111864,111864 +111865,111865 +111866,111866 +111867,111867 +111868,111868 +191623,111868 +111869,111869 +111870,111870 +111871,111871 +111872,111872 +111873,111873 +111874,111874 +111875,111875 +111876,111876 +81351,111876 +111877,111877 +111878,111878 +111879,111879 +111880,111880 +131959,111880 +124372,111880 +111881,111881 +111882,111882 +111883,111883 +111884,111884 +111885,111885 +111886,111886 +111887,111887 +111888,111888 +111889,111889 +111890,111890 +111891,111891 +111892,111892 +111893,111893 +111894,111894 +111895,111895 +36932,111895 +96206,111895 +111896,111896 +29906,111896 +111897,111897 +111898,111898 +111899,111899 +111900,111900 +111901,111901 +111902,111902 +111903,111903 +165189,111903 +111904,111904 +205120,111904 +111905,111905 +111906,111906 +111907,111907 +111908,111908 +111909,111909 +111910,111910 +111911,111911 +111912,111912 +111913,111913 +82548,111913 +111914,111914 +111915,111915 +111916,111916 +111917,111917 +111918,111918 +111919,111919 +113347,111919 +111920,111920 +158783,111920 +111921,111921 +111922,111922 +29066,111922 +111923,111923 +111924,111924 +111925,111925 +111926,111926 +52482,111926 +44378,111926 +111927,111927 +111928,111928 +111929,111929 +111930,111930 +111931,111931 +111932,111932 +111933,111933 +111934,111934 +111935,111935 +111936,111936 +111937,111937 +111938,111938 +111939,111939 +111940,111940 +111941,111941 +111942,111942 +111943,111943 +111944,111944 +111945,111945 +120261,111946 +111946,111946 +111947,111947 +111948,111948 +111949,111949 +111950,111950 +111951,111951 +111952,111952 +111953,111953 +111954,111954 +111955,111955 +111956,111956 +111957,111957 +111958,111958 +111959,111959 +111960,111960 +111268,111960 +111961,111961 +111962,111962 +64960,111962 +111963,111963 +111964,111964 +111965,111965 +249071,111965 +111966,111966 +111967,111967 +111968,111968 +156835,111968 +111969,111969 +111970,111970 +111971,111971 +203531,111971 +111972,111972 +111973,111973 +135072,111974 +127485,111974 +111974,111974 +111975,111975 +111976,111976 +111977,111977 +111978,111978 +111979,111979 +37053,111979 +111980,111980 +111981,111981 +111982,111982 +111983,111983 +111984,111984 +111985,111985 +111986,111986 +111987,111987 +111988,111988 +111989,111989 +111990,111990 +111991,111991 +111992,111992 +111993,111993 +111994,111994 +207639,111994 +111995,111995 +159382,111995 +111996,111996 +111997,111997 +92718,111997 +111998,111998 +111999,111999 +112000,112000 +112001,112001 +112002,112002 +112003,112003 +112004,112004 +112005,112005 +112006,112006 +112007,112007 +112008,112008 +151499,112008 +112009,112009 +112010,112010 +112011,112011 +112012,112012 +235890,112012 +112013,112013 +112014,112014 +112015,112015 +112016,112016 +112017,112017 +112018,112018 +112019,112019 +112020,112020 +112021,112021 +112022,112022 +138943,112022 +75487,112022 +112023,112023 +112024,112024 +112025,112025 +112026,112026 +112027,112027 +112028,112028 +112029,112029 +112030,112030 +112031,112031 +112032,112032 +112033,112033 +112034,112034 +112035,112035 +112036,112036 +112037,112037 +112038,112038 +173902,112038 +167938,112038 +112039,112039 +112040,112040 +112041,112041 +112042,112042 +112043,112043 +112044,112044 +112045,112045 +112046,112046 +112047,112047 +112048,112048 +112049,112049 +112050,112050 +112051,112051 +112052,112052 +112053,112053 +112054,112054 +112055,112055 +112056,112056 +112057,112057 +112058,112058 +112059,112059 +112060,112060 +112061,112061 +112062,112062 +112063,112063 +112064,112064 +112065,112065 +112066,112066 +112067,112067 +112068,112068 +112069,112069 +112070,112070 +112071,112071 +112072,112072 +112073,112073 +112074,112074 +112075,112075 +112076,112076 +112077,112077 +112078,112078 +112079,112079 +112080,112080 +112081,112081 +112082,112082 +112083,112083 +112084,112084 +112085,112085 +112086,112086 +112087,112087 +112088,112088 +112089,112089 +112090,112090 +232900,112090 +106316,112090 +112091,112091 +112092,112092 +112093,112093 +112094,112094 +5658,112094 +112095,112095 +112096,112096 +112097,112097 +112098,112098 +112099,112099 +112100,112100 +124083,112100 +112101,112101 +112102,112102 +112103,112103 +112104,112104 +112105,112105 +112106,112106 +180101,112106 +112107,112107 +112108,112108 +112109,112109 +112110,112110 +112111,112111 +112112,112112 +112113,112113 +112114,112114 +112115,112115 +112116,112116 +112117,112117 +112118,112118 +112119,112119 +112120,112120 +112121,112121 +112122,112122 +112123,112123 +194965,112123 +112124,112124 +112125,112125 +112126,112126 +112127,112127 +112128,112128 +112129,112129 +112130,112130 +112131,112131 +112132,112132 +112133,112133 +112134,112134 +112135,112135 +112136,112136 +112137,112137 +112138,112138 +112139,112139 +224974,112139 +112140,112140 +112141,112141 +112142,112142 +112143,112143 +112144,112144 +112145,112145 +112146,112146 +112147,112147 +112148,112148 +112149,112149 +112150,112150 +112151,112151 +112152,112152 +112153,112153 +112154,112154 +112155,112155 +112156,112156 +112157,112157 +112158,112158 +112159,112159 +112160,112160 +112161,112161 +112162,112162 +112163,112163 +112164,112164 +112165,112165 +112166,112166 +193365,112166 +112167,112167 +112168,112168 +112169,112169 +112170,112170 +112171,112171 +112172,112172 +112173,112173 +112174,112174 +112175,112175 +112176,112176 +112177,112177 +112178,112178 +112179,112179 +112180,112180 +112181,112181 +112182,112182 +39904,112182 +112183,112183 +112184,112184 +112185,112185 +112186,112186 +108436,112186 +112187,112187 +112188,112188 +112189,112189 +112190,112190 +112191,112191 +112192,112192 +112193,112193 +112194,112194 +112195,112195 +112196,112196 +112197,112197 +112198,112198 +112199,112199 +112200,112200 +112201,112201 +112202,112202 +112203,112203 +112204,112204 +112205,112205 +112206,112206 +112207,112207 +112208,112208 +112209,112209 +112210,112210 +112211,112211 +112212,112212 +9106,112212 +112213,112213 +112214,112214 +112215,112215 +112216,112216 +112217,112217 +112218,112218 +112219,112219 +112220,112220 +112221,112221 +112222,112222 +112223,112223 +112224,112224 +112225,112225 +112226,112226 +112227,112227 +112228,112228 +112229,112229 +107328,112229 +112230,112230 +112231,112231 +112232,112232 +112233,112233 +112234,112234 +112235,112235 +26320,112235 +112236,112236 +112237,112237 +112238,112238 +112239,112239 +112240,112240 +112241,112241 +112242,112242 +112243,112243 +112244,112244 +112245,112245 +112246,112246 +112247,112247 +112248,112248 +100569,112248 +112249,112249 +112250,112250 +112251,112251 +112252,112252 +112253,112253 +112254,112254 +112255,112255 +112256,112256 +112257,112257 +68197,112257 +112258,112258 +112259,112259 +112260,112260 +112261,112261 +112262,112262 +112263,112263 +112264,112264 +112265,112265 +112266,112266 +112267,112267 +112268,112268 +112269,112269 +112270,112270 +210083,112270 +112271,112271 +112272,112272 +163535,112272 +112273,112273 +112274,112274 +112275,112275 +112276,112276 +112277,112277 +112278,112278 +112279,112279 +112280,112280 +112281,112281 +112282,112282 +112283,112283 +112284,112284 +112285,112285 +112286,112286 +112287,112287 +112288,112288 +112289,112289 +112290,112290 +112291,112291 +112292,112292 +112293,112293 +112294,112294 +211391,112294 +112295,112295 +112296,112296 +112297,112297 +112298,112298 +112299,112299 +112300,112300 +112301,112301 +112302,112302 +112303,112303 +60928,112303 +112304,112304 +24705,112304 +112305,112305 +112306,112306 +112307,112307 +208746,112307 +112308,112308 +112309,112309 +112310,112310 +112311,112311 +112312,112312 +204622,112312 +112313,112313 +112314,112314 +112315,112315 +112316,112316 +112317,112317 +112318,112318 +208470,112318 +112319,112319 +112320,112320 +112321,112321 +112322,112322 +112323,112323 +112324,112324 +112325,112325 +112326,112326 +112327,112327 +112328,112328 +112430,112430 +112431,112431 +112432,112432 +112433,112433 +112434,112434 +112435,112435 +112436,112436 +112437,112437 +112438,112438 +112439,112439 +112440,112440 +112441,112441 +112442,112442 +112443,112443 +243922,112443 +112444,112444 +112445,112445 +112446,112446 +112447,112447 +141396,112447 +112448,112448 +112449,112449 +112450,112450 +112451,112451 +112452,112452 +112453,112453 +132105,112453 +124518,112453 +112454,112454 +112455,112455 +112456,112456 +112457,112457 +112458,112458 +112459,112459 +112460,112460 +119167,112460 +112461,112461 +141410,112461 +112462,112462 +141411,112462 +112463,112463 +112464,112464 +112465,112465 +112466,112466 +112467,112467 +112468,112468 +112469,112469 +112470,112470 +112471,112471 +112472,112472 +113157,113157 +199825,113157 +113158,113158 +113299,113299 +113300,113300 +113301,113301 +32419,113301 +113302,113302 +113303,113303 +113304,113304 +113305,113305 +113306,113306 +242602,113306 +113307,113307 +113308,113308 +113309,113309 +113310,113310 +113311,113311 +113312,113312 +113313,113313 +113314,113314 +113315,113315 +113316,113316 +113317,113317 +113318,113318 +113319,113319 +113320,113320 +113321,113321 +113322,113322 +113323,113323 +113324,113324 +113325,113325 +113326,113326 +113327,113327 +113328,113328 +80615,113328 +113329,113329 +113330,113330 +113331,113331 +113332,113332 +65823,113332 +113333,113333 +113334,113334 +202744,113334 +113335,113335 +113336,113336 +113337,113337 +113338,113338 +113339,113339 +113340,113340 +113341,113341 +113342,113342 +75154,113342 +113343,113343 +113344,113344 +113345,113345 +166472,113345 +113346,113346 +113347,113347 +111919,113347 +113348,113348 +113349,113349 +113350,113350 +113351,113351 +113352,113352 +113353,113353 +113354,113354 +113355,113355 +113356,113356 +113357,113357 +113358,113358 +113359,113359 +113360,113360 +142489,113360 +113361,113361 +113362,113362 +113363,113363 +113364,113364 +113365,113365 +113366,113366 +113367,113367 +113368,113368 +113369,113369 +113370,113370 +113371,113371 +113372,113372 +113373,113373 +113374,113374 +113375,113375 +113376,113376 +113377,113377 +113378,113378 +113379,113379 +113380,113380 +113381,113381 +113382,113382 +113383,113383 +113384,113384 +113385,113385 +113386,113386 +113387,113387 +113388,113388 +113389,113389 +113390,113390 +113391,113391 +113392,113392 +113393,113393 +113394,113394 +113395,113395 +113396,113396 +113397,113397 +113398,113398 +113399,113399 +113400,113400 +185096,113400 +7986,113400 +113401,113401 +58071,113401 +113402,113402 +113403,113403 +113404,113404 +113405,113405 +113406,113406 +113407,113407 +113408,113408 +113409,113409 +113410,113410 +113411,113411 +113412,113412 +113413,113413 +93389,113413 +113414,113414 +113415,113415 +113416,113416 +113417,113417 +113418,113418 +106756,113418 +113419,113419 +157771,113419 +113420,113420 +113421,113421 +113422,113422 +113423,113423 +61366,113423 +113424,113424 +113425,113425 +113426,113426 +94951,113426 +113427,113427 +113428,113428 +113429,113429 +113430,113430 +113431,113431 +113432,113432 +113433,113433 +113434,113434 +113435,113435 +113436,113436 +113437,113437 +113438,113438 +113439,113439 +113440,113440 +113441,113441 +113442,113442 +113443,113443 +113444,113444 +113445,113445 +113446,113446 +113447,113447 +113448,113448 +113449,113449 +195130,113449 +19959,113449 +113450,113450 +113451,113451 +113452,113452 +113453,113453 +113454,113454 +113455,113455 +113456,113456 +113457,113457 +113458,113458 +113459,113459 +113460,113460 +113461,113461 +113462,113462 +113463,113463 +113464,113464 +113465,113465 +113466,113466 +113467,113467 +113468,113468 +113469,113469 +113470,113470 +113471,113471 +113472,113472 +113473,113473 +113474,113474 +113475,113475 +113476,113476 +113477,113477 +113478,113478 +113479,113479 +113480,113480 +113481,113481 +113482,113482 +113483,113483 +113484,113484 +113485,113485 +113486,113486 +113487,113487 +113488,113488 +113489,113489 +113490,113490 +113491,113491 +113492,113492 +113493,113493 +134137,113493 +126550,113493 +113494,113494 +113495,113495 +113496,113496 +113497,113497 +113498,113498 +113499,113499 +113500,113500 +113501,113501 +113502,113502 +113503,113503 +113504,113504 +113505,113505 +6584,113505 +113506,113506 +113507,113507 +113508,113508 +23551,113508 +113509,113509 +113510,113510 +113511,113511 +113512,113512 +113513,113513 +113514,113514 +113515,113515 +113516,113516 +113517,113517 +113518,113518 +113519,113519 +113520,113520 +113521,113521 +113522,113522 +113523,113523 +113524,113524 +113525,113525 +9060,113525 +113526,113526 +113527,113527 +113528,113528 +113529,113529 +113530,113530 +113531,113531 +113532,113532 +113533,113533 +113534,113534 +113535,113535 +113536,113536 +113537,113537 +113538,113538 +113539,113539 +113540,113540 +113541,113541 +113542,113542 +113543,113543 +113544,113544 +113545,113545 +113546,113546 +113547,113547 +141389,113547 +113548,113548 +113549,113549 +113550,113550 +113551,113551 +113552,113552 +113553,113553 +113554,113554 +113555,113555 +113556,113556 +113557,113557 +113558,113558 +113559,113559 +113560,113560 +113561,113561 +113562,113562 +113563,113563 +113564,113564 +56733,113564 +113565,113565 +113566,113566 +113567,113567 +113568,113568 +113569,113569 +113570,113570 +113571,113571 +113572,113572 +113573,113573 +113574,113574 +113575,113575 +113576,113576 +113577,113577 +113578,113578 +113579,113579 +113580,113580 +113581,113581 +113582,113582 +113583,113583 +113584,113584 +113585,113585 +113586,113586 +113587,113587 +194969,113587 +113588,113588 +113589,113589 +113590,113590 +113591,113591 +113592,113592 +113593,113593 +113594,113594 +113595,113595 +113910,113910 +113911,113911 +113912,113912 +114156,114156 +114157,114157 +26231,114157 +76022,114157 +114158,114158 +114159,114159 +114160,114160 +114161,114161 +114162,114162 +114163,114163 +114164,114164 +114165,114165 +114166,114166 +114167,114167 +114168,114168 +114169,114169 +247,114169 +114170,114170 +114171,114171 +109819,114171 +114172,114172 +114221,114221 +91008,114221 +114222,114222 +114223,114223 +114224,114224 +114225,114225 +114226,114226 +114227,114227 +178920,114227 +114228,114228 +114229,114229 +114230,114230 +114231,114231 +114232,114232 +114233,114233 +114234,114234 +114235,114235 +114236,114236 +114237,114237 +114238,114238 +114239,114239 +114240,114240 +114241,114241 +114242,114242 +114243,114243 +114244,114244 +114245,114245 +2180,114245 +114246,114246 +114247,114247 +114248,114248 +114249,114249 +114250,114250 +114251,114251 +114252,114252 +114253,114253 +114254,114254 +114255,114255 +114306,114306 +114307,114307 +114308,114308 +114309,114309 +114310,114310 +114311,114311 +114312,114312 +114313,114313 +114314,114314 +114315,114315 +114316,114316 +114317,114317 +114318,114318 +114319,114319 +114320,114320 +148122,114320 +114321,114321 +114322,114322 +114323,114323 +114324,114324 +114325,114325 +114326,114326 +114327,114327 +618,114327 +114328,114328 +114329,114329 +114330,114330 +114331,114331 +114332,114332 +114333,114333 +114334,114334 +114335,114335 +114336,114336 +114337,114337 +114338,114338 +114339,114339 +114340,114340 +114341,114341 +114342,114342 +114343,114343 +114344,114344 +114345,114345 +114346,114346 +114347,114347 +114348,114348 +148143,114348 +114349,114349 +114350,114350 +114351,114351 +114352,114352 +114353,114353 +114354,114354 +114355,114355 +114356,114356 +114357,114357 +114358,114358 +215399,114358 +5742,114358 +114359,114359 +67996,114359 +114360,114360 +114361,114361 +114362,114362 +114363,114363 +114364,114364 +114365,114365 +114366,114366 +114367,114367 +114368,114368 +114369,114369 +114370,114370 +114371,114371 +114372,114372 +114373,114373 +114374,114374 +114375,114375 +114376,114376 +114377,114377 +209489,114377 +114378,114378 +114379,114379 +114380,114380 +114381,114381 +114382,114382 +114383,114383 +114384,114384 +114385,114385 +114386,114386 +20109,114386 +114387,114387 +114388,114388 +114389,114389 +77713,114389 +114390,114390 +114391,114391 +203519,114391 +114392,114392 +114393,114393 +114394,114394 +114395,114395 +114396,114396 +114397,114397 +114398,114398 +114399,114399 +114400,114400 +114401,114401 +114402,114402 +114403,114403 +114404,114404 +114405,114405 +114406,114406 +114407,114407 +114408,114408 +114409,114409 +114410,114410 +114411,114411 +114412,114412 +114413,114413 +114414,114414 +114415,114415 +114416,114416 +114417,114417 +114418,114418 +114419,114419 +114420,114420 +57066,114420 +114421,114421 +114422,114422 +114423,114423 +114424,114424 +132987,114424 +125400,114424 +114425,114425 +114426,114426 +114427,114427 +12655,114427 +114428,114428 +114429,114429 +114430,114430 +114431,114431 +114432,114432 +114433,114433 +114434,114434 +114435,114435 +114436,114436 +114437,114437 +114438,114438 +114439,114439 +114440,114440 +114441,114441 +215934,114441 +114442,114442 +114443,114443 +183068,114443 +114444,114444 +191388,114444 +220376,114444 +114445,114445 +224976,114445 +114446,114446 +114447,114447 +114448,114448 +114449,114449 +114450,114450 +114451,114451 +114452,114452 +114453,114453 +114454,114454 +114455,114455 +192541,114455 +114456,114456 +114457,114457 +114458,114458 +114459,114459 +114460,114460 +114461,114461 +114462,114462 +114463,114463 +84611,114463 +114464,114464 +114465,114465 +114466,114466 +114467,114467 +114468,114468 +114469,114469 +114470,114470 +114471,114471 +114472,114472 +114473,114473 +114474,114474 +114475,114475 +114476,114476 +114477,114477 +114478,114478 +114479,114479 +114480,114480 +114481,114481 +114482,114482 +114483,114483 +100756,114483 +114484,114484 +114485,114485 +114486,114486 +114487,114487 +165290,114487 +114666,114666 +114667,114667 +114668,114668 +114669,114669 +114670,114670 +114671,114671 +114672,114672 +114673,114673 +114674,114674 +114675,114675 +114676,114676 +141273,114676 +114677,114677 +114678,114678 +114679,114679 +114680,114680 +114681,114681 +114682,114682 +114683,114683 +114684,114684 +114685,114685 +139932,114685 +114686,114686 +114687,114687 +114688,114688 +114689,114689 +114690,114690 +114691,114691 +114692,114692 +114693,114693 +83048,114693 +114694,114694 +114695,114695 +114696,114696 +114697,114697 +114698,114698 +114699,114699 +114700,114700 +114701,114701 +114702,114702 +114703,114703 +114704,114704 +114705,114705 +114706,114706 +114707,114707 +114708,114708 +114709,114709 +114710,114710 +114711,114711 +114712,114712 +114713,114713 +114714,114714 +114715,114715 +114716,114716 +114717,114717 +114718,114718 +114719,114719 +114720,114720 +114721,114721 +114722,114722 +114723,114723 +114724,114724 +114725,114725 +114726,114726 +149860,114726 +114727,114727 +114728,114728 +114729,114729 +114730,114730 +114731,114731 +114732,114732 +178947,114732 +114733,114733 +114734,114734 +114735,114735 +201781,114735 +114736,114736 +114737,114737 +114738,114738 +183116,114738 +114739,114739 +114740,114740 +114741,114741 +114742,114742 +114743,114743 +114744,114744 +114745,114745 +114746,114746 +114747,114747 +114748,114748 +114749,114749 +114750,114750 +114751,114751 +114752,114752 +114753,114753 +177495,114753 +171531,114753 +114754,114754 +114755,114755 +134196,114755 +126609,114755 +114756,114756 +114757,114757 +114758,114758 +114759,114759 +114760,114760 +141408,114760 +114761,114761 +114762,114762 +114763,114763 +114764,114764 +114765,114765 +114766,114766 +114767,114767 +114768,114768 +114769,114769 +114770,114770 +114771,114771 +114772,114772 +114773,114773 +114774,114774 +114775,114775 +114776,114776 +114777,114777 +114778,114778 +114779,114779 +114780,114780 +114781,114781 +114782,114782 +95052,114782 +114783,114783 +114784,114784 +114785,114785 +114786,114786 +114787,114787 +114788,114788 +114789,114789 +114790,114790 +114791,114791 +114792,114792 +114793,114793 +114794,114794 +114795,114795 +114796,114796 +114797,114797 +114798,114798 +114799,114799 +114800,114800 +114801,114801 +114802,114802 +207985,114802 +114803,114803 +114804,114804 +114805,114805 +220614,114805 +76456,114805 +114806,114806 +114807,114807 +114808,114808 +114809,114809 +114810,114810 +114811,114811 +114812,114812 +114813,114813 +114814,114814 +114815,114815 +114816,114816 +114817,114817 +114818,114818 +114819,114819 +114820,114820 +114821,114821 +114822,114822 +114823,114823 +114824,114824 +114825,114825 +114826,114826 +114827,114827 +157825,114827 +114828,114828 +114829,114829 +114830,114830 +114831,114831 +114832,114832 +114833,114833 +114834,114834 +114835,114835 +114836,114836 +111382,114836 +114837,114837 +162353,114837 +114838,114838 +114839,114839 +114840,114840 +114841,114841 +114842,114842 +114843,114843 +114844,114844 +114845,114845 +178666,114845 +114846,114846 +114847,114847 +114848,114848 +114849,114849 +154105,114849 +114850,114850 +114851,114851 +114852,114852 +114853,114853 +114854,114854 +114855,114855 +114856,114856 +114857,114857 +114858,114858 +114859,114859 +114860,114860 +114861,114861 +114862,114862 +114863,114863 +114864,114864 +114865,114865 +114866,114866 +114867,114867 +114868,114868 +114869,114869 +114870,114870 +114871,114871 +114872,114872 +21676,114872 +114873,114873 +114874,114874 +114875,114875 +198689,114875 +114876,114876 +114877,114877 +114878,114878 +174362,114878 +168398,114878 +114879,114879 +114880,114880 +114881,114881 +114882,114882 +114883,114883 +114884,114884 +114885,114885 +114886,114886 +114887,114887 +114888,114888 +114889,114889 +114890,114890 +114891,114891 +114892,114892 +114893,114893 +150808,114893 +59178,114893 +58804,114893 +114894,114894 +75921,114894 +114895,114895 +114896,114896 +114897,114897 +114898,114898 +114899,114899 +114900,114900 +163013,114900 +114901,114901 +114902,114902 +114903,114903 +77216,114903 +236892,115469 +115469,115469 +188512,115469 +115470,115470 +56219,115470 +48115,115470 +236894,115471 +115471,115471 +115472,115472 +115736,115736 +115737,115737 +115738,115738 +115739,115739 +115740,115740 +115741,115741 +115778,115778 +200892,115778 +115779,115779 +115780,115780 +115781,115781 +115782,115782 +115783,115783 +115784,115784 +115785,115785 +115786,115786 +60444,115786 +81549,115786 +115787,115787 +115788,115788 +115789,115789 +115790,115790 +115791,115791 +115792,115792 +115793,115793 +115794,115794 +115795,115795 +233998,115795 +115796,115796 +115797,115797 +115798,115798 +115799,115799 +115800,115800 +115801,115801 +115802,115802 +115803,115803 +115804,115804 +115805,115805 +115806,115806 +115807,115807 +115808,115808 +115809,115809 +115810,115810 +115932,115932 +137099,115932 +129512,115932 +115933,115933 +115934,115934 +115935,115935 +115936,115936 +115937,115937 +115938,115938 +115939,115939 +115940,115940 +211094,115940 +115941,115941 +196317,115941 +115942,115942 +115943,115943 +180215,115943 +115944,115944 +115945,115945 +115946,115946 +115947,115947 +115948,115948 +115949,115949 +115950,115950 +115951,115951 +115952,115952 +115953,115953 +115954,115954 +115955,115955 +115956,115956 +115957,115957 +107654,115957 +115958,115958 +59910,115958 +115959,115959 +115960,115960 +115961,115961 +115962,115962 +2833,115962 +115963,115963 +115964,115964 +115965,115965 +115966,115966 +115967,115967 +115968,115968 +176283,115968 +170319,115968 +115969,115969 +90250,115969 +115970,115970 +115971,115971 +115972,115972 +115973,115973 +115974,115974 +115975,115975 +115976,115976 +115977,115977 +115978,115978 +115979,115979 +115980,115980 +115981,115981 +115982,115982 +188342,115982 +115983,115983 +115984,115984 +115985,115985 +115986,115986 +115987,115987 +115988,115988 +115989,115989 +115990,115990 +115991,115991 +115992,115992 +115993,115993 +115994,115994 +115995,115995 +115996,115996 +85612,115996 +115997,115997 +115998,115998 +115999,115999 +116000,116000 +116001,116001 +116002,116002 +116003,116003 +116004,116004 +116005,116005 +116006,116006 +116007,116007 +116008,116008 +116009,116009 +116010,116010 +116011,116011 +116012,116012 +116013,116013 +116014,116014 +116015,116015 +206678,116015 +116016,116016 +147662,116016 +116017,116017 +116018,116018 +116209,116209 +116210,116210 +116211,116211 +249430,116211 +116212,116212 +116213,116213 +116214,116214 +116335,116335 +36629,116335 +116336,116336 +181297,116336 +116337,116337 +116338,116338 +116339,116339 +116340,116340 +116341,116341 +918,116341 +116342,116342 +116343,116343 +116344,116344 +116345,116345 +116346,116346 +216398,116346 +116347,116347 +116348,116348 +116349,116349 +116350,116350 +116351,116351 +116352,116352 +116353,116353 +116354,116354 +116355,116355 +116356,116356 +116357,116357 +116358,116358 +214677,116358 +116359,116359 +116360,116360 +116361,116361 +116362,116362 +116363,116363 +116364,116364 +116365,116365 +116366,116366 +116367,116367 +116368,116368 +78073,116368 +116369,116369 +116370,116370 +116371,116371 +116372,116372 +55957,116372 +47853,116372 +116373,116373 +116374,116374 +116375,116375 +116376,116376 +116377,116377 +116378,116378 +210211,116378 +116379,116379 +116380,116380 +116381,116381 +56106,116381 +48002,116381 +116382,116382 +116383,116383 +116384,116384 +116385,116385 +116386,116386 +116387,116387 +116388,116388 +227809,116388 +116389,116389 +116390,116390 +116391,116391 +116392,116392 +116393,116393 +164075,116393 +116394,116394 +116395,116395 +116396,116396 +116397,116397 +116398,116398 +116399,116399 +116400,116400 +33894,116400 +116401,116401 +116402,116402 +116403,116403 +70400,116403 +116545,116545 +116546,116546 +116547,116547 +116548,116548 +116549,116549 +18996,116549 +116550,116550 +116551,116551 +116552,116552 +116553,116553 +116554,116554 +116555,116555 +191236,116555 +116556,116556 +116557,116557 +116558,116558 +116559,116559 +116560,116560 +116561,116561 +116562,116562 +116563,116563 +116564,116564 +116565,116565 +116566,116566 +116567,116567 +116568,116568 +116569,116569 +116570,116570 +116571,116571 +116572,116572 +116573,116573 +116574,116574 +116575,116575 +116576,116576 +90170,116576 +116577,116577 +81336,116577 +116578,116578 +116579,116579 +116580,116580 +116581,116581 +116582,116582 +116583,116583 +116584,116584 +116585,116585 +116586,116586 +116587,116587 +116588,116588 +116589,116589 +116590,116590 +116591,116591 +116592,116592 +116593,116593 +116594,116594 +116595,116595 +116596,116596 +116597,116597 +116598,116598 +155859,116598 +116599,116599 +116600,116600 +116601,116601 +116602,116602 +116603,116603 +116604,116604 +116605,116605 +116606,116606 +116607,116607 +116608,116608 +116609,116609 +116610,116610 +116611,116611 +49611,116611 +41418,116611 +116612,116612 +116613,116613 +116614,116614 +116615,116615 +116616,116616 +116617,116617 +116618,116618 +116619,116619 +116620,116620 +116621,116621 +178996,116622 +116622,116622 +116623,116623 +116624,116624 +116625,116625 +106315,116625 +116626,116626 +116627,116627 +116628,116628 +116629,116629 +116630,116630 +116631,116631 +116632,116632 +116633,116633 +116634,116634 +116635,116635 +116636,116636 +116738,116738 +116739,116739 +116740,116740 +116741,116741 +116742,116742 +248739,116742 +116743,116743 +116744,116744 +116745,116745 +116746,116746 +116747,116747 +116748,116748 +116749,116749 +116750,116750 +116751,116751 +110871,116751 +116752,116752 +116753,116753 +116754,116754 +116791,116791 +116792,116792 +116793,116793 +116794,116794 +116795,116795 +116796,116796 +116797,116797 +116798,116798 +116799,116799 +116800,116800 +116801,116801 +116802,116802 +116803,116803 +116804,116804 +116805,116805 +116806,116806 +116807,116807 +116808,116808 +116809,116809 +102266,116809 +116810,116810 +116811,116811 +116812,116812 +116813,116813 +116814,116814 +152212,116814 +116815,116815 +116816,116816 +116817,116817 +116818,116818 +116819,116819 +116820,116820 +116821,116821 +116822,116822 +116823,116823 +116824,116824 +116825,116825 +116826,116826 +116827,116827 +116828,116828 +116829,116829 +116830,116830 +116831,116831 +116832,116832 +116833,116833 +116834,116834 +116835,116835 +116836,116836 +116837,116837 +116838,116838 +116839,116839 +116840,116840 +116841,116841 +116842,116842 +116843,116843 +116844,116844 +116845,116845 +116846,116846 +116847,116847 +116848,116848 +116849,116849 +116850,116850 +116851,116851 +116852,116852 +116853,116853 +116854,116854 +116855,116855 +116856,116856 +116857,116857 +116858,116858 +156816,116858 +116994,116994 +116995,116995 +116996,116996 +116997,116997 +116998,116998 +116999,116999 +104876,116999 +117000,117000 +117001,117001 +214239,117001 +117002,117002 +117003,117003 +117004,117004 +117005,117005 +117006,117006 +117007,117007 +117008,117008 +117009,117009 +89204,117009 +161237,117009 +117010,117010 +117011,117011 +117012,117012 +205263,117012 +117013,117013 +117014,117014 +117015,117015 +117016,117016 +117017,117017 +117018,117018 +87247,117018 +117019,117019 +117020,117020 +117021,117021 +117022,117022 +117023,117023 +117024,117024 +117025,117025 +117026,117026 +117027,117027 +117028,117028 +117029,117029 +117030,117030 +117031,117031 +111460,117031 +117032,117032 +117033,117033 +117034,117034 +117035,117035 +117036,117036 +117037,117037 +117038,117038 +117039,117039 +117040,117040 +117041,117041 +117042,117042 +117043,117043 +117044,117044 +117045,117045 +137973,117045 +130386,117045 +117046,117046 +117047,117047 +117048,117048 +117049,117049 +117050,117050 +117051,117051 +117052,117052 +5726,117052 +117053,117053 +117054,117054 +117055,117055 +117056,117056 +117057,117057 +117058,117058 +117059,117059 +117060,117060 +117061,117061 +117062,117062 +117063,117063 +117064,117064 +117065,117065 +117066,117066 +117067,117067 +117068,117068 +117069,117069 +117070,117070 +117071,117071 +117072,117072 +117073,117073 +117074,117074 +117075,117075 +117076,117076 +117077,117077 +117078,117078 +117079,117079 +117080,117080 +117081,117081 +117082,117082 +117083,117083 +117084,117084 +117085,117085 +117086,117086 +117087,117087 +117088,117088 +117089,117089 +222212,117090 +117090,117090 +86962,117090 +117091,117091 +117441,117441 +117442,117442 +117443,117443 +117444,117444 +164303,117444 +117445,117445 +117446,117446 +117447,117447 +117448,117448 +117449,117449 +214525,117449 +117450,117450 +117451,117451 +117681,117681 +117682,117682 +117683,117683 +117684,117684 +117685,117685 +15700,117685 +117686,117686 +117687,117687 +117688,117688 +33001,117688 +117689,117689 +117690,117690 +117691,117691 +117692,117692 +117693,117693 +117694,117694 +117695,117695 +117696,117696 +117697,117697 +117698,117698 +117699,117699 +117700,117700 +117701,117701 +117702,117702 +117703,117703 +117704,117704 +117705,117705 +117706,117706 +33373,117706 +117707,117707 +117708,117708 +117709,117709 +194765,117709 +117710,117710 +117711,117711 +117712,117712 +117713,117713 +117714,117714 +117715,117715 +138527,117715 +117716,117716 +117717,117717 +208841,117717 +117718,117718 +117719,117719 +117720,117720 +117721,117721 +117722,117722 +117723,117723 +117724,117724 +117725,117725 +117726,117726 +117727,117727 +117728,117728 +117729,117729 +117730,117730 +117731,117731 +117732,117732 +117733,117733 +117734,117734 +117735,117735 +155604,117735 +117736,117736 +117737,117737 +35496,117737 +117738,117738 +117739,117739 +117740,117740 +117741,117741 +117742,117742 +117743,117743 +117744,117744 +117745,117745 +117802,117802 +117803,117803 +117804,117804 +117805,117805 +117806,117806 +117807,117807 +117808,117808 +117809,117809 +117810,117810 +117811,117811 +117812,117812 +117813,117813 +117814,117814 +117815,117815 +190638,117815 +117816,117816 +117817,117817 +117818,117818 +117819,117819 +117820,117820 +117821,117821 +117822,117822 +144839,117822 +117823,117823 +197562,117823 +117824,117824 +117825,117825 +117826,117826 +117827,117827 +117828,117828 +117829,117829 +117830,117830 +117831,117831 +117832,117832 +118114,118114 +118115,118115 +31784,118115 +118116,118116 +118117,118117 +118118,118118 +30224,118118 +118119,118119 +118120,118120 +118121,118121 +118122,118122 +118123,118123 +10875,118123 +118124,118124 +118125,118125 +118126,118126 +118127,118127 +118128,118128 +118129,118129 +118130,118130 +118184,118184 +118185,118185 +118186,118186 +107111,118186 +118187,118187 +118188,118188 +118189,118189 +118190,118190 +118191,118191 +118192,118192 +118193,118193 +118194,118194 +118195,118195 +118196,118196 +118197,118197 +118198,118198 +118199,118199 +118200,118200 +213935,118200 +118201,118201 +17139,118201 +118202,118202 +118203,118203 +118204,118204 +118205,118205 +118206,118206 +118207,118207 +118208,118208 +118209,118209 +118210,118210 +118211,118211 +118212,118212 +118213,118213 +118214,118214 +118215,118215 +118216,118216 +118217,118217 +118218,118218 +118301,118301 +118302,118302 +118303,118303 +118304,118304 +118305,118305 +118306,118306 +118307,118307 +118308,118308 +118309,118309 +118310,118310 +118311,118311 +118312,118312 +118313,118313 +118314,118314 +118315,118315 +243938,118315 +118316,118316 +118317,118317 +118318,118318 +118319,118319 +118320,118320 +136256,118320 +128669,118320 +118391,118391 +118392,118392 +118432,118432 +159430,118432 +3587,118432 +118433,118433 +118434,118434 +118435,118435 +118436,118436 +118437,118437 +118438,118438 +118439,118439 +118440,118440 +118441,118441 +118442,118442 +118443,118443 +118444,118444 +118445,118445 +118446,118446 +118447,118447 +118448,118448 +81452,118448 +118449,118449 +118450,118450 +118451,118451 +118452,118452 +118453,118453 +118454,118454 +118455,118455 +118456,118456 +118457,118457 +118458,118458 +118459,118459 +183421,118459 +118460,118460 +118461,118461 +118462,118462 +118463,118463 +118464,118464 +118465,118465 +118466,118466 +118467,118467 +118468,118468 +185289,118468 +118469,118469 +118470,118470 +118471,118471 +118472,118472 +118511,118511 +118512,118512 +118513,118513 +50247,118513 +42055,118513 +118514,118514 +118515,118515 +118516,118516 +118517,118517 +118518,118518 +118519,118519 +118520,118520 +118521,118521 +118522,118522 +118523,118523 +118524,118524 +118525,118525 +118526,118526 +118527,118527 +118528,118528 +118529,118529 +118612,118612 +118613,118613 +118614,118614 +118615,118615 +118616,118616 +118617,118617 +149193,118617 +118618,118618 +118619,118619 +118620,118620 +118621,118621 +77991,118621 +180581,118621 +118622,118622 +118623,118623 +118624,118624 +118625,118625 +118626,118626 +118627,118627 +118628,118628 +118629,118629 +118630,118630 +118631,118631 +118632,118632 +118633,118633 +118634,118634 +118635,118635 +118636,118636 +118637,118637 +118638,118638 +118639,118639 +118640,118640 +118641,118641 +118642,118642 +118643,118643 +118644,118644 +118645,118645 +118646,118646 +231947,118646 +231203,118646 +118647,118647 +118648,118648 +118649,118649 +118650,118650 +118651,118651 +118652,118652 +118653,118653 +118654,118654 +118655,118655 +118656,118656 +118657,118657 +118658,118658 +118659,118659 +118660,118660 +118988,118988 +118989,118989 +118990,118990 +118991,118991 +118992,118992 +118993,118993 +118994,118994 +118995,118995 +118996,118996 +118997,118997 +118998,118998 +118999,118999 +119000,119000 +119001,119001 +119002,119002 +119003,119003 +119004,119004 +119005,119005 +138125,119005 +130538,119005 +119006,119006 +119162,119162 +119163,119163 +119164,119164 +119165,119165 +119166,119166 +119167,119167 +112460,119167 +119168,119168 +119169,119169 +119170,119170 +119171,119171 +119172,119172 +65026,119172 +119173,119173 +119174,119174 +20797,119174 +119175,119175 +119176,119176 +119177,119177 +119178,119178 +119179,119179 +77774,119179 +119221,119221 +119222,119222 +78525,119222 +119223,119223 +119224,119224 +242937,119224 +119225,119225 +119226,119226 +119227,119227 +119228,119228 +119229,119229 +119230,119230 +148120,119230 +85213,119230 +119231,119231 +119232,119232 +119233,119233 +119234,119234 +190757,119234 +119235,119235 +119236,119236 +119237,119237 +119238,119238 +119239,119239 +119240,119240 +119241,119241 +119242,119242 +119243,119243 +119244,119244 +119245,119245 +236647,119245 +119246,119246 +119247,119247 +119248,119248 +98323,119248 +119249,119249 +119250,119250 +119251,119251 +119252,119252 +119253,119253 +119254,119254 +119255,119255 +119256,119256 +119257,119257 +119258,119258 +119259,119259 +119260,119260 +119261,119261 +119262,119262 +242299,119262 +119263,119263 +119264,119264 +119265,119265 +119266,119266 +119267,119267 +119268,119268 +132915,119268 +125328,119268 +119269,119269 +119270,119270 +119271,119271 +119272,119272 +119273,119273 +79256,119273 +119274,119274 +119275,119275 +119276,119276 +119277,119277 +119278,119278 +119279,119279 +119280,119280 +119281,119281 +119282,119282 +119283,119283 +119284,119284 +119285,119285 +119286,119286 +119287,119287 +233083,119287 +119288,119288 +11500,119288 +119289,119289 +119290,119290 +49476,119290 +41283,119290 +119291,119291 +119292,119292 +121056,119292 +119584,119584 +119585,119585 +119586,119586 +30514,119586 +119587,119587 +119588,119588 +119589,119589 +119590,119590 +119591,119591 +119592,119592 +119593,119593 +119594,119594 +236561,119594 +119751,119751 +119752,119752 +119753,119753 +119754,119754 +119755,119755 +68628,119755 +119756,119756 +119757,119757 +119758,119758 +119759,119759 +247992,119760 +119760,119760 +119761,119761 +119762,119762 +164534,119762 +119763,119763 +119764,119764 +88143,119764 +119765,119765 +119766,119766 +119767,119767 +119768,119768 +119769,119769 +119770,119770 +119771,119771 +131704,119771 +124117,119771 +119772,119772 +119773,119773 +119774,119774 +119775,119775 +119776,119776 +119777,119777 +201019,119778 +119778,119778 +119779,119779 +119780,119780 +119781,119781 +14543,119781 +119782,119782 +119783,119783 +119784,119784 +119785,119785 +120054,120054 +120055,120055 +120056,120056 +120057,120057 +120058,120058 +120059,120059 +56780,120059 +120060,120060 +214406,120060 +120061,120061 +120062,120062 +120063,120063 +25038,120063 +83216,120063 +120166,120166 +120167,120167 +120168,120168 +120169,120169 +120170,120170 +120221,120170 +120171,120171 +120172,120172 +120173,120173 +120174,120174 +120175,120175 +180587,120175 +120176,120176 +120177,120177 +120178,120178 +120179,120179 +70543,120179 +120180,120180 +86869,120180 +120181,120181 +120182,120182 +120183,120183 +120184,120184 +120185,120185 +120186,120186 +120187,120187 +120188,120188 +120189,120189 +120190,120190 +75206,120190 +120191,120191 +120192,120192 +111425,120192 +120193,120193 +120194,120194 +120195,120195 +120196,120196 +120197,120197 +120198,120198 +120199,120199 +120200,120200 +120201,120201 +120202,120202 +120203,120203 +120204,120204 +120205,120205 +120206,120206 +120207,120207 +120208,120208 +120209,120209 +120210,120210 +120211,120211 +120212,120212 +111630,120212 +120213,120213 +120214,120214 +120215,120215 +120216,120216 +120217,120217 +120218,120218 +120219,120219 +120220,120220 +120221,120221 +120170,120221 +120222,120222 +120223,120223 +120224,120224 +120225,120225 +120226,120226 +120227,120227 +120228,120228 +120229,120229 +120230,120230 +120231,120231 +120232,120232 +120233,120233 +120234,120234 +120235,120235 +120236,120236 +158075,120236 +120237,120237 +120238,120238 +120239,120239 +120240,120240 +120241,120241 +120242,120242 +120243,120243 +120244,120244 +120245,120245 +120246,120246 +120247,120247 +120248,120248 +120249,120249 +120250,120250 +120251,120251 +120252,120252 +120253,120253 +120254,120254 +120255,120255 +120256,120256 +120257,120257 +120258,120258 +120259,120259 +201335,120259 +120260,120260 +205456,120260 +120261,120261 +111946,120261 +120262,120262 +120263,120263 +120264,120264 +120265,120265 +9085,120265 +120266,120266 +120267,120267 +120268,120268 +120269,120269 +120270,120270 +120271,120271 +120272,120272 +120273,120273 +120274,120274 +134095,120274 +126508,120274 +120275,120275 +120276,120276 +120277,120277 +120278,120278 +120279,120279 +120280,120280 +177814,120280 +171850,120280 +120281,120281 +120282,120282 +120283,120283 +120284,120284 +120285,120285 +120286,120286 +120287,120287 +120288,120288 +120289,120289 +120290,120290 +120291,120291 +120292,120292 +120293,120293 +120294,120294 +120295,120295 +120296,120296 +120297,120297 +120298,120298 +120299,120299 +120300,120300 +120301,120301 +120302,120302 +6881,120302 +120303,120303 +120304,120304 +120305,120305 +120306,120306 +120307,120307 +120308,120308 +120309,120309 +120310,120310 +120311,120311 +120312,120312 +198807,120312 +120313,120313 +120314,120314 +2732,120314 +120315,120315 +120316,120316 +120317,120317 +120318,120318 +120319,120319 +120320,120320 +120321,120321 +120322,120322 +120323,120323 +120324,120324 +120386,120386 +97862,120386 +120387,120387 +120388,120388 +120389,120389 +120390,120390 +104886,120390 +120391,120391 +120392,120392 +120393,120393 +120394,120394 +120395,120395 +120396,120396 +120397,120397 +120398,120398 +120399,120399 +120400,120400 +120401,120401 +120402,120402 +120403,120403 +120404,120404 +120405,120405 +120406,120406 +185280,120406 +120407,120407 +120408,120408 +120409,120409 +120410,120410 +120411,120411 +120412,120412 +120413,120413 +120414,120414 +120415,120415 +74193,120415 +120416,120416 +120417,120417 +120418,120418 +120419,120419 +120420,120420 +120421,120421 +120422,120422 +120423,120423 +120424,120424 +120425,120425 +120426,120426 +120427,120427 +120428,120428 +120429,120429 +120430,120430 +184058,120430 +120431,120431 +120432,120432 +120433,120433 +120434,120434 +32340,120434 +120435,120435 +120436,120436 +120437,120437 +120438,120438 +204904,120438 +120439,120439 +120440,120440 +120441,120441 +120442,120442 +120443,120443 +120444,120444 +120445,120445 +120446,120446 +120447,120447 +120448,120448 +120449,120449 +120450,120450 +120451,120451 +120452,120452 +120453,120453 +120454,120454 +120455,120455 +120456,120456 +64175,120456 +120457,120457 +120458,120458 +120459,120459 +120460,120460 +120461,120461 +120462,120462 +120681,120681 +83942,120681 +120682,120682 +120683,120683 +120684,120684 +120685,120685 +194247,120685 +120686,120686 +120687,120687 +120688,120688 +120689,120689 +2760,120689 +120690,120690 +183875,120690 +120691,120691 +120692,120692 +120693,120693 +120694,120694 +120695,120695 +120696,120696 +120697,120697 +140501,120697 +120698,120698 +120699,120699 +120700,120700 +120701,120701 +120702,120702 +120703,120703 +120704,120704 +120705,120705 +13468,120705 +120706,120706 +35485,120706 +120789,120789 +120790,120790 +120791,120791 +120792,120792 +120793,120793 +120794,120794 +120795,120795 +120895,120895 +120896,120896 +120897,120897 +120898,120898 +120899,120899 +120900,120900 +120901,120901 +120902,120902 +120903,120903 +120904,120904 +120947,120947 +164109,120947 +120948,120948 +120949,120949 +120950,120950 +120951,120951 +120952,120952 +120953,120953 +120954,120954 +120955,120955 +120956,120956 +120957,120957 +120958,120958 +33369,120958 +120959,120959 +120960,120960 +120961,120961 +120962,120962 +120963,120963 +120964,120964 +120965,120965 +152909,120965 +120966,120966 +120967,120967 +120968,120968 +133713,120968 +126126,120968 +121052,121052 +209359,121052 +121053,121053 +121054,121054 +121055,121055 +121056,121056 +119292,121056 +121057,121057 +121058,121058 +121059,121059 +121060,121060 +121061,121061 +219817,121061 +121062,121062 +121063,121063 +121064,121064 +121206,121206 +121207,121207 +30787,121207 +121305,121305 +185676,121305 +121306,121306 +121307,121307 +121308,121308 +121309,121309 +161938,121309 +121310,121310 +121311,121311 +65956,121311 +121312,121312 +121313,121313 +121314,121314 +121315,121315 +121316,121316 +239348,121316 +121317,121317 +121318,121318 +121319,121319 +121320,121320 +121321,121321 +121322,121322 +121323,121323 +121324,121324 +121325,121325 +121326,121326 +121327,121327 +121328,121328 +121329,121329 +121330,121330 +121331,121331 +121332,121332 +121333,121333 +121334,121334 +121335,121335 +121336,121336 +121337,121337 +121338,121338 +121339,121339 +121340,121340 +121341,121341 +121342,121342 +121343,121343 +121344,121344 +121345,121345 +121346,121346 +121347,121347 +121348,121348 +121349,121349 +121350,121350 +121351,121351 +121352,121352 +121353,121353 +121354,121354 +121355,121355 +121356,121356 +121357,121357 +121358,121358 +121359,121359 +121360,121360 +121361,121361 +121362,121362 +121363,121363 +121364,121364 +121365,121365 +188372,121365 +121366,121366 +121367,121367 +121368,121368 +121369,121369 +121370,121370 +121371,121371 +121372,121372 +121373,121373 +121374,121374 +121375,121375 +121376,121376 +121377,121377 +121378,121378 +121379,121379 +121380,121380 +121381,121381 +121382,121382 +121383,121383 +121384,121384 +2852,121384 +121385,121385 +121386,121386 +121387,121387 +71066,121387 +121388,121388 +121389,121389 +121390,121390 +121391,121391 +121392,121392 +121393,121393 +121394,121394 +121395,121395 +121396,121396 +121397,121397 +121398,121398 +121399,121399 +121400,121400 +121401,121401 +121402,121402 +121403,121403 +121404,121404 +121405,121405 +121406,121406 +121407,121407 +121408,121408 +233630,121408 +121409,121409 +121410,121410 +121411,121411 +121412,121412 +121413,121413 +121414,121414 +121415,121415 +121416,121416 +121417,121417 +161198,121417 +121418,121418 +121419,121419 +121420,121420 +121421,121421 +121422,121422 +121423,121423 +121424,121424 +121425,121425 +121426,121426 +121427,121427 +121428,121428 +121429,121429 +3842,121429 +121430,121430 +102250,121430 +121431,121431 +121432,121432 +121433,121433 +121434,121434 +121435,121435 +121436,121436 +61771,121436 +121437,121437 +121438,121438 +121439,121439 +206713,121439 +121440,121440 +121441,121441 +121442,121442 +140991,121442 +121443,121443 +121444,121444 +121445,121445 +121446,121446 +121447,121447 +121448,121448 +121449,121449 +121450,121450 +121451,121451 +121452,121452 +121453,121453 +121454,121454 +121455,121455 +121456,121456 +40239,121456 +121457,121457 +121458,121458 +121459,121459 +121460,121460 +121461,121461 +121462,121462 +121463,121463 +121464,121464 +121465,121465 +121466,121466 +121467,121467 +86123,121467 +121468,121468 +121469,121469 +38208,121469 +121470,121470 +121471,121471 +121472,121472 +121473,121473 +121474,121474 +121475,121475 +121476,121476 +141169,121476 +121477,121477 +121478,121478 +121479,121479 +121480,121480 +121481,121481 +121482,121482 +121483,121483 +147569,121483 +121484,121484 +121485,121485 +121486,121486 +121487,121487 +121488,121488 +121489,121489 +121490,121490 +121491,121491 +121492,121492 +121493,121493 +121494,121494 +121495,121495 +121496,121496 +121497,121497 +121498,121498 +121499,121499 +121500,121500 +121501,121501 +192126,121501 +121502,121502 +121503,121503 +121504,121504 +121505,121505 +121506,121506 +121507,121507 +121508,121508 +121509,121509 +121510,121510 +121511,121511 +121512,121512 +121513,121513 +121514,121514 +121515,121515 +121516,121516 +121517,121517 +121518,121518 +121519,121519 +121520,121520 +121521,121521 +121522,121522 +121523,121523 +121524,121524 +121525,121525 +121526,121526 +121527,121527 +121528,121528 +121529,121529 +121530,121530 +121531,121531 +121532,121532 +121533,121533 +121534,121534 +121535,121535 +121536,121536 +121537,121537 +121538,121538 +121539,121539 +121540,121540 +121541,121541 +121542,121542 +57881,121542 +121543,121543 +121544,121544 +121545,121545 +121546,121546 +121547,121547 +121548,121548 +121549,121549 +121550,121550 +121551,121551 +121552,121552 +121553,121553 +121554,121554 +121555,121555 +121556,121556 +121557,121557 +121558,121558 +121559,121559 +121560,121560 +121561,121561 +121562,121562 +121563,121563 +81574,121563 +121564,121564 +121565,121565 +3973,121565 +121566,121566 +121567,121567 +256,121567 +121568,121568 +121569,121569 +220236,121569 +230361,121569 +121612,121612 +121613,121613 +121614,121614 +121615,121615 +121616,121616 +121690,121690 +121691,121691 +121692,121692 +121693,121693 +121694,121694 +121695,121695 +76586,121695 +121696,121696 +155621,121696 +121697,121697 +121698,121698 +121699,121699 +121700,121700 +121701,121701 +121702,121702 +121703,121703 +121704,121704 +121705,121705 +121706,121706 +121872,121872 +121873,121873 +121955,121955 +121993,121993 +121994,121994 +122063,122063 +122064,122064 +122065,122065 +122066,122066 +122067,122067 +122068,122068 +229719,122068 +122121,122121 +122122,122122 +122123,122123 +122124,122124 +122125,122125 +122126,122126 +122127,122127 +122128,122128 +122129,122129 +122130,122130 +122168,122168 +122169,122169 +122170,122170 +122171,122171 +122172,122172 +211394,122172 +242297,122172 +33918,122172 +122173,122173 +60976,122173 +122174,122174 +122175,122175 +232544,122175 +122176,122176 +122177,122177 +122178,122178 +122179,122179 +122180,122180 +122181,122181 +122182,122182 +122183,122183 +122184,122184 +122185,122185 +138863,122185 +5265,122185 +122186,122186 +122187,122187 +122188,122188 +122189,122189 +122190,122190 +242890,122190 +122191,122191 +122192,122192 +122193,122193 +122194,122194 +122195,122195 +122196,122196 +122197,122197 +58131,122197 +122198,122198 +122199,122199 +122200,122200 +122201,122201 +122202,122202 +122203,122203 +122204,122204 +122205,122205 +122206,122206 +122207,122207 +122208,122208 +122209,122209 +122210,122210 +58635,122210 +122211,122211 +122212,122212 +122213,122213 +122214,122214 +122215,122215 +122216,122216 +122217,122217 +122218,122218 +122219,122219 +122220,122220 +122221,122221 +122222,122222 +122223,122223 +122224,122224 +122225,122225 +122226,122226 +122227,122227 +122228,122228 +122229,122229 +122230,122230 +122231,122231 +122232,122232 +122233,122233 +122234,122234 +52737,122234 +44633,122234 +122235,122235 +122236,122236 +122237,122237 +122238,122238 +122239,122239 +122240,122240 +122241,122241 +122242,122242 +122243,122243 +122244,122244 +135241,122244 +127654,122244 +122245,122245 +122246,122246 +122247,122247 +122248,122248 +122249,122249 +122250,122250 +57972,122250 +122251,122251 +122252,122252 +122253,122253 +102062,122253 +122254,122254 +56781,122254 +110671,122254 +122255,122255 +122256,122256 +122257,122257 +122258,122258 +36850,122258 +32773,122258 +122259,122259 +122444,122444 +122445,122445 +122592,122592 +122593,122593 +122594,122594 +122595,122595 +122596,122596 +122597,122597 +187381,122597 +8082,122597 +122598,122598 +122599,122599 +122600,122600 +122601,122601 +133224,122601 +125637,122601 +122602,122602 +122603,122603 +21459,122603 +122604,122604 +122605,122605 +122606,122606 +122607,122607 +133230,122607 +125643,122607 +122608,122608 +122609,122609 +122610,122610 +122611,122611 +122612,122612 +122613,122613 +122614,122614 +122615,122615 +122616,122616 +122617,122617 +122618,122618 +122619,122619 +122620,122620 +122621,122621 +122622,122622 +122623,122623 +122624,122624 +122625,122625 +122626,122626 +122665,122665 +122666,122666 +122667,122667 +122668,122668 +122669,122669 +122670,122670 +122671,122671 +122672,122672 +122673,122673 +122674,122674 +122675,122675 +122676,122676 +122677,122677 +122678,122678 +122679,122679 +122680,122680 +122681,122681 +122682,122682 +122683,122683 +122684,122684 +122685,122685 +122686,122686 +122687,122687 +122688,122688 +122689,122689 +122690,122690 +122691,122691 +122692,122692 +122693,122693 +122694,122694 +122695,122695 +122696,122696 +122697,122697 +122698,122698 +122699,122699 +122700,122700 +122701,122701 +122702,122702 +148183,122702 +122703,122703 +122704,122704 +122705,122705 +104978,122705 +122706,122706 +122707,122707 +122708,122708 +122709,122709 +122710,122710 +122711,122711 +122712,122712 +122713,122713 +122714,122714 +122715,122715 +122716,122716 +122717,122717 +122718,122718 +122719,122719 +122720,122720 +122721,122721 +122722,122722 +122723,122723 +122724,122724 +122725,122725 +122726,122726 +122727,122727 +122728,122728 +209625,122728 +122729,122729 +122730,122730 +122731,122731 +122732,122732 +122733,122733 +122734,122734 +122735,122735 +122736,122736 +122737,122737 +122738,122738 +122739,122739 +122740,122740 +122741,122741 +122742,122742 +122743,122743 +122744,122744 +122745,122745 +122746,122746 +122747,122747 +122748,122748 +122749,122749 +122750,122750 +122751,122751 +122752,122752 +122753,122753 +122754,122754 +122755,122755 +122756,122756 +185568,122756 +122757,122757 +122758,122758 +122759,122759 +122760,122760 +122761,122761 +122762,122762 +122763,122763 +122764,122764 +122765,122765 +122766,122766 +122767,122767 +122768,122768 +122769,122769 +122770,122770 +122771,122771 +122772,122772 +122773,122773 +122774,122774 +122775,122775 +122776,122776 +122777,122777 +122778,122778 +122779,122779 +122780,122780 +122781,122781 +122782,122782 +122783,122783 +122784,122784 +122785,122785 +122786,122786 +122787,122787 +122788,122788 +122789,122789 +122790,122790 +122791,122791 +122792,122792 +122793,122793 +122794,122794 +122795,122795 +122796,122796 +122797,122797 +122798,122798 +122799,122799 +122800,122800 +122801,122801 +122802,122802 +122803,122803 +122804,122804 +187186,122804 +122805,122805 +122806,122806 +122807,122807 +122808,122808 +122809,122809 +122810,122810 +122811,122811 +214572,122811 +122812,122812 +122813,122813 +122814,122814 +122815,122815 +122816,122816 +199540,122816 +201780,122816 +122817,122817 +68055,122817 +137057,122817 +129470,122817 +122818,122818 +122819,122819 +122820,122820 +122821,122821 +122822,122822 +122823,122823 +122824,122824 +122825,122825 +122826,122826 +5538,122826 +122827,122827 +122828,122828 +122829,122829 +122830,122830 +122831,122831 +122832,122832 +122833,122833 +122834,122834 +122835,122835 +122836,122836 +122837,122837 +122838,122838 +122839,122839 +122840,122840 +5385,122840 +122841,122841 +122842,122842 +122843,122843 +122844,122844 +122845,122845 +122846,122846 +122847,122847 +159511,122847 +122848,122848 +122849,122849 +122850,122850 +122851,122851 +122852,122852 +122853,122853 +178836,122853 +122854,122854 +122855,122855 +122856,122856 +122857,122857 +122858,122858 +122859,122859 +198444,122859 +122860,122860 +122861,122861 +122862,122862 +122863,122863 +77359,122863 +122864,122864 +122865,122865 +122866,122866 +122867,122867 +122868,122868 +122869,122869 +122870,122870 +122871,122871 +122872,122872 +122873,122873 +122874,122874 +122875,122875 +122876,122876 +122877,122877 +122878,122878 +122879,122879 +122880,122880 +122881,122881 +122882,122882 +122883,122883 +122884,122884 +122885,122885 +122886,122886 +122887,122887 +122888,122888 +59555,122888 +122889,122889 +122890,122890 +122891,122891 +122892,122892 +122893,122893 +122894,122894 +122895,122895 +122896,122896 +122897,122897 +122898,122898 +122899,122899 +132832,122899 +125245,122899 +122900,122900 +122901,122901 +122902,122902 +122903,122903 +122904,122904 +122905,122905 +122906,122906 +122907,122907 +122908,122908 +122909,122909 +21034,122909 +122910,122910 +122911,122911 +122912,122912 +122913,122913 +122914,122914 +122915,122915 +122916,122916 +122917,122917 +122918,122918 +122919,122919 +122920,122920 +122921,122921 +122922,122922 +122923,122923 +122924,122924 +35629,122924 +122925,122925 +122926,122926 +122927,122927 +122928,122928 +122929,122929 +122930,122930 +122931,122931 +122932,122932 +82331,122932 +122933,122933 +151532,122933 +122934,122934 +122935,122935 +232134,122935 +122936,122936 +122937,122937 +122938,122938 +122939,122939 +122940,122940 +122941,122941 +122942,122942 +122943,122943 +122944,122944 +122945,122945 +122946,122946 +122947,122947 +122948,122948 +122949,122949 +122950,122950 +122951,122951 +122952,122952 +122953,122953 +194231,122953 +122954,122954 +122955,122955 +122956,122956 +122957,122957 +90651,122957 +122958,122958 +122959,122959 +122960,122960 +122961,122961 +122962,122962 +122963,122963 +122964,122964 +122965,122965 +122966,122966 +122967,122967 +122968,122968 +122969,122969 +122970,122970 +122971,122971 +122972,122972 +122973,122973 +122974,122974 +122975,122975 +122976,122976 +122977,122977 +122978,122978 +122979,122979 +122980,122980 +122981,122981 +141441,122981 +122982,122982 +122983,122983 +122984,122984 +122985,122985 +122986,122986 +122987,122987 +122988,122988 +53783,122988 +45679,122988 +122989,122989 +122990,122990 +122991,122991 +122992,122992 +122993,122993 +225932,122993 +122994,122994 +14555,122994 +122995,122995 +122996,122996 +122997,122997 +122998,122998 +122999,122999 +197573,122999 +123000,123000 +123001,123001 +123002,123002 +123003,123003 +123004,123004 +123005,123005 +123006,123006 +123007,123007 +123008,123008 +123009,123009 +123010,123010 +123011,123011 +123012,123012 +123013,123013 +123014,123014 +123015,123015 +123016,123016 +123017,123017 +123018,123018 +123019,123019 +123020,123020 +123021,123021 +123022,123022 +123023,123023 +213396,123023 +123024,123024 +123025,123025 +123026,123026 +123027,123027 +123028,123028 +123029,123029 +123030,123030 +123031,123031 +183452,123031 +123032,123032 +190660,123032 +123033,123033 +123034,123034 +123035,123035 +123036,123036 +123037,123037 +123038,123038 +123039,123039 +145419,123039 +123040,123040 +123041,123041 +123042,123042 +123043,123043 +123044,123044 +123045,123045 +123046,123046 +123047,123047 +123048,123048 +123049,123049 +123050,123050 +123051,123051 +107552,123051 +123052,123052 +123053,123053 +123054,123054 +123055,123055 +123056,123056 +123057,123057 +59166,123057 +58792,123057 +123058,123058 +123059,123059 +123060,123060 +123061,123061 +123062,123062 +123063,123063 +123064,123064 +123065,123065 +123066,123066 +123067,123067 +123068,123068 +123069,123069 +123070,123070 +123071,123071 +123072,123072 +123073,123073 +200766,123073 +123074,123074 +149251,123074 +123075,123075 +123076,123076 +423,123076 +123077,123077 +123078,123078 +123079,123079 +123080,123080 +123081,123081 +123082,123082 +123083,123083 +123084,123084 +123085,123085 +123086,123086 +123087,123087 +123088,123088 +123089,123089 +123090,123090 +123091,123091 +123092,123092 +123093,123093 +123094,123094 +123095,123095 +123096,123096 +123097,123097 +123098,123098 +123099,123099 +123100,123100 +123101,123101 +123102,123102 +20401,123102 +20216,123102 +123103,123103 +123104,123104 +123105,123105 +123106,123106 +123107,123107 +225489,123107 +123108,123108 +123109,123109 +123110,123110 +123111,123111 +3122,123111 +123112,123112 +123113,123113 +123114,123114 +123115,123115 +123116,123116 +123117,123117 +8214,123117 +123118,123118 +123119,123119 +123120,123120 +123121,123121 +123122,123122 +123123,123123 +123124,123124 +123125,123125 +123126,123126 +123127,123127 +123128,123128 +123129,123129 +123130,123130 +123131,123131 +123132,123132 +123133,123133 +123134,123134 +123135,123135 +123136,123136 +123137,123137 +123138,123138 +123139,123139 +123140,123140 +123141,123141 +123142,123142 +123143,123143 +123144,123144 +123145,123145 +123146,123146 +123147,123147 +123148,123148 +123149,123149 +123150,123150 +123151,123151 +123152,123152 +123153,123153 +123154,123154 +123155,123155 +123156,123156 +123157,123157 +123158,123158 +123159,123159 +123160,123160 +123161,123161 +123162,123162 +123163,123163 +123164,123164 +123165,123165 +178109,123165 +172145,123165 +123166,123166 +123167,123167 +123168,123168 +123169,123169 +123170,123170 +123171,123171 +123172,123172 +123173,123173 +123174,123174 +123175,123175 +123176,123176 +123177,123177 +123178,123178 +216221,123178 +123179,123179 +123180,123180 +123181,123181 +123182,123182 +123183,123183 +123184,123184 +123185,123185 +123186,123186 +123187,123187 +123188,123188 +67202,123188 +123189,123189 +123190,123190 +123191,123191 +123192,123192 +123193,123193 +123194,123194 +123264,123264 +123265,123265 +123266,123266 +123267,123267 +238483,123267 +123268,123268 +123269,123269 +123270,123270 +123271,123271 +123272,123272 +123273,123273 +123274,123274 +123275,123275 +64691,123275 +123276,123276 +145261,123276 +123277,123277 +123278,123278 +123279,123279 +142017,123279 +123280,123280 +123281,123281 +123282,123282 +123283,123283 +123284,123284 +123285,123285 +123286,123286 +123287,123287 +249581,123287 +123288,123288 +168,123288 +123289,123289 +123290,123290 +123291,123291 +123292,123292 +123293,123293 +210144,123293 +123294,123294 +123295,123295 +123296,123296 +51521,123296 +43417,123296 +123297,123297 +123298,123298 +123299,123299 +210001,123299 +123347,123347 +123348,123348 +153893,123348 +123349,123349 +123350,123350 +123351,123351 +123352,123352 +123353,123353 +123354,123354 +123355,123355 +123356,123356 +123357,123357 +123358,123358 +123359,123359 +123360,123360 +123361,123361 +123362,123362 +160888,123362 +123363,123363 +123364,123364 +123365,123365 +123366,123366 +123367,123367 +123368,123368 +211673,123368 +123369,123369 +123370,123370 +123371,123371 +123372,123372 +123373,123373 +123374,123374 +123375,123375 +123376,123376 +123377,123377 +123378,123378 +123379,123379 +123380,123380 +123381,123381 +123382,123382 +123383,123383 +123384,123384 +123385,123385 +123386,123386 +123387,123387 +123388,123388 +123389,123389 +123390,123390 +123391,123391 +123392,123392 +123393,123393 +110613,123393 +123394,123394 +123395,123395 +123396,123396 +123397,123397 +123398,123398 +123399,123399 +110972,123399 +123400,123400 +123401,123401 +123446,123446 +123494,123494 +123495,123495 +123496,123496 +31476,123496 +123497,123497 +131913,123497 +124326,123497 +123498,123498 +131914,123498 +124327,123498 +123499,123499 +123500,123500 +145713,123500 +123501,123501 +29001,123501 +123502,123502 +123503,123503 +123504,123504 +123505,123505 +123506,123506 +123507,123507 +185391,123507 +123508,123508 +123509,123509 +12063,123509 +123510,123510 +123511,123511 +123512,123512 +123513,123513 +123514,123514 +123515,123515 +123516,123516 +123517,123517 +123518,123518 +55841,123518 +47737,123518 +123519,123519 +123520,123520 +123691,123691 +123692,123692 +123693,123693 +123928,123928 +123929,123929 +123930,123930 +123931,123931 +123932,123932 +123933,123933 +123934,123934 +123935,123935 +123936,123936 +123937,123937 +123938,123938 +216288,123938 +123939,123939 +123940,123940 +123941,123941 +123942,123942 +123943,123943 +139837,123944 +123944,123944 +123945,123945 +123946,123946 +154071,123946 +123947,123947 +123948,123948 +123949,123949 +123950,123950 +123951,123951 +123952,123952 +82571,123952 +123953,123953 +123954,123954 +123955,123955 +123956,123956 +123957,123957 +123958,123958 +123959,123959 +123960,123960 +123961,123961 +123962,123962 +123963,123963 +123964,123964 +180737,123964 +123965,123965 +123966,123966 +123967,123967 +123968,123968 +123969,123969 +123970,123970 +123971,123971 +123972,123972 +123973,123973 +123974,123974 +123975,123975 +123976,123976 +123977,123977 +123978,123978 +123979,123979 +123980,123980 +123981,123981 +123982,123982 +123983,123983 +63762,123983 +123984,123984 +123985,123985 +123986,123986 +67379,123986 +123987,123987 +123988,123988 +124028,124028 +124067,124067 +124068,124068 +124069,124069 +124070,124070 +124071,124071 +124072,124072 +124073,124073 +124074,124074 +54985,124074 +46881,124074 +124075,124075 +124076,124076 +124077,124077 +124078,124078 +124079,124079 +124080,124080 +5293,124080 +124081,124081 +124082,124082 +124083,124083 +112100,124083 +124084,124084 +124085,124085 +124086,124086 +124087,124087 +135955,124087 +128368,124087 +124088,124088 +89464,124088 +124089,124089 +124090,124090 +124091,124091 +124092,124092 +124093,124093 +124094,124094 +124095,124095 +124096,124096 +124097,124097 +124098,124098 +124099,124099 +124100,124100 +248698,124101 +124101,124101 +242283,124101 +124102,124102 +124103,124103 +124104,124104 +131692,124105 +124105,124105 +131693,124106 +124106,124106 +131694,124107 +124107,124107 +131695,124108 +124108,124108 +131696,124109 +124109,124109 +131697,124110 +124110,124110 +131698,124111 +124111,124111 +131699,124112 +124112,124112 +131700,124113 +124113,124113 +131701,124114 +124114,124114 +131702,124115 +124115,124115 +131703,124116 +124116,124116 +131704,124117 +124117,124117 +119771,124117 +131705,124118 +124118,124118 +131706,124119 +124119,124119 +131707,124120 +124120,124120 +131708,124121 +124121,124121 +131709,124122 +124122,124122 +131710,124123 +124123,124123 +131711,124124 +124124,124124 +131712,124125 +124125,124125 +131713,124126 +124126,124126 +131714,124127 +124127,124127 +131715,124128 +124128,124128 +131716,124129 +124129,124129 +131717,124130 +124130,124130 +181268,124130 +131718,124131 +124131,124131 +131719,124132 +124132,124132 +131720,124133 +124133,124133 +131721,124134 +124134,124134 +131722,124135 +124135,124135 +131723,124136 +124136,124136 +131724,124137 +124137,124137 +132115,124138 +131725,124138 +124528,124138 +124138,124138 +131726,124139 +124139,124139 +131727,124140 +124140,124140 +131728,124141 +124141,124141 +131729,124142 +124142,124142 +131730,124143 +124143,124143 +131731,124144 +124144,124144 +131732,124145 +124145,124145 +131754,124145 +124167,124145 +131755,124146 +131733,124146 +124168,124146 +124146,124146 +131734,124147 +124147,124147 +131735,124148 +124148,124148 +131736,124149 +124149,124149 +131737,124150 +124150,124150 +131738,124151 +124151,124151 +131739,124152 +124152,124152 +131740,124153 +124153,124153 +131741,124154 +124154,124154 +131742,124155 +124155,124155 +131743,124156 +124156,124156 +131744,124157 +124157,124157 +131745,124158 +124158,124158 +131746,124159 +124159,124159 +131747,124160 +124160,124160 +131748,124161 +124161,124161 +131749,124162 +124162,124162 +131750,124163 +124163,124163 +131751,124164 +124164,124164 +131752,124165 +124165,124165 +131753,124166 +124166,124166 +162851,124166 +131754,124167 +124167,124167 +131732,124167 +124145,124167 +131755,124168 +131733,124168 +124168,124168 +124146,124168 +131756,124169 +124169,124169 +131757,124170 +124170,124170 +131758,124171 +124171,124171 +131759,124172 +124172,124172 +131760,124173 +124173,124173 +131761,124174 +124174,124174 +131762,124175 +124175,124175 +131763,124176 +124176,124176 +131764,124177 +124177,124177 +131765,124178 +124178,124178 +131766,124179 +124179,124179 +131767,124180 +124180,124180 +131768,124181 +124181,124181 +131769,124182 +124182,124182 +131770,124183 +124183,124183 +131771,124184 +124184,124184 +131772,124185 +124185,124185 +131773,124186 +124186,124186 +131774,124187 +124187,124187 +78630,124187 +131775,124188 +124188,124188 +131776,124189 +124189,124189 +131777,124190 +124190,124190 +131778,124191 +124191,124191 +131779,124192 +124192,124192 +131780,124193 +124193,124193 +131781,124194 +124194,124194 +131782,124195 +124195,124195 +131783,124196 +124196,124196 +131784,124197 +124197,124197 +131785,124198 +124198,124198 +131786,124199 +124199,124199 +131787,124200 +124200,124200 +131788,124201 +124201,124201 +131789,124202 +124202,124202 +131790,124203 +124203,124203 +131791,124204 +124204,124204 +131792,124205 +124205,124205 +131793,124206 +124206,124206 +131794,124207 +124207,124207 +131795,124208 +124208,124208 +131796,124209 +124209,124209 +133088,124210 +131797,124210 +125501,124210 +124210,124210 +131798,124211 +124211,124211 +131799,124212 +124212,124212 +131800,124213 +124213,124213 +131801,124214 +124214,124214 +131896,124309 +124309,124309 +131897,124310 +124310,124310 +131898,124311 +124311,124311 +131899,124312 +124312,124312 +131900,124313 +124313,124313 +131901,124314 +124314,124314 +131902,124315 +124315,124315 +131903,124316 +124316,124316 +131904,124317 +124317,124317 +131905,124318 +124318,124318 +131906,124319 +124319,124319 +131907,124320 +124320,124320 +131908,124321 +124321,124321 +84948,124321 +131909,124322 +124322,124322 +132102,124322 +124515,124322 +131910,124323 +124323,124323 +131911,124324 +124324,124324 +132104,124324 +124517,124324 +131912,124325 +124325,124325 +136999,124325 +129412,124325 +131913,124326 +124326,124326 +123497,124326 +131914,124327 +124327,124327 +123498,124327 +131915,124328 +124328,124328 +131916,124329 +124329,124329 +215648,124329 +131917,124330 +124330,124330 +132111,124331 +131918,124331 +124524,124331 +124331,124331 +131919,124332 +124332,124332 +132113,124333 +131920,124333 +124526,124333 +124333,124333 +131921,124334 +124334,124334 +131922,124335 +124335,124335 +131923,124336 +124336,124336 +132117,124337 +131924,124337 +124530,124337 +124337,124337 +67591,124337 +131925,124338 +124338,124338 +132118,124338 +124531,124338 +131926,124339 +124339,124339 +131927,124340 +124340,124340 +55947,124340 +47843,124340 +131928,124341 +124341,124341 +131929,124342 +124342,124342 +131930,124343 +124343,124343 +131931,124344 +124344,124344 +132124,124344 +124537,124344 +64913,124344 +131932,124345 +124345,124345 +131933,124346 +124346,124346 +132126,124346 +124539,124346 +131934,124347 +124347,124347 +132127,124347 +124540,124347 +131935,124348 +124348,124348 +131936,124349 +124349,124349 +131937,124350 +124350,124350 +205090,124350 +131938,124351 +124351,124351 +131939,124352 +124352,124352 +131940,124353 +124353,124353 +131941,124354 +124354,124354 +131942,124355 +124355,124355 +131943,124356 +124356,124356 +131944,124357 +124357,124357 +131945,124358 +124358,124358 +131946,124359 +124359,124359 +131947,124360 +124360,124360 +131948,124361 +124361,124361 +131949,124362 +124362,124362 +131950,124363 +124363,124363 +131951,124364 +124364,124364 +131952,124365 +124365,124365 +131953,124366 +124366,124366 +131954,124367 +124367,124367 +131955,124368 +124368,124368 +131956,124369 +124369,124369 +131957,124370 +124370,124370 +131958,124371 +124371,124371 +131959,124372 +124372,124372 +111880,124372 +131960,124373 +124373,124373 +131961,124374 +124374,124374 +131962,124375 +124375,124375 +131963,124376 +124376,124376 +131964,124377 +124377,124377 +131965,124378 +124378,124378 +131966,124379 +124379,124379 +131967,124380 +124380,124380 +131968,124381 +124381,124381 +131969,124382 +124382,124382 +131970,124383 +124383,124383 +246669,124383 +131971,124384 +124384,124384 +131972,124385 +124385,124385 +131973,124386 +124386,124386 +131974,124387 +124387,124387 +131975,124388 +124388,124388 +131976,124389 +124389,124389 +187378,124389 +131977,124390 +124390,124390 +131978,124391 +124391,124391 +131979,124392 +124392,124392 +131980,124393 +124393,124393 +243144,124393 +133079,124394 +131981,124394 +125492,124394 +124394,124394 +136322,124394 +128735,124394 +131982,124395 +124395,124395 +131983,124396 +124396,124396 +131984,124397 +124397,124397 +131985,124398 +124398,124398 +131986,124399 +124399,124399 +131987,124400 +124400,124400 +131988,124401 +124401,124401 +131989,124402 +124402,124402 +131990,124403 +124403,124403 +131991,124404 +124404,124404 +131992,124405 +124405,124405 +131993,124406 +124406,124406 +139150,124406 +131994,124407 +124407,124407 +131995,124408 +124408,124408 +131996,124409 +124409,124409 +132100,124513 +124513,124513 +132101,124514 +124514,124514 +229163,124514 +132102,124515 +124515,124515 +131909,124515 +124322,124515 +132103,124516 +124516,124516 +132104,124517 +124517,124517 +131911,124517 +124324,124517 +132105,124518 +124518,124518 +112453,124518 +132106,124519 +124519,124519 +132107,124520 +124520,124520 +132108,124521 +124521,124521 +16456,124521 +132109,124522 +124522,124522 +145713,124522 +132110,124523 +124523,124523 +132111,124524 +131918,124524 +124524,124524 +124331,124524 +132112,124525 +124525,124525 +249575,124525 +132113,124526 +131920,124526 +124526,124526 +124333,124526 +132114,124527 +124527,124527 +132115,124528 +131725,124528 +124528,124528 +124138,124528 +132116,124529 +124529,124529 +132117,124530 +131924,124530 +124530,124530 +124337,124530 +67591,124530 +132118,124531 +124531,124531 +131925,124531 +124338,124531 +132119,124532 +124532,124532 +132120,124533 +124533,124533 +132121,124534 +124534,124534 +132122,124535 +124535,124535 +132123,124536 +124536,124536 +132124,124537 +124537,124537 +131931,124537 +124344,124537 +132125,124538 +124538,124538 +132126,124539 +124539,124539 +28096,124539 +131933,124539 +124346,124539 +132127,124540 +124540,124540 +131934,124540 +124347,124540 +132128,124541 +124541,124541 +132129,124542 +124542,124542 +132130,124543 +124543,124543 +132131,124544 +124544,124544 +132132,124545 +124545,124545 +132133,124546 +124546,124546 +132134,124547 +124547,124547 +132135,124548 +124548,124548 +132136,124549 +124549,124549 +132137,124550 +124550,124550 +132138,124551 +124551,124551 +132286,124699 +124699,124699 +132287,124700 +124700,124700 +132288,124701 +124701,124701 +132289,124702 +124702,124702 +132290,124703 +124703,124703 +132291,124704 +124704,124704 +34052,124704 +157555,124704 +237996,124704 +132292,124705 +124705,124705 +132293,124706 +124706,124706 +132294,124707 +124707,124707 +132295,124708 +124708,124708 +132296,124709 +124709,124709 +132297,124710 +124710,124710 +132298,124711 +124711,124711 +132299,124712 +124712,124712 +132300,124713 +124713,124713 +34061,124713 +132301,124714 +124714,124714 +132302,124715 +124715,124715 +147640,124715 +132303,124716 +124716,124716 +132304,124717 +124717,124717 +132305,124718 +124718,124718 +132306,124719 +124719,124719 +132307,124720 +124720,124720 +132308,124721 +124721,124721 +132309,124722 +124722,124722 +146422,124722 +132310,124723 +124723,124723 +226661,124723 +132311,124724 +124724,124724 +132312,124725 +124725,124725 +132313,124726 +124726,124726 +132350,124763 +124763,124763 +132351,124764 +124764,124764 +132352,124765 +124765,124765 +132353,124766 +124766,124766 +132354,124767 +124767,124767 +140980,124767 +132355,124768 +124768,124768 +132356,124769 +124769,124769 +132357,124770 +124770,124770 +132358,124771 +124771,124771 +132359,124772 +124772,124772 +132360,124773 +124773,124773 +142862,124773 +132361,124774 +124774,124774 +132362,124775 +124775,124775 +132363,124776 +124776,124776 +132364,124777 +124777,124777 +132365,124778 +124778,124778 +132366,124779 +124779,124779 +132367,124780 +124780,124780 +132368,124781 +124781,124781 +132369,124782 +124782,124782 +142855,124782 +132370,124783 +124783,124783 +139985,124783 +132371,124784 +124784,124784 +132372,124785 +124785,124785 +132373,124786 +124786,124786 +83044,124786 +132374,124787 +124787,124787 +132375,124788 +124788,124788 +132376,124789 +124789,124789 +132377,124790 +124790,124790 +132378,124791 +124791,124791 +132379,124792 +124792,124792 +141741,124792 +132380,124793 +124793,124793 +132381,124794 +124794,124794 +132382,124795 +124795,124795 +132383,124796 +124796,124796 +231828,124796 +132384,124797 +124797,124797 +132385,124798 +124798,124798 +132386,124799 +124799,124799 +132387,124800 +124800,124800 +132388,124801 +124801,124801 +146482,124801 +132389,124802 +124802,124802 +132390,124803 +124803,124803 +132391,124804 +124804,124804 +132392,124805 +124805,124805 +132393,124806 +124806,124806 +132394,124807 +124807,124807 +132395,124808 +124808,124808 +132396,124809 +124809,124809 +182320,124809 +66487,124809 +132397,124810 +124810,124810 +142465,124810 +132398,124811 +124811,124811 +31656,124811 +132399,124812 +124812,124812 +132400,124813 +124813,124813 +132401,124814 +124814,124814 +132402,124815 +124815,124815 +132403,124816 +124816,124816 +132404,124817 +124817,124817 +57604,124817 +132405,124818 +124818,124818 +132406,124819 +124819,124819 +132407,124820 +124820,124820 +132408,124821 +124821,124821 +54986,124821 +46882,124821 +132409,124822 +124822,124822 +132410,124823 +124823,124823 +132411,124824 +124824,124824 +132412,124825 +124825,124825 +132451,124864 +124864,124864 +132452,124865 +124865,124865 +132453,124866 +124866,124866 +132497,124910 +124910,124910 +132498,124911 +124911,124911 +142119,124911 +132499,124912 +124912,124912 +132500,124913 +124913,124913 +98239,124913 +132501,124914 +124914,124914 +132502,124915 +124915,124915 +132503,124916 +124916,124916 +132504,124917 +124917,124917 +132505,124918 +124918,124918 +132506,124919 +124919,124919 +132507,124920 +124920,124920 +132508,124921 +124921,124921 +132509,124922 +124922,124922 +132510,124923 +124923,124923 +132511,124924 +124924,124924 +132512,124925 +124925,124925 +132513,124926 +124926,124926 +132514,124927 +124927,124927 +132515,124928 +124928,124928 +132516,124929 +124929,124929 +132517,124930 +124930,124930 +132518,124931 +124931,124931 +132519,124932 +124932,124932 +132520,124933 +124933,124933 +132521,124934 +124934,124934 +132522,124935 +124935,124935 +132523,124936 +124936,124936 +132524,124937 +124937,124937 +132525,124938 +124938,124938 +132526,124939 +124939,124939 +132527,124940 +124940,124940 +132528,124941 +124941,124941 +132529,124942 +124942,124942 +132530,124943 +124943,124943 +132531,124944 +124944,124944 +132532,124945 +124945,124945 +132533,124946 +124946,124946 +132534,124947 +124947,124947 +132596,125009 +125009,125009 +132597,125010 +125010,125010 +11561,125010 +132598,125011 +125011,125011 +132599,125012 +125012,125012 +132600,125013 +125013,125013 +132601,125014 +125014,125014 +132602,125015 +125015,125015 +223447,125015 +132603,125016 +125016,125016 +132604,125017 +125017,125017 +132605,125018 +125018,125018 +132606,125019 +125019,125019 +132607,125020 +125020,125020 +218039,125020 +132608,125021 +125021,125021 +5116,125021 +132609,125022 +125022,125022 +132610,125023 +125023,125023 +132611,125024 +125024,125024 +132612,125025 +125025,125025 +132613,125026 +125026,125026 +132614,125027 +125027,125027 +132615,125028 +125028,125028 +132616,125029 +125029,125029 +132617,125030 +125030,125030 +132618,125031 +125031,125031 +134051,125031 +126464,125031 +132619,125032 +125032,125032 +132620,125033 +125033,125033 +132621,125034 +125034,125034 +132622,125035 +125035,125035 +132623,125036 +125036,125036 +132624,125037 +125037,125037 +132625,125038 +125038,125038 +132626,125039 +125039,125039 +111018,125039 +132627,125040 +125040,125040 +132628,125041 +125041,125041 +132629,125042 +125042,125042 +132630,125043 +125043,125043 +132631,125044 +125044,125044 +132632,125045 +125045,125045 +132633,125046 +125046,125046 +132634,125047 +125047,125047 +132635,125048 +125048,125048 +132636,125049 +125049,125049 +132637,125050 +125050,125050 +132638,125051 +125051,125051 +132639,125052 +125052,125052 +132640,125053 +125053,125053 +132641,125054 +125054,125054 +132642,125055 +125055,125055 +88047,125055 +132643,125056 +125056,125056 +132644,125057 +125057,125057 +132645,125058 +125058,125058 +132646,125059 +125059,125059 +132647,125060 +125060,125060 +132648,125061 +125061,125061 +132649,125062 +125062,125062 +132650,125063 +125063,125063 +132651,125064 +125064,125064 +132652,125065 +125065,125065 +132701,125114 +125114,125114 +132702,125115 +125115,125115 +132703,125116 +125116,125116 +132704,125117 +125117,125117 +132705,125118 +125118,125118 +132706,125119 +125119,125119 +132707,125120 +125120,125120 +132708,125121 +125121,125121 +132709,125122 +125122,125122 +132710,125123 +125123,125123 +132711,125124 +125124,125124 +132712,125125 +125125,125125 +2208,125125 +132713,125126 +125126,125126 +132714,125127 +125127,125127 +132715,125128 +125128,125128 +247362,125128 +132716,125129 +125129,125129 +192035,125129 +132717,125130 +125130,125130 +132718,125131 +125131,125131 +132719,125132 +125132,125132 +132720,125133 +125133,125133 +132721,125134 +125134,125134 +132722,125135 +125135,125135 +132723,125136 +125136,125136 +132724,125137 +125137,125137 +132725,125138 +125138,125138 +132726,125139 +125139,125139 +132727,125140 +125140,125140 +132728,125141 +125141,125141 +132729,125142 +125142,125142 +132730,125143 +125143,125143 +132731,125144 +125144,125144 +132732,125145 +125145,125145 +132733,125146 +125146,125146 +132734,125147 +125147,125147 +132735,125148 +125148,125148 +132736,125149 +125149,125149 +132737,125150 +125150,125150 +132738,125151 +125151,125151 +132739,125152 +125152,125152 +132740,125153 +125153,125153 +132741,125154 +125154,125154 +132742,125155 +125155,125155 +132743,125156 +125156,125156 +132744,125157 +125157,125157 +132745,125158 +125158,125158 +132746,125159 +125159,125159 +132747,125160 +125160,125160 +132748,125161 +125161,125161 +132749,125162 +125162,125162 +132750,125163 +125163,125163 +132751,125164 +125164,125164 +132752,125165 +125165,125165 +132753,125166 +125166,125166 +132754,125167 +125167,125167 +238930,125167 +213582,125167 +132755,125168 +125168,125168 +132756,125169 +125169,125169 +88179,125169 +132757,125170 +125170,125170 +132758,125171 +125171,125171 +132759,125172 +125172,125172 +132760,125173 +125173,125173 +132761,125174 +125174,125174 +132762,125175 +125175,125175 +132763,125176 +125176,125176 +132764,125177 +125177,125177 +132765,125178 +125178,125178 +132766,125179 +125179,125179 +132767,125180 +125180,125180 +132768,125181 +125181,125181 +57730,125181 +132769,125182 +125182,125182 +132770,125183 +125183,125183 +132771,125184 +125184,125184 +132772,125185 +125185,125185 +132773,125186 +125186,125186 +132774,125187 +125187,125187 +191284,125187 +132775,125188 +125188,125188 +132776,125189 +125189,125189 +132777,125190 +125190,125190 +132778,125191 +125191,125191 +132779,125192 +125192,125192 +132780,125193 +125193,125193 +132781,125194 +125194,125194 +132782,125195 +125195,125195 +132783,125196 +125196,125196 +132784,125197 +125197,125197 +132785,125198 +125198,125198 +132786,125199 +125199,125199 +132787,125200 +125200,125200 +201468,125200 +132788,125201 +125201,125201 +132789,125202 +125202,125202 +132790,125203 +125203,125203 +132791,125204 +125204,125204 +132792,125205 +125205,125205 +132793,125206 +125206,125206 +178306,125206 +172342,125206 +132794,125207 +125207,125207 +132795,125208 +125208,125208 +132796,125209 +125209,125209 +236432,125210 +132797,125210 +125210,125210 +132798,125211 +125211,125211 +132799,125212 +125212,125212 +152977,125212 +132800,125213 +125213,125213 +132801,125214 +125214,125214 +242926,125214 +132802,125215 +125215,125215 +132803,125216 +125216,125216 +132804,125217 +125217,125217 +132805,125218 +125218,125218 +132806,125219 +125219,125219 +132807,125220 +125220,125220 +132808,125221 +125221,125221 +132809,125222 +125222,125222 +132810,125223 +125223,125223 +132811,125224 +125224,125224 +132812,125225 +125225,125225 +132813,125226 +125226,125226 +132814,125227 +125227,125227 +132815,125228 +125228,125228 +132816,125229 +125229,125229 +132817,125230 +125230,125230 +132818,125231 +125231,125231 +132819,125232 +125232,125232 +132820,125233 +125233,125233 +132821,125234 +125234,125234 +132822,125235 +125235,125235 +132823,125236 +125236,125236 +132824,125237 +125237,125237 +132825,125238 +125238,125238 +132826,125239 +125239,125239 +132827,125240 +125240,125240 +132828,125241 +125241,125241 +132829,125242 +125242,125242 +132830,125243 +125243,125243 +132831,125244 +125244,125244 +36116,125244 +132832,125245 +125245,125245 +122899,125245 +132833,125246 +125246,125246 +132834,125247 +125247,125247 +132835,125248 +125248,125248 +207850,125248 +132836,125249 +125249,125249 +132837,125250 +125250,125250 +132838,125251 +125251,125251 +132839,125252 +125252,125252 +132840,125253 +125253,125253 +132841,125254 +125254,125254 +132842,125255 +125255,125255 +132843,125256 +125256,125256 +132913,125326 +125326,125326 +132914,125327 +125327,125327 +132915,125328 +125328,125328 +25075,125328 +119268,125328 +132916,125329 +125329,125329 +132917,125330 +125330,125330 +132918,125331 +125331,125331 +132919,125332 +125332,125332 +132920,125333 +125333,125333 +132921,125334 +125334,125334 +132922,125335 +125335,125335 +132923,125336 +125336,125336 +132924,125337 +125337,125337 +132925,125338 +125338,125338 +132926,125339 +125339,125339 +132927,125340 +125340,125340 +132928,125341 +125341,125341 +206858,125341 +132929,125342 +125342,125342 +50150,125342 +132930,125343 +125343,125343 +132931,125344 +125344,125344 +132932,125345 +125345,125345 +132933,125346 +125346,125346 +132934,125347 +125347,125347 +132935,125348 +125348,125348 +132936,125349 +125349,125349 +33507,125349 +234861,125349 +132937,125350 +125350,125350 +132938,125351 +125351,125351 +132939,125352 +125352,125352 +132940,125353 +125353,125353 +132941,125354 +125354,125354 +132942,125355 +125355,125355 +132943,125356 +125356,125356 +30227,125356 +132944,125357 +125357,125357 +214671,125357 +132945,125358 +125358,125358 +132946,125359 +125359,125359 +132947,125360 +125360,125360 +132948,125361 +125361,125361 +132949,125362 +125362,125362 +132950,125363 +125363,125363 +132951,125364 +125364,125364 +132952,125365 +125365,125365 +132953,125366 +125366,125366 +132954,125367 +125367,125367 +21971,125367 +132955,125368 +125368,125368 +132956,125369 +125369,125369 +132957,125370 +125370,125370 +132958,125371 +125371,125371 +132959,125372 +125372,125372 +132960,125373 +125373,125373 +132961,125374 +125374,125374 +132962,125375 +125375,125375 +132963,125376 +125376,125376 +132964,125377 +125377,125377 +194975,125378 +132965,125378 +125378,125378 +132966,125379 +125379,125379 +4577,125379 +132967,125380 +125380,125380 +132968,125381 +125381,125381 +132969,125382 +125382,125382 +132970,125383 +125383,125383 +132971,125384 +125384,125384 +132972,125385 +125385,125385 +132973,125386 +125386,125386 +132974,125387 +125387,125387 +132975,125388 +125388,125388 +132976,125389 +125389,125389 +132977,125390 +125390,125390 +132978,125391 +125391,125391 +132979,125392 +125392,125392 +132980,125393 +125393,125393 +132981,125394 +125394,125394 +132982,125395 +125395,125395 +132983,125396 +125396,125396 +132984,125397 +125397,125397 +132985,125398 +125398,125398 +132986,125399 +125399,125399 +132987,125400 +125400,125400 +114424,125400 +132988,125401 +125401,125401 +132989,125402 +125402,125402 +132990,125403 +125403,125403 +132991,125404 +125404,125404 +132992,125405 +125405,125405 +133031,125444 +125444,125444 +133032,125445 +125445,125445 +232548,125445 +133033,125446 +125446,125446 +133034,125447 +125447,125447 +31643,125447 +133035,125448 +125448,125448 +133036,125449 +125449,125449 +133037,125450 +125450,125450 +209943,125450 +133038,125451 +125451,125451 +149849,125451 +97739,125451 +133039,125452 +125452,125452 +133040,125453 +125453,125453 +193271,125453 +133041,125454 +125454,125454 +162960,125454 +133042,125455 +125455,125455 +133043,125456 +125456,125456 +133044,125457 +125457,125457 +133045,125458 +125458,125458 +133046,125459 +125459,125459 +133047,125460 +125460,125460 +133048,125461 +125461,125461 +133049,125462 +125462,125462 +133050,125463 +125463,125463 +133051,125464 +125464,125464 +133052,125465 +125465,125465 +133053,125466 +125466,125466 +133054,125467 +125467,125467 +133055,125468 +125468,125468 +133056,125469 +125469,125469 +133057,125470 +125470,125470 +133058,125471 +125471,125471 +133059,125472 +125472,125472 +57332,125472 +133060,125473 +125473,125473 +133061,125474 +125474,125474 +133062,125475 +125475,125475 +133063,125476 +125476,125476 +133064,125477 +125477,125477 +133065,125478 +125478,125478 +133066,125479 +125479,125479 +133067,125480 +125480,125480 +133068,125481 +125481,125481 +133069,125482 +125482,125482 +133070,125483 +125483,125483 +133071,125484 +125484,125484 +133072,125485 +125485,125485 +133073,125486 +125486,125486 +133074,125487 +125487,125487 +133075,125488 +125488,125488 +133076,125489 +125489,125489 +133077,125490 +125490,125490 +111836,125490 +133078,125491 +125491,125491 +133079,125492 +131981,125492 +125492,125492 +124394,125492 +136322,125492 +128735,125492 +133080,125493 +125493,125493 +133081,125494 +125494,125494 +133082,125495 +125495,125495 +133083,125496 +125496,125496 +133084,125497 +125497,125497 +133085,125498 +125498,125498 +133086,125499 +125499,125499 +133087,125500 +125500,125500 +133088,125501 +131797,125501 +125501,125501 +124210,125501 +133089,125502 +125502,125502 +133090,125503 +125503,125503 +133091,125504 +125504,125504 +133092,125505 +125505,125505 +133093,125506 +125506,125506 +133094,125507 +125507,125507 +133095,125508 +125508,125508 +58665,125508 +133096,125509 +125509,125509 +133097,125510 +125510,125510 +133098,125511 +125511,125511 +67626,125511 +133099,125512 +125512,125512 +133100,125513 +125513,125513 +133101,125514 +125514,125514 +62005,125514 +57317,125514 +133102,125515 +125515,125515 +133103,125516 +125516,125516 +133104,125517 +125517,125517 +133105,125518 +125518,125518 +133106,125519 +125519,125519 +133107,125520 +125520,125520 +133108,125521 +125521,125521 +133109,125522 +125522,125522 +133110,125523 +125523,125523 +133111,125524 +125524,125524 +133112,125525 +125525,125525 +133113,125526 +125526,125526 +133114,125527 +125527,125527 +133181,125594 +125594,125594 +133182,125595 +125595,125595 +133183,125596 +125596,125596 +133184,125597 +125597,125597 +133185,125598 +125598,125598 +133186,125599 +125599,125599 +149100,125599 +133187,125600 +125600,125600 +56078,125600 +47974,125600 +133188,125601 +125601,125601 +133189,125602 +125602,125602 +133190,125603 +125603,125603 +133191,125604 +125604,125604 +133192,125605 +125605,125605 +133193,125606 +125606,125606 +59375,125606 +2443,125606 +133194,125607 +125607,125607 +133195,125608 +125608,125608 +2445,125608 +133196,125609 +125609,125609 +133197,125610 +125610,125610 +133198,125611 +125611,125611 +133199,125612 +125612,125612 +133200,125613 +125613,125613 +133201,125614 +125614,125614 +133202,125615 +125615,125615 +133203,125616 +125616,125616 +161209,125616 +133204,125617 +125617,125617 +133205,125618 +125618,125618 +133206,125619 +125619,125619 +133207,125620 +125620,125620 +133208,125621 +125621,125621 +133209,125622 +125622,125622 +133210,125623 +125623,125623 +133211,125624 +125624,125624 +10974,125624 +133212,125625 +125625,125625 +133213,125626 +125626,125626 +133214,125627 +125627,125627 +133215,125628 +125628,125628 +133216,125629 +125629,125629 +133217,125630 +125630,125630 +133218,125631 +125631,125631 +133219,125632 +125632,125632 +5357,125632 +133220,125633 +125633,125633 +223446,125633 +133221,125634 +125634,125634 +20095,125634 +133222,125635 +125635,125635 +133223,125636 +125636,125636 +133224,125637 +125637,125637 +122601,125637 +133225,125638 +125638,125638 +133226,125639 +125639,125639 +133227,125640 +125640,125640 +133228,125641 +125641,125641 +133229,125642 +125642,125642 +133230,125643 +125643,125643 +122607,125643 +133231,125644 +125644,125644 +133232,125645 +125645,125645 +67127,125645 +133233,125646 +125646,125646 +133234,125647 +125647,125647 +133235,125648 +125648,125648 +133236,125649 +125649,125649 +133237,125650 +125650,125650 +133238,125651 +125651,125651 +133239,125652 +125652,125652 +133240,125653 +125653,125653 +133241,125654 +125654,125654 +133242,125655 +125655,125655 +133243,125656 +125656,125656 +133244,125657 +125657,125657 +133245,125658 +125658,125658 +133246,125659 +125659,125659 +133247,125660 +125660,125660 +133542,125955 +125955,125955 +133543,125956 +125956,125956 +133544,125957 +125957,125957 +102651,125957 +133545,125958 +125958,125958 +141422,125958 +133546,125959 +125959,125959 +133547,125960 +125960,125960 +133548,125961 +125961,125961 +133549,125962 +125962,125962 +5611,125962 +133550,125963 +125963,125963 +133551,125964 +125964,125964 +133552,125965 +125965,125965 +233374,125965 +133553,125966 +125966,125966 +133554,125967 +125967,125967 +133555,125968 +125968,125968 +133556,125969 +125969,125969 +133557,125970 +125970,125970 +133558,125971 +125971,125971 +133559,125972 +125972,125972 +133560,125973 +125973,125973 +133561,125974 +125974,125974 +133562,125975 +125975,125975 +133563,125976 +125976,125976 +133564,125977 +125977,125977 +133565,125978 +125978,125978 +133566,125979 +125979,125979 +133567,125980 +125980,125980 +133568,125981 +125981,125981 +133569,125982 +125982,125982 +133570,125983 +125983,125983 +133571,125984 +125984,125984 +133572,125985 +125985,125985 +133573,125986 +125986,125986 +133574,125987 +125987,125987 +133575,125988 +125988,125988 +133576,125989 +125989,125989 +133577,125990 +125990,125990 +133578,125991 +125991,125991 +133579,125992 +125992,125992 +133672,126085 +126085,126085 +133673,126086 +126086,126086 +133674,126087 +126087,126087 +133675,126088 +126088,126088 +133676,126089 +126089,126089 +133677,126090 +126090,126090 +133678,126091 +126091,126091 +133679,126092 +126092,126092 +133680,126093 +126093,126093 +133681,126094 +126094,126094 +133682,126095 +126095,126095 +133683,126096 +126096,126096 +133684,126097 +126097,126097 +133685,126098 +126098,126098 +133686,126099 +126099,126099 +133687,126100 +126100,126100 +133688,126101 +126101,126101 +133689,126102 +126102,126102 +133690,126103 +126103,126103 +133691,126104 +126104,126104 +133692,126105 +126105,126105 +133693,126106 +126106,126106 +133694,126107 +126107,126107 +133695,126108 +126108,126108 +133696,126109 +126109,126109 +133697,126110 +126110,126110 +133698,126111 +126111,126111 +136108,126112 +133699,126112 +128521,126112 +126112,126112 +133700,126113 +126113,126113 +133701,126114 +126114,126114 +133702,126115 +126115,126115 +133703,126116 +126116,126116 +133704,126117 +126117,126117 +133705,126118 +126118,126118 +133706,126119 +126119,126119 +133707,126120 +126120,126120 +133708,126121 +126121,126121 +133709,126122 +126122,126122 +133710,126123 +126123,126123 +133711,126124 +126124,126124 +133712,126125 +126125,126125 +133713,126126 +126126,126126 +120968,126126 +133714,126127 +126127,126127 +73043,126127 +133715,126128 +126128,126128 +133716,126129 +126129,126129 +133717,126130 +126130,126130 +133718,126131 +126131,126131 +133719,126132 +126132,126132 +133720,126133 +126133,126133 +133721,126134 +126134,126134 +133722,126135 +126135,126135 +133723,126136 +126136,126136 +139696,126136 +133724,126137 +126137,126137 +133725,126138 +126138,126138 +133726,126139 +126139,126139 +133727,126140 +126140,126140 +133728,126141 +126141,126141 +133729,126142 +126142,126142 +133730,126143 +126143,126143 +133731,126144 +126144,126144 +133732,126145 +126145,126145 +139290,126145 +133733,126146 +126146,126146 +139707,126147 +133734,126147 +126147,126147 +17208,126147 +133735,126148 +126148,126148 +133736,126149 +126149,126149 +133737,126150 +126150,126150 +133738,126151 +126151,126151 +133739,126152 +126152,126152 +133740,126153 +126153,126153 +133741,126154 +126154,126154 +133742,126155 +126155,126155 +133743,126156 +126156,126156 +133744,126157 +126157,126157 +133745,126158 +126158,126158 +152221,126158 +133746,126159 +126159,126159 +133747,126160 +126160,126160 +133748,126161 +126161,126161 +133749,126162 +126162,126162 +133750,126163 +126163,126163 +77977,126163 +133751,126164 +126164,126164 +133752,126165 +126165,126165 +133753,126166 +126166,126166 +219814,126166 +133754,126167 +126167,126167 +133755,126168 +126168,126168 +133756,126169 +126169,126169 +133757,126170 +126170,126170 +133758,126171 +126171,126171 +229578,126171 +133759,126172 +126172,126172 +133760,126173 +126173,126173 +133761,126174 +126174,126174 +133762,126175 +126175,126175 +133763,126176 +126176,126176 +133764,126177 +126177,126177 +133982,126395 +126395,126395 +133983,126396 +126396,126396 +133984,126397 +126397,126397 +133985,126398 +126398,126398 +133986,126399 +126399,126399 +133987,126400 +126400,126400 +133988,126401 +126401,126401 +133989,126402 +126402,126402 +133990,126403 +126403,126403 +133991,126404 +126404,126404 +133992,126405 +126405,126405 +140696,126406 +133993,126406 +126406,126406 +28943,126406 +133994,126407 +126407,126407 +133995,126408 +126408,126408 +133996,126409 +126409,126409 +133997,126410 +126410,126410 +133998,126411 +126411,126411 +133999,126412 +126412,126412 +134000,126413 +126413,126413 +134001,126414 +126414,126414 +134002,126415 +126415,126415 +134003,126416 +126416,126416 +134004,126417 +126417,126417 +134005,126418 +126418,126418 +134006,126419 +126419,126419 +134007,126420 +126420,126420 +134008,126421 +126421,126421 +209512,126421 +134009,126422 +126422,126422 +134010,126423 +126423,126423 +134011,126424 +126424,126424 +134012,126425 +126425,126425 +134013,126426 +126426,126426 +134014,126427 +126427,126427 +134015,126428 +126428,126428 +134016,126429 +126429,126429 +134017,126430 +126430,126430 +134018,126431 +126431,126431 +134019,126432 +126432,126432 +134020,126433 +126433,126433 +134021,126434 +126434,126434 +134022,126435 +126435,126435 +134023,126436 +126436,126436 +134024,126437 +126437,126437 +134025,126438 +126438,126438 +134026,126439 +126439,126439 +134027,126440 +126440,126440 +134028,126441 +126441,126441 +134029,126442 +126442,126442 +134030,126443 +126443,126443 +134031,126444 +126444,126444 +134032,126445 +126445,126445 +134033,126446 +126446,126446 +134034,126447 +126447,126447 +134035,126448 +126448,126448 +134036,126449 +126449,126449 +134037,126450 +126450,126450 +134038,126451 +126451,126451 +134039,126452 +126452,126452 +134040,126453 +126453,126453 +134041,126454 +126454,126454 +134042,126455 +126455,126455 +134043,126456 +126456,126456 +134044,126457 +126457,126457 +134045,126458 +126458,126458 +146448,126458 +134046,126459 +126459,126459 +134047,126460 +126460,126460 +207848,126460 +207618,126460 +134048,126461 +126461,126461 +134049,126462 +126462,126462 +134050,126463 +126463,126463 +134051,126464 +126464,126464 +132618,126464 +125031,126464 +208425,126464 +134052,126465 +126465,126465 +134053,126466 +126466,126466 +134054,126467 +126467,126467 +134055,126468 +126468,126468 +134056,126469 +126469,126469 +134057,126470 +126470,126470 +134058,126471 +126471,126471 +134059,126472 +126472,126472 +134060,126473 +126473,126473 +134061,126474 +126474,126474 +24409,126474 +134062,126475 +126475,126475 +134063,126476 +126476,126476 +134064,126477 +126477,126477 +134065,126478 +126478,126478 +134066,126479 +126479,126479 +134067,126480 +126480,126480 +134068,126481 +126481,126481 +151591,126481 +134069,126482 +126482,126482 +134070,126483 +126483,126483 +134071,126484 +126484,126484 +134072,126485 +126485,126485 +134073,126486 +126486,126486 +231830,126486 +134074,126487 +126487,126487 +134075,126488 +126488,126488 +134076,126489 +126489,126489 +63706,126489 +134077,126490 +126490,126490 +134078,126491 +126491,126491 +134079,126492 +126492,126492 +134080,126493 +126493,126493 +134081,126494 +126494,126494 +134082,126495 +126495,126495 +134083,126496 +126496,126496 +134084,126497 +126497,126497 +134085,126498 +126498,126498 +134086,126499 +126499,126499 +80377,126499 +202703,126499 +134087,126500 +126500,126500 +134088,126501 +126501,126501 +134089,126502 +126502,126502 +76071,126502 +33216,126502 +134090,126503 +126503,126503 +134091,126504 +126504,126504 +134092,126505 +126505,126505 +134093,126506 +126506,126506 +134094,126507 +126507,126507 +134095,126508 +126508,126508 +120274,126508 +134096,126509 +126509,126509 +134097,126510 +126510,126510 +134098,126511 +126511,126511 +134099,126512 +126512,126512 +134100,126513 +126513,126513 +134101,126514 +126514,126514 +134102,126515 +126515,126515 +134103,126516 +126516,126516 +134104,126517 +126517,126517 +134105,126518 +126518,126518 +134106,126519 +126519,126519 +134107,126520 +126520,126520 +134108,126521 +126521,126521 +134109,126522 +126522,126522 +134110,126523 +126523,126523 +134111,126524 +126524,126524 +134112,126525 +126525,126525 +134113,126526 +126526,126526 +134114,126527 +126527,126527 +134115,126528 +126528,126528 +134116,126529 +126529,126529 +185827,126529 +134117,126530 +126530,126530 +134118,126531 +126531,126531 +134119,126532 +126532,126532 +134120,126533 +126533,126533 +134121,126534 +126534,126534 +134122,126535 +126535,126535 +134123,126536 +126536,126536 +134124,126537 +126537,126537 +134125,126538 +126538,126538 +134126,126539 +126539,126539 +134127,126540 +126540,126540 +104316,126540 +60136,126540 +134128,126541 +126541,126541 +140019,126541 +134129,126542 +126542,126542 +134130,126543 +126543,126543 +134131,126544 +126544,126544 +134132,126545 +126545,126545 +134133,126546 +126546,126546 +134134,126547 +126547,126547 +134135,126548 +126548,126548 +134136,126549 +126549,126549 +134137,126550 +126550,126550 +113493,126550 +134138,126551 +126551,126551 +134139,126552 +126552,126552 +134140,126553 +126553,126553 +134141,126554 +126554,126554 +134142,126555 +126555,126555 +134143,126556 +126556,126556 +134144,126557 +126557,126557 +134145,126558 +126558,126558 +134146,126559 +126559,126559 +134147,126560 +126560,126560 +134148,126561 +126561,126561 +134149,126562 +126562,126562 +134150,126563 +126563,126563 +134151,126564 +126564,126564 +134152,126565 +126565,126565 +134153,126566 +126566,126566 +134154,126567 +126567,126567 +134155,126568 +126568,126568 +134156,126569 +126569,126569 +134157,126570 +126570,126570 +134158,126571 +126571,126571 +134159,126572 +126572,126572 +202784,126572 +134160,126573 +126573,126573 +134161,126574 +126574,126574 +134162,126575 +126575,126575 +134163,126576 +126576,126576 +134164,126577 +126577,126577 +135491,126577 +127904,126577 +134165,126578 +126578,126578 +134166,126579 +126579,126579 +134167,126580 +126580,126580 +134168,126581 +126581,126581 +134169,126582 +126582,126582 +90710,126582 +161750,126583 +134170,126583 +126583,126583 +134171,126584 +126584,126584 +134172,126585 +126585,126585 +134173,126586 +126586,126586 +134174,126587 +126587,126587 +134175,126588 +126588,126588 +134176,126589 +126589,126589 +134177,126590 +126590,126590 +134178,126591 +126591,126591 +134179,126592 +126592,126592 +134180,126593 +126593,126593 +134181,126594 +126594,126594 +187084,126594 +134182,126595 +126595,126595 +134183,126596 +126596,126596 +134184,126597 +126597,126597 +134185,126598 +126598,126598 +134186,126599 +126599,126599 +5222,126599 +134187,126600 +126600,126600 +134188,126601 +126601,126601 +3999,126601 +134189,126602 +126602,126602 +134190,126603 +126603,126603 +134191,126604 +126604,126604 +108669,126604 +134192,126605 +126605,126605 +134193,126606 +126606,126606 +134194,126607 +126607,126607 +134195,126608 +126608,126608 +134196,126609 +126609,126609 +180963,126609 +114755,126609 +134197,126610 +126610,126610 +134198,126611 +126611,126611 +134199,126612 +126612,126612 +134200,126613 +126613,126613 +56262,126613 +48158,126613 +134201,126614 +126614,126614 +134202,126615 +126615,126615 +134203,126616 +126616,126616 +134204,126617 +126617,126617 +134205,126618 +126618,126618 +70205,126618 +134206,126619 +126619,126619 +57731,126619 +134207,126620 +126620,126620 +134208,126621 +126621,126621 +142692,126621 +134209,126622 +126622,126622 +134210,126623 +126623,126623 +134211,126624 +126624,126624 +134212,126625 +126625,126625 +134213,126626 +126626,126626 +134214,126627 +126627,126627 +134215,126628 +126628,126628 +134216,126629 +126629,126629 +134217,126630 +126630,126630 +159458,126630 +134218,126631 +126631,126631 +134219,126632 +126632,126632 +134220,126633 +126633,126633 +134221,126634 +126634,126634 +134222,126635 +126635,126635 +134223,126636 +126636,126636 +134224,126637 +126637,126637 +134225,126638 +126638,126638 +111201,126638 +134226,126639 +126639,126639 +134227,126640 +126640,126640 +247217,126640 +134228,126641 +126641,126641 +134229,126642 +126642,126642 +134230,126643 +126643,126643 +134231,126644 +126644,126644 +134232,126645 +126645,126645 +134233,126646 +126646,126646 +134234,126647 +126647,126647 +134235,126648 +126648,126648 +134236,126649 +126649,126649 +134237,126650 +126650,126650 +134238,126651 +126651,126651 +134239,126652 +126652,126652 +134240,126653 +126653,126653 +134241,126654 +126654,126654 +134242,126655 +126655,126655 +134243,126656 +126656,126656 +204185,126656 +134244,126657 +126657,126657 +134245,126658 +126658,126658 +190925,126658 +134246,126659 +126659,126659 +134247,126660 +126660,126660 +135078,126660 +127491,126660 +134248,126661 +126661,126661 +134249,126662 +126662,126662 +232917,126662 +134250,126663 +126663,126663 +134251,126664 +126664,126664 +134252,126665 +126665,126665 +211554,126665 +134253,126666 +126666,126666 +134254,126667 +126667,126667 +217254,126667 +233749,126668 +134255,126668 +126668,126668 +134256,126669 +126669,126669 +134257,126670 +126670,126670 +134258,126671 +126671,126671 +134259,126672 +126672,126672 +134260,126673 +126673,126673 +134261,126674 +126674,126674 +134262,126675 +126675,126675 +134263,126676 +126676,126676 +134264,126677 +126677,126677 +134265,126678 +126678,126678 +190981,126678 +134266,126679 +126679,126679 +134267,126680 +126680,126680 +134268,126681 +126681,126681 +134269,126682 +126682,126682 +134270,126683 +126683,126683 +134271,126684 +126684,126684 +134272,126685 +126685,126685 +134273,126686 +126686,126686 +134274,126687 +126687,126687 +134275,126688 +126688,126688 +134276,126689 +126689,126689 +134277,126690 +126690,126690 +23673,126690 +134278,126691 +126691,126691 +134279,126692 +126692,126692 +134280,126693 +126693,126693 +134281,126694 +126694,126694 +134282,126695 +126695,126695 +134283,126696 +126696,126696 +134284,126697 +126697,126697 +134285,126698 +126698,126698 +203806,126698 +134286,126699 +126699,126699 +134287,126700 +126700,126700 +134288,126701 +126701,126701 +134289,126702 +126702,126702 +134290,126703 +126703,126703 +134291,126704 +126704,126704 +134292,126705 +126705,126705 +134293,126706 +126706,126706 +134294,126707 +126707,126707 +134295,126708 +126708,126708 +134296,126709 +126709,126709 +134297,126710 +126710,126710 +134298,126711 +126711,126711 +134299,126712 +126712,126712 +134300,126713 +126713,126713 +134301,126714 +126714,126714 +134302,126715 +126715,126715 +134303,126716 +126716,126716 +134304,126717 +126717,126717 +111752,126717 +134305,126718 +126718,126718 +134306,126719 +126719,126719 +36191,126719 +134307,126720 +126720,126720 +134308,126721 +126721,126721 +134309,126722 +126722,126722 +134310,126723 +126723,126723 +134311,126724 +126724,126724 +134312,126725 +126725,126725 +134313,126726 +126726,126726 +134314,126727 +126727,126727 +134315,126728 +126728,126728 +134316,126729 +126729,126729 +134317,126730 +126730,126730 +134318,126731 +126731,126731 +134319,126732 +126732,126732 +134320,126733 +126733,126733 +134321,126734 +126734,126734 +134322,126735 +126735,126735 +134323,126736 +126736,126736 +134324,126737 +126737,126737 +134325,126738 +126738,126738 +134326,126739 +126739,126739 +134327,126740 +126740,126740 +134328,126741 +126741,126741 +65743,126741 +134329,126742 +126742,126742 +134330,126743 +126743,126743 +134331,126744 +126744,126744 +134332,126745 +126745,126745 +134333,126746 +126746,126746 +134334,126747 +126747,126747 +134335,126748 +126748,126748 +134336,126749 +126749,126749 +134337,126750 +126750,126750 +134338,126751 +126751,126751 +134339,126752 +126752,126752 +134340,126753 +126753,126753 +134341,126754 +126754,126754 +134342,126755 +126755,126755 +134343,126756 +126756,126756 +134344,126757 +126757,126757 +134345,126758 +126758,126758 +134346,126759 +126759,126759 +134347,126760 +126760,126760 +146587,126760 +134385,126798 +126798,126798 +135061,127474 +127474,127474 +135062,127475 +127475,127475 +135063,127476 +127476,127476 +135064,127477 +127477,127477 +135065,127478 +127478,127478 +135066,127479 +127479,127479 +135067,127480 +127480,127480 +135068,127481 +127481,127481 +135069,127482 +127482,127482 +135070,127483 +127483,127483 +2449,127483 +135071,127484 +127484,127484 +135072,127485 +127485,127485 +111974,127485 +135073,127486 +127486,127486 +135074,127487 +127487,127487 +135075,127488 +127488,127488 +135076,127489 +127489,127489 +135077,127490 +127490,127490 +135078,127491 +127491,127491 +134247,127491 +126660,127491 +135079,127492 +127492,127492 +135080,127493 +127493,127493 +135081,127494 +127494,127494 +135082,127495 +127495,127495 +135083,127496 +127496,127496 +135084,127497 +127497,127497 +135085,127498 +127498,127498 +135086,127499 +127499,127499 +135087,127500 +127500,127500 +135088,127501 +127501,127501 +135089,127502 +127502,127502 +135090,127503 +127503,127503 +135091,127504 +127504,127504 +135092,127505 +127505,127505 +135093,127506 +127506,127506 +29522,127506 +135094,127507 +127507,127507 +135095,127508 +127508,127508 +135096,127509 +127509,127509 +135097,127510 +127510,127510 +135098,127511 +127511,127511 +194361,127511 +135099,127512 +127512,127512 +135100,127513 +127513,127513 +135101,127514 +127514,127514 +135102,127515 +127515,127515 +135103,127516 +127516,127516 +135104,127517 +127517,127517 +135105,127518 +127518,127518 +135106,127519 +127519,127519 +135107,127520 +127520,127520 +135108,127521 +127521,127521 +135109,127522 +127522,127522 +135110,127523 +127523,127523 +135111,127524 +127524,127524 +135112,127525 +127525,127525 +135113,127526 +127526,127526 +10488,127526 +135114,127527 +127527,127527 +135115,127528 +127528,127528 +135116,127529 +127529,127529 +135117,127530 +127530,127530 +135118,127531 +127531,127531 +135119,127532 +127532,127532 +135120,127533 +127533,127533 +135121,127534 +127534,127534 +135122,127535 +127535,127535 +135123,127536 +127536,127536 +135124,127537 +127537,127537 +135125,127538 +127538,127538 +135126,127539 +127539,127539 +135127,127540 +127540,127540 +135128,127541 +127541,127541 +5555,127541 +135129,127542 +127542,127542 +135130,127543 +127543,127543 +135131,127544 +127544,127544 +135132,127545 +127545,127545 +135200,127613 +127613,127613 +135201,127614 +127614,127614 +135202,127615 +127615,127615 +135203,127616 +127616,127616 +135204,127617 +127617,127617 +135205,127618 +127618,127618 +135206,127619 +127619,127619 +135207,127620 +127620,127620 +135208,127621 +127621,127621 +135209,127622 +127622,127622 +135210,127623 +127623,127623 +135211,127624 +127624,127624 +135212,127625 +127625,127625 +135213,127626 +127626,127626 +135214,127627 +127627,127627 +135215,127628 +127628,127628 +135216,127629 +127629,127629 +135217,127630 +127630,127630 +135218,127631 +127631,127631 +135219,127632 +127632,127632 +135220,127633 +127633,127633 +135221,127634 +127634,127634 +135222,127635 +127635,127635 +135223,127636 +127636,127636 +135224,127637 +127637,127637 +135225,127638 +127638,127638 +135226,127639 +127639,127639 +50026,127639 +41833,127639 +135227,127640 +127640,127640 +135228,127641 +127641,127641 +135229,127642 +127642,127642 +135230,127643 +127643,127643 +135231,127644 +127644,127644 +135232,127645 +127645,127645 +35645,127645 +135233,127646 +127646,127646 +135234,127647 +127647,127647 +135235,127648 +127648,127648 +135236,127649 +127649,127649 +135237,127650 +127650,127650 +135238,127651 +127651,127651 +135239,127652 +127652,127652 +135240,127653 +127653,127653 +135241,127654 +127654,127654 +122244,127654 +135242,127655 +127655,127655 +135243,127656 +127656,127656 +135244,127657 +127657,127657 +135245,127658 +127658,127658 +135246,127659 +127659,127659 +136062,127660 +135247,127660 +128475,127660 +127660,127660 +135248,127661 +127661,127661 +135249,127662 +127662,127662 +135250,127663 +127663,127663 +135251,127664 +127664,127664 +236058,127664 +135252,127665 +127665,127665 +135253,127666 +127666,127666 +135254,127667 +127667,127667 +135255,127668 +127668,127668 +135256,127669 +127669,127669 +135257,127670 +127670,127670 +135258,127671 +127671,127671 +135259,127672 +127672,127672 +64901,127672 +135260,127673 +127673,127673 +135261,127674 +127674,127674 +135262,127675 +127675,127675 +60758,127675 +135263,127676 +127676,127676 +236712,127676 +3537,127676 +135264,127677 +127677,127677 +135265,127678 +127678,127678 +135266,127679 +127679,127679 +135267,127680 +127680,127680 +135268,127681 +127681,127681 +55932,127681 +47828,127681 +135269,127682 +127682,127682 +135270,127683 +127683,127683 +164081,127683 +135271,127684 +127684,127684 +135272,127685 +127685,127685 +2053,127685 +135273,127686 +127686,127686 +27590,127686 +135274,127687 +127687,127687 +135275,127688 +127688,127688 +135276,127689 +127689,127689 +135277,127690 +127690,127690 +135278,127691 +127691,127691 +135279,127692 +127692,127692 +207780,127692 +9258,127692 +142561,127692 +135280,127693 +127693,127693 +135281,127694 +127694,127694 +135282,127695 +127695,127695 +135283,127696 +127696,127696 +166726,127697 +135284,127697 +127697,127697 +150799,127697 +135285,127698 +127698,127698 +243297,127698 +36066,127698 +135286,127699 +127699,127699 +135287,127700 +127700,127700 +186557,127700 +109549,127700 +135288,127701 +127701,127701 +135289,127702 +127702,127702 +135290,127703 +127703,127703 +135291,127704 +127704,127704 +135292,127705 +127705,127705 +222198,127706 +135293,127706 +127706,127706 +135294,127707 +127707,127707 +135295,127708 +127708,127708 +135296,127709 +127709,127709 +135297,127710 +127710,127710 +135298,127711 +127711,127711 +135416,127829 +127829,127829 +135417,127830 +127830,127830 +135418,127831 +127831,127831 +135419,127832 +127832,127832 +135420,127833 +127833,127833 +135421,127834 +127834,127834 +135422,127835 +127835,127835 +135423,127836 +127836,127836 +135424,127837 +127837,127837 +89224,127837 +135425,127838 +127838,127838 +93859,127838 +135426,127839 +127839,127839 +135427,127840 +127840,127840 +135428,127841 +127841,127841 +135429,127842 +127842,127842 +135430,127843 +127843,127843 +135431,127844 +127844,127844 +135432,127845 +127845,127845 +135433,127846 +127846,127846 +176537,127846 +170573,127846 +135434,127847 +127847,127847 +135435,127848 +127848,127848 +135436,127849 +127849,127849 +135437,127850 +127850,127850 +135438,127851 +127851,127851 +135439,127852 +127852,127852 +135440,127853 +127853,127853 +135441,127854 +127854,127854 +135442,127855 +127855,127855 +135443,127856 +127856,127856 +135444,127857 +127857,127857 +135445,127858 +127858,127858 +135446,127859 +127859,127859 +135447,127860 +127860,127860 +135448,127861 +127861,127861 +135449,127862 +127862,127862 +135450,127863 +127863,127863 +135451,127864 +127864,127864 +135452,127865 +127865,127865 +135453,127866 +127866,127866 +135454,127867 +127867,127867 +135455,127868 +127868,127868 +135456,127869 +127869,127869 +135457,127870 +127870,127870 +135458,127871 +127871,127871 +135459,127872 +127872,127872 +135460,127873 +127873,127873 +203684,127873 +135461,127874 +127874,127874 +135462,127875 +127875,127875 +135463,127876 +127876,127876 +135464,127877 +127877,127877 +135465,127878 +127878,127878 +135466,127879 +127879,127879 +135467,127880 +127880,127880 +135468,127881 +127881,127881 +135469,127882 +127882,127882 +135470,127883 +127883,127883 +135471,127884 +127884,127884 +135472,127885 +127885,127885 +135473,127886 +127886,127886 +135474,127887 +127887,127887 +135475,127888 +127888,127888 +135476,127889 +127889,127889 +77583,127889 +135477,127890 +127890,127890 +135478,127891 +127891,127891 +135479,127892 +127892,127892 +135480,127893 +127893,127893 +135481,127894 +127894,127894 +135482,127895 +127895,127895 +135483,127896 +127896,127896 +135484,127897 +127897,127897 +36350,127897 +135485,127898 +127898,127898 +135486,127899 +127899,127899 +135487,127900 +127900,127900 +135488,127901 +127901,127901 +135489,127902 +127902,127902 +135490,127903 +127903,127903 +135491,127904 +127904,127904 +134164,127904 +126577,127904 +135492,127905 +127905,127905 +135493,127906 +127906,127906 +135494,127907 +127907,127907 +135495,127908 +127908,127908 +135496,127909 +127909,127909 +135497,127910 +127910,127910 +135498,127911 +127911,127911 +135499,127912 +127912,127912 +135500,127913 +127913,127913 +135501,127914 +127914,127914 +135502,127915 +127915,127915 +135503,127916 +127916,127916 +135504,127917 +127917,127917 +33258,127917 +135505,127918 +127918,127918 +135506,127919 +127919,127919 +135507,127920 +127920,127920 +135508,127921 +127921,127921 +135509,127922 +127922,127922 +135510,127923 +127923,127923 +244127,127923 +135511,127924 +127924,127924 +135512,127925 +127925,127925 +135513,127926 +127926,127926 +135514,127927 +127927,127927 +15726,127927 +135515,127928 +127928,127928 +135516,127929 +127929,127929 +135517,127930 +127930,127930 +135518,127931 +127931,127931 +150114,127932 +135519,127932 +127932,127932 +135520,127933 +127933,127933 +135521,127934 +127934,127934 +135584,127997 +127997,127997 +135585,127998 +127998,127998 +135586,127999 +127999,127999 +135587,128000 +128000,128000 +135588,128001 +128001,128001 +135589,128002 +128002,128002 +135590,128003 +128003,128003 +135591,128004 +128004,128004 +135592,128005 +128005,128005 +135593,128006 +128006,128006 +135594,128007 +128007,128007 +135595,128008 +128008,128008 +135596,128009 +128009,128009 +135597,128010 +128010,128010 +135598,128011 +128011,128011 +135676,128089 +128089,128089 +135677,128090 +128090,128090 +56181,128090 +48077,128090 +135678,128091 +128091,128091 +135679,128092 +128092,128092 +135680,128093 +128093,128093 +135681,128094 +128094,128094 +135682,128095 +128095,128095 +135683,128096 +128096,128096 +135684,128097 +128097,128097 +135685,128098 +128098,128098 +135686,128099 +128099,128099 +135687,128100 +128100,128100 +135688,128101 +128101,128101 +135689,128102 +128102,128102 +135690,128103 +128103,128103 +135743,128156 +128156,128156 +135744,128157 +128157,128157 +135745,128158 +128158,128158 +135746,128159 +128159,128159 +135747,128160 +128160,128160 +135748,128161 +128161,128161 +135749,128162 +128162,128162 +148825,128162 +135750,128163 +128163,128163 +135751,128164 +128164,128164 +135752,128165 +128165,128165 +236410,128165 +135753,128166 +128166,128166 +135754,128167 +128167,128167 +135755,128168 +128168,128168 +135756,128169 +128169,128169 +109828,128169 +155936,128169 +135757,128170 +128170,128170 +66813,128170 +135758,128171 +128171,128171 +135759,128172 +128172,128172 +135760,128173 +128173,128173 +135761,128174 +128174,128174 +135762,128175 +128175,128175 +135763,128176 +128176,128176 +135764,128177 +128177,128177 +135765,128178 +128178,128178 +36751,128178 +135766,128179 +128179,128179 +135767,128180 +128180,128180 +227849,128180 +135768,128181 +128181,128181 +135769,128182 +128182,128182 +135770,128183 +128183,128183 +135771,128184 +128184,128184 +135772,128185 +128185,128185 +135773,128186 +128186,128186 +135774,128187 +128187,128187 +135775,128188 +128188,128188 +135776,128189 +128189,128189 +135777,128190 +128190,128190 +135778,128191 +128191,128191 +135779,128192 +128192,128192 +135780,128193 +128193,128193 +135781,128194 +128194,128194 +135782,128195 +128195,128195 +135783,128196 +128196,128196 +233082,128196 +135784,128197 +128197,128197 +135785,128198 +128198,128198 +135786,128199 +128199,128199 +135787,128200 +128200,128200 +135788,128201 +128201,128201 +135789,128202 +128202,128202 +135790,128203 +128203,128203 +135791,128204 +128204,128204 +135792,128205 +128205,128205 +135793,128206 +128206,128206 +135794,128207 +128207,128207 +135795,128208 +128208,128208 +135796,128209 +128209,128209 +135797,128210 +128210,128210 +135798,128211 +128211,128211 +135799,128212 +128212,128212 +135800,128213 +128213,128213 +135801,128214 +128214,128214 +135802,128215 +128215,128215 +135803,128216 +128216,128216 +135804,128217 +128217,128217 +135805,128218 +128218,128218 +135806,128219 +128219,128219 +135807,128220 +128220,128220 +135808,128221 +128221,128221 +135809,128222 +128222,128222 +135810,128223 +128223,128223 +135811,128224 +128224,128224 +135812,128225 +128225,128225 +135813,128226 +128226,128226 +135814,128227 +128227,128227 +135815,128228 +128228,128228 +203620,128228 +135816,128229 +128229,128229 +135817,128230 +128230,128230 +135818,128231 +128231,128231 +135819,128232 +128232,128232 +135820,128233 +128233,128233 +137958,128233 +130371,128233 +135821,128234 +128234,128234 +135822,128235 +128235,128235 +135823,128236 +128236,128236 +135824,128237 +128237,128237 +135825,128238 +128238,128238 +135826,128239 +128239,128239 +135827,128240 +128240,128240 +135828,128241 +128241,128241 +135829,128242 +128242,128242 +135830,128243 +128243,128243 +135831,128244 +128244,128244 +135832,128245 +128245,128245 +135833,128246 +128246,128246 +135834,128247 +128247,128247 +84835,128247 +135835,128248 +128248,128248 +135836,128249 +128249,128249 +135837,128250 +128250,128250 +135838,128251 +128251,128251 +135839,128252 +128252,128252 +135840,128253 +128253,128253 +135841,128254 +128254,128254 +135842,128255 +128255,128255 +135843,128256 +128256,128256 +135844,128257 +128257,128257 +135845,128258 +128258,128258 +135846,128259 +128259,128259 +135847,128260 +128260,128260 +135848,128261 +128261,128261 +135849,128262 +128262,128262 +135850,128263 +128263,128263 +135851,128264 +128264,128264 +135852,128265 +128265,128265 +135853,128266 +128266,128266 +135854,128267 +128267,128267 +135855,128268 +128268,128268 +135856,128269 +128269,128269 +135857,128270 +128270,128270 +135913,128326 +128326,128326 +135914,128327 +128327,128327 +135915,128328 +128328,128328 +15432,128328 +135916,128329 +128329,128329 +135917,128330 +128330,128330 +135918,128331 +128331,128331 +135919,128332 +128332,128332 +135920,128333 +128333,128333 +135921,128334 +128334,128334 +135922,128335 +128335,128335 +135923,128336 +128336,128336 +135924,128337 +128337,128337 +135925,128338 +128338,128338 +135926,128339 +128339,128339 +176675,128339 +170711,128339 +135927,128340 +128340,128340 +135928,128341 +128341,128341 +3041,128341 +135929,128342 +128342,128342 +135930,128343 +128343,128343 +135931,128344 +128344,128344 +240519,128344 +135932,128345 +128345,128345 +135933,128346 +128346,128346 +135934,128347 +128347,128347 +135935,128348 +128348,128348 +135936,128349 +128349,128349 +135937,128350 +128350,128350 +135938,128351 +128351,128351 +135939,128352 +128352,128352 +135940,128353 +128353,128353 +135941,128354 +128354,128354 +135942,128355 +128355,128355 +135943,128356 +128356,128356 +135944,128357 +128357,128357 +135945,128358 +128358,128358 +135946,128359 +128359,128359 +135947,128360 +128360,128360 +135948,128361 +128361,128361 +135949,128362 +128362,128362 +220375,128362 +5773,128362 +135950,128363 +128363,128363 +135951,128364 +128364,128364 +135952,128365 +128365,128365 +135953,128366 +128366,128366 +80804,128366 +135954,128367 +128367,128367 +28951,128367 +135955,128368 +128368,128368 +124087,128368 +135956,128369 +128369,128369 +135957,128370 +128370,128370 +136025,128438 +128438,128438 +136026,128439 +128439,128439 +136027,128440 +128440,128440 +136028,128441 +128441,128441 +136029,128442 +128442,128442 +136030,128443 +128443,128443 +136031,128444 +128444,128444 +136032,128445 +128445,128445 +136033,128446 +128446,128446 +136034,128447 +128447,128447 +136035,128448 +128448,128448 +136036,128449 +128449,128449 +136037,128450 +128450,128450 +217041,128451 +136038,128451 +128451,128451 +68193,128451 +136039,128452 +128452,128452 +136040,128453 +128453,128453 +136041,128454 +128454,128454 +136042,128455 +128455,128455 +136043,128456 +128456,128456 +136044,128457 +128457,128457 +74157,128457 +136045,128458 +128458,128458 +136046,128459 +128459,128459 +136047,128460 +128460,128460 +136048,128461 +128461,128461 +684,128461 +136049,128462 +128462,128462 +136050,128463 +128463,128463 +136051,128464 +128464,128464 +136052,128465 +128465,128465 +136053,128466 +128466,128466 +136054,128467 +128467,128467 +136055,128468 +128468,128468 +136056,128469 +128469,128469 +136057,128470 +128470,128470 +136058,128471 +128471,128471 +136059,128472 +128472,128472 +136060,128473 +128473,128473 +136061,128474 +128474,128474 +136062,128475 +135247,128475 +128475,128475 +127660,128475 +136063,128476 +128476,128476 +136064,128477 +128477,128477 +136065,128478 +128478,128478 +136066,128479 +128479,128479 +136067,128480 +128480,128480 +136068,128481 +128481,128481 +64119,128481 +136069,128482 +128482,128482 +136070,128483 +128483,128483 +136071,128484 +128484,128484 +136072,128485 +128485,128485 +156494,128485 +136073,128486 +128486,128486 +136074,128487 +128487,128487 +136075,128488 +128488,128488 +136076,128489 +128489,128489 +136077,128490 +128490,128490 +136078,128491 +128491,128491 +136079,128492 +128492,128492 +136080,128493 +128493,128493 +150798,128493 +136081,128494 +128494,128494 +136082,128495 +128495,128495 +136083,128496 +128496,128496 +136084,128497 +128497,128497 +136085,128498 +128498,128498 +136086,128499 +128499,128499 +136087,128500 +128500,128500 +136088,128501 +128501,128501 +136089,128502 +128502,128502 +136090,128503 +128503,128503 +136091,128504 +128504,128504 +136092,128505 +128505,128505 +136093,128506 +128506,128506 +136094,128507 +128507,128507 +136095,128508 +128508,128508 +136096,128509 +128509,128509 +136097,128510 +128510,128510 +136098,128511 +128511,128511 +136099,128512 +128512,128512 +136100,128513 +128513,128513 +136101,128514 +128514,128514 +136102,128515 +128515,128515 +136103,128516 +128516,128516 +136104,128517 +128517,128517 +136105,128518 +128518,128518 +136106,128519 +128519,128519 +136107,128520 +128520,128520 +136108,128521 +133699,128521 +128521,128521 +126112,128521 +136109,128522 +128522,128522 +136110,128523 +128523,128523 +136111,128524 +128524,128524 +136112,128525 +128525,128525 +136113,128526 +128526,128526 +136114,128527 +128527,128527 +136115,128528 +128528,128528 +136116,128529 +128529,128529 +199190,128529 +136117,128530 +128530,128530 +136118,128531 +128531,128531 +136119,128532 +128532,128532 +136120,128533 +128533,128533 +136121,128534 +128534,128534 +136122,128535 +128535,128535 +136123,128536 +128536,128536 +191558,128536 +136124,128537 +128537,128537 +136125,128538 +128538,128538 +136126,128539 +128539,128539 +136127,128540 +128540,128540 +136128,128541 +128541,128541 +136129,128542 +128542,128542 +136130,128543 +128543,128543 +136131,128544 +128544,128544 +136132,128545 +128545,128545 +136133,128546 +128546,128546 +136134,128547 +128547,128547 +136135,128548 +128548,128548 +136136,128549 +128549,128549 +136137,128550 +128550,128550 +136138,128551 +128551,128551 +136139,128552 +128552,128552 +32333,128552 +136140,128553 +128553,128553 +136141,128554 +128554,128554 +107531,128554 +136142,128555 +128555,128555 +136143,128556 +128556,128556 +136144,128557 +128557,128557 +136145,128558 +128558,128558 +136146,128559 +128559,128559 +136147,128560 +128560,128560 +136148,128561 +128561,128561 +136149,128562 +128562,128562 +136150,128563 +128563,128563 +136151,128564 +128564,128564 +136152,128565 +128565,128565 +136153,128566 +128566,128566 +136154,128567 +128567,128567 +136155,128568 +128568,128568 +136156,128569 +128569,128569 +136157,128570 +128570,128570 +136158,128571 +128571,128571 +136159,128572 +128572,128572 +136160,128573 +128573,128573 +136161,128574 +128574,128574 +136162,128575 +128575,128575 +136163,128576 +128576,128576 +136164,128577 +128577,128577 +136165,128578 +128578,128578 +136166,128579 +128579,128579 +136167,128580 +128580,128580 +136168,128581 +128581,128581 +68021,128581 +136169,128582 +128582,128582 +136170,128583 +128583,128583 +136171,128584 +128584,128584 +136172,128585 +128585,128585 +136173,128586 +128586,128586 +136174,128587 +128587,128587 +136175,128588 +128588,128588 +136176,128589 +128589,128589 +136177,128590 +128590,128590 +136178,128591 +128591,128591 +136179,128592 +128592,128592 +136180,128593 +128593,128593 +136181,128594 +128594,128594 +136182,128595 +128595,128595 +136183,128596 +128596,128596 +136184,128597 +128597,128597 +136185,128598 +128598,128598 +136186,128599 +128599,128599 +136187,128600 +128600,128600 +136188,128601 +128601,128601 +136189,128602 +128602,128602 +136190,128603 +128603,128603 +136191,128604 +128604,128604 +136192,128605 +128605,128605 +136193,128606 +128606,128606 +136194,128607 +128607,128607 +136195,128608 +128608,128608 +136196,128609 +128609,128609 +136197,128610 +128610,128610 +136198,128611 +128611,128611 +136199,128612 +128612,128612 +136200,128613 +128613,128613 +136201,128614 +128614,128614 +136202,128615 +128615,128615 +225535,128615 +136203,128616 +128616,128616 +136204,128617 +128617,128617 +136205,128618 +128618,128618 +136206,128619 +128619,128619 +136207,128620 +128620,128620 +136208,128621 +128621,128621 +136209,128622 +128622,128622 +136210,128623 +128623,128623 +136211,128624 +128624,128624 +136981,128624 +129394,128624 +136212,128625 +128625,128625 +136213,128626 +128626,128626 +136214,128627 +128627,128627 +136215,128628 +128628,128628 +136216,128629 +128629,128629 +136217,128630 +128630,128630 +136218,128631 +128631,128631 +136219,128632 +128632,128632 +136220,128633 +128633,128633 +136221,128634 +128634,128634 +136222,128635 +128635,128635 +136223,128636 +128636,128636 +136224,128637 +128637,128637 +136225,128638 +128638,128638 +136226,128639 +128639,128639 +225987,128639 +4034,128639 +136227,128640 +128640,128640 +136228,128641 +128641,128641 +136229,128642 +128642,128642 +136230,128643 +128643,128643 +136231,128644 +128644,128644 +136232,128645 +128645,128645 +136233,128646 +128646,128646 +136234,128647 +128647,128647 +136235,128648 +128648,128648 +136236,128649 +128649,128649 +136237,128650 +128650,128650 +136238,128651 +128651,128651 +136239,128652 +128652,128652 +136240,128653 +128653,128653 +136241,128654 +128654,128654 +136242,128655 +128655,128655 +136243,128656 +128656,128656 +136244,128657 +128657,128657 +136245,128658 +128658,128658 +136246,128659 +128659,128659 +136247,128660 +128660,128660 +136248,128661 +128661,128661 +105805,128661 +136249,128662 +128662,128662 +136250,128663 +128663,128663 +82175,128663 +136251,128664 +128664,128664 +71387,128664 +207189,128664 +136252,128665 +128665,128665 +136253,128666 +128666,128666 +136254,128667 +128667,128667 +136255,128668 +128668,128668 +136256,128669 +128669,128669 +118320,128669 +136257,128670 +128670,128670 +136258,128671 +128671,128671 +30906,128671 +36750,128671 +136259,128672 +128672,128672 +136260,128673 +128673,128673 +136261,128674 +128674,128674 +136303,128716 +128716,128716 +136304,128717 +128717,128717 +136305,128718 +128718,128718 +136306,128719 +128719,128719 +136307,128720 +128720,128720 +136308,128721 +128721,128721 +178211,128721 +172247,128721 +136309,128722 +128722,128722 +136310,128723 +128723,128723 +136311,128724 +128724,128724 +111387,128724 +136312,128725 +128725,128725 +136313,128726 +128726,128726 +192236,128726 +136314,128727 +128727,128727 +136315,128728 +128728,128728 +104764,128728 +136316,128729 +128729,128729 +136317,128730 +128730,128730 +136318,128731 +128731,128731 +136319,128732 +128732,128732 +136320,128733 +128733,128733 +136321,128734 +128734,128734 +136322,128735 +128735,128735 +133079,128735 +131981,128735 +125492,128735 +124394,128735 +136323,128736 +128736,128736 +136324,128737 +128737,128737 +136325,128738 +128738,128738 +136326,128739 +128739,128739 +136327,128740 +128740,128740 +136328,128741 +128741,128741 +140639,128741 +136329,128742 +128742,128742 +136330,128743 +128743,128743 +136331,128744 +128744,128744 +136332,128745 +128745,128745 +136333,128746 +128746,128746 +136334,128747 +128747,128747 +136335,128748 +128748,128748 +136336,128749 +128749,128749 +136337,128750 +128750,128750 +136338,128751 +128751,128751 +136339,128752 +128752,128752 +136340,128753 +128753,128753 +136341,128754 +128754,128754 +136342,128755 +128755,128755 +136343,128756 +128756,128756 +136344,128757 +128757,128757 +136345,128758 +128758,128758 +136346,128759 +128759,128759 +136347,128760 +128760,128760 +136348,128761 +128761,128761 +136349,128762 +128762,128762 +136350,128763 +128763,128763 +136351,128764 +128764,128764 +136352,128765 +128765,128765 +136353,128766 +128766,128766 +136354,128767 +128767,128767 +136355,128768 +128768,128768 +136356,128769 +128769,128769 +136357,128770 +128770,128770 +136358,128771 +128771,128771 +136359,128772 +128772,128772 +136360,128773 +128773,128773 +136361,128774 +128774,128774 +136362,128775 +128775,128775 +136363,128776 +128776,128776 +136364,128777 +128777,128777 +136365,128778 +128778,128778 +136366,128779 +128779,128779 +136367,128780 +128780,128780 +136368,128781 +128781,128781 +136369,128782 +128782,128782 +136370,128783 +128783,128783 +136371,128784 +128784,128784 +136372,128785 +128785,128785 +136373,128786 +128786,128786 +136374,128787 +128787,128787 +136375,128788 +128788,128788 +136376,128789 +128789,128789 +136377,128790 +128790,128790 +136378,128791 +128791,128791 +80651,128791 +136379,128792 +128792,128792 +198414,128792 +136380,128793 +128793,128793 +217694,128793 +136381,128794 +128794,128794 +216091,128794 +136382,128795 +128795,128795 +136383,128796 +128796,128796 +136384,128797 +128797,128797 +136385,128798 +128798,128798 +136386,128799 +128799,128799 +136387,128800 +128800,128800 +136388,128801 +128801,128801 +136389,128802 +128802,128802 +136390,128803 +128803,128803 +136391,128804 +128804,128804 +136392,128805 +128805,128805 +136393,128806 +128806,128806 +136394,128807 +128807,128807 +136395,128808 +128808,128808 +136396,128809 +128809,128809 +136397,128810 +128810,128810 +136398,128811 +128811,128811 +136399,128812 +128812,128812 +136400,128813 +128813,128813 +136401,128814 +128814,128814 +243331,128814 +136402,128815 +128815,128815 +136403,128816 +128816,128816 +136404,128817 +128817,128817 +76564,128817 +136405,128818 +128818,128818 +136406,128819 +128819,128819 +136407,128820 +128820,128820 +136408,128821 +128821,128821 +136409,128822 +128822,128822 +136410,128823 +128823,128823 +136411,128824 +128824,128824 +136412,128825 +128825,128825 +136413,128826 +128826,128826 +136414,128827 +128827,128827 +136415,128828 +128828,128828 +136416,128829 +128829,128829 +136454,128867 +128867,128867 +136455,128868 +128868,128868 +136456,128869 +128869,128869 +136457,128870 +128870,128870 +136458,128871 +128871,128871 +136459,128872 +128872,128872 +136460,128873 +128873,128873 +136461,128874 +128874,128874 +136462,128875 +128875,128875 +136463,128876 +128876,128876 +136464,128877 +128877,128877 +136465,128878 +128878,128878 +136466,128879 +128879,128879 +136467,128880 +128880,128880 +136468,128881 +128881,128881 +136469,128882 +128882,128882 +136470,128883 +128883,128883 +136471,128884 +128884,128884 +136472,128885 +128885,128885 +136473,128886 +128886,128886 +203003,128886 +136474,128887 +128887,128887 +246961,128887 +136475,128888 +128888,128888 +136476,128889 +128889,128889 +136477,128890 +128890,128890 +136478,128891 +128891,128891 +136479,128892 +128892,128892 +136480,128893 +128893,128893 +136481,128894 +128894,128894 +136482,128895 +128895,128895 +136483,128896 +128896,128896 +136484,128897 +128897,128897 +136485,128898 +128898,128898 +136486,128899 +128899,128899 +136487,128900 +128900,128900 +136488,128901 +128901,128901 +136489,128902 +128902,128902 +136490,128903 +128903,128903 +136491,128904 +128904,128904 +136492,128905 +128905,128905 +136493,128906 +128906,128906 +136494,128907 +128907,128907 +136495,128908 +128908,128908 +136496,128909 +128909,128909 +136497,128910 +128910,128910 +136498,128911 +128911,128911 +136499,128912 +128912,128912 +136500,128913 +128913,128913 +136501,128914 +128914,128914 +136502,128915 +128915,128915 +136503,128916 +128916,128916 +136504,128917 +128917,128917 +136505,128918 +128918,128918 +136506,128919 +128919,128919 +136507,128920 +128920,128920 +136508,128921 +128921,128921 +136509,128922 +128922,128922 +136510,128923 +128923,128923 +136511,128924 +128924,128924 +136512,128925 +128925,128925 +136513,128926 +128926,128926 +136514,128927 +128927,128927 +136515,128928 +128928,128928 +136516,128929 +128929,128929 +136517,128930 +128930,128930 +136518,128931 +128931,128931 +136519,128932 +128932,128932 +136520,128933 +128933,128933 +136521,128934 +128934,128934 +136522,128935 +128935,128935 +136523,128936 +128936,128936 +136524,128937 +128937,128937 +136525,128938 +128938,128938 +136526,128939 +128939,128939 +136527,128940 +128940,128940 +136528,128941 +128941,128941 +150671,128941 +136529,128942 +128942,128942 +136530,128943 +128943,128943 +136531,128944 +128944,128944 +136532,128945 +128945,128945 +86750,128945 +136533,128946 +128946,128946 +65122,128946 +136534,128947 +128947,128947 +136535,128948 +128948,128948 +136536,128949 +128949,128949 +136537,128950 +128950,128950 +136538,128951 +128951,128951 +136539,128952 +128952,128952 +136540,128953 +128953,128953 +136541,128954 +128954,128954 +136542,128955 +128955,128955 +3879,128955 +136543,128956 +128956,128956 +136544,128957 +128957,128957 +136545,128958 +128958,128958 +245426,128958 +136546,128959 +128959,128959 +136547,128960 +128960,128960 +136548,128961 +128961,128961 +136549,128962 +128962,128962 +136550,128963 +128963,128963 +136551,128964 +128964,128964 +136552,128965 +128965,128965 +136553,128966 +128966,128966 +136554,128967 +128967,128967 +136555,128968 +128968,128968 +136556,128969 +128969,128969 +136557,128970 +128970,128970 +136558,128971 +128971,128971 +136559,128972 +128972,128972 +136560,128973 +128973,128973 +38751,128973 +136561,128974 +128974,128974 +136562,128975 +128975,128975 +136563,128976 +128976,128976 +136564,128977 +128977,128977 +136565,128978 +128978,128978 +27575,128978 +136566,128979 +128979,128979 +136567,128980 +128980,128980 +136568,128981 +128981,128981 +136569,128982 +128982,128982 +136570,128983 +128983,128983 +136571,128984 +128984,128984 +136572,128985 +128985,128985 +136573,128986 +128986,128986 +232735,128986 +136574,128987 +128987,128987 +136575,128988 +128988,128988 +136576,128989 +128989,128989 +136577,128990 +128990,128990 +136578,128991 +128991,128991 +136579,128992 +128992,128992 +136580,128993 +128993,128993 +136581,128994 +128994,128994 +136582,128995 +128995,128995 +136583,128996 +128996,128996 +151596,128996 +136584,128997 +128997,128997 +97903,128997 +136585,128998 +128998,128998 +136586,128999 +128999,128999 +136587,129000 +129000,129000 +136588,129001 +129001,129001 +136589,129002 +129002,129002 +136590,129003 +129003,129003 +136591,129004 +129004,129004 +59988,129004 +136592,129005 +129005,129005 +136593,129006 +129006,129006 +136594,129007 +129007,129007 +136595,129008 +129008,129008 +136596,129009 +129009,129009 +136766,129179 +129179,129179 +136767,129180 +129180,129180 +136768,129181 +129181,129181 +136769,129182 +129182,129182 +136770,129183 +129183,129183 +136771,129184 +129184,129184 +136772,129185 +129185,129185 +136773,129186 +129186,129186 +136774,129187 +129187,129187 +136775,129188 +129188,129188 +136776,129189 +129189,129189 +136777,129190 +129190,129190 +223487,129190 +136778,129191 +129191,129191 +136779,129192 +129192,129192 +136780,129193 +129193,129193 +136781,129194 +129194,129194 +136782,129195 +129195,129195 +136783,129196 +129196,129196 +136784,129197 +129197,129197 +136785,129198 +129198,129198 +136786,129199 +129199,129199 +136787,129200 +129200,129200 +136788,129201 +129201,129201 +136789,129202 +129202,129202 +136790,129203 +129203,129203 +136791,129204 +129204,129204 +136942,129355 +129355,129355 +84000,129355 +136943,129356 +129356,129356 +136944,129357 +129357,129357 +136945,129358 +129358,129358 +241088,129358 +136946,129359 +129359,129359 +136947,129360 +129360,129360 +136948,129361 +129361,129361 +136949,129362 +129362,129362 +136950,129363 +129363,129363 +136951,129364 +129364,129364 +136952,129365 +129365,129365 +16471,129365 +136953,129366 +129366,129366 +136954,129367 +129367,129367 +136955,129368 +129368,129368 +136956,129369 +129369,129369 +136957,129370 +129370,129370 +136958,129371 +129371,129371 +136959,129372 +129372,129372 +136960,129373 +129373,129373 +136961,129374 +129374,129374 +136962,129375 +129375,129375 +136963,129376 +129376,129376 +136964,129377 +129377,129377 +136965,129378 +129378,129378 +136966,129379 +129379,129379 +136967,129380 +129380,129380 +136968,129381 +129381,129381 +136969,129382 +129382,129382 +136970,129383 +129383,129383 +136971,129384 +129384,129384 +136972,129385 +129385,129385 +136973,129386 +129386,129386 +136974,129387 +129387,129387 +136975,129388 +129388,129388 +136976,129389 +129389,129389 +136977,129390 +129390,129390 +136978,129391 +129391,129391 +136979,129392 +129392,129392 +136980,129393 +129393,129393 +109228,129393 +136981,129394 +129394,129394 +136211,129394 +128624,129394 +136982,129395 +129395,129395 +136983,129396 +129396,129396 +136984,129397 +129397,129397 +136985,129398 +129398,129398 +136986,129399 +129399,129399 +136987,129400 +129400,129400 +136988,129401 +129401,129401 +136989,129402 +129402,129402 +136990,129403 +129403,129403 +136991,129404 +129404,129404 +136992,129405 +129405,129405 +136993,129406 +129406,129406 +151121,129406 +82333,129406 +136994,129407 +129407,129407 +136995,129408 +129408,129408 +214459,129408 +136996,129409 +129409,129409 +136997,129410 +129410,129410 +8863,129410 +136998,129411 +129411,129411 +136999,129412 +129412,129412 +131912,129412 +124325,129412 +137000,129413 +129413,129413 +137001,129414 +129414,129414 +137002,129415 +129415,129415 +137003,129416 +129416,129416 +137004,129417 +129417,129417 +137005,129418 +129418,129418 +137006,129419 +129419,129419 +137007,129420 +129420,129420 +137008,129421 +129421,129421 +137009,129422 +129422,129422 +137010,129423 +129423,129423 +137011,129424 +129424,129424 +137012,129425 +129425,129425 +38463,129425 +215417,129425 +137013,129426 +129426,129426 +137014,129427 +129427,129427 +222880,129427 +137015,129428 +129428,129428 +137016,129429 +129429,129429 +137017,129430 +129430,129430 +137018,129431 +129431,129431 +137019,129432 +129432,129432 +137020,129433 +129433,129433 +137021,129434 +129434,129434 +137022,129435 +129435,129435 +137023,129436 +129436,129436 +137024,129437 +129437,129437 +137025,129438 +129438,129438 +137026,129439 +129439,129439 +185722,129439 +137027,129440 +129440,129440 +137028,129441 +129441,129441 +137029,129442 +129442,129442 +137030,129443 +129443,129443 +137031,129444 +129444,129444 +137032,129445 +129445,129445 +137033,129446 +129446,129446 +137034,129447 +129447,129447 +137035,129448 +129448,129448 +137036,129449 +129449,129449 +137037,129450 +129450,129450 +137038,129451 +129451,129451 +137039,129452 +129452,129452 +137040,129453 +129453,129453 +137041,129454 +129454,129454 +137042,129455 +129455,129455 +137043,129456 +129456,129456 +137044,129457 +129457,129457 +137045,129458 +129458,129458 +137046,129459 +129459,129459 +190969,129459 +187164,129459 +137047,129460 +129460,129460 +137048,129461 +129461,129461 +137049,129462 +129462,129462 +137050,129463 +129463,129463 +137051,129464 +129464,129464 +137052,129465 +129465,129465 +137053,129466 +129466,129466 +137054,129467 +129467,129467 +137055,129468 +129468,129468 +137056,129469 +129469,129469 +137057,129470 +129470,129470 +122817,129470 +137058,129471 +129471,129471 +137059,129472 +129472,129472 +137060,129473 +129473,129473 +137061,129474 +129474,129474 +137062,129475 +129475,129475 +137063,129476 +129476,129476 +137064,129477 +129477,129477 +137065,129478 +129478,129478 +145352,129478 +137066,129479 +129479,129479 +137067,129480 +129480,129480 +158443,129481 +137068,129481 +129481,129481 +137069,129482 +129482,129482 +137070,129483 +129483,129483 +137071,129484 +129484,129484 +137072,129485 +129485,129485 +137073,129486 +129486,129486 +137074,129487 +129487,129487 +137075,129488 +129488,129488 +137076,129489 +129489,129489 +137077,129490 +129490,129490 +137078,129491 +129491,129491 +137079,129492 +129492,129492 +5653,129492 +137080,129493 +129493,129493 +23548,129493 +137081,129494 +129494,129494 +137082,129495 +129495,129495 +137083,129496 +129496,129496 +137084,129497 +129497,129497 +137085,129498 +129498,129498 +137086,129499 +129499,129499 +137087,129500 +129500,129500 +137088,129501 +129501,129501 +137089,129502 +129502,129502 +137090,129503 +129503,129503 +137091,129504 +129504,129504 +137092,129505 +129505,129505 +60687,129505 +137093,129506 +129506,129506 +137094,129507 +129507,129507 +218487,129507 +137095,129508 +129508,129508 +185036,129509 +137096,129509 +129509,129509 +137097,129510 +129510,129510 +137098,129511 +129511,129511 +137099,129512 +129512,129512 +115932,129512 +137100,129513 +129513,129513 +137101,129514 +129514,129514 +137102,129515 +129515,129515 +137103,129516 +129516,129516 +137104,129517 +129517,129517 +137105,129518 +129518,129518 +137106,129519 +129519,129519 +137107,129520 +129520,129520 +137108,129521 +129521,129521 +137109,129522 +129522,129522 +137110,129523 +129523,129523 +137111,129524 +129524,129524 +137112,129525 +129525,129525 +137113,129526 +129526,129526 +137114,129527 +129527,129527 +137151,129564 +129564,129564 +137152,129565 +129565,129565 +5548,129565 +137153,129566 +129566,129566 +137154,129567 +129567,129567 +137155,129568 +129568,129568 +3134,129568 +137156,129569 +129569,129569 +137157,129570 +129570,129570 +188520,129570 +137158,129571 +129571,129571 +137159,129572 +129572,129572 +137160,129573 +129573,129573 +137161,129574 +129574,129574 +137162,129575 +129575,129575 +137163,129576 +129576,129576 +137164,129577 +129577,129577 +137165,129578 +129578,129578 +137166,129579 +129579,129579 +137167,129580 +129580,129580 +137168,129581 +129581,129581 +137169,129582 +129582,129582 +137170,129583 +129583,129583 +137171,129584 +129584,129584 +137172,129585 +129585,129585 +220058,129585 +137173,129586 +129586,129586 +137174,129587 +129587,129587 +137175,129588 +129588,129588 +137176,129589 +129589,129589 +137177,129590 +129590,129590 +137178,129591 +129591,129591 +226674,129591 +137179,129592 +129592,129592 +137180,129593 +129593,129593 +137181,129594 +129594,129594 +137182,129595 +129595,129595 +137183,129596 +129596,129596 +137184,129597 +129597,129597 +137185,129598 +129598,129598 +137186,129599 +129599,129599 +137187,129600 +129600,129600 +137188,129601 +129601,129601 +137189,129602 +129602,129602 +137190,129603 +129603,129603 +137313,129726 +129726,129726 +137720,129726 +130133,129726 +137314,129727 +129727,129727 +137315,129728 +129728,129728 +137723,129729 +137316,129729 +130136,129729 +129729,129729 +137724,129730 +137317,129730 +130137,129730 +129730,129730 +137318,129731 +129731,129731 +137725,129731 +130138,129731 +137726,129732 +137319,129732 +130139,129732 +129732,129732 +137320,129733 +129733,129733 +137727,129733 +130140,129733 +137321,129734 +129734,129734 +137728,129734 +130141,129734 +137322,129735 +129735,129735 +137729,129735 +130142,129735 +137730,129736 +137323,129736 +130143,129736 +129736,129736 +137731,129737 +137324,129737 +130144,129737 +129737,129737 +137732,129738 +137325,129738 +130145,129738 +129738,129738 +137733,129739 +137532,129739 +137326,129739 +130146,129739 +129945,129739 +129739,129739 +137734,129740 +137327,129740 +130147,129740 +129740,129740 +137735,129741 +137328,129741 +130148,129741 +129741,129741 +137736,129742 +137329,129742 +130149,129742 +129742,129742 +137737,129743 +137330,129743 +130150,129743 +129743,129743 +137738,129744 +137331,129744 +130151,129744 +129744,129744 +151798,129744 +137739,129745 +137332,129745 +130152,129745 +129745,129745 +137333,129746 +129746,129746 +137540,129747 +137334,129747 +129953,129747 +129747,129747 +137335,129748 +129748,129748 +137336,129749 +129749,129749 +137337,129750 +129750,129750 +137338,129751 +129751,129751 +137339,129752 +129752,129752 +137746,129752 +130159,129752 +137340,129753 +129753,129753 +137341,129754 +129754,129754 +137748,129754 +130161,129754 +137342,129755 +129755,129755 +137343,129756 +129756,129756 +137549,129756 +129962,129756 +137344,129757 +129757,129757 +137345,129758 +129758,129758 +137346,129759 +129759,129759 +150095,129759 +137347,129760 +129760,129760 +155443,129760 +137348,129761 +129761,129761 +137349,129762 +129762,129762 +137350,129763 +129763,129763 +137351,129764 +129764,129764 +137352,129765 +129765,129765 +137353,129766 +129766,129766 +137354,129767 +129767,129767 +137355,129768 +129768,129768 +137356,129769 +129769,129769 +137357,129770 +129770,129770 +137358,129771 +129771,129771 +137359,129772 +129772,129772 +137360,129773 +129773,129773 +190655,129773 +137361,129774 +129774,129774 +137362,129775 +129775,129775 +137363,129776 +129776,129776 +137528,129941 +129941,129941 +137529,129942 +129942,129942 +137530,129943 +129943,129943 +137531,129944 +129944,129944 +137733,129945 +137532,129945 +137326,129945 +130146,129945 +129945,129945 +129739,129945 +137533,129946 +129946,129946 +137534,129947 +129947,129947 +137535,129948 +129948,129948 +1064,129948 +137536,129949 +129949,129949 +137537,129950 +129950,129950 +137538,129951 +129951,129951 +137539,129952 +129952,129952 +137540,129953 +137334,129953 +129953,129953 +129747,129953 +146139,129954 +137541,129954 +129954,129954 +137542,129955 +129955,129955 +155432,129955 +151803,129955 +137543,129956 +129956,129956 +137544,129957 +129957,129957 +137545,129958 +129958,129958 +137546,129959 +129959,129959 +137547,129960 +129960,129960 +137749,129961 +137548,129961 +130162,129961 +129961,129961 +137549,129962 +129962,129962 +137343,129962 +129756,129962 +137550,129963 +129963,129963 +137752,129964 +137551,129964 +130165,129964 +129964,129964 +137552,129965 +129965,129965 +27194,129965 +137553,129966 +129966,129966 +137554,129967 +129967,129967 +224970,129967 +155445,129968 +146153,129968 +137555,129968 +129968,129968 +150099,129969 +146154,129969 +137556,129969 +129969,129969 +137557,129970 +129970,129970 +137558,129971 +129971,129971 +150293,129971 +137559,129972 +129972,129972 +137560,129973 +129973,129973 +137561,129974 +129974,129974 +137716,130129 +130129,130129 +137717,130130 +130130,130130 +137718,130131 +130131,130131 +137719,130132 +130132,130132 +137720,130133 +130133,130133 +148222,130133 +137313,130133 +129726,130133 +137721,130134 +130134,130134 +137722,130135 +130135,130135 +137723,130136 +137316,130136 +130136,130136 +129729,130136 +137724,130137 +137317,130137 +130137,130137 +129730,130137 +137725,130138 +130138,130138 +137318,130138 +129731,130138 +137726,130139 +137319,130139 +130139,130139 +129732,130139 +137727,130140 +130140,130140 +137320,130140 +129733,130140 +137728,130141 +130141,130141 +137321,130141 +129734,130141 +137729,130142 +130142,130142 +137322,130142 +129735,130142 +137730,130143 +137323,130143 +130143,130143 +129736,130143 +137731,130144 +137324,130144 +130144,130144 +129737,130144 +137732,130145 +137325,130145 +130145,130145 +129738,130145 +137733,130146 +137532,130146 +137326,130146 +130146,130146 +129945,130146 +129739,130146 +137734,130147 +137327,130147 +130147,130147 +129740,130147 +137735,130148 +137328,130148 +130148,130148 +129741,130148 +137736,130149 +137329,130149 +130149,130149 +129742,130149 +137737,130150 +137330,130150 +130150,130150 +129743,130150 +137738,130151 +137331,130151 +130151,130151 +129744,130151 +151798,130151 +137739,130152 +137332,130152 +130152,130152 +129745,130152 +137740,130153 +130153,130153 +137741,130154 +130154,130154 +137742,130155 +130155,130155 +137743,130156 +130156,130156 +137744,130157 +130157,130157 +155433,130157 +137745,130158 +130158,130158 +137746,130159 +130159,130159 +137339,130159 +129752,130159 +137747,130160 +130160,130160 +137748,130161 +130161,130161 +155437,130161 +137341,130161 +129754,130161 +137749,130162 +137548,130162 +130162,130162 +129961,130162 +137750,130163 +130163,130163 +137751,130164 +130164,130164 +137752,130165 +137551,130165 +130165,130165 +129964,130165 +137753,130166 +130166,130166 +137754,130167 +130167,130167 +137755,130168 +130168,130168 +203933,130168 +150097,130168 +137756,130169 +130169,130169 +137757,130170 +130170,130170 +137758,130171 +130171,130171 +75601,130171 +137759,130172 +130172,130172 +137760,130173 +130173,130173 +137761,130174 +130174,130174 +137762,130175 +130175,130175 +150105,130176 +137763,130176 +130176,130176 +146160,130176 +137922,130335 +130335,130335 +137923,130336 +130336,130336 +88242,130336 +137924,130337 +130337,130337 +137925,130338 +130338,130338 +137926,130339 +130339,130339 +137927,130340 +130340,130340 +137928,130341 +130341,130341 +137929,130342 +130342,130342 +137930,130343 +130343,130343 +137931,130344 +130344,130344 +84566,130344 +137932,130345 +130345,130345 +137933,130346 +130346,130346 +137934,130347 +130347,130347 +137935,130348 +130348,130348 +137936,130349 +130349,130349 +137937,130350 +130350,130350 +137938,130351 +130351,130351 +206664,130351 +137939,130352 +130352,130352 +179181,130352 +137940,130353 +130353,130353 +137941,130354 +130354,130354 +137942,130355 +130355,130355 +137943,130356 +130356,130356 +177817,130356 +171853,130356 +137944,130357 +130357,130357 +137945,130358 +130358,130358 +137946,130359 +130359,130359 +137947,130360 +130360,130360 +137948,130361 +130361,130361 +137949,130362 +130362,130362 +137950,130363 +130363,130363 +137951,130364 +130364,130364 +137952,130365 +130365,130365 +137953,130366 +130366,130366 +92673,130366 +137954,130367 +130367,130367 +137955,130368 +130368,130368 +137956,130369 +130369,130369 +137957,130370 +130370,130370 +137958,130371 +130371,130371 +135820,130371 +128233,130371 +137959,130372 +130372,130372 +137960,130373 +130373,130373 +137961,130374 +130374,130374 +137962,130375 +130375,130375 +137963,130376 +130376,130376 +137964,130377 +130377,130377 +137965,130378 +130378,130378 +137966,130379 +130379,130379 +137967,130380 +130380,130380 +137968,130381 +130381,130381 +33428,130381 +137969,130382 +130382,130382 +137970,130383 +130383,130383 +137971,130384 +130384,130384 +137972,130385 +130385,130385 +137973,130386 +130386,130386 +117045,130386 +137974,130387 +130387,130387 +137975,130388 +130388,130388 +137976,130389 +130389,130389 +137977,130390 +130390,130390 +137978,130391 +130391,130391 +137979,130392 +130392,130392 +137980,130393 +130393,130393 +63457,130393 +137981,130394 +130394,130394 +137982,130395 +130395,130395 +137983,130396 +130396,130396 +137984,130397 +130397,130397 +137985,130398 +130398,130398 +137986,130399 +130399,130399 +137987,130400 +130400,130400 +137988,130401 +130401,130401 +137989,130402 +130402,130402 +137990,130403 +130403,130403 +137991,130404 +130404,130404 +137992,130405 +130405,130405 +137993,130406 +130406,130406 +137994,130407 +130407,130407 +137995,130408 +130408,130408 +137996,130409 +130409,130409 +137997,130410 +130410,130410 +137998,130411 +130411,130411 +137999,130412 +130412,130412 +138000,130413 +130413,130413 +158109,130414 +138001,130414 +130414,130414 +138002,130415 +130415,130415 +138003,130416 +130416,130416 +138004,130417 +130417,130417 +138005,130418 +130418,130418 +138006,130419 +130419,130419 +138007,130420 +130420,130420 +138008,130421 +130421,130421 +138009,130422 +130422,130422 +138010,130423 +130423,130423 +138011,130424 +130424,130424 +138012,130425 +130425,130425 +138013,130426 +130426,130426 +138014,130427 +130427,130427 +138015,130428 +130428,130428 +138016,130429 +130429,130429 +138017,130430 +130430,130430 +138018,130431 +130431,130431 +138019,130432 +130432,130432 +138020,130433 +130433,130433 +138021,130434 +130434,130434 +138022,130435 +130435,130435 +138023,130436 +130436,130436 +138024,130437 +130437,130437 +138025,130438 +130438,130438 +138026,130439 +130439,130439 +138027,130440 +130440,130440 +138028,130441 +130441,130441 +138029,130442 +130442,130442 +32320,130442 +138030,130443 +130443,130443 +138031,130444 +130444,130444 +138032,130445 +130445,130445 +138033,130446 +130446,130446 +138034,130447 +130447,130447 +138035,130448 +130448,130448 +246740,130448 +138077,130490 +130490,130490 +138078,130491 +130491,130491 +138079,130492 +130492,130492 +138080,130493 +130493,130493 +138081,130494 +130494,130494 +138082,130495 +130495,130495 +138083,130496 +130496,130496 +138084,130497 +130497,130497 +138085,130498 +130498,130498 +138086,130499 +130499,130499 +138087,130500 +130500,130500 +138088,130501 +130501,130501 +138089,130502 +130502,130502 +138090,130503 +130503,130503 +138091,130504 +130504,130504 +138092,130505 +130505,130505 +138093,130506 +130506,130506 +138094,130507 +130507,130507 +138095,130508 +130508,130508 +138096,130509 +130509,130509 +138097,130510 +130510,130510 +138098,130511 +130511,130511 +138099,130512 +130512,130512 +138100,130513 +130513,130513 +138101,130514 +130514,130514 +138102,130515 +130515,130515 +138103,130516 +130516,130516 +138104,130517 +130517,130517 +138105,130518 +130518,130518 +138106,130519 +130519,130519 +138107,130520 +130520,130520 +138108,130521 +130521,130521 +138109,130522 +130522,130522 +22081,130522 +138110,130523 +130523,130523 +138111,130524 +130524,130524 +138112,130525 +130525,130525 +138113,130526 +130526,130526 +138114,130527 +130527,130527 +138115,130528 +130528,130528 +138116,130529 +130529,130529 +138117,130530 +130530,130530 +138118,130531 +130531,130531 +138119,130532 +130532,130532 +138120,130533 +130533,130533 +138121,130534 +130534,130534 +138122,130535 +130535,130535 +138123,130536 +130536,130536 +138124,130537 +130537,130537 +138125,130538 +130538,130538 +119005,130538 +138126,130539 +130539,130539 +138127,130540 +130540,130540 +138128,130541 +130541,130541 +138129,130542 +130542,130542 +138130,130543 +130543,130543 +138131,130544 +130544,130544 +138132,130545 +130545,130545 +138133,130546 +130546,130546 +138134,130547 +130547,130547 +138135,130548 +130548,130548 +138136,130549 +130549,130549 +138137,130550 +130550,130550 +138138,130551 +130551,130551 +22651,130551 +138139,130552 +130552,130552 +138140,130553 +130553,130553 +138141,130554 +130554,130554 +138142,130555 +130555,130555 +138143,130556 +130556,130556 +138144,130557 +130557,130557 +138145,130558 +130558,130558 +138146,130559 +130559,130559 +138147,130560 +130560,130560 +138148,130561 +130561,130561 +138149,130562 +130562,130562 +138150,130563 +130563,130563 +139171,130563 +214745,130563 +138151,130564 +130564,130564 +138152,130565 +130565,130565 +138153,130566 +130566,130566 +138154,130567 +130567,130567 +138155,130568 +130568,130568 +138156,130569 +130569,130569 +138157,130570 +130570,130570 +138158,130571 +130571,130571 +138159,130572 +130572,130572 +138160,130573 +130573,130573 +138161,130574 +130574,130574 +138162,130575 +130575,130575 +138163,130576 +130576,130576 +138164,130577 +130577,130577 +138165,130578 +130578,130578 +234788,130579 +138166,130579 +130579,130579 +138167,130580 +130580,130580 +138168,130581 +130581,130581 +138169,130582 +130582,130582 +138170,130583 +130583,130583 +242944,130583 +138171,130584 +130584,130584 +138172,130585 +130585,130585 +138173,130586 +130586,130586 +138174,130587 +130587,130587 +28369,130587 +131692,131692 +124105,131692 +131693,131693 +124106,131693 +131694,131694 +124107,131694 +131695,131695 +124108,131695 +131696,131696 +124109,131696 +131697,131697 +124110,131697 +131698,131698 +124111,131698 +131699,131699 +124112,131699 +131700,131700 +124113,131700 +131701,131701 +124114,131701 +131702,131702 +124115,131702 +131703,131703 +124116,131703 +131704,131704 +124117,131704 +119771,131704 +131705,131705 +124118,131705 +131706,131706 +124119,131706 +131707,131707 +124120,131707 +131708,131708 +124121,131708 +131709,131709 +124122,131709 +131710,131710 +124123,131710 +131711,131711 +124124,131711 +131712,131712 +124125,131712 +131713,131713 +124126,131713 +131714,131714 +124127,131714 +131715,131715 +124128,131715 +131716,131716 +124129,131716 +131717,131717 +124130,131717 +181268,131717 +131718,131718 +124131,131718 +131719,131719 +124132,131719 +131720,131720 +124133,131720 +131721,131721 +124134,131721 +131722,131722 +124135,131722 +131723,131723 +124136,131723 +131724,131724 +124137,131724 +132115,131725 +131725,131725 +124528,131725 +124138,131725 +131726,131726 +124139,131726 +131727,131727 +124140,131727 +131728,131728 +124141,131728 +131729,131729 +124142,131729 +131730,131730 +124143,131730 +131731,131731 +124144,131731 +131732,131732 +124145,131732 +131754,131732 +124167,131732 +131755,131733 +131733,131733 +124168,131733 +124146,131733 +131734,131734 +124147,131734 +131735,131735 +124148,131735 +131736,131736 +124149,131736 +131737,131737 +124150,131737 +131738,131738 +124151,131738 +131739,131739 +124152,131739 +131740,131740 +124153,131740 +131741,131741 +124154,131741 +131742,131742 +124155,131742 +131743,131743 +124156,131743 +131744,131744 +124157,131744 +131745,131745 +124158,131745 +131746,131746 +124159,131746 +131747,131747 +124160,131747 +131748,131748 +124161,131748 +131749,131749 +124162,131749 +131750,131750 +124163,131750 +131751,131751 +124164,131751 +131752,131752 +124165,131752 +131753,131753 +124166,131753 +162851,131753 +131754,131754 +124167,131754 +131732,131754 +124145,131754 +131755,131755 +131733,131755 +124168,131755 +124146,131755 +131756,131756 +124169,131756 +131757,131757 +124170,131757 +131758,131758 +124171,131758 +131759,131759 +124172,131759 +131760,131760 +124173,131760 +131761,131761 +124174,131761 +131762,131762 +124175,131762 +131763,131763 +124176,131763 +131764,131764 +124177,131764 +131765,131765 +124178,131765 +131766,131766 +124179,131766 +131767,131767 +124180,131767 +131768,131768 +124181,131768 +131769,131769 +124182,131769 +131770,131770 +124183,131770 +131771,131771 +124184,131771 +131772,131772 +124185,131772 +131773,131773 +124186,131773 +131774,131774 +124187,131774 +78630,131774 +131775,131775 +124188,131775 +131776,131776 +124189,131776 +131777,131777 +124190,131777 +131778,131778 +124191,131778 +131779,131779 +124192,131779 +131780,131780 +124193,131780 +131781,131781 +124194,131781 +131782,131782 +124195,131782 +131783,131783 +124196,131783 +131784,131784 +124197,131784 +131785,131785 +124198,131785 +131786,131786 +124199,131786 +131787,131787 +124200,131787 +131788,131788 +124201,131788 +131789,131789 +124202,131789 +131790,131790 +124203,131790 +131791,131791 +124204,131791 +131792,131792 +124205,131792 +131793,131793 +124206,131793 +131794,131794 +124207,131794 +131795,131795 +124208,131795 +131796,131796 +124209,131796 +133088,131797 +131797,131797 +125501,131797 +124210,131797 +131798,131798 +124211,131798 +131799,131799 +124212,131799 +131800,131800 +124213,131800 +131801,131801 +124214,131801 +131896,131896 +124309,131896 +131897,131897 +124310,131897 +131898,131898 +124311,131898 +131899,131899 +124312,131899 +131900,131900 +124313,131900 +131901,131901 +124314,131901 +131902,131902 +124315,131902 +131903,131903 +124316,131903 +131904,131904 +124317,131904 +131905,131905 +124318,131905 +131906,131906 +124319,131906 +131907,131907 +124320,131907 +131908,131908 +124321,131908 +84948,131908 +131909,131909 +124322,131909 +132102,131909 +124515,131909 +131910,131910 +124323,131910 +131911,131911 +124324,131911 +132104,131911 +124517,131911 +131912,131912 +124325,131912 +136999,131912 +129412,131912 +131913,131913 +124326,131913 +123497,131913 +131914,131914 +124327,131914 +123498,131914 +131915,131915 +124328,131915 +131916,131916 +124329,131916 +215648,131916 +131917,131917 +124330,131917 +132111,131918 +131918,131918 +124524,131918 +124331,131918 +131919,131919 +124332,131919 +132113,131920 +131920,131920 +124526,131920 +124333,131920 +131921,131921 +124334,131921 +131922,131922 +124335,131922 +131923,131923 +124336,131923 +132117,131924 +131924,131924 +124530,131924 +124337,131924 +67591,131924 +131925,131925 +124338,131925 +132118,131925 +124531,131925 +131926,131926 +124339,131926 +131927,131927 +124340,131927 +55947,131927 +47843,131927 +131928,131928 +124341,131928 +131929,131929 +124342,131929 +131930,131930 +124343,131930 +131931,131931 +124344,131931 +132124,131931 +124537,131931 +64913,131931 +131932,131932 +124345,131932 +131933,131933 +124346,131933 +132126,131933 +124539,131933 +131934,131934 +124347,131934 +132127,131934 +124540,131934 +131935,131935 +124348,131935 +131936,131936 +124349,131936 +131937,131937 +124350,131937 +205090,131937 +131938,131938 +124351,131938 +131939,131939 +124352,131939 +131940,131940 +124353,131940 +131941,131941 +124354,131941 +131942,131942 +124355,131942 +131943,131943 +124356,131943 +131944,131944 +124357,131944 +131945,131945 +124358,131945 +131946,131946 +124359,131946 +131947,131947 +124360,131947 +131948,131948 +124361,131948 +131949,131949 +124362,131949 +131950,131950 +124363,131950 +131951,131951 +124364,131951 +131952,131952 +124365,131952 +131953,131953 +124366,131953 +131954,131954 +124367,131954 +131955,131955 +124368,131955 +131956,131956 +124369,131956 +131957,131957 +124370,131957 +131958,131958 +124371,131958 +131959,131959 +124372,131959 +111880,131959 +131960,131960 +124373,131960 +131961,131961 +124374,131961 +131962,131962 +124375,131962 +131963,131963 +124376,131963 +131964,131964 +124377,131964 +131965,131965 +124378,131965 +131966,131966 +124379,131966 +131967,131967 +124380,131967 +131968,131968 +124381,131968 +131969,131969 +124382,131969 +131970,131970 +124383,131970 +246669,131970 +131971,131971 +124384,131971 +131972,131972 +124385,131972 +131973,131973 +124386,131973 +131974,131974 +124387,131974 +131975,131975 +124388,131975 +131976,131976 +124389,131976 +187378,131976 +131977,131977 +124390,131977 +131978,131978 +124391,131978 +131979,131979 +124392,131979 +131980,131980 +124393,131980 +243144,131980 +133079,131981 +131981,131981 +125492,131981 +124394,131981 +136322,131981 +128735,131981 +131982,131982 +124395,131982 +131983,131983 +124396,131983 +131984,131984 +124397,131984 +131985,131985 +124398,131985 +131986,131986 +124399,131986 +131987,131987 +124400,131987 +131988,131988 +124401,131988 +131989,131989 +124402,131989 +131990,131990 +124403,131990 +131991,131991 +124404,131991 +131992,131992 +124405,131992 +131993,131993 +124406,131993 +139150,131993 +131994,131994 +124407,131994 +131995,131995 +124408,131995 +131996,131996 +124409,131996 +132100,132100 +124513,132100 +132101,132101 +124514,132101 +229163,132101 +132102,132102 +124515,132102 +131909,132102 +124322,132102 +132103,132103 +124516,132103 +132104,132104 +124517,132104 +131911,132104 +124324,132104 +132105,132105 +124518,132105 +112453,132105 +132106,132106 +124519,132106 +132107,132107 +124520,132107 +132108,132108 +124521,132108 +16456,132108 +132109,132109 +124522,132109 +145713,132109 +132110,132110 +124523,132110 +132111,132111 +131918,132111 +124524,132111 +124331,132111 +132112,132112 +124525,132112 +249575,132112 +132113,132113 +131920,132113 +124526,132113 +124333,132113 +132114,132114 +124527,132114 +132115,132115 +131725,132115 +124528,132115 +124138,132115 +132116,132116 +124529,132116 +132117,132117 +131924,132117 +124530,132117 +124337,132117 +67591,132117 +132118,132118 +124531,132118 +131925,132118 +124338,132118 +132119,132119 +124532,132119 +132120,132120 +124533,132120 +132121,132121 +124534,132121 +132122,132122 +124535,132122 +132123,132123 +124536,132123 +132124,132124 +124537,132124 +131931,132124 +124344,132124 +132125,132125 +124538,132125 +132126,132126 +124539,132126 +28096,132126 +131933,132126 +124346,132126 +132127,132127 +124540,132127 +131934,132127 +124347,132127 +132128,132128 +124541,132128 +132129,132129 +124542,132129 +132130,132130 +124543,132130 +132131,132131 +124544,132131 +132132,132132 +124545,132132 +132133,132133 +124546,132133 +132134,132134 +124547,132134 +132135,132135 +124548,132135 +132136,132136 +124549,132136 +132137,132137 +124550,132137 +132138,132138 +124551,132138 +132286,132286 +124699,132286 +132287,132287 +124700,132287 +132288,132288 +124701,132288 +132289,132289 +124702,132289 +132290,132290 +124703,132290 +132291,132291 +124704,132291 +34052,132291 +157555,132291 +237996,132291 +132292,132292 +124705,132292 +132293,132293 +124706,132293 +132294,132294 +124707,132294 +132295,132295 +124708,132295 +132296,132296 +124709,132296 +132297,132297 +124710,132297 +132298,132298 +124711,132298 +132299,132299 +124712,132299 +132300,132300 +124713,132300 +34061,132300 +132301,132301 +124714,132301 +132302,132302 +124715,132302 +147640,132302 +132303,132303 +124716,132303 +132304,132304 +124717,132304 +132305,132305 +124718,132305 +132306,132306 +124719,132306 +132307,132307 +124720,132307 +132308,132308 +124721,132308 +132309,132309 +124722,132309 +146422,132309 +132310,132310 +124723,132310 +226661,132310 +132311,132311 +124724,132311 +132312,132312 +124725,132312 +132313,132313 +124726,132313 +132350,132350 +124763,132350 +132351,132351 +124764,132351 +132352,132352 +124765,132352 +132353,132353 +124766,132353 +132354,132354 +124767,132354 +140980,132354 +132355,132355 +124768,132355 +132356,132356 +124769,132356 +132357,132357 +124770,132357 +132358,132358 +124771,132358 +132359,132359 +124772,132359 +132360,132360 +124773,132360 +142862,132360 +132361,132361 +124774,132361 +132362,132362 +124775,132362 +132363,132363 +124776,132363 +132364,132364 +124777,132364 +132365,132365 +124778,132365 +132366,132366 +124779,132366 +132367,132367 +124780,132367 +132368,132368 +124781,132368 +132369,132369 +124782,132369 +142855,132369 +132370,132370 +124783,132370 +139985,132370 +132371,132371 +124784,132371 +132372,132372 +124785,132372 +132373,132373 +124786,132373 +83044,132373 +132374,132374 +124787,132374 +132375,132375 +124788,132375 +132376,132376 +124789,132376 +132377,132377 +124790,132377 +132378,132378 +124791,132378 +132379,132379 +124792,132379 +141741,132379 +132380,132380 +124793,132380 +132381,132381 +124794,132381 +132382,132382 +124795,132382 +132383,132383 +124796,132383 +231828,132383 +132384,132384 +124797,132384 +132385,132385 +124798,132385 +132386,132386 +124799,132386 +132387,132387 +124800,132387 +132388,132388 +124801,132388 +146482,132388 +132389,132389 +124802,132389 +132390,132390 +124803,132390 +132391,132391 +124804,132391 +132392,132392 +124805,132392 +132393,132393 +124806,132393 +132394,132394 +124807,132394 +132395,132395 +124808,132395 +132396,132396 +124809,132396 +182320,132396 +66487,132396 +132397,132397 +124810,132397 +142465,132397 +132398,132398 +124811,132398 +31656,132398 +132399,132399 +124812,132399 +132400,132400 +124813,132400 +132401,132401 +124814,132401 +132402,132402 +124815,132402 +132403,132403 +124816,132403 +132404,132404 +124817,132404 +57604,132404 +132405,132405 +124818,132405 +132406,132406 +124819,132406 +132407,132407 +124820,132407 +132408,132408 +124821,132408 +54986,132408 +46882,132408 +132409,132409 +124822,132409 +132410,132410 +124823,132410 +132411,132411 +124824,132411 +132412,132412 +124825,132412 +132451,132451 +124864,132451 +132452,132452 +124865,132452 +132453,132453 +124866,132453 +132497,132497 +124910,132497 +132498,132498 +124911,132498 +142119,132498 +132499,132499 +124912,132499 +132500,132500 +124913,132500 +98239,132500 +132501,132501 +124914,132501 +132502,132502 +124915,132502 +132503,132503 +124916,132503 +132504,132504 +124917,132504 +132505,132505 +124918,132505 +132506,132506 +124919,132506 +132507,132507 +124920,132507 +132508,132508 +124921,132508 +132509,132509 +124922,132509 +132510,132510 +124923,132510 +132511,132511 +124924,132511 +132512,132512 +124925,132512 +132513,132513 +124926,132513 +132514,132514 +124927,132514 +132515,132515 +124928,132515 +132516,132516 +124929,132516 +132517,132517 +124930,132517 +132518,132518 +124931,132518 +132519,132519 +124932,132519 +132520,132520 +124933,132520 +132521,132521 +124934,132521 +132522,132522 +124935,132522 +132523,132523 +124936,132523 +132524,132524 +124937,132524 +132525,132525 +124938,132525 +132526,132526 +124939,132526 +132527,132527 +124940,132527 +132528,132528 +124941,132528 +132529,132529 +124942,132529 +132530,132530 +124943,132530 +132531,132531 +124944,132531 +132532,132532 +124945,132532 +132533,132533 +124946,132533 +132534,132534 +124947,132534 +132596,132596 +125009,132596 +132597,132597 +125010,132597 +11561,132597 +132598,132598 +125011,132598 +132599,132599 +125012,132599 +132600,132600 +125013,132600 +132601,132601 +125014,132601 +132602,132602 +125015,132602 +223447,132602 +132603,132603 +125016,132603 +132604,132604 +125017,132604 +132605,132605 +125018,132605 +132606,132606 +125019,132606 +132607,132607 +125020,132607 +218039,132607 +132608,132608 +125021,132608 +5116,132608 +132609,132609 +125022,132609 +132610,132610 +125023,132610 +132611,132611 +125024,132611 +132612,132612 +125025,132612 +132613,132613 +125026,132613 +132614,132614 +125027,132614 +132615,132615 +125028,132615 +132616,132616 +125029,132616 +132617,132617 +125030,132617 +132618,132618 +125031,132618 +134051,132618 +126464,132618 +132619,132619 +125032,132619 +132620,132620 +125033,132620 +132621,132621 +125034,132621 +132622,132622 +125035,132622 +132623,132623 +125036,132623 +132624,132624 +125037,132624 +132625,132625 +125038,132625 +132626,132626 +125039,132626 +111018,132626 +132627,132627 +125040,132627 +132628,132628 +125041,132628 +132629,132629 +125042,132629 +132630,132630 +125043,132630 +132631,132631 +125044,132631 +132632,132632 +125045,132632 +132633,132633 +125046,132633 +132634,132634 +125047,132634 +132635,132635 +125048,132635 +132636,132636 +125049,132636 +132637,132637 +125050,132637 +132638,132638 +125051,132638 +132639,132639 +125052,132639 +132640,132640 +125053,132640 +132641,132641 +125054,132641 +132642,132642 +125055,132642 +88047,132642 +132643,132643 +125056,132643 +132644,132644 +125057,132644 +132645,132645 +125058,132645 +132646,132646 +125059,132646 +132647,132647 +125060,132647 +132648,132648 +125061,132648 +132649,132649 +125062,132649 +132650,132650 +125063,132650 +132651,132651 +125064,132651 +132652,132652 +125065,132652 +132701,132701 +125114,132701 +132702,132702 +125115,132702 +132703,132703 +125116,132703 +132704,132704 +125117,132704 +132705,132705 +125118,132705 +132706,132706 +125119,132706 +132707,132707 +125120,132707 +132708,132708 +125121,132708 +132709,132709 +125122,132709 +132710,132710 +125123,132710 +132711,132711 +125124,132711 +132712,132712 +125125,132712 +2208,132712 +132713,132713 +125126,132713 +132714,132714 +125127,132714 +132715,132715 +125128,132715 +247362,132715 +132716,132716 +125129,132716 +192035,132716 +132717,132717 +125130,132717 +132718,132718 +125131,132718 +132719,132719 +125132,132719 +132720,132720 +125133,132720 +132721,132721 +125134,132721 +132722,132722 +125135,132722 +132723,132723 +125136,132723 +132724,132724 +125137,132724 +132725,132725 +125138,132725 +132726,132726 +125139,132726 +132727,132727 +125140,132727 +132728,132728 +125141,132728 +132729,132729 +125142,132729 +132730,132730 +125143,132730 +132731,132731 +125144,132731 +132732,132732 +125145,132732 +132733,132733 +125146,132733 +132734,132734 +125147,132734 +132735,132735 +125148,132735 +132736,132736 +125149,132736 +132737,132737 +125150,132737 +132738,132738 +125151,132738 +132739,132739 +125152,132739 +132740,132740 +125153,132740 +132741,132741 +125154,132741 +132742,132742 +125155,132742 +132743,132743 +125156,132743 +132744,132744 +125157,132744 +132745,132745 +125158,132745 +132746,132746 +125159,132746 +132747,132747 +125160,132747 +132748,132748 +125161,132748 +132749,132749 +125162,132749 +132750,132750 +125163,132750 +132751,132751 +125164,132751 +132752,132752 +125165,132752 +132753,132753 +125166,132753 +132754,132754 +125167,132754 +238930,132754 +213582,132754 +132755,132755 +125168,132755 +132756,132756 +125169,132756 +88179,132756 +132757,132757 +125170,132757 +132758,132758 +125171,132758 +132759,132759 +125172,132759 +132760,132760 +125173,132760 +132761,132761 +125174,132761 +132762,132762 +125175,132762 +132763,132763 +125176,132763 +132764,132764 +125177,132764 +132765,132765 +125178,132765 +132766,132766 +125179,132766 +132767,132767 +125180,132767 +132768,132768 +125181,132768 +57730,132768 +132769,132769 +125182,132769 +132770,132770 +125183,132770 +132771,132771 +125184,132771 +132772,132772 +125185,132772 +132773,132773 +125186,132773 +132774,132774 +125187,132774 +191284,132774 +132775,132775 +125188,132775 +132776,132776 +125189,132776 +132777,132777 +125190,132777 +132778,132778 +125191,132778 +132779,132779 +125192,132779 +132780,132780 +125193,132780 +132781,132781 +125194,132781 +132782,132782 +125195,132782 +132783,132783 +125196,132783 +132784,132784 +125197,132784 +132785,132785 +125198,132785 +132786,132786 +125199,132786 +132787,132787 +125200,132787 +201468,132787 +132788,132788 +125201,132788 +132789,132789 +125202,132789 +132790,132790 +125203,132790 +132791,132791 +125204,132791 +132792,132792 +125205,132792 +132793,132793 +125206,132793 +178306,132793 +172342,132793 +132794,132794 +125207,132794 +132795,132795 +125208,132795 +132796,132796 +125209,132796 +236432,132797 +132797,132797 +125210,132797 +132798,132798 +125211,132798 +132799,132799 +125212,132799 +152977,132799 +132800,132800 +125213,132800 +132801,132801 +125214,132801 +242926,132801 +132802,132802 +125215,132802 +132803,132803 +125216,132803 +132804,132804 +125217,132804 +132805,132805 +125218,132805 +132806,132806 +125219,132806 +132807,132807 +125220,132807 +132808,132808 +125221,132808 +132809,132809 +125222,132809 +132810,132810 +125223,132810 +132811,132811 +125224,132811 +132812,132812 +125225,132812 +132813,132813 +125226,132813 +132814,132814 +125227,132814 +132815,132815 +125228,132815 +132816,132816 +125229,132816 +132817,132817 +125230,132817 +132818,132818 +125231,132818 +132819,132819 +125232,132819 +132820,132820 +125233,132820 +132821,132821 +125234,132821 +132822,132822 +125235,132822 +132823,132823 +125236,132823 +132824,132824 +125237,132824 +132825,132825 +125238,132825 +132826,132826 +125239,132826 +132827,132827 +125240,132827 +132828,132828 +125241,132828 +132829,132829 +125242,132829 +132830,132830 +125243,132830 +132831,132831 +125244,132831 +36116,132831 +132832,132832 +125245,132832 +122899,132832 +132833,132833 +125246,132833 +132834,132834 +125247,132834 +132835,132835 +125248,132835 +207850,132835 +132836,132836 +125249,132836 +132837,132837 +125250,132837 +132838,132838 +125251,132838 +132839,132839 +125252,132839 +132840,132840 +125253,132840 +132841,132841 +125254,132841 +132842,132842 +125255,132842 +132843,132843 +125256,132843 +132913,132913 +125326,132913 +132914,132914 +125327,132914 +132915,132915 +125328,132915 +25075,132915 +119268,132915 +132916,132916 +125329,132916 +132917,132917 +125330,132917 +132918,132918 +125331,132918 +132919,132919 +125332,132919 +132920,132920 +125333,132920 +132921,132921 +125334,132921 +132922,132922 +125335,132922 +132923,132923 +125336,132923 +132924,132924 +125337,132924 +132925,132925 +125338,132925 +132926,132926 +125339,132926 +132927,132927 +125340,132927 +132928,132928 +125341,132928 +206858,132928 +132929,132929 +125342,132929 +50150,132929 +132930,132930 +125343,132930 +132931,132931 +125344,132931 +132932,132932 +125345,132932 +132933,132933 +125346,132933 +132934,132934 +125347,132934 +132935,132935 +125348,132935 +132936,132936 +125349,132936 +33507,132936 +234861,132936 +132937,132937 +125350,132937 +132938,132938 +125351,132938 +132939,132939 +125352,132939 +132940,132940 +125353,132940 +132941,132941 +125354,132941 +132942,132942 +125355,132942 +132943,132943 +125356,132943 +30227,132943 +132944,132944 +125357,132944 +214671,132944 +132945,132945 +125358,132945 +132946,132946 +125359,132946 +132947,132947 +125360,132947 +132948,132948 +125361,132948 +132949,132949 +125362,132949 +132950,132950 +125363,132950 +132951,132951 +125364,132951 +132952,132952 +125365,132952 +132953,132953 +125366,132953 +132954,132954 +125367,132954 +21971,132954 +132955,132955 +125368,132955 +132956,132956 +125369,132956 +132957,132957 +125370,132957 +132958,132958 +125371,132958 +132959,132959 +125372,132959 +132960,132960 +125373,132960 +132961,132961 +125374,132961 +132962,132962 +125375,132962 +132963,132963 +125376,132963 +132964,132964 +125377,132964 +194975,132965 +132965,132965 +125378,132965 +132966,132966 +125379,132966 +4577,132966 +132967,132967 +125380,132967 +132968,132968 +125381,132968 +132969,132969 +125382,132969 +132970,132970 +125383,132970 +132971,132971 +125384,132971 +132972,132972 +125385,132972 +132973,132973 +125386,132973 +132974,132974 +125387,132974 +132975,132975 +125388,132975 +132976,132976 +125389,132976 +132977,132977 +125390,132977 +132978,132978 +125391,132978 +132979,132979 +125392,132979 +132980,132980 +125393,132980 +132981,132981 +125394,132981 +132982,132982 +125395,132982 +132983,132983 +125396,132983 +132984,132984 +125397,132984 +132985,132985 +125398,132985 +132986,132986 +125399,132986 +132987,132987 +125400,132987 +114424,132987 +132988,132988 +125401,132988 +132989,132989 +125402,132989 +132990,132990 +125403,132990 +132991,132991 +125404,132991 +132992,132992 +125405,132992 +133031,133031 +125444,133031 +133032,133032 +125445,133032 +232548,133032 +133033,133033 +125446,133033 +133034,133034 +125447,133034 +31643,133034 +133035,133035 +125448,133035 +133036,133036 +125449,133036 +133037,133037 +125450,133037 +209943,133037 +133038,133038 +125451,133038 +149849,133038 +97739,133038 +133039,133039 +125452,133039 +133040,133040 +125453,133040 +193271,133040 +133041,133041 +125454,133041 +162960,133041 +133042,133042 +125455,133042 +133043,133043 +125456,133043 +133044,133044 +125457,133044 +133045,133045 +125458,133045 +133046,133046 +125459,133046 +133047,133047 +125460,133047 +133048,133048 +125461,133048 +133049,133049 +125462,133049 +133050,133050 +125463,133050 +133051,133051 +125464,133051 +133052,133052 +125465,133052 +133053,133053 +125466,133053 +133054,133054 +125467,133054 +133055,133055 +125468,133055 +133056,133056 +125469,133056 +133057,133057 +125470,133057 +133058,133058 +125471,133058 +133059,133059 +125472,133059 +57332,133059 +133060,133060 +125473,133060 +133061,133061 +125474,133061 +133062,133062 +125475,133062 +133063,133063 +125476,133063 +133064,133064 +125477,133064 +133065,133065 +125478,133065 +133066,133066 +125479,133066 +133067,133067 +125480,133067 +133068,133068 +125481,133068 +133069,133069 +125482,133069 +133070,133070 +125483,133070 +133071,133071 +125484,133071 +133072,133072 +125485,133072 +133073,133073 +125486,133073 +133074,133074 +125487,133074 +133075,133075 +125488,133075 +133076,133076 +125489,133076 +133077,133077 +125490,133077 +111836,133077 +133078,133078 +125491,133078 +133079,133079 +131981,133079 +125492,133079 +124394,133079 +136322,133079 +128735,133079 +133080,133080 +125493,133080 +133081,133081 +125494,133081 +133082,133082 +125495,133082 +133083,133083 +125496,133083 +133084,133084 +125497,133084 +133085,133085 +125498,133085 +133086,133086 +125499,133086 +133087,133087 +125500,133087 +133088,133088 +131797,133088 +125501,133088 +124210,133088 +133089,133089 +125502,133089 +133090,133090 +125503,133090 +133091,133091 +125504,133091 +133092,133092 +125505,133092 +133093,133093 +125506,133093 +133094,133094 +125507,133094 +133095,133095 +125508,133095 +58665,133095 +133096,133096 +125509,133096 +133097,133097 +125510,133097 +133098,133098 +125511,133098 +67626,133098 +133099,133099 +125512,133099 +133100,133100 +125513,133100 +133101,133101 +125514,133101 +62005,133101 +57317,133101 +133102,133102 +125515,133102 +133103,133103 +125516,133103 +133104,133104 +125517,133104 +133105,133105 +125518,133105 +133106,133106 +125519,133106 +133107,133107 +125520,133107 +133108,133108 +125521,133108 +133109,133109 +125522,133109 +133110,133110 +125523,133110 +133111,133111 +125524,133111 +133112,133112 +125525,133112 +133113,133113 +125526,133113 +133114,133114 +125527,133114 +133181,133181 +125594,133181 +133182,133182 +125595,133182 +133183,133183 +125596,133183 +133184,133184 +125597,133184 +133185,133185 +125598,133185 +133186,133186 +125599,133186 +149100,133186 +133187,133187 +125600,133187 +56078,133187 +47974,133187 +133188,133188 +125601,133188 +133189,133189 +125602,133189 +133190,133190 +125603,133190 +133191,133191 +125604,133191 +133192,133192 +125605,133192 +133193,133193 +125606,133193 +59375,133193 +2443,133193 +133194,133194 +125607,133194 +133195,133195 +125608,133195 +2445,133195 +133196,133196 +125609,133196 +133197,133197 +125610,133197 +133198,133198 +125611,133198 +133199,133199 +125612,133199 +133200,133200 +125613,133200 +133201,133201 +125614,133201 +133202,133202 +125615,133202 +133203,133203 +125616,133203 +161209,133203 +133204,133204 +125617,133204 +133205,133205 +125618,133205 +133206,133206 +125619,133206 +133207,133207 +125620,133207 +133208,133208 +125621,133208 +133209,133209 +125622,133209 +133210,133210 +125623,133210 +133211,133211 +125624,133211 +10974,133211 +133212,133212 +125625,133212 +133213,133213 +125626,133213 +133214,133214 +125627,133214 +133215,133215 +125628,133215 +133216,133216 +125629,133216 +133217,133217 +125630,133217 +133218,133218 +125631,133218 +133219,133219 +125632,133219 +5357,133219 +133220,133220 +125633,133220 +223446,133220 +133221,133221 +125634,133221 +20095,133221 +133222,133222 +125635,133222 +133223,133223 +125636,133223 +133224,133224 +125637,133224 +122601,133224 +133225,133225 +125638,133225 +133226,133226 +125639,133226 +133227,133227 +125640,133227 +133228,133228 +125641,133228 +133229,133229 +125642,133229 +133230,133230 +125643,133230 +122607,133230 +133231,133231 +125644,133231 +133232,133232 +125645,133232 +67127,133232 +133233,133233 +125646,133233 +133234,133234 +125647,133234 +133235,133235 +125648,133235 +133236,133236 +125649,133236 +133237,133237 +125650,133237 +133238,133238 +125651,133238 +133239,133239 +125652,133239 +133240,133240 +125653,133240 +133241,133241 +125654,133241 +133242,133242 +125655,133242 +133243,133243 +125656,133243 +133244,133244 +125657,133244 +133245,133245 +125658,133245 +133246,133246 +125659,133246 +133247,133247 +125660,133247 +133542,133542 +125955,133542 +133543,133543 +125956,133543 +133544,133544 +125957,133544 +102651,133544 +133545,133545 +125958,133545 +141422,133545 +133546,133546 +125959,133546 +133547,133547 +125960,133547 +133548,133548 +125961,133548 +133549,133549 +125962,133549 +5611,133549 +133550,133550 +125963,133550 +133551,133551 +125964,133551 +133552,133552 +125965,133552 +233374,133552 +133553,133553 +125966,133553 +133554,133554 +125967,133554 +133555,133555 +125968,133555 +133556,133556 +125969,133556 +133557,133557 +125970,133557 +133558,133558 +125971,133558 +133559,133559 +125972,133559 +133560,133560 +125973,133560 +133561,133561 +125974,133561 +133562,133562 +125975,133562 +133563,133563 +125976,133563 +133564,133564 +125977,133564 +133565,133565 +125978,133565 +133566,133566 +125979,133566 +133567,133567 +125980,133567 +133568,133568 +125981,133568 +133569,133569 +125982,133569 +133570,133570 +125983,133570 +133571,133571 +125984,133571 +133572,133572 +125985,133572 +133573,133573 +125986,133573 +133574,133574 +125987,133574 +133575,133575 +125988,133575 +133576,133576 +125989,133576 +133577,133577 +125990,133577 +133578,133578 +125991,133578 +133579,133579 +125992,133579 +133672,133672 +126085,133672 +133673,133673 +126086,133673 +133674,133674 +126087,133674 +133675,133675 +126088,133675 +133676,133676 +126089,133676 +133677,133677 +126090,133677 +133678,133678 +126091,133678 +133679,133679 +126092,133679 +133680,133680 +126093,133680 +133681,133681 +126094,133681 +133682,133682 +126095,133682 +133683,133683 +126096,133683 +133684,133684 +126097,133684 +133685,133685 +126098,133685 +133686,133686 +126099,133686 +133687,133687 +126100,133687 +133688,133688 +126101,133688 +133689,133689 +126102,133689 +133690,133690 +126103,133690 +133691,133691 +126104,133691 +133692,133692 +126105,133692 +133693,133693 +126106,133693 +133694,133694 +126107,133694 +133695,133695 +126108,133695 +133696,133696 +126109,133696 +133697,133697 +126110,133697 +133698,133698 +126111,133698 +136108,133699 +133699,133699 +128521,133699 +126112,133699 +133700,133700 +126113,133700 +133701,133701 +126114,133701 +133702,133702 +126115,133702 +133703,133703 +126116,133703 +133704,133704 +126117,133704 +133705,133705 +126118,133705 +133706,133706 +126119,133706 +133707,133707 +126120,133707 +133708,133708 +126121,133708 +133709,133709 +126122,133709 +133710,133710 +126123,133710 +133711,133711 +126124,133711 +133712,133712 +126125,133712 +133713,133713 +126126,133713 +120968,133713 +133714,133714 +126127,133714 +73043,133714 +133715,133715 +126128,133715 +133716,133716 +126129,133716 +133717,133717 +126130,133717 +133718,133718 +126131,133718 +133719,133719 +126132,133719 +133720,133720 +126133,133720 +133721,133721 +126134,133721 +133722,133722 +126135,133722 +133723,133723 +126136,133723 +139696,133723 +133724,133724 +126137,133724 +133725,133725 +126138,133725 +133726,133726 +126139,133726 +133727,133727 +126140,133727 +133728,133728 +126141,133728 +133729,133729 +126142,133729 +133730,133730 +126143,133730 +133731,133731 +126144,133731 +133732,133732 +126145,133732 +139290,133732 +133733,133733 +126146,133733 +139707,133734 +133734,133734 +126147,133734 +17208,133734 +133735,133735 +126148,133735 +133736,133736 +126149,133736 +133737,133737 +126150,133737 +133738,133738 +126151,133738 +133739,133739 +126152,133739 +133740,133740 +126153,133740 +133741,133741 +126154,133741 +133742,133742 +126155,133742 +133743,133743 +126156,133743 +133744,133744 +126157,133744 +133745,133745 +126158,133745 +152221,133745 +133746,133746 +126159,133746 +133747,133747 +126160,133747 +133748,133748 +126161,133748 +133749,133749 +126162,133749 +133750,133750 +126163,133750 +77977,133750 +133751,133751 +126164,133751 +133752,133752 +126165,133752 +133753,133753 +126166,133753 +219814,133753 +133754,133754 +126167,133754 +133755,133755 +126168,133755 +133756,133756 +126169,133756 +133757,133757 +126170,133757 +133758,133758 +126171,133758 +229578,133758 +133759,133759 +126172,133759 +133760,133760 +126173,133760 +133761,133761 +126174,133761 +133762,133762 +126175,133762 +133763,133763 +126176,133763 +133764,133764 +126177,133764 +133982,133982 +126395,133982 +133983,133983 +126396,133983 +133984,133984 +126397,133984 +133985,133985 +126398,133985 +133986,133986 +126399,133986 +133987,133987 +126400,133987 +133988,133988 +126401,133988 +133989,133989 +126402,133989 +133990,133990 +126403,133990 +133991,133991 +126404,133991 +133992,133992 +126405,133992 +140696,133993 +133993,133993 +126406,133993 +28943,133993 +133994,133994 +126407,133994 +133995,133995 +126408,133995 +133996,133996 +126409,133996 +133997,133997 +126410,133997 +133998,133998 +126411,133998 +133999,133999 +126412,133999 +134000,134000 +126413,134000 +134001,134001 +126414,134001 +134002,134002 +126415,134002 +134003,134003 +126416,134003 +134004,134004 +126417,134004 +134005,134005 +126418,134005 +134006,134006 +126419,134006 +134007,134007 +126420,134007 +134008,134008 +126421,134008 +209512,134008 +134009,134009 +126422,134009 +134010,134010 +126423,134010 +134011,134011 +126424,134011 +134012,134012 +126425,134012 +134013,134013 +126426,134013 +134014,134014 +126427,134014 +134015,134015 +126428,134015 +134016,134016 +126429,134016 +134017,134017 +126430,134017 +134018,134018 +126431,134018 +134019,134019 +126432,134019 +134020,134020 +126433,134020 +134021,134021 +126434,134021 +134022,134022 +126435,134022 +134023,134023 +126436,134023 +134024,134024 +126437,134024 +134025,134025 +126438,134025 +134026,134026 +126439,134026 +134027,134027 +126440,134027 +134028,134028 +126441,134028 +134029,134029 +126442,134029 +134030,134030 +126443,134030 +134031,134031 +126444,134031 +134032,134032 +126445,134032 +134033,134033 +126446,134033 +134034,134034 +126447,134034 +134035,134035 +126448,134035 +134036,134036 +126449,134036 +134037,134037 +126450,134037 +134038,134038 +126451,134038 +134039,134039 +126452,134039 +134040,134040 +126453,134040 +134041,134041 +126454,134041 +134042,134042 +126455,134042 +134043,134043 +126456,134043 +134044,134044 +126457,134044 +134045,134045 +126458,134045 +146448,134045 +134046,134046 +126459,134046 +134047,134047 +126460,134047 +207848,134047 +207618,134047 +134048,134048 +126461,134048 +134049,134049 +126462,134049 +134050,134050 +126463,134050 +134051,134051 +126464,134051 +132618,134051 +125031,134051 +208425,134051 +134052,134052 +126465,134052 +134053,134053 +126466,134053 +134054,134054 +126467,134054 +134055,134055 +126468,134055 +134056,134056 +126469,134056 +134057,134057 +126470,134057 +134058,134058 +126471,134058 +134059,134059 +126472,134059 +134060,134060 +126473,134060 +134061,134061 +126474,134061 +24409,134061 +134062,134062 +126475,134062 +134063,134063 +126476,134063 +134064,134064 +126477,134064 +134065,134065 +126478,134065 +134066,134066 +126479,134066 +134067,134067 +126480,134067 +134068,134068 +126481,134068 +151591,134068 +134069,134069 +126482,134069 +134070,134070 +126483,134070 +134071,134071 +126484,134071 +134072,134072 +126485,134072 +134073,134073 +126486,134073 +231830,134073 +134074,134074 +126487,134074 +134075,134075 +126488,134075 +134076,134076 +126489,134076 +63706,134076 +134077,134077 +126490,134077 +134078,134078 +126491,134078 +134079,134079 +126492,134079 +134080,134080 +126493,134080 +134081,134081 +126494,134081 +134082,134082 +126495,134082 +134083,134083 +126496,134083 +134084,134084 +126497,134084 +134085,134085 +126498,134085 +134086,134086 +126499,134086 +80377,134086 +202703,134086 +134087,134087 +126500,134087 +134088,134088 +126501,134088 +134089,134089 +126502,134089 +76071,134089 +33216,134089 +134090,134090 +126503,134090 +134091,134091 +126504,134091 +134092,134092 +126505,134092 +134093,134093 +126506,134093 +134094,134094 +126507,134094 +134095,134095 +126508,134095 +120274,134095 +134096,134096 +126509,134096 +134097,134097 +126510,134097 +134098,134098 +126511,134098 +134099,134099 +126512,134099 +134100,134100 +126513,134100 +134101,134101 +126514,134101 +134102,134102 +126515,134102 +134103,134103 +126516,134103 +134104,134104 +126517,134104 +134105,134105 +126518,134105 +134106,134106 +126519,134106 +134107,134107 +126520,134107 +134108,134108 +126521,134108 +134109,134109 +126522,134109 +134110,134110 +126523,134110 +134111,134111 +126524,134111 +134112,134112 +126525,134112 +134113,134113 +126526,134113 +134114,134114 +126527,134114 +134115,134115 +126528,134115 +134116,134116 +126529,134116 +185827,134116 +134117,134117 +126530,134117 +134118,134118 +126531,134118 +134119,134119 +126532,134119 +134120,134120 +126533,134120 +134121,134121 +126534,134121 +134122,134122 +126535,134122 +134123,134123 +126536,134123 +134124,134124 +126537,134124 +134125,134125 +126538,134125 +134126,134126 +126539,134126 +134127,134127 +126540,134127 +104316,134127 +60136,134127 +134128,134128 +126541,134128 +140019,134128 +134129,134129 +126542,134129 +134130,134130 +126543,134130 +134131,134131 +126544,134131 +134132,134132 +126545,134132 +134133,134133 +126546,134133 +134134,134134 +126547,134134 +134135,134135 +126548,134135 +134136,134136 +126549,134136 +134137,134137 +126550,134137 +113493,134137 +134138,134138 +126551,134138 +134139,134139 +126552,134139 +134140,134140 +126553,134140 +134141,134141 +126554,134141 +134142,134142 +126555,134142 +134143,134143 +126556,134143 +134144,134144 +126557,134144 +134145,134145 +126558,134145 +134146,134146 +126559,134146 +134147,134147 +126560,134147 +134148,134148 +126561,134148 +134149,134149 +126562,134149 +134150,134150 +126563,134150 +134151,134151 +126564,134151 +134152,134152 +126565,134152 +134153,134153 +126566,134153 +134154,134154 +126567,134154 +134155,134155 +126568,134155 +134156,134156 +126569,134156 +134157,134157 +126570,134157 +134158,134158 +126571,134158 +134159,134159 +126572,134159 +202784,134159 +134160,134160 +126573,134160 +134161,134161 +126574,134161 +134162,134162 +126575,134162 +134163,134163 +126576,134163 +134164,134164 +126577,134164 +135491,134164 +127904,134164 +134165,134165 +126578,134165 +134166,134166 +126579,134166 +134167,134167 +126580,134167 +134168,134168 +126581,134168 +134169,134169 +126582,134169 +90710,134169 +161750,134170 +134170,134170 +126583,134170 +134171,134171 +126584,134171 +134172,134172 +126585,134172 +134173,134173 +126586,134173 +134174,134174 +126587,134174 +134175,134175 +126588,134175 +134176,134176 +126589,134176 +134177,134177 +126590,134177 +134178,134178 +126591,134178 +134179,134179 +126592,134179 +134180,134180 +126593,134180 +134181,134181 +126594,134181 +187084,134181 +134182,134182 +126595,134182 +134183,134183 +126596,134183 +134184,134184 +126597,134184 +134185,134185 +126598,134185 +134186,134186 +126599,134186 +5222,134186 +134187,134187 +126600,134187 +134188,134188 +126601,134188 +3999,134188 +134189,134189 +126602,134189 +134190,134190 +126603,134190 +134191,134191 +126604,134191 +108669,134191 +134192,134192 +126605,134192 +134193,134193 +126606,134193 +134194,134194 +126607,134194 +134195,134195 +126608,134195 +134196,134196 +126609,134196 +180963,134196 +114755,134196 +134197,134197 +126610,134197 +134198,134198 +126611,134198 +134199,134199 +126612,134199 +134200,134200 +126613,134200 +56262,134200 +48158,134200 +134201,134201 +126614,134201 +134202,134202 +126615,134202 +134203,134203 +126616,134203 +134204,134204 +126617,134204 +134205,134205 +126618,134205 +70205,134205 +134206,134206 +126619,134206 +57731,134206 +134207,134207 +126620,134207 +134208,134208 +126621,134208 +142692,134208 +134209,134209 +126622,134209 +134210,134210 +126623,134210 +134211,134211 +126624,134211 +134212,134212 +126625,134212 +134213,134213 +126626,134213 +134214,134214 +126627,134214 +134215,134215 +126628,134215 +134216,134216 +126629,134216 +134217,134217 +126630,134217 +159458,134217 +134218,134218 +126631,134218 +134219,134219 +126632,134219 +134220,134220 +126633,134220 +134221,134221 +126634,134221 +134222,134222 +126635,134222 +134223,134223 +126636,134223 +134224,134224 +126637,134224 +134225,134225 +126638,134225 +111201,134225 +134226,134226 +126639,134226 +134227,134227 +126640,134227 +247217,134227 +134228,134228 +126641,134228 +134229,134229 +126642,134229 +134230,134230 +126643,134230 +134231,134231 +126644,134231 +134232,134232 +126645,134232 +134233,134233 +126646,134233 +134234,134234 +126647,134234 +134235,134235 +126648,134235 +134236,134236 +126649,134236 +134237,134237 +126650,134237 +134238,134238 +126651,134238 +134239,134239 +126652,134239 +134240,134240 +126653,134240 +134241,134241 +126654,134241 +134242,134242 +126655,134242 +134243,134243 +126656,134243 +204185,134243 +134244,134244 +126657,134244 +134245,134245 +126658,134245 +190925,134245 +134246,134246 +126659,134246 +134247,134247 +126660,134247 +135078,134247 +127491,134247 +134248,134248 +126661,134248 +134249,134249 +126662,134249 +232917,134249 +134250,134250 +126663,134250 +134251,134251 +126664,134251 +134252,134252 +126665,134252 +211554,134252 +134253,134253 +126666,134253 +134254,134254 +126667,134254 +217254,134254 +233749,134255 +134255,134255 +126668,134255 +134256,134256 +126669,134256 +134257,134257 +126670,134257 +134258,134258 +126671,134258 +134259,134259 +126672,134259 +134260,134260 +126673,134260 +134261,134261 +126674,134261 +134262,134262 +126675,134262 +134263,134263 +126676,134263 +134264,134264 +126677,134264 +134265,134265 +126678,134265 +190981,134265 +134266,134266 +126679,134266 +134267,134267 +126680,134267 +134268,134268 +126681,134268 +134269,134269 +126682,134269 +134270,134270 +126683,134270 +134271,134271 +126684,134271 +134272,134272 +126685,134272 +134273,134273 +126686,134273 +134274,134274 +126687,134274 +134275,134275 +126688,134275 +134276,134276 +126689,134276 +134277,134277 +126690,134277 +23673,134277 +134278,134278 +126691,134278 +134279,134279 +126692,134279 +134280,134280 +126693,134280 +134281,134281 +126694,134281 +134282,134282 +126695,134282 +134283,134283 +126696,134283 +134284,134284 +126697,134284 +134285,134285 +126698,134285 +203806,134285 +134286,134286 +126699,134286 +134287,134287 +126700,134287 +134288,134288 +126701,134288 +134289,134289 +126702,134289 +134290,134290 +126703,134290 +134291,134291 +126704,134291 +134292,134292 +126705,134292 +134293,134293 +126706,134293 +134294,134294 +126707,134294 +134295,134295 +126708,134295 +134296,134296 +126709,134296 +134297,134297 +126710,134297 +134298,134298 +126711,134298 +134299,134299 +126712,134299 +134300,134300 +126713,134300 +134301,134301 +126714,134301 +134302,134302 +126715,134302 +134303,134303 +126716,134303 +134304,134304 +126717,134304 +111752,134304 +134305,134305 +126718,134305 +134306,134306 +126719,134306 +36191,134306 +134307,134307 +126720,134307 +134308,134308 +126721,134308 +134309,134309 +126722,134309 +134310,134310 +126723,134310 +134311,134311 +126724,134311 +134312,134312 +126725,134312 +134313,134313 +126726,134313 +134314,134314 +126727,134314 +134315,134315 +126728,134315 +134316,134316 +126729,134316 +134317,134317 +126730,134317 +134318,134318 +126731,134318 +134319,134319 +126732,134319 +134320,134320 +126733,134320 +134321,134321 +126734,134321 +134322,134322 +126735,134322 +134323,134323 +126736,134323 +134324,134324 +126737,134324 +134325,134325 +126738,134325 +134326,134326 +126739,134326 +134327,134327 +126740,134327 +134328,134328 +126741,134328 +65743,134328 +134329,134329 +126742,134329 +134330,134330 +126743,134330 +134331,134331 +126744,134331 +134332,134332 +126745,134332 +134333,134333 +126746,134333 +134334,134334 +126747,134334 +134335,134335 +126748,134335 +134336,134336 +126749,134336 +134337,134337 +126750,134337 +134338,134338 +126751,134338 +134339,134339 +126752,134339 +134340,134340 +126753,134340 +134341,134341 +126754,134341 +134342,134342 +126755,134342 +134343,134343 +126756,134343 +134344,134344 +126757,134344 +134345,134345 +126758,134345 +134346,134346 +126759,134346 +134347,134347 +126760,134347 +146587,134347 +134385,134385 +126798,134385 +135061,135061 +127474,135061 +135062,135062 +127475,135062 +135063,135063 +127476,135063 +135064,135064 +127477,135064 +135065,135065 +127478,135065 +135066,135066 +127479,135066 +135067,135067 +127480,135067 +135068,135068 +127481,135068 +135069,135069 +127482,135069 +135070,135070 +127483,135070 +2449,135070 +135071,135071 +127484,135071 +135072,135072 +127485,135072 +111974,135072 +135073,135073 +127486,135073 +135074,135074 +127487,135074 +135075,135075 +127488,135075 +135076,135076 +127489,135076 +135077,135077 +127490,135077 +135078,135078 +127491,135078 +134247,135078 +126660,135078 +135079,135079 +127492,135079 +135080,135080 +127493,135080 +135081,135081 +127494,135081 +135082,135082 +127495,135082 +135083,135083 +127496,135083 +135084,135084 +127497,135084 +135085,135085 +127498,135085 +135086,135086 +127499,135086 +135087,135087 +127500,135087 +135088,135088 +127501,135088 +135089,135089 +127502,135089 +135090,135090 +127503,135090 +135091,135091 +127504,135091 +135092,135092 +127505,135092 +135093,135093 +127506,135093 +29522,135093 +135094,135094 +127507,135094 +135095,135095 +127508,135095 +135096,135096 +127509,135096 +135097,135097 +127510,135097 +135098,135098 +127511,135098 +194361,135098 +135099,135099 +127512,135099 +135100,135100 +127513,135100 +135101,135101 +127514,135101 +135102,135102 +127515,135102 +135103,135103 +127516,135103 +135104,135104 +127517,135104 +135105,135105 +127518,135105 +135106,135106 +127519,135106 +135107,135107 +127520,135107 +135108,135108 +127521,135108 +135109,135109 +127522,135109 +135110,135110 +127523,135110 +135111,135111 +127524,135111 +135112,135112 +127525,135112 +135113,135113 +127526,135113 +10488,135113 +135114,135114 +127527,135114 +135115,135115 +127528,135115 +135116,135116 +127529,135116 +135117,135117 +127530,135117 +135118,135118 +127531,135118 +135119,135119 +127532,135119 +135120,135120 +127533,135120 +135121,135121 +127534,135121 +135122,135122 +127535,135122 +135123,135123 +127536,135123 +135124,135124 +127537,135124 +135125,135125 +127538,135125 +135126,135126 +127539,135126 +135127,135127 +127540,135127 +135128,135128 +127541,135128 +5555,135128 +135129,135129 +127542,135129 +135130,135130 +127543,135130 +135131,135131 +127544,135131 +135132,135132 +127545,135132 +135200,135200 +127613,135200 +135201,135201 +127614,135201 +135202,135202 +127615,135202 +135203,135203 +127616,135203 +135204,135204 +127617,135204 +135205,135205 +127618,135205 +135206,135206 +127619,135206 +135207,135207 +127620,135207 +135208,135208 +127621,135208 +135209,135209 +127622,135209 +135210,135210 +127623,135210 +135211,135211 +127624,135211 +135212,135212 +127625,135212 +135213,135213 +127626,135213 +135214,135214 +127627,135214 +135215,135215 +127628,135215 +135216,135216 +127629,135216 +135217,135217 +127630,135217 +135218,135218 +127631,135218 +135219,135219 +127632,135219 +135220,135220 +127633,135220 +135221,135221 +127634,135221 +135222,135222 +127635,135222 +135223,135223 +127636,135223 +135224,135224 +127637,135224 +135225,135225 +127638,135225 +135226,135226 +127639,135226 +50026,135226 +41833,135226 +135227,135227 +127640,135227 +135228,135228 +127641,135228 +135229,135229 +127642,135229 +135230,135230 +127643,135230 +135231,135231 +127644,135231 +135232,135232 +127645,135232 +35645,135232 +135233,135233 +127646,135233 +135234,135234 +127647,135234 +135235,135235 +127648,135235 +135236,135236 +127649,135236 +135237,135237 +127650,135237 +135238,135238 +127651,135238 +135239,135239 +127652,135239 +135240,135240 +127653,135240 +135241,135241 +127654,135241 +122244,135241 +135242,135242 +127655,135242 +135243,135243 +127656,135243 +135244,135244 +127657,135244 +135245,135245 +127658,135245 +135246,135246 +127659,135246 +136062,135247 +135247,135247 +128475,135247 +127660,135247 +135248,135248 +127661,135248 +135249,135249 +127662,135249 +135250,135250 +127663,135250 +135251,135251 +127664,135251 +236058,135251 +135252,135252 +127665,135252 +135253,135253 +127666,135253 +135254,135254 +127667,135254 +135255,135255 +127668,135255 +135256,135256 +127669,135256 +135257,135257 +127670,135257 +135258,135258 +127671,135258 +135259,135259 +127672,135259 +64901,135259 +135260,135260 +127673,135260 +135261,135261 +127674,135261 +135262,135262 +127675,135262 +60758,135262 +135263,135263 +127676,135263 +236712,135263 +3537,135263 +135264,135264 +127677,135264 +135265,135265 +127678,135265 +135266,135266 +127679,135266 +135267,135267 +127680,135267 +135268,135268 +127681,135268 +55932,135268 +47828,135268 +135269,135269 +127682,135269 +135270,135270 +127683,135270 +164081,135270 +135271,135271 +127684,135271 +135272,135272 +127685,135272 +2053,135272 +135273,135273 +127686,135273 +27590,135273 +135274,135274 +127687,135274 +135275,135275 +127688,135275 +135276,135276 +127689,135276 +135277,135277 +127690,135277 +135278,135278 +127691,135278 +135279,135279 +127692,135279 +207780,135279 +9258,135279 +142561,135279 +135280,135280 +127693,135280 +135281,135281 +127694,135281 +135282,135282 +127695,135282 +135283,135283 +127696,135283 +166726,135284 +135284,135284 +127697,135284 +150799,135284 +135285,135285 +127698,135285 +243297,135285 +36066,135285 +135286,135286 +127699,135286 +135287,135287 +127700,135287 +186557,135287 +109549,135287 +135288,135288 +127701,135288 +135289,135289 +127702,135289 +135290,135290 +127703,135290 +135291,135291 +127704,135291 +135292,135292 +127705,135292 +222198,135293 +135293,135293 +127706,135293 +135294,135294 +127707,135294 +135295,135295 +127708,135295 +135296,135296 +127709,135296 +135297,135297 +127710,135297 +135298,135298 +127711,135298 +135416,135416 +127829,135416 +135417,135417 +127830,135417 +135418,135418 +127831,135418 +135419,135419 +127832,135419 +135420,135420 +127833,135420 +135421,135421 +127834,135421 +135422,135422 +127835,135422 +135423,135423 +127836,135423 +135424,135424 +127837,135424 +89224,135424 +135425,135425 +127838,135425 +93859,135425 +135426,135426 +127839,135426 +135427,135427 +127840,135427 +135428,135428 +127841,135428 +135429,135429 +127842,135429 +135430,135430 +127843,135430 +135431,135431 +127844,135431 +135432,135432 +127845,135432 +135433,135433 +127846,135433 +176537,135433 +170573,135433 +135434,135434 +127847,135434 +135435,135435 +127848,135435 +135436,135436 +127849,135436 +135437,135437 +127850,135437 +135438,135438 +127851,135438 +135439,135439 +127852,135439 +135440,135440 +127853,135440 +135441,135441 +127854,135441 +135442,135442 +127855,135442 +135443,135443 +127856,135443 +135444,135444 +127857,135444 +135445,135445 +127858,135445 +135446,135446 +127859,135446 +135447,135447 +127860,135447 +135448,135448 +127861,135448 +135449,135449 +127862,135449 +135450,135450 +127863,135450 +135451,135451 +127864,135451 +135452,135452 +127865,135452 +135453,135453 +127866,135453 +135454,135454 +127867,135454 +135455,135455 +127868,135455 +135456,135456 +127869,135456 +135457,135457 +127870,135457 +135458,135458 +127871,135458 +135459,135459 +127872,135459 +135460,135460 +127873,135460 +203684,135460 +135461,135461 +127874,135461 +135462,135462 +127875,135462 +135463,135463 +127876,135463 +135464,135464 +127877,135464 +135465,135465 +127878,135465 +135466,135466 +127879,135466 +135467,135467 +127880,135467 +135468,135468 +127881,135468 +135469,135469 +127882,135469 +135470,135470 +127883,135470 +135471,135471 +127884,135471 +135472,135472 +127885,135472 +135473,135473 +127886,135473 +135474,135474 +127887,135474 +135475,135475 +127888,135475 +135476,135476 +127889,135476 +77583,135476 +135477,135477 +127890,135477 +135478,135478 +127891,135478 +135479,135479 +127892,135479 +135480,135480 +127893,135480 +135481,135481 +127894,135481 +135482,135482 +127895,135482 +135483,135483 +127896,135483 +135484,135484 +127897,135484 +36350,135484 +135485,135485 +127898,135485 +135486,135486 +127899,135486 +135487,135487 +127900,135487 +135488,135488 +127901,135488 +135489,135489 +127902,135489 +135490,135490 +127903,135490 +135491,135491 +127904,135491 +134164,135491 +126577,135491 +135492,135492 +127905,135492 +135493,135493 +127906,135493 +135494,135494 +127907,135494 +135495,135495 +127908,135495 +135496,135496 +127909,135496 +135497,135497 +127910,135497 +135498,135498 +127911,135498 +135499,135499 +127912,135499 +135500,135500 +127913,135500 +135501,135501 +127914,135501 +135502,135502 +127915,135502 +135503,135503 +127916,135503 +135504,135504 +127917,135504 +33258,135504 +135505,135505 +127918,135505 +135506,135506 +127919,135506 +135507,135507 +127920,135507 +135508,135508 +127921,135508 +135509,135509 +127922,135509 +135510,135510 +127923,135510 +244127,135510 +135511,135511 +127924,135511 +135512,135512 +127925,135512 +135513,135513 +127926,135513 +135514,135514 +127927,135514 +15726,135514 +135515,135515 +127928,135515 +135516,135516 +127929,135516 +135517,135517 +127930,135517 +135518,135518 +127931,135518 +150114,135519 +135519,135519 +127932,135519 +135520,135520 +127933,135520 +135521,135521 +127934,135521 +135584,135584 +127997,135584 +135585,135585 +127998,135585 +135586,135586 +127999,135586 +135587,135587 +128000,135587 +135588,135588 +128001,135588 +135589,135589 +128002,135589 +135590,135590 +128003,135590 +135591,135591 +128004,135591 +135592,135592 +128005,135592 +135593,135593 +128006,135593 +135594,135594 +128007,135594 +135595,135595 +128008,135595 +135596,135596 +128009,135596 +135597,135597 +128010,135597 +135598,135598 +128011,135598 +135676,135676 +128089,135676 +135677,135677 +128090,135677 +56181,135677 +48077,135677 +135678,135678 +128091,135678 +135679,135679 +128092,135679 +135680,135680 +128093,135680 +135681,135681 +128094,135681 +135682,135682 +128095,135682 +135683,135683 +128096,135683 +135684,135684 +128097,135684 +135685,135685 +128098,135685 +135686,135686 +128099,135686 +135687,135687 +128100,135687 +135688,135688 +128101,135688 +135689,135689 +128102,135689 +135690,135690 +128103,135690 +135743,135743 +128156,135743 +135744,135744 +128157,135744 +135745,135745 +128158,135745 +135746,135746 +128159,135746 +135747,135747 +128160,135747 +135748,135748 +128161,135748 +135749,135749 +128162,135749 +148825,135749 +135750,135750 +128163,135750 +135751,135751 +128164,135751 +135752,135752 +128165,135752 +236410,135752 +135753,135753 +128166,135753 +135754,135754 +128167,135754 +135755,135755 +128168,135755 +135756,135756 +128169,135756 +109828,135756 +155936,135756 +135757,135757 +128170,135757 +66813,135757 +135758,135758 +128171,135758 +135759,135759 +128172,135759 +135760,135760 +128173,135760 +135761,135761 +128174,135761 +135762,135762 +128175,135762 +135763,135763 +128176,135763 +135764,135764 +128177,135764 +135765,135765 +128178,135765 +36751,135765 +135766,135766 +128179,135766 +135767,135767 +128180,135767 +227849,135767 +135768,135768 +128181,135768 +135769,135769 +128182,135769 +135770,135770 +128183,135770 +135771,135771 +128184,135771 +135772,135772 +128185,135772 +135773,135773 +128186,135773 +135774,135774 +128187,135774 +135775,135775 +128188,135775 +135776,135776 +128189,135776 +135777,135777 +128190,135777 +135778,135778 +128191,135778 +135779,135779 +128192,135779 +135780,135780 +128193,135780 +135781,135781 +128194,135781 +135782,135782 +128195,135782 +135783,135783 +128196,135783 +233082,135783 +135784,135784 +128197,135784 +135785,135785 +128198,135785 +135786,135786 +128199,135786 +135787,135787 +128200,135787 +135788,135788 +128201,135788 +135789,135789 +128202,135789 +135790,135790 +128203,135790 +135791,135791 +128204,135791 +135792,135792 +128205,135792 +135793,135793 +128206,135793 +135794,135794 +128207,135794 +135795,135795 +128208,135795 +135796,135796 +128209,135796 +135797,135797 +128210,135797 +135798,135798 +128211,135798 +135799,135799 +128212,135799 +135800,135800 +128213,135800 +135801,135801 +128214,135801 +135802,135802 +128215,135802 +135803,135803 +128216,135803 +135804,135804 +128217,135804 +135805,135805 +128218,135805 +135806,135806 +128219,135806 +135807,135807 +128220,135807 +135808,135808 +128221,135808 +135809,135809 +128222,135809 +135810,135810 +128223,135810 +135811,135811 +128224,135811 +135812,135812 +128225,135812 +135813,135813 +128226,135813 +135814,135814 +128227,135814 +135815,135815 +128228,135815 +203620,135815 +135816,135816 +128229,135816 +135817,135817 +128230,135817 +135818,135818 +128231,135818 +135819,135819 +128232,135819 +135820,135820 +128233,135820 +137958,135820 +130371,135820 +135821,135821 +128234,135821 +135822,135822 +128235,135822 +135823,135823 +128236,135823 +135824,135824 +128237,135824 +135825,135825 +128238,135825 +135826,135826 +128239,135826 +135827,135827 +128240,135827 +135828,135828 +128241,135828 +135829,135829 +128242,135829 +135830,135830 +128243,135830 +135831,135831 +128244,135831 +135832,135832 +128245,135832 +135833,135833 +128246,135833 +135834,135834 +128247,135834 +84835,135834 +135835,135835 +128248,135835 +135836,135836 +128249,135836 +135837,135837 +128250,135837 +135838,135838 +128251,135838 +135839,135839 +128252,135839 +135840,135840 +128253,135840 +135841,135841 +128254,135841 +135842,135842 +128255,135842 +135843,135843 +128256,135843 +135844,135844 +128257,135844 +135845,135845 +128258,135845 +135846,135846 +128259,135846 +135847,135847 +128260,135847 +135848,135848 +128261,135848 +135849,135849 +128262,135849 +135850,135850 +128263,135850 +135851,135851 +128264,135851 +135852,135852 +128265,135852 +135853,135853 +128266,135853 +135854,135854 +128267,135854 +135855,135855 +128268,135855 +135856,135856 +128269,135856 +135857,135857 +128270,135857 +135913,135913 +128326,135913 +135914,135914 +128327,135914 +135915,135915 +128328,135915 +15432,135915 +135916,135916 +128329,135916 +135917,135917 +128330,135917 +135918,135918 +128331,135918 +135919,135919 +128332,135919 +135920,135920 +128333,135920 +135921,135921 +128334,135921 +135922,135922 +128335,135922 +135923,135923 +128336,135923 +135924,135924 +128337,135924 +135925,135925 +128338,135925 +135926,135926 +128339,135926 +176675,135926 +170711,135926 +135927,135927 +128340,135927 +135928,135928 +128341,135928 +3041,135928 +135929,135929 +128342,135929 +135930,135930 +128343,135930 +135931,135931 +128344,135931 +240519,135931 +135932,135932 +128345,135932 +135933,135933 +128346,135933 +135934,135934 +128347,135934 +135935,135935 +128348,135935 +135936,135936 +128349,135936 +135937,135937 +128350,135937 +135938,135938 +128351,135938 +135939,135939 +128352,135939 +135940,135940 +128353,135940 +135941,135941 +128354,135941 +135942,135942 +128355,135942 +135943,135943 +128356,135943 +135944,135944 +128357,135944 +135945,135945 +128358,135945 +135946,135946 +128359,135946 +135947,135947 +128360,135947 +135948,135948 +128361,135948 +135949,135949 +128362,135949 +220375,135949 +5773,135949 +135950,135950 +128363,135950 +135951,135951 +128364,135951 +135952,135952 +128365,135952 +135953,135953 +128366,135953 +80804,135953 +135954,135954 +128367,135954 +28951,135954 +135955,135955 +128368,135955 +124087,135955 +135956,135956 +128369,135956 +135957,135957 +128370,135957 +136025,136025 +128438,136025 +136026,136026 +128439,136026 +136027,136027 +128440,136027 +136028,136028 +128441,136028 +136029,136029 +128442,136029 +136030,136030 +128443,136030 +136031,136031 +128444,136031 +136032,136032 +128445,136032 +136033,136033 +128446,136033 +136034,136034 +128447,136034 +136035,136035 +128448,136035 +136036,136036 +128449,136036 +136037,136037 +128450,136037 +217041,136038 +136038,136038 +128451,136038 +68193,136038 +136039,136039 +128452,136039 +136040,136040 +128453,136040 +136041,136041 +128454,136041 +136042,136042 +128455,136042 +136043,136043 +128456,136043 +136044,136044 +128457,136044 +74157,136044 +136045,136045 +128458,136045 +136046,136046 +128459,136046 +136047,136047 +128460,136047 +136048,136048 +128461,136048 +684,136048 +136049,136049 +128462,136049 +136050,136050 +128463,136050 +136051,136051 +128464,136051 +136052,136052 +128465,136052 +136053,136053 +128466,136053 +136054,136054 +128467,136054 +136055,136055 +128468,136055 +136056,136056 +128469,136056 +136057,136057 +128470,136057 +136058,136058 +128471,136058 +136059,136059 +128472,136059 +136060,136060 +128473,136060 +136061,136061 +128474,136061 +136062,136062 +135247,136062 +128475,136062 +127660,136062 +136063,136063 +128476,136063 +136064,136064 +128477,136064 +136065,136065 +128478,136065 +136066,136066 +128479,136066 +136067,136067 +128480,136067 +136068,136068 +128481,136068 +64119,136068 +136069,136069 +128482,136069 +136070,136070 +128483,136070 +136071,136071 +128484,136071 +136072,136072 +128485,136072 +156494,136072 +136073,136073 +128486,136073 +136074,136074 +128487,136074 +136075,136075 +128488,136075 +136076,136076 +128489,136076 +136077,136077 +128490,136077 +136078,136078 +128491,136078 +136079,136079 +128492,136079 +136080,136080 +128493,136080 +150798,136080 +136081,136081 +128494,136081 +136082,136082 +128495,136082 +136083,136083 +128496,136083 +136084,136084 +128497,136084 +136085,136085 +128498,136085 +136086,136086 +128499,136086 +136087,136087 +128500,136087 +136088,136088 +128501,136088 +136089,136089 +128502,136089 +136090,136090 +128503,136090 +136091,136091 +128504,136091 +136092,136092 +128505,136092 +136093,136093 +128506,136093 +136094,136094 +128507,136094 +136095,136095 +128508,136095 +136096,136096 +128509,136096 +136097,136097 +128510,136097 +136098,136098 +128511,136098 +136099,136099 +128512,136099 +136100,136100 +128513,136100 +136101,136101 +128514,136101 +136102,136102 +128515,136102 +136103,136103 +128516,136103 +136104,136104 +128517,136104 +136105,136105 +128518,136105 +136106,136106 +128519,136106 +136107,136107 +128520,136107 +136108,136108 +133699,136108 +128521,136108 +126112,136108 +136109,136109 +128522,136109 +136110,136110 +128523,136110 +136111,136111 +128524,136111 +136112,136112 +128525,136112 +136113,136113 +128526,136113 +136114,136114 +128527,136114 +136115,136115 +128528,136115 +136116,136116 +128529,136116 +199190,136116 +136117,136117 +128530,136117 +136118,136118 +128531,136118 +136119,136119 +128532,136119 +136120,136120 +128533,136120 +136121,136121 +128534,136121 +136122,136122 +128535,136122 +136123,136123 +128536,136123 +191558,136123 +136124,136124 +128537,136124 +136125,136125 +128538,136125 +136126,136126 +128539,136126 +136127,136127 +128540,136127 +136128,136128 +128541,136128 +136129,136129 +128542,136129 +136130,136130 +128543,136130 +136131,136131 +128544,136131 +136132,136132 +128545,136132 +136133,136133 +128546,136133 +136134,136134 +128547,136134 +136135,136135 +128548,136135 +136136,136136 +128549,136136 +136137,136137 +128550,136137 +136138,136138 +128551,136138 +136139,136139 +128552,136139 +32333,136139 +136140,136140 +128553,136140 +136141,136141 +128554,136141 +107531,136141 +136142,136142 +128555,136142 +136143,136143 +128556,136143 +136144,136144 +128557,136144 +136145,136145 +128558,136145 +136146,136146 +128559,136146 +136147,136147 +128560,136147 +136148,136148 +128561,136148 +136149,136149 +128562,136149 +136150,136150 +128563,136150 +136151,136151 +128564,136151 +136152,136152 +128565,136152 +136153,136153 +128566,136153 +136154,136154 +128567,136154 +136155,136155 +128568,136155 +136156,136156 +128569,136156 +136157,136157 +128570,136157 +136158,136158 +128571,136158 +136159,136159 +128572,136159 +136160,136160 +128573,136160 +136161,136161 +128574,136161 +136162,136162 +128575,136162 +136163,136163 +128576,136163 +136164,136164 +128577,136164 +136165,136165 +128578,136165 +136166,136166 +128579,136166 +136167,136167 +128580,136167 +136168,136168 +128581,136168 +68021,136168 +136169,136169 +128582,136169 +136170,136170 +128583,136170 +136171,136171 +128584,136171 +136172,136172 +128585,136172 +136173,136173 +128586,136173 +136174,136174 +128587,136174 +136175,136175 +128588,136175 +136176,136176 +128589,136176 +136177,136177 +128590,136177 +136178,136178 +128591,136178 +136179,136179 +128592,136179 +136180,136180 +128593,136180 +136181,136181 +128594,136181 +136182,136182 +128595,136182 +136183,136183 +128596,136183 +136184,136184 +128597,136184 +136185,136185 +128598,136185 +136186,136186 +128599,136186 +136187,136187 +128600,136187 +136188,136188 +128601,136188 +136189,136189 +128602,136189 +136190,136190 +128603,136190 +136191,136191 +128604,136191 +136192,136192 +128605,136192 +136193,136193 +128606,136193 +136194,136194 +128607,136194 +136195,136195 +128608,136195 +136196,136196 +128609,136196 +136197,136197 +128610,136197 +136198,136198 +128611,136198 +136199,136199 +128612,136199 +136200,136200 +128613,136200 +136201,136201 +128614,136201 +136202,136202 +128615,136202 +225535,136202 +136203,136203 +128616,136203 +136204,136204 +128617,136204 +136205,136205 +128618,136205 +136206,136206 +128619,136206 +136207,136207 +128620,136207 +136208,136208 +128621,136208 +136209,136209 +128622,136209 +136210,136210 +128623,136210 +136211,136211 +128624,136211 +136981,136211 +129394,136211 +136212,136212 +128625,136212 +136213,136213 +128626,136213 +136214,136214 +128627,136214 +136215,136215 +128628,136215 +136216,136216 +128629,136216 +136217,136217 +128630,136217 +136218,136218 +128631,136218 +136219,136219 +128632,136219 +136220,136220 +128633,136220 +136221,136221 +128634,136221 +136222,136222 +128635,136222 +136223,136223 +128636,136223 +136224,136224 +128637,136224 +136225,136225 +128638,136225 +136226,136226 +128639,136226 +225987,136226 +4034,136226 +136227,136227 +128640,136227 +136228,136228 +128641,136228 +136229,136229 +128642,136229 +136230,136230 +128643,136230 +136231,136231 +128644,136231 +136232,136232 +128645,136232 +136233,136233 +128646,136233 +136234,136234 +128647,136234 +136235,136235 +128648,136235 +136236,136236 +128649,136236 +136237,136237 +128650,136237 +136238,136238 +128651,136238 +136239,136239 +128652,136239 +136240,136240 +128653,136240 +136241,136241 +128654,136241 +136242,136242 +128655,136242 +136243,136243 +128656,136243 +136244,136244 +128657,136244 +136245,136245 +128658,136245 +136246,136246 +128659,136246 +136247,136247 +128660,136247 +136248,136248 +128661,136248 +105805,136248 +136249,136249 +128662,136249 +136250,136250 +128663,136250 +82175,136250 +136251,136251 +128664,136251 +71387,136251 +207189,136251 +136252,136252 +128665,136252 +136253,136253 +128666,136253 +136254,136254 +128667,136254 +136255,136255 +128668,136255 +136256,136256 +128669,136256 +118320,136256 +136257,136257 +128670,136257 +136258,136258 +128671,136258 +30906,136258 +36750,136258 +136259,136259 +128672,136259 +136260,136260 +128673,136260 +136261,136261 +128674,136261 +136303,136303 +128716,136303 +136304,136304 +128717,136304 +136305,136305 +128718,136305 +136306,136306 +128719,136306 +136307,136307 +128720,136307 +136308,136308 +128721,136308 +178211,136308 +172247,136308 +136309,136309 +128722,136309 +136310,136310 +128723,136310 +136311,136311 +128724,136311 +111387,136311 +136312,136312 +128725,136312 +136313,136313 +128726,136313 +192236,136313 +136314,136314 +128727,136314 +136315,136315 +128728,136315 +104764,136315 +136316,136316 +128729,136316 +136317,136317 +128730,136317 +136318,136318 +128731,136318 +136319,136319 +128732,136319 +136320,136320 +128733,136320 +136321,136321 +128734,136321 +136322,136322 +128735,136322 +133079,136322 +131981,136322 +125492,136322 +124394,136322 +136323,136323 +128736,136323 +136324,136324 +128737,136324 +136325,136325 +128738,136325 +136326,136326 +128739,136326 +136327,136327 +128740,136327 +136328,136328 +128741,136328 +140639,136328 +136329,136329 +128742,136329 +136330,136330 +128743,136330 +136331,136331 +128744,136331 +136332,136332 +128745,136332 +136333,136333 +128746,136333 +136334,136334 +128747,136334 +136335,136335 +128748,136335 +136336,136336 +128749,136336 +136337,136337 +128750,136337 +136338,136338 +128751,136338 +136339,136339 +128752,136339 +136340,136340 +128753,136340 +136341,136341 +128754,136341 +136342,136342 +128755,136342 +136343,136343 +128756,136343 +136344,136344 +128757,136344 +136345,136345 +128758,136345 +136346,136346 +128759,136346 +136347,136347 +128760,136347 +136348,136348 +128761,136348 +136349,136349 +128762,136349 +136350,136350 +128763,136350 +136351,136351 +128764,136351 +136352,136352 +128765,136352 +136353,136353 +128766,136353 +136354,136354 +128767,136354 +136355,136355 +128768,136355 +136356,136356 +128769,136356 +136357,136357 +128770,136357 +136358,136358 +128771,136358 +136359,136359 +128772,136359 +136360,136360 +128773,136360 +136361,136361 +128774,136361 +136362,136362 +128775,136362 +136363,136363 +128776,136363 +136364,136364 +128777,136364 +136365,136365 +128778,136365 +136366,136366 +128779,136366 +136367,136367 +128780,136367 +136368,136368 +128781,136368 +136369,136369 +128782,136369 +136370,136370 +128783,136370 +136371,136371 +128784,136371 +136372,136372 +128785,136372 +136373,136373 +128786,136373 +136374,136374 +128787,136374 +136375,136375 +128788,136375 +136376,136376 +128789,136376 +136377,136377 +128790,136377 +136378,136378 +128791,136378 +80651,136378 +136379,136379 +128792,136379 +198414,136379 +136380,136380 +128793,136380 +217694,136380 +136381,136381 +128794,136381 +216091,136381 +136382,136382 +128795,136382 +136383,136383 +128796,136383 +136384,136384 +128797,136384 +136385,136385 +128798,136385 +136386,136386 +128799,136386 +136387,136387 +128800,136387 +136388,136388 +128801,136388 +136389,136389 +128802,136389 +136390,136390 +128803,136390 +136391,136391 +128804,136391 +136392,136392 +128805,136392 +136393,136393 +128806,136393 +136394,136394 +128807,136394 +136395,136395 +128808,136395 +136396,136396 +128809,136396 +136397,136397 +128810,136397 +136398,136398 +128811,136398 +136399,136399 +128812,136399 +136400,136400 +128813,136400 +136401,136401 +128814,136401 +243331,136401 +136402,136402 +128815,136402 +136403,136403 +128816,136403 +136404,136404 +128817,136404 +76564,136404 +136405,136405 +128818,136405 +136406,136406 +128819,136406 +136407,136407 +128820,136407 +136408,136408 +128821,136408 +136409,136409 +128822,136409 +136410,136410 +128823,136410 +136411,136411 +128824,136411 +136412,136412 +128825,136412 +136413,136413 +128826,136413 +136414,136414 +128827,136414 +136415,136415 +128828,136415 +136416,136416 +128829,136416 +136454,136454 +128867,136454 +136455,136455 +128868,136455 +136456,136456 +128869,136456 +136457,136457 +128870,136457 +136458,136458 +128871,136458 +136459,136459 +128872,136459 +136460,136460 +128873,136460 +136461,136461 +128874,136461 +136462,136462 +128875,136462 +136463,136463 +128876,136463 +136464,136464 +128877,136464 +136465,136465 +128878,136465 +136466,136466 +128879,136466 +136467,136467 +128880,136467 +136468,136468 +128881,136468 +136469,136469 +128882,136469 +136470,136470 +128883,136470 +136471,136471 +128884,136471 +136472,136472 +128885,136472 +136473,136473 +128886,136473 +203003,136473 +136474,136474 +128887,136474 +246961,136474 +136475,136475 +128888,136475 +136476,136476 +128889,136476 +136477,136477 +128890,136477 +136478,136478 +128891,136478 +136479,136479 +128892,136479 +136480,136480 +128893,136480 +136481,136481 +128894,136481 +136482,136482 +128895,136482 +136483,136483 +128896,136483 +136484,136484 +128897,136484 +136485,136485 +128898,136485 +136486,136486 +128899,136486 +136487,136487 +128900,136487 +136488,136488 +128901,136488 +136489,136489 +128902,136489 +136490,136490 +128903,136490 +136491,136491 +128904,136491 +136492,136492 +128905,136492 +136493,136493 +128906,136493 +136494,136494 +128907,136494 +136495,136495 +128908,136495 +136496,136496 +128909,136496 +136497,136497 +128910,136497 +136498,136498 +128911,136498 +136499,136499 +128912,136499 +136500,136500 +128913,136500 +136501,136501 +128914,136501 +136502,136502 +128915,136502 +136503,136503 +128916,136503 +136504,136504 +128917,136504 +136505,136505 +128918,136505 +136506,136506 +128919,136506 +136507,136507 +128920,136507 +136508,136508 +128921,136508 +136509,136509 +128922,136509 +136510,136510 +128923,136510 +136511,136511 +128924,136511 +136512,136512 +128925,136512 +136513,136513 +128926,136513 +136514,136514 +128927,136514 +136515,136515 +128928,136515 +136516,136516 +128929,136516 +136517,136517 +128930,136517 +136518,136518 +128931,136518 +136519,136519 +128932,136519 +136520,136520 +128933,136520 +136521,136521 +128934,136521 +136522,136522 +128935,136522 +136523,136523 +128936,136523 +136524,136524 +128937,136524 +136525,136525 +128938,136525 +136526,136526 +128939,136526 +136527,136527 +128940,136527 +136528,136528 +128941,136528 +150671,136528 +136529,136529 +128942,136529 +136530,136530 +128943,136530 +136531,136531 +128944,136531 +136532,136532 +128945,136532 +86750,136532 +136533,136533 +128946,136533 +65122,136533 +136534,136534 +128947,136534 +136535,136535 +128948,136535 +136536,136536 +128949,136536 +136537,136537 +128950,136537 +136538,136538 +128951,136538 +136539,136539 +128952,136539 +136540,136540 +128953,136540 +136541,136541 +128954,136541 +136542,136542 +128955,136542 +3879,136542 +136543,136543 +128956,136543 +136544,136544 +128957,136544 +136545,136545 +128958,136545 +245426,136545 +136546,136546 +128959,136546 +136547,136547 +128960,136547 +136548,136548 +128961,136548 +136549,136549 +128962,136549 +136550,136550 +128963,136550 +136551,136551 +128964,136551 +136552,136552 +128965,136552 +136553,136553 +128966,136553 +136554,136554 +128967,136554 +136555,136555 +128968,136555 +136556,136556 +128969,136556 +136557,136557 +128970,136557 +136558,136558 +128971,136558 +136559,136559 +128972,136559 +136560,136560 +128973,136560 +38751,136560 +136561,136561 +128974,136561 +136562,136562 +128975,136562 +136563,136563 +128976,136563 +136564,136564 +128977,136564 +136565,136565 +128978,136565 +27575,136565 +136566,136566 +128979,136566 +136567,136567 +128980,136567 +136568,136568 +128981,136568 +136569,136569 +128982,136569 +136570,136570 +128983,136570 +136571,136571 +128984,136571 +136572,136572 +128985,136572 +136573,136573 +128986,136573 +232735,136573 +136574,136574 +128987,136574 +136575,136575 +128988,136575 +136576,136576 +128989,136576 +136577,136577 +128990,136577 +136578,136578 +128991,136578 +136579,136579 +128992,136579 +136580,136580 +128993,136580 +136581,136581 +128994,136581 +136582,136582 +128995,136582 +136583,136583 +128996,136583 +151596,136583 +136584,136584 +128997,136584 +97903,136584 +136585,136585 +128998,136585 +136586,136586 +128999,136586 +136587,136587 +129000,136587 +136588,136588 +129001,136588 +136589,136589 +129002,136589 +136590,136590 +129003,136590 +136591,136591 +129004,136591 +59988,136591 +136592,136592 +129005,136592 +136593,136593 +129006,136593 +136594,136594 +129007,136594 +136595,136595 +129008,136595 +136596,136596 +129009,136596 +136766,136766 +129179,136766 +136767,136767 +129180,136767 +136768,136768 +129181,136768 +136769,136769 +129182,136769 +136770,136770 +129183,136770 +136771,136771 +129184,136771 +136772,136772 +129185,136772 +136773,136773 +129186,136773 +136774,136774 +129187,136774 +136775,136775 +129188,136775 +136776,136776 +129189,136776 +136777,136777 +129190,136777 +223487,136777 +136778,136778 +129191,136778 +136779,136779 +129192,136779 +136780,136780 +129193,136780 +136781,136781 +129194,136781 +136782,136782 +129195,136782 +136783,136783 +129196,136783 +136784,136784 +129197,136784 +136785,136785 +129198,136785 +136786,136786 +129199,136786 +136787,136787 +129200,136787 +136788,136788 +129201,136788 +136789,136789 +129202,136789 +136790,136790 +129203,136790 +136791,136791 +129204,136791 +136942,136942 +129355,136942 +84000,136942 +136943,136943 +129356,136943 +136944,136944 +129357,136944 +136945,136945 +129358,136945 +241088,136945 +136946,136946 +129359,136946 +136947,136947 +129360,136947 +136948,136948 +129361,136948 +136949,136949 +129362,136949 +136950,136950 +129363,136950 +136951,136951 +129364,136951 +136952,136952 +129365,136952 +16471,136952 +136953,136953 +129366,136953 +136954,136954 +129367,136954 +136955,136955 +129368,136955 +136956,136956 +129369,136956 +136957,136957 +129370,136957 +136958,136958 +129371,136958 +136959,136959 +129372,136959 +136960,136960 +129373,136960 +136961,136961 +129374,136961 +136962,136962 +129375,136962 +136963,136963 +129376,136963 +136964,136964 +129377,136964 +136965,136965 +129378,136965 +136966,136966 +129379,136966 +136967,136967 +129380,136967 +136968,136968 +129381,136968 +136969,136969 +129382,136969 +136970,136970 +129383,136970 +136971,136971 +129384,136971 +136972,136972 +129385,136972 +136973,136973 +129386,136973 +136974,136974 +129387,136974 +136975,136975 +129388,136975 +136976,136976 +129389,136976 +136977,136977 +129390,136977 +136978,136978 +129391,136978 +136979,136979 +129392,136979 +136980,136980 +129393,136980 +109228,136980 +136981,136981 +129394,136981 +136211,136981 +128624,136981 +136982,136982 +129395,136982 +136983,136983 +129396,136983 +136984,136984 +129397,136984 +136985,136985 +129398,136985 +136986,136986 +129399,136986 +136987,136987 +129400,136987 +136988,136988 +129401,136988 +136989,136989 +129402,136989 +136990,136990 +129403,136990 +136991,136991 +129404,136991 +136992,136992 +129405,136992 +136993,136993 +129406,136993 +151121,136993 +82333,136993 +136994,136994 +129407,136994 +136995,136995 +129408,136995 +214459,136995 +136996,136996 +129409,136996 +136997,136997 +129410,136997 +8863,136997 +136998,136998 +129411,136998 +136999,136999 +129412,136999 +131912,136999 +124325,136999 +137000,137000 +129413,137000 +137001,137001 +129414,137001 +137002,137002 +129415,137002 +137003,137003 +129416,137003 +137004,137004 +129417,137004 +137005,137005 +129418,137005 +137006,137006 +129419,137006 +137007,137007 +129420,137007 +137008,137008 +129421,137008 +137009,137009 +129422,137009 +137010,137010 +129423,137010 +137011,137011 +129424,137011 +137012,137012 +129425,137012 +38463,137012 +215417,137012 +137013,137013 +129426,137013 +137014,137014 +129427,137014 +222880,137014 +137015,137015 +129428,137015 +137016,137016 +129429,137016 +137017,137017 +129430,137017 +137018,137018 +129431,137018 +137019,137019 +129432,137019 +137020,137020 +129433,137020 +137021,137021 +129434,137021 +137022,137022 +129435,137022 +137023,137023 +129436,137023 +137024,137024 +129437,137024 +137025,137025 +129438,137025 +137026,137026 +129439,137026 +185722,137026 +137027,137027 +129440,137027 +137028,137028 +129441,137028 +137029,137029 +129442,137029 +137030,137030 +129443,137030 +137031,137031 +129444,137031 +137032,137032 +129445,137032 +137033,137033 +129446,137033 +137034,137034 +129447,137034 +137035,137035 +129448,137035 +137036,137036 +129449,137036 +137037,137037 +129450,137037 +137038,137038 +129451,137038 +137039,137039 +129452,137039 +137040,137040 +129453,137040 +137041,137041 +129454,137041 +137042,137042 +129455,137042 +137043,137043 +129456,137043 +137044,137044 +129457,137044 +137045,137045 +129458,137045 +137046,137046 +129459,137046 +190969,137046 +187164,137046 +137047,137047 +129460,137047 +137048,137048 +129461,137048 +137049,137049 +129462,137049 +137050,137050 +129463,137050 +137051,137051 +129464,137051 +137052,137052 +129465,137052 +137053,137053 +129466,137053 +137054,137054 +129467,137054 +137055,137055 +129468,137055 +137056,137056 +129469,137056 +137057,137057 +129470,137057 +122817,137057 +137058,137058 +129471,137058 +137059,137059 +129472,137059 +137060,137060 +129473,137060 +137061,137061 +129474,137061 +137062,137062 +129475,137062 +137063,137063 +129476,137063 +137064,137064 +129477,137064 +137065,137065 +129478,137065 +145352,137065 +137066,137066 +129479,137066 +137067,137067 +129480,137067 +158443,137068 +137068,137068 +129481,137068 +137069,137069 +129482,137069 +137070,137070 +129483,137070 +137071,137071 +129484,137071 +137072,137072 +129485,137072 +137073,137073 +129486,137073 +137074,137074 +129487,137074 +137075,137075 +129488,137075 +137076,137076 +129489,137076 +137077,137077 +129490,137077 +137078,137078 +129491,137078 +137079,137079 +129492,137079 +5653,137079 +137080,137080 +129493,137080 +23548,137080 +137081,137081 +129494,137081 +137082,137082 +129495,137082 +137083,137083 +129496,137083 +137084,137084 +129497,137084 +137085,137085 +129498,137085 +137086,137086 +129499,137086 +137087,137087 +129500,137087 +137088,137088 +129501,137088 +137089,137089 +129502,137089 +137090,137090 +129503,137090 +137091,137091 +129504,137091 +137092,137092 +129505,137092 +60687,137092 +137093,137093 +129506,137093 +137094,137094 +129507,137094 +218487,137094 +137095,137095 +129508,137095 +185036,137096 +137096,137096 +129509,137096 +137097,137097 +129510,137097 +137098,137098 +129511,137098 +137099,137099 +129512,137099 +115932,137099 +137100,137100 +129513,137100 +137101,137101 +129514,137101 +137102,137102 +129515,137102 +137103,137103 +129516,137103 +137104,137104 +129517,137104 +137105,137105 +129518,137105 +137106,137106 +129519,137106 +137107,137107 +129520,137107 +137108,137108 +129521,137108 +137109,137109 +129522,137109 +137110,137110 +129523,137110 +137111,137111 +129524,137111 +137112,137112 +129525,137112 +137113,137113 +129526,137113 +137114,137114 +129527,137114 +137151,137151 +129564,137151 +137152,137152 +129565,137152 +5548,137152 +137153,137153 +129566,137153 +137154,137154 +129567,137154 +137155,137155 +129568,137155 +3134,137155 +137156,137156 +129569,137156 +137157,137157 +129570,137157 +188520,137157 +137158,137158 +129571,137158 +137159,137159 +129572,137159 +137160,137160 +129573,137160 +137161,137161 +129574,137161 +137162,137162 +129575,137162 +137163,137163 +129576,137163 +137164,137164 +129577,137164 +137165,137165 +129578,137165 +137166,137166 +129579,137166 +137167,137167 +129580,137167 +137168,137168 +129581,137168 +137169,137169 +129582,137169 +137170,137170 +129583,137170 +137171,137171 +129584,137171 +137172,137172 +129585,137172 +220058,137172 +137173,137173 +129586,137173 +137174,137174 +129587,137174 +137175,137175 +129588,137175 +137176,137176 +129589,137176 +137177,137177 +129590,137177 +137178,137178 +129591,137178 +226674,137178 +137179,137179 +129592,137179 +137180,137180 +129593,137180 +137181,137181 +129594,137181 +137182,137182 +129595,137182 +137183,137183 +129596,137183 +137184,137184 +129597,137184 +137185,137185 +129598,137185 +137186,137186 +129599,137186 +137187,137187 +129600,137187 +137188,137188 +129601,137188 +137189,137189 +129602,137189 +137190,137190 +129603,137190 +137313,137313 +129726,137313 +137720,137313 +130133,137313 +137314,137314 +129727,137314 +137315,137315 +129728,137315 +137723,137316 +137316,137316 +130136,137316 +129729,137316 +137724,137317 +137317,137317 +130137,137317 +129730,137317 +137318,137318 +129731,137318 +137725,137318 +130138,137318 +137726,137319 +137319,137319 +130139,137319 +129732,137319 +137320,137320 +129733,137320 +137727,137320 +130140,137320 +137321,137321 +129734,137321 +137728,137321 +130141,137321 +137322,137322 +129735,137322 +137729,137322 +130142,137322 +137730,137323 +137323,137323 +130143,137323 +129736,137323 +137731,137324 +137324,137324 +130144,137324 +129737,137324 +137732,137325 +137325,137325 +130145,137325 +129738,137325 +137733,137326 +137532,137326 +137326,137326 +130146,137326 +129945,137326 +129739,137326 +137734,137327 +137327,137327 +130147,137327 +129740,137327 +137735,137328 +137328,137328 +130148,137328 +129741,137328 +137736,137329 +137329,137329 +130149,137329 +129742,137329 +137737,137330 +137330,137330 +130150,137330 +129743,137330 +137738,137331 +137331,137331 +130151,137331 +129744,137331 +151798,137331 +137739,137332 +137332,137332 +130152,137332 +129745,137332 +137333,137333 +129746,137333 +137540,137334 +137334,137334 +129953,137334 +129747,137334 +137335,137335 +129748,137335 +137336,137336 +129749,137336 +137337,137337 +129750,137337 +137338,137338 +129751,137338 +137339,137339 +129752,137339 +137746,137339 +130159,137339 +137340,137340 +129753,137340 +137341,137341 +129754,137341 +137748,137341 +130161,137341 +137342,137342 +129755,137342 +137343,137343 +129756,137343 +137549,137343 +129962,137343 +137344,137344 +129757,137344 +137345,137345 +129758,137345 +137346,137346 +129759,137346 +150095,137346 +137347,137347 +129760,137347 +155443,137347 +137348,137348 +129761,137348 +137349,137349 +129762,137349 +137350,137350 +129763,137350 +137351,137351 +129764,137351 +137352,137352 +129765,137352 +137353,137353 +129766,137353 +137354,137354 +129767,137354 +137355,137355 +129768,137355 +137356,137356 +129769,137356 +137357,137357 +129770,137357 +137358,137358 +129771,137358 +137359,137359 +129772,137359 +137360,137360 +129773,137360 +190655,137360 +137361,137361 +129774,137361 +137362,137362 +129775,137362 +137363,137363 +129776,137363 +137528,137528 +129941,137528 +137529,137529 +129942,137529 +137530,137530 +129943,137530 +137531,137531 +129944,137531 +137733,137532 +137532,137532 +137326,137532 +130146,137532 +129945,137532 +129739,137532 +137533,137533 +129946,137533 +137534,137534 +129947,137534 +137535,137535 +129948,137535 +1064,137535 +137536,137536 +129949,137536 +137537,137537 +129950,137537 +137538,137538 +129951,137538 +137539,137539 +129952,137539 +137540,137540 +137334,137540 +129953,137540 +129747,137540 +146139,137541 +137541,137541 +129954,137541 +137542,137542 +129955,137542 +155432,137542 +151803,137542 +137543,137543 +129956,137543 +137544,137544 +129957,137544 +137545,137545 +129958,137545 +137546,137546 +129959,137546 +137547,137547 +129960,137547 +137749,137548 +137548,137548 +130162,137548 +129961,137548 +137549,137549 +129962,137549 +137343,137549 +129756,137549 +137550,137550 +129963,137550 +137752,137551 +137551,137551 +130165,137551 +129964,137551 +137552,137552 +129965,137552 +27194,137552 +137553,137553 +129966,137553 +137554,137554 +129967,137554 +224970,137554 +155445,137555 +146153,137555 +137555,137555 +129968,137555 +150099,137556 +146154,137556 +137556,137556 +129969,137556 +137557,137557 +129970,137557 +137558,137558 +129971,137558 +150293,137558 +137559,137559 +129972,137559 +137560,137560 +129973,137560 +137561,137561 +129974,137561 +137716,137716 +130129,137716 +137717,137717 +130130,137717 +137718,137718 +130131,137718 +137719,137719 +130132,137719 +137720,137720 +130133,137720 +148222,137720 +137313,137720 +129726,137720 +137721,137721 +130134,137721 +137722,137722 +130135,137722 +137723,137723 +137316,137723 +130136,137723 +129729,137723 +137724,137724 +137317,137724 +130137,137724 +129730,137724 +137725,137725 +130138,137725 +137318,137725 +129731,137725 +137726,137726 +137319,137726 +130139,137726 +129732,137726 +137727,137727 +130140,137727 +137320,137727 +129733,137727 +137728,137728 +130141,137728 +137321,137728 +129734,137728 +137729,137729 +130142,137729 +137322,137729 +129735,137729 +137730,137730 +137323,137730 +130143,137730 +129736,137730 +137731,137731 +137324,137731 +130144,137731 +129737,137731 +137732,137732 +137325,137732 +130145,137732 +129738,137732 +137733,137733 +137532,137733 +137326,137733 +130146,137733 +129945,137733 +129739,137733 +137734,137734 +137327,137734 +130147,137734 +129740,137734 +137735,137735 +137328,137735 +130148,137735 +129741,137735 +137736,137736 +137329,137736 +130149,137736 +129742,137736 +137737,137737 +137330,137737 +130150,137737 +129743,137737 +137738,137738 +137331,137738 +130151,137738 +129744,137738 +151798,137738 +137739,137739 +137332,137739 +130152,137739 +129745,137739 +137740,137740 +130153,137740 +137741,137741 +130154,137741 +137742,137742 +130155,137742 +137743,137743 +130156,137743 +137744,137744 +130157,137744 +155433,137744 +137745,137745 +130158,137745 +137746,137746 +130159,137746 +137339,137746 +129752,137746 +137747,137747 +130160,137747 +137748,137748 +130161,137748 +155437,137748 +137341,137748 +129754,137748 +137749,137749 +137548,137749 +130162,137749 +129961,137749 +137750,137750 +130163,137750 +137751,137751 +130164,137751 +137752,137752 +137551,137752 +130165,137752 +129964,137752 +137753,137753 +130166,137753 +137754,137754 +130167,137754 +137755,137755 +130168,137755 +203933,137755 +150097,137755 +137756,137756 +130169,137756 +137757,137757 +130170,137757 +137758,137758 +130171,137758 +75601,137758 +137759,137759 +130172,137759 +137760,137760 +130173,137760 +137761,137761 +130174,137761 +137762,137762 +130175,137762 +150105,137763 +137763,137763 +130176,137763 +146160,137763 +137922,137922 +130335,137922 +137923,137923 +130336,137923 +88242,137923 +137924,137924 +130337,137924 +137925,137925 +130338,137925 +137926,137926 +130339,137926 +137927,137927 +130340,137927 +137928,137928 +130341,137928 +137929,137929 +130342,137929 +137930,137930 +130343,137930 +137931,137931 +130344,137931 +84566,137931 +137932,137932 +130345,137932 +137933,137933 +130346,137933 +137934,137934 +130347,137934 +137935,137935 +130348,137935 +137936,137936 +130349,137936 +137937,137937 +130350,137937 +137938,137938 +130351,137938 +206664,137938 +137939,137939 +130352,137939 +179181,137939 +137940,137940 +130353,137940 +137941,137941 +130354,137941 +137942,137942 +130355,137942 +137943,137943 +130356,137943 +177817,137943 +171853,137943 +137944,137944 +130357,137944 +137945,137945 +130358,137945 +137946,137946 +130359,137946 +137947,137947 +130360,137947 +137948,137948 +130361,137948 +137949,137949 +130362,137949 +137950,137950 +130363,137950 +137951,137951 +130364,137951 +137952,137952 +130365,137952 +137953,137953 +130366,137953 +92673,137953 +137954,137954 +130367,137954 +137955,137955 +130368,137955 +137956,137956 +130369,137956 +137957,137957 +130370,137957 +137958,137958 +130371,137958 +135820,137958 +128233,137958 +137959,137959 +130372,137959 +137960,137960 +130373,137960 +137961,137961 +130374,137961 +137962,137962 +130375,137962 +137963,137963 +130376,137963 +137964,137964 +130377,137964 +137965,137965 +130378,137965 +137966,137966 +130379,137966 +137967,137967 +130380,137967 +137968,137968 +130381,137968 +33428,137968 +137969,137969 +130382,137969 +137970,137970 +130383,137970 +137971,137971 +130384,137971 +137972,137972 +130385,137972 +137973,137973 +130386,137973 +117045,137973 +137974,137974 +130387,137974 +137975,137975 +130388,137975 +137976,137976 +130389,137976 +137977,137977 +130390,137977 +137978,137978 +130391,137978 +137979,137979 +130392,137979 +137980,137980 +130393,137980 +63457,137980 +137981,137981 +130394,137981 +137982,137982 +130395,137982 +137983,137983 +130396,137983 +137984,137984 +130397,137984 +137985,137985 +130398,137985 +137986,137986 +130399,137986 +137987,137987 +130400,137987 +137988,137988 +130401,137988 +137989,137989 +130402,137989 +137990,137990 +130403,137990 +137991,137991 +130404,137991 +137992,137992 +130405,137992 +137993,137993 +130406,137993 +137994,137994 +130407,137994 +137995,137995 +130408,137995 +137996,137996 +130409,137996 +137997,137997 +130410,137997 +137998,137998 +130411,137998 +137999,137999 +130412,137999 +138000,138000 +130413,138000 +158109,138001 +138001,138001 +130414,138001 +138002,138002 +130415,138002 +138003,138003 +130416,138003 +138004,138004 +130417,138004 +138005,138005 +130418,138005 +138006,138006 +130419,138006 +138007,138007 +130420,138007 +138008,138008 +130421,138008 +138009,138009 +130422,138009 +138010,138010 +130423,138010 +138011,138011 +130424,138011 +138012,138012 +130425,138012 +138013,138013 +130426,138013 +138014,138014 +130427,138014 +138015,138015 +130428,138015 +138016,138016 +130429,138016 +138017,138017 +130430,138017 +138018,138018 +130431,138018 +138019,138019 +130432,138019 +138020,138020 +130433,138020 +138021,138021 +130434,138021 +138022,138022 +130435,138022 +138023,138023 +130436,138023 +138024,138024 +130437,138024 +138025,138025 +130438,138025 +138026,138026 +130439,138026 +138027,138027 +130440,138027 +138028,138028 +130441,138028 +138029,138029 +130442,138029 +32320,138029 +138030,138030 +130443,138030 +138031,138031 +130444,138031 +138032,138032 +130445,138032 +138033,138033 +130446,138033 +138034,138034 +130447,138034 +138035,138035 +130448,138035 +246740,138035 +138077,138077 +130490,138077 +138078,138078 +130491,138078 +138079,138079 +130492,138079 +138080,138080 +130493,138080 +138081,138081 +130494,138081 +138082,138082 +130495,138082 +138083,138083 +130496,138083 +138084,138084 +130497,138084 +138085,138085 +130498,138085 +138086,138086 +130499,138086 +138087,138087 +130500,138087 +138088,138088 +130501,138088 +138089,138089 +130502,138089 +138090,138090 +130503,138090 +138091,138091 +130504,138091 +138092,138092 +130505,138092 +138093,138093 +130506,138093 +138094,138094 +130507,138094 +138095,138095 +130508,138095 +138096,138096 +130509,138096 +138097,138097 +130510,138097 +138098,138098 +130511,138098 +138099,138099 +130512,138099 +138100,138100 +130513,138100 +138101,138101 +130514,138101 +138102,138102 +130515,138102 +138103,138103 +130516,138103 +138104,138104 +130517,138104 +138105,138105 +130518,138105 +138106,138106 +130519,138106 +138107,138107 +130520,138107 +138108,138108 +130521,138108 +138109,138109 +130522,138109 +22081,138109 +138110,138110 +130523,138110 +138111,138111 +130524,138111 +138112,138112 +130525,138112 +138113,138113 +130526,138113 +138114,138114 +130527,138114 +138115,138115 +130528,138115 +138116,138116 +130529,138116 +138117,138117 +130530,138117 +138118,138118 +130531,138118 +138119,138119 +130532,138119 +138120,138120 +130533,138120 +138121,138121 +130534,138121 +138122,138122 +130535,138122 +138123,138123 +130536,138123 +138124,138124 +130537,138124 +138125,138125 +130538,138125 +119005,138125 +138126,138126 +130539,138126 +138127,138127 +130540,138127 +138128,138128 +130541,138128 +138129,138129 +130542,138129 +138130,138130 +130543,138130 +138131,138131 +130544,138131 +138132,138132 +130545,138132 +138133,138133 +130546,138133 +138134,138134 +130547,138134 +138135,138135 +130548,138135 +138136,138136 +130549,138136 +138137,138137 +130550,138137 +138138,138138 +130551,138138 +22651,138138 +138139,138139 +130552,138139 +138140,138140 +130553,138140 +138141,138141 +130554,138141 +138142,138142 +130555,138142 +138143,138143 +130556,138143 +138144,138144 +130557,138144 +138145,138145 +130558,138145 +138146,138146 +130559,138146 +138147,138147 +130560,138147 +138148,138148 +130561,138148 +138149,138149 +130562,138149 +138150,138150 +130563,138150 +139171,138150 +214745,138150 +138151,138151 +130564,138151 +138152,138152 +130565,138152 +138153,138153 +130566,138153 +138154,138154 +130567,138154 +138155,138155 +130568,138155 +138156,138156 +130569,138156 +138157,138157 +130570,138157 +138158,138158 +130571,138158 +138159,138159 +130572,138159 +138160,138160 +130573,138160 +138161,138161 +130574,138161 +138162,138162 +130575,138162 +138163,138163 +130576,138163 +138164,138164 +130577,138164 +138165,138165 +130578,138165 +234788,138166 +138166,138166 +130579,138166 +138167,138167 +130580,138167 +138168,138168 +130581,138168 +138169,138169 +130582,138169 +138170,138170 +130583,138170 +242944,138170 +138171,138171 +130584,138171 +138172,138172 +130585,138172 +138173,138173 +130586,138173 +138174,138174 +130587,138174 +28369,138174 +138527,138527 +117715,138527 +138528,138528 +138529,138529 +138530,138530 +138531,138531 +138532,138532 +138533,138533 +138534,138534 +138535,138535 +138536,138536 +138537,138537 +138538,138538 +138539,138539 +138540,138540 +138541,138541 +138542,138542 +138543,138543 +138544,138544 +138545,138545 +138546,138546 +59492,138546 +138547,138547 +138548,138548 +138549,138549 +138639,138639 +138640,138640 +138641,138641 +138642,138642 +138643,138643 +138644,138644 +138645,138645 +138646,138646 +138647,138647 +138648,138648 +138649,138649 +27055,138649 +138650,138650 +138651,138651 +138652,138652 +138653,138653 +138654,138654 +138655,138655 +138656,138656 +138657,138657 +138658,138658 +138659,138659 +138660,138660 +138661,138661 +36469,138661 +138662,138662 +138663,138663 +138664,138664 +138665,138665 +138666,138666 +138667,138667 +138668,138668 +138669,138669 +138670,138670 +138671,138671 +138672,138672 +138673,138673 +138674,138674 +138675,138675 +138676,138676 +234282,138676 +148225,138676 +138677,138677 +138678,138678 +185368,138678 +138679,138679 +138680,138680 +138681,138681 +138682,138682 +138683,138683 +88131,138683 +138684,138684 +138685,138685 +35816,138685 +138686,138686 +138687,138687 +138688,138688 +138689,138689 +138690,138690 +138691,138691 +138692,138692 +138693,138693 +138694,138694 +138695,138695 +138696,138696 +138697,138697 +138698,138698 +138699,138699 +138700,138700 +138701,138701 +138702,138702 +138703,138703 +138704,138704 +138705,138705 +138706,138706 +138707,138707 +138708,138708 +138709,138709 +138710,138710 +138711,138711 +138712,138712 +138713,138713 +138714,138714 +103491,138714 +138715,138715 +138716,138716 +138717,138717 +138718,138718 +138719,138719 +75905,138719 +138720,138720 +138721,138721 +138722,138722 +138723,138723 +38516,138723 +138724,138724 +17397,138724 +138725,138725 +138726,138726 +138727,138727 +138728,138728 +138729,138729 +138730,138730 +138731,138731 +138732,138732 +138733,138733 +79972,138733 +138734,138734 +138735,138735 +138736,138736 +138737,138737 +138738,138738 +138739,138739 +138740,138740 +138741,138741 +138742,138742 +138743,138743 +151479,138743 +138744,138744 +138745,138745 +138746,138746 +138747,138747 +155076,138747 +138748,138748 +138749,138749 +67014,138749 +138750,138750 +138751,138751 +138752,138752 +138753,138753 +138754,138754 +138755,138755 +138756,138756 +138757,138757 +138758,138758 +138759,138759 +138760,138760 +138761,138761 +138762,138762 +138763,138763 +138764,138764 +138765,138765 +138766,138766 +138767,138767 +194292,138767 +138768,138768 +138769,138769 +138770,138770 +138771,138771 +138772,138772 +138773,138773 +138774,138774 +74975,138774 +138775,138775 +138776,138776 +138777,138777 +138778,138778 +155162,138778 +138779,138779 +138780,138780 +138781,138781 +138782,138782 +138783,138783 +138784,138784 +138849,138849 +138850,138850 +138851,138851 +138852,138852 +8086,138852 +138853,138853 +138854,138854 +138855,138855 +138856,138856 +138857,138857 +138858,138858 +138859,138859 +138860,138860 +138861,138861 +138862,138862 +138863,138863 +122185,138863 +138864,138864 +138865,138865 +138866,138866 +138867,138867 +138868,138868 +138869,138869 +138870,138870 +138871,138871 +64145,138871 +138872,138872 +138873,138873 +209627,138873 +138874,138874 +138875,138875 +138876,138876 +138877,138877 +138878,138878 +138879,138879 +138880,138880 +138881,138881 +235916,138881 +138882,138882 +138883,138883 +138884,138884 +138885,138885 +138886,138886 +138887,138887 +138888,138888 +25110,138888 +138889,138889 +138890,138890 +158508,138890 +138891,138891 +138892,138892 +138893,138893 +138894,138894 +4502,138894 +138895,138895 +208767,138896 +138896,138896 +160430,138896 +138897,138897 +138898,138898 +138899,138899 +138900,138900 +138901,138901 +138902,138902 +138903,138903 +138904,138904 +138905,138905 +138906,138906 +138907,138907 +138908,138908 +138909,138909 +138910,138910 +138911,138911 +138912,138912 +138913,138913 +138914,138914 +138915,138915 +138916,138916 +138917,138917 +138918,138918 +138919,138919 +138920,138920 +138921,138921 +138922,138922 +138923,138923 +138924,138924 +138925,138925 +138926,138926 +138927,138927 +138928,138928 +138929,138929 +138930,138930 +138931,138931 +138932,138932 +138933,138933 +138934,138934 +138935,138935 +138936,138936 +138937,138937 +138938,138938 +138939,138939 +138940,138940 +138941,138941 +138942,138942 +138943,138943 +112022,138943 +138944,138944 +138945,138945 +138946,138946 +107640,138946 +138947,138947 +138948,138948 +138949,138949 +201559,138949 +138950,138950 +138951,138951 +138952,138952 +138953,138953 +138954,138954 +138955,138955 +138956,138956 +189423,138956 +138957,138957 +138958,138958 +138959,138959 +138960,138960 +138961,138961 +138962,138962 +138963,138963 +138964,138964 +138965,138965 +138966,138966 +11535,138966 +138967,138967 +138968,138968 +138969,138969 +138970,138970 +138971,138971 +138972,138972 +138973,138973 +138974,138974 +139035,139035 +139036,139036 +139037,139037 +139038,139038 +139039,139039 +139040,139040 +139041,139041 +139042,139042 +139043,139043 +139044,139044 +139045,139045 +139046,139046 +23729,139046 +139047,139047 +139048,139048 +139049,139049 +139050,139050 +139051,139051 +139052,139052 +29885,139052 +139053,139053 +139054,139054 +139055,139055 +139056,139056 +139057,139057 +139058,139058 +139059,139059 +139060,139060 +139061,139061 +139062,139062 +139063,139063 +139064,139064 +139065,139065 +139066,139066 +139067,139067 +139068,139068 +139069,139069 +139070,139070 +139071,139071 +908,139071 +139072,139072 +139073,139073 +139074,139074 +139075,139075 +139076,139076 +139077,139077 +139119,139119 +139120,139120 +104920,139120 +139121,139121 +139122,139122 +139123,139123 +139124,139124 +139125,139125 +139126,139126 +33502,139126 +139127,139127 +139128,139128 +139129,139129 +139130,139130 +139131,139131 +139132,139132 +139133,139133 +139134,139134 +139135,139135 +139136,139136 +139137,139137 +139138,139138 +139139,139139 +139140,139140 +139141,139141 +139142,139142 +139143,139143 +139144,139144 +193771,139145 +139145,139145 +139146,139146 +139147,139147 +139148,139148 +139149,139149 +139150,139150 +131993,139150 +124406,139150 +139151,139151 +139152,139152 +139153,139153 +139154,139154 +158766,139154 +139155,139155 +139156,139156 +139157,139157 +139158,139158 +82576,139158 +60991,139158 +139159,139159 +139160,139160 +139161,139161 +139162,139162 +139163,139163 +139164,139164 +139165,139165 +139166,139166 +139167,139167 +139168,139168 +139169,139169 +139170,139170 +139171,139171 +138150,139171 +130563,139171 +139172,139172 +139173,139173 +139174,139174 +139175,139175 +245403,139175 +139176,139176 +139177,139177 +139178,139178 +139179,139179 +139180,139180 +9463,139180 +28232,139180 +139181,139181 +139182,139182 +139183,139183 +5346,139183 +139184,139184 +139185,139185 +139186,139186 +139187,139187 +139188,139188 +139189,139189 +72896,139189 +139190,139190 +139191,139191 +139192,139192 +139193,139193 +241527,139193 +139194,139194 +139195,139195 +139259,139259 +139260,139260 +139261,139261 +139262,139262 +139263,139263 +139264,139264 +139265,139265 +139266,139266 +139267,139267 +139268,139268 +141689,139268 +139269,139269 +139270,139270 +139271,139271 +139272,139272 +139273,139273 +139274,139274 +139275,139275 +139276,139276 +139277,139277 +139278,139278 +139279,139279 +139280,139280 +139281,139281 +139282,139282 +139283,139283 +139284,139284 +139285,139285 +139286,139286 +139287,139287 +139288,139288 +139289,139289 +139290,139290 +133732,139290 +126145,139290 +139291,139291 +139292,139292 +139293,139293 +139294,139294 +139295,139295 +139296,139296 +139297,139297 +139298,139298 +139299,139299 +139300,139300 +139301,139301 +139302,139302 +139303,139303 +139304,139304 +139305,139305 +139599,139599 +74710,139599 +249835,139599 +139600,139600 +139601,139601 +139602,139602 +139603,139603 +139604,139604 +139669,139669 +139670,139670 +139671,139671 +139672,139672 +139673,139673 +139674,139674 +139675,139675 +139676,139676 +139677,139677 +139678,139678 +139679,139679 +139680,139680 +139681,139681 +139682,139682 +139683,139683 +163699,139683 +22421,139683 +139684,139684 +139685,139685 +107135,139685 +139686,139686 +139687,139687 +139688,139688 +139689,139689 +139690,139690 +139691,139691 +139692,139692 +139693,139693 +139694,139694 +139695,139695 +139696,139696 +133723,139696 +126136,139696 +139697,139697 +139698,139698 +139699,139699 +139700,139700 +139701,139701 +139702,139702 +139703,139703 +139704,139704 +139705,139705 +139706,139706 +164020,139706 +139707,139707 +133734,139707 +126147,139707 +17208,139707 +139708,139708 +139709,139709 +139710,139710 +139711,139711 +139712,139712 +139713,139713 +229591,139713 +139714,139714 +139715,139715 +139716,139716 +139717,139717 +150497,139718 +139718,139718 +139719,139719 +139720,139720 +139721,139721 +139722,139722 +139723,139723 +139724,139724 +139725,139725 +139726,139726 +139727,139727 +139728,139728 +139729,139729 +139730,139730 +139731,139731 +139732,139732 +139733,139733 +162387,139733 +139734,139734 +139735,139735 +139736,139736 +139737,139737 +139738,139738 +64036,139738 +139739,139739 +139740,139740 +139741,139741 +139742,139742 +139743,139743 +139744,139744 +139745,139745 +139746,139746 +139747,139747 +139748,139748 +160840,139748 +139749,139749 +139750,139750 +139751,139751 +139752,139752 +139753,139753 +139754,139754 +139755,139755 +139756,139756 +139757,139757 +139758,139758 +139759,139759 +139760,139760 +109817,139760 +139761,139761 +139762,139762 +139763,139763 +139764,139764 +139765,139765 +139766,139766 +139767,139767 +139768,139768 +139769,139769 +139770,139770 +203021,139770 +139771,139771 +139772,139772 +139773,139773 +23553,139773 +139774,139774 +104049,139774 +139775,139775 +139776,139776 +139777,139777 +187461,139777 +139778,139778 +139779,139779 +139780,139780 +139781,139781 +208544,139782 +139782,139782 +139783,139783 +139784,139784 +139785,139785 +139786,139786 +139836,139836 +83680,139836 +139837,139837 +123944,139837 +139838,139838 +139839,139839 +139840,139840 +139841,139841 +139842,139842 +139843,139843 +139844,139844 +139845,139845 +139883,139883 +203701,139883 +139884,139884 +139885,139885 +139886,139886 +139887,139887 +139888,139888 +139889,139889 +139890,139890 +139891,139891 +139892,139892 +139893,139893 +139894,139894 +139895,139895 +139896,139896 +139897,139897 +139898,139898 +139899,139899 +106112,139899 +139900,139900 +139901,139901 +139902,139902 +139903,139903 +139904,139904 +83010,139904 +139905,139905 +139906,139906 +67386,139906 +139907,139907 +139908,139908 +139909,139909 +139910,139910 +139911,139911 +139912,139912 +139913,139913 +139914,139914 +139915,139915 +139916,139916 +139917,139917 +139918,139918 +139919,139919 +139920,139920 +139921,139921 +139922,139922 +139923,139923 +139924,139924 +139925,139925 +139926,139926 +139927,139927 +139928,139928 +139929,139929 +139930,139930 +139931,139931 +139932,139932 +114685,139932 +139933,139933 +139934,139934 +142694,139934 +139935,139935 +139936,139936 +139937,139937 +82023,139937 +139938,139938 +139939,139939 +139940,139940 +139941,139941 +139942,139942 +139943,139943 +139944,139944 +139945,139945 +139946,139946 +139947,139947 +139948,139948 +188035,139948 +139949,139949 +241649,139949 +139950,139950 +139951,139951 +98217,139951 +139952,139952 +139953,139953 +139954,139954 +139955,139955 +20455,139955 +139956,139956 +139957,139957 +70336,139957 +139958,139958 +139959,139959 +139960,139960 +139961,139961 +139962,139962 +139963,139963 +139964,139964 +139965,139965 +139966,139966 +192623,139966 +139967,139967 +139968,139968 +139969,139969 +139970,139970 +139971,139971 +139972,139972 +139973,139973 +139974,139974 +139975,139975 +183667,139975 +139976,139976 +139977,139977 +139978,139978 +139979,139979 +139980,139980 +139981,139981 +139982,139982 +194189,139982 +139983,139983 +139984,139984 +139985,139985 +132370,139985 +124783,139985 +139986,139986 +139987,139987 +139988,139988 +139989,139989 +242781,139989 +139990,139990 +139991,139991 +139992,139992 +139993,139993 +139994,139994 +139995,139995 +139996,139996 +139997,139997 +98038,139997 +139998,139998 +139999,139999 +140000,140000 +140001,140001 +140002,140002 +140003,140003 +140004,140004 +140005,140005 +140006,140006 +98087,140006 +140007,140007 +140008,140008 +140009,140009 +140010,140010 +140011,140011 +140012,140012 +140013,140013 +140014,140014 +140015,140015 +140016,140016 +140017,140017 +140018,140018 +140019,140019 +134128,140019 +126541,140019 +140020,140020 +140021,140021 +70400,140021 +140022,140022 +140023,140023 +140024,140024 +140025,140025 +140026,140026 +140027,140027 +140028,140028 +27190,140028 +140029,140029 +140030,140030 +140031,140031 +140032,140032 +100600,140032 +140033,140033 +140034,140034 +140035,140035 +140036,140036 +140037,140037 +140038,140038 +140039,140039 +140040,140040 +140041,140041 +140042,140042 +140043,140043 +140044,140044 +140045,140045 +140046,140046 +144084,140046 +140047,140047 +140048,140048 +140049,140049 +140050,140050 +140051,140051 +140052,140052 +140053,140053 +140054,140054 +140055,140055 +140056,140056 +140057,140057 +140058,140058 +140059,140059 +140060,140060 +140061,140061 +140062,140062 +140063,140063 +140064,140064 +140065,140065 +140066,140066 +140104,140104 +87,140104 +140105,140105 +140106,140106 +140107,140107 +140108,140108 +140109,140109 +140110,140110 +140111,140111 +140112,140112 +140113,140113 +140114,140114 +140115,140115 +140116,140116 +140117,140117 +140118,140118 +140119,140119 +140120,140120 +51647,140120 +43543,140120 +140121,140121 +140122,140122 +140123,140123 +140124,140124 +140125,140125 +140126,140126 +140127,140127 +140128,140128 +77723,140128 +140129,140129 +140130,140130 +140131,140131 +8001,140131 +140132,140132 +140133,140133 +140134,140134 +140135,140135 +140136,140136 +140137,140137 +140138,140138 +140139,140139 +140140,140140 +140141,140141 +140142,140142 +140143,140143 +142494,140143 +140144,140144 +140145,140145 +140146,140146 +142497,140146 +140147,140147 +142498,140147 +140148,140148 +142499,140148 +140149,140149 +142500,140149 +140150,140150 +6496,140150 +140151,140151 +140152,140152 +142503,140152 +142504,140153 +140153,140153 +142505,140154 +140154,140154 +142506,140155 +140155,140155 +142507,140156 +140156,140156 +140157,140157 +142508,140157 +58639,140157 +237990,140157 +140158,140158 +142509,140158 +140159,140159 +142510,140159 +140160,140160 +142511,140160 +142512,140161 +140161,140161 +140162,140162 +142513,140162 +209406,140162 +140163,140163 +140164,140164 +140165,140165 +140166,140166 +140167,140167 +140168,140168 +140169,140169 +140170,140170 +140171,140171 +140172,140172 +142523,140172 +140173,140173 +140174,140174 +140175,140175 +140176,140176 +142527,140176 +140177,140177 +140178,140178 +140179,140179 +140180,140180 +140229,140229 +140230,140230 +216502,140230 +140231,140231 +140232,140232 +140233,140233 +140234,140234 +140235,140235 +140236,140236 +140237,140237 +140238,140238 +140239,140239 +140240,140240 +140241,140241 +140242,140242 +140243,140243 +140244,140244 +140245,140245 +59723,140245 +140246,140246 +140247,140247 +140248,140248 +140249,140249 +140250,140250 +88650,140250 +140251,140251 +140252,140252 +140253,140253 +140254,140254 +140255,140255 +105687,140255 +140256,140256 +140257,140257 +80739,140257 +140258,140258 +161141,140258 +140259,140259 +140260,140260 +140261,140261 +140262,140262 +140263,140263 +140264,140264 +140265,140265 +140266,140266 +140267,140267 +140268,140268 +140269,140269 +140270,140270 +140271,140271 +140272,140272 +140273,140273 +233947,140273 +140274,140274 +140275,140275 +140276,140276 +140277,140277 +140278,140278 +140279,140279 +90712,140279 +140280,140280 +140281,140281 +140282,140282 +140283,140283 +140284,140284 +140285,140285 +140286,140286 +140287,140287 +140288,140288 +176237,140288 +170273,140288 +140289,140289 +140290,140290 +140291,140291 +140292,140292 +140293,140293 +140294,140294 +140295,140295 +140296,140296 +140297,140297 +140298,140298 +140299,140299 +140300,140300 +140301,140301 +140302,140302 +35545,140302 +140303,140303 +140304,140304 +140305,140305 +140306,140306 +140307,140307 +140308,140308 +140309,140309 +140310,140310 +140311,140311 +140312,140312 +140313,140313 +140314,140314 +215638,140314 +140315,140315 +140316,140316 +140317,140317 +140318,140318 +140319,140319 +140320,140320 +56778,140320 +140321,140321 +140322,140322 +140323,140323 +140324,140324 +240192,140324 +140325,140325 +140326,140326 +140327,140327 +140328,140328 +140329,140329 +140330,140330 +140331,140331 +140332,140332 +140333,140333 +140334,140334 +140335,140335 +140376,140376 +140377,140377 +140378,140378 +140379,140379 +140380,140380 +140381,140381 +140382,140382 +942,140382 +140383,140383 +140384,140384 +140385,140385 +140386,140386 +140387,140387 +140388,140388 +78814,140388 +140389,140389 +140390,140390 +140391,140391 +140392,140392 +235131,140392 +140393,140393 +140394,140394 +140395,140395 +56649,140395 +48545,140395 +140396,140396 +140397,140397 +140398,140398 +140399,140399 +140400,140400 +140401,140401 +56695,140401 +140402,140402 +140403,140403 +140404,140404 +140405,140405 +140406,140406 +140407,140407 +140408,140408 +140409,140409 +140410,140410 +140411,140411 +140412,140412 +140413,140413 +140414,140414 +140415,140415 +140416,140416 +140417,140417 +140418,140418 +140419,140419 +142877,140419 +140420,140420 +236141,140420 +140421,140421 +140422,140422 +207307,140422 +189141,140422 +140423,140423 +142881,140423 +140424,140424 +140425,140425 +140426,140426 +140427,140427 +195571,140427 +140428,140428 +140429,140429 +140430,140430 +140431,140431 +140432,140432 +140433,140433 +142891,140433 +140434,140434 +140435,140435 +140436,140436 +140437,140437 +140438,140438 +140439,140439 +140440,140440 +140441,140441 +140442,140442 +140443,140443 +140444,140444 +140445,140445 +140446,140446 +140447,140447 +140448,140448 +140449,140449 +140450,140450 +140451,140451 +140452,140452 +235832,140452 +140453,140453 +183836,140453 +140454,140454 +140455,140455 +140456,140456 +140457,140457 +140458,140458 +140459,140459 +140460,140460 +140461,140461 +140462,140462 +142920,140462 +140463,140463 +140464,140464 +140465,140465 +140466,140466 +142924,140466 +140467,140467 +140468,140468 +140469,140469 +140470,140470 +140471,140471 +140472,140472 +140473,140473 +140474,140474 +140475,140475 +140476,140476 +140477,140477 +140478,140478 +140479,140479 +140480,140480 +35088,140480 +140481,140481 +140482,140482 +204471,140482 +140483,140483 +140484,140484 +176481,140484 +170517,140484 +140485,140485 +140486,140486 +63799,140486 +214552,140486 +140487,140487 +140488,140488 +140489,140489 +140490,140490 +140491,140491 +140492,140492 +199233,140492 +140493,140493 +140494,140494 +140495,140495 +140496,140496 +140497,140497 +140498,140498 +140499,140499 +140500,140500 +140501,140501 +120697,140501 +225378,140501 +140502,140502 +140503,140503 +140504,140504 +140505,140505 +140506,140506 +140507,140507 +140508,140508 +191599,140508 +140509,140509 +140510,140510 +30584,140510 +140511,140511 +140512,140512 +140513,140513 +140514,140514 +140515,140515 +140516,140516 +140517,140517 +140518,140518 +140519,140519 +140520,140520 +140521,140521 +140522,140522 +140523,140523 +140524,140524 +140525,140525 +140599,140599 +140600,140600 +140601,140601 +140602,140602 +140603,140603 +140604,140604 +140605,140605 +140606,140606 +140607,140607 +140608,140608 +140609,140609 +140610,140610 +140611,140611 +140612,140612 +140613,140613 +140614,140614 +140615,140615 +140616,140616 +140617,140617 +30524,140617 +140618,140618 +140619,140619 +140620,140620 +140621,140621 +140622,140622 +140623,140623 +140624,140624 +140625,140625 +216142,140625 +140626,140626 +140627,140627 +140628,140628 +232735,140628 +140629,140629 +140630,140630 +140631,140631 +140632,140632 +140633,140633 +140634,140634 +140635,140635 +140636,140636 +140637,140637 +140638,140638 +140639,140639 +136328,140639 +128741,140639 +60035,140639 +140640,140640 +140641,140641 +140642,140642 +140643,140643 +140644,140644 +140645,140645 +140646,140646 +140647,140647 +140648,140648 +17412,140648 +140649,140649 +213960,140649 +140650,140650 +140651,140651 +140652,140652 +140653,140653 +140654,140654 +140655,140655 +140656,140656 +140657,140657 +140658,140658 +140659,140659 +140660,140660 +140661,140661 +140662,140662 +140663,140663 +140664,140664 +140665,140665 +140666,140666 +140667,140667 +140668,140668 +140669,140669 +140670,140670 +33432,140670 +6476,140670 +140671,140671 +98320,140671 +140672,140672 +140673,140673 +140674,140674 +140675,140675 +140676,140676 +140677,140677 +140678,140678 +163564,140678 +140679,140679 +140680,140680 +140681,140681 +140682,140682 +140683,140683 +140684,140684 +51811,140684 +43707,140684 +140685,140685 +140686,140686 +140687,140687 +140688,140688 +140689,140689 +140690,140690 +140691,140691 +140692,140692 +140693,140693 +140694,140694 +242866,140694 +140695,140695 +140696,140696 +133993,140696 +126406,140696 +28943,140696 +140697,140697 +5732,140697 +202,140697 +140698,140698 +140699,140699 +140700,140700 +140701,140701 +140702,140702 +140703,140703 +140704,140704 +140705,140705 +140706,140706 +140707,140707 +103,140707 +140708,140708 +36622,140708 +30484,140708 +140709,140709 +140710,140710 +140711,140711 +140712,140712 +140713,140713 +140714,140714 +188217,140714 +140715,140715 +140754,140754 +140755,140755 +140756,140756 +140801,140801 +140802,140802 +140803,140803 +140804,140804 +140805,140805 +140806,140806 +140807,140807 +140808,140808 +140809,140809 +140810,140810 +140811,140811 +140812,140812 +140813,140813 +140814,140814 +140815,140815 +140816,140816 +140817,140817 +140818,140818 +140819,140819 +140820,140820 +140821,140821 +140822,140822 +140823,140823 +140824,140824 +140825,140825 +140826,140826 +140827,140827 +140828,140828 +140829,140829 +140830,140830 +140831,140831 +140832,140832 +140833,140833 +140834,140834 +140835,140835 +140836,140836 +140837,140837 +140838,140838 +140839,140839 +140840,140840 +140841,140841 +140842,140842 +140843,140843 +140844,140844 +140845,140845 +140846,140846 +140847,140847 +140848,140848 +140849,140849 +29877,140849 +140850,140850 +140851,140851 +140852,140852 +140853,140853 +140854,140854 +140855,140855 +165,140855 +140856,140856 +140857,140857 +140858,140858 +140859,140859 +140860,140860 +140861,140861 +140862,140862 +140863,140863 +140864,140864 +140865,140865 +140866,140866 +140867,140867 +140868,140868 +140869,140869 +140870,140870 +15743,140870 +140871,140871 +140872,140872 +140873,140873 +140874,140874 +140929,140929 +140930,140930 +140931,140931 +140932,140932 +140933,140933 +140934,140934 +140935,140935 +140936,140936 +140937,140937 +140938,140938 +140979,140979 +140980,140980 +132354,140980 +124767,140980 +140981,140981 +140982,140982 +140983,140983 +140984,140984 +140985,140985 +140986,140986 +140987,140987 +140988,140988 +140989,140989 +140990,140990 +140991,140991 +121442,140991 +140992,140992 +140993,140993 +140994,140994 +140995,140995 +59175,140995 +58801,140995 +140996,140996 +140997,140997 +140998,140998 +191274,140998 +140999,140999 +141000,141000 +141001,141001 +141002,141002 +141003,141003 +141004,141004 +141005,141005 +141006,141006 +141007,141007 +141008,141008 +141009,141009 +141010,141010 +141011,141011 +141012,141012 +102089,141012 +141013,141013 +141014,141014 +141015,141015 +141016,141016 +141017,141017 +141018,141018 +141019,141019 +141020,141020 +141021,141021 +235178,141021 +141022,141022 +141023,141023 +141024,141024 +141025,141025 +141134,141134 +141135,141135 +141136,141136 +141137,141137 +141138,141138 +231299,141139 +141139,141139 +141140,141140 +141141,141141 +141142,141142 +141143,141143 +141144,141144 +141145,141145 +141146,141146 +141147,141147 +141148,141148 +248754,141148 +141149,141149 +141150,141150 +141151,141151 +141152,141152 +141153,141153 +141154,141154 +141155,141155 +141156,141156 +141157,141157 +141158,141158 +141159,141159 +141160,141160 +141161,141161 +141162,141162 +141163,141163 +141164,141164 +141165,141165 +141166,141166 +141167,141167 +141168,141168 +141169,141169 +121476,141169 +141170,141170 +141171,141171 +141172,141172 +141173,141173 +141174,141174 +141175,141175 +141176,141176 +141177,141177 +141178,141178 +141179,141179 +179642,141179 +141180,141180 +141181,141181 +141182,141182 +141183,141183 +189564,141183 +141184,141184 +141185,141185 +141186,141186 +141187,141187 +141188,141188 +141189,141189 +141190,141190 +141191,141191 +180756,141192 +141192,141192 +142078,141192 +141193,141193 +141194,141194 +141195,141195 +141196,141196 +141197,141197 +141198,141198 +141199,141199 +141200,141200 +141201,141201 +141202,141202 +141203,141203 +141204,141204 +141205,141205 +141206,141206 +141207,141207 +141208,141208 +162347,141208 +141209,141209 +76410,141209 +141210,141210 +141211,141211 +141212,141212 +141213,141213 +141214,141214 +141215,141215 +141216,141216 +141217,141217 +141218,141218 +141219,141219 +141220,141220 +141221,141221 +141222,141222 +141223,141223 +141224,141224 +141225,141225 +141226,141226 +141227,141227 +141228,141228 +141229,141229 +141230,141230 +141231,141231 +141232,141232 +141233,141233 +141234,141234 +141235,141235 +141236,141236 +243338,141236 +141237,141237 +141238,141238 +141239,141239 +141240,141240 +141241,141241 +141242,141242 +141243,141243 +141244,141244 +72507,141244 +141245,141245 +141246,141246 +141247,141247 +141248,141248 +141249,141249 +141250,141250 +141251,141251 +141252,141252 +141253,141253 +141254,141254 +75396,141254 +141255,141255 +141256,141256 +141257,141257 +141258,141258 +141259,141259 +141260,141260 +141261,141261 +141262,141262 +141263,141263 +141264,141264 +54714,141264 +46610,141264 +141265,141265 +141266,141266 +141267,141267 +141268,141268 +141269,141269 +141270,141270 +34147,141270 +141271,141271 +141272,141272 +141273,141273 +114676,141273 +141274,141274 +141275,141275 +141276,141276 +141277,141277 +141278,141278 +141279,141279 +141280,141280 +141281,141281 +141282,141282 +141283,141283 +141284,141284 +141285,141285 +141286,141286 +141287,141287 +141288,141288 +141289,141289 +141290,141290 +141291,141291 +141292,141292 +141293,141293 +141294,141294 +141295,141295 +141296,141296 +4366,141296 +179023,141296 +141297,141297 +141298,141298 +141299,141299 +141300,141300 +141301,141301 +141302,141302 +141303,141303 +141304,141304 +141305,141305 +141306,141306 +141307,141307 +141308,141308 +141309,141309 +141310,141310 +141311,141311 +141312,141312 +141313,141313 +141314,141314 +141315,141315 +141316,141316 +141317,141317 +141318,141318 +141319,141319 +373,141319 +141320,141320 +141321,141321 +141322,141322 +141323,141323 +177707,141323 +171743,141323 +141324,141324 +141325,141325 +141326,141326 +141327,141327 +141328,141328 +141329,141329 +78937,141329 +141330,141330 +141331,141331 +141332,141332 +141333,141333 +141334,141334 +34234,141334 +233922,141334 +141335,141335 +141336,141336 +180006,141336 +141337,141337 +141338,141338 +141339,141339 +141340,141340 +141341,141341 +141342,141342 +141343,141343 +21525,141343 +141344,141344 +141345,141345 +35706,141345 +141346,141346 +141347,141347 +141348,141348 +141349,141349 +141350,141350 +141351,141351 +141352,141352 +141353,141353 +141354,141354 +183868,141355 +141355,141355 +141356,141356 +141357,141357 +141358,141358 +214676,141358 +141359,141359 +141360,141360 +210588,141360 +141361,141361 +141362,141362 +141363,141363 +234780,141363 +234845,141363 +141364,141364 +141365,141365 +141366,141366 +141367,141367 +36728,141367 +141368,141368 +141369,141369 +141370,141370 +141371,141371 +141372,141372 +208553,141372 +141373,141373 +141374,141374 +141375,141375 +141376,141376 +141377,141377 +141378,141378 +141379,141379 +141380,141380 +141381,141381 +141382,141382 +141383,141383 +141384,141384 +141385,141385 +141386,141386 +141387,141387 +141388,141388 +61762,141388 +141389,141389 +113547,141389 +141390,141390 +141391,141391 +141392,141392 +141393,141393 +141394,141394 +227084,141394 +141395,141395 +141396,141396 +112447,141396 +141397,141397 +163513,141397 +141398,141398 +141399,141399 +91466,141399 +141400,141400 +141401,141401 +141402,141402 +83054,141402 +141403,141403 +141404,141404 +141405,141405 +141406,141406 +91473,141406 +141407,141407 +141408,141408 +114760,141408 +141409,141409 +141410,141410 +36146,141410 +112461,141410 +141411,141411 +112462,141411 +141412,141412 +141413,141413 +141414,141414 +141415,141415 +141416,141416 +223406,141416 +141417,141417 +141418,141418 +141419,141419 +141420,141420 +141421,141421 +141422,141422 +133545,141422 +125958,141422 +162347,141422 +141423,141423 +141424,141424 +141425,141425 +141426,141426 +141427,141427 +141428,141428 +141429,141429 +141430,141430 +141431,141431 +141432,141432 +141433,141433 +141434,141434 +141435,141435 +141436,141436 +141437,141437 +141438,141438 +141439,141439 +141440,141440 +141441,141441 +122981,141441 +141442,141442 +141443,141443 +77412,141443 +141444,141444 +141445,141445 +141446,141446 +141447,141447 +141448,141448 +141449,141449 +141450,141450 +141451,141451 +141452,141452 +141453,141453 +141454,141454 +141455,141455 +141456,141456 +141457,141457 +141458,141458 +141459,141459 +141460,141460 +141461,141461 +141462,141462 +141463,141463 +141464,141464 +141465,141465 +141466,141466 +141467,141467 +141468,141468 +8630,141468 +141469,141469 +141470,141470 +3276,141470 +141471,141471 +141472,141472 +141473,141473 +141474,141474 +141475,141475 +141476,141476 +81856,141476 +141477,141477 +141478,141478 +143066,141478 +143067,141479 +141479,141479 +141480,141480 +141481,141481 +141482,141482 +88743,141482 +141483,141483 +141484,141484 +141485,141485 +141486,141486 +141487,141487 +141488,141488 +141489,141489 +141490,141490 +141491,141491 +141492,141492 +141493,141493 +141494,141494 +141495,141495 +141496,141496 +141497,141497 +141498,141498 +141499,141499 +141500,141500 +152917,141500 +141501,141501 +143089,141501 +141502,141502 +141503,141503 +141504,141504 +84089,141504 +141505,141505 +221180,141505 +141506,141506 +178625,141506 +141507,141507 +141508,141508 +141509,141509 +141510,141510 +141511,141511 +141512,141512 +141513,141513 +141514,141514 +242907,141514 +141515,141515 +141516,141516 +141517,141517 +141518,141518 +141519,141519 +141520,141520 +211510,141520 +141521,141521 +141522,141522 +141523,141523 +145602,141523 +141524,141524 +141525,141525 +141526,141526 +141527,141527 +81772,141527 +141528,141528 +141529,141529 +141530,141530 +141531,141531 +141532,141532 +141533,141533 +141534,141534 +141535,141535 +141536,141536 +141537,141537 +141538,141538 +141539,141539 +141540,141540 +141541,141541 +141542,141542 +141543,141543 +141544,141544 +163993,141544 +30215,141544 +141545,141545 +141546,141546 +141547,141547 +141548,141548 +143137,141549 +141549,141549 +3102,141549 +143138,141550 +141550,141550 +141551,141551 +141552,141552 +143141,141553 +141553,141553 +143142,141554 +141554,141554 +141555,141555 +143144,141556 +141556,141556 +141557,141557 +141558,141558 +141559,141559 +143148,141560 +141560,141560 +107302,141560 +143149,141561 +141561,141561 +141562,141562 +143150,141562 +141563,141563 +84097,141563 +141564,141564 +141565,141565 +141566,141566 +141567,141567 +141568,141568 +143156,141568 +141569,141569 +141570,141570 +155939,141570 +143158,141570 +141571,141571 +141572,141572 +141573,141573 +141574,141574 +141575,141575 +141576,141576 +141577,141577 +141578,141578 +141579,141579 +141580,141580 +141581,141581 +141582,141582 +141583,141583 +141584,141584 +141585,141585 +141586,141586 +141587,141587 +141588,141588 +141589,141589 +141590,141590 +141591,141591 +141592,141592 +141593,141593 +141594,141594 +141595,141595 +141596,141596 +141597,141597 +141598,141598 +141599,141599 +141600,141600 +141601,141601 +141602,141602 +141603,141603 +141604,141604 +141605,141605 +141606,141606 +141607,141607 +141608,141608 +111592,141608 +141609,141609 +141610,141610 +141611,141611 +141612,141612 +9086,141612 +141613,141613 +141614,141614 +141615,141615 +141616,141616 +141617,141617 +141618,141618 +141619,141619 +141620,141620 +141621,141621 +141622,141622 +141623,141623 +141624,141624 +141625,141625 +141626,141626 +141627,141627 +141628,141628 +141629,141629 +33525,141629 +141630,141630 +141631,141631 +141632,141632 +141633,141633 +141634,141634 +141635,141635 +141636,141636 +141637,141637 +161970,141637 +141638,141638 +141639,141639 +141640,141640 +143228,141640 +141641,141641 +141642,141642 +141643,141643 +143231,141643 +141644,141644 +158331,141644 +216497,141644 +141645,141645 +218661,141645 +141646,141646 +62852,141646 +141647,141647 +143235,141647 +141648,141648 +141649,141649 +143237,141649 +141650,141650 +56220,141650 +48116,141650 +141651,141651 +143239,141651 +141652,141652 +141653,141653 +141654,141654 +141655,141655 +143243,141655 +143244,141656 +141656,141656 +141657,141657 +141658,141658 +143246,141658 +141659,141659 +141660,141660 +141661,141661 +141662,141662 +141663,141663 +141664,141664 +141665,141665 +141666,141666 +143255,141667 +141667,141667 +141668,141668 +141669,141669 +141670,141670 +141671,141671 +141672,141672 +141673,141673 +141674,141674 +141675,141675 +141676,141676 +141677,141677 +141678,141678 +141679,141679 +141680,141680 +141681,141681 +141682,141682 +141683,141683 +141684,141684 +141685,141685 +141686,141686 +141687,141687 +141688,141688 +141689,141689 +139268,141689 +141690,141690 +141691,141691 +108729,141691 +141692,141692 +141693,141693 +141694,141694 +110542,141694 +141695,141695 +141696,141696 +141697,141697 +141698,141698 +141699,141699 +3582,141699 +141700,141700 +141701,141701 +141702,141702 +141703,141703 +141704,141704 +162325,141704 +141705,141705 +141706,141706 +141707,141707 +141708,141708 +143296,141708 +141709,141709 +141710,141710 +141711,141711 +143300,141712 +141712,141712 +143301,141713 +141713,141713 +143302,141714 +141714,141714 +143303,141715 +141715,141715 +143304,141716 +141716,141716 +141717,141717 +143305,141717 +141718,141718 +141719,141719 +141720,141720 +141721,141721 +141722,141722 +141723,141723 +141724,141724 +143312,141724 +141725,141725 +141726,141726 +141727,141727 +141728,141728 +141729,141729 +141730,141730 +141731,141731 +141732,141732 +141733,141733 +141734,141734 +141735,141735 +141736,141736 +141737,141737 +141738,141738 +141739,141739 +141740,141740 +141741,141741 +132379,141741 +124792,141741 +141742,141742 +141743,141743 +141744,141744 +141745,141745 +141746,141746 +141747,141747 +141748,141748 +141749,141749 +141750,141750 +141751,141751 +141752,141752 +141753,141753 +141754,141754 +141755,141755 +141756,141756 +141757,141757 +141758,141758 +141759,141759 +141760,141760 +141761,141761 +237948,141761 +141762,141762 +141763,141763 +141764,141764 +141765,141765 +141766,141766 +141767,141767 +141768,141768 +141769,141769 +141770,141770 +141771,141771 +141772,141772 +141773,141773 +141774,141774 +141775,141775 +141776,141776 +141777,141777 +141778,141778 +141779,141779 +141780,141780 +141781,141781 +141782,141782 +141783,141783 +141784,141784 +141785,141785 +141786,141786 +141787,141787 +141788,141788 +141789,141789 +141790,141790 +141791,141791 +61771,141791 +141792,141792 +141793,141793 +38136,141793 +37724,141793 +34632,141793 +141794,141794 +141795,141795 +141796,141796 +81319,141796 +141797,141797 +27090,141797 +141798,141798 +141799,141799 +141800,141800 +141801,141801 +141802,141802 +141803,141803 +141804,141804 +141805,141805 +141806,141806 +144840,141806 +141807,141807 +141808,141808 +141809,141809 +141810,141810 +141811,141811 +141812,141812 +141813,141813 +141814,141814 +141815,141815 +141816,141816 +141817,141817 +141818,141818 +141819,141819 +141820,141820 +206280,141820 +141821,141821 +141822,141822 +141823,141823 +141824,141824 +141825,141825 +141826,141826 +141827,141827 +141828,141828 +141829,141829 +141830,141830 +178331,141830 +172367,141830 +141831,141831 +141832,141832 +141833,141833 +141834,141834 +141835,141835 +141836,141836 +141837,141837 +141838,141838 +141839,141839 +141840,141840 +141841,141841 +141842,141842 +141843,141843 +141844,141844 +141845,141845 +141846,141846 +141847,141847 +97501,141847 +141848,141848 +141849,141849 +141850,141850 +141851,141851 +141852,141852 +141853,141853 +141854,141854 +141855,141855 +141856,141856 +141857,141857 +141858,141858 +141859,141859 +55571,141859 +47467,141859 +141860,141860 +141861,141861 +141862,141862 +141863,141863 +141864,141864 +141865,141865 +141866,141866 +141867,141867 +165306,141868 +141868,141868 +141869,141869 +141870,141870 +58427,141870 +141871,141871 +141872,141872 +141873,141873 +141874,141874 +100658,141874 +141875,141875 +178763,141875 +141876,141876 +141877,141877 +141878,141878 +38997,141878 +141879,141879 +141880,141880 +141881,141881 +141882,141882 +141883,141883 +141884,141884 +141885,141885 +59105,141885 +58731,141885 +141886,141886 +141887,141887 +141888,141888 +141889,141889 +164888,141890 +141890,141890 +141891,141891 +141892,141892 +141893,141893 +141894,141894 +141895,141895 +141896,141896 +141897,141897 +141898,141898 +141899,141899 +211326,141899 +141900,141900 +160162,141900 +141901,141901 +141902,141902 +141903,141903 +141904,141904 +141905,141905 +141906,141906 +141907,141907 +141908,141908 +141909,141909 +141910,141910 +141911,141911 +141912,141912 +141913,141913 +141914,141914 +141915,141915 +141916,141916 +141917,141917 +141918,141918 +141919,141919 +141920,141920 +141921,141921 +141922,141922 +141923,141923 +141924,141924 +141925,141925 +141926,141926 +141970,141970 +141971,141971 +141972,141972 +141973,141973 +141974,141974 +141975,141975 +141976,141976 +141977,141977 +141978,141978 +141979,141979 +141980,141980 +141981,141981 +141982,141982 +141983,141983 +141984,141984 +141985,141985 +141986,141986 +141987,141987 +141988,141988 +141989,141989 +141990,141990 +141991,141991 +141992,141992 +141993,141993 +141994,141994 +106907,141994 +141995,141995 +141996,141996 +141997,141997 +141998,141998 +141999,141999 +142000,142000 +142001,142001 +142002,142002 +142003,142003 +142004,142004 +142005,142005 +142006,142006 +210282,142006 +142007,142007 +142008,142008 +142009,142009 +142010,142010 +104742,142010 +142011,142011 +142012,142012 +142013,142013 +142014,142014 +142015,142015 +142016,142016 +142017,142017 +123279,142017 +142018,142018 +142019,142019 +188368,142019 +142020,142020 +142021,142021 +142022,142022 +142023,142023 +142024,142024 +142025,142025 +142026,142026 +142027,142027 +142028,142028 +142029,142029 +142030,142030 +142031,142031 +142032,142032 +142033,142033 +142034,142034 +142035,142035 +142036,142036 +142037,142037 +142038,142038 +142039,142039 +142040,142040 +142041,142041 +154862,142041 +142042,142042 +142043,142043 +142044,142044 +142045,142045 +142046,142046 +142047,142047 +142048,142048 +142049,142049 +142050,142050 +142051,142051 +209523,142051 +142052,142052 +142053,142053 +142054,142054 +142055,142055 +142056,142056 +142057,142057 +142058,142058 +142059,142059 +142060,142060 +142061,142061 +142062,142062 +142063,142063 +142064,142064 +142065,142065 +142066,142066 +142067,142067 +142068,142068 +142069,142069 +142070,142070 +142071,142071 +142072,142072 +164770,142072 +142073,142073 +142074,142074 +24014,142074 +142075,142075 +142076,142076 +142077,142077 +142078,142078 +180756,142078 +141192,142078 +142079,142079 +142080,142080 +142081,142081 +142082,142082 +142083,142083 +142084,142084 +142085,142085 +142086,142086 +142087,142087 +142088,142088 +142089,142089 +156290,142089 +142090,142090 +142091,142091 +142092,142092 +142093,142093 +142094,142094 +142095,142095 +142096,142096 +142097,142097 +142098,142098 +142099,142099 +142100,142100 +142101,142101 +142102,142102 +142103,142103 +4559,142103 +142104,142104 +142105,142105 +86423,142105 +142106,142106 +142107,142107 +142108,142108 +142109,142109 +142110,142110 +142111,142111 +142112,142112 +142113,142113 +142114,142114 +142115,142115 +142116,142116 +142117,142117 +142118,142118 +142119,142119 +132498,142119 +124911,142119 +59777,142119 +142120,142120 +142121,142121 +142122,142122 +142123,142123 +142124,142124 +142125,142125 +142126,142126 +142127,142127 +142128,142128 +142129,142129 +142130,142130 +142131,142131 +142132,142132 +142133,142133 +142134,142134 +142135,142135 +142136,142136 +142137,142137 +142138,142138 +163417,142138 +157592,142138 +142139,142139 +142140,142140 +142141,142141 +142142,142142 +142143,142143 +142144,142144 +142145,142145 +142146,142146 +142147,142147 +142148,142148 +142149,142149 +6297,142149 +142150,142150 +142151,142151 +142152,142152 +142153,142153 +142154,142154 +142155,142155 +142156,142156 +142157,142157 +142158,142158 +142159,142159 +142160,142160 +142161,142161 +142162,142162 +142163,142163 +142164,142164 +142165,142165 +231778,142165 +142166,142166 +189134,142166 +142167,142167 +142168,142168 +142169,142169 +142170,142170 +142171,142171 +142172,142172 +142173,142173 +144252,142173 +142174,142174 +142175,142175 +107003,142175 +142176,142176 +142177,142177 +142178,142178 +142179,142179 +142180,142180 +142181,142181 +142182,142182 +142183,142183 +142184,142184 +142185,142185 +142186,142186 +142187,142187 +145167,142187 +142188,142188 +142189,142189 +142190,142190 +142191,142191 +57508,142191 +142192,142192 +155932,142192 +142193,142193 +142194,142194 +142195,142195 +142196,142196 +142197,142197 +142198,142198 +142199,142199 +142200,142200 +142201,142201 +142202,142202 +142203,142203 +36913,142203 +142204,142204 +142205,142205 +84230,142205 +142206,142206 +142207,142207 +142208,142208 +142209,142209 +142210,142210 +142211,142211 +142212,142212 +142213,142213 +145193,142213 +142214,142214 +145064,142214 +142215,142215 +142216,142216 +142217,142217 +142218,142218 +179979,142218 +142219,142219 +142220,142220 +142221,142221 +142222,142222 +142223,142223 +142224,142224 +142225,142225 +142226,142226 +142227,142227 +154872,142227 +142228,142228 +142229,142229 +233453,142229 +142230,142230 +142231,142231 +142232,142232 +142233,142233 +209899,142233 +142234,142234 +142235,142235 +207261,142235 +3100,142235 +142236,142236 +142237,142237 +142238,142238 +142239,142239 +142240,142240 +142241,142241 +142242,142242 +142243,142243 +142244,142244 +142245,142245 +142246,142246 +142247,142247 +142248,142248 +142249,142249 +142250,142250 +142251,142251 +142252,142252 +142253,142253 +142254,142254 +142255,142255 +142256,142256 +142257,142257 +142258,142258 +142259,142259 +142260,142260 +142261,142261 +142262,142262 +142263,142263 +142264,142264 +142265,142265 +142266,142266 +142267,142267 +142268,142268 +142269,142269 +142270,142270 +142271,142271 +142272,142272 +191029,142272 +142273,142273 +142274,142274 +142275,142275 +142276,142276 +142277,142277 +142278,142278 +142279,142279 +145260,142280 +142280,142280 +142281,142281 +600,142281 +145261,142281 +142282,142282 +145262,142282 +142283,142283 +145263,142283 +142284,142284 +142285,142285 +142286,142286 +145266,142286 +145267,142287 +142287,142287 +145268,142288 +142288,142288 +142289,142289 +145269,142289 +142290,142290 +145270,142290 +142291,142291 +145271,142291 +142292,142292 +142293,142293 +145273,142293 +142294,142294 +142295,142295 +142296,142296 +76633,142296 +142297,142297 +142298,142298 +142299,142299 +142300,142300 +142301,142301 +142302,142302 +142303,142303 +142340,142340 +142341,142341 +142342,142342 +142343,142343 +142344,142344 +142345,142345 +145326,142346 +142346,142346 +142347,142347 +142348,142348 +145328,142348 +142349,142349 +142350,142350 +142351,142351 +185287,142351 +39735,142351 +142352,142352 +142353,142353 +145333,142353 +142354,142354 +142355,142355 +142356,142356 +142357,142357 +142358,142358 +65020,142358 +142359,142359 +142360,142360 +180372,142360 +142361,142361 +142362,142362 +142363,142363 +142364,142364 +142365,142365 +3513,142365 +142366,142366 +142367,142367 +142368,142368 +142369,142369 +142370,142370 +142371,142371 +142372,142372 +142373,142373 +145353,142373 +142374,142374 +142375,142375 +142376,142376 +142377,142377 +225173,142377 +142378,142378 +142379,142379 +142380,142380 +142381,142381 +178600,142382 +142382,142382 +142383,142383 +142422,142422 +142423,142423 +142424,142424 +142425,142425 +142426,142426 +67281,142426 +142427,142427 +142428,142428 +142429,142429 +142430,142430 +142431,142431 +142432,142432 +142433,142433 +142434,142434 +142435,142435 +142436,142436 +142437,142437 +142438,142438 +142439,142439 +142440,142440 +142441,142441 +160782,142441 +142442,142442 +142443,142443 +142444,142444 +142445,142445 +142446,142446 +142447,142447 +142448,142448 +164087,142448 +142449,142449 +142450,142450 +142451,142451 +142452,142452 +142453,142453 +142454,142454 +142455,142455 +142456,142456 +142457,142457 +142458,142458 +142459,142459 +142460,142460 +142461,142461 +142462,142462 +8878,142462 +142463,142463 +142464,142464 +142465,142465 +132397,142465 +124810,142465 +142466,142466 +154933,142466 +142467,142467 +234320,142468 +142468,142468 +142469,142469 +142470,142470 +142471,142471 +142472,142472 +142473,142473 +142474,142474 +142475,142475 +15731,142475 +142476,142476 +142477,142477 +145473,142477 +142478,142478 +142479,142479 +142480,142480 +142481,142481 +145477,142481 +142482,142482 +145478,142482 +142483,142483 +142484,142484 +142485,142485 +142486,142486 +2763,142486 +142487,142487 +142488,142488 +142489,142489 +113360,142489 +142490,142490 +111744,142490 +142491,142491 +244206,142492 +142492,142492 +145488,142492 +142493,142493 +190873,142493 +142494,142494 +140143,142494 +142495,142495 +142496,142496 +145492,142496 +142497,142497 +140146,142497 +142498,142498 +140147,142498 +142499,142499 +140148,142499 +246755,142499 +142500,142500 +140149,142500 +142501,142501 +142502,142502 +142503,142503 +140152,142503 +142504,142504 +140153,142504 +142505,142505 +140154,142505 +142506,142506 +140155,142506 +142507,142507 +140156,142507 +142508,142508 +140157,142508 +142509,142509 +140158,142509 +142510,142510 +140159,142510 +142511,142511 +140160,142511 +142512,142512 +140161,142512 +142513,142513 +140162,142513 +142514,142514 +142515,142515 +142516,142516 +142517,142517 +142518,142518 +142519,142519 +142520,142520 +142521,142521 +142522,142522 +142523,142523 +140172,142523 +142524,142524 +142525,142525 +142526,142526 +142527,142527 +140176,142527 +142528,142528 +142529,142529 +142530,142530 +142531,142531 +142532,142532 +142533,142533 +142534,142534 +142535,142535 +142536,142536 +142537,142537 +142538,142538 +142539,142539 +87596,142539 +142540,142540 +142541,142541 +199722,142541 +142542,142542 +142543,142543 +142544,142544 +142545,142545 +142546,142546 +142547,142547 +142548,142548 +142549,142549 +142550,142550 +142551,142551 +142552,142552 +142553,142553 +142554,142554 +142555,142555 +142556,142556 +142557,142557 +28413,142557 +142558,142558 +142559,142559 +57958,142559 +142560,142560 +185456,142560 +142561,142561 +135279,142561 +127692,142561 +142562,142562 +142563,142563 +142564,142564 +142565,142565 +142566,142566 +142567,142567 +111099,142567 +142568,142568 +142569,142569 +142570,142570 +7750,142570 +142571,142571 +142572,142572 +142573,142573 +142574,142574 +153300,142574 +142575,142575 +142576,142576 +142577,142577 +142578,142578 +142579,142579 +142580,142580 +142581,142581 +142582,142582 +142583,142583 +142584,142584 +142585,142585 +142586,142586 +142587,142587 +142588,142588 +142589,142589 +142590,142590 +159508,142591 +142591,142591 +142592,142592 +934,142592 +142593,142593 +160631,142593 +142594,142594 +142595,142595 +142596,142596 +142597,142597 +63149,142597 +142598,142598 +142599,142599 +142600,142600 +142601,142601 +142602,142602 +53690,142602 +45586,142602 +33741,142602 +2712,142602 +142603,142603 +142604,142604 +142605,142605 +142606,142606 +54691,142606 +46587,142606 +142607,142607 +142608,142608 +142609,142609 +142610,142610 +142611,142611 +142612,142612 +142613,142613 +142614,142614 +142615,142615 +142616,142616 +142617,142617 +142618,142618 +216068,142618 +142619,142619 +142620,142620 +142621,142621 +142622,142622 +142623,142623 +142624,142624 +142625,142625 +142626,142626 +142627,142627 +86828,142627 +142628,142628 +142629,142629 +142630,142630 +142631,142631 +185516,142632 +142632,142632 +142633,142633 +189958,142633 +142634,142634 +142635,142635 +235199,142635 +142636,142636 +142637,142637 +142638,142638 +206640,142639 +142639,142639 +142640,142640 +227758,142640 +142641,142641 +142642,142642 +142643,142643 +142644,142644 +142645,142645 +76277,142645 +142646,142646 +142647,142647 +142648,142648 +142649,142649 +142650,142650 +142651,142651 +142652,142652 +142653,142653 +142654,142654 +142655,142655 +142656,142656 +142657,142657 +142658,142658 +142659,142659 +142660,142660 +142661,142661 +142662,142662 +142663,142663 +161866,142663 +142664,142664 +142665,142665 +142666,142666 +57057,142666 +142667,142667 +142668,142668 +142669,142669 +207338,142670 +142670,142670 +180274,142670 +142671,142671 +142672,142672 +242380,142672 +142673,142673 +3667,142673 +142674,142674 +142675,142675 +142676,142676 +142677,142677 +142678,142678 +142679,142679 +142680,142680 +142681,142681 +142682,142682 +142683,142683 +142684,142684 +142685,142685 +142686,142686 +187749,142686 +142687,142687 +142688,142688 +142689,142689 +142690,142690 +142691,142691 +142692,142692 +134208,142692 +126621,142692 +142693,142693 +142694,142694 +139934,142694 +142695,142695 +6698,142695 +142696,142696 +142697,142697 +142698,142698 +142699,142699 +142700,142700 +142701,142701 +142702,142702 +142703,142703 +142704,142704 +142743,142743 +142744,142744 +142745,142745 +142746,142746 +142747,142747 +142748,142748 +187327,142748 +77183,142748 +193405,142748 +142749,142749 +142750,142750 +142751,142751 +142752,142752 +142753,142753 +142754,142754 +142755,142755 +142756,142756 +142757,142757 +142758,142758 +142759,142759 +142760,142760 +142761,142761 +142762,142762 +15609,142762 +142763,142763 +142764,142764 +142765,142765 +142766,142766 +142767,142767 +142768,142768 +142769,142769 +142770,142770 +142771,142771 +142772,142772 +142773,142773 +142774,142774 +142775,142775 +142776,142776 +142777,142777 +142778,142778 +142779,142779 +142780,142780 +231708,142780 +142781,142781 +142782,142782 +142783,142783 +142784,142784 +142785,142785 +65152,142785 +142786,142786 +142787,142787 +142788,142788 +142789,142789 +142790,142790 +142791,142791 +142792,142792 +142793,142793 +142794,142794 +142795,142795 +142796,142796 +142797,142797 +142798,142798 +142799,142799 +192562,142799 +23583,142799 +142800,142800 +142801,142801 +207536,142801 +142802,142802 +142803,142803 +142804,142804 +27576,142804 +142805,142805 +142806,142806 +142807,142807 +142808,142808 +142809,142809 +142810,142810 +142811,142811 +147053,142811 +142812,142812 +142813,142813 +142814,142814 +142815,142815 +142816,142816 +142855,142855 +132369,142855 +124782,142855 +142856,142856 +142857,142857 +142858,142858 +142859,142859 +142860,142860 +142861,142861 +142862,142862 +132360,142862 +124773,142862 +142863,142863 +142864,142864 +242535,142864 +142865,142865 +142866,142866 +142867,142867 +142868,142868 +142869,142869 +142870,142870 +142871,142871 +142872,142872 +142873,142873 +142874,142874 +142875,142875 +142876,142876 +142877,142877 +140419,142877 +142878,142878 +202777,142878 +142879,142879 +142880,142880 +142881,142881 +140423,142881 +142882,142882 +142883,142883 +142884,142884 +106529,142884 +142885,142885 +142886,142886 +142887,142887 +142888,142888 +6615,142888 +13461,142888 +142889,142889 +142890,142890 +233132,142890 +142891,142891 +140433,142891 +142892,142892 +142893,142893 +142894,142894 +142895,142895 +142896,142896 +142897,142897 +142898,142898 +142899,142899 +142900,142900 +142901,142901 +142902,142902 +142903,142903 +142904,142904 +142905,142905 +142906,142906 +225468,142906 +142907,142907 +142908,142908 +142909,142909 +142910,142910 +142911,142911 +142912,142912 +142913,142913 +213609,142913 +142914,142914 +142915,142915 +142916,142916 +142917,142917 +142918,142918 +142919,142919 +142920,142920 +102186,142920 +140462,142920 +142921,142921 +142922,142922 +142923,142923 +142924,142924 +140466,142924 +201132,142925 +142925,142925 +142926,142926 +142927,142927 +142928,142928 +105870,142928 +142929,142929 +142930,142930 +142931,142931 +142932,142932 +142933,142933 +142934,142934 +142935,142935 +142936,142936 +142937,142937 +142938,142938 +142939,142939 +142940,142940 +142941,142941 +142942,142942 +142943,142943 +142944,142944 +142945,142945 +142946,142946 +206854,142946 +142947,142947 +211157,142947 +142948,142948 +142949,142949 +142950,142950 +142951,142951 +142952,142952 +142953,142953 +142954,142954 +142955,142955 +142956,142956 +142957,142957 +49616,142957 +41423,142957 +142958,142958 +142959,142959 +142960,142960 +142961,142961 +142962,142962 +142963,142963 +142964,142964 +142965,142965 +142966,142966 +142967,142967 +142968,142968 +142969,142969 +142970,142970 +142971,142971 +142972,142972 +142973,142973 +29876,142973 +142974,142974 +142975,142975 +142976,142976 +142977,142977 +142978,142978 +142979,142979 +142980,142980 +142981,142981 +142982,142982 +142983,142983 +142984,142984 +142985,142985 +142986,142986 +142987,142987 +142988,142988 +142989,142989 +142990,142990 +142991,142991 +142992,142992 +142993,142993 +142994,142994 +142995,142995 +142996,142996 +142997,142997 +142998,142998 +142999,142999 +143000,143000 +143001,143001 +143002,143002 +143003,143003 +143004,143004 +143005,143005 +143006,143006 +143007,143007 +143008,143008 +143009,143009 +143010,143010 +110806,143010 +143011,143011 +110807,143011 +143012,143012 +143013,143013 +143014,143014 +243338,143014 +143015,143015 +143016,143016 +14632,143016 +143017,143017 +143018,143018 +143019,143019 +143020,143020 +143021,143021 +143022,143022 +143023,143023 +143024,143024 +143025,143025 +143026,143026 +143027,143027 +143028,143028 +143029,143029 +143030,143030 +143031,143031 +143032,143032 +143033,143033 +143034,143034 +143035,143035 +143036,143036 +143037,143037 +143038,143038 +143039,143039 +143040,143040 +85878,143040 +143041,143041 +143042,143042 +143043,143043 +143044,143044 +235697,143044 +143045,143045 +111320,143045 +143046,143046 +143047,143047 +17151,143047 +143048,143048 +143049,143049 +143050,143050 +82580,143050 +143051,143051 +143052,143052 +143053,143053 +143054,143054 +143055,143055 +143056,143056 +143057,143057 +143058,143058 +143059,143059 +201163,143059 +143060,143060 +143061,143061 +143062,143062 +143063,143063 +216547,143063 +143064,143064 +143065,143065 +143066,143066 +141478,143066 +143067,143067 +141479,143067 +143068,143068 +143069,143069 +246627,143069 +143070,143070 +191667,143071 +143071,143071 +143072,143072 +143073,143073 +143074,143074 +143075,143075 +143076,143076 +143077,143077 +143078,143078 +86752,143078 +143079,143079 +143080,143080 +143081,143081 +143082,143082 +143083,143083 +229559,143083 +143084,143084 +143085,143085 +143086,143086 +143087,143087 +143088,143088 +143089,143089 +141501,143089 +143090,143090 +143091,143091 +143092,143092 +143093,143093 +143094,143094 +143095,143095 +143135,143135 +143136,143136 +143137,143137 +141549,143137 +3102,143137 +143138,143138 +141550,143138 +143139,143139 +143140,143140 +143141,143141 +141553,143141 +143142,143142 +141554,143142 +143143,143143 +143144,143144 +141556,143144 +143145,143145 +143146,143146 +143147,143147 +143148,143148 +141560,143148 +107302,143148 +143149,143149 +141561,143149 +143150,143150 +141562,143150 +143151,143151 +84097,143151 +143152,143152 +191422,143152 +143153,143153 +143154,143154 +143155,143155 +143156,143156 +141568,143156 +143157,143157 +143158,143158 +141570,143158 +143159,143159 +143160,143160 +143161,143161 +143162,143162 +143163,143163 +220294,143163 +143164,143164 +143165,143165 +88229,143165 +143166,143166 +143167,143167 +143168,143168 +143169,143169 +143170,143170 +309,143170 +143171,143171 +15404,143171 +143172,143172 +143173,143173 +143174,143174 +143175,143175 +143176,143176 +143177,143177 +143178,143178 +143179,143179 +143180,143180 +143181,143181 +143182,143182 +143183,143183 +143184,143184 +143185,143185 +143186,143186 +143187,143187 +22068,143187 +143188,143188 +143189,143189 +143190,143190 +143191,143191 +143192,143192 +4288,143192 +143193,143193 +143194,143194 +143195,143195 +143196,143196 +143197,143197 +143198,143198 +143199,143199 +143200,143200 +143201,143201 +143202,143202 +143203,143203 +143204,143204 +143205,143205 +143206,143206 +143207,143207 +242351,143207 +143208,143208 +143209,143209 +243409,143209 +143210,143210 +143211,143211 +143212,143212 +143213,143213 +7605,143213 +143214,143214 +143215,143215 +158521,143215 +143216,143216 +143217,143217 +143218,143218 +143219,143219 +143220,143220 +37120,143220 +143221,143221 +143222,143222 +143223,143223 +143224,143224 +143225,143225 +143226,143226 +143227,143227 +143228,143228 +141640,143228 +143229,143229 +143230,143230 +143231,143231 +141643,143231 +184284,143231 +143232,143232 +69820,143232 +143233,143233 +143234,143234 +143235,143235 +141647,143235 +143236,143236 +143237,143237 +141649,143237 +143238,143238 +143239,143239 +141651,143239 +143240,143240 +143241,143241 +143242,143242 +143243,143243 +141655,143243 +143244,143244 +141656,143244 +143245,143245 +7973,143245 +143246,143246 +141658,143246 +143247,143247 +143248,143248 +143249,143249 +143250,143250 +143251,143251 +143252,143252 +143253,143253 +143254,143254 +143255,143255 +141667,143255 +143256,143256 +143257,143257 +203678,143257 +143258,143258 +143259,143259 +143260,143260 +143261,143261 +143262,143262 +143263,143263 +143264,143264 +143265,143265 +143266,143266 +143267,143267 +143268,143268 +143269,143269 +143270,143270 +143271,143271 +143272,143272 +143273,143273 +143274,143274 +143275,143275 +143276,143276 +143277,143277 +143278,143278 +143279,143279 +143280,143280 +220200,143280 +163054,143280 +143281,143281 +143282,143282 +143283,143283 +56081,143283 +47977,143283 +143284,143284 +143285,143285 +143286,143286 +143287,143287 +143288,143288 +143289,143289 +143290,143290 +143291,143291 +143292,143292 +143293,143293 +143294,143294 +143295,143295 +143296,143296 +141708,143296 +143297,143297 +143298,143298 +143299,143299 +143300,143300 +141712,143300 +143301,143301 +141713,143301 +143302,143302 +141714,143302 +143303,143303 +141715,143303 +143304,143304 +141716,143304 +143305,143305 +141717,143305 +143306,143306 +143307,143307 +143308,143308 +143309,143309 +143310,143310 +143311,143311 +143312,143312 +141724,143312 +143313,143313 +143314,143314 +9911,143314 +143315,143315 +143316,143316 +143317,143317 +143318,143318 +143319,143319 +143320,143320 +143321,143321 +143322,143322 +143323,143323 +143324,143324 +143325,143325 +143326,143326 +143327,143327 +143328,143328 +143329,143329 +143330,143330 +143331,143331 +78706,143331 +143332,143332 +143333,143333 +143334,143334 +143335,143335 +143336,143336 +143337,143337 +143338,143338 +143339,143339 +143340,143340 +143341,143341 +143342,143342 +143343,143343 +240079,143343 +143344,143344 +228052,143344 +143345,143345 +87885,143345 +143346,143346 +143347,143347 +143348,143348 +143349,143349 +143350,143350 +143351,143351 +143352,143352 +143353,143353 +143354,143354 +143355,143355 +143356,143356 +143357,143357 +143358,143358 +143359,143359 +143360,143360 +143361,143361 +143362,143362 +143363,143363 +143364,143364 +143365,143365 +143366,143366 +143367,143367 +75985,143367 +143368,143368 +143369,143369 +143686,143686 +143687,143687 +143688,143688 +143689,143689 +143690,143690 +143691,143691 +143692,143692 +143693,143693 +68053,143693 +143694,143694 +143695,143695 +143696,143696 +162437,143696 +143697,143697 +143698,143698 +143699,143699 +143700,143700 +143701,143701 +143702,143702 +143703,143703 +143704,143704 +143705,143705 +143706,143706 +143707,143707 +143708,143708 +143709,143709 +143710,143710 +143711,143711 +143712,143712 +143713,143713 +143714,143714 +143715,143715 +143716,143716 +143717,143717 +143718,143718 +143719,143719 +143720,143720 +143721,143721 +143722,143722 +143882,143882 +143883,143883 +143884,143884 +143885,143885 +143886,143886 +143887,143887 +248163,143887 +143888,143888 +143889,143889 +143890,143890 +143891,143891 +143892,143892 +143893,143893 +143894,143894 +143895,143895 +143896,143896 +143897,143897 +143898,143898 +143899,143899 +143900,143900 +143901,143901 +143902,143902 +143903,143903 +218409,143904 +143904,143904 +143905,143905 +26383,143905 +143906,143906 +143907,143907 +143908,143908 +143909,143909 +15748,143909 +143910,143910 +146334,143910 +40,143910 +143911,143911 +159229,143911 +2681,143911 +143912,143912 +143959,143959 +143960,143960 +143961,143961 +143962,143962 +143963,143963 +143964,143964 +143965,143965 +143966,143966 +143967,143967 +143968,143968 +143991,143969 +143969,143969 +143970,143970 +143993,143971 +143971,143971 +143994,143972 +143972,143972 +144015,143972 +144016,143973 +143973,143973 +143974,143974 +143975,143975 +143976,143976 +143977,143977 +143978,143978 +143979,143979 +52747,143979 +44643,143979 +143980,143980 +143981,143981 +143982,143982 +143983,143983 +143984,143984 +143985,143985 +143986,143986 +143987,143987 +143988,143988 +143989,143989 +143990,143990 +143991,143991 +143969,143991 +143992,143992 +143993,143993 +143971,143993 +143994,143994 +143972,143994 +144015,143994 +143995,143995 +143996,143996 +143997,143997 +143998,143998 +143999,143999 +144000,144000 +22035,144000 +144001,144001 +144002,144002 +144003,144003 +144004,144004 +144005,144005 +144006,144006 +144007,144007 +144008,144008 +144009,144009 +144010,144010 +144011,144011 +144012,144012 +144013,144013 +144014,144014 +144015,144015 +143994,144015 +143972,144015 +144016,144016 +143973,144016 +144017,144017 +144018,144018 +144019,144019 +144020,144020 +144021,144021 +79548,144021 +144022,144022 +144023,144023 +144024,144024 +144025,144025 +144026,144026 +144027,144027 +144028,144028 +8990,144028 +8528,144028 +144029,144029 +144030,144030 +15418,144030 +144031,144031 +144032,144032 +144033,144033 +144034,144034 +144035,144035 +144036,144036 +11902,144036 +144037,144037 +144038,144038 +144039,144039 +144040,144040 +144041,144041 +144042,144042 +144043,144043 +144044,144044 +144045,144045 +144046,144046 +144047,144047 +234818,144047 +144048,144048 +144049,144049 +144050,144050 +144051,144051 +144052,144052 +144053,144053 +144054,144054 +144055,144055 +144056,144056 +144057,144057 +144058,144058 +144059,144059 +64973,144059 +144060,144060 +144061,144061 +144062,144062 +144063,144063 +144064,144064 +144065,144065 +144066,144066 +144067,144067 +144068,144068 +144069,144069 +144070,144070 +144071,144071 +106126,144071 +144072,144072 +206900,144072 +144073,144073 +144074,144074 +144075,144075 +144076,144076 +144077,144077 +144078,144078 +144079,144079 +144080,144080 +144081,144081 +144082,144082 +144083,144083 +28938,144083 +144084,144084 +140046,144084 +144085,144085 +144086,144086 +144087,144087 +144088,144088 +144089,144089 +144090,144090 +144091,144091 +144092,144092 +144093,144093 +144094,144094 +144095,144095 +144096,144096 +218063,144096 +144097,144097 +144098,144098 +144099,144099 +144100,144100 +144101,144101 +144147,144147 +144148,144148 +144149,144149 +144150,144150 +144151,144151 +103623,144151 +144152,144152 +144153,144153 +144154,144154 +144155,144155 +144156,144156 +144157,144157 +144158,144158 +144159,144159 +144160,144160 +144161,144161 +144162,144162 +144163,144163 +144164,144164 +144165,144165 +144166,144166 +144167,144167 +144168,144168 +144169,144169 +144170,144170 +144171,144171 +144172,144172 +144173,144173 +144174,144174 +144175,144175 +144176,144176 +144177,144177 +144178,144178 +144179,144179 +144180,144180 +144181,144181 +144182,144182 +144183,144183 +144184,144184 +144185,144185 +211789,144185 +144186,144186 +144187,144187 +144188,144188 +144189,144189 +144190,144190 +144191,144191 +144192,144192 +144193,144193 +149432,144193 +144194,144194 +144195,144195 +12145,144195 +144196,144196 +144197,144197 +144198,144198 +144199,144199 +144200,144200 +144201,144201 +24685,144201 +144202,144202 +144203,144203 +144204,144204 +144205,144205 +144206,144206 +144207,144207 +144208,144208 +144209,144209 +144210,144210 +144211,144211 +144212,144212 +144213,144213 +144214,144214 +144215,144215 +144216,144216 +144217,144217 +144218,144218 +217590,144218 +144219,144219 +144220,144220 +144221,144221 +144222,144222 +144223,144223 +144224,144224 +144225,144225 +144226,144226 +144227,144227 +144228,144228 +144229,144229 +144230,144230 +144231,144231 +144232,144232 +144233,144233 +144234,144234 +144235,144235 +144236,144236 +144237,144237 +144238,144238 +240904,144238 +144239,144239 +144240,144240 +144241,144241 +144242,144242 +144243,144243 +144244,144244 +158673,144244 +144245,144245 +144246,144246 +144247,144247 +144248,144248 +94956,144248 +144249,144249 +144250,144250 +144251,144251 +144252,144252 +142173,144252 +144253,144253 +144254,144254 +144255,144255 +144256,144256 +144257,144257 +144258,144258 +144259,144259 +144260,144260 +144261,144261 +144262,144262 +144263,144263 +92549,144263 +144264,144264 +144265,144265 +144266,144266 +144267,144267 +144268,144268 +144269,144269 +144270,144270 +144271,144271 +144272,144272 +189559,144272 +144273,144273 +144274,144274 +144275,144275 +144276,144276 +144277,144277 +144278,144278 +144279,144279 +144280,144280 +144281,144281 +144282,144282 +144283,144283 +144284,144284 +176264,144284 +170300,144284 +144334,144334 +144335,144335 +144336,144336 +144337,144337 +144338,144338 +144339,144339 +144340,144340 +144341,144341 +144342,144342 +144343,144343 +144344,144344 +144345,144345 +144346,144346 +144347,144347 +144348,144348 +144388,144388 +144389,144389 +144390,144390 +144433,144433 +144434,144434 +144435,144435 +144436,144436 +144437,144437 +144438,144438 +144439,144439 +144440,144440 +144441,144441 +144442,144442 +144443,144443 +144444,144444 +144445,144445 +180980,144445 +144446,144446 +144447,144447 +144448,144448 +144449,144449 +144450,144450 +144451,144451 +144452,144452 +144453,144453 +144454,144454 +144455,144455 +144456,144456 +144457,144457 +144458,144458 +144459,144459 +144460,144460 +144461,144461 +144678,144678 +144716,144716 +144717,144717 +144718,144718 +144719,144719 +86369,144719 +144720,144720 +144839,144839 +117822,144839 +243849,144839 +144840,144840 +141806,144840 +144841,144841 +177801,144841 +171837,144841 +144842,144842 +144843,144843 +144844,144844 +144845,144845 +144846,144846 +144847,144847 +144848,144848 +144849,144849 +144850,144850 +144851,144851 +144852,144852 +144853,144853 +85652,144853 +144854,144854 +144855,144855 +144856,144856 +144857,144857 +144858,144858 +144859,144859 +144860,144860 +144861,144861 +144862,144862 +144863,144863 +144864,144864 +144865,144865 +144866,144866 +144867,144867 +144868,144868 +144869,144869 +144870,144870 +144871,144871 +144872,144872 +144873,144873 +144874,144874 +144875,144875 +144876,144876 +144877,144877 +144878,144878 +144879,144879 +144880,144880 +144881,144881 +144882,144882 +144883,144883 +144884,144884 +144885,144885 +144886,144886 +144887,144887 +226690,144887 +144888,144888 +144889,144889 +144890,144890 +144891,144891 +144892,144892 +144893,144893 +144894,144894 +144895,144895 +144896,144896 +144897,144897 +144898,144898 +144899,144899 +144900,144900 +144901,144901 +144902,144902 +144903,144903 +144904,144904 +144905,144905 +144906,144906 +144907,144907 +144908,144908 +144909,144909 +144910,144910 +144911,144911 +144912,144912 +241103,144912 +144913,144913 +144914,144914 +144915,144915 +144916,144916 +144917,144917 +144918,144918 +144919,144919 +144920,144920 +144921,144921 +144922,144922 +144923,144923 +144924,144924 +144925,144925 +144926,144926 +144927,144927 +144928,144928 +144929,144929 +144930,144930 +144931,144931 +144932,144932 +148210,144932 +144933,144933 +144934,144934 +144935,144935 +144936,144936 +144937,144937 +144938,144938 +144939,144939 +144940,144940 +144941,144941 +144942,144942 +144943,144943 +144944,144944 +144945,144945 +145638,144945 +144946,144946 +144947,144947 +144948,144948 +144949,144949 +144950,144950 +144951,144951 +144952,144952 +144953,144953 +144954,144954 +144955,144955 +144956,144956 +144957,144957 +211967,144957 +144958,144958 +144959,144959 +144960,144960 +144961,144961 +144962,144962 +144963,144963 +144964,144964 +144965,144965 +144966,144966 +144967,144967 +144968,144968 +144969,144969 +144970,144970 +144971,144971 +144972,144972 +144973,144973 +144974,144974 +144975,144975 +144976,144976 +144977,144977 +144978,144978 +144979,144979 +144980,144980 +144981,144981 +144982,144982 +144983,144983 +144984,144984 +144985,144985 +144986,144986 +144987,144987 +144988,144988 +144989,144989 +144990,144990 +144991,144991 +144992,144992 +144993,144993 +144994,144994 +144995,144995 +144996,144996 +144997,144997 +144998,144998 +144999,144999 +145044,145044 +166440,145044 +145045,145045 +145046,145046 +145047,145047 +145048,145048 +145049,145049 +145050,145050 +145051,145051 +247536,145051 +145052,145052 +145053,145053 +145054,145054 +145055,145055 +145056,145056 +52478,145056 +44374,145056 +145057,145057 +145058,145058 +145059,145059 +145060,145060 +145061,145061 +106795,145061 +145062,145062 +145063,145063 +145064,145064 +142214,145064 +145065,145065 +145066,145066 +145067,145067 +145068,145068 +145069,145069 +145070,145070 +145071,145071 +206737,145071 +145072,145072 +145073,145073 +179765,145073 +145074,145074 +145075,145075 +145076,145076 +145077,145077 +4642,145077 +145078,145078 +145079,145079 +145080,145080 +145081,145081 +145082,145082 +145083,145083 +145084,145084 +145085,145085 +145086,145086 +145087,145087 +145088,145088 +145089,145089 +145090,145090 +145091,145091 +60254,145091 +145092,145092 +145093,145093 +231303,145094 +145094,145094 +226304,145094 +145095,145095 +145096,145096 +145097,145097 +145098,145098 +145099,145099 +145100,145100 +145101,145101 +145102,145102 +145103,145103 +50217,145103 +42025,145103 +145104,145104 +145105,145105 +145106,145106 +145107,145107 +145108,145108 +145109,145109 +145110,145110 +145111,145111 +145112,145112 +145113,145113 +145114,145114 +145115,145115 +145116,145116 +145117,145117 +221871,145117 +145118,145118 +145119,145119 +145120,145120 +145121,145121 +145122,145122 +145123,145123 +145124,145124 +145125,145125 +145126,145126 +111513,145126 +187294,145126 +145127,145127 +145128,145128 +145129,145129 +145130,145130 +145131,145131 +145132,145132 +111519,145132 +145133,145133 +145134,145134 +145135,145135 +9476,145135 +145136,145136 +145137,145137 +145138,145138 +145139,145139 +145140,145140 +145141,145141 +145142,145142 +145143,145143 +145144,145144 +145145,145145 +145146,145146 +145147,145147 +145148,145148 +145149,145149 +145150,145150 +145151,145151 +145152,145152 +145153,145153 +145154,145154 +145155,145155 +145156,145156 +145157,145157 +145158,145158 +145159,145159 +145160,145160 +145161,145161 +145162,145162 +145163,145163 +145164,145164 +145165,145165 +145166,145166 +145167,145167 +155640,145167 +142187,145167 +145168,145168 +145169,145169 +145170,145170 +145171,145171 +145172,145172 +145173,145173 +145174,145174 +145175,145175 +145176,145176 +145177,145177 +145178,145178 +145179,145179 +145180,145180 +145181,145181 +145182,145182 +145183,145183 +145184,145184 +145185,145185 +145186,145186 +145187,145187 +51547,145187 +43443,145187 +145188,145188 +145189,145189 +145190,145190 +145191,145191 +145192,145192 +163365,145192 +145193,145193 +142213,145193 +145194,145194 +145195,145195 +145196,145196 +145197,145197 +145198,145198 +145199,145199 +79148,145199 +145200,145200 +145201,145201 +145202,145202 +145203,145203 +145204,145204 +145205,145205 +145206,145206 +207261,145206 +3100,145206 +145207,145207 +145208,145208 +145209,145209 +26371,145209 +145210,145210 +145211,145211 +145212,145212 +145213,145213 +145214,145214 +145215,145215 +145216,145216 +145217,145217 +145218,145218 +145219,145219 +145220,145220 +145221,145221 +145222,145222 +211643,145222 +145223,145223 +145224,145224 +213707,145224 +145225,145225 +7753,145225 +145226,145226 +145227,145227 +145228,145228 +145229,145229 +145230,145230 +145231,145231 +145232,145232 +145233,145233 +145234,145234 +111761,145234 +145235,145235 +145236,145236 +145237,145237 +145238,145238 +145239,145239 +145240,145240 +145241,145241 +145242,145242 +145243,145243 +145244,145244 +145245,145245 +145246,145246 +145247,145247 +145248,145248 +145249,145249 +145250,145250 +145251,145251 +145252,145252 +145253,145253 +145254,145254 +145255,145255 +145256,145256 +145257,145257 +145258,145258 +145259,145259 +145260,145260 +142280,145260 +145261,145261 +123276,145261 +142281,145261 +600,145261 +145262,145262 +142282,145262 +145263,145263 +142283,145263 +145264,145264 +212051,145264 +145265,145265 +145266,145266 +142286,145266 +145267,145267 +142287,145267 +145268,145268 +142288,145268 +145269,145269 +142289,145269 +145270,145270 +142290,145270 +145271,145271 +203837,145271 +142291,145271 +145272,145272 +145273,145273 +142293,145273 +145274,145274 +145275,145275 +145276,145276 +145277,145277 +145278,145278 +145279,145279 +145280,145280 +151581,145280 +145281,145281 +104963,145281 +145282,145282 +145325,145325 +55079,145325 +46975,145325 +145326,145326 +142346,145326 +145327,145327 +145328,145328 +142348,145328 +145329,145329 +145330,145330 +145331,145331 +145332,145332 +145333,145333 +142353,145333 +145334,145334 +145335,145335 +145336,145336 +145337,145337 +145338,145338 +145339,145339 +145340,145340 +145341,145341 +145342,145342 +145343,145343 +145344,145344 +145345,145345 +145346,145346 +160430,145346 +145347,145347 +145348,145348 +145349,145349 +145350,145350 +145351,145351 +215388,145351 +145352,145352 +137065,145352 +129478,145352 +145353,145353 +142373,145353 +145354,145354 +145355,145355 +145356,145356 +145357,145357 +145358,145358 +145359,145359 +145360,145360 +145361,145361 +145362,145362 +145363,145363 +145364,145364 +145365,145365 +145366,145366 +178122,145366 +172158,145366 +156398,145366 +107557,145366 +145367,145367 +145368,145368 +145369,145369 +90616,145369 +145370,145370 +145371,145371 +145372,145372 +247920,145372 +145373,145373 +145374,145374 +145375,145375 +145376,145376 +145377,145377 +145378,145378 +145379,145379 +145380,145380 +145381,145381 +145382,145382 +145383,145383 +145384,145384 +145385,145385 +145386,145386 +11840,145386 +145387,145387 +145388,145388 +145389,145389 +145390,145390 +145391,145391 +145392,145392 +145393,145393 +72033,145393 +145394,145394 +194777,145394 +145395,145395 +145396,145396 +145397,145397 +145398,145398 +145399,145399 +145400,145400 +145401,145401 +145402,145402 +145403,145403 +145404,145404 +145405,145405 +79219,145405 +145406,145406 +145407,145407 +145408,145408 +145409,145409 +145410,145410 +145411,145411 +145412,145412 +145413,145413 +145414,145414 +145415,145415 +145416,145416 +145417,145417 +64188,145417 +145418,145418 +145419,145419 +123039,145419 +145420,145420 +145421,145421 +145422,145422 +19690,145422 +145423,145423 +145424,145424 +145425,145425 +145426,145426 +145427,145427 +145428,145428 +145429,145429 +145430,145430 +145431,145431 +145432,145432 +145433,145433 +145434,145434 +145435,145435 +179847,145435 +145436,145436 +150703,145436 +145437,145437 +145438,145438 +145439,145439 +145440,145440 +145441,145441 +145442,145442 +145443,145443 +145444,145444 +145445,145445 +145446,145446 +145447,145447 +145448,145448 +145449,145449 +145450,145450 +145451,145451 +145452,145452 +145453,145453 +145454,145454 +145455,145455 +145456,145456 +145457,145457 +145458,145458 +145459,145459 +192459,145459 +145460,145460 +145461,145461 +226313,145461 +145462,145462 +145463,145463 +145464,145464 +199709,145464 +145465,145465 +145466,145466 +145467,145467 +145468,145468 +145469,145469 +145470,145470 +145471,145471 +145472,145472 +76583,145472 +145473,145473 +142477,145473 +145474,145474 +145475,145475 +145476,145476 +61432,145476 +145477,145477 +142481,145477 +145478,145478 +142482,145478 +145479,145479 +33764,145479 +145480,145480 +145481,145481 +145482,145482 +145483,145483 +145484,145484 +65618,145484 +145485,145485 +145486,145486 +145487,145487 +145488,145488 +244206,145488 +142492,145488 +145489,145489 +145490,145490 +80000,145490 +145491,145491 +145492,145492 +142496,145492 +145493,145493 +145494,145494 +145495,145495 +145496,145496 +145497,145497 +145498,145498 +145499,145499 +145500,145500 +145501,145501 +145502,145502 +145503,145503 +225927,145504 +145504,145504 +145505,145505 +145506,145506 +145507,145507 +145508,145508 +145509,145509 +145510,145510 +145511,145511 +145512,145512 +145513,145513 +145514,145514 +145515,145515 +145516,145516 +145517,145517 +145518,145518 +145519,145519 +145520,145520 +145521,145521 +145522,145522 +145523,145523 +191510,145523 +145524,145524 +145525,145525 +202561,145525 +145526,145526 +145527,145527 +145528,145528 +145529,145529 +145530,145530 +145531,145531 +145532,145532 +145533,145533 +145534,145534 +145535,145535 +145536,145536 +145537,145537 +145538,145538 +145539,145539 +145540,145540 +234816,145540 +145541,145541 +52750,145541 +44646,145541 +145542,145542 +145543,145543 +145544,145544 +145597,145597 +145598,145598 +145599,145599 +145600,145600 +145601,145601 +145602,145602 +141523,145602 +145603,145603 +145604,145604 +145605,145605 +145606,145606 +145607,145607 +145608,145608 +192568,145609 +145609,145609 +191753,145609 +145610,145610 +145611,145611 +145612,145612 +145613,145613 +145614,145614 +145615,145615 +145616,145616 +145617,145617 +145618,145618 +145619,145619 +145620,145620 +145621,145621 +145622,145622 +145623,145623 +145624,145624 +145625,145625 +145626,145626 +145627,145627 +145628,145628 +145629,145629 +145630,145630 +145631,145631 +145632,145632 +145633,145633 +145634,145634 +145635,145635 +145636,145636 +197325,145636 +145637,145637 +145638,145638 +144945,145638 +145639,145639 +145640,145640 +145641,145641 +145642,145642 +145643,145643 +145644,145644 +145699,145699 +145700,145700 +145701,145701 +164616,145701 +145702,145702 +145703,145703 +145704,145704 +145705,145705 +145706,145706 +145707,145707 +145708,145708 +145709,145709 +145710,145710 +145711,145711 +248147,145711 +145712,145712 +20654,145712 +145713,145713 +123500,145713 +132109,145713 +124522,145713 +145714,145714 +145715,145715 +145716,145716 +145717,145717 +145718,145718 +145719,145719 +145720,145720 +145721,145721 +145722,145722 +145723,145723 +145724,145724 +145725,145725 +145726,145726 +61738,145726 +145727,145727 +145728,145728 +145729,145729 +145730,145730 +145731,145731 +145732,145732 +145733,145733 +145734,145734 +34094,145734 +145735,145735 +145736,145736 +145737,145737 +145738,145738 +207156,145738 +145739,145739 +145740,145740 +145741,145741 +145742,145742 +160554,145742 +145743,145743 +242327,145743 +145744,145744 +145745,145745 +145746,145746 +145747,145747 +145748,145748 +145749,145749 +145750,145750 +145751,145751 +145752,145752 +145753,145753 +145754,145754 +145755,145755 +145756,145756 +36636,145756 +145757,145757 +145758,145758 +145759,145759 +145760,145760 +145761,145761 +145762,145762 +145763,145763 +145764,145764 +145765,145765 +145766,145766 +145767,145767 +145768,145768 +145769,145769 +145770,145770 +145771,145771 +145772,145772 +145773,145773 +145774,145774 +163576,145774 +145775,145775 +145776,145776 +145777,145777 +145778,145778 +145779,145779 +51060,145779 +42956,145779 +145780,145780 +145781,145781 +145782,145782 +145783,145783 +145784,145784 +2312,145784 +145785,145785 +145786,145786 +145787,145787 +145788,145788 +145789,145789 +145790,145790 +145791,145791 +145792,145792 +145793,145793 +145794,145794 +145795,145795 +145796,145796 +145797,145797 +145798,145798 +145799,145799 +145800,145800 +145801,145801 +145802,145802 +145803,145803 +145804,145804 +145805,145805 +145806,145806 +145807,145807 +145808,145808 +145809,145809 +145810,145810 +145811,145811 +145812,145812 +145813,145813 +145814,145814 +145815,145815 +145816,145816 +145817,145817 +145818,145818 +145819,145819 +145820,145820 +145821,145821 +145822,145822 +155436,145822 +145823,145823 +145824,145824 +145825,145825 +145826,145826 +145827,145827 +145828,145828 +145829,145829 +145830,145830 +145831,145831 +145832,145832 +145833,145833 +145834,145834 +145835,145835 +145836,145836 +145837,145837 +145838,145838 +145839,145839 +145840,145840 +145841,145841 +145842,145842 +145843,145843 +145844,145844 +145845,145845 +145846,145846 +145847,145847 +36118,145847 +145848,145848 +145849,145849 +146104,146104 +146105,146105 +146106,146106 +213579,146106 +155399,146107 +146107,146107 +155400,146108 +146108,146108 +146109,146109 +155401,146109 +146110,146110 +146111,146111 +146112,146112 +933,146112 +146113,146113 +155406,146114 +146114,146114 +146115,146115 +146116,146116 +155408,146116 +146117,146117 +146118,146118 +146119,146119 +146120,146120 +146121,146121 +146122,146122 +146123,146123 +146124,146124 +146125,146125 +155417,146125 +146126,146126 +146127,146127 +151790,146127 +151791,146128 +146128,146128 +155421,146129 +151792,146129 +146129,146129 +151793,146130 +146130,146130 +155422,146130 +146131,146131 +151794,146131 +146132,146132 +151795,146132 +146133,146133 +146134,146134 +146135,146135 +146136,146136 +146137,146137 +146138,146138 +146139,146139 +137541,146139 +129954,146139 +146140,146140 +146141,146141 +146142,146142 +151805,146142 +146143,146143 +146144,146144 +146145,146145 +146146,146146 +146147,146147 +146148,146148 +146149,146149 +146150,146150 +146151,146151 +146152,146152 +155445,146153 +146153,146153 +137555,146153 +129968,146153 +150099,146154 +146154,146154 +137556,146154 +129969,146154 +146155,146155 +146156,146156 +146157,146157 +146158,146158 +146159,146159 +146160,146160 +150105,146160 +137763,146160 +130176,146160 +146161,146161 +146162,146162 +146163,146163 +146326,146326 +146327,146327 +146328,146328 +146329,146329 +146330,146330 +146331,146331 +146332,146332 +146333,146333 +240274,146333 +146334,146334 +143910,146334 +146335,146335 +146336,146336 +146337,146337 +146338,146338 +57972,146338 +146339,146339 +146340,146340 +146341,146341 +146342,146342 +146343,146343 +146344,146344 +146345,146345 +146346,146346 +146347,146347 +146348,146348 +146349,146349 +146350,146350 +146351,146351 +146352,146352 +146353,146353 +146354,146354 +146355,146355 +146356,146356 +146357,146357 +146358,146358 +146359,146359 +146360,146360 +146361,146361 +146362,146362 +146363,146363 +146364,146364 +146365,146365 +236829,146365 +146366,146366 +146367,146367 +146368,146368 +146369,146369 +146370,146370 +146371,146371 +146372,146372 +146373,146373 +146374,146374 +146375,146375 +146376,146376 +146377,146377 +146378,146378 +146379,146379 +146380,146380 +146381,146381 +146382,146382 +146383,146383 +146384,146384 +146385,146385 +146386,146386 +146387,146387 +146388,146388 +146389,146389 +146390,146390 +146391,146391 +146392,146392 +146393,146393 +242639,146393 +146394,146394 +146395,146395 +146396,146396 +146397,146397 +146398,146398 +146399,146399 +146400,146400 +146401,146401 +146402,146402 +146403,146403 +146404,146404 +146405,146405 +146406,146406 +186167,146406 +146407,146407 +146408,146408 +146409,146409 +146410,146410 +146411,146411 +146412,146412 +146413,146413 +146414,146414 +146415,146415 +146416,146416 +146417,146417 +146418,146418 +146419,146419 +146420,146420 +146421,146421 +146422,146422 +132309,146422 +124722,146422 +146423,146423 +146424,146424 +146425,146425 +146426,146426 +146427,146427 +146428,146428 +146429,146429 +249592,146429 +146430,146430 +146431,146431 +146432,146432 +146433,146433 +103615,146433 +146434,146434 +146435,146435 +146436,146436 +146437,146437 +146438,146438 +146439,146439 +83797,146439 +146440,146440 +146441,146441 +146442,146442 +146443,146443 +2306,146443 +146444,146444 +146445,146445 +146446,146446 +146447,146447 +146448,146448 +134045,146448 +126458,146448 +150890,146448 +146449,146449 +146450,146450 +146451,146451 +146452,146452 +146453,146453 +146454,146454 +146455,146455 +70447,146455 +146456,146456 +146457,146457 +146458,146458 +146459,146459 +146460,146460 +146461,146461 +146462,146462 +146463,146463 +146464,146464 +146465,146465 +146466,146466 +146467,146467 +146468,146468 +146469,146469 +153170,146469 +146470,146470 +146471,146471 +146472,146472 +146473,146473 +146474,146474 +146475,146475 +146476,146476 +146477,146477 +146478,146478 +146479,146479 +146480,146480 +146481,146481 +146482,146482 +132388,146482 +124801,146482 +146483,146483 +146484,146484 +146485,146485 +146486,146486 +213674,146487 +146487,146487 +146488,146488 +210287,146488 +146489,146489 +146490,146490 +146491,146491 +146492,146492 +146493,146493 +158435,146493 +146494,146494 +30572,146494 +146495,146495 +146496,146496 +146497,146497 +146498,146498 +146499,146499 +146500,146500 +146501,146501 +146502,146502 +164095,146502 +146503,146503 +146504,146504 +146505,146505 +146506,146506 +146507,146507 +146508,146508 +146509,146509 +7099,146509 +146510,146510 +146511,146511 +188221,146511 +146512,146512 +146513,146513 +146514,146514 +146515,146515 +146516,146516 +146517,146517 +182318,146518 +146518,146518 +146519,146519 +146520,146520 +146521,146521 +146522,146522 +189171,146522 +146523,146523 +185452,146523 +146524,146524 +146525,146525 +146526,146526 +146527,146527 +146528,146528 +146529,146529 +68252,146529 +146530,146530 +146531,146531 +146532,146532 +35913,146532 +146533,146533 +146534,146534 +146535,146535 +146536,146536 +146537,146537 +146538,146538 +159518,146538 +146539,146539 +146540,146540 +146541,146541 +146542,146542 +146543,146543 +146544,146544 +146545,146545 +146546,146546 +146547,146547 +146548,146548 +146549,146549 +146550,146550 +146551,146551 +146552,146552 +146553,146553 +146554,146554 +146555,146555 +146556,146556 +146557,146557 +146558,146558 +146559,146559 +146560,146560 +146561,146561 +146562,146562 +36770,146562 +81479,146562 +146563,146563 +146564,146564 +146565,146565 +146566,146566 +146567,146567 +192150,146567 +146568,146568 +146569,146569 +146570,146570 +66885,146570 +146571,146571 +146572,146572 +177422,146572 +171458,146572 +146573,146573 +146574,146574 +146575,146575 +146576,146576 +146577,146577 +146578,146578 +146579,146579 +146580,146580 +146581,146581 +146582,146582 +146583,146583 +146584,146584 +146585,146585 +146586,146586 +146587,146587 +134347,146587 +126760,146587 +146588,146588 +146589,146589 +146590,146590 +146591,146591 +146592,146592 +180092,146592 +146593,146593 +146594,146594 +146595,146595 +146596,146596 +146597,146597 +146598,146598 +146599,146599 +146600,146600 +146601,146601 +146602,146602 +146603,146603 +146604,146604 +146605,146605 +146606,146606 +62013,146606 +146607,146607 +146608,146608 +146609,146609 +146610,146610 +146611,146611 +146612,146612 +146613,146613 +146614,146614 +146615,146615 +247223,146615 +146616,146616 +146617,146617 +146618,146618 +146619,146619 +146620,146620 +146621,146621 +30499,146621 +146622,146622 +146623,146623 +146624,146624 +29972,146624 +146625,146625 +146626,146626 +146627,146627 +146628,146628 +146673,146673 +146674,146674 +146675,146675 +146676,146676 +146677,146677 +146678,146678 +146679,146679 +146680,146680 +146681,146681 +146682,146682 +146683,146683 +146684,146684 +146685,146685 +146686,146686 +146687,146687 +146688,146688 +146689,146689 +146690,146690 +146691,146691 +146692,146692 +146693,146693 +146694,146694 +58663,146694 +146695,146695 +146696,146696 +146697,146697 +146698,146698 +146699,146699 +231284,146699 +146700,146700 +146701,146701 +146702,146702 +146703,146703 +146704,146704 +146705,146705 +244209,146705 +146706,146706 +146707,146707 +146708,146708 +146709,146709 +146710,146710 +146711,146711 +146712,146712 +192174,146712 +146713,146713 +3996,146713 +146714,146714 +214495,146714 +146715,146715 +81637,146715 +146716,146716 +146717,146717 +146718,146718 +146719,146719 +146720,146720 +146721,146721 +176551,146721 +170587,146721 +146722,146722 +146723,146723 +146724,146724 +146725,146725 +146726,146726 +146727,146727 +190485,146727 +146728,146728 +146729,146729 +146730,146730 +146731,146731 +107079,146731 +146732,146732 +146733,146733 +146734,146734 +146735,146735 +146736,146736 +146737,146737 +146738,146738 +146739,146739 +146740,146740 +79052,146740 +146741,146741 +146742,146742 +198689,146742 +146743,146743 +146744,146744 +146745,146745 +146746,146746 +146747,146747 +146748,146748 +146749,146749 +146750,146750 +146751,146751 +146752,146752 +146753,146753 +146754,146754 +146755,146755 +146756,146756 +146757,146757 +242901,146757 +38450,146757 +146758,146758 +146759,146759 +146760,146760 +146761,146761 +146762,146762 +146763,146763 +146764,146764 +146765,146765 +146766,146766 +146767,146767 +146768,146768 +146769,146769 +146770,146770 +146771,146771 +146772,146772 +146773,146773 +146774,146774 +146775,146775 +146776,146776 +146777,146777 +146778,146778 +146779,146779 +146780,146780 +146781,146781 +146782,146782 +146783,146783 +146784,146784 +146785,146785 +146786,146786 +146787,146787 +146788,146788 +146789,146789 +146790,146790 +146791,146791 +146792,146792 +146793,146793 +146794,146794 +146795,146795 +146796,146796 +146797,146797 +146798,146798 +146799,146799 +146800,146800 +146801,146801 +146802,146802 +146803,146803 +146804,146804 +146805,146805 +146806,146806 +146807,146807 +146808,146808 +146809,146809 +146810,146810 +146811,146811 +146812,146812 +146813,146813 +146814,146814 +146815,146815 +146816,146816 +146817,146817 +146818,146818 +89504,146818 +146819,146819 +179533,146819 +146820,146820 +146821,146821 +146822,146822 +146823,146823 +146824,146824 +146825,146825 +146826,146826 +146827,146827 +146828,146828 +146829,146829 +146830,146830 +146831,146831 +146832,146832 +146833,146833 +146834,146834 +146835,146835 +109001,146835 +146836,146836 +146837,146837 +146838,146838 +146839,146839 +146840,146840 +146841,146841 +146842,146842 +183687,146842 +146843,146843 +146844,146844 +146845,146845 +146846,146846 +146847,146847 +146848,146848 +3953,146848 +146849,146849 +146850,146850 +146851,146851 +146852,146852 +146853,146853 +146854,146854 +146855,146855 +146856,146856 +146857,146857 +146858,146858 +146859,146859 +146860,146860 +146861,146861 +146862,146862 +191950,146862 +146863,146863 +146864,146864 +146865,146865 +146866,146866 +146867,146867 +146868,146868 +146869,146869 +160653,146869 +146870,146870 +146871,146871 +146872,146872 +146873,146873 +146874,146874 +146875,146875 +146876,146876 +146877,146877 +146878,146878 +146879,146879 +146880,146880 +146881,146881 +146882,146882 +146883,146883 +146884,146884 +146885,146885 +146886,146886 +146887,146887 +146888,146888 +146889,146889 +146890,146890 +146891,146891 +146892,146892 +146893,146893 +111117,146893 +146894,146894 +146895,146895 +146896,146896 +146897,146897 +146898,146898 +146899,146899 +146900,146900 +146901,146901 +146902,146902 +146903,146903 +146904,146904 +146905,146905 +146906,146906 +146907,146907 +146908,146908 +146909,146909 +146910,146910 +590,146910 +146911,146911 +146912,146912 +146913,146913 +146914,146914 +146915,146915 +146916,146916 +146917,146917 +200631,146917 +80137,146917 +146918,146918 +146919,146919 +146920,146920 +161637,146920 +146921,146921 +146922,146922 +146923,146923 +146924,146924 +146925,146925 +146926,146926 +146927,146927 +146928,146928 +20125,146928 +146929,146929 +146930,146930 +146931,146931 +146932,146932 +146933,146933 +146934,146934 +146935,146935 +146936,146936 +146937,146937 +146938,146938 +146939,146939 +146940,146940 +146941,146941 +146942,146942 +146943,146943 +146944,146944 +146945,146945 +146946,146946 +146947,146947 +146948,146948 +146949,146949 +146950,146950 +146951,146951 +146952,146952 +146953,146953 +146954,146954 +52884,146954 +44780,146954 +146955,146955 +146956,146956 +146957,146957 +146958,146958 +146959,146959 +205586,146959 +146960,146960 +146961,146961 +146962,146962 +146963,146963 +66136,146963 +146964,146964 +146965,146965 +146966,146966 +146967,146967 +146968,146968 +146969,146969 +205113,146969 +146970,146970 +146971,146971 +146972,146972 +146973,146973 +146974,146974 +110306,146974 +146975,146975 +146976,146976 +146977,146977 +146978,146978 +146979,146979 +146980,146980 +146981,146981 +146982,146982 +146983,146983 +83638,146983 +146984,146984 +146985,146985 +227654,146985 +146986,146986 +76013,146986 +146987,146987 +146988,146988 +146989,146989 +146990,146990 +146991,146991 +146992,146992 +146993,146993 +146994,146994 +146995,146995 +146996,146996 +146997,146997 +146998,146998 +146999,146999 +147000,147000 +147001,147001 +147002,147002 +147003,147003 +147004,147004 +147005,147005 +147006,147006 +147007,147007 +147008,147008 +147009,147009 +147010,147010 +147011,147011 +147012,147012 +147013,147013 +147014,147014 +147015,147015 +216276,147015 +147016,147016 +147017,147017 +147018,147018 +147019,147019 +147020,147020 +147021,147021 +147022,147022 +147023,147023 +147024,147024 +147025,147025 +147026,147026 +147027,147027 +147028,147028 +147029,147029 +147030,147030 +147031,147031 +147032,147032 +147033,147033 +147034,147034 +147035,147035 +147036,147036 +147037,147037 +147038,147038 +147039,147039 +147040,147040 +147041,147041 +147042,147042 +147043,147043 +147044,147044 +35545,147044 +214461,147044 +147045,147045 +147046,147046 +147047,147047 +147048,147048 +147049,147049 +147050,147050 +147051,147051 +147052,147052 +147053,147053 +142811,147053 +147054,147054 +147055,147055 +158335,147055 +147056,147056 +147057,147057 +226845,147057 +147058,147058 +147059,147059 +147060,147060 +147061,147061 +147062,147062 +147063,147063 +147064,147064 +147065,147065 +147066,147066 +147067,147067 +147068,147068 +147069,147069 +147070,147070 +147071,147071 +147072,147072 +147073,147073 +147074,147074 +147075,147075 +147076,147076 +147077,147077 +147078,147078 +147079,147079 +2133,147079 +147080,147080 +147081,147081 +29994,147081 +147082,147082 +147083,147083 +147084,147084 +147085,147085 +147086,147086 +147087,147087 +147088,147088 +147089,147089 +232553,147089 +147090,147090 +147091,147091 +147092,147092 +147093,147093 +147094,147094 +147095,147095 +162317,147095 +147096,147096 +147097,147097 +147098,147098 +147099,147099 +147100,147100 +147101,147101 +147102,147102 +147103,147103 +147104,147104 +147105,147105 +147106,147106 +147107,147107 +147108,147108 +147109,147109 +150869,147109 +147110,147110 +147111,147111 +147112,147112 +147113,147113 +147114,147114 +21182,147114 +147115,147115 +147116,147116 +147117,147117 +147118,147118 +147119,147119 +147120,147120 +147121,147121 +147122,147122 +147123,147123 +147124,147124 +147125,147125 +147126,147126 +147127,147127 +147128,147128 +147129,147129 +147130,147130 +147131,147131 +147132,147132 +147133,147133 +147134,147134 +148241,147134 +147135,147135 +147136,147136 +147137,147137 +147138,147138 +147139,147139 +147140,147140 +51141,147140 +43037,147140 +147141,147141 +147142,147142 +147143,147143 +147144,147144 +147145,147145 +147146,147146 +147147,147147 +147148,147148 +147149,147149 +147150,147150 +147151,147151 +108931,147151 +147152,147152 +147153,147153 +147154,147154 +147155,147155 +147156,147156 +147157,147157 +147158,147158 +147159,147159 +147160,147160 +163035,147160 +147161,147161 +147162,147162 +147163,147163 +147164,147164 +147165,147165 +147166,147166 +147167,147167 +147168,147168 +147169,147169 +147170,147170 +147316,147316 +147317,147317 +161199,147317 +147318,147318 +147319,147319 +147320,147320 +147321,147321 +147322,147322 +201770,147322 +147323,147323 +147324,147324 +147325,147325 +241663,147325 +147326,147326 +243482,147326 +147327,147327 +147328,147328 +147329,147329 +147330,147330 +36498,147330 +147331,147331 +147332,147332 +147333,147333 +147334,147334 +147335,147335 +147336,147336 +147337,147337 +161930,147337 +147338,147338 +147339,147339 +147340,147340 +147341,147341 +202625,147341 +147342,147342 +147343,147343 +147344,147344 +147345,147345 +147346,147346 +147347,147347 +147348,147348 +147349,147349 +203978,147349 +147350,147350 +147351,147351 +147352,147352 +147353,147353 +147354,147354 +147355,147355 +147515,147515 +147516,147516 +147517,147517 +147518,147518 +147519,147519 +147520,147520 +30950,147520 +147521,147521 +147522,147522 +147523,147523 +147524,147524 +147525,147525 +147526,147526 +147527,147527 +147528,147528 +147529,147529 +147530,147530 +147531,147531 +147532,147532 +147533,147533 +147534,147534 +100700,147534 +147535,147535 +147536,147536 +147537,147537 +2844,147537 +147538,147538 +147539,147539 +147540,147540 +147541,147541 +147542,147542 +183967,147543 +147543,147543 +215893,147543 +147544,147544 +147545,147545 +147546,147546 +246633,147546 +147547,147547 +147548,147548 +147549,147549 +147550,147550 +147551,147551 +31349,147551 +147552,147552 +147553,147553 +147554,147554 +147555,147555 +147556,147556 +224868,147556 +147557,147557 +147558,147558 +147559,147559 +147560,147560 +147561,147561 +147562,147562 +147563,147563 +147564,147564 +147565,147565 +147566,147566 +147567,147567 +147568,147568 +147569,147569 +121483,147569 +147570,147570 +147571,147571 +147572,147572 +4893,147572 +147573,147573 +147574,147574 +147575,147575 +147576,147576 +147577,147577 +147578,147578 +147579,147579 +147580,147580 +147581,147581 +147582,147582 +147583,147583 +147584,147584 +147585,147585 +147586,147586 +147587,147587 +50221,147587 +42029,147587 +147588,147588 +147589,147589 +147590,147590 +147591,147591 +147592,147592 +147593,147593 +147594,147594 +147595,147595 +81132,147595 +147596,147596 +147597,147597 +147598,147598 +147640,147640 +132302,147640 +124715,147640 +147641,147641 +147642,147642 +207924,147642 +147643,147643 +147644,147644 +147645,147645 +147646,147646 +147647,147647 +147648,147648 +147649,147649 +147650,147650 +147651,147651 +76048,147651 +147652,147652 +147653,147653 +147654,147654 +147655,147655 +147656,147656 +147657,147657 +147658,147658 +147659,147659 +147660,147660 +147661,147661 +147662,147662 +116016,147662 +147663,147663 +147664,147664 +147665,147665 +147666,147666 +147667,147667 +147668,147668 +147669,147669 +147670,147670 +147671,147671 +147672,147672 +147673,147673 +147674,147674 +147675,147675 +208655,147675 +147676,147676 +147677,147677 +147678,147678 +147679,147679 +147680,147680 +147681,147681 +147682,147682 +5936,147682 +147683,147683 +147684,147684 +147685,147685 +62985,147685 +147686,147686 +147687,147687 +147688,147688 +147689,147689 +147690,147690 +147691,147691 +147692,147692 +147693,147693 +147694,147694 +147695,147695 +147696,147696 +147697,147697 +147698,147698 +147699,147699 +147700,147700 +147701,147701 +147702,147702 +147703,147703 +147704,147704 +147705,147705 +147706,147706 +147879,147879 +147880,147880 +147881,147881 +147882,147882 +147883,147883 +147884,147884 +165263,147885 +147885,147885 +147886,147886 +147887,147887 +147888,147888 +147889,147889 +147890,147890 +147891,147891 +147892,147892 +147893,147893 +147894,147894 +147895,147895 +147896,147896 +147897,147897 +147898,147898 +147899,147899 +147900,147900 +147901,147901 +191048,147901 +147902,147902 +147903,147903 +147904,147904 +147905,147905 +147906,147906 +147907,147907 +150115,147907 +147908,147908 +147909,147909 +148108,148108 +148109,148109 +148110,148110 +148111,148111 +148112,148112 +148113,148113 +148114,148114 +148115,148115 +148116,148116 +148117,148117 +148118,148118 +148119,148119 +148120,148120 +119230,148120 +148121,148121 +148122,148122 +114320,148122 +148123,148123 +148124,148124 +160629,148125 +148125,148125 +148126,148126 +148127,148127 +148128,148128 +148129,148129 +226626,148129 +148130,148130 +220325,148130 +148131,148131 +148132,148132 +148133,148133 +148134,148134 +148135,148135 +148136,148136 +148137,148137 +148138,148138 +148139,148139 +148140,148140 +148141,148141 +148142,148142 +148143,148143 +114348,148143 +148144,148144 +148145,148145 +148146,148146 +148147,148147 +20907,148147 +148148,148148 +148149,148149 +148150,148150 +148151,148151 +148152,148152 +148153,148153 +148154,148154 +148155,148155 +148156,148156 +148157,148157 +90552,148157 +148158,148158 +148159,148159 +148160,148160 +148161,148161 +148162,148162 +148163,148163 +148164,148164 +148165,148165 +148166,148166 +148167,148167 +148168,148168 +148169,148169 +200587,148169 +148170,148170 +148171,148171 +148172,148172 +148173,148173 +148174,148174 +148175,148175 +148176,148176 +13461,148176 +148177,148177 +148178,148178 +148179,148179 +148180,148180 +148181,148181 +148182,148182 +148183,148183 +122702,148183 +148184,148184 +148185,148185 +148186,148186 +148187,148187 +148188,148188 +148189,148189 +148190,148190 +148191,148191 +227060,148191 +148192,148192 +148193,148193 +148194,148194 +148195,148195 +148196,148196 +148197,148197 +148198,148198 +148199,148199 +148200,148200 +148201,148201 +148202,148202 +148203,148203 +148204,148204 +155118,148204 +148205,148205 +148206,148206 +148207,148207 +148208,148208 +148209,148209 +148210,148210 +144932,148210 +148211,148211 +148212,148212 +148213,148213 +148214,148214 +148215,148215 +148216,148216 +148217,148217 +148218,148218 +148219,148219 +148220,148220 +148221,148221 +148222,148222 +137720,148222 +130133,148222 +148223,148223 +148224,148224 +234282,148225 +148225,148225 +138676,148225 +148226,148226 +148227,148227 +148228,148228 +148229,148229 +148230,148230 +148231,148231 +148232,148232 +148233,148233 +148234,148234 +148235,148235 +148236,148236 +148237,148237 +148238,148238 +148239,148239 +148240,148240 +148241,148241 +147134,148241 +148242,148242 +148243,148243 +148244,148244 +148245,148245 +148246,148246 +148247,148247 +107100,148247 +148248,148248 +148249,148249 +148250,148250 +148251,148251 +148252,148252 +148253,148253 +148254,148254 +148255,148255 +148256,148256 +148257,148257 +148258,148258 +148259,148259 +148260,148260 +148261,148261 +148262,148262 +148263,148263 +148264,148264 +148265,148265 +148266,148266 +4790,148266 +148267,148267 +148268,148268 +148269,148269 +148270,148270 +148271,148271 +148272,148272 +148273,148273 +148274,148274 +148275,148275 +148276,148276 +148277,148277 +148278,148278 +148279,148279 +148280,148280 +148281,148281 +148282,148282 +148283,148283 +148284,148284 +148285,148285 +148286,148286 +148287,148287 +148288,148288 +148289,148289 +148290,148290 +35636,148290 +148291,148291 +148292,148292 +148293,148293 +148294,148294 +148295,148295 +148296,148296 +148297,148297 +148298,148298 +148299,148299 +148300,148300 +148301,148301 +148302,148302 +148303,148303 +108313,148303 +148304,148304 +148305,148305 +148306,148306 +148307,148307 +148308,148308 +148309,148309 +148310,148310 +148311,148311 +148312,148312 +5854,148312 +148313,148313 +148314,148314 +148315,148315 +148316,148316 +148317,148317 +148318,148318 +148319,148319 +75555,148319 +148320,148320 +148321,148321 +148322,148322 +148323,148323 +148324,148324 +205295,148324 +148325,148325 +148326,148326 +148327,148327 +148328,148328 +148329,148329 +191148,148329 +148330,148330 +148331,148331 +148332,148332 +148333,148333 +148334,148334 +148335,148335 +148336,148336 +148337,148337 +148338,148338 +148339,148339 +148340,148340 +148341,148341 +148342,148342 +148343,148343 +148344,148344 +148345,148345 +148346,148346 +148347,148347 +148348,148348 +148349,148349 +148350,148350 +148351,148351 +148352,148352 +148353,148353 +148354,148354 +148355,148355 +148356,148356 +148357,148357 +148358,148358 +148359,148359 +148360,148360 +148361,148361 +148362,148362 +148363,148363 +52745,148363 +44641,148363 +148364,148364 +148365,148365 +148366,148366 +148367,148367 +148368,148368 +148369,148369 +148370,148370 +148371,148371 +148372,148372 +51518,148372 +43414,148372 +148373,148373 +148374,148374 +148375,148375 +148376,148376 +148377,148377 +148378,148378 +214189,148378 +148379,148379 +86037,148379 +148380,148380 +148381,148381 +148382,148382 +109031,148382 +148383,148383 +160356,148383 +148384,148384 +148385,148385 +148386,148386 +148387,148387 +148388,148388 +148389,148389 +148390,148390 +148391,148391 +148392,148392 +148393,148393 +148394,148394 +148395,148395 +148396,148396 +148397,148397 +148398,148398 +148399,148399 +148400,148400 +161759,148400 +148401,148401 +148402,148402 +148403,148403 +148404,148404 +148405,148405 +148406,148406 +148407,148407 +148408,148408 +148409,148409 +148410,148410 +148411,148411 +148412,148412 +148413,148413 +148414,148414 +148415,148415 +148416,148416 +148417,148417 +148418,148418 +148419,148419 +148420,148420 +148421,148421 +148422,148422 +148423,148423 +148424,148424 +148425,148425 +148426,148426 +148427,148427 +148428,148428 +148429,148429 +148430,148430 +148431,148431 +148432,148432 +40201,148432 +148433,148433 +148434,148434 +148435,148435 +148436,148436 +148437,148437 +148438,148438 +63514,148438 +148439,148439 +232545,148439 +148440,148440 +202579,148440 +148441,148441 +230558,148441 +148442,148442 +148443,148443 +148444,148444 +2216,148444 +148445,148445 +148446,148446 +148447,148447 +148448,148448 +148449,148449 +148450,148450 +148451,148451 +148452,148452 +148453,148453 +148454,148454 +148455,148455 +148456,148456 +148457,148457 +148458,148458 +148459,148459 +148460,148460 +148461,148461 +148462,148462 +148463,148463 +244934,148463 +148464,148464 +148465,148465 +148466,148466 +148467,148467 +148468,148468 +4221,148468 +191825,148468 +148469,148469 +148470,148470 +148471,148471 +148472,148472 +148473,148473 +148474,148474 +148475,148475 +148476,148476 +148477,148477 +148478,148478 +148479,148479 +148480,148480 +148481,148481 +148482,148482 +148483,148483 +148484,148484 +148485,148485 +148486,148486 +148487,148487 +148488,148488 +148489,148489 +148490,148490 +148491,148491 +148492,148492 +148493,148493 +148494,148494 +148495,148495 +148496,148496 +148497,148497 +148498,148498 +148499,148499 +148500,148500 +148501,148501 +148502,148502 +148503,148503 +148504,148504 +148505,148505 +148506,148506 +148507,148507 +148508,148508 +148509,148509 +148510,148510 +148511,148511 +148512,148512 +148513,148513 +148514,148514 +148515,148515 +148516,148516 +148517,148517 +148518,148518 +148519,148519 +148520,148520 +148521,148521 +148522,148522 +148618,148618 +148619,148619 +148620,148620 +148621,148621 +148622,148622 +36626,148622 +166620,148622 +148623,148623 +148624,148624 +148625,148625 +148626,148626 +148627,148627 +148628,148628 +148629,148629 +192312,148629 +148630,148630 +148631,148631 +148632,148632 +148633,148633 +148634,148634 +148635,148635 +148636,148636 +148637,148637 +148638,148638 +148639,148639 +148640,148640 +148641,148641 +148642,148642 +148643,148643 +148644,148644 +148645,148645 +148646,148646 +148647,148647 +148648,148648 +148649,148649 +148650,148650 +148651,148651 +148652,148652 +148653,148653 +148654,148654 +148655,148655 +16019,148655 +148656,148656 +239529,148656 +234000,148656 +148657,148657 +148658,148658 +5590,148658 +148659,148659 +211320,148659 +148660,148660 +148661,148661 +85597,148661 +148662,148662 +148663,148663 +148664,148664 +148665,148665 +148666,148666 +148667,148667 +148668,148668 +148669,148669 +148670,148670 +148800,148800 +148801,148801 +148802,148802 +148803,148803 +148804,148804 +148805,148805 +148806,148806 +148807,148807 +148808,148808 +148809,148809 +13298,148809 +148810,148810 +148811,148811 +148812,148812 +148813,148813 +148814,148814 +148815,148815 +148816,148816 +148817,148817 +148818,148818 +148819,148819 +148820,148820 +148821,148821 +204265,148822 +148822,148822 +148823,148823 +148824,148824 +148825,148825 +135749,148825 +128162,148825 +148826,148826 +148827,148827 +148828,148828 +148982,148982 +148983,148983 +148984,148984 +148985,148985 +148986,148986 +148987,148987 +148988,148988 +148989,148989 +148990,148990 +148991,148991 +148992,148992 +148993,148993 +148994,148994 +148995,148995 +148996,148996 +148997,148997 +148998,148998 +34033,148998 +148999,148999 +177674,148999 +171710,148999 +149000,149000 +149001,149001 +149002,149002 +149003,149003 +149004,149004 +149005,149005 +149006,149006 +149007,149007 +149008,149008 +149009,149009 +149010,149010 +149011,149011 +149012,149012 +149013,149013 +149014,149014 +149015,149015 +149016,149016 +149017,149017 +149018,149018 +149019,149019 +149020,149020 +149021,149021 +149022,149022 +149023,149023 +149024,149024 +149025,149025 +149026,149026 +149027,149027 +149028,149028 +149029,149029 +149030,149030 +149031,149031 +154615,149031 +149032,149032 +149033,149033 +149034,149034 +149035,149035 +149036,149036 +149037,149037 +149038,149038 +193811,149038 +149039,149039 +164889,149039 +164890,149040 +149040,149040 +149041,149041 +149042,149042 +149043,149043 +149044,149044 +149045,149045 +149046,149046 +149047,149047 +149048,149048 +149049,149049 +149050,149050 +183488,149050 +149051,149051 +149052,149052 +149053,149053 +149054,149054 +149055,149055 +149056,149056 +149057,149057 +149058,149058 +149059,149059 +149060,149060 +149061,149061 +149062,149062 +149063,149063 +149064,149064 +149065,149065 +149066,149066 +149067,149067 +149068,149068 +149069,149069 +149070,149070 +149071,149071 +149072,149072 +149073,149073 +149074,149074 +149075,149075 +149076,149076 +149077,149077 +149078,149078 +149079,149079 +149080,149080 +149081,149081 +149082,149082 +149083,149083 +149084,149084 +78025,149084 +149085,149085 +30621,149085 +149086,149086 +77131,149086 +149087,149087 +149088,149088 +149089,149089 +149090,149090 +149091,149091 +149092,149092 +149093,149093 +149094,149094 +149095,149095 +88122,149095 +149096,149096 +149097,149097 +149098,149098 +149099,149099 +149100,149100 +133186,149100 +125599,149100 +149101,149101 +149102,149102 +149103,149103 +149104,149104 +158066,149104 +149105,149105 +149106,149106 +149107,149107 +15561,149107 +149108,149108 +149109,149109 +149110,149110 +149111,149111 +149112,149112 +149113,149113 +149114,149114 +149115,149115 +149116,149116 +149117,149117 +149118,149118 +149119,149119 +149120,149120 +149121,149121 +200637,149122 +149122,149122 +149123,149123 +149124,149124 +149125,149125 +149126,149126 +149127,149127 +149128,149128 +149129,149129 +149130,149130 +149131,149131 +149132,149132 +149133,149133 +149134,149134 +149135,149135 +149136,149136 +149137,149137 +149138,149138 +149139,149139 +149140,149140 +149141,149141 +149142,149142 +149143,149143 +149144,149144 +149145,149145 +154789,149145 +247595,149145 +149146,149146 +149147,149147 +149148,149148 +149149,149149 +149150,149150 +149151,149151 +149152,149152 +149153,149153 +149191,149191 +149192,149192 +149193,149193 +118617,149193 +149194,149194 +149195,149195 +149196,149196 +149197,149197 +149198,149198 +149199,149199 +149200,149200 +149201,149201 +149202,149202 +149203,149203 +149204,149204 +149205,149205 +149206,149206 +149207,149207 +149208,149208 +149209,149209 +149210,149210 +149211,149211 +149212,149212 +149213,149213 +149214,149214 +149215,149215 +149216,149216 +149217,149217 +149218,149218 +149219,149219 +149220,149220 +149221,149221 +149222,149222 +149223,149223 +149224,149224 +149225,149225 +149226,149226 +149227,149227 +149228,149228 +149229,149229 +149230,149230 +149231,149231 +149232,149232 +149233,149233 +149234,149234 +149235,149235 +149236,149236 +67316,149236 +149237,149237 +149238,149238 +149239,149239 +149240,149240 +149241,149241 +149242,149242 +149243,149243 +149244,149244 +149245,149245 +149246,149246 +149247,149247 +149248,149248 +86688,149248 +149249,149249 +149250,149250 +149251,149251 +123074,149251 +149252,149252 +149253,149253 +149254,149254 +149255,149255 +149256,149256 +149257,149257 +149258,149258 +216662,149258 +149259,149259 +149260,149260 +149261,149261 +149262,149262 +149263,149263 +9025,149263 +149264,149264 +149265,149265 +152773,149265 +149266,149266 +149267,149267 +149268,149268 +149269,149269 +149270,149270 +151596,149270 +149271,149271 +149272,149272 +149273,149273 +149274,149274 +149275,149275 +149276,149276 +149277,149277 +149278,149278 +161054,149278 +149279,149279 +149280,149280 +149281,149281 +149282,149282 +149283,149283 +149284,149284 +149285,149285 +149286,149286 +149287,149287 +149288,149288 +149289,149289 +149290,149290 +149291,149291 +149292,149292 +149293,149293 +149294,149294 +77195,149294 +149295,149295 +149296,149296 +149297,149297 +149298,149298 +149299,149299 +149300,149300 +149301,149301 +149302,149302 +149303,149303 +149304,149304 +149305,149305 +149306,149306 +149307,149307 +149308,149308 +149309,149309 +149310,149310 +149311,149311 +179368,149311 +149312,149312 +149313,149313 +149314,149314 +214711,149314 +149315,149315 +149316,149316 +149317,149317 +149318,149318 +149319,149319 +149320,149320 +149321,149321 +149322,149322 +149323,149323 +149324,149324 +180225,149324 +149325,149325 +149326,149326 +149327,149327 +149328,149328 +149329,149329 +149330,149330 +149331,149331 +149332,149332 +149333,149333 +149334,149334 +149335,149335 +149336,149336 +149337,149337 +149338,149338 +149339,149339 +149340,149340 +62780,149340 +149341,149341 +149342,149342 +214685,149342 +149343,149343 +149344,149344 +89207,149344 +149345,149345 +149346,149346 +149347,149347 +149348,149348 +149349,149349 +208772,149349 +149350,149350 +149351,149351 +149352,149352 +149353,149353 +149354,149354 +149355,149355 +149356,149356 +149357,149357 +149358,149358 +149359,149359 +149360,149360 +149361,149361 +149362,149362 +190991,149362 +149363,149363 +149364,149364 +149365,149365 +149366,149366 +149367,149367 +149368,149368 +149369,149369 +149370,149370 +149371,149371 +149372,149372 +17395,149372 +149373,149373 +149374,149374 +149375,149375 +87227,149375 +149376,149376 +149377,149377 +149378,149378 +149379,149379 +149380,149380 +149381,149381 +149382,149382 +149383,149383 +149384,149384 +149385,149385 +149386,149386 +149387,149387 +149388,149388 +149389,149389 +149390,149390 +206464,149391 +149391,149391 +149392,149392 +149393,149393 +149394,149394 +149395,149395 +149396,149396 +149397,149397 +149398,149398 +149399,149399 +149400,149400 +149401,149401 +149402,149402 +178852,149402 +149403,149403 +149404,149404 +2332,149404 +149405,149405 +149406,149406 +149407,149407 +149408,149408 +149409,149409 +149410,149410 +149411,149411 +149412,149412 +26945,149412 +149413,149413 +149414,149414 +149415,149415 +149416,149416 +149417,149417 +149418,149418 +149419,149419 +149420,149420 +149421,149421 +149422,149422 +102196,149422 +149423,149423 +149424,149424 +149425,149425 +149426,149426 +149427,149427 +149428,149428 +149429,149429 +149430,149430 +149431,149431 +149432,149432 +144193,149432 +149433,149433 +149434,149434 +149435,149435 +235656,149435 +149436,149436 +149437,149437 +149438,149438 +149439,149439 +84834,149439 +149440,149440 +149441,149441 +149442,149442 +149443,149443 +149444,149444 +149445,149445 +149446,149446 +149447,149447 +207614,149448 +149448,149448 +149449,149449 +149450,149450 +149451,149451 +149452,149452 +149453,149453 +149454,149454 +149455,149455 +149456,149456 +149611,149611 +149612,149612 +149613,149613 +149614,149614 +149615,149615 +149616,149616 +149617,149617 +149618,149618 +149619,149619 +149620,149620 +149621,149621 +149622,149622 +149623,149623 +149624,149624 +149625,149625 +149626,149626 +149627,149627 +149628,149628 +149629,149629 +149630,149630 +149631,149631 +149632,149632 +149633,149633 +149634,149634 +149635,149635 +53722,149635 +45618,149635 +149636,149636 +149637,149637 +149638,149638 +149639,149639 +149640,149640 +149641,149641 +149642,149642 +149643,149643 +149644,149644 +149645,149645 +149646,149646 +149647,149647 +149648,149648 +149649,149649 +149650,149650 +149651,149651 +149652,149652 +149653,149653 +162319,149653 +149654,149654 +149655,149655 +149656,149656 +149657,149657 +149658,149658 +149659,149659 +149660,149660 +149661,149661 +149662,149662 +149663,149663 +149664,149664 +149665,149665 +58159,149665 +149666,149666 +70665,149666 +181295,149666 +149667,149667 +149668,149668 +149669,149669 +149670,149670 +149671,149671 +149672,149672 +149673,149673 +149674,149674 +149675,149675 +149676,149676 +149677,149677 +149678,149678 +149679,149679 +149680,149680 +149681,149681 +149682,149682 +149683,149683 +149684,149684 +149685,149685 +149686,149686 +149687,149687 +149688,149688 +149689,149689 +149690,149690 +149691,149691 +149692,149692 +84238,149692 +191788,149692 +149693,149693 +149694,149694 +149695,149695 +149696,149696 +149697,149697 +149698,149698 +149699,149699 +149700,149700 +149701,149701 +149702,149702 +149703,149703 +149704,149704 +149705,149705 +149706,149706 +149707,149707 +149708,149708 +149709,149709 +249206,149709 +149710,149710 +149711,149711 +149712,149712 +149713,149713 +149714,149714 +149715,149715 +149716,149716 +149717,149717 +149718,149718 +177526,149755 +171562,149755 +149755,149755 +149756,149756 +149757,149757 +149758,149758 +149759,149759 +161103,149759 +149760,149760 +149761,149761 +149762,149762 +149763,149763 +149764,149764 +149765,149765 +149766,149766 +149767,149767 +149768,149768 +149769,149769 +149770,149770 +149771,149771 +149772,149772 +190970,149772 +149773,149773 +65166,149773 +63997,149773 +149774,149774 +149775,149775 +149776,149776 +149777,149777 +149778,149778 +149779,149779 +180656,149779 +149780,149780 +149781,149781 +164089,149781 +149782,149782 +149783,149783 +149784,149784 +149785,149785 +149786,149786 +149787,149787 +149788,149788 +149789,149789 +149790,149790 +149791,149791 +149792,149792 +149793,149793 +149794,149794 +31675,149794 +149795,149795 +149796,149796 +83037,149796 +149797,149797 +149798,149798 +149799,149799 +149800,149800 +149801,149801 +150298,149801 +149802,149802 +149803,149803 +149804,149804 +149805,149805 +149806,149806 +149807,149807 +149808,149808 +149809,149809 +149810,149810 +149811,149811 +149812,149812 +149813,149813 +149814,149814 +149815,149815 +149816,149816 +149817,149817 +149818,149818 +149819,149819 +149820,149820 +149821,149821 +149822,149822 +149823,149823 +149824,149824 +149825,149825 +149826,149826 +149827,149827 +149828,149828 +149829,149829 +149830,149830 +149831,149831 +149832,149832 +149833,149833 +149834,149834 +149835,149835 +64397,149835 +149836,149836 +149837,149837 +149838,149838 +149839,149839 +149840,149840 +205930,149840 +149841,149841 +86848,149841 +149842,149842 +149843,149843 +149844,149844 +149845,149845 +149846,149846 +149847,149847 +149848,149848 +152213,149848 +149849,149849 +133038,149849 +125451,149849 +149850,149850 +149851,149851 +149852,149852 +149853,149853 +149854,149854 +149855,149855 +149856,149856 +149857,149857 +149858,149858 +218467,149858 +149859,149859 +149860,149860 +114726,149860 +149861,149861 +149862,149862 +149863,149863 +180113,149863 +149864,149864 +149865,149865 +149866,149866 +149867,149867 +149868,149868 +149869,149869 +201000,149869 +149870,149870 +149871,149871 +149872,149872 +149873,149873 +149874,149874 +110720,149874 +149875,149875 +149876,149876 +197706,149876 +149877,149877 +149878,149878 +149879,149879 +149880,149880 +149881,149881 +149882,149882 +149883,149883 +149884,149884 +149885,149885 +149886,149886 +149887,149887 +149888,149888 +149889,149889 +149890,149890 +149891,149891 +149892,149892 +149893,149893 +149894,149894 +161897,149894 +149895,149895 +149896,149896 +149897,149897 +185906,149897 +149898,149898 +149899,149899 +149900,149900 +149901,149901 +81193,149901 +149902,149902 +149903,149903 +149904,149904 +149905,149905 +149906,149906 +149907,149907 +149908,149908 +161881,149908 +149909,149909 +149910,149910 +149911,149911 +149912,149912 +149913,149913 +149914,149914 +149915,149915 +150089,150089 +150090,150090 +150282,150090 +150091,150091 +150092,150092 +150093,150093 +150094,150094 +150095,150095 +137346,150095 +129759,150095 +150096,150096 +150097,150097 +137755,150097 +130168,150097 +150098,150098 +150099,150099 +146154,150099 +137556,150099 +129969,150099 +150100,150100 +150101,150101 +150102,150102 +150103,150103 +150104,150104 +150105,150105 +137763,150105 +130176,150105 +146160,150105 +150106,150106 +150107,150107 +150108,150108 +150109,150109 +150110,150110 +150111,150111 +150112,150112 +150113,150113 +150114,150114 +135519,150114 +127932,150114 +150115,150115 +147907,150115 +150116,150116 +2488,150116 +150117,150117 +150279,150279 +150280,150280 +150281,150281 +150282,150282 +150090,150282 +150283,150283 +150284,150284 +150285,150285 +150286,150286 +150287,150287 +63750,150287 +150288,150288 +150289,150289 +150290,150290 +150291,150291 +150292,150292 +150293,150293 +137558,150293 +129971,150293 +150294,150294 +150295,150295 +150296,150296 +150297,150297 +150298,150298 +149801,150298 +150299,150299 +150300,150300 +150301,150301 +150459,150459 +150460,150460 +222056,150460 +150461,150461 +150462,150462 +150463,150463 +150464,150464 +150465,150465 +28999,150465 +150466,150466 +150467,150467 +206059,150467 +150468,150468 +150469,150469 +233226,150469 +150470,150470 +243208,150470 +150471,150471 +150472,150472 +150473,150473 +150474,150474 +150475,150475 +150476,150476 +150477,150477 +150478,150478 +150479,150479 +150480,150480 +150481,150481 +150482,150482 +150483,150483 +150484,150484 +150485,150485 +150486,150486 +150487,150487 +150488,150488 +150489,150489 +150490,150490 +150491,150491 +150492,150492 +150493,150493 +150494,150494 +150495,150495 +150496,150496 +150497,150497 +139718,150497 +150498,150498 +150499,150499 +150500,150500 +150501,150501 +150502,150502 +150503,150503 +150504,150504 +150505,150505 +150506,150506 +150553,150553 +150554,150554 +150555,150555 +150556,150556 +87526,150556 +150557,150557 +150558,150558 +227550,150558 +150559,150559 +150560,150560 +150561,150561 +150562,150562 +150563,150563 +150564,150564 +150565,150565 +150566,150566 +150567,150567 +150568,150568 +150569,150569 +56250,150569 +48146,150569 +150570,150570 +150571,150571 +150572,150572 +150573,150573 +150574,150574 +150575,150575 +150576,150576 +86409,150576 +150577,150577 +150578,150578 +150579,150579 +192825,150579 +150580,150580 +150581,150581 +150582,150582 +150583,150583 +150584,150584 +150585,150585 +150586,150586 +150587,150587 +150588,150588 +150589,150589 +163396,150589 +150590,150590 +150591,150591 +150592,150592 +150593,150593 +214696,150593 +150594,150594 +150595,150595 +150596,150596 +150597,150597 +150598,150598 +150599,150599 +150600,150600 +150601,150601 +150602,150602 +150603,150603 +150604,150604 +150605,150605 +150606,150606 +150607,150607 +150608,150608 +150609,150609 +150610,150610 +150611,150611 +150612,150612 +150613,150613 +150614,150614 +150615,150615 +150616,150616 +150617,150617 +150618,150618 +150619,150619 +150620,150620 +150621,150621 +150622,150622 +150623,150623 +150624,150624 +150625,150625 +150626,150626 +12240,150626 +150627,150627 +91027,150627 +158491,150627 +150628,150628 +150629,150629 +150630,150630 +150631,150631 +79860,150631 +150632,150632 +150633,150633 +150634,150634 +150635,150635 +150636,150636 +150637,150637 +150638,150638 +150639,150639 +150640,150640 +150641,150641 +150642,150642 +150643,150643 +150644,150644 +75546,150644 +150645,150645 +150646,150646 +150647,150647 +150648,150648 +150649,150649 +150650,150650 +150651,150651 +150652,150652 +150653,150653 +150654,150654 +150655,150655 +150656,150656 +150657,150657 +150658,150658 +150659,150659 +150660,150660 +150661,150661 +150662,150662 +150663,150663 +150664,150664 +150665,150665 +150666,150666 +150667,150667 +161785,150667 +150668,150668 +150669,150669 +150670,150670 +150671,150671 +136528,150671 +128941,150671 +150672,150672 +150673,150673 +150674,150674 +150675,150675 +150676,150676 +150677,150677 +79245,150677 +150678,150678 +20456,150678 +150679,150679 +150680,150680 +150681,150681 +150682,150682 +150683,150683 +150684,150684 +216091,150684 +150685,150685 +150686,150686 +150687,150687 +150688,150688 +150689,150689 +150690,150690 +150691,150691 +150692,150692 +150693,150693 +150694,150694 +150695,150695 +208254,150695 +150696,150696 +150697,150697 +150698,150698 +150699,150699 +150700,150700 +150701,150701 +150702,150702 +160051,150702 +150703,150703 +33829,150703 +145436,150703 +150704,150704 +150705,150705 +150706,150706 +150707,150707 +150708,150708 +150709,150709 +150710,150710 +230345,150710 +150711,150711 +150712,150712 +150713,150713 +150714,150714 +150715,150715 +150716,150716 +150717,150717 +150718,150718 +14536,150718 +150719,150719 +150720,150720 +150721,150721 +150722,150722 +150723,150723 +150724,150724 +150725,150725 +150726,150726 +150727,150727 +207028,150727 +150728,150728 +150729,150729 +150730,150730 +150731,150731 +150732,150732 +150733,150733 +150734,150734 +150735,150735 +150736,150736 +150737,150737 +150738,150738 +150739,150739 +193776,150739 +150740,150740 +150741,150741 +150742,150742 +150743,150743 +150744,150744 +150745,150745 +150746,150746 +150747,150747 +73767,150747 +150748,150748 +150749,150749 +150750,150750 +150751,150751 +150752,150752 +150753,150753 +150754,150754 +78681,150754 +150755,150755 +150756,150756 +150757,150757 +150758,150758 +150759,150759 +150760,150760 +150761,150761 +150762,150762 +150763,150763 +150764,150764 +150765,150765 +150766,150766 +150767,150767 +150768,150768 +150769,150769 +150770,150770 +150771,150771 +150772,150772 +150773,150773 +150774,150774 +150775,150775 +150776,150776 +150777,150777 +150778,150778 +150779,150779 +150780,150780 +150781,150781 +150782,150782 +150783,150783 +150784,150784 +150785,150785 +150786,150786 +150787,150787 +150788,150788 +150789,150789 +150790,150790 +150791,150791 +102359,150791 +150792,150792 +150793,150793 +150794,150794 +150795,150795 +150796,150796 +150797,150797 +150798,150798 +136080,150798 +128493,150798 +150799,150799 +166726,150799 +135284,150799 +127697,150799 +150800,150800 +150801,150801 +150802,150802 +150803,150803 +150804,150804 +150805,150805 +107564,150805 +150806,150806 +150807,150807 +60438,150807 +150808,150808 +114893,150808 +150809,150809 +150810,150810 +150811,150811 +150812,150812 +150813,150813 +150814,150814 +150815,150815 +150816,150816 +150817,150817 +150818,150818 +150819,150819 +150820,150820 +150821,150821 +150822,150822 +150823,150823 +150824,150824 +150825,150825 +150826,150826 +150827,150827 +150828,150828 +150829,150829 +150830,150830 +150831,150831 +150832,150832 +150833,150833 +150834,150834 +227019,150834 +150835,150835 +150836,150836 +150837,150837 +150838,150838 +150839,150839 +150840,150840 +150841,150841 +150842,150842 +150843,150843 +150844,150844 +150845,150845 +150846,150846 +150847,150847 +150848,150848 +150849,150849 +150850,150850 +150851,150851 +150852,150852 +150853,150853 +150854,150854 +150855,150855 +150856,150856 +150857,150857 +150858,150858 +150859,150859 +150860,150860 +150861,150861 +150862,150862 +150863,150863 +213447,150863 +150864,150864 +150865,150865 +150866,150866 +150867,150867 +150868,150868 +150869,150869 +147109,150869 +150870,150870 +150871,150871 +150872,150872 +150873,150873 +150874,150874 +150875,150875 +150876,150876 +150877,150877 +150878,150878 +150879,150879 +150880,150880 +173915,150880 +167951,150880 +247955,150880 +150881,150881 +150882,150882 +150883,150883 +150884,150884 +150885,150885 +150886,150886 +150887,150887 +150888,150888 +150889,150889 +150890,150890 +146448,150890 +150891,150891 +150892,150892 +150893,150893 +150894,150894 +150895,150895 +150896,150896 +150897,150897 +150898,150898 +151111,151111 +151112,151112 +151113,151113 +151114,151114 +151115,151115 +151116,151116 +151117,151117 +151118,151118 +151119,151119 +151120,151120 +1960,151120 +151121,151121 +204751,151121 +136993,151121 +129406,151121 +214446,151121 +151122,151122 +151123,151123 +151124,151124 +151125,151125 +151126,151126 +151127,151127 +151128,151128 +151129,151129 +151130,151130 +151131,151131 +165250,151131 +151132,151132 +25017,151132 +151133,151133 +151134,151134 +151135,151135 +151136,151136 +151137,151137 +227555,151137 +151138,151138 +28752,151138 +151139,151139 +151140,151140 +151141,151141 +151142,151142 +151143,151143 +151144,151144 +151145,151145 +151146,151146 +151147,151147 +151148,151148 +151149,151149 +151150,151150 +151151,151151 +151152,151152 +151153,151153 +151154,151154 +151155,151155 +151156,151156 +29906,151156 +151157,151157 +234286,151157 +151158,151158 +151159,151159 +151160,151160 +151161,151161 +151162,151162 +151163,151163 +151164,151164 +151165,151165 +151166,151166 +151167,151167 +151168,151168 +151169,151169 +151170,151170 +151171,151171 +151172,151172 +151173,151173 +151174,151174 +151175,151175 +151176,151176 +151177,151177 +151178,151178 +151179,151179 +151180,151180 +151181,151181 +151182,151182 +151183,151183 +151184,151184 +151185,151185 +151186,151186 +151187,151187 +151188,151188 +151189,151189 +227607,151189 +151190,151190 +151191,151191 +151192,151192 +151193,151193 +227612,151194 +151194,151194 +151195,151195 +227614,151196 +151196,151196 +151197,151197 +151198,151198 +227616,151198 +151199,151199 +151200,151200 +151201,151201 +151202,151202 +151203,151203 +227621,151203 +151204,151204 +151205,151205 +151206,151206 +151207,151207 +151208,151208 +151209,151209 +151210,151210 +151211,151211 +151212,151212 +165021,151212 +151213,151213 +151214,151214 +151215,151215 +151319,151319 +151320,151320 +151321,151321 +151322,151322 +151323,151323 +151324,151324 +102363,151324 +151325,151325 +151326,151326 +151327,151327 +151328,151328 +151329,151329 +151330,151330 +151331,151331 +151332,151332 +151333,151333 +151334,151334 +151335,151335 +151336,151336 +151337,151337 +151338,151338 +151339,151339 +234905,151339 +151340,151340 +205278,151340 +151341,151341 +151342,151342 +151343,151343 +151344,151344 +151345,151345 +151346,151346 +151347,151347 +151348,151348 +151349,151349 +208836,151349 +151350,151350 +151351,151351 +151352,151352 +151353,151353 +151354,151354 +151355,151355 +151356,151356 +151357,151357 +151358,151358 +151359,151359 +151360,151360 +79402,151360 +151361,151361 +151362,151362 +151363,151363 +151364,151364 +151365,151365 +109592,151365 +162593,151365 +151366,151366 +151367,151367 +151368,151368 +151369,151369 +151370,151370 +151371,151371 +151372,151372 +151373,151373 +151374,151374 +151375,151375 +151376,151376 +151377,151377 +151378,151378 +184456,151378 +33986,151378 +151379,151379 +151380,151380 +151381,151381 +151382,151382 +151383,151383 +151384,151384 +151385,151385 +151386,151386 +151387,151387 +151388,151388 +151389,151389 +151390,151390 +151391,151391 +151392,151392 +151393,151393 +151394,151394 +36338,151394 +151395,151395 +151396,151396 +151397,151397 +151398,151398 +151399,151399 +151400,151400 +151401,151401 +151402,151402 +110284,151402 +151403,151403 +151404,151404 +151405,151405 +151406,151406 +151407,151407 +151408,151408 +151409,151409 +151410,151410 +151411,151411 +151412,151412 +191161,151412 +151413,151413 +151414,151414 +151415,151415 +151416,151416 +151417,151417 +151418,151418 +187408,151418 +151419,151419 +151420,151420 +151421,151421 +151422,151422 +151423,151423 +151424,151424 +151425,151425 +151426,151426 +151427,151427 +151428,151428 +151429,151429 +151430,151430 +151431,151431 +151432,151432 +151433,151433 +163359,151433 +151434,151434 +151435,151435 +151436,151436 +151437,151437 +151438,151438 +151439,151439 +151440,151440 +152999,151440 +151441,151441 +151442,151442 +151443,151443 +151444,151444 +151445,151445 +151446,151446 +151447,151447 +151448,151448 +151449,151449 +151450,151450 +151451,151451 +151452,151452 +56118,151452 +48014,151452 +28453,151452 +151453,151453 +151454,151454 +151455,151455 +151456,151456 +5814,151456 +151457,151457 +159458,151457 +151458,151458 +151459,151459 +151460,151460 +151461,151461 +151462,151462 +151463,151463 +151464,151464 +4606,151464 +151465,151465 +151466,151466 +151467,151467 +151468,151468 +26368,151468 +151469,151469 +151470,151470 +151471,151471 +151472,151472 +151473,151473 +151474,151474 +151475,151475 +151476,151476 +151477,151477 +151478,151478 +151479,151479 +189844,151479 +138743,151479 +151480,151480 +151481,151481 +151482,151482 +151483,151483 +151484,151484 +204452,151484 +151485,151485 +151486,151486 +151487,151487 +151488,151488 +151489,151489 +151490,151490 +151491,151491 +151492,151492 +151493,151493 +151494,151494 +151495,151495 +151496,151496 +151497,151497 +151498,151498 +151499,151499 +112008,151499 +151500,151500 +220,151500 +151501,151501 +151502,151502 +151503,151503 +151504,151504 +151505,151505 +151506,151506 +151507,151507 +151508,151508 +151509,151509 +151510,151510 +244744,151510 +151511,151511 +151512,151512 +151513,151513 +151514,151514 +151515,151515 +151516,151516 +151517,151517 +151518,151518 +151519,151519 +151520,151520 +151521,151521 +151522,151522 +151523,151523 +151524,151524 +151525,151525 +151526,151526 +151527,151527 +151528,151528 +151529,151529 +151530,151530 +151531,151531 +151532,151532 +122933,151532 +151533,151533 +151534,151534 +151535,151535 +151536,151536 +151537,151537 +151538,151538 +151539,151539 +11967,151539 +151540,151540 +151541,151541 +151542,151542 +151543,151543 +151544,151544 +242676,151544 +151545,151545 +207189,151545 +151546,151546 +151547,151547 +151548,151548 +151549,151549 +151550,151550 +151551,151551 +151552,151552 +151553,151553 +151554,151554 +151555,151555 +151556,151556 +111212,151556 +151557,151557 +151558,151558 +151559,151559 +151560,151560 +151561,151561 +151562,151562 +151563,151563 +151564,151564 +151565,151565 +151566,151566 +151567,151567 +151568,151568 +5269,151568 +151569,151569 +9219,151569 +151570,151570 +151571,151571 +151572,151572 +151573,151573 +151574,151574 +151575,151575 +151576,151576 +151577,151577 +151578,151578 +151579,151579 +194345,151579 +151580,151580 +151581,151581 +145280,151581 +151582,151582 +151583,151583 +151584,151584 +151585,151585 +151586,151586 +151587,151587 +151588,151588 +151589,151589 +151590,151590 +151591,151591 +134068,151591 +126481,151591 +151592,151592 +151593,151593 +151594,151594 +151595,151595 +161370,151595 +151596,151596 +136583,151596 +128996,151596 +149270,151596 +151597,151597 +151598,151598 +151599,151599 +151600,151600 +151601,151601 +151602,151602 +151603,151603 +222356,151603 +151604,151604 +151605,151605 +151606,151606 +151607,151607 +151608,151608 +151609,151609 +151610,151610 +151611,151611 +151612,151612 +151613,151613 +151614,151614 +151615,151615 +151616,151616 +151617,151617 +151618,151618 +151619,151619 +151620,151620 +151621,151621 +151622,151622 +151623,151623 +190686,151623 +151624,151624 +151625,151625 +151626,151626 +97397,151626 +151627,151627 +151628,151628 +151629,151629 +151630,151630 +14128,151630 +151631,151631 +151632,151632 +151633,151633 +151634,151634 +151635,151635 +151790,151790 +146127,151790 +151791,151791 +146128,151791 +155421,151792 +151792,151792 +146129,151792 +151793,151793 +146130,151793 +155422,151793 +151794,151794 +146131,151794 +151795,151795 +146132,151795 +151796,151796 +151797,151797 +151798,151798 +137738,151798 +137331,151798 +130151,151798 +129744,151798 +151799,151799 +155428,151799 +77627,151799 +151800,151800 +151801,151801 +151802,151802 +155432,151803 +151803,151803 +137542,151803 +129955,151803 +151804,151804 +151805,151805 +146142,151805 +151806,151806 +151807,151807 +50811,151807 +42707,151807 +151808,151808 +151809,151809 +151810,151810 +151811,151811 +151812,151812 +151813,151813 +151814,151814 +151815,151815 +151816,151816 +151817,151817 +151818,151818 +152202,152202 +152203,152203 +152204,152204 +152205,152205 +152206,152206 +152207,152207 +152208,152208 +152209,152209 +152210,152210 +152211,152211 +152212,152212 +116814,152212 +152213,152213 +149848,152213 +152214,152214 +152215,152215 +152216,152216 +152217,152217 +152218,152218 +152219,152219 +152220,152220 +152221,152221 +133745,152221 +126158,152221 +152222,152222 +152223,152223 +152224,152224 +152225,152225 +152226,152226 +152227,152227 +152228,152228 +92082,152228 +152229,152229 +152230,152230 +152231,152231 +152232,152232 +152233,152233 +152234,152234 +152235,152235 +152236,152236 +152237,152237 +152238,152238 +152239,152239 +152240,152240 +220181,152240 +152241,152241 +152242,152242 +152243,152243 +152244,152244 +152245,152245 +152246,152246 +152247,152247 +152248,152248 +152249,152249 +179475,152249 +152250,152250 +211774,152250 +152251,152251 +152252,152252 +193750,152252 +152253,152253 +152254,152254 +152255,152255 +77684,152255 +152256,152256 +152257,152257 +152258,152258 +152259,152259 +152260,152260 +152261,152261 +152262,152262 +152301,152301 +152302,152302 +51786,152302 +43682,152302 +152303,152303 +152772,152772 +152773,152773 +149265,152773 +152774,152774 +152775,152775 +152776,152776 +152777,152777 +152778,152778 +152779,152779 +152780,152780 +152781,152781 +152782,152782 +152783,152783 +152784,152784 +152785,152785 +152786,152786 +152787,152787 +152788,152788 +152789,152789 +152790,152790 +152791,152791 +152792,152792 +233486,152792 +214814,152792 +152835,152835 +152836,152836 +152837,152837 +152838,152838 +152839,152839 +152840,152840 +152841,152841 +152842,152842 +152843,152843 +152844,152844 +152845,152845 +152846,152846 +152847,152847 +152848,152848 +152849,152849 +152850,152850 +152851,152851 +152852,152852 +152853,152853 +152854,152854 +152855,152855 +222014,152855 +152856,152856 +152857,152857 +152858,152858 +152859,152859 +152860,152860 +152861,152861 +152862,152862 +152863,152863 +152864,152864 +152865,152865 +152866,152866 +152867,152867 +152868,152868 +152869,152869 +152870,152870 +152871,152871 +207884,152871 +152872,152872 +152873,152873 +26047,152873 +152874,152874 +179423,152874 +152875,152875 +152876,152876 +152877,152877 +152878,152878 +152879,152879 +152880,152880 +152881,152881 +104303,152881 +152882,152882 +152883,152883 +152884,152884 +152885,152885 +152886,152886 +152887,152887 +152888,152888 +152889,152889 +152890,152890 +152891,152891 +152892,152892 +152893,152893 +152894,152894 +2415,152894 +152895,152895 +102662,152895 +152896,152896 +152897,152897 +152898,152898 +152899,152899 +152900,152900 +152901,152901 +152902,152902 +152903,152903 +223497,152903 +152904,152904 +152905,152905 +152906,152906 +152907,152907 +152908,152908 +75308,152908 +152909,152909 +120965,152909 +152910,152910 +152911,152911 +152912,152912 +152913,152913 +152914,152914 +152915,152915 +152916,152916 +152917,152917 +141500,152917 +152918,152918 +152919,152919 +152920,152920 +152921,152921 +152922,152922 +152923,152923 +152924,152924 +152925,152925 +26936,152925 +152926,152926 +152927,152927 +152928,152928 +152929,152929 +152930,152930 +152931,152931 +152932,152932 +152933,152933 +152934,152934 +152935,152935 +152936,152936 +152937,152937 +152938,152938 +152939,152939 +152940,152940 +152941,152941 +152942,152942 +152943,152943 +152944,152944 +152945,152945 +152946,152946 +152947,152947 +152948,152948 +152949,152949 +152950,152950 +230394,152950 +152951,152951 +152952,152952 +152953,152953 +152954,152954 +152955,152955 +152956,152956 +152957,152957 +152958,152958 +152959,152959 +152960,152960 +55760,152960 +47656,152960 +152961,152961 +152962,152962 +152963,152963 +152964,152964 +152965,152965 +5754,152965 +152966,152966 +152967,152967 +152968,152968 +152969,152969 +152970,152970 +152971,152971 +152972,152972 +152973,152973 +152974,152974 +152975,152975 +152976,152976 +152977,152977 +132799,152977 +125212,152977 +152978,152978 +152979,152979 +152980,152980 +152981,152981 +152982,152982 +152983,152983 +152984,152984 +152985,152985 +152986,152986 +152987,152987 +152988,152988 +152989,152989 +152990,152990 +152991,152991 +152992,152992 +152993,152993 +152994,152994 +152995,152995 +223270,152995 +244094,152995 +152996,152996 +76358,152996 +152997,152997 +152998,152998 +152999,152999 +151440,152999 +153000,153000 +153001,153001 +153002,153002 +153003,153003 +153004,153004 +153005,153005 +153006,153006 +153007,153007 +153008,153008 +153009,153009 +153010,153010 +153011,153011 +153012,153012 +153013,153013 +191884,153013 +153014,153014 +153015,153015 +153016,153016 +153017,153017 +153018,153018 +153019,153019 +186045,153019 +153020,153020 +153021,153021 +153022,153022 +33969,153022 +153023,153023 +153024,153024 +153025,153025 +153026,153026 +153027,153027 +153028,153028 +35633,153028 +153029,153029 +153030,153030 +153031,153031 +153032,153032 +153033,153033 +153034,153034 +153035,153035 +153036,153036 +153037,153037 +153079,153079 +153080,153080 +153081,153081 +153082,153082 +153083,153083 +153084,153084 +153085,153085 +153086,153086 +153087,153087 +153088,153088 +153089,153089 +153090,153090 +153091,153091 +153092,153092 +153093,153093 +153094,153094 +153095,153095 +153096,153096 +153097,153097 +153098,153098 +208223,153098 +153099,153099 +36479,153099 +153100,153100 +153101,153101 +153102,153102 +57872,153102 +153103,153103 +153104,153104 +153105,153105 +153106,153106 +153107,153107 +153108,153108 +153109,153109 +243463,153109 +153110,153110 +153111,153111 +153112,153112 +153113,153113 +153114,153114 +153115,153115 +24959,153115 +153116,153116 +153117,153117 +153118,153118 +153119,153119 +153120,153120 +153121,153121 +153122,153122 +153123,153123 +153124,153124 +153125,153125 +153126,153126 +153127,153127 +153128,153128 +153129,153129 +153130,153130 +153131,153131 +153132,153132 +153133,153133 +153134,153134 +153135,153135 +153136,153136 +153137,153137 +153138,153138 +153139,153139 +153140,153140 +153141,153141 +153142,153142 +153143,153143 +153144,153144 +153145,153145 +29879,153145 +153146,153146 +153147,153147 +153148,153148 +153149,153149 +153150,153150 +86659,153150 +153151,153151 +184772,153151 +153152,153152 +37206,153152 +153153,153153 +230399,153153 +153154,153154 +153155,153155 +153156,153156 +153157,153157 +153158,153158 +153159,153159 +242077,153159 +153160,153160 +192446,153160 +153161,153161 +153162,153162 +153163,153163 +153164,153164 +153165,153165 +153166,153166 +153167,153167 +153168,153168 +153169,153169 +153170,153170 +146469,153170 +153171,153171 +153172,153172 +153173,153173 +153174,153174 +153175,153175 +153176,153176 +153177,153177 +153178,153178 +153179,153179 +60145,153179 +153180,153180 +153181,153181 +153182,153182 +153183,153183 +153184,153184 +153185,153185 +153186,153186 +153187,153187 +153188,153188 +153189,153189 +153190,153190 +153191,153191 +153192,153192 +153193,153193 +153194,153194 +153195,153195 +153196,153196 +153197,153197 +153198,153198 +82357,153198 +153199,153199 +153200,153200 +153201,153201 +153202,153202 +77288,153202 +153203,153203 +153204,153204 +153205,153205 +153297,153297 +153298,153298 +153299,153299 +153300,153300 +142574,153300 +76586,153300 +153301,153301 +153302,153302 +153303,153303 +153304,153304 +153305,153305 +153306,153306 +153307,153307 +153308,153308 +153309,153309 +153310,153310 +153311,153311 +153312,153312 +153313,153313 +153314,153314 +153315,153315 +153316,153316 +153317,153317 +153318,153318 +153319,153319 +153320,153320 +153321,153321 +153322,153322 +153323,153323 +153324,153324 +153325,153325 +164664,153325 +210334,153325 +216583,153325 +153326,153326 +153327,153327 +153328,153328 +153329,153329 +153330,153330 +153331,153331 +153647,153331 +153332,153332 +153333,153333 +153334,153334 +153335,153335 +153336,153336 +153337,153337 +153338,153338 +153339,153339 +153340,153340 +153341,153341 +153342,153342 +153343,153343 +153344,153344 +153345,153345 +153346,153346 +153347,153347 +153632,153632 +87555,153632 +153633,153633 +153634,153634 +198819,153634 +153635,153635 +198820,153635 +88864,153635 +153636,153636 +153637,153637 +179700,153637 +153638,153638 +59776,153638 +153639,153639 +153640,153640 +153641,153641 +230869,153641 +153642,153642 +153643,153643 +153644,153644 +153645,153645 +153646,153646 +153647,153647 +153331,153647 +153648,153648 +153649,153649 +153650,153650 +153651,153651 +153652,153652 +153653,153653 +153654,153654 +153655,153655 +153656,153656 +153657,153657 +153658,153658 +153659,153659 +153660,153660 +153661,153661 +76189,153661 +153662,153662 +153663,153663 +153664,153664 +153665,153665 +153865,153865 +153866,153866 +153867,153867 +153868,153868 +153869,153869 +153870,153870 +153871,153871 +153872,153872 +153873,153873 +153874,153874 +153875,153875 +153876,153876 +153877,153877 +153878,153878 +153879,153879 +153880,153880 +153881,153881 +153882,153882 +153883,153883 +153884,153884 +153885,153885 +153886,153886 +153887,153887 +153888,153888 +153889,153889 +153890,153890 +153891,153891 +153892,153892 +153893,153893 +123348,153893 +153894,153894 +153895,153895 +153896,153896 +153897,153897 +153898,153898 +153899,153899 +153900,153900 +153901,153901 +153902,153902 +153903,153903 +153904,153904 +153905,153905 +153906,153906 +153907,153907 +153908,153908 +153909,153909 +220284,153909 +153910,153910 +153911,153911 +31289,153911 +153912,153912 +153913,153913 +153914,153914 +153915,153915 +153916,153916 +153917,153917 +153918,153918 +153919,153919 +153920,153920 +153921,153921 +153922,153922 +220380,153922 +153923,153923 +153924,153924 +60246,153924 +153925,153925 +153926,153926 +153927,153927 +15839,153927 +153928,153928 +153929,153929 +104350,153929 +153930,153930 +153931,153931 +153932,153932 +153933,153933 +153934,153934 +153935,153935 +153936,153936 +153937,153937 +153938,153938 +92650,153938 +153939,153939 +153940,153940 +153941,153941 +153942,153942 +153943,153943 +153944,153944 +153945,153945 +153946,153946 +153947,153947 +153948,153948 +153949,153949 +188748,153949 +153950,153950 +153951,153951 +153952,153952 +211765,153952 +153953,153953 +153954,153954 +153955,153955 +153956,153956 +153957,153957 +153958,153958 +153959,153959 +153960,153960 +153961,153961 +70511,153961 +153962,153962 +153963,153963 +153964,153964 +153965,153965 +153966,153966 +153967,153967 +190839,153967 +153968,153968 +205114,153968 +153969,153969 +153970,153970 +153971,153971 +153972,153972 +153973,153973 +153974,153974 +153975,153975 +153976,153976 +153977,153977 +153978,153978 +153979,153979 +153980,153980 +153981,153981 +153982,153982 +153983,153983 +153984,153984 +153985,153985 +153986,153986 +153987,153987 +153988,153988 +153989,153989 +153990,153990 +153991,153991 +153992,153992 +153993,153993 +153994,153994 +153995,153995 +153996,153996 +153997,153997 +153998,153998 +153999,153999 +154000,154000 +154001,154001 +154002,154002 +154003,154003 +154004,154004 +154005,154005 +154006,154006 +154007,154007 +154008,154008 +154009,154009 +154010,154010 +154011,154011 +154012,154012 +154013,154013 +154014,154014 +154015,154015 +154016,154016 +154017,154017 +154018,154018 +154019,154019 +154020,154020 +154021,154021 +154022,154022 +83395,154022 +154023,154023 +154024,154024 +154025,154025 +154026,154026 +154027,154027 +154028,154028 +154029,154029 +154030,154030 +154031,154031 +154032,154032 +34269,154032 +154033,154033 +154034,154034 +154035,154035 +154036,154036 +154037,154037 +154038,154038 +154039,154039 +154040,154040 +154041,154041 +154042,154042 +154043,154043 +154044,154044 +154045,154045 +154046,154046 +154047,154047 +154048,154048 +154049,154049 +154050,154050 +154051,154051 +154052,154052 +154053,154053 +154054,154054 +154055,154055 +154056,154056 +154057,154057 +154058,154058 +154059,154059 +154060,154060 +154061,154061 +154062,154062 +154063,154063 +154064,154064 +154065,154065 +154066,154066 +154067,154067 +154068,154068 +154069,154069 +154070,154070 +154071,154071 +123946,154071 +154072,154072 +154073,154073 +154074,154074 +154075,154075 +154076,154076 +154077,154077 +154078,154078 +154079,154079 +154080,154080 +154081,154081 +154082,154082 +179917,154082 +154083,154083 +154084,154084 +154085,154085 +154086,154086 +154087,154087 +154088,154088 +154089,154089 +154090,154090 +154091,154091 +154092,154092 +154093,154093 +154094,154094 +154095,154095 +154096,154096 +154097,154097 +154098,154098 +5130,154098 +154099,154099 +154100,154100 +154101,154101 +154102,154102 +154103,154103 +154104,154104 +154105,154105 +114849,154105 +154106,154106 +154107,154107 +21750,154107 +154108,154108 +157069,154108 +157101,154108 +154109,154109 +154110,154110 +11159,154110 +154111,154111 +154112,154112 +154113,154113 +154114,154114 +174006,154114 +168042,154114 +154115,154115 +154116,154116 +154117,154117 +154118,154118 +154119,154119 +154120,154120 +154121,154121 +157153,154121 +154122,154122 +154123,154123 +154124,154124 +154125,154125 +154126,154126 +154127,154127 +154128,154128 +154129,154129 +154130,154130 +154131,154131 +154132,154132 +154133,154133 +154134,154134 +154135,154135 +154136,154136 +154137,154137 +154138,154138 +154139,154139 +154201,154201 +154202,154202 +154203,154203 +154204,154204 +154205,154205 +154206,154206 +205622,154206 +207628,154206 +154207,154207 +154208,154208 +154209,154209 +178112,154209 +172148,154209 +154210,154210 +154211,154211 +154212,154212 +154213,154213 +154214,154214 +154215,154215 +154216,154216 +154217,154217 +154218,154218 +154219,154219 +154220,154220 +154221,154221 +154222,154222 +154322,154322 +154323,154323 +154324,154324 +154325,154325 +154326,154326 +154327,154327 +154328,154328 +154329,154329 +154330,154330 +76060,154330 +154331,154331 +154332,154332 +154333,154333 +154334,154334 +154335,154335 +154336,154336 +154337,154337 +154338,154338 +154339,154339 +154340,154340 +154341,154341 +154342,154342 +9867,154342 +154343,154343 +154344,154344 +154345,154345 +154346,154346 +154347,154347 +206945,154347 +154348,154348 +154349,154349 +154350,154350 +154351,154351 +154352,154352 +154353,154353 +224605,154353 +154354,154354 +154355,154355 +154356,154356 +154357,154357 +154358,154358 +154359,154359 +154360,154360 +154361,154361 +154362,154362 +154363,154363 +154364,154364 +154365,154365 +154366,154366 +154367,154367 +154368,154368 +154369,154369 +154370,154370 +154371,154371 +192607,154371 +154372,154372 +154373,154373 +154498,154498 +154499,154499 +154500,154500 +154501,154501 +6419,154501 +154502,154502 +154503,154503 +154504,154504 +191565,154504 +84107,154504 +154505,154505 +154506,154506 +154507,154507 +154508,154508 +51552,154508 +43448,154508 +154509,154509 +154510,154510 +154511,154511 +154512,154512 +154513,154513 +154514,154514 +65947,154514 +154515,154515 +154516,154516 +85510,154516 +85478,154516 +154517,154517 +154518,154518 +154519,154519 +154520,154520 +154521,154521 +154522,154522 +154523,154523 +154524,154524 +154525,154525 +154526,154526 +154527,154527 +154528,154528 +154529,154529 +154530,154530 +154531,154531 +154532,154532 +154533,154533 +154534,154534 +154535,154535 +154536,154536 +154537,154537 +154538,154538 +154539,154539 +104829,154539 +154540,154540 +154541,154541 +154542,154542 +154543,154543 +62529,154543 +154544,154544 +154545,154545 +154546,154546 +202844,154546 +154547,154547 +154548,154548 +154549,154549 +154550,154550 +154551,154551 +154552,154552 +154553,154553 +212095,154553 +154554,154554 +154555,154555 +154556,154556 +154557,154557 +154558,154558 +154559,154559 +154560,154560 +154561,154561 +154562,154562 +154563,154563 +154564,154564 +154565,154565 +154566,154566 +154567,154567 +154568,154568 +154569,154569 +154570,154570 +154571,154571 +154572,154572 +154573,154573 +154574,154574 +154575,154575 +154576,154576 +154577,154577 +154578,154578 +154579,154579 +154580,154580 +154581,154581 +154582,154582 +154583,154583 +154584,154584 +154585,154585 +24514,154585 +154586,154586 +154587,154587 +154588,154588 +154589,154589 +154590,154590 +154591,154591 +154592,154592 +68024,154592 +154593,154593 +154594,154594 +154595,154595 +154596,154596 +154597,154597 +154598,154598 +154599,154599 +154600,154600 +154601,154601 +154602,154602 +154603,154603 +154604,154604 +154605,154605 +154606,154606 +154607,154607 +154608,154608 +11467,154608 +154609,154609 +154610,154610 +154611,154611 +154612,154612 +154613,154613 +154614,154614 +154615,154615 +149031,154615 +154616,154616 +154617,154617 +154618,154618 +154619,154619 +154620,154620 +154621,154621 +154622,154622 +154707,154707 +154708,154708 +154709,154709 +154710,154710 +154711,154711 +154712,154712 +154713,154713 +154714,154714 +176756,154714 +170792,154714 +154715,154715 +154716,154716 +154717,154717 +154718,154718 +154719,154719 +154720,154720 +154721,154721 +154722,154722 +154723,154723 +154724,154724 +154725,154725 +154726,154726 +154727,154727 +154728,154728 +211592,154728 +154729,154729 +154730,154730 +154731,154731 +154732,154732 +154733,154733 +154734,154734 +154735,154735 +154736,154736 +154737,154737 +154738,154738 +154739,154739 +154740,154740 +154741,154741 +62765,154741 +103394,154741 +154742,154742 +154743,154743 +218206,154743 +154744,154744 +154745,154745 +154746,154746 +154747,154747 +211208,154747 +154748,154748 +154749,154749 +154750,154750 +154751,154751 +154752,154752 +154753,154753 +154754,154754 +154755,154755 +154756,154756 +154757,154757 +154758,154758 +154759,154759 +154760,154760 +154761,154761 +211765,154761 +154762,154762 +154763,154763 +192828,154763 +154764,154764 +154765,154765 +154766,154766 +154767,154767 +154768,154768 +154769,154769 +13520,154769 +154770,154770 +6634,154770 +154771,154771 +154772,154772 +154773,154773 +154774,154774 +154775,154775 +154776,154776 +191256,154776 +154777,154777 +154778,154778 +154779,154779 +154780,154780 +154781,154781 +154782,154782 +154783,154783 +154784,154784 +62267,154784 +194341,154784 +154785,154785 +154786,154786 +154787,154787 +162445,154787 +154788,154788 +184083,154788 +154789,154789 +149145,154789 +154790,154790 +154791,154791 +109217,154791 +154792,154792 +154793,154793 +154794,154794 +154795,154795 +154796,154796 +154797,154797 +154798,154798 +154799,154799 +154800,154800 +154801,154801 +202076,154802 +154802,154802 +154803,154803 +154804,154804 +154805,154805 +154806,154806 +154807,154807 +154808,154808 +154809,154809 +154810,154810 +154811,154811 +154812,154812 +154813,154813 +154814,154814 +50927,154814 +42823,154814 +154815,154815 +154816,154816 +9040,154816 +154817,154817 +154818,154818 +154819,154819 +154820,154820 +154821,154821 +154822,154822 +154823,154823 +154824,154824 +154825,154825 +154826,154826 +154827,154827 +154828,154828 +154829,154829 +154830,154830 +154831,154831 +154832,154832 +194921,154832 +154833,154833 +154834,154834 +154835,154835 +154836,154836 +154837,154837 +154838,154838 +154839,154839 +154840,154840 +154841,154841 +154842,154842 +154843,154843 +154844,154844 +154845,154845 +154846,154846 +154847,154847 +154848,154848 +106945,154848 +154849,154849 +196512,154849 +79888,154849 +154850,154850 +154851,154851 +227558,154852 +154852,154852 +154853,154853 +154854,154854 +154855,154855 +208750,154856 +154856,154856 +154857,154857 +208071,154857 +154858,154858 +154859,154859 +154860,154860 +154861,154861 +154862,154862 +142041,154862 +154863,154863 +154864,154864 +154865,154865 +154866,154866 +154867,154867 +154868,154868 +154869,154869 +154870,154870 +154871,154871 +154872,154872 +142227,154872 +80130,154872 +154873,154873 +154874,154874 +154875,154875 +80133,154875 +154876,154876 +154877,154877 +154878,154878 +154879,154879 +154880,154880 +154881,154881 +154882,154882 +154883,154883 +154884,154884 +154885,154885 +154886,154886 +154887,154887 +154888,154888 +154889,154889 +219406,154889 +154890,154890 +154891,154891 +154892,154892 +154893,154893 +154894,154894 +154895,154895 +154896,154896 +154897,154897 +154898,154898 +154899,154899 +154900,154900 +34011,154900 +154901,154901 +154902,154902 +154903,154903 +154904,154904 +154905,154905 +154906,154906 +154907,154907 +154908,154908 +154909,154909 +154910,154910 +32502,154910 +154911,154911 +154912,154912 +154913,154913 +154914,154914 +154915,154915 +154916,154916 +154917,154917 +154918,154918 +154919,154919 +154920,154920 +154921,154921 +154922,154922 +154923,154923 +154924,154924 +50105,154924 +41912,154924 +154925,154925 +154926,154926 +154927,154927 +154928,154928 +154929,154929 +154930,154930 +154931,154931 +154932,154932 +154933,154933 +142466,154933 +154934,154934 +154935,154935 +154936,154936 +154937,154937 +154938,154938 +154939,154939 +154940,154940 +185407,154940 +154941,154941 +154942,154942 +154943,154943 +154982,154982 +201552,154982 +154983,154983 +154984,154984 +154985,154985 +154986,154986 +154987,154987 +154988,154988 +154989,154989 +154990,154990 +154991,154991 +154992,154992 +154993,154993 +154994,154994 +154995,154995 +154996,154996 +154997,154997 +154998,154998 +164159,154998 +154999,154999 +155000,155000 +155001,155001 +155002,155002 +155003,155003 +155004,155004 +155005,155005 +155006,155006 +155007,155007 +155008,155008 +155009,155009 +155010,155010 +155011,155011 +155012,155012 +155013,155013 +155014,155014 +155015,155015 +155016,155016 +155017,155017 +155018,155018 +155019,155019 +155020,155020 +155021,155021 +155022,155022 +17410,155022 +155023,155023 +155024,155024 +155025,155025 +155026,155026 +155027,155027 +155028,155028 +155029,155029 +155030,155030 +155031,155031 +155032,155032 +155033,155033 +155034,155034 +155035,155035 +155036,155036 +155037,155037 +155038,155038 +155039,155039 +229567,155039 +155040,155040 +155041,155041 +155042,155042 +155043,155043 +155044,155044 +155045,155045 +155046,155046 +155047,155047 +155048,155048 +155049,155049 +155050,155050 +155051,155051 +155052,155052 +155053,155053 +155054,155054 +155055,155055 +155056,155056 +155057,155057 +155058,155058 +155059,155059 +247302,155060 +155060,155060 +155061,155061 +219415,155062 +155062,155062 +155063,155063 +155064,155064 +155065,155065 +155066,155066 +155067,155067 +155068,155068 +155069,155069 +155070,155070 +155071,155071 +36301,155071 +155072,155072 +155073,155073 +155074,155074 +155075,155075 +155076,155076 +138747,155076 +155077,155077 +155078,155078 +155079,155079 +155080,155080 +155081,155081 +155082,155082 +155083,155083 +155084,155084 +155085,155085 +155086,155086 +155087,155087 +155088,155088 +155089,155089 +155090,155090 +155091,155091 +155092,155092 +155093,155093 +155094,155094 +155095,155095 +155096,155096 +155097,155097 +58300,155097 +155098,155098 +155099,155099 +155100,155100 +155101,155101 +155102,155102 +155103,155103 +155104,155104 +155105,155105 +196678,155105 +155106,155106 +155107,155107 +155108,155108 +155109,155109 +155110,155110 +155111,155111 +155112,155112 +155113,155113 +155114,155114 +155115,155115 +155116,155116 +155117,155117 +155118,155118 +148204,155118 +155119,155119 +155120,155120 +155121,155121 +155122,155122 +155123,155123 +155124,155124 +155125,155125 +155126,155126 +155127,155127 +155128,155128 +155129,155129 +155130,155130 +155131,155131 +155132,155132 +155133,155133 +155134,155134 +155135,155135 +226770,155135 +155136,155136 +155137,155137 +155138,155138 +155139,155139 +155140,155140 +155141,155141 +155142,155142 +155143,155143 +155144,155144 +155145,155145 +155146,155146 +155147,155147 +155148,155148 +155149,155149 +155150,155150 +155151,155151 +156394,155152 +155152,155152 +155153,155153 +155154,155154 +155155,155155 +155156,155156 +155157,155157 +195552,155157 +155158,155158 +155159,155159 +155160,155160 +155161,155161 +155162,155162 +138778,155162 +155163,155163 +155164,155164 +194368,155164 +155165,155165 +155166,155166 +194370,155166 +155167,155167 +155168,155168 +155169,155169 +155170,155170 +155171,155171 +155172,155172 +155173,155173 +155174,155174 +155175,155175 +155176,155176 +155177,155177 +155178,155178 +155179,155179 +155180,155180 +164991,155180 +155181,155181 +155182,155182 +155183,155183 +155184,155184 +155185,155185 +155186,155186 +155187,155187 +155188,155188 +224402,155189 +155189,155189 +155190,155190 +232984,155190 +24458,155190 +155191,155191 +155192,155192 +155193,155193 +87791,155193 +155194,155194 +155195,155195 +155196,155196 +155197,155197 +155198,155198 +155199,155199 +155200,155200 +155201,155201 +155202,155202 +155203,155203 +203437,155203 +155204,155204 +155205,155205 +155206,155206 +155207,155207 +155208,155208 +155209,155209 +155210,155210 +155211,155211 +155212,155212 +155213,155213 +155214,155214 +155215,155215 +155216,155216 +155217,155217 +155218,155218 +155219,155219 +155220,155220 +155221,155221 +155222,155222 +155223,155223 +155224,155224 +155225,155225 +155226,155226 +155227,155227 +155228,155228 +155229,155229 +155230,155230 +155231,155231 +155232,155232 +155233,155233 +155234,155234 +155235,155235 +155236,155236 +155237,155237 +155238,155238 +155239,155239 +155240,155240 +191142,155240 +155241,155241 +155242,155242 +155243,155243 +155244,155244 +155245,155245 +107544,155245 +155246,155246 +155247,155247 +155248,155248 +155249,155249 +155250,155250 +155251,155251 +155399,155399 +146107,155399 +155400,155400 +146108,155400 +155401,155401 +146109,155401 +155402,155402 +155403,155403 +155404,155404 +155405,155405 +155406,155406 +146114,155406 +155407,155407 +155408,155408 +146116,155408 +155409,155409 +155410,155410 +155411,155411 +155412,155412 +155413,155413 +155414,155414 +155415,155415 +155416,155416 +155417,155417 +146125,155417 +155418,155418 +155419,155419 +155420,155420 +155421,155421 +151792,155421 +146129,155421 +155422,155422 +248720,155422 +151793,155422 +146130,155422 +155423,155423 +63851,155423 +155424,155424 +155425,155425 +155426,155426 +155427,155427 +155428,155428 +77627,155428 +151799,155428 +155429,155429 +155430,155430 +155431,155431 +155432,155432 +151803,155432 +137542,155432 +129955,155432 +155433,155433 +137744,155433 +130157,155433 +155434,155434 +155435,155435 +155436,155436 +145822,155436 +155437,155437 +137748,155437 +130161,155437 +155438,155438 +155439,155439 +155440,155440 +155441,155441 +155442,155442 +155443,155443 +137347,155443 +129760,155443 +155444,155444 +155445,155445 +146153,155445 +137555,155445 +129968,155445 +155446,155446 +155447,155447 +155448,155448 +25130,155448 +155449,155449 +155450,155450 +155451,155451 +178871,155451 +155452,155452 +155453,155453 +155454,155454 +155455,155455 +155456,155456 +155603,155603 +155604,155604 +117735,155604 +32446,155604 +155605,155605 +155606,155606 +230400,155606 +155607,155607 +155608,155608 +155609,155609 +155610,155610 +155611,155611 +155612,155612 +182321,155613 +155613,155613 +2691,155613 +155614,155614 +155615,155615 +155616,155616 +155617,155617 +155618,155618 +155619,155619 +89566,155619 +155620,155620 +155621,155621 +121696,155621 +155622,155622 +155623,155623 +155624,155624 +155625,155625 +155626,155626 +155627,155627 +155628,155628 +155629,155629 +155630,155630 +155631,155631 +155632,155632 +155633,155633 +155634,155634 +155635,155635 +155636,155636 +155637,155637 +155638,155638 +222465,155638 +155639,155639 +155640,155640 +145167,155640 +155641,155641 +155642,155642 +155643,155643 +155644,155644 +155853,155853 +155854,155854 +155855,155855 +155856,155856 +155857,155857 +155858,155858 +155859,155859 +116598,155859 +155860,155860 +155861,155861 +155862,155862 +155863,155863 +155864,155864 +155865,155865 +155866,155866 +155867,155867 +155868,155868 +155869,155869 +155870,155870 +155871,155871 +155872,155872 +155873,155873 +155874,155874 +155875,155875 +155876,155876 +155877,155877 +207861,155877 +207771,155877 +155878,155878 +155879,155879 +155880,155880 +155881,155881 +155882,155882 +155883,155883 +83774,155883 +190510,155883 +155884,155884 +155885,155885 +155886,155886 +155887,155887 +155888,155888 +155889,155889 +155890,155890 +155891,155891 +155892,155892 +155893,155893 +164853,155893 +155894,155894 +155895,155895 +155896,155896 +155897,155897 +155898,155898 +159501,155898 +155899,155899 +155900,155900 +155901,155901 +155902,155902 +155903,155903 +155904,155904 +155905,155905 +155906,155906 +155907,155907 +155908,155908 +155909,155909 +155910,155910 +71176,155910 +155911,155911 +155912,155912 +155913,155913 +155914,155914 +155915,155915 +155916,155916 +155917,155917 +155918,155918 +155919,155919 +155920,155920 +155921,155921 +155922,155922 +155923,155923 +155924,155924 +207707,155924 +155925,155925 +190442,155925 +155926,155926 +155927,155927 +155928,155928 +155929,155929 +155930,155930 +155931,155931 +155932,155932 +142192,155932 +155933,155933 +15842,155933 +155934,155934 +155935,155935 +155936,155936 +135756,155936 +128169,155936 +109828,155936 +155937,155937 +155938,155938 +155939,155939 +141570,155939 +155940,155940 +155941,155941 +155942,155942 +155943,155943 +77225,155943 +155944,155944 +155945,155945 +155946,155946 +155947,155947 +155948,155948 +155949,155949 +218378,155949 +155950,155950 +155951,155951 +155952,155952 +155953,155953 +155954,155954 +155955,155955 +155956,155956 +155957,155957 +155958,155958 +155959,155959 +155960,155960 +155961,155961 +156102,156102 +156103,156103 +160740,156103 +156104,156104 +156105,156105 +156106,156106 +156107,156107 +156108,156108 +156109,156109 +156190,156190 +156258,156258 +156259,156259 +156260,156260 +156261,156261 +156262,156262 +156263,156263 +156264,156264 +156265,156265 +156266,156266 +156267,156267 +156268,156268 +156269,156269 +156270,156270 +59424,156270 +156271,156271 +156272,156272 +156273,156273 +156274,156274 +61358,156274 +156275,156275 +156276,156276 +156277,156277 +156278,156278 +156279,156279 +156280,156280 +156281,156281 +156282,156282 +156283,156283 +156284,156284 +156285,156285 +156286,156286 +156287,156287 +156288,156288 +156289,156289 +156290,156290 +142089,156290 +156291,156291 +156292,156292 +156293,156293 +156294,156294 +211380,156294 +156295,156295 +156296,156296 +156297,156297 +7649,156297 +156298,156298 +156299,156299 +156300,156300 +156301,156301 +156302,156302 +178606,156302 +156303,156303 +156304,156304 +156305,156305 +156306,156306 +156307,156307 +156308,156308 +156309,156309 +156310,156310 +81329,156310 +156311,156311 +156312,156312 +156313,156313 +156314,156314 +156315,156315 +156316,156316 +156317,156317 +156318,156318 +156319,156319 +156320,156320 +190332,156320 +156321,156321 +156322,156322 +156323,156323 +156324,156324 +156325,156325 +156326,156326 +156327,156327 +156328,156328 +156329,156329 +156330,156330 +156331,156331 +156332,156332 +156333,156333 +156334,156334 +213290,156334 +156335,156335 +156336,156336 +156337,156337 +156338,156338 +156339,156339 +156340,156340 +156341,156341 +156342,156342 +156343,156343 +156344,156344 +156345,156345 +156346,156346 +156347,156347 +156348,156348 +156349,156349 +156350,156350 +156351,156351 +156352,156352 +156353,156353 +156354,156354 +156355,156355 +156356,156356 +156357,156357 +156358,156358 +164859,156358 +156359,156359 +191186,156359 +156360,156360 +156361,156361 +156362,156362 +156363,156363 +156364,156364 +156365,156365 +156366,156366 +156367,156367 +156368,156368 +156369,156369 +156370,156370 +156371,156371 +156372,156372 +156373,156373 +156374,156374 +156375,156375 +156376,156376 +156377,156377 +156378,156378 +156379,156379 +65037,156379 +156380,156380 +156381,156381 +156382,156382 +156383,156383 +156384,156384 +156385,156385 +2793,156385 +156386,156386 +156387,156387 +156388,156388 +156389,156389 +156390,156390 +156391,156391 +156392,156392 +156393,156393 +156394,156394 +155152,156394 +156395,156395 +156396,156396 +156397,156397 +156398,156398 +107557,156398 +145366,156398 +156399,156399 +156400,156400 +156401,156401 +224353,156401 +156402,156402 +59994,156402 +156403,156403 +156404,156404 +156405,156405 +156406,156406 +156407,156407 +156408,156408 +156409,156409 +156410,156410 +156411,156411 +65604,156411 +84379,156411 +156412,156412 +156413,156413 +156414,156414 +156415,156415 +220631,156415 +156416,156416 +156417,156417 +156418,156418 +60343,156418 +156419,156419 +156420,156420 +156421,156421 +156422,156422 +156423,156423 +156424,156424 +156425,156425 +156426,156426 +156427,156427 +156428,156428 +156429,156429 +156430,156430 +156431,156431 +156432,156432 +156433,156433 +156434,156434 +156435,156435 +156436,156436 +156437,156437 +156438,156438 +156439,156439 +156440,156440 +156441,156441 +156442,156442 +156443,156443 +156444,156444 +81475,156444 +156445,156445 +97891,156445 +156446,156446 +156447,156447 +156448,156448 +156449,156449 +156450,156450 +156451,156451 +156452,156452 +156453,156453 +156454,156454 +156455,156455 +156456,156456 +156457,156457 +156458,156458 +156459,156459 +156460,156460 +156461,156461 +156462,156462 +156463,156463 +156464,156464 +156465,156465 +156466,156466 +156467,156467 +156468,156468 +156469,156469 +156470,156470 +156471,156471 +156472,156472 +156473,156473 +156474,156474 +2064,156474 +156475,156475 +156476,156476 +156477,156477 +156478,156478 +156479,156479 +156480,156480 +156481,156481 +86103,156481 +156482,156482 +156483,156483 +156484,156484 +156485,156485 +156486,156486 +156487,156487 +156488,156488 +156489,156489 +185091,156489 +156490,156490 +156491,156491 +156492,156492 +156493,156493 +156494,156494 +136072,156494 +128485,156494 +156495,156495 +156496,156496 +156497,156497 +156498,156498 +156499,156499 +156500,156500 +156501,156501 +156502,156502 +156503,156503 +156504,156504 +156505,156505 +156506,156506 +156507,156507 +156508,156508 +156509,156509 +156510,156510 +156574,156574 +62277,156574 +156575,156575 +156576,156576 +156577,156577 +197595,156578 +156578,156578 +156579,156579 +156580,156580 +156581,156581 +156582,156582 +156583,156583 +156584,156584 +156585,156585 +156586,156586 +156587,156587 +156588,156588 +156627,156627 +156628,156628 +59627,156628 +156629,156629 +156630,156630 +156631,156631 +156632,156632 +156633,156633 +156634,156634 +156635,156635 +156636,156636 +386,156636 +49496,156636 +41303,156636 +156637,156637 +156638,156638 +4995,156638 +156639,156639 +156640,156640 +156641,156641 +156642,156642 +156643,156643 +156644,156644 +156645,156645 +156646,156646 +156647,156647 +156648,156648 +156649,156649 +156650,156650 +156651,156651 +156652,156652 +156653,156653 +156654,156654 +156655,156655 +156656,156656 +156657,156657 +156658,156658 +156659,156659 +111606,156659 +156660,156660 +156661,156661 +156662,156662 +156663,156663 +156664,156664 +156665,156665 +156666,156666 +156667,156667 +156668,156668 +156669,156669 +156670,156670 +156671,156671 +156672,156672 +156673,156673 +156674,156674 +237478,156674 +156675,156675 +156676,156676 +156677,156677 +156678,156678 +156679,156679 +156680,156680 +156681,156681 +156682,156682 +156683,156683 +156684,156684 +156685,156685 +156686,156686 +156687,156687 +156688,156688 +156689,156689 +110776,156689 +156690,156690 +156691,156691 +156692,156692 +156693,156693 +156694,156694 +156695,156695 +156696,156696 +156697,156697 +156698,156698 +156699,156699 +156700,156700 +156701,156701 +156702,156702 +156703,156703 +156704,156704 +156705,156705 +156706,156706 +156707,156707 +156708,156708 +156709,156709 +156793,156793 +156794,156794 +156795,156795 +156796,156796 +156797,156797 +156798,156798 +156799,156799 +156800,156800 +156801,156801 +156802,156802 +156803,156803 +156804,156804 +156805,156805 +156806,156806 +156807,156807 +156808,156808 +156809,156809 +156810,156810 +156811,156811 +156812,156812 +156813,156813 +156814,156814 +156815,156815 +156816,156816 +116858,156816 +156817,156817 +156818,156818 +156819,156819 +156820,156820 +156821,156821 +156822,156822 +156823,156823 +156824,156824 +156825,156825 +156826,156826 +156827,156827 +156828,156828 +156829,156829 +26225,156829 +156830,156830 +156831,156831 +186795,156831 +156832,156832 +156833,156833 +240945,156833 +156834,156834 +156835,156835 +111968,156835 +156836,156836 +156837,156837 +156838,156838 +156839,156839 +205101,156839 +156840,156840 +156841,156841 +156842,156842 +156843,156843 +157013,157013 +157014,157014 +15384,157014 +157015,157015 +157016,157016 +157017,157017 +157018,157018 +157019,157019 +198470,157019 +157020,157020 +157021,157021 +157022,157022 +157023,157023 +157024,157024 +157025,157025 +157026,157026 +157027,157027 +157028,157028 +157029,157029 +157030,157030 +157031,157031 +157032,157032 +157033,157033 +157034,157034 +157035,157035 +157036,157036 +157037,157037 +157038,157038 +157039,157039 +157040,157040 +157041,157041 +157042,157042 +157043,157043 +211556,157044 +157044,157044 +157045,157045 +77365,157045 +157046,157046 +157047,157047 +157048,157048 +157049,157049 +157050,157050 +157051,157051 +157052,157052 +157053,157053 +157054,157054 +157055,157055 +157056,157056 +157057,157057 +157058,157058 +157817,157059 +157059,157059 +157060,157060 +157061,157061 +75086,157061 +157062,157062 +157063,157063 +157064,157064 +157065,157065 +157066,157066 +157067,157067 +157068,157068 +29063,157068 +157069,157069 +154108,157069 +157070,157070 +157071,157071 +157072,157072 +157073,157073 +157074,157074 +157075,157075 +157076,157076 +157077,157077 +157078,157078 +157079,157079 +157080,157080 +157081,157081 +157082,157082 +157083,157083 +157084,157084 +157085,157085 +178843,157085 +157086,157086 +157087,157087 +157088,157088 +157089,157089 +157090,157090 +157091,157091 +211419,157092 +157092,157092 +157093,157093 +104302,157093 +157094,157094 +157095,157095 +76877,157095 +157096,157096 +157097,157097 +157098,157098 +157099,157099 +157100,157100 +157101,157101 +154108,157101 +157102,157102 +157103,157103 +157104,157104 +157105,157105 +157106,157106 +157107,157107 +157108,157108 +157109,157109 +157110,157110 +157111,157111 +157112,157112 +157113,157113 +157114,157114 +157115,157115 +248111,157115 +157116,157116 +157117,157117 +157118,157118 +157119,157119 +157120,157120 +157121,157121 +157122,157122 +157123,157123 +238617,157123 +157124,157124 +157125,157125 +157126,157126 +157127,157127 +157128,157128 +157129,157129 +157130,157130 +157131,157131 +157132,157132 +157133,157133 +157134,157134 +157135,157135 +157136,157136 +157137,157137 +101755,157137 +157138,157138 +157139,157139 +157140,157140 +157141,157141 +157142,157142 +157143,157143 +157144,157144 +219918,157144 +157145,157145 +157146,157146 +157147,157147 +157148,157148 +71373,157148 +157149,157149 +157150,157150 +157151,157151 +157152,157152 +226532,157152 +226369,157152 +157153,157153 +154121,157153 +157154,157154 +157155,157155 +157156,157156 +157157,157157 +157158,157158 +157159,157159 +157160,157160 +157161,157161 +157162,157162 +157163,157163 +157164,157164 +157165,157165 +157166,157166 +157167,157167 +157168,157168 +157169,157169 +157170,157170 +157171,157171 +157172,157172 +157173,157173 +157174,157174 +157175,157175 +157176,157176 +157177,157177 +157178,157178 +157179,157179 +157180,157180 +157181,157181 +157182,157182 +157183,157183 +157184,157184 +157185,157185 +157186,157186 +157187,157187 +157188,157188 +157189,157189 +157190,157190 +157191,157191 +157192,157192 +157193,157193 +157194,157194 +157195,157195 +157196,157196 +157197,157197 +193256,157197 +157198,157198 +157199,157199 +157200,157200 +157201,157201 +27139,157201 +157202,157202 +157203,157203 +157204,157204 +157205,157205 +157206,157206 +157207,157207 +157208,157208 +27146,157208 +157209,157209 +157210,157210 +193679,157210 +157211,157211 +157212,157212 +157213,157213 +157214,157214 +157215,157215 +157216,157216 +157217,157217 +157218,157218 +157219,157219 +157220,157220 +157221,157221 +157222,157222 +157223,157223 +157224,157224 +157225,157225 +157226,157226 +157227,157227 +157228,157228 +157229,157229 +157230,157230 +157231,157231 +78722,157231 +157267,157267 +157268,157268 +157269,157269 +157270,157270 +157370,157370 +157371,157371 +157372,157372 +157373,157373 +103594,157373 +157374,157374 +222152,157374 +157375,157375 +157376,157376 +192177,157376 +157377,157377 +157378,157378 +157379,157379 +157380,157380 +157381,157381 +157382,157382 +157383,157383 +157384,157384 +157385,157385 +157386,157386 +157387,157387 +157388,157388 +157389,157389 +75834,157389 +157390,157390 +157391,157391 +157392,157392 +157393,157393 +157394,157394 +157395,157395 +157396,157396 +157397,157397 +157398,157398 +157399,157399 +157400,157400 +157401,157401 +5127,157401 +157402,157402 +157403,157403 +157404,157404 +157405,157405 +157531,157531 +157532,157532 +157533,157533 +157534,157534 +176496,157534 +170532,157534 +157535,157535 +110541,157535 +157536,157536 +60063,157536 +157537,157537 +157538,157538 +57835,157538 +157539,157539 +157540,157540 +157541,157541 +157542,157542 +157543,157543 +157544,157544 +157545,157545 +157546,157546 +157547,157547 +164520,157547 +157548,157548 +157549,157549 +157550,157550 +157551,157551 +157552,157552 +157553,157553 +157554,157554 +157555,157555 +132291,157555 +124704,157555 +157556,157556 +157557,157557 +157558,157558 +157559,157559 +157560,157560 +157561,157561 +157562,157562 +157563,157563 +157564,157564 +157565,157565 +157566,157566 +157567,157567 +157568,157568 +157569,157569 +157570,157570 +157571,157571 +157572,157572 +157573,157573 +157574,157574 +157575,157575 +157576,157576 +157577,157577 +157578,157578 +157579,157579 +165171,157579 +157580,157580 +157581,157581 +157582,157582 +157583,157583 +157584,157584 +157585,157585 +157586,157586 +157587,157587 +157588,157588 +157589,157589 +157590,157590 +157591,157591 +157592,157592 +142138,157592 +157593,157593 +228719,157594 +157594,157594 +157595,157595 +157596,157596 +157597,157597 +157598,157598 +157599,157599 +157752,157752 +157753,157753 +157754,157754 +157755,157755 +157756,157756 +157757,157757 +157758,157758 +157759,157759 +249600,157759 +157760,157760 +157761,157761 +157762,157762 +157763,157763 +157764,157764 +157765,157765 +157766,157766 +157767,157767 +157768,157768 +157769,157769 +190891,157769 +157770,157770 +157771,157771 +113419,157771 +157772,157772 +157773,157773 +240935,157773 +157774,157774 +157775,157775 +157776,157776 +157777,157777 +157778,157778 +157779,157779 +157780,157780 +157781,157781 +157782,157782 +157783,157783 +157784,157784 +157785,157785 +157786,157786 +157787,157787 +180662,157787 +157788,157788 +82561,157788 +157789,157789 +157790,157790 +157791,157791 +157792,157792 +157793,157793 +157794,157794 +157795,157795 +157796,157796 +157797,157797 +157798,157798 +157799,157799 +157800,157800 +221747,157800 +157801,157801 +157802,157802 +157803,157803 +157804,157804 +157805,157805 +157806,157806 +157807,157807 +157808,157808 +208845,157808 +226683,157808 +157809,157809 +157810,157810 +157811,157811 +77976,157811 +157812,157812 +77247,157812 +157813,157813 +157814,157814 +157815,157815 +157816,157816 +157817,157817 +157059,157817 +157818,157818 +157819,157819 +157820,157820 +157821,157821 +157822,157822 +157823,157823 +157824,157824 +157825,157825 +114827,157825 +157826,157826 +157827,157827 +157828,157828 +157829,157829 +157830,157830 +157831,157831 +38469,157831 +157832,157832 +157833,157833 +157834,157834 +157835,157835 +157836,157836 +228979,157836 +157837,157837 +157838,157838 +214477,157838 +157839,157839 +157840,157840 +157841,157841 +157842,157842 +157843,157843 +157844,157844 +157845,157845 +157846,157846 +157847,157847 +74715,157847 +157848,157848 +157849,157849 +157850,157850 +157851,157851 +157852,157852 +157853,157853 +157854,157854 +157855,157855 +157856,157856 +157857,157857 +157858,157858 +157859,157859 +157860,157860 +157861,157861 +157862,157862 +157863,157863 +276,157863 +157864,157864 +157865,157865 +157866,157866 +157867,157867 +157868,157868 +157869,157869 +157870,157870 +157871,157871 +157872,157872 +157873,157873 +157874,157874 +157875,157875 +157876,157876 +157877,157877 +157878,157878 +157879,157879 +157880,157880 +157881,157881 +157882,157882 +157883,157883 +157884,157884 +157885,157885 +157886,157886 +157887,157887 +157888,157888 +157889,157889 +157890,157890 +157891,157891 +157892,157892 +157893,157893 +157894,157894 +157895,157895 +157896,157896 +157897,157897 +157898,157898 +157899,157899 +157900,157900 +157901,157901 +157902,157902 +157903,157903 +157904,157904 +157905,157905 +157906,157906 +157907,157907 +57038,157907 +157908,157908 +157909,157909 +157910,157910 +157911,157911 +157912,157912 +157913,157913 +157914,157914 +157915,157915 +157916,157916 +157917,157917 +185149,157917 +157918,157918 +157919,157919 +157920,157920 +157921,157921 +157922,157922 +157923,157923 +157924,157924 +157993,157993 +225981,157993 +157994,157994 +157995,157995 +157996,157996 +157997,157997 +157998,157998 +157999,157999 +158000,158000 +158001,158001 +158002,158002 +158003,158003 +158004,158004 +158005,158005 +158006,158006 +158007,158007 +158008,158008 +158009,158009 +158010,158010 +81623,158010 +158011,158011 +158012,158012 +158013,158013 +158014,158014 +158015,158015 +158016,158016 +158017,158017 +158018,158018 +158019,158019 +158020,158020 +158021,158021 +158022,158022 +158023,158023 +158024,158024 +158025,158025 +158026,158026 +158027,158027 +158028,158028 +158029,158029 +158030,158030 +158031,158031 +158032,158032 +158033,158033 +158034,158034 +158035,158035 +158036,158036 +158037,158037 +158038,158038 +158039,158039 +158040,158040 +109874,158040 +95156,158040 +158041,158041 +158042,158042 +158043,158043 +158044,158044 +158045,158045 +158046,158046 +158047,158047 +158048,158048 +158049,158049 +158050,158050 +158051,158051 +158052,158052 +158053,158053 +206728,158053 +158054,158054 +186800,158054 +158055,158055 +158056,158056 +158057,158057 +158058,158058 +80455,158058 +158059,158059 +158060,158060 +158061,158061 +158062,158062 +158063,158063 +158064,158064 +158065,158065 +158066,158066 +149104,158066 +158067,158067 +158068,158068 +158069,158069 +158070,158070 +158071,158071 +158072,158072 +158073,158073 +158074,158074 +158075,158075 +244025,158075 +120236,158075 +158076,158076 +158077,158077 +158078,158078 +3510,158078 +158079,158079 +158080,158080 +158081,158081 +158082,158082 +158083,158083 +158084,158084 +158085,158085 +158086,158086 +158087,158087 +158088,158088 +158089,158089 +158090,158090 +158091,158091 +158092,158092 +158093,158093 +158094,158094 +158095,158095 +158096,158096 +158097,158097 +158098,158098 +158099,158099 +158100,158100 +158101,158101 +158102,158102 +158103,158103 +158104,158104 +158105,158105 +158106,158106 +102118,158106 +158107,158107 +158108,158108 +158109,158109 +138001,158109 +130414,158109 +158110,158110 +158111,158111 +158112,158112 +158113,158113 +158114,158114 +158115,158115 +158116,158116 +158117,158117 +158118,158118 +158119,158119 +158120,158120 +158121,158121 +158122,158122 +158123,158123 +158124,158124 +158125,158125 +158126,158126 +158127,158127 +158128,158128 +158129,158129 +158130,158130 +211543,158130 +158131,158131 +158132,158132 +158133,158133 +158134,158134 +158135,158135 +158136,158136 +158137,158137 +67536,158137 +158138,158138 +158139,158139 +158140,158140 +158141,158141 +158142,158142 +158143,158143 +158144,158144 +158145,158145 +158146,158146 +158147,158147 +158148,158148 +158149,158149 +158150,158150 +158151,158151 +158152,158152 +158153,158153 +158154,158154 +158155,158155 +158156,158156 +158157,158157 +158158,158158 +158159,158159 +158160,158160 +158161,158161 +158162,158162 +158163,158163 +158164,158164 +158165,158165 +158166,158166 +158167,158167 +158168,158168 +158169,158169 +158170,158170 +158209,158209 +248158,158209 +158210,158210 +158211,158211 +158212,158212 +158213,158213 +158214,158214 +158215,158215 +158216,158216 +158217,158217 +158218,158218 +158219,158219 +158220,158220 +158221,158221 +212113,158221 +158222,158222 +214076,158222 +158223,158223 +158224,158224 +158225,158225 +158226,158226 +158227,158227 +158228,158228 +158229,158229 +98233,158229 +158230,158230 +158231,158231 +158232,158232 +158233,158233 +158234,158234 +158235,158235 +158236,158236 +158237,158237 +70372,158237 +158238,158238 +158239,158239 +158240,158240 +158241,158241 +158242,158242 +158243,158243 +158244,158244 +158245,158245 +158246,158246 +158247,158247 +158297,158297 +158298,158298 +158299,158299 +158300,158300 +158301,158301 +193410,158301 +158302,158302 +158303,158303 +158304,158304 +158305,158305 +158306,158306 +158307,158307 +199879,158307 +158308,158308 +158309,158309 +158310,158310 +158311,158311 +158312,158312 +158313,158313 +158314,158314 +86666,158314 +158315,158315 +158316,158316 +158317,158317 +158318,158318 +158319,158319 +158320,158320 +158321,158321 +158322,158322 +158323,158323 +65115,158323 +163343,158323 +158324,158324 +158325,158325 +158326,158326 +158327,158327 +158328,158328 +158329,158329 +158330,158330 +158331,158331 +141644,158331 +158332,158332 +158333,158333 +158334,158334 +158335,158335 +147055,158335 +158336,158336 +158337,158337 +158338,158338 +158339,158339 +65093,158339 +158340,158340 +158341,158341 +158342,158342 +158343,158343 +158344,158344 +158345,158345 +158346,158346 +158417,158417 +158418,158418 +158419,158419 +158420,158420 +158421,158421 +158422,158422 +158423,158423 +76189,158423 +158424,158424 +85432,158424 +158425,158425 +158426,158426 +158427,158427 +158428,158428 +158429,158429 +158430,158430 +158431,158431 +158432,158432 +158433,158433 +158434,158434 +158435,158435 +146493,158435 +158436,158436 +158437,158437 +158438,158438 +185173,158438 +158439,158439 +158440,158440 +158441,158441 +158442,158442 +158443,158443 +137068,158443 +129481,158443 +158444,158444 +158445,158445 +158446,158446 +158447,158447 +158448,158448 +158449,158449 +158450,158450 +158451,158451 +158452,158452 +158453,158453 +243487,158453 +158454,158454 +158455,158455 +158456,158456 +60952,158456 +158457,158457 +158458,158458 +158459,158459 +158460,158460 +158461,158461 +158462,158462 +192450,158462 +158463,158463 +60959,158463 +215944,158463 +158464,158464 +158465,158465 +158466,158466 +158467,158467 +158468,158468 +158469,158469 +158470,158470 +60019,158470 +158471,158471 +158472,158472 +158473,158473 +158474,158474 +158475,158475 +158476,158476 +158477,158477 +158478,158478 +158479,158479 +158480,158480 +158481,158481 +158482,158482 +158483,158483 +158484,158484 +158485,158485 +158486,158486 +158487,158487 +158488,158488 +158489,158489 +91025,158489 +158490,158490 +158491,158491 +150627,158491 +91027,158491 +158492,158492 +6698,158492 +158493,158493 +158494,158494 +162870,158494 +158495,158495 +158496,158496 +158497,158497 +158498,158498 +158499,158499 +158500,158500 +158501,158501 +158502,158502 +158503,158503 +158504,158504 +158505,158505 +158506,158506 +158507,158507 +64405,158507 +158508,158508 +138890,158508 +158509,158509 +158510,158510 +158511,158511 +158512,158512 +158513,158513 +158514,158514 +158515,158515 +158516,158516 +158517,158517 +158518,158518 +158519,158519 +158520,158520 +158521,158521 +143215,158521 +158584,158584 +158585,158585 +158586,158586 +158587,158587 +158588,158588 +158589,158589 +158590,158590 +231590,158590 +158591,158591 +158592,158592 +158593,158593 +158594,158594 +158595,158595 +158596,158596 +158597,158597 +158598,158598 +158599,158599 +158600,158600 +158601,158601 +158602,158602 +158603,158603 +158604,158604 +158605,158605 +56163,158605 +48059,158605 +158606,158606 +83331,158606 +31822,158606 +158607,158607 +158608,158608 +158609,158609 +87775,158609 +158610,158610 +158611,158611 +158612,158612 +158613,158613 +105020,158613 +158614,158614 +158615,158615 +158616,158616 +158617,158617 +158618,158618 +158619,158619 +158620,158620 +158621,158621 +158622,158622 +158623,158623 +158624,158624 +158625,158625 +214053,158625 +162659,158626 +158626,158626 +158627,158627 +158628,158628 +158629,158629 +158630,158630 +158631,158631 +158632,158632 +158633,158633 +158634,158634 +158635,158635 +158636,158636 +158637,158637 +158638,158638 +184539,158638 +158639,158639 +158640,158640 +158641,158641 +158642,158642 +158643,158643 +158644,158644 +158645,158645 +190056,158645 +158646,158646 +243386,158647 +158647,158647 +158648,158648 +158649,158649 +158650,158650 +158651,158651 +158652,158652 +158653,158653 +158654,158654 +158655,158655 +158656,158656 +158657,158657 +158658,158658 +158659,158659 +158660,158660 +158661,158661 +158662,158662 +158663,158663 +158664,158664 +158665,158665 +158666,158666 +158667,158667 +158668,158668 +158669,158669 +158670,158670 +158671,158671 +158672,158672 +158673,158673 +144244,158673 +158674,158674 +158675,158675 +158676,158676 +158677,158677 +158678,158678 +158679,158679 +158680,158680 +158681,158681 +68057,158681 +158682,158682 +158683,158683 +202733,158683 +158684,158684 +158685,158685 +158686,158686 +158687,158687 +158688,158688 +158689,158689 +158690,158690 +158691,158691 +158692,158692 +158693,158693 +82743,158693 +158694,158694 +158695,158695 +158696,158696 +158697,158697 +158698,158698 +158699,158699 +158700,158700 +158701,158701 +158702,158702 +158703,158703 +158704,158704 +158705,158705 +158706,158706 +158707,158707 +158708,158708 +158709,158709 +158710,158710 +158711,158711 +158712,158712 +158713,158713 +158714,158714 +158715,158715 +158716,158716 +158717,158717 +158718,158718 +158719,158719 +158720,158720 +158721,158721 +158722,158722 +158723,158723 +158724,158724 +158725,158725 +158726,158726 +158727,158727 +158728,158728 +158729,158729 +158730,158730 +158731,158731 +158732,158732 +158733,158733 +158734,158734 +158735,158735 +249986,158735 +165000,158735 +158736,158736 +158737,158737 +38999,158737 +158738,158738 +158739,158739 +158740,158740 +158741,158741 +158742,158742 +38373,158742 +158743,158743 +158744,158744 +158745,158745 +158746,158746 +158747,158747 +158748,158748 +158749,158749 +158750,158750 +158751,158751 +158752,158752 +160744,158752 +158753,158753 +160773,158753 +158754,158754 +158755,158755 +158756,158756 +158757,158757 +158758,158758 +214459,158758 +158759,158759 +158760,158760 +158761,158761 +158762,158762 +158763,158763 +158764,158764 +158765,158765 +158766,158766 +139154,158766 +158767,158767 +158768,158768 +158769,158769 +158770,158770 +158771,158771 +158772,158772 +158773,158773 +158774,158774 +158775,158775 +158776,158776 +158777,158777 +158778,158778 +158779,158779 +185871,158779 +158780,158780 +158781,158781 +158782,158782 +158783,158783 +111920,158783 +158784,158784 +158785,158785 +158786,158786 +158787,158787 +98813,158787 +158788,158788 +158789,158789 +158790,158790 +158791,158791 +158792,158792 +158793,158793 +158794,158794 +176607,158794 +170643,158794 +158795,158795 +205085,158795 +158796,158796 +158797,158797 +158798,158798 +62460,158798 +208474,158799 +158799,158799 +158800,158800 +158801,158801 +158802,158802 +33232,158802 +158803,158803 +158804,158804 +212130,158804 +158805,158805 +158806,158806 +158807,158807 +158808,158808 +158809,158809 +158810,158810 +158971,158971 +158972,158972 +51711,158972 +43607,158972 +158973,158973 +158974,158974 +158975,158975 +158976,158976 +158977,158977 +95891,158977 +158978,158978 +158979,158979 +158980,158980 +158981,158981 +158982,158982 +6773,158982 +158983,158983 +158984,158984 +158985,158985 +158986,158986 +158987,158987 +158988,158988 +158989,158989 +158990,158990 +51306,158990 +43202,158990 +158991,158991 +158992,158992 +158993,158993 +158994,158994 +158995,158995 +158996,158996 +159117,159117 +159118,159118 +159119,159119 +159120,159120 +159121,159121 +159122,159122 +159123,159123 +159124,159124 +159125,159125 +159126,159126 +159127,159127 +159128,159128 +159129,159129 +159130,159130 +159131,159131 +159132,159132 +159133,159133 +159134,159134 +159135,159135 +159136,159136 +159137,159137 +159138,159138 +159139,159139 +159188,159188 +159189,159189 +159190,159190 +159191,159191 +159192,159192 +159229,159229 +143911,159229 +159230,159230 +159231,159231 +159232,159232 +190919,159232 +159233,159233 +159234,159234 +159235,159235 +159236,159236 +159237,159237 +159238,159238 +159239,159239 +159240,159240 +159241,159241 +159242,159242 +159243,159243 +159244,159244 +159245,159245 +159246,159246 +159247,159247 +159248,159248 +159249,159249 +159250,159250 +159251,159251 +159252,159252 +159253,159253 +159254,159254 +30349,159254 +159255,159255 +217568,159255 +159256,159256 +159257,159257 +159258,159258 +159259,159259 +159260,159260 +159261,159261 +159262,159262 +159263,159263 +159264,159264 +159265,159265 +159266,159266 +159267,159267 +159268,159268 +159269,159269 +159270,159270 +159271,159271 +159272,159272 +159273,159273 +159274,159274 +159275,159275 +159276,159276 +159277,159277 +159340,159340 +159341,159341 +159342,159342 +159343,159343 +159344,159344 +183866,159344 +159345,159345 +159346,159346 +159347,159347 +159348,159348 +159349,159349 +159350,159350 +159351,159351 +159352,159352 +159353,159353 +159354,159354 +159355,159355 +159356,159356 +159357,159357 +159358,159358 +159359,159359 +159360,159360 +159361,159361 +159362,159362 +35716,159362 +4169,159362 +159363,159363 +159364,159364 +159365,159365 +159366,159366 +159367,159367 +159368,159368 +83644,159368 +79473,159368 +159369,159369 +159370,159370 +96485,159370 +159371,159371 +106797,159371 +159372,159372 +97948,159372 +159373,159373 +159374,159374 +226670,159374 +159375,159375 +159376,159376 +159377,159377 +159378,159378 +159379,159379 +159380,159380 +159381,159381 +159382,159382 +111995,159382 +159383,159383 +159384,159384 +159385,159385 +159386,159386 +159387,159387 +233399,159387 +159388,159388 +159389,159389 +159390,159390 +100584,159390 +159391,159391 +159392,159392 +159393,159393 +159394,159394 +159395,159395 +159396,159396 +159397,159397 +159398,159398 +159399,159399 +159400,159400 +217966,159400 +159401,159401 +159402,159402 +159403,159403 +159404,159404 +159405,159405 +159406,159406 +190284,159406 +159407,159407 +159408,159408 +215526,159408 +159409,159409 +159410,159410 +159411,159411 +159412,159412 +159413,159413 +159414,159414 +159415,159415 +159416,159416 +159417,159417 +159418,159418 +159419,159419 +159420,159420 +159421,159421 +159422,159422 +159423,159423 +189066,159423 +159424,159424 +159425,159425 +159426,159426 +159427,159427 +159428,159428 +159429,159429 +161776,159429 +159430,159430 +118432,159430 +159431,159431 +159432,159432 +159433,159433 +159434,159434 +159435,159435 +159436,159436 +210087,159436 +159437,159437 +159438,159438 +159439,159439 +159440,159440 +159441,159441 +159442,159442 +159443,159443 +159444,159444 +159445,159445 +159446,159446 +159447,159447 +159448,159448 +184155,159448 +159449,159449 +159450,159450 +159451,159451 +14106,159451 +159452,159452 +159453,159453 +159454,159454 +159455,159455 +60041,159455 +159456,159456 +159457,159457 +159458,159458 +151457,159458 +134217,159458 +126630,159458 +159459,159459 +159460,159460 +159461,159461 +51001,159461 +42897,159461 +159462,159462 +216543,159462 +159463,159463 +84176,159463 +159464,159464 +159465,159465 +159466,159466 +159467,159467 +159468,159468 +159469,159469 +159470,159470 +159471,159471 +159472,159472 +159473,159473 +159474,159474 +159475,159475 +159476,159476 +159477,159477 +15404,159477 +159478,159478 +159479,159479 +159480,159480 +159481,159481 +159482,159482 +159483,159483 +159484,159484 +159485,159485 +245415,159485 +29262,159485 +159486,159486 +159487,159487 +159488,159488 +205020,159488 +159489,159489 +159490,159490 +159491,159491 +159492,159492 +159493,159493 +159494,159494 +159495,159495 +159496,159496 +159497,159497 +159498,159498 +159499,159499 +159500,159500 +159501,159501 +155898,159501 +159502,159502 +159503,159503 +159504,159504 +159505,159505 +159506,159506 +159507,159507 +159508,159508 +142591,159508 +159509,159509 +159510,159510 +159511,159511 +122847,159511 +159512,159512 +159513,159513 +159514,159514 +159515,159515 +159516,159516 +159517,159517 +159518,159518 +146538,159518 +159519,159519 +159520,159520 +159521,159521 +100578,159521 +159522,159522 +159523,159523 +159524,159524 +159525,159525 +159526,159526 +159527,159527 +159528,159528 +159529,159529 +159590,159590 +199708,159590 +159591,159591 +159592,159592 +159593,159593 +159907,159907 +37877,159907 +159908,159908 +159909,159909 +159910,159910 +160046,160046 +180745,160046 +160047,160047 +160048,160048 +160049,160049 +160050,160050 +160051,160051 +150702,160051 +160052,160052 +160053,160053 +160054,160054 +160055,160055 +160056,160056 +160057,160057 +160058,160058 +160059,160059 +160060,160060 +160061,160061 +160062,160062 +160063,160063 +160064,160064 +160065,160065 +160066,160066 +160067,160067 +160068,160068 +160069,160069 +160070,160070 +160071,160071 +160072,160072 +160073,160073 +160074,160074 +160075,160075 +160076,160076 +160077,160077 +160078,160078 +160079,160079 +160080,160080 +160158,160080 +160081,160081 +160082,160082 +160083,160083 +160151,160083 +160084,160084 +30560,160084 +160085,160085 +160086,160086 +160154,160086 +160087,160087 +160088,160088 +160089,160089 +160090,160090 +160091,160091 +160092,160092 +160093,160093 +160094,160094 +162154,160094 +160095,160095 +160096,160096 +160097,160097 +160098,160098 +160099,160099 +160100,160100 +160101,160101 +160102,160102 +160103,160103 +160104,160104 +160105,160105 +160106,160106 +160107,160107 +160108,160108 +160109,160109 +160110,160110 +160111,160111 +160112,160112 +160113,160113 +160114,160114 +160115,160115 +208054,160115 +160116,160116 +160117,160117 +160118,160118 +160119,160119 +160120,160120 +79773,160120 +211332,160120 +160121,160121 +160122,160122 +160123,160123 +160124,160124 +160125,160125 +160126,160126 +160127,160127 +160128,160128 +160129,160129 +160130,160130 +160131,160131 +160132,160132 +160133,160133 +160134,160134 +160135,160135 +160136,160136 +160137,160137 +160138,160138 +160139,160139 +160140,160140 +160141,160141 +160142,160142 +160143,160143 +160144,160144 +225079,160144 +160145,160145 +160146,160146 +160147,160147 +160148,160148 +208195,160148 +160149,160149 +160150,160150 +160151,160151 +160083,160151 +160152,160152 +160153,160153 +160154,160154 +160086,160154 +160155,160155 +160156,160156 +160157,160157 +160158,160158 +160080,160158 +160159,160159 +160160,160160 +160161,160161 +160162,160162 +141900,160162 +160163,160163 +49135,160163 +40943,160163 +160164,160164 +160165,160165 +184258,160165 +160166,160166 +160167,160167 +160168,160168 +160169,160169 +160170,160170 +160171,160171 +160172,160172 +160173,160173 +184806,160173 +160174,160174 +160175,160175 +160352,160352 +160353,160353 +160354,160354 +160355,160355 +160356,160356 +148383,160356 +160357,160357 +160358,160358 +160359,160359 +160360,160360 +160361,160361 +230564,160361 +160362,160362 +160363,160363 +160364,160364 +160365,160365 +160366,160366 +160367,160367 +160368,160368 +160369,160369 +160370,160370 +160371,160371 +160372,160372 +160373,160373 +160374,160374 +160375,160375 +160376,160376 +160377,160377 +160378,160378 +160379,160379 +160380,160380 +160381,160381 +160382,160382 +160383,160383 +160384,160384 +160385,160385 +160386,160386 +160387,160387 +160388,160388 +160389,160389 +160390,160390 +160391,160391 +160392,160392 +160393,160393 +160394,160394 +160395,160395 +160396,160396 +160397,160397 +160398,160398 +160399,160399 +160400,160400 +160401,160401 +160402,160402 +160403,160403 +160404,160404 +160405,160405 +62596,160405 +160406,160406 +160407,160407 +160408,160408 +160409,160409 +209431,160409 +160410,160410 +160411,160411 +90659,160411 +160412,160412 +160413,160413 +160414,160414 +160415,160415 +160416,160416 +20655,160416 +20483,160416 +160417,160417 +160418,160418 +160419,160419 +160420,160420 +160421,160421 +160422,160422 +160423,160423 +65605,160423 +160424,160424 +160425,160425 +160426,160426 +160427,160427 +160428,160428 +160429,160429 +160430,160430 +145346,160430 +208767,160430 +138896,160430 +160431,160431 +160432,160432 +160433,160433 +160434,160434 +160435,160435 +160436,160436 +160437,160437 +160438,160438 +160439,160439 +160440,160440 +160441,160441 +101039,160441 +160442,160442 +248158,160442 +160443,160443 +160444,160444 +160445,160445 +160446,160446 +160447,160447 +160448,160448 +160449,160449 +160450,160450 +160451,160451 +160452,160452 +160453,160453 +160454,160454 +160455,160455 +34251,160455 +160456,160456 +160457,160457 +164772,160457 +160458,160458 +2078,160458 +160459,160459 +160460,160460 +160461,160461 +160462,160462 +160463,160463 +160464,160464 +160465,160465 +160466,160466 +160467,160467 +160468,160468 +160469,160469 +160470,160470 +160471,160471 +160472,160472 +160473,160473 +160474,160474 +160475,160475 +160551,160551 +160552,160552 +160553,160553 +160554,160554 +145742,160554 +160555,160555 +160556,160556 +160557,160557 +160558,160558 +160559,160559 +160560,160560 +160561,160561 +160562,160562 +160563,160563 +160564,160564 +160565,160565 +160572,160565 +160566,160566 +7753,160566 +160567,160567 +160568,160568 +160569,160569 +160570,160570 +160571,160571 +160572,160572 +160565,160572 +160573,160573 +160574,160574 +160575,160575 +160576,160576 +160620,160620 +160621,160621 +160622,160622 +160623,160623 +160624,160624 +7737,160624 +160625,160625 +160626,160626 +160627,160627 +160628,160628 +160629,160629 +148125,160629 +160630,160630 +160631,160631 +142593,160631 +160632,160632 +160633,160633 +160634,160634 +160635,160635 +160636,160636 +160637,160637 +160638,160638 +160639,160639 +162800,160639 +160640,160640 +160641,160641 +160642,160642 +160643,160643 +160644,160644 +160645,160645 +160646,160646 +160647,160647 +160648,160648 +90958,160648 +28064,160648 +160649,160649 +160650,160650 +160651,160651 +160652,160652 +160653,160653 +146869,160653 +160654,160654 +160655,160655 +160656,160656 +87121,160656 +160657,160657 +160658,160658 +160659,160659 +160660,160660 +6574,160660 +160661,160661 +160662,160662 +160663,160663 +160664,160664 +160665,160665 +88134,160665 +160666,160666 +160667,160667 +160668,160668 +160669,160669 +160670,160670 +160671,160671 +160672,160672 +160673,160673 +160674,160674 +160675,160675 +160676,160676 +3713,160676 +160677,160677 +160678,160678 +38519,160678 +160679,160679 +160680,160680 +160681,160681 +160682,160682 +160683,160683 +160684,160684 +160685,160685 +160686,160686 +160687,160687 +160688,160688 +189884,160688 +160689,160689 +160690,160690 +160691,160691 +160692,160692 +160693,160693 +160694,160694 +199583,160694 +160695,160695 +32410,160695 +160696,160696 +160697,160697 +160698,160698 +80046,160698 +160699,160699 +160700,160700 +225219,160700 +160701,160701 +160702,160702 +160703,160703 +160704,160704 +160705,160705 +160706,160706 +160707,160707 +160708,160708 +160709,160709 +160710,160710 +160711,160711 +160712,160712 +62005,160712 +160713,160713 +160714,160714 +160715,160715 +160716,160716 +160717,160717 +160718,160718 +160719,160719 +160720,160720 +160721,160721 +160722,160722 +160723,160723 +160724,160724 +110088,160724 +160725,160725 +160726,160726 +216604,160726 +160727,160727 +160728,160728 +160750,160728 +160729,160729 +160730,160730 +160731,160731 +160732,160732 +160733,160733 +160734,160734 +160735,160735 +160736,160736 +160737,160737 +160738,160738 +160739,160739 +160740,160740 +156103,160740 +160741,160741 +160742,160742 +160792,160742 +160743,160743 +160744,160744 +158752,160744 +160745,160745 +160746,160746 +160796,160746 +160747,160747 +160775,160747 +160748,160748 +160749,160749 +160750,160750 +160728,160750 +160751,160751 +160752,160752 +160753,160753 +160754,160754 +108716,160754 +160755,160755 +160756,160756 +160757,160757 +160758,160758 +160759,160759 +160760,160760 +160761,160761 +160762,160762 +160763,160763 +160764,160764 +160765,160765 +160766,160766 +160767,160767 +160768,160768 +160769,160769 +160770,160770 +160771,160771 +160772,160772 +160773,160773 +158753,160773 +160774,160774 +160775,160775 +160747,160775 +160776,160776 +187055,160776 +160777,160777 +160778,160778 +160779,160779 +160780,160780 +160781,160781 +160782,160782 +142441,160782 +160783,160783 +160784,160784 +160785,160785 +160786,160786 +160787,160787 +160788,160788 +31650,160788 +160789,160789 +160790,160790 +160791,160791 +160792,160792 +160742,160792 +160793,160793 +160794,160794 +160795,160795 +160796,160796 +160746,160796 +160797,160797 +160798,160798 +160799,160799 +160800,160800 +160801,160801 +160802,160802 +160803,160803 +160804,160804 +160805,160805 +160806,160806 +160807,160807 +160808,160808 +160809,160809 +160810,160810 +160811,160811 +160812,160812 +160813,160813 +160814,160814 +160815,160815 +160816,160816 +160817,160817 +86561,160817 +160818,160818 +160819,160819 +160820,160820 +160821,160821 +160822,160822 +160823,160823 +160824,160824 +160825,160825 +160826,160826 +160827,160827 +160828,160828 +160829,160829 +160830,160830 +160831,160831 +160832,160832 +160833,160833 +160834,160834 +36639,160834 +160835,160835 +160836,160836 +160837,160837 +160838,160838 +160839,160839 +160840,160840 +139748,160840 +160841,160841 +160842,160842 +160843,160843 +160844,160844 +160845,160845 +160846,160846 +160847,160847 +160848,160848 +160849,160849 +160850,160850 +160851,160851 +160852,160852 +160853,160853 +160854,160854 +160855,160855 +204883,160855 +160856,160856 +160857,160857 +198711,160858 +160858,160858 +160859,160859 +160860,160860 +160861,160861 +160862,160862 +160863,160863 +160864,160864 +160865,160865 +160866,160866 +160867,160867 +160868,160868 +160869,160869 +160870,160870 +160871,160871 +160872,160872 +160873,160873 +160874,160874 +160875,160875 +160876,160876 +160877,160877 +160878,160878 +160879,160879 +160880,160880 +160881,160881 +160882,160882 +160883,160883 +160884,160884 +203526,160884 +160885,160885 +160886,160886 +160887,160887 +160888,160888 +123362,160888 +160889,160889 +160890,160890 +160891,160891 +160892,160892 +160893,160893 +160894,160894 +160895,160895 +160896,160896 +160897,160897 +160898,160898 +160899,160899 +160900,160900 +160901,160901 +160902,160902 +160903,160903 +160904,160904 +160905,160905 +160906,160906 +160907,160907 +160908,160908 +160909,160909 +160910,160910 +160911,160911 +160912,160912 +160913,160913 +160914,160914 +160915,160915 +160916,160916 +160917,160917 +160918,160918 +160919,160919 +160920,160920 +160921,160921 +160922,160922 +160923,160923 +160924,160924 +160925,160925 +160926,160926 +160927,160927 +160928,160928 +160929,160929 +160930,160930 +160931,160931 +160968,160968 +160969,160969 +160970,160970 +160971,160971 +2909,160971 +160972,160972 +160973,160973 +160974,160974 +206282,160974 +160975,160975 +160976,160976 +160977,160977 +241642,160977 +160978,160978 +160979,160979 +111480,160979 +160980,160980 +160981,160981 +160982,160982 +160983,160983 +160984,160984 +160985,160985 +160986,160986 +160987,160987 +160988,160988 +160989,160989 +160990,160990 +160991,160991 +160992,160992 +160993,160993 +160994,160994 +160995,160995 +160996,160996 +160997,160997 +160998,160998 +160999,160999 +161000,161000 +161001,161001 +161002,161002 +161003,161003 +161004,161004 +161005,161005 +161006,161006 +161007,161007 +161008,161008 +161009,161009 +161010,161010 +161011,161011 +161012,161012 +161013,161013 +161014,161014 +161015,161015 +200573,161015 +161016,161016 +161017,161017 +161018,161018 +161019,161019 +192321,161019 +161020,161020 +161021,161021 +57038,161021 +161022,161022 +161023,161023 +161024,161024 +161025,161025 +190178,161025 +161026,161026 +161027,161027 +161028,161028 +161029,161029 +161030,161030 +161031,161031 +161032,161032 +161033,161033 +161034,161034 +161035,161035 +161036,161036 +161037,161037 +161038,161038 +24537,161038 +161039,161039 +161040,161040 +161041,161041 +161042,161042 +161043,161043 +161044,161044 +161045,161045 +161046,161046 +161047,161047 +161048,161048 +161049,161049 +161050,161050 +161051,161051 +161052,161052 +161053,161053 +56636,161053 +48532,161053 +161054,161054 +149278,161054 +161055,161055 +161056,161056 +161057,161057 +161058,161058 +161059,161059 +161060,161060 +161061,161061 +161062,161062 +161063,161063 +161064,161064 +161065,161065 +161066,161066 +161067,161067 +161068,161068 +161069,161069 +161070,161070 +161071,161071 +161072,161072 +161073,161073 +161074,161074 +232058,161074 +161075,161075 +161076,161076 +161077,161077 +161078,161078 +161079,161079 +161080,161080 +161081,161081 +161082,161082 +161083,161083 +5141,161083 +161084,161084 +161085,161085 +161086,161086 +161087,161087 +161088,161088 +161089,161089 +5800,161089 +161090,161090 +161091,161091 +161092,161092 +161093,161093 +161094,161094 +161095,161095 +83915,161095 +161096,161096 +161097,161097 +161098,161098 +161099,161099 +161100,161100 +161101,161101 +161102,161102 +161103,161103 +149759,161103 +161104,161104 +161105,161105 +161106,161106 +161107,161107 +161108,161108 +161109,161109 +161110,161110 +161111,161111 +161112,161112 +179357,161112 +161113,161113 +161114,161114 +161115,161115 +161116,161116 +161117,161117 +161118,161118 +161119,161119 +161120,161120 +161121,161121 +161122,161122 +161123,161123 +161124,161124 +161125,161125 +161126,161126 +161127,161127 +102062,161127 +161128,161128 +161129,161129 +161130,161130 +161131,161131 +161132,161132 +161133,161133 +161134,161134 +161135,161135 +161136,161136 +161137,161137 +161138,161138 +161139,161139 +161140,161140 +220229,161140 +161141,161141 +140258,161141 +161142,161142 +161143,161143 +161144,161144 +161145,161145 +161146,161146 +161147,161147 +161148,161148 +161185,161185 +161186,161186 +161187,161187 +73059,161187 +161188,161188 +161189,161189 +161190,161190 +161191,161191 +161192,161192 +161193,161193 +161194,161194 +161195,161195 +161196,161196 +161197,161197 +161198,161198 +121417,161198 +161199,161199 +147317,161199 +161200,161200 +190960,161200 +161201,161201 +161202,161202 +161203,161203 +161204,161204 +161205,161205 +161206,161206 +161207,161207 +161208,161208 +161209,161209 +133203,161209 +125616,161209 +161210,161210 +161211,161211 +161212,161212 +161213,161213 +161214,161214 +161215,161215 +161216,161216 +161217,161217 +161218,161218 +161219,161219 +161220,161220 +161221,161221 +161222,161222 +161223,161223 +161224,161224 +161225,161225 +161226,161226 +161227,161227 +161228,161228 +161229,161229 +161230,161230 +161231,161231 +161232,161232 +161233,161233 +161234,161234 +161235,161235 +161236,161236 +161237,161237 +117009,161237 +161238,161238 +82578,161238 +161239,161239 +161240,161240 +161241,161241 +161242,161242 +161243,161243 +161244,161244 +161245,161245 +161246,161246 +161247,161247 +161248,161248 +161249,161249 +161250,161250 +161251,161251 +161252,161252 +161253,161253 +161254,161254 +161255,161255 +161256,161256 +161257,161257 +161258,161258 +161259,161259 +161260,161260 +161261,161261 +161262,161262 +161263,161263 +161264,161264 +161265,161265 +161266,161266 +161267,161267 +161268,161268 +161269,161269 +161270,161270 +161271,161271 +161272,161272 +161273,161273 +249068,161273 +161274,161274 +67774,161274 +161275,161275 +161276,161276 +51734,161276 +43630,161276 +161277,161277 +161278,161278 +161279,161279 +161280,161280 +161281,161281 +161282,161282 +161283,161283 +161284,161284 +161285,161285 +161286,161286 +161287,161287 +161288,161288 +161289,161289 +181322,161289 +161290,161290 +161291,161291 +10572,161291 +161356,161356 +161357,161357 +161358,161358 +161359,161359 +161360,161360 +161361,161361 +161362,161362 +161363,161363 +161364,161364 +161365,161365 +161366,161366 +161367,161367 +161368,161368 +161369,161369 +161370,161370 +151595,161370 +161371,161371 +161372,161372 +161373,161373 +161374,161374 +161375,161375 +161376,161376 +161377,161377 +161378,161378 +161379,161379 +161380,161380 +161381,161381 +161382,161382 +161383,161383 +161384,161384 +161385,161385 +161386,161386 +161387,161387 +161388,161388 +161389,161389 +161390,161390 +161391,161391 +161392,161392 +220186,161392 +161393,161393 +161394,161394 +161395,161395 +199559,161395 +161396,161396 +3939,161396 +161397,161397 +161398,161398 +161399,161399 +161400,161400 +161401,161401 +161402,161402 +161512,161512 +161513,161513 +161514,161514 +161515,161515 +161516,161516 +161517,161517 +161518,161518 +161519,161519 +161520,161520 +161521,161521 +161522,161522 +161523,161523 +161524,161524 +161525,161525 +161526,161526 +161527,161527 +161528,161528 +161529,161529 +161530,161530 +161531,161531 +161532,161532 +161533,161533 +161534,161534 +57582,161534 +161535,161535 +161536,161536 +161537,161537 +161538,161538 +161539,161539 +161540,161540 +161541,161541 +161542,161542 +161543,161543 +161544,161544 +196722,161545 +161545,161545 +161546,161546 +194402,161546 +161547,161547 +85847,161547 +224894,161547 +161548,161548 +161549,161549 +161550,161550 +161551,161551 +161552,161552 +161553,161553 +161554,161554 +161555,161555 +161556,161556 +161557,161557 +161558,161558 +161559,161559 +161560,161560 +161561,161561 +161562,161562 +161563,161563 +161564,161564 +161565,161565 +161566,161566 +161567,161567 +161568,161568 +161569,161569 +161570,161570 +161571,161571 +161572,161572 +161573,161573 +161574,161574 +161575,161575 +249611,161575 +161576,161576 +161577,161577 +161578,161578 +161579,161579 +161580,161580 +107478,161580 +161581,161581 +161582,161582 +161583,161583 +161584,161584 +161585,161585 +161586,161586 +161587,161587 +161588,161588 +161589,161589 +161590,161590 +161591,161591 +161592,161592 +90419,161592 +161593,161593 +161594,161594 +161595,161595 +161596,161596 +161597,161597 +161598,161598 +161599,161599 +161600,161600 +161601,161601 +161602,161602 +161603,161603 +161604,161604 +161605,161605 +161606,161606 +161607,161607 +161608,161608 +161609,161609 +161610,161610 +161611,161611 +161612,161612 +161613,161613 +161614,161614 +161615,161615 +161616,161616 +161617,161617 +161618,161618 +161619,161619 +161620,161620 +161621,161621 +161622,161622 +161623,161623 +161624,161624 +161625,161625 +161626,161626 +161627,161627 +74967,161627 +161628,161628 +161629,161629 +161630,161630 +161631,161631 +161632,161632 +161633,161633 +161634,161634 +161635,161635 +161636,161636 +161637,161637 +146920,161637 +161638,161638 +161639,161639 +161640,161640 +161641,161641 +64116,161641 +161642,161642 +161643,161643 +161644,161644 +161645,161645 +161646,161646 +161647,161647 +161648,161648 +161732,161732 +161733,161733 +161734,161734 +161735,161735 +161736,161736 +161737,161737 +161738,161738 +161739,161739 +161740,161740 +161741,161741 +161742,161742 +161743,161743 +2872,161743 +161744,161744 +161745,161745 +161746,161746 +161747,161747 +161748,161748 +161749,161749 +161750,161750 +134170,161750 +126583,161750 +161751,161751 +161752,161752 +161753,161753 +161754,161754 +161755,161755 +161756,161756 +245454,161757 +161757,161757 +161758,161758 +161759,161759 +148400,161759 +161760,161760 +6182,161760 +161761,161761 +161762,161762 +161763,161763 +161764,161764 +161765,161765 +161766,161766 +161767,161767 +217185,161767 +161768,161768 +161769,161769 +74215,161769 +161770,161770 +161771,161771 +161772,161772 +161773,161773 +2855,161773 +161774,161774 +161775,161775 +161776,161776 +159429,161776 +161777,161777 +161778,161778 +161779,161779 +161780,161780 +161781,161781 +161782,161782 +161783,161783 +161784,161784 +161785,161785 +150667,161785 +161786,161786 +161787,161787 +161788,161788 +161789,161789 +161790,161790 +161791,161791 +161792,161792 +161793,161793 +214245,161794 +161794,161794 +161795,161795 +101945,161795 +161796,161796 +161797,161797 +161798,161798 +161799,161799 +161800,161800 +161801,161801 +161802,161802 +161803,161803 +4946,161803 +161804,161804 +201030,161805 +161805,161805 +201763,161805 +161806,161806 +209635,161806 +161807,161807 +161808,161808 +161809,161809 +161810,161810 +161811,161811 +161812,161812 +161813,161813 +161814,161814 +161815,161815 +194316,161815 +161816,161816 +161817,161817 +161818,161818 +90637,161818 +161819,161819 +161820,161820 +234753,161820 +161821,161821 +161822,161822 +161823,161823 +161824,161824 +161825,161825 +161826,161826 +161827,161827 +161828,161828 +161829,161829 +161830,161830 +161831,161831 +161832,161832 +161833,161833 +161834,161834 +161835,161835 +161836,161836 +161837,161837 +21404,161837 +161838,161838 +161839,161839 +161840,161840 +161841,161841 +161842,161842 +161843,161843 +161844,161844 +161845,161845 +161846,161846 +161847,161847 +161848,161848 +161849,161849 +161850,161850 +161851,161851 +161852,161852 +161853,161853 +224937,161853 +161854,161854 +161855,161855 +161856,161856 +161857,161857 +161858,161858 +161859,161859 +161860,161860 +161861,161861 +161862,161862 +161863,161863 +161864,161864 +161865,161865 +161866,161866 +142663,161866 +161867,161867 +161868,161868 +161869,161869 +161870,161870 +161871,161871 +161872,161872 +240960,161872 +161873,161873 +161874,161874 +161875,161875 +161876,161876 +161877,161877 +161878,161878 +161879,161879 +161880,161880 +161881,161881 +149908,161881 +161882,161882 +161883,161883 +161884,161884 +161885,161885 +36236,161885 +161886,161886 +161887,161887 +20131,161887 +20503,161887 +161888,161888 +161889,161889 +161890,161890 +161891,161891 +161892,161892 +161893,161893 +161894,161894 +161895,161895 +161896,161896 +161897,161897 +149894,161897 +161898,161898 +161899,161899 +161900,161900 +161901,161901 +161902,161902 +161903,161903 +161904,161904 +161905,161905 +161906,161906 +161907,161907 +161908,161908 +161909,161909 +161910,161910 +161911,161911 +161912,161912 +161913,161913 +161914,161914 +161915,161915 +161916,161916 +161917,161917 +161918,161918 +161919,161919 +161920,161920 +161921,161921 +161922,161922 +161923,161923 +161924,161924 +161925,161925 +161926,161926 +161927,161927 +161928,161928 +161929,161929 +161930,161930 +147337,161930 +161931,161931 +161932,161932 +161933,161933 +161934,161934 +161935,161935 +161936,161936 +161937,161937 +161938,161938 +121309,161938 +161939,161939 +161940,161940 +161941,161941 +161942,161942 +161943,161943 +161944,161944 +161945,161945 +161946,161946 +247327,161947 +161947,161947 +161948,161948 +161949,161949 +161950,161950 +161951,161951 +161952,161952 +161953,161953 +161954,161954 +161955,161955 +161956,161956 +161957,161957 +102669,161957 +161958,161958 +161959,161959 +161960,161960 +161961,161961 +161962,161962 +161963,161963 +161964,161964 +161965,161965 +161966,161966 +161967,161967 +161968,161968 +161969,161969 +161970,161970 +141637,161970 +161971,161971 +161972,161972 +161973,161973 +161974,161974 +161975,161975 +161976,161976 +161977,161977 +161978,161978 +196284,161978 +161979,161979 +161980,161980 +161981,161981 +161982,161982 +161983,161983 +161984,161984 +162053,162053 +162054,162054 +162147,162147 +162148,162148 +162149,162149 +162150,162150 +162151,162151 +162152,162152 +162153,162153 +162154,162154 +160094,162154 +162155,162155 +162156,162156 +162157,162157 +162158,162158 +162159,162159 +162160,162160 +162161,162161 +162162,162162 +162163,162163 +162164,162164 +162165,162165 +162166,162166 +162167,162167 +211120,162167 +162168,162168 +162169,162169 +162170,162170 +162171,162171 +162172,162172 +162173,162173 +162174,162174 +162175,162175 +193276,162175 +162176,162176 +162177,162177 +162178,162178 +162179,162179 +162180,162180 +162181,162181 +162182,162182 +162183,162183 +162184,162184 +64222,162184 +4811,162184 +162185,162185 +162186,162186 +223507,162186 +162187,162187 +162188,162188 +79355,162188 +162189,162189 +162228,162228 +162229,162229 +109907,162229 +162230,162230 +162231,162231 +162232,162232 +162233,162233 +162234,162234 +162235,162235 +162236,162236 +162237,162237 +162238,162238 +205568,162238 +162239,162239 +162240,162240 +19185,162240 +162241,162241 +162242,162242 +162243,162243 +162244,162244 +162245,162245 +162246,162246 +162247,162247 +162248,162248 +162249,162249 +162250,162250 +162251,162251 +162252,162252 +162253,162253 +162254,162254 +162255,162255 +162256,162256 +162257,162257 +162258,162258 +79885,162258 +162259,162259 +162260,162260 +162261,162261 +162262,162262 +162263,162263 +162264,162264 +162265,162265 +162266,162266 +162267,162267 +162268,162268 +162269,162269 +162270,162270 +162271,162271 +162272,162272 +162273,162273 +162311,162311 +162312,162312 +214048,162312 +162313,162313 +162314,162314 +162315,162315 +162316,162316 +162317,162317 +147095,162317 +162318,162318 +162319,162319 +149653,162319 +162320,162320 +162321,162321 +162322,162322 +162323,162323 +162324,162324 +162325,162325 +141704,162325 +162326,162326 +162327,162327 +162328,162328 +106692,162328 +162329,162329 +162330,162330 +162331,162331 +162332,162332 +203675,162332 +162333,162333 +162334,162334 +162335,162335 +162336,162336 +162337,162337 +162338,162338 +162339,162339 +162340,162340 +162341,162341 +162342,162342 +162343,162343 +162344,162344 +162345,162345 +162346,162346 +162347,162347 +141208,162347 +102020,162347 +141422,162347 +162348,162348 +162349,162349 +162350,162350 +162351,162351 +162352,162352 +162353,162353 +114837,162353 +162354,162354 +162355,162355 +162356,162356 +162357,162357 +162358,162358 +162359,162359 +162360,162360 +162361,162361 +162362,162362 +162363,162363 +162364,162364 +225580,162364 +162365,162365 +162366,162366 +162367,162367 +162368,162368 +162369,162369 +162370,162370 +162371,162371 +162372,162372 +162373,162373 +162374,162374 +162375,162375 +162376,162376 +178939,162376 +162377,162377 +162378,162378 +162379,162379 +162380,162380 +60946,162380 +162381,162381 +97480,162381 +162382,162382 +162383,162383 +162384,162384 +162385,162385 +162386,162386 +162387,162387 +139733,162387 +162388,162388 +162389,162389 +162390,162390 +162391,162391 +162392,162392 +162393,162393 +162394,162394 +162395,162395 +162396,162396 +162397,162397 +162398,162398 +162399,162399 +162400,162400 +162401,162401 +162402,162402 +162403,162403 +162404,162404 +162405,162405 +162406,162406 +162407,162407 +162408,162408 +233519,162408 +162409,162409 +162410,162410 +162411,162411 +162412,162412 +162413,162413 +211091,162413 +162414,162414 +162415,162415 +162416,162416 +162417,162417 +162418,162418 +106263,162418 +162419,162419 +162420,162420 +162421,162421 +162422,162422 +162423,162423 +162424,162424 +220844,162424 +162425,162425 +162426,162426 +162427,162427 +162428,162428 +12161,162428 +162429,162429 +162430,162430 +162431,162431 +162432,162432 +179401,162432 +75998,162432 +162433,162433 +176723,162433 +170759,162433 +162434,162434 +162435,162435 +162436,162436 +162437,162437 +143696,162437 +162438,162438 +217250,162438 +162439,162439 +162440,162440 +162441,162441 +162442,162442 +162443,162443 +162444,162444 +162445,162445 +154787,162445 +162446,162446 +162447,162447 +162448,162448 +162449,162449 +162450,162450 +162451,162451 +162452,162452 +162453,162453 +162454,162454 +162455,162455 +162456,162456 +162457,162457 +162458,162458 +162459,162459 +162460,162460 +162461,162461 +162462,162462 +162463,162463 +162464,162464 +162465,162465 +162466,162466 +162467,162467 +162468,162468 +162469,162469 +162470,162470 +162471,162471 +162472,162472 +162473,162473 +162474,162474 +162475,162475 +162476,162476 +162477,162477 +162478,162478 +162479,162479 +162480,162480 +162481,162481 +162482,162482 +162483,162483 +180423,162483 +162484,162484 +162485,162485 +162486,162486 +162487,162487 +162488,162488 +162489,162489 +162490,162490 +162491,162491 +162492,162492 +162493,162493 +162494,162494 +162495,162495 +162496,162496 +162497,162497 +162498,162498 +162499,162499 +162500,162500 +162501,162501 +162502,162502 +162503,162503 +162504,162504 +162505,162505 +162506,162506 +162507,162507 +162508,162508 +162509,162509 +162510,162510 +162511,162511 +162512,162512 +162513,162513 +162514,162514 +162515,162515 +162516,162516 +104130,162516 +162517,162517 +162518,162518 +5014,162518 +162519,162519 +191559,162519 +6577,162519 +162520,162520 +187983,162520 +162521,162521 +216304,162521 +162522,162522 +162523,162523 +162524,162524 +191614,162524 +162525,162525 +162526,162526 +162527,162527 +162528,162528 +162529,162529 +107660,162529 +162530,162530 +162531,162531 +162532,162532 +162533,162533 +162534,162534 +162535,162535 +162536,162536 +5745,162536 +55029,162536 +46925,162536 +193394,162536 +162537,162537 +62795,162537 +162538,162538 +162539,162539 +162540,162540 +162541,162541 +162542,162542 +162543,162543 +162544,162544 +162545,162545 +162546,162546 +162547,162547 +162548,162548 +162549,162549 +162550,162550 +162551,162551 +162552,162552 +162553,162553 +162554,162554 +162555,162555 +162556,162556 +162557,162557 +162558,162558 +162559,162559 +162560,162560 +162561,162561 +162562,162562 +162563,162563 +162564,162564 +162565,162565 +241644,162565 +162566,162566 +162567,162567 +162568,162568 +162569,162569 +162570,162570 +162571,162571 +162572,162572 +162573,162573 +162574,162574 +162575,162575 +10872,162575 +162576,162576 +162577,162577 +162578,162578 +162579,162579 +162580,162580 +28875,162580 +162581,162581 +162582,162582 +162583,162583 +162584,162584 +162585,162585 +178109,162585 +172145,162585 +162586,162586 +162587,162587 +162588,162588 +162589,162589 +12958,162589 +162590,162590 +162591,162591 +162592,162592 +162593,162593 +151365,162593 +162594,162594 +162595,162595 +162596,162596 +162597,162597 +162598,162598 +4942,162598 +162599,162599 +162600,162600 +162601,162601 +162602,162602 +162603,162603 +162604,162604 +162605,162605 +162606,162606 +162607,162607 +162608,162608 +162609,162609 +162610,162610 +162611,162611 +190614,162611 +162612,162612 +162613,162613 +162614,162614 +162615,162615 +162616,162616 +162617,162617 +10527,162617 +162618,162618 +162619,162619 +102271,162619 +162620,162620 +220944,162620 +162621,162621 +162622,162622 +162623,162623 +162624,162624 +162625,162625 +162626,162626 +162627,162627 +162628,162628 +162629,162629 +162630,162630 +162631,162631 +162632,162632 +162633,162633 +162634,162634 +162635,162635 +162636,162636 +162637,162637 +162638,162638 +162639,162639 +162640,162640 +162641,162641 +162642,162642 +162643,162643 +162644,162644 +56648,162644 +48544,162644 +162645,162645 +162646,162646 +162647,162647 +162648,162648 +162649,162649 +162650,162650 +162651,162651 +162652,162652 +162653,162653 +162654,162654 +162655,162655 +162656,162656 +162657,162657 +162658,162658 +162659,162659 +158626,162659 +162660,162660 +162661,162661 +162662,162662 +162663,162663 +162664,162664 +162665,162665 +162666,162666 +162667,162667 +162668,162668 +162669,162669 +162670,162670 +224573,162670 +162671,162671 +208588,162671 +162672,162672 +208589,162672 +162673,162673 +162674,162674 +231281,162674 +162675,162675 +162676,162676 +162677,162677 +162678,162678 +162679,162679 +162680,162680 +162681,162681 +162682,162682 +162683,162683 +162684,162684 +162685,162685 +180365,162685 +162686,162686 +162687,162687 +162688,162688 +162689,162689 +162690,162690 +162691,162691 +162692,162692 +162693,162693 +162694,162694 +162695,162695 +162696,162696 +162697,162697 +162698,162698 +162699,162699 +162700,162700 +59144,162700 +58770,162700 +162701,162701 +162702,162702 +162703,162703 +162704,162704 +162705,162705 +162706,162706 +162707,162707 +162748,162748 +162749,162749 +162750,162750 +162751,162751 +162752,162752 +162753,162753 +162754,162754 +162755,162755 +162756,162756 +162757,162757 +162758,162758 +162759,162759 +162760,162760 +162761,162761 +162762,162762 +216170,162762 +162763,162763 +162764,162764 +162765,162765 +162766,162766 +162767,162767 +162768,162768 +162769,162769 +88500,162769 +162770,162770 +162771,162771 +162772,162772 +162773,162773 +162774,162774 +162775,162775 +162776,162776 +180066,162777 +162777,162777 +162778,162778 +162779,162779 +162780,162780 +162781,162781 +5341,162781 +162782,162782 +162783,162783 +162784,162784 +162785,162785 +162786,162786 +162787,162787 +162788,162788 +162789,162789 +202712,162789 +162790,162790 +162791,162791 +162792,162792 +162793,162793 +162794,162794 +162795,162795 +162796,162796 +162797,162797 +162798,162798 +162799,162799 +162800,162800 +160639,162800 +162801,162801 +162802,162802 +162803,162803 +162804,162804 +162805,162805 +162806,162806 +162807,162807 +162808,162808 +162809,162809 +162810,162810 +54567,162810 +46463,162810 +162811,162811 +162812,162812 +162813,162813 +162814,162814 +162815,162815 +162816,162816 +162817,162817 +162818,162818 +162819,162819 +162820,162820 +162821,162821 +162822,162822 +162823,162823 +162824,162824 +162825,162825 +162826,162826 +162827,162827 +162828,162828 +162829,162829 +162830,162830 +162831,162831 +162832,162832 +162833,162833 +162834,162834 +18959,162834 +162835,162835 +162836,162836 +162837,162837 +162838,162838 +162839,162839 +162840,162840 +162841,162841 +162842,162842 +162843,162843 +178718,162843 +162844,162844 +162845,162845 +162846,162846 +164850,162846 +162847,162847 +162848,162848 +162849,162849 +162850,162850 +162851,162851 +131753,162851 +124166,162851 +162852,162852 +162853,162853 +162854,162854 +162855,162855 +162856,162856 +162857,162857 +162858,162858 +162859,162859 +162860,162860 +162861,162861 +162862,162862 +210090,162862 +162863,162863 +162864,162864 +162865,162865 +58484,162865 +162866,162866 +162867,162867 +162868,162868 +680,162868 +162869,162869 +162870,162870 +158494,162870 +162871,162871 +94082,162871 +82334,162871 +162872,162872 +162873,162873 +162874,162874 +240962,162874 +162875,162875 +92058,162875 +205924,162875 +162876,162876 +162877,162877 +162878,162878 +162879,162879 +107354,162879 +162880,162880 +162881,162881 +162882,162882 +162883,162883 +59167,162883 +58793,162883 +162884,162884 +162885,162885 +162886,162886 +162887,162887 +162888,162888 +162889,162889 +162890,162890 +162891,162891 +162892,162892 +162893,162893 +162937,162937 +162938,162938 +162939,162939 +162940,162940 +162941,162941 +162942,162942 +162943,162943 +162944,162944 +162945,162945 +162946,162946 +162947,162947 +162948,162948 +162949,162949 +162950,162950 +162951,162951 +33667,162951 +162952,162952 +162953,162953 +162954,162954 +162955,162955 +162956,162956 +162957,162957 +162958,162958 +62785,162958 +162959,162959 +162960,162960 +133041,162960 +125454,162960 +162961,162961 +162962,162962 +162963,162963 +162964,162964 +162965,162965 +162966,162966 +162967,162967 +162968,162968 +162969,162969 +162970,162970 +162971,162971 +162972,162972 +162973,162973 +162974,162974 +162975,162975 +162976,162976 +162977,162977 +162978,162978 +162979,162979 +162980,162980 +162981,162981 +162982,162982 +162983,162983 +162984,162984 +162985,162985 +162986,162986 +162987,162987 +162988,162988 +162989,162989 +162990,162990 +162991,162991 +229985,162991 +193546,162991 +162992,162992 +162993,162993 +54716,162993 +46612,162993 +162994,162994 +189120,162995 +162995,162995 +162996,162996 +30514,162996 +162997,162997 +162998,162998 +162999,162999 +163000,163000 +163001,163001 +27197,163001 +163002,163002 +163003,163003 +163004,163004 +163005,163005 +163006,163006 +163007,163007 +163008,163008 +163009,163009 +163010,163010 +163011,163011 +84306,163011 +163012,163012 +163013,163013 +114900,163013 +163014,163014 +163015,163015 +163016,163016 +163017,163017 +242787,163017 +163018,163018 +163019,163019 +163020,163020 +163021,163021 +163022,163022 +163023,163023 +163024,163024 +163025,163025 +163026,163026 +163027,163027 +163028,163028 +163029,163029 +163030,163030 +163031,163031 +163032,163032 +163033,163033 +163034,163034 +163035,163035 +147160,163035 +163036,163036 +163037,163037 +163038,163038 +163039,163039 +163040,163040 +163041,163041 +163042,163042 +163043,163043 +211906,163043 +163044,163044 +163045,163045 +163046,163046 +163047,163047 +163048,163048 +163049,163049 +163050,163050 +163051,163051 +163052,163052 +163053,163053 +220200,163054 +163054,163054 +143280,163054 +188406,163054 +163055,163055 +163056,163056 +163057,163057 +163058,163058 +201697,163059 +163059,163059 +163060,163060 +163061,163061 +163062,163062 +163063,163063 +163064,163064 +64015,163064 +163065,163065 +163066,163066 +163067,163067 +163068,163068 +163069,163069 +163070,163070 +163071,163071 +181317,163071 +163072,163072 +163073,163073 +163074,163074 +80095,163074 +163075,163075 +163076,163076 +225528,163076 +163077,163077 +163078,163078 +163079,163079 +163080,163080 +163081,163081 +188230,163081 +163082,163082 +163083,163083 +163084,163084 +3274,163084 +163085,163085 +163086,163086 +163137,163137 +163138,163138 +163139,163139 +163140,163140 +163141,163141 +163142,163142 +163143,163143 +37326,163143 +163144,163144 +84101,163144 +163145,163145 +163146,163146 +163147,163147 +163148,163148 +163149,163149 +163150,163150 +163331,163331 +246592,163331 +163332,163332 +247948,163332 +163333,163333 +245213,163333 +163334,163334 +247950,163334 +163335,163335 +163336,163336 +163337,163337 +163338,163338 +163339,163339 +163340,163340 +163341,163341 +163342,163342 +163343,163343 +158323,163343 +163344,163344 +163345,163345 +163346,163346 +163347,163347 +163348,163348 +163349,163349 +163350,163350 +163351,163351 +163352,163352 +163353,163353 +163354,163354 +163355,163355 +163356,163356 +163357,163357 +163358,163358 +163359,163359 +151433,163359 +163360,163360 +163361,163361 +163362,163362 +163363,163363 +163364,163364 +163365,163365 +145192,163365 +163366,163366 +163367,163367 +111072,163367 +163368,163368 +163369,163369 +163370,163370 +163371,163371 +163372,163372 +163373,163373 +163374,163374 +163375,163375 +163376,163376 +178704,163376 +163377,163377 +163378,163378 +163379,163379 +163380,163380 +163381,163381 +163382,163382 +163383,163383 +163384,163384 +163385,163385 +163386,163386 +163387,163387 +163388,163388 +163389,163389 +163390,163390 +163391,163391 +163392,163392 +163393,163393 +163394,163394 +163395,163395 +163396,163396 +150589,163396 +163397,163397 +101766,163397 +163398,163398 +189184,163398 +163399,163399 +163400,163400 +163401,163401 +163402,163402 +163403,163403 +9721,163403 +163404,163404 +163405,163405 +163406,163406 +30584,163406 +163407,163407 +163408,163408 +163409,163409 +163410,163410 +163411,163411 +163412,163412 +163413,163413 +163414,163414 +163415,163415 +247450,163416 +163416,163416 +163417,163417 +142138,163417 +163418,163418 +163419,163419 +163420,163420 +163421,163421 +163422,163422 +163423,163423 +163424,163424 +163425,163425 +163426,163426 +163427,163427 +163428,163428 +163429,163429 +163465,163465 +163466,163466 +163467,163467 +210172,163467 +163468,163468 +163469,163469 +163470,163470 +163471,163471 +163472,163472 +163473,163473 +77789,163473 +163474,163474 +163475,163475 +163476,163476 +163477,163477 +163478,163478 +163479,163479 +163480,163480 +163481,163481 +163482,163482 +163483,163483 +163484,163484 +163485,163485 +163486,163486 +163487,163487 +163488,163488 +163489,163489 +163490,163490 +163491,163491 +163492,163492 +163493,163493 +163494,163494 +163495,163495 +163496,163496 +163497,163497 +163498,163498 +163499,163499 +163500,163500 +163501,163501 +163502,163502 +163503,163503 +163504,163504 +163505,163505 +163506,163506 +163507,163507 +163508,163508 +163509,163509 +163510,163510 +163511,163511 +163512,163512 +163513,163513 +141397,163513 +163514,163514 +163515,163515 +163516,163516 +163517,163517 +163518,163518 +163519,163519 +163520,163520 +163521,163521 +58435,163521 +163522,163522 +163523,163523 +163524,163524 +163525,163525 +163526,163526 +163527,163527 +163528,163528 +163529,163529 +163530,163530 +163531,163531 +163532,163532 +163533,163533 +163534,163534 +163535,163535 +112272,163535 +163536,163536 +211250,163536 +163537,163537 +163538,163538 +163539,163539 +220307,163539 +163540,163540 +163541,163541 +163542,163542 +163543,163543 +163544,163544 +163545,163545 +163546,163546 +163547,163547 +163548,163548 +163549,163549 +163550,163550 +163551,163551 +163552,163552 +163553,163553 +163554,163554 +163555,163555 +163556,163556 +163557,163557 +239881,163557 +163558,163558 +163559,163559 +163560,163560 +163561,163561 +163562,163562 +163563,163563 +163564,163564 +140678,163564 +163565,163565 +191813,163565 +163566,163566 +163567,163567 +163568,163568 +163569,163569 +163570,163570 +163571,163571 +163572,163572 +163573,163573 +163574,163574 +163575,163575 +163576,163576 +145774,163576 +219402,163576 +163577,163577 +163680,163680 +163681,163681 +163682,163682 +163683,163683 +163684,163684 +163685,163685 +163686,163686 +163687,163687 +194964,163687 +163688,163688 +238612,163688 +163689,163689 +163690,163690 +163691,163691 +163692,163692 +163693,163693 +163694,163694 +163695,163695 +163696,163696 +163697,163697 +163698,163698 +163699,163699 +22421,163699 +139683,163699 +163700,163700 +163701,163701 +163702,163702 +163703,163703 +190218,163703 +163704,163704 +77078,163704 +163705,163705 +163706,163706 +163707,163707 +163708,163708 +163709,163709 +163710,163710 +163711,163711 +19210,163711 +163712,163712 +163713,163713 +163714,163714 +163715,163715 +163716,163716 +163717,163717 +80649,163717 +163718,163718 +163719,163719 +163720,163720 +163721,163721 +163722,163722 +163723,163723 +163724,163724 +163725,163725 +163763,163763 +163764,163764 +163765,163765 +163766,163766 +163767,163767 +163768,163768 +163769,163769 +163770,163770 +163771,163771 +163772,163772 +163773,163773 +163774,163774 +163775,163775 +163776,163776 +163777,163777 +163778,163778 +163837,163837 +163838,163838 +163839,163839 +163840,163840 +163841,163841 +163842,163842 +163843,163843 +163844,163844 +163845,163845 +163846,163846 +163847,163847 +102479,163847 +163848,163848 +163849,163849 +163964,163964 +163965,163965 +204614,163965 +163966,163966 +163967,163967 +163968,163968 +163969,163969 +163970,163970 +163971,163971 +163972,163972 +163973,163973 +163974,163974 +163975,163975 +216601,163975 +163976,163976 +163977,163977 +163978,163978 +163979,163979 +163980,163980 +163981,163981 +163982,163982 +163983,163983 +163984,163984 +163985,163985 +163986,163986 +163987,163987 +163988,163988 +163989,163989 +163990,163990 +187049,163990 +163991,163991 +163992,163992 +163993,163993 +141544,163993 +163994,163994 +163995,163995 +163996,163996 +163997,163997 +163998,163998 +163999,163999 +164000,164000 +111658,164000 +164001,164001 +164002,164002 +164003,164003 +164004,164004 +164005,164005 +164006,164006 +164007,164007 +164008,164008 +164009,164009 +164010,164010 +164011,164011 +164012,164012 +164013,164013 +164014,164014 +164015,164015 +164016,164016 +164017,164017 +164018,164018 +164019,164019 +164020,164020 +139706,164020 +164021,164021 +164022,164022 +164023,164023 +164024,164024 +164025,164025 +164026,164026 +164027,164027 +164028,164028 +164029,164029 +164030,164030 +164031,164031 +164032,164032 +164033,164033 +164034,164034 +164035,164035 +164036,164036 +164037,164037 +164038,164038 +164039,164039 +164040,164040 +164041,164041 +164042,164042 +164043,164043 +164044,164044 +164045,164045 +164046,164046 +164047,164047 +164048,164048 +164049,164049 +164050,164050 +164051,164051 +164052,164052 +164053,164053 +164054,164054 +164055,164055 +164056,164056 +164057,164057 +164058,164058 +164059,164059 +53467,164059 +45363,164059 +164060,164060 +164061,164061 +164062,164062 +164063,164063 +164064,164064 +164065,164065 +164066,164066 +164067,164067 +164068,164068 +164069,164069 +164070,164070 +164071,164071 +164072,164072 +164073,164073 +164074,164074 +107040,164074 +164075,164075 +116393,164075 +164076,164076 +164077,164077 +164078,164078 +164079,164079 +164080,164080 +164081,164081 +135270,164081 +127683,164081 +164082,164082 +164083,164083 +164084,164084 +164085,164085 +164086,164086 +164087,164087 +142448,164087 +164088,164088 +164089,164089 +149781,164089 +164090,164090 +164091,164091 +164092,164092 +164093,164093 +233508,164093 +164094,164094 +164095,164095 +146502,164095 +164096,164096 +164097,164097 +164098,164098 +164099,164099 +164100,164100 +24655,164100 +164101,164101 +164102,164102 +164103,164103 +164104,164104 +164105,164105 +164106,164106 +164107,164107 +164108,164108 +164109,164109 +120947,164109 +164110,164110 +164111,164111 +164112,164112 +164113,164113 +164114,164114 +164115,164115 +164116,164116 +164117,164117 +236051,164117 +164118,164118 +164119,164119 +164120,164120 +164121,164121 +164122,164122 +164123,164123 +164124,164124 +164125,164125 +164126,164126 +164127,164127 +164128,164128 +164129,164129 +164130,164130 +164131,164131 +164132,164132 +164133,164133 +164134,164134 +164135,164135 +164136,164136 +164137,164137 +164138,164138 +164139,164139 +164140,164140 +164141,164141 +164142,164142 +164143,164143 +164144,164144 +164145,164145 +164146,164146 +164147,164147 +164148,164148 +164149,164149 +164150,164150 +164151,164151 +164152,164152 +164153,164153 +164154,164154 +164155,164155 +164156,164156 +164157,164157 +164158,164158 +164159,164159 +154998,164159 +164160,164160 +164161,164161 +164162,164162 +164163,164163 +80877,164163 +164164,164164 +164165,164165 +164302,164302 +164303,164303 +117444,164303 +164304,164304 +164305,164305 +164306,164306 +164307,164307 +111681,164307 +164308,164308 +164309,164309 +164310,164310 +164311,164311 +164382,164382 +164383,164383 +164384,164384 +164385,164385 +164386,164386 +164387,164387 +229657,164388 +164388,164388 +164389,164389 +164390,164390 +164391,164391 +164392,164392 +164393,164393 +164435,164435 +164436,164436 +164437,164437 +212758,164437 +164438,164438 +164439,164439 +164440,164440 +164441,164441 +164442,164442 +164443,164443 +164444,164444 +164445,164445 +164446,164446 +76581,164446 +164447,164447 +164448,164448 +164449,164449 +164450,164450 +164451,164451 +164452,164452 +932,164452 +164453,164453 +164454,164454 +164455,164455 +164456,164456 +164457,164457 +20650,164457 +19944,164457 +164458,164458 +164459,164459 +7652,164459 +164460,164460 +164461,164461 +164462,164462 +164463,164463 +83813,164463 +164464,164464 +164465,164465 +164466,164466 +245018,164466 +164467,164467 +164468,164468 +164469,164469 +164470,164470 +164471,164471 +164472,164472 +164473,164473 +88222,164473 +164474,164474 +164475,164475 +164476,164476 +164477,164477 +164478,164478 +164479,164479 +164480,164480 +164481,164481 +164482,164482 +164483,164483 +190480,164483 +164484,164484 +164485,164485 +164486,164486 +164487,164487 +164488,164488 +164489,164489 +164490,164490 +164491,164491 +181268,164491 +164492,164492 +164493,164493 +164494,164494 +164495,164495 +164496,164496 +164497,164497 +164498,164498 +164499,164499 +164500,164500 +164501,164501 +164502,164502 +164503,164503 +164504,164504 +164505,164505 +164506,164506 +208554,164506 +164507,164507 +26224,164507 +164508,164508 +164509,164509 +164510,164510 +216161,164510 +164511,164511 +164512,164512 +164513,164513 +164514,164514 +164515,164515 +164516,164516 +164517,164517 +164518,164518 +164519,164519 +164520,164520 +157547,164520 +164521,164521 +4663,164521 +33918,164521 +164522,164522 +164523,164523 +164524,164524 +164525,164525 +164526,164526 +164527,164527 +37019,164527 +164528,164528 +25104,164528 +164529,164529 +164530,164530 +164531,164531 +164532,164532 +164533,164533 +164534,164534 +119762,164534 +164535,164535 +164536,164536 +164537,164537 +164538,164538 +245019,164538 +164539,164539 +164540,164540 +164541,164541 +164542,164542 +23918,164542 +164595,164595 +164596,164596 +164597,164597 +164598,164598 +164599,164599 +164600,164600 +164601,164601 +164602,164602 +164603,164603 +164604,164604 +164605,164605 +164606,164606 +164607,164607 +164608,164608 +164609,164609 +87497,164609 +164610,164610 +164611,164611 +164612,164612 +164613,164613 +164614,164614 +164615,164615 +164616,164616 +145701,164616 +164617,164617 +164618,164618 +164619,164619 +164620,164620 +164621,164621 +164622,164622 +164623,164623 +164624,164624 +164625,164625 +164626,164626 +164627,164627 +164628,164628 +164629,164629 +164630,164630 +164631,164631 +164632,164632 +164633,164633 +164634,164634 +164635,164635 +164636,164636 +164637,164637 +164638,164638 +164639,164639 +164640,164640 +164641,164641 +164642,164642 +164643,164643 +164644,164644 +164645,164645 +164646,164646 +164647,164647 +164648,164648 +164649,164649 +164650,164650 +164651,164651 +164652,164652 +164653,164653 +164654,164654 +164655,164655 +164656,164656 +164657,164657 +164658,164658 +164659,164659 +164660,164660 +164661,164661 +164662,164662 +164663,164663 +211946,164663 +164664,164664 +69966,164664 +153325,164664 +164665,164665 +164666,164666 +164667,164667 +164668,164668 +164669,164669 +164670,164670 +201036,164670 +164671,164671 +164672,164672 +164673,164673 +164674,164674 +164675,164675 +164676,164676 +11993,164676 +164677,164677 +164751,164751 +164752,164752 +164753,164753 +164754,164754 +37294,164754 +164755,164755 +164756,164756 +164757,164757 +164758,164758 +164759,164759 +164760,164760 +164761,164761 +164762,164762 +164763,164763 +164764,164764 +164765,164765 +56794,164765 +164766,164766 +164767,164767 +164768,164768 +108927,164768 +164769,164769 +164770,164770 +142072,164770 +164771,164771 +164772,164772 +197820,164772 +160457,164772 +164773,164773 +164774,164774 +164775,164775 +164776,164776 +164777,164777 +164778,164778 +164779,164779 +164780,164780 +164781,164781 +164782,164782 +164783,164783 +164784,164784 +164785,164785 +164786,164786 +164787,164787 +164788,164788 +164789,164789 +164790,164790 +164791,164791 +164792,164792 +164793,164793 +164794,164794 +164795,164795 +164796,164796 +164797,164797 +164798,164798 +164799,164799 +164800,164800 +164801,164801 +164802,164802 +164803,164803 +164804,164804 +164805,164805 +164806,164806 +179858,164806 +164807,164807 +164808,164808 +164809,164809 +164810,164810 +164811,164811 +164812,164812 +164813,164813 +164814,164814 +164815,164815 +164816,164816 +164817,164817 +164818,164818 +164819,164819 +164820,164820 +164821,164821 +164822,164822 +164823,164823 +164824,164824 +164825,164825 +164826,164826 +164827,164827 +164828,164828 +164829,164829 +164830,164830 +164831,164831 +164832,164832 +164833,164833 +89329,164833 +164834,164834 +164835,164835 +164836,164836 +164837,164837 +164838,164838 +164839,164839 +164840,164840 +164841,164841 +164842,164842 +164843,164843 +164844,164844 +164845,164845 +75841,164845 +164846,164846 +164847,164847 +64739,164847 +164848,164848 +164849,164849 +164850,164850 +104395,164850 +162846,164850 +164851,164851 +164852,164852 +164853,164853 +155893,164853 +164854,164854 +164855,164855 +164856,164856 +164857,164857 +164858,164858 +164859,164859 +156358,164859 +164860,164860 +164861,164861 +164862,164862 +164863,164863 +164864,164864 +235650,164864 +164865,164865 +164866,164866 +164867,164867 +164868,164868 +164869,164869 +164870,164870 +164871,164871 +164872,164872 +164873,164873 +164874,164874 +164875,164875 +97872,164875 +164876,164876 +164877,164877 +164878,164878 +164879,164879 +164880,164880 +164881,164881 +164882,164882 +107630,164882 +164883,164883 +164884,164884 +164885,164885 +164886,164886 +164887,164887 +164888,164888 +141890,164888 +164889,164889 +149039,164889 +164890,164890 +149040,164890 +164891,164891 +164892,164892 +164893,164893 +164894,164894 +164895,164895 +186366,164895 +164896,164896 +164897,164897 +164898,164898 +164899,164899 +164900,164900 +164901,164901 +3587,164901 +164902,164902 +164903,164903 +187046,164903 +164904,164904 +164905,164905 +164906,164906 +164907,164907 +164908,164908 +164909,164909 +164910,164910 +77103,164910 +164911,164911 +164912,164912 +164913,164913 +164914,164914 +164915,164915 +164916,164916 +164917,164917 +164918,164918 +164919,164919 +164920,164920 +164921,164921 +164922,164922 +164923,164923 +164924,164924 +164925,164925 +164926,164926 +164927,164927 +164928,164928 +164929,164929 +164930,164930 +164931,164931 +164932,164932 +164933,164933 +164934,164934 +164935,164935 +164936,164936 +196710,164936 +164937,164937 +227020,164937 +164938,164938 +164939,164939 +164940,164940 +201154,164940 +164941,164941 +164942,164942 +164943,164943 +164944,164944 +164945,164945 +204047,164945 +164946,164946 +164947,164947 +164948,164948 +164949,164949 +164950,164950 +164951,164951 +164952,164952 +164953,164953 +164954,164954 +13835,164954 +164955,164955 +83511,164955 +164956,164956 +164957,164957 +164958,164958 +164959,164959 +164960,164960 +164961,164961 +164962,164962 +164963,164963 +164964,164964 +164965,164965 +164966,164966 +93404,164966 +164967,164967 +164968,164968 +164969,164969 +164970,164970 +164971,164971 +164972,164972 +164973,164973 +164974,164974 +164975,164975 +164976,164976 +164977,164977 +164978,164978 +164979,164979 +164980,164980 +35631,164980 +164981,164981 +164982,164982 +164983,164983 +183834,164983 +164984,164984 +164985,164985 +164986,164986 +164987,164987 +164988,164988 +164989,164989 +164990,164990 +164991,164991 +155180,164991 +164992,164992 +164993,164993 +164994,164994 +164995,164995 +164996,164996 +108480,164996 +164997,164997 +164998,164998 +90693,164998 +164999,164999 +165000,165000 +158735,165000 +165001,165001 +165002,165002 +165003,165003 +165004,165004 +165005,165005 +208494,165005 +165006,165006 +165007,165007 +165008,165008 +165009,165009 +165010,165010 +165011,165011 +165012,165012 +165013,165013 +165014,165014 +165015,165015 +165016,165016 +165017,165017 +165018,165018 +165019,165019 +165020,165020 +215477,165020 +165021,165021 +151212,165021 +165022,165022 +165023,165023 +165024,165024 +165025,165025 +165026,165026 +165027,165027 +165028,165028 +165029,165029 +165030,165030 +246193,165030 +165031,165031 +165032,165032 +165033,165033 +165034,165034 +165035,165035 +165036,165036 +165037,165037 +227025,165038 +165038,165038 +165039,165039 +165040,165040 +165041,165041 +165042,165042 +213953,165042 +165043,165043 +165044,165044 +165045,165045 +165046,165046 +165047,165047 +165048,165048 +165049,165049 +165050,165050 +165051,165051 +165052,165052 +165053,165053 +165054,165054 +165055,165055 +165056,165056 +165057,165057 +165058,165058 +165059,165059 +10580,165059 +165060,165060 +165061,165061 +165062,165062 +165063,165063 +165064,165064 +165065,165065 +165066,165066 +60572,165066 +165067,165067 +165068,165068 +165069,165069 +165070,165070 +165071,165071 +165072,165072 +165073,165073 +165074,165074 +165075,165075 +165076,165076 +165077,165077 +165078,165078 +165079,165079 +165080,165080 +165081,165081 +5612,165081 +174349,165081 +168385,165081 +165082,165082 +165083,165083 +165084,165084 +165085,165085 +76648,165085 +165086,165086 +165087,165087 +165088,165088 +165167,165167 +165168,165168 +165169,165169 +165170,165170 +165171,165171 +157579,165171 +165172,165172 +165173,165173 +165174,165174 +165175,165175 +165176,165176 +165177,165177 +74707,165177 +165178,165178 +165179,165179 +165180,165180 +165181,165181 +165182,165182 +165183,165183 +165184,165184 +165185,165185 +165186,165186 +165187,165187 +165188,165188 +165189,165189 +111903,165189 +165190,165190 +165191,165191 +165192,165192 +165193,165193 +165194,165194 +165195,165195 +108652,165195 +165196,165196 +165197,165197 +165198,165198 +165199,165199 +2023,165199 +248014,165199 +165200,165200 +165201,165201 +165202,165202 +165203,165203 +177418,165204 +171454,165204 +165204,165204 +165205,165205 +165206,165206 +165207,165207 +165208,165208 +107277,165208 +165209,165209 +165210,165210 +165211,165211 +165212,165212 +165213,165213 +165214,165214 +165215,165215 +165216,165216 +165217,165217 +32378,165217 +165218,165218 +206724,165218 +165219,165219 +178523,165219 +172559,165219 +180979,165219 +165220,165220 +165221,165221 +165222,165222 +165223,165223 +101037,165223 +165224,165224 +165225,165225 +37032,165225 +165226,165226 +165227,165227 +165228,165228 +165229,165229 +165230,165230 +165231,165231 +165232,165232 +211451,165232 +165233,165233 +165234,165234 +165235,165235 +177427,165235 +171463,165235 +165236,165236 +165237,165237 +165238,165238 +165239,165239 +165240,165240 +165241,165241 +165242,165242 +165243,165243 +165244,165244 +165245,165245 +165246,165246 +165247,165247 +165248,165248 +165249,165249 +165250,165250 +151131,165250 +165251,165251 +165252,165252 +165253,165253 +165254,165254 +165255,165255 +165256,165256 +165257,165257 +165258,165258 +165259,165259 +165260,165260 +165261,165261 +165262,165262 +165263,165263 +147885,165263 +165264,165264 +165265,165265 +165266,165266 +84016,165266 +165267,165267 +165268,165268 +165269,165269 +165270,165270 +165271,165271 +165272,165272 +87918,165272 +165273,165273 +165274,165274 +165275,165275 +165276,165276 +165277,165277 +165278,165278 +165279,165279 +165280,165280 +64085,165280 +165281,165281 +165282,165282 +165283,165283 +165284,165284 +165285,165285 +165286,165286 +165287,165287 +165288,165288 +165289,165289 +165290,165290 +114487,165290 +165291,165291 +165292,165292 +165293,165293 +165294,165294 +165295,165295 +165296,165296 +165297,165297 +165298,165298 +165299,165299 +165300,165300 +165301,165301 +165302,165302 +165303,165303 +165304,165304 +105665,165304 +165305,165305 +165306,165306 +141868,165306 +165307,165307 +165308,165308 +188331,165308 +165309,165309 +165310,165310 +165311,165311 +165312,165312 +165313,165313 +165314,165314 +165315,165315 +165316,165316 +165317,165317 +165318,165318 +165319,165319 +165320,165320 +165321,165321 +165322,165322 +165323,165323 +165324,165324 +165325,165325 +165326,165326 +165327,165327 +165328,165328 +165329,165329 +205169,165329 +165330,165330 +165331,165331 +165332,165332 +165333,165333 +165334,165334 +165335,165335 +165336,165336 +165337,165337 +165338,165338 +165339,165339 +165340,165340 +165341,165341 +165342,165342 +165343,165343 +165344,165344 +193680,165344 +165345,165345 +165346,165346 +165347,165347 +165348,165348 +165349,165349 +165350,165350 +165351,165351 +165352,165352 +165353,165353 +165354,165354 +165355,165355 +165356,165356 +165357,165357 +165358,165358 +165359,165359 +165360,165360 +165361,165361 +165362,165362 +165363,165363 +165364,165364 +165365,165365 +165366,165366 +165367,165367 +165368,165368 +165369,165369 +165370,165370 +165371,165371 +165372,165372 +165373,165373 +165374,165374 +165375,165375 +165376,165376 +165377,165377 +165378,165378 +165379,165379 +165380,165380 +165381,165381 +165382,165382 +165383,165383 +165384,165384 +165385,165385 +165386,165386 +165387,165387 +26893,165387 +165388,165388 +165389,165389 +165390,165390 +165391,165391 +165392,165392 +165393,165393 +20460,165393 +14633,165393 +165394,165394 +7715,165394 +165395,165395 +165396,165396 +165397,165397 +165398,165398 +165399,165399 +165400,165400 +165401,165401 +165402,165402 +165403,165403 +165404,165404 +165405,165405 +165406,165406 +165407,165407 +165408,165408 +165409,165409 +165410,165410 +165411,165411 +165412,165412 +165413,165413 +165414,165414 +165415,165415 +165416,165416 +165417,165417 +165418,165418 +165419,165419 +165420,165420 +165421,165421 +165422,165422 +165423,165423 +165424,165424 +165425,165425 +165426,165426 +165427,165427 +165467,165467 +165468,165468 +165469,165469 +165470,165470 +165471,165471 +165472,165472 +165473,165473 +165474,165474 +165603,165603 +165604,165604 +165605,165605 +165606,165606 +165607,165607 +165608,165608 +165609,165609 +4780,165609 +165610,165610 +165611,165611 +165612,165612 +165613,165613 +165684,165684 +41947,165684 +165685,165685 +165686,165686 +165757,165757 +165758,165758 +165759,165759 +165760,165760 +165761,165761 +165762,165762 +165763,165763 +165764,165764 +165765,165765 +165766,165766 +165767,165767 +165768,165768 +165769,165769 +184725,165769 +165770,165770 +165771,165771 +37296,165771 +165842,165842 +165843,165843 +165844,165844 +165845,165845 +166005,166005 +166006,166006 +166007,166007 +166008,166008 +166009,166009 +166010,166010 +166011,166011 +166012,166012 +166013,166013 +166014,166014 +166015,166015 +166016,166016 +207588,166016 +166017,166017 +166018,166018 +166019,166019 +166269,166269 +166270,166270 +166271,166271 +193444,166271 +166272,166272 +166432,166432 +98040,166432 +166433,166433 +166434,166434 +166435,166435 +166436,166436 +166437,166437 +166438,166438 +166439,166439 +166440,166440 +145044,166440 +166441,166441 +20804,166441 +166442,166442 +166443,166443 +50246,166443 +42054,166443 +166444,166444 +166445,166445 +166446,166446 +166447,166447 +166448,166448 +166449,166449 +166450,166450 +166451,166451 +166452,166452 +166453,166453 +166454,166454 +52627,166454 +44523,166454 +166455,166455 +166456,166456 +166457,166457 +166458,166458 +51852,166458 +43748,166458 +166459,166459 +166460,166460 +166461,166461 +166462,166462 +166463,166463 +166464,166464 +166465,166465 +166466,166466 +166467,166467 +166468,166468 +166469,166469 +166470,166470 +166471,166471 +166472,166472 +113345,166472 +166473,166473 +166474,166474 +166475,166475 +166476,166476 +166477,166477 +166478,166478 +166479,166479 +176754,166479 +170790,166479 +166480,166480 +166481,166481 +166482,166482 +166483,166483 +75004,166483 +166484,166484 +166485,166485 +166486,166486 +166487,166487 +166488,166488 +166489,166489 +166490,166490 +166491,166491 +166492,166492 +166493,166493 +166494,166494 +166495,166495 +166496,166496 +166497,166497 +166498,166498 +166499,166499 +166500,166500 +166501,166501 +166502,166502 +166503,166503 +166504,166504 +166505,166505 +166506,166506 +166507,166507 +166508,166508 +166509,166509 +166510,166510 +166511,166511 +166512,166512 +166513,166513 +166514,166514 +166515,166515 +166516,166516 +166517,166517 +166518,166518 +166519,166519 +166520,166520 +166521,166521 +166522,166522 +166523,166523 +166524,166524 +166525,166525 +166526,166526 +166527,166527 +166528,166528 +166529,166529 +166530,166530 +166531,166531 +166532,166532 +166533,166533 +166534,166534 +166535,166535 +166536,166536 +166537,166537 +166538,166538 +107093,166538 +166539,166539 +166576,166576 +166577,166577 +166578,166578 +166579,166579 +166580,166580 +166581,166581 +166582,166582 +166583,166583 +166584,166584 +166585,166585 +166586,166586 +166587,166587 +166588,166588 +166589,166589 +166590,166590 +166591,166591 +166592,166592 +166593,166593 +166594,166594 +166595,166595 +166596,166596 +166597,166597 +166598,166598 +166599,166599 +166600,166600 +166601,166601 +166602,166602 +166603,166603 +166604,166604 +166605,166605 +166606,166606 +166607,166607 +166608,166608 +166609,166609 +166610,166610 +2542,166610 +166611,166611 +166612,166612 +166613,166613 +5170,166613 +166614,166614 +166615,166615 +166616,166616 +166617,166617 +166618,166618 +166619,166619 +166620,166620 +148622,166620 +166621,166621 +166622,166622 +166623,166623 +166624,166624 +62279,166624 +166625,166625 +166626,166626 +166627,166627 +166628,166628 +166629,166629 +166630,166630 +166631,166631 +166632,166632 +166633,166633 +166634,166634 +166635,166635 +166636,166636 +166637,166637 +166638,166638 +166639,166639 +166640,166640 +166641,166641 +166642,166642 +166643,166643 +166644,166644 +166645,166645 +166646,166646 +166647,166647 +166648,166648 +166649,166649 +166650,166650 +166651,166651 +166652,166652 +166653,166653 +166654,166654 +166655,166655 +166656,166656 +166657,166657 +166658,166658 +166659,166659 +166660,166660 +166661,166661 +166662,166662 +166663,166663 +166664,166664 +166665,166665 +166666,166666 +166667,166667 +166668,166668 +166669,166669 +166670,166670 +166671,166671 +166672,166672 +166673,166673 +166674,166674 +166675,166675 +166676,166676 +166677,166677 +74153,166677 +166678,166678 +166679,166679 +166680,166680 +166681,166681 +166682,166682 +166683,166683 +166684,166684 +66129,166684 +166685,166685 +166686,166686 +166687,166687 +166688,166688 +166689,166689 +166690,166690 +166691,166691 +166692,166692 +166693,166693 +166694,166694 +166695,166695 +166696,166696 +166697,166697 +166698,166698 +166699,166699 +166700,166700 +166701,166701 +166702,166702 +166703,166703 +166704,166704 +166705,166705 +166706,166706 +166707,166707 +103477,166707 +166708,166708 +166709,166709 +166710,166710 +166711,166711 +166712,166712 +166713,166713 +166714,166714 +166715,166715 +166716,166716 +166717,166717 +166718,166718 +166719,166719 +243549,166719 +166720,166720 +166721,166721 +166722,166722 +92121,166722 +166723,166723 +166724,166724 +166725,166725 +2036,166725 +166726,166726 +135284,166726 +127697,166726 +150799,166726 +166727,166727 +166728,166728 +166729,166729 +166730,166730 +57568,166730 +166731,166731 +166732,166732 +166733,166733 +166734,166734 +166735,166735 +166736,166736 +166737,166737 +166738,166738 +166739,166739 +166740,166740 +166741,166741 +166742,166742 +166743,166743 +166744,166744 +166745,166745 +166746,166746 +166747,166747 +20412,166747 +20227,166747 +166748,166748 +166749,166749 +166750,166750 +166751,166751 +166752,166752 +166753,166753 +166754,166754 +166755,166755 +166756,166756 +166757,166757 +166758,166758 +166759,166759 +166760,166760 +166761,166761 +166762,166762 +166763,166763 +166764,166764 +107340,166764 +166765,166765 +166766,166766 +233566,166766 +166767,166767 +166768,166768 +166769,166769 +166770,166770 +166771,166771 +166772,166772 +166773,166773 +166774,166774 +166775,166775 +166776,166776 +166777,166777 +166778,166778 +166779,166779 +166780,166780 +166781,166781 +17380,166781 +166782,166782 +166783,166783 +166784,166784 +166785,166785 +166786,166786 +166787,166787 +166788,166788 +166789,166789 +166790,166790 +173894,167930 +167930,167930 +173895,167931 +167931,167931 +173896,167932 +167932,167932 +173897,167933 +167933,167933 +173898,167934 +167934,167934 +173899,167935 +167935,167935 +88260,167935 +173900,167936 +167936,167936 +173901,167937 +167937,167937 +173902,167938 +167938,167938 +112038,167938 +173903,167939 +167939,167939 +173904,167940 +167940,167940 +173905,167941 +167941,167941 +173906,167942 +167942,167942 +173907,167943 +167943,167943 +173908,167944 +167944,167944 +173909,167945 +167945,167945 +173910,167946 +167946,167946 +173911,167947 +167947,167947 +173912,167948 +167948,167948 +173913,167949 +167949,167949 +173914,167950 +167950,167950 +173915,167951 +167951,167951 +150880,167951 +173916,167952 +167952,167952 +206649,167952 +173917,167953 +167953,167953 +173918,167954 +167954,167954 +173919,167955 +167955,167955 +173920,167956 +167956,167956 +173921,167957 +167957,167957 +85781,167957 +173922,167958 +167958,167958 +173923,167959 +167959,167959 +173924,167960 +167960,167960 +173925,167961 +167961,167961 +83022,167961 +173926,167962 +167962,167962 +173927,167963 +167963,167963 +93263,167963 +173968,168004 +168004,168004 +107055,168004 +173969,168005 +168005,168005 +173970,168006 +168006,168006 +193711,168006 +173971,168007 +168007,168007 +216965,168007 +173972,168008 +168008,168008 +173973,168009 +168009,168009 +14548,168009 +173974,168010 +168010,168010 +173975,168011 +168011,168011 +173976,168012 +168012,168012 +173977,168013 +168013,168013 +173978,168014 +168014,168014 +173979,168015 +168015,168015 +173980,168016 +168016,168016 +173981,168017 +168017,168017 +173982,168018 +168018,168018 +173983,168019 +168019,168019 +173984,168020 +168020,168020 +173985,168021 +168021,168021 +173986,168022 +168022,168022 +173987,168023 +168023,168023 +110617,168023 +173988,168024 +168024,168024 +173989,168025 +168025,168025 +173990,168026 +168026,168026 +173991,168027 +168027,168027 +173992,168028 +168028,168028 +173993,168029 +168029,168029 +173994,168030 +168030,168030 +173995,168031 +168031,168031 +173996,168032 +168032,168032 +173997,168033 +168033,168033 +173998,168034 +168034,168034 +67269,168034 +173999,168035 +168035,168035 +174000,168036 +168036,168036 +174001,168037 +168037,168037 +174002,168038 +168038,168038 +174003,168039 +168039,168039 +174004,168040 +168040,168040 +174005,168041 +168041,168041 +174006,168042 +168042,168042 +183573,168042 +154114,168042 +174007,168043 +168043,168043 +174008,168044 +168044,168044 +174009,168045 +168045,168045 +174010,168046 +168046,168046 +174011,168047 +168047,168047 +174012,168048 +168048,168048 +174013,168049 +168049,168049 +174014,168050 +168050,168050 +174015,168051 +168051,168051 +174016,168052 +168052,168052 +174017,168053 +168053,168053 +174018,168054 +168054,168054 +174019,168055 +168055,168055 +174020,168056 +168056,168056 +174021,168057 +168057,168057 +174022,168058 +168058,168058 +174023,168059 +168059,168059 +174024,168060 +168060,168060 +174025,168061 +168061,168061 +174026,168062 +168062,168062 +174027,168063 +168063,168063 +174155,168191 +168191,168191 +174156,168192 +168192,168192 +174157,168193 +168193,168193 +174158,168194 +168194,168194 +174159,168195 +168195,168195 +174160,168196 +168196,168196 +174161,168197 +168197,168197 +174162,168198 +168198,168198 +174163,168199 +168199,168199 +174164,168200 +168200,168200 +174165,168201 +168201,168201 +63403,168201 +227703,168201 +19003,168201 +174166,168202 +168202,168202 +174167,168203 +168203,168203 +174168,168204 +168204,168204 +174169,168205 +168205,168205 +174170,168206 +168206,168206 +174171,168207 +168207,168207 +224774,168207 +174172,168208 +168208,168208 +174173,168209 +168209,168209 +174174,168210 +168210,168210 +174175,168211 +168211,168211 +174176,168212 +168212,168212 +174177,168213 +168213,168213 +6370,168213 +174178,168214 +168214,168214 +174179,168215 +168215,168215 +174180,168216 +168216,168216 +174181,168217 +168217,168217 +174182,168218 +168218,168218 +174183,168219 +168219,168219 +174184,168220 +168220,168220 +174185,168221 +168221,168221 +174186,168222 +168222,168222 +174187,168223 +168223,168223 +174255,168291 +168291,168291 +174256,168292 +168292,168292 +246601,168292 +174257,168293 +168293,168293 +174258,168294 +168294,168294 +174259,168295 +168295,168295 +174260,168296 +168296,168296 +174261,168297 +168297,168297 +174262,168298 +168298,168298 +174263,168299 +168299,168299 +174264,168300 +168300,168300 +174265,168301 +168301,168301 +174266,168302 +168302,168302 +174267,168303 +168303,168303 +174268,168304 +168304,168304 +174269,168305 +168305,168305 +174270,168306 +168306,168306 +174271,168307 +168307,168307 +174272,168308 +168308,168308 +174273,168309 +168309,168309 +174274,168310 +168310,168310 +174275,168311 +168311,168311 +174276,168312 +168312,168312 +174277,168313 +168313,168313 +174278,168314 +168314,168314 +174279,168315 +168315,168315 +174280,168316 +168316,168316 +174281,168317 +168317,168317 +174282,168318 +168318,168318 +174283,168319 +168319,168319 +174284,168320 +168320,168320 +174285,168321 +168321,168321 +174286,168322 +168322,168322 +174287,168323 +168323,168323 +174288,168324 +168324,168324 +174289,168325 +168325,168325 +174290,168326 +168326,168326 +174291,168327 +168327,168327 +174292,168328 +168328,168328 +174293,168329 +168329,168329 +174294,168330 +168330,168330 +174295,168331 +168331,168331 +174296,168332 +168332,168332 +174297,168333 +168333,168333 +174298,168334 +168334,168334 +174299,168335 +168335,168335 +96028,168335 +174300,168336 +168336,168336 +174336,168372 +168372,168372 +174337,168373 +168373,168373 +174338,168374 +168374,168374 +100590,168374 +174339,168375 +168375,168375 +174340,168376 +168376,168376 +174341,168377 +168377,168377 +174342,168378 +168378,168378 +174343,168379 +168379,168379 +174344,168380 +168380,168380 +174345,168381 +168381,168381 +174346,168382 +168382,168382 +174347,168383 +168383,168383 +174348,168384 +168384,168384 +174349,168385 +168385,168385 +165081,168385 +5612,168385 +174350,168386 +168386,168386 +174351,168387 +168387,168387 +174352,168388 +168388,168388 +174353,168389 +168389,168389 +174354,168390 +168390,168390 +174355,168391 +168391,168391 +174356,168392 +168392,168392 +60924,168392 +174357,168393 +168393,168393 +9081,168393 +174358,168394 +168394,168394 +174359,168395 +168395,168395 +174360,168396 +168396,168396 +174361,168397 +168397,168397 +174362,168398 +168398,168398 +114878,168398 +174363,168399 +168399,168399 +174364,168400 +168400,168400 +84663,168400 +174365,168401 +168401,168401 +174366,168402 +168402,168402 +77327,168402 +174367,168403 +168403,168403 +174368,168404 +168404,168404 +174369,168405 +168405,168405 +174472,168508 +168508,168508 +174473,168509 +168509,168509 +174474,168510 +168510,168510 +174475,168511 +168511,168511 +174476,168512 +168512,168512 +174477,168513 +168513,168513 +174478,168514 +168514,168514 +174606,168642 +168642,168642 +174607,168643 +168643,168643 +174608,168644 +168644,168644 +174609,168645 +168645,168645 +174610,168646 +168646,168646 +174611,168647 +168647,168647 +174612,168648 +168648,168648 +174613,168649 +168649,168649 +174614,168650 +168650,168650 +191059,168650 +174615,168651 +168651,168651 +174616,168652 +168652,168652 +174617,168653 +168653,168653 +174618,168654 +168654,168654 +174619,168655 +168655,168655 +174620,168656 +168656,168656 +174621,168657 +168657,168657 +174622,168658 +168658,168658 +174623,168659 +168659,168659 +174624,168660 +168660,168660 +174625,168661 +168661,168661 +174626,168662 +168662,168662 +174627,168663 +168663,168663 +174628,168664 +168664,168664 +174629,168665 +168665,168665 +174630,168666 +168666,168666 +57998,168666 +174631,168667 +168667,168667 +174632,168668 +168668,168668 +174633,168669 +168669,168669 +174634,168670 +168670,168670 +174635,168671 +168671,168671 +174636,168672 +168672,168672 +174637,168673 +168673,168673 +174638,168674 +168674,168674 +174639,168675 +168675,168675 +39790,168675 +174640,168676 +168676,168676 +174641,168677 +168677,168677 +174642,168678 +168678,168678 +174643,168679 +168679,168679 +174644,168680 +168680,168680 +174645,168681 +168681,168681 +175994,170030 +170030,170030 +175995,170031 +170031,170031 +175996,170032 +170032,170032 +175997,170033 +170033,170033 +175998,170034 +170034,170034 +175999,170035 +170035,170035 +176000,170036 +170036,170036 +176001,170037 +170037,170037 +176002,170038 +170038,170038 +176003,170039 +170039,170039 +176004,170040 +170040,170040 +176005,170041 +170041,170041 +107095,170041 +176006,170042 +170042,170042 +176007,170043 +170043,170043 +176008,170044 +170044,170044 +176009,170045 +170045,170045 +176010,170046 +170046,170046 +176011,170047 +170047,170047 +176012,170048 +170048,170048 +176013,170049 +170049,170049 +176014,170050 +170050,170050 +176230,170266 +170266,170266 +176231,170267 +170267,170267 +176232,170268 +170268,170268 +176233,170269 +170269,170269 +176234,170270 +170270,170270 +176235,170271 +170271,170271 +176236,170272 +170272,170272 +176237,170273 +170273,170273 +140288,170273 +176238,170274 +170274,170274 +176239,170275 +170275,170275 +176240,170276 +170276,170276 +176241,170277 +170277,170277 +51819,170277 +43715,170277 +176242,170278 +170278,170278 +176243,170279 +170279,170279 +176244,170280 +170280,170280 +176245,170281 +170281,170281 +176246,170282 +170282,170282 +176247,170283 +170283,170283 +176248,170284 +170284,170284 +176249,170285 +170285,170285 +176250,170286 +170286,170286 +176251,170287 +170287,170287 +176252,170288 +170288,170288 +176253,170289 +170289,170289 +176254,170290 +170290,170290 +176255,170291 +170291,170291 +176256,170292 +170292,170292 +176257,170293 +170293,170293 +176258,170294 +170294,170294 +176259,170295 +170295,170295 +176260,170296 +170296,170296 +176261,170297 +170297,170297 +176262,170298 +170298,170298 +176263,170299 +170299,170299 +176264,170300 +170300,170300 +144284,170300 +176265,170301 +170301,170301 +176266,170302 +170302,170302 +176267,170303 +170303,170303 +176268,170304 +170304,170304 +176269,170305 +170305,170305 +176270,170306 +170306,170306 +176271,170307 +170307,170307 +176272,170308 +170308,170308 +176273,170309 +170309,170309 +176274,170310 +170310,170310 +176275,170311 +170311,170311 +176276,170312 +170312,170312 +176277,170313 +170313,170313 +176278,170314 +170314,170314 +103613,170314 +176279,170315 +170315,170315 +176280,170316 +170316,170316 +176281,170317 +170317,170317 +176282,170318 +170318,170318 +176283,170319 +170319,170319 +115968,170319 +176284,170320 +170320,170320 +15382,170320 +96471,170320 +176285,170321 +170321,170321 +15383,170321 +9458,170321 +176286,170322 +170322,170322 +176287,170323 +170323,170323 +176288,170324 +170324,170324 +176289,170325 +170325,170325 +176410,170446 +170446,170446 +176411,170447 +170447,170447 +176412,170448 +170448,170448 +176413,170449 +170449,170449 +176414,170450 +170450,170450 +14654,170450 +176415,170451 +170451,170451 +176416,170452 +170452,170452 +176417,170453 +170453,170453 +176418,170454 +170454,170454 +176419,170455 +170455,170455 +176420,170456 +170456,170456 +176421,170457 +170457,170457 +176422,170458 +170458,170458 +176423,170459 +170459,170459 +176424,170460 +170460,170460 +176425,170461 +170461,170461 +176426,170462 +170462,170462 +176427,170463 +170463,170463 +205454,170464 +176428,170464 +170464,170464 +176429,170465 +170465,170465 +176430,170466 +170466,170466 +176431,170467 +170467,170467 +176432,170468 +170468,170468 +176433,170469 +170469,170469 +176434,170470 +170470,170470 +176435,170471 +170471,170471 +176436,170472 +170472,170472 +176437,170473 +170473,170473 +176438,170474 +170474,170474 +176439,170475 +170475,170475 +176440,170476 +170476,170476 +176441,170477 +170477,170477 +176442,170478 +170478,170478 +176443,170479 +170479,170479 +176444,170480 +170480,170480 +176445,170481 +170481,170481 +176446,170482 +170482,170482 +176447,170483 +170483,170483 +176448,170484 +170484,170484 +176449,170485 +170485,170485 +176450,170486 +170486,170486 +176451,170487 +170487,170487 +176452,170488 +170488,170488 +176453,170489 +170489,170489 +176454,170490 +170490,170490 +176455,170491 +170491,170491 +176456,170492 +170492,170492 +176457,170493 +170493,170493 +176458,170494 +170494,170494 +176459,170495 +170495,170495 +176460,170496 +170496,170496 +176461,170497 +170497,170497 +176462,170498 +170498,170498 +176463,170499 +170499,170499 +241592,170499 +176464,170500 +170500,170500 +176465,170501 +170501,170501 +176466,170502 +170502,170502 +176467,170503 +170503,170503 +176468,170504 +170504,170504 +68275,170504 +176469,170505 +170505,170505 +82952,170505 +176470,170506 +170506,170506 +176471,170507 +170507,170507 +176472,170508 +170508,170508 +176473,170509 +170509,170509 +176474,170510 +170510,170510 +176475,170511 +170511,170511 +176476,170512 +170512,170512 +176477,170513 +170513,170513 +57687,170513 +176478,170514 +170514,170514 +176479,170515 +170515,170515 +176480,170516 +170516,170516 +176481,170517 +170517,170517 +140484,170517 +176482,170518 +170518,170518 +176483,170519 +170519,170519 +176484,170520 +170520,170520 +176485,170521 +170521,170521 +176486,170522 +170522,170522 +176487,170523 +170523,170523 +176488,170524 +170524,170524 +176489,170525 +170525,170525 +176490,170526 +170526,170526 +176491,170527 +170527,170527 +176492,170528 +170528,170528 +176493,170529 +170529,170529 +176494,170530 +170530,170530 +176495,170531 +170531,170531 +176496,170532 +170532,170532 +157534,170532 +176497,170533 +170533,170533 +176498,170534 +170534,170534 +176499,170535 +170535,170535 +176500,170536 +170536,170536 +176501,170537 +170537,170537 +87352,170537 +176502,170538 +170538,170538 +176503,170539 +170539,170539 +176504,170540 +170540,170540 +176505,170541 +170541,170541 +176531,170567 +170567,170567 +176532,170568 +170568,170568 +176533,170569 +170569,170569 +176534,170570 +170570,170570 +176535,170571 +170571,170571 +176536,170572 +170572,170572 +51807,170572 +43703,170572 +176537,170573 +170573,170573 +135433,170573 +127846,170573 +7241,170573 +176538,170574 +170574,170574 +176539,170575 +170575,170575 +176540,170576 +170576,170576 +88499,170576 +176541,170577 +170577,170577 +176542,170578 +170578,170578 +233823,170578 +176543,170579 +170579,170579 +176544,170580 +170580,170580 +176545,170581 +170581,170581 +176546,170582 +170582,170582 +176547,170583 +170583,170583 +176548,170584 +170584,170584 +176549,170585 +170585,170585 +176550,170586 +170586,170586 +59998,170586 +176551,170587 +170587,170587 +146721,170587 +176552,170588 +170588,170588 +176553,170589 +170589,170589 +176554,170590 +170590,170590 +246607,170590 +176555,170591 +170591,170591 +176556,170592 +170592,170592 +213770,170593 +176557,170593 +170593,170593 +176558,170594 +170594,170594 +176559,170595 +170595,170595 +176560,170596 +170596,170596 +176561,170597 +170597,170597 +176562,170598 +170598,170598 +188242,170598 +176563,170599 +170599,170599 +176564,170600 +170600,170600 +176565,170601 +170601,170601 +176566,170602 +170602,170602 +176567,170603 +170603,170603 +176568,170604 +170604,170604 +176569,170605 +170605,170605 +176570,170606 +170606,170606 +176571,170607 +170607,170607 +176572,170608 +170608,170608 +176573,170609 +170609,170609 +109267,170609 +176574,170610 +170610,170610 +176575,170611 +170611,170611 +176576,170612 +170612,170612 +176577,170613 +170613,170613 +176578,170614 +170614,170614 +176579,170615 +170615,170615 +176580,170616 +170616,170616 +176581,170617 +170617,170617 +176582,170618 +170618,170618 +176583,170619 +170619,170619 +176584,170620 +170620,170620 +176585,170621 +170621,170621 +176586,170622 +170622,170622 +86910,170622 +176587,170623 +170623,170623 +176588,170624 +170624,170624 +176589,170625 +170625,170625 +176590,170626 +170626,170626 +110606,170626 +176591,170627 +170627,170627 +6317,170627 +176592,170628 +170628,170628 +176593,170629 +170629,170629 +176594,170630 +170630,170630 +176595,170631 +170631,170631 +176596,170632 +170632,170632 +49140,170632 +40948,170632 +176597,170633 +170633,170633 +176598,170634 +170634,170634 +176599,170635 +170635,170635 +176600,170636 +170636,170636 +176601,170637 +170637,170637 +176602,170638 +170638,170638 +176603,170639 +170639,170639 +176604,170640 +170640,170640 +176605,170641 +170641,170641 +233668,170641 +176606,170642 +170642,170642 +176607,170643 +170643,170643 +158794,170643 +176608,170644 +170644,170644 +176609,170645 +170645,170645 +176610,170646 +170646,170646 +176611,170647 +170647,170647 +176612,170648 +170648,170648 +176613,170649 +170649,170649 +176614,170650 +170650,170650 +176615,170651 +170651,170651 +176616,170652 +170652,170652 +176617,170653 +170653,170653 +176618,170654 +170654,170654 +176619,170655 +170655,170655 +176620,170656 +170656,170656 +176621,170657 +170657,170657 +176622,170658 +170658,170658 +176623,170659 +170659,170659 +176624,170660 +170660,170660 +176625,170661 +170661,170661 +176626,170662 +170662,170662 +176627,170663 +170663,170663 +191868,170663 +176628,170664 +170664,170664 +176629,170665 +170665,170665 +176630,170666 +170666,170666 +176631,170667 +170667,170667 +191851,170667 +176632,170668 +170668,170668 +176633,170669 +170669,170669 +176634,170670 +170670,170670 +176635,170671 +170671,170671 +176636,170672 +170672,170672 +176637,170673 +170673,170673 +176638,170674 +170674,170674 +176639,170675 +170675,170675 +176640,170676 +170676,170676 +176641,170677 +170677,170677 +176642,170678 +170678,170678 +79356,170678 +176643,170679 +170679,170679 +176644,170680 +170680,170680 +58247,170680 +176645,170681 +170681,170681 +176646,170682 +170682,170682 +176647,170683 +170683,170683 +176648,170684 +170684,170684 +176649,170685 +170685,170685 +176650,170686 +170686,170686 +176651,170687 +170687,170687 +176652,170688 +170688,170688 +176653,170689 +170689,170689 +176654,170690 +170690,170690 +176655,170691 +170691,170691 +176656,170692 +170692,170692 +176657,170693 +170693,170693 +176658,170694 +170694,170694 +176659,170695 +170695,170695 +176660,170696 +170696,170696 +204556,170696 +176661,170697 +170697,170697 +176662,170698 +170698,170698 +208875,170698 +176663,170699 +170699,170699 +176664,170700 +170700,170700 +176665,170701 +170701,170701 +176666,170702 +170702,170702 +176667,170703 +170703,170703 +176668,170704 +170704,170704 +176669,170705 +170705,170705 +176670,170706 +170706,170706 +176671,170707 +170707,170707 +176672,170708 +170708,170708 +176673,170709 +170709,170709 +176674,170710 +170710,170710 +176675,170711 +170711,170711 +135926,170711 +128339,170711 +176676,170712 +170712,170712 +176677,170713 +170713,170713 +176678,170714 +170714,170714 +176679,170715 +170715,170715 +176680,170716 +170716,170716 +176681,170717 +170717,170717 +176682,170718 +170718,170718 +176683,170719 +170719,170719 +36686,170719 +176684,170720 +170720,170720 +176685,170721 +170721,170721 +176686,170722 +170722,170722 +176687,170723 +170723,170723 +176688,170724 +170724,170724 +176689,170725 +170725,170725 +176690,170726 +170726,170726 +176691,170727 +170727,170727 +176692,170728 +170728,170728 +176693,170729 +170729,170729 +176694,170730 +170730,170730 +176695,170731 +170731,170731 +176696,170732 +170732,170732 +176697,170733 +170733,170733 +176698,170734 +170734,170734 +176699,170735 +170735,170735 +176700,170736 +170736,170736 +176701,170737 +170737,170737 +176702,170738 +170738,170738 +176703,170739 +170739,170739 +176704,170740 +170740,170740 +176705,170741 +170741,170741 +176706,170742 +170742,170742 +176707,170743 +170743,170743 +176708,170744 +170744,170744 +176709,170745 +170745,170745 +6508,170745 +176710,170746 +170746,170746 +176711,170747 +170747,170747 +176712,170748 +170748,170748 +176713,170749 +170749,170749 +176714,170750 +170750,170750 +176715,170751 +170751,170751 +176716,170752 +170752,170752 +176717,170753 +170753,170753 +176718,170754 +170754,170754 +176719,170755 +170755,170755 +176720,170756 +170756,170756 +176721,170757 +170757,170757 +176722,170758 +170758,170758 +176723,170759 +170759,170759 +162433,170759 +176724,170760 +170760,170760 +176725,170761 +170761,170761 +5339,170761 +176726,170762 +170762,170762 +5304,170762 +176727,170763 +170763,170763 +176728,170764 +170764,170764 +176729,170765 +170765,170765 +110707,170765 +176730,170766 +170766,170766 +176731,170767 +170767,170767 +176732,170768 +170768,170768 +176733,170769 +170769,170769 +176734,170770 +170770,170770 +176735,170771 +170771,170771 +176736,170772 +170772,170772 +176737,170773 +170773,170773 +176738,170774 +170774,170774 +176739,170775 +170775,170775 +176740,170776 +170776,170776 +176741,170777 +170777,170777 +176742,170778 +170778,170778 +176743,170779 +170779,170779 +176744,170780 +170780,170780 +176745,170781 +170781,170781 +176746,170782 +170782,170782 +189795,170782 +176747,170783 +170783,170783 +176748,170784 +170784,170784 +176749,170785 +170785,170785 +176750,170786 +170786,170786 +176751,170787 +170787,170787 +176752,170788 +170788,170788 +176753,170789 +170789,170789 +176754,170790 +170790,170790 +166479,170790 +176755,170791 +170791,170791 +176756,170792 +170792,170792 +154714,170792 +176757,170793 +170793,170793 +176758,170794 +170794,170794 +176759,170795 +170795,170795 +176760,170796 +170796,170796 +176761,170797 +170797,170797 +176762,170798 +170798,170798 +176763,170799 +170799,170799 +176764,170800 +170800,170800 +222147,170800 +176765,170801 +170801,170801 +176766,170802 +170802,170802 +176767,170803 +170803,170803 +176768,170804 +170804,170804 +176769,170805 +170805,170805 +233737,170805 +176770,170806 +170806,170806 +176771,170807 +170807,170807 +176772,170808 +170808,170808 +176773,170809 +170809,170809 +188047,170809 +176774,170810 +170810,170810 +176775,170811 +170811,170811 +176776,170812 +170812,170812 +176777,170813 +170813,170813 +176778,170814 +170814,170814 +13484,170814 +176779,170815 +170815,170815 +176780,170816 +170816,170816 +176781,170817 +170817,170817 +176782,170818 +170818,170818 +176783,170819 +170819,170819 +176784,170820 +170820,170820 +176785,170821 +170821,170821 +176786,170822 +170822,170822 +89206,170822 +176787,170823 +170823,170823 +176788,170824 +170824,170824 +176789,170825 +170825,170825 +176790,170826 +170826,170826 +176791,170827 +170827,170827 +176792,170828 +170828,170828 +176793,170829 +170829,170829 +176794,170830 +170830,170830 +176795,170831 +170831,170831 +176796,170832 +170832,170832 +177418,171454 +171454,171454 +165204,171454 +177419,171455 +171455,171455 +177420,171456 +171456,171456 +177421,171457 +171457,171457 +177422,171458 +171458,171458 +146572,171458 +177423,171459 +171459,171459 +81451,171459 +177424,171460 +171460,171460 +177425,171461 +171461,171461 +177426,171462 +171462,171462 +177427,171463 +171463,171463 +165235,171463 +177428,171464 +171464,171464 +177429,171465 +171465,171465 +177430,171466 +171466,171466 +177431,171467 +171467,171467 +177432,171468 +171468,171468 +177433,171469 +171469,171469 +177434,171470 +171470,171470 +177435,171471 +171471,171471 +177436,171472 +171472,171472 +177437,171473 +171473,171473 +177438,171474 +171474,171474 +178076,171474 +172112,171474 +177476,171512 +171512,171512 +177477,171513 +171513,171513 +177478,171514 +171514,171514 +108695,171514 +177479,171515 +171515,171515 +177480,171516 +171516,171516 +177481,171517 +171517,171517 +177482,171518 +171518,171518 +177483,171519 +171519,171519 +177484,171520 +171520,171520 +177485,171521 +171521,171521 +177486,171522 +171522,171522 +177487,171523 +171523,171523 +177488,171524 +171524,171524 +225824,171524 +177489,171525 +171525,171525 +178331,171525 +172367,171525 +177490,171526 +171526,171526 +177491,171527 +171527,171527 +177492,171528 +171528,171528 +177493,171529 +171529,171529 +177494,171530 +171530,171530 +177495,171531 +171531,171531 +114753,171531 +177496,171532 +171532,171532 +177497,171533 +171533,171533 +177498,171534 +171534,171534 +177499,171535 +171535,171535 +177500,171536 +171536,171536 +106707,171536 +177501,171537 +171537,171537 +177502,171538 +171538,171538 +194168,171538 +177503,171539 +171539,171539 +6900,171539 +177504,171540 +171540,171540 +33928,171540 +177505,171541 +171541,171541 +177506,171542 +171542,171542 +177507,171543 +171543,171543 +177508,171544 +171544,171544 +177509,171545 +171545,171545 +73750,171545 +177510,171546 +171546,171546 +177511,171547 +171547,171547 +177512,171548 +171548,171548 +177513,171549 +171549,171549 +177514,171550 +171550,171550 +177515,171551 +171551,171551 +177516,171552 +171552,171552 +177517,171553 +171553,171553 +177518,171554 +171554,171554 +177519,171555 +171555,171555 +177520,171556 +171556,171556 +203595,171556 +177521,171557 +171557,171557 +177522,171558 +171558,171558 +177523,171559 +171559,171559 +177524,171560 +171560,171560 +177525,171561 +171561,171561 +177526,171562 +171562,171562 +149755,171562 +177527,171563 +171563,171563 +177528,171564 +171564,171564 +177529,171565 +171565,171565 +79573,171565 +177530,171566 +171566,171566 +177531,171567 +171567,171567 +177532,171568 +171568,171568 +177533,171569 +171569,171569 +177534,171570 +171570,171570 +177535,171571 +171571,171571 +177536,171572 +171572,171572 +177537,171573 +171573,171573 +177538,171574 +171574,171574 +177539,171575 +171575,171575 +177540,171576 +171576,171576 +177541,171577 +171577,171577 +177542,171578 +171578,171578 +67020,171578 +177543,171579 +171579,171579 +177544,171580 +171580,171580 +177545,171581 +171581,171581 +177546,171582 +171582,171582 +177547,171583 +171583,171583 +177548,171584 +171584,171584 +177549,171585 +171585,171585 +177550,171586 +171586,171586 +177551,171587 +171587,171587 +177552,171588 +171588,171588 +177553,171589 +171589,171589 +177554,171590 +171590,171590 +177555,171591 +171591,171591 +177556,171592 +171592,171592 +177557,171593 +171593,171593 +177558,171594 +171594,171594 +177559,171595 +171595,171595 +177560,171596 +171596,171596 +177561,171597 +171597,171597 +177562,171598 +171598,171598 +177563,171599 +171599,171599 +177564,171600 +171600,171600 +177565,171601 +171601,171601 +177566,171602 +171602,171602 +177567,171603 +171603,171603 +177568,171604 +171604,171604 +177569,171605 +171605,171605 +177570,171606 +171606,171606 +177571,171607 +171607,171607 +177572,171608 +171608,171608 +177573,171609 +171609,171609 +177574,171610 +171610,171610 +177575,171611 +171611,171611 +177576,171612 +171612,171612 +177577,171613 +171613,171613 +177578,171614 +171614,171614 +177579,171615 +171615,171615 +177580,171616 +171616,171616 +177581,171617 +171617,171617 +104400,171617 +177582,171618 +171618,171618 +177583,171619 +171619,171619 +177584,171620 +171620,171620 +177585,171621 +171621,171621 +187006,171621 +177586,171622 +171622,171622 +177587,171623 +171623,171623 +177588,171624 +171624,171624 +177589,171625 +171625,171625 +177590,171626 +171626,171626 +177591,171627 +171627,171627 +177592,171628 +171628,171628 +177593,171629 +171629,171629 +177594,171630 +171630,171630 +177595,171631 +171631,171631 +177596,171632 +171632,171632 +177597,171633 +171633,171633 +177598,171634 +171634,171634 +177599,171635 +171635,171635 +177600,171636 +171636,171636 +177601,171637 +171637,171637 +177602,171638 +171638,171638 +177603,171639 +171639,171639 +177604,171640 +171640,171640 +177605,171641 +171641,171641 +177606,171642 +171642,171642 +177607,171643 +171643,171643 +177608,171644 +171644,171644 +177609,171645 +171645,171645 +177610,171646 +171646,171646 +177611,171647 +171647,171647 +177612,171648 +171648,171648 +177613,171649 +171649,171649 +177614,171650 +171650,171650 +177615,171651 +171651,171651 +177616,171652 +171652,171652 +177617,171653 +171653,171653 +177618,171654 +171654,171654 +177619,171655 +171655,171655 +177620,171656 +171656,171656 +177621,171657 +171657,171657 +177622,171658 +171658,171658 +177623,171659 +171659,171659 +177624,171660 +171660,171660 +177625,171661 +171661,171661 +177626,171662 +171662,171662 +177627,171663 +171663,171663 +177628,171664 +171664,171664 +212097,171664 +177629,171665 +171665,171665 +177630,171666 +171666,171666 +202306,171666 +177631,171667 +171667,171667 +208631,171667 +177632,171668 +171668,171668 +177633,171669 +171669,171669 +177634,171670 +171670,171670 +177635,171671 +171671,171671 +177636,171672 +171672,171672 +177637,171673 +171673,171673 +177638,171674 +171674,171674 +177639,171675 +171675,171675 +177640,171676 +171676,171676 +177641,171677 +171677,171677 +177642,171678 +171678,171678 +177643,171679 +171679,171679 +177644,171680 +171680,171680 +177645,171681 +171681,171681 +180542,171681 +177646,171682 +171682,171682 +177647,171683 +171683,171683 +177648,171684 +171684,171684 +177649,171685 +171685,171685 +177650,171686 +171686,171686 +211680,171686 +177651,171687 +171687,171687 +177652,171688 +171688,171688 +177653,171689 +171689,171689 +177654,171690 +171690,171690 +177655,171691 +171691,171691 +177656,171692 +171692,171692 +177657,171693 +171693,171693 +177658,171694 +171694,171694 +177659,171695 +171695,171695 +177660,171696 +171696,171696 +177661,171697 +171697,171697 +177662,171698 +171698,171698 +177663,171699 +171699,171699 +177664,171700 +171700,171700 +177665,171701 +171701,171701 +177666,171702 +171702,171702 +177667,171703 +171703,171703 +177668,171704 +171704,171704 +177669,171705 +171705,171705 +177670,171706 +171706,171706 +177671,171707 +171707,171707 +177672,171708 +171708,171708 +177673,171709 +171709,171709 +177674,171710 +171710,171710 +148999,171710 +53388,171710 +45284,171710 +177675,171711 +171711,171711 +177676,171712 +171712,171712 +177677,171713 +171713,171713 +177678,171714 +171714,171714 +177679,171715 +171715,171715 +177680,171716 +171716,171716 +177681,171717 +171717,171717 +177682,171718 +171718,171718 +177683,171719 +171719,171719 +177684,171720 +171720,171720 +177685,171721 +171721,171721 +177686,171722 +171722,171722 +177687,171723 +171723,171723 +177688,171724 +171724,171724 +177689,171725 +171725,171725 +177690,171726 +171726,171726 +177691,171727 +171727,171727 +177692,171728 +171728,171728 +177693,171729 +171729,171729 +177694,171730 +171730,171730 +177695,171731 +171731,171731 +177696,171732 +171732,171732 +177697,171733 +171733,171733 +177698,171734 +171734,171734 +177699,171735 +171735,171735 +177700,171736 +171736,171736 +177701,171737 +171737,171737 +177702,171738 +171738,171738 +177703,171739 +171739,171739 +177704,171740 +171740,171740 +177705,171741 +171741,171741 +177706,171742 +171742,171742 +177707,171743 +171743,171743 +141323,171743 +177708,171744 +171744,171744 +177709,171745 +171745,171745 +177710,171746 +171746,171746 +177711,171747 +171747,171747 +177712,171748 +171748,171748 +177713,171749 +171749,171749 +101842,171749 +177714,171750 +171750,171750 +177715,171751 +171751,171751 +216516,171751 +177716,171752 +171752,171752 +177717,171753 +171753,171753 +177718,171754 +171754,171754 +211600,171754 +177719,171755 +171755,171755 +177720,171756 +171756,171756 +177721,171757 +171757,171757 +177722,171758 +171758,171758 +177723,171759 +171759,171759 +177724,171760 +171760,171760 +177725,171761 +171761,171761 +177726,171762 +171762,171762 +177727,171763 +171763,171763 +177728,171764 +171764,171764 +177729,171765 +171765,171765 +177730,171766 +171766,171766 +177731,171767 +171767,171767 +177732,171768 +171768,171768 +177733,171769 +171769,171769 +177734,171770 +171770,171770 +177735,171771 +171771,171771 +177736,171772 +171772,171772 +177737,171773 +171773,171773 +177738,171774 +171774,171774 +177739,171775 +171775,171775 +177740,171776 +171776,171776 +177741,171777 +171777,171777 +177742,171778 +171778,171778 +177743,171779 +171779,171779 +177744,171780 +171780,171780 +177745,171781 +171781,171781 +177746,171782 +171782,171782 +177747,171783 +171783,171783 +177748,171784 +171784,171784 +177749,171785 +171785,171785 +177750,171786 +171786,171786 +177751,171787 +171787,171787 +177752,171788 +171788,171788 +177753,171789 +171789,171789 +177754,171790 +171790,171790 +177755,171791 +171791,171791 +177756,171792 +171792,171792 +177757,171793 +171793,171793 +177758,171794 +171794,171794 +177759,171795 +171795,171795 +177760,171796 +171796,171796 +180969,171796 +177761,171797 +171797,171797 +177762,171798 +171798,171798 +177763,171799 +171799,171799 +177764,171800 +171800,171800 +177765,171801 +171801,171801 +177766,171802 +171802,171802 +177767,171803 +171803,171803 +177768,171804 +171804,171804 +179949,171805 +177769,171805 +171805,171805 +177770,171806 +171806,171806 +180831,171806 +33821,171806 +177771,171807 +171807,171807 +33822,171807 +177772,171808 +171808,171808 +177773,171809 +171809,171809 +177774,171810 +171810,171810 +177775,171811 +171811,171811 +177776,171812 +171812,171812 +177777,171813 +171813,171813 +177778,171814 +171814,171814 +177779,171815 +171815,171815 +177780,171816 +171816,171816 +177781,171817 +171817,171817 +177782,171818 +171818,171818 +177783,171819 +171819,171819 +177784,171820 +171820,171820 +177785,171821 +171821,171821 +177786,171822 +171822,171822 +30573,171822 +177787,171823 +171823,171823 +177788,171824 +171824,171824 +177789,171825 +171825,171825 +177790,171826 +171826,171826 +177791,171827 +171827,171827 +177792,171828 +171828,171828 +177793,171829 +171829,171829 +177794,171830 +171830,171830 +177795,171831 +171831,171831 +177796,171832 +171832,171832 +177797,171833 +171833,171833 +177798,171834 +171834,171834 +177799,171835 +171835,171835 +177800,171836 +171836,171836 +177801,171837 +171837,171837 +144841,171837 +177802,171838 +171838,171838 +177803,171839 +171839,171839 +177804,171840 +171840,171840 +177805,171841 +171841,171841 +177806,171842 +171842,171842 +177807,171843 +171843,171843 +177808,171844 +171844,171844 +177809,171845 +171845,171845 +177810,171846 +171846,171846 +177811,171847 +171847,171847 +177812,171848 +171848,171848 +177813,171849 +171849,171849 +177814,171850 +171850,171850 +120280,171850 +177815,171851 +171851,171851 +177816,171852 +171852,171852 +177817,171853 +171853,171853 +137943,171853 +130356,171853 +177818,171854 +171854,171854 +177819,171855 +171855,171855 +177820,171856 +171856,171856 +177821,171857 +171857,171857 +177822,171858 +171858,171858 +177823,171859 +171859,171859 +177824,171860 +171860,171860 +177825,171861 +171861,171861 +177826,171862 +171862,171862 +177827,171863 +171863,171863 +177828,171864 +171864,171864 +177829,171865 +171865,171865 +177830,171866 +171866,171866 +177831,171867 +171867,171867 +177832,171868 +171868,171868 +177833,171869 +171869,171869 +177834,171870 +171870,171870 +177835,171871 +171871,171871 +177836,171872 +171872,171872 +177837,171873 +171873,171873 +177838,171874 +171874,171874 +177960,171996 +171996,171996 +177961,171997 +171997,171997 +177962,171998 +171998,171998 +108645,171998 +177963,171999 +171999,171999 +178016,172052 +172052,172052 +178017,172053 +172053,172053 +178059,172095 +172095,172095 +178060,172096 +172096,172096 +108449,172096 +211776,172096 +178061,172097 +172097,172097 +178062,172098 +172098,172098 +178063,172099 +172099,172099 +178064,172100 +172100,172100 +178065,172101 +172101,172101 +178066,172102 +172102,172102 +178067,172103 +172103,172103 +178068,172104 +172104,172104 +178069,172105 +172105,172105 +76343,172105 +178070,172106 +172106,172106 +178071,172107 +172107,172107 +178072,172108 +172108,172108 +178073,172109 +172109,172109 +178074,172110 +172110,172110 +178075,172111 +172111,172111 +178076,172112 +172112,172112 +178723,172112 +177438,172112 +171474,172112 +178077,172113 +172113,172113 +178078,172114 +172114,172114 +37052,172114 +178079,172115 +172115,172115 +63790,172115 +178080,172116 +172116,172116 +178081,172117 +172117,172117 +178082,172118 +172118,172118 +178083,172119 +172119,172119 +178084,172120 +172120,172120 +178085,172121 +172121,172121 +216489,172121 +178086,172122 +172122,172122 +178087,172123 +172123,172123 +178088,172124 +172124,172124 +178089,172125 +172125,172125 +178090,172126 +172126,172126 +39909,172126 +178091,172127 +172127,172127 +178092,172128 +172128,172128 +178093,172129 +172129,172129 +178094,172130 +172130,172130 +178095,172131 +172131,172131 +178096,172132 +172132,172132 +178097,172133 +172133,172133 +178098,172134 +172134,172134 +178099,172135 +172135,172135 +178100,172136 +172136,172136 +178101,172137 +172137,172137 +178102,172138 +172138,172138 +178103,172139 +172139,172139 +178104,172140 +172140,172140 +178105,172141 +172141,172141 +178106,172142 +172142,172142 +178107,172143 +172143,172143 +178108,172144 +172144,172144 +178109,172145 +172145,172145 +162585,172145 +123165,172145 +178110,172146 +172146,172146 +178111,172147 +172147,172147 +178112,172148 +172148,172148 +242336,172148 +154209,172148 +178113,172149 +172149,172149 +107548,172149 +178114,172150 +172150,172150 +178115,172151 +172151,172151 +178116,172152 +172152,172152 +178117,172153 +172153,172153 +178118,172154 +172154,172154 +178119,172155 +172155,172155 +178120,172156 +172156,172156 +107555,172156 +178121,172157 +172157,172157 +178122,172158 +172158,172158 +145366,172158 +193723,172158 +178123,172159 +172159,172159 +178124,172160 +172160,172160 +178125,172161 +172161,172161 +178126,172162 +172162,172162 +178127,172163 +172163,172163 +178128,172164 +172164,172164 +178129,172165 +172165,172165 +178130,172166 +172166,172166 +178131,172167 +172167,172167 +178132,172168 +172168,172168 +178133,172169 +172169,172169 +178134,172170 +172170,172170 +178135,172171 +172171,172171 +178136,172172 +172172,172172 +178137,172173 +172173,172173 +178138,172174 +172174,172174 +178139,172175 +172175,172175 +178140,172176 +172176,172176 +67773,172176 +178141,172177 +172177,172177 +178142,172178 +172178,172178 +110760,172178 +178143,172179 +172179,172179 +197543,172179 +178144,172180 +172180,172180 +178145,172181 +172181,172181 +178146,172182 +172182,172182 +178147,172183 +172183,172183 +178148,172184 +172184,172184 +178149,172185 +172185,172185 +178150,172186 +172186,172186 +178151,172187 +172187,172187 +178152,172188 +172188,172188 +178153,172189 +172189,172189 +178154,172190 +172190,172190 +178155,172191 +172191,172191 +178156,172192 +172192,172192 +178157,172193 +172193,172193 +178158,172194 +172194,172194 +178159,172195 +172195,172195 +178160,172196 +172196,172196 +178161,172197 +172197,172197 +178162,172198 +172198,172198 +178163,172199 +172199,172199 +178164,172200 +172200,172200 +178165,172201 +172201,172201 +178166,172202 +172202,172202 +178167,172203 +172203,172203 +178168,172204 +172204,172204 +178169,172205 +172205,172205 +178170,172206 +172206,172206 +178171,172207 +172207,172207 +178172,172208 +172208,172208 +178173,172209 +172209,172209 +178174,172210 +172210,172210 +178175,172211 +172211,172211 +178176,172212 +172212,172212 +178177,172213 +172213,172213 +178178,172214 +172214,172214 +178179,172215 +172215,172215 +178180,172216 +172216,172216 +178181,172217 +172217,172217 +178182,172218 +172218,172218 +178183,172219 +172219,172219 +178184,172220 +172220,172220 +178185,172221 +172221,172221 +178186,172222 +172222,172222 +178187,172223 +172223,172223 +206578,172223 +178188,172224 +172224,172224 +178189,172225 +172225,172225 +178190,172226 +172226,172226 +178191,172227 +172227,172227 +193276,172227 +178192,172228 +172228,172228 +178193,172229 +172229,172229 +178194,172230 +172230,172230 +178195,172231 +172231,172231 +178196,172232 +172232,172232 +178197,172233 +172233,172233 +178198,172234 +172234,172234 +178199,172235 +172235,172235 +178200,172236 +172236,172236 +178201,172237 +172237,172237 +178202,172238 +172238,172238 +178203,172239 +172239,172239 +178204,172240 +172240,172240 +178205,172241 +172241,172241 +178206,172242 +172242,172242 +83661,172242 +178207,172243 +172243,172243 +76394,172243 +178208,172244 +172244,172244 +178209,172245 +172245,172245 +178210,172246 +172246,172246 +178211,172247 +172247,172247 +136308,172247 +128721,172247 +178212,172248 +172248,172248 +178213,172249 +172249,172249 +178214,172250 +172250,172250 +179222,172250 +178215,172251 +172251,172251 +178216,172252 +172252,172252 +178217,172253 +172253,172253 +178218,172254 +172254,172254 +178219,172255 +172255,172255 +178220,172256 +172256,172256 +178221,172257 +172257,172257 +178222,172258 +172258,172258 +178223,172259 +172259,172259 +178224,172260 +172260,172260 +178225,172261 +172261,172261 +178226,172262 +172262,172262 +178227,172263 +172263,172263 +178228,172264 +172264,172264 +178229,172265 +172265,172265 +178230,172266 +172266,172266 +178231,172267 +172267,172267 +178232,172268 +172268,172268 +178233,172269 +172269,172269 +178234,172270 +172270,172270 +178235,172271 +172271,172271 +86869,172271 +178236,172272 +172272,172272 +178237,172273 +172273,172273 +178238,172274 +172274,172274 +178239,172275 +172275,172275 +178240,172276 +172276,172276 +178241,172277 +172277,172277 +178242,172278 +172278,172278 +248675,172278 +178243,172279 +172279,172279 +178244,172280 +172280,172280 +178245,172281 +172281,172281 +178246,172282 +172282,172282 +178247,172283 +172283,172283 +178248,172284 +172284,172284 +178249,172285 +172285,172285 +178250,172286 +172286,172286 +178251,172287 +172287,172287 +178252,172288 +172288,172288 +179950,172288 +178253,172289 +172289,172289 +178254,172290 +172290,172290 +178255,172291 +172291,172291 +178256,172292 +172292,172292 +178257,172293 +172293,172293 +178258,172294 +172294,172294 +178259,172295 +172295,172295 +178260,172296 +172296,172296 +178261,172297 +172297,172297 +178262,172298 +172298,172298 +178263,172299 +172299,172299 +178264,172300 +172300,172300 +178265,172301 +172301,172301 +178266,172302 +172302,172302 +178267,172303 +172303,172303 +178268,172304 +172304,172304 +178269,172305 +172305,172305 +178270,172306 +172306,172306 +219941,172306 +178271,172307 +172307,172307 +106199,172307 +178272,172308 +172308,172308 +178273,172309 +172309,172309 +178274,172310 +172310,172310 +178275,172311 +172311,172311 +178276,172312 +172312,172312 +178277,172313 +172313,172313 +178278,172314 +172314,172314 +178279,172315 +172315,172315 +107490,172315 +233933,172315 +178280,172316 +172316,172316 +178281,172317 +172317,172317 +178282,172318 +172318,172318 +178283,172319 +172319,172319 +178284,172320 +172320,172320 +178285,172321 +172321,172321 +178286,172322 +172322,172322 +178287,172323 +172323,172323 +178288,172324 +172324,172324 +178289,172325 +172325,172325 +178290,172326 +172326,172326 +178291,172327 +172327,172327 +178292,172328 +172328,172328 +178293,172329 +172329,172329 +178294,172330 +172330,172330 +178295,172331 +172331,172331 +178296,172332 +172332,172332 +178297,172333 +172333,172333 +178298,172334 +172334,172334 +178299,172335 +172335,172335 +178300,172336 +172336,172336 +178301,172337 +172337,172337 +178302,172338 +172338,172338 +178303,172339 +172339,172339 +178304,172340 +172340,172340 +178305,172341 +172341,172341 +178306,172342 +172342,172342 +132793,172342 +125206,172342 +178307,172343 +172343,172343 +178308,172344 +172344,172344 +235678,172344 +178309,172345 +172345,172345 +178310,172346 +172346,172346 +77137,172346 +178311,172347 +172347,172347 +178312,172348 +172348,172348 +178313,172349 +172349,172349 +178314,172350 +172350,172350 +178315,172351 +172351,172351 +178316,172352 +172352,172352 +178317,172353 +172353,172353 +178318,172354 +172354,172354 +178319,172355 +172355,172355 +178320,172356 +172356,172356 +178321,172357 +172357,172357 +206298,172357 +178322,172358 +172358,172358 +178323,172359 +172359,172359 +178324,172360 +172360,172360 +178325,172361 +172361,172361 +178326,172362 +172362,172362 +178327,172363 +172363,172363 +5232,172363 +178328,172364 +172364,172364 +178329,172365 +172365,172365 +178330,172366 +172366,172366 +178331,172367 +172367,172367 +189750,172367 +141830,172367 +177489,172367 +171525,172367 +178332,172368 +172368,172368 +24538,172368 +178333,172369 +172369,172369 +178334,172370 +172370,172370 +178335,172371 +172371,172371 +178336,172372 +172372,172372 +178337,172373 +172373,172373 +178338,172374 +172374,172374 +178339,172375 +172375,172375 +178340,172376 +172376,172376 +178341,172377 +172377,172377 +178342,172378 +172378,172378 +178343,172379 +172379,172379 +178344,172380 +172380,172380 +178345,172381 +172381,172381 +211236,172381 +178346,172382 +172382,172382 +178347,172383 +172383,172383 +178348,172384 +172384,172384 +178349,172385 +172385,172385 +220035,172385 +178350,172386 +172386,172386 +178351,172387 +172387,172387 +178352,172388 +172388,172388 +178353,172389 +172389,172389 +178354,172390 +172390,172390 +178355,172391 +172391,172391 +178356,172392 +172392,172392 +178357,172393 +172393,172393 +178358,172394 +172394,172394 +23675,172394 +178359,172395 +172395,172395 +178360,172396 +172396,172396 +178361,172397 +172397,172397 +178362,172398 +172398,172398 +178363,172399 +172399,172399 +178364,172400 +172400,172400 +178365,172401 +172401,172401 +178366,172402 +172402,172402 +178367,172403 +172403,172403 +178368,172404 +172404,172404 +178369,172405 +172405,172405 +178370,172406 +172406,172406 +205969,172406 +178371,172407 +172407,172407 +178372,172408 +172408,172408 +178373,172409 +172409,172409 +178374,172410 +172410,172410 +178375,172411 +172411,172411 +178376,172412 +172412,172412 +178377,172413 +172413,172413 +178378,172414 +172414,172414 +178379,172415 +172415,172415 +178380,172416 +172416,172416 +178381,172417 +172417,172417 +178382,172418 +172418,172418 +178383,172419 +172419,172419 +178384,172420 +172420,172420 +178385,172421 +172421,172421 +178386,172422 +172422,172422 +178387,172423 +172423,172423 +83985,172423 +178388,172424 +172424,172424 +178389,172425 +172425,172425 +178390,172426 +172426,172426 +178391,172427 +172427,172427 +178392,172428 +172428,172428 +178393,172429 +172429,172429 +30487,172429 +178394,172430 +172430,172430 +178395,172431 +172431,172431 +178396,172432 +172432,172432 +178397,172433 +172433,172433 +36456,172433 +178465,172501 +172501,172501 +178466,172502 +172502,172502 +178467,172503 +172503,172503 +178468,172504 +172504,172504 +178469,172505 +172505,172505 +178470,172506 +172506,172506 +178471,172507 +172507,172507 +178472,172508 +172508,172508 +178473,172509 +172509,172509 +178474,172510 +172510,172510 +178475,172511 +172511,172511 +178476,172512 +172512,172512 +178477,172513 +172513,172513 +187294,172513 +178478,172514 +172514,172514 +178479,172515 +172515,172515 +178480,172516 +172516,172516 +178481,172517 +172517,172517 +178482,172518 +172518,172518 +178483,172519 +172519,172519 +178484,172520 +172520,172520 +178485,172521 +172521,172521 +50894,172521 +42790,172521 +178486,172522 +172522,172522 +178487,172523 +172523,172523 +178488,172524 +172524,172524 +178489,172525 +172525,172525 +178490,172526 +172526,172526 +178491,172527 +172527,172527 +178492,172528 +172528,172528 +178493,172529 +172529,172529 +36612,172529 +30474,172529 +178494,172530 +172530,172530 +178495,172531 +172531,172531 +178496,172532 +172532,172532 +178497,172533 +172533,172533 +178498,172534 +172534,172534 +178499,172535 +172535,172535 +178500,172536 +172536,172536 +178501,172537 +172537,172537 +178502,172538 +172538,172538 +178503,172539 +172539,172539 +178504,172540 +172540,172540 +178505,172541 +172541,172541 +178506,172542 +172542,172542 +178507,172543 +172543,172543 +178508,172544 +172544,172544 +178509,172545 +172545,172545 +204011,172545 +178510,172546 +172546,172546 +178511,172547 +172547,172547 +178512,172548 +172548,172548 +178513,172549 +172549,172549 +178514,172550 +172550,172550 +178515,172551 +172551,172551 +178516,172552 +172552,172552 +178517,172553 +172553,172553 +178518,172554 +172554,172554 +178519,172555 +172555,172555 +178520,172556 +172556,172556 +178521,172557 +172557,172557 +80441,172557 +178522,172558 +172558,172558 +178523,172559 +172559,172559 +165219,172559 +178524,172560 +172560,172560 +178525,172561 +172561,172561 +178526,172562 +172562,172562 +178527,172563 +172563,172563 +178528,172564 +172564,172564 +50764,172564 +42660,172564 +178529,172565 +172565,172565 +178530,172566 +172566,172566 +178531,172567 +172567,172567 +178532,172568 +172568,172568 +178533,172569 +172569,172569 +178534,172570 +172570,172570 +178535,172571 +172571,172571 +178536,172572 +172572,172572 +178537,172573 +172573,172573 +178538,172574 +172574,172574 +178539,172575 +172575,172575 +178540,172576 +172576,172576 +178541,172577 +172577,172577 +178542,172578 +172578,172578 +178543,172579 +172579,172579 +178544,172580 +172580,172580 +178545,172581 +172581,172581 +178546,172582 +172582,172582 +178547,172583 +172583,172583 +178548,172584 +172584,172584 +178549,172585 +172585,172585 +178550,172586 +172586,172586 +178551,172587 +172587,172587 +178552,172588 +172588,172588 +178553,172589 +172589,172589 +178554,172590 +172590,172590 +178555,172591 +172591,172591 +178556,172592 +172592,172592 +178557,172593 +172593,172593 +178558,172594 +172594,172594 +178559,172595 +172595,172595 +178560,172596 +172596,172596 +178561,172597 +172597,172597 +178562,172598 +172598,172598 +178563,172599 +172599,172599 +65859,172599 +178564,172600 +172600,172600 +178565,172601 +172601,172601 +178566,172602 +172602,172602 +178567,172603 +172603,172603 +178568,172604 +172604,172604 +173894,173894 +167930,173894 +173895,173895 +167931,173895 +173896,173896 +167932,173896 +173897,173897 +167933,173897 +173898,173898 +167934,173898 +173899,173899 +167935,173899 +88260,173899 +173900,173900 +167936,173900 +173901,173901 +167937,173901 +173902,173902 +167938,173902 +112038,173902 +173903,173903 +167939,173903 +173904,173904 +167940,173904 +173905,173905 +167941,173905 +173906,173906 +167942,173906 +173907,173907 +167943,173907 +173908,173908 +167944,173908 +173909,173909 +167945,173909 +173910,173910 +167946,173910 +173911,173911 +167947,173911 +173912,173912 +167948,173912 +173913,173913 +167949,173913 +173914,173914 +167950,173914 +173915,173915 +167951,173915 +150880,173915 +173916,173916 +167952,173916 +206649,173916 +173917,173917 +167953,173917 +173918,173918 +167954,173918 +173919,173919 +167955,173919 +173920,173920 +167956,173920 +173921,173921 +167957,173921 +85781,173921 +173922,173922 +167958,173922 +173923,173923 +167959,173923 +173924,173924 +167960,173924 +173925,173925 +167961,173925 +83022,173925 +173926,173926 +167962,173926 +173927,173927 +167963,173927 +93263,173927 +173968,173968 +168004,173968 +107055,173968 +173969,173969 +168005,173969 +173970,173970 +168006,173970 +193711,173970 +173971,173971 +168007,173971 +216965,173971 +173972,173972 +168008,173972 +173973,173973 +168009,173973 +14548,173973 +173974,173974 +168010,173974 +173975,173975 +168011,173975 +173976,173976 +168012,173976 +173977,173977 +168013,173977 +173978,173978 +168014,173978 +173979,173979 +168015,173979 +173980,173980 +168016,173980 +173981,173981 +168017,173981 +173982,173982 +168018,173982 +173983,173983 +168019,173983 +173984,173984 +168020,173984 +173985,173985 +168021,173985 +173986,173986 +168022,173986 +173987,173987 +168023,173987 +110617,173987 +173988,173988 +168024,173988 +173989,173989 +168025,173989 +173990,173990 +168026,173990 +173991,173991 +168027,173991 +173992,173992 +168028,173992 +173993,173993 +168029,173993 +173994,173994 +168030,173994 +173995,173995 +168031,173995 +173996,173996 +168032,173996 +173997,173997 +168033,173997 +173998,173998 +168034,173998 +67269,173998 +173999,173999 +168035,173999 +174000,174000 +168036,174000 +174001,174001 +168037,174001 +174002,174002 +168038,174002 +174003,174003 +168039,174003 +174004,174004 +168040,174004 +174005,174005 +168041,174005 +174006,174006 +168042,174006 +183573,174006 +154114,174006 +174007,174007 +168043,174007 +174008,174008 +168044,174008 +174009,174009 +168045,174009 +174010,174010 +168046,174010 +174011,174011 +168047,174011 +174012,174012 +168048,174012 +174013,174013 +168049,174013 +174014,174014 +168050,174014 +174015,174015 +168051,174015 +174016,174016 +168052,174016 +174017,174017 +168053,174017 +174018,174018 +168054,174018 +174019,174019 +168055,174019 +174020,174020 +168056,174020 +174021,174021 +168057,174021 +174022,174022 +168058,174022 +174023,174023 +168059,174023 +174024,174024 +168060,174024 +174025,174025 +168061,174025 +174026,174026 +168062,174026 +174027,174027 +168063,174027 +174155,174155 +168191,174155 +174156,174156 +168192,174156 +174157,174157 +168193,174157 +174158,174158 +168194,174158 +174159,174159 +168195,174159 +174160,174160 +168196,174160 +174161,174161 +168197,174161 +174162,174162 +168198,174162 +174163,174163 +168199,174163 +174164,174164 +168200,174164 +174165,174165 +168201,174165 +63403,174165 +227703,174165 +19003,174165 +174166,174166 +168202,174166 +174167,174167 +168203,174167 +174168,174168 +168204,174168 +174169,174169 +168205,174169 +174170,174170 +168206,174170 +174171,174171 +168207,174171 +224774,174171 +174172,174172 +168208,174172 +174173,174173 +168209,174173 +174174,174174 +168210,174174 +174175,174175 +168211,174175 +174176,174176 +168212,174176 +174177,174177 +168213,174177 +6370,174177 +174178,174178 +168214,174178 +174179,174179 +168215,174179 +174180,174180 +168216,174180 +174181,174181 +168217,174181 +174182,174182 +168218,174182 +174183,174183 +168219,174183 +174184,174184 +168220,174184 +174185,174185 +168221,174185 +174186,174186 +168222,174186 +174187,174187 +168223,174187 +174255,174255 +168291,174255 +174256,174256 +168292,174256 +246601,174256 +174257,174257 +168293,174257 +174258,174258 +168294,174258 +174259,174259 +168295,174259 +174260,174260 +168296,174260 +174261,174261 +168297,174261 +174262,174262 +168298,174262 +174263,174263 +168299,174263 +174264,174264 +168300,174264 +174265,174265 +168301,174265 +174266,174266 +168302,174266 +174267,174267 +168303,174267 +174268,174268 +168304,174268 +174269,174269 +168305,174269 +174270,174270 +168306,174270 +174271,174271 +168307,174271 +174272,174272 +168308,174272 +174273,174273 +168309,174273 +174274,174274 +168310,174274 +174275,174275 +168311,174275 +174276,174276 +168312,174276 +174277,174277 +168313,174277 +174278,174278 +168314,174278 +174279,174279 +168315,174279 +174280,174280 +168316,174280 +174281,174281 +168317,174281 +174282,174282 +168318,174282 +174283,174283 +168319,174283 +174284,174284 +168320,174284 +174285,174285 +168321,174285 +174286,174286 +168322,174286 +174287,174287 +168323,174287 +174288,174288 +168324,174288 +174289,174289 +168325,174289 +174290,174290 +168326,174290 +174291,174291 +168327,174291 +174292,174292 +168328,174292 +174293,174293 +168329,174293 +174294,174294 +168330,174294 +174295,174295 +168331,174295 +174296,174296 +168332,174296 +174297,174297 +168333,174297 +174298,174298 +168334,174298 +174299,174299 +168335,174299 +96028,174299 +174300,174300 +168336,174300 +174336,174336 +168372,174336 +174337,174337 +168373,174337 +174338,174338 +168374,174338 +100590,174338 +174339,174339 +168375,174339 +174340,174340 +168376,174340 +174341,174341 +168377,174341 +174342,174342 +168378,174342 +174343,174343 +168379,174343 +174344,174344 +168380,174344 +174345,174345 +168381,174345 +174346,174346 +168382,174346 +174347,174347 +168383,174347 +174348,174348 +168384,174348 +174349,174349 +168385,174349 +165081,174349 +5612,174349 +174350,174350 +168386,174350 +174351,174351 +168387,174351 +174352,174352 +168388,174352 +174353,174353 +168389,174353 +174354,174354 +168390,174354 +174355,174355 +168391,174355 +174356,174356 +168392,174356 +60924,174356 +174357,174357 +168393,174357 +9081,174357 +174358,174358 +168394,174358 +174359,174359 +168395,174359 +174360,174360 +168396,174360 +174361,174361 +168397,174361 +174362,174362 +168398,174362 +114878,174362 +174363,174363 +168399,174363 +174364,174364 +168400,174364 +84663,174364 +174365,174365 +168401,174365 +174366,174366 +168402,174366 +77327,174366 +174367,174367 +168403,174367 +174368,174368 +168404,174368 +174369,174369 +168405,174369 +174472,174472 +168508,174472 +174473,174473 +168509,174473 +174474,174474 +168510,174474 +174475,174475 +168511,174475 +174476,174476 +168512,174476 +174477,174477 +168513,174477 +174478,174478 +168514,174478 +174606,174606 +168642,174606 +174607,174607 +168643,174607 +174608,174608 +168644,174608 +174609,174609 +168645,174609 +174610,174610 +168646,174610 +174611,174611 +168647,174611 +174612,174612 +168648,174612 +174613,174613 +168649,174613 +174614,174614 +168650,174614 +191059,174614 +174615,174615 +168651,174615 +174616,174616 +168652,174616 +174617,174617 +168653,174617 +174618,174618 +168654,174618 +174619,174619 +168655,174619 +174620,174620 +168656,174620 +174621,174621 +168657,174621 +174622,174622 +168658,174622 +174623,174623 +168659,174623 +174624,174624 +168660,174624 +174625,174625 +168661,174625 +174626,174626 +168662,174626 +174627,174627 +168663,174627 +174628,174628 +168664,174628 +174629,174629 +168665,174629 +174630,174630 +168666,174630 +57998,174630 +174631,174631 +168667,174631 +174632,174632 +168668,174632 +174633,174633 +168669,174633 +174634,174634 +168670,174634 +174635,174635 +168671,174635 +174636,174636 +168672,174636 +174637,174637 +168673,174637 +174638,174638 +168674,174638 +174639,174639 +168675,174639 +39790,174639 +174640,174640 +168676,174640 +174641,174641 +168677,174641 +174642,174642 +168678,174642 +174643,174643 +168679,174643 +174644,174644 +168680,174644 +174645,174645 +168681,174645 +175994,175994 +170030,175994 +175995,175995 +170031,175995 +175996,175996 +170032,175996 +175997,175997 +170033,175997 +175998,175998 +170034,175998 +175999,175999 +170035,175999 +176000,176000 +170036,176000 +176001,176001 +170037,176001 +176002,176002 +170038,176002 +176003,176003 +170039,176003 +176004,176004 +170040,176004 +176005,176005 +170041,176005 +107095,176005 +176006,176006 +170042,176006 +176007,176007 +170043,176007 +176008,176008 +170044,176008 +176009,176009 +170045,176009 +176010,176010 +170046,176010 +176011,176011 +170047,176011 +176012,176012 +170048,176012 +176013,176013 +170049,176013 +176014,176014 +170050,176014 +176230,176230 +170266,176230 +176231,176231 +170267,176231 +176232,176232 +170268,176232 +176233,176233 +170269,176233 +176234,176234 +170270,176234 +176235,176235 +170271,176235 +176236,176236 +170272,176236 +176237,176237 +170273,176237 +140288,176237 +176238,176238 +170274,176238 +176239,176239 +170275,176239 +176240,176240 +170276,176240 +176241,176241 +170277,176241 +51819,176241 +43715,176241 +176242,176242 +170278,176242 +176243,176243 +170279,176243 +176244,176244 +170280,176244 +176245,176245 +170281,176245 +176246,176246 +170282,176246 +176247,176247 +170283,176247 +176248,176248 +170284,176248 +176249,176249 +170285,176249 +176250,176250 +170286,176250 +176251,176251 +170287,176251 +176252,176252 +170288,176252 +176253,176253 +170289,176253 +176254,176254 +170290,176254 +176255,176255 +170291,176255 +176256,176256 +170292,176256 +176257,176257 +170293,176257 +176258,176258 +170294,176258 +176259,176259 +170295,176259 +176260,176260 +170296,176260 +176261,176261 +170297,176261 +176262,176262 +170298,176262 +176263,176263 +170299,176263 +176264,176264 +170300,176264 +144284,176264 +176265,176265 +170301,176265 +176266,176266 +170302,176266 +176267,176267 +170303,176267 +176268,176268 +170304,176268 +176269,176269 +170305,176269 +176270,176270 +170306,176270 +176271,176271 +170307,176271 +176272,176272 +170308,176272 +176273,176273 +170309,176273 +176274,176274 +170310,176274 +176275,176275 +170311,176275 +176276,176276 +170312,176276 +176277,176277 +170313,176277 +176278,176278 +170314,176278 +103613,176278 +176279,176279 +170315,176279 +176280,176280 +170316,176280 +176281,176281 +170317,176281 +176282,176282 +170318,176282 +176283,176283 +170319,176283 +115968,176283 +176284,176284 +170320,176284 +15382,176284 +96471,176284 +176285,176285 +170321,176285 +15383,176285 +9458,176285 +176286,176286 +170322,176286 +176287,176287 +170323,176287 +176288,176288 +170324,176288 +176289,176289 +170325,176289 +176410,176410 +170446,176410 +176411,176411 +170447,176411 +176412,176412 +170448,176412 +176413,176413 +170449,176413 +176414,176414 +170450,176414 +14654,176414 +176415,176415 +170451,176415 +176416,176416 +170452,176416 +176417,176417 +170453,176417 +176418,176418 +170454,176418 +176419,176419 +170455,176419 +176420,176420 +170456,176420 +176421,176421 +170457,176421 +176422,176422 +170458,176422 +176423,176423 +170459,176423 +176424,176424 +170460,176424 +176425,176425 +170461,176425 +176426,176426 +170462,176426 +176427,176427 +170463,176427 +205454,176428 +176428,176428 +170464,176428 +176429,176429 +170465,176429 +176430,176430 +170466,176430 +176431,176431 +170467,176431 +176432,176432 +170468,176432 +176433,176433 +170469,176433 +176434,176434 +170470,176434 +176435,176435 +170471,176435 +176436,176436 +170472,176436 +176437,176437 +170473,176437 +176438,176438 +170474,176438 +176439,176439 +170475,176439 +176440,176440 +170476,176440 +176441,176441 +170477,176441 +176442,176442 +170478,176442 +176443,176443 +170479,176443 +176444,176444 +170480,176444 +176445,176445 +170481,176445 +176446,176446 +170482,176446 +176447,176447 +170483,176447 +176448,176448 +170484,176448 +176449,176449 +170485,176449 +176450,176450 +170486,176450 +176451,176451 +170487,176451 +176452,176452 +170488,176452 +176453,176453 +170489,176453 +176454,176454 +170490,176454 +176455,176455 +170491,176455 +176456,176456 +170492,176456 +176457,176457 +170493,176457 +176458,176458 +170494,176458 +176459,176459 +170495,176459 +176460,176460 +170496,176460 +176461,176461 +170497,176461 +176462,176462 +170498,176462 +176463,176463 +170499,176463 +241592,176463 +176464,176464 +170500,176464 +176465,176465 +170501,176465 +176466,176466 +170502,176466 +176467,176467 +170503,176467 +176468,176468 +170504,176468 +68275,176468 +176469,176469 +170505,176469 +82952,176469 +176470,176470 +170506,176470 +176471,176471 +170507,176471 +176472,176472 +170508,176472 +176473,176473 +170509,176473 +176474,176474 +170510,176474 +176475,176475 +170511,176475 +176476,176476 +170512,176476 +176477,176477 +170513,176477 +57687,176477 +176478,176478 +170514,176478 +176479,176479 +170515,176479 +176480,176480 +170516,176480 +176481,176481 +170517,176481 +140484,176481 +176482,176482 +170518,176482 +176483,176483 +170519,176483 +176484,176484 +170520,176484 +176485,176485 +170521,176485 +176486,176486 +170522,176486 +176487,176487 +170523,176487 +176488,176488 +170524,176488 +176489,176489 +170525,176489 +176490,176490 +170526,176490 +176491,176491 +170527,176491 +176492,176492 +170528,176492 +176493,176493 +170529,176493 +176494,176494 +170530,176494 +176495,176495 +170531,176495 +176496,176496 +170532,176496 +157534,176496 +176497,176497 +170533,176497 +176498,176498 +170534,176498 +176499,176499 +170535,176499 +176500,176500 +170536,176500 +176501,176501 +170537,176501 +87352,176501 +176502,176502 +170538,176502 +176503,176503 +170539,176503 +176504,176504 +170540,176504 +176505,176505 +170541,176505 +176531,176531 +170567,176531 +176532,176532 +170568,176532 +176533,176533 +170569,176533 +176534,176534 +170570,176534 +176535,176535 +170571,176535 +176536,176536 +170572,176536 +51807,176536 +43703,176536 +176537,176537 +170573,176537 +135433,176537 +127846,176537 +7241,176537 +176538,176538 +170574,176538 +176539,176539 +170575,176539 +176540,176540 +170576,176540 +88499,176540 +176541,176541 +170577,176541 +176542,176542 +170578,176542 +233823,176542 +176543,176543 +170579,176543 +176544,176544 +170580,176544 +176545,176545 +170581,176545 +176546,176546 +170582,176546 +176547,176547 +170583,176547 +176548,176548 +170584,176548 +176549,176549 +170585,176549 +176550,176550 +170586,176550 +59998,176550 +176551,176551 +170587,176551 +146721,176551 +176552,176552 +170588,176552 +176553,176553 +170589,176553 +176554,176554 +170590,176554 +246607,176554 +176555,176555 +170591,176555 +176556,176556 +170592,176556 +213770,176557 +176557,176557 +170593,176557 +176558,176558 +170594,176558 +176559,176559 +170595,176559 +176560,176560 +170596,176560 +176561,176561 +170597,176561 +176562,176562 +170598,176562 +188242,176562 +176563,176563 +170599,176563 +176564,176564 +170600,176564 +176565,176565 +170601,176565 +176566,176566 +170602,176566 +176567,176567 +170603,176567 +176568,176568 +170604,176568 +176569,176569 +170605,176569 +176570,176570 +170606,176570 +176571,176571 +170607,176571 +176572,176572 +170608,176572 +176573,176573 +170609,176573 +109267,176573 +176574,176574 +170610,176574 +176575,176575 +170611,176575 +176576,176576 +170612,176576 +176577,176577 +170613,176577 +176578,176578 +170614,176578 +176579,176579 +170615,176579 +176580,176580 +170616,176580 +176581,176581 +170617,176581 +176582,176582 +170618,176582 +176583,176583 +170619,176583 +176584,176584 +170620,176584 +176585,176585 +170621,176585 +176586,176586 +170622,176586 +86910,176586 +176587,176587 +170623,176587 +176588,176588 +170624,176588 +176589,176589 +170625,176589 +176590,176590 +170626,176590 +110606,176590 +176591,176591 +170627,176591 +6317,176591 +176592,176592 +170628,176592 +176593,176593 +170629,176593 +176594,176594 +170630,176594 +176595,176595 +170631,176595 +176596,176596 +170632,176596 +49140,176596 +40948,176596 +176597,176597 +170633,176597 +176598,176598 +170634,176598 +176599,176599 +170635,176599 +176600,176600 +170636,176600 +176601,176601 +170637,176601 +176602,176602 +170638,176602 +176603,176603 +170639,176603 +176604,176604 +170640,176604 +176605,176605 +170641,176605 +233668,176605 +176606,176606 +170642,176606 +176607,176607 +170643,176607 +158794,176607 +176608,176608 +170644,176608 +176609,176609 +170645,176609 +176610,176610 +170646,176610 +176611,176611 +170647,176611 +176612,176612 +170648,176612 +176613,176613 +170649,176613 +176614,176614 +170650,176614 +176615,176615 +170651,176615 +176616,176616 +170652,176616 +176617,176617 +170653,176617 +176618,176618 +170654,176618 +176619,176619 +170655,176619 +176620,176620 +170656,176620 +176621,176621 +170657,176621 +176622,176622 +170658,176622 +176623,176623 +170659,176623 +176624,176624 +170660,176624 +176625,176625 +170661,176625 +176626,176626 +170662,176626 +176627,176627 +170663,176627 +191868,176627 +176628,176628 +170664,176628 +176629,176629 +170665,176629 +176630,176630 +170666,176630 +176631,176631 +170667,176631 +191851,176631 +176632,176632 +170668,176632 +176633,176633 +170669,176633 +176634,176634 +170670,176634 +176635,176635 +170671,176635 +176636,176636 +170672,176636 +176637,176637 +170673,176637 +176638,176638 +170674,176638 +176639,176639 +170675,176639 +176640,176640 +170676,176640 +176641,176641 +170677,176641 +176642,176642 +170678,176642 +79356,176642 +176643,176643 +170679,176643 +176644,176644 +170680,176644 +58247,176644 +176645,176645 +170681,176645 +176646,176646 +170682,176646 +176647,176647 +170683,176647 +176648,176648 +170684,176648 +176649,176649 +170685,176649 +176650,176650 +170686,176650 +176651,176651 +170687,176651 +176652,176652 +170688,176652 +176653,176653 +170689,176653 +176654,176654 +170690,176654 +176655,176655 +170691,176655 +176656,176656 +170692,176656 +176657,176657 +170693,176657 +176658,176658 +170694,176658 +176659,176659 +170695,176659 +176660,176660 +170696,176660 +204556,176660 +176661,176661 +170697,176661 +176662,176662 +170698,176662 +208875,176662 +176663,176663 +170699,176663 +176664,176664 +170700,176664 +176665,176665 +170701,176665 +176666,176666 +170702,176666 +176667,176667 +170703,176667 +176668,176668 +170704,176668 +176669,176669 +170705,176669 +176670,176670 +170706,176670 +176671,176671 +170707,176671 +176672,176672 +170708,176672 +176673,176673 +170709,176673 +176674,176674 +170710,176674 +176675,176675 +170711,176675 +135926,176675 +128339,176675 +176676,176676 +170712,176676 +176677,176677 +170713,176677 +176678,176678 +170714,176678 +176679,176679 +170715,176679 +176680,176680 +170716,176680 +176681,176681 +170717,176681 +176682,176682 +170718,176682 +176683,176683 +170719,176683 +36686,176683 +176684,176684 +170720,176684 +176685,176685 +170721,176685 +176686,176686 +170722,176686 +176687,176687 +170723,176687 +176688,176688 +170724,176688 +176689,176689 +170725,176689 +176690,176690 +170726,176690 +176691,176691 +170727,176691 +176692,176692 +170728,176692 +176693,176693 +170729,176693 +176694,176694 +170730,176694 +176695,176695 +170731,176695 +176696,176696 +170732,176696 +176697,176697 +170733,176697 +176698,176698 +170734,176698 +176699,176699 +170735,176699 +176700,176700 +170736,176700 +176701,176701 +170737,176701 +176702,176702 +170738,176702 +176703,176703 +170739,176703 +176704,176704 +170740,176704 +176705,176705 +170741,176705 +176706,176706 +170742,176706 +176707,176707 +170743,176707 +176708,176708 +170744,176708 +176709,176709 +170745,176709 +6508,176709 +176710,176710 +170746,176710 +176711,176711 +170747,176711 +176712,176712 +170748,176712 +176713,176713 +170749,176713 +176714,176714 +170750,176714 +176715,176715 +170751,176715 +176716,176716 +170752,176716 +176717,176717 +170753,176717 +176718,176718 +170754,176718 +176719,176719 +170755,176719 +176720,176720 +170756,176720 +176721,176721 +170757,176721 +176722,176722 +170758,176722 +176723,176723 +170759,176723 +162433,176723 +176724,176724 +170760,176724 +176725,176725 +170761,176725 +5339,176725 +176726,176726 +170762,176726 +5304,176726 +176727,176727 +170763,176727 +176728,176728 +170764,176728 +176729,176729 +170765,176729 +110707,176729 +176730,176730 +170766,176730 +176731,176731 +170767,176731 +176732,176732 +170768,176732 +176733,176733 +170769,176733 +176734,176734 +170770,176734 +176735,176735 +170771,176735 +176736,176736 +170772,176736 +176737,176737 +170773,176737 +176738,176738 +170774,176738 +176739,176739 +170775,176739 +176740,176740 +170776,176740 +176741,176741 +170777,176741 +176742,176742 +170778,176742 +176743,176743 +170779,176743 +176744,176744 +170780,176744 +176745,176745 +170781,176745 +176746,176746 +170782,176746 +189795,176746 +176747,176747 +170783,176747 +176748,176748 +170784,176748 +176749,176749 +170785,176749 +176750,176750 +170786,176750 +176751,176751 +170787,176751 +176752,176752 +170788,176752 +176753,176753 +170789,176753 +176754,176754 +170790,176754 +166479,176754 +176755,176755 +170791,176755 +176756,176756 +170792,176756 +154714,176756 +176757,176757 +170793,176757 +176758,176758 +170794,176758 +176759,176759 +170795,176759 +176760,176760 +170796,176760 +176761,176761 +170797,176761 +176762,176762 +170798,176762 +176763,176763 +170799,176763 +176764,176764 +170800,176764 +222147,176764 +176765,176765 +170801,176765 +176766,176766 +170802,176766 +176767,176767 +170803,176767 +176768,176768 +170804,176768 +176769,176769 +170805,176769 +233737,176769 +176770,176770 +170806,176770 +176771,176771 +170807,176771 +176772,176772 +170808,176772 +176773,176773 +170809,176773 +188047,176773 +176774,176774 +170810,176774 +176775,176775 +170811,176775 +176776,176776 +170812,176776 +176777,176777 +170813,176777 +176778,176778 +170814,176778 +13484,176778 +176779,176779 +170815,176779 +176780,176780 +170816,176780 +176781,176781 +170817,176781 +176782,176782 +170818,176782 +176783,176783 +170819,176783 +176784,176784 +170820,176784 +176785,176785 +170821,176785 +176786,176786 +170822,176786 +89206,176786 +176787,176787 +170823,176787 +176788,176788 +170824,176788 +176789,176789 +170825,176789 +176790,176790 +170826,176790 +176791,176791 +170827,176791 +176792,176792 +170828,176792 +176793,176793 +170829,176793 +176794,176794 +170830,176794 +176795,176795 +170831,176795 +176796,176796 +170832,176796 +177418,177418 +171454,177418 +165204,177418 +177419,177419 +171455,177419 +177420,177420 +171456,177420 +177421,177421 +171457,177421 +177422,177422 +171458,177422 +146572,177422 +177423,177423 +171459,177423 +81451,177423 +177424,177424 +171460,177424 +177425,177425 +171461,177425 +177426,177426 +171462,177426 +177427,177427 +171463,177427 +165235,177427 +177428,177428 +171464,177428 +177429,177429 +171465,177429 +177430,177430 +171466,177430 +177431,177431 +171467,177431 +177432,177432 +171468,177432 +177433,177433 +171469,177433 +177434,177434 +171470,177434 +177435,177435 +171471,177435 +177436,177436 +171472,177436 +177437,177437 +171473,177437 +177438,177438 +171474,177438 +178076,177438 +172112,177438 +177476,177476 +171512,177476 +177477,177477 +171513,177477 +177478,177478 +171514,177478 +108695,177478 +177479,177479 +171515,177479 +177480,177480 +171516,177480 +177481,177481 +171517,177481 +177482,177482 +171518,177482 +177483,177483 +171519,177483 +177484,177484 +171520,177484 +177485,177485 +171521,177485 +177486,177486 +171522,177486 +177487,177487 +171523,177487 +177488,177488 +171524,177488 +225824,177488 +177489,177489 +171525,177489 +178331,177489 +172367,177489 +177490,177490 +171526,177490 +177491,177491 +171527,177491 +177492,177492 +171528,177492 +177493,177493 +171529,177493 +177494,177494 +171530,177494 +177495,177495 +171531,177495 +114753,177495 +177496,177496 +171532,177496 +177497,177497 +171533,177497 +177498,177498 +171534,177498 +177499,177499 +171535,177499 +177500,177500 +171536,177500 +106707,177500 +177501,177501 +171537,177501 +177502,177502 +171538,177502 +194168,177502 +177503,177503 +171539,177503 +6900,177503 +177504,177504 +171540,177504 +33928,177504 +177505,177505 +171541,177505 +177506,177506 +171542,177506 +177507,177507 +171543,177507 +177508,177508 +171544,177508 +177509,177509 +171545,177509 +73750,177509 +177510,177510 +171546,177510 +177511,177511 +171547,177511 +177512,177512 +171548,177512 +177513,177513 +171549,177513 +177514,177514 +171550,177514 +177515,177515 +171551,177515 +177516,177516 +171552,177516 +177517,177517 +171553,177517 +177518,177518 +171554,177518 +177519,177519 +171555,177519 +177520,177520 +171556,177520 +203595,177520 +177521,177521 +171557,177521 +177522,177522 +171558,177522 +177523,177523 +171559,177523 +177524,177524 +171560,177524 +177525,177525 +171561,177525 +177526,177526 +171562,177526 +149755,177526 +177527,177527 +171563,177527 +177528,177528 +171564,177528 +177529,177529 +171565,177529 +79573,177529 +177530,177530 +171566,177530 +177531,177531 +171567,177531 +177532,177532 +171568,177532 +177533,177533 +171569,177533 +177534,177534 +171570,177534 +177535,177535 +171571,177535 +177536,177536 +171572,177536 +177537,177537 +171573,177537 +177538,177538 +171574,177538 +177539,177539 +171575,177539 +177540,177540 +171576,177540 +177541,177541 +171577,177541 +177542,177542 +171578,177542 +67020,177542 +177543,177543 +171579,177543 +177544,177544 +171580,177544 +177545,177545 +171581,177545 +177546,177546 +171582,177546 +177547,177547 +171583,177547 +177548,177548 +171584,177548 +177549,177549 +171585,177549 +177550,177550 +171586,177550 +177551,177551 +171587,177551 +177552,177552 +171588,177552 +177553,177553 +171589,177553 +177554,177554 +171590,177554 +177555,177555 +171591,177555 +177556,177556 +171592,177556 +177557,177557 +171593,177557 +177558,177558 +171594,177558 +177559,177559 +171595,177559 +177560,177560 +171596,177560 +177561,177561 +171597,177561 +177562,177562 +171598,177562 +177563,177563 +171599,177563 +177564,177564 +171600,177564 +177565,177565 +171601,177565 +177566,177566 +171602,177566 +177567,177567 +171603,177567 +177568,177568 +171604,177568 +177569,177569 +171605,177569 +177570,177570 +171606,177570 +177571,177571 +171607,177571 +177572,177572 +171608,177572 +177573,177573 +171609,177573 +177574,177574 +171610,177574 +177575,177575 +171611,177575 +177576,177576 +171612,177576 +177577,177577 +171613,177577 +177578,177578 +171614,177578 +177579,177579 +171615,177579 +177580,177580 +171616,177580 +177581,177581 +171617,177581 +104400,177581 +177582,177582 +171618,177582 +177583,177583 +171619,177583 +177584,177584 +171620,177584 +177585,177585 +171621,177585 +187006,177585 +177586,177586 +171622,177586 +177587,177587 +171623,177587 +177588,177588 +171624,177588 +177589,177589 +171625,177589 +177590,177590 +171626,177590 +177591,177591 +171627,177591 +177592,177592 +171628,177592 +177593,177593 +171629,177593 +177594,177594 +171630,177594 +177595,177595 +171631,177595 +177596,177596 +171632,177596 +177597,177597 +171633,177597 +177598,177598 +171634,177598 +177599,177599 +171635,177599 +177600,177600 +171636,177600 +177601,177601 +171637,177601 +177602,177602 +171638,177602 +177603,177603 +171639,177603 +177604,177604 +171640,177604 +177605,177605 +171641,177605 +177606,177606 +171642,177606 +177607,177607 +171643,177607 +177608,177608 +171644,177608 +177609,177609 +171645,177609 +177610,177610 +171646,177610 +177611,177611 +171647,177611 +177612,177612 +171648,177612 +177613,177613 +171649,177613 +177614,177614 +171650,177614 +177615,177615 +171651,177615 +177616,177616 +171652,177616 +177617,177617 +171653,177617 +177618,177618 +171654,177618 +177619,177619 +171655,177619 +177620,177620 +171656,177620 +177621,177621 +171657,177621 +177622,177622 +171658,177622 +177623,177623 +171659,177623 +177624,177624 +171660,177624 +177625,177625 +171661,177625 +177626,177626 +171662,177626 +177627,177627 +171663,177627 +177628,177628 +171664,177628 +212097,177628 +177629,177629 +171665,177629 +177630,177630 +171666,177630 +202306,177630 +177631,177631 +171667,177631 +208631,177631 +177632,177632 +171668,177632 +177633,177633 +171669,177633 +177634,177634 +171670,177634 +177635,177635 +171671,177635 +177636,177636 +171672,177636 +177637,177637 +171673,177637 +177638,177638 +171674,177638 +177639,177639 +171675,177639 +177640,177640 +171676,177640 +177641,177641 +171677,177641 +177642,177642 +171678,177642 +177643,177643 +171679,177643 +177644,177644 +171680,177644 +177645,177645 +171681,177645 +180542,177645 +177646,177646 +171682,177646 +177647,177647 +171683,177647 +177648,177648 +171684,177648 +177649,177649 +171685,177649 +177650,177650 +171686,177650 +211680,177650 +177651,177651 +171687,177651 +177652,177652 +171688,177652 +177653,177653 +171689,177653 +177654,177654 +171690,177654 +177655,177655 +171691,177655 +177656,177656 +171692,177656 +177657,177657 +171693,177657 +177658,177658 +171694,177658 +177659,177659 +171695,177659 +177660,177660 +171696,177660 +177661,177661 +171697,177661 +177662,177662 +171698,177662 +177663,177663 +171699,177663 +177664,177664 +171700,177664 +177665,177665 +171701,177665 +177666,177666 +171702,177666 +177667,177667 +171703,177667 +177668,177668 +171704,177668 +177669,177669 +171705,177669 +177670,177670 +171706,177670 +177671,177671 +171707,177671 +177672,177672 +171708,177672 +177673,177673 +171709,177673 +177674,177674 +171710,177674 +148999,177674 +53388,177674 +45284,177674 +177675,177675 +171711,177675 +177676,177676 +171712,177676 +177677,177677 +171713,177677 +177678,177678 +171714,177678 +177679,177679 +171715,177679 +177680,177680 +171716,177680 +177681,177681 +171717,177681 +177682,177682 +171718,177682 +177683,177683 +171719,177683 +177684,177684 +171720,177684 +177685,177685 +171721,177685 +177686,177686 +171722,177686 +177687,177687 +171723,177687 +177688,177688 +171724,177688 +177689,177689 +171725,177689 +177690,177690 +171726,177690 +177691,177691 +171727,177691 +177692,177692 +171728,177692 +177693,177693 +171729,177693 +177694,177694 +171730,177694 +177695,177695 +171731,177695 +177696,177696 +171732,177696 +177697,177697 +171733,177697 +177698,177698 +171734,177698 +177699,177699 +171735,177699 +177700,177700 +171736,177700 +177701,177701 +171737,177701 +177702,177702 +171738,177702 +177703,177703 +171739,177703 +177704,177704 +171740,177704 +177705,177705 +171741,177705 +177706,177706 +171742,177706 +177707,177707 +171743,177707 +141323,177707 +177708,177708 +171744,177708 +177709,177709 +171745,177709 +177710,177710 +171746,177710 +177711,177711 +171747,177711 +177712,177712 +171748,177712 +177713,177713 +171749,177713 +101842,177713 +177714,177714 +171750,177714 +177715,177715 +171751,177715 +216516,177715 +177716,177716 +171752,177716 +177717,177717 +171753,177717 +177718,177718 +171754,177718 +211600,177718 +177719,177719 +171755,177719 +177720,177720 +171756,177720 +177721,177721 +171757,177721 +177722,177722 +171758,177722 +177723,177723 +171759,177723 +177724,177724 +171760,177724 +177725,177725 +171761,177725 +177726,177726 +171762,177726 +177727,177727 +171763,177727 +177728,177728 +171764,177728 +177729,177729 +171765,177729 +177730,177730 +171766,177730 +177731,177731 +171767,177731 +177732,177732 +171768,177732 +177733,177733 +171769,177733 +177734,177734 +171770,177734 +177735,177735 +171771,177735 +177736,177736 +171772,177736 +177737,177737 +171773,177737 +177738,177738 +171774,177738 +177739,177739 +171775,177739 +177740,177740 +171776,177740 +177741,177741 +171777,177741 +177742,177742 +171778,177742 +177743,177743 +171779,177743 +177744,177744 +171780,177744 +177745,177745 +171781,177745 +177746,177746 +171782,177746 +177747,177747 +171783,177747 +177748,177748 +171784,177748 +177749,177749 +171785,177749 +177750,177750 +171786,177750 +177751,177751 +171787,177751 +177752,177752 +171788,177752 +177753,177753 +171789,177753 +177754,177754 +171790,177754 +177755,177755 +171791,177755 +177756,177756 +171792,177756 +177757,177757 +171793,177757 +177758,177758 +171794,177758 +177759,177759 +171795,177759 +177760,177760 +171796,177760 +180969,177760 +177761,177761 +171797,177761 +177762,177762 +171798,177762 +177763,177763 +171799,177763 +177764,177764 +171800,177764 +177765,177765 +171801,177765 +177766,177766 +171802,177766 +177767,177767 +171803,177767 +177768,177768 +171804,177768 +179949,177769 +177769,177769 +171805,177769 +177770,177770 +171806,177770 +180831,177770 +33821,177770 +177771,177771 +171807,177771 +33822,177771 +177772,177772 +171808,177772 +177773,177773 +171809,177773 +177774,177774 +171810,177774 +177775,177775 +171811,177775 +177776,177776 +171812,177776 +177777,177777 +171813,177777 +177778,177778 +171814,177778 +177779,177779 +171815,177779 +177780,177780 +171816,177780 +177781,177781 +171817,177781 +177782,177782 +171818,177782 +177783,177783 +171819,177783 +177784,177784 +171820,177784 +177785,177785 +171821,177785 +177786,177786 +171822,177786 +30573,177786 +177787,177787 +171823,177787 +177788,177788 +171824,177788 +177789,177789 +171825,177789 +177790,177790 +171826,177790 +177791,177791 +171827,177791 +177792,177792 +171828,177792 +177793,177793 +171829,177793 +177794,177794 +171830,177794 +177795,177795 +171831,177795 +177796,177796 +171832,177796 +177797,177797 +171833,177797 +177798,177798 +171834,177798 +177799,177799 +171835,177799 +177800,177800 +171836,177800 +177801,177801 +171837,177801 +144841,177801 +177802,177802 +171838,177802 +177803,177803 +171839,177803 +177804,177804 +171840,177804 +177805,177805 +171841,177805 +177806,177806 +171842,177806 +177807,177807 +171843,177807 +177808,177808 +171844,177808 +177809,177809 +171845,177809 +177810,177810 +171846,177810 +177811,177811 +171847,177811 +177812,177812 +171848,177812 +177813,177813 +171849,177813 +177814,177814 +171850,177814 +120280,177814 +177815,177815 +171851,177815 +177816,177816 +171852,177816 +177817,177817 +171853,177817 +137943,177817 +130356,177817 +177818,177818 +171854,177818 +177819,177819 +171855,177819 +177820,177820 +171856,177820 +177821,177821 +171857,177821 +177822,177822 +171858,177822 +177823,177823 +171859,177823 +177824,177824 +171860,177824 +177825,177825 +171861,177825 +177826,177826 +171862,177826 +177827,177827 +171863,177827 +177828,177828 +171864,177828 +177829,177829 +171865,177829 +177830,177830 +171866,177830 +177831,177831 +171867,177831 +177832,177832 +171868,177832 +177833,177833 +171869,177833 +177834,177834 +171870,177834 +177835,177835 +171871,177835 +177836,177836 +171872,177836 +177837,177837 +171873,177837 +177838,177838 +171874,177838 +177960,177960 +171996,177960 +177961,177961 +171997,177961 +177962,177962 +171998,177962 +108645,177962 +177963,177963 +171999,177963 +178016,178016 +172052,178016 +178017,178017 +172053,178017 +178059,178059 +172095,178059 +178060,178060 +172096,178060 +108449,178060 +211776,178060 +178061,178061 +172097,178061 +178062,178062 +172098,178062 +178063,178063 +172099,178063 +178064,178064 +172100,178064 +178065,178065 +172101,178065 +178066,178066 +172102,178066 +178067,178067 +172103,178067 +178068,178068 +172104,178068 +178069,178069 +172105,178069 +76343,178069 +178070,178070 +172106,178070 +178071,178071 +172107,178071 +178072,178072 +172108,178072 +178073,178073 +172109,178073 +178074,178074 +172110,178074 +178075,178075 +172111,178075 +178076,178076 +172112,178076 +178723,178076 +177438,178076 +171474,178076 +178077,178077 +172113,178077 +178078,178078 +172114,178078 +37052,178078 +178079,178079 +172115,178079 +63790,178079 +178080,178080 +172116,178080 +178081,178081 +172117,178081 +178082,178082 +172118,178082 +178083,178083 +172119,178083 +178084,178084 +172120,178084 +178085,178085 +172121,178085 +216489,178085 +178086,178086 +172122,178086 +178087,178087 +172123,178087 +178088,178088 +172124,178088 +178089,178089 +172125,178089 +178090,178090 +172126,178090 +39909,178090 +178091,178091 +172127,178091 +178092,178092 +172128,178092 +178093,178093 +172129,178093 +178094,178094 +172130,178094 +178095,178095 +172131,178095 +178096,178096 +172132,178096 +178097,178097 +172133,178097 +178098,178098 +172134,178098 +178099,178099 +172135,178099 +178100,178100 +172136,178100 +178101,178101 +172137,178101 +178102,178102 +172138,178102 +178103,178103 +172139,178103 +178104,178104 +172140,178104 +178105,178105 +172141,178105 +178106,178106 +172142,178106 +178107,178107 +172143,178107 +178108,178108 +172144,178108 +178109,178109 +172145,178109 +162585,178109 +123165,178109 +178110,178110 +172146,178110 +178111,178111 +172147,178111 +178112,178112 +172148,178112 +242336,178112 +154209,178112 +178113,178113 +172149,178113 +107548,178113 +178114,178114 +172150,178114 +178115,178115 +172151,178115 +178116,178116 +172152,178116 +178117,178117 +172153,178117 +178118,178118 +172154,178118 +178119,178119 +172155,178119 +178120,178120 +172156,178120 +107555,178120 +178121,178121 +172157,178121 +178122,178122 +172158,178122 +145366,178122 +193723,178122 +178123,178123 +172159,178123 +178124,178124 +172160,178124 +178125,178125 +172161,178125 +178126,178126 +172162,178126 +178127,178127 +172163,178127 +178128,178128 +172164,178128 +178129,178129 +172165,178129 +178130,178130 +172166,178130 +178131,178131 +172167,178131 +178132,178132 +172168,178132 +178133,178133 +172169,178133 +178134,178134 +172170,178134 +178135,178135 +172171,178135 +178136,178136 +172172,178136 +178137,178137 +172173,178137 +178138,178138 +172174,178138 +178139,178139 +172175,178139 +178140,178140 +172176,178140 +67773,178140 +178141,178141 +172177,178141 +178142,178142 +172178,178142 +110760,178142 +178143,178143 +172179,178143 +197543,178143 +178144,178144 +172180,178144 +178145,178145 +172181,178145 +178146,178146 +172182,178146 +178147,178147 +172183,178147 +178148,178148 +172184,178148 +178149,178149 +172185,178149 +178150,178150 +172186,178150 +178151,178151 +172187,178151 +178152,178152 +172188,178152 +178153,178153 +172189,178153 +178154,178154 +172190,178154 +178155,178155 +172191,178155 +178156,178156 +172192,178156 +178157,178157 +172193,178157 +178158,178158 +172194,178158 +178159,178159 +172195,178159 +178160,178160 +172196,178160 +178161,178161 +172197,178161 +178162,178162 +172198,178162 +178163,178163 +172199,178163 +178164,178164 +172200,178164 +178165,178165 +172201,178165 +178166,178166 +172202,178166 +178167,178167 +172203,178167 +178168,178168 +172204,178168 +178169,178169 +172205,178169 +178170,178170 +172206,178170 +178171,178171 +172207,178171 +178172,178172 +172208,178172 +178173,178173 +172209,178173 +178174,178174 +172210,178174 +178175,178175 +172211,178175 +178176,178176 +172212,178176 +178177,178177 +172213,178177 +178178,178178 +172214,178178 +178179,178179 +172215,178179 +178180,178180 +172216,178180 +178181,178181 +172217,178181 +178182,178182 +172218,178182 +178183,178183 +172219,178183 +178184,178184 +172220,178184 +178185,178185 +172221,178185 +178186,178186 +172222,178186 +178187,178187 +172223,178187 +206578,178187 +178188,178188 +172224,178188 +178189,178189 +172225,178189 +178190,178190 +172226,178190 +178191,178191 +172227,178191 +193276,178191 +178192,178192 +172228,178192 +178193,178193 +172229,178193 +178194,178194 +172230,178194 +178195,178195 +172231,178195 +178196,178196 +172232,178196 +178197,178197 +172233,178197 +178198,178198 +172234,178198 +178199,178199 +172235,178199 +178200,178200 +172236,178200 +178201,178201 +172237,178201 +178202,178202 +172238,178202 +178203,178203 +172239,178203 +178204,178204 +172240,178204 +178205,178205 +172241,178205 +178206,178206 +172242,178206 +83661,178206 +178207,178207 +172243,178207 +76394,178207 +178208,178208 +172244,178208 +178209,178209 +172245,178209 +178210,178210 +172246,178210 +178211,178211 +172247,178211 +136308,178211 +128721,178211 +178212,178212 +172248,178212 +178213,178213 +172249,178213 +178214,178214 +172250,178214 +179222,178214 +178215,178215 +172251,178215 +178216,178216 +172252,178216 +178217,178217 +172253,178217 +178218,178218 +172254,178218 +178219,178219 +172255,178219 +178220,178220 +172256,178220 +178221,178221 +172257,178221 +178222,178222 +172258,178222 +178223,178223 +172259,178223 +178224,178224 +172260,178224 +178225,178225 +172261,178225 +178226,178226 +172262,178226 +178227,178227 +172263,178227 +178228,178228 +172264,178228 +178229,178229 +172265,178229 +178230,178230 +172266,178230 +178231,178231 +172267,178231 +178232,178232 +172268,178232 +178233,178233 +172269,178233 +178234,178234 +172270,178234 +178235,178235 +172271,178235 +86869,178235 +178236,178236 +172272,178236 +178237,178237 +172273,178237 +178238,178238 +172274,178238 +178239,178239 +172275,178239 +178240,178240 +172276,178240 +178241,178241 +172277,178241 +178242,178242 +172278,178242 +248675,178242 +178243,178243 +172279,178243 +178244,178244 +172280,178244 +178245,178245 +172281,178245 +178246,178246 +172282,178246 +178247,178247 +172283,178247 +178248,178248 +172284,178248 +178249,178249 +172285,178249 +178250,178250 +172286,178250 +178251,178251 +172287,178251 +178252,178252 +172288,178252 +179950,178252 +178253,178253 +172289,178253 +178254,178254 +172290,178254 +178255,178255 +172291,178255 +178256,178256 +172292,178256 +178257,178257 +172293,178257 +178258,178258 +172294,178258 +178259,178259 +172295,178259 +178260,178260 +172296,178260 +178261,178261 +172297,178261 +178262,178262 +172298,178262 +178263,178263 +172299,178263 +178264,178264 +172300,178264 +178265,178265 +172301,178265 +178266,178266 +172302,178266 +178267,178267 +172303,178267 +178268,178268 +172304,178268 +178269,178269 +172305,178269 +178270,178270 +172306,178270 +219941,178270 +178271,178271 +172307,178271 +106199,178271 +178272,178272 +172308,178272 +178273,178273 +172309,178273 +178274,178274 +172310,178274 +178275,178275 +172311,178275 +178276,178276 +172312,178276 +178277,178277 +172313,178277 +178278,178278 +172314,178278 +178279,178279 +172315,178279 +107490,178279 +233933,178279 +178280,178280 +172316,178280 +178281,178281 +172317,178281 +178282,178282 +172318,178282 +178283,178283 +172319,178283 +178284,178284 +172320,178284 +178285,178285 +172321,178285 +178286,178286 +172322,178286 +178287,178287 +172323,178287 +178288,178288 +172324,178288 +178289,178289 +172325,178289 +178290,178290 +172326,178290 +178291,178291 +172327,178291 +178292,178292 +172328,178292 +178293,178293 +172329,178293 +178294,178294 +172330,178294 +178295,178295 +172331,178295 +178296,178296 +172332,178296 +178297,178297 +172333,178297 +178298,178298 +172334,178298 +178299,178299 +172335,178299 +178300,178300 +172336,178300 +178301,178301 +172337,178301 +178302,178302 +172338,178302 +178303,178303 +172339,178303 +178304,178304 +172340,178304 +178305,178305 +172341,178305 +178306,178306 +172342,178306 +132793,178306 +125206,178306 +178307,178307 +172343,178307 +178308,178308 +172344,178308 +235678,178308 +178309,178309 +172345,178309 +178310,178310 +172346,178310 +77137,178310 +178311,178311 +172347,178311 +178312,178312 +172348,178312 +178313,178313 +172349,178313 +178314,178314 +172350,178314 +178315,178315 +172351,178315 +178316,178316 +172352,178316 +178317,178317 +172353,178317 +178318,178318 +172354,178318 +178319,178319 +172355,178319 +178320,178320 +172356,178320 +178321,178321 +172357,178321 +206298,178321 +178322,178322 +172358,178322 +178323,178323 +172359,178323 +178324,178324 +172360,178324 +178325,178325 +172361,178325 +178326,178326 +172362,178326 +178327,178327 +172363,178327 +5232,178327 +178328,178328 +172364,178328 +178329,178329 +172365,178329 +178330,178330 +172366,178330 +178331,178331 +172367,178331 +189750,178331 +141830,178331 +177489,178331 +171525,178331 +178332,178332 +172368,178332 +24538,178332 +178333,178333 +172369,178333 +178334,178334 +172370,178334 +178335,178335 +172371,178335 +178336,178336 +172372,178336 +178337,178337 +172373,178337 +178338,178338 +172374,178338 +178339,178339 +172375,178339 +178340,178340 +172376,178340 +178341,178341 +172377,178341 +178342,178342 +172378,178342 +178343,178343 +172379,178343 +178344,178344 +172380,178344 +178345,178345 +172381,178345 +211236,178345 +178346,178346 +172382,178346 +178347,178347 +172383,178347 +178348,178348 +172384,178348 +178349,178349 +172385,178349 +220035,178349 +178350,178350 +172386,178350 +178351,178351 +172387,178351 +178352,178352 +172388,178352 +178353,178353 +172389,178353 +178354,178354 +172390,178354 +178355,178355 +172391,178355 +178356,178356 +172392,178356 +178357,178357 +172393,178357 +178358,178358 +172394,178358 +23675,178358 +178359,178359 +172395,178359 +178360,178360 +172396,178360 +178361,178361 +172397,178361 +178362,178362 +172398,178362 +178363,178363 +172399,178363 +178364,178364 +172400,178364 +178365,178365 +172401,178365 +178366,178366 +172402,178366 +178367,178367 +172403,178367 +178368,178368 +172404,178368 +178369,178369 +172405,178369 +178370,178370 +172406,178370 +205969,178370 +178371,178371 +172407,178371 +178372,178372 +172408,178372 +178373,178373 +172409,178373 +178374,178374 +172410,178374 +178375,178375 +172411,178375 +178376,178376 +172412,178376 +178377,178377 +172413,178377 +178378,178378 +172414,178378 +178379,178379 +172415,178379 +178380,178380 +172416,178380 +178381,178381 +172417,178381 +178382,178382 +172418,178382 +178383,178383 +172419,178383 +178384,178384 +172420,178384 +178385,178385 +172421,178385 +178386,178386 +172422,178386 +178387,178387 +172423,178387 +83985,178387 +178388,178388 +172424,178388 +178389,178389 +172425,178389 +178390,178390 +172426,178390 +178391,178391 +172427,178391 +178392,178392 +172428,178392 +178393,178393 +172429,178393 +30487,178393 +178394,178394 +172430,178394 +178395,178395 +172431,178395 +178396,178396 +172432,178396 +178397,178397 +172433,178397 +36456,178397 +178465,178465 +172501,178465 +178466,178466 +172502,178466 +178467,178467 +172503,178467 +178468,178468 +172504,178468 +178469,178469 +172505,178469 +178470,178470 +172506,178470 +178471,178471 +172507,178471 +178472,178472 +172508,178472 +178473,178473 +172509,178473 +178474,178474 +172510,178474 +178475,178475 +172511,178475 +178476,178476 +172512,178476 +178477,178477 +172513,178477 +187294,178477 +178478,178478 +172514,178478 +178479,178479 +172515,178479 +178480,178480 +172516,178480 +178481,178481 +172517,178481 +178482,178482 +172518,178482 +178483,178483 +172519,178483 +178484,178484 +172520,178484 +178485,178485 +172521,178485 +50894,178485 +42790,178485 +178486,178486 +172522,178486 +178487,178487 +172523,178487 +178488,178488 +172524,178488 +178489,178489 +172525,178489 +178490,178490 +172526,178490 +178491,178491 +172527,178491 +178492,178492 +172528,178492 +178493,178493 +172529,178493 +36612,178493 +30474,178493 +178494,178494 +172530,178494 +178495,178495 +172531,178495 +178496,178496 +172532,178496 +178497,178497 +172533,178497 +178498,178498 +172534,178498 +178499,178499 +172535,178499 +178500,178500 +172536,178500 +178501,178501 +172537,178501 +178502,178502 +172538,178502 +178503,178503 +172539,178503 +178504,178504 +172540,178504 +178505,178505 +172541,178505 +178506,178506 +172542,178506 +178507,178507 +172543,178507 +178508,178508 +172544,178508 +178509,178509 +172545,178509 +204011,178509 +178510,178510 +172546,178510 +178511,178511 +172547,178511 +178512,178512 +172548,178512 +178513,178513 +172549,178513 +178514,178514 +172550,178514 +178515,178515 +172551,178515 +178516,178516 +172552,178516 +178517,178517 +172553,178517 +178518,178518 +172554,178518 +178519,178519 +172555,178519 +178520,178520 +172556,178520 +178521,178521 +172557,178521 +80441,178521 +178522,178522 +172558,178522 +178523,178523 +172559,178523 +165219,178523 +178524,178524 +172560,178524 +178525,178525 +172561,178525 +178526,178526 +172562,178526 +178527,178527 +172563,178527 +178528,178528 +172564,178528 +50764,178528 +42660,178528 +178529,178529 +172565,178529 +178530,178530 +172566,178530 +178531,178531 +172567,178531 +178532,178532 +172568,178532 +178533,178533 +172569,178533 +178534,178534 +172570,178534 +178535,178535 +172571,178535 +178536,178536 +172572,178536 +178537,178537 +172573,178537 +178538,178538 +172574,178538 +178539,178539 +172575,178539 +178540,178540 +172576,178540 +178541,178541 +172577,178541 +178542,178542 +172578,178542 +178543,178543 +172579,178543 +178544,178544 +172580,178544 +178545,178545 +172581,178545 +178546,178546 +172582,178546 +178547,178547 +172583,178547 +178548,178548 +172584,178548 +178549,178549 +172585,178549 +178550,178550 +172586,178550 +178551,178551 +172587,178551 +178552,178552 +172588,178552 +178553,178553 +172589,178553 +178554,178554 +172590,178554 +178555,178555 +172591,178555 +178556,178556 +172592,178556 +178557,178557 +172593,178557 +178558,178558 +172594,178558 +178559,178559 +172595,178559 +178560,178560 +172596,178560 +178561,178561 +172597,178561 +178562,178562 +172598,178562 +178563,178563 +172599,178563 +65859,178563 +178564,178564 +172600,178564 +178565,178565 +172601,178565 +178566,178566 +172602,178566 +178567,178567 +172603,178567 +178568,178568 +172604,178568 +178569,178569 +221862,178570 +178570,178570 +178571,178571 +207724,178571 +178572,178572 +178573,178573 +209794,178573 +178574,178574 +178575,178575 +178576,178576 +79476,178576 +178577,178577 +178578,178578 +178579,178579 +178580,178580 +178581,178581 +178582,178582 +178583,178583 +178584,178584 +178585,178585 +178586,178586 +178587,178587 +178588,178588 +212282,178588 +178589,178589 +178590,178590 +178591,178591 +178592,178592 +178593,178593 +178594,178594 +178595,178595 +178596,178596 +178597,178597 +178598,178598 +178599,178599 +178600,178600 +142382,178600 +178601,178601 +178602,178602 +178603,178603 +178604,178604 +178605,178605 +178606,178606 +156302,178606 +178607,178607 +178608,178608 +178609,178609 +178610,178610 +178611,178611 +178612,178612 +178613,178613 +178614,178614 +178615,178615 +178616,178616 +178617,178617 +178618,178618 +178619,178619 +178620,178620 +178621,178621 +178622,178622 +178623,178623 +178624,178624 +80853,178624 +178625,178625 +141506,178625 +179191,178625 +178626,178626 +178627,178627 +178628,178628 +178629,178629 +178630,178630 +178631,178631 +178632,178632 +178633,178633 +178634,178634 +178635,178635 +178636,178636 +178637,178637 +178638,178638 +178639,178639 +178640,178640 +178641,178641 +178642,178642 +178643,178643 +178644,178644 +178645,178645 +178646,178646 +178647,178647 +178648,178648 +178649,178649 +178650,178650 +178651,178651 +178652,178652 +178653,178653 +178654,178654 +197533,178654 +178655,178655 +178656,178656 +178657,178657 +178658,178658 +178659,178659 +178660,178660 +178661,178661 +178662,178662 +178663,178663 +178664,178664 +178665,178665 +108901,178665 +178666,178666 +114845,178666 +178667,178667 +178668,178668 +178669,178669 +178670,178670 +178671,178671 +178672,178672 +178673,178673 +178674,178674 +178675,178675 +178676,178676 +178677,178677 +178678,178678 +178679,178679 +178680,178680 +178681,178681 +178682,178682 +178683,178683 +178684,178684 +178685,178685 +178686,178686 +178687,178687 +178688,178688 +178689,178689 +178690,178690 +178691,178691 +178692,178692 +178693,178693 +178694,178694 +178695,178695 +53371,178695 +45267,178695 +178696,178696 +178697,178697 +178698,178698 +178699,178699 +211777,178699 +178700,178700 +178701,178701 +178702,178702 +178703,178703 +178704,178704 +163376,178704 +178705,178705 +178706,178706 +178707,178707 +235847,178707 +178708,178708 +178709,178709 +178710,178710 +189496,178710 +178711,178711 +178712,178712 +178713,178713 +178714,178714 +178715,178715 +178716,178716 +178717,178717 +178718,178718 +162843,178718 +178719,178719 +178720,178720 +178721,178721 +178722,178722 +178723,178723 +178076,178723 +172112,178723 +178724,178724 +178725,178725 +178726,178726 +178727,178727 +82018,178727 +178728,178728 +178729,178729 +178730,178730 +178731,178731 +178732,178732 +178733,178733 +178734,178734 +178735,178735 +178736,178736 +178737,178737 +178738,178738 +178739,178739 +178740,178740 +178741,178741 +178742,178742 +178743,178743 +178744,178744 +178745,178745 +178746,178746 +178747,178747 +178748,178748 +178749,178749 +178750,178750 +178751,178751 +178752,178752 +96035,178752 +178753,178753 +178754,178754 +178755,178755 +178756,178756 +226384,178756 +178757,178757 +178758,178758 +178759,178759 +178760,178760 +178761,178761 +178762,178762 +178763,178763 +141875,178763 +178764,178764 +178765,178765 +178766,178766 +178767,178767 +178768,178768 +178769,178769 +178770,178770 +178771,178771 +178772,178772 +178773,178773 +178774,178774 +57502,178774 +178775,178775 +178776,178776 +178777,178777 +178778,178778 +178779,178779 +178780,178780 +178781,178781 +178782,178782 +178783,178783 +178784,178784 +178785,178785 +178786,178786 +178787,178787 +178788,178788 +178789,178789 +225141,178789 +178790,178790 +178791,178791 +178792,178792 +178793,178793 +178794,178794 +178795,178795 +178796,178796 +178797,178797 +178798,178798 +178799,178799 +178800,178800 +178801,178801 +178802,178802 +178803,178803 +178804,178804 +178805,178805 +178806,178806 +178807,178807 +178808,178808 +178809,178809 +178810,178810 +178811,178811 +178812,178812 +178813,178813 +178814,178814 +178815,178815 +178816,178816 +178817,178817 +178818,178818 +84378,178818 +211699,178818 +178819,178819 +96472,178819 +178820,178820 +178821,178821 +178822,178822 +178823,178823 +178824,178824 +178825,178825 +178826,178826 +178827,178827 +33197,178827 +178828,178828 +178829,178829 +178830,178830 +178831,178831 +178832,178832 +178833,178833 +178834,178834 +208076,178834 +178835,178835 +178836,178836 +122853,178836 +178837,178837 +178838,178838 +184134,178838 +178839,178839 +178840,178840 +204258,178840 +178841,178841 +178842,178842 +178843,178843 +157085,178843 +178844,178844 +178845,178845 +75913,178845 +178846,178846 +178847,178847 +178848,178848 +178849,178849 +178850,178850 +178851,178851 +178852,178852 +149402,178852 +178853,178853 +178854,178854 +178855,178855 +178856,178856 +178857,178857 +178858,178858 +178859,178859 +178860,178860 +178861,178861 +193689,178861 +178862,178862 +34235,178862 +178863,178863 +178864,178864 +178865,178865 +178866,178866 +178867,178867 +98053,178867 +178868,178868 +178869,178869 +178870,178870 +178871,178871 +155451,178871 +178872,178872 +178873,178873 +178874,178874 +178875,178875 +178876,178876 +178877,178877 +178878,178878 +178879,178879 +178880,178880 +178881,178881 +178882,178882 +178883,178883 +178884,178884 +185065,178884 +178885,178885 +178886,178886 +178887,178887 +178888,178888 +178889,178889 +178890,178890 +178891,178891 +178892,178892 +178893,178893 +178894,178894 +178895,178895 +178896,178896 +178897,178897 +178898,178898 +178899,178899 +178900,178900 +178901,178901 +178902,178902 +107332,178902 +178903,178903 +178904,178904 +178905,178905 +178906,178906 +178907,178907 +178908,178908 +178909,178909 +178910,178910 +178911,178911 +178912,178912 +178913,178913 +178914,178914 +178915,178915 +178916,178916 +178917,178917 +178918,178918 +178919,178919 +178920,178920 +114227,178920 +178921,178921 +86107,178921 +178922,178922 +178923,178923 +178924,178924 +34211,178924 +178925,178925 +178926,178926 +178927,178927 +178928,178928 +178929,178929 +178930,178930 +178931,178931 +178932,178932 +178933,178933 +178934,178934 +178935,178935 +178936,178936 +178937,178937 +178938,178938 +178939,178939 +162376,178939 +178940,178940 +178941,178941 +79321,178941 +178942,178942 +178943,178943 +28229,178943 +178944,178944 +178945,178945 +178946,178946 +178947,178947 +114732,178947 +178948,178948 +178949,178949 +178950,178950 +178951,178951 +178952,178952 +178953,178953 +178954,178954 +178955,178955 +178956,178956 +178957,178957 +187277,178957 +178958,178958 +178959,178959 +178960,178960 +178961,178961 +178962,178962 +178963,178963 +178964,178964 +178965,178965 +178966,178966 +237991,178966 +178967,178967 +178968,178968 +178969,178969 +178970,178970 +178971,178971 +178972,178972 +178973,178973 +178974,178974 +178975,178975 +178976,178976 +178977,178977 +178978,178978 +178979,178979 +178980,178980 +178981,178981 +178982,178982 +62451,178982 +178983,178983 +178984,178984 +178985,178985 +178986,178986 +178987,178987 +178988,178988 +178989,178989 +178990,178990 +178991,178991 +178992,178992 +178993,178993 +178994,178994 +178995,178995 +178996,178996 +116622,178996 +178997,178997 +178998,178998 +178999,178999 +179000,179000 +179001,179001 +179002,179002 +179003,179003 +179004,179004 +179005,179005 +179006,179006 +179007,179007 +179008,179008 +179009,179009 +179010,179010 +179011,179011 +179012,179012 +20581,179012 +179013,179013 +179014,179014 +179015,179015 +179016,179016 +179017,179017 +108727,179017 +179018,179018 +179019,179019 +179020,179020 +179021,179021 +211786,179021 +179022,179022 +179023,179023 +141296,179023 +179024,179024 +179025,179025 +179026,179026 +179027,179027 +179028,179028 +179029,179029 +179030,179030 +180876,179030 +179031,179031 +179032,179032 +179033,179033 +179034,179034 +179035,179035 +179036,179036 +179037,179037 +179038,179038 +179039,179039 +179040,179040 +179041,179041 +179042,179042 +179043,179043 +179044,179044 +179045,179045 +179046,179046 +179047,179047 +179048,179048 +15363,179048 +179049,179049 +179050,179050 +179051,179051 +179052,179052 +179053,179053 +179054,179054 +179055,179055 +179056,179056 +179057,179057 +179058,179058 +179059,179059 +179060,179060 +179061,179061 +179062,179062 +179063,179063 +179064,179064 +179065,179065 +179066,179066 +179067,179067 +179068,179068 +179069,179069 +179070,179070 +179071,179071 +74921,179071 +179072,179072 +179073,179073 +179074,179074 +179075,179075 +179076,179076 +179077,179077 +179078,179078 +179079,179079 +179080,179080 +179081,179081 +179082,179082 +179083,179083 +179084,179084 +179085,179085 +191623,179085 +190799,179085 +179086,179086 +179087,179087 +179088,179088 +179089,179089 +179090,179090 +68016,179090 +179091,179091 +179092,179092 +179093,179093 +179094,179094 +179095,179095 +179096,179096 +179097,179097 +179098,179098 +179099,179099 +179100,179100 +179101,179101 +179102,179102 +179103,179103 +179104,179104 +179105,179105 +179106,179106 +179107,179107 +179108,179108 +179109,179109 +179110,179110 +179111,179111 +179112,179112 +179113,179113 +179114,179114 +179115,179115 +179116,179116 +179117,179117 +179118,179118 +11449,179118 +179119,179119 +179120,179120 +179121,179121 +179122,179122 +179123,179123 +179124,179124 +179125,179125 +179126,179126 +179127,179127 +179128,179128 +179129,179129 +179130,179130 +179131,179131 +179132,179132 +179133,179133 +179134,179134 +179135,179135 +179136,179136 +179137,179137 +179138,179138 +179139,179139 +179140,179140 +179141,179141 +179142,179142 +179143,179143 +57694,179143 +179144,179144 +179145,179145 +179146,179146 +179147,179147 +179148,179148 +179149,179149 +179150,179150 +179151,179151 +179152,179152 +110653,179152 +179153,179153 +179154,179154 +179155,179155 +179156,179156 +179157,179157 +179158,179158 +188977,179158 +220430,179158 +179159,179159 +179160,179160 +179161,179161 +179162,179162 +179163,179163 +179164,179164 +179165,179165 +179166,179166 +179167,179167 +179491,179167 +179168,179168 +179169,179169 +211131,179169 +179170,179170 +179171,179171 +179172,179172 +179173,179173 +179174,179174 +179175,179175 +179176,179176 +179177,179177 +179178,179178 +179179,179179 +179180,179180 +179181,179181 +137939,179181 +130352,179181 +17202,179181 +179182,179182 +179183,179183 +179184,179184 +179185,179185 +179186,179186 +179187,179187 +82568,179187 +180233,179187 +179188,179188 +179189,179189 +179190,179190 +35026,179190 +179191,179191 +178625,179191 +179192,179192 +179193,179193 +179194,179194 +179195,179195 +179196,179196 +179197,179197 +179198,179198 +179199,179199 +179200,179200 +179201,179201 +9092,179201 +179202,179202 +179203,179203 +107263,179203 +179204,179204 +179205,179205 +179206,179206 +179207,179207 +179208,179208 +179209,179209 +179210,179210 +179211,179211 +179212,179212 +179213,179213 +179214,179214 +179215,179215 +179216,179216 +179217,179217 +190330,179217 +179218,179218 +108783,179218 +179219,179219 +179220,179220 +179221,179221 +179222,179222 +178214,179222 +172250,179222 +179223,179223 +179224,179224 +179225,179225 +179226,179226 +179227,179227 +179228,179228 +179229,179229 +179230,179230 +179231,179231 +179232,179232 +87517,179232 +179233,179233 +179234,179234 +179235,179235 +179236,179236 +179237,179237 +179238,179238 +179239,179239 +179240,179240 +179241,179241 +179242,179242 +179243,179243 +179244,179244 +179245,179245 +179246,179246 +179247,179247 +179248,179248 +179249,179249 +179250,179250 +179251,179251 +179252,179252 +179253,179253 +179254,179254 +179255,179255 +225637,179255 +179256,179256 +179257,179257 +179258,179258 +85589,179258 +179259,179259 +179260,179260 +179261,179261 +179262,179262 +179263,179263 +179264,179264 +179265,179265 +179266,179266 +179267,179267 +179268,179268 +179269,179269 +179270,179270 +179271,179271 +179272,179272 +179273,179273 +179274,179274 +179275,179275 +179276,179276 +179277,179277 +179278,179278 +179279,179279 +179280,179280 +179281,179281 +179282,179282 +179283,179283 +179284,179284 +85900,179284 +179285,179285 +179286,179286 +179287,179287 +179288,179288 +179289,179289 +179290,179290 +104637,179290 +179291,179291 +179292,179292 +179293,179293 +179294,179294 +84832,179294 +179295,179295 +179296,179296 +179297,179297 +179298,179298 +179299,179299 +179300,179300 +179301,179301 +179302,179302 +179303,179303 +179304,179304 +179305,179305 +179306,179306 +179307,179307 +179308,179308 +179309,179309 +179310,179310 +179311,179311 +179312,179312 +179313,179313 +179314,179314 +179315,179315 +179316,179316 +179317,179317 +179318,179318 +179319,179319 +179320,179320 +179321,179321 +179322,179322 +179323,179323 +179324,179324 +179325,179325 +179326,179326 +179327,179327 +21610,179327 +89533,179327 +179328,179328 +179329,179329 +179330,179330 +179331,179331 +74179,179331 +179332,179332 +179333,179333 +179334,179334 +179335,179335 +179336,179336 +179337,179337 +179338,179338 +179339,179339 +179340,179340 +179341,179341 +179342,179342 +179343,179343 +179344,179344 +179345,179345 +179346,179346 +179347,179347 +179348,179348 +179349,179349 +179350,179350 +179351,179351 +179352,179352 +179353,179353 +179354,179354 +179355,179355 +179356,179356 +179357,179357 +161112,179357 +179358,179358 +224729,179359 +179359,179359 +179360,179360 +179361,179361 +179362,179362 +179363,179363 +78521,179363 +179364,179364 +179365,179365 +179366,179366 +179367,179367 +179368,179368 +149311,179368 +179369,179369 +179370,179370 +179371,179371 +179372,179372 +179373,179373 +179374,179374 +179375,179375 +179376,179376 +179377,179377 +179378,179378 +179379,179379 +179380,179380 +179381,179381 +179382,179382 +179383,179383 +179384,179384 +179385,179385 +8005,179385 +179386,179386 +179387,179387 +179388,179388 +179389,179389 +179390,179390 +13850,179390 +179391,179391 +179392,179392 +179393,179393 +179394,179394 +179395,179395 +179396,179396 +179397,179397 +179398,179398 +89334,179398 +179399,179399 +179400,179400 +179401,179401 +162432,179401 +179402,179402 +179403,179403 +179404,179404 +179405,179405 +179406,179406 +179407,179407 +179408,179408 +179409,179409 +179410,179410 +179411,179411 +179412,179412 +179413,179413 +179414,179414 +179415,179415 +179416,179416 +179417,179417 +179418,179418 +179419,179419 +179420,179420 +179421,179421 +179422,179422 +179423,179423 +152874,179423 +179424,179424 +179425,179425 +179426,179426 +179427,179427 +179428,179428 +179429,179429 +111793,179429 +179430,179430 +179431,179431 +179432,179432 +179433,179433 +85432,179433 +179434,179434 +179435,179435 +179436,179436 +179437,179437 +85488,179437 +179438,179438 +179439,179439 +179440,179440 +179441,179441 +179442,179442 +179443,179443 +179444,179444 +179445,179445 +180996,179445 +179446,179446 +179447,179447 +179448,179448 +179449,179449 +179450,179450 +179451,179451 +179452,179452 +179453,179453 +179454,179454 +179455,179455 +179456,179456 +179457,179457 +179458,179458 +179459,179459 +179460,179460 +179461,179461 +179462,179462 +179463,179463 +179464,179464 +179465,179465 +179466,179466 +179467,179467 +179468,179468 +179469,179469 +179470,179470 +179471,179471 +179472,179472 +179473,179473 +179474,179474 +179475,179475 +152249,179475 +179476,179476 +179477,179477 +179478,179478 +179479,179479 +66901,179479 +179480,179480 +179481,179481 +179482,179482 +179483,179483 +179484,179484 +179485,179485 +179486,179486 +179487,179487 +179488,179488 +179489,179489 +179490,179490 +179491,179491 +179167,179491 +179492,179492 +179493,179493 +179494,179494 +179495,179495 +179496,179496 +179497,179497 +179498,179498 +203988,179498 +179499,179499 +179500,179500 +179501,179501 +179502,179502 +179503,179503 +179504,179504 +179505,179505 +179506,179506 +249770,179506 +53305,179506 +45201,179506 +179507,179507 +179508,179508 +179509,179509 +179510,179510 +179511,179511 +179512,179512 +179513,179513 +179514,179514 +179515,179515 +179516,179516 +179517,179517 +179518,179518 +179519,179519 +179520,179520 +179521,179521 +179522,179522 +179523,179523 +179524,179524 +109226,179524 +179525,179525 +179526,179526 +179527,179527 +179528,179528 +179529,179529 +179530,179530 +9795,179530 +179531,179531 +179532,179532 +179533,179533 +146819,179533 +179534,179534 +179535,179535 +179536,179536 +179537,179537 +179538,179538 +2485,179538 +179539,179539 +179540,179540 +179541,179541 +179542,179542 +179543,179543 +179544,179544 +179545,179545 +179546,179546 +179547,179547 +179548,179548 +179549,179549 +179550,179550 +179551,179551 +179552,179552 +179553,179553 +179554,179554 +38426,179554 +179555,179555 +179556,179556 +179557,179557 +179558,179558 +179559,179559 +179560,179560 +574,179560 +179561,179561 +179562,179562 +179563,179563 +179564,179564 +179565,179565 +179566,179566 +15600,179566 +179567,179567 +179568,179568 +179569,179569 +179570,179570 +179571,179571 +179572,179572 +179612,179612 +179613,179613 +179614,179614 +179615,179615 +179616,179616 +179617,179617 +179618,179618 +179619,179619 +179620,179620 +179621,179621 +179622,179622 +179623,179623 +179624,179624 +179625,179625 +179626,179626 +179627,179627 +179628,179628 +179629,179629 +179630,179630 +179631,179631 +179632,179632 +179633,179633 +179634,179634 +179635,179635 +179636,179636 +179637,179637 +179638,179638 +179639,179639 +179640,179640 +179641,179641 +179642,179642 +141179,179642 +28267,179642 +179643,179643 +179644,179644 +179645,179645 +207400,179645 +179646,179646 +179647,179647 +179648,179648 +179649,179649 +179650,179650 +179651,179651 +106294,179651 +179652,179652 +179653,179653 +179654,179654 +179655,179655 +179656,179656 +179657,179657 +179658,179658 +179659,179659 +179660,179660 +179661,179661 +179662,179662 +179663,179663 +179664,179664 +179665,179665 +179666,179666 +179667,179667 +179668,179668 +179669,179669 +179670,179670 +179671,179671 +57170,179671 +179672,179672 +179673,179673 +179674,179674 +179675,179675 +179676,179676 +179677,179677 +179678,179678 +179679,179679 +71218,179679 +179680,179680 +179681,179681 +179682,179682 +179683,179683 +179684,179684 +179685,179685 +57604,179685 +179686,179686 +179687,179687 +179688,179688 +179689,179689 +179690,179690 +231283,179690 +179691,179691 +179692,179692 +179693,179693 +179694,179694 +179695,179695 +179696,179696 +179697,179697 +179698,179698 +179699,179699 +179700,179700 +153637,179700 +179701,179701 +179702,179702 +179703,179703 +179704,179704 +179705,179705 +179706,179706 +179707,179707 +179708,179708 +179709,179709 +179710,179710 +179711,179711 +179712,179712 +179713,179713 +179714,179714 +179715,179715 +179716,179716 +179717,179717 +179718,179718 +179719,179719 +179720,179720 +179721,179721 +179722,179722 +179723,179723 +179724,179724 +179725,179725 +179726,179726 +179727,179727 +179728,179728 +179729,179729 +179730,179730 +179731,179731 +179732,179732 +57855,179732 +179733,179733 +31669,179733 +179734,179734 +179735,179735 +179736,179736 +179737,179737 +179738,179738 +179739,179739 +179740,179740 +179741,179741 +179742,179742 +179743,179743 +179744,179744 +179745,179745 +179746,179746 +179747,179747 +179748,179748 +179749,179749 +179750,179750 +179751,179751 +179752,179752 +179753,179753 +179754,179754 +179755,179755 +179756,179756 +179757,179757 +179758,179758 +179759,179759 +179760,179760 +179761,179761 +179762,179762 +179763,179763 +179764,179764 +179765,179765 +145073,179765 +179766,179766 +179767,179767 +179768,179768 +179769,179769 +179770,179770 +179771,179771 +179772,179772 +179773,179773 +179774,179774 +179775,179775 +179776,179776 +179777,179777 +179778,179778 +179779,179779 +179780,179780 +179781,179781 +179782,179782 +179783,179783 +179784,179784 +179785,179785 +179786,179786 +179787,179787 +179788,179788 +179789,179789 +224724,179789 +179790,179790 +179791,179791 +179792,179792 +179793,179793 +179794,179794 +184726,179794 +179795,179795 +179796,179796 +179797,179797 +179798,179798 +179799,179799 +179800,179800 +179801,179801 +179802,179802 +179803,179803 +179804,179804 +179805,179805 +179806,179806 +179807,179807 +179808,179808 +179809,179809 +179810,179810 +179811,179811 +179812,179812 +179813,179813 +179814,179814 +179815,179815 +179816,179816 +206440,179816 +179817,179817 +179818,179818 +179819,179819 +179820,179820 +179821,179821 +179822,179822 +179823,179823 +84064,179823 +240229,179823 +179824,179824 +179825,179825 +179826,179826 +179827,179827 +179828,179828 +179829,179829 +179830,179830 +179831,179831 +179832,179832 +179833,179833 +179834,179834 +179835,179835 +179836,179836 +179837,179837 +64928,179837 +179838,179838 +179839,179839 +179840,179840 +179841,179841 +179842,179842 +179843,179843 +220624,179843 +179844,179844 +179845,179845 +179846,179846 +179847,179847 +145435,179847 +179848,179848 +179849,179849 +179850,179850 +179851,179851 +179852,179852 +179853,179853 +179854,179854 +179855,179855 +179856,179856 +179857,179857 +179858,179858 +164806,179858 +179859,179859 +179860,179860 +179861,179861 +179862,179862 +179863,179863 +179864,179864 +179865,179865 +179866,179866 +179867,179867 +179868,179868 +179869,179869 +179870,179870 +179871,179871 +179872,179872 +179873,179873 +65600,179873 +179874,179874 +179875,179875 +179876,179876 +179877,179877 +179878,179878 +39936,179878 +179879,179879 +179880,179880 +179881,179881 +10190,179881 +179882,179882 +179883,179883 +179884,179884 +179885,179885 +179886,179886 +179887,179887 +179888,179888 +179889,179889 +179890,179890 +179891,179891 +179892,179892 +179893,179893 +179894,179894 +179895,179895 +179896,179896 +179897,179897 +179898,179898 +179899,179899 +179900,179900 +179901,179901 +179902,179902 +179903,179903 +179904,179904 +179905,179905 +179906,179906 +84647,179906 +179907,179907 +179908,179908 +179909,179909 +179910,179910 +179911,179911 +49133,179911 +40941,179911 +179912,179912 +179913,179913 +179914,179914 +179915,179915 +179916,179916 +179917,179917 +154082,179917 +179918,179918 +179919,179919 +179920,179920 +179921,179921 +179922,179922 +179923,179923 +179924,179924 +179925,179925 +179926,179926 +179927,179927 +179928,179928 +179929,179929 +179930,179930 +179931,179931 +179932,179932 +179933,179933 +179934,179934 +179935,179935 +179936,179936 +179937,179937 +179938,179938 +179939,179939 +179940,179940 +179941,179941 +179942,179942 +179943,179943 +179944,179944 +179945,179945 +179946,179946 +179947,179947 +179948,179948 +179949,179949 +177769,179949 +171805,179949 +179950,179950 +178252,179950 +172288,179950 +179951,179951 +179952,179952 +179953,179953 +179954,179954 +179955,179955 +179956,179956 +179957,179957 +179958,179958 +179959,179959 +179960,179960 +179961,179961 +179962,179962 +179963,179963 +179964,179964 +179965,179965 +179966,179966 +179967,179967 +179968,179968 +100744,179968 +179969,179969 +179970,179970 +179971,179971 +179972,179972 +179973,179973 +179974,179974 +179975,179975 +179976,179976 +179977,179977 +179978,179978 +179979,179979 +142218,179979 +179980,179980 +179981,179981 +179982,179982 +179983,179983 +179984,179984 +179985,179985 +179986,179986 +179987,179987 +179988,179988 +179989,179989 +179990,179990 +179991,179991 +179992,179992 +179993,179993 +179994,179994 +179995,179995 +179996,179996 +179997,179997 +179998,179998 +179999,179999 +180000,180000 +180001,180001 +180002,180002 +180003,180003 +180004,180004 +180005,180005 +180006,180006 +141336,180006 +180007,180007 +180008,180008 +180009,180009 +180010,180010 +180011,180011 +180012,180012 +180013,180013 +180014,180014 +180015,180015 +180016,180016 +180017,180017 +180018,180018 +180019,180019 +180020,180020 +180021,180021 +180022,180022 +180023,180023 +180543,180023 +180024,180024 +180025,180025 +180026,180026 +180027,180027 +220574,180027 +180028,180028 +180029,180029 +180030,180030 +180031,180031 +180032,180032 +180033,180033 +180034,180034 +180035,180035 +180036,180036 +180037,180037 +180038,180038 +180039,180039 +180040,180040 +180041,180041 +180042,180042 +180043,180043 +180044,180044 +189073,180044 +180045,180045 +180046,180046 +180047,180047 +180048,180048 +180049,180049 +180050,180050 +180051,180051 +180052,180052 +180053,180053 +180054,180054 +180055,180055 +180056,180056 +180057,180057 +180058,180058 +180059,180059 +180060,180060 +180061,180061 +10499,180061 +180062,180062 +180063,180063 +180064,180064 +180065,180065 +180066,180066 +162777,180066 +180067,180067 +180068,180068 +180069,180069 +180070,180070 +180071,180071 +180072,180072 +180073,180073 +180074,180074 +180075,180075 +180076,180076 +180077,180077 +180078,180078 +180079,180079 +180080,180080 +180081,180081 +180082,180082 +215514,180082 +180083,180083 +180084,180084 +180085,180085 +180086,180086 +180087,180087 +180088,180088 +180089,180089 +180090,180090 +180091,180091 +180092,180092 +146592,180092 +222059,180092 +180093,180093 +180094,180094 +180095,180095 +180096,180096 +180097,180097 +94061,180097 +180098,180098 +180099,180099 +180100,180100 +3500,180100 +180101,180101 +112106,180101 +180102,180102 +180103,180103 +180104,180104 +180105,180105 +79989,180105 +180106,180106 +180107,180107 +180108,180108 +180109,180109 +180110,180110 +180111,180111 +181261,180111 +180112,180112 +180113,180113 +149863,180113 +180114,180114 +180115,180115 +180116,180116 +180117,180117 +180118,180118 +180119,180119 +180120,180120 +180121,180121 +180122,180122 +180123,180123 +180124,180124 +180125,180125 +180126,180126 +180127,180127 +180128,180128 +180129,180129 +180130,180130 +180131,180131 +180132,180132 +180133,180133 +180134,180134 +180135,180135 +180136,180136 +180137,180137 +180138,180138 +180139,180139 +180140,180140 +180141,180141 +180142,180142 +180143,180143 +180144,180144 +180145,180145 +180146,180146 +180147,180147 +180148,180148 +180149,180149 +180150,180150 +180151,180151 +180152,180152 +79152,180152 +180153,180153 +180154,180154 +180155,180155 +180156,180156 +180157,180157 +180158,180158 +180159,180159 +180160,180160 +180161,180161 +180162,180162 +180163,180163 +180164,180164 +180165,180165 +180166,180166 +180167,180167 +180168,180168 +180169,180169 +180170,180170 +180171,180171 +180172,180172 +180173,180173 +180174,180174 +180175,180175 +180213,180213 +84712,180213 +180214,180214 +180215,180215 +115943,180215 +180216,180216 +180217,180217 +180218,180218 +180219,180219 +9346,180219 +180220,180220 +180221,180221 +180222,180222 +180223,180223 +180224,180224 +180225,180225 +149324,180225 +180226,180226 +180227,180227 +180228,180228 +180229,180229 +180230,180230 +180231,180231 +180232,180232 +180233,180233 +179187,180233 +180234,180234 +180235,180235 +180236,180236 +180237,180237 +180238,180238 +180239,180239 +180240,180240 +180241,180241 +180242,180242 +180243,180243 +180244,180244 +180245,180245 +180246,180246 +180247,180247 +180248,180248 +6833,180248 +180249,180249 +180250,180250 +180251,180251 +180252,180252 +180253,180253 +242660,180253 +79478,180253 +180254,180254 +180255,180255 +180256,180256 +180257,180257 +180258,180258 +203738,180258 +180259,180259 +180260,180260 +180261,180261 +180262,180262 +180263,180263 +180264,180264 +180265,180265 +105631,180265 +180266,180266 +180267,180267 +180268,180268 +180269,180269 +180270,180270 +180271,180271 +180272,180272 +180273,180273 +180274,180274 +207338,180274 +142670,180274 +180275,180275 +180276,180276 +180277,180277 +180278,180278 +180279,180279 +180280,180280 +180281,180281 +180282,180282 +180283,180283 +180284,180284 +180285,180285 +180286,180286 +180287,180287 +180288,180288 +180289,180289 +180290,180290 +56216,180290 +48112,180290 +180291,180291 +180292,180292 +180293,180293 +180294,180294 +180295,180295 +180296,180296 +180297,180297 +180298,180298 +180299,180299 +180300,180300 +180301,180301 +180302,180302 +180303,180303 +180304,180304 +180305,180305 +180306,180306 +180307,180307 +180308,180308 +180309,180309 +180310,180310 +180311,180311 +180312,180312 +180313,180313 +180314,180314 +180315,180315 +180316,180316 +180317,180317 +180318,180318 +180319,180319 +180320,180320 +180321,180321 +180322,180322 +180323,180323 +180324,180324 +180325,180325 +180326,180326 +180327,180327 +180328,180328 +180329,180329 +180330,180330 +180331,180331 +180332,180332 +180333,180333 +180334,180334 +180335,180335 +180336,180336 +21200,180336 +180337,180337 +180338,180338 +180339,180339 +180340,180340 +180341,180341 +180342,180342 +9349,180342 +180343,180343 +180344,180344 +39865,180344 +180345,180345 +180346,180346 +248102,180346 +180347,180347 +180348,180348 +180349,180349 +180350,180350 +180351,180351 +180352,180352 +180353,180353 +180354,180354 +180355,180355 +180356,180356 +180357,180357 +180358,180358 +180359,180359 +97964,180359 +180360,180360 +180361,180361 +180362,180362 +180363,180363 +180364,180364 +180365,180365 +162685,180365 +180366,180366 +180367,180367 +180368,180368 +180369,180369 +180370,180370 +180371,180371 +180372,180372 +142360,180372 +180373,180373 +180374,180374 +180375,180375 +180376,180376 +180377,180377 +180378,180378 +180379,180379 +181144,180379 +180380,180380 +180381,180381 +180382,180382 +180383,180383 +180384,180384 +180385,180385 +180386,180386 +180387,180387 +180388,180388 +180389,180389 +180390,180390 +180391,180391 +180392,180392 +180393,180393 +180394,180394 +180395,180395 +180396,180396 +180397,180397 +180398,180398 +180399,180399 +180400,180400 +180401,180401 +188495,180401 +180402,180402 +180403,180403 +180404,180404 +180405,180405 +180406,180406 +180407,180407 +180408,180408 +180409,180409 +248141,180409 +180410,180410 +180411,180411 +180412,180412 +180413,180413 +180414,180414 +180415,180415 +180416,180416 +180417,180417 +180418,180418 +81447,180418 +180419,180419 +180420,180420 +180421,180421 +180422,180422 +180423,180423 +162483,180423 +180424,180424 +180425,180425 +180426,180426 +180427,180427 +180428,180428 +180429,180429 +180430,180430 +180431,180431 +180432,180432 +180433,180433 +180434,180434 +180435,180435 +180436,180436 +180437,180437 +180438,180438 +180439,180439 +180440,180440 +180441,180441 +180442,180442 +180443,180443 +180444,180444 +180445,180445 +180446,180446 +180447,180447 +180448,180448 +180449,180449 +180450,180450 +180451,180451 +180452,180452 +180453,180453 +180454,180454 +180455,180455 +180456,180456 +180457,180457 +62667,180457 +180458,180458 +180459,180459 +180460,180460 +180461,180461 +180462,180462 +180463,180463 +180464,180464 +180465,180465 +180466,180466 +180467,180467 +180468,180468 +180469,180469 +180470,180470 +180471,180471 +180472,180472 +180473,180473 +180474,180474 +180475,180475 +5562,180475 +180476,180476 +180477,180477 +180478,180478 +180479,180479 +180480,180480 +180481,180481 +180482,180482 +180483,180483 +180484,180484 +180485,180485 +180486,180486 +59141,180486 +58767,180486 +217039,180486 +180487,180487 +180488,180488 +180489,180489 +180490,180490 +180491,180491 +180492,180492 +180493,180493 +180494,180494 +180495,180495 +180496,180496 +180497,180497 +180498,180498 +180499,180499 +180500,180500 +180501,180501 +180502,180502 +180503,180503 +180504,180504 +214442,180505 +180505,180505 +180506,180506 +180507,180507 +180508,180508 +180509,180509 +107400,180509 +180510,180510 +180511,180511 +180512,180512 +180513,180513 +180514,180514 +180515,180515 +180516,180516 +180517,180517 +180518,180518 +180519,180519 +180520,180520 +180521,180521 +180522,180522 +180523,180523 +180524,180524 +180525,180525 +180526,180526 +180527,180527 +180528,180528 +180529,180529 +180530,180530 +180531,180531 +222264,180532 +180532,180532 +180533,180533 +180534,180534 +180535,180535 +180536,180536 +180537,180537 +180538,180538 +180539,180539 +180540,180540 +180541,180541 +180542,180542 +177645,180542 +171681,180542 +180543,180543 +180023,180543 +180544,180544 +180545,180545 +180546,180546 +180547,180547 +180548,180548 +180549,180549 +180550,180550 +180551,180551 +180552,180552 +180553,180553 +180554,180554 +180555,180555 +180556,180556 +180557,180557 +19041,180557 +180558,180558 +180559,180559 +180560,180560 +180561,180561 +180562,180562 +180563,180563 +180564,180564 +71884,180564 +180565,180565 +180566,180566 +180567,180567 +180568,180568 +180569,180569 +180570,180570 +180571,180571 +180572,180572 +180573,180573 +180574,180574 +180575,180575 +180576,180576 +180577,180577 +243146,180577 +180578,180578 +180579,180579 +180580,180580 +180581,180581 +118621,180581 +180582,180582 +180583,180583 +180584,180584 +180585,180585 +180586,180586 +180587,180587 +120175,180587 +180588,180588 +180589,180589 +180590,180590 +180591,180591 +201412,180591 +180592,180592 +180593,180593 +180594,180594 +180595,180595 +180596,180596 +180597,180597 +180598,180598 +180599,180599 +180600,180600 +180601,180601 +180602,180602 +180603,180603 +180604,180604 +180605,180605 +180606,180606 +180607,180607 +180608,180608 +180609,180609 +180610,180610 +221797,180610 +180611,180611 +180612,180612 +180613,180613 +180614,180614 +180615,180615 +180616,180616 +180617,180617 +180618,180618 +180619,180619 +180620,180620 +180621,180621 +180622,180622 +180623,180623 +77290,180623 +180624,180624 +180625,180625 +180626,180626 +180627,180627 +180628,180628 +180629,180629 +180630,180630 +180631,180631 +180632,180632 +180633,180633 +180634,180634 +180635,180635 +180636,180636 +180637,180637 +180638,180638 +180639,180639 +180640,180640 +180641,180641 +180642,180642 +180643,180643 +180644,180644 +180645,180645 +180646,180646 +180647,180647 +180648,180648 +180649,180649 +180650,180650 +180651,180651 +180652,180652 +180653,180653 +180654,180654 +180655,180655 +60435,180655 +103617,180655 +180656,180656 +149779,180656 +180657,180657 +180658,180658 +180659,180659 +180660,180660 +180661,180661 +180662,180662 +192436,180662 +157787,180662 +180663,180663 +180664,180664 +180665,180665 +180666,180666 +180667,180667 +180668,180668 +180669,180669 +180670,180670 +180671,180671 +180672,180672 +180673,180673 +180674,180674 +180675,180675 +85988,180675 +180676,180676 +180677,180677 +180678,180678 +180679,180679 +180680,180680 +180681,180681 +180682,180682 +180683,180683 +180684,180684 +180685,180685 +180686,180686 +180687,180687 +180688,180688 +180689,180689 +180690,180690 +180691,180691 +180692,180692 +180693,180693 +180694,180694 +180695,180695 +180696,180696 +180697,180697 +180698,180698 +180699,180699 +180700,180700 +192267,180700 +180701,180701 +188485,180701 +180702,180702 +180703,180703 +180704,180704 +180705,180705 +180706,180706 +207828,180707 +180707,180707 +180708,180708 +180709,180709 +180710,180710 +180711,180711 +180712,180712 +180713,180713 +180714,180714 +249762,180714 +180715,180715 +111626,180715 +180716,180716 +180717,180717 +180718,180718 +180719,180719 +180720,180720 +180721,180721 +180722,180722 +109604,180722 +180723,180723 +180724,180724 +180725,180725 +180726,180726 +180727,180727 +180728,180728 +180729,180729 +180730,180730 +180731,180731 +180732,180732 +180733,180733 +180734,180734 +180735,180735 +180736,180736 +180737,180737 +123964,180737 +180738,180738 +180739,180739 +180740,180740 +180741,180741 +180742,180742 +180743,180743 +180744,180744 +180745,180745 +160046,180745 +180746,180746 +180747,180747 +180748,180748 +180749,180749 +180750,180750 +180751,180751 +180752,180752 +180753,180753 +180754,180754 +180755,180755 +180756,180756 +141192,180756 +142078,180756 +180757,180757 +180758,180758 +180759,180759 +180760,180760 +180761,180761 +180762,180762 +180763,180763 +219968,180763 +180764,180764 +180765,180765 +180766,180766 +180767,180767 +180768,180768 +180769,180769 +180809,180809 +241120,180809 +180810,180810 +180811,180811 +246652,180811 +180812,180812 +180813,180813 +180814,180814 +180815,180815 +180816,180816 +180817,180817 +180818,180818 +202588,180818 +180819,180819 +180820,180820 +180821,180821 +180822,180822 +180823,180823 +180824,180824 +180825,180825 +180826,180826 +180827,180827 +180828,180828 +180829,180829 +180830,180830 +108768,180830 +180831,180831 +33821,180831 +177770,180831 +171806,180831 +180832,180832 +190337,180832 +180833,180833 +109028,180833 +180834,180834 +180835,180835 +180836,180836 +180837,180837 +180838,180838 +180839,180839 +180840,180840 +180841,180841 +180842,180842 +180843,180843 +180844,180844 +180845,180845 +216614,180845 +180846,180846 +180847,180847 +180848,180848 +180849,180849 +180850,180850 +180851,180851 +180852,180852 +180853,180853 +180854,180854 +180855,180855 +180856,180856 +180857,180857 +180858,180858 +180859,180859 +180860,180860 +180861,180861 +180862,180862 +180863,180863 +180864,180864 +180865,180865 +180866,180866 +180867,180867 +180868,180868 +180869,180869 +180870,180870 +180871,180871 +180872,180872 +180873,180873 +180874,180874 +180875,180875 +52057,180875 +43953,180875 +180876,180876 +179030,180876 +180877,180877 +180878,180878 +52060,180878 +43956,180878 +180879,180879 +180880,180880 +180881,180881 +180882,180882 +180883,180883 +180884,180884 +180885,180885 +180886,180886 +180887,180887 +180888,180888 +180889,180889 +180890,180890 +180891,180891 +180892,180892 +180893,180893 +180894,180894 +180895,180895 +180896,180896 +180897,180897 +180898,180898 +180899,180899 +180900,180900 +180901,180901 +180902,180902 +180903,180903 +180904,180904 +180905,180905 +180906,180906 +103618,180906 +180907,180907 +180908,180908 +108859,180908 +180909,180909 +180910,180910 +180911,180911 +180912,180912 +180913,180913 +180914,180914 +180915,180915 +180916,180916 +180917,180917 +180918,180918 +180961,180961 +38453,180961 +180962,180962 +180963,180963 +134196,180963 +126609,180963 +180964,180964 +180965,180965 +180966,180966 +180967,180967 +180968,180968 +180969,180969 +177760,180969 +171796,180969 +180970,180970 +180971,180971 +180972,180972 +180973,180973 +180974,180974 +180975,180975 +180976,180976 +180977,180977 +180978,180978 +56550,180978 +48446,180978 +180979,180979 +165219,180979 +180980,180980 +144445,180980 +180981,180981 +180982,180982 +180983,180983 +180984,180984 +180985,180985 +180986,180986 +180987,180987 +180988,180988 +180989,180989 +180990,180990 +180991,180991 +180992,180992 +180993,180993 +180994,180994 +180995,180995 +180996,180996 +179445,180996 +180997,180997 +180998,180998 +104348,180998 +100631,180998 +180999,180999 +181000,181000 +181001,181001 +181002,181002 +181003,181003 +181004,181004 +181005,181005 +181006,181006 +181007,181007 +181008,181008 +181009,181009 +181010,181010 +181011,181011 +181012,181012 +181013,181013 +181014,181014 +181015,181015 +225391,181015 +181016,181016 +181017,181017 +181018,181018 +181019,181019 +223102,181019 +181020,181020 +181021,181021 +181022,181022 +181023,181023 +181024,181024 +181025,181025 +40215,181025 +181026,181026 +181027,181027 +181028,181028 +181029,181029 +181030,181030 +234217,181030 +181031,181031 +181032,181032 +181033,181033 +181034,181034 +181035,181035 +181036,181036 +181037,181037 +181038,181038 +181039,181039 +181040,181040 +181041,181041 +181042,181042 +181043,181043 +181044,181044 +181045,181045 +181046,181046 +181047,181047 +181048,181048 +181049,181049 +181050,181050 +181051,181051 +181052,181052 +181053,181053 +181054,181054 +181055,181055 +181056,181056 +181057,181057 +181058,181058 +181059,181059 +181060,181060 +181061,181061 +108948,181061 +181062,181062 +181063,181063 +181064,181064 +181065,181065 +181066,181066 +181067,181067 +181068,181068 +181069,181069 +181070,181070 +181071,181071 +181072,181072 +181073,181073 +181074,181074 +181075,181075 +181076,181076 +181077,181077 +99391,181077 +181078,181078 +181079,181079 +181080,181080 +181081,181081 +181082,181082 +181083,181083 +181084,181084 +181085,181085 +181086,181086 +181087,181087 +181088,181088 +181089,181089 +181090,181090 +181091,181091 +181092,181092 +181093,181093 +181094,181094 +181095,181095 +181096,181096 +58678,181096 +181097,181097 +181098,181098 +181099,181099 +181126,181099 +181100,181100 +181101,181101 +77245,181101 +181102,181102 +234190,181102 +181103,181103 +181104,181104 +181105,181105 +110952,181105 +181106,181106 +181107,181107 +181108,181108 +181109,181109 +181110,181110 +181111,181111 +181112,181112 +181113,181113 +181114,181114 +181115,181115 +181116,181116 +181117,181117 +181118,181118 +181119,181119 +181120,181120 +181121,181121 +181122,181122 +181123,181123 +181124,181124 +181125,181125 +181126,181126 +181099,181126 +181127,181127 +80724,181127 +181128,181128 +181129,181129 +181130,181130 +181131,181131 +181132,181132 +181133,181133 +181134,181134 +181135,181135 +181136,181136 +181137,181137 +181138,181138 +181139,181139 +181140,181140 +181141,181141 +181142,181142 +181143,181143 +181144,181144 +180379,181144 +181145,181145 +181217,181217 +181218,181218 +181219,181219 +181220,181220 +181221,181221 +181222,181222 +181223,181223 +181224,181224 +181225,181225 +181226,181226 +181227,181227 +181228,181228 +181229,181229 +181230,181230 +181231,181231 +181232,181232 +181233,181233 +181234,181234 +181235,181235 +181236,181236 +181237,181237 +181238,181238 +9117,181238 +181239,181239 +181240,181240 +181241,181241 +181242,181242 +181243,181243 +181244,181244 +181245,181245 +181246,181246 +181247,181247 +181248,181248 +65827,181248 +181249,181249 +181250,181250 +181251,181251 +181252,181252 +181253,181253 +181254,181254 +181255,181255 +181256,181256 +181257,181257 +181258,181258 +181259,181259 +181260,181260 +181261,181261 +180111,181261 +181262,181262 +181263,181263 +181264,181264 +181265,181265 +181266,181266 +220048,181266 +181267,181267 +181268,181268 +164491,181268 +131717,181268 +124130,181268 +181269,181269 +181270,181270 +181271,181271 +181272,181272 +181273,181273 +181274,181274 +181275,181275 +181276,181276 +181277,181277 +181278,181278 +181279,181279 +181280,181280 +181281,181281 +181282,181282 +181283,181283 +181284,181284 +181285,181285 +181286,181286 +181287,181287 +181288,181288 +97,181288 +181289,181289 +181290,181290 +181291,181291 +181292,181292 +181293,181293 +181294,181294 +181295,181295 +149666,181295 +181296,181296 +181297,181297 +116336,181297 +181298,181298 +181299,181299 +181300,181300 +181301,181301 +181302,181302 +181303,181303 +181304,181304 +181305,181305 +181306,181306 +181307,181307 +181308,181308 +181309,181309 +181310,181310 +208182,181310 +181311,181311 +11368,181311 +181312,181312 +181313,181313 +181314,181314 +181315,181315 +181316,181316 +181317,181317 +163071,181317 +181318,181318 +181319,181319 +54174,181319 +46070,181319 +181320,181320 +181321,181321 +181322,181322 +161289,181322 +181323,181323 +181324,181324 +181325,181325 +181326,181326 +181327,181327 +181328,181328 +181329,181329 +181330,181330 +181331,181331 +181332,181332 +181333,181333 +181334,181334 +75539,181334 +181335,181335 +181336,181336 +181337,181337 +181338,181338 +181339,181339 +181340,181340 +236899,181340 +181341,181341 +181342,181342 +181343,181343 +181344,181344 +181345,181345 +181346,181346 +181347,181347 +181348,181348 +181349,181349 +241408,181349 +181350,181350 +181351,181351 +181352,181352 +181353,181353 +181354,181354 +181355,181355 +181356,181356 +181357,181357 +181358,181358 +181359,181359 +181360,181360 +181361,181361 +181362,181362 +181363,181363 +181364,181364 +181365,181365 +181366,181366 +181367,181367 +181368,181368 +181369,181369 +181370,181370 +182310,182310 +182311,182311 +182312,182312 +182313,182313 +182314,182314 +182315,182315 +182316,182316 +182317,182317 +182318,182318 +146518,182318 +182319,182319 +182320,182320 +132396,182320 +124809,182320 +182321,182321 +155613,182321 +2691,182321 +182322,182322 +182323,182323 +182324,182324 +182325,182325 +182326,182326 +182327,182327 +182328,182328 +182329,182329 +182330,182330 +182491,182491 +182492,182492 +182493,182493 +182494,182494 +182495,182495 +182496,182496 +182497,182497 +182498,182498 +182994,182994 +182995,182995 +182996,182996 +182997,182997 +182998,182998 +182999,182999 +183000,183000 +183001,183001 +183002,183002 +183064,183002 +183003,183003 +183004,183004 +183005,183005 +183006,183006 +183007,183007 +183008,183008 +183009,183009 +183010,183010 +183011,183011 +183012,183012 +183013,183013 +183014,183014 +109816,183014 +183015,183015 +183053,183053 +183054,183054 +183055,183055 +183056,183056 +183057,183057 +4336,183057 +183058,183058 +183059,183059 +183060,183060 +183061,183061 +183062,183062 +183063,183063 +183064,183064 +183002,183064 +183065,183065 +183066,183066 +183067,183067 +183068,183068 +114443,183068 +183069,183069 +220376,183069 +183070,183070 +209752,183070 +183071,183071 +183072,183072 +183073,183073 +183074,183074 +183112,183112 +183113,183113 +183114,183114 +183115,183115 +183116,183116 +114738,183116 +183117,183117 +183118,183118 +183119,183119 +183120,183120 +19922,183120 +183121,183121 +183122,183122 +183123,183123 +183124,183124 +183125,183125 +183126,183126 +183127,183127 +183128,183128 +183129,183129 +183231,183231 +183232,183232 +183233,183233 +183234,183234 +183235,183235 +183236,183236 +103550,183236 +183237,183237 +183238,183238 +183239,183239 +183240,183240 +183241,183241 +183242,183242 +206637,183242 +183243,183243 +183244,183244 +183245,183245 +183246,183246 +183247,183247 +183248,183248 +183249,183249 +183250,183250 +183251,183251 +183252,183252 +183253,183253 +183254,183254 +183255,183255 +183256,183256 +183257,183257 +183258,183258 +183259,183259 +183260,183260 +183261,183261 +183262,183262 +183263,183263 +183264,183264 +183265,183265 +183266,183266 +183267,183267 +183268,183268 +183269,183269 +183270,183270 +183271,183271 +183344,183344 +183345,183345 +183346,183346 +183347,183347 +183348,183348 +243572,183348 +183349,183349 +183350,183350 +183351,183351 +183352,183352 +183353,183353 +183354,183354 +183355,183355 +183356,183356 +183357,183357 +183358,183358 +183359,183359 +183360,183360 +183361,183361 +183362,183362 +183363,183363 +183364,183364 +37028,183364 +183365,183365 +183366,183366 +183367,183367 +183368,183368 +183369,183369 +183370,183370 +183371,183371 +183372,183372 +183373,183373 +183374,183374 +183375,183375 +183376,183376 +183377,183377 +183378,183378 +183379,183379 +183380,183380 +183381,183381 +183382,183382 +183383,183383 +104897,183383 +183384,183384 +183385,183385 +183386,183386 +183387,183387 +183388,183388 +183389,183389 +183390,183390 +183391,183391 +183392,183392 +183393,183393 +183394,183394 +183395,183395 +183396,183396 +183397,183397 +183398,183398 +183399,183399 +183400,183400 +183401,183401 +183402,183402 +183403,183403 +183404,183404 +183405,183405 +183406,183406 +183407,183407 +183408,183408 +183409,183409 +183410,183410 +183411,183411 +183412,183412 +183413,183413 +183414,183414 +183415,183415 +183416,183416 +183417,183417 +183418,183418 +183419,183419 +192919,183420 +183420,183420 +183421,183421 +118459,183421 +183422,183422 +2015,183422 +183423,183423 +183424,183424 +183425,183425 +183426,183426 +183427,183427 +183428,183428 +106661,183428 +183429,183429 +183430,183430 +183431,183431 +183432,183432 +183433,183433 +183434,183434 +183435,183435 +183436,183436 +183437,183437 +183438,183438 +183439,183439 +183440,183440 +183441,183441 +183442,183442 +183443,183443 +183444,183444 +183445,183445 +183446,183446 +183447,183447 +183448,183448 +183449,183449 +183450,183450 +183451,183451 +183452,183452 +123031,183452 +183453,183453 +244838,183453 +183454,183454 +90966,183454 +183455,183455 +183456,183456 +183457,183457 +183458,183458 +183459,183459 +183460,183460 +183461,183461 +183462,183462 +183463,183463 +183464,183464 +24702,183464 +183465,183465 +183466,183466 +183467,183467 +183468,183468 +183469,183469 +183470,183470 +183471,183471 +183472,183472 +21561,183472 +183473,183473 +183474,183474 +183475,183475 +183476,183476 +183477,183477 +183478,183478 +183479,183479 +183480,183480 +183481,183481 +183482,183482 +183483,183483 +183484,183484 +183485,183485 +183486,183486 +183487,183487 +183488,183488 +149050,183488 +183489,183489 +183490,183490 +183491,183491 +183492,183492 +183493,183493 +183494,183494 +183495,183495 +183496,183496 +183497,183497 +183498,183498 +183499,183499 +183500,183500 +183501,183501 +183502,183502 +183503,183503 +183504,183504 +183505,183505 +183506,183506 +183507,183507 +183508,183508 +183509,183509 +183510,183510 +56801,183510 +183511,183511 +183512,183512 +183513,183513 +183514,183514 +183515,183515 +183516,183516 +183517,183517 +183518,183518 +183519,183519 +183520,183520 +183521,183521 +183522,183522 +183523,183523 +183524,183524 +235606,183524 +183525,183525 +183526,183526 +183527,183527 +183528,183528 +183529,183529 +183530,183530 +183531,183531 +183532,183532 +183533,183533 +183534,183534 +183535,183535 +183536,183536 +183537,183537 +183538,183538 +183539,183539 +183540,183540 +183541,183541 +183542,183542 +183543,183543 +63493,183543 +183544,183544 +183545,183545 +183546,183546 +183547,183547 +183548,183548 +183549,183549 +183550,183550 +183551,183551 +207890,183551 +183552,183552 +183553,183553 +183554,183554 +183555,183555 +183556,183556 +183557,183557 +183558,183558 +183559,183559 +183560,183560 +183561,183561 +183562,183562 +183563,183563 +183564,183564 +11190,183564 +183565,183565 +183566,183566 +183567,183567 +183568,183568 +183569,183569 +183570,183570 +183571,183571 +183572,183572 +183573,183573 +174006,183573 +168042,183573 +183574,183574 +183575,183575 +183576,183576 +183577,183577 +183578,183578 +183579,183579 +183580,183580 +183581,183581 +183582,183582 +183583,183583 +183584,183584 +183585,183585 +183586,183586 +183587,183587 +183588,183588 +183589,183589 +183590,183590 +183591,183591 +183592,183592 +183593,183593 +183594,183594 +183595,183595 +183596,183596 +183597,183597 +183598,183598 +183599,183599 +183600,183600 +183601,183601 +183602,183602 +183603,183603 +183604,183604 +183605,183605 +183606,183606 +55560,183606 +47456,183606 +183607,183607 +183608,183608 +183609,183609 +183610,183610 +183611,183611 +183612,183612 +183613,183613 +183614,183614 +183615,183615 +183616,183616 +183617,183617 +183618,183618 +183619,183619 +75113,183619 +183620,183620 +183659,183659 +183660,183660 +183661,183661 +183662,183662 +183663,183663 +183664,183664 +224879,183664 +183665,183665 +183666,183666 +183667,183667 +139975,183667 +183668,183668 +183669,183669 +183670,183670 +183671,183671 +183672,183672 +183673,183673 +183674,183674 +183675,183675 +183676,183676 +183677,183677 +183678,183678 +183679,183679 +183680,183680 +183681,183681 +183682,183682 +183683,183683 +183684,183684 +183685,183685 +183686,183686 +183687,183687 +146842,183687 +183688,183688 +183689,183689 +183690,183690 +183731,183731 +183732,183732 +183733,183733 +183734,183734 +183735,183735 +183736,183736 +183737,183737 +183738,183738 +183739,183739 +183740,183740 +183741,183741 +183742,183742 +183743,183743 +183744,183744 +183745,183745 +183746,183746 +183747,183747 +183748,183748 +183749,183749 +183750,183750 +183751,183751 +183752,183752 +183753,183753 +183754,183754 +183755,183755 +183756,183756 +183757,183757 +183758,183758 +183759,183759 +183760,183760 +183761,183761 +207329,183761 +183762,183762 +183763,183763 +183764,183764 +183829,183829 +183830,183830 +206294,183830 +183831,183831 +183832,183832 +98227,183832 +183833,183833 +183834,183834 +164983,183834 +183835,183835 +183836,183836 +140453,183836 +183837,183837 +183838,183838 +183839,183839 +183840,183840 +183841,183841 +183842,183842 +183843,183843 +183844,183844 +183845,183845 +183846,183846 +183847,183847 +183848,183848 +183849,183849 +183850,183850 +183851,183851 +183852,183852 +183853,183853 +183854,183854 +183855,183855 +183856,183856 +183857,183857 +183858,183858 +183859,183859 +183860,183860 +183861,183861 +183862,183862 +183863,183863 +183864,183864 +183865,183865 +183866,183866 +159344,183866 +183867,183867 +207055,183867 +183868,183868 +141355,183868 +183869,183869 +183870,183870 +183871,183871 +183872,183872 +183873,183873 +183874,183874 +183875,183875 +120690,183875 +183876,183876 +183877,183877 +183878,183878 +183879,183879 +183880,183880 +183881,183881 +183882,183882 +183883,183883 +183884,183884 +183885,183885 +183886,183886 +244055,183886 +201406,183886 +183887,183887 +183888,183888 +183889,183889 +183890,183890 +183891,183891 +183892,183892 +205974,183892 +183893,183893 +183894,183894 +185084,183894 +183895,183895 +183896,183896 +183897,183897 +183898,183898 +183899,183899 +183900,183900 +183901,183901 +183902,183902 +183903,183903 +225083,183903 +183904,183904 +183905,183905 +107115,183905 +183906,183906 +183907,183907 +183908,183908 +184135,183908 +183909,183909 +183910,183910 +183911,183911 +183912,183912 +183913,183913 +183914,183914 +183915,183915 +183916,183916 +183917,183917 +183918,183918 +183919,183919 +183920,183920 +183921,183921 +183922,183922 +183923,183923 +183924,183924 +183925,183925 +183926,183926 +183927,183927 +183928,183928 +183929,183929 +183930,183930 +225200,183930 +183931,183931 +183932,183932 +106811,183932 +183933,183933 +183934,183934 +183935,183935 +183936,183936 +183937,183937 +183938,183938 +183939,183939 +183940,183940 +183941,183941 +183942,183942 +183943,183943 +183944,183944 +183945,183945 +183946,183946 +183947,183947 +183948,183948 +183949,183949 +183950,183950 +183951,183951 +183952,183952 +183953,183953 +183954,183954 +183955,183955 +247823,183955 +183956,183956 +183957,183957 +183958,183958 +183959,183959 +183960,183960 +183961,183961 +37291,183961 +183962,183962 +183963,183963 +183964,183964 +183965,183965 +29873,183965 +183966,183966 +183967,183967 +147543,183967 +215893,183967 +183968,183968 +183969,183969 +183970,183970 +183971,183971 +183972,183972 +184040,184040 +184041,184041 +184042,184042 +184043,184043 +184044,184044 +184045,184045 +184046,184046 +184047,184047 +184048,184048 +184049,184049 +184050,184050 +184051,184051 +184052,184052 +184053,184053 +184054,184054 +184055,184055 +184056,184056 +184057,184057 +184058,184058 +120430,184058 +184059,184059 +184060,184060 +184061,184061 +184062,184062 +184063,184063 +184064,184064 +184065,184065 +184066,184066 +184067,184067 +184068,184068 +184069,184069 +184070,184070 +184071,184071 +184072,184072 +184073,184073 +184074,184074 +184075,184075 +184076,184076 +233786,184076 +233048,184077 +184077,184077 +184078,184078 +184079,184079 +184080,184080 +184081,184081 +184082,184082 +14523,184082 +184083,184083 +154788,184083 +184084,184084 +20618,184084 +184123,184123 +184124,184124 +184125,184125 +184126,184126 +184127,184127 +184128,184128 +184129,184129 +184130,184130 +184131,184131 +184132,184132 +184133,184133 +184134,184134 +178838,184134 +184135,184135 +183908,184135 +184136,184136 +184137,184137 +184138,184138 +184139,184139 +184140,184140 +184141,184141 +220442,184141 +213818,184141 +233301,184141 +184142,184142 +184143,184143 +85599,184143 +110656,184143 +184144,184144 +184145,184145 +184146,184146 +184147,184147 +184148,184148 +184149,184149 +184150,184150 +184151,184151 +184152,184152 +184153,184153 +184154,184154 +184155,184155 +159448,184155 +184156,184156 +184157,184157 +184158,184158 +184159,184159 +184160,184160 +184161,184161 +92692,184161 +184162,184162 +184163,184163 +184164,184164 +184165,184165 +184166,184166 +184167,184167 +184168,184168 +184169,184169 +184170,184170 +184171,184171 +184172,184172 +184173,184173 +184174,184174 +184175,184175 +184176,184176 +184177,184177 +184178,184178 +184179,184179 +184180,184180 +184181,184181 +184182,184182 +184183,184183 +184184,184184 +184185,184185 +107106,184185 +184186,184186 +184187,184187 +184188,184188 +184189,184189 +184190,184190 +184191,184191 +184192,184192 +184193,184193 +184194,184194 +184195,184195 +184196,184196 +184197,184197 +184198,184198 +184199,184199 +184200,184200 +184201,184201 +184202,184202 +184203,184203 +184204,184204 +184205,184205 +184206,184206 +184207,184207 +184208,184208 +184209,184209 +184210,184210 +184211,184211 +184212,184212 +89171,184212 +184213,184213 +184214,184214 +184215,184215 +184216,184216 +184217,184217 +184218,184218 +184219,184219 +184220,184220 +184221,184221 +184222,184222 +184223,184223 +242320,184223 +184224,184224 +184225,184225 +184226,184226 +184227,184227 +184228,184228 +184229,184229 +184230,184230 +184231,184231 +184232,184232 +184233,184233 +184234,184234 +184235,184235 +184236,184236 +184237,184237 +184238,184238 +184239,184239 +184240,184240 +184241,184241 +184242,184242 +184243,184243 +184244,184244 +184245,184245 +235454,184245 +184246,184246 +184247,184247 +184248,184248 +184249,184249 +184250,184250 +184251,184251 +184252,184252 +184253,184253 +184254,184254 +77276,184254 +184255,184255 +184256,184256 +184257,184257 +184258,184258 +160165,184258 +184259,184259 +184260,184260 +184261,184261 +184262,184262 +184263,184263 +184264,184264 +184265,184265 +184266,184266 +184267,184267 +184268,184268 +184269,184269 +184270,184270 +184271,184271 +184272,184272 +184273,184273 +184274,184274 +184275,184275 +184276,184276 +184277,184277 +184278,184278 +184279,184279 +184280,184280 +184281,184281 +184282,184282 +184283,184283 +184284,184284 +143231,184284 +184285,184285 +184286,184286 +223441,184286 +184287,184287 +184288,184288 +184289,184289 +184290,184290 +184291,184291 +184292,184292 +184293,184293 +184294,184294 +184295,184295 +184296,184296 +184297,184297 +184298,184298 +184299,184299 +184300,184300 +184301,184301 +184302,184302 +184303,184303 +184304,184304 +184305,184305 +184306,184306 +184307,184307 +184308,184308 +229799,184308 +184309,184309 +184310,184310 +184311,184311 +184312,184312 +184313,184313 +184314,184314 +185927,184314 +184315,184315 +184316,184316 +184317,184317 +184318,184318 +184319,184319 +184320,184320 +185453,184320 +184321,184321 +111550,184321 +184322,184322 +184323,184323 +184324,184324 +184325,184325 +184326,184326 +184327,184327 +184328,184328 +184329,184329 +184330,184330 +184331,184331 +184332,184332 +184333,184333 +184334,184334 +184335,184335 +184336,184336 +184337,184337 +184338,184338 +184339,184339 +184340,184340 +184341,184341 +184342,184342 +184343,184343 +184344,184344 +184345,184345 +184346,184346 +184347,184347 +184348,184348 +184349,184349 +184350,184350 +184351,184351 +184352,184352 +184353,184353 +243,184353 +184354,184354 +184393,184393 +184394,184394 +184395,184395 +184396,184396 +184397,184397 +184398,184398 +184399,184399 +184400,184400 +184401,184401 +184402,184402 +184403,184403 +184404,184404 +184405,184405 +184406,184406 +184407,184407 +184408,184408 +184409,184409 +184410,184410 +184411,184411 +184412,184412 +184413,184413 +184414,184414 +184415,184415 +184416,184416 +184417,184417 +184418,184418 +184419,184419 +184420,184420 +184421,184421 +184422,184422 +51814,184422 +43710,184422 +184423,184423 +184424,184424 +184425,184425 +184426,184426 +184427,184427 +184428,184428 +184429,184429 +184430,184430 +184431,184431 +184432,184432 +184433,184433 +184434,184434 +184435,184435 +184436,184436 +10166,184436 +184437,184437 +184438,184438 +184439,184439 +184440,184440 +62391,184440 +184441,184441 +184442,184442 +184443,184443 +83943,184443 +184444,184444 +184445,184445 +184446,184446 +3198,184446 +184447,184447 +184448,184448 +184449,184449 +184450,184450 +184451,184451 +227964,184451 +184452,184452 +184453,184453 +196535,184453 +184454,184454 +184455,184455 +184456,184456 +151378,184456 +184457,184457 +184458,184458 +184459,184459 +184460,184460 +184461,184461 +184462,184462 +184463,184463 +184464,184464 +184465,184465 +184466,184466 +184467,184467 +184468,184468 +184469,184469 +111421,184469 +184470,184470 +184471,184471 +184472,184472 +184473,184473 +184474,184474 +184475,184475 +184476,184476 +81355,184476 +184477,184477 +184478,184478 +184479,184479 +184480,184480 +184481,184481 +184482,184482 +184483,184483 +110797,184483 +184484,184484 +184485,184485 +184486,184486 +184487,184487 +184488,184488 +184489,184489 +184490,184490 +184491,184491 +184492,184492 +233504,184492 +184493,184493 +184494,184494 +184495,184495 +184496,184496 +184497,184497 +184498,184498 +184499,184499 +184500,184500 +184501,184501 +184502,184502 +184503,184503 +184504,184504 +184505,184505 +184506,184506 +184507,184507 +184508,184508 +184509,184509 +184510,184510 +184511,184511 +184512,184512 +184513,184513 +184514,184514 +34037,184514 +184515,184515 +184516,184516 +184517,184517 +184518,184518 +184519,184519 +184520,184520 +184521,184521 +184522,184522 +184523,184523 +184524,184524 +184525,184525 +184526,184526 +184527,184527 +184528,184528 +184529,184529 +184530,184530 +184531,184531 +184532,184532 +184533,184533 +184534,184534 +184535,184535 +184536,184536 +184537,184537 +184538,184538 +184539,184539 +158638,184539 +184540,184540 +184541,184541 +184542,184542 +184543,184543 +184544,184544 +184545,184545 +184546,184546 +184547,184547 +184548,184548 +184549,184549 +184550,184550 +184551,184551 +184552,184552 +184553,184553 +184554,184554 +184555,184555 +184556,184556 +184557,184557 +184558,184558 +184559,184559 +184560,184560 +184561,184561 +217009,184561 +184562,184562 +184563,184563 +184564,184564 +184565,184565 +184566,184566 +184567,184567 +184568,184568 +225136,184568 +184569,184569 +200591,184569 +184570,184570 +184571,184571 +184572,184572 +184573,184573 +184574,184574 +184575,184575 +184576,184576 +184577,184577 +184578,184578 +184579,184579 +184580,184580 +184581,184581 +184582,184582 +184583,184583 +184584,184584 +184585,184585 +184586,184586 +184587,184587 +184588,184588 +184589,184589 +184590,184590 +184591,184591 +184592,184592 +184593,184593 +184594,184594 +184595,184595 +184596,184596 +184597,184597 +184598,184598 +184599,184599 +184600,184600 +184601,184601 +184602,184602 +184603,184603 +184604,184604 +184605,184605 +184606,184606 +184607,184607 +184608,184608 +184609,184609 +184610,184610 +184611,184611 +184612,184612 +184613,184613 +184614,184614 +184615,184615 +184616,184616 +184617,184617 +184618,184618 +184619,184619 +184620,184620 +184621,184621 +184622,184622 +184623,184623 +184624,184624 +184625,184625 +184626,184626 +184627,184627 +184628,184628 +184629,184629 +184630,184630 +184631,184631 +184632,184632 +184633,184633 +184634,184634 +184635,184635 +184636,184636 +184637,184637 +184638,184638 +184639,184639 +184640,184640 +184641,184641 +184642,184642 +184643,184643 +184644,184644 +184645,184645 +184646,184646 +184647,184647 +184648,184648 +184649,184649 +104033,184649 +184650,184650 +184651,184651 +184652,184652 +184653,184653 +184654,184654 +184655,184655 +184656,184656 +184657,184657 +51459,184657 +43355,184657 +184658,184658 +184659,184659 +184660,184660 +184661,184661 +184662,184662 +184663,184663 +184664,184664 +184665,184665 +184707,184707 +628,184707 +184708,184708 +184709,184709 +184710,184710 +184711,184711 +184712,184712 +184713,184713 +184714,184714 +184715,184715 +184716,184716 +184717,184717 +184718,184718 +184719,184719 +184720,184720 +184721,184721 +184722,184722 +184723,184723 +184724,184724 +184725,184725 +165769,184725 +184726,184726 +179794,184726 +184727,184727 +184728,184728 +184729,184729 +184730,184730 +184731,184731 +184732,184732 +184733,184733 +184734,184734 +184735,184735 +184736,184736 +184737,184737 +184738,184738 +184739,184739 +184740,184740 +184741,184741 +184742,184742 +184743,184743 +184744,184744 +184745,184745 +184746,184746 +184747,184747 +184748,184748 +184749,184749 +184750,184750 +184751,184751 +184752,184752 +184753,184753 +184754,184754 +184755,184755 +184756,184756 +184757,184757 +184758,184758 +184759,184759 +201143,184760 +184760,184760 +184761,184761 +184762,184762 +184763,184763 +184764,184764 +225241,184764 +184765,184765 +184766,184766 +78670,184766 +184767,184767 +204783,184767 +184768,184768 +184769,184769 +184770,184770 +184771,184771 +184772,184772 +153151,184772 +184773,184773 +184774,184774 +184775,184775 +184776,184776 +184777,184777 +184778,184778 +184779,184779 +14124,184779 +184780,184780 +184781,184781 +184782,184782 +184783,184783 +184784,184784 +184785,184785 +184786,184786 +184787,184787 +184788,184788 +184789,184789 +184790,184790 +184791,184791 +184792,184792 +184793,184793 +184794,184794 +184795,184795 +184796,184796 +184797,184797 +6140,184797 +184798,184798 +184799,184799 +184800,184800 +184801,184801 +184802,184802 +184803,184803 +184804,184804 +184805,184805 +184806,184806 +160173,184806 +184807,184807 +184808,184808 +184809,184809 +184810,184810 +184811,184811 +9097,184811 +184812,184812 +184813,184813 +184814,184814 +184815,184815 +184816,184816 +184817,184817 +39985,184817 +184818,184818 +184819,184819 +184820,184820 +184972,184972 +184973,184973 +7589,184973 +184974,184974 +184975,184975 +184976,184976 +184977,184977 +184978,184978 +184979,184979 +184980,184980 +184981,184981 +184982,184982 +184983,184983 +184984,184984 +184985,184985 +184986,184986 +184987,184987 +184988,184988 +184989,184989 +184990,184990 +184991,184991 +184992,184992 +184993,184993 +184994,184994 +184995,184995 +184996,184996 +184997,184997 +184998,184998 +184999,184999 +185000,185000 +185001,185001 +185002,185002 +185003,185003 +185004,185004 +185005,185005 +185006,185006 +185007,185007 +185008,185008 +185009,185009 +185010,185010 +87941,185010 +185011,185011 +185012,185012 +185013,185013 +185014,185014 +185015,185015 +185016,185016 +185017,185017 +185018,185018 +185019,185019 +185020,185020 +185021,185021 +185022,185022 +185023,185023 +185024,185024 +185025,185025 +185026,185026 +185027,185027 +185028,185028 +185029,185029 +185030,185030 +185031,185031 +185032,185032 +185033,185033 +185034,185034 +185035,185035 +185036,185036 +137096,185036 +129509,185036 +185037,185037 +185038,185038 +185039,185039 +185040,185040 +185041,185041 +185042,185042 +185043,185043 +185044,185044 +185045,185045 +185046,185046 +185047,185047 +185048,185048 +185049,185049 +185050,185050 +185051,185051 +185052,185052 +185053,185053 +185054,185054 +216178,185054 +185055,185055 +185056,185056 +185057,185057 +185058,185058 +185059,185059 +185060,185060 +185061,185061 +185062,185062 +185063,185063 +185064,185064 +185065,185065 +178884,185065 +185066,185066 +185067,185067 +185068,185068 +185069,185069 +221918,185069 +185070,185070 +185071,185071 +185072,185072 +185073,185073 +185074,185074 +185075,185075 +185076,185076 +185077,185077 +185078,185078 +185079,185079 +185080,185080 +185081,185081 +185082,185082 +185083,185083 +185084,185084 +183894,185084 +185085,185085 +185086,185086 +185087,185087 +185088,185088 +185089,185089 +185090,185090 +185091,185091 +156489,185091 +185092,185092 +185093,185093 +185094,185094 +185095,185095 +185096,185096 +213559,185096 +113400,185096 +185097,185097 +185098,185098 +185099,185099 +185100,185100 +185101,185101 +185102,185102 +185103,185103 +185145,185145 +185146,185146 +185147,185147 +185148,185148 +185149,185149 +157917,185149 +185150,185150 +185151,185151 +185152,185152 +185153,185153 +185154,185154 +185155,185155 +185156,185156 +185157,185157 +185158,185158 +185159,185159 +185160,185160 +185161,185161 +185162,185162 +185163,185163 +185164,185164 +185165,185165 +81622,185165 +185166,185166 +185167,185167 +185168,185168 +185169,185169 +185170,185170 +185171,185171 +185172,185172 +185173,185173 +158438,185173 +185174,185174 +185175,185175 +8581,185175 +8401,185175 +185176,185176 +185177,185177 +185178,185178 +185179,185179 +185180,185180 +185181,185181 +199863,185181 +185182,185182 +185183,185183 +185184,185184 +185185,185185 +185186,185186 +185187,185187 +185188,185188 +185189,185189 +185190,185190 +185191,185191 +185192,185192 +49323,185192 +41130,185192 +185193,185193 +185194,185194 +185195,185195 +17230,185195 +185196,185196 +185197,185197 +185198,185198 +185276,185276 +185277,185277 +103412,185277 +185278,185278 +185279,185279 +185280,185280 +120406,185280 +185281,185281 +185282,185282 +185283,185283 +185284,185284 +185285,185285 +185286,185286 +572,185286 +185287,185287 +142351,185287 +185288,185288 +185289,185289 +118468,185289 +185290,185290 +185291,185291 +185292,185292 +185293,185293 +211350,185293 +185294,185294 +185295,185295 +185296,185296 +185297,185297 +185298,185298 +185299,185299 +185300,185300 +185301,185301 +185302,185302 +185303,185303 +185304,185304 +185305,185305 +80043,185305 +185306,185306 +185307,185307 +185308,185308 +185309,185309 +185310,185310 +185311,185311 +185312,185312 +185313,185313 +185314,185314 +185315,185315 +185316,185316 +185317,185317 +185318,185318 +185319,185319 +185320,185320 +185321,185321 +185322,185322 +185323,185323 +185324,185324 +185325,185325 +185326,185326 +185327,185327 +185328,185328 +185329,185329 +185330,185330 +185331,185331 +92068,185331 +185332,185332 +185333,185333 +185334,185334 +185335,185335 +185336,185336 +185337,185337 +185338,185338 +185339,185339 +185340,185340 +185341,185341 +185342,185342 +185343,185343 +185344,185344 +185345,185345 +185346,185346 +185347,185347 +185348,185348 +187207,185348 +185349,185349 +185350,185350 +185351,185351 +185352,185352 +185353,185353 +79867,185353 +185354,185354 +185355,185355 +185356,185356 +185357,185357 +185358,185358 +185359,185359 +185360,185360 +185361,185361 +100590,185361 +185362,185362 +185363,185363 +185364,185364 +185365,185365 +185366,185366 +185367,185367 +185368,185368 +138678,185368 +185369,185369 +185370,185370 +5525,185370 +185371,185371 +185372,185372 +185373,185373 +185374,185374 +185375,185375 +185376,185376 +185377,185377 +185378,185378 +185379,185379 +185380,185380 +185381,185381 +185382,185382 +216770,185382 +185383,185383 +185384,185384 +185385,185385 +185386,185386 +185387,185387 +185388,185388 +185389,185389 +185390,185390 +185391,185391 +123507,185391 +106125,185391 +185392,185392 +185393,185393 +185394,185394 +185395,185395 +185396,185396 +185397,185397 +185398,185398 +185399,185399 +185400,185400 +185401,185401 +185402,185402 +185403,185403 +215605,185403 +5828,185403 +52424,185403 +44320,185403 +36352,185403 +185404,185404 +185405,185405 +226769,185405 +185406,185406 +14427,185406 +185407,185407 +154940,185407 +185408,185408 +185409,185409 +242345,185409 +185410,185410 +100859,185410 +185411,185411 +185412,185412 +185413,185413 +185414,185414 +185415,185415 +185416,185416 +185417,185417 +185418,185418 +185419,185419 +185420,185420 +185421,185421 +185422,185422 +185423,185423 +185424,185424 +185425,185425 +185426,185426 +185427,185427 +185428,185428 +185429,185429 +185430,185430 +185431,185431 +185432,185432 +185433,185433 +185434,185434 +185435,185435 +185436,185436 +185437,185437 +185438,185438 +6700,185438 +4588,185438 +185439,185439 +185440,185440 +185441,185441 +185442,185442 +4581,185442 +185443,185443 +185444,185444 +185445,185445 +231846,185445 +185446,185446 +185447,185447 +185448,185448 +185449,185449 +185450,185450 +185451,185451 +185452,185452 +146523,185452 +185453,185453 +184320,185453 +185454,185454 +185455,185455 +185456,185456 +54693,185456 +46589,185456 +142560,185456 +185457,185457 +185458,185458 +185459,185459 +185460,185460 +185461,185461 +185462,185462 +50018,185462 +41825,185462 +185463,185463 +185464,185464 +185465,185465 +185466,185466 +185467,185467 +185468,185468 +185469,185469 +185470,185470 +185471,185471 +185472,185472 +185473,185473 +185474,185474 +185475,185475 +185476,185476 +185477,185477 +185478,185478 +185479,185479 +185480,185480 +185481,185481 +185482,185482 +185483,185483 +185484,185484 +185485,185485 +185486,185486 +185487,185487 +185488,185488 +185489,185489 +185490,185490 +185491,185491 +185492,185492 +185493,185493 +185494,185494 +185495,185495 +185496,185496 +185497,185497 +222018,185497 +185498,185498 +185499,185499 +185500,185500 +10550,185500 +185501,185501 +185502,185502 +185503,185503 +185504,185504 +185505,185505 +185506,185506 +185507,185507 +185508,185508 +185509,185509 +193698,185509 +185510,185510 +185511,185511 +185512,185512 +185513,185513 +185514,185514 +185515,185515 +185516,185516 +142632,185516 +185517,185517 +75145,185517 +185518,185518 +185519,185519 +185520,185520 +185521,185521 +185522,185522 +185558,185558 +185559,185559 +185560,185560 +185561,185561 +185562,185562 +185563,185563 +185564,185564 +28135,185564 +185565,185565 +185566,185566 +214438,185566 +185567,185567 +185568,185568 +122756,185568 +185569,185569 +185570,185570 +185571,185571 +185572,185572 +185573,185573 +185574,185574 +185575,185575 +185576,185576 +185577,185577 +185578,185578 +185579,185579 +75067,185579 +185580,185580 +185581,185581 +185582,185582 +185583,185583 +185584,185584 +185585,185585 +185586,185586 +185587,185587 +185588,185588 +185589,185589 +185590,185590 +185591,185591 +185592,185592 +185593,185593 +185594,185594 +185595,185595 +185596,185596 +185597,185597 +185598,185598 +185599,185599 +185600,185600 +185601,185601 +185602,185602 +185603,185603 +185604,185604 +185605,185605 +185606,185606 +185607,185607 +185608,185608 +185609,185609 +185610,185610 +185611,185611 +185612,185612 +185613,185613 +185614,185614 +185615,185615 +185616,185616 +185617,185617 +185618,185618 +89228,185618 +185619,185619 +7738,185619 +185620,185620 +185621,185621 +185622,185622 +185623,185623 +185624,185624 +185625,185625 +185669,185669 +185670,185670 +185671,185671 +185672,185672 +185673,185673 +185674,185674 +185675,185675 +65104,185675 +185676,185676 +121305,185676 +185677,185677 +185678,185678 +185679,185679 +185680,185680 +185681,185681 +185682,185682 +185683,185683 +185684,185684 +185685,185685 +185686,185686 +185687,185687 +185688,185688 +185689,185689 +185690,185690 +185691,185691 +185692,185692 +185693,185693 +185694,185694 +185695,185695 +185696,185696 +185697,185697 +185698,185698 +27123,185698 +185699,185699 +185700,185700 +249588,185700 +185701,185701 +185702,185702 +247879,185702 +185703,185703 +185704,185704 +185705,185705 +185706,185706 +185707,185707 +185708,185708 +185709,185709 +185710,185710 +185711,185711 +185712,185712 +185713,185713 +185714,185714 +185715,185715 +185716,185716 +185717,185717 +185718,185718 +185719,185719 +185720,185720 +187051,185720 +185721,185721 +185722,185722 +137026,185722 +129439,185722 +185723,185723 +185724,185724 +185725,185725 +185726,185726 +185727,185727 +185728,185728 +185729,185729 +185730,185730 +185731,185731 +185732,185732 +185733,185733 +185734,185734 +81534,185734 +185735,185735 +185736,185736 +185737,185737 +185738,185738 +185739,185739 +185740,185740 +185741,185741 +185742,185742 +226102,185742 +185743,185743 +185744,185744 +185745,185745 +185746,185746 +232039,185746 +185747,185747 +185748,185748 +185749,185749 +185750,185750 +185751,185751 +185752,185752 +185817,185817 +185818,185818 +185819,185819 +185820,185820 +185821,185821 +185822,185822 +185823,185823 +185824,185824 +185825,185825 +185826,185826 +185827,185827 +134116,185827 +126529,185827 +185828,185828 +185829,185829 +185830,185830 +185831,185831 +185832,185832 +185833,185833 +185834,185834 +185835,185835 +185836,185836 +185837,185837 +185838,185838 +185839,185839 +185840,185840 +185841,185841 +185842,185842 +185843,185843 +185844,185844 +185845,185845 +185846,185846 +185847,185847 +185848,185848 +185849,185849 +185850,185850 +185851,185851 +185852,185852 +185853,185853 +185854,185854 +185855,185855 +185856,185856 +185857,185857 +185858,185858 +185859,185859 +185860,185860 +185861,185861 +185862,185862 +185863,185863 +185864,185864 +185865,185865 +185866,185866 +185867,185867 +185868,185868 +185869,185869 +185870,185870 +185871,185871 +158779,185871 +185872,185872 +185873,185873 +185874,185874 +185875,185875 +185876,185876 +185877,185877 +185878,185878 +185879,185879 +185880,185880 +185881,185881 +185882,185882 +222049,185882 +185883,185883 +185884,185884 +185885,185885 +185886,185886 +222183,185887 +185887,185887 +185888,185888 +185889,185889 +185890,185890 +185891,185891 +185892,185892 +232984,185892 +185893,185893 +185894,185894 +222190,185894 +185895,185895 +92086,185895 +185896,185896 +185897,185897 +185898,185898 +185899,185899 +185900,185900 +185901,185901 +103650,185901 +185902,185902 +185903,185903 +185904,185904 +185905,185905 +185906,185906 +149897,185906 +185907,185907 +185908,185908 +185909,185909 +185910,185910 +105015,185910 +185911,185911 +185912,185912 +185913,185913 +185914,185914 +185915,185915 +185916,185916 +185917,185917 +185918,185918 +185919,185919 +185920,185920 +185921,185921 +54174,185921 +46070,185921 +185922,185922 +185923,185923 +185924,185924 +185925,185925 +185926,185926 +185927,185927 +184314,185927 +185928,185928 +185929,185929 +185930,185930 +185931,185931 +185932,185932 +185933,185933 +185934,185934 +185935,185935 +185936,185936 +185937,185937 +185938,185938 +185939,185939 +65459,185939 +185940,185940 +185941,185941 +218446,185941 +185942,185942 +185943,185943 +2922,185943 +185944,185944 +218214,185944 +185945,185945 +185946,185946 +185947,185947 +185948,185948 +185949,185949 +185950,185950 +185951,185951 +185952,185952 +185953,185953 +185954,185954 +185955,185955 +186043,186043 +186044,186044 +186045,186045 +153019,186045 +186046,186046 +186047,186047 +208820,186047 +186048,186048 +186049,186049 +186050,186050 +186051,186051 +51161,186051 +43057,186051 +186052,186052 +186053,186053 +186054,186054 +186055,186055 +186056,186056 +22042,186056 +186057,186057 +186058,186058 +221291,186058 +186059,186059 +186164,186164 +186165,186165 +186166,186166 +186167,186167 +146406,186167 +186168,186168 +186169,186169 +186283,186283 +186284,186284 +186285,186285 +186286,186286 +186287,186287 +186288,186288 +186289,186289 +186290,186290 +186291,186291 +186292,186292 +186293,186293 +186294,186294 +186295,186295 +186296,186296 +186297,186297 +186298,186298 +186299,186299 +186300,186300 +186301,186301 +186302,186302 +186303,186303 +186304,186304 +186305,186305 +186306,186306 +186307,186307 +186308,186308 +186309,186309 +186310,186310 +186311,186311 +186312,186312 +186313,186313 +186314,186314 +186315,186315 +186316,186316 +227606,186316 +186317,186317 +199217,186317 +186318,186318 +186319,186319 +186320,186320 +11465,186320 +186361,186361 +186362,186362 +186363,186363 +186364,186364 +186365,186365 +186366,186366 +164895,186366 +186367,186367 +186368,186368 +186369,186369 +186370,186370 +186371,186371 +186372,186372 +186373,186373 +186374,186374 +186375,186375 +186376,186376 +186377,186377 +186378,186378 +186379,186379 +186380,186380 +186381,186381 +186382,186382 +186383,186383 +186384,186384 +186385,186385 +186386,186386 +19875,186386 +186387,186387 +186388,186388 +186389,186389 +186390,186390 +197506,186390 +186391,186391 +186392,186392 +186393,186393 +186394,186394 +186395,186395 +186396,186396 +186397,186397 +186398,186398 +186399,186399 +186400,186400 +186401,186401 +186402,186402 +186403,186403 +186404,186404 +186405,186405 +186406,186406 +186407,186407 +186408,186408 +186409,186409 +186410,186410 +186411,186411 +186412,186412 +186413,186413 +186414,186414 +186415,186415 +186416,186416 +87531,186416 +186417,186417 +186418,186418 +186419,186419 +95890,186419 +186420,186420 +186421,186421 +186422,186422 +186423,186423 +186424,186424 +186425,186425 +186426,186426 +186427,186427 +186554,186554 +186555,186555 +186556,186556 +186557,186557 +135287,186557 +127700,186557 +186703,186703 +186704,186704 +186705,186705 +186706,186706 +186707,186707 +186793,186793 +186794,186794 +186795,186795 +231298,186795 +76375,186795 +156831,186795 +186796,186796 +186797,186797 +186798,186798 +186799,186799 +186800,186800 +158054,186800 +186801,186801 +186802,186802 +186803,186803 +186804,186804 +186805,186805 +186806,186806 +186807,186807 +186808,186808 +186809,186809 +186918,186918 +186919,186919 +186920,186920 +186921,186921 +186922,186922 +186960,186960 +76318,186960 +186961,186961 +186962,186962 +187003,187003 +187004,187004 +187005,187005 +187006,187006 +177585,187006 +171621,187006 +187007,187007 +187008,187008 +187009,187009 +187010,187010 +187011,187011 +187012,187012 +187013,187013 +187014,187014 +187015,187015 +81855,187015 +187016,187016 +187017,187017 +187018,187018 +187019,187019 +187020,187020 +187021,187021 +187022,187022 +187023,187023 +187024,187024 +187025,187025 +187026,187026 +187027,187027 +62690,187027 +187028,187028 +187029,187029 +187030,187030 +187031,187031 +187032,187032 +187033,187033 +187034,187034 +187035,187035 +187036,187036 +187037,187037 +187038,187038 +187039,187039 +187040,187040 +187041,187041 +187042,187042 +187043,187043 +187044,187044 +187045,187045 +187046,187046 +164903,187046 +187047,187047 +187048,187048 +207434,187048 +187049,187049 +163990,187049 +187050,187050 +187051,187051 +185720,187051 +187052,187052 +187053,187053 +187054,187054 +187055,187055 +160776,187055 +187056,187056 +187057,187057 +187058,187058 +187059,187059 +187060,187060 +187061,187061 +187062,187062 +187063,187063 +187064,187064 +187065,187065 +187066,187066 +187067,187067 +187068,187068 +187069,187069 +187070,187070 +187071,187071 +187072,187072 +187073,187073 +187074,187074 +187075,187075 +187076,187076 +187077,187077 +187078,187078 +187079,187079 +187080,187080 +187081,187081 +187082,187082 +187083,187083 +187084,187084 +134181,187084 +126594,187084 +51062,187084 +42958,187084 +187085,187085 +187086,187086 +8770,187086 +187087,187087 +187088,187088 +187089,187089 +187090,187090 +187091,187091 +951,187091 +187092,187092 +187093,187093 +187094,187094 +224411,187094 +187095,187095 +187096,187096 +204353,187096 +187097,187097 +242926,187097 +20217,187097 +187098,187098 +187099,187099 +187100,187100 +187101,187101 +187102,187102 +187103,187103 +187104,187104 +187105,187105 +105780,187105 +187106,187106 +187107,187107 +187108,187108 +187109,187109 +187110,187110 +187111,187111 +187112,187112 +187113,187113 +187114,187114 +187115,187115 +247300,187115 +187116,187116 +187117,187117 +187118,187118 +187119,187119 +187120,187120 +198700,187121 +187121,187121 +193687,187121 +187122,187122 +187123,187123 +187124,187124 +187125,187125 +187126,187126 +187127,187127 +187128,187128 +187129,187129 +6015,187129 +187130,187130 +187131,187131 +187132,187132 +187133,187133 +187134,187134 +187135,187135 +187136,187136 +187137,187137 +187138,187138 +187139,187139 +187140,187140 +187141,187141 +187142,187142 +187143,187143 +187144,187144 +187145,187145 +187146,187146 +187147,187147 +187148,187148 +187149,187149 +187150,187150 +187151,187151 +187152,187152 +187153,187153 +187154,187154 +187155,187155 +187156,187156 +187157,187157 +187158,187158 +187159,187159 +187160,187160 +187161,187161 +187162,187162 +187163,187163 +187164,187164 +137046,187164 +129459,187164 +187165,187165 +6909,187165 +187166,187166 +187167,187167 +187168,187168 +187169,187169 +187170,187170 +187171,187171 +187172,187172 +187173,187173 +187174,187174 +187175,187175 +187176,187176 +187177,187177 +187178,187178 +187179,187179 +187180,187180 +187181,187181 +187182,187182 +187183,187183 +187184,187184 +187185,187185 +187186,187186 +122804,187186 +187187,187187 +187188,187188 +187189,187189 +187190,187190 +187191,187191 +187192,187192 +187193,187193 +187194,187194 +187195,187195 +38989,187195 +187196,187196 +187197,187197 +187198,187198 +187199,187199 +187200,187200 +187201,187201 +187202,187202 +187203,187203 +187204,187204 +192170,187204 +187205,187205 +20918,187205 +187206,187206 +187207,187207 +185348,187207 +96373,187207 +187208,187208 +187209,187209 +187210,187210 +92126,187210 +187211,187211 +187212,187212 +187213,187213 +187214,187214 +187215,187215 +187216,187216 +187217,187217 +187218,187218 +187219,187219 +187220,187220 +187221,187221 +187222,187222 +187223,187223 +187224,187224 +187225,187225 +187226,187226 +187227,187227 +187228,187228 +187229,187229 +191850,187229 +187230,187230 +187231,187231 +187232,187232 +187233,187233 +187234,187234 +187235,187235 +187236,187236 +187237,187237 +187238,187238 +187239,187239 +187240,187240 +187241,187241 +81602,187241 +187242,187242 +187243,187243 +187244,187244 +187245,187245 +62868,187245 +187246,187246 +26756,187246 +187247,187247 +187248,187248 +187249,187249 +187250,187250 +187251,187251 +187252,187252 +187253,187253 +187254,187254 +187255,187255 +187256,187256 +187257,187257 +187258,187258 +187259,187259 +88243,187259 +187260,187260 +187261,187261 +187262,187262 +187263,187263 +187961,187263 +187264,187264 +187265,187265 +187266,187266 +187267,187267 +187268,187268 +187269,187269 +187270,187270 +107437,187270 +187271,187271 +187272,187272 +187273,187273 +187274,187274 +187275,187275 +187276,187276 +187277,187277 +178957,187277 +187278,187278 +187279,187279 +187280,187280 +187281,187281 +33640,187281 +187282,187282 +187283,187283 +244817,187283 +187284,187284 +187285,187285 +187286,187286 +187287,187287 +187288,187288 +187289,187289 +187290,187290 +187291,187291 +187292,187292 +187293,187293 +187294,187294 +145126,187294 +178477,187294 +172513,187294 +187295,187295 +187296,187296 +190016,187296 +187297,187297 +187298,187298 +187299,187299 +187300,187300 +187301,187301 +187302,187302 +187303,187303 +187304,187304 +187305,187305 +187306,187306 +187307,187307 +187308,187308 +187309,187309 +187310,187310 +187311,187311 +187312,187312 +187313,187313 +187314,187314 +187315,187315 +187316,187316 +52892,187316 +44788,187316 +187317,187317 +82354,187317 +187318,187318 +187319,187319 +187320,187320 +187321,187321 +221163,187321 +187322,187322 +187323,187323 +187324,187324 +187325,187325 +187326,187326 +187327,187327 +142748,187327 +187328,187328 +187329,187329 +187330,187330 +187331,187331 +187332,187332 +187333,187333 +187334,187334 +187335,187335 +187336,187336 +187337,187337 +187338,187338 +187339,187339 +187340,187340 +187341,187341 +187342,187342 +187343,187343 +187344,187344 +187345,187345 +187346,187346 +187347,187347 +187348,187348 +187349,187349 +187350,187350 +187351,187351 +187352,187352 +187353,187353 +187354,187354 +187355,187355 +187356,187356 +187357,187357 +187358,187358 +187359,187359 +187360,187360 +187361,187361 +187362,187362 +187363,187363 +187364,187364 +118,187364 +187365,187365 +187366,187366 +187367,187367 +187368,187368 +187369,187369 +187370,187370 +187371,187371 +187372,187372 +4369,187372 +187373,187373 +187374,187374 +187375,187375 +79337,187375 +187376,187376 +187377,187377 +187378,187378 +131976,187378 +124389,187378 +221879,187378 +187379,187379 +187380,187380 +187381,187381 +73072,187381 +122597,187381 +187382,187382 +187383,187383 +187384,187384 +206752,187384 +187385,187385 +187386,187386 +187387,187387 +187388,187388 +187389,187389 +187390,187390 +187391,187391 +187392,187392 +187393,187393 +187394,187394 +187395,187395 +31348,187395 +187396,187396 +187397,187397 +187398,187398 +187399,187399 +240072,187399 +187400,187400 +187401,187401 +187402,187402 +187403,187403 +187404,187404 +187405,187405 +90423,187405 +187406,187406 +187407,187407 +187408,187408 +151418,187408 +187409,187409 +187410,187410 +187411,187411 +187412,187412 +187413,187413 +187414,187414 +187415,187415 +187416,187416 +187417,187417 +187418,187418 +229,187418 +187419,187419 +187420,187420 +187421,187421 +187422,187422 +187423,187423 +187424,187424 +187425,187425 +187426,187426 +187427,187427 +187428,187428 +187429,187429 +187430,187430 +15734,187430 +224564,187430 +187431,187431 +187432,187432 +56680,187432 +48576,187432 +187433,187433 +187434,187434 +187435,187435 +187436,187436 +187437,187437 +187438,187438 +187439,187439 +187440,187440 +187441,187441 +187442,187442 +187443,187443 +187444,187444 +187445,187445 +187446,187446 +53243,187446 +45139,187446 +187447,187447 +187448,187448 +187449,187449 +187450,187450 +75400,187450 +187451,187451 +187452,187452 +187453,187453 +187454,187454 +187455,187455 +187456,187456 +187457,187457 +187458,187458 +187459,187459 +187460,187460 +187461,187461 +139777,187461 +187462,187462 +187463,187463 +187464,187464 +187465,187465 +187466,187466 +187467,187467 +55950,187467 +47846,187467 +187468,187468 +56251,187468 +48147,187468 +187469,187469 +12348,187469 +187470,187470 +187471,187471 +187472,187472 +187473,187473 +187474,187474 +187475,187475 +187476,187476 +187477,187477 +187478,187478 +187479,187479 +187480,187480 +187481,187481 +187482,187482 +187483,187483 +187642,187642 +187745,187745 +187746,187746 +187747,187747 +187748,187748 +229177,187748 +187749,187749 +142686,187749 +187750,187750 +187751,187751 +187752,187752 +187753,187753 +187754,187754 +187755,187755 +187756,187756 +187757,187757 +93178,187757 +187758,187758 +245440,187758 +187759,187759 +187760,187760 +187761,187761 +187762,187762 +187763,187763 +187764,187764 +187765,187765 +187766,187766 +187767,187767 +187768,187768 +187769,187769 +187770,187770 +187771,187771 +187772,187772 +187773,187773 +187774,187774 +187775,187775 +187776,187776 +187777,187777 +36928,187777 +187778,187778 +187779,187779 +187780,187780 +187781,187781 +187872,187872 +187873,187873 +187874,187874 +187875,187875 +187945,187945 +187946,187946 +187947,187947 +187948,187948 +187949,187949 +187950,187950 +187951,187951 +187952,187952 +110095,187952 +187953,187953 +187954,187954 +187955,187955 +187956,187956 +187957,187957 +187958,187958 +187959,187959 +187960,187960 +187961,187961 +187263,187961 +187962,187962 +187963,187963 +187964,187964 +187965,187965 +187966,187966 +187967,187967 +187968,187968 +187969,187969 +187970,187970 +187971,187971 +187972,187972 +187973,187973 +187974,187974 +187975,187975 +187976,187976 +187977,187977 +187978,187978 +187979,187979 +187980,187980 +187981,187981 +187982,187982 +187983,187983 +162520,187983 +187984,187984 +187985,187985 +187986,187986 +187987,187987 +187988,187988 +187989,187989 +187990,187990 +187991,187991 +187992,187992 +187993,187993 +187994,187994 +187995,187995 +187996,187996 +187997,187997 +187998,187998 +187999,187999 +188020,187999 +188000,188000 +188001,188001 +188002,188002 +188003,188003 +188004,188004 +188005,188005 +188006,188006 +188007,188007 +188008,188008 +188009,188009 +188010,188010 +188011,188011 +188012,188012 +188013,188013 +188014,188014 +188015,188015 +188016,188016 +188017,188017 +188018,188018 +188019,188019 +188020,188020 +187999,188020 +188021,188021 +188022,188022 +188023,188023 +188024,188024 +188025,188025 +188026,188026 +188027,188027 +188028,188028 +211946,188028 +188029,188029 +188030,188030 +188031,188031 +188032,188032 +110112,188032 +188033,188033 +188034,188034 +188035,188035 +38389,188035 +139948,188035 +188036,188036 +188037,188037 +188038,188038 +188039,188039 +188040,188040 +188041,188041 +188042,188042 +188043,188043 +188044,188044 +111787,188044 +188045,188045 +188046,188046 +234330,188046 +188047,188047 +176773,188047 +170809,188047 +188048,188048 +188049,188049 +188050,188050 +188051,188051 +188052,188052 +188090,188090 +188091,188091 +188092,188092 +188093,188093 +188094,188094 +188095,188095 +188185,188185 +188186,188186 +188187,188187 +188188,188188 +188189,188189 +188190,188190 +188191,188191 +188192,188192 +188193,188193 +4954,188193 +188194,188194 +188195,188195 +248703,188195 +188196,188196 +188197,188197 +188198,188198 +188199,188199 +188200,188200 +188201,188201 +188202,188202 +188203,188203 +188204,188204 +188205,188205 +188206,188206 +188207,188207 +188208,188208 +188209,188209 +188210,188210 +188211,188211 +188212,188212 +188213,188213 +188214,188214 +188215,188215 +188216,188216 +188217,188217 +140714,188217 +188218,188218 +188219,188219 +188552,188219 +188220,188220 +188221,188221 +146511,188221 +188222,188222 +188223,188223 +188224,188224 +188225,188225 +188226,188226 +188227,188227 +188228,188228 +209665,188228 +188229,188229 +188230,188230 +163081,188230 +188231,188231 +188232,188232 +188233,188233 +188234,188234 +188235,188235 +188236,188236 +188237,188237 +188238,188238 +188239,188239 +188240,188240 +188241,188241 +188242,188242 +176562,188242 +170598,188242 +188243,188243 +188244,188244 +188245,188245 +188246,188246 +188247,188247 +188248,188248 +188249,188249 +188250,188250 +188251,188251 +188252,188252 +188253,188253 +188254,188254 +188255,188255 +188256,188256 +30932,188256 +188257,188257 +188258,188258 +188259,188259 +188260,188260 +188261,188261 +188262,188262 +188263,188263 +188264,188264 +188265,188265 +188266,188266 +188267,188267 +188268,188268 +188269,188269 +188270,188270 +188271,188271 +188272,188272 +188273,188273 +188274,188274 +188275,188275 +188276,188276 +188277,188277 +188317,188317 +188318,188318 +188319,188319 +188320,188320 +188321,188321 +188322,188322 +188323,188323 +188324,188324 +188325,188325 +188326,188326 +188327,188327 +188328,188328 +188329,188329 +188330,188330 +188331,188331 +165308,188331 +188332,188332 +188333,188333 +188334,188334 +188335,188335 +188336,188336 +188337,188337 +188338,188338 +102194,188338 +188339,188339 +188340,188340 +188341,188341 +188342,188342 +115982,188342 +188343,188343 +188344,188344 +188345,188345 +188346,188346 +188347,188347 +188348,188348 +188349,188349 +188350,188350 +188351,188351 +188352,188352 +188353,188353 +188354,188354 +188355,188355 +209395,188355 +188356,188356 +188357,188357 +188358,188358 +188359,188359 +188360,188360 +188361,188361 +188362,188362 +188363,188363 +188364,188364 +188365,188365 +188366,188366 +188367,188367 +188368,188368 +142019,188368 +188369,188369 +188370,188370 +188371,188371 +188372,188372 +121365,188372 +188373,188373 +188374,188374 +188375,188375 +188376,188376 +188377,188377 +188378,188378 +188379,188379 +188380,188380 +188381,188381 +188382,188382 +188383,188383 +188384,188384 +188385,188385 +188386,188386 +188387,188387 +188388,188388 +188389,188389 +188390,188390 +188391,188391 +188392,188392 +188393,188393 +188394,188394 +188395,188395 +188396,188396 +188397,188397 +188398,188398 +188399,188399 +188400,188400 +188401,188401 +188402,188402 +188403,188403 +188404,188404 +188405,188405 +188406,188406 +220200,188406 +163054,188406 +188407,188407 +188408,188408 +188409,188409 +188410,188410 +188411,188411 +188473,188473 +188474,188474 +188475,188475 +188476,188476 +188477,188477 +188478,188478 +188479,188479 +188480,188480 +188481,188481 +188482,188482 +38362,188482 +188483,188483 +188484,188484 +188485,188485 +180701,188485 +188486,188486 +188487,188487 +84676,188487 +188488,188488 +188489,188489 +188490,188490 +188491,188491 +188492,188492 +188493,188493 +188494,188494 +188495,188495 +180401,188495 +188496,188496 +188497,188497 +188498,188498 +188499,188499 +188500,188500 +188501,188501 +188502,188502 +188503,188503 +188504,188504 +188505,188505 +188506,188506 +188507,188507 +188508,188508 +188509,188509 +60365,188509 +188510,188510 +188511,188511 +188512,188512 +236892,188512 +115469,188512 +188513,188513 +188514,188514 +188515,188515 +188516,188516 +188517,188517 +188518,188518 +188519,188519 +188520,188520 +137157,188520 +129570,188520 +188521,188521 +188522,188522 +188523,188523 +188524,188524 +188525,188525 +188526,188526 +188527,188527 +188528,188528 +188529,188529 +226729,188529 +188530,188530 +188531,188531 +188532,188532 +188533,188533 +188534,188534 +188535,188535 +188536,188536 +188537,188537 +188538,188538 +188539,188539 +188540,188540 +188541,188541 +188542,188542 +188543,188543 +188544,188544 +188545,188545 +188546,188546 +188547,188547 +188548,188548 +188549,188549 +188550,188550 +188551,188551 +188552,188552 +188219,188552 +188553,188553 +188554,188554 +188555,188555 +188556,188556 +188557,188557 +188558,188558 +188559,188559 +188560,188560 +247447,188560 +188596,188596 +188597,188597 +188691,188691 +188692,188692 +188693,188693 +222415,188693 +220895,188693 +188694,188694 +188695,188695 +188696,188696 +201676,188697 +188697,188697 +188698,188698 +188699,188699 +188700,188700 +188701,188701 +60811,188701 +188702,188702 +188703,188703 +188704,188704 +188705,188705 +247902,188705 +188706,188706 +188707,188707 +188708,188708 +188709,188709 +188710,188710 +188748,188748 +153949,188748 +188749,188749 +188970,188970 +188971,188971 +188972,188972 +188973,188973 +188974,188974 +188975,188975 +188976,188976 +188977,188977 +179158,188977 +188978,188978 +188979,188979 +188980,188980 +188981,188981 +110612,188981 +188982,188982 +188983,188983 +188984,188984 +188985,188985 +188986,188986 +188987,188987 +188988,188988 +188989,188989 +188990,188990 +188991,188991 +188992,188992 +188993,188993 +188994,188994 +222055,188994 +62563,188994 +188995,188995 +188996,188996 +188997,188997 +188998,188998 +188999,188999 +189000,189000 +189001,189001 +189002,189002 +189003,189003 +189004,189004 +189005,189005 +189006,189006 +189007,189007 +189008,189008 +189009,189009 +189010,189010 +189011,189011 +189012,189012 +189013,189013 +189014,189014 +189015,189015 +189016,189016 +189017,189017 +189018,189018 +189019,189019 +189020,189020 +189021,189021 +247226,189021 +189022,189022 +189023,189023 +189024,189024 +189025,189025 +189026,189026 +189027,189027 +228853,189027 +189028,189028 +189029,189029 +189030,189030 +189031,189031 +21209,189031 +189032,189032 +82849,189032 +189033,189033 +189034,189034 +189035,189035 +189036,189036 +189037,189037 +189038,189038 +189039,189039 +83012,189039 +189040,189040 +189041,189041 +189042,189042 +189043,189043 +189044,189044 +189045,189045 +189046,189046 +189047,189047 +189048,189048 +189049,189049 +189050,189050 +189051,189051 +189052,189052 +189053,189053 +189054,189054 +189055,189055 +189056,189056 +189057,189057 +189058,189058 +189059,189059 +189060,189060 +189061,189061 +189062,189062 +189063,189063 +189064,189064 +189065,189065 +189066,189066 +159423,189066 +189067,189067 +189068,189068 +189069,189069 +189070,189070 +189071,189071 +189072,189072 +189073,189073 +180044,189073 +189074,189074 +189075,189075 +189076,189076 +189077,189077 +189078,189078 +189079,189079 +189080,189080 +189081,189081 +189082,189082 +189083,189083 +189084,189084 +189085,189085 +189086,189086 +189087,189087 +189088,189088 +189089,189089 +189090,189090 +189091,189091 +189092,189092 +189093,189093 +189094,189094 +189095,189095 +189096,189096 +189097,189097 +189098,189098 +189099,189099 +189100,189100 +189101,189101 +189102,189102 +189103,189103 +189104,189104 +189105,189105 +189106,189106 +189107,189107 +189108,189108 +189109,189109 +14649,189109 +189110,189110 +189111,189111 +189112,189112 +189113,189113 +189114,189114 +189115,189115 +189116,189116 +189117,189117 +189118,189118 +189119,189119 +189120,189120 +162995,189120 +189121,189121 +189122,189122 +189123,189123 +189124,189124 +189125,189125 +189126,189126 +189127,189127 +189128,189128 +189129,189129 +189130,189130 +189131,189131 +189132,189132 +189133,189133 +189134,189134 +142166,189134 +189135,189135 +189136,189136 +189137,189137 +189138,189138 +189139,189139 +189140,189140 +207307,189141 +189141,189141 +140422,189141 +189142,189142 +189143,189143 +189144,189144 +189145,189145 +189146,189146 +104677,189146 +189147,189147 +189148,189148 +189149,189149 +189150,189150 +189151,189151 +189152,189152 +189153,189153 +189154,189154 +189155,189155 +189156,189156 +189157,189157 +189158,189158 +189159,189159 +189160,189160 +189161,189161 +189162,189162 +189163,189163 +189164,189164 +189165,189165 +189166,189166 +189167,189167 +189168,189168 +189169,189169 +189170,189170 +189171,189171 +146522,189171 +189172,189172 +189173,189173 +189174,189174 +189175,189175 +189176,189176 +189177,189177 +189178,189178 +189179,189179 +189180,189180 +189181,189181 +189182,189182 +189183,189183 +189184,189184 +163398,189184 +189185,189185 +189186,189186 +189187,189187 +189188,189188 +106532,189188 +76657,189188 +189189,189189 +189190,189190 +189191,189191 +189192,189192 +189193,189193 +189194,189194 +189195,189195 +189196,189196 +189197,189197 +189198,189198 +189199,189199 +189200,189200 +189201,189201 +189202,189202 +189203,189203 +189204,189204 +189205,189205 +189206,189206 +189207,189207 +189208,189208 +189209,189209 +189210,189210 +189211,189211 +189212,189212 +189213,189213 +189214,189214 +189215,189215 +189216,189216 +189217,189217 +192010,189217 +189218,189218 +189219,189219 +189220,189220 +189221,189221 +189222,189222 +189223,189223 +189224,189224 +189225,189225 +189412,189412 +189413,189413 +189414,189414 +189415,189415 +189416,189416 +217485,189416 +189417,189417 +189418,189418 +189419,189419 +189420,189420 +189421,189421 +189422,189422 +189423,189423 +138956,189423 +76008,189423 +189424,189424 +189425,189425 +189426,189426 +189427,189427 +189428,189428 +189429,189429 +189430,189430 +189431,189431 +242918,189431 +189432,189432 +189433,189433 +189434,189434 +189435,189435 +189436,189436 +189437,189437 +189438,189438 +189439,189439 +189440,189440 +189441,189441 +189442,189442 +189443,189443 +189444,189444 +189445,189445 +189446,189446 +189447,189447 +189448,189448 +189449,189449 +189450,189450 +189451,189451 +234980,189451 +189452,189452 +189453,189453 +189454,189454 +189455,189455 +189456,189456 +77530,189456 +189457,189457 +189458,189458 +189459,189459 +189460,189460 +189461,189461 +189462,189462 +189463,189463 +189464,189464 +189465,189465 +189466,189466 +189467,189467 +189468,189468 +189469,189469 +189470,189470 +189471,189471 +189472,189472 +189473,189473 +189474,189474 +189475,189475 +189476,189476 +189477,189477 +189478,189478 +211262,189478 +189479,189479 +189480,189480 +189481,189481 +85994,189481 +189482,189482 +189483,189483 +189484,189484 +229559,189484 +189485,189485 +60254,189485 +189486,189486 +189487,189487 +189488,189488 +189489,189489 +189490,189490 +189491,189491 +189492,189492 +189493,189493 +189494,189494 +189495,189495 +189496,189496 +178710,189496 +189497,189497 +189498,189498 +189499,189499 +189500,189500 +21349,189500 +189501,189501 +189502,189502 +189503,189503 +189504,189504 +189505,189505 +189506,189506 +189507,189507 +189508,189508 +189509,189509 +189510,189510 +189511,189511 +189512,189512 +189513,189513 +189514,189514 +189515,189515 +189516,189516 +189517,189517 +189518,189518 +189519,189519 +189520,189520 +189521,189521 +189522,189522 +189523,189523 +189524,189524 +189525,189525 +189526,189526 +189527,189527 +189528,189528 +189529,189529 +189530,189530 +189531,189531 +189532,189532 +189533,189533 +189534,189534 +189535,189535 +189536,189536 +189537,189537 +189538,189538 +189539,189539 +189540,189540 +189541,189541 +189542,189542 +189543,189543 +189544,189544 +189545,189545 +189546,189546 +189547,189547 +189548,189548 +189549,189549 +189550,189550 +189551,189551 +189552,189552 +189553,189553 +189554,189554 +189555,189555 +189556,189556 +189557,189557 +189558,189558 +189559,189559 +144272,189559 +189560,189560 +189561,189561 +189562,189562 +189563,189563 +189564,189564 +141183,189564 +189565,189565 +189566,189566 +189567,189567 +189568,189568 +189569,189569 +189570,189570 +189571,189571 +189572,189572 +189573,189573 +189574,189574 +189575,189575 +189576,189576 +189577,189577 +189578,189578 +189579,189579 +189580,189580 +189581,189581 +189582,189582 +189583,189583 +189584,189584 +189585,189585 +189586,189586 +189587,189587 +189588,189588 +189589,189589 +189590,189590 +189591,189591 +38725,189591 +189592,189592 +189593,189593 +189594,189594 +189595,189595 +189596,189596 +189597,189597 +189598,189598 +189599,189599 +189600,189600 +189601,189601 +189602,189602 +189603,189603 +189604,189604 +189605,189605 +189606,189606 +189607,189607 +12013,189607 +189608,189608 +189609,189609 +189610,189610 +189611,189611 +189612,189612 +189613,189613 +189614,189614 +189615,189615 +189616,189616 +189617,189617 +189618,189618 +189619,189619 +189620,189620 +189621,189621 +189622,189622 +189623,189623 +189662,189662 +189663,189663 +189664,189664 +189665,189665 +189666,189666 +189667,189667 +189668,189668 +189669,189669 +189670,189670 +189671,189671 +189672,189672 +189673,189673 +189674,189674 +189675,189675 +189676,189676 +189677,189677 +189678,189678 +189679,189679 +189680,189680 +189681,189681 +189682,189682 +189683,189683 +189684,189684 +189685,189685 +189686,189686 +189687,189687 +189688,189688 +189689,189689 +189690,189690 +189691,189691 +189692,189692 +189693,189693 +189694,189694 +189695,189695 +189696,189696 +189697,189697 +189698,189698 +21540,189698 +189699,189699 +189700,189700 +189900,189700 +189701,189701 +189702,189702 +189703,189703 +189704,189704 +189705,189705 +189706,189706 +189707,189707 +189708,189708 +189709,189709 +189710,189710 +189711,189711 +207038,189711 +189712,189712 +189713,189713 +189714,189714 +189715,189715 +189716,189716 +189717,189717 +189718,189718 +189719,189719 +189720,189720 +189721,189721 +189722,189722 +189723,189723 +189724,189724 +189725,189725 +189726,189726 +189727,189727 +189728,189728 +189729,189729 +189730,189730 +189731,189731 +189732,189732 +207862,189733 +189733,189733 +207712,189733 +207772,189733 +189734,189734 +189735,189735 +189736,189736 +189737,189737 +189738,189738 +189739,189739 +189940,189739 +189740,189740 +190122,189740 +189741,189741 +189742,189742 +189743,189743 +189744,189744 +189745,189745 +189746,189746 +189747,189747 +189748,189748 +189749,189749 +189750,189750 +24537,189750 +178331,189750 +172367,189750 +189751,189751 +189752,189752 +190013,189752 +189753,189753 +189754,189754 +189755,189755 +189756,189756 +189757,189757 +189758,189758 +223159,189758 +189759,189759 +189760,189760 +189761,189761 +189762,189762 +30367,189762 +189933,189763 +189763,189763 +189764,189764 +189765,189765 +190106,189765 +189766,189766 +189767,189767 +190209,189767 +189768,189768 +190220,189768 +189769,189769 +189770,189770 +189819,189770 +189771,189771 +190224,189772 +189772,189772 +189773,189773 +189774,189774 +189775,189775 +189776,189776 +189777,189777 +189778,189778 +189779,189779 +189780,189780 +189781,189781 +189782,189782 +189783,189783 +189784,189784 +189785,189785 +189786,189786 +189787,189787 +189788,189788 +189789,189789 +189790,189790 +189791,189791 +35558,189791 +189792,189792 +226609,189792 +189793,189793 +189794,189794 +189795,189795 +176746,189795 +170782,189795 +189796,189796 +189797,189797 +189798,189798 +189799,189799 +189800,189800 +189801,189801 +189802,189802 +189803,189803 +189804,189804 +189805,189805 +189806,189806 +189807,189807 +189808,189808 +189809,189809 +189810,189810 +190223,189810 +189811,189811 +189812,189812 +189813,189813 +189814,189814 +189815,189815 +190218,189815 +189816,189816 +189817,189817 +190220,189817 +189818,189818 +189980,189818 +189819,189819 +189770,189819 +189820,189820 +189821,189821 +108457,189821 +189822,189822 +189934,189823 +189823,189823 +189824,189824 +189825,189825 +189826,189826 +189827,189827 +189828,189828 +189829,189829 +189830,189830 +189941,189830 +189831,189831 +189832,189832 +189833,189833 +189834,189834 +189835,189835 +189836,189836 +189837,189837 +189838,189838 +189839,189839 +189840,189840 +189841,189841 +189842,189842 +189843,189843 +189844,189844 +151479,189844 +189845,189845 +189846,189846 +189847,189847 +189848,189848 +189849,189849 +64482,189849 +189850,189850 +189851,189851 +189852,189852 +189853,189853 +189954,189854 +189854,189854 +189855,189855 +189956,189856 +189856,189856 +189906,189856 +189857,189857 +189858,189858 +189859,189859 +189860,189860 +189861,189861 +189862,189862 +189863,189863 +189864,189864 +189865,189865 +216360,189865 +189866,189866 +190209,189866 +189867,189867 +189868,189868 +189869,189869 +204567,189869 +189870,189870 +189871,189871 +189872,189872 +189873,189873 +189874,189874 +10887,189874 +189875,189875 +189876,189876 +189877,189877 +189878,189878 +189879,189879 +189880,189880 +189881,189881 +189882,189882 +189883,189883 +246579,189883 +189884,189884 +3138,189884 +160688,189884 +189885,189885 +189886,189886 +189887,189887 +189888,189888 +189889,189889 +189890,189890 +189891,189891 +189892,189892 +189893,189893 +189894,189894 +189895,189895 +189896,189896 +189897,189897 +189898,189898 +189899,189899 +189900,189900 +189700,189900 +189901,189901 +189902,189902 +189903,189903 +189904,189904 +189905,189905 +189906,189906 +189956,189906 +189856,189906 +189907,189907 +189908,189908 +189909,189909 +189910,189910 +189911,189911 +189912,189912 +189913,189913 +189914,189914 +189915,189915 +189916,189916 +189917,189917 +190097,189917 +189918,189918 +189919,189919 +189920,189920 +189921,189921 +189922,189922 +189923,189923 +189924,189924 +189925,189925 +189926,189926 +189927,189927 +189928,189928 +189929,189929 +189930,189930 +189931,189931 +189932,189932 +189933,189933 +189763,189933 +189934,189934 +189823,189934 +189935,189935 +189936,189936 +216350,189936 +189937,189937 +189938,189938 +190220,189938 +189939,189939 +189940,189940 +189739,189940 +189941,189941 +189830,189941 +189942,189942 +189943,189943 +190124,189943 +189944,189944 +189945,189945 +189946,189946 +189947,189947 +189948,189948 +189949,189949 +189950,189950 +189951,189951 +189952,189952 +189953,189953 +189954,189954 +189854,189954 +189955,189955 +189956,189956 +189856,189956 +189906,189956 +189957,189957 +189958,189958 +142633,189958 +189959,189959 +189960,189960 +189961,189961 +189962,189962 +189963,189963 +189964,189964 +189965,189965 +189966,189966 +189967,189967 +189968,189968 +189969,189969 +189970,189970 +189971,189971 +189972,189972 +189973,189973 +189974,189974 +189975,189975 +189976,189976 +189977,189977 +189978,189978 +189979,189979 +189980,189980 +189818,189980 +189981,189981 +191621,189981 +189982,189982 +189983,189983 +189984,189984 +190143,189984 +189985,189985 +189986,189986 +189987,189987 +196482,189987 +110817,189987 +189988,189988 +21494,189988 +189989,189989 +189990,189990 +189991,189991 +189992,189992 +189993,189993 +189994,189994 +189995,189995 +189996,189996 +189997,189997 +189998,189998 +189999,189999 +190000,190000 +190001,190001 +190002,190002 +190003,190003 +190004,190004 +190005,190005 +190006,190006 +190007,190007 +190008,190008 +190009,190009 +190010,190010 +190011,190011 +190012,190012 +190013,190013 +189752,190013 +190014,190014 +190015,190015 +190016,190016 +187296,190016 +190017,190017 +190018,190018 +190019,190019 +217482,190019 +190020,190020 +190021,190021 +190022,190022 +190023,190023 +190024,190024 +190025,190025 +190026,190026 +190027,190027 +190028,190028 +190029,190029 +12874,190029 +190030,190030 +190031,190031 +190032,190032 +190033,190033 +190034,190034 +190035,190035 +190036,190036 +190037,190037 +190038,190038 +190039,190039 +190040,190040 +190041,190041 +190042,190042 +190043,190043 +190044,190044 +190045,190045 +190046,190046 +190047,190047 +190048,190048 +217945,190048 +190049,190049 +36927,190049 +190050,190050 +190051,190051 +190052,190052 +191772,190053 +190053,190053 +190054,190054 +5009,190054 +190055,190055 +190056,190056 +158645,190056 +190057,190057 +190058,190058 +190059,190059 +190060,190060 +190061,190061 +190062,190062 +190063,190063 +190064,190064 +190065,190065 +190066,190066 +190067,190067 +190068,190068 +190069,190069 +190070,190070 +190071,190071 +190072,190072 +190073,190073 +190074,190074 +190075,190075 +190076,190076 +190077,190077 +190078,190078 +190079,190079 +190080,190080 +190081,190081 +190082,190082 +190083,190083 +190084,190084 +190085,190085 +190086,190086 +6413,190086 +190087,190087 +190088,190088 +190089,190089 +190090,190090 +190091,190091 +190092,190092 +190124,190092 +190093,190093 +190094,190094 +190095,190095 +190096,190096 +190097,190097 +189917,190097 +190098,190098 +190099,190099 +190100,190100 +190101,190101 +190102,190102 +190103,190103 +190104,190104 +190105,190105 +190106,190106 +189765,190106 +190107,190107 +190108,190108 +190209,190108 +190109,190109 +190110,190110 +190111,190111 +190112,190112 +190113,190113 +190114,190114 +190115,190115 +190116,190116 +190117,190117 +190118,190118 +190119,190119 +190120,190120 +190121,190121 +190122,190122 +189740,190122 +190123,190123 +190124,190124 +189943,190124 +190092,190124 +190125,190125 +190126,190126 +190127,190127 +190128,190128 +190129,190129 +190130,190130 +190131,190131 +190132,190132 +190133,190133 +190134,190134 +190135,190135 +190136,190136 +190137,190137 +190138,190138 +190139,190139 +190140,190140 +190141,190141 +190142,190142 +190143,190143 +189984,190143 +72184,190143 +190144,190144 +190145,190145 +190146,190146 +190147,190147 +190148,190148 +190149,190149 +190150,190150 +190151,190151 +190152,190152 +190153,190153 +203744,190153 +221559,190153 +190154,190154 +190155,190155 +190156,190156 +190157,190157 +190158,190158 +190159,190159 +190160,190160 +190161,190161 +190162,190162 +190163,190163 +190164,190164 +57603,190164 +190165,190165 +190166,190166 +190167,190167 +190168,190168 +190169,190169 +190170,190170 +190171,190171 +190172,190172 +190173,190173 +190174,190174 +190175,190175 +190176,190176 +190177,190177 +190178,190178 +161025,190178 +190179,190179 +190180,190180 +190181,190181 +190182,190182 +190183,190183 +190184,190184 +190185,190185 +190186,190186 +190187,190187 +190188,190188 +190189,190189 +190190,190190 +190191,190191 +190192,190192 +80360,190192 +190193,190193 +190194,190194 +190195,190195 +190196,190196 +190197,190197 +190198,190198 +190199,190199 +190200,190200 +190201,190201 +190202,190202 +190203,190203 +190204,190204 +190205,190205 +190206,190206 +190207,190207 +190208,190208 +190209,190209 +189767,190209 +189866,190209 +190108,190209 +190210,190210 +190211,190211 +190212,190212 +190213,190213 +190214,190214 +190215,190215 +190216,190216 +104633,190216 +190217,190217 +190237,190217 +190218,190218 +189815,190218 +163703,190218 +190219,190219 +190220,190220 +189768,190220 +189817,190220 +189938,190220 +190221,190221 +190222,190222 +216522,190222 +190223,190223 +189810,190223 +190224,190224 +189772,190224 +190225,190225 +190226,190226 +190227,190227 +190228,190228 +216538,190228 +190229,190229 +190230,190230 +190231,190231 +190232,190232 +190233,190233 +190234,190234 +190235,190235 +190236,190236 +213324,190236 +190237,190237 +190217,190237 +190238,190238 +190239,190239 +190240,190240 +190241,190241 +190242,190242 +190243,190243 +190244,190244 +190245,190245 +190246,190246 +190247,190247 +190248,190248 +190249,190249 +190250,190250 +190251,190251 +190252,190252 +83482,190252 +190253,190253 +190254,190254 +190255,190255 +76547,190255 +190256,190256 +190257,190257 +190258,190258 +190259,190259 +190260,190260 +190261,190261 +190262,190262 +190263,190263 +190264,190264 +190265,190265 +190266,190266 +190267,190267 +195790,190267 +190268,190268 +190269,190269 +190270,190270 +190271,190271 +190272,190272 +190273,190273 +190274,190274 +190275,190275 +190276,190276 +190277,190277 +190278,190278 +20913,190278 +249239,190278 +190279,190279 +190280,190280 +190281,190281 +190282,190282 +190283,190283 +190284,190284 +159406,190284 +190285,190285 +190286,190286 +190287,190287 +190288,190288 +190289,190289 +190290,190290 +190291,190291 +190292,190292 +190293,190293 +190294,190294 +208803,190295 +190295,190295 +190296,190296 +190297,190297 +190298,190298 +190299,190299 +190300,190300 +190301,190301 +190302,190302 +190303,190303 +190304,190304 +190305,190305 +190306,190306 +190307,190307 +190308,190308 +190309,190309 +190310,190310 +190311,190311 +190312,190312 +190313,190313 +190314,190314 +190315,190315 +190316,190316 +190317,190317 +190318,190318 +190319,190319 +190320,190320 +190321,190321 +190322,190322 +190323,190323 +190324,190324 +190325,190325 +190326,190326 +190327,190327 +190328,190328 +190329,190329 +190330,190330 +179217,190330 +190331,190331 +190332,190332 +156320,190332 +190333,190333 +190334,190334 +190335,190335 +190336,190336 +190337,190337 +180832,190337 +190338,190338 +190339,190339 +190340,190340 +190341,190341 +190342,190342 +190343,190343 +190344,190344 +190345,190345 +190346,190346 +190347,190347 +190348,190348 +190349,190349 +190350,190350 +190351,190351 +190352,190352 +190353,190353 +190354,190354 +190355,190355 +190356,190356 +190357,190357 +190358,190358 +190359,190359 +190360,190360 +190361,190361 +190362,190362 +190363,190363 +190364,190364 +190365,190365 +190366,190366 +190367,190367 +190368,190368 +190369,190369 +190370,190370 +190371,190371 +190372,190372 +190373,190373 +190374,190374 +190375,190375 +190376,190376 +190377,190377 +190378,190378 +190379,190379 +190380,190380 +190381,190381 +190382,190382 +190383,190383 +190384,190384 +190385,190385 +190386,190386 +190387,190387 +190388,190388 +190389,190389 +190390,190390 +190391,190391 +190392,190392 +190393,190393 +190394,190394 +190395,190395 +190396,190396 +190397,190397 +194282,190397 +190398,190398 +190399,190399 +190400,190400 +190401,190401 +190402,190402 +190403,190403 +190404,190404 +190405,190405 +86830,190405 +190406,190406 +190407,190407 +190408,190408 +190409,190409 +190410,190410 +102062,190410 +190411,190411 +190412,190412 +190413,190413 +190414,190414 +190415,190415 +190416,190416 +190417,190417 +190418,190418 +190419,190419 +190420,190420 +190421,190421 +190422,190422 +190423,190423 +190424,190424 +190425,190425 +190426,190426 +190427,190427 +190428,190428 +190429,190429 +64853,190429 +190430,190430 +190431,190431 +190432,190432 +190433,190433 +190434,190434 +190435,190435 +190436,190436 +190437,190437 +190438,190438 +28962,190438 +190439,190439 +190440,190440 +190441,190441 +190442,190442 +155925,190442 +190443,190443 +190444,190444 +190445,190445 +190446,190446 +190447,190447 +190448,190448 +190449,190449 +190450,190450 +190451,190451 +190452,190452 +190453,190453 +190454,190454 +190455,190455 +190456,190456 +190457,190457 +190458,190458 +190459,190459 +190460,190460 +190461,190461 +190462,190462 +190463,190463 +190464,190464 +190465,190465 +190466,190466 +190467,190467 +190468,190468 +190469,190469 +190470,190470 +190471,190471 +190472,190472 +190473,190473 +190474,190474 +190475,190475 +190476,190476 +190477,190477 +190478,190478 +190479,190479 +190480,190480 +164483,190480 +190481,190481 +209447,190481 +190482,190482 +190483,190483 +190484,190484 +190485,190485 +146727,190485 +190486,190486 +190487,190487 +190488,190488 +190489,190489 +190490,190490 +190491,190491 +190492,190492 +190493,190493 +190494,190494 +190495,190495 +190496,190496 +190497,190497 +190498,190498 +190499,190499 +190500,190500 +190501,190501 +190502,190502 +190503,190503 +190504,190504 +190505,190505 +190506,190506 +190507,190507 +190508,190508 +214518,190508 +190509,190509 +190510,190510 +155883,190510 +83774,190510 +190511,190511 +190512,190512 +190513,190513 +190514,190514 +190515,190515 +190516,190516 +190517,190517 +190518,190518 +190519,190519 +190520,190520 +190521,190521 +190522,190522 +190523,190523 +190524,190524 +190525,190525 +190526,190526 +190527,190527 +190528,190528 +190529,190529 +190530,190530 +28271,190530 +190531,190531 +190532,190532 +190533,190533 +190534,190534 +190535,190535 +190536,190536 +190537,190537 +190538,190538 +190539,190539 +190540,190540 +190541,190541 +190542,190542 +190543,190543 +190544,190544 +190545,190545 +190546,190546 +190547,190547 +190548,190548 +190549,190549 +190550,190550 +190551,190551 +190552,190552 +190553,190553 +190554,190554 +190555,190555 +190556,190556 +190557,190557 +202528,190557 +190558,190558 +190559,190559 +190560,190560 +190561,190561 +190562,190562 +190563,190563 +190564,190564 +190565,190565 +190566,190566 +190567,190567 +190568,190568 +190569,190569 +190570,190570 +190571,190571 +190572,190572 +190573,190573 +190574,190574 +190575,190575 +95051,190575 +190576,190576 +190577,190577 +190578,190578 +190579,190579 +4502,190579 +190580,190580 +190581,190581 +190582,190582 +190583,190583 +190584,190584 +190585,190585 +190586,190586 +190587,190587 +190588,190588 +190589,190589 +190590,190590 +190591,190591 +190592,190592 +190593,190593 +190594,190594 +190595,190595 +190596,190596 +190597,190597 +190598,190598 +190599,190599 +190600,190600 +190601,190601 +190602,190602 +214575,190602 +190603,190603 +190604,190604 +190605,190605 +190606,190606 +227968,190606 +190607,190607 +190608,190608 +190609,190609 +190610,190610 +190611,190611 +190612,190612 +190613,190613 +190614,190614 +162611,190614 +190615,190615 +190616,190616 +222068,190616 +190617,190617 +190618,190618 +190619,190619 +190620,190620 +190621,190621 +190622,190622 +190623,190623 +190624,190624 +190625,190625 +214476,190625 +190626,190626 +190627,190627 +190628,190628 +190629,190629 +235071,190629 +190630,190630 +190631,190631 +190632,190632 +190633,190633 +190634,190634 +190635,190635 +190636,190636 +190637,190637 +190638,190638 +117815,190638 +190639,190639 +190640,190640 +190641,190641 +190642,190642 +190643,190643 +190644,190644 +190645,190645 +190646,190646 +190647,190647 +216207,190647 +190648,190648 +190649,190649 +190650,190650 +190651,190651 +190652,190652 +190653,190653 +190654,190654 +190655,190655 +137360,190655 +129773,190655 +190656,190656 +190657,190657 +190658,190658 +190659,190659 +190660,190660 +123032,190660 +190661,190661 +190662,190662 +190663,190663 +190664,190664 +190665,190665 +190666,190666 +190667,190667 +59778,190667 +190668,190668 +190669,190669 +190670,190670 +190671,190671 +190672,190672 +190673,190673 +190674,190674 +190675,190675 +190676,190676 +190677,190677 +190678,190678 +190679,190679 +190680,190680 +190681,190681 +190682,190682 +190683,190683 +190684,190684 +190685,190685 +190686,190686 +151623,190686 +190687,190687 +190688,190688 +209532,190688 +190689,190689 +190690,190690 +190691,190691 +190692,190692 +190743,190743 +190744,190744 +190745,190745 +190746,190746 +190747,190747 +190748,190748 +190749,190749 +190750,190750 +190751,190751 +190752,190752 +190753,190753 +190754,190754 +190755,190755 +108590,190755 +190756,190756 +190757,190757 +119234,190757 +190758,190758 +190759,190759 +190760,190760 +190761,190761 +190762,190762 +190763,190763 +190764,190764 +190765,190765 +190766,190766 +190767,190767 +190768,190768 +190769,190769 +190770,190770 +190771,190771 +190772,190772 +8617,190772 +8437,190772 +190773,190773 +190774,190774 +190775,190775 +190776,190776 +190777,190777 +190778,190778 +190779,190779 +190780,190780 +190781,190781 +190782,190782 +190783,190783 +190784,190784 +190785,190785 +190786,190786 +190787,190787 +190788,190788 +190789,190789 +190790,190790 +190791,190791 +190792,190792 +190793,190793 +190794,190794 +190795,190795 +190796,190796 +190797,190797 +190798,190798 +190799,190799 +179085,190799 +190800,190800 +190801,190801 +190802,190802 +190803,190803 +190804,190804 +190805,190805 +190806,190806 +190807,190807 +190808,190808 +190809,190809 +190810,190810 +190811,190811 +190812,190812 +190813,190813 +190814,190814 +190815,190815 +224757,190815 +190816,190816 +190817,190817 +190818,190818 +190819,190819 +190820,190820 +190821,190821 +190822,190822 +190823,190823 +190824,190824 +190825,190825 +190826,190826 +190827,190827 +190828,190828 +190829,190829 +190830,190830 +190831,190831 +190832,190832 +190833,190833 +190834,190834 +190835,190835 +190836,190836 +190837,190837 +190838,190838 +190839,190839 +153967,190839 +190840,190840 +242136,190840 +190841,190841 +190842,190842 +190843,190843 +190844,190844 +190845,190845 +190846,190846 +190847,190847 +190848,190848 +190849,190849 +190850,190850 +190851,190851 +190852,190852 +190853,190853 +190854,190854 +190855,190855 +98337,190855 +190856,190856 +190857,190857 +190858,190858 +190859,190859 +190860,190860 +190861,190861 +190862,190862 +190863,190863 +190864,190864 +190865,190865 +190866,190866 +190867,190867 +190868,190868 +190869,190869 +190870,190870 +190871,190871 +190872,190872 +190873,190873 +142493,190873 +190874,190874 +190875,190875 +190876,190876 +190877,190877 +190878,190878 +202632,190878 +190879,190879 +190880,190880 +190881,190881 +190882,190882 +190883,190883 +190884,190884 +190885,190885 +190886,190886 +190887,190887 +190888,190888 +190889,190889 +190890,190890 +190891,190891 +157769,190891 +190892,190892 +190893,190893 +190894,190894 +190895,190895 +190896,190896 +190897,190897 +190898,190898 +190899,190899 +190900,190900 +190901,190901 +190902,190902 +190903,190903 +78370,190903 +190904,190904 +190905,190905 +190906,190906 +190907,190907 +247422,190908 +190908,190908 +190909,190909 +190910,190910 +190911,190911 +190912,190912 +190913,190913 +190914,190914 +190915,190915 +190916,190916 +77402,190916 +190917,190917 +190918,190918 +190919,190919 +159232,190919 +190920,190920 +190921,190921 +190922,190922 +190923,190923 +190924,190924 +190925,190925 +217830,190925 +134245,190925 +126658,190925 +190926,190926 +190927,190927 +190928,190928 +190929,190929 +190930,190930 +190931,190931 +190932,190932 +190933,190933 +67255,190933 +190934,190934 +190935,190935 +190936,190936 +190937,190937 +190938,190938 +190939,190939 +190940,190940 +190941,190941 +190942,190942 +190943,190943 +190944,190944 +190945,190945 +190946,190946 +190947,190947 +190948,190948 +190949,190949 +190950,190950 +190951,190951 +190952,190952 +190953,190953 +190954,190954 +190955,190955 +190956,190956 +190957,190957 +190958,190958 +190959,190959 +190960,190960 +161200,190960 +190961,190961 +190962,190962 +84711,190962 +190963,190963 +190964,190964 +190965,190965 +190966,190966 +190967,190967 +190968,190968 +35095,190968 +190969,190969 +137046,190969 +129459,190969 +190970,190970 +149772,190970 +190971,190971 +190972,190972 +190973,190973 +190974,190974 +190975,190975 +190976,190976 +190977,190977 +190978,190978 +190979,190979 +190980,190980 +95159,190980 +190981,190981 +134265,190981 +126678,190981 +190982,190982 +190983,190983 +190984,190984 +190985,190985 +190986,190986 +190987,190987 +190988,190988 +190989,190989 +190990,190990 +190991,190991 +149362,190991 +190992,190992 +190993,190993 +190994,190994 +190995,190995 +190996,190996 +190997,190997 +190998,190998 +190999,190999 +191000,191000 +191001,191001 +191002,191002 +191003,191003 +191004,191004 +191005,191005 +191006,191006 +191007,191007 +191008,191008 +191009,191009 +191010,191010 +191011,191011 +191012,191012 +191013,191013 +191014,191014 +191015,191015 +191016,191016 +191017,191017 +191018,191018 +191019,191019 +191020,191020 +191021,191021 +191022,191022 +191023,191023 +191024,191024 +191025,191025 +191026,191026 +191027,191027 +191028,191028 +191029,191029 +142272,191029 +191030,191030 +191031,191031 +191032,191032 +191033,191033 +191034,191034 +191035,191035 +191036,191036 +191037,191037 +191038,191038 +191039,191039 +191040,191040 +191041,191041 +191042,191042 +191043,191043 +191044,191044 +191045,191045 +191046,191046 +191047,191047 +30916,191047 +191048,191048 +147901,191048 +191049,191049 +191050,191050 +191051,191051 +191052,191052 +191053,191053 +191054,191054 +191055,191055 +191056,191056 +191057,191057 +191058,191058 +191059,191059 +174614,191059 +168650,191059 +191060,191060 +191061,191061 +191062,191062 +191063,191063 +191064,191064 +191065,191065 +191066,191066 +191067,191067 +191068,191068 +192242,191068 +191069,191069 +191070,191070 +191071,191071 +191072,191072 +191073,191073 +191074,191074 +191075,191075 +191076,191076 +191077,191077 +191078,191078 +191079,191079 +191080,191080 +191081,191081 +191082,191082 +191083,191083 +191084,191084 +191085,191085 +191086,191086 +191087,191087 +191088,191088 +191089,191089 +191090,191090 +191091,191091 +191092,191092 +191093,191093 +191094,191094 +191095,191095 +191096,191096 +191097,191097 +191098,191098 +191099,191099 +191100,191100 +191101,191101 +191102,191102 +191103,191103 +191104,191104 +191105,191105 +4728,191105 +191106,191106 +191107,191107 +191108,191108 +191109,191109 +191110,191110 +191111,191111 +191112,191112 +191113,191113 +191114,191114 +191115,191115 +191116,191116 +192029,191116 +191117,191117 +191118,191118 +191119,191119 +191120,191120 +191121,191121 +191122,191122 +9090,191122 +191123,191123 +191124,191124 +191125,191125 +191126,191126 +191127,191127 +191128,191128 +191129,191129 +191130,191130 +191131,191131 +191132,191132 +191133,191133 +191134,191134 +191135,191135 +191136,191136 +191137,191137 +191138,191138 +191139,191139 +191140,191140 +191141,191141 +191142,191142 +155240,191142 +191143,191143 +191144,191144 +191145,191145 +191146,191146 +191147,191147 +191148,191148 +148329,191148 +191149,191149 +191150,191150 +191151,191151 +191152,191152 +191153,191153 +191154,191154 +191155,191155 +191156,191156 +191157,191157 +191158,191158 +209772,191158 +191159,191159 +191160,191160 +191161,191161 +151412,191161 +191162,191162 +191163,191163 +38258,191163 +191164,191164 +191165,191165 +191166,191166 +191167,191167 +191168,191168 +191169,191169 +191170,191170 +191171,191171 +191172,191172 +191173,191173 +191174,191174 +191175,191175 +191176,191176 +191177,191177 +2716,191177 +191178,191178 +191179,191179 +191180,191180 +191181,191181 +191182,191182 +191183,191183 +191184,191184 +191185,191185 +191186,191186 +156359,191186 +191187,191187 +191188,191188 +191189,191189 +191190,191190 +191191,191191 +191192,191192 +191193,191193 +191194,191194 +191195,191195 +191196,191196 +191197,191197 +191198,191198 +191199,191199 +191200,191200 +191201,191201 +191202,191202 +191203,191203 +191204,191204 +191205,191205 +57720,191205 +191206,191206 +191207,191207 +191208,191208 +191209,191209 +191210,191210 +191211,191211 +191212,191212 +191213,191213 +31230,191213 +191214,191214 +191215,191215 +191216,191216 +191217,191217 +191218,191218 +191219,191219 +191220,191220 +191221,191221 +191222,191222 +225995,191222 +191223,191223 +191224,191224 +191225,191225 +191226,191226 +191227,191227 +191228,191228 +191229,191229 +191230,191230 +191231,191231 +191232,191232 +191233,191233 +191234,191234 +191235,191235 +191236,191236 +116555,191236 +191237,191237 +191238,191238 +191239,191239 +191240,191240 +191241,191241 +191242,191242 +191243,191243 +191244,191244 +191245,191245 +191246,191246 +191247,191247 +191248,191248 +191249,191249 +191250,191250 +191251,191251 +33550,191251 +191252,191252 +191253,191253 +82594,191253 +191254,191254 +191255,191255 +191256,191256 +154776,191256 +191257,191257 +191258,191258 +191259,191259 +191260,191260 +191261,191261 +191262,191262 +191263,191263 +102386,191263 +191264,191264 +191265,191265 +191266,191266 +191267,191267 +191268,191268 +191269,191269 +191270,191270 +191271,191271 +191272,191272 +191273,191273 +191274,191274 +140998,191274 +191275,191275 +191276,191276 +191277,191277 +191278,191278 +86749,191278 +84801,191278 +191279,191279 +191280,191280 +191281,191281 +191282,191282 +191283,191283 +191284,191284 +132774,191284 +125187,191284 +191285,191285 +191286,191286 +191287,191287 +191288,191288 +191289,191289 +191290,191290 +191291,191291 +191292,191292 +191293,191293 +191294,191294 +191295,191295 +191296,191296 +191297,191297 +191298,191298 +191299,191299 +191300,191300 +191301,191301 +191302,191302 +191303,191303 +191304,191304 +191305,191305 +191379,191379 +191380,191380 +191381,191381 +191382,191382 +191874,191382 +191383,191383 +191384,191384 +191385,191385 +191386,191386 +191387,191387 +191388,191388 +114444,191388 +91984,191388 +191389,191389 +191390,191390 +191391,191391 +191392,191392 +191393,191393 +191394,191394 +191395,191395 +191396,191396 +191397,191397 +191398,191398 +191399,191399 +191400,191400 +191401,191401 +191402,191402 +191857,191402 +191403,191403 +191404,191404 +191528,191404 +191405,191405 +191406,191406 +191610,191406 +191407,191407 +191408,191408 +191409,191409 +191794,191409 +191410,191410 +191411,191411 +191866,191411 +191412,191412 +191413,191413 +191799,191414 +191414,191414 +191415,191415 +191800,191415 +191416,191416 +207887,191416 +191801,191416 +191417,191417 +191418,191418 +191803,191418 +191522,191418 +191419,191419 +191420,191420 +191421,191421 +191422,191422 +143152,191422 +191502,191502 +191503,191503 +93258,191503 +191504,191504 +191505,191505 +216691,191505 +191506,191506 +191507,191507 +191508,191508 +191509,191509 +191510,191510 +145523,191510 +191511,191511 +3534,191511 +191512,191512 +191513,191513 +191514,191514 +191515,191515 +191516,191516 +191517,191517 +191518,191518 +191519,191519 +191520,191520 +191801,191520 +191521,191521 +191522,191522 +191418,191522 +191523,191523 +191524,191524 +191525,191525 +191526,191526 +191527,191527 +191528,191528 +191404,191528 +191529,191529 +191530,191530 +191531,191531 +191532,191532 +201620,191532 +191533,191533 +191534,191534 +191535,191535 +191876,191535 +191666,191535 +191536,191536 +191537,191537 +191668,191537 +191798,191537 +191538,191538 +191539,191539 +191540,191540 +191541,191541 +191542,191542 +191543,191543 +191544,191544 +191545,191545 +191546,191546 +191547,191547 +191548,191548 +191549,191549 +191550,191550 +191551,191551 +191552,191552 +191553,191553 +191554,191554 +191555,191555 +95163,191555 +191556,191556 +220232,191556 +191557,191557 +191558,191558 +191859,191558 +136123,191558 +128536,191558 +191559,191559 +162519,191559 +191560,191560 +191561,191561 +216304,191561 +191562,191562 +191563,191563 +191564,191564 +207871,191564 +207781,191564 +207911,191564 +191565,191565 +84107,191565 +154504,191565 +191566,191566 +191567,191567 +191568,191568 +191569,191569 +191570,191570 +191571,191571 +191572,191572 +191573,191573 +222273,191573 +191795,191574 +191574,191574 +191575,191575 +191576,191576 +191577,191577 +191578,191578 +191579,191579 +191880,191579 +191800,191579 +191580,191580 +191581,191581 +191802,191581 +191582,191582 +191583,191583 +191584,191584 +191585,191585 +191586,191586 +191587,191587 +191588,191588 +191589,191589 +191800,191589 +191590,191590 +191591,191591 +201437,191592 +191592,191592 +191593,191593 +191594,191594 +191816,191595 +191595,191595 +191596,191596 +191597,191597 +191829,191598 +191608,191598 +191598,191598 +191599,191599 +140508,191599 +191600,191600 +191601,191601 +243477,191601 +191823,191602 +191602,191602 +191603,191603 +216296,191603 +191604,191604 +191605,191605 +191827,191606 +191606,191606 +191828,191607 +191607,191607 +191829,191608 +191608,191608 +191598,191608 +191830,191609 +191609,191609 +216302,191609 +6577,191609 +191610,191610 +191831,191610 +191406,191610 +191812,191611 +191611,191611 +191832,191611 +191612,191612 +191613,191613 +191614,191614 +191835,191614 +162524,191614 +191615,191615 +191837,191616 +191616,191616 +191617,191617 +207884,191617 +191618,191618 +191839,191618 +78008,191618 +62463,191618 +191619,191619 +191620,191620 +191621,191621 +189981,191621 +191622,191622 +191623,191623 +179085,191623 +111868,191623 +191624,191624 +191625,191625 +191626,191626 +191627,191627 +191663,191663 +191664,191664 +207910,191664 +84095,191664 +191665,191665 +191666,191666 +191535,191666 +191667,191667 +143071,191667 +191668,191668 +191537,191668 +191669,191669 +191736,191736 +191737,191737 +191738,191738 +191739,191739 +191740,191740 +191741,191741 +191742,191742 +191743,191743 +191744,191744 +191745,191745 +191746,191746 +191747,191747 +246690,191747 +191748,191748 +191749,191749 +191750,191750 +215499,191750 +191751,191751 +191752,191752 +191753,191753 +192568,191753 +145609,191753 +191754,191754 +191755,191755 +191756,191756 +191757,191757 +191758,191758 +191759,191759 +191760,191760 +191761,191761 +191762,191762 +191763,191763 +191764,191764 +191765,191765 +191766,191766 +191767,191767 +191768,191768 +191769,191769 +191770,191770 +191771,191771 +191772,191772 +190053,191772 +191773,191773 +191774,191774 +191775,191775 +191776,191776 +191777,191777 +191778,191778 +191779,191779 +191780,191780 +191781,191781 +191782,191782 +191783,191783 +191784,191784 +191785,191785 +205584,191785 +191786,191786 +191787,191787 +191857,191787 +191788,191788 +149692,191788 +191789,191789 +210278,191789 +191790,191790 +191791,191791 +191792,191792 +191793,191793 +191794,191794 +191409,191794 +214722,191794 +191795,191795 +191574,191795 +191796,191796 +207883,191797 +191797,191797 +207583,191797 +207613,191797 +84098,191797 +191798,191798 +191537,191798 +191799,191799 +191414,191799 +191800,191800 +191415,191800 +191579,191800 +191589,191800 +20522,191800 +191801,191801 +84102,191801 +191520,191801 +191416,191801 +191802,191802 +191581,191802 +191803,191803 +191418,191803 +226596,191803 +191804,191804 +191805,191805 +191835,191805 +191806,191806 +191807,191807 +191808,191808 +191809,191809 +233975,191809 +191810,191810 +191811,191811 +191812,191812 +191611,191812 +191832,191812 +191813,191813 +163565,191813 +191814,191814 +191844,191814 +191815,191815 +191816,191816 +191595,191816 +191817,191817 +191818,191818 +191819,191819 +191820,191820 +191821,191821 +191822,191822 +191823,191823 +191602,191823 +191824,191824 +191825,191825 +148468,191825 +191826,191826 +191827,191827 +191606,191827 +191828,191828 +191607,191828 +191829,191829 +191608,191829 +191598,191829 +191830,191830 +191609,191830 +216302,191830 +6577,191830 +191831,191831 +191610,191831 +191832,191832 +191812,191832 +191611,191832 +191842,191832 +191833,191833 +191834,191834 +51120,191834 +43016,191834 +191835,191835 +191614,191835 +191805,191835 +191836,191836 +191837,191837 +191616,191837 +191838,191838 +191839,191839 +78008,191839 +191618,191839 +78080,191839 +191840,191840 +191841,191841 +191842,191842 +191832,191842 +207858,191842 +207558,191842 +191843,191843 +191844,191844 +191814,191844 +191845,191845 +191846,191846 +191847,191847 +191848,191848 +191849,191849 +191850,191850 +187229,191850 +191851,191851 +176631,191851 +170667,191851 +191852,191852 +191853,191853 +191854,191854 +191855,191855 +191856,191856 +191857,191857 +207603,191857 +191787,191857 +191402,191857 +191858,191858 +191859,191859 +191558,191859 +191860,191860 +191861,191861 +191862,191862 +191863,191863 +62762,191863 +191864,191864 +191865,191865 +191866,191866 +191411,191866 +191867,191867 +191868,191868 +176627,191868 +170663,191868 +191869,191869 +191870,191870 +191871,191871 +191872,191872 +191873,191873 +2769,191873 +191874,191874 +191382,191874 +191875,191875 +191876,191876 +191535,191876 +191877,191877 +191878,191878 +191879,191879 +191880,191880 +191579,191880 +191881,191881 +191882,191882 +191883,191883 +191884,191884 +153013,191884 +210273,191884 +191885,191885 +191886,191886 +191887,191887 +191888,191888 +191889,191889 +191890,191890 +191891,191891 +191892,191892 +207858,191892 +191893,191893 +191894,191894 +214722,191894 +207721,191895 +191895,191895 +207911,191895 +207871,191895 +207781,191895 +207935,191895 +191896,191896 +231759,191896 +191897,191897 +191898,191898 +191942,191942 +191943,191943 +191944,191944 +191945,191945 +191946,191946 +111384,191946 +191947,191947 +191948,191948 +191949,191949 +191950,191950 +146862,191950 +191951,191951 +191952,191952 +191953,191953 +191954,191954 +191955,191955 +191956,191956 +191957,191957 +191958,191958 +191959,191959 +59706,191959 +191960,191960 +191961,191961 +222475,191961 +220955,191961 +191962,191962 +191963,191963 +191964,191964 +191965,191965 +191966,191966 +191967,191967 +191968,191968 +191969,191969 +191970,191970 +191971,191971 +191972,191972 +191973,191973 +191974,191974 +191975,191975 +191976,191976 +191977,191977 +191978,191978 +191979,191979 +33603,191979 +191980,191980 +191981,191981 +191982,191982 +191983,191983 +191984,191984 +191985,191985 +191986,191986 +191987,191987 +191988,191988 +191989,191989 +191990,191990 +191991,191991 +191992,191992 +191993,191993 +191994,191994 +80709,191994 +191995,191995 +191996,191996 +191997,191997 +191998,191998 +191999,191999 +19913,191999 +192000,192000 +192001,192001 +192002,192002 +192003,192003 +192004,192004 +192005,192005 +192006,192006 +192007,192007 +192008,192008 +30580,192008 +192009,192009 +192010,192010 +189217,192010 +192011,192011 +192012,192012 +192013,192013 +192014,192014 +192015,192015 +192016,192016 +192017,192017 +192018,192018 +192019,192019 +192020,192020 +192021,192021 +192022,192022 +192023,192023 +192024,192024 +192025,192025 +192026,192026 +192027,192027 +192028,192028 +192029,192029 +191116,192029 +192030,192030 +192031,192031 +192032,192032 +192033,192033 +192034,192034 +192035,192035 +132716,192035 +125129,192035 +192036,192036 +192037,192037 +192038,192038 +192039,192039 +192040,192040 +192041,192041 +192042,192042 +192043,192043 +192044,192044 +192045,192045 +192046,192046 +192047,192047 +192048,192048 +192125,192125 +192126,192126 +121501,192126 +192127,192127 +192128,192128 +192129,192129 +192130,192130 +192131,192131 +192132,192132 +192133,192133 +192134,192134 +192135,192135 +192136,192136 +194460,192136 +192137,192137 +192138,192138 +192139,192139 +192140,192140 +192141,192141 +192142,192142 +192143,192143 +192144,192144 +212025,192144 +192145,192145 +192146,192146 +192147,192147 +192148,192148 +192149,192149 +192150,192150 +146567,192150 +192151,192151 +192152,192152 +192153,192153 +192154,192154 +192155,192155 +192156,192156 +192157,192157 +192158,192158 +192159,192159 +192160,192160 +192161,192161 +192162,192162 +192163,192163 +192164,192164 +192165,192165 +192166,192166 +192167,192167 +192168,192168 +192169,192169 +192170,192170 +187204,192170 +192171,192171 +192172,192172 +192173,192173 +192174,192174 +146712,192174 +192175,192175 +192176,192176 +192177,192177 +157376,192177 +192178,192178 +192179,192179 +192180,192180 +192181,192181 +192182,192182 +192183,192183 +192184,192184 +192185,192185 +192186,192186 +192187,192187 +192188,192188 +192189,192189 +192190,192190 +192191,192191 +192192,192192 +192193,192193 +192194,192194 +81489,192194 +192195,192195 +192196,192196 +192197,192197 +192198,192198 +192199,192199 +192200,192200 +192201,192201 +192202,192202 +192203,192203 +192204,192204 +192205,192205 +192206,192206 +192207,192207 +192208,192208 +192209,192209 +192210,192210 +35814,192210 +192211,192211 +192212,192212 +192213,192213 +192214,192214 +192215,192215 +192216,192216 +192217,192217 +192218,192218 +192219,192219 +192220,192220 +192221,192221 +192222,192222 +192223,192223 +192224,192224 +192225,192225 +192226,192226 +192227,192227 +192228,192228 +192229,192229 +192230,192230 +192231,192231 +192232,192232 +192233,192233 +192234,192234 +192235,192235 +192236,192236 +136313,192236 +128726,192236 +192237,192237 +192238,192238 +192239,192239 +192240,192240 +192241,192241 +192242,192242 +77230,192242 +191068,192242 +192243,192243 +192244,192244 +192245,192245 +192246,192246 +192247,192247 +192248,192248 +192249,192249 +192250,192250 +192251,192251 +192252,192252 +204011,192252 +192253,192253 +192254,192254 +192255,192255 +77357,192255 +192256,192256 +192257,192257 +192258,192258 +192259,192259 +192260,192260 +192261,192261 +192262,192262 +192263,192263 +192264,192264 +192265,192265 +192266,192266 +192267,192267 +180700,192267 +192268,192268 +56821,192268 +192269,192269 +192270,192270 +192271,192271 +192272,192272 +192273,192273 +192274,192274 +192275,192275 +192276,192276 +192277,192277 +192278,192278 +192279,192279 +192280,192280 +192281,192281 +58612,192281 +192282,192282 +192283,192283 +192284,192284 +192285,192285 +192286,192286 +192287,192287 +192288,192288 +192289,192289 +192290,192290 +192291,192291 +192292,192292 +192293,192293 +192294,192294 +192295,192295 +192296,192296 +192297,192297 +192298,192298 +192299,192299 +192300,192300 +192301,192301 +192302,192302 +193287,192302 +192303,192303 +192304,192304 +192305,192305 +192306,192306 +192307,192307 +192308,192308 +192309,192309 +192310,192310 +192311,192311 +192312,192312 +148629,192312 +192313,192313 +192314,192314 +192315,192315 +192316,192316 +192317,192317 +192318,192318 +192319,192319 +192320,192320 +192321,192321 +161019,192321 +192322,192322 +192323,192323 +192324,192324 +192325,192325 +192326,192326 +192327,192327 +192328,192328 +192329,192329 +192330,192330 +198034,192330 +192331,192331 +192332,192332 +60757,192332 +192333,192333 +192334,192334 +192335,192335 +192336,192336 +192337,192337 +192338,192338 +192339,192339 +192340,192340 +192341,192341 +243581,192341 +192342,192342 +192343,192343 +192344,192344 +192345,192345 +192346,192346 +192347,192347 +192348,192348 +192349,192349 +192350,192350 +192351,192351 +192352,192352 +192353,192353 +192354,192354 +192355,192355 +192356,192356 +192357,192357 +192358,192358 +192359,192359 +192360,192360 +192361,192361 +192362,192362 +192363,192363 +199259,192363 +192364,192364 +192365,192365 +192366,192366 +192367,192367 +192368,192368 +192369,192369 +192370,192370 +192371,192371 +192372,192372 +192373,192373 +192374,192374 +192375,192375 +192376,192376 +205838,192376 +192377,192377 +192378,192378 +192379,192379 +192380,192380 +111537,192380 +192381,192381 +192382,192382 +192383,192383 +192384,192384 +192385,192385 +70229,192385 +192386,192386 +192387,192387 +192388,192388 +192389,192389 +192390,192390 +192391,192391 +192392,192392 +192393,192393 +192394,192394 +192395,192395 +192396,192396 +192397,192397 +192398,192398 +192399,192399 +192400,192400 +107259,192400 +192401,192401 +192402,192402 +192403,192403 +192404,192404 +192405,192405 +192406,192406 +192407,192407 +192408,192408 +192409,192409 +192410,192410 +192411,192411 +192412,192412 +192413,192413 +192414,192414 +192415,192415 +192416,192416 +192417,192417 +192418,192418 +192419,192419 +192420,192420 +192421,192421 +192422,192422 +192423,192423 +83612,192423 +192424,192424 +192425,192425 +192426,192426 +29769,192426 +192427,192427 +192428,192428 +192429,192429 +192430,192430 +192431,192431 +192432,192432 +192433,192433 +192434,192434 +192435,192435 +192436,192436 +180662,192436 +192437,192437 +192438,192438 +192439,192439 +192440,192440 +192441,192441 +192442,192442 +192443,192443 +192444,192444 +192445,192445 +192446,192446 +153160,192446 +192447,192447 +192448,192448 +192449,192449 +192450,192450 +158462,192450 +192451,192451 +192452,192452 +192453,192453 +192454,192454 +192455,192455 +192456,192456 +192457,192457 +192458,192458 +192459,192459 +145459,192459 +192460,192460 +192461,192461 +192462,192462 +192463,192463 +192464,192464 +192465,192465 +192466,192466 +192467,192467 +192468,192468 +192469,192469 +192470,192470 +192471,192471 +192472,192472 +192473,192473 +192474,192474 +192475,192475 +192476,192476 +192477,192477 +192519,192519 +194122,192520 +192520,192520 +192521,192521 +192522,192522 +192523,192523 +192524,192524 +192525,192525 +192526,192526 +192527,192527 +192528,192528 +192529,192529 +192530,192530 +192531,192531 +192532,192532 +192533,192533 +192534,192534 +192535,192535 +192536,192536 +192537,192537 +192538,192538 +192539,192539 +192540,192540 +192541,192541 +114455,192541 +192542,192542 +192543,192543 +192544,192544 +192545,192545 +192546,192546 +192547,192547 +192548,192548 +192549,192549 +102387,192549 +192550,192550 +106348,192550 +192551,192551 +192552,192552 +220036,192552 +192553,192553 +192554,192554 +192555,192555 +192556,192556 +192557,192557 +192558,192558 +71142,192558 +192559,192559 +192560,192560 +192561,192561 +192562,192562 +142799,192562 +192563,192563 +213104,192563 +192564,192564 +192565,192565 +192566,192566 +192567,192567 +192568,192568 +145609,192568 +191753,192568 +192569,192569 +192570,192570 +192571,192571 +220014,192572 +192572,192572 +30596,192572 +192573,192573 +192574,192574 +192575,192575 +192576,192576 +192577,192577 +192578,192578 +192579,192579 +192580,192580 +192581,192581 +192582,192582 +192583,192583 +192584,192584 +192585,192585 +192586,192586 +192587,192587 +192588,192588 +192589,192589 +192590,192590 +192591,192591 +192592,192592 +192593,192593 +192594,192594 +192595,192595 +192596,192596 +192597,192597 +192598,192598 +192599,192599 +192600,192600 +192601,192601 +192602,192602 +192603,192603 +192604,192604 +192605,192605 +192606,192606 +192607,192607 +154371,192607 +192608,192608 +192609,192609 +192610,192610 +192611,192611 +59635,192611 +192612,192612 +192613,192613 +192614,192614 +192615,192615 +192616,192616 +192617,192617 +192618,192618 +192619,192619 +192620,192620 +192621,192621 +192622,192622 +192623,192623 +139966,192623 +192624,192624 +192625,192625 +192626,192626 +192627,192627 +81385,192627 +192628,192628 +192731,192731 +192732,192732 +192733,192733 +107984,192733 +192734,192734 +108067,192734 +192735,192735 +76017,192735 +192736,192736 +192824,192824 +192825,192825 +150579,192825 +192826,192826 +192827,192827 +192828,192828 +154763,192828 +192829,192829 +221946,192829 +192830,192830 +192831,192831 +192832,192832 +192833,192833 +192919,192919 +183420,192919 +192920,192920 +192921,192921 +192922,192922 +192923,192923 +192924,192924 +192925,192925 +192926,192926 +192927,192927 +192928,192928 +195109,192928 +192929,192929 +193038,193038 +193039,193039 +193040,193040 +9748,193040 +193041,193041 +193042,193042 +193043,193043 +193044,193044 +193045,193045 +193046,193046 +193047,193047 +193048,193048 +193049,193049 +193050,193050 +193051,193051 +193052,193052 +193053,193053 +193054,193054 +193055,193055 +193056,193056 +242299,193056 +193057,193057 +193058,193058 +193059,193059 +193060,193060 +193061,193061 +193062,193062 +193063,193063 +193064,193064 +193065,193065 +193066,193066 +193067,193067 +193068,193068 +193069,193069 +193070,193070 +193071,193071 +193072,193072 +193073,193073 +193074,193074 +193075,193075 +193076,193076 +193077,193077 +193140,193140 +193141,193141 +193142,193142 +193143,193143 +16186,193143 +193144,193144 +193145,193145 +193146,193146 +193147,193147 +30171,193147 +193148,193148 +7346,193148 +193149,193149 +193150,193150 +77996,193150 +193151,193151 +193152,193152 +193153,193153 +193154,193154 +213289,193154 +193155,193155 +193156,193156 +193157,193157 +201116,193157 +193158,193158 +193159,193159 +193160,193160 +193161,193161 +193162,193162 +193163,193163 +193164,193164 +193165,193165 +193166,193166 +193167,193167 +193168,193168 +193169,193169 +193170,193170 +193255,193255 +193256,193256 +157197,193256 +193257,193257 +193258,193258 +193259,193259 +204012,193259 +193260,193260 +92112,193260 +193261,193261 +193262,193262 +193263,193263 +193264,193264 +193265,193265 +193266,193266 +193267,193267 +193268,193268 +193269,193269 +193270,193270 +193271,193271 +133040,193271 +125453,193271 +193272,193272 +193273,193273 +193274,193274 +193275,193275 +193276,193276 +178191,193276 +172227,193276 +162175,193276 +193277,193277 +193278,193278 +62769,193278 +193279,193279 +193280,193280 +193281,193281 +193282,193282 +193283,193283 +193284,193284 +193285,193285 +193286,193286 +193287,193287 +192302,193287 +193288,193288 +193289,193289 +193290,193290 +193291,193291 +193292,193292 +193293,193293 +193294,193294 +193295,193295 +193296,193296 +193297,193297 +193298,193298 +193299,193299 +193300,193300 +193301,193301 +193302,193302 +220840,193302 +193303,193303 +193304,193304 +193305,193305 +193306,193306 +193307,193307 +193346,193346 +193347,193347 +193348,193348 +193349,193349 +193350,193350 +193351,193351 +193352,193352 +193353,193353 +193354,193354 +193355,193355 +193356,193356 +193357,193357 +193358,193358 +193359,193359 +193360,193360 +193361,193361 +193362,193362 +193363,193363 +193364,193364 +193365,193365 +112166,193365 +193366,193366 +193367,193367 +193368,193368 +193369,193369 +193370,193370 +193371,193371 +193372,193372 +193373,193373 +193374,193374 +193375,193375 +193376,193376 +193377,193377 +193378,193378 +193379,193379 +193380,193380 +193381,193381 +193382,193382 +193383,193383 +193384,193384 +193385,193385 +225084,193385 +193386,193386 +193387,193387 +193388,193388 +193389,193389 +193390,193390 +193391,193391 +193392,193392 +193393,193393 +193394,193394 +162536,193394 +193395,193395 +193396,193396 +193397,193397 +193398,193398 +193399,193399 +193400,193400 +193401,193401 +193402,193402 +193403,193403 +193404,193404 +193405,193405 +142748,193405 +193406,193406 +193407,193407 +193408,193408 +193409,193409 +193410,193410 +158301,193410 +193411,193411 +193412,193412 +193413,193413 +193414,193414 +59537,193414 +193415,193415 +193416,193416 +193417,193417 +193418,193418 +193419,193419 +193420,193420 +193421,193421 +59544,193421 +5609,193421 +193422,193422 +193423,193423 +193424,193424 +203804,193424 +193425,193425 +193426,193426 +193427,193427 +193428,193428 +8592,193428 +8412,193428 +193429,193429 +242618,193429 +193430,193430 +193431,193431 +193432,193432 +193433,193433 +193434,193434 +193435,193435 +193436,193436 +193437,193437 +193438,193438 +193439,193439 +193440,193440 +193441,193441 +193442,193442 +193443,193443 +193444,193444 +166271,193444 +193445,193445 +193446,193446 +193447,193447 +193448,193448 +193449,193449 +193450,193450 +193451,193451 +193452,193452 +193453,193453 +193454,193454 +193455,193455 +193456,193456 +193457,193457 +193458,193458 +193459,193459 +193460,193460 +193542,193542 +193543,193543 +201083,193544 +193544,193544 +193545,193545 +56682,193545 +48578,193545 +193546,193546 +162991,193546 +193547,193547 +193548,193548 +193549,193549 +193550,193550 +193551,193551 +193552,193552 +193553,193553 +193554,193554 +193555,193555 +193556,193556 +193557,193557 +193558,193558 +193559,193559 +193560,193560 +193561,193561 +193562,193562 +193563,193563 +193564,193564 +193565,193565 +193566,193566 +193567,193567 +193568,193568 +193569,193569 +11484,193569 +193570,193570 +193571,193571 +193572,193572 +193573,193573 +193574,193574 +4547,193574 +193575,193575 +193576,193576 +193577,193577 +193578,193578 +193670,193670 +193671,193671 +193672,193672 +193673,193673 +193674,193674 +193675,193675 +193676,193676 +193677,193677 +193678,193678 +193679,193679 +157210,193679 +193680,193680 +165344,193680 +193681,193681 +193682,193682 +193683,193683 +193684,193684 +193685,193685 +193686,193686 +193687,193687 +31145,193687 +198700,193687 +187121,193687 +38245,193687 +35011,193687 +193688,193688 +193689,193689 +178861,193689 +193690,193690 +193691,193691 +193692,193692 +26092,193692 +193693,193693 +193694,193694 +193695,193695 +193696,193696 +193697,193697 +193698,193698 +185509,193698 +193699,193699 +193700,193700 +193701,193701 +193702,193702 +193703,193703 +193704,193704 +193705,193705 +193706,193706 +193707,193707 +193708,193708 +193709,193709 +193710,193710 +193711,193711 +173970,193711 +168006,193711 +193712,193712 +193713,193713 +193714,193714 +193715,193715 +193716,193716 +213746,193717 +193717,193717 +193718,193718 +193719,193719 +193720,193720 +193721,193721 +193722,193722 +193723,193723 +178122,193723 +172158,193723 +193724,193724 +193725,193725 +193726,193726 +193727,193727 +193728,193728 +193729,193729 +193730,193730 +193731,193731 +88075,193731 +193732,193732 +193733,193733 +193734,193734 +193735,193735 +193736,193736 +193737,193737 +193738,193738 +193739,193739 +193740,193740 +193741,193741 +193742,193742 +193743,193743 +193744,193744 +193745,193745 +193746,193746 +193747,193747 +193748,193748 +193749,193749 +193750,193750 +152252,193750 +193751,193751 +193752,193752 +193753,193753 +193754,193754 +193755,193755 +193756,193756 +193757,193757 +193758,193758 +207990,193758 +193759,193759 +193760,193760 +193761,193761 +193762,193762 +193763,193763 +193764,193764 +193765,193765 +193766,193766 +193767,193767 +193768,193768 +193769,193769 +193770,193770 +193771,193771 +139145,193771 +193772,193772 +193773,193773 +193774,193774 +193775,193775 +193776,193776 +150739,193776 +193777,193777 +193778,193778 +193779,193779 +8766,193779 +193780,193780 +193781,193781 +193782,193782 +193783,193783 +193784,193784 +193785,193785 +193786,193786 +193787,193787 +81588,193787 +193788,193788 +193789,193789 +193790,193790 +193791,193791 +193792,193792 +193793,193793 +193794,193794 +193795,193795 +193796,193796 +193797,193797 +193798,193798 +193799,193799 +193800,193800 +193801,193801 +193802,193802 +193803,193803 +193804,193804 +193805,193805 +193806,193806 +193807,193807 +193808,193808 +193809,193809 +193810,193810 +33370,193810 +193811,193811 +149038,193811 +193812,193812 +193813,193813 +193814,193814 +193815,193815 +193816,193816 +193817,193817 +193818,193818 +84370,193818 +211341,193818 +193819,193819 +193820,193820 +193821,193821 +193822,193822 +193823,193823 +193824,193824 +193825,193825 +193826,193826 +193827,193827 +246980,193827 +193828,193828 +193829,193829 +193830,193830 +193831,193831 +193832,193832 +193833,193833 +57750,193833 +193834,193834 +193835,193835 +193836,193836 +193837,193837 +193838,193838 +193839,193839 +196711,193839 +193840,193840 +193841,193841 +193842,193842 +193843,193843 +193844,193844 +193845,193845 +193846,193846 +193847,193847 +193848,193848 +193849,193849 +194013,194013 +194014,194014 +194015,194015 +194016,194016 +194017,194017 +194018,194018 +194019,194019 +194020,194020 +194021,194021 +194022,194022 +194023,194023 +194024,194024 +194025,194025 +194026,194026 +3176,194026 +194027,194027 +194098,194098 +194099,194099 +110070,194099 +194100,194100 +194101,194101 +194102,194102 +194103,194103 +194104,194104 +110648,194104 +194105,194105 +963,194105 +194106,194106 +194107,194107 +194108,194108 +194109,194109 +194110,194110 +194111,194111 +194112,194112 +194113,194113 +194114,194114 +213613,194114 +194115,194115 +194116,194116 +194117,194117 +194118,194118 +194119,194119 +194120,194120 +194121,194121 +194122,194122 +192520,194122 +194123,194123 +194124,194124 +194125,194125 +33390,194125 +194126,194126 +194127,194127 +194128,194128 +194129,194129 +194130,194130 +194131,194131 +194132,194132 +194133,194133 +194134,194134 +194135,194135 +194136,194136 +194137,194137 +194138,194138 +194139,194139 +194140,194140 +194141,194141 +194142,194142 +194143,194143 +194144,194144 +194145,194145 +194146,194146 +194147,194147 +194148,194148 +194149,194149 +194150,194150 +194151,194151 +194152,194152 +194153,194153 +194154,194154 +194155,194155 +194156,194156 +194157,194157 +194158,194158 +194159,194159 +194160,194160 +194161,194161 +194162,194162 +194163,194163 +194164,194164 +194165,194165 +206563,194166 +194166,194166 +194167,194167 +194168,194168 +177502,194168 +171538,194168 +76279,194168 +194169,194169 +194170,194170 +194171,194171 +194172,194172 +194173,194173 +194174,194174 +194175,194175 +67277,194175 +194176,194176 +194177,194177 +194178,194178 +194179,194179 +194180,194180 +194181,194181 +194182,194182 +194183,194183 +194184,194184 +194185,194185 +194186,194186 +194187,194187 +110050,194187 +194188,194188 +194189,194189 +139982,194189 +194190,194190 +194191,194191 +194192,194192 +194193,194193 +194194,194194 +194195,194195 +194196,194196 +194197,194197 +194198,194198 +194199,194199 +194200,194200 +194201,194201 +194202,194202 +194203,194203 +194204,194204 +194205,194205 +194206,194206 +194207,194207 +194208,194208 +194209,194209 +194210,194210 +194211,194211 +194212,194212 +194213,194213 +194214,194214 +194215,194215 +194216,194216 +194217,194217 +194218,194218 +194219,194219 +194220,194220 +194221,194221 +194222,194222 +194223,194223 +194224,194224 +194225,194225 +194226,194226 +194227,194227 +194228,194228 +216277,194228 +194229,194229 +194230,194230 +194231,194231 +122953,194231 +194232,194232 +194233,194233 +194234,194234 +194235,194235 +194236,194236 +194237,194237 +194238,194238 +194239,194239 +194240,194240 +194241,194241 +194242,194242 +194243,194243 +194244,194244 +194245,194245 +194246,194246 +194247,194247 +120685,194247 +194248,194248 +194249,194249 +194250,194250 +194251,194251 +194252,194252 +194253,194253 +194254,194254 +194255,194255 +194256,194256 +194257,194257 +194258,194258 +194259,194259 +194260,194260 +194261,194261 +194262,194262 +194263,194263 +194264,194264 +33731,194264 +194265,194265 +194266,194266 +194267,194267 +194268,194268 +194269,194269 +194270,194270 +194271,194271 +194272,194272 +194273,194273 +194274,194274 +194275,194275 +194276,194276 +194277,194277 +194278,194278 +194279,194279 +194280,194280 +194281,194281 +194282,194282 +190397,194282 +194283,194283 +59523,194283 +194284,194284 +194285,194285 +194286,194286 +194287,194287 +194288,194288 +194289,194289 +194290,194290 +55675,194290 +47571,194290 +194291,194291 +194292,194292 +138767,194292 +194293,194293 +194294,194294 +194295,194295 +194296,194296 +194297,194297 +194298,194298 +194299,194299 +194300,194300 +194301,194301 +194302,194302 +194303,194303 +194304,194304 +194305,194305 +194306,194306 +194307,194307 +194308,194308 +194309,194309 +194310,194310 +194311,194311 +194312,194312 +194313,194313 +194314,194314 +194315,194315 +194316,194316 +161815,194316 +194317,194317 +194318,194318 +194319,194319 +194320,194320 +194321,194321 +194322,194322 +194323,194323 +194324,194324 +194325,194325 +194326,194326 +194327,194327 +194328,194328 +194329,194329 +194330,194330 +99320,194330 +194331,194331 +194332,194332 +194333,194333 +194334,194334 +194335,194335 +194336,194336 +194337,194337 +194338,194338 +194339,194339 +194340,194340 +111221,194340 +194341,194341 +154784,194341 +240519,194341 +194342,194342 +194343,194343 +194344,194344 +194345,194345 +151579,194345 +194346,194346 +194347,194347 +194348,194348 +194349,194349 +194350,194350 +194351,194351 +194352,194352 +194353,194353 +194354,194354 +194355,194355 +194356,194356 +194357,194357 +194358,194358 +194359,194359 +194360,194360 +194361,194361 +135098,194361 +127511,194361 +194362,194362 +194363,194363 +194364,194364 +194365,194365 +194366,194366 +194367,194367 +194368,194368 +155164,194368 +194369,194369 +194370,194370 +155166,194370 +194371,194371 +194372,194372 +194373,194373 +194374,194374 +194375,194375 +194376,194376 +194377,194377 +194378,194378 +194379,194379 +194380,194380 +194381,194381 +194382,194382 +194383,194383 +194384,194384 +194385,194385 +194386,194386 +194387,194387 +194388,194388 +194389,194389 +194390,194390 +194391,194391 +194392,194392 +194393,194393 +194394,194394 +194395,194395 +194396,194396 +194397,194397 +194398,194398 +194399,194399 +194400,194400 +194401,194401 +194402,194402 +161546,194402 +194403,194403 +194404,194404 +194405,194405 +194406,194406 +194407,194407 +104898,194407 +194408,194408 +194409,194409 +194410,194410 +194411,194411 +194412,194412 +194413,194413 +194414,194414 +194415,194415 +194416,194416 +194417,194417 +194418,194418 +194419,194419 +194420,194420 +194421,194421 +194422,194422 +194423,194423 +194424,194424 +194425,194425 +194426,194426 +194427,194427 +194428,194428 +194429,194429 +194430,194430 +194431,194431 +194432,194432 +194433,194433 +194434,194434 +194435,194435 +87251,194435 +194436,194436 +194437,194437 +194438,194438 +194439,194439 +194440,194440 +194441,194441 +194442,194442 +194443,194443 +220289,194444 +194444,194444 +194445,194445 +194446,194446 +194447,194447 +968,194447 +194448,194448 +242920,194448 +194449,194449 +194450,194450 +194451,194451 +194452,194452 +194453,194453 +194454,194454 +194455,194455 +30000,194455 +194456,194456 +194457,194457 +194458,194458 +194459,194459 +194460,194460 +192136,194460 +194461,194461 +194462,194462 +194463,194463 +194464,194464 +194465,194465 +194466,194466 +194467,194467 +194468,194468 +194469,194469 +194470,194470 +194471,194471 +194472,194472 +194473,194473 +194474,194474 +194475,194475 +194476,194476 +194477,194477 +194478,194478 +194479,194479 +194480,194480 +194481,194481 +194482,194482 +194483,194483 +194543,194543 +194544,194544 +194545,194545 +194709,194709 +194710,194710 +194711,194711 +194712,194712 +194713,194713 +194714,194714 +194715,194715 +194716,194716 +194717,194717 +232030,194717 +194718,194718 +194719,194719 +90750,194719 +194720,194720 +194721,194721 +65464,194721 +194722,194722 +194723,194723 +194724,194724 +194725,194725 +10161,194725 +194726,194726 +103630,194726 +194727,194727 +194728,194728 +194729,194729 +194730,194730 +194731,194731 +194732,194732 +194733,194733 +194734,194734 +194735,194735 +194736,194736 +194737,194737 +194738,194738 +194739,194739 +194740,194740 +194741,194741 +194742,194742 +194743,194743 +194744,194744 +194745,194745 +194746,194746 +194747,194747 +101988,194747 +194748,194748 +194749,194749 +212120,194750 +194750,194750 +194751,194751 +220311,194751 +194752,194752 +194753,194753 +194754,194754 +194755,194755 +194756,194756 +194757,194757 +194758,194758 +194759,194759 +194760,194760 +194761,194761 +194762,194762 +194763,194763 +194764,194764 +194765,194765 +117709,194765 +194766,194766 +194767,194767 +194768,194768 +103378,194768 +194769,194769 +194770,194770 +194771,194771 +194772,194772 +194773,194773 +194774,194774 +194775,194775 +194776,194776 +194777,194777 +145394,194777 +194778,194778 +194779,194779 +194780,194780 +194781,194781 +194782,194782 +194783,194783 +194784,194784 +194785,194785 +194786,194786 +194787,194787 +194788,194788 +83557,194788 +194789,194789 +194790,194790 +194852,194852 +194853,194853 +194854,194854 +194855,194855 +194856,194856 +201068,194856 +194857,194857 +194858,194858 +194909,194909 +83619,194909 +194910,194910 +194911,194911 +194912,194912 +194913,194913 +194914,194914 +194915,194915 +207984,194916 +194916,194916 +194917,194917 +194918,194918 +194919,194919 +194920,194920 +194921,194921 +154832,194921 +194922,194922 +194923,194923 +194924,194924 +194925,194925 +194926,194926 +194927,194927 +194928,194928 +194929,194929 +194930,194930 +194931,194931 +194932,194932 +194933,194933 +76279,194933 +194934,194934 +194935,194935 +194936,194936 +194937,194937 +194938,194938 +194939,194939 +194940,194940 +194941,194941 +194942,194942 +194943,194943 +194944,194944 +194945,194945 +194946,194946 +194947,194947 +194948,194948 +194949,194949 +98141,194949 +194950,194950 +194951,194951 +194952,194952 +194953,194953 +194954,194954 +194955,194955 +194956,194956 +194957,194957 +194958,194958 +194959,194959 +194960,194960 +194961,194961 +194962,194962 +194963,194963 +194964,194964 +163687,194964 +194965,194965 +112123,194965 +194966,194966 +194967,194967 +194968,194968 +194969,194969 +113587,194969 +194970,194970 +194971,194971 +194972,194972 +194973,194973 +194974,194974 +194975,194975 +132965,194975 +125378,194975 +194976,194976 +194977,194977 +194978,194978 +195031,195031 +195032,195032 +195033,195033 +195034,195034 +211759,195034 +195035,195035 +195036,195036 +195037,195037 +195038,195038 +195039,195039 +195040,195040 +195041,195041 +195042,195042 +195043,195043 +195044,195044 +195092,195092 +195093,195093 +195094,195094 +195095,195095 +195096,195096 +67606,195096 +195097,195097 +195098,195098 +195099,195099 +195100,195100 +195101,195101 +195102,195102 +195103,195103 +195104,195104 +195105,195105 +195106,195106 +195107,195107 +195108,195108 +195109,195109 +192928,195109 +195110,195110 +195111,195111 +195112,195112 +195113,195113 +195114,195114 +195115,195115 +195116,195116 +195117,195117 +195118,195118 +195119,195119 +195120,195120 +195121,195121 +195122,195122 +204647,195122 +195123,195123 +195124,195124 +195125,195125 +195126,195126 +195127,195127 +195128,195128 +195129,195129 +195130,195130 +113449,195130 +195131,195131 +195132,195132 +195133,195133 +195134,195134 +195135,195135 +195136,195136 +195137,195137 +195138,195138 +195139,195139 +195140,195140 +195141,195141 +195142,195142 +195143,195143 +195144,195144 +195145,195145 +195146,195146 +195147,195147 +195148,195148 +95881,195148 +195149,195149 +195150,195150 +195151,195151 +195152,195152 +195153,195153 +195154,195154 +195155,195155 +195156,195156 +195157,195157 +195158,195158 +195159,195159 +195160,195160 +195161,195161 +195162,195162 +26907,195162 +195163,195163 +195164,195164 +195165,195165 +66819,195165 +195166,195166 +195167,195167 +195168,195168 +195169,195169 +195170,195170 +195171,195171 +195172,195172 +195173,195173 +195174,195174 +39977,195174 +195175,195175 +195176,195176 +195177,195177 +195178,195178 +195179,195179 +195180,195180 +2185,195180 +195181,195181 +195182,195182 +195183,195183 +195184,195184 +100656,195184 +195185,195185 +195186,195186 +195187,195187 +195188,195188 +195189,195189 +195190,195190 +195191,195191 +195192,195192 +195193,195193 +195194,195194 +195195,195195 +195196,195196 +195197,195197 +195198,195198 +195199,195199 +195200,195200 +195201,195201 +103462,195201 +195202,195202 +195203,195203 +195204,195204 +195205,195205 +195206,195206 +195207,195207 +195208,195208 +195209,195209 +195210,195210 +195211,195211 +195301,195301 +195302,195302 +195303,195303 +195304,195304 +195305,195305 +195306,195306 +195307,195307 +195308,195308 +195397,195397 +195398,195398 +195440,195440 +195441,195441 +195515,195515 +195516,195516 +195517,195517 +231245,195517 +195518,195518 +3492,195518 +195519,195519 +195520,195520 +195521,195521 +195522,195522 +195523,195523 +195524,195524 +195525,195525 +195526,195526 +195527,195527 +195528,195528 +195529,195529 +195530,195530 +195531,195531 +195532,195532 +195533,195533 +195534,195534 +195535,195535 +195536,195536 +195537,195537 +195538,195538 +195539,195539 +195540,195540 +195541,195541 +195542,195542 +195543,195543 +195544,195544 +195545,195545 +195546,195546 +195547,195547 +195548,195548 +195549,195549 +195550,195550 +195551,195551 +195552,195552 +155157,195552 +195553,195553 +195554,195554 +195555,195555 +195556,195556 +195557,195557 +195558,195558 +195559,195559 +195560,195560 +195561,195561 +195562,195562 +217592,195563 +195563,195563 +195564,195564 +195565,195565 +195566,195566 +195567,195567 +195568,195568 +99579,195568 +195569,195569 +195570,195570 +195571,195571 +140427,195571 +195572,195572 +195573,195573 +195574,195574 +195575,195575 +195576,195576 +195689,195689 +195690,195690 +195691,195691 +195692,195692 +195693,195693 +195694,195694 +195695,195695 +195696,195696 +195697,195697 +195698,195698 +195699,195699 +195700,195700 +195701,195701 +195702,195702 +195703,195703 +195704,195704 +195705,195705 +195706,195706 +195707,195707 +195708,195708 +195709,195709 +195710,195710 +195711,195711 +195712,195712 +195713,195713 +195714,195714 +195715,195715 +195716,195716 +195717,195717 +195718,195718 +195719,195719 +195720,195720 +195721,195721 +195722,195722 +195775,195775 +195776,195776 +195777,195777 +195778,195778 +195779,195779 +195780,195780 +195781,195781 +195782,195782 +195783,195783 +195784,195784 +195785,195785 +195786,195786 +195787,195787 +195788,195788 +195789,195789 +195790,195790 +190267,195790 +195791,195791 +195792,195792 +29879,195792 +195793,195793 +195794,195794 +195795,195795 +195796,195796 +195797,195797 +195798,195798 +195799,195799 +195800,195800 +195801,195801 +195802,195802 +195803,195803 +195804,195804 +195805,195805 +195806,195806 +195807,195807 +195808,195808 +195809,195809 +195810,195810 +195811,195811 +4495,195811 +195812,195812 +234920,195812 +195813,195813 +195814,195814 +195815,195815 +195868,195868 +195869,195869 +195870,195870 +195871,195871 +195872,195872 +195873,195873 +195874,195874 +195875,195875 +195876,195876 +195877,195877 +196259,196259 +196260,196260 +196261,196261 +196262,196262 +196263,196263 +196264,196264 +196265,196265 +196266,196266 +196267,196267 +196268,196268 +196269,196269 +196270,196270 +7684,196270 +196271,196271 +196272,196272 +196273,196273 +196274,196274 +196275,196275 +196276,196276 +196277,196277 +79914,196277 +196278,196278 +196279,196279 +196280,196280 +196281,196281 +196282,196282 +196283,196283 +196284,196284 +161978,196284 +196285,196285 +196286,196286 +196287,196287 +196288,196288 +196289,196289 +196290,196290 +196291,196291 +196292,196292 +196293,196293 +78049,196293 +196294,196294 +196295,196295 +196296,196296 +196297,196297 +196298,196298 +196299,196299 +196300,196300 +196301,196301 +196302,196302 +196303,196303 +196304,196304 +196305,196305 +50218,196305 +42026,196305 +196306,196306 +196307,196307 +196308,196308 +196309,196309 +196310,196310 +196311,196311 +196312,196312 +196313,196313 +196314,196314 +196315,196315 +196316,196316 +196317,196317 +115941,196317 +196318,196318 +196319,196319 +196320,196320 +196321,196321 +196322,196322 +196323,196323 +196324,196324 +196325,196325 +196326,196326 +196327,196327 +196328,196328 +196329,196329 +196330,196330 +196331,196331 +196332,196332 +196333,196333 +196334,196334 +196478,196478 +196479,196479 +196480,196480 +196481,196481 +196482,196482 +110817,196482 +189987,196482 +196483,196483 +196484,196484 +196485,196485 +196486,196486 +196487,196487 +196488,196488 +196489,196489 +196490,196490 +196491,196491 +196492,196492 +196493,196493 +196494,196494 +196495,196495 +196496,196496 +196497,196497 +196498,196498 +196499,196499 +75436,196499 +196500,196500 +196501,196501 +196502,196502 +196503,196503 +205083,196503 +196504,196504 +196505,196505 +196506,196506 +196507,196507 +196508,196508 +196509,196509 +196510,196510 +196511,196511 +196512,196512 +79888,196512 +154849,196512 +59642,196512 +196513,196513 +196514,196514 +196515,196515 +196516,196516 +196517,196517 +196518,196518 +196519,196519 +196520,196520 +209894,196521 +196521,196521 +196522,196522 +196523,196523 +196524,196524 +196525,196525 +196526,196526 +196527,196527 +196528,196528 +196529,196529 +196530,196530 +196531,196531 +196532,196532 +196533,196533 +196534,196534 +196535,196535 +184453,196535 +196536,196536 +196537,196537 +196538,196538 +196539,196539 +196540,196540 +39790,196540 +196541,196541 +196542,196542 +196543,196543 +196544,196544 +196545,196545 +196546,196546 +196547,196547 +196548,196548 +196549,196549 +196550,196550 +196551,196551 +196552,196552 +196553,196553 +196554,196554 +196555,196555 +196556,196556 +196557,196557 +196558,196558 +196559,196559 +196560,196560 +246967,196560 +196561,196561 +196562,196562 +111208,196562 +196563,196563 +202387,196563 +196564,196564 +196565,196565 +196566,196566 +244833,196566 +196567,196567 +196568,196568 +196569,196569 +196570,196570 +196571,196571 +196572,196572 +196573,196573 +217916,196573 +196574,196574 +39741,196574 +196575,196575 +196576,196576 +196577,196577 +196578,196578 +196579,196579 +196580,196580 +196581,196581 +196582,196582 +196583,196583 +51522,196583 +43418,196583 +196584,196584 +196585,196585 +196586,196586 +196587,196587 +196588,196588 +196589,196589 +196590,196590 +196591,196591 +196592,196592 +196593,196593 +196594,196594 +196595,196595 +196596,196596 +196597,196597 +196598,196598 +196599,196599 +196600,196600 +196601,196601 +196602,196602 +196603,196603 +196604,196604 +196605,196605 +196606,196606 +196607,196607 +196608,196608 +196609,196609 +196610,196610 +196611,196611 +196612,196612 +196613,196613 +196614,196614 +196615,196615 +196616,196616 +196617,196617 +196669,196669 +196670,196670 +196671,196671 +196672,196672 +196673,196673 +196674,196674 +196675,196675 +196676,196676 +196677,196677 +196678,196678 +155105,196678 +196679,196679 +196680,196680 +213643,196680 +196681,196681 +196682,196682 +196683,196683 +196684,196684 +196685,196685 +196686,196686 +196687,196687 +196688,196688 +196689,196689 +196690,196690 +196691,196691 +196692,196692 +196693,196693 +196694,196694 +196695,196695 +196696,196696 +196697,196697 +196698,196698 +101866,196698 +196699,196699 +196700,196700 +196701,196701 +196702,196702 +196703,196703 +196704,196704 +196705,196705 +196706,196706 +196707,196707 +196708,196708 +196709,196709 +196710,196710 +164936,196710 +196711,196711 +193839,196711 +221980,196711 +196712,196712 +107583,196712 +196713,196713 +196714,196714 +196715,196715 +51801,196715 +43697,196715 +196716,196716 +196717,196717 +196718,196718 +196719,196719 +196720,196720 +196721,196721 +196722,196722 +161545,196722 +196723,196723 +196724,196724 +196725,196725 +196726,196726 +196727,196727 +196728,196728 +196729,196729 +196730,196730 +196731,196731 +196732,196732 +196733,196733 +196734,196734 +196735,196735 +196736,196736 +196737,196737 +196738,196738 +235478,196738 +196739,196739 +196740,196740 +196741,196741 +196742,196742 +196743,196743 +196744,196744 +196745,196745 +196746,196746 +196747,196747 +196748,196748 +196749,196749 +196750,196750 +196751,196751 +196752,196752 +196753,196753 +196754,196754 +196755,196755 +81600,196755 +196756,196756 +196757,196757 +196758,196758 +196759,196759 +196760,196760 +196761,196761 +196810,196810 +36455,196810 +196811,196811 +196812,196812 +196813,196813 +196814,196814 +196815,196815 +196816,196816 +196918,196918 +196919,196919 +202435,196919 +196920,196920 +196921,196921 +196922,196922 +196923,196923 +196924,196924 +196925,196925 +196926,196926 +196927,196927 +196928,196928 +196929,196929 +196930,196930 +196931,196931 +19164,196931 +196932,196932 +196933,196933 +196934,196934 +196935,196935 +196936,196936 +196937,196937 +197056,197056 +197057,197057 +197058,197058 +197059,197059 +197060,197060 +197098,197098 +197099,197099 +197100,197100 +197101,197101 +197102,197102 +197314,197314 +197315,197315 +197316,197316 +197317,197317 +199859,197317 +197318,197318 +197319,197319 +197320,197320 +197321,197321 +197322,197322 +197323,197323 +111828,197323 +197324,197324 +197325,197325 +145636,197325 +197326,197326 +197327,197327 +197328,197328 +197329,197329 +197330,197330 +197331,197331 +197332,197332 +197333,197333 +197334,197334 +197335,197335 +197336,197336 +197337,197337 +205819,197337 +197338,197338 +197339,197339 +206816,197339 +197340,197340 +197341,197341 +197342,197342 +197343,197343 +197386,197386 +197387,197387 +202376,197387 +197388,197388 +197428,197428 +197429,197429 +197430,197430 +197431,197431 +197432,197432 +197433,197433 +197434,197434 +197435,197435 +197436,197436 +197437,197437 +197438,197438 +197439,197439 +197440,197440 +197441,197441 +197442,197442 +197492,197492 +197493,197493 +197494,197494 +197495,197495 +197496,197496 +197497,197497 +197498,197498 +197499,197499 +197500,197500 +197501,197501 +197502,197502 +197503,197503 +197504,197504 +197505,197505 +197506,197506 +186390,197506 +197507,197507 +197508,197508 +197509,197509 +197510,197510 +197511,197511 +197512,197512 +197513,197513 +197514,197514 +197515,197515 +197516,197516 +197517,197517 +197518,197518 +197519,197519 +197520,197520 +197521,197521 +197522,197522 +197523,197523 +197524,197524 +197525,197525 +197526,197526 +197527,197527 +197528,197528 +197529,197529 +197530,197530 +197531,197531 +197532,197532 +197533,197533 +178654,197533 +197534,197534 +197535,197535 +197536,197536 +197537,197537 +197538,197538 +197539,197539 +2481,197539 +197540,197540 +197541,197541 +197542,197542 +197543,197543 +178143,197543 +172179,197543 +197544,197544 +197545,197545 +197546,197546 +197547,197547 +197548,197548 +197549,197549 +197550,197550 +197551,197551 +197552,197552 +197553,197553 +197554,197554 +197555,197555 +197556,197556 +197557,197557 +197558,197558 +197559,197559 +197560,197560 +197561,197561 +197562,197562 +117823,197562 +197563,197563 +197564,197564 +197565,197565 +197566,197566 +197567,197567 +197568,197568 +197569,197569 +197570,197570 +197571,197571 +197572,197572 +197573,197573 +122999,197573 +197574,197574 +197575,197575 +197576,197576 +197577,197577 +197578,197578 +197579,197579 +197580,197580 +197581,197581 +197582,197582 +197583,197583 +197584,197584 +197585,197585 +197586,197586 +197587,197587 +197588,197588 +197589,197589 +197590,197590 +197591,197591 +197592,197592 +197593,197593 +197594,197594 +197595,197595 +156578,197595 +197596,197596 +197597,197597 +197598,197598 +197599,197599 +197600,197600 +197601,197601 +197602,197602 +197603,197603 +197604,197604 +197605,197605 +222130,197605 +197606,197606 +197607,197607 +226505,197607 +197608,197608 +197609,197609 +197610,197610 +197611,197611 +197612,197612 +197613,197613 +197614,197614 +197615,197615 +197616,197616 +197617,197617 +197618,197618 +197619,197619 +197620,197620 +197621,197621 +197622,197622 +197623,197623 +197624,197624 +197625,197625 +197626,197626 +197627,197627 +197628,197628 +197629,197629 +197675,197675 +197676,197676 +197677,197677 +197678,197678 +197679,197679 +197680,197680 +197681,197681 +197682,197682 +197683,197683 +197684,197684 +197685,197685 +214568,197685 +197686,197686 +197687,197687 +197688,197688 +197689,197689 +84080,197689 +197690,197690 +197691,197691 +197692,197692 +197693,197693 +4317,197693 +197694,197694 +197695,197695 +85816,197695 +197696,197696 +197697,197697 +197698,197698 +197699,197699 +197700,197700 +197701,197701 +197702,197702 +197703,197703 +197704,197704 +197705,197705 +197706,197706 +149876,197706 +197707,197707 +197708,197708 +197709,197709 +197710,197710 +197711,197711 +197712,197712 +197713,197713 +197714,197714 +197715,197715 +197716,197716 +197717,197717 +197718,197718 +197719,197719 +197720,197720 +197721,197721 +4204,197721 +197722,197722 +197723,197723 +197724,197724 +197725,197725 +197726,197726 +197727,197727 +197728,197728 +197729,197729 +197730,197730 +197731,197731 +242097,197731 +197732,197732 +197733,197733 +197734,197734 +197735,197735 +197736,197736 +197737,197737 +197738,197738 +197739,197739 +197740,197740 +197741,197741 +197742,197742 +197743,197743 +197744,197744 +197745,197745 +197746,197746 +197747,197747 +197748,197748 +197749,197749 +70339,197749 +197750,197750 +197751,197751 +197752,197752 +197753,197753 +197754,197754 +197755,197755 +197756,197756 +197757,197757 +197758,197758 +197759,197759 +197760,197760 +197761,197761 +215604,197761 +197808,197808 +197809,197809 +197810,197810 +197811,197811 +197812,197812 +197813,197813 +197814,197814 +197815,197815 +197816,197816 +197817,197817 +197818,197818 +197819,197819 +197820,197820 +164772,197820 +197821,197821 +197822,197822 +197823,197823 +197824,197824 +197825,197825 +197826,197826 +197827,197827 +36887,197827 +197828,197828 +197829,197829 +197830,197830 +197831,197831 +197832,197832 +197833,197833 +197834,197834 +197835,197835 +197836,197836 +197837,197837 +197838,197838 +197888,197888 +198003,198003 +198004,198004 +198005,198005 +198006,198006 +198007,198007 +198008,198008 +198009,198009 +198010,198010 +198011,198011 +198012,198012 +198013,198013 +198014,198014 +198015,198015 +198016,198016 +198017,198017 +198018,198018 +198019,198019 +203763,198019 +198020,198020 +198021,198021 +198022,198022 +198023,198023 +198024,198024 +198025,198025 +198026,198026 +198027,198027 +30824,198027 +235738,198027 +198028,198028 +103370,198028 +198029,198029 +198030,198030 +198031,198031 +198032,198032 +198033,198033 +198034,198034 +192330,198034 +416,198034 +198035,198035 +198036,198036 +198037,198037 +11503,198037 +198038,198038 +198039,198039 +198395,198395 +198396,198396 +198397,198397 +198398,198398 +198399,198399 +198400,198400 +198401,198401 +198402,198402 +198403,198403 +198404,198404 +84104,198404 +198405,198405 +198406,198406 +198407,198407 +198408,198408 +198409,198409 +198410,198410 +220017,198410 +198411,198411 +198412,198412 +198413,198413 +198414,198414 +136379,198414 +128792,198414 +198415,198415 +36432,198415 +53426,198415 +45322,198415 +198416,198416 +198417,198417 +198418,198418 +198419,198419 +198420,198420 +198421,198421 +198422,198422 +198423,198423 +198424,198424 +198425,198425 +198426,198426 +198427,198427 +198428,198428 +198429,198429 +198430,198430 +198431,198431 +198432,198432 +198433,198433 +198434,198434 +198435,198435 +198436,198436 +198437,198437 +198438,198438 +198439,198439 +198440,198440 +198441,198441 +198442,198442 +198443,198443 +198444,198444 +122859,198444 +198445,198445 +198446,198446 +198447,198447 +198448,198448 +198449,198449 +198450,198450 +198451,198451 +198452,198452 +198453,198453 +198454,198454 +83671,198454 +198455,198455 +198456,198456 +198457,198457 +198458,198458 +198459,198459 +198460,198460 +78383,198460 +198461,198461 +198462,198462 +198463,198463 +198464,198464 +198465,198465 +198466,198466 +198467,198467 +198468,198468 +198469,198469 +198470,198470 +239524,198470 +157019,198470 +198471,198471 +198472,198472 +198473,198473 +198474,198474 +198580,198580 +198581,198581 +198654,198654 +198655,198655 +198656,198656 +198657,198657 +198658,198658 +246001,198658 +198659,198659 +198660,198660 +198661,198661 +198662,198662 +198663,198663 +198664,198664 +198665,198665 +198666,198666 +198667,198667 +198668,198668 +198669,198669 +198670,198670 +198671,198671 +198672,198672 +198673,198673 +198674,198674 +198675,198675 +198676,198676 +198677,198677 +198678,198678 +198679,198679 +198680,198680 +198681,198681 +198682,198682 +198683,198683 +198684,198684 +198685,198685 +198686,198686 +198687,198687 +198688,198688 +198689,198689 +146742,198689 +114875,198689 +198690,198690 +198691,198691 +198692,198692 +198693,198693 +198694,198694 +198695,198695 +198696,198696 +198697,198697 +198698,198698 +198699,198699 +198700,198700 +187121,198700 +193687,198700 +198701,198701 +198702,198702 +227083,198702 +198703,198703 +198704,198704 +198705,198705 +198706,198706 +198707,198707 +198708,198708 +89512,198708 +198709,198709 +198710,198710 +198711,198711 +160858,198711 +198712,198712 +198713,198713 +198714,198714 +198715,198715 +198716,198716 +198717,198717 +198718,198718 +198719,198719 +218015,198719 +198720,198720 +198721,198721 +198722,198722 +198723,198723 +198724,198724 +5390,198724 +198725,198725 +198726,198726 +198727,198727 +198728,198728 +198729,198729 +198730,198730 +198731,198731 +198732,198732 +198733,198733 +198734,198734 +198735,198735 +198736,198736 +106881,198736 +198737,198737 +198738,198738 +49818,198738 +41625,198738 +198739,198739 +198740,198740 +198741,198741 +67606,198741 +198742,198742 +198743,198743 +80320,198743 +198744,198744 +198745,198745 +198746,198746 +198747,198747 +198748,198748 +49818,198748 +41625,198748 +198749,198749 +198750,198750 +198751,198751 +198752,198752 +198753,198753 +198754,198754 +198755,198755 +198756,198756 +198757,198757 +198758,198758 +198759,198759 +198760,198760 +198801,198801 +198802,198802 +198803,198803 +87127,198803 +198804,198804 +198805,198805 +198806,198806 +198807,198807 +120312,198807 +198808,198808 +198809,198809 +198810,198810 +198811,198811 +198812,198812 +198813,198813 +198814,198814 +198815,198815 +198816,198816 +198817,198817 +198818,198818 +198819,198819 +153634,198819 +198820,198820 +58080,198820 +153635,198820 +198821,198821 +198822,198822 +198916,198916 +198917,198917 +198918,198918 +198919,198919 +57323,198919 +198920,198920 +198977,198977 +198978,198978 +198979,198979 +198980,198980 +198981,198981 +199125,199125 +199126,199126 +199127,199127 +199128,199128 +199129,199129 +199130,199130 +199131,199131 +199132,199132 +199133,199133 +199134,199134 +199135,199135 +199136,199136 +199137,199137 +199138,199138 +199184,199184 +199185,199185 +199186,199186 +199187,199187 +199188,199188 +199189,199189 +199190,199190 +136116,199190 +128529,199190 +199191,199191 +199192,199192 +199193,199193 +199194,199194 +199195,199195 +243222,199195 +199196,199196 +199197,199197 +199198,199198 +199199,199199 +199200,199200 +199201,199201 +199202,199202 +199203,199203 +199204,199204 +199205,199205 +199206,199206 +199207,199207 +70437,199207 +199208,199208 +199209,199209 +199210,199210 +199211,199211 +199212,199212 +199213,199213 +199214,199214 +199215,199215 +199216,199216 +199217,199217 +186317,199217 +199218,199218 +199219,199219 +199220,199220 +199221,199221 +199222,199222 +199223,199223 +199224,199224 +199225,199225 +199226,199226 +199227,199227 +199228,199228 +199229,199229 +199230,199230 +199231,199231 +199232,199232 +199233,199233 +140492,199233 +199234,199234 +51002,199234 +42898,199234 +199235,199235 +199236,199236 +199237,199237 +199238,199238 +199239,199239 +199240,199240 +199241,199241 +199242,199242 +199243,199243 +199244,199244 +199245,199245 +199246,199246 +199247,199247 +199248,199248 +199249,199249 +199250,199250 +199251,199251 +199252,199252 +199253,199253 +248006,199253 +199254,199254 +199255,199255 +199256,199256 +199257,199257 +199258,199258 +199259,199259 +192363,199259 +199260,199260 +199261,199261 +199262,199262 +199263,199263 +199264,199264 +199265,199265 +2836,199265 +199266,199266 +199267,199267 +199268,199268 +199269,199269 +199270,199270 +199271,199271 +199272,199272 +199273,199273 +199274,199274 +199275,199275 +199276,199276 +199277,199277 +199278,199278 +199279,199279 +199280,199280 +199281,199281 +199282,199282 +199283,199283 +199284,199284 +199285,199285 +199286,199286 +199287,199287 +199288,199288 +199289,199289 +199290,199290 +199291,199291 +15821,199291 +199292,199292 +199293,199293 +235162,199293 +199294,199294 +107184,199294 +199295,199295 +199296,199296 +199297,199297 +199298,199298 +92721,199298 +199299,199299 +199300,199300 +199301,199301 +199302,199302 +199303,199303 +199304,199304 +199341,199341 +199342,199342 +199343,199343 +199344,199344 +199345,199345 +199346,199346 +199347,199347 +199348,199348 +199349,199349 +199350,199350 +199351,199351 +199352,199352 +199353,199353 +199354,199354 +199530,199530 +199531,199531 +199532,199532 +199533,199533 +199534,199534 +199535,199535 +199536,199536 +199537,199537 +199538,199538 +199539,199539 +199540,199540 +122816,199540 +199541,199541 +199542,199542 +199543,199543 +199544,199544 +199545,199545 +199546,199546 +199547,199547 +199548,199548 +199549,199549 +9362,199549 +199550,199550 +199551,199551 +199552,199552 +199553,199553 +199554,199554 +199555,199555 +199556,199556 +199557,199557 +199558,199558 +199559,199559 +161395,199559 +199560,199560 +109215,199560 +216033,199560 +199561,199561 +199562,199562 +199563,199563 +199564,199564 +199565,199565 +199566,199566 +199567,199567 +199568,199568 +199569,199569 +199570,199570 +230956,199570 +199571,199571 +199572,199572 +199573,199573 +199574,199574 +199575,199575 +199576,199576 +199577,199577 +199578,199578 +199579,199579 +199580,199580 +199581,199581 +199582,199582 +199583,199583 +160694,199583 +199584,199584 +87139,199584 +199585,199585 +199586,199586 +199587,199587 +199588,199588 +199589,199589 +219919,199589 +199590,199590 +199591,199591 +199592,199592 +199593,199593 +199594,199594 +199595,199595 +199596,199596 +199597,199597 +199598,199598 +202864,199599 +199599,199599 +199600,199600 +199601,199601 +199602,199602 +199603,199603 +199604,199604 +199605,199605 +199606,199606 +199607,199607 +199608,199608 +199609,199609 +199610,199610 +199611,199611 +199612,199612 +199613,199613 +199614,199614 +199615,199615 +199708,199708 +159590,199708 +199709,199709 +145464,199709 +199710,199710 +199711,199711 +199712,199712 +199713,199713 +84629,199713 +199714,199714 +199715,199715 +199716,199716 +199717,199717 +199718,199718 +199719,199719 +199720,199720 +199721,199721 +199722,199722 +142541,199722 +199723,199723 +199724,199724 +75738,199724 +199725,199725 +199726,199726 +199727,199727 +199728,199728 +199729,199729 +199730,199730 +199731,199731 +199732,199732 +59975,199732 +199733,199733 +199821,199821 +199822,199822 +199823,199823 +199824,199824 +199825,199825 +113157,199825 +199826,199826 +199827,199827 +199828,199828 +199829,199829 +199830,199830 +199831,199831 +199832,199832 +199833,199833 +199834,199834 +23894,199834 +199835,199835 +34244,199835 +199836,199836 +199837,199837 +199838,199838 +199839,199839 +199840,199840 +199841,199841 +199842,199842 +199843,199843 +199844,199844 +199845,199845 +199846,199846 +199847,199847 +205111,199847 +75583,199847 +199848,199848 +199849,199849 +199850,199850 +199851,199851 +199852,199852 +199853,199853 +199854,199854 +21814,199854 +199855,199855 +199856,199856 +199857,199857 +199858,199858 +199859,199859 +197317,199859 +199860,199860 +199861,199861 +199862,199862 +199863,199863 +185181,199863 +199864,199864 +199865,199865 +199866,199866 +199867,199867 +199868,199868 +199869,199869 +199870,199870 +199871,199871 +199872,199872 +199873,199873 +199874,199874 +199875,199875 +199876,199876 +199877,199877 +199878,199878 +199879,199879 +158307,199879 +199880,199880 +199881,199881 +199882,199882 +199883,199883 +199884,199884 +200025,200025 +200026,200026 +200337,200337 +200460,200460 +200461,200461 +200462,200462 +200463,200463 +226999,200463 +200464,200464 +86692,200464 +200465,200465 +200466,200466 +200467,200467 +200468,200468 +200469,200469 +200470,200470 +200471,200471 +200472,200472 +200473,200473 +200474,200474 +200475,200475 +200476,200476 +200477,200477 +200478,200478 +200479,200479 +200480,200480 +200481,200481 +200482,200482 +200483,200483 +200484,200484 +200485,200485 +200486,200486 +200487,200487 +200488,200488 +200489,200489 +200490,200490 +200491,200491 +200492,200492 +200493,200493 +81585,200493 +200494,200494 +200495,200495 +200496,200496 +200497,200497 +200498,200498 +200499,200499 +200500,200500 +200501,200501 +200502,200502 +200503,200503 +200504,200504 +200505,200505 +200506,200506 +200507,200507 +200508,200508 +200509,200509 +200510,200510 +200511,200511 +200512,200512 +200513,200513 +200514,200514 +200515,200515 +200516,200516 +200517,200517 +200518,200518 +200519,200519 +200520,200520 +200521,200521 +200522,200522 +200523,200523 +200524,200524 +200525,200525 +200526,200526 +200527,200527 +200528,200528 +200529,200529 +200530,200530 +200531,200531 +200532,200532 +200533,200533 +200534,200534 +200535,200535 +200536,200536 +32369,200536 +200537,200537 +200538,200538 +200539,200539 +200540,200540 +200541,200541 +200542,200542 +200543,200543 +200544,200544 +200545,200545 +200546,200546 +200547,200547 +200548,200548 +200549,200549 +200550,200550 +200551,200551 +11101,200551 +200552,200552 +200553,200553 +200554,200554 +200555,200555 +200556,200556 +200557,200557 +200558,200558 +200559,200559 +200560,200560 +200561,200561 +200562,200562 +200563,200563 +200564,200564 +200565,200565 +200566,200566 +200567,200567 +200568,200568 +200569,200569 +200570,200570 +200571,200571 +200572,200572 +200573,200573 +161015,200573 +200574,200574 +200575,200575 +200576,200576 +200577,200577 +200578,200578 +200579,200579 +200580,200580 +200581,200581 +200582,200582 +200583,200583 +200584,200584 +200585,200585 +200586,200586 +200587,200587 +148169,200587 +200588,200588 +200589,200589 +200590,200590 +200591,200591 +184569,200591 +200592,200592 +220897,200592 +200593,200593 +200594,200594 +200595,200595 +200596,200596 +200597,200597 +200598,200598 +200599,200599 +200600,200600 +200601,200601 +200602,200602 +200603,200603 +200604,200604 +200605,200605 +200606,200606 +200607,200607 +200608,200608 +200609,200609 +200610,200610 +200611,200611 +200612,200612 +200613,200613 +200614,200614 +200615,200615 +200616,200616 +200617,200617 +200618,200618 +200619,200619 +200620,200620 +200621,200621 +200622,200622 +200623,200623 +200624,200624 +200625,200625 +200626,200626 +200627,200627 +200628,200628 +74021,200628 +200629,200629 +200630,200630 +200631,200631 +146917,200631 +200632,200632 +200633,200633 +200634,200634 +200635,200635 +200636,200636 +200637,200637 +149122,200637 +200638,200638 +200729,200729 +200730,200730 +200731,200731 +200732,200732 +200733,200733 +200734,200734 +200735,200735 +200736,200736 +200737,200737 +200738,200738 +200739,200739 +200740,200740 +200741,200741 +200742,200742 +200743,200743 +200744,200744 +53700,200744 +45596,200744 +200745,200745 +200746,200746 +200747,200747 +200748,200748 +200749,200749 +31824,200749 +200750,200750 +200751,200751 +200752,200752 +200753,200753 +82437,200753 +200754,200754 +200755,200755 +200756,200756 +200757,200757 +200758,200758 +200759,200759 +200760,200760 +200761,200761 +200762,200762 +200763,200763 +200764,200764 +200765,200765 +200766,200766 +123073,200766 +200767,200767 +200768,200768 +200769,200769 +200770,200770 +200771,200771 +200772,200772 +200773,200773 +200774,200774 +107551,200774 +200775,200775 +200776,200776 +200777,200777 +200778,200778 +200779,200779 +200780,200780 +200781,200781 +200782,200782 +200783,200783 +200784,200784 +200785,200785 +200786,200786 +200787,200787 +200788,200788 +200789,200789 +200790,200790 +200791,200791 +200792,200792 +200793,200793 +200794,200794 +200795,200795 +200796,200796 +200797,200797 +200798,200798 +200799,200799 +200800,200800 +200801,200801 +200802,200802 +205257,200802 +200803,200803 +200804,200804 +200805,200805 +200806,200806 +213700,200806 +200807,200807 +200808,200808 +242759,200808 +200809,200809 +200810,200810 +200811,200811 +200812,200812 +200856,200856 +200857,200857 +200858,200858 +200859,200859 +200860,200860 +200861,200861 +200862,200862 +200863,200863 +200864,200864 +200865,200865 +200866,200866 +200867,200867 +200868,200868 +4316,200868 +200869,200869 +200870,200870 +200871,200871 +200872,200872 +14111,200872 +200873,200873 +200874,200874 +200875,200875 +200876,200876 +200877,200877 +200878,200878 +73892,200878 +200879,200879 +200880,200880 +200881,200881 +200882,200882 +200883,200883 +200884,200884 +200885,200885 +200886,200886 +200887,200887 +200888,200888 +200889,200889 +200890,200890 +243571,200890 +200891,200891 +200892,200892 +115778,200892 +200893,200893 +200970,200970 +200971,200971 +200972,200972 +200973,200973 +200974,200974 +200975,200975 +200976,200976 +200977,200977 +200978,200978 +200979,200979 +1964,200979 +200980,200980 +200981,200981 +200982,200982 +76543,200982 +200983,200983 +200984,200984 +200985,200985 +200986,200986 +200987,200987 +200988,200988 +200989,200989 +200990,200990 +200991,200991 +200992,200992 +200993,200993 +200994,200994 +200995,200995 +200996,200996 +200997,200997 +200998,200998 +200999,200999 +201000,201000 +149869,201000 +201001,201001 +201002,201002 +201003,201003 +201004,201004 +201005,201005 +201006,201006 +201007,201007 +201008,201008 +201009,201009 +201010,201010 +201011,201011 +201012,201012 +201013,201013 +201014,201014 +201015,201015 +64106,201015 +201016,201016 +201017,201017 +6718,201017 +201018,201018 +201019,201019 +119778,201019 +201020,201020 +201021,201021 +201022,201022 +201023,201023 +201024,201024 +201025,201025 +201026,201026 +201027,201027 +201028,201028 +201029,201029 +201030,201030 +161805,201030 +201763,201030 +201031,201031 +201032,201032 +218526,201032 +201033,201033 +201034,201034 +201035,201035 +201036,201036 +164670,201036 +201037,201037 +201038,201038 +201039,201039 +201040,201040 +201041,201041 +201042,201042 +201043,201043 +201044,201044 +201045,201045 +201046,201046 +201047,201047 +201048,201048 +201049,201049 +201050,201050 +201051,201051 +201052,201052 +201053,201053 +201054,201054 +201055,201055 +201056,201056 +201057,201057 +201058,201058 +201059,201059 +201060,201060 +201061,201061 +201062,201062 +201063,201063 +201064,201064 +5282,201064 +201065,201065 +201066,201066 +201067,201067 +201068,201068 +194856,201068 +201069,201069 +201070,201070 +201071,201071 +201072,201072 +201073,201073 +201074,201074 +68005,201074 +201075,201075 +1109,201075 +201076,201076 +201077,201077 +201078,201078 +201079,201079 +108306,201079 +201080,201080 +201081,201081 +201082,201082 +91488,201082 +201083,201083 +193544,201083 +201084,201084 +201085,201085 +201086,201086 +201087,201087 +91485,201087 +201088,201088 +201089,201089 +201090,201090 +201091,201091 +201092,201092 +201093,201093 +201094,201094 +201095,201095 +201096,201096 +201097,201097 +201098,201098 +201099,201099 +201100,201100 +201101,201101 +201102,201102 +201103,201103 +34280,201103 +201104,201104 +201105,201105 +201106,201106 +201107,201107 +201108,201108 +53729,201108 +45625,201108 +201109,201109 +201110,201110 +201111,201111 +201112,201112 +53733,201112 +45629,201112 +201113,201113 +201114,201114 +201115,201115 +201116,201116 +58421,201116 +193157,201116 +201117,201117 +201118,201118 +201119,201119 +201120,201120 +201121,201121 +201122,201122 +201123,201123 +201124,201124 +9062,201124 +77452,201124 +207461,201124 +201125,201125 +201126,201126 +201127,201127 +201128,201128 +201129,201129 +201130,201130 +201131,201131 +201132,201132 +142925,201132 +201133,201133 +32967,201133 +201134,201134 +201135,201135 +201136,201136 +201137,201137 +201138,201138 +201139,201139 +201140,201140 +201141,201141 +201142,201142 +201143,201143 +184760,201143 +201144,201144 +201145,201145 +201146,201146 +201147,201147 +85689,201147 +201148,201148 +201149,201149 +201150,201150 +201151,201151 +201152,201152 +201153,201153 +201154,201154 +164940,201154 +201155,201155 +201156,201156 +201157,201157 +201158,201158 +89230,201158 +201159,201159 +201160,201160 +201161,201161 +201162,201162 +201163,201163 +143059,201163 +201164,201164 +201165,201165 +201166,201166 +201167,201167 +201168,201168 +201169,201169 +80332,201169 +201170,201170 +201171,201171 +201172,201172 +201173,201173 +201174,201174 +201175,201175 +201176,201176 +201177,201177 +201178,201178 +201179,201179 +201180,201180 +201181,201181 +201182,201182 +21349,201182 +201183,201183 +201184,201184 +201185,201185 +201186,201186 +201187,201187 +201188,201188 +201189,201189 +201190,201190 +201191,201191 +201192,201192 +201193,201193 +201194,201194 +201330,201330 +201331,201331 +201332,201332 +201333,201333 +201334,201334 +201335,201335 +120259,201335 +201336,201336 +201337,201337 +201338,201338 +201339,201339 +2413,201339 +201340,201340 +2675,201340 +201341,201341 +201342,201342 +201343,201343 +201344,201344 +201345,201345 +201346,201346 +201347,201347 +201348,201348 +201349,201349 +201350,201350 +201389,201389 +201390,201390 +59421,201390 +201391,201391 +201392,201392 +201393,201393 +201394,201394 +54744,201394 +46640,201394 +201395,201395 +201396,201396 +201397,201397 +201398,201398 +201399,201399 +201400,201400 +201401,201401 +201402,201402 +103511,201402 +201403,201403 +201404,201404 +201405,201405 +244055,201406 +201406,201406 +183886,201406 +201407,201407 +201408,201408 +201409,201409 +201410,201410 +201411,201411 +201412,201412 +180591,201412 +201413,201413 +201414,201414 +201415,201415 +201416,201416 +201417,201417 +201418,201418 +201419,201419 +201420,201420 +201421,201421 +201422,201422 +201423,201423 +201424,201424 +201425,201425 +201426,201426 +201427,201427 +201428,201428 +201429,201429 +201430,201430 +201431,201431 +201432,201432 +201433,201433 +201434,201434 +201435,201435 +201436,201436 +59535,201436 +201437,201437 +191592,201437 +201438,201438 +201439,201439 +201440,201440 +201441,201441 +21206,201441 +201442,201442 +201443,201443 +201444,201444 +201445,201445 +201446,201446 +201447,201447 +201448,201448 +201449,201449 +201450,201450 +211083,201450 +201451,201451 +201452,201452 +201453,201453 +201454,201454 +201455,201455 +201456,201456 +201457,201457 +201458,201458 +201459,201459 +201460,201460 +201461,201461 +201462,201462 +201463,201463 +201464,201464 +201465,201465 +201466,201466 +201467,201467 +201468,201468 +214698,201468 +132787,201468 +125200,201468 +201469,201469 +201470,201470 +19230,201470 +201471,201471 +201472,201472 +211404,201472 +201473,201473 +201474,201474 +201475,201475 +201476,201476 +201477,201477 +201478,201478 +201479,201479 +201480,201480 +201481,201481 +201482,201482 +201483,201483 +201484,201484 +201485,201485 +23454,201485 +201486,201486 +201487,201487 +201488,201488 +201489,201489 +201490,201490 +201491,201491 +201492,201492 +201493,201493 +201494,201494 +201495,201495 +239677,201495 +201496,201496 +201497,201497 +201498,201498 +201499,201499 +201500,201500 +201501,201501 +201502,201502 +201503,201503 +201504,201504 +201505,201505 +201506,201506 +201507,201507 +201508,201508 +201509,201509 +205037,201509 +201510,201510 +201511,201511 +201512,201512 +201513,201513 +90974,201513 +201514,201514 +201515,201515 +201516,201516 +201517,201517 +201518,201518 +201519,201519 +201520,201520 +201521,201521 +201522,201522 +201523,201523 +201524,201524 +201525,201525 +201526,201526 +206886,201526 +201527,201527 +201528,201528 +201529,201529 +201530,201530 +201531,201531 +201532,201532 +201533,201533 +201534,201534 +201535,201535 +201536,201536 +201537,201537 +201538,201538 +201539,201539 +201540,201540 +201541,201541 +201542,201542 +201543,201543 +201544,201544 +249607,201544 +201545,201545 +201546,201546 +201547,201547 +201548,201548 +201549,201549 +201550,201550 +201551,201551 +201552,201552 +154982,201552 +201553,201553 +201554,201554 +201555,201555 +201556,201556 +201557,201557 +201558,201558 +201559,201559 +138949,201559 +201560,201560 +201561,201561 +201562,201562 +201563,201563 +201564,201564 +201565,201565 +201566,201566 +201567,201567 +201568,201568 +201569,201569 +201570,201570 +201571,201571 +201572,201572 +201573,201573 +201574,201574 +201575,201575 +201576,201576 +201577,201577 +201578,201578 +201616,201616 +201617,201617 +201618,201618 +201619,201619 +201620,201620 +191532,201620 +201621,201621 +201622,201622 +201623,201623 +201624,201624 +201625,201625 +201626,201626 +201627,201627 +201628,201628 +201629,201629 +201630,201630 +201631,201631 +201670,201670 +201671,201671 +201672,201672 +201673,201673 +201674,201674 +201675,201675 +201676,201676 +188697,201676 +201677,201677 +3071,201677 +201678,201678 +201679,201679 +201680,201680 +201681,201681 +201682,201682 +201683,201683 +201684,201684 +201685,201685 +201686,201686 +201687,201687 +201688,201688 +201689,201689 +201690,201690 +201691,201691 +201692,201692 +201693,201693 +11979,201693 +201694,201694 +201695,201695 +201696,201696 +59450,201696 +201697,201697 +163059,201697 +201698,201698 +201699,201699 +201700,201700 +201701,201701 +201702,201702 +201703,201703 +201704,201704 +201705,201705 +201706,201706 +201707,201707 +201708,201708 +201709,201709 +201756,201756 +201757,201757 +201758,201758 +2823,201758 +201759,201759 +201760,201760 +201761,201761 +201762,201762 +201763,201763 +201030,201763 +161805,201763 +201764,201764 +201765,201765 +201766,201766 +201767,201767 +201768,201768 +201769,201769 +201770,201770 +67545,201770 +147322,201770 +201771,201771 +201772,201772 +101767,201772 +201773,201773 +201774,201774 +201775,201775 +201776,201776 +203776,201776 +201777,201777 +201778,201778 +201779,201779 +201780,201780 +122816,201780 +201781,201781 +114735,201781 +201782,201782 +201783,201783 +201784,201784 +201785,201785 +57496,201785 +201786,201786 +202025,202025 +202026,202026 +202027,202027 +202028,202028 +202029,202029 +202030,202030 +202031,202031 +202032,202032 +211765,202032 +202033,202033 +110846,202033 +202034,202034 +202035,202035 +202036,202036 +202037,202037 +202038,202038 +202039,202039 +202040,202040 +202041,202041 +202042,202042 +202043,202043 +202044,202044 +202045,202045 +202046,202046 +202047,202047 +202048,202048 +202049,202049 +202050,202050 +202051,202051 +202052,202052 +3051,202052 +202053,202053 +202054,202054 +202055,202055 +202056,202056 +202057,202057 +202058,202058 +202059,202059 +202060,202060 +202061,202061 +202062,202062 +202063,202063 +202064,202064 +202065,202065 +202066,202066 +79217,202066 +202067,202067 +202068,202068 +202069,202069 +202070,202070 +202071,202071 +202072,202072 +202073,202073 +202074,202074 +202075,202075 +202076,202076 +154802,202076 +202077,202077 +202078,202078 +202079,202079 +202080,202080 +202081,202081 +202082,202082 +202083,202083 +202084,202084 +202085,202085 +202086,202086 +202087,202087 +202088,202088 +202089,202089 +63858,202089 +202090,202090 +202091,202091 +202092,202092 +202093,202093 +202094,202094 +89840,202094 +202095,202095 +202096,202096 +202097,202097 +202098,202098 +202099,202099 +202100,202100 +202101,202101 +202102,202102 +202103,202103 +202104,202104 +202105,202105 +202106,202106 +202107,202107 +202108,202108 +202109,202109 +202110,202110 +202111,202111 +202112,202112 +202113,202113 +202114,202114 +202115,202115 +202116,202116 +202117,202117 +202118,202118 +21679,202118 +202119,202119 +202120,202120 +202121,202121 +202122,202122 +202123,202123 +202124,202124 +202125,202125 +202126,202126 +202127,202127 +202128,202128 +202129,202129 +202296,202296 +202297,202297 +202298,202298 +202299,202299 +202300,202300 +202301,202301 +202302,202302 +224771,202302 +202303,202303 +202304,202304 +202305,202305 +202306,202306 +177630,202306 +171666,202306 +202307,202307 +202308,202308 +202309,202309 +202310,202310 +202311,202311 +202312,202312 +202313,202313 +202314,202314 +202315,202315 +202316,202316 +202317,202317 +202318,202318 +202319,202319 +202320,202320 +202321,202321 +202322,202322 +202761,202322 +202323,202323 +202324,202324 +202325,202325 +202326,202326 +202327,202327 +202328,202328 +202329,202329 +202330,202330 +202373,202373 +202374,202374 +202375,202375 +202376,202376 +197387,202376 +243328,202376 +202377,202377 +202378,202378 +202379,202379 +202380,202380 +4296,202380 +202381,202381 +202382,202382 +202383,202383 +202384,202384 +202385,202385 +202386,202386 +202387,202387 +196563,202387 +202388,202388 +202389,202389 +202390,202390 +202391,202391 +202392,202392 +202393,202393 +202394,202394 +109526,202394 +202395,202395 +202396,202396 +202397,202397 +202398,202398 +202399,202399 +202400,202400 +202401,202401 +202402,202402 +202403,202403 +202404,202404 +202405,202405 +202406,202406 +202407,202407 +202408,202408 +202409,202409 +202410,202410 +202411,202411 +202412,202412 +99289,202412 +202413,202413 +202414,202414 +202415,202415 +202416,202416 +202417,202417 +202418,202418 +202419,202419 +202420,202420 +202421,202421 +202422,202422 +202423,202423 +202424,202424 +202425,202425 +202426,202426 +202427,202427 +202428,202428 +202429,202429 +202430,202430 +202431,202431 +202432,202432 +202433,202433 +202434,202434 +202435,202435 +196919,202435 +202436,202436 +202437,202437 +202438,202438 +202439,202439 +202440,202440 +202441,202441 +202442,202442 +202443,202443 +202444,202444 +202445,202445 +202446,202446 +202447,202447 +202448,202448 +202449,202449 +77094,202449 +202450,202450 +202487,202487 +202488,202488 +202489,202489 +202490,202490 +202491,202491 +23464,202491 +202492,202492 +202493,202493 +202494,202494 +202495,202495 +202496,202496 +209395,202496 +202497,202497 +202498,202498 +106820,202498 +202499,202499 +202500,202500 +202501,202501 +58424,202501 +202502,202502 +202503,202503 +202504,202504 +202505,202505 +202506,202506 +202507,202507 +202508,202508 +233764,202508 +202509,202509 +202510,202510 +202511,202511 +202512,202512 +202513,202513 +202514,202514 +202515,202515 +202516,202516 +202517,202517 +202518,202518 +202519,202519 +202520,202520 +202521,202521 +202522,202522 +202523,202523 +207058,202523 +202524,202524 +202525,202525 +202526,202526 +202527,202527 +202528,202528 +190557,202528 +202529,202529 +202530,202530 +202531,202531 +202532,202532 +202533,202533 +202534,202534 +202535,202535 +202536,202536 +202537,202537 +202538,202538 +202539,202539 +202540,202540 +202541,202541 +202542,202542 +202543,202543 +202544,202544 +202545,202545 +202546,202546 +202547,202547 +202548,202548 +202549,202549 +202550,202550 +202551,202551 +202552,202552 +211428,202552 +202553,202553 +202554,202554 +202555,202555 +202556,202556 +202557,202557 +202558,202558 +202559,202559 +202560,202560 +202561,202561 +145525,202561 +202562,202562 +202563,202563 +202564,202564 +202565,202565 +202566,202566 +202567,202567 +202568,202568 +202569,202569 +202570,202570 +202571,202571 +202572,202572 +202573,202573 +202574,202574 +202575,202575 +202576,202576 +202577,202577 +202578,202578 +202579,202579 +148440,202579 +202580,202580 +202581,202581 +202582,202582 +231454,202582 +202583,202583 +226328,202583 +202584,202584 +202585,202585 +202586,202586 +202587,202587 +202588,202588 +180818,202588 +202589,202589 +202590,202590 +202591,202591 +202592,202592 +202593,202593 +202594,202594 +202595,202595 +202596,202596 +202597,202597 +202598,202598 +202599,202599 +202600,202600 +202601,202601 +202602,202602 +202603,202603 +202604,202604 +202605,202605 +202606,202606 +202607,202607 +202608,202608 +202609,202609 +202610,202610 +66906,202610 +202611,202611 +202612,202612 +202613,202613 +216428,202613 +202614,202614 +202615,202615 +202616,202616 +202617,202617 +202618,202618 +202619,202619 +223057,202619 +202620,202620 +202621,202621 +202622,202622 +202623,202623 +202624,202624 +202625,202625 +147341,202625 +202626,202626 +202627,202627 +202628,202628 +202629,202629 +202630,202630 +202631,202631 +202632,202632 +190878,202632 +202633,202633 +202634,202634 +202635,202635 +202636,202636 +202637,202637 +202638,202638 +202639,202639 +202640,202640 +202641,202641 +202642,202642 +202643,202643 +202644,202644 +202645,202645 +202646,202646 +202647,202647 +202648,202648 +202649,202649 +202650,202650 +202651,202651 +202652,202652 +202653,202653 +202654,202654 +202655,202655 +202656,202656 +97650,202656 +202657,202657 +202658,202658 +202659,202659 +202660,202660 +90162,202660 +202661,202661 +202662,202662 +202663,202663 +202664,202664 +202665,202665 +202666,202666 +202667,202667 +202668,202668 +202669,202669 +202670,202670 +202671,202671 +202672,202672 +202673,202673 +202674,202674 +202675,202675 +202676,202676 +202677,202677 +202678,202678 +202679,202679 +202680,202680 +202681,202681 +202682,202682 +202683,202683 +202684,202684 +202685,202685 +202686,202686 +202687,202687 +202688,202688 +202689,202689 +202690,202690 +18544,202690 +202691,202691 +202692,202692 +208495,202692 +202693,202693 +202694,202694 +202695,202695 +202696,202696 +202697,202697 +202698,202698 +202699,202699 +202700,202700 +202701,202701 +202702,202702 +202703,202703 +134086,202703 +126499,202703 +202704,202704 +202705,202705 +202706,202706 +202707,202707 +202708,202708 +52732,202708 +44628,202708 +202709,202709 +202710,202710 +202711,202711 +202712,202712 +162789,202712 +202713,202713 +202714,202714 +202715,202715 +202716,202716 +202717,202717 +202718,202718 +202719,202719 +202720,202720 +202721,202721 +202722,202722 +202723,202723 +10552,202723 +202724,202724 +202725,202725 +202726,202726 +202727,202727 +202728,202728 +202729,202729 +202730,202730 +202731,202731 +202732,202732 +202733,202733 +158683,202733 +202734,202734 +202735,202735 +202736,202736 +36713,202736 +202737,202737 +202738,202738 +202739,202739 +202740,202740 +202741,202741 +202742,202742 +202743,202743 +202744,202744 +113334,202744 +202745,202745 +202746,202746 +202747,202747 +202748,202748 +202749,202749 +202750,202750 +202751,202751 +202752,202752 +202753,202753 +202754,202754 +202755,202755 +202756,202756 +202757,202757 +202758,202758 +202759,202759 +202760,202760 +202761,202761 +202322,202761 +202762,202762 +202763,202763 +202764,202764 +202765,202765 +202766,202766 +202767,202767 +9225,202767 +202768,202768 +202769,202769 +202770,202770 +202771,202771 +202772,202772 +202773,202773 +202774,202774 +202775,202775 +202776,202776 +202777,202777 +142878,202777 +202778,202778 +202779,202779 +202780,202780 +202781,202781 +202782,202782 +202783,202783 +202784,202784 +8604,202784 +8424,202784 +134159,202784 +126572,202784 +202785,202785 +202786,202786 +202787,202787 +202788,202788 +202789,202789 +202790,202790 +202791,202791 +202792,202792 +202793,202793 +202794,202794 +202795,202795 +202796,202796 +50346,202796 +42154,202796 +202797,202797 +202798,202798 +202799,202799 +202800,202800 +202801,202801 +202802,202802 +216743,202802 +202803,202803 +93996,202803 +229584,202803 +202804,202804 +202805,202805 +202806,202806 +110087,202806 +202807,202807 +202808,202808 +202809,202809 +202810,202810 +202811,202811 +202812,202812 +202813,202813 +202814,202814 +202815,202815 +202816,202816 +202817,202817 +202818,202818 +202819,202819 +100645,202819 +202820,202820 +202821,202821 +202822,202822 +89824,202822 +202823,202823 +202824,202824 +202825,202825 +202826,202826 +202827,202827 +202828,202828 +202829,202829 +202830,202830 +202831,202831 +202832,202832 +202833,202833 +202834,202834 +202835,202835 +202836,202836 +202837,202837 +202838,202838 +202839,202839 +202840,202840 +202841,202841 +202842,202842 +202843,202843 +202844,202844 +154546,202844 +202845,202845 +202846,202846 +202847,202847 +202848,202848 +202849,202849 +202850,202850 +202851,202851 +202852,202852 +202853,202853 +202854,202854 +202855,202855 +202856,202856 +202857,202857 +202858,202858 +202859,202859 +202860,202860 +202861,202861 +202862,202862 +202863,202863 +202864,202864 +199599,202864 +202865,202865 +202866,202866 +202867,202867 +202868,202868 +202869,202869 +202870,202870 +202871,202871 +202872,202872 +202873,202873 +202874,202874 +202875,202875 +35724,202875 +202876,202876 +202877,202877 +202878,202878 +202879,202879 +202880,202880 +224413,202880 +202881,202881 +202882,202882 +202883,202883 +202884,202884 +216226,202884 +202885,202885 +202886,202886 +240422,202886 +202887,202887 +202888,202888 +202889,202889 +202890,202890 +2164,202890 +202891,202891 +202892,202892 +655,202892 +202893,202893 +202894,202894 +202895,202895 +202896,202896 +202897,202897 +202898,202898 +202899,202899 +202900,202900 +202901,202901 +202902,202902 +202903,202903 +202904,202904 +202905,202905 +202906,202906 +202907,202907 +202908,202908 +202946,202946 +202947,202947 +202997,202997 +202998,202998 +202999,202999 +203000,203000 +203001,203001 +203002,203002 +203003,203003 +136473,203003 +128886,203003 +247146,203004 +203004,203004 +203005,203005 +110099,203005 +203006,203006 +247148,203006 +203007,203007 +203008,203008 +203009,203009 +203010,203010 +203011,203011 +203012,203012 +203013,203013 +203014,203014 +203015,203015 +203016,203016 +203017,203017 +203018,203018 +203019,203019 +203305,203019 +203020,203020 +203021,203021 +28944,203021 +139770,203021 +203022,203022 +227653,203022 +203023,203023 +203024,203024 +59650,203024 +203025,203025 +203026,203026 +203027,203027 +203028,203028 +203029,203029 +203305,203305 +203019,203305 +203306,203306 +203307,203307 +203308,203308 +203309,203309 +203434,203434 +203435,203435 +203436,203436 +203437,203437 +155203,203437 +203438,203438 +203439,203439 +203440,203440 +203441,203441 +203442,203442 +203443,203443 +203444,203444 +203445,203445 +203446,203446 +203447,203447 +203448,203448 +203449,203449 +203450,203450 +203451,203451 +203452,203452 +203453,203453 +203454,203454 +203455,203455 +203456,203456 +203457,203457 +203458,203458 +203459,203459 +203460,203460 +203461,203461 +203462,203462 +203463,203463 +203464,203464 +203465,203465 +203466,203466 +203467,203467 +203468,203468 +203469,203469 +203470,203470 +203471,203471 +203472,203472 +203473,203473 +203474,203474 +226631,203474 +203475,203475 +203476,203476 +203477,203477 +203478,203478 +203479,203479 +203480,203480 +203481,203481 +203482,203482 +203483,203483 +203484,203484 +203485,203485 +203486,203486 +203487,203487 +203488,203488 +203489,203489 +203490,203490 +203491,203491 +203492,203492 +203493,203493 +203494,203494 +203495,203495 +203496,203496 +203497,203497 +203498,203498 +203499,203499 +203500,203500 +203501,203501 +203502,203502 +203503,203503 +203504,203504 +203505,203505 +203506,203506 +203507,203507 +203508,203508 +203509,203509 +203510,203510 +203511,203511 +203512,203512 +203513,203513 +203514,203514 +203515,203515 +203516,203516 +203517,203517 +203518,203518 +203519,203519 +114391,203519 +203520,203520 +203521,203521 +203522,203522 +203523,203523 +203524,203524 +203525,203525 +203526,203526 +160884,203526 +203527,203527 +203528,203528 +203529,203529 +203530,203530 +203531,203531 +111971,203531 +62471,203531 +203532,203532 +203533,203533 +203534,203534 +203535,203535 +203536,203536 +203537,203537 +203538,203538 +203539,203539 +203540,203540 +203541,203541 +203542,203542 +203543,203543 +203544,203544 +203545,203545 +203546,203546 +203547,203547 +100680,203547 +241775,203547 +203548,203548 +203549,203549 +203550,203550 +203551,203551 +203552,203552 +203553,203553 +203554,203554 +203555,203555 +203556,203556 +203557,203557 +210332,203557 +203558,203558 +203559,203559 +203560,203560 +203561,203561 +203562,203562 +203563,203563 +203564,203564 +203565,203565 +203566,203566 +203567,203567 +203568,203568 +203569,203569 +203570,203570 +203571,203571 +229583,203571 +203572,203572 +203573,203573 +203574,203574 +203575,203575 +203576,203576 +203577,203577 +203578,203578 +203579,203579 +203580,203580 +203581,203581 +212267,203581 +203582,203582 +203583,203583 +203584,203584 +203585,203585 +203586,203586 +203587,203587 +203588,203588 +203589,203589 +203590,203590 +203591,203591 +203592,203592 +203593,203593 +84366,203593 +203594,203594 +203595,203595 +177520,203595 +171556,203595 +203596,203596 +203597,203597 +203598,203598 +203599,203599 +203600,203600 +203601,203601 +203602,203602 +203603,203603 +203604,203604 +203605,203605 +203606,203606 +203607,203607 +203608,203608 +203609,203609 +203610,203610 +203611,203611 +203612,203612 +231827,203612 +203613,203613 +203614,203614 +203615,203615 +203616,203616 +203617,203617 +203618,203618 +203619,203619 +203620,203620 +135815,203620 +128228,203620 +203621,203621 +203622,203622 +203623,203623 +203624,203624 +203625,203625 +203626,203626 +203627,203627 +203628,203628 +203629,203629 +203630,203630 +203631,203631 +203632,203632 +203671,203671 +203672,203672 +203673,203673 +203674,203674 +203675,203675 +162332,203675 +203676,203676 +51851,203676 +43747,203676 +203677,203677 +203678,203678 +76035,203678 +143257,203678 +203679,203679 +203680,203680 +203681,203681 +203682,203682 +203683,203683 +203684,203684 +135460,203684 +127873,203684 +203685,203685 +203686,203686 +203687,203687 +203688,203688 +203689,203689 +203690,203690 +203691,203691 +203692,203692 +203693,203693 +203694,203694 +231289,203694 +203695,203695 +203696,203696 +203697,203697 +203698,203698 +203699,203699 +203700,203700 +203701,203701 +139883,203701 +203702,203702 +203703,203703 +203704,203704 +203705,203705 +203706,203706 +203707,203707 +203708,203708 +203709,203709 +203710,203710 +203711,203711 +203712,203712 +203713,203713 +203714,203714 +203715,203715 +203716,203716 +203717,203717 +203718,203718 +203719,203719 +203720,203720 +203721,203721 +203722,203722 +203723,203723 +203724,203724 +203725,203725 +203726,203726 +203727,203727 +203728,203728 +203729,203729 +203730,203730 +203731,203731 +203732,203732 +203733,203733 +203734,203734 +203735,203735 +203736,203736 +203737,203737 +203738,203738 +180258,203738 +203739,203739 +105678,203739 +203740,203740 +203741,203741 +78617,203741 +203742,203742 +203743,203743 +203744,203744 +190153,203744 +203745,203745 +203746,203746 +203747,203747 +203748,203748 +203749,203749 +203750,203750 +203751,203751 +203752,203752 +203753,203753 +203754,203754 +203755,203755 +203756,203756 +203757,203757 +203758,203758 +203759,203759 +203760,203760 +203761,203761 +203762,203762 +203763,203763 +198019,203763 +203764,203764 +203765,203765 +203766,203766 +203767,203767 +203768,203768 +203769,203769 +203770,203770 +203771,203771 +203772,203772 +203773,203773 +203774,203774 +203775,203775 +203776,203776 +201776,203776 +203777,203777 +203778,203778 +203779,203779 +203780,203780 +203781,203781 +203782,203782 +203783,203783 +203784,203784 +203785,203785 +203786,203786 +203787,203787 +85817,203787 +203788,203788 +203789,203789 +203790,203790 +203791,203791 +203792,203792 +203793,203793 +203794,203794 +203795,203795 +203796,203796 +203797,203797 +203798,203798 +203799,203799 +203800,203800 +203801,203801 +203802,203802 +203803,203803 +203804,203804 +193424,203804 +203805,203805 +203806,203806 +134285,203806 +126698,203806 +203807,203807 +203808,203808 +203809,203809 +203810,203810 +203811,203811 +203812,203812 +203813,203813 +203814,203814 +203815,203815 +203816,203816 +203817,203817 +203818,203818 +203819,203819 +203820,203820 +24022,203820 +203821,203821 +203822,203822 +203823,203823 +203824,203824 +203825,203825 +203826,203826 +203827,203827 +7689,203827 +203828,203828 +203829,203829 +203830,203830 +203831,203831 +203832,203832 +203833,203833 +203834,203834 +203835,203835 +203836,203836 +203837,203837 +145271,203837 +111861,203837 +203838,203838 +203839,203839 +203840,203840 +203841,203841 +203842,203842 +203843,203843 +203844,203844 +80353,203844 +203845,203845 +203846,203846 +203847,203847 +203848,203848 +203849,203849 +203850,203850 +203851,203851 +203852,203852 +203853,203853 +203854,203854 +203855,203855 +203856,203856 +203857,203857 +40246,203857 +203858,203858 +203859,203859 +203860,203860 +235859,203860 +203861,203861 +203862,203862 +203863,203863 +203864,203864 +203865,203865 +203866,203866 +227774,203866 +203867,203867 +203868,203868 +203869,203869 +203870,203870 +203871,203871 +203872,203872 +203873,203873 +203874,203874 +203875,203875 +203876,203876 +81543,203876 +203877,203877 +209995,203877 +203878,203878 +203879,203879 +203880,203880 +203881,203881 +203882,203882 +203883,203883 +86033,203883 +203884,203884 +203885,203885 +203886,203886 +203887,203887 +203888,203888 +203889,203889 +203890,203890 +203891,203891 +203892,203892 +203893,203893 +203894,203894 +203895,203895 +203896,203896 +203897,203897 +203898,203898 +203899,203899 +203900,203900 +203901,203901 +203902,203902 +203903,203903 +203904,203904 +203905,203905 +203906,203906 +203907,203907 +203908,203908 +203909,203909 +203910,203910 +203911,203911 +203912,203912 +203913,203913 +203914,203914 +203915,203915 +203916,203916 +203917,203917 +203918,203918 +203919,203919 +203920,203920 +203921,203921 +203922,203922 +203923,203923 +203924,203924 +203925,203925 +203926,203926 +203927,203927 +203928,203928 +203929,203929 +203930,203930 +203931,203931 +203932,203932 +203933,203933 +137755,203933 +130168,203933 +203934,203934 +203935,203935 +203936,203936 +203937,203937 +203938,203938 +203939,203939 +203940,203940 +80216,203940 +203941,203941 +203942,203942 +80700,203942 +203943,203943 +203944,203944 +203945,203945 +203946,203946 +203947,203947 +203948,203948 +203949,203949 +203950,203950 +203951,203951 +203952,203952 +203953,203953 +203954,203954 +203955,203955 +203956,203956 +203957,203957 +203958,203958 +203959,203959 +203960,203960 +203961,203961 +203962,203962 +223101,203962 +203963,203963 +203964,203964 +203965,203965 +107594,203965 +203966,203966 +203967,203967 +203968,203968 +203969,203969 +203970,203970 +203971,203971 +203972,203972 +203973,203973 +203974,203974 +203975,203975 +203976,203976 +203977,203977 +203978,203978 +147349,203978 +203979,203979 +203980,203980 +203981,203981 +203982,203982 +203983,203983 +203984,203984 +203985,203985 +203986,203986 +203987,203987 +203988,203988 +179498,203988 +203989,203989 +203990,203990 +203991,203991 +203992,203992 +203993,203993 +203994,203994 +203995,203995 +203996,203996 +203997,203997 +203998,203998 +203999,203999 +204000,204000 +204001,204001 +204002,204002 +204003,204003 +20550,204003 +204004,204004 +204005,204005 +204006,204006 +204007,204007 +204008,204008 +204009,204009 +204010,204010 +204011,204011 +178509,204011 +172545,204011 +192252,204011 +204012,204012 +193259,204012 +204013,204013 +204014,204014 +204015,204015 +204016,204016 +204017,204017 +204018,204018 +204019,204019 +204020,204020 +204021,204021 +204022,204022 +204023,204023 +204024,204024 +204025,204025 +204026,204026 +204027,204027 +204028,204028 +204029,204029 +204030,204030 +204031,204031 +204032,204032 +204033,204033 +204034,204034 +204035,204035 +204036,204036 +204037,204037 +204038,204038 +204039,204039 +204040,204040 +204041,204041 +204042,204042 +204043,204043 +204044,204044 +204045,204045 +204046,204046 +204047,204047 +164945,204047 +204048,204048 +204049,204049 +204050,204050 +96424,204050 +204051,204051 +204052,204052 +204053,204053 +204054,204054 +204055,204055 +204056,204056 +204057,204057 +204058,204058 +204059,204059 +204060,204060 +204061,204061 +204062,204062 +204063,204063 +204064,204064 +204065,204065 +204066,204066 +204067,204067 +204068,204068 +204069,204069 +204070,204070 +204071,204071 +204072,204072 +204073,204073 +204074,204074 +204075,204075 +204076,204076 +204077,204077 +204078,204078 +204079,204079 +204080,204080 +204081,204081 +204082,204082 +204122,204122 +204123,204123 +204124,204124 +204125,204125 +204126,204126 +204127,204127 +204128,204128 +204129,204129 +204130,204130 +204131,204131 +204132,204132 +204133,204133 +204134,204134 +204135,204135 +204136,204136 +204137,204137 +204138,204138 +204139,204139 +204140,204140 +204141,204141 +204142,204142 +204143,204143 +204144,204144 +204145,204145 +204146,204146 +204147,204147 +204148,204148 +204149,204149 +204150,204150 +204151,204151 +204152,204152 +204153,204153 +204154,204154 +204155,204155 +204156,204156 +204157,204157 +204158,204158 +204159,204159 +204160,204160 +204161,204161 +204162,204162 +204163,204163 +204164,204164 +204165,204165 +204166,204166 +204167,204167 +204168,204168 +204169,204169 +204170,204170 +204171,204171 +204172,204172 +204173,204173 +204174,204174 +204175,204175 +204176,204176 +204177,204177 +204178,204178 +204179,204179 +204180,204180 +204181,204181 +204182,204182 +204183,204183 +204184,204184 +204185,204185 +134243,204185 +126656,204185 +204186,204186 +204187,204187 +204188,204188 +204189,204189 +204190,204190 +204191,204191 +204192,204192 +204193,204193 +204194,204194 +204195,204195 +204196,204196 +36171,204196 +204197,204197 +204198,204198 +204250,204250 +204251,204251 +204252,204252 +204253,204253 +204254,204254 +204255,204255 +204256,204256 +204257,204257 +204258,204258 +178840,204258 +204259,204259 +204260,204260 +204261,204261 +204262,204262 +204263,204263 +204264,204264 +204265,204265 +148822,204265 +204266,204266 +204267,204267 +204268,204268 +204269,204269 +204270,204270 +204271,204271 +204272,204272 +204273,204273 +204274,204274 +204275,204275 +204276,204276 +204277,204277 +204278,204278 +15844,204278 +204279,204279 +204280,204280 +204281,204281 +204282,204282 +204283,204283 +204284,204284 +204285,204285 +204286,204286 +204287,204287 +204288,204288 +204351,204351 +204352,204352 +204353,204353 +187096,204353 +204354,204354 +204452,204354 +204453,204355 +204355,204355 +204356,204356 +204357,204357 +204455,204357 +204358,204358 +204359,204359 +204360,204360 +204361,204361 +204362,204362 +65036,204362 +204363,204363 +204364,204364 +204365,204365 +204366,204366 +204367,204367 +204368,204368 +224389,204368 +204369,204369 +204370,204370 +204371,204371 +204372,204372 +204415,204415 +233677,204415 +204416,204416 +204417,204417 +204418,204418 +204419,204419 +204420,204420 +204421,204421 +204422,204422 +204423,204423 +204424,204424 +204425,204425 +204426,204426 +204427,204427 +204428,204428 +204429,204429 +204430,204430 +204431,204431 +204432,204432 +204433,204433 +204434,204434 +204435,204435 +204436,204436 +204437,204437 +204438,204438 +204439,204439 +204440,204440 +204441,204441 +204442,204442 +204443,204443 +204444,204444 +204445,204445 +204446,204446 +89181,204446 +204447,204447 +58142,204447 +204448,204448 +204449,204449 +204450,204450 +204451,204451 +204452,204452 +204354,204452 +151484,204452 +204453,204453 +204355,204453 +204454,204454 +204455,204455 +204357,204455 +204456,204456 +204457,204457 +204458,204458 +204459,204459 +204460,204460 +204461,204461 +204462,204462 +204463,204463 +204464,204464 +204465,204465 +204466,204466 +204467,204467 +204468,204468 +204469,204469 +204470,204470 +204471,204471 +140482,204471 +204472,204472 +204473,204473 +204474,204474 +204475,204475 +204476,204476 +204477,204477 +204478,204478 +95168,204478 +204479,204479 +204480,204480 +20668,204480 +20496,204480 +204481,204481 +204482,204482 +204483,204483 +204484,204484 +204485,204485 +204486,204486 +204487,204487 +204488,204488 +204489,204489 +204490,204490 +204491,204491 +244317,204492 +204492,204492 +204493,204493 +204494,204494 +204495,204495 +204496,204496 +204497,204497 +208069,204497 +204498,204498 +204499,204499 +204500,204500 +204501,204501 +204502,204502 +204503,204503 +204504,204504 +204505,204505 +204506,204506 +204507,204507 +204508,204508 +204509,204509 +204510,204510 +204511,204511 +204512,204512 +204513,204513 +204514,204514 +204515,204515 +204516,204516 +204517,204517 +204518,204518 +204519,204519 +204520,204520 +204521,204521 +204522,204522 +204523,204523 +204524,204524 +204525,204525 +204526,204526 +204527,204527 +204528,204528 +204529,204529 +204530,204530 +204531,204531 +204532,204532 +204533,204533 +204534,204534 +204535,204535 +204536,204536 +204537,204537 +204538,204538 +204539,204539 +204540,204540 +204541,204541 +67173,204541 +204542,204542 +204543,204543 +204544,204544 +204545,204545 +204546,204546 +204547,204547 +204548,204548 +204549,204549 +204550,204550 +204551,204551 +204552,204552 +204553,204553 +204554,204554 +204555,204555 +204556,204556 +176660,204556 +170696,204556 +204557,204557 +204558,204558 +204559,204559 +204560,204560 +204561,204561 +204562,204562 +204563,204563 +204564,204564 +204565,204565 +204566,204566 +204567,204567 +189869,204567 +204568,204568 +204569,204569 +204570,204570 +204571,204571 +204572,204572 +204573,204573 +204574,204574 +204575,204575 +204576,204576 +204577,204577 +204578,204578 +204579,204579 +204580,204580 +204581,204581 +204582,204582 +204583,204583 +204584,204584 +204585,204585 +204586,204586 +204587,204587 +204588,204588 +204589,204589 +204590,204590 +204591,204591 +204592,204592 +204593,204593 +204594,204594 +204595,204595 +204596,204596 +204597,204597 +204598,204598 +204599,204599 +204600,204600 +204601,204601 +63970,204601 +204602,204602 +204603,204603 +204604,204604 +204605,204605 +204606,204606 +204607,204607 +204608,204608 +204609,204609 +204610,204610 +204611,204611 +204612,204612 +204613,204613 +204614,204614 +163965,204614 +204615,204615 +204616,204616 +204617,204617 +204618,204618 +204619,204619 +204620,204620 +204621,204621 +204622,204622 +112312,204622 +204623,204623 +204624,204624 +59941,204624 +204625,204625 +204626,204626 +204627,204627 +204628,204628 +204629,204629 +204630,204630 +204631,204631 +204632,204632 +204633,204633 +204634,204634 +204635,204635 +204636,204636 +204637,204637 +204638,204638 +204639,204639 +204640,204640 +204641,204641 +204642,204642 +204643,204643 +204644,204644 +204645,204645 +204646,204646 +204647,204647 +195122,204647 +204648,204648 +204649,204649 +204650,204650 +204651,204651 +204652,204652 +204653,204653 +204654,204654 +204655,204655 +204656,204656 +204657,204657 +204658,204658 +204659,204659 +204660,204660 +204661,204661 +204662,204662 +204663,204663 +62392,204663 +204664,204664 +204665,204665 +204666,204666 +204667,204667 +204668,204668 +204669,204669 +204670,204670 +204707,204707 +204708,204708 +204709,204709 +204710,204710 +234443,204710 +204711,204711 +204712,204712 +204713,204713 +204714,204714 +204715,204715 +204716,204716 +204717,204717 +204718,204718 +204719,204719 +204720,204720 +204721,204721 +204722,204722 +204723,204723 +204724,204724 +204725,204725 +204726,204726 +204727,204727 +204728,204728 +204729,204729 +204730,204730 +204731,204731 +204732,204732 +204733,204733 +204734,204734 +204735,204735 +204736,204736 +204737,204737 +204738,204738 +204739,204739 +204740,204740 +204741,204741 +204742,204742 +204743,204743 +204744,204744 +204745,204745 +204746,204746 +204747,204747 +79061,204747 +204748,204748 +204749,204749 +205282,204749 +204750,204750 +204751,204751 +151121,204751 +204752,204752 +204753,204753 +204754,204754 +204755,204755 +107560,204755 +67221,204755 +204756,204756 +204757,204757 +204758,204758 +204759,204759 +204760,204760 +204761,204761 +204762,204762 +204763,204763 +204764,204764 +204765,204765 +204766,204766 +204767,204767 +204768,204768 +204769,204769 +204770,204770 +204771,204771 +204772,204772 +204773,204773 +204774,204774 +204775,204775 +204776,204776 +204777,204777 +204778,204778 +204779,204779 +204780,204780 +204781,204781 +204782,204782 +204783,204783 +184767,204783 +204784,204784 +204785,204785 +204786,204786 +204830,204830 +204831,204831 +204832,204832 +204833,204833 +204834,204834 +204835,204835 +204836,204836 +204837,204837 +204838,204838 +204839,204839 +204840,204840 +204841,204841 +204842,204842 +204843,204843 +204844,204844 +204845,204845 +204846,204846 +204847,204847 +204848,204848 +204849,204849 +204850,204850 +204851,204851 +204852,204852 +248775,204852 +204853,204853 +204854,204854 +204855,204855 +204856,204856 +204857,204857 +204858,204858 +204859,204859 +204860,204860 +204861,204861 +204862,204862 +204863,204863 +204864,204864 +204865,204865 +204866,204866 +204867,204867 +204868,204868 +204869,204869 +204870,204870 +204871,204871 +204872,204872 +204873,204873 +59893,204873 +204874,204874 +204875,204875 +204876,204876 +204877,204877 +204878,204878 +84675,204878 +204879,204879 +204880,204880 +204881,204881 +204882,204882 +204883,204883 +160855,204883 +204884,204884 +204885,204885 +204886,204886 +204887,204887 +204888,204888 +204889,204889 +204890,204890 +204891,204891 +204892,204892 +204893,204893 +204894,204894 +204895,204895 +204896,204896 +204897,204897 +204898,204898 +204899,204899 +204900,204900 +204901,204901 +204902,204902 +204903,204903 +204904,204904 +120438,204904 +204905,204905 +204906,204906 +204907,204907 +204908,204908 +204909,204909 +204910,204910 +204911,204911 +52547,204911 +44443,204911 +204912,204912 +204913,204913 +204914,204914 +204915,204915 +204916,204916 +239521,204916 +204917,204917 +204918,204918 +204919,204919 +204920,204920 +204921,204921 +204922,204922 +204923,204923 +204924,204924 +204925,204925 +204926,204926 +204927,204927 +204928,204928 +204929,204929 +204930,204930 +204931,204931 +204932,204932 +204933,204933 +204934,204934 +204935,204935 +204936,204936 +204937,204937 +204938,204938 +205019,205019 +205020,205020 +159488,205020 +205021,205021 +205022,205022 +205023,205023 +205024,205024 +205025,205025 +205026,205026 +205027,205027 +205028,205028 +205029,205029 +205030,205030 +205031,205031 +72566,205031 +205032,205032 +205033,205033 +205034,205034 +205035,205035 +205036,205036 +205037,205037 +201509,205037 +205038,205038 +205039,205039 +205040,205040 +205041,205041 +205042,205042 +205043,205043 +205044,205044 +205045,205045 +205046,205046 +205047,205047 +205048,205048 +205049,205049 +205050,205050 +205051,205051 +205052,205052 +205053,205053 +80420,205053 +205054,205054 +205055,205055 +205056,205056 +205057,205057 +205058,205058 +205059,205059 +205060,205060 +205061,205061 +205062,205062 +205063,205063 +205064,205064 +205065,205065 +205066,205066 +205067,205067 +205068,205068 +205069,205069 +205070,205070 +205071,205071 +205072,205072 +80115,205072 +205073,205073 +205074,205074 +205075,205075 +205076,205076 +205077,205077 +205078,205078 +205079,205079 +205080,205080 +205081,205081 +205082,205082 +205083,205083 +196503,205083 +248908,205083 +205084,205084 +205085,205085 +158795,205085 +205086,205086 +205087,205087 +205088,205088 +205089,205089 +205090,205090 +131937,205090 +124350,205090 +205091,205091 +205092,205092 +205093,205093 +205094,205094 +39939,205094 +205095,205095 +205096,205096 +205097,205097 +205098,205098 +9475,205098 +205099,205099 +205100,205100 +205101,205101 +156839,205101 +205102,205102 +205103,205103 +205104,205104 +205105,205105 +205106,205106 +205107,205107 +205108,205108 +205109,205109 +205110,205110 +205111,205111 +199847,205111 +205112,205112 +205113,205113 +146969,205113 +205114,205114 +153968,205114 +205115,205115 +205116,205116 +205117,205117 +205118,205118 +205119,205119 +205120,205120 +111904,205120 +234299,205120 +205121,205121 +205122,205122 +205123,205123 +205124,205124 +205125,205125 +205126,205126 +205127,205127 +205128,205128 +205129,205129 +205130,205130 +205131,205131 +205132,205132 +205133,205133 +205134,205134 +205135,205135 +205136,205136 +205137,205137 +205138,205138 +205139,205139 +205140,205140 +205141,205141 +205142,205142 +205143,205143 +205144,205144 +205145,205145 +205146,205146 +205147,205147 +205148,205148 +205149,205149 +205150,205150 +205151,205151 +205152,205152 +205153,205153 +205154,205154 +205155,205155 +205156,205156 +205157,205157 +205158,205158 +205159,205159 +205160,205160 +205161,205161 +205162,205162 +205163,205163 +205164,205164 +205165,205165 +205166,205166 +205167,205167 +205168,205168 +205169,205169 +165329,205169 +205170,205170 +205171,205171 +205172,205172 +205173,205173 +205174,205174 +205175,205175 +205176,205176 +205177,205177 +205178,205178 +205179,205179 +205180,205180 +205181,205181 +205182,205182 +205183,205183 +205184,205184 +205185,205185 +205186,205186 +205187,205187 +205188,205188 +99357,205188 +205189,205189 +205190,205190 +205191,205191 +205192,205192 +205193,205193 +243216,205193 +205194,205194 +205195,205195 +108980,205195 +205196,205196 +205197,205197 +205198,205198 +205199,205199 +205200,205200 +205201,205201 +205202,205202 +205203,205203 +205204,205204 +205205,205205 +205206,205206 +205207,205207 +205208,205208 +205209,205209 +205210,205210 +205211,205211 +205212,205212 +205213,205213 +205214,205214 +205215,205215 +205216,205216 +205217,205217 +205218,205218 +205219,205219 +205220,205220 +109283,205220 +205221,205221 +205222,205222 +205223,205223 +205224,205224 +205225,205225 +205226,205226 +205227,205227 +205228,205228 +205229,205229 +205230,205230 +205231,205231 +205232,205232 +205233,205233 +205234,205234 +205235,205235 +205236,205236 +205237,205237 +205238,205238 +205239,205239 +50762,205239 +42658,205239 +205240,205240 +205241,205241 +205242,205242 +76013,205242 +205243,205243 +205244,205244 +205245,205245 +205246,205246 +11831,205246 +205247,205247 +205248,205248 +234593,205248 +205249,205249 +205250,205250 +205251,205251 +205252,205252 +205253,205253 +218311,205253 +205254,205254 +205255,205255 +224941,205256 +205256,205256 +205257,205257 +200802,205257 +205258,205258 +205259,205259 +205260,205260 +205261,205261 +205262,205262 +205263,205263 +117012,205263 +205264,205264 +205265,205265 +205266,205266 +205267,205267 +205268,205268 +205269,205269 +205270,205270 +205271,205271 +205272,205272 +205273,205273 +205274,205274 +205275,205275 +3281,205275 +205276,205276 +205277,205277 +205278,205278 +151340,205278 +205279,205279 +205280,205280 +205281,205281 +205282,205282 +204749,205282 +205283,205283 +205284,205284 +205285,205285 +205286,205286 +205287,205287 +205288,205288 +205289,205289 +205290,205290 +205291,205291 +218485,205292 +205292,205292 +205293,205293 +205294,205294 +205295,205295 +148324,205295 +205296,205296 +205297,205297 +205298,205298 +89213,205298 +205299,205299 +205300,205300 +205301,205301 +205302,205302 +205303,205303 +205304,205304 +205305,205305 +205306,205306 +205307,205307 +205308,205308 +205309,205309 +205310,205310 +9684,205310 +205311,205311 +205312,205312 +205313,205313 +205314,205314 +205315,205315 +205316,205316 +205317,205317 +205318,205318 +205319,205319 +205320,205320 +205321,205321 +205322,205322 +205323,205323 +205324,205324 +205325,205325 +205326,205326 +205327,205327 +205328,205328 +205329,205329 +205330,205330 +205331,205331 +205332,205332 +205333,205333 +205334,205334 +205335,205335 +205336,205336 +205337,205337 +205338,205338 +205339,205339 +205340,205340 +205341,205341 +205379,205379 +80285,205379 +205380,205380 +205381,205381 +205382,205382 +205383,205383 +205384,205384 +205385,205385 +205386,205386 +205387,205387 +205388,205388 +205389,205389 +205428,205428 +205429,205429 +8862,205429 +205430,205430 +205431,205431 +205432,205432 +205433,205433 +205434,205434 +205435,205435 +205436,205436 +205437,205437 +89550,205437 +205438,205438 +205439,205439 +205440,205440 +205441,205441 +205442,205442 +205443,205443 +205444,205444 +205445,205445 +205446,205446 +205447,205447 +205448,205448 +205449,205449 +205450,205450 +205451,205451 +205452,205452 +205453,205453 +205454,205454 +176428,205454 +170464,205454 +205455,205455 +205456,205456 +120260,205456 +205457,205457 +205458,205458 +205459,205459 +205460,205460 +205461,205461 +205462,205462 +205463,205463 +205464,205464 +205465,205465 +33361,205465 +205466,205466 +205467,205467 +205468,205468 +205469,205469 +205470,205470 +205471,205471 +205472,205472 +205473,205473 +205474,205474 +205475,205475 +205476,205476 +205477,205477 +205478,205478 +205479,205479 +205480,205480 +205481,205481 +205482,205482 +205483,205483 +205484,205484 +205485,205485 +205486,205486 +205487,205487 +205488,205488 +205489,205489 +205490,205490 +205491,205491 +205492,205492 +205493,205493 +205494,205494 +205495,205495 +205496,205496 +205497,205497 +205498,205498 +205499,205499 +205500,205500 +205501,205501 +205502,205502 +205503,205503 +205504,205504 +205505,205505 +205506,205506 +235817,205506 +205507,205507 +205508,205508 +205509,205509 +205510,205510 +205511,205511 +205512,205512 +205513,205513 +205514,205514 +205515,205515 +205516,205516 +205558,205558 +205559,205559 +205560,205560 +205561,205561 +205562,205562 +205563,205563 +205564,205564 +205565,205565 +205566,205566 +205567,205567 +205568,205568 +162238,205568 +205569,205569 +205570,205570 +205571,205571 +60142,205571 +205572,205572 +205573,205573 +205574,205574 +205575,205575 +205576,205576 +205577,205577 +205578,205578 +205579,205579 +37184,205579 +205580,205580 +205581,205581 +205582,205582 +205583,205583 +205584,205584 +191785,205584 +205585,205585 +205586,205586 +146959,205586 +205587,205587 +205588,205588 +205589,205589 +205590,205590 +205591,205591 +205592,205592 +205593,205593 +205594,205594 +205595,205595 +205596,205596 +236141,205596 +205597,205597 +205598,205598 +205599,205599 +205600,205600 +205601,205601 +205602,205602 +205603,205603 +205604,205604 +205605,205605 +205606,205606 +205607,205607 +205608,205608 +205609,205609 +80411,205609 +205610,205610 +205611,205611 +205612,205612 +205613,205613 +205614,205614 +244707,205614 +205615,205615 +205616,205616 +205617,205617 +205618,205618 +205619,205619 +205620,205620 +205621,205621 +236955,205621 +205622,205622 +154206,205622 +205623,205623 +205624,205624 +205625,205625 +205626,205626 +205627,205627 +205628,205628 +205629,205629 +205630,205630 +205631,205631 +205632,205632 +205633,205633 +205634,205634 +205635,205635 +213579,205635 +205676,205676 +205677,205677 +205678,205678 +205679,205679 +205680,205680 +205681,205681 +205682,205682 +205683,205683 +205684,205684 +205685,205685 +206058,205685 +205686,205686 +205687,205687 +205688,205688 +205689,205689 +205690,205690 +206063,205690 +205691,205691 +206065,205692 +205692,205692 +205693,205693 +205694,205694 +206067,205694 +206068,205695 +205695,205695 +205696,205696 +205697,205697 +206071,205698 +205698,205698 +205699,205699 +206072,205699 +205700,205700 +205701,205701 +205702,205702 +205703,205703 +205704,205704 +205705,205705 +206078,205705 +206079,205706 +205706,205706 +205707,205707 +233941,205707 +206080,205707 +206081,205708 +205708,205708 +206082,205709 +205709,205709 +222843,205709 +205710,205710 +206083,205710 +205711,205711 +206084,205711 +231204,205711 +205808,205808 +205809,205809 +59654,205809 +205810,205810 +205811,205811 +205812,205812 +205813,205813 +205814,205814 +205815,205815 +205816,205816 +205817,205817 +205818,205818 +205819,205819 +197337,205819 +205820,205820 +205821,205821 +205822,205822 +205823,205823 +205824,205824 +205825,205825 +205826,205826 +40238,205826 +205827,205827 +205828,205828 +205829,205829 +205830,205830 +205831,205831 +205832,205832 +205833,205833 +205834,205834 +205835,205835 +205836,205836 +205837,205837 +205838,205838 +192376,205838 +205839,205839 +205840,205840 +205841,205841 +205842,205842 +205843,205843 +205882,205882 +205883,205883 +205884,205884 +205885,205885 +205886,205886 +205887,205887 +205888,205888 +205889,205889 +205890,205890 +205891,205891 +224541,205891 +205892,205892 +205893,205893 +205894,205894 +205895,205895 +205896,205896 +205897,205897 +205898,205898 +205899,205899 +205900,205900 +108485,205900 +205901,205901 +205902,205902 +205903,205903 +205904,205904 +205905,205905 +205906,205906 +205907,205907 +205908,205908 +205909,205909 +205910,205910 +205911,205911 +205912,205912 +205913,205913 +205914,205914 +205915,205915 +205916,205916 +205917,205917 +205918,205918 +205919,205919 +205920,205920 +205921,205921 +205922,205922 +205923,205923 +205924,205924 +162875,205924 +205925,205925 +205926,205926 +205927,205927 +205928,205928 +205929,205929 +205930,205930 +149840,205930 +205931,205931 +205932,205932 +205933,205933 +205934,205934 +205935,205935 +205936,205936 +205937,205937 +209795,205937 +205938,205938 +205939,205939 +205940,205940 +205941,205941 +205942,205942 +205943,205943 +205944,205944 +205945,205945 +205946,205946 +205947,205947 +205948,205948 +205949,205949 +214449,205949 +205950,205950 +205951,205951 +205952,205952 +205953,205953 +205954,205954 +78729,205954 +205955,205955 +205956,205956 +78731,205956 +205957,205957 +205958,205958 +244754,205958 +205959,205959 +205960,205960 +205961,205961 +205962,205962 +205963,205963 +205964,205964 +205965,205965 +205966,205966 +205967,205967 +205968,205968 +205969,205969 +178370,205969 +172406,205969 +205970,205970 +205971,205971 +205972,205972 +205973,205973 +205974,205974 +183892,205974 +205975,205975 +205976,205976 +205977,205977 +205978,205978 +205979,205979 +205980,205980 +205981,205981 +205982,205982 +205983,205983 +205984,205984 +205985,205985 +205986,205986 +205987,205987 +205988,205988 +205989,205989 +205990,205990 +205991,205991 +6204,205991 +205992,205992 +205993,205993 +205994,205994 +205995,205995 +225041,205995 +205996,205996 +205997,205997 +205998,205998 +205999,205999 +206000,206000 +206001,206001 +51723,206001 +43619,206001 +206002,206002 +206003,206003 +206004,206004 +206005,206005 +206052,206052 +206053,206053 +206054,206054 +35121,206054 +206055,206055 +206056,206056 +206057,206057 +206058,206058 +205685,206058 +206059,206059 +150467,206059 +206060,206060 +206061,206061 +206062,206062 +206063,206063 +205690,206063 +206064,206064 +206065,206065 +205692,206065 +206066,206066 +206067,206067 +205694,206067 +206068,206068 +205695,206068 +206069,206069 +206070,206070 +206071,206071 +205698,206071 +206072,206072 +10457,206072 +205699,206072 +206073,206073 +206074,206074 +206075,206075 +206076,206076 +206077,206077 +206078,206078 +205705,206078 +66349,206078 +206079,206079 +205706,206079 +206080,206080 +205707,206080 +206081,206081 +205708,206081 +206082,206082 +205709,206082 +222843,206082 +206083,206083 +205710,206083 +206084,206084 +205711,206084 +206085,206085 +206086,206086 +206087,206087 +206088,206088 +206089,206089 +206090,206090 +206091,206091 +206092,206092 +229804,206092 +206093,206093 +206094,206094 +206095,206095 +206096,206096 +206097,206097 +206098,206098 +206099,206099 +206100,206100 +206101,206101 +206102,206102 +206103,206103 +206104,206104 +206105,206105 +211400,206105 +206106,206106 +206107,206107 +206108,206108 +206109,206109 +206110,206110 +206111,206111 +206112,206112 +206113,206113 +62713,206113 +206114,206114 +206115,206115 +206116,206116 +102233,206116 +206117,206117 +206118,206118 +206119,206119 +206120,206120 +206121,206121 +206122,206122 +206123,206123 +206124,206124 +206125,206125 +206126,206126 +206127,206127 +206128,206128 +206129,206129 +206130,206130 +206195,206195 +206196,206196 +92600,206196 +236822,206196 +206197,206197 +92601,206197 +236823,206197 +206198,206198 +34047,206198 +206199,206199 +206200,206200 +206201,206201 +206202,206202 +206203,206203 +206204,206204 +206205,206205 +206206,206206 +206207,206207 +206208,206208 +206209,206209 +206210,206210 +206211,206211 +206212,206212 +206213,206213 +206214,206214 +206215,206215 +206216,206216 +206217,206217 +206218,206218 +206219,206219 +206220,206220 +247948,206220 +206262,206262 +206263,206263 +206264,206264 +206265,206265 +206266,206266 +206267,206267 +206268,206268 +206269,206269 +206270,206270 +206271,206271 +206272,206272 +206273,206273 +65764,206273 +206274,206274 +206275,206275 +206276,206276 +206277,206277 +206278,206278 +206279,206279 +206280,206280 +141820,206280 +206281,206281 +206282,206282 +234694,206282 +160974,206282 +206283,206283 +206284,206284 +206285,206285 +206286,206286 +206287,206287 +206288,206288 +206289,206289 +206290,206290 +206291,206291 +206292,206292 +206293,206293 +206294,206294 +183830,206294 +111397,206294 +206295,206295 +206296,206296 +206297,206297 +206298,206298 +67273,206298 +178321,206298 +172357,206298 +206299,206299 +206300,206300 +206301,206301 +206302,206302 +206303,206303 +206304,206304 +206305,206305 +206383,206383 +206384,206384 +206385,206385 +206386,206386 +206387,206387 +206388,206388 +206389,206389 +64906,206389 +206390,206390 +206391,206391 +206392,206392 +206393,206393 +206394,206394 +206395,206395 +206396,206396 +206397,206397 +206398,206398 +206399,206399 +206400,206400 +206401,206401 +206402,206402 +206403,206403 +206404,206404 +206405,206405 +206406,206406 +206407,206407 +206408,206408 +206409,206409 +206410,206410 +206411,206411 +206412,206412 +206413,206413 +206414,206414 +206415,206415 +206416,206416 +206417,206417 +206418,206418 +206419,206419 +206420,206420 +206421,206421 +206422,206422 +206423,206423 +206424,206424 +206425,206425 +206426,206426 +206427,206427 +206428,206428 +206429,206429 +206430,206430 +206431,206431 +206432,206432 +206433,206433 +206434,206434 +206435,206435 +206436,206436 +206437,206437 +206438,206438 +206439,206439 +206440,206440 +179816,206440 +206441,206441 +206442,206442 +206443,206443 +206444,206444 +206445,206445 +206446,206446 +206447,206447 +206448,206448 +206449,206449 +206450,206450 +206451,206451 +206452,206452 +206453,206453 +206454,206454 +206455,206455 +206456,206456 +206457,206457 +206458,206458 +206459,206459 +206460,206460 +206461,206461 +206462,206462 +206463,206463 +206464,206464 +149391,206464 +206465,206465 +206466,206466 +206467,206467 +206468,206468 +206469,206469 +206470,206470 +206471,206471 +206472,206472 +206473,206473 +206474,206474 +206475,206475 +206476,206476 +206477,206477 +206478,206478 +206479,206479 +206480,206480 +206481,206481 +206482,206482 +206483,206483 +206522,206522 +206523,206523 +206524,206524 +206525,206525 +206526,206526 +206527,206527 +206528,206528 +206529,206529 +206530,206530 +206531,206531 +245471,206531 +206532,206532 +206533,206533 +206534,206534 +206535,206535 +206536,206536 +206537,206537 +206538,206538 +9462,206538 +206539,206539 +206540,206540 +206541,206541 +206542,206542 +206543,206543 +206544,206544 +206545,206545 +206546,206546 +206547,206547 +206548,206548 +206549,206549 +206550,206550 +206551,206551 +206552,206552 +206553,206553 +55763,206553 +47659,206553 +206554,206554 +206555,206555 +206556,206556 +206557,206557 +206558,206558 +206559,206559 +206560,206560 +100935,206560 +206561,206561 +206562,206562 +206563,206563 +194166,206563 +206564,206564 +206565,206565 +206566,206566 +206567,206567 +206568,206568 +206569,206569 +206570,206570 +206571,206571 +206572,206572 +206573,206573 +206574,206574 +206575,206575 +206576,206576 +206577,206577 +2207,206577 +206578,206578 +178187,206578 +172223,206578 +206579,206579 +206580,206580 +206581,206581 +206582,206582 +206583,206583 +206584,206584 +53368,206584 +45264,206584 +206585,206585 +206586,206586 +206587,206587 +206588,206588 +206589,206589 +206590,206590 +84194,206590 +206591,206591 +206592,206592 +206593,206593 +206594,206594 +206595,206595 +206596,206596 +98119,206596 +206597,206597 +206598,206598 +206599,206599 +206600,206600 +206601,206601 +206602,206602 +6591,206602 +206603,206603 +206604,206604 +206605,206605 +206606,206606 +206607,206607 +206608,206608 +206609,206609 +206610,206610 +206611,206611 +206612,206612 +206613,206613 +206614,206614 +206615,206615 +206616,206616 +206617,206617 +206618,206618 +206619,206619 +206620,206620 +206621,206621 +206622,206622 +206623,206623 +206624,206624 +206625,206625 +206626,206626 +206627,206627 +206628,206628 +206629,206629 +206630,206630 +206631,206631 +206632,206632 +206633,206633 +206634,206634 +206635,206635 +206636,206636 +9338,206636 +206637,206637 +183242,206637 +206638,206638 +206639,206639 +206640,206640 +142639,206640 +206641,206641 +206642,206642 +206643,206643 +206644,206644 +206645,206645 +206646,206646 +206647,206647 +206648,206648 +206649,206649 +173916,206649 +167952,206649 +206650,206650 +206651,206651 +206652,206652 +206653,206653 +206654,206654 +206655,206655 +206656,206656 +206657,206657 +206658,206658 +206659,206659 +206660,206660 +206661,206661 +206662,206662 +206663,206663 +206664,206664 +137938,206664 +130351,206664 +206665,206665 +206666,206666 +206667,206667 +206668,206668 +206669,206669 +206670,206670 +206671,206671 +206672,206672 +206673,206673 +111280,206673 +95003,206673 +206674,206674 +206675,206675 +206676,206676 +206677,206677 +206678,206678 +116015,206678 +206679,206679 +206680,206680 +206681,206681 +206682,206682 +206683,206683 +206684,206684 +206685,206685 +206686,206686 +206687,206687 +206688,206688 +206689,206689 +206690,206690 +206691,206691 +206692,206692 +206693,206693 +206694,206694 +206695,206695 +206696,206696 +206697,206697 +206698,206698 +206699,206699 +206700,206700 +206701,206701 +206702,206702 +207308,206702 +206703,206703 +206704,206704 +206705,206705 +206706,206706 +206707,206707 +206708,206708 +206709,206709 +206710,206710 +206711,206711 +206712,206712 +206713,206713 +121439,206713 +206714,206714 +84051,206714 +206715,206715 +206716,206716 +206717,206717 +206718,206718 +206719,206719 +206720,206720 +206721,206721 +206722,206722 +206723,206723 +83905,206723 +206724,206724 +165218,206724 +206725,206725 +206726,206726 +206727,206727 +206728,206728 +158053,206728 +206729,206729 +206730,206730 +206731,206731 +206732,206732 +206733,206733 +83526,206733 +206734,206734 +206735,206735 +206736,206736 +206737,206737 +145071,206737 +206738,206738 +206739,206739 +206740,206740 +81396,206740 +206741,206741 +206742,206742 +206743,206743 +206744,206744 +206745,206745 +206746,206746 +206747,206747 +206748,206748 +206749,206749 +206750,206750 +206751,206751 +206752,206752 +187384,206752 +206753,206753 +9774,206753 +206754,206754 +206755,206755 +206756,206756 +206757,206757 +206758,206758 +206759,206759 +206760,206760 +206761,206761 +206762,206762 +206763,206763 +206764,206764 +206765,206765 +206766,206766 +206767,206767 +206768,206768 +206769,206769 +206770,206770 +206771,206771 +206772,206772 +206773,206773 +206774,206774 +206775,206775 +206776,206776 +206777,206777 +206778,206778 +206779,206779 +206780,206780 +206781,206781 +206782,206782 +206783,206783 +206784,206784 +206785,206785 +206786,206786 +206787,206787 +206788,206788 +206789,206789 +206790,206790 +206791,206791 +206792,206792 +206793,206793 +206794,206794 +206795,206795 +206796,206796 +206797,206797 +206798,206798 +206799,206799 +206800,206800 +206801,206801 +206802,206802 +206803,206803 +218402,206803 +206804,206804 +206805,206805 +206806,206806 +206807,206807 +206808,206808 +206809,206809 +206810,206810 +206811,206811 +206812,206812 +206813,206813 +206814,206814 +206815,206815 +206816,206816 +197339,206816 +243571,206816 +206817,206817 +206818,206818 +206819,206819 +206820,206820 +206821,206821 +206822,206822 +206823,206823 +206824,206824 +213613,206824 +206825,206825 +206826,206826 +206827,206827 +206828,206828 +206829,206829 +206830,206830 +206831,206831 +206832,206832 +206833,206833 +206834,206834 +206835,206835 +206836,206836 +206837,206837 +206838,206838 +206839,206839 +206840,206840 +206841,206841 +206842,206842 +206843,206843 +206844,206844 +206845,206845 +206846,206846 +206847,206847 +206848,206848 +206849,206849 +206850,206850 +206851,206851 +206852,206852 +206853,206853 +234674,206853 +206854,206854 +142946,206854 +206855,206855 +15699,206855 +206856,206856 +206857,206857 +206858,206858 +132928,206858 +125341,206858 +206859,206859 +206860,206860 +206861,206861 +206862,206862 +22039,206862 +206863,206863 +206864,206864 +206865,206865 +206866,206866 +206867,206867 +206868,206868 +206869,206869 +206870,206870 +206871,206871 +206872,206872 +206873,206873 +206874,206874 +206875,206875 +206876,206876 +206877,206877 +206878,206878 +206879,206879 +206880,206880 +206881,206881 +206882,206882 +206883,206883 +206884,206884 +206885,206885 +206886,206886 +201526,206886 +206887,206887 +206888,206888 +206889,206889 +206890,206890 +206891,206891 +206892,206892 +206893,206893 +206894,206894 +206895,206895 +206896,206896 +206897,206897 +206898,206898 +206899,206899 +247536,206899 +206900,206900 +144072,206900 +206901,206901 +206902,206902 +206903,206903 +206904,206904 +206905,206905 +206906,206906 +206907,206907 +206908,206908 +206909,206909 +206910,206910 +206911,206911 +206912,206912 +206913,206913 +59782,206913 +206914,206914 +206915,206915 +206916,206916 +206917,206917 +206918,206918 +5786,206918 +206919,206919 +206920,206920 +206921,206921 +206922,206922 +206923,206923 +206924,206924 +206925,206925 +206926,206926 +206927,206927 +206928,206928 +206929,206929 +206930,206930 +206931,206931 +206932,206932 +206933,206933 +206934,206934 +80535,206934 +206935,206935 +206936,206936 +206937,206937 +33187,206937 +206938,206938 +206939,206939 +206940,206940 +206941,206941 +206942,206942 +206943,206943 +59400,206943 +206944,206944 +206945,206945 +154347,206945 +206946,206946 +76397,206946 +206947,206947 +206948,206948 +233831,206948 +206949,206949 +206950,206950 +206951,206951 +206952,206952 +206953,206953 +206954,206954 +206955,206955 +206956,206956 +206957,206957 +207456,206958 +206958,206958 +206959,206959 +206998,206998 +206999,206999 +207000,207000 +207001,207001 +207002,207002 +207003,207003 +207004,207004 +11193,207004 +207005,207005 +207006,207006 +207007,207007 +207008,207008 +207009,207009 +207010,207010 +207011,207011 +207012,207012 +207013,207013 +207014,207014 +207015,207015 +207016,207016 +207017,207017 +207018,207018 +207019,207019 +207020,207020 +207021,207021 +207022,207022 +207023,207023 +207024,207024 +207025,207025 +207026,207026 +207027,207027 +21936,207027 +207028,207028 +150727,207028 +207029,207029 +207030,207030 +207031,207031 +207032,207032 +207033,207033 +207034,207034 +207035,207035 +207036,207036 +207037,207037 +207038,207038 +189711,207038 +207039,207039 +244361,207040 +207040,207040 +207041,207041 +207042,207042 +207043,207043 +207044,207044 +207045,207045 +207046,207046 +207047,207047 +207048,207048 +207049,207049 +207050,207050 +207051,207051 +211689,207051 +207052,207052 +207053,207053 +207054,207054 +207055,207055 +183867,207055 +207056,207056 +207057,207057 +207058,207058 +202523,207058 +207059,207059 +207060,207060 +207061,207061 +207062,207062 +207063,207063 +207064,207064 +207065,207065 +207066,207066 +207067,207067 +207068,207068 +207105,207105 +207106,207106 +207107,207107 +207108,207108 +207109,207109 +207110,207110 +207111,207111 +207112,207112 +89326,207112 +207113,207113 +7338,207113 +207114,207114 +207115,207115 +207116,207116 +207117,207117 +207118,207118 +207119,207119 +207120,207120 +207121,207121 +207122,207122 +207123,207123 +207124,207124 +207125,207125 +207126,207126 +207127,207127 +207128,207128 +207129,207129 +207130,207130 +216270,207130 +207131,207131 +207132,207132 +207133,207133 +207134,207134 +207135,207135 +207136,207136 +207137,207137 +207138,207138 +207139,207139 +207140,207140 +207141,207141 +207142,207142 +207143,207143 +207144,207144 +207145,207145 +207146,207146 +207147,207147 +207148,207148 +207149,207149 +207150,207150 +207151,207151 +207152,207152 +207153,207153 +207154,207154 +207155,207155 +207156,207156 +145738,207156 +207157,207157 +207158,207158 +207159,207159 +207160,207160 +207161,207161 +207162,207162 +109149,207162 +207163,207163 +207164,207164 +249227,207164 +207165,207165 +207166,207166 +207167,207167 +207168,207168 +207169,207169 +207170,207170 +207171,207171 +207172,207172 +207173,207173 +207174,207174 +207175,207175 +207176,207176 +207177,207177 +207178,207178 +207179,207179 +207180,207180 +207181,207181 +207182,207182 +207183,207183 +207184,207184 +207185,207185 +207186,207186 +207187,207187 +207188,207188 +207189,207189 +136251,207189 +128664,207189 +151545,207189 +207190,207190 +207191,207191 +207192,207192 +207193,207193 +207194,207194 +207195,207195 +207196,207196 +207197,207197 +207198,207198 +78683,207198 +207199,207199 +207200,207200 +207201,207201 +207202,207202 +207203,207203 +207204,207204 +207205,207205 +207206,207206 +207207,207207 +207208,207208 +207209,207209 +207210,207210 +207211,207211 +207212,207212 +207213,207213 +207214,207214 +207215,207215 +207216,207216 +207217,207217 +207218,207218 +207219,207219 +21327,207219 +207220,207220 +207221,207221 +207222,207222 +207223,207223 +207224,207224 +207225,207225 +207226,207226 +207227,207227 +107136,207227 +207228,207228 +207229,207229 +207230,207230 +207231,207231 +207232,207232 +207233,207233 +207234,207234 +207235,207235 +207236,207236 +207237,207237 +207238,207238 +207239,207239 +207240,207240 +207241,207241 +207242,207242 +207243,207243 +207244,207244 +207245,207245 +207246,207246 +207247,207247 +79071,207247 +207248,207248 +207249,207249 +14340,207249 +207250,207250 +207251,207251 +207252,207252 +207253,207253 +207254,207254 +207255,207255 +207256,207256 +207257,207257 +207258,207258 +207259,207259 +207260,207260 +207261,207261 +3100,207261 +142235,207261 +145206,207261 +207262,207262 +207263,207263 +207264,207264 +207265,207265 +207266,207266 +207267,207267 +207268,207268 +207269,207269 +207270,207270 +207271,207271 +207272,207272 +207273,207273 +207274,207274 +207275,207275 +207276,207276 +231326,207276 +207277,207277 +207278,207278 +207279,207279 +207280,207280 +207281,207281 +207282,207282 +207283,207283 +207284,207284 +207285,207285 +207286,207286 +207287,207287 +207288,207288 +207289,207289 +207290,207290 +10560,207290 +207291,207291 +207292,207292 +207293,207293 +207294,207294 +207295,207295 +207296,207296 +207297,207297 +207298,207298 +207299,207299 +207300,207300 +207301,207301 +207302,207302 +207303,207303 +207304,207304 +207305,207305 +207306,207306 +207307,207307 +189141,207307 +140422,207307 +207308,207308 +206702,207308 +207309,207309 +207310,207310 +207311,207311 +207312,207312 +207313,207313 +207314,207314 +207315,207315 +207316,207316 +207317,207317 +207318,207318 +207319,207319 +207320,207320 +207321,207321 +207322,207322 +207323,207323 +207324,207324 +207325,207325 +207326,207326 +207327,207327 +207328,207328 +207329,207329 +183761,207329 +207330,207330 +207331,207331 +207332,207332 +207333,207333 +207334,207334 +207335,207335 +207336,207336 +207337,207337 +207338,207338 +142670,207338 +180274,207338 +207339,207339 +207340,207340 +207341,207341 +103704,207341 +207342,207342 +207343,207343 +207344,207344 +207345,207345 +207346,207346 +207347,207347 +207348,207348 +207349,207349 +207350,207350 +207351,207351 +207352,207352 +103959,207352 +207353,207353 +207354,207354 +207355,207355 +207356,207356 +207357,207357 +207358,207358 +207359,207359 +207360,207360 +207361,207361 +207362,207362 +207363,207363 +207364,207364 +207365,207365 +207366,207366 +207367,207367 +207368,207368 +207369,207369 +233948,207369 +207370,207370 +207371,207371 +207372,207372 +207373,207373 +207374,207374 +207375,207375 +207376,207376 +207377,207377 +207378,207378 +207379,207379 +207380,207380 +207381,207381 +207382,207382 +207383,207383 +207384,207384 +207385,207385 +207386,207386 +207387,207387 +207388,207388 +207389,207389 +207390,207390 +207391,207391 +207392,207392 +207393,207393 +207394,207394 +207395,207395 +207396,207396 +207397,207397 +207398,207398 +207399,207399 +207400,207400 +179645,207400 +207401,207401 +207402,207402 +207403,207403 +207404,207404 +207405,207405 +207406,207406 +207407,207407 +207408,207408 +207409,207409 +207410,207410 +207411,207411 +207412,207412 +207413,207413 +207414,207414 +81338,207414 +207415,207415 +207416,207416 +207417,207417 +207418,207418 +207419,207419 +207420,207420 +207421,207421 +207422,207422 +207423,207423 +207424,207424 +83996,207424 +207425,207425 +207426,207426 +207427,207427 +207428,207428 +207429,207429 +207430,207430 +207431,207431 +207432,207432 +207433,207433 +207434,207434 +36074,207434 +187048,207434 +207435,207435 +207436,207436 +207437,207437 +102077,207437 +207438,207438 +207439,207439 +207440,207440 +207441,207441 +207442,207442 +207443,207443 +207444,207444 +207445,207445 +207446,207446 +207447,207447 +207448,207448 +207449,207449 +207450,207450 +207451,207451 +207452,207452 +207453,207453 +207454,207454 +207455,207455 +207456,207456 +206958,207456 +207457,207457 +207458,207458 +81834,207458 +207459,207459 +207460,207460 +207461,207461 +201124,207461 +207462,207462 +207463,207463 +207464,207464 +207465,207465 +226351,207465 +207466,207466 +207467,207467 +207468,207468 +75109,207468 +207469,207469 +207470,207470 +207471,207471 +207472,207472 +207473,207473 +207474,207474 +207475,207475 +207476,207476 +207477,207477 +207478,207478 +207479,207479 +207480,207480 +207481,207481 +207482,207482 +207483,207483 +207484,207484 +207485,207485 +207486,207486 +207487,207487 +207488,207488 +207489,207489 +207490,207490 +207491,207491 +207492,207492 +207493,207493 +207494,207494 +207495,207495 +207496,207496 +207497,207497 +207498,207498 +207499,207499 +207500,207500 +207501,207501 +207502,207502 +207503,207503 +207504,207504 +207505,207505 +207506,207506 +207507,207507 +207508,207508 +207509,207509 +207510,207510 +207511,207511 +207512,207512 +207513,207513 +207514,207514 +207515,207515 +207516,207516 +207517,207517 +207518,207518 +207519,207519 +207520,207520 +207521,207521 +207522,207522 +207523,207523 +207524,207524 +207525,207525 +207526,207526 +207527,207527 +207528,207528 +207529,207529 +207530,207530 +207531,207531 +207532,207532 +207533,207533 +207534,207534 +207535,207535 +207536,207536 +142801,207536 +207537,207537 +207538,207538 +207539,207539 +207540,207540 +207541,207541 +207542,207542 +207543,207543 +207544,207544 +207545,207545 +207546,207546 +207547,207547 +207548,207548 +207549,207549 +207550,207550 +207551,207551 +207552,207552 +207553,207553 +207554,207554 +207555,207555 +207556,207556 +207557,207557 +207558,207558 +191842,207558 +207559,207559 +207560,207560 +207690,207560 +207561,207561 +207562,207562 +207563,207563 +233953,207563 +207564,207564 +207565,207565 +207566,207566 +207567,207567 +207568,207568 +207569,207569 +207933,207569 +207570,207570 +91978,207570 +207571,207571 +207572,207572 +207573,207573 +207574,207574 +207575,207575 +207576,207576 +207577,207577 +207578,207578 +207579,207579 +207580,207580 +207581,207581 +207792,207582 +207582,207582 +207732,207582 +207583,207583 +207883,207583 +191797,207583 +207733,207583 +207584,207584 +207585,207585 +207586,207586 +207796,207586 +207587,207587 +207887,207587 +207588,207588 +216235,207588 +166016,207588 +207848,207588 +207589,207589 +207590,207590 +207591,207591 +207592,207592 +207593,207593 +207594,207594 +207595,207595 +207805,207595 +5822,207595 +207596,207596 +207597,207597 +207598,207598 +207599,207599 +216257,207599 +207600,207600 +207601,207601 +207602,207602 +207603,207603 +191857,207603 +207604,207604 +207874,207604 +227718,207604 +207605,207605 +39194,207605 +207606,207606 +207607,207607 +207608,207608 +207609,207609 +207610,207610 +207611,207611 +207612,207612 +207613,207613 +207883,207613 +191797,207613 +207614,207614 +149448,207614 +207615,207615 +207616,207616 +207617,207617 +207618,207618 +207888,207618 +134047,207618 +126460,207618 +207619,207619 +207620,207620 +207621,207621 +207622,207622 +207623,207623 +211493,207623 +207624,207624 +207625,207625 +207626,207626 +207627,207627 +207628,207628 +154206,207628 +207629,207629 +207630,207630 +207631,207631 +207632,207632 +207633,207633 +207634,207634 +207635,207635 +207636,207636 +207637,207637 +207638,207638 +80141,207638 +207639,207639 +111994,207639 +207640,207640 +207641,207641 +207642,207642 +207643,207643 +207644,207644 +207645,207645 +207646,207646 +207647,207647 +207648,207648 +207649,207649 +207685,207685 +207686,207686 +207687,207687 +207688,207688 +207689,207689 +207690,207690 +207560,207690 +207691,207691 +207692,207692 +207693,207693 +207694,207694 +207824,207694 +207695,207695 +207696,207696 +207697,207697 +207698,207698 +207699,207699 +207700,207700 +207701,207701 +207702,207702 +207703,207703 +207704,207704 +207705,207705 +207706,207706 +207707,207707 +155924,207707 +207708,207708 +207709,207709 +207710,207710 +207860,207710 +207711,207711 +207712,207712 +207862,207712 +189733,207712 +207713,207713 +207714,207714 +207864,207714 +207774,207714 +207715,207715 +207939,207715 +207716,207716 +207717,207717 +207868,207718 +207718,207718 +207932,207718 +207778,207718 +207869,207719 +207719,207719 +207779,207719 +207720,207720 +207721,207721 +191895,207721 +207911,207721 +207871,207721 +207781,207721 +207935,207721 +207722,207722 +207723,207723 +207724,207724 +178571,207724 +207725,207725 +207726,207726 +207727,207727 +207787,207727 +207728,207728 +207879,207729 +207789,207729 +207729,207729 +207730,207730 +207731,207731 +207881,207731 +207791,207731 +207732,207732 +207792,207732 +207582,207732 +207733,207733 +207583,207733 +207769,207769 +207770,207770 +207861,207771 +207771,207771 +155877,207771 +207772,207772 +207862,207772 +189733,207772 +207773,207773 +207863,207773 +207864,207774 +207774,207774 +207714,207774 +207865,207775 +207775,207775 +207939,207775 +207776,207776 +207866,207776 +207896,207776 +207867,207777 +207777,207777 +207778,207778 +207868,207778 +207718,207778 +207779,207779 +207869,207779 +207719,207779 +207780,207780 +9258,207780 +135279,207780 +127692,207780 +11197,207780 +207870,207780 +207871,207781 +207781,207781 +191564,207781 +207721,207781 +191895,207781 +207872,207782 +207782,207782 +207783,207783 +207784,207784 +207785,207785 +207786,207786 +207787,207787 +207727,207787 +207788,207788 +207879,207789 +207789,207789 +207729,207789 +207790,207790 +207881,207791 +207791,207791 +207731,207791 +207792,207792 +207582,207792 +207732,207792 +207793,207793 +207794,207794 +207795,207795 +207796,207796 +207586,207796 +207797,207797 +207798,207798 +207799,207799 +207800,207800 +207801,207801 +207802,207802 +207803,207803 +207804,207804 +207805,207805 +207595,207805 +207806,207806 +207807,207807 +207808,207808 +207809,207809 +207810,207810 +207811,207811 +207812,207812 +207813,207813 +207814,207814 +207815,207815 +107659,207815 +207816,207816 +207817,207817 +207818,207818 +207819,207819 +207933,207819 +207820,207820 +207821,207821 +207822,207822 +207823,207823 +207824,207824 +207694,207824 +207825,207825 +207826,207826 +207827,207827 +207828,207828 +180707,207828 +207829,207829 +207830,207830 +207831,207831 +207832,207832 +207833,207833 +207834,207834 +207835,207835 +207836,207836 +231747,207837 +207837,207837 +207838,207838 +207839,207839 +207840,207840 +207841,207841 +207842,207842 +207843,207843 +207844,207844 +103378,207844 +207845,207845 +207846,207846 +207847,207847 +207848,207848 +134047,207848 +126460,207848 +207888,207848 +207588,207848 +207849,207849 +207850,207850 +132835,207850 +125248,207850 +207851,207851 +207852,207852 +207853,207853 +207854,207854 +207855,207855 +207856,207856 +207857,207857 +207858,207858 +191892,207858 +191842,207858 +207859,207859 +207860,207860 +207710,207860 +207861,207861 +207771,207861 +155877,207861 +207862,207862 +189733,207862 +207712,207862 +207772,207862 +207863,207863 +207773,207863 +207864,207864 +207774,207864 +207714,207864 +207865,207865 +207775,207865 +207939,207865 +207866,207866 +207776,207866 +207867,207867 +207777,207867 +207868,207868 +207718,207868 +207932,207868 +207778,207868 +207869,207869 +207719,207869 +207779,207869 +207870,207870 +207780,207870 +9258,207870 +207871,207871 +207781,207871 +191564,207871 +207721,207871 +191895,207871 +207872,207872 +207782,207872 +207873,207873 +207874,207874 +207604,207874 +207875,207875 +216529,207875 +207876,207876 +227720,207876 +207877,207877 +207878,207878 +207879,207879 +207789,207879 +207729,207879 +207880,207880 +207881,207881 +207791,207881 +207731,207881 +207882,207882 +13311,207882 +207883,207883 +191797,207883 +207583,207883 +207613,207883 +84098,207883 +207884,207884 +152871,207884 +191617,207884 +207885,207885 +207886,207886 +207887,207887 +191416,207887 +207587,207887 +207888,207888 +207618,207888 +207848,207888 +207889,207889 +207890,207890 +183551,207890 +207891,207891 +87449,207891 +207892,207892 +207893,207893 +208606,207893 +207894,207894 +207895,207895 +207896,207896 +207776,207896 +207897,207897 +207898,207898 +207899,207899 +207900,207900 +207901,207901 +207902,207902 +207903,207903 +207904,207904 +207905,207905 +207906,207906 +207907,207907 +207908,207908 +207909,207909 +83890,207909 +58689,207909 +207910,207910 +191664,207910 +207911,207911 +207721,207911 +191895,207911 +191564,207911 +207912,207912 +207913,207913 +20304,207913 +207914,207914 +207915,207915 +207916,207916 +207917,207917 +207918,207918 +207919,207919 +231865,207919 +207920,207920 +207921,207921 +207922,207922 +207923,207923 +207924,207924 +147642,207924 +207925,207925 +207926,207926 +207927,207927 +107523,207927 +207928,207928 +207929,207929 +207930,207930 +207931,207931 +207932,207932 +7334,207932 +207868,207932 +207718,207932 +207933,207933 +207819,207933 +214829,207933 +1067,207933 +207569,207933 +207934,207934 +207935,207935 +207721,207935 +191895,207935 +207936,207936 +207937,207937 +207938,207938 +207939,207939 +207865,207939 +207775,207939 +207715,207939 +207940,207940 +207941,207941 +207942,207942 +207943,207943 +247999,207943 +207982,207982 +207983,207983 +207984,207984 +194916,207984 +207985,207985 +114802,207985 +207986,207986 +207987,207987 +207988,207988 +207989,207989 +207990,207990 +193758,207990 +207991,207991 +207992,207992 +207993,207993 +207994,207994 +207995,207995 +207996,207996 +207997,207997 +207998,207998 +207999,207999 +208000,208000 +208001,208001 +208002,208002 +83524,208002 +208003,208003 +208004,208004 +208005,208005 +208006,208006 +208007,208007 +208008,208008 +208009,208009 +208010,208010 +208011,208011 +208012,208012 +208013,208013 +19031,208013 +208014,208014 +208015,208015 +208016,208016 +208017,208017 +208018,208018 +234204,208018 +208019,208019 +208020,208020 +208021,208021 +208022,208022 +208023,208023 +211206,208023 +208024,208024 +208025,208025 +208026,208026 +224552,208026 +208027,208027 +208028,208028 +208029,208029 +208030,208030 +208031,208031 +208032,208032 +208033,208033 +208034,208034 +208035,208035 +208036,208036 +208037,208037 +208038,208038 +77543,208038 +208039,208039 +208040,208040 +208041,208041 +208042,208042 +208144,208042 +208043,208043 +208044,208044 +208045,208045 +208046,208046 +208047,208047 +64932,208047 +208048,208048 +208049,208049 +208050,208050 +208051,208051 +208052,208052 +208053,208053 +208054,208054 +160115,208054 +208055,208055 +79880,208055 +208056,208056 +208057,208057 +208058,208058 +208059,208059 +208060,208060 +208061,208061 +208062,208062 +208063,208063 +209604,208063 +208064,208064 +208065,208065 +208066,208066 +208067,208067 +216814,208067 +208068,208068 +208069,208069 +204497,208069 +208070,208070 +208071,208071 +154857,208071 +208072,208072 +208073,208073 +233790,208074 +208074,208074 +94987,208074 +208075,208075 +208076,208076 +178834,208076 +208077,208077 +208078,208078 +208079,208079 +208080,208080 +208081,208081 +208082,208082 +208083,208083 +208084,208084 +208085,208085 +208086,208086 +208087,208087 +208088,208088 +208089,208089 +208090,208090 +208091,208091 +208092,208092 +208093,208093 +208094,208094 +208095,208095 +208096,208096 +208097,208097 +208098,208098 +208099,208099 +208100,208100 +208101,208101 +208102,208102 +208103,208103 +208104,208104 +208105,208105 +208106,208106 +208107,208107 +208108,208108 +208109,208109 +208110,208110 +208111,208111 +208112,208112 +208113,208113 +234252,208113 +208114,208114 +208115,208115 +208116,208116 +211298,208116 +208117,208117 +208118,208118 +208119,208119 +208120,208120 +208121,208121 +208122,208122 +208123,208123 +208124,208124 +208125,208125 +208126,208126 +208127,208127 +208128,208128 +208129,208129 +208130,208130 +208131,208131 +208132,208132 +208133,208133 +15096,208133 +208134,208134 +208135,208135 +208136,208136 +208137,208137 +208138,208138 +208139,208139 +208140,208140 +208141,208141 +208142,208142 +108677,208142 +208143,208143 +208144,208144 +208042,208144 +208145,208145 +208146,208146 +208147,208147 +208148,208148 +208149,208149 +208150,208150 +208151,208151 +34257,208151 +208152,208152 +208153,208153 +208154,208154 +208155,208155 +208156,208156 +208157,208157 +208158,208158 +208159,208159 +208160,208160 +208161,208161 +208162,208162 +208163,208163 +208164,208164 +208165,208165 +208166,208166 +208167,208167 +208168,208168 +208169,208169 +208170,208170 +208171,208171 +208172,208172 +208173,208173 +208174,208174 +208175,208175 +208176,208176 +208177,208177 +208178,208178 +208179,208179 +208180,208180 +208181,208181 +208182,208182 +181310,208182 +208183,208183 +208184,208184 +208185,208185 +208186,208186 +208187,208187 +208188,208188 +208189,208189 +208190,208190 +208191,208191 +208192,208192 +208193,208193 +208194,208194 +84636,208194 +107235,208194 +208195,208195 +160148,208195 +208196,208196 +208197,208197 +208198,208198 +208199,208199 +208200,208200 +208201,208201 +208202,208202 +208203,208203 +208204,208204 +208205,208205 +104036,208205 +208206,208206 +208207,208207 +208208,208208 +208209,208209 +208210,208210 +7703,208210 +208211,208211 +208212,208212 +208213,208213 +208214,208214 +208215,208215 +208216,208216 +208217,208217 +208218,208218 +208219,208219 +208220,208220 +208221,208221 +208222,208222 +208223,208223 +153098,208223 +208224,208224 +208225,208225 +208226,208226 +208227,208227 +208228,208228 +208229,208229 +89493,208229 +208230,208230 +208231,208231 +208232,208232 +208233,208233 +208234,208234 +208235,208235 +208236,208236 +208237,208237 +208238,208238 +208239,208239 +208240,208240 +208241,208241 +208242,208242 +208243,208243 +208244,208244 +208245,208245 +208246,208246 +208247,208247 +208248,208248 +208249,208249 +208250,208250 +208251,208251 +208252,208252 +208253,208253 +208254,208254 +150695,208254 +208255,208255 +208256,208256 +208257,208257 +208258,208258 +208259,208259 +208260,208260 +208261,208261 +208262,208262 +208263,208263 +208264,208264 +27177,208264 +208265,208265 +208266,208266 +241047,208266 +208267,208267 +208268,208268 +208269,208269 +208270,208270 +208271,208271 +208272,208272 +208273,208273 +208274,208274 +208275,208275 +208276,208276 +208277,208277 +208278,208278 +208279,208279 +208280,208280 +208281,208281 +208282,208282 +57150,208282 +208283,208283 +208284,208284 +208285,208285 +208286,208286 +77963,208286 +208287,208287 +208288,208288 +208289,208289 +208290,208290 +208291,208291 +208292,208292 +208293,208293 +208294,208294 +208295,208295 +208296,208296 +230365,208296 +208297,208297 +208298,208298 +208299,208299 +208300,208300 +208301,208301 +208302,208302 +208303,208303 +107096,208303 +208304,208304 +208305,208305 +208306,208306 +208307,208307 +208308,208308 +208309,208309 +208310,208310 +208311,208311 +208312,208312 +208313,208313 +208314,208314 +208315,208315 +208316,208316 +208317,208317 +208318,208318 +208319,208319 +208320,208320 +208321,208321 +208322,208322 +208323,208323 +208324,208324 +208325,208325 +208326,208326 +208327,208327 +208328,208328 +208329,208329 +208330,208330 +208331,208331 +208332,208332 +208333,208333 +208334,208334 +208335,208335 +208336,208336 +208337,208337 +208338,208338 +208339,208339 +208340,208340 +208341,208341 +208342,208342 +208343,208343 +208344,208344 +104675,208344 +208345,208345 +208346,208346 +208347,208347 +208348,208348 +208349,208349 +208350,208350 +208351,208351 +208352,208352 +208353,208353 +208354,208354 +208355,208355 +208356,208356 +208357,208357 +208358,208358 +208359,208359 +208360,208360 +208361,208361 +208362,208362 +208363,208363 +208364,208364 +208365,208365 +208366,208366 +208367,208367 +208368,208368 +208369,208369 +208370,208370 +208371,208371 +208372,208372 +208373,208373 +208374,208374 +208375,208375 +208376,208376 +208377,208377 +208378,208378 +208379,208379 +208380,208380 +208381,208381 +208382,208382 +208383,208383 +76391,208383 +208384,208384 +208385,208385 +208386,208386 +208387,208387 +208388,208388 +208389,208389 +208390,208390 +208391,208391 +208392,208392 +208393,208393 +208394,208394 +208395,208395 +208396,208396 +208397,208397 +208398,208398 +208399,208399 +208400,208400 +208401,208401 +208402,208402 +208403,208403 +208404,208404 +208405,208405 +208406,208406 +102226,208406 +208407,208407 +208408,208408 +208409,208409 +208410,208410 +208411,208411 +208412,208412 +208413,208413 +208414,208414 +210181,208415 +208415,208415 +208416,208416 +208417,208417 +208418,208418 +208419,208419 +208420,208420 +208421,208421 +59777,208421 +208422,208422 +224945,208422 +208423,208423 +208424,208424 +208425,208425 +134051,208425 +126464,208425 +208426,208426 +208427,208427 +235905,208427 +208428,208428 +208429,208429 +208430,208430 +208431,208431 +208432,208432 +208433,208433 +208434,208434 +208435,208435 +215346,208435 +208436,208436 +208437,208437 +208438,208438 +208439,208439 +208440,208440 +208441,208441 +208442,208442 +208443,208443 +208444,208444 +208445,208445 +208446,208446 +208447,208447 +208448,208448 +208449,208449 +208450,208450 +208451,208451 +208452,208452 +208453,208453 +208454,208454 +208455,208455 +208456,208456 +208457,208457 +208458,208458 +208459,208459 +208460,208460 +25036,208460 +208461,208461 +208462,208462 +208463,208463 +208464,208464 +26335,208464 +208465,208465 +208466,208466 +208467,208467 +208468,208468 +208469,208469 +208470,208470 +112318,208470 +208471,208471 +229161,208471 +208472,208472 +208473,208473 +208474,208474 +158799,208474 +208475,208475 +208476,208476 +33815,208476 +208477,208477 +208478,208478 +208479,208479 +208480,208480 +95881,208480 +208481,208481 +208482,208482 +208483,208483 +208484,208484 +208485,208485 +208486,208486 +208487,208487 +208488,208488 +208489,208489 +208490,208490 +208491,208491 +208492,208492 +208493,208493 +208494,208494 +165005,208494 +208495,208495 +202692,208495 +208496,208496 +208497,208497 +208498,208498 +208499,208499 +208500,208500 +208501,208501 +208502,208502 +5773,208502 +208503,208503 +208504,208504 +208505,208505 +208506,208506 +208507,208507 +208508,208508 +208509,208509 +208510,208510 +208511,208511 +208512,208512 +208513,208513 +208514,208514 +208515,208515 +208516,208516 +208517,208517 +208518,208518 +208519,208519 +208520,208520 +84197,208520 +208521,208521 +208522,208522 +208523,208523 +208524,208524 +208525,208525 +208526,208526 +208527,208527 +208528,208528 +208529,208529 +208530,208530 +33870,208530 +208531,208531 +208532,208532 +208533,208533 +208534,208534 +208535,208535 +208536,208536 +208537,208537 +208538,208538 +208539,208539 +208540,208540 +208541,208541 +208542,208542 +208543,208543 +208544,208544 +139782,208544 +208545,208545 +208546,208546 +21929,208546 +208547,208547 +208548,208548 +208549,208549 +208550,208550 +208551,208551 +208552,208552 +208553,208553 +141372,208553 +208554,208554 +71063,208554 +164506,208554 +208555,208555 +208556,208556 +208557,208557 +208558,208558 +208559,208559 +208560,208560 +208561,208561 +208562,208562 +208563,208563 +208564,208564 +208565,208565 +208566,208566 +208567,208567 +208568,208568 +208569,208569 +208570,208570 +208571,208571 +208572,208572 +208573,208573 +208574,208574 +208575,208575 +208576,208576 +208577,208577 +208578,208578 +208579,208579 +208580,208580 +208581,208581 +208582,208582 +208583,208583 +208584,208584 +29250,208584 +208585,208585 +208586,208586 +208587,208587 +208588,208588 +162671,208588 +208589,208589 +162672,208589 +73623,208589 +208590,208590 +208591,208591 +208592,208592 +208593,208593 +208594,208594 +208595,208595 +208596,208596 +208597,208597 +208598,208598 +208599,208599 +208600,208600 +208601,208601 +208602,208602 +208603,208603 +208604,208604 +208605,208605 +208606,208606 +207893,208606 +208607,208607 +208608,208608 +70164,208608 +208609,208609 +208610,208610 +208611,208611 +208612,208612 +208613,208613 +208614,208614 +208615,208615 +208616,208616 +208617,208617 +208618,208618 +208619,208619 +208620,208620 +208621,208621 +208622,208622 +208623,208623 +208624,208624 +208625,208625 +208626,208626 +208627,208627 +208628,208628 +208629,208629 +208630,208630 +85209,208630 +208631,208631 +177631,208631 +171667,208631 +208632,208632 +208633,208633 +208634,208634 +208635,208635 +208636,208636 +208637,208637 +208638,208638 +208639,208639 +208640,208640 +208641,208641 +208642,208642 +208643,208643 +208644,208644 +208645,208645 +211998,208645 +208646,208646 +208647,208647 +9338,208647 +208648,208648 +208649,208649 +208650,208650 +208651,208651 +208652,208652 +208653,208653 +208654,208654 +208655,208655 +147675,208655 +208656,208656 +208657,208657 +208658,208658 +208659,208659 +208660,208660 +208661,208661 +208662,208662 +208663,208663 +208664,208664 +208665,208665 +208666,208666 +208667,208667 +208668,208668 +208669,208669 +208670,208670 +208671,208671 +208672,208672 +208673,208673 +208674,208674 +208675,208675 +208676,208676 +208677,208677 +208678,208678 +208679,208679 +208680,208680 +208681,208681 +211516,208681 +208682,208682 +208683,208683 +208684,208684 +208685,208685 +208686,208686 +208687,208687 +208688,208688 +208689,208689 +208690,208690 +208691,208691 +208692,208692 +208693,208693 +208694,208694 +208695,208695 +208696,208696 +208697,208697 +208698,208698 +208699,208699 +208700,208700 +208701,208701 +208702,208702 +208703,208703 +208704,208704 +208705,208705 +223605,208706 +208706,208706 +208707,208707 +208708,208708 +208709,208709 +208710,208710 +208711,208711 +208712,208712 +208713,208713 +208714,208714 +208715,208715 +208716,208716 +208717,208717 +208718,208718 +208719,208719 +208720,208720 +208721,208721 +208722,208722 +208723,208723 +208724,208724 +208725,208725 +208726,208726 +208727,208727 +208728,208728 +208729,208729 +208730,208730 +208731,208731 +208732,208732 +208733,208733 +208734,208734 +208735,208735 +208736,208736 +208737,208737 +208738,208738 +208739,208739 +74991,208739 +208740,208740 +208741,208741 +208742,208742 +208743,208743 +208744,208744 +208745,208745 +208746,208746 +112307,208746 +208747,208747 +224637,208747 +208748,208748 +208749,208749 +80734,208749 +208750,208750 +154856,208750 +208751,208751 +208752,208752 +208753,208753 +208754,208754 +208755,208755 +208756,208756 +208757,208757 +208758,208758 +208759,208759 +208760,208760 +208761,208761 +208762,208762 +208763,208763 +208764,208764 +208765,208765 +208766,208766 +208767,208767 +138896,208767 +160430,208767 +208768,208768 +208769,208769 +208770,208770 +208771,208771 +208772,208772 +149349,208772 +208773,208773 +208774,208774 +208775,208775 +208776,208776 +219829,208776 +208777,208777 +9465,208777 +208778,208778 +11191,208778 +208779,208779 +208780,208780 +108703,208780 +208781,208781 +208782,208782 +208783,208783 +208784,208784 +208785,208785 +208786,208786 +208787,208787 +208788,208788 +234719,208788 +208789,208789 +231242,208789 +208790,208790 +208791,208791 +208792,208792 +208793,208793 +208794,208794 +208795,208795 +208796,208796 +208797,208797 +208798,208798 +208799,208799 +208800,208800 +208801,208801 +208802,208802 +208803,208803 +190295,208803 +208804,208804 +208805,208805 +208806,208806 +208807,208807 +208808,208808 +208809,208809 +208810,208810 +208811,208811 +208812,208812 +208813,208813 +208814,208814 +208815,208815 +208816,208816 +208817,208817 +208818,208818 +208819,208819 +28975,208819 +208820,208820 +186047,208820 +208821,208821 +208822,208822 +208823,208823 +208824,208824 +208825,208825 +208826,208826 +208827,208827 +208828,208828 +208829,208829 +208830,208830 +208831,208831 +208832,208832 +208833,208833 +208834,208834 +208835,208835 +208836,208836 +151349,208836 +208837,208837 +208838,208838 +208839,208839 +208840,208840 +208841,208841 +117717,208841 +208842,208842 +208843,208843 +208844,208844 +208845,208845 +157808,208845 +208846,208846 +208847,208847 +208848,208848 +208849,208849 +208850,208850 +208851,208851 +208852,208852 +208853,208853 +208854,208854 +67589,208854 +208855,208855 +208856,208856 +208857,208857 +208858,208858 +208859,208859 +208860,208860 +208861,208861 +208862,208862 +208863,208863 +208864,208864 +208865,208865 +208866,208866 +208867,208867 +208868,208868 +208869,208869 +208870,208870 +208871,208871 +208872,208872 +208873,208873 +208874,208874 +87503,208874 +208875,208875 +176662,208875 +170698,208875 +208876,208876 +208877,208877 +208878,208878 +208879,208879 +208880,208880 +208881,208881 +208882,208882 +208883,208883 +208884,208884 +208885,208885 +208886,208886 +208887,208887 +208888,208888 +208889,208889 +208890,208890 +208891,208891 +208892,208892 +208893,208893 +208894,208894 +208895,208895 +208896,208896 +208897,208897 +208898,208898 +208899,208899 +208900,208900 +208901,208901 +208902,208902 +233870,208902 +208903,208903 +208904,208904 +208905,208905 +208906,208906 +209232,209232 +209233,209233 +209234,209234 +209235,209235 +209339,209339 +209340,209340 +209341,209341 +209342,209342 +209343,209343 +209344,209344 +82882,209344 +209345,209345 +209346,209346 +209347,209347 +209348,209348 +209349,209349 +209350,209350 +209351,209351 +209352,209352 +83968,209352 +209353,209353 +209354,209354 +209355,209355 +209356,209356 +209357,209357 +209358,209358 +209359,209359 +121052,209359 +209360,209360 +209361,209361 +209362,209362 +209363,209363 +235173,209363 +209364,209364 +209365,209365 +209366,209366 +209367,209367 +209368,209368 +209369,209369 +222459,209370 +220607,209370 +209370,209370 +209371,209371 +209372,209372 +209373,209373 +209374,209374 +209375,209375 +209376,209376 +209377,209377 +209378,209378 +209379,209379 +209380,209380 +209381,209381 +209382,209382 +209383,209383 +209384,209384 +209385,209385 +209386,209386 +209387,209387 +209388,209388 +209389,209389 +209390,209390 +209391,209391 +209392,209392 +209393,209393 +209394,209394 +209395,209395 +202496,209395 +188355,209395 +209396,209396 +209397,209397 +235132,209398 +209398,209398 +209399,209399 +209400,209400 +209401,209401 +75414,209401 +209402,209402 +209403,209403 +209404,209404 +209405,209405 +209406,209406 +140162,209406 +209407,209407 +209408,209408 +209409,209409 +209410,209410 +209411,209411 +209412,209412 +209413,209413 +209414,209414 +209415,209415 +209416,209416 +209417,209417 +209418,209418 +209419,209419 +209420,209420 +209421,209421 +209422,209422 +209423,209423 +209424,209424 +209425,209425 +209426,209426 +209427,209427 +209428,209428 +209429,209429 +209430,209430 +209431,209431 +160409,209431 +209432,209432 +209433,209433 +209434,209434 +209435,209435 +209436,209436 +209437,209437 +209438,209438 +221757,209439 +209439,209439 +209440,209440 +209441,209441 +209442,209442 +209443,209443 +209444,209444 +209445,209445 +209446,209446 +209447,209447 +190481,209447 +209448,209448 +209449,209449 +209450,209450 +209451,209451 +209452,209452 +209453,209453 +209454,209454 +209455,209455 +209456,209456 +209457,209457 +209458,209458 +209459,209459 +209460,209460 +67596,209460 +209461,209461 +209462,209462 +209463,209463 +209464,209464 +209465,209465 +209466,209466 +209467,209467 +209468,209468 +209469,209469 +209470,209470 +209471,209471 +209472,209472 +209473,209473 +209474,209474 +209475,209475 +209476,209476 +209477,209477 +209478,209478 +209479,209479 +30813,209479 +209480,209480 +209481,209481 +209482,209482 +209483,209483 +209484,209484 +209485,209485 +209486,209486 +209487,209487 +209488,209488 +209489,209489 +114377,209489 +209490,209490 +209491,209491 +209492,209492 +209493,209493 +209494,209494 +209495,209495 +209496,209496 +209497,209497 +209498,209498 +209499,209499 +209500,209500 +209501,209501 +209502,209502 +209503,209503 +209504,209504 +209505,209505 +209506,209506 +209507,209507 +209508,209508 +209509,209509 +209510,209510 +209511,209511 +209512,209512 +134008,209512 +126421,209512 +209513,209513 +209514,209514 +234157,209514 +209515,209515 +209516,209516 +209517,209517 +209518,209518 +209519,209519 +209520,209520 +209521,209521 +209522,209522 +209523,209523 +142051,209523 +209524,209524 +209525,209525 +209526,209526 +209527,209527 +209528,209528 +209529,209529 +209530,209530 +209531,209531 +209532,209532 +190688,209532 +209533,209533 +209534,209534 +209535,209535 +209536,209536 +209537,209537 +209538,209538 +209539,209539 +209540,209540 +209541,209541 +209542,209542 +209543,209543 +209544,209544 +209545,209545 +209546,209546 +209547,209547 +209548,209548 +209549,209549 +209550,209550 +209551,209551 +209552,209552 +209553,209553 +210144,209553 +209554,209554 +209555,209555 +209556,209556 +209557,209557 +209558,209558 +209559,209559 +209560,209560 +209561,209561 +209562,209562 +209563,209563 +209564,209564 +209565,209565 +209566,209566 +209567,209567 +209568,209568 +209569,209569 +209570,209570 +209571,209571 +209572,209572 +236125,209572 +209573,209573 +209574,209574 +209575,209575 +209576,209576 +209577,209577 +209578,209578 +209579,209579 +209580,209580 +209581,209581 +209582,209582 +209583,209583 +209584,209584 +209585,209585 +209586,209586 +209587,209587 +209588,209588 +209589,209589 +209590,209590 +209591,209591 +209592,209592 +209593,209593 +209594,209594 +209595,209595 +209596,209596 +28226,209596 +209597,209597 +209598,209598 +209599,209599 +209600,209600 +209601,209601 +209602,209602 +209603,209603 +106737,209603 +209604,209604 +208063,209604 +209605,209605 +209606,209606 +209607,209607 +209608,209608 +209609,209609 +209610,209610 +209611,209611 +209612,209612 +209613,209613 +209614,209614 +209615,209615 +209616,209616 +209617,209617 +209618,209618 +209619,209619 +209620,209620 +209621,209621 +209622,209622 +209623,209623 +81311,209623 +209624,209624 +21633,209624 +209625,209625 +122728,209625 +209626,209626 +209627,209627 +210002,209627 +138873,209627 +209628,209628 +209629,209629 +209630,209630 +209631,209631 +209632,209632 +209633,209633 +209634,209634 +209635,209635 +161806,209635 +209636,209636 +209637,209637 +209638,209638 +209639,209639 +209640,209640 +209641,209641 +209642,209642 +209643,209643 +209644,209644 +209645,209645 +209646,209646 +209647,209647 +209648,209648 +209649,209649 +209650,209650 +209651,209651 +209652,209652 +16258,209652 +209653,209653 +209654,209654 +209655,209655 +209656,209656 +209657,209657 +209658,209658 +209659,209659 +209660,209660 +209661,209661 +209662,209662 +209663,209663 +209664,209664 +209665,209665 +188228,209665 +209666,209666 +226668,209666 +209667,209667 +209668,209668 +209669,209669 +209670,209670 +209671,209671 +209672,209672 +209673,209673 +209674,209674 +209675,209675 +209676,209676 +209677,209677 +209678,209678 +209679,209679 +209680,209680 +212046,209680 +209681,209681 +209682,209682 +209683,209683 +209684,209684 +209685,209685 +212051,209685 +209686,209686 +209687,209687 +209688,209688 +209689,209689 +209690,209690 +209691,209691 +209692,209692 +209693,209693 +209694,209694 +209695,209695 +209696,209696 +209697,209697 +209698,209698 +209699,209699 +209700,209700 +209701,209701 +209702,209702 +209703,209703 +209704,209704 +209705,209705 +209706,209706 +209707,209707 +209708,209708 +209709,209709 +209710,209710 +209711,209711 +209712,209712 +209713,209713 +209714,209714 +209715,209715 +209716,209716 +209717,209717 +209718,209718 +209719,209719 +209720,209720 +209721,209721 +209722,209722 +209723,209723 +209724,209724 +209725,209725 +209726,209726 +209727,209727 +209728,209728 +209729,209729 +209730,209730 +209731,209731 +26276,209731 +209732,209732 +209733,209733 +209734,209734 +209735,209735 +209736,209736 +227791,209736 +209737,209737 +209738,209738 +209739,209739 +209740,209740 +209741,209741 +209742,209742 +209743,209743 +209744,209744 +209745,209745 +209746,209746 +209747,209747 +209748,209748 +83019,209748 +59505,209748 +209749,209749 +209750,209750 +209751,209751 +209752,209752 +183070,209752 +209753,209753 +209754,209754 +209755,209755 +209756,209756 +209757,209757 +209758,209758 +209759,209759 +209760,209760 +209761,209761 +209762,209762 +209763,209763 +209764,209764 +209765,209765 +209766,209766 +209767,209767 +209768,209768 +209769,209769 +209770,209770 +209771,209771 +209772,209772 +191158,209772 +209773,209773 +78078,209773 +209774,209774 +209775,209775 +209776,209776 +209777,209777 +209778,209778 +209779,209779 +209780,209780 +209781,209781 +209782,209782 +209783,209783 +209784,209784 +209785,209785 +209786,209786 +209787,209787 +209788,209788 +209789,209789 +209790,209790 +209791,209791 +209792,209792 +209793,209793 +209794,209794 +178573,209794 +209795,209795 +205937,209795 +209796,209796 +209797,209797 +209798,209798 +26197,209798 +209799,209799 +209800,209800 +209801,209801 +209802,209802 +31188,209802 +209803,209803 +209804,209804 +224596,209804 +209805,209805 +209806,209806 +209807,209807 +209808,209808 +209809,209809 +209810,209810 +209811,209811 +209812,209812 +85451,209812 +209813,209813 +209814,209814 +209815,209815 +209816,209816 +209817,209817 +209818,209818 +209819,209819 +209820,209820 +209821,209821 +209822,209822 +209823,209823 +209824,209824 +209825,209825 +209826,209826 +209827,209827 +209828,209828 +209829,209829 +209830,209830 +209831,209831 +209832,209832 +209833,209833 +209834,209834 +209835,209835 +209874,209874 +209875,209875 +209876,209876 +209877,209877 +209878,209878 +209879,209879 +209880,209880 +209881,209881 +209882,209882 +209883,209883 +209884,209884 +209885,209885 +209886,209886 +209887,209887 +79837,209887 +209888,209888 +209889,209889 +209890,209890 +234299,209890 +209891,209891 +209892,209892 +209893,209893 +209894,209894 +196521,209894 +209895,209895 +209896,209896 +209897,209897 +209898,209898 +209899,209899 +142233,209899 +209900,209900 +209901,209901 +209902,209902 +209903,209903 +209904,209904 +209905,209905 +209906,209906 +209907,209907 +209908,209908 +209909,209909 +209910,209910 +246999,209910 +209911,209911 +209912,209912 +209913,209913 +209914,209914 +209915,209915 +209916,209916 +209917,209917 +209918,209918 +209919,209919 +209920,209920 +209921,209921 +209922,209922 +209923,209923 +209924,209924 +209925,209925 +209926,209926 +209927,209927 +213762,209927 +209928,209928 +209929,209929 +209930,209930 +209931,209931 +209932,209932 +209933,209933 +209934,209934 +209935,209935 +209936,209936 +209937,209937 +209938,209938 +209939,209939 +209940,209940 +209941,209941 +209942,209942 +209943,209943 +133037,209943 +125450,209943 +209944,209944 +209945,209945 +209946,209946 +209947,209947 +209948,209948 +209949,209949 +209950,209950 +209951,209951 +224555,209951 +209952,209952 +209953,209953 +209954,209954 +209955,209955 +209956,209956 +209957,209957 +216671,209957 +209958,209958 +209959,209959 +209960,209960 +209961,209961 +227121,209962 +209962,209962 +209963,209963 +223497,209963 +9003,209963 +209964,209964 +209965,209965 +93830,209965 +209966,209966 +211128,209966 +209967,209967 +209968,209968 +209969,209969 +209970,209970 +209971,209971 +209972,209972 +209973,209973 +209974,209974 +209975,209975 +209976,209976 +209977,209977 +209978,209978 +209979,209979 +209980,209980 +209981,209981 +209982,209982 +209983,209983 +209984,209984 +89543,209984 +209985,209985 +209986,209986 +209987,209987 +209988,209988 +209989,209989 +209990,209990 +209991,209991 +209992,209992 +209993,209993 +209994,209994 +209995,209995 +203877,209995 +209996,209996 +209997,209997 +72259,209997 +209998,209998 +209999,209999 +210000,210000 +210001,210001 +123299,210001 +210002,210002 +209627,210002 +210003,210003 +210004,210004 +210005,210005 +210006,210006 +210007,210007 +210008,210008 +210009,210009 +210010,210010 +210011,210011 +210012,210012 +210013,210013 +210014,210014 +210015,210015 +210016,210016 +210017,210017 +210018,210018 +210019,210019 +210020,210020 +210021,210021 +210022,210022 +210023,210023 +210024,210024 +210025,210025 +210026,210026 +210027,210027 +210028,210028 +105699,210028 +210029,210029 +210030,210030 +210031,210031 +210032,210032 +210033,210033 +210034,210034 +210035,210035 +210036,210036 +210037,210037 +25559,210037 +210038,210038 +210039,210039 +210040,210040 +210041,210041 +210042,210042 +210043,210043 +210044,210044 +210045,210045 +210046,210046 +210047,210047 +210048,210048 +210049,210049 +210050,210050 +210051,210051 +210052,210052 +210053,210053 +210054,210054 +210055,210055 +210056,210056 +210057,210057 +210058,210058 +210059,210059 +210060,210060 +210061,210061 +210062,210062 +210063,210063 +210064,210064 +210065,210065 +210066,210066 +210067,210067 +210068,210068 +210069,210069 +210070,210070 +210071,210071 +210072,210072 +210073,210073 +210074,210074 +86691,210074 +210075,210075 +210076,210076 +210077,210077 +210078,210078 +210079,210079 +210080,210080 +67225,210080 +210081,210081 +210082,210082 +210083,210083 +112270,210083 +210084,210084 +80556,210084 +210085,210085 +210086,210086 +210087,210087 +159436,210087 +210088,210088 +210089,210089 +210090,210090 +162862,210090 +210091,210091 +210092,210092 +210093,210093 +210094,210094 +210095,210095 +210096,210096 +210097,210097 +210098,210098 +210099,210099 +210100,210100 +210101,210101 +210102,210102 +210103,210103 +210104,210104 +210105,210105 +210106,210106 +210107,210107 +210108,210108 +210109,210109 +210110,210110 +210111,210111 +210112,210112 +210113,210113 +210114,210114 +210115,210115 +210116,210116 +210117,210117 +210118,210118 +210119,210119 +210120,210120 +210121,210121 +210122,210122 +210123,210123 +210124,210124 +210125,210125 +210126,210126 +210127,210127 +210128,210128 +210129,210129 +210130,210130 +210131,210131 +210132,210132 +210133,210133 +210134,210134 +210135,210135 +210136,210136 +210137,210137 +210138,210138 +210139,210139 +210140,210140 +210141,210141 +210142,210142 +210143,210143 +210144,210144 +209553,210144 +123293,210144 +210145,210145 +210146,210146 +210147,210147 +210148,210148 +210149,210149 +210150,210150 +210151,210151 +210152,210152 +210153,210153 +210154,210154 +210155,210155 +210156,210156 +210157,210157 +210158,210158 +210159,210159 +210160,210160 +210161,210161 +210162,210162 +210163,210163 +210164,210164 +210165,210165 +210166,210166 +210167,210167 +210168,210168 +210169,210169 +210170,210170 +210171,210171 +210172,210172 +163467,210172 +210173,210173 +210174,210174 +210175,210175 +210176,210176 +210177,210177 +210178,210178 +210179,210179 +79242,210179 +210180,210180 +210181,210181 +208415,210181 +210182,210182 +210183,210183 +210184,210184 +210185,210185 +210186,210186 +210187,210187 +210188,210188 +210189,210189 +210190,210190 +210191,210191 +210192,210192 +210193,210193 +210194,210194 +210195,210195 +216013,210195 +210196,210196 +210197,210197 +210198,210198 +210199,210199 +9741,210199 +210200,210200 +26874,210200 +210201,210201 +210202,210202 +210203,210203 +210204,210204 +210205,210205 +210206,210206 +210207,210207 +210208,210208 +210209,210209 +210210,210210 +210211,210211 +116378,210211 +210212,210212 +210213,210213 +56220,210213 +48116,210213 +3208,210213 +210214,210214 +210215,210215 +210216,210216 +210217,210217 +210218,210218 +210219,210219 +210220,210220 +210221,210221 +210222,210222 +210223,210223 +210224,210224 +210225,210225 +210226,210226 +210227,210227 +210228,210228 +210229,210229 +210230,210230 +210231,210231 +210232,210232 +210233,210233 +210234,210234 +210235,210235 +210236,210236 +210237,210237 +210238,210238 +210239,210239 +210240,210240 +210241,210241 +210242,210242 +210243,210243 +210244,210244 +386,210244 +210245,210245 +210246,210246 +210247,210247 +210248,210248 +210249,210249 +210250,210250 +210251,210251 +210252,210252 +210253,210253 +210254,210254 +210255,210255 +210256,210256 +210257,210257 +210258,210258 +210259,210259 +210260,210260 +2419,210260 +210261,210261 +210262,210262 +210263,210263 +210264,210264 +210265,210265 +210266,210266 +210267,210267 +210268,210268 +210269,210269 +210270,210270 +210271,210271 +210272,210272 +210273,210273 +191884,210273 +210274,210274 +210275,210275 +210276,210276 +210277,210277 +84099,210277 +210278,210278 +191789,210278 +210279,210279 +210280,210280 +210281,210281 +210282,210282 +142006,210282 +210283,210283 +210284,210284 +210285,210285 +210286,210286 +210287,210287 +146488,210287 +210288,210288 +210289,210289 +210290,210290 +210291,210291 +210292,210292 +210293,210293 +210294,210294 +210295,210295 +210296,210296 +210297,210297 +210298,210298 +210299,210299 +210300,210300 +210301,210301 +210302,210302 +210303,210303 +210304,210304 +214546,210304 +210305,210305 +210306,210306 +210307,210307 +210308,210308 +210309,210309 +210310,210310 +210311,210311 +210312,210312 +210313,210313 +210314,210314 +210315,210315 +210316,210316 +210317,210317 +210318,210318 +210319,210319 +210320,210320 +104817,210320 +210321,210321 +210322,210322 +210323,210323 +210324,210324 +210325,210325 +210326,210326 +210327,210327 +210328,210328 +210329,210329 +210330,210330 +216409,210330 +210331,210331 +210332,210332 +203557,210332 +210333,210333 +210334,210334 +25023,210334 +153325,210334 +210335,210335 +210336,210336 +224408,210336 +210337,210337 +210338,210338 +210339,210339 +210340,210340 +210341,210341 +210342,210342 +210343,210343 +210344,210344 +210345,210345 +210346,210346 +210347,210347 +210348,210348 +210349,210349 +83643,210349 +210350,210350 +5822,210350 +210351,210351 +210352,210352 +210353,210353 +210354,210354 +210355,210355 +210356,210356 +210357,210357 +210358,210358 +210359,210359 +210360,210360 +210361,210361 +210362,210362 +210363,210363 +210364,210364 +210365,210365 +210366,210366 +210367,210367 +210368,210368 +210369,210369 +210370,210370 +210371,210371 +210372,210372 +210431,210431 +210432,210432 +210433,210433 +210434,210434 +210588,210588 +141360,210588 +211078,211078 +211079,211079 +211080,211080 +211081,211081 +211082,211082 +211083,211083 +201450,211083 +211084,211084 +211085,211085 +211086,211086 +211087,211087 +211088,211088 +211089,211089 +211090,211090 +211091,211091 +162413,211091 +211092,211092 +211093,211093 +211094,211094 +115940,211094 +211095,211095 +211096,211096 +211097,211097 +211098,211098 +211099,211099 +211100,211100 +211101,211101 +211102,211102 +211103,211103 +211104,211104 +211105,211105 +211106,211106 +211107,211107 +211108,211108 +211109,211109 +211110,211110 +211111,211111 +211112,211112 +211113,211113 +211114,211114 +211115,211115 +211116,211116 +211117,211117 +211118,211118 +211119,211119 +211120,211120 +162167,211120 +211121,211121 +211122,211122 +211123,211123 +211124,211124 +211125,211125 +211126,211126 +211127,211127 +211128,211128 +209966,211128 +211129,211129 +211130,211130 +211131,211131 +179169,211131 +211132,211132 +211133,211133 +211134,211134 +35776,211134 +211135,211135 +50037,211135 +41844,211135 +231305,211135 +211136,211136 +211137,211137 +79914,211137 +211138,211138 +211139,211139 +211140,211140 +106212,211140 +211141,211141 +211142,211142 +239539,211142 +211143,211143 +211144,211144 +211145,211145 +211146,211146 +211147,211147 +211148,211148 +211149,211149 +211150,211150 +211151,211151 +211152,211152 +211153,211153 +211154,211154 +211155,211155 +211156,211156 +211157,211157 +142947,211157 +211158,211158 +211159,211159 +211160,211160 +211161,211161 +211162,211162 +211163,211163 +211164,211164 +211165,211165 +211166,211166 +211167,211167 +90978,211167 +211168,211168 +211169,211169 +79805,211169 +211170,211170 +211171,211171 +211172,211172 +211173,211173 +211174,211174 +211175,211175 +211176,211176 +211177,211177 +211178,211178 +211179,211179 +76337,211179 +211180,211180 +211181,211181 +211182,211182 +211183,211183 +211184,211184 +76399,211184 +211185,211185 +74949,211185 +211186,211186 +231290,211186 +32317,211186 +211187,211187 +211188,211188 +211189,211189 +211190,211190 +110822,211190 +211191,211191 +211192,211192 +77547,211192 +211193,211193 +211194,211194 +211689,211194 +211195,211195 +211196,211196 +211197,211197 +211198,211198 +211199,211199 +211200,211200 +211201,211201 +211202,211202 +211203,211203 +211204,211204 +211205,211205 +211206,211206 +213290,211206 +208023,211206 +211207,211207 +211208,211208 +154747,211208 +211209,211209 +211210,211210 +211211,211211 +211212,211212 +86038,211212 +211213,211213 +211214,211214 +211215,211215 +211216,211216 +242878,211216 +211217,211217 +211218,211218 +211219,211219 +211220,211220 +211221,211221 +211222,211222 +211223,211223 +211224,211224 +211225,211225 +211226,211226 +211227,211227 +211228,211228 +211229,211229 +211230,211230 +211231,211231 +211232,211232 +211233,211233 +211234,211234 +211235,211235 +211236,211236 +178345,211236 +172381,211236 +211237,211237 +211238,211238 +211239,211239 +211240,211240 +211241,211241 +211242,211242 +211243,211243 +211244,211244 +211245,211245 +211246,211246 +211247,211247 +211248,211248 +211249,211249 +211250,211250 +163536,211250 +211251,211251 +211252,211252 +211253,211253 +211254,211254 +111511,211254 +211255,211255 +211256,211256 +211257,211257 +211258,211258 +211259,211259 +211260,211260 +211261,211261 +211262,211262 +189478,211262 +211263,211263 +211264,211264 +211265,211265 +216399,211265 +211266,211266 +211267,211267 +211268,211268 +211269,211269 +211270,211270 +211271,211271 +211272,211272 +211273,211273 +211274,211274 +211275,211275 +211276,211276 +211277,211277 +211278,211278 +211279,211279 +211280,211280 +211281,211281 +211282,211282 +211283,211283 +211284,211284 +211285,211285 +211286,211286 +211287,211287 +211288,211288 +211289,211289 +211290,211290 +211291,211291 +211292,211292 +211293,211293 +211294,211294 +211295,211295 +211296,211296 +211297,211297 +211298,211298 +208116,211298 +211299,211299 +211300,211300 +211301,211301 +211302,211302 +211303,211303 +211304,211304 +234221,211304 +211305,211305 +211306,211306 +211307,211307 +211308,211308 +211309,211309 +211310,211310 +211311,211311 +211312,211312 +211313,211313 +211314,211314 +84294,211314 +211315,211315 +211316,211316 +211317,211317 +211318,211318 +211319,211319 +211320,211320 +148659,211320 +211321,211321 +211322,211322 +211323,211323 +227839,211324 +211324,211324 +211325,211325 +211326,211326 +141899,211326 +211327,211327 +211328,211328 +62612,211328 +211329,211329 +211330,211330 +211331,211331 +211332,211332 +160120,211332 +211333,211333 +211334,211334 +50230,211334 +42038,211334 +211335,211335 +211336,211336 +211337,211337 +211338,211338 +211339,211339 +211340,211340 +211341,211341 +193818,211341 +211342,211342 +211343,211343 +211344,211344 +211345,211345 +211346,211346 +211347,211347 +211348,211348 +211349,211349 +211350,211350 +185293,211350 +211351,211351 +211352,211352 +211353,211353 +211354,211354 +211355,211355 +211356,211356 +211357,211357 +211358,211358 +211359,211359 +211360,211360 +211361,211361 +211362,211362 +211363,211363 +211364,211364 +211365,211365 +235282,211365 +211366,211366 +211367,211367 +211368,211368 +211369,211369 +211370,211370 +211371,211371 +211372,211372 +211373,211373 +211374,211374 +211375,211375 +211376,211376 +211377,211377 +211378,211378 +211379,211379 +211380,211380 +156294,211380 +211381,211381 +211382,211382 +211383,211383 +211384,211384 +211385,211385 +211386,211386 +211387,211387 +211388,211388 +211389,211389 +211390,211390 +211391,211391 +112294,211391 +211392,211392 +211393,211393 +211394,211394 +122172,211394 +211395,211395 +211396,211396 +211397,211397 +211398,211398 +211399,211399 +211400,211400 +206105,211400 +211401,211401 +211402,211402 +211403,211403 +211404,211404 +201472,211404 +211405,211405 +211406,211406 +211407,211407 +211408,211408 +211409,211409 +211410,211410 +211411,211411 +211412,211412 +211413,211413 +70211,211413 +211414,211414 +211415,211415 +211416,211416 +211417,211417 +211418,211418 +211419,211419 +157092,211419 +211420,211420 +211421,211421 +211422,211422 +211423,211423 +211424,211424 +211425,211425 +211426,211426 +211427,211427 +211428,211428 +202552,211428 +211429,211429 +211430,211430 +211431,211431 +211432,211432 +211433,211433 +211434,211434 +211435,211435 +211436,211436 +211437,211437 +211438,211438 +211439,211439 +211440,211440 +211441,211441 +211442,211442 +211443,211443 +211444,211444 +211445,211445 +211446,211446 +211447,211447 +52752,211447 +44648,211447 +211448,211448 +232110,211448 +211449,211449 +22036,211449 +211450,211450 +211451,211451 +165232,211451 +211452,211452 +211453,211453 +211454,211454 +211455,211455 +211456,211456 +211457,211457 +211458,211458 +211459,211459 +211460,211460 +211461,211461 +211462,211462 +211463,211463 +211464,211464 +211465,211465 +211466,211466 +211467,211467 +211468,211468 +211469,211469 +211470,211470 +211471,211471 +211472,211472 +211473,211473 +108492,211473 +211474,211474 +211475,211475 +49390,211475 +41197,211475 +211476,211476 +211477,211477 +211478,211478 +211479,211479 +211942,211479 +211480,211480 +211481,211481 +211482,211482 +211483,211483 +211484,211484 +81407,211484 +211485,211485 +211486,211486 +211487,211487 +211488,211488 +211489,211489 +211490,211490 +211491,211491 +211492,211492 +211493,211493 +207623,211493 +211494,211494 +211495,211495 +211496,211496 +211497,211497 +211498,211498 +211499,211499 +211500,211500 +211501,211501 +211502,211502 +211503,211503 +211504,211504 +211505,211505 +211506,211506 +211507,211507 +211508,211508 +211509,211509 +211510,211510 +141520,211510 +211511,211511 +211512,211512 +211513,211513 +211514,211514 +211515,211515 +211516,211516 +208681,211516 +211517,211517 +211518,211518 +211519,211519 +211520,211520 +211521,211521 +211522,211522 +211523,211523 +211524,211524 +211525,211525 +211526,211526 +211527,211527 +211528,211528 +211529,211529 +211530,211530 +211531,211531 +211532,211532 +211533,211533 +211534,211534 +211535,211535 +211536,211536 +211537,211537 +211538,211538 +211539,211539 +88185,211539 +211540,211540 +211541,211541 +211542,211542 +211543,211543 +158130,211543 +211544,211544 +211545,211545 +211546,211546 +211547,211547 +211548,211548 +211549,211549 +211550,211550 +211551,211551 +211552,211552 +232972,211552 +211553,211553 +211554,211554 +134252,211554 +126665,211554 +211555,211555 +211556,211556 +157044,211556 +211557,211557 +211558,211558 +240898,211558 +211559,211559 +211560,211560 +211561,211561 +211562,211562 +211563,211563 +211564,211564 +211565,211565 +211566,211566 +211567,211567 +211568,211568 +211569,211569 +211570,211570 +211571,211571 +211572,211572 +211573,211573 +211574,211574 +211575,211575 +211576,211576 +211577,211577 +211578,211578 +211579,211579 +211580,211580 +211581,211581 +211582,211582 +211583,211583 +211584,211584 +211585,211585 +211586,211586 +211587,211587 +211588,211588 +211589,211589 +211590,211590 +211591,211591 +211592,211592 +154728,211592 +211593,211593 +211594,211594 +211595,211595 +75746,211595 +211596,211596 +211597,211597 +211598,211598 +216230,211598 +211599,211599 +211600,211600 +177718,211600 +171754,211600 +211601,211601 +211602,211602 +211603,211603 +211604,211604 +2449,211604 +211605,211605 +211606,211606 +211607,211607 +77235,211607 +211608,211608 +211609,211609 +211610,211610 +211611,211611 +211612,211612 +211613,211613 +211614,211614 +211615,211615 +211616,211616 +211617,211617 +211618,211618 +211619,211619 +211620,211620 +211621,211621 +211622,211622 +211623,211623 +211624,211624 +211625,211625 +211626,211626 +211627,211627 +211628,211628 +211629,211629 +211630,211630 +211631,211631 +211632,211632 +211633,211633 +211634,211634 +211635,211635 +211636,211636 +211637,211637 +211638,211638 +211639,211639 +211640,211640 +211641,211641 +211642,211642 +211643,211643 +145222,211643 +211644,211644 +211645,211645 +211646,211646 +211647,211647 +211648,211648 +211649,211649 +211650,211650 +211651,211651 +211652,211652 +211653,211653 +211654,211654 +211655,211655 +211656,211656 +211657,211657 +211658,211658 +211659,211659 +211660,211660 +211661,211661 +211662,211662 +211663,211663 +211664,211664 +211665,211665 +211666,211666 +211667,211667 +211668,211668 +211669,211669 +78701,211669 +211670,211670 +211671,211671 +211672,211672 +211673,211673 +123368,211673 +211674,211674 +211675,211675 +211676,211676 +211677,211677 +26219,211677 +211678,211678 +211679,211679 +211680,211680 +177650,211680 +171686,211680 +211681,211681 +211682,211682 +211683,211683 +211684,211684 +211685,211685 +211686,211686 +211687,211687 +211688,211688 +211689,211689 +211194,211689 +207051,211689 +211690,211690 +233983,211690 +211691,211691 +211692,211692 +211693,211693 +211694,211694 +211695,211695 +211696,211696 +211697,211697 +211698,211698 +211699,211699 +178818,211699 +211700,211700 +211701,211701 +211702,211702 +211703,211703 +211704,211704 +211758,211758 +54988,211758 +46884,211758 +211759,211759 +195034,211759 +211760,211760 +211761,211761 +211762,211762 +211763,211763 +211764,211764 +211765,211765 +202032,211765 +154761,211765 +153952,211765 +211766,211766 +211767,211767 +211768,211768 +6843,211768 +211769,211769 +211770,211770 +211771,211771 +211772,211772 +211773,211773 +211774,211774 +152250,211774 +211775,211775 +211776,211776 +97961,211776 +178060,211776 +172096,211776 +108449,211776 +211777,211777 +178699,211777 +211778,211778 +211779,211779 +211780,211780 +211781,211781 +211782,211782 +211783,211783 +211784,211784 +211785,211785 +211786,211786 +179021,211786 +211787,211787 +211788,211788 +211789,211789 +144185,211789 +211790,211790 +211791,211791 +211792,211792 +211793,211793 +211794,211794 +211795,211795 +211796,211796 +211797,211797 +211798,211798 +211799,211799 +211800,211800 +211801,211801 +211802,211802 +211803,211803 +109261,211803 +211804,211804 +9465,211804 +211805,211805 +211806,211806 +211807,211807 +211808,211808 +211809,211809 +211810,211810 +211811,211811 +211812,211812 +211813,211813 +211814,211814 +211815,211815 +211816,211816 +211817,211817 +211818,211818 +211819,211819 +211820,211820 +211821,211821 +211822,211822 +211823,211823 +211824,211824 +211825,211825 +211826,211826 +211893,211893 +221672,211893 +211894,211894 +211895,211895 +211896,211896 +38226,211896 +211897,211897 +211898,211898 +211899,211899 +211900,211900 +211901,211901 +211902,211902 +211903,211903 +211904,211904 +211905,211905 +211906,211906 +163043,211906 +211907,211907 +211908,211908 +211909,211909 +211910,211910 +211911,211911 +211912,211912 +211913,211913 +211914,211914 +211915,211915 +211916,211916 +211917,211917 +211918,211918 +211919,211919 +211920,211920 +211921,211921 +211922,211922 +211923,211923 +211924,211924 +88153,211924 +211925,211925 +211926,211926 +211927,211927 +211928,211928 +211929,211929 +211930,211930 +211931,211931 +211932,211932 +211933,211933 +211934,211934 +211935,211935 +211936,211936 +226823,211937 +211937,211937 +211938,211938 +211939,211939 +211940,211940 +211941,211941 +211942,211942 +103514,211942 +211479,211942 +211943,211943 +211944,211944 +211945,211945 +211946,211946 +188028,211946 +214454,211946 +164663,211946 +211947,211947 +211948,211948 +211949,211949 +211950,211950 +211951,211951 +211952,211952 +211953,211953 +211954,211954 +211955,211955 +211956,211956 +211957,211957 +211958,211958 +211959,211959 +20611,211959 +211960,211960 +33521,211960 +211961,211961 +211962,211962 +211963,211963 +211964,211964 +211965,211965 +211966,211966 +211967,211967 +144957,211967 +211968,211968 +211969,211969 +211970,211970 +211971,211971 +211972,211972 +218026,211972 +211973,211973 +211974,211974 +211975,211975 +211976,211976 +211977,211977 +211978,211978 +211979,211979 +211980,211980 +65026,211980 +233364,211981 +211981,211981 +215527,211981 +211982,211982 +211983,211983 +211984,211984 +211985,211985 +211986,211986 +211987,211987 +211988,211988 +211989,211989 +211990,211990 +211991,211991 +211992,211992 +211993,211993 +211994,211994 +211995,211995 +211996,211996 +211997,211997 +211998,211998 +208645,211998 +211999,211999 +68062,211999 +212000,212000 +212001,212001 +212002,212002 +212003,212003 +212004,212004 +212005,212005 +212006,212006 +76328,212006 +212007,212007 +78575,212007 +212008,212008 +212009,212009 +212010,212010 +212011,212011 +212012,212012 +212013,212013 +212014,212014 +212015,212015 +212016,212016 +212017,212017 +212018,212018 +248711,212018 +212019,212019 +212020,212020 +212021,212021 +212022,212022 +212023,212023 +212024,212024 +212025,212025 +192144,212025 +212026,212026 +212027,212027 +212028,212028 +212029,212029 +212030,212030 +212031,212031 +212032,212032 +212033,212033 +212034,212034 +212035,212035 +212036,212036 +212037,212037 +212038,212038 +89701,212038 +212039,212039 +212303,212039 +212040,212040 +212041,212041 +212042,212042 +212043,212043 +212044,212044 +212045,212045 +212046,212046 +209680,212046 +212047,212047 +212048,212048 +212049,212049 +212050,212050 +212051,212051 +145264,212051 +209685,212051 +212052,212052 +212053,212053 +212054,212054 +212055,212055 +236898,212055 +212056,212056 +212057,212057 +212058,212058 +212059,212059 +212060,212060 +212061,212061 +246655,212061 +212062,212062 +212063,212063 +212064,212064 +212065,212065 +212066,212066 +212067,212067 +212068,212068 +212069,212069 +212070,212070 +212071,212071 +212072,212072 +212073,212073 +212074,212074 +212075,212075 +212076,212076 +212077,212077 +212078,212078 +212079,212079 +212080,212080 +212081,212081 +212082,212082 +212083,212083 +212084,212084 +212085,212085 +212086,212086 +212087,212087 +212088,212088 +212089,212089 +212090,212090 +212091,212091 +212092,212092 +212093,212093 +212094,212094 +212095,212095 +154553,212095 +212096,212096 +212097,212097 +177628,212097 +171664,212097 +212098,212098 +212099,212099 +212100,212100 +212101,212101 +212102,212102 +212103,212103 +212104,212104 +212105,212105 +212106,212106 +212107,212107 +214384,212107 +212108,212108 +212109,212109 +212110,212110 +212111,212111 +212112,212112 +212113,212113 +158221,212113 +212114,212114 +212115,212115 +212116,212116 +29886,212116 +212117,212117 +212118,212118 +5097,212118 +212119,212119 +212120,212120 +194750,212120 +212121,212121 +212122,212122 +212123,212123 +212124,212124 +212125,212125 +212126,212126 +212127,212127 +212128,212128 +212129,212129 +212130,212130 +158804,212130 +212131,212131 +212132,212132 +212133,212133 +212134,212134 +212135,212135 +212136,212136 +212137,212137 +212258,212258 +78379,212258 +212259,212259 +212260,212260 +212261,212261 +212262,212262 +212263,212263 +212264,212264 +212265,212265 +212266,212266 +212267,212267 +203581,212267 +212268,212268 +212269,212269 +212270,212270 +212271,212271 +212272,212272 +212273,212273 +212274,212274 +212275,212275 +212276,212276 +212277,212277 +212278,212278 +212279,212279 +212280,212280 +212281,212281 +212282,212282 +178588,212282 +212283,212283 +212284,212284 +212285,212285 +212286,212286 +212287,212287 +212288,212288 +212289,212289 +212290,212290 +212291,212291 +212292,212292 +212293,212293 +212294,212294 +212295,212295 +212296,212296 +212297,212297 +212298,212298 +212299,212299 +212300,212300 +212301,212301 +60268,212301 +212302,212302 +212303,212303 +212039,212303 +212304,212304 +212305,212305 +212306,212306 +212307,212307 +212308,212308 +212309,212309 +30932,212309 +212310,212310 +212311,212311 +212312,212312 +212313,212313 +212314,212314 +212315,212315 +212316,212316 +212317,212317 +212318,212318 +212319,212319 +212320,212320 +212321,212321 +57510,212321 +212322,212322 +212323,212323 +212324,212324 +212325,212325 +212326,212326 +212327,212327 +212328,212328 +212329,212329 +212330,212330 +212331,212331 +212332,212332 +212333,212333 +212334,212334 +212335,212335 +212336,212336 +212337,212337 +212338,212338 +212339,212339 +212340,212340 +74184,212340 +212341,212341 +212342,212342 +212343,212343 +212344,212344 +212345,212345 +111033,212345 +212346,212346 +212347,212347 +212348,212348 +212349,212349 +212350,212350 +212351,212351 +212352,212352 +212353,212353 +212354,212354 +212355,212355 +212356,212356 +212357,212357 +212358,212358 +212359,212359 +33205,212359 +212397,212397 +212398,212398 +212399,212399 +212754,212754 +65159,212754 +212755,212755 +212756,212756 +212757,212757 +103884,212757 +212758,212758 +164437,212758 +212759,212759 +212760,212760 +212761,212761 +212895,212895 +212896,212896 +212897,212897 +212898,212898 +227652,212898 +212899,212899 +212900,212900 +212901,212901 +212902,212902 +212903,212903 +212904,212904 +212905,212905 +212906,212906 +212907,212907 +212908,212908 +212909,212909 +212910,212910 +212911,212911 +78394,212911 +212912,212912 +212913,212913 +213064,213064 +213065,213065 +213104,213104 +192563,213104 +213105,213105 +213106,213106 +213107,213107 +213108,213108 +213109,213109 +213110,213110 +213111,213111 +213112,213112 +213113,213113 +213114,213114 +213115,213115 +213277,213277 +213278,213278 +213279,213279 +213280,213280 +213281,213281 +213282,213282 +213283,213283 +213284,213284 +213285,213285 +213286,213286 +213287,213287 +213288,213288 +213289,213289 +193154,213289 +213290,213290 +211206,213290 +50743,213290 +42639,213290 +156334,213290 +213291,213291 +213292,213292 +213293,213293 +213294,213294 +213295,213295 +213296,213296 +213297,213297 +213298,213298 +213299,213299 +213300,213300 +213301,213301 +213302,213302 +213303,213303 +213304,213304 +213305,213305 +213306,213306 +213307,213307 +213308,213308 +213309,213309 +213310,213310 +213311,213311 +213312,213312 +213313,213313 +213314,213314 +213315,213315 +213316,213316 +213317,213317 +213318,213318 +213319,213319 +213320,213320 +213321,213321 +213322,213322 +213323,213323 +213324,213324 +190236,213324 +213325,213325 +213326,213326 +213327,213327 +213328,213328 +213329,213329 +213330,213330 +213331,213331 +213332,213332 +213333,213333 +213334,213334 +213335,213335 +213385,213385 +213386,213386 +213387,213387 +213388,213388 +213389,213389 +213390,213390 +213391,213391 +213392,213392 +213393,213393 +213394,213394 +109049,213394 +213395,213395 +213396,213396 +123023,213396 +213397,213397 +213438,213438 +213439,213439 +213440,213440 +213441,213441 +213442,213442 +213443,213443 +6193,213443 +213444,213444 +213445,213445 +213446,213446 +213447,213447 +150863,213447 +213448,213448 +213449,213449 +213450,213450 +213551,213551 +213552,213552 +213553,213553 +213554,213554 +213555,213555 +213556,213556 +213557,213557 +247454,213557 +218661,213557 +213558,213558 +213559,213559 +185096,213559 +213560,213560 +213561,213561 +213562,213562 +213563,213563 +213564,213564 +213565,213565 +77272,213565 +213566,213566 +213567,213567 +213568,213568 +60681,213568 +213569,213569 +213570,213570 +213571,213571 +213572,213572 +213573,213573 +213574,213574 +213575,213575 +213576,213576 +213577,213577 +213578,213578 +213579,213579 +146106,213579 +205635,213579 +213580,213580 +213581,213581 +238930,213582 +213582,213582 +132754,213582 +125167,213582 +213583,213583 +60984,213583 +213584,213584 +213585,213585 +213586,213586 +213587,213587 +213588,213588 +213589,213589 +213590,213590 +213591,213591 +213592,213592 +213593,213593 +213594,213594 +213595,213595 +213596,213596 +213597,213597 +213598,213598 +213599,213599 +213600,213600 +213601,213601 +213602,213602 +213603,213603 +56269,213603 +48165,213603 +213604,213604 +213605,213605 +213606,213606 +213607,213607 +213608,213608 +213609,213609 +142913,213609 +213610,213610 +213611,213611 +213612,213612 +213613,213613 +206824,213613 +194114,213613 +213614,213614 +213615,213615 +213616,213616 +213617,213617 +213618,213618 +213619,213619 +213620,213620 +213621,213621 +213622,213622 +213623,213623 +86393,213623 +213624,213624 +213625,213625 +213626,213626 +213627,213627 +213628,213628 +213629,213629 +213630,213630 +213631,213631 +213632,213632 +213633,213633 +213634,213634 +213635,213635 +213636,213636 +213637,213637 +213638,213638 +213639,213639 +213640,213640 +213641,213641 +213642,213642 +213643,213643 +196680,213643 +213644,213644 +213645,213645 +213646,213646 +213647,213647 +213648,213648 +213649,213649 +213650,213650 +213651,213651 +213652,213652 +213653,213653 +235071,213653 +213654,213654 +213655,213655 +11996,213655 +213656,213656 +213657,213657 +213658,213658 +213659,213659 +213660,213660 +213661,213661 +213662,213662 +213663,213663 +213664,213664 +213665,213665 +213666,213666 +213667,213667 +213668,213668 +213669,213669 +213670,213670 +213671,213671 +213672,213672 +213673,213673 +213674,213674 +146487,213674 +213675,213675 +213676,213676 +213677,213677 +213678,213678 +213679,213679 +213680,213680 +213681,213681 +213682,213682 +213683,213683 +213684,213684 +213685,213685 +213686,213686 +213687,213687 +213688,213688 +213689,213689 +213825,213689 +213690,213690 +213691,213691 +213692,213692 +213693,213693 +77594,213693 +213694,213694 +213695,213695 +213696,213696 +107565,213696 +213697,213697 +213698,213698 +213699,213699 +213700,213700 +200806,213700 +213701,213701 +222274,213701 +213702,213702 +213703,213703 +213704,213704 +213705,213705 +213706,213706 +213707,213707 +213775,213707 +145224,213707 +213708,213708 +213709,213709 +213710,213710 +213711,213711 +213712,213712 +213713,213713 +213714,213714 +213715,213715 +213716,213716 +213717,213717 +213718,213718 +213719,213719 +242544,213719 +213720,213720 +213721,213721 +213722,213722 +213723,213723 +213724,213724 +213725,213725 +213726,213726 +213727,213727 +213728,213728 +213729,213729 +213730,213730 +213731,213731 +213732,213732 +213733,213733 +213734,213734 +213735,213735 +213736,213736 +213737,213737 +213738,213738 +213739,213739 +213740,213740 +213741,213741 +213742,213742 +213743,213743 +213744,213744 +213745,213745 +213746,213746 +193717,213746 +213747,213747 +213748,213748 +213749,213749 +213750,213750 +213751,213751 +213752,213752 +213753,213753 +213754,213754 +213755,213755 +213756,213756 +213757,213757 +213758,213758 +213759,213759 +213760,213760 +213761,213761 +213762,213762 +209927,213762 +213763,213763 +213764,213764 +213765,213765 +213766,213766 +213767,213767 +213768,213768 +213769,213769 +213770,213770 +176557,213770 +170593,213770 +213771,213771 +213772,213772 +213773,213773 +213774,213774 +213775,213775 +213707,213775 +213776,213776 +213777,213777 +213778,213778 +213779,213779 +213780,213780 +213781,213781 +213782,213782 +213783,213783 +213784,213784 +213785,213785 +213786,213786 +213787,213787 +213788,213788 +213789,213789 +213790,213790 +213791,213791 +213792,213792 +213793,213793 +213794,213794 +213795,213795 +213796,213796 +213797,213797 +213798,213798 +213799,213799 +213800,213800 +213801,213801 +213802,213802 +213803,213803 +213804,213804 +213805,213805 +213806,213806 +213807,213807 +213808,213808 +213809,213809 +213810,213810 +213811,213811 +213812,213812 +213813,213813 +213814,213814 +213815,213815 +213816,213816 +213817,213817 +213818,213818 +184141,213818 +213819,213819 +213820,213820 +213821,213821 +213822,213822 +213823,213823 +213824,213824 +213825,213825 +105007,213825 +213689,213825 +213826,213826 +213827,213827 +213828,213828 +213829,213829 +213830,213830 +213831,213831 +213832,213832 +213833,213833 +213834,213834 +213835,213835 +213836,213836 +213837,213837 +213838,213838 +213839,213839 +213840,213840 +213841,213841 +213842,213842 +213843,213843 +213844,213844 +213845,213845 +213846,213846 +213847,213847 +213848,213848 +213849,213849 +213850,213850 +213851,213851 +3164,213851 +213852,213852 +35793,213852 +213853,213853 +213854,213854 +213855,213855 +213856,213856 +213857,213857 +213858,213858 +213859,213859 +213860,213860 +213861,213861 +213862,213862 +215626,213862 +213863,213863 +213864,213864 +213865,213865 +213866,213866 +213867,213867 +213868,213868 +213869,213869 +213870,213870 +53698,213870 +45594,213870 +213871,213871 +213872,213872 +213873,213873 +21830,213873 +213874,213874 +213875,213875 +213876,213876 +213877,213877 +213921,213921 +4826,213921 +213922,213922 +213923,213923 +213924,213924 +213925,213925 +213926,213926 +213927,213927 +213928,213928 +213929,213929 +213930,213930 +213931,213931 +213932,213932 +213933,213933 +213934,213934 +213935,213935 +118200,213935 +213936,213936 +213937,213937 +213938,213938 +213939,213939 +213940,213940 +213941,213941 +213942,213942 +213943,213943 +213944,213944 +213945,213945 +213946,213946 +213947,213947 +213948,213948 +213949,213949 +213950,213950 +213951,213951 +213952,213952 +213953,213953 +165042,213953 +213954,213954 +213955,213955 +213956,213956 +213957,213957 +213958,213958 +213959,213959 +213960,213960 +140649,213960 +213961,213961 +213962,213962 +213963,213963 +213964,213964 +213965,213965 +249988,213965 +213966,213966 +213967,213967 +213968,213968 +213969,213969 +213970,213970 +213971,213971 +213972,213972 +214040,214040 +214041,214041 +214257,214042 +214042,214042 +214043,214043 +214044,214044 +214045,214045 +214046,214046 +214047,214047 +214048,214048 +162312,214048 +214049,214049 +214050,214050 +214051,214051 +214052,214052 +214053,214053 +158625,214053 +214054,214054 +214055,214055 +214056,214056 +214057,214057 +62082,214057 +214058,214058 +214059,214059 +214060,214060 +214061,214061 +214062,214062 +214063,214063 +214064,214064 +214065,214065 +214066,214066 +214067,214067 +214068,214068 +214069,214069 +214070,214070 +214071,214071 +214072,214072 +214073,214073 +214074,214074 +214075,214075 +214076,214076 +158222,214076 +214077,214077 +214078,214078 +214079,214079 +214080,214080 +214081,214081 +214082,214082 +214083,214083 +214084,214084 +214085,214085 +214086,214086 +214087,214087 +214088,214088 +214089,214089 +214090,214090 +106985,214090 +214091,214091 +214092,214092 +214139,214139 +214140,214140 +214141,214141 +214142,214142 +214143,214143 +214144,214144 +214145,214145 +214146,214146 +214147,214147 +214148,214148 +214149,214149 +214150,214150 +214719,214150 +214151,214151 +214152,214152 +214153,214153 +214154,214154 +214155,214155 +214156,214156 +214157,214157 +214158,214158 +214159,214159 +214160,214160 +214161,214161 +214162,214162 +214163,214163 +214164,214164 +214165,214165 +214166,214166 +214167,214167 +214168,214168 +214169,214169 +214170,214170 +214171,214171 +214172,214172 +214173,214173 +214174,214174 +214175,214175 +214176,214176 +214177,214177 +214178,214178 +214179,214179 +214180,214180 +214181,214181 +214182,214182 +214183,214183 +214184,214184 +214185,214185 +214186,214186 +214187,214187 +214188,214188 +214189,214189 +148378,214189 +214190,214190 +55003,214190 +46899,214190 +214191,214191 +223069,214191 +214192,214192 +214193,214193 +214194,214194 +214195,214195 +214196,214196 +214197,214197 +214198,214198 +214199,214199 +214200,214200 +214201,214201 +214202,214202 +214203,214203 +214204,214204 +214205,214205 +214206,214206 +214207,214207 +214208,214208 +214209,214209 +214210,214210 +214211,214211 +214212,214212 +214213,214213 +214214,214214 +214215,214215 +214216,214216 +214217,214217 +214218,214218 +214219,214219 +214220,214220 +214221,214221 +214222,214222 +214223,214223 +214224,214224 +214225,214225 +214226,214226 +214227,214227 +214228,214228 +214229,214229 +214230,214230 +214231,214231 +214232,214232 +214233,214233 +214234,214234 +214235,214235 +214236,214236 +214237,214237 +214238,214238 +214239,214239 +117001,214239 +214240,214240 +214241,214241 +214242,214242 +214243,214243 +39,214243 +214244,214244 +214245,214245 +161794,214245 +214246,214246 +214247,214247 +214248,214248 +214249,214249 +11967,214249 +214250,214250 +214251,214251 +214252,214252 +214253,214253 +214254,214254 +214255,214255 +88112,214255 +214256,214256 +214257,214257 +214042,214257 +214258,214258 +214259,214259 +228048,214259 +214260,214260 +214261,214261 +214262,214262 +214263,214263 +214264,214264 +214265,214265 +214266,214266 +214267,214267 +214268,214268 +214269,214269 +214270,214270 +214271,214271 +214272,214272 +214273,214273 +214274,214274 +214275,214275 +214276,214276 +214277,214277 +214278,214278 +214279,214279 +214280,214280 +214281,214281 +214317,214317 +214318,214318 +214319,214319 +214320,214320 +214321,214321 +214322,214322 +214323,214323 +214324,214324 +214325,214325 +214326,214326 +214327,214327 +214328,214328 +214329,214329 +214330,214330 +214331,214331 +214332,214332 +214333,214333 +214334,214334 +214335,214335 +214336,214336 +214337,214337 +214338,214338 +214339,214339 +214340,214340 +214341,214341 +214342,214342 +214343,214343 +214344,214344 +214345,214345 +214346,214346 +214347,214347 +214348,214348 +214349,214349 +214350,214350 +214351,214351 +214352,214352 +214353,214353 +214354,214354 +214355,214355 +214356,214356 +214357,214357 +214358,214358 +214359,214359 +214360,214360 +214361,214361 +214362,214362 +214363,214363 +214364,214364 +214365,214365 +214366,214366 +214367,214367 +214368,214368 +214369,214369 +214370,214370 +214371,214371 +214372,214372 +214373,214373 +214374,214374 +214375,214375 +214376,214376 +214377,214377 +214378,214378 +214379,214379 +214380,214380 +214381,214381 +214382,214382 +214383,214383 +102391,214383 +214384,214384 +212107,214384 +214385,214385 +214386,214386 +214387,214387 +214388,214388 +214389,214389 +214390,214390 +214391,214391 +214392,214392 +214393,214393 +214394,214394 +214395,214395 +214396,214396 +214397,214397 +61015,214397 +214398,214398 +214399,214399 +214400,214400 +214401,214401 +214402,214402 +214403,214403 +214404,214404 +214405,214405 +214406,214406 +120060,214406 +214407,214407 +5263,214407 +214408,214408 +214409,214409 +107969,214409 +34,214409 +214410,214410 +214411,214411 +214412,214412 +214413,214413 +214414,214414 +214415,214415 +214416,214416 +214417,214417 +214418,214418 +214419,214419 +214420,214420 +214421,214421 +214422,214422 +214423,214423 +13662,214423 +214424,214424 +214425,214425 +89197,214425 +214426,214426 +214427,214427 +214428,214428 +214429,214429 +214430,214430 +232972,214430 +214431,214431 +214432,214432 +70201,214432 +214433,214433 +214434,214434 +214435,214435 +214436,214436 +214437,214437 +214438,214438 +185566,214438 +214439,214439 +214440,214440 +214441,214441 +214442,214442 +180505,214442 +214443,214443 +214444,214444 +214445,214445 +214446,214446 +151121,214446 +214447,214447 +214448,214448 +214449,214449 +205949,214449 +214450,214450 +57611,214450 +214451,214451 +81475,214451 +214452,214452 +214453,214453 +214454,214454 +211946,214454 +214455,214455 +214456,214456 +214457,214457 +214458,214458 +214459,214459 +136995,214459 +129408,214459 +158758,214459 +214460,214460 +88161,214460 +214461,214461 +147044,214461 +214462,214462 +214463,214463 +214464,214464 +214465,214465 +214466,214466 +214467,214467 +214468,214468 +214469,214469 +214470,214470 +214471,214471 +214472,214472 +214473,214473 +214474,214474 +214475,214475 +214476,214476 +190625,214476 +214477,214477 +157838,214477 +214478,214478 +214479,214479 +214480,214480 +214481,214481 +214482,214482 +214483,214483 +214484,214484 +214485,214485 +214486,214486 +214487,214487 +98063,214487 +214488,214488 +214489,214489 +214490,214490 +214491,214491 +214492,214492 +214493,214493 +214494,214494 +214495,214495 +146714,214495 +214496,214496 +214497,214497 +214498,214498 +214499,214499 +214500,214500 +214501,214501 +214502,214502 +214503,214503 +214504,214504 +214505,214505 +214506,214506 +214507,214507 +108983,214507 +214508,214508 +214509,214509 +214510,214510 +214511,214511 +214512,214512 +214513,214513 +214514,214514 +214515,214515 +214516,214516 +214517,214517 +214518,214518 +190508,214518 +214519,214519 +214520,214520 +214521,214521 +214522,214522 +214523,214523 +214524,214524 +214525,214525 +117449,214525 +214526,214526 +214527,214527 +214528,214528 +214529,214529 +214530,214530 +214531,214531 +54621,214531 +46517,214531 +214532,214532 +214533,214533 +214534,214534 +214535,214535 +214536,214536 +214537,214537 +214538,214538 +214539,214539 +214540,214540 +214541,214541 +214542,214542 +214543,214543 +214544,214544 +214545,214545 +214546,214546 +210304,214546 +214547,214547 +214548,214548 +214549,214549 +214550,214550 +214551,214551 +214552,214552 +140486,214552 +214553,214553 +214554,214554 +214555,214555 +214556,214556 +214557,214557 +214558,214558 +96445,214558 +214559,214559 +214560,214560 +214561,214561 +214562,214562 +214563,214563 +214564,214564 +214565,214565 +214566,214566 +214567,214567 +214568,214568 +197685,214568 +214569,214569 +214570,214570 +214571,214571 +214572,214572 +122811,214572 +214573,214573 +214574,214574 +214575,214575 +190602,214575 +214576,214576 +214577,214577 +214578,214578 +214579,214579 +214580,214580 +214625,214625 +214626,214626 +214627,214627 +214628,214628 +214629,214629 +214630,214630 +233239,214630 +214631,214631 +214632,214632 +214633,214633 +214634,214634 +214635,214635 +214636,214636 +214637,214637 +214638,214638 +214639,214639 +214640,214640 +214641,214641 +214642,214642 +214643,214643 +214644,214644 +214645,214645 +243981,214645 +214646,214646 +214647,214647 +214648,214648 +214649,214649 +214650,214650 +214651,214651 +214652,214652 +214653,214653 +214654,214654 +214655,214655 +214656,214656 +214657,214657 +214658,214658 +214659,214659 +214660,214660 +214661,214661 +214662,214662 +214663,214663 +214664,214664 +214665,214665 +214666,214666 +214667,214667 +214668,214668 +214669,214669 +214670,214670 +214671,214671 +132944,214671 +125357,214671 +214672,214672 +214673,214673 +214674,214674 +214675,214675 +214676,214676 +141358,214676 +214677,214677 +116358,214677 +214678,214678 +214679,214679 +214680,214680 +214681,214681 +214682,214682 +214683,214683 +214684,214684 +214685,214685 +149342,214685 +214686,214686 +214687,214687 +245856,214687 +241864,214687 +214688,214688 +214689,214689 +214690,214690 +214691,214691 +214692,214692 +60007,214692 +214693,214693 +214694,214694 +214695,214695 +67625,214695 +214696,214696 +150593,214696 +214697,214697 +214698,214698 +201468,214698 +214699,214699 +214700,214700 +214701,214701 +214702,214702 +214703,214703 +214704,214704 +214705,214705 +214706,214706 +214707,214707 +214708,214708 +214709,214709 +214710,214710 +214711,214711 +149314,214711 +214712,214712 +214713,214713 +214714,214714 +214715,214715 +214716,214716 +214717,214717 +214718,214718 +214719,214719 +214150,214719 +214720,214720 +33817,214720 +214721,214721 +214722,214722 +191894,214722 +191794,214722 +214723,214723 +214724,214724 +214725,214725 +214726,214726 +214727,214727 +86868,214727 +214728,214728 +214729,214729 +214730,214730 +214731,214731 +214732,214732 +214733,214733 +214734,214734 +214735,214735 +214736,214736 +214737,214737 +214738,214738 +214739,214739 +214740,214740 +214741,214741 +214742,214742 +214743,214743 +214744,214744 +214745,214745 +138150,214745 +130563,214745 +214746,214746 +214747,214747 +214788,214788 +214789,214789 +214790,214790 +214791,214791 +214792,214792 +214793,214793 +214794,214794 +214795,214795 +214796,214796 +214797,214797 +214798,214798 +214799,214799 +214800,214800 +214801,214801 +214802,214802 +214803,214803 +214804,214804 +214805,214805 +214806,214806 +214807,214807 +214808,214808 +214809,214809 +214810,214810 +104416,214810 +214811,214811 +214812,214812 +214813,214813 +214814,214814 +152792,214814 +214815,214815 +214816,214816 +214817,214817 +214818,214818 +111627,214818 +214819,214819 +214820,214820 +214821,214821 +214822,214822 +214823,214823 +214824,214824 +214825,214825 +214826,214826 +214827,214827 +214828,214828 +214829,214829 +207933,214829 +214830,214830 +103401,214830 +214970,214970 +214971,214971 +214972,214972 +214973,214973 +214974,214974 +215344,215344 +215345,215345 +215346,215346 +208435,215346 +215347,215347 +215348,215348 +215349,215349 +215350,215350 +215351,215351 +215352,215352 +108552,215352 +215353,215353 +215354,215354 +215355,215355 +215356,215356 +215357,215357 +215358,215358 +215359,215359 +215360,215360 +215361,215361 +215362,215362 +215363,215363 +215364,215364 +215365,215365 +215366,215366 +215367,215367 +215368,215368 +215369,215369 +215370,215370 +84660,215370 +215371,215371 +215372,215372 +215373,215373 +215374,215374 +215375,215375 +215376,215376 +215377,215377 +215378,215378 +215379,215379 +85867,215379 +215380,215380 +215381,215381 +215382,215382 +215383,215383 +215384,215384 +215385,215385 +215386,215386 +215387,215387 +215388,215388 +145351,215388 +215389,215389 +215390,215390 +215391,215391 +215392,215392 +215393,215393 +215394,215394 +215395,215395 +215396,215396 +215397,215397 +215398,215398 +215399,215399 +5742,215399 +114358,215399 +215400,215400 +215401,215401 +215402,215402 +215403,215403 +215404,215404 +215405,215405 +215406,215406 +215407,215407 +215408,215408 +215409,215409 +215410,215410 +215411,215411 +215412,215412 +215413,215413 +215414,215414 +244067,215414 +215415,215415 +215416,215416 +215417,215417 +137012,215417 +129425,215417 +215418,215418 +83403,215418 +215419,215419 +215420,215420 +215421,215421 +215422,215422 +215423,215423 +215424,215424 +215425,215425 +215426,215426 +215427,215427 +215428,215428 +215429,215429 +215430,215430 +215431,215431 +215432,215432 +215433,215433 +215434,215434 +215435,215435 +215436,215436 +215437,215437 +215438,215438 +215439,215439 +215440,215440 +215441,215441 +215442,215442 +215443,215443 +215444,215444 +215445,215445 +215446,215446 +215447,215447 +215448,215448 +215449,215449 +215450,215450 +215451,215451 +215452,215452 +215453,215453 +215454,215454 +215455,215455 +215456,215456 +215457,215457 +215458,215458 +215459,215459 +215460,215460 +215461,215461 +215462,215462 +215463,215463 +215464,215464 +215465,215465 +215466,215466 +215467,215467 +215468,215468 +215469,215469 +215470,215470 +215471,215471 +215472,215472 +215473,215473 +215474,215474 +215475,215475 +215476,215476 +215477,215477 +165020,215477 +20665,215477 +20493,215477 +215478,215478 +215479,215479 +215480,215480 +215481,215481 +215482,215482 +215483,215483 +215484,215484 +215485,215485 +215486,215486 +215487,215487 +215488,215488 +215489,215489 +215490,215490 +215491,215491 +215492,215492 +215493,215493 +215494,215494 +215495,215495 +215496,215496 +215497,215497 +215498,215498 +215499,215499 +191750,215499 +215500,215500 +14437,215500 +215501,215501 +215502,215502 +215503,215503 +215504,215504 +215505,215505 +215506,215506 +215507,215507 +26241,215507 +215508,215508 +215509,215509 +215510,215510 +215511,215511 +215512,215512 +215513,215513 +215514,215514 +180082,215514 +215515,215515 +215516,215516 +215517,215517 +215518,215518 +215519,215519 +215520,215520 +215521,215521 +215522,215522 +215523,215523 +215524,215524 +215525,215525 +215526,215526 +159408,215526 +215527,215527 +233364,215527 +211981,215527 +215528,215528 +215529,215529 +215530,215530 +215531,215531 +215532,215532 +215533,215533 +215534,215534 +215535,215535 +215536,215536 +215537,215537 +215538,215538 +215539,215539 +215540,215540 +215541,215541 +215542,215542 +215543,215543 +215544,215544 +215545,215545 +215546,215546 +215547,215547 +215548,215548 +215549,215549 +215550,215550 +247032,215550 +215551,215551 +215552,215552 +215553,215553 +215554,215554 +215595,215595 +215596,215596 +215597,215597 +215598,215598 +215599,215599 +215600,215600 +56360,215600 +48256,215600 +223169,215600 +215601,215601 +215602,215602 +215603,215603 +215604,215604 +197761,215604 +56705,215604 +215605,215605 +185403,215605 +232906,215605 +215606,215606 +215607,215607 +215608,215608 +215609,215609 +215610,215610 +215611,215611 +215612,215612 +215613,215613 +215614,215614 +215615,215615 +215616,215616 +215617,215617 +215618,215618 +215619,215619 +215620,215620 +215621,215621 +215622,215622 +215623,215623 +215624,215624 +215625,215625 +215626,215626 +213862,215626 +215627,215627 +215628,215628 +215629,215629 +215630,215630 +215631,215631 +215632,215632 +215633,215633 +215634,215634 +215635,215635 +215636,215636 +215637,215637 +215638,215638 +6629,215638 +140314,215638 +215639,215639 +215640,215640 +215641,215641 +215642,215642 +215643,215643 +215644,215644 +215645,215645 +215646,215646 +215647,215647 +215648,215648 +131916,215648 +124329,215648 +215649,215649 +215650,215650 +215651,215651 +215652,215652 +215653,215653 +215654,215654 +215655,215655 +215656,215656 +215657,215657 +215658,215658 +215659,215659 +215893,215893 +183967,215893 +147543,215893 +215894,215894 +215895,215895 +215896,215896 +215897,215897 +215898,215898 +215899,215899 +215900,215900 +215901,215901 +215902,215902 +215903,215903 +215904,215904 +215905,215905 +215906,215906 +215907,215907 +215908,215908 +215909,215909 +215910,215910 +108540,215910 +215911,215911 +215912,215912 +215913,215913 +215914,215914 +215915,215915 +215916,215916 +215917,215917 +215918,215918 +215919,215919 +215920,215920 +215921,215921 +215922,215922 +215923,215923 +215924,215924 +215925,215925 +215926,215926 +215927,215927 +215928,215928 +215929,215929 +215930,215930 +215931,215931 +215932,215932 +24483,215932 +215933,215933 +215934,215934 +114441,215934 +215935,215935 +215936,215936 +215937,215937 +215938,215938 +108880,215938 +215939,215939 +215940,215940 +215941,215941 +215942,215942 +215943,215943 +215944,215944 +158463,215944 +215945,215945 +215946,215946 +215947,215947 +215948,215948 +215949,215949 +247414,215950 +215950,215950 +215951,215951 +215952,215952 +215953,215953 +215954,215954 +215955,215955 +215956,215956 +215957,215957 +215958,215958 +215959,215959 +215960,215960 +215961,215961 +215962,215962 +215963,215963 +215964,215964 +215965,215965 +215966,215966 +215967,215967 +215968,215968 +215969,215969 +215970,215970 +215971,215971 +215972,215972 +215973,215973 +215974,215974 +215975,215975 +215976,215976 +215977,215977 +215978,215978 +215979,215979 +215980,215980 +215981,215981 +215982,215982 +215983,215983 +215984,215984 +215985,215985 +215986,215986 +111808,215986 +215987,215987 +215988,215988 +215989,215989 +215990,215990 +215991,215991 +215992,215992 +24440,215992 +215993,215993 +215994,215994 +215995,215995 +215996,215996 +215997,215997 +215998,215998 +215999,215999 +216000,216000 +216001,216001 +216002,216002 +216003,216003 +216004,216004 +216005,216005 +216006,216006 +216007,216007 +216008,216008 +216009,216009 +216010,216010 +216011,216011 +216012,216012 +216013,216013 +210195,216013 +216014,216014 +216015,216015 +216016,216016 +216017,216017 +216018,216018 +216019,216019 +216020,216020 +216021,216021 +216022,216022 +216023,216023 +5274,216023 +216024,216024 +216025,216025 +216026,216026 +216027,216027 +216028,216028 +216029,216029 +216030,216030 +216031,216031 +216032,216032 +216033,216033 +199560,216033 +216034,216034 +216035,216035 +216036,216036 +232018,216036 +216037,216037 +216038,216038 +216039,216039 +216040,216040 +216041,216041 +216042,216042 +216043,216043 +226751,216043 +216044,216044 +216045,216045 +216046,216046 +216047,216047 +216048,216048 +216049,216049 +216050,216050 +216051,216051 +216052,216052 +216053,216053 +216054,216054 +216055,216055 +216056,216056 +216057,216057 +216058,216058 +216059,216059 +216060,216060 +216061,216061 +216062,216062 +216063,216063 +216064,216064 +216065,216065 +216066,216066 +216067,216067 +216068,216068 +142618,216068 +216069,216069 +216070,216070 +216071,216071 +216072,216072 +216073,216073 +216074,216074 +216075,216075 +216076,216076 +216077,216077 +216078,216078 +216079,216079 +216080,216080 +216081,216081 +216082,216082 +216083,216083 +216084,216084 +216085,216085 +216086,216086 +216087,216087 +62496,216087 +216088,216088 +216089,216089 +216090,216090 +216091,216091 +136381,216091 +128794,216091 +150684,216091 +216092,216092 +216093,216093 +216094,216094 +216095,216095 +216096,216096 +216097,216097 +216098,216098 +216099,216099 +216100,216100 +216101,216101 +216102,216102 +216103,216103 +216104,216104 +216105,216105 +216106,216106 +216107,216107 +216108,216108 +216109,216109 +216110,216110 +216111,216111 +216112,216112 +216113,216113 +216114,216114 +216115,216115 +216116,216116 +216117,216117 +216118,216118 +216119,216119 +216120,216120 +216121,216121 +216122,216122 +216123,216123 +216124,216124 +216125,216125 +216126,216126 +216127,216127 +216128,216128 +248142,216128 +216129,216129 +216130,216130 +216131,216131 +30882,216131 +216132,216132 +216133,216133 +216134,216134 +216135,216135 +216136,216136 +216137,216137 +216138,216138 +216139,216139 +216140,216140 +216141,216141 +216142,216142 +140625,216142 +216143,216143 +216144,216144 +216145,216145 +216146,216146 +216147,216147 +216148,216148 +216149,216149 +216150,216150 +216151,216151 +216152,216152 +76009,216152 +216153,216153 +216154,216154 +216155,216155 +216156,216156 +216157,216157 +216158,216158 +216159,216159 +216160,216160 +216161,216161 +164510,216161 +216162,216162 +216163,216163 +216164,216164 +216165,216165 +216166,216166 +216167,216167 +216168,216168 +216169,216169 +216170,216170 +162762,216170 +216171,216171 +216172,216172 +216173,216173 +216174,216174 +216175,216175 +216176,216176 +216177,216177 +216178,216178 +185054,216178 +216179,216179 +216180,216180 +216181,216181 +216182,216182 +216183,216183 +216184,216184 +216185,216185 +216186,216186 +216187,216187 +216188,216188 +216189,216189 +216190,216190 +216191,216191 +216192,216192 +216193,216193 +216194,216194 +216195,216195 +216196,216196 +216197,216197 +216198,216198 +216199,216199 +216200,216200 +216201,216201 +216202,216202 +216203,216203 +216204,216204 +216205,216205 +216206,216206 +216207,216207 +190647,216207 +80596,216207 +216208,216208 +216209,216209 +216210,216210 +216211,216211 +216212,216212 +14628,216212 +216213,216213 +216214,216214 +216215,216215 +216216,216216 +216217,216217 +216218,216218 +216219,216219 +216220,216220 +216221,216221 +123178,216221 +216222,216222 +216223,216223 +216224,216224 +216225,216225 +216226,216226 +202884,216226 +216227,216227 +216228,216228 +216229,216229 +216230,216230 +211598,216230 +216231,216231 +216232,216232 +216233,216233 +216234,216234 +216235,216235 +207588,216235 +216236,216236 +216237,216237 +216238,216238 +216239,216239 +216240,216240 +216241,216241 +216242,216242 +6287,216242 +216243,216243 +216244,216244 +216245,216245 +216246,216246 +216247,216247 +216248,216248 +216249,216249 +75735,216249 +216250,216250 +216251,216251 +216252,216252 +216253,216253 +216254,216254 +216255,216255 +216256,216256 +216257,216257 +207599,216257 +216258,216258 +216259,216259 +216260,216260 +216261,216261 +2783,216261 +216262,216262 +216263,216263 +216264,216264 +216265,216265 +4943,216265 +216266,216266 +216267,216267 +216268,216268 +216269,216269 +216270,216270 +207130,216270 +216271,216271 +216272,216272 +216273,216273 +216274,216274 +216275,216275 +216276,216276 +147015,216276 +216277,216277 +194228,216277 +216278,216278 +216279,216279 +216280,216280 +216281,216281 +216282,216282 +216283,216283 +216284,216284 +216285,216285 +53415,216285 +45311,216285 +216286,216286 +216287,216287 +216288,216288 +123938,216288 +216289,216289 +216290,216290 +216291,216291 +216292,216292 +216293,216293 +216294,216294 +216295,216295 +216296,216296 +191603,216296 +216297,216297 +216298,216298 +216299,216299 +216300,216300 +216301,216301 +216302,216302 +191830,216302 +191609,216302 +216303,216303 +216304,216304 +191561,216304 +162521,216304 +216343,216343 +216344,216344 +216345,216345 +216346,216346 +216347,216347 +216348,216348 +216349,216349 +35479,216349 +216350,216350 +189936,216350 +216351,216351 +216352,216352 +216353,216353 +216354,216354 +216355,216355 +216356,216356 +216357,216357 +216358,216358 +216359,216359 +216360,216360 +189865,216360 +216361,216361 +216362,216362 +216363,216363 +216364,216364 +216365,216365 +216366,216366 +216367,216367 +216368,216368 +216369,216369 +216370,216370 +216371,216371 +216372,216372 +216373,216373 +216374,216374 +216375,216375 +216376,216376 +216377,216377 +216378,216378 +216379,216379 +216380,216380 +216577,216380 +216381,216381 +216382,216382 +216383,216383 +216384,216384 +216385,216385 +216386,216386 +216387,216387 +216388,216388 +216389,216389 +216390,216390 +216391,216391 +216392,216392 +216393,216393 +216394,216394 +216395,216395 +216396,216396 +216397,216397 +216398,216398 +79865,216398 +116346,216398 +216399,216399 +211265,216399 +216400,216400 +216401,216401 +216402,216402 +216403,216403 +243599,216403 +216404,216404 +216405,216405 +216406,216406 +216407,216407 +216408,216408 +216409,216409 +210330,216409 +216410,216410 +216411,216411 +216412,216412 +216413,216413 +216414,216414 +231942,216415 +216415,216415 +216416,216416 +216417,216417 +216418,216418 +216419,216419 +216420,216420 +216421,216421 +216422,216422 +216423,216423 +216424,216424 +103961,216424 +216425,216425 +216426,216426 +216427,216427 +216428,216428 +202613,216428 +216429,216429 +76120,216429 +216430,216430 +216431,216431 +216432,216432 +216433,216433 +216434,216434 +216435,216435 +216436,216436 +216437,216437 +216438,216438 +242888,216438 +216439,216439 +216440,216440 +216441,216441 +216442,216442 +216443,216443 +216444,216444 +216445,216445 +216446,216446 +216447,216447 +216484,216484 +216485,216485 +216486,216486 +216487,216487 +216488,216488 +216489,216489 +178085,216489 +172121,216489 +216490,216490 +216491,216491 +216492,216492 +216493,216493 +216494,216494 +216495,216495 +216496,216496 +216497,216497 +141644,216497 +216498,216498 +216499,216499 +216500,216500 +216501,216501 +216502,216502 +140230,216502 +216503,216503 +216504,216504 +216505,216505 +216506,216506 +216507,216507 +216508,216508 +216509,216509 +216510,216510 +216511,216511 +216512,216512 +216513,216513 +216514,216514 +216515,216515 +216516,216516 +177715,216516 +171751,216516 +216517,216517 +216518,216518 +216519,216519 +216520,216520 +216521,216521 +216522,216522 +190222,216522 +216523,216523 +216524,216524 +216525,216525 +216526,216526 +237608,216526 +216527,216527 +224453,216527 +216528,216528 +216529,216529 +207875,216529 +216530,216530 +216531,216531 +216532,216532 +216533,216533 +216534,216534 +216535,216535 +216536,216536 +216537,216537 +216538,216538 +190228,216538 +216539,216539 +216540,216540 +216541,216541 +216542,216542 +216543,216543 +159462,216543 +216544,216544 +216545,216545 +216546,216546 +216547,216547 +143063,216547 +216548,216548 +216549,216549 +216550,216550 +84092,216550 +216551,216551 +216552,216552 +216553,216553 +216554,216554 +216555,216555 +216556,216556 +216557,216557 +216558,216558 +216559,216559 +38752,216559 +216560,216560 +216561,216561 +84024,216561 +216562,216562 +216563,216563 +216564,216564 +216565,216565 +216566,216566 +216567,216567 +216568,216568 +216569,216569 +216570,216570 +216571,216571 +216572,216572 +216573,216573 +216574,216574 +216575,216575 +216576,216576 +216577,216577 +216380,216577 +216578,216578 +216579,216579 +216580,216580 +216581,216581 +216582,216582 +216583,216583 +153325,216583 +216584,216584 +216585,216585 +216586,216586 +216587,216587 +2795,216587 +216588,216588 +216589,216589 +216590,216590 +216591,216591 +216592,216592 +216593,216593 +216594,216594 +216595,216595 +216596,216596 +216597,216597 +216598,216598 +216599,216599 +216600,216600 +216601,216601 +163975,216601 +216602,216602 +87579,216602 +216603,216603 +216604,216604 +160726,216604 +216605,216605 +216606,216606 +216607,216607 +216608,216608 +216609,216609 +216610,216610 +216611,216611 +216612,216612 +216613,216613 +216614,216614 +180845,216614 +216615,216615 +216616,216616 +216617,216617 +216618,216618 +216619,216619 +216620,216620 +216621,216621 +216622,216622 +216623,216623 +216624,216624 +216625,216625 +216626,216626 +216627,216627 +216628,216628 +216629,216629 +216630,216630 +216631,216631 +216632,216632 +216633,216633 +216634,216634 +216635,216635 +216636,216636 +216637,216637 +216638,216638 +216639,216639 +216640,216640 +216641,216641 +216642,216642 +216643,216643 +216644,216644 +216645,216645 +216646,216646 +216647,216647 +216648,216648 +216649,216649 +216650,216650 +216651,216651 +216652,216652 +216653,216653 +216654,216654 +100934,216654 +216655,216655 +216656,216656 +216657,216657 +216658,216658 +216659,216659 +216660,216660 +216661,216661 +16827,216661 +216662,216662 +149258,216662 +216663,216663 +216664,216664 +216665,216665 +216666,216666 +216667,216667 +216668,216668 +227072,216668 +216669,216669 +216670,216670 +216671,216671 +209957,216671 +216672,216672 +216673,216673 +216674,216674 +216675,216675 +216676,216676 +216677,216677 +216678,216678 +220101,216678 +216679,216679 +216680,216680 +216681,216681 +70156,216681 +216682,216682 +216683,216683 +216684,216684 +216685,216685 +216686,216686 +216687,216687 +216688,216688 +216689,216689 +216690,216690 +216691,216691 +191505,216691 +86684,216691 +216692,216692 +216693,216693 +216694,216694 +216695,216695 +216696,216696 +216697,216697 +216698,216698 +216699,216699 +216700,216700 +216701,216701 +216702,216702 +216703,216703 +216704,216704 +216705,216705 +216706,216706 +216707,216707 +216708,216708 +216709,216709 +216710,216710 +216711,216711 +216712,216712 +216713,216713 +216714,216714 +216715,216715 +216716,216716 +216717,216717 +216718,216718 +216719,216719 +216720,216720 +216721,216721 +216722,216722 +216723,216723 +216724,216724 +216725,216725 +216726,216726 +216727,216727 +216728,216728 +216729,216729 +216730,216730 +216731,216731 +216732,216732 +83778,216732 +216733,216733 +216734,216734 +216735,216735 +216736,216736 +216737,216737 +216738,216738 +216739,216739 +216740,216740 +216741,216741 +216742,216742 +216743,216743 +202802,216743 +216744,216744 +216745,216745 +216746,216746 +216747,216747 +216748,216748 +216749,216749 +216750,216750 +216751,216751 +216752,216752 +216753,216753 +216754,216754 +216755,216755 +216756,216756 +97748,216756 +216757,216757 +216758,216758 +216759,216759 +216760,216760 +216761,216761 +216762,216762 +216763,216763 +216764,216764 +216765,216765 +216766,216766 +216767,216767 +216768,216768 +216769,216769 +216770,216770 +185382,216770 +216771,216771 +216772,216772 +216773,216773 +216774,216774 +216775,216775 +216776,216776 +216777,216777 +92598,216777 +216778,216778 +216779,216779 +216780,216780 +216781,216781 +216782,216782 +216783,216783 +216784,216784 +216785,216785 +216786,216786 +216787,216787 +216788,216788 +216789,216789 +216790,216790 +216791,216791 +216792,216792 +216793,216793 +216794,216794 +216795,216795 +216796,216796 +194,216796 +216797,216797 +216798,216798 +216799,216799 +216800,216800 +216801,216801 +216802,216802 +216803,216803 +216804,216804 +216805,216805 +216806,216806 +216807,216807 +216808,216808 +216809,216809 +28284,216809 +216810,216810 +216811,216811 +216812,216812 +216813,216813 +79243,216813 +231449,216813 +216814,216814 +208067,216814 +216815,216815 +216816,216816 +216817,216817 +216818,216818 +216819,216819 +216820,216820 +216821,216821 +216822,216822 +216823,216823 +216860,216860 +216861,216861 +216862,216862 +216863,216863 +216864,216864 +77401,216864 +216865,216865 +12641,216865 +216866,216866 +216867,216867 +216868,216868 +216869,216869 +216870,216870 +216871,216871 +240180,216871 +216872,216872 +216873,216873 +216874,216874 +216875,216875 +216916,216916 +216917,216917 +216918,216918 +216919,216919 +216920,216920 +216921,216921 +216922,216922 +216923,216923 +216924,216924 +216925,216925 +216926,216926 +216927,216927 +216928,216928 +216929,216929 +216930,216930 +216931,216931 +216932,216932 +216933,216933 +216934,216934 +216935,216935 +216936,216936 +216937,216937 +216938,216938 +216939,216939 +216940,216940 +216941,216941 +216942,216942 +216943,216943 +216944,216944 +216945,216945 +216946,216946 +216947,216947 +216948,216948 +216949,216949 +216950,216950 +216951,216951 +216952,216952 +216953,216953 +216954,216954 +216955,216955 +216956,216956 +216957,216957 +216958,216958 +216959,216959 +216960,216960 +216961,216961 +216962,216962 +216963,216963 +216964,216964 +216965,216965 +173971,216965 +168007,216965 +216966,216966 +216967,216967 +216968,216968 +216969,216969 +216970,216970 +216971,216971 +216972,216972 +216973,216973 +216974,216974 +216975,216975 +216976,216976 +216977,216977 +216978,216978 +216979,216979 +216980,216980 +216981,216981 +216982,216982 +216983,216983 +216984,216984 +216985,216985 +216986,216986 +216987,216987 +216988,216988 +97209,216988 +216989,216989 +216990,216990 +216991,216991 +216992,216992 +216993,216993 +216994,216994 +216995,216995 +216996,216996 +216997,216997 +216998,216998 +87773,216998 +216999,216999 +217000,217000 +217001,217001 +217002,217002 +217003,217003 +217004,217004 +217005,217005 +217006,217006 +217007,217007 +217008,217008 +217009,217009 +184561,217009 +233270,217009 +217010,217010 +217011,217011 +217012,217012 +217013,217013 +217014,217014 +217015,217015 +217016,217016 +217017,217017 +217018,217018 +217019,217019 +217020,217020 +217021,217021 +217022,217022 +217023,217023 +217024,217024 +217025,217025 +217026,217026 +217027,217027 +80475,217027 +217028,217028 +217029,217029 +217030,217030 +33996,217030 +217031,217031 +217032,217032 +217033,217033 +51659,217033 +43555,217033 +217034,217034 +217035,217035 +217036,217036 +217037,217037 +217038,217038 +217039,217039 +180486,217039 +217040,217040 +217041,217041 +136038,217041 +128451,217041 +68193,217041 +217083,217083 +217084,217084 +217085,217085 +217086,217086 +217087,217087 +234918,217087 +217088,217088 +217089,217089 +217090,217090 +217091,217091 +217092,217092 +217093,217093 +217094,217094 +217095,217095 +8566,217095 +8386,217095 +217096,217096 +217097,217097 +217098,217098 +217099,217099 +217100,217100 +217101,217101 +217102,217102 +217103,217103 +217104,217104 +217105,217105 +217106,217106 +217107,217107 +217108,217108 +217109,217109 +217110,217110 +217111,217111 +217112,217112 +217113,217113 +217156,217156 +217157,217157 +217158,217158 +217159,217159 +217160,217160 +217161,217161 +217162,217162 +217163,217163 +217164,217164 +217165,217165 +217166,217166 +217167,217167 +217168,217168 +217169,217169 +217170,217170 +217171,217171 +217172,217172 +217173,217173 +217174,217174 +4804,217174 +217175,217175 +217176,217176 +217177,217177 +217178,217178 +217179,217179 +217180,217180 +217181,217181 +217182,217182 +217183,217183 +217184,217184 +217185,217185 +161767,217185 +217228,217228 +247893,217228 +217229,217229 +217230,217230 +217231,217231 +217232,217232 +217233,217233 +217234,217234 +217235,217235 +217236,217236 +217237,217237 +87596,217237 +217238,217238 +217239,217239 +217240,217240 +217241,217241 +217242,217242 +217243,217243 +217244,217244 +217245,217245 +217246,217246 +217247,217247 +217248,217248 +217249,217249 +217250,217250 +162438,217250 +217251,217251 +217252,217252 +217253,217253 +217254,217254 +134254,217254 +126667,217254 +217255,217255 +217256,217256 +217257,217257 +217258,217258 +217259,217259 +217260,217260 +217261,217261 +217262,217262 +217263,217263 +217264,217264 +217265,217265 +217266,217266 +217267,217267 +217268,217268 +217269,217269 +217270,217270 +217271,217271 +217272,217272 +217273,217273 +217274,217274 +217275,217275 +217276,217276 +217277,217277 +217278,217278 +217279,217279 +217280,217280 +217281,217281 +217282,217282 +217283,217283 +217284,217284 +217285,217285 +217286,217286 +217287,217287 +217288,217288 +217289,217289 +217290,217290 +231639,217290 +217291,217291 +217292,217292 +217293,217293 +217294,217294 +217472,217472 +217473,217473 +217474,217474 +217475,217475 +7738,217475 +217476,217476 +217477,217477 +217478,217478 +217479,217479 +217480,217480 +217481,217481 +52959,217481 +44855,217481 +217482,217482 +190019,217482 +217483,217483 +217484,217484 +217485,217485 +189416,217485 +217486,217486 +217487,217487 +217488,217488 +217489,217489 +217490,217490 +217491,217491 +217492,217492 +217493,217493 +217494,217494 +217495,217495 +217496,217496 +217497,217497 +217559,217559 +217560,217560 +217561,217561 +9093,217561 +217562,217562 +217563,217563 +217564,217564 +217565,217565 +217566,217566 +217567,217567 +217568,217568 +159255,217568 +217569,217569 +217570,217570 +217571,217571 +217572,217572 +217573,217573 +49809,217573 +41616,217573 +217574,217574 +217575,217575 +217576,217576 +217577,217577 +217578,217578 +217579,217579 +217580,217580 +82176,217580 +217581,217581 +217582,217582 +217583,217583 +217584,217584 +217585,217585 +217586,217586 +217587,217587 +217588,217588 +217589,217589 +217590,217590 +144218,217590 +217591,217591 +217592,217592 +195563,217592 +217593,217593 +217594,217594 +217595,217595 +217596,217596 +217597,217597 +241393,217598 +217598,217598 +217599,217599 +217600,217600 +217601,217601 +217602,217602 +217603,217603 +217682,217682 +217683,217683 +217684,217684 +217685,217685 +217686,217686 +86716,217686 +217687,217687 +217688,217688 +217689,217689 +217690,217690 +11595,217690 +217691,217691 +217692,217692 +217693,217693 +217694,217694 +136380,217694 +128793,217694 +217695,217695 +217696,217696 +217697,217697 +217698,217698 +217699,217699 +217700,217700 +217701,217701 +217702,217702 +217703,217703 +217704,217704 +217705,217705 +217706,217706 +217707,217707 +217708,217708 +217709,217709 +217710,217710 +217711,217711 +217712,217712 +241666,217712 +217713,217713 +217714,217714 +217715,217715 +217716,217716 +217717,217717 +217718,217718 +217719,217719 +217720,217720 +217721,217721 +217722,217722 +217723,217723 +217724,217724 +217725,217725 +217726,217726 +217727,217727 +217728,217728 +217729,217729 +217730,217730 +217731,217731 +217732,217732 +217733,217733 +237610,217733 +217734,217734 +217735,217735 +217736,217736 +3291,217736 +217737,217737 +217738,217738 +217739,217739 +217740,217740 +217741,217741 +217742,217742 +217743,217743 +217744,217744 +217745,217745 +217746,217746 +217747,217747 +217748,217748 +217749,217749 +106222,217749 +217750,217750 +217751,217751 +217752,217752 +217753,217753 +102265,217753 +217754,217754 +217755,217755 +217756,217756 +217757,217757 +217758,217758 +217759,217759 +217760,217760 +217761,217761 +217805,217805 +217806,217806 +217807,217807 +217808,217808 +217809,217809 +217810,217810 +217811,217811 +243003,217811 +217812,217812 +217813,217813 +217814,217814 +223492,217814 +217815,217815 +217816,217816 +217817,217817 +217818,217818 +217819,217819 +217820,217820 +217821,217821 +217822,217822 +217823,217823 +217824,217824 +217825,217825 +217826,217826 +217827,217827 +217828,217828 +217829,217829 +2745,217829 +217830,217830 +190925,217830 +217831,217831 +217832,217832 +217833,217833 +61002,217833 +217834,217834 +58484,217834 +217835,217835 +217836,217836 +217837,217837 +217838,217838 +217839,217839 +217840,217840 +217841,217841 +217842,217842 +217843,217843 +217844,217844 +217845,217845 +217846,217846 +217847,217847 +217848,217848 +217849,217849 +217850,217850 +217851,217851 +217852,217852 +217853,217853 +217854,217854 +217855,217855 +217856,217856 +217857,217857 +217858,217858 +5758,217858 +217859,217859 +217860,217860 +217861,217861 +217862,217862 +217863,217863 +217864,217864 +217903,217903 +217904,217904 +217905,217905 +217906,217906 +217907,217907 +217908,217908 +217909,217909 +217910,217910 +217911,217911 +217912,217912 +217913,217913 +217914,217914 +217915,217915 +217916,217916 +196573,217916 +217917,217917 +217918,217918 +217919,217919 +217920,217920 +217921,217921 +217922,217922 +217923,217923 +217924,217924 +217925,217925 +217926,217926 +234991,217926 +217927,217927 +217928,217928 +217929,217929 +217930,217930 +217931,217931 +18833,217931 +217932,217932 +217933,217933 +217934,217934 +217935,217935 +217936,217936 +36930,217936 +217937,217937 +217938,217938 +217939,217939 +217940,217940 +217941,217941 +217942,217942 +217943,217943 +217944,217944 +217945,217945 +190048,217945 +217946,217946 +233358,217946 +217947,217947 +217948,217948 +217949,217949 +233357,217949 +217950,217950 +217951,217951 +217952,217952 +217953,217953 +20788,217953 +217954,217954 +217955,217955 +217956,217956 +217957,217957 +217958,217958 +224735,217958 +217959,217959 +217960,217960 +217961,217961 +217962,217962 +217963,217963 +217964,217964 +217965,217965 +217966,217966 +159400,217966 +217967,217967 +39752,217967 +217968,217968 +217969,217969 +217970,217970 +217971,217971 +217972,217972 +217973,217973 +217974,217974 +217975,217975 +217976,217976 +217977,217977 +217978,217978 +217979,217979 +217980,217980 +217981,217981 +217982,217982 +22085,217982 +217983,217983 +217984,217984 +217985,217985 +217986,217986 +217987,217987 +217988,217988 +217989,217989 +217990,217990 +217991,217991 +217992,217992 +217993,217993 +217994,217994 +217995,217995 +217996,217996 +217997,217997 +217998,217998 +217999,217999 +218000,218000 +218001,218001 +218002,218002 +218003,218003 +218004,218004 +218005,218005 +218006,218006 +218007,218007 +218008,218008 +218009,218009 +218010,218010 +218011,218011 +218012,218012 +218013,218013 +218014,218014 +218015,218015 +198719,218015 +218016,218016 +218017,218017 +218018,218018 +218019,218019 +218020,218020 +30895,218020 +218021,218021 +218022,218022 +218023,218023 +218024,218024 +218025,218025 +218026,218026 +211972,218026 +218027,218027 +218028,218028 +218029,218029 +218030,218030 +218031,218031 +218032,218032 +218033,218033 +218034,218034 +218035,218035 +218036,218036 +218037,218037 +218038,218038 +218039,218039 +132607,218039 +125020,218039 +218040,218040 +218041,218041 +218042,218042 +218043,218043 +218044,218044 +218045,218045 +218046,218046 +218047,218047 +218048,218048 +218049,218049 +233473,218049 +218050,218050 +218051,218051 +218052,218052 +218053,218053 +218054,218054 +218055,218055 +218056,218056 +218057,218057 +218058,218058 +218059,218059 +218060,218060 +218061,218061 +218062,218062 +218063,218063 +144096,218063 +218064,218064 +218065,218065 +218066,218066 +218067,218067 +218068,218068 +218069,218069 +218070,218070 +218071,218071 +218072,218072 +218073,218073 +106274,218073 +218074,218074 +218075,218075 +218076,218076 +75120,218076 +218077,218077 +218078,218078 +218079,218079 +218080,218080 +218196,218196 +218197,218197 +218198,218198 +218199,218199 +218200,218200 +218201,218201 +218202,218202 +218203,218203 +218204,218204 +218205,218205 +218206,218206 +154743,218206 +218207,218207 +218208,218208 +218209,218209 +218210,218210 +218211,218211 +218212,218212 +218213,218213 +218214,218214 +185944,218214 +218215,218215 +218216,218216 +218217,218217 +35732,218217 +218218,218218 +218219,218219 +218220,218220 +218221,218221 +218222,218222 +218223,218223 +218224,218224 +218225,218225 +218226,218226 +218227,218227 +218228,218228 +218229,218229 +218230,218230 +218231,218231 +218232,218232 +218233,218233 +218234,218234 +218235,218235 +218236,218236 +218237,218237 +218238,218238 +218239,218239 +218240,218240 +218241,218241 +218242,218242 +218243,218243 +218244,218244 +218245,218245 +218246,218246 +218247,218247 +218248,218248 +218249,218249 +218287,218287 +218288,218288 +218289,218289 +218290,218290 +218291,218291 +218292,218292 +218293,218293 +218294,218294 +218295,218295 +218296,218296 +218297,218297 +218298,218298 +218299,218299 +81323,218299 +218300,218300 +20820,218300 +218301,218301 +218302,218302 +218303,218303 +218304,218304 +12251,218304 +218305,218305 +218306,218306 +218307,218307 +218308,218308 +218309,218309 +218310,218310 +218311,218311 +205253,218311 +218312,218312 +218313,218313 +218314,218314 +78857,218314 +218315,218315 +218316,218316 +218317,218317 +218318,218318 +218319,218319 +218320,218320 +83756,218320 +218321,218321 +218322,218322 +218323,218323 +218324,218324 +218325,218325 +102246,218325 +218326,218326 +218327,218327 +218375,218375 +218376,218376 +218377,218377 +218378,218378 +155949,218378 +231249,218378 +218379,218379 +218380,218380 +218381,218381 +218382,218382 +218383,218383 +218384,218384 +218385,218385 +218386,218386 +218387,218387 +100826,218387 +218388,218388 +218389,218389 +218390,218390 +218391,218391 +218392,218392 +218393,218393 +218394,218394 +218395,218395 +218396,218396 +218397,218397 +218398,218398 +218399,218399 +218400,218400 +218401,218401 +218402,218402 +206803,218402 +218403,218403 +218404,218404 +218405,218405 +218406,218406 +218407,218407 +218408,218408 +218409,218409 +143904,218409 +218410,218410 +218411,218411 +218412,218412 +218413,218413 +218414,218414 +218415,218415 +218416,218416 +63008,218416 +218417,218417 +218418,218418 +218419,218419 +218420,218420 +33527,218420 +218421,218421 +218422,218422 +218423,218423 +218424,218424 +218425,218425 +218426,218426 +218427,218427 +218428,218428 +218429,218429 +218430,218430 +218431,218431 +218432,218432 +218433,218433 +218434,218434 +218435,218435 +218436,218436 +218437,218437 +218438,218438 +218439,218439 +218440,218440 +218441,218441 +218442,218442 +218443,218443 +218444,218444 +218445,218445 +218446,218446 +185941,218446 +218447,218447 +218448,218448 +218449,218449 +218450,218450 +218451,218451 +218452,218452 +218453,218453 +96681,218453 +218454,218454 +218455,218455 +218456,218456 +218457,218457 +218458,218458 +218459,218459 +218460,218460 +218461,218461 +218462,218462 +218463,218463 +218464,218464 +218465,218465 +218466,218466 +218467,218467 +149858,218467 +218468,218468 +218469,218469 +218470,218470 +218471,218471 +218472,218472 +218473,218473 +218474,218474 +218475,218475 +218476,218476 +218477,218477 +218478,218478 +241508,218478 +218479,218479 +218480,218480 +222270,218481 +218481,218481 +218482,218482 +218483,218483 +218484,218484 +218485,218485 +205292,218485 +218486,218486 +218487,218487 +137094,218487 +129507,218487 +218488,218488 +218489,218489 +218490,218490 +218491,218491 +218492,218492 +218493,218493 +218494,218494 +218495,218495 +218496,218496 +218497,218497 +218498,218498 +218499,218499 +218500,218500 +218501,218501 +218502,218502 +218503,218503 +218504,218504 +218505,218505 +218506,218506 +218507,218507 +218508,218508 +218509,218509 +218510,218510 +218511,218511 +218512,218512 +218513,218513 +218514,218514 +218515,218515 +249230,218515 +218516,218516 +218517,218517 +218518,218518 +218519,218519 +218520,218520 +218521,218521 +218522,218522 +218523,218523 +218524,218524 +218525,218525 +218526,218526 +201032,218526 +218527,218527 +218528,218528 +218529,218529 +37505,218529 +218654,218654 +218655,218655 +218656,218656 +218657,218657 +218658,218658 +218659,218659 +218660,218660 +218661,218661 +213557,218661 +141645,218661 +57743,218661 +218662,218662 +218663,218663 +59776,218663 +218664,218664 +218665,218665 +218666,218666 +29764,218666 +218667,218667 +218668,218668 +218669,218669 +218670,218670 +218671,218671 +2901,218671 +218672,218672 +218673,218673 +218674,218674 +219394,219394 +219395,219395 +219396,219396 +219397,219397 +219398,219398 +219399,219399 +219400,219400 +219401,219401 +219402,219402 +163576,219402 +219403,219403 +219404,219404 +219405,219405 +219406,219406 +154889,219406 +242633,219406 +219407,219407 +219408,219408 +219409,219409 +219410,219410 +219411,219411 +219412,219412 +219413,219413 +219414,219414 +219415,219415 +155062,219415 +219416,219416 +219417,219417 +219418,219418 +219419,219419 +219420,219420 +219702,219702 +219703,219703 +219794,219794 +219795,219795 +219796,219796 +219797,219797 +219798,219798 +219799,219799 +219800,219800 +237505,219800 +219801,219801 +219802,219802 +219803,219803 +219804,219804 +219805,219805 +219806,219806 +219807,219807 +219808,219808 +219809,219809 +219810,219810 +219811,219811 +219812,219812 +219813,219813 +219814,219814 +133753,219814 +126166,219814 +219815,219815 +219816,219816 +219817,219817 +121061,219817 +219818,219818 +219819,219819 +5130,219819 +219820,219820 +219821,219821 +219822,219822 +219823,219823 +219824,219824 +247543,219824 +219825,219825 +219826,219826 +219827,219827 +219828,219828 +219829,219829 +208776,219829 +219830,219830 +219831,219831 +219832,219832 +219833,219833 +219834,219834 +219835,219835 +219836,219836 +219837,219837 +219838,219838 +219839,219839 +219840,219840 +219841,219841 +219842,219842 +219843,219843 +219844,219844 +53447,219844 +45343,219844 +219917,219917 +219918,219918 +157144,219918 +219919,219919 +199589,219919 +219920,219920 +219921,219921 +219922,219922 +219923,219923 +219924,219924 +219925,219925 +219926,219926 +219927,219927 +219928,219928 +219929,219929 +2222,219929 +219930,219930 +219931,219931 +219932,219932 +219933,219933 +219934,219934 +219935,219935 +219936,219936 +219937,219937 +219938,219938 +219939,219939 +219940,219940 +219941,219941 +178270,219941 +172306,219941 +219942,219942 +219943,219943 +219944,219944 +219945,219945 +219946,219946 +219947,219947 +219948,219948 +57075,219948 +219949,219949 +219950,219950 +219951,219951 +219952,219952 +219953,219953 +219954,219954 +219955,219955 +219956,219956 +219957,219957 +219958,219958 +219959,219959 +31239,219959 +219960,219960 +219961,219961 +219962,219962 +219963,219963 +219964,219964 +219965,219965 +219966,219966 +219967,219967 +219968,219968 +180763,219968 +219969,219969 +219970,219970 +219971,219971 +219972,219972 +219973,219973 +219974,219974 +235158,219974 +219975,219975 +104698,219975 +219976,219976 +219977,219977 +219978,219978 +219979,219979 +219980,219980 +219981,219981 +219982,219982 +219983,219983 +219984,219984 +219985,219985 +219986,219986 +219987,219987 +219988,219988 +219989,219989 +219990,219990 +219991,219991 +219992,219992 +219993,219993 +219994,219994 +219995,219995 +219996,219996 +219997,219997 +219998,219998 +219999,219999 +221986,219999 +220000,220000 +220001,220001 +220002,220002 +220003,220003 +220004,220004 +220005,220005 +220006,220006 +220007,220007 +220008,220008 +220009,220009 +220010,220010 +220011,220011 +220012,220012 +220013,220013 +220014,220014 +192572,220014 +30596,220014 +220015,220015 +220016,220016 +220017,220017 +198410,220017 +220018,220018 +220019,220019 +220020,220020 +87921,220020 +220021,220021 +220022,220022 +220023,220023 +220024,220024 +220025,220025 +220026,220026 +220027,220027 +220028,220028 +220029,220029 +220030,220030 +220031,220031 +220032,220032 +220033,220033 +220034,220034 +220035,220035 +106943,220035 +178349,220035 +172385,220035 +220036,220036 +192552,220036 +220037,220037 +220038,220038 +220039,220039 +220040,220040 +220041,220041 +220042,220042 +220043,220043 +220044,220044 +220045,220045 +220046,220046 +220047,220047 +220048,220048 +181266,220048 +220049,220049 +220050,220050 +220051,220051 +220052,220052 +220053,220053 +220054,220054 +220055,220055 +220056,220056 +220057,220057 +220058,220058 +137172,220058 +129585,220058 +220059,220059 +220060,220060 +220061,220061 +220062,220062 +220063,220063 +220064,220064 +220065,220065 +220066,220066 +220067,220067 +220068,220068 +220069,220069 +220070,220070 +220071,220071 +220072,220072 +220073,220073 +220074,220074 +220075,220075 +220076,220076 +220077,220077 +220078,220078 +220079,220079 +220080,220080 +220081,220081 +226403,220081 +220082,220082 +220083,220083 +220084,220084 +220085,220085 +220086,220086 +220087,220087 +220088,220088 +220089,220089 +220090,220090 +220091,220091 +220092,220092 +220093,220093 +220094,220094 +220095,220095 +220096,220096 +220097,220097 +220098,220098 +220099,220099 +220100,220100 +220101,220101 +216678,220101 +220102,220102 +220103,220103 +220104,220104 +220105,220105 +220106,220106 +220107,220107 +220108,220108 +220109,220109 +220110,220110 +220111,220111 +86528,220111 +220112,220112 +234668,220112 +220113,220113 +220114,220114 +220115,220115 +220116,220116 +220117,220117 +220118,220118 +220119,220119 +220120,220120 +220121,220121 +220122,220122 +220123,220123 +220165,220165 +220166,220166 +220167,220167 +220168,220168 +220169,220169 +220170,220170 +220171,220171 +220172,220172 +220173,220173 +220174,220174 +220175,220175 +220176,220176 +220177,220177 +220178,220178 +220179,220179 +220180,220180 +220181,220181 +152240,220181 +220182,220182 +220183,220183 +220184,220184 +50241,220184 +42049,220184 +220185,220185 +220186,220186 +161392,220186 +220187,220187 +220188,220188 +220189,220189 +220190,220190 +220191,220191 +11935,220191 +220192,220192 +220193,220193 +220194,220194 +220195,220195 +220196,220196 +220197,220197 +220198,220198 +220199,220199 +220200,220200 +163054,220200 +143280,220200 +188406,220200 +220201,220201 +220202,220202 +220203,220203 +220204,220204 +220205,220205 +220206,220206 +220207,220207 +220208,220208 +220209,220209 +220210,220210 +220211,220211 +220212,220212 +220213,220213 +220214,220214 +220215,220215 +220216,220216 +220217,220217 +220218,220218 +233127,220218 +220219,220219 +220220,220220 +220221,220221 +220222,220222 +220223,220223 +220224,220224 +220225,220225 +220226,220226 +220227,220227 +220228,220228 +220229,220229 +161140,220229 +220230,220230 +220231,220231 +220232,220232 +191556,220232 +220233,220233 +220234,220234 +220235,220235 +220236,220236 +121569,220236 +220237,220237 +220238,220238 +220239,220239 +220240,220240 +220241,220241 +220242,220242 +220243,220243 +220282,220282 +220283,220283 +220284,220284 +153909,220284 +220285,220285 +220286,220286 +220287,220287 +220288,220288 +220289,220289 +194444,220289 +220290,220290 +220291,220291 +220292,220292 +220293,220293 +220294,220294 +143163,220294 +220295,220295 +67262,220295 +220296,220296 +220297,220297 +220298,220298 +220299,220299 +220300,220300 +28208,220300 +220301,220301 +220302,220302 +220303,220303 +220304,220304 +220305,220305 +220306,220306 +220307,220307 +163539,220307 +220308,220308 +220309,220309 +220310,220310 +220311,220311 +194751,220311 +220312,220312 +220313,220313 +220314,220314 +220315,220315 +220316,220316 +220317,220317 +220318,220318 +220319,220319 +220320,220320 +220321,220321 +220322,220322 +220323,220323 +220324,220324 +220325,220325 +148130,220325 +220326,220326 +220327,220327 +220328,220328 +220329,220329 +220330,220330 +220331,220331 +220332,220332 +220333,220333 +220334,220334 +220335,220335 +220336,220336 +65695,220336 +220337,220337 +220338,220338 +220339,220339 +220340,220340 +220341,220341 +220342,220342 +220343,220343 +220344,220344 +220345,220345 +220346,220346 +220347,220347 +220348,220348 +220349,220349 +220350,220350 +220351,220351 +220352,220352 +220353,220353 +220354,220354 +220355,220355 +220356,220356 +220357,220357 +220358,220358 +220359,220359 +220360,220360 +220361,220361 +220362,220362 +220363,220363 +220364,220364 +220365,220365 +220366,220366 +220367,220367 +220368,220368 +220369,220369 +220370,220370 +220371,220371 +220372,220372 +220373,220373 +220374,220374 +220375,220375 +135949,220375 +128362,220375 +220376,220376 +114444,220376 +183069,220376 +220377,220377 +220378,220378 +220379,220379 +220380,220380 +153922,220380 +220381,220381 +220382,220382 +220383,220383 +220384,220384 +220385,220385 +226102,220385 +220386,220386 +220387,220387 +220388,220388 +220389,220389 +220390,220390 +220391,220391 +220392,220392 +220393,220393 +220394,220394 +220395,220395 +220396,220396 +220397,220397 +220398,220398 +220399,220399 +220400,220400 +220401,220401 +220402,220402 +220403,220403 +220404,220404 +220405,220405 +220406,220406 +220407,220407 +220408,220408 +220409,220409 +220410,220410 +220411,220411 +220412,220412 +220413,220413 +220414,220414 +220415,220415 +220416,220416 +220417,220417 +220418,220418 +220419,220419 +220420,220420 +220421,220421 +220422,220422 +220423,220423 +220424,220424 +220425,220425 +220426,220426 +220427,220427 +220428,220428 +220429,220429 +220430,220430 +179158,220430 +220431,220431 +220432,220432 +220433,220433 +220434,220434 +220435,220435 +220436,220436 +220437,220437 +220438,220438 +220439,220439 +220440,220440 +220441,220441 +220442,220442 +184141,220442 +220443,220443 +220444,220444 +220445,220445 +220446,220446 +220447,220447 +220448,220448 +220449,220449 +220450,220450 +220451,220451 +220452,220452 +220453,220453 +220454,220454 +220455,220455 +220456,220456 +75217,220456 +220457,220457 +220458,220458 +220459,220459 +220460,220460 +220461,220461 +220462,220462 +220463,220463 +220464,220464 +220465,220465 +220466,220466 +220467,220467 +220468,220468 +220469,220469 +220470,220470 +220471,220471 +220472,220472 +220473,220473 +220474,220474 +220475,220475 +220476,220476 +220477,220477 +220478,220478 +220479,220479 +220480,220480 +220481,220481 +220563,220563 +220564,220564 +220565,220565 +220566,220566 +220567,220567 +220568,220568 +220569,220569 +220570,220570 +220571,220571 +220572,220572 +220573,220573 +220574,220574 +180027,220574 +220575,220575 +220576,220576 +220577,220577 +220578,220578 +220579,220579 +220580,220580 +220581,220581 +220582,220582 +5505,220582 +220583,220583 +220584,220584 +220585,220585 +23557,220585 +220586,220586 +220587,220587 +220588,220588 +220589,220589 +220590,220590 +220591,220591 +220592,220592 +220593,220593 +220594,220594 +220595,220595 +249409,220595 +220596,220596 +220597,220597 +220598,220598 +220599,220599 +220600,220600 +220601,220601 +220602,220602 +220603,220603 +220604,220604 +220605,220605 +220606,220606 +51298,220606 +43194,220606 +15706,220606 +222459,220607 +220607,220607 +209370,220607 +220608,220608 +220609,220609 +222461,220609 +220610,220610 +220611,220611 +220943,220611 +220612,220612 +220944,220612 +220613,220613 +220614,220614 +114805,220614 +220615,220615 +220947,220615 +220616,220616 +220617,220617 +220950,220618 +220618,220618 +222470,220618 +2694,220618 +220619,220619 +231251,220620 +220620,220620 +220621,220621 +220622,220622 +220954,220622 +220623,220623 +220624,220624 +179843,220624 +32426,220624 +220625,220625 +220626,220626 +220958,220626 +220627,220627 +220628,220628 +220629,220629 +24508,220629 +220630,220630 +220631,220631 +156415,220631 +220632,220632 +220633,220633 +220634,220634 +220635,220635 +220636,220636 +220637,220637 +220638,220638 +220639,220639 +220640,220640 +220641,220641 +220642,220642 +220643,220643 +220644,220644 +220745,220745 +220746,220746 +220747,220747 +220748,220748 +220749,220749 +240307,220749 +220833,220833 +220834,220834 +247427,220834 +220835,220835 +220836,220836 +220837,220837 +220838,220838 +220839,220839 +220840,220840 +193302,220840 +220841,220841 +220842,220842 +4196,220842 +220843,220843 +220844,220844 +162424,220844 +220845,220845 +222365,220845 +220846,220846 +220847,220847 +220888,220888 +222408,220888 +220889,220889 +222409,220889 +222410,220890 +220890,220890 +220891,220891 +222411,220891 +222412,220892 +220892,220892 +220893,220893 +222413,220893 +222414,220894 +220894,220894 +222415,220895 +220895,220895 +188693,220895 +220896,220896 +222416,220896 +220897,220897 +222417,220897 +200592,220897 +222418,220898 +220898,220898 +220940,220940 +220941,220941 +220942,220942 +220943,220943 +220611,220943 +220944,220944 +220612,220944 +162620,220944 +220945,220945 +222466,220946 +220946,220946 +220947,220947 +220615,220947 +222468,220948 +220948,220948 +220949,220949 +220950,220950 +220618,220950 +222470,220950 +2694,220950 +220951,220951 +222471,220951 +220952,220952 +220953,220953 +220954,220954 +220622,220954 +222474,220954 +222475,220955 +220955,220955 +191961,220955 +220956,220956 +222477,220957 +220957,220957 +220958,220958 +220626,220958 +220959,220959 +220960,220960 +222481,220961 +220961,220961 +222482,220962 +220962,220962 +221019,221019 +221020,221020 +221021,221021 +221022,221022 +221023,221023 +221024,221024 +221025,221025 +221026,221026 +221027,221027 +221028,221028 +221029,221029 +221030,221030 +221031,221031 +221032,221032 +221033,221033 +221034,221034 +221035,221035 +221036,221036 +221037,221037 +221038,221038 +221039,221039 +221040,221040 +221158,221158 +221159,221159 +221160,221160 +221161,221161 +221162,221162 +221163,221163 +187321,221163 +221164,221164 +226780,221164 +221165,221165 +221166,221166 +221167,221167 +221168,221168 +221169,221169 +221170,221170 +221171,221171 +221172,221172 +221173,221173 +221174,221174 +221175,221175 +221176,221176 +221177,221177 +221178,221178 +221179,221179 +221180,221180 +141505,221180 +221181,221181 +221182,221182 +221183,221183 +221184,221184 +221185,221185 +221186,221186 +221187,221187 +221188,221188 +221189,221189 +221190,221190 +221191,221191 +221192,221192 +221193,221193 +221194,221194 +221195,221195 +221196,221196 +109890,221196 +221197,221197 +221198,221198 +221199,221199 +221200,221200 +221201,221201 +221202,221202 +221203,221203 +79399,221203 +221204,221204 +221205,221205 +221206,221206 +221207,221207 +221208,221208 +221209,221209 +221210,221210 +221211,221211 +221289,221289 +51541,221289 +43437,221289 +221290,221290 +221291,221291 +186058,221291 +221292,221292 +3880,221292 +221293,221293 +221294,221294 +221295,221295 +221296,221296 +221297,221297 +221424,221424 +221425,221425 +221426,221426 +221427,221427 +221548,221548 +221549,221549 +102424,221549 +221550,221550 +221551,221551 +102426,221551 +221552,221552 +221553,221553 +221554,221554 +221555,221555 +221556,221556 +221557,221557 +221558,221558 +221559,221559 +190153,221559 +221560,221560 +221561,221561 +221562,221562 +221563,221563 +221649,221649 +221650,221650 +221651,221651 +221652,221652 +221653,221653 +221654,221654 +221655,221655 +221656,221656 +15385,221656 +221657,221657 +221658,221658 +221659,221659 +221660,221660 +221661,221661 +221662,221662 +221663,221663 +221664,221664 +221665,221665 +221666,221666 +221667,221667 +221668,221668 +221669,221669 +221670,221670 +221671,221671 +221672,221672 +211893,221672 +221673,221673 +221674,221674 +221675,221675 +221676,221676 +22081,221676 +221677,221677 +221678,221678 +221679,221679 +221680,221680 +221681,221681 +221682,221682 +221683,221683 +221684,221684 +221685,221685 +221686,221686 +221687,221687 +221688,221688 +221689,221689 +221690,221690 +221691,221691 +221692,221692 +221693,221693 +221694,221694 +221695,221695 +99252,221695 +221696,221696 +221697,221697 +221698,221698 +221699,221699 +221700,221700 +221701,221701 +221702,221702 +221703,221703 +221704,221704 +221705,221705 +221706,221706 +221707,221707 +221708,221708 +221709,221709 +221710,221710 +221711,221711 +221712,221712 +221713,221713 +221714,221714 +221715,221715 +221716,221716 +221717,221717 +221718,221718 +221719,221719 +221720,221720 +221721,221721 +221722,221722 +221723,221723 +221724,221724 +221725,221725 +87450,221725 +221726,221726 +221727,221727 +221728,221728 +221729,221729 +221730,221730 +221731,221731 +221732,221732 +221733,221733 +221734,221734 +221735,221735 +221736,221736 +221737,221737 +221738,221738 +221739,221739 +89863,221739 +221740,221740 +221741,221741 +221742,221742 +221743,221743 +221744,221744 +221745,221745 +221746,221746 +221747,221747 +157800,221747 +221748,221748 +221749,221749 +221750,221750 +221751,221751 +221752,221752 +221753,221753 +221754,221754 +221755,221755 +221756,221756 +221757,221757 +209439,221757 +221758,221758 +221759,221759 +221760,221760 +221761,221761 +221762,221762 +221763,221763 +221764,221764 +221765,221765 +221766,221766 +221767,221767 +221768,221768 +221769,221769 +221770,221770 +221771,221771 +221772,221772 +221773,221773 +221774,221774 +221775,221775 +221776,221776 +221777,221777 +221778,221778 +221779,221779 +221780,221780 +221781,221781 +221782,221782 +221783,221783 +221784,221784 +221785,221785 +221786,221786 +221787,221787 +221788,221788 +221789,221789 +221790,221790 +221791,221791 +221792,221792 +221793,221793 +221794,221794 +221795,221795 +221796,221796 +221797,221797 +180610,221797 +221798,221798 +221799,221799 +221800,221800 +221801,221801 +221802,221802 +221803,221803 +221804,221804 +221805,221805 +221806,221806 +221807,221807 +221808,221808 +221809,221809 +221810,221810 +221811,221811 +221812,221812 +221813,221813 +221814,221814 +221815,221815 +221816,221816 +221817,221817 +221818,221818 +221819,221819 +221820,221820 +221821,221821 +221822,221822 +221823,221823 +221824,221824 +221825,221825 +221826,221826 +221827,221827 +221828,221828 +221829,221829 +221830,221830 +221831,221831 +221832,221832 +221833,221833 +221834,221834 +221835,221835 +221836,221836 +221837,221837 +221838,221838 +221839,221839 +221840,221840 +221841,221841 +221842,221842 +221843,221843 +221844,221844 +221845,221845 +221846,221846 +221847,221847 +221848,221848 +221849,221849 +221850,221850 +221851,221851 +221852,221852 +221853,221853 +221854,221854 +221855,221855 +221856,221856 +221857,221857 +221858,221858 +221859,221859 +221860,221860 +221861,221861 +221862,221862 +178570,221862 +221863,221863 +221864,221864 +80547,221864 +221865,221865 +221866,221866 +221867,221867 +221868,221868 +221869,221869 +221870,221870 +221871,221871 +145117,221871 +221872,221872 +221873,221873 +221874,221874 +221875,221875 +221876,221876 +221877,221877 +221878,221878 +29278,221878 +221879,221879 +187378,221879 +221880,221880 +221881,221881 +221882,221882 +221883,221883 +221884,221884 +221885,221885 +221886,221886 +221887,221887 +221888,221888 +221889,221889 +221890,221890 +221891,221891 +221892,221892 +221893,221893 +221894,221894 +221895,221895 +221896,221896 +221897,221897 +221898,221898 +221899,221899 +221900,221900 +221901,221901 +221902,221902 +221903,221903 +221904,221904 +221905,221905 +221906,221906 +221907,221907 +221908,221908 +221909,221909 +221910,221910 +221911,221911 +221912,221912 +221913,221913 +234736,221913 +221914,221914 +221915,221915 +221916,221916 +221917,221917 +224754,221917 +221918,221918 +185069,221918 +221919,221919 +221920,221920 +221921,221921 +221922,221922 +221923,221923 +221924,221924 +221925,221925 +221926,221926 +221927,221927 +221928,221928 +5529,221928 +240300,221928 +221929,221929 +221930,221930 +221931,221931 +221932,221932 +221933,221933 +221934,221934 +221935,221935 +221936,221936 +221937,221937 +221938,221938 +221939,221939 +221940,221940 +221941,221941 +221942,221942 +221943,221943 +221944,221944 +221945,221945 +221946,221946 +192829,221946 +241600,221946 +221947,221947 +221948,221948 +221949,221949 +221950,221950 +221951,221951 +221952,221952 +221953,221953 +221954,221954 +221955,221955 +221956,221956 +221957,221957 +221958,221958 +221959,221959 +221960,221960 +221961,221961 +221962,221962 +221963,221963 +221964,221964 +245418,221964 +221965,221965 +221966,221966 +221967,221967 +221968,221968 +221969,221969 +221970,221970 +221971,221971 +221972,221972 +221973,221973 +221974,221974 +221975,221975 +221976,221976 +221977,221977 +221978,221978 +221979,221979 +80407,221979 +221980,221980 +196711,221980 +221981,221981 +221982,221982 +221983,221983 +221984,221984 +221985,221985 +221986,221986 +219999,221986 +221987,221987 +221988,221988 +221989,221989 +221990,221990 +221991,221991 +84106,221991 +221992,221992 +221993,221993 +88692,221993 +221994,221994 +221995,221995 +221996,221996 +28097,221996 +221997,221997 +221998,221998 +221999,221999 +222000,222000 +222001,222001 +222002,222002 +222003,222003 +222004,222004 +222005,222005 +222006,222006 +222007,222007 +222008,222008 +222009,222009 +222010,222010 +222011,222011 +222012,222012 +222013,222013 +222014,222014 +152855,222014 +222015,222015 +222016,222016 +222017,222017 +222018,222018 +185497,222018 +10071,222018 +222019,222019 +222020,222020 +222021,222021 +222022,222022 +222023,222023 +222024,222024 +222025,222025 +222026,222026 +222027,222027 +222028,222028 +241078,222028 +222029,222029 +222030,222030 +222031,222031 +222032,222032 +222033,222033 +222034,222034 +222035,222035 +222036,222036 +222037,222037 +222038,222038 +222039,222039 +222040,222040 +222041,222041 +222042,222042 +222043,222043 +222044,222044 +100844,222044 +222045,222045 +222046,222046 +222047,222047 +222048,222048 +222049,222049 +185882,222049 +222050,222050 +222051,222051 +222052,222052 +222053,222053 +222054,222054 +235171,222054 +222055,222055 +188994,222055 +222056,222056 +150460,222056 +222057,222057 +222058,222058 +222059,222059 +180092,222059 +222060,222060 +222061,222061 +222062,222062 +222063,222063 +222064,222064 +222065,222065 +222066,222066 +222067,222067 +222068,222068 +190616,222068 +222069,222069 +222070,222070 +222071,222071 +222072,222072 +222073,222073 +222074,222074 +222075,222075 +222076,222076 +222077,222077 +222078,222078 +222079,222079 +222080,222080 +222081,222081 +222082,222082 +222083,222083 +222084,222084 +222085,222085 +222086,222086 +222087,222087 +222088,222088 +222089,222089 +222090,222090 +222091,222091 +222092,222092 +222093,222093 +222094,222094 +222095,222095 +88508,222095 +222096,222096 +222097,222097 +222098,222098 +222099,222099 +222100,222100 +222101,222101 +222102,222102 +222103,222103 +222104,222104 +222105,222105 +222106,222106 +222107,222107 +222108,222108 +60921,222108 +222109,222109 +222110,222110 +222111,222111 +222112,222112 +235789,222112 +222113,222113 +222114,222114 +222115,222115 +222116,222116 +222117,222117 +222118,222118 +222119,222119 +222120,222120 +222121,222121 +222122,222122 +71339,222122 +222123,222123 +222124,222124 +222125,222125 +222126,222126 +222127,222127 +222128,222128 +222129,222129 +222130,222130 +197605,222130 +222131,222131 +222132,222132 +222133,222133 +222134,222134 +222135,222135 +222136,222136 +222137,222137 +222138,222138 +222139,222139 +222140,222140 +222141,222141 +222142,222142 +222143,222143 +222144,222144 +222145,222145 +222146,222146 +222147,222147 +176764,222147 +170800,222147 +222148,222148 +222149,222149 +222150,222150 +222151,222151 +222152,222152 +157374,222152 +222153,222153 +222154,222154 +222155,222155 +222156,222156 +222157,222157 +222158,222158 +222159,222159 +222160,222160 +222161,222161 +222162,222162 +222163,222163 +222164,222164 +222165,222165 +222166,222166 +222167,222167 +222168,222168 +222169,222169 +222170,222170 +222171,222171 +225484,222171 +222172,222172 +222173,222173 +222174,222174 +36138,222174 +222175,222175 +222176,222176 +222177,222177 +61783,222177 +222178,222178 +222179,222179 +222180,222180 +222181,222181 +222182,222182 +222183,222183 +185887,222183 +222184,222184 +222185,222185 +222186,222186 +222187,222187 +222188,222188 +222189,222189 +107558,222189 +222190,222190 +185894,222190 +222191,222191 +222192,222192 +222193,222193 +222194,222194 +222195,222195 +222196,222196 +222197,222197 +222198,222198 +135293,222198 +127706,222198 +222199,222199 +222200,222200 +222201,222201 +222202,222202 +222203,222203 +222204,222204 +249233,222204 +222205,222205 +222206,222206 +222207,222207 +222208,222208 +222209,222209 +222210,222210 +222211,222211 +222212,222212 +117090,222212 +86962,222212 +222213,222213 +222214,222214 +222215,222215 +222216,222216 +222217,222217 +222218,222218 +222219,222219 +222220,222220 +222221,222221 +222222,222222 +88251,222222 +222223,222223 +222224,222224 +222225,222225 +222226,222226 +222227,222227 +222228,222228 +222229,222229 +222230,222230 +222231,222231 +222232,222232 +222233,222233 +222234,222234 +222235,222235 +222236,222236 +222237,222237 +222238,222238 +222239,222239 +222240,222240 +222241,222241 +222242,222242 +222243,222243 +222244,222244 +222245,222245 +222246,222246 +222247,222247 +222248,222248 +222249,222249 +222250,222250 +222251,222251 +222252,222252 +222253,222253 +222254,222254 +222255,222255 +222256,222256 +222257,222257 +222258,222258 +222259,222259 +222260,222260 +222261,222261 +222262,222262 +222263,222263 +222264,222264 +180532,222264 +222265,222265 +222266,222266 +222267,222267 +35728,222267 +222268,222268 +222269,222269 +222270,222270 +218481,222270 +222271,222271 +222272,222272 +222273,222273 +191573,222273 +222274,222274 +213701,222274 +222275,222275 +222276,222276 +222277,222277 +222278,222278 +222279,222279 +222280,222280 +222343,222343 +222344,222344 +222345,222345 +222346,222346 +222347,222347 +222348,222348 +222349,222349 +222350,222350 +222351,222351 +222352,222352 +79106,222352 +222353,222353 +222354,222354 +222355,222355 +222356,222356 +151603,222356 +222357,222357 +222358,222358 +222359,222359 +222360,222360 +222361,222361 +222362,222362 +222363,222363 +222364,222364 +222365,222365 +27192,222365 +220845,222365 +222366,222366 +222367,222367 +222368,222368 +222369,222369 +222408,222408 +220888,222408 +222409,222409 +220889,222409 +222410,222410 +220890,222410 +222411,222411 +220891,222411 +222412,222412 +220892,222412 +222413,222413 +220893,222413 +222414,222414 +220894,222414 +222415,222415 +220895,222415 +188693,222415 +222416,222416 +220896,222416 +222417,222417 +220897,222417 +222418,222418 +220898,222418 +222458,222458 +222459,222459 +220607,222459 +209370,222459 +222460,222460 +222461,222461 +220609,222461 +222462,222462 +222463,222463 +222464,222464 +222465,222465 +155638,222465 +222466,222466 +220946,222466 +222467,222467 +222468,222468 +220948,222468 +222469,222469 +222470,222470 +220950,222470 +220618,222470 +222471,222471 +220951,222471 +222472,222472 +222473,222473 +222474,222474 +220954,222474 +222475,222475 +220955,222475 +191961,222475 +222476,222476 +21815,222476 +222477,222477 +220957,222477 +222478,222478 +222479,222479 +222480,222480 +222481,222481 +220961,222481 +222482,222482 +220962,222482 +222653,222653 +222654,222654 +222655,222655 +222656,222656 +222657,222657 +222658,222658 +222659,222659 +222660,222660 +222661,222661 +222662,222662 +222663,222663 +222664,222664 +222665,222665 +222666,222666 +222667,222667 +222668,222668 +222669,222669 +222670,222670 +222671,222671 +222766,222766 +222767,222767 +222768,222768 +222769,222769 +222770,222770 +222771,222771 +222772,222772 +222773,222773 +222774,222774 +222775,222775 +222776,222776 +222777,222777 +222778,222778 +222779,222779 +222780,222780 +222781,222781 +222782,222782 +222783,222783 +222784,222784 +222785,222785 +222786,222786 +222787,222787 +222788,222788 +222789,222789 +222790,222790 +222791,222791 +222792,222792 +222793,222793 +222794,222794 +222795,222795 +222796,222796 +222797,222797 +222798,222798 +222799,222799 +222800,222800 +77203,222800 +222801,222801 +222841,222841 +222842,222842 +222843,222843 +206082,222843 +205709,222843 +222844,222844 +222845,222845 +222846,222846 +222847,222847 +222848,222848 +222849,222849 +222850,222850 +222851,222851 +222852,222852 +222853,222853 +222854,222854 +222855,222855 +225518,222855 +222856,222856 +81189,222856 +222857,222857 +222858,222858 +222859,222859 +222860,222860 +222861,222861 +222862,222862 +222863,222863 +222864,222864 +222865,222865 +222866,222866 +222867,222867 +87392,222867 +222868,222868 +222869,222869 +222870,222870 +222871,222871 +222872,222872 +224968,222872 +222873,222873 +222874,222874 +222875,222875 +222876,222876 +222877,222877 +222878,222878 +222879,222879 +222880,222880 +137014,222880 +129427,222880 +222881,222881 +222882,222882 +222883,222883 +222884,222884 +222885,222885 +222886,222886 +222887,222887 +222888,222888 +222889,222889 +222890,222890 +222891,222891 +222892,222892 +222893,222893 +222894,222894 +222895,222895 +222896,222896 +222897,222897 +222898,222898 +222899,222899 +222900,222900 +222901,222901 +222902,222902 +222903,222903 +222904,222904 +222905,222905 +222906,222906 +222907,222907 +222908,222908 +222909,222909 +222910,222910 +222911,222911 +222912,222912 +222913,222913 +222914,222914 +222915,222915 +222916,222916 +222917,222917 +222918,222918 +222919,222919 +222920,222920 +222921,222921 +222922,222922 +222923,222923 +222924,222924 +222925,222925 +222926,222926 +231961,222926 +222927,222927 +222928,222928 +222929,222929 +222930,222930 +222931,222931 +222932,222932 +222933,222933 +222934,222934 +222935,222935 +76916,222935 +222936,222936 +222937,222937 +222938,222938 +222939,222939 +222940,222940 +222941,222941 +222942,222942 +222943,222943 +110088,222943 +7055,222943 +222944,222944 +222945,222945 +222946,222946 +222947,222947 +222948,222948 +89814,222948 +222949,222949 +222950,222950 +40173,222950 +222990,222990 +222991,222991 +222992,222992 +222993,222993 +110840,222993 +222994,222994 +222995,222995 +222996,222996 +222997,222997 +222998,222998 +222999,222999 +223000,223000 +223001,223001 +223002,223002 +223003,223003 +223004,223004 +223005,223005 +223006,223006 +223007,223007 +223008,223008 +223009,223009 +223010,223010 +223011,223011 +223012,223012 +223013,223013 +223014,223014 +100593,223014 +223015,223015 +223016,223016 +223017,223017 +223018,223018 +223019,223019 +223020,223020 +223021,223021 +223022,223022 +84529,223022 +223023,223023 +80319,223023 +223024,223024 +223025,223025 +223026,223026 +223027,223027 +223028,223028 +223029,223029 +223030,223030 +223031,223031 +223032,223032 +223033,223033 +223034,223034 +30806,223034 +223035,223035 +223036,223036 +223037,223037 +223038,223038 +223039,223039 +223040,223040 +223041,223041 +223042,223042 +223043,223043 +223044,223044 +223045,223045 +223046,223046 +223047,223047 +223048,223048 +223049,223049 +102522,223049 +223050,223050 +223051,223051 +223052,223052 +223053,223053 +223054,223054 +223055,223055 +223056,223056 +223057,223057 +202619,223057 +223058,223058 +223059,223059 +223060,223060 +223061,223061 +223062,223062 +223063,223063 +223064,223064 +223065,223065 +223066,223066 +223067,223067 +223068,223068 +223069,223069 +214191,223069 +223070,223070 +223071,223071 +223072,223072 +223073,223073 +223074,223074 +223075,223075 +223076,223076 +223077,223077 +223078,223078 +223079,223079 +223080,223080 +223081,223081 +223082,223082 +223083,223083 +223084,223084 +223085,223085 +223086,223086 +223087,223087 +223088,223088 +223089,223089 +223090,223090 +223091,223091 +223092,223092 +223093,223093 +223094,223094 +223095,223095 +223096,223096 +223097,223097 +223098,223098 +223099,223099 +223100,223100 +223101,223101 +203962,223101 +223102,223102 +181019,223102 +223103,223103 +223104,223104 +223105,223105 +6225,223105 +223106,223106 +223107,223107 +223108,223108 +223109,223109 +223110,223110 +223154,223154 +223155,223155 +223156,223156 +223157,223157 +223158,223158 +223159,223159 +189758,223159 +223160,223160 +223161,223161 +223162,223162 +223163,223163 +223164,223164 +223165,223165 +223166,223166 +223167,223167 +223168,223168 +223169,223169 +233370,223169 +215600,223169 +223170,223170 +223171,223171 +223172,223172 +223173,223173 +223174,223174 +223175,223175 +223176,223176 +223177,223177 +223178,223178 +223179,223179 +223180,223180 +223181,223181 +223182,223182 +223183,223183 +223184,223184 +223185,223185 +223186,223186 +223187,223187 +223188,223188 +223189,223189 +223190,223190 +223191,223191 +223192,223192 +223193,223193 +223194,223194 +223195,223195 +223196,223196 +223197,223197 +223198,223198 +223199,223199 +223200,223200 +223201,223201 +223202,223202 +223203,223203 +223204,223204 +223205,223205 +223206,223206 +223245,223245 +223246,223246 +223247,223247 +223248,223248 +223249,223249 +223250,223250 +223251,223251 +223252,223252 +223253,223253 +223254,223254 +223255,223255 +223256,223256 +223257,223257 +223258,223258 +223259,223259 +223260,223260 +223261,223261 +223262,223262 +223263,223263 +223264,223264 +223265,223265 +223266,223266 +223267,223267 +223268,223268 +223269,223269 +223270,223270 +152995,223270 +223271,223271 +223272,223272 +223341,223341 +223342,223342 +223343,223343 +223344,223344 +223345,223345 +223346,223346 +223347,223347 +223348,223348 +223349,223349 +223350,223350 +223351,223351 +223352,223352 +223353,223353 +223354,223354 +223355,223355 +223356,223356 +223395,223395 +223396,223396 +223397,223397 +223398,223398 +223399,223399 +223400,223400 +223401,223401 +223402,223402 +223403,223403 +223404,223404 +223405,223405 +223406,223406 +141416,223406 +223407,223407 +223408,223408 +223409,223409 +223410,223410 +223411,223411 +223412,223412 +223413,223413 +223414,223414 +223415,223415 +223416,223416 +223417,223417 +223418,223418 +223419,223419 +223420,223420 +223421,223421 +223422,223422 +223423,223423 +223424,223424 +223425,223425 +223426,223426 +223427,223427 +223428,223428 +223429,223429 +223430,223430 +223431,223431 +223432,223432 +223433,223433 +223434,223434 +223435,223435 +223436,223436 +223437,223437 +223438,223438 +223439,223439 +223440,223440 +223441,223441 +184286,223441 +223442,223442 +223443,223443 +223444,223444 +223445,223445 +223446,223446 +133220,223446 +125633,223446 +223447,223447 +132602,223447 +125015,223447 +223448,223448 +223449,223449 +223450,223450 +223451,223451 +223452,223452 +223453,223453 +223454,223454 +223455,223455 +223456,223456 +223457,223457 +223458,223458 +77156,223458 +223459,223459 +223460,223460 +223461,223461 +223462,223462 +223463,223463 +111753,223463 +223464,223464 +223465,223465 +223466,223466 +223467,223467 +223468,223468 +223469,223469 +223470,223470 +223471,223471 +104323,223471 +223472,223472 +223473,223473 +223474,223474 +223475,223475 +223476,223476 +223477,223477 +223478,223478 +223479,223479 +223480,223480 +3676,223480 +223481,223481 +223482,223482 +223483,223483 +223484,223484 +223485,223485 +223486,223486 +223487,223487 +136777,223487 +129190,223487 +223488,223488 +223489,223489 +223490,223490 +223491,223491 +223492,223492 +217814,223492 +223493,223493 +223494,223494 +223495,223495 +223496,223496 +223497,223497 +209963,223497 +106132,223497 +152903,223497 +223498,223498 +223499,223499 +223500,223500 +223501,223501 +223502,223502 +223503,223503 +235846,223503 +223504,223504 +223505,223505 +234733,223505 +223506,223506 +223507,223507 +162186,223507 +223556,223556 +223601,223601 +223602,223602 +223603,223603 +223604,223604 +223605,223605 +208706,223605 +223606,223606 +223607,223607 +223608,223608 +223609,223609 +223610,223610 +223611,223611 +223612,223612 +223613,223613 +223614,223614 +223615,223615 +8593,223615 +8413,223615 +223616,223616 +223617,223617 +223618,223618 +223619,223619 +223620,223620 +223621,223621 +223622,223622 +223623,223623 +223624,223624 +223625,223625 +223626,223626 +223627,223627 +223628,223628 +223629,223629 +4687,223629 +223630,223630 +223631,223631 +223632,223632 +223633,223633 +223634,223634 +223635,223635 +90130,223635 +223636,223636 +223759,223759 +223760,223760 +223761,223761 +223762,223762 +223763,223763 +223764,223764 +244113,223764 +223765,223765 +223766,223766 +223767,223767 +223768,223768 +223769,223769 +223770,223770 +223771,223771 +223772,223772 +72561,223772 +223773,223773 +223774,223774 +223775,223775 +223776,223776 +223777,223777 +223778,223778 +30817,223778 +224348,224348 +224349,224349 +224350,224350 +224351,224351 +224352,224352 +224353,224353 +156401,224353 +224354,224354 +224355,224355 +224356,224356 +224357,224357 +224358,224358 +224359,224359 +224360,224360 +224361,224361 +224362,224362 +224363,224363 +224364,224364 +87797,224364 +224365,224365 +224366,224366 +224367,224367 +224368,224368 +224369,224369 +224370,224370 +224371,224371 +224372,224372 +30733,224372 +224373,224373 +224374,224374 +224375,224375 +224376,224376 +224377,224377 +224378,224378 +224379,224379 +224380,224380 +224381,224381 +224382,224382 +224383,224383 +224384,224384 +224385,224385 +224386,224386 +224387,224387 +224388,224388 +224389,224389 +204368,224389 +224390,224390 +224391,224391 +5547,224391 +224392,224392 +224393,224393 +224394,224394 +224395,224395 +224396,224396 +224397,224397 +224398,224398 +102387,224398 +224399,224399 +224400,224400 +224401,224401 +224402,224402 +155189,224402 +224403,224403 +224404,224404 +224405,224405 +224406,224406 +224407,224407 +224408,224408 +210336,224408 +224409,224409 +224410,224410 +224411,224411 +187094,224411 +224412,224412 +225997,224412 +224413,224413 +202880,224413 +224414,224414 +224415,224415 +224416,224416 +224417,224417 +224418,224418 +224419,224419 +224420,224420 +224421,224421 +224422,224422 +224423,224423 +224424,224424 +224425,224425 +224426,224426 +224427,224427 +224428,224428 +224429,224429 +224430,224430 +224431,224431 +224432,224432 +224433,224433 +224434,224434 +224435,224435 +224436,224436 +224437,224437 +224438,224438 +224439,224439 +224440,224440 +224441,224441 +224442,224442 +224443,224443 +224444,224444 +224445,224445 +224446,224446 +224447,224447 +224448,224448 +224449,224449 +224450,224450 +224451,224451 +224452,224452 +224453,224453 +216527,224453 +224454,224454 +224455,224455 +224456,224456 +224457,224457 +224541,224541 +205891,224541 +224542,224542 +224543,224543 +224544,224544 +224545,224545 +224546,224546 +224547,224547 +224548,224548 +224549,224549 +224550,224550 +224551,224551 +224552,224552 +208026,224552 +224553,224553 +224554,224554 +224555,224555 +209951,224555 +224556,224556 +224557,224557 +224558,224558 +224559,224559 +224560,224560 +224561,224561 +224562,224562 +224563,224563 +224564,224564 +187430,224564 +224565,224565 +224566,224566 +224567,224567 +224568,224568 +224569,224569 +224570,224570 +224571,224571 +224572,224572 +224573,224573 +162670,224573 +224574,224574 +224575,224575 +224576,224576 +224577,224577 +224578,224578 +224579,224579 +80529,224579 +224580,224580 +224581,224581 +224582,224582 +80532,224582 +224583,224583 +80536,224583 +224584,224584 +224585,224585 +224586,224586 +93833,224586 +224587,224587 +224588,224588 +224589,224589 +224590,224590 +224591,224591 +224592,224592 +224593,224593 +224594,224594 +224595,224595 +224596,224596 +209804,224596 +224597,224597 +93844,224597 +224598,224598 +224599,224599 +224600,224600 +224601,224601 +224602,224602 +224603,224603 +224604,224604 +28736,224604 +224605,224605 +154353,224605 +224606,224606 +55568,224606 +47464,224606 +224607,224607 +224608,224608 +224609,224609 +224610,224610 +224611,224611 +224612,224612 +224613,224613 +224614,224614 +224615,224615 +224616,224616 +224617,224617 +224618,224618 +224619,224619 +224620,224620 +224621,224621 +224622,224622 +224623,224623 +224624,224624 +224625,224625 +68006,224625 +20642,224625 +224626,224626 +224627,224627 +224628,224628 +224629,224629 +224630,224630 +224631,224631 +224632,224632 +224633,224633 +224634,224634 +224635,224635 +224636,224636 +224637,224637 +208747,224637 +224638,224638 +224639,224639 +224640,224640 +224641,224641 +92634,224641 +224642,224642 +84827,224642 +224643,224643 +224719,224719 +224720,224720 +224721,224721 +224722,224722 +224723,224723 +224724,224724 +179789,224724 +224725,224725 +224726,224726 +224727,224727 +224728,224728 +224729,224729 +179359,224729 +224730,224730 +224731,224731 +224732,224732 +224733,224733 +224734,224734 +224735,224735 +217958,224735 +224736,224736 +224737,224737 +36071,224737 +224738,224738 +224739,224739 +224740,224740 +224741,224741 +81489,224741 +224742,224742 +224743,224743 +224744,224744 +224745,224745 +224746,224746 +224747,224747 +224748,224748 +224749,224749 +224750,224750 +224751,224751 +224752,224752 +224753,224753 +224754,224754 +221917,224754 +224755,224755 +224756,224756 +224757,224757 +190815,224757 +224758,224758 +224759,224759 +224760,224760 +224761,224761 +224762,224762 +224763,224763 +224764,224764 +224765,224765 +224766,224766 +224767,224767 +224768,224768 +224769,224769 +55294,224769 +47190,224769 +224770,224770 +224771,224771 +202302,224771 +224772,224772 +73754,224772 +224773,224773 +224774,224774 +174171,224774 +168207,224774 +224775,224775 +224776,224776 +224777,224777 +224778,224778 +224779,224779 +224780,224780 +224781,224781 +224782,224782 +224783,224783 +224784,224784 +81860,224784 +224785,224785 +224786,224786 +224787,224787 +224788,224788 +224789,224789 +224790,224790 +224791,224791 +224792,224792 +224793,224793 +224794,224794 +224795,224795 +224796,224796 +224797,224797 +224798,224798 +224799,224799 +224867,224867 +224868,224868 +147556,224868 +224869,224869 +224870,224870 +224871,224871 +224872,224872 +224873,224873 +224874,224874 +224875,224875 +224876,224876 +224877,224877 +224878,224878 +83838,224878 +224879,224879 +183664,224879 +224880,224880 +224881,224881 +224882,224882 +224883,224883 +224884,224884 +224885,224885 +224886,224886 +224887,224887 +224888,224888 +224889,224889 +224890,224890 +224891,224891 +85417,224891 +224892,224892 +224893,224893 +224894,224894 +161547,224894 +224895,224895 +224896,224896 +224897,224897 +224898,224898 +224899,224899 +224900,224900 +224901,224901 +224902,224902 +224903,224903 +224904,224904 +224905,224905 +224906,224906 +224907,224907 +224908,224908 +224909,224909 +224910,224910 +224911,224911 +224912,224912 +224913,224913 +19048,224913 +224914,224914 +224915,224915 +224916,224916 +224917,224917 +224918,224918 +224919,224919 +224920,224920 +224921,224921 +224922,224922 +224923,224923 +224924,224924 +224925,224925 +224926,224926 +224927,224927 +224928,224928 +224929,224929 +224930,224930 +224931,224931 +224932,224932 +224933,224933 +224934,224934 +224935,224935 +224936,224936 +224937,224937 +161853,224937 +224938,224938 +224939,224939 +79263,224939 +224940,224940 +224941,224941 +205256,224941 +224942,224942 +224943,224943 +224944,224944 +224945,224945 +208422,224945 +224946,224946 +224947,224947 +232638,224947 +224948,224948 +224949,224949 +224950,224950 +224951,224951 +56694,224951 +224952,224952 +224953,224953 +224954,224954 +224955,224955 +224956,224956 +224957,224957 +224958,224958 +224959,224959 +224960,224960 +224961,224961 +224962,224962 +224963,224963 +224964,224964 +224965,224965 +224966,224966 +224967,224967 +224968,224968 +222872,224968 +224969,224969 +224970,224970 +137554,224970 +129967,224970 +224971,224971 +224972,224972 +224973,224973 +224974,224974 +112139,224974 +224975,224975 +224976,224976 +114445,224976 +224977,224977 +224978,224978 +225023,225023 +225024,225024 +225025,225025 +225026,225026 +7104,225026 +225027,225027 +225028,225028 +225029,225029 +225030,225030 +225031,225031 +225032,225032 +225033,225033 +225034,225034 +225035,225035 +225036,225036 +225037,225037 +225038,225038 +225039,225039 +225040,225040 +225041,225041 +205995,225041 +225079,225079 +160144,225079 +225080,225080 +225081,225081 +225082,225082 +225083,225083 +183903,225083 +225084,225084 +193385,225084 +225085,225085 +225086,225086 +225087,225087 +225088,225088 +225089,225089 +225090,225090 +225091,225091 +225092,225092 +225093,225093 +225132,225132 +225133,225133 +225134,225134 +225135,225135 +225136,225136 +184568,225136 +225137,225137 +225138,225138 +225139,225139 +225140,225140 +225141,225141 +178789,225141 +225142,225142 +225143,225143 +225144,225144 +225145,225145 +91463,225145 +225146,225146 +225147,225147 +225148,225148 +225149,225149 +225150,225150 +225151,225151 +225152,225152 +225153,225153 +225154,225154 +225155,225155 +225156,225156 +225157,225157 +225158,225158 +225159,225159 +225160,225160 +225161,225161 +225162,225162 +225163,225163 +225164,225164 +225165,225165 +225166,225166 +225167,225167 +225168,225168 +225169,225169 +225170,225170 +111137,225170 +225171,225171 +225172,225172 +225173,225173 +142377,225173 +225174,225174 +225175,225175 +225176,225176 +225177,225177 +225178,225178 +225179,225179 +225180,225180 +225181,225181 +225182,225182 +225183,225183 +225184,225184 +225185,225185 +225186,225186 +225187,225187 +225188,225188 +225189,225189 +225190,225190 +225191,225191 +225192,225192 +225193,225193 +225194,225194 +225195,225195 +225196,225196 +225197,225197 +225198,225198 +225199,225199 +225200,225200 +183930,225200 +225201,225201 +225202,225202 +225203,225203 +225204,225204 +225205,225205 +225206,225206 +225207,225207 +225208,225208 +225209,225209 +225210,225210 +225211,225211 +79472,225211 +225212,225212 +225213,225213 +225214,225214 +225215,225215 +225216,225216 +225217,225217 +225218,225218 +225219,225219 +160700,225219 +225220,225220 +58055,225220 +225221,225221 +225222,225222 +225223,225223 +225224,225224 +225225,225225 +225226,225226 +225227,225227 +225228,225228 +225229,225229 +19166,225229 +225230,225230 +225231,225231 +225232,225232 +225233,225233 +225234,225234 +225235,225235 +225236,225236 +225237,225237 +225238,225238 +225239,225239 +225240,225240 +225241,225241 +184764,225241 +225242,225242 +225243,225243 +225244,225244 +225245,225245 +225246,225246 +225247,225247 +225248,225248 +225249,225249 +225250,225250 +225251,225251 +14455,225251 +225252,225252 +225253,225253 +225254,225254 +225255,225255 +225256,225256 +225257,225257 +225258,225258 +225259,225259 +225260,225260 +225261,225261 +225262,225262 +225263,225263 +225264,225264 +225265,225265 +225266,225266 +225267,225267 +225268,225268 +225269,225269 +225270,225270 +225271,225271 +225272,225272 +225273,225273 +225274,225274 +225275,225275 +108682,225275 +225276,225276 +225277,225277 +30440,225277 +225278,225278 +225279,225279 +225280,225280 +225281,225281 +225282,225282 +225283,225283 +225284,225284 +225285,225285 +225286,225286 +225287,225287 +225288,225288 +225289,225289 +225290,225290 +225291,225291 +225292,225292 +225293,225293 +225294,225294 +225295,225295 +225296,225296 +225297,225297 +225298,225298 +225299,225299 +225300,225300 +225301,225301 +225302,225302 +225303,225303 +225304,225304 +4360,225304 +225305,225305 +225306,225306 +225307,225307 +225308,225308 +225309,225309 +225310,225310 +225311,225311 +225312,225312 +225313,225313 +225314,225314 +225315,225315 +225316,225316 +19703,225316 +225317,225317 +245010,225317 +225318,225318 +225319,225319 +225320,225320 +225321,225321 +225322,225322 +225323,225323 +225324,225324 +225325,225325 +225326,225326 +225327,225327 +225328,225328 +225329,225329 +225330,225330 +225331,225331 +57252,225331 +225332,225332 +225333,225333 +6690,225333 +225372,225372 +225373,225373 +225374,225374 +225375,225375 +225376,225376 +225377,225377 +225378,225378 +140501,225378 +225379,225379 +225380,225380 +225381,225381 +225382,225382 +225383,225383 +225384,225384 +225385,225385 +225386,225386 +14092,225386 +225387,225387 +225388,225388 +225389,225389 +225390,225390 +225391,225391 +181015,225391 +225392,225392 +225393,225393 +225394,225394 +225395,225395 +225396,225396 +111714,225396 +225397,225397 +225398,225398 +225399,225399 +225460,225460 +225461,225461 +225462,225462 +225463,225463 +225464,225464 +225465,225465 +225466,225466 +225467,225467 +225468,225468 +142906,225468 +225469,225469 +225470,225470 +225471,225471 +225472,225472 +225473,225473 +225474,225474 +61427,225474 +225475,225475 +87256,225475 +225476,225476 +225477,225477 +225478,225478 +225479,225479 +225480,225480 +225481,225481 +225482,225482 +225483,225483 +225484,225484 +222171,225484 +225485,225485 +225486,225486 +225487,225487 +225488,225488 +225489,225489 +123107,225489 +225490,225490 +225491,225491 +225492,225492 +225493,225493 +225494,225494 +225495,225495 +225496,225496 +24995,225496 +225497,225497 +225498,225498 +225499,225499 +225500,225500 +225501,225501 +225502,225502 +225503,225503 +225504,225504 +225505,225505 +225506,225506 +225507,225507 +225508,225508 +225509,225509 +225510,225510 +225511,225511 +225512,225512 +225513,225513 +225514,225514 +225515,225515 +225516,225516 +225517,225517 +225518,225518 +222855,225518 +225519,225519 +225520,225520 +225521,225521 +225522,225522 +225523,225523 +225524,225524 +225525,225525 +225526,225526 +225527,225527 +225528,225528 +163076,225528 +225529,225529 +225530,225530 +225531,225531 +225532,225532 +225533,225533 +225534,225534 +225535,225535 +136202,225535 +128615,225535 +225536,225536 +225537,225537 +225538,225538 +225539,225539 +225540,225540 +225541,225541 +225542,225542 +225543,225543 +225544,225544 +225545,225545 +225546,225546 +225547,225547 +225548,225548 +67321,225548 +225549,225549 +225550,225550 +225551,225551 +225552,225552 +225553,225553 +225554,225554 +225555,225555 +225556,225556 +2871,225556 +225557,225557 +225558,225558 +225559,225559 +225560,225560 +225561,225561 +225562,225562 +225563,225563 +225564,225564 +225565,225565 +225566,225566 +225567,225567 +225568,225568 +225569,225569 +225570,225570 +225571,225571 +225572,225572 +225573,225573 +225574,225574 +225575,225575 +225576,225576 +225577,225577 +225578,225578 +225579,225579 +225580,225580 +162364,225580 +225581,225581 +225582,225582 +225583,225583 +225584,225584 +225585,225585 +225634,225634 +225635,225635 +57812,225635 +225636,225636 +225637,225637 +106746,225637 +179255,225637 +225638,225638 +225639,225639 +225640,225640 +225641,225641 +225642,225642 +225643,225643 +225644,225644 +225645,225645 +225646,225646 +225647,225647 +225648,225648 +225649,225649 +225650,225650 +225651,225651 +225652,225652 +225653,225653 +225654,225654 +225655,225655 +225656,225656 +225789,225789 +225790,225790 +225791,225791 +225792,225792 +225793,225793 +225794,225794 +225795,225795 +225796,225796 +225797,225797 +225798,225798 +225799,225799 +225800,225800 +225801,225801 +225802,225802 +225803,225803 +225804,225804 +225805,225805 +225806,225806 +225807,225807 +225808,225808 +225809,225809 +225810,225810 +69840,225810 +225811,225811 +225812,225812 +69842,225812 +225813,225813 +69843,225813 +225814,225814 +225815,225815 +225816,225816 +225817,225817 +225818,225818 +225819,225819 +225820,225820 +225821,225821 +225822,225822 +225823,225823 +225824,225824 +177488,225824 +171524,225824 +225825,225825 +225826,225826 +225827,225827 +225828,225828 +225829,225829 +77765,225829 +225830,225830 +225831,225831 +225921,225921 +225922,225922 +225923,225923 +225924,225924 +225925,225925 +225926,225926 +225927,225927 +145504,225927 +225928,225928 +225929,225929 +225930,225930 +225931,225931 +225932,225932 +122993,225932 +225933,225933 +225934,225934 +225935,225935 +225936,225936 +225937,225937 +225938,225938 +225981,225981 +157993,225981 +225982,225982 +225983,225983 +225984,225984 +225985,225985 +225986,225986 +225987,225987 +136226,225987 +128639,225987 +247960,225987 +225988,225988 +225989,225989 +225990,225990 +225991,225991 +225992,225992 +225993,225993 +225994,225994 +225995,225995 +191222,225995 +225996,225996 +225997,225997 +224412,225997 +225998,225998 +226102,226102 +185742,226102 +220385,226102 +226103,226103 +226104,226104 +226105,226105 +226106,226106 +226107,226107 +226108,226108 +226109,226109 +226110,226110 +226111,226111 +226112,226112 +226113,226113 +226114,226114 +226115,226115 +226116,226116 +226117,226117 +226118,226118 +226119,226119 +226120,226120 +111103,226120 +226121,226121 +226122,226122 +226123,226123 +226124,226124 +226125,226125 +226126,226126 +67137,226126 +226127,226127 +226128,226128 +226129,226129 +226130,226130 +226131,226131 +226132,226132 +226133,226133 +78525,226133 +226134,226134 +226135,226135 +226136,226136 +226137,226137 +226138,226138 +226139,226139 +226140,226140 +226141,226141 +226142,226142 +102166,226142 +226143,226143 +68503,226143 +226144,226144 +226145,226145 +226146,226146 +68506,226146 +226147,226147 +226148,226148 +226149,226149 +226150,226150 +226151,226151 +226152,226152 +226153,226153 +226154,226154 +226155,226155 +226156,226156 +226157,226157 +226158,226158 +226159,226159 +226160,226160 +226161,226161 +226162,226162 +226163,226163 +226164,226164 +226165,226165 +226166,226166 +226167,226167 +226168,226168 +226169,226169 +226170,226170 +226171,226171 +226172,226172 +226173,226173 +226174,226174 +226175,226175 +226176,226176 +226177,226177 +226178,226178 +226179,226179 +69843,226179 +226180,226180 +226181,226181 +226182,226182 +226183,226183 +226297,226297 +226298,226298 +226299,226299 +226300,226300 +226301,226301 +226302,226302 +226303,226303 +226304,226304 +231303,226304 +145094,226304 +226305,226305 +226306,226306 +226307,226307 +226308,226308 +244350,226308 +226309,226309 +226310,226310 +226311,226311 +76119,226311 +226312,226312 +226313,226313 +145461,226313 +226314,226314 +36293,226314 +77097,226314 +226315,226315 +226316,226316 +226317,226317 +226318,226318 +226319,226319 +226320,226320 +226321,226321 +226322,226322 +226323,226323 +226324,226324 +226325,226325 +226326,226326 +226327,226327 +226328,226328 +202583,226328 +226329,226329 +226330,226330 +226331,226331 +226332,226332 +226333,226333 +226334,226334 +226335,226335 +226336,226336 +226337,226337 +226338,226338 +226339,226339 +226340,226340 +226341,226341 +226342,226342 +226343,226343 +226344,226344 +226345,226345 +226346,226346 +56363,226346 +48259,226346 +226347,226347 +226348,226348 +226349,226349 +226350,226350 +226351,226351 +207465,226351 +226352,226352 +226353,226353 +226354,226354 +226355,226355 +226356,226356 +226357,226357 +226358,226358 +226359,226359 +226360,226360 +226361,226361 +226362,226362 +226363,226363 +52727,226363 +44623,226363 +226364,226364 +226365,226365 +226366,226366 +226367,226367 +226368,226368 +226369,226369 +157152,226369 +226370,226370 +226371,226371 +226372,226372 +226373,226373 +226374,226374 +226375,226375 +226376,226376 +226377,226377 +226378,226378 +226379,226379 +226380,226380 +226381,226381 +24450,226381 +226382,226382 +226383,226383 +226384,226384 +178756,226384 +226385,226385 +226386,226386 +226387,226387 +226388,226388 +226389,226389 +226390,226390 +226391,226391 +226392,226392 +226393,226393 +226394,226394 +226395,226395 +226396,226396 +226397,226397 +226398,226398 +226399,226399 +226400,226400 +226401,226401 +226402,226402 +226403,226403 +220081,226403 +226404,226404 +226405,226405 +226406,226406 +226407,226407 +226408,226408 +226409,226409 +226410,226410 +226411,226411 +226412,226412 +226413,226413 +226414,226414 +226415,226415 +226416,226416 +226417,226417 +226418,226418 +226419,226419 +226420,226420 +84766,226420 +226421,226421 +226422,226422 +226423,226423 +226424,226424 +226425,226425 +226426,226426 +226427,226427 +226428,226428 +226429,226429 +226430,226430 +226431,226431 +226432,226432 +226433,226433 +226434,226434 +226435,226435 +226472,226472 +226473,226473 +226474,226474 +226475,226475 +226476,226476 +226477,226477 +226478,226478 +226479,226479 +226480,226480 +226481,226481 +71099,226481 +226482,226482 +226483,226483 +226484,226484 +226485,226485 +226486,226486 +226487,226487 +226488,226488 +226489,226489 +226490,226490 +226491,226491 +226492,226492 +226493,226493 +226494,226494 +226495,226495 +226496,226496 +226497,226497 +226498,226498 +226499,226499 +226500,226500 +226501,226501 +226502,226502 +226503,226503 +226504,226504 +226505,226505 +197607,226505 +226506,226506 +226507,226507 +226508,226508 +226509,226509 +226510,226510 +226511,226511 +226512,226512 +226513,226513 +226514,226514 +226515,226515 +226516,226516 +80429,226516 +226517,226517 +226518,226518 +226519,226519 +226520,226520 +226521,226521 +226522,226522 +226523,226523 +226524,226524 +226525,226525 +78796,226525 +226526,226526 +226527,226527 +226528,226528 +226529,226529 +226530,226530 +226531,226531 +226532,226532 +157152,226532 +226533,226533 +226534,226534 +226535,226535 +226536,226536 +226537,226537 +226538,226538 +226539,226539 +226540,226540 +226541,226541 +101996,226541 +226542,226542 +226543,226543 +226544,226544 +226545,226545 +226546,226546 +226547,226547 +226548,226548 +226549,226549 +226550,226550 +226593,226593 +226594,226594 +226595,226595 +51118,226595 +43014,226595 +226596,226596 +191803,226596 +226597,226597 +226598,226598 +226599,226599 +226600,226600 +226601,226601 +226602,226602 +226603,226603 +226604,226604 +226605,226605 +226606,226606 +226607,226607 +226608,226608 +226609,226609 +189792,226609 +226610,226610 +108188,226610 +226611,226611 +226612,226612 +226613,226613 +226614,226614 +226615,226615 +226616,226616 +226617,226617 +226618,226618 +226619,226619 +226620,226620 +226621,226621 +226622,226622 +226623,226623 +226624,226624 +226625,226625 +226626,226626 +148129,226626 +86436,226626 +226627,226627 +226628,226628 +226629,226629 +226630,226630 +226631,226631 +203474,226631 +226632,226632 +226633,226633 +226634,226634 +226635,226635 +226636,226636 +226637,226637 +81501,226637 +226638,226638 +226639,226639 +226640,226640 +226641,226641 +226642,226642 +9624,226642 +226643,226643 +226644,226644 +226645,226645 +226646,226646 +101840,226646 +226647,226647 +226648,226648 +226649,226649 +226650,226650 +226651,226651 +226652,226652 +226653,226653 +226654,226654 +226655,226655 +226656,226656 +226657,226657 +226658,226658 +226659,226659 +226660,226660 +70370,226660 +226661,226661 +132310,226661 +124723,226661 +226662,226662 +226663,226663 +226664,226664 +263,226664 +226665,226665 +226666,226666 +36909,226666 +226667,226667 +226668,226668 +209666,226668 +226669,226669 +226670,226670 +77781,226670 +159374,226670 +226671,226671 +226672,226672 +226673,226673 +226674,226674 +137178,226674 +129591,226674 +226675,226675 +226676,226676 +226677,226677 +226678,226678 +226679,226679 +226680,226680 +226681,226681 +226682,226682 +226683,226683 +157808,226683 +226684,226684 +226685,226685 +226686,226686 +226687,226687 +226688,226688 +226689,226689 +226690,226690 +144887,226690 +226691,226691 +226692,226692 +226693,226693 +226694,226694 +226695,226695 +226696,226696 +226697,226697 +226698,226698 +226699,226699 +226700,226700 +226701,226701 +226702,226702 +226703,226703 +226704,226704 +226705,226705 +226706,226706 +226707,226707 +226708,226708 +226709,226709 +226710,226710 +226711,226711 +226712,226712 +226713,226713 +226714,226714 +226715,226715 +226716,226716 +226717,226717 +226718,226718 +80845,226718 +226719,226719 +226720,226720 +226721,226721 +226722,226722 +226723,226723 +226724,226724 +226725,226725 +226726,226726 +226727,226727 +226728,226728 +226729,226729 +188529,226729 +226730,226730 +226731,226731 +226732,226732 +226733,226733 +226734,226734 +226735,226735 +226736,226736 +226737,226737 +226738,226738 +226739,226739 +226740,226740 +226741,226741 +226742,226742 +226743,226743 +226744,226744 +226745,226745 +226746,226746 +226747,226747 +226748,226748 +226749,226749 +226750,226750 +226751,226751 +216043,226751 +226752,226752 +226753,226753 +226754,226754 +226755,226755 +226756,226756 +226757,226757 +226758,226758 +226759,226759 +29947,226759 +226760,226760 +226761,226761 +226762,226762 +226763,226763 +226764,226764 +226765,226765 +226766,226766 +226767,226767 +226768,226768 +87602,226768 +226769,226769 +185405,226769 +226770,226770 +155135,226770 +226771,226771 +226772,226772 +226773,226773 +226774,226774 +226775,226775 +226776,226776 +226777,226777 +226778,226778 +226779,226779 +226780,226780 +221164,226780 +226781,226781 +226782,226782 +226783,226783 +226784,226784 +226785,226785 +226786,226786 +226787,226787 +226788,226788 +226789,226789 +226790,226790 +226791,226791 +226792,226792 +226793,226793 +226794,226794 +226795,226795 +226796,226796 +226797,226797 +226798,226798 +226799,226799 +226800,226800 +226801,226801 +226802,226802 +226803,226803 +226804,226804 +226805,226805 +226806,226806 +226807,226807 +226808,226808 +226809,226809 +226810,226810 +226811,226811 +226812,226812 +226813,226813 +226814,226814 +226815,226815 +226816,226816 +226817,226817 +226818,226818 +226819,226819 +226820,226820 +226821,226821 +226822,226822 +226823,226823 +211937,226823 +226824,226824 +226825,226825 +226826,226826 +226827,226827 +226828,226828 +226829,226829 +15068,226829 +226830,226830 +226831,226831 +226832,226832 +226833,226833 +226834,226834 +226835,226835 +226836,226836 +226837,226837 +226838,226838 +246701,226838 +226839,226839 +226840,226840 +226841,226841 +226842,226842 +226843,226843 +226844,226844 +226845,226845 +147057,226845 +226846,226846 +226847,226847 +226848,226848 +226849,226849 +226850,226850 +226851,226851 +226852,226852 +226853,226853 +226854,226854 +226855,226855 +226856,226856 +226857,226857 +226858,226858 +226859,226859 +226860,226860 +226861,226861 +226862,226862 +226863,226863 +226864,226864 +226865,226865 +226866,226866 +226867,226867 +226868,226868 +226869,226869 +226870,226870 +226871,226871 +226872,226872 +226873,226873 +226874,226874 +226875,226875 +226876,226876 +226877,226877 +226878,226878 +226879,226879 +226880,226880 +226881,226881 +226882,226882 +226883,226883 +226884,226884 +226885,226885 +226886,226886 +226887,226887 +226888,226888 +226889,226889 +226890,226890 +226891,226891 +226892,226892 +226893,226893 +226894,226894 +226895,226895 +226896,226896 +226897,226897 +226898,226898 +226899,226899 +226900,226900 +226901,226901 +226902,226902 +226903,226903 +226904,226904 +226984,226984 +226985,226985 +226986,226986 +226987,226987 +226988,226988 +226989,226989 +226990,226990 +226991,226991 +226992,226992 +226993,226993 +226994,226994 +226995,226995 +226996,226996 +226997,226997 +226998,226998 +226999,226999 +200463,226999 +227000,227000 +227001,227001 +227002,227002 +227003,227003 +227004,227004 +227005,227005 +227006,227006 +227007,227007 +227008,227008 +227009,227009 +227010,227010 +227011,227011 +227012,227012 +227013,227013 +227014,227014 +227015,227015 +227016,227016 +227017,227017 +227018,227018 +227019,227019 +150834,227019 +227020,227020 +164937,227020 +227021,227021 +227022,227022 +227023,227023 +227024,227024 +227025,227025 +165038,227025 +227026,227026 +227027,227027 +227028,227028 +227029,227029 +227030,227030 +227031,227031 +227032,227032 +227033,227033 +227034,227034 +227035,227035 +227036,227036 +227037,227037 +227038,227038 +227039,227039 +227040,227040 +227041,227041 +227042,227042 +227043,227043 +227044,227044 +227045,227045 +227046,227046 +227047,227047 +227048,227048 +227049,227049 +227050,227050 +227051,227051 +227052,227052 +227053,227053 +227054,227054 +227055,227055 +227056,227056 +227057,227057 +71193,227057 +227058,227058 +227059,227059 +227060,227060 +148191,227060 +227061,227061 +227062,227062 +227063,227063 +227064,227064 +227065,227065 +227066,227066 +227067,227067 +227068,227068 +227069,227069 +227070,227070 +227071,227071 +227072,227072 +216668,227072 +227073,227073 +227074,227074 +227075,227075 +227076,227076 +227077,227077 +227078,227078 +227079,227079 +227080,227080 +227081,227081 +227082,227082 +227083,227083 +198702,227083 +227084,227084 +141394,227084 +227085,227085 +227086,227086 +227087,227087 +227088,227088 +227089,227089 +227090,227090 +227091,227091 +227092,227092 +227093,227093 +227094,227094 +227095,227095 +227096,227096 +227097,227097 +227098,227098 +227099,227099 +227100,227100 +58119,227100 +57968,227100 +227101,227101 +227102,227102 +227103,227103 +227104,227104 +227105,227105 +227106,227106 +227107,227107 +227108,227108 +227109,227109 +227110,227110 +227111,227111 +227112,227112 +227113,227113 +227114,227114 +227115,227115 +227116,227116 +67282,227116 +77330,227116 +227117,227117 +227118,227118 +227119,227119 +227120,227120 +227121,227121 +209962,227121 +227122,227122 +227123,227123 +227124,227124 +227125,227125 +227126,227126 +227127,227127 +227165,227165 +227166,227166 +227167,227167 +227168,227168 +227494,227494 +227495,227495 +77145,227495 +227496,227496 +227497,227497 +227498,227498 +227499,227499 +227500,227500 +227501,227501 +227502,227502 +227503,227503 +227504,227504 +227505,227505 +227506,227506 +227507,227507 +227508,227508 +227509,227509 +227510,227510 +227511,227511 +227512,227512 +227513,227513 +227514,227514 +71376,227514 +227515,227515 +227516,227516 +58478,227516 +227517,227517 +227518,227518 +227519,227519 +227520,227520 +227521,227521 +227522,227522 +227523,227523 +227524,227524 +227525,227525 +227526,227526 +227527,227527 +227528,227528 +227529,227529 +227530,227530 +227531,227531 +227532,227532 +227533,227533 +227534,227534 +227535,227535 +227536,227536 +227537,227537 +227538,227538 +227539,227539 +227540,227540 +227541,227541 +227542,227542 +227543,227543 +227544,227544 +227545,227545 +227546,227546 +227547,227547 +227548,227548 +227549,227549 +227550,227550 +150558,227550 +227551,227551 +227552,227552 +227553,227553 +227554,227554 +227555,227555 +151137,227555 +227556,227556 +28752,227556 +227557,227557 +227558,227558 +154852,227558 +227559,227559 +227560,227560 +227561,227561 +227562,227562 +227563,227563 +234304,227563 +227564,227564 +227565,227565 +227566,227566 +227567,227567 +227568,227568 +227569,227569 +227570,227570 +108704,227570 +227571,227571 +227572,227572 +227573,227573 +227574,227574 +227575,227575 +227576,227576 +28763,227576 +227577,227577 +227578,227578 +227579,227579 +227580,227580 +227581,227581 +227582,227582 +227583,227583 +227584,227584 +227585,227585 +227586,227586 +227587,227587 +227588,227588 +227589,227589 +227590,227590 +227591,227591 +227592,227592 +227593,227593 +227594,227594 +227595,227595 +227596,227596 +227597,227597 +227598,227598 +227599,227599 +227600,227600 +227601,227601 +63477,227601 +227602,227602 +64202,227602 +227603,227603 +227604,227604 +227605,227605 +227606,227606 +186316,227606 +227607,227607 +151189,227607 +227608,227608 +227609,227609 +227610,227610 +227611,227611 +227612,227612 +151194,227612 +227613,227613 +227614,227614 +151196,227614 +227615,227615 +227616,227616 +151198,227616 +227617,227617 +227618,227618 +227619,227619 +74247,227619 +227620,227620 +227621,227621 +151203,227621 +227622,227622 +227623,227623 +227624,227624 +227625,227625 +227626,227626 +227627,227627 +35826,227627 +227628,227628 +227629,227629 +227630,227630 +227631,227631 +227632,227632 +227633,227633 +227634,227634 +227635,227635 +227636,227636 +227637,227637 +227638,227638 +227639,227639 +227640,227640 +227641,227641 +227642,227642 +227643,227643 +227644,227644 +227645,227645 +227646,227646 +227647,227647 +227648,227648 +227649,227649 +16249,227649 +227650,227650 +227651,227651 +227652,227652 +212898,227652 +227653,227653 +203022,227653 +227654,227654 +146985,227654 +227703,227703 +174165,227703 +168201,227703 +63403,227703 +227704,227704 +241778,227704 +227705,227705 +227706,227706 +227707,227707 +227708,227708 +227709,227709 +227710,227710 +227711,227711 +227712,227712 +227713,227713 +227714,227714 +81870,227714 +227715,227715 +227716,227716 +227717,227717 +227718,227718 +207604,227718 +227719,227719 +227720,227720 +207876,227720 +227721,227721 +227722,227722 +227723,227723 +227724,227724 +74708,227724 +227725,227725 +227726,227726 +227727,227727 +97855,227727 +227728,227728 +227729,227729 +227730,227730 +227731,227731 +227732,227732 +227733,227733 +227734,227734 +227735,227735 +227736,227736 +227737,227737 +227738,227738 +227739,227739 +227740,227740 +227741,227741 +227742,227742 +227743,227743 +227744,227744 +227745,227745 +227746,227746 +227747,227747 +227748,227748 +227749,227749 +227750,227750 +227751,227751 +227752,227752 +227753,227753 +227754,227754 +227755,227755 +56122,227755 +48018,227755 +227756,227756 +227757,227757 +227758,227758 +142640,227758 +227759,227759 +227760,227760 +7202,227760 +227761,227761 +227762,227762 +227763,227763 +80425,227763 +227764,227764 +227765,227765 +227766,227766 +227767,227767 +227768,227768 +227769,227769 +227770,227770 +227771,227771 +227772,227772 +227773,227773 +227774,227774 +203866,227774 +227775,227775 +227776,227776 +227777,227777 +227778,227778 +227779,227779 +227780,227780 +227781,227781 +88530,227781 +227782,227782 +227783,227783 +227784,227784 +227785,227785 +227786,227786 +227787,227787 +227788,227788 +227789,227789 +227790,227790 +227791,227791 +209736,227791 +227792,227792 +227793,227793 +227794,227794 +227795,227795 +227796,227796 +227797,227797 +227798,227798 +227799,227799 +227800,227800 +227801,227801 +227802,227802 +227803,227803 +227804,227804 +227805,227805 +227806,227806 +227807,227807 +227808,227808 +227809,227809 +116388,227809 +227810,227810 +227811,227811 +227812,227812 +227813,227813 +227814,227814 +227815,227815 +227816,227816 +227817,227817 +227818,227818 +227819,227819 +227820,227820 +227821,227821 +227822,227822 +227823,227823 +227824,227824 +227825,227825 +227826,227826 +227827,227827 +227828,227828 +227829,227829 +227830,227830 +227831,227831 +227832,227832 +227833,227833 +227834,227834 +227835,227835 +227836,227836 +227837,227837 +227838,227838 +227839,227839 +211324,227839 +227840,227840 +227841,227841 +227842,227842 +227843,227843 +227844,227844 +227845,227845 +227846,227846 +227847,227847 +227848,227848 +227849,227849 +135767,227849 +128180,227849 +227850,227850 +227851,227851 +227852,227852 +227853,227853 +227854,227854 +227855,227855 +227856,227856 +227857,227857 +227858,227858 +227859,227859 +227860,227860 +227861,227861 +227862,227862 +227863,227863 +227864,227864 +227865,227865 +227866,227866 +227867,227867 +227868,227868 +227869,227869 +227870,227870 +227871,227871 +227872,227872 +227873,227873 +227874,227874 +227875,227875 +227876,227876 +227877,227877 +227878,227878 +227954,227954 +227955,227955 +227956,227956 +227957,227957 +227958,227958 +227959,227959 +227960,227960 +227961,227961 +227962,227962 +227963,227963 +227964,227964 +184451,227964 +227965,227965 +227966,227966 +239519,227966 +227967,227967 +227968,227968 +190606,227968 +227969,227969 +227970,227970 +227971,227971 +227972,227972 +227973,227973 +227974,227974 +227975,227975 +227976,227976 +228012,228012 +228013,228013 +228014,228014 +228015,228015 +228016,228016 +228017,228017 +83013,228017 +228018,228018 +228019,228019 +228020,228020 +228021,228021 +228022,228022 +228023,228023 +228024,228024 +236943,228024 +228025,228025 +228026,228026 +228027,228027 +228028,228028 +228029,228029 +228030,228030 +228031,228031 +228032,228032 +228033,228033 +228034,228034 +228035,228035 +35746,228035 +228036,228036 +228037,228037 +228038,228038 +228039,228039 +228040,228040 +228041,228041 +228042,228042 +228043,228043 +228044,228044 +228045,228045 +228046,228046 +228047,228047 +228048,228048 +214259,228048 +228049,228049 +228050,228050 +228051,228051 +228052,228052 +143344,228052 +228053,228053 +228054,228054 +228055,228055 +228056,228056 +228057,228057 +228058,228058 +228059,228059 +228060,228060 +228061,228061 +228062,228062 +228700,228700 +228701,228701 +61468,228701 +228702,228702 +228703,228703 +228704,228704 +228705,228705 +228706,228706 +228707,228707 +228708,228708 +228709,228709 +228710,228710 +228711,228711 +228712,228712 +228713,228713 +228714,228714 +68202,228714 +228715,228715 +228716,228716 +228717,228717 +30516,228717 +228718,228718 +78647,228718 +228719,228719 +157594,228719 +228720,228720 +228721,228721 +228722,228722 +228723,228723 +228724,228724 +72585,228724 +228725,228725 +50904,228725 +42800,228725 +228726,228726 +228727,228727 +228728,228728 +228729,228729 +9069,228729 +228730,228730 +228731,228731 +228732,228732 +228733,228733 +228734,228734 +228735,228735 +228736,228736 +228737,228737 +228738,228738 +228739,228739 +228740,228740 +228741,228741 +228742,228742 +228743,228743 +228744,228744 +228745,228745 +228746,228746 +228747,228747 +228748,228748 +228749,228749 +228750,228750 +228751,228751 +228752,228752 +228753,228753 +228754,228754 +228755,228755 +228756,228756 +228757,228757 +228758,228758 +228759,228759 +228760,228760 +228761,228761 +228843,228843 +228844,228844 +228845,228845 +228846,228846 +228847,228847 +228848,228848 +39937,228848 +228849,228849 +228850,228850 +228851,228851 +228852,228852 +228853,228853 +189027,228853 +228854,228854 +228966,228966 +228967,228967 +228968,228968 +8947,228968 +228969,228969 +228970,228970 +228971,228971 +228972,228972 +228973,228973 +228974,228974 +228975,228975 +228976,228976 +228977,228977 +228978,228978 +228979,228979 +157836,228979 +228980,228980 +228981,228981 +228982,228982 +228983,228983 +228984,228984 +228985,228985 +229105,229105 +229106,229106 +229107,229107 +229108,229108 +229109,229109 +229796,229109 +229110,229110 +109038,229110 +229111,229111 +229112,229112 +229113,229113 +229114,229114 +229115,229115 +229116,229116 +232120,229116 +229117,229117 +229118,229118 +229119,229119 +229120,229120 +229160,229160 +229161,229161 +208471,229161 +229162,229162 +229163,229163 +132101,229163 +124514,229163 +229164,229164 +103945,229164 +105185,229164 +229165,229165 +229166,229166 +229167,229167 +229168,229168 +229169,229169 +229170,229170 +229171,229171 +229172,229172 +229173,229173 +229174,229174 +229175,229175 +229176,229176 +229177,229177 +187748,229177 +229178,229178 +229179,229179 +229180,229180 +229181,229181 +229220,229220 +229221,229221 +229222,229222 +229223,229223 +229224,229224 +229225,229225 +229226,229226 +229556,229556 +229557,229557 +229558,229558 +229559,229559 +143083,229559 +189484,229559 +229560,229560 +229561,229561 +229562,229562 +229563,229563 +229564,229564 +229565,229565 +229566,229566 +229567,229567 +155039,229567 +229568,229568 +229569,229569 +3723,229569 +229570,229570 +229571,229571 +229572,229572 +229573,229573 +62609,229573 +229574,229574 +62610,229574 +229575,229575 +54987,229575 +46883,229575 +229576,229576 +229577,229577 +229578,229578 +133758,229578 +126171,229578 +229579,229579 +229580,229580 +229581,229581 +229582,229582 +229583,229583 +203571,229583 +229584,229584 +202803,229584 +229585,229585 +229586,229586 +229587,229587 +229588,229588 +38205,229588 +229589,229589 +229590,229590 +229591,229591 +139713,229591 +229592,229592 +229655,229655 +229656,229656 +229657,229657 +164388,229657 +229658,229658 +229659,229659 +229660,229660 +229661,229661 +229662,229662 +229700,229700 +229701,229701 +229702,229702 +229703,229703 +60227,229703 +229704,229704 +229705,229705 +229706,229706 +229707,229707 +242897,229707 +229708,229708 +229709,229709 +229710,229710 +229711,229711 +229712,229712 +229713,229713 +229714,229714 +229715,229715 +229716,229716 +229717,229717 +229718,229718 +229719,229719 +122068,229719 +229720,229720 +229721,229721 +233359,229721 +229722,229722 +229723,229723 +229724,229724 +229725,229725 +229766,229766 +229767,229767 +229768,229768 +229769,229769 +229770,229770 +229771,229771 +229772,229772 +4301,229772 +229773,229773 +229774,229774 +229775,229775 +229776,229776 +229777,229777 +229778,229778 +249437,229778 +229779,229779 +229780,229780 +229781,229781 +229782,229782 +229783,229783 +229784,229784 +229785,229785 +229786,229786 +229787,229787 +229788,229788 +229789,229789 +229790,229790 +229791,229791 +229792,229792 +229793,229793 +229794,229794 +229795,229795 +229796,229796 +229109,229796 +229797,229797 +229798,229798 +229799,229799 +184308,229799 +229800,229800 +229801,229801 +229802,229802 +2873,229802 +229803,229803 +229804,229804 +206092,229804 +229805,229805 +229806,229806 +10536,229806 +229807,229807 +229808,229808 +20455,229808 +229809,229809 +229810,229810 +229811,229811 +229812,229812 +229813,229813 +229814,229814 +229815,229815 +229860,229860 +229861,229861 +229862,229862 +229863,229863 +229864,229864 +229865,229865 +229866,229866 +229867,229867 +229868,229868 +229869,229869 +229870,229870 +229871,229871 +229872,229872 +229873,229873 +229874,229874 +229965,229965 +229966,229966 +229967,229967 +229968,229968 +229969,229969 +229970,229970 +229971,229971 +229972,229972 +229973,229973 +229974,229974 +229975,229975 +229976,229976 +229977,229977 +229978,229978 +229979,229979 +229980,229980 +229981,229981 +229982,229982 +229983,229983 +229984,229984 +229985,229985 +162991,229985 +229986,229986 +229987,229987 +229988,229988 +229989,229989 +229990,229990 +229991,229991 +229992,229992 +229993,229993 +229994,229994 +229995,229995 +229996,229996 +229997,229997 +229998,229998 +229999,229999 +230000,230000 +230001,230001 +230002,230002 +230003,230003 +230004,230004 +230005,230005 +230006,230006 +230007,230007 +230008,230008 +230009,230009 +230244,230244 +230245,230245 +230246,230246 +230247,230247 +230248,230248 +230249,230249 +230250,230250 +230251,230251 +230252,230252 +230253,230253 +230254,230254 +230255,230255 +230256,230256 +230257,230257 +230258,230258 +230259,230259 +60064,230259 +230340,230340 +230341,230341 +230342,230342 +230343,230343 +230344,230344 +33891,230344 +59647,230344 +230345,230345 +150710,230345 +230346,230346 +230347,230347 +230348,230348 +230349,230349 +230350,230350 +230351,230351 +230352,230352 +230353,230353 +230354,230354 +230355,230355 +230356,230356 +238275,230356 +230357,230357 +76034,230357 +233998,230357 +230358,230358 +109818,230358 +230359,230359 +230360,230360 +230361,230361 +121569,230361 +230362,230362 +88965,230362 +230363,230363 +230364,230364 +230365,230365 +208296,230365 +230366,230366 +230367,230367 +230368,230368 +230369,230369 +230370,230370 +230371,230371 +230372,230372 +75204,230372 +230373,230373 +230374,230374 +230375,230375 +230376,230376 +230377,230377 +230378,230378 +230379,230379 +32868,230379 +230380,230380 +230381,230381 +230382,230382 +230383,230383 +230384,230384 +230385,230385 +230386,230386 +230387,230387 +230388,230388 +230389,230389 +230390,230390 +230391,230391 +230392,230392 +230393,230393 +230394,230394 +152950,230394 +230395,230395 +230396,230396 +230397,230397 +90752,230397 +230398,230398 +70200,230398 +1961,230398 +230399,230399 +153153,230399 +230400,230400 +155606,230400 +230401,230401 +230402,230402 +230539,230539 +230540,230540 +230541,230541 +230542,230542 +230543,230543 +230544,230544 +230545,230545 +230546,230546 +230547,230547 +230548,230548 +6774,230548 +230549,230549 +230550,230550 +230551,230551 +230552,230552 +230553,230553 +230554,230554 +230555,230555 +230556,230556 +230557,230557 +230558,230558 +148441,230558 +230559,230559 +230560,230560 +230561,230561 +230562,230562 +230563,230563 +230564,230564 +160361,230564 +230565,230565 +230566,230566 +230567,230567 +230568,230568 +230569,230569 +230570,230570 +230571,230571 +230572,230572 +230573,230573 +230574,230574 +230575,230575 +230576,230576 +230577,230577 +230578,230578 +230579,230579 +230580,230580 +230581,230581 +230582,230582 +230583,230583 +240773,230584 +230584,230584 +230697,230697 +230698,230698 +230699,230699 +230700,230700 +230701,230701 +230702,230702 +230703,230703 +11186,230703 +230704,230704 +230705,230705 +230706,230706 +230707,230707 +230708,230708 +230709,230709 +230710,230710 +230749,230749 +230866,230866 +230867,230867 +230868,230868 +230869,230869 +153641,230869 +230870,230870 +78652,230870 +230871,230871 +230872,230872 +230873,230873 +230874,230874 +230875,230875 +230876,230876 +230877,230877 +230878,230878 +230879,230879 +230880,230880 +230881,230881 +247902,230881 +230882,230882 +230922,230922 +230923,230923 +230924,230924 +230925,230925 +230926,230926 +230927,230927 +230928,230928 +230929,230929 +230930,230930 +230931,230931 +230932,230932 +230933,230933 +230934,230934 +230935,230935 +230936,230936 +230937,230937 +230938,230938 +230939,230939 +230940,230940 +230941,230941 +230942,230942 +230943,230943 +230944,230944 +230945,230945 +230946,230946 +230947,230947 +230948,230948 +230949,230949 +230950,230950 +230951,230951 +230952,230952 +230953,230953 +230954,230954 +230955,230955 +230956,230956 +199570,230956 +230957,230957 +230958,230958 +40223,230958 +230959,230959 +231197,231197 +231198,231198 +231199,231199 +107648,231199 +231200,231200 +231201,231201 +231202,231202 +231947,231203 +231203,231203 +118646,231203 +231204,231204 +205711,231204 +231205,231205 +231206,231206 +6587,231206 +231207,231207 +231208,231208 +231209,231209 +88970,231209 +231210,231210 +231211,231211 +76925,231211 +231212,231212 +249206,231212 +231213,231213 +231214,231214 +231215,231215 +231216,231216 +231217,231217 +231218,231218 +241041,231218 +231219,231219 +231220,231220 +231221,231221 +231222,231222 +231223,231223 +231224,231224 +231225,231225 +231226,231226 +231227,231227 +231228,231228 +231229,231229 +231230,231230 +103506,231230 +231231,231231 +231232,231232 +231233,231233 +231234,231234 +231235,231235 +231236,231236 +231237,231237 +231238,231238 +231239,231239 +231240,231240 +9847,231240 +231241,231241 +231242,231242 +208789,231242 +231243,231243 +90504,231243 +231244,231244 +231245,231245 +195517,231245 +231246,231246 +231247,231247 +231248,231248 +231249,231249 +218378,231249 +231250,231250 +231251,231251 +220620,231251 +231252,231252 +231253,231253 +231254,231254 +231255,231255 +231256,231256 +231257,231257 +231258,231258 +231259,231259 +231260,231260 +231261,231261 +231262,231262 +231263,231263 +231264,231264 +231265,231265 +231266,231266 +231267,231267 +231268,231268 +231269,231269 +231270,231270 +231271,231271 +231272,231272 +231273,231273 +231274,231274 +231275,231275 +231276,231276 +231277,231277 +231278,231278 +231279,231279 +231280,231280 +231281,231281 +162674,231281 +231282,231282 +231283,231283 +179690,231283 +231284,231284 +146699,231284 +231285,231285 +231286,231286 +231287,231287 +231288,231288 +231289,231289 +203694,231289 +231290,231290 +211186,231290 +231291,231291 +231292,231292 +242363,231292 +231293,231293 +231294,231294 +231295,231295 +231296,231296 +231297,231297 +231298,231298 +186795,231298 +231299,231299 +141139,231299 +231300,231300 +231301,231301 +231302,231302 +40773,231302 +231303,231303 +145094,231303 +226304,231303 +231304,231304 +231305,231305 +211135,231305 +231306,231306 +231307,231307 +231308,231308 +231309,231309 +231310,231310 +231311,231311 +231312,231312 +231313,231313 +231314,231314 +18941,231314 +231315,231315 +231316,231316 +231317,231317 +231318,231318 +231319,231319 +231320,231320 +231321,231321 +231322,231322 +231323,231323 +231324,231324 +231325,231325 +231326,231326 +207276,231326 +231327,231327 +231328,231328 +231329,231329 +231330,231330 +231331,231331 +231332,231332 +231333,231333 +231334,231334 +231335,231335 +231336,231336 +231337,231337 +231338,231338 +231339,231339 +231340,231340 +231341,231341 +231342,231342 +231343,231343 +231344,231344 +231345,231345 +231346,231346 +231347,231347 +231348,231348 +231349,231349 +92608,231349 +231350,231350 +231351,231351 +231352,231352 +231353,231353 +231354,231354 +231355,231355 +231356,231356 +231382,231382 +231383,231383 +80545,231383 +231384,231384 +231385,231385 +231386,231386 +231387,231387 +231388,231388 +231389,231389 +231390,231390 +231391,231391 +231392,231392 +231393,231393 +231394,231394 +231395,231395 +231396,231396 +231397,231397 +231398,231398 +231399,231399 +231400,231400 +231401,231401 +231402,231402 +231403,231403 +231404,231404 +231405,231405 +231406,231406 +231407,231407 +231408,231408 +231409,231409 +231410,231410 +231411,231411 +231412,231412 +231413,231413 +231414,231414 +231415,231415 +231416,231416 +231417,231417 +231418,231418 +231419,231419 +249430,231419 +231420,231420 +231421,231421 +233357,231421 +231422,231422 +231423,231423 +231424,231424 +231425,231425 +231426,231426 +231427,231427 +231428,231428 +231429,231429 +7664,231429 +231430,231430 +231431,231431 +231432,231432 +231433,231433 +231434,231434 +231435,231435 +231436,231436 +231437,231437 +231438,231438 +18522,231438 +231439,231439 +231440,231440 +231441,231441 +231442,231442 +231443,231443 +231444,231444 +231445,231445 +231446,231446 +231447,231447 +231448,231448 +231449,231449 +216813,231449 +231450,231450 +231451,231451 +231452,231452 +231453,231453 +231454,231454 +202582,231454 +14554,231454 +231455,231455 +231456,231456 +231457,231457 +231458,231458 +231459,231459 +231460,231460 +231461,231461 +231462,231462 +231463,231463 +231464,231464 +231465,231465 +231466,231466 +231467,231467 +231468,231468 +231469,231469 +231470,231470 +231471,231471 +231472,231472 +92091,231472 +231473,231473 +231474,231474 +231475,231475 +231476,231476 +231477,231477 +231478,231478 +231479,231479 +231480,231480 +231588,231588 +231589,231589 +231590,231590 +158590,231590 +231591,231591 +231639,231639 +217290,231639 +231640,231640 +231641,231641 +231642,231642 +231643,231643 +231644,231644 +231645,231645 +231646,231646 +231647,231647 +231648,231648 +231649,231649 +231650,231650 +231651,231651 +231652,231652 +231653,231653 +231654,231654 +231655,231655 +231656,231656 +231657,231657 +231658,231658 +231659,231659 +231660,231660 +231661,231661 +104441,231661 +231662,231662 +231663,231663 +231664,231664 +231665,231665 +231704,231704 +231705,231705 +231706,231706 +231707,231707 +231708,231708 +142780,231708 +231709,231709 +231710,231710 +231711,231711 +231712,231712 +231713,231713 +231714,231714 +231715,231715 +231716,231716 +231717,231717 +231718,231718 +231719,231719 +231720,231720 +231721,231721 +231722,231722 +231723,231723 +231724,231724 +231725,231725 +231726,231726 +231727,231727 +231728,231728 +231729,231729 +231730,231730 +231731,231731 +231732,231732 +58054,231732 +231733,231733 +231734,231734 +231735,231735 +231736,231736 +231737,231737 +231738,231738 +231739,231739 +231740,231740 +9278,231740 +231741,231741 +231742,231742 +231743,231743 +231744,231744 +231745,231745 +231746,231746 +231747,231747 +207837,231747 +231748,231748 +231749,231749 +231750,231750 +231751,231751 +231752,231752 +231753,231753 +231754,231754 +231755,231755 +231756,231756 +68621,231756 +231757,231757 +231758,231758 +231759,231759 +191896,231759 +231760,231760 +231761,231761 +231762,231762 +231763,231763 +231764,231764 +231765,231765 +231766,231766 +231767,231767 +231768,231768 +231769,231769 +80645,231769 +231770,231770 +231771,231771 +231772,231772 +231773,231773 +231774,231774 +231775,231775 +231776,231776 +231777,231777 +231778,231778 +142165,231778 +231779,231779 +231780,231780 +231781,231781 +231782,231782 +231783,231783 +231784,231784 +231785,231785 +231786,231786 +231787,231787 +231788,231788 +231789,231789 +231790,231790 +231791,231791 +231792,231792 +231793,231793 +231794,231794 +231795,231795 +19194,231795 +231796,231796 +231797,231797 +57614,231797 +231798,231798 +231799,231799 +57616,231799 +231800,231800 +57617,231800 +231801,231801 +57618,231801 +231802,231802 +231803,231803 +231804,231804 +231805,231805 +231806,231806 +231807,231807 +231808,231808 +231809,231809 +231810,231810 +231811,231811 +231812,231812 +65869,231812 +231813,231813 +231814,231814 +231815,231815 +231816,231816 +231817,231817 +231818,231818 +231819,231819 +231820,231820 +231821,231821 +231822,231822 +231823,231823 +231824,231824 +231825,231825 +88615,231825 +59536,231825 +231826,231826 +231827,231827 +203612,231827 +231828,231828 +132383,231828 +124796,231828 +231829,231829 +231830,231830 +134073,231830 +126486,231830 +231831,231831 +231832,231832 +231833,231833 +231834,231834 +231835,231835 +231836,231836 +231837,231837 +231838,231838 +231839,231839 +231840,231840 +231841,231841 +231842,231842 +231843,231843 +231844,231844 +231845,231845 +231846,231846 +185445,231846 +231847,231847 +231848,231848 +231849,231849 +231850,231850 +231851,231851 +231852,231852 +231853,231853 +231854,231854 +231855,231855 +231856,231856 +231857,231857 +231858,231858 +231859,231859 +231860,231860 +231861,231861 +244098,231861 +231862,231862 +231863,231863 +231864,231864 +231865,231865 +207919,231865 +231866,231866 +231867,231867 +231868,231868 +231869,231869 +231870,231870 +231871,231871 +231872,231872 +231873,231873 +231874,231874 +231875,231875 +231876,231876 +231877,231877 +63656,231877 +231878,231878 +231879,231879 +231880,231880 +231881,231881 +231882,231882 +231883,231883 +88672,231883 +231884,231884 +231885,231885 +231927,231927 +231928,231928 +231929,231929 +231930,231930 +231931,231931 +231932,231932 +3692,231932 +231933,231933 +231934,231934 +231935,231935 +231936,231936 +231937,231937 +231938,231938 +231939,231939 +231940,231940 +231941,231941 +231942,231942 +216415,231942 +231943,231943 +231944,231944 +231945,231945 +231946,231946 +6217,231946 +231947,231947 +231203,231947 +118646,231947 +231948,231948 +231949,231949 +231950,231950 +231951,231951 +231952,231952 +231953,231953 +231954,231954 +231955,231955 +6841,231955 +231956,231956 +58676,231956 +231957,231957 +231958,231958 +231959,231959 +5370,231959 +231960,231960 +231961,231961 +222926,231961 +231962,231962 +231963,231963 +231964,231964 +231965,231965 +231966,231966 +231967,231967 +231968,231968 +85437,231968 +231969,231969 +231970,231970 +231971,231971 +231972,231972 +231973,231973 +231974,231974 +231975,231975 +231976,231976 +231977,231977 +231978,231978 +231979,231979 +231980,231980 +231981,231981 +231982,231982 +231983,231983 +231984,231984 +231985,231985 +231986,231986 +231987,231987 +231988,231988 +231989,231989 +231990,231990 +231991,231991 +231992,231992 +231993,231993 +231994,231994 +231995,231995 +231996,231996 +231997,231997 +231998,231998 +6725,231998 +231999,231999 +232000,232000 +232001,232001 +232002,232002 +232003,232003 +11628,232003 +232004,232004 +232005,232005 +232006,232006 +232007,232007 +232008,232008 +232009,232009 +232010,232010 +232011,232011 +232012,232012 +232013,232013 +232014,232014 +232015,232015 +232016,232016 +232017,232017 +232018,232018 +216036,232018 +232019,232019 +57843,232019 +232020,232020 +232021,232021 +232022,232022 +232023,232023 +232024,232024 +232025,232025 +232026,232026 +232027,232027 +232028,232028 +232029,232029 +232030,232030 +194717,232030 +232031,232031 +232032,232032 +232033,232033 +232034,232034 +232035,232035 +232036,232036 +232037,232037 +77405,232037 +232038,232038 +232039,232039 +185746,232039 +232040,232040 +232041,232041 +232042,232042 +232043,232043 +232044,232044 +232045,232045 +232046,232046 +232047,232047 +232048,232048 +232049,232049 +232050,232050 +232051,232051 +232052,232052 +232053,232053 +232054,232054 +232055,232055 +232056,232056 +232057,232057 +249585,232057 +232058,232058 +161074,232058 +232059,232059 +232060,232060 +232061,232061 +232103,232103 +232104,232104 +232105,232105 +232106,232106 +232107,232107 +104031,232107 +232108,232108 +82745,232108 +232109,232109 +232110,232110 +211448,232110 +232111,232111 +232112,232112 +232113,232113 +232114,232114 +232115,232115 +232116,232116 +232117,232117 +232118,232118 +232119,232119 +232120,232120 +229116,232120 +232121,232121 +232122,232122 +232123,232123 +232124,232124 +232125,232125 +232126,232126 +232127,232127 +232128,232128 +232129,232129 +232130,232130 +232131,232131 +232132,232132 +232133,232133 +232134,232134 +122935,232134 +232135,232135 +232136,232136 +232137,232137 +80104,232137 +35089,232137 +232138,232138 +232139,232139 +72541,232139 +232140,232140 +232141,232141 +232142,232142 +232143,232143 +80145,232143 +232144,232144 +232145,232145 +232146,232146 +232147,232147 +232148,232148 +232149,232149 +232150,232150 +232151,232151 +232152,232152 +232153,232153 +232154,232154 +232234,232234 +232235,232235 +232543,232543 +232544,232544 +122175,232544 +232545,232545 +148439,232545 +232546,232546 +97149,232546 +232547,232547 +232548,232548 +133032,232548 +125445,232548 +232549,232549 +232550,232550 +232551,232551 +232552,232552 +232553,232553 +147089,232553 +232554,232554 +232555,232555 +57733,232555 +232556,232556 +232637,232637 +232638,232638 +224947,232638 +232639,232639 +232640,232640 +232641,232641 +232642,232642 +232643,232643 +232644,232644 +232645,232645 +6864,232645 +232646,232646 +232647,232647 +232648,232648 +232649,232649 +232650,232650 +232651,232651 +232652,232652 +232722,232722 +232723,232723 +232724,232724 +85426,232724 +232725,232725 +232726,232726 +232727,232727 +232728,232728 +232729,232729 +232730,232730 +232731,232731 +232732,232732 +232733,232733 +240180,232733 +232734,232734 +232735,232735 +140628,232735 +136573,232735 +128986,232735 +232736,232736 +232737,232737 +232738,232738 +232739,232739 +232740,232740 +232741,232741 +232742,232742 +232743,232743 +232744,232744 +232745,232745 +232790,232790 +232791,232791 +232792,232792 +232793,232793 +232794,232794 +232795,232795 +232796,232796 +232797,232797 +232798,232798 +232799,232799 +232800,232800 +232887,232887 +232888,232888 +232889,232889 +232890,232890 +232891,232891 +232892,232892 +232893,232893 +232894,232894 +232895,232895 +232896,232896 +232897,232897 +232898,232898 +232899,232899 +232900,232900 +112090,232900 +232901,232901 +232902,232902 +232903,232903 +232904,232904 +232905,232905 +232906,232906 +215605,232906 +232907,232907 +232908,232908 +232909,232909 +232910,232910 +232911,232911 +232912,232912 +232913,232913 +240185,232914 +232914,232914 +232915,232915 +232916,232916 +232917,232917 +134249,232917 +126662,232917 +232918,232918 +232919,232919 +232920,232920 +232921,232921 +232922,232922 +232923,232923 +232924,232924 +232925,232925 +232926,232926 +232927,232927 +232928,232928 +232929,232929 +232930,232930 +232931,232931 +232932,232932 +232933,232933 +232934,232934 +232935,232935 +232936,232936 +232937,232937 +232938,232938 +110349,232938 +232939,232939 +232940,232940 +232941,232941 +232942,232942 +232943,232943 +232944,232944 +232945,232945 +232946,232946 +232947,232947 +232948,232948 +232949,232949 +4278,232949 +232950,232950 +232951,232951 +232952,232952 +232953,232953 +232954,232954 +232955,232955 +60032,232955 +232956,232956 +232957,232957 +232958,232958 +232959,232959 +232960,232960 +232961,232961 +232962,232962 +232963,232963 +232964,232964 +232965,232965 +232966,232966 +232967,232967 +232968,232968 +232969,232969 +232970,232970 +232971,232971 +232972,232972 +214430,232972 +211552,232972 +232973,232973 +5474,232973 +232974,232974 +232975,232975 +232976,232976 +232977,232977 +232978,232978 +232979,232979 +232980,232980 +232981,232981 +232982,232982 +232983,232983 +232984,232984 +155190,232984 +185892,232984 +232985,232985 +232986,232986 +232987,232987 +233025,233025 +233026,233026 +233027,233027 +233028,233028 +233029,233029 +233030,233030 +233031,233031 +233032,233032 +233033,233033 +233034,233034 +233035,233035 +233036,233036 +233037,233037 +233038,233038 +233039,233039 +105007,233039 +233040,233040 +233041,233041 +233042,233042 +233043,233043 +233044,233044 +233045,233045 +233046,233046 +233047,233047 +233048,233048 +184077,233048 +233049,233049 +233050,233050 +233051,233051 +233052,233052 +233053,233053 +233054,233054 +233055,233055 +233056,233056 +233057,233057 +233058,233058 +233059,233059 +233060,233060 +233061,233061 +233062,233062 +233063,233063 +233064,233064 +233065,233065 +233066,233066 +233067,233067 +233068,233068 +233069,233069 +233070,233070 +233071,233071 +233072,233072 +233073,233073 +233074,233074 +233075,233075 +233076,233076 +233077,233077 +233078,233078 +233079,233079 +233080,233080 +233081,233081 +233082,233082 +135783,233082 +128196,233082 +233083,233083 +119287,233083 +233084,233084 +233085,233085 +233086,233086 +233087,233087 +233088,233088 +233089,233089 +233090,233090 +233091,233091 +233092,233092 +233093,233093 +233094,233094 +233095,233095 +233096,233096 +233097,233097 +233098,233098 +233099,233099 +233100,233100 +233101,233101 +233102,233102 +233103,233103 +233104,233104 +233105,233105 +233106,233106 +233107,233107 +233108,233108 +233109,233109 +233110,233110 +233111,233111 +233112,233112 +233113,233113 +233114,233114 +233115,233115 +233116,233116 +233117,233117 +233118,233118 +233119,233119 +233120,233120 +233121,233121 +233122,233122 +233123,233123 +233124,233124 +233125,233125 +233126,233126 +233127,233127 +220218,233127 +233128,233128 +79154,233128 +233129,233129 +233130,233130 +233131,233131 +233132,233132 +142890,233132 +233133,233133 +233134,233134 +233135,233135 +233136,233136 +233137,233137 +233138,233138 +233139,233139 +233140,233140 +233141,233141 +233142,233142 +233143,233143 +233144,233144 +233145,233145 +84797,233145 +233215,233215 +233216,233216 +233217,233217 +233218,233218 +233219,233219 +233220,233220 +233221,233221 +233222,233222 +233223,233223 +233224,233224 +233225,233225 +233226,233226 +150469,233226 +233227,233227 +233228,233228 +233229,233229 +233230,233230 +233231,233231 +64153,233231 +233232,233232 +233233,233233 +233234,233234 +233235,233235 +233236,233236 +233237,233237 +233238,233238 +233239,233239 +214630,233239 +233240,233240 +233241,233241 +233242,233242 +233243,233243 +107042,233243 +233244,233244 +233245,233245 +233246,233246 +233247,233247 +233248,233248 +233249,233249 +233250,233250 +233251,233251 +77059,233251 +233252,233252 +233253,233253 +233254,233254 +66679,233254 +66315,233254 +233255,233255 +233256,233256 +233257,233257 +233258,233258 +233259,233259 +92690,233259 +233260,233260 +233261,233261 +233262,233262 +233263,233263 +233264,233264 +233265,233265 +233266,233266 +233267,233267 +233268,233268 +233269,233269 +233270,233270 +217009,233270 +233271,233271 +233272,233272 +233273,233273 +233274,233274 +233275,233275 +233276,233276 +233277,233277 +233278,233278 +233279,233279 +233280,233280 +233281,233281 +233282,233282 +233283,233283 +233284,233284 +233285,233285 +233286,233286 +234562,233286 +233287,233287 +233288,233288 +233289,233289 +233290,233290 +233291,233291 +233292,233292 +233293,233293 +233294,233294 +233295,233295 +233296,233296 +233297,233297 +233298,233298 +233299,233299 +30283,233299 +233300,233300 +233301,233301 +184141,233301 +233302,233302 +233303,233303 +233304,233304 +233305,233305 +98246,233305 +233306,233306 +233307,233307 +233308,233308 +233309,233309 +233310,233310 +233311,233311 +233312,233312 +233313,233313 +233314,233314 +233315,233315 +76962,233315 +233316,233316 +233317,233317 +233318,233318 +233319,233319 +233320,233320 +233321,233321 +233322,233322 +233323,233323 +233324,233324 +233325,233325 +233326,233326 +233327,233327 +233328,233328 +233329,233329 +233330,233330 +233331,233331 +233332,233332 +233333,233333 +233334,233334 +233335,233335 +233336,233336 +233337,233337 +233338,233338 +233339,233339 +233340,233340 +233341,233341 +233342,233342 +233343,233343 +233344,233344 +233345,233345 +233346,233346 +233347,233347 +233348,233348 +233349,233349 +233350,233350 +233351,233351 +233352,233352 +233353,233353 +233354,233354 +65940,233354 +233355,233355 +233356,233356 +233357,233357 +217949,233357 +231421,233357 +233358,233358 +217946,233358 +233359,233359 +229721,233359 +233360,233360 +233361,233361 +233362,233362 +233363,233363 +2956,233363 +233364,233364 +211981,233364 +215527,233364 +233365,233365 +233366,233366 +233367,233367 +233368,233368 +233369,233369 +233370,233370 +223169,233370 +233371,233371 +233372,233372 +233373,233373 +233374,233374 +133552,233374 +125965,233374 +233375,233375 +233376,233376 +233377,233377 +233378,233378 +233379,233379 +233380,233380 +233381,233381 +233382,233382 +233383,233383 +233384,233384 +233385,233385 +233386,233386 +233387,233387 +233388,233388 +233389,233389 +233390,233390 +233391,233391 +233392,233392 +233393,233393 +233394,233394 +233395,233395 +233396,233396 +233397,233397 +233398,233398 +233399,233399 +92660,233399 +159387,233399 +233400,233400 +233401,233401 +233402,233402 +233403,233403 +233404,233404 +233405,233405 +233406,233406 +233407,233407 +233408,233408 +233409,233409 +233410,233410 +233411,233411 +233412,233412 +233413,233413 +233414,233414 +233415,233415 +233416,233416 +233417,233417 +233418,233418 +233419,233419 +104955,233419 +233420,233420 +233421,233421 +233422,233422 +233423,233423 +233424,233424 +233425,233425 +233426,233426 +233427,233427 +233428,233428 +233429,233429 +97408,233429 +233430,233430 +233431,233431 +233432,233432 +233433,233433 +233434,233434 +233435,233435 +233436,233436 +233437,233437 +233438,233438 +233439,233439 +233440,233440 +233441,233441 +233442,233442 +233443,233443 +233444,233444 +233445,233445 +233446,233446 +233447,233447 +233448,233448 +233449,233449 +233450,233450 +233451,233451 +233452,233452 +233453,233453 +95003,233453 +142229,233453 +233454,233454 +233455,233455 +233456,233456 +233457,233457 +233458,233458 +233459,233459 +233460,233460 +233461,233461 +233462,233462 +233463,233463 +233464,233464 +233465,233465 +233466,233466 +233467,233467 +233468,233468 +233469,233469 +233470,233470 +233471,233471 +233472,233472 +233473,233473 +218049,233473 +233474,233474 +233475,233475 +233476,233476 +233477,233477 +233478,233478 +233479,233479 +233480,233480 +233481,233481 +233482,233482 +233483,233483 +233484,233484 +233485,233485 +233486,233486 +152792,233486 +233487,233487 +233488,233488 +233489,233489 +233490,233490 +233491,233491 +233492,233492 +233493,233493 +69800,233493 +233494,233494 +233495,233495 +233496,233496 +233497,233497 +233498,233498 +233499,233499 +233500,233500 +233501,233501 +233502,233502 +233503,233503 +233504,233504 +184492,233504 +233505,233505 +233506,233506 +233507,233507 +233508,233508 +164093,233508 +233509,233509 +9023,233509 +233510,233510 +233511,233511 +233512,233512 +233513,233513 +233514,233514 +233515,233515 +233516,233516 +233517,233517 +233518,233518 +233519,233519 +162408,233519 +233520,233520 +233521,233521 +233522,233522 +233523,233523 +233524,233524 +233525,233525 +233526,233526 +233527,233527 +233528,233528 +233529,233529 +233530,233530 +233531,233531 +233532,233532 +233533,233533 +233534,233534 +233535,233535 +233536,233536 +233537,233537 +233538,233538 +233539,233539 +59562,233539 +233540,233540 +233541,233541 +233542,233542 +237952,233542 +233543,233543 +233544,233544 +233545,233545 +233546,233546 +233547,233547 +233548,233548 +233549,233549 +233550,233550 +233551,233551 +233552,233552 +233553,233553 +233554,233554 +233555,233555 +233556,233556 +233557,233557 +233558,233558 +233559,233559 +233560,233560 +233561,233561 +233562,233562 +233563,233563 +233564,233564 +233565,233565 +233566,233566 +166766,233566 +233567,233567 +233568,233568 +6853,233568 +233569,233569 +233570,233570 +233571,233571 +233572,233572 +233573,233573 +233574,233574 +233575,233575 +233576,233576 +233577,233577 +233578,233578 +233579,233579 +233580,233580 +233581,233581 +233582,233582 +233583,233583 +233584,233584 +233585,233585 +233586,233586 +21587,233586 +233587,233587 +233588,233588 +233589,233589 +233590,233590 +233591,233591 +233592,233592 +233593,233593 +233594,233594 +233595,233595 +233596,233596 +233597,233597 +233598,233598 +233599,233599 +96383,233599 +233600,233600 +233601,233601 +233602,233602 +233603,233603 +233604,233604 +233605,233605 +233606,233606 +233607,233607 +233608,233608 +233609,233609 +233610,233610 +233611,233611 +233612,233612 +233613,233613 +233614,233614 +233615,233615 +233616,233616 +233617,233617 +233618,233618 +233619,233619 +233620,233620 +233621,233621 +233622,233622 +233623,233623 +233624,233624 +233625,233625 +233626,233626 +233627,233627 +233628,233628 +233629,233629 +233630,233630 +121408,233630 +233631,233631 +233632,233632 +233633,233633 +233634,233634 +233635,233635 +233636,233636 +233637,233637 +233638,233638 +233639,233639 +233640,233640 +233641,233641 +233642,233642 +233643,233643 +233644,233644 +233645,233645 +233646,233646 +233647,233647 +233648,233648 +233649,233649 +233650,233650 +81686,233650 +233651,233651 +233652,233652 +233653,233653 +90551,233653 +233654,233654 +233655,233655 +233656,233656 +233657,233657 +233658,233658 +233659,233659 +233660,233660 +233661,233661 +233662,233662 +233663,233663 +233664,233664 +233665,233665 +233666,233666 +233667,233667 +233668,233668 +176605,233668 +170641,233668 +233669,233669 +233670,233670 +233671,233671 +233672,233672 +233673,233673 +233674,233674 +233675,233675 +233676,233676 +233677,233677 +204415,233677 +233678,233678 +233679,233679 +233680,233680 +233681,233681 +233682,233682 +233683,233683 +233684,233684 +233685,233685 +233686,233686 +233687,233687 +233688,233688 +233689,233689 +233690,233690 +233691,233691 +233692,233692 +233693,233693 +233694,233694 +233695,233695 +233696,233696 +233697,233697 +233698,233698 +233699,233699 +233700,233700 +233701,233701 +233702,233702 +233703,233703 +233704,233704 +233705,233705 +233706,233706 +233707,233707 +233708,233708 +233709,233709 +233710,233710 +233711,233711 +233712,233712 +233713,233713 +233714,233714 +233715,233715 +11603,233715 +11378,233715 +233716,233716 +233717,233717 +233718,233718 +233719,233719 +233720,233720 +233721,233721 +233722,233722 +233723,233723 +233724,233724 +233725,233725 +233726,233726 +233727,233727 +233728,233728 +233729,233729 +233730,233730 +233731,233731 +233732,233732 +233733,233733 +233734,233734 +233735,233735 +233736,233736 +233737,233737 +176769,233737 +170805,233737 +233738,233738 +233739,233739 +233740,233740 +233741,233741 +233742,233742 +233743,233743 +233744,233744 +233745,233745 +233746,233746 +233747,233747 +233748,233748 +233749,233749 +134255,233749 +126668,233749 +233750,233750 +110944,233750 +233751,233751 +233752,233752 +233753,233753 +233754,233754 +233755,233755 +233756,233756 +233757,233757 +233758,233758 +233759,233759 +233760,233760 +233761,233761 +233762,233762 +233763,233763 +233764,233764 +202508,233764 +233765,233765 +233766,233766 +233767,233767 +233768,233768 +233769,233769 +233770,233770 +233771,233771 +245847,233771 +241855,233771 +239168,233771 +233772,233772 +233773,233773 +233774,233774 +233775,233775 +233776,233776 +233777,233777 +233778,233778 +233779,233779 +233780,233780 +233781,233781 +233782,233782 +233783,233783 +233784,233784 +233785,233785 +233786,233786 +184076,233786 +233787,233787 +233788,233788 +233789,233789 +233790,233790 +208074,233790 +94987,233790 +233791,233791 +78938,233791 +233792,233792 +233793,233793 +233794,233794 +22066,233794 +233795,233795 +233796,233796 +233797,233797 +233798,233798 +233799,233799 +233800,233800 +233801,233801 +233802,233802 +233803,233803 +233804,233804 +233805,233805 +233806,233806 +233807,233807 +233808,233808 +233809,233809 +233810,233810 +233811,233811 +233812,233812 +233813,233813 +233814,233814 +233815,233815 +233816,233816 +233817,233817 +233818,233818 +233819,233819 +233820,233820 +233821,233821 +233822,233822 +233823,233823 +176542,233823 +170578,233823 +233824,233824 +233825,233825 +233826,233826 +233827,233827 +233828,233828 +233829,233829 +233830,233830 +233831,233831 +206948,233831 +233832,233832 +233833,233833 +233834,233834 +233835,233835 +81462,233835 +233836,233836 +233837,233837 +233838,233838 +233839,233839 +233840,233840 +233841,233841 +233842,233842 +233843,233843 +233844,233844 +233845,233845 +233846,233846 +233847,233847 +233848,233848 +233849,233849 +233850,233850 +233851,233851 +233852,233852 +233853,233853 +233854,233854 +233855,233855 +233856,233856 +233857,233857 +233858,233858 +233859,233859 +233860,233860 +233861,233861 +233862,233862 +233863,233863 +233864,233864 +233865,233865 +233866,233866 +32896,233866 +233867,233867 +233868,233868 +233869,233869 +233870,233870 +208902,233870 +233871,233871 +233872,233872 +233910,233910 +233911,233911 +233912,233912 +233913,233913 +233914,233914 +233915,233915 +233916,233916 +233917,233917 +233918,233918 +233919,233919 +233920,233920 +233921,233921 +233922,233922 +141334,233922 +233923,233923 +233924,233924 +233925,233925 +233926,233926 +233927,233927 +233928,233928 +233929,233929 +233930,233930 +233931,233931 +233932,233932 +233933,233933 +178279,233933 +172315,233933 +233934,233934 +233935,233935 +233936,233936 +233937,233937 +233938,233938 +233939,233939 +233940,233940 +233941,233941 +205707,233941 +233942,233942 +233943,233943 +233944,233944 +233945,233945 +233946,233946 +233947,233947 +140273,233947 +233948,233948 +207369,233948 +233949,233949 +233950,233950 +233951,233951 +233952,233952 +233953,233953 +207563,233953 +233954,233954 +233955,233955 +233956,233956 +63981,233956 +233957,233957 +233958,233958 +233959,233959 +233960,233960 +233961,233961 +233962,233962 +233963,233963 +233964,233964 +233965,233965 +233966,233966 +233967,233967 +233968,233968 +233969,233969 +233970,233970 +233971,233971 +233972,233972 +233973,233973 +233974,233974 +233975,233975 +191809,233975 +233976,233976 +233977,233977 +31159,233977 +38259,233977 +233978,233978 +233979,233979 +233980,233980 +233981,233981 +233982,233982 +233983,233983 +211690,233983 +233984,233984 +233985,233985 +233986,233986 +233987,233987 +233988,233988 +233989,233989 +233990,233990 +233991,233991 +233992,233992 +233993,233993 +233994,233994 +233995,233995 +233996,233996 +233997,233997 +233998,233998 +115795,233998 +230357,233998 +233999,233999 +234000,234000 +148656,234000 +234001,234001 +234002,234002 +234003,234003 +234004,234004 +234005,234005 +234006,234006 +234149,234149 +234150,234150 +234151,234151 +234152,234152 +234153,234153 +234154,234154 +234155,234155 +234156,234156 +234157,234157 +209514,234157 +234158,234158 +234159,234159 +234160,234160 +234161,234161 +234162,234162 +234163,234163 +234164,234164 +234165,234165 +39030,234165 +234166,234166 +242064,234166 +234167,234167 +234168,234168 +234169,234169 +234170,234170 +234171,234171 +234172,234172 +234173,234173 +234174,234174 +234175,234175 +234176,234176 +234177,234177 +243980,234177 +234178,234178 +234179,234179 +234180,234180 +234181,234181 +234182,234182 +234183,234183 +234184,234184 +234185,234185 +234186,234186 +234187,234187 +234188,234188 +234189,234189 +234190,234190 +181102,234190 +234191,234191 +234192,234192 +234193,234193 +30808,234193 +51654,234193 +43550,234193 +234194,234194 +234195,234195 +234196,234196 +234197,234197 +234198,234198 +234199,234199 +234200,234200 +234201,234201 +234202,234202 +234203,234203 +234204,234204 +208018,234204 +234205,234205 +234206,234206 +234207,234207 +234208,234208 +234209,234209 +234210,234210 +234211,234211 +234212,234212 +234213,234213 +234214,234214 +234215,234215 +234216,234216 +234217,234217 +181030,234217 +234218,234218 +234219,234219 +234220,234220 +234221,234221 +211304,234221 +234222,234222 +234223,234223 +234224,234224 +234225,234225 +234226,234226 +234227,234227 +234228,234228 +234229,234229 +234230,234230 +234231,234231 +234232,234232 +234233,234233 +234234,234234 +234235,234235 +234236,234236 +234237,234237 +234238,234238 +234239,234239 +234240,234240 +234241,234241 +234242,234242 +234243,234243 +234244,234244 +234245,234245 +234246,234246 +234247,234247 +234248,234248 +234249,234249 +234250,234250 +234251,234251 +234252,234252 +208113,234252 +234253,234253 +234254,234254 +234255,234255 +234256,234256 +234257,234257 +234258,234258 +234259,234259 +234260,234260 +234261,234261 +234262,234262 +234263,234263 +234264,234264 +234265,234265 +234266,234266 +234267,234267 +234268,234268 +234269,234269 +234270,234270 +234271,234271 +234272,234272 +234273,234273 +234274,234274 +234275,234275 +234276,234276 +234277,234277 +234278,234278 +234279,234279 +97722,234279 +234280,234280 +234281,234281 +234282,234282 +148225,234282 +138676,234282 +234283,234283 +234284,234284 +234285,234285 +234286,234286 +151157,234286 +234287,234287 +234288,234288 +234289,234289 +234290,234290 +234291,234291 +234292,234292 +234293,234293 +234294,234294 +234295,234295 +234296,234296 +234297,234297 +234298,234298 +234299,234299 +209890,234299 +205120,234299 +234300,234300 +234301,234301 +234302,234302 +234303,234303 +234304,234304 +227563,234304 +234305,234305 +234306,234306 +234307,234307 +234308,234308 +234309,234309 +234310,234310 +234311,234311 +234312,234312 +234313,234313 +234314,234314 +234315,234315 +234316,234316 +234317,234317 +234318,234318 +234319,234319 +234320,234320 +142468,234320 +234321,234321 +234322,234322 +234323,234323 +234324,234324 +234325,234325 +234326,234326 +234327,234327 +234328,234328 +234329,234329 +234330,234330 +188046,234330 +234331,234331 +237969,234331 +234332,234332 +234333,234333 +234334,234334 +234335,234335 +234336,234336 +234337,234337 +234338,234338 +234403,234403 +238484,234403 +234404,234404 +5386,234404 +234405,234405 +234406,234406 +234407,234407 +234408,234408 +88189,234408 +78915,234408 +234409,234409 +234410,234410 +234411,234411 +234412,234412 +234413,234413 +234414,234414 +234415,234415 +234416,234416 +234417,234417 +84367,234417 +234418,234418 +234419,234419 +234420,234420 +234421,234421 +234422,234422 +234423,234423 +234424,234424 +234425,234425 +234426,234426 +234427,234427 +234428,234428 +234429,234429 +234430,234430 +234431,234431 +91471,234431 +234432,234432 +234433,234433 +234434,234434 +234435,234435 +234436,234436 +234437,234437 +234438,234438 +234439,234439 +234440,234440 +234441,234441 +234442,234442 +234443,234443 +204710,234443 +234444,234444 +74257,234444 +234445,234445 +234446,234446 +234447,234447 +234448,234448 +234449,234449 +234450,234450 +234451,234451 +234452,234452 +234453,234453 +234454,234454 +234455,234455 +234456,234456 +234457,234457 +234458,234458 +234459,234459 +234460,234460 +234461,234461 +102181,234461 +234462,234462 +234463,234463 +234464,234464 +234465,234465 +234466,234466 +234467,234467 +234468,234468 +234469,234469 +234470,234470 +234471,234471 +234472,234472 +234473,234473 +234474,234474 +234475,234475 +234476,234476 +234477,234477 +234478,234478 +234479,234479 +234480,234480 +234481,234481 +234482,234482 +234483,234483 +234484,234484 +234485,234485 +234486,234486 +234487,234487 +234488,234488 +234489,234489 +234490,234490 +234491,234491 +234492,234492 +234493,234493 +234494,234494 +234495,234495 +234496,234496 +234497,234497 +234498,234498 +234499,234499 +234500,234500 +234501,234501 +234502,234502 +234503,234503 +234504,234504 +234505,234505 +234506,234506 +234507,234507 +17203,234507 +234508,234508 +234509,234509 +234510,234510 +234511,234511 +234512,234512 +234513,234513 +234514,234514 +234515,234515 +234516,234516 +234517,234517 +234518,234518 +234519,234519 +234520,234520 +234521,234521 +234522,234522 +234523,234523 +234524,234524 +234525,234525 +234526,234526 +234527,234527 +234528,234528 +234529,234529 +234530,234530 +234531,234531 +234532,234532 +234533,234533 +234534,234534 +234535,234535 +234536,234536 +234537,234537 +234538,234538 +234539,234539 +234540,234540 +234541,234541 +234542,234542 +234543,234543 +234544,234544 +234545,234545 +234546,234546 +234547,234547 +234548,234548 +17180,234548 +234549,234549 +242547,234550 +234550,234550 +234551,234551 +234552,234552 +234553,234553 +234554,234554 +234555,234555 +234556,234556 +234557,234557 +234558,234558 +234559,234559 +234560,234560 +234561,234561 +234562,234562 +233286,234562 +234563,234563 +234564,234564 +234565,234565 +234566,234566 +234567,234567 +234568,234568 +234569,234569 +234570,234570 +234571,234571 +234572,234572 +234573,234573 +234574,234574 +234575,234575 +234576,234576 +234577,234577 +234578,234578 +234579,234579 +234580,234580 +234581,234581 +234582,234582 +234583,234583 +234584,234584 +234585,234585 +234586,234586 +234587,234587 +234588,234588 +234589,234589 +28478,234589 +234590,234590 +234591,234591 +234592,234592 +234593,234593 +83062,234593 +205248,234593 +234594,234594 +234595,234595 +234596,234596 +234597,234597 +234598,234598 +234599,234599 +234600,234600 +234601,234601 +234602,234602 +234603,234603 +234604,234604 +234605,234605 +234606,234606 +234607,234607 +234608,234608 +234609,234609 +234610,234610 +234611,234611 +238247,234611 +234612,234612 +234613,234613 +234614,234614 +234615,234615 +234616,234616 +234653,234653 +234654,234654 +234655,234655 +234656,234656 +234657,234657 +234658,234658 +234659,234659 +234660,234660 +234661,234661 +234662,234662 +234663,234663 +234664,234664 +234665,234665 +234666,234666 +234667,234667 +234668,234668 +220112,234668 +234669,234669 +234670,234670 +234671,234671 +234672,234672 +234673,234673 +234674,234674 +206853,234674 +234675,234675 +234676,234676 +234677,234677 +234678,234678 +234679,234679 +234680,234680 +234681,234681 +234682,234682 +234683,234683 +234684,234684 +234685,234685 +234686,234686 +234687,234687 +234688,234688 +234689,234689 +234690,234690 +234691,234691 +234692,234692 +234693,234693 +234694,234694 +206282,234694 +10147,234694 +234695,234695 +234696,234696 +234697,234697 +234698,234698 +234699,234699 +234700,234700 +234701,234701 +234702,234702 +234703,234703 +234704,234704 +234705,234705 +234706,234706 +234707,234707 +234708,234708 +234709,234709 +13485,234709 +234710,234710 +234711,234711 +234712,234712 +234713,234713 +234714,234714 +7064,234714 +234715,234715 +234716,234716 +234717,234717 +234718,234718 +234719,234719 +208788,234719 +234720,234720 +234721,234721 +234722,234722 +234723,234723 +234724,234724 +234725,234725 +234726,234726 +234727,234727 +234728,234728 +30250,234728 +50905,234728 +42801,234728 +234729,234729 +234730,234730 +234731,234731 +234732,234732 +234733,234733 +223505,234733 +234734,234734 +110800,234734 +234735,234735 +234736,234736 +221913,234736 +234737,234737 +234738,234738 +79489,234738 +234739,234739 +234740,234740 +234741,234741 +234742,234742 +234902,234743 +234743,234743 +234744,234744 +234745,234745 +234746,234746 +234747,234747 +234748,234748 +234749,234749 +234750,234750 +234751,234751 +234752,234752 +234753,234753 +161820,234753 +234754,234754 +234755,234755 +234756,234756 +234757,234757 +234758,234758 +234759,234759 +234760,234760 +234761,234761 +234762,234762 +234763,234763 +234764,234764 +234765,234765 +234766,234766 +234767,234767 +234768,234768 +234769,234769 +234770,234770 +234771,234771 +234791,234772 +234772,234772 +234792,234773 +234773,234773 +234774,234774 +234794,234775 +234775,234775 +234776,234776 +234777,234777 +234778,234778 +234779,234779 +234780,234780 +141363,234780 +234781,234781 +234782,234782 +234783,234783 +234784,234784 +234785,234785 +234786,234786 +234787,234787 +234788,234788 +138166,234788 +130579,234788 +234789,234789 +234790,234790 +234791,234791 +234772,234791 +234792,234792 +234773,234792 +234793,234793 +234794,234794 +234775,234794 +234795,234795 +234796,234796 +234797,234797 +234798,234798 +6066,234798 +234799,234799 +234800,234800 +234801,234801 +234802,234802 +234803,234803 +59383,234803 +234804,234804 +234805,234805 +234806,234806 +234807,234807 +234808,234808 +234809,234809 +234810,234810 +234811,234811 +234812,234812 +234813,234813 +234814,234814 +234815,234815 +234816,234816 +145540,234816 +234817,234817 +234818,234818 +144047,234818 +234819,234819 +234820,234820 +234821,234821 +234822,234822 +234823,234823 +234824,234824 +234825,234825 +83230,234825 +234826,234826 +234827,234827 +234828,234828 +234829,234829 +234830,234830 +68020,234830 +234831,234831 +234832,234832 +234833,234833 +234834,234834 +234835,234835 +234836,234836 +234837,234837 +234838,234838 +234839,234839 +234840,234840 +234841,234841 +234842,234842 +234843,234843 +234844,234844 +234845,234845 +141363,234845 +234846,234846 +234847,234847 +234848,234848 +234933,234848 +234849,234849 +234850,234850 +234851,234851 +234852,234852 +234853,234853 +234854,234854 +234940,234855 +234855,234855 +234856,234856 +234857,234857 +234858,234858 +50229,234858 +42037,234858 +234859,234859 +234860,234860 +234861,234861 +234946,234861 +132936,234861 +125349,234861 +234862,234862 +234947,234862 +234863,234863 +234864,234864 +103978,234864 +234949,234864 +234865,234865 +234866,234866 +234867,234867 +234953,234868 +234868,234868 +234954,234869 +234869,234869 +234870,234870 +234955,234870 +234956,234871 +234871,234871 +234872,234872 +234957,234872 +234873,234873 +234958,234873 +234874,234874 +234959,234874 +234960,234875 +234875,234875 +234876,234876 +234877,234877 +234878,234878 +234879,234879 +234880,234880 +234881,234881 +234882,234882 +234883,234883 +234884,234884 +234885,234885 +234886,234886 +234887,234887 +234888,234888 +234889,234889 +234890,234890 +234891,234891 +234892,234892 +234893,234893 +234894,234894 +234895,234895 +234896,234896 +1103,234896 +234897,234897 +234898,234898 +110528,234898 +234899,234899 +234900,234900 +234901,234901 +234902,234902 +234743,234902 +234903,234903 +234904,234904 +234905,234905 +151339,234905 +234906,234906 +234907,234907 +234908,234908 +236128,234908 +234909,234909 +234910,234910 +234911,234911 +234912,234912 +234913,234913 +234914,234914 +234915,234915 +234916,234916 +234917,234917 +234918,234918 +217087,234918 +234919,234919 +234920,234920 +195812,234920 +28403,234920 +234921,234921 +234922,234922 +234923,234923 +234924,234924 +234925,234925 +234926,234926 +234927,234927 +234928,234928 +234929,234929 +234930,234930 +234931,234931 +234932,234932 +234933,234933 +234848,234933 +234934,234934 +234935,234935 +234936,234936 +234937,234937 +234938,234938 +234939,234939 +234940,234940 +234855,234940 +234941,234941 +234942,234942 +234943,234943 +234944,234944 +234945,234945 +234946,234946 +234861,234946 +234947,234947 +234862,234947 +234948,234948 +234949,234949 +234864,234949 +234950,234950 +234951,234951 +234952,234952 +234953,234953 +234868,234953 +234954,234954 +234869,234954 +234955,234955 +234870,234955 +234956,234956 +234871,234956 +234957,234957 +234872,234957 +234958,234958 +234873,234958 +234959,234959 +234874,234959 +234960,234960 +234875,234960 +234961,234961 +234962,234962 +234963,234963 +234964,234964 +234965,234965 +234966,234966 +234967,234967 +234968,234968 +234969,234969 +234970,234970 +234971,234971 +234972,234972 +234973,234973 +234974,234974 +234975,234975 +234976,234976 +234977,234977 +234978,234978 +234979,234979 +234980,234980 +189451,234980 +234981,234981 +234982,234982 +234983,234983 +234984,234984 +107266,234984 +234985,234985 +234986,234986 +234987,234987 +234988,234988 +234989,234989 +234990,234990 +234991,234991 +217926,234991 +234992,234992 +234993,234993 +234994,234994 +234995,234995 +234996,234996 +234997,234997 +234998,234998 +234999,234999 +235000,235000 +235001,235001 +235002,235002 +235003,235003 +235004,235004 +235005,235005 +235006,235006 +102562,235006 +235007,235007 +6141,235007 +235008,235008 +235009,235009 +235010,235010 +235011,235011 +3974,235011 +235012,235012 +235013,235013 +235014,235014 +235015,235015 +235016,235016 +235017,235017 +235018,235018 +235019,235019 +9742,235019 +235020,235020 +73019,235020 +235021,235021 +235022,235022 +235023,235023 +235024,235024 +235025,235025 +235026,235026 +235027,235027 +235028,235028 +18948,235028 +235029,235029 +235030,235030 +235031,235031 +235032,235032 +235033,235033 +235034,235034 +235035,235035 +83748,235035 +235036,235036 +235037,235037 +60629,235037 +235038,235038 +235039,235039 +235040,235040 +235041,235041 +235042,235042 +235043,235043 +235044,235044 +235045,235045 +235046,235046 +235047,235047 +235048,235048 +235049,235049 +235050,235050 +235051,235051 +235052,235052 +235053,235053 +235054,235054 +235055,235055 +235056,235056 +235057,235057 +235058,235058 +235059,235059 +235060,235060 +235061,235061 +235062,235062 +235063,235063 +235064,235064 +235065,235065 +235066,235066 +235067,235067 +235068,235068 +235069,235069 +235070,235070 +235071,235071 +213653,235071 +190629,235071 +235072,235072 +235073,235073 +235074,235074 +235075,235075 +235076,235076 +235077,235077 +235078,235078 +235079,235079 +11861,235079 +235080,235080 +235081,235081 +235082,235082 +235083,235083 +235084,235084 +54579,235084 +46475,235084 +235123,235123 +235124,235124 +235125,235125 +235126,235126 +235127,235127 +235128,235128 +235129,235129 +235130,235130 +235131,235131 +140392,235131 +235132,235132 +209398,235132 +235133,235133 +60365,235133 +235134,235134 +235135,235135 +235136,235136 +235137,235137 +235138,235138 +235139,235139 +235140,235140 +235141,235141 +235142,235142 +235143,235143 +235144,235144 +103937,235144 +235145,235145 +235146,235146 +235147,235147 +235148,235148 +235149,235149 +235150,235150 +235151,235151 +235152,235152 +235153,235153 +235154,235154 +235155,235155 +235156,235156 +235157,235157 +235158,235158 +219974,235158 +235159,235159 +235160,235160 +235161,235161 +235162,235162 +199293,235162 +235163,235163 +235164,235164 +235165,235165 +63772,235165 +235166,235166 +235167,235167 +235168,235168 +20148,235168 +235169,235169 +235170,235170 +235171,235171 +222054,235171 +235172,235172 +235173,235173 +209363,235173 +235174,235174 +242962,235174 +235175,235175 +235176,235176 +235177,235177 +235178,235178 +141021,235178 +235179,235179 +235180,235180 +235181,235181 +243851,235181 +235182,235182 +235183,235183 +235184,235184 +235185,235185 +235186,235186 +235187,235187 +235188,235188 +235189,235189 +235190,235190 +235191,235191 +235192,235192 +235193,235193 +235194,235194 +235195,235195 +235196,235196 +235197,235197 +235198,235198 +31651,235198 +235199,235199 +142635,235199 +235200,235200 +57360,235200 +235201,235201 +235202,235202 +235203,235203 +235204,235204 +235205,235205 +235206,235206 +235207,235207 +235208,235208 +235209,235209 +235210,235210 +235211,235211 +235212,235212 +235213,235213 +235214,235214 +235258,235258 +235259,235259 +235260,235260 +235261,235261 +235262,235262 +235263,235263 +6913,235263 +235264,235264 +235265,235265 +235266,235266 +235267,235267 +235268,235268 +235269,235269 +235270,235270 +235271,235271 +23707,235271 +235272,235272 +235273,235273 +235274,235274 +235275,235275 +235276,235276 +235277,235277 +235278,235278 +235279,235279 +235280,235280 +235281,235281 +235282,235282 +211365,235282 +235283,235283 +235284,235284 +235285,235285 +235286,235286 +235287,235287 +235288,235288 +235289,235289 +235290,235290 +235291,235291 +235292,235292 +235293,235293 +235294,235294 +235295,235295 +235296,235296 +235297,235297 +235298,235298 +235299,235299 +235300,235300 +235301,235301 +235359,235359 +235360,235360 +235361,235361 +235362,235362 +235363,235363 +235364,235364 +235365,235365 +235366,235366 +235367,235367 +235368,235368 +235369,235369 +235370,235370 +235371,235371 +235372,235372 +235373,235373 +235374,235374 +235375,235375 +235376,235376 +235377,235377 +235378,235378 +235379,235379 +235380,235380 +235381,235381 +235382,235382 +235383,235383 +235384,235384 +235385,235385 +235386,235386 +235387,235387 +235388,235388 +235433,235433 +235434,235434 +235435,235435 +235436,235436 +235437,235437 +235438,235438 +235439,235439 +235440,235440 +235441,235441 +235442,235442 +235443,235443 +235444,235444 +235445,235445 +235446,235446 +235447,235447 +235448,235448 +235449,235449 +235450,235450 +235451,235451 +235452,235452 +235453,235453 +235454,235454 +184245,235454 +235455,235455 +235456,235456 +235457,235457 +235458,235458 +235459,235459 +235460,235460 +235461,235461 +235462,235462 +235463,235463 +235464,235464 +235465,235465 +235466,235466 +235467,235467 +235468,235468 +235469,235469 +235470,235470 +235471,235471 +235472,235472 +235473,235473 +235474,235474 +235475,235475 +235476,235476 +235477,235477 +235478,235478 +196738,235478 +235479,235479 +235480,235480 +235481,235481 +235482,235482 +235483,235483 +235484,235484 +235485,235485 +235486,235486 +235605,235605 +235606,235606 +183524,235606 +235607,235607 +235608,235608 +235609,235609 +235610,235610 +235611,235611 +235612,235612 +235613,235613 +235614,235614 +235615,235615 +235616,235616 +235617,235617 +51866,235617 +43762,235617 +235618,235618 +235619,235619 +235620,235620 +235621,235621 +235622,235622 +235623,235623 +235624,235624 +235625,235625 +235626,235626 +235627,235627 +235628,235628 +235629,235629 +235630,235630 +235631,235631 +235632,235632 +235633,235633 +235634,235634 +235635,235635 +235636,235636 +235637,235637 +235638,235638 +235639,235639 +235640,235640 +235641,235641 +235642,235642 +235643,235643 +235644,235644 +235645,235645 +235646,235646 +235647,235647 +235648,235648 +235649,235649 +235650,235650 +164864,235650 +235651,235651 +235652,235652 +235653,235653 +235654,235654 +235655,235655 +235656,235656 +149435,235656 +235657,235657 +235658,235658 +235659,235659 +235660,235660 +235661,235661 +235662,235662 +104778,235662 +235663,235663 +235664,235664 +235665,235665 +235666,235666 +235667,235667 +235668,235668 +235669,235669 +235670,235670 +235671,235671 +235672,235672 +68185,235672 +235673,235673 +235674,235674 +235675,235675 +235676,235676 +235677,235677 +235678,235678 +178308,235678 +172344,235678 +235679,235679 +109253,235679 +80670,235679 +235680,235680 +235681,235681 +235682,235682 +235683,235683 +235684,235684 +235685,235685 +235686,235686 +235687,235687 +235688,235688 +235689,235689 +235690,235690 +235691,235691 +235692,235692 +235693,235693 +235694,235694 +235695,235695 +235696,235696 +235697,235697 +143044,235697 +235698,235698 +235699,235699 +235700,235700 +235701,235701 +235702,235702 +235703,235703 +235704,235704 +235705,235705 +235706,235706 +235707,235707 +235708,235708 +235709,235709 +235710,235710 +235711,235711 +235712,235712 +235713,235713 +235714,235714 +235715,235715 +235716,235716 +235717,235717 +235718,235718 +235719,235719 +235720,235720 +13301,235720 +235721,235721 +235722,235722 +235723,235723 +235724,235724 +235725,235725 +76365,235725 +235726,235726 +235727,235727 +235728,235728 +235729,235729 +235730,235730 +235731,235731 +235732,235732 +235733,235733 +235734,235734 +235735,235735 +21914,235735 +235736,235736 +235737,235737 +235738,235738 +198027,235738 +235739,235739 +235740,235740 +235741,235741 +235787,235787 +235788,235788 +235789,235789 +222112,235789 +235790,235790 +235791,235791 +235792,235792 +75851,235792 +235793,235793 +235794,235794 +235795,235795 +235796,235796 +235797,235797 +235798,235798 +20429,235798 +235799,235799 +235800,235800 +235801,235801 +235802,235802 +235803,235803 +235804,235804 +235805,235805 +235806,235806 +235807,235807 +235808,235808 +235809,235809 +235810,235810 +235811,235811 +235812,235812 +235813,235813 +235814,235814 +235815,235815 +235816,235816 +235817,235817 +205506,235817 +235818,235818 +235819,235819 +235820,235820 +235821,235821 +235822,235822 +235823,235823 +8609,235823 +8429,235823 +235824,235824 +235825,235825 +235826,235826 +235827,235827 +235828,235828 +235829,235829 +235830,235830 +235831,235831 +235832,235832 +140452,235832 +235833,235833 +235834,235834 +235835,235835 +235836,235836 +235837,235837 +235838,235838 +235839,235839 +235840,235840 +235841,235841 +235842,235842 +235843,235843 +15709,235843 +235844,235844 +235845,235845 +235846,235846 +223503,235846 +235847,235847 +178707,235847 +235848,235848 +235849,235849 +235850,235850 +235851,235851 +235852,235852 +235853,235853 +235854,235854 +235855,235855 +235856,235856 +235857,235857 +235858,235858 +235859,235859 +203860,235859 +235860,235860 +235861,235861 +235862,235862 +235863,235863 +235864,235864 +235865,235865 +235866,235866 +247335,235866 +235867,235867 +235868,235868 +75040,235868 +235869,235869 +235870,235870 +111257,235870 +235871,235871 +235872,235872 +235873,235873 +235874,235874 +235875,235875 +235876,235876 +235877,235877 +235878,235878 +235879,235879 +235880,235880 +235881,235881 +235882,235882 +235883,235883 +235884,235884 +235885,235885 +235886,235886 +235887,235887 +235888,235888 +235889,235889 +235890,235890 +112012,235890 +235891,235891 +235892,235892 +235893,235893 +235894,235894 +235895,235895 +235896,235896 +235897,235897 +235898,235898 +235899,235899 +235900,235900 +235901,235901 +235902,235902 +235903,235903 +235904,235904 +235905,235905 +208427,235905 +235906,235906 +235907,235907 +235908,235908 +235909,235909 +235910,235910 +235911,235911 +235912,235912 +235913,235913 +235914,235914 +235915,235915 +235916,235916 +138881,235916 +235954,235954 +235955,235955 +93828,235955 +235956,235956 +235957,235957 +235958,235958 +235959,235959 +235960,235960 +235961,235961 +235962,235962 +235963,235963 +235964,235964 +235965,235965 +9072,235965 +235966,235966 +235967,235967 +235968,235968 +235969,235969 +235970,235970 +235971,235971 +235972,235972 +235973,235973 +235974,235974 +235975,235975 +235976,235976 +235977,235977 +235978,235978 +235979,235979 +235980,235980 +235981,235981 +235982,235982 +235983,235983 +235984,235984 +235985,235985 +235986,235986 +235987,235987 +235988,235988 +235989,235989 +235990,235990 +235991,235991 +235992,235992 +235993,235993 +235994,235994 +235995,235995 +235996,235996 +235997,235997 +235998,235998 +235999,235999 +236042,236042 +236043,236043 +236044,236044 +236045,236045 +236046,236046 +236047,236047 +236048,236048 +236049,236049 +236050,236050 +236051,236051 +164117,236051 +236052,236052 +236053,236053 +236054,236054 +236055,236055 +236056,236056 +236057,236057 +236058,236058 +135251,236058 +127664,236058 +236059,236059 +236060,236060 +236061,236061 +236062,236062 +236063,236063 +236064,236064 +236065,236065 +236066,236066 +236067,236067 +236107,236107 +236108,236108 +236109,236109 +236110,236110 +236111,236111 +236112,236112 +236113,236113 +236114,236114 +236115,236115 +236116,236116 +236117,236117 +22081,236117 +246001,236117 +236118,236118 +236119,236119 +236120,236120 +236121,236121 +236122,236122 +236123,236123 +236124,236124 +236125,236125 +209572,236125 +236126,236126 +236127,236127 +236128,236128 +234908,236128 +236129,236129 +236130,236130 +236131,236131 +236132,236132 +236133,236133 +236134,236134 +236135,236135 +236136,236136 +236137,236137 +236138,236138 +236139,236139 +236140,236140 +236141,236141 +205596,236141 +140420,236141 +236142,236142 +236143,236143 +236144,236144 +56544,236144 +48440,236144 +236145,236145 +236146,236146 +236147,236147 +236148,236148 +236149,236149 +236150,236150 +236151,236151 +83047,236151 +236152,236152 +236153,236153 +236154,236154 +236155,236155 +236156,236156 +236157,236157 +236158,236158 +236334,236334 +236335,236335 +103309,236335 +236336,236336 +236337,236337 +236338,236338 +236339,236339 +236340,236340 +236341,236341 +236342,236342 +236343,236343 +236344,236344 +236345,236345 +236346,236346 +236347,236347 +236348,236348 +236349,236349 +109245,236349 +108376,236349 +236350,236350 +236351,236351 +236352,236352 +236353,236353 +236354,236354 +236355,236355 +236356,236356 +236404,236404 +236405,236405 +236406,236406 +236407,236407 +236408,236408 +236409,236409 +236410,236410 +135752,236410 +128165,236410 +236411,236411 +236412,236412 +236413,236413 +236414,236414 +236415,236415 +236416,236416 +236417,236417 +236418,236418 +236419,236419 +236420,236420 +236421,236421 +236422,236422 +236423,236423 +14462,236423 +236424,236424 +236425,236425 +236426,236426 +236427,236427 +236428,236428 +236429,236429 +236430,236430 +236431,236431 +236432,236432 +132797,236432 +125210,236432 +236433,236433 +236434,236434 +236435,236435 +38393,236435 +236476,236476 +236477,236477 +68096,236477 +236478,236478 +236479,236479 +236480,236480 +236481,236481 +236482,236482 +236483,236483 +242906,236483 +236484,236484 +236485,236485 +236486,236486 +236487,236487 +236488,236488 +241400,236488 +236489,236489 +236490,236490 +236491,236491 +236492,236492 +236493,236493 +236494,236494 +236495,236495 +236496,236496 +236497,236497 +236498,236498 +236499,236499 +236500,236500 +236501,236501 +236502,236502 +236503,236503 +236504,236504 +236505,236505 +236506,236506 +236507,236507 +236508,236508 +236509,236509 +236510,236510 +236511,236511 +236512,236512 +236513,236513 +28129,236513 +236514,236514 +236515,236515 +236516,236516 +236517,236517 +236518,236518 +236519,236519 +236520,236520 +7657,236520 +236521,236521 +236522,236522 +236523,236523 +236524,236524 +236525,236525 +236526,236526 +236527,236527 +236528,236528 +236529,236529 +236530,236530 +236531,236531 +236532,236532 +236533,236533 +236534,236534 +236535,236535 +56247,236535 +48143,236535 +236536,236536 +236537,236537 +236538,236538 +236539,236539 +236540,236540 +236541,236541 +236542,236542 +236543,236543 +236544,236544 +236545,236545 +236546,236546 +236547,236547 +236548,236548 +236549,236549 +236550,236550 +236551,236551 +236552,236552 +236553,236553 +236554,236554 +236555,236555 +236556,236556 +236557,236557 +236558,236558 +236559,236559 +236560,236560 +236561,236561 +16245,236561 +119594,236561 +236562,236562 +236563,236563 +236564,236564 +259,236564 +236565,236565 +236629,236629 +236630,236630 +236631,236631 +236632,236632 +236633,236633 +236634,236634 +236635,236635 +20799,236635 +236636,236636 +236637,236637 +236638,236638 +236639,236639 +236640,236640 +236641,236641 +236642,236642 +236643,236643 +236644,236644 +236645,236645 +236646,236646 +236647,236647 +119245,236647 +236648,236648 +236649,236649 +24006,236649 +236650,236650 +236651,236651 +236652,236652 +236653,236653 +236654,236654 +236655,236655 +236656,236656 +236657,236657 +236658,236658 +236702,236702 +236703,236703 +236704,236704 +236705,236705 +236706,236706 +236707,236707 +236708,236708 +236709,236709 +236710,236710 +236711,236711 +22985,236711 +236712,236712 +135263,236712 +127676,236712 +236713,236713 +236714,236714 +236715,236715 +236716,236716 +236717,236717 +236760,236760 +236761,236761 +236762,236762 +236763,236763 +236808,236808 +236809,236809 +236810,236810 +236811,236811 +236812,236812 +236813,236813 +236814,236814 +236815,236815 +236816,236816 +236817,236817 +236818,236818 +22983,236818 +60375,236818 +236819,236819 +236820,236820 +236821,236821 +236822,236822 +206196,236822 +92600,236822 +236823,236823 +206197,236823 +92601,236823 +236824,236824 +236825,236825 +236826,236826 +236827,236827 +236828,236828 +236829,236829 +146365,236829 +236830,236830 +236874,236874 +236875,236875 +236876,236876 +236877,236877 +236878,236878 +236879,236879 +236880,236880 +236881,236881 +236882,236882 +236883,236883 +236884,236884 +236885,236885 +236886,236886 +236887,236887 +236888,236888 +236889,236889 +236890,236890 +236891,236891 +236892,236892 +115469,236892 +188512,236892 +236893,236893 +236894,236894 +115471,236894 +236895,236895 +236896,236896 +236897,236897 +236898,236898 +212055,236898 +236899,236899 +181340,236899 +236938,236938 +110604,236938 +236939,236939 +236940,236940 +236941,236941 +236942,236942 +236943,236943 +14085,236943 +228024,236943 +236944,236944 +236945,236945 +236946,236946 +236947,236947 +236948,236948 +236949,236949 +236950,236950 +236951,236951 +236952,236952 +236953,236953 +236954,236954 +236955,236955 +205621,236955 +236956,236956 +236957,236957 +236958,236958 +236959,236959 +236960,236960 +236961,236961 +236962,236962 +236963,236963 +236964,236964 +236965,236965 +236966,236966 +236967,236967 +236968,236968 +237466,237466 +237467,237467 +237468,237468 +237469,237469 +79344,237469 +237470,237470 +237471,237471 +237472,237472 +237473,237473 +237474,237474 +237475,237475 +237476,237476 +237477,237477 +237478,237478 +156674,237478 +237479,237479 +237480,237480 +237481,237481 +237482,237482 +237483,237483 +237484,237484 +237485,237485 +237486,237486 +72892,237486 +237487,237487 +237488,237488 +237489,237489 +237490,237490 +237491,237491 +237492,237492 +237493,237493 +237494,237494 +237495,237495 +237496,237496 +237497,237497 +237498,237498 +237499,237499 +237500,237500 +237501,237501 +237502,237502 +237503,237503 +237504,237504 +237505,237505 +219800,237505 +237506,237506 +237507,237507 +237508,237508 +237509,237509 +237600,237600 +237601,237601 +244935,237601 +237602,237602 +237603,237603 +237604,237604 +237605,237605 +237606,237606 +237607,237607 +237608,237608 +216526,237608 +237609,237609 +237610,237610 +217733,237610 +237611,237611 +237612,237612 +237613,237613 +237614,237614 +237656,237656 +237657,237657 +237658,237658 +237696,237696 +237697,237697 +237698,237698 +237699,237699 +237700,237700 +237701,237701 +237947,237947 +237948,237948 +141761,237948 +237949,237949 +237950,237950 +237951,237951 +237952,237952 +233542,237952 +237953,237953 +237954,237954 +237955,237955 +237956,237956 +237957,237957 +237958,237958 +237959,237959 +237960,237960 +237961,237961 +237962,237962 +237963,237963 +237964,237964 +237965,237965 +237966,237966 +237967,237967 +237968,237968 +237969,237969 +234331,237969 +237970,237970 +237971,237971 +237972,237972 +237973,237973 +237974,237974 +237975,237975 +237976,237976 +237977,237977 +32886,237977 +237978,237978 +237979,237979 +237980,237980 +237981,237981 +237982,237982 +237983,237983 +237984,237984 +237985,237985 +237986,237986 +237987,237987 +237988,237988 +237989,237989 +237990,237990 +140157,237990 +237991,237991 +178966,237991 +237992,237992 +237993,237993 +237994,237994 +237995,237995 +29057,237995 +237996,237996 +132291,237996 +124704,237996 +237997,237997 +237998,237998 +237999,237999 +238000,238000 +238001,238001 +58691,238001 +238002,238002 +2058,238002 +238003,238003 +26270,238003 +238004,238004 +238005,238005 +238006,238006 +238007,238007 +238008,238008 +87790,238008 +238009,238009 +238010,238010 +238011,238011 +238012,238012 +238013,238013 +238014,238014 +238015,238015 +238016,238016 +238017,238017 +238018,238018 +238019,238019 +238020,238020 +238021,238021 +238022,238022 +238073,238073 +238074,238074 +238075,238075 +238076,238076 +238077,238077 +238078,238078 +61198,238078 +238079,238079 +238080,238080 +238081,238081 +238082,238082 +238083,238083 +238084,238084 +238085,238085 +238228,238228 +238229,238229 +238230,238230 +238231,238231 +238232,238232 +238233,238233 +238234,238234 +238235,238235 +238236,238236 +238237,238237 +238238,238238 +238239,238239 +238240,238240 +238241,238241 +238242,238242 +238243,238243 +238244,238244 +238245,238245 +238246,238246 +238247,238247 +234611,238247 +238248,238248 +238249,238249 +238250,238250 +238251,238251 +238252,238252 +238253,238253 +238254,238254 +238255,238255 +238256,238256 +238257,238257 +238258,238258 +238259,238259 +238260,238260 +238261,238261 +238262,238262 +242795,238262 +238263,238263 +238264,238264 +238265,238265 +238266,238266 +238267,238267 +238268,238268 +238269,238269 +238270,238270 +238271,238271 +238272,238272 +238273,238273 +238274,238274 +238275,238275 +230356,238275 +238276,238276 +62068,238276 +238277,238277 +71887,238277 +238278,238278 +238279,238279 +238280,238280 +238281,238281 +238282,238282 +238283,238283 +238284,238284 +238285,238285 +238286,238286 +238427,238427 +238428,238428 +238429,238429 +238430,238430 +238431,238431 +238432,238432 +238433,238433 +238434,238434 +59106,238434 +58732,238434 +238482,238482 +238483,238483 +123267,238483 +238484,238484 +234403,238484 +238485,238485 +238486,238486 +238487,238487 +238606,238606 +238607,238607 +238608,238608 +238609,238609 +238610,238610 +238611,238611 +238612,238612 +163688,238612 +238613,238613 +238614,238614 +238615,238615 +238616,238616 +238617,238617 +157123,238617 +238618,238618 +238619,238619 +238620,238620 +238621,238621 +238622,238622 +238623,238623 +238624,238624 +238625,238625 +238626,238626 +238627,238627 +238628,238628 +238666,238666 +238667,238667 +238668,238668 +238669,238669 +238670,238670 +238671,238671 +238672,238672 +238673,238673 +238674,238674 +238675,238675 +238676,238676 +238815,238815 +238816,238816 +238817,238817 +238818,238818 +238819,238819 +238820,238820 +238821,238821 +238822,238822 +238823,238823 +238824,238824 +110608,238824 +238825,238825 +238826,238826 +238827,238827 +110514,238827 +238828,238828 +238829,238829 +238830,238830 +238831,238831 +238832,238832 +238833,238833 +238834,238834 +238835,238835 +238836,238836 +238837,238837 +238838,238838 +238839,238839 +238840,238840 +238841,238841 +238842,238842 +238843,238843 +238844,238844 +238845,238845 +238924,238924 +238925,238925 +238926,238926 +238927,238927 +238928,238928 +238929,238929 +238930,238930 +213582,238930 +132754,238930 +125167,238930 +238931,238931 +81195,238931 +238932,238932 +239161,239161 +239162,239162 +239163,239163 +105807,239163 +239164,239164 +239165,239165 +239166,239166 +239167,239167 +245847,239168 +241855,239168 +239168,239168 +233771,239168 +239169,239169 +239170,239170 +239171,239171 +239344,239344 +239345,239345 +239346,239346 +239347,239347 +239348,239348 +121316,239348 +239349,239349 +239350,239350 +239351,239351 +239352,239352 +239353,239353 +239354,239354 +239355,239355 +239356,239356 +239357,239357 +239358,239358 +239359,239359 +239360,239360 +239361,239361 +239362,239362 +239363,239363 +239364,239364 +239365,239365 +239366,239366 +239367,239367 +239368,239368 +239369,239369 +239370,239370 +70167,239370 +239411,239411 +239412,239412 +239413,239413 +239414,239414 +104,239414 +239415,239415 +239416,239416 +239417,239417 +239418,239418 +239419,239419 +239420,239420 +239421,239421 +239422,239422 +239423,239423 +239492,239492 +56742,239492 +41957,239492 +239493,239493 +239494,239494 +239495,239495 +239496,239496 +239497,239497 +239498,239498 +239499,239499 +239500,239500 +239501,239501 +239502,239502 +239503,239503 +239504,239504 +239505,239505 +239506,239506 +239507,239507 +239508,239508 +239509,239509 +239510,239510 +239511,239511 +239512,239512 +239513,239513 +239514,239514 +239515,239515 +239516,239516 +239517,239517 +239518,239518 +239519,239519 +227966,239519 +239520,239520 +239521,239521 +204916,239521 +239522,239522 +239523,239523 +239524,239524 +198470,239524 +239525,239525 +239526,239526 +239527,239527 +239528,239528 +239529,239529 +148656,239529 +239530,239530 +8987,239530 +239531,239531 +239532,239532 +239533,239533 +71892,239533 +239534,239534 +239535,239535 +239536,239536 +239537,239537 +27108,239537 +239538,239538 +239539,239539 +211142,239539 +239540,239540 +239671,239671 +239672,239672 +239673,239673 +239674,239674 +239675,239675 +239676,239676 +239677,239677 +201495,239677 +239678,239678 +239679,239679 +239680,239680 +239681,239681 +239682,239682 +239683,239683 +239684,239684 +239685,239685 +239686,239686 +239687,239687 +239688,239688 +239689,239689 +239690,239690 +239878,239690 +239879,239691 +239691,239691 +239692,239692 +239693,239693 +239694,239694 +239695,239695 +239696,239696 +239697,239697 +239698,239698 +239699,239699 +239700,239700 +239701,239701 +239702,239702 +239703,239703 +239704,239704 +239865,239865 +239866,239866 +239867,239867 +239868,239868 +239869,239869 +239870,239870 +239871,239871 +239872,239872 +239873,239873 +239874,239874 +239875,239875 +239876,239876 +248208,239876 +239877,239877 +239878,239878 +239690,239878 +239879,239879 +239691,239879 +239880,239880 +239881,239881 +163557,239881 +239882,239882 +239883,239883 +240060,240060 +240061,240061 +240062,240062 +240063,240063 +240064,240064 +240065,240065 +240066,240066 +240067,240067 +240068,240068 +240069,240069 +240070,240070 +240071,240071 +240072,240072 +187399,240072 +240073,240073 +240074,240074 +240075,240075 +240076,240076 +240077,240077 +240078,240078 +240079,240079 +143343,240079 +240080,240080 +240081,240081 +240082,240082 +40186,240082 +240083,240083 +240084,240084 +240085,240085 +240177,240177 +240178,240178 +248762,240178 +240179,240179 +240180,240180 +232733,240180 +216871,240180 +240181,240181 +240182,240182 +240183,240183 +240184,240184 +240185,240185 +232914,240185 +240186,240186 +240187,240187 +240188,240188 +240189,240189 +240190,240190 +240191,240191 +240192,240192 +140324,240192 +240193,240193 +240229,240229 +179823,240229 +240266,240266 +240267,240267 +240268,240268 +240269,240269 +78618,240269 +240270,240270 +240271,240271 +240272,240272 +240273,240273 +240274,240274 +146333,240274 +240275,240275 +240276,240276 +240277,240277 +240278,240278 +240279,240279 +240280,240280 +240281,240281 +240282,240282 +20900,240282 +240283,240283 +240284,240284 +240285,240285 +240286,240286 +240287,240287 +240288,240288 +240289,240289 +240290,240290 +240291,240291 +240292,240292 +240293,240293 +240294,240294 +240295,240295 +240296,240296 +240297,240297 +240298,240298 +240299,240299 +240300,240300 +221928,240300 +240301,240301 +240302,240302 +240303,240303 +240304,240304 +240305,240305 +240306,240306 +240307,240307 +220749,240307 +240308,240308 +240309,240309 +240411,240411 +240412,240412 +240413,240413 +240414,240414 +240415,240415 +240416,240416 +240417,240417 +240418,240418 +240419,240419 +57513,240419 +240420,240420 +248728,240420 +240421,240421 +240422,240422 +202886,240422 +240423,240423 +240424,240424 +11629,240424 +240425,240425 +240426,240426 +68626,240426 +240427,240427 +240428,240428 +240429,240429 +240430,240430 +80748,240430 +240431,240431 +240432,240432 +240433,240433 +240434,240434 +240435,240435 +240436,240436 +240517,240517 +240518,240518 +240519,240519 +135931,240519 +128344,240519 +194341,240519 +240520,240520 +240521,240521 +240522,240522 +240523,240523 +7985,240523 +240524,240524 +20471,240524 +240525,240525 +240526,240526 +240527,240527 +240528,240528 +240529,240529 +240530,240530 +240531,240531 +240532,240532 +240773,240773 +230584,240773 +240774,240774 +240775,240775 +240776,240776 +240882,240882 +240883,240883 +240884,240884 +240885,240885 +240886,240886 +240887,240887 +240888,240888 +240889,240889 +240890,240890 +240891,240891 +240892,240892 +240893,240893 +240894,240894 +240895,240895 +96354,240895 +240896,240896 +240897,240897 +240898,240898 +211558,240898 +240899,240899 +240900,240900 +111785,240900 +240901,240901 +240902,240902 +240903,240903 +240904,240904 +144238,240904 +240905,240905 +240906,240906 +240907,240907 +240908,240908 +240909,240909 +240910,240910 +240911,240911 +240912,240912 +240913,240913 +240914,240914 +240915,240915 +91464,240915 +240916,240916 +240917,240917 +240918,240918 +240919,240919 +240920,240920 +240921,240921 +240922,240922 +240923,240923 +240924,240924 +240925,240925 +240926,240926 +240927,240927 +240928,240928 +240929,240929 +240930,240930 +240931,240931 +240932,240932 +240933,240933 +240934,240934 +55647,240934 +47543,240934 +240935,240935 +157773,240935 +240936,240936 +240937,240937 +240938,240938 +240939,240939 +240940,240940 +240941,240941 +76006,240941 +240942,240942 +240943,240943 +240944,240944 +240945,240945 +156833,240945 +240946,240946 +240947,240947 +240948,240948 +240949,240949 +240950,240950 +240951,240951 +240952,240952 +240953,240953 +240954,240954 +240955,240955 +240956,240956 +240957,240957 +240958,240958 +240959,240959 +240960,240960 +161872,240960 +240961,240961 +240962,240962 +162874,240962 +240963,240963 +240964,240964 +240965,240965 +240966,240966 +240967,240967 +240968,240968 +240969,240969 +240970,240970 +240971,240971 +240972,240972 +240973,240973 +240974,240974 +240975,240975 +241016,241016 +241017,241017 +241018,241018 +241019,241019 +241020,241020 +241021,241021 +241022,241022 +241023,241023 +241024,241024 +241025,241025 +241026,241026 +241027,241027 +241028,241028 +241029,241029 +241030,241030 +241031,241031 +241032,241032 +241033,241033 +241034,241034 +241035,241035 +241036,241036 +241037,241037 +241038,241038 +241039,241039 +241040,241040 +241041,241041 +231218,241041 +241042,241042 +241043,241043 +241044,241044 +241045,241045 +241046,241046 +241047,241047 +208266,241047 +241048,241048 +241049,241049 +241050,241050 +241051,241051 +241052,241052 +241053,241053 +241054,241054 +241055,241055 +241056,241056 +241057,241057 +241058,241058 +241059,241059 +241060,241060 +241061,241061 +241062,241062 +241063,241063 +241064,241064 +241065,241065 +241066,241066 +241067,241067 +241068,241068 +241069,241069 +241070,241070 +241071,241071 +241072,241072 +241073,241073 +241074,241074 +241075,241075 +241076,241076 +241077,241077 +241078,241078 +222028,241078 +241079,241079 +241080,241080 +241081,241081 +241082,241082 +241083,241083 +241084,241084 +241085,241085 +241086,241086 +241087,241087 +241088,241088 +136945,241088 +129358,241088 +241089,241089 +241090,241090 +26243,241090 +241091,241091 +241092,241092 +241093,241093 +241094,241094 +241095,241095 +241096,241096 +241097,241097 +241098,241098 +241099,241099 +241100,241100 +241101,241101 +241102,241102 +241103,241103 +144912,241103 +241104,241104 +84710,241104 +241105,241105 +241106,241106 +241107,241107 +241108,241108 +241109,241109 +241110,241110 +241111,241111 +53166,241111 +45062,241111 +241112,241112 +241113,241113 +35030,241113 +241114,241114 +241115,241115 +241116,241116 +241117,241117 +241118,241118 +241119,241119 +241120,241120 +180809,241120 +241121,241121 +241122,241122 +241294,241294 +241295,241295 +241296,241296 +241297,241297 +241381,241381 +241382,241382 +241383,241383 +241384,241384 +241385,241385 +77965,241385 +241386,241386 +241387,241387 +241388,241388 +66799,241388 +241389,241389 +241390,241390 +241391,241391 +241392,241392 +241393,241393 +217598,241393 +241394,241394 +241395,241395 +241396,241396 +241397,241397 +241398,241398 +241399,241399 +241400,241400 +236488,241400 +79334,241400 +241401,241401 +241402,241402 +241403,241403 +241404,241404 +241405,241405 +241406,241406 +241407,241407 +241408,241408 +181349,241408 +241409,241409 +241448,241448 +241449,241449 +241450,241450 +241451,241451 +241452,241452 +241453,241453 +241454,241454 +241455,241455 +241456,241456 +241457,241457 +241458,241458 +241459,241459 +241460,241460 +241461,241461 +241462,241462 +241505,241505 +241506,241506 +29533,241506 +241507,241507 +241508,241508 +218478,241508 +241509,241509 +241510,241510 +241511,241511 +241512,241512 +241513,241513 +241514,241514 +241515,241515 +241516,241516 +241517,241517 +241518,241518 +241519,241519 +165,241519 +241520,241520 +241521,241521 +72549,241521 +241522,241522 +241523,241523 +241524,241524 +241525,241525 +241526,241526 +241527,241527 +139193,241527 +241528,241528 +241529,241529 +241530,241530 +241531,241531 +241532,241532 +241571,241571 +241572,241572 +241573,241573 +241574,241574 +241575,241575 +59663,241575 +241576,241576 +241577,241577 +241578,241578 +241579,241579 +241580,241580 +241581,241581 +241582,241582 +241583,241583 +241584,241584 +241585,241585 +241586,241586 +241587,241587 +241588,241588 +241589,241589 +87553,241589 +241590,241590 +241591,241591 +241592,241592 +176463,241592 +170499,241592 +241593,241593 +241594,241594 +241595,241595 +241596,241596 +241655,241596 +241597,241597 +241598,241598 +241599,241599 +241600,241600 +221946,241600 +241601,241601 +241637,241637 +241638,241638 +241639,241639 +241640,241640 +241641,241641 +241642,241642 +160977,241642 +241643,241643 +241644,241644 +162565,241644 +241645,241645 +241646,241646 +241647,241647 +241648,241648 +241649,241649 +139949,241649 +241650,241650 +241651,241651 +241652,241652 +241653,241653 +241654,241654 +241655,241655 +241596,241655 +241656,241656 +241657,241657 +241658,241658 +241659,241659 +241660,241660 +241661,241661 +241662,241662 +241663,241663 +147325,241663 +241664,241664 +241665,241665 +241666,241666 +217712,241666 +241667,241667 +241668,241668 +241669,241669 +241670,241670 +241713,241713 +241714,241714 +247945,241714 +241715,241715 +241716,241716 +241717,241717 +241718,241718 +241719,241719 +245756,241764 +241764,241764 +241765,241765 +245758,241766 +241766,241766 +245759,241767 +241767,241767 +241768,241768 +241769,241769 +245762,241770 +241770,241770 +245763,241771 +241771,241771 +241772,241772 +986,241772 +241773,241773 +245765,241773 +241774,241774 +241775,241775 +203547,241775 +100680,241775 +241776,241776 +241777,241777 +245769,241777 +241778,241778 +227704,241778 +241779,241779 +245771,241779 +241780,241780 +245772,241780 +241781,241781 +245773,241781 +245774,241782 +241782,241782 +241783,241783 +241784,241784 +245776,241784 +245777,241785 +241785,241785 +106672,241785 +241786,241786 +245778,241786 +241787,241787 +245780,241788 +241788,241788 +241789,241789 +245781,241789 +241790,241790 +241791,241791 +241792,241792 +241793,241793 +241794,241794 +245787,241795 +241795,241795 +241796,241796 +245788,241796 +245789,241797 +241797,241797 +241798,241798 +245790,241798 +245791,241799 +241799,241799 +245792,241800 +241800,241800 +241801,241801 +245793,241801 +241802,241802 +245794,241802 +245795,241803 +241803,241803 +107174,241803 +245796,241804 +241804,241804 +241805,241805 +245797,241805 +245798,241806 +241806,241806 +241807,241807 +241808,241808 +241809,241809 +241810,241810 +241811,241811 +241812,241812 +241813,241813 +241814,241814 +241815,241815 +241816,241816 +241817,241817 +241818,241818 +84941,241818 +57026,241818 +241819,241819 +241820,241820 +5129,241820 +241821,241821 +241822,241822 +241823,241823 +241824,241824 +241825,241825 +241826,241826 +241827,241827 +241828,241828 +241829,241829 +241830,241830 +241831,241831 +241832,241832 +241833,241833 +241834,241834 +38199,241834 +34695,241834 +241835,241835 +241836,241836 +241837,241837 +241838,241838 +241839,241839 +241840,241840 +241841,241841 +241842,241842 +245835,241843 +241843,241843 +245836,241844 +241844,241844 +245837,241845 +241845,241845 +245838,241846 +241846,241846 +14575,241846 +245839,241847 +241847,241847 +245840,241848 +241848,241848 +245841,241849 +241849,241849 +245842,241850 +241850,241850 +245843,241851 +241851,241851 +245844,241852 +241852,241852 +245845,241853 +241853,241853 +245846,241854 +241854,241854 +245847,241855 +241855,241855 +239168,241855 +233771,241855 +245848,241856 +241856,241856 +245849,241857 +241857,241857 +245850,241858 +241858,241858 +245851,241859 +241859,241859 +245852,241860 +241860,241860 +245853,241861 +241861,241861 +245854,241862 +241862,241862 +245855,241863 +241863,241863 +245856,241864 +241864,241864 +214687,241864 +245857,241865 +241865,241865 +245858,241866 +241866,241866 +245859,241867 +241867,241867 +245860,241868 +241868,241868 +245861,241869 +241869,241869 +245862,241870 +241870,241870 +245912,241920 +241920,241920 +245913,241921 +241921,241921 +245914,241922 +241922,241922 +245915,241923 +241923,241923 +245916,241924 +241924,241924 +245917,241925 +241925,241925 +245918,241926 +241926,241926 +245919,241927 +241927,241927 +245920,241928 +241928,241928 +245921,241929 +241929,241929 +245922,241930 +241930,241930 +245923,241931 +241931,241931 +245924,241932 +241932,241932 +245925,241933 +241933,241933 +245926,241934 +241934,241934 +245927,241935 +241935,241935 +242040,242040 +242041,242041 +242042,242042 +242043,242043 +242044,242044 +242045,242045 +242046,242046 +242047,242047 +242048,242048 +85716,242048 +242049,242049 +4708,242049 +242050,242050 +242051,242051 +242052,242052 +242053,242053 +242054,242054 +242055,242055 +242056,242056 +242057,242057 +242058,242058 +242059,242059 +242060,242060 +242061,242061 +242062,242062 +242063,242063 +242064,242064 +234166,242064 +242065,242065 +242066,242066 +242067,242067 +242068,242068 +242069,242069 +57413,242069 +242070,242070 +242071,242071 +242072,242072 +242073,242073 +242074,242074 +88627,242074 +242075,242075 +242076,242076 +242077,242077 +153159,242077 +242078,242078 +242079,242079 +242080,242080 +242081,242081 +242082,242082 +242083,242083 +242084,242084 +242085,242085 +242086,242086 +242087,242087 +242088,242088 +242089,242089 +242090,242090 +941,242090 +242091,242091 +242092,242092 +242093,242093 +242094,242094 +242095,242095 +242096,242096 +242097,242097 +197731,242097 +242098,242098 +242099,242099 +242100,242100 +242101,242101 +242102,242102 +242103,242103 +242104,242104 +242105,242105 +242106,242106 +242107,242107 +242108,242108 +242109,242109 +242110,242110 +242111,242111 +242112,242112 +242113,242113 +242114,242114 +242115,242115 +242116,242116 +242117,242117 +242118,242118 +242119,242119 +242120,242120 +242121,242121 +242122,242122 +242123,242123 +242124,242124 +242125,242125 +242126,242126 +242127,242127 +242128,242128 +242129,242129 +242130,242130 +242131,242131 +242132,242132 +242133,242133 +242134,242134 +242135,242135 +242136,242136 +190840,242136 +242137,242137 +242138,242138 +242139,242139 +242140,242140 +242141,242141 +242142,242142 +242143,242143 +242144,242144 +242145,242145 +242146,242146 +242147,242147 +242148,242148 +242149,242149 +242150,242150 +242151,242151 +242152,242152 +242153,242153 +242154,242154 +242155,242155 +242267,242267 +242268,242268 +242269,242269 +242270,242270 +242271,242271 +242272,242272 +242273,242273 +242274,242274 +242275,242275 +242276,242276 +242277,242277 +242278,242278 +242279,242279 +242280,242280 +242281,242281 +242282,242282 +242283,242283 +248698,242283 +124101,242283 +242284,242284 +242285,242285 +242286,242286 +242287,242287 +242288,242288 +110299,242288 +242289,242289 +242290,242290 +242291,242291 +242292,242292 +242293,242293 +242294,242294 +242295,242295 +242296,242296 +242297,242297 +122172,242297 +242298,242298 +242299,242299 +119262,242299 +193056,242299 +242300,242300 +242301,242301 +245192,242301 +242302,242302 +242303,242303 +242304,242304 +242305,242305 +242306,242306 +242307,242307 +242308,242308 +242309,242309 +242310,242310 +242311,242311 +242312,242312 +242313,242313 +53348,242313 +45244,242313 +242314,242314 +242315,242315 +242316,242316 +242317,242317 +242318,242318 +242319,242319 +242320,242320 +184223,242320 +242321,242321 +242322,242322 +242323,242323 +242324,242324 +242325,242325 +242326,242326 +6134,242326 +242327,242327 +145743,242327 +242328,242328 +242329,242329 +242330,242330 +242331,242331 +242332,242332 +242333,242333 +242334,242334 +243968,242334 +242335,242335 +242336,242336 +178112,242336 +172148,242336 +242337,242337 +242338,242338 +242339,242339 +242340,242340 +242341,242341 +242342,242342 +242343,242343 +242344,242344 +242345,242345 +185409,242345 +242346,242346 +242347,242347 +242348,242348 +242349,242349 +242350,242350 +14646,242350 +242351,242351 +143207,242351 +242352,242352 +242353,242353 +242354,242354 +242355,242355 +242356,242356 +242357,242357 +242358,242358 +242359,242359 +242360,242360 +242361,242361 +242362,242362 +242363,242363 +231292,242363 +242364,242364 +242365,242365 +242366,242366 +242367,242367 +242368,242368 +242369,242369 +242370,242370 +242371,242371 +242372,242372 +242373,242373 +242374,242374 +242375,242375 +242376,242376 +242377,242377 +242378,242378 +242379,242379 +242380,242380 +142672,242380 +242381,242381 +242382,242382 +242383,242383 +242384,242384 +242385,242385 +242386,242386 +242387,242387 +242434,242434 +242435,242435 +242436,242436 +242437,242437 +242438,242438 +242534,242534 +242535,242535 +142864,242535 +242536,242536 +242537,242537 +242538,242538 +242539,242539 +242540,242540 +242541,242541 +242542,242542 +242543,242543 +242544,242544 +213719,242544 +242545,242545 +242546,242546 +242547,242547 +234550,242547 +242586,242586 +242798,242586 +242587,242587 +242588,242588 +242589,242589 +242590,242590 +242591,242591 +242592,242592 +242593,242593 +242594,242594 +242595,242595 +242596,242596 +242597,242597 +242598,242598 +242599,242599 +242600,242600 +242601,242601 +242602,242602 +113306,242602 +2744,242602 +242603,242603 +242604,242604 +242605,242605 +60985,242605 +242606,242606 +242607,242607 +242608,242608 +242609,242609 +242610,242610 +242611,242611 +242612,242612 +242613,242613 +242614,242614 +242615,242615 +242616,242616 +3942,242616 +242617,242617 +242618,242618 +193429,242618 +242619,242619 +242620,242620 +242621,242621 +242622,242622 +242623,242623 +242624,242624 +242625,242625 +242626,242626 +242627,242627 +242628,242628 +242629,242629 +242630,242630 +242631,242631 +242632,242632 +242633,242633 +219406,242633 +242634,242634 +242635,242635 +242636,242636 +242637,242637 +242638,242638 +242639,242639 +146393,242639 +242640,242640 +242641,242641 +242642,242642 +242643,242643 +242644,242644 +242645,242645 +61454,242645 +242646,242646 +242647,242647 +242648,242648 +242649,242649 +242650,242650 +242651,242651 +242652,242652 +242653,242653 +242654,242654 +242655,242655 +97982,242655 +242656,242656 +242657,242657 +242658,242658 +242659,242659 +242660,242660 +180253,242660 +242661,242661 +242662,242662 +242663,242663 +242664,242664 +242665,242665 +242666,242666 +242667,242667 +242668,242668 +242669,242669 +242670,242670 +242671,242671 +242672,242672 +242673,242673 +242674,242674 +242675,242675 +242676,242676 +151544,242676 +242677,242677 +242678,242678 +242679,242679 +242680,242680 +242681,242681 +242682,242682 +242683,242683 +242684,242684 +242749,242749 +242750,242750 +242751,242751 +242752,242752 +242753,242753 +242754,242754 +242755,242755 +242756,242756 +242757,242757 +242758,242758 +242759,242759 +200808,242759 +242760,242760 +242761,242761 +242762,242762 +242763,242763 +242764,242764 +54618,242764 +46514,242764 +242765,242765 +242766,242766 +242767,242767 +242768,242768 +242769,242769 +242770,242770 +88076,242770 +242771,242771 +242772,242772 +242773,242773 +17382,242773 +242774,242774 +242775,242775 +242776,242776 +242777,242777 +242778,242778 +242779,242779 +242780,242780 +88189,242780 +242781,242781 +139989,242781 +242782,242782 +9216,242782 +242783,242783 +86835,242783 +242784,242784 +242785,242785 +242786,242786 +242787,242787 +163017,242787 +242788,242788 +242789,242789 +242790,242790 +242791,242791 +242792,242792 +242793,242793 +242794,242794 +242795,242795 +238262,242795 +242796,242796 +242797,242797 +242798,242798 +242586,242798 +242799,242799 +242800,242800 +242801,242801 +242838,242838 +242839,242839 +242840,242840 +242841,242841 +242842,242842 +242843,242843 +242844,242844 +242845,242845 +75471,242845 +242846,242846 +242847,242847 +242848,242848 +242849,242849 +242850,242850 +242851,242851 +242852,242852 +242853,242853 +242854,242854 +242855,242855 +242856,242856 +242857,242857 +242858,242858 +242859,242859 +242860,242860 +242861,242861 +242862,242862 +242863,242863 +28091,242863 +242864,242864 +242865,242865 +242866,242866 +140694,242866 +242867,242867 +242868,242868 +76099,242868 +242869,242869 +242870,242870 +242871,242871 +242872,242872 +242873,242873 +29532,242873 +242874,242874 +242875,242875 +242876,242876 +242877,242877 +242878,242878 +211216,242878 +242879,242879 +242880,242880 +242881,242881 +242882,242882 +242883,242883 +242884,242884 +81445,242884 +242885,242885 +242886,242886 +242887,242887 +242888,242888 +216438,242888 +242889,242889 +242890,242890 +122190,242890 +242891,242891 +242892,242892 +242893,242893 +242894,242894 +242895,242895 +242896,242896 +242897,242897 +229707,242897 +242898,242898 +242899,242899 +242900,242900 +242901,242901 +146757,242901 +242902,242902 +242903,242903 +242904,242904 +242905,242905 +242906,242906 +236483,242906 +242907,242907 +141514,242907 +242908,242908 +242909,242909 +242910,242910 +242911,242911 +242912,242912 +242913,242913 +35602,242913 +242914,242914 +105631,242914 +242915,242915 +242916,242916 +242917,242917 +242918,242918 +189431,242918 +242919,242919 +242920,242920 +194448,242920 +242921,242921 +242922,242922 +242923,242923 +242924,242924 +242925,242925 +242926,242926 +187097,242926 +132801,242926 +125214,242926 +242927,242927 +242928,242928 +242929,242929 +242930,242930 +242931,242931 +65740,242931 +242932,242932 +242933,242933 +242934,242934 +70413,242934 +242935,242935 +242936,242936 +242937,242937 +119224,242937 +242938,242938 +242939,242939 +242940,242940 +242941,242941 +242942,242942 +242943,242943 +242944,242944 +57861,242944 +138170,242944 +130583,242944 +242945,242945 +242946,242946 +242947,242947 +242948,242948 +242949,242949 +242950,242950 +242951,242951 +242952,242952 +242953,242953 +242954,242954 +242955,242955 +242956,242956 +242957,242957 +242958,242958 +242959,242959 +242960,242960 +242961,242961 +242962,242962 +235174,242962 +242963,242963 +242964,242964 +242965,242965 +242966,242966 +242967,242967 +242968,242968 +242969,242969 +242970,242970 +242971,242971 +242972,242972 +242973,242973 +242974,242974 +242975,242975 +242976,242976 +242977,242977 +242978,242978 +242979,242979 +242980,242980 +242981,242981 +242982,242982 +242983,242983 +242984,242984 +242985,242985 +242986,242986 +242987,242987 +242988,242988 +242989,242989 +242990,242990 +242991,242991 +242992,242992 +242993,242993 +246959,242993 +242994,242994 +242995,242995 +242996,242996 +242997,242997 +242998,242998 +242999,242999 +243000,243000 +243001,243001 +243002,243002 +243003,243003 +217811,243003 +243004,243004 +243005,243005 +243006,243006 +243007,243007 +243008,243008 +243009,243009 +243133,243133 +243134,243134 +243135,243135 +243136,243136 +243137,243137 +243138,243138 +243139,243139 +243140,243140 +243141,243141 +243142,243142 +243143,243143 +243144,243144 +131980,243144 +124393,243144 +98093,243144 +243145,243145 +243146,243146 +180577,243146 +243147,243147 +243148,243148 +243149,243149 +243207,243207 +243208,243208 +150470,243208 +243209,243209 +243210,243210 +243211,243211 +243212,243212 +243213,243213 +243214,243214 +243215,243215 +243216,243216 +205193,243216 +243217,243217 +243218,243218 +243219,243219 +243220,243220 +243221,243221 +243222,243222 +199195,243222 +243223,243223 +243224,243224 +243225,243225 +243226,243226 +243227,243227 +243228,243228 +243229,243229 +243230,243230 +243231,243231 +243232,243232 +243233,243233 +243234,243234 +243235,243235 +243236,243236 +243274,243274 +243275,243275 +243276,243276 +243277,243277 +243278,243278 +243279,243279 +243280,243280 +243281,243281 +243282,243282 +243283,243283 +243284,243284 +243285,243285 +243286,243286 +243287,243287 +243288,243288 +243289,243289 +243290,243290 +243291,243291 +243292,243292 +243293,243293 +243294,243294 +243295,243295 +243296,243296 +243297,243297 +135285,243297 +127698,243297 +243298,243298 +243299,243299 +243300,243300 +243301,243301 +243302,243302 +243303,243303 +243304,243304 +243305,243305 +243306,243306 +243307,243307 +243308,243308 +243309,243309 +243310,243310 +243311,243311 +9908,243311 +243312,243312 +243313,243313 +243314,243314 +243315,243315 +243316,243316 +2763,243316 +243317,243317 +243318,243318 +243319,243319 +243320,243320 +243321,243321 +243322,243322 +243323,243323 +243324,243324 +243325,243325 +243326,243326 +243327,243327 +243328,243328 +202376,243328 +243329,243329 +49302,243329 +41109,243329 +243330,243330 +243331,243331 +136401,243331 +128814,243331 +244115,243331 +243332,243332 +243333,243333 +243334,243334 +64996,243334 +243335,243335 +243336,243336 +243337,243337 +243338,243338 +143014,243338 +141236,243338 +243339,243339 +243340,243340 +243341,243341 +243342,243342 +243343,243343 +243344,243344 +243345,243345 +243346,243346 +243347,243347 +243348,243348 +243349,243349 +243350,243350 +243351,243351 +243352,243352 +243353,243353 +243354,243354 +243355,243355 +243356,243356 +243357,243357 +243358,243358 +243359,243359 +243360,243360 +243361,243361 +243362,243362 +243363,243363 +243364,243364 +243365,243365 +243366,243366 +243367,243367 +243368,243368 +243369,243369 +243370,243370 +243371,243371 +243372,243372 +243373,243373 +243374,243374 +243375,243375 +243376,243376 +243377,243377 +243378,243378 +243379,243379 +243380,243380 +243381,243381 +243382,243382 +243383,243383 +12164,243383 +243384,243384 +243385,243385 +243386,243386 +158647,243386 +243387,243387 +243388,243388 +243389,243389 +243390,243390 +243391,243391 +243392,243392 +243393,243393 +243394,243394 +243395,243395 +243396,243396 +4001,243396 +243397,243397 +248769,243397 +243398,243398 +243399,243399 +243400,243400 +243401,243401 +243402,243402 +59412,243402 +243403,243403 +243404,243404 +243405,243405 +243406,243406 +243407,243407 +243408,243408 +243409,243409 +143209,243409 +243410,243410 +243411,243411 +243412,243412 +243413,243413 +6691,243413 +243414,243414 +243415,243415 +243416,243416 +243417,243417 +243418,243418 +243419,243419 +243420,243420 +243421,243421 +243422,243422 +243423,243423 +243424,243424 +243425,243425 +243426,243426 +243427,243427 +243428,243428 +243429,243429 +243430,243430 +243431,243431 +243432,243432 +243433,243433 +243434,243434 +243435,243435 +243436,243436 +243437,243437 +243438,243438 +243439,243439 +243440,243440 +243441,243441 +54565,243441 +46461,243441 +243442,243442 +243443,243443 +243444,243444 +243445,243445 +243446,243446 +243447,243447 +243448,243448 +243449,243449 +86362,243449 +243450,243450 +243451,243451 +243452,243452 +243453,243453 +243454,243454 +243455,243455 +243456,243456 +243457,243457 +243458,243458 +243459,243459 +243460,243460 +243461,243461 +243462,243462 +243463,243463 +153109,243463 +243464,243464 +243465,243465 +243466,243466 +243467,243467 +243468,243468 +243469,243469 +243470,243470 +243471,243471 +243472,243472 +243473,243473 +243474,243474 +243475,243475 +243476,243476 +243477,243477 +191601,243477 +243478,243478 +243479,243479 +243480,243480 +70340,243480 +243481,243481 +243482,243482 +147326,243482 +243483,243483 +64473,243483 +243484,243484 +13511,243484 +243485,243485 +243486,243486 +243487,243487 +158453,243487 +243488,243488 +243489,243489 +243490,243490 +243491,243491 +243492,243492 +243493,243493 +243494,243494 +243495,243495 +243496,243496 +243497,243497 +243498,243498 +243499,243499 +652,243499 +243500,243500 +243501,243501 +243502,243502 +243503,243503 +243504,243504 +243505,243505 +243547,243547 +243548,243548 +243549,243549 +166719,243549 +243550,243550 +243551,243551 +243552,243552 +243553,243553 +243554,243554 +243555,243555 +243556,243556 +243557,243557 +243558,243558 +243559,243559 +243560,243560 +243561,243561 +243562,243562 +243563,243563 +243564,243564 +243565,243565 +243566,243566 +243567,243567 +243568,243568 +243569,243569 +243570,243570 +243571,243571 +200890,243571 +206816,243571 +243572,243572 +183348,243572 +243573,243573 +243574,243574 +243575,243575 +243576,243576 +243577,243577 +243578,243578 +243579,243579 +243580,243580 +243581,243581 +192341,243581 +243582,243582 +243583,243583 +243584,243584 +243585,243585 +243586,243586 +243587,243587 +243588,243588 +243589,243589 +243590,243590 +243591,243591 +243592,243592 +243593,243593 +243594,243594 +243595,243595 +243596,243596 +243597,243597 +243598,243598 +243599,243599 +216403,243599 +243794,243794 +243795,243795 +243796,243796 +243797,243797 +243798,243798 +243799,243799 +243800,243800 +243801,243801 +243802,243802 +243803,243803 +243804,243804 +243805,243805 +243806,243806 +243807,243807 +243808,243808 +243809,243809 +243810,243810 +243811,243811 +243812,243812 +243813,243813 +243814,243814 +243815,243815 +243816,243816 +243817,243817 +243818,243818 +243819,243819 +243820,243820 +243821,243821 +243822,243822 +243823,243823 +36543,243823 +243824,243824 +243825,243825 +243826,243826 +243827,243827 +243828,243828 +243829,243829 +243830,243830 +243831,243831 +243832,243832 +243833,243833 +243834,243834 +243835,243835 +243836,243836 +243837,243837 +243838,243838 +243839,243839 +243840,243840 +243841,243841 +243842,243842 +243843,243843 +243844,243844 +243845,243845 +243846,243846 +76952,243846 +243847,243847 +5492,243847 +243848,243848 +243849,243849 +144839,243849 +243850,243850 +243851,243851 +235181,243851 +243852,243852 +243853,243853 +243854,243854 +243855,243855 +243856,243856 +243857,243857 +102112,243857 +70263,243857 +243858,243858 +243859,243859 +243860,243860 +243861,243861 +243862,243862 +243863,243863 +243864,243864 +243865,243865 +243866,243866 +243867,243867 +53361,243867 +45257,243867 +243868,243868 +243869,243869 +243870,243870 +243871,243871 +243872,243872 +243873,243873 +243874,243874 +243875,243875 +243876,243876 +243877,243877 +243878,243878 +54686,243878 +46582,243878 +243879,243879 +243880,243880 +243881,243881 +243882,243882 +243883,243883 +243921,243921 +243922,243922 +112443,243922 +243923,243923 +243924,243924 +243925,243925 +243926,243926 +243927,243927 +243928,243928 +243929,243929 +243930,243930 +243931,243931 +243932,243932 +243933,243933 +243934,243934 +243935,243935 +243936,243936 +243937,243937 +243938,243938 +118315,243938 +243939,243939 +243940,243940 +243941,243941 +243942,243942 +243943,243943 +243944,243944 +243945,243945 +243946,243946 +243947,243947 +243948,243948 +243949,243949 +243950,243950 +243951,243951 +243952,243952 +75127,243952 +28261,243952 +243953,243953 +243954,243954 +243955,243955 +243956,243956 +243957,243957 +243958,243958 +243959,243959 +243960,243960 +243961,243961 +243962,243962 +243963,243963 +243964,243964 +243965,243965 +243966,243966 +243967,243967 +243968,243968 +242334,243968 +243969,243969 +243970,243970 +243971,243971 +243972,243972 +243973,243973 +243974,243974 +243975,243975 +243976,243976 +243977,243977 +243978,243978 +243979,243979 +243980,243980 +234177,243980 +243981,243981 +214645,243981 +244022,244022 +244023,244023 +244024,244024 +244025,244025 +158075,244025 +244026,244026 +244027,244027 +244028,244028 +244029,244029 +244030,244030 +244031,244031 +244032,244032 +244033,244033 +244034,244034 +244035,244035 +88536,244035 +244036,244036 +244037,244037 +244038,244038 +244039,244039 +244040,244040 +244041,244041 +244042,244042 +244043,244043 +244044,244044 +244045,244045 +244046,244046 +244047,244047 +244048,244048 +244049,244049 +56686,244049 +48582,244049 +244050,244050 +244051,244051 +244052,244052 +244053,244053 +244054,244054 +244055,244055 +201406,244055 +183886,244055 +244056,244056 +244057,244057 +244058,244058 +244059,244059 +244060,244060 +244061,244061 +244062,244062 +244063,244063 +244064,244064 +244065,244065 +244066,244066 +244067,244067 +215414,244067 +244068,244068 +244069,244069 +244070,244070 +244071,244071 +244072,244072 +244073,244073 +244074,244074 +39889,244074 +244075,244075 +244076,244076 +244077,244077 +244078,244078 +244079,244079 +244080,244080 +244081,244081 +88101,244081 +244082,244082 +244083,244083 +244084,244084 +244085,244085 +15740,244085 +244086,244086 +244087,244087 +244088,244088 +244089,244089 +244090,244090 +244091,244091 +65959,244091 +244092,244092 +244093,244093 +244094,244094 +152995,244094 +244095,244095 +244096,244096 +244097,244097 +244098,244098 +231861,244098 +244099,244099 +244100,244100 +244101,244101 +244102,244102 +244103,244103 +244104,244104 +54727,244104 +46623,244104 +244105,244105 +244106,244106 +244107,244107 +244108,244108 +244109,244109 +244110,244110 +244111,244111 +244112,244112 +244113,244113 +223764,244113 +244114,244114 +244115,244115 +243331,244115 +244116,244116 +37326,244116 +244117,244117 +244118,244118 +244119,244119 +244120,244120 +244121,244121 +244122,244122 +244123,244123 +244124,244124 +244125,244125 +244126,244126 +244127,244127 +135510,244127 +127923,244127 +244336,244127 +244128,244128 +244167,244167 +244168,244168 +244169,244169 +244170,244170 +244171,244171 +244172,244172 +244173,244173 +244174,244174 +244175,244175 +244176,244176 +244177,244177 +244178,244178 +244179,244179 +244180,244180 +244181,244181 +244182,244182 +244183,244183 +244184,244184 +244185,244185 +244186,244186 +244187,244187 +244188,244188 +244189,244189 +244190,244190 +244191,244191 +244192,244192 +244193,244193 +244194,244194 +244195,244195 +244196,244196 +244197,244197 +244198,244198 +244199,244199 +244200,244200 +244201,244201 +244202,244202 +244203,244203 +244204,244204 +244205,244205 +244206,244206 +142492,244206 +145488,244206 +244207,244207 +244208,244208 +244209,244209 +146705,244209 +244210,244210 +244307,244307 +244308,244308 +244309,244309 +244310,244310 +244311,244311 +244312,244312 +244313,244313 +244314,244314 +244315,244315 +244316,244316 +244317,244317 +204492,244317 +244318,244318 +244319,244319 +244320,244320 +244321,244321 +244322,244322 +244323,244323 +244324,244324 +244325,244325 +244326,244326 +244327,244327 +244328,244328 +244329,244329 +244330,244330 +244331,244331 +244332,244332 +244333,244333 +244334,244334 +244335,244335 +244336,244336 +244127,244336 +244337,244337 +244338,244338 +244339,244339 +244340,244340 +244341,244341 +244342,244342 +244343,244343 +244344,244344 +244345,244345 +244346,244346 +244347,244347 +244348,244348 +244349,244349 +244350,244350 +226308,244350 +244351,244351 +244352,244352 +244353,244353 +244354,244354 +244355,244355 +2666,244355 +244356,244356 +244357,244357 +6199,244357 +244358,244358 +244359,244359 +244360,244360 +244361,244361 +207040,244361 +244362,244362 +244545,244545 +244546,244546 +244547,244547 +244548,244548 +244549,244549 +244698,244698 +244699,244699 +244700,244700 +244701,244701 +244702,244702 +244703,244703 +244704,244704 +244705,244705 +244706,244706 +244707,244707 +205614,244707 +244708,244708 +244709,244709 +244710,244710 +244711,244711 +244712,244712 +244713,244713 +244714,244714 +244715,244715 +244716,244716 +244717,244717 +244718,244718 +244719,244719 +244720,244720 +244721,244721 +244722,244722 +244723,244723 +107544,244723 +244724,244724 +244725,244725 +244726,244726 +244727,244727 +244728,244728 +244729,244729 +244730,244730 +244731,244731 +244732,244732 +244733,244733 +244734,244734 +244735,244735 +244736,244736 +244737,244737 +244738,244738 +244739,244739 +244740,244740 +244741,244741 +244742,244742 +244743,244743 +53334,244743 +45230,244743 +244744,244744 +151510,244744 +244745,244745 +244746,244746 +244747,244747 +244748,244748 +244749,244749 +244750,244750 +244751,244751 +244752,244752 +244753,244753 +244754,244754 +205958,244754 +244755,244755 +244756,244756 +244757,244757 +244758,244758 +244759,244759 +244760,244760 +244761,244761 +244762,244762 +244763,244763 +9087,244763 +244764,244764 +244804,244804 +244805,244805 +244806,244806 +244807,244807 +244808,244808 +244809,244809 +244810,244810 +244811,244811 +244812,244812 +244813,244813 +244814,244814 +244815,244815 +244816,244816 +244817,244817 +187283,244817 +244818,244818 +244819,244819 +244820,244820 +244821,244821 +244822,244822 +244823,244823 +30228,244823 +244824,244824 +244825,244825 +244826,244826 +108864,244826 +244827,244827 +244828,244828 +244829,244829 +244830,244830 +244831,244831 +244832,244832 +244833,244833 +196566,244833 +244834,244834 +244835,244835 +244836,244836 +244837,244837 +14464,244837 +244838,244838 +183453,244838 +244839,244839 +244840,244840 +244841,244841 +244842,244842 +244843,244843 +244844,244844 +244845,244845 +245015,244845 +244894,244894 +244895,244895 +244896,244896 +244897,244897 +244898,244898 +244899,244899 +244900,244900 +244901,244901 +244902,244902 +244903,244903 +244904,244904 +244905,244905 +244906,244906 +244907,244907 +244908,244908 +244909,244909 +244910,244910 +244911,244911 +244912,244912 +244913,244913 +244914,244914 +67611,244914 +244915,244915 +244916,244916 +244917,244917 +244918,244918 +244919,244919 +244920,244920 +244921,244921 +244922,244922 +244923,244923 +66332,244923 +244924,244924 +244925,244925 +244926,244926 +244927,244927 +244928,244928 +244929,244929 +244930,244930 +244931,244931 +244932,244932 +244933,244933 +244934,244934 +148463,244934 +244935,244935 +66344,244935 +237601,244935 +244936,244936 +244937,244937 +244938,244938 +245001,245001 +245002,245002 +245003,245003 +245004,245004 +245005,245005 +245006,245006 +245007,245007 +245008,245008 +245009,245009 +245010,245010 +225317,245010 +245011,245011 +245012,245012 +245013,245013 +245014,245014 +245015,245015 +523,245015 +244845,245015 +245016,245016 +245017,245017 +245018,245018 +164466,245018 +245019,245019 +164538,245019 +245020,245020 +245070,245070 +245071,245071 +245072,245072 +245073,245073 +245074,245074 +245075,245075 +245076,245076 +245077,245077 +245078,245078 +245079,245079 +245080,245080 +245081,245081 +245082,245082 +245083,245083 +245084,245084 +245085,245085 +245086,245086 +245087,245087 +245088,245088 +245089,245089 +245090,245090 +245091,245091 +245092,245092 +245093,245093 +245094,245094 +245095,245095 +245096,245096 +245191,245191 +245192,245192 +242301,245192 +245193,245193 +245194,245194 +245195,245195 +245196,245196 +245197,245197 +245198,245198 +245199,245199 +245200,245200 +245201,245201 +245202,245202 +245203,245203 +245204,245204 +245205,245205 +245206,245206 +245207,245207 +245208,245208 +245209,245209 +245210,245210 +245211,245211 +245212,245212 +245213,245213 +163333,245213 +245214,245214 +245347,245347 +245348,245348 +37717,245348 +34625,245348 +245349,245349 +245350,245350 +245351,245351 +245398,245398 +245399,245399 +245400,245400 +245401,245401 +245402,245402 +245403,245403 +139175,245403 +245404,245404 +245405,245405 +245406,245406 +245407,245407 +245408,245408 +245409,245409 +245410,245410 +245411,245411 +245412,245412 +245413,245413 +12150,245413 +245414,245414 +245415,245415 +29262,245415 +159485,245415 +245416,245416 +245417,245417 +245418,245418 +221964,245418 +245419,245419 +245420,245420 +245421,245421 +245422,245422 +245423,245423 +245424,245424 +245425,245425 +245426,245426 +136545,245426 +128958,245426 +245427,245427 +245428,245428 +245429,245429 +64881,245429 +245430,245430 +245431,245431 +245432,245432 +245433,245433 +245434,245434 +245435,245435 +245436,245436 +245437,245437 +245438,245438 +245439,245439 +245440,245440 +187758,245440 +245441,245441 +245442,245442 +245443,245443 +245444,245444 +245445,245445 +245446,245446 +245447,245447 +245448,245448 +245449,245449 +245450,245450 +245451,245451 +245452,245452 +29505,245452 +245453,245453 +245454,245454 +161757,245454 +245455,245455 +245456,245456 +245457,245457 +245458,245458 +245459,245459 +245460,245460 +245461,245461 +245462,245462 +245463,245463 +245464,245464 +245465,245465 +245466,245466 +245467,245467 +245468,245468 +245469,245469 +245470,245470 +245471,245471 +206531,245471 +245472,245472 +245473,245473 +245474,245474 +245475,245475 +245476,245476 +245477,245477 +245478,245478 +245479,245479 +245480,245480 +245481,245481 +245482,245482 +245483,245483 +245484,245484 +245485,245485 +245486,245486 +245487,245487 +245488,245488 +245489,245489 +245490,245490 +245491,245491 +245492,245492 +36081,245492 +245703,245703 +245704,245704 +245705,245705 +245706,245706 +245707,245707 +245708,245708 +245709,245709 +245710,245710 +245711,245711 +52407,245711 +44303,245711 +245756,245756 +241764,245756 +245757,245757 +245758,245758 +241766,245758 +245759,245759 +241767,245759 +245760,245760 +245761,245761 +245762,245762 +241770,245762 +245763,245763 +241771,245763 +245764,245764 +245765,245765 +241773,245765 +245766,245766 +245767,245767 +245768,245768 +245769,245769 +241777,245769 +245770,245770 +245771,245771 +241779,245771 +245772,245772 +241780,245772 +245773,245773 +241781,245773 +245774,245774 +241782,245774 +245775,245775 +245776,245776 +241784,245776 +245777,245777 +241785,245777 +106672,245777 +245778,245778 +241786,245778 +245779,245779 +245780,245780 +241788,245780 +245781,245781 +241789,245781 +245782,245782 +245783,245783 +245784,245784 +245785,245785 +245786,245786 +245787,245787 +241795,245787 +245788,245788 +241796,245788 +245789,245789 +241797,245789 +245790,245790 +241798,245790 +245791,245791 +241799,245791 +245792,245792 +241800,245792 +245793,245793 +241801,245793 +245794,245794 +241802,245794 +245795,245795 +241803,245795 +107174,245795 +245796,245796 +241804,245796 +245797,245797 +241805,245797 +245798,245798 +241806,245798 +245835,245835 +241843,245835 +245836,245836 +241844,245836 +245837,245837 +241845,245837 +245838,245838 +241846,245838 +14575,245838 +245839,245839 +241847,245839 +245840,245840 +241848,245840 +245841,245841 +241849,245841 +245842,245842 +241850,245842 +245843,245843 +241851,245843 +245844,245844 +241852,245844 +245845,245845 +241853,245845 +245846,245846 +241854,245846 +245847,245847 +241855,245847 +239168,245847 +233771,245847 +245848,245848 +241856,245848 +245849,245849 +241857,245849 +245850,245850 +241858,245850 +245851,245851 +241859,245851 +245852,245852 +241860,245852 +245853,245853 +241861,245853 +245854,245854 +241862,245854 +245855,245855 +241863,245855 +245856,245856 +241864,245856 +214687,245856 +245857,245857 +241865,245857 +245858,245858 +241866,245858 +245859,245859 +241867,245859 +245860,245860 +241868,245860 +245861,245861 +241869,245861 +245862,245862 +241870,245862 +245912,245912 +241920,245912 +245913,245913 +241921,245913 +245914,245914 +241922,245914 +245915,245915 +241923,245915 +245916,245916 +241924,245916 +245917,245917 +241925,245917 +245918,245918 +241926,245918 +245919,245919 +241927,245919 +245920,245920 +241928,245920 +245921,245921 +241929,245921 +245922,245922 +241930,245922 +245923,245923 +241931,245923 +245924,245924 +241932,245924 +245925,245925 +241933,245925 +245926,245926 +241934,245926 +245927,245927 +241935,245927 +246000,246000 +246001,246001 +198658,246001 +236117,246001 +246002,246002 +246003,246003 +246004,246004 +246005,246005 +246006,246006 +246007,246007 +246008,246008 +246009,246009 +246010,246010 +246011,246011 +246012,246012 +246013,246013 +246014,246014 +246015,246015 +246016,246016 +246017,246017 +246018,246018 +246019,246019 +246020,246020 +246021,246021 +246022,246022 +246023,246023 +246024,246024 +246025,246025 +246026,246026 +246027,246027 +246028,246028 +246029,246029 +246030,246030 +246031,246031 +246032,246032 +246033,246033 +246034,246034 +246035,246035 +246036,246036 +246037,246037 +246038,246038 +246039,246039 +246040,246040 +246041,246041 +246042,246042 +246043,246043 +246044,246044 +246045,246045 +108509,246045 +246046,246046 +246047,246047 +246048,246048 +246049,246049 +246050,246050 +246051,246051 +246187,246187 +246188,246188 +105655,246188 +246189,246189 +246190,246190 +246191,246191 +246192,246192 +246193,246193 +165030,246193 +246194,246194 +246195,246195 +246370,246370 +94080,246370 +246371,246371 +59785,246371 +246372,246372 +246373,246373 +246374,246374 +246375,246375 +246376,246376 +246377,246377 +246378,246378 +246379,246379 +246380,246380 +246381,246381 +246382,246382 +246383,246383 +102620,246383 +246384,246384 +246385,246385 +246386,246386 +246387,246387 +246562,246562 +246563,246563 +246564,246564 +246565,246565 +246566,246566 +246567,246567 +246568,246568 +246569,246569 +246570,246570 +246571,246571 +246572,246572 +246573,246573 +246574,246574 +246575,246575 +246576,246576 +246577,246577 +246578,246578 +246579,246579 +189883,246579 +246580,246580 +246581,246581 +246582,246582 +246583,246583 +246584,246584 +246585,246585 +246586,246586 +246587,246587 +246588,246588 +246589,246589 +246590,246590 +246591,246591 +246592,246592 +163331,246592 +246593,246593 +246594,246594 +246595,246595 +246596,246596 +246597,246597 +246598,246598 +246599,246599 +246600,246600 +246601,246601 +174256,246601 +168292,246601 +246602,246602 +246603,246603 +246604,246604 +246605,246605 +246606,246606 +246607,246607 +176554,246607 +170590,246607 +15831,246607 +246608,246608 +12642,246608 +246609,246609 +246610,246610 +246611,246611 +246612,246612 +246613,246613 +246614,246614 +248811,246614 +246615,246615 +246616,246616 +246617,246617 +246618,246618 +246619,246619 +246620,246620 +246621,246621 +246622,246622 +246623,246623 +246624,246624 +246625,246625 +246626,246626 +246627,246627 +143069,246627 +246628,246628 +246629,246629 +246630,246630 +246631,246631 +246632,246632 +246633,246633 +147546,246633 +246634,246634 +246635,246635 +111619,246635 +246636,246636 +246637,246637 +246638,246638 +246639,246639 +246640,246640 +246641,246641 +8606,246641 +8426,246641 +246642,246642 +246643,246643 +246644,246644 +246645,246645 +246646,246646 +246647,246647 +246648,246648 +246649,246649 +7099,246649 +246650,246650 +28944,246650 +246651,246651 +246652,246652 +180811,246652 +246653,246653 +246654,246654 +246655,246655 +212061,246655 +246656,246656 +246657,246657 +246658,246658 +246659,246659 +246660,246660 +246661,246661 +246662,246662 +246663,246663 +246664,246664 +246665,246665 +246666,246666 +246667,246667 +57143,246667 +246668,246668 +246669,246669 +131970,246669 +124383,246669 +246670,246670 +246671,246671 +246672,246672 +246673,246673 +246674,246674 +246675,246675 +246676,246676 +246677,246677 +246678,246678 +246679,246679 +246680,246680 +246681,246681 +246682,246682 +246683,246683 +246684,246684 +246685,246685 +246686,246686 +246687,246687 +246688,246688 +246689,246689 +246690,246690 +191747,246690 +246691,246691 +246692,246692 +246693,246693 +246694,246694 +29893,246694 +246695,246695 +246696,246696 +246697,246697 +246698,246698 +246699,246699 +111502,246699 +246700,246700 +246701,246701 +226838,246701 +246702,246702 +246703,246703 +246704,246704 +246705,246705 +246706,246706 +246707,246707 +246708,246708 +246709,246709 +246710,246710 +246711,246711 +246712,246712 +246713,246713 +246714,246714 +246715,246715 +246716,246716 +246717,246717 +246718,246718 +246719,246719 +246720,246720 +52635,246720 +44531,246720 +246721,246721 +246722,246722 +246723,246723 +246724,246724 +246725,246725 +246726,246726 +246727,246727 +246728,246728 +246729,246729 +246730,246730 +246731,246731 +248786,246731 +246732,246732 +246733,246733 +246734,246734 +246735,246735 +246736,246736 +246737,246737 +246738,246738 +246739,246739 +246740,246740 +138035,246740 +130448,246740 +246741,246741 +246742,246742 +246743,246743 +246744,246744 +246745,246745 +246746,246746 +246747,246747 +246748,246748 +246749,246749 +246750,246750 +246751,246751 +246752,246752 +246753,246753 +12253,246753 +246754,246754 +246755,246755 +142499,246755 +246931,246931 +246932,246932 +246933,246933 +246934,246934 +246935,246935 +246936,246936 +246937,246937 +10355,246937 +246938,246938 +246939,246939 +246940,246940 +246941,246941 +246942,246942 +246943,246943 +49481,246943 +41288,246943 +246944,246944 +246945,246945 +246946,246946 +246947,246947 +7060,246947 +246948,246948 +246949,246949 +246950,246950 +246951,246951 +246952,246952 +246953,246953 +246954,246954 +246955,246955 +15823,246955 +246956,246956 +246957,246957 +246958,246958 +7371,246958 +246959,246959 +242993,246959 +246960,246960 +246961,246961 +136474,246961 +128887,246961 +246962,246962 +246963,246963 +246964,246964 +246965,246965 +246966,246966 +246967,246967 +5265,246967 +196560,246967 +246968,246968 +246969,246969 +246970,246970 +246971,246971 +66948,246971 +246972,246972 +246973,246973 +246974,246974 +108793,246974 +246975,246975 +246976,246976 +246977,246977 +80905,246977 +246978,246978 +246979,246979 +246980,246980 +193827,246980 +246981,246981 +246982,246982 +246983,246983 +246984,246984 +246985,246985 +246986,246986 +246987,246987 +246988,246988 +246989,246989 +246990,246990 +246991,246991 +246992,246992 +246993,246993 +246994,246994 +246995,246995 +246996,246996 +246997,246997 +246998,246998 +246999,246999 +209910,246999 +247000,247000 +247001,247001 +247002,247002 +247003,247003 +247004,247004 +247005,247005 +13290,247005 +247006,247006 +247007,247007 +247008,247008 +247009,247009 +247010,247010 +247011,247011 +247012,247012 +247013,247013 +247014,247014 +247015,247015 +247016,247016 +247017,247017 +247018,247018 +247019,247019 +247020,247020 +247021,247021 +247022,247022 +247023,247023 +247024,247024 +247025,247025 +76897,247025 +247026,247026 +247027,247027 +247028,247028 +247029,247029 +247030,247030 +247031,247031 +247032,247032 +215550,247032 +247033,247033 +247034,247034 +247035,247035 +247036,247036 +247037,247037 +247038,247038 +247039,247039 +247143,247143 +247144,247144 +247145,247145 +247146,247146 +203004,247146 +247147,247147 +68154,247147 +247148,247148 +203006,247148 +247149,247149 +247150,247150 +247151,247151 +247152,247152 +247153,247153 +247154,247154 +247155,247155 +247200,247200 +247201,247201 +247202,247202 +40201,247202 +247203,247203 +247204,247204 +247205,247205 +247206,247206 +247207,247207 +247208,247208 +247209,247209 +247210,247210 +247211,247211 +247212,247212 +247213,247213 +247214,247214 +247215,247215 +247216,247216 +70360,247216 +247217,247217 +134227,247217 +126640,247217 +247218,247218 +247219,247219 +247220,247220 +247221,247221 +247222,247222 +247223,247223 +146615,247223 +247224,247224 +247225,247225 +247226,247226 +189021,247226 +247227,247227 +247228,247228 +247229,247229 +57991,247229 +247230,247230 +247231,247231 +104342,247231 +247232,247232 +247233,247233 +247234,247234 +247235,247235 +247236,247236 +247237,247237 +57052,247237 +247238,247238 +247239,247239 +247291,247291 +247292,247292 +247293,247293 +64848,247293 +247294,247294 +247295,247295 +247296,247296 +247297,247297 +247298,247298 +247299,247299 +247300,247300 +187115,247300 +247301,247301 +247302,247302 +155060,247302 +247303,247303 +247304,247304 +247305,247305 +247306,247306 +247307,247307 +247308,247308 +111170,247308 +247309,247309 +247310,247310 +247311,247311 +247312,247312 +247313,247313 +247314,247314 +247315,247315 +247316,247316 +57989,247316 +247317,247317 +247318,247318 +247319,247319 +247320,247320 +247321,247321 +247322,247322 +247323,247323 +247324,247324 +247325,247325 +247326,247326 +247327,247327 +161947,247327 +247328,247328 +247329,247329 +247330,247330 +247331,247331 +247332,247332 +247333,247333 +247334,247334 +247335,247335 +235866,247335 +247336,247336 +247337,247337 +247338,247338 +247339,247339 +247340,247340 +247341,247341 +247342,247342 +247343,247343 +247344,247344 +247345,247345 +247346,247346 +247347,247347 +247348,247348 +247349,247349 +247350,247350 +247351,247351 +247352,247352 +247353,247353 +247354,247354 +247355,247355 +247356,247356 +247357,247357 +247358,247358 +53921,247358 +45817,247358 +247359,247359 +247360,247360 +247361,247361 +247362,247362 +59370,247362 +132715,247362 +125128,247362 +247363,247363 +247364,247364 +247365,247365 +247366,247366 +247367,247367 +247368,247368 +247369,247369 +247370,247370 +247371,247371 +247372,247372 +247414,247414 +215950,247414 +247415,247415 +247416,247416 +247417,247417 +247418,247418 +247419,247419 +247420,247420 +247421,247421 +247422,247422 +190908,247422 +247423,247423 +247424,247424 +247425,247425 +247426,247426 +247427,247427 +220834,247427 +247428,247428 +247429,247429 +247430,247430 +247431,247431 +34138,247431 +247432,247432 +247433,247433 +247434,247434 +247435,247435 +247436,247436 +247437,247437 +247438,247438 +247439,247439 +247440,247440 +247441,247441 +247442,247442 +247443,247443 +247444,247444 +247445,247445 +247446,247446 +247447,247447 +188560,247447 +247448,247448 +247449,247449 +247450,247450 +163416,247450 +247451,247451 +247452,247452 +247453,247453 +247454,247454 +213557,247454 +247455,247455 +247456,247456 +247457,247457 +247458,247458 +247459,247459 +247460,247460 +247461,247461 +247462,247462 +247463,247463 +247464,247464 +247465,247465 +247527,247527 +247528,247528 +247529,247529 +247530,247530 +247531,247531 +247532,247532 +247533,247533 +247534,247534 +247535,247535 +247536,247536 +206899,247536 +145051,247536 +247537,247537 +247538,247538 +247539,247539 +247540,247540 +247541,247541 +247542,247542 +247543,247543 +219824,247543 +247544,247544 +247545,247545 +247546,247546 +247547,247547 +247548,247548 +247549,247549 +247550,247550 +247551,247551 +247552,247552 +247553,247553 +247554,247554 +247555,247555 +247556,247556 +247557,247557 +247558,247558 +247559,247559 +247560,247560 +247561,247561 +247562,247562 +247563,247563 +247564,247564 +247565,247565 +247566,247566 +247567,247567 +247568,247568 +247569,247569 +247570,247570 +247571,247571 +247572,247572 +247573,247573 +247574,247574 +247575,247575 +247576,247576 +247577,247577 +247578,247578 +247579,247579 +247580,247580 +247581,247581 +247582,247582 +247583,247583 +247584,247584 +247585,247585 +247586,247586 +247587,247587 +247588,247588 +247589,247589 +247590,247590 +74250,247590 +247591,247591 +247592,247592 +247593,247593 +247594,247594 +247595,247595 +149145,247595 +247596,247596 +247597,247597 +247598,247598 +247599,247599 +247600,247600 +247601,247601 +247602,247602 +247603,247603 +247604,247604 +247605,247605 +247606,247606 +247607,247607 +247608,247608 +247609,247609 +247610,247610 +68500,247610 +247611,247611 +247791,247791 +247792,247792 +34219,247792 +247793,247793 +247794,247794 +247795,247795 +247796,247796 +5349,247796 +247797,247797 +247798,247798 +247799,247799 +247800,247800 +247801,247801 +247802,247802 +247803,247803 +247804,247804 +54718,247804 +46614,247804 +247805,247805 +247806,247806 +247807,247807 +247808,247808 +247809,247809 +247810,247810 +247811,247811 +247812,247812 +247813,247813 +247814,247814 +247815,247815 +247816,247816 +51717,247816 +43613,247816 +247817,247817 +247818,247818 +247819,247819 +247820,247820 +247821,247821 +247822,247822 +247823,247823 +183955,247823 +247824,247824 +247825,247825 +247826,247826 +247827,247827 +247828,247828 +247829,247829 +247830,247830 +247831,247831 +247832,247832 +247833,247833 +247834,247834 +247835,247835 +247836,247836 +247837,247837 +247838,247838 +247839,247839 +247840,247840 +247841,247841 +247842,247842 +247843,247843 +247844,247844 +247845,247845 +247846,247846 +247847,247847 +247848,247848 +247849,247849 +247850,247850 +247851,247851 +247852,247852 +247853,247853 +247854,247854 +247855,247855 +247856,247856 +247857,247857 +247858,247858 +247859,247859 +247860,247860 +247861,247861 +247862,247862 +247863,247863 +247864,247864 +247865,247865 +247866,247866 +247867,247867 +247868,247868 +247869,247869 +247870,247870 +247871,247871 +247872,247872 +247873,247873 +247874,247874 +247875,247875 +247876,247876 +247877,247877 +247878,247878 +247879,247879 +185702,247879 +247880,247880 +247881,247881 +247882,247882 +247883,247883 +247884,247884 +247885,247885 +247886,247886 +247887,247887 +247888,247888 +247889,247889 +247890,247890 +247891,247891 +247892,247892 +247893,247893 +217228,247893 +247894,247894 +81321,247894 +247895,247895 +247896,247896 +247897,247897 +80161,247897 +247898,247898 +247899,247899 +247900,247900 +247901,247901 +247902,247902 +188705,247902 +230881,247902 +247903,247903 +247904,247904 +247905,247905 +247906,247906 +247907,247907 +247908,247908 +247909,247909 +247910,247910 +247911,247911 +247912,247912 +247913,247913 +247914,247914 +247915,247915 +247916,247916 +247917,247917 +247918,247918 +247919,247919 +247920,247920 +145372,247920 +247921,247921 +247922,247922 +247923,247923 +247924,247924 +247925,247925 +247926,247926 +247927,247927 +247928,247928 +247929,247929 +247930,247930 +247931,247931 +247932,247932 +247933,247933 +247934,247934 +247935,247935 +247936,247936 +247937,247937 +247938,247938 +247939,247939 +247940,247940 +247941,247941 +247942,247942 +247943,247943 +247944,247944 +247945,247945 +241714,247945 +247946,247946 +247947,247947 +247948,247948 +206220,247948 +163332,247948 +247949,247949 +247950,247950 +163334,247950 +247951,247951 +247952,247952 +247953,247953 +2161,247953 +247954,247954 +247955,247955 +150880,247955 +247956,247956 +247957,247957 +247958,247958 +247959,247959 +247960,247960 +4034,247960 +225987,247960 +247961,247961 +247962,247962 +248045,247962 +247963,247963 +247964,247964 +247965,247965 +247966,247966 +247967,247967 +247968,247968 +247969,247969 +247970,247970 +247971,247971 +247972,247972 +247973,247973 +247974,247974 +34431,247974 +247975,247975 +247976,247976 +61186,247976 +247977,247977 +247978,247978 +247979,247979 +247980,247980 +247981,247981 +247982,247982 +247983,247983 +247984,247984 +247985,247985 +247986,247986 +247987,247987 +247988,247988 +247989,247989 +247990,247990 +247991,247991 +247992,247992 +119760,247992 +247993,247993 +247994,247994 +247995,247995 +247996,247996 +247997,247997 +247998,247998 +247999,247999 +207943,247999 +248000,248000 +248001,248001 +248002,248002 +248003,248003 +248004,248004 +248005,248005 +248006,248006 +199253,248006 +248007,248007 +248008,248008 +248009,248009 +248010,248010 +248011,248011 +248012,248012 +248013,248013 +248014,248014 +165199,248014 +2023,248014 +248015,248015 +248016,248016 +248017,248017 +248018,248018 +248019,248019 +248020,248020 +248021,248021 +248022,248022 +248023,248023 +248024,248024 +248025,248025 +248026,248026 +248027,248027 +248028,248028 +248029,248029 +248030,248030 +248031,248031 +248032,248032 +248033,248033 +248034,248034 +248035,248035 +248036,248036 +248037,248037 +248038,248038 +248039,248039 +248040,248040 +248041,248041 +248042,248042 +248043,248043 +248044,248044 +248045,248045 +247962,248045 +248046,248046 +248047,248047 +248048,248048 +102625,248048 +104438,248048 +6179,248048 +11625,248048 +248049,248049 +248050,248050 +104440,248050 +248051,248051 +248052,248052 +248053,248053 +248054,248054 +248055,248055 +248056,248056 +4794,248056 +248057,248057 +248058,248058 +248059,248059 +248060,248060 +248061,248061 +248062,248062 +248063,248063 +248102,248102 +180346,248102 +248103,248103 +248104,248104 +248105,248105 +248106,248106 +248107,248107 +248108,248108 +248109,248109 +248110,248110 +248111,248111 +157115,248111 +248112,248112 +248113,248113 +248114,248114 +248115,248115 +248116,248116 +248117,248117 +248118,248118 +248119,248119 +248120,248120 +248121,248121 +248122,248122 +248123,248123 +248124,248124 +248125,248125 +248126,248126 +248127,248127 +248128,248128 +248129,248129 +248130,248130 +248131,248131 +2348,248131 +248132,248132 +55940,248132 +47836,248132 +248133,248133 +65882,248133 +248134,248134 +248135,248135 +248136,248136 +248137,248137 +248138,248138 +248139,248139 +248140,248140 +248141,248141 +180409,248141 +248142,248142 +216128,248142 +248143,248143 +248144,248144 +248145,248145 +248146,248146 +248147,248147 +145711,248147 +248148,248148 +248149,248149 +248150,248150 +248151,248151 +248152,248152 +248153,248153 +248154,248154 +248155,248155 +248156,248156 +248157,248157 +248158,248158 +158209,248158 +160442,248158 +248159,248159 +248160,248160 +248161,248161 +248162,248162 +248163,248163 +143887,248163 +248164,248164 +248165,248165 +248166,248166 +248167,248167 +248168,248168 +182,248168 +248169,248169 +248207,248207 +248208,248208 +239876,248208 +248209,248209 +248210,248210 +248211,248211 +248212,248212 +248213,248213 +248214,248214 +248215,248215 +248216,248216 +248217,248217 +248373,248373 +248663,248663 +248664,248664 +248665,248665 +248666,248666 +248667,248667 +248668,248668 +248669,248669 +76854,248669 +248670,248670 +248671,248671 +248672,248672 +248673,248673 +248674,248674 +248675,248675 +178242,248675 +172278,248675 +248676,248676 +248677,248677 +248678,248678 +248679,248679 +248680,248680 +248681,248681 +248682,248682 +248683,248683 +248684,248684 +248685,248685 +248686,248686 +248687,248687 +248688,248688 +248689,248689 +248690,248690 +248691,248691 +248692,248692 +248693,248693 +248694,248694 +248695,248695 +248696,248696 +248697,248697 +248698,248698 +124101,248698 +242283,248698 +248699,248699 +248700,248700 +248701,248701 +248702,248702 +248703,248703 +188195,248703 +248704,248704 +248705,248705 +248706,248706 +248707,248707 +248708,248708 +248709,248709 +248710,248710 +248711,248711 +212018,248711 +248712,248712 +248713,248713 +248714,248714 +248715,248715 +248716,248716 +248717,248717 +248718,248718 +248719,248719 +248720,248720 +155422,248720 +248721,248721 +248722,248722 +248723,248723 +248724,248724 +248725,248725 +59525,248725 +8062,248725 +248726,248726 +248727,248727 +248728,248728 +240420,248728 +248729,248729 +248730,248730 +248731,248731 +248732,248732 +248733,248733 +248734,248734 +248735,248735 +9344,248735 +248736,248736 +248737,248737 +248738,248738 +248739,248739 +116742,248739 +248740,248740 +248741,248741 +248742,248742 +248743,248743 +248744,248744 +248745,248745 +248746,248746 +248747,248747 +248748,248748 +248749,248749 +248750,248750 +83215,248750 +248751,248751 +248752,248752 +248753,248753 +248754,248754 +141148,248754 +248755,248755 +248756,248756 +248757,248757 +248758,248758 +248759,248759 +248760,248760 +248761,248761 +248762,248762 +33273,248762 +240178,248762 +248763,248763 +248764,248764 +248765,248765 +248766,248766 +248767,248767 +248768,248768 +248769,248769 +243397,248769 +248770,248770 +248771,248771 +248772,248772 +248773,248773 +248774,248774 +248775,248775 +204852,248775 +248776,248776 +248777,248777 +248778,248778 +248779,248779 +248780,248780 +20049,248780 +248781,248781 +248782,248782 +14338,248782 +248783,248783 +248784,248784 +248785,248785 +248786,248786 +246731,248786 +248787,248787 +248788,248788 +248789,248789 +248790,248790 +248791,248791 +248792,248792 +51308,248792 +43204,248792 +248793,248793 +248794,248794 +248795,248795 +248796,248796 +248797,248797 +248798,248798 +248799,248799 +248800,248800 +248801,248801 +248802,248802 +248803,248803 +248804,248804 +248805,248805 +248806,248806 +248807,248807 +248808,248808 +248809,248809 +248810,248810 +248811,248811 +246614,248811 +248898,248898 +248899,248899 +248900,248900 +248901,248901 +248902,248902 +248903,248903 +80442,248903 +248904,248904 +248905,248905 +248906,248906 +248907,248907 +248908,248908 +205083,248908 +248909,248909 +248910,248910 +248911,248911 +248912,248912 +58416,248912 +248913,248913 +248914,248914 +248915,248915 +248916,248916 +248917,248917 +248918,248918 +248919,248919 +248920,248920 +248921,248921 +248922,248922 +248923,248923 +76404,248923 +248924,248924 +248925,248925 +30255,248925 +248926,248926 +62763,248926 +248927,248927 +248928,248928 +61484,248928 +248929,248929 +248930,248930 +248931,248931 +248932,248932 +249059,249059 +249060,249060 +249061,249061 +249062,249062 +249063,249063 +249064,249064 +249065,249065 +249066,249066 +249067,249067 +249068,249068 +161273,249068 +249069,249069 +249070,249070 +249071,249071 +24450,249071 +111965,249071 +249072,249072 +249073,249073 +249074,249074 +249075,249075 +249076,249076 +249077,249077 +249078,249078 +249079,249079 +249080,249080 +249081,249081 +249082,249082 +249083,249083 +249084,249084 +249085,249085 +249086,249086 +249087,249087 +249088,249088 +249089,249089 +249090,249090 +249091,249091 +249092,249092 +249093,249093 +249094,249094 +249095,249095 +249096,249096 +249097,249097 +249098,249098 +249099,249099 +249100,249100 +249101,249101 +249102,249102 +249103,249103 +249104,249104 +249106,249104 +249105,249105 +249106,249106 +4668,249106 +249104,249106 +249107,249107 +249108,249108 +249109,249109 +249110,249110 +32,249110 +249111,249111 +249112,249112 +249113,249113 +249114,249114 +249115,249115 +249116,249116 +249117,249117 +7232,249117 +249118,249118 +249119,249119 +249120,249120 +249121,249121 +249122,249122 +249123,249123 +249124,249124 +249125,249125 +39864,249125 +249126,249126 +77169,249126 +249127,249127 +249128,249128 +249129,249129 +249130,249130 +249131,249131 +249132,249132 +249206,249206 +231212,249206 +149709,249206 +249207,249207 +249208,249208 +249209,249209 +33961,249209 +249210,249210 +249211,249211 +249212,249212 +249213,249213 +89625,249213 +249214,249214 +249215,249215 +249216,249216 +249217,249217 +86132,249217 +249218,249218 +249219,249219 +249220,249220 +249221,249221 +249222,249222 +249223,249223 +249224,249224 +249225,249225 +249226,249226 +249227,249227 +207164,249227 +249228,249228 +249229,249229 +249230,249230 +218515,249230 +249231,249231 +249232,249232 +249233,249233 +222204,249233 +249234,249234 +249235,249235 +249236,249236 +249237,249237 +249238,249238 +249239,249239 +190278,249239 +249240,249240 +249241,249241 +249242,249242 +249243,249243 +249244,249244 +249245,249245 +249246,249246 +249247,249247 +249248,249248 +33404,249248 +249249,249249 +249250,249250 +249251,249251 +249252,249252 +249253,249253 +249254,249254 +249255,249255 +249256,249256 +249257,249257 +249258,249258 +249259,249259 +249260,249260 +249261,249261 +249262,249262 +249393,249393 +82577,249393 +249394,249394 +249395,249395 +249396,249396 +249397,249397 +249398,249398 +249399,249399 +249400,249400 +249401,249401 +249402,249402 +249403,249403 +249404,249404 +249405,249405 +249406,249406 +249407,249407 +19828,249407 +249408,249408 +249409,249409 +220595,249409 +249410,249410 +249411,249411 +249412,249412 +249413,249413 +249414,249414 +249415,249415 +249416,249416 +249417,249417 +110759,249417 +249418,249418 +249419,249419 +249420,249420 +249421,249421 +249422,249422 +249423,249423 +249424,249424 +249425,249425 +249426,249426 +249427,249427 +249428,249428 +249429,249429 +249430,249430 +116211,249430 +231419,249430 +249431,249431 +249432,249432 +249433,249433 +249434,249434 +249435,249435 +249436,249436 +249437,249437 +229778,249437 +249438,249438 +249439,249439 +249440,249440 +249441,249441 +249442,249442 +249443,249443 +54594,249443 +46490,249443 +249444,249444 +249445,249445 +249446,249446 +249447,249447 +249448,249448 +249449,249449 +249450,249450 +249566,249566 +249567,249567 +249568,249568 +249569,249569 +249570,249570 +6650,249570 +249571,249571 +249572,249572 +249573,249573 +249574,249574 +249575,249575 +132112,249575 +124525,249575 +249576,249576 +249577,249577 +249578,249578 +249579,249579 +249580,249580 +249581,249581 +123287,249581 +249582,249582 +249583,249583 +249584,249584 +249585,249585 +232057,249585 +249586,249586 +249587,249587 +249588,249588 +185700,249588 +249589,249589 +249590,249590 +249591,249591 +249592,249592 +146429,249592 +249593,249593 +249594,249594 +249595,249595 +249596,249596 +249597,249597 +249598,249598 +249599,249599 +249600,249600 +157759,249600 +249601,249601 +249602,249602 +249603,249603 +249604,249604 +249605,249605 +249606,249606 +249607,249607 +201544,249607 +249608,249608 +249609,249609 +249610,249610 +249611,249611 +161575,249611 +249612,249612 +249613,249613 +249614,249614 +9203,249614 +249615,249615 +249616,249616 +249617,249617 +249618,249618 +249619,249619 +249657,249657 +249658,249658 +249659,249659 +249660,249660 +6706,249660 +249661,249661 +249662,249662 +7754,249662 +249663,249663 +249755,249755 +249756,249756 +249757,249757 +249758,249758 +249759,249759 +249760,249760 +249761,249761 +249762,249762 +180714,249762 +249763,249763 +249764,249764 +249765,249765 +249766,249766 +249767,249767 +249768,249768 +249769,249769 +249770,249770 +179506,249770 +249835,249835 +139599,249835 +249836,249836 +249837,249837 +5144,249837 +249838,249838 +249839,249839 +249920,249920 +249921,249921 +249922,249922 +249923,249923 +249924,249924 +249925,249925 +249926,249926 +249927,249927 +249928,249928 +249929,249929 +249930,249930 +249931,249931 +249978,249978 +249979,249979 +249980,249980 +249981,249981 +249982,249982 +249983,249983 +249984,249984 +249985,249985 +249986,249986 +158735,249986 +249987,249987 +249988,249988 +213965,249988 +249989,249989 +249990,249990 +249991,249991 +249992,249992 +249993,249993 +249994,249994 +249995,249995 +249996,249996 +249997,249997 +249998,249998 diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index cf31b2ff0..aa8113440 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -46,7 +46,7 @@ namespace cudaungappedextender magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ - std::string anchors_file_path = "../data/anchors.csv"; + std::string anchors_file_path = "../data/example_hits.csv"; std::vector h_hits; // Following function loops through all hits in the anchors.csv and returns results in diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index 104f43138..7ec3363d1 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -44,7 +44,7 @@ namespace cudaungappedextender magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ - std::string anchors_file_path = "../data/anchors.csv"; + std::string anchors_file_path = "../data/example_hits.csv"; std::vector h_hits; // Following function loops through all hits in the anchors.csv and returns results in From 83748b4f4a689d4aec0f0fb96567e8420585d998 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 21 Aug 2020 18:18:00 -0700 Subject: [PATCH 023/281] [cudaungappedextender] Correct obviously wrong data types --- cudaungappedextender/samples/device_sample.cu | 2 +- cudaungappedextender/samples/host_sample.cu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index aa8113440..e16a87d5f 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -40,7 +40,7 @@ namespace cudaungappedextender // Assumes that only one sequence is present per file std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); - magic_sequence query_file_path = "../data/example.fa"; + std::string query_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index 7ec3363d1..363361c5c 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -38,7 +38,7 @@ namespace cudaungappedextender // Assumes that only one sequence is present per file std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); - magic_sequence query_file_path = "../data/example.fa"; + std::string query_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); From ee2e3dd58c2ef92092cb91e5df9fa645352be8c0 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 21 Aug 2020 18:27:20 -0700 Subject: [PATCH 024/281] [cudaue] Add return 0 --- cudaungappedextender/samples/device_sample.cu | 2 ++ cudaungappedextender/samples/host_sample.cu | 1 + 2 files changed, 3 insertions(+) diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index e16a87d5f..b0a4fc15f 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -120,6 +120,8 @@ namespace cudaungappedextender } } + + return 0; } } // namespace cudaungappedextender diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index 363361c5c..a8ff2567d 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -87,6 +87,7 @@ namespace cudaungappedextender i++; } + return 0; } } From e28449b92d79a07f9fad0d4cc169b2193b1e620b Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 21 Aug 2020 18:35:08 -0700 Subject: [PATCH 025/281] [cudaue] namespace scope change --- cudaungappedextender/samples/device_sample.cu | 186 +++++++++--------- cudaungappedextender/samples/host_sample.cu | 129 ++++++------ 2 files changed, 151 insertions(+), 164 deletions(-) diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index b0a4fc15f..868018f27 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -23,107 +23,101 @@ #include -namespace claraparabricks -{ -namespace genomeworks -{ -namespace cudaungappedextender +using namespace claraparabricks::genomeworks; +using namespace claraparabricks::genomeworks::cudaungappedextender; + +int main(int argc, char* argv[]) { - int main(int argc, char* argv[]) - { - const int32_t input_xdrop = 10; - const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; - // Fasta query and target files - std::string target_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); - // Assumes that only one sequence is present per file - std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); - - std::string query_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); - // Assumes that only one sequence is present per file - magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - - // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ - std::string anchors_file_path = "../data/example_hits.csv"; - - std::vector h_hits; - // Following function loops through all hits in the anchors.csv and returns results in - // the passed array - parse_anchors(anchors_file_path, h_hits); + const int32_t input_xdrop = 10; + const int32_t input_no_entropy = 0; + const int32_t hsp_threshold = 20000; + // Fasta query and target files + std::string target_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); + // Assumes that only one sequence is present per file + std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); + + std::string query_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); + // Assumes that only one sequence is present per file + magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); + + // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ + std::string anchors_file_path = "../data/example_hits.csv"; + + std::vector h_hits; + // Following function loops through all hits in the anchors.csv and returns results in + // the passed array + parse_anchors(anchors_file_path, h_hits); - // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; - std::string h_encoded_target = magic_encode(magic_base, target_sequence); - std::string h_encoded_query = magic_encode(magic_base, query_sequence); + // Following sections TBD based on encoding + ScoreMatrix = magic_number_matrix; + std::string h_encoded_target = magic_encode(magic_base, target_sequence); + std::string h_encoded_query = magic_encode(magic_base, query_sequence); - // Create a stream for async use - cudaStream_t stream0; - cudaStreamCreateWithFlags(&stream0, cudaStreamNonBlocking); - // Allocate space on device for target and query sequences, hits, - // high scoring segment pairs (hsps) and num_hsps. - char* d_query; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_query, sizeof(char)*h_encoded_query.size())); - char* d_target; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_target, sizeof(char)*h_target_query.size())); - Anchor* d_hits; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_hits, sizeof(Anchor)*h_hits.size())); - // Allocate a minimum of num_hits as all hits could be hsps in the worst case - int32_t h_num_hsps = 0; - ScoredSegment* d_hsps; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_hsps, sizeof(ScoredSegment)*h_hits.size())); - int32_t* d_num_hsps; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_num_hsps, sizeof(int32_t)); + // Create a stream for async use + cudaStream_t stream0; + cudaStreamCreateWithFlags(&stream0, cudaStreamNonBlocking); + // Allocate space on device for target and query sequences, hits, + // high scoring segment pairs (hsps) and num_hsps. + char* d_query; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_query, sizeof(char)*h_encoded_query.size())); + char* d_target; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_target, sizeof(char)*h_target_query.size())); + Anchor* d_hits; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_hits, sizeof(Anchor)*h_hits.size())); + // Allocate a minimum of num_hits as all hits could be hsps in the worst case + int32_t h_num_hsps = 0; + ScoredSegment* d_hsps; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_hsps, sizeof(ScoredSegment)*h_hits.size())); + int32_t* d_num_hsps; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_num_hsps, sizeof(int32_t)); - // Async Memcopy all the input values to device - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query.c_str(), sizeof(char)*h_encoded_query.size(), cudaMemcpyHostToDevice, stream0)); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target.c_str(), sizeof(char)*h_encoded_target.size(), cudaMemcpyHostToDevice, stream0)); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_hits, &h_hits[0], sizeof(Anchor)*h_hits.size(), cudaMemcpyHostToDevice, stream0)); - - // Create an ungapped extender object - std::unique_ptr ungapped_extender = std::make_unique(0, - magic_number_matrix, - input_xdrop, - input_no_entropy, - stream0); - // Launch the ungapped extender device function - ungapped_extender->ungapped_extend(d_query, // Type TBD based on encoding - encoded_query.size(), - d_target.c_str(), - encoded_target.size(), - hsp_threshold, - d_hits, - h_hits.size(), - d_hsps, - d_num_hsps); - // Copy back the number of hsps to host - GW_CU_CHECK_ERR(cudaMemcpyAsync(&h_num_hsps, d_num_hsps, sizeof(int32_t), cudaMemcpyDeviceToHost, stream0)); - - // Wait for ungapped extender to finish - GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0)); + // Async Memcopy all the input values to device + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query.c_str(), sizeof(char)*h_encoded_query.size(), cudaMemcpyHostToDevice, stream0)); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target.c_str(), sizeof(char)*h_encoded_target.size(), cudaMemcpyHostToDevice, stream0)); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_hits, &h_hits[0], sizeof(Anchor)*h_hits.size(), cudaMemcpyHostToDevice, stream0)); + + // Create an ungapped extender object + std::unique_ptr ungapped_extender = std::make_unique(0, + magic_number_matrix, + input_xdrop, + input_no_entropy, + stream0); + // Launch the ungapped extender device function + ungapped_extender->ungapped_extend(d_query, // Type TBD based on encoding + encoded_query.size(), + d_target.c_str(), + encoded_target.size(), + hsp_threshold, + d_hits, + h_hits.size(), + d_hsps, + d_num_hsps); + // Copy back the number of hsps to host + GW_CU_CHECK_ERR(cudaMemcpyAsync(&h_num_hsps, d_num_hsps, sizeof(int32_t), cudaMemcpyDeviceToHost, stream0)); + + // Wait for ungapped extender to finish + GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0)); + + //Get results + if(h_num_hsps > 0) + { + std::vector h_hsps(h_num_hsps); + // Don't care about asynchronous copies here + GW_CU_CHECK_ERR(cudaMemcpy(&h_hsps[0], d_hsps, sizeof(ScoredSegment)*h_num_hsps, cudaMemcpyDeviceToHost)); - //Get results - if(h_num_hsps > 0) + int32_t i = 0; + for (const auto& segment : h_hsps) { - std::vector h_hsps(h_num_hsps); - // Don't care about asynchronous copies here - GW_CU_CHECK_ERR(cudaMemcpy(&h_hsps[0], d_hsps, sizeof(ScoredSegment)*h_num_hsps, cudaMemcpyDeviceToHost)); - - int32_t i = 0; - for (const auto& segment : h_hsps) - { - std::cout << "Segment: " << i << "Length: " << segment.len << "Score: " << segment.score << std::endl; - std::cout << "Position in query: " << segment.anchor.query_position_in_read_< #include - -namespace claraparabricks -{ -namespace genomeworks -{ -namespace cudaungappedextender +using namespace claraparabricks::genomeworks; +using namespace claraparabricks::genomeworks::cudaungappedextender; + +int main(int argc, char* argv[]) { - int main(int argc, char* argv[]) - { - const int32_t input_xdrop = 10; - const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; - // Fasta query and target files - std::string target_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); - // Assumes that only one sequence is present per file - std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); - - std::string query_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); - // Assumes that only one sequence is present per file - magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - - // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ - std::string anchors_file_path = "../data/example_hits.csv"; - - std::vector h_hits; - // Following function loops through all hits in the anchors.csv and returns results in - // the passed array - parse_anchors(anchors_file_path, h_hits); + const int32_t input_xdrop = 10; + const int32_t input_no_entropy = 0; + const int32_t hsp_threshold = 20000; + // Fasta query and target files + std::string target_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); + // Assumes that only one sequence is present per file + std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); + + std::string query_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); + // Assumes that only one sequence is present per file + magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); + + // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ + std::string anchors_file_path = "../data/example_hits.csv"; + + std::vector h_hits; + // Following function loops through all hits in the anchors.csv and returns results in + // the passed array + parse_anchors(anchors_file_path, h_hits); - // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; - std::string encoded_target = magic_encode(magic_base, target_sequence); - std::string encoded_query = magic_encode(magic_base, query_sequence); + // Following sections TBD based on encoding + ScoreMatrix = magic_number_matrix; + std::string encoded_target = magic_encode(magic_base, target_sequence); + std::string encoded_query = magic_encode(magic_base, query_sequence); - // Create a stream for async use - cudaStream_t stream0; - cudaStreamCreateWithFlags(&stream0, cudaStreamNonBlocking); - // Create an ungapped extender object - std::unique_ptr ungapped_extender = std::make_unique(0, - magic_number_matrix, - input_xdrop, - input_no_entropy, - stream0); - // Launch the ungapped extender host function - ungapped_extender->ungapped_extend(encoded_query.c_str(), // Type TBD based on encoding - encoded_query.size(), - encoded_target.c_str(), - encoded_target.size(), - hsp_threshold, - h_hits); - - // Wait for ungapped extender to finish - ungapped_extender->sync_extensions(); + // Create a stream for async use + cudaStream_t stream0; + cudaStreamCreateWithFlags(&stream0, cudaStreamNonBlocking); + // Create an ungapped extender object + std::unique_ptr ungapped_extender = std::make_unique(0, + magic_number_matrix, + input_xdrop, + input_no_entropy, + stream0); + // Launch the ungapped extender host function + ungapped_extender->ungapped_extend(encoded_query.c_str(), // Type TBD based on encoding + encoded_query.size(), + encoded_target.c_str(), + encoded_target.size(), + hsp_threshold, + h_hits); + + // Wait for ungapped extender to finish + ungapped_extender->sync_extensions(); - //Get results - const std::vector>& segments = ungapped_extender->get_scored_segments(); - int32_t i = 0; - for (const auto& segment : segments) - { - std::cout << "Segment: " << i << "Length: " << segment->len << "Score: " << segment->score << std::endl; - std::cout << "Position in query: " << segment->anchor.query_position_in_read_<anchor.target_position_in_read_<>& segments = ungapped_extender->get_scored_segments(); + int32_t i = 0; + for (const auto& segment : segments) + { + std::cout << "Segment: " << i << "Length: " << segment->len << "Score: " << segment->score << std::endl; + std::cout << "Position in query: " << segment->anchor.query_position_in_read_<anchor.target_position_in_read_< Date: Mon, 24 Aug 2020 16:28:24 +0200 Subject: [PATCH 026/281] [cudamapper] Renamed create_cache() to create_host_copy() in IndexHostCopyBase --- .../claraparabricks/genomeworks/cudamapper/index.hpp | 10 +++++----- cudamapper/src/index.cu | 10 +++++----- cudamapper/src/index_cache.cu | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index f4a0144c2..f4e95805e 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -203,11 +203,11 @@ class IndexHostCopyBase /// \param window_size the number of adjacent k-mers in a window, adjacent = shifted by one basepair /// \param cuda_stream D2H copy is done on this stream /// \return - an instance of IndexHostCopyBase - static std::unique_ptr create_cache(const Index& index, - const read_id_t first_read_id, - const std::uint64_t kmer_size, - const std::uint64_t window_size, - const cudaStream_t cuda_stream = 0); + static std::unique_ptr create_host_copy(const Index& index, + const read_id_t first_read_id, + const std::uint64_t kmer_size, + const std::uint64_t window_size, + const cudaStream_t cuda_stream = 0); }; } // namespace cudamapper diff --git a/cudamapper/src/index.cu b/cudamapper/src/index.cu index 19fe1da9b..b6aff3c18 100644 --- a/cudamapper/src/index.cu +++ b/cudamapper/src/index.cu @@ -52,11 +52,11 @@ std::unique_ptr Index::create_index(DefaultDeviceAllocator allocator, cuda_stream_copy); } -std::unique_ptr IndexHostCopyBase::create_cache(const Index& index, - const read_id_t first_read_id, - const std::uint64_t kmer_size, - const std::uint64_t window_size, - const cudaStream_t cuda_stream) +std::unique_ptr IndexHostCopyBase::create_host_copy(const Index& index, + const read_id_t first_read_id, + const std::uint64_t kmer_size, + const std::uint64_t window_size, + const cudaStream_t cuda_stream) { GW_NVTX_RANGE(profiler, "cache_D2H"); return std::make_unique(index, diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 6a13001be..809d11f0c 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -196,11 +196,11 @@ void IndexCache::generate_content_host(const std::vector& descr index_on_device_from_previous_step = nullptr; } - index_on_host = IndexHostCopy::create_cache(*index_on_device, - descriptor_of_index_to_cache.first_read(), - kmer_size_, - window_size_, - cuda_stream_copy_); + index_on_host = IndexHostCopy::create_host_copy(*index_on_device, + descriptor_of_index_to_cache.first_read(), + kmer_size_, + window_size_, + cuda_stream_copy_); started_copy = true; // index is being copied to host memory, a sync will be needed } } From a7a3f07a5645f190e7142dc5bf79afb94de3a362 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Mon, 24 Aug 2020 09:09:33 -0700 Subject: [PATCH 027/281] [cudaue] Free CUDA memory in device sample --- cudaungappedextender/samples/device_sample.cu | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index 868018f27..9a4df97b8 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -117,6 +117,13 @@ int main(int argc, char* argv[]) } } + + // Free all allocated memory on the GPU + GW_CU_CHECK_ERROR(cudaFree(d_query); + GW_CU_CHECK_ERROR(cudaFree(d_target); + GW_CU_CHECK_ERROR(cudaFree(d_hsps); + GW_CU_CHECK_ERROR(cudaFree(d_hits); + GW_CU_CHECK_ERROR(cudaFree(d_num_hsps); return 0; From 03781e36b5a926ff7edf2bc6dbfe51b52564bb7f Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Mon, 24 Aug 2020 09:12:03 -0700 Subject: [PATCH 028/281] [cudaue] Fix comment --- cudaungappedextender/samples/device_sample.cu | 4 ++-- cudaungappedextender/samples/host_sample.cu | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index 9a4df97b8..fcdeaebd1 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -46,8 +46,8 @@ int main(int argc, char* argv[]) std::string anchors_file_path = "../data/example_hits.csv"; std::vector h_hits; - // Following function loops through all hits in the anchors.csv and returns results in - // the passed array + // Following function loops through all hits in the anchors csv and returns results in + // the passed vector parse_anchors(anchors_file_path, h_hits); // Following sections TBD based on encoding diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index 50be98dd1..b38206371 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -43,8 +43,8 @@ int main(int argc, char* argv[]) std::string anchors_file_path = "../data/example_hits.csv"; std::vector h_hits; - // Following function loops through all hits in the anchors.csv and returns results in - // the passed array + // Following function loops through all hits in the anchors csv and returns results in + // the passed vector parse_anchors(anchors_file_path, h_hits); // Following sections TBD based on encoding From a940113c435b92a6807bcdcc07e5d991850e4011 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Mon, 24 Aug 2020 09:31:42 -0700 Subject: [PATCH 029/281] [cudaue] Primary formatting --- CMakeLists.txt | 1 + cudaungappedextender/CMakeLists.txt | 3 + cudaungappedextender/include/CMakeLists.txt | 3 + .../cudaungappedextender.hpp | 49 +-- cudaungappedextender/samples/CMakeLists.txt | 3 + cudaungappedextender/samples/device_sample.cu | 67 ++-- cudaungappedextender/samples/host_sample.cu | 80 +++-- .../samples/segalign_seedfilter.cu | 335 ++++++++++-------- 8 files changed, 304 insertions(+), 237 deletions(-) create mode 100644 cudaungappedextender/CMakeLists.txt create mode 100644 cudaungappedextender/include/CMakeLists.txt create mode 100644 cudaungappedextender/samples/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 786714ee0..686129871 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,7 @@ add_subdirectory(common/io) add_subdirectory(cudapoa) add_subdirectory(cudamapper) add_subdirectory(cudaaligner) +add_subdirectory(cudaungappedextender) # Add documentation generation. validate_boolean(gw_generate_docs) diff --git a/cudaungappedextender/CMakeLists.txt b/cudaungappedextender/CMakeLists.txt new file mode 100644 index 000000000..a76075e5f --- /dev/null +++ b/cudaungappedextender/CMakeLists.txt @@ -0,0 +1,3 @@ +# Enable formatting +gw_enable_auto_formatting("${CMAKE_CURRENT_SOURCE_DIR}") + diff --git a/cudaungappedextender/include/CMakeLists.txt b/cudaungappedextender/include/CMakeLists.txt new file mode 100644 index 000000000..a76075e5f --- /dev/null +++ b/cudaungappedextender/include/CMakeLists.txt @@ -0,0 +1,3 @@ +# Enable formatting +gw_enable_auto_formatting("${CMAKE_CURRENT_SOURCE_DIR}") + diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index 34cb44f49..abdfc3cb6 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -30,7 +30,8 @@ struct Anchor position_in_read_t target_position_in_read_; }; -struct ScoredSegment { +struct ScoredSegment +{ /// Anchor for the segment Anchor anchor; /// length of the segment @@ -42,57 +43,45 @@ struct ScoredSegment { /// CUDA Ungapped Extension object class UngappedExtender { - public: +public: /// \brief Destructor for UngappedExtender. ~UngappedExtender() = default; /// \brief Constructor Prototype - UngappedExtender(int* h_sub_mat, - int input_xdrop, - bool input_noentropy, - int gpu_id=0, - cudaStream_t stream=0); - - - + UngappedExtender(int* h_sub_mat, int input_xdrop, bool input_noentropy, + int gpu_id = 0, cudaStream_t stream = 0); + /// \brief Host pointer prototype for ungapped extension /// /// Takes values from host data structures, - /// copies them over to device, + /// copies them over to device, /// launches async extension kernels on specified stream. Filters /// segments on device based on input_hspthresh. - StatusType ungapped_extend(const char* h_query, - int32_t query_length, - const char* h_target, - int32_t target_length, + StatusType ungapped_extend(const char* h_query, int32_t query_length, + const char* h_target, int32_t target_length, int32_t input_hspthresh, std::vector& h_seedHits); - + /// \brief Device pointer prototype for ungapped extension /// - /// Memcopies to device memory are assumed to be done before this + /// Memcopies to device memory are assumed to be done before this /// function. Output array d_hsp_out must be pre-allocated on device. - /// Launches async extension kernel. Filters segments on device + /// Launches async extension kernel. Filters segments on device /// based on input_hspthresh. - StatusType ungapped_extend(const char* d_query, - int32_t query_length, - const char* d_target, - int32_t target_length, - int32_t input_hspthresh, - Anchor* d_seed_hits, - int32_t num_hits, - ScoredSegment* d_hsp_out, + StatusType ungapped_extend(const char* d_query, int32_t query_length, + const char* d_target, int32_t target_length, + int32_t input_hspthresh, Anchor* d_seed_hits, + int32_t num_hits, ScoredSegment* d_hsp_out, int32_t* d_num_hsps); - + /// \brief Waits for CUDA accelerated extension to finish /// /// Blocking call that waits for all the extensions scheduled /// on the GPU to come to completion. StatusType sync_extensions(); - - + /// \brief Return the computed segments /// - /// \return Vector of Scored Segments + /// \return Vector of Scored Segments const std::vector>& get_scored_segments(); /// \brief Reset UngappedExtender object. diff --git a/cudaungappedextender/samples/CMakeLists.txt b/cudaungappedextender/samples/CMakeLists.txt new file mode 100644 index 000000000..a76075e5f --- /dev/null +++ b/cudaungappedextender/samples/CMakeLists.txt @@ -0,0 +1,3 @@ +# Enable formatting +gw_enable_auto_formatting("${CMAKE_CURRENT_SOURCE_DIR}") + diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index fcdeaebd1..ec2ea5f28 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -13,47 +13,50 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include +#include #include #include -#include -#include -#include -#include #include - +#include +#include using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::cudaungappedextender; int main(int argc, char* argv[]) { - const int32_t input_xdrop = 10; + const int32_t input_xdrop = 10; const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; + const int32_t hsp_threshold = 20000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); + std::unique_ptr fasta_parser_target = + io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file - std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); - + std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); + std::string query_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); + std::unique_ptr fasta_parser_query = + io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file - magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - - // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ + magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); + + // CSV Anchors file - Each row -> query_position_in_read_, + // target_position_in_read_ std::string anchors_file_path = "../data/example_hits.csv"; - + std::vector h_hits; - // Following function loops through all hits in the anchors csv and returns results in + // Following function loops through all hits in the anchors csv and returns + // results in // the passed vector parse_anchors(anchors_file_path, h_hits); // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; + ScoreMatrix = magic_number_matrix; std::string h_encoded_target = magic_encode(magic_base, target_sequence); - std::string h_encoded_query = magic_encode(magic_base, query_sequence); + std::string h_encoded_query = magic_encode(magic_base, query_sequence); // Create a stream for async use cudaStream_t stream0; @@ -61,15 +64,19 @@ int main(int argc, char* argv[]) // Allocate space on device for target and query sequences, hits, // high scoring segment pairs (hsps) and num_hsps. char* d_query; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_query, sizeof(char)*h_encoded_query.size())); + GW_CU_CHECK_ERROR( + cudaMalloc((void**)&d_query, sizeof(char) * h_encoded_query.size())); char* d_target; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_target, sizeof(char)*h_target_query.size())); + GW_CU_CHECK_ERROR( + cudaMalloc((void**)&d_target, sizeof(char) * h_target_query.size())); Anchor* d_hits; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_hits, sizeof(Anchor)*h_hits.size())); + GW_CU_CHECK_ERROR( + cudaMalloc((void**)&d_hits, sizeof(Anchor) * h_hits.size())); // Allocate a minimum of num_hits as all hits could be hsps in the worst case int32_t h_num_hsps = 0; ScoredSegment* d_hsps; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_hsps, sizeof(ScoredSegment)*h_hits.size())); + GW_CU_CHECK_ERROR( + cudaMalloc((void**)&d_hsps, sizeof(ScoredSegment) * h_hits.size())); int32_t* d_num_hsps; GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_num_hsps, sizeof(int32_t)); @@ -105,14 +112,19 @@ int main(int argc, char* argv[]) { std::vector h_hsps(h_num_hsps); // Don't care about asynchronous copies here - GW_CU_CHECK_ERR(cudaMemcpy(&h_hsps[0], d_hsps, sizeof(ScoredSegment)*h_num_hsps, cudaMemcpyDeviceToHost)); - + GW_CU_CHECK_ERR(cudaMemcpy(&h_hsps[0], d_hsps, + sizeof(ScoredSegment) * h_num_hsps, + cudaMemcpyDeviceToHost)); + int32_t i = 0; for (const auto& segment : h_hsps) { - std::cout << "Segment: " << i << "Length: " << segment.len << "Score: " << segment.score << std::endl; - std::cout << "Position in query: " << segment.anchor.query_position_in_read_< +#include +#include +#include #include #include -#include -#include -#include -#include using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::cudaungappedextender; - + int main(int argc, char* argv[]) { - const int32_t input_xdrop = 10; + const int32_t input_xdrop = 10; const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; + const int32_t hsp_threshold = 20000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); + std::unique_ptr fasta_parser_target = + io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file - std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); - + std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); + std::string query_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); + std::unique_ptr fasta_parser_query = + io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file - magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - - // CSV Anchors file - Each row -> query_position_in_read_, target_position_in_read_ + magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); + + // CSV Anchors file - Each row -> query_position_in_read_, + // target_position_in_read_ std::string anchors_file_path = "../data/example_hits.csv"; - + std::vector h_hits; - // Following function loops through all hits in the anchors csv and returns results in + // Following function loops through all hits in the anchors csv and returns + // results in // the passed vector parse_anchors(anchors_file_path, h_hits); // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; - std::string encoded_target = magic_encode(magic_base, target_sequence); - std::string encoded_query = magic_encode(magic_base, query_sequence); + ScoreMatrix = magic_number_matrix; + std::string encoded_target = magic_encode(magic_base, target_sequence); + std::string encoded_query = magic_encode(magic_base, query_sequence); // Create a stream for async use cudaStream_t stream0; cudaStreamCreateWithFlags(&stream0, cudaStreamNonBlocking); // Create an ungapped extender object - std::unique_ptr ungapped_extender = std::make_unique(0, - magic_number_matrix, - input_xdrop, - input_no_entropy, - stream0); + std::unique_ptr ungapped_extender = + std::make_unique(0, magic_number_matrix, input_xdrop, + input_no_entropy, stream0); // Launch the ungapped extender host function - ungapped_extender->ungapped_extend(encoded_query.c_str(), // Type TBD based on encoding - encoded_query.size(), - encoded_target.c_str(), - encoded_target.size(), - hsp_threshold, - h_hits); - + ungapped_extender->ungapped_extend( + encoded_query.c_str(), // Type TBD based on encoding + encoded_query.size(), encoded_target.c_str(), encoded_target.size(), + hsp_threshold, h_hits); + // Wait for ungapped extender to finish ungapped_extender->sync_extensions(); - //Get results - const std::vector>& segments = ungapped_extender->get_scored_segments(); + // Get results + const std::vector>& segments = + ungapped_extender->get_scored_segments(); int32_t i = 0; for (const auto& segment : segments) { - std::cout << "Segment: " << i << "Length: " << segment->len << "Score: " << segment->score << std::endl; - std::cout << "Position in query: " << segment->anchor.query_position_in_read_<anchor.target_position_in_read_<len + << "Score: " << segment->score << std::endl; + std::cout << "Position in query: " + << segment->anchor.query_position_in_read_ << std::endl; + std::cout << "Position in target: " + << segment->anchor.target_position_in_read_ << std::endl; i++; } - + return 0; - } - diff --git a/cudaungappedextender/samples/segalign_seedfilter.cu b/cudaungappedextender/samples/segalign_seedfilter.cu index ab8cf0430..2f2a9c55f 100644 --- a/cudaungappedextender/samples/segalign_seedfilter.cu +++ b/cudaungappedextender/samples/segalign_seedfilter.cu @@ -1,3 +1,5 @@ +#include "parameters.h" +#include "seed_filter.h" #include #include #include @@ -5,8 +7,6 @@ #include #include #include -#include "parameters.h" -#include "seed_filter.h" #define MAX_SEED_HITS_PER_GB 8388608 #define MAX_UNGAPPED_PER_GB 4194304 @@ -14,14 +14,12 @@ using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::cudaungappedextender; - // Control Variables std::mutex mu; std::condition_variable cv; std::vector available_gpus; std::vector g_cuda_ungapped_extenders; // indexed by gpu_id -std::vector g_cuda_streams; // indexed by gpu_id - +std::vector g_cuda_streams; // indexed by gpu_id int NUM_DEVICES; @@ -43,65 +41,77 @@ uint32_t** d_pos_table; uint64_t** d_seed_offsets; uint32_t** d_hit_num_array; -std::vector > d_hit_num_vec; +std::vector> d_hit_num_vec; Anchor** d_hit; -std::vector > d_hit_vec; +std::vector> d_hit_vec; ScoredSegment** d_hsp; -std::vector > d_hsp_vec; +std::vector> d_hsp_vec; -//UngappedExtend Variables (ideally not visible to the user in the API) -uint32_t MAX_UNGAPPED; //maximum extensions per iteration in the UngappedExtension function - -int **d_sub_mat; // substitution score matrix -int xdrop; // xdrop parameter for the UngappedExtension function -int hspthresh; // score threshold for qualifying as an HSP -bool noentropy; // whether or not to adjust scores of segments as a factor of the Shannon entropy - -// convert input sequence from alphabet to integers -__global__ -void compress_string (char* dst_seq, char* src_seq, uint32_t len){ -... -} - -// convert input sequence to its reverse complement and convert from alphabet to integers -__global__ -void compress_rev_comp_string (char* dst_seq, char* src_seq, uint32_t len){ -... -} +// UngappedExtend Variables (ideally not visible to the user in the API) +uint32_t MAX_UNGAPPED; // maximum extensions per iteration in the + // UngappedExtension function +int** d_sub_mat; // substitution score matrix +int xdrop; // xdrop parameter for the UngappedExtension function +int hspthresh; // score threshold for qualifying as an HSP +bool noentropy; // whether or not to adjust scores of segments as a factor of + // the Shannon entropy -///////////////////// End Ungapped Extension related functions executed on the GPU /////////////// - -__global__ -void find_num_hits (int num_seeds, const uint32_t* __restrict__ d_index_table, uint64_t* seed_offsets, uint32_t* seed_hit_num){ - ... -} - -__global__ -void find_hits (const uint32_t* __restrict__ d_index_table, const uint32_t* __restrict__ d_pos_table, uint64_t* d_seed_offsets, uint32_t seed_size, uint32_t* seed_hit_num, int num_hits, Anchor* d_hit, uint32_t start_seed_index, uint32_t start_hit_index){ +// convert input sequence from alphabet to integers +__global__ void compress_string(char* dst_seq, char* src_seq, uint32_t len){ + ...} + +// convert input sequence to its reverse complement and convert from alphabet to +// integers +__global__ void compress_rev_comp_string(char* dst_seq, char* src_seq, + uint32_t len){ + ...} + +///////////////////// End Ungapped Extension related functions executed on the +/// GPU /////////////// + +__global__ void find_num_hits(int num_seeds, + const uint32_t* __restrict__ d_index_table, + uint64_t* seed_offsets, uint32_t* seed_hit_num){ + ...} + +__global__ void find_hits(const uint32_t* __restrict__ d_index_table, + const uint32_t* __restrict__ d_pos_table, + uint64_t* d_seed_offsets, uint32_t seed_size, + uint32_t* seed_hit_num, int num_hits, Anchor* d_hit, + uint32_t start_seed_index, uint32_t start_hit_index) +{ ... } -int InitializeProcessor (int num_gpu, bool transition, uint32_t WGA_CHUNK, uint32_t input_seed_size, int* sub_mat, int input_xdrop, int input_hspthresh, bool input_noentropy){ +int InitializeProcessor(int num_gpu, bool transition, uint32_t WGA_CHUNK, + uint32_t input_seed_size, int* sub_mat, int input_xdrop, + int input_hspthresh, bool input_noentropy) +{ int nDevices; - + cudaError_t err = cudaGetDeviceCount(&nDevices); - if (err != cudaSuccess) { + if (err != cudaSuccess) + { fprintf(stderr, "Error: No GPU device found!\n"); exit(1); } - if(num_gpu == -1){ - NUM_DEVICES = nDevices; + if (num_gpu == -1) + { + NUM_DEVICES = nDevices; } - else{ - if(num_gpu <= nDevices){ + else + { + if (num_gpu <= nDevices) + { NUM_DEVICES = num_gpu; } - else{ + else + { fprintf(stderr, "Requested GPUs greater than available GPUs\n"); exit(10); } @@ -111,49 +121,52 @@ int InitializeProcessor (int num_gpu, bool transition, uint32_t WGA_CHUNK, uint3 seed_size = input_seed_size; - if(transition) - MAX_SEEDS = 13*WGA_CHUNK; + if (transition) + MAX_SEEDS = 13 * WGA_CHUNK; else MAX_SEEDS = WGA_CHUNK; cudaDeviceProp deviceProp; cudaGetDeviceProperties(&deviceProp, 0); - float global_mem_gb = static_cast(deviceProp.totalGlobalMem / 1073741824.0f); - MAX_SEED_HITS = global_mem_gb*MAX_SEED_HITS_PER_GB; + float global_mem_gb = + static_cast(deviceProp.totalGlobalMem / 1073741824.0f); + MAX_SEED_HITS = global_mem_gb * MAX_SEED_HITS_PER_GB; Anchor zeroHit; - zeroHit.query_position_in_read_ = 0; + zeroHit.query_position_in_read_ = 0; zeroHit.target_position_in_read_ = 0; ScoredSegment zeroHsp; - zeroHsp.anchor.query_position_in_read_ = 0; + zeroHsp.anchor.query_position_in_read_ = 0; zeroHsp.anchor.target_position_in_read_ = 0; - zeroHsp.len = 0; - zeroHsp.score = 0; + zeroHsp.len = 0; + zeroHsp.score = 0; + + d_ref_seq = (char**)malloc(NUM_DEVICES * sizeof(char*)); + d_query_seq = (char**)malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char*)); + d_query_rc_seq = (char**)malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char*)); - d_ref_seq = (char**) malloc(NUM_DEVICES*sizeof(char*)); - d_query_seq = (char**) malloc(BUFFER_DEPTH*NUM_DEVICES*sizeof(char*)); - d_query_rc_seq = (char**) malloc(BUFFER_DEPTH*NUM_DEVICES*sizeof(char*)); - - d_index_table = (uint32_t**) malloc(NUM_DEVICES*sizeof(uint32_t*)); - d_pos_table = (uint32_t**) malloc(NUM_DEVICES*sizeof(uint32_t*)); + d_index_table = (uint32_t**)malloc(NUM_DEVICES * sizeof(uint32_t*)); + d_pos_table = (uint32_t**)malloc(NUM_DEVICES * sizeof(uint32_t*)); - d_seed_offsets = (uint64_t**) malloc(NUM_DEVICES*sizeof(uint64_t*)); + d_seed_offsets = (uint64_t**)malloc(NUM_DEVICES * sizeof(uint64_t*)); - d_hit_num_array = (uint32_t**) malloc(NUM_DEVICES*sizeof(int32_t*)); + d_hit_num_array = (uint32_t**)malloc(NUM_DEVICES * sizeof(int32_t*)); d_hit_num_vec.reserve(NUM_DEVICES); - d_hit = (seedHit**) malloc(NUM_DEVICES*sizeof(Anchor*)); + d_hit = (seedHit**)malloc(NUM_DEVICES * sizeof(Anchor*)); d_hit_vec.reserve(NUM_DEVICES); - d_hsp = (segment**) malloc(NUM_DEVICES*sizeof(ScoredSegment*)); + d_hsp = (segment**)malloc(NUM_DEVICES * sizeof(ScoredSegment*)); d_hsp_vec.reserve(NUM_DEVICES); - for(int g = 0; g < NUM_DEVICES; g++){ + for (int g = 0; g < NUM_DEVICES; g++) + { check_cuda_setDevice(g, "InitializeProcessor"); - check_cuda_malloc((void**)&d_seed_offsets[g], MAX_SEEDS*sizeof(uint64_t), "seed_offsets"); + check_cuda_malloc((void**)&d_seed_offsets[g], MAX_SEEDS * sizeof(uint64_t), + "seed_offsets"); d_hit_num_vec.emplace_back(MAX_SEEDS, 0); d_hit_num_array[g] = thrust::raw_pointer_cast(d_hit_num_vec.at(g).data()); @@ -166,44 +179,54 @@ int InitializeProcessor (int num_gpu, bool transition, uint32_t WGA_CHUNK, uint3 cudaStream_t stream; cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking); g_cuda_streams.push_back(stream); - g_cuda_ungapped_extenders.emplace_back(g, sub_mat, input_xdrop, input_noentropy, stream); + g_cuda_ungapped_extenders.emplace_back(g, sub_mat, input_xdrop, + input_noentropy, stream); available_gpus.push_back(g); } return NUM_DEVICES; } -void InclusivePrefixScan (uint32_t* data, uint32_t len) { -... +void InclusivePrefixScan(uint32_t* data, uint32_t len) +{ + ... } -void SendSeedPosTable (uint32_t* index_table, uint32_t index_table_size, uint32_t* pos_table, uint32_t num_index, uint32_t max_pos_index){ -... +void SendSeedPosTable(uint32_t* index_table, uint32_t index_table_size, + uint32_t* pos_table, uint32_t num_index, + uint32_t max_pos_index) +{ + ... } -void SendRefWriteRequest (size_t start_addr, uint32_t len){ -... +void SendRefWriteRequest(size_t start_addr, uint32_t len) +{ + ... } -void SendQueryWriteRequest (size_t start_addr, uint32_t len, uint32_t buffer){ -... -} +void SendQueryWriteRequest(size_t start_addr, uint32_t len, uint32_t buffer){ + ...} -std::vector SeedAndFilter (std::vector seed_offset_vector, bool rev, uint32_t buffer, int input_hspthresh){ +std::vector SeedAndFilter(std::vector seed_offset_vector, + bool rev, uint32_t buffer, + int input_hspthresh) +{ - uint32_t num_hits = 0; + uint32_t num_hits = 0; uint32_t total_anchors = 0; uint32_t num_seeds = seed_offset_vector.size(); - uint64_t* tmp_offset = (uint64_t*) malloc(num_seeds*sizeof(uint64_t)); - for (uint32_t i = 0; i < num_seeds; i++) { + uint64_t* tmp_offset = (uint64_t*)malloc(num_seeds * sizeof(uint64_t)); + for (uint32_t i = 0; i < num_seeds; i++) + { tmp_offset[i] = seed_offset_vector[i]; } int g; std::unique_lock locker(mu); - if (available_gpus.empty()) { - cv.wait(locker, [](){return !available_gpus.empty();}); + if (available_gpus.empty()) + { + cv.wait(locker, []() { return !available_gpus.empty(); }); } g = available_gpus.back(); available_gpus.pop_back(); @@ -211,70 +234,99 @@ std::vector SeedAndFilter (std::vector seed_offset_vector, bo check_cuda_setDevice(g, "SeedAndFilter"); - check_cuda_memcpy((void*)d_seed_offsets[g], (void*)tmp_offset, num_seeds*sizeof(uint64_t), cudaMemcpyHostToDevice, "seed_offsets"); + check_cuda_memcpy((void*)d_seed_offsets[g], (void*)tmp_offset, + num_seeds * sizeof(uint64_t), cudaMemcpyHostToDevice, + "seed_offsets"); + + find_num_hits<<>>( + num_seeds, d_index_table[g], d_seed_offsets[g], d_hit_num_array[g]); - find_num_hits <<>> (num_seeds, d_index_table[g], d_seed_offsets[g], d_hit_num_array[g]); + thrust::inclusive_scan(d_hit_num_vec[g].begin(), + d_hit_num_vec[g].begin() + num_seeds, + d_hit_num_vec[g].begin()); - thrust::inclusive_scan(d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, d_hit_num_vec[g].begin()); + check_cuda_memcpy((void*)&num_hits, + (void*)(d_hit_num_array[g] + num_seeds - 1), + sizeof(uint32_t), cudaMemcpyDeviceToHost, "num_hits"); - check_cuda_memcpy((void*)&num_hits, (void*)(d_hit_num_array[g]+num_seeds-1), sizeof(uint32_t), cudaMemcpyDeviceToHost, "num_hits"); - - int num_iter = num_hits/MAX_UNGAPPED+1; + int num_iter = num_hits / MAX_UNGAPPED + 1; uint32_t iter_hit_limit = MAX_UNGAPPED; - thrust::device_vector limit_pos (num_iter); + thrust::device_vector limit_pos(num_iter); - for(int i = 0; i < num_iter-1; i++){ - thrust::device_vector::iterator result_end = thrust::lower_bound(d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin()+num_seeds, iter_hit_limit); - uint32_t pos = thrust::distance(d_hit_num_vec[g].begin(), result_end)-1; - iter_hit_limit = d_hit_num_vec[g][pos]+MAX_UNGAPPED; - limit_pos[i] = pos; + for (int i = 0; i < num_iter - 1; i++) + { + thrust::device_vector::iterator result_end = thrust::lower_bound( + d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, + iter_hit_limit); + uint32_t pos = thrust::distance(d_hit_num_vec[g].begin(), result_end) - 1; + iter_hit_limit = d_hit_num_vec[g][pos] + MAX_UNGAPPED; + limit_pos[i] = pos; } - limit_pos[num_iter-1] = num_seeds-1; + limit_pos[num_iter - 1] = num_seeds - 1; - segment** h_hsp = (segment**) malloc(num_iter*sizeof(segment*)); - uint32_t* num_anchors = (uint32_t*) calloc(num_iter, sizeof(uint32_t)); + segment** h_hsp = (segment**)malloc(num_iter * sizeof(segment*)); + uint32_t* num_anchors = (uint32_t*)calloc(num_iter, sizeof(uint32_t)); uint32_t start_seed_index = 0; - uint32_t start_hit_val = 0; + uint32_t start_hit_val = 0; uint32_t iter_num_seeds, iter_num_hits; int32_t* d_num_anchors; GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_anchors, sizeof(int32_t))); - if(num_hits > 0){ - - for(int i = 0; i < num_iter; i++){ + if (num_hits > 0) + { + + for (int i = 0; i < num_iter; i++) + { iter_num_seeds = limit_pos[i] + 1 - start_seed_index; iter_num_hits = d_hit_num_vec[g][limit_pos[i]] - start_hit_val; - find_hits <<>> (d_index_table[g], d_pos_table[g], d_seed_offsets[g], seed_size, d_hit_num_array[g], iter_num_hits, d_hit[g], start_seed_index, start_hit_val); - - if(rev){ - if(!g_cuda_ungapped_extenders[g].ungapped_extend(d_query_rc_seq[buffer*NUM_DEVICES+g], query_length[buffer], d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, d_hsp[g], d_num_anchors)); + find_hits<<>>( + d_index_table[g], d_pos_table[g], d_seed_offsets[g], seed_size, + d_hit_num_array[g], iter_num_hits, d_hit[g], start_seed_index, + start_hit_val); + + if (rev) + { + if (!g_cuda_ungapped_extenders[g].ungapped_extend( + d_query_rc_seq[buffer * NUM_DEVICES + g], query_length[buffer], + d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, + d_hsp[g], d_num_anchors)) + ; { - //err... + // err... } } - else{ - if(!g_cuda_ungapped_extenders[g].ungapped_extend(d_query_seq[buffer*NUM_DEVICES+g], query_length[buffer], d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, d_hsp[g], d_num_anchors)); + else + { + if (!g_cuda_ungapped_extenders[g].ungapped_extend( + d_query_seq[buffer * NUM_DEVICES + g], query_length[buffer], + d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, + d_hsp[g], d_num_anchors)) + ; { - //err... + // err... } } - GW_CU_CHECK_ERR(cudaMemcpyAsync(&num_anchors[i], d_num_anchors, sizeof(int32_t), cudaMemcpyDeviceToHost, g_cuda_streams[g])); + GW_CU_CHECK_ERR(cudaMemcpyAsync(&num_anchors[i], d_num_anchors, + sizeof(int32_t), cudaMemcpyDeviceToHost, + g_cuda_streams[g])); cudaStreamSynchronize(g_cuda_streams[g]); - total_anchors += num_anchors[i]; - if(num_anchors[i] > 0){ - h_hsp[i] = (segment*) calloc(num_anchors[i], sizeof(segment)); + if (num_anchors[i] > 0) + { + h_hsp[i] = (segment*)calloc(num_anchors[i], sizeof(segment)); - check_cuda_memcpy((void*)h_hsp[i], (void*)d_hsp[g], num_anchors[i]*sizeof(segment), cudaMemcpyDeviceToHost, "hsp_output"); + check_cuda_memcpy((void*)h_hsp[i], (void*)d_hsp[g], + num_anchors[i] * sizeof(segment), + cudaMemcpyDeviceToHost, "hsp_output"); } start_seed_index = limit_pos[i] + 1; - start_hit_val = d_hit_num_vec[g][limit_pos[i]]; + start_hit_val = d_hit_num_vec[g][limit_pos[i]]; } } @@ -290,40 +342,46 @@ std::vector SeedAndFilter (std::vector seed_offset_vector, bo std::vector gpu_filter_output; segment first_el; - first_el.len = total_anchors; + first_el.len = total_anchors; first_el.score = num_hits; gpu_filter_output.push_back(first_el); - if(total_anchors > 0){ - for(int it = 0; it < num_iter; it++){ + if (total_anchors > 0) + { + for (int it = 0; it < num_iter; it++) + { - for(int i = 0; i < num_anchors[it]; i++){ + for (int i = 0; i < num_anchors[it]; i++) + { gpu_filter_output.push_back(h_hsp[it][i]); } } free(h_hsp); } - + free(tmp_offset); return gpu_filter_output; } -void clearRef(){ -... +void clearRef() +{ + ... } -void clearQuery(uint32_t buffer){ -... +void clearQuery(uint32_t buffer) +{ + ... } -void ShutdownProcessor(){ +void ShutdownProcessor() +{ d_hit_num_vec.clear(); d_hit_vec.clear(); d_hsp_vec.clear(); g_cuda_ungapped_extenders.clear(); - for(auto& cudaStream: g_cuda_streams) + for (auto& cudaStream : g_cuda_streams) { cudaStreamDestroy(cudaStream); } @@ -333,27 +391,24 @@ void ShutdownProcessor(){ ///// Start Ungapped Extension related functions ///// -void CompressSeq(char* input_seq, char* output_seq, uint32_t len){ -... -} - -void CompressRevCompSeq(char* input_seq, char* output_seq, uint32_t len){ -... +void CompressSeq(char* input_seq, char* output_seq, uint32_t len) +{ + ... } +void CompressRevCompSeq(char* input_seq, char* output_seq, uint32_t len){...} - -CompressSeq_ptr g_CompressSeq = CompressSeq; +CompressSeq_ptr g_CompressSeq = CompressSeq; CompressRevCompSeq_ptr g_CompressRevCompSeq = CompressRevCompSeq; ///// End Ungapped Extension related functions ///// -InitializeProcessor_ptr g_InitializeProcessor = InitializeProcessor; -InclusivePrefixScan_ptr g_InclusivePrefixScan = InclusivePrefixScan; -SendSeedPosTable_ptr g_SendSeedPosTable = SendSeedPosTable; -SendRefWriteRequest_ptr g_SendRefWriteRequest = SendRefWriteRequest; +InitializeProcessor_ptr g_InitializeProcessor = InitializeProcessor; +InclusivePrefixScan_ptr g_InclusivePrefixScan = InclusivePrefixScan; +SendSeedPosTable_ptr g_SendSeedPosTable = SendSeedPosTable; +SendRefWriteRequest_ptr g_SendRefWriteRequest = SendRefWriteRequest; SendQueryWriteRequest_ptr g_SendQueryWriteRequest = SendQueryWriteRequest; -SeedAndFilter_ptr g_SeedAndFilter = SeedAndFilter; -clearRef_ptr g_clearRef = clearRef; -clearQuery_ptr g_clearQuery = clearQuery; -ShutdownProcessor_ptr g_ShutdownProcessor = ShutdownProcessor; +SeedAndFilter_ptr g_SeedAndFilter = SeedAndFilter; +clearRef_ptr g_clearRef = clearRef; +clearQuery_ptr g_clearQuery = clearQuery; +ShutdownProcessor_ptr g_ShutdownProcessor = ShutdownProcessor; From 401df0e4d6e6811c73d3925d139f634e0b338f02 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Tue, 25 Aug 2020 18:58:22 +0200 Subject: [PATCH 030/281] [cudamapper] Calling finish_copying() on index_on_host_from_previous_step instead of synchronizing stream --- cudamapper/src/index_cache.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 809d11f0c..4b631e6f6 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -190,7 +190,7 @@ void IndexCache::generate_content_host(const std::vector& descr // if a D2H copy has been been started in the previous step wait for it to finish if (started_copy_from_previous_step) { - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy_)); + index_on_host_from_previous_step->finish_copying(); // copying is done, these pointer are not needed anymore index_on_host_from_previous_step = nullptr; index_on_device_from_previous_step = nullptr; From 68a36dca86edff36d98a92b8a1fe1035ceabb511 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 26 Aug 2020 12:14:19 +0200 Subject: [PATCH 031/281] [cudmapper] Calling finish_copying() on index_on_host_from_previous_step instead of synchronizing stream for the last index as well --- cudamapper/src/index_cache.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 4b631e6f6..ce9a0475c 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -247,7 +247,7 @@ void IndexCache::generate_content_host(const std::vector& descr // wait for the last copy to finish if (started_copy_from_previous_step) { - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy_)); + index_on_host_from_previous_step->finish_copying(); } std::swap(new_cache, this_cache); From b174de7eb80a682acdd2a8c765dbe3f9c67d3afa Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 27 Aug 2020 19:07:10 +0200 Subject: [PATCH 032/281] [cudamapper] Set started_copy to falls at the end of IndexCache::generate_content_host() --- cudamapper/src/index_cache.cu | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index ce9a0475c..2548a920e 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -144,9 +144,10 @@ void IndexCache::generate_content_host(const std::vector& descr bool started_copy = false; // if index is found on host copy is not needed bool started_copy_from_previous_step = false; + const bool host_copy_needed = !skip_copy_to_host; + for (const IndexDescriptor& descriptor_of_index_to_cache : descriptors_of_indices_to_cache) { - const bool host_copy_needed = !skip_copy_to_host; const bool device_copy_needed = descriptors_of_indices_to_keep_on_device_set.count(descriptor_of_index_to_cache) != 0; // check if host copy already exists @@ -188,6 +189,7 @@ void IndexCache::generate_content_host(const std::vector& descr if (host_copy_needed) { // if a D2H copy has been been started in the previous step wait for it to finish + // TODO: do this sync using an event if (started_copy_from_previous_step) { index_on_host_from_previous_step->finish_copying(); @@ -213,7 +215,7 @@ void IndexCache::generate_content_host(const std::vector& descr // Device copy of index is only saved if is already exists, i.e. if the index has been generated // If the index has been found on host it won't be copied back to device at this point - // TODO: check device caches from this index in that case, this is not expected to happen frequently so performance gains are going to be small + // TODO: check whether this index is already present in device cache, this is not expected to happen frequently so performance gains are going to be small if (device_copy_needed && index_on_device) { indices_kept_on_device[descriptor_of_index_to_cache] = index_on_device; @@ -222,7 +224,7 @@ void IndexCache::generate_content_host(const std::vector& descr // if a D2H copy has been been started in the previous step and it has not been waited for yet wait for it to finish if (started_copy_from_previous_step && index_on_host_from_previous_step && index_on_device_from_previous_step) { - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_copy_)); + index_on_host_from_previous_step->finish_copying(); // copying is done, these pointer are not needed anymore index_on_host_from_previous_step = nullptr; index_on_device_from_previous_step = nullptr; @@ -242,6 +244,7 @@ void IndexCache::generate_content_host(const std::vector& descr } index_on_host = nullptr; index_on_device = nullptr; + started_copy = false; } // wait for the last copy to finish From 0a71edf95e19e3756459dc3e8c92c6f86fb3be88 Mon Sep 17 00:00:00 2001 From: atadkase Date: Thu, 27 Aug 2020 10:11:24 -0700 Subject: [PATCH 033/281] [cudaungappedextender]Address comments --- cudaungappedextender/include/CMakeLists.txt | 3 - .../cudaungappedextender.hpp | 122 ++++----- cudaungappedextender/samples/CMakeLists.txt | 3 - cudaungappedextender/samples/device_sample.cu | 103 ++++--- cudaungappedextender/samples/host_sample.cu | 47 ++-- .../samples/segalign_seedfilter.cu | 259 ++++++++---------- 6 files changed, 245 insertions(+), 292 deletions(-) delete mode 100644 cudaungappedextender/include/CMakeLists.txt delete mode 100644 cudaungappedextender/samples/CMakeLists.txt diff --git a/cudaungappedextender/include/CMakeLists.txt b/cudaungappedextender/include/CMakeLists.txt deleted file mode 100644 index a76075e5f..000000000 --- a/cudaungappedextender/include/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Enable formatting -gw_enable_auto_formatting("${CMAKE_CURRENT_SOURCE_DIR}") - diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index abdfc3cb6..7eec48593 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -15,78 +15,70 @@ */ #pragma once -namespace claraparabricks -{ -namespace genomeworks -{ -namespace cudaungappedextender -{ +namespace claraparabricks { + namespace genomeworks { + namespace cudaungappedextender { -struct Anchor -{ - /// position of first sketch element in query_read_id_ - position_in_read_t query_position_in_read_; - /// position of second sketch element in target_read_id_ - position_in_read_t target_position_in_read_; -}; + struct Seed { + /// position of first sketch element in query_read_id_ + position_in_read_t query_position_in_read_; + /// position of second sketch element in target_read_id_ + position_in_read_t target_position_in_read_; + }; -struct ScoredSegment -{ - /// Anchor for the segment - Anchor anchor; - /// length of the segment - int32_t len; - /// score of the segment - int32_t score; -}; + struct ScoredSegment { + /// Seed for the segment + Seed seed; + /// length of the segment + int32_t len; + /// score of the segment + int32_t score; + }; /// CUDA Ungapped Extension object -class UngappedExtender -{ -public: - /// \brief Destructor for UngappedExtender. - ~UngappedExtender() = default; - /// \brief Constructor Prototype - UngappedExtender(int* h_sub_mat, int input_xdrop, bool input_noentropy, - int gpu_id = 0, cudaStream_t stream = 0); + class UngappedExtender { + public: + /// \brief Constructor Prototype + UngappedExtender(int *h_sub_mat, int input_xdrop, bool input_noentropy, + int gpu_id = 0, cudaStream_t stream = 0); - /// \brief Host pointer prototype for ungapped extension - /// - /// Takes values from host data structures, - /// copies them over to device, - /// launches async extension kernels on specified stream. Filters - /// segments on device based on input_hspthresh. - StatusType ungapped_extend(const char* h_query, int32_t query_length, - const char* h_target, int32_t target_length, - int32_t input_hspthresh, - std::vector& h_seedHits); + /// \brief Host pointer prototype for ungapped extension + /// + /// Takes values from host data structures, + /// copies them over to device, + /// launches async extension kernels on specified stream. Filters + /// segments on device based on input_hspthresh. + StatusType extend_async(const char *h_query, int32_t query_length, + const char *h_target, int32_t target_length, + int32_t input_hspthresh, + std::vector &h_seedHits); - /// \brief Device pointer prototype for ungapped extension - /// - /// Memcopies to device memory are assumed to be done before this - /// function. Output array d_hsp_out must be pre-allocated on device. - /// Launches async extension kernel. Filters segments on device - /// based on input_hspthresh. - StatusType ungapped_extend(const char* d_query, int32_t query_length, - const char* d_target, int32_t target_length, - int32_t input_hspthresh, Anchor* d_seed_hits, - int32_t num_hits, ScoredSegment* d_hsp_out, - int32_t* d_num_hsps); + /// \brief Device pointer prototype for ungapped extension + /// + /// Memcopies to device memory are assumed to be done before this + /// function. Output array d_hsp_out must be pre-allocated on device. + /// Launches async extension kernel. Filters segments on device + /// based on input_hspthresh. + StatusType extend_async(const char *d_query, int32_t query_length, + const char *d_target, int32_t target_length, + int32_t input_hspthresh, Anchor *d_seed_hits, + int32_t num_hits, ScoredSegment *d_hsp_out, + int32_t *d_num_hsps); - /// \brief Waits for CUDA accelerated extension to finish - /// - /// Blocking call that waits for all the extensions scheduled - /// on the GPU to come to completion. - StatusType sync_extensions(); + /// \brief Waits for CUDA accelerated extension to finish + /// + /// Blocking call that waits for all the extensions scheduled + /// on the GPU to come to completion. + StatusType sync(); - /// \brief Return the computed segments - /// - /// \return Vector of Scored Segments - const std::vector>& get_scored_segments(); + /// \brief Return the computed segments + /// + /// \return Vector of Scored Segments + const std::vector &get_scored_segments(); - /// \brief Reset UngappedExtender object. - void reset(); -}; -} // namespace cudaungappedextender -} // namespace genomeworks + /// \brief Reset UngappedExtender object and free device/host memory + void reset(); + }; + } // namespace cudaungappedextender + } // namespace genomeworks } // namespace claraparabricks diff --git a/cudaungappedextender/samples/CMakeLists.txt b/cudaungappedextender/samples/CMakeLists.txt deleted file mode 100644 index a76075e5f..000000000 --- a/cudaungappedextender/samples/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Enable formatting -gw_enable_auto_formatting("${CMAKE_CURRENT_SOURCE_DIR}") - diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index ec2ea5f28..ab24f3dcf 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -25,21 +25,20 @@ using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::cudaungappedextender; -int main(int argc, char* argv[]) -{ - const int32_t input_xdrop = 10; +int main(int argc, char *argv[]) { + const int32_t input_xdrop = 10; const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; + const int32_t hsp_threshold = 20000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_target = - io::create_kseq_fasta_parser(target_file_path, 0, false); + io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); std::string query_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_query = - io::create_kseq_fasta_parser(query_file_path, 0, false); + io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); @@ -54,62 +53,63 @@ int main(int argc, char* argv[]) parse_anchors(anchors_file_path, h_hits); // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; + ScoreMatrix = magic_number_matrix; std::string h_encoded_target = magic_encode(magic_base, target_sequence); - std::string h_encoded_query = magic_encode(magic_base, query_sequence); + std::string h_encoded_query = magic_encode(magic_base, query_sequence); // Create a stream for async use - cudaStream_t stream0; - cudaStreamCreateWithFlags(&stream0, cudaStreamNonBlocking); + CudaStream stream0 = make_cuda_stream(); // Allocate space on device for target and query sequences, hits, // high scoring segment pairs (hsps) and num_hsps. - char* d_query; + char *d_query; GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_query, sizeof(char) * h_encoded_query.size())); - char* d_target; + cudaMalloc((void **) &d_query, sizeof(char) * h_encoded_query.size())); + char *d_target; GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_target, sizeof(char) * h_target_query.size())); - Anchor* d_hits; + cudaMalloc((void **) &d_target, sizeof(char) * h_target_query.size())); + Anchor *d_hits; GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_hits, sizeof(Anchor) * h_hits.size())); + cudaMalloc((void **) &d_hits, sizeof(Anchor) * h_hits.size())); // Allocate a minimum of num_hits as all hits could be hsps in the worst case int32_t h_num_hsps = 0; - ScoredSegment* d_hsps; + ScoredSegment *d_hsps; GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_hsps, sizeof(ScoredSegment) * h_hits.size())); - int32_t* d_num_hsps; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_num_hsps, sizeof(int32_t)); + cudaMalloc((void **) &d_hsps, sizeof(ScoredSegment) * h_hits.size())); + int32_t *d_num_hsps; + GW_CU_CHECK_ERROR(cudaMalloc((void **) &d_num_hsps, sizeof(int32_t)); // Async Memcopy all the input values to device - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query.c_str(), sizeof(char)*h_encoded_query.size(), cudaMemcpyHostToDevice, stream0)); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target.c_str(), sizeof(char)*h_encoded_target.size(), cudaMemcpyHostToDevice, stream0)); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_hits, &h_hits[0], sizeof(Anchor)*h_hits.size(), cudaMemcpyHostToDevice, stream0)); - + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query.c_str(), sizeof(char) * h_encoded_query.size(), + cudaMemcpyHostToDevice, stream0.get())); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target.c_str(), sizeof(char) * h_encoded_target.size(), + cudaMemcpyHostToDevice, stream0.get())); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_hits, &h_hits[0], sizeof(Anchor) * h_hits.size(), cudaMemcpyHostToDevice, + stream0.get()))); + // Create an ungapped extender object - std::unique_ptr ungapped_extender = std::make_unique(0, - magic_number_matrix, - input_xdrop, - input_no_entropy, - stream0); + std::unique_ptr ungapped_extender = std::make_unique(0, + magic_number_matrix, + input_xdrop, + input_no_entropy, + stream0.get()); // Launch the ungapped extender device function - ungapped_extender->ungapped_extend(d_query, // Type TBD based on encoding - encoded_query.size(), - d_target.c_str(), - encoded_target.size(), - hsp_threshold, - d_hits, - h_hits.size(), - d_hsps, - d_num_hsps); + ungapped_extender->extend_async(d_query, // Type TBD based on encoding + encoded_query.size(), + d_target.c_str(), + encoded_target.size(), + hsp_threshold, + d_hits, + h_hits.size(), + d_hsps, + d_num_hsps); // Copy back the number of hsps to host - GW_CU_CHECK_ERR(cudaMemcpyAsync(&h_num_hsps, d_num_hsps, sizeof(int32_t), cudaMemcpyDeviceToHost, stream0)); - + GW_CU_CHECK_ERR(cudaMemcpyAsync(&h_num_hsps, d_num_hsps, sizeof(int32_t), cudaMemcpyDeviceToHost, stream0.get())); + // Wait for ungapped extender to finish - GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0)); - + GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0.get())); + //Get results - if(h_num_hsps > 0) - { + if (h_num_hsps > 0) { std::vector h_hsps(h_num_hsps); // Don't care about asynchronous copies here GW_CU_CHECK_ERR(cudaMemcpy(&h_hsps[0], d_hsps, @@ -117,8 +117,7 @@ int main(int argc, char* argv[]) cudaMemcpyDeviceToHost)); int32_t i = 0; - for (const auto& segment : h_hsps) - { + for (const auto &segment : h_hsps) { std::cout << "Segment: " << i << "Length: " << segment.len << "Score: " << segment.score << std::endl; std::cout << "Position in query: " @@ -127,14 +126,14 @@ int main(int argc, char* argv[]) << segment.anchor.target_position_in_read_ << std::endl; i++; } - + } - + // Free all allocated memory on the GPU - GW_CU_CHECK_ERROR(cudaFree(d_query); - GW_CU_CHECK_ERROR(cudaFree(d_target); - GW_CU_CHECK_ERROR(cudaFree(d_hsps); - GW_CU_CHECK_ERROR(cudaFree(d_hits); + GW_CU_CHECK_ERROR(cudaFree(d_query); + GW_CU_CHECK_ERROR(cudaFree(d_target); + GW_CU_CHECK_ERROR(cudaFree(d_hsps); + GW_CU_CHECK_ERROR(cudaFree(d_hits); GW_CU_CHECK_ERROR(cudaFree(d_num_hsps); return 0; diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index 6b16067a3..ac70e0389 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -23,21 +23,20 @@ using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::cudaungappedextender; -int main(int argc, char* argv[]) -{ - const int32_t input_xdrop = 10; +int main(int argc, char *argv[]) { + const int32_t input_xdrop = 10; const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; + const int32_t hsp_threshold = 20000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_target = - io::create_kseq_fasta_parser(target_file_path, 0, false); + io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); std::string query_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_query = - io::create_kseq_fasta_parser(query_file_path, 0, false); + io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); @@ -52,38 +51,36 @@ int main(int argc, char* argv[]) parse_anchors(anchors_file_path, h_hits); // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; + ScoreMatrix = magic_number_matrix; std::string encoded_target = magic_encode(magic_base, target_sequence); - std::string encoded_query = magic_encode(magic_base, query_sequence); + std::string encoded_query = magic_encode(magic_base, query_sequence); // Create a stream for async use - cudaStream_t stream0; - cudaStreamCreateWithFlags(&stream0, cudaStreamNonBlocking); + CudaStream stream0 = make_cuda_stream(); // Create an ungapped extender object std::unique_ptr ungapped_extender = - std::make_unique(0, magic_number_matrix, input_xdrop, - input_no_entropy, stream0); + std::make_unique(0, magic_number_matrix, input_xdrop, + input_no_entropy, stream0.get()); // Launch the ungapped extender host function - ungapped_extender->ungapped_extend( - encoded_query.c_str(), // Type TBD based on encoding - encoded_query.size(), encoded_target.c_str(), encoded_target.size(), - hsp_threshold, h_hits); + ungapped_extender->extend_async( + encoded_query.c_str(), // Type TBD based on encoding + encoded_query.size(), encoded_target.c_str(), encoded_target.size(), + hsp_threshold, h_hits); // Wait for ungapped extender to finish - ungapped_extender->sync_extensions(); + ungapped_extender->sync(); // Get results - const std::vector>& segments = - ungapped_extender->get_scored_segments(); + const std::vector &segments = + ungapped_extender->get_scored_segments(); int32_t i = 0; - for (const auto& segment : segments) - { - std::cout << "Segment: " << i << "Length: " << segment->len - << "Score: " << segment->score << std::endl; + for (const auto &segment : segments) { + std::cout << "Segment: " << i << "Length: " << segment.len + << "Score: " << segment.score << std::endl; std::cout << "Position in query: " - << segment->anchor.query_position_in_read_ << std::endl; + << segment.anchor.query_position_in_read_ << std::endl; std::cout << "Position in target: " - << segment->anchor.target_position_in_read_ << std::endl; + << segment.anchor.target_position_in_read_ << std::endl; i++; } diff --git a/cudaungappedextender/samples/segalign_seedfilter.cu b/cudaungappedextender/samples/segalign_seedfilter.cu index 2f2a9c55f..138fb4bf8 100644 --- a/cudaungappedextender/samples/segalign_seedfilter.cu +++ b/cudaungappedextender/samples/segalign_seedfilter.cu @@ -27,91 +27,85 @@ int NUM_DEVICES; uint32_t MAX_SEEDS; uint32_t MAX_SEED_HITS; -char** d_ref_seq; +char **d_ref_seq; uint32_t ref_len; -char** d_query_seq; -char** d_query_rc_seq; +char **d_query_seq; +char **d_query_rc_seq; uint32_t query_length[BUFFER_DEPTH]; uint32_t seed_size; -uint32_t** d_index_table; -uint32_t** d_pos_table; +uint32_t **d_index_table; +uint32_t **d_pos_table; -uint64_t** d_seed_offsets; +uint64_t **d_seed_offsets; -uint32_t** d_hit_num_array; +uint32_t **d_hit_num_array; std::vector> d_hit_num_vec; -Anchor** d_hit; +Anchor **d_hit; std::vector> d_hit_vec; -ScoredSegment** d_hsp; +ScoredSegment **d_hsp; std::vector> d_hsp_vec; // UngappedExtend Variables (ideally not visible to the user in the API) uint32_t MAX_UNGAPPED; // maximum extensions per iteration in the - // UngappedExtension function +// UngappedExtension function -int** d_sub_mat; // substitution score matrix +int **d_sub_mat; // substitution score matrix int xdrop; // xdrop parameter for the UngappedExtension function int hspthresh; // score threshold for qualifying as an HSP bool noentropy; // whether or not to adjust scores of segments as a factor of - // the Shannon entropy +// the Shannon entropy // convert input sequence from alphabet to integers -__global__ void compress_string(char* dst_seq, char* src_seq, uint32_t len){ - ...} +__global__ void compress_string(char *dst_seq, char *src_seq, uint32_t len) { + ... +} // convert input sequence to its reverse complement and convert from alphabet to // integers -__global__ void compress_rev_comp_string(char* dst_seq, char* src_seq, - uint32_t len){ - ...} +__global__ void compress_rev_comp_string(char *dst_seq, char *src_seq, + uint32_t len) { + ... +} ///////////////////// End Ungapped Extension related functions executed on the /// GPU /////////////// __global__ void find_num_hits(int num_seeds, - const uint32_t* __restrict__ d_index_table, - uint64_t* seed_offsets, uint32_t* seed_hit_num){ - ...} - -__global__ void find_hits(const uint32_t* __restrict__ d_index_table, - const uint32_t* __restrict__ d_pos_table, - uint64_t* d_seed_offsets, uint32_t seed_size, - uint32_t* seed_hit_num, int num_hits, Anchor* d_hit, - uint32_t start_seed_index, uint32_t start_hit_index) -{ + const uint32_t *__restrict__ d_index_table, + uint64_t *seed_offsets, uint32_t *seed_hit_num) { + ... +} + +__global__ void find_hits(const uint32_t *__restrict__ d_index_table, + const uint32_t *__restrict__ d_pos_table, + uint64_t *d_seed_offsets, uint32_t seed_size, + uint32_t *seed_hit_num, int num_hits, Anchor *d_hit, + uint32_t start_seed_index, uint32_t start_hit_index) { ... } int InitializeProcessor(int num_gpu, bool transition, uint32_t WGA_CHUNK, - uint32_t input_seed_size, int* sub_mat, int input_xdrop, - int input_hspthresh, bool input_noentropy) -{ + uint32_t input_seed_size, int *sub_mat, int input_xdrop, + int input_hspthresh, bool input_noentropy) { int nDevices; cudaError_t err = cudaGetDeviceCount(&nDevices); - if (err != cudaSuccess) - { + if (err != cudaSuccess) { fprintf(stderr, "Error: No GPU device found!\n"); exit(1); } - if (num_gpu == -1) - { + if (num_gpu == -1) { NUM_DEVICES = nDevices; - } - else - { - if (num_gpu <= nDevices) - { + } else { + if (num_gpu <= nDevices) { NUM_DEVICES = num_gpu; - } - else - { + } else { fprintf(stderr, "Requested GPUs greater than available GPUs\n"); exit(10); } @@ -129,43 +123,42 @@ int InitializeProcessor(int num_gpu, bool transition, uint32_t WGA_CHUNK, cudaDeviceProp deviceProp; cudaGetDeviceProperties(&deviceProp, 0); float global_mem_gb = - static_cast(deviceProp.totalGlobalMem / 1073741824.0f); + static_cast(deviceProp.totalGlobalMem / 1073741824.0f); MAX_SEED_HITS = global_mem_gb * MAX_SEED_HITS_PER_GB; Anchor zeroHit; - zeroHit.query_position_in_read_ = 0; + zeroHit.query_position_in_read_ = 0; zeroHit.target_position_in_read_ = 0; ScoredSegment zeroHsp; - zeroHsp.anchor.query_position_in_read_ = 0; + zeroHsp.anchor.query_position_in_read_ = 0; zeroHsp.anchor.target_position_in_read_ = 0; - zeroHsp.len = 0; - zeroHsp.score = 0; + zeroHsp.len = 0; + zeroHsp.score = 0; - d_ref_seq = (char**)malloc(NUM_DEVICES * sizeof(char*)); - d_query_seq = (char**)malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char*)); - d_query_rc_seq = (char**)malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char*)); + d_ref_seq = (char **) malloc(NUM_DEVICES * sizeof(char *)); + d_query_seq = (char **) malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char *)); + d_query_rc_seq = (char **) malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char *)); - d_index_table = (uint32_t**)malloc(NUM_DEVICES * sizeof(uint32_t*)); - d_pos_table = (uint32_t**)malloc(NUM_DEVICES * sizeof(uint32_t*)); + d_index_table = (uint32_t **) malloc(NUM_DEVICES * sizeof(uint32_t *)); + d_pos_table = (uint32_t **) malloc(NUM_DEVICES * sizeof(uint32_t *)); - d_seed_offsets = (uint64_t**)malloc(NUM_DEVICES * sizeof(uint64_t*)); + d_seed_offsets = (uint64_t **) malloc(NUM_DEVICES * sizeof(uint64_t *)); - d_hit_num_array = (uint32_t**)malloc(NUM_DEVICES * sizeof(int32_t*)); + d_hit_num_array = (uint32_t **) malloc(NUM_DEVICES * sizeof(int32_t *)); d_hit_num_vec.reserve(NUM_DEVICES); - d_hit = (seedHit**)malloc(NUM_DEVICES * sizeof(Anchor*)); + d_hit = (seedHit **) malloc(NUM_DEVICES * sizeof(Anchor * )); d_hit_vec.reserve(NUM_DEVICES); - d_hsp = (segment**)malloc(NUM_DEVICES * sizeof(ScoredSegment*)); + d_hsp = (segment **) malloc(NUM_DEVICES * sizeof(ScoredSegment * )); d_hsp_vec.reserve(NUM_DEVICES); - for (int g = 0; g < NUM_DEVICES; g++) - { + for (int g = 0; g < NUM_DEVICES; g++) { check_cuda_setDevice(g, "InitializeProcessor"); - check_cuda_malloc((void**)&d_seed_offsets[g], MAX_SEEDS * sizeof(uint64_t), + check_cuda_malloc((void **) &d_seed_offsets[g], MAX_SEEDS * sizeof(uint64_t), "seed_offsets"); d_hit_num_vec.emplace_back(MAX_SEEDS, 0); @@ -186,46 +179,41 @@ int InitializeProcessor(int num_gpu, bool transition, uint32_t WGA_CHUNK, return NUM_DEVICES; } -void InclusivePrefixScan(uint32_t* data, uint32_t len) -{ +void InclusivePrefixScan(uint32_t *data, uint32_t len) { ... } -void SendSeedPosTable(uint32_t* index_table, uint32_t index_table_size, - uint32_t* pos_table, uint32_t num_index, - uint32_t max_pos_index) -{ +void SendSeedPosTable(uint32_t *index_table, uint32_t index_table_size, + uint32_t *pos_table, uint32_t num_index, + uint32_t max_pos_index) { ... } -void SendRefWriteRequest(size_t start_addr, uint32_t len) -{ +void SendRefWriteRequest(size_t start_addr, uint32_t len) { ... } -void SendQueryWriteRequest(size_t start_addr, uint32_t len, uint32_t buffer){ - ...} +void SendQueryWriteRequest(size_t start_addr, uint32_t len, uint32_t buffer) { + ... +} std::vector SeedAndFilter(std::vector seed_offset_vector, bool rev, uint32_t buffer, - int input_hspthresh) -{ + int input_hspthresh) { - uint32_t num_hits = 0; + uint32_t num_hits = 0; uint32_t total_anchors = 0; uint32_t num_seeds = seed_offset_vector.size(); - uint64_t* tmp_offset = (uint64_t*)malloc(num_seeds * sizeof(uint64_t)); - for (uint32_t i = 0; i < num_seeds; i++) - { + uint64_t *tmp_offset = (uint64_t *) malloc(num_seeds * sizeof(uint64_t)); + for (uint32_t i = 0; i < num_seeds; i++) { tmp_offset[i] = seed_offset_vector[i]; } int g; std::unique_lock locker(mu); - if (available_gpus.empty()) - { + if (available_gpus.empty()) { cv.wait(locker, []() { return !available_gpus.empty(); }); } g = available_gpus.back(); @@ -234,77 +222,69 @@ std::vector SeedAndFilter(std::vector seed_offset_vector, check_cuda_setDevice(g, "SeedAndFilter"); - check_cuda_memcpy((void*)d_seed_offsets[g], (void*)tmp_offset, + check_cuda_memcpy((void *) d_seed_offsets[g], (void *) tmp_offset, num_seeds * sizeof(uint64_t), cudaMemcpyHostToDevice, "seed_offsets"); - find_num_hits<<>>( - num_seeds, d_index_table[g], d_seed_offsets[g], d_hit_num_array[g]); + find_num_hits << < MAX_BLOCKS, MAX_THREADS >> > ( + num_seeds, d_index_table[g], d_seed_offsets[g], d_hit_num_array[g]); thrust::inclusive_scan(d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, d_hit_num_vec[g].begin()); - check_cuda_memcpy((void*)&num_hits, - (void*)(d_hit_num_array[g] + num_seeds - 1), + check_cuda_memcpy((void *) &num_hits, + (void *) (d_hit_num_array[g] + num_seeds - 1), sizeof(uint32_t), cudaMemcpyDeviceToHost, "num_hits"); - int num_iter = num_hits / MAX_UNGAPPED + 1; + int num_iter = num_hits / MAX_UNGAPPED + 1; uint32_t iter_hit_limit = MAX_UNGAPPED; thrust::device_vector limit_pos(num_iter); - for (int i = 0; i < num_iter - 1; i++) - { + for (int i = 0; i < num_iter - 1; i++) { thrust::device_vector::iterator result_end = thrust::lower_bound( - d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, - iter_hit_limit); - uint32_t pos = thrust::distance(d_hit_num_vec[g].begin(), result_end) - 1; + d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, + iter_hit_limit); + uint32_t pos = thrust::distance(d_hit_num_vec[g].begin(), result_end) - 1; iter_hit_limit = d_hit_num_vec[g][pos] + MAX_UNGAPPED; - limit_pos[i] = pos; + limit_pos[i] = pos; } limit_pos[num_iter - 1] = num_seeds - 1; - segment** h_hsp = (segment**)malloc(num_iter * sizeof(segment*)); - uint32_t* num_anchors = (uint32_t*)calloc(num_iter, sizeof(uint32_t)); + segment **h_hsp = (segment **) malloc(num_iter * sizeof(segment * )); + uint32_t *num_anchors = (uint32_t *) calloc(num_iter, sizeof(uint32_t)); uint32_t start_seed_index = 0; - uint32_t start_hit_val = 0; + uint32_t start_hit_val = 0; uint32_t iter_num_seeds, iter_num_hits; - int32_t* d_num_anchors; - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_anchors, sizeof(int32_t))); + int32_t *d_num_anchors; + GW_CU_CHECK_ERR(cudaMalloc((void **) &d_num_anchors, sizeof(int32_t))); - if (num_hits > 0) - { + if (num_hits > 0) { - for (int i = 0; i < num_iter; i++) - { + for (int i = 0; i < num_iter; i++) { iter_num_seeds = limit_pos[i] + 1 - start_seed_index; - iter_num_hits = d_hit_num_vec[g][limit_pos[i]] - start_hit_val; + iter_num_hits = d_hit_num_vec[g][limit_pos[i]] - start_hit_val; - find_hits<<>>( - d_index_table[g], d_pos_table[g], d_seed_offsets[g], seed_size, - d_hit_num_array[g], iter_num_hits, d_hit[g], start_seed_index, - start_hit_val); + find_hits << < iter_num_seeds, BLOCK_SIZE >> > ( + d_index_table[g], d_pos_table[g], d_seed_offsets[g], seed_size, + d_hit_num_array[g], iter_num_hits, d_hit[g], start_seed_index, + start_hit_val); - if (rev) - { + if (rev) { if (!g_cuda_ungapped_extenders[g].ungapped_extend( d_query_rc_seq[buffer * NUM_DEVICES + g], query_length[buffer], d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, - d_hsp[g], d_num_anchors)) - ; + d_hsp[g], d_num_anchors)); { // err... } - } - else - { + } else { if (!g_cuda_ungapped_extenders[g].ungapped_extend( d_query_seq[buffer * NUM_DEVICES + g], query_length[buffer], d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, - d_hsp[g], d_num_anchors)) - ; + d_hsp[g], d_num_anchors)); { // err... } @@ -316,17 +296,16 @@ std::vector SeedAndFilter(std::vector seed_offset_vector, total_anchors += num_anchors[i]; - if (num_anchors[i] > 0) - { - h_hsp[i] = (segment*)calloc(num_anchors[i], sizeof(segment)); + if (num_anchors[i] > 0) { + h_hsp[i] = (segment *) calloc(num_anchors[i], sizeof(segment)); - check_cuda_memcpy((void*)h_hsp[i], (void*)d_hsp[g], + check_cuda_memcpy((void *) h_hsp[i], (void *) d_hsp[g], num_anchors[i] * sizeof(segment), cudaMemcpyDeviceToHost, "hsp_output"); } start_seed_index = limit_pos[i] + 1; - start_hit_val = d_hit_num_vec[g][limit_pos[i]]; + start_hit_val = d_hit_num_vec[g][limit_pos[i]]; } } @@ -342,17 +321,14 @@ std::vector SeedAndFilter(std::vector seed_offset_vector, std::vector gpu_filter_output; segment first_el; - first_el.len = total_anchors; + first_el.len = total_anchors; first_el.score = num_hits; gpu_filter_output.push_back(first_el); - if (total_anchors > 0) - { - for (int it = 0; it < num_iter; it++) - { + if (total_anchors > 0) { + for (int it = 0; it < num_iter; it++) { - for (int i = 0; i < num_anchors[it]; i++) - { + for (int i = 0; i < num_anchors[it]; i++) { gpu_filter_output.push_back(h_hsp[it][i]); } } @@ -363,26 +339,22 @@ std::vector SeedAndFilter(std::vector seed_offset_vector, return gpu_filter_output; } -void clearRef() -{ +void clearRef() { ... } -void clearQuery(uint32_t buffer) -{ +void clearQuery(uint32_t buffer) { ... } -void ShutdownProcessor() -{ +void ShutdownProcessor() { d_hit_num_vec.clear(); d_hit_vec.clear(); d_hsp_vec.clear(); g_cuda_ungapped_extenders.clear(); - for (auto& cudaStream : g_cuda_streams) - { + for (auto &cudaStream : g_cuda_streams) { cudaStreamDestroy(cudaStream); } g_cuda_streams.clear(); @@ -391,24 +363,23 @@ void ShutdownProcessor() ///// Start Ungapped Extension related functions ///// -void CompressSeq(char* input_seq, char* output_seq, uint32_t len) -{ +void CompressSeq(char *input_seq, char *output_seq, uint32_t len) { ... } -void CompressRevCompSeq(char* input_seq, char* output_seq, uint32_t len){...} +void CompressRevCompSeq(char *input_seq, char *output_seq, uint32_t len) { ... } -CompressSeq_ptr g_CompressSeq = CompressSeq; +CompressSeq_ptr g_CompressSeq = CompressSeq; CompressRevCompSeq_ptr g_CompressRevCompSeq = CompressRevCompSeq; ///// End Ungapped Extension related functions ///// -InitializeProcessor_ptr g_InitializeProcessor = InitializeProcessor; -InclusivePrefixScan_ptr g_InclusivePrefixScan = InclusivePrefixScan; -SendSeedPosTable_ptr g_SendSeedPosTable = SendSeedPosTable; -SendRefWriteRequest_ptr g_SendRefWriteRequest = SendRefWriteRequest; +InitializeProcessor_ptr g_InitializeProcessor = InitializeProcessor; +InclusivePrefixScan_ptr g_InclusivePrefixScan = InclusivePrefixScan; +SendSeedPosTable_ptr g_SendSeedPosTable = SendSeedPosTable; +SendRefWriteRequest_ptr g_SendRefWriteRequest = SendRefWriteRequest; SendQueryWriteRequest_ptr g_SendQueryWriteRequest = SendQueryWriteRequest; -SeedAndFilter_ptr g_SeedAndFilter = SeedAndFilter; -clearRef_ptr g_clearRef = clearRef; -clearQuery_ptr g_clearQuery = clearQuery; -ShutdownProcessor_ptr g_ShutdownProcessor = ShutdownProcessor; +SeedAndFilter_ptr g_SeedAndFilter = SeedAndFilter; +clearRef_ptr g_clearRef = clearRef; +clearQuery_ptr g_clearQuery = clearQuery; +ShutdownProcessor_ptr g_ShutdownProcessor = ShutdownProcessor; From 1e5a4fe9b757789bbe81125ad415cf6496ceb11a Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 27 Aug 2020 10:25:57 -0700 Subject: [PATCH 034/281] [cudaungappedextender] Change Anchors to Seeds and clang-format --- .../cudaungappedextender.hpp | 120 +++++---- cudaungappedextender/samples/device_sample.cu | 84 +++--- cudaungappedextender/samples/host_sample.cu | 46 ++-- .../samples/segalign_seedfilter.cu | 255 ++++++++++-------- 4 files changed, 272 insertions(+), 233 deletions(-) diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index 7eec48593..5e676bcf6 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -15,70 +15,76 @@ */ #pragma once -namespace claraparabricks { - namespace genomeworks { - namespace cudaungappedextender { +namespace claraparabricks +{ +namespace genomeworks +{ +namespace cudaungappedextender +{ - struct Seed { - /// position of first sketch element in query_read_id_ - position_in_read_t query_position_in_read_; - /// position of second sketch element in target_read_id_ - position_in_read_t target_position_in_read_; - }; +struct Seed +{ + /// position of first sketch element in query_read_id_ + position_in_read_t query_position_in_read; + /// position of second sketch element in target_read_id_ + position_in_read_t target_position_in_read; +}; - struct ScoredSegment { - /// Seed for the segment - Seed seed; - /// length of the segment - int32_t len; - /// score of the segment - int32_t score; - }; +struct ScoredSegment +{ + /// Seed for the segment + Seed seed; + /// length of the segment + int32_t len; + /// score of the segment + int32_t score; +}; /// CUDA Ungapped Extension object - class UngappedExtender { - public: - /// \brief Constructor Prototype - UngappedExtender(int *h_sub_mat, int input_xdrop, bool input_noentropy, - int gpu_id = 0, cudaStream_t stream = 0); +class UngappedExtender +{ +public: + /// \brief Constructor Prototype + UngappedExtender(int* h_sub_mat, int input_xdrop, bool input_noentropy, + int gpu_id = 0, cudaStream_t stream = 0); - /// \brief Host pointer prototype for ungapped extension - /// - /// Takes values from host data structures, - /// copies them over to device, - /// launches async extension kernels on specified stream. Filters - /// segments on device based on input_hspthresh. - StatusType extend_async(const char *h_query, int32_t query_length, - const char *h_target, int32_t target_length, - int32_t input_hspthresh, - std::vector &h_seedHits); + /// \brief Host pointer prototype for ungapped extension + /// + /// Takes values from host data structures, + /// copies them over to device, + /// launches async extension kernels on specified stream. Filters + /// segments on device based on input_hspthresh. + StatusType extend_async(const char* h_query, int32_t query_length, + const char* h_target, int32_t target_length, + int32_t input_hspthresh, + std::vector& h_seedHits); - /// \brief Device pointer prototype for ungapped extension - /// - /// Memcopies to device memory are assumed to be done before this - /// function. Output array d_hsp_out must be pre-allocated on device. - /// Launches async extension kernel. Filters segments on device - /// based on input_hspthresh. - StatusType extend_async(const char *d_query, int32_t query_length, - const char *d_target, int32_t target_length, - int32_t input_hspthresh, Anchor *d_seed_hits, - int32_t num_hits, ScoredSegment *d_hsp_out, - int32_t *d_num_hsps); + /// \brief Device pointer prototype for ungapped extension + /// + /// Memcopies to device memory are assumed to be done before this + /// function. Output array d_hsp_out must be pre-allocated on device. + /// Launches async extension kernel. Filters segments on device + /// based on input_hspthresh. + StatusType extend_async(const char* d_query, int32_t query_length, + const char* d_target, int32_t target_length, + int32_t input_hspthresh, Seed* d_seed_hits, + int32_t num_hits, ScoredSegment* d_hsp_out, + int32_t* d_num_hsps); - /// \brief Waits for CUDA accelerated extension to finish - /// - /// Blocking call that waits for all the extensions scheduled - /// on the GPU to come to completion. - StatusType sync(); + /// \brief Waits for CUDA accelerated extension to finish + /// + /// Blocking call that waits for all the extensions scheduled + /// on the GPU to come to completion. + StatusType sync(); - /// \brief Return the computed segments - /// - /// \return Vector of Scored Segments - const std::vector &get_scored_segments(); + /// \brief Return the computed segments + /// + /// \return Vector of Scored Segments + const std::vector& get_scored_segments(); - /// \brief Reset UngappedExtender object and free device/host memory - void reset(); - }; - } // namespace cudaungappedextender - } // namespace genomeworks + /// \brief Reset UngappedExtender object and free device/host memory + void reset(); +}; +} // namespace cudaungappedextender +} // namespace genomeworks } // namespace claraparabricks diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index ab24f3dcf..335fc47e2 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -25,66 +25,67 @@ using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::cudaungappedextender; -int main(int argc, char *argv[]) { - const int32_t input_xdrop = 10; +int main(int argc, char* argv[]) +{ + const int32_t input_xdrop = 10; const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; + const int32_t hsp_threshold = 20000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_target = - io::create_kseq_fasta_parser(target_file_path, 0, false); + io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); std::string query_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_query = - io::create_kseq_fasta_parser(query_file_path, 0, false); + io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - // CSV Anchors file - Each row -> query_position_in_read_, + // CSV Seeds file - Each row -> query_position_in_read_, // target_position_in_read_ - std::string anchors_file_path = "../data/example_hits.csv"; + std::string seeds_file_path = "../data/example_seeds.csv"; - std::vector h_hits; - // Following function loops through all hits in the anchors csv and returns + std::vector h_seeds; + // Following function loops through all seeds in the example_seeds.csv and returns // results in // the passed vector - parse_anchors(anchors_file_path, h_hits); + parse_seeds(seeds_file_path, h_seeds); // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; + ScoreMatrix = magic_number_matrix; std::string h_encoded_target = magic_encode(magic_base, target_sequence); - std::string h_encoded_query = magic_encode(magic_base, query_sequence); + std::string h_encoded_query = magic_encode(magic_base, query_sequence); // Create a stream for async use CudaStream stream0 = make_cuda_stream(); - // Allocate space on device for target and query sequences, hits, + // Allocate space on device for target and query sequences, seeds, // high scoring segment pairs (hsps) and num_hsps. - char *d_query; + char* d_query; GW_CU_CHECK_ERROR( - cudaMalloc((void **) &d_query, sizeof(char) * h_encoded_query.size())); - char *d_target; + cudaMalloc((void**)&d_query, sizeof(char) * h_encoded_query.size())); + char* d_target; GW_CU_CHECK_ERROR( - cudaMalloc((void **) &d_target, sizeof(char) * h_target_query.size())); - Anchor *d_hits; + cudaMalloc((void**)&d_target, sizeof(char) * h_target_query.size())); + Seed* d_seeds; GW_CU_CHECK_ERROR( - cudaMalloc((void **) &d_hits, sizeof(Anchor) * h_hits.size())); - // Allocate a minimum of num_hits as all hits could be hsps in the worst case + cudaMalloc((void**)&d_seeds, sizeof(Seed) * h_seeds.size())); + // Allocate a minimum of num_seeds as all seeds could be hsps in the worst case int32_t h_num_hsps = 0; - ScoredSegment *d_hsps; + ScoredSegment* d_hsps; GW_CU_CHECK_ERROR( - cudaMalloc((void **) &d_hsps, sizeof(ScoredSegment) * h_hits.size())); - int32_t *d_num_hsps; - GW_CU_CHECK_ERROR(cudaMalloc((void **) &d_num_hsps, sizeof(int32_t)); - - // Async Memcopy all the input values to device - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query.c_str(), sizeof(char) * h_encoded_query.size(), - cudaMemcpyHostToDevice, stream0.get())); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target.c_str(), sizeof(char) * h_encoded_target.size(), - cudaMemcpyHostToDevice, stream0.get())); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_hits, &h_hits[0], sizeof(Anchor) * h_hits.size(), cudaMemcpyHostToDevice, - stream0.get()))); + cudaMalloc((void**)&d_hsps, sizeof(ScoredSegment) * h_seeds.size())); + int32_t* d_num_hsps; + GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_num_hsps, sizeof(int32_t)); + + // Async Memcopy all the input values to device + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query.c_str(), sizeof(char) * h_encoded_query.size(), + cudaMemcpyHostToDevice, stream0.get())); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target.c_str(), sizeof(char) * h_encoded_target.size(), + cudaMemcpyHostToDevice, stream0.get())); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_seeds, &h_seeds[0], sizeof(Seed) * h_seeds.size(), cudaMemcpyHostToDevice, + stream0.get()))); // Create an ungapped extender object std::unique_ptr ungapped_extender = std::make_unique(0, @@ -93,13 +94,13 @@ int main(int argc, char *argv[]) { input_no_entropy, stream0.get()); // Launch the ungapped extender device function - ungapped_extender->extend_async(d_query, // Type TBD based on encoding + ungapped_extender->extend_async(d_query, // Type TBD based on encoding encoded_query.size(), d_target.c_str(), encoded_target.size(), hsp_threshold, - d_hits, - h_hits.size(), + d_seeds, + h_seeds.size(), d_hsps, d_num_hsps); // Copy back the number of hsps to host @@ -109,7 +110,8 @@ int main(int argc, char *argv[]) { GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0.get())); //Get results - if (h_num_hsps > 0) { + if (h_num_hsps > 0) + { std::vector h_hsps(h_num_hsps); // Don't care about asynchronous copies here GW_CU_CHECK_ERR(cudaMemcpy(&h_hsps[0], d_hsps, @@ -117,23 +119,23 @@ int main(int argc, char *argv[]) { cudaMemcpyDeviceToHost)); int32_t i = 0; - for (const auto &segment : h_hsps) { + for (const auto& segment : h_hsps) + { std::cout << "Segment: " << i << "Length: " << segment.len << "Score: " << segment.score << std::endl; std::cout << "Position in query: " - << segment.anchor.query_position_in_read_ << std::endl; + << segment.seed.query_position_in_read_ << std::endl; std::cout << "Position in target: " - << segment.anchor.target_position_in_read_ << std::endl; + << segment.seed.target_position_in_read_ << std::endl; i++; } - } // Free all allocated memory on the GPU GW_CU_CHECK_ERROR(cudaFree(d_query); GW_CU_CHECK_ERROR(cudaFree(d_target); GW_CU_CHECK_ERROR(cudaFree(d_hsps); - GW_CU_CHECK_ERROR(cudaFree(d_hits); + GW_CU_CHECK_ERROR(cudaFree(d_seeds); GW_CU_CHECK_ERROR(cudaFree(d_num_hsps); return 0; diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index ac70e0389..9251fe35e 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -23,64 +23,66 @@ using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::cudaungappedextender; -int main(int argc, char *argv[]) { - const int32_t input_xdrop = 10; +int main(int argc, char* argv[]) +{ + const int32_t input_xdrop = 10; const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; + const int32_t hsp_threshold = 20000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_target = - io::create_kseq_fasta_parser(target_file_path, 0, false); + io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); std::string query_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_query = - io::create_kseq_fasta_parser(query_file_path, 0, false); + io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - // CSV Anchors file - Each row -> query_position_in_read_, + // CSV Seeds file - Each row -> query_position_in_read_, // target_position_in_read_ - std::string anchors_file_path = "../data/example_hits.csv"; + std::string seeds_file_path = "../data/example_seeds.csv"; - std::vector h_hits; - // Following function loops through all hits in the anchors csv and returns + std::vector h_seeds; + // Following function loops through all seeds in the Seeds csv and returns // results in // the passed vector - parse_anchors(anchors_file_path, h_hits); + parse_Seeds(seeds_file_path, h_seeds); // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; + ScoreMatrix = magic_number_matrix; std::string encoded_target = magic_encode(magic_base, target_sequence); - std::string encoded_query = magic_encode(magic_base, query_sequence); + std::string encoded_query = magic_encode(magic_base, query_sequence); // Create a stream for async use CudaStream stream0 = make_cuda_stream(); // Create an ungapped extender object std::unique_ptr ungapped_extender = - std::make_unique(0, magic_number_matrix, input_xdrop, - input_no_entropy, stream0.get()); + std::make_unique(0, magic_number_matrix, input_xdrop, + input_no_entropy, stream0.get()); // Launch the ungapped extender host function ungapped_extender->extend_async( - encoded_query.c_str(), // Type TBD based on encoding - encoded_query.size(), encoded_target.c_str(), encoded_target.size(), - hsp_threshold, h_hits); + encoded_query.c_str(), // Type TBD based on encoding + encoded_query.size(), encoded_target.c_str(), encoded_target.size(), + hsp_threshold, h_seeds); // Wait for ungapped extender to finish ungapped_extender->sync(); // Get results - const std::vector &segments = - ungapped_extender->get_scored_segments(); + const std::vector& segments = + ungapped_extender->get_scored_segments(); int32_t i = 0; - for (const auto &segment : segments) { + for (const auto& segment : segments) + { std::cout << "Segment: " << i << "Length: " << segment.len << "Score: " << segment.score << std::endl; std::cout << "Position in query: " - << segment.anchor.query_position_in_read_ << std::endl; + << segment.seed.query_position_in_read_ << std::endl; std::cout << "Position in target: " - << segment.anchor.target_position_in_read_ << std::endl; + << segment.seed.target_position_in_read_ << std::endl; i++; } diff --git a/cudaungappedextender/samples/segalign_seedfilter.cu b/cudaungappedextender/samples/segalign_seedfilter.cu index 138fb4bf8..152719373 100644 --- a/cudaungappedextender/samples/segalign_seedfilter.cu +++ b/cudaungappedextender/samples/segalign_seedfilter.cu @@ -27,85 +27,91 @@ int NUM_DEVICES; uint32_t MAX_SEEDS; uint32_t MAX_SEED_HITS; -char **d_ref_seq; +char** d_ref_seq; uint32_t ref_len; -char **d_query_seq; -char **d_query_rc_seq; +char** d_query_seq; +char** d_query_rc_seq; uint32_t query_length[BUFFER_DEPTH]; uint32_t seed_size; -uint32_t **d_index_table; -uint32_t **d_pos_table; +uint32_t** d_index_table; +uint32_t** d_pos_table; -uint64_t **d_seed_offsets; +uint64_t** d_seed_offsets; -uint32_t **d_hit_num_array; +uint32_t** d_hit_num_array; std::vector> d_hit_num_vec; -Anchor **d_hit; +Anchor** d_hit; std::vector> d_hit_vec; -ScoredSegment **d_hsp; +ScoredSegment** d_hsp; std::vector> d_hsp_vec; // UngappedExtend Variables (ideally not visible to the user in the API) uint32_t MAX_UNGAPPED; // maximum extensions per iteration in the // UngappedExtension function -int **d_sub_mat; // substitution score matrix +int** d_sub_mat; // substitution score matrix int xdrop; // xdrop parameter for the UngappedExtension function int hspthresh; // score threshold for qualifying as an HSP bool noentropy; // whether or not to adjust scores of segments as a factor of // the Shannon entropy // convert input sequence from alphabet to integers -__global__ void compress_string(char *dst_seq, char *src_seq, uint32_t len) { - ... -} +__global__ void compress_string(char* dst_seq, char* src_seq, uint32_t len){ + ...} // convert input sequence to its reverse complement and convert from alphabet to // integers -__global__ void compress_rev_comp_string(char *dst_seq, char *src_seq, - uint32_t len) { - ... -} +__global__ void compress_rev_comp_string(char* dst_seq, char* src_seq, + uint32_t len){ + ...} ///////////////////// End Ungapped Extension related functions executed on the /// GPU /////////////// __global__ void find_num_hits(int num_seeds, - const uint32_t *__restrict__ d_index_table, - uint64_t *seed_offsets, uint32_t *seed_hit_num) { - ... -} - -__global__ void find_hits(const uint32_t *__restrict__ d_index_table, - const uint32_t *__restrict__ d_pos_table, - uint64_t *d_seed_offsets, uint32_t seed_size, - uint32_t *seed_hit_num, int num_hits, Anchor *d_hit, - uint32_t start_seed_index, uint32_t start_hit_index) { + const uint32_t* __restrict__ d_index_table, + uint64_t* seed_offsets, uint32_t* seed_hit_num){ + ...} + +__global__ void find_hits(const uint32_t* __restrict__ d_index_table, + const uint32_t* __restrict__ d_pos_table, + uint64_t* d_seed_offsets, uint32_t seed_size, + uint32_t* seed_hit_num, int num_hits, Anchor* d_hit, + uint32_t start_seed_index, uint32_t start_hit_index) +{ ... } int InitializeProcessor(int num_gpu, bool transition, uint32_t WGA_CHUNK, - uint32_t input_seed_size, int *sub_mat, int input_xdrop, - int input_hspthresh, bool input_noentropy) { + uint32_t input_seed_size, int* sub_mat, int input_xdrop, + int input_hspthresh, bool input_noentropy) +{ int nDevices; cudaError_t err = cudaGetDeviceCount(&nDevices); - if (err != cudaSuccess) { + if (err != cudaSuccess) + { fprintf(stderr, "Error: No GPU device found!\n"); exit(1); } - if (num_gpu == -1) { + if (num_gpu == -1) + { NUM_DEVICES = nDevices; - } else { - if (num_gpu <= nDevices) { + } + else + { + if (num_gpu <= nDevices) + { NUM_DEVICES = num_gpu; - } else { + } + else + { fprintf(stderr, "Requested GPUs greater than available GPUs\n"); exit(10); } @@ -123,42 +129,43 @@ int InitializeProcessor(int num_gpu, bool transition, uint32_t WGA_CHUNK, cudaDeviceProp deviceProp; cudaGetDeviceProperties(&deviceProp, 0); float global_mem_gb = - static_cast(deviceProp.totalGlobalMem / 1073741824.0f); + static_cast(deviceProp.totalGlobalMem / 1073741824.0f); MAX_SEED_HITS = global_mem_gb * MAX_SEED_HITS_PER_GB; Anchor zeroHit; - zeroHit.query_position_in_read_ = 0; + zeroHit.query_position_in_read_ = 0; zeroHit.target_position_in_read_ = 0; ScoredSegment zeroHsp; - zeroHsp.anchor.query_position_in_read_ = 0; + zeroHsp.anchor.query_position_in_read_ = 0; zeroHsp.anchor.target_position_in_read_ = 0; - zeroHsp.len = 0; - zeroHsp.score = 0; + zeroHsp.len = 0; + zeroHsp.score = 0; - d_ref_seq = (char **) malloc(NUM_DEVICES * sizeof(char *)); - d_query_seq = (char **) malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char *)); - d_query_rc_seq = (char **) malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char *)); + d_ref_seq = (char**)malloc(NUM_DEVICES * sizeof(char*)); + d_query_seq = (char**)malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char*)); + d_query_rc_seq = (char**)malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char*)); - d_index_table = (uint32_t **) malloc(NUM_DEVICES * sizeof(uint32_t *)); - d_pos_table = (uint32_t **) malloc(NUM_DEVICES * sizeof(uint32_t *)); + d_index_table = (uint32_t**)malloc(NUM_DEVICES * sizeof(uint32_t*)); + d_pos_table = (uint32_t**)malloc(NUM_DEVICES * sizeof(uint32_t*)); - d_seed_offsets = (uint64_t **) malloc(NUM_DEVICES * sizeof(uint64_t *)); + d_seed_offsets = (uint64_t**)malloc(NUM_DEVICES * sizeof(uint64_t*)); - d_hit_num_array = (uint32_t **) malloc(NUM_DEVICES * sizeof(int32_t *)); + d_hit_num_array = (uint32_t**)malloc(NUM_DEVICES * sizeof(int32_t*)); d_hit_num_vec.reserve(NUM_DEVICES); - d_hit = (seedHit **) malloc(NUM_DEVICES * sizeof(Anchor * )); + d_hit = (seedHit**)malloc(NUM_DEVICES * sizeof(Anchor*)); d_hit_vec.reserve(NUM_DEVICES); - d_hsp = (segment **) malloc(NUM_DEVICES * sizeof(ScoredSegment * )); + d_hsp = (segment**)malloc(NUM_DEVICES * sizeof(ScoredSegment*)); d_hsp_vec.reserve(NUM_DEVICES); - for (int g = 0; g < NUM_DEVICES; g++) { + for (int g = 0; g < NUM_DEVICES; g++) + { check_cuda_setDevice(g, "InitializeProcessor"); - check_cuda_malloc((void **) &d_seed_offsets[g], MAX_SEEDS * sizeof(uint64_t), + check_cuda_malloc((void**)&d_seed_offsets[g], MAX_SEEDS * sizeof(uint64_t), "seed_offsets"); d_hit_num_vec.emplace_back(MAX_SEEDS, 0); @@ -179,41 +186,46 @@ int InitializeProcessor(int num_gpu, bool transition, uint32_t WGA_CHUNK, return NUM_DEVICES; } -void InclusivePrefixScan(uint32_t *data, uint32_t len) { +void InclusivePrefixScan(uint32_t* data, uint32_t len) +{ ... } -void SendSeedPosTable(uint32_t *index_table, uint32_t index_table_size, - uint32_t *pos_table, uint32_t num_index, - uint32_t max_pos_index) { +void SendSeedPosTable(uint32_t* index_table, uint32_t index_table_size, + uint32_t* pos_table, uint32_t num_index, + uint32_t max_pos_index) +{ ... } -void SendRefWriteRequest(size_t start_addr, uint32_t len) { +void SendRefWriteRequest(size_t start_addr, uint32_t len) +{ ... } -void SendQueryWriteRequest(size_t start_addr, uint32_t len, uint32_t buffer) { - ... -} +void SendQueryWriteRequest(size_t start_addr, uint32_t len, uint32_t buffer){ + ...} std::vector SeedAndFilter(std::vector seed_offset_vector, bool rev, uint32_t buffer, - int input_hspthresh) { + int input_hspthresh) +{ - uint32_t num_hits = 0; + uint32_t num_hits = 0; uint32_t total_anchors = 0; uint32_t num_seeds = seed_offset_vector.size(); - uint64_t *tmp_offset = (uint64_t *) malloc(num_seeds * sizeof(uint64_t)); - for (uint32_t i = 0; i < num_seeds; i++) { + uint64_t* tmp_offset = (uint64_t*)malloc(num_seeds * sizeof(uint64_t)); + for (uint32_t i = 0; i < num_seeds; i++) + { tmp_offset[i] = seed_offset_vector[i]; } int g; std::unique_lock locker(mu); - if (available_gpus.empty()) { + if (available_gpus.empty()) + { cv.wait(locker, []() { return !available_gpus.empty(); }); } g = available_gpus.back(); @@ -222,69 +234,77 @@ std::vector SeedAndFilter(std::vector seed_offset_vector, check_cuda_setDevice(g, "SeedAndFilter"); - check_cuda_memcpy((void *) d_seed_offsets[g], (void *) tmp_offset, + check_cuda_memcpy((void*)d_seed_offsets[g], (void*)tmp_offset, num_seeds * sizeof(uint64_t), cudaMemcpyHostToDevice, "seed_offsets"); - find_num_hits << < MAX_BLOCKS, MAX_THREADS >> > ( - num_seeds, d_index_table[g], d_seed_offsets[g], d_hit_num_array[g]); + find_num_hits<<>>( + num_seeds, d_index_table[g], d_seed_offsets[g], d_hit_num_array[g]); thrust::inclusive_scan(d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, d_hit_num_vec[g].begin()); - check_cuda_memcpy((void *) &num_hits, - (void *) (d_hit_num_array[g] + num_seeds - 1), + check_cuda_memcpy((void*)&num_hits, + (void*)(d_hit_num_array[g] + num_seeds - 1), sizeof(uint32_t), cudaMemcpyDeviceToHost, "num_hits"); - int num_iter = num_hits / MAX_UNGAPPED + 1; + int num_iter = num_hits / MAX_UNGAPPED + 1; uint32_t iter_hit_limit = MAX_UNGAPPED; thrust::device_vector limit_pos(num_iter); - for (int i = 0; i < num_iter - 1; i++) { + for (int i = 0; i < num_iter - 1; i++) + { thrust::device_vector::iterator result_end = thrust::lower_bound( - d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, - iter_hit_limit); - uint32_t pos = thrust::distance(d_hit_num_vec[g].begin(), result_end) - 1; + d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, + iter_hit_limit); + uint32_t pos = thrust::distance(d_hit_num_vec[g].begin(), result_end) - 1; iter_hit_limit = d_hit_num_vec[g][pos] + MAX_UNGAPPED; - limit_pos[i] = pos; + limit_pos[i] = pos; } limit_pos[num_iter - 1] = num_seeds - 1; - segment **h_hsp = (segment **) malloc(num_iter * sizeof(segment * )); - uint32_t *num_anchors = (uint32_t *) calloc(num_iter, sizeof(uint32_t)); + segment** h_hsp = (segment**)malloc(num_iter * sizeof(segment*)); + uint32_t* num_anchors = (uint32_t*)calloc(num_iter, sizeof(uint32_t)); uint32_t start_seed_index = 0; - uint32_t start_hit_val = 0; + uint32_t start_hit_val = 0; uint32_t iter_num_seeds, iter_num_hits; - int32_t *d_num_anchors; - GW_CU_CHECK_ERR(cudaMalloc((void **) &d_num_anchors, sizeof(int32_t))); + int32_t* d_num_anchors; + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_anchors, sizeof(int32_t))); - if (num_hits > 0) { + if (num_hits > 0) + { - for (int i = 0; i < num_iter; i++) { + for (int i = 0; i < num_iter; i++) + { iter_num_seeds = limit_pos[i] + 1 - start_seed_index; - iter_num_hits = d_hit_num_vec[g][limit_pos[i]] - start_hit_val; + iter_num_hits = d_hit_num_vec[g][limit_pos[i]] - start_hit_val; - find_hits << < iter_num_seeds, BLOCK_SIZE >> > ( - d_index_table[g], d_pos_table[g], d_seed_offsets[g], seed_size, - d_hit_num_array[g], iter_num_hits, d_hit[g], start_seed_index, - start_hit_val); + find_hits<<>>( + d_index_table[g], d_pos_table[g], d_seed_offsets[g], seed_size, + d_hit_num_array[g], iter_num_hits, d_hit[g], start_seed_index, + start_hit_val); - if (rev) { + if (rev) + { if (!g_cuda_ungapped_extenders[g].ungapped_extend( d_query_rc_seq[buffer * NUM_DEVICES + g], query_length[buffer], d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, - d_hsp[g], d_num_anchors)); + d_hsp[g], d_num_anchors)) + ; { // err... } - } else { + } + else + { if (!g_cuda_ungapped_extenders[g].ungapped_extend( d_query_seq[buffer * NUM_DEVICES + g], query_length[buffer], d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, - d_hsp[g], d_num_anchors)); + d_hsp[g], d_num_anchors)) + ; { // err... } @@ -296,16 +316,17 @@ std::vector SeedAndFilter(std::vector seed_offset_vector, total_anchors += num_anchors[i]; - if (num_anchors[i] > 0) { - h_hsp[i] = (segment *) calloc(num_anchors[i], sizeof(segment)); + if (num_anchors[i] > 0) + { + h_hsp[i] = (segment*)calloc(num_anchors[i], sizeof(segment)); - check_cuda_memcpy((void *) h_hsp[i], (void *) d_hsp[g], + check_cuda_memcpy((void*)h_hsp[i], (void*)d_hsp[g], num_anchors[i] * sizeof(segment), cudaMemcpyDeviceToHost, "hsp_output"); } start_seed_index = limit_pos[i] + 1; - start_hit_val = d_hit_num_vec[g][limit_pos[i]]; + start_hit_val = d_hit_num_vec[g][limit_pos[i]]; } } @@ -321,14 +342,17 @@ std::vector SeedAndFilter(std::vector seed_offset_vector, std::vector gpu_filter_output; segment first_el; - first_el.len = total_anchors; + first_el.len = total_anchors; first_el.score = num_hits; gpu_filter_output.push_back(first_el); - if (total_anchors > 0) { - for (int it = 0; it < num_iter; it++) { + if (total_anchors > 0) + { + for (int it = 0; it < num_iter; it++) + { - for (int i = 0; i < num_anchors[it]; i++) { + for (int i = 0; i < num_anchors[it]; i++) + { gpu_filter_output.push_back(h_hsp[it][i]); } } @@ -339,22 +363,26 @@ std::vector SeedAndFilter(std::vector seed_offset_vector, return gpu_filter_output; } -void clearRef() { +void clearRef() +{ ... } -void clearQuery(uint32_t buffer) { +void clearQuery(uint32_t buffer) +{ ... } -void ShutdownProcessor() { +void ShutdownProcessor() +{ d_hit_num_vec.clear(); d_hit_vec.clear(); d_hsp_vec.clear(); g_cuda_ungapped_extenders.clear(); - for (auto &cudaStream : g_cuda_streams) { + for (auto& cudaStream : g_cuda_streams) + { cudaStreamDestroy(cudaStream); } g_cuda_streams.clear(); @@ -363,23 +391,24 @@ void ShutdownProcessor() { ///// Start Ungapped Extension related functions ///// -void CompressSeq(char *input_seq, char *output_seq, uint32_t len) { +void CompressSeq(char* input_seq, char* output_seq, uint32_t len) +{ ... } -void CompressRevCompSeq(char *input_seq, char *output_seq, uint32_t len) { ... } +void CompressRevCompSeq(char* input_seq, char* output_seq, uint32_t len){...} -CompressSeq_ptr g_CompressSeq = CompressSeq; +CompressSeq_ptr g_CompressSeq = CompressSeq; CompressRevCompSeq_ptr g_CompressRevCompSeq = CompressRevCompSeq; ///// End Ungapped Extension related functions ///// -InitializeProcessor_ptr g_InitializeProcessor = InitializeProcessor; -InclusivePrefixScan_ptr g_InclusivePrefixScan = InclusivePrefixScan; -SendSeedPosTable_ptr g_SendSeedPosTable = SendSeedPosTable; -SendRefWriteRequest_ptr g_SendRefWriteRequest = SendRefWriteRequest; +InitializeProcessor_ptr g_InitializeProcessor = InitializeProcessor; +InclusivePrefixScan_ptr g_InclusivePrefixScan = InclusivePrefixScan; +SendSeedPosTable_ptr g_SendSeedPosTable = SendSeedPosTable; +SendRefWriteRequest_ptr g_SendRefWriteRequest = SendRefWriteRequest; SendQueryWriteRequest_ptr g_SendQueryWriteRequest = SendQueryWriteRequest; -SeedAndFilter_ptr g_SeedAndFilter = SeedAndFilter; -clearRef_ptr g_clearRef = clearRef; -clearQuery_ptr g_clearQuery = clearQuery; -ShutdownProcessor_ptr g_ShutdownProcessor = ShutdownProcessor; +SeedAndFilter_ptr g_SeedAndFilter = SeedAndFilter; +clearRef_ptr g_clearRef = clearRef; +clearQuery_ptr g_clearQuery = clearQuery; +ShutdownProcessor_ptr g_ShutdownProcessor = ShutdownProcessor; From 550d0a9e49785c3798b0a8ad5f7de74d6bbacd67 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 27 Aug 2020 10:36:09 -0700 Subject: [PATCH 035/281] [cudaungappedextender] update length --- .../genomeworks/cudaungappedextender/cudaungappedextender.hpp | 2 +- cudaungappedextender/samples/device_sample.cu | 2 +- cudaungappedextender/samples/host_sample.cu | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index 5e676bcf6..6d5795728 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -35,7 +35,7 @@ struct ScoredSegment /// Seed for the segment Seed seed; /// length of the segment - int32_t len; + int32_t length; /// score of the segment int32_t score; }; diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index 335fc47e2..1def3befd 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -121,7 +121,7 @@ int main(int argc, char* argv[]) int32_t i = 0; for (const auto& segment : h_hsps) { - std::cout << "Segment: " << i << "Length: " << segment.len + std::cout << "Segment: " << i << "Length: " << segment.length << "Score: " << segment.score << std::endl; std::cout << "Position in query: " << segment.seed.query_position_in_read_ << std::endl; diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index 9251fe35e..b4bb74ca7 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -77,7 +77,7 @@ int main(int argc, char* argv[]) int32_t i = 0; for (const auto& segment : segments) { - std::cout << "Segment: " << i << "Length: " << segment.len + std::cout << "Segment: " << i << "Length: " << segment.length << "Score: " << segment.score << std::endl; std::cout << "Position in query: " << segment.seed.query_position_in_read_ << std::endl; From d863ca1adb632ef71b363c3882264cb998a8a83b Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 27 Aug 2020 10:39:42 -0700 Subject: [PATCH 036/281] [cudaungappedextender] Change seedHits to seeds --- .../cudaungappedextender/cudaungappedextender.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index 6d5795728..3d148a942 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -57,7 +57,7 @@ class UngappedExtender StatusType extend_async(const char* h_query, int32_t query_length, const char* h_target, int32_t target_length, int32_t input_hspthresh, - std::vector& h_seedHits); + std::vector& h_seeds); /// \brief Device pointer prototype for ungapped extension /// @@ -67,8 +67,8 @@ class UngappedExtender /// based on input_hspthresh. StatusType extend_async(const char* d_query, int32_t query_length, const char* d_target, int32_t target_length, - int32_t input_hspthresh, Seed* d_seed_hits, - int32_t num_hits, ScoredSegment* d_hsp_out, + int32_t input_hspthresh, Seed* d_seeds, + int32_t num_seeds, ScoredSegment* d_hsp_out, int32_t* d_num_hsps); /// \brief Waits for CUDA accelerated extension to finish From 93c8e39665fa15557afbea5976ff2ef86f9ad85a Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 27 Aug 2020 10:43:41 -0700 Subject: [PATCH 037/281] [cudaungappedextender] const method --- .../genomeworks/cudaungappedextender/cudaungappedextender.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index 3d148a942..ff2d9f40f 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -80,7 +80,7 @@ class UngappedExtender /// \brief Return the computed segments /// /// \return Vector of Scored Segments - const std::vector& get_scored_segments(); + const std::vector& get_scored_segments() const; /// \brief Reset UngappedExtender object and free device/host memory void reset(); From c2e243f5dc0d07be00fe4b8c031fc77619ba90ce Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Mon, 31 Aug 2020 10:27:47 -0400 Subject: [PATCH 038/281] [pygw] fix copyright header binding file --- pygenomeworks/genomeworks/cuda/cuda.pxd | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pygenomeworks/genomeworks/cuda/cuda.pxd b/pygenomeworks/genomeworks/cuda/cuda.pxd index d1dd2f48e..f8bf69850 100644 --- a/pygenomeworks/genomeworks/cuda/cuda.pxd +++ b/pygenomeworks/genomeworks/cuda/cuda.pxd @@ -1,11 +1,17 @@ # -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019-2020 NVIDIA CORPORATION. # -# NVIDIA CORPORATION and its licensors retain all intellectual property -# and proprietary rights in and to this software, related documentation -# and any modifications thereto. Any use, reproduction, disclosure or -# distribution of this software and related documentation without an express -# license agreement from NVIDIA CORPORATION is strictly prohibited. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # cython: profile=False From 15a672840af65aaf7a04b044fe770f03f477b62b Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 31 Aug 2020 10:44:03 -0700 Subject: [PATCH 039/281] [cudaungappedextender] Address review comments - default values; rename hsp_threshold to score_threshold --- .../cudaungappedextender/cudaungappedextender.hpp | 4 ++-- cudaungappedextender/samples/device_sample.cu | 8 ++++---- cudaungappedextender/samples/host_sample.cu | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index ff2d9f40f..ad8560a05 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -56,7 +56,7 @@ class UngappedExtender /// segments on device based on input_hspthresh. StatusType extend_async(const char* h_query, int32_t query_length, const char* h_target, int32_t target_length, - int32_t input_hspthresh, + int32_t score_threshold, std::vector& h_seeds); /// \brief Device pointer prototype for ungapped extension @@ -67,7 +67,7 @@ class UngappedExtender /// based on input_hspthresh. StatusType extend_async(const char* d_query, int32_t query_length, const char* d_target, int32_t target_length, - int32_t input_hspthresh, Seed* d_seeds, + int32_t score_threshold, Seed* d_seeds, int32_t num_seeds, ScoredSegment* d_hsp_out, int32_t* d_num_hsps); diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index 1def3befd..cdf6230d4 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -27,9 +27,9 @@ using namespace claraparabricks::genomeworks::cudaungappedextender; int main(int argc, char* argv[]) { - const int32_t input_xdrop = 10; - const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; + const int32_t input_xdrop = 910; + const bool input_no_entropy = false; + const int32_t score_threshold = 3000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_target = @@ -98,7 +98,7 @@ int main(int argc, char* argv[]) encoded_query.size(), d_target.c_str(), encoded_target.size(), - hsp_threshold, + score_threshold, d_seeds, h_seeds.size(), d_hsps, diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index b4bb74ca7..0dbd86aa5 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -25,9 +25,9 @@ using namespace claraparabricks::genomeworks::cudaungappedextender; int main(int argc, char* argv[]) { - const int32_t input_xdrop = 10; - const int32_t input_no_entropy = 0; - const int32_t hsp_threshold = 20000; + const int32_t input_xdrop = 910; + const bool input_no_entropy = false; + const int32_t score_threshold = 3000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_target = @@ -66,7 +66,7 @@ int main(int argc, char* argv[]) ungapped_extender->extend_async( encoded_query.c_str(), // Type TBD based on encoding encoded_query.size(), encoded_target.c_str(), encoded_target.size(), - hsp_threshold, h_seeds); + score_threshold, h_seeds); // Wait for ungapped extender to finish ungapped_extender->sync(); From 6a1a185fc4cf05b74a0738a2749ff4fed27e5f46 Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 1 Sep 2020 13:58:19 -0700 Subject: [PATCH 040/281] [cudaungappedextender] Rename seed -> seed_pair; scored_segment -> scored_segment_pair --- .../cudaungappedextender.hpp | 6 +-- cudaungappedextender/samples/device_sample.cu | 38 +++++++++---------- cudaungappedextender/samples/host_sample.cu | 18 ++++----- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index ad8560a05..6e8e29a12 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -22,7 +22,7 @@ namespace genomeworks namespace cudaungappedextender { -struct Seed +struct SeedPair { /// position of first sketch element in query_read_id_ position_in_read_t query_position_in_read; @@ -30,10 +30,10 @@ struct Seed position_in_read_t target_position_in_read; }; -struct ScoredSegment +struct ScoredSegmentPair { /// Seed for the segment - Seed seed; + Seed seed_pair; /// length of the segment int32_t length; /// score of the segment diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaungappedextender/samples/device_sample.cu index cdf6230d4..7dae9085c 100644 --- a/cudaungappedextender/samples/device_sample.cu +++ b/cudaungappedextender/samples/device_sample.cu @@ -43,15 +43,15 @@ int main(int argc, char* argv[]) // Assumes that only one sequence is present per file magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - // CSV Seeds file - Each row -> query_position_in_read_, + // CSV SeedPairs file - Each row -> query_position_in_read_, // target_position_in_read_ - std::string seeds_file_path = "../data/example_seeds.csv"; + std::string seed_pairs_file_path = "../data/example_seed_pairs.csv"; - std::vector h_seeds; - // Following function loops through all seeds in the example_seeds.csv and returns + std::vector h_seed_pairs; + // Following function loops through all seed_pairs in the example_seed_pairs.csv and returns // results in // the passed vector - parse_seeds(seeds_file_path, h_seeds); + parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); // Following sections TBD based on encoding ScoreMatrix = magic_number_matrix; @@ -60,7 +60,7 @@ int main(int argc, char* argv[]) // Create a stream for async use CudaStream stream0 = make_cuda_stream(); - // Allocate space on device for target and query sequences, seeds, + // Allocate space on device for target and query sequences, seed_pairs, // high scoring segment pairs (hsps) and num_hsps. char* d_query; GW_CU_CHECK_ERROR( @@ -68,14 +68,14 @@ int main(int argc, char* argv[]) char* d_target; GW_CU_CHECK_ERROR( cudaMalloc((void**)&d_target, sizeof(char) * h_target_query.size())); - Seed* d_seeds; + SeedPair* d_seed_pairs; GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_seeds, sizeof(Seed) * h_seeds.size())); - // Allocate a minimum of num_seeds as all seeds could be hsps in the worst case + cudaMalloc((void**)&d_seed_pairs, sizeof(SeedPair) * h_seed_pairs.size())); + // Allocate a minimum of num_seed_pairs as all seed_pairs could be hsps in the worst case int32_t h_num_hsps = 0; - ScoredSegment* d_hsps; + ScoredSegmentPair* d_hsps; GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_hsps, sizeof(ScoredSegment) * h_seeds.size())); + cudaMalloc((void**)&d_hsps, sizeof(ScoredSegmentPair) * h_seed_pairs.size())); int32_t* d_num_hsps; GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_num_hsps, sizeof(int32_t)); @@ -84,7 +84,7 @@ int main(int argc, char* argv[]) cudaMemcpyHostToDevice, stream0.get())); GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target.c_str(), sizeof(char) * h_encoded_target.size(), cudaMemcpyHostToDevice, stream0.get())); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_seeds, &h_seeds[0], sizeof(Seed) * h_seeds.size(), cudaMemcpyHostToDevice, + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_seed_pairs, &h_seed_pairs[0], sizeof(SeedPair) * h_seed_pairs.size(), cudaMemcpyHostToDevice, stream0.get()))); // Create an ungapped extender object @@ -99,8 +99,8 @@ int main(int argc, char* argv[]) d_target.c_str(), encoded_target.size(), score_threshold, - d_seeds, - h_seeds.size(), + d_seed_pairs, + h_seed_pairs.size(), d_hsps, d_num_hsps); // Copy back the number of hsps to host @@ -112,10 +112,10 @@ int main(int argc, char* argv[]) //Get results if (h_num_hsps > 0) { - std::vector h_hsps(h_num_hsps); + std::vector h_hsps(h_num_hsps); // Don't care about asynchronous copies here GW_CU_CHECK_ERR(cudaMemcpy(&h_hsps[0], d_hsps, - sizeof(ScoredSegment) * h_num_hsps, + sizeof(ScoredSegmentPair) * h_num_hsps, cudaMemcpyDeviceToHost)); int32_t i = 0; @@ -124,9 +124,9 @@ int main(int argc, char* argv[]) std::cout << "Segment: " << i << "Length: " << segment.length << "Score: " << segment.score << std::endl; std::cout << "Position in query: " - << segment.seed.query_position_in_read_ << std::endl; + << segment.seed_pair.query_position_in_read << std::endl; std::cout << "Position in target: " - << segment.seed.target_position_in_read_ << std::endl; + << segment.seed_pair.target_position_in_read << std::endl; i++; } } @@ -135,7 +135,7 @@ int main(int argc, char* argv[]) GW_CU_CHECK_ERROR(cudaFree(d_query); GW_CU_CHECK_ERROR(cudaFree(d_target); GW_CU_CHECK_ERROR(cudaFree(d_hsps); - GW_CU_CHECK_ERROR(cudaFree(d_seeds); + GW_CU_CHECK_ERROR(cudaFree(d_seed_pairs); GW_CU_CHECK_ERROR(cudaFree(d_num_hsps); return 0; diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index 0dbd86aa5..14f76049f 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -41,15 +41,15 @@ int main(int argc, char* argv[]) // Assumes that only one sequence is present per file magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - // CSV Seeds file - Each row -> query_position_in_read_, + // CSV SeedPairs file - Each row -> query_position_in_read_, // target_position_in_read_ - std::string seeds_file_path = "../data/example_seeds.csv"; + std::string seed_pairs_file_path = "../data/example_seed_pairs.csv"; - std::vector h_seeds; - // Following function loops through all seeds in the Seeds csv and returns + std::vector h_seed_pairs; + // Following function loops through all seed_pairs in the SeedPairs csv and returns // results in // the passed vector - parse_Seeds(seeds_file_path, h_seeds); + parse_SeedPairs(seed_pairs_file_path, h_seed_pairs); // Following sections TBD based on encoding ScoreMatrix = magic_number_matrix; @@ -66,13 +66,13 @@ int main(int argc, char* argv[]) ungapped_extender->extend_async( encoded_query.c_str(), // Type TBD based on encoding encoded_query.size(), encoded_target.c_str(), encoded_target.size(), - score_threshold, h_seeds); + score_threshold, h_seed_pairs); // Wait for ungapped extender to finish ungapped_extender->sync(); // Get results - const std::vector& segments = + const std::vector& segments = ungapped_extender->get_scored_segments(); int32_t i = 0; for (const auto& segment : segments) @@ -80,9 +80,9 @@ int main(int argc, char* argv[]) std::cout << "Segment: " << i << "Length: " << segment.length << "Score: " << segment.score << std::endl; std::cout << "Position in query: " - << segment.seed.query_position_in_read_ << std::endl; + << segment.seed_pair.query_position_in_read << std::endl; std::cout << "Position in target: " - << segment.seed.target_position_in_read_ << std::endl; + << segment.seed_pair.target_position_in_read << std::endl; i++; } From 5a91b4362ee2bc914e8e7a69ced743fb06322c04 Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 1 Sep 2020 14:01:18 -0700 Subject: [PATCH 041/281] [cudaungappedextender] propagate renaming changes --- .../cudaungappedextender/cudaungappedextender.hpp | 8 ++++---- cudaungappedextender/samples/host_sample.cu | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp index 6e8e29a12..fc765cdac 100644 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp @@ -57,7 +57,7 @@ class UngappedExtender StatusType extend_async(const char* h_query, int32_t query_length, const char* h_target, int32_t target_length, int32_t score_threshold, - std::vector& h_seeds); + std::vector& h_seed_pairs); /// \brief Device pointer prototype for ungapped extension /// @@ -67,8 +67,8 @@ class UngappedExtender /// based on input_hspthresh. StatusType extend_async(const char* d_query, int32_t query_length, const char* d_target, int32_t target_length, - int32_t score_threshold, Seed* d_seeds, - int32_t num_seeds, ScoredSegment* d_hsp_out, + int32_t score_threshold, SeedPair* d_seed_pairs, + int32_t num_seed_pairs, ScoredSegmentPair* d_hsp_out, int32_t* d_num_hsps); /// \brief Waits for CUDA accelerated extension to finish @@ -80,7 +80,7 @@ class UngappedExtender /// \brief Return the computed segments /// /// \return Vector of Scored Segments - const std::vector& get_scored_segments() const; + const std::vector& get_scored_segment_pairs() const; /// \brief Reset UngappedExtender object and free device/host memory void reset(); diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaungappedextender/samples/host_sample.cu index 14f76049f..7e5c26682 100644 --- a/cudaungappedextender/samples/host_sample.cu +++ b/cudaungappedextender/samples/host_sample.cu @@ -73,7 +73,7 @@ int main(int argc, char* argv[]) // Get results const std::vector& segments = - ungapped_extender->get_scored_segments(); + ungapped_extender->get_scored_segment_pairs(); int32_t i = 0; for (const auto& segment : segments) { From 0e57503d72031ccb2e32875bdeae065719b55706 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 4 Sep 2020 18:00:12 -0700 Subject: [PATCH 042/281] [cudaextender] Rename ungappedextender to extender; Add skeleton for implementation --- CMakeLists.txt | 2 +- cudaextender/CMakeLists.txt | 79 ++++ cudaextender/data/CMakeLists.txt | 6 + .../data/example.fa | 0 .../data/example_hits.csv | 0 cudaextender/data/file_location.hpp.in | 4 + .../genomeworks/cudaextender/cudaextender.hpp | 54 +++ .../genomeworks/cudaextender/extender.hpp | 100 +++++ cudaextender/samples/CMakeLists.txt | 33 ++ .../samples/device_sample.cu | 0 .../samples/host_sample.cu | 0 cudaextender/src/cudaextender.cpp | 39 ++ cudaextender/src/extender.cpp | 46 ++ cudaextender/src/ungapped_xdrop.cpp | 0 cudaextender/src/ungapped_xdrop.hpp | 66 +++ cudaungappedextender/CMakeLists.txt | 3 - .../cudaungappedextender.hpp | 90 ---- .../samples/segalign_seedfilter.cu | 414 ------------------ 18 files changed, 428 insertions(+), 508 deletions(-) create mode 100644 cudaextender/CMakeLists.txt create mode 100644 cudaextender/data/CMakeLists.txt rename {cudaungappedextender => cudaextender}/data/example.fa (100%) rename {cudaungappedextender => cudaextender}/data/example_hits.csv (100%) create mode 100644 cudaextender/data/file_location.hpp.in create mode 100644 cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp create mode 100644 cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp create mode 100644 cudaextender/samples/CMakeLists.txt rename {cudaungappedextender => cudaextender}/samples/device_sample.cu (100%) rename {cudaungappedextender => cudaextender}/samples/host_sample.cu (100%) create mode 100644 cudaextender/src/cudaextender.cpp create mode 100644 cudaextender/src/extender.cpp create mode 100644 cudaextender/src/ungapped_xdrop.cpp create mode 100644 cudaextender/src/ungapped_xdrop.hpp delete mode 100644 cudaungappedextender/CMakeLists.txt delete mode 100644 cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp delete mode 100644 cudaungappedextender/samples/segalign_seedfilter.cu diff --git a/CMakeLists.txt b/CMakeLists.txt index 686129871..262dc10d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,7 +83,7 @@ add_subdirectory(common/io) add_subdirectory(cudapoa) add_subdirectory(cudamapper) add_subdirectory(cudaaligner) -add_subdirectory(cudaungappedextender) +add_subdirectory(cudaextender) # Add documentation generation. validate_boolean(gw_generate_docs) diff --git a/cudaextender/CMakeLists.txt b/cudaextender/CMakeLists.txt new file mode 100644 index 000000000..f7943368f --- /dev/null +++ b/cudaextender/CMakeLists.txt @@ -0,0 +1,79 @@ +# +# Copyright 2019-2020 NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +project(cudaextender) + +# Process data subdirectory first +add_subdirectory(data) + +GitVersion() + +configure_file(${CMAKE_SOURCE_DIR}/common/base/src/version.cpp.in + ${CMAKE_CURRENT_BINARY_DIR}/version.cpp) + +# Optimization flags interfere with nvcc and cause compilation issues with reworked +# file structure. +if(NOT DEFINED gw_cuda_before_10_0) + message(FATAL_ERROR "${PROJECT_NAME} : variable gw_cuda_before_10_0 is not defined yet. Please make sure CUDA.cmake is loaded first.") +elseif(gw_cuda_before_10_0) + message(STATUS "${PROJECT_NAME} : Remove -O optimization when building for CUDA < 10 as it causes compilation issues.") + string(REGEX REPLACE "-O[0-3]" "" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + string(REGEX REPLACE "-O[0-3]" "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) +endif() + +# Project specific NVCC flags, --expt-relaxed-constexpr is being added to allow using numeric_limits inside device kernels. +# Using in CUDA 10.1+ to avoid compilation issues with CUDA <= 10.0. +set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14") +if(gw_cuda_after_10_0) + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} --expt-relaxed-constexpr") +endif() +message(STATUS "nvcc flags for ${PROJECT_NAME}: ${CUDA_NVCC_FLAGS}") + +get_property(gw_library_type GLOBAL PROPERTY gw_library_type) +cuda_add_library(cudaextender ${gw_library_type} + src/cudaextender.cpp + src/extender.cpp + ${CMAKE_CURRENT_BINARY_DIR}/version.cpp + ) + +target_link_libraries(cudaextender gwbase gwio) + +target_compile_options(cudaextender PRIVATE -Werror -Wall -Wextra) +if (gw_optimize_for_native_cpu) + target_compile_options(cudaextender PRIVATE -march=native) +endif() + +target_include_directories(cudaextender + PUBLIC + $ + $ + ) + +add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) + +install(TARGETS cudaextender + COMPONENT gwlogging + EXPORT cudaextender + DESTINATION lib + INCLUDES DESTINATION include + ) +install(DIRECTORY include/ DESTINATION include) +install(EXPORT cudaextender DESTINATION cmake) + +# Add other folders. +add_subdirectory(samples) + +# Add auto formatting. +gw_enable_auto_formatting("${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/cudaextender/data/CMakeLists.txt b/cudaextender/data/CMakeLists.txt new file mode 100644 index 000000000..1df0a07a6 --- /dev/null +++ b/cudaextender/data/CMakeLists.txt @@ -0,0 +1,6 @@ + + +set(CUDAUNGAPPEDEXTENDER_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../data") +configure_file(file_location.hpp.in ${PROJECT_BINARY_DIR}/data/file_location.hpp @ONLY) + +set_property(GLOBAL PROPERTY cudaungappedextender_data_include_dir "${PROJECT_BINARY_DIR}/data") diff --git a/cudaungappedextender/data/example.fa b/cudaextender/data/example.fa similarity index 100% rename from cudaungappedextender/data/example.fa rename to cudaextender/data/example.fa diff --git a/cudaungappedextender/data/example_hits.csv b/cudaextender/data/example_hits.csv similarity index 100% rename from cudaungappedextender/data/example_hits.csv rename to cudaextender/data/example_hits.csv diff --git a/cudaextender/data/file_location.hpp.in b/cudaextender/data/file_location.hpp.in new file mode 100644 index 000000000..3338868b6 --- /dev/null +++ b/cudaextender/data/file_location.hpp.in @@ -0,0 +1,4 @@ + +#pragma once + +#cmakedefine CUDAUNGAPPEDEXTENDER_DATA_DIR "@CUDAUNGAPPEDEXTENDER_DATA_DIR@" diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp new file mode 100644 index 000000000..00dfc446d --- /dev/null +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp @@ -0,0 +1,54 @@ +/* +* Copyright 2019-2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#pragma once + +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ +/// \defgroup cudaextender CUDA Extender package +/// Base docs for the cudaextender package (tbd) +/// \{ + +/// CUDA Extender status type +enum StatusType +{ + success = 0, + generic_error +}; + +/// Extension types +/// ungapped_xdrop performs ungapped extension with an x-drop threshold +/// +enum ExtensionType +{ + ungapped_xdrop = 0, +}; + +/// Initialize CUDA Extender context. +StatusType Init(); + +/// \} +} // namespace cudaextender + +} // namespace genomeworks + +} // namespace claraparabricks diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp new file mode 100644 index 000000000..bed005085 --- /dev/null +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -0,0 +1,100 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#pragma once +#include +#include + +#include +#include +#include + +namespace claraparabricks +{ +namespace genomeworks +{ +namespace cudaextender +{ + +typedef struct SeedPair +{ + /// position of first sketch element in query_read_id_ + position_in_read_t query_position_in_read; + /// position of second sketch element in target_read_id_ + position_in_read_t target_position_in_read; +} SeedPair; + +typedef struct ScoredSegmentPair +{ + /// Seed for the segment + SeedPair seed_pair; + /// length of the segment + int32_t length; + /// score of the segment + int32_t score; +} ScoredSegmentPair; + +/// CUDA Extension object +class Extender +{ +public: + /// \brief Virtual destructor for Extender. + virtual ~Extender() = default; + + /// \brief Host pointer prototype for extension + /// + /// Takes values from host data structures, + /// copies them over to device, + /// launches async extension kernels on specified stream. Filters + /// segments on device based on score_threshold + virtual StatusType extend_async(const char* h_query, int32_t query_length, + const char* h_target, int32_t target_length, + int32_t score_threshold, + std::vector& h_seed_pairs) = 0; + + /// \brief Device pointer prototype for extension + /// + /// Memcopies to device memory are assumed to be done before this + /// function. Output array d_hsp_out must be pre-allocated on device. + /// Launches async extension kernel. Filters segments on device + /// based on input_hspthresh. + virtual StatusType extend_async(const char* d_query, int32_t query_length, + const char* d_target, int32_t target_length, + int32_t score_threshold, SeedPair* d_seed_pairs, + int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, + int32_t* d_num_scored_segment_pairs) = 0; + + /// \brief Waits for CUDA accelerated extension to finish + /// + /// Blocking call that waits for all the extensions scheduled + /// on the GPU to come to completion. + virtual StatusType sync() = 0; + + /// \brief Return the computed segments + /// + /// \return Vector of Scored Segments + virtual const std::vector& get_scored_segment_pairs() const = 0; + + /// \brief Reset Extender object and free device/host memory + virtual void reset() = 0; +}; + +std::unique_ptr create_extender(int32_t* h_sub_mat, bool no_entropy = false, + cudaStream_t stream = 0, int32_t device_id = 0, + ExtensionType type = ExtensionType::ungapped_xdrop); + +} // namespace cudaextender +} // namespace genomeworks +} // namespace claraparabricks diff --git a/cudaextender/samples/CMakeLists.txt b/cudaextender/samples/CMakeLists.txt new file mode 100644 index 000000000..9649e6a4a --- /dev/null +++ b/cudaextender/samples/CMakeLists.txt @@ -0,0 +1,33 @@ +# +# Copyright 2019-2020 NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +project(sample_cudaextender) + +get_property(cudaextender_data_include_dir GLOBAL PROPERTY cudaextender_data_include_dir) +include_directories(${cudaextender_data_include_dir}) + +add_executable(${PROJECT_NAME} + host_sample.cu + ) + +target_link_libraries(${PROJECT_NAME} + cudaextender + ) + +install(TARGETS ${PROJECT_NAME} + DESTINATION samples) diff --git a/cudaungappedextender/samples/device_sample.cu b/cudaextender/samples/device_sample.cu similarity index 100% rename from cudaungappedextender/samples/device_sample.cu rename to cudaextender/samples/device_sample.cu diff --git a/cudaungappedextender/samples/host_sample.cu b/cudaextender/samples/host_sample.cu similarity index 100% rename from cudaungappedextender/samples/host_sample.cu rename to cudaextender/samples/host_sample.cu diff --git a/cudaextender/src/cudaextender.cpp b/cudaextender/src/cudaextender.cpp new file mode 100644 index 000000000..9a8e349b5 --- /dev/null +++ b/cudaextender/src/cudaextender.cpp @@ -0,0 +1,39 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#include +#include + +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ + +StatusType Init() +{ + if (logging::LoggingStatus::success != logging::Init()) + return StatusType::generic_error; + + return StatusType::success; +} +} // namespace cudaextender + +} // namespace genomeworks + +} // namespace claraparabricks \ No newline at end of file diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp new file mode 100644 index 000000000..aaaab53c5 --- /dev/null +++ b/cudaextender/src/extender.cpp @@ -0,0 +1,46 @@ +/* +* Copyright 2019-2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include + +#include "ungapped_xdrop.hpp" + +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ + +std::unique_ptr create_extender(int32_t* h_sub_mat, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type) +{ + if (type == ExtensionType::ungapped_xdrop) + { + return std::make_unique(h_sub_mat, no_entropy, stream, device_id); + } + else + { + throw std::runtime_error("Extender for specified type not implemented yet."); + } +} + +} // namespace cudaaligner + +} // namespace genomeworks + +} // namespace claraparabricks diff --git a/cudaextender/src/ungapped_xdrop.cpp b/cudaextender/src/ungapped_xdrop.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/cudaextender/src/ungapped_xdrop.hpp b/cudaextender/src/ungapped_xdrop.hpp new file mode 100644 index 000000000..a545324d0 --- /dev/null +++ b/cudaextender/src/ungapped_xdrop.hpp @@ -0,0 +1,66 @@ +/* +* Copyright 2019-2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#pragma once + +#include + +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ + +class UngappedXDrop : public Extender +{ +public: + UngappedXDrop(int32_t* h_sub_mat, bool no_entropy = false, cudaStream_t stream = 0, int32_t device_id = 0); + ~UngappedXDrop() override; + + StatusType extend_async(const char* h_query, int32_t query_length, + const char* h_target, int32_t target_length, + int32_t score_threshold, + std::vector& h_seed_pairs) override; + + StatusType extend_async(const char* d_query, int32_t query_length, + const char* d_target, int32_t target_length, + int32_t score_threshold, SeedPair* d_seed_pairs, + int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, + int32_t* d_num_scored_segment_pairs) override; + + StatusType sync() override; + void reset() override; + + + const std::vector& get_scored_segment_pairs() const override; + +private: + + int32_t* h_sub_mat_; + cudaStream_t stream_; + int32_t device_id_; + bool no_entropy_; + std::vector scored_segment_pairs_; +}; + +} // namespace cudaaligner + +} // namespace genomeworks + +} // namespace claraparabricks diff --git a/cudaungappedextender/CMakeLists.txt b/cudaungappedextender/CMakeLists.txt deleted file mode 100644 index a76075e5f..000000000 --- a/cudaungappedextender/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Enable formatting -gw_enable_auto_formatting("${CMAKE_CURRENT_SOURCE_DIR}") - diff --git a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp b/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp deleted file mode 100644 index fc765cdac..000000000 --- a/cudaungappedextender/include/claraparabricks/genomeworks/cudaungappedextender/cudaungappedextender.hpp +++ /dev/null @@ -1,90 +0,0 @@ -/* -* Copyright 2020 NVIDIA CORPORATION. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#pragma once - -namespace claraparabricks -{ -namespace genomeworks -{ -namespace cudaungappedextender -{ - -struct SeedPair -{ - /// position of first sketch element in query_read_id_ - position_in_read_t query_position_in_read; - /// position of second sketch element in target_read_id_ - position_in_read_t target_position_in_read; -}; - -struct ScoredSegmentPair -{ - /// Seed for the segment - Seed seed_pair; - /// length of the segment - int32_t length; - /// score of the segment - int32_t score; -}; - -/// CUDA Ungapped Extension object -class UngappedExtender -{ -public: - /// \brief Constructor Prototype - UngappedExtender(int* h_sub_mat, int input_xdrop, bool input_noentropy, - int gpu_id = 0, cudaStream_t stream = 0); - - /// \brief Host pointer prototype for ungapped extension - /// - /// Takes values from host data structures, - /// copies them over to device, - /// launches async extension kernels on specified stream. Filters - /// segments on device based on input_hspthresh. - StatusType extend_async(const char* h_query, int32_t query_length, - const char* h_target, int32_t target_length, - int32_t score_threshold, - std::vector& h_seed_pairs); - - /// \brief Device pointer prototype for ungapped extension - /// - /// Memcopies to device memory are assumed to be done before this - /// function. Output array d_hsp_out must be pre-allocated on device. - /// Launches async extension kernel. Filters segments on device - /// based on input_hspthresh. - StatusType extend_async(const char* d_query, int32_t query_length, - const char* d_target, int32_t target_length, - int32_t score_threshold, SeedPair* d_seed_pairs, - int32_t num_seed_pairs, ScoredSegmentPair* d_hsp_out, - int32_t* d_num_hsps); - - /// \brief Waits for CUDA accelerated extension to finish - /// - /// Blocking call that waits for all the extensions scheduled - /// on the GPU to come to completion. - StatusType sync(); - - /// \brief Return the computed segments - /// - /// \return Vector of Scored Segments - const std::vector& get_scored_segment_pairs() const; - - /// \brief Reset UngappedExtender object and free device/host memory - void reset(); -}; -} // namespace cudaungappedextender -} // namespace genomeworks -} // namespace claraparabricks diff --git a/cudaungappedextender/samples/segalign_seedfilter.cu b/cudaungappedextender/samples/segalign_seedfilter.cu deleted file mode 100644 index 152719373..000000000 --- a/cudaungappedextender/samples/segalign_seedfilter.cu +++ /dev/null @@ -1,414 +0,0 @@ -#include "parameters.h" -#include "seed_filter.h" -#include -#include -#include -#include -#include -#include -#include - -#define MAX_SEED_HITS_PER_GB 8388608 -#define MAX_UNGAPPED_PER_GB 4194304 - -using namespace claraparabricks::genomeworks; -using namespace claraparabricks::genomeworks::cudaungappedextender; - -// Control Variables -std::mutex mu; -std::condition_variable cv; -std::vector available_gpus; -std::vector g_cuda_ungapped_extenders; // indexed by gpu_id -std::vector g_cuda_streams; // indexed by gpu_id - -int NUM_DEVICES; - -// Seed Variables -uint32_t MAX_SEEDS; -uint32_t MAX_SEED_HITS; - -char** d_ref_seq; -uint32_t ref_len; - -char** d_query_seq; -char** d_query_rc_seq; -uint32_t query_length[BUFFER_DEPTH]; - -uint32_t seed_size; -uint32_t** d_index_table; -uint32_t** d_pos_table; - -uint64_t** d_seed_offsets; - -uint32_t** d_hit_num_array; -std::vector> d_hit_num_vec; - -Anchor** d_hit; -std::vector> d_hit_vec; - -ScoredSegment** d_hsp; -std::vector> d_hsp_vec; - -// UngappedExtend Variables (ideally not visible to the user in the API) -uint32_t MAX_UNGAPPED; // maximum extensions per iteration in the -// UngappedExtension function - -int** d_sub_mat; // substitution score matrix -int xdrop; // xdrop parameter for the UngappedExtension function -int hspthresh; // score threshold for qualifying as an HSP -bool noentropy; // whether or not to adjust scores of segments as a factor of -// the Shannon entropy - -// convert input sequence from alphabet to integers -__global__ void compress_string(char* dst_seq, char* src_seq, uint32_t len){ - ...} - -// convert input sequence to its reverse complement and convert from alphabet to -// integers -__global__ void compress_rev_comp_string(char* dst_seq, char* src_seq, - uint32_t len){ - ...} - -///////////////////// End Ungapped Extension related functions executed on the -/// GPU /////////////// - -__global__ void find_num_hits(int num_seeds, - const uint32_t* __restrict__ d_index_table, - uint64_t* seed_offsets, uint32_t* seed_hit_num){ - ...} - -__global__ void find_hits(const uint32_t* __restrict__ d_index_table, - const uint32_t* __restrict__ d_pos_table, - uint64_t* d_seed_offsets, uint32_t seed_size, - uint32_t* seed_hit_num, int num_hits, Anchor* d_hit, - uint32_t start_seed_index, uint32_t start_hit_index) -{ - ... -} - -int InitializeProcessor(int num_gpu, bool transition, uint32_t WGA_CHUNK, - uint32_t input_seed_size, int* sub_mat, int input_xdrop, - int input_hspthresh, bool input_noentropy) -{ - - int nDevices; - - cudaError_t err = cudaGetDeviceCount(&nDevices); - if (err != cudaSuccess) - { - fprintf(stderr, "Error: No GPU device found!\n"); - exit(1); - } - - if (num_gpu == -1) - { - NUM_DEVICES = nDevices; - } - else - { - if (num_gpu <= nDevices) - { - NUM_DEVICES = num_gpu; - } - else - { - fprintf(stderr, "Requested GPUs greater than available GPUs\n"); - exit(10); - } - } - - fprintf(stderr, "Using %d GPU(s)\n", NUM_DEVICES); - - seed_size = input_seed_size; - - if (transition) - MAX_SEEDS = 13 * WGA_CHUNK; - else - MAX_SEEDS = WGA_CHUNK; - - cudaDeviceProp deviceProp; - cudaGetDeviceProperties(&deviceProp, 0); - float global_mem_gb = - static_cast(deviceProp.totalGlobalMem / 1073741824.0f); - MAX_SEED_HITS = global_mem_gb * MAX_SEED_HITS_PER_GB; - - Anchor zeroHit; - zeroHit.query_position_in_read_ = 0; - zeroHit.target_position_in_read_ = 0; - - ScoredSegment zeroHsp; - zeroHsp.anchor.query_position_in_read_ = 0; - zeroHsp.anchor.target_position_in_read_ = 0; - zeroHsp.len = 0; - zeroHsp.score = 0; - - d_ref_seq = (char**)malloc(NUM_DEVICES * sizeof(char*)); - d_query_seq = (char**)malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char*)); - d_query_rc_seq = (char**)malloc(BUFFER_DEPTH * NUM_DEVICES * sizeof(char*)); - - d_index_table = (uint32_t**)malloc(NUM_DEVICES * sizeof(uint32_t*)); - d_pos_table = (uint32_t**)malloc(NUM_DEVICES * sizeof(uint32_t*)); - - d_seed_offsets = (uint64_t**)malloc(NUM_DEVICES * sizeof(uint64_t*)); - - d_hit_num_array = (uint32_t**)malloc(NUM_DEVICES * sizeof(int32_t*)); - d_hit_num_vec.reserve(NUM_DEVICES); - - d_hit = (seedHit**)malloc(NUM_DEVICES * sizeof(Anchor*)); - d_hit_vec.reserve(NUM_DEVICES); - - d_hsp = (segment**)malloc(NUM_DEVICES * sizeof(ScoredSegment*)); - d_hsp_vec.reserve(NUM_DEVICES); - - for (int g = 0; g < NUM_DEVICES; g++) - { - - check_cuda_setDevice(g, "InitializeProcessor"); - - check_cuda_malloc((void**)&d_seed_offsets[g], MAX_SEEDS * sizeof(uint64_t), - "seed_offsets"); - - d_hit_num_vec.emplace_back(MAX_SEEDS, 0); - d_hit_num_array[g] = thrust::raw_pointer_cast(d_hit_num_vec.at(g).data()); - - d_hit_vec.emplace_back(MAX_SEED_HITS, zeroHit); - d_hit[g] = thrust::raw_pointer_cast(d_hit_vec.at(g).data()); - - d_hsp_vec.emplace_back(MAX_SEED_HITS, zeroHsp); - d_hsp[g] = thrust::raw_pointer_cast(d_hsp_vec.at(g).data()); - cudaStream_t stream; - cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking); - g_cuda_streams.push_back(stream); - g_cuda_ungapped_extenders.emplace_back(g, sub_mat, input_xdrop, - input_noentropy, stream); - available_gpus.push_back(g); - } - return NUM_DEVICES; -} - -void InclusivePrefixScan(uint32_t* data, uint32_t len) -{ - ... -} - -void SendSeedPosTable(uint32_t* index_table, uint32_t index_table_size, - uint32_t* pos_table, uint32_t num_index, - uint32_t max_pos_index) -{ - ... -} - -void SendRefWriteRequest(size_t start_addr, uint32_t len) -{ - ... -} - -void SendQueryWriteRequest(size_t start_addr, uint32_t len, uint32_t buffer){ - ...} - -std::vector SeedAndFilter(std::vector seed_offset_vector, - bool rev, uint32_t buffer, - int input_hspthresh) -{ - - uint32_t num_hits = 0; - uint32_t total_anchors = 0; - - uint32_t num_seeds = seed_offset_vector.size(); - - uint64_t* tmp_offset = (uint64_t*)malloc(num_seeds * sizeof(uint64_t)); - for (uint32_t i = 0; i < num_seeds; i++) - { - tmp_offset[i] = seed_offset_vector[i]; - } - - int g; - std::unique_lock locker(mu); - if (available_gpus.empty()) - { - cv.wait(locker, []() { return !available_gpus.empty(); }); - } - g = available_gpus.back(); - available_gpus.pop_back(); - locker.unlock(); - - check_cuda_setDevice(g, "SeedAndFilter"); - - check_cuda_memcpy((void*)d_seed_offsets[g], (void*)tmp_offset, - num_seeds * sizeof(uint64_t), cudaMemcpyHostToDevice, - "seed_offsets"); - - find_num_hits<<>>( - num_seeds, d_index_table[g], d_seed_offsets[g], d_hit_num_array[g]); - - thrust::inclusive_scan(d_hit_num_vec[g].begin(), - d_hit_num_vec[g].begin() + num_seeds, - d_hit_num_vec[g].begin()); - - check_cuda_memcpy((void*)&num_hits, - (void*)(d_hit_num_array[g] + num_seeds - 1), - sizeof(uint32_t), cudaMemcpyDeviceToHost, "num_hits"); - - int num_iter = num_hits / MAX_UNGAPPED + 1; - uint32_t iter_hit_limit = MAX_UNGAPPED; - thrust::device_vector limit_pos(num_iter); - - for (int i = 0; i < num_iter - 1; i++) - { - thrust::device_vector::iterator result_end = thrust::lower_bound( - d_hit_num_vec[g].begin(), d_hit_num_vec[g].begin() + num_seeds, - iter_hit_limit); - uint32_t pos = thrust::distance(d_hit_num_vec[g].begin(), result_end) - 1; - iter_hit_limit = d_hit_num_vec[g][pos] + MAX_UNGAPPED; - limit_pos[i] = pos; - } - - limit_pos[num_iter - 1] = num_seeds - 1; - - segment** h_hsp = (segment**)malloc(num_iter * sizeof(segment*)); - uint32_t* num_anchors = (uint32_t*)calloc(num_iter, sizeof(uint32_t)); - - uint32_t start_seed_index = 0; - uint32_t start_hit_val = 0; - uint32_t iter_num_seeds, iter_num_hits; - int32_t* d_num_anchors; - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_anchors, sizeof(int32_t))); - - if (num_hits > 0) - { - - for (int i = 0; i < num_iter; i++) - { - iter_num_seeds = limit_pos[i] + 1 - start_seed_index; - iter_num_hits = d_hit_num_vec[g][limit_pos[i]] - start_hit_val; - - find_hits<<>>( - d_index_table[g], d_pos_table[g], d_seed_offsets[g], seed_size, - d_hit_num_array[g], iter_num_hits, d_hit[g], start_seed_index, - start_hit_val); - - if (rev) - { - if (!g_cuda_ungapped_extenders[g].ungapped_extend( - d_query_rc_seq[buffer * NUM_DEVICES + g], query_length[buffer], - d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, - d_hsp[g], d_num_anchors)) - ; - { - // err... - } - } - else - { - if (!g_cuda_ungapped_extenders[g].ungapped_extend( - d_query_seq[buffer * NUM_DEVICES + g], query_length[buffer], - d_ref_seq[g], ref_len, input_hspthresh, d_hit[g], iter_num_hits, - d_hsp[g], d_num_anchors)) - ; - { - // err... - } - } - GW_CU_CHECK_ERR(cudaMemcpyAsync(&num_anchors[i], d_num_anchors, - sizeof(int32_t), cudaMemcpyDeviceToHost, - g_cuda_streams[g])); - cudaStreamSynchronize(g_cuda_streams[g]); - - total_anchors += num_anchors[i]; - - if (num_anchors[i] > 0) - { - h_hsp[i] = (segment*)calloc(num_anchors[i], sizeof(segment)); - - check_cuda_memcpy((void*)h_hsp[i], (void*)d_hsp[g], - num_anchors[i] * sizeof(segment), - cudaMemcpyDeviceToHost, "hsp_output"); - } - - start_seed_index = limit_pos[i] + 1; - start_hit_val = d_hit_num_vec[g][limit_pos[i]]; - } - } - - limit_pos.clear(); - - { - std::unique_lock locker(mu); - available_gpus.push_back(g); - locker.unlock(); - cv.notify_one(); - } - - std::vector gpu_filter_output; - - segment first_el; - first_el.len = total_anchors; - first_el.score = num_hits; - gpu_filter_output.push_back(first_el); - - if (total_anchors > 0) - { - for (int it = 0; it < num_iter; it++) - { - - for (int i = 0; i < num_anchors[it]; i++) - { - gpu_filter_output.push_back(h_hsp[it][i]); - } - } - free(h_hsp); - } - - free(tmp_offset); - return gpu_filter_output; -} - -void clearRef() -{ - ... -} - -void clearQuery(uint32_t buffer) -{ - ... -} - -void ShutdownProcessor() -{ - - d_hit_num_vec.clear(); - d_hit_vec.clear(); - d_hsp_vec.clear(); - - g_cuda_ungapped_extenders.clear(); - for (auto& cudaStream : g_cuda_streams) - { - cudaStreamDestroy(cudaStream); - } - g_cuda_streams.clear(); - cudaDeviceReset(); -} - -///// Start Ungapped Extension related functions ///// - -void CompressSeq(char* input_seq, char* output_seq, uint32_t len) -{ - ... -} - -void CompressRevCompSeq(char* input_seq, char* output_seq, uint32_t len){...} - -CompressSeq_ptr g_CompressSeq = CompressSeq; -CompressRevCompSeq_ptr g_CompressRevCompSeq = CompressRevCompSeq; - -///// End Ungapped Extension related functions ///// - -InitializeProcessor_ptr g_InitializeProcessor = InitializeProcessor; -InclusivePrefixScan_ptr g_InclusivePrefixScan = InclusivePrefixScan; -SendSeedPosTable_ptr g_SendSeedPosTable = SendSeedPosTable; -SendRefWriteRequest_ptr g_SendRefWriteRequest = SendRefWriteRequest; -SendQueryWriteRequest_ptr g_SendQueryWriteRequest = SendQueryWriteRequest; -SeedAndFilter_ptr g_SeedAndFilter = SeedAndFilter; -clearRef_ptr g_clearRef = clearRef; -clearQuery_ptr g_clearQuery = clearQuery; -ShutdownProcessor_ptr g_ShutdownProcessor = ShutdownProcessor; From 3a52056bfe129373b2b1e836c52a1e39b956011c Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 4 Sep 2020 18:01:30 -0700 Subject: [PATCH 043/281] [cudaextender] Make format --- cudaextender/CMakeLists.txt | 12 ++++++------ cudaextender/samples/CMakeLists.txt | 11 +++++------ cudaextender/samples/device_sample.cu | 4 ++-- cudaextender/samples/host_sample.cu | 4 ++-- cudaextender/src/extender.cpp | 2 +- cudaextender/src/ungapped_xdrop.hpp | 10 ++++------ 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/cudaextender/CMakeLists.txt b/cudaextender/CMakeLists.txt index f7943368f..ff797df4e 100644 --- a/cudaextender/CMakeLists.txt +++ b/cudaextender/CMakeLists.txt @@ -25,20 +25,20 @@ configure_file(${CMAKE_SOURCE_DIR}/common/base/src/version.cpp.in # Optimization flags interfere with nvcc and cause compilation issues with reworked # file structure. -if(NOT DEFINED gw_cuda_before_10_0) +if (NOT DEFINED gw_cuda_before_10_0) message(FATAL_ERROR "${PROJECT_NAME} : variable gw_cuda_before_10_0 is not defined yet. Please make sure CUDA.cmake is loaded first.") -elseif(gw_cuda_before_10_0) +elseif (gw_cuda_before_10_0) message(STATUS "${PROJECT_NAME} : Remove -O optimization when building for CUDA < 10 as it causes compilation issues.") string(REGEX REPLACE "-O[0-3]" "" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) string(REGEX REPLACE "-O[0-3]" "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) -endif() +endif () # Project specific NVCC flags, --expt-relaxed-constexpr is being added to allow using numeric_limits inside device kernels. # Using in CUDA 10.1+ to avoid compilation issues with CUDA <= 10.0. set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14") -if(gw_cuda_after_10_0) +if (gw_cuda_after_10_0) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} --expt-relaxed-constexpr") -endif() +endif () message(STATUS "nvcc flags for ${PROJECT_NAME}: ${CUDA_NVCC_FLAGS}") get_property(gw_library_type GLOBAL PROPERTY gw_library_type) @@ -53,7 +53,7 @@ target_link_libraries(cudaextender gwbase gwio) target_compile_options(cudaextender PRIVATE -Werror -Wall -Wextra) if (gw_optimize_for_native_cpu) target_compile_options(cudaextender PRIVATE -march=native) -endif() +endif () target_include_directories(cudaextender PUBLIC diff --git a/cudaextender/samples/CMakeLists.txt b/cudaextender/samples/CMakeLists.txt index 9649e6a4a..09bda87b3 100644 --- a/cudaextender/samples/CMakeLists.txt +++ b/cudaextender/samples/CMakeLists.txt @@ -15,19 +15,18 @@ # - project(sample_cudaextender) get_property(cudaextender_data_include_dir GLOBAL PROPERTY cudaextender_data_include_dir) include_directories(${cudaextender_data_include_dir}) add_executable(${PROJECT_NAME} - host_sample.cu - ) + host_sample.cu + ) target_link_libraries(${PROJECT_NAME} - cudaextender - ) + cudaextender + ) install(TARGETS ${PROJECT_NAME} - DESTINATION samples) + DESTINATION samples) diff --git a/cudaextender/samples/device_sample.cu b/cudaextender/samples/device_sample.cu index 7dae9085c..8ec002b12 100644 --- a/cudaextender/samples/device_sample.cu +++ b/cudaextender/samples/device_sample.cu @@ -27,8 +27,8 @@ using namespace claraparabricks::genomeworks::cudaungappedextender; int main(int argc, char* argv[]) { - const int32_t input_xdrop = 910; - const bool input_no_entropy = false; + const int32_t input_xdrop = 910; + const bool input_no_entropy = false; const int32_t score_threshold = 3000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; diff --git a/cudaextender/samples/host_sample.cu b/cudaextender/samples/host_sample.cu index 7e5c26682..4e670b604 100644 --- a/cudaextender/samples/host_sample.cu +++ b/cudaextender/samples/host_sample.cu @@ -25,8 +25,8 @@ using namespace claraparabricks::genomeworks::cudaungappedextender; int main(int argc, char* argv[]) { - const int32_t input_xdrop = 910; - const bool input_no_entropy = false; + const int32_t input_xdrop = 910; + const bool input_no_entropy = false; const int32_t score_threshold = 3000; // Fasta query and target files std::string target_file_path = "../data/example.fa"; diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index aaaab53c5..eee7cc18c 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -39,7 +39,7 @@ std::unique_ptr create_extender(int32_t* h_sub_mat, bool no_entropy, c } } -} // namespace cudaaligner +} // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/src/ungapped_xdrop.hpp b/cudaextender/src/ungapped_xdrop.hpp index a545324d0..19feb47c9 100644 --- a/cudaextender/src/ungapped_xdrop.hpp +++ b/cudaextender/src/ungapped_xdrop.hpp @@ -34,9 +34,9 @@ class UngappedXDrop : public Extender ~UngappedXDrop() override; StatusType extend_async(const char* h_query, int32_t query_length, - const char* h_target, int32_t target_length, - int32_t score_threshold, - std::vector& h_seed_pairs) override; + const char* h_target, int32_t target_length, + int32_t score_threshold, + std::vector& h_seed_pairs) override; StatusType extend_async(const char* d_query, int32_t query_length, const char* d_target, int32_t target_length, @@ -47,11 +47,9 @@ class UngappedXDrop : public Extender StatusType sync() override; void reset() override; - const std::vector& get_scored_segment_pairs() const override; private: - int32_t* h_sub_mat_; cudaStream_t stream_; int32_t device_id_; @@ -59,7 +57,7 @@ class UngappedXDrop : public Extender std::vector scored_segment_pairs_; }; -} // namespace cudaaligner +} // namespace cudaextender } // namespace genomeworks From 58fd25ea5b199f0f15b91ee4a7975721448fe746 Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Tue, 8 Sep 2020 12:18:13 -0700 Subject: [PATCH 044/281] create_index accepts IndexDescriptor --- .../genomeworks/cudamapper/index.hpp | 96 +++++++++---------- cudamapper/samples/sample_cudamapper.cpp | 6 +- cudamapper/src/index.cu | 7 +- cudamapper/src/index_cache.cu | 3 +- cudamapper/tests/Test_CudamapperMatcherGPU.cu | 8 +- 5 files changed, 57 insertions(+), 63 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index 001cbc829..dddd0760b 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -35,6 +35,51 @@ namespace cudamapper /// \addtogroup cudamapper /// \{ +/// IndexDescriptor Implementation - Every Index is defined by its first read and the number of reads +/// Used in conjunction with a fasta parser object to create an index +class IndexDescriptor +{ +public: + /// \brief constructor + IndexDescriptor(read_id_t first_read, + number_of_reads_t number_of_reads); + + /// \brief getter + read_id_t first_read() const; + + /// \brief getter + number_of_reads_t number_of_reads() const; + + /// \brief returns hash value + std::size_t get_hash() const; + +private: + /// \brief generates hash + void generate_hash(); + + /// first read in index + read_id_t first_read_; + /// number of reads in index + number_of_reads_t number_of_reads_; + /// hash of this object + std::size_t hash_; +}; + +/// \brief equality operator +bool operator==(const IndexDescriptor& lhs, + const IndexDescriptor& rhs); + +/// \brief inequality operator +bool operator!=(const IndexDescriptor& lhs, + const IndexDescriptor& rhs); + +/// IndexDescriptorHash - operator() calculates hash of a given IndexDescriptor +struct IndexDescriptorHash +{ + /// \brief caclulates hash of given IndexDescriptor + std::size_t operator()(const IndexDescriptor& index_descriptor) const; +}; + /// Index - manages mapping of (k,w)-kmer-representation and all its occurences class Index { @@ -92,8 +137,7 @@ class Index /// \brief generates a mapping of (k,w)-kmer-representation to all of its occurrences for one or more sequences /// \param allocator The device memory allocator to use for temporary buffer allocations /// \param parser parser for the whole input file (part that goes into this index is determined by first_read_id and past_the_last_read_id) - /// \param first_read_id read_id of the first read to the included in this index - /// \param past_the_last_read_id read_id+1 of the last read to be included in this index + /// \param descriptor object holding the first read, size of the read, and hash value of the index being created /// \param kmer_size k - the kmer length /// \param window_size w - the length of the sliding window used to find sketch elements (i.e. the number of adjacent kmers in a window, adjacent = shifted by one basepair) /// \param hash_representations if true, hash kmer representations @@ -103,8 +147,7 @@ class Index static std::unique_ptr create_index(DefaultDeviceAllocator allocator, const io::FastaParser& parser, - const read_id_t first_read_id, - const read_id_t past_the_last_read_id, + const IndexDescriptor& descriptor, const std::uint64_t kmer_size, const std::uint64_t window_size, const bool hash_representations = true, @@ -187,51 +230,6 @@ class IndexHostCopyBase const cudaStream_t cuda_stream = 0); }; -/// IndexDescriptor Implementation - Every Index is defined by its first read and the number of reads -/// Used in conjunction with a fasta parser object to create an index -class IndexDescriptor -{ -public: - /// \brief constructor - IndexDescriptor(read_id_t first_read, - number_of_reads_t number_of_reads); - - /// \brief getter - read_id_t first_read() const; - - /// \brief getter - number_of_reads_t number_of_reads() const; - - /// \brief returns hash value - std::size_t get_hash() const; - -private: - /// \brief generates hash - void generate_hash(); - - /// first read in index - read_id_t first_read_; - /// number of reads in index - number_of_reads_t number_of_reads_; - /// hash of this object - std::size_t hash_; -}; - -/// \brief equality operator -bool operator==(const IndexDescriptor& lhs, - const IndexDescriptor& rhs); - -/// \brief inequality operator -bool operator!=(const IndexDescriptor& lhs, - const IndexDescriptor& rhs); - -/// IndexDescriptorHash - operator() calculates hash of a given IndexDescriptor -struct IndexDescriptorHash -{ - /// \brief caclulates hash of given IndexDescriptor - std::size_t operator()(const IndexDescriptor& index_descriptor) const; -}; - } // namespace cudamapper } // namespace genomeworks diff --git a/cudamapper/samples/sample_cudamapper.cpp b/cudamapper/samples/sample_cudamapper.cpp index c35ebe766..61492852c 100644 --- a/cudamapper/samples/sample_cudamapper.cpp +++ b/cudamapper/samples/sample_cudamapper.cpp @@ -84,8 +84,7 @@ void process_batch(std::vector& query_index_descriptors, { std::unique_ptr query_index = Index::create_index(allocator, *query_parser, - query_index_descriptor.first_read(), - query_index_descriptor.first_read() + query_index_descriptor.number_of_reads(), + query_index_descriptor, KMER_SIZE, WINDOWS_SIZE, true, // hash representations @@ -98,8 +97,7 @@ void process_batch(std::vector& query_index_descriptors, { std::unique_ptr target_index = Index::create_index(allocator, *target_parser, - target_index_descriptor.first_read(), - target_index_descriptor.first_read() + target_index_descriptor.number_of_reads(), + target_index_descriptor, KMER_SIZE, WINDOWS_SIZE, true, // hash representations diff --git a/cudamapper/src/index.cu b/cudamapper/src/index.cu index 98c7064b4..1e9cf0564 100644 --- a/cudamapper/src/index.cu +++ b/cudamapper/src/index.cu @@ -30,8 +30,7 @@ namespace cudamapper std::unique_ptr Index::create_index(DefaultDeviceAllocator allocator, const io::FastaParser& parser, - const read_id_t first_read_id, - const read_id_t past_the_last_read_id, + const IndexDescriptor& descriptor, const std::uint64_t kmer_size, const std::uint64_t window_size, const bool hash_representations, @@ -41,8 +40,8 @@ std::unique_ptr Index::create_index(DefaultDeviceAllocator allocator, GW_NVTX_RANGE(profiler, "create_index"); return std::make_unique>(allocator, parser, - first_read_id, - past_the_last_read_id, + descriptor.first_read(), + descriptor.first_read() + descriptor.number_of_reads(), kmer_size, window_size, hash_representations, diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 4e82c7e31..d3f37182f 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -155,8 +155,7 @@ void IndexCacheHost::generate_cache_content(const std::vector& // create index index_on_device = Index::create_index(allocator_, *parser, - descriptor_of_index_to_cache.first_read(), - descriptor_of_index_to_cache.first_read() + descriptor_of_index_to_cache.number_of_reads(), + descriptor_of_index_to_cache, kmer_size_, window_size_, hash_representations_, diff --git a/cudamapper/tests/Test_CudamapperMatcherGPU.cu b/cudamapper/tests/Test_CudamapperMatcherGPU.cu index ffe6165e9..cb6eee362 100644 --- a/cudamapper/tests/Test_CudamapperMatcherGPU.cu +++ b/cudamapper/tests/Test_CudamapperMatcherGPU.cu @@ -639,8 +639,8 @@ TEST(TestCudamapperMatcherGPU, OneReadOneMinimizer) { DefaultDeviceAllocator allocator = create_default_device_allocator(); std::unique_ptr parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/gatt.fasta"); - std::unique_ptr query_index = Index::create_index(allocator, *parser, 0, parser->get_num_seqences(), 4, 1); - std::unique_ptr target_index = Index::create_index(allocator, *parser, 0, parser->get_num_seqences(), 4, 1); + std::unique_ptr query_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_numseqences()), 4, 1); + std::unique_ptr target_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_numseqences()), 4, 1); MatcherGPU matcher(allocator, *query_index, *target_index); thrust::host_vector anchors(matcher.anchors().size()); @@ -652,8 +652,8 @@ TEST(TestCudamapperMatcherGPU, AtLeastOneIndexEmpty) { DefaultDeviceAllocator allocator = create_default_device_allocator(); std::unique_ptr parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/gatt.fasta"); - std::unique_ptr index_full = Index::create_index(allocator, *parser, 0, parser->get_num_seqences(), 4, 1); - std::unique_ptr index_empty = Index::create_index(allocator, *parser, 0, parser->get_num_seqences(), 5, 1); // kmer longer than read + std::unique_ptr index_full = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_numseqences()), 4, 1); + std::unique_ptr index_empty = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_numseqences()), 5, 1); // kmer longer than read { MatcherGPU matcher(allocator, *index_full, *index_empty); From 58d15be094e9b43cbf25994fa707d083b6e81122 Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 8 Sep 2020 17:13:17 -0700 Subject: [PATCH 045/281] [cudaextender] Add more files --- cudaextender/CMakeLists.txt | 3 +- .../genomeworks/cudaextender/extender.hpp | 4 +- cudaextender/src/extender.cpp | 4 +- cudaextender/src/ungapped_extender_kernels.cu | 397 ++++++++++++++++++ cudaextender/src/ungapped_xdrop.cpp | 104 +++++ cudaextender/src/ungapped_xdrop.hpp | 10 +- cudaextender/src/ungapped_xdrop_kernels.cuh | 24 ++ 7 files changed, 538 insertions(+), 8 deletions(-) create mode 100644 cudaextender/src/ungapped_extender_kernels.cu create mode 100644 cudaextender/src/ungapped_xdrop_kernels.cuh diff --git a/cudaextender/CMakeLists.txt b/cudaextender/CMakeLists.txt index ff797df4e..2cd479eac 100644 --- a/cudaextender/CMakeLists.txt +++ b/cudaextender/CMakeLists.txt @@ -45,8 +45,9 @@ get_property(gw_library_type GLOBAL PROPERTY gw_library_type) cuda_add_library(cudaextender ${gw_library_type} src/cudaextender.cpp src/extender.cpp + src/ungapped_xdrop.cpp ${CMAKE_CURRENT_BINARY_DIR}/version.cpp - ) + src/ungapped_extender_kernels.cu) target_link_libraries(cudaextender gwbase gwio) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index bed005085..cd3228662 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -91,9 +91,7 @@ class Extender virtual void reset() = 0; }; -std::unique_ptr create_extender(int32_t* h_sub_mat, bool no_entropy = false, - cudaStream_t stream = 0, int32_t device_id = 0, - ExtensionType type = ExtensionType::ungapped_xdrop); +std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_size, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type); } // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index eee7cc18c..2988a9841 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -27,11 +27,11 @@ namespace genomeworks namespace cudaextender { -std::unique_ptr create_extender(int32_t* h_sub_mat, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type) +std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_size, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type) { if (type == ExtensionType::ungapped_xdrop) { - return std::make_unique(h_sub_mat, no_entropy, stream, device_id); + return std::make_unique(h_sub_mat, sub_mat_size, xdrop_threshold, no_entropy, stream, device_id); } else { diff --git a/cudaextender/src/ungapped_extender_kernels.cu b/cudaextender/src/ungapped_extender_kernels.cu new file mode 100644 index 000000000..1b5695e25 --- /dev/null +++ b/cudaextender/src/ungapped_extender_kernels.cu @@ -0,0 +1,397 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#include "ungapped_xdrop_kernels.cuh" + + +// extend the hits to a segment by ungapped x-drop method, adjust low-scoring +// segment scores based on entropy factor, compare resulting segment scores +// to hspthresh and update the d_hsp and d_done vectors +__global__ +void find_hsps (const char* __restrict__ d_ref_seq, const char* __restrict__ d_query_seq, uint32_t ref_len, uint32_t query_len, int *d_sub_mat, bool noentropy, int xdrop, int hspthresh, int num_hits, seedHit* d_hit, uint32_t start_index, segment* d_hsp, uint32_t* d_done){ + + int thread_id = threadIdx.x; + int block_id = blockIdx.x; + int num_blocks = gridDim.x; + int warp_size = warpSize; + int lane_id = thread_id%warp_size; + int warp_id = (thread_id-lane_id)/warp_size; + + __shared__ uint32_t ref_loc[NUM_WARPS]; + __shared__ uint32_t query_loc[NUM_WARPS]; + __shared__ int total_score[NUM_WARPS]; + __shared__ int prev_score[NUM_WARPS]; + __shared__ int prev_max_score[NUM_WARPS]; + __shared__ uint32_t prev_max_pos[NUM_WARPS]; + __shared__ bool edge_found[NUM_WARPS]; + __shared__ bool xdrop_found[NUM_WARPS]; + __shared__ bool new_max_found[NUM_WARPS]; + __shared__ uint32_t left_extent[NUM_WARPS]; + __shared__ uint32_t extent[NUM_WARPS]; + __shared__ uint32_t tile[NUM_WARPS]; + __shared__ double entropy[NUM_WARPS]; + + int thread_score; + int max_thread_score; + uint32_t max_pos; + uint32_t temp_pos; + bool xdrop_done; + int temp; + short count[4]; + short count_del[4]; + char r_chr; + char q_chr; + uint32_t ref_pos; + uint32_t query_pos; + uint32_t pos_offset; + + __shared__ int sub_mat[NUC2]; + + if(thread_id < NUC2){ + sub_mat[thread_id] = d_sub_mat[thread_id]; + } + __syncthreads(); + + for(int hid0 = block_id*NUM_WARPS; hid0 < num_hits; hid0 += NUM_WARPS*num_blocks){ + int hid = hid0 + warp_id + start_index; + + if(hid < num_hits){ + if(lane_id == 0){ + ref_loc[warp_id] = d_hit[hid].ref_start; + query_loc[warp_id] = d_hit[hid].query_start; + total_score[warp_id] = 0; + } + } + else{ + if(lane_id == 0){ + + ref_loc[warp_id] = d_hit[hid0].ref_start; + query_loc[warp_id] = d_hit[hid0].query_start; + total_score[warp_id] = 0; + } + } + __syncwarp(); + + + ////////////////////////////////////////////////////////////////// + //Right extension + + if(lane_id ==0){ + tile[warp_id] = 0; + xdrop_found[warp_id] = false; + edge_found[warp_id] = false; + new_max_found[warp_id] = false; + entropy[warp_id] = 1.0f; + prev_score[warp_id] = 0; + prev_max_score[warp_id] = -1000; + prev_max_pos[warp_id] = 0; + extent[warp_id] = 0; + } + + count[0] = 0; + count[1] = 0; + count[2] = 0; + count[3] = 0; + count_del[0] = 0; + count_del[1] = 0; + count_del[2] = 0; + count_del[3] = 0; + max_pos = 0; + + __syncwarp(); + + while(!xdrop_found[warp_id] && !edge_found[warp_id]){ + pos_offset = lane_id + tile[warp_id]; + ref_pos = ref_loc[warp_id] + pos_offset; + query_pos = query_loc[warp_id] + pos_offset; + thread_score = 0; + + if(ref_pos < ref_len && query_pos < query_len){ + r_chr = d_ref_seq[ref_pos]; + q_chr = d_query_seq[query_pos]; + thread_score = sub_mat[r_chr*NUC+q_chr]; + } + __syncwarp(); + + +#pragma unroll + for (int offset = 1; offset < warp_size; offset = offset << 1){ + temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); + + if(lane_id >= offset){ + thread_score += temp; + } + } + + + thread_score += prev_score[warp_id]; + if(thread_score > prev_max_score[warp_id]){ + max_thread_score = thread_score; + max_pos = pos_offset; + } + else{ + max_thread_score = prev_max_score[warp_id]; + max_pos = prev_max_pos[warp_id]; + } + + __syncwarp(); + +#pragma unroll + for (int offset = 1; offset < warp_size; offset = offset << 1){ + temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); + temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); + + if(lane_id >= offset){ + if(temp >= max_thread_score){ + max_thread_score = temp; + max_pos = temp_pos; + } + } + } + + xdrop_done = ((max_thread_score-thread_score) > xdrop); + __syncwarp(); + +#pragma unroll + for (int offset = 1; offset < warp_size; offset = offset << 1){ + xdrop_done |= __shfl_up_sync(0xFFFFFFFF, xdrop_done, offset); + } + + if(lane_id == warp_size-1){ + + if(max_pos > prev_max_pos[warp_id]) + new_max_found[warp_id] = true; + else + new_max_found[warp_id] = false; + + if(xdrop_done){ + total_score[warp_id] += max_thread_score; + xdrop_found[warp_id] = true; + extent[warp_id] = max_pos; + prev_max_pos[warp_id] = max_pos; + tile[warp_id] = max_pos; + } + else if(ref_pos >= ref_len || query_pos >= query_len){ + total_score[warp_id] += max_thread_score; + edge_found[warp_id] = true; + extent[warp_id] = max_pos; + prev_max_pos[warp_id] = max_pos; + tile[warp_id] = max_pos; + } + else{ + prev_score[warp_id] = thread_score; + prev_max_score[warp_id] = max_thread_score; + prev_max_pos[warp_id] = max_pos; + tile[warp_id]+= warp_size; + } + } + __syncwarp(); + + if(new_max_found[warp_id]){ + for(int i = 0; i < 4; i++){ + count[i] = count[i] + count_del[i]; + count_del[i] = 0; + } + } + __syncwarp(); + + if(r_chr == q_chr){ + if(pos_offset <= prev_max_pos[warp_id]){ + count[r_chr] += 1; + } + else{ + count_del[r_chr] += 1; + } + } + __syncwarp(); + + } + + __syncwarp(); + + //////////////////////////////////////////////////////////////// + //Left extension + + if(lane_id ==0){ + tile[warp_id] = 0; + xdrop_found[warp_id] = false; + edge_found[warp_id] = false; + new_max_found[warp_id] = false; + prev_score[warp_id] = 0; + prev_max_score[warp_id] = -1000; + prev_max_pos[warp_id] = 0; + left_extent[warp_id] = 0; + } + + count_del[0] = 0; + count_del[1] = 0; + count_del[2] = 0; + count_del[3] = 0; + max_pos = 0; + __syncwarp(); + + while(!xdrop_found[warp_id] && !edge_found[warp_id]){ + pos_offset = lane_id+1+tile[warp_id]; + thread_score = 0; + + if(ref_loc[warp_id] >= pos_offset && query_loc[warp_id] >= pos_offset){ + ref_pos = ref_loc[warp_id] - pos_offset; + query_pos = query_loc[warp_id] - pos_offset; + r_chr = d_ref_seq[ref_pos]; + q_chr = d_query_seq[query_pos]; + thread_score = sub_mat[r_chr*NUC+q_chr]; + + } + +#pragma unroll + for (int offset = 1; offset < warp_size; offset = offset << 1){ + temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); + + if(lane_id >= offset){ + thread_score += temp; + } + } + + thread_score += prev_score[warp_id]; + if(thread_score > prev_max_score[warp_id]){ + max_thread_score = thread_score; + max_pos = pos_offset; + } + else{ + max_thread_score = prev_max_score[warp_id]; + max_pos = prev_max_pos[warp_id]; + } + __syncwarp(); + +#pragma unroll + for (int offset = 1; offset < warp_size; offset = offset << 1){ + temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); + temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); + + if(lane_id >= offset){ + if(temp >= max_thread_score){ + max_thread_score = temp; + max_pos = temp_pos; + } + } + } + + xdrop_done = ((max_thread_score-thread_score) > xdrop); + __syncwarp(); + +#pragma unroll + for (int offset = 1; offset < warp_size; offset = offset << 1){ + xdrop_done |= __shfl_up_sync(0xFFFFFFFF, xdrop_done, offset); + } + + if(lane_id == warp_size-1){ + + if(max_pos > prev_max_pos[warp_id]) + new_max_found[warp_id] = true; + else + new_max_found[warp_id] = false; + + if(xdrop_done){ + total_score[warp_id]+=max_thread_score; + xdrop_found[warp_id] = true; + left_extent[warp_id] = max_pos; + extent[warp_id] += left_extent[warp_id]; + prev_max_pos[warp_id] = max_pos; + tile[warp_id] = max_pos; + } + else if(ref_loc[warp_id] < pos_offset || query_loc[warp_id] < pos_offset){ + total_score[warp_id]+=max_thread_score; + edge_found[warp_id] = true; + left_extent[warp_id] = max_pos; + extent[warp_id] += left_extent[warp_id]; + prev_max_pos[warp_id] = max_pos; + tile[warp_id] = max_pos; + } + else{ + prev_score[warp_id] = thread_score; + prev_max_score[warp_id] = max_thread_score; + prev_max_pos[warp_id] = max_pos; + tile[warp_id]+=warp_size; + } + } + __syncwarp(); + + if(new_max_found[warp_id]){ + for(int i = 0; i < 4; i++){ + count[i] = count[i] + count_del[i]; + count_del[i] = 0; + } + } + __syncwarp(); + + if(r_chr == q_chr){ + if(pos_offset <= prev_max_pos[warp_id]){ + count[r_chr] += 1; + } + else{ + count_del[r_chr] += 1; + } + } + __syncwarp(); + + } + + ////////////////////////////////////////////////////////////////// + + if(total_score[warp_id] >= hspthresh && total_score[warp_id] <= 3*hspthresh && !noentropy){ + for(int i = 0; i < 4; i++){ +#pragma unroll + for (int offset = 1; offset < warp_size; offset = offset << 1){ + count[i] += __shfl_up_sync(0xFFFFFFFF, count[i], offset); + } + } + __syncwarp(); + + if(lane_id == warp_size-1 && ((count[0]+count[1]+count[2]+count[3]) >= 20)){ + + entropy[warp_id] = 0.f; + for(int i = 0; i < 4; i++){ + entropy[warp_id] += ((double) count[i])/((double) (extent[warp_id]+1)) * ((count[i] != 0) ? log(((double) count[i]) / ((double) (extent[warp_id]+1))): 0.f); + } + entropy[warp_id] = -entropy[warp_id]/log(4.0f); + } + } + __syncwarp(); + + ////////////////////////////////////////////////////////////////// + + if(hid < num_hits){ + if(lane_id == 0){ + + if( ((int) (((float) total_score[warp_id]) * entropy[warp_id])) >= hspthresh){ + d_hsp[hid].ref_start = ref_loc[warp_id] - left_extent[warp_id]; + d_hsp[hid].query_start = query_loc[warp_id] - left_extent[warp_id]; + d_hsp[hid].len = extent[warp_id]; + if(entropy[warp_id] > 0) + d_hsp[hid].score = total_score[warp_id]*entropy[warp_id]; + d_done[hid-start_index] = 1; + } + else{ + d_hsp[hid].ref_start = ref_loc[warp_id]; + d_hsp[hid].query_start = query_loc[warp_id]; + d_hsp[hid].len = 0; + d_hsp[hid].score = 0; + d_done[hid-start_index] = 0; + } + } + } + __syncwarp(); + } +} \ No newline at end of file diff --git a/cudaextender/src/ungapped_xdrop.cpp b/cudaextender/src/ungapped_xdrop.cpp index e69de29bb..04a784050 100644 --- a/cudaextender/src/ungapped_xdrop.cpp +++ b/cudaextender/src/ungapped_xdrop.cpp @@ -0,0 +1,104 @@ +/* +* Copyright 2019-2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#include +#include +#include "ungapped_xdrop.hpp" +#include "ungapped_xdrop_kernels.cuh" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ + +using namespace cudautils; + +UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id) + : h_sub_mat_(h_sub_mat) + , sub_mat_dim_(sub_mat_dim) + , xdrop_threshold_(xdrop_threshold) + , no_entropy_(no_entropy) + , stream_(stream) + , device_id_(device_id) +{ + // Calculate the max limits on the number of extensions we can do on + // this GPU + cudaDeviceProp device_prop; + cudaGetDeviceProperties(&device_prop, device_id_); + const float global_mem_gb = static_cast(device_prop.totalGlobalMem / 1073741824.0f); + const int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures + max_ungapped_extensions_ = static_cast(global_mem_gb * max_ungapped_per_gb); + + // Switch to device for copying over initial structures + scoped_device_switch dev(device_id_); + + // Allocate space on device for scoring matrix and + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_sub_mat_, sub_mat_dim_ * sizeof(int32_t))); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, max_ungapped_extensions_ * sizeof(int32_t))); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_tmp_hsp_, max_ungapped_extensions_ * sizeof(ScoredSegmentPair))); + + // FIXME - Pinned host memory registration for proper async behavior + device_copy_n(h_sub_mat_, sub_mat_dim_ * sizeof(int32_t), d_sub_mat_, stream_); + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_, 0, max_ungapped_extensions_ * sizeof(int32_t), stream_)); + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_hsp_, 0, max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); +} + +StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, const char* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) +{ + // Switch to configured GPU + scoped_device_switch dev(device_id_); + + int32_t curr_num_pairs = 0; + int32_t num_anchors = 0; + for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += max_ungapped_extensions_) + { + curr_num_pairs = std::min(max_ungapped_extensions_, num_seed_pairs - seed_pair_start); + // TODO- Extricate the kernel params? + find_high_scoring_segment_pairs<<<1024, 128, stream_>>>(d_query, query_length, d_target, target_length, d_sub_mat_, no_entropy_, xdrop_threshold_, score_threshold, curr_num_pairs, d_seed_pairs, seed_pair_start, d_scored_segment_pairs, d_done); + thrust::device_ptr d_done_dev_ptr = thrust::device_pointer_cast(d_done_scratch); + // TODO- Make thrust use caching allocator or change kernel + thrust::inclusive_scan(thrust::cuda::par.on(stream_), d_done_dev_ptr, d_done_dev_ptr + curr_num_pairs, d_done_dev_ptr); + device_copy_n((void*)(d_done[gpu_id]+curr_num_hits-1), sizeof(int32_t), &num_anchors, stream_); + // TODO- Make async + cudaStreamSynchronize(stream_); + if(num_anchors > 0){ + compress_output<<<1024, 1024, stream_>>>(d_done_, seed_pair_start, scored_segment_pairs_, d_tmp_hsp_, curr_num_pairs); // TODO- Need configurability for kernel? + + thrust::stable_sort(thrust::cuda::par.on(stream_), d_tmp_hsp_vec[gpu_id].begin(), d_tmp_hsp_vec[gpu_id].begin()+num_anchors, hspComp()); + thrust::device_vector::iterator result_end = thrust::unique_copy(thrust::cuda::par.on(stream_), d_tmp_hsp_vec[gpu_id].begin(), d_tmp_hsp_vec[gpu_id].begin()+num_anchors, d_hsp_vec[gpu_id].begin()+total_anchors, hspEqual()); + num_anchors = thrust::distance(thrust::cuda::par.on(stream_), d_hsp_vec[gpu_id].begin()+total_anchors, result_end), num_anchors; + total_anchors += num_anchors; + } + } + return success; +} + +} // namespace cudaextender + +} // namespace genomeworks + +} // namespace claraparabricks \ No newline at end of file diff --git a/cudaextender/src/ungapped_xdrop.hpp b/cudaextender/src/ungapped_xdrop.hpp index 19feb47c9..628db13be 100644 --- a/cudaextender/src/ungapped_xdrop.hpp +++ b/cudaextender/src/ungapped_xdrop.hpp @@ -30,7 +30,7 @@ namespace cudaextender class UngappedXDrop : public Extender { public: - UngappedXDrop(int32_t* h_sub_mat, bool no_entropy = false, cudaStream_t stream = 0, int32_t device_id = 0); + UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id); ~UngappedXDrop() override; StatusType extend_async(const char* h_query, int32_t query_length, @@ -51,10 +51,16 @@ class UngappedXDrop : public Extender private: int32_t* h_sub_mat_; + int32_t sub_mat_dim_; // Assume matrix is square + int32_t xdrop_threshold_; + bool no_entropy_; cudaStream_t stream_; int32_t device_id_; - bool no_entropy_; std::vector scored_segment_pairs_; + int32_t max_ungapped_extensions_; + int32_t* d_sub_mat_; // Pointer to device substitution matrix + int32_t* d_done_; // TODO- Rename scratch space + ScoredSegmentPair* d_tmp_hsp_; // TODO- Rename Scratch space 2 }; } // namespace cudaextender diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh new file mode 100644 index 000000000..df7d36736 --- /dev/null +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -0,0 +1,24 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#pragma once + +#include +#include "ungapped_xdrop.hpp" + +/ extend the hits to a segment by ungapped x-drop method, adjust low-scoring +// segment scores based on entropy factor, compare resulting segment scores +// to hspthresh and update the d_hsp and d_done vectors +__global__ void find_hsps (const char* __restrict__ d_ref_seq, const char* __restrict__ d_query_seq, uint32_t ref_len, uint32_t query_len, int *d_sub_mat, bool noentropy, int xdrop, int hspthresh, int num_hits, seedHit* d_hit, uint32_t start_index, segment* d_hsp, uint32_t* d_done); From 4bb2e2ab502866671508ecdeaaea1ce254945de7 Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 8 Sep 2020 17:42:14 -0700 Subject: [PATCH 046/281] [cudaextender] refactor find_hsps --- cudaextender/src/ungapped_extender_kernels.cu | 414 ++++++++++-------- cudaextender/src/ungapped_xdrop_kernels.cuh | 20 +- 2 files changed, 258 insertions(+), 176 deletions(-) diff --git a/cudaextender/src/ungapped_extender_kernels.cu b/cudaextender/src/ungapped_extender_kernels.cu index 1b5695e25..02c015d47 100644 --- a/cudaextender/src/ungapped_extender_kernels.cu +++ b/cudaextender/src/ungapped_extender_kernels.cu @@ -15,40 +15,51 @@ */ #include "ungapped_xdrop_kernels.cuh" +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ // extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores -// to hspthresh and update the d_hsp and d_done vectors -__global__ -void find_hsps (const char* __restrict__ d_ref_seq, const char* __restrict__ d_query_seq, uint32_t ref_len, uint32_t query_len, int *d_sub_mat, bool noentropy, int xdrop, int hspthresh, int num_hits, seedHit* d_hit, uint32_t start_index, segment* d_hsp, uint32_t* d_done){ - - int thread_id = threadIdx.x; - int block_id = blockIdx.x; - int num_blocks = gridDim.x; - int warp_size = warpSize; - int lane_id = thread_id%warp_size; - int warp_id = (thread_id-lane_id)/warp_size; - - __shared__ uint32_t ref_loc[NUM_WARPS]; - __shared__ uint32_t query_loc[NUM_WARPS]; - __shared__ int total_score[NUM_WARPS]; - __shared__ int prev_score[NUM_WARPS]; - __shared__ int prev_max_score[NUM_WARPS]; - __shared__ uint32_t prev_max_pos[NUM_WARPS]; - __shared__ bool edge_found[NUM_WARPS]; - __shared__ bool xdrop_found[NUM_WARPS]; - __shared__ bool new_max_found[NUM_WARPS]; - __shared__ uint32_t left_extent[NUM_WARPS]; - __shared__ uint32_t extent[NUM_WARPS]; - __shared__ uint32_t tile[NUM_WARPS]; - __shared__ double entropy[NUM_WARPS]; - - int thread_score; - int max_thread_score; +// to score_threshold and update the d_hsp and d_done vectors +__global__ void find_high_score_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int32_t* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, uint32_t* d_done) +{ + + int32_t thread_id = threadIdx.x; + int32_t block_id = blockIdx.x; + int32_t num_blocks = gridDim.x; + int32_t warp_size = warpSize; + int32_t lane_id = thread_id % warp_size; + int32_t warp_id = (thread_id - lane_id) / warp_size; + constexpr int32_t num_warps = 4; // TODO - move out? + constexpr int32_t nuc = 8; // TODO - remove hardcode - pass in + constexpr int32_t nuc2 = 64; // TODO - remove hardcode + + __shared__ uint32_t ref_loc[num_warps]; + __shared__ uint32_t query_loc[num_warps]; + __shared__ int32_t total_score[num_warps]; + __shared__ int32_t prev_score[num_warps]; + __shared__ int32_t prev_max_score[num_warps]; + __shared__ uint32_t prev_max_pos[num_warps]; + __shared__ bool edge_found[num_warps]; + __shared__ bool xdrop_found[num_warps]; + __shared__ bool new_max_found[num_warps]; + __shared__ uint32_t left_extent[num_warps]; + __shared__ uint32_t extent[num_warps]; + __shared__ uint32_t tile[num_warps]; + __shared__ double entropy[num_warps]; + + int32_t thread_score; + int32_t max_thread_score; uint32_t max_pos; uint32_t temp_pos; bool xdrop_done; - int temp; + int32_t temp; short count[4]; short count_del[4]; char r_chr; @@ -57,166 +68,188 @@ void find_hsps (const char* __restrict__ d_ref_seq, const char* __restrict__ d uint32_t query_pos; uint32_t pos_offset; - __shared__ int sub_mat[NUC2]; + __shared__ int32_t sub_mat[nuc2]; - if(thread_id < NUC2){ + if (thread_id < nuc2) + { sub_mat[thread_id] = d_sub_mat[thread_id]; } __syncthreads(); - for(int hid0 = block_id*NUM_WARPS; hid0 < num_hits; hid0 += NUM_WARPS*num_blocks){ - int hid = hid0 + warp_id + start_index; + for (int32_t hid0 = block_id * num_warps; hid0 < num_seed_pairs; hid0 += num_warps * num_blocks) + { + int32_t hid = hid0 + warp_id + start_index; - if(hid < num_hits){ - if(lane_id == 0){ - ref_loc[warp_id] = d_hit[hid].ref_start; - query_loc[warp_id] = d_hit[hid].query_start; + if (hid < num_seed_pairs) + { + if (lane_id == 0) + { + ref_loc[warp_id] = d_seed_pairs[hid].target_position_in_read; + query_loc[warp_id] = d_seed_pairs[hid].query_position_in_read; total_score[warp_id] = 0; } } - else{ - if(lane_id == 0){ + else + { + if (lane_id == 0) + { - ref_loc[warp_id] = d_hit[hid0].ref_start; - query_loc[warp_id] = d_hit[hid0].query_start; + ref_loc[warp_id] = d_seed_pairs[hid0].target_position_in_read; + query_loc[warp_id] = d_seed_pairs[hid0].query_position_in_read; total_score[warp_id] = 0; } } __syncwarp(); - ////////////////////////////////////////////////////////////////// //Right extension - if(lane_id ==0){ - tile[warp_id] = 0; - xdrop_found[warp_id] = false; - edge_found[warp_id] = false; - new_max_found[warp_id] = false; - entropy[warp_id] = 1.0f; - prev_score[warp_id] = 0; + if (lane_id == 0) + { + tile[warp_id] = 0; + xdrop_found[warp_id] = false; + edge_found[warp_id] = false; + new_max_found[warp_id] = false; + entropy[warp_id] = 1.0f; + prev_score[warp_id] = 0; prev_max_score[warp_id] = -1000; - prev_max_pos[warp_id] = 0; - extent[warp_id] = 0; + prev_max_pos[warp_id] = 0; + extent[warp_id] = 0; } - count[0] = 0; - count[1] = 0; - count[2] = 0; - count[3] = 0; + count[0] = 0; + count[1] = 0; + count[2] = 0; + count[3] = 0; count_del[0] = 0; count_del[1] = 0; count_del[2] = 0; count_del[3] = 0; - max_pos = 0; + max_pos = 0; __syncwarp(); - while(!xdrop_found[warp_id] && !edge_found[warp_id]){ - pos_offset = lane_id + tile[warp_id]; - ref_pos = ref_loc[warp_id] + pos_offset; - query_pos = query_loc[warp_id] + pos_offset; + while (!xdrop_found[warp_id] && !edge_found[warp_id]) + { + pos_offset = lane_id + tile[warp_id]; + ref_pos = ref_loc[warp_id] + pos_offset; + query_pos = query_loc[warp_id] + pos_offset; thread_score = 0; - if(ref_pos < ref_len && query_pos < query_len){ - r_chr = d_ref_seq[ref_pos]; - q_chr = d_query_seq[query_pos]; - thread_score = sub_mat[r_chr*NUC+q_chr]; + if (ref_pos < target_length && query_pos < query_length) + { + r_chr = d_target[ref_pos]; + q_chr = d_query[query_pos]; + thread_score = sub_mat[r_chr * nuc + q_chr]; } __syncwarp(); - #pragma unroll - for (int offset = 1; offset < warp_size; offset = offset << 1){ + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + { temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); - if(lane_id >= offset){ + if (lane_id >= offset) + { thread_score += temp; } } - thread_score += prev_score[warp_id]; - if(thread_score > prev_max_score[warp_id]){ + if (thread_score > prev_max_score[warp_id]) + { max_thread_score = thread_score; - max_pos = pos_offset; + max_pos = pos_offset; } - else{ + else + { max_thread_score = prev_max_score[warp_id]; - max_pos = prev_max_pos[warp_id]; + max_pos = prev_max_pos[warp_id]; } __syncwarp(); #pragma unroll - for (int offset = 1; offset < warp_size; offset = offset << 1){ - temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + { + temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); - if(lane_id >= offset){ - if(temp >= max_thread_score){ + if (lane_id >= offset) + { + if (temp >= max_thread_score) + { max_thread_score = temp; - max_pos = temp_pos; + max_pos = temp_pos; } } } - xdrop_done = ((max_thread_score-thread_score) > xdrop); + xdrop_done = ((max_thread_score - thread_score) > xdrop_threshold); __syncwarp(); #pragma unroll - for (int offset = 1; offset < warp_size; offset = offset << 1){ + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + { xdrop_done |= __shfl_up_sync(0xFFFFFFFF, xdrop_done, offset); } - if(lane_id == warp_size-1){ + if (lane_id == warp_size - 1) + { - if(max_pos > prev_max_pos[warp_id]) + if (max_pos > prev_max_pos[warp_id]) new_max_found[warp_id] = true; else new_max_found[warp_id] = false; - if(xdrop_done){ + if (xdrop_done) + { total_score[warp_id] += max_thread_score; - xdrop_found[warp_id] = true; - extent[warp_id] = max_pos; + xdrop_found[warp_id] = true; + extent[warp_id] = max_pos; prev_max_pos[warp_id] = max_pos; - tile[warp_id] = max_pos; + tile[warp_id] = max_pos; } - else if(ref_pos >= ref_len || query_pos >= query_len){ + else if (ref_pos >= target_length || query_pos >= query_length) + { total_score[warp_id] += max_thread_score; - edge_found[warp_id] = true; - extent[warp_id] = max_pos; + edge_found[warp_id] = true; + extent[warp_id] = max_pos; prev_max_pos[warp_id] = max_pos; - tile[warp_id] = max_pos; + tile[warp_id] = max_pos; } - else{ - prev_score[warp_id] = thread_score; + else + { + prev_score[warp_id] = thread_score; prev_max_score[warp_id] = max_thread_score; - prev_max_pos[warp_id] = max_pos; - tile[warp_id]+= warp_size; + prev_max_pos[warp_id] = max_pos; + tile[warp_id] += warp_size; } } __syncwarp(); - if(new_max_found[warp_id]){ - for(int i = 0; i < 4; i++){ - count[i] = count[i] + count_del[i]; + if (new_max_found[warp_id]) + { + for (int32_t i = 0; i < 4; i++) + { + count[i] = count[i] + count_del[i]; count_del[i] = 0; } } __syncwarp(); - if(r_chr == q_chr){ - if(pos_offset <= prev_max_pos[warp_id]){ + if (r_chr == q_chr) + { + if (pos_offset <= prev_max_pos[warp_id]) + { count[r_chr] += 1; } - else{ + else + { count_del[r_chr] += 1; } } __syncwarp(); - } __syncwarp(); @@ -224,174 +257,207 @@ void find_hsps (const char* __restrict__ d_ref_seq, const char* __restrict__ d //////////////////////////////////////////////////////////////// //Left extension - if(lane_id ==0){ - tile[warp_id] = 0; - xdrop_found[warp_id] = false; - edge_found[warp_id] = false; - new_max_found[warp_id] = false; - prev_score[warp_id] = 0; + if (lane_id == 0) + { + tile[warp_id] = 0; + xdrop_found[warp_id] = false; + edge_found[warp_id] = false; + new_max_found[warp_id] = false; + prev_score[warp_id] = 0; prev_max_score[warp_id] = -1000; - prev_max_pos[warp_id] = 0; - left_extent[warp_id] = 0; + prev_max_pos[warp_id] = 0; + left_extent[warp_id] = 0; } count_del[0] = 0; count_del[1] = 0; count_del[2] = 0; count_del[3] = 0; - max_pos = 0; + max_pos = 0; __syncwarp(); - while(!xdrop_found[warp_id] && !edge_found[warp_id]){ - pos_offset = lane_id+1+tile[warp_id]; + while (!xdrop_found[warp_id] && !edge_found[warp_id]) + { + pos_offset = lane_id + 1 + tile[warp_id]; thread_score = 0; - if(ref_loc[warp_id] >= pos_offset && query_loc[warp_id] >= pos_offset){ - ref_pos = ref_loc[warp_id] - pos_offset; - query_pos = query_loc[warp_id] - pos_offset; - r_chr = d_ref_seq[ref_pos]; - q_chr = d_query_seq[query_pos]; - thread_score = sub_mat[r_chr*NUC+q_chr]; - + if (ref_loc[warp_id] >= pos_offset && query_loc[warp_id] >= pos_offset) + { + ref_pos = ref_loc[warp_id] - pos_offset; + query_pos = query_loc[warp_id] - pos_offset; + r_chr = d_target[ref_pos]; + q_chr = d_query[query_pos]; + thread_score = sub_mat[r_chr * nuc + q_chr]; } #pragma unroll - for (int offset = 1; offset < warp_size; offset = offset << 1){ + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + { temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); - if(lane_id >= offset){ + if (lane_id >= offset) + { thread_score += temp; } } thread_score += prev_score[warp_id]; - if(thread_score > prev_max_score[warp_id]){ + if (thread_score > prev_max_score[warp_id]) + { max_thread_score = thread_score; - max_pos = pos_offset; + max_pos = pos_offset; } - else{ + else + { max_thread_score = prev_max_score[warp_id]; - max_pos = prev_max_pos[warp_id]; + max_pos = prev_max_pos[warp_id]; } __syncwarp(); #pragma unroll - for (int offset = 1; offset < warp_size; offset = offset << 1){ - temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + { + temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); - if(lane_id >= offset){ - if(temp >= max_thread_score){ + if (lane_id >= offset) + { + if (temp >= max_thread_score) + { max_thread_score = temp; - max_pos = temp_pos; + max_pos = temp_pos; } } } - xdrop_done = ((max_thread_score-thread_score) > xdrop); + xdrop_done = ((max_thread_score - thread_score) > xdrop_threshold); __syncwarp(); #pragma unroll - for (int offset = 1; offset < warp_size; offset = offset << 1){ + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + { xdrop_done |= __shfl_up_sync(0xFFFFFFFF, xdrop_done, offset); } - if(lane_id == warp_size-1){ + if (lane_id == warp_size - 1) + { - if(max_pos > prev_max_pos[warp_id]) + if (max_pos > prev_max_pos[warp_id]) new_max_found[warp_id] = true; else new_max_found[warp_id] = false; - if(xdrop_done){ - total_score[warp_id]+=max_thread_score; + if (xdrop_done) + { + total_score[warp_id] += max_thread_score; xdrop_found[warp_id] = true; left_extent[warp_id] = max_pos; extent[warp_id] += left_extent[warp_id]; prev_max_pos[warp_id] = max_pos; - tile[warp_id] = max_pos; + tile[warp_id] = max_pos; } - else if(ref_loc[warp_id] < pos_offset || query_loc[warp_id] < pos_offset){ - total_score[warp_id]+=max_thread_score; - edge_found[warp_id] = true; + else if (ref_loc[warp_id] < pos_offset || query_loc[warp_id] < pos_offset) + { + total_score[warp_id] += max_thread_score; + edge_found[warp_id] = true; left_extent[warp_id] = max_pos; extent[warp_id] += left_extent[warp_id]; prev_max_pos[warp_id] = max_pos; - tile[warp_id] = max_pos; + tile[warp_id] = max_pos; } - else{ - prev_score[warp_id] = thread_score; + else + { + prev_score[warp_id] = thread_score; prev_max_score[warp_id] = max_thread_score; - prev_max_pos[warp_id] = max_pos; - tile[warp_id]+=warp_size; + prev_max_pos[warp_id] = max_pos; + tile[warp_id] += warp_size; } } __syncwarp(); - if(new_max_found[warp_id]){ - for(int i = 0; i < 4; i++){ - count[i] = count[i] + count_del[i]; + if (new_max_found[warp_id]) + { + for (int32_t i = 0; i < 4; i++) + { + count[i] = count[i] + count_del[i]; count_del[i] = 0; } } __syncwarp(); - if(r_chr == q_chr){ - if(pos_offset <= prev_max_pos[warp_id]){ + if (r_chr == q_chr) + { + if (pos_offset <= prev_max_pos[warp_id]) + { count[r_chr] += 1; } - else{ + else + { count_del[r_chr] += 1; } } __syncwarp(); - } ////////////////////////////////////////////////////////////////// - if(total_score[warp_id] >= hspthresh && total_score[warp_id] <= 3*hspthresh && !noentropy){ - for(int i = 0; i < 4; i++){ + if (total_score[warp_id] >= score_threshold && total_score[warp_id] <= 3 * score_threshold && !no_entropy) + { + for (int32_t i = 0; i < 4; i++) + { #pragma unroll - for (int offset = 1; offset < warp_size; offset = offset << 1){ + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + { count[i] += __shfl_up_sync(0xFFFFFFFF, count[i], offset); } } __syncwarp(); - if(lane_id == warp_size-1 && ((count[0]+count[1]+count[2]+count[3]) >= 20)){ + if (lane_id == warp_size - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) + { entropy[warp_id] = 0.f; - for(int i = 0; i < 4; i++){ - entropy[warp_id] += ((double) count[i])/((double) (extent[warp_id]+1)) * ((count[i] != 0) ? log(((double) count[i]) / ((double) (extent[warp_id]+1))): 0.f); + for (int32_t i = 0; i < 4; i++) + { + entropy[warp_id] += ((double)count[i]) / ((double)(extent[warp_id] + 1)) * ((count[i] != 0) ? log(((double)count[i]) / ((double)(extent[warp_id] + 1))) : 0.f); } - entropy[warp_id] = -entropy[warp_id]/log(4.0f); + entropy[warp_id] = -entropy[warp_id] / log(4.0f); } } __syncwarp(); ////////////////////////////////////////////////////////////////// - if(hid < num_hits){ - if(lane_id == 0){ - - if( ((int) (((float) total_score[warp_id]) * entropy[warp_id])) >= hspthresh){ - d_hsp[hid].ref_start = ref_loc[warp_id] - left_extent[warp_id]; - d_hsp[hid].query_start = query_loc[warp_id] - left_extent[warp_id]; - d_hsp[hid].len = extent[warp_id]; - if(entropy[warp_id] > 0) - d_hsp[hid].score = total_score[warp_id]*entropy[warp_id]; - d_done[hid-start_index] = 1; + if (hid < num_seed_pairs) + { + if (lane_id == 0) + { + + if (((int32_t)(((float)total_score[warp_id]) * entropy[warp_id])) >= score_threshold) + { + d_scored_segment[hid].seed_pair.target_position_in_read = ref_loc[warp_id] - left_extent[warp_id]; + d_scored_segment[hid].seed_pair.query_position_in_read = query_loc[warp_id] - left_extent[warp_id]; + d_scored_segment[hid].length = extent[warp_id]; + if (entropy[warp_id] > 0) + d_scored_segment[hid].score = total_score[warp_id] * entropy[warp_id]; + d_done[hid - start_index] = 1; } - else{ - d_hsp[hid].ref_start = ref_loc[warp_id]; - d_hsp[hid].query_start = query_loc[warp_id]; - d_hsp[hid].len = 0; - d_hsp[hid].score = 0; - d_done[hid-start_index] = 0; + else + { + d_scored_segment[hid].seed_pair.target_position_in_read = ref_loc[warp_id]; + d_scored_segment[hid].seed_pair.query_position_in_read = query_loc[warp_id]; + d_scored_segment[hid].length = 0; + d_scored_segment[hid].score = 0; + d_done[hid - start_index] = 0; } } } __syncwarp(); } -} \ No newline at end of file +} + +} // namespace cudaextender + +} // namespace genomeworks + +} // namespace claraparabricks \ No newline at end of file diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index df7d36736..48dd344d1 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -18,7 +18,23 @@ #include #include "ungapped_xdrop.hpp" -/ extend the hits to a segment by ungapped x-drop method, adjust low-scoring +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ + +// extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to hspthresh and update the d_hsp and d_done vectors -__global__ void find_hsps (const char* __restrict__ d_ref_seq, const char* __restrict__ d_query_seq, uint32_t ref_len, uint32_t query_len, int *d_sub_mat, bool noentropy, int xdrop, int hspthresh, int num_hits, seedHit* d_hit, uint32_t start_index, segment* d_hsp, uint32_t* d_done); +__global__ void find_high_score_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, uint32_t* d_done); + + +} + +} + +} From 2867834e43b0f9f9fb9577a1b49cb165894de0db Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 9 Sep 2020 11:25:20 -0700 Subject: [PATCH 047/281] [cudaextender] rename function --- cudaextender/src/ungapped_extender_kernels.cu | 2 +- cudaextender/src/ungapped_xdrop.cpp | 2 +- cudaextender/src/ungapped_xdrop_kernels.cuh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cudaextender/src/ungapped_extender_kernels.cu b/cudaextender/src/ungapped_extender_kernels.cu index 02c015d47..df786ccb3 100644 --- a/cudaextender/src/ungapped_extender_kernels.cu +++ b/cudaextender/src/ungapped_extender_kernels.cu @@ -27,7 +27,7 @@ namespace cudaextender // extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to score_threshold and update the d_hsp and d_done vectors -__global__ void find_high_score_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int32_t* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, uint32_t* d_done) +__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int32_t* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, uint32_t* d_done) { int32_t thread_id = threadIdx.x; diff --git a/cudaextender/src/ungapped_xdrop.cpp b/cudaextender/src/ungapped_xdrop.cpp index 04a784050..7d5038509 100644 --- a/cudaextender/src/ungapped_xdrop.cpp +++ b/cudaextender/src/ungapped_xdrop.cpp @@ -79,7 +79,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length curr_num_pairs = std::min(max_ungapped_extensions_, num_seed_pairs - seed_pair_start); // TODO- Extricate the kernel params? find_high_scoring_segment_pairs<<<1024, 128, stream_>>>(d_query, query_length, d_target, target_length, d_sub_mat_, no_entropy_, xdrop_threshold_, score_threshold, curr_num_pairs, d_seed_pairs, seed_pair_start, d_scored_segment_pairs, d_done); - thrust::device_ptr d_done_dev_ptr = thrust::device_pointer_cast(d_done_scratch); + thrust::device_ptr d_done_dev_ptr = thrust::device_pointer_cast(d_done_scratch_); // TODO- Make thrust use caching allocator or change kernel thrust::inclusive_scan(thrust::cuda::par.on(stream_), d_done_dev_ptr, d_done_dev_ptr + curr_num_pairs, d_done_dev_ptr); device_copy_n((void*)(d_done[gpu_id]+curr_num_hits-1), sizeof(int32_t), &num_anchors, stream_); diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index 48dd344d1..e380ae449 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -30,7 +30,7 @@ namespace cudaextender // extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to hspthresh and update the d_hsp and d_done vectors -__global__ void find_high_score_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, uint32_t* d_done); +__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, uint32_t* d_done); } From f06f553e28664884ff3ed5e28241202d0cd220d1 Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Fri, 11 Sep 2020 13:48:04 -0700 Subject: [PATCH 048/281] Make IndexGPU takes IndexDescriptor and fix tests --- cudamapper/src/index.cu | 3 +-- cudamapper/src/index_gpu.cuh | 10 ++++------ cudamapper/tests/Test_CudamapperIndexGPU.cu | 6 ++++-- cudamapper/tests/Test_CudamapperMatcherGPU.cu | 8 ++++---- cudamapper/tests/mock_index.cuh | 3 +-- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/cudamapper/src/index.cu b/cudamapper/src/index.cu index 1e9cf0564..6c0ff9f92 100644 --- a/cudamapper/src/index.cu +++ b/cudamapper/src/index.cu @@ -40,8 +40,7 @@ std::unique_ptr Index::create_index(DefaultDeviceAllocator allocator, GW_NVTX_RANGE(profiler, "create_index"); return std::make_unique>(allocator, parser, - descriptor.first_read(), - descriptor.first_read() + descriptor.number_of_reads(), + descriptor, kmer_size, window_size, hash_representations, diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index 7b0f7a427..084b3f4d0 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -74,8 +74,7 @@ public: /// \param cuda_stream CUDA stream on which the work is to be done. Device arrays are also associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor is done IndexGPU(DefaultDeviceAllocator allocator, const io::FastaParser& parser, - const read_id_t first_read_id, - const read_id_t past_the_last_read_id, + const IndexDescriptor& descriptor, const std::uint64_t kmer_size, const std::uint64_t window_size, const bool hash_representations = true, @@ -555,14 +554,13 @@ void filter_out_most_common_representations(DefaultDeviceAllocator allocator, template IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, const io::FastaParser& parser, - const read_id_t first_read_id, - const read_id_t past_the_last_read_id, + const IndexDescriptor& descriptor, const std::uint64_t kmer_size, const std::uint64_t window_size, const bool hash_representations, const double filtering_parameter, const cudaStream_t cuda_stream) - : first_read_id_(first_read_id) + : first_read_id_(descriptor.first_read()) , kmer_size_(kmer_size) , window_size_(window_size) , number_of_reads_(0) @@ -578,7 +576,7 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, { generate_index(parser, first_read_id_, - past_the_last_read_id, + first_read_id_ + descriptor.number_of_reads(), hash_representations, filtering_parameter); diff --git a/cudamapper/tests/Test_CudamapperIndexGPU.cu b/cudamapper/tests/Test_CudamapperIndexGPU.cu index 4b27afb3e..642e80785 100644 --- a/cudamapper/tests/Test_CudamapperIndexGPU.cu +++ b/cudamapper/tests/Test_CudamapperIndexGPU.cu @@ -1334,10 +1334,12 @@ void test_function(const std::string& filename, CudaStream cuda_stream = make_cuda_stream(); + // IndexDescriptors take the first_read and num_reads, get num_reads from past_the_last_read_id + IndexDescriptor descriptor = {first_read_id, (past_the_last_read_id - first_read_id)}; + IndexGPU index(allocator, *parser, - first_read_id, - past_the_last_read_id, + descriptor, kmer_size, window_size, false, diff --git a/cudamapper/tests/Test_CudamapperMatcherGPU.cu b/cudamapper/tests/Test_CudamapperMatcherGPU.cu index cb6eee362..618c0bfa8 100644 --- a/cudamapper/tests/Test_CudamapperMatcherGPU.cu +++ b/cudamapper/tests/Test_CudamapperMatcherGPU.cu @@ -639,8 +639,8 @@ TEST(TestCudamapperMatcherGPU, OneReadOneMinimizer) { DefaultDeviceAllocator allocator = create_default_device_allocator(); std::unique_ptr parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/gatt.fasta"); - std::unique_ptr query_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_numseqences()), 4, 1); - std::unique_ptr target_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_numseqences()), 4, 1); + std::unique_ptr query_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); + std::unique_ptr target_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); MatcherGPU matcher(allocator, *query_index, *target_index); thrust::host_vector anchors(matcher.anchors().size()); @@ -652,8 +652,8 @@ TEST(TestCudamapperMatcherGPU, AtLeastOneIndexEmpty) { DefaultDeviceAllocator allocator = create_default_device_allocator(); std::unique_ptr parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/gatt.fasta"); - std::unique_ptr index_full = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_numseqences()), 4, 1); - std::unique_ptr index_empty = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_numseqences()), 5, 1); // kmer longer than read + std::unique_ptr index_full = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); + std::unique_ptr index_empty = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 5, 1); // kmer longer than read { MatcherGPU matcher(allocator, *index_full, *index_empty); diff --git a/cudamapper/tests/mock_index.cuh b/cudamapper/tests/mock_index.cuh index da89eb782..4515b3a0d 100644 --- a/cudamapper/tests/mock_index.cuh +++ b/cudamapper/tests/mock_index.cuh @@ -39,8 +39,7 @@ public: MockIndex(DefaultDeviceAllocator allocator) : IndexGPU(allocator, *genomeworks::io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/gatt.fasta"), - 0, - 0, + IndexDescriptor(0, 0), 0, 0, true) From 701b404f17b0452b1a35fe05e64ba4c5d3ce9bee Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 2 Sep 2020 18:36:35 -0400 Subject: [PATCH 049/281] [cudapoa] added option -l to API, this determines maximum graph length --- .../genomeworks/cudapoa/batch.hpp | 3 ++- .../genomeworks/cudapoa/utils.hpp | 1 + cudapoa/src/application_parameters.cpp | 9 ++++++++- cudapoa/src/application_parameters.hpp | 1 + cudapoa/src/batch.cu | 17 ++++++++--------- cudapoa/src/main.cpp | 1 + cudapoa/src/utils.cu | 10 +++++++--- 7 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp index 3b8b416a2..0d8b56c6d 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp @@ -76,7 +76,8 @@ struct BatchConfig BandMode band_mode; /// constructor- set upper limit parameters based on max_seq_sz and band_width - BatchConfig(int32_t max_seq_sz = 1024, int32_t max_seq_per_poa = 100, int32_t band_width = 256, BandMode banding = BandMode::full_band, float adapive_storage_factor = 2.0); + BatchConfig(int32_t max_seq_sz = 1024, int32_t max_seq_per_poa = 100, int32_t band_width = 256, BandMode banding = BandMode::full_band, + float adapive_storage_factor = 2.0, float graph_length_factor = 3.0); /// constructor- set all parameters separately BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t max_nodes_per_w, diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/utils.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/utils.hpp index 9ed9b49c4..af651d389 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/utils.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/utils.hpp @@ -59,6 +59,7 @@ void get_multi_batch_sizes(std::vector& list_of_batch_sizes, int32_t band_width = 256, BandMode band_mode = BandMode::adaptive_band, float adaptive_storage_factor = 2.0f, + float graph_length_factor = 3.0f, std::vector* bins_capacity = nullptr, float gpu_memory_usage_quota = 0.9, int32_t mismatch_score = -6, diff --git a/cudapoa/src/application_parameters.cpp b/cudapoa/src/application_parameters.cpp index 9860b63ea..73b5b7d48 100644 --- a/cudapoa/src/application_parameters.cpp +++ b/cudapoa/src/application_parameters.cpp @@ -40,6 +40,7 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) {"band-mode", required_argument, 0, 'b'}, {"band-width", required_argument, 0, 'w'}, {"adaptive-storage", required_argument, 0, 's'}, + {"graph-length", required_argument, 0, 'l'}, {"dot", required_argument, 0, 'd'}, {"max-groups", required_argument, 0, 'M'}, {"gpu-mem-alloc", required_argument, 0, 'R'}, @@ -50,7 +51,7 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) {"help", no_argument, 0, 'h'}, }; - std::string optstring = "i:ab:w:s:d:M:R:m:n:g:vh"; + std::string optstring = "i:ab:w:s:l:d:M:R:m:n:g:vh"; int32_t argument = 0; while ((argument = getopt_long(argc, argv, optstring.c_str(), options, nullptr)) != -1) @@ -76,6 +77,9 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) case 's': adaptive_storage = std::stof(optarg); break; + case 'l': + graph_length = std::stof(optarg); + break; case 'd': graph_output_path = std::string(optarg); break; @@ -192,6 +196,9 @@ void ApplicationParameters::help(int32_t exit_code) -s, --adaptive-storage factor to accommodate extra memory for adaptive score matrix. The factor represents ratio of adaptive-banded score matrix to static-banded score matrix [2.0])" << R"( + -l, --graph-length + factor to determine maximum length of POA graph. The factor represents ratio of graph length to maximum sequence length in POA group [3.0])" + << R"( -d, --dot output path for printing graph in DOT format [disabled])" << R"( diff --git a/cudapoa/src/application_parameters.hpp b/cudapoa/src/application_parameters.hpp index f5e1b7fc3..bc712c082 100644 --- a/cudapoa/src/application_parameters.hpp +++ b/cudapoa/src/application_parameters.hpp @@ -49,6 +49,7 @@ class ApplicationParameters int32_t match_score = 8; double gpu_mem_allocation = 0.9; float adaptive_storage = 2.0f; + float graph_length = 3.0f; private: /// \brief verifies input file formats diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index bf24667a0..a2cb96443 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -32,7 +32,7 @@ namespace cudapoa /// constructor- set other parameters based on a minimum set of input arguments BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa /*= 100*/, int32_t band_width /*= 256*/, - BandMode banding /*= BandMode::full_band*/, float adapive_storage_factor /*= 2.0*/) + BandMode banding /*= BandMode::full_band*/, float adapive_storage_factor /*= 2.0*/, float graph_length_factor /*= 3.0*/) /// ensure a 4-byte boundary alignment for any allocated buffer : max_sequence_size(max_seq_sz) , max_consensus_size(2 * max_sequence_size) @@ -43,23 +43,22 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa { if (banding == BandMode::full_band) { - max_nodes_per_graph = cudautils::align(3 * max_sequence_size); + max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); matrix_graph_dimension = cudautils::align(max_nodes_per_graph); matrix_sequence_dimension = cudautils::align(max_sequence_size); } else if (banding == BandMode::static_band) { - max_nodes_per_graph = cudautils::align(4 * max_sequence_size); - matrix_graph_dimension = cudautils::align(max_nodes_per_graph); - // 8 = CUDAPOA_BANDED_MATRIX_RIGHT_PADDING - matrix_sequence_dimension = cudautils::align(alignment_band_width + 8); + max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); + matrix_graph_dimension = cudautils::align(max_nodes_per_graph); + matrix_sequence_dimension = cudautils::align(alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); } else // BandMode::adaptive_band { - max_nodes_per_graph = cudautils::align(4 * max_sequence_size); + max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); matrix_graph_dimension = cudautils::align(max_nodes_per_graph); - // 8 = CUDAPOA_BANDED_MATRIX_RIGHT_PADDING, *2 is to reserve extra memory for cases with extended band-width - matrix_sequence_dimension = cudautils::align(adapive_storage_factor * (alignment_band_width + 8)); + // adapive_storage_factor is to reserve extra memory for cases with extended band-width + matrix_sequence_dimension = cudautils::align(adapive_storage_factor * (alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING)); } throw_on_negative(max_seq_sz, "max_sequence_size cannot be negative."); diff --git a/cudapoa/src/main.cpp b/cudapoa/src/main.cpp index 3baac575f..b49cae2ba 100644 --- a/cudapoa/src/main.cpp +++ b/cudapoa/src/main.cpp @@ -185,6 +185,7 @@ int main(int argc, char* argv[]) parameters.band_width, parameters.band_mode, parameters.adaptive_storage, + parameters.graph_length, nullptr, parameters.gpu_mem_allocation, parameters.mismatch_score, diff --git a/cudapoa/src/utils.cu b/cudapoa/src/utils.cu index 8f26ff9e5..a14a8a081 100644 --- a/cudapoa/src/utils.cu +++ b/cudapoa/src/utils.cu @@ -34,6 +34,7 @@ void get_multi_batch_sizes(std::vector& list_of_batch_sizes, int32_t band_width /*= 256*/, BandMode band_mode /*= adaptive_band*/, float adaptive_storage_factor /*= 2.0f*/, + float graph_length_factor /*= 3.0f*/, std::vector* bins_capacity /*= nullptr*/, float gpu_memory_usage_quota /*= 0.9*/, int32_t mismatch_score /*= -6*/, @@ -52,9 +53,12 @@ void get_multi_batch_sizes(std::vector& list_of_batch_sizes, { max_read_length = std::max(max_read_length, entry.length); } - max_poas[i] = BatchBlock::estimate_max_poas(BatchConfig(max_read_length, get_size(poa_groups[i]), band_width, band_mode, adaptive_storage_factor), - msa_flag, gpu_memory_usage_quota, + BatchConfig batch_size = BatchConfig(max_read_length, get_size(poa_groups[i]), band_width, band_mode, + adaptive_storage_factor, graph_length_factor); + + max_poas[i] = BatchBlock::estimate_max_poas(batch_size, msa_flag, gpu_memory_usage_quota, mismatch_score, gap_score, match_score); + max_lengths[i] = max_read_length; } @@ -118,7 +122,7 @@ void get_multi_batch_sizes(std::vector& list_of_batch_sizes, { if (bins_frequency[j] > 0) { - list_of_batch_sizes.emplace_back(bins_max_length[j], bins_num_reads[j], band_width, band_mode, adaptive_storage_factor); + list_of_batch_sizes.emplace_back(bins_max_length[j], bins_num_reads[j], band_width, band_mode, adaptive_storage_factor, graph_length_factor); list_of_groups_per_batch.push_back(bins_group_list[j]); // check if poa_groups in the following bins can be merged into the current bin for (int32_t k = j + 1; k < num_bins; k++) From c1388bd5a4a219550f1a7adb4208cba4d37ee931 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 2 Sep 2020 20:26:55 -0400 Subject: [PATCH 050/281] [cudapoa] change in BatchConfig in .pxd file to reflect previous commit changes --- pygenomeworks/genomeworks/cudapoa/cudapoa.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd b/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd index bed26a4bb..904490618 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd @@ -82,7 +82,7 @@ cdef extern from "claraparabricks/genomeworks/cudapoa/batch.hpp" namespace "clar int32_t max_sequences_per_poa BandMode band_mode - BatchConfig(int32_t, int32_t, int32_t, BandMode) + BatchConfig(int32_t, int32_t, int32_t, BandMode, float, float) BatchConfig(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, BandMode) From e925b9be938eae136220447a2af5b6e16015e485 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 2 Sep 2020 20:49:11 -0400 Subject: [PATCH 051/281] [cudapoa] added a new pointer TraceT* backtrack in AlignmentDetails. This will be used to store matrix that contains info to complete backtracking. TraceT will be either 16-bit ot 8-bit signed int. --- cudapoa/src/allocate_block.hpp | 30 ++++++++++----------- cudapoa/src/batch.cu | 48 ++++++++++++++++----------------- cudapoa/src/cudapoa_batch.cuh | 46 +++++++++++++++---------------- cudapoa/src/cudapoa_kernels.cuh | 4 +-- cudapoa/src/cudapoa_structs.cuh | 4 ++- cudapoa/src/utils.cu | 4 +-- 6 files changed, 69 insertions(+), 67 deletions(-) diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 0da1ae6c2..ab07821e8 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -50,7 +50,7 @@ namespace genomeworks namespace cudapoa { -template +template class BatchBlock { public: @@ -185,13 +185,13 @@ class BatchBlock *input_details_d_p = input_details_d; } - void get_alignment_details(AlignmentDetails** alignment_details_d_p) + void get_alignment_details(AlignmentDetails** alignment_details_d_p) { - AlignmentDetails* alignment_details_d{}; + AlignmentDetails* alignment_details_d{}; // on host - alignment_details_d = reinterpret_cast*>(&block_data_h_[offset_h_]); - offset_h_ += sizeof(AlignmentDetails); + alignment_details_d = reinterpret_cast*>(&block_data_h_[offset_h_]); + offset_h_ += sizeof(AlignmentDetails); // on device; alignment_details_d->alignment_graph = reinterpret_castalignment_graph)>(&block_data_d_[offset_d_]); @@ -338,12 +338,12 @@ class BatchBlock device_size_per_poa += (msa_flag) ? sizeof(*GraphDetails::outgoing_edges_coverage_count) * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES : 0; // graph_details_d_->outgoing_edges_coverage_count device_size_per_poa += (msa_flag) ? sizeof(*GraphDetails::node_id_to_msa_pos) * max_nodes_per_graph : 0; // graph_details_d_->node_id_to_msa_pos // for alignment - device - device_size_per_poa += sizeof(*AlignmentDetails::alignment_graph) * max_nodes_per_graph; // alignment_details_d_->alignment_graph - device_size_per_poa += sizeof(*AlignmentDetails::alignment_read) * max_nodes_per_graph; // alignment_details_d_->alignment_read - device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_starts) * max_nodes_per_graph : 0; // alignment_details_d_->band_starts - device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_widths) * max_nodes_per_graph : 0; // alignment_details_d_->band_widths - device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_head_indices) * max_nodes_per_graph : 0; // alignment_details_d_->band_head_indices - device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_max_indices) * max_nodes_per_graph : 0; // alignment_details_d_->band_max_indices + device_size_per_poa += sizeof(*AlignmentDetails::alignment_graph) * max_nodes_per_graph; // alignment_details_d_->alignment_graph + device_size_per_poa += sizeof(*AlignmentDetails::alignment_read) * max_nodes_per_graph; // alignment_details_d_->alignment_read + device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_starts) * max_nodes_per_graph : 0; // alignment_details_d_->band_starts + device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_widths) * max_nodes_per_graph : 0; // alignment_details_d_->band_widths + device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_head_indices) * max_nodes_per_graph : 0; // alignment_details_d_->band_head_indices + device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_max_indices) * max_nodes_per_graph : 0; // alignment_details_d_->band_max_indices return device_size_per_poa; } @@ -388,17 +388,17 @@ class BatchBlock sizeof_ScoreT = 4; if (use32bitSize(batch_size)) { - device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); + device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); } else { - device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); + device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); } } else { // if ScoreT is 16-bit, it's safe to assume SizeT is also 16-bit - device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); + device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); } // Compute required memory for score matrix @@ -432,7 +432,7 @@ class BatchBlock host_size_fixed += sizeof(GraphDetails); // graph_details_h_ host_size_fixed += sizeof(GraphDetails); // graph_details_d_ // for alignment - host - host_size_fixed += sizeof(AlignmentDetails); // alignment_details_d_ + host_size_fixed += sizeof(AlignmentDetails); // alignment_details_d_ return std::make_tuple(host_size_fixed, device_size_fixed, host_size_per_poa, device_size_per_poa); } diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index a2cb96443..5e1ceed3c 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -115,38 +115,38 @@ std::unique_ptr create_batch(int32_t device_id, { if (use32bitSize(batch_size)) { - return std::make_unique>(device_id, - stream, - max_mem, - output_mask, - batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + return std::make_unique>(device_id, + stream, + max_mem, + output_mask, + batch_size, + (int32_t)gap_score, + (int32_t)mismatch_score, + (int32_t)match_score); } else { - return std::make_unique>(device_id, - stream, - max_mem, - output_mask, - batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + return std::make_unique>(device_id, + stream, + max_mem, + output_mask, + batch_size, + (int32_t)gap_score, + (int32_t)mismatch_score, + (int32_t)match_score); } } else { // if ScoreT is 16-bit, then it's safe to assume SizeT is 16-bit - return std::make_unique>(device_id, - stream, - max_mem, - output_mask, - batch_size, - gap_score, - mismatch_score, - match_score); + return std::make_unique>(device_id, + stream, + max_mem, + output_mask, + batch_size, + gap_score, + mismatch_score, + match_score); } } diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index c6e938db2..c73b7656a 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -59,7 +59,7 @@ namespace cudapoa /// \class /// Batched GPU CUDA POA object -template +template class CudapoaBatch : public Batch { public: @@ -75,10 +75,10 @@ public: , match_score_(match_score) , banded_alignment_(batch_size.band_mode == BandMode::static_band) , adaptive_banded_(batch_size.band_mode == BandMode::adaptive_band) - , batch_block_(new BatchBlock(device_id, - max_gpu_mem, - output_mask, - batch_size_)) + , batch_block_(new BatchBlock(device_id, + max_gpu_mem, + output_mask, + batch_size_)) , max_poas_(batch_block_->get_max_poas()) { // Set CUDA device @@ -178,20 +178,20 @@ public: std::string msg = " Launching kernel for " + std::to_string(poa_count_) + " on device "; print_batch_debug_message(msg); - generatePOA(output_details_d_, - input_details_d_, - poa_count_, - stream_, - alignment_details_d_, - graph_details_d_, - gap_score_, - mismatch_score_, - match_score_, - banded_alignment_, - adaptive_banded_, - max_sequences_per_poa_, - output_mask_, - batch_size_); + generatePOA(output_details_d_, + input_details_d_, + poa_count_, + stream_, + alignment_details_d_, + graph_details_d_, + gap_score_, + mismatch_score_, + match_score_, + banded_alignment_, + adaptive_banded_, + max_sequences_per_poa_, + output_mask_, + batch_size_); msg = " Launched kernel on device "; print_batch_debug_message(msg); @@ -602,7 +602,7 @@ protected: InputDetails* input_details_h_; // Device buffer struct for alignment details - AlignmentDetails* alignment_details_d_; + AlignmentDetails* alignment_details_d_; // Device buffer struct for graph details GraphDetails* graph_details_d_; @@ -635,7 +635,7 @@ protected: bool variable_band_ = false; // Pointer of a seperate class BatchBlock that implements details on calculating and allocating the memory for each batch - std::unique_ptr> batch_block_; + std::unique_ptr> batch_block_; // Maximum POAs to process in batch. int32_t max_poas_ = 0; @@ -645,8 +645,8 @@ public: static int32_t batches; }; -template -int32_t CudapoaBatch::batches = 0; +template +int32_t CudapoaBatch::batches = 0; /// \} diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index aaa48c1e3..606982951 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -423,12 +423,12 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) } } -template +template void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, genomeworks::cudapoa::InputDetails* input_details_d, int32_t total_windows, cudaStream_t stream, - genomeworks::cudapoa::AlignmentDetails* alignment_details_d, + genomeworks::cudapoa::AlignmentDetails* alignment_details_d, genomeworks::cudapoa::GraphDetails* graph_details_d, int32_t gap_score, int32_t mismatch_score, diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index d2158a2f3..c52bda367 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -99,11 +99,13 @@ struct InputDetails SizeT* sequence_begin_nodes_ids; }; -template +template struct AlignmentDetails { // Device buffer for the scoring matrix for all windows. ScoreT* scores; + // Device buffer for the backtrack matrix for all windows. + TraceT* backtrack; /// Buffer for storing per row band start location in absolute score matrix for adaptive banding SizeT* band_starts; /// Buffer for storing per row band widths for adaptive banding diff --git a/cudapoa/src/utils.cu b/cudapoa/src/utils.cu index a14a8a081..462e3b814 100644 --- a/cudapoa/src/utils.cu +++ b/cudapoa/src/utils.cu @@ -56,8 +56,8 @@ void get_multi_batch_sizes(std::vector& list_of_batch_sizes, BatchConfig batch_size = BatchConfig(max_read_length, get_size(poa_groups[i]), band_width, band_mode, adaptive_storage_factor, graph_length_factor); - max_poas[i] = BatchBlock::estimate_max_poas(batch_size, msa_flag, gpu_memory_usage_quota, - mismatch_score, gap_score, match_score); + max_poas[i] = BatchBlock::estimate_max_poas(batch_size, msa_flag, gpu_memory_usage_quota, + mismatch_score, gap_score, match_score); max_lengths[i] = max_read_length; } From 1bd3a5ca4e72e5659510851ec1348a85ce29aef4 Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 3 Sep 2020 10:48:46 -0400 Subject: [PATCH 052/281] [cudapoa] added a new parameter max_pred_distance_in_banded_mode to BatchConfig --- .../claraparabricks/genomeworks/cudapoa/batch.hpp | 8 +++++--- cudapoa/src/batch.cu | 10 +++++++--- pygenomeworks/genomeworks/cudapoa/cudapoa.pxd | 6 +++--- pygenomeworks/genomeworks/cudapoa/cudapoa.pyx | 3 ++- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp index 0d8b56c6d..c29cb9af9 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp @@ -74,14 +74,16 @@ struct BatchConfig int32_t max_sequences_per_poa; /// Banding mode: full, static, adaptive BandMode band_mode; + /// Maximum distance of predecessor node used in NW computations for static or adaptive-banded + int32_t max_pred_distance_in_banded_mode; /// constructor- set upper limit parameters based on max_seq_sz and band_width BatchConfig(int32_t max_seq_sz = 1024, int32_t max_seq_per_poa = 100, int32_t band_width = 256, BandMode banding = BandMode::full_band, - float adapive_storage_factor = 2.0, float graph_length_factor = 3.0); + float adapive_storage_factor = 2.0, float graph_length_factor = 3.0, int32_t max_pred_dist = 0); /// constructor- set all parameters separately - BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t max_nodes_per_w, - int32_t band_width, int32_t max_seq_per_poa, int32_t matrix_seq_dim, BandMode banding); + BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t max_nodes_per_w, int32_t band_width, + int32_t max_seq_per_poa, int32_t matrix_seq_dim, BandMode banding, int32_t max_pred_dist); }; /// \class Batch diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index 5e1ceed3c..9807ea1ca 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -32,7 +32,8 @@ namespace cudapoa /// constructor- set other parameters based on a minimum set of input arguments BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa /*= 100*/, int32_t band_width /*= 256*/, - BandMode banding /*= BandMode::full_band*/, float adapive_storage_factor /*= 2.0*/, float graph_length_factor /*= 3.0*/) + BandMode banding /*= BandMode::full_band*/, float adapive_storage_factor /*= 2.0*/, float graph_length_factor /*= 3.0*/, + int32_t max_pred_dist /*= 0*/) /// ensure a 4-byte boundary alignment for any allocated buffer : max_sequence_size(max_seq_sz) , max_consensus_size(2 * max_sequence_size) @@ -40,6 +41,7 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa , alignment_band_width(cudautils::align(band_width)) , max_sequences_per_poa(max_seq_per_poa) , band_mode(banding) + , max_pred_distance_in_banded_mode(max_pred_dist > 0 ? max_pred_dist : cudautils::align(band_width)) { if (banding == BandMode::full_band) { @@ -71,8 +73,8 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa } /// constructor- set all parameters separately -BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t max_nodes_per_w, - int32_t band_width, int32_t max_seq_per_poa, int32_t matrix_seq_dim, BandMode banding) +BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t max_nodes_per_w, int32_t band_width, + int32_t max_seq_per_poa, int32_t matrix_seq_dim, BandMode banding, int32_t max_pred_distance) /// ensure a 4-byte boundary alignment for any allocated buffer : max_sequence_size(max_seq_sz) , max_consensus_size(max_consensus_sz) @@ -83,12 +85,14 @@ BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t m , alignment_band_width(cudautils::align(band_width)) , max_sequences_per_poa(max_seq_per_poa) , band_mode(banding) + , max_pred_distance_in_banded_mode(max_pred_distance) { throw_on_negative(max_seq_sz, "max_sequence_size cannot be negative."); throw_on_negative(max_consensus_sz, "max_consensus_size cannot be negative."); throw_on_negative(max_nodes_per_w, "max_nodes_per_graph cannot be negative."); throw_on_negative(max_seq_per_poa, "max_sequences_per_poa cannot be negative."); throw_on_negative(band_width, "alignment_band_width cannot be negative."); + throw_on_negative(max_pred_distance, "max_pred_distance_in_banded_mode cannot be negative."); if (max_nodes_per_graph < max_sequence_size) throw std::invalid_argument("max_nodes_per_graph should be greater than or equal to max_sequence_size."); diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd b/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd index 904490618..2d2e88145 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd @@ -82,9 +82,9 @@ cdef extern from "claraparabricks/genomeworks/cudapoa/batch.hpp" namespace "clar int32_t max_sequences_per_poa BandMode band_mode - BatchConfig(int32_t, int32_t, int32_t, BandMode, float, float) - BatchConfig(int32_t, int32_t, int32_t, - int32_t, int32_t, int32_t, BandMode) + BatchConfig(int32_t, int32_t, int32_t, BandMode, float, float, int32t_t) + BatchConfig(int32_t, int32_t, int32_t, int32_t, + int32_t, int32_t, BandMode, int32_t) ctypedef vector[Entry] Group diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx index e7051493d..cffa9769e 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx @@ -131,6 +131,7 @@ cdef class CudaPoaBatch: 2 * max_sequence_size if max_consensus_size is None else max_consensus_size cdef int32_t mx_nodes_per_w cdef int32_t matrix_seq_dim + cdef int32_t mx_pred_dist cdef BandMode batch_band_mode if (band_mode == "full_band"): batch_band_mode = BandMode.full_band @@ -150,7 +151,7 @@ cdef class CudaPoaBatch: raise RuntimeError("Unknown band_mode provided. Must be full_band/static_band/adaptive_band.") self.batch_size = make_unique[cudapoa.BatchConfig]( - mx_seq_sz, mx_consensus_sz, mx_nodes_per_w, band_width_sz, mx_seq_per_poa, matrix_seq_dim, batch_band_mode) + mx_seq_sz, mx_consensus_sz, mx_nodes_per_w, band_width_sz, mx_seq_per_poa, matrix_seq_dim, batch_band_mode, mx_pred_dist) self.batch = cudapoa.create_batch( device_id, From 5a57046be4bffc5c7678ed54a149b87c08f9a31b Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 3 Sep 2020 11:40:01 -0400 Subject: [PATCH 053/281] [cudapoa] added predecessor_distance to binary API and get_multi_batch_sizes() --- .../genomeworks/cudapoa/utils.hpp | 1 + cudapoa/src/application_parameters.cpp | 14 ++++++++++- cudapoa/src/application_parameters.hpp | 23 ++++++++++--------- cudapoa/src/main.cpp | 1 + cudapoa/src/utils.cu | 5 ++-- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/utils.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/utils.hpp index af651d389..ac23d22a2 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/utils.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/utils.hpp @@ -60,6 +60,7 @@ void get_multi_batch_sizes(std::vector& list_of_batch_sizes, BandMode band_mode = BandMode::adaptive_band, float adaptive_storage_factor = 2.0f, float graph_length_factor = 3.0f, + int32_t max_pred_distance = 0, std::vector* bins_capacity = nullptr, float gpu_memory_usage_quota = 0.9, int32_t mismatch_score = -6, diff --git a/cudapoa/src/application_parameters.cpp b/cudapoa/src/application_parameters.cpp index 73b5b7d48..8cdc302c9 100644 --- a/cudapoa/src/application_parameters.cpp +++ b/cudapoa/src/application_parameters.cpp @@ -41,6 +41,7 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) {"band-width", required_argument, 0, 'w'}, {"adaptive-storage", required_argument, 0, 's'}, {"graph-length", required_argument, 0, 'l'}, + {"pred-distance", required_argument, 0, 'H'}, {"dot", required_argument, 0, 'd'}, {"max-groups", required_argument, 0, 'M'}, {"gpu-mem-alloc", required_argument, 0, 'R'}, @@ -51,7 +52,7 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) {"help", no_argument, 0, 'h'}, }; - std::string optstring = "i:ab:w:s:l:d:M:R:m:n:g:vh"; + std::string optstring = "i:ab:w:s:l:H:d:M:R:m:n:g:vh"; int32_t argument = 0; while ((argument = getopt_long(argc, argv, optstring.c_str(), options, nullptr)) != -1) @@ -80,6 +81,9 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) case 'l': graph_length = std::stof(optarg); break; + case 'H': + predecessor_disance = std::stoi(optarg); + break; case 'd': graph_output_path = std::string(optarg); break; @@ -117,6 +121,11 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) throw std::runtime_error("band-width must be positive"); } + if (band_mode != BandMode::full_band && predecessor_disance < 0) + { + throw std::runtime_error("pred-distance must be positive"); + } + if (match_score < 0) { throw std::runtime_error("match score must be positive"); @@ -199,6 +208,9 @@ void ApplicationParameters::help(int32_t exit_code) -l, --graph-length factor to determine maximum length of POA graph. The factor represents ratio of graph length to maximum sequence length in POA group [3.0])" << R"( + -H, --pred-distance + maximum distance of predecessor nodes that are considered in Needleman-Wunsch computations for static or adaptive-banded. If 0, it will be set equal to band-width [0])" + << R"( -d, --dot output path for printing graph in DOT format [disabled])" << R"( diff --git a/cudapoa/src/application_parameters.hpp b/cudapoa/src/application_parameters.hpp index bc712c082..d2e38856f 100644 --- a/cudapoa/src/application_parameters.hpp +++ b/cudapoa/src/application_parameters.hpp @@ -39,17 +39,18 @@ class ApplicationParameters std::vector input_paths; std::string graph_output_path; - bool all_fasta = true; - bool msa = false; // consensus by default - BandMode band_mode = BandMode::adaptive_band; - int32_t band_width = 256; // Band width for banded mode - int32_t max_groups = -1; // -1 => infinite - int32_t mismatch_score = -6; - int32_t gap_score = -8; - int32_t match_score = 8; - double gpu_mem_allocation = 0.9; - float adaptive_storage = 2.0f; - float graph_length = 3.0f; + bool all_fasta = true; + bool msa = false; // consensus by default + BandMode band_mode = BandMode::adaptive_band; + int32_t band_width = 256; // Band width for banded mode + int32_t max_groups = -1; // -1 => infinite + int32_t mismatch_score = -6; + int32_t gap_score = -8; + int32_t match_score = 8; + double gpu_mem_allocation = 0.9; + float adaptive_storage = 2.0f; + float graph_length = 3.0f; + int32_t predecessor_disance = 0; private: /// \brief verifies input file formats diff --git a/cudapoa/src/main.cpp b/cudapoa/src/main.cpp index b49cae2ba..3668f65cc 100644 --- a/cudapoa/src/main.cpp +++ b/cudapoa/src/main.cpp @@ -186,6 +186,7 @@ int main(int argc, char* argv[]) parameters.band_mode, parameters.adaptive_storage, parameters.graph_length, + parameters.predecessor_disance, nullptr, parameters.gpu_mem_allocation, parameters.mismatch_score, diff --git a/cudapoa/src/utils.cu b/cudapoa/src/utils.cu index 462e3b814..ba37bf71e 100644 --- a/cudapoa/src/utils.cu +++ b/cudapoa/src/utils.cu @@ -35,6 +35,7 @@ void get_multi_batch_sizes(std::vector& list_of_batch_sizes, BandMode band_mode /*= adaptive_band*/, float adaptive_storage_factor /*= 2.0f*/, float graph_length_factor /*= 3.0f*/, + int32_t max_pred_distance /*= 0*/, std::vector* bins_capacity /*= nullptr*/, float gpu_memory_usage_quota /*= 0.9*/, int32_t mismatch_score /*= -6*/, @@ -54,7 +55,7 @@ void get_multi_batch_sizes(std::vector& list_of_batch_sizes, max_read_length = std::max(max_read_length, entry.length); } BatchConfig batch_size = BatchConfig(max_read_length, get_size(poa_groups[i]), band_width, band_mode, - adaptive_storage_factor, graph_length_factor); + adaptive_storage_factor, graph_length_factor, max_pred_distance); max_poas[i] = BatchBlock::estimate_max_poas(batch_size, msa_flag, gpu_memory_usage_quota, mismatch_score, gap_score, match_score); @@ -122,7 +123,7 @@ void get_multi_batch_sizes(std::vector& list_of_batch_sizes, { if (bins_frequency[j] > 0) { - list_of_batch_sizes.emplace_back(bins_max_length[j], bins_num_reads[j], band_width, band_mode, adaptive_storage_factor, graph_length_factor); + list_of_batch_sizes.emplace_back(bins_max_length[j], bins_num_reads[j], band_width, band_mode, adaptive_storage_factor, graph_length_factor, max_pred_distance); list_of_groups_per_batch.push_back(bins_group_list[j]); // check if poa_groups in the following bins can be merged into the current bin for (int32_t k = j + 1; k < num_bins; k++) From acf5b064d529132c73f8e3acda47a533ce04305f Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 14 Sep 2020 10:55:59 -0400 Subject: [PATCH 054/281] [cudapoa-backtracking] changed default max_pred_distance to double band-width; modified API option -H to -D --- cudapoa/src/application_parameters.cpp | 10 +++++----- cudapoa/src/batch.cu | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cudapoa/src/application_parameters.cpp b/cudapoa/src/application_parameters.cpp index 8cdc302c9..8a487f42b 100644 --- a/cudapoa/src/application_parameters.cpp +++ b/cudapoa/src/application_parameters.cpp @@ -41,7 +41,7 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) {"band-width", required_argument, 0, 'w'}, {"adaptive-storage", required_argument, 0, 's'}, {"graph-length", required_argument, 0, 'l'}, - {"pred-distance", required_argument, 0, 'H'}, + {"pred-distance", required_argument, 0, 'D'}, {"dot", required_argument, 0, 'd'}, {"max-groups", required_argument, 0, 'M'}, {"gpu-mem-alloc", required_argument, 0, 'R'}, @@ -52,7 +52,7 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) {"help", no_argument, 0, 'h'}, }; - std::string optstring = "i:ab:w:s:l:H:d:M:R:m:n:g:vh"; + std::string optstring = "i:ab:w:s:l:D:d:M:R:m:n:g:vh"; int32_t argument = 0; while ((argument = getopt_long(argc, argv, optstring.c_str(), options, nullptr)) != -1) @@ -81,7 +81,7 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) case 'l': graph_length = std::stof(optarg); break; - case 'H': + case 'D': predecessor_disance = std::stoi(optarg); break; case 'd': @@ -208,8 +208,8 @@ void ApplicationParameters::help(int32_t exit_code) -l, --graph-length factor to determine maximum length of POA graph. The factor represents ratio of graph length to maximum sequence length in POA group [3.0])" << R"( - -H, --pred-distance - maximum distance of predecessor nodes that are considered in Needleman-Wunsch computations for static or adaptive-banded. If 0, it will be set equal to band-width [0])" + -D, --pred-distance + maximum distance of predecessor nodes that are considered in Needleman-Wunsch computations for static or adaptive-banded. If 0, it will be set equal to 2 x band-width [0])" << R"( -d, --dot output path for printing graph in DOT format [disabled])" diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index 9807ea1ca..fb9d27d44 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -41,7 +41,7 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa , alignment_band_width(cudautils::align(band_width)) , max_sequences_per_poa(max_seq_per_poa) , band_mode(banding) - , max_pred_distance_in_banded_mode(max_pred_dist > 0 ? max_pred_dist : cudautils::align(band_width)) + , max_pred_distance_in_banded_mode(max_pred_dist > 0 ? max_pred_dist : 2 * cudautils::align(band_width)) { if (banding == BandMode::full_band) { From d763479bad29653500995ecf400dfa95060840c0 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 14 Sep 2020 11:42:59 -0400 Subject: [PATCH 055/281] [cudapoa-backtracking] modified estimation of max_poas based on scores matrix and backtracking matrix. --- cudapoa/src/allocate_block.hpp | 41 ++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index ab07821e8..d31e71da7 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -78,9 +78,23 @@ class BatchBlock } // Calculate max POAs possible based on available memory. - int64_t device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.matrix_graph_dimension) * sizeof(ScoreT); - max_poas_ = avail_mem / (device_size_per_poa + device_size_per_score_matrix); + int64_t device_size_per_score_matrix = 0; + int64_t device_size_per_backtracking_matrix = 0; + + if (batch_size.band_mode == BandMode::full_band) + { + device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * + static_cast(batch_size.matrix_graph_dimension) * sizeof(ScoreT); + } + else + { + device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * + static_cast(batch_size.max_pred_distance_in_banded_mode) * sizeof(ScoreT); + device_size_per_backtracking_matrix = static_cast(batch_size.matrix_sequence_dimension) * + static_cast(batch_size.matrix_graph_dimension) * sizeof(TraceT); + } + + max_poas_ = avail_mem / (device_size_per_poa + device_size_per_score_matrix + device_size_per_backtracking_matrix); // Update final sizes for block based on calculated maximum POAs. output_size_ = max_poas_ * static_cast(batch_size.max_consensus_size); @@ -401,12 +415,25 @@ class BatchBlock device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); } - // Compute required memory for score matrix - int64_t device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.matrix_graph_dimension) * sizeof_ScoreT; + // Compute required memory for score matrix and backtracking matrix + int64_t device_size_per_score_matrix = 0; + int64_t device_size_per_backtracking_matrix = 0; + + if (batch_size.band_mode == BandMode::full_band) + { + device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * + static_cast(batch_size.matrix_graph_dimension) * sizeof(ScoreT); + } + else + { + device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * + static_cast(batch_size.max_pred_distance_in_banded_mode) * sizeof(ScoreT); + device_size_per_backtracking_matrix = static_cast(batch_size.matrix_sequence_dimension) * + static_cast(batch_size.matrix_graph_dimension) * sizeof(TraceT); + } // Calculate max POAs possible based on available memory. - int64_t max_poas = mem_per_batch / (device_size_per_poa + device_size_per_score_matrix); + int64_t max_poas = mem_per_batch / (device_size_per_poa + device_size_per_score_matrix + device_size_per_backtracking_matrix); return max_poas; } From 20fc68d1eec39d2546982f5c0f4452a5e90a520f Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 3 Sep 2020 12:21:43 -0400 Subject: [PATCH 056/281] [cudapoa] added use16bitTrace() to determine type size of TraceT --- cudapoa/src/allocate_block.hpp | 7 +-- cudapoa/src/batch.cu | 83 ++++++++++++++++++++++++++-------- cudapoa/src/cudapoa_limits.hpp | 6 +++ 3 files changed, 73 insertions(+), 23 deletions(-) diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index d31e71da7..6f284d8a6 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -395,6 +395,7 @@ class BatchBlock size_t mem_per_batch = memory_usage_quota * free; // Using memory_usage_quota of GPU available memory for cudapoa batch. int64_t sizeof_ScoreT = 2; + int64_t sizeof_TraceT = use16bitTrace(batch_size) ? 2 : 1; int64_t device_size_per_poa = 0; if (use32bitScore(batch_size, gap_score, mismatch_score, match_score)) @@ -422,14 +423,14 @@ class BatchBlock if (batch_size.band_mode == BandMode::full_band) { device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.matrix_graph_dimension) * sizeof(ScoreT); + static_cast(batch_size.matrix_graph_dimension) * sizeof_ScoreT; } else { device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.max_pred_distance_in_banded_mode) * sizeof(ScoreT); + static_cast(batch_size.max_pred_distance_in_banded_mode) * sizeof_ScoreT; device_size_per_backtracking_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.matrix_graph_dimension) * sizeof(TraceT); + static_cast(batch_size.matrix_graph_dimension) * sizeof_TraceT; } // Calculate max POAs possible based on available memory. diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index fb9d27d44..038787bd5 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -119,39 +119,82 @@ std::unique_ptr create_batch(int32_t device_id, { if (use32bitSize(batch_size)) { - return std::make_unique>(device_id, + if (use16bitTrace(batch_size)) + { + return std::make_unique>(device_id, + stream, + max_mem, + output_mask, + batch_size, + (int32_t)gap_score, + (int32_t)mismatch_score, + (int32_t)match_score); + } + else + { + return std::make_unique>(device_id, + stream, + max_mem, + output_mask, + batch_size, + (int32_t)gap_score, + (int32_t)mismatch_score, + (int32_t)match_score); + } + } + else + { + if (use16bitTrace(batch_size)) + { + return std::make_unique>(device_id, + stream, + max_mem, + output_mask, + batch_size, + (int32_t)gap_score, + (int32_t)mismatch_score, + (int32_t)match_score); + } + else + { + return std::make_unique>(device_id, + stream, + max_mem, + output_mask, + batch_size, + (int32_t)gap_score, + (int32_t)mismatch_score, + (int32_t)match_score); + } + + } + } + else + { + // if ScoreT is 16-bit, then it's safe to assume SizeT is 16-bit + if (use16bitTrace(batch_size)) + { + return std::make_unique>(device_id, stream, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } else { - return std::make_unique>(device_id, + return std::make_unique>(device_id, stream, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } } - else - { - // if ScoreT is 16-bit, then it's safe to assume SizeT is 16-bit - return std::make_unique>(device_id, - stream, - max_mem, - output_mask, - batch_size, - gap_score, - mismatch_score, - match_score); - } } } // namespace cudapoa diff --git a/cudapoa/src/cudapoa_limits.hpp b/cudapoa/src/cudapoa_limits.hpp index bbe058561..6fd508b7e 100644 --- a/cudapoa/src/cudapoa_limits.hpp +++ b/cudapoa/src/cudapoa_limits.hpp @@ -52,6 +52,12 @@ static bool use32bitSize(const BatchConfig& batch_size) return (max_length > INT16_MAX); } +static bool use16bitTrace(const BatchConfig& batch_size) +{ + //if max_pred_distance_in_banded_mode exceeds the range represented by int8_t, then int16_t should be used + return (batch_size.max_pred_distance_in_banded_mode > INT8_MAX); +} + } // namespace cudapoa } // namespace genomeworks } // namespace claraparabricks From bceb7117d3632cce77b69871065379aad066463a Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 3 Sep 2020 15:46:41 -0400 Subject: [PATCH 057/281] [cudapoa-backtracking] started adding changes related to backtrace buffer allocation --- cudapoa/src/allocate_block.hpp | 29 ++++++++++++++++++---- cudapoa/src/batch.cu | 43 ++++++++++++++++----------------- cudapoa/src/cudapoa_batch.cuh | 27 +++++++++++---------- cudapoa/src/cudapoa_kernels.cuh | 42 ++++++++++++++++++++------------ cudapoa/src/cudapoa_nw.cuh | 8 +++--- cudapoa/src/cudapoa_structs.cuh | 4 +-- 6 files changed, 91 insertions(+), 62 deletions(-) diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 6f284d8a6..377f18192 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -60,7 +60,9 @@ class BatchBlock , output_mask_(output_mask) { scoped_device_switch dev(device_id_); - max_nodes_per_window_ = batch_size.max_nodes_per_graph; + max_nodes_per_window_ = batch_size.max_nodes_per_graph; + full_matrix_alignment_ = batch_size.band_mode == BandMode::full_band; + score_matrix_height_ = full_matrix_alignment_ ? batch_size.max_nodes_per_graph : batch_size.max_pred_distance_in_banded_mode; // calculate static and dynamic sizes of buffers needed per POA entry. int64_t host_size_fixed, device_size_fixed; @@ -212,6 +214,7 @@ class BatchBlock offset_d_ += cudautils::align(sizeof(*alignment_details_d->alignment_graph) * max_nodes_per_window_ * max_poas_); alignment_details_d->alignment_read = reinterpret_castalignment_read)>(&block_data_d_[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->alignment_read) * max_nodes_per_window_ * max_poas_); + if (variable_bands_) { alignment_details_d->band_starts = reinterpret_castband_starts)>(&block_data_d_[offset_d_]); @@ -224,10 +227,24 @@ class BatchBlock offset_d_ += cudautils::align(sizeof(*alignment_details_d->band_max_indices) * max_nodes_per_window_ * max_poas_); } - // rest of the available memory is assigned to scores buffer - alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; - alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); - *alignment_details_d_p = alignment_details_d; + if (full_matrix_alignment_) + { + // in full-band mode, where only scores buffer is used and backtrace buffer size is 0, rest of the available memory is assigned to scores buffer + alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; + alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); + alignment_details_d->backtrace = nullptr; + } + else + { + // in banded mode, we store only part of scores matrix for forward computation in NW + alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); + offset_d_ += cudautils::align(sizeof(*alignment_details_d->scores) * score_matrix_height_ * max_poas_); + // rest of the available memory is assigned to backtrace buffer + alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; + alignment_details_d->backtrace = reinterpret_castbacktrace)>(&block_data_d_[offset_d_]); + } + + *alignment_details_d_p = alignment_details_d; } void get_graph_details(GraphDetails** graph_details_d_p, GraphDetails** graph_details_h_p) @@ -491,6 +508,8 @@ class BatchBlock int64_t input_size_ = 0; int64_t output_size_ = 0; int32_t max_nodes_per_window_ = 0; + int32_t score_matrix_height_ = 0; + bool full_matrix_alignment_ = false; int32_t device_id_; // Bit field for output type diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index 038787bd5..1e870e7be 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -133,13 +133,13 @@ std::unique_ptr create_batch(int32_t device_id, else { return std::make_unique>(device_id, - stream, - max_mem, - output_mask, - batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + stream, + max_mem, + output_mask, + batch_size, + (int32_t)gap_score, + (int32_t)mismatch_score, + (int32_t)match_score); } } else @@ -158,15 +158,14 @@ std::unique_ptr create_batch(int32_t device_id, else { return std::make_unique>(device_id, - stream, - max_mem, - output_mask, - batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + stream, + max_mem, + output_mask, + batch_size, + (int32_t)gap_score, + (int32_t)mismatch_score, + (int32_t)match_score); } - } } else @@ -186,13 +185,13 @@ std::unique_ptr create_batch(int32_t device_id, else { return std::make_unique>(device_id, - stream, - max_mem, - output_mask, - batch_size, - gap_score, - mismatch_score, - match_score); + stream, + max_mem, + output_mask, + batch_size, + gap_score, + mismatch_score, + match_score); } } } diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index c73b7656a..87454646b 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -404,7 +404,7 @@ public: num_nucleotides_copied_ = 0; global_sequence_idx_ = 0; next_scores_offset_ = 0; - avail_scorebuf_mem_ = alignment_details_d_->scorebuf_alloc_size; + avail_buf_mem_ = alignment_details_d_->scorebuf_alloc_size; } protected: @@ -547,27 +547,28 @@ protected: return StatusType::success; } - // Check if seq length can fit in available scoring matrix memory. + // Check if intermediate data for seq length can fit in available scoring/backtrace buffer bool reserve_buf(int32_t max_seq_length) { - int32_t max_graph_dimension = batch_size_.matrix_graph_dimension; + int32_t matrix_height = batch_size_.matrix_graph_dimension; + int32_t matrix_width = (banded_alignment_ || adaptive_banded_) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); + // in full-band, avail_buf_mem_ is dedicated to scores matrix and in static or adaptive band modes, avail_buf_mem_ is dedicated to backtrace matrix + size_t required_size = static_cast(matrix_width) * static_cast(matrix_height); + required_size *= batch_size_.band_mode == BandMode::full_band ? sizeof(ScoreT) : sizeof(TraceT); - int32_t scores_width = (banded_alignment_ || adaptive_banded_) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); - size_t scores_size = static_cast(scores_width) * static_cast(max_graph_dimension) * sizeof(ScoreT); - - if (scores_size > avail_scorebuf_mem_) + if (required_size > avail_buf_mem_) { if (get_total_poas() == 0) { - std::cout << "Memory available " << std::fixed << std::setprecision(2) << (static_cast(avail_scorebuf_mem_)) / 1024. / 1024. / 1024.; - std::cout << "GB, Memory required " << (static_cast(scores_size)) / 1024. / 1024. / 1024.; - std::cout << "GB (sequence length " << max_seq_length << ", graph length " << max_graph_dimension << ")" << std::endl; + std::cout << "Memory available " << std::fixed << std::setprecision(2) << (static_cast(avail_buf_mem_)) / 1024. / 1024. / 1024.; + std::cout << "GB, Memory required " << (static_cast(required_size)) / 1024. / 1024. / 1024.; + std::cout << "GB (sequence length " << max_seq_length << ", graph length " << matrix_height << ")" << std::endl; } return false; } else { - avail_scorebuf_mem_ -= scores_size; + avail_buf_mem_ -= required_size; return true; } } @@ -620,8 +621,8 @@ protected: // Global sequence index. int32_t global_sequence_idx_ = 0; - // Remaining scores buffer memory available for use. - size_t avail_scorebuf_mem_ = 0; + // Remaining buffer memory available for scores matrices in case of full alignment, and for backtrace matrices in case of banded alignment + size_t avail_buf_mem_ = 0; // Temporary variable to compute the offset to scorebuf. size_t next_scores_offset_ = 0; diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 606982951..f9c10256a 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -72,7 +72,7 @@ namespace cudapoa * @param[in] mismatch_score Score for finding a mismatch in alignment * @param[in] match_score Score for finding a match in alignment */ -template +template __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) __global__ void generatePOAKernel(uint8_t* consensus_d, uint8_t* sequences_d, @@ -110,7 +110,9 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) int32_t scores_matrix_width, int32_t max_limit_consensus_size, int32_t TPB = 64, - int32_t static_band_width = 256) + int32_t static_band_width = 256, + int32_t max_pred_distance = 0, + TraceT* backtrace_d = nullptr) { // shared error indicator within a warp bool warp_error = false; @@ -137,6 +139,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) int32_t scores_width = window_details_d[window_idx].scores_width; ScoreT* scores; + TraceT* backtrace; float banded_score_matrix_size; // using float instead of int64_t to minimize register if (BM == BandMode::adaptive_band || BM == BandMode::static_band) { @@ -453,6 +456,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, // unpack alignment details ScoreT* scores = alignment_details_d->scores; + TraceT* backtrace = alignment_details_d->backtrace; SizeT* alignment_graph = alignment_details_d->alignment_graph; SizeT* alignment_read = alignment_details_d->alignment_read; @@ -494,7 +498,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, { if (static_banded) { - generatePOAKernel + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -531,11 +535,13 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_seq_dimension, batch_size.max_consensus_size, TPB, - batch_size.alignment_band_width); + batch_size.alignment_band_width, + batch_size.max_pred_distance_in_banded_mode, + backtrace); } else if (adaptive_banded) { - generatePOAKernel + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -572,11 +578,13 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_seq_dimension, batch_size.max_consensus_size, TPB, - batch_size.alignment_band_width); + batch_size.alignment_band_width, + batch_size.max_pred_distance_in_banded_mode, + backtrace); } else { - generatePOAKernel + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -612,15 +620,14 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_graph_dimension, matrix_seq_dimension, batch_size.max_consensus_size, - TPB, - batch_size.alignment_band_width); + TPB); } } else { if (static_banded) { - generatePOAKernel + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -657,11 +664,13 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_seq_dimension, batch_size.max_consensus_size, TPB, - batch_size.alignment_band_width); + batch_size.alignment_band_width, + batch_size.max_pred_distance_in_banded_mode, + backtrace); } else if (adaptive_banded) { - generatePOAKernel + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -698,11 +707,13 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_seq_dimension, batch_size.max_consensus_size, TPB, - batch_size.alignment_band_width); + batch_size.alignment_band_width, + batch_size.max_pred_distance_in_banded_mode, + backtrace); } else { - generatePOAKernel + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -738,8 +749,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_graph_dimension, matrix_seq_dimension, batch_size.max_consensus_size, - TPB, - batch_size.alignment_band_width); + TPB); } } GW_CU_CHECK_ERR(cudaPeekAtLastError()); diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index a71886209..e3dfddaad 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -358,11 +358,11 @@ __device__ __forceinline__ int32_t prev_i = 0; int32_t prev_j = 0; - // Trace back from maximum score position to generate alignment. - // Trace back is done by re-calculating the score at each cell + // backtrack from maximum score position to generate alignment. + // backtracking is done by re-calculating the score at each cell // along the path to see which preceding cell the move could have - // come from. This seems computaitonally more expensive, but doesn't - // require storing any traceback buffer during alignment. + // come from. This seems computationally more expensive, but doesn't + // require storing any backtrace buffer during alignment. int32_t loop_count = 0; while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) { diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index c52bda367..3c85186de 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -104,8 +104,8 @@ struct AlignmentDetails { // Device buffer for the scoring matrix for all windows. ScoreT* scores; - // Device buffer for the backtrack matrix for all windows. - TraceT* backtrack; + // Device buffer for the backtracking matrix for all windows. + TraceT* backtrace; /// Buffer for storing per row band start location in absolute score matrix for adaptive banding SizeT* band_starts; /// Buffer for storing per row band widths for adaptive banding From 17a9a9aad04cabb1c4182fbfb936bd3bfd1dbb5e Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 3 Sep 2020 17:12:01 -0400 Subject: [PATCH 058/281] [cudapoa] adding backtrace buffer in cudapoa_kernels, wip --- cudapoa/src/cudapoa_kernels.cuh | 121 +++++++++++---------- cudapoa/src/cudapoa_nw_adaptive_banded.cuh | 12 +- cudapoa/src/cudapoa_nw_banded.cuh | 4 +- 3 files changed, 75 insertions(+), 62 deletions(-) diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index f9c10256a..ba215fb84 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -139,18 +139,24 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) int32_t scores_width = window_details_d[window_idx].scores_width; ScoreT* scores; - TraceT* backtrace; - float banded_score_matrix_size; // using float instead of int64_t to minimize register + TraceT* backtrace = backtrace_d; + float banded_buffer_size; // using float instead of int64_t to minimize register if (BM == BandMode::adaptive_band || BM == BandMode::static_band) { - banded_score_matrix_size = static_cast(scores_matrix_height) * static_cast(scores_matrix_width); - int64_t scores_offset = static_cast(banded_score_matrix_size) * static_cast(window_idx); - scores = &scores_d[scores_offset]; + // buffer size for scores, in banded we only need to store part of the scores matrix +// banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); + banded_buffer_size = static_cast(scores_matrix_height) * static_cast(scores_matrix_width); + int64_t offset = static_cast(banded_buffer_size) * static_cast(window_idx); + scores = &scores_d[offset]; + // buffer size for backtrace + banded_buffer_size = static_cast(scores_matrix_height) * static_cast(scores_matrix_width); + offset = static_cast(banded_buffer_size) * static_cast(window_idx); + backtrace = &backtrace_d[offset]; } else { - int64_t scores_offset = static_cast(window_details_d[window_idx].scores_offset) * static_cast(scores_matrix_height); - scores = &scores_d[scores_offset]; + int64_t offset = static_cast(window_details_d[window_idx].scores_offset) * static_cast(scores_matrix_height); + scores = &scores_d[offset]; } SizeT* alignment_graph = &alignment_graph_d[max_nodes_per_graph * window_idx]; @@ -252,7 +258,56 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) if (BM == BandMode::adaptive_band) { - alignment_length = runNeedlemanWunschAdaptiveBanded(nodes, + alignment_length = runNeedlemanWunschAdaptiveBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + backtrace, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + 0); + + __syncwarp(); + + if (alignment_length < -2) + { + // rerun with extended band-width + alignment_length = runNeedlemanWunschAdaptiveBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + backtrace, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + alignment_length); + __syncwarp(); + } + } + else if (BM == BandMode::static_band) + { + alignment_length = runNeedlemanWunschBanded(nodes, sorted_poa, node_id_to_pos, sequence_lengths[0], @@ -262,59 +317,13 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) sequence, seq_len, scores, - banded_score_matrix_size, + backtrace, alignment_graph, alignment_read, static_band_width, gap_score, mismatch_score, - match_score, - 0); - - __syncwarp(); - - if (alignment_length < -2) - { - // rerun with extended band-width - alignment_length = runNeedlemanWunschAdaptiveBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - banded_score_matrix_size, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score, - alignment_length); - __syncwarp(); - } - } - else if (BM == BandMode::static_band) - { - alignment_length = runNeedlemanWunschBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score); + match_score); __syncwarp(); } else // BM == BandMode::full_band diff --git a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh index c0b3570fc..9bb4d04d4 100644 --- a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh +++ b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh @@ -164,7 +164,8 @@ __device__ __forceinline__ void initialize_band_adaptive(ScoreT* scores, int32_t template + typename SizeT, + typename TraceT> __device__ __forceinline__ int32_t runNeedlemanWunschAdaptiveBanded(SeqT* nodes, @@ -177,7 +178,8 @@ __device__ __forceinline__ SeqT* read, int32_t read_length, ScoreT* scores, - float scores_size, + TraceT* backtrace, + float max_buffer_size, SizeT* alignment_graph, SizeT* alignment_read, int32_t static_band_width, @@ -237,10 +239,10 @@ __device__ __forceinline__ } //--------------------------------------------------------- - // check required memory and return error if exceeding scores_size + // check required memory and return error if exceeding max_buffer_size // using float to avoid 64-bit - float required_scores_size = static_cast(graph_count) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - if (required_scores_size > scores_size) + float required_buffer_size = static_cast(graph_count) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + if (required_buffer_size > max_buffer_size) { return -2; } diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index adbc5b295..4b02d8354 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -172,7 +172,8 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, template + typename SizeT, + typename TraceT> __device__ __forceinline__ int32_t runNeedlemanWunschBanded(SeqT* nodes, @@ -185,6 +186,7 @@ __device__ __forceinline__ SeqT* read, int32_t read_length, ScoreT* scores, + TraceT* backtrace, SizeT* alignment_graph, SizeT* alignment_read, int32_t band_width, From 93b74f6385269d8427c302cffdb97f003ea1ee38 Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 3 Sep 2020 17:32:48 -0400 Subject: [PATCH 059/281] [cudapoa] got rid of redundant matrix_graph_dimension in BatchConfig. matrix_graph_dimension and max_nodes_per_graph were always the same --- .../genomeworks/cudapoa/batch.hpp | 4 +-- cudapoa/src/allocate_block.hpp | 8 +++--- cudapoa/src/batch.cu | 12 +++----- cudapoa/src/cudapoa_batch.cuh | 2 +- cudapoa/src/cudapoa_kernels.cuh | 28 +++++++------------ cudapoa/tests/Test_CudapoaNW.cu | 4 +-- pygenomeworks/genomeworks/cudapoa/cudapoa.pxd | 5 +--- pygenomeworks/genomeworks/cudapoa/cudapoa.pyx | 10 +++---- 8 files changed, 28 insertions(+), 45 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp index c29cb9af9..5305cc6d4 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp @@ -64,8 +64,6 @@ struct BatchConfig int32_t max_consensus_size; /// Maximum number of nodes in a POA graph, one graph per window int32_t max_nodes_per_graph; - /// Maximum vertical dimension of scoring matrix, which stores POA graph - int32_t matrix_graph_dimension; /// Maximum horizontal dimension of scoring matrix, which stores part of sequences used in scores matrix computation int32_t matrix_sequence_dimension; /// Band-width used in banded alignment, it also defines minimum band-width in adaptive alignment @@ -82,7 +80,7 @@ struct BatchConfig float adapive_storage_factor = 2.0, float graph_length_factor = 3.0, int32_t max_pred_dist = 0); /// constructor- set all parameters separately - BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t max_nodes_per_w, int32_t band_width, + BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t max_nodes_per_poa, int32_t band_width, int32_t max_seq_per_poa, int32_t matrix_seq_dim, BandMode banding, int32_t max_pred_dist); }; diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 377f18192..2e19ff210 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -86,14 +86,14 @@ class BatchBlock if (batch_size.band_mode == BandMode::full_band) { device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.matrix_graph_dimension) * sizeof(ScoreT); + static_cast(batch_size.max_nodes_per_graph) * sizeof(ScoreT); } else { device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * static_cast(batch_size.max_pred_distance_in_banded_mode) * sizeof(ScoreT); device_size_per_backtracking_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.matrix_graph_dimension) * sizeof(TraceT); + static_cast(batch_size.max_nodes_per_graph) * sizeof(TraceT); } max_poas_ = avail_mem / (device_size_per_poa + device_size_per_score_matrix + device_size_per_backtracking_matrix); @@ -440,14 +440,14 @@ class BatchBlock if (batch_size.band_mode == BandMode::full_band) { device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.matrix_graph_dimension) * sizeof_ScoreT; + static_cast(batch_size.max_nodes_per_graph) * sizeof_ScoreT; } else { device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * static_cast(batch_size.max_pred_distance_in_banded_mode) * sizeof_ScoreT; device_size_per_backtracking_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.matrix_graph_dimension) * sizeof_TraceT; + static_cast(batch_size.max_nodes_per_graph) * sizeof_TraceT; } // Calculate max POAs possible based on available memory. diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index 1e870e7be..de6a71780 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -46,19 +46,16 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa if (banding == BandMode::full_band) { max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); - matrix_graph_dimension = cudautils::align(max_nodes_per_graph); matrix_sequence_dimension = cudautils::align(max_sequence_size); } else if (banding == BandMode::static_band) { max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); - matrix_graph_dimension = cudautils::align(max_nodes_per_graph); matrix_sequence_dimension = cudautils::align(alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); } else // BandMode::adaptive_band { - max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); - matrix_graph_dimension = cudautils::align(max_nodes_per_graph); + max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); // adapive_storage_factor is to reserve extra memory for cases with extended band-width matrix_sequence_dimension = cudautils::align(adapive_storage_factor * (alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING)); } @@ -73,13 +70,12 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa } /// constructor- set all parameters separately -BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t max_nodes_per_w, int32_t band_width, +BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t max_nodes_per_poa, int32_t band_width, int32_t max_seq_per_poa, int32_t matrix_seq_dim, BandMode banding, int32_t max_pred_distance) /// ensure a 4-byte boundary alignment for any allocated buffer : max_sequence_size(max_seq_sz) , max_consensus_size(max_consensus_sz) - , max_nodes_per_graph(cudautils::align(max_nodes_per_w)) - , matrix_graph_dimension(cudautils::align(max_nodes_per_graph)) + , max_nodes_per_graph(cudautils::align(max_nodes_per_poa)) , matrix_sequence_dimension(cudautils::align(matrix_seq_dim)) /// ensure 128-alignment for band_width size , alignment_band_width(cudautils::align(band_width)) @@ -89,7 +85,7 @@ BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t m { throw_on_negative(max_seq_sz, "max_sequence_size cannot be negative."); throw_on_negative(max_consensus_sz, "max_consensus_size cannot be negative."); - throw_on_negative(max_nodes_per_w, "max_nodes_per_graph cannot be negative."); + throw_on_negative(max_nodes_per_poa, "max_nodes_per_graph cannot be negative."); throw_on_negative(max_seq_per_poa, "max_sequences_per_poa cannot be negative."); throw_on_negative(band_width, "alignment_band_width cannot be negative."); throw_on_negative(max_pred_distance, "max_pred_distance_in_banded_mode cannot be negative."); diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 87454646b..63119398b 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -550,7 +550,7 @@ protected: // Check if intermediate data for seq length can fit in available scoring/backtrace buffer bool reserve_buf(int32_t max_seq_length) { - int32_t matrix_height = batch_size_.matrix_graph_dimension; + int32_t matrix_height = batch_size_.max_nodes_per_graph; int32_t matrix_width = (banded_alignment_ || adaptive_banded_) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); // in full-band, avail_buf_mem_ is dedicated to scores matrix and in static or adaptive band modes, avail_buf_mem_ is dedicated to backtrace matrix size_t required_size = static_cast(matrix_width) * static_cast(matrix_height); diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index ba215fb84..374632ef6 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -106,7 +106,6 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) uint16_t* outgoing_edges_coverage_d, uint16_t* outgoing_edges_coverage_count_d, int32_t max_nodes_per_graph, - int32_t scores_matrix_height, int32_t scores_matrix_width, int32_t max_limit_consensus_size, int32_t TPB = 64, @@ -144,18 +143,18 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) if (BM == BandMode::adaptive_band || BM == BandMode::static_band) { // buffer size for scores, in banded we only need to store part of the scores matrix -// banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); - banded_buffer_size = static_cast(scores_matrix_height) * static_cast(scores_matrix_width); + // banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); + banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); int64_t offset = static_cast(banded_buffer_size) * static_cast(window_idx); scores = &scores_d[offset]; // buffer size for backtrace - banded_buffer_size = static_cast(scores_matrix_height) * static_cast(scores_matrix_width); + banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); offset = static_cast(banded_buffer_size) * static_cast(window_idx); backtrace = &backtrace_d[offset]; } else { - int64_t offset = static_cast(window_details_d[window_idx].scores_offset) * static_cast(scores_matrix_height); + int64_t offset = static_cast(window_details_d[window_idx].scores_offset) * static_cast(max_nodes_per_graph); scores = &scores_d[offset]; } @@ -491,13 +490,12 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, uint16_t* outgoing_edges_coverage_count = graph_details_d->outgoing_edges_coverage_count; SizeT* node_id_to_msa_pos = graph_details_d->node_id_to_msa_pos; - int32_t nwindows_per_block = CUDAPOA_THREADS_PER_BLOCK / WARP_SIZE; - int32_t nblocks = (static_banded || adaptive_banded) ? total_windows : (total_windows + nwindows_per_block - 1) / nwindows_per_block; - int32_t TPB = (static_banded || adaptive_banded) ? CUDAPOA_BANDED_THREADS_PER_BLOCK : CUDAPOA_THREADS_PER_BLOCK; - int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; - int32_t matrix_graph_dimension = batch_size.matrix_graph_dimension; - int32_t matrix_seq_dimension = batch_size.matrix_sequence_dimension; - bool msa = output_mask & OutputType::msa; + int32_t nwindows_per_block = CUDAPOA_THREADS_PER_BLOCK / WARP_SIZE; + int32_t nblocks = (static_banded || adaptive_banded) ? total_windows : (total_windows + nwindows_per_block - 1) / nwindows_per_block; + int32_t TPB = (static_banded || adaptive_banded) ? CUDAPOA_BANDED_THREADS_PER_BLOCK : CUDAPOA_THREADS_PER_BLOCK; + int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; + int32_t matrix_seq_dimension = batch_size.matrix_sequence_dimension; + bool msa = output_mask & OutputType::msa; GW_CU_CHECK_ERR(cudaDeviceSetCacheConfig(cudaFuncCachePreferL1)); @@ -540,7 +538,6 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, outgoing_edges_coverage, outgoing_edges_coverage_count, max_nodes_per_graph, - matrix_graph_dimension, matrix_seq_dimension, batch_size.max_consensus_size, TPB, @@ -583,7 +580,6 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, outgoing_edges_coverage, outgoing_edges_coverage_count, max_nodes_per_graph, - matrix_graph_dimension, matrix_seq_dimension, batch_size.max_consensus_size, TPB, @@ -626,7 +622,6 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, outgoing_edges_coverage, outgoing_edges_coverage_count, max_nodes_per_graph, - matrix_graph_dimension, matrix_seq_dimension, batch_size.max_consensus_size, TPB); @@ -669,7 +664,6 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, outgoing_edges_coverage, outgoing_edges_coverage_count, max_nodes_per_graph, - matrix_graph_dimension, matrix_seq_dimension, batch_size.max_consensus_size, TPB, @@ -712,7 +706,6 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, outgoing_edges_coverage, outgoing_edges_coverage_count, max_nodes_per_graph, - matrix_graph_dimension, matrix_seq_dimension, batch_size.max_consensus_size, TPB, @@ -755,7 +748,6 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, outgoing_edges_coverage, outgoing_edges_coverage_count, max_nodes_per_graph, - matrix_graph_dimension, matrix_seq_dimension, batch_size.max_consensus_size, TPB); diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index 87856507e..586fb4a9d 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -218,7 +218,7 @@ NWAnswer testNW(const BasicNW& obj) GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.matrix_graph_dimension * batch_size.matrix_sequence_dimension * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&read, batch_size.max_sequence_size * sizeof(uint8_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_nodes_per_graph * sizeof(SizeT))); @@ -228,7 +228,7 @@ NWAnswer testNW(const BasicNW& obj) memset((void**)incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); memset((void**)outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); memset((void**)node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(SizeT)); - memset((void**)scores, 0, batch_size.matrix_graph_dimension * batch_size.matrix_sequence_dimension * sizeof(int16_t)); + memset((void**)scores, 0, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t)); //calculate edge counts on host obj.get_graph_buffers(incoming_edges, incoming_edge_count, diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd b/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd index 2d2e88145..eff1b0310 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd @@ -74,10 +74,7 @@ cdef extern from "claraparabricks/genomeworks/cudapoa/batch.hpp" namespace "clar int32_t max_sequence_size int32_t max_consensus_size int32_t max_nodes_per_graph - int32_t max_nodes_per_graph_banded - int32_t max_matrix_graph_dimension - int32_t max_matrix_graph_dimension_banded - int32_t max_matrix_sequence_dimension + int32_t matrix_sequence_dimension int32_t alignment_band_width int32_t max_sequences_per_poa BandMode band_mode diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx index cffa9769e..674cab32b 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx @@ -129,29 +129,29 @@ cdef class CudaPoaBatch: cdef int32_t mx_seq_per_poa = max_sequences_per_poa cdef int32_t mx_consensus_sz = \ 2 * max_sequence_size if max_consensus_size is None else max_consensus_size - cdef int32_t mx_nodes_per_w + cdef int32_t mx_nodes_per_poa cdef int32_t matrix_seq_dim cdef int32_t mx_pred_dist cdef BandMode batch_band_mode if (band_mode == "full_band"): batch_band_mode = BandMode.full_band - mx_nodes_per_w = 3 * max_sequence_size if max_nodes_per_graph is None else max_nodes_per_graph + mx_nodes_per_poa = 3 * max_sequence_size if max_nodes_per_graph is None else max_nodes_per_graph matrix_seq_dim = max_sequence_size if matrix_sequence_dimension is None else matrix_sequence_dimension elif (band_mode == "static_band"): batch_band_mode = BandMode.static_band - mx_nodes_per_w = 4 * max_sequence_size if max_nodes_per_graph is None else max_nodes_per_graph + mx_nodes_per_poa = 4 * max_sequence_size if max_nodes_per_graph is None else max_nodes_per_graph matrix_seq_dim = ((alignment_band_width + 8) if matrix_sequence_dimension is None else matrix_sequence_dimension) elif (band_mode == "adaptive_band"): batch_band_mode = BandMode.adaptive_band - mx_nodes_per_w = 4 * max_sequence_size if max_nodes_per_graph is None else max_nodes_per_graph + mx_nodes_per_poa = 4 * max_sequence_size if max_nodes_per_graph is None else max_nodes_per_graph matrix_seq_dim = (2*(alignment_band_width + 8) if matrix_sequence_dimension is None else matrix_sequence_dimension) else: raise RuntimeError("Unknown band_mode provided. Must be full_band/static_band/adaptive_band.") self.batch_size = make_unique[cudapoa.BatchConfig]( - mx_seq_sz, mx_consensus_sz, mx_nodes_per_w, band_width_sz, mx_seq_per_poa, matrix_seq_dim, batch_band_mode, mx_pred_dist) + mx_seq_sz, mx_consensus_sz, mx_nodes_per_poa, band_width_sz, mx_seq_per_poa, matrix_seq_dim, batch_band_mode, mx_pred_dist) self.batch = cudapoa.create_batch( device_id, From 4628d76c95d9f1a2badcaa22caa729754397012a Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 4 Sep 2020 11:31:52 -0400 Subject: [PATCH 060/281] [cudapoa] updating trace in forward computation in NW banded, WIP --- cudapoa/src/cudapoa_nw_banded.cuh | 101 +++++++++++++++++++++++------- cudapoa/src/cudapoa_structs.cuh | 18 ++++++ 2 files changed, 98 insertions(+), 21 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 4b02d8354..e1add387d 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -116,16 +116,18 @@ __device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t } } -template +template __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, - int32_t node, + int32_t pred_node, + int32_t current_node, int32_t read_pos, float gradient, int32_t band_width, int32_t max_column, ScoreT default_value, int32_t gap_score, - ScoreT4& char_profile) + ScoreT4& char_profile, + TraceT4& trace) { // The load instructions typically load data in 4B or 8B chunks. @@ -134,9 +136,9 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, // as each read of 16b issues a separate load command. // Instead it is better to load a 4B or 8B chunk into a register // using a single load inst, and then extracting necessary part of - // of the data using bit arithmatic. Also reduces register count. + // of the data using bit arithmetic. Also reduces register count. - int32_t band_start = get_band_start_for_row(node, gradient, band_width, max_column); + int32_t band_start = get_band_start_for_row(pred_node, gradient, band_width, max_column); // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds int32_t band_end = static_cast(band_start + band_width - CELLS_PER_THREAD); @@ -147,7 +149,7 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, } else { - ScoreT4* pred_scores = (ScoreT4*)get_score_ptr(scores, node, read_pos, band_start, band_width); + ScoreT4* pred_scores = (ScoreT4*)get_score_ptr(scores, pred_node, read_pos, band_start, band_width); // loads 8/16 consecutive bytes (4 ScoreT) ScoreT4 score4 = pred_scores[0]; @@ -157,14 +159,50 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, ScoreT4 score; - score.s0 = max(score4.s0 + char_profile.s0, - score4.s1 + gap_score); - score.s1 = max(score4.s1 + char_profile.s1, - score4.s2 + gap_score); - score.s2 = max(score4.s2 + char_profile.s2, - score4.s3 + gap_score); - score.s3 = max(score4.s3 + char_profile.s3, - score4_next.s0 + gap_score); + // if trace is diogonal, its value is positive and if vertical, negative + if ((score4.s0 + char_profile.s0) > (score4.s1 + gap_score)) + { + score.s0 = score4.s0 + char_profile.s0; + trace.t0 = current_node - pred_node; + } + else + { + score.s0 = score4.s1 + gap_score; + trace.t0 = -(current_node - pred_node); + } + + if ((score4.s1 + char_profile.s1) > (score4.s2 + gap_score)) + { + score.s1 = score4.s1 + char_profile.s1; + trace.t1 = current_node - pred_node; + } + else + { + score.s1 = score4.s2 + gap_score; + trace.t1 = -(current_node - pred_node); + } + + if ((score4.s2 + char_profile.s2) > (score4.s3 + gap_score)) + { + score.s2 = score4.s2 + char_profile.s2; + trace.t2 = current_node - pred_node; + } + else + { + score.s2 = score4.s3 + gap_score; + trace.t2 = -(current_node - pred_node); + } + + if ((score4.s3 + char_profile.s3) > (score4_next.s0 + gap_score)) + { + score.s3 = score4.s3 + char_profile.s3; + trace.t3 = current_node - pred_node; + } + else + { + score.s3 = score4_next.s0 + gap_score; + trace.t3 = -(current_node - pred_node); + } return score; } @@ -280,19 +318,36 @@ __device__ __forceinline__ char_profile.s2 = (graph_base == read4.r2 ? match_score : mismatch_score); char_profile.s3 = (graph_base == read4.r3 ? match_score : mismatch_score); - ScoreT4 score = get_scores(scores, pred_idx, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile); + TraceT4 trace; + ScoreT4 score = get_scores(scores, pred_idx, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile, trace); // Perform same score updates as above, but for rest of predecessors. for (int32_t p = 1; p < pred_count; p++) { int32_t pred_idx2 = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; + TraceT4 traces_4; + ScoreT4 scores_4 = get_scores(scores, pred_idx2, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile, traces_4); - ScoreT4 scores_4 = get_scores(scores, pred_idx2, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile); - - score.s0 = max(score.s0, scores_4.s0); - score.s1 = max(score.s1, scores_4.s1); - score.s2 = max(score.s2, scores_4.s2); - score.s3 = max(score.s3, scores_4.s3); + if (score.s0 < scores_4.s0) + { + score.s0 = scores_4.s0; + trace.t0 = traces_4.t0; + } + if (score.s1 < scores_4.s1) + { + score.s1 = scores_4.s1; + trace.t1 = traces_4.t1; + } + if (score.s2 < scores_4.s2) + { + score.s2 = scores_4.s2; + trace.t2 = traces_4.t2; + } + if (score.s3 < scores_4.s3) + { + score.s3 = scores_4.s3; + trace.t3 = traces_4.t3; + } } // While there are changes to the horizontal score values, keep updating the matrix. @@ -320,6 +375,7 @@ __device__ __forceinline__ if (tscore > score.s0) { score.s0 = tscore; + trace.t0 = 0; loop = true; } @@ -327,6 +383,7 @@ __device__ __forceinline__ if (tscore > score.s1) { score.s1 = tscore; + trace.t1 = 0; loop = true; } @@ -334,6 +391,7 @@ __device__ __forceinline__ if (tscore > score.s2) { score.s2 = tscore; + trace.t2 = 0; loop = true; } @@ -341,6 +399,7 @@ __device__ __forceinline__ if (tscore > score.s3) { score.s3 = tscore; + trace.t3 = 0; loop = true; } } diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index 3c85186de..ed4a54ed5 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -201,6 +201,24 @@ struct __align__(8) ScoreT4 int16_t s0, s1, s2, s3; }; +template +struct TraceT4 +{ + TraceT t0, t1, t2, t3; +}; + +template <> +struct __align__(8) TraceT4 +{ + int16_t t0, t1, t2, t3; +}; + +template <> +struct __align__(4) TraceT4 +{ + int8_t t0, t1, t2, t3; +}; + } // namespace cudapoa } // namespace genomeworks From 16462b32d94cd502a016aff0368f1fb213bf918b Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 4 Sep 2020 15:05:33 -0400 Subject: [PATCH 061/281] [cudapoa] modified updating trace to give proiority to diagonal movement. The hierarchy of priorities is: diagonal (match/mismatch) > vertical (indel) > horizontal (indel) --- cudapoa/src/cudapoa_nw_banded.cuh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index e1add387d..f295e048d 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -44,7 +44,7 @@ __device__ __forceinline__ int32_t get_band_start_for_row(int32_t row_idx, float if (end_pos > max_column) { start_pos = max_column - band_width + CELLS_PER_THREAD; - }; + } start_pos = max(start_pos, 0); @@ -61,7 +61,7 @@ __device__ __forceinline__ ScoreT* get_score_ptr(ScoreT* scores, int32_t row, in int64_t score_index = static_cast(column) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); return &scores[score_index]; -}; +} template __device__ __forceinline__ void set_score(ScoreT* scores, int32_t row, int32_t column, int32_t value, float gradient, int32_t band_width, int32_t max_column) @@ -99,6 +99,13 @@ __device__ __forceinline__ void initialize_band(ScoreT* scores, int32_t row, int } } +template +__device__ TraceT get_trace(TraceT* backtrace, int32_t row, int32_t column, int32_t band_start, int32_t band_width) +{ + int64_t trace_index = static_cast(column - band_start) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + return backtrace[trace_index]; +} + template __device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t column, float gradient, int32_t band_width, int32_t max_column, const ScoreT min_score_value) { @@ -160,7 +167,7 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, ScoreT4 score; // if trace is diogonal, its value is positive and if vertical, negative - if ((score4.s0 + char_profile.s0) > (score4.s1 + gap_score)) + if ((score4.s0 + char_profile.s0) >= (score4.s1 + gap_score)) { score.s0 = score4.s0 + char_profile.s0; trace.t0 = current_node - pred_node; @@ -171,7 +178,7 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, trace.t0 = -(current_node - pred_node); } - if ((score4.s1 + char_profile.s1) > (score4.s2 + gap_score)) + if ((score4.s1 + char_profile.s1) >= (score4.s2 + gap_score)) { score.s1 = score4.s1 + char_profile.s1; trace.t1 = current_node - pred_node; @@ -182,7 +189,7 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, trace.t1 = -(current_node - pred_node); } - if ((score4.s2 + char_profile.s2) > (score4.s3 + gap_score)) + if ((score4.s2 + char_profile.s2) >= (score4.s3 + gap_score)) { score.s2 = score4.s2 + char_profile.s2; trace.t2 = current_node - pred_node; @@ -193,7 +200,7 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, trace.t2 = -(current_node - pred_node); } - if ((score4.s3 + char_profile.s3) > (score4_next.s0 + gap_score)) + if ((score4.s3 + char_profile.s3) >= (score4_next.s0 + gap_score)) { score.s3 = score4.s3 + char_profile.s3; trace.t3 = current_node - pred_node; @@ -519,9 +526,8 @@ __device__ __forceinline__ // Check if move is horizontal. if (!pred_found && scores_ij == get_score(scores, i, j - 1, gradient, band_width, max_column, min_score_value) + gap_score) { - prev_i = i; - prev_j = j - 1; - pred_found = true; + prev_i = i; + prev_j = j - 1; } next_node_id = graph[prev_i - 1]; From 3624425abd406f754657601d8c172d1c93592f80 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Mon, 14 Sep 2020 18:29:55 -0400 Subject: [PATCH 062/281] [pygw] reorder *args position to avoid positional arguments bug --- pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx b/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx index 59a5c4418..88fa820ca 100644 --- a/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx +++ b/pygenomeworks/genomeworks/cudaaligner/cudaaligner.pyx @@ -137,11 +137,11 @@ cdef class CudaAlignerBatch: max_query_length, max_target_length, max_alignments, - *args, alignment_type="global", stream=None, device_id=0, max_device_memory_allocator_caching_size=-1, + *args, **kwargs): """Construct a CudaAligner object to run CUDA-accelerated sequence to sequence alignment across all pairs in a batch. @@ -194,11 +194,11 @@ cdef class CudaAlignerBatch: max_query_length, max_target_length, max_alignments, - *args, alignment_type="global", stream=None, device_id=0, max_device_memory_allocator_caching_size=-1, + *args, **kwargs): """Dummy implementation of __init__ function to allow for Python subclassing. From 751254563f67628c40a40d2528c1637d255c6888 Mon Sep 17 00:00:00 2001 From: ramin Date: Sun, 6 Sep 2020 20:53:37 -0400 Subject: [PATCH 063/281] [cudapoa] fixed a bug in memory allocation for backtracking buffer and slightly refactored the code --- cudapoa/src/allocate_block.hpp | 57 +++++++++++++++++----------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 2e19ff210..ca2a69cb8 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -63,6 +63,7 @@ class BatchBlock max_nodes_per_window_ = batch_size.max_nodes_per_graph; full_matrix_alignment_ = batch_size.band_mode == BandMode::full_band; score_matrix_height_ = full_matrix_alignment_ ? batch_size.max_nodes_per_graph : batch_size.max_pred_distance_in_banded_mode; + score_matrix_width_ = batch_size.matrix_sequence_dimension; // calculate static and dynamic sizes of buffers needed per POA entry. int64_t host_size_fixed, device_size_fixed; @@ -79,24 +80,21 @@ class BatchBlock throw std::runtime_error(msg); } - // Calculate max POAs possible based on available memory. - int64_t device_size_per_score_matrix = 0; - int64_t device_size_per_backtracking_matrix = 0; - + // compute device memory used for matrix buffer + // in full alignment, this is for scores matrix buffer, in static or adaptive banded alignment, it is for backtracking matrix buffer + int64_t device_size_per_matrix = static_cast(batch_size.matrix_sequence_dimension) * + static_cast(batch_size.max_nodes_per_graph); if (batch_size.band_mode == BandMode::full_band) { - device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.max_nodes_per_graph) * sizeof(ScoreT); + device_size_per_matrix *= sizeof(ScoreT); } else { - device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.max_pred_distance_in_banded_mode) * sizeof(ScoreT); - device_size_per_backtracking_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.max_nodes_per_graph) * sizeof(TraceT); + device_size_per_matrix *= sizeof(TraceT); } - max_poas_ = avail_mem / (device_size_per_poa + device_size_per_score_matrix + device_size_per_backtracking_matrix); + // Calculate max POAs possible based on available memory + max_poas_ = avail_mem / (device_size_per_poa + device_size_per_matrix); // Update final sizes for block based on calculated maximum POAs. output_size_ = max_poas_ * static_cast(batch_size.max_consensus_size); @@ -214,6 +212,12 @@ class BatchBlock offset_d_ += cudautils::align(sizeof(*alignment_details_d->alignment_graph) * max_nodes_per_window_ * max_poas_); alignment_details_d->alignment_read = reinterpret_castalignment_read)>(&block_data_d_[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->alignment_read) * max_nodes_per_window_ * max_poas_); + // in banded mode, we store only part of scores matrix for forward computation in NW + if (!full_matrix_alignment_) + { + alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); + offset_d_ += cudautils::align(sizeof(*alignment_details_d->scores) * score_matrix_width_ * score_matrix_height_ * max_poas_); + } if (variable_bands_) { @@ -236,10 +240,7 @@ class BatchBlock } else { - // in banded mode, we store only part of scores matrix for forward computation in NW - alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); - offset_d_ += cudautils::align(sizeof(*alignment_details_d->scores) * score_matrix_height_ * max_poas_); - // rest of the available memory is assigned to backtrace buffer + // in banded alignment, rest of the available memory is assigned to backtrace buffer alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; alignment_details_d->backtrace = reinterpret_castbacktrace)>(&block_data_d_[offset_d_]); } @@ -334,8 +335,10 @@ class BatchBlock static int64_t compute_device_memory_per_poa(const BatchConfig& batch_size, const bool msa_flag, const bool variable_bands = false) { - int64_t device_size_per_poa = 0; - int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; + int64_t device_size_per_poa = 0; + int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; + bool banded = batch_size.band_mode == BandMode::static_band || batch_size.band_mode == BandMode::adaptive_band; + int32_t banded_score_matrix_size = batch_size.matrix_sequence_dimension * batch_size.max_pred_distance_in_banded_mode; // for output - device device_size_per_poa += batch_size.max_consensus_size * sizeof(*OutputDetails::consensus); // output_details_d_->consensus @@ -375,7 +378,7 @@ class BatchBlock device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_widths) * max_nodes_per_graph : 0; // alignment_details_d_->band_widths device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_head_indices) * max_nodes_per_graph : 0; // alignment_details_d_->band_head_indices device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_max_indices) * max_nodes_per_graph : 0; // alignment_details_d_->band_max_indices - + device_size_per_poa += banded ? sizeof(*AlignmentDetails::scores) * banded_score_matrix_size : 0; // alignment_details_d_->scores (only for static and adaptive band modes) return device_size_per_poa; } @@ -433,25 +436,20 @@ class BatchBlock device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); } - // Compute required memory for score matrix and backtracking matrix - int64_t device_size_per_score_matrix = 0; - int64_t device_size_per_backtracking_matrix = 0; - + // Compute required memory for score or backtracking matrix + int64_t device_size_per_matrix = static_cast(batch_size.matrix_sequence_dimension) * + static_cast(batch_size.max_nodes_per_graph); if (batch_size.band_mode == BandMode::full_band) { - device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.max_nodes_per_graph) * sizeof_ScoreT; + device_size_per_matrix *= sizeof_ScoreT; } else { - device_size_per_score_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.max_pred_distance_in_banded_mode) * sizeof_ScoreT; - device_size_per_backtracking_matrix = static_cast(batch_size.matrix_sequence_dimension) * - static_cast(batch_size.max_nodes_per_graph) * sizeof_TraceT; + device_size_per_matrix *= sizeof_TraceT; } // Calculate max POAs possible based on available memory. - int64_t max_poas = mem_per_batch / (device_size_per_poa + device_size_per_score_matrix + device_size_per_backtracking_matrix); + int64_t max_poas = mem_per_batch / (device_size_per_poa + device_size_per_matrix); return max_poas; } @@ -509,6 +507,7 @@ class BatchBlock int64_t output_size_ = 0; int32_t max_nodes_per_window_ = 0; int32_t score_matrix_height_ = 0; + int32_t score_matrix_width_ = 0; bool full_matrix_alignment_ = false; int32_t device_id_; From c7d694a4029db3ae23310f659af15fae19323a78 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 14 Sep 2020 20:07:56 -0400 Subject: [PATCH 064/281] [cudapoa] storing trace in forward computation in NW banded and reading back, this commit increased reg count from 65 to 77. WIP --- cudapoa/src/cudapoa_nw_banded.cuh | 110 ++++++++++++++---------------- 1 file changed, 51 insertions(+), 59 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index f295e048d..bd385c3e0 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -422,6 +422,17 @@ __device__ __forceinline__ scores[score_index + 2L] = score.s2; scores[score_index + 3L] = score.s3; + /* + if (score_gIdx == 1 && lane_idx == 0) + { + printf(">> %d %d %d %d score_index %ld\n", trace.t0, trace.t1, trace.t2, trace.t3, score_index); + }*/ + + backtrace[score_index] = trace.t0; + backtrace[score_index + 1L] = trace.t1; + backtrace[score_index + 2L] = trace.t2; + backtrace[score_index + 3L] = trace.t3; + __syncwarp(); } } @@ -448,96 +459,77 @@ __device__ __forceinline__ } // Fill in backtrace - int32_t prev_i = 0; - int32_t prev_j = 0; - int32_t next_node_id = i > 0 ? graph[i - 1] : 0; - int32_t loop_count = 0; while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) { loop_count++; - int32_t scores_ij = get_score(scores, i, j, gradient, band_width, max_column, min_score_value); - bool pred_found = false; - // Check if move is diagonal. - if (i != 0 && j != 0) - { + //printf("i %3d j %3d \n", i, j); - int32_t node_id = next_node_id; - - int32_t match_cost = (nodes[node_id] == read[j - 1] ? match_score : mismatch_score); - uint16_t pred_count = incoming_edge_count[node_id]; - int32_t pred_i = (pred_count == 0 ? 0 : (node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]] + 1)); + if (j > 0) + { + int32_t band_start = get_band_start_for_row(i, gradient, band_width, max_column); + TraceT trace = get_trace(backtrace, i, j, band_start, band_width); - if (scores_ij == (get_score(scores, pred_i, j - 1, gradient, band_width, max_column, min_score_value) + match_cost)) + if (trace == 0) { - prev_i = pred_i; - prev_j = j - 1; - pred_found = true; + // horizontal path (indel) + alignment_graph[aligned_nodes] = -1; + alignment_read[aligned_nodes] = j - 1; + j--; } - - if (!pred_found) + else if (trace < 0) { - for (int32_t p = 1; p < pred_count; p++) - { - pred_i = (node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1); - - if (scores_ij == (get_score(scores, pred_i, j - 1, gradient, band_width, max_column, min_score_value) + match_cost)) - { - prev_i = pred_i; - prev_j = j - 1; - pred_found = true; - break; - } - } + // vertical path (indel) + alignment_graph[aligned_nodes] = graph[i - 1]; + alignment_read[aligned_nodes] = -1; + i += trace; + } + else + { + // diagonal path (match/mismatch) + alignment_graph[aligned_nodes] = graph[i - 1]; + alignment_read[aligned_nodes] = j - 1; + i -= trace; + j--; } } - - // Check if move is vertical. - if (!pred_found && i != 0) + // backtrace data is not available for the first column, find the vertical path by checking predecessors + if (j == 0 && i != 0) { + int32_t scores_ij = get_score(scores, i, j, gradient, band_width, max_column, min_score_value); int32_t node_id = graph[i - 1]; uint16_t pred_count = incoming_edge_count[node_id]; int32_t pred_i = (pred_count == 0 ? 0 : node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]] + 1); - + bool pred_found = false; if (scores_ij == get_score(scores, pred_i, j, gradient, band_width, max_column, min_score_value) + gap_score) { - prev_i = pred_i; - prev_j = j; - pred_found = true; + alignment_graph[aligned_nodes] = graph[i - 1]; + alignment_read[aligned_nodes] = -1; + i = pred_i; + pred_found = true; } - - if (!pred_found) + else { for (int32_t p = 1; p < pred_count; p++) { pred_i = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; - if (scores_ij == get_score(scores, pred_i, j, gradient, band_width, max_column, min_score_value) + gap_score) { - prev_i = pred_i; - prev_j = j; - pred_found = true; + alignment_graph[aligned_nodes] = graph[i - 1]; + alignment_read[aligned_nodes] = -1; + i = pred_i; + pred_found = true; break; } } } + if (!pred_found) + { + //ToDo throw a new error code indicating max_pred_distance is not large enough + } } - // Check if move is horizontal. - if (!pred_found && scores_ij == get_score(scores, i, j - 1, gradient, band_width, max_column, min_score_value) + gap_score) - { - prev_i = i; - prev_j = j - 1; - } - - next_node_id = graph[prev_i - 1]; - - alignment_graph[aligned_nodes] = (i == prev_i ? -1 : graph[i - 1]); - alignment_read[aligned_nodes] = (j == prev_j ? -1 : j - 1); aligned_nodes++; - - i = prev_i; - j = prev_j; } if (loop_count >= (read_length + graph_count + 2)) From 1d40f64e8601ecc9adcca62619c4b45390a15861 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 14 Sep 2020 20:17:07 -0400 Subject: [PATCH 065/281] [cudapoa-backtracking] changed get_scores() to remove ScoreT4 char_profile, which in turn reduced register count to 72. --- cudapoa/src/batch.cu | 2 ++ cudapoa/src/cudapoa_nw_banded.cuh | 39 +++++++++++++++++-------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index de6a71780..d71b0540a 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -63,6 +63,8 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa throw_on_negative(max_seq_sz, "max_sequence_size cannot be negative."); throw_on_negative(max_seq_per_poa, "max_sequences_per_poa cannot be negative."); throw_on_negative(band_width, "alignment_band_width cannot be negative."); + throw_on_negative(max_nodes_per_graph, "max_nodes_per_graph cannot be negative."); + if (alignment_band_width != band_width) { std::cerr << "Band-width should be multiple of 128. The input was changed from " << band_width << " to " << alignment_band_width << std::endl; diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index bd385c3e0..f93f54952 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -123,7 +123,7 @@ __device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t } } -template +template __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, int32_t pred_node, int32_t current_node, @@ -133,7 +133,10 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, int32_t max_column, ScoreT default_value, int32_t gap_score, - ScoreT4& char_profile, + int32_t match_score, + int32_t mismatch_score, + SeqT4 read4, + SeqT graph_base, TraceT4& trace) { @@ -165,11 +168,12 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, ScoreT4 score4_next = pred_scores[1]; ScoreT4 score; + ScoreT char_profile = (graph_base == read4.r0 ? match_score : mismatch_score); // if trace is diogonal, its value is positive and if vertical, negative - if ((score4.s0 + char_profile.s0) >= (score4.s1 + gap_score)) + if ((score4.s0 + char_profile) >= (score4.s1 + gap_score)) { - score.s0 = score4.s0 + char_profile.s0; + score.s0 = score4.s0 + char_profile; trace.t0 = current_node - pred_node; } else @@ -178,9 +182,10 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, trace.t0 = -(current_node - pred_node); } - if ((score4.s1 + char_profile.s1) >= (score4.s2 + gap_score)) + char_profile = (graph_base == read4.r1 ? match_score : mismatch_score); + if ((score4.s1 + char_profile) >= (score4.s2 + gap_score)) { - score.s1 = score4.s1 + char_profile.s1; + score.s1 = score4.s1 + char_profile; trace.t1 = current_node - pred_node; } else @@ -189,9 +194,10 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, trace.t1 = -(current_node - pred_node); } - if ((score4.s2 + char_profile.s2) >= (score4.s3 + gap_score)) + char_profile = (graph_base == read4.r2 ? match_score : mismatch_score); + if ((score4.s2 + char_profile) >= (score4.s3 + gap_score)) { - score.s2 = score4.s2 + char_profile.s2; + score.s2 = score4.s2 + char_profile; trace.t2 = current_node - pred_node; } else @@ -200,9 +206,10 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, trace.t2 = -(current_node - pred_node); } - if ((score4.s3 + char_profile.s3) >= (score4_next.s0 + gap_score)) + char_profile = (graph_base == read4.r3 ? match_score : mismatch_score); + if ((score4.s3 + char_profile) >= (score4_next.s0 + gap_score)) { - score.s3 = score4.s3 + char_profile.s3; + score.s3 = score4.s3 + char_profile; trace.t3 = current_node - pred_node; } else @@ -319,21 +326,17 @@ __device__ __forceinline__ SeqT4* d_read4 = (SeqT4*)read; SeqT4 read4 = d_read4[read_pos / CELLS_PER_THREAD]; - ScoreT4 char_profile; - char_profile.s0 = (graph_base == read4.r0 ? match_score : mismatch_score); - char_profile.s1 = (graph_base == read4.r1 ? match_score : mismatch_score); - char_profile.s2 = (graph_base == read4.r2 ? match_score : mismatch_score); - char_profile.s3 = (graph_base == read4.r3 ? match_score : mismatch_score); - TraceT4 trace; - ScoreT4 score = get_scores(scores, pred_idx, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile, trace); + ScoreT4 score = get_scores(scores, pred_idx, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, + gap_score, match_score, mismatch_score, read4, graph_base, trace); // Perform same score updates as above, but for rest of predecessors. for (int32_t p = 1; p < pred_count; p++) { int32_t pred_idx2 = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; TraceT4 traces_4; - ScoreT4 scores_4 = get_scores(scores, pred_idx2, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile, traces_4); + ScoreT4 scores_4 = get_scores(scores, pred_idx2, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, + gap_score, match_score, mismatch_score, read4, graph_base, traces_4); if (score.s0 < scores_4.s0) { From 68365618046ead75bb3d84ae2a27d2c250d77c2a Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 14 Sep 2020 20:24:25 -0400 Subject: [PATCH 066/281] [cudapoa-backtracking] some more changes in get_scores(), did not change register count, still 72. --- cudapoa/src/cudapoa_nw_banded.cuh | 128 +++++++++++++++--------------- 1 file changed, 65 insertions(+), 63 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index f93f54952..7c41fdf6f 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -124,20 +124,21 @@ __device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t } template -__device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, - int32_t pred_node, - int32_t current_node, - int32_t read_pos, - float gradient, - int32_t band_width, - int32_t max_column, - ScoreT default_value, - int32_t gap_score, - int32_t match_score, - int32_t mismatch_score, - SeqT4 read4, - SeqT graph_base, - TraceT4& trace) +__device__ __forceinline__ void get_scores(ScoreT* scores, + int32_t pred_node, + int32_t current_node, + int32_t read_pos, + float gradient, + int32_t band_width, + int32_t max_column, + ScoreT default_value, + int32_t gap_score, + int32_t match_score, + int32_t mismatch_score, + SeqT4 read4, + SeqT graph_base, + ScoreT4& score, + TraceT4& trace) { // The load instructions typically load data in 4B or 8B chunks. @@ -155,7 +156,7 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, if ((read_pos > band_end || read_pos < band_start) && read_pos != -1) { - return ScoreT4{default_value, default_value, default_value, default_value}; + return; } else { @@ -167,58 +168,80 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, // need to load the next chunk of memory as well ScoreT4 score4_next = pred_scores[1]; - ScoreT4 score; ScoreT char_profile = (graph_base == read4.r0 ? match_score : mismatch_score); // if trace is diogonal, its value is positive and if vertical, negative + // update score.s0, trace.t0 ---------- if ((score4.s0 + char_profile) >= (score4.s1 + gap_score)) { - score.s0 = score4.s0 + char_profile; - trace.t0 = current_node - pred_node; + if ((score4.s0 + char_profile) > score.s0) + { + score.s0 = score4.s0 + char_profile; + trace.t0 = current_node - pred_node; + } } else { - score.s0 = score4.s1 + gap_score; - trace.t0 = -(current_node - pred_node); + if ((score4.s1 + gap_score) > score.s0) + { + score.s0 = score4.s1 + gap_score; + trace.t0 = -(current_node - pred_node); + } } - + // update score.s1, trace.t1 ---------- char_profile = (graph_base == read4.r1 ? match_score : mismatch_score); if ((score4.s1 + char_profile) >= (score4.s2 + gap_score)) { - score.s1 = score4.s1 + char_profile; - trace.t1 = current_node - pred_node; + if ((score4.s1 + char_profile) > score.s1) + { + score.s1 = score4.s1 + char_profile; + trace.t1 = current_node - pred_node; + } } else { - score.s1 = score4.s2 + gap_score; - trace.t1 = -(current_node - pred_node); + if ((score4.s2 + gap_score) > score.s1) + { + score.s1 = score4.s2 + gap_score; + trace.t1 = -(current_node - pred_node); + } } - + // update score.s2, trace.t2 ---------- char_profile = (graph_base == read4.r2 ? match_score : mismatch_score); if ((score4.s2 + char_profile) >= (score4.s3 + gap_score)) { - score.s2 = score4.s2 + char_profile; - trace.t2 = current_node - pred_node; + if ((score4.s2 + char_profile) > score.s2) + { + score.s2 = score4.s2 + char_profile; + trace.t2 = current_node - pred_node; + } } else { - score.s2 = score4.s3 + gap_score; - trace.t2 = -(current_node - pred_node); + if ((score4.s3 + gap_score) > score.s2) + { + score.s2 = score4.s3 + gap_score; + trace.t2 = -(current_node - pred_node); + } } - + // update score.s3, trace.t3 ---------- char_profile = (graph_base == read4.r3 ? match_score : mismatch_score); if ((score4.s3 + char_profile) >= (score4_next.s0 + gap_score)) { - score.s3 = score4.s3 + char_profile; - trace.t3 = current_node - pred_node; + if ((score4.s3 + char_profile) > score.s3) + { + score.s3 = score4.s3 + char_profile; + trace.t3 = current_node - pred_node; + } } else { - score.s3 = score4_next.s0 + gap_score; - trace.t3 = -(current_node - pred_node); + if ((score4_next.s0 + gap_score) > score.s3) + { + score.s3 = score4_next.s0 + gap_score; + trace.t3 = -(current_node - pred_node); + } } - - return score; } } @@ -327,37 +350,16 @@ __device__ __forceinline__ SeqT4 read4 = d_read4[read_pos / CELLS_PER_THREAD]; TraceT4 trace; - ScoreT4 score = get_scores(scores, pred_idx, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, - gap_score, match_score, mismatch_score, read4, graph_base, trace); + ScoreT4 score = {min_score_value, min_score_value, min_score_value, min_score_value}; + get_scores(scores, pred_idx, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, + gap_score, match_score, mismatch_score, read4, graph_base, score, trace); // Perform same score updates as above, but for rest of predecessors. for (int32_t p = 1; p < pred_count; p++) { int32_t pred_idx2 = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; - TraceT4 traces_4; - ScoreT4 scores_4 = get_scores(scores, pred_idx2, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, - gap_score, match_score, mismatch_score, read4, graph_base, traces_4); - - if (score.s0 < scores_4.s0) - { - score.s0 = scores_4.s0; - trace.t0 = traces_4.t0; - } - if (score.s1 < scores_4.s1) - { - score.s1 = scores_4.s1; - trace.t1 = traces_4.t1; - } - if (score.s2 < scores_4.s2) - { - score.s2 = scores_4.s2; - trace.t2 = traces_4.t2; - } - if (score.s3 < scores_4.s3) - { - score.s3 = scores_4.s3; - trace.t3 = traces_4.t3; - } + get_scores(scores, pred_idx2, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, + gap_score, match_score, mismatch_score, read4, graph_base, score, trace); } // While there are changes to the horizontal score values, keep updating the matrix. From 70f80ff54ad433291aa0ae4d188924f9584c0529 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 14 Sep 2020 20:38:08 -0400 Subject: [PATCH 067/281] [cudapoa-backtracking] padding sequence lengths to be aligned with SeqT4 (uchar4) size --- cudapoa/src/cudapoa_batch.cuh | 4 +++- cudapoa/src/cudapoa_kernels.cuh | 8 ++++---- cudapoa/src/cudapoa_structs.cuh | 9 +++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 63119398b..3ae2b6a79 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -541,7 +541,9 @@ protected: } input_details_h_->sequence_lengths[global_sequence_idx_] = seq_len; - num_nucleotides_copied_ += seq_len; + // to be aligned with uchar4 size, pad sequence length to be multiple of 32 + num_nucleotides_copied_ += cudautils::align(seq_len); + ; global_sequence_idx_++; return StatusType::success; diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 374632ef6..7ad43a167 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -233,8 +233,8 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) for (int32_t s = 1; s < num_sequences; s++) { int32_t seq_len = sequence_lengths[s]; - sequence += sequence_lengths[s - 1]; // increment the pointer so it is pointing to correct sequence data - base_weights += sequence_lengths[s - 1]; // increment the pointer so it is pointing to correct sequence data + sequence += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data + base_weights += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data if (lane_idx == 0) { @@ -275,7 +275,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) gap_score, mismatch_score, match_score, - 0); + 0); __syncwarp(); @@ -300,7 +300,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) gap_score, mismatch_score, match_score, - alignment_length); + alignment_length); __syncwarp(); } } diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index ed4a54ed5..ce6c4bc88 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -26,6 +26,9 @@ // Maximum number of nodes aligned to each other. #define CUDAPOA_MAX_NODE_ALIGNMENTS 50 +// Size of read chunks for reading sequences +#define SIZE_OF_SeqT4 32 + // Dimensions for Banded alignment score matrix #define WARP_SIZE 32 #define CELLS_PER_THREAD 4 @@ -189,6 +192,12 @@ struct SeqT4 SeqT r0, r1, r2, r3; }; +template <> +struct __align__(4) SeqT4 +{ + uint8_t r0, r1, r2, r3; +}; + template struct ScoreT4 { From b77dc2e436dfb3272d6b7eb1d13da964faadbd19 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 14 Sep 2020 17:59:46 -0700 Subject: [PATCH 068/281] [cudaextender] Working device_ptr sample --- CMakeLists.txt | 2 +- cudaextender/CMakeLists.txt | 12 +- .../genomeworks/cudaextender/extender.hpp | 4 +- cudaextender/samples/CMakeLists.txt | 4 +- cudaextender/samples/device_sample.cpp | 180 ++++++++++++++++++ cudaextender/samples/device_sample.cu | 142 -------------- cudaextender/src/extender.cpp | 6 +- cudaextender/src/ungapped_xdrop.cpp | 104 ---------- cudaextender/src/ungapped_xdrop.cu | 174 +++++++++++++++++ ...{ungapped_xdrop.hpp => ungapped_xdrop.cuh} | 11 +- ...r_kernels.cu => ungapped_xdrop_kernels.cu} | 55 ++++-- cudaextender/src/ungapped_xdrop_kernels.cuh | 54 +++++- 12 files changed, 471 insertions(+), 277 deletions(-) create mode 100644 cudaextender/samples/device_sample.cpp delete mode 100644 cudaextender/samples/device_sample.cu delete mode 100644 cudaextender/src/ungapped_xdrop.cpp create mode 100644 cudaextender/src/ungapped_xdrop.cu rename cudaextender/src/{ungapped_xdrop.hpp => ungapped_xdrop.cuh} (82%) rename cudaextender/src/{ungapped_extender_kernels.cu => ungapped_xdrop_kernels.cu} (91%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 262dc10d9..bf4006e2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,7 +40,7 @@ option(gw_enable_cudapoa_nw_print "Enable verbose prints within cudapoa NW kerne option(gw_profiling "Compile a binary for profiling with NVTX markers." OFF) option(gw_enable_caching_allocator "Enable caching allocator." ON) option(gw_generate_docs "Generate Doxygen documentation" ON) -option(gw_cuda_gen_all_arch "ON: Generate optimized CUDA code for all architectures | OFF: for detected architectures only" ON) +option(gw_cuda_gen_all_arch "ON: Generate optimized CUDA code for all architectures | OFF: for detected architectures only" OFF) # Must be included before others for options value validation include(cmake/Utils.cmake) diff --git a/cudaextender/CMakeLists.txt b/cudaextender/CMakeLists.txt index 2cd479eac..2fa927dd0 100644 --- a/cudaextender/CMakeLists.txt +++ b/cudaextender/CMakeLists.txt @@ -39,15 +39,21 @@ set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14") if (gw_cuda_after_10_0) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} --expt-relaxed-constexpr") endif () +if (gw_cuda_gen_all_arch) + CUDA_SELECT_NVCC_ARCH_FLAGS(ARCH_FLAGS "Common") +else() + CUDA_SELECT_NVCC_ARCH_FLAGS(ARCH_FLAGS "Auto") +endif() +set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} ${ARCH_FLAGS}") message(STATUS "nvcc flags for ${PROJECT_NAME}: ${CUDA_NVCC_FLAGS}") get_property(gw_library_type GLOBAL PROPERTY gw_library_type) cuda_add_library(cudaextender ${gw_library_type} src/cudaextender.cpp src/extender.cpp - src/ungapped_xdrop.cpp - ${CMAKE_CURRENT_BINARY_DIR}/version.cpp - src/ungapped_extender_kernels.cu) + src/ungapped_xdrop.cu + src/ungapped_xdrop_kernels.cu + ${CMAKE_CURRENT_BINARY_DIR}/version.cpp) target_link_libraries(cudaextender gwbase gwio) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index cd3228662..42c25b89b 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -74,7 +74,7 @@ class Extender const char* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, - int32_t* d_num_scored_segment_pairs) = 0; + int32_t& num_scored_segment_pairs) = 0; /// \brief Waits for CUDA accelerated extension to finish /// @@ -91,7 +91,7 @@ class Extender virtual void reset() = 0; }; -std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_size, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type); +std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type=ExtensionType::ungapped_xdrop); } // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/samples/CMakeLists.txt b/cudaextender/samples/CMakeLists.txt index 09bda87b3..623f6ead4 100644 --- a/cudaextender/samples/CMakeLists.txt +++ b/cudaextender/samples/CMakeLists.txt @@ -21,11 +21,13 @@ get_property(cudaextender_data_include_dir GLOBAL PROPERTY cudaextender_data_inc include_directories(${cudaextender_data_include_dir}) add_executable(${PROJECT_NAME} - host_sample.cu + device_sample.cpp ) target_link_libraries(${PROJECT_NAME} cudaextender + gwio + gwbase ) install(TARGETS ${PROJECT_NAME} diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp new file mode 100644 index 000000000..a518c4bb2 --- /dev/null +++ b/cudaextender/samples/device_sample.cpp @@ -0,0 +1,180 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace claraparabricks::genomeworks; +using namespace claraparabricks::genomeworks::cudaextender; + +constexpr int8_t A_NT = 0; +constexpr int8_t C_NT = 1; +constexpr int8_t G_NT = 2; +constexpr int8_t T_NT = 3; +constexpr int8_t L_NT = 4; +constexpr int8_t N_NT = 5; +constexpr int8_t X_NT = 6; +constexpr int8_t E_NT = 7; +constexpr int8_t NUC = 8; +constexpr int8_t NUC2 = NUC * NUC; + +// Really simple parser with no error checks +void parse_seed_pairs(const std::string& filepath, std::vector& seed_pairs) +{ + std::ifstream seed_pair_file(filepath); + if (!seed_pair_file.is_open()) + throw std::runtime_error("Cannot open file"); + if (seed_pair_file.good()) + { + std::string line; + while (std::getline(seed_pair_file, line, ',')) + { + SeedPair seed_pair; + seed_pair.target_position_in_read = std::atoi(line.c_str()); + std::getline(seed_pair_file, line, ','); // Get the next value + seed_pair.query_position_in_read = std::atoi(line.c_str()); + seed_pairs.push_back(seed_pair); + } + } +} + +// convert input sequence from alphabet to integers +void encode_string(char* dst_seq, const char* src_seq, int32_t len) +{ + for (int32_t i = 0; i < len; i++) + { + char ch = src_seq[i]; + char dst = X_NT; + if (ch == 'A') + dst = A_NT; + else if (ch == 'C') + dst = C_NT; + else if (ch == 'G') + dst = G_NT; + else if (ch == 'T') + dst = T_NT; + else if ((ch == 'a') || (ch == 'c') || (ch == 'g') || (ch == 't')) + dst = L_NT; + else if ((ch == 'n') || (ch == 'N')) + dst = N_NT; + else if (ch == '&') + dst = E_NT; + dst_seq[i] = dst; + } +} + +int main(int argc, char* argv[]) +{ + const int32_t xdrop_threshold = 910; + const bool input_no_entropy = false; + const int32_t score_threshold = 3000; + // Fasta query and target files + std::string target_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); + // Assumes that only one sequence is present per file + std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; + + std::string query_file_path = "../data/example.fa"; + std::unique_ptr fasta_parser_query = + io::create_kseq_fasta_parser(query_file_path, 0, false); + // Assumes that only one sequence is present per file + std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; + + // CSV SeedPairs file - Each row -> query_position_in_read_, + // target_position_in_read_ + std::string seed_pairs_file_path = "../data/example_hits.csv"; + + std::vector h_seed_pairs; + // Following function loops through all seed_pairs in the example_seed_pairs.csv and returns + // results in + // the passed vector + parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); + std::cout << h_seed_pairs[0].query_position_in_read << std::endl; + + // Define Scoring Matrix + int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, + -114, 100, -125, -31, -1000, -1000, -100, -9100, + -31, -125, 100, -114, -1000, -1000, -100, -9100, + -123, -31, -114, 91, -1000, -1000, -100, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -100, -100, -100, -100, -1000, -1000, -100, -9100, + -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; + + char* h_encoded_target = (char*)malloc(sizeof(char) * target_sequence.length()); + char* h_encoded_query = (char*)malloc(sizeof(char) * query_sequence.length()); + encode_string(h_encoded_target, target_sequence.c_str(), target_sequence.length()); + encode_string(h_encoded_query, query_sequence.c_str(), query_sequence.length()); + // Create a stream for async use + CudaStream stream0 = make_cuda_stream(); + // Allocate space on device for target and query sequences, seed_pairs, + // high scoring segment pairs (hsps) and num_hsps. + char* d_query; + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_query, sizeof(char) * query_sequence.length())); + char* d_target; + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_target, sizeof(char) * target_sequence.length())); + SeedPair* d_seed_pairs; + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_seed_pairs, sizeof(SeedPair) * h_seed_pairs.size())); + // Allocate a minimum of num_seed_pairs as all seed_pairs could be hsps in the worst case + int32_t h_num_hsps = 0; + ScoredSegmentPair* d_hsps; + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_hsps, sizeof(ScoredSegmentPair) * h_seed_pairs.size())); + + // Async Memcopy all the input values to device + // TODO - Convert to pinned memory for true async copy + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query, sizeof(char) * query_sequence.length(), + cudaMemcpyHostToDevice, stream0.get())); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target, sizeof(char) * target_sequence.length(), + cudaMemcpyHostToDevice, stream0.get())); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_seed_pairs, &h_seed_pairs[0], sizeof(SeedPair) * h_seed_pairs.size(), cudaMemcpyHostToDevice, + stream0.get())); + + // Create an ungapped extender object + std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0); + + // Launch the ungapped extender device function + ungapped_extender->extend_async(d_query, // Type TBD based on encoding + query_sequence.length(), + d_target, + target_sequence.length(), + score_threshold, + d_seed_pairs, + h_seed_pairs.size(), + d_hsps, + h_num_hsps); + + // Wait for ungapped extender to finish + GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0.get())); + + //Get results + std::cout << "h_num_hsps=" << h_num_hsps << std::endl; + + free(h_encoded_target); + free(h_encoded_query); + // Free all allocated memory on the GPU + GW_CU_CHECK_ERR(cudaFree(d_query)); + GW_CU_CHECK_ERR(cudaFree(d_target)); + GW_CU_CHECK_ERR(cudaFree(d_hsps)); + GW_CU_CHECK_ERR(cudaFree(d_seed_pairs)); + + return 0; +} diff --git a/cudaextender/samples/device_sample.cu b/cudaextender/samples/device_sample.cu deleted file mode 100644 index 8ec002b12..000000000 --- a/cudaextender/samples/device_sample.cu +++ /dev/null @@ -1,142 +0,0 @@ -/* -* Copyright 2020 NVIDIA CORPORATION. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#include -#include -#include -#include - -#include -#include -#include - -using namespace claraparabricks::genomeworks; -using namespace claraparabricks::genomeworks::cudaungappedextender; - -int main(int argc, char* argv[]) -{ - const int32_t input_xdrop = 910; - const bool input_no_entropy = false; - const int32_t score_threshold = 3000; - // Fasta query and target files - std::string target_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_target = - io::create_kseq_fasta_parser(target_file_path, 0, false); - // Assumes that only one sequence is present per file - std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); - - std::string query_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_query = - io::create_kseq_fasta_parser(query_file_path, 0, false); - // Assumes that only one sequence is present per file - magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - - // CSV SeedPairs file - Each row -> query_position_in_read_, - // target_position_in_read_ - std::string seed_pairs_file_path = "../data/example_seed_pairs.csv"; - - std::vector h_seed_pairs; - // Following function loops through all seed_pairs in the example_seed_pairs.csv and returns - // results in - // the passed vector - parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); - - // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; - std::string h_encoded_target = magic_encode(magic_base, target_sequence); - std::string h_encoded_query = magic_encode(magic_base, query_sequence); - - // Create a stream for async use - CudaStream stream0 = make_cuda_stream(); - // Allocate space on device for target and query sequences, seed_pairs, - // high scoring segment pairs (hsps) and num_hsps. - char* d_query; - GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_query, sizeof(char) * h_encoded_query.size())); - char* d_target; - GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_target, sizeof(char) * h_target_query.size())); - SeedPair* d_seed_pairs; - GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_seed_pairs, sizeof(SeedPair) * h_seed_pairs.size())); - // Allocate a minimum of num_seed_pairs as all seed_pairs could be hsps in the worst case - int32_t h_num_hsps = 0; - ScoredSegmentPair* d_hsps; - GW_CU_CHECK_ERROR( - cudaMalloc((void**)&d_hsps, sizeof(ScoredSegmentPair) * h_seed_pairs.size())); - int32_t* d_num_hsps; - GW_CU_CHECK_ERROR(cudaMalloc((void**)&d_num_hsps, sizeof(int32_t)); - - // Async Memcopy all the input values to device - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query.c_str(), sizeof(char) * h_encoded_query.size(), - cudaMemcpyHostToDevice, stream0.get())); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target.c_str(), sizeof(char) * h_encoded_target.size(), - cudaMemcpyHostToDevice, stream0.get())); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_seed_pairs, &h_seed_pairs[0], sizeof(SeedPair) * h_seed_pairs.size(), cudaMemcpyHostToDevice, - stream0.get()))); - - // Create an ungapped extender object - std::unique_ptr ungapped_extender = std::make_unique(0, - magic_number_matrix, - input_xdrop, - input_no_entropy, - stream0.get()); - // Launch the ungapped extender device function - ungapped_extender->extend_async(d_query, // Type TBD based on encoding - encoded_query.size(), - d_target.c_str(), - encoded_target.size(), - score_threshold, - d_seed_pairs, - h_seed_pairs.size(), - d_hsps, - d_num_hsps); - // Copy back the number of hsps to host - GW_CU_CHECK_ERR(cudaMemcpyAsync(&h_num_hsps, d_num_hsps, sizeof(int32_t), cudaMemcpyDeviceToHost, stream0.get())); - - // Wait for ungapped extender to finish - GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0.get())); - - //Get results - if (h_num_hsps > 0) - { - std::vector h_hsps(h_num_hsps); - // Don't care about asynchronous copies here - GW_CU_CHECK_ERR(cudaMemcpy(&h_hsps[0], d_hsps, - sizeof(ScoredSegmentPair) * h_num_hsps, - cudaMemcpyDeviceToHost)); - - int32_t i = 0; - for (const auto& segment : h_hsps) - { - std::cout << "Segment: " << i << "Length: " << segment.length - << "Score: " << segment.score << std::endl; - std::cout << "Position in query: " - << segment.seed_pair.query_position_in_read << std::endl; - std::cout << "Position in target: " - << segment.seed_pair.target_position_in_read << std::endl; - i++; - } - } - - // Free all allocated memory on the GPU - GW_CU_CHECK_ERROR(cudaFree(d_query); - GW_CU_CHECK_ERROR(cudaFree(d_target); - GW_CU_CHECK_ERROR(cudaFree(d_hsps); - GW_CU_CHECK_ERROR(cudaFree(d_seed_pairs); - GW_CU_CHECK_ERROR(cudaFree(d_num_hsps); - - return 0; -} diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index 2988a9841..532d550b7 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -16,7 +16,7 @@ #include -#include "ungapped_xdrop.hpp" +#include "ungapped_xdrop.cuh" namespace claraparabricks { @@ -27,11 +27,11 @@ namespace genomeworks namespace cudaextender { -std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_size, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type) +std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type) { if (type == ExtensionType::ungapped_xdrop) { - return std::make_unique(h_sub_mat, sub_mat_size, xdrop_threshold, no_entropy, stream, device_id); + return std::make_unique(h_sub_mat, sub_mat_dim, xdrop_threshold, no_entropy, stream, device_id); } else { diff --git a/cudaextender/src/ungapped_xdrop.cpp b/cudaextender/src/ungapped_xdrop.cpp deleted file mode 100644 index 7d5038509..000000000 --- a/cudaextender/src/ungapped_xdrop.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* -* Copyright 2019-2020 NVIDIA CORPORATION. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#include -#include -#include "ungapped_xdrop.hpp" -#include "ungapped_xdrop_kernels.cuh" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace claraparabricks -{ - -namespace genomeworks -{ - -namespace cudaextender -{ - -using namespace cudautils; - -UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id) - : h_sub_mat_(h_sub_mat) - , sub_mat_dim_(sub_mat_dim) - , xdrop_threshold_(xdrop_threshold) - , no_entropy_(no_entropy) - , stream_(stream) - , device_id_(device_id) -{ - // Calculate the max limits on the number of extensions we can do on - // this GPU - cudaDeviceProp device_prop; - cudaGetDeviceProperties(&device_prop, device_id_); - const float global_mem_gb = static_cast(device_prop.totalGlobalMem / 1073741824.0f); - const int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures - max_ungapped_extensions_ = static_cast(global_mem_gb * max_ungapped_per_gb); - - // Switch to device for copying over initial structures - scoped_device_switch dev(device_id_); - - // Allocate space on device for scoring matrix and - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_sub_mat_, sub_mat_dim_ * sizeof(int32_t))); - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, max_ungapped_extensions_ * sizeof(int32_t))); - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_tmp_hsp_, max_ungapped_extensions_ * sizeof(ScoredSegmentPair))); - - // FIXME - Pinned host memory registration for proper async behavior - device_copy_n(h_sub_mat_, sub_mat_dim_ * sizeof(int32_t), d_sub_mat_, stream_); - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_, 0, max_ungapped_extensions_ * sizeof(int32_t), stream_)); - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_hsp_, 0, max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); -} - -StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, const char* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) -{ - // Switch to configured GPU - scoped_device_switch dev(device_id_); - - int32_t curr_num_pairs = 0; - int32_t num_anchors = 0; - for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += max_ungapped_extensions_) - { - curr_num_pairs = std::min(max_ungapped_extensions_, num_seed_pairs - seed_pair_start); - // TODO- Extricate the kernel params? - find_high_scoring_segment_pairs<<<1024, 128, stream_>>>(d_query, query_length, d_target, target_length, d_sub_mat_, no_entropy_, xdrop_threshold_, score_threshold, curr_num_pairs, d_seed_pairs, seed_pair_start, d_scored_segment_pairs, d_done); - thrust::device_ptr d_done_dev_ptr = thrust::device_pointer_cast(d_done_scratch_); - // TODO- Make thrust use caching allocator or change kernel - thrust::inclusive_scan(thrust::cuda::par.on(stream_), d_done_dev_ptr, d_done_dev_ptr + curr_num_pairs, d_done_dev_ptr); - device_copy_n((void*)(d_done[gpu_id]+curr_num_hits-1), sizeof(int32_t), &num_anchors, stream_); - // TODO- Make async - cudaStreamSynchronize(stream_); - if(num_anchors > 0){ - compress_output<<<1024, 1024, stream_>>>(d_done_, seed_pair_start, scored_segment_pairs_, d_tmp_hsp_, curr_num_pairs); // TODO- Need configurability for kernel? - - thrust::stable_sort(thrust::cuda::par.on(stream_), d_tmp_hsp_vec[gpu_id].begin(), d_tmp_hsp_vec[gpu_id].begin()+num_anchors, hspComp()); - thrust::device_vector::iterator result_end = thrust::unique_copy(thrust::cuda::par.on(stream_), d_tmp_hsp_vec[gpu_id].begin(), d_tmp_hsp_vec[gpu_id].begin()+num_anchors, d_hsp_vec[gpu_id].begin()+total_anchors, hspEqual()); - num_anchors = thrust::distance(thrust::cuda::par.on(stream_), d_hsp_vec[gpu_id].begin()+total_anchors, result_end), num_anchors; - total_anchors += num_anchors; - } - } - return success; -} - -} // namespace cudaextender - -} // namespace genomeworks - -} // namespace claraparabricks \ No newline at end of file diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu new file mode 100644 index 000000000..3d76a0aae --- /dev/null +++ b/cudaextender/src/ungapped_xdrop.cu @@ -0,0 +1,174 @@ +/* +* Copyright 2019-2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#include +#include +#include "ungapped_xdrop.cuh" +#include "ungapped_xdrop_kernels.cuh" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ + +using namespace cudautils; + +UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id) + : h_sub_mat_(h_sub_mat) + , sub_mat_dim_(sub_mat_dim) + , xdrop_threshold_(xdrop_threshold) + , no_entropy_(no_entropy) + , stream_(stream) + , device_id_(device_id) +{ + // Calculate the max limits on the number of extensions we can do on + // this GPU + cudaDeviceProp device_prop; + cudaGetDeviceProperties(&device_prop, device_id_); + constexpr int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures + constexpr int32_t max_seed_pairs_per_gb = 8388608; // FIXME: Calculate using sizeof datastructures + const float global_mem_gb = static_cast(device_prop.totalGlobalMem / 1073741824.0f); + batch_max_ungapped_extensions_ = static_cast(global_mem_gb * max_ungapped_per_gb); + int32_t max_seed_pairs = static_cast(global_mem_gb * max_seed_pairs_per_gb); + // Switch to device for copying over initial structures + scoped_device_switch dev(device_id_); + + // Allocate space on device for scoring matrix and + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_sub_mat_, sub_mat_dim_ * sizeof(int32_t))); + // FIXME - Pinned host memory registration for proper async behavior + device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_, stream_); + + d_done_.resize(batch_max_ungapped_extensions_, 0); + ScoredSegmentPair dummy_zero_pair = {{0, 0}, 0, 0}; + d_tmp_hsp_.resize(batch_max_ungapped_extensions_, dummy_zero_pair); + d_hsp_.resize(max_seed_pairs, dummy_zero_pair); + //GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, max_ungapped_extensions_ * sizeof(int32_t))); + //GW_CU_CHECK_ERR(cudaMalloc((void**)&d_tmp_hsp_, max_ungapped_extensions_ * sizeof(ScoredSegmentPair))); + + //GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_, 0, max_ungapped_extensions_ * sizeof(int32_t), stream_)); + //GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_hsp_, 0, max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); +} + +StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, + const char* d_target, int32_t target_length, + int32_t score_threshold, SeedPair* d_seed_pairs, + int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, + int32_t& num_scored_segment_pairs) +{ + // Switch to configured GPU + scoped_device_switch dev(device_id_); + + int32_t curr_num_pairs = 0; + num_scored_segment_pairs = 0; + total_scored_segment_pairs_ = 0; + for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += batch_max_ungapped_extensions_) + { + curr_num_pairs = std::min(batch_max_ungapped_extensions_, num_seed_pairs - seed_pair_start); + int32_t* d_done_raw_pointer = thrust::raw_pointer_cast(d_done_.data()); + // TODO- Extricate the kernel params? + find_high_scoring_segment_pairs<<<1024, 128, 0, stream_>>>(d_target, + target_length, + d_query, + query_length, + d_sub_mat_, + no_entropy_, + xdrop_threshold_, + score_threshold, + d_seed_pairs, + curr_num_pairs, + seed_pair_start, + thrust::raw_pointer_cast(d_hsp_.data()), + d_done_raw_pointer); + // TODO- Make thrust use caching allocator or change kernel + thrust::inclusive_scan(thrust::cuda::par.on(stream_), d_done_.begin(), d_done_.begin() + curr_num_pairs, d_done_.begin()); + // TODO- Make async + device_copy_n((d_done_raw_pointer + curr_num_pairs - 1), 1, &num_scored_segment_pairs, stream_); + cudaStreamSynchronize(stream_); + std::cout< 0) + { + compress_output<<<1024, 1024, 0, stream_>>>(d_done_raw_pointer, + seed_pair_start, + thrust::raw_pointer_cast(d_hsp_.data()), + thrust::raw_pointer_cast(d_tmp_hsp_.data()), + curr_num_pairs); // TODO- Need configurability for kernel? + thrust::stable_sort(thrust::cuda::par.on(stream_), + d_tmp_hsp_.begin(), + d_tmp_hsp_.begin() + num_scored_segment_pairs, + scored_segment_pair_comp()); + thrust::device_vector::iterator result_end = thrust::unique_copy(thrust::cuda::par.on(stream_), + d_tmp_hsp_.begin(), + d_tmp_hsp_.begin() + num_scored_segment_pairs, + d_hsp_.begin() + total_scored_segment_pairs_, + scored_segment_pair_equal()); + std::cout<<"Before: "<& UngappedXDrop::get_scored_segment_pairs() const +{ + std::vector placeholder; + return placeholder; +} + +StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length, + const char* h_target, int32_t target_length, + int32_t score_threshold, + std::vector& h_seed_pairs) +{ + return success; +} + +StatusType UngappedXDrop::sync() +{ + return success; +} + +void UngappedXDrop::reset() +{ + d_done_.clear(); + d_tmp_hsp_.clear(); + d_hsp_.clear(); +}; + +UngappedXDrop::~UngappedXDrop() +{ + GW_CU_CHECK_ERR(cudaFree(d_sub_mat_)); +}; + +} // namespace cudaextender + +} // namespace genomeworks + +} // namespace claraparabricks \ No newline at end of file diff --git a/cudaextender/src/ungapped_xdrop.hpp b/cudaextender/src/ungapped_xdrop.cuh similarity index 82% rename from cudaextender/src/ungapped_xdrop.hpp rename to cudaextender/src/ungapped_xdrop.cuh index 628db13be..29aa5188c 100644 --- a/cudaextender/src/ungapped_xdrop.hpp +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -17,6 +17,7 @@ #pragma once #include +#include namespace claraparabricks { @@ -42,7 +43,7 @@ class UngappedXDrop : public Extender const char* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, - int32_t* d_num_scored_segment_pairs) override; + int32_t& num_scored_segment_pairs) override; StatusType sync() override; void reset() override; @@ -57,10 +58,12 @@ class UngappedXDrop : public Extender cudaStream_t stream_; int32_t device_id_; std::vector scored_segment_pairs_; - int32_t max_ungapped_extensions_; + int32_t batch_max_ungapped_extensions_; int32_t* d_sub_mat_; // Pointer to device substitution matrix - int32_t* d_done_; // TODO- Rename scratch space - ScoredSegmentPair* d_tmp_hsp_; // TODO- Rename Scratch space 2 + thrust::device_vector d_done_; // TODO- Rename scratch space + thrust::device_vector d_tmp_hsp_; // TODO- Rename Scratch space 2 + thrust::device_vector d_hsp_; // TODO- Rename output variable + int32_t total_scored_segment_pairs_; }; } // namespace cudaextender diff --git a/cudaextender/src/ungapped_extender_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu similarity index 91% rename from cudaextender/src/ungapped_extender_kernels.cu rename to cudaextender/src/ungapped_xdrop_kernels.cu index df786ccb3..bbee1a067 100644 --- a/cudaextender/src/ungapped_extender_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -27,7 +27,7 @@ namespace cudaextender // extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to score_threshold and update the d_hsp and d_done vectors -__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int32_t* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, uint32_t* d_done) +__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int32_t* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, int32_t* d_done) { int32_t thread_id = threadIdx.x; @@ -40,33 +40,33 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe constexpr int32_t nuc = 8; // TODO - remove hardcode - pass in constexpr int32_t nuc2 = 64; // TODO - remove hardcode - __shared__ uint32_t ref_loc[num_warps]; - __shared__ uint32_t query_loc[num_warps]; + __shared__ int32_t ref_loc[num_warps]; + __shared__ int32_t query_loc[num_warps]; __shared__ int32_t total_score[num_warps]; __shared__ int32_t prev_score[num_warps]; __shared__ int32_t prev_max_score[num_warps]; - __shared__ uint32_t prev_max_pos[num_warps]; + __shared__ int32_t prev_max_pos[num_warps]; __shared__ bool edge_found[num_warps]; __shared__ bool xdrop_found[num_warps]; __shared__ bool new_max_found[num_warps]; - __shared__ uint32_t left_extent[num_warps]; - __shared__ uint32_t extent[num_warps]; - __shared__ uint32_t tile[num_warps]; + __shared__ int32_t left_extent[num_warps]; + __shared__ int32_t extent[num_warps]; + __shared__ int32_t tile[num_warps]; __shared__ double entropy[num_warps]; int32_t thread_score; int32_t max_thread_score; - uint32_t max_pos; - uint32_t temp_pos; + int32_t max_pos; + int32_t temp_pos; bool xdrop_done; int32_t temp; short count[4]; short count_del[4]; char r_chr; char q_chr; - uint32_t ref_pos; - uint32_t query_pos; - uint32_t pos_offset; + int32_t ref_pos; + int32_t query_pos; + int32_t pos_offset; __shared__ int32_t sub_mat[nuc2]; @@ -456,6 +456,37 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } } +// gather only the HSPs from the resulting segments to the beginning of the +// tmp_hsp vector +__global__ +void compress_output (int32_t* d_done, int32_t start_index, ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, int32_t num_hits){ + + int thread_id = threadIdx.x; + int block_dim = blockDim.x; + int grid_dim = gridDim.x; + int block_id = blockIdx.x; + + int stride = block_dim * grid_dim; + int32_t start = block_dim * block_id + thread_id; + int32_t reduced_index = 0; + int32_t index = 0; + + for (int32_t id = start; id < num_hits; id += stride) { + reduced_index = d_done[id]; + index = id + start_index; + if(index > 0){ + if(reduced_index > d_done[index-1]){ + d_tmp_hsp[reduced_index-1] = d_hsp[index]; + } + } + else{ + if(reduced_index == 1){ + d_tmp_hsp[0] = d_hsp[start_index]; + } + } + } +} + } // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index e380ae449..07ba7d5d9 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -16,7 +16,7 @@ #pragma once #include -#include "ungapped_xdrop.hpp" +#include "ungapped_xdrop.cuh" namespace claraparabricks { @@ -30,11 +30,55 @@ namespace cudaextender // extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to hspthresh and update the d_hsp and d_done vectors -__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, uint32_t* d_done); +__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int32_t * d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, int32_t* d_done); +// gather only the HSPs from the resulting segments to the beginning of the +// tmp_hsp vector +__global__ void compress_output(int32_t* d_done, int32_t start_index, ScoredSegmentPair * d_hsp, ScoredSegmentPair* d_tmp_hsp, int num_hits); -} +// Binary Predicate for sorting the ScoredSegmentPairs +struct scored_segment_pair_comp +{ + __host__ __device__ bool operator()(ScoredSegmentPair x, ScoredSegmentPair y) + { + if (x.seed_pair.query_position_in_read < y.seed_pair.query_position_in_read) + return true; + else if (x.seed_pair.query_position_in_read == y.seed_pair.query_position_in_read) + { + if (x.length > y.length) + return true; + else if (x.length == y.length) + { + if (x.seed_pair.target_position_in_read < y.seed_pair.target_position_in_read) + return true; + else if (x.seed_pair.target_position_in_read == y.seed_pair.target_position_in_read) + { + if (x.score > y.score) + return true; + else + return false; + } + else + return false; + } + else + return false; + } + else + return false; + } +}; + +struct scored_segment_pair_equal +{ + __host__ __device__ bool operator()(ScoredSegmentPair x, ScoredSegmentPair y) + { + return ((x.seed_pair.target_position_in_read == y.seed_pair.target_position_in_read) && (x.seed_pair.query_position_in_read == y.seed_pair.query_position_in_read) && (x.length == y.length) && (x.score == y.score)); + } +}; + +} // namespace cudaextender -} +} // namespace genomeworks -} +} // namespace claraparabricks From 4369712ecf81b1bf9e600eb0ea3d0640b431af6b Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Tue, 15 Sep 2020 09:07:37 -0400 Subject: [PATCH 069/281] [readme] clarify gcc support in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6d751551c..fd7f1f509 100644 --- a/README.md +++ b/README.md @@ -104,9 +104,9 @@ Minimum requirements - 1. Ubuntu 16.04 or Ubuntu 18.04 2. CUDA 9.0+ (official instructions for installing CUDA are available [here](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html)) -3. gcc/g++ 5.4.0+ +3. gcc/g++ 5.4.0+ / 7.x.x 4. Python 3.6.7+ -5. CMake (>= 3.0) +5. CMake (>= 3.10.2) ## GenomeWorks Setup From 8321c7b4431edfbc7db18485d102372df530d22d Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 15 Sep 2020 14:49:05 -0700 Subject: [PATCH 070/281] [cudaextender] debug changes --- cudaextender/samples/device_sample.cpp | 2 +- cudaextender/src/ungapped_xdrop.cu | 10 +++++++++- cudaextender/src/ungapped_xdrop_kernels.cu | 5 ++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp index a518c4bb2..8215aeaee 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/device_sample.cpp @@ -96,7 +96,7 @@ int main(int argc, char* argv[]) std::string query_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); - // Assumes that only one sequence is present per file + // Assumes that only one sequence is present per file std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; // CSV SeedPairs file - Each row -> query_position_in_read_, diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 3d76a0aae..1ed1e45a2 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -89,7 +89,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length { curr_num_pairs = std::min(batch_max_ungapped_extensions_, num_seed_pairs - seed_pair_start); int32_t* d_done_raw_pointer = thrust::raw_pointer_cast(d_done_.data()); - // TODO- Extricate the kernel params? + // TODO- Extricate the kernel launch params? find_high_scoring_segment_pairs<<<1024, 128, 0, stream_>>>(d_target, target_length, d_query, @@ -133,6 +133,14 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length } num_scored_segment_pairs = total_scored_segment_pairs_; + thrust::host_vector h_vec = d_hsp_; +// + // std::cout<<"Ouptut: "<= score_threshold) { d_scored_segment[hid].seed_pair.target_position_in_read = ref_loc[warp_id] - left_extent[warp_id]; From 623711ceb15b8023fdce637c76dc6d686fda7c6a Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 15 Sep 2020 15:17:51 -0700 Subject: [PATCH 071/281] [cudaextender] Remove redundant comma --- cudaextender/src/ungapped_xdrop.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 1ed1e45a2..4446e3fbc 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -126,7 +126,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length d_hsp_.begin() + total_scored_segment_pairs_, scored_segment_pair_equal()); std::cout<<"Before: "< Date: Tue, 15 Sep 2020 15:34:04 -0700 Subject: [PATCH 072/281] [cudaextender] correct casts --- cudaextender/src/ungapped_xdrop.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 4446e3fbc..e2d857403 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -51,9 +51,9 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd cudaGetDeviceProperties(&device_prop, device_id_); constexpr int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures constexpr int32_t max_seed_pairs_per_gb = 8388608; // FIXME: Calculate using sizeof datastructures - const float global_mem_gb = static_cast(device_prop.totalGlobalMem / 1073741824.0f); - batch_max_ungapped_extensions_ = static_cast(global_mem_gb * max_ungapped_per_gb); - int32_t max_seed_pairs = static_cast(global_mem_gb * max_seed_pairs_per_gb); + const float global_mem_gb = static_cast(device_prop.totalGlobalMem) / 1073741824.0f; + batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; + int32_t max_seed_pairs = static_cast(global_mem_gb) * max_seed_pairs_per_gb; // Switch to device for copying over initial structures scoped_device_switch dev(device_id_); From 1855e04e39c0e617b663717c599e3aa47c60a962 Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 15 Sep 2020 21:51:31 -0400 Subject: [PATCH 073/281] [cudapoa-backtracking] backtrace buffer 1st column computation is added. --- cudapoa/src/cudapoa_nw_banded.cuh | 106 ++++++++++-------------------- 1 file changed, 36 insertions(+), 70 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 7c41fdf6f..2a55e7b78 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -100,7 +100,7 @@ __device__ __forceinline__ void initialize_band(ScoreT* scores, int32_t row, int } template -__device__ TraceT get_trace(TraceT* backtrace, int32_t row, int32_t column, int32_t band_start, int32_t band_width) +__device__ __forceinline__ TraceT get_trace(TraceT* backtrace, int32_t row, int32_t column, int32_t band_start, int32_t band_width) { int64_t trace_index = static_cast(column - band_start) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); return backtrace[trace_index]; @@ -311,14 +311,19 @@ __device__ __forceinline__ { // Initialise the vertical boundary of the score matrix int32_t penalty; - pred_count = incoming_edge_count[node_id]; + pred_count = incoming_edge_count[node_id]; + int64_t trace_index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); if (pred_count == 0) { - set_score(scores, score_gIdx, -1, gap_score, gradient, band_width, max_column); + scores[trace_index] = gap_score; + backtrace[trace_index] = -score_gIdx; } else { pred_idx = node_id_to_pos[pred_node_id] + 1; + // fill in first column of backtrace buffer + backtrace[trace_index] = -(score_gIdx - pred_idx); + if (band_start > CELLS_PER_THREAD && pred_count == 1) { first_element_prev_score = min_score_value + gap_score; @@ -331,10 +336,16 @@ __device__ __forceinline__ { pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; - penalty = max(penalty, get_score(scores, pred_idx_tmp, -1, gradient, band_width, max_column, min_score_value)); + if (penalty < get_score(scores, pred_idx_tmp, -1, gradient, band_width, max_column, min_score_value)) + { + penalty = get_score(scores, pred_idx_tmp, -1, gradient, band_width, max_column, min_score_value); + // fill in first column of backtrace buffer + backtrace[trace_index] = -(score_gIdx - pred_idx_tmp); + } } first_element_prev_score = penalty + gap_score; } + set_score(scores, score_gIdx, -1, first_element_prev_score, gradient, band_width, max_column); } } @@ -427,12 +438,6 @@ __device__ __forceinline__ scores[score_index + 2L] = score.s2; scores[score_index + 3L] = score.s3; - /* - if (score_gIdx == 1 && lane_idx == 0) - { - printf(">> %d %d %d %d score_index %ld\n", trace.t0, trace.t1, trace.t2, trace.t3, score_index); - }*/ - backtrace[score_index] = trace.t0; backtrace[score_index + 1L] = trace.t1; backtrace[score_index + 2L] = trace.t2; @@ -468,70 +473,31 @@ __device__ __forceinline__ while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) { loop_count++; - //printf("i %3d j %3d \n", i, j); - if (j > 0) - { - int32_t band_start = get_band_start_for_row(i, gradient, band_width, max_column); - TraceT trace = get_trace(backtrace, i, j, band_start, band_width); + int32_t band_start = get_band_start_for_row(i, gradient, band_width, max_column); + TraceT trace = get_trace(backtrace, i, j, band_start, band_width); - if (trace == 0) - { - // horizontal path (indel) - alignment_graph[aligned_nodes] = -1; - alignment_read[aligned_nodes] = j - 1; - j--; - } - else if (trace < 0) - { - // vertical path (indel) - alignment_graph[aligned_nodes] = graph[i - 1]; - alignment_read[aligned_nodes] = -1; - i += trace; - } - else - { - // diagonal path (match/mismatch) - alignment_graph[aligned_nodes] = graph[i - 1]; - alignment_read[aligned_nodes] = j - 1; - i -= trace; - j--; - } + if (trace == 0) + { + // horizontal path (indel) + alignment_graph[aligned_nodes] = -1; + alignment_read[aligned_nodes] = j - 1; + j--; } - // backtrace data is not available for the first column, find the vertical path by checking predecessors - if (j == 0 && i != 0) + else if (trace < 0) { - int32_t scores_ij = get_score(scores, i, j, gradient, band_width, max_column, min_score_value); - int32_t node_id = graph[i - 1]; - uint16_t pred_count = incoming_edge_count[node_id]; - int32_t pred_i = (pred_count == 0 ? 0 : node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]] + 1); - bool pred_found = false; - if (scores_ij == get_score(scores, pred_i, j, gradient, band_width, max_column, min_score_value) + gap_score) - { - alignment_graph[aligned_nodes] = graph[i - 1]; - alignment_read[aligned_nodes] = -1; - i = pred_i; - pred_found = true; - } - else - { - for (int32_t p = 1; p < pred_count; p++) - { - pred_i = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; - if (scores_ij == get_score(scores, pred_i, j, gradient, band_width, max_column, min_score_value) + gap_score) - { - alignment_graph[aligned_nodes] = graph[i - 1]; - alignment_read[aligned_nodes] = -1; - i = pred_i; - pred_found = true; - break; - } - } - } - if (!pred_found) - { - //ToDo throw a new error code indicating max_pred_distance is not large enough - } + // vertical path (indel) + alignment_graph[aligned_nodes] = graph[i - 1]; + alignment_read[aligned_nodes] = -1; + i += trace; + } + else + { + // diagonal path (match/mismatch) + alignment_graph[aligned_nodes] = graph[i - 1]; + alignment_read[aligned_nodes] = j - 1; + i -= trace; + j--; } aligned_nodes++; From 6d3a2d141899ab1190b02fa0fa30a176662417b0 Mon Sep 17 00:00:00 2001 From: Andreas Hehn Date: Wed, 16 Sep 2020 14:44:13 +0200 Subject: [PATCH 074/281] [common] Fixed a compiler warning about non-initialized class constant This fixes #554. --- common/base/CMakeLists.txt | 4 ++- .../genomeworks/utils/graph.hpp | 2 +- common/base/src/graph.cpp | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 common/base/src/graph.cpp diff --git a/common/base/CMakeLists.txt b/common/base/CMakeLists.txt index f301c375c..f0e6ac51c 100644 --- a/common/base/CMakeLists.txt +++ b/common/base/CMakeLists.txt @@ -26,7 +26,9 @@ message(STATUS "nvcc flags for ${MODULE_NAME}: ${CUDA_NVCC_FLAGS}") get_property(gw_library_type GLOBAL PROPERTY gw_library_type) add_library(${MODULE_NAME} ${gw_library_type} src/cudautils.cpp - src/logging.cpp) + src/logging.cpp + src/graph.cpp + ) target_link_libraries(${MODULE_NAME} PUBLIC spdlog ${CUDA_LIBRARIES}) if (gw_profiling) diff --git a/common/base/include/claraparabricks/genomeworks/utils/graph.hpp b/common/base/include/claraparabricks/genomeworks/utils/graph.hpp index ba28bf23d..ef9a6da86 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/graph.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/graph.hpp @@ -185,7 +185,7 @@ class Graph std::unordered_map node_labels_; /// An empty list representing no connectivity - const std::vector empty_; + static const std::vector empty_; }; /// \brief DirectedGraph Object representing a directed graph structure diff --git a/common/base/src/graph.cpp b/common/base/src/graph.cpp new file mode 100644 index 000000000..c7779603c --- /dev/null +++ b/common/base/src/graph.cpp @@ -0,0 +1,29 @@ +/* +* Copyright 2019-2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include + +namespace claraparabricks +{ + +namespace genomeworks +{ + +const std::vector Graph::empty_ = {}; + +} // namespace genomeworks + +} // namespace claraparabricks From 925edca3062783f9251bdec3605adf2f9e4600e2 Mon Sep 17 00:00:00 2001 From: "Eric T. Dawson" Date: Wed, 16 Sep 2020 14:53:07 -0400 Subject: [PATCH 075/281] [pygenomeworks] Fix incorrect output of number of mismatched starts/ends in evaluate_paf. The `evaluate_paf` script now reports mismatched starts/ends only from the best match, rather than adding on every iteration. --- pygenomeworks/bin/evaluate_paf | 44 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/pygenomeworks/bin/evaluate_paf b/pygenomeworks/bin/evaluate_paf index b29aacbd1..f86f66954 100755 --- a/pygenomeworks/bin/evaluate_paf +++ b/pygenomeworks/bin/evaluate_paf @@ -90,8 +90,8 @@ def match_overlaps(record, other, pos_tolerance, min_reciprocal_overlap): """ equal, query_start_valid, query_end_valid, target_start_valid, target_end_valid, strands_equal = records_equal(record, other, pos_tolerance) - - reciprocal = calculate_reciprocal_overlap(record, other) > min_reciprocal_overlap + pct_recip = calculate_reciprocal_overlap(record, other) + reciprocal = pct_recip > min_reciprocal_overlap match = equal or reciprocal @@ -100,6 +100,7 @@ def match_overlaps(record, other, pos_tolerance, min_reciprocal_overlap): "target_start_valid": target_start_valid, "target_end_valid": target_end_valid, "reciprocal_overlaps": reciprocal, + "percent_reciprocal": pct_recip, "strands_equal" : strands_equal, "equal" : equal, "match": match} @@ -186,28 +187,35 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip # seen_test_overlap_keys.add(key) # seen_test_overlap_keys.add(key_reversed) - + #incorrect_query_start += not match_statistics["query_start_valid"] + #incorrect_query_end += not match_statistics["query_end_valid"] + #incorrect_target_start += not match_statistics["target_start_valid"] + #incorrect_target_end += not match_statistics["target_end_valid"] + best_pct_match = 0.0 + best_ends = [1, 1, 1, 1] + found_match = False if key in truth_keys: for truth_interval in truth_query_intervals[test_overlap.query_sequence_name]: truth_overlap = truth_interval.data match_statistics = match_overlaps(truth_overlap, test_overlap, pos_tolerance, min_reciprocal) - incorrect_query_start += not match_statistics["query_start_valid"] - incorrect_query_end += not match_statistics["query_end_valid"] - incorrect_target_start += not match_statistics["target_start_valid"] - incorrect_target_end += not match_statistics["target_end_valid"] if match_statistics["match"]: true_positive_count += 1 found_match = True + best_pct_match = match_statistics["percent_reciprocal"] + best_ends = [0, 0, 0, 0] break + pct_match = match_statistics["percent_reciprocal"] + if pct_match > best_pct_match: + best_pct_match = pct_match + best_ends[0] = 1 if not match_statistics["query_start_valid"] else 0 + best_ends[1] = 1 if not match_statistics["query_end_valid"] else 0 + best_ends[2] = 1 if not match_statistics["target_start_valid"] else 0 + best_ends[3] = 1 if not match_statistics["target_end_valid"] else 0 if not found_match: for truth_interval in truth_target_intervals[test_overlap.target_sequence_name]: truth_overlap = truth_interval.data match_statistics = match_overlaps(truth_overlap, test_overlap, pos_tolerance, min_reciprocal) - incorrect_query_start += not match_statistics["query_start_valid"] - incorrect_query_end += not match_statistics["query_end_valid"] - incorrect_target_start += not match_statistics["target_start_valid"] - incorrect_target_end += not match_statistics["target_end_valid"] if match_statistics["match"]: true_positive_count += 1 found_match = True @@ -217,10 +225,6 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip for truth_interval in truth_query_intervals[key_reversed]: truth_overlap = truth_interval.data match_statistics = match_overlaps(truth_overlap, test_overlap) - incorrect_query_start += not match_statistics["query_start_valid"] - incorrect_query_end += not match_statistics["query_end_valid"] - incorrect_target_start += not match_statistics["target_start_valid"] - incorrect_target_end += not match_statistics["target_end_valid"] if match_statistics["match"]: true_positive_count += 1 found_match = True @@ -229,15 +233,15 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip for truth_interval in truth_target_intervals[key_reversed]: truth_overlap = truth_interval.data match_statistics = match_overlaps(truth_overlap, test_overlap) - incorrect_query_start += not match_statistics["query_start_valid"] - incorrect_query_end += not match_statistics["query_end_valid"] - incorrect_target_start += not match_statistics["target_start_valid"] - incorrect_target_end += not match_statistics["target_end_valid"] if match_statistics["match"]: true_positive_count += 1 found_match = True break - + incorrect_query_start += best_ends[0] + incorrect_query_end += best_ends[1] + incorrect_target_start += best_ends[2] + incorrect_target_end += best_ends[3] + if not found_match: false_positive_count += 1 From a47641ffcb7cec55f3cfb1818ded7f5229897555 Mon Sep 17 00:00:00 2001 From: "Eric T. Dawson" Date: Wed, 16 Sep 2020 14:59:27 -0400 Subject: [PATCH 076/281] [pygenomeworks] Add tabulation for mismatched starts/ends to target and searches where the query/target keys are reversed. --- pygenomeworks/bin/evaluate_paf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pygenomeworks/bin/evaluate_paf b/pygenomeworks/bin/evaluate_paf index f86f66954..2d6a47daf 100755 --- a/pygenomeworks/bin/evaluate_paf +++ b/pygenomeworks/bin/evaluate_paf @@ -220,6 +220,13 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip true_positive_count += 1 found_match = True break + pct_match = match_statistics["percent_reciprocal"] + if pct_match > best_pct_match: + best_pct_match = pct_match + best_ends[0] = 1 if not match_statistics["query_start_valid"] else 0 + best_ends[1] = 1 if not match_statistics["query_end_valid"] else 0 + best_ends[2] = 1 if not match_statistics["target_start_valid"] else 0 + best_ends[3] = 1 if not match_statistics["target_end_valid"] else 0 if not found_match and key_reversed in truth_keys: test_overlap = reverse_record(test_overlap) for truth_interval in truth_query_intervals[key_reversed]: @@ -229,6 +236,13 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip true_positive_count += 1 found_match = True break + pct_match = match_statistics["percent_reciprocal"] + if pct_match > best_pct_match: + best_pct_match = pct_match + best_ends[0] = 1 if not match_statistics["query_start_valid"] else 0 + best_ends[1] = 1 if not match_statistics["query_end_valid"] else 0 + best_ends[2] = 1 if not match_statistics["target_start_valid"] else 0 + best_ends[3] = 1 if not match_statistics["target_end_valid"] else 0 if not found_match: for truth_interval in truth_target_intervals[key_reversed]: truth_overlap = truth_interval.data @@ -237,6 +251,13 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip true_positive_count += 1 found_match = True break + pct_match = match_statistics["percent_reciprocal"] + if pct_match > best_pct_match: + best_pct_match = pct_match + best_ends[0] = 1 if not match_statistics["query_start_valid"] else 0 + best_ends[1] = 1 if not match_statistics["query_end_valid"] else 0 + best_ends[2] = 1 if not match_statistics["target_start_valid"] else 0 + best_ends[3] = 1 if not match_statistics["target_end_valid"] else 0 incorrect_query_start += best_ends[0] incorrect_query_end += best_ends[1] incorrect_target_start += best_ends[2] From 1376dd5175d031cac9fdb668c6316a5822e232eb Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 16 Sep 2020 14:45:16 -0700 Subject: [PATCH 077/281] [cudaextender] Fix parser copy-pasta; Remove debug prints --- cudaextender/samples/device_sample.cpp | 24 +++++++++++----------- cudaextender/src/ungapped_xdrop.cu | 10 --------- cudaextender/src/ungapped_xdrop_kernels.cu | 4 ---- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp index 8215aeaee..aff093bcc 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/device_sample.cpp @@ -26,16 +26,16 @@ using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::cudaextender; -constexpr int8_t A_NT = 0; -constexpr int8_t C_NT = 1; -constexpr int8_t G_NT = 2; -constexpr int8_t T_NT = 3; -constexpr int8_t L_NT = 4; -constexpr int8_t N_NT = 5; -constexpr int8_t X_NT = 6; -constexpr int8_t E_NT = 7; -constexpr int8_t NUC = 8; -constexpr int8_t NUC2 = NUC * NUC; +constexpr char A_NT = 0; +constexpr char C_NT = 1; +constexpr char G_NT = 2; +constexpr char T_NT = 3; +constexpr char L_NT = 4; +constexpr char N_NT = 5; +constexpr char X_NT = 6; +constexpr char E_NT = 7; +constexpr char NUC = 8; +constexpr char NUC2 = NUC * NUC; // Really simple parser with no error checks void parse_seed_pairs(const std::string& filepath, std::vector& seed_pairs) @@ -50,7 +50,7 @@ void parse_seed_pairs(const std::string& filepath, std::vector& seed_p { SeedPair seed_pair; seed_pair.target_position_in_read = std::atoi(line.c_str()); - std::getline(seed_pair_file, line, ','); // Get the next value + std::getline(seed_pair_file, line); // Get the next value seed_pair.query_position_in_read = std::atoi(line.c_str()); seed_pairs.push_back(seed_pair); } @@ -108,7 +108,7 @@ int main(int argc, char* argv[]) // results in // the passed vector parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); - std::cout << h_seed_pairs[0].query_position_in_read << std::endl; + std::cout <<"Number of seed pairs: "< 0) { compress_output<<<1024, 1024, 0, stream_>>>(d_done_raw_pointer, @@ -125,22 +124,13 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length d_tmp_hsp_.begin() + num_scored_segment_pairs, d_hsp_.begin() + total_scored_segment_pairs_, scored_segment_pair_equal()); - std::cout<<"Before: "< h_vec = d_hsp_; -// - // std::cout<<"Ouptut: "<= score_threshold) { d_scored_segment[hid].seed_pair.target_position_in_read = ref_loc[warp_id] - left_extent[warp_id]; From 017b58d243a346884f432da3ae121ff6c8ef44eb Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 16 Sep 2020 17:37:47 -0700 Subject: [PATCH 078/281] [cudaextender] replaced thrust with cub --- .../genomeworks/cudaextender/extender.hpp | 5 + cudaextender/samples/device_sample.cpp | 11 ++- cudaextender/src/ungapped_xdrop.cu | 94 ++++++++++++------- cudaextender/src/ungapped_xdrop.cuh | 8 +- 4 files changed, 82 insertions(+), 36 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index 42c25b89b..652362a06 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -44,6 +44,11 @@ typedef struct ScoredSegmentPair int32_t length; /// score of the segment int32_t score; + + __host__ __device__ bool operator==(const ScoredSegmentPair& other) const + { + return ((seed_pair.target_position_in_read == other.seed_pair.target_position_in_read) && (seed_pair.query_position_in_read == other.seed_pair.query_position_in_read) && (length == other.length) && (score == other.score)); + } } ScoredSegmentPair; /// CUDA Extension object diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp index aff093bcc..4d7461041 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/device_sample.cpp @@ -167,7 +167,16 @@ int main(int argc, char* argv[]) //Get results std::cout << "h_num_hsps=" << h_num_hsps << std::endl; - + ScoredSegmentPair *h_ssps =(ScoredSegmentPair*) malloc(h_num_hsps*sizeof(ScoredSegmentPair)); + cudaMemcpy(h_ssps, d_hsps, h_num_hsps*sizeof(ScoredSegmentPair), cudaMemcpyDeviceToHost); +// for (int i=0; i #include #include +#include +#include namespace claraparabricks { @@ -62,15 +64,30 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd // FIXME - Pinned host memory registration for proper async behavior device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_, stream_); - d_done_.resize(batch_max_ungapped_extensions_, 0); ScoredSegmentPair dummy_zero_pair = {{0, 0}, 0, 0}; - d_tmp_hsp_.resize(batch_max_ungapped_extensions_, dummy_zero_pair); - d_hsp_.resize(max_seed_pairs, dummy_zero_pair); - //GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, max_ungapped_extensions_ * sizeof(int32_t))); - //GW_CU_CHECK_ERR(cudaMalloc((void**)&d_tmp_hsp_, max_ungapped_extensions_ * sizeof(ScoredSegmentPair))); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, batch_max_ungapped_extensions_ * sizeof(int32_t))); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_tmp_ssp_, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair))); + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_, 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_, 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); + + // Experimental cub usage + // Get size of temporary storage + int32_t* num_out; + size_t temp_storage = 0; + storage_ = 0; + cub::DeviceSelect::Unique(NULL, temp_storage, d_tmp_ssp_, d_tmp_ssp_, num_out, batch_max_ungapped_extensions_); + storage_ = std::max(temp_storage, storage_); + std::cout<<"Storage requirement: "<< storage_<>>(d_target, target_length, @@ -101,36 +117,51 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length d_seed_pairs, curr_num_pairs, seed_pair_start, - thrust::raw_pointer_cast(d_hsp_.data()), - d_done_raw_pointer); + d_scored_segment_pairs, + d_done_); // TODO- Make thrust use caching allocator or change kernel - thrust::inclusive_scan(thrust::cuda::par.on(stream_), d_done_.begin(), d_done_.begin() + curr_num_pairs, d_done_.begin()); + thrust::device_ptr d_done_dev_ptr(d_done_); + //cudaStreamSynchronize(stream_); + //auto start = std::chrono::high_resolution_clock::now(); + //thrust::inclusive_scan(thrust::cuda::par.on(stream_), d_done_dev_ptr, d_done_dev_ptr + curr_num_pairs, d_done_dev_ptr); + cub::DeviceScan::InclusiveSum(d_temp_storage_unique_, storage_, d_done_, d_done_, curr_num_pairs, stream_); // TODO- Make async - device_copy_n((d_done_raw_pointer + curr_num_pairs - 1), 1, &num_scored_segment_pairs, stream_); + device_copy_n((d_done_ + curr_num_pairs - 1), 1, &num_scored_segment_pairs, stream_); cudaStreamSynchronize(stream_); + // auto stop = std::chrono::high_resolution_clock::now(); + //auto duration = std::chrono::duration_cast(stop - start); + //std::cout<<"Duration: "< 0) { - compress_output<<<1024, 1024, 0, stream_>>>(d_done_raw_pointer, + compress_output<<<1024, 1024, 0, stream_>>>(d_done_, seed_pair_start, - thrust::raw_pointer_cast(d_hsp_.data()), - thrust::raw_pointer_cast(d_tmp_hsp_.data()), + d_scored_segment_pairs, + d_tmp_ssp_, curr_num_pairs); // TODO- Need configurability for kernel? + thrust::device_ptr d_tmp_hsp_dev_ptr(d_tmp_ssp_); thrust::stable_sort(thrust::cuda::par.on(stream_), - d_tmp_hsp_.begin(), - d_tmp_hsp_.begin() + num_scored_segment_pairs, + d_tmp_hsp_dev_ptr, + d_tmp_hsp_dev_ptr + num_scored_segment_pairs, scored_segment_pair_comp()); - thrust::device_vector::iterator result_end = thrust::unique_copy(thrust::cuda::par.on(stream_), - d_tmp_hsp_.begin(), - d_tmp_hsp_.begin() + num_scored_segment_pairs, - d_hsp_.begin() + total_scored_segment_pairs_, - scored_segment_pair_equal()); - num_scored_segment_pairs = thrust::distance(d_hsp_.begin()+total_scored_segment_pairs_, result_end); + thrust::device_ptr d_scored_segment_pairs_dev_ptr(d_scored_segment_pairs); + +// thrust::device_ptr result_end = thrust::unique_copy(thrust::cuda::par.on(stream_), +// d_tmp_hsp_dev_ptr, +// d_tmp_hsp_dev_ptr + num_scored_segment_pairs, +// d_scored_segment_pairs_dev_ptr + total_scored_segment_pairs_, +// scored_segment_pair_equal()); + cub::DeviceSelect::Unique(d_temp_storage_unique_,storage_, d_tmp_ssp_, d_scored_segment_pairs+total_scored_segment_pairs_, d_num_hsps_temp, num_scored_segment_pairs, stream_); + num_scored_segment_pairs = get_value_from_device(d_num_hsps_temp, stream_); + + //num_scored_segment_pairs = thrust::distance(d_scored_segment_pairs_dev_ptr + total_scored_segment_pairs_, result_end); total_scored_segment_pairs_ += num_scored_segment_pairs; } - } num_scored_segment_pairs = total_scored_segment_pairs_; - thrust::host_vector h_vec = d_hsp_; + auto stop1 = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(stop1 - start1); + std::cout<<"Duration: "< scored_segment_pairs_; int32_t batch_max_ungapped_extensions_; int32_t* d_sub_mat_; // Pointer to device substitution matrix - thrust::device_vector d_done_; // TODO- Rename scratch space - thrust::device_vector d_tmp_hsp_; // TODO- Rename Scratch space 2 - thrust::device_vector d_hsp_; // TODO- Rename output variable + int32_t* d_done_; // TODO- Rename scratch space + ScoredSegmentPair* d_tmp_ssp_; // TODO- Rename Scratch space 2 int32_t total_scored_segment_pairs_; + void* d_temp_storage_unique_; // temporary storage for cub functions + int32_t* d_num_hsps_temp; // TODO - move out to sample + size_t storage_; }; } // namespace cudaextender From 1c3e199ea83af64b207ef61be0dd63740ffa347d Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 17 Sep 2020 16:15:37 +0200 Subject: [PATCH 079/281] [common] Allocators' constructors are explicit --- .../include/claraparabricks/genomeworks/utils/allocator.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp index 81742feb2..ddc0b1d4b 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp @@ -45,7 +45,7 @@ class CudaMallocAllocator /// \brief default constructor /// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead, ignored in this allocator - CudaMallocAllocator(const std::vector& default_streams = {{0}}) + explicit CudaMallocAllocator(const std::vector& default_streams = {{0}}) { static_cast(default_streams); } @@ -177,8 +177,8 @@ class CachingDeviceAllocator /// \brief Constructor /// \param max_cached_bytes max bytes used by memory resource /// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead - CachingDeviceAllocator(size_t max_cached_bytes, - const std::vector& default_streams = {{0}}) + explicit CachingDeviceAllocator(size_t max_cached_bytes, + const std::vector& default_streams = {{0}}) : memory_resource_(std::make_shared(max_cached_bytes)) , default_streams_(default_streams) { From 5b253df6f9cb6a616547ce61e5ded6cfc50b2801 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 17 Sep 2020 16:35:37 +0200 Subject: [PATCH 080/281] [cudamapper] Renamed ArrayView -> Span --- .../genomeworks/cudamapper/index.hpp | 18 +++++++------- cudamapper/src/index_host_copy.cu | 12 +++++----- cudamapper/src/index_host_copy.cuh | 24 +++++++++---------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index a9d106d71..cb45cf105 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -135,7 +135,7 @@ class Index } /// \brief checks if index is ready to be used, index might not be ready if its creation is asynchronous - /// \return whether the index is ready to be used + /// \return true if the index is ready to be used virtual bool is_ready() const = 0; /// \brief if is_ready() is true returns immediately, blocks until it becomes ready otherwise @@ -170,10 +170,10 @@ class Index class IndexHostCopyBase { public: - /// ArrayView - helper struct that provides a view of part of underlying array + /// Span - helper struct that provides a view of part of underlying array /// \tparam T type of data in this view template - struct ArrayView + struct Span { /// pointer to first element in view T* data; @@ -196,27 +196,27 @@ class IndexHostCopyBase /// \brief returns an array of representations of sketch elements (stored on host) /// \return an array of representations of sketch elements - virtual const ArrayView representations() const = 0; + virtual const Span representations() const = 0; /// \brief returns an array of reads ids for sketch elements (stored on host) /// \return an array of reads ids for sketch elements - virtual const ArrayView read_ids() const = 0; + virtual const Span read_ids() const = 0; /// \brief returns an array of starting positions of sketch elements in their reads (stored on host) /// \return an array of starting positions of sketch elements in their reads - virtual const ArrayView positions_in_reads() const = 0; + virtual const Span positions_in_reads() const = 0; /// \brief returns an array of directions in which sketch elements were read (stored on host) /// \return an array of directions in which sketch elements were read - virtual const ArrayView directions_of_reads() const = 0; + virtual const Span directions_of_reads() const = 0; /// \brief returns an array where each representation is recorded only once, sorted by representation (stored on host) /// \return an array where each representation is recorded only once, sorted by representation - virtual const ArrayView unique_representations() const = 0; + virtual const Span unique_representations() const = 0; /// \brief returns first occurrence of corresponding representation from unique_representations(), plus one more element with the total number of sketch elements (stored on host) /// \return first occurrence of corresponding representation from unique_representations(), plus one more element with the total number of sketch elements - virtual const ArrayView first_occurrence_of_representations() const = 0; + virtual const Span first_occurrence_of_representations() const = 0; /// \brief returns number of reads in input data /// \return number of reads in input data diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 847331e32..191e6a4b9 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -135,32 +135,32 @@ std::unique_ptr IndexHostCopy::copy_index_to_device(DefaultDeviceAllocato // no stream synchronization, synchronization done in finish_copying() } -const IndexHostCopyBase::ArrayView IndexHostCopy::representations() const +const IndexHostCopyBase::Span IndexHostCopy::representations() const { return representations_; } -const IndexHostCopyBase::ArrayView IndexHostCopy::read_ids() const +const IndexHostCopyBase::Span IndexHostCopy::read_ids() const { return read_ids_; } -const IndexHostCopyBase::ArrayView IndexHostCopy::positions_in_reads() const +const IndexHostCopyBase::Span IndexHostCopy::positions_in_reads() const { return positions_in_reads_; } -const IndexHostCopyBase::ArrayView IndexHostCopy::directions_of_reads() const +const IndexHostCopyBase::Span IndexHostCopy::directions_of_reads() const { return directions_of_reads_; } -const IndexHostCopyBase::ArrayView IndexHostCopy::unique_representations() const +const IndexHostCopyBase::Span IndexHostCopy::unique_representations() const { return unique_representations_; } -const IndexHostCopyBase::ArrayView IndexHostCopy::first_occurrence_of_representations() const +const IndexHostCopyBase::Span IndexHostCopy::first_occurrence_of_representations() const { return first_occurrence_of_representations_; } diff --git a/cudamapper/src/index_host_copy.cuh b/cudamapper/src/index_host_copy.cuh index b4a5f5b4e..8aec285d5 100644 --- a/cudamapper/src/index_host_copy.cuh +++ b/cudamapper/src/index_host_copy.cuh @@ -64,27 +64,27 @@ public: /// \brief returns an array of representations of sketch elements (stored on host) /// \return an array of representations of sketch elements - const ArrayView representations() const override; + const Span representations() const override; /// \brief returns an array of reads ids for sketch elements (stored on host) /// \return an array of reads ids for sketch elements - const ArrayView read_ids() const override; + const Span read_ids() const override; /// \brief returns an array of starting positions of sketch elements in their reads (stored on host) /// \return an array of starting positions of sketch elements in their reads - const ArrayView positions_in_reads() const override; + const Span positions_in_reads() const override; /// \brief returns an array of directions in which sketch elements were read (stored on host) /// \return an array of directions in which sketch elements were read - const ArrayView directions_of_reads() const override; + const Span directions_of_reads() const override; /// \brief returns an array where each representation is recorded only once, sorted by representation (stored on host) /// \return an array where each representation is recorded only once, sorted by representation - const ArrayView unique_representations() const override; + const Span unique_representations() const override; /// \brief returns first occurrence of corresponding representation from unique_representations(), plus one more element with the total number of sketch elements (stored on host) /// \return first occurrence of corresponding representation from unique_representations(), plus one more element with the total number of sketch elements - const ArrayView first_occurrence_of_representations() const override; + const Span first_occurrence_of_representations() const override; /// \brief returns number of reads in input data /// \return number of reads in input data @@ -140,12 +140,12 @@ private: // use a single underlying vector in order to reduce memory fragmentation when using pool allocators std::vector underlying_array_; - ArrayView representations_; - ArrayView read_ids_; - ArrayView positions_in_reads_; - ArrayView directions_of_reads_; - ArrayView unique_representations_; - ArrayView first_occurrence_of_representations_; + Span representations_; + Span read_ids_; + Span positions_in_reads_; + Span directions_of_reads_; + Span unique_representations_; + Span first_occurrence_of_representations_; mutable IndexHostMemoryPinner memory_pinner_; From 3148200dcdf106eeb534f5d5b27000f76cbf77ce Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 17 Sep 2020 16:41:32 +0200 Subject: [PATCH 081/281] [cudamapper] Removed forward declaration for IndexHostMemoryPinner --- cudamapper/src/index_host_copy.cuh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cudamapper/src/index_host_copy.cuh b/cudamapper/src/index_host_copy.cuh index 8aec285d5..f474795d2 100644 --- a/cudamapper/src/index_host_copy.cuh +++ b/cudamapper/src/index_host_copy.cuh @@ -27,11 +27,6 @@ namespace genomeworks namespace cudamapper { -namespace details -{ -class IndexHostMemoryPinner; -} // namespace details - /// IndexHostCopy - Creates and maintains a copy of computed IndexGPU elements on the host /// /// From 2fc0a1b65840cd1a4c529ca4c28f82e8e529e36a Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 17 Sep 2020 16:44:28 +0200 Subject: [PATCH 082/281] [cudamapper] Removed unnecessary constructors/destructor from IndexNotFoundException --- cudamapper/src/index_cache.cuh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index 4f94fe3b0..4128ad36c 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -242,12 +242,6 @@ public: IndexNotFoundException(IndexDescriptor index_descriptor, IndexType index_type); - IndexNotFoundException(const IndexNotFoundException&) = default; - IndexNotFoundException& operator=(const IndexNotFoundException&) = default; - IndexNotFoundException(IndexNotFoundException&&) = default; - IndexNotFoundException& operator=(IndexNotFoundException&&) = default; - virtual ~IndexNotFoundException() = default; - /// Returns the error message of the exception virtual const char* what() const noexcept; From e28eab19f017f2d3b09758297a319ef917278bc4 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 17 Sep 2020 16:50:42 +0200 Subject: [PATCH 083/281] [cudamapper] calling .clear() instead of .empty() --- cudamapper/src/index_cache.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index cf66d63a3..067639fda 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -129,7 +129,7 @@ void IndexCache::generate_content_host(const std::vector& descr end(descriptors_of_indices_to_keep_on_device)); host_cache_t new_cache; - indices_kept_on_device.empty(); // normally this should be empty anyway + indices_kept_on_device.clear(); // normally this should be empty anyway // In most cases index is generated on device and then moved to host. These two operations can be overlapped, i.e. while one index is being copied // to host the next index can be generated. From 2d93a55154cfeac4f5e04803b4af93beab7444bc Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 17 Sep 2020 16:53:57 +0200 Subject: [PATCH 084/281] [cudamapper] Taking references instead of values while looping over cache --- cudamapper/src/index_cache.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 067639fda..b57fb2e9e 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -318,7 +318,7 @@ void IndexCache::finish_generating_content_device(const CacheSelector which_cach device_cache_t& this_device_cache = (CacheSelector::query_cache == which_cache) ? query_device_cache_ : target_device_cache_; device_cache_t& new_cache = (CacheSelector::query_cache == which_cache) ? next_query_device_cache_ : next_target_device_cache_; - for (const auto device_index : new_cache) + for (const auto& device_index : new_cache) { device_index.second->wait_to_be_ready(); } From 468e345abea820ebcdbbeaed51ba58fe2891a98b Mon Sep 17 00:00:00 2001 From: atadkase Date: Thu, 17 Sep 2020 16:28:18 -0700 Subject: [PATCH 085/281] [cudaextender] Clean API working sample --- .../genomeworks/cudaextender/extender.hpp | 2 +- cudaextender/samples/device_sample.cpp | 93 ++++++++++++++----- cudaextender/src/ungapped_xdrop.cu | 86 +++++++---------- cudaextender/src/ungapped_xdrop.cuh | 7 +- 4 files changed, 104 insertions(+), 84 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index 652362a06..79f70729c 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -79,7 +79,7 @@ class Extender const char* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, - int32_t& num_scored_segment_pairs) = 0; + int32_t* d_num_scored_segment_pairs) = 0; /// \brief Waits for CUDA accelerated extension to finish /// diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp index 4d7461041..0c44b127a 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/device_sample.cpp @@ -87,6 +87,31 @@ int main(int argc, char* argv[]) const int32_t xdrop_threshold = 910; const bool input_no_entropy = false; const int32_t score_threshold = 3000; + char c; + bool print=false, help=false; + while ((c = getopt(argc, argv, "p")) != -1) + { + switch (c) + { + case 'p': + print = true; + break; + case 'h': + default: + help = true; + break; + } + } + + if(help) + { + std::cout << "CUDAExtender API sample program. Runs ungapped extender on canned data." << std::endl; + std::cout << "-p : Print the Scored Segment Pair output to stdout" << std::endl; + std::cout << "-h : Print help message" << std::endl; + std::exit(0); + } + + // Fasta query and target files std::string target_file_path = "../data/example.fa"; std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); @@ -108,7 +133,7 @@ int main(int argc, char* argv[]) // results in // the passed vector parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); - std::cout <<"Number of seed pairs: "<(device_prop.totalGlobalMem) / 1073741824.0f; batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; - int32_t max_seed_pairs = static_cast(global_mem_gb) * max_seed_pairs_per_gb; + const int32_t max_seed_pairs = static_cast(global_mem_gb) * max_seed_pairs_per_gb; // Switch to device for copying over initial structures scoped_device_switch dev(device_id_); - // Allocate space on device for scoring matrix and - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_sub_mat_, sub_mat_dim_ * sizeof(int32_t))); - // FIXME - Pinned host memory registration for proper async behavior - device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_, stream_); + //Figure out memory requirements for cub functions + int32_t* dummy_num_out; + size_t temp_storage_bytes = 0; + cub_storage_bytes_ = 0; + cub::DeviceSelect::Unique(NULL, temp_storage_bytes, d_tmp_ssp_, d_tmp_ssp_, dummy_num_out, batch_max_ungapped_extensions_); + cub::DeviceScan::InclusiveSum(NULL, cub_storage_bytes_, d_done_,d_done_+batch_max_ungapped_extensions_, batch_max_ungapped_extensions_); + cub_storage_bytes_ = std::max(temp_storage_bytes, cub_storage_bytes_); - ScoredSegmentPair dummy_zero_pair = {{0, 0}, 0, 0}; + + // Allocate space on device for scoring matrix and intermediate results + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_sub_mat_, sub_mat_dim_ * sizeof(int32_t))); GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, batch_max_ungapped_extensions_ * sizeof(int32_t))); GW_CU_CHECK_ERR(cudaMalloc((void**)&d_tmp_ssp_, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair))); + // Allocate temporary storage for cub + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_temp_storage_cub_, cub_storage_bytes_)); + + // Requires pinned host memory registration for proper async behavior + device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_, stream_); GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_, 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_, 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); - // Experimental cub usage - // Get size of temporary storage - int32_t* num_out; - size_t temp_storage = 0; - storage_ = 0; - cub::DeviceSelect::Unique(NULL, temp_storage, d_tmp_ssp_, d_tmp_ssp_, num_out, batch_max_ungapped_extensions_); - storage_ = std::max(temp_storage, storage_); - std::cout<<"Storage requirement: "<< storage_< d_done_dev_ptr(d_done_); - //cudaStreamSynchronize(stream_); - //auto start = std::chrono::high_resolution_clock::now(); - //thrust::inclusive_scan(thrust::cuda::par.on(stream_), d_done_dev_ptr, d_done_dev_ptr + curr_num_pairs, d_done_dev_ptr); - cub::DeviceScan::InclusiveSum(d_temp_storage_unique_, storage_, d_done_, d_done_, curr_num_pairs, stream_); + cub::DeviceScan::InclusiveSum(d_temp_storage_cub_, cub_storage_bytes_, d_done_, d_done_, curr_num_pairs, stream_); // TODO- Make async - device_copy_n((d_done_ + curr_num_pairs - 1), 1, &num_scored_segment_pairs, stream_); - cudaStreamSynchronize(stream_); - // auto stop = std::chrono::high_resolution_clock::now(); - //auto duration = std::chrono::duration_cast(stop - start); - //std::cout<<"Duration: "< 0) { compress_output<<<1024, 1024, 0, stream_>>>(d_done_, @@ -140,28 +122,24 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length d_tmp_ssp_, curr_num_pairs); // TODO- Need configurability for kernel? thrust::device_ptr d_tmp_hsp_dev_ptr(d_tmp_ssp_); + // TODO- Make thrust use caching allocator or change kernel thrust::stable_sort(thrust::cuda::par.on(stream_), d_tmp_hsp_dev_ptr, d_tmp_hsp_dev_ptr + num_scored_segment_pairs, scored_segment_pair_comp()); thrust::device_ptr d_scored_segment_pairs_dev_ptr(d_scored_segment_pairs); - -// thrust::device_ptr result_end = thrust::unique_copy(thrust::cuda::par.on(stream_), -// d_tmp_hsp_dev_ptr, -// d_tmp_hsp_dev_ptr + num_scored_segment_pairs, -// d_scored_segment_pairs_dev_ptr + total_scored_segment_pairs_, -// scored_segment_pair_equal()); - cub::DeviceSelect::Unique(d_temp_storage_unique_,storage_, d_tmp_ssp_, d_scored_segment_pairs+total_scored_segment_pairs_, d_num_hsps_temp, num_scored_segment_pairs, stream_); - num_scored_segment_pairs = get_value_from_device(d_num_hsps_temp, stream_); - - //num_scored_segment_pairs = thrust::distance(d_scored_segment_pairs_dev_ptr + total_scored_segment_pairs_, result_end); + cub::DeviceSelect::Unique(d_temp_storage_cub_, + cub_storage_bytes_, + d_tmp_ssp_, + d_scored_segment_pairs+total_scored_segment_pairs_, + d_num_scored_segment_pairs, + num_scored_segment_pairs, + stream_); + num_scored_segment_pairs = get_value_from_device(d_num_scored_segment_pairs, stream_); total_scored_segment_pairs_ += num_scored_segment_pairs; } } - num_scored_segment_pairs = total_scored_segment_pairs_; - auto stop1 = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(stop1 - start1); - std::cout<<"Duration: "< Date: Thu, 17 Sep 2020 17:32:36 -0700 Subject: [PATCH 086/281] [cudaextender] Add host sample --- cudaextender/samples/device_sample.cpp | 1 + cudaextender/src/ungapped_xdrop.cu | 94 ++++++++++++++++++-------- cudaextender/src/ungapped_xdrop.cuh | 17 ++++- 3 files changed, 80 insertions(+), 32 deletions(-) diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp index 0c44b127a..cf9e5ed84 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/device_sample.cpp @@ -128,6 +128,7 @@ int main(int argc, char* argv[]) // target_position_in_read_ std::string seed_pairs_file_path = "../data/example_hits.csv"; + //TODO - pinned seed_pairs std::vector h_seed_pairs; // Following function loops through all seed_pairs in the example_seed_pairs.csv and returns // results in diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 28c4d0eaa..0b5ce2aa1 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -47,27 +47,26 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd , stream_(stream) , device_id_(device_id) { + //TODO - Check bounds // Calculate the max limits on the number of extensions we can do on // this GPU cudaDeviceProp device_prop; cudaGetDeviceProperties(&device_prop, device_id_); const int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures const int32_t max_seed_pairs_per_gb = 8388608; // FIXME: Calculate using sizeof datastructures - const float global_mem_gb = static_cast(device_prop.totalGlobalMem) / 1073741824.0f; - batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; - const int32_t max_seed_pairs = static_cast(global_mem_gb) * max_seed_pairs_per_gb; + const float global_mem_gb = static_cast(device_prop.totalGlobalMem) / 1073741824.0f; + batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; + const int32_t max_seed_pairs = static_cast(global_mem_gb) * max_seed_pairs_per_gb; // Switch to device for copying over initial structures scoped_device_switch dev(device_id_); //Figure out memory requirements for cub functions - int32_t* dummy_num_out; size_t temp_storage_bytes = 0; - cub_storage_bytes_ = 0; - cub::DeviceSelect::Unique(NULL, temp_storage_bytes, d_tmp_ssp_, d_tmp_ssp_, dummy_num_out, batch_max_ungapped_extensions_); - cub::DeviceScan::InclusiveSum(NULL, cub_storage_bytes_, d_done_,d_done_+batch_max_ungapped_extensions_, batch_max_ungapped_extensions_); + cub_storage_bytes_ = 0; + GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(nullptr, temp_storage_bytes, d_tmp_ssp_, d_tmp_ssp_, (int32_t*)nullptr, batch_max_ungapped_extensions_)); + GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(nullptr, cub_storage_bytes_, d_done_, d_done_ + batch_max_ungapped_extensions_, batch_max_ungapped_extensions_)); cub_storage_bytes_ = std::max(temp_storage_bytes, cub_storage_bytes_); - // Allocate space on device for scoring matrix and intermediate results GW_CU_CHECK_ERR(cudaMalloc((void**)&d_sub_mat_, sub_mat_dim_ * sizeof(int32_t))); GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, batch_max_ungapped_extensions_ * sizeof(int32_t))); @@ -79,8 +78,6 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_, stream_); GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_, 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_, 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); - - } StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, @@ -89,11 +86,12 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) { + //TODO - Check bounds // Switch to configured GPU scoped_device_switch dev(device_id_); - int32_t curr_num_pairs = 0; - int32_t num_scored_segment_pairs = 0; - total_scored_segment_pairs_ = 0; + int32_t curr_num_pairs = 0; + int32_t num_scored_segment_pairs = 0; + total_scored_segment_pairs_ = 0; for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += batch_max_ungapped_extensions_) { curr_num_pairs = std::min(batch_max_ungapped_extensions_, num_seed_pairs - seed_pair_start); @@ -111,7 +109,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length seed_pair_start, d_scored_segment_pairs, d_done_); - cub::DeviceScan::InclusiveSum(d_temp_storage_cub_, cub_storage_bytes_, d_done_, d_done_, curr_num_pairs, stream_); + GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(d_temp_storage_cub_, cub_storage_bytes_, d_done_, d_done_, curr_num_pairs, stream_)); // TODO- Make async num_scored_segment_pairs = get_value_from_device(d_done_ + curr_num_pairs - 1, stream_); if (num_scored_segment_pairs > 0) @@ -127,14 +125,13 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length d_tmp_hsp_dev_ptr, d_tmp_hsp_dev_ptr + num_scored_segment_pairs, scored_segment_pair_comp()); - thrust::device_ptr d_scored_segment_pairs_dev_ptr(d_scored_segment_pairs); - cub::DeviceSelect::Unique(d_temp_storage_cub_, - cub_storage_bytes_, - d_tmp_ssp_, - d_scored_segment_pairs+total_scored_segment_pairs_, - d_num_scored_segment_pairs, - num_scored_segment_pairs, - stream_); + GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(d_temp_storage_cub_, + cub_storage_bytes_, + d_tmp_ssp_, + d_scored_segment_pairs + total_scored_segment_pairs_, + d_num_scored_segment_pairs, + num_scored_segment_pairs, + stream_)); num_scored_segment_pairs = get_value_from_device(d_num_scored_segment_pairs, stream_); total_scored_segment_pairs_ += num_scored_segment_pairs; } @@ -143,31 +140,70 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length return success; } -const std::vector& UngappedXDrop::get_scored_segment_pairs() const -{ - std::vector placeholder; - return placeholder; -} - StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length, const char* h_target, int32_t target_length, int32_t score_threshold, std::vector& h_seed_pairs) { + // Allocate space on device for target and query sequences, seed_pairs, + // high scoring segment pairs (ssp) and num_ssp. + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_query_, sizeof(char) * query_length)); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_target_, sizeof(char) * target_length)); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_seed_pairs_, sizeof(SeedPair) * h_seed_pairs.size())); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_ssp_, sizeof(ScoredSegmentPair) * h_seed_pairs.size())); + + // Async Memcopy all the input values to device + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query_, h_query, sizeof(char) * query_length, + cudaMemcpyHostToDevice, stream_)); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target_, h_target, sizeof(char) * target_length, + cudaMemcpyHostToDevice, stream_)); + GW_CU_CHECK_ERR(cudaMemcpyAsync(d_seed_pairs_, &h_seed_pairs[0], sizeof(SeedPair) * h_seed_pairs.size(), cudaMemcpyHostToDevice, + stream_)); + + // Launch the ungapped extender device function + if (!extend_async(d_query_, query_length, d_target_, target_length, score_threshold, d_seed_pairs_, h_seed_pairs.size(), d_ssp_, d_num_ssp_)) + { + GW_LOG_ERROR("Error running cudaextender"); + } + return success; } StatusType UngappedXDrop::sync() { + h_num_ssp_ = get_value_from_device(d_num_ssp_, stream_); + if (h_num_ssp_ > 0) + { + h_ssp_.resize(h_num_ssp_); + device_copy_n(d_ssp_, h_num_ssp_, &h_ssp_[0], stream_); + cudaStreamSynchronize(stream_); + } + return success; } -void UngappedXDrop::reset(){ +const std::vector& UngappedXDrop::get_scored_segment_pairs() const +{ + return h_ssp_; +} +void UngappedXDrop::reset() +{ + // TODO - Add flag for host ptr mode + // TODO - Add checks for prev free + h_ssp_.clear(); + GW_CU_CHECK_ERR(cudaFree(d_query_)); + GW_CU_CHECK_ERR(cudaFree(d_target_)); + GW_CU_CHECK_ERR(cudaFree(d_seed_pairs_)); + GW_CU_CHECK_ERR(cudaFree(d_num_ssp_)); + GW_CU_CHECK_ERR(cudaFree(d_ssp_)); }; UngappedXDrop::~UngappedXDrop() { + // TODO - Check flag for host pointer mode + reset(); GW_CU_CHECK_ERR(cudaFree(d_sub_mat_)); GW_CU_CHECK_ERR(cudaFree(d_tmp_ssp_)); GW_CU_CHECK_ERR(cudaFree(d_done_)); diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index ecdafb8ed..37f63c89c 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -51,6 +51,8 @@ public: const std::vector& get_scored_segment_pairs() const override; private: + // TODO - Smart pointers + // Device ptr required variables int32_t* h_sub_mat_; int32_t sub_mat_dim_; // Assume matrix is square int32_t xdrop_threshold_; @@ -59,12 +61,21 @@ private: int32_t device_id_; std::vector scored_segment_pairs_; int32_t batch_max_ungapped_extensions_; - int32_t* d_sub_mat_; // Pointer to device substitution matrix - int32_t* d_done_; // TODO- Rename scratch space - ScoredSegmentPair* d_tmp_ssp_; // TODO- Rename Scratch space 2 + int32_t* d_sub_mat_; // Pointer to device substitution matrix + int32_t* d_done_; // TODO- Rename scratch space + ScoredSegmentPair* d_tmp_ssp_; // TODO- Rename Scratch space 2 int32_t total_scored_segment_pairs_; void* d_temp_storage_cub_; // temporary storage for cub functions size_t cub_storage_bytes_; + + // Host ptr additional required variables + char *d_query_, *d_target_; + SeedPair* d_seed_pairs_; + int32_t* d_num_ssp_; + ScoredSegmentPair* d_ssp_; + int32_t h_num_ssp_; + std::vector h_ssp_; + }; } // namespace cudaextender From 2712e5257cc752aa17910e00c1e73eeaf053dd52 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 18 Sep 2020 12:07:10 +0200 Subject: [PATCH 087/281] [common] Added gw_byte_t --- common/base/include/claraparabricks/genomeworks/types.hpp | 7 +++++-- cudamapper/src/index_host_copy.cuh | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/types.hpp b/common/base/include/claraparabricks/genomeworks/types.hpp index b61d7c0e6..469699be8 100644 --- a/common/base/include/claraparabricks/genomeworks/types.hpp +++ b/common/base/include/claraparabricks/genomeworks/types.hpp @@ -21,6 +21,7 @@ #if __cplusplus >= 201703 #include #include +#include #else #include #include @@ -45,8 +46,8 @@ using position_in_read_t = std::uint32_t; using number_of_basepairs_t = position_in_read_t; // TODO: Once minimal supported GCC version is moved to GCC 7.1 -// or higher, thegw_optional_t and gw_string_view_t aliases -// can be removed and std::optional and std::string_view can +// or higher, gw_optional_t, gw_string_view_t and gw_byte_t aliases +// can be removed and std::optional, std::string_view and std::byte can // be used directly instead #if __cplusplus >= 201703 template @@ -54,12 +55,14 @@ using gw_optional_t = std::optional; using gw_nullopt_t = std::nullopt_t; constexpr gw_nullopt_t gw_nullopt = std::nullopt; using gw_string_view_t = std::string_view; +using gw_byte_t = std::byte; #else template using gw_optional_t = std::experimental::optional; using gw_nullopt_t = std::experimental::nullopt_t; constexpr gw_nullopt_t gw_nullopt = std::experimental::nullopt; using gw_string_view_t = std::experimental::string_view; +using gw_byte_t = unsigned char; #endif } // namespace genomeworks diff --git a/cudamapper/src/index_host_copy.cuh b/cudamapper/src/index_host_copy.cuh index f474795d2..17ed9918d 100644 --- a/cudamapper/src/index_host_copy.cuh +++ b/cudamapper/src/index_host_copy.cuh @@ -133,7 +133,7 @@ private: }; // use a single underlying vector in order to reduce memory fragmentation when using pool allocators - std::vector underlying_array_; + std::vector underlying_array_; Span representations_; Span read_ids_; From fa285b90c723b2779049db1a15fc6a2a8eaa9283 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 18 Sep 2020 12:07:45 +0200 Subject: [PATCH 088/281] [cudamapper] Improved alignment in IndexHostCopy --- cudamapper/src/index_host_copy.cu | 56 ++++++++++++++++--------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 191e6a4b9..671858e22 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -43,38 +43,40 @@ IndexHostCopy::IndexHostCopy(const Index& index, GW_NVTX_RANGE(profiler, "index_host_copy::constructor"); // Use only one large array to store all arrays in order to reduce fragmentation when using pool allocators - // Align all arrays by 64 bits - const std::size_t representations_bits = claraparabricks::genomeworks::ceiling_divide(index.representations().size() * sizeof(representation_t), sizeof(std::uint64_t)) * sizeof(uint64_t); - const std::size_t read_ids_bits = claraparabricks::genomeworks::ceiling_divide(index.read_ids().size() * sizeof(read_id_t), sizeof(std::uint64_t)) * sizeof(uint64_t); - const std::size_t positions_in_reads_bits = claraparabricks::genomeworks::ceiling_divide(index.positions_in_reads().size() * sizeof(position_in_read_t), sizeof(std::uint64_t)) * sizeof(uint64_t); - const std::size_t directions_of_reads_bits = claraparabricks::genomeworks::ceiling_divide(index.directions_of_reads().size() * sizeof(SketchElement::DirectionOfRepresentation), sizeof(std::uint64_t)) * sizeof(uint64_t); - const std::size_t unique_representations_bits = claraparabricks::genomeworks::ceiling_divide(index.unique_representations().size() * sizeof(representation_t), sizeof(std::uint64_t)) * sizeof(uint64_t); - const std::size_t first_occurrence_of_representations_bits = claraparabricks::genomeworks::ceiling_divide(index.first_occurrence_of_representations().size() * sizeof(std::uint32_t), sizeof(std::uint64_t)) * sizeof(uint64_t); - - const std::size_t total_bits = representations_bits + - read_ids_bits + - positions_in_reads_bits + - directions_of_reads_bits + - unique_representations_bits + - first_occurrence_of_representations_bits; + // Align all arrays according to the largest type + constexpr size_t alignment_bytes = std::max({alignof(representation_t), alignof(read_id_t), alignof(position_in_read_t), alignof(SketchElement::DirectionOfRepresentation), alignof(std::uint32_t)}); + + const std::size_t representations_bytes = claraparabricks::genomeworks::ceiling_divide(index.representations().size() * sizeof(representation_t), alignment_bytes) * alignment_bytes; + const std::size_t read_ids_bytes = claraparabricks::genomeworks::ceiling_divide(index.read_ids().size() * sizeof(read_id_t), alignment_bytes) * alignment_bytes; + const std::size_t positions_in_reads_bytes = claraparabricks::genomeworks::ceiling_divide(index.positions_in_reads().size() * sizeof(position_in_read_t), alignment_bytes) * alignment_bytes; + const std::size_t directions_of_reads_bytes = claraparabricks::genomeworks::ceiling_divide(index.directions_of_reads().size() * sizeof(SketchElement::DirectionOfRepresentation), alignment_bytes) * alignment_bytes; + const std::size_t unique_representations_bytes = claraparabricks::genomeworks::ceiling_divide(index.unique_representations().size() * sizeof(representation_t), alignment_bytes) * alignment_bytes; + const std::size_t first_occurrence_of_representations_bytes = claraparabricks::genomeworks::ceiling_divide(index.first_occurrence_of_representations().size() * sizeof(std::uint32_t), alignment_bytes) * alignment_bytes; + + const std::size_t total_bytes = representations_bytes + + read_ids_bytes + + positions_in_reads_bytes + + directions_of_reads_bytes + + unique_representations_bytes + + first_occurrence_of_representations_bytes; { GW_NVTX_RANGE(profiler, "index_host_copy::constructor::allocate_host_memory"); - underlying_array_.resize(total_bits); + underlying_array_.resize(total_bytes); } - std::size_t current_bit = 0; - representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.representations().size()}; - current_bit += representations_bits; - read_ids_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.read_ids().size()}; - current_bit += read_ids_bits; - positions_in_reads_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.positions_in_reads().size()}; - current_bit += positions_in_reads_bits; - directions_of_reads_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.directions_of_reads().size()}; - current_bit += directions_of_reads_bits; - unique_representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.unique_representations().size()}; - current_bit += unique_representations_bits; - first_occurrence_of_representations_ = {reinterpret_cast(underlying_array_.data() + current_bit), index.first_occurrence_of_representations().size()}; + std::size_t current_byte = 0; + representations_ = {reinterpret_cast(underlying_array_.data() + current_byte), index.representations().size()}; + current_byte += representations_bytes; + read_ids_ = {reinterpret_cast(underlying_array_.data() + current_byte), index.read_ids().size()}; + current_byte += read_ids_bytes; + positions_in_reads_ = {reinterpret_cast(underlying_array_.data() + current_byte), index.positions_in_reads().size()}; + current_byte += positions_in_reads_bytes; + directions_of_reads_ = {reinterpret_cast(underlying_array_.data() + current_byte), index.directions_of_reads().size()}; + current_byte += directions_of_reads_bytes; + unique_representations_ = {reinterpret_cast(underlying_array_.data() + current_byte), index.unique_representations().size()}; + current_byte += unique_representations_bytes; + first_occurrence_of_representations_ = {reinterpret_cast(underlying_array_.data() + current_byte), index.first_occurrence_of_representations().size()}; // register pinned memory, memory gets unpinned in finish_copying() memory_pinner_.register_pinned_memory(); From 9e7fb2f487c5729f328c7050d04ce714ccdfc069 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 18 Sep 2020 12:57:32 +0200 Subject: [PATCH 089/281] [cudamapper] Pass cuda_stream_copy_ as well when generating Index so the arrays are associated properly --- cudamapper/src/index_cache.cu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index b57fb2e9e..eb1fefe93 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -179,7 +179,8 @@ void IndexCache::generate_content_host(const std::vector& descr window_size_, hash_representations_, filtering_parameter_, - cuda_stream_generation_); + cuda_stream_generation_, + cuda_stream_copy_); // wait for index to be generated on cuda_stream_generation_ before copying it on cuda_stream_copy_ // TODO: do this sync using an event From 52d6d69369805c4782f20edc4029a8004c571ec9 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 18 Sep 2020 13:01:12 +0200 Subject: [PATCH 090/281] [cudamapper] Fixing formatting --- cudamapper/src/index_host_copy.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 671858e22..3487264e5 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -66,7 +66,7 @@ IndexHostCopy::IndexHostCopy(const Index& index, } std::size_t current_byte = 0; - representations_ = {reinterpret_cast(underlying_array_.data() + current_byte), index.representations().size()}; + representations_ = {reinterpret_cast(underlying_array_.data() + current_byte), index.representations().size()}; current_byte += representations_bytes; read_ids_ = {reinterpret_cast(underlying_array_.data() + current_byte), index.read_ids().size()}; current_byte += read_ids_bytes; From 881bb9d15c8c931f48098e4e1f35744811b2d595 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 09:13:36 -0700 Subject: [PATCH 091/281] [cudaextender] Drop typedefs; replace memcpy with device_copy --- .../genomeworks/cudaextender/extender.hpp | 12 +++++------ cudaextender/src/ungapped_xdrop.cu | 20 +++++++------------ 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index 79f70729c..80266bad8 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -28,15 +28,15 @@ namespace genomeworks namespace cudaextender { -typedef struct SeedPair +struct SeedPair { /// position of first sketch element in query_read_id_ position_in_read_t query_position_in_read; /// position of second sketch element in target_read_id_ position_in_read_t target_position_in_read; -} SeedPair; +}; -typedef struct ScoredSegmentPair +struct ScoredSegmentPair { /// Seed for the segment SeedPair seed_pair; @@ -49,7 +49,7 @@ typedef struct ScoredSegmentPair { return ((seed_pair.target_position_in_read == other.seed_pair.target_position_in_read) && (seed_pair.query_position_in_read == other.seed_pair.query_position_in_read) && (length == other.length) && (score == other.score)); } -} ScoredSegmentPair; +}; /// CUDA Extension object class Extender @@ -72,9 +72,9 @@ class Extender /// \brief Device pointer prototype for extension /// /// Memcopies to device memory are assumed to be done before this - /// function. Output array d_hsp_out must be pre-allocated on device. + /// function. Output array d_scored_segment_pairs must be pre-allocated on device. /// Launches async extension kernel. Filters segments on device - /// based on input_hspthresh. + /// based on score_threshold. virtual StatusType extend_async(const char* d_query, int32_t query_length, const char* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 0b5ce2aa1..015ba5f0b 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -89,12 +89,10 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length //TODO - Check bounds // Switch to configured GPU scoped_device_switch dev(device_id_); - int32_t curr_num_pairs = 0; - int32_t num_scored_segment_pairs = 0; total_scored_segment_pairs_ = 0; for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += batch_max_ungapped_extensions_) { - curr_num_pairs = std::min(batch_max_ungapped_extensions_, num_seed_pairs - seed_pair_start); + const int32_t curr_num_pairs = std::min(batch_max_ungapped_extensions_, num_seed_pairs - seed_pair_start); // TODO- Extricate the kernel launch params? find_high_scoring_segment_pairs<<<1024, 128, 0, stream_>>>(d_target, target_length, @@ -111,7 +109,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length d_done_); GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(d_temp_storage_cub_, cub_storage_bytes_, d_done_, d_done_, curr_num_pairs, stream_)); // TODO- Make async - num_scored_segment_pairs = get_value_from_device(d_done_ + curr_num_pairs - 1, stream_); + const int32_t num_scored_segment_pairs = get_value_from_device(d_done_ + curr_num_pairs - 1, stream_); if (num_scored_segment_pairs > 0) { compress_output<<<1024, 1024, 0, stream_>>>(d_done_, @@ -132,8 +130,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length d_num_scored_segment_pairs, num_scored_segment_pairs, stream_)); - num_scored_segment_pairs = get_value_from_device(d_num_scored_segment_pairs, stream_); - total_scored_segment_pairs_ += num_scored_segment_pairs; + total_scored_segment_pairs_ += get_value_from_device(d_num_scored_segment_pairs, stream_); } } set_device_value_async(d_num_scored_segment_pairs, &total_scored_segment_pairs_, stream_); @@ -153,13 +150,10 @@ StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))); GW_CU_CHECK_ERR(cudaMalloc((void**)&d_ssp_, sizeof(ScoredSegmentPair) * h_seed_pairs.size())); - // Async Memcopy all the input values to device - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query_, h_query, sizeof(char) * query_length, - cudaMemcpyHostToDevice, stream_)); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target_, h_target, sizeof(char) * target_length, - cudaMemcpyHostToDevice, stream_)); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_seed_pairs_, &h_seed_pairs[0], sizeof(SeedPair) * h_seed_pairs.size(), cudaMemcpyHostToDevice, - stream_)); + // Async memcopy all the input values to device + device_copy_n(h_query, query_length, d_query_, stream_); + device_copy_n(h_target, target_length, d_target_, stream_); + device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs_, stream_); // Launch the ungapped extender device function if (!extend_async(d_query_, query_length, d_target_, target_length, score_threshold, d_seed_pairs_, h_seed_pairs.size(), d_ssp_, d_num_ssp_)) From 0e534d45c4992059c7d6a30b42711497dbe88c82 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 09:16:24 -0700 Subject: [PATCH 092/281] [cudaextender] drop h_ssp_ member var --- cudaextender/src/ungapped_xdrop.cu | 8 ++++---- cudaextender/src/ungapped_xdrop.cuh | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 015ba5f0b..392d64a2e 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -166,11 +166,11 @@ StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length StatusType UngappedXDrop::sync() { - h_num_ssp_ = get_value_from_device(d_num_ssp_, stream_); - if (h_num_ssp_ > 0) + const int32_t h_num_ssp = get_value_from_device(d_num_ssp_, stream_); + if (h_num_ssp > 0) { - h_ssp_.resize(h_num_ssp_); - device_copy_n(d_ssp_, h_num_ssp_, &h_ssp_[0], stream_); + h_ssp_.resize(h_num_ssp); + device_copy_n(d_ssp_, h_num_ssp, &h_ssp_[0], stream_); cudaStreamSynchronize(stream_); } diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 37f63c89c..63c86807d 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -73,7 +73,6 @@ private: SeedPair* d_seed_pairs_; int32_t* d_num_ssp_; ScoredSegmentPair* d_ssp_; - int32_t h_num_ssp_; std::vector h_ssp_; }; From 6f380fdd78a8d6a25dc98eec2d27da2d0cf00d67 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 09:18:23 -0700 Subject: [PATCH 093/281] [cudaextender] Replace pointer access with data() --- cudaextender/src/ungapped_xdrop.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 392d64a2e..d9b74649a 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -170,7 +170,7 @@ StatusType UngappedXDrop::sync() if (h_num_ssp > 0) { h_ssp_.resize(h_num_ssp); - device_copy_n(d_ssp_, h_num_ssp, &h_ssp_[0], stream_); + device_copy_n(d_ssp_, h_num_ssp, h_ssp_.data(), stream_); cudaStreamSynchronize(stream_); } From 569bfcebca3ad9c15886a1ad6c3453ca089a104e Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 09:24:33 -0700 Subject: [PATCH 094/281] [cudaextender] Remove extra include; var declaration update; --- cudaextender/src/ungapped_xdrop.cuh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 63c86807d..06985abda 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -17,7 +17,6 @@ #pragma once #include -#include namespace claraparabricks { @@ -46,13 +45,14 @@ public: int32_t* d_num_scored_segment_pairs) override; StatusType sync() override; + void reset() override; const std::vector& get_scored_segment_pairs() const override; private: // TODO - Smart pointers - // Device ptr required variables + // Device ptr API required variables int32_t* h_sub_mat_; int32_t sub_mat_dim_; // Assume matrix is square int32_t xdrop_threshold_; @@ -68,13 +68,13 @@ private: void* d_temp_storage_cub_; // temporary storage for cub functions size_t cub_storage_bytes_; - // Host ptr additional required variables - char *d_query_, *d_target_; + // Host ptr API additional required variables + char* d_query_; + char* d_target_; SeedPair* d_seed_pairs_; int32_t* d_num_ssp_; ScoredSegmentPair* d_ssp_; std::vector h_ssp_; - }; } // namespace cudaextender From 91fdfe3a02c7d7db2f89786eb4ed0460f528931e Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 09:33:52 -0700 Subject: [PATCH 095/281] [cudaextender] Add flag for host_pointer_mode check --- cudaextender/src/ungapped_xdrop.cu | 29 +++++++++++++++++++---------- cudaextender/src/ungapped_xdrop.cuh | 1 + 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index d9b74649a..f2ffc0f67 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -46,6 +46,7 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd , no_entropy_(no_entropy) , stream_(stream) , device_id_(device_id) + , host_ptr_api_mode_(false) { //TODO - Check bounds // Calculate the max limits on the number of extensions we can do on @@ -88,6 +89,9 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length { //TODO - Check bounds // Switch to configured GPU + // If host pointer API mode was used before this mode, reset data structures + if(host_ptr_api_mode_) + reset(); scoped_device_switch dev(device_id_); total_scored_segment_pairs_ = 0; for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += batch_max_ungapped_extensions_) @@ -142,6 +146,8 @@ StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length int32_t score_threshold, std::vector& h_seed_pairs) { + // Set host pointer mode on + host_ptr_api_mode_=true; // Allocate space on device for target and query sequences, seed_pairs, // high scoring segment pairs (ssp) and num_ssp. GW_CU_CHECK_ERR(cudaMalloc((void**)&d_query_, sizeof(char) * query_length)); @@ -184,24 +190,27 @@ const std::vector& UngappedXDrop::get_scored_segment_pairs() void UngappedXDrop::reset() { - // TODO - Add flag for host ptr mode // TODO - Add checks for prev free - h_ssp_.clear(); - GW_CU_CHECK_ERR(cudaFree(d_query_)); - GW_CU_CHECK_ERR(cudaFree(d_target_)); - GW_CU_CHECK_ERR(cudaFree(d_seed_pairs_)); - GW_CU_CHECK_ERR(cudaFree(d_num_ssp_)); - GW_CU_CHECK_ERR(cudaFree(d_ssp_)); -}; + if(host_ptr_api_mode_) + { + h_ssp_.clear(); + GW_CU_CHECK_ERR(cudaFree(d_query_)); + GW_CU_CHECK_ERR(cudaFree(d_target_)); + GW_CU_CHECK_ERR(cudaFree(d_seed_pairs_)); + GW_CU_CHECK_ERR(cudaFree(d_num_ssp_)); + GW_CU_CHECK_ERR(cudaFree(d_ssp_)); + host_ptr_api_mode_ = false; + } + +} UngappedXDrop::~UngappedXDrop() { - // TODO - Check flag for host pointer mode reset(); GW_CU_CHECK_ERR(cudaFree(d_sub_mat_)); GW_CU_CHECK_ERR(cudaFree(d_tmp_ssp_)); GW_CU_CHECK_ERR(cudaFree(d_done_)); -}; +} } // namespace cudaextender diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 06985abda..ce564e354 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -69,6 +69,7 @@ private: size_t cub_storage_bytes_; // Host ptr API additional required variables + bool host_ptr_api_mode_; char* d_query_; char* d_target_; SeedPair* d_seed_pairs_; From 9719f43d2f93f53397a7bf785fb3ab9fc7a589d1 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 10:51:18 -0700 Subject: [PATCH 096/281] [cudaextender] Remove local vars that store language defaults --- .../genomeworks/cudaextender/cudaextender.hpp | 1 + cudaextender/src/ungapped_xdrop.cu | 86 ++++++++++--------- cudaextender/src/ungapped_xdrop_kernels.cu | 51 +++++------ 3 files changed, 66 insertions(+), 72 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp index 00dfc446d..0f18be8a1 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp @@ -32,6 +32,7 @@ namespace cudaextender enum StatusType { success = 0, + error_invalid_operation = 1, generic_error }; diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index f2ffc0f67..4d17422b2 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -54,31 +54,30 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd cudaDeviceProp device_prop; cudaGetDeviceProperties(&device_prop, device_id_); const int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures - const int32_t max_seed_pairs_per_gb = 8388608; // FIXME: Calculate using sizeof datastructures + //const int32_t max_seed_pairs_per_gb = 8388608; // FIXME: Calculate using sizeof datastructures // TODO- Do we need this? const float global_mem_gb = static_cast(device_prop.totalGlobalMem) / 1073741824.0f; batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; - const int32_t max_seed_pairs = static_cast(global_mem_gb) * max_seed_pairs_per_gb; // Switch to device for copying over initial structures scoped_device_switch dev(device_id_); //Figure out memory requirements for cub functions size_t temp_storage_bytes = 0; cub_storage_bytes_ = 0; - GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(nullptr, temp_storage_bytes, d_tmp_ssp_, d_tmp_ssp_, (int32_t*)nullptr, batch_max_ungapped_extensions_)); - GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(nullptr, cub_storage_bytes_, d_done_, d_done_ + batch_max_ungapped_extensions_, batch_max_ungapped_extensions_)); + GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(nullptr, temp_storage_bytes, d_tmp_ssp_, d_tmp_ssp_, (int32_t*)nullptr, batch_max_ungapped_extensions_)) + GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(nullptr, cub_storage_bytes_, d_done_, d_done_ + batch_max_ungapped_extensions_, batch_max_ungapped_extensions_)) cub_storage_bytes_ = std::max(temp_storage_bytes, cub_storage_bytes_); // Allocate space on device for scoring matrix and intermediate results - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_sub_mat_, sub_mat_dim_ * sizeof(int32_t))); - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, batch_max_ungapped_extensions_ * sizeof(int32_t))); - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_tmp_ssp_, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair))); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_sub_mat_, sub_mat_dim_ * sizeof(int32_t))) + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, batch_max_ungapped_extensions_ * sizeof(int32_t))) + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_tmp_ssp_, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair))) // Allocate temporary storage for cub - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_temp_storage_cub_, cub_storage_bytes_)); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_temp_storage_cub_, cub_storage_bytes_)) // Requires pinned host memory registration for proper async behavior device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_, stream_); - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_, 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_, 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_, 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)) + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_, 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)) } StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, @@ -111,7 +110,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length seed_pair_start, d_scored_segment_pairs, d_done_); - GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(d_temp_storage_cub_, cub_storage_bytes_, d_done_, d_done_, curr_num_pairs, stream_)); + GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(d_temp_storage_cub_, cub_storage_bytes_, d_done_, d_done_, curr_num_pairs, stream_)) // TODO- Make async const int32_t num_scored_segment_pairs = get_value_from_device(d_done_ + curr_num_pairs - 1, stream_); if (num_scored_segment_pairs > 0) @@ -133,7 +132,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length d_scored_segment_pairs + total_scored_segment_pairs_, d_num_scored_segment_pairs, num_scored_segment_pairs, - stream_)); + stream_)) total_scored_segment_pairs_ += get_value_from_device(d_num_scored_segment_pairs, stream_); } } @@ -150,11 +149,11 @@ StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length host_ptr_api_mode_=true; // Allocate space on device for target and query sequences, seed_pairs, // high scoring segment pairs (ssp) and num_ssp. - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_query_, sizeof(char) * query_length)); - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_target_, sizeof(char) * target_length)); - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_seed_pairs_, sizeof(SeedPair) * h_seed_pairs.size())); - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))); - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_ssp_, sizeof(ScoredSegmentPair) * h_seed_pairs.size())); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_query_, sizeof(char) * query_length)) + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_target_, sizeof(char) * target_length)) + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_seed_pairs_, sizeof(SeedPair) * h_seed_pairs.size())) + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))) + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_ssp_, sizeof(ScoredSegmentPair) * h_seed_pairs.size())) // Async memcopy all the input values to device device_copy_n(h_query, query_length, d_query_, stream_); @@ -162,54 +161,59 @@ StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs_, stream_); // Launch the ungapped extender device function - if (!extend_async(d_query_, query_length, d_target_, target_length, score_threshold, d_seed_pairs_, h_seed_pairs.size(), d_ssp_, d_num_ssp_)) - { - GW_LOG_ERROR("Error running cudaextender"); - } - - return success; + return extend_async(d_query_, query_length, d_target_, target_length, score_threshold, d_seed_pairs_, h_seed_pairs.size(), d_ssp_, d_num_ssp_); } StatusType UngappedXDrop::sync() { - const int32_t h_num_ssp = get_value_from_device(d_num_ssp_, stream_); - if (h_num_ssp > 0) + if(host_ptr_api_mode_) { - h_ssp_.resize(h_num_ssp); - device_copy_n(d_ssp_, h_num_ssp, h_ssp_.data(), stream_); - cudaStreamSynchronize(stream_); + const int32_t h_num_ssp = get_value_from_device(d_num_ssp_, stream_); + if (h_num_ssp > 0) + { + h_ssp_.resize(h_num_ssp); + device_copy_n(d_ssp_, h_num_ssp, h_ssp_.data(), stream_); + cudaStreamSynchronize(stream_); + } + return success; } - return success; + // If this function was called without using the host_ptr_api, throw error + return error_invalid_operation; + } const std::vector& UngappedXDrop::get_scored_segment_pairs() const { - return h_ssp_; + if(host_ptr_api_mode_) + { + return h_ssp_; + } + // If this function was called using the host_ptr_api, throw error + throw std::runtime_error("Invalid API call. Getting scored segment pairs without calling extend_async host ptr API"); } void UngappedXDrop::reset() { - // TODO - Add checks for prev free + // Reset only if host pointer API was used earlier if(host_ptr_api_mode_) { h_ssp_.clear(); - GW_CU_CHECK_ERR(cudaFree(d_query_)); - GW_CU_CHECK_ERR(cudaFree(d_target_)); - GW_CU_CHECK_ERR(cudaFree(d_seed_pairs_)); - GW_CU_CHECK_ERR(cudaFree(d_num_ssp_)); - GW_CU_CHECK_ERR(cudaFree(d_ssp_)); + GW_CU_CHECK_ERR(cudaFree(d_query_)) + GW_CU_CHECK_ERR(cudaFree(d_target_)) + GW_CU_CHECK_ERR(cudaFree(d_seed_pairs_)) + GW_CU_CHECK_ERR(cudaFree(d_num_ssp_)) + GW_CU_CHECK_ERR(cudaFree(d_ssp_)) host_ptr_api_mode_ = false; } - } UngappedXDrop::~UngappedXDrop() { - reset(); - GW_CU_CHECK_ERR(cudaFree(d_sub_mat_)); - GW_CU_CHECK_ERR(cudaFree(d_tmp_ssp_)); - GW_CU_CHECK_ERR(cudaFree(d_done_)); + UngappedXDrop::reset(); + GW_CU_CHECK_ERR(cudaFree(d_sub_mat_)) + GW_CU_CHECK_ERR(cudaFree(d_tmp_ssp_)) + GW_CU_CHECK_ERR(cudaFree(d_done_)) } } // namespace cudaextender diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 2a5429b2a..7bf1b99f0 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -29,13 +29,8 @@ namespace cudaextender // to score_threshold and update the d_hsp and d_done vectors __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int32_t* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, int32_t* d_done) { - - int32_t thread_id = threadIdx.x; - int32_t block_id = blockIdx.x; - int32_t num_blocks = gridDim.x; - int32_t warp_size = warpSize; - int32_t lane_id = thread_id % warp_size; - int32_t warp_id = (thread_id - lane_id) / warp_size; + int32_t lane_id = threadIdx.x % warpSize; + int32_t warp_id = (threadIdx.x - lane_id) / warpSize; constexpr int32_t num_warps = 4; // TODO - move out? constexpr int32_t nuc = 8; // TODO - remove hardcode - pass in constexpr int32_t nuc2 = 64; // TODO - remove hardcode @@ -70,13 +65,13 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __shared__ int32_t sub_mat[nuc2]; - if (thread_id < nuc2) + if (threadIdx.x < nuc2) { - sub_mat[thread_id] = d_sub_mat[thread_id]; + sub_mat[threadIdx.x] = d_sub_mat[threadIdx.x]; } __syncthreads(); - for (int32_t hid0 = block_id * num_warps; hid0 < num_seed_pairs; hid0 += num_warps * num_blocks) + for (int32_t hid0 = blockIdx.x * num_warps; hid0 < num_seed_pairs; hid0 += num_warps * gridDim.x) { int32_t hid = hid0 + warp_id + start_index; @@ -145,7 +140,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); @@ -170,7 +165,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); @@ -189,12 +184,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { xdrop_done |= __shfl_up_sync(0xFFFFFFFF, xdrop_done, offset); } - if (lane_id == warp_size - 1) + if (lane_id == warpSize - 1) { if (max_pos > prev_max_pos[warp_id]) @@ -223,7 +218,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe prev_score[warp_id] = thread_score; prev_max_score[warp_id] = max_thread_score; prev_max_pos[warp_id] = max_pos; - tile[warp_id] += warp_size; + tile[warp_id] += warpSize; } } __syncwarp(); @@ -291,7 +286,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } #pragma unroll - for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); @@ -315,7 +310,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); @@ -334,12 +329,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { xdrop_done |= __shfl_up_sync(0xFFFFFFFF, xdrop_done, offset); } - if (lane_id == warp_size - 1) + if (lane_id == warpSize - 1) { if (max_pos > prev_max_pos[warp_id]) @@ -370,7 +365,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe prev_score[warp_id] = thread_score; prev_max_score[warp_id] = max_thread_score; prev_max_pos[warp_id] = max_pos; - tile[warp_id] += warp_size; + tile[warp_id] += warpSize; } } __syncwarp(); @@ -406,14 +401,14 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe for (int32_t i = 0; i < 4; i++) { #pragma unroll - for (int32_t offset = 1; offset < warp_size; offset = offset << 1) + for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { count[i] += __shfl_up_sync(0xFFFFFFFF, count[i], offset); } } __syncwarp(); - if (lane_id == warp_size - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) + if (lane_id == warpSize - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) { entropy[warp_id] = 0.f; @@ -458,15 +453,9 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe // gather only the HSPs from the resulting segments to the beginning of the // tmp_hsp vector __global__ -void compress_output (int32_t* d_done, int32_t start_index, ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, int32_t num_hits){ - - int thread_id = threadIdx.x; - int block_dim = blockDim.x; - int grid_dim = gridDim.x; - int block_id = blockIdx.x; - - int stride = block_dim * grid_dim; - int32_t start = block_dim * block_id + thread_id; +void compress_output (int32_t* d_done, const int32_t start_index, ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, const int32_t num_hits){ + const int32_t stride = blockDim.x * gridDim.x; + const int32_t start = blockDim.x * blockIdx.x + threadIdx.x; int32_t reduced_index = 0; int32_t index = 0; From 28a47087e904dfe44cf814c8f059398132cf1d78 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 10:56:38 -0700 Subject: [PATCH 097/281] [cudaextender] Limit scope of vars in compress output --- cudaextender/src/ungapped_xdrop_kernels.cu | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 7bf1b99f0..51b482c3c 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -456,12 +456,9 @@ __global__ void compress_output (int32_t* d_done, const int32_t start_index, ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, const int32_t num_hits){ const int32_t stride = blockDim.x * gridDim.x; const int32_t start = blockDim.x * blockIdx.x + threadIdx.x; - int32_t reduced_index = 0; - int32_t index = 0; - for (int32_t id = start; id < num_hits; id += stride) { - reduced_index = d_done[id]; - index = id + start_index; + const int32_t reduced_index = d_done[id]; + const int32_t index = id + start_index; if(index > 0){ if(reduced_index > d_done[index-1]){ d_tmp_hsp[reduced_index-1] = d_hsp[index]; From 409316922f6a668a98261279d2ce2f5fa1c1e5d6 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 11:35:21 -0700 Subject: [PATCH 098/281] [cudaextender] constants and logs --- cudaextender/src/ungapped_xdrop.cu | 7 +++++++ cudaextender/src/ungapped_xdrop_kernels.cu | 22 ++++++++++----------- cudaextender/src/ungapped_xdrop_kernels.cuh | 4 ++-- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 4d17422b2..3d3bfc297 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -89,6 +89,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length //TODO - Check bounds // Switch to configured GPU // If host pointer API mode was used before this mode, reset data structures + auto t1 = std::chrono::high_resolution_clock::now(); if(host_ptr_api_mode_) reset(); scoped_device_switch dev(device_id_); @@ -136,6 +137,12 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length total_scored_segment_pairs_ += get_value_from_device(d_num_scored_segment_pairs, stream_); } } + auto t2 = std::chrono::high_resolution_clock::now(); + + auto duration = std::chrono::duration_cast( t2 - t1 ).count(); + + std::cout <<"Time: " < prev_max_pos[warp_id]) - new_max_found[warp_id] = true; - else - new_max_found[warp_id] = false; - + new_max_found[warp_id] = max_pos > prev_max_pos[warp_id]; if (xdrop_done) { total_score[warp_id] += max_thread_score; @@ -372,6 +370,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe if (new_max_found[warp_id]) { +#pragma unroll for (int32_t i = 0; i < 4; i++) { count[i] = count[i] + count_del[i]; @@ -412,11 +411,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe { entropy[warp_id] = 0.f; +#pragma unroll for (int32_t i = 0; i < 4; i++) { entropy[warp_id] += ((double)count[i]) / ((double)(extent[warp_id] + 1)) * ((count[i] != 0) ? log(((double)count[i]) / ((double)(extent[warp_id] + 1))) : 0.f); } - entropy[warp_id] = -entropy[warp_id] / log(4.0f); + entropy[warp_id] = -entropy[warp_id] / log_4; } } __syncwarp(); @@ -453,7 +453,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe // gather only the HSPs from the resulting segments to the beginning of the // tmp_hsp vector __global__ -void compress_output (int32_t* d_done, const int32_t start_index, ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, const int32_t num_hits){ +void compress_output (const int32_t* d_done, const int32_t start_index, ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, const int32_t num_hits){ const int32_t stride = blockDim.x * gridDim.x; const int32_t start = blockDim.x * blockIdx.x + threadIdx.x; for (int32_t id = start; id < num_hits; id += stride) { diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index 07ba7d5d9..507fe182b 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -34,12 +34,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe // gather only the HSPs from the resulting segments to the beginning of the // tmp_hsp vector -__global__ void compress_output(int32_t* d_done, int32_t start_index, ScoredSegmentPair * d_hsp, ScoredSegmentPair* d_tmp_hsp, int num_hits); +__global__ void compress_output(const int32_t* d_done, int32_t start_index, ScoredSegmentPair * d_hsp, ScoredSegmentPair* d_tmp_hsp, int num_hits); // Binary Predicate for sorting the ScoredSegmentPairs struct scored_segment_pair_comp { - __host__ __device__ bool operator()(ScoredSegmentPair x, ScoredSegmentPair y) + __host__ __device__ bool operator()(const ScoredSegmentPair& x, const ScoredSegmentPair& y) { if (x.seed_pair.query_position_in_read < y.seed_pair.query_position_in_read) return true; From 173688fbf548702078e825dfd1e17232c2680bb3 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 12:21:58 -0700 Subject: [PATCH 099/281] [cudaextender] Scoped some vars locally --- cudaextender/src/ungapped_xdrop_kernels.cu | 71 +++++++++++----------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index b0c722908..db6a1c3d9 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -49,20 +49,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __shared__ int32_t tile[num_warps]; __shared__ double entropy[num_warps]; - - - int32_t max_pos; - int32_t temp_pos; bool xdrop_done; - int32_t temp; short count[4]; short count_del[4]; char r_chr; char q_chr; - int32_t ref_pos; - int32_t query_pos; - int32_t pos_offset; - const float log_4= log(4.0f); + const float log_4 = log(4.0f); __shared__ int32_t sub_mat[nuc2]; @@ -74,8 +66,6 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe for (int32_t hid0 = blockIdx.x * num_warps; hid0 < num_seed_pairs; hid0 += num_warps * gridDim.x) { - int32_t thread_score; - int32_t max_thread_score; const int32_t hid = hid0 + warp_id + start_index; if (hid < num_seed_pairs) @@ -123,16 +113,17 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe count_del[1] = 0; count_del[2] = 0; count_del[3] = 0; - max_pos = 0; __syncwarp(); while (!xdrop_found[warp_id] && !edge_found[warp_id]) { - pos_offset = lane_id + tile[warp_id]; - ref_pos = ref_loc[warp_id] + pos_offset; - query_pos = query_loc[warp_id] + pos_offset; - thread_score = 0; + int32_t max_pos; + int32_t thread_score = 0; + int32_t max_thread_score; + const int32_t pos_offset = lane_id + tile[warp_id]; + const int32_t ref_pos = ref_loc[warp_id] + pos_offset; + const int32_t query_pos = query_loc[warp_id] + pos_offset; if (ref_pos < target_length && query_pos < query_length) { @@ -145,7 +136,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe #pragma unroll for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { - temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); + int32_t temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); if (lane_id >= offset) { @@ -170,8 +161,8 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe #pragma unroll for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { - temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); - temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); + int32_t temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); + int32_t temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); if (lane_id >= offset) { @@ -271,18 +262,19 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe count_del[1] = 0; count_del[2] = 0; count_del[3] = 0; - max_pos = 0; __syncwarp(); while (!xdrop_found[warp_id] && !edge_found[warp_id]) { - pos_offset = lane_id + 1 + tile[warp_id]; - thread_score = 0; + int32_t max_pos; + int32_t thread_score = 0; + int32_t max_thread_score; + const int32_t pos_offset = lane_id + 1 + tile[warp_id]; if (ref_loc[warp_id] >= pos_offset && query_loc[warp_id] >= pos_offset) { - ref_pos = ref_loc[warp_id] - pos_offset; - query_pos = query_loc[warp_id] - pos_offset; + const int32_t ref_pos = ref_loc[warp_id] - pos_offset; + const int32_t query_pos = query_loc[warp_id] - pos_offset; r_chr = d_target[ref_pos]; q_chr = d_query[query_pos]; thread_score = sub_mat[r_chr * nuc + q_chr]; @@ -291,7 +283,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe #pragma unroll for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { - temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); + int32_t temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); if (lane_id >= offset) { @@ -315,8 +307,8 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe #pragma unroll for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { - temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); - temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); + int32_t temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); + int32_t temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); if (lane_id >= offset) { @@ -452,20 +444,25 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe // gather only the HSPs from the resulting segments to the beginning of the // tmp_hsp vector -__global__ -void compress_output (const int32_t* d_done, const int32_t start_index, ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, const int32_t num_hits){ +__global__ void compress_output(const int32_t* d_done, const int32_t start_index, ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, const int32_t num_hits) +{ const int32_t stride = blockDim.x * gridDim.x; - const int32_t start = blockDim.x * blockIdx.x + threadIdx.x; - for (int32_t id = start; id < num_hits; id += stride) { + const int32_t start = blockDim.x * blockIdx.x + threadIdx.x; + for (int32_t id = start; id < num_hits; id += stride) + { const int32_t reduced_index = d_done[id]; - const int32_t index = id + start_index; - if(index > 0){ - if(reduced_index > d_done[index-1]){ - d_tmp_hsp[reduced_index-1] = d_hsp[index]; + const int32_t index = id + start_index; + if (index > 0) + { + if (reduced_index > d_done[index - 1]) + { + d_tmp_hsp[reduced_index - 1] = d_hsp[index]; } } - else{ - if(reduced_index == 1){ + else + { + if (reduced_index == 1) + { d_tmp_hsp[0] = d_hsp[start_index]; } } From 7f112c5c06f3de54638a74815119b549ab9e8629 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 12:23:23 -0700 Subject: [PATCH 100/281] [cudaextender] Forgotten pragma --- cudaextender/src/ungapped_xdrop_kernels.cu | 1 + 1 file changed, 1 insertion(+) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index db6a1c3d9..71b32d5cb 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -219,6 +219,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe if (new_max_found[warp_id]) { +#pragma unroll for (int32_t i = 0; i < 4; i++) { count[i] = count[i] + count_del[i]; From 249f826ff3e6fd3b9099f920a0de07757774b500 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 13:00:49 -0700 Subject: [PATCH 101/281] [cudaextender] More constants --- cudaextender/src/ungapped_xdrop_kernels.cu | 82 ++++++++++----------- cudaextender/src/ungapped_xdrop_kernels.cuh | 20 ++++- 2 files changed, 57 insertions(+), 45 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 71b32d5cb..2520dd0ff 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -27,14 +27,26 @@ namespace cudaextender // extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to score_threshold and update the d_hsp and d_done vectors -__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int32_t* d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, int32_t* d_done) +__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, + const int32_t target_length, + const char* __restrict__ d_query, + const int32_t query_length, + const int32_t* d_sub_mat, + const bool no_entropy, + const int32_t xdrop_threshold, + const int32_t score_threshold, + const SeedPair* d_seed_pairs, + const int32_t num_seed_pairs, + const int32_t start_index, + ScoredSegmentPair* d_scored_segment, + int32_t* d_done) { - int32_t lane_id = threadIdx.x % warpSize; - int32_t warp_id = (threadIdx.x - lane_id) / warpSize; constexpr int32_t num_warps = 4; // TODO - move out? constexpr int32_t nuc = 8; // TODO - remove hardcode - pass in constexpr int32_t nuc2 = 64; // TODO - remove hardcode - + const int32_t lane_id = threadIdx.x % warpSize; + const int32_t warp_id = (threadIdx.x - lane_id) / warpSize; + const float log_4 = log(4.0f); __shared__ int32_t ref_loc[num_warps]; __shared__ int32_t query_loc[num_warps]; __shared__ int32_t total_score[num_warps]; @@ -48,14 +60,6 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __shared__ int32_t extent[num_warps]; __shared__ int32_t tile[num_warps]; __shared__ double entropy[num_warps]; - - bool xdrop_done; - short count[4]; - short count_del[4]; - char r_chr; - char q_chr; - const float log_4 = log(4.0f); - __shared__ int32_t sub_mat[nuc2]; if (threadIdx.x < nuc2) @@ -66,7 +70,9 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe for (int32_t hid0 = blockIdx.x * num_warps; hid0 < num_seed_pairs; hid0 += num_warps * gridDim.x) { - const int32_t hid = hid0 + warp_id + start_index; + short count[4] = {0}; + short count_del[4] = {0}; + const int32_t hid = hid0 + warp_id + start_index; if (hid < num_seed_pairs) { @@ -105,15 +111,6 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe extent[warp_id] = 0; } - count[0] = 0; - count[1] = 0; - count[2] = 0; - count[3] = 0; - count_del[0] = 0; - count_del[1] = 0; - count_del[2] = 0; - count_del[3] = 0; - __syncwarp(); while (!xdrop_found[warp_id] && !edge_found[warp_id]) @@ -124,6 +121,8 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe const int32_t pos_offset = lane_id + tile[warp_id]; const int32_t ref_pos = ref_loc[warp_id] + pos_offset; const int32_t query_pos = query_loc[warp_id] + pos_offset; + char r_chr; + char q_chr; if (ref_pos < target_length && query_pos < query_length) { @@ -136,7 +135,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe #pragma unroll for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { - int32_t temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); + const int32_t temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); if (lane_id >= offset) { @@ -161,8 +160,8 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe #pragma unroll for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { - int32_t temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); - int32_t temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); + const int32_t temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); + const int32_t temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); if (lane_id >= offset) { @@ -174,7 +173,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } } - xdrop_done = ((max_thread_score - thread_score) > xdrop_threshold); + bool xdrop_done = ((max_thread_score - thread_score) > xdrop_threshold); __syncwarp(); #pragma unroll @@ -185,12 +184,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe if (lane_id == warpSize - 1) { - - if (max_pos > prev_max_pos[warp_id]) - new_max_found[warp_id] = true; - else - new_max_found[warp_id] = false; - + new_max_found[warp_id] = max_pos > prev_max_pos[warp_id]; if (xdrop_done) { total_score[warp_id] += max_thread_score; @@ -271,20 +265,22 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe int32_t thread_score = 0; int32_t max_thread_score; const int32_t pos_offset = lane_id + 1 + tile[warp_id]; + char r_chr; + char q_chr; if (ref_loc[warp_id] >= pos_offset && query_loc[warp_id] >= pos_offset) { - const int32_t ref_pos = ref_loc[warp_id] - pos_offset; - const int32_t query_pos = query_loc[warp_id] - pos_offset; - r_chr = d_target[ref_pos]; - q_chr = d_query[query_pos]; - thread_score = sub_mat[r_chr * nuc + q_chr]; + const int32_t ref_pos = ref_loc[warp_id] - pos_offset; + const int32_t query_pos = query_loc[warp_id] - pos_offset; + r_chr = d_target[ref_pos]; + q_chr = d_query[query_pos]; + thread_score = sub_mat[r_chr * nuc + q_chr]; } #pragma unroll for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { - int32_t temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); + const int32_t temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); if (lane_id >= offset) { @@ -308,8 +304,8 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe #pragma unroll for (int32_t offset = 1; offset < warpSize; offset = offset << 1) { - int32_t temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); - int32_t temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); + const int32_t temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); + const int32_t temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); if (lane_id >= offset) { @@ -321,7 +317,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } } - xdrop_done = ((max_thread_score - thread_score) > xdrop_threshold); + bool xdrop_done = ((max_thread_score - thread_score) > xdrop_threshold); __syncwarp(); #pragma unroll @@ -425,7 +421,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe d_scored_segment[hid].seed_pair.target_position_in_read = ref_loc[warp_id] - left_extent[warp_id]; d_scored_segment[hid].seed_pair.query_position_in_read = query_loc[warp_id] - left_extent[warp_id]; d_scored_segment[hid].length = extent[warp_id]; - if (entropy[warp_id] > 0) + if (entropy[warp_id] > 0) // TODO - Is this necessary? d_scored_segment[hid].score = total_score[warp_id] * entropy[warp_id]; d_done[hid - start_index] = 1; } @@ -445,7 +441,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe // gather only the HSPs from the resulting segments to the beginning of the // tmp_hsp vector -__global__ void compress_output(const int32_t* d_done, const int32_t start_index, ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, const int32_t num_hits) +__global__ void compress_output(const int32_t* d_done, const int32_t start_index, const ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, const int32_t num_hits) { const int32_t stride = blockDim.x * gridDim.x; const int32_t start = blockDim.x * blockIdx.x + threadIdx.x; diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index 507fe182b..31ced10e1 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -30,11 +30,27 @@ namespace cudaextender // extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to hspthresh and update the d_hsp and d_done vectors -__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, const int32_t * d_sub_mat, bool no_entropy, int32_t xdrop_threshold, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, int32_t start_index, ScoredSegmentPair* d_scored_segment, int32_t* d_done); +__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, + const int32_t target_length, + const char* __restrict__ d_query, + const int32_t query_length, + const int32_t * d_sub_mat, + const bool no_entropy, + const int32_t xdrop_threshold, + const int32_t score_threshold, + const SeedPair* d_seed_pairs, + const int32_t num_seed_pairs, + const int32_t start_index, + ScoredSegmentPair* d_scored_segment, + int32_t* d_done); // gather only the HSPs from the resulting segments to the beginning of the // tmp_hsp vector -__global__ void compress_output(const int32_t* d_done, int32_t start_index, ScoredSegmentPair * d_hsp, ScoredSegmentPair* d_tmp_hsp, int num_hits); +__global__ void compress_output(const int32_t* d_done, + const int32_t start_index, + const ScoredSegmentPair * d_hsp, + ScoredSegmentPair* d_tmp_hsp, + int num_hits); // Binary Predicate for sorting the ScoredSegmentPairs struct scored_segment_pair_comp From f391ec28906d00d39f04d4ed01ebd58847293c0d Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 14:45:48 -0700 Subject: [PATCH 102/281] [cudaextender] Fix hardcoded file location --- cudaextender/data/CMakeLists.txt | 4 ++-- cudaextender/data/file_location.hpp.in | 2 +- cudaextender/data/{example.fa => sample.fa} | 0 .../data/{example_hits.csv => sample_seed_pairs.csv} | 0 cudaextender/samples/device_sample.cpp | 10 +++++----- 5 files changed, 8 insertions(+), 8 deletions(-) rename cudaextender/data/{example.fa => sample.fa} (100%) rename cudaextender/data/{example_hits.csv => sample_seed_pairs.csv} (100%) diff --git a/cudaextender/data/CMakeLists.txt b/cudaextender/data/CMakeLists.txt index 1df0a07a6..45b5a8405 100644 --- a/cudaextender/data/CMakeLists.txt +++ b/cudaextender/data/CMakeLists.txt @@ -1,6 +1,6 @@ -set(CUDAUNGAPPEDEXTENDER_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../data") +set(CUDAEXTENDER_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../data") configure_file(file_location.hpp.in ${PROJECT_BINARY_DIR}/data/file_location.hpp @ONLY) -set_property(GLOBAL PROPERTY cudaungappedextender_data_include_dir "${PROJECT_BINARY_DIR}/data") +set_property(GLOBAL PROPERTY cudaextender_data_include_dir "${PROJECT_BINARY_DIR}/data") diff --git a/cudaextender/data/file_location.hpp.in b/cudaextender/data/file_location.hpp.in index 3338868b6..0b894f664 100644 --- a/cudaextender/data/file_location.hpp.in +++ b/cudaextender/data/file_location.hpp.in @@ -1,4 +1,4 @@ #pragma once -#cmakedefine CUDAUNGAPPEDEXTENDER_DATA_DIR "@CUDAUNGAPPEDEXTENDER_DATA_DIR@" +#cmakedefine CUDAEXTENDER_DATA_DIR "@CUDAEXTENDER_DATA_DIR@" diff --git a/cudaextender/data/example.fa b/cudaextender/data/sample.fa similarity index 100% rename from cudaextender/data/example.fa rename to cudaextender/data/sample.fa diff --git a/cudaextender/data/example_hits.csv b/cudaextender/data/sample_seed_pairs.csv similarity index 100% rename from cudaextender/data/example_hits.csv rename to cudaextender/data/sample_seed_pairs.csv diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp index cf9e5ed84..a89027aa5 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/device_sample.cpp @@ -18,7 +18,7 @@ #include #include #include - +#include #include #include #include @@ -113,12 +113,12 @@ int main(int argc, char* argv[]) // Fasta query and target files - std::string target_file_path = "../data/example.fa"; + std::string target_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; - std::string query_file_path = "../data/example.fa"; + std::string query_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa";; std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file @@ -126,11 +126,11 @@ int main(int argc, char* argv[]) // CSV SeedPairs file - Each row -> query_position_in_read_, // target_position_in_read_ - std::string seed_pairs_file_path = "../data/example_hits.csv"; + std::string seed_pairs_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample_seed_pairs.csv"; //TODO - pinned seed_pairs std::vector h_seed_pairs; - // Following function loops through all seed_pairs in the example_seed_pairs.csv and returns + // Following function loops through all seed_pairs in the sample_seed_pairs.csv and returns // results in // the passed vector parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); From d42f1aef60e8cd4d2aa26607a2705ee2f49f20ad Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 15:04:09 -0700 Subject: [PATCH 103/281] [cudaextender] Flip if/else --- cudaextender/src/ungapped_xdrop_kernels.cu | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 2520dd0ff..6600c0784 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -73,25 +73,19 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe short count[4] = {0}; short count_del[4] = {0}; const int32_t hid = hid0 + warp_id + start_index; - - if (hid < num_seed_pairs) + if (lane_id == 0) { - if (lane_id == 0) + if (hid < num_seed_pairs) { ref_loc[warp_id] = d_seed_pairs[hid].target_position_in_read; query_loc[warp_id] = d_seed_pairs[hid].query_position_in_read; - total_score[warp_id] = 0; } - } - else - { - if (lane_id == 0) + else { - ref_loc[warp_id] = d_seed_pairs[hid0].target_position_in_read; query_loc[warp_id] = d_seed_pairs[hid0].query_position_in_read; - total_score[warp_id] = 0; } + total_score[warp_id] = 0; } __syncwarp(); From 3822cb690508de9425085736d3f4abfeb5ff00b9 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 15:06:47 -0700 Subject: [PATCH 104/281] [cudaextender] merge ifs --- cudaextender/src/ungapped_xdrop_kernels.cu | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 6600c0784..3285118f8 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -86,14 +86,6 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe query_loc[warp_id] = d_seed_pairs[hid0].query_position_in_read; } total_score[warp_id] = 0; - } - __syncwarp(); - - ////////////////////////////////////////////////////////////////// - //Right extension - - if (lane_id == 0) - { tile[warp_id] = 0; xdrop_found[warp_id] = false; edge_found[warp_id] = false; @@ -104,9 +96,9 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe prev_max_pos[warp_id] = 0; extent[warp_id] = 0; } - __syncwarp(); - + ////////////////////////////////////////////////////////////////// + //Right extension while (!xdrop_found[warp_id] && !edge_found[warp_id]) { int32_t max_pos; From b49d6d89d4c3fe892ff1f9d552e44ce570cf2829 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 15:14:18 -0700 Subject: [PATCH 105/281] [cudaextender] Hoisted out common params from if/else --- cudaextender/src/ungapped_xdrop_kernels.cu | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 3285118f8..bc955885a 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -77,15 +77,15 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe { if (hid < num_seed_pairs) { - ref_loc[warp_id] = d_seed_pairs[hid].target_position_in_read; - query_loc[warp_id] = d_seed_pairs[hid].query_position_in_read; + ref_loc[warp_id] = d_seed_pairs[hid].target_position_in_read; + query_loc[warp_id] = d_seed_pairs[hid].query_position_in_read; } else { - ref_loc[warp_id] = d_seed_pairs[hid0].target_position_in_read; - query_loc[warp_id] = d_seed_pairs[hid0].query_position_in_read; + ref_loc[warp_id] = d_seed_pairs[hid0].target_position_in_read; + query_loc[warp_id] = d_seed_pairs[hid0].query_position_in_read; } - total_score[warp_id] = 0; + total_score[warp_id] = 0; tile[warp_id] = 0; xdrop_found[warp_id] = false; edge_found[warp_id] = false; @@ -171,12 +171,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe if (lane_id == warpSize - 1) { new_max_found[warp_id] = max_pos > prev_max_pos[warp_id]; + prev_max_pos[warp_id] = max_pos; if (xdrop_done) { total_score[warp_id] += max_thread_score; xdrop_found[warp_id] = true; extent[warp_id] = max_pos; - prev_max_pos[warp_id] = max_pos; tile[warp_id] = max_pos; } else if (ref_pos >= target_length || query_pos >= query_length) @@ -184,14 +184,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe total_score[warp_id] += max_thread_score; edge_found[warp_id] = true; extent[warp_id] = max_pos; - prev_max_pos[warp_id] = max_pos; tile[warp_id] = max_pos; } else { prev_score[warp_id] = thread_score; prev_max_score[warp_id] = max_thread_score; - prev_max_pos[warp_id] = max_pos; tile[warp_id] += warpSize; } } @@ -315,13 +313,13 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe if (lane_id == warpSize - 1) { new_max_found[warp_id] = max_pos > prev_max_pos[warp_id]; + prev_max_pos[warp_id] = max_pos; if (xdrop_done) { total_score[warp_id] += max_thread_score; xdrop_found[warp_id] = true; left_extent[warp_id] = max_pos; extent[warp_id] += left_extent[warp_id]; - prev_max_pos[warp_id] = max_pos; tile[warp_id] = max_pos; } else if (ref_loc[warp_id] < pos_offset || query_loc[warp_id] < pos_offset) @@ -330,14 +328,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe edge_found[warp_id] = true; left_extent[warp_id] = max_pos; extent[warp_id] += left_extent[warp_id]; - prev_max_pos[warp_id] = max_pos; tile[warp_id] = max_pos; } else { prev_score[warp_id] = thread_score; prev_max_score[warp_id] = max_thread_score; - prev_max_pos[warp_id] = max_pos; tile[warp_id] += warpSize; } } From cd96e583cd56c5704ea5b587b59464a024579d96 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 15:36:32 -0700 Subject: [PATCH 106/281] [cudaextender] simplified entropy calculation --- cudaextender/src/ungapped_xdrop_kernels.cu | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index bc955885a..235e0b14a 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -378,14 +378,18 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } __syncwarp(); - if (lane_id == warpSize - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) + if (lane_id == warpSize - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) // TODO - MAGIC NUMBER ALERT! { entropy[warp_id] = 0.f; #pragma unroll for (int32_t i = 0; i < 4; i++) { - entropy[warp_id] += ((double)count[i]) / ((double)(extent[warp_id] + 1)) * ((count[i] != 0) ? log(((double)count[i]) / ((double)(extent[warp_id] + 1))) : 0.f); + if(count[i]!=0) + { + const double probability = static_cast(count[i]) / static_cast(extent[warp_id] + 1); + entropy[warp_id] += (probability) * log(probability); + } } entropy[warp_id] = -entropy[warp_id] / log_4; } From bed9513dd8f238811947f1c8052a52dc96f31317 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 15:46:52 -0700 Subject: [PATCH 107/281] [cudaextender] C style casts to static cast --- cudaextender/src/ungapped_xdrop_kernels.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 235e0b14a..9396a3639 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -402,7 +402,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe { if (lane_id == 0) { - if (((int32_t)(((float)total_score[warp_id]) * entropy[warp_id])) >= score_threshold) + if (static_cast(static_cast(total_score[warp_id]) * entropy[warp_id]) >= score_threshold) { d_scored_segment[hid].seed_pair.target_position_in_read = ref_loc[warp_id] - left_extent[warp_id]; d_scored_segment[hid].seed_pair.query_position_in_read = query_loc[warp_id] - left_extent[warp_id]; From f21275487d230010feebdfd95886735490e1d1f1 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 15:58:13 -0700 Subject: [PATCH 108/281] [cudaextender] switch case --- cudaextender/samples/device_sample.cpp | 94 +++++++++++++--------- cudaextender/src/ungapped_xdrop_kernels.cu | 2 +- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp index a89027aa5..d05ccc302 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/device_sample.cpp @@ -62,23 +62,39 @@ void encode_string(char* dst_seq, const char* src_seq, int32_t len) { for (int32_t i = 0; i < len; i++) { - char ch = src_seq[i]; - char dst = X_NT; - if (ch == 'A') - dst = A_NT; - else if (ch == 'C') - dst = C_NT; - else if (ch == 'G') - dst = G_NT; - else if (ch == 'T') - dst = T_NT; - else if ((ch == 'a') || (ch == 'c') || (ch == 'g') || (ch == 't')) - dst = L_NT; - else if ((ch == 'n') || (ch == 'N')) - dst = N_NT; - else if (ch == '&') - dst = E_NT; - dst_seq[i] = dst; + char ch = src_seq[i]; + char dst; + switch (ch) + { + case 'A': + dst_seq[i] = A_NT; + break; + case 'C': + dst_seq[i] = C_NT; + break; + case 'G': + dst_seq[i] = G_NT; + break; + case 'T': + dst_seq[i] = T_NT; + break; + case '&': + dst_seq[i] = E_NT; + break; + case 'n': + case 'N': + dst_seq[i] = N_NT; + break; + case 'a': + case 'c': + case 'g': + case 't': + dst_seq[i] = L_NT; + break; + default: + dst_seq[i] = X_NT; + break; + } } } @@ -88,7 +104,7 @@ int main(int argc, char* argv[]) const bool input_no_entropy = false; const int32_t score_threshold = 3000; char c; - bool print=false, help=false; + bool print = false, help = false; while ((c = getopt(argc, argv, "p")) != -1) { switch (c) @@ -103,7 +119,7 @@ int main(int argc, char* argv[]) } } - if(help) + if (help) { std::cout << "CUDAExtender API sample program. Runs ungapped extender on canned data." << std::endl; std::cout << "-p : Print the Scored Segment Pair output to stdout" << std::endl; @@ -111,17 +127,17 @@ int main(int argc, char* argv[]) std::exit(0); } - // Fasta query and target files - std::string target_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + std::string target_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; - std::string query_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa";; + std::string query_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + ; std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); - // Assumes that only one sequence is present per file + // Assumes that only one sequence is present per file std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; // CSV SeedPairs file - Each row -> query_position_in_read_, @@ -134,17 +150,17 @@ int main(int argc, char* argv[]) // results in // the passed vector parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); - std::cerr <<"Number of seed pairs: "< Date: Mon, 21 Sep 2020 12:32:21 -0400 Subject: [PATCH 109/281] [pygenomeworks] Remove commented out code from evaluate_paf. --- pygenomeworks/bin/evaluate_paf | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pygenomeworks/bin/evaluate_paf b/pygenomeworks/bin/evaluate_paf index 2d6a47daf..f248311c6 100755 --- a/pygenomeworks/bin/evaluate_paf +++ b/pygenomeworks/bin/evaluate_paf @@ -176,21 +176,10 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip (test_overlap.query_sequence_name == test_overlap.target_sequence_name): continue test_overlap_count += 1 - # query_0 = (test_overlap.query_start, test_overlap.query_end) - # target_0 = (test_overlap.target_start, test_overlap.target_end) key = generate_key(test_overlap.query_sequence_name, test_overlap.target_sequence_name) key_reversed = generate_key(test_overlap.target_sequence_name, test_overlap.query_sequence_name) - # if (key in seen_test_overlap_keys) or (key_reversed in seen_test_overlap_keys): - # continue - - # seen_test_overlap_keys.add(key) - # seen_test_overlap_keys.add(key_reversed) - #incorrect_query_start += not match_statistics["query_start_valid"] - #incorrect_query_end += not match_statistics["query_end_valid"] - #incorrect_target_start += not match_statistics["target_start_valid"] - #incorrect_target_end += not match_statistics["target_end_valid"] best_pct_match = 0.0 best_ends = [1, 1, 1, 1] From dfd40546fb45e377011b65e86dbd152bb09dcab3 Mon Sep 17 00:00:00 2001 From: atadkase Date: Fri, 18 Sep 2020 14:05:56 -0700 Subject: [PATCH 110/281] [cmake] Turn default gen_all_arch flag off --- CMakeLists.txt | 2 +- ci/common/build-test-sdk.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 786714ee0..3e6ffaffe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,7 +40,7 @@ option(gw_enable_cudapoa_nw_print "Enable verbose prints within cudapoa NW kerne option(gw_profiling "Compile a binary for profiling with NVTX markers." OFF) option(gw_enable_caching_allocator "Enable caching allocator." ON) option(gw_generate_docs "Generate Doxygen documentation" ON) -option(gw_cuda_gen_all_arch "ON: Generate optimized CUDA code for all architectures | OFF: for detected architectures only" ON) +option(gw_cuda_gen_all_arch "ON: Generate optimized CUDA code for all architectures | OFF: for detected architectures only" OFF) # Must be included before others for options value validation include(cmake/Utils.cmake) diff --git a/ci/common/build-test-sdk.sh b/ci/common/build-test-sdk.sh index 216c9746f..e1cbb3edd 100644 --- a/ci/common/build-test-sdk.sh +++ b/ci/common/build-test-sdk.sh @@ -31,6 +31,7 @@ cmake .. "${CMAKE_COMMON_VARIABLES[@]}" \ -Dgw_enable_tests=ON \ -Dgw_enable_benchmarks=ON \ -Dgw_build_shared=ON \ + -Dgw_cuda_gen_all_arch=ON \ -DCMAKE_INSTALL_PREFIX="${LOCAL_BUILD_DIR}/install" \ -GNinja From 0ae2342787e153496f0327a74d5535caffb1239d Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 11:41:11 -0700 Subject: [PATCH 111/281] [cudaextender] Replace with device buffer and caching allocator --- .../genomeworks/cudaextender/extender.hpp | 9 +- cudaextender/samples/device_sample.cpp | 4 +- cudaextender/src/extender.cpp | 4 +- cudaextender/src/ungapped_xdrop.cu | 90 +++++++++---------- cudaextender/src/ungapped_xdrop.cuh | 13 +-- 5 files changed, 60 insertions(+), 60 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index 80266bad8..30a975e48 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -14,12 +14,12 @@ * limitations under the License. */ #pragma once -#include #include - +#include +#include +#include #include #include -#include namespace claraparabricks { @@ -96,7 +96,8 @@ class Extender virtual void reset() = 0; }; -std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type=ExtensionType::ungapped_xdrop); +std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator, ExtensionType type=ExtensionType::ungapped_xdrop); + } // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp index d05ccc302..64c607a15 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/device_sample.cpp @@ -199,7 +199,9 @@ int main(int argc, char* argv[]) stream0.get())); // Create an ungapped extender object - std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0); + const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); + DefaultDeviceAllocator allocator = create_default_device_allocator(max_gpu_memory); + std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0, allocator); // Launch the ungapped extender device function ungapped_extender->extend_async(d_query, // Type TBD based on encoding diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index 532d550b7..e22de6a4a 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -27,11 +27,11 @@ namespace genomeworks namespace cudaextender { -std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, ExtensionType type) +std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator, ExtensionType type) { if (type == ExtensionType::ungapped_xdrop) { - return std::make_unique(h_sub_mat, sub_mat_dim, xdrop_threshold, no_entropy, stream, device_id); + return std::make_unique(h_sub_mat, sub_mat_dim, xdrop_threshold, no_entropy, stream, device_id, allocator); } else { diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 3d3bfc297..f9065b749 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -39,7 +39,7 @@ namespace cudaextender using namespace cudautils; -UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id) +UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator) : h_sub_mat_(h_sub_mat) , sub_mat_dim_(sub_mat_dim) , xdrop_threshold_(xdrop_threshold) @@ -47,6 +47,7 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd , stream_(stream) , device_id_(device_id) , host_ptr_api_mode_(false) + , allocator_(allocator) { //TODO - Check bounds // Calculate the max limits on the number of extensions we can do on @@ -62,22 +63,21 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd //Figure out memory requirements for cub functions size_t temp_storage_bytes = 0; - cub_storage_bytes_ = 0; - GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(nullptr, temp_storage_bytes, d_tmp_ssp_, d_tmp_ssp_, (int32_t*)nullptr, batch_max_ungapped_extensions_)) - GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(nullptr, cub_storage_bytes_, d_done_, d_done_ + batch_max_ungapped_extensions_, batch_max_ungapped_extensions_)) - cub_storage_bytes_ = std::max(temp_storage_bytes, cub_storage_bytes_); + size_t cub_storage_bytes = 0; + GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(nullptr, temp_storage_bytes, d_tmp_ssp_.data(), d_tmp_ssp_.data(), (int32_t*)nullptr, batch_max_ungapped_extensions_, stream_)); + GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(nullptr, cub_storage_bytes, d_done_.data(), d_done_.data() + batch_max_ungapped_extensions_, batch_max_ungapped_extensions_, stream_)); + cub_storage_bytes = std::max(temp_storage_bytes, cub_storage_bytes); // Allocate space on device for scoring matrix and intermediate results - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_sub_mat_, sub_mat_dim_ * sizeof(int32_t))) - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_done_, batch_max_ungapped_extensions_ * sizeof(int32_t))) - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_tmp_ssp_, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair))) - // Allocate temporary storage for cub - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_temp_storage_cub_, cub_storage_bytes_)) + d_sub_mat_ = device_buffer(sub_mat_dim_, allocator_, stream_); + d_done_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); + d_tmp_ssp_ = device_buffer(batch_max_ungapped_extensions_, allocator_,stream_); + d_temp_storage_cub_ = device_buffer(cub_storage_bytes, allocator_, stream_); // Requires pinned host memory registration for proper async behavior - device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_, stream_); - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_, 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)) - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_, 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)) + device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_.data(), stream_); + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_.data(), 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_.data(), 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); } StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, @@ -102,7 +102,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length target_length, d_query, query_length, - d_sub_mat_, + d_sub_mat_.data(), no_entropy_, xdrop_threshold_, score_threshold, @@ -110,26 +110,25 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length curr_num_pairs, seed_pair_start, d_scored_segment_pairs, - d_done_); - GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(d_temp_storage_cub_, cub_storage_bytes_, d_done_, d_done_, curr_num_pairs, stream_)) + d_done_.data()); + size_t cub_storage_bytes = d_temp_storage_cub_.size(); + GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(d_temp_storage_cub_.data(), cub_storage_bytes, d_done_.data(), d_done_.data(), curr_num_pairs, stream_)) // TODO- Make async - const int32_t num_scored_segment_pairs = get_value_from_device(d_done_ + curr_num_pairs - 1, stream_); + const int32_t num_scored_segment_pairs = get_value_from_device(d_done_.data() + curr_num_pairs - 1, stream_); if (num_scored_segment_pairs > 0) { - compress_output<<<1024, 1024, 0, stream_>>>(d_done_, + compress_output<<<1024, 1024, 0, stream_>>>(d_done_.data(), seed_pair_start, d_scored_segment_pairs, - d_tmp_ssp_, + d_tmp_ssp_.data(), curr_num_pairs); // TODO- Need configurability for kernel? - thrust::device_ptr d_tmp_hsp_dev_ptr(d_tmp_ssp_); - // TODO- Make thrust use caching allocator or change kernel - thrust::stable_sort(thrust::cuda::par.on(stream_), - d_tmp_hsp_dev_ptr, - d_tmp_hsp_dev_ptr + num_scored_segment_pairs, + thrust::stable_sort(thrust::cuda::par(allocator_).on(stream_), + d_tmp_ssp_.begin(), + d_tmp_ssp_.begin() + num_scored_segment_pairs, scored_segment_pair_comp()); - GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(d_temp_storage_cub_, - cub_storage_bytes_, - d_tmp_ssp_, + GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(d_temp_storage_cub_.data(), + cub_storage_bytes, + d_tmp_ssp_.data(), d_scored_segment_pairs + total_scored_segment_pairs_, d_num_scored_segment_pairs, num_scored_segment_pairs, @@ -152,23 +151,23 @@ StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length int32_t score_threshold, std::vector& h_seed_pairs) { - // Set host pointer mode on - host_ptr_api_mode_=true; - // Allocate space on device for target and query sequences, seed_pairs, - // high scoring segment pairs (ssp) and num_ssp. - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_query_, sizeof(char) * query_length)) - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_target_, sizeof(char) * target_length)) - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_seed_pairs_, sizeof(SeedPair) * h_seed_pairs.size())) - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))) - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_ssp_, sizeof(ScoredSegmentPair) * h_seed_pairs.size())) - - // Async memcopy all the input values to device - device_copy_n(h_query, query_length, d_query_, stream_); - device_copy_n(h_target, target_length, d_target_, stream_); - device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs_, stream_); - - // Launch the ungapped extender device function - return extend_async(d_query_, query_length, d_target_, target_length, score_threshold, d_seed_pairs_, h_seed_pairs.size(), d_ssp_, d_num_ssp_); +// // Set host pointer mode on +// host_ptr_api_mode_=true; +// // Allocate space on device for target and query sequences, seed_pairs, +// // high scoring segment pairs (ssp) and num_ssp. +// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_query_, sizeof(char) * query_length)) +// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_target_, sizeof(char) * target_length)) +// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_seed_pairs_, sizeof(SeedPair) * h_seed_pairs.size())) +// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))) +// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_ssp_, sizeof(ScoredSegmentPair) * h_seed_pairs.size())) +// +// // Async memcopy all the input values to device +// device_copy_n(h_query, query_length, d_query_, stream_); +// device_copy_n(h_target, target_length, d_target_, stream_); +// device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs_, stream_); +// +// // Launch the ungapped extender device function +// return extend_async(d_query_, query_length, d_target_, target_length, score_threshold, d_seed_pairs_, h_seed_pairs.size(), d_ssp_, d_num_ssp_); } StatusType UngappedXDrop::sync() @@ -218,9 +217,6 @@ void UngappedXDrop::reset() UngappedXDrop::~UngappedXDrop() { UngappedXDrop::reset(); - GW_CU_CHECK_ERR(cudaFree(d_sub_mat_)) - GW_CU_CHECK_ERR(cudaFree(d_tmp_ssp_)) - GW_CU_CHECK_ERR(cudaFree(d_done_)) } } // namespace cudaextender diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index ce564e354..37ca41b12 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -17,6 +17,7 @@ #pragma once #include +#include namespace claraparabricks { @@ -30,7 +31,7 @@ namespace cudaextender class UngappedXDrop : public Extender { public: - UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id); + UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator); ~UngappedXDrop() override; StatusType extend_async(const char* h_query, int32_t query_length, @@ -52,6 +53,7 @@ public: private: // TODO - Smart pointers + DefaultDeviceAllocator allocator_; // Device ptr API required variables int32_t* h_sub_mat_; int32_t sub_mat_dim_; // Assume matrix is square @@ -61,12 +63,11 @@ private: int32_t device_id_; std::vector scored_segment_pairs_; int32_t batch_max_ungapped_extensions_; - int32_t* d_sub_mat_; // Pointer to device substitution matrix - int32_t* d_done_; // TODO- Rename scratch space - ScoredSegmentPair* d_tmp_ssp_; // TODO- Rename Scratch space 2 + device_buffer d_sub_mat_; // Pointer to device substitution matrix + device_buffer d_done_; // TODO- Rename scratch space + device_buffer d_tmp_ssp_; // TODO- Rename Scratch space 2 int32_t total_scored_segment_pairs_; - void* d_temp_storage_cub_; // temporary storage for cub functions - size_t cub_storage_bytes_; + device_buffer d_temp_storage_cub_; // temporary storage for cub functions // Host ptr API additional required variables bool host_ptr_api_mode_; From 45611e2fbfba5f380c7e2e4b7aa8c1a1bb594dad Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 12:23:01 -0700 Subject: [PATCH 112/281] [cudaextender] Get rid of manual memory allocation for device_sample --- cudaextender/samples/device_sample.cpp | 82 ++++++++++---------------- cudaextender/src/ungapped_xdrop.cu | 11 ++-- cudaextender/src/ungapped_xdrop.cuh | 1 - 3 files changed, 38 insertions(+), 56 deletions(-) diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/device_sample.cpp index 64c607a15..2c450ae6d 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/device_sample.cpp @@ -22,8 +22,11 @@ #include #include #include +#include +#include using namespace claraparabricks::genomeworks; +using namespace claraparabricks::genomeworks::cudautils; using namespace claraparabricks::genomeworks::cudaextender; constexpr char A_NT = 0; @@ -163,68 +166,56 @@ int main(int argc, char* argv[]) -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; // Allocate pinned memory for query and target strings - char* h_encoded_target; - char* h_encoded_query; - GW_CU_CHECK_ERR(cudaHostAlloc((void**)&h_encoded_target, sizeof(char) * target_sequence.length(), cudaHostAllocPortable)); - GW_CU_CHECK_ERR(cudaHostAlloc((void**)&h_encoded_query, sizeof(char) * query_sequence.length(), cudaHostAllocPortable)); + pinned_host_vector h_encoded_target(target_sequence.length()); + pinned_host_vector h_encoded_query(target_sequence.length()); - encode_string(h_encoded_target, target_sequence.c_str(), target_sequence.length()); - encode_string(h_encoded_query, query_sequence.c_str(), query_sequence.length()); + encode_string(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); + encode_string(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); // Create a stream for async use CudaStream stream0 = make_cuda_stream(); - // Allocate space on device for target and query sequences, seed_pairs, - // high scoring segment pairs (ssp) and num_ssp. - char* d_query; - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_query, sizeof(char) * query_sequence.length())); - char* d_target; - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_target, sizeof(char) * target_sequence.length())); - - // Allocate a minimum of num_seed_pairs as all seed_pairs could satisfy the threshold in the worst case - SeedPair* d_seed_pairs; - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_seed_pairs, sizeof(SeedPair) * h_seed_pairs.size())); + // Allocate space on device for target and query sequences, seed_pairs, + // scored segment pairs (ssp) and num_ssp using default allocator (caching) + const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); + DefaultDeviceAllocator allocator = create_default_device_allocator(max_gpu_memory); + // Allocate space for query and target sequences + device_buffer d_query(query_sequence.length(), allocator, stream0.get()); + device_buffer d_target(target_sequence.length(), allocator, stream0.get()); + // Allocate space for SeedPair input + device_buffer d_seed_pairs(h_seed_pairs.size(), allocator, stream0.get()); + // Allocate space for ScoredSegmentPair output + device_buffer d_ssp(h_seed_pairs.size(), allocator, stream0.get()); + // TODO - Keep this as a malloc for single int? int32_t* d_num_ssp; GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp, sizeof(int32_t))); - ScoredSegmentPair* d_ssp; - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_ssp, sizeof(ScoredSegmentPair) * h_seed_pairs.size())); - // Async Memcopy all the input values to device - // TODO - Convert to pinned memory for true async copy - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_query, h_encoded_query, sizeof(char) * query_sequence.length(), - cudaMemcpyHostToDevice, stream0.get())); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_target, h_encoded_target, sizeof(char) * target_sequence.length(), - cudaMemcpyHostToDevice, stream0.get())); - GW_CU_CHECK_ERR(cudaMemcpyAsync(d_seed_pairs, &h_seed_pairs[0], sizeof(SeedPair) * h_seed_pairs.size(), cudaMemcpyHostToDevice, - stream0.get())); + device_copy_n(h_encoded_query.data(), query_sequence.length(), d_query.data(), stream0.get()); + device_copy_n(h_encoded_target.data(), target_sequence.length(), d_target.data(), stream0.get()); + device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs.data(), stream0.get()); // Create an ungapped extender object - const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); - DefaultDeviceAllocator allocator = create_default_device_allocator(max_gpu_memory); std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0, allocator); // Launch the ungapped extender device function - ungapped_extender->extend_async(d_query, // Type TBD based on encoding - query_sequence.length(), - d_target, - target_sequence.length(), + ungapped_extender->extend_async(d_query.data(), // Type TBD based on encoding + d_query.size(), + d_target.data(), + d_target.size(), score_threshold, - d_seed_pairs, - h_seed_pairs.size(), - d_ssp, + d_seed_pairs.data(), + d_seed_pairs.size(), + d_ssp.data(), d_num_ssp); // Wait for ungapped extender to finish GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0.get())); - - // Following function has synchronization built-in, but the above sync is kept for demoing the async nature int32_t h_num_ssp = cudautils::get_value_from_device(d_num_ssp, stream0.get()); - //Get results std::cerr << "Number of ScoredSegmentPairs found: " << h_num_ssp << std::endl; - ScoredSegmentPair* h_ssp = (ScoredSegmentPair*)malloc(h_num_ssp * sizeof(ScoredSegmentPair)); - cudaMemcpy(h_ssp, d_ssp, h_num_ssp * sizeof(ScoredSegmentPair), cudaMemcpyDeviceToHost); - + std::vector h_ssp(h_num_ssp); + // Copy data synchronously + device_copy_n(d_ssp.data(), h_num_ssp, h_ssp.data()); if (print) { std::cout << "Target Position, Query Position, Length, Score" << std::endl; @@ -236,16 +227,7 @@ int main(int argc, char* argv[]) } } - // Free memory on host - free(h_ssp); // Free all CUDA allocated memory - GW_CU_CHECK_ERR(cudaFreeHost(h_encoded_target)); - GW_CU_CHECK_ERR(cudaFreeHost(h_encoded_query)); - GW_CU_CHECK_ERR(cudaFree(d_query)); - GW_CU_CHECK_ERR(cudaFree(d_target)); - GW_CU_CHECK_ERR(cudaFree(d_ssp)); GW_CU_CHECK_ERR(cudaFree(d_num_ssp)); - GW_CU_CHECK_ERR(cudaFree(d_seed_pairs)); - return 0; } diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index f9065b749..a664d7fcd 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -76,8 +76,7 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd // Requires pinned host memory registration for proper async behavior device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_.data(), stream_); - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_.data(), 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_.data(), 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); + } StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, @@ -90,8 +89,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length // Switch to configured GPU // If host pointer API mode was used before this mode, reset data structures auto t1 = std::chrono::high_resolution_clock::now(); - if(host_ptr_api_mode_) - reset(); + reset(); scoped_device_switch dev(device_id_); total_scored_segment_pairs_ = 0; for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += batch_max_ungapped_extensions_) @@ -201,7 +199,10 @@ const std::vector& UngappedXDrop::get_scored_segment_pairs() void UngappedXDrop::reset() { - // Reset only if host pointer API was used earlier + // TODO - Do we need these? + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_.data(), 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_.data(), 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); + // Reset these only if host pointer API was used earlier if(host_ptr_api_mode_) { h_ssp_.clear(); diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 37ca41b12..269a35566 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -52,7 +52,6 @@ public: const std::vector& get_scored_segment_pairs() const override; private: - // TODO - Smart pointers DefaultDeviceAllocator allocator_; // Device ptr API required variables int32_t* h_sub_mat_; From f78a833756821369509fe7a3ee3595b752a8183a Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 13:04:36 -0700 Subject: [PATCH 113/281] [cudaextender] Add host ptr API code with device buffer --- cudaextender/src/extender.cpp | 1 + cudaextender/src/ungapped_xdrop.cu | 44 ++++++++++----------- cudaextender/src/ungapped_xdrop.cuh | 9 +++-- cudaextender/src/ungapped_xdrop_kernels.cuh | 10 +---- 4 files changed, 29 insertions(+), 35 deletions(-) diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index e22de6a4a..d95a88d46 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -27,6 +27,7 @@ namespace genomeworks namespace cudaextender { +// TODO - Provide an allocator-less API for creation? std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator, ExtensionType type) { if (type == ExtensionType::ungapped_xdrop) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index a664d7fcd..7c3bb1fca 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -149,23 +149,27 @@ StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length int32_t score_threshold, std::vector& h_seed_pairs) { -// // Set host pointer mode on -// host_ptr_api_mode_=true; -// // Allocate space on device for target and query sequences, seed_pairs, -// // high scoring segment pairs (ssp) and num_ssp. -// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_query_, sizeof(char) * query_length)) -// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_target_, sizeof(char) * target_length)) -// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_seed_pairs_, sizeof(SeedPair) * h_seed_pairs.size())) -// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))) -// GW_CU_CHECK_ERR(cudaMalloc((void**)&d_ssp_, sizeof(ScoredSegmentPair) * h_seed_pairs.size())) -// -// // Async memcopy all the input values to device -// device_copy_n(h_query, query_length, d_query_, stream_); -// device_copy_n(h_target, target_length, d_target_, stream_); -// device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs_, stream_); -// -// // Launch the ungapped extender device function -// return extend_async(d_query_, query_length, d_target_, target_length, score_threshold, d_seed_pairs_, h_seed_pairs.size(), d_ssp_, d_num_ssp_); + // Set host pointer mode on + host_ptr_api_mode_=true; +// Allocate space for query and target sequences + device_buffer d_query(query_length, allocator_, stream_); + device_buffer d_target(target_length, allocator_, stream_); + // Allocate space for SeedPair input + device_buffer d_seed_pairs(h_seed_pairs.size(), allocator_, stream_); + // Allocate space for ScoredSegmentPair output + device_buffer d_ssp(h_seed_pairs.size(), allocator_, stream_); + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))) + // Async memcopy all the input values to device + device_copy_n(h_query, query_length, d_query_.data(), stream_); + device_copy_n(h_target, target_length, d_target_.data(), stream_); + device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs_.data(), stream_); + + // Launch the ungapped extender device function + return extend_async(d_query_.data(), query_length, + d_target_.data(), target_length, + score_threshold, d_seed_pairs_.data(), + h_seed_pairs.size(), d_ssp_.data(), + d_num_ssp_); } StatusType UngappedXDrop::sync() @@ -176,7 +180,7 @@ StatusType UngappedXDrop::sync() if (h_num_ssp > 0) { h_ssp_.resize(h_num_ssp); - device_copy_n(d_ssp_, h_num_ssp, h_ssp_.data(), stream_); + device_copy_n(d_ssp_.data(), h_num_ssp, h_ssp_.data(), stream_); cudaStreamSynchronize(stream_); } return success; @@ -206,11 +210,7 @@ void UngappedXDrop::reset() if(host_ptr_api_mode_) { h_ssp_.clear(); - GW_CU_CHECK_ERR(cudaFree(d_query_)) - GW_CU_CHECK_ERR(cudaFree(d_target_)) - GW_CU_CHECK_ERR(cudaFree(d_seed_pairs_)) GW_CU_CHECK_ERR(cudaFree(d_num_ssp_)) - GW_CU_CHECK_ERR(cudaFree(d_ssp_)) host_ptr_api_mode_ = false; } } diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 269a35566..77de0851b 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -18,6 +18,7 @@ #include #include +#include namespace claraparabricks { @@ -70,11 +71,11 @@ private: // Host ptr API additional required variables bool host_ptr_api_mode_; - char* d_query_; - char* d_target_; - SeedPair* d_seed_pairs_; + device_buffer d_query_; + device_buffer d_target_; + device_buffer d_seed_pairs_; int32_t* d_num_ssp_; - ScoredSegmentPair* d_ssp_; + device_buffer d_ssp_; std::vector h_ssp_; }; diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index 31ced10e1..49b1f08af 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -38,7 +38,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe const bool no_entropy, const int32_t xdrop_threshold, const int32_t score_threshold, - const SeedPair* d_seed_pairs, + const SeedPair* __restrict__ d_seed_pairs, const int32_t num_seed_pairs, const int32_t start_index, ScoredSegmentPair* d_scored_segment, @@ -85,14 +85,6 @@ struct scored_segment_pair_comp } }; -struct scored_segment_pair_equal -{ - __host__ __device__ bool operator()(ScoredSegmentPair x, ScoredSegmentPair y) - { - return ((x.seed_pair.target_position_in_read == y.seed_pair.target_position_in_read) && (x.seed_pair.query_position_in_read == y.seed_pair.query_position_in_read) && (x.length == y.length) && (x.score == y.score)); - } -}; - } // namespace cudaextender } // namespace genomeworks From e8b711b56de676974b087c08bcbb06a75a8f1724 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 13:09:56 -0700 Subject: [PATCH 114/281] [cudaextender] Fix output iterator of size check --- cudaextender/src/ungapped_xdrop.cu | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 7c3bb1fca..43790501a 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -65,7 +65,7 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd size_t temp_storage_bytes = 0; size_t cub_storage_bytes = 0; GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(nullptr, temp_storage_bytes, d_tmp_ssp_.data(), d_tmp_ssp_.data(), (int32_t*)nullptr, batch_max_ungapped_extensions_, stream_)); - GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(nullptr, cub_storage_bytes, d_done_.data(), d_done_.data() + batch_max_ungapped_extensions_, batch_max_ungapped_extensions_, stream_)); + GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(nullptr, cub_storage_bytes, d_done_.data(), d_done_.data(), batch_max_ungapped_extensions_, stream_)); cub_storage_bytes = std::max(temp_storage_bytes, cub_storage_bytes); // Allocate space on device for scoring matrix and intermediate results @@ -86,10 +86,11 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length int32_t* d_num_scored_segment_pairs) { //TODO - Check bounds - // Switch to configured GPU - // If host pointer API mode was used before this mode, reset data structures + auto t1 = std::chrono::high_resolution_clock::now(); reset(); + // Switch to configured GPU + // If host pointer API mode was used before this mode, reset data structures scoped_device_switch dev(device_id_); total_scored_segment_pairs_ = 0; for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += batch_max_ungapped_extensions_) From 16e38d6e74ce819d5fcddfb95da964bd95161f64 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 13:59:41 -0700 Subject: [PATCH 115/281] [cudaextender] Enable host_ptr_sample --- cudaextender/CMakeLists.txt | 34 ++--- .../genomeworks/cudaextender/cudaextender.hpp | 2 +- .../genomeworks/cudaextender/extender.hpp | 11 +- cudaextender/samples/CMakeLists.txt | 12 +- cudaextender/samples/host_sample.cu | 90 ------------- ...ice_sample.cpp => sample_cudaextender.cpp} | 127 ++++++++++-------- cudaextender/src/extender.cpp | 2 +- cudaextender/src/ungapped_xdrop.cu | 57 ++++---- cudaextender/src/ungapped_xdrop.cuh | 10 +- cudaextender/src/ungapped_xdrop_kernels.cu | 24 ++-- cudaextender/src/ungapped_xdrop_kernels.cuh | 4 +- 11 files changed, 146 insertions(+), 227 deletions(-) delete mode 100644 cudaextender/samples/host_sample.cu rename cudaextender/samples/{device_sample.cpp => sample_cudaextender.cpp} (61%) diff --git a/cudaextender/CMakeLists.txt b/cudaextender/CMakeLists.txt index 2fa927dd0..359b84e5f 100644 --- a/cudaextender/CMakeLists.txt +++ b/cudaextender/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright 2019-2020 NVIDIA CORPORATION. +# Copyright 2020 NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -project(cudaextender) +set(MODULE_NAME cudaextender) # Process data subdirectory first add_subdirectory(data) @@ -23,16 +23,6 @@ GitVersion() configure_file(${CMAKE_SOURCE_DIR}/common/base/src/version.cpp.in ${CMAKE_CURRENT_BINARY_DIR}/version.cpp) -# Optimization flags interfere with nvcc and cause compilation issues with reworked -# file structure. -if (NOT DEFINED gw_cuda_before_10_0) - message(FATAL_ERROR "${PROJECT_NAME} : variable gw_cuda_before_10_0 is not defined yet. Please make sure CUDA.cmake is loaded first.") -elseif (gw_cuda_before_10_0) - message(STATUS "${PROJECT_NAME} : Remove -O optimization when building for CUDA < 10 as it causes compilation issues.") - string(REGEX REPLACE "-O[0-3]" "" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) - string(REGEX REPLACE "-O[0-3]" "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) -endif () - # Project specific NVCC flags, --expt-relaxed-constexpr is being added to allow using numeric_limits inside device kernels. # Using in CUDA 10.1+ to avoid compilation issues with CUDA <= 10.0. set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14") @@ -41,28 +31,28 @@ if (gw_cuda_after_10_0) endif () if (gw_cuda_gen_all_arch) CUDA_SELECT_NVCC_ARCH_FLAGS(ARCH_FLAGS "Common") -else() +else () CUDA_SELECT_NVCC_ARCH_FLAGS(ARCH_FLAGS "Auto") -endif() +endif () set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} ${ARCH_FLAGS}") message(STATUS "nvcc flags for ${PROJECT_NAME}: ${CUDA_NVCC_FLAGS}") get_property(gw_library_type GLOBAL PROPERTY gw_library_type) -cuda_add_library(cudaextender ${gw_library_type} +cuda_add_library(${MODULE_NAME} ${gw_library_type} src/cudaextender.cpp src/extender.cpp src/ungapped_xdrop.cu src/ungapped_xdrop_kernels.cu ${CMAKE_CURRENT_BINARY_DIR}/version.cpp) -target_link_libraries(cudaextender gwbase gwio) +target_link_libraries(${MODULE_NAME} gwbase gwio) -target_compile_options(cudaextender PRIVATE -Werror -Wall -Wextra) +target_compile_options(${MODULE_NAME} PRIVATE -Werror -Wall -Wextra) if (gw_optimize_for_native_cpu) - target_compile_options(cudaextender PRIVATE -march=native) + target_compile_options(${MODULE_NAME} PRIVATE -march=native) endif () -target_include_directories(cudaextender +target_include_directories(${MODULE_NAME} PUBLIC $ $ @@ -70,14 +60,14 @@ target_include_directories(cudaextender add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) -install(TARGETS cudaextender +install(TARGETS ${MODULE_NAME} COMPONENT gwlogging - EXPORT cudaextender + EXPORT ${MODULE_NAME} DESTINATION lib INCLUDES DESTINATION include ) install(DIRECTORY include/ DESTINATION include) -install(EXPORT cudaextender DESTINATION cmake) +install(EXPORT ${MODULE_NAME} DESTINATION cmake) # Add other folders. add_subdirectory(samples) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp index 0f18be8a1..adbd45f06 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp @@ -31,7 +31,7 @@ namespace cudaextender /// CUDA Extender status type enum StatusType { - success = 0, + success = 0, error_invalid_operation = 1, generic_error }; diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index 30a975e48..d10faeb53 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -64,10 +64,10 @@ class Extender /// copies them over to device, /// launches async extension kernels on specified stream. Filters /// segments on device based on score_threshold - virtual StatusType extend_async(const char* h_query, int32_t query_length, - const char* h_target, int32_t target_length, - int32_t score_threshold, - std::vector& h_seed_pairs) = 0; + virtual StatusType extend_async(const char* h_query, const int32_t& query_length, + const char* h_target, const int32_t& target_length, + const int32_t& score_threshold, + const std::vector& h_seed_pairs) = 0; /// \brief Device pointer prototype for extension /// @@ -96,8 +96,7 @@ class Extender virtual void reset() = 0; }; -std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator, ExtensionType type=ExtensionType::ungapped_xdrop); - +std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator, ExtensionType type = ExtensionType::ungapped_xdrop); } // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/samples/CMakeLists.txt b/cudaextender/samples/CMakeLists.txt index 623f6ead4..43be7eadd 100644 --- a/cudaextender/samples/CMakeLists.txt +++ b/cudaextender/samples/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright 2019-2020 NVIDIA CORPORATION. +# Copyright 2020 NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,20 +15,20 @@ # -project(sample_cudaextender) +set(MODULE_NAME sample_cudaextender) get_property(cudaextender_data_include_dir GLOBAL PROPERTY cudaextender_data_include_dir) include_directories(${cudaextender_data_include_dir}) -add_executable(${PROJECT_NAME} - device_sample.cpp +add_executable(${MODULE_NAME} + sample_cudaextender.cpp ) -target_link_libraries(${PROJECT_NAME} +target_link_libraries(${MODULE_NAME} cudaextender gwio gwbase ) -install(TARGETS ${PROJECT_NAME} +install(TARGETS ${MODULE_NAME} DESTINATION samples) diff --git a/cudaextender/samples/host_sample.cu b/cudaextender/samples/host_sample.cu deleted file mode 100644 index 4e670b604..000000000 --- a/cudaextender/samples/host_sample.cu +++ /dev/null @@ -1,90 +0,0 @@ -/* -* Copyright 2020 NVIDIA CORPORATION. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#include -#include -#include -#include -#include -#include - -using namespace claraparabricks::genomeworks; -using namespace claraparabricks::genomeworks::cudaungappedextender; - -int main(int argc, char* argv[]) -{ - const int32_t input_xdrop = 910; - const bool input_no_entropy = false; - const int32_t score_threshold = 3000; - // Fasta query and target files - std::string target_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_target = - io::create_kseq_fasta_parser(target_file_path, 0, false); - // Assumes that only one sequence is present per file - std::string target_sequence = fasta_parser_target->get_sequence_by_id(0); - - std::string query_file_path = "../data/example.fa"; - std::unique_ptr fasta_parser_query = - io::create_kseq_fasta_parser(query_file_path, 0, false); - // Assumes that only one sequence is present per file - magic_sequence query_sequence = fasta_parser_query->get_sequence_by_id(0); - - // CSV SeedPairs file - Each row -> query_position_in_read_, - // target_position_in_read_ - std::string seed_pairs_file_path = "../data/example_seed_pairs.csv"; - - std::vector h_seed_pairs; - // Following function loops through all seed_pairs in the SeedPairs csv and returns - // results in - // the passed vector - parse_SeedPairs(seed_pairs_file_path, h_seed_pairs); - - // Following sections TBD based on encoding - ScoreMatrix = magic_number_matrix; - std::string encoded_target = magic_encode(magic_base, target_sequence); - std::string encoded_query = magic_encode(magic_base, query_sequence); - - // Create a stream for async use - CudaStream stream0 = make_cuda_stream(); - // Create an ungapped extender object - std::unique_ptr ungapped_extender = - std::make_unique(0, magic_number_matrix, input_xdrop, - input_no_entropy, stream0.get()); - // Launch the ungapped extender host function - ungapped_extender->extend_async( - encoded_query.c_str(), // Type TBD based on encoding - encoded_query.size(), encoded_target.c_str(), encoded_target.size(), - score_threshold, h_seed_pairs); - - // Wait for ungapped extender to finish - ungapped_extender->sync(); - - // Get results - const std::vector& segments = - ungapped_extender->get_scored_segment_pairs(); - int32_t i = 0; - for (const auto& segment : segments) - { - std::cout << "Segment: " << i << "Length: " << segment.length - << "Score: " << segment.score << std::endl; - std::cout << "Position in query: " - << segment.seed_pair.query_position_in_read << std::endl; - std::cout << "Position in target: " - << segment.seed_pair.target_position_in_read << std::endl; - i++; - } - - return 0; -} diff --git a/cudaextender/samples/device_sample.cpp b/cudaextender/samples/sample_cudaextender.cpp similarity index 61% rename from cudaextender/samples/device_sample.cpp rename to cudaextender/samples/sample_cudaextender.cpp index 2c450ae6d..c07731f0e 100644 --- a/cudaextender/samples/device_sample.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -101,20 +101,35 @@ void encode_string(char* dst_seq, const char* src_seq, int32_t len) } } +void print_scored_segment_pairs(std::vector scored_segment_pairs) +{ + std::cout << "Target Position, Query Position, Length, Score" << std::endl; + for (auto& segment : scored_segment_pairs) + { + std::cout << segment.seed_pair.target_position_in_read << "," << segment.seed_pair.query_position_in_read + << "," << segment.length << "," << segment.score << std::endl; + } +} + int main(int argc, char* argv[]) { const int32_t xdrop_threshold = 910; const bool input_no_entropy = false; const int32_t score_threshold = 3000; char c; - bool print = false, help = false; - while ((c = getopt(argc, argv, "p")) != -1) + bool print = false; + bool help = false; + bool device_ptr_api_mode = false; + while ((c = getopt(argc, argv, "pdh")) != -1) { switch (c) { case 'p': print = true; break; + case 'd': + device_ptr_api_mode = true; + break; case 'h': default: help = true; @@ -173,61 +188,67 @@ int main(int argc, char* argv[]) encode_string(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); // Create a stream for async use CudaStream stream0 = make_cuda_stream(); - - // Allocate space on device for target and query sequences, seed_pairs, - // scored segment pairs (ssp) and num_ssp using default allocator (caching) + // Create an allocator for use with both APIs const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); DefaultDeviceAllocator allocator = create_default_device_allocator(max_gpu_memory); - // Allocate space for query and target sequences - device_buffer d_query(query_sequence.length(), allocator, stream0.get()); - device_buffer d_target(target_sequence.length(), allocator, stream0.get()); - // Allocate space for SeedPair input - device_buffer d_seed_pairs(h_seed_pairs.size(), allocator, stream0.get()); - // Allocate space for ScoredSegmentPair output - device_buffer d_ssp(h_seed_pairs.size(), allocator, stream0.get()); - // TODO - Keep this as a malloc for single int? - int32_t* d_num_ssp; - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp, sizeof(int32_t))); - - // Async Memcopy all the input values to device - device_copy_n(h_encoded_query.data(), query_sequence.length(), d_query.data(), stream0.get()); - device_copy_n(h_encoded_target.data(), target_sequence.length(), d_target.data(), stream0.get()); - device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs.data(), stream0.get()); - - // Create an ungapped extender object - std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0, allocator); - - // Launch the ungapped extender device function - ungapped_extender->extend_async(d_query.data(), // Type TBD based on encoding - d_query.size(), - d_target.data(), - d_target.size(), - score_threshold, - d_seed_pairs.data(), - d_seed_pairs.size(), - d_ssp.data(), - d_num_ssp); - - // Wait for ungapped extender to finish - GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0.get())); - int32_t h_num_ssp = cudautils::get_value_from_device(d_num_ssp, stream0.get()); - //Get results - std::cerr << "Number of ScoredSegmentPairs found: " << h_num_ssp << std::endl; - std::vector h_ssp(h_num_ssp); - // Copy data synchronously - device_copy_n(d_ssp.data(), h_num_ssp, h_ssp.data()); - if (print) + + if (!device_ptr_api_mode) { - std::cout << "Target Position, Query Position, Length, Score" << std::endl; - for (int i = 0; i < h_num_ssp; i++) - { - ScoredSegmentPair segment = h_ssp[i]; - std::cout << segment.seed_pair.target_position_in_read << "," << segment.seed_pair.query_position_in_read - << "," << segment.length << "," << segment.score << std::endl; - } + std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0, allocator); + ungapped_extender->extend_async(h_encoded_query.data(), h_encoded_query.size(), h_encoded_target.data(), h_encoded_target.size(), score_threshold, h_seed_pairs); + ungapped_extender->sync(); + std::vector h_ssp = ungapped_extender->get_scored_segment_pairs(); + std::cerr << "Number of ScoredSegmentPairs found: " << h_ssp.size() << std::endl; + if(print) + print_scored_segment_pairs(h_ssp); + } + else + { + // Allocate space on device for target and query sequences, seed_pairs, + // scored segment pairs (ssp) and num_ssp using default allocator (caching) + // Allocate space for query and target sequences + device_buffer d_query(query_sequence.length(), allocator, stream0.get()); + device_buffer d_target(target_sequence.length(), allocator, stream0.get()); + // Allocate space for SeedPair input + device_buffer d_seed_pairs(h_seed_pairs.size(), allocator, stream0.get()); + // Allocate space for ScoredSegmentPair output + device_buffer d_ssp(h_seed_pairs.size(), allocator, stream0.get()); + // TODO - Keep this as a malloc for single int? + int32_t* d_num_ssp; + GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp, sizeof(int32_t))); + + // Async Memcopy all the input values to device + device_copy_n(h_encoded_query.data(), query_sequence.length(), d_query.data(), stream0.get()); + device_copy_n(h_encoded_target.data(), target_sequence.length(), d_target.data(), stream0.get()); + device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs.data(), stream0.get()); + + // Create an ungapped extender object + std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0, allocator); + + // Launch the ungapped extender device function + ungapped_extender->extend_async(d_query.data(), // Type TBD based on encoding + d_query.size(), + d_target.data(), + d_target.size(), + score_threshold, + d_seed_pairs.data(), + d_seed_pairs.size(), + d_ssp.data(), + d_num_ssp); + + // Wait for ungapped extender to finish + GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0.get())); + int32_t h_num_ssp = cudautils::get_value_from_device(d_num_ssp, stream0.get()); + //Get results + std::cerr << "Number of ScoredSegmentPairs found: " << h_num_ssp << std::endl; + std::vector h_ssp(h_num_ssp); + // Copy data synchronously + device_copy_n(d_ssp.data(), h_num_ssp, h_ssp.data()); + if (print) + print_scored_segment_pairs(h_ssp); + // Free all CUDA allocated memory + GW_CU_CHECK_ERR(cudaFree(d_num_ssp)); } - // Free all CUDA allocated memory - GW_CU_CHECK_ERR(cudaFree(d_num_ssp)); return 0; } diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index d95a88d46..3a9f19b35 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -28,7 +28,7 @@ namespace cudaextender { // TODO - Provide an allocator-less API for creation? -std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator, ExtensionType type) +std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator, ExtensionType type) { if (type == ExtensionType::ungapped_xdrop) { diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 43790501a..0299999d9 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -54,10 +54,10 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd // this GPU cudaDeviceProp device_prop; cudaGetDeviceProperties(&device_prop, device_id_); - const int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures + const int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures //const int32_t max_seed_pairs_per_gb = 8388608; // FIXME: Calculate using sizeof datastructures // TODO- Do we need this? - const float global_mem_gb = static_cast(device_prop.totalGlobalMem) / 1073741824.0f; - batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; + const float global_mem_gb = static_cast(device_prop.totalGlobalMem) / 1073741824.0f; + batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; // Switch to device for copying over initial structures scoped_device_switch dev(device_id_); @@ -69,14 +69,13 @@ UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xd cub_storage_bytes = std::max(temp_storage_bytes, cub_storage_bytes); // Allocate space on device for scoring matrix and intermediate results - d_sub_mat_ = device_buffer(sub_mat_dim_, allocator_, stream_); - d_done_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); - d_tmp_ssp_ = device_buffer(batch_max_ungapped_extensions_, allocator_,stream_); + d_sub_mat_ = device_buffer(sub_mat_dim_, allocator_, stream_); + d_done_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); + d_tmp_ssp_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); d_temp_storage_cub_ = device_buffer(cub_storage_bytes, allocator_, stream_); // Requires pinned host memory registration for proper async behavior device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_.data(), stream_); - } StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, @@ -88,13 +87,15 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length //TODO - Check bounds auto t1 = std::chrono::high_resolution_clock::now(); - reset(); // Switch to configured GPU // If host pointer API mode was used before this mode, reset data structures scoped_device_switch dev(device_id_); - total_scored_segment_pairs_ = 0; + total_scored_segment_pairs_ = 0; for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += batch_max_ungapped_extensions_) { + // TODO - Do we need these? It seems we don't! + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_.data(), 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); + GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_.data(), 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); const int32_t curr_num_pairs = std::min(batch_max_ungapped_extensions_, num_seed_pairs - seed_pair_start); // TODO- Extricate the kernel launch params? find_high_scoring_segment_pairs<<<1024, 128, 0, stream_>>>(d_target, @@ -137,28 +138,30 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length } auto t2 = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast( t2 - t1 ).count(); + auto duration = std::chrono::duration_cast(t2 - t1).count(); - std::cout <<"Time: " <& h_seed_pairs) +StatusType UngappedXDrop::extend_async(const char* h_query, const int32_t& query_length, + const char* h_target, const int32_t& target_length, + const int32_t& score_threshold, + const std::vector& h_seed_pairs) { + // Reset the extender if it was used before in this mode + reset(); // Set host pointer mode on - host_ptr_api_mode_=true; -// Allocate space for query and target sequences - device_buffer d_query(query_length, allocator_, stream_); - device_buffer d_target(target_length, allocator_, stream_); + host_ptr_api_mode_ = true; + // Allocate space for query and target sequences + d_query_ = device_buffer(query_length, allocator_, stream_); + d_target_ = device_buffer(target_length, allocator_, stream_); // Allocate space for SeedPair input - device_buffer d_seed_pairs(h_seed_pairs.size(), allocator_, stream_); + d_seed_pairs_ = device_buffer(h_seed_pairs.size(), allocator_, stream_); // Allocate space for ScoredSegmentPair output - device_buffer d_ssp(h_seed_pairs.size(), allocator_, stream_); + d_ssp_ = device_buffer(h_seed_pairs.size(), allocator_, stream_); GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))) // Async memcopy all the input values to device device_copy_n(h_query, query_length, d_query_.data(), stream_); @@ -169,13 +172,13 @@ StatusType UngappedXDrop::extend_async(const char* h_query, int32_t query_length return extend_async(d_query_.data(), query_length, d_target_.data(), target_length, score_threshold, d_seed_pairs_.data(), - h_seed_pairs.size(), d_ssp_.data(), + d_seed_pairs_.size(), d_ssp_.data(), d_num_ssp_); } StatusType UngappedXDrop::sync() { - if(host_ptr_api_mode_) + if (host_ptr_api_mode_) { const int32_t h_num_ssp = get_value_from_device(d_num_ssp_, stream_); if (h_num_ssp > 0) @@ -189,12 +192,11 @@ StatusType UngappedXDrop::sync() // If this function was called without using the host_ptr_api, throw error return error_invalid_operation; - } const std::vector& UngappedXDrop::get_scored_segment_pairs() const { - if(host_ptr_api_mode_) + if (host_ptr_api_mode_) { return h_ssp_; } @@ -204,11 +206,8 @@ const std::vector& UngappedXDrop::get_scored_segment_pairs() void UngappedXDrop::reset() { - // TODO - Do we need these? - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_.data(), 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); - GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_.data(), 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); // Reset these only if host pointer API was used earlier - if(host_ptr_api_mode_) + if (host_ptr_api_mode_) { h_ssp_.clear(); GW_CU_CHECK_ERR(cudaFree(d_num_ssp_)) diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 77de0851b..c04f499c0 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -35,10 +35,10 @@ public: UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator); ~UngappedXDrop() override; - StatusType extend_async(const char* h_query, int32_t query_length, - const char* h_target, int32_t target_length, - int32_t score_threshold, - std::vector& h_seed_pairs) override; + StatusType extend_async(const char* h_query, const int32_t& query_length, + const char* h_target, const int32_t& target_length, + const int32_t& score_threshold, + const std::vector& h_seed_pairs) override; StatusType extend_async(const char* d_query, int32_t query_length, const char* d_target, int32_t target_length, @@ -53,7 +53,7 @@ public: const std::vector& get_scored_segment_pairs() const override; private: - DefaultDeviceAllocator allocator_; + DefaultDeviceAllocator allocator_; // Device ptr API required variables int32_t* h_sub_mat_; int32_t sub_mat_dim_; // Assume matrix is square diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 9959c196d..908623c0d 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -171,20 +171,20 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe if (lane_id == warpSize - 1) { new_max_found[warp_id] = max_pos > prev_max_pos[warp_id]; - prev_max_pos[warp_id] = max_pos; + prev_max_pos[warp_id] = max_pos; if (xdrop_done) { total_score[warp_id] += max_thread_score; - xdrop_found[warp_id] = true; - extent[warp_id] = max_pos; - tile[warp_id] = max_pos; + xdrop_found[warp_id] = true; + extent[warp_id] = max_pos; + tile[warp_id] = max_pos; } else if (ref_pos >= target_length || query_pos >= query_length) { total_score[warp_id] += max_thread_score; - edge_found[warp_id] = true; - extent[warp_id] = max_pos; - tile[warp_id] = max_pos; + edge_found[warp_id] = true; + extent[warp_id] = max_pos; + tile[warp_id] = max_pos; } else { @@ -313,14 +313,14 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe if (lane_id == warpSize - 1) { new_max_found[warp_id] = max_pos > prev_max_pos[warp_id]; - prev_max_pos[warp_id] = max_pos; + prev_max_pos[warp_id] = max_pos; if (xdrop_done) { total_score[warp_id] += max_thread_score; xdrop_found[warp_id] = true; left_extent[warp_id] = max_pos; extent[warp_id] += left_extent[warp_id]; - tile[warp_id] = max_pos; + tile[warp_id] = max_pos; } else if (ref_loc[warp_id] < pos_offset || query_loc[warp_id] < pos_offset) { @@ -328,7 +328,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe edge_found[warp_id] = true; left_extent[warp_id] = max_pos; extent[warp_id] += left_extent[warp_id]; - tile[warp_id] = max_pos; + tile[warp_id] = max_pos; } else { @@ -385,10 +385,10 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe #pragma unroll for (int32_t i = 0; i < 4; i++) { - if(count[i]!=0) + if (count[i] != 0) { const double probability = static_cast(count[i]) / static_cast(extent[warp_id] + 1); - entropy[warp_id] += (probability) * log(probability); + entropy[warp_id] += (probability)*log(probability); } } entropy[warp_id] = -entropy[warp_id] / log_4; diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index 49b1f08af..7f3b3bb0c 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -34,7 +34,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe const int32_t target_length, const char* __restrict__ d_query, const int32_t query_length, - const int32_t * d_sub_mat, + const int32_t* d_sub_mat, const bool no_entropy, const int32_t xdrop_threshold, const int32_t score_threshold, @@ -48,7 +48,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe // tmp_hsp vector __global__ void compress_output(const int32_t* d_done, const int32_t start_index, - const ScoredSegmentPair * d_hsp, + const ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, int num_hits); From 233871f1aa463e0170fc95f462258b830e841c17 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 14:05:31 -0700 Subject: [PATCH 116/281] [cudaextender] update flag --- cudaextender/samples/sample_cudaextender.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index c07731f0e..c0abd03d0 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -140,8 +140,9 @@ int main(int argc, char* argv[]) if (help) { std::cout << "CUDAExtender API sample program. Runs ungapped extender on canned data." << std::endl; - std::cout << "-p : Print the Scored Segment Pair output to stdout" << std::endl; - std::cout << "-h : Print help message" << std::endl; + std::cout << "-p : Print the Scored Segment Pair output to stdout." << std::endl; + std::cout << "-d : Use Device Pointer API. If not provided uses Host Pointer API." << std::endl; + std::cout << "-h : Print help message." << std::endl; std::exit(0); } From ed824aed7a5a8c6716d2f034157fb1e6b4b3c0dd Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 14:10:01 -0700 Subject: [PATCH 117/281] [cudaextender] Copyright date fix --- cudaextender/data/CMakeLists.txt | 17 +++++++++++++++-- cudaextender/data/file_location.hpp.in | 16 +++++++++++++++- .../genomeworks/cudaextender/cudaextender.hpp | 2 +- .../genomeworks/cudaextender/extender.hpp | 1 + cudaextender/src/extender.cpp | 2 +- cudaextender/src/ungapped_xdrop.cu | 2 +- cudaextender/src/ungapped_xdrop.cuh | 2 +- 7 files changed, 35 insertions(+), 7 deletions(-) diff --git a/cudaextender/data/CMakeLists.txt b/cudaextender/data/CMakeLists.txt index 45b5a8405..e5c4355dc 100644 --- a/cudaextender/data/CMakeLists.txt +++ b/cudaextender/data/CMakeLists.txt @@ -1,5 +1,18 @@ - - +# +# Copyright 2020 NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# set(CUDAEXTENDER_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../data") configure_file(file_location.hpp.in ${PROJECT_BINARY_DIR}/data/file_location.hpp @ONLY) diff --git a/cudaextender/data/file_location.hpp.in b/cudaextender/data/file_location.hpp.in index 0b894f664..56487d77a 100644 --- a/cudaextender/data/file_location.hpp.in +++ b/cudaextender/data/file_location.hpp.in @@ -1,4 +1,18 @@ - +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ #pragma once #cmakedefine CUDAEXTENDER_DATA_DIR "@CUDAEXTENDER_DATA_DIR@" diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp index adbd45f06..1c1161bce 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp @@ -1,5 +1,5 @@ /* -* Copyright 2019-2020 NVIDIA CORPORATION. +* Copyright 2020 NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index d10faeb53..e6e023cc9 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #pragma once #include #include diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index 3a9f19b35..e90617806 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -1,5 +1,5 @@ /* -* Copyright 2019-2020 NVIDIA CORPORATION. +* Copyright 2020 NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 0299999d9..f2c52de98 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -1,5 +1,5 @@ /* -* Copyright 2019-2020 NVIDIA CORPORATION. +* Copyright 2020 NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index c04f499c0..43ef27846 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -1,5 +1,5 @@ /* -* Copyright 2019-2020 NVIDIA CORPORATION. +* Copyright 2020 NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 7f67c0ca0b3436a2b8a9b58f1a7fb45018c40f0c Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 14:20:08 -0700 Subject: [PATCH 118/281] [cudaextender] sorted some imports --- cudaextender/src/ungapped_xdrop.cu | 16 ++++++++-------- cudaextender/src/ungapped_xdrop.cuh | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index f2c52de98..42fd2e73d 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include #include "ungapped_xdrop.cuh" #include "ungapped_xdrop_kernels.cuh" -#include -#include + +#include +#include + #include -#include #include -#include -#include -#include + #include +#include + +// Temporary timing code #include namespace claraparabricks diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 43ef27846..bc6d13a3f 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -18,7 +18,6 @@ #include #include -#include namespace claraparabricks { From d3642050a4913a2bc351390fe8e13c5d6182e258 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 15:42:56 -0700 Subject: [PATCH 119/281] [cudaextender] Code style fix --- cudaextender/samples/sample_cudaextender.cpp | 17 +++++++++-------- cudaextender/src/extender.cpp | 3 +-- cudaextender/src/ungapped_xdrop_kernels.cu | 8 ++++---- cudaextender/src/ungapped_xdrop_kernels.cuh | 7 ++++--- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index c0abd03d0..eab860ddc 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -13,11 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include #include #include #include @@ -25,9 +20,15 @@ #include #include +#include +#include +#include +#include +#include + using namespace claraparabricks::genomeworks; -using namespace claraparabricks::genomeworks::cudautils; -using namespace claraparabricks::genomeworks::cudaextender; +using namespace cudautils; +using namespace cudaextender; constexpr char A_NT = 0; constexpr char C_NT = 1; @@ -200,7 +201,7 @@ int main(int argc, char* argv[]) ungapped_extender->sync(); std::vector h_ssp = ungapped_extender->get_scored_segment_pairs(); std::cerr << "Number of ScoredSegmentPairs found: " << h_ssp.size() << std::endl; - if(print) + if (print) print_scored_segment_pairs(h_ssp); } else diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index e90617806..b54226768 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "ungapped_xdrop.cuh" #include -#include "ungapped_xdrop.cuh" - namespace claraparabricks { diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 908623c0d..970598ed7 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -427,11 +427,11 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe // gather only the HSPs from the resulting segments to the beginning of the // tmp_hsp vector -__global__ void compress_output(const int32_t* d_done, const int32_t start_index, const ScoredSegmentPair* d_hsp, ScoredSegmentPair* d_tmp_hsp, const int32_t num_hits) +__global__ void compress_output(const int32_t* d_done, const int32_t start_index, const ScoredSegmentPair* d_ssp, ScoredSegmentPair* d_tmp_ssp, const int32_t num_seed_pairs) { const int32_t stride = blockDim.x * gridDim.x; const int32_t start = blockDim.x * blockIdx.x + threadIdx.x; - for (int32_t id = start; id < num_hits; id += stride) + for (int32_t id = start; id < num_seed_pairs; id += stride) { const int32_t reduced_index = d_done[id]; const int32_t index = id + start_index; @@ -439,14 +439,14 @@ __global__ void compress_output(const int32_t* d_done, const int32_t start_index { if (reduced_index > d_done[index - 1]) { - d_tmp_hsp[reduced_index - 1] = d_hsp[index]; + d_tmp_ssp[reduced_index - 1] = d_ssp[index]; } } else { if (reduced_index == 1) { - d_tmp_hsp[0] = d_hsp[start_index]; + d_tmp_ssp[0] = d_ssp[start_index]; } } } diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index 7f3b3bb0c..6d8e87f57 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -15,9 +15,10 @@ */ #pragma once -#include #include "ungapped_xdrop.cuh" +#include + namespace claraparabricks { @@ -48,8 +49,8 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe // tmp_hsp vector __global__ void compress_output(const int32_t* d_done, const int32_t start_index, - const ScoredSegmentPair* d_hsp, - ScoredSegmentPair* d_tmp_hsp, + const ScoredSegmentPair* d_ssp, + ScoredSegmentPair* d_tmp_ssp, int num_hits); // Binary Predicate for sorting the ScoredSegmentPairs From d77371be047123e1c07657d4e04e8d5561688e89 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 21 Sep 2020 15:57:38 -0700 Subject: [PATCH 120/281] [cudaextender] Rename ssps --- cudaextender/src/ungapped_xdrop_kernels.cu | 18 +++++++++--------- cudaextender/src/ungapped_xdrop_kernels.cuh | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 970598ed7..44704b8fc 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -38,7 +38,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe const SeedPair* d_seed_pairs, const int32_t num_seed_pairs, const int32_t start_index, - ScoredSegmentPair* d_scored_segment, + ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_done) { constexpr int32_t num_warps = 4; // TODO - move out? @@ -404,19 +404,19 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe { if (static_cast(static_cast(total_score[warp_id]) * entropy[warp_id]) >= score_threshold) { - d_scored_segment[hid].seed_pair.target_position_in_read = ref_loc[warp_id] - left_extent[warp_id]; - d_scored_segment[hid].seed_pair.query_position_in_read = query_loc[warp_id] - left_extent[warp_id]; - d_scored_segment[hid].length = extent[warp_id]; + d_scored_segment_pairs[hid].seed_pair.target_position_in_read = ref_loc[warp_id] - left_extent[warp_id]; + d_scored_segment_pairs[hid].seed_pair.query_position_in_read = query_loc[warp_id] - left_extent[warp_id]; + d_scored_segment_pairs[hid].length = extent[warp_id]; if (entropy[warp_id] > 0) // TODO - Is this necessary? - d_scored_segment[hid].score = total_score[warp_id] * entropy[warp_id]; + d_scored_segment_pairs[hid].score = total_score[warp_id] * entropy[warp_id]; d_done[hid - start_index] = 1; } else { - d_scored_segment[hid].seed_pair.target_position_in_read = ref_loc[warp_id]; - d_scored_segment[hid].seed_pair.query_position_in_read = query_loc[warp_id]; - d_scored_segment[hid].length = 0; - d_scored_segment[hid].score = 0; + d_scored_segment_pairs[hid].seed_pair.target_position_in_read = ref_loc[warp_id]; + d_scored_segment_pairs[hid].seed_pair.query_position_in_read = query_loc[warp_id]; + d_scored_segment_pairs[hid].length = 0; + d_scored_segment_pairs[hid].score = 0; d_done[hid - start_index] = 0; } } diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index 6d8e87f57..f03d06828 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -42,7 +42,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe const SeedPair* __restrict__ d_seed_pairs, const int32_t num_seed_pairs, const int32_t start_index, - ScoredSegmentPair* d_scored_segment, + ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_done); // gather only the HSPs from the resulting segments to the beginning of the From 763de9dbac6952deaf1c428a9f2fad28ad4a7fb0 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Tue, 22 Sep 2020 09:16:44 -0400 Subject: [PATCH 121/281] [pygw] update cmake command for pygw builds enable generate for all archs --- pygenomeworks/setup_pygenomeworks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pygenomeworks/setup_pygenomeworks.py b/pygenomeworks/setup_pygenomeworks.py index 4e88437ba..a46bd2f6c 100644 --- a/pygenomeworks/setup_pygenomeworks.py +++ b/pygenomeworks/setup_pygenomeworks.py @@ -83,7 +83,8 @@ def _run_cmake_cmd(self): cmake_args = ['-DCMAKE_INSTALL_PREFIX=' + self.gw_install_dir, '-DCMAKE_BUILD_TYPE=' + 'Release', '-DCMAKE_INSTALL_RPATH=' + os.path.join(self.gw_install_dir, "lib"), - '-Dgw_generate_docs=OFF'] + '-Dgw_generate_docs=OFF', + '-Dgw_cuda_gen_all_arch=ON'] cmake_args += [self.cmake_extra_args] if self.cmake_extra_args else [] if self.cuda_toolkit_root_dir: From b4b8bf76ea2ce44452d3a1107e66d47968414adb Mon Sep 17 00:00:00 2001 From: gpuCI <38199262+GPUtester@users.noreply.github.com> Date: Tue, 22 Sep 2020 07:00:13 -0700 Subject: [PATCH 122/281] REL v0.5.3 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index cb0c939a9..be14282b7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.2 +0.5.3 From b2fa6b252d66ae2f84ed767c3c79ae371efc99ec Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 22 Sep 2020 13:35:41 -0700 Subject: [PATCH 123/281] [cudaextender] Move vars closer to use; Use intermediate local variable for accum --- cudaextender/samples/sample_cudaextender.cpp | 3 +-- cudaextender/src/ungapped_xdrop_kernels.cu | 21 +++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index eab860ddc..347dabf62 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -66,8 +66,7 @@ void encode_string(char* dst_seq, const char* src_seq, int32_t len) { for (int32_t i = 0; i < len; i++) { - char ch = src_seq[i]; - char dst; + const char ch = src_seq[i]; switch (ch) { case 'A': diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 44704b8fc..8ec49481f 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -46,7 +46,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe constexpr int32_t nuc2 = 64; // TODO - remove hardcode const int32_t lane_id = threadIdx.x % warpSize; const int32_t warp_id = (threadIdx.x - lane_id) / warpSize; - const float log_4 = log(4.0f); + const float ln_4 = log(4.0f); __shared__ int32_t ref_loc[num_warps]; __shared__ int32_t query_loc[num_warps]; __shared__ int32_t total_score[num_warps]; @@ -101,9 +101,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe //Right extension while (!xdrop_found[warp_id] && !edge_found[warp_id]) { - int32_t max_pos; int32_t thread_score = 0; - int32_t max_thread_score; const int32_t pos_offset = lane_id + tile[warp_id]; const int32_t ref_pos = ref_loc[warp_id] + pos_offset; const int32_t query_pos = query_loc[warp_id] + pos_offset; @@ -130,6 +128,10 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } thread_score += prev_score[warp_id]; + + int32_t max_pos; + int32_t max_thread_score; + if (thread_score > prev_max_score[warp_id]) { max_thread_score = thread_score; @@ -245,9 +247,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe while (!xdrop_found[warp_id] && !edge_found[warp_id]) { - int32_t max_pos; int32_t thread_score = 0; - int32_t max_thread_score; const int32_t pos_offset = lane_id + 1 + tile[warp_id]; char r_chr; char q_chr; @@ -273,6 +273,10 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } thread_score += prev_score[warp_id]; + + int32_t max_pos; + int32_t max_thread_score; + if (thread_score > prev_max_score[warp_id]) { max_thread_score = thread_score; @@ -380,18 +384,17 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe if (lane_id == warpSize - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) // TODO - MAGIC NUMBER ALERT! { - - entropy[warp_id] = 0.f; + double entropy_ln = 0.f; #pragma unroll for (int32_t i = 0; i < 4; i++) { if (count[i] != 0) { const double probability = static_cast(count[i]) / static_cast(extent[warp_id] + 1); - entropy[warp_id] += (probability)*log(probability); + entropy_ln += (probability)*log(probability); } } - entropy[warp_id] = -entropy[warp_id] / log_4; + entropy[warp_id] = -entropy_ln / ln_4; // Store Entropy with log base 4 } } __syncwarp(); From a29d64d9c62e869ce50785c2423361df1dcf1554 Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 22 Sep 2020 13:40:56 -0700 Subject: [PATCH 124/281] [cudaextender] replace warpSize with warp_size constexpr --- cudaextender/src/ungapped_xdrop_kernels.cu | 31 +++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 8ec49481f..68bfbd4aa 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -43,9 +43,10 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe { constexpr int32_t num_warps = 4; // TODO - move out? constexpr int32_t nuc = 8; // TODO - remove hardcode - pass in - constexpr int32_t nuc2 = 64; // TODO - remove hardcode - const int32_t lane_id = threadIdx.x % warpSize; - const int32_t warp_id = (threadIdx.x - lane_id) / warpSize; + constexpr int32_t nuc2 = 64; // TODO - remove hardcode - TBD based on encoding + constexpr int32_t warp_size = 32; + const int32_t lane_id = threadIdx.x % warp_size; + const int32_t warp_id = (threadIdx.x - lane_id) / warp_size; const float ln_4 = log(4.0f); __shared__ int32_t ref_loc[num_warps]; __shared__ int32_t query_loc[num_warps]; @@ -117,7 +118,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warpSize; offset = offset << 1) + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) { const int32_t temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); @@ -146,7 +147,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warpSize; offset = offset << 1) + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) { const int32_t temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); const int32_t temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); @@ -165,12 +166,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warpSize; offset = offset << 1) + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) { xdrop_done |= __shfl_up_sync(0xFFFFFFFF, xdrop_done, offset); } - if (lane_id == warpSize - 1) + if (lane_id == warp_size - 1) { new_max_found[warp_id] = max_pos > prev_max_pos[warp_id]; prev_max_pos[warp_id] = max_pos; @@ -192,7 +193,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe { prev_score[warp_id] = thread_score; prev_max_score[warp_id] = max_thread_score; - tile[warp_id] += warpSize; + tile[warp_id] += warp_size; } } __syncwarp(); @@ -262,7 +263,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } #pragma unroll - for (int32_t offset = 1; offset < warpSize; offset = offset << 1) + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) { const int32_t temp = __shfl_up_sync(0xFFFFFFFF, thread_score, offset); @@ -290,7 +291,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warpSize; offset = offset << 1) + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) { const int32_t temp = __shfl_up_sync(0xFFFFFFFF, max_thread_score, offset); const int32_t temp_pos = __shfl_up_sync(0xFFFFFFFF, max_pos, offset); @@ -309,12 +310,12 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe __syncwarp(); #pragma unroll - for (int32_t offset = 1; offset < warpSize; offset = offset << 1) + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) { xdrop_done |= __shfl_up_sync(0xFFFFFFFF, xdrop_done, offset); } - if (lane_id == warpSize - 1) + if (lane_id == warp_size - 1) { new_max_found[warp_id] = max_pos > prev_max_pos[warp_id]; prev_max_pos[warp_id] = max_pos; @@ -338,7 +339,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe { prev_score[warp_id] = thread_score; prev_max_score[warp_id] = max_thread_score; - tile[warp_id] += warpSize; + tile[warp_id] += warp_size; } } __syncwarp(); @@ -375,14 +376,14 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe for (int32_t i = 0; i < 4; i++) { #pragma unroll - for (int32_t offset = 1; offset < warpSize; offset = offset << 1) + for (int32_t offset = 1; offset < warp_size; offset = offset << 1) { count[i] += __shfl_up_sync(0xFFFFFFFF, count[i], offset); } } __syncwarp(); - if (lane_id == warpSize - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) // TODO - MAGIC NUMBER ALERT! + if (lane_id == warp_size - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) // TODO - MAGIC NUMBER ALERT! { double entropy_ln = 0.f; #pragma unroll From 9e9edf36ca4dc67f00377bce2fc7c825ee5ca197 Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 22 Sep 2020 14:49:45 -0700 Subject: [PATCH 125/281] [ci] Robust copyright detection --- ci/checks/check_copyright.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/checks/check_copyright.py b/ci/checks/check_copyright.py index ef58d3dc5..458be5b34 100755 --- a/ci/checks/check_copyright.py +++ b/ci/checks/check_copyright.py @@ -145,7 +145,7 @@ def copyright_present(f): f - Path to file """ with io.open(f, "r", encoding="utf-8") as fh: - return re.search('Copyright 20[0-9]+-20[0-9]+', fh.read()) + return re.search('Copyright (20[0-9][0-9]-)?20[0-9][0-9] NVIDIA CORPORATION', fh.read()) def parse_args(): From 4638ca8aaf3bb171e9d11a6754e49c542cd538f6 Mon Sep 17 00:00:00 2001 From: atadkase Date: Tue, 22 Sep 2020 14:52:33 -0700 Subject: [PATCH 126/281] [cudaextender] make format --- cudaextender/src/ungapped_xdrop_kernels.cu | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 68bfbd4aa..5782131e7 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -47,7 +47,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe constexpr int32_t warp_size = 32; const int32_t lane_id = threadIdx.x % warp_size; const int32_t warp_id = (threadIdx.x - lane_id) / warp_size; - const float ln_4 = log(4.0f); + const float ln_4 = log(4.0f); __shared__ int32_t ref_loc[num_warps]; __shared__ int32_t query_loc[num_warps]; __shared__ int32_t total_score[num_warps]; @@ -102,7 +102,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe //Right extension while (!xdrop_found[warp_id] && !edge_found[warp_id]) { - int32_t thread_score = 0; + int32_t thread_score = 0; const int32_t pos_offset = lane_id + tile[warp_id]; const int32_t ref_pos = ref_loc[warp_id] + pos_offset; const int32_t query_pos = query_loc[warp_id] + pos_offset; @@ -248,7 +248,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe while (!xdrop_found[warp_id] && !edge_found[warp_id]) { - int32_t thread_score = 0; + int32_t thread_score = 0; const int32_t pos_offset = lane_id + 1 + tile[warp_id]; char r_chr; char q_chr; @@ -421,7 +421,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe d_scored_segment_pairs[hid].seed_pair.query_position_in_read = query_loc[warp_id]; d_scored_segment_pairs[hid].length = 0; d_scored_segment_pairs[hid].score = 0; - d_done[hid - start_index] = 0; + d_done[hid - start_index] = 0; } } } From 76338dfb25fb649274f96af536afdb124a2f77fc Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 22 Sep 2020 17:59:39 -0400 Subject: [PATCH 127/281] [cudapoa-backtracking] shrink score matrix height --- cudapoa/src/cudapoa_kernels.cuh | 1 + cudapoa/src/cudapoa_nw_banded.cuh | 167 +++++++++++++++++++++--------- 2 files changed, 119 insertions(+), 49 deletions(-) diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 7ad43a167..9767d67a7 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -320,6 +320,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) alignment_graph, alignment_read, static_band_width, + max_pred_distance, gap_score, mismatch_score, match_score); diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 2a55e7b78..0b003e650 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -54,17 +54,25 @@ __device__ __forceinline__ int32_t get_band_start_for_row(int32_t row_idx, float } template -__device__ __forceinline__ ScoreT* get_score_ptr(ScoreT* scores, int32_t row, int32_t column, int32_t band_start, int32_t band_width) +__device__ __forceinline__ ScoreT* get_score_ptr(ScoreT* scores, int32_t score_row, int32_t column, int32_t band_start, int32_t band_width) { column = column == -1 ? 0 : column - band_start; - int64_t score_index = static_cast(column) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + int64_t score_index = static_cast(column) + + static_cast(score_row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); return &scores[score_index]; } template -__device__ __forceinline__ void set_score(ScoreT* scores, int32_t row, int32_t column, int32_t value, float gradient, int32_t band_width, int32_t max_column) +__device__ __forceinline__ void set_score(ScoreT* scores, + int32_t row, + int32_t column, + int32_t score_matrix_height, + int32_t value, + float gradient, + int32_t band_width, + int32_t max_column) { int32_t band_start = get_band_start_for_row(row, gradient, band_width, max_column); @@ -78,36 +86,52 @@ __device__ __forceinline__ void set_score(ScoreT* scores, int32_t row, int32_t c col_idx = column - band_start; } - int64_t score_index = static_cast(col_idx) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + int64_t score_index = static_cast(col_idx) + + static_cast(row % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[score_index] = value; } template -__device__ __forceinline__ void initialize_band(ScoreT* scores, int32_t row, int32_t value, float gradient, int32_t band_width, int32_t max_column, int32_t lane_idx) +__device__ __forceinline__ void initialize_band(ScoreT* scores, + int32_t row, + int32_t score_matrix_height, + int32_t value, + float gradient, + int32_t band_width, + int32_t max_column, + int32_t lane_idx) { int32_t band_start = get_band_start_for_row(row, gradient, band_width, max_column); int32_t band_end = band_start + band_width; int32_t initialization_offset = (band_start == 0) ? 1 : band_start; - set_score(scores, row, initialization_offset, value, gradient, band_width, max_column); + set_score(scores, row, initialization_offset, score_matrix_height, value, gradient, band_width, max_column); // note: as long as CUDAPOA_BANDED_MATRIX_RIGHT_PADDING < WARP_SIZE, no need for a for loop for (int32_t j = lane_idx + band_end; j < band_end + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) { - set_score(scores, row, j, value, gradient, band_width, max_column); + set_score(scores, row, j, score_matrix_height, value, gradient, band_width, max_column); } } template __device__ __forceinline__ TraceT get_trace(TraceT* backtrace, int32_t row, int32_t column, int32_t band_start, int32_t band_width) { - int64_t trace_index = static_cast(column - band_start) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + int64_t trace_index = static_cast(column - band_start) + + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); return backtrace[trace_index]; } template -__device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t column, float gradient, int32_t band_width, int32_t max_column, const ScoreT min_score_value) +__device__ __forceinline__ ScoreT get_score(ScoreT* scores, + int32_t row, + int32_t column, + int32_t score_matrix_height, + float gradient, + int32_t band_width, + int32_t max_column, + const ScoreT min_score_value) { int32_t band_start = get_band_start_for_row(row, gradient, band_width, max_column); int32_t band_end = band_start + band_width; @@ -119,7 +143,7 @@ __device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t } else { - return *get_score_ptr(scores, row, column, band_start, band_width); + return *get_score_ptr(scores, row % score_matrix_height, column, band_start, band_width); } } @@ -128,10 +152,10 @@ __device__ __forceinline__ void get_scores(ScoreT* scores, int32_t pred_node, int32_t current_node, int32_t read_pos, + int32_t score_matrix_height, float gradient, int32_t band_width, int32_t max_column, - ScoreT default_value, int32_t gap_score, int32_t match_score, int32_t mismatch_score, @@ -160,7 +184,7 @@ __device__ __forceinline__ void get_scores(ScoreT* scores, } else { - ScoreT4* pred_scores = (ScoreT4*)get_score_ptr(scores, pred_node, read_pos, band_start, band_width); + ScoreT4* pred_scores = (ScoreT4*)get_score_ptr(scores, pred_node % score_matrix_height, read_pos, band_start, band_width); // loads 8/16 consecutive bytes (4 ScoreT) ScoreT4 score4 = pred_scores[0]; @@ -265,6 +289,7 @@ __device__ __forceinline__ SizeT* alignment_graph, SizeT* alignment_read, int32_t band_width, + int32_t score_matrix_height, int32_t gap_score, int32_t mismatch_score, int32_t match_score) @@ -281,7 +306,7 @@ __device__ __forceinline__ // Initialise the horizontal boundary of the score matrix for (int32_t j = lane_idx; j < band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) { - set_score(scores, 0, j, j * gap_score, gradient, band_width, max_column); + set_score(scores, 0, j, score_matrix_height, j * gap_score, gradient, band_width, max_column); } if (lane_idx == 0) @@ -301,7 +326,7 @@ __device__ __forceinline__ int32_t band_start = get_band_start_for_row(score_gIdx, gradient, band_width, max_column); int32_t pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]; - initialize_band(scores, score_gIdx, min_score_value, gradient, band_width, max_column, lane_idx); + initialize_band(scores, score_gIdx, score_matrix_height, min_score_value, gradient, band_width, max_column, lane_idx); int32_t first_element_prev_score = 0; uint16_t pred_count = 0; @@ -311,42 +336,74 @@ __device__ __forceinline__ { // Initialise the vertical boundary of the score matrix int32_t penalty; - pred_count = incoming_edge_count[node_id]; - int64_t trace_index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + pred_count = incoming_edge_count[node_id]; if (pred_count == 0) { - scores[trace_index] = gap_score; - backtrace[trace_index] = -score_gIdx; + int64_t index = static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + scores[index] = gap_score; + index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + backtrace[index] = -score_gIdx; } else { + int64_t index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + pred_idx = node_id_to_pos[pred_node_id] + 1; - // fill in first column of backtrace buffer - backtrace[trace_index] = -(score_gIdx - pred_idx); - if (band_start > CELLS_PER_THREAD && pred_count == 1) + if ((graph_pos - pred_idx) < score_matrix_height) { - first_element_prev_score = min_score_value + gap_score; + // fill in first column of backtrace buffer + backtrace[index] = -(score_gIdx - pred_idx); + + if (band_start > CELLS_PER_THREAD && pred_count == 1) + { + first_element_prev_score = min_score_value + gap_score; + } + else + { + penalty = max(min_score_value, get_score(scores, pred_idx, -1, score_matrix_height, gradient, band_width, max_column, min_score_value)); + // if pred_num > 1 keep checking to find max score as penalty + for (int32_t p = 1; p < pred_count; p++) + { + pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; + int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; + if ((score_gIdx - pred_idx_tmp) < score_matrix_height) + { + int32_t trace_tmp = -(score_gIdx - pred_idx_tmp); + int32_t score_tmp = get_score(scores, pred_idx_tmp, -1, score_matrix_height, gradient, band_width, max_column, min_score_value); + if (penalty < score_tmp) + { + penalty = score_tmp; + backtrace[index] = trace_tmp; + } + } + } + first_element_prev_score = penalty + gap_score; + set_score(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, gradient, band_width, max_column); + } } else { - penalty = max(min_score_value, get_score(scores, pred_idx, -1, gradient, band_width, max_column, min_score_value)); - // if pred_num > 1 keep checking to find max score as penalty - for (int32_t p = 0; p < pred_count; p++) + penalty = min_score_value; + // pick the predecessor which falls within the limit of score_matrix_height + for (int32_t p = 1; p < pred_count; p++) { pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; - if (penalty < get_score(scores, pred_idx_tmp, -1, gradient, band_width, max_column, min_score_value)) + if ((score_gIdx - pred_idx_tmp) < score_matrix_height) { - penalty = get_score(scores, pred_idx_tmp, -1, gradient, band_width, max_column, min_score_value); - // fill in first column of backtrace buffer - backtrace[trace_index] = -(score_gIdx - pred_idx_tmp); + int32_t trace_tmp = -(score_gIdx - pred_idx_tmp); + int32_t score_tmp = get_score(scores, pred_idx_tmp, -1, score_matrix_height, gradient, band_width, max_column, min_score_value); + if (penalty < score_tmp) + { + penalty = score_tmp; + backtrace[index] = trace_tmp; + } } } first_element_prev_score = penalty + gap_score; + set_score(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, gradient, band_width, max_column); } - - set_score(scores, score_gIdx, -1, first_element_prev_score, gradient, band_width, max_column); } } pred_count = __shfl_sync(FULL_MASK, pred_count, 0); @@ -362,15 +419,18 @@ __device__ __forceinline__ TraceT4 trace; ScoreT4 score = {min_score_value, min_score_value, min_score_value, min_score_value}; - get_scores(scores, pred_idx, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, + get_scores(scores, pred_idx, score_gIdx, read_pos, score_matrix_height, gradient, band_width, max_column, gap_score, match_score, mismatch_score, read4, graph_base, score, trace); // Perform same score updates as above, but for rest of predecessors. for (int32_t p = 1; p < pred_count; p++) { - int32_t pred_idx2 = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; - get_scores(scores, pred_idx2, score_gIdx, read_pos, gradient, band_width, max_column, min_score_value, - gap_score, match_score, mismatch_score, read4, graph_base, score, trace); + int32_t pred_idx_tmp = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; + if ((score_gIdx - pred_idx_tmp) < score_matrix_height) + { + get_scores(scores, pred_idx_tmp, score_gIdx, read_pos, score_matrix_height, gradient, band_width, max_column, + gap_score, match_score, mismatch_score, read4, graph_base, score, trace); + } } // While there are changes to the horizontal score values, keep updating the matrix. @@ -431,17 +491,17 @@ __device__ __forceinline__ // which can be used to compute the first cell of the next warp. first_element_prev_score = __shfl_sync(FULL_MASK, score.s3, WARP_SIZE - 1); - int64_t score_index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - - scores[score_index] = score.s0; - scores[score_index + 1L] = score.s1; - scores[score_index + 2L] = score.s2; - scores[score_index + 3L] = score.s3; + int64_t index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + scores[index] = score.s0; + scores[index + 1L] = score.s1; + scores[index + 2L] = score.s2; + scores[index + 3L] = score.s3; - backtrace[score_index] = trace.t0; - backtrace[score_index + 1L] = trace.t1; - backtrace[score_index + 2L] = trace.t2; - backtrace[score_index + 3L] = trace.t3; + index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + backtrace[index] = trace.t0; + backtrace[index + 1L] = trace.t1; + backtrace[index + 2L] = trace.t2; + backtrace[index + 3L] = trace.t3; __syncwarp(); } @@ -459,15 +519,24 @@ __device__ __forceinline__ { if (outgoing_edge_count[graph[idx - 1]] == 0) { - int32_t s = get_score(scores, idx, j, gradient, band_width, max_column, min_score_value); - if (mscore < s) + if ((graph_count - idx) < score_matrix_height) { - mscore = s; - i = idx; + int32_t s = get_score(scores, idx, j, score_matrix_height, gradient, band_width, max_column, min_score_value); + if (mscore < s) + { + mscore = s; + i = idx; + } + } + else + { + ///ToDo throw an error indicating selected score_matrix_height (i.e. max predecessor distance) is too small } } } + //------------------------------------------------------------------------ + // Fill in backtrace int32_t loop_count = 0; while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) From f1747f5683ab4aa2a1a81139efdc5852af9d2ba5 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 23 Sep 2020 11:51:22 -0400 Subject: [PATCH 128/281] [cudapoa-backtracking] changed warning for invalid input for option -D (--pred-distance) --- cudapoa/src/application_parameters.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cudapoa/src/application_parameters.cpp b/cudapoa/src/application_parameters.cpp index 8a487f42b..c8532ea8d 100644 --- a/cudapoa/src/application_parameters.cpp +++ b/cudapoa/src/application_parameters.cpp @@ -82,6 +82,10 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) graph_length = std::stof(optarg); break; case 'D': + if (std::stoi(optarg) <= 0) + { + throw std::runtime_error("pred-distance must be an integer greater than 0"); + } predecessor_disance = std::stoi(optarg); break; case 'd': @@ -121,11 +125,6 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) throw std::runtime_error("band-width must be positive"); } - if (band_mode != BandMode::full_band && predecessor_disance < 0) - { - throw std::runtime_error("pred-distance must be positive"); - } - if (match_score < 0) { throw std::runtime_error("match score must be positive"); From 2f2614a6ec09055291c84a5d2e24fefc1990b701 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 23 Sep 2020 12:56:44 -0400 Subject: [PATCH 129/281] [cudapoa-backtracking] fixed a bug- removed a change which was intended for debugging --- cudapoa/src/cudapoa_kernels.cuh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 9767d67a7..bc3668b07 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -143,8 +143,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) if (BM == BandMode::adaptive_band || BM == BandMode::static_band) { // buffer size for scores, in banded we only need to store part of the scores matrix - // banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); - banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); + banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); int64_t offset = static_cast(banded_buffer_size) * static_cast(window_idx); scores = &scores_d[offset]; // buffer size for backtrace From 71d035d0b5df4bc74afca9e4e592c665ab47099f Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 10:33:03 -0700 Subject: [PATCH 130/281] [cudaextender] More consts + device buffer for num_ssp --- .../genomeworks/cudaextender/extender.hpp | 2 +- cudaextender/samples/sample_cudaextender.cpp | 61 ++++++++----------- cudaextender/src/extender.cpp | 2 +- cudaextender/src/ungapped_xdrop.cu | 10 +-- cudaextender/src/ungapped_xdrop.cuh | 16 ++--- 5 files changed, 41 insertions(+), 50 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index e6e023cc9..89d886ad1 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -97,7 +97,7 @@ class Extender virtual void reset() = 0; }; -std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator, ExtensionType type = ExtensionType::ungapped_xdrop); +std::unique_ptr create_extender(const int32_t* h_sub_mat, const int32_t sub_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator, const ExtensionType type = ExtensionType::ungapped_xdrop); } // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index 347dabf62..378061bcf 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -62,7 +62,7 @@ void parse_seed_pairs(const std::string& filepath, std::vector& seed_p } // convert input sequence from alphabet to integers -void encode_string(char* dst_seq, const char* src_seq, int32_t len) +void encode_string(char* dst_seq, const char* src_seq, const int32_t& len) { for (int32_t i = 0; i < len; i++) { @@ -101,7 +101,7 @@ void encode_string(char* dst_seq, const char* src_seq, int32_t len) } } -void print_scored_segment_pairs(std::vector scored_segment_pairs) +void print_scored_segment_pairs(const std::vector& scored_segment_pairs) { std::cout << "Target Position, Query Position, Length, Score" << std::endl; for (auto& segment : scored_segment_pairs) @@ -147,21 +147,19 @@ int main(int argc, char* argv[]) } // Fasta query and target files - std::string target_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + const std::string target_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file - std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; + const std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; - std::string query_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; - ; - std::unique_ptr fasta_parser_query = - io::create_kseq_fasta_parser(query_file_path, 0, false); + const std::string query_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file - std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; + const std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; // CSV SeedPairs file - Each row -> query_position_in_read_, // target_position_in_read_ - std::string seed_pairs_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample_seed_pairs.csv"; + const std::string seed_pairs_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample_seed_pairs.csv"; //TODO - pinned seed_pairs std::vector h_seed_pairs; @@ -172,14 +170,14 @@ int main(int argc, char* argv[]) std::cerr << "Number of seed pairs: " << h_seed_pairs.size() << std::endl; // Define Scoring Matrix - int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, - -114, 100, -125, -31, -1000, -1000, -100, -9100, - -31, -125, 100, -114, -1000, -1000, -100, -9100, - -123, -31, -114, 91, -1000, -1000, -100, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -100, -100, -100, -100, -1000, -1000, -100, -9100, - -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; + const int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, + -114, 100, -125, -31, -1000, -1000, -100, -9100, + -31, -125, 100, -114, -1000, -1000, -100, -9100, + -123, -31, -114, 91, -1000, -1000, -100, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -100, -100, -100, -100, -1000, -1000, -100, -9100, + -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; // Allocate pinned memory for query and target strings pinned_host_vector h_encoded_target(target_sequence.length()); @@ -192,16 +190,14 @@ int main(int argc, char* argv[]) // Create an allocator for use with both APIs const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); DefaultDeviceAllocator allocator = create_default_device_allocator(max_gpu_memory); - + // Reference for output + std::vector h_ssp; if (!device_ptr_api_mode) { std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0, allocator); ungapped_extender->extend_async(h_encoded_query.data(), h_encoded_query.size(), h_encoded_target.data(), h_encoded_target.size(), score_threshold, h_seed_pairs); ungapped_extender->sync(); - std::vector h_ssp = ungapped_extender->get_scored_segment_pairs(); - std::cerr << "Number of ScoredSegmentPairs found: " << h_ssp.size() << std::endl; - if (print) - print_scored_segment_pairs(h_ssp); + h_ssp = ungapped_extender->get_scored_segment_pairs(); } else { @@ -214,9 +210,7 @@ int main(int argc, char* argv[]) device_buffer d_seed_pairs(h_seed_pairs.size(), allocator, stream0.get()); // Allocate space for ScoredSegmentPair output device_buffer d_ssp(h_seed_pairs.size(), allocator, stream0.get()); - // TODO - Keep this as a malloc for single int? - int32_t* d_num_ssp; - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp, sizeof(int32_t))); + device_buffer d_num_ssp(1, allocator, stream0.get()); // Async Memcopy all the input values to device device_copy_n(h_encoded_query.data(), query_sequence.length(), d_query.data(), stream0.get()); @@ -235,21 +229,18 @@ int main(int argc, char* argv[]) d_seed_pairs.data(), d_seed_pairs.size(), d_ssp.data(), - d_num_ssp); + d_num_ssp.data()); // Wait for ungapped extender to finish GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0.get())); - int32_t h_num_ssp = cudautils::get_value_from_device(d_num_ssp, stream0.get()); - //Get results - std::cerr << "Number of ScoredSegmentPairs found: " << h_num_ssp << std::endl; - std::vector h_ssp(h_num_ssp); + const int32_t h_num_ssp = cudautils::get_value_from_device(d_num_ssp.data(), stream0.get()); + h_ssp.resize(h_num_ssp); // Copy data synchronously device_copy_n(d_ssp.data(), h_num_ssp, h_ssp.data()); - if (print) - print_scored_segment_pairs(h_ssp); - // Free all CUDA allocated memory - GW_CU_CHECK_ERR(cudaFree(d_num_ssp)); } + std::cerr << "Number of ScoredSegmentPairs found: " << h_ssp.size() << std::endl; + if (print) + print_scored_segment_pairs(h_ssp); return 0; } diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index b54226768..a3dd8a96e 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -27,7 +27,7 @@ namespace cudaextender { // TODO - Provide an allocator-less API for creation? -std::unique_ptr create_extender(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator, ExtensionType type) +std::unique_ptr create_extender(const int32_t* h_sub_mat, const int32_t sub_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator, const ExtensionType type) { if (type == ExtensionType::ungapped_xdrop) { diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 42fd2e73d..dfedde8df 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -39,7 +39,7 @@ namespace cudaextender using namespace cudautils; -UngappedXDrop::UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator) +UngappedXDrop::UngappedXDrop(const int32_t* h_sub_mat, const int32_t sub_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator) : h_sub_mat_(h_sub_mat) , sub_mat_dim_(sub_mat_dim) , xdrop_threshold_(xdrop_threshold) @@ -162,7 +162,8 @@ StatusType UngappedXDrop::extend_async(const char* h_query, const int32_t& query d_seed_pairs_ = device_buffer(h_seed_pairs.size(), allocator_, stream_); // Allocate space for ScoredSegmentPair output d_ssp_ = device_buffer(h_seed_pairs.size(), allocator_, stream_); - GW_CU_CHECK_ERR(cudaMalloc((void**)&d_num_ssp_, sizeof(int32_t))) + d_num_ssp_ = device_buffer(1, allocator_, stream_); + // Async memcopy all the input values to device device_copy_n(h_query, query_length, d_query_.data(), stream_); device_copy_n(h_target, target_length, d_target_.data(), stream_); @@ -173,14 +174,14 @@ StatusType UngappedXDrop::extend_async(const char* h_query, const int32_t& query d_target_.data(), target_length, score_threshold, d_seed_pairs_.data(), d_seed_pairs_.size(), d_ssp_.data(), - d_num_ssp_); + d_num_ssp_.data()); } StatusType UngappedXDrop::sync() { if (host_ptr_api_mode_) { - const int32_t h_num_ssp = get_value_from_device(d_num_ssp_, stream_); + const int32_t h_num_ssp = get_value_from_device(d_num_ssp_.data(), stream_); if (h_num_ssp > 0) { h_ssp_.resize(h_num_ssp); @@ -210,7 +211,6 @@ void UngappedXDrop::reset() if (host_ptr_api_mode_) { h_ssp_.clear(); - GW_CU_CHECK_ERR(cudaFree(d_num_ssp_)) host_ptr_api_mode_ = false; } } diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index bc6d13a3f..5c9c215b7 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -31,7 +31,7 @@ namespace cudaextender class UngappedXDrop : public Extender { public: - UngappedXDrop(int32_t* h_sub_mat, int32_t sub_mat_dim, int32_t xdrop_threshold, bool no_entropy, cudaStream_t stream, int32_t device_id, DefaultDeviceAllocator allocator); + UngappedXDrop(const int32_t* h_sub_mat, const int32_t sub_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator); ~UngappedXDrop() override; StatusType extend_async(const char* h_query, const int32_t& query_length, @@ -54,14 +54,14 @@ public: private: DefaultDeviceAllocator allocator_; // Device ptr API required variables - int32_t* h_sub_mat_; - int32_t sub_mat_dim_; // Assume matrix is square - int32_t xdrop_threshold_; - bool no_entropy_; + const int32_t* h_sub_mat_; + const int32_t sub_mat_dim_; // Assume matrix is square + const int32_t xdrop_threshold_; + const bool no_entropy_; cudaStream_t stream_; - int32_t device_id_; + const int32_t device_id_; std::vector scored_segment_pairs_; - int32_t batch_max_ungapped_extensions_; + int32_t batch_max_ungapped_extensions_; // TODO - Make const device_buffer d_sub_mat_; // Pointer to device substitution matrix device_buffer d_done_; // TODO- Rename scratch space device_buffer d_tmp_ssp_; // TODO- Rename Scratch space 2 @@ -73,7 +73,7 @@ private: device_buffer d_query_; device_buffer d_target_; device_buffer d_seed_pairs_; - int32_t* d_num_ssp_; + device_buffer d_num_ssp_; device_buffer d_ssp_; std::vector h_ssp_; }; From dd9eed9563d86981ff5b31629532a9e44bebeb69 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 10:37:24 -0700 Subject: [PATCH 131/281] [cudaextender] make format --- cudaextender/samples/sample_cudaextender.cpp | 4 ++-- cudaextender/src/ungapped_xdrop.cu | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index 378061bcf..4ace26269 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -147,12 +147,12 @@ int main(int argc, char* argv[]) } // Fasta query and target files - const std::string target_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + const std::string target_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(target_file_path, 0, false); // Assumes that only one sequence is present per file const std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; - const std::string query_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + const std::string query_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(query_file_path, 0, false); // Assumes that only one sequence is present per file const std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index dfedde8df..de0549d55 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -161,7 +161,7 @@ StatusType UngappedXDrop::extend_async(const char* h_query, const int32_t& query // Allocate space for SeedPair input d_seed_pairs_ = device_buffer(h_seed_pairs.size(), allocator_, stream_); // Allocate space for ScoredSegmentPair output - d_ssp_ = device_buffer(h_seed_pairs.size(), allocator_, stream_); + d_ssp_ = device_buffer(h_seed_pairs.size(), allocator_, stream_); d_num_ssp_ = device_buffer(1, allocator_, stream_); // Async memcopy all the input values to device From 8a9b6314a60dea0a602c481a84f0e11051b2a641 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 10:53:06 -0700 Subject: [PATCH 132/281] [cudaextender] Partial docs --- .../genomeworks/cudaextender/extender.hpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index 89d886ad1..c279daed6 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -29,6 +29,7 @@ namespace genomeworks namespace cudaextender { +/// Seed positions in target and query reads struct SeedPair { /// position of first sketch element in query_read_id_ @@ -37,15 +38,16 @@ struct SeedPair position_in_read_t target_position_in_read; }; +/// Segment pairs in target & query reads with associated length & score struct ScoredSegmentPair { - /// Seed for the segment + /// seed for the segment SeedPair seed_pair; /// length of the segment int32_t length; /// score of the segment int32_t score; - + /// equality operator __host__ __device__ bool operator==(const ScoredSegmentPair& other) const { return ((seed_pair.target_position_in_read == other.seed_pair.target_position_in_read) && (seed_pair.query_position_in_read == other.seed_pair.query_position_in_read) && (length == other.length) && (score == other.score)); @@ -70,7 +72,7 @@ class Extender const int32_t& score_threshold, const std::vector& h_seed_pairs) = 0; - /// \brief Device pointer prototype for extension + /// \brief Device pointer prototype for extension /// /// Memcopies to device memory are assumed to be done before this /// function. Output array d_scored_segment_pairs must be pre-allocated on device. @@ -88,9 +90,9 @@ class Extender /// on the GPU to come to completion. virtual StatusType sync() = 0; - /// \brief Return the computed segments + /// \brief Return the computed segment pairs /// - /// \return Vector of Scored Segments + /// \return Vector of Scored Segment Pairs virtual const std::vector& get_scored_segment_pairs() const = 0; /// \brief Reset Extender object and free device/host memory From 073327467f8090d4e6e7d6a9b560ab34c32869d1 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 12:11:30 -0700 Subject: [PATCH 133/281] [cudaextender] Add end2end host ptr api test --- cudaextender/CMakeLists.txt | 1 + .../data/sample_scored_segment_pairs.csv | 1377 +++++++++++++++++ .../genomeworks/cudaextender/utils.hpp | 149 ++ cudaextender/samples/sample_cudaextender.cpp | 80 +- cudaextender/tests/CMakeLists.txt | 29 + .../tests/Test_CudaextenderEnd2End.cu | 126 ++ cudaextender/tests/main.cpp | 25 + 7 files changed, 1711 insertions(+), 76 deletions(-) create mode 100644 cudaextender/data/sample_scored_segment_pairs.csv create mode 100644 cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp create mode 100644 cudaextender/tests/CMakeLists.txt create mode 100644 cudaextender/tests/Test_CudaextenderEnd2End.cu create mode 100644 cudaextender/tests/main.cpp diff --git a/cudaextender/CMakeLists.txt b/cudaextender/CMakeLists.txt index 359b84e5f..d4ab14443 100644 --- a/cudaextender/CMakeLists.txt +++ b/cudaextender/CMakeLists.txt @@ -71,6 +71,7 @@ install(EXPORT ${MODULE_NAME} DESTINATION cmake) # Add other folders. add_subdirectory(samples) +add_subdirectory(tests) # Add auto formatting. gw_enable_auto_formatting("${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/cudaextender/data/sample_scored_segment_pairs.csv b/cudaextender/data/sample_scored_segment_pairs.csv new file mode 100644 index 000000000..6a5aadb69 --- /dev/null +++ b/cudaextender/data/sample_scored_segment_pairs.csv @@ -0,0 +1,1377 @@ +0,0,322,30338 +348,348,100,9497 +469,469,55,4828 +542,542,155,14790 +884,884,113,10752 +1018,1018,98,9279 +1985,1985,271,25535 +2278,2278,85,7354 +2383,2383,173,16392 +2607,2607,369,34723 +3005,3005,318,30037 +3456,3456,144,13546 +3646,3646,85,7519 +3761,3761,39,3417 +3824,3824,215,20331 +4118,4118,115,10880 +4256,4256,131,12354 +4450,4450,289,27227 +4760,4760,66,5652 +4852,4852,48,4447 +4924,4924,251,23661 +5202,5202,226,21323 +5449,5449,420,39418 +5903,5903,112,10643 +6037,6037,201,18958 +6260,6260,173,16284 +6457,6457,55,4805 +6553,6553,371,34860 +6996,6996,375,35152 +7544,7544,220,20651 +7954,7954,136,12944 +8151,8151,75,6802 +8304,8304,158,15180 +8511,8331,131,11794 +8511,8331,130,11825 +8491,8491,151,14471 +8671,8491,62,3179 +8331,8511,131,11794 +8331,8511,130,11825 +8671,8671,132,12634 +8491,8671,62,3179 +8837,8837,67,6316 +8924,8924,215,20520 +9170,9170,120,11452 +9317,9317,49,4565 +9436,9436,64,5977 +9649,9649,221,21120 +9889,9889,54,5011 +9974,9974,47,4575 +10042,10042,38,3652 +10123,10123,85,7916 +10230,10230,35,3399 +10438,10438,142,13616 +10837,10837,56,5343 +10934,10934,52,4830 +11051,11051,50,4804 +11137,11137,65,6059 +11325,11325,61,5700 +11427,11427,203,19203 +11806,11806,213,20311 +12027,12027,65,5787 +12122,12122,71,6525 +12218,12218,39,3479 +12324,12324,84,7902 +12433,12433,61,5700 +12617,12617,65,6059 +12718,12718,50,4842 +12826,12826,51,4727 +12919,12919,57,5465 +13232,13232,88,7841 +13350,13350,42,3887 +13436,13436,85,7677 +13638,13638,41,3671 +13808,13808,44,4141 +13979,13979,38,3490 +14046,14046,105,10114 +14311,14311,55,4996 +14387,14387,196,18674 +14603,14603,65,6276 +15028,15028,69,6164 +15315,15315,132,12562 +15467,15467,146,13908 +15667,15667,88,7441 +15799,15799,50,4496 +15969,15969,60,5819 +16157,16157,40,3683 +16218,16218,60,5005 +16433,16433,47,4492 +16785,16785,45,4246 +17087,17087,146,14061 +17353,17353,61,5589 +18502,18502,51,4504 +18896,18896,178,17153 +19142,19142,91,8738 +19672,19672,76,6933 +19808,19808,160,15164 +20026,20026,135,12799 +20182,20182,72,6751 +20272,20272,37,3519 +20357,20357,311,29544 +20614,20442,54,4309 +20442,20614,54,4309 +20768,20768,75,6765 +20867,20867,63,6063 +21161,21161,52,4852 +21248,21248,118,11486 +21386,21386,33,3163 +21441,21441,240,23218 +21711,21711,39,3611 +21780,21780,95,9231 +21894,21894,78,6761 +22003,22003,94,8401 +22377,22377,50,4702 +22609,22609,54,5113 +23429,23429,78,7417 +23528,23528,56,5072 +23635,23635,104,9960 +23793,23793,136,12962 +23966,23966,65,6084 +24052,24052,34,3213 +24377,24377,217,19665 +24377,24377,216,19574 +24377,24377,215,19483 +24377,24377,214,19383 +24377,24377,213,19292 +24377,24377,212,19192 +24377,24377,211,19092 +24377,24377,210,18992 +24377,24377,209,18892 +24377,24377,208,18801 +24377,24377,207,18710 +24377,24377,206,18610 +24377,24377,205,18519 +24377,24377,204,18428 +24377,24377,203,18328 +24377,24377,202,18237 +24377,24377,201,18146 +24377,24377,200,18046 +24377,24377,199,17955 +24377,24377,198,17864 +24377,24377,197,17764 +24377,24377,185,17709 +24532,24532,77,6283 +24533,24533,76,6202 +24534,24534,75,6096 +24535,24535,74,6013 +24536,24536,73,5929 +24537,24537,72,5833 +24538,24538,71,5726 +24539,24539,70,5641 +24540,24540,69,5531 +24541,24541,68,5436 +24542,24542,67,5323 +24543,24543,66,5238 +24544,24544,65,5142 +24545,24545,64,5050 +24550,24550,59,4508 +24551,24551,58,4412 +24552,24552,57,4286 +24564,24564,45,4230 +24629,24629,87,8402 +24872,24872,259,24857 +25805,25805,34,3219 +26023,26023,73,6691 +26150,26150,235,22367 +26443,26443,47,4288 +26854,26854,99,9415 +27016,27016,183,17545 +27327,27327,33,3220 +27556,27556,37,3487 +27709,27709,36,3446 +27896,27896,65,5631 +28041,28041,105,10123 +28183,28183,101,9714 +28347,28347,66,6283 +28432,28432,54,5146 +28706,28706,58,5394 +28856,28856,40,3834 +28916,28916,95,9006 +29032,29032,36,3166 +29205,29205,74,6595 +29395,29395,34,3137 +29472,29472,71,5809 +29744,29744,69,6267 +29854,29854,149,14127 +36118,29928,73,4536 +30118,30118,181,17318 +33375,30118,64,3958 +36336,30201,92,4948 +30323,30323,74,6767 +30419,30419,222,21130 +36562,30424,74,3865 +33692,30435,176,6484 +36663,30525,112,6106 +30671,30671,322,30599 +36809,30671,232,12529 +33917,30671,89,4156 +34034,30815,61,3445 +31014,31014,59,5503 +31106,31106,190,18002 +38209,31109,63,3562 +31451,31451,43,4025 +31604,31604,111,10606 +31747,31747,38,3551 +32282,32282,223,21194 +32828,32828,81,7665 +32941,32941,111,10606 +33168,33168,116,11106 +36110,33216,68,3773 +33313,33313,126,12016 +36215,33322,115,5444 +30118,33375,64,3958 +33469,33469,83,7617 +36353,33469,72,4502 +33570,33570,99,9451 +33692,33692,403,38114 +30435,33692,176,6484 +36803,33911,95,4291 +30671,33917,89,4156 +30815,34034,61,3445 +34114,34114,43,3947 +34181,34181,104,9942 +34545,34545,94,8584 +37697,34605,34,3036 +34980,34980,62,5952 +38214,34980,49,3647 +35062,35062,81,7601 +35166,35166,65,6240 +35460,35460,40,3232 +35526,35526,143,13590 +35687,35687,144,13636 +35951,35951,34,3192 +36037,36037,154,14672 +33216,36110,68,3773 +29928,36118,73,4536 +36214,36214,426,40396 +33322,36215,115,5444 +30201,36336,92,4948 +33469,36353,72,4502 +30424,36562,74,3865 +36662,36662,114,10897 +30525,36663,112,6106 +36803,36803,265,25097 +33911,36803,95,4291 +30671,36809,232,12529 +37087,37087,55,4995 +37166,37166,41,3787 +37269,37269,63,5896 +38171,37296,36,3041 +37466,37466,45,4223 +34605,37697,34,3036 +37697,37697,34,3036 +37857,37857,64,6062 +39341,37857,64,5843 +38109,38109,33,3048 +38171,38171,113,10770 +37296,38171,36,3041 +31109,38209,63,3562 +34980,38214,49,3647 +38337,38337,134,12735 +38687,38687,86,8022 +38970,38970,62,5694 +39164,39164,41,3884 +39341,39341,80,7543 +37857,39341,64,5843 +39691,39691,108,10342 +39843,39843,193,18590 +40144,40144,127,12116 +40744,40744,38,3445 +40917,40917,52,4765 +49109,40917,52,4765 +41087,41087,114,10726 +49280,41087,114,10726 +41250,41250,80,6765 +49443,41250,80,6765 +41393,41393,35,3142 +49586,41393,35,3142 +41592,41592,45,4006 +49785,41592,45,4006 +41771,41771,86,8088 +49964,41771,86,8088 +41877,41877,95,9042 +50070,41877,70,4895 +42000,42000,60,5479 +50192,42000,60,5479 +42079,42079,88,8363 +50271,42079,88,8363 +42596,42596,114,10870 +50700,42596,114,10626 +42767,42767,89,8255 +50871,42767,89,8255 +42929,42929,131,12426 +51033,42929,131,12426 +43143,43143,136,12944 +51247,43143,136,12944 +43330,43330,124,11996 +51434,43330,124,11996 +43521,43521,36,3203 +51625,43521,36,3203 +43588,43588,56,5375 +51692,43588,56,5375 +43662,43662,104,9969 +51766,43662,104,9969 +43935,43935,66,6017 +52039,43935,66,6017 +44039,44039,49,4750 +52143,44039,49,4750 +44169,44169,50,4364 +52273,44169,50,4364 +44280,44280,42,3924 +52384,44280,42,3924 +44423,44423,47,4434 +52527,44423,47,4434 +44492,44492,59,5566 +52596,44492,59,5566 +44572,44572,83,7553 +52676,44572,83,7553 +44761,44761,46,4388 +52865,44761,46,4388 +44831,44831,40,3863 +52935,44831,40,3863 +45029,45029,38,3547 +53133,45029,38,3547 +45103,45103,43,4119 +53207,45103,43,4119 +45181,45181,186,17710 +53285,45181,186,17710 +45557,45557,143,13635 +53661,45557,143,13635 +45721,45721,99,9460 +53825,45721,99,9460 +46433,46433,87,8009 +54537,46433,87,8009 +46538,46538,149,14190 +54642,46538,149,14190 +46842,46842,84,7553 +54946,46842,84,7553 +46944,46944,66,6273 +55048,46944,66,6273 +47165,47165,37,3242 +55269,47165,37,3242 +47227,47227,106,10133 +55331,47227,106,10133 +47414,47414,91,8357 +55518,47414,91,8357 +47525,47525,65,5932 +55629,47525,65,5932 +47608,47608,51,4933 +55712,47608,51,4933 +47708,47708,31,3038 +55812,47708,31,3038 +47804,47804,67,6074 +55908,47804,67,6074 +47956,47956,236,22287 +56060,47956,236,22287 +48211,48211,51,4762 +56315,48211,51,4762 +48420,48420,37,3440 +56524,48420,37,3440 +48514,48514,71,6525 +56618,48514,71,6525 +49094,49094,67,6027 +40917,49109,52,4765 +41087,49280,114,10726 +49280,49280,114,10726 +41250,49443,80,6765 +49443,49443,80,6765 +41393,49586,35,3142 +49586,49586,35,3142 +41592,49785,45,4006 +49785,49785,45,4006 +41771,49964,86,8088 +49964,49964,86,8088 +50070,50070,94,7735 +41877,50070,70,4895 +42000,50192,60,5479 +50192,50192,60,5479 +42079,50271,88,8363 +50271,50271,88,8363 +42596,50700,114,10626 +50700,50700,114,10888 +42767,50871,89,8255 +50871,50871,89,8255 +42929,51033,131,12426 +51033,51033,131,12426 +43143,51247,136,12944 +51247,51247,136,12944 +43330,51434,124,11996 +51434,51434,124,11996 +43521,51625,36,3203 +51625,51625,36,3203 +43588,51692,56,5375 +51692,51692,56,5375 +43662,51766,104,9969 +51766,51766,104,9969 +43935,52039,66,6017 +52039,52039,66,6017 +44039,52143,49,4750 +52143,52143,49,4750 +44169,52273,50,4364 +52273,52273,50,4364 +44280,52384,42,3924 +52384,52384,42,3924 +44423,52527,47,4434 +52527,52527,47,4434 +44492,52596,59,5566 +52596,52596,59,5566 +44572,52676,83,7553 +52676,52676,83,7553 +44761,52865,46,4388 +52865,52865,46,4388 +44831,52935,40,3863 +52935,52935,40,3863 +45029,53133,38,3547 +53133,53133,38,3547 +45103,53207,43,4119 +53207,53207,43,4119 +45181,53285,186,17710 +53285,53285,186,17710 +45557,53661,143,13635 +53661,53661,143,13635 +45721,53825,99,9460 +53825,53825,99,9460 +46433,54537,87,8009 +54537,54537,87,8009 +46538,54642,149,14190 +54642,54642,149,14190 +46842,54946,84,7553 +54946,54946,84,7553 +46944,55048,66,6273 +55048,55048,66,6273 +47165,55269,37,3242 +55269,55269,37,3242 +47227,55331,106,10133 +55331,55331,106,10133 +47414,55518,91,8357 +55518,55518,91,8357 +47525,55629,65,5932 +55629,55629,65,5932 +47608,55712,51,4933 +55712,55712,51,4933 +47708,55812,31,3038 +55812,55812,31,3038 +47804,55908,67,6074 +55908,55908,67,6074 +47956,56060,236,22287 +56060,56060,236,22287 +48211,56315,51,4762 +56315,56315,51,4762 +48420,56524,37,3440 +56524,56524,37,3440 +56618,56618,207,19657 +48514,56618,71,6525 +56998,56998,89,8275 +57121,57121,52,4642 +57272,57272,92,8481 +57393,57393,36,3204 +57477,57477,45,4019 +57548,57548,70,6735 +57636,57636,262,24698 +57786,57640,111,4147 +57937,57643,90,4372 +57640,57786,111,4147 +57937,57789,87,3504 +58099,57800,55,3108 +57937,57937,144,13663 +57643,57937,90,4372 +57789,57937,87,3504 +58099,58099,84,7689 +57800,58099,55,3108 +58212,58212,151,14507 +58382,58382,104,10086 +58587,58587,106,10061 +58712,58712,118,11225 +59086,58712,118,11225 +58987,58987,50,4679 +58712,59086,118,11225 +59086,59086,118,11225 +59330,59330,121,11480 +59459,59459,122,11706 +59600,59600,188,17919 +59849,59849,218,20883 +60086,60086,68,6568 +60184,60184,92,8683 +60323,60323,135,12862 +60542,60542,145,13952 +60712,60712,51,4577 +60791,60791,60,5721 +60901,60901,143,13644 +61165,61165,37,3397 +61328,61328,44,4191 +61409,61409,84,7803 +61708,61708,81,7490 +61970,61970,45,4101 +62041,62041,57,5058 +62248,62248,35,3194 +62352,62352,266,25404 +62649,62649,229,22010 +62936,62936,81,7377 +63125,63125,47,3993 +63380,63380,140,13542 +63679,63679,198,19099 +63936,63936,109,10505 +64063,64063,161,15498 +64286,64286,41,3659 +64442,64442,62,5651 +64658,64658,85,7480 +64775,64775,408,39019 +65554,65554,71,6311 +65677,65677,97,9269 +65797,65797,89,8255 +65911,65911,60,5446 +66090,66090,78,7343 +66270,66270,85,7936 +66634,66270,54,4807 +66451,66451,62,5953 +66534,66534,42,3377 +66270,66634,54,4807 +66634,66634,54,5113 +66778,66778,48,4065 +66864,66864,48,4225 +66930,66930,52,4621 +66990,66990,41,3913 +67094,67094,231,22255 +67348,67348,41,3582 +67518,67518,111,10543 +67723,67723,55,5284 +67929,67929,172,16454 +68122,68122,179,17145 +68465,68465,52,4819 +69770,68465,37,3342 +68594,68594,68,6103 +69615,69615,43,4096 +69768,69768,81,7814 +68465,69770,37,3342 +70132,70132,423,40447 +70623,70623,43,3965 +71033,71033,66,5978 +71121,71121,122,11742 +71294,71294,97,9323 +71854,71854,40,3826 +72164,72164,34,3295 +72241,72241,64,5907 +72460,72460,145,14051 +72855,72855,50,4506 +73001,73001,86,7973 +73730,73730,40,3537 +73998,73998,93,8210 +74109,74109,202,19472 +74676,74676,52,4707 +74889,74889,726,69487 +75672,75672,75,6694 +75767,75767,176,16836 +75962,75962,159,15118 +76256,76256,408,39100 +76776,76776,715,68405 +77512,77512,214,20627 +77746,77746,57,5217 +77823,77823,35,3072 +77943,77943,140,13263 +78351,78351,51,4672 +78494,78494,33,3077 +78547,78547,193,18482 +78761,78761,103,9941 +78882,78882,294,28060 +79195,79195,77,7326 +79293,79293,64,6176 +79377,79377,112,10760 +79516,79516,74,6909 +79749,79749,731,70059 +80501,80501,424,40853 +81105,81105,32,3118 +81169,81169,42,4044 +81292,81292,353,33978 +81740,81740,48,4592 +81812,81812,62,5965 +81994,81994,57,5381 +82301,82301,57,5194 +82518,82518,119,11460 +82715,82715,60,5765 +82830,82830,52,4641 +82926,82926,39,3697 +82990,82990,86,8075 +83105,83105,48,4605 +83191,83191,140,13452 +83352,83352,90,8434 +83464,83464,93,9076 +83587,83587,257,24603 +83865,83865,260,24831 +84144,84144,174,16573 +84343,84343,46,4113 +84485,84485,252,24067 +84748,84748,102,9877 +84890,84890,60,5691 +85172,85172,68,6150 +85395,85395,137,13188 +85555,85555,161,15390 +85760,85760,145,13925 +85935,85935,115,11087 +86071,86071,62,5914 +86329,86329,139,13370 +86503,86503,76,7205 +86601,86601,159,15163 +86800,86800,117,11296 +86937,86937,34,3070 +87078,87078,96,9232 +87197,87197,62,5217 +87328,87328,79,7065 +87478,87478,142,13499 +87734,87734,66,6290 +87862,87862,79,7606 +88015,88015,251,23805 +88371,88371,56,5312 +88480,88480,57,5123 +88590,88590,165,15817 +88821,88821,43,4170 +88921,88921,86,7683 +89124,89124,109,10460 +89300,89300,35,3283 +89424,89424,203,19383 +89681,89681,38,3508 +89792,89792,76,7255 +89923,89923,51,4737 +90097,90097,103,9941 +90232,90232,57,5337 +90528,90528,224,21465 +90938,90938,95,9159 +91443,91443,47,4484 +91909,91909,85,8211 +92035,92035,99,9541 +92483,92483,37,3420 +92572,92572,152,14598 +92746,92746,48,4652 +93368,93368,40,3771 +93802,93802,75,6991 +93959,93959,58,5275 +94042,94042,63,6038 +94460,94460,42,3849 +94922,94922,83,8017 +95023,95023,47,4517 +95863,95863,66,5740 +95996,95996,59,5586 +96177,96177,33,3045 +96314,96314,69,6675 +96405,96405,81,7500 +97375,97375,61,5338 +97458,97458,46,3995 +97628,97628,130,12614 +97837,97837,330,31615 +98188,98188,59,5659 +98300,98300,44,3908 +98733,98733,32,3045 +99197,99197,198,19045 +99541,99541,41,3749 +100533,100533,223,21347 +100778,100778,85,8031 +100896,100896,42,3937 +101005,101005,48,4352 +101697,101697,72,6575 +101810,101810,473,45366 +102313,102313,115,11051 +102459,102459,115,10943 +102601,102601,69,6326 +103351,103351,355,33863 +103918,103918,62,5852 +104010,104010,50,4531 +104111,104111,38,3610 +104279,104279,163,15644 +104611,104611,468,44947 +105105,105105,41,3779 +105604,105604,206,19809 +105849,105849,41,3994 +106093,106093,46,4167 +106160,106160,192,18463 +106497,106497,54,5217 +106623,106623,1038,99256 +107943,107943,66,6209 +108157,108157,35,3219 +108275,108275,59,5584 +108413,108413,393,37600 +108825,108825,348,33262 +109194,109194,95,9114 +109498,109498,118,11288 +109782,109782,50,4625 +109855,109855,73,6787 +109950,109950,39,3583 +110022,110022,90,8528 +110256,110256,135,12817 +110495,110495,47,4084 +110560,110560,1768,168764 +112412,112412,60,5416 +113281,113281,314,29952 +114203,114203,52,4718 +114288,114288,199,19109 +114648,114648,255,24286 +115760,115760,50,4439 +115914,115914,104,9951 +116317,116317,86,7812 +116527,116527,109,10433 +116720,116720,34,3130 +116773,116773,85,8174 +116976,116976,115,11042 +117663,117663,82,7620 +117784,117784,48,4647 +118166,118166,52,4470 +118283,118283,37,3056 +118414,118414,58,5210 +118493,118493,36,3271 +118594,118594,66,6291 +118970,118970,36,3041 +119144,119144,35,3146 +119203,119203,89,8249 +119733,119733,52,4375 +120148,120148,176,16764 +120368,120368,94,9059 +120663,120663,43,3529 +120929,120929,39,3250 +121287,121287,282,26860 +121672,121672,34,3182 +122150,122150,109,10343 +122574,122574,52,4952 +122647,122647,547,52217 +123246,123246,53,4737 +123329,123329,72,6966 +123476,123476,44,4281 +123910,123910,78,7361 +124049,124049,165,15610 +131674,124087,127,12071 +124291,124291,118,11288 +131878,124291,118,11288 +124496,124303,55,3384 +132083,124303,55,3384 +124495,124495,56,5205 +132082,124495,56,5205 +124303,124496,55,3384 +131890,124496,55,3384 +124681,124681,45,4133 +132268,124681,45,4133 +124745,124745,80,7637 +132332,124745,80,7637 +124892,124892,55,5227 +132479,124892,55,5227 +124991,124991,74,6877 +132578,124991,74,6877 +125096,125096,160,15200 +132683,125096,160,15200 +125308,125308,97,9305 +132895,125308,97,9305 +125426,125426,101,9606 +133013,125426,101,9606 +125576,125576,84,7775 +133163,125576,84,7775 +125937,125937,55,5267 +133524,125937,55,5267 +126067,126067,110,10443 +133654,126067,110,10443 +126377,126377,383,36303 +133964,126377,383,36303 +127456,127456,89,8252 +135043,127456,89,8252 +127595,127595,116,11070 +135182,127595,116,11070 +127811,127811,123,11743 +135398,127811,123,11743 +127979,127979,32,3145 +135566,127979,32,3145 +128071,128071,32,3008 +135658,128071,32,3008 +128138,128138,132,12589 +135725,128138,132,12589 +128308,128308,62,5752 +135895,128308,62,5752 +128420,128420,254,24195 +136007,128420,254,24195 +128698,128698,131,12597 +136285,128698,131,12597 +128849,128849,160,15290 +136436,128849,160,15290 +129161,129161,43,3985 +136748,129161,43,3985 +129337,129337,190,18128 +136924,129337,190,18128 +129546,129546,57,5075 +137133,129546,57,5075 +129708,129708,68,6478 +137295,129708,68,6478 +130115,129708,52,4083 +137702,129708,52,4083 +129923,129923,51,4754 +137510,129923,51,4754 +130125,129924,50,3428 +137712,129924,50,3428 +130111,130111,65,6202 +137698,130111,65,6202 +129708,130115,52,4083 +137295,130115,52,4083 +129924,130125,50,3428 +137511,130125,50,3428 +130317,130317,131,12462 +137904,130317,131,12462 +130472,130472,115,11015 +138059,130472,115,11015 +124087,131674,127,12071 +131674,131674,127,12071 +124291,131878,118,11288 +131878,131878,118,11288 +124496,131890,55,3384 +132083,131890,55,3384 +124495,132082,56,5205 +132082,132082,56,5205 +124303,132083,55,3384 +131890,132083,55,3384 +124681,132268,45,4133 +132268,132268,45,4133 +124745,132332,80,7637 +132332,132332,80,7637 +124892,132479,55,5227 +132479,132479,55,5227 +124991,132578,74,6877 +132578,132578,74,6877 +125096,132683,160,15200 +132683,132683,160,15200 +125308,132895,97,9305 +132895,132895,97,9305 +125426,133013,101,9606 +133013,133013,101,9606 +125576,133163,84,7775 +133163,133163,84,7775 +125937,133524,55,5267 +133524,133524,55,5267 +126067,133654,110,10443 +133654,133654,110,10443 +126377,133964,383,36303 +133964,133964,383,36303 +127456,135043,89,8252 +135043,135043,89,8252 +127595,135182,116,11070 +135182,135182,116,11070 +127811,135398,123,11743 +135398,135398,123,11743 +127979,135566,32,3145 +135566,135566,32,3145 +128071,135658,32,3008 +135658,135658,32,3008 +128138,135725,132,12589 +135725,135725,132,12589 +128308,135895,62,5752 +135895,135895,62,5752 +128420,136007,254,24195 +136007,136007,254,24195 +128698,136285,131,12597 +136285,136285,131,12597 +128849,136436,160,15290 +136436,136436,160,15290 +129161,136748,43,3985 +136748,136748,43,3985 +129337,136924,190,18128 +136924,136924,190,18128 +129546,137133,57,5075 +137133,137133,57,5075 +129708,137295,68,6478 +137295,137295,68,6478 +130115,137295,52,4083 +137702,137295,52,4083 +129923,137510,51,4754 +137510,137510,51,4754 +130125,137511,50,3428 +137712,137511,50,3428 +130111,137698,65,6202 +137698,137698,65,6202 +129708,137702,52,4083 +137295,137702,52,4083 +129924,137712,50,3428 +137511,137712,50,3428 +130317,137904,131,12462 +137904,137904,131,12462 +130472,138059,115,11015 +138059,138059,115,11015 +138509,138509,40,3722 +138621,138621,163,15518 +138831,138831,143,13743 +139017,139017,60,5658 +139101,139101,94,8712 +139241,139241,64,6078 +139651,139651,135,12898 +139865,139865,201,19183 +140086,140086,94,8590 +142476,140125,55,3783 +140211,140211,124,11861 +140358,140358,167,15873 +140581,140581,134,12717 +140783,140783,91,7980 +140961,140961,64,5785 +141116,141116,810,76996 +142940,141352,155,5045 +143119,141531,213,10995 +141952,141952,351,33445 +145143,142163,139,5128 +142322,142322,61,5751 +142404,142404,300,28264 +145433,142437,76,4196 +140125,142476,55,3783 +142725,142725,91,8414 +142837,142837,258,24676 +141352,142940,155,5045 +143117,143117,252,23959 +141531,143119,213,10995 +143668,143668,54,4596 +143864,143864,48,4207 +143941,143941,160,15083 +144129,144129,155,14826 +144415,144415,46,4042 +144821,144821,178,16955 +145026,145026,256,24386 +142163,145143,139,5128 +145307,145307,237,22675 +142437,145433,76,4196 +145579,145579,65,5793 +145681,145681,168,15973 +146086,146086,77,7345 +155381,146089,73,3880 +146308,146308,320,30453 +146655,146655,515,49089 +147298,147298,57,5183 +147497,147497,101,9651 +147622,147622,84,7779 +147861,147861,48,4644 +148090,148090,432,41023 +148600,148600,70,6723 +148782,148782,46,4405 +148964,148964,189,18037 +149173,149173,283,27068 +149593,149593,125,11961 +149737,149737,178,17063 +150071,150071,46,4365 +150261,150261,40,3707 +150441,150441,65,5927 +150535,150535,363,34636 +151093,151093,122,11778 +151301,151301,334,31844 +151772,151772,46,4485 +152184,152184,78,7495 +152754,152754,38,3707 +152817,152817,220,20966 +153061,153061,144,13762 +153279,153279,68,6476 +153614,153614,51,4781 +153847,153847,292,27959 +154183,154183,39,3674 +154304,154304,69,6619 +154480,154480,142,13724 +154689,154689,254,24258 +154964,154964,287,27324 +155381,155381,75,7195 +146089,155381,73,3880 +155585,155585,59,5617 +155835,155835,126,11998 +156240,156240,270,25813 +156609,156609,100,9596 +156775,156775,68,6325 +156995,156995,236,22566 +157352,157352,53,5115 +157513,157513,86,7911 +157734,157734,190,18056 +157975,157975,195,18529 +158191,158191,56,5095 +158279,158279,67,6223 +158399,158399,122,11679 +158566,158566,244,23456 +158953,158953,43,3812 +159099,159099,40,3826 +159211,159211,66,6060 +159322,159322,207,19657 +160028,160028,147,14188 +160334,160334,141,13390 +160533,160533,43,3580 +160602,160602,329,31362 +160950,160950,198,18946 +161167,161167,124,11870 +161338,161338,64,6161 +161494,161494,154,14951 +161714,161714,270,25750 +162129,162129,60,5784 +162210,162210,63,5767 +162293,162293,414,39367 +162730,162730,163,15599 +162919,162919,167,15972 +163313,163313,116,11016 +163447,163447,130,12479 +163662,163662,63,5803 +163745,163745,33,3071 +163946,163946,219,20857 +164417,164417,125,11844 +164577,164577,100,9560 +164733,164733,355,33854 +165149,165149,278,26676 +165739,165739,32,3066 +166414,166414,125,12042 +166558,166558,232,22211 +167912,167912,51,4940 +173876,167912,51,4940 +167986,167986,77,7273 +173950,167986,77,7273 +168173,168173,50,4562 +174137,168173,50,4562 +168273,168273,63,5889 +174237,168273,63,5889 +168354,168354,51,4857 +174318,168354,51,4857 +168624,168624,57,5412 +174588,168624,57,5412 +170012,170012,38,3655 +175976,170012,38,3655 +170248,170248,77,7340 +176212,170248,77,7340 +170428,170428,113,10815 +176392,170428,113,10815 +170549,170549,283,27077 +176513,170549,283,27077 +171436,171436,38,3399 +177400,171436,38,3399 +171494,171494,380,36462 +177458,171494,380,36462 +172077,172077,356,34260 +178041,172077,356,34260 +172483,172483,121,11588 +178447,172483,121,11588 +167912,173876,51,4940 +173876,173876,51,4940 +167986,173950,77,7273 +173950,173950,77,7273 +168173,174137,50,4562 +174137,174137,50,4562 +168273,174237,63,5889 +174237,174237,63,5889 +168354,174318,51,4857 +174318,174318,51,4857 +168624,174588,57,5412 +174588,174588,57,5412 +170012,175976,38,3655 +175976,175976,38,3655 +170248,176212,77,7340 +176212,176212,77,7340 +170428,176392,113,10815 +176392,176392,113,10815 +170549,176513,283,27077 +176513,176513,283,27077 +171436,177400,38,3399 +177400,177400,38,3399 +171494,177458,380,36462 +177458,177458,380,36462 +172077,178041,356,34260 +178041,178041,356,34260 +178447,178447,1125,107281 +172483,178447,121,11588 +179594,179594,581,55545 +180195,180195,574,54836 +180791,180791,127,12215 +180943,180943,202,19418 +181199,181199,171,16345 +182292,182292,38,3305 +182976,182976,39,3723 +183035,183035,39,3750 +183094,183094,35,3176 +183213,183213,58,5636 +183326,183326,294,28123 +183641,183641,49,4559 +183713,183713,51,4822 +183811,183811,161,15435 +184022,184022,62,5871 +184105,184105,249,23902 +184375,184375,290,27795 +184689,184689,131,12579 +184954,184954,149,14271 +185127,185127,71,6847 +185258,185258,264,25240 +185540,185540,85,8146 +185651,185651,101,9768 +185799,185799,156,14980 +186025,186025,34,3160 +186265,186265,55,5207 +186343,186343,84,7693 +186775,186775,34,3036 +186985,186985,498,47308 +187727,187727,54,4897 +187927,187927,125,11970 +188167,188167,110,10587 +188299,188299,112,10787 +188455,188455,105,9943 +188673,188673,37,3382 +188952,188952,273,26320 +189394,189394,229,22019 +189644,189644,1048,100238 +190096,189915,115,4397 +189915,190096,115,4397 +190725,190725,580,55454 +191361,191361,61,5764 +191484,191484,143,13635 +191719,191498,122,4111 +191779,191498,114,3315 +191718,191718,180,17119 +191498,191719,122,4111 +207754,191778,118,3115 +191498,191779,114,3315 +191924,191924,124,11879 +192107,192107,370,35462 +192501,192501,127,12170 +193020,193020,57,5292 +193122,193122,48,4467 +193237,193237,70,6552 +193328,193328,132,12733 +193524,193524,54,4857 +193652,193652,197,18873 +193995,193995,32,3038 +194080,194080,403,38636 +194691,194691,99,9496 +194891,194891,87,8272 +195074,195074,137,13035 +195497,195497,79,6527 +195671,195671,51,4698 +195757,195757,58,5561 +196241,196241,93,8717 +196460,196460,157,15134 +196651,196651,110,10704 +196900,196900,37,3544 +197296,197296,47,4129 +197474,197474,155,14826 +197657,197657,104,9825 +197790,197790,48,4082 +197985,197985,54,4596 +198377,198377,97,9269 +198636,198636,124,11915 +198783,198783,39,3626 +199166,199166,138,13216 +199512,199512,103,9878 +199690,199690,43,3993 +199803,199803,81,7785 +200442,200442,196,18890 +200711,200711,101,9624 +200838,200838,55,4890 +200952,200952,242,23112 +201312,201312,38,3639 +201371,201371,207,19738 +201598,201598,33,3129 +201652,201652,57,5139 +201738,201738,48,4338 +202007,202007,122,11841 +202278,202278,52,4910 +202355,202355,95,9132 +202469,202469,439,42065 +202979,202979,50,4711 +203416,203416,216,20773 +203653,203653,429,41263 +204104,204104,94,9050 +204232,204232,56,5253 +204333,204333,39,3446 +204397,204397,273,26230 +204689,204689,97,9530 +204812,204812,126,12268 +205001,205001,340,32813 +205410,205410,106,10169 +205540,205540,95,9096 +205658,205658,53,4726 +206040,205667,44,3243 +205790,205790,53,4734 +205864,205864,141,13552 +206034,206034,96,9178 +205667,206040,44,3243 +206177,206177,43,3928 +206244,206244,61,5856 +206365,206365,118,11333 +206504,206504,455,43476 +206980,206980,88,8222 +207087,207087,562,53717 +207753,207543,95,3594 +207667,207667,66,6355 +207818,207668,64,3496 +207751,207751,192,18301 +207793,207753,119,3189 +207543,207753,95,3594 +207843,207753,67,3862 +191778,207754,118,3115 +207753,207793,119,3189 +207668,207818,64,3496 +207753,207843,67,3862 +207964,207964,942,89854 +209321,209321,514,49151 +209856,209856,516,49414 +211060,211060,644,61476 +211740,211740,86,8026 +211875,211875,262,24950 +212240,212240,119,11532 +212877,212877,36,3404 +213259,213259,76,7228 +213533,213533,344,33051 +213903,213903,69,6288 +214022,214022,70,6091 +214121,214121,160,15371 +214299,214299,281,26922 +214607,214607,140,13371 +214770,214770,60,5764 +215326,215326,228,21748 +215577,215577,82,7075 +215875,215875,429,41011 +216325,216325,122,11706 +216466,216466,357,34243 +216898,216898,143,13878 +217065,217065,48,4571 +217138,217138,47,4419 +217210,217210,84,7394 +217454,217454,43,4037 +217541,217541,62,5557 +217664,217664,97,9323 +217787,217787,77,7051 +217885,217885,195,18601 +218178,218178,71,6505 +218269,218269,58,5399 +218357,218357,172,16364 +218636,218636,38,3587 +219376,219376,44,3898 +219776,219776,68,6486 +219899,219899,224,21537 +220147,220147,96,9277 +220264,220264,217,20927 +220545,220545,99,9406 +220815,220815,32,3101 +220922,220922,40,3801 +222442,220922,40,3169 +221001,221001,39,3731 +221140,221140,71,6529 +221631,221631,649,62246 +222325,222325,44,4252 +222440,222440,42,4012 +220922,222442,40,3169 +222635,222635,36,3140 +222748,222748,53,5092 +222823,222823,127,12152 +222972,222972,138,13360 +223136,223136,70,6768 +223227,223227,45,4271 +223323,223323,33,3133 +223377,223377,130,12362 +223583,223583,53,4842 +223741,223741,37,3070 +224330,224330,127,12125 +224523,224523,120,11506 +224701,224701,98,9405 +224849,224849,129,12487 +225005,225005,36,3281 +225061,225061,32,3077 +225114,225114,219,20857 +225354,225354,45,4202 +225442,225442,143,13752 +225616,225616,40,3655 +225771,225771,60,5560 +225903,225903,35,3306 +225963,225963,35,3257 +226084,226084,99,9595 +226279,226279,156,14953 +226454,226454,96,9259 +226575,226575,329,31389 +226966,226966,161,15381 +227476,227476,178,17189 +227685,227685,193,18374 +227936,227936,40,3861 +227994,227994,68,6001 +228682,228682,79,7253 +228948,228948,37,3560 +229142,229142,39,3525 +229538,229538,54,4742 +229682,229682,43,3873 +229748,229748,67,6430 +229947,229947,62,5754 +230226,230226,33,3077 +230322,230322,80,7041 +230521,230521,63,6011 +230848,230848,34,3003 +230904,230904,55,5332 +231179,231179,177,16909 +231364,231364,116,11214 +231621,231621,44,3927 +231686,231686,199,18992 +231909,231909,152,14364 +232085,232085,69,6558 +232619,232619,33,3133 +232704,232704,41,3500 +232869,232869,118,11207 +233007,233007,138,13279 +233197,233197,675,64738 +233892,233892,114,10870 +234131,234131,207,19810 +234385,234385,231,22210 +234635,234635,449,42696 +234914,234829,46,3273 +234829,234914,46,3273 +235105,235105,109,10415 +235240,235240,61,5819 +235341,235341,47,4013 +235415,235415,71,6600 +235587,235587,154,14825 +235769,235769,147,14116 +235936,235936,63,6077 +236024,236024,43,4147 +236089,236089,69,6310 +236316,236316,40,3514 +236386,236386,49,4549 +236458,236458,107,10188 +236611,236611,47,4350 +236790,236790,40,3746 +236856,236856,43,3919 +236920,236920,48,4361 +237448,237448,61,5870 +237582,237582,32,3056 +237929,237929,93,8155 +238210,238210,76,7276 +238588,238588,40,3871 +238797,238797,48,4358 +239326,239326,44,4215 +239474,239474,66,6062 +239653,239653,51,4772 +239847,239847,36,3495 +240042,240042,43,3831 +240248,240248,61,5063 +240393,240393,43,3956 +240499,240499,33,3077 +240864,240864,111,10651 +240998,240998,124,11789 +241363,241363,46,3910 +241487,241487,45,3819 +241553,241553,48,4238 +241619,241619,51,4490 +241746,241746,124,11807 +245738,241746,60,4729 +245817,241825,45,3661 +242022,242022,133,12671 +242249,242249,138,13063 +242568,242568,116,11016 +242731,242731,70,5695 +242820,242820,189,17893 +243115,243115,34,3050 +243189,243189,47,4350 +243256,243256,249,23605 +243529,243529,70,6329 +243776,243776,107,10215 +243903,243903,78,6658 +244004,244004,124,11780 +244149,244149,61,5464 +244289,244289,73,6735 +244680,244680,84,7982 +244786,244786,59,5447 +244876,244876,62,5975 +244983,244983,37,3429 +245052,245052,44,4296 +245173,245173,41,3835 +245380,245380,112,10643 +241746,245738,60,4729 +245738,245738,60,5449 +241825,245817,45,3661 +245817,245817,45,3661 +245982,245982,69,6239 +246352,246352,35,3333 +246544,246544,211,19967 +246913,246913,126,12061 +247182,247182,57,5303 +247273,247273,99,9532 +247396,247396,69,6417 +247509,247509,102,9868 +247773,247773,290,27498 +248084,248084,85,7278 +248645,248645,166,15782 +248880,248880,52,4691 +249041,249041,91,8100 +249188,249188,74,6757 +249375,249375,75,7153 +249548,249548,71,6601 +249737,249737,33,3187 +249960,249960,39,3655 diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp new file mode 100644 index 000000000..56d6e714c --- /dev/null +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp @@ -0,0 +1,149 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#pragma once +#include + +#include +#include +#include +#include + +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ + +constexpr char A_NT = 0; +constexpr char C_NT = 1; +constexpr char G_NT = 2; +constexpr char T_NT = 3; +constexpr char L_NT = 4; +constexpr char N_NT = 5; +constexpr char X_NT = 6; +constexpr char E_NT = 7; +constexpr char NUC = 8; +constexpr char NUC2 = NUC * NUC; + +/// \brief Parses seed pairs from a csv file in the following format: +/// target_position_in_read_1, query_position_in_read_1 +/// target_position_in_read_2, query_position_in_read_2 +/// target_position_in_read_n, query_position_in_read_n +/// +/// \param[out] seed_pairs Reference to vector into which parsed seed pairs are saved +/// \param[in] filepath Reference to the string containing the path of the seed pairs csv +inline void parse_seed_pairs(const std::string& filepath, std::vector& seed_pairs) +{ + std::ifstream seed_pairs_file(filepath); + if (!seed_pairs_file.is_open()) + throw std::runtime_error("Cannot open file"); + if (seed_pairs_file.good()) + { + std::string line; + while (std::getline(seed_pairs_file, line, ',')) + { + SeedPair seed_pair; + seed_pair.target_position_in_read = std::atoi(line.c_str()); + std::getline(seed_pairs_file, line); // Get the next value + seed_pair.query_position_in_read = std::atoi(line.c_str()); + seed_pairs.push_back(seed_pair); + } + } +} + +/// \brief Parses scored segment pairs from a csv file in the following format: +/// target_position_in_read_1, query_position_in_read_1, score_1, length_1 +/// target_position_in_read_2, query_position_in_read_2, score_2, length_2 +/// target_position_in_read_n, query_position_in_read_n, score_n, length_n +/// +/// \param[out] scored_segment_pairs Reference to vector into which parsed scored segment pairs are saved +/// \param[in] filepath Reference to the string containing the path of the scored segment pairs csv +inline void parse_scored_segment_pairs(const std::string& filepath, std::vector& scored_segment_pairs) +{ + std::ifstream scored_segment_pairs_file(filepath); + if (!scored_segment_pairs_file.is_open()) + throw std::runtime_error("Cannot open file"); + if (scored_segment_pairs_file.good()) + { + std::string line; + while (std::getline(scored_segment_pairs_file, line, ',')) + { + ScoredSegmentPair scored_segment_pair; + scored_segment_pair.seed_pair.target_position_in_read = std::atoi(line.c_str()); + std::getline(scored_segment_pairs_file, line, ','); + scored_segment_pair.seed_pair.query_position_in_read = std::atoi(line.c_str()); + std::getline(scored_segment_pairs_file, line, ','); + scored_segment_pair.score = std::atoi(line.c_str()); + std::getline(scored_segment_pairs_file, line); + scored_segment_pair.length = std::atoi(line.c_str()); + scored_segment_pairs.push_back(scored_segment_pair); + } + } +} + +/// \brief Encodes character sequence as integer sequence +/// +/// \param[out] dst_seq Pointer to pre-allocated storage for encoded sequence +/// \param[in] src_seq Pointer to input sequence +/// \param[in] length Length of the sequence +inline void encode_sequence(char* dst_seq, const char* src_seq, const int32_t& length) +{ + for (int32_t i = 0; i < length; i++) + { + const char ch = src_seq[i]; + switch (ch) + { + case 'A': + dst_seq[i] = A_NT; + break; + case 'C': + dst_seq[i] = C_NT; + break; + case 'G': + dst_seq[i] = G_NT; + break; + case 'T': + dst_seq[i] = T_NT; + break; + case '&': + dst_seq[i] = E_NT; + break; + case 'n': + case 'N': + dst_seq[i] = N_NT; + break; + case 'a': + case 'c': + case 'g': + case 't': + dst_seq[i] = L_NT; + break; + default: + dst_seq[i] = X_NT; + break; + } + } +} + +} // namespace cudapoa + +} // namespace genomeworks + +} // namespace claraparabricks diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index 4ace26269..18b8d0e79 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -15,6 +15,7 @@ */ #include #include +#include #include #include #include @@ -23,84 +24,12 @@ #include #include #include -#include #include using namespace claraparabricks::genomeworks; using namespace cudautils; using namespace cudaextender; -constexpr char A_NT = 0; -constexpr char C_NT = 1; -constexpr char G_NT = 2; -constexpr char T_NT = 3; -constexpr char L_NT = 4; -constexpr char N_NT = 5; -constexpr char X_NT = 6; -constexpr char E_NT = 7; -constexpr char NUC = 8; -constexpr char NUC2 = NUC * NUC; - -// Really simple parser with no error checks -void parse_seed_pairs(const std::string& filepath, std::vector& seed_pairs) -{ - std::ifstream seed_pair_file(filepath); - if (!seed_pair_file.is_open()) - throw std::runtime_error("Cannot open file"); - if (seed_pair_file.good()) - { - std::string line; - while (std::getline(seed_pair_file, line, ',')) - { - SeedPair seed_pair; - seed_pair.target_position_in_read = std::atoi(line.c_str()); - std::getline(seed_pair_file, line); // Get the next value - seed_pair.query_position_in_read = std::atoi(line.c_str()); - seed_pairs.push_back(seed_pair); - } - } -} - -// convert input sequence from alphabet to integers -void encode_string(char* dst_seq, const char* src_seq, const int32_t& len) -{ - for (int32_t i = 0; i < len; i++) - { - const char ch = src_seq[i]; - switch (ch) - { - case 'A': - dst_seq[i] = A_NT; - break; - case 'C': - dst_seq[i] = C_NT; - break; - case 'G': - dst_seq[i] = G_NT; - break; - case 'T': - dst_seq[i] = T_NT; - break; - case '&': - dst_seq[i] = E_NT; - break; - case 'n': - case 'N': - dst_seq[i] = N_NT; - break; - case 'a': - case 'c': - case 'g': - case 't': - dst_seq[i] = L_NT; - break; - default: - dst_seq[i] = X_NT; - break; - } - } -} - void print_scored_segment_pairs(const std::vector& scored_segment_pairs) { std::cout << "Target Position, Query Position, Length, Score" << std::endl; @@ -164,8 +93,7 @@ int main(int argc, char* argv[]) //TODO - pinned seed_pairs std::vector h_seed_pairs; // Following function loops through all seed_pairs in the sample_seed_pairs.csv and returns - // results in - // the passed vector + // results in the passed vector parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); std::cerr << "Number of seed pairs: " << h_seed_pairs.size() << std::endl; @@ -183,8 +111,8 @@ int main(int argc, char* argv[]) pinned_host_vector h_encoded_target(target_sequence.length()); pinned_host_vector h_encoded_query(target_sequence.length()); - encode_string(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); - encode_string(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); + encode_sequence(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); + encode_sequence(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); // Create a stream for async use CudaStream stream0 = make_cuda_stream(); // Create an allocator for use with both APIs diff --git a/cudaextender/tests/CMakeLists.txt b/cudaextender/tests/CMakeLists.txt new file mode 100644 index 000000000..fb1a33dc3 --- /dev/null +++ b/cudaextender/tests/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# Copyright 2020 NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +set(TARGET_NAME cudaextendertests) + +set(SOURCES + main.cpp + Test_CudaextenderEnd2End.cu) + +get_property(cudextender_data_include_dir GLOBAL PROPERTY cudaextender_data_include_dir) +include_directories(${cudextender_data_include_dir}) + +set(LIBS cudaextender) + +gw_add_tests(${TARGET_NAME} "${SOURCES}" "${LIBS}") diff --git a/cudaextender/tests/Test_CudaextenderEnd2End.cu b/cudaextender/tests/Test_CudaextenderEnd2End.cu new file mode 100644 index 000000000..d3fe5aefc --- /dev/null +++ b/cudaextender/tests/Test_CudaextenderEnd2End.cu @@ -0,0 +1,126 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#include "file_location.hpp" + +#include "gtest/gtest.h" +#include +#include +#include + +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudaextender +{ + +typedef struct +{ + std::string query_file; + std::string target_file; + std::string seed_pairs_file; + std::string golden_scored_segment_pairs_file; + int32_t score_threshold; +} End2EndTestParam; + +std::vector getCudaextenderEnd2EndTestCases() +{ + std::vector test_cases; + + End2EndTestParam test1{}; + test1.query_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + test1.target_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + test1.seed_pairs_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample_seed_pairs.csv"; + test1.golden_scored_segment_pairs_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample_scored_segment_pairs.csv"; + test1.score_threshold = 3000; + test_cases.push_back(test1); + + return test_cases; +} + +// Testing correctness of UngappedXDrop extension End2End with default scoring matrix +class TestCudaextenderEnd2End : public ::testing::TestWithParam +{ +public: + void SetUp() + { + param_ = GetParam(); + // Define Scoring Matrix + const int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, + -114, 100, -125, -31, -1000, -1000, -100, -9100, + -31, -125, 100, -114, -1000, -1000, -100, -9100, + -123, -31, -114, 91, -1000, -1000, -100, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -100, -100, -100, -100, -1000, -1000, -100, -9100, + -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; + const int32_t xdrop_threshold = 910; + const int32_t device_id = 0; + const bool no_entropy = false; + const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); + allocator_ = create_default_device_allocator(max_gpu_memory); + stream_ = make_cuda_stream(); + ungapped_extender_ = create_extender(score_matrix, NUC2, xdrop_threshold, no_entropy, stream_.get(), device_id, allocator_); + } + + void TearDown() + { + ungapped_extender_.reset(); + } + + void run_test() + { + std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(param_.target_file, 0, false); + const std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; + std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(param_.query_file, 0, false); + const std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; + std::vector h_seed_pairs; + parse_seed_pairs(param_.seed_pairs_file, h_seed_pairs); + // Allocate pinned memory for query and target strings + pinned_host_vector h_encoded_target(target_sequence.length()); + pinned_host_vector h_encoded_query(target_sequence.length()); + encode_sequence(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); + encode_sequence(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); + ungapped_extender_->extend_async(h_encoded_query.data(), h_encoded_query.size(), h_encoded_target.data(), h_encoded_target.size(), param_.score_threshold, h_seed_pairs); + // Parse golden scored_segment_pairs while extension is in progress + std::vector golden_scored_segment_pairs; + parse_scored_segment_pairs(param_.golden_scored_segment_pairs_file, golden_scored_segment_pairs); + ungapped_extender_->sync(); + std::vector scored_segment_pairs = ungapped_extender_->get_scored_segment_pairs(); + ASSERT_EQ(golden_scored_segment_pairs, scored_segment_pairs); + } + +private: + std::unique_ptr ungapped_extender_; + End2EndTestParam param_; + CudaStream stream_; + DefaultDeviceAllocator allocator_; +}; + +TEST_P(TestCudaextenderEnd2End, TestCorrectness) +{ + run_test(); +} + +INSTANTIATE_TEST_SUITE_P(TestEnd2End, TestCudaextenderEnd2End, ::testing::ValuesIn(getCudaextenderEnd2EndTestCases())); + +} // namespace cudaextender + +} // namespace genomeworks + +} // namespace claraparabricks diff --git a/cudaextender/tests/main.cpp b/cudaextender/tests/main.cpp new file mode 100644 index 000000000..f8017e551 --- /dev/null +++ b/cudaextender/tests/main.cpp @@ -0,0 +1,25 @@ +/* +* Copyright 2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "gtest/gtest.h" + +// ----------------------------------------------------------------------------- +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} From 9c61b8f6b3bc62f98a8a00afe6160a964ac870db Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 12:22:19 -0700 Subject: [PATCH 134/281] [cudaextender] Fix test ordering --- .../genomeworks/cudaextender/utils.hpp | 11 ++++++----- cudaextender/tests/Test_CudaextenderEnd2End.cu | 9 ++++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp index 56d6e714c..2b67de7fa 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include @@ -69,9 +70,9 @@ inline void parse_seed_pairs(const std::string& filepath, std::vector& } /// \brief Parses scored segment pairs from a csv file in the following format: -/// target_position_in_read_1, query_position_in_read_1, score_1, length_1 -/// target_position_in_read_2, query_position_in_read_2, score_2, length_2 -/// target_position_in_read_n, query_position_in_read_n, score_n, length_n +/// target_position_in_read_1, query_position_in_read_1, length_1, score_1 +/// target_position_in_read_2, query_position_in_read_2, length_2, score_2 +/// target_position_in_read_n, query_position_in_read_n, length_n, score_n /// /// \param[out] scored_segment_pairs Reference to vector into which parsed scored segment pairs are saved /// \param[in] filepath Reference to the string containing the path of the scored segment pairs csv @@ -90,9 +91,9 @@ inline void parse_scored_segment_pairs(const std::string& filepath, std::vector< std::getline(scored_segment_pairs_file, line, ','); scored_segment_pair.seed_pair.query_position_in_read = std::atoi(line.c_str()); std::getline(scored_segment_pairs_file, line, ','); - scored_segment_pair.score = std::atoi(line.c_str()); - std::getline(scored_segment_pairs_file, line); scored_segment_pair.length = std::atoi(line.c_str()); + std::getline(scored_segment_pairs_file, line); + scored_segment_pair.score = std::atoi(line.c_str()); scored_segment_pairs.push_back(scored_segment_pair); } } diff --git a/cudaextender/tests/Test_CudaextenderEnd2End.cu b/cudaextender/tests/Test_CudaextenderEnd2End.cu index d3fe5aefc..4f76f33c6 100644 --- a/cudaextender/tests/Test_CudaextenderEnd2End.cu +++ b/cudaextender/tests/Test_CudaextenderEnd2End.cu @@ -102,7 +102,14 @@ public: parse_scored_segment_pairs(param_.golden_scored_segment_pairs_file, golden_scored_segment_pairs); ungapped_extender_->sync(); std::vector scored_segment_pairs = ungapped_extender_->get_scored_segment_pairs(); - ASSERT_EQ(golden_scored_segment_pairs, scored_segment_pairs); + ASSERT_EQ(golden_scored_segment_pairs.size(), scored_segment_pairs.size()); + for(int32_t i=0; i Date: Wed, 23 Sep 2020 12:32:22 -0700 Subject: [PATCH 135/281] [cudaextender] Add equality operator --- .../genomeworks/cudaextender/extender.hpp | 11 ++++++----- cudaextender/tests/Test_CudaextenderEnd2End.cu | 9 +-------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index c279daed6..bef943608 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -47,13 +47,14 @@ struct ScoredSegmentPair int32_t length; /// score of the segment int32_t score; - /// equality operator - __host__ __device__ bool operator==(const ScoredSegmentPair& other) const - { - return ((seed_pair.target_position_in_read == other.seed_pair.target_position_in_read) && (seed_pair.query_position_in_read == other.seed_pair.query_position_in_read) && (length == other.length) && (score == other.score)); - } }; +/// Equality operator for ScoredSegmentPair +__host__ __device__ inline bool operator==(const ScoredSegmentPair& x, const ScoredSegmentPair& y) +{ + return ((x.seed_pair.target_position_in_read == y.seed_pair.target_position_in_read) && (x.seed_pair.query_position_in_read == y.seed_pair.query_position_in_read) && (x.length == y.length) && (x.score == y.score)); +} + /// CUDA Extension object class Extender { diff --git a/cudaextender/tests/Test_CudaextenderEnd2End.cu b/cudaextender/tests/Test_CudaextenderEnd2End.cu index 4f76f33c6..d3fe5aefc 100644 --- a/cudaextender/tests/Test_CudaextenderEnd2End.cu +++ b/cudaextender/tests/Test_CudaextenderEnd2End.cu @@ -102,14 +102,7 @@ public: parse_scored_segment_pairs(param_.golden_scored_segment_pairs_file, golden_scored_segment_pairs); ungapped_extender_->sync(); std::vector scored_segment_pairs = ungapped_extender_->get_scored_segment_pairs(); - ASSERT_EQ(golden_scored_segment_pairs.size(), scored_segment_pairs.size()); - for(int32_t i=0; i Date: Wed, 23 Sep 2020 12:34:17 -0700 Subject: [PATCH 136/281] [cudaextender] Make format --- .../genomeworks/cudaextender/utils.hpp | 2 +- cudaextender/tests/CMakeLists.txt | 4 +-- .../tests/Test_CudaextenderEnd2End.cu | 26 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp index 2b67de7fa..7355d7a8c 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp @@ -143,7 +143,7 @@ inline void encode_sequence(char* dst_seq, const char* src_seq, const int32_t& l } } -} // namespace cudapoa +} // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/tests/CMakeLists.txt b/cudaextender/tests/CMakeLists.txt index fb1a33dc3..7c774c8cc 100644 --- a/cudaextender/tests/CMakeLists.txt +++ b/cudaextender/tests/CMakeLists.txt @@ -18,8 +18,8 @@ set(TARGET_NAME cudaextendertests) set(SOURCES - main.cpp - Test_CudaextenderEnd2End.cu) + main.cpp + Test_CudaextenderEnd2End.cu) get_property(cudextender_data_include_dir GLOBAL PROPERTY cudaextender_data_include_dir) include_directories(${cudextender_data_include_dir}) diff --git a/cudaextender/tests/Test_CudaextenderEnd2End.cu b/cudaextender/tests/Test_CudaextenderEnd2End.cu index d3fe5aefc..7c6f01dd7 100644 --- a/cudaextender/tests/Test_CudaextenderEnd2End.cu +++ b/cudaextender/tests/Test_CudaextenderEnd2End.cu @@ -43,11 +43,11 @@ std::vector getCudaextenderEnd2EndTestCases() std::vector test_cases; End2EndTestParam test1{}; - test1.query_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; - test1.target_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; - test1.seed_pairs_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample_seed_pairs.csv"; + test1.query_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + test1.target_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample.fa"; + test1.seed_pairs_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample_seed_pairs.csv"; test1.golden_scored_segment_pairs_file = std::string(CUDAEXTENDER_DATA_DIR) + "/sample_scored_segment_pairs.csv"; - test1.score_threshold = 3000; + test1.score_threshold = 3000; test_cases.push_back(test1); return test_cases; @@ -59,7 +59,7 @@ class TestCudaextenderEnd2End : public ::testing::TestWithParam fasta_parser_target = io::create_kseq_fasta_parser(param_.target_file, 0, false); - const std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; - std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(param_.query_file, 0, false); - const std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; + const std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; + std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(param_.query_file, 0, false); + const std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; std::vector h_seed_pairs; parse_seed_pairs(param_.seed_pairs_file, h_seed_pairs); // Allocate pinned memory for query and target strings From 3596286fccb5cda82c55105b0ee3f72066c1f2b6 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 13:10:38 -0700 Subject: [PATCH 137/281] [cudaextender] Resolve file location conflict with cudapoa --- cudaextender/data/CMakeLists.txt | 2 +- ...file_location.hpp.in => cudaextender_file_location.hpp.in} | 0 cudaextender/tests/Test_CudaextenderEnd2End.cu | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename cudaextender/data/{file_location.hpp.in => cudaextender_file_location.hpp.in} (100%) diff --git a/cudaextender/data/CMakeLists.txt b/cudaextender/data/CMakeLists.txt index e5c4355dc..11ce93724 100644 --- a/cudaextender/data/CMakeLists.txt +++ b/cudaextender/data/CMakeLists.txt @@ -14,6 +14,6 @@ # limitations under the License. # set(CUDAEXTENDER_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../data") -configure_file(file_location.hpp.in ${PROJECT_BINARY_DIR}/data/file_location.hpp @ONLY) +configure_file(cudaextender_file_location.hpp.in ${PROJECT_BINARY_DIR}/data/cudaextender_file_location.hpp @ONLY) set_property(GLOBAL PROPERTY cudaextender_data_include_dir "${PROJECT_BINARY_DIR}/data") diff --git a/cudaextender/data/file_location.hpp.in b/cudaextender/data/cudaextender_file_location.hpp.in similarity index 100% rename from cudaextender/data/file_location.hpp.in rename to cudaextender/data/cudaextender_file_location.hpp.in diff --git a/cudaextender/tests/Test_CudaextenderEnd2End.cu b/cudaextender/tests/Test_CudaextenderEnd2End.cu index 7c6f01dd7..b1fdc07d6 100644 --- a/cudaextender/tests/Test_CudaextenderEnd2End.cu +++ b/cudaextender/tests/Test_CudaextenderEnd2End.cu @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "file_location.hpp" - #include "gtest/gtest.h" + +#include #include #include #include From 32f276180f142740fd1c5b70a9dc6a86a9272306 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 13:31:53 -0700 Subject: [PATCH 138/281] [cudaextender] Try removing cub includes --- cudaextender/src/ungapped_xdrop.cu | 3 --- 1 file changed, 3 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index de0549d55..fc12939fa 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -22,9 +22,6 @@ #include #include -#include -#include - // Temporary timing code #include From b49f26601cb458df3b5df6c21a0bf6d7b5fde7ea Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 13:41:14 -0700 Subject: [PATCH 139/281] [cudaextender] Replace cub include --- cudaextender/src/ungapped_xdrop.cu | 1 + 1 file changed, 1 insertion(+) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index fc12939fa..ff3adfb58 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -21,6 +21,7 @@ #include #include +#include // Temporary timing code #include From 57e4c1990b8653cd9fc9f1420ea9afebdb77209c Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 14:00:02 -0700 Subject: [PATCH 140/281] [cudaextender] Add cub as a linked lib --- cudaextender/CMakeLists.txt | 2 +- cudaextender/src/ungapped_xdrop.cu | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cudaextender/CMakeLists.txt b/cudaextender/CMakeLists.txt index d4ab14443..82d282374 100644 --- a/cudaextender/CMakeLists.txt +++ b/cudaextender/CMakeLists.txt @@ -45,7 +45,7 @@ cuda_add_library(${MODULE_NAME} ${gw_library_type} src/ungapped_xdrop_kernels.cu ${CMAKE_CURRENT_BINARY_DIR}/version.cpp) -target_link_libraries(${MODULE_NAME} gwbase gwio) +target_link_libraries(${MODULE_NAME} gwbase gwio cub) target_compile_options(${MODULE_NAME} PRIVATE -Werror -Wall -Wextra) if (gw_optimize_for_native_cpu) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index ff3adfb58..de0549d55 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -21,7 +21,9 @@ #include #include -#include + +#include +#include // Temporary timing code #include From 641f6cdb154204f53bced3ef8b42ad8b0a1e0809 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 14:36:56 -0700 Subject: [PATCH 141/281] [cudaextender] Add cudaextender file location --- cudaextender/samples/sample_cudaextender.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index 18b8d0e79..a8160fb8c 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include +#include #include #include #include From 1939264d8c681b5b31d356edb1523f90794f382b Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 15:29:55 -0700 Subject: [PATCH 142/281] [cudaextender] Update external docs --- .../genomeworks/cudaextender/cudaextender.hpp | 2 +- .../genomeworks/cudaextender/extender.hpp | 68 +++++++++++++++---- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp index 1c1161bce..f0026e920 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp @@ -25,7 +25,7 @@ namespace genomeworks namespace cudaextender { /// \defgroup cudaextender CUDA Extender package -/// Base docs for the cudaextender package (tbd) +/// Base docs for the cudaextender package /// \{ /// CUDA Extender status type diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index bef943608..08eb33299 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -29,6 +29,9 @@ namespace genomeworks namespace cudaextender { +/// \addtogroup cudaextender +/// \{ + /// Seed positions in target and query reads struct SeedPair { @@ -55,30 +58,53 @@ __host__ __device__ inline bool operator==(const ScoredSegmentPair& x, const Sco return ((x.seed_pair.target_position_in_read == y.seed_pair.target_position_in_read) && (x.seed_pair.query_position_in_read == y.seed_pair.query_position_in_read) && (x.length == y.length) && (x.score == y.score)); } -/// CUDA Extension object +/// CUDA Extender Class class Extender { public: /// \brief Virtual destructor for Extender. virtual ~Extender() = default; - /// \brief Host pointer prototype for extension + /// \brief Host pointer API for extension /// - /// Takes values from host data structures, - /// copies them over to device, + /// Takes values from host data structures, copies them over to device, /// launches async extension kernels on specified stream. Filters /// segments on device based on score_threshold + /// + /// \param[in] h_query Host pointer to encoded query sequence + /// \param[in] query_length Length of query sequence + /// \param[in] h_target Host pointer to encoded target sequence + /// \param[in] target_length Length of target sequence + /// \param[in] score_threshold Score threshold for filtering extended segments + /// \param[in] h_seed_pairs Vector of seed pairs mapping to query and target + /// sequences + /// \return Status of the async extension launch virtual StatusType extend_async(const char* h_query, const int32_t& query_length, const char* h_target, const int32_t& target_length, const int32_t& score_threshold, const std::vector& h_seed_pairs) = 0; - /// \brief Device pointer prototype for extension + /// \brief Device pointer API for extension + /// + /// All inputs to this function are expected to be resident on the device + /// before launch. Output array d_scored_segment_pairs must be pre-allocated on device. + /// Launches async extension kernel. Filters segments on device based on score_threshold. + /// The stream associated with this function call must be synchronized to before accessing + /// any output parameters. /// - /// Memcopies to device memory are assumed to be done before this - /// function. Output array d_scored_segment_pairs must be pre-allocated on device. - /// Launches async extension kernel. Filters segments on device - /// based on score_threshold. + /// \param[in] d_query Device pointer to encoded query sequence + /// \param[in] query_length Length of query sequence + /// \param[in] d_target Device pointer to encoded target sequence + /// \param[in] target_length Length of target sequence + /// \param[in] score_threshold Score threshold for filtering extended segments + /// \param[in] d_seed_pairs Device pointer to array of seed pairs mapping between + /// target and query sequences + /// \param[in] num_seed_pairs Length of d_seed_pairs array + /// \param[out] d_scored_segment_pairs Pointer to a pre-allocated device location for + /// storing extension output + /// \param[out] d_num_scored_segment_pairs Pointer to pre-allocated device location for + /// storing length of extension output + /// \return Status of the async extension launch virtual StatusType extend_async(const char* d_query, int32_t query_length, const char* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, @@ -87,21 +113,39 @@ class Extender /// \brief Waits for CUDA accelerated extension to finish /// + /// To be used with the host pointer extend_async API. /// Blocking call that waits for all the extensions scheduled /// on the GPU to come to completion. + /// + /// \return Synchronization and memory copy status virtual StatusType sync() = 0; - /// \brief Return the computed segment pairs + /// \brief Returns the computed scored segment pairs + /// + /// To be used with the host pointer extend_async API. sync() must + /// be called after launching async extension and before calling this function + /// or the returned results will not be valid. /// - /// \return Vector of Scored Segment Pairs + /// \return Vector of Scored Segment Pairs that have a score >= score_threshold virtual const std::vector& get_scored_segment_pairs() const = 0; /// \brief Reset Extender object and free device/host memory virtual void reset() = 0; }; +/// \brief Create Extender object +/// +/// \param h_sub_mat Host pointer to square scoring/substitution matrix for use during extension +/// \param sub_mat_dim Length of the square matrix +/// \param xdrop_threshold Threshold for performing X-Drop +/// \param no_entropy Flag indicating whether to use entropy during extension +/// \param stream CUDA Stream to be used with extension +/// \param device_id GPU to be used for extension +/// \param allocator DeviceAllocator to be used for allocating/freeing memory +/// \param type Type of extension to be performed +/// \return Unique pointer to Extender object. std::unique_ptr create_extender(const int32_t* h_sub_mat, const int32_t sub_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator, const ExtensionType type = ExtensionType::ungapped_xdrop); - +/// \} } // namespace cudaextender } // namespace genomeworks } // namespace claraparabricks From faceee52e30a0393ddcdd4e039b3b30f67dfbfb9 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 15:49:32 -0700 Subject: [PATCH 143/281] [cudaextender] Additional error types --- .../genomeworks/cudaextender/cudaextender.hpp | 5 ++- cudaextender/src/ungapped_xdrop.cu | 38 ++++++++++--------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp index f0026e920..0642007d0 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/cudaextender.hpp @@ -31,8 +31,9 @@ namespace cudaextender /// CUDA Extender status type enum StatusType { - success = 0, - error_invalid_operation = 1, + success = 0, + invalid_operation = 1, + invalid_input = 2, generic_error }; diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index de0549d55..402653086 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -25,9 +25,6 @@ #include #include -// Temporary timing code -#include - namespace claraparabricks { @@ -49,9 +46,12 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_sub_mat, const int32_t sub_mat_dim , host_ptr_api_mode_(false) , allocator_(allocator) { - //TODO - Check bounds - // Calculate the max limits on the number of extensions we can do on - // this GPU + if(h_sub_mat_ == nullptr) + { + throw std::runtime_error("Substitution matrix cannot be null"); + } + // TODO - check sub_mat_dim based on Sequence Encoder API + // Calculate the max limits on the number of extensions we can do on this GPU cudaDeviceProp device_prop; cudaGetDeviceProperties(&device_prop, device_id_); const int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures @@ -84,11 +84,17 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) { - //TODO - Check bounds - - auto t1 = std::chrono::high_resolution_clock::now(); + if(d_query == nullptr || d_target == nullptr || d_seed_pairs == nullptr) + { + GW_LOG_ERROR("Invalid input pointers"); + return StatusType::invalid_input; + } + if(d_scored_segment_pairs == nullptr || d_num_scored_segment_pairs == nullptr) + { + GW_LOG_ERROR("Invalid output pointers"); + return StatusType::invalid_input; + } // Switch to configured GPU - // If host pointer API mode was used before this mode, reset data structures scoped_device_switch dev(device_id_); total_scored_segment_pairs_ = 0; for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += batch_max_ungapped_extensions_) @@ -136,14 +142,10 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length total_scored_segment_pairs_ += get_value_from_device(d_num_scored_segment_pairs, stream_); } } - auto t2 = std::chrono::high_resolution_clock::now(); - - auto duration = std::chrono::duration_cast(t2 - t1).count(); - - std::cout << "Time: " << duration << std::endl; set_device_value_async(d_num_scored_segment_pairs, &total_scored_segment_pairs_, stream_); - return success; + + return StatusType::success; } StatusType UngappedXDrop::extend_async(const char* h_query, const int32_t& query_length, @@ -188,11 +190,11 @@ StatusType UngappedXDrop::sync() device_copy_n(d_ssp_.data(), h_num_ssp, h_ssp_.data(), stream_); cudaStreamSynchronize(stream_); } - return success; + return StatusType::success; } // If this function was called without using the host_ptr_api, throw error - return error_invalid_operation; + return StatusType::invalid_operation; } const std::vector& UngappedXDrop::get_scored_segment_pairs() const From af7aa0630f24ef2d0e0d7c1067fa8eedd5e45848 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 15:56:09 -0700 Subject: [PATCH 144/281] [cudaextender] Make format --- cudaextender/src/ungapped_xdrop.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 402653086..22d5598a4 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -46,7 +46,7 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_sub_mat, const int32_t sub_mat_dim , host_ptr_api_mode_(false) , allocator_(allocator) { - if(h_sub_mat_ == nullptr) + if (h_sub_mat_ == nullptr) { throw std::runtime_error("Substitution matrix cannot be null"); } @@ -84,12 +84,12 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) { - if(d_query == nullptr || d_target == nullptr || d_seed_pairs == nullptr) + if (d_query == nullptr || d_target == nullptr || d_seed_pairs == nullptr) { GW_LOG_ERROR("Invalid input pointers"); return StatusType::invalid_input; } - if(d_scored_segment_pairs == nullptr || d_num_scored_segment_pairs == nullptr) + if (d_scored_segment_pairs == nullptr || d_num_scored_segment_pairs == nullptr) { GW_LOG_ERROR("Invalid output pointers"); return StatusType::invalid_input; From 59d0846cc36cd8a9456347794312d4960c12ed89 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 16:06:49 -0700 Subject: [PATCH 145/281] [cudaextender] Refactor and newlines --- cudaextender/samples/sample_cudaextender.cpp | 28 +++++++++++++------- cudaextender/src/cudaextender.cpp | 2 +- cudaextender/src/ungapped_xdrop.cu | 2 +- cudaextender/src/ungapped_xdrop_kernels.cu | 2 +- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index a8160fb8c..08a6518ac 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -95,7 +95,7 @@ int main(int argc, char* argv[]) // Following function loops through all seed_pairs in the sample_seed_pairs.csv and returns // results in the passed vector parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); - std::cerr << "Number of seed pairs: " << h_seed_pairs.size() << std::endl; + std::cerr << "Number of Seed Pairs: " << h_seed_pairs.size() << std::endl; // Define Scoring Matrix const int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, @@ -120,10 +120,22 @@ int main(int argc, char* argv[]) DefaultDeviceAllocator allocator = create_default_device_allocator(max_gpu_memory); // Reference for output std::vector h_ssp; + // Create Ungapped Extender Object for both API modes + std::unique_ptr ungapped_extender = create_extender(score_matrix, + NUC2, + xdrop_threshold, + input_no_entropy, + stream0.get(), + 0, + allocator); if (!device_ptr_api_mode) { - std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0, allocator); - ungapped_extender->extend_async(h_encoded_query.data(), h_encoded_query.size(), h_encoded_target.data(), h_encoded_target.size(), score_threshold, h_seed_pairs); + ungapped_extender->extend_async(h_encoded_query.data(), + h_encoded_query.size(), + h_encoded_target.data(), + h_encoded_target.size(), + score_threshold, + h_seed_pairs); ungapped_extender->sync(); h_ssp = ungapped_extender->get_scored_segment_pairs(); } @@ -145,11 +157,8 @@ int main(int argc, char* argv[]) device_copy_n(h_encoded_target.data(), target_sequence.length(), d_target.data(), stream0.get()); device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs.data(), stream0.get()); - // Create an ungapped extender object - std::unique_ptr ungapped_extender = create_extender(score_matrix, NUC2, xdrop_threshold, input_no_entropy, stream0.get(), 0, allocator); - - // Launch the ungapped extender device function - ungapped_extender->extend_async(d_query.data(), // Type TBD based on encoding + // Launch the ungapped extender device pointer function + ungapped_extender->extend_async(d_query.data(), d_query.size(), d_target.data(), d_target.size(), @@ -166,9 +175,8 @@ int main(int argc, char* argv[]) // Copy data synchronously device_copy_n(d_ssp.data(), h_num_ssp, h_ssp.data()); } - std::cerr << "Number of ScoredSegmentPairs found: " << h_ssp.size() << std::endl; + std::cerr << "Number of Scored Segment Pairs found: " << h_ssp.size() << std::endl; if (print) print_scored_segment_pairs(h_ssp); - return 0; } diff --git a/cudaextender/src/cudaextender.cpp b/cudaextender/src/cudaextender.cpp index 9a8e349b5..c345a4007 100644 --- a/cudaextender/src/cudaextender.cpp +++ b/cudaextender/src/cudaextender.cpp @@ -36,4 +36,4 @@ StatusType Init() } // namespace genomeworks -} // namespace claraparabricks \ No newline at end of file +} // namespace claraparabricks diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 22d5598a4..cb13d2f9a 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -226,4 +226,4 @@ UngappedXDrop::~UngappedXDrop() } // namespace genomeworks -} // namespace claraparabricks \ No newline at end of file +} // namespace claraparabricks diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 5782131e7..003f94a2d 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -460,4 +460,4 @@ __global__ void compress_output(const int32_t* d_done, const int32_t start_index } // namespace genomeworks -} // namespace claraparabricks \ No newline at end of file +} // namespace claraparabricks From e5828afd786eaa25a0153f46b0aefd6340250441 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 16:14:46 -0700 Subject: [PATCH 146/281] [cudaextender] Fix comments --- cudaextender/src/ungapped_xdrop_kernels.cu | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 003f94a2d..919ca7a44 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -24,9 +24,9 @@ namespace genomeworks namespace cudaextender { -// extend the hits to a segment by ungapped x-drop method, adjust low-scoring +// extend the seed values to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores -// to score_threshold and update the d_hsp and d_done vectors +// to score_threshold and update the d_scored_segment_pairs and d_done vectors __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, @@ -429,8 +429,7 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } } -// gather only the HSPs from the resulting segments to the beginning of the -// tmp_hsp vector +// Gather the SSPs from the resulting segments to the beginning of the tmp_ssp array __global__ void compress_output(const int32_t* d_done, const int32_t start_index, const ScoredSegmentPair* d_ssp, ScoredSegmentPair* d_tmp_ssp, const int32_t num_seed_pairs) { const int32_t stride = blockDim.x * gridDim.x; From 56586ca2942da47a65ae282d0ab08e54b0b40cd3 Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 16:17:05 -0700 Subject: [PATCH 147/281] [cudaextender] Add error check to stream synchronize --- cudaextender/src/ungapped_xdrop.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index cb13d2f9a..ad44833e0 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -188,7 +188,7 @@ StatusType UngappedXDrop::sync() { h_ssp_.resize(h_num_ssp); device_copy_n(d_ssp_.data(), h_num_ssp, h_ssp_.data(), stream_); - cudaStreamSynchronize(stream_); + GW_CU_CHECK_ERR(cudaStreamSynchronize(stream_)); } return StatusType::success; } From c120c47fb2cbf21046f08ae54ae17540ecc06b1b Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 23 Sep 2020 16:34:50 -0700 Subject: [PATCH 148/281] [cudaextender] Rename sub_mat as score_mat --- .../genomeworks/cudaextender/extender.hpp | 6 +++--- cudaextender/src/extender.cpp | 4 ++-- cudaextender/src/ungapped_xdrop.cu | 14 +++++++------- cudaextender/src/ungapped_xdrop.cuh | 8 ++++---- cudaextender/src/ungapped_xdrop_kernels.cuh | 8 ++++---- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index 08eb33299..452b7bcfe 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -135,8 +135,8 @@ class Extender /// \brief Create Extender object /// -/// \param h_sub_mat Host pointer to square scoring/substitution matrix for use during extension -/// \param sub_mat_dim Length of the square matrix +/// \param h_score_mat Host pointer to scoring matrix for use during extension +/// \param score_mat_dim Dimension of the scoring matrix /// \param xdrop_threshold Threshold for performing X-Drop /// \param no_entropy Flag indicating whether to use entropy during extension /// \param stream CUDA Stream to be used with extension @@ -144,7 +144,7 @@ class Extender /// \param allocator DeviceAllocator to be used for allocating/freeing memory /// \param type Type of extension to be performed /// \return Unique pointer to Extender object. -std::unique_ptr create_extender(const int32_t* h_sub_mat, const int32_t sub_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator, const ExtensionType type = ExtensionType::ungapped_xdrop); +std::unique_ptr create_extender(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator, const ExtensionType type = ExtensionType::ungapped_xdrop); /// \} } // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index a3dd8a96e..040ed75ed 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -27,11 +27,11 @@ namespace cudaextender { // TODO - Provide an allocator-less API for creation? -std::unique_ptr create_extender(const int32_t* h_sub_mat, const int32_t sub_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator, const ExtensionType type) +std::unique_ptr create_extender(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator, const ExtensionType type) { if (type == ExtensionType::ungapped_xdrop) { - return std::make_unique(h_sub_mat, sub_mat_dim, xdrop_threshold, no_entropy, stream, device_id, allocator); + return std::make_unique(h_score_mat, score_mat_dim, xdrop_threshold, no_entropy, stream, device_id, allocator); } else { diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index ad44833e0..360c881eb 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -36,9 +36,9 @@ namespace cudaextender using namespace cudautils; -UngappedXDrop::UngappedXDrop(const int32_t* h_sub_mat, const int32_t sub_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator) - : h_sub_mat_(h_sub_mat) - , sub_mat_dim_(sub_mat_dim) +UngappedXDrop::UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator) + : h_score_mat_(h_score_mat) + , score_mat_dim_(score_mat_dim) , xdrop_threshold_(xdrop_threshold) , no_entropy_(no_entropy) , stream_(stream) @@ -46,7 +46,7 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_sub_mat, const int32_t sub_mat_dim , host_ptr_api_mode_(false) , allocator_(allocator) { - if (h_sub_mat_ == nullptr) + if (h_score_mat_ == nullptr) { throw std::runtime_error("Substitution matrix cannot be null"); } @@ -69,13 +69,13 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_sub_mat, const int32_t sub_mat_dim cub_storage_bytes = std::max(temp_storage_bytes, cub_storage_bytes); // Allocate space on device for scoring matrix and intermediate results - d_sub_mat_ = device_buffer(sub_mat_dim_, allocator_, stream_); + d_score_mat_ = device_buffer(score_mat_dim_, allocator_, stream_); d_done_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); d_tmp_ssp_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); d_temp_storage_cub_ = device_buffer(cub_storage_bytes, allocator_, stream_); // Requires pinned host memory registration for proper async behavior - device_copy_n(h_sub_mat_, sub_mat_dim_, d_sub_mat_.data(), stream_); + device_copy_n(h_score_mat_, score_mat_dim_, d_score_mat_.data(), stream_); } StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, @@ -108,7 +108,7 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length target_length, d_query, query_length, - d_sub_mat_.data(), + d_score_mat_.data(), no_entropy_, xdrop_threshold_, score_threshold, diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 5c9c215b7..b4726f4fc 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -31,7 +31,7 @@ namespace cudaextender class UngappedXDrop : public Extender { public: - UngappedXDrop(const int32_t* h_sub_mat, const int32_t sub_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator); + UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator); ~UngappedXDrop() override; StatusType extend_async(const char* h_query, const int32_t& query_length, @@ -54,15 +54,15 @@ public: private: DefaultDeviceAllocator allocator_; // Device ptr API required variables - const int32_t* h_sub_mat_; - const int32_t sub_mat_dim_; // Assume matrix is square + const int32_t* h_score_mat_; + const int32_t score_mat_dim_; // Assume matrix is square const int32_t xdrop_threshold_; const bool no_entropy_; cudaStream_t stream_; const int32_t device_id_; std::vector scored_segment_pairs_; int32_t batch_max_ungapped_extensions_; // TODO - Make const - device_buffer d_sub_mat_; // Pointer to device substitution matrix + device_buffer d_score_mat_; // Pointer to device substitution matrix device_buffer d_done_; // TODO- Rename scratch space device_buffer d_tmp_ssp_; // TODO- Rename Scratch space 2 int32_t total_scored_segment_pairs_; diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index f03d06828..2ad9464e5 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -30,7 +30,7 @@ namespace cudaextender // extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores -// to hspthresh and update the d_hsp and d_done vectors +// to score_threshold and update the d_scored_segment_pairs and d_done vectors __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, const int32_t target_length, const char* __restrict__ d_query, @@ -45,15 +45,15 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_done); -// gather only the HSPs from the resulting segments to the beginning of the -// tmp_hsp vector +// Gathers the SSPs from the resulting segments to the beginning of the +// tmp_ssp vector __global__ void compress_output(const int32_t* d_done, const int32_t start_index, const ScoredSegmentPair* d_ssp, ScoredSegmentPair* d_tmp_ssp, int num_hits); -// Binary Predicate for sorting the ScoredSegmentPairs +// Binary predicate for sorting the ScoredSegmentPairs struct scored_segment_pair_comp { __host__ __device__ bool operator()(const ScoredSegmentPair& x, const ScoredSegmentPair& y) From 744401956636e76c8dcf583a7af4009e83f90fa1 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 23 Sep 2020 21:52:33 -0400 Subject: [PATCH 149/281] [cudapoa-backtracking] moving new banded NW with traceback buffer to a separate file, renaming its kernels. WIP --- cudapoa/src/cudapoa_kernels.cuh | 37 +- cudapoa/src/cudapoa_nw_banded.cuh | 397 +++++++----------- cudapoa/src/cudapoa_nw_tb_banded.cuh | 592 +++++++++++++++++++++++++++ 3 files changed, 760 insertions(+), 266 deletions(-) create mode 100644 cudapoa/src/cudapoa_nw_tb_banded.cuh diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index bc3668b07..93387da81 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -20,6 +20,7 @@ #include "cudapoa_nw.cuh" #include "cudapoa_nw_banded.cuh" #include "cudapoa_nw_adaptive_banded.cuh" +#include "cudapoa_nw_tb_banded.cuh" #include "cudapoa_topsort.cuh" #include "cudapoa_add_alignment.cuh" #include "cudapoa_generate_consensus.cuh" @@ -305,24 +306,24 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) } else if (BM == BandMode::static_band) { - alignment_length = runNeedlemanWunschBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - backtrace, - alignment_graph, - alignment_read, - static_band_width, - max_pred_distance, - gap_score, - mismatch_score, - match_score); + alignment_length = runNeedlemanWunschBandedTraceBack(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + backtrace, + alignment_graph, + alignment_read, + static_band_width, + max_pred_distance, + gap_score, + mismatch_score, + match_score); __syncwarp(); } else // BM == BandMode::full_band diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 0b003e650..adbc5b295 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -44,7 +44,7 @@ __device__ __forceinline__ int32_t get_band_start_for_row(int32_t row_idx, float if (end_pos > max_column) { start_pos = max_column - band_width + CELLS_PER_THREAD; - } + }; start_pos = max(start_pos, 0); @@ -54,25 +54,17 @@ __device__ __forceinline__ int32_t get_band_start_for_row(int32_t row_idx, float } template -__device__ __forceinline__ ScoreT* get_score_ptr(ScoreT* scores, int32_t score_row, int32_t column, int32_t band_start, int32_t band_width) +__device__ __forceinline__ ScoreT* get_score_ptr(ScoreT* scores, int32_t row, int32_t column, int32_t band_start, int32_t band_width) { column = column == -1 ? 0 : column - band_start; - int64_t score_index = static_cast(column) + - static_cast(score_row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + int64_t score_index = static_cast(column) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); return &scores[score_index]; -} +}; template -__device__ __forceinline__ void set_score(ScoreT* scores, - int32_t row, - int32_t column, - int32_t score_matrix_height, - int32_t value, - float gradient, - int32_t band_width, - int32_t max_column) +__device__ __forceinline__ void set_score(ScoreT* scores, int32_t row, int32_t column, int32_t value, float gradient, int32_t band_width, int32_t max_column) { int32_t band_start = get_band_start_for_row(row, gradient, band_width, max_column); @@ -86,52 +78,29 @@ __device__ __forceinline__ void set_score(ScoreT* scores, col_idx = column - band_start; } - int64_t score_index = static_cast(col_idx) + - static_cast(row % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + int64_t score_index = static_cast(col_idx) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[score_index] = value; } template -__device__ __forceinline__ void initialize_band(ScoreT* scores, - int32_t row, - int32_t score_matrix_height, - int32_t value, - float gradient, - int32_t band_width, - int32_t max_column, - int32_t lane_idx) +__device__ __forceinline__ void initialize_band(ScoreT* scores, int32_t row, int32_t value, float gradient, int32_t band_width, int32_t max_column, int32_t lane_idx) { int32_t band_start = get_band_start_for_row(row, gradient, band_width, max_column); int32_t band_end = band_start + band_width; int32_t initialization_offset = (band_start == 0) ? 1 : band_start; - set_score(scores, row, initialization_offset, score_matrix_height, value, gradient, band_width, max_column); + set_score(scores, row, initialization_offset, value, gradient, band_width, max_column); // note: as long as CUDAPOA_BANDED_MATRIX_RIGHT_PADDING < WARP_SIZE, no need for a for loop for (int32_t j = lane_idx + band_end; j < band_end + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) { - set_score(scores, row, j, score_matrix_height, value, gradient, band_width, max_column); + set_score(scores, row, j, value, gradient, band_width, max_column); } } -template -__device__ __forceinline__ TraceT get_trace(TraceT* backtrace, int32_t row, int32_t column, int32_t band_start, int32_t band_width) -{ - int64_t trace_index = static_cast(column - band_start) + - static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - return backtrace[trace_index]; -} - template -__device__ __forceinline__ ScoreT get_score(ScoreT* scores, - int32_t row, - int32_t column, - int32_t score_matrix_height, - float gradient, - int32_t band_width, - int32_t max_column, - const ScoreT min_score_value) +__device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t column, float gradient, int32_t band_width, int32_t max_column, const ScoreT min_score_value) { int32_t band_start = get_band_start_for_row(row, gradient, band_width, max_column); int32_t band_end = band_start + band_width; @@ -143,26 +112,20 @@ __device__ __forceinline__ ScoreT get_score(ScoreT* scores, } else { - return *get_score_ptr(scores, row % score_matrix_height, column, band_start, band_width); + return *get_score_ptr(scores, row, column, band_start, band_width); } } -template -__device__ __forceinline__ void get_scores(ScoreT* scores, - int32_t pred_node, - int32_t current_node, - int32_t read_pos, - int32_t score_matrix_height, - float gradient, - int32_t band_width, - int32_t max_column, - int32_t gap_score, - int32_t match_score, - int32_t mismatch_score, - SeqT4 read4, - SeqT graph_base, - ScoreT4& score, - TraceT4& trace) +template +__device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, + int32_t node, + int32_t read_pos, + float gradient, + int32_t band_width, + int32_t max_column, + ScoreT default_value, + int32_t gap_score, + ScoreT4& char_profile) { // The load instructions typically load data in 4B or 8B chunks. @@ -171,20 +134,20 @@ __device__ __forceinline__ void get_scores(ScoreT* scores, // as each read of 16b issues a separate load command. // Instead it is better to load a 4B or 8B chunk into a register // using a single load inst, and then extracting necessary part of - // of the data using bit arithmetic. Also reduces register count. + // of the data using bit arithmatic. Also reduces register count. - int32_t band_start = get_band_start_for_row(pred_node, gradient, band_width, max_column); + int32_t band_start = get_band_start_for_row(node, gradient, band_width, max_column); // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds int32_t band_end = static_cast(band_start + band_width - CELLS_PER_THREAD); if ((read_pos > band_end || read_pos < band_start) && read_pos != -1) { - return; + return ScoreT4{default_value, default_value, default_value, default_value}; } else { - ScoreT4* pred_scores = (ScoreT4*)get_score_ptr(scores, pred_node % score_matrix_height, read_pos, band_start, band_width); + ScoreT4* pred_scores = (ScoreT4*)get_score_ptr(scores, node, read_pos, band_start, band_width); // loads 8/16 consecutive bytes (4 ScoreT) ScoreT4 score4 = pred_scores[0]; @@ -192,87 +155,24 @@ __device__ __forceinline__ void get_scores(ScoreT* scores, // need to load the next chunk of memory as well ScoreT4 score4_next = pred_scores[1]; - ScoreT char_profile = (graph_base == read4.r0 ? match_score : mismatch_score); + ScoreT4 score; - // if trace is diogonal, its value is positive and if vertical, negative - // update score.s0, trace.t0 ---------- - if ((score4.s0 + char_profile) >= (score4.s1 + gap_score)) - { - if ((score4.s0 + char_profile) > score.s0) - { - score.s0 = score4.s0 + char_profile; - trace.t0 = current_node - pred_node; - } - } - else - { - if ((score4.s1 + gap_score) > score.s0) - { - score.s0 = score4.s1 + gap_score; - trace.t0 = -(current_node - pred_node); - } - } - // update score.s1, trace.t1 ---------- - char_profile = (graph_base == read4.r1 ? match_score : mismatch_score); - if ((score4.s1 + char_profile) >= (score4.s2 + gap_score)) - { - if ((score4.s1 + char_profile) > score.s1) - { - score.s1 = score4.s1 + char_profile; - trace.t1 = current_node - pred_node; - } - } - else - { - if ((score4.s2 + gap_score) > score.s1) - { - score.s1 = score4.s2 + gap_score; - trace.t1 = -(current_node - pred_node); - } - } - // update score.s2, trace.t2 ---------- - char_profile = (graph_base == read4.r2 ? match_score : mismatch_score); - if ((score4.s2 + char_profile) >= (score4.s3 + gap_score)) - { - if ((score4.s2 + char_profile) > score.s2) - { - score.s2 = score4.s2 + char_profile; - trace.t2 = current_node - pred_node; - } - } - else - { - if ((score4.s3 + gap_score) > score.s2) - { - score.s2 = score4.s3 + gap_score; - trace.t2 = -(current_node - pred_node); - } - } - // update score.s3, trace.t3 ---------- - char_profile = (graph_base == read4.r3 ? match_score : mismatch_score); - if ((score4.s3 + char_profile) >= (score4_next.s0 + gap_score)) - { - if ((score4.s3 + char_profile) > score.s3) - { - score.s3 = score4.s3 + char_profile; - trace.t3 = current_node - pred_node; - } - } - else - { - if ((score4_next.s0 + gap_score) > score.s3) - { - score.s3 = score4_next.s0 + gap_score; - trace.t3 = -(current_node - pred_node); - } - } + score.s0 = max(score4.s0 + char_profile.s0, + score4.s1 + gap_score); + score.s1 = max(score4.s1 + char_profile.s1, + score4.s2 + gap_score); + score.s2 = max(score4.s2 + char_profile.s2, + score4.s3 + gap_score); + score.s3 = max(score4.s3 + char_profile.s3, + score4_next.s0 + gap_score); + + return score; } } template + typename SizeT> __device__ __forceinline__ int32_t runNeedlemanWunschBanded(SeqT* nodes, @@ -285,11 +185,9 @@ __device__ __forceinline__ SeqT* read, int32_t read_length, ScoreT* scores, - TraceT* backtrace, SizeT* alignment_graph, SizeT* alignment_read, int32_t band_width, - int32_t score_matrix_height, int32_t gap_score, int32_t mismatch_score, int32_t match_score) @@ -306,7 +204,7 @@ __device__ __forceinline__ // Initialise the horizontal boundary of the score matrix for (int32_t j = lane_idx; j < band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) { - set_score(scores, 0, j, score_matrix_height, j * gap_score, gradient, band_width, max_column); + set_score(scores, 0, j, j * gap_score, gradient, band_width, max_column); } if (lane_idx == 0) @@ -326,7 +224,7 @@ __device__ __forceinline__ int32_t band_start = get_band_start_for_row(score_gIdx, gradient, band_width, max_column); int32_t pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]; - initialize_band(scores, score_gIdx, score_matrix_height, min_score_value, gradient, band_width, max_column, lane_idx); + initialize_band(scores, score_gIdx, min_score_value, gradient, band_width, max_column, lane_idx); int32_t first_element_prev_score = 0; uint16_t pred_count = 0; @@ -339,71 +237,28 @@ __device__ __forceinline__ pred_count = incoming_edge_count[node_id]; if (pred_count == 0) { - int64_t index = static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - scores[index] = gap_score; - index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - backtrace[index] = -score_gIdx; + set_score(scores, score_gIdx, -1, gap_score, gradient, band_width, max_column); } else { - int64_t index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - pred_idx = node_id_to_pos[pred_node_id] + 1; - - if ((graph_pos - pred_idx) < score_matrix_height) + if (band_start > CELLS_PER_THREAD && pred_count == 1) { - // fill in first column of backtrace buffer - backtrace[index] = -(score_gIdx - pred_idx); - - if (band_start > CELLS_PER_THREAD && pred_count == 1) - { - first_element_prev_score = min_score_value + gap_score; - } - else - { - penalty = max(min_score_value, get_score(scores, pred_idx, -1, score_matrix_height, gradient, band_width, max_column, min_score_value)); - // if pred_num > 1 keep checking to find max score as penalty - for (int32_t p = 1; p < pred_count; p++) - { - pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; - int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; - if ((score_gIdx - pred_idx_tmp) < score_matrix_height) - { - int32_t trace_tmp = -(score_gIdx - pred_idx_tmp); - int32_t score_tmp = get_score(scores, pred_idx_tmp, -1, score_matrix_height, gradient, band_width, max_column, min_score_value); - if (penalty < score_tmp) - { - penalty = score_tmp; - backtrace[index] = trace_tmp; - } - } - } - first_element_prev_score = penalty + gap_score; - set_score(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, gradient, band_width, max_column); - } + first_element_prev_score = min_score_value + gap_score; } else { - penalty = min_score_value; - // pick the predecessor which falls within the limit of score_matrix_height - for (int32_t p = 1; p < pred_count; p++) + penalty = max(min_score_value, get_score(scores, pred_idx, -1, gradient, band_width, max_column, min_score_value)); + // if pred_num > 1 keep checking to find max score as penalty + for (int32_t p = 0; p < pred_count; p++) { pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; - if ((score_gIdx - pred_idx_tmp) < score_matrix_height) - { - int32_t trace_tmp = -(score_gIdx - pred_idx_tmp); - int32_t score_tmp = get_score(scores, pred_idx_tmp, -1, score_matrix_height, gradient, band_width, max_column, min_score_value); - if (penalty < score_tmp) - { - penalty = score_tmp; - backtrace[index] = trace_tmp; - } - } + penalty = max(penalty, get_score(scores, pred_idx_tmp, -1, gradient, band_width, max_column, min_score_value)); } first_element_prev_score = penalty + gap_score; - set_score(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, gradient, band_width, max_column); } + set_score(scores, score_gIdx, -1, first_element_prev_score, gradient, band_width, max_column); } } pred_count = __shfl_sync(FULL_MASK, pred_count, 0); @@ -417,20 +272,25 @@ __device__ __forceinline__ SeqT4* d_read4 = (SeqT4*)read; SeqT4 read4 = d_read4[read_pos / CELLS_PER_THREAD]; - TraceT4 trace; - ScoreT4 score = {min_score_value, min_score_value, min_score_value, min_score_value}; - get_scores(scores, pred_idx, score_gIdx, read_pos, score_matrix_height, gradient, band_width, max_column, - gap_score, match_score, mismatch_score, read4, graph_base, score, trace); + ScoreT4 char_profile; + char_profile.s0 = (graph_base == read4.r0 ? match_score : mismatch_score); + char_profile.s1 = (graph_base == read4.r1 ? match_score : mismatch_score); + char_profile.s2 = (graph_base == read4.r2 ? match_score : mismatch_score); + char_profile.s3 = (graph_base == read4.r3 ? match_score : mismatch_score); + + ScoreT4 score = get_scores(scores, pred_idx, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile); // Perform same score updates as above, but for rest of predecessors. for (int32_t p = 1; p < pred_count; p++) { - int32_t pred_idx_tmp = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; - if ((score_gIdx - pred_idx_tmp) < score_matrix_height) - { - get_scores(scores, pred_idx_tmp, score_gIdx, read_pos, score_matrix_height, gradient, band_width, max_column, - gap_score, match_score, mismatch_score, read4, graph_base, score, trace); - } + int32_t pred_idx2 = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; + + ScoreT4 scores_4 = get_scores(scores, pred_idx2, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile); + + score.s0 = max(score.s0, scores_4.s0); + score.s1 = max(score.s1, scores_4.s1); + score.s2 = max(score.s2, scores_4.s2); + score.s3 = max(score.s3, scores_4.s3); } // While there are changes to the horizontal score values, keep updating the matrix. @@ -458,7 +318,6 @@ __device__ __forceinline__ if (tscore > score.s0) { score.s0 = tscore; - trace.t0 = 0; loop = true; } @@ -466,7 +325,6 @@ __device__ __forceinline__ if (tscore > score.s1) { score.s1 = tscore; - trace.t1 = 0; loop = true; } @@ -474,7 +332,6 @@ __device__ __forceinline__ if (tscore > score.s2) { score.s2 = tscore; - trace.t2 = 0; loop = true; } @@ -482,7 +339,6 @@ __device__ __forceinline__ if (tscore > score.s3) { score.s3 = tscore; - trace.t3 = 0; loop = true; } } @@ -491,17 +347,12 @@ __device__ __forceinline__ // which can be used to compute the first cell of the next warp. first_element_prev_score = __shfl_sync(FULL_MASK, score.s3, WARP_SIZE - 1); - int64_t index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - scores[index] = score.s0; - scores[index + 1L] = score.s1; - scores[index + 2L] = score.s2; - scores[index + 3L] = score.s3; + int64_t score_index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - backtrace[index] = trace.t0; - backtrace[index + 1L] = trace.t1; - backtrace[index + 2L] = trace.t2; - backtrace[index + 3L] = trace.t3; + scores[score_index] = score.s0; + scores[score_index + 1L] = score.s1; + scores[score_index + 2L] = score.s2; + scores[score_index + 3L] = score.s3; __syncwarp(); } @@ -519,57 +370,107 @@ __device__ __forceinline__ { if (outgoing_edge_count[graph[idx - 1]] == 0) { - if ((graph_count - idx) < score_matrix_height) - { - int32_t s = get_score(scores, idx, j, score_matrix_height, gradient, band_width, max_column, min_score_value); - if (mscore < s) - { - mscore = s; - i = idx; - } - } - else + int32_t s = get_score(scores, idx, j, gradient, band_width, max_column, min_score_value); + if (mscore < s) { - ///ToDo throw an error indicating selected score_matrix_height (i.e. max predecessor distance) is too small + mscore = s; + i = idx; } } } - //------------------------------------------------------------------------ - // Fill in backtrace + int32_t prev_i = 0; + int32_t prev_j = 0; + int32_t next_node_id = i > 0 ? graph[i - 1] : 0; + int32_t loop_count = 0; while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) { loop_count++; + int32_t scores_ij = get_score(scores, i, j, gradient, band_width, max_column, min_score_value); + bool pred_found = false; + // Check if move is diagonal. + if (i != 0 && j != 0) + { - int32_t band_start = get_band_start_for_row(i, gradient, band_width, max_column); - TraceT trace = get_trace(backtrace, i, j, band_start, band_width); + int32_t node_id = next_node_id; - if (trace == 0) - { - // horizontal path (indel) - alignment_graph[aligned_nodes] = -1; - alignment_read[aligned_nodes] = j - 1; - j--; + int32_t match_cost = (nodes[node_id] == read[j - 1] ? match_score : mismatch_score); + uint16_t pred_count = incoming_edge_count[node_id]; + int32_t pred_i = (pred_count == 0 ? 0 : (node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]] + 1)); + + if (scores_ij == (get_score(scores, pred_i, j - 1, gradient, band_width, max_column, min_score_value) + match_cost)) + { + prev_i = pred_i; + prev_j = j - 1; + pred_found = true; + } + + if (!pred_found) + { + for (int32_t p = 1; p < pred_count; p++) + { + pred_i = (node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1); + + if (scores_ij == (get_score(scores, pred_i, j - 1, gradient, band_width, max_column, min_score_value) + match_cost)) + { + prev_i = pred_i; + prev_j = j - 1; + pred_found = true; + break; + } + } + } } - else if (trace < 0) + + // Check if move is vertical. + if (!pred_found && i != 0) { - // vertical path (indel) - alignment_graph[aligned_nodes] = graph[i - 1]; - alignment_read[aligned_nodes] = -1; - i += trace; + int32_t node_id = graph[i - 1]; + uint16_t pred_count = incoming_edge_count[node_id]; + int32_t pred_i = (pred_count == 0 ? 0 : node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]] + 1); + + if (scores_ij == get_score(scores, pred_i, j, gradient, band_width, max_column, min_score_value) + gap_score) + { + prev_i = pred_i; + prev_j = j; + pred_found = true; + } + + if (!pred_found) + { + for (int32_t p = 1; p < pred_count; p++) + { + pred_i = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; + + if (scores_ij == get_score(scores, pred_i, j, gradient, band_width, max_column, min_score_value) + gap_score) + { + prev_i = pred_i; + prev_j = j; + pred_found = true; + break; + } + } + } } - else + + // Check if move is horizontal. + if (!pred_found && scores_ij == get_score(scores, i, j - 1, gradient, band_width, max_column, min_score_value) + gap_score) { - // diagonal path (match/mismatch) - alignment_graph[aligned_nodes] = graph[i - 1]; - alignment_read[aligned_nodes] = j - 1; - i -= trace; - j--; + prev_i = i; + prev_j = j - 1; + pred_found = true; } + next_node_id = graph[prev_i - 1]; + + alignment_graph[aligned_nodes] = (i == prev_i ? -1 : graph[i - 1]); + alignment_read[aligned_nodes] = (j == prev_j ? -1 : j - 1); aligned_nodes++; + + i = prev_i; + j = prev_j; } if (loop_count >= (read_length + graph_count + 2)) diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh new file mode 100644 index 000000000..59c2bc388 --- /dev/null +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -0,0 +1,592 @@ +/* +* Copyright 2019-2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#pragma once + +#include "cudapoa_structs.cuh" + +#include +#include + +#include + +namespace claraparabricks +{ + +namespace genomeworks +{ + +namespace cudapoa +{ + +__device__ __forceinline__ int32_t get_band_start_for_row_tb(int32_t row_idx, float gradient, int32_t band_width, int32_t max_column) +{ + + int32_t start_pos = int32_t(row_idx * gradient) - band_width / 2; + + start_pos = max(start_pos, 0); + + int32_t end_pos = start_pos + band_width; + + if (end_pos > max_column) + { + start_pos = max_column - band_width + CELLS_PER_THREAD; + } + + start_pos = max(start_pos, 0); + + start_pos = start_pos - (start_pos % CELLS_PER_THREAD); + + return start_pos; +} + +template +__device__ __forceinline__ ScoreT* get_score_ptr_tb(ScoreT* scores, int32_t score_row, int32_t column, int32_t band_start, int32_t band_width) +{ + column = column == -1 ? 0 : column - band_start; + + int64_t score_index = static_cast(column) + + static_cast(score_row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + + return &scores[score_index]; +} + +template +__device__ __forceinline__ void set_score_tb(ScoreT* scores, + int32_t row, + int32_t column, + int32_t score_matrix_height, + int32_t value, + float gradient, + int32_t band_width, + int32_t max_column) +{ + int32_t band_start = get_band_start_for_row_tb(row, gradient, band_width, max_column); + + int32_t col_idx; + if (column == -1) + { + col_idx = band_start; + } + else + { + col_idx = column - band_start; + } + + int64_t score_index = static_cast(col_idx) + + static_cast(row % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + scores[score_index] = value; +} + +template +__device__ __forceinline__ void initialize_band_tb(ScoreT* scores, + int32_t row, + int32_t score_matrix_height, + int32_t value, + float gradient, + int32_t band_width, + int32_t max_column, + int32_t lane_idx) +{ + int32_t band_start = get_band_start_for_row_tb(row, gradient, band_width, max_column); + int32_t band_end = band_start + band_width; + + int32_t initialization_offset = (band_start == 0) ? 1 : band_start; + + set_score_tb(scores, row, initialization_offset, score_matrix_height, value, gradient, band_width, max_column); + + // note: as long as CUDAPOA_BANDED_MATRIX_RIGHT_PADDING < WARP_SIZE, no need for a for loop + for (int32_t j = lane_idx + band_end; j < band_end + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) + { + set_score_tb(scores, row, j, score_matrix_height, value, gradient, band_width, max_column); + } +} + +template +__device__ __forceinline__ TraceT get_trace(TraceT* backtrace, int32_t row, int32_t column, int32_t band_start, int32_t band_width) +{ + int64_t trace_index = static_cast(column - band_start) + + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + return backtrace[trace_index]; +} + +template +__device__ __forceinline__ ScoreT get_score_tb(ScoreT* scores, + int32_t row, + int32_t column, + int32_t score_matrix_height, + float gradient, + int32_t band_width, + int32_t max_column, + const ScoreT min_score_value) +{ + int32_t band_start = get_band_start_for_row_tb(row, gradient, band_width, max_column); + int32_t band_end = band_start + band_width; + band_end = min(band_end, max_column); + + if (((column > band_end) || (column < band_start)) && column != -1) + { + return min_score_value; + } + else + { + return *get_score_ptr_tb(scores, row % score_matrix_height, column, band_start, band_width); + } +} + +template +__device__ __forceinline__ void get_scores_tb(ScoreT* scores, + int32_t pred_node, + int32_t current_node, + int32_t read_pos, + int32_t score_matrix_height, + float gradient, + int32_t band_width, + int32_t max_column, + int32_t gap_score, + int32_t match_score, + int32_t mismatch_score, + SeqT4 read4, + SeqT graph_base, + ScoreT4& score, + TraceT4& trace) +{ + + // The load instructions typically load data in 4B or 8B chunks. + // If data is 16b (2B), then a 4B load chunk is loaded into register + // and the necessary bits are extracted before returning. This wastes cycles + // as each read of 16b issues a separate load command. + // Instead it is better to load a 4B or 8B chunk into a register + // using a single load inst, and then extracting necessary part of + // of the data using bit arithmetic. Also reduces register count. + + int32_t band_start = get_band_start_for_row_tb(pred_node, gradient, band_width, max_column); + + // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds + int32_t band_end = static_cast(band_start + band_width - CELLS_PER_THREAD); + + if ((read_pos > band_end || read_pos < band_start) && read_pos != -1) + { + return; + } + else + { + ScoreT4* pred_scores = (ScoreT4*)get_score_ptr_tb(scores, pred_node % score_matrix_height, read_pos, band_start, band_width); + + // loads 8/16 consecutive bytes (4 ScoreT) + ScoreT4 score4 = pred_scores[0]; + + // need to load the next chunk of memory as well + ScoreT4 score4_next = pred_scores[1]; + + ScoreT char_profile = (graph_base == read4.r0 ? match_score : mismatch_score); + + // if trace is diogonal, its value is positive and if vertical, negative + // update score.s0, trace.t0 ---------- + if ((score4.s0 + char_profile) >= (score4.s1 + gap_score)) + { + if ((score4.s0 + char_profile) > score.s0) + { + score.s0 = score4.s0 + char_profile; + trace.t0 = current_node - pred_node; + } + } + else + { + if ((score4.s1 + gap_score) > score.s0) + { + score.s0 = score4.s1 + gap_score; + trace.t0 = -(current_node - pred_node); + } + } + // update score.s1, trace.t1 ---------- + char_profile = (graph_base == read4.r1 ? match_score : mismatch_score); + if ((score4.s1 + char_profile) >= (score4.s2 + gap_score)) + { + if ((score4.s1 + char_profile) > score.s1) + { + score.s1 = score4.s1 + char_profile; + trace.t1 = current_node - pred_node; + } + } + else + { + if ((score4.s2 + gap_score) > score.s1) + { + score.s1 = score4.s2 + gap_score; + trace.t1 = -(current_node - pred_node); + } + } + // update score.s2, trace.t2 ---------- + char_profile = (graph_base == read4.r2 ? match_score : mismatch_score); + if ((score4.s2 + char_profile) >= (score4.s3 + gap_score)) + { + if ((score4.s2 + char_profile) > score.s2) + { + score.s2 = score4.s2 + char_profile; + trace.t2 = current_node - pred_node; + } + } + else + { + if ((score4.s3 + gap_score) > score.s2) + { + score.s2 = score4.s3 + gap_score; + trace.t2 = -(current_node - pred_node); + } + } + // update score.s3, trace.t3 ---------- + char_profile = (graph_base == read4.r3 ? match_score : mismatch_score); + if ((score4.s3 + char_profile) >= (score4_next.s0 + gap_score)) + { + if ((score4.s3 + char_profile) > score.s3) + { + score.s3 = score4.s3 + char_profile; + trace.t3 = current_node - pred_node; + } + } + else + { + if ((score4_next.s0 + gap_score) > score.s3) + { + score.s3 = score4_next.s0 + gap_score; + trace.t3 = -(current_node - pred_node); + } + } + } +} + +template +__device__ __forceinline__ + int32_t + runNeedlemanWunschBandedTraceBack(SeqT* nodes, + SizeT* graph, + SizeT* node_id_to_pos, + int32_t graph_count, + uint16_t* incoming_edge_count, + SizeT* incoming_edges, + uint16_t* outgoing_edge_count, + SeqT* read, + int32_t read_length, + ScoreT* scores, + TraceT* backtrace, + SizeT* alignment_graph, + SizeT* alignment_read, + int32_t band_width, + int32_t score_matrix_height, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score) +{ + + const ScoreT min_score_value = 2 * abs(min(min(gap_score, mismatch_score), -match_score) - 1) + numeric_limits::min(); + + int32_t lane_idx = threadIdx.x % WARP_SIZE; + + //Calculate gradient for the scores matrix + float gradient = float(read_length + 1) / float(graph_count + 1); + + int32_t max_column = read_length + 1; + // Initialise the horizontal boundary of the score matrix + for (int32_t j = lane_idx; j < band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) + { + set_score_tb(scores, 0, j, score_matrix_height, j * gap_score, gradient, band_width, max_column); + } + + if (lane_idx == 0) + { +#ifdef NW_VERBOSE_PRINT + printf("graph %d, read %d\n", graph_count, read_length); +#endif + } + + __syncwarp(); + + // compute vertical and diagonal values in parallel. + for (int32_t graph_pos = 0; graph_pos < graph_count; graph_pos++) + { + int32_t node_id = graph[graph_pos]; + int32_t score_gIdx = graph_pos + 1; + int32_t band_start = get_band_start_for_row_tb(score_gIdx, gradient, band_width, max_column); + int32_t pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]; + + initialize_band_tb(scores, score_gIdx, score_matrix_height, min_score_value, gradient, band_width, max_column, lane_idx); + + int32_t first_element_prev_score = 0; + uint16_t pred_count = 0; + int32_t pred_idx = 0; + + if (lane_idx == 0) + { + // Initialise the vertical boundary of the score matrix + int32_t penalty; + pred_count = incoming_edge_count[node_id]; + if (pred_count == 0) + { + int64_t index = static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + scores[index] = gap_score; + index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + backtrace[index] = -score_gIdx; + } + else + { + int64_t index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + + pred_idx = node_id_to_pos[pred_node_id] + 1; + + if ((graph_pos - pred_idx) < score_matrix_height) + { + // fill in first column of backtrace buffer + backtrace[index] = -(score_gIdx - pred_idx); + + if (band_start > CELLS_PER_THREAD && pred_count == 1) + { + first_element_prev_score = min_score_value + gap_score; + } + else + { + penalty = max(min_score_value, get_score_tb(scores, pred_idx, -1, score_matrix_height, gradient, band_width, max_column, min_score_value)); + // if pred_num > 1 keep checking to find max score as penalty + for (int32_t p = 1; p < pred_count; p++) + { + pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; + int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; + if ((score_gIdx - pred_idx_tmp) < score_matrix_height) + { + int32_t trace_tmp = -(score_gIdx - pred_idx_tmp); + int32_t score_tmp = get_score_tb(scores, pred_idx_tmp, -1, score_matrix_height, gradient, band_width, max_column, min_score_value); + if (penalty < score_tmp) + { + penalty = score_tmp; + backtrace[index] = trace_tmp; + } + } + } + first_element_prev_score = penalty + gap_score; + set_score_tb(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, gradient, band_width, max_column); + } + } + else + { + penalty = min_score_value; + // pick the predecessor which falls within the limit of score_matrix_height + for (int32_t p = 1; p < pred_count; p++) + { + pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; + int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; + if ((score_gIdx - pred_idx_tmp) < score_matrix_height) + { + int32_t trace_tmp = -(score_gIdx - pred_idx_tmp); + int32_t score_tmp = get_score_tb(scores, pred_idx_tmp, -1, score_matrix_height, gradient, band_width, max_column, min_score_value); + if (penalty < score_tmp) + { + penalty = score_tmp; + backtrace[index] = trace_tmp; + } + } + } + first_element_prev_score = penalty + gap_score; + set_score_tb(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, gradient, band_width, max_column); + } + } + } + pred_count = __shfl_sync(FULL_MASK, pred_count, 0); + pred_idx = __shfl_sync(FULL_MASK, pred_idx, 0); + //------------------------------------------------------------- + + SeqT graph_base = nodes[node_id]; + + for (int32_t read_pos = lane_idx * CELLS_PER_THREAD + band_start; read_pos < band_start + band_width; read_pos += CUDAPOA_MIN_BAND_WIDTH) + { + SeqT4* d_read4 = (SeqT4*)read; + SeqT4 read4 = d_read4[read_pos / CELLS_PER_THREAD]; + + TraceT4 trace; + ScoreT4 score = {min_score_value, min_score_value, min_score_value, min_score_value}; + get_scores_tb(scores, pred_idx, score_gIdx, read_pos, score_matrix_height, gradient, band_width, max_column, + gap_score, match_score, mismatch_score, read4, graph_base, score, trace); + + // Perform same score updates as above, but for rest of predecessors. + for (int32_t p = 1; p < pred_count; p++) + { + int32_t pred_idx_tmp = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; + if ((score_gIdx - pred_idx_tmp) < score_matrix_height) + { + get_scores_tb(scores, pred_idx_tmp, score_gIdx, read_pos, score_matrix_height, gradient, band_width, max_column, + gap_score, match_score, mismatch_score, read4, graph_base, score, trace); + } + } + + // While there are changes to the horizontal score values, keep updating the matrix. + // So loop will only run the number of time there are corrections in the matrix. + // The any_sync warp primitive lets us easily check if any of the threads had an update. + bool loop = true; + + while (__any_sync(FULL_MASK, loop)) + { + loop = false; + // Note: computation of s3 depends on s2, s2 depends on s1 and s1 on s0. + // If we reverse the order of computation in this loop from s3 to s0, it will increase + // ILP. However, in longer reads where indels are more frequent, this reverse computations + // results in larger number of iterations. Since if s0 is changed, value of s1, s2 and s3 which + // already have been computed in parallel need to be updated again. + + // The shfl_up lets us grab a value from the lane below. + int32_t last_score = __shfl_up_sync(FULL_MASK, score.s3, 1); + if (lane_idx == 0) + { + last_score = first_element_prev_score; + } + + int32_t tscore = max(last_score + gap_score, score.s0); + if (tscore > score.s0) + { + score.s0 = tscore; + trace.t0 = 0; + loop = true; + } + + tscore = max(score.s0 + gap_score, score.s1); + if (tscore > score.s1) + { + score.s1 = tscore; + trace.t1 = 0; + loop = true; + } + + tscore = max(score.s1 + gap_score, score.s2); + if (tscore > score.s2) + { + score.s2 = tscore; + trace.t2 = 0; + loop = true; + } + + tscore = max(score.s2 + gap_score, score.s3); + if (tscore > score.s3) + { + score.s3 = tscore; + trace.t3 = 0; + loop = true; + } + } + + // Copy over the last element score of the last lane into a register of first lane + // which can be used to compute the first cell of the next warp. + first_element_prev_score = __shfl_sync(FULL_MASK, score.s3, WARP_SIZE - 1); + + int64_t index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + scores[index] = score.s0; + scores[index + 1L] = score.s1; + scores[index + 2L] = score.s2; + scores[index + 3L] = score.s3; + + index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + backtrace[index] = trace.t0; + backtrace[index + 1L] = trace.t1; + backtrace[index + 2L] = trace.t2; + backtrace[index + 3L] = trace.t3; + + __syncwarp(); + } + } + + int32_t aligned_nodes = 0; + if (lane_idx == 0) + { + // Find location of the maximum score in the matrix. + int32_t i = 0; + int32_t j = read_length; + int32_t mscore = min_score_value; + + for (int32_t idx = 1; idx <= graph_count; idx++) + { + if (outgoing_edge_count[graph[idx - 1]] == 0) + { + if ((graph_count - idx) < score_matrix_height) + { + int32_t s = get_score_tb(scores, idx, j, score_matrix_height, gradient, band_width, max_column, min_score_value); + if (mscore < s) + { + mscore = s; + i = idx; + } + } + else + { + ///ToDo throw an error indicating selected score_matrix_height (i.e. max predecessor distance) is too small + } + } + } + + //------------------------------------------------------------------------ + + // Fill in backtrace + int32_t loop_count = 0; + while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) + { + loop_count++; + + int32_t band_start = get_band_start_for_row_tb(i, gradient, band_width, max_column); + TraceT trace = get_trace(backtrace, i, j, band_start, band_width); + + if (trace == 0) + { + // horizontal path (indel) + alignment_graph[aligned_nodes] = -1; + alignment_read[aligned_nodes] = j - 1; + j--; + } + else if (trace < 0) + { + // vertical path (indel) + alignment_graph[aligned_nodes] = graph[i - 1]; + alignment_read[aligned_nodes] = -1; + i += trace; + } + else + { + // diagonal path (match/mismatch) + alignment_graph[aligned_nodes] = graph[i - 1]; + alignment_read[aligned_nodes] = j - 1; + i -= trace; + j--; + } + + aligned_nodes++; + } + + if (loop_count >= (read_length + graph_count + 2)) + { + aligned_nodes = -1; + } + +#ifdef NW_VERBOSE_PRINT + printf("aligned nodes %d, loop count %d\n", aligned_nodes, loop_count); +#endif + } + aligned_nodes = __shfl_sync(FULL_MASK, aligned_nodes, 0); + return aligned_nodes; +} + +} // namespace cudapoa + +} // namespace genomeworks + +} // namespace claraparabricks From 7df7bcf7a228791d565cd6959c74ace672f9ab15 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 23 Sep 2020 21:58:10 -0400 Subject: [PATCH 150/281] [cudapoa-backtracking] variable renaming --- cudapoa/src/allocate_block.hpp | 8 +++---- cudapoa/src/cudapoa_batch.cuh | 6 ++--- cudapoa/src/cudapoa_kernels.cuh | 26 ++++++++++---------- cudapoa/src/cudapoa_nw.cuh | 8 +++---- cudapoa/src/cudapoa_nw_adaptive_banded.cuh | 9 ++++--- cudapoa/src/cudapoa_nw_banded.cuh | 2 +- cudapoa/src/cudapoa_nw_tb_banded.cuh | 28 +++++++++++----------- cudapoa/src/cudapoa_structs.cuh | 4 ++-- 8 files changed, 44 insertions(+), 47 deletions(-) diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index ca2a69cb8..095f980a3 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -233,16 +233,16 @@ class BatchBlock if (full_matrix_alignment_) { - // in full-band mode, where only scores buffer is used and backtrace buffer size is 0, rest of the available memory is assigned to scores buffer + // in full-band mode, where only scores buffer is used and traceback buffer size is 0, rest of the available memory is assigned to scores buffer alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); - alignment_details_d->backtrace = nullptr; + alignment_details_d->traceback = nullptr; } else { - // in banded alignment, rest of the available memory is assigned to backtrace buffer + // in banded alignment, rest of the available memory is assigned to traceback buffer alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; - alignment_details_d->backtrace = reinterpret_castbacktrace)>(&block_data_d_[offset_d_]); + alignment_details_d->traceback = reinterpret_casttraceback)>(&block_data_d_[offset_d_]); } *alignment_details_d_p = alignment_details_d; diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 3ae2b6a79..8b6f43f49 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -549,12 +549,12 @@ protected: return StatusType::success; } - // Check if intermediate data for seq length can fit in available scoring/backtrace buffer + // Check if intermediate data for seq length can fit in available scoring/traceback buffer bool reserve_buf(int32_t max_seq_length) { int32_t matrix_height = batch_size_.max_nodes_per_graph; int32_t matrix_width = (banded_alignment_ || adaptive_banded_) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); - // in full-band, avail_buf_mem_ is dedicated to scores matrix and in static or adaptive band modes, avail_buf_mem_ is dedicated to backtrace matrix + // in full-band, avail_buf_mem_ is dedicated to scores matrix and in static or adaptive band modes, avail_buf_mem_ is dedicated to traceback matrix size_t required_size = static_cast(matrix_width) * static_cast(matrix_height); required_size *= batch_size_.band_mode == BandMode::full_band ? sizeof(ScoreT) : sizeof(TraceT); @@ -623,7 +623,7 @@ protected: // Global sequence index. int32_t global_sequence_idx_ = 0; - // Remaining buffer memory available for scores matrices in case of full alignment, and for backtrace matrices in case of banded alignment + // Remaining buffer memory available for scores matrices in case of full alignment, and for traceback matrices in case of banded alignment size_t avail_buf_mem_ = 0; // Temporary variable to compute the offset to scorebuf. diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 93387da81..318130497 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -52,8 +52,8 @@ namespace cudapoa * @param[in] window_details_d Device buffer with structs encapsulating sequence details per window * @param[in] total_windows Total number of windows to process * @param[in] scores_d Device scratch space that scores alignment matrix score - * @param[in] alignment_graph_d Device scratch space for backtrace alignment of graph - * @param[in] alignment_read_d Device scratch space for backtrace alignment of sequence + * @param[in] alignment_graph_d Device scratch space for traceback alignment of graph + * @param[in] alignment_read_d Device scratch space for traceback alignment of sequence * @param[in] nodes_d Device scratch space for storing unique nodes in graph * @param[in] incoming_edges_d Device scratch space for storing incoming edges per node * @param[in] incoming_edges_count_d Device scratch space for storing number of incoming edges per node @@ -112,7 +112,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) int32_t TPB = 64, int32_t static_band_width = 256, int32_t max_pred_distance = 0, - TraceT* backtrace_d = nullptr) + TraceT* traceback_d = nullptr) { // shared error indicator within a warp bool warp_error = false; @@ -139,7 +139,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) int32_t scores_width = window_details_d[window_idx].scores_width; ScoreT* scores; - TraceT* backtrace = backtrace_d; + TraceT* traceback = traceback_d; float banded_buffer_size; // using float instead of int64_t to minimize register if (BM == BandMode::adaptive_band || BM == BandMode::static_band) { @@ -147,10 +147,10 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); int64_t offset = static_cast(banded_buffer_size) * static_cast(window_idx); scores = &scores_d[offset]; - // buffer size for backtrace + // buffer size for traceback banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); offset = static_cast(banded_buffer_size) * static_cast(window_idx); - backtrace = &backtrace_d[offset]; + traceback = &traceback_d[offset]; } else { @@ -267,7 +267,6 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) sequence, seq_len, scores, - backtrace, banded_buffer_size, alignment_graph, alignment_read, @@ -292,7 +291,6 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) sequence, seq_len, scores, - backtrace, banded_buffer_size, alignment_graph, alignment_read, @@ -316,7 +314,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) sequence, seq_len, scores, - backtrace, + traceback, alignment_graph, alignment_read, static_band_width, @@ -465,7 +463,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, // unpack alignment details ScoreT* scores = alignment_details_d->scores; - TraceT* backtrace = alignment_details_d->backtrace; + TraceT* traceback = alignment_details_d->traceback; SizeT* alignment_graph = alignment_details_d->alignment_graph; SizeT* alignment_read = alignment_details_d->alignment_read; @@ -544,7 +542,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, TPB, batch_size.alignment_band_width, batch_size.max_pred_distance_in_banded_mode, - backtrace); + traceback); } else if (adaptive_banded) { @@ -586,7 +584,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, TPB, batch_size.alignment_band_width, batch_size.max_pred_distance_in_banded_mode, - backtrace); + traceback); } else { @@ -670,7 +668,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, TPB, batch_size.alignment_band_width, batch_size.max_pred_distance_in_banded_mode, - backtrace); + traceback); } else if (adaptive_banded) { @@ -712,7 +710,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, TPB, batch_size.alignment_band_width, batch_size.max_pred_distance_in_banded_mode, - backtrace); + traceback); } else { diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index e3dfddaad..7fb6e7dd5 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -138,8 +138,8 @@ __device__ __forceinline__ * @param[in] read Device buffer with sequence (read) to align * @param[in] read_length Number of bases in read * @param[out] scores Device scratch space that scores alignment matrix score - * @param[out] alignment_graph Device scratch space for backtrace alignment of graph - * @param[out] alignment_read Device scratch space for backtrace alignment of sequence + * @param[out] alignment_graph Device scratch space for traceback alignment of graph + * @param[out] alignment_read Device scratch space for traceback alignment of sequence * @param[in] gap_score Score for inserting gap into alignment * @param[in] mismatch_score Score for finding a mismatch in alignment * @param[in] match_score Score for finding a match in alignment @@ -353,7 +353,7 @@ __device__ __forceinline__ } } - // Fill in backtrace + // Fill in traceback int32_t prev_i = 0; int32_t prev_j = 0; @@ -362,7 +362,7 @@ __device__ __forceinline__ // backtracking is done by re-calculating the score at each cell // along the path to see which preceding cell the move could have // come from. This seems computationally more expensive, but doesn't - // require storing any backtrace buffer during alignment. + // require storing any traceback buffer during alignment. int32_t loop_count = 0; while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) { diff --git a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh index 9bb4d04d4..05fd31199 100644 --- a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh +++ b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh @@ -178,7 +178,6 @@ __device__ __forceinline__ SeqT* read, int32_t read_length, ScoreT* scores, - TraceT* backtrace, float max_buffer_size, SizeT* alignment_graph, SizeT* alignment_read, @@ -221,8 +220,8 @@ __device__ __forceinline__ // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 int32_t band_shift = band_width / 2; // rerun code is defined in backtracking loop from previous alignment try - // -3 means backtrace path was too close to the left bound of band - // -4 means backtrace path was too close to the right bound of band + // -3 means traceback path was too close to the left bound of band + // -4 means traceback path was too close to the right bound of band // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift // further to the left for rerun == -3, and further to the right for rerun == -4. if (rerun == -3) @@ -425,7 +424,7 @@ __device__ __forceinline__ } } - // Fill in backtrace + // Fill in traceback int32_t prev_i = 0; int32_t prev_j = 0; int32_t next_node_id = i > 0 ? graph[i - 1] : 0; @@ -441,7 +440,7 @@ __device__ __forceinline__ { if (rerun == 0) { - // check if backtrace gets too close or hits the band limits, if so stop and rerun with extended band-width + // check if traceback gets too close or hits the band limits, if so stop and rerun with extended band-width // threshold for proximity to band limits works better if defined proportionate to the sequence length int32_t threshold = max(1, max_column / 1024); // ad-hoc rule 7 if (j > threshold && j < max_column - threshold) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index adbc5b295..2cb70d06c 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -379,7 +379,7 @@ __device__ __forceinline__ } } - // Fill in backtrace + // Fill in traceback int32_t prev_i = 0; int32_t prev_j = 0; int32_t next_node_id = i > 0 ? graph[i - 1] : 0; diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index 59c2bc388..4a7d001f7 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -116,11 +116,11 @@ __device__ __forceinline__ void initialize_band_tb(ScoreT* scores, } template -__device__ __forceinline__ TraceT get_trace(TraceT* backtrace, int32_t row, int32_t column, int32_t band_start, int32_t band_width) +__device__ __forceinline__ TraceT get_trace(TraceT* traceback, int32_t row, int32_t column, int32_t band_start, int32_t band_width) { int64_t trace_index = static_cast(column - band_start) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - return backtrace[trace_index]; + return traceback[trace_index]; } template @@ -285,7 +285,7 @@ __device__ __forceinline__ SeqT* read, int32_t read_length, ScoreT* scores, - TraceT* backtrace, + TraceT* traceback, SizeT* alignment_graph, SizeT* alignment_read, int32_t band_width, @@ -342,7 +342,7 @@ __device__ __forceinline__ int64_t index = static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[index] = gap_score; index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - backtrace[index] = -score_gIdx; + traceback[index] = -score_gIdx; } else { @@ -352,8 +352,8 @@ __device__ __forceinline__ if ((graph_pos - pred_idx) < score_matrix_height) { - // fill in first column of backtrace buffer - backtrace[index] = -(score_gIdx - pred_idx); + // fill in first column of traceback buffer + traceback[index] = -(score_gIdx - pred_idx); if (band_start > CELLS_PER_THREAD && pred_count == 1) { @@ -374,7 +374,7 @@ __device__ __forceinline__ if (penalty < score_tmp) { penalty = score_tmp; - backtrace[index] = trace_tmp; + traceback[index] = trace_tmp; } } } @@ -397,7 +397,7 @@ __device__ __forceinline__ if (penalty < score_tmp) { penalty = score_tmp; - backtrace[index] = trace_tmp; + traceback[index] = trace_tmp; } } } @@ -498,10 +498,10 @@ __device__ __forceinline__ scores[index + 3L] = score.s3; index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - backtrace[index] = trace.t0; - backtrace[index + 1L] = trace.t1; - backtrace[index + 2L] = trace.t2; - backtrace[index + 3L] = trace.t3; + traceback[index] = trace.t0; + traceback[index + 1L] = trace.t1; + traceback[index + 2L] = trace.t2; + traceback[index + 3L] = trace.t3; __syncwarp(); } @@ -537,14 +537,14 @@ __device__ __forceinline__ //------------------------------------------------------------------------ - // Fill in backtrace + // Fill in traceback int32_t loop_count = 0; while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) { loop_count++; int32_t band_start = get_band_start_for_row_tb(i, gradient, band_width, max_column); - TraceT trace = get_trace(backtrace, i, j, band_start, band_width); + TraceT trace = get_trace(traceback, i, j, band_start, band_width); if (trace == 0) { diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index ce6c4bc88..9f5883619 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -108,7 +108,7 @@ struct AlignmentDetails // Device buffer for the scoring matrix for all windows. ScoreT* scores; // Device buffer for the backtracking matrix for all windows. - TraceT* backtrace; + TraceT* traceback; /// Buffer for storing per row band start location in absolute score matrix for adaptive banding SizeT* band_starts; /// Buffer for storing per row band widths for adaptive banding @@ -121,7 +121,7 @@ struct AlignmentDetails // preallocated size of scores buffer size_t scorebuf_alloc_size = 0; - // Device buffers for alignment backtrace + // Device buffers for alignment traceback SizeT* alignment_graph; SizeT* alignment_read; }; From 601ffa71e144d98007fbe019c46b54be59f8e5e0 Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 24 Sep 2020 11:58:02 -0400 Subject: [PATCH 151/281] [cudapoa-backtracking] adding a new banding mode: static_band_traceback, WIP --- .../genomeworks/cudapoa/cudapoa.hpp | 13 +++++++++---- cudapoa/samples/sample_cudapoa.cpp | 6 +++--- cudapoa/src/cudapoa_batch.cuh | 10 +--------- cudapoa/src/cudapoa_kernels.cuh | 14 ++++++-------- 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp index 655859bb5..acfa00861 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp @@ -45,15 +45,20 @@ enum StatusType }; /// Banding mode used in Needleman-Wunch algorithm -/// full_band performs computations on full scores matrix, highest accuracy -/// static_band performs computations on a fixed band along scores matrix diagonal, fastest implementation -/// adaptive_band, similar to static_band performs computations on a band along diagonal, but the band-width +/// - full_band performs computations on full scores matrix, highest accuracy +/// - static_band performs computations on a fixed band along scores matrix diagonal, fastest implementation +/// - adaptive_band, similar to static_band performs computations on a band along diagonal, but the band-width /// can vary per alignment's score matrix, faster than full_band and more accurate than static_band +/// - static_band_traceback computations are on a fixed band, score matrix is stored partially, instead traceback +/// information is stored in a buffer and used in backtracking. This can be useful for long-read data GPU memory is +/// limiting GPU parallelism. Traceback matrix requires 2x to 4x smaller memory compared to score matrix + enum BandMode { full_band = 0, static_band, - adaptive_band + adaptive_band, + static_band_traceback }; /// Initialize CUDA POA context. diff --git a/cudapoa/samples/sample_cudapoa.cpp b/cudapoa/samples/sample_cudapoa.cpp index b58be0187..3d9eeac87 100644 --- a/cudapoa/samples/sample_cudapoa.cpp +++ b/cudapoa/samples/sample_cudapoa.cpp @@ -131,7 +131,7 @@ int main(int argc, char** argv) int c = 0; bool msa = false; bool long_read = false; - BandMode band_mode = BandMode::adaptive_band; // 0: full, 1: static-band, 2: adaptive-band + BandMode band_mode = BandMode::adaptive_band; // 0: full, 1: static-band, 2: adaptive-band, 3- static-band-traceback bool help = false; bool print = false; bool print_graph = false; @@ -148,9 +148,9 @@ int main(int argc, char** argv) long_read = true; break; case 'b': - if (std::stoi(optarg) < 0 || std::stoi(optarg) > 2) + if (std::stoi(optarg) < 0 || std::stoi(optarg) > 3) { - throw std::runtime_error("band-mode must be either 0 for full bands, 1 for static bands or 2 for adaptive bands"); + throw std::runtime_error("band-mode must be either 0 for full bands, 1 for static bands, 2 for adaptive bands or 3 for static band with traceback"); } band_mode = static_cast(std::stoi(optarg)); break; diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 8b6f43f49..dd13ccf1f 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -73,8 +73,6 @@ public: , gap_score_(gap_score) , mismatch_score_(mismatch_score) , match_score_(match_score) - , banded_alignment_(batch_size.band_mode == BandMode::static_band) - , adaptive_banded_(batch_size.band_mode == BandMode::adaptive_band) , batch_block_(new BatchBlock(device_id, max_gpu_mem, output_mask, @@ -187,8 +185,6 @@ public: gap_score_, mismatch_score_, match_score_, - banded_alignment_, - adaptive_banded_, max_sequences_per_poa_, output_mask_, batch_size_); @@ -553,7 +549,7 @@ protected: bool reserve_buf(int32_t max_seq_length) { int32_t matrix_height = batch_size_.max_nodes_per_graph; - int32_t matrix_width = (banded_alignment_ || adaptive_banded_) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); + int32_t matrix_width = (batch_size_.band_mode != BandMode::full_band) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); // in full-band, avail_buf_mem_ is dedicated to scores matrix and in static or adaptive band modes, avail_buf_mem_ is dedicated to traceback matrix size_t required_size = static_cast(matrix_width) * static_cast(matrix_height); required_size *= batch_size_.band_mode == BandMode::full_band ? sizeof(ScoreT) : sizeof(TraceT); @@ -629,10 +625,6 @@ protected: // Temporary variable to compute the offset to scorebuf. size_t next_scores_offset_ = 0; - // Use banded POA alignment - bool banded_alignment_; - bool adaptive_banded_; - // flag that enables some extra buffers to accommodate fully adaptive bands with variable width and arbitrary location // disabled for current implementation, can be enabled for possible future variants of adaptive alignment algorithm bool variable_band_ = false; diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 318130497..92ac22b08 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -443,8 +443,6 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, int32_t gap_score, int32_t mismatch_score, int32_t match_score, - bool static_banded, - bool adaptive_banded, uint32_t max_sequences_per_poa, int8_t output_mask, const BatchConfig& batch_size) @@ -490,8 +488,8 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, SizeT* node_id_to_msa_pos = graph_details_d->node_id_to_msa_pos; int32_t nwindows_per_block = CUDAPOA_THREADS_PER_BLOCK / WARP_SIZE; - int32_t nblocks = (static_banded || adaptive_banded) ? total_windows : (total_windows + nwindows_per_block - 1) / nwindows_per_block; - int32_t TPB = (static_banded || adaptive_banded) ? CUDAPOA_BANDED_THREADS_PER_BLOCK : CUDAPOA_THREADS_PER_BLOCK; + int32_t nblocks = (batch_size.band_mode != BandMode::full_band) ? total_windows : (total_windows + nwindows_per_block - 1) / nwindows_per_block; + int32_t TPB = (batch_size.band_mode != BandMode::full_band) ? CUDAPOA_BANDED_THREADS_PER_BLOCK : CUDAPOA_THREADS_PER_BLOCK; int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; int32_t matrix_seq_dimension = batch_size.matrix_sequence_dimension; bool msa = output_mask & OutputType::msa; @@ -502,7 +500,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, if (msa) { - if (static_banded) + if (batch_size.band_mode == BandMode::static_band) { generatePOAKernel <<>>(consensus_d, @@ -544,7 +542,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_pred_distance_in_banded_mode, traceback); } - else if (adaptive_banded) + else if (batch_size.band_mode == BandMode::adaptive_band) { generatePOAKernel <<>>(consensus_d, @@ -628,7 +626,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, } else { - if (static_banded) + if (batch_size.band_mode == BandMode::static_band) { generatePOAKernel <<>>(consensus_d, @@ -670,7 +668,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_pred_distance_in_banded_mode, traceback); } - else if (adaptive_banded) + else if (batch_size.band_mode == BandMode::adaptive_band) { generatePOAKernel <<>>(consensus_d, From 008932b5798d4a424ce7942109a1ea0beb9dc8ea Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 24 Sep 2020 15:36:25 -0400 Subject: [PATCH 152/281] [cudapoa-backtracking] adding the new banding mode: static_band_traceback --- cudapoa/src/allocate_block.hpp | 44 +++++----- cudapoa/src/application_parameters.cpp | 4 +- cudapoa/src/batch.cu | 7 +- cudapoa/src/cudapoa_batch.cuh | 2 +- cudapoa/src/cudapoa_kernels.cuh | 116 +++++++++++++++++++++++-- 5 files changed, 139 insertions(+), 34 deletions(-) diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 095f980a3..4fdb90a0f 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -60,10 +60,10 @@ class BatchBlock , output_mask_(output_mask) { scoped_device_switch dev(device_id_); - max_nodes_per_window_ = batch_size.max_nodes_per_graph; - full_matrix_alignment_ = batch_size.band_mode == BandMode::full_band; - score_matrix_height_ = full_matrix_alignment_ ? batch_size.max_nodes_per_graph : batch_size.max_pred_distance_in_banded_mode; - score_matrix_width_ = batch_size.matrix_sequence_dimension; + max_nodes_per_window_ = batch_size.max_nodes_per_graph; + traceback_alignment_ = batch_size.band_mode == BandMode::static_band_traceback; + score_matrix_height_ = traceback_alignment_ ? batch_size.max_pred_distance_in_banded_mode : batch_size.max_nodes_per_graph; + score_matrix_width_ = batch_size.matrix_sequence_dimension; // calculate static and dynamic sizes of buffers needed per POA entry. int64_t host_size_fixed, device_size_fixed; @@ -81,16 +81,16 @@ class BatchBlock } // compute device memory used for matrix buffer - // in full alignment, this is for scores matrix buffer, in static or adaptive banded alignment, it is for backtracking matrix buffer + // in banded traceback alignment, this is for traceback matrix, otherwise it is for score matrix int64_t device_size_per_matrix = static_cast(batch_size.matrix_sequence_dimension) * static_cast(batch_size.max_nodes_per_graph); - if (batch_size.band_mode == BandMode::full_band) + if (batch_size.band_mode == BandMode::static_band_traceback) { - device_size_per_matrix *= sizeof(ScoreT); + device_size_per_matrix *= sizeof(TraceT); } else { - device_size_per_matrix *= sizeof(TraceT); + device_size_per_matrix *= sizeof(ScoreT); } // Calculate max POAs possible based on available memory @@ -212,8 +212,8 @@ class BatchBlock offset_d_ += cudautils::align(sizeof(*alignment_details_d->alignment_graph) * max_nodes_per_window_ * max_poas_); alignment_details_d->alignment_read = reinterpret_castalignment_read)>(&block_data_d_[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->alignment_read) * max_nodes_per_window_ * max_poas_); - // in banded mode, we store only part of scores matrix for forward computation in NW - if (!full_matrix_alignment_) + // in traceback banded mode, we store only part of scores matrix for forward computation in NW + if (traceback_alignment_) { alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->scores) * score_matrix_width_ * score_matrix_height_ * max_poas_); @@ -231,16 +231,16 @@ class BatchBlock offset_d_ += cudautils::align(sizeof(*alignment_details_d->band_max_indices) * max_nodes_per_window_ * max_poas_); } - if (full_matrix_alignment_) + if (!traceback_alignment_) { - // in full-band mode, where only scores buffer is used and traceback buffer size is 0, rest of the available memory is assigned to scores buffer + // in non-traceback alignments, where only scores buffer is used and traceback buffer size is 0, rest of the available memory is assigned to scores buffer alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); alignment_details_d->traceback = nullptr; } else { - // in banded alignment, rest of the available memory is assigned to traceback buffer + // in traceback alignment, rest of the available memory is assigned to traceback buffer alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; alignment_details_d->traceback = reinterpret_casttraceback)>(&block_data_d_[offset_d_]); } @@ -335,10 +335,10 @@ class BatchBlock static int64_t compute_device_memory_per_poa(const BatchConfig& batch_size, const bool msa_flag, const bool variable_bands = false) { - int64_t device_size_per_poa = 0; - int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; - bool banded = batch_size.band_mode == BandMode::static_band || batch_size.band_mode == BandMode::adaptive_band; - int32_t banded_score_matrix_size = batch_size.matrix_sequence_dimension * batch_size.max_pred_distance_in_banded_mode; + int64_t device_size_per_poa = 0; + int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; + bool traceback = batch_size.band_mode == BandMode::static_band_traceback; + int32_t traceback_score_matrix_size = batch_size.matrix_sequence_dimension * batch_size.max_pred_distance_in_banded_mode; // for output - device device_size_per_poa += batch_size.max_consensus_size * sizeof(*OutputDetails::consensus); // output_details_d_->consensus @@ -378,7 +378,7 @@ class BatchBlock device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_widths) * max_nodes_per_graph : 0; // alignment_details_d_->band_widths device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_head_indices) * max_nodes_per_graph : 0; // alignment_details_d_->band_head_indices device_size_per_poa += variable_bands ? sizeof(*AlignmentDetails::band_max_indices) * max_nodes_per_graph : 0; // alignment_details_d_->band_max_indices - device_size_per_poa += banded ? sizeof(*AlignmentDetails::scores) * banded_score_matrix_size : 0; // alignment_details_d_->scores (only for static and adaptive band modes) + device_size_per_poa += traceback ? sizeof(*AlignmentDetails::scores) * traceback_score_matrix_size : 0; // alignment_details_d_->scores (only for traceback banded modes) return device_size_per_poa; } @@ -439,13 +439,13 @@ class BatchBlock // Compute required memory for score or backtracking matrix int64_t device_size_per_matrix = static_cast(batch_size.matrix_sequence_dimension) * static_cast(batch_size.max_nodes_per_graph); - if (batch_size.band_mode == BandMode::full_band) + if (batch_size.band_mode == BandMode::static_band_traceback) { - device_size_per_matrix *= sizeof_ScoreT; + device_size_per_matrix *= sizeof_TraceT; } else { - device_size_per_matrix *= sizeof_TraceT; + device_size_per_matrix *= sizeof_ScoreT; } // Calculate max POAs possible based on available memory. @@ -508,7 +508,7 @@ class BatchBlock int32_t max_nodes_per_window_ = 0; int32_t score_matrix_height_ = 0; int32_t score_matrix_width_ = 0; - bool full_matrix_alignment_ = false; + bool traceback_alignment_ = false; int32_t device_id_; // Bit field for output type diff --git a/cudapoa/src/application_parameters.cpp b/cudapoa/src/application_parameters.cpp index c8532ea8d..aebe8141e 100644 --- a/cudapoa/src/application_parameters.cpp +++ b/cudapoa/src/application_parameters.cpp @@ -66,9 +66,9 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) msa = true; break; case 'b': - if (std::stoi(optarg) < 0 || std::stoi(optarg) > 2) + if (std::stoi(optarg) < 0 || std::stoi(optarg) > 3) { - throw std::runtime_error("band-mode must be either 0 for full bands, 1 for static bands or 2 for adaptive bands"); + throw std::runtime_error("band-mode must be either 0 for full bands, 1 for static bands, 2 for adaptive bands or 3 for static band with traceback"); } band_mode = static_cast(std::stoi(optarg)); break; diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index d71b0540a..01aded35f 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -43,19 +43,18 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa , band_mode(banding) , max_pred_distance_in_banded_mode(max_pred_dist > 0 ? max_pred_dist : 2 * cudautils::align(band_width)) { + max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); + if (banding == BandMode::full_band) { - max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); matrix_sequence_dimension = cudautils::align(max_sequence_size); } - else if (banding == BandMode::static_band) + else if (banding == BandMode::static_band || banding == BandMode::static_band_traceback) { - max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); matrix_sequence_dimension = cudautils::align(alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); } else // BandMode::adaptive_band { - max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); // adapive_storage_factor is to reserve extra memory for cases with extended band-width matrix_sequence_dimension = cudautils::align(adapive_storage_factor * (alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING)); } diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index dd13ccf1f..c4cac03ba 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -552,7 +552,7 @@ protected: int32_t matrix_width = (batch_size_.band_mode != BandMode::full_band) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); // in full-band, avail_buf_mem_ is dedicated to scores matrix and in static or adaptive band modes, avail_buf_mem_ is dedicated to traceback matrix size_t required_size = static_cast(matrix_width) * static_cast(matrix_height); - required_size *= batch_size_.band_mode == BandMode::full_band ? sizeof(ScoreT) : sizeof(TraceT); + required_size *= batch_size_.band_mode == BandMode::static_band_traceback ? sizeof(TraceT) : sizeof(ScoreT); if (required_size > avail_buf_mem_) { diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 92ac22b08..7812435f2 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -142,6 +142,12 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) TraceT* traceback = traceback_d; float banded_buffer_size; // using float instead of int64_t to minimize register if (BM == BandMode::adaptive_band || BM == BandMode::static_band) + { + banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); + int64_t scores_offset = static_cast(banded_buffer_size) * static_cast(window_idx); + scores = &scores_d[scores_offset]; + } + else if (BM == BandMode::static_band_traceback) { // buffer size for scores, in banded we only need to store part of the scores matrix banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); @@ -229,7 +235,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) __syncwarp(); - // Align each subsequent read, add alignment to graph, run topoligical sort. + // Align each subsequent read, add alignment to graph, run topological sort. for (int32_t s = 1; s < num_sequences; s++) { int32_t seq_len = sequence_lengths[s]; @@ -303,6 +309,26 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) } } else if (BM == BandMode::static_band) + { + alignment_length = runNeedlemanWunschBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score); + __syncwarp(); + } + else if (BM == BandMode::static_band_traceback) { alignment_length = runNeedlemanWunschBandedTraceBack(nodes, sorted_poa, @@ -539,12 +565,52 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode, - traceback); + batch_size.max_pred_distance_in_banded_mode); } else if (batch_size.band_mode == BandMode::adaptive_band) { generatePOAKernel + <<>>(consensus_d, + sequences_d, + base_weights_d, + sequence_lengths_d, + window_details_d, + total_windows, + scores, + alignment_graph, + alignment_read, + nodes, + incoming_edges, + incoming_edge_count, + outgoing_edges, + outgoing_edge_count, + incoming_edge_w, + sorted_poa, + node_id_to_pos, + node_alignments, + node_alignment_count, + sorted_poa_local_edge_count, + node_marks, + check_aligned_nodes, + nodes_to_visit, + node_coverage_counts, + gap_score, + mismatch_score, + match_score, + max_sequences_per_poa, + sequence_begin_nodes_ids, + outgoing_edges_coverage, + outgoing_edges_coverage_count, + max_nodes_per_graph, + matrix_seq_dimension, + batch_size.max_consensus_size, + TPB, + batch_size.alignment_band_width, + batch_size.max_pred_distance_in_banded_mode); + } + else if (batch_size.band_mode == BandMode::static_band_traceback) + { + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -665,12 +731,52 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode, - traceback); + batch_size.max_pred_distance_in_banded_mode); } else if (batch_size.band_mode == BandMode::adaptive_band) { generatePOAKernel + <<>>(consensus_d, + sequences_d, + base_weights_d, + sequence_lengths_d, + window_details_d, + total_windows, + scores, + alignment_graph, + alignment_read, + nodes, + incoming_edges, + incoming_edge_count, + outgoing_edges, + outgoing_edge_count, + incoming_edge_w, + sorted_poa, + node_id_to_pos, + node_alignments, + node_alignment_count, + sorted_poa_local_edge_count, + node_marks, + check_aligned_nodes, + nodes_to_visit, + node_coverage_counts, + gap_score, + mismatch_score, + match_score, + max_sequences_per_poa, + sequence_begin_nodes_ids, + outgoing_edges_coverage, + outgoing_edges_coverage_count, + max_nodes_per_graph, + matrix_seq_dimension, + batch_size.max_consensus_size, + TPB, + batch_size.alignment_band_width, + batch_size.max_pred_distance_in_banded_mode); + } + else if (batch_size.band_mode == BandMode::static_band_traceback) + { + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, From c1f69e9d1a9b2e28d96d3b44e822721e69337bda Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 24 Sep 2020 19:40:14 -0400 Subject: [PATCH 153/281] [cudapoa-backtracking] removed unnecessary checks from NW horizontal loop --- cudapoa/src/cudapoa_nw.cuh | 25 +++---------------- cudapoa/src/cudapoa_nw_adaptive_banded.cuh | 25 +++---------------- cudapoa/src/cudapoa_nw_banded.cuh | 25 +++---------------- cudapoa/src/cudapoa_nw_tb_banded.cuh | 20 ++++++--------- pygenomeworks/genomeworks/cudapoa/cudapoa.pyx | 3 ++- 5 files changed, 21 insertions(+), 77 deletions(-) diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index 7fb6e7dd5..d30455624 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -285,28 +285,11 @@ __device__ __forceinline__ last_score = first_element_prev_score; } - int32_t tscore = max(last_score + gap_score, score.s0); - if (tscore > score.s0) - { - score.s0 = tscore; - loop = true; - } - - tscore = max(score.s0 + gap_score, score.s1); - if (tscore > score.s1) - { - score.s1 = tscore; - loop = true; - } - - tscore = max(score.s1 + gap_score, score.s2); - if (tscore > score.s2) - { - score.s2 = tscore; - loop = true; - } + score.s0 = max(last_score + gap_score, score.s0); + score.s1 = max(score.s0 + gap_score, score.s1); + score.s2 = max(score.s1 + gap_score, score.s2); - tscore = max(score.s2 + gap_score, score.s3); + int32_t tscore = max(score.s2 + gap_score, score.s3); if (tscore > score.s3) { score.s3 = tscore; diff --git a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh index 05fd31199..5b7f1a805 100644 --- a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh +++ b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh @@ -359,28 +359,11 @@ __device__ __forceinline__ last_score = first_element_prev_score; } - int32_t tscore = max(last_score + gap_score, score.s0); - if (tscore > score.s0) - { - score.s0 = tscore; - loop = true; - } - - tscore = max(score.s0 + gap_score, score.s1); - if (tscore > score.s1) - { - score.s1 = tscore; - loop = true; - } - - tscore = max(score.s1 + gap_score, score.s2); - if (tscore > score.s2) - { - score.s2 = tscore; - loop = true; - } + score.s0 = max(last_score + gap_score, score.s0); + score.s1 = max(score.s0 + gap_score, score.s1); + score.s2 = max(score.s1 + gap_score, score.s2); - tscore = max(score.s2 + gap_score, score.s3); + int32_t tscore = max(score.s2 + gap_score, score.s3); if (tscore > score.s3) { score.s3 = tscore; diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 2cb70d06c..4970d35d9 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -314,28 +314,11 @@ __device__ __forceinline__ last_score = first_element_prev_score; } - int32_t tscore = max(last_score + gap_score, score.s0); - if (tscore > score.s0) - { - score.s0 = tscore; - loop = true; - } - - tscore = max(score.s0 + gap_score, score.s1); - if (tscore > score.s1) - { - score.s1 = tscore; - loop = true; - } - - tscore = max(score.s1 + gap_score, score.s2); - if (tscore > score.s2) - { - score.s2 = tscore; - loop = true; - } + score.s0 = max(last_score + gap_score, score.s0); + score.s1 = max(score.s0 + gap_score, score.s1); + score.s2 = max(score.s1 + gap_score, score.s2); - tscore = max(score.s2 + gap_score, score.s3); + int32_t tscore = max(score.s2 + gap_score, score.s3); if (tscore > score.s3) { score.s3 = tscore; diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index 4a7d001f7..bd79b6714 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -454,31 +454,25 @@ __device__ __forceinline__ last_score = first_element_prev_score; } - int32_t tscore = max(last_score + gap_score, score.s0); - if (tscore > score.s0) + if (score.s0 < last_score + gap_score) { - score.s0 = tscore; + score.s0 = last_score + gap_score; trace.t0 = 0; - loop = true; } - tscore = max(score.s0 + gap_score, score.s1); - if (tscore > score.s1) + if (score.s1 < score.s0 + gap_score) { - score.s1 = tscore; + score.s1 = score.s0 + gap_score; trace.t1 = 0; - loop = true; } - tscore = max(score.s1 + gap_score, score.s2); - if (tscore > score.s2) + if (score.s2 < score.s1 + gap_score) { - score.s2 = tscore; + score.s2 = score.s1 + gap_score; trace.t2 = 0; - loop = true; } - tscore = max(score.s2 + gap_score, score.s3); + int32_t tscore = max(score.s2 + gap_score, score.s3); if (tscore > score.s3) { score.s3 = tscore; diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx index a6f21fbbf..c78a638a7 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx @@ -159,7 +159,8 @@ cdef class CudaPoaBatch: raise RuntimeError("Unknown band_mode provided. Must be full_band/static_band/adaptive_band.") self.batch_size = make_unique[cudapoa.BatchConfig]( - mx_seq_sz, mx_consensus_sz, mx_nodes_per_poa, band_width_sz, mx_seq_per_poa, matrix_seq_dim, batch_band_mode, mx_pred_dist) + mx_seq_sz, mx_consensus_sz, mx_nodes_per_poa, band_width_sz, mx_seq_per_poa, matrix_seq_dim, + batch_band_mode, mx_pred_dist) self.batch = cudapoa.create_batch( device_id, From 6f08dd5368813cef3f5370049a6972414432aa72 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 25 Sep 2020 10:44:32 -0700 Subject: [PATCH 154/281] [cudaextender] char to int8_t and reformat --- CMakeLists.txt | 2 +- .../genomeworks/cudaextender/extender.hpp | 17 +++++++++----- .../genomeworks/cudaextender/utils.hpp | 22 +++++++++---------- cudaextender/samples/sample_cudaextender.cpp | 10 +++++---- cudaextender/src/ungapped_xdrop.cu | 14 ++++++------ cudaextender/src/ungapped_xdrop.cuh | 14 ++++++------ cudaextender/src/ungapped_xdrop_kernels.cu | 18 +++++++++------ cudaextender/src/ungapped_xdrop_kernels.cuh | 4 ++-- .../tests/Test_CudaextenderEnd2End.cu | 4 ++-- 9 files changed, 59 insertions(+), 46 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bf4006e2c..262dc10d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,7 +40,7 @@ option(gw_enable_cudapoa_nw_print "Enable verbose prints within cudapoa NW kerne option(gw_profiling "Compile a binary for profiling with NVTX markers." OFF) option(gw_enable_caching_allocator "Enable caching allocator." ON) option(gw_generate_docs "Generate Doxygen documentation" ON) -option(gw_cuda_gen_all_arch "ON: Generate optimized CUDA code for all architectures | OFF: for detected architectures only" OFF) +option(gw_cuda_gen_all_arch "ON: Generate optimized CUDA code for all architectures | OFF: for detected architectures only" ON) # Must be included before others for options value validation include(cmake/Utils.cmake) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index 452b7bcfe..b211b3a47 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -79,8 +79,8 @@ class Extender /// \param[in] h_seed_pairs Vector of seed pairs mapping to query and target /// sequences /// \return Status of the async extension launch - virtual StatusType extend_async(const char* h_query, const int32_t& query_length, - const char* h_target, const int32_t& target_length, + virtual StatusType extend_async(const int8_t* h_query, const int32_t& query_length, + const int8_t* h_target, const int32_t& target_length, const int32_t& score_threshold, const std::vector& h_seed_pairs) = 0; @@ -105,8 +105,8 @@ class Extender /// \param[out] d_num_scored_segment_pairs Pointer to pre-allocated device location for /// storing length of extension output /// \return Status of the async extension launch - virtual StatusType extend_async(const char* d_query, int32_t query_length, - const char* d_target, int32_t target_length, + virtual StatusType extend_async(const int8_t* d_query, int32_t query_length, + const int8_t* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) = 0; @@ -144,7 +144,14 @@ class Extender /// \param allocator DeviceAllocator to be used for allocating/freeing memory /// \param type Type of extension to be performed /// \return Unique pointer to Extender object. -std::unique_ptr create_extender(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator, const ExtensionType type = ExtensionType::ungapped_xdrop); +std::unique_ptr create_extender(const int32_t* h_score_mat, + const int32_t score_mat_dim, + const int32_t xdrop_threshold, + const bool no_entropy, + cudaStream_t stream, + const int32_t device_id, + DefaultDeviceAllocator allocator, + const ExtensionType type = ExtensionType::ungapped_xdrop); /// \} } // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp index 7355d7a8c..91bfec640 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp @@ -32,16 +32,16 @@ namespace genomeworks namespace cudaextender { -constexpr char A_NT = 0; -constexpr char C_NT = 1; -constexpr char G_NT = 2; -constexpr char T_NT = 3; -constexpr char L_NT = 4; -constexpr char N_NT = 5; -constexpr char X_NT = 6; -constexpr char E_NT = 7; -constexpr char NUC = 8; -constexpr char NUC2 = NUC * NUC; +constexpr int8_t A_NT = 0; +constexpr int8_t C_NT = 1; +constexpr int8_t G_NT = 2; +constexpr int8_t T_NT = 3; +constexpr int8_t L_NT = 4; +constexpr int8_t N_NT = 5; +constexpr int8_t X_NT = 6; +constexpr int8_t E_NT = 7; +constexpr int8_t NUC = 8; +constexpr int8_t NUC2 = NUC * NUC; /// \brief Parses seed pairs from a csv file in the following format: /// target_position_in_read_1, query_position_in_read_1 @@ -104,7 +104,7 @@ inline void parse_scored_segment_pairs(const std::string& filepath, std::vector< /// \param[out] dst_seq Pointer to pre-allocated storage for encoded sequence /// \param[in] src_seq Pointer to input sequence /// \param[in] length Length of the sequence -inline void encode_sequence(char* dst_seq, const char* src_seq, const int32_t& length) +inline void encode_sequence(int8_t* dst_seq, const char* src_seq, const int32_t& length) { for (int32_t i = 0; i < length; i++) { diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index 08a6518ac..d22e44784 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -108,8 +108,8 @@ int main(int argc, char* argv[]) -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; // Allocate pinned memory for query and target strings - pinned_host_vector h_encoded_target(target_sequence.length()); - pinned_host_vector h_encoded_query(target_sequence.length()); + pinned_host_vector h_encoded_target(target_sequence.length()); + pinned_host_vector h_encoded_query(target_sequence.length()); encode_sequence(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); encode_sequence(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); @@ -144,8 +144,8 @@ int main(int argc, char* argv[]) // Allocate space on device for target and query sequences, seed_pairs, // scored segment pairs (ssp) and num_ssp using default allocator (caching) // Allocate space for query and target sequences - device_buffer d_query(query_sequence.length(), allocator, stream0.get()); - device_buffer d_target(target_sequence.length(), allocator, stream0.get()); + device_buffer d_query(query_sequence.length(), allocator, stream0.get()); + device_buffer d_target(target_sequence.length(), allocator, stream0.get()); // Allocate space for SeedPair input device_buffer d_seed_pairs(h_seed_pairs.size(), allocator, stream0.get()); // Allocate space for ScoredSegmentPair output @@ -177,6 +177,8 @@ int main(int argc, char* argv[]) } std::cerr << "Number of Scored Segment Pairs found: " << h_ssp.size() << std::endl; if (print) + { print_scored_segment_pairs(h_ssp); + } return 0; } diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 360c881eb..92f0361d0 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -72,14 +72,14 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat d_score_mat_ = device_buffer(score_mat_dim_, allocator_, stream_); d_done_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); d_tmp_ssp_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); - d_temp_storage_cub_ = device_buffer(cub_storage_bytes, allocator_, stream_); + d_temp_storage_cub_ = device_buffer(cub_storage_bytes, allocator_, stream_); // Requires pinned host memory registration for proper async behavior device_copy_n(h_score_mat_, score_mat_dim_, d_score_mat_.data(), stream_); } -StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length, - const char* d_target, int32_t target_length, +StatusType UngappedXDrop::extend_async(const int8_t* d_query, int32_t query_length, + const int8_t* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) @@ -148,8 +148,8 @@ StatusType UngappedXDrop::extend_async(const char* d_query, int32_t query_length return StatusType::success; } -StatusType UngappedXDrop::extend_async(const char* h_query, const int32_t& query_length, - const char* h_target, const int32_t& target_length, +StatusType UngappedXDrop::extend_async(const int8_t* h_query, const int32_t& query_length, + const int8_t* h_target, const int32_t& target_length, const int32_t& score_threshold, const std::vector& h_seed_pairs) { @@ -158,8 +158,8 @@ StatusType UngappedXDrop::extend_async(const char* h_query, const int32_t& query // Set host pointer mode on host_ptr_api_mode_ = true; // Allocate space for query and target sequences - d_query_ = device_buffer(query_length, allocator_, stream_); - d_target_ = device_buffer(target_length, allocator_, stream_); + d_query_ = device_buffer(query_length, allocator_, stream_); + d_target_ = device_buffer(target_length, allocator_, stream_); // Allocate space for SeedPair input d_seed_pairs_ = device_buffer(h_seed_pairs.size(), allocator_, stream_); // Allocate space for ScoredSegmentPair output diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index b4726f4fc..002c035bf 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -34,13 +34,13 @@ public: UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator); ~UngappedXDrop() override; - StatusType extend_async(const char* h_query, const int32_t& query_length, - const char* h_target, const int32_t& target_length, + StatusType extend_async(const int8_t* h_query, const int32_t& query_length, + const int8_t* h_target, const int32_t& target_length, const int32_t& score_threshold, const std::vector& h_seed_pairs) override; - StatusType extend_async(const char* d_query, int32_t query_length, - const char* d_target, int32_t target_length, + StatusType extend_async(const int8_t* d_query, int32_t query_length, + const int8_t* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) override; @@ -66,12 +66,12 @@ private: device_buffer d_done_; // TODO- Rename scratch space device_buffer d_tmp_ssp_; // TODO- Rename Scratch space 2 int32_t total_scored_segment_pairs_; - device_buffer d_temp_storage_cub_; // temporary storage for cub functions + device_buffer d_temp_storage_cub_; // temporary storage for cub functions // Host ptr API additional required variables bool host_ptr_api_mode_; - device_buffer d_query_; - device_buffer d_target_; + device_buffer d_query_; + device_buffer d_target_; device_buffer d_seed_pairs_; device_buffer d_num_ssp_; device_buffer d_ssp_; diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 919ca7a44..3e5038b6b 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -27,9 +27,9 @@ namespace cudaextender // extend the seed values to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to score_threshold and update the d_scored_segment_pairs and d_done vectors -__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, +__global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_target, const int32_t target_length, - const char* __restrict__ d_query, + const int8_t* __restrict__ d_query, const int32_t query_length, const int32_t* d_sub_mat, const bool no_entropy, @@ -106,8 +106,8 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe const int32_t pos_offset = lane_id + tile[warp_id]; const int32_t ref_pos = ref_loc[warp_id] + pos_offset; const int32_t query_pos = query_loc[warp_id] + pos_offset; - char r_chr; - char q_chr; + int8_t r_chr; + int8_t q_chr; if (ref_pos < target_length && query_pos < query_length) { @@ -250,8 +250,8 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe { int32_t thread_score = 0; const int32_t pos_offset = lane_id + 1 + tile[warp_id]; - char r_chr; - char q_chr; + int8_t r_chr; + int8_t q_chr; if (ref_loc[warp_id] >= pos_offset && query_loc[warp_id] >= pos_offset) { @@ -430,7 +430,11 @@ __global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_targe } // Gather the SSPs from the resulting segments to the beginning of the tmp_ssp array -__global__ void compress_output(const int32_t* d_done, const int32_t start_index, const ScoredSegmentPair* d_ssp, ScoredSegmentPair* d_tmp_ssp, const int32_t num_seed_pairs) +__global__ void compress_output(const int32_t* d_done, + const int32_t start_index, + const ScoredSegmentPair* d_ssp, + ScoredSegmentPair* d_tmp_ssp, + const int32_t num_seed_pairs) { const int32_t stride = blockDim.x * gridDim.x; const int32_t start = blockDim.x * blockIdx.x + threadIdx.x; diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index 2ad9464e5..782fc6fc0 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -31,9 +31,9 @@ namespace cudaextender // extend the hits to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to score_threshold and update the d_scored_segment_pairs and d_done vectors -__global__ void find_high_scoring_segment_pairs(const char* __restrict__ d_target, +__global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_target, const int32_t target_length, - const char* __restrict__ d_query, + const int8_t* __restrict__ d_query, const int32_t query_length, const int32_t* d_sub_mat, const bool no_entropy, diff --git a/cudaextender/tests/Test_CudaextenderEnd2End.cu b/cudaextender/tests/Test_CudaextenderEnd2End.cu index b1fdc07d6..25cde2680 100644 --- a/cudaextender/tests/Test_CudaextenderEnd2End.cu +++ b/cudaextender/tests/Test_CudaextenderEnd2End.cu @@ -92,8 +92,8 @@ public: std::vector h_seed_pairs; parse_seed_pairs(param_.seed_pairs_file, h_seed_pairs); // Allocate pinned memory for query and target strings - pinned_host_vector h_encoded_target(target_sequence.length()); - pinned_host_vector h_encoded_query(target_sequence.length()); + pinned_host_vector h_encoded_target(target_sequence.length()); + pinned_host_vector h_encoded_query(target_sequence.length()); encode_sequence(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); encode_sequence(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); ungapped_extender_->extend_async(h_encoded_query.data(), h_encoded_query.size(), h_encoded_target.data(), h_encoded_target.size(), param_.score_threshold, h_seed_pairs); From e56c9c8b053ecf83d76db57d11c7ce58ef84f8dd Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 25 Sep 2020 17:13:41 -0400 Subject: [PATCH 155/281] [cudapoa] started unifying NW banded and NW banded adaptive, WIP --- cudapoa/src/cudapoa_nw_adaptive_banded.cuh | 109 ++++++++++++--------- 1 file changed, 60 insertions(+), 49 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh index 5b7f1a805..8ef4e8ce5 100644 --- a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh +++ b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh @@ -165,7 +165,8 @@ __device__ __forceinline__ void initialize_band_adaptive(ScoreT* scores, int32_t template + typename TraceT, + bool ADAPTIVE = false> __device__ __forceinline__ int32_t runNeedlemanWunschAdaptiveBanded(SeqT* nodes, @@ -203,48 +204,55 @@ __device__ __forceinline__ //--------------------------------------------------------- int32_t band_width = static_band_width; - if (gradient > 1.1) // ad-hoc rule 1.a + if (ADAPTIVE) { - // ad-hoc rule 1.b - band_width = max(band_width, cudautils::align(max_column * 0.08 * gradient)); - } - if (gradient < 0.8) // ad-hoc rule 2.a - { - // ad-hoc rule 2.b - band_width = max(band_width, cudautils::align(max_column * 0.1 / gradient)); + if (gradient > 1.1) // ad-hoc rule 1.a + { + // ad-hoc rule 1.b + band_width = max(band_width, cudautils::align(max_column * 0.08 * gradient)); + } + if (gradient < 0.8) // ad-hoc rule 2.a + { + // ad-hoc rule 2.b + band_width = max(band_width, cudautils::align(max_column * 0.1 / gradient)); + } + + // limit band-width for very large reads, ad-hoc rule 3 + ///ToDo add a check for cases where static_band_width > CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH + band_width = min(band_width, CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH); } - // limit band-width for very large reads, ad-hoc rule 3 - ///ToDo add a check for cases where static_band_width > CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH - band_width = min(band_width, CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH); // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 int32_t band_shift = band_width / 2; - // rerun code is defined in backtracking loop from previous alignment try - // -3 means traceback path was too close to the left bound of band - // -4 means traceback path was too close to the right bound of band - // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift - // further to the left for rerun == -3, and further to the right for rerun == -4. - if (rerun == -3) - { - // ad-hoc rule 5 - band_width *= 2; - band_shift *= 2.5; - } - if (rerun == -4) - { - // ad-hoc rule 6 - band_width *= 2; - band_shift *= 1.5; - } - //--------------------------------------------------------- - // check required memory and return error if exceeding max_buffer_size - // using float to avoid 64-bit - float required_buffer_size = static_cast(graph_count) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - if (required_buffer_size > max_buffer_size) + if(ADAPTIVE) { - return -2; + // rerun code is defined in backtracking loop from previous alignment try + // -3 means traceback path was too close to the left bound of band + // -4 means traceback path was too close to the right bound of band + // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift + // further to the left for rerun == -3, and further to the right for rerun == -4. + if (rerun == -3) + { + // ad-hoc rule 5 + band_width *= 2; + band_shift *= 2.5; + } + if (rerun == -4) + { + // ad-hoc rule 6 + band_width *= 2; + band_shift *= 1.5; + } + // check required memory and return error if exceeding max_buffer_size + // using float to avoid 64-bit + float required_buffer_size = static_cast(graph_count) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + if (required_buffer_size > max_buffer_size) + { + return -2; + } } + //--------------------------------------------------------- // Initialise the horizontal boundary of the score matrix, initialising of the vertical boundary is done within the main for loop for (int32_t j = lane_idx; j < band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) @@ -421,23 +429,26 @@ __device__ __forceinline__ // Check if move is diagonal. if (i != 0 && j != 0) { - if (rerun == 0) + if(ADAPTIVE) { - // check if traceback gets too close or hits the band limits, if so stop and rerun with extended band-width - // threshold for proximity to band limits works better if defined proportionate to the sequence length - int32_t threshold = max(1, max_column / 1024); // ad-hoc rule 7 - if (j > threshold && j < max_column - threshold) + if (rerun == 0) { - int32_t band_start = get_band_start_for_row_adaptive(i, gradient, band_width, band_shift, max_column); - if (j <= band_start + threshold) // ad-hoc rule 8-a, too close to left bound + // check if traceback gets too close or hits the band limits, if so stop and rerun with extended band-width + // threshold for proximity to band limits works better if defined proportionate to the sequence length + int32_t threshold = max(1, max_column / 1024); // ad-hoc rule 7 + if (j > threshold && j < max_column - threshold) { - aligned_nodes = -3; - break; - } - if (j >= (band_start + band_width - threshold)) // ad-hoc rule 8-b, too close to right bound - { - aligned_nodes = -4; - break; + int32_t band_start = get_band_start_for_row_adaptive(i, gradient, band_width, band_shift, max_column); + if (j <= band_start + threshold) // ad-hoc rule 8-a, too close to left bound + { + aligned_nodes = -3; + break; + } + if (j >= (band_start + band_width - threshold)) // ad-hoc rule 8-b, too close to right bound + { + aligned_nodes = -4; + break; + } } } } From 020585d73c6c6a6b201bafbd3f9d2d94bca734c9 Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 25 Sep 2020 18:53:58 -0400 Subject: [PATCH 156/281] [cudapoa] unifying NW banded and NW banded adaptive, removed cudapoa_nw_banded.cuh, renamed NW adaptive kernels, WIP --- cudapoa/src/cudapoa_kernels.cuh | 107 ++--- cudapoa/src/cudapoa_nw_adaptive_banded.cuh | 121 +++--- cudapoa/src/cudapoa_nw_banded.cuh | 476 --------------------- 3 files changed, 114 insertions(+), 590 deletions(-) delete mode 100644 cudapoa/src/cudapoa_nw_banded.cuh diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 7812435f2..496c10c64 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -18,7 +18,6 @@ #pragma once #include "cudapoa_nw.cuh" -#include "cudapoa_nw_banded.cuh" #include "cudapoa_nw_adaptive_banded.cuh" #include "cudapoa_nw_tb_banded.cuh" #include "cudapoa_topsort.cuh" @@ -263,69 +262,71 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) if (BM == BandMode::adaptive_band) { - alignment_length = runNeedlemanWunschAdaptiveBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score, - 0); + alignment_length = runNeedlemanWunschBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + 0); __syncwarp(); if (alignment_length < -2) { // rerun with extended band-width - alignment_length = runNeedlemanWunschAdaptiveBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score, - alignment_length); + alignment_length = runNeedlemanWunschBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + alignment_length); __syncwarp(); } } else if (BM == BandMode::static_band) { - alignment_length = runNeedlemanWunschBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score); + alignment_length = runNeedlemanWunschBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + alignment_length); __syncwarp(); } else if (BM == BandMode::static_band_traceback) diff --git a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh index 8ef4e8ce5..e14b5e25f 100644 --- a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh +++ b/cudapoa/src/cudapoa_nw_adaptive_banded.cuh @@ -33,7 +33,7 @@ namespace cudapoa { template -__device__ __forceinline__ ScoreT* get_score_ptr_adaptive(ScoreT* scores, int32_t row, int32_t column, int32_t band_start, int32_t band_width) +__device__ __forceinline__ ScoreT* get_score_ptr(ScoreT* scores, int32_t row, int32_t column, int32_t band_start, int32_t band_width) { column = column == -1 ? 0 : column - band_start; @@ -43,7 +43,7 @@ __device__ __forceinline__ ScoreT* get_score_ptr_adaptive(ScoreT* scores, int32_ } template -__device__ __forceinline__ void set_score_adaptive(ScoreT* scores, int32_t row, int32_t column, int32_t value, int32_t band_start, int32_t band_width) +__device__ __forceinline__ void set_score(ScoreT* scores, int32_t row, int32_t column, int32_t value, int32_t band_start, int32_t band_width) { if (column == -1) { @@ -59,7 +59,7 @@ __device__ __forceinline__ void set_score_adaptive(ScoreT* scores, int32_t row, scores[score_index] = value; } -__device__ __forceinline__ int32_t get_band_start_for_row_adaptive(int32_t row, float gradient, int32_t band_width, int32_t band_shift, int32_t max_column) +__device__ __forceinline__ int32_t get_band_start_for_row(int32_t row, float gradient, int32_t band_width, int32_t band_shift, int32_t max_column) { int32_t diagonal_index = int32_t(row * gradient); @@ -74,11 +74,11 @@ __device__ __forceinline__ int32_t get_band_start_for_row_adaptive(int32_t row, } template -__device__ __forceinline__ ScoreT get_score_adaptive(ScoreT* scores, int32_t row, int32_t column, - int32_t band_width, int32_t band_shift, float gradient, - int32_t max_column, const ScoreT min_score_value) +__device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t column, + int32_t band_width, int32_t band_shift, float gradient, + int32_t max_column, const ScoreT min_score_value) { - int32_t band_start = get_band_start_for_row_adaptive(row, gradient, band_width, band_shift, max_column); + int32_t band_start = get_band_start_for_row(row, gradient, band_width, band_shift, max_column); int32_t band_end = band_start + band_width; band_end = min(band_end, max_column); @@ -88,21 +88,21 @@ __device__ __forceinline__ ScoreT get_score_adaptive(ScoreT* scores, int32_t row } else { - return *get_score_ptr_adaptive(scores, row, column, band_start, band_width); + return *get_score_ptr(scores, row, column, band_start, band_width); } } template -__device__ __forceinline__ ScoreT4 get_scores_adaptive(ScoreT* scores, - int32_t row, - int32_t column, - int32_t band_width, - int32_t band_shift, - float gradient, - int32_t max_column, - ScoreT default_value, - int32_t gap_score, - ScoreT4& char_profile) +__device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, + int32_t row, + int32_t column, + int32_t band_width, + int32_t band_shift, + float gradient, + int32_t max_column, + ScoreT default_value, + int32_t gap_score, + ScoreT4& char_profile) { // The load instructions typically load data in 4B or 8B chunks. @@ -114,7 +114,7 @@ __device__ __forceinline__ ScoreT4 get_scores_adaptive(ScoreT* scores, // of the data using bit arithmatic. Also reduces register count. // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds - int32_t band_start = get_band_start_for_row_adaptive(row, gradient, band_width, band_shift, max_column); + int32_t band_start = get_band_start_for_row(row, gradient, band_width, band_shift, max_column); // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds int32_t band_end = band_start + band_width - CELLS_PER_THREAD; band_end = min(band_end, max_column); @@ -125,7 +125,7 @@ __device__ __forceinline__ ScoreT4 get_scores_adaptive(ScoreT* scores, } else { - ScoreT4* pred_scores = (ScoreT4*)get_score_ptr_adaptive(scores, row, column, band_start, band_width); + ScoreT4* pred_scores = (ScoreT4*)get_score_ptr(scores, row, column, band_start, band_width); // loads 8/16 consecutive bytes (4 ScoreTs) ScoreT4 score4 = pred_scores[0]; @@ -149,44 +149,43 @@ __device__ __forceinline__ ScoreT4 get_scores_adaptive(ScoreT* scores, } template -__device__ __forceinline__ void initialize_band_adaptive(ScoreT* scores, int32_t row, int32_t min_score_value, int32_t band_start, int32_t band_width, int32_t lane_idx) +__device__ __forceinline__ void initialize_band(ScoreT* scores, int32_t row, int32_t min_score_value, int32_t band_start, int32_t band_width, int32_t lane_idx) { int32_t band_end = band_start + band_width; band_start = max(1, band_start); - set_score_adaptive(scores, row, band_start, min_score_value, band_start, band_width); + set_score(scores, row, band_start, min_score_value, band_start, band_width); if (lane_idx < CUDAPOA_BANDED_MATRIX_RIGHT_PADDING) { - set_score_adaptive(scores, row, lane_idx + band_end, min_score_value, band_start, band_width); + set_score(scores, row, lane_idx + band_end, min_score_value, band_start, band_width); } } template + bool ADAPTIVE = true> __device__ __forceinline__ int32_t - runNeedlemanWunschAdaptiveBanded(SeqT* nodes, - SizeT* graph, - SizeT* node_id_to_pos, - int32_t graph_count, - uint16_t* incoming_edge_count, - SizeT* incoming_edges, - uint16_t* outgoing_edge_count, - SeqT* read, - int32_t read_length, - ScoreT* scores, - float max_buffer_size, - SizeT* alignment_graph, - SizeT* alignment_read, - int32_t static_band_width, - int32_t gap_score, - int32_t mismatch_score, - int32_t match_score, - int32_t rerun) + runNeedlemanWunschBanded(SeqT* nodes, + SizeT* graph, + SizeT* node_id_to_pos, + int32_t graph_count, + uint16_t* incoming_edge_count, + SizeT* incoming_edges, + uint16_t* outgoing_edge_count, + SeqT* read, + int32_t read_length, + ScoreT* scores, + float max_buffer_size, + SizeT* alignment_graph, + SizeT* alignment_read, + int32_t static_band_width, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, + int32_t rerun) { // in adaptive bands, there can be cases where multiple rows happen to have a band with start index // smaller than band-start index of a row above. If min_value is too close to numeric_limits::min(), @@ -225,7 +224,7 @@ __device__ __forceinline__ // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 int32_t band_shift = band_width / 2; - if(ADAPTIVE) + if (ADAPTIVE) { // rerun code is defined in backtracking loop from previous alignment try // -3 means traceback path was too close to the left bound of band @@ -274,10 +273,10 @@ __device__ __forceinline__ { int32_t node_id = graph[graph_pos]; int32_t score_gIdx = graph_pos + 1; - int32_t band_start = get_band_start_for_row_adaptive(score_gIdx, gradient, band_width, band_shift, max_column); + int32_t band_start = get_band_start_for_row(score_gIdx, gradient, band_width, band_shift, max_column); int32_t pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]; - initialize_band_adaptive(scores, score_gIdx, min_score_value, band_start, band_width, lane_idx); + initialize_band(scores, score_gIdx, min_score_value, band_start, band_width, lane_idx); int32_t first_element_prev_score = 0; @@ -290,7 +289,7 @@ __device__ __forceinline__ pred_count = incoming_edge_count[node_id]; if (pred_count == 0) { - set_score_adaptive(scores, score_gIdx, -1, gap_score, band_start, band_width); + set_score(scores, score_gIdx, -1, gap_score, band_start, band_width); } else { @@ -301,17 +300,17 @@ __device__ __forceinline__ } else { - penalty = max(min_score_value, get_score_adaptive(scores, pred_idx, -1, band_width, band_shift, gradient, max_column, min_score_value)); + penalty = max(min_score_value, get_score(scores, pred_idx, -1, band_width, band_shift, gradient, max_column, min_score_value)); // if pred_num > 1 keep checking to find max score as penalty for (int32_t p = 0; p < pred_count; p++) { pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; - penalty = max(penalty, get_score_adaptive(scores, pred_idx_tmp, -1, band_width, band_shift, gradient, max_column, min_score_value)); + penalty = max(penalty, get_score(scores, pred_idx_tmp, -1, band_width, band_shift, gradient, max_column, min_score_value)); } first_element_prev_score = penalty + gap_score; } - set_score_adaptive(scores, score_gIdx, -1, first_element_prev_score, band_start, band_width); + set_score(scores, score_gIdx, -1, first_element_prev_score, band_start, band_width); } } pred_count = __shfl_sync(FULL_MASK, pred_count, 0); @@ -331,14 +330,14 @@ __device__ __forceinline__ char_profile.s2 = (graph_base == read4.r2 ? match_score : mismatch_score); char_profile.s3 = (graph_base == read4.r3 ? match_score : mismatch_score); - ScoreT4 score = get_scores_adaptive(scores, pred_idx, read_pos, band_width, band_shift, gradient, max_column, min_score_value, gap_score, char_profile); + ScoreT4 score = get_scores(scores, pred_idx, read_pos, band_width, band_shift, gradient, max_column, min_score_value, gap_score, char_profile); // Perform same score updates as above, but for rest of predecessors. for (int32_t p = 1; p < pred_count; p++) { int32_t pred_idx2 = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; - ScoreT4 scores_4 = get_scores_adaptive(scores, pred_idx2, read_pos, band_width, band_shift, gradient, max_column, min_score_value, gap_score, char_profile); + ScoreT4 scores_4 = get_scores(scores, pred_idx2, read_pos, band_width, band_shift, gradient, max_column, min_score_value, gap_score, char_profile); score.s0 = max(score.s0, scores_4.s0); score.s1 = max(score.s1, scores_4.s1); @@ -406,7 +405,7 @@ __device__ __forceinline__ { if (outgoing_edge_count[graph[idx - 1]] == 0) { - int32_t s = get_score_adaptive(scores, idx, j, band_width, band_shift, gradient, max_column, min_score_value); + int32_t s = get_score(scores, idx, j, band_width, band_shift, gradient, max_column, min_score_value); if (mscore < s) { mscore = s; @@ -424,12 +423,12 @@ __device__ __forceinline__ while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) { loop_count++; - int32_t scores_ij = get_score_adaptive(scores, i, j, band_width, band_shift, gradient, max_column, min_score_value); + int32_t scores_ij = get_score(scores, i, j, band_width, band_shift, gradient, max_column, min_score_value); bool pred_found = false; // Check if move is diagonal. if (i != 0 && j != 0) { - if(ADAPTIVE) + if (ADAPTIVE) { if (rerun == 0) { @@ -438,7 +437,7 @@ __device__ __forceinline__ int32_t threshold = max(1, max_column / 1024); // ad-hoc rule 7 if (j > threshold && j < max_column - threshold) { - int32_t band_start = get_band_start_for_row_adaptive(i, gradient, band_width, band_shift, max_column); + int32_t band_start = get_band_start_for_row(i, gradient, band_width, band_shift, max_column); if (j <= band_start + threshold) // ad-hoc rule 8-a, too close to left bound { aligned_nodes = -3; @@ -459,7 +458,7 @@ __device__ __forceinline__ uint16_t pred_count = incoming_edge_count[node_id]; int32_t pred_i = (pred_count == 0 ? 0 : (node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]] + 1)); - if (scores_ij == (get_score_adaptive(scores, pred_i, j - 1, band_width, band_shift, gradient, max_column, min_score_value) + match_cost)) + if (scores_ij == (get_score(scores, pred_i, j - 1, band_width, band_shift, gradient, max_column, min_score_value) + match_cost)) { prev_i = pred_i; prev_j = j - 1; @@ -472,7 +471,7 @@ __device__ __forceinline__ { pred_i = (node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1); - if (scores_ij == (get_score_adaptive(scores, pred_i, j - 1, band_width, band_shift, gradient, max_column, min_score_value) + match_cost)) + if (scores_ij == (get_score(scores, pred_i, j - 1, band_width, band_shift, gradient, max_column, min_score_value) + match_cost)) { prev_i = pred_i; prev_j = j - 1; @@ -490,7 +489,7 @@ __device__ __forceinline__ uint16_t pred_count = incoming_edge_count[node_id]; int32_t pred_i = (pred_count == 0 ? 0 : node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]] + 1); - if (scores_ij == get_score_adaptive(scores, pred_i, j, band_width, band_shift, gradient, max_column, min_score_value) + gap_score) + if (scores_ij == get_score(scores, pred_i, j, band_width, band_shift, gradient, max_column, min_score_value) + gap_score) { prev_i = pred_i; prev_j = j; @@ -503,7 +502,7 @@ __device__ __forceinline__ { pred_i = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; - if (scores_ij == get_score_adaptive(scores, pred_i, j, band_width, band_shift, gradient, max_column, min_score_value) + gap_score) + if (scores_ij == get_score(scores, pred_i, j, band_width, band_shift, gradient, max_column, min_score_value) + gap_score) { prev_i = pred_i; prev_j = j; @@ -515,7 +514,7 @@ __device__ __forceinline__ } // Check if move is horizontal. - if (!pred_found && scores_ij == get_score_adaptive(scores, i, j - 1, band_width, band_shift, gradient, max_column, min_score_value) + gap_score) + if (!pred_found && scores_ij == get_score(scores, i, j - 1, band_width, band_shift, gradient, max_column, min_score_value) + gap_score) { prev_i = i; prev_j = j - 1; diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh deleted file mode 100644 index 4970d35d9..000000000 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ /dev/null @@ -1,476 +0,0 @@ -/* -* Copyright 2019-2020 NVIDIA CORPORATION. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -#pragma once - -#include "cudapoa_structs.cuh" - -#include -#include - -#include - -namespace claraparabricks -{ - -namespace genomeworks -{ - -namespace cudapoa -{ - -__device__ __forceinline__ int32_t get_band_start_for_row(int32_t row_idx, float gradient, int32_t band_width, int32_t max_column) -{ - - int32_t start_pos = int32_t(row_idx * gradient) - band_width / 2; - - start_pos = max(start_pos, 0); - - int32_t end_pos = start_pos + band_width; - - if (end_pos > max_column) - { - start_pos = max_column - band_width + CELLS_PER_THREAD; - }; - - start_pos = max(start_pos, 0); - - start_pos = start_pos - (start_pos % CELLS_PER_THREAD); - - return start_pos; -} - -template -__device__ __forceinline__ ScoreT* get_score_ptr(ScoreT* scores, int32_t row, int32_t column, int32_t band_start, int32_t band_width) -{ - column = column == -1 ? 0 : column - band_start; - - int64_t score_index = static_cast(column) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - - return &scores[score_index]; -}; - -template -__device__ __forceinline__ void set_score(ScoreT* scores, int32_t row, int32_t column, int32_t value, float gradient, int32_t band_width, int32_t max_column) -{ - int32_t band_start = get_band_start_for_row(row, gradient, band_width, max_column); - - int32_t col_idx; - if (column == -1) - { - col_idx = band_start; - } - else - { - col_idx = column - band_start; - } - - int64_t score_index = static_cast(col_idx) + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - scores[score_index] = value; -} - -template -__device__ __forceinline__ void initialize_band(ScoreT* scores, int32_t row, int32_t value, float gradient, int32_t band_width, int32_t max_column, int32_t lane_idx) -{ - int32_t band_start = get_band_start_for_row(row, gradient, band_width, max_column); - int32_t band_end = band_start + band_width; - - int32_t initialization_offset = (band_start == 0) ? 1 : band_start; - - set_score(scores, row, initialization_offset, value, gradient, band_width, max_column); - - // note: as long as CUDAPOA_BANDED_MATRIX_RIGHT_PADDING < WARP_SIZE, no need for a for loop - for (int32_t j = lane_idx + band_end; j < band_end + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) - { - set_score(scores, row, j, value, gradient, band_width, max_column); - } -} - -template -__device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t column, float gradient, int32_t band_width, int32_t max_column, const ScoreT min_score_value) -{ - int32_t band_start = get_band_start_for_row(row, gradient, band_width, max_column); - int32_t band_end = band_start + band_width; - band_end = min(band_end, max_column); - - if (((column > band_end) || (column < band_start)) && column != -1) - { - return min_score_value; - } - else - { - return *get_score_ptr(scores, row, column, band_start, band_width); - } -} - -template -__device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, - int32_t node, - int32_t read_pos, - float gradient, - int32_t band_width, - int32_t max_column, - ScoreT default_value, - int32_t gap_score, - ScoreT4& char_profile) -{ - - // The load instructions typically load data in 4B or 8B chunks. - // If data is 16b (2B), then a 4B load chunk is loaded into register - // and the necessary bits are extracted before returning. This wastes cycles - // as each read of 16b issues a separate load command. - // Instead it is better to load a 4B or 8B chunk into a register - // using a single load inst, and then extracting necessary part of - // of the data using bit arithmatic. Also reduces register count. - - int32_t band_start = get_band_start_for_row(node, gradient, band_width, max_column); - - // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds - int32_t band_end = static_cast(band_start + band_width - CELLS_PER_THREAD); - - if ((read_pos > band_end || read_pos < band_start) && read_pos != -1) - { - return ScoreT4{default_value, default_value, default_value, default_value}; - } - else - { - ScoreT4* pred_scores = (ScoreT4*)get_score_ptr(scores, node, read_pos, band_start, band_width); - - // loads 8/16 consecutive bytes (4 ScoreT) - ScoreT4 score4 = pred_scores[0]; - - // need to load the next chunk of memory as well - ScoreT4 score4_next = pred_scores[1]; - - ScoreT4 score; - - score.s0 = max(score4.s0 + char_profile.s0, - score4.s1 + gap_score); - score.s1 = max(score4.s1 + char_profile.s1, - score4.s2 + gap_score); - score.s2 = max(score4.s2 + char_profile.s2, - score4.s3 + gap_score); - score.s3 = max(score4.s3 + char_profile.s3, - score4_next.s0 + gap_score); - - return score; - } -} - -template -__device__ __forceinline__ - int32_t - runNeedlemanWunschBanded(SeqT* nodes, - SizeT* graph, - SizeT* node_id_to_pos, - int32_t graph_count, - uint16_t* incoming_edge_count, - SizeT* incoming_edges, - uint16_t* outgoing_edge_count, - SeqT* read, - int32_t read_length, - ScoreT* scores, - SizeT* alignment_graph, - SizeT* alignment_read, - int32_t band_width, - int32_t gap_score, - int32_t mismatch_score, - int32_t match_score) -{ - - const ScoreT min_score_value = 2 * abs(min(min(gap_score, mismatch_score), -match_score) - 1) + numeric_limits::min(); - - int32_t lane_idx = threadIdx.x % WARP_SIZE; - - //Calculate gradient for the scores matrix - float gradient = float(read_length + 1) / float(graph_count + 1); - - int32_t max_column = read_length + 1; - // Initialise the horizontal boundary of the score matrix - for (int32_t j = lane_idx; j < band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) - { - set_score(scores, 0, j, j * gap_score, gradient, band_width, max_column); - } - - if (lane_idx == 0) - { -#ifdef NW_VERBOSE_PRINT - printf("graph %d, read %d\n", graph_count, read_length); -#endif - } - - __syncwarp(); - - // compute vertical and diagonal values in parallel. - for (int32_t graph_pos = 0; graph_pos < graph_count; graph_pos++) - { - int32_t node_id = graph[graph_pos]; - int32_t score_gIdx = graph_pos + 1; - int32_t band_start = get_band_start_for_row(score_gIdx, gradient, band_width, max_column); - int32_t pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]; - - initialize_band(scores, score_gIdx, min_score_value, gradient, band_width, max_column, lane_idx); - - int32_t first_element_prev_score = 0; - uint16_t pred_count = 0; - int32_t pred_idx = 0; - - if (lane_idx == 0) - { - // Initialise the vertical boundary of the score matrix - int32_t penalty; - pred_count = incoming_edge_count[node_id]; - if (pred_count == 0) - { - set_score(scores, score_gIdx, -1, gap_score, gradient, band_width, max_column); - } - else - { - pred_idx = node_id_to_pos[pred_node_id] + 1; - if (band_start > CELLS_PER_THREAD && pred_count == 1) - { - first_element_prev_score = min_score_value + gap_score; - } - else - { - penalty = max(min_score_value, get_score(scores, pred_idx, -1, gradient, band_width, max_column, min_score_value)); - // if pred_num > 1 keep checking to find max score as penalty - for (int32_t p = 0; p < pred_count; p++) - { - pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; - int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; - penalty = max(penalty, get_score(scores, pred_idx_tmp, -1, gradient, band_width, max_column, min_score_value)); - } - first_element_prev_score = penalty + gap_score; - } - set_score(scores, score_gIdx, -1, first_element_prev_score, gradient, band_width, max_column); - } - } - pred_count = __shfl_sync(FULL_MASK, pred_count, 0); - pred_idx = __shfl_sync(FULL_MASK, pred_idx, 0); - //------------------------------------------------------------- - - SeqT graph_base = nodes[node_id]; - - for (int32_t read_pos = lane_idx * CELLS_PER_THREAD + band_start; read_pos < band_start + band_width; read_pos += CUDAPOA_MIN_BAND_WIDTH) - { - SeqT4* d_read4 = (SeqT4*)read; - SeqT4 read4 = d_read4[read_pos / CELLS_PER_THREAD]; - - ScoreT4 char_profile; - char_profile.s0 = (graph_base == read4.r0 ? match_score : mismatch_score); - char_profile.s1 = (graph_base == read4.r1 ? match_score : mismatch_score); - char_profile.s2 = (graph_base == read4.r2 ? match_score : mismatch_score); - char_profile.s3 = (graph_base == read4.r3 ? match_score : mismatch_score); - - ScoreT4 score = get_scores(scores, pred_idx, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile); - - // Perform same score updates as above, but for rest of predecessors. - for (int32_t p = 1; p < pred_count; p++) - { - int32_t pred_idx2 = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; - - ScoreT4 scores_4 = get_scores(scores, pred_idx2, read_pos, gradient, band_width, max_column, min_score_value, gap_score, char_profile); - - score.s0 = max(score.s0, scores_4.s0); - score.s1 = max(score.s1, scores_4.s1); - score.s2 = max(score.s2, scores_4.s2); - score.s3 = max(score.s3, scores_4.s3); - } - - // While there are changes to the horizontal score values, keep updating the matrix. - // So loop will only run the number of time there are corrections in the matrix. - // The any_sync warp primitive lets us easily check if any of the threads had an update. - bool loop = true; - - while (__any_sync(FULL_MASK, loop)) - { - loop = false; - // Note: computation of s3 depends on s2, s2 depends on s1 and s1 on s0. - // If we reverse the order of computation in this loop from s3 to s0, it will increase - // ILP. However, in longer reads where indels are more frequent, this reverse computations - // results in larger number of iterations. Since if s0 is changed, value of s1, s2 and s3 which - // already have been computed in parallel need to be updated again. - - // The shfl_up lets us grab a value from the lane below. - int32_t last_score = __shfl_up_sync(FULL_MASK, score.s3, 1); - if (lane_idx == 0) - { - last_score = first_element_prev_score; - } - - score.s0 = max(last_score + gap_score, score.s0); - score.s1 = max(score.s0 + gap_score, score.s1); - score.s2 = max(score.s1 + gap_score, score.s2); - - int32_t tscore = max(score.s2 + gap_score, score.s3); - if (tscore > score.s3) - { - score.s3 = tscore; - loop = true; - } - } - - // Copy over the last element score of the last lane into a register of first lane - // which can be used to compute the first cell of the next warp. - first_element_prev_score = __shfl_sync(FULL_MASK, score.s3, WARP_SIZE - 1); - - int64_t score_index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - - scores[score_index] = score.s0; - scores[score_index + 1L] = score.s1; - scores[score_index + 2L] = score.s2; - scores[score_index + 3L] = score.s3; - - __syncwarp(); - } - } - - int32_t aligned_nodes = 0; - if (lane_idx == 0) - { - // Find location of the maximum score in the matrix. - int32_t i = 0; - int32_t j = read_length; - int32_t mscore = min_score_value; - - for (int32_t idx = 1; idx <= graph_count; idx++) - { - if (outgoing_edge_count[graph[idx - 1]] == 0) - { - int32_t s = get_score(scores, idx, j, gradient, band_width, max_column, min_score_value); - if (mscore < s) - { - mscore = s; - i = idx; - } - } - } - - // Fill in traceback - int32_t prev_i = 0; - int32_t prev_j = 0; - int32_t next_node_id = i > 0 ? graph[i - 1] : 0; - - int32_t loop_count = 0; - while (!(i == 0 && j == 0) && loop_count < static_cast(read_length + graph_count + 2)) - { - loop_count++; - int32_t scores_ij = get_score(scores, i, j, gradient, band_width, max_column, min_score_value); - bool pred_found = false; - // Check if move is diagonal. - if (i != 0 && j != 0) - { - - int32_t node_id = next_node_id; - - int32_t match_cost = (nodes[node_id] == read[j - 1] ? match_score : mismatch_score); - uint16_t pred_count = incoming_edge_count[node_id]; - int32_t pred_i = (pred_count == 0 ? 0 : (node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]] + 1)); - - if (scores_ij == (get_score(scores, pred_i, j - 1, gradient, band_width, max_column, min_score_value) + match_cost)) - { - prev_i = pred_i; - prev_j = j - 1; - pred_found = true; - } - - if (!pred_found) - { - for (int32_t p = 1; p < pred_count; p++) - { - pred_i = (node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1); - - if (scores_ij == (get_score(scores, pred_i, j - 1, gradient, band_width, max_column, min_score_value) + match_cost)) - { - prev_i = pred_i; - prev_j = j - 1; - pred_found = true; - break; - } - } - } - } - - // Check if move is vertical. - if (!pred_found && i != 0) - { - int32_t node_id = graph[i - 1]; - uint16_t pred_count = incoming_edge_count[node_id]; - int32_t pred_i = (pred_count == 0 ? 0 : node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]] + 1); - - if (scores_ij == get_score(scores, pred_i, j, gradient, band_width, max_column, min_score_value) + gap_score) - { - prev_i = pred_i; - prev_j = j; - pred_found = true; - } - - if (!pred_found) - { - for (int32_t p = 1; p < pred_count; p++) - { - pred_i = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; - - if (scores_ij == get_score(scores, pred_i, j, gradient, band_width, max_column, min_score_value) + gap_score) - { - prev_i = pred_i; - prev_j = j; - pred_found = true; - break; - } - } - } - } - - // Check if move is horizontal. - if (!pred_found && scores_ij == get_score(scores, i, j - 1, gradient, band_width, max_column, min_score_value) + gap_score) - { - prev_i = i; - prev_j = j - 1; - pred_found = true; - } - - next_node_id = graph[prev_i - 1]; - - alignment_graph[aligned_nodes] = (i == prev_i ? -1 : graph[i - 1]); - alignment_read[aligned_nodes] = (j == prev_j ? -1 : j - 1); - aligned_nodes++; - - i = prev_i; - j = prev_j; - } - - if (loop_count >= (read_length + graph_count + 2)) - { - aligned_nodes = -1; - } - -#ifdef NW_VERBOSE_PRINT - printf("aligned nodes %d, loop count %d\n", aligned_nodes, loop_count); -#endif - } - aligned_nodes = __shfl_sync(FULL_MASK, aligned_nodes, 0); - return aligned_nodes; -} - -} // namespace cudapoa - -} // namespace genomeworks - -} // namespace claraparabricks From 887ceedec8e9daf59d6983fbae73cf0c0f757840 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 28 Sep 2020 10:45:00 -0400 Subject: [PATCH 157/281] [cudapoa] addressed ToDo item for adaptive NW, in case that static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH --- cudapoa/src/cudapoa_kernels.cuh | 10 +++++----- ...ptive_banded.cuh => cudapoa_nw_banded.cuh} | 4 ---- cudapoa/src/cudapoa_nw_tb_banded.cuh | 19 ++++++++++++++----- 3 files changed, 19 insertions(+), 14 deletions(-) rename cudapoa/src/{cudapoa_nw_adaptive_banded.cuh => cudapoa_nw_banded.cuh} (98%) diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 496c10c64..202e8b4ea 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -18,7 +18,7 @@ #pragma once #include "cudapoa_nw.cuh" -#include "cudapoa_nw_adaptive_banded.cuh" +#include "cudapoa_nw_banded.cuh" #include "cudapoa_nw_tb_banded.cuh" #include "cudapoa_topsort.cuh" #include "cudapoa_add_alignment.cuh" @@ -260,8 +260,9 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) // Run Needleman-Wunsch alignment between graph and new sequence. int32_t alignment_length; - if (BM == BandMode::adaptive_band) + if (BM == BandMode::adaptive_band && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) { + // run in adaptive mode only if static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH alignment_length = runNeedlemanWunschBanded(nodes, sorted_poa, node_id_to_pos, @@ -280,7 +281,6 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) mismatch_score, match_score, 0); - __syncwarp(); if (alignment_length < -2) @@ -307,7 +307,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) __syncwarp(); } } - else if (BM == BandMode::static_band) + else if (BM == BandMode::static_band || (BM == BandMode::adaptive_band && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) { alignment_length = runNeedlemanWunschBanded(nodes, sorted_poa, @@ -331,7 +331,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) } else if (BM == BandMode::static_band_traceback) { - alignment_length = runNeedlemanWunschBandedTraceBack(nodes, + alignment_length = runNeedlemanWunschBandedTraceback(nodes, sorted_poa, node_id_to_pos, sequence_lengths[0], diff --git a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh similarity index 98% rename from cudapoa/src/cudapoa_nw_adaptive_banded.cuh rename to cudapoa/src/cudapoa_nw_banded.cuh index e14b5e25f..10763e5da 100644 --- a/cudapoa/src/cudapoa_nw_adaptive_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -187,9 +187,6 @@ __device__ __forceinline__ int32_t match_score, int32_t rerun) { - // in adaptive bands, there can be cases where multiple rows happen to have a band with start index - // smaller than band-start index of a row above. If min_value is too close to numeric_limits::min(), - // this can cause overflow, therefore min_score_value is selected far enough const ScoreT min_score_value = numeric_limits::min() / 2; int32_t lane_idx = threadIdx.x % WARP_SIZE; @@ -217,7 +214,6 @@ __device__ __forceinline__ } // limit band-width for very large reads, ad-hoc rule 3 - ///ToDo add a check for cases where static_band_width > CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH band_width = min(band_width, CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH); } diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index bd79b6714..fa9a762c1 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -17,6 +17,7 @@ #pragma once #include "cudapoa_structs.cuh" +#include "cudapoa_nw_banded.cuh" #include #include @@ -272,10 +273,11 @@ __device__ __forceinline__ void get_scores_tb(ScoreT* scores, template + typename TraceT, + bool ADAPTIVE = true> __device__ __forceinline__ int32_t - runNeedlemanWunschBandedTraceBack(SeqT* nodes, + runNeedlemanWunschBandedTraceback(SeqT* nodes, SizeT* graph, SizeT* node_id_to_pos, int32_t graph_count, @@ -294,15 +296,22 @@ __device__ __forceinline__ int32_t mismatch_score, int32_t match_score) { - - const ScoreT min_score_value = 2 * abs(min(min(gap_score, mismatch_score), -match_score) - 1) + numeric_limits::min(); + const ScoreT min_score_value = numeric_limits::min() / 2; int32_t lane_idx = threadIdx.x % WARP_SIZE; - //Calculate gradient for the scores matrix + //Calculate aspect ratio for the scores matrix float gradient = float(read_length + 1) / float(graph_count + 1); int32_t max_column = read_length + 1; + + // Set band-width based on scores matrix aspect ratio + //--------------------------------------------------------- + //adaptive additions here... + // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 + //int32_t band_shift = band_width / 2; + //--------------------------------------------------------- + // Initialise the horizontal boundary of the score matrix for (int32_t j = lane_idx; j < band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) { From d3b55d091044357bd8edc1e099ee31e5d18ab85a Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 28 Sep 2020 12:49:10 -0400 Subject: [PATCH 158/281] [cudapoa] working on adding adaptive NW with traceback buffer, WIP --- cudapoa/src/cudapoa_nw_banded.cuh | 37 +++++--- cudapoa/src/cudapoa_nw_tb_banded.cuh | 124 +++++++++++++-------------- 2 files changed, 82 insertions(+), 79 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 10763e5da..5f0c186e4 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -43,7 +43,12 @@ __device__ __forceinline__ ScoreT* get_score_ptr(ScoreT* scores, int32_t row, in } template -__device__ __forceinline__ void set_score(ScoreT* scores, int32_t row, int32_t column, int32_t value, int32_t band_start, int32_t band_width) +__device__ __forceinline__ void set_score(ScoreT* scores, + int32_t row, + int32_t column, + int32_t value, + int32_t band_start, + int32_t band_width) { if (column == -1) { @@ -61,7 +66,6 @@ __device__ __forceinline__ void set_score(ScoreT* scores, int32_t row, int32_t c __device__ __forceinline__ int32_t get_band_start_for_row(int32_t row, float gradient, int32_t band_width, int32_t band_shift, int32_t max_column) { - int32_t diagonal_index = int32_t(row * gradient); int32_t start_pos = max(0, diagonal_index - band_shift); if (max_column < start_pos + band_width) @@ -74,9 +78,14 @@ __device__ __forceinline__ int32_t get_band_start_for_row(int32_t row, float gra } template -__device__ __forceinline__ ScoreT get_score(ScoreT* scores, int32_t row, int32_t column, - int32_t band_width, int32_t band_shift, float gradient, - int32_t max_column, const ScoreT min_score_value) +__device__ __forceinline__ ScoreT get_score(ScoreT* scores, + int32_t row, + int32_t column, + int32_t band_width, + int32_t band_shift, + float gradient, + int32_t max_column, + const ScoreT min_score_value) { int32_t band_start = get_band_start_for_row(row, gradient, band_width, band_shift, max_column); int32_t band_end = band_start + band_width; @@ -104,7 +113,6 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, int32_t gap_score, ScoreT4& char_profile) { - // The load instructions typically load data in 4B or 8B chunks. // If data is 16b (2B), then a 4B load chunk is loaded into register // and the necessary bits are extracted before returning. This wastes cycles @@ -113,7 +121,6 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, // using a single load inst, and then extracting necessary part of // of the data using bit arithmatic. Also reduces register count. - // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds int32_t band_start = get_band_start_for_row(row, gradient, band_width, band_shift, max_column); // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds int32_t band_end = band_start + band_width - CELLS_PER_THREAD; @@ -149,7 +156,12 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, } template -__device__ __forceinline__ void initialize_band(ScoreT* scores, int32_t row, int32_t min_score_value, int32_t band_start, int32_t band_width, int32_t lane_idx) +__device__ __forceinline__ void initialize_band(ScoreT* scores, + int32_t row, + int32_t min_score_value, + int32_t band_start, + int32_t band_width, + int32_t lane_idx) { int32_t band_end = band_start + band_width; @@ -181,7 +193,7 @@ __device__ __forceinline__ float max_buffer_size, SizeT* alignment_graph, SizeT* alignment_read, - int32_t static_band_width, + int32_t band_width, int32_t gap_score, int32_t mismatch_score, int32_t match_score, @@ -198,8 +210,6 @@ __device__ __forceinline__ // Set band-width based on scores matrix aspect ratio //--------------------------------------------------------- - int32_t band_width = static_band_width; - if (ADAPTIVE) { if (gradient > 1.1) // ad-hoc rule 1.a @@ -275,9 +285,8 @@ __device__ __forceinline__ initialize_band(scores, score_gIdx, min_score_value, band_start, band_width, lane_idx); int32_t first_element_prev_score = 0; - - uint16_t pred_count = 0; - int32_t pred_idx = 0; + uint16_t pred_count = 0; + int32_t pred_idx = 0; if (lane_idx == 0) { diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index fa9a762c1..d24bbf7b9 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -33,27 +33,6 @@ namespace genomeworks namespace cudapoa { -__device__ __forceinline__ int32_t get_band_start_for_row_tb(int32_t row_idx, float gradient, int32_t band_width, int32_t max_column) -{ - - int32_t start_pos = int32_t(row_idx * gradient) - band_width / 2; - - start_pos = max(start_pos, 0); - - int32_t end_pos = start_pos + band_width; - - if (end_pos > max_column) - { - start_pos = max_column - band_width + CELLS_PER_THREAD; - } - - start_pos = max(start_pos, 0); - - start_pos = start_pos - (start_pos % CELLS_PER_THREAD); - - return start_pos; -} - template __device__ __forceinline__ ScoreT* get_score_ptr_tb(ScoreT* scores, int32_t score_row, int32_t column, int32_t band_start, int32_t band_width) { @@ -71,12 +50,9 @@ __device__ __forceinline__ void set_score_tb(ScoreT* scores, int32_t column, int32_t score_matrix_height, int32_t value, - float gradient, - int32_t band_width, - int32_t max_column) + int32_t band_start, + int32_t band_width) { - int32_t band_start = get_band_start_for_row_tb(row, gradient, band_width, max_column); - int32_t col_idx; if (column == -1) { @@ -86,33 +62,44 @@ __device__ __forceinline__ void set_score_tb(ScoreT* scores, { col_idx = column - band_start; } - + // in NW with traceback buffer, score matrix is stored partially, hence row is mapped to [0, score_matrix_height) span int64_t score_index = static_cast(col_idx) + static_cast(row % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[score_index] = value; } +__device__ __forceinline__ int32_t get_band_start_for_row_tb(int32_t row, float gradient, int32_t band_width, int32_t band_shift, int32_t max_column) +{ + int32_t diagonal_index = int32_t(row * gradient); + int32_t start_pos = max(0, diagonal_index - band_shift); + if (max_column < start_pos + band_width) + { + start_pos = max(0, max_column - band_width + CELLS_PER_THREAD); + } + start_pos = start_pos - (start_pos % CELLS_PER_THREAD); + + return start_pos; +} + template __device__ __forceinline__ void initialize_band_tb(ScoreT* scores, int32_t row, int32_t score_matrix_height, - int32_t value, - float gradient, + int32_t min_score_value, + int32_t band_start, int32_t band_width, - int32_t max_column, int32_t lane_idx) { - int32_t band_start = get_band_start_for_row_tb(row, gradient, band_width, max_column); - int32_t band_end = band_start + band_width; + int32_t band_end = band_start + band_width; - int32_t initialization_offset = (band_start == 0) ? 1 : band_start; + band_start = max(1, band_start); - set_score_tb(scores, row, initialization_offset, score_matrix_height, value, gradient, band_width, max_column); + set_score_tb(scores, row, band_start, score_matrix_height, min_score_value, band_start, band_width); // note: as long as CUDAPOA_BANDED_MATRIX_RIGHT_PADDING < WARP_SIZE, no need for a for loop for (int32_t j = lane_idx + band_end; j < band_end + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) { - set_score_tb(scores, row, j, score_matrix_height, value, gradient, band_width, max_column); + set_score_tb(scores, row, j, score_matrix_height, min_score_value, band_start, band_width); } } @@ -129,12 +116,13 @@ __device__ __forceinline__ ScoreT get_score_tb(ScoreT* scores, int32_t row, int32_t column, int32_t score_matrix_height, - float gradient, int32_t band_width, + int32_t band_shift, + float gradient, int32_t max_column, const ScoreT min_score_value) { - int32_t band_start = get_band_start_for_row_tb(row, gradient, band_width, max_column); + int32_t band_start = get_band_start_for_row_tb(row, gradient, band_width, band_shift, max_column); int32_t band_end = band_start + band_width; band_end = min(band_end, max_column); @@ -144,6 +132,7 @@ __device__ __forceinline__ ScoreT get_score_tb(ScoreT* scores, } else { + // in NW with traceback buffer, score matrix is stored partially, hence row is mapped to [0, score_matrix_height) span return *get_score_ptr_tb(scores, row % score_matrix_height, column, band_start, band_width); } } @@ -152,10 +141,11 @@ template __device__ __forceinline__ void get_scores_tb(ScoreT* scores, int32_t pred_node, int32_t current_node, - int32_t read_pos, + int32_t column, int32_t score_matrix_height, - float gradient, int32_t band_width, + int32_t band_shift, + float gradient, int32_t max_column, int32_t gap_score, int32_t match_score, @@ -165,7 +155,6 @@ __device__ __forceinline__ void get_scores_tb(ScoreT* scores, ScoreT4& score, TraceT4& trace) { - // The load instructions typically load data in 4B or 8B chunks. // If data is 16b (2B), then a 4B load chunk is loaded into register // and the necessary bits are extracted before returning. This wastes cycles @@ -174,20 +163,22 @@ __device__ __forceinline__ void get_scores_tb(ScoreT* scores, // using a single load inst, and then extracting necessary part of // of the data using bit arithmetic. Also reduces register count. - int32_t band_start = get_band_start_for_row_tb(pred_node, gradient, band_width, max_column); + int32_t band_start = get_band_start_for_row_tb(pred_node, gradient, band_width, band_shift, max_column); // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds - int32_t band_end = static_cast(band_start + band_width - CELLS_PER_THREAD); + int32_t band_end = band_start + band_width - CELLS_PER_THREAD; + band_end = min(band_end, max_column); - if ((read_pos > band_end || read_pos < band_start) && read_pos != -1) + if ((column > band_end || column < band_start) && column != -1) { return; } else { - ScoreT4* pred_scores = (ScoreT4*)get_score_ptr_tb(scores, pred_node % score_matrix_height, read_pos, band_start, band_width); + // row is mapped to [0, score_matrix_height) span + ScoreT4* pred_scores = (ScoreT4*)get_score_ptr_tb(scores, pred_node % score_matrix_height, column, band_start, band_width); - // loads 8/16 consecutive bytes (4 ScoreT) + // loads 8/16 consecutive bytes (4 ScoreTs) ScoreT4 score4 = pred_scores[0]; // need to load the next chunk of memory as well @@ -309,13 +300,13 @@ __device__ __forceinline__ //--------------------------------------------------------- //adaptive additions here... // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 - //int32_t band_shift = band_width / 2; + int32_t band_shift = band_width / 2; //--------------------------------------------------------- - // Initialise the horizontal boundary of the score matrix + // Initialise the horizontal boundary of the score matrix, initialising of the vertical boundary is done within the main for loop for (int32_t j = lane_idx; j < band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING; j += WARP_SIZE) { - set_score_tb(scores, 0, j, score_matrix_height, j * gap_score, gradient, band_width, max_column); + set_score_tb(scores, 0, j, score_matrix_height, j * gap_score, 0, band_width); } if (lane_idx == 0) @@ -332,10 +323,10 @@ __device__ __forceinline__ { int32_t node_id = graph[graph_pos]; int32_t score_gIdx = graph_pos + 1; - int32_t band_start = get_band_start_for_row_tb(score_gIdx, gradient, band_width, max_column); + int32_t band_start = get_band_start_for_row_tb(score_gIdx, gradient, band_width, band_shift, max_column); int32_t pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES]; - initialize_band_tb(scores, score_gIdx, score_matrix_height, min_score_value, gradient, band_width, max_column, lane_idx); + initialize_band_tb(scores, score_gIdx, score_matrix_height, min_score_value, band_start, band_width, lane_idx); int32_t first_element_prev_score = 0; uint16_t pred_count = 0; @@ -356,9 +347,8 @@ __device__ __forceinline__ else { int64_t index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); - - pred_idx = node_id_to_pos[pred_node_id] + 1; - + pred_idx = node_id_to_pos[pred_node_id] + 1; + // only predecessors that are less than score_matrix_height distant can be taken into account if ((graph_pos - pred_idx) < score_matrix_height) { // fill in first column of traceback buffer @@ -370,16 +360,19 @@ __device__ __forceinline__ } else { - penalty = max(min_score_value, get_score_tb(scores, pred_idx, -1, score_matrix_height, gradient, band_width, max_column, min_score_value)); + penalty = max(min_score_value, + get_score_tb(scores, pred_idx, -1, score_matrix_height, band_width, band_shift, gradient, max_column, min_score_value)); // if pred_num > 1 keep checking to find max score as penalty for (int32_t p = 1; p < pred_count; p++) { pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; + // only predecessors that are less than score_matrix_height distant can be taken into account if ((score_gIdx - pred_idx_tmp) < score_matrix_height) { int32_t trace_tmp = -(score_gIdx - pred_idx_tmp); - int32_t score_tmp = get_score_tb(scores, pred_idx_tmp, -1, score_matrix_height, gradient, band_width, max_column, min_score_value); + int32_t score_tmp = get_score_tb(scores, pred_idx_tmp, -1, score_matrix_height, band_width, band_shift, + gradient, max_column, min_score_value); if (penalty < score_tmp) { penalty = score_tmp; @@ -388,21 +381,22 @@ __device__ __forceinline__ } } first_element_prev_score = penalty + gap_score; - set_score_tb(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, gradient, band_width, max_column); + set_score_tb(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, band_start, band_width); } } else { penalty = min_score_value; - // pick the predecessor which falls within the limit of score_matrix_height + // look for a predecessor which is within score_matrix_height limit for (int32_t p = 1; p < pred_count; p++) { pred_node_id = incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]; int32_t pred_idx_tmp = node_id_to_pos[pred_node_id] + 1; + // only predecessors that are less than score_matrix_height distant can be taken into account if ((score_gIdx - pred_idx_tmp) < score_matrix_height) { int32_t trace_tmp = -(score_gIdx - pred_idx_tmp); - int32_t score_tmp = get_score_tb(scores, pred_idx_tmp, -1, score_matrix_height, gradient, band_width, max_column, min_score_value); + int32_t score_tmp = get_score_tb(scores, pred_idx_tmp, -1, score_matrix_height, band_width, band_shift, gradient, max_column, min_score_value); if (penalty < score_tmp) { penalty = score_tmp; @@ -411,7 +405,7 @@ __device__ __forceinline__ } } first_element_prev_score = penalty + gap_score; - set_score_tb(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, gradient, band_width, max_column); + set_score_tb(scores, score_gIdx, -1, score_matrix_height, first_element_prev_score, band_start, band_width); } } } @@ -428,8 +422,8 @@ __device__ __forceinline__ TraceT4 trace; ScoreT4 score = {min_score_value, min_score_value, min_score_value, min_score_value}; - get_scores_tb(scores, pred_idx, score_gIdx, read_pos, score_matrix_height, gradient, band_width, max_column, - gap_score, match_score, mismatch_score, read4, graph_base, score, trace); + get_scores_tb(scores, pred_idx, score_gIdx, read_pos, score_matrix_height, band_width, band_shift, gradient, + max_column, gap_score, match_score, mismatch_score, read4, graph_base, score, trace); // Perform same score updates as above, but for rest of predecessors. for (int32_t p = 1; p < pred_count; p++) @@ -437,8 +431,8 @@ __device__ __forceinline__ int32_t pred_idx_tmp = node_id_to_pos[incoming_edges[node_id * CUDAPOA_MAX_NODE_EDGES + p]] + 1; if ((score_gIdx - pred_idx_tmp) < score_matrix_height) { - get_scores_tb(scores, pred_idx_tmp, score_gIdx, read_pos, score_matrix_height, gradient, band_width, max_column, - gap_score, match_score, mismatch_score, read4, graph_base, score, trace); + get_scores_tb(scores, pred_idx_tmp, score_gIdx, read_pos, score_matrix_height, band_width, band_shift, gradient, + max_column, gap_score, match_score, mismatch_score, read4, graph_base, score, trace); } } @@ -524,7 +518,7 @@ __device__ __forceinline__ { if ((graph_count - idx) < score_matrix_height) { - int32_t s = get_score_tb(scores, idx, j, score_matrix_height, gradient, band_width, max_column, min_score_value); + int32_t s = get_score_tb(scores, idx, j, score_matrix_height, band_width, band_shift, gradient, max_column, min_score_value); if (mscore < s) { mscore = s; @@ -546,7 +540,7 @@ __device__ __forceinline__ { loop_count++; - int32_t band_start = get_band_start_for_row_tb(i, gradient, band_width, max_column); + int32_t band_start = get_band_start_for_row_tb(i, gradient, band_width, band_shift, max_column); TraceT trace = get_trace(traceback, i, j, band_start, band_width); if (trace == 0) From 4cca27ba9a752bde381d6512011a939dbef924d5 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 10:28:32 -0700 Subject: [PATCH 159/281] [cudaextender] Const removal; signature updates with ref removal; size()->get_size<>(); --- .../genomeworks/cudaextender/extender.hpp | 22 ++++----- .../genomeworks/cudaextender/utils.hpp | 2 +- cudaextender/samples/sample_cudaextender.cpp | 32 ++++++------ cudaextender/src/extender.cpp | 9 +++- cudaextender/src/ungapped_xdrop.cu | 49 ++++++++++++------- cudaextender/src/ungapped_xdrop.cuh | 21 +++++--- .../tests/Test_CudaextenderEnd2End.cu | 12 +++-- 7 files changed, 92 insertions(+), 55 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index b211b3a47..142dd9085 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -79,9 +79,9 @@ class Extender /// \param[in] h_seed_pairs Vector of seed pairs mapping to query and target /// sequences /// \return Status of the async extension launch - virtual StatusType extend_async(const int8_t* h_query, const int32_t& query_length, - const int8_t* h_target, const int32_t& target_length, - const int32_t& score_threshold, + virtual StatusType extend_async(int8_t* h_query, int32_t query_length, + int8_t* h_target, int32_t target_length, + int32_t score_threshold, const std::vector& h_seed_pairs) = 0; /// \brief Device pointer API for extension @@ -105,8 +105,8 @@ class Extender /// \param[out] d_num_scored_segment_pairs Pointer to pre-allocated device location for /// storing length of extension output /// \return Status of the async extension launch - virtual StatusType extend_async(const int8_t* d_query, int32_t query_length, - const int8_t* d_target, int32_t target_length, + virtual StatusType extend_async(int8_t* d_query, int32_t query_length, + int8_t* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) = 0; @@ -144,14 +144,14 @@ class Extender /// \param allocator DeviceAllocator to be used for allocating/freeing memory /// \param type Type of extension to be performed /// \return Unique pointer to Extender object. -std::unique_ptr create_extender(const int32_t* h_score_mat, - const int32_t score_mat_dim, - const int32_t xdrop_threshold, - const bool no_entropy, +std::unique_ptr create_extender(int32_t* h_score_mat, + int32_t score_mat_dim, + int32_t xdrop_threshold, + bool no_entropy, cudaStream_t stream, - const int32_t device_id, + int32_t device_id, DefaultDeviceAllocator allocator, - const ExtensionType type = ExtensionType::ungapped_xdrop); + ExtensionType type = ExtensionType::ungapped_xdrop); /// \} } // namespace cudaextender } // namespace genomeworks diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp index 91bfec640..e8d561a1d 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp @@ -104,7 +104,7 @@ inline void parse_scored_segment_pairs(const std::string& filepath, std::vector< /// \param[out] dst_seq Pointer to pre-allocated storage for encoded sequence /// \param[in] src_seq Pointer to input sequence /// \param[in] length Length of the sequence -inline void encode_sequence(int8_t* dst_seq, const char* src_seq, const int32_t& length) +inline void encode_sequence(int8_t* dst_seq, const char* src_seq, const int32_t length) { for (int32_t i = 0; i < length; i++) { diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index d22e44784..f84216c2e 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -30,7 +31,7 @@ using namespace claraparabricks::genomeworks; using namespace cudautils; using namespace cudaextender; -void print_scored_segment_pairs(const std::vector& scored_segment_pairs) +static void print_scored_segment_pairs(const std::vector& scored_segment_pairs) { std::cout << "Target Position, Query Position, Length, Score" << std::endl; for (auto& segment : scored_segment_pairs) @@ -98,14 +99,14 @@ int main(int argc, char* argv[]) std::cerr << "Number of Seed Pairs: " << h_seed_pairs.size() << std::endl; // Define Scoring Matrix - const int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, - -114, 100, -125, -31, -1000, -1000, -100, -9100, - -31, -125, 100, -114, -1000, -1000, -100, -9100, - -123, -31, -114, 91, -1000, -1000, -100, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -100, -100, -100, -100, -1000, -1000, -100, -9100, - -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; + int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, + -114, 100, -125, -31, -1000, -1000, -100, -9100, + -31, -125, 100, -114, -1000, -1000, -100, -9100, + -123, -31, -114, 91, -1000, -1000, -100, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -100, -100, -100, -100, -1000, -1000, -100, -9100, + -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; // Allocate pinned memory for query and target strings pinned_host_vector h_encoded_target(target_sequence.length()); @@ -131,9 +132,9 @@ int main(int argc, char* argv[]) if (!device_ptr_api_mode) { ungapped_extender->extend_async(h_encoded_query.data(), - h_encoded_query.size(), + get_size(h_encoded_query), h_encoded_target.data(), - h_encoded_target.size(), + get_size(h_encoded_target), score_threshold, h_seed_pairs); ungapped_extender->sync(); @@ -159,9 +160,9 @@ int main(int argc, char* argv[]) // Launch the ungapped extender device pointer function ungapped_extender->extend_async(d_query.data(), - d_query.size(), + get_size(d_query), d_target.data(), - d_target.size(), + get_size(d_target), score_threshold, d_seed_pairs.data(), d_seed_pairs.size(), @@ -172,8 +173,9 @@ int main(int argc, char* argv[]) GW_CU_CHECK_ERR(cudaStreamSynchronize(stream0.get())); const int32_t h_num_ssp = cudautils::get_value_from_device(d_num_ssp.data(), stream0.get()); h_ssp.resize(h_num_ssp); - // Copy data synchronously - device_copy_n(d_ssp.data(), h_num_ssp, h_ssp.data()); + // Copy data asynchronously + device_copy_n(d_ssp.data(), h_num_ssp, h_ssp.data(), stream0.get()); + cudaStreamSynchronize(stream0.get()); } std::cerr << "Number of Scored Segment Pairs found: " << h_ssp.size() << std::endl; if (print) diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index 040ed75ed..d4aaf8dd1 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -27,7 +27,14 @@ namespace cudaextender { // TODO - Provide an allocator-less API for creation? -std::unique_ptr create_extender(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator, const ExtensionType type) +std::unique_ptr create_extender(int32_t* h_score_mat, + int32_t score_mat_dim, + int32_t xdrop_threshold, + bool no_entropy, + cudaStream_t stream, + int32_t device_id, + DefaultDeviceAllocator allocator, + ExtensionType type) { if (type == ExtensionType::ungapped_xdrop) { diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 92f0361d0..0cdb5248d 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -36,8 +36,11 @@ namespace cudaextender using namespace cudautils; -UngappedXDrop::UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator) - : h_score_mat_(h_score_mat) +UngappedXDrop::UngappedXDrop(int32_t* h_score_mat, const int32_t score_mat_dim, + const int32_t xdrop_threshold, const bool no_entropy, + cudaStream_t stream, const int32_t device_id, + DefaultDeviceAllocator allocator) + : h_score_mat_(h_score_mat, h_score_mat + score_mat_dim) , score_mat_dim_(score_mat_dim) , xdrop_threshold_(xdrop_threshold) , no_entropy_(no_entropy) @@ -46,10 +49,6 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat , host_ptr_api_mode_(false) , allocator_(allocator) { - if (h_score_mat_ == nullptr) - { - throw std::runtime_error("Substitution matrix cannot be null"); - } // TODO - check sub_mat_dim based on Sequence Encoder API // Calculate the max limits on the number of extensions we can do on this GPU cudaDeviceProp device_prop; @@ -64,8 +63,19 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat //Figure out memory requirements for cub functions size_t temp_storage_bytes = 0; size_t cub_storage_bytes = 0; - GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(nullptr, temp_storage_bytes, d_tmp_ssp_.data(), d_tmp_ssp_.data(), (int32_t*)nullptr, batch_max_ungapped_extensions_, stream_)); - GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(nullptr, cub_storage_bytes, d_done_.data(), d_done_.data(), batch_max_ungapped_extensions_, stream_)); + GW_CU_CHECK_ERR(cub::DeviceSelect::Unique(nullptr, + temp_storage_bytes, + d_tmp_ssp_.data(), + d_tmp_ssp_.data(), + (int32_t*)nullptr, + batch_max_ungapped_extensions_, + stream_)); + GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(nullptr, + cub_storage_bytes, + d_done_.data(), + d_done_.data(), + batch_max_ungapped_extensions_, + stream_)); cub_storage_bytes = std::max(temp_storage_bytes, cub_storage_bytes); // Allocate space on device for scoring matrix and intermediate results @@ -75,13 +85,13 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat d_temp_storage_cub_ = device_buffer(cub_storage_bytes, allocator_, stream_); // Requires pinned host memory registration for proper async behavior - device_copy_n(h_score_mat_, score_mat_dim_, d_score_mat_.data(), stream_); + device_copy_n(h_score_mat_.data(), score_mat_dim_, d_score_mat_.data(), stream_); } -StatusType UngappedXDrop::extend_async(const int8_t* d_query, int32_t query_length, - const int8_t* d_target, int32_t target_length, - int32_t score_threshold, SeedPair* d_seed_pairs, - int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, +StatusType UngappedXDrop::extend_async(int8_t* d_query, const int32_t query_length, + int8_t* d_target, const int32_t target_length, + const int32_t score_threshold, SeedPair* d_seed_pairs, + const int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) { if (d_query == nullptr || d_target == nullptr || d_seed_pairs == nullptr) @@ -118,7 +128,12 @@ StatusType UngappedXDrop::extend_async(const int8_t* d_query, int32_t query_leng d_scored_segment_pairs, d_done_.data()); size_t cub_storage_bytes = d_temp_storage_cub_.size(); - GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(d_temp_storage_cub_.data(), cub_storage_bytes, d_done_.data(), d_done_.data(), curr_num_pairs, stream_)) + GW_CU_CHECK_ERR(cub::DeviceScan::InclusiveSum(d_temp_storage_cub_.data(), + cub_storage_bytes, + d_done_.data(), + d_done_.data(), + curr_num_pairs, + stream_)) // TODO- Make async const int32_t num_scored_segment_pairs = get_value_from_device(d_done_.data() + curr_num_pairs - 1, stream_); if (num_scored_segment_pairs > 0) @@ -148,9 +163,9 @@ StatusType UngappedXDrop::extend_async(const int8_t* d_query, int32_t query_leng return StatusType::success; } -StatusType UngappedXDrop::extend_async(const int8_t* h_query, const int32_t& query_length, - const int8_t* h_target, const int32_t& target_length, - const int32_t& score_threshold, +StatusType UngappedXDrop::extend_async(int8_t* h_query, const int32_t query_length, + int8_t* h_target, const int32_t target_length, + const int32_t score_threshold, const std::vector& h_seed_pairs) { // Reset the extender if it was used before in this mode diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 002c035bf..bff14dec4 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -18,6 +18,7 @@ #include #include +#include namespace claraparabricks { @@ -31,16 +32,22 @@ namespace cudaextender class UngappedXDrop : public Extender { public: - UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator); + UngappedXDrop(int32_t* h_score_mat, + int32_t score_mat_dim, + int32_t xdrop_threshold, + bool no_entropy, + cudaStream_t stream, + int32_t device_id, + DefaultDeviceAllocator allocator); ~UngappedXDrop() override; - StatusType extend_async(const int8_t* h_query, const int32_t& query_length, - const int8_t* h_target, const int32_t& target_length, - const int32_t& score_threshold, + StatusType extend_async(int8_t* h_query, int32_t query_length, + int8_t* h_target, int32_t target_length, + int32_t score_threshold, const std::vector& h_seed_pairs) override; - StatusType extend_async(const int8_t* d_query, int32_t query_length, - const int8_t* d_target, int32_t target_length, + StatusType extend_async(int8_t* d_query, int32_t query_length, + int8_t* d_target, int32_t target_length, int32_t score_threshold, SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) override; @@ -54,7 +61,7 @@ public: private: DefaultDeviceAllocator allocator_; // Device ptr API required variables - const int32_t* h_score_mat_; + pinned_host_vector h_score_mat_; const int32_t score_mat_dim_; // Assume matrix is square const int32_t xdrop_threshold_; const bool no_entropy_; diff --git a/cudaextender/tests/Test_CudaextenderEnd2End.cu b/cudaextender/tests/Test_CudaextenderEnd2End.cu index 25cde2680..0bebc5989 100644 --- a/cudaextender/tests/Test_CudaextenderEnd2End.cu +++ b/cudaextender/tests/Test_CudaextenderEnd2End.cu @@ -19,6 +19,7 @@ #include #include #include +#include namespace claraparabricks { @@ -29,14 +30,14 @@ namespace genomeworks namespace cudaextender { -typedef struct +struct End2EndTestParam { std::string query_file; std::string target_file; std::string seed_pairs_file; std::string golden_scored_segment_pairs_file; int32_t score_threshold; -} End2EndTestParam; +}; std::vector getCudaextenderEnd2EndTestCases() { @@ -96,7 +97,12 @@ public: pinned_host_vector h_encoded_query(target_sequence.length()); encode_sequence(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); encode_sequence(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); - ungapped_extender_->extend_async(h_encoded_query.data(), h_encoded_query.size(), h_encoded_target.data(), h_encoded_target.size(), param_.score_threshold, h_seed_pairs); + ungapped_extender_->extend_async(h_encoded_query.data(), + get_size(h_encoded_query), + h_encoded_target.data(), + get_size(h_encoded_target), + param_.score_threshold, + h_seed_pairs); // Parse golden scored_segment_pairs while extension is in progress std::vector golden_scored_segment_pairs; parse_scored_segment_pairs(param_.golden_scored_segment_pairs_file, golden_scored_segment_pairs); From cc65d19ce7bdf83fd0f4966e0214b0a7b8a843f1 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 28 Sep 2020 13:55:05 -0400 Subject: [PATCH 160/281] [cudapoa] added adaptive NW with traceback buffer --- .../genomeworks/cudapoa/cudapoa.hpp | 14 +- cudapoa/samples/sample_cudapoa.cpp | 6 +- cudapoa/src/allocate_block.hpp | 8 +- cudapoa/src/application_parameters.cpp | 4 +- cudapoa/src/batch.cu | 2 +- cudapoa/src/cudapoa_batch.cuh | 2 +- cudapoa/src/cudapoa_kernels.cuh | 182 ++++++++++++++++-- cudapoa/src/cudapoa_nw_tb_banded.cuh | 75 +++++++- 8 files changed, 252 insertions(+), 41 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp index acfa00861..105db9c4a 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp @@ -44,21 +44,23 @@ enum StatusType generic_error }; -/// Banding mode used in Needleman-Wunch algorithm +/// Banding mode used in Needleman-Wunsch algorithm /// - full_band performs computations on full scores matrix, highest accuracy /// - static_band performs computations on a fixed band along scores matrix diagonal, fastest implementation /// - adaptive_band, similar to static_band performs computations on a band along diagonal, but the band-width /// can vary per alignment's score matrix, faster than full_band and more accurate than static_band -/// - static_band_traceback computations are on a fixed band, score matrix is stored partially, instead traceback -/// information is stored in a buffer and used in backtracking. This can be useful for long-read data GPU memory is -/// limiting GPU parallelism. Traceback matrix requires 2x to 4x smaller memory compared to score matrix - +/// - static_band_traceback computations are on a fixed band, score matrix is stored partially and there is an +/// additional traceback matrix. Accuracy is similar to static_band. This mode can particularly be useful +/// for long-read data where GPU memory is limiting parallelism. Traceback matrix requires 2x to 4x smaller +/// memory compared to score matrix +/// - adaptive_band_traceback, similar to static_band_traceback but with varying band-width size enum BandMode { full_band = 0, static_band, adaptive_band, - static_band_traceback + static_band_traceback, + adaptive_band_traceback }; /// Initialize CUDA POA context. diff --git a/cudapoa/samples/sample_cudapoa.cpp b/cudapoa/samples/sample_cudapoa.cpp index 3d9eeac87..d4b66f3ea 100644 --- a/cudapoa/samples/sample_cudapoa.cpp +++ b/cudapoa/samples/sample_cudapoa.cpp @@ -131,7 +131,7 @@ int main(int argc, char** argv) int c = 0; bool msa = false; bool long_read = false; - BandMode band_mode = BandMode::adaptive_band; // 0: full, 1: static-band, 2: adaptive-band, 3- static-band-traceback + BandMode band_mode = BandMode::adaptive_band; // 0: full, 1: static-band, 2: adaptive-band, 3- static-band-traceback 4- adaptive-band-traceback bool help = false; bool print = false; bool print_graph = false; @@ -148,9 +148,9 @@ int main(int argc, char** argv) long_read = true; break; case 'b': - if (std::stoi(optarg) < 0 || std::stoi(optarg) > 3) + if (std::stoi(optarg) < 0 || std::stoi(optarg) > 4) { - throw std::runtime_error("band-mode must be either 0 for full bands, 1 for static bands, 2 for adaptive bands or 3 for static band with traceback"); + throw std::runtime_error("band-mode must be either 0 for full bands, 1 for static bands, 2 for adaptive bands, 3 and 4 for static and adaptive bands with traceback"); } band_mode = static_cast(std::stoi(optarg)); break; diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 4fdb90a0f..cacd4e88c 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -61,7 +61,7 @@ class BatchBlock { scoped_device_switch dev(device_id_); max_nodes_per_window_ = batch_size.max_nodes_per_graph; - traceback_alignment_ = batch_size.band_mode == BandMode::static_band_traceback; + traceback_alignment_ = batch_size.band_mode == BandMode::static_band_traceback || batch_size.band_mode == BandMode::adaptive_band_traceback; score_matrix_height_ = traceback_alignment_ ? batch_size.max_pred_distance_in_banded_mode : batch_size.max_nodes_per_graph; score_matrix_width_ = batch_size.matrix_sequence_dimension; @@ -84,7 +84,7 @@ class BatchBlock // in banded traceback alignment, this is for traceback matrix, otherwise it is for score matrix int64_t device_size_per_matrix = static_cast(batch_size.matrix_sequence_dimension) * static_cast(batch_size.max_nodes_per_graph); - if (batch_size.band_mode == BandMode::static_band_traceback) + if (traceback_alignment_) { device_size_per_matrix *= sizeof(TraceT); } @@ -337,7 +337,7 @@ class BatchBlock { int64_t device_size_per_poa = 0; int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; - bool traceback = batch_size.band_mode == BandMode::static_band_traceback; + bool traceback = batch_size.band_mode == static_band_traceback || batch_size.band_mode == adaptive_band_traceback; int32_t traceback_score_matrix_size = batch_size.matrix_sequence_dimension * batch_size.max_pred_distance_in_banded_mode; // for output - device @@ -439,7 +439,7 @@ class BatchBlock // Compute required memory for score or backtracking matrix int64_t device_size_per_matrix = static_cast(batch_size.matrix_sequence_dimension) * static_cast(batch_size.max_nodes_per_graph); - if (batch_size.band_mode == BandMode::static_band_traceback) + if (batch_size.band_mode == BandMode::static_band_traceback || batch_size.band_mode == BandMode::adaptive_band_traceback) { device_size_per_matrix *= sizeof_TraceT; } diff --git a/cudapoa/src/application_parameters.cpp b/cudapoa/src/application_parameters.cpp index aebe8141e..4942aae90 100644 --- a/cudapoa/src/application_parameters.cpp +++ b/cudapoa/src/application_parameters.cpp @@ -66,9 +66,9 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) msa = true; break; case 'b': - if (std::stoi(optarg) < 0 || std::stoi(optarg) > 3) + if (std::stoi(optarg) < 0 || std::stoi(optarg) > 4) { - throw std::runtime_error("band-mode must be either 0 for full bands, 1 for static bands, 2 for adaptive bands or 3 for static band with traceback"); + throw std::runtime_error("band-mode must be either 0 for full bands, 1 for static bands, 2 for adaptive bands, 3 and 4 for static and adaptive bands with traceback"); } band_mode = static_cast(std::stoi(optarg)); break; diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index 01aded35f..edb32c552 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -53,7 +53,7 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa { matrix_sequence_dimension = cudautils::align(alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); } - else // BandMode::adaptive_band + else // BandMode::adaptive_band || BandMode::adaptive_band_traceback { // adapive_storage_factor is to reserve extra memory for cases with extended band-width matrix_sequence_dimension = cudautils::align(adapive_storage_factor * (alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING)); diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index c4cac03ba..f31187c4e 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -552,7 +552,7 @@ protected: int32_t matrix_width = (batch_size_.band_mode != BandMode::full_band) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); // in full-band, avail_buf_mem_ is dedicated to scores matrix and in static or adaptive band modes, avail_buf_mem_ is dedicated to traceback matrix size_t required_size = static_cast(matrix_width) * static_cast(matrix_height); - required_size *= batch_size_.band_mode == BandMode::static_band_traceback ? sizeof(TraceT) : sizeof(ScoreT); + required_size *= (batch_size_.band_mode == static_band_traceback || batch_size_.band_mode == adaptive_band_traceback) ? sizeof(TraceT) : sizeof(ScoreT); if (required_size > avail_buf_mem_) { diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 202e8b4ea..606e509bc 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -146,9 +146,9 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) int64_t scores_offset = static_cast(banded_buffer_size) * static_cast(window_idx); scores = &scores_d[scores_offset]; } - else if (BM == BandMode::static_band_traceback) + else if (BM == BandMode::static_band_traceback || BM == BandMode::adaptive_band_traceback) { - // buffer size for scores, in banded we only need to store part of the scores matrix + // buffer size for scores, in traceback we only need to store part of the scores matrix banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); int64_t offset = static_cast(banded_buffer_size) * static_cast(window_idx); scores = &scores_d[offset]; @@ -260,6 +260,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) // Run Needleman-Wunsch alignment between graph and new sequence. int32_t alignment_length; + // Adaptive band --------------------------------------------------------------------------------------- if (BM == BandMode::adaptive_band && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) { // run in adaptive mode only if static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH @@ -307,6 +308,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) __syncwarp(); } } + // Static band --------------------------------------------------------------------------------------- else if (BM == BandMode::static_band || (BM == BandMode::adaptive_band && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) { alignment_length = runNeedlemanWunschBanded(nodes, @@ -329,28 +331,82 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) alignment_length); __syncwarp(); } - else if (BM == BandMode::static_band_traceback) + // Adaptive band with traceback ------------------------------------------------------------------------ + else if (BM == BandMode::adaptive_band_traceback && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) { - alignment_length = runNeedlemanWunschBandedTraceback(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - traceback, - alignment_graph, - alignment_read, - static_band_width, - max_pred_distance, - gap_score, - mismatch_score, - match_score); + alignment_length = runNeedlemanWunschBandedTraceback(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + max_pred_distance, + gap_score, + mismatch_score, + match_score, + 0); __syncwarp(); + if (alignment_length < -2) + { + // rerun with extended band-width + alignment_length = runNeedlemanWunschBandedTraceback(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + max_pred_distance, + gap_score, + mismatch_score, + match_score, + alignment_length); + __syncwarp(); + } } + // Static band with traceback -------------------------------------------------------------------------- + else if (BM == BandMode::static_band_traceback || (BM == BandMode::adaptive_band_traceback && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) + { + alignment_length = runNeedlemanWunschBandedTraceback(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + max_pred_distance, + gap_score, + mismatch_score, + match_score, + alignment_length); + __syncwarp(); + } + // Full band ------------------------------------------------------------------------------------------- else // BM == BandMode::full_band { alignment_length = runNeedlemanWunsch(nodes, @@ -651,6 +707,48 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_pred_distance_in_banded_mode, traceback); } + else if (batch_size.band_mode == BandMode::adaptive_band_traceback) + { + generatePOAKernel + <<>>(consensus_d, + sequences_d, + base_weights_d, + sequence_lengths_d, + window_details_d, + total_windows, + scores, + alignment_graph, + alignment_read, + nodes, + incoming_edges, + incoming_edge_count, + outgoing_edges, + outgoing_edge_count, + incoming_edge_w, + sorted_poa, + node_id_to_pos, + node_alignments, + node_alignment_count, + sorted_poa_local_edge_count, + node_marks, + check_aligned_nodes, + nodes_to_visit, + node_coverage_counts, + gap_score, + mismatch_score, + match_score, + max_sequences_per_poa, + sequence_begin_nodes_ids, + outgoing_edges_coverage, + outgoing_edges_coverage_count, + max_nodes_per_graph, + matrix_seq_dimension, + batch_size.max_consensus_size, + TPB, + batch_size.alignment_band_width, + batch_size.max_pred_distance_in_banded_mode, + traceback); + } else { generatePOAKernel @@ -817,6 +915,48 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_pred_distance_in_banded_mode, traceback); } + else if (batch_size.band_mode == BandMode::adaptive_band_traceback) + { + generatePOAKernel + <<>>(consensus_d, + sequences_d, + base_weights_d, + sequence_lengths_d, + window_details_d, + total_windows, + scores, + alignment_graph, + alignment_read, + nodes, + incoming_edges, + incoming_edge_count, + outgoing_edges, + outgoing_edge_count, + incoming_edge_w, + sorted_poa, + node_id_to_pos, + node_alignments, + node_alignment_count, + sorted_poa_local_edge_count, + node_marks, + check_aligned_nodes, + nodes_to_visit, + node_coverage_counts, + gap_score, + mismatch_score, + match_score, + max_sequences_per_poa, + sequence_begin_nodes_ids, + outgoing_edges_coverage, + outgoing_edges_coverage_count, + max_nodes_per_graph, + matrix_seq_dimension, + batch_size.max_consensus_size, + TPB, + batch_size.alignment_band_width, + batch_size.max_pred_distance_in_banded_mode, + traceback); + } else { generatePOAKernel diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index d24bbf7b9..8e9d33742 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -17,7 +17,6 @@ #pragma once #include "cudapoa_structs.cuh" -#include "cudapoa_nw_banded.cuh" #include #include @@ -279,13 +278,15 @@ __device__ __forceinline__ int32_t read_length, ScoreT* scores, TraceT* traceback, + float max_buffer_size, SizeT* alignment_graph, SizeT* alignment_read, int32_t band_width, int32_t score_matrix_height, int32_t gap_score, int32_t mismatch_score, - int32_t match_score) + int32_t match_score, + int32_t rerun) { const ScoreT min_score_value = numeric_limits::min() / 2; @@ -298,9 +299,53 @@ __device__ __forceinline__ // Set band-width based on scores matrix aspect ratio //--------------------------------------------------------- - //adaptive additions here... + if (ADAPTIVE) + { + if (gradient > 1.1) // ad-hoc rule 1.a + { + // ad-hoc rule 1.b + band_width = max(band_width, cudautils::align(max_column * 0.08 * gradient)); + } + if (gradient < 0.8) // ad-hoc rule 2.a + { + // ad-hoc rule 2.b + band_width = max(band_width, cudautils::align(max_column * 0.1 / gradient)); + } + + // limit band-width for very large reads, ad-hoc rule 3 + band_width = min(band_width, CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH); + } + // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 int32_t band_shift = band_width / 2; + + if (ADAPTIVE) + { + // rerun code is defined in backtracking loop from previous alignment try + // -3 means traceback path was too close to the left bound of band + // -4 means traceback path was too close to the right bound of band + // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift + // further to the left for rerun == -3, and further to the right for rerun == -4. + if (rerun == -3) + { + // ad-hoc rule 5 + band_width *= 2; + band_shift *= 2.5; + } + if (rerun == -4) + { + // ad-hoc rule 6 + band_width *= 2; + band_shift *= 1.5; + } + // check required memory and return error if exceeding max_buffer_size + // using float to avoid 64-bit + float required_buffer_size = static_cast(graph_count) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + if (required_buffer_size > max_buffer_size) + { + return -2; + } + } //--------------------------------------------------------- // Initialise the horizontal boundary of the score matrix, initialising of the vertical boundary is done within the main for loop @@ -564,6 +609,30 @@ __device__ __forceinline__ alignment_read[aligned_nodes] = j - 1; i -= trace; j--; + + if (ADAPTIVE) + { + if (rerun == 0) + { + // check if traceback gets too close or hits the band limits, if so stop and rerun with extended band-width + // threshold for proximity to band limits works better if defined proportionate to the sequence length + int32_t threshold = max(1, max_column / 1024); // ad-hoc rule 7 + if (j > threshold && j < max_column - threshold) + { + int32_t band_start = get_band_start_for_row(i, gradient, band_width, band_shift, max_column); + if (j <= band_start + threshold) // ad-hoc rule 8-a, too close to left bound + { + aligned_nodes = -3; + break; + } + if (j >= (band_start + band_width - threshold)) // ad-hoc rule 8-b, too close to right bound + { + aligned_nodes = -4; + break; + } + } + } + } } aligned_nodes++; From cfb1daa00e5c1e5d825e79e27bece95f397fe7c6 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 11:08:15 -0700 Subject: [PATCH 161/281] [cudaextender] enum class --- .../genomeworks/cudaextender/utils.hpp | 41 ++++++++++--------- cudaextender/samples/sample_cudaextender.cpp | 16 ++++---- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp index e8d561a1d..44ffef162 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp @@ -32,17 +32,6 @@ namespace genomeworks namespace cudaextender { -constexpr int8_t A_NT = 0; -constexpr int8_t C_NT = 1; -constexpr int8_t G_NT = 2; -constexpr int8_t T_NT = 3; -constexpr int8_t L_NT = 4; -constexpr int8_t N_NT = 5; -constexpr int8_t X_NT = 6; -constexpr int8_t E_NT = 7; -constexpr int8_t NUC = 8; -constexpr int8_t NUC2 = NUC * NUC; - /// \brief Parses seed pairs from a csv file in the following format: /// target_position_in_read_1, query_position_in_read_1 /// target_position_in_read_2, query_position_in_read_2 @@ -98,6 +87,20 @@ inline void parse_scored_segment_pairs(const std::string& filepath, std::vector< } } } +/// \brief Integer encoding scheme for character sequence +enum class Encoding : int8_t +{ + A_NT = 0, + C_NT, + G_NT, + T_NT, + L_NT, + N_NT, + X_NT, + E_NT, + NUC, + NUC2 = NUC * NUC, +}; /// \brief Encodes character sequence as integer sequence /// @@ -112,32 +115,32 @@ inline void encode_sequence(int8_t* dst_seq, const char* src_seq, const int32_t switch (ch) { case 'A': - dst_seq[i] = A_NT; + dst_seq[i] = static_cast(Encoding::A_NT); break; case 'C': - dst_seq[i] = C_NT; + dst_seq[i] = static_cast(Encoding::C_NT); break; case 'G': - dst_seq[i] = G_NT; + dst_seq[i] = static_cast(Encoding::G_NT); break; case 'T': - dst_seq[i] = T_NT; + dst_seq[i] = static_cast(Encoding::T_NT); break; case '&': - dst_seq[i] = E_NT; + dst_seq[i] = static_cast(Encoding::E_NT); break; case 'n': case 'N': - dst_seq[i] = N_NT; + dst_seq[i] = static_cast(Encoding::N_NT); break; case 'a': case 'c': case 'g': case 't': - dst_seq[i] = L_NT; + dst_seq[i] = static_cast(Encoding::L_NT); break; default: - dst_seq[i] = X_NT; + dst_seq[i] = static_cast(Encoding::X_NT); break; } } diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index f84216c2e..eb04c4ed2 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -99,14 +99,14 @@ int main(int argc, char* argv[]) std::cerr << "Number of Seed Pairs: " << h_seed_pairs.size() << std::endl; // Define Scoring Matrix - int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, - -114, 100, -125, -31, -1000, -1000, -100, -9100, - -31, -125, 100, -114, -1000, -1000, -100, -9100, - -123, -31, -114, 91, -1000, -1000, -100, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -100, -100, -100, -100, -1000, -1000, -100, -9100, - -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; + int32_t score_matrix[static_cast(Encoding::NUC2)] = {91, -114, -31, -123, -1000, -1000, -100, -9100, + -114, 100, -125, -31, -1000, -1000, -100, -9100, + -31, -125, 100, -114, -1000, -1000, -100, -9100, + -123, -31, -114, 91, -1000, -1000, -100, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -100, -100, -100, -100, -1000, -1000, -100, -9100, + -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; // Allocate pinned memory for query and target strings pinned_host_vector h_encoded_target(target_sequence.length()); From b64c01862d3cf21ac1a989b4afbb236b4f4dfb6e Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 11:19:25 -0700 Subject: [PATCH 162/281] [cudaextender] Add NUC2 as enum --- cudaextender/samples/sample_cudaextender.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index eb04c4ed2..ff62f3976 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -123,7 +123,7 @@ int main(int argc, char* argv[]) std::vector h_ssp; // Create Ungapped Extender Object for both API modes std::unique_ptr ungapped_extender = create_extender(score_matrix, - NUC2, + static_cast(Encoding::NUC2), xdrop_threshold, input_no_entropy, stream0.get(), From 1e258cf5453a3f3af76a6bf0c6e6fb9e48d192de Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 11:22:57 -0700 Subject: [PATCH 163/281] [cudaextender] size_t->int64_t --- cudaextender/samples/sample_cudaextender.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index ff62f3976..e61db0a59 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -117,7 +117,7 @@ int main(int argc, char* argv[]) // Create a stream for async use CudaStream stream0 = make_cuda_stream(); // Create an allocator for use with both APIs - const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); + const int64_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); DefaultDeviceAllocator allocator = create_default_device_allocator(max_gpu_memory); // Reference for output std::vector h_ssp; From 4d93eb0934b334ffabcce18f137dbf708f7c40f7 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 11:23:34 -0700 Subject: [PATCH 164/281] [cudaextender] Make format --- cudaextender/samples/sample_cudaextender.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index e61db0a59..0c7194f59 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -117,7 +117,7 @@ int main(int argc, char* argv[]) // Create a stream for async use CudaStream stream0 = make_cuda_stream(); // Create an allocator for use with both APIs - const int64_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); + const int64_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); DefaultDeviceAllocator allocator = create_default_device_allocator(max_gpu_memory); // Reference for output std::vector h_ssp; From bca8fe74bba189475a6313d759f7903255b2f366 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 11:29:58 -0700 Subject: [PATCH 165/281] [cudaextender] Make protected --- .../tests/Test_CudaextenderEnd2End.cu | 83 +++++++++---------- 1 file changed, 39 insertions(+), 44 deletions(-) diff --git a/cudaextender/tests/Test_CudaextenderEnd2End.cu b/cudaextender/tests/Test_CudaextenderEnd2End.cu index 0bebc5989..6ac8cec3f 100644 --- a/cudaextender/tests/Test_CudaextenderEnd2End.cu +++ b/cudaextender/tests/Test_CudaextenderEnd2End.cu @@ -62,21 +62,21 @@ public: { param_ = GetParam(); // Define Scoring Matrix - const int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, - -114, 100, -125, -31, -1000, -1000, -100, -9100, - -31, -125, 100, -114, -1000, -1000, -100, -9100, - -123, -31, -114, 91, -1000, -1000, -100, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -100, -100, -100, -100, -1000, -1000, -100, -9100, - -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; - const int32_t xdrop_threshold = 910; - const int32_t device_id = 0; - const bool no_entropy = false; - const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); - allocator_ = create_default_device_allocator(max_gpu_memory); - stream_ = make_cuda_stream(); - ungapped_extender_ = create_extender(score_matrix, NUC2, xdrop_threshold, no_entropy, stream_.get(), device_id, allocator_); + int32_t score_matrix[static_cast(Encoding::NUC2)] = {91, -114, -31, -123, -1000, -1000, -100, -9100, + -114, 100, -125, -31, -1000, -1000, -100, -9100, + -31, -125, 100, -114, -1000, -1000, -100, -9100, + -123, -31, -114, 91, -1000, -1000, -100, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -100, -100, -100, -100, -1000, -1000, -100, -9100, + -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; + const int32_t xdrop_threshold = 910; + const int32_t device_id = 0; + const bool no_entropy = false; + const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); + allocator_ = create_default_device_allocator(max_gpu_memory); + stream_ = make_cuda_stream(); + ungapped_extender_ = create_extender(score_matrix, static_cast(Encoding::NUC2), xdrop_threshold, no_entropy, stream_.get(), device_id, allocator_); } void TearDown() @@ -84,34 +84,7 @@ public: ungapped_extender_.reset(); } - void run_test() - { - std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(param_.target_file, 0, false); - const std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; - std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(param_.query_file, 0, false); - const std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; - std::vector h_seed_pairs; - parse_seed_pairs(param_.seed_pairs_file, h_seed_pairs); - // Allocate pinned memory for query and target strings - pinned_host_vector h_encoded_target(target_sequence.length()); - pinned_host_vector h_encoded_query(target_sequence.length()); - encode_sequence(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); - encode_sequence(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); - ungapped_extender_->extend_async(h_encoded_query.data(), - get_size(h_encoded_query), - h_encoded_target.data(), - get_size(h_encoded_target), - param_.score_threshold, - h_seed_pairs); - // Parse golden scored_segment_pairs while extension is in progress - std::vector golden_scored_segment_pairs; - parse_scored_segment_pairs(param_.golden_scored_segment_pairs_file, golden_scored_segment_pairs); - ungapped_extender_->sync(); - std::vector scored_segment_pairs = ungapped_extender_->get_scored_segment_pairs(); - ASSERT_EQ(golden_scored_segment_pairs, scored_segment_pairs); - } - -private: +protected: std::unique_ptr ungapped_extender_; End2EndTestParam param_; CudaStream stream_; @@ -120,7 +93,29 @@ private: TEST_P(TestCudaextenderEnd2End, TestCorrectness) { - run_test(); + std::unique_ptr fasta_parser_target = io::create_kseq_fasta_parser(param_.target_file, 0, false); + const std::string target_sequence = fasta_parser_target->get_sequence_by_id(0).seq; + std::unique_ptr fasta_parser_query = io::create_kseq_fasta_parser(param_.query_file, 0, false); + const std::string query_sequence = fasta_parser_query->get_sequence_by_id(0).seq; + std::vector h_seed_pairs; + parse_seed_pairs(param_.seed_pairs_file, h_seed_pairs); + // Allocate pinned memory for query and target strings + pinned_host_vector h_encoded_target(target_sequence.length()); + pinned_host_vector h_encoded_query(target_sequence.length()); + encode_sequence(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); + encode_sequence(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); + ungapped_extender_->extend_async(h_encoded_query.data(), + get_size(h_encoded_query), + h_encoded_target.data(), + get_size(h_encoded_target), + param_.score_threshold, + h_seed_pairs); + // Parse golden scored_segment_pairs while extension is in progress + std::vector golden_scored_segment_pairs; + parse_scored_segment_pairs(param_.golden_scored_segment_pairs_file, golden_scored_segment_pairs); + ungapped_extender_->sync(); + std::vector scored_segment_pairs = ungapped_extender_->get_scored_segment_pairs(); + ASSERT_EQ(golden_scored_segment_pairs, scored_segment_pairs); } INSTANTIATE_TEST_SUITE_P(TestEnd2End, TestCudaextenderEnd2End, ::testing::ValuesIn(getCudaextenderEnd2EndTestCases())); From c705cd5c4328730aa66b81ab1f8268bcac0627bb Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 11:57:29 -0700 Subject: [PATCH 166/281] [cudaextender] Add attribution --- cudaextender/src/ungapped_xdrop.cu | 6 ++++++ cudaextender/src/ungapped_xdrop.cuh | 6 ++++++ cudaextender/src/ungapped_xdrop_kernels.cu | 7 ++++++- cudaextender/src/ungapped_xdrop_kernels.cuh | 6 ++++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 0cdb5248d..9803927a3 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -13,6 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* +* This algorithm was adapted from SegAlign's Ungapped Extender authored by +* Sneha Goenka (gsneha@stanford.edu) and Yatish Turakhia (yturakhi@uscs.edu). +* Source code for original implementation and use in SegAlign can be found +* here: https://github.com/gsneha26/SegAlign +*/ #include "ungapped_xdrop.cuh" #include "ungapped_xdrop_kernels.cuh" diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index bff14dec4..bb16901f4 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -13,6 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* +* This algorithm was adapted from SegAlign's Ungapped Extender authored by +* Sneha Goenka (gsneha@stanford.edu) and Yatish Turakhia (yturakhi@uscs.edu). +* Source code for original implementation and use in SegAlign can be found +* here: https://github.com/gsneha26/SegAlign +*/ #pragma once diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 3e5038b6b..7561e2952 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -13,6 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* +* This algorithm was adapted from SegAlign's Ungapped Extender authored by +* Sneha Goenka (gsneha@stanford.edu) and Yatish Turakhia (yturakhi@uscs.edu). +* Source code for original implementation and use in SegAlign can be found +* here: https://github.com/gsneha26/SegAlign +*/ #include "ungapped_xdrop_kernels.cuh" namespace claraparabricks @@ -23,7 +29,6 @@ namespace genomeworks namespace cudaextender { - // extend the seed values to a segment by ungapped x-drop method, adjust low-scoring // segment scores based on entropy factor, compare resulting segment scores // to score_threshold and update the d_scored_segment_pairs and d_done vectors diff --git a/cudaextender/src/ungapped_xdrop_kernels.cuh b/cudaextender/src/ungapped_xdrop_kernels.cuh index 782fc6fc0..befd9bea0 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cuh +++ b/cudaextender/src/ungapped_xdrop_kernels.cuh @@ -13,6 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/* +* This algorithm was adapted from SegAlign's Ungapped Extender authored by +* Sneha Goenka (gsneha@stanford.edu) and Yatish Turakhia (yturakhi@uscs.edu). +* Source code for original implementation and use in SegAlign can be found +* here: https://github.com/gsneha26/SegAlign +*/ #pragma once #include "ungapped_xdrop.cuh" From abedff8a77294e3969e9c4cd23e4c574fca66a56 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 12:02:04 -0700 Subject: [PATCH 167/281] Revert "[cudaextender] Add NUC2 as enum" This reverts commit b64c01862d3cf21ac1a989b4afbb236b4f4dfb6e. --- cudaextender/samples/sample_cudaextender.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index 0c7194f59..c4959157c 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -123,7 +123,7 @@ int main(int argc, char* argv[]) std::vector h_ssp; // Create Ungapped Extender Object for both API modes std::unique_ptr ungapped_extender = create_extender(score_matrix, - static_cast(Encoding::NUC2), + NUC2, xdrop_threshold, input_no_entropy, stream0.get(), From 4e54c99e9615e8a331e87857d191fc90cc268515 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 12:02:09 -0700 Subject: [PATCH 168/281] Revert "[cudaextender] enum class" This reverts commit cfb1daa00e5c1e5d825e79e27bece95f397fe7c6. --- .../genomeworks/cudaextender/utils.hpp | 41 +++++++++---------- cudaextender/samples/sample_cudaextender.cpp | 16 ++++---- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp index 44ffef162..e8d561a1d 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp @@ -32,6 +32,17 @@ namespace genomeworks namespace cudaextender { +constexpr int8_t A_NT = 0; +constexpr int8_t C_NT = 1; +constexpr int8_t G_NT = 2; +constexpr int8_t T_NT = 3; +constexpr int8_t L_NT = 4; +constexpr int8_t N_NT = 5; +constexpr int8_t X_NT = 6; +constexpr int8_t E_NT = 7; +constexpr int8_t NUC = 8; +constexpr int8_t NUC2 = NUC * NUC; + /// \brief Parses seed pairs from a csv file in the following format: /// target_position_in_read_1, query_position_in_read_1 /// target_position_in_read_2, query_position_in_read_2 @@ -87,20 +98,6 @@ inline void parse_scored_segment_pairs(const std::string& filepath, std::vector< } } } -/// \brief Integer encoding scheme for character sequence -enum class Encoding : int8_t -{ - A_NT = 0, - C_NT, - G_NT, - T_NT, - L_NT, - N_NT, - X_NT, - E_NT, - NUC, - NUC2 = NUC * NUC, -}; /// \brief Encodes character sequence as integer sequence /// @@ -115,32 +112,32 @@ inline void encode_sequence(int8_t* dst_seq, const char* src_seq, const int32_t switch (ch) { case 'A': - dst_seq[i] = static_cast(Encoding::A_NT); + dst_seq[i] = A_NT; break; case 'C': - dst_seq[i] = static_cast(Encoding::C_NT); + dst_seq[i] = C_NT; break; case 'G': - dst_seq[i] = static_cast(Encoding::G_NT); + dst_seq[i] = G_NT; break; case 'T': - dst_seq[i] = static_cast(Encoding::T_NT); + dst_seq[i] = T_NT; break; case '&': - dst_seq[i] = static_cast(Encoding::E_NT); + dst_seq[i] = E_NT; break; case 'n': case 'N': - dst_seq[i] = static_cast(Encoding::N_NT); + dst_seq[i] = N_NT; break; case 'a': case 'c': case 'g': case 't': - dst_seq[i] = static_cast(Encoding::L_NT); + dst_seq[i] = L_NT; break; default: - dst_seq[i] = static_cast(Encoding::X_NT); + dst_seq[i] = X_NT; break; } } diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index c4959157c..cefefa634 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -99,14 +99,14 @@ int main(int argc, char* argv[]) std::cerr << "Number of Seed Pairs: " << h_seed_pairs.size() << std::endl; // Define Scoring Matrix - int32_t score_matrix[static_cast(Encoding::NUC2)] = {91, -114, -31, -123, -1000, -1000, -100, -9100, - -114, 100, -125, -31, -1000, -1000, -100, -9100, - -31, -125, 100, -114, -1000, -1000, -100, -9100, - -123, -31, -114, 91, -1000, -1000, -100, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -100, -100, -100, -100, -1000, -1000, -100, -9100, - -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; + int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, + -114, 100, -125, -31, -1000, -1000, -100, -9100, + -31, -125, 100, -114, -1000, -1000, -100, -9100, + -123, -31, -114, 91, -1000, -1000, -100, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -100, -100, -100, -100, -1000, -1000, -100, -9100, + -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; // Allocate pinned memory for query and target strings pinned_host_vector h_encoded_target(target_sequence.length()); From c618f23517952f345a734f4d2151a850f029e826 Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 28 Sep 2020 12:04:44 -0700 Subject: [PATCH 169/281] [cudaextender] Revert enum to constexpr globals --- cudaextender/src/ungapped_xdrop.cuh | 1 - .../tests/Test_CudaextenderEnd2End.cu | 30 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index bb16901f4..3308e6ec1 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -19,7 +19,6 @@ * Source code for original implementation and use in SegAlign can be found * here: https://github.com/gsneha26/SegAlign */ - #pragma once #include diff --git a/cudaextender/tests/Test_CudaextenderEnd2End.cu b/cudaextender/tests/Test_CudaextenderEnd2End.cu index 6ac8cec3f..c818d77d6 100644 --- a/cudaextender/tests/Test_CudaextenderEnd2End.cu +++ b/cudaextender/tests/Test_CudaextenderEnd2End.cu @@ -62,21 +62,21 @@ public: { param_ = GetParam(); // Define Scoring Matrix - int32_t score_matrix[static_cast(Encoding::NUC2)] = {91, -114, -31, -123, -1000, -1000, -100, -9100, - -114, 100, -125, -31, -1000, -1000, -100, -9100, - -31, -125, 100, -114, -1000, -1000, -100, -9100, - -123, -31, -114, 91, -1000, -1000, -100, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -100, -100, -100, -100, -1000, -1000, -100, -9100, - -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; - const int32_t xdrop_threshold = 910; - const int32_t device_id = 0; - const bool no_entropy = false; - const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); - allocator_ = create_default_device_allocator(max_gpu_memory); - stream_ = make_cuda_stream(); - ungapped_extender_ = create_extender(score_matrix, static_cast(Encoding::NUC2), xdrop_threshold, no_entropy, stream_.get(), device_id, allocator_); + int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, + -114, 100, -125, -31, -1000, -1000, -100, -9100, + -31, -125, 100, -114, -1000, -1000, -100, -9100, + -123, -31, -114, 91, -1000, -1000, -100, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -100, -100, -100, -100, -1000, -1000, -100, -9100, + -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; + const int32_t xdrop_threshold = 910; + const int32_t device_id = 0; + const bool no_entropy = false; + const std::size_t max_gpu_memory = cudautils::find_largest_contiguous_device_memory_section(); + allocator_ = create_default_device_allocator(max_gpu_memory); + stream_ = make_cuda_stream(); + ungapped_extender_ = create_extender(score_matrix, NUC2, xdrop_threshold, no_entropy, stream_.get(), device_id, allocator_); } void TearDown() From 6528bebe407fa098c57a3bec7e73d10e88ca23d9 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Tue, 29 Sep 2020 08:58:11 -0700 Subject: [PATCH 170/281] [cudaextender] Add consts and make get_size() uniform --- .../genomeworks/cudaextender/extender.hpp | 12 ++--- cudaextender/samples/sample_cudaextender.cpp | 44 +++++++++---------- cudaextender/src/extender.cpp | 2 +- cudaextender/src/ungapped_xdrop.cu | 14 +++--- cudaextender/src/ungapped_xdrop.cuh | 12 ++--- 5 files changed, 41 insertions(+), 43 deletions(-) diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp index 142dd9085..ee4fcc05e 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/extender.hpp @@ -79,8 +79,8 @@ class Extender /// \param[in] h_seed_pairs Vector of seed pairs mapping to query and target /// sequences /// \return Status of the async extension launch - virtual StatusType extend_async(int8_t* h_query, int32_t query_length, - int8_t* h_target, int32_t target_length, + virtual StatusType extend_async(const int8_t* h_query, int32_t query_length, + const int8_t* h_target, int32_t target_length, int32_t score_threshold, const std::vector& h_seed_pairs) = 0; @@ -105,9 +105,9 @@ class Extender /// \param[out] d_num_scored_segment_pairs Pointer to pre-allocated device location for /// storing length of extension output /// \return Status of the async extension launch - virtual StatusType extend_async(int8_t* d_query, int32_t query_length, - int8_t* d_target, int32_t target_length, - int32_t score_threshold, SeedPair* d_seed_pairs, + virtual StatusType extend_async(const int8_t* d_query, int32_t query_length, + const int8_t* d_target, int32_t target_length, + int32_t score_threshold, const SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) = 0; @@ -144,7 +144,7 @@ class Extender /// \param allocator DeviceAllocator to be used for allocating/freeing memory /// \param type Type of extension to be performed /// \return Unique pointer to Extender object. -std::unique_ptr create_extender(int32_t* h_score_mat, +std::unique_ptr create_extender(const int32_t* h_score_mat, int32_t score_mat_dim, int32_t xdrop_threshold, bool no_entropy, diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index cefefa634..5d4e83507 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -34,7 +34,7 @@ using namespace cudaextender; static void print_scored_segment_pairs(const std::vector& scored_segment_pairs) { std::cout << "Target Position, Query Position, Length, Score" << std::endl; - for (auto& segment : scored_segment_pairs) + for (const auto& segment : scored_segment_pairs) { std::cout << segment.seed_pair.target_position_in_read << "," << segment.seed_pair.query_position_in_read << "," << segment.length << "," << segment.score << std::endl; @@ -99,14 +99,14 @@ int main(int argc, char* argv[]) std::cerr << "Number of Seed Pairs: " << h_seed_pairs.size() << std::endl; // Define Scoring Matrix - int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, - -114, 100, -125, -31, -1000, -1000, -100, -9100, - -31, -125, 100, -114, -1000, -1000, -100, -9100, - -123, -31, -114, 91, -1000, -1000, -100, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, - -100, -100, -100, -100, -1000, -1000, -100, -9100, - -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; + const int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, + -114, 100, -125, -31, -1000, -1000, -100, -9100, + -31, -125, 100, -114, -1000, -1000, -100, -9100, + -123, -31, -114, 91, -1000, -1000, -100, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -9100, + -100, -100, -100, -100, -1000, -1000, -100, -9100, + -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; // Allocate pinned memory for query and target strings pinned_host_vector h_encoded_target(target_sequence.length()); @@ -132,9 +132,9 @@ int main(int argc, char* argv[]) if (!device_ptr_api_mode) { ungapped_extender->extend_async(h_encoded_query.data(), - get_size(h_encoded_query), + get_size(h_encoded_query), h_encoded_target.data(), - get_size(h_encoded_target), + get_size(h_encoded_target), score_threshold, h_seed_pairs); ungapped_extender->sync(); @@ -145,27 +145,27 @@ int main(int argc, char* argv[]) // Allocate space on device for target and query sequences, seed_pairs, // scored segment pairs (ssp) and num_ssp using default allocator (caching) // Allocate space for query and target sequences - device_buffer d_query(query_sequence.length(), allocator, stream0.get()); - device_buffer d_target(target_sequence.length(), allocator, stream0.get()); + device_buffer d_query(get_size(query_sequence), allocator, stream0.get()); + device_buffer d_target(get_size(target_sequence), allocator, stream0.get()); // Allocate space for SeedPair input - device_buffer d_seed_pairs(h_seed_pairs.size(), allocator, stream0.get()); + device_buffer d_seed_pairs(get_size(h_seed_pairs), allocator, stream0.get()); // Allocate space for ScoredSegmentPair output - device_buffer d_ssp(h_seed_pairs.size(), allocator, stream0.get()); + device_buffer d_ssp(get_size(h_seed_pairs), allocator, stream0.get()); device_buffer d_num_ssp(1, allocator, stream0.get()); // Async Memcopy all the input values to device - device_copy_n(h_encoded_query.data(), query_sequence.length(), d_query.data(), stream0.get()); - device_copy_n(h_encoded_target.data(), target_sequence.length(), d_target.data(), stream0.get()); - device_copy_n(h_seed_pairs.data(), h_seed_pairs.size(), d_seed_pairs.data(), stream0.get()); + device_copy_n(h_encoded_query.data(), get_size(query_sequence), d_query.data(), stream0.get()); + device_copy_n(h_encoded_target.data(), get_size(target_sequence), d_target.data(), stream0.get()); + device_copy_n(h_seed_pairs.data(), get_size(h_seed_pairs), d_seed_pairs.data(), stream0.get()); // Launch the ungapped extender device pointer function ungapped_extender->extend_async(d_query.data(), - get_size(d_query), + get_size(d_query), d_target.data(), - get_size(d_target), + get_size(d_target), score_threshold, d_seed_pairs.data(), - d_seed_pairs.size(), + get_size(d_seed_pairs), d_ssp.data(), d_num_ssp.data()); @@ -177,7 +177,7 @@ int main(int argc, char* argv[]) device_copy_n(d_ssp.data(), h_num_ssp, h_ssp.data(), stream0.get()); cudaStreamSynchronize(stream0.get()); } - std::cerr << "Number of Scored Segment Pairs found: " << h_ssp.size() << std::endl; + std::cerr << "Number of Scored Segment Pairs found: " << get_size(h_ssp) << std::endl; if (print) { print_scored_segment_pairs(h_ssp); diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index d4aaf8dd1..442cf2d94 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -27,7 +27,7 @@ namespace cudaextender { // TODO - Provide an allocator-less API for creation? -std::unique_ptr create_extender(int32_t* h_score_mat, +std::unique_ptr create_extender(const int32_t* h_score_mat, int32_t score_mat_dim, int32_t xdrop_threshold, bool no_entropy, diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 9803927a3..0e122c6f2 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -42,7 +42,7 @@ namespace cudaextender using namespace cudautils; -UngappedXDrop::UngappedXDrop(int32_t* h_score_mat, const int32_t score_mat_dim, +UngappedXDrop::UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat_dim, const int32_t xdrop_threshold, const bool no_entropy, cudaStream_t stream, const int32_t device_id, DefaultDeviceAllocator allocator) @@ -89,14 +89,12 @@ UngappedXDrop::UngappedXDrop(int32_t* h_score_mat, const int32_t score_mat_dim, d_done_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); d_tmp_ssp_ = device_buffer(batch_max_ungapped_extensions_, allocator_, stream_); d_temp_storage_cub_ = device_buffer(cub_storage_bytes, allocator_, stream_); - - // Requires pinned host memory registration for proper async behavior device_copy_n(h_score_mat_.data(), score_mat_dim_, d_score_mat_.data(), stream_); } -StatusType UngappedXDrop::extend_async(int8_t* d_query, const int32_t query_length, - int8_t* d_target, const int32_t target_length, - const int32_t score_threshold, SeedPair* d_seed_pairs, +StatusType UngappedXDrop::extend_async(const int8_t* d_query, const int32_t query_length, + const int8_t* d_target, const int32_t target_length, + const int32_t score_threshold, const SeedPair* d_seed_pairs, const int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) { @@ -169,8 +167,8 @@ StatusType UngappedXDrop::extend_async(int8_t* d_query, const int32_t query_leng return StatusType::success; } -StatusType UngappedXDrop::extend_async(int8_t* h_query, const int32_t query_length, - int8_t* h_target, const int32_t target_length, +StatusType UngappedXDrop::extend_async(const int8_t* h_query, const int32_t query_length, + const int8_t* h_target, const int32_t target_length, const int32_t score_threshold, const std::vector& h_seed_pairs) { diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 3308e6ec1..f19d750f9 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -37,7 +37,7 @@ namespace cudaextender class UngappedXDrop : public Extender { public: - UngappedXDrop(int32_t* h_score_mat, + UngappedXDrop(const int32_t* h_score_mat, int32_t score_mat_dim, int32_t xdrop_threshold, bool no_entropy, @@ -46,14 +46,14 @@ public: DefaultDeviceAllocator allocator); ~UngappedXDrop() override; - StatusType extend_async(int8_t* h_query, int32_t query_length, - int8_t* h_target, int32_t target_length, + StatusType extend_async(const int8_t* h_query, int32_t query_length, + const int8_t* h_target, int32_t target_length, int32_t score_threshold, const std::vector& h_seed_pairs) override; - StatusType extend_async(int8_t* d_query, int32_t query_length, - int8_t* d_target, int32_t target_length, - int32_t score_threshold, SeedPair* d_seed_pairs, + StatusType extend_async(const int8_t* d_query, int32_t query_length, + const int8_t* d_target, int32_t target_length, + int32_t score_threshold, const SeedPair* d_seed_pairs, int32_t num_seed_pairs, ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_num_scored_segment_pairs) override; From e011a73eb9003b78c06514cbaa1c95e1b5a59b56 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Tue, 29 Sep 2020 09:09:09 -0700 Subject: [PATCH 171/281] [cudaextender] Replace length() with get_size(); Fix bug with query/target length swap --- cudaextender/samples/sample_cudaextender.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index 5d4e83507..a91c21415 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -96,7 +96,7 @@ int main(int argc, char* argv[]) // Following function loops through all seed_pairs in the sample_seed_pairs.csv and returns // results in the passed vector parse_seed_pairs(seed_pairs_file_path, h_seed_pairs); - std::cerr << "Number of Seed Pairs: " << h_seed_pairs.size() << std::endl; + std::cerr << "Number of Seed Pairs: " << get_size(h_seed_pairs) << std::endl; // Define Scoring Matrix const int32_t score_matrix[NUC2] = {91, -114, -31, -123, -1000, -1000, -100, -9100, @@ -109,11 +109,11 @@ int main(int argc, char* argv[]) -9100, -9100, -9100, -9100, -9100, -9100, -9100, -9100}; // Allocate pinned memory for query and target strings - pinned_host_vector h_encoded_target(target_sequence.length()); - pinned_host_vector h_encoded_query(target_sequence.length()); + pinned_host_vector h_encoded_target(get_size(target_sequence)); + pinned_host_vector h_encoded_query(get_size(query_sequence)); - encode_sequence(h_encoded_target.data(), target_sequence.c_str(), target_sequence.length()); - encode_sequence(h_encoded_query.data(), query_sequence.c_str(), query_sequence.length()); + encode_sequence(h_encoded_target.data(), target_sequence.c_str(), get_size(target_sequence)); + encode_sequence(h_encoded_query.data(), query_sequence.c_str(), get_size(query_sequence)); // Create a stream for async use CudaStream stream0 = make_cuda_stream(); // Create an allocator for use with both APIs From fdba890ed89fd42cc5b116cd8377ea868be1430a Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 29 Sep 2020 17:37:40 -0400 Subject: [PATCH 172/281] [cudapoa] a couple of small changes reducing 4 to 12 registers in nw-adaptive-tb --- cudapoa/src/cudapoa_kernels.cuh | 2 +- cudapoa/src/cudapoa_nw_tb_banded.cuh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 606e509bc..99a2373f5 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -407,7 +407,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) __syncwarp(); } // Full band ------------------------------------------------------------------------------------------- - else // BM == BandMode::full_band + else if (BM == BandMode::full_band) { alignment_length = runNeedlemanWunsch(nodes, sorted_poa, diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index 8e9d33742..334f4a12c 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -62,8 +62,9 @@ __device__ __forceinline__ void set_score_tb(ScoreT* scores, col_idx = column - band_start; } // in NW with traceback buffer, score matrix is stored partially, hence row is mapped to [0, score_matrix_height) span + row = row % score_matrix_height; int64_t score_index = static_cast(col_idx) + - static_cast(row % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + static_cast(row) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[score_index] = value; } From 68ba6ed0a6a99397161957a23a07f1a4fb342b8f Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 29 Sep 2020 18:58:03 -0400 Subject: [PATCH 173/281] [cudapoa] further reduced up to 6 registers in nw-adaptive-tb with a minor change --- cudapoa/src/cudapoa_nw_tb_banded.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index 334f4a12c..ce4e002c4 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -184,7 +184,7 @@ __device__ __forceinline__ void get_scores_tb(ScoreT* scores, // need to load the next chunk of memory as well ScoreT4 score4_next = pred_scores[1]; - ScoreT char_profile = (graph_base == read4.r0 ? match_score : mismatch_score); + int32_t char_profile = (graph_base == read4.r0 ? match_score : mismatch_score); // if trace is diogonal, its value is positive and if vertical, negative // update score.s0, trace.t0 ---------- From 15de0f8fdafa0bab5661b9089f1eaeb7ebb135a8 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 30 Sep 2020 19:31:14 +0200 Subject: [PATCH 174/281] [cudamapper] IndexCache split into HostIndexCache and DeviceIndexCache --- cudamapper/src/index_cache.cu | 255 ++++++++------ cudamapper/src/index_cache.cuh | 286 +++++++++------- cudamapper/src/main.cu | 72 ++-- cudamapper/tests/Test_CudamapperIndexCache.cu | 320 +++++++++--------- 4 files changed, 513 insertions(+), 420 deletions(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index eb1fefe93..1ef7abe4d 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -32,97 +32,119 @@ namespace genomeworks namespace cudamapper { -IndexCache::IndexCache(bool same_query_and_target, - genomeworks::DefaultDeviceAllocator allocator, - std::shared_ptr query_parser, - std::shared_ptr target_parser, - std::uint64_t kmer_size, - std::uint64_t window_size, - bool hash_representations, - double filtering_parameter, - cudaStream_t cuda_stream_generation, - cudaStream_t cuda_stream_copy) - : same_query_and_target_(same_query_and_target) - , allocator_(allocator) - , query_parser_(query_parser) - , target_parser_(target_parser) - , kmer_size_(kmer_size) - , window_size_(window_size) - , hash_representations_(hash_representations) - , filtering_parameter_(filtering_parameter) - , cuda_stream_generation_(cuda_stream_generation) - , cuda_stream_copy_(cuda_stream_copy) +DeviceIndexCache::DeviceIndexCache(const CacheType cache_type, + HostIndexCache* host_cache) + : cache_type_(cache_type) + , host_cache_(host_cache) + , is_ready_(false) { + host_cache_->register_device_cache(cache_type_, + this); } -void IndexCache::generate_content_query_host(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - const bool skip_copy_to_host) +DeviceIndexCache::~DeviceIndexCache() { - generate_content_host(descriptors_of_indices_to_cache, - descriptors_of_indices_to_keep_on_device, - skip_copy_to_host, - CacheSelector::query_cache); + host_cache_->deregister_device_cache(cache_type_, + this); } -void IndexCache::generate_content_target_host(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - const bool skip_copy_to_host) +void DeviceIndexCache::add_index(const IndexDescriptor index_descriptor, + std::shared_ptr device_index) { - generate_content_host(descriptors_of_indices_to_cache, - descriptors_of_indices_to_keep_on_device, - skip_copy_to_host, - CacheSelector::target_cache); + cache_[index_descriptor] = device_index; } -void IndexCache::start_generating_content_query_device(const std::vector& descriptors_of_indices_to_cache) +std::shared_ptr DeviceIndexCache::get_index(const IndexDescriptor index_descriptor) const { - start_generating_content_device(descriptors_of_indices_to_cache, - CacheSelector::query_cache); + if (!is_ready_) + { + throw DeviceCacheNotReadyException(cache_type_, + index_descriptor); + } + + const auto index_iter = cache_.find(index_descriptor); + if (index_iter == cache_.end()) + { + throw IndexNotFoundException(cache_type_, + IndexNotFoundException::IndexLocation::device_cache, + index_descriptor); + } + return index_iter->second; } -void IndexCache::finish_generating_content_query_device() +std::shared_ptr DeviceIndexCache::get_index_no_check_if_ready(IndexDescriptor index_descriptor) const { - finish_generating_content_device(CacheSelector::query_cache); + const auto index_iter = cache_.find(index_descriptor); + if (index_iter == cache_.end()) + { + throw IndexNotFoundException(cache_type_, + IndexNotFoundException::IndexLocation::device_cache, + index_descriptor); + } + return index_iter->second; } -void IndexCache::start_generating_content_target_device(const std::vector& descriptors_of_indices_to_cache) +bool DeviceIndexCache::has_index(const IndexDescriptor index_descriptor) const { - start_generating_content_device(descriptors_of_indices_to_cache, - CacheSelector::target_cache); + // TODO: use optional instead + return 0 != cache_.count(index_descriptor); } -void IndexCache::finish_generating_content_target_device() +void DeviceIndexCache::wait_for_data_to_be_ready() { - finish_generating_content_device(CacheSelector::target_cache); + if (!is_ready_) + { + for (const auto& index_it : cache_) + { + index_it.second->wait_to_be_ready(); + } + + is_ready_ = true; + } } -std::shared_ptr IndexCache::get_index_from_query_cache(const IndexDescriptor& index_descriptor) const +bool DeviceIndexCache::is_ready() const { - return get_index_from_cache(index_descriptor, - CacheSelector::query_cache); + return is_ready_; } -std::shared_ptr IndexCache::get_index_from_target_cache(const IndexDescriptor& index_descriptor) const +HostIndexCache::HostIndexCache(const bool same_query_and_target, + genomeworks::DefaultDeviceAllocator allocator, + std::shared_ptr query_parser, + std::shared_ptr target_parser, + const std::uint64_t kmer_size, + const std::uint64_t window_size, + const bool hash_representations, + const double filtering_parameter, + cudaStream_t cuda_stream_generation, + cudaStream_t cuda_stream_copy) + : same_query_and_target_(same_query_and_target) + , allocator_(allocator) + , query_parser_(query_parser) + , target_parser_(target_parser) + , kmer_size_(kmer_size) + , window_size_(window_size) + , hash_representations_(hash_representations) + , filtering_parameter_(filtering_parameter) + , cuda_stream_generation_(cuda_stream_generation) + , cuda_stream_copy_(cuda_stream_copy) { - return get_index_from_cache(index_descriptor, - CacheSelector::target_cache); } -void IndexCache::generate_content_host(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - const bool skip_copy_to_host, - const CacheSelector which_cache) +void HostIndexCache::generate_content(const CacheType cache_type, + const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device, + const bool skip_copy_to_host) { // skip_copy_to_host only makes sense if descriptors_of_indices_to_cache and descriptors_of_indices_to_keep_on_device are the same // otherwise some indices would be created and not saved on either host or device assert(!skip_copy_to_host || (descriptors_of_indices_to_cache == descriptors_of_indices_to_keep_on_device)); assert(!descriptors_of_indices_to_cache.empty()); - host_cache_t& this_cache = (CacheSelector::query_cache == which_cache) ? query_host_cache_ : target_host_cache_; - const host_cache_t& other_cache = (CacheSelector::query_cache == which_cache) ? target_host_cache_ : query_host_cache_; - device_cache_t& indices_kept_on_device = (CacheSelector::query_cache == which_cache) ? query_indices_kept_on_device_ : target_indices_kept_on_device_; - const genomeworks::io::FastaParser* parser = (CacheSelector::query_cache == which_cache) ? query_parser_.get() : target_parser_.get(); + host_cache_t& this_cache = (CacheType::query_cache == cache_type) ? query_host_cache_ : target_host_cache_; + const host_cache_t& other_cache = (CacheType::query_cache == cache_type) ? target_host_cache_ : query_host_cache_; + device_cache_t& indices_kept_on_device = (CacheType::query_cache == cache_type) ? query_indices_kept_on_device_ : target_indices_kept_on_device_; + const genomeworks::io::FastaParser* parser = (CacheType::query_cache == cache_type) ? query_parser_.get() : target_parser_.get(); // convert descriptors_of_indices_to_keep_on_device into set for faster search std::unordered_set descriptors_of_indices_to_keep_on_device_set(begin(descriptors_of_indices_to_keep_on_device), @@ -207,7 +229,7 @@ void IndexCache::generate_content_host(const std::vector& descr } } - if (host_copy_needed) + if (index_on_host) { assert(index_on_host); new_cache[descriptor_of_index_to_cache] = index_on_host; @@ -256,100 +278,129 @@ void IndexCache::generate_content_host(const std::vector& descr std::swap(new_cache, this_cache); } -void IndexCache::start_generating_content_device(const std::vector& descriptors_of_indices_to_cache, - const CacheSelector which_cache) +std::shared_ptr HostIndexCache::start_copying_indices_to_device(const CacheType cache_type, + const std::vector& descriptors_of_indices_to_cache) { - const host_cache_t& host_cache = (CacheSelector::query_cache == which_cache) ? query_host_cache_ : target_host_cache_; - device_cache_t& this_device_cache = (CacheSelector::query_cache == which_cache) ? query_device_cache_ : target_device_cache_; - const device_cache_t& other_device_cache = (CacheSelector::query_cache == which_cache) ? target_device_cache_ : query_device_cache_; - device_cache_t& indices_kept_on_device = (CacheSelector::query_cache == which_cache) ? query_indices_kept_on_device_ : target_indices_kept_on_device_; - device_cache_t& new_cache = (CacheSelector::query_cache == which_cache) ? next_query_device_cache_ : next_target_device_cache_; + const host_cache_t& host_cache = (CacheType::query_cache == cache_type) ? query_host_cache_ : target_host_cache_; + const std::vector& this_device_caches = (CacheType::query_cache == cache_type) ? device_caches_query_ : device_caches_target_; + const std::vector& other_device_caches = (CacheType::query_cache == cache_type) ? device_caches_target_ : device_caches_query_; + device_cache_t& indices_kept_on_device = (CacheType::query_cache == cache_type) ? query_indices_kept_on_device_ : target_indices_kept_on_device_; - for (const IndexDescriptor& index_descriptor : descriptors_of_indices_to_cache) + std::shared_ptr device_cache = std::make_shared(cache_type, + this); + + for (const IndexDescriptor& descriptor : descriptors_of_indices_to_cache) { std::shared_ptr device_index = nullptr; // check if index was kept on device after creation - auto index_kept_on_device = indices_kept_on_device.find(index_descriptor); + auto index_kept_on_device = indices_kept_on_device.find(descriptor); if (index_kept_on_device != indices_kept_on_device.end()) { device_index = index_kept_on_device->second; - indices_kept_on_device.erase(index_descriptor); + indices_kept_on_device.erase(descriptor); } // check if value is already in this cache if (!device_index) { - auto index_in_this_cache = this_device_cache.find(index_descriptor); - if (index_in_this_cache != this_device_cache.end()) + for (const DeviceIndexCache* const existing_cache : this_device_caches) { - device_index = index_in_this_cache->second; + if (existing_cache != device_cache.get() && existing_cache->has_index(descriptor)) + { + device_index = existing_cache->get_index_no_check_if_ready(descriptor); + break; + } } } // if query and target files are the same check the other index as well if (!device_index && same_query_and_target_) { - auto index_in_other_cache = other_device_cache.find(index_descriptor); - if (index_in_other_cache != other_device_cache.end()) + for (const DeviceIndexCache* const existing_cache : other_device_caches) { - device_index = index_in_other_cache->second; + if (existing_cache->has_index(descriptor)) + { + device_index = existing_cache->get_index_no_check_if_ready(descriptor); + break; + } } } // if index has not been found on device copy it from host if (!device_index) { - const auto index_on_host_iter = host_cache.find(index_descriptor); + const auto index_on_host_iter = host_cache.find(descriptor); if (index_on_host_iter == host_cache.end()) { - throw IndexNotFoundException(index_descriptor, - IndexNotFoundException::IndexType::host_cache); + throw IndexNotFoundException(cache_type, + IndexNotFoundException::IndexLocation::host_cache, + descriptor); } device_index = index_on_host_iter->second->copy_index_to_device(allocator_, cuda_stream_copy_); } assert(device_index); - new_cache[index_descriptor] = device_index; + device_cache->add_index(descriptor, + device_index); } + + return device_cache; } -void IndexCache::finish_generating_content_device(const CacheSelector which_cache) +void HostIndexCache::register_device_cache(const CacheType cache_type, + DeviceIndexCache* index_cache) { - device_cache_t& this_device_cache = (CacheSelector::query_cache == which_cache) ? query_device_cache_ : target_device_cache_; - device_cache_t& new_cache = (CacheSelector::query_cache == which_cache) ? next_query_device_cache_ : next_target_device_cache_; + assert(cache_type == CacheType::query_cache || cache_type == CacheType::target_cache); - for (const auto& device_index : new_cache) - { - device_index.second->wait_to_be_ready(); - } + std::vector& device_caches = cache_type == CacheType::query_cache ? device_caches_query_ : device_caches_target_; - this_device_cache.clear(); - std::swap(this_device_cache, new_cache); + device_caches.push_back(index_cache); } -std::shared_ptr IndexCache::get_index_from_cache(const IndexDescriptor& index_descriptor, - const CacheSelector which_cache) const +void HostIndexCache::deregister_device_cache(const CacheType cache_type, + DeviceIndexCache* index_cache) { - const device_cache_t& this_device_cache = (CacheSelector::query_cache == which_cache) ? query_device_cache_ : target_device_cache_; - const auto index_iter = this_device_cache.find(index_descriptor); - if (index_iter == this_device_cache.end()) - { - throw IndexNotFoundException(index_descriptor, - IndexNotFoundException::IndexType::device_cache); - } - return index_iter->second; + assert(cache_type == CacheType::query_cache || cache_type == CacheType::target_cache); + + std::vector& device_caches = cache_type == CacheType::query_cache ? device_caches_query_ : device_caches_target_; + + auto new_end = std::remove(begin(device_caches), end(device_caches), index_cache); + device_caches.erase(new_end, end(device_caches)); } -IndexNotFoundException::IndexNotFoundException(IndexDescriptor index_descriptor, - IndexType index_type) - : error_message_("Index not found in " + std::string(index_type == IndexType::host_cache ? "host" : "device") + " cache. First read: " + std::to_string(index_descriptor.first_read()) + ", number of reads: " + std::to_string(index_descriptor.number_of_reads())) +IndexNotFoundException::IndexNotFoundException(const CacheType cache_type, + const IndexLocation index_location, + const IndexDescriptor index_descriptor) + : message_(std::string(cache_type == CacheType::query_cache ? "Query " : "Target ") + + "index not found in " + + std::string(index_location == IndexLocation::host_cache ? "host " : "device ") + + "cache. First read: " + + std::to_string(index_descriptor.first_read()) + + ", number of reads: " + + std::to_string(index_descriptor.number_of_reads())) { } const char* IndexNotFoundException::what() const noexcept { - return error_message_.c_str(); + return message_.c_str(); +} + +DeviceCacheNotReadyException::DeviceCacheNotReadyException(const CacheType cache_type, + const IndexDescriptor index_descriptor) + : message_("Cache for " + + std::string(cache_type == CacheType::query_cache ? "query " : "target ") + + "index is not ready. First read: " + + std::to_string(index_descriptor.first_read()) + + ", number of reads: " + + std::to_string(index_descriptor.number_of_reads())) +{ +} + +const char* DeviceCacheNotReadyException::what() const noexcept +{ + return message_.c_str(); } } // namespace cudamapper diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index 4128ad36c..eefbace20 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -40,20 +40,96 @@ namespace cudamapper { class IndexHostCopyBase; +class HostIndexCache; -/// IndexCache - Creates indices, stores them in host memory and on demands moves them to device memory +/// CacheType - Specifies if query or target cache +enum class CacheType +{ + query_cache, + target_cache +}; + +/// DeviceIndexCache - Owns copies of indices on device +/// +/// These object are created by HostIndexCache::start_copying_indices_to_device() +class DeviceIndexCache +{ +public: + /// \brief Constructor + /// \param cache_type + /// \param host_cache HostIndexCache that created this object + DeviceIndexCache(CacheType cache_type, + HostIndexCache* host_cache); + + DeviceIndexCache(const DeviceIndexCache&) = delete; + DeviceIndexCache(DeviceIndexCache&&) = delete; + DeviceIndexCache& operator=(const DeviceIndexCache&) = delete; + DeviceIndexCache& operator=(DeviceIndexCache&&) = delete; + + /// \brief Destructor + ~DeviceIndexCache(); + + /// \brief Adds index to host, should only be called by HostIndexCache::start_copying_indices_to_device() + /// \param index_descriptor + /// \param device_index + void add_index(IndexDescriptor index_descriptor, + std::shared_ptr device_index); + + /// \brief Returns requested index + /// Calling this function before wait_for_data_to_be_ready() results in an exception + /// \param index_descriptor + /// \throw IndexNotFoundException if requested index is not cached + /// \throw DeviceCacheNotReadyException is cache is not ready, i.e. wait_for_data_to_be_ready() has not been called yet + /// \return requested index + std::shared_ptr get_index(IndexDescriptor index_descriptor) const; + + /// \brief Returns requested index, returned index might not be ready and has to be synchronized directly + /// \param index_descriptor + /// \throw IndexNotFoundException if requested index is not cached + /// \return requested index + std::shared_ptr get_index_no_check_if_ready(IndexDescriptor index_descriptor) const; + + /// \brief Returns whether given index is present in cache + /// \param index_descriptor + /// \return is given index present in cache + bool has_index(IndexDescriptor index_descriptor) const; + + /// \brief Waits for indices to be copied from host memory. Must be called before get_index() + void wait_for_data_to_be_ready(); + + /// \brief Returns whether indices have been copied to device and get_index() can be called, i.e. whether wait_for_data_to_be_ready() has already been called + /// \return whether indices have been copied to device + bool is_ready() const; + +private: + using device_cache_t = std::unordered_map, + IndexDescriptorHash>; + + device_cache_t cache_; + + CacheType cache_type_; + // HostIndexCache which created this DeviceIndexCache + HostIndexCache* host_cache_; + + // wait_for_data_to_be_ready + bool is_ready_; +}; + +/// HostIndexCache - Creates indices, stores them in host memory and on demands moves them to device memory /// -/// Class contains separate caches for query and target. In this example query cache is going to be used, target cache is equivalent. -/// The user generates indices and stores them in host memory using generate_content_query_host(). The user then copies some of those indices -/// to device memory using start_generating_content_query_device() and finish_generating_content_query_device(). It is user's responsibility -/// to make sure that indices requested by start_generating_content_query_device() were generated by generate_content_query_host(). -/// Memory copy to device is done asynchronously, the user should make sure that every call to start_generating_content_query_device() is -/// followed by a call finish_generating_content_query_device(). +/// Class contains separate caches for query and target. The user chooses between query and target by specifying CacheType in function calls. +/// The user generates indices and stores them in host memory using generate_content(). The user then copies some of those indices +/// to device memory using start_copying_indices_to_device() and the function returns a pointer to DeviceIndexCache. To wait for indices to be +/// fully copied one should call DeviceIndexCache::wait_for_data_to_be_ready(). +/// It is user's responsibility to make sure that indices requested by start_copying_indices_to_device() were generated by generate_content(). +/// Memory copy to device is done asynchronously, the user should make sure that every call to start_copying_indices_to_device() is +/// accompanied by a call DeviceIndexCache::wait_for_data_to_be_ready(). /// The class tries to minimize the number of index creation and movemens, e.g. by reusing already existing indices, but not guarantees are given. -class IndexCache +class HostIndexCache { public: - /// \brief Constructor only initializes cache, no index is generated at this point + /// \brief Constructor only initializes cache, no index is generated at this point, generate_content() does that /// /// \param same_query_and_target true means that both query and target files are the same, meaning that if some index exists in query cache it can also be used by target cache directly /// \param allocator allocator to use for device arrays @@ -65,82 +141,66 @@ public: /// \param filtering_parameter see Index /// \param cuda_stream_generation index generation is done one this stream, device memory in resulting device copies of index will only we freed once all previously scheduled work on this stream has finished /// \param cuda_stream_copy D2H and H2D copies of indices will be done on this stream, device memory in resulting device copies of index will only we freed once all previously scheduled work on this stream has finished - IndexCache(bool same_query_and_target, - genomeworks::DefaultDeviceAllocator allocator, - std::shared_ptr query_parser, - std::shared_ptr target_parser, - std::uint64_t kmer_size, - std::uint64_t window_size, - bool hash_representations = true, - double filtering_parameter = 1.0, - cudaStream_t cuda_stream_generation = 0, - cudaStream_t cuda_stream_copy = 0); + HostIndexCache(bool same_query_and_target, + genomeworks::DefaultDeviceAllocator allocator, + std::shared_ptr query_parser, + std::shared_ptr target_parser, + std::uint64_t kmer_size, + std::uint64_t window_size, + bool hash_representations = true, + double filtering_parameter = 1.0, + cudaStream_t cuda_stream_generation = 0, + cudaStream_t cuda_stream_copy = 0); + + HostIndexCache(const HostIndexCache&) = delete; + HostIndexCache(HostIndexCache&&) = delete; + HostIndexCache& operator=(const HostIndexCache&) = delete; + HostIndexCache& operator=(HostIndexCache&&) = delete; + ~HostIndexCache() = default; /// \brief Generates indices on device and copies them to host memory /// /// If index already exists on host is may be reused. /// Indices from descriptors_of_indices_to_keep_on_device will be kept on device in addition to being to host. This is useful if the same indices - /// are going to be requested by start_generating_content_query_device() immediately after this call. - /// If skip_copy_to_host is ture indices are going to be kept on device and not copied to host. In that case descriptors_of_indices_to_cache must - /// be equal to descriptors_of_indices_to_keep_on_device and there must be only one call to start_generating_content_query_device() - /// - /// \param descriptors_of_indices_to_cache - /// \param descriptors_of_indices_to_keep_on_device - /// \param skip_copy_to_host - void generate_content_query_host(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device = {}, - bool skip_copy_to_host = false); - - /// \brief Generates indices on device and copies them to host memory - /// - /// If index already exists it is not going to be regenerated. - /// Indices from descriptors_of_indices_to_keep_on_device will be kept on device in addition to being to host. This is useful if the same indices - /// are going to be requested by start_generating_content_target_device() immediately after this call. - /// If skip_copy_to_host is ture indices are going to be kept on device and not copied to host. In that case descriptors_of_indices_to_cache must - /// be equal to descriptors_of_indices_to_keep_on_device and there must be only one call to start_generating_content_target_device() + /// are going to be requested by start_copying_indices_to_device() immediately after this call + /// If skip_copy_to_host is true indices are going to be kept on device and not copied to host. In that case descriptors_of_indices_to_cache must + /// be equal to descriptors_of_indices_to_keep_on_device and there must be only one call to start_copying_indices_to_device() with exactly these indices + /// Calling this function invalidates any previously cached data for the same cache type /// + /// \param cache_type /// \param descriptors_of_indices_to_cache /// \param descriptors_of_indices_to_keep_on_device /// \param skip_copy_to_host - void generate_content_target_host(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device = {}, - bool skip_copy_to_host = false); - - /// \brief Begins copying indices to device - /// - /// If index already exists on device it may be reused. - /// This copy is done asynchronously, copy is finised by calling finish_generating_content_query_device(). - /// The user should make sure that every call to start_generating_content_query_device() is followed by a call to finish_generating_content_query_device() - /// - /// \param descriptors_of_indices_to_cache - void start_generating_content_query_device(const std::vector& descriptors_of_indices_to_cache); - - /// \brief Waits for copy started in start_generating_content_query_device() - /// - /// The user should make sure that there was a call to start_generating_content_query_device() before this call - void finish_generating_content_query_device(); + void generate_content(CacheType cache_type, + const std::vector& descriptors_of_indices_to_cache, + const std::vector& descriptors_of_indices_to_keep_on_device = {}, + bool skip_copy_to_host = false); /// \brief Begins copying indices to device /// /// If index already exists on device it may be reused. - /// This copy is done asynchronously, copy is finised by calling finish_generating_content_target_device(). - /// The user should make sure that every call to start_generating_content_target_device() is followed by a call to finish_generating_content_target_device() + /// This copy is done asynchronously. Function returns a DeviceIndexCache object which should be used to access the indices. + /// Copy to device is finised by calling DeviceIndexCache::wait_for_data_to_be_ready(). + /// The user should make sure that every call to start_copying_indices_to_device() is accompanied by a call to DeviceIndexCache::wait_for_data_to_be_ready() /// + /// \param cache_type /// \param descriptors_of_indices_to_cache - void start_generating_content_target_device(const std::vector& descriptors_of_indices_to_cache); - - /// \brief Waits for copy started in start_generating_content_target_device() - /// - /// The user should make sure that there was a call to start_generating_content_target_device() before this call - void finish_generating_content_target_device(); - - /// \brief Returns a pointer to requested index - /// \return a pointer to requested index - std::shared_ptr get_index_from_query_cache(const IndexDescriptor& index_descriptor) const; - - /// \brief Returns a pointer to requested index - /// \return a pointer to requested index - std::shared_ptr get_index_from_target_cache(const IndexDescriptor& index_descriptor) const; + /// \throw IndexNotFoundException if an index that is not cached by call to generate_content() is requested + /// \return DeviceIndexCache object + std::shared_ptr start_copying_indices_to_device(CacheType cache_type, + const std::vector& descriptors_of_indices_to_cache); + + /// \brief Registers DeviceIndexCache object + /// \param cache_type + /// \param index_cache + void register_device_cache(CacheType cache_type, + DeviceIndexCache* index_cache); + + /// \brief Deregisters DeviceIndexCache object + /// \param cache_type + /// \param index_cache + void deregister_device_cache(CacheType cache_type, + DeviceIndexCache* index_cache); private: using host_cache_t = std::unordered_map, IndexDescriptorHash>; - enum class CacheSelector - { - query_cache, - target_cache - }; - - /// \brief Generates indices on device and copies them to host memory - /// - /// If index already exists on host is may be reused. - /// Indices from descriptors_of_indices_to_keep_on_device will be kept on device in addition to being to host. This is useful if the same indices - /// are going to be requested by start_generating_content_device() immediately after this call. - /// If skip_copy_to_host is ture indices are going to be kept on device and not copied to host. In that case descriptors_of_indices_to_cache must - /// be equal to descriptors_of_indices_to_keep_on_device and there must be only one call to start_generating_content_device() - /// - /// \param descriptors_of_indices_to_cache - /// \param descriptors_of_indices_to_keep_on_device - /// \param skip_copy_to_host - /// \param which_cache - void generate_content_host(const std::vector& descriptors_of_indices_to_cache, - const std::vector& descriptors_of_indices_to_keep_on_device, - bool skip_copy_to_host, - CacheSelector which_cache); - - /// \brief Begins copying indices to device - /// - /// If index already exists on device it may be reused. - /// This copy is done asynchronously, copy is finised by calling finish_generating_content_device(). - /// The user should make sure that every call to start_generating_content_device() is followed by a call to finish_generating_content_device() - /// - /// \param descriptors_of_indices_to_cache - /// \param which_cache - void start_generating_content_device(const std::vector& descriptors_of_indices_to_cache, - CacheSelector which_cache); - - /// \brief Waits for copy started in start_generating_content_device() - /// - /// The user should make sure that there was a call to start_generating_content_device() before this call - /// \param which_cache - void finish_generating_content_device(CacheSelector which_cache); - - /// \brief Returns a pointer to requested index - /// \return a pointer to requested index - std::shared_ptr get_index_from_cache(const IndexDescriptor& index_descriptor, - CacheSelector which_cache) const; - // Indices kept on host host_cache_t query_host_cache_; host_cache_t target_host_cache_; @@ -204,14 +219,9 @@ private: device_cache_t query_indices_kept_on_device_; device_cache_t target_indices_kept_on_device_; - // Indices on device ready to be used - device_cache_t query_device_cache_; - device_cache_t target_device_cache_; - - // Indices currently being copied to device, they are going to replace the content of query_device_cache_ - // and target_device_cache_ after the next call to finish_generating_content_device( - device_cache_t next_query_device_cache_; - device_cache_t next_target_device_cache_; + // Currently existing DeviceIndexCaches created by this HostIndexCache + std::vector device_caches_query_; + std::vector device_caches_target_; const bool same_query_and_target_; genomeworks::DefaultDeviceAllocator allocator_; @@ -229,8 +239,8 @@ private: class IndexNotFoundException : public std::exception { public: - /// IndexType - Was the Index requested from host or device cache - enum class IndexType + /// IndexLocation - Was the Index requested from host or device cache + enum class IndexLocation { host_cache, device_cache @@ -239,14 +249,32 @@ public: /// \brief constructor /// \param index_descriptor /// \param index_type was Index equested from host or device cache - IndexNotFoundException(IndexDescriptor index_descriptor, - IndexType index_type); + IndexNotFoundException(CacheType cache_type, + IndexLocation index_location, + IndexDescriptor index_descriptor); + + /// \brief Returns the error message of the exception + virtual const char* what() const noexcept; + +private: + const std::string message_; +}; + +/// DeviceCacheNotReadyException - Exception ot be thrown when an index is requested before it has been copied completely +class DeviceCacheNotReadyException : public std::exception +{ +public: + /// \brief constructor + /// \param cache_type + /// \param index_descriptor + DeviceCacheNotReadyException(CacheType cache_type, + IndexDescriptor index_descriptor); - /// Returns the error message of the exception + /// \brief Returns the error message of the exception virtual const char* what() const noexcept; private: - const std::string error_message_; + const std::string message_; }; } // namespace cudamapper diff --git a/cudamapper/src/main.cu b/cudamapper/src/main.cu index a128cb398..9e06c0f9a 100644 --- a/cudamapper/src/main.cu +++ b/cudamapper/src/main.cu @@ -196,14 +196,16 @@ struct OverlapsAndCigars /// \brief does overlapping and matching for pairs of query and target indices from device_batch /// \param device_batch list of query and target indices to be processed -/// \param index_cache indices should be loaded into cache beforehand +/// \param query_index_cache indices should be loaded into cache beforehand +/// \param target_index_cache indices should be loaded into cache beforehand /// \param application_parameters /// \param device_allocator /// \param overlaps_and_cigars_to_process overlaps and cigars are output here and then consumed by another thread /// \param number_of_skipped_pairs_of_indices number of pairs of indices skipped due to OOM error, variable shared between all threads, each call increases the number by the number of skipped pairs /// \param cuda_stream cuda stream on which the computation should be done void process_one_device_batch(const IndexBatch& device_batch, - IndexCache& index_cache, + std::shared_ptr query_index_cache, + std::shared_ptr target_index_cache, const ApplicationParameters& application_parameters, DefaultDeviceAllocator device_allocator, ThreadsafeProducerConsumer& overlaps_and_cigars_to_process, @@ -223,8 +225,8 @@ void process_one_device_batch(const IndexBatch& device_batch, // if doing all-to-all skip pairs in which target batch has smaller id than query batch as it will be covered by symmetry if (!application_parameters.all_to_all || target_index_descriptor.first_read() >= query_index_descriptor.first_read()) { - std::shared_ptr query_index = index_cache.get_index_from_query_cache(query_index_descriptor); - std::shared_ptr target_index = index_cache.get_index_from_target_cache(target_index_descriptor); + std::shared_ptr query_index = query_index_cache->get_index(query_index_descriptor); + std::shared_ptr target_index = target_index_cache->get_index(target_index_descriptor); try { @@ -278,14 +280,14 @@ void process_one_device_batch(const IndexBatch& device_batch, /// \brief loads one batch into host memory and then processes its device batches one by one /// \param batch list of all query and target indices that belong to this batch, as well as subgroups of those indices that belong to device batches /// \param application_parameters -/// \param index_cache data will be loaded into cache within the function +/// \param host_index_cache data will be loaded into cache within the function /// \param overlaps_and_cigars_to_process overlaps and cigars are output to this structure and the then consumed by another thread /// \param number_of_skipped_pairs_of_indices number of pairs of indices skipped due to OOM error, variable shared between all threads, each call increases the number by the number of skipped pairs /// \param cuda_stream void process_one_batch(const BatchOfIndices& batch, const ApplicationParameters& application_parameters, DefaultDeviceAllocator device_allocator, - IndexCache& index_cache, + HostIndexCache& host_index_cache, ThreadsafeProducerConsumer& overlaps_and_cigars_to_process, std::atomic& number_of_skipped_pairs_of_indices, cudaStream_t cuda_stream) @@ -304,12 +306,14 @@ void process_one_batch(const BatchOfIndices& batch, GW_NVTX_RANGE(profiler, "main::process_one_batch::generte_host_indices"); assert(!host_batch.query_indices.empty() && !host_batch.target_indices.empty() && !device_batches.empty()); - index_cache.generate_content_query_host(host_batch.query_indices, - device_batches.front().query_indices, - skip_copy_to_host); - index_cache.generate_content_target_host(host_batch.target_indices, - device_batches.front().target_indices, - skip_copy_to_host); + host_index_cache.generate_content(CacheType::query_cache, + host_batch.query_indices, + device_batches.front().query_indices, + skip_copy_to_host); + host_index_cache.generate_content(CacheType::target_cache, + host_batch.target_indices, + device_batches.front().target_indices, + skip_copy_to_host); } // process device batches one by one @@ -317,14 +321,20 @@ void process_one_batch(const BatchOfIndices& batch, // Loop uses copy_device_batch_index as its index, compute_device_batch_index is equal to copy_device_batch_index - 1 // (compute_device_batch_index does not actually exist). // copy_device_batch_index loops over one more element because for such element compute_device_batch_index would have been equal to the last batch + std::shared_ptr query_cache_being_copied_into; + std::shared_ptr query_cache_with_ready_data; + std::shared_ptr target_cache_being_copied_into; + std::shared_ptr target_cache_with_ready_data; for (int32_t copy_device_batch_index = 0; copy_device_batch_index < get_size(batch.device_batches) + 1; ++copy_device_batch_index) { if (copy_device_batch_index > 0) { // if not the first batch wait for previous batch to finish copying GW_NVTX_RANGE(profiler, "main::process_one_batch::finish_generating_device_indices"); - index_cache.finish_generating_content_query_device(); - index_cache.finish_generating_content_target_device(); + query_cache_being_copied_into->wait_for_data_to_be_ready(); + target_cache_being_copied_into->wait_for_data_to_be_ready(); + query_cache_with_ready_data = query_cache_being_copied_into; + target_cache_with_ready_data = target_cache_being_copied_into; } if (copy_device_batch_index < get_size(batch.device_batches)) @@ -333,8 +343,11 @@ void process_one_batch(const BatchOfIndices& batch, GW_NVTX_RANGE(profiler, "main::process_one_batch::start_generating_device_indices"); const std::vector& query_index_batch_to_start_copying = batch.device_batches[copy_device_batch_index].query_indices; const std::vector& target_index_batch_to_start_copying = batch.device_batches[copy_device_batch_index].target_indices; - index_cache.start_generating_content_query_device(query_index_batch_to_start_copying); - index_cache.start_generating_content_target_device(target_index_batch_to_start_copying); + + query_cache_being_copied_into = host_index_cache.start_copying_indices_to_device(CacheType::query_cache, + query_index_batch_to_start_copying); + target_cache_being_copied_into = host_index_cache.start_copying_indices_to_device(CacheType::target_cache, + target_index_batch_to_start_copying); } if (copy_device_batch_index > 0) @@ -342,13 +355,16 @@ void process_one_batch(const BatchOfIndices& batch, // when copy_device_batch_index == 0 then compute_device_batch_index == -1, so there is no batch to process GW_NVTX_RANGE(profiler, "main::process_one_batch::process_device_batch"); process_one_device_batch(batch.device_batches[copy_device_batch_index - 1], - index_cache, + query_cache_with_ready_data, + target_cache_with_ready_data, application_parameters, device_allocator, overlaps_and_cigars_to_process, number_of_skipped_pairs_of_indices, cuda_stream); } + + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); } } @@ -440,16 +456,16 @@ void worker_thread_function(const int32_t device_id, {cuda_stream_computation}); // create index_cache, indices are not created at this point but later as each batch gets processed - IndexCache index_cache(application_parameters.all_to_all, - device_allocator, - application_parameters.query_parser, - application_parameters.target_parser, - application_parameters.kmer_size, - application_parameters.windows_size, - true, // hash_representations - application_parameters.filtering_parameter, - cuda_stream_computation, - cuda_stream_copy); + HostIndexCache host_index_cache(application_parameters.all_to_all, + device_allocator, + application_parameters.query_parser, + application_parameters.target_parser, + application_parameters.kmer_size, + application_parameters.windows_size, + true, // hash_representations + application_parameters.filtering_parameter, + cuda_stream_computation, + cuda_stream_copy); // data structure used to exchange data with postprocess_and_write_thread ThreadsafeProducerConsumer overlaps_and_cigars_to_process; @@ -483,7 +499,7 @@ void worker_thread_function(const int32_t device_id, process_one_batch(batch_of_indices.value(), application_parameters, device_allocator, - index_cache, + host_index_cache, overlaps_and_cigars_to_process, number_of_skipped_pairs_of_indices, cuda_stream_computation); diff --git a/cudamapper/tests/Test_CudamapperIndexCache.cu b/cudamapper/tests/Test_CudamapperIndexCache.cu index c66b28a97..8f0acfb95 100644 --- a/cudamapper/tests/Test_CudamapperIndexCache.cu +++ b/cudamapper/tests/Test_CudamapperIndexCache.cu @@ -335,22 +335,24 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) std::vector catcaag_aagcta_separate_index_descriptors({catcaag_index_descriptor, aagcta_index_descriptor}); std::vector catcaag_aagcta_one_index_descriptors({catcaag_aagcta_index_descriptor}); - IndexCache index_cache(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate.get(), - cuda_stream_copy.get()); - - index_cache.generate_content_query_host(catcaag_index_descriptors); - index_cache.start_generating_content_query_device(catcaag_index_descriptors); - index_cache.finish_generating_content_query_device(); - - auto index_query_catcaag = index_cache.get_index_from_query_cache(catcaag_index_descriptor); + HostIndexCache host_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate.get(), + cuda_stream_copy.get()); + + host_cache.generate_content(CacheType::query_cache, + catcaag_index_descriptors); + + std::shared_ptr device_cache_query = host_cache.start_copying_indices_to_device(CacheType::query_cache, catcaag_index_descriptors); + device_cache_query->wait_for_data_to_be_ready(); + + auto index_query_catcaag = device_cache_query->get_index(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag, catcaag_representations, catcaag_read_ids, @@ -365,17 +367,16 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) catcaag_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_1"); - ASSERT_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_query->get_index(aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_query->get_index(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(host_cache.start_copying_indices_to_device(CacheType::target_cache, aagcta_index_descriptors), IndexNotFoundException); + ASSERT_THROW(host_cache.start_copying_indices_to_device(CacheType::target_cache, aagcta_index_descriptors), IndexNotFoundException); - index_cache.generate_content_target_host(aagcta_index_descriptors); - index_cache.start_generating_content_target_device(aagcta_index_descriptors); - index_cache.finish_generating_content_target_device(); + host_cache.generate_content(CacheType::target_cache, aagcta_index_descriptors); + std::shared_ptr device_cache_target = host_cache.start_copying_indices_to_device(CacheType::target_cache, aagcta_index_descriptors); + device_cache_target->wait_for_data_to_be_ready(); - index_query_catcaag = index_cache.get_index_from_query_cache(catcaag_index_descriptor); + index_query_catcaag = device_cache_query->get_index(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag, catcaag_representations, catcaag_read_ids, @@ -390,10 +391,10 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) catcaag_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_2"); - ASSERT_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); - auto index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); + ASSERT_THROW(device_cache_query->get_index(aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_query->get_index(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_target->get_index(catcaag_index_descriptor), IndexNotFoundException); + auto index_target_aagcta = device_cache_target->get_index(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, aagcta_read_ids, @@ -408,14 +409,15 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_3"); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_query->get_index(catcaag_aagcta_index_descriptor), IndexNotFoundException); - index_cache.generate_content_query_host(aagcta_index_descriptors); - index_cache.start_generating_content_query_device(aagcta_index_descriptors); - index_cache.finish_generating_content_query_device(); + device_cache_query = nullptr; + host_cache.generate_content(CacheType::query_cache, aagcta_index_descriptors); + device_cache_query = host_cache.start_copying_indices_to_device(CacheType::query_cache, aagcta_index_descriptors); + device_cache_query->wait_for_data_to_be_ready(); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_index_descriptor), IndexNotFoundException); - auto index_query_aagcta = index_cache.get_index_from_query_cache(aagcta_index_descriptor); + ASSERT_THROW(device_cache_query->get_index(catcaag_index_descriptor), IndexNotFoundException); + auto index_query_aagcta = device_cache_query->get_index(aagcta_index_descriptor); check_if_index_is_correct(index_query_aagcta, aagcta_representations, aagcta_read_ids, @@ -430,9 +432,9 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_4"); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); - index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); + ASSERT_THROW(device_cache_query->get_index(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_target->get_index(catcaag_index_descriptor), IndexNotFoundException); + index_target_aagcta = device_cache_target->get_index(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, aagcta_read_ids, @@ -447,13 +449,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_5"); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_query->get_index(catcaag_aagcta_index_descriptor), IndexNotFoundException); - index_cache.generate_content_query_host(catcaag_aagcta_separate_index_descriptors); - index_cache.start_generating_content_query_device(catcaag_aagcta_separate_index_descriptors); - index_cache.finish_generating_content_query_device(); + device_cache_query = nullptr; + host_cache.generate_content(CacheType::query_cache, catcaag_aagcta_separate_index_descriptors); + device_cache_query = host_cache.start_copying_indices_to_device(CacheType::query_cache, catcaag_aagcta_separate_index_descriptors); + device_cache_query->wait_for_data_to_be_ready(); - auto index_query_catcaag_separate = index_cache.get_index_from_query_cache(catcaag_index_descriptor); + auto index_query_catcaag_separate = device_cache_query->get_index(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag_separate, catcaag_representations, catcaag_read_ids, @@ -468,7 +471,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) catcaag_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_6"); - auto index_query_aagcta_separate = index_cache.get_index_from_query_cache(aagcta_index_descriptor); + auto index_query_aagcta_separate = device_cache_query->get_index(aagcta_index_descriptor); check_if_index_is_correct(index_query_aagcta_separate, aagcta_representations, aagcta_read_ids, @@ -483,9 +486,9 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_7"); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); - index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); + ASSERT_THROW(device_cache_query->get_index(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_target->get_index(catcaag_index_descriptor), IndexNotFoundException); + index_target_aagcta = device_cache_target->get_index(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, aagcta_read_ids, @@ -500,13 +503,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_8"); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_query->get_index(catcaag_aagcta_index_descriptor), IndexNotFoundException); - index_cache.generate_content_target_host(catcaag_aagcta_one_index_descriptors); - index_cache.start_generating_content_target_device(catcaag_aagcta_one_index_descriptors); - index_cache.finish_generating_content_target_device(); + device_cache_target = nullptr; + host_cache.generate_content(CacheType::target_cache, catcaag_aagcta_one_index_descriptors); + device_cache_target = host_cache.start_copying_indices_to_device(CacheType::target_cache, catcaag_aagcta_one_index_descriptors); + device_cache_target->wait_for_data_to_be_ready(); - index_query_catcaag_separate = index_cache.get_index_from_query_cache(catcaag_index_descriptor); + index_query_catcaag_separate = device_cache_query->get_index(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag_separate, catcaag_representations, catcaag_read_ids, @@ -521,7 +525,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) catcaag_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_9"); - index_query_aagcta_separate = index_cache.get_index_from_query_cache(aagcta_index_descriptor); + index_query_aagcta_separate = device_cache_query->get_index(aagcta_index_descriptor); check_if_index_is_correct(index_query_aagcta_separate, aagcta_representations, aagcta_read_ids, @@ -536,10 +540,10 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_10"); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_aagcta_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor), IndexNotFoundException); - auto catcaag_aagcta_target_aagcta = index_cache.get_index_from_target_cache(catcaag_aagcta_index_descriptor); + ASSERT_THROW(device_cache_query->get_index(catcaag_aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_target->get_index(catcaag_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_target->get_index(aagcta_index_descriptor), IndexNotFoundException); + auto catcaag_aagcta_target_aagcta = device_cache_target->get_index(catcaag_aagcta_index_descriptor); check_if_index_is_correct(catcaag_aagcta_target_aagcta, catcaag_aagcta_representations, catcaag_aagcta_read_ids, @@ -707,22 +711,22 @@ TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target) IndexDescriptor index_descriptor(0, 1); std::vector index_descriptors({index_descriptor}); - IndexCache index_cache(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate.get(), - cuda_stream_copy.get()); - - index_cache.generate_content_query_host(index_descriptors); - index_cache.start_generating_content_query_device(index_descriptors); - index_cache.finish_generating_content_query_device(); - - auto index_query_aagcta = index_cache.get_index_from_query_cache(index_descriptor); + HostIndexCache host_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate.get(), + cuda_stream_copy.get()); + + host_cache.generate_content(CacheType::query_cache, index_descriptors); + std::shared_ptr device_cache_query = host_cache.start_copying_indices_to_device(CacheType::query_cache, index_descriptors); + device_cache_query->wait_for_data_to_be_ready(); + + auto index_query_aagcta = device_cache_query->get_index(index_descriptor); check_if_index_is_correct(index_query_aagcta, aagcta_representations, aagcta_read_ids, @@ -737,13 +741,13 @@ TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_not_the_same_query_and_target_1"); - ASSERT_THROW(index_cache.get_index_from_target_cache(index_descriptor), IndexNotFoundException); + ASSERT_THROW(host_cache.start_copying_indices_to_device(CacheType::target_cache, index_descriptors), IndexNotFoundException); - index_cache.generate_content_target_host(index_descriptors); - index_cache.start_generating_content_target_device(index_descriptors); - index_cache.finish_generating_content_target_device(); + host_cache.generate_content(CacheType::target_cache, index_descriptors); + std::shared_ptr device_cache_target = host_cache.start_copying_indices_to_device(CacheType::target_cache, index_descriptors); + device_cache_target->wait_for_data_to_be_ready(); - index_query_aagcta = index_cache.get_index_from_query_cache(index_descriptor); + index_query_aagcta = device_cache_query->get_index(index_descriptor); check_if_index_is_correct(index_query_aagcta, aagcta_representations, aagcta_read_ids, @@ -758,7 +762,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target) aagcta_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_not_the_same_query_and_target_2"); - auto index_target_catcaag = index_cache.get_index_from_target_cache(index_descriptor); + auto index_target_catcaag = device_cache_target->get_index(index_descriptor); check_if_index_is_correct(index_target_catcaag, catcaag_representations, catcaag_read_ids, @@ -836,34 +840,34 @@ TEST(TestCudamapperIndexCaching, test_index_cache_keep_on_device) IndexDescriptor index_descriptor(0, 1); std::vector index_descriptors({index_descriptor}); - IndexCache index_cache(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate.get(), - cuda_stream_copy.get()); - - index_cache.generate_content_query_host(index_descriptors, index_descriptors); - index_cache.start_generating_content_query_device(index_descriptors); // 1st copy, kept on device - index_cache.generate_content_target_host(index_descriptors, index_descriptors); - index_cache.start_generating_content_target_device(index_descriptors); - index_cache.finish_generating_content_query_device(); - index_cache.finish_generating_content_target_device(); - - auto index_query_temp_device_cache = index_cache.get_index_from_query_cache(index_descriptor); - auto index_target_temp_device_cache = index_cache.get_index_from_target_cache(index_descriptor); - - index_cache.start_generating_content_query_device(index_descriptors); - index_cache.start_generating_content_target_device(index_descriptors); - index_cache.finish_generating_content_query_device(); - index_cache.finish_generating_content_target_device(); - - auto index_query_copy_from_host = index_cache.get_index_from_query_cache(index_descriptor); // 2nd copy, copied from host - auto index_target_copy_from_host = index_cache.get_index_from_target_cache(index_descriptor); + HostIndexCache host_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate.get(), + cuda_stream_copy.get()); + + host_cache.generate_content(CacheType::query_cache, index_descriptors, index_descriptors); + std::shared_ptr device_cache_query_1 = host_cache.start_copying_indices_to_device(CacheType::query_cache, index_descriptors); // 1st copy, kept on device + host_cache.generate_content(CacheType::target_cache, index_descriptors, index_descriptors); + std::shared_ptr device_cache_target_1 = host_cache.start_copying_indices_to_device(CacheType::target_cache, index_descriptors); + device_cache_query_1->wait_for_data_to_be_ready(); + device_cache_target_1->wait_for_data_to_be_ready(); + + auto index_query_temp_device_cache = device_cache_query_1->get_index(index_descriptor); + auto index_target_temp_device_cache = device_cache_target_1->get_index(index_descriptor); + + std::shared_ptr device_cache_query_2 = host_cache.start_copying_indices_to_device(CacheType::query_cache, index_descriptors); + std::shared_ptr device_cache_target_2 = host_cache.start_copying_indices_to_device(CacheType::target_cache, index_descriptors); + device_cache_query_2->wait_for_data_to_be_ready(); + device_cache_target_2->wait_for_data_to_be_ready(); + + auto index_query_copy_from_host = device_cache_query_2->get_index(index_descriptor); // 2nd copy, copied from host + auto index_target_copy_from_host = device_cache_target_2->get_index(index_descriptor); check_if_index_is_correct(index_query_temp_device_cache, aagcta_representations, @@ -1038,22 +1042,21 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target_2) std::vector aagcta_index_descriptors({aagcta_index_descriptor}); std::vector catcaag_aagcta_index_descriptors({catcaag_index_descriptor, aagcta_index_descriptor}); - IndexCache index_cache(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate.get(), - cuda_stream_copy.get()); - - index_cache.generate_content_query_host(catcaag_index_descriptors); - ASSERT_THROW(index_cache.get_index_from_query_cache(catcaag_index_descriptor), IndexNotFoundException); - index_cache.start_generating_content_query_device(catcaag_index_descriptors); - index_cache.finish_generating_content_query_device(); - auto index_query_catcaag = index_cache.get_index_from_query_cache(catcaag_index_descriptor); + HostIndexCache host_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate.get(), + cuda_stream_copy.get()); + + host_cache.generate_content(CacheType::query_cache, catcaag_index_descriptors); + std::shared_ptr device_cache_query = host_cache.start_copying_indices_to_device(CacheType::query_cache, catcaag_index_descriptors); + device_cache_query->wait_for_data_to_be_ready(); + auto index_query_catcaag = device_cache_query->get_index(catcaag_index_descriptor); check_if_index_is_correct(index_query_catcaag, catcaag_representations, catcaag_read_ids, @@ -1068,17 +1071,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target_2) catcaag_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_same_query_and_target_2_1"); - ASSERT_THROW(index_cache.get_index_from_query_cache(aagcta_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(device_cache_query->get_index(aagcta_index_descriptor), IndexNotFoundException); + ASSERT_THROW(host_cache.start_copying_indices_to_device(CacheType::target_cache, catcaag_aagcta_index_descriptors), IndexNotFoundException); - index_cache.generate_content_target_host(catcaag_aagcta_index_descriptors); - ASSERT_THROW(index_cache.get_index_from_target_cache(catcaag_index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(aagcta_index_descriptor), IndexNotFoundException); - index_cache.start_generating_content_target_device(catcaag_aagcta_index_descriptors); - index_cache.finish_generating_content_target_device(); + host_cache.generate_content(CacheType::target_cache, catcaag_aagcta_index_descriptors); + std::shared_ptr device_cache_target = host_cache.start_copying_indices_to_device(CacheType::target_cache, catcaag_aagcta_index_descriptors); + device_cache_target->wait_for_data_to_be_ready(); - auto index_target_catcaag = index_cache.get_index_from_target_cache(catcaag_index_descriptor); + auto index_target_catcaag = device_cache_target->get_index(catcaag_index_descriptor); ASSERT_EQ(index_query_catcaag, index_target_catcaag); // check same object is used because same_query_and_target == true check_if_index_is_correct(index_target_catcaag, catcaag_representations, @@ -1095,7 +1095,7 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target_2) cuda_stream_generate.get(), "test_index_cache_same_query_and_target_2_2"); - auto index_target_aagcta = index_cache.get_index_from_target_cache(aagcta_index_descriptor); + auto index_target_aagcta = device_cache_target->get_index(aagcta_index_descriptor); check_if_index_is_correct(index_target_aagcta, aagcta_representations, aagcta_read_ids, @@ -1112,8 +1112,8 @@ TEST(TestCudamapperIndexCaching, test_index_cache_same_query_and_target_2) "test_index_cache_same_query_and_target_2_3"); // get the same query and target indices again and make sure they point to the same objects as the last time - auto index_query_catcaag_1 = index_cache.get_index_from_query_cache(catcaag_index_descriptor); - auto index_target_aagcta_1 = index_cache.get_index_from_target_cache(aagcta_index_descriptor); + auto index_query_catcaag_1 = device_cache_query->get_index(catcaag_index_descriptor); + auto index_target_aagcta_1 = device_cache_target->get_index(aagcta_index_descriptor); ASSERT_EQ(index_query_catcaag, index_query_catcaag_1); ASSERT_EQ(index_target_aagcta, index_target_aagcta_1); } @@ -1226,24 +1226,22 @@ TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target_ IndexDescriptor index_descriptor(0, 1); std::vector index_descriptors({index_descriptor}); - IndexCache index_cache(same_query_and_target, - allocator, - query_parser, - target_parser, - k, - w, - hash_representations, - filtering_parameter, - cuda_stream_generate.get(), - cuda_stream_copy.get()); - - index_cache.generate_content_query_host(index_descriptors); - ASSERT_THROW(index_cache.get_index_from_query_cache(index_descriptor), IndexNotFoundException); - ASSERT_THROW(index_cache.get_index_from_target_cache(index_descriptor), IndexNotFoundException); - - index_cache.start_generating_content_query_device(index_descriptors); - index_cache.finish_generating_content_query_device(); - auto index_query = index_cache.get_index_from_query_cache(index_descriptor); + HostIndexCache host_cache(same_query_and_target, + allocator, + query_parser, + target_parser, + k, + w, + hash_representations, + filtering_parameter, + cuda_stream_generate.get(), + cuda_stream_copy.get()); + + host_cache.generate_content(CacheType::query_cache, index_descriptors); + ASSERT_THROW(host_cache.start_copying_indices_to_device(CacheType::target_cache, index_descriptors), IndexNotFoundException); + std::shared_ptr device_cache_query = host_cache.start_copying_indices_to_device(CacheType::query_cache, index_descriptors); + device_cache_query->wait_for_data_to_be_ready(); + auto index_query = device_cache_query->get_index(index_descriptor); check_if_index_is_correct(index_query, aagcta_representations, aagcta_read_ids, @@ -1258,14 +1256,14 @@ TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target_ aagcta_maximum_kmer_size, cuda_stream_generate.get(), "test_index_cache_not_the_same_query_and_target_2_1"); - ASSERT_THROW(index_cache.get_index_from_target_cache(index_descriptor), IndexNotFoundException); + ASSERT_THROW(host_cache.start_copying_indices_to_device(CacheType::target_cache, index_descriptors), IndexNotFoundException); - index_cache.generate_content_target_host(index_descriptors); - index_cache.start_generating_content_target_device(index_descriptors); - index_cache.finish_generating_content_target_device(); + host_cache.generate_content(CacheType::target_cache, index_descriptors); + std::shared_ptr device_cache_target = host_cache.start_copying_indices_to_device(CacheType::target_cache, index_descriptors); + device_cache_target->wait_for_data_to_be_ready(); - index_query = index_cache.get_index_from_query_cache(index_descriptor); - auto index_target = index_cache.get_index_from_target_cache(index_descriptor); + index_query = device_cache_query->get_index(index_descriptor); + auto index_target = device_cache_target->get_index(index_descriptor); ASSERT_NE(index_query, index_target); check_if_index_is_correct(index_query, aagcta_representations, @@ -1297,8 +1295,8 @@ TEST(TestCudamapperIndexCaching, test_index_cache_not_the_same_query_and_target_ "test_index_cache_not_the_same_query_and_target_2_3"); // get the same query and target indices again and make sure they point to the same objects as the last time - auto index_query_1 = index_cache.get_index_from_query_cache(index_descriptor); - auto index_target_1 = index_cache.get_index_from_target_cache(index_descriptor); + auto index_query_1 = device_cache_query->get_index(index_descriptor); + auto index_target_1 = device_cache_target->get_index(index_descriptor); ASSERT_EQ(index_query, index_query_1); ASSERT_EQ(index_target, index_target_1); } From 80a1ebd0ece7937bced3454ad5930a4067adcb0b Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 1 Oct 2020 10:57:25 +0200 Subject: [PATCH 175/281] [cudamapper] HostIndexCache and DeviceIndexCache friends, methods moved to private --- cudamapper/src/index_cache.cuh | 51 +++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index eefbace20..7861073b9 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -69,13 +69,7 @@ public: /// \brief Destructor ~DeviceIndexCache(); - /// \brief Adds index to host, should only be called by HostIndexCache::start_copying_indices_to_device() - /// \param index_descriptor - /// \param device_index - void add_index(IndexDescriptor index_descriptor, - std::shared_ptr device_index); - - /// \brief Returns requested index + /// \brief Returns requested index for which it is guarateed that it is ready, i.e. is has been fully copied from host memory is needed /// Calling this function before wait_for_data_to_be_ready() results in an exception /// \param index_descriptor /// \throw IndexNotFoundException if requested index is not cached @@ -83,12 +77,6 @@ public: /// \return requested index std::shared_ptr get_index(IndexDescriptor index_descriptor) const; - /// \brief Returns requested index, returned index might not be ready and has to be synchronized directly - /// \param index_descriptor - /// \throw IndexNotFoundException if requested index is not cached - /// \return requested index - std::shared_ptr get_index_no_check_if_ready(IndexDescriptor index_descriptor) const; - /// \brief Returns whether given index is present in cache /// \param index_descriptor /// \return is given index present in cache @@ -102,10 +90,25 @@ public: bool is_ready() const; private: + friend HostIndexCache; + using device_cache_t = std::unordered_map, IndexDescriptorHash>; + /// \brief Adds index to cache + /// To be called by HostIndexCache::start_copying_indices_to_device() + /// \param index_descriptor + /// \param device_index + void add_index(IndexDescriptor index_descriptor, + std::shared_ptr device_index); + + /// \brief Returns requested index, returned index might not be ready and has to be synchronized directly + /// \param index_descriptor + /// \throw IndexNotFoundException if requested index is not cached + /// \return requested index + std::shared_ptr get_index_no_check_if_ready(IndexDescriptor index_descriptor) const; + device_cache_t cache_; CacheType cache_type_; @@ -190,27 +193,31 @@ public: std::shared_ptr start_copying_indices_to_device(CacheType cache_type, const std::vector& descriptors_of_indices_to_cache); +private: + friend DeviceIndexCache; + + using host_cache_t = std::unordered_map, + IndexDescriptorHash>; + + using device_cache_t = std::unordered_map, + IndexDescriptorHash>; + /// \brief Registers DeviceIndexCache object + /// To be called by the constructor of DeviceIndexCache /// \param cache_type /// \param index_cache void register_device_cache(CacheType cache_type, DeviceIndexCache* index_cache); /// \brief Deregisters DeviceIndexCache object + /// To be called by the destructor of DeviceIndexCache /// \param cache_type /// \param index_cache void deregister_device_cache(CacheType cache_type, DeviceIndexCache* index_cache); -private: - using host_cache_t = std::unordered_map, - IndexDescriptorHash>; - - using device_cache_t = std::unordered_map, - IndexDescriptorHash>; - // Indices kept on host host_cache_t query_host_cache_; host_cache_t target_host_cache_; From d0a38b34d3c5c51502bd96edb50450edbafef099 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 1 Oct 2020 15:52:00 +0200 Subject: [PATCH 176/281] [cudamapper] Better names in HostIndexCache and DeviceIndexCache --- cudamapper/src/index_cache.cu | 8 ++++---- cudamapper/src/index_cache.cuh | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 1ef7abe4d..f8ce82bf2 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -282,8 +282,8 @@ std::shared_ptr HostIndexCache::start_copying_indices_to_devic const std::vector& descriptors_of_indices_to_cache) { const host_cache_t& host_cache = (CacheType::query_cache == cache_type) ? query_host_cache_ : target_host_cache_; - const std::vector& this_device_caches = (CacheType::query_cache == cache_type) ? device_caches_query_ : device_caches_target_; - const std::vector& other_device_caches = (CacheType::query_cache == cache_type) ? device_caches_target_ : device_caches_query_; + const std::vector& this_device_caches = (CacheType::query_cache == cache_type) ? query_device_caches_ : target_device_caches_; + const std::vector& other_device_caches = (CacheType::query_cache == cache_type) ? target_device_caches_ : query_device_caches_; device_cache_t& indices_kept_on_device = (CacheType::query_cache == cache_type) ? query_indices_kept_on_device_ : target_indices_kept_on_device_; std::shared_ptr device_cache = std::make_shared(cache_type, @@ -353,7 +353,7 @@ void HostIndexCache::register_device_cache(const CacheType cache_type, { assert(cache_type == CacheType::query_cache || cache_type == CacheType::target_cache); - std::vector& device_caches = cache_type == CacheType::query_cache ? device_caches_query_ : device_caches_target_; + std::vector& device_caches = cache_type == CacheType::query_cache ? query_device_caches_ : target_device_caches_; device_caches.push_back(index_cache); } @@ -363,7 +363,7 @@ void HostIndexCache::deregister_device_cache(const CacheType cache_type, { assert(cache_type == CacheType::query_cache || cache_type == CacheType::target_cache); - std::vector& device_caches = cache_type == CacheType::query_cache ? device_caches_query_ : device_caches_target_; + std::vector& device_caches = cache_type == CacheType::query_cache ? query_device_caches_ : target_device_caches_; auto new_end = std::remove(begin(device_caches), end(device_caches), index_cache); device_caches.erase(new_end, end(device_caches)); diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index 7861073b9..85406eeab 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -104,6 +104,7 @@ private: std::shared_ptr device_index); /// \brief Returns requested index, returned index might not be ready and has to be synchronized directly + /// To be called by HostIndexCache::start_copying_indices_to_device() /// \param index_descriptor /// \throw IndexNotFoundException if requested index is not cached /// \return requested index @@ -227,8 +228,8 @@ private: device_cache_t target_indices_kept_on_device_; // Currently existing DeviceIndexCaches created by this HostIndexCache - std::vector device_caches_query_; - std::vector device_caches_target_; + std::vector query_device_caches_; + std::vector target_device_caches_; const bool same_query_and_target_; genomeworks::DefaultDeviceAllocator allocator_; From 69604f15b8cfb7b067746a8e47fb89a812da6d8f Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 1 Oct 2020 09:16:10 -0700 Subject: [PATCH 177/281] [cudaextender] Add issues for hardcoded vars --- cudaextender/src/ungapped_xdrop_kernels.cu | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 7561e2952..e90ac1f54 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -46,9 +46,13 @@ __global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_tar ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_done) { - constexpr int32_t num_warps = 4; // TODO - move out? - constexpr int32_t nuc = 8; // TODO - remove hardcode - pass in - constexpr int32_t nuc2 = 64; // TODO - remove hardcode - TBD based on encoding + // TODO - Following variables are an artifact of the hardcoded encoding scheme with a fixed + // scoring matrix and a fixed alphabet. Will be replaced with cudasequence API. + // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/574 + constexpr int32_t nuc = 8; + constexpr int32_t nuc2 = 64; + + constexpr int32_t num_warps = 4; constexpr int32_t warp_size = 32; const int32_t lane_id = threadIdx.x % warp_size; const int32_t warp_id = (threadIdx.x - lane_id) / warp_size; @@ -65,9 +69,11 @@ __global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_tar __shared__ int32_t left_extent[num_warps]; __shared__ int32_t extent[num_warps]; __shared__ int32_t tile[num_warps]; - __shared__ double entropy[num_warps]; // TODO - Can this be a float? __shared__ int32_t sub_mat[nuc2]; - + // TODO - Accuracy exploration required to check if the following member can be a float + // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/575 + __shared__ double entropy[num_warps]; + if (threadIdx.x < nuc2) { sub_mat[threadIdx.x] = d_sub_mat[threadIdx.x]; From 3481796db5457c0b7b4de6bd46f1d3b308103fa7 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 1 Oct 2020 11:01:59 -0700 Subject: [PATCH 178/281] [cudaextender] Add issues for kernel optimization and other kernel issues --- cudaextender/src/ungapped_xdrop.cu | 30 ++++++++++++++-------- cudaextender/src/ungapped_xdrop_kernels.cu | 8 ++++-- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 0e122c6f2..fb845cab7 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -55,17 +55,22 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat , host_ptr_api_mode_(false) , allocator_(allocator) { - // TODO - check sub_mat_dim based on Sequence Encoder API + // Switch to device for copying over initial structures + scoped_device_switch dev(device_id_); + // Calculate the max limits on the number of extensions we can do on this GPU cudaDeviceProp device_prop; cudaGetDeviceProperties(&device_prop, device_id_); - const int32_t max_ungapped_per_gb = 4194304; // FIXME: Calculate using sizeof datastructures - //const int32_t max_seed_pairs_per_gb = 8388608; // FIXME: Calculate using sizeof datastructures // TODO- Do we need this? + + // TODO - Currently element and memory limits are artifacts of hardcoded global memory limits in + // SegAlign. To be replaced with actual calculation of memory requirements with sizes of + // datastructures taken into consideration. Also currently the max limit is based on total + // global memory, which should be replaced with memory available from the passed in allocator. + // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/576 + const int32_t max_ungapped_per_gb = 4194304; const float global_mem_gb = static_cast(device_prop.totalGlobalMem) / 1073741824.0f; batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; - // Switch to device for copying over initial structures - scoped_device_switch dev(device_id_); - + //Figure out memory requirements for cub functions size_t temp_storage_bytes = 0; size_t cub_storage_bytes = 0; @@ -113,11 +118,11 @@ StatusType UngappedXDrop::extend_async(const int8_t* d_query, const int32_t quer total_scored_segment_pairs_ = 0; for (int32_t seed_pair_start = 0; seed_pair_start < num_seed_pairs; seed_pair_start += batch_max_ungapped_extensions_) { - // TODO - Do we need these? It seems we don't! + // TODO - Kernel optimizations [Unnecessary memset?] + // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/579 GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_done_.data(), 0, batch_max_ungapped_extensions_ * sizeof(int32_t), stream_)); GW_CU_CHECK_ERR(cudaMemsetAsync((void*)d_tmp_ssp_.data(), 0, batch_max_ungapped_extensions_ * sizeof(ScoredSegmentPair), stream_)); const int32_t curr_num_pairs = std::min(batch_max_ungapped_extensions_, num_seed_pairs - seed_pair_start); - // TODO- Extricate the kernel launch params? find_high_scoring_segment_pairs<<<1024, 128, 0, stream_>>>(d_target, target_length, d_query, @@ -138,15 +143,20 @@ StatusType UngappedXDrop::extend_async(const int8_t* d_query, const int32_t quer d_done_.data(), curr_num_pairs, stream_)) - // TODO- Make async + // TODO- Make output compression async. Currently synchronocity is arising due to + // thrust::stable_sort. Dynamic parallelism or an equivalent sort with cub can be used + // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/578 const int32_t num_scored_segment_pairs = get_value_from_device(d_done_.data() + curr_num_pairs - 1, stream_); if (num_scored_segment_pairs > 0) { + // TODO - Explore scaling up/down launch config based on workload. Also explore making + // this accessible to the user for configuration + // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/577 compress_output<<<1024, 1024, 0, stream_>>>(d_done_.data(), seed_pair_start, d_scored_segment_pairs, d_tmp_ssp_.data(), - curr_num_pairs); // TODO- Need configurability for kernel? + curr_num_pairs); thrust::stable_sort(thrust::cuda::par(allocator_).on(stream_), d_tmp_ssp_.begin(), d_tmp_ssp_.begin() + num_scored_segment_pairs, diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index e90ac1f54..b7320f644 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -46,6 +46,9 @@ __global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_tar ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_done) { + // TODO - Kernel optimizations - + // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/579 + // TODO - Following variables are an artifact of the hardcoded encoding scheme with a fixed // scoring matrix and a fixed alphabet. Will be replaced with cudasequence API. // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/574 @@ -70,7 +73,8 @@ __global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_tar __shared__ int32_t extent[num_warps]; __shared__ int32_t tile[num_warps]; __shared__ int32_t sub_mat[nuc2]; - // TODO - Accuracy exploration required to check if the following member can be a float + // TODO - Accuracy exploration required to check if the following member can be a float and to + // check the condition for entropy calculation. // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/575 __shared__ double entropy[num_warps]; @@ -394,7 +398,7 @@ __global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_tar } __syncwarp(); - if (lane_id == warp_size - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) // TODO - MAGIC NUMBER ALERT! + if (lane_id == warp_size - 1 && ((count[0] + count[1] + count[2] + count[3]) >= 20)) { double entropy_ln = 0.f; #pragma unroll From 8f414854c519c9eb11d3982f6c92b47a9cdd7173 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 1 Oct 2020 11:03:20 -0700 Subject: [PATCH 179/281] [cudaextender] make format --- cudaextender/src/ungapped_xdrop.cu | 18 +++++++++--------- cudaextender/src/ungapped_xdrop_kernels.cu | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index fb845cab7..72f0401b0 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -61,16 +61,16 @@ UngappedXDrop::UngappedXDrop(const int32_t* h_score_mat, const int32_t score_mat // Calculate the max limits on the number of extensions we can do on this GPU cudaDeviceProp device_prop; cudaGetDeviceProperties(&device_prop, device_id_); - + // TODO - Currently element and memory limits are artifacts of hardcoded global memory limits in - // SegAlign. To be replaced with actual calculation of memory requirements with sizes of - // datastructures taken into consideration. Also currently the max limit is based on total + // SegAlign. To be replaced with actual calculation of memory requirements with sizes of + // datastructures taken into consideration. Also currently the max limit is based on total // global memory, which should be replaced with memory available from the passed in allocator. // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/576 - const int32_t max_ungapped_per_gb = 4194304; - const float global_mem_gb = static_cast(device_prop.totalGlobalMem) / 1073741824.0f; - batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; - + const int32_t max_ungapped_per_gb = 4194304; + const float global_mem_gb = static_cast(device_prop.totalGlobalMem) / 1073741824.0f; + batch_max_ungapped_extensions_ = static_cast(global_mem_gb) * max_ungapped_per_gb; + //Figure out memory requirements for cub functions size_t temp_storage_bytes = 0; size_t cub_storage_bytes = 0; @@ -143,7 +143,7 @@ StatusType UngappedXDrop::extend_async(const int8_t* d_query, const int32_t quer d_done_.data(), curr_num_pairs, stream_)) - // TODO- Make output compression async. Currently synchronocity is arising due to + // TODO- Make output compression async. Currently synchronocity is arising due to // thrust::stable_sort. Dynamic parallelism or an equivalent sort with cub can be used // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/578 const int32_t num_scored_segment_pairs = get_value_from_device(d_done_.data() + curr_num_pairs - 1, stream_); @@ -151,7 +151,7 @@ StatusType UngappedXDrop::extend_async(const int8_t* d_query, const int32_t quer { // TODO - Explore scaling up/down launch config based on workload. Also explore making // this accessible to the user for configuration - // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/577 + // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/577 compress_output<<<1024, 1024, 0, stream_>>>(d_done_.data(), seed_pair_start, d_scored_segment_pairs, diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index b7320f644..2d8763a8c 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -46,15 +46,15 @@ __global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_tar ScoredSegmentPair* d_scored_segment_pairs, int32_t* d_done) { - // TODO - Kernel optimizations - + // TODO - Kernel optimizations - // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/579 - + // TODO - Following variables are an artifact of the hardcoded encoding scheme with a fixed // scoring matrix and a fixed alphabet. Will be replaced with cudasequence API. // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/574 - constexpr int32_t nuc = 8; - constexpr int32_t nuc2 = 64; - + constexpr int32_t nuc = 8; + constexpr int32_t nuc2 = 64; + constexpr int32_t num_warps = 4; constexpr int32_t warp_size = 32; const int32_t lane_id = threadIdx.x % warp_size; @@ -73,11 +73,11 @@ __global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_tar __shared__ int32_t extent[num_warps]; __shared__ int32_t tile[num_warps]; __shared__ int32_t sub_mat[nuc2]; - // TODO - Accuracy exploration required to check if the following member can be a float and to + // TODO - Accuracy exploration required to check if the following member can be a float and to // check the condition for entropy calculation. // Github Issue: https://github.com/clara-parabricks/GenomeWorks/issues/575 __shared__ double entropy[num_warps]; - + if (threadIdx.x < nuc2) { sub_mat[threadIdx.x] = d_sub_mat[threadIdx.x]; From 961c2a563fb493c5cdcab86e89f088fd278f6aba Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 1 Oct 2020 11:06:26 -0700 Subject: [PATCH 180/281] [cudaextender] Uniform comment names for host pointer API --- cudaextender/src/ungapped_xdrop.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cu b/cudaextender/src/ungapped_xdrop.cu index 72f0401b0..6aa2b438f 100644 --- a/cudaextender/src/ungapped_xdrop.cu +++ b/cudaextender/src/ungapped_xdrop.cu @@ -222,7 +222,7 @@ StatusType UngappedXDrop::sync() return StatusType::success; } - // If this function was called without using the host_ptr_api, throw error + // If this function was called without using the host pointer API, throw error return StatusType::invalid_operation; } @@ -232,7 +232,7 @@ const std::vector& UngappedXDrop::get_scored_segment_pairs() { return h_ssp_; } - // If this function was called using the host_ptr_api, throw error + // If this function was called using the host pointer API, throw error throw std::runtime_error("Invalid API call. Getting scored segment pairs without calling extend_async host ptr API"); } From 86e71b1da0a2ebd9fe8207a3d78cc7bf4f314dcb Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 1 Oct 2020 12:24:37 -0700 Subject: [PATCH 181/281] [cudaextender] Remove TODO --- cudaextender/src/extender.cpp | 1 - cudaextender/src/ungapped_xdrop.cuh | 8 ++++---- cudaextender/src/ungapped_xdrop_kernels.cu | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cudaextender/src/extender.cpp b/cudaextender/src/extender.cpp index 442cf2d94..6e37a037e 100644 --- a/cudaextender/src/extender.cpp +++ b/cudaextender/src/extender.cpp @@ -26,7 +26,6 @@ namespace genomeworks namespace cudaextender { -// TODO - Provide an allocator-less API for creation? std::unique_ptr create_extender(const int32_t* h_score_mat, int32_t score_mat_dim, int32_t xdrop_threshold, diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index f19d750f9..07195e91a 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -64,8 +64,8 @@ public: const std::vector& get_scored_segment_pairs() const override; private: - DefaultDeviceAllocator allocator_; // Device ptr API required variables + DefaultDeviceAllocator allocator_; pinned_host_vector h_score_mat_; const int32_t score_mat_dim_; // Assume matrix is square const int32_t xdrop_threshold_; @@ -73,10 +73,10 @@ private: cudaStream_t stream_; const int32_t device_id_; std::vector scored_segment_pairs_; - int32_t batch_max_ungapped_extensions_; // TODO - Make const + int32_t batch_max_ungapped_extensions_; device_buffer d_score_mat_; // Pointer to device substitution matrix - device_buffer d_done_; // TODO- Rename scratch space - device_buffer d_tmp_ssp_; // TODO- Rename Scratch space 2 + device_buffer d_done_; + device_buffer d_tmp_ssp_; int32_t total_scored_segment_pairs_; device_buffer d_temp_storage_cub_; // temporary storage for cub functions diff --git a/cudaextender/src/ungapped_xdrop_kernels.cu b/cudaextender/src/ungapped_xdrop_kernels.cu index 2d8763a8c..bf0367e48 100644 --- a/cudaextender/src/ungapped_xdrop_kernels.cu +++ b/cudaextender/src/ungapped_xdrop_kernels.cu @@ -426,7 +426,7 @@ __global__ void find_high_scoring_segment_pairs(const int8_t* __restrict__ d_tar d_scored_segment_pairs[hid].seed_pair.target_position_in_read = ref_loc[warp_id] - left_extent[warp_id]; d_scored_segment_pairs[hid].seed_pair.query_position_in_read = query_loc[warp_id] - left_extent[warp_id]; d_scored_segment_pairs[hid].length = extent[warp_id]; - if (entropy[warp_id] > 0) // TODO - Is this necessary? + if (entropy[warp_id] > 0) d_scored_segment_pairs[hid].score = total_score[warp_id] * entropy[warp_id]; d_done[hid - start_index] = 1; } From e6ffd17ff4205a4f145e12bcfdceeb4af592c63b Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 1 Oct 2020 12:26:41 -0700 Subject: [PATCH 182/281] [cudaextender] Remove todo --- cudaextender/samples/sample_cudaextender.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index a91c21415..87ad85793 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -91,7 +91,6 @@ int main(int argc, char* argv[]) // target_position_in_read_ const std::string seed_pairs_file_path = std::string(CUDAEXTENDER_DATA_DIR) + "/sample_seed_pairs.csv"; - //TODO - pinned seed_pairs std::vector h_seed_pairs; // Following function loops through all seed_pairs in the sample_seed_pairs.csv and returns // results in the passed vector From 7993dd26fb36152b1a533efcae4ee9f842d7e504 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 1 Oct 2020 12:27:11 -0700 Subject: [PATCH 183/281] [cudaextender] Make format --- cudaextender/src/ungapped_xdrop.cuh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cudaextender/src/ungapped_xdrop.cuh b/cudaextender/src/ungapped_xdrop.cuh index 07195e91a..79684779d 100644 --- a/cudaextender/src/ungapped_xdrop.cuh +++ b/cudaextender/src/ungapped_xdrop.cuh @@ -74,8 +74,8 @@ private: const int32_t device_id_; std::vector scored_segment_pairs_; int32_t batch_max_ungapped_extensions_; - device_buffer d_score_mat_; // Pointer to device substitution matrix - device_buffer d_done_; + device_buffer d_score_mat_; // Pointer to device substitution matrix + device_buffer d_done_; device_buffer d_tmp_ssp_; int32_t total_scored_segment_pairs_; device_buffer d_temp_storage_cub_; // temporary storage for cub functions From 3db3bc2e7a96e0f9d7aefd809519138d107d7a7d Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 1 Oct 2020 15:28:35 -0400 Subject: [PATCH 184/281] [cudapoa] breaking global kernel into 2 different kernels to be able to use different launch bound settings --- cudapoa/src/cudapoa_kernels.cuh | 335 ++++++++++++++++++++++++++++---- 1 file changed, 294 insertions(+), 41 deletions(-) diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 99a2373f5..aef6b74c4 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -29,7 +29,8 @@ #include // considering 65536 register file size, __launch_bounds__(1024) translates to max 64 register usage -#define GW_POA_KERNELS_MAX_THREADS_PER_BLOCK 1024 +#define MAX_THREADS_PER_BLOCK_64 1024 +#define MAX_THREADS_PER_BLOCK_72 896 namespace claraparabricks { @@ -73,7 +74,7 @@ namespace cudapoa * @param[in] match_score Score for finding a match in alignment */ template -__launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) +__launch_bounds__(MAX_THREADS_PER_BLOCK_64) __global__ void generatePOAKernel(uint8_t* consensus_d, uint8_t* sequences_d, int8_t* base_weights_d, @@ -146,18 +147,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) int64_t scores_offset = static_cast(banded_buffer_size) * static_cast(window_idx); scores = &scores_d[scores_offset]; } - else if (BM == BandMode::static_band_traceback || BM == BandMode::adaptive_band_traceback) - { - // buffer size for scores, in traceback we only need to store part of the scores matrix - banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); - int64_t offset = static_cast(banded_buffer_size) * static_cast(window_idx); - scores = &scores_d[offset]; - // buffer size for traceback - banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); - offset = static_cast(banded_buffer_size) * static_cast(window_idx); - traceback = &traceback_d[offset]; - } - else + else if (BM == BandMode::full_band) { int64_t offset = static_cast(window_details_d[window_idx].scores_offset) * static_cast(max_nodes_per_graph); scores = &scores_d[offset]; @@ -258,7 +248,7 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) } // Run Needleman-Wunsch alignment between graph and new sequence. - int32_t alignment_length; + SizeT alignment_length; // Adaptive band --------------------------------------------------------------------------------------- if (BM == BandMode::adaptive_band && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) @@ -331,8 +321,292 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) alignment_length); __syncwarp(); } + // Full band ------------------------------------------------------------------------------------------- + else if (BM == BandMode::full_band) + { + alignment_length = runNeedlemanWunsch(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + scores_width, + alignment_graph, + alignment_read, + gap_score, + mismatch_score, + match_score); + __syncwarp(); + } + + if (alignment_length == -1) + { + if (lane_idx == 0) + { + consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; + consensus[1] = static_cast(StatusType::loop_count_exceeded_upper_bound); + } + return; + } + else if (alignment_length == -2) + { + if (lane_idx == 0) + { + consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; + consensus[1] = static_cast(StatusType::exceeded_adaptive_banded_matrix_size); + } + return; + } + + if (lane_idx == 0) + { + + // Add alignment to graph. + SizeT new_node_count; + uint8_t error_code = addAlignmentToGraph(new_node_count, + nodes, sequence_lengths[0], + node_alignments, node_alignment_count, + incoming_edges, incoming_edge_count, + outgoing_edges, outgoing_edge_count, + incoming_edge_weights, + alignment_length, + sorted_poa, alignment_graph, + sequence, alignment_read, + node_coverage_counts, + base_weights, + (sequence_begin_nodes_ids + s), + outgoing_edges_coverage, + outgoing_edges_coverage_count, + s, + max_sequences_per_poa, + max_nodes_per_graph); + + if (error_code != 0) + { + consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; + consensus[1] = error_code; + warp_error = true; + } + else + { + sequence_lengths[0] = new_node_count; + // Run a topsort on the graph. +#ifdef SPOA_ACCURATE + // Exactly matches racon CPU results + raconTopologicalSortDeviceUtil(sorted_poa, + node_id_to_pos, + new_node_count, + incoming_edge_count, + incoming_edges, + node_alignment_count, + node_alignments, + node_marks, + check_aligned_nodes, + nodes_to_visit, + (uint16_t)max_nodes_per_graph); +#else + // Faster top sort + topologicalSortDeviceUtil(sorted_poa, + node_id_to_pos, + new_node_count, + incoming_edge_count, + outgoing_edges, + outgoing_edge_count, + sorted_poa_local_edge_count); +#endif + } + } + + __syncwarp(); + + warp_error = __shfl_sync(FULL_MASK, warp_error, 0); + if (warp_error) + { + return; + } + } +} + +// Similar to generatePOAKernel(), but with a different launch bounds setting, used to call kernels using traceback buffer +// i.e. BM == BandMode::static_band_traceback or BM == BandMode::adaptive_band_traceback +template +__launch_bounds__(MAX_THREADS_PER_BLOCK_72) + __global__ void generatePOAKernelTB(uint8_t* consensus_d, + uint8_t* sequences_d, + int8_t* base_weights_d, + SizeT* sequence_lengths_d, + genomeworks::cudapoa::WindowDetails* window_details_d, + int32_t total_windows, + ScoreT* scores_d, + SizeT* alignment_graph_d, + SizeT* alignment_read_d, + uint8_t* nodes_d, + SizeT* incoming_edges_d, + uint16_t* incoming_edge_count_d, + SizeT* outgoing_edges_d, + uint16_t* outgoing_edge_count_d, + uint16_t* incoming_edge_w_d, + SizeT* sorted_poa_d, + SizeT* node_id_to_pos_d, + SizeT* node_alignments_d, + uint16_t* node_alignment_count_d, + uint16_t* sorted_poa_local_edge_count_d, + uint8_t* node_marks_d_, + bool* check_aligned_nodes_d_, + SizeT* nodes_to_visit_d_, + uint16_t* node_coverage_counts_d_, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, + uint32_t max_sequences_per_poa, + SizeT* sequence_begin_nodes_ids_d, + uint16_t* outgoing_edges_coverage_d, + uint16_t* outgoing_edges_coverage_count_d, + int32_t max_nodes_per_graph, + int32_t scores_matrix_width, + int32_t max_limit_consensus_size, + int32_t TPB = 64, + int32_t static_band_width = 256, + int32_t max_pred_distance = 0, + TraceT* traceback_d = nullptr) +{ + // shared error indicator within a warp + bool warp_error = false; + + uint32_t lane_idx = threadIdx.x % WARP_SIZE; + uint32_t window_idx = blockIdx.x * TPB / WARP_SIZE + threadIdx.x / WARP_SIZE; + + if (window_idx >= total_windows) + return; + + // Find the buffer offsets for each thread within the global memory buffers. + uint8_t* nodes = &nodes_d[max_nodes_per_graph * window_idx]; + SizeT* incoming_edges = &incoming_edges_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES]; + uint16_t* incoming_edge_count = &incoming_edge_count_d[window_idx * max_nodes_per_graph]; + SizeT* outgoing_edges = &outgoing_edges_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES]; + uint16_t* outgoing_edge_count = &outgoing_edge_count_d[window_idx * max_nodes_per_graph]; + uint16_t* incoming_edge_weights = &incoming_edge_w_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES]; + SizeT* sorted_poa = &sorted_poa_d[window_idx * max_nodes_per_graph]; + SizeT* node_id_to_pos = &node_id_to_pos_d[window_idx * max_nodes_per_graph]; + SizeT* node_alignments = &node_alignments_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS]; + uint16_t* node_alignment_count = &node_alignment_count_d[window_idx * max_nodes_per_graph]; + uint16_t* sorted_poa_local_edge_count = &sorted_poa_local_edge_count_d[window_idx * max_nodes_per_graph]; + + ScoreT* scores; + TraceT* traceback = traceback_d; + float banded_buffer_size; // using float instead of int64_t to minimize register + // buffer size for scores, in traceback we only need to store part of the scores matrix + banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); + int64_t offset = static_cast(banded_buffer_size) * static_cast(window_idx); + scores = &scores_d[offset]; + // buffer size for traceback + banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); + offset = static_cast(banded_buffer_size) * static_cast(window_idx); + traceback = &traceback_d[offset]; + + SizeT* alignment_graph = &alignment_graph_d[max_nodes_per_graph * window_idx]; + SizeT* alignment_read = &alignment_read_d[max_nodes_per_graph * window_idx]; + uint16_t* node_coverage_counts = &node_coverage_counts_d_[max_nodes_per_graph * window_idx]; + +#ifdef SPOA_ACCURATE + uint8_t* node_marks = &node_marks_d_[max_nodes_per_graph * window_idx]; + bool* check_aligned_nodes = &check_aligned_nodes_d_[max_nodes_per_graph * window_idx]; + SizeT* nodes_to_visit = &nodes_to_visit_d_[max_nodes_per_graph * window_idx]; +#endif + + SizeT* sequence_lengths = &sequence_lengths_d[window_details_d[window_idx].seq_len_buffer_offset]; + + uint16_t num_sequences = window_details_d[window_idx].num_seqs; + uint8_t* sequence = &sequences_d[window_details_d[window_idx].seq_starts]; + int8_t* base_weights = &base_weights_d[window_details_d[window_idx].seq_starts]; + + uint8_t* consensus = &consensus_d[window_idx * max_limit_consensus_size]; + + SizeT* sequence_begin_nodes_ids = nullptr; + uint16_t* outgoing_edges_coverage = nullptr; + uint16_t* outgoing_edges_coverage_count = nullptr; + + if (MSA) + { + sequence_begin_nodes_ids = &sequence_begin_nodes_ids_d[window_idx * max_sequences_per_poa]; + outgoing_edges_coverage = &outgoing_edges_coverage_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * max_sequences_per_poa]; + outgoing_edges_coverage_count = &outgoing_edges_coverage_count_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES]; + } + + if (lane_idx == 0) + { + // Create backbone for window based on first sequence in window. + nodes[0] = sequence[0]; + sorted_poa[0] = 0; + incoming_edge_count[0] = 0; + node_alignment_count[0] = 0; + node_id_to_pos[0] = 0; + outgoing_edge_count[sequence_lengths[0] - 1] = 0; + incoming_edge_weights[0] = base_weights[0]; + node_coverage_counts[0] = 1; + if (MSA) + { + sequence_begin_nodes_ids[0] = 0; + } + + //Build the rest of the graphs + for (SizeT nucleotide_idx = 1; nucleotide_idx < sequence_lengths[0]; nucleotide_idx++) + { + nodes[nucleotide_idx] = sequence[nucleotide_idx]; + sorted_poa[nucleotide_idx] = nucleotide_idx; + outgoing_edges[(nucleotide_idx - 1) * CUDAPOA_MAX_NODE_EDGES] = nucleotide_idx; + outgoing_edge_count[nucleotide_idx - 1] = 1; + incoming_edges[nucleotide_idx * CUDAPOA_MAX_NODE_EDGES] = nucleotide_idx - 1; + incoming_edge_weights[nucleotide_idx * CUDAPOA_MAX_NODE_EDGES] = base_weights[nucleotide_idx - 1] + base_weights[nucleotide_idx]; + incoming_edge_count[nucleotide_idx] = 1; + node_alignment_count[nucleotide_idx] = 0; + node_id_to_pos[nucleotide_idx] = nucleotide_idx; + node_coverage_counts[nucleotide_idx] = 1; + if (MSA) + { + outgoing_edges_coverage[(nucleotide_idx - 1) * CUDAPOA_MAX_NODE_EDGES * max_sequences_per_poa] = 0; + outgoing_edges_coverage_count[(nucleotide_idx - 1) * CUDAPOA_MAX_NODE_EDGES] = 1; + } + } + + // Clear error code for window. + consensus[0] = CUDAPOA_KERNEL_NOERROR_ENCOUNTERED; + } + + __syncwarp(); + + // Align each subsequent read, add alignment to graph, run topological sort. + for (int32_t s = 1; s < num_sequences; s++) + { + int32_t seq_len = sequence_lengths[s]; + sequence += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data + base_weights += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data + + if (lane_idx == 0) + { + if (sequence_lengths[0] >= max_nodes_per_graph) + { + consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; + consensus[1] = static_cast(StatusType::node_count_exceeded_maximum_graph_size); + warp_error = true; + } + } + + warp_error = __shfl_sync(FULL_MASK, warp_error, 0); + if (warp_error) + { + return; + } + + // Run Needleman-Wunsch alignment between graph and new sequence. + SizeT alignment_length; + // Adaptive band with traceback ------------------------------------------------------------------------ - else if (BM == BandMode::adaptive_band_traceback && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) + if (BM == BandMode::adaptive_band_traceback && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) { alignment_length = runNeedlemanWunschBandedTraceback(nodes, sorted_poa, @@ -406,27 +680,6 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK) alignment_length); __syncwarp(); } - // Full band ------------------------------------------------------------------------------------------- - else if (BM == BandMode::full_band) - { - alignment_length = runNeedlemanWunsch(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - scores_width, - alignment_graph, - alignment_read, - gap_score, - mismatch_score, - match_score); - __syncwarp(); - } if (alignment_length == -1) { @@ -667,7 +920,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, } else if (batch_size.band_mode == BandMode::static_band_traceback) { - generatePOAKernel + generatePOAKernelTB <<>>(consensus_d, sequences_d, base_weights_d, @@ -709,7 +962,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, } else if (batch_size.band_mode == BandMode::adaptive_band_traceback) { - generatePOAKernel + generatePOAKernelTB <<>>(consensus_d, sequences_d, base_weights_d, @@ -875,7 +1128,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, } else if (batch_size.band_mode == BandMode::static_band_traceback) { - generatePOAKernel + generatePOAKernelTB <<>>(consensus_d, sequences_d, base_weights_d, @@ -917,7 +1170,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, } else if (batch_size.band_mode == BandMode::adaptive_band_traceback) { - generatePOAKernel + generatePOAKernelTB <<>>(consensus_d, sequences_d, base_weights_d, From 3fb71821977fd7cd34af4e4c11cd47717ee10be3 Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 2 Oct 2020 16:34:00 -0400 Subject: [PATCH 185/281] [cudapoa] addressing PR comments --- .../genomeworks/cudapoa/batch.hpp | 2 +- .../genomeworks/cudapoa/cudapoa.hpp | 12 ++++---- cudapoa/src/allocate_block.hpp | 10 +++---- cudapoa/src/batch.cu | 30 +++++++++---------- cudapoa/src/cudapoa_batch.cuh | 12 ++++---- cudapoa/src/cudapoa_kernels.cuh | 12 ++++---- cudapoa/src/cudapoa_limits.hpp | 4 +-- cudapoa/src/cudapoa_nw.cuh | 12 ++++---- cudapoa/src/cudapoa_nw_tb_banded.cuh | 6 ++++ cudapoa/src/cudapoa_structs.cuh | 2 +- cudapoa/tests/Test_CudapoaNW.cu | 6 ++-- 11 files changed, 57 insertions(+), 51 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp index 5305cc6d4..4da84cceb 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp @@ -73,7 +73,7 @@ struct BatchConfig /// Banding mode: full, static, adaptive BandMode band_mode; /// Maximum distance of predecessor node used in NW computations for static or adaptive-banded - int32_t max_pred_distance_in_banded_mode; + int32_t max_banded_pred_distance; /// constructor- set upper limit parameters based on max_seq_sz and band_width BatchConfig(int32_t max_seq_sz = 1024, int32_t max_seq_per_poa = 100, int32_t band_width = 256, BandMode banding = BandMode::full_band, diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp index acfa00861..5dcff0c30 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp @@ -44,15 +44,15 @@ enum StatusType generic_error }; -/// Banding mode used in Needleman-Wunch algorithm +/// Banding mode used in Needleman-Wunsch algorithm /// - full_band performs computations on full scores matrix, highest accuracy /// - static_band performs computations on a fixed band along scores matrix diagonal, fastest implementation /// - adaptive_band, similar to static_band performs computations on a band along diagonal, but the band-width -/// can vary per alignment's score matrix, faster than full_band and more accurate than static_band -/// - static_band_traceback computations are on a fixed band, score matrix is stored partially, instead traceback -/// information is stored in a buffer and used in backtracking. This can be useful for long-read data GPU memory is -/// limiting GPU parallelism. Traceback matrix requires 2x to 4x smaller memory compared to score matrix - +/// can vary per alignment's score matrix, faster than full_band and more accurate than static_band +/// - static_band_traceback similar to static_band, but uses traceback matrix. In this mode, score matrix is only +/// partially stored. The height of score matrix is equivalent to maximum predecessors distance and this maximum +/// distance is limited and smaller than full POA graph length. Traceback matrix requires less memory compared to +/// score matrix, and this banding mode can be useful for long-read cases where GPU memory is limiting parallelism. enum BandMode { full_band = 0, diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 4fdb90a0f..f008cc97e 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -62,7 +62,7 @@ class BatchBlock scoped_device_switch dev(device_id_); max_nodes_per_window_ = batch_size.max_nodes_per_graph; traceback_alignment_ = batch_size.band_mode == BandMode::static_band_traceback; - score_matrix_height_ = traceback_alignment_ ? batch_size.max_pred_distance_in_banded_mode : batch_size.max_nodes_per_graph; + score_matrix_height_ = traceback_alignment_ ? batch_size.max_banded_pred_distance : batch_size.max_nodes_per_graph; score_matrix_width_ = batch_size.matrix_sequence_dimension; // calculate static and dynamic sizes of buffers needed per POA entry. @@ -338,7 +338,7 @@ class BatchBlock int64_t device_size_per_poa = 0; int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; bool traceback = batch_size.band_mode == BandMode::static_band_traceback; - int32_t traceback_score_matrix_size = batch_size.matrix_sequence_dimension * batch_size.max_pred_distance_in_banded_mode; + int32_t traceback_score_matrix_size = batch_size.matrix_sequence_dimension * batch_size.max_banded_pred_distance; // for output - device device_size_per_poa += batch_size.max_consensus_size * sizeof(*OutputDetails::consensus); // output_details_d_->consensus @@ -414,13 +414,13 @@ class BatchBlock cudaMemGetInfo(&free, &total); size_t mem_per_batch = memory_usage_quota * free; // Using memory_usage_quota of GPU available memory for cudapoa batch. - int64_t sizeof_ScoreT = 2; - int64_t sizeof_TraceT = use16bitTrace(batch_size) ? 2 : 1; + int64_t sizeof_ScoreT = sizeof(int16_t); + int64_t sizeof_TraceT = use16bitTrace(batch_size) ? sizeof(int16_t) : sizeof(int8_t); int64_t device_size_per_poa = 0; if (use32bitScore(batch_size, gap_score, mismatch_score, match_score)) { - sizeof_ScoreT = 4; + sizeof_ScoreT = sizeof(int32_t); if (use32bitSize(batch_size)) { device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index 01aded35f..88c9966e1 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -41,7 +41,7 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa , alignment_band_width(cudautils::align(band_width)) , max_sequences_per_poa(max_seq_per_poa) , band_mode(banding) - , max_pred_distance_in_banded_mode(max_pred_dist > 0 ? max_pred_dist : 2 * cudautils::align(band_width)) + , max_banded_pred_distance(max_pred_dist > 0 ? max_pred_dist : 2 * cudautils::align(band_width)) { max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); @@ -82,14 +82,14 @@ BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t m , alignment_band_width(cudautils::align(band_width)) , max_sequences_per_poa(max_seq_per_poa) , band_mode(banding) - , max_pred_distance_in_banded_mode(max_pred_distance) + , max_banded_pred_distance(max_pred_distance) { throw_on_negative(max_seq_sz, "max_sequence_size cannot be negative."); throw_on_negative(max_consensus_sz, "max_consensus_size cannot be negative."); throw_on_negative(max_nodes_per_poa, "max_nodes_per_graph cannot be negative."); throw_on_negative(max_seq_per_poa, "max_sequences_per_poa cannot be negative."); throw_on_negative(band_width, "alignment_band_width cannot be negative."); - throw_on_negative(max_pred_distance, "max_pred_distance_in_banded_mode cannot be negative."); + throw_on_negative(max_pred_distance, "max_banded_pred_distance cannot be negative."); if (max_nodes_per_graph < max_sequence_size) throw std::invalid_argument("max_nodes_per_graph should be greater than or equal to max_sequence_size."); @@ -123,9 +123,9 @@ std::unique_ptr create_batch(int32_t device_id, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } else { @@ -134,9 +134,9 @@ std::unique_ptr create_batch(int32_t device_id, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } } else @@ -148,9 +148,9 @@ std::unique_ptr create_batch(int32_t device_id, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } else { @@ -159,9 +159,9 @@ std::unique_ptr create_batch(int32_t device_id, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } } } diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index c4cac03ba..a861b6b62 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -64,7 +64,7 @@ class CudapoaBatch : public Batch { public: CudapoaBatch(int32_t device_id, cudaStream_t stream, size_t max_gpu_mem, int8_t output_mask, - const BatchConfig& batch_size, ScoreT gap_score = -8, ScoreT mismatch_score = -6, ScoreT match_score = 8) + const BatchConfig& batch_size, int32_t gap_score = -8, int32_t mismatch_score = -6, int32_t match_score = 8) : max_sequences_per_poa_(throw_on_negative(batch_size.max_sequences_per_poa, "Maximum sequences per POA has to be non-negative")) , device_id_(throw_on_negative(device_id, "Device ID has to be non-negative")) , stream_(stream) @@ -537,9 +537,9 @@ protected: } input_details_h_->sequence_lengths[global_sequence_idx_] = seq_len; - // to be aligned with uchar4 size, pad sequence length to be multiple of 32 + // to be aligned with uchar4 size, pad sequence length to be multiple of 4 num_nucleotides_copied_ += cudautils::align(seq_len); - ; + global_sequence_idx_++; return StatusType::success; @@ -588,9 +588,9 @@ protected: BatchConfig batch_size_; // Gap, mismatch and match scores for NW dynamic programming loop. - ScoreT gap_score_; - ScoreT mismatch_score_; - ScoreT match_score_; + int32_t gap_score_; + int32_t mismatch_score_; + int32_t match_score_; // Host and device buffer for output data. OutputDetails* output_details_h_; diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 7812435f2..03be70470 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -565,7 +565,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode); + batch_size.max_banded_pred_distance); } else if (batch_size.band_mode == BandMode::adaptive_band) { @@ -606,7 +606,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode); + batch_size.max_banded_pred_distance); } else if (batch_size.band_mode == BandMode::static_band_traceback) { @@ -647,7 +647,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode, + batch_size.max_banded_pred_distance, traceback); } else @@ -731,7 +731,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode); + batch_size.max_banded_pred_distance); } else if (batch_size.band_mode == BandMode::adaptive_band) { @@ -772,7 +772,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode); + batch_size.max_banded_pred_distance); } else if (batch_size.band_mode == BandMode::static_band_traceback) { @@ -813,7 +813,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode, + batch_size.max_banded_pred_distance, traceback); } else diff --git a/cudapoa/src/cudapoa_limits.hpp b/cudapoa/src/cudapoa_limits.hpp index 6fd508b7e..cf1ffaa0f 100644 --- a/cudapoa/src/cudapoa_limits.hpp +++ b/cudapoa/src/cudapoa_limits.hpp @@ -54,8 +54,8 @@ static bool use32bitSize(const BatchConfig& batch_size) static bool use16bitTrace(const BatchConfig& batch_size) { - //if max_pred_distance_in_banded_mode exceeds the range represented by int8_t, then int16_t should be used - return (batch_size.max_pred_distance_in_banded_mode > INT8_MAX); + //if max_banded_pred_distance exceeds the range represented by int8_t, then int16_t should be used + return (batch_size.max_banded_pred_distance > INT8_MAX); } } // namespace cudapoa diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index d30455624..4bdd267a2 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -467,9 +467,9 @@ __global__ void runNeedlemanWunschKernel(uint8_t* nodes, int32_t scores_width, SizeT* alignment_graph, SizeT* alignment_read, - int16_t gap_score, - int16_t mismatch_score, - int16_t match_score, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, SizeT* aligned_nodes) { *aligned_nodes = runNeedlemanWunsch(nodes, @@ -505,9 +505,9 @@ void runNW(uint8_t* nodes, int32_t scores_width, SizeT* alignment_graph, SizeT* alignment_read, - int16_t gap_score, - int16_t mismatch_score, - int16_t match_score, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, SizeT* aligned_nodes) { runNeedlemanWunschKernel<<<1, 64>>>(nodes, diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index bd79b6714..0f0871748 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -86,6 +86,7 @@ __device__ __forceinline__ void set_score_tb(ScoreT* scores, col_idx = column - band_start; } + // row is mapped to [0, score_matrix_height) span int64_t score_index = static_cast(col_idx) + static_cast(row % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[score_index] = value; @@ -143,6 +144,7 @@ __device__ __forceinline__ ScoreT get_score_tb(ScoreT* scores, } else { + // row is mapped to [0, score_matrix_height) span return *get_score_ptr_tb(scores, row % score_matrix_height, column, band_start, band_width); } } @@ -184,6 +186,7 @@ __device__ __forceinline__ void get_scores_tb(ScoreT* scores, } else { + // row is mapped to [0, score_matrix_height) span ScoreT4* pred_scores = (ScoreT4*)get_score_ptr_tb(scores, pred_node % score_matrix_height, read_pos, band_start, band_width); // loads 8/16 consecutive bytes (4 ScoreT) @@ -339,6 +342,7 @@ __device__ __forceinline__ pred_count = incoming_edge_count[node_id]; if (pred_count == 0) { + // row is mapped to [0, score_matrix_height) span int64_t index = static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[index] = gap_score; index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); @@ -419,6 +423,7 @@ __device__ __forceinline__ TraceT4 trace; ScoreT4 score = {min_score_value, min_score_value, min_score_value, min_score_value}; + // note that whenever accessing a score matrix row, the row needs to be mapped to [0, score_matrix_height) get_scores_tb(scores, pred_idx, score_gIdx, read_pos, score_matrix_height, gradient, band_width, max_column, gap_score, match_score, mismatch_score, read4, graph_base, score, trace); @@ -485,6 +490,7 @@ __device__ __forceinline__ // which can be used to compute the first cell of the next warp. first_element_prev_score = __shfl_sync(FULL_MASK, score.s3, WARP_SIZE - 1); + // row is mapped to [0, score_matrix_height) span int64_t index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[index] = score.s0; scores[index + 1L] = score.s1; diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index 9f5883619..7076d48b5 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -27,7 +27,7 @@ #define CUDAPOA_MAX_NODE_ALIGNMENTS 50 // Size of read chunks for reading sequences -#define SIZE_OF_SeqT4 32 +#define SIZE_OF_SeqT4 4 // Dimensions for Banded alignment score matrix #define WARP_SIZE 32 diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index 586fb4a9d..da6d8fbeb 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -204,9 +204,9 @@ NWAnswer testNW(const BasicNW& obj) int16_t* scores; SizeT* alignment_graph; SizeT* alignment_read; - int16_t gap_score; - int16_t mismatch_score; - int16_t match_score; + int32_t gap_score; + int32_t mismatch_score; + int32_t match_score; SizeT* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) BatchConfig batch_size; //default max_sequence_size = 1024, max_sequences_per_poa = 100 From 94625bae700d9a7542cc4459c20bc08ea77bcbc0 Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 2 Oct 2020 16:50:20 -0400 Subject: [PATCH 186/281] [cudapoa] addressing PR comments --- .../genomeworks/cudapoa/batch.hpp | 2 +- .../genomeworks/cudapoa/cudapoa.hpp | 10 +++---- cudapoa/src/allocate_block.hpp | 10 +++---- cudapoa/src/batch.cu | 30 +++++++++---------- cudapoa/src/cudapoa_batch.cuh | 12 ++++---- cudapoa/src/cudapoa_kernels.cuh | 25 ++++++---------- cudapoa/src/cudapoa_limits.hpp | 4 +-- cudapoa/src/cudapoa_nw.cuh | 12 ++++---- cudapoa/src/cudapoa_nw_tb_banded.cuh | 3 ++ cudapoa/src/cudapoa_structs.cuh | 2 +- cudapoa/tests/Test_CudapoaNW.cu | 6 ++-- 11 files changed, 56 insertions(+), 60 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp index 5305cc6d4..4da84cceb 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp @@ -73,7 +73,7 @@ struct BatchConfig /// Banding mode: full, static, adaptive BandMode band_mode; /// Maximum distance of predecessor node used in NW computations for static or adaptive-banded - int32_t max_pred_distance_in_banded_mode; + int32_t max_banded_pred_distance; /// constructor- set upper limit parameters based on max_seq_sz and band_width BatchConfig(int32_t max_seq_sz = 1024, int32_t max_seq_per_poa = 100, int32_t band_width = 256, BandMode banding = BandMode::full_band, diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp index 105db9c4a..5c09d3ef4 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp @@ -48,11 +48,11 @@ enum StatusType /// - full_band performs computations on full scores matrix, highest accuracy /// - static_band performs computations on a fixed band along scores matrix diagonal, fastest implementation /// - adaptive_band, similar to static_band performs computations on a band along diagonal, but the band-width -/// can vary per alignment's score matrix, faster than full_band and more accurate than static_band -/// - static_band_traceback computations are on a fixed band, score matrix is stored partially and there is an -/// additional traceback matrix. Accuracy is similar to static_band. This mode can particularly be useful -/// for long-read data where GPU memory is limiting parallelism. Traceback matrix requires 2x to 4x smaller -/// memory compared to score matrix +/// can vary per alignment's score matrix, faster than full_band and more accurate than static_band +/// - static_band_traceback is similar to static_band, but uses traceback matrix. In this mode, score matrix is only +/// partially stored. The height of score matrix is equivalent to maximum predecessors distance and this maximum +/// distance is limited and smaller than full POA graph length. Traceback matrix requires less memory compared to +/// score matrix, and this banding mode can be useful for long-read cases where GPU memory is limiting parallelism. /// - adaptive_band_traceback, similar to static_band_traceback but with varying band-width size enum BandMode { diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index cacd4e88c..1d6a42d54 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -62,7 +62,7 @@ class BatchBlock scoped_device_switch dev(device_id_); max_nodes_per_window_ = batch_size.max_nodes_per_graph; traceback_alignment_ = batch_size.band_mode == BandMode::static_band_traceback || batch_size.band_mode == BandMode::adaptive_band_traceback; - score_matrix_height_ = traceback_alignment_ ? batch_size.max_pred_distance_in_banded_mode : batch_size.max_nodes_per_graph; + score_matrix_height_ = traceback_alignment_ ? batch_size.max_banded_pred_distance : batch_size.max_nodes_per_graph; score_matrix_width_ = batch_size.matrix_sequence_dimension; // calculate static and dynamic sizes of buffers needed per POA entry. @@ -338,7 +338,7 @@ class BatchBlock int64_t device_size_per_poa = 0; int32_t max_nodes_per_graph = batch_size.max_nodes_per_graph; bool traceback = batch_size.band_mode == static_band_traceback || batch_size.band_mode == adaptive_band_traceback; - int32_t traceback_score_matrix_size = batch_size.matrix_sequence_dimension * batch_size.max_pred_distance_in_banded_mode; + int32_t traceback_score_matrix_size = batch_size.matrix_sequence_dimension * batch_size.max_banded_pred_distance; // for output - device device_size_per_poa += batch_size.max_consensus_size * sizeof(*OutputDetails::consensus); // output_details_d_->consensus @@ -414,13 +414,13 @@ class BatchBlock cudaMemGetInfo(&free, &total); size_t mem_per_batch = memory_usage_quota * free; // Using memory_usage_quota of GPU available memory for cudapoa batch. - int64_t sizeof_ScoreT = 2; - int64_t sizeof_TraceT = use16bitTrace(batch_size) ? 2 : 1; + int64_t sizeof_ScoreT = sizeof(int16_t); + int64_t sizeof_TraceT = use16bitTrace(batch_size) ? sizeof(int16_t) : sizeof(int8_t); int64_t device_size_per_poa = 0; if (use32bitScore(batch_size, gap_score, mismatch_score, match_score)) { - sizeof_ScoreT = 4; + sizeof_ScoreT = sizeof(int32_t); if (use32bitSize(batch_size)) { device_size_per_poa = BatchBlock::compute_device_memory_per_poa(batch_size, msa_flag); diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index edb32c552..5b577e064 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -41,7 +41,7 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa , alignment_band_width(cudautils::align(band_width)) , max_sequences_per_poa(max_seq_per_poa) , band_mode(banding) - , max_pred_distance_in_banded_mode(max_pred_dist > 0 ? max_pred_dist : 2 * cudautils::align(band_width)) + , max_banded_pred_distance(max_pred_dist > 0 ? max_pred_dist : 2 * cudautils::align(band_width)) { max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); @@ -82,14 +82,14 @@ BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t m , alignment_band_width(cudautils::align(band_width)) , max_sequences_per_poa(max_seq_per_poa) , band_mode(banding) - , max_pred_distance_in_banded_mode(max_pred_distance) + , max_banded_pred_distance(max_pred_distance) { throw_on_negative(max_seq_sz, "max_sequence_size cannot be negative."); throw_on_negative(max_consensus_sz, "max_consensus_size cannot be negative."); throw_on_negative(max_nodes_per_poa, "max_nodes_per_graph cannot be negative."); throw_on_negative(max_seq_per_poa, "max_sequences_per_poa cannot be negative."); throw_on_negative(band_width, "alignment_band_width cannot be negative."); - throw_on_negative(max_pred_distance, "max_pred_distance_in_banded_mode cannot be negative."); + throw_on_negative(max_pred_distance, "max_banded_pred_distance cannot be negative."); if (max_nodes_per_graph < max_sequence_size) throw std::invalid_argument("max_nodes_per_graph should be greater than or equal to max_sequence_size."); @@ -123,9 +123,9 @@ std::unique_ptr create_batch(int32_t device_id, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } else { @@ -134,9 +134,9 @@ std::unique_ptr create_batch(int32_t device_id, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } } else @@ -148,9 +148,9 @@ std::unique_ptr create_batch(int32_t device_id, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } else { @@ -159,9 +159,9 @@ std::unique_ptr create_batch(int32_t device_id, max_mem, output_mask, batch_size, - (int32_t)gap_score, - (int32_t)mismatch_score, - (int32_t)match_score); + gap_score, + mismatch_score, + match_score); } } } diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index f31187c4e..aa331b5ff 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -64,7 +64,7 @@ class CudapoaBatch : public Batch { public: CudapoaBatch(int32_t device_id, cudaStream_t stream, size_t max_gpu_mem, int8_t output_mask, - const BatchConfig& batch_size, ScoreT gap_score = -8, ScoreT mismatch_score = -6, ScoreT match_score = 8) + const BatchConfig& batch_size, int32_t gap_score = -8, int32_t mismatch_score = -6, int32_t match_score = 8) : max_sequences_per_poa_(throw_on_negative(batch_size.max_sequences_per_poa, "Maximum sequences per POA has to be non-negative")) , device_id_(throw_on_negative(device_id, "Device ID has to be non-negative")) , stream_(stream) @@ -537,9 +537,9 @@ protected: } input_details_h_->sequence_lengths[global_sequence_idx_] = seq_len; - // to be aligned with uchar4 size, pad sequence length to be multiple of 32 + // to be aligned with uchar4 size, pad sequence length to be multiple of 4 num_nucleotides_copied_ += cudautils::align(seq_len); - ; + global_sequence_idx_++; return StatusType::success; @@ -588,9 +588,9 @@ protected: BatchConfig batch_size_; // Gap, mismatch and match scores for NW dynamic programming loop. - ScoreT gap_score_; - ScoreT mismatch_score_; - ScoreT match_score_; + int32_t gap_score_; + int32_t mismatch_score_; + int32_t match_score_; // Host and device buffer for output data. OutputDetails* output_details_h_; diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index aef6b74c4..868abd1ac 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -110,9 +110,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) int32_t scores_matrix_width, int32_t max_limit_consensus_size, int32_t TPB = 64, - int32_t static_band_width = 256, - int32_t max_pred_distance = 0, - TraceT* traceback_d = nullptr) + int32_t static_band_width = 256) { // shared error indicator within a warp bool warp_error = false; @@ -139,7 +137,6 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) int32_t scores_width = window_details_d[window_idx].scores_width; ScoreT* scores; - TraceT* traceback = traceback_d; float banded_buffer_size; // using float instead of int64_t to minimize register if (BM == BandMode::adaptive_band || BM == BandMode::static_band) { @@ -874,8 +871,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_seq_dimension, batch_size.max_consensus_size, TPB, - batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode); + batch_size.alignment_band_width); } else if (batch_size.band_mode == BandMode::adaptive_band) { @@ -915,8 +911,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_seq_dimension, batch_size.max_consensus_size, TPB, - batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode); + batch_size.alignment_band_width); } else if (batch_size.band_mode == BandMode::static_band_traceback) { @@ -957,7 +952,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode, + batch_size.max_banded_pred_distance, traceback); } else if (batch_size.band_mode == BandMode::adaptive_band_traceback) @@ -999,7 +994,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode, + batch_size.max_banded_pred_distance, traceback); } else @@ -1082,8 +1077,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_seq_dimension, batch_size.max_consensus_size, TPB, - batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode); + batch_size.alignment_band_width); } else if (batch_size.band_mode == BandMode::adaptive_band) { @@ -1123,8 +1117,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, matrix_seq_dimension, batch_size.max_consensus_size, TPB, - batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode); + batch_size.alignment_band_width); } else if (batch_size.band_mode == BandMode::static_band_traceback) { @@ -1165,7 +1158,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode, + batch_size.max_banded_pred_distance, traceback); } else if (batch_size.band_mode == BandMode::adaptive_band_traceback) @@ -1207,7 +1200,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, batch_size.max_consensus_size, TPB, batch_size.alignment_band_width, - batch_size.max_pred_distance_in_banded_mode, + batch_size.max_banded_pred_distance, traceback); } else diff --git a/cudapoa/src/cudapoa_limits.hpp b/cudapoa/src/cudapoa_limits.hpp index 6fd508b7e..cf1ffaa0f 100644 --- a/cudapoa/src/cudapoa_limits.hpp +++ b/cudapoa/src/cudapoa_limits.hpp @@ -54,8 +54,8 @@ static bool use32bitSize(const BatchConfig& batch_size) static bool use16bitTrace(const BatchConfig& batch_size) { - //if max_pred_distance_in_banded_mode exceeds the range represented by int8_t, then int16_t should be used - return (batch_size.max_pred_distance_in_banded_mode > INT8_MAX); + //if max_banded_pred_distance exceeds the range represented by int8_t, then int16_t should be used + return (batch_size.max_banded_pred_distance > INT8_MAX); } } // namespace cudapoa diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index d30455624..4bdd267a2 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -467,9 +467,9 @@ __global__ void runNeedlemanWunschKernel(uint8_t* nodes, int32_t scores_width, SizeT* alignment_graph, SizeT* alignment_read, - int16_t gap_score, - int16_t mismatch_score, - int16_t match_score, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, SizeT* aligned_nodes) { *aligned_nodes = runNeedlemanWunsch(nodes, @@ -505,9 +505,9 @@ void runNW(uint8_t* nodes, int32_t scores_width, SizeT* alignment_graph, SizeT* alignment_read, - int16_t gap_score, - int16_t mismatch_score, - int16_t match_score, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, SizeT* aligned_nodes) { runNeedlemanWunschKernel<<<1, 64>>>(nodes, diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index ce4e002c4..478de653b 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -385,6 +385,7 @@ __device__ __forceinline__ pred_count = incoming_edge_count[node_id]; if (pred_count == 0) { + // row is mapped to [0, score_matrix_height) span int64_t index = static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[index] = gap_score; index = static_cast(score_gIdx) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); @@ -468,6 +469,7 @@ __device__ __forceinline__ TraceT4 trace; ScoreT4 score = {min_score_value, min_score_value, min_score_value, min_score_value}; + // note that whenever accessing a score matrix row, the row needs to be mapped to [0, score_matrix_height) get_scores_tb(scores, pred_idx, score_gIdx, read_pos, score_matrix_height, band_width, band_shift, gradient, max_column, gap_score, match_score, mismatch_score, read4, graph_base, score, trace); @@ -534,6 +536,7 @@ __device__ __forceinline__ // which can be used to compute the first cell of the next warp. first_element_prev_score = __shfl_sync(FULL_MASK, score.s3, WARP_SIZE - 1); + // row is mapped to [0, score_matrix_height) span int64_t index = static_cast(read_pos + 1 - band_start) + static_cast(score_gIdx % score_matrix_height) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); scores[index] = score.s0; scores[index + 1L] = score.s1; diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index 9f5883619..7076d48b5 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -27,7 +27,7 @@ #define CUDAPOA_MAX_NODE_ALIGNMENTS 50 // Size of read chunks for reading sequences -#define SIZE_OF_SeqT4 32 +#define SIZE_OF_SeqT4 4 // Dimensions for Banded alignment score matrix #define WARP_SIZE 32 diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index 586fb4a9d..da6d8fbeb 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -204,9 +204,9 @@ NWAnswer testNW(const BasicNW& obj) int16_t* scores; SizeT* alignment_graph; SizeT* alignment_read; - int16_t gap_score; - int16_t mismatch_score; - int16_t match_score; + int32_t gap_score; + int32_t mismatch_score; + int32_t match_score; SizeT* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) BatchConfig batch_size; //default max_sequence_size = 1024, max_sequences_per_poa = 100 From 5b292b4b459abda13f4953bf39b0809e42112643 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Mon, 5 Oct 2020 17:19:56 +0200 Subject: [PATCH 187/281] [common] Add default stream in CachingDeviceAllocator's default constructor as well --- .../claraparabricks/genomeworks/utils/allocator.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp index ddc0b1d4b..250d51ab3 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp @@ -172,13 +172,16 @@ class CachingDeviceAllocator /// Constructs an invalid CachingDeviceAllocator to allow default-construction of containers. /// A container using this allocator needs obtain a non-default constructed CachingDeviceAllocator object before performing any allocations. /// This can be achieved through through container assignment for example. - CachingDeviceAllocator() = default; + CachingDeviceAllocator() + : default_streams_(std::vector(1, 0)) + { + } /// \brief Constructor /// \param max_cached_bytes max bytes used by memory resource /// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead explicit CachingDeviceAllocator(size_t max_cached_bytes, - const std::vector& default_streams = {{0}}) + const std::vector& default_streams = std::vector(1, 0)) : memory_resource_(std::make_shared(max_cached_bytes)) , default_streams_(default_streams) { From a182f5f4bcc5d8e949dd2e0a33a3044e54dd3396 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Mon, 5 Oct 2020 17:30:24 +0200 Subject: [PATCH 188/281] [common] Copying/moving default_streams in CachingDeviceAllocator --- .../claraparabricks/genomeworks/utils/allocator.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp index 250d51ab3..cc9365101 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp @@ -198,6 +198,7 @@ class CachingDeviceAllocator template CachingDeviceAllocator(const CachingDeviceAllocator& rhs) : memory_resource_(rhs.memory_resource()) + , default_streams_(rhs.default_streams()) { } @@ -215,6 +216,7 @@ class CachingDeviceAllocator CachingDeviceAllocator& operator=(const CachingDeviceAllocator& rhs) { memory_resource_ = rhs.memory_resource(); + default_streams_ = rhs.default_streams(); return *this; } @@ -229,6 +231,7 @@ class CachingDeviceAllocator template CachingDeviceAllocator(CachingDeviceAllocator&& rhs) : memory_resource_(rhs.memory_resource()) + , default_streams_(std::move(rhs.default_streams())) { } @@ -246,6 +249,7 @@ class CachingDeviceAllocator CachingDeviceAllocator& operator=(CachingDeviceAllocator&& rhs) { memory_resource_ = rhs.memory_resource(); + default_streams_ = std::move(rhs.default_streams()); return *this; } @@ -307,6 +311,10 @@ class CachingDeviceAllocator /// \return a shared pointer to memory_resource std::shared_ptr memory_resource() const { return memory_resource_; } + /// \brief returns default streams + /// \return default streams + const std::vector default_streams() const { return default_streams_; } + private: std::shared_ptr memory_resource_; std::vector default_streams_; From e796e095ce31fc26571b66743f431d375b6af27d Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Mon, 5 Oct 2020 18:41:40 +0200 Subject: [PATCH 189/281] [common] Only one default stream in CachingDeviceAllocator --- .../genomeworks/utils/allocator.hpp | 47 +++++++++---------- cudamapper/src/main.cu | 2 +- .../Test_CudamapperOverlapperTriggered.cu | 6 +-- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp index cc9365101..c2d0d0e4c 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp @@ -44,10 +44,10 @@ class CudaMallocAllocator using pointer = T*; /// \brief default constructor - /// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead, ignored in this allocator - explicit CudaMallocAllocator(const std::vector& default_streams = {{0}}) + /// \param default_stream if a call to allocate() does not specify any streams this stream will be used instead, ignored in this allocator + explicit CudaMallocAllocator(cudaStream_t default_stream = 0) { - static_cast(default_streams); + static_cast(default_stream); } /// \brief copy constructor @@ -173,17 +173,17 @@ class CachingDeviceAllocator /// A container using this allocator needs obtain a non-default constructed CachingDeviceAllocator object before performing any allocations. /// This can be achieved through through container assignment for example. CachingDeviceAllocator() - : default_streams_(std::vector(1, 0)) + : default_stream_(0) { } /// \brief Constructor /// \param max_cached_bytes max bytes used by memory resource - /// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead + /// \param default_stream if a call to allocate() does not specify any streams this stream will be used instead explicit CachingDeviceAllocator(size_t max_cached_bytes, - const std::vector& default_streams = std::vector(1, 0)) + cudaStream_t default_stream = 0) : memory_resource_(std::make_shared(max_cached_bytes)) - , default_streams_(default_streams) + , default_stream_(default_stream) { } @@ -198,7 +198,7 @@ class CachingDeviceAllocator template CachingDeviceAllocator(const CachingDeviceAllocator& rhs) : memory_resource_(rhs.memory_resource()) - , default_streams_(rhs.default_streams()) + , default_stream_(rhs.default_stream()) { } @@ -216,7 +216,7 @@ class CachingDeviceAllocator CachingDeviceAllocator& operator=(const CachingDeviceAllocator& rhs) { memory_resource_ = rhs.memory_resource(); - default_streams_ = rhs.default_streams(); + default_stream_ = rhs.default_stream(); return *this; } @@ -231,7 +231,7 @@ class CachingDeviceAllocator template CachingDeviceAllocator(CachingDeviceAllocator&& rhs) : memory_resource_(rhs.memory_resource()) - , default_streams_(std::move(rhs.default_streams())) + , default_stream_(rhs.default_stream()) { } @@ -249,7 +249,7 @@ class CachingDeviceAllocator CachingDeviceAllocator& operator=(CachingDeviceAllocator&& rhs) { memory_resource_ = rhs.memory_resource(); - default_streams_ = std::move(rhs.default_streams()); + default_stream_ = rhs.default_stream(); return *this; } @@ -258,24 +258,23 @@ class CachingDeviceAllocator /// \brief asynchronously allocates a device array with enough space for n elements of value_type /// \param n number of elements to allocate the array for - /// \param streams on deallocation this memory block is guaranteed to live at least until all previously scheduled work in these streams has finished, if no streams are default_streams from constructor are used, if no default_streams were specified in constructor default stream is used + /// \param streams on deallocation this memory block is guaranteed to live at least until all previously scheduled work in these streams has finished, if no streams are specified default_stream from constructor are used, if no default_stream was specified in constructor default stream is used /// \return pointer to allocated memory /// \throw device_memory_allocation_exception if allocation was not successful pointer allocate(std::size_t n, const std::vector& streams = {}) { - assert(!streams.empty() || !default_streams_.empty()); - if (!memory_resource_) { GW_LOG_ERROR("{}\n", "ERROR:: Trying to allocate memory from an default-constructed CachingDeviceAllocator. Please assign a non-default-constructed CachingDeviceAllocator before performing any memory operations."); assert(false); std::abort(); } + void* ptr = nullptr; cudaError_t err = memory_resource_->DeviceAllocate(&ptr, n * sizeof(T), - streams.empty() ? default_streams_ : streams); // if no streams have been specified use default_streams_ + streams.empty() ? std::vector(1, default_stream_) : streams); // if no streams have been specified use default_stream_ if (err == cudaErrorMemoryAllocation) { throw device_memory_allocation_exception(); @@ -311,13 +310,13 @@ class CachingDeviceAllocator /// \return a shared pointer to memory_resource std::shared_ptr memory_resource() const { return memory_resource_; } - /// \brief returns default streams - /// \return default streams - const std::vector default_streams() const { return default_streams_; } + /// \brief returns default stream + /// \return default stream + cudaStream_t default_stream() const { return default_stream_; } private: std::shared_ptr memory_resource_; - std::vector default_streams_; + cudaStream_t default_stream_; }; #ifdef GW_ENABLE_CACHING_ALLOCATOR @@ -344,16 +343,16 @@ inline int64_t get_size_of_largest_free_memory_block(DefaultDeviceAllocator cons /// Default constuction of CachingDeviceAllocator yields an dummy object /// which cannot allocate memory. /// \param max_cached_bytes max bytes used by memory resource used by CachingDeviceAllocator (default: 2GiB, unused for CudaMallocAllocator) -/// \param default_streams if a call to allocate() does not specify any streams these streams will be used instead (unused for CudaMallocAllocator) -inline DefaultDeviceAllocator create_default_device_allocator(std::size_t max_caching_size = 2ull * 1024 * 1024 * 1024, - const std::vector& default_streams = {}) +/// \param default_stream if a call to allocate() does not specify any streams this stream will be used instead (unused for CudaMallocAllocator) +inline DefaultDeviceAllocator create_default_device_allocator(std::size_t max_caching_size = 2ull * 1024 * 1024 * 1024, + cudaStream_t default_stream = 0) { #ifdef GW_ENABLE_CACHING_ALLOCATOR return DefaultDeviceAllocator(max_caching_size, - default_streams); + default_stream); #else static_cast(max_caching_size); - static_cast(default_streams); + static_cast(default_stream); return DefaultDeviceAllocator(); #endif } diff --git a/cudamapper/src/main.cu b/cudamapper/src/main.cu index 9e06c0f9a..f40b84b61 100644 --- a/cudamapper/src/main.cu +++ b/cudamapper/src/main.cu @@ -453,7 +453,7 @@ void worker_thread_function(const int32_t device_id, // Whenever device_allocator is used directly (e.g. in Thrust library) it will be associated with cuda_stream_computation DefaultDeviceAllocator device_allocator = create_default_device_allocator(application_parameters.max_cached_memory_bytes, - {cuda_stream_computation}); + cuda_stream_computation); // create index_cache, indices are not created at this point but later as each batch gets processed HostIndexCache host_index_cache(application_parameters.all_to_all, diff --git a/cudamapper/tests/Test_CudamapperOverlapperTriggered.cu b/cudamapper/tests/Test_CudamapperOverlapperTriggered.cu index 9d8510415..cab478b2e 100644 --- a/cudamapper/tests/Test_CudamapperOverlapperTriggered.cu +++ b/cudamapper/tests/Test_CudamapperOverlapperTriggered.cu @@ -306,7 +306,7 @@ TEST(TestCudamapperOverlapperTriggerred, ReverseStrand) TEST(TestCudamapperOverlapperTriggerred, OverlapPostProcessingTwoForwardOverlapsTwoFusable) { - DefaultDeviceAllocator allocator; + DefaultDeviceAllocator allocator = create_default_device_allocator(); OverlapperTriggered overlapper(allocator); std::vector overlaps; @@ -359,7 +359,7 @@ TEST(TestCudamapperOverlapperTriggerred, OverlapPostProcessingTwoForwardOverlaps TEST(TestCudamapperOverlapperTriggerred, OverlapPostProcessingTwoForwardOverlapsOneFusable) { - DefaultDeviceAllocator allocator; + DefaultDeviceAllocator allocator = create_default_device_allocator(); OverlapperTriggered overlapper(allocator); std::vector overlaps; @@ -412,7 +412,7 @@ TEST(TestCudamapperOverlapperTriggerred, OverlapPostProcessingTwoForwardOverlaps TEST(TestCudamapperOverlapperTriggerred, OverlapPostProcessingOneForwardOneReverseBothFuasble) { - DefaultDeviceAllocator allocator; + DefaultDeviceAllocator allocator = create_default_device_allocator(); OverlapperTriggered overlapper(allocator); std::vector overlaps; From 34c0734c4038febfdd1227c54c401cfbd8f0ce56 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 5 Oct 2020 18:34:21 -0400 Subject: [PATCH 190/281] [cudapoa-benchmark] addressed PR comments --- cudapoa/src/cudapoa_batch.cuh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index a861b6b62..97b602636 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -549,8 +549,10 @@ protected: bool reserve_buf(int32_t max_seq_length) { int32_t matrix_height = batch_size_.max_nodes_per_graph; - int32_t matrix_width = (batch_size_.band_mode != BandMode::full_band) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); - // in full-band, avail_buf_mem_ is dedicated to scores matrix and in static or adaptive band modes, avail_buf_mem_ is dedicated to traceback matrix + // matrix width for full_band is based on the current group max_seq_length as opposed to batch_size_.matrix_sequence_dimension. + // The latter is based on the largest group in the batch and is more conservative + int32_t matrix_width = (batch_size_.band_mode != BandMode::full_band) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); + // in traceback alignments avail_buf_mem_ is dedicated to traceback matrix, otherwise it is being used for score matrix size_t required_size = static_cast(matrix_width) * static_cast(matrix_height); required_size *= batch_size_.band_mode == BandMode::static_band_traceback ? sizeof(TraceT) : sizeof(ScoreT); From 1c180e20503795c395858f210a35c0e79be79c1d Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Tue, 6 Oct 2020 09:45:03 +0200 Subject: [PATCH 191/281] [common] Additional asserts to make sure that associated_streams are not empty --- .../genomeworks/utils/device_preallocated_allocator.cuh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 9517c15b3..a21f3a976 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -194,6 +194,7 @@ private: return cudaErrorMemoryAllocation; } + assert(!associated_streams.empty()); MemoryBlock new_memory_block{block_to_get_memory_from_iter->begin, bytes_needed, associated_streams}; @@ -247,6 +248,7 @@ private: assert(block_to_be_freed_iter != std::end(used_blocks_)); // ** wait for all work on associated_streams to finish before freeing up this memory block + assert(!block_to_be_freed_iter->associated_streams.empty()); for (cudaStream_t associated_stream : block_to_be_freed_iter->associated_streams) { // WARNING: The way and place this synchronization is done might change in the future, do not rely on this cudaStreamSynchronize() in the caller. From 94c065ffb199aedb5bd156b9668561ec94f94f09 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 5 Oct 2020 18:34:21 -0400 Subject: [PATCH 192/281] [cudapoa-benchmark] addressed PR comments --- cudapoa/src/cudapoa_batch.cuh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index aa331b5ff..48c434c64 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -549,8 +549,10 @@ protected: bool reserve_buf(int32_t max_seq_length) { int32_t matrix_height = batch_size_.max_nodes_per_graph; - int32_t matrix_width = (batch_size_.band_mode != BandMode::full_band) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); - // in full-band, avail_buf_mem_ is dedicated to scores matrix and in static or adaptive band modes, avail_buf_mem_ is dedicated to traceback matrix + // matrix width for full_band is based on the current group max_seq_length as opposed to batch_size_.matrix_sequence_dimension. + // The latter is based on the largest group in the batch and is more conservative + int32_t matrix_width = (batch_size_.band_mode != BandMode::full_band) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); + // in traceback alignments avail_buf_mem_ is dedicated to traceback matrix, otherwise it is being used for score matrix size_t required_size = static_cast(matrix_width) * static_cast(matrix_height); required_size *= (batch_size_.band_mode == static_band_traceback || batch_size_.band_mode == adaptive_band_traceback) ? sizeof(TraceT) : sizeof(ScoreT); From f173764f1d9122339f49c0a192a33f4bf8bbd7e5 Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 6 Oct 2020 18:14:45 -0400 Subject: [PATCH 193/281] [cudapoa] updated comments to highlight implications of modifying sequence length alignment size in both host and device code --- cudapoa/src/cudapoa_batch.cuh | 4 +++- cudapoa/src/cudapoa_kernels.cuh | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 48c434c64..0dff2a4b6 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -537,7 +537,9 @@ protected: } input_details_h_->sequence_lengths[global_sequence_idx_] = seq_len; - // to be aligned with uchar4 size, pad sequence length to be multiple of 4 + // to be aligned with SeqT4 struct size, pad sequence length to be multiple of 4. + // Note: num_nucleotides_copied_ is used to define allocated space per read on device as well as on host, + // therefore it is important to reflect any changes in the following line in the corresponding device code as well num_nucleotides_copied_ += cudautils::align(seq_len); global_sequence_idx_++; diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 868abd1ac..4e9f7ca0c 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -225,6 +225,8 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) for (int32_t s = 1; s < num_sequences; s++) { int32_t seq_len = sequence_lengths[s]; + // Note: the following 2 lines correspond to num_nucleotides_copied_ value on the host side + // therefore it is important to reflect any changes in the following 2 lines in the corresponding host code as well sequence += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data base_weights += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data @@ -429,7 +431,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) } // Similar to generatePOAKernel(), but with a different launch bounds setting, used to call kernels using traceback buffer -// i.e. BM == BandMode::static_band_traceback or BM == BandMode::adaptive_band_traceback +// i.e. for BM == BandMode::static_band_traceback or BM == BandMode::adaptive_band_traceback template __launch_bounds__(MAX_THREADS_PER_BLOCK_72) __global__ void generatePOAKernelTB(uint8_t* consensus_d, @@ -580,6 +582,8 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_72) for (int32_t s = 1; s < num_sequences; s++) { int32_t seq_len = sequence_lengths[s]; + // Note: the following 2 lines correspond to num_nucleotides_copied_ value on the host side + // therefore it is important to reflect any changes in the following 2 lines in the corresponding host code as well sequence += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data base_weights += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data From 4d2f3d6eea931e1a2fedded6736ed4630c25b33d Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 6 Oct 2020 19:14:42 -0400 Subject: [PATCH 194/281] [cudapoa] added a new error code for possible scenario where traceback NW can fail. Defined some macros for NW error codes and banded rerun codes. --- .../genomeworks/cudapoa/cudapoa.hpp | 3 +- cudapoa/src/cudapoa_batch.cuh | 4 +++ cudapoa/src/cudapoa_kernels.cuh | 25 ++++++++++----- cudapoa/src/cudapoa_nw.cuh | 2 +- cudapoa/src/cudapoa_nw_banded.cuh | 20 ++++++------ cudapoa/src/cudapoa_nw_tb_banded.cuh | 31 ++++++++++--------- cudapoa/src/cudapoa_structs.cuh | 9 ++++++ 7 files changed, 60 insertions(+), 34 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp index acba1aea0..5df1e9249 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp @@ -41,7 +41,8 @@ enum StatusType seq_len_exceeded_maximum_nodes_per_window, loop_count_exceeded_upper_bound, output_type_unavailable, - generic_error + generic_error, + exceeded_maximum_predecessor_distance }; /// Banding mode used in Needleman-Wunsch algorithm diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 0dff2a4b6..8adca6927 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -465,6 +465,10 @@ protected: GW_LOG_WARN("Kernel Error:: Consensus/MSA sequence size exceeded max sequence size in batch {}\n", bid_); output_status.emplace_back(error_type); break; + case genomeworks::cudapoa::StatusType::exceeded_maximum_predecessor_distance: + GW_LOG_WARN("Kernel Error:: Set value for maximum predecessor distance in traceback NW is too small {}\n", bid_); + output_status.emplace_back(error_type); + break; default: GW_LOG_WARN("Kernel Error:: Unknown error in batch {}\n", bid_); output_status.emplace_back(error_type); diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 4e9f7ca0c..a9c4443dd 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -273,9 +273,9 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) 0); __syncwarp(); - if (alignment_length < -2) + if (alignment_length == SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == SHIFT_ADAPTIVE_BAND_TO_RIGHT) { - // rerun with extended band-width + // rerun with extended and shifted band-width alignment_length = runNeedlemanWunschBanded(nodes, sorted_poa, node_id_to_pos, @@ -342,7 +342,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) __syncwarp(); } - if (alignment_length == -1) + if (alignment_length == NW_BACKTRACKING_LOOP_FAILED) { if (lane_idx == 0) { @@ -351,7 +351,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) } return; } - else if (alignment_length == -2) + else if (alignment_length == NW_ADAPTIVE_STORAGE_FAILED) { if (lane_idx == 0) { @@ -630,9 +630,9 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_72) match_score, 0); __syncwarp(); - if (alignment_length < -2) + if (alignment_length == SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == SHIFT_ADAPTIVE_BAND_TO_RIGHT) { - // rerun with extended band-width + // rerun with extended and shifted band-width alignment_length = runNeedlemanWunschBandedTraceback(nodes, sorted_poa, node_id_to_pos, @@ -682,7 +682,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_72) __syncwarp(); } - if (alignment_length == -1) + if (alignment_length == NW_BACKTRACKING_LOOP_FAILED) { if (lane_idx == 0) { @@ -691,7 +691,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_72) } return; } - else if (alignment_length == -2) + else if (alignment_length == NW_ADAPTIVE_STORAGE_FAILED) { if (lane_idx == 0) { @@ -700,6 +700,15 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_72) } return; } + else if (alignment_length == NW_TRACEBACK_BUFFER_FAILED) + { + if (lane_idx == 0) + { + consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; + consensus[1] = static_cast(StatusType::exceeded_maximum_predecessor_distance); + } + return; + } if (lane_idx == 0) { diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index 4bdd267a2..97cf03215 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -441,7 +441,7 @@ __device__ __forceinline__ if (loop_count >= (read_length + graph_count + 2)) { - aligned_nodes = -1; + aligned_nodes = NW_BACKTRACKING_LOOP_FAILED; } #ifdef NW_VERBOSE_PRINT diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 5f0c186e4..53e3b945c 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -233,17 +233,17 @@ __device__ __forceinline__ if (ADAPTIVE) { // rerun code is defined in backtracking loop from previous alignment try - // -3 means traceback path was too close to the left bound of band - // -4 means traceback path was too close to the right bound of band - // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift - // further to the left for rerun == -3, and further to the right for rerun == -4. - if (rerun == -3) + // SHIFT_ADAPTIVE_BAND_TO_LEFT means traceback path was too close to the left bound of band + // SHIFT_ADAPTIVE_BAND_TO_RIGHT means traceback path was too close to the right bound of band + // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift further to + // the left for rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT, and further to the right for rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT. + if (rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT) { // ad-hoc rule 5 band_width *= 2; band_shift *= 2.5; } - if (rerun == -4) + if (rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT) { // ad-hoc rule 6 band_width *= 2; @@ -254,7 +254,7 @@ __device__ __forceinline__ float required_buffer_size = static_cast(graph_count) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); if (required_buffer_size > max_buffer_size) { - return -2; + return NW_ADAPTIVE_STORAGE_FAILED; } } //--------------------------------------------------------- @@ -445,12 +445,12 @@ __device__ __forceinline__ int32_t band_start = get_band_start_for_row(i, gradient, band_width, band_shift, max_column); if (j <= band_start + threshold) // ad-hoc rule 8-a, too close to left bound { - aligned_nodes = -3; + aligned_nodes = SHIFT_ADAPTIVE_BAND_TO_LEFT; break; } if (j >= (band_start + band_width - threshold)) // ad-hoc rule 8-b, too close to right bound { - aligned_nodes = -4; + aligned_nodes = SHIFT_ADAPTIVE_BAND_TO_RIGHT; break; } } @@ -538,7 +538,7 @@ __device__ __forceinline__ if (loop_count >= (read_length + graph_count + 2)) { - aligned_nodes = -1; + aligned_nodes = NW_BACKTRACKING_LOOP_FAILED; } #ifdef NW_VERBOSE_PRINT diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index 258dc9803..e78440586 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -323,17 +323,17 @@ __device__ __forceinline__ if (ADAPTIVE) { // rerun code is defined in backtracking loop from previous alignment try - // -3 means traceback path was too close to the left bound of band - // -4 means traceback path was too close to the right bound of band - // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift - // further to the left for rerun == -3, and further to the right for rerun == -4. - if (rerun == -3) + // SHIFT_ADAPTIVE_BAND_TO_LEFT means traceback path was too close to the left bound of band + // SHIFT_ADAPTIVE_BAND_TO_RIGHT means traceback path was too close to the right bound of band + // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift further to + // the left for rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT, and further to the right for rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT. + if (rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT) { // ad-hoc rule 5 band_width *= 2; band_shift *= 2.5; } - if (rerun == -4) + if (rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT) { // ad-hoc rule 6 band_width *= 2; @@ -344,7 +344,7 @@ __device__ __forceinline__ float required_buffer_size = static_cast(graph_count) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); if (required_buffer_size > max_buffer_size) { - return -2; + return NW_ADAPTIVE_STORAGE_FAILED; } } //--------------------------------------------------------- @@ -574,13 +574,16 @@ __device__ __forceinline__ i = idx; } } - else - { - ///ToDo throw an error indicating selected score_matrix_height (i.e. max predecessor distance) is too small - } } } + // if i was not set, throw an error indicating selected score_matrix_height (i.e. max predecessor distance) is too small + if (i == 0) + { + j = 0; + aligned_nodes = NW_TRACEBACK_BUFFER_FAILED; + } + //------------------------------------------------------------------------ // Fill in traceback @@ -626,12 +629,12 @@ __device__ __forceinline__ int32_t band_start = get_band_start_for_row(i, gradient, band_width, band_shift, max_column); if (j <= band_start + threshold) // ad-hoc rule 8-a, too close to left bound { - aligned_nodes = -3; + aligned_nodes = SHIFT_ADAPTIVE_BAND_TO_LEFT; break; } if (j >= (band_start + band_width - threshold)) // ad-hoc rule 8-b, too close to right bound { - aligned_nodes = -4; + aligned_nodes = SHIFT_ADAPTIVE_BAND_TO_RIGHT; break; } } @@ -644,7 +647,7 @@ __device__ __forceinline__ if (loop_count >= (read_length + graph_count + 2)) { - aligned_nodes = -1; + aligned_nodes = NW_BACKTRACKING_LOOP_FAILED; } #ifdef NW_VERBOSE_PRINT diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index 7076d48b5..c90e3aa07 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -37,6 +37,10 @@ // ad-hoc maximum band-width defined in adaptive banding #define CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH 1536 +// rerun codes used in adaptive banding +#define SHIFT_ADAPTIVE_BAND_TO_LEFT -10 +#define SHIFT_ADAPTIVE_BAND_TO_RIGHT -11 + #define CUDAPOA_THREADS_PER_BLOCK 64 #define CUDAPOA_BANDED_THREADS_PER_BLOCK WARP_SIZE #define CUDAPOA_MAX_CONSENSUS_PER_BLOCK 512 @@ -45,6 +49,11 @@ #define CUDAPOA_KERNEL_ERROR_ENCOUNTERED UINT8_MAX #define CUDAPOA_KERNEL_NOERROR_ENCOUNTERED 0 +// NW return error codes +#define NW_BACKTRACKING_LOOP_FAILED -1 +#define NW_ADAPTIVE_STORAGE_FAILED -2 +#define NW_TRACEBACK_BUFFER_FAILED -3 + namespace claraparabricks { From 58693fef38840d03c78f7d180ab5d09c47bc53cc Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 6 Oct 2020 19:16:46 -0400 Subject: [PATCH 195/281] [cudapoa] updated help message in cudapoa --- cudapoa/src/application_parameters.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudapoa/src/application_parameters.cpp b/cudapoa/src/application_parameters.cpp index 4942aae90..a1216a711 100644 --- a/cudapoa/src/application_parameters.cpp +++ b/cudapoa/src/application_parameters.cpp @@ -196,7 +196,7 @@ void ApplicationParameters::help(int32_t exit_code) generates msa if this flag is passed [default: consensus])" << R"( -b, --band-mode - selects banding mode, 0: full-alignment, 1: static band, 2: adaptive band [2])" + selects banding mode, 0: full-alignment, 1: static band, 2: adaptive band, 3: traceback static band, 4: traceback adaptive band [2])" << R"( -w, --band-width band-width for banded alignment (must be multiple of 128) [256])" From 833b19fccbf56f5563b9882833dc4b9ea6e9874f Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Thu, 8 Oct 2020 19:25:16 -0400 Subject: [PATCH 196/281] [cmake] push fixed for cmake 1. fix usage of CMAKE_SOURCE_DIR with PROJECT_SOURCE_DIR 2. make tests and benchmarks build by default --- CMakeLists.txt | 4 ++-- cudaextender/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bf4006e2c..59cafc621 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,8 +23,8 @@ string(STRIP ${GW_VERSION} GW_VERSION) project(${GW_PROJECT_NAME}) # Process options. -option(gw_enable_tests "Build GenomeWorks unit tests" OFF) -option(gw_enable_benchmarks "Build GenomeWorks benchmarks" OFF) +option(gw_enable_tests "Build GenomeWorks unit tests" ON) +option(gw_enable_benchmarks "Build GenomeWorks benchmarks" ON) option(gw_build_shared "Build GenomeWorks libraries as shared objects" OFF) option(gw_device_synchronize_kernels "Run cudaDeviceSynchronize() in GW_CU_CHECK_ERR calls" OFF) option(gw_optimize_for_native_cpu "Build with march=native" ON) diff --git a/cudaextender/CMakeLists.txt b/cudaextender/CMakeLists.txt index 82d282374..adf51a0ff 100644 --- a/cudaextender/CMakeLists.txt +++ b/cudaextender/CMakeLists.txt @@ -20,7 +20,7 @@ add_subdirectory(data) GitVersion() -configure_file(${CMAKE_SOURCE_DIR}/common/base/src/version.cpp.in +configure_file(${PROJECT_SOURCE_DIR}/common/base/src/version.cpp.in ${CMAKE_CURRENT_BINARY_DIR}/version.cpp) # Project specific NVCC flags, --expt-relaxed-constexpr is being added to allow using numeric_limits inside device kernels. From 01099fcb0e0c9f9b1d53823049a0b9029cbfc10a Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 9 Oct 2020 18:49:52 -0400 Subject: [PATCH 197/281] [cudapoa] modified adaptive banding to avoid rerun if it has already executed NW with CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH; may need to tune CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH value --- cudapoa/src/cudapoa_nw_banded.cuh | 10 ++++++++-- cudapoa/src/cudapoa_nw_tb_banded.cuh | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 53e3b945c..2a61ce2cf 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -225,6 +225,12 @@ __device__ __forceinline__ // limit band-width for very large reads, ad-hoc rule 3 band_width = min(band_width, CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH); + + if (band_width == CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH && rerun != 0) + { + // already we have tried with maximum allowed band-width, rerun won't help + return rerun; + } } // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 @@ -237,13 +243,13 @@ __device__ __forceinline__ // SHIFT_ADAPTIVE_BAND_TO_RIGHT means traceback path was too close to the right bound of band // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift further to // the left for rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT, and further to the right for rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT. - if (rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT) + if (rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) { // ad-hoc rule 5 band_width *= 2; band_shift *= 2.5; } - if (rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT) + if (rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) { // ad-hoc rule 6 band_width *= 2; diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index e78440586..e0a4a4461 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -315,6 +315,12 @@ __device__ __forceinline__ // limit band-width for very large reads, ad-hoc rule 3 band_width = min(band_width, CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH); + + if (band_width == CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH && rerun != 0) + { + // already we have tried with maximum allowed band-width, rerun won't help + return rerun; + } } // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 @@ -327,13 +333,13 @@ __device__ __forceinline__ // SHIFT_ADAPTIVE_BAND_TO_RIGHT means traceback path was too close to the right bound of band // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift further to // the left for rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT, and further to the right for rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT. - if (rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT) + if (rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) { // ad-hoc rule 5 band_width *= 2; band_shift *= 2.5; } - if (rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT) + if (rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) { // ad-hoc rule 6 band_width *= 2; From 6cc4316f12496f82b75d87f04d8ba134b3ebd45c Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 12 Oct 2020 09:59:46 -0400 Subject: [PATCH 198/281] [cudapoa] modified the conditions to set rerun flag in adaptive, it won't request rerun if it already runs with max bw --- cudapoa/src/cudapoa_nw_banded.cuh | 3 ++- cudapoa/src/cudapoa_nw_tb_banded.cuh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 2a61ce2cf..51342edcf 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -441,7 +441,8 @@ __device__ __forceinline__ { if (ADAPTIVE) { - if (rerun == 0) + // no need to request rerun if (a) it's not the first run, (b) band_width == CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH already + if (rerun == 0 && band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) { // check if traceback gets too close or hits the band limits, if so stop and rerun with extended band-width // threshold for proximity to band limits works better if defined proportionate to the sequence length diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index e0a4a4461..a94a9be1a 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -625,7 +625,8 @@ __device__ __forceinline__ if (ADAPTIVE) { - if (rerun == 0) + // no need to request rerun if (a) it's not the first run, (b) band_width == CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH already + if (rerun == 0 && band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) { // check if traceback gets too close or hits the band limits, if so stop and rerun with extended band-width // threshold for proximity to band limits works better if defined proportionate to the sequence length From 09138dd7fe2ad7041d995eaffe05191d323c74d6 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 12 Oct 2020 11:00:47 -0400 Subject: [PATCH 199/281] [cudapoa] in adaptive traceback, removed adjusting bw based on gradient, this could cause bw get bigger than x2, causing memory allocationed for score buffer --- cudapoa/src/cudapoa_nw_tb_banded.cuh | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index a94a9be1a..956eb8cae 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -300,29 +300,6 @@ __device__ __forceinline__ // Set band-width based on scores matrix aspect ratio //--------------------------------------------------------- - if (ADAPTIVE) - { - if (gradient > 1.1) // ad-hoc rule 1.a - { - // ad-hoc rule 1.b - band_width = max(band_width, cudautils::align(max_column * 0.08 * gradient)); - } - if (gradient < 0.8) // ad-hoc rule 2.a - { - // ad-hoc rule 2.b - band_width = max(band_width, cudautils::align(max_column * 0.1 / gradient)); - } - - // limit band-width for very large reads, ad-hoc rule 3 - band_width = min(band_width, CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH); - - if (band_width == CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH && rerun != 0) - { - // already we have tried with maximum allowed band-width, rerun won't help - return rerun; - } - } - // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 int32_t band_shift = band_width / 2; From cf6326c230e9d3bc33a8236f84b05fb2bdfade29 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Fri, 16 Oct 2020 01:27:10 -0400 Subject: [PATCH 200/281] [pygw] add alignment visualizer tool --- pygenomeworks/bin/alignment_visualizer | 116 +++++++++++++++++++++++++ pygenomeworks/bin/evaluate_paf | 55 ++++++++---- pygenomeworks/requirements.txt | 1 + pygenomeworks/style_check | 1 + 4 files changed, 156 insertions(+), 17 deletions(-) create mode 100755 pygenomeworks/bin/alignment_visualizer diff --git a/pygenomeworks/bin/alignment_visualizer b/pygenomeworks/bin/alignment_visualizer new file mode 100755 index 000000000..e3b99e408 --- /dev/null +++ b/pygenomeworks/bin/alignment_visualizer @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Tool to visualize pairwise alignments.""" + +import argparse +import numpy as np +import matplotlib.pyplot as plt +import csv +import math + + +def expand_cigar(cigar): + """ + Expand CIGAR string into array of CIGAR operations. + """ + start_of_num = 0 + str_idx = 0 + expanded_cigar = [] + for c in cigar: + if c.isdigit(): + str_idx += 1 + else: + temp_arr = [c] * int(cigar[start_of_num: str_idx]) + expanded_cigar.extend(temp_arr) + str_idx += 1 + start_of_num = str_idx + return expanded_cigar + + +def fill_alignment_matrix_with_cigar(alignment_matrix, cigar, val): + """ + Based on CIGAR string, fill backtrce matrix with alignment + path, storing value val at each cell. + """ + # Starting the alignment at top left of matrix + i = 0 + j = 0 + alignment_matrix[i, j] = val + + cigar = expand_cigar(cigar) + for op in cigar: + if op == 'M': + i += 1 + j += 1 + alignment_matrix[i, j] = val + elif op == 'I': + i += 0 + j += 1 + alignment_matrix[i, j] = val + elif op == 'D': + i += 1 + j += 0 + alignment_matrix[i, j] = val + else: + raise RuntimeError(f"Unknown operation type - {op}") + + +def viz_alignments(axes, target, query, cigars): + """ + Visualize alignment(s) between target and query on the + subplot defined by axes. Supports rendering multiple + alignments per pair. + """ + target = list(target) + query = list(query) + + alignment_matrix = np.zeros((len(query) + 1, len(target) + 1)) + for i, cigar in enumerate(cigars): + fill_alignment_matrix_with_cigar(alignment_matrix, cigar, i+1) + + cax = axes.matshow(alignment_matrix) + figure.colorbar(cax) + + target = [''] * 2 + target + query = [''] * 2 + query + axes.set_xticklabels(target) + axes.set_yticklabels(query) + + +def parse_args(): + parser = argparse.ArgumentParser(description="A tool to visualize alignment for pairwise alignments.") + parser.add_argument("alignment_file", + help="A CSV file of the format target,query,cigar1,cigar2,...", + type=str) + args = parser.parse_args() + return args + + +if __name__ == "__main__": + targets = [] + queries = [] + cigars = [] + + # Parse arguments + args = parse_args() + + # Parse in all alignments to be visualized. + with open(args.alignment_file, newline='') as fh: + reader = csv.reader(fh) + for row in reader: + targets.append(row[0]) + queries.append(row[1]) + cigars.append(row[2:]) + + # Generate figure dimensions. + plot_width = int(math.sqrt(len(targets))) + 1 + plot_height = (len(targets) // plot_width) + 1 + + # Iterate through every pair of alignment and visualize it. + figure = plt.figure() + for i in range(len(targets)): + subplot_idx = i + 1 + subplot = figure.add_subplot(plot_width, plot_height, subplot_idx) + viz_alignments(subplot, targets[i], queries[i], cigars[i]) + + # Generate final plot. + plt.show() diff --git a/pygenomeworks/bin/evaluate_paf b/pygenomeworks/bin/evaluate_paf index f248311c6..52a5be034 100755 --- a/pygenomeworks/bin/evaluate_paf +++ b/pygenomeworks/bin/evaluate_paf @@ -30,9 +30,11 @@ from genomeworks.io import pafio def points_equal(fixed, point, slop): return max(0, int(fixed) - slop) <= int(point) <= (int(fixed) + slop) + def within(val, target, tolerance=0.05): return abs(float(val) - float(target)) <= tolerance + def calculate_reciprocal_overlap(record, other): q_overlap = min(record.query_end, other.query_end) - max(record.query_start, other.query_start) q_total_len = max(record.query_start, other.query_start) - min(record.query_start, other.query_start) @@ -46,6 +48,7 @@ def calculate_reciprocal_overlap(record, other): def _gen_interval(start, end, value, tolerance): return intervaltree.Interval(max(0, int(start) - tolerance), end + tolerance, value) + def construct_interval_dictionaries(paf_record_list, tolerance): """ Constructs a dictionary[string->IntervalTree], where the keys of the dictionary are @@ -64,6 +67,7 @@ def construct_interval_dictionaries(paf_record_list, tolerance): return query_paf_dict, target_paf_dict + def records_equal(record, other, pos_tolerance): query_start_valid = points_equal(record.query_start, other.query_start, pos_tolerance) query_end_valid = points_equal(record.query_end, other.query_end, pos_tolerance) @@ -76,6 +80,7 @@ def records_equal(record, other, pos_tolerance): return equal, query_start_valid, query_end_valid, target_start_valid, target_end_valid, strands_equal + def match_overlaps(record, other, pos_tolerance, min_reciprocal_overlap): """Given two sets of query and target ranges, check if the query and target ranges fall within a specified tolerance of each other. @@ -89,9 +94,10 @@ def match_overlaps(record, other, pos_tolerance, min_reciprocal_overlap): Returns: Boolean indicating query and target match. """ - equal, query_start_valid, query_end_valid, target_start_valid, target_end_valid, strands_equal = records_equal(record, other, pos_tolerance) + equal, query_start_valid, query_end_valid, target_start_valid, target_end_valid, \ + strands_equal = records_equal(record, other, pos_tolerance) pct_recip = calculate_reciprocal_overlap(record, other) - reciprocal = pct_recip > min_reciprocal_overlap + reciprocal = pct_recip > min_reciprocal_overlap match = equal or reciprocal @@ -101,8 +107,8 @@ def match_overlaps(record, other, pos_tolerance, min_reciprocal_overlap): "target_end_valid": target_end_valid, "reciprocal_overlaps": reciprocal, "percent_reciprocal": pct_recip, - "strands_equal" : strands_equal, - "equal" : equal, + "strands_equal": strands_equal, + "equal": equal, "match": match} return match @@ -118,16 +124,31 @@ def generate_key(name_1, name_2): """ return "{}_{}".format(name_1, name_2) + def _swap(val, dest): return dest, val + def reverse_record(record): query_sequence_name, target_sequence_name = _swap(record.query_sequence_name, record.target_sequence_name) query_sequence_length, target_sequence_length = _swap(record.query_sequence_length, record.target_sequence_length) query_start, target_start = _swap(record.query_start, record.target_start) query_end, target_end = _swap(record.query_end, record.target_end) - return pafio.Overlap(query_sequence_name, query_sequence_length, query_start, query_end, record.relative_strand, target_sequence_name, target_sequence_length, target_start, target_end, record.num_residue_matches, record.alignment_block_length, record.mapping_quality, record.tags) + return pafio.Overlap(query_sequence_name, + query_sequence_length, + query_start, + query_end, + record.relative_strand, + target_sequence_name, + target_sequence_length, + target_start, + target_end, + record.num_residue_matches, + record.alignment_block_length, + record.mapping_quality, + record.tags) + def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_reciprocal, skip_self_mappings=True): """Given a truth and test set PAF file, count number of in/incorrectly detected, and non-detected overlaps @@ -164,8 +185,6 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip test_overlap_count = 0 print("{} true overlaps in truth set".format(num_true_overlaps)) - seen_test_overlap_keys = set() - incorrect_query_start = 0 incorrect_query_end = 0 incorrect_target_start = 0 @@ -182,7 +201,7 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip best_pct_match = 0.0 best_ends = [1, 1, 1, 1] - + found_match = False if key in truth_keys: for truth_interval in truth_query_intervals[test_overlap.query_sequence_name]: @@ -251,7 +270,7 @@ def evaluate_paf(truth_paf_filepath, test_paf_filepath, pos_tolerance, min_recip incorrect_query_end += best_ends[1] incorrect_target_start += best_ends[2] incorrect_target_end += best_ends[3] - + if not found_match: false_positive_count += 1 @@ -281,21 +300,23 @@ if __name__ == "__main__": parser.add_argument("-s", '--slop', dest="pos_tolerance", type=int, default=200, - help="Number of basepairs to tolerate on either side of an interval (or record) to consider the two records equal [200].") + help="Number of basepairs to tolerate on either side of an interval (or record) \ + to consider the two records equal [200].") parser.add_argument("-r", "--reciprocal-cutoff", dest="min_reciprocal", type=float, - required=False, default=0.9, help="Amount of reciprocal overlap required to consider two overlaps the same [0.9]") + required=False, default=0.9, help="Amount of reciprocal overlap required \ + to consider two overlaps the same [0.9]") parser.add_argument('--skip_self_mapping', action="store_true", help="Skip checking overlaps where query/target name are same") args = parser.parse_args() - true_positives, false_positives, false_negatives, incorrect_query_start, incorrect_query_end, incorrect_target_start, \ - incorrect_target_end, total_test_records = evaluate_paf(args.truth, - args.test, - args.pos_tolerance, - args.min_reciprocal, - args.skip_self_mapping) + true_positives, false_positives, false_negatives, incorrect_query_start, incorrect_query_end, \ + incorrect_target_start, incorrect_target_end, total_test_records = evaluate_paf(args.truth, + args.test, + args.pos_tolerance, + args.min_reciprocal, + args.skip_self_mapping) print("Total test records:", total_test_records) print("True positives: ", true_positives) print("False positives: ", false_positives) diff --git a/pygenomeworks/requirements.txt b/pygenomeworks/requirements.txt index 61e0674b5..2413134dd 100644 --- a/pygenomeworks/requirements.txt +++ b/pygenomeworks/requirements.txt @@ -18,6 +18,7 @@ Cython==0.29.12 intervaltree==3.1.0 +matplotlib==3.3.2 networkx==2.4 numpy==1.16.3 pytest==4.4.1 diff --git a/pygenomeworks/style_check b/pygenomeworks/style_check index 5ef962dd5..a4bc28e33 100755 --- a/pygenomeworks/style_check +++ b/pygenomeworks/style_check @@ -23,6 +23,7 @@ set -e PYGW_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") echo "Run Python formatting check..." flake8 "$PYGW_DIR" +flake8 "$PYGW_DIR" $PYGW_DIR/bin/* echo "Run Cython formatting check..." flake8 "$PYGW_DIR" --config "$PYGW_DIR"/.flake8.cython echo "Run Python docstring conventions check..." From 3813e35a74502aaf1be4b88d3f8b083653ba0355 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 16 Oct 2020 12:45:57 -0700 Subject: [PATCH 201/281] [cudaextender] Add and update READMEs --- README.md | 5 ++++ cudaextender/README.md | 27 +++++++++++++++++++ .../genomeworks/cudaextender/utils.hpp | 1 + 3 files changed, 33 insertions(+) create mode 100644 cudaextender/README.md diff --git a/README.md b/README.md index fd7f1f509..06fcf89a5 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ For more detailed API documentation please refer to the [documentation](#enable- * [cudamapper](#cudamapper) - CUDA-accelerated sequence to sequence mapping * [cudapoa](#cudapoa) - CUDA-accelerated partial order alignment * [cudaaligner](#cudaaligner) - CUDA-accelerated pairwise sequence alignment + * [cudaextender](#cudaextender) - CUDA-accelerated seed extension * Setup GenomeWorks * [Clone GenomeWorks](#clone-genomeworks) * [System Requirements](#system-requirements) @@ -81,6 +82,10 @@ The `cudaaligner` package provides GPU-accelerated global alignment. Features in A prototypical binary to showcase the use of `libcudaaligner.so` APIs. +### cudaextender +The `cudaextender` package provides GPU-accelerated seed-extension. Details can be found in +the package's [Readme](cudaextender/README.md). + ## Clone GenomeWorks ### Latest released version diff --git a/cudaextender/README.md b/cudaextender/README.md new file mode 100644 index 000000000..078eb5376 --- /dev/null +++ b/cudaextender/README.md @@ -0,0 +1,27 @@ +# cudaextender + +## Overview +This package implements CUDA-accelerated seed-extension algorithms that use seed positions in +encoded input strands to extend and compute the alignment between the strands. +Currently this module implements the ungapped X-drop algorithm, adapted from +[SegAlign's](https://github.com/gsneha26/SegAlign) Ungapped Extender authored by +Sneha Goenka (gsneha@stanford.edu) and Yatish Turakhia (yturakhi@uscs.edu). + +### Encoded Input +`cudaextender` expects the input strands to be encoded as integer sequences. +This encoding scheme is documented here: [utils.hpp](include/claraparabricks/genomeworks/cudaextender/utils.hpp) +file. The provided `encode_sequence()` helper function will encode the input strands on CPU with +the expected scheme. + +### API +`cudaextender` provides host and device pointer APIs to enable ease of integration with other +producer/consumer modules. The user is expected to handle all memory transactions and device +sychronizations for the device pointer API. The host pointer API abstracts those operations away. +Both APIs are documented here: [extender.hpp](include/claraparabricks/genomeworks/cudaextender/extender.hpp) + +### Library - *libcudaextender.so* +Features: +* Ungapped X-Drop extension + +### Sample - *[sample_cudaextender.cpp](samples/sample_cudaextender.cpp)* +Protoype to show the usage of host and device pointer APIs on FASTA sequences. diff --git a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp index e8d561a1d..f99f14483 100644 --- a/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp +++ b/cudaextender/include/claraparabricks/genomeworks/cudaextender/utils.hpp @@ -32,6 +32,7 @@ namespace genomeworks namespace cudaextender { +/// Sequence encoding scheme: constexpr int8_t A_NT = 0; constexpr int8_t C_NT = 1; constexpr int8_t G_NT = 2; From 4a8eca9ad219505ed01b2d18e31ce39ddc0691fd Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Fri, 16 Oct 2020 13:57:18 -0700 Subject: [PATCH 202/281] Output Overlaps in SAM/BAM format --- .gitmodules | 3 + 3rdparty/htslib | 1 + CMakeLists.txt | 2 + README.md | 6 + ci/common/build-test-sdk.sh | 1 + cmake/3rdparty.cmake | 7 + cmake/BuildHTSLib.cmake | 42 ++++ cudamapper/CMakeLists.txt | 10 +- .../genomeworks/cudamapper/utils.hpp | 32 +++ cudamapper/samples/sample_cudamapper.cpp | 44 +++- cudamapper/src/application_parameters.cpp | 22 +- cudamapper/src/application_parameters.hpp | 6 +- cudamapper/src/main.cu | 33 ++- cudamapper/src/utils.cpp | 197 ++++++++++++++++++ 14 files changed, 387 insertions(+), 19 deletions(-) create mode 160000 3rdparty/htslib create mode 100644 cmake/BuildHTSLib.cmake diff --git a/.gitmodules b/.gitmodules index 974a62fae..e62836a80 100644 --- a/.gitmodules +++ b/.gitmodules @@ -33,3 +33,6 @@ [submodule "3rdparty/kseqpp"] path = 3rdparty/kseqpp url = https://github.com/cartoonist/kseqpp.git +[submodule "3rdparty/htslib"] + path = 3rdparty/htslib + url = https://github.com/samtools/htslib.git diff --git a/3rdparty/htslib b/3rdparty/htslib new file mode 160000 index 000000000..a79009b38 --- /dev/null +++ b/3rdparty/htslib @@ -0,0 +1 @@ +Subproject commit a79009b38ce83e39bcbc8f54c00cf203621aa5bb diff --git a/CMakeLists.txt b/CMakeLists.txt index 59cafc621..0438319ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,8 @@ option(gw_profiling "Compile a binary for profiling with NVTX markers." OFF) option(gw_enable_caching_allocator "Enable caching allocator." ON) option(gw_generate_docs "Generate Doxygen documentation" ON) option(gw_cuda_gen_all_arch "ON: Generate optimized CUDA code for all architectures | OFF: for detected architectures only" OFF) +# Optionally build htslib for SAM/BAM support. Requires autoconf to be installed +option(gw_build_htslib "Build 3rdparty htslib that allows output in SAM/BAM format" OFF) # Must be included before others for options value validation include(cmake/Utils.cmake) diff --git a/README.md b/README.md index fd7f1f509..ceb337506 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,10 @@ Minimum requirements - 4. Python 3.6.7+ 5. CMake (>= 3.10.2) +Optional requirements - + +1. autoconf (required to output SAM/BAM files) + ## GenomeWorks Setup ### Build and Install @@ -124,6 +128,8 @@ NOTE : The `gw_cuda_gen_all_arch=OFF` option pre-generates optimized code only f For building a binary that pre-generates opimized code for all common GPU architectures, please remove the option or set it to `ON`. +NOTE : (OPTIONAL) To enable outputting overlaps in SAM/BAM format, pass the `gw_build_htslib=ON` option. + ### Package generation Package generation puts the libraries, headers and binaries built by the `make` command above into a `.deb`/`.rpm` for portability and easy installation. The package generation itself doesn't diff --git a/ci/common/build-test-sdk.sh b/ci/common/build-test-sdk.sh index e1cbb3edd..6574882a6 100644 --- a/ci/common/build-test-sdk.sh +++ b/ci/common/build-test-sdk.sh @@ -32,6 +32,7 @@ cmake .. "${CMAKE_COMMON_VARIABLES[@]}" \ -Dgw_enable_benchmarks=ON \ -Dgw_build_shared=ON \ -Dgw_cuda_gen_all_arch=ON \ + -Dgw_build_htslib=ON \ -DCMAKE_INSTALL_PREFIX="${LOCAL_BUILD_DIR}/install" \ -GNinja diff --git a/cmake/3rdparty.cmake b/cmake/3rdparty.cmake index fbe90b1e2..f26ce2af5 100644 --- a/cmake/3rdparty.cmake +++ b/cmake/3rdparty.cmake @@ -50,3 +50,10 @@ set_property(TARGET cub APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${CUB_DIR set(KSEQPP_DIR ${PROJECT_SOURCE_DIR}/3rdparty/kseqpp/src CACHE STRING "Path to kseqpp repo") + +if (gw_build_htslib) + include(cmake/BuildHTSLib.cmake) + build_htslib_source() +else() + message(STATUS "Not building htslib, overlap output to SAM & BAM unavailable. Enable with -Dgw_build_htslib=ON") +endif() diff --git a/cmake/BuildHTSLib.cmake b/cmake/BuildHTSLib.cmake new file mode 100644 index 000000000..e8dc02201 --- /dev/null +++ b/cmake/BuildHTSLib.cmake @@ -0,0 +1,42 @@ +# +# Copyright 2019-2020 NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function(build_htslib_source) + message(STATUS "Building htslib") + set(HTSLIB_DIR ${PROJECT_SOURCE_DIR}/3rdparty/htslib/ CACHE STRING + "Path to htslib repo") + set(MAKE_COMMAND make) + set(HTSLIB_INSTALL ${MAKE_COMMAND} install prefix=${CMAKE_BINARY_DIR}/3rdparty/htslib) + set(htslib_PREFIX ${CMAKE_BINARY_DIR}/3rdparty/htslib) + include(ExternalProject) + ExternalProject_Add(htslib_project + PREFIX ${htslib_PREFIX} + SOURCE_DIR ${PROJECT_SOURCE_DIR}/3rdparty/htslib + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND autoheader && autoconf && ${HTSLIB_DIR}configure --disable-bz2 --disable-lzma --disable-libcurl --disable-s3 --disable-gcs + BUILD_COMMAND "${HTSLIB_INSTALL}" + INSTALL_COMMAND "" + BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/3rdparty/htslib/lib/libhts.a + LOG_CONFIGURE 0 + LOG_BUILD 0 + LOG_TEST 0 + LOG_INSTALL 0 + ) + + include_directories(${CMAKE_BINARY_DIR}/3rdparty/htslib/include/htslib) + add_library(htslib STATIC IMPORTED) + set_property(TARGET htslib APPEND PROPERTY IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/3rdparty/htslib/lib/libhts.a) +endfunction() diff --git a/cudamapper/CMakeLists.txt b/cudamapper/CMakeLists.txt index d50bd6bf1..5b6d3be99 100644 --- a/cudamapper/CMakeLists.txt +++ b/cudamapper/CMakeLists.txt @@ -67,7 +67,15 @@ target_include_directories(${MODULE_NAME} $ $ ) -target_link_libraries(${MODULE_NAME} gwbase gwio cub) + +# optionally link against htslib +if (TARGET htslib_project) + add_compile_definitions(GW_BUILD_HTSLIB) + add_dependencies(${MODULE_NAME} htslib_project) + target_link_libraries(${MODULE_NAME} htslib gwbase gwio cub) +else() + target_link_libraries(${MODULE_NAME} gwbase gwio cub) +endif() target_compile_options(${MODULE_NAME} PRIVATE -Werror) add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/utils.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/utils.hpp index bc6a7bd61..227bf097c 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/utils.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/utils.hpp @@ -20,6 +20,10 @@ #include #include +#ifdef GW_BUILD_HTSLIB +#include "sam.h" +#endif + #include #include @@ -37,6 +41,14 @@ class FastaParser; namespace cudamapper { +// enum to determine output format +enum class OutputFormat +{ + PAF, + SAM, + BAM +}; + /// \brief prints overlaps to stdout in PAF format /// \param overlaps vector of overlap objects /// \param cigars CIGAR strings. Empty vector if none exist @@ -51,6 +63,25 @@ void print_paf(const std::vector& overlaps, int32_t kmer_size, std::mutex& write_output_mutex); +#ifdef GW_BUILD_HTSLIB +/// \brief prints overlaps to stdout in BAM format +/// \param overlaps vector of overlap objects +/// \param cigars CIGAR strings. Empty vector if none exist +/// \param query_parser needed for read names and lengths +/// \param target_parser needed for read names and lengths +/// \param format print in either BAM or SAM +/// \param write_output_mutex mutex that enables exclusive access to output stream +/// \param argc (optional) number of command line arguments used to generated the @PG CL sections +/// \param argv (optional) command line arguments used to generated the @PG CL sections +void print_sam(const std::vector& overlaps, + const std::vector& cigars, + const io::FastaParser& query_parser, + const io::FastaParser& target_parser, + OutputFormat format, + std::mutex& write_output_mutex, + int argc = -1, + char* argv[] = nullptr); +#endif /// \brief returns a vector of IndexDescriptors in which the sum of basepairs of all reads in one IndexDescriptor is at most max_basepairs_per_index /// If a single read exceeds max_chunk_size it will be placed in its own IndexDescriptor. /// @@ -59,6 +90,7 @@ void print_paf(const std::vector& overlaps, /// \return vector of IndexDescriptors std::vector group_reads_into_indices(const io::FastaParser& parser, number_of_basepairs_t max_basepairs_per_index = 1000000); + } // namespace cudamapper } // namespace genomeworks diff --git a/cudamapper/samples/sample_cudamapper.cpp b/cudamapper/samples/sample_cudamapper.cpp index 61492852c..51add049f 100644 --- a/cudamapper/samples/sample_cudamapper.cpp +++ b/cudamapper/samples/sample_cudamapper.cpp @@ -73,7 +73,8 @@ void process_batch(std::vector& query_index_descriptors, const std::shared_ptr query_parser, const std::shared_ptr target_parser, DefaultDeviceAllocator allocator, - bool print) + const bool print, + const OutputFormat format) { // extra variables used in print_paf. Note "cigars" are typically found during alignment. const std::vector cigars(0); @@ -126,7 +127,19 @@ void process_batch(std::vector& query_index_descriptors, // print overlaps if (print) { - print_paf(overlaps, cigars, *query_parser, *target_parser, KMER_SIZE, print_mutex); +#ifdef GW_BUILD_HTSLIB + if (format == OutputFormat::PAF) + { +#endif + print_paf(overlaps, cigars, *query_parser, *target_parser, KMER_SIZE, print_mutex); +#ifdef GW_BUILD_HTSLIB + } + // SAM or BAM, depends on type of format + else + { + print_sam(overlaps, cigars, *query_parser, *target_parser, format, print_mutex); + } +#endif } } } @@ -138,11 +151,12 @@ void process_batch(std::vector& query_index_descriptors, int main(int argc, char** argv) { // parse command line options - int c = 0; - bool help = false; - bool print = false; + int c = 0; + bool help = false; + bool print = false; + OutputFormat format = OutputFormat::PAF; - while ((c = getopt(argc, argv, "hp")) != -1) + while ((c = getopt(argc, argv, "hpSB")) != -1) { switch (c) { @@ -152,6 +166,18 @@ int main(int argc, char** argv) case 'h': help = true; break; + case 'S': +#ifndef GW_BUILD_HTSLIB + throw std::runtime_error("ERROR: Argument -S cannot be used without htslib"); +#endif + format = OutputFormat::SAM; + break; + case 'B': +#ifndef GW_BUILD_HTSLIB + throw std::runtime_error("ERROR: Argument -B cannot be used without htslib"); +#endif + format = OutputFormat::BAM; + break; } } @@ -160,9 +186,11 @@ int main(int argc, char** argv) { std::cout << "CUDA Mapper API sample program. Runs minimizer-based approximate mapping" << std::endl; std::cout << "Usage:" << std::endl; - std::cout << "./sample_cudamapper [-p] [-h]" << std::endl; + std::cout << "./sample_cudamapper [-p] [-h] [-S] [-B]" << std::endl; std::cout << "-p : Print the overlaps to stdout" << std::endl; std::cout << "-h : Print help message" << std::endl; + std::cout << "-S : Print in SAM format" << std::endl; + std::cout << "-B : Print in BAM format" << std::endl; std::exit(0); } @@ -183,7 +211,7 @@ int main(int argc, char** argv) std::vector query_index_descriptors = initialize_batch(query_parser, allocator); std::vector target_index_descriptors = initialize_batch(target_parser, allocator); - process_batch(query_index_descriptors, target_index_descriptors, query_parser, target_parser, allocator, print); + process_batch(query_index_descriptors, target_index_descriptors, query_parser, target_parser, allocator, print, format); return 0; } diff --git a/cudamapper/src/application_parameters.cpp b/cudamapper/src/application_parameters.cpp index ee6d013bc..999a5c539 100644 --- a/cudamapper/src/application_parameters.cpp +++ b/cudamapper/src/application_parameters.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include namespace claraparabricks @@ -59,12 +60,13 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) {"help", no_argument, 0, 'h'}, }; - std::string optstring = "k:w:d:m:i:t:F:a:r:l:b:z:RDQ:q:C:c:vh"; + std::string optstring = "k:w:d:m:i:t:F:a:r:l:b:z:RDQ:q:C:c:BSvh"; bool target_indices_in_host_memory_set = false; bool target_indices_in_device_memory_set = false; bool custom_filtering_parameter = false; int32_t argument = 0; + format = OutputFormat::PAF; while ((argument = getopt_long(argc, argv, optstring.c_str(), options, nullptr)) != -1) { switch (argument) @@ -131,6 +133,18 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) target_indices_in_device_memory = std::stoi(optarg); target_indices_in_device_memory_set = true; break; + case 'S': +#ifndef GW_BUILD_HTSLIB + throw std::runtime_error("ERROR: Argument -S cannot be used without htslib"); +#endif + format = OutputFormat::SAM; + break; + case 'B': +#ifndef GW_BUILD_HTSLIB + throw std::runtime_error("ERROR: Argument -B cannot be used without htslib"); +#endif + format = OutputFormat::BAM; + break; case 'v': print_version(); case 'h': @@ -355,6 +369,12 @@ void ApplicationParameters::help(int32_t exit_code) -c, --target-indices-in-device-memory number of target indices to keep in device memory [5])" << R"( + -S + print overlaps in SAM format" + << R"( + -B + print overlaps in BAM format" + << R"( -v, --version Version information)" << std::endl; diff --git a/cudamapper/src/application_parameters.hpp b/cudamapper/src/application_parameters.hpp index 22acfbfad..705d95c45 100644 --- a/cudamapper/src/application_parameters.hpp +++ b/cudamapper/src/application_parameters.hpp @@ -32,6 +32,9 @@ class FastaParser; namespace cudamapper { +// Forward declare enum for output type +enum class OutputFormat; + /// @brief application parameteres, default or passed through command line class ApplicationParameters { @@ -59,7 +62,8 @@ class ApplicationParameters int32_t query_indices_in_device_memory = 5; // q int32_t target_indices_in_host_memory = 10; // C int32_t target_indices_in_device_memory = 5; // c - bool all_to_all = false; + OutputFormat format; // default is PAF, S for SAM, and B for BAM + bool all_to_all = false; std::string query_filepath; std::string target_filepath; std::shared_ptr query_parser; diff --git a/cudamapper/src/main.cu b/cudamapper/src/main.cu index e39447302..ca8879226 100644 --- a/cudamapper/src/main.cu +++ b/cudamapper/src/main.cu @@ -370,15 +370,32 @@ void postprocess_and_write_thread_function(const int32_t device_id, 0.5); } - // write to output + // write to output PAF, SAM, or BAM format { - GW_NVTX_RANGE(profiler, "main::postprocess_and_write_thread::print_paf"); - print_paf(overlaps, - cigars, - *application_parameters.query_parser, - *application_parameters.target_parser, - application_parameters.kmer_size, - output_mutex); + GW_NVTX_RANGE(profiler, "main::postprocess_and_write_thread::print_function"); +#ifdef GW_BUILD_HTSLIB + if (application_parameters.format == OutputFormat::PAF) + { +#endif + print_paf(overlaps, + cigars, + *application_parameters.query_parser, + *application_parameters.target_parser, + application_parameters.kmer_size, + output_mutex); +#ifdef GW_BUILD_HTSLIB + } + // SAM or BAM, depends on type of format + else + { + print_sam(overlaps, + cigars, + *application_parameters.query_parser, + *application_parameters.target_parser, + application_parameters.format, + output_mutex); + } +#endif } } } diff --git a/cudamapper/src/utils.cpp b/cudamapper/src/utils.cpp index 1f4e040f3..24bcb9817 100644 --- a/cudamapper/src/utils.cpp +++ b/cudamapper/src/utils.cpp @@ -15,13 +15,19 @@ */ #include +#include #include #include #include +#include #include +#ifdef GW_BUILD_HTSLIB +#include "kroundup.h" +#endif + namespace claraparabricks { @@ -115,6 +121,197 @@ void print_paf(const std::vector& overlaps, } } +#ifdef GW_BUILD_HTSLIB +namespace +{ + +static inline void encode_cigar(bam1_t* const alignment, const std::string& cigar) +{ + uint32_t* bam_cigar_start = bam_get_cigar(alignment); + memcpy(bam_cigar_start, cigar.c_str(), cigar.length()); + alignment->core.n_cigar = cigar.length(); + + return; +} + +static inline void encode_seq(bam1_t* const alignment, const std::string& seq) +{ + // This table was taken from bam_construct_seq() for base -> nibble encoding + static const char L[256] = + { + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 15, 15, + 15, 1, 14, 2, 13, 15, 15, 4, 11, 15, 15, 12, 15, 3, 15, 15, + 15, 15, 5, 6, 8, 15, 7, 9, 15, 10, 15, 15, 15, 15, 15, 15, + 15, 1, 14, 2, 13, 15, 15, 4, 11, 15, 15, 12, 15, 3, 15, 15, + 15, 15, 5, 6, 8, 15, 7, 9, 15, 10, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; + + const size_t len = seq.length(); + + // encode bases as nibbles and write them into the alignment struct 2 bases at a time + // 4 high bits are first base, 4 low bits are second base + uint8_t* bam_seq_start = bam_get_seq(alignment); + int i = 0; + for (i; (i + 1) < len; i += 2) + { + *bam_seq_start++ = (L[static_cast(seq[i])] << 4) + L[static_cast(seq[i + 1])]; + } + // odd number of bases, so we have one left over + if (i < len) + { + *bam_seq_start++ = L[static_cast(seq[i])] << 4; + } + + alignment->core.l_qseq = len; + + return; +} + +static inline void encode_qual(bam1_t* alignment) +{ + // if htslib encounters 0xff at the start of the QUAL section, it writes default value (*) + auto bam_qual_start = bam_get_qual(alignment); + *bam_qual_start = 0xff; +} + +} // namespace + +void print_sam(const std::vector& overlaps, + const std::vector& cigars, + const io::FastaParser& query_parser, + const io::FastaParser& target_parser, + const OutputFormat format, + std::mutex& write_output_mutex, + int argc, + char* argv[]) +{ + GW_NVTX_RANGE(profiler, "print_sam::formatting_output"); + + // Assume output is stdout, and assume SAM output by default + // Rest of the add()/write() functions choose based on samFile format + // use hts_close here directly, because sam_close is a macro for hts_close, which the compiler did not like + using samFilePtr_t = std::unique_ptr; + + auto file = (format == OutputFormat::SAM) ? samFilePtr_t(sam_open("-", "wh"), &hts_close) : samFilePtr_t(sam_open("-", "bwh"), &hts_close); + auto header = std::unique_ptr(sam_hdr_init(), &sam_hdr_destroy); + int result = 0; + + if (file == nullptr) + { + throw std::runtime_error("print_sam: cannot open file stream for writing"); + } + + const int64_t number_of_overlaps_to_print = get_size(overlaps); + for (int64_t i = 0; i < number_of_overlaps_to_print; ++i) + { + const std::string& query_read_name = query_parser.get_sequence_by_id(overlaps[i].query_read_id_).name; + const std::string& target_read_name = target_parser.get_sequence_by_id(overlaps[i].target_read_id_).name; + const std::string length = std::to_string(target_parser.get_sequence_by_id(overlaps[i].target_read_id_).seq.length()); + + result = sam_hdr_add_line(header.get(), "SQ", "SN", target_read_name.c_str(), "LN", length.c_str(), + "CL", NULL); + if (result < 0) + { + fprintf(stderr, "print_sam: could not add header value"); + } + } + + // If we have the information to write out the CL section, do it + if ((argc != -1) && (argv != nullptr)) + { + auto command_line = std::unique_ptr(stringify_argv(argc, argv), &free); + result = sam_hdr_add_pg(header.get(), "cudamapper", "VN", claraparabricks_genomeworks_version().c_str(), + "CL", *command_line.get(), NULL); + } + else + { + result = sam_hdr_add_pg(header.get(), "cudamapper", "VN", claraparabricks_genomeworks_version().c_str(), NULL); + } + if (result < 0) + { + fprintf(stderr, "print_sam: could not add PG header line"); + } + + { + result = sam_hdr_write(file.get(), header.get()); + if (result < 0) + { + fprintf(stderr, "print_sam: could not add PG header line"); + } + } + + // write out mandatory fields for alignments + for (int64_t i = 0; i < number_of_overlaps_to_print; ++i) + { + const std::string& query_read_name = query_parser.get_sequence_by_id(overlaps[i].query_read_id_).name; + auto alignment = std::unique_ptr(bam_init1(), &bam_destroy1); + + // write QNAME field + bam_set_qname(alignment.get(), query_read_name.c_str()); + + // At this point we need to handle mem allocs ourselves, find necessary data size + size_t total_size = 0; + if (!cigars.empty()) + { + total_size = alignment->core.l_qname + alignment->core.l_extranul // len query name + how many padding '\0' chars + + cigars[i].length() + ((query_parser.get_sequence_by_id(overlaps[i].query_read_id_).seq.length() + 1) / 2); // sequence length and div by 2 because bases in the struct are encoded as nibbles + } + else + { + // case where we don't have CIGAR strings. See above calculation for details + total_size = alignment->core.l_qname + alignment->core.l_extranul + ((query_parser.get_sequence_by_id(overlaps[i].query_read_id_).seq.length() + 1) / 2); + } + const size_t max_data = kroundup64(total_size); + if (total_size >= alignment->m_data) + { + alignment->data = static_cast(realloc(alignment->data, max_data)); + } + alignment->l_data = total_size; + alignment->m_data = max_data; + + // write FLAG field + alignment->core.flag = 0; + + // write POS, MAPQ + // NOTE hardcode MAPQ to 255 like in print_paf, but SAM specification says we shouldn't do this + alignment->core.pos = static_cast(overlaps[i].query_start_position_in_read_); + alignment->core.qual = 255; + + // write CIGAR + if (!cigars.empty()) + { + encode_cigar(alignment.get(), cigars[i]); + } + + // write RNEXT, PNEXT, and TLEN + // all left as blank/default for now + + // write SEQ + encode_seq(alignment.get(), query_parser.get_sequence_by_id(overlaps[i].query_read_id_).seq); + + // QUAL is set to * by default + encode_qual(alignment.get()); + + // TODO: write AUX data if available + + std::lock_guard lg(write_output_mutex); + int out = sam_write1(file.get(), header.get(), alignment.get()); + } + + return; +} +#endif + std::vector group_reads_into_indices(const io::FastaParser& parser, const number_of_basepairs_t max_basepairs_per_index) { From 03ce56d709b3dc430024f4bd87c7c29771e54bea Mon Sep 17 00:00:00 2001 From: atadkase Date: Mon, 19 Oct 2020 09:28:45 -0700 Subject: [PATCH 203/281] [cudaextender] Fix external readme link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 06fcf89a5..c95c9878a 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ A prototypical binary to showcase the use of `libcudaaligner.so` APIs. ### cudaextender The `cudaextender` package provides GPU-accelerated seed-extension. Details can be found in -the package's [Readme](cudaextender/README.md). +the package's readme. ## Clone GenomeWorks From 79e67aa5a8500387504447f5973d5c55d07792b5 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Mon, 19 Oct 2020 22:42:20 -0400 Subject: [PATCH 204/281] [pygw] add copyright header --- pygenomeworks/bin/alignment_visualizer | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pygenomeworks/bin/alignment_visualizer b/pygenomeworks/bin/alignment_visualizer index e3b99e408..f62998dd2 100755 --- a/pygenomeworks/bin/alignment_visualizer +++ b/pygenomeworks/bin/alignment_visualizer @@ -1,4 +1,21 @@ #!/usr/bin/env python3 + +# +# Copyright 2020 NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Tool to visualize pairwise alignments.""" import argparse From bd0897baba951e1fbe7419fb89b693ea2ded1342 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 10:37:43 +0200 Subject: [PATCH 205/281] [cudamapper] Addressed smaller comments from PR #523 --- .../genomeworks/cudamapper/index.hpp | 2 +- cudamapper/src/index_cache.cuh | 6 +++--- cudamapper/src/index_gpu.cuh | 13 +------------ cudamapper/src/index_host_copy.cu | 2 +- 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index cb45cf105..c257fe7b4 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -134,7 +134,7 @@ class Index return sizeof(representation_t) * CHAR_BIT / 2; } - /// \brief checks if index is ready to be used, index might not be ready if its creation is asynchronous + /// \brief checks if index is ready to be used on this device, index might not be ready if its creation or copy from host is asynchronous /// \return true if the index is ready to be used virtual bool is_ready() const = 0; diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index 85406eeab..28900bc56 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -129,7 +129,7 @@ private: /// It is user's responsibility to make sure that indices requested by start_copying_indices_to_device() were generated by generate_content(). /// Memory copy to device is done asynchronously, the user should make sure that every call to start_copying_indices_to_device() is /// accompanied by a call DeviceIndexCache::wait_for_data_to_be_ready(). -/// The class tries to minimize the number of index creation and movemens, e.g. by reusing already existing indices, but not guarantees are given. +/// The class tries to minimize the number of index creation and movements, e.g. by reusing already existing indices, but not guarantees are given. class HostIndexCache { public: @@ -262,7 +262,7 @@ public: IndexDescriptor index_descriptor); /// \brief Returns the error message of the exception - virtual const char* what() const noexcept; + const char* what() const noexcept override; private: const std::string message_; @@ -279,7 +279,7 @@ public: IndexDescriptor index_descriptor); /// \brief Returns the error message of the exception - virtual const char* what() const noexcept; + virtual const char* what() const noexcept override; private: const std::string message_; diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index d8b14cd86..742b66065 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -132,7 +132,7 @@ public: /// \return length of the longest read in this index position_in_read_t number_of_basepairs_in_longest_read() const override; - /// \brief checks if index is ready to be used, index might not be ready if its creation is asynchronous + /// \brief checks if index is ready to be used on this device, index might not be ready if its creation or copy from host is asynchronous /// \return whether the index is ready to be used bool is_ready() const override; @@ -620,22 +620,11 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, number_of_reads_ = index_host_copy->number_of_reads(); number_of_basepairs_in_longest_read_ = index_host_copy->number_of_basepairs_in_longest_read(); - //H2D- representations_d_ = index_host_copy->representations(); cudautils::device_copy_n(index_host_copy->representations().data, index_host_copy->representations().size, representations_d_.data(), cuda_stream); - - //H2D- read_ids_d_ = index_host_copy->read_ids(); cudautils::device_copy_n(index_host_copy->read_ids().data, index_host_copy->read_ids().size, read_ids_d_.data(), cuda_stream); - - //H2D- positions_in_reads_d_ = index_host_copy->positions_in_reads(); cudautils::device_copy_n(index_host_copy->positions_in_reads().data, index_host_copy->positions_in_reads().size, positions_in_reads_d_.data(), cuda_stream); - - //H2D- directions_of_reads_d_ = index_host_copy->directions_of_reads(); cudautils::device_copy_n(index_host_copy->directions_of_reads().data, index_host_copy->directions_of_reads().size, directions_of_reads_d_.data(), cuda_stream); - - //H2D- unique_representations_d_ = index_host_copy->unique_representations(); cudautils::device_copy_n(index_host_copy->unique_representations().data, index_host_copy->unique_representations().size, unique_representations_d_.data(), cuda_stream); - - //H2D- first_occurrence_of_representations_d_ = index_host_copy->first_occurrence_of_representations(); cudautils::device_copy_n(index_host_copy->first_occurrence_of_representations().data, index_host_copy->first_occurrence_of_representations().size, first_occurrence_of_representations_d_.data(), cuda_stream); } diff --git a/cudamapper/src/index_host_copy.cu b/cudamapper/src/index_host_copy.cu index 3487264e5..5a83e50dd 100644 --- a/cudamapper/src/index_host_copy.cu +++ b/cudamapper/src/index_host_copy.cu @@ -216,7 +216,7 @@ void IndexHostCopy::IndexHostMemoryPinner::register_pinned_memory() { GW_NVTX_RANGE(profiler, "index_host_memory_pinner::register_pinned_memory"); GW_CU_CHECK_ERR(cudaHostRegister(index_host_copy_.underlying_array_.data(), - index_host_copy_.underlying_array_.size() * sizeof(unsigned char), + index_host_copy_.underlying_array_.size() * sizeof(gw_byte_t), cudaHostRegisterDefault)); } ++times_memory_pinned_; From 1a5e449373ec5fb3246c82fad5f41a640bde4242 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 10:44:13 +0200 Subject: [PATCH 206/281] [cudamapper] Not creating query_index_batch_to_start_copying and target_index_batch_to_start_copying in main.cu --- cudamapper/src/main.cu | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cudamapper/src/main.cu b/cudamapper/src/main.cu index ebb503159..f01449ac4 100644 --- a/cudamapper/src/main.cu +++ b/cudamapper/src/main.cu @@ -341,13 +341,10 @@ void process_one_batch(const BatchOfIndices& batch, { // if not pass-the-last batch start copying the batch GW_NVTX_RANGE(profiler, "main::process_one_batch::start_generating_device_indices"); - const std::vector& query_index_batch_to_start_copying = batch.device_batches[copy_device_batch_index].query_indices; - const std::vector& target_index_batch_to_start_copying = batch.device_batches[copy_device_batch_index].target_indices; - query_cache_being_copied_into = host_index_cache.start_copying_indices_to_device(CacheType::query_cache, - query_index_batch_to_start_copying); + batch.device_batches[copy_device_batch_index].query_indices); target_cache_being_copied_into = host_index_cache.start_copying_indices_to_device(CacheType::target_cache, - target_index_batch_to_start_copying); + batch.device_batches[copy_device_batch_index].target_indices); } if (copy_device_batch_index > 0) From f6398495f1348c863beebab870299b808706135c Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 10:52:03 +0200 Subject: [PATCH 207/281] [cudamapper] Calling DeviceIndexCache::get_index_no_check_if_ready() from DeviceIndexCache::get_index() --- cudamapper/src/index_cache.cu | 9 +-------- cudamapper/src/index_cache.cuh | 1 - 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index f8ce82bf2..7bfc95bea 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -62,14 +62,7 @@ std::shared_ptr DeviceIndexCache::get_index(const IndexDescriptor index_d index_descriptor); } - const auto index_iter = cache_.find(index_descriptor); - if (index_iter == cache_.end()) - { - throw IndexNotFoundException(cache_type_, - IndexNotFoundException::IndexLocation::device_cache, - index_descriptor); - } - return index_iter->second; + return get_index_no_check_if_ready(index_descriptor); } std::shared_ptr DeviceIndexCache::get_index_no_check_if_ready(IndexDescriptor index_descriptor) const diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index 28900bc56..dcba9eeb1 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -104,7 +104,6 @@ private: std::shared_ptr device_index); /// \brief Returns requested index, returned index might not be ready and has to be synchronized directly - /// To be called by HostIndexCache::start_copying_indices_to_device() /// \param index_descriptor /// \throw IndexNotFoundException if requested index is not cached /// \return requested index From 617001eb1a3ce08e8528f4ad3ebc7948c24727d5 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 11:59:09 +0200 Subject: [PATCH 208/281] [cudamapper] IndexGPUwait_to_be_ready() not const anymore --- .../claraparabricks/genomeworks/cudamapper/index.hpp | 2 +- cudamapper/src/index_gpu.cuh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index c257fe7b4..60ded6edf 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -139,7 +139,7 @@ class Index virtual bool is_ready() const = 0; /// \brief if is_ready() is true returns immediately, blocks until it becomes ready otherwise - virtual void wait_to_be_ready() const = 0; + virtual void wait_to_be_ready() = 0; /// \brief generates a mapping of (k,w)-kmer-representation to all of its occurrences for one or more sequences /// \param allocator The device memory allocator to use for temporary buffer allocations diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index 742b66065..80f89de46 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -137,7 +137,7 @@ public: bool is_ready() const override; /// \brief if is_ready() is true returns immediately, blocks until it becomes ready otherwise - void wait_to_be_ready() const override; + void wait_to_be_ready() override; private: /// \brief generates the index @@ -166,7 +166,7 @@ private: position_in_read_t number_of_basepairs_in_longest_read_ = 0; // has index generation finished - mutable bool is_ready_; + bool is_ready_; // if index is not generated but copied from the host a pointer to it is saved here const IndexHostCopy* const index_host_copy_source_; }; @@ -695,7 +695,7 @@ bool IndexGPU::is_ready() const } template -void IndexGPU::wait_to_be_ready() const +void IndexGPU::wait_to_be_ready() { // if index is not ready for usage wait for it to become ready if (!is_ready()) From 9e3bb0c741d93884783cdc599f006c098f50af67 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 14:11:11 +0200 Subject: [PATCH 209/281] [cudamapper] Removed unnecessary assert --- cudamapper/src/index_cache.cu | 1 - 1 file changed, 1 deletion(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 7bfc95bea..3147ed201 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -224,7 +224,6 @@ void HostIndexCache::generate_content(const CacheType cache_type, if (index_on_host) { - assert(index_on_host); new_cache[descriptor_of_index_to_cache] = index_on_host; } From 4fc64c474e13edc068f0e26e678649fc7ee72b8b Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 17:48:42 +0200 Subject: [PATCH 210/281] [cudamapper] Simplied the way index copies are synchronized in HostIndexCache::generate_content() --- cudamapper/src/index_cache.cu | 60 +++++++++++------------------------ 1 file changed, 19 insertions(+), 41 deletions(-) diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 3147ed201..a0c344b38 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -152,17 +152,20 @@ void HostIndexCache::generate_content(const CacheType cache_type, // being copied to host. In this implementation only two copies of index are kept on device: the one currently being generated and the one currently // being copied to host (from the previous step). - std::shared_ptr index_on_host = nullptr; - std::shared_ptr index_on_host_from_previous_step = nullptr; - std::shared_ptr index_on_device = nullptr; - std::shared_ptr index_on_device_from_previous_step = nullptr; - bool started_copy = false; // if index is found on host copy is not needed - bool started_copy_from_previous_step = false; + std::shared_ptr index_on_host = nullptr; + std::shared_ptr index_on_device = nullptr; + // Only one pair of (host, device) indices can be copied at a time. Whenever a copy should be started if these pointers are not null (= copy is in progress) + // wait for that current copy to be done first + std::shared_ptr index_on_host_copy_in_flight = nullptr; + std::shared_ptr index_on_device_copy_in_flight = nullptr; const bool host_copy_needed = !skip_copy_to_host; for (const IndexDescriptor& descriptor_of_index_to_cache : descriptors_of_indices_to_cache) { + index_on_host = nullptr; + index_on_device = nullptr; + const bool device_copy_needed = descriptors_of_indices_to_keep_on_device_set.count(descriptor_of_index_to_cache) != 0; // check if host copy already exists @@ -203,14 +206,11 @@ void HostIndexCache::generate_content(const CacheType cache_type, if (host_copy_needed) { - // if a D2H copy has been been started in the previous step wait for it to finish - // TODO: do this sync using an event - if (started_copy_from_previous_step) + // if a copy is already in progress wait for it to finish + if (index_on_host_copy_in_flight) { - index_on_host_from_previous_step->finish_copying(); - // copying is done, these pointer are not needed anymore - index_on_host_from_previous_step = nullptr; - index_on_device_from_previous_step = nullptr; + assert(index_on_host_copy_in_flight && index_on_device_copy_in_flight); + index_on_host_copy_in_flight->finish_copying(); } index_on_host = IndexHostCopy::create_host_copy(*index_on_device, @@ -218,7 +218,9 @@ void HostIndexCache::generate_content(const CacheType cache_type, kmer_size_, window_size_, cuda_stream_copy_); - started_copy = true; // index is being copied to host memory, a sync will be needed + + index_on_host_copy_in_flight = index_on_host; + index_on_device_copy_in_flight = index_on_device; } } @@ -234,37 +236,13 @@ void HostIndexCache::generate_content(const CacheType cache_type, { indices_kept_on_device[descriptor_of_index_to_cache] = index_on_device; } - - // if a D2H copy has been been started in the previous step and it has not been waited for yet wait for it to finish - if (started_copy_from_previous_step && index_on_host_from_previous_step && index_on_device_from_previous_step) - { - index_on_host_from_previous_step->finish_copying(); - // copying is done, these pointer are not needed anymore - index_on_host_from_previous_step = nullptr; - index_on_device_from_previous_step = nullptr; - } - - // prepare for next step - started_copy_from_previous_step = started_copy; - if (started_copy) - { - index_on_host_from_previous_step = index_on_host; - index_on_device_from_previous_step = index_on_device; - } - else - { - index_on_host_from_previous_step = nullptr; - index_on_device_from_previous_step = nullptr; - } - index_on_host = nullptr; - index_on_device = nullptr; - started_copy = false; } // wait for the last copy to finish - if (started_copy_from_previous_step) + if (index_on_host_copy_in_flight) { - index_on_host_from_previous_step->finish_copying(); + assert(index_on_host_copy_in_flight && index_on_device_copy_in_flight); + index_on_host_copy_in_flight->finish_copying(); } std::swap(new_cache, this_cache); From 6b88d6bd1f7b0a16717fae598939af7bf5a27b11 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 18:14:15 +0200 Subject: [PATCH 211/281] [cudamapper] Calling wait_to_be_ready() after every index generation --- .../claraparabricks/genomeworks/cudamapper/index.hpp | 1 + cudamapper/samples/sample_cudamapper.cpp | 2 ++ cudamapper/src/index_cache.cu | 5 +---- cudamapper/src/index_gpu.cuh | 9 ++------- cudamapper/tests/Test_CudamapperIndexGPU.cu | 2 +- cudamapper/tests/Test_CudamapperMatcherGPU.cu | 8 ++++++-- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index 60ded6edf..c506ed7ba 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -142,6 +142,7 @@ class Index virtual void wait_to_be_ready() = 0; /// \brief generates a mapping of (k,w)-kmer-representation to all of its occurrences for one or more sequences + /// Generation is done asynchronously and one should call wait_to_be_ready() before using the object /// \param allocator The device memory allocator to use for temporary buffer allocations /// \param parser parser for the whole input file (part that goes into this index is determined by first_read_id and past_the_last_read_id) /// \param descriptor object holding the first read, size of the read, and hash value of the index being created diff --git a/cudamapper/samples/sample_cudamapper.cpp b/cudamapper/samples/sample_cudamapper.cpp index 51add049f..2905d97e1 100644 --- a/cudamapper/samples/sample_cudamapper.cpp +++ b/cudamapper/samples/sample_cudamapper.cpp @@ -90,6 +90,7 @@ void process_batch(std::vector& query_index_descriptors, WINDOWS_SIZE, true, // hash representations FILTERING_PARAMETER); // filter parameter + query_index->wait_to_be_ready(); for (const IndexDescriptor& target_index_descriptor : target_index_descriptors) { @@ -103,6 +104,7 @@ void process_batch(std::vector& query_index_descriptors, WINDOWS_SIZE, true, // hash representations FILTERING_PARAMETER); // filter parameter + target_index->wait_to_be_ready(); // find anchors & find overlaps auto matcher = Matcher::create_matcher(allocator, diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index a0c344b38..5cb3ffad6 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -199,10 +199,7 @@ void HostIndexCache::generate_content(const CacheType cache_type, filtering_parameter_, cuda_stream_generation_, cuda_stream_copy_); - - // wait for index to be generated on cuda_stream_generation_ before copying it on cuda_stream_copy_ - // TODO: do this sync using an event - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation_)); + index_on_device->wait_to_be_ready(); if (host_copy_needed) { diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index 80f89de46..20585534e 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -583,7 +583,7 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, , directions_of_reads_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) , unique_representations_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) , first_occurrence_of_representations_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) - , is_ready_(false) // set to true at the end of generate_index() + , is_ready_(false) , index_host_copy_source_(nullptr) { generate_index(parser, @@ -594,9 +594,8 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, allocator, cuda_stream_generation); - // This is not completely necessary, but if removed one has to make sure that the next step - // uses the same stream or that sync is done in caller GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation)); + is_ready_ = true; } template @@ -878,10 +877,6 @@ void IndexGPU::generate_index(const io::FastaParser& parser, first_occurrence_of_representations_d_, cuda_stream); } - - // TODO: replace with stream callback - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream)); - is_ready_ = true; } } // namespace cudamapper diff --git a/cudamapper/tests/Test_CudamapperIndexGPU.cu b/cudamapper/tests/Test_CudamapperIndexGPU.cu index 642e80785..9350fcba8 100644 --- a/cudamapper/tests/Test_CudamapperIndexGPU.cu +++ b/cudamapper/tests/Test_CudamapperIndexGPU.cu @@ -1345,7 +1345,7 @@ void test_function(const std::string& filename, false, filtering_parameter, cuda_stream.get()); - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream.get())); + index.wait_to_be_ready(); ASSERT_EQ(index.number_of_reads(), expected_number_of_reads); if (0 == expected_number_of_reads) diff --git a/cudamapper/tests/Test_CudamapperMatcherGPU.cu b/cudamapper/tests/Test_CudamapperMatcherGPU.cu index 618c0bfa8..59945897e 100644 --- a/cudamapper/tests/Test_CudamapperMatcherGPU.cu +++ b/cudamapper/tests/Test_CudamapperMatcherGPU.cu @@ -640,7 +640,9 @@ TEST(TestCudamapperMatcherGPU, OneReadOneMinimizer) DefaultDeviceAllocator allocator = create_default_device_allocator(); std::unique_ptr parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/gatt.fasta"); std::unique_ptr query_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); - std::unique_ptr target_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); + query_index->wait_to_be_ready(); + std::unique_ptr target_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); + target_index->wait_to_be_ready(); MatcherGPU matcher(allocator, *query_index, *target_index); thrust::host_vector anchors(matcher.anchors().size()); @@ -653,7 +655,9 @@ TEST(TestCudamapperMatcherGPU, AtLeastOneIndexEmpty) DefaultDeviceAllocator allocator = create_default_device_allocator(); std::unique_ptr parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/gatt.fasta"); std::unique_ptr index_full = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); - std::unique_ptr index_empty = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 5, 1); // kmer longer than read + index_full->wait_to_be_ready(); + std::unique_ptr index_empty = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 5, 1); // kmer longer than read + index_empty->wait_to_be_ready(); { MatcherGPU matcher(allocator, *index_full, *index_empty); From b3fb10ce2097151ea5c80e47f780b7004853a2e7 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 18:19:32 +0200 Subject: [PATCH 212/281] [cudamapper] Asserting is_ready() before accessing members of IndexGPU --- cudamapper/src/index_gpu.cuh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index 20585534e..fef8329a6 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -630,60 +630,70 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, template const device_buffer& IndexGPU::representations() const { + assert(is_ready()); return representations_d_; }; template const device_buffer& IndexGPU::read_ids() const { + assert(is_ready()); return read_ids_d_; } template const device_buffer& IndexGPU::positions_in_reads() const { + assert(is_ready()); return positions_in_reads_d_; } template const device_buffer& IndexGPU::directions_of_reads() const { + assert(is_ready()); return directions_of_reads_d_; } template const device_buffer& IndexGPU::unique_representations() const { + assert(is_ready()); return unique_representations_d_; } template const device_buffer& IndexGPU::first_occurrence_of_representations() const { + assert(is_ready()); return first_occurrence_of_representations_d_; } template read_id_t IndexGPU::number_of_reads() const { + assert(is_ready()); return number_of_reads_; } template read_id_t IndexGPU::smallest_read_id() const { + assert(is_ready()); return number_of_reads_ > 0 ? first_read_id_ : 0; } template read_id_t IndexGPU::largest_read_id() const { + assert(is_ready()); return number_of_reads_ > 0 ? first_read_id_ + number_of_reads_ - 1 : 0; } template position_in_read_t IndexGPU::number_of_basepairs_in_longest_read() const { + assert(is_ready()); return number_of_basepairs_in_longest_read_; } From f68f82da0a9d86eae9a0b6084b534b29b289fa8c Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 19:12:44 +0200 Subject: [PATCH 213/281] [cudamapper] Using IndexNotReadyException instead of asserts in Index --- .../genomeworks/cudamapper/index.hpp | 28 +++++++- cudamapper/src/index.cu | 10 +++ cudamapper/src/index_cache.cu | 4 +- cudamapper/src/index_cache.cuh | 3 +- cudamapper/src/index_gpu.cuh | 70 ++++++++++++++++--- 5 files changed, 101 insertions(+), 14 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index c506ed7ba..33447e75c 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -16,9 +16,9 @@ #pragma once +#include #include #include -#include #include #include #include @@ -88,46 +88,57 @@ class Index virtual ~Index() = default; /// \brief returns an array of representations of sketch elements + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array of representations of sketch elements virtual const device_buffer& representations() const = 0; /// \brief returns an array of reads ids for sketch elements + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array of reads ids for sketch elements virtual const device_buffer& read_ids() const = 0; /// \brief returns an array of starting positions of sketch elements in their reads + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array of starting positions of sketch elements in their reads virtual const device_buffer& positions_in_reads() const = 0; /// \brief returns an array of directions in which sketch elements were read + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array of directions in which sketch elements were read virtual const device_buffer& directions_of_reads() const = 0; /// \brief returns an array where each representation is recorder only once, sorted by representation + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array where each representation is recorder only once, sorted by representation virtual const device_buffer& unique_representations() const = 0; /// \brief returns first occurrence of corresponding representation from unique_representations() in data arrays + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return first occurrence of corresponding representation from unique_representations() in data arrays virtual const device_buffer& first_occurrence_of_representations() const = 0; /// \brief returns number of reads in input data + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return number of reads in input data virtual read_id_t number_of_reads() const = 0; /// \brief returns smallest read_id in index + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return smallest read_id in index (0 if empty index) virtual read_id_t smallest_read_id() const = 0; /// \brief returns largest read_id in index + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return largest read_id in index (0 if empty index) virtual read_id_t largest_read_id() const = 0; /// \brief returns length of the longest read in this index + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return length of the longest read in this index virtual position_in_read_t number_of_basepairs_in_longest_read() const = 0; /// \brief Return the maximum kmer length allowable + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return Return the maximum kmer length allowable static uint64_t maximum_kmer_size() { @@ -254,6 +265,21 @@ class IndexHostCopyBase const cudaStream_t cuda_stream = 0); }; +/// IndexNotReadyException - Exception to be thrown if a member of Index is accessed, but is_read() == false +class IndexNotReadyException : public std::exception +{ +public: + /// \brief Constructor + /// \param function_name name of funciton which has been accessed + IndexNotReadyException(const std::string& function_name); + + /// \brief Returns the error message of the exception + const char* what() const noexcept override; + +private: + const std::string message_; +}; + } // namespace cudamapper } // namespace genomeworks diff --git a/cudamapper/src/index.cu b/cudamapper/src/index.cu index 2b57e3459..49fb89e42 100644 --- a/cudamapper/src/index.cu +++ b/cudamapper/src/index.cu @@ -64,6 +64,16 @@ std::unique_ptr IndexHostCopyBase::create_host_copy(const Ind cuda_stream); } +IndexNotReadyException::IndexNotReadyException(const std::string& function_name) + : message_("Index::" + function_name + "() has been accessed before a call to wait_to_be_ready()") +{ +} + +const char* IndexNotReadyException::what() const noexcept +{ + return message_.c_str(); +} + } // namespace cudamapper } // namespace genomeworks diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 5cb3ffad6..7e75a5213 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -56,7 +56,7 @@ void DeviceIndexCache::add_index(const IndexDescriptor index_descriptor, std::shared_ptr DeviceIndexCache::get_index(const IndexDescriptor index_descriptor) const { - if (!is_ready_) + if (!is_ready()) { throw DeviceCacheNotReadyException(cache_type_, index_descriptor); @@ -85,7 +85,7 @@ bool DeviceIndexCache::has_index(const IndexDescriptor index_descriptor) const void DeviceIndexCache::wait_for_data_to_be_ready() { - if (!is_ready_) + if (!is_ready()) { for (const auto& index_it : cache_) { diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index dcba9eeb1..34ae7286c 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -254,8 +254,9 @@ public: }; /// \brief constructor + /// \param cache_type was Index requested from host or device cache + /// \param index_location /// \param index_descriptor - /// \param index_type was Index equested from host or device cache IndexNotFoundException(CacheType cache_type, IndexLocation index_location, IndexDescriptor index_descriptor); diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index fef8329a6..a82e257d1 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -93,42 +93,52 @@ public: const cudaStream_t cuda_stream = 0); /// \brief returns an array of representations of sketch elements + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array of representations of sketch elements const device_buffer& representations() const override; /// \brief returns an array of reads ids for sketch elements + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array of reads ids for sketch elements const device_buffer& read_ids() const override; /// \brief returns an array of starting positions of sketch elements in their reads + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array of starting positions of sketch elements in their reads const device_buffer& positions_in_reads() const override; /// \brief returns an array of directions in which sketch elements were read + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array of directions in which sketch elements were read const device_buffer& directions_of_reads() const override; /// \brief returns an array where each representation is recorder only once, sorted by representation + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return an array where each representation is recorder only once, sorted by representation const device_buffer& unique_representations() const override; /// \brief returns first occurrence of corresponding representation from unique_representations() in data arrays, plus one more element with the total number of sketch elements + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return first occurrence of corresponding representation from unique_representations() in data arrays, plus one more element with the total number of sketch elements const device_buffer& first_occurrence_of_representations() const override; /// \brief returns number of reads in input data + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return number of reads in input data read_id_t number_of_reads() const override; /// \brief returns smallest read_id in index + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return smallest read_id in index (0 if empty index) read_id_t smallest_read_id() const override; /// \brief returns largest read_id in index + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return largest read_id in index (0 if empty index) read_id_t largest_read_id() const override; /// \brief returns length of the longest read in this index + /// \throw IndexNotReadyException if called before wait_to_be_ready() /// \return length of the longest read in this index position_in_read_t number_of_basepairs_in_longest_read() const override; @@ -630,70 +640,110 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, template const device_buffer& IndexGPU::representations() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("representations"); + } + return representations_d_; }; template const device_buffer& IndexGPU::read_ids() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("read_ids"); + } + return read_ids_d_; } template const device_buffer& IndexGPU::positions_in_reads() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("positions_in_reads"); + } + return positions_in_reads_d_; } template const device_buffer& IndexGPU::directions_of_reads() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("directions_of_reads"); + } + return directions_of_reads_d_; } template const device_buffer& IndexGPU::unique_representations() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("unique_representations"); + } + return unique_representations_d_; } template const device_buffer& IndexGPU::first_occurrence_of_representations() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("first_occurrence_of_representations"); + } + return first_occurrence_of_representations_d_; } template read_id_t IndexGPU::number_of_reads() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("number_of_reads"); + } + return number_of_reads_; } template read_id_t IndexGPU::smallest_read_id() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("smallest_read_id"); + } + return number_of_reads_ > 0 ? first_read_id_ : 0; } template read_id_t IndexGPU::largest_read_id() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("largest_read_id"); + } + return number_of_reads_ > 0 ? first_read_id_ + number_of_reads_ - 1 : 0; } template position_in_read_t IndexGPU::number_of_basepairs_in_longest_read() const { - assert(is_ready()); + if (!is_ready()) + { + throw IndexNotReadyException("number_of_basepairs_in_longest_read"); + } + return number_of_basepairs_in_longest_read_; } From 86ee16b8321a1c5c43c209ac37edb4e07bcd36d4 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Wed, 21 Oct 2020 19:31:38 +0200 Subject: [PATCH 214/281] [cudamapper] IndexGPU's generating constructor is now asynchronous --- cudamapper/src/index_gpu.cuh | 42 ++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/cudamapper/src/index_gpu.cuh b/cudamapper/src/index_gpu.cuh index a82e257d1..37e2ac487 100644 --- a/cudamapper/src/index_gpu.cuh +++ b/cudamapper/src/index_gpu.cuh @@ -62,7 +62,9 @@ template class IndexGPU : public Index { public: - /// \brief Constructor whcih constucts a new index + /// \brief Constructor which generates a new index + /// + /// Generation is done asynchronously and one should call wait_to_be_ready() before using the object /// /// \param parser parser for the whole input file (part that goes into this index is determined by first_read_id and past_the_last_read_id) /// \param first_read_id read_id of the first read to the included in this index @@ -85,6 +87,8 @@ public: /// \brief Constructor which copies the index from host copy /// + /// Copy is done asynchronously and one should call wait_to_be_ready() before using the object + /// /// \param allocator is pointer to asynchronous device allocator /// \param index_host_copy is a copy of index for a set of reads which has been previously computed and stored on the host. /// \param cuda_stream CUDA stream on which the work is to be done. Device arrays are also associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor is done @@ -150,6 +154,13 @@ public: void wait_to_be_ready() override; private: + /// WayOfConstruciton - has IndexGPU been created by generation or copy from host copy + enum class WayOfConstruciton + { + generation, + copy + }; + /// \brief generates the index void generate_index(const io::FastaParser& query_parser, const read_id_t first_read_id, @@ -174,11 +185,15 @@ private: const std::uint64_t window_size_ = 0; read_id_t number_of_reads_ = 0; position_in_read_t number_of_basepairs_in_longest_read_ = 0; + const WayOfConstruciton way_of_construciton_; // has index generation finished bool is_ready_; // if index is not generated but copied from the host a pointer to it is saved here const IndexHostCopy* const index_host_copy_source_; + + // only important is using generating constructor, ignored in copying constructor + cudaStream_t cuda_stream_generation_; }; namespace details @@ -593,8 +608,10 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, , directions_of_reads_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) , unique_representations_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) , first_occurrence_of_representations_d_(0, allocator, cuda_stream_generation, cuda_stream_copy) - , is_ready_(false) + , way_of_construciton_(WayOfConstruciton::generation) + , is_ready_(false) // set to true in wait_to_be_ready() , index_host_copy_source_(nullptr) + , cuda_stream_generation_(cuda_stream_generation) { generate_index(parser, first_read_id_, @@ -603,9 +620,6 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, filtering_parameter, allocator, cuda_stream_generation); - - GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation)); - is_ready_ = true; } template @@ -621,8 +635,10 @@ IndexGPU::IndexGPU(DefaultDeviceAllocator allocator, , directions_of_reads_d_(index_host_copy->directions_of_reads().size, allocator, cuda_stream) , unique_representations_d_(index_host_copy->unique_representations().size, allocator, cuda_stream) , first_occurrence_of_representations_d_(index_host_copy->first_occurrence_of_representations().size, allocator, cuda_stream) + , way_of_construciton_(WayOfConstruciton::copy) , is_ready_(false) // set to true in wait_to_be_ready() , index_host_copy_source_(index_host_copy) + , cuda_stream_generation_(cuda_stream) { GW_NVTX_RANGE(profiler, "IndexGPU::constructor_copy_from_IndexHostCopy"); @@ -759,8 +775,20 @@ void IndexGPU::wait_to_be_ready() // if index is not ready for usage wait for it to become ready if (!is_ready()) { - assert(index_host_copy_source_); - index_host_copy_source_->finish_copying(); + if (WayOfConstruciton::generation == way_of_construciton_) + { + GW_CU_CHECK_ERR(cudaStreamSynchronize(cuda_stream_generation_)); + } + else if (WayOfConstruciton::copy == way_of_construciton_) + { + assert(index_host_copy_source_); + index_host_copy_source_->finish_copying(); + } + else + { + assert(!"unsupported value of WayOfConstruciton"); + } + is_ready_ = true; } } From bc4a6653016224a3fa8c0f67e72ffdea5e4dd690 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 21 Oct 2020 16:27:28 -0400 Subject: [PATCH 215/281] [debug] renaming some macros --- cudapoa/src/cudapoa_kernels.cuh | 18 +++++++++--------- cudapoa/src/cudapoa_nw.cuh | 2 +- cudapoa/src/cudapoa_nw_banded.cuh | 4 ++-- cudapoa/src/cudapoa_nw_tb_banded.cuh | 6 +++--- cudapoa/src/cudapoa_structs.cuh | 6 +++--- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index a9c4443dd..551706c44 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -29,8 +29,8 @@ #include // considering 65536 register file size, __launch_bounds__(1024) translates to max 64 register usage -#define MAX_THREADS_PER_BLOCK_64 1024 -#define MAX_THREADS_PER_BLOCK_72 896 +#define GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_64_REGISTERS 1024 +#define GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS 896 namespace claraparabricks { @@ -74,7 +74,7 @@ namespace cudapoa * @param[in] match_score Score for finding a match in alignment */ template -__launch_bounds__(MAX_THREADS_PER_BLOCK_64) +__launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_64_REGISTERS) __global__ void generatePOAKernel(uint8_t* consensus_d, uint8_t* sequences_d, int8_t* base_weights_d, @@ -342,7 +342,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) __syncwarp(); } - if (alignment_length == NW_BACKTRACKING_LOOP_FAILED) + if (alignment_length == CUDAPOA_KERNEL_NW_BACKTRACKING_LOOP_FAILED) { if (lane_idx == 0) { @@ -351,7 +351,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) } return; } - else if (alignment_length == NW_ADAPTIVE_STORAGE_FAILED) + else if (alignment_length == CUDAPOA_KERNEL_NW_ADAPTIVE_STORAGE_FAILED) { if (lane_idx == 0) { @@ -433,7 +433,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_64) // Similar to generatePOAKernel(), but with a different launch bounds setting, used to call kernels using traceback buffer // i.e. for BM == BandMode::static_band_traceback or BM == BandMode::adaptive_band_traceback template -__launch_bounds__(MAX_THREADS_PER_BLOCK_72) +__launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS) __global__ void generatePOAKernelTB(uint8_t* consensus_d, uint8_t* sequences_d, int8_t* base_weights_d, @@ -682,7 +682,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_72) __syncwarp(); } - if (alignment_length == NW_BACKTRACKING_LOOP_FAILED) + if (alignment_length == CUDAPOA_KERNEL_NW_BACKTRACKING_LOOP_FAILED) { if (lane_idx == 0) { @@ -691,7 +691,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_72) } return; } - else if (alignment_length == NW_ADAPTIVE_STORAGE_FAILED) + else if (alignment_length == CUDAPOA_KERNEL_NW_ADAPTIVE_STORAGE_FAILED) { if (lane_idx == 0) { @@ -700,7 +700,7 @@ __launch_bounds__(MAX_THREADS_PER_BLOCK_72) } return; } - else if (alignment_length == NW_TRACEBACK_BUFFER_FAILED) + else if (alignment_length == CUDAPOA_KERNEL_NW_TRACEBACK_BUFFER_FAILED) { if (lane_idx == 0) { diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index 97cf03215..76bdc4474 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -441,7 +441,7 @@ __device__ __forceinline__ if (loop_count >= (read_length + graph_count + 2)) { - aligned_nodes = NW_BACKTRACKING_LOOP_FAILED; + aligned_nodes = CUDAPOA_KERNEL_NW_BACKTRACKING_LOOP_FAILED; } #ifdef NW_VERBOSE_PRINT diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 51342edcf..d81490453 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -260,7 +260,7 @@ __device__ __forceinline__ float required_buffer_size = static_cast(graph_count) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); if (required_buffer_size > max_buffer_size) { - return NW_ADAPTIVE_STORAGE_FAILED; + return CUDAPOA_KERNEL_NW_ADAPTIVE_STORAGE_FAILED; } } //--------------------------------------------------------- @@ -545,7 +545,7 @@ __device__ __forceinline__ if (loop_count >= (read_length + graph_count + 2)) { - aligned_nodes = NW_BACKTRACKING_LOOP_FAILED; + aligned_nodes = CUDAPOA_KERNEL_NW_BACKTRACKING_LOOP_FAILED; } #ifdef NW_VERBOSE_PRINT diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index 956eb8cae..e672c151b 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -327,7 +327,7 @@ __device__ __forceinline__ float required_buffer_size = static_cast(graph_count) * static_cast(band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); if (required_buffer_size > max_buffer_size) { - return NW_ADAPTIVE_STORAGE_FAILED; + return CUDAPOA_KERNEL_NW_ADAPTIVE_STORAGE_FAILED; } } //--------------------------------------------------------- @@ -564,7 +564,7 @@ __device__ __forceinline__ if (i == 0) { j = 0; - aligned_nodes = NW_TRACEBACK_BUFFER_FAILED; + aligned_nodes = CUDAPOA_KERNEL_NW_TRACEBACK_BUFFER_FAILED; } //------------------------------------------------------------------------ @@ -631,7 +631,7 @@ __device__ __forceinline__ if (loop_count >= (read_length + graph_count + 2)) { - aligned_nodes = NW_BACKTRACKING_LOOP_FAILED; + aligned_nodes = CUDAPOA_KERNEL_NW_BACKTRACKING_LOOP_FAILED; } #ifdef NW_VERBOSE_PRINT diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index c90e3aa07..78d85fd7e 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -50,9 +50,9 @@ #define CUDAPOA_KERNEL_NOERROR_ENCOUNTERED 0 // NW return error codes -#define NW_BACKTRACKING_LOOP_FAILED -1 -#define NW_ADAPTIVE_STORAGE_FAILED -2 -#define NW_TRACEBACK_BUFFER_FAILED -3 +#define CUDAPOA_KERNEL_NW_BACKTRACKING_LOOP_FAILED -1 +#define CUDAPOA_KERNEL_NW_ADAPTIVE_STORAGE_FAILED -2 +#define CUDAPOA_KERNEL_NW_TRACEBACK_BUFFER_FAILED -3 namespace claraparabricks { From 0e96eb1565732caaf13c7a2176fdcba6e8311f2f Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 22 Oct 2020 11:16:09 +0200 Subject: [PATCH 216/281] [cudamapper] Removed unnecessary virutal from DeviceCacheNotReadyException::what() --- cudamapper/src/index_cache.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudamapper/src/index_cache.cuh b/cudamapper/src/index_cache.cuh index 34ae7286c..4dec7d9a7 100644 --- a/cudamapper/src/index_cache.cuh +++ b/cudamapper/src/index_cache.cuh @@ -279,7 +279,7 @@ public: IndexDescriptor index_descriptor); /// \brief Returns the error message of the exception - virtual const char* what() const noexcept override; + const char* what() const noexcept override; private: const std::string message_; From 04998ae3eeb2304b75dc3cca639c8e7172314bfc Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 22 Oct 2020 12:30:26 +0200 Subject: [PATCH 217/281] [cudamapper] Added _async to create_index() and create_host_copy() --- .../genomeworks/cudamapper/index.hpp | 28 ++++++++--------- cudamapper/samples/sample_cudamapper.cpp | 28 ++++++++--------- cudamapper/src/index.cu | 30 +++++++++---------- cudamapper/src/index_cache.cu | 28 ++++++++--------- cudamapper/tests/Test_CudamapperMatcherGPU.cu | 8 ++--- 5 files changed, 61 insertions(+), 61 deletions(-) diff --git a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp index 33447e75c..abb142e20 100644 --- a/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp +++ b/cudamapper/include/claraparabricks/genomeworks/cudamapper/index.hpp @@ -165,15 +165,15 @@ class Index /// \param cuda_stream_copy CUDA stream on which the index is copied to host if needed. Device arrays are associated with this stream and will not be freed at least until all work issued on this stream before calling their destructor has been done /// \return instance of Index static std::unique_ptr - create_index(DefaultDeviceAllocator allocator, - const io::FastaParser& parser, - const IndexDescriptor& descriptor, - const std::uint64_t kmer_size, - const std::uint64_t window_size, - const bool hash_representations = true, - const double filtering_parameter = 1.0, - const cudaStream_t cuda_stream_generation = 0, - const cudaStream_t cuda_stream_copy = 0); + create_index_async(DefaultDeviceAllocator allocator, + const io::FastaParser& parser, + const IndexDescriptor& descriptor, + const std::uint64_t kmer_size, + const std::uint64_t window_size, + const bool hash_representations = true, + const double filtering_parameter = 1.0, + const cudaStream_t cuda_stream_generation = 0, + const cudaStream_t cuda_stream_copy = 0); }; /// IndexHostCopyBase - Creates and maintains a copy of computed IndexGPU elements on the host, then allows to retrieve target @@ -258,11 +258,11 @@ class IndexHostCopyBase /// \param window_size the number of adjacent k-mers in a window, adjacent = shifted by one basepair /// \param cuda_stream D2H copy is done on this stream /// \return - an instance of IndexHostCopyBase - static std::unique_ptr create_host_copy(const Index& index, - const read_id_t first_read_id, - const std::uint64_t kmer_size, - const std::uint64_t window_size, - const cudaStream_t cuda_stream = 0); + static std::unique_ptr create_host_copy_async(const Index& index, + const read_id_t first_read_id, + const std::uint64_t kmer_size, + const std::uint64_t window_size, + const cudaStream_t cuda_stream = 0); }; /// IndexNotReadyException - Exception to be thrown if a member of Index is accessed, but is_read() == false diff --git a/cudamapper/samples/sample_cudamapper.cpp b/cudamapper/samples/sample_cudamapper.cpp index 2905d97e1..cfe2d59f9 100644 --- a/cudamapper/samples/sample_cudamapper.cpp +++ b/cudamapper/samples/sample_cudamapper.cpp @@ -83,13 +83,13 @@ void process_batch(std::vector& query_index_descriptors, // process the pairs of query and target indices for (const IndexDescriptor& query_index_descriptor : query_index_descriptors) { - std::unique_ptr query_index = Index::create_index(allocator, - *query_parser, - query_index_descriptor, - KMER_SIZE, - WINDOWS_SIZE, - true, // hash representations - FILTERING_PARAMETER); // filter parameter + std::unique_ptr query_index = Index::create_index_async(allocator, + *query_parser, + query_index_descriptor, + KMER_SIZE, + WINDOWS_SIZE, + true, // hash representations + FILTERING_PARAMETER); // filter parameter query_index->wait_to_be_ready(); for (const IndexDescriptor& target_index_descriptor : target_index_descriptors) @@ -97,13 +97,13 @@ void process_batch(std::vector& query_index_descriptors, // skip pairs in which target batch has smaller id than query batch as it will be covered by symmetry if (target_index_descriptor.first_read() >= query_index_descriptor.first_read()) { - std::unique_ptr target_index = Index::create_index(allocator, - *target_parser, - target_index_descriptor, - KMER_SIZE, - WINDOWS_SIZE, - true, // hash representations - FILTERING_PARAMETER); // filter parameter + std::unique_ptr target_index = Index::create_index_async(allocator, + *target_parser, + target_index_descriptor, + KMER_SIZE, + WINDOWS_SIZE, + true, // hash representations + FILTERING_PARAMETER); // filter parameter target_index->wait_to_be_ready(); // find anchors & find overlaps diff --git a/cudamapper/src/index.cu b/cudamapper/src/index.cu index 49fb89e42..168dbedd6 100644 --- a/cudamapper/src/index.cu +++ b/cudamapper/src/index.cu @@ -28,17 +28,17 @@ namespace genomeworks namespace cudamapper { -std::unique_ptr Index::create_index(DefaultDeviceAllocator allocator, - const io::FastaParser& parser, - const IndexDescriptor& descriptor, - const std::uint64_t kmer_size, - const std::uint64_t window_size, - const bool hash_representations, - const double filtering_parameter, - const cudaStream_t cuda_stream_generation, - const cudaStream_t cuda_stream_copy) +std::unique_ptr Index::create_index_async(DefaultDeviceAllocator allocator, + const io::FastaParser& parser, + const IndexDescriptor& descriptor, + const std::uint64_t kmer_size, + const std::uint64_t window_size, + const bool hash_representations, + const double filtering_parameter, + const cudaStream_t cuda_stream_generation, + const cudaStream_t cuda_stream_copy) { - GW_NVTX_RANGE(profiler, "create_index"); + GW_NVTX_RANGE(profiler, "create_index_async"); return std::make_unique>(allocator, parser, descriptor, @@ -50,11 +50,11 @@ std::unique_ptr Index::create_index(DefaultDeviceAllocator allocator, cuda_stream_copy); } -std::unique_ptr IndexHostCopyBase::create_host_copy(const Index& index, - const read_id_t first_read_id, - const std::uint64_t kmer_size, - const std::uint64_t window_size, - const cudaStream_t cuda_stream) +std::unique_ptr IndexHostCopyBase::create_host_copy_async(const Index& index, + const read_id_t first_read_id, + const std::uint64_t kmer_size, + const std::uint64_t window_size, + const cudaStream_t cuda_stream) { GW_NVTX_RANGE(profiler, "cache_D2H"); return std::make_unique(index, diff --git a/cudamapper/src/index_cache.cu b/cudamapper/src/index_cache.cu index 7e75a5213..b128d35a4 100644 --- a/cudamapper/src/index_cache.cu +++ b/cudamapper/src/index_cache.cu @@ -190,15 +190,15 @@ void HostIndexCache::generate_content(const CacheType cache_type, if (!index_on_host) { // create index - index_on_device = Index::create_index(allocator_, - *parser, - descriptor_of_index_to_cache, - kmer_size_, - window_size_, - hash_representations_, - filtering_parameter_, - cuda_stream_generation_, - cuda_stream_copy_); + index_on_device = Index::create_index_async(allocator_, + *parser, + descriptor_of_index_to_cache, + kmer_size_, + window_size_, + hash_representations_, + filtering_parameter_, + cuda_stream_generation_, + cuda_stream_copy_); index_on_device->wait_to_be_ready(); if (host_copy_needed) @@ -210,11 +210,11 @@ void HostIndexCache::generate_content(const CacheType cache_type, index_on_host_copy_in_flight->finish_copying(); } - index_on_host = IndexHostCopy::create_host_copy(*index_on_device, - descriptor_of_index_to_cache.first_read(), - kmer_size_, - window_size_, - cuda_stream_copy_); + index_on_host = IndexHostCopy::create_host_copy_async(*index_on_device, + descriptor_of_index_to_cache.first_read(), + kmer_size_, + window_size_, + cuda_stream_copy_); index_on_host_copy_in_flight = index_on_host; index_on_device_copy_in_flight = index_on_device; diff --git a/cudamapper/tests/Test_CudamapperMatcherGPU.cu b/cudamapper/tests/Test_CudamapperMatcherGPU.cu index 59945897e..58aa7e042 100644 --- a/cudamapper/tests/Test_CudamapperMatcherGPU.cu +++ b/cudamapper/tests/Test_CudamapperMatcherGPU.cu @@ -639,9 +639,9 @@ TEST(TestCudamapperMatcherGPU, OneReadOneMinimizer) { DefaultDeviceAllocator allocator = create_default_device_allocator(); std::unique_ptr parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/gatt.fasta"); - std::unique_ptr query_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); + std::unique_ptr query_index = Index::create_index_async(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); query_index->wait_to_be_ready(); - std::unique_ptr target_index = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); + std::unique_ptr target_index = Index::create_index_async(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); target_index->wait_to_be_ready(); MatcherGPU matcher(allocator, *query_index, *target_index); @@ -654,9 +654,9 @@ TEST(TestCudamapperMatcherGPU, AtLeastOneIndexEmpty) { DefaultDeviceAllocator allocator = create_default_device_allocator(); std::unique_ptr parser = io::create_kseq_fasta_parser(std::string(CUDAMAPPER_BENCHMARK_DATA_DIR) + "/gatt.fasta"); - std::unique_ptr index_full = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); + std::unique_ptr index_full = Index::create_index_async(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 4, 1); index_full->wait_to_be_ready(); - std::unique_ptr index_empty = Index::create_index(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 5, 1); // kmer longer than read + std::unique_ptr index_empty = Index::create_index_async(allocator, *parser, IndexDescriptor(0, parser->get_num_seqences()), 5, 1); // kmer longer than read index_empty->wait_to_be_ready(); { From 1b62f46680e23fed413cc67f7c60a27d1c627945 Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 22 Oct 2020 11:59:18 -0400 Subject: [PATCH 218/281] [cudapoa-unify] merged generatePOAKernel() and generatePOAKernelTB() into a single global kernel, removed the latter kernel. --- cudapoa/src/cudapoa_kernels.cuh | 602 ++++++++++---------------------- 1 file changed, 182 insertions(+), 420 deletions(-) diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 551706c44..6f0ba7839 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -73,8 +73,8 @@ namespace cudapoa * @param[in] mismatch_score Score for finding a mismatch in alignment * @param[in] match_score Score for finding a match in alignment */ -template -__launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_64_REGISTERS) +template +__launch_bounds__(TRACEBACK ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS : GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_64_REGISTERS) __global__ void generatePOAKernel(uint8_t* consensus_d, uint8_t* sequences_d, int8_t* base_weights_d, @@ -110,7 +110,9 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_64_REGISTERS) int32_t scores_matrix_width, int32_t max_limit_consensus_size, int32_t TPB = 64, - int32_t static_band_width = 256) + int32_t static_band_width = 256, + int32_t max_pred_distance = 0, + TraceT* traceback_d = nullptr) { // shared error indicator within a warp bool warp_error = false; @@ -134,378 +136,37 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_64_REGISTERS) uint16_t* node_alignment_count = &node_alignment_count_d[window_idx * max_nodes_per_graph]; uint16_t* sorted_poa_local_edge_count = &sorted_poa_local_edge_count_d[window_idx * max_nodes_per_graph]; - int32_t scores_width = window_details_d[window_idx].scores_width; - ScoreT* scores; float banded_buffer_size; // using float instead of int64_t to minimize register - if (BM == BandMode::adaptive_band || BM == BandMode::static_band) - { - banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); - int64_t scores_offset = static_cast(banded_buffer_size) * static_cast(window_idx); - scores = &scores_d[scores_offset]; - } - else if (BM == BandMode::full_band) - { - int64_t offset = static_cast(window_details_d[window_idx].scores_offset) * static_cast(max_nodes_per_graph); - scores = &scores_d[offset]; - } - - SizeT* alignment_graph = &alignment_graph_d[max_nodes_per_graph * window_idx]; - SizeT* alignment_read = &alignment_read_d[max_nodes_per_graph * window_idx]; - uint16_t* node_coverage_counts = &node_coverage_counts_d_[max_nodes_per_graph * window_idx]; - -#ifdef SPOA_ACCURATE - uint8_t* node_marks = &node_marks_d_[max_nodes_per_graph * window_idx]; - bool* check_aligned_nodes = &check_aligned_nodes_d_[max_nodes_per_graph * window_idx]; - SizeT* nodes_to_visit = &nodes_to_visit_d_[max_nodes_per_graph * window_idx]; -#endif - - SizeT* sequence_lengths = &sequence_lengths_d[window_details_d[window_idx].seq_len_buffer_offset]; - uint16_t num_sequences = window_details_d[window_idx].num_seqs; - uint8_t* sequence = &sequences_d[window_details_d[window_idx].seq_starts]; - int8_t* base_weights = &base_weights_d[window_details_d[window_idx].seq_starts]; - - uint8_t* consensus = &consensus_d[window_idx * max_limit_consensus_size]; - - SizeT* sequence_begin_nodes_ids = nullptr; - uint16_t* outgoing_edges_coverage = nullptr; - uint16_t* outgoing_edges_coverage_count = nullptr; + TraceT* traceback = traceback_d; // only used in traceback + int32_t scores_width = window_details_d[window_idx].scores_width; // only used in non-traceback - if (MSA) - { - sequence_begin_nodes_ids = &sequence_begin_nodes_ids_d[window_idx * max_sequences_per_poa]; - outgoing_edges_coverage = &outgoing_edges_coverage_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * max_sequences_per_poa]; - outgoing_edges_coverage_count = &outgoing_edges_coverage_count_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES]; - } - - if (lane_idx == 0) + if (TRACEBACK) { - // Create backbone for window based on first sequence in window. - nodes[0] = sequence[0]; - sorted_poa[0] = 0; - incoming_edge_count[0] = 0; - node_alignment_count[0] = 0; - node_id_to_pos[0] = 0; - outgoing_edge_count[sequence_lengths[0] - 1] = 0; - incoming_edge_weights[0] = base_weights[0]; - node_coverage_counts[0] = 1; - if (MSA) - { - sequence_begin_nodes_ids[0] = 0; - } - - //Build the rest of the graphs - for (SizeT nucleotide_idx = 1; nucleotide_idx < sequence_lengths[0]; nucleotide_idx++) - { - nodes[nucleotide_idx] = sequence[nucleotide_idx]; - sorted_poa[nucleotide_idx] = nucleotide_idx; - outgoing_edges[(nucleotide_idx - 1) * CUDAPOA_MAX_NODE_EDGES] = nucleotide_idx; - outgoing_edge_count[nucleotide_idx - 1] = 1; - incoming_edges[nucleotide_idx * CUDAPOA_MAX_NODE_EDGES] = nucleotide_idx - 1; - incoming_edge_weights[nucleotide_idx * CUDAPOA_MAX_NODE_EDGES] = base_weights[nucleotide_idx - 1] + base_weights[nucleotide_idx]; - incoming_edge_count[nucleotide_idx] = 1; - node_alignment_count[nucleotide_idx] = 0; - node_id_to_pos[nucleotide_idx] = nucleotide_idx; - node_coverage_counts[nucleotide_idx] = 1; - if (MSA) - { - outgoing_edges_coverage[(nucleotide_idx - 1) * CUDAPOA_MAX_NODE_EDGES * max_sequences_per_poa] = 0; - outgoing_edges_coverage_count[(nucleotide_idx - 1) * CUDAPOA_MAX_NODE_EDGES] = 1; - } - } - - // Clear error code for window. - consensus[0] = CUDAPOA_KERNEL_NOERROR_ENCOUNTERED; + // buffer size for scores, in traceback we only need to store part of the scores matrix + banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); + int64_t offset = static_cast(banded_buffer_size) * static_cast(window_idx); + scores = &scores_d[offset]; + // buffer size for traceback + banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); + offset = static_cast(banded_buffer_size) * static_cast(window_idx); + traceback = &traceback_d[offset]; } - - __syncwarp(); - - // Align each subsequent read, add alignment to graph, run topological sort. - for (int32_t s = 1; s < num_sequences; s++) + else { - int32_t seq_len = sequence_lengths[s]; - // Note: the following 2 lines correspond to num_nucleotides_copied_ value on the host side - // therefore it is important to reflect any changes in the following 2 lines in the corresponding host code as well - sequence += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data - base_weights += cudautils::align(sequence_lengths[s - 1]); // increment the pointer so it is pointing to correct sequence data - - if (lane_idx == 0) - { - if (sequence_lengths[0] >= max_nodes_per_graph) - { - consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; - consensus[1] = static_cast(StatusType::node_count_exceeded_maximum_graph_size); - warp_error = true; - } - } - - warp_error = __shfl_sync(FULL_MASK, warp_error, 0); - if (warp_error) - { - return; - } - - // Run Needleman-Wunsch alignment between graph and new sequence. - SizeT alignment_length; - - // Adaptive band --------------------------------------------------------------------------------------- - if (BM == BandMode::adaptive_band && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) - { - // run in adaptive mode only if static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH - alignment_length = runNeedlemanWunschBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score, - 0); - __syncwarp(); - - if (alignment_length == SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == SHIFT_ADAPTIVE_BAND_TO_RIGHT) - { - // rerun with extended and shifted band-width - alignment_length = runNeedlemanWunschBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score, - alignment_length); - __syncwarp(); - } - } - // Static band --------------------------------------------------------------------------------------- - else if (BM == BandMode::static_band || (BM == BandMode::adaptive_band && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) + if (BM == BandMode::adaptive_band || BM == BandMode::static_band) { - alignment_length = runNeedlemanWunschBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score, - alignment_length); - __syncwarp(); + banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); + int64_t scores_offset = static_cast(banded_buffer_size) * static_cast(window_idx); + scores = &scores_d[scores_offset]; } - // Full band ------------------------------------------------------------------------------------------- else if (BM == BandMode::full_band) { - alignment_length = runNeedlemanWunsch(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - scores_width, - alignment_graph, - alignment_read, - gap_score, - mismatch_score, - match_score); - __syncwarp(); - } - - if (alignment_length == CUDAPOA_KERNEL_NW_BACKTRACKING_LOOP_FAILED) - { - if (lane_idx == 0) - { - consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; - consensus[1] = static_cast(StatusType::loop_count_exceeded_upper_bound); - } - return; - } - else if (alignment_length == CUDAPOA_KERNEL_NW_ADAPTIVE_STORAGE_FAILED) - { - if (lane_idx == 0) - { - consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; - consensus[1] = static_cast(StatusType::exceeded_adaptive_banded_matrix_size); - } - return; - } - - if (lane_idx == 0) - { - - // Add alignment to graph. - SizeT new_node_count; - uint8_t error_code = addAlignmentToGraph(new_node_count, - nodes, sequence_lengths[0], - node_alignments, node_alignment_count, - incoming_edges, incoming_edge_count, - outgoing_edges, outgoing_edge_count, - incoming_edge_weights, - alignment_length, - sorted_poa, alignment_graph, - sequence, alignment_read, - node_coverage_counts, - base_weights, - (sequence_begin_nodes_ids + s), - outgoing_edges_coverage, - outgoing_edges_coverage_count, - s, - max_sequences_per_poa, - max_nodes_per_graph); - - if (error_code != 0) - { - consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; - consensus[1] = error_code; - warp_error = true; - } - else - { - sequence_lengths[0] = new_node_count; - // Run a topsort on the graph. -#ifdef SPOA_ACCURATE - // Exactly matches racon CPU results - raconTopologicalSortDeviceUtil(sorted_poa, - node_id_to_pos, - new_node_count, - incoming_edge_count, - incoming_edges, - node_alignment_count, - node_alignments, - node_marks, - check_aligned_nodes, - nodes_to_visit, - (uint16_t)max_nodes_per_graph); -#else - // Faster top sort - topologicalSortDeviceUtil(sorted_poa, - node_id_to_pos, - new_node_count, - incoming_edge_count, - outgoing_edges, - outgoing_edge_count, - sorted_poa_local_edge_count); -#endif - } - } - - __syncwarp(); - - warp_error = __shfl_sync(FULL_MASK, warp_error, 0); - if (warp_error) - { - return; + int64_t offset = static_cast(window_details_d[window_idx].scores_offset) * static_cast(max_nodes_per_graph); + scores = &scores_d[offset]; } } -} - -// Similar to generatePOAKernel(), but with a different launch bounds setting, used to call kernels using traceback buffer -// i.e. for BM == BandMode::static_band_traceback or BM == BandMode::adaptive_band_traceback -template -__launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS) - __global__ void generatePOAKernelTB(uint8_t* consensus_d, - uint8_t* sequences_d, - int8_t* base_weights_d, - SizeT* sequence_lengths_d, - genomeworks::cudapoa::WindowDetails* window_details_d, - int32_t total_windows, - ScoreT* scores_d, - SizeT* alignment_graph_d, - SizeT* alignment_read_d, - uint8_t* nodes_d, - SizeT* incoming_edges_d, - uint16_t* incoming_edge_count_d, - SizeT* outgoing_edges_d, - uint16_t* outgoing_edge_count_d, - uint16_t* incoming_edge_w_d, - SizeT* sorted_poa_d, - SizeT* node_id_to_pos_d, - SizeT* node_alignments_d, - uint16_t* node_alignment_count_d, - uint16_t* sorted_poa_local_edge_count_d, - uint8_t* node_marks_d_, - bool* check_aligned_nodes_d_, - SizeT* nodes_to_visit_d_, - uint16_t* node_coverage_counts_d_, - int32_t gap_score, - int32_t mismatch_score, - int32_t match_score, - uint32_t max_sequences_per_poa, - SizeT* sequence_begin_nodes_ids_d, - uint16_t* outgoing_edges_coverage_d, - uint16_t* outgoing_edges_coverage_count_d, - int32_t max_nodes_per_graph, - int32_t scores_matrix_width, - int32_t max_limit_consensus_size, - int32_t TPB = 64, - int32_t static_band_width = 256, - int32_t max_pred_distance = 0, - TraceT* traceback_d = nullptr) -{ - // shared error indicator within a warp - bool warp_error = false; - - uint32_t lane_idx = threadIdx.x % WARP_SIZE; - uint32_t window_idx = blockIdx.x * TPB / WARP_SIZE + threadIdx.x / WARP_SIZE; - - if (window_idx >= total_windows) - return; - - // Find the buffer offsets for each thread within the global memory buffers. - uint8_t* nodes = &nodes_d[max_nodes_per_graph * window_idx]; - SizeT* incoming_edges = &incoming_edges_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES]; - uint16_t* incoming_edge_count = &incoming_edge_count_d[window_idx * max_nodes_per_graph]; - SizeT* outgoing_edges = &outgoing_edges_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES]; - uint16_t* outgoing_edge_count = &outgoing_edge_count_d[window_idx * max_nodes_per_graph]; - uint16_t* incoming_edge_weights = &incoming_edge_w_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES]; - SizeT* sorted_poa = &sorted_poa_d[window_idx * max_nodes_per_graph]; - SizeT* node_id_to_pos = &node_id_to_pos_d[window_idx * max_nodes_per_graph]; - SizeT* node_alignments = &node_alignments_d[window_idx * max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS]; - uint16_t* node_alignment_count = &node_alignment_count_d[window_idx * max_nodes_per_graph]; - uint16_t* sorted_poa_local_edge_count = &sorted_poa_local_edge_count_d[window_idx * max_nodes_per_graph]; - - ScoreT* scores; - TraceT* traceback = traceback_d; - float banded_buffer_size; // using float instead of int64_t to minimize register - // buffer size for scores, in traceback we only need to store part of the scores matrix - banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); - int64_t offset = static_cast(banded_buffer_size) * static_cast(window_idx); - scores = &scores_d[offset]; - // buffer size for traceback - banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_matrix_width); - offset = static_cast(banded_buffer_size) * static_cast(window_idx); - traceback = &traceback_d[offset]; SizeT* alignment_graph = &alignment_graph_d[max_nodes_per_graph * window_idx]; SizeT* alignment_read = &alignment_read_d[max_nodes_per_graph * window_idx]; @@ -606,33 +267,11 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS) // Run Needleman-Wunsch alignment between graph and new sequence. SizeT alignment_length; - // Adaptive band with traceback ------------------------------------------------------------------------ - if (BM == BandMode::adaptive_band_traceback && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) + if (TRACEBACK) { - alignment_length = runNeedlemanWunschBandedTraceback(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - traceback, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - max_pred_distance, - gap_score, - mismatch_score, - match_score, - 0); - __syncwarp(); - if (alignment_length == SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == SHIFT_ADAPTIVE_BAND_TO_RIGHT) + // Adaptive band with traceback ------------------------------------------------------------------------ + if (BM == BandMode::adaptive_band_traceback && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) { - // rerun with extended and shifted band-width alignment_length = runNeedlemanWunschBandedTraceback(nodes, sorted_poa, node_id_to_pos, @@ -652,34 +291,154 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS) gap_score, mismatch_score, match_score, - alignment_length); + 0); + __syncwarp(); + if (alignment_length == SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == SHIFT_ADAPTIVE_BAND_TO_RIGHT) + { + // rerun with extended and shifted band-width + alignment_length = runNeedlemanWunschBandedTraceback(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + max_pred_distance, + gap_score, + mismatch_score, + match_score, + alignment_length); + __syncwarp(); + } + } + // Static band with traceback -------------------------------------------------------------------------- + else if (BM == BandMode::static_band_traceback || (BM == BandMode::adaptive_band_traceback && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) + { + alignment_length = runNeedlemanWunschBandedTraceback(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + max_pred_distance, + gap_score, + mismatch_score, + match_score, + alignment_length); __syncwarp(); } } - // Static band with traceback -------------------------------------------------------------------------- - else if (BM == BandMode::static_band_traceback || (BM == BandMode::adaptive_band_traceback && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) + else { - alignment_length = runNeedlemanWunschBandedTraceback(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - traceback, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - max_pred_distance, - gap_score, - mismatch_score, - match_score, - alignment_length); - __syncwarp(); + // Adaptive band --------------------------------------------------------------------------------------- + if (BM == BandMode::adaptive_band && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) + { + // run in adaptive mode only if static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH + alignment_length = runNeedlemanWunschBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + 0); + __syncwarp(); + + if (alignment_length == SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == SHIFT_ADAPTIVE_BAND_TO_RIGHT) + { + // rerun with extended and shifted band-width + alignment_length = runNeedlemanWunschBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + alignment_length); + __syncwarp(); + } + } + // Static band --------------------------------------------------------------------------------------- + else if (BM == BandMode::static_band || (BM == BandMode::adaptive_band && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) + { + alignment_length = runNeedlemanWunschBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + alignment_length); + __syncwarp(); + } + // Full band ------------------------------------------------------------------------------------------- + else if (BM == BandMode::full_band) + { + alignment_length = runNeedlemanWunsch(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + scores_width, + alignment_graph, + alignment_read, + gap_score, + mismatch_score, + match_score); + __syncwarp(); + } } if (alignment_length == CUDAPOA_KERNEL_NW_BACKTRACKING_LOOP_FAILED) @@ -700,14 +459,17 @@ __launch_bounds__(GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS) } return; } - else if (alignment_length == CUDAPOA_KERNEL_NW_TRACEBACK_BUFFER_FAILED) + if (TRACEBACK) { - if (lane_idx == 0) + if (alignment_length == CUDAPOA_KERNEL_NW_TRACEBACK_BUFFER_FAILED) { - consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; - consensus[1] = static_cast(StatusType::exceeded_maximum_predecessor_distance); + if (lane_idx == 0) + { + consensus[0] = CUDAPOA_KERNEL_ERROR_ENCOUNTERED; + consensus[1] = static_cast(StatusType::exceeded_maximum_predecessor_distance); + } + return; } - return; } if (lane_idx == 0) @@ -928,7 +690,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, } else if (batch_size.band_mode == BandMode::static_band_traceback) { - generatePOAKernelTB + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -970,7 +732,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, } else if (batch_size.band_mode == BandMode::adaptive_band_traceback) { - generatePOAKernelTB + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -1134,7 +896,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, } else if (batch_size.band_mode == BandMode::static_band_traceback) { - generatePOAKernelTB + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, @@ -1176,7 +938,7 @@ void generatePOA(genomeworks::cudapoa::OutputDetails* output_details_d, } else if (batch_size.band_mode == BandMode::adaptive_band_traceback) { - generatePOAKernelTB + generatePOAKernel <<>>(consensus_d, sequences_d, base_weights_d, From 3775aefc2e0461fbf71de29ad27d1dba23cddca9 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Mon, 26 Oct 2020 13:55:13 -0400 Subject: [PATCH 219/281] [logging] fix spdlog incompatibility with gcc9 1. WAR fix for spdlog with gcc9. disable logging for gcc9 2. enable SAM support by default and fix cmake compile option for SAM --- CMakeLists.txt | 2 +- .../genomeworks/logging/logging.hpp | 14 ++++++++++++++ cudamapper/CMakeLists.txt | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0438319ce..414c78dfd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ option(gw_enable_caching_allocator "Enable caching allocator." ON) option(gw_generate_docs "Generate Doxygen documentation" ON) option(gw_cuda_gen_all_arch "ON: Generate optimized CUDA code for all architectures | OFF: for detected architectures only" OFF) # Optionally build htslib for SAM/BAM support. Requires autoconf to be installed -option(gw_build_htslib "Build 3rdparty htslib that allows output in SAM/BAM format" OFF) +option(gw_build_htslib "Build 3rdparty htslib that allows output in SAM/BAM format" ON) # Must be included before others for options value validation include(cmake/Utils.cmake) diff --git a/common/base/include/claraparabricks/genomeworks/logging/logging.hpp b/common/base/include/claraparabricks/genomeworks/logging/logging.hpp index d4e2ac777..822987894 100644 --- a/common/base/include/claraparabricks/genomeworks/logging/logging.hpp +++ b/common/base/include/claraparabricks/genomeworks/logging/logging.hpp @@ -94,7 +94,11 @@ #ifdef GW_CUDA_BEFORE_9_2 // Due to a header file incompatibility with nvcc in CUDA 9.0 // logging through the logger class in GW is disabled for any .cu files. +#elif __GNUC__ >= 9 #pragma message("Logging disabled for CUDA Toolkit < 9.2") +// Due to a ISO C++ standard incompatibility the spdlog fails to pass +// pedantic requirements. +#pragma message("Logging disabled for GCC >= 9") #else #include #endif @@ -137,6 +141,8 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_DEBUG(...) +#elif __GNUC__ >= 9 +#define GW_LOG_DEBUG(...) #else #define GW_LOG_DEBUG(...) SPDLOG_DEBUG(__VA_ARGS__) #endif @@ -148,6 +154,8 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_INFO(...) +#elif __GNUC__ >= 9 +#define GW_LOG_INFO(...) #else #define GW_LOG_INFO(...) SPDLOG_INFO(__VA_ARGS__) #endif @@ -159,6 +167,8 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_WARN(...) +#elif __GNUC__ >= 9 +#define GW_LOG_WARN(...) #else #define GW_LOG_WARN(...) SPDLOG_WARN(__VA_ARGS__) #endif @@ -170,6 +180,8 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_ERROR(...) +#elif __GNUC__ >= 9 +#define GW_LOG_ERROR(...) #else #define GW_LOG_ERROR(...) SPDLOG_ERROR(__VA_ARGS__) #endif @@ -181,6 +193,8 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_CRITICAL(...) +#elif __GNUC__ >= 9 +#define GW_LOG_CRITICAL(...) #else #define GW_LOG_CRITICAL(...) SPDLOG_CRITICAL(__VA_ARGS__) #endif diff --git a/cudamapper/CMakeLists.txt b/cudamapper/CMakeLists.txt index 5b6d3be99..86f584fa1 100644 --- a/cudamapper/CMakeLists.txt +++ b/cudamapper/CMakeLists.txt @@ -70,7 +70,7 @@ target_include_directories(${MODULE_NAME} # optionally link against htslib if (TARGET htslib_project) - add_compile_definitions(GW_BUILD_HTSLIB) + target_compile_definitions(${MODULE_NAME} PUBLIC GW_BUILD_HTSLIB) add_dependencies(${MODULE_NAME} htslib_project) target_link_libraries(${MODULE_NAME} htslib gwbase gwio cub) else() From c0898075e2595bd3157e8a2d9f25c3741fd8e108 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Mon, 26 Oct 2020 14:01:43 -0400 Subject: [PATCH 220/281] [logging] fix formatting --- .../claraparabricks/genomeworks/logging/logging.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/logging/logging.hpp b/common/base/include/claraparabricks/genomeworks/logging/logging.hpp index 822987894..bb7505294 100644 --- a/common/base/include/claraparabricks/genomeworks/logging/logging.hpp +++ b/common/base/include/claraparabricks/genomeworks/logging/logging.hpp @@ -94,7 +94,7 @@ #ifdef GW_CUDA_BEFORE_9_2 // Due to a header file incompatibility with nvcc in CUDA 9.0 // logging through the logger class in GW is disabled for any .cu files. -#elif __GNUC__ >= 9 +#elif __GNUC__ >= 9 #pragma message("Logging disabled for CUDA Toolkit < 9.2") // Due to a ISO C++ standard incompatibility the spdlog fails to pass // pedantic requirements. @@ -141,7 +141,7 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_DEBUG(...) -#elif __GNUC__ >= 9 +#elif __GNUC__ >= 9 #define GW_LOG_DEBUG(...) #else #define GW_LOG_DEBUG(...) SPDLOG_DEBUG(__VA_ARGS__) @@ -154,7 +154,7 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_INFO(...) -#elif __GNUC__ >= 9 +#elif __GNUC__ >= 9 #define GW_LOG_INFO(...) #else #define GW_LOG_INFO(...) SPDLOG_INFO(__VA_ARGS__) @@ -167,7 +167,7 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_WARN(...) -#elif __GNUC__ >= 9 +#elif __GNUC__ >= 9 #define GW_LOG_WARN(...) #else #define GW_LOG_WARN(...) SPDLOG_WARN(__VA_ARGS__) @@ -180,7 +180,7 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_ERROR(...) -#elif __GNUC__ >= 9 +#elif __GNUC__ >= 9 #define GW_LOG_ERROR(...) #else #define GW_LOG_ERROR(...) SPDLOG_ERROR(__VA_ARGS__) @@ -193,7 +193,7 @@ LoggingStatus SetHeader(bool logTime, bool logLocation); /// parameters as per https://github.com/gabime/spdlog/blob/v1.x/README.md #ifdef GW_CUDA_BEFORE_9_2 #define GW_LOG_CRITICAL(...) -#elif __GNUC__ >= 9 +#elif __GNUC__ >= 9 #define GW_LOG_CRITICAL(...) #else #define GW_LOG_CRITICAL(...) SPDLOG_CRITICAL(__VA_ARGS__) From ddd9be5599560a13c1ae91946f7fef9bb1ed25a0 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Mon, 26 Oct 2020 16:05:41 -0400 Subject: [PATCH 221/281] fix comments from review 1. update cmake calls for sam generation 2. fix ifdef statement ordering --- .../include/claraparabricks/genomeworks/logging/logging.hpp | 2 +- cudamapper/CMakeLists.txt | 3 +++ cudamapper/samples/CMakeLists.txt | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/common/base/include/claraparabricks/genomeworks/logging/logging.hpp b/common/base/include/claraparabricks/genomeworks/logging/logging.hpp index bb7505294..962ca2325 100644 --- a/common/base/include/claraparabricks/genomeworks/logging/logging.hpp +++ b/common/base/include/claraparabricks/genomeworks/logging/logging.hpp @@ -94,8 +94,8 @@ #ifdef GW_CUDA_BEFORE_9_2 // Due to a header file incompatibility with nvcc in CUDA 9.0 // logging through the logger class in GW is disabled for any .cu files. -#elif __GNUC__ >= 9 #pragma message("Logging disabled for CUDA Toolkit < 9.2") +#elif __GNUC__ >= 9 // Due to a ISO C++ standard incompatibility the spdlog fails to pass // pedantic requirements. #pragma message("Logging disabled for GCC >= 9") diff --git a/cudamapper/CMakeLists.txt b/cudamapper/CMakeLists.txt index 86f584fa1..87e9bcac2 100644 --- a/cudamapper/CMakeLists.txt +++ b/cudamapper/CMakeLists.txt @@ -87,6 +87,9 @@ cuda_add_executable(${MODULE_NAME}-bin target_compile_options(${MODULE_NAME}-bin PRIVATE -Werror) target_link_libraries(${MODULE_NAME}-bin ${MODULE_NAME} cudaaligner) set_target_properties(${MODULE_NAME}-bin PROPERTIES OUTPUT_NAME ${MODULE_NAME}) +if (TARGET htslib_project) + target_compile_definitions(${MODULE_NAME}-bin PUBLIC GW_BUILD_HTSLIB) +endif() # Add tests folder diff --git a/cudamapper/samples/CMakeLists.txt b/cudamapper/samples/CMakeLists.txt index 503cbfb28..ca0684314 100644 --- a/cudamapper/samples/CMakeLists.txt +++ b/cudamapper/samples/CMakeLists.txt @@ -27,5 +27,11 @@ target_link_libraries(${MODULE_NAME} cudamapper ) +if (TARGET htslib_project) + target_compile_definitions(${MODULE_NAME} + PUBLIC + GW_BUILD_HTSLIB) +endif() + install(TARGETS ${MODULE_NAME} DESTINATION samples) From d76fa68ac0ac875d03016a4e9a00fbbdc9dd8387 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Fri, 30 Oct 2020 12:53:38 -0400 Subject: [PATCH 222/281] [samples] fix sample compilation for gcc9 --- cudaaligner/samples/sample_cudaaligner.cpp | 1 + cudamapper/samples/sample_cudamapper.cpp | 1 + cudapoa/samples/sample_cudapoa.cpp | 1 + 3 files changed, 3 insertions(+) diff --git a/cudaaligner/samples/sample_cudaaligner.cpp b/cudaaligner/samples/sample_cudaaligner.cpp index db147faab..fb147876e 100644 --- a/cudaaligner/samples/sample_cudaaligner.cpp +++ b/cudaaligner/samples/sample_cudaaligner.cpp @@ -26,6 +26,7 @@ #include #include #include +#include using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::genomeutils; diff --git a/cudamapper/samples/sample_cudamapper.cpp b/cudamapper/samples/sample_cudamapper.cpp index cfe2d59f9..142439d4b 100644 --- a/cudamapper/samples/sample_cudamapper.cpp +++ b/cudamapper/samples/sample_cudamapper.cpp @@ -29,6 +29,7 @@ #include #include #include +#include // define constants. See cudamapper/src/application_parameters.hpp for more. // constants used in multiple places diff --git a/cudapoa/samples/sample_cudapoa.cpp b/cudapoa/samples/sample_cudapoa.cpp index 3d9eeac87..4f370ad33 100644 --- a/cudapoa/samples/sample_cudapoa.cpp +++ b/cudapoa/samples/sample_cudapoa.cpp @@ -25,6 +25,7 @@ #include #include #include +#include using namespace claraparabricks::genomeworks; using namespace claraparabricks::genomeworks::cudapoa; From 15eccfb1bf4146a72ab8000ae8e626a641cd7ddf Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Fri, 30 Oct 2020 17:18:18 -0400 Subject: [PATCH 223/281] [sample] add missing header to cudaextender sample --- cudaextender/samples/sample_cudaextender.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cudaextender/samples/sample_cudaextender.cpp b/cudaextender/samples/sample_cudaextender.cpp index 87ad85793..53d4bd8e4 100644 --- a/cudaextender/samples/sample_cudaextender.cpp +++ b/cudaextender/samples/sample_cudaextender.cpp @@ -26,6 +26,7 @@ #include #include #include +#include using namespace claraparabricks::genomeworks; using namespace cudautils; From 5b04b8079bb7bb2934ff53aacf6e50f65caf1193 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Wed, 4 Nov 2020 12:20:51 +0200 Subject: [PATCH 224/281] Fix failing tests for CUDA 9.0 --- ci/common/prep-init-env.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ci/common/prep-init-env.sh b/ci/common/prep-init-env.sh index 378457abe..fcd06803c 100644 --- a/ci/common/prep-init-env.sh +++ b/ci/common/prep-init-env.sh @@ -53,8 +53,19 @@ python --version logger "Conda install GenomeWorks custom packages - clang-format" conda install --override-channels -c sarcasm clang-format -logger "Conda install GenomeWorks custom packages - doxygen ninja cmake" -conda install --override-channels -c conda-forge doxygen ninja cmake">=3.10.2" +logger "Conda install GenomeWorks custom packages - doxygen ninja" +conda install --override-channels -c conda-forge doxygen ninja + +# Building cudamapper, using CUDA_SELECT_NVCC_ARCH_FLAGS() with the 'common' argument, generates +# the "-gencode;arch=compute_72,code=sm_72" arch flag which is incompatible with CUDA 9.0 and causes nvcc command to fail +# Using cmake with a version older than 3.18 does not create 'compute_72' arch. +if [ "$(echo ${CUDA_VERSION} | cut -d"." -f1-2)" = "9.0" ]; then + logger "Conda install cmake for CUDA 9.0" + conda install --override-channels -c conda-forge cmake"=3.17" +else + logger "Conda install cmake" + conda install --override-channels -c conda-forge cmake">=3.10.2" +fi logger "Update LD_LIBRARY_PATH" export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib From 9d2e89f244f194eaf97c613613b848e319949a00 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Wed, 4 Nov 2020 13:03:35 +0200 Subject: [PATCH 225/281] Change conda environment name from 'gdf' to 'parabricks' --- ci/cpu/prebuild.sh | 2 +- ci/gpu/prebuild.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh index 912845ef9..bd26f5f00 100755 --- a/ci/cpu/prebuild.sh +++ b/ci/cpu/prebuild.sh @@ -22,4 +22,4 @@ # Note we still _BUILD_ for GPU, we just don't (can't) test on it export BUILD_FOR_GPU=1 export TEST_ON_GPU=0 -export CONDA_ENV_NAME="gdf" +export CONDA_ENV_NAME="parabricks" diff --git a/ci/gpu/prebuild.sh b/ci/gpu/prebuild.sh index 9ce52e560..606a33e3b 100644 --- a/ci/gpu/prebuild.sh +++ b/ci/gpu/prebuild.sh @@ -21,4 +21,4 @@ export BUILD_FOR_GPU=1 export TEST_ON_GPU=1 -export CONDA_ENV_NAME="gdf" +export CONDA_ENV_NAME="parabricks" From f6e5cf114569efad0de5e973db6670a544ac18ba Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 4 Nov 2020 09:06:22 -0500 Subject: [PATCH 226/281] [cudapoa] renaming some macros --- cudapoa/src/batch.cu | 12 ++++++------ cudapoa/src/cudapoa_batch.cuh | 4 ++-- cudapoa/src/cudapoa_kernels.cuh | 4 ++-- cudapoa/src/cudapoa_nw_banded.cuh | 20 ++++++++++---------- cudapoa/src/cudapoa_nw_tb_banded.cuh | 20 ++++++++++---------- cudapoa/src/cudapoa_structs.cuh | 10 +++++----- 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index 5b577e064..e66bc7cf4 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -43,20 +43,20 @@ BatchConfig::BatchConfig(int32_t max_seq_sz /*= 1024*/, int32_t max_seq_per_poa , band_mode(banding) , max_banded_pred_distance(max_pred_dist > 0 ? max_pred_dist : 2 * cudautils::align(band_width)) { - max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); + max_nodes_per_graph = cudautils::align(graph_length_factor * max_sequence_size); if (banding == BandMode::full_band) { - matrix_sequence_dimension = cudautils::align(max_sequence_size); + matrix_sequence_dimension = cudautils::align(max_sequence_size); } else if (banding == BandMode::static_band || banding == BandMode::static_band_traceback) { - matrix_sequence_dimension = cudautils::align(alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); + matrix_sequence_dimension = cudautils::align(alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING); } else // BandMode::adaptive_band || BandMode::adaptive_band_traceback { // adapive_storage_factor is to reserve extra memory for cases with extended band-width - matrix_sequence_dimension = cudautils::align(adapive_storage_factor * (alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING)); + matrix_sequence_dimension = cudautils::align(adapive_storage_factor * (alignment_band_width + CUDAPOA_BANDED_MATRIX_RIGHT_PADDING)); } throw_on_negative(max_seq_sz, "max_sequence_size cannot be negative."); @@ -76,8 +76,8 @@ BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t m /// ensure a 4-byte boundary alignment for any allocated buffer : max_sequence_size(max_seq_sz) , max_consensus_size(max_consensus_sz) - , max_nodes_per_graph(cudautils::align(max_nodes_per_poa)) - , matrix_sequence_dimension(cudautils::align(matrix_seq_dim)) + , max_nodes_per_graph(cudautils::align(max_nodes_per_poa)) + , matrix_sequence_dimension(cudautils::align(matrix_seq_dim)) /// ensure 128-alignment for band_width size , alignment_band_width(cudautils::align(band_width)) , max_sequences_per_poa(max_seq_per_poa) diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 8adca6927..1a9d7ac2f 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -504,7 +504,7 @@ protected: } WindowDetails* window_details = &(input_details_h_->window_details[poa_count_ - 1]); - int32_t scores_width_ = cudautils::align(seq_len + 1 + CELLS_PER_THREAD); + int32_t scores_width_ = cudautils::align(seq_len + 1 + CUDAPOA_CELLS_PER_THREAD); if (scores_width_ > window_details->scores_width) { next_scores_offset_ += (scores_width_ - window_details->scores_width); @@ -557,7 +557,7 @@ protected: int32_t matrix_height = batch_size_.max_nodes_per_graph; // matrix width for full_band is based on the current group max_seq_length as opposed to batch_size_.matrix_sequence_dimension. // The latter is based on the largest group in the batch and is more conservative - int32_t matrix_width = (batch_size_.band_mode != BandMode::full_band) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CELLS_PER_THREAD); + int32_t matrix_width = (batch_size_.band_mode != BandMode::full_band) ? batch_size_.matrix_sequence_dimension : cudautils::align(max_seq_length + 1 + CUDAPOA_CELLS_PER_THREAD); // in traceback alignments avail_buf_mem_ is dedicated to traceback matrix, otherwise it is being used for score matrix size_t required_size = static_cast(matrix_width) * static_cast(matrix_height); required_size *= (batch_size_.band_mode == static_band_traceback || batch_size_.band_mode == adaptive_band_traceback) ? sizeof(TraceT) : sizeof(ScoreT); diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index 6f0ba7839..e42a3f95d 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -293,7 +293,7 @@ __launch_bounds__(TRACEBACK ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS match_score, 0); __syncwarp(); - if (alignment_length == SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == SHIFT_ADAPTIVE_BAND_TO_RIGHT) + if (alignment_length == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_RIGHT) { // rerun with extended and shifted band-width alignment_length = runNeedlemanWunschBandedTraceback(nodes, @@ -371,7 +371,7 @@ __launch_bounds__(TRACEBACK ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS 0); __syncwarp(); - if (alignment_length == SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == SHIFT_ADAPTIVE_BAND_TO_RIGHT) + if (alignment_length == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_RIGHT) { // rerun with extended and shifted band-width alignment_length = runNeedlemanWunschBanded(nodes, diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index d81490453..1b4218277 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -70,9 +70,9 @@ __device__ __forceinline__ int32_t get_band_start_for_row(int32_t row, float gra int32_t start_pos = max(0, diagonal_index - band_shift); if (max_column < start_pos + band_width) { - start_pos = max(0, max_column - band_width + CELLS_PER_THREAD); + start_pos = max(0, max_column - band_width + CUDAPOA_CELLS_PER_THREAD); } - start_pos = start_pos - (start_pos % CELLS_PER_THREAD); + start_pos = start_pos - (start_pos % CUDAPOA_CELLS_PER_THREAD); return start_pos; } @@ -123,7 +123,7 @@ __device__ __forceinline__ ScoreT4 get_scores(ScoreT* scores, int32_t band_start = get_band_start_for_row(row, gradient, band_width, band_shift, max_column); // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds - int32_t band_end = band_start + band_width - CELLS_PER_THREAD; + int32_t band_end = band_start + band_width - CUDAPOA_CELLS_PER_THREAD; band_end = min(band_end, max_column); if ((column > band_end || column < band_start) && column != -1) @@ -243,13 +243,13 @@ __device__ __forceinline__ // SHIFT_ADAPTIVE_BAND_TO_RIGHT means traceback path was too close to the right bound of band // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift further to // the left for rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT, and further to the right for rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT. - if (rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) + if (rerun == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_LEFT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) { // ad-hoc rule 5 band_width *= 2; band_shift *= 2.5; } - if (rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) + if (rerun == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_RIGHT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) { // ad-hoc rule 6 band_width *= 2; @@ -305,7 +305,7 @@ __device__ __forceinline__ else { pred_idx = node_id_to_pos[pred_node_id] + 1; - if (band_start > CELLS_PER_THREAD && pred_count == 1) + if (band_start > CUDAPOA_CELLS_PER_THREAD && pred_count == 1) { first_element_prev_score = min_score_value + gap_score; } @@ -330,10 +330,10 @@ __device__ __forceinline__ SeqT graph_base = nodes[node_id]; - for (int32_t read_pos = lane_idx * CELLS_PER_THREAD + band_start; read_pos < band_start + band_width; read_pos += WARP_SIZE * CELLS_PER_THREAD) + for (int32_t read_pos = lane_idx * CUDAPOA_CELLS_PER_THREAD + band_start; read_pos < band_start + band_width; read_pos += WARP_SIZE * CUDAPOA_CELLS_PER_THREAD) { SeqT4* d_read4 = (SeqT4*)read; - SeqT4 read4 = d_read4[read_pos / CELLS_PER_THREAD]; + SeqT4 read4 = d_read4[read_pos / CUDAPOA_CELLS_PER_THREAD]; ScoreT4 char_profile; char_profile.s0 = (graph_base == read4.r0 ? match_score : mismatch_score); @@ -452,12 +452,12 @@ __device__ __forceinline__ int32_t band_start = get_band_start_for_row(i, gradient, band_width, band_shift, max_column); if (j <= band_start + threshold) // ad-hoc rule 8-a, too close to left bound { - aligned_nodes = SHIFT_ADAPTIVE_BAND_TO_LEFT; + aligned_nodes = CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_LEFT; break; } if (j >= (band_start + band_width - threshold)) // ad-hoc rule 8-b, too close to right bound { - aligned_nodes = SHIFT_ADAPTIVE_BAND_TO_RIGHT; + aligned_nodes = CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_RIGHT; break; } } diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index e672c151b..120487163 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -74,9 +74,9 @@ __device__ __forceinline__ int32_t get_band_start_for_row_tb(int32_t row, float int32_t start_pos = max(0, diagonal_index - band_shift); if (max_column < start_pos + band_width) { - start_pos = max(0, max_column - band_width + CELLS_PER_THREAD); + start_pos = max(0, max_column - band_width + CUDAPOA_CELLS_PER_THREAD); } - start_pos = start_pos - (start_pos % CELLS_PER_THREAD); + start_pos = start_pos - (start_pos % CUDAPOA_CELLS_PER_THREAD); return start_pos; } @@ -166,7 +166,7 @@ __device__ __forceinline__ void get_scores_tb(ScoreT* scores, int32_t band_start = get_band_start_for_row_tb(pred_node, gradient, band_width, band_shift, max_column); // subtract by CELLS_PER_THREAD to ensure score4_next is not pointing out of the corresponding band bounds - int32_t band_end = band_start + band_width - CELLS_PER_THREAD; + int32_t band_end = band_start + band_width - CUDAPOA_CELLS_PER_THREAD; band_end = min(band_end, max_column); if ((column > band_end || column < band_start) && column != -1) @@ -310,13 +310,13 @@ __device__ __forceinline__ // SHIFT_ADAPTIVE_BAND_TO_RIGHT means traceback path was too close to the right bound of band // Therefore we rerun alignment of the same read, but this time with double band-width and band_shift further to // the left for rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT, and further to the right for rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT. - if (rerun == SHIFT_ADAPTIVE_BAND_TO_LEFT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) + if (rerun == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_LEFT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) { // ad-hoc rule 5 band_width *= 2; band_shift *= 2.5; } - if (rerun == SHIFT_ADAPTIVE_BAND_TO_RIGHT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) + if (rerun == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_RIGHT && band_width <= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH / 2) { // ad-hoc rule 6 band_width *= 2; @@ -384,7 +384,7 @@ __device__ __forceinline__ // fill in first column of traceback buffer traceback[index] = -(score_gIdx - pred_idx); - if (band_start > CELLS_PER_THREAD && pred_count == 1) + if (band_start > CUDAPOA_CELLS_PER_THREAD && pred_count == 1) { first_element_prev_score = min_score_value + gap_score; } @@ -445,10 +445,10 @@ __device__ __forceinline__ SeqT graph_base = nodes[node_id]; - for (int32_t read_pos = lane_idx * CELLS_PER_THREAD + band_start; read_pos < band_start + band_width; read_pos += CUDAPOA_MIN_BAND_WIDTH) + for (int32_t read_pos = lane_idx * CUDAPOA_CELLS_PER_THREAD + band_start; read_pos < band_start + band_width; read_pos += CUDAPOA_MIN_BAND_WIDTH) { SeqT4* d_read4 = (SeqT4*)read; - SeqT4 read4 = d_read4[read_pos / CELLS_PER_THREAD]; + SeqT4 read4 = d_read4[read_pos / CUDAPOA_CELLS_PER_THREAD]; TraceT4 trace; ScoreT4 score = {min_score_value, min_score_value, min_score_value, min_score_value}; @@ -613,12 +613,12 @@ __device__ __forceinline__ int32_t band_start = get_band_start_for_row(i, gradient, band_width, band_shift, max_column); if (j <= band_start + threshold) // ad-hoc rule 8-a, too close to left bound { - aligned_nodes = SHIFT_ADAPTIVE_BAND_TO_LEFT; + aligned_nodes = CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_LEFT; break; } if (j >= (band_start + band_width - threshold)) // ad-hoc rule 8-b, too close to right bound { - aligned_nodes = SHIFT_ADAPTIVE_BAND_TO_RIGHT; + aligned_nodes = CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_RIGHT; break; } } diff --git a/cudapoa/src/cudapoa_structs.cuh b/cudapoa/src/cudapoa_structs.cuh index 78d85fd7e..9a663de0d 100644 --- a/cudapoa/src/cudapoa_structs.cuh +++ b/cudapoa/src/cudapoa_structs.cuh @@ -31,15 +31,15 @@ // Dimensions for Banded alignment score matrix #define WARP_SIZE 32 -#define CELLS_PER_THREAD 4 -#define CUDAPOA_MIN_BAND_WIDTH (CELLS_PER_THREAD * WARP_SIZE) -#define CUDAPOA_BANDED_MATRIX_RIGHT_PADDING (CELLS_PER_THREAD * 2) +#define CUDAPOA_CELLS_PER_THREAD 4 +#define CUDAPOA_MIN_BAND_WIDTH (CUDAPOA_CELLS_PER_THREAD * WARP_SIZE) +#define CUDAPOA_BANDED_MATRIX_RIGHT_PADDING (CUDAPOA_CELLS_PER_THREAD * 2) // ad-hoc maximum band-width defined in adaptive banding #define CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH 1536 // rerun codes used in adaptive banding -#define SHIFT_ADAPTIVE_BAND_TO_LEFT -10 -#define SHIFT_ADAPTIVE_BAND_TO_RIGHT -11 +#define CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_LEFT -10 +#define CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_RIGHT -11 #define CUDAPOA_THREADS_PER_BLOCK 64 #define CUDAPOA_BANDED_THREADS_PER_BLOCK WARP_SIZE From d800c2555d05fd080b635c82d529bd6cb53a7f2a Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Fri, 6 Nov 2020 12:28:27 -0800 Subject: [PATCH 227/281] typo in cmake for cudaaligner optimization --- cudaaligner/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudaaligner/CMakeLists.txt b/cudaaligner/CMakeLists.txt index 6500a0fad..d2a2f5712 100644 --- a/cudaaligner/CMakeLists.txt +++ b/cudaaligner/CMakeLists.txt @@ -53,7 +53,7 @@ target_link_libraries(${MODULE_NAME} gwbase cub) target_compile_options(${MODULE_NAME} PRIVATE -Werror -Wall -Wextra) if (gw_optimize_for_native_cpu) - target_compile_options(cudapoa PRIVATE -march=native) + target_compile_options(${MODULE_NAME} PRIVATE -march=native) endif() target_include_directories(${MODULE_NAME} From 58515a0bb939e6b7c04607735f61731585fdca3c Mon Sep 17 00:00:00 2001 From: atadkase Date: Wed, 11 Nov 2020 13:03:42 -0800 Subject: [PATCH 228/281] [cudapoa] Add default device allocator --- .../genomeworks/cudapoa/batch.hpp | 7 +- cudapoa/src/allocate_block.hpp | 90 ++++++++++--------- cudapoa/src/batch.cu | 21 +++-- cudapoa/src/cudapoa_batch.cuh | 8 +- 4 files changed, 71 insertions(+), 55 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp index 4da84cceb..82ff0b7cf 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,7 @@ #include #include + namespace claraparabricks { @@ -170,7 +172,7 @@ class Batch /// \param gap_score score to be assigned to a gap /// \param mismatch_score score to be assigned to a mismatch /// \param match_score score to be assigned for a match -/// +/// \param allocator allocator to be used /// \return Returns a unique pointer to a new Batch object std::unique_ptr create_batch(int32_t device_id, cudaStream_t stream, @@ -179,7 +181,8 @@ std::unique_ptr create_batch(int32_t device_id, const BatchConfig& batch_size, int16_t gap_score, int16_t mismatch_score, - int16_t match_score); + int16_t match_score, + DefaultDeviceAllocator allocator); /// \} diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 1d6a42d54..86aa9409f 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #ifndef GW_LOG_LEVEL #ifndef NDEBUG @@ -54,7 +55,7 @@ template class BatchBlock { public: - BatchBlock(int32_t device_id, size_t avail_mem, int8_t output_mask, const BatchConfig& batch_size) + BatchBlock(int32_t device_id, size_t avail_mem, int8_t output_mask, const BatchConfig& batch_size, DefaultDeviceAllocator allocator) : max_sequences_per_poa_(throw_on_negative(batch_size.max_sequences_per_poa, "Maximum sequences per POA has to be non-negative")) , device_id_(throw_on_negative(device_id, "Device ID has to be non-negative")) , output_mask_(output_mask) @@ -104,12 +105,11 @@ class BatchBlock // Allocate. GW_CU_CHECK_ERR(cudaHostAlloc((void**)&block_data_h_, total_h_, cudaHostAllocDefault)); - GW_CU_CHECK_ERR(cudaMalloc((void**)&block_data_d_, total_d_)); + block_data_d_ = device_buffer(total_d_, allocator); } ~BatchBlock() { - GW_CU_CHECK_ERR(cudaFree(block_data_d_)); GW_CU_CHECK_ERR(cudaFreeHost(block_data_h_)); } @@ -117,6 +117,7 @@ class BatchBlock { OutputDetails* output_details_h{}; OutputDetails* output_details_d{}; + uint8_t * block_data_d = block_data_d_.data(); // on host output_details_h = reinterpret_cast(&block_data_h_[offset_h_]); @@ -138,16 +139,16 @@ class BatchBlock offset_h_ += sizeof(OutputDetails); // on device - output_details_d->consensus = &block_data_d_[offset_d_]; + output_details_d->consensus = &block_data_d[offset_d_]; offset_d_ += cudautils::align(output_size_ * sizeof(*output_details_d->consensus)); if (output_mask_ & OutputType::consensus) { - output_details_d->coverage = reinterpret_castcoverage)>(&block_data_d_[offset_d_]); + output_details_d->coverage = reinterpret_castcoverage)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(output_size_ * sizeof(*output_details_d->coverage)); } if (output_mask_ & OutputType::msa) { - output_details_d->multiple_sequence_alignments = reinterpret_castmultiple_sequence_alignments)>(&block_data_d_[offset_d_]); + output_details_d->multiple_sequence_alignments = reinterpret_castmultiple_sequence_alignments)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(output_size_ * max_sequences_per_poa_ * sizeof(*output_details_d->multiple_sequence_alignments)); } @@ -160,6 +161,7 @@ class BatchBlock // on host InputDetails* input_details_h{}; InputDetails* input_details_d{}; + uint8_t * block_data_d = block_data_d_.data(); input_details_h = reinterpret_cast*>(&block_data_h_[offset_h_]); offset_h_ += sizeof(InputDetails); @@ -181,17 +183,17 @@ class BatchBlock offset_h_ += sizeof(InputDetails); // on device - input_details_d->sequences = &block_data_d_[offset_d_]; + input_details_d->sequences = &block_data_d[offset_d_]; offset_d_ += cudautils::align(input_size_ * sizeof(*input_details_d->sequences)); - input_details_d->base_weights = reinterpret_castbase_weights)>(&block_data_d_[offset_d_]); + input_details_d->base_weights = reinterpret_castbase_weights)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(input_size_ * sizeof(*input_details_d->base_weights)); - input_details_d->sequence_lengths = reinterpret_castsequence_lengths)>(&block_data_d_[offset_d_]); + input_details_d->sequence_lengths = reinterpret_castsequence_lengths)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(max_poas_ * max_sequences_per_poa_ * sizeof(*input_details_d->sequence_lengths)); - input_details_d->window_details = reinterpret_castwindow_details)>(&block_data_d_[offset_d_]); + input_details_d->window_details = reinterpret_castwindow_details)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(max_poas_ * sizeof(*input_details_d->window_details)); if (output_mask_ & OutputType::msa) { - input_details_d->sequence_begin_nodes_ids = reinterpret_castsequence_begin_nodes_ids)>(&block_data_d_[offset_d_]); + input_details_d->sequence_begin_nodes_ids = reinterpret_castsequence_begin_nodes_ids)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(max_poas_ * max_sequences_per_poa_ * sizeof(*input_details_d->sequence_begin_nodes_ids)); } @@ -202,32 +204,32 @@ class BatchBlock void get_alignment_details(AlignmentDetails** alignment_details_d_p) { AlignmentDetails* alignment_details_d{}; - + uint8_t * block_data_d = block_data_d_.data(); // on host alignment_details_d = reinterpret_cast*>(&block_data_h_[offset_h_]); offset_h_ += sizeof(AlignmentDetails); // on device; - alignment_details_d->alignment_graph = reinterpret_castalignment_graph)>(&block_data_d_[offset_d_]); + alignment_details_d->alignment_graph = reinterpret_castalignment_graph)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->alignment_graph) * max_nodes_per_window_ * max_poas_); - alignment_details_d->alignment_read = reinterpret_castalignment_read)>(&block_data_d_[offset_d_]); + alignment_details_d->alignment_read = reinterpret_castalignment_read)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->alignment_read) * max_nodes_per_window_ * max_poas_); // in traceback banded mode, we store only part of scores matrix for forward computation in NW if (traceback_alignment_) { - alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); + alignment_details_d->scores = reinterpret_castscores)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->scores) * score_matrix_width_ * score_matrix_height_ * max_poas_); } if (variable_bands_) { - alignment_details_d->band_starts = reinterpret_castband_starts)>(&block_data_d_[offset_d_]); + alignment_details_d->band_starts = reinterpret_castband_starts)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->band_starts) * max_nodes_per_window_ * max_poas_); - alignment_details_d->band_widths = reinterpret_castband_widths)>(&block_data_d_[offset_d_]); + alignment_details_d->band_widths = reinterpret_castband_widths)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->band_widths) * max_nodes_per_window_ * max_poas_); - alignment_details_d->band_head_indices = reinterpret_castband_head_indices)>(&block_data_d_[offset_d_]); + alignment_details_d->band_head_indices = reinterpret_castband_head_indices)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->band_head_indices) * max_nodes_per_window_ * max_poas_); - alignment_details_d->band_max_indices = reinterpret_castband_max_indices)>(&block_data_d_[offset_d_]); + alignment_details_d->band_max_indices = reinterpret_castband_max_indices)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*alignment_details_d->band_max_indices) * max_nodes_per_window_ * max_poas_); } @@ -235,14 +237,14 @@ class BatchBlock { // in non-traceback alignments, where only scores buffer is used and traceback buffer size is 0, rest of the available memory is assigned to scores buffer alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; - alignment_details_d->scores = reinterpret_castscores)>(&block_data_d_[offset_d_]); + alignment_details_d->scores = reinterpret_castscores)>(&block_data_d[offset_d_]); alignment_details_d->traceback = nullptr; } else { // in traceback alignment, rest of the available memory is assigned to traceback buffer alignment_details_d->scorebuf_alloc_size = total_d_ - offset_d_; - alignment_details_d->traceback = reinterpret_casttraceback)>(&block_data_d_[offset_d_]); + alignment_details_d->traceback = reinterpret_casttraceback)>(&block_data_d[offset_d_]); } *alignment_details_d_p = alignment_details_d; @@ -252,7 +254,7 @@ class BatchBlock { GraphDetails* graph_details_d{}; GraphDetails* graph_details_h{}; - + uint8_t * block_data_d = block_data_d_.data(); // on host graph_details_h = reinterpret_cast*>(&block_data_h_[offset_h_]); offset_h_ += sizeof(GraphDetails); @@ -269,51 +271,51 @@ class BatchBlock graph_details_d->nodes = &block_data_h_[offset_h_]; // on device - graph_details_d->nodes = &block_data_d_[offset_d_]; + graph_details_d->nodes = &block_data_d[offset_d_]; offset_d_ += cudautils::align(sizeof(*graph_details_d->nodes) * max_nodes_per_window_ * max_poas_); - graph_details_d->node_alignments = reinterpret_castnode_alignments)>(&block_data_d_[offset_d_]); + graph_details_d->node_alignments = reinterpret_castnode_alignments)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->node_alignments) * max_nodes_per_window_ * CUDAPOA_MAX_NODE_ALIGNMENTS * max_poas_); - graph_details_d->node_alignment_count = reinterpret_castnode_alignment_count)>(&block_data_d_[offset_d_]); + graph_details_d->node_alignment_count = reinterpret_castnode_alignment_count)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->node_alignment_count) * max_nodes_per_window_ * max_poas_); - graph_details_d->incoming_edges = reinterpret_castincoming_edges)>(&block_data_d_[offset_d_]); + graph_details_d->incoming_edges = reinterpret_castincoming_edges)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->incoming_edges) * max_nodes_per_window_ * CUDAPOA_MAX_NODE_EDGES * max_poas_); - graph_details_d->incoming_edge_count = reinterpret_castincoming_edge_count)>(&block_data_d_[offset_d_]); + graph_details_d->incoming_edge_count = reinterpret_castincoming_edge_count)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->incoming_edge_count) * max_nodes_per_window_ * max_poas_); - graph_details_d->outgoing_edges = reinterpret_castoutgoing_edges)>(&block_data_d_[offset_d_]); + graph_details_d->outgoing_edges = reinterpret_castoutgoing_edges)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->outgoing_edges) * max_nodes_per_window_ * CUDAPOA_MAX_NODE_EDGES * max_poas_); - graph_details_d->outgoing_edge_count = reinterpret_castoutgoing_edge_count)>(&block_data_d_[offset_d_]); + graph_details_d->outgoing_edge_count = reinterpret_castoutgoing_edge_count)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->outgoing_edge_count) * max_nodes_per_window_ * max_poas_); - graph_details_d->incoming_edge_weights = reinterpret_castincoming_edge_weights)>(&block_data_d_[offset_d_]); + graph_details_d->incoming_edge_weights = reinterpret_castincoming_edge_weights)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->incoming_edge_weights) * max_nodes_per_window_ * CUDAPOA_MAX_NODE_EDGES * max_poas_); - graph_details_d->sorted_poa = reinterpret_castsorted_poa)>(&block_data_d_[offset_d_]); + graph_details_d->sorted_poa = reinterpret_castsorted_poa)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->sorted_poa) * max_nodes_per_window_ * max_poas_); - graph_details_d->sorted_poa_node_map = reinterpret_castsorted_poa_node_map)>(&block_data_d_[offset_d_]); + graph_details_d->sorted_poa_node_map = reinterpret_castsorted_poa_node_map)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->sorted_poa_node_map) * max_nodes_per_window_ * max_poas_); - graph_details_d->sorted_poa_local_edge_count = reinterpret_castsorted_poa_local_edge_count)>(&block_data_d_[offset_d_]); + graph_details_d->sorted_poa_local_edge_count = reinterpret_castsorted_poa_local_edge_count)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->sorted_poa_local_edge_count) * max_nodes_per_window_ * max_poas_); if (output_mask_ & OutputType::consensus) { - graph_details_d->consensus_scores = reinterpret_castconsensus_scores)>(&block_data_d_[offset_d_]); + graph_details_d->consensus_scores = reinterpret_castconsensus_scores)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->consensus_scores) * max_nodes_per_window_ * max_poas_); - graph_details_d->consensus_predecessors = reinterpret_castconsensus_predecessors)>(&block_data_d_[offset_d_]); + graph_details_d->consensus_predecessors = reinterpret_castconsensus_predecessors)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->consensus_predecessors) * max_nodes_per_window_ * max_poas_); } - graph_details_d->node_marks = reinterpret_castnode_marks)>(&block_data_d_[offset_d_]); + graph_details_d->node_marks = reinterpret_castnode_marks)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->node_marks) * max_nodes_per_window_ * max_poas_); - graph_details_d->check_aligned_nodes = reinterpret_castcheck_aligned_nodes)>(&block_data_d_[offset_d_]); + graph_details_d->check_aligned_nodes = reinterpret_castcheck_aligned_nodes)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->check_aligned_nodes) * max_nodes_per_window_ * max_poas_); - graph_details_d->nodes_to_visit = reinterpret_castnodes_to_visit)>(&block_data_d_[offset_d_]); + graph_details_d->nodes_to_visit = reinterpret_castnodes_to_visit)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->nodes_to_visit) * max_nodes_per_window_ * max_poas_); - graph_details_d->node_coverage_counts = reinterpret_castnode_coverage_counts)>(&block_data_d_[offset_d_]); + graph_details_d->node_coverage_counts = reinterpret_castnode_coverage_counts)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->node_coverage_counts) * max_nodes_per_window_ * max_poas_); if (output_mask_ & OutputType::msa) { - graph_details_d->outgoing_edges_coverage = reinterpret_castoutgoing_edges_coverage)>(&block_data_d_[offset_d_]); + graph_details_d->outgoing_edges_coverage = reinterpret_castoutgoing_edges_coverage)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->outgoing_edges_coverage) * max_nodes_per_window_ * CUDAPOA_MAX_NODE_EDGES * max_sequences_per_poa_ * max_poas_); - graph_details_d->outgoing_edges_coverage_count = reinterpret_castoutgoing_edges_coverage_count)>(&block_data_d_[offset_d_]); + graph_details_d->outgoing_edges_coverage_count = reinterpret_castoutgoing_edges_coverage_count)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->outgoing_edges_coverage_count) * max_nodes_per_window_ * CUDAPOA_MAX_NODE_EDGES * max_poas_); - graph_details_d->node_id_to_msa_pos = reinterpret_castnode_id_to_msa_pos)>(&block_data_d_[offset_d_]); + graph_details_d->node_id_to_msa_pos = reinterpret_castnode_id_to_msa_pos)>(&block_data_d[offset_d_]); offset_d_ += cudautils::align(sizeof(*graph_details_d->node_id_to_msa_pos) * max_nodes_per_window_ * max_poas_); } @@ -328,7 +330,7 @@ class BatchBlock uint8_t* get_block_device() { - return block_data_d_; + return block_data_d_.data(); } int32_t get_max_poas() const { return max_poas_; }; @@ -493,7 +495,7 @@ class BatchBlock // Pointer for block data on host and device uint8_t* block_data_h_; - uint8_t* block_data_d_; + device_buffer block_data_d_; // Accumulator for the memory size int64_t total_h_ = 0; diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index e66bc7cf4..fa92e2e3d 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -110,7 +110,8 @@ std::unique_ptr create_batch(int32_t device_id, const BatchConfig& batch_size, int16_t gap_score, int16_t mismatch_score, - int16_t match_score) + int16_t match_score, + DefaultDeviceAllocator allocator) { if (use32bitScore(batch_size, gap_score, mismatch_score, match_score)) { @@ -125,7 +126,8 @@ std::unique_ptr create_batch(int32_t device_id, batch_size, gap_score, mismatch_score, - match_score); + match_score, + allocator); } else { @@ -136,7 +138,8 @@ std::unique_ptr create_batch(int32_t device_id, batch_size, gap_score, mismatch_score, - match_score); + match_score, + allocator); } } else @@ -150,7 +153,8 @@ std::unique_ptr create_batch(int32_t device_id, batch_size, gap_score, mismatch_score, - match_score); + match_score, + allocator); } else { @@ -161,7 +165,8 @@ std::unique_ptr create_batch(int32_t device_id, batch_size, gap_score, mismatch_score, - match_score); + match_score, + allocator); } } } @@ -177,7 +182,8 @@ std::unique_ptr create_batch(int32_t device_id, batch_size, gap_score, mismatch_score, - match_score); + match_score, + allocator); } else { @@ -188,7 +194,8 @@ std::unique_ptr create_batch(int32_t device_id, batch_size, gap_score, mismatch_score, - match_score); + match_score, + allocator); } } } diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 1a9d7ac2f..4beeb49e0 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -64,7 +64,7 @@ class CudapoaBatch : public Batch { public: CudapoaBatch(int32_t device_id, cudaStream_t stream, size_t max_gpu_mem, int8_t output_mask, - const BatchConfig& batch_size, int32_t gap_score = -8, int32_t mismatch_score = -6, int32_t match_score = 8) + const BatchConfig& batch_size, int32_t gap_score = -8, int32_t mismatch_score = -6, int32_t match_score = 8, DefaultDeviceAllocator allocator=DefaultDeviceAllocator()) : max_sequences_per_poa_(throw_on_negative(batch_size.max_sequences_per_poa, "Maximum sequences per POA has to be non-negative")) , device_id_(throw_on_negative(device_id, "Device ID has to be non-negative")) , stream_(stream) @@ -76,8 +76,10 @@ public: , batch_block_(new BatchBlock(device_id, max_gpu_mem, output_mask, - batch_size_)) + batch_size_, + allocator)) , max_poas_(batch_block_->get_max_poas()) + , allocator_(allocator) { // Set CUDA device scoped_device_switch dev(device_id_); @@ -643,6 +645,8 @@ protected: // Maximum POAs to process in batch. int32_t max_poas_ = 0; + DefaultDeviceAllocator allocator_; + public: // Static batch count used to generate batch IDs. static int32_t batches; From 6f931e78c7021800c8a9c5c2e9cd5d0db8c3b8c4 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 12 Nov 2020 11:30:45 -0800 Subject: [PATCH 229/281] [cudapoa] cleanup device memory limit and device allocator interface --- cudapoa/benchmarks/multi_batch.hpp | 2 +- .../genomeworks/cudapoa/batch.hpp | 28 +++++++-- cudapoa/samples/sample_cudapoa.cpp | 2 +- cudapoa/src/allocate_block.hpp | 10 +-- cudapoa/src/batch.cu | 61 ++++++++++++++----- cudapoa/src/cudapoa_batch.cuh | 13 ++-- cudapoa/src/main.cpp | 2 +- cudapoa/tests/Test_CudapoaSerializeGraph.cpp | 2 +- 8 files changed, 84 insertions(+), 36 deletions(-) diff --git a/cudapoa/benchmarks/multi_batch.hpp b/cudapoa/benchmarks/multi_batch.hpp index a88c0d30d..194a38a33 100644 --- a/cudapoa/benchmarks/multi_batch.hpp +++ b/cudapoa/benchmarks/multi_batch.hpp @@ -51,7 +51,7 @@ class MultiBatch size_t total = 0, free = 0; cudaSetDevice(0); cudaMemGetInfo(&free, &total); - size_t mem_per_batch = 0.9 * free / num_batches_; + int64_t mem_per_batch = 0.9 * free / num_batches_; for (int32_t batch = 0; batch < num_batches_; batch++) { streams_.emplace_back(make_cuda_stream()); diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp index 82ff0b7cf..c386ea3d2 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp @@ -166,23 +166,43 @@ class Batch /// /// \param device_id GPU device on which to run CUDA POA algorithm /// \param stream CUDA stream to use on GPU +/// \param allocator Allocator to use for internal device memory allocations /// \param max_gpu_mem Maximum GPU memory to use for this batch. /// \param output_mask which outputs to produce from POA (msa, consensus) /// \param batch_size defines upper limits for size of a POA batch, i.e. sequence length and other related parameters /// \param gap_score score to be assigned to a gap /// \param mismatch_score score to be assigned to a mismatch /// \param match_score score to be assigned for a match -/// \param allocator allocator to be used /// \return Returns a unique pointer to a new Batch object std::unique_ptr create_batch(int32_t device_id, cudaStream_t stream, - size_t max_gpu_mem, + DefaultDeviceAllocator allocator, + int64_t max_gpu_mem, int8_t output_mask, const BatchConfig& batch_size, int16_t gap_score, int16_t mismatch_score, - int16_t match_score, - DefaultDeviceAllocator allocator); + int16_t match_score); + +/// \brief Creates a new CUDA Batch object. +/// +/// \param device_id GPU device on which to run CUDA POA algorithm +/// \param stream CUDA stream to use on GPU +/// \param max_gpu_mem Maximum GPU memory to use for this batch. +/// \param output_mask which outputs to produce from POA (msa, consensus) +/// \param batch_size defines upper limits for size of a POA batch, i.e. sequence length and other related parameters +/// \param gap_score score to be assigned to a gap +/// \param mismatch_score score to be assigned to a mismatch +/// \param match_score score to be assigned for a match +/// \return Returns a unique pointer to a new Batch object +std::unique_ptr create_batch(int32_t device_id, + cudaStream_t stream, + int64_t max_gpu_mem, + int8_t output_mask, + const BatchConfig& batch_size, + int16_t gap_score, + int16_t mismatch_score, + int16_t match_score); /// \} diff --git a/cudapoa/samples/sample_cudapoa.cpp b/cudapoa/samples/sample_cudapoa.cpp index ed1f3f20d..3851ee093 100644 --- a/cudapoa/samples/sample_cudapoa.cpp +++ b/cudapoa/samples/sample_cudapoa.cpp @@ -47,7 +47,7 @@ std::unique_ptr initialize_batch(bool msa, const BatchConfig& batch_size) // Initialize CUDAPOA batch object for batched processing of POAs on the GPU. const int32_t device_id = 0; cudaStream_t stream = 0; - size_t mem_per_batch = 0.9 * free; // Using 90% of GPU available memory for CUDAPOA batch. + int64_t mem_per_batch = 0.9 * free; // Using 90% of GPU available memory for CUDAPOA batch. const int32_t mismatch_score = -6, gap_score = -8, match_score = 8; std::unique_ptr batch = create_batch(device_id, stream, diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 86aa9409f..747429a97 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -55,7 +55,7 @@ template class BatchBlock { public: - BatchBlock(int32_t device_id, size_t avail_mem, int8_t output_mask, const BatchConfig& batch_size, DefaultDeviceAllocator allocator) + BatchBlock(int32_t device_id, DefaultDeviceAllocator allocator, int64_t max_mem, int8_t output_mask, const BatchConfig& batch_size) : max_sequences_per_poa_(throw_on_negative(batch_size.max_sequences_per_poa, "Maximum sequences per POA has to be non-negative")) , device_id_(throw_on_negative(device_id, "Device ID has to be non-negative")) , output_mask_(output_mask) @@ -70,12 +70,13 @@ class BatchBlock int64_t host_size_fixed, device_size_fixed; int64_t host_size_per_poa, device_size_per_poa; std::tie(host_size_fixed, device_size_fixed, host_size_per_poa, device_size_per_poa) = calculate_space_per_poa(batch_size); - // Check minimum requirement for device memory - size_t minimum_device_mem = device_size_fixed + device_size_per_poa; + // Max memory that can be used = min(available free mem, upper limit on memory use) + int64_t avail_mem = std::min(get_size_of_largest_free_memory_block(allocator), max_mem); + int64_t minimum_device_mem = device_size_fixed + device_size_per_poa; if (avail_mem < minimum_device_mem) { - std::string msg = std::string("Require at least ") + std::string msg = std::string("Requires at least ") .append(std::to_string(minimum_device_mem)) .append(" bytes of device memory per CUDAPOA batch to process correctly."); throw std::runtime_error(msg); @@ -512,7 +513,6 @@ class BatchBlock int32_t score_matrix_width_ = 0; bool traceback_alignment_ = false; int32_t device_id_; - // Bit field for output type int8_t output_mask_; }; diff --git a/cudapoa/src/batch.cu b/cudapoa/src/batch.cu index fa92e2e3d..18361fd63 100644 --- a/cudapoa/src/batch.cu +++ b/cudapoa/src/batch.cu @@ -105,13 +105,13 @@ BatchConfig::BatchConfig(int32_t max_seq_sz, int32_t max_consensus_sz, int32_t m std::unique_ptr create_batch(int32_t device_id, cudaStream_t stream, - size_t max_mem, + DefaultDeviceAllocator allocator, + int64_t max_mem, int8_t output_mask, const BatchConfig& batch_size, int16_t gap_score, int16_t mismatch_score, - int16_t match_score, - DefaultDeviceAllocator allocator) + int16_t match_score) { if (use32bitScore(batch_size, gap_score, mismatch_score, match_score)) { @@ -121,25 +121,25 @@ std::unique_ptr create_batch(int32_t device_id, { return std::make_unique>(device_id, stream, + allocator, max_mem, output_mask, batch_size, gap_score, mismatch_score, - match_score, - allocator); + match_score); } else { return std::make_unique>(device_id, stream, + allocator, max_mem, output_mask, batch_size, gap_score, mismatch_score, - match_score, - allocator); + match_score); } } else @@ -148,25 +148,25 @@ std::unique_ptr create_batch(int32_t device_id, { return std::make_unique>(device_id, stream, + allocator, max_mem, output_mask, batch_size, gap_score, mismatch_score, - match_score, - allocator); + match_score); } else { return std::make_unique>(device_id, stream, + allocator, max_mem, output_mask, batch_size, gap_score, mismatch_score, - match_score, - allocator); + match_score); } } } @@ -177,29 +177,60 @@ std::unique_ptr create_batch(int32_t device_id, { return std::make_unique>(device_id, stream, + allocator, max_mem, output_mask, batch_size, gap_score, mismatch_score, - match_score, - allocator); + match_score); } else { return std::make_unique>(device_id, stream, + allocator, max_mem, output_mask, batch_size, gap_score, mismatch_score, - match_score, - allocator); + match_score); } } } +std::unique_ptr create_batch(int32_t device_id, + cudaStream_t stream, + int64_t max_mem, + int8_t output_mask, + const BatchConfig& batch_size, + int16_t gap_score, + int16_t mismatch_score, + int16_t match_score) +{ + if (max_mem < -1) + { + throw std::invalid_argument("max_mem has to be either -1 (=all available GPU memory) or greater or equal than 0."); + } +#ifdef GW_ENABLE_CACHING_ALLOCATOR + // uses CachingDeviceAllocator + if (max_mem == -1) + { + max_mem = claraparabricks::genomeworks::cudautils::find_largest_contiguous_device_memory_section(); + if (max_mem == 0) + { + throw std::runtime_error("No memory available for caching"); + } + } + claraparabricks::genomeworks::DefaultDeviceAllocator allocator(max_mem); +#else + // uses CudaMallocAllocator + claraparabricks::genomeworks::DefaultDeviceAllocator allocator; +#endif + return create_batch(device_id, stream, allocator, max_mem, output_mask, batch_size, gap_score, mismatch_score, match_score); +} + } // namespace cudapoa } // namespace genomeworks diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 4beeb49e0..690d1628e 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -63,8 +63,8 @@ template class CudapoaBatch : public Batch { public: - CudapoaBatch(int32_t device_id, cudaStream_t stream, size_t max_gpu_mem, int8_t output_mask, - const BatchConfig& batch_size, int32_t gap_score = -8, int32_t mismatch_score = -6, int32_t match_score = 8, DefaultDeviceAllocator allocator=DefaultDeviceAllocator()) + CudapoaBatch(int32_t device_id, cudaStream_t stream, DefaultDeviceAllocator allocator, int64_t max_mem, int8_t output_mask, + const BatchConfig& batch_size, int32_t gap_score = -8, int32_t mismatch_score = -6, int32_t match_score = 8) : max_sequences_per_poa_(throw_on_negative(batch_size.max_sequences_per_poa, "Maximum sequences per POA has to be non-negative")) , device_id_(throw_on_negative(device_id, "Device ID has to be non-negative")) , stream_(stream) @@ -74,12 +74,11 @@ public: , mismatch_score_(mismatch_score) , match_score_(match_score) , batch_block_(new BatchBlock(device_id, - max_gpu_mem, + allocator, + max_mem, output_mask, - batch_size_, - allocator)) + batch_size_)) , max_poas_(batch_block_->get_max_poas()) - , allocator_(allocator) { // Set CUDA device scoped_device_switch dev(device_id_); @@ -645,8 +644,6 @@ protected: // Maximum POAs to process in batch. int32_t max_poas_ = 0; - DefaultDeviceAllocator allocator_; - public: // Static batch count used to generate batch IDs. static int32_t batches; diff --git a/cudapoa/src/main.cpp b/cudapoa/src/main.cpp index 3668f65cc..65f22339e 100644 --- a/cudapoa/src/main.cpp +++ b/cudapoa/src/main.cpp @@ -50,7 +50,7 @@ std::unique_ptr initialize_batch(int32_t mismatch_score, // Initialize CUDAPOA batch object for batched processing of POAs on the GPU. const int32_t device_id = 0; cudaStream_t stream = 0; - size_t mem_per_batch = gpu_mem_allocation * free; // Using 90% of GPU available memory for CUDAPOA batch. + int64_t mem_per_batch = gpu_mem_allocation * free; // Using 90% of GPU available memory for CUDAPOA batch. std::unique_ptr batch = create_batch(device_id, stream, diff --git a/cudapoa/tests/Test_CudapoaSerializeGraph.cpp b/cudapoa/tests/Test_CudapoaSerializeGraph.cpp index 5db2dba69..7866df5bb 100644 --- a/cudapoa/tests/Test_CudapoaSerializeGraph.cpp +++ b/cudapoa/tests/Test_CudapoaSerializeGraph.cpp @@ -48,7 +48,7 @@ class GraphTest : public ::testing::Test size_t total = 0, free = 0; cudaSetDevice(device_id); cudaMemGetInfo(&free, &total); - size_t mem_per_batch = 0.9 * free; + int64_t mem_per_batch = 0.9 * free; BatchConfig batch_size(1024, max_sequences_per_poa); cudapoa_batch = genomeworks::cudapoa::create_batch(device_id, stream, mem_per_batch, output_mask, batch_size, gap_score, mismatch_score, match_score); From 67827766e2f5dc28c5665b221b7b6557782ece88 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 12 Nov 2020 11:31:54 -0800 Subject: [PATCH 230/281] [cudapoa] Make format --- .../claraparabricks/genomeworks/cudapoa/batch.hpp | 1 - cudapoa/samples/sample_cudapoa.cpp | 2 +- cudapoa/src/allocate_block.hpp | 12 ++++++------ cudapoa/src/main.cpp | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp index c386ea3d2..664a0c451 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/batch.hpp @@ -30,7 +30,6 @@ #include #include - namespace claraparabricks { diff --git a/cudapoa/samples/sample_cudapoa.cpp b/cudapoa/samples/sample_cudapoa.cpp index 3851ee093..116995f9e 100644 --- a/cudapoa/samples/sample_cudapoa.cpp +++ b/cudapoa/samples/sample_cudapoa.cpp @@ -47,7 +47,7 @@ std::unique_ptr initialize_batch(bool msa, const BatchConfig& batch_size) // Initialize CUDAPOA batch object for batched processing of POAs on the GPU. const int32_t device_id = 0; cudaStream_t stream = 0; - int64_t mem_per_batch = 0.9 * free; // Using 90% of GPU available memory for CUDAPOA batch. + int64_t mem_per_batch = 0.9 * free; // Using 90% of GPU available memory for CUDAPOA batch. const int32_t mismatch_score = -6, gap_score = -8, match_score = 8; std::unique_ptr batch = create_batch(device_id, stream, diff --git a/cudapoa/src/allocate_block.hpp b/cudapoa/src/allocate_block.hpp index 747429a97..2b5771021 100644 --- a/cudapoa/src/allocate_block.hpp +++ b/cudapoa/src/allocate_block.hpp @@ -71,8 +71,8 @@ class BatchBlock int64_t host_size_per_poa, device_size_per_poa; std::tie(host_size_fixed, device_size_fixed, host_size_per_poa, device_size_per_poa) = calculate_space_per_poa(batch_size); // Check minimum requirement for device memory - // Max memory that can be used = min(available free mem, upper limit on memory use) - int64_t avail_mem = std::min(get_size_of_largest_free_memory_block(allocator), max_mem); + // Max memory that can be used = min(available free mem, upper limit on memory use) + int64_t avail_mem = std::min(get_size_of_largest_free_memory_block(allocator), max_mem); int64_t minimum_device_mem = device_size_fixed + device_size_per_poa; if (avail_mem < minimum_device_mem) { @@ -118,7 +118,7 @@ class BatchBlock { OutputDetails* output_details_h{}; OutputDetails* output_details_d{}; - uint8_t * block_data_d = block_data_d_.data(); + uint8_t* block_data_d = block_data_d_.data(); // on host output_details_h = reinterpret_cast(&block_data_h_[offset_h_]); @@ -162,7 +162,7 @@ class BatchBlock // on host InputDetails* input_details_h{}; InputDetails* input_details_d{}; - uint8_t * block_data_d = block_data_d_.data(); + uint8_t* block_data_d = block_data_d_.data(); input_details_h = reinterpret_cast*>(&block_data_h_[offset_h_]); offset_h_ += sizeof(InputDetails); @@ -205,7 +205,7 @@ class BatchBlock void get_alignment_details(AlignmentDetails** alignment_details_d_p) { AlignmentDetails* alignment_details_d{}; - uint8_t * block_data_d = block_data_d_.data(); + uint8_t* block_data_d = block_data_d_.data(); // on host alignment_details_d = reinterpret_cast*>(&block_data_h_[offset_h_]); offset_h_ += sizeof(AlignmentDetails); @@ -255,7 +255,7 @@ class BatchBlock { GraphDetails* graph_details_d{}; GraphDetails* graph_details_h{}; - uint8_t * block_data_d = block_data_d_.data(); + uint8_t* block_data_d = block_data_d_.data(); // on host graph_details_h = reinterpret_cast*>(&block_data_h_[offset_h_]); offset_h_ += sizeof(GraphDetails); diff --git a/cudapoa/src/main.cpp b/cudapoa/src/main.cpp index 65f22339e..e68d0d2f6 100644 --- a/cudapoa/src/main.cpp +++ b/cudapoa/src/main.cpp @@ -50,7 +50,7 @@ std::unique_ptr initialize_batch(int32_t mismatch_score, // Initialize CUDAPOA batch object for batched processing of POAs on the GPU. const int32_t device_id = 0; cudaStream_t stream = 0; - int64_t mem_per_batch = gpu_mem_allocation * free; // Using 90% of GPU available memory for CUDAPOA batch. + int64_t mem_per_batch = gpu_mem_allocation * free; // Using 90% of GPU available memory for CUDAPOA batch. std::unique_ptr batch = create_batch(device_id, stream, From dd0a776b375958d7ad5bae48e092d33ee970e396 Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Thu, 12 Nov 2020 12:56:31 -0800 Subject: [PATCH 231/281] update readme to include automake for building --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fc4e9b981..a667f3129 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ Minimum requirements - Optional requirements - 1. autoconf (required to output SAM/BAM files) +2. automake (required to output SAM/BAM files) ## GenomeWorks Setup From eb6de409e1f2d778d861067e0bbb03d9732d2e01 Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Thu, 12 Nov 2020 13:12:57 -0800 Subject: [PATCH 232/281] make htslib build tools required because htslib is build by default --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a667f3129..fb8b027ba 100644 --- a/README.md +++ b/README.md @@ -112,11 +112,8 @@ Minimum requirements - 3. gcc/g++ 5.4.0+ / 7.x.x 4. Python 3.6.7+ 5. CMake (>= 3.10.2) - -Optional requirements - - -1. autoconf (required to output SAM/BAM files) -2. automake (required to output SAM/BAM files) +6. autoconf (required to output SAM/BAM files) +7. automake (required to output SAM/BAM files) ## GenomeWorks Setup From fd8cb5c9a1b74451ec8586fb918ef0ddae396eae Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Wed, 4 Nov 2020 22:49:27 -0500 Subject: [PATCH 233/281] [cudamapper] add important gcc flags --- cudamapper/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cudamapper/CMakeLists.txt b/cudamapper/CMakeLists.txt index 87e9bcac2..16463c21f 100644 --- a/cudamapper/CMakeLists.txt +++ b/cudamapper/CMakeLists.txt @@ -76,7 +76,11 @@ if (TARGET htslib_project) else() target_link_libraries(${MODULE_NAME} gwbase gwio cub) endif() -target_compile_options(${MODULE_NAME} PRIVATE -Werror) + +target_compile_options(${MODULE_NAME} PRIVATE -Werror -Wall -Wextra) +if (gw_optimize_for_native_cpu) + target_compile_options(cudapoa PRIVATE -march=native) +endif() add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) From 026859ae9ae0aa81e7c12190138b14a7e73ac896 Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Thu, 5 Nov 2020 13:52:38 -0800 Subject: [PATCH 234/281] fix gcc compile issues --- cudamapper/src/application_parameters.cpp | 2 ++ cudamapper/src/cudamapper_utils.cpp | 2 +- cudamapper/src/overlapper.cpp | 13 ++++++------- cudamapper/src/utils.cpp | 13 ++++++++----- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/cudamapper/src/application_parameters.cpp b/cudamapper/src/application_parameters.cpp index 999a5c539..f7642ea60 100644 --- a/cudamapper/src/application_parameters.cpp +++ b/cudamapper/src/application_parameters.cpp @@ -147,8 +147,10 @@ ApplicationParameters::ApplicationParameters(int argc, char* argv[]) break; case 'v': print_version(); + exit(1); case 'h': help(0); + exit(1); default: exit(1); } diff --git a/cudamapper/src/cudamapper_utils.cpp b/cudamapper/src/cudamapper_utils.cpp index 8125874e8..a7b8466fb 100644 --- a/cudamapper/src/cudamapper_utils.cpp +++ b/cudamapper/src/cudamapper_utils.cpp @@ -38,7 +38,7 @@ std::vector split_into_kmers(const gw_string_view_t& s, const const std::size_t kmer_count = s.length() - kmer_size + 1; std::vector kmers; - if (s.length() < kmer_size) + if (get_size(s) < kmer_size) { kmers.push_back(s); return kmers; diff --git a/cudamapper/src/overlapper.cpp b/cudamapper/src/overlapper.cpp index 1b6bd9341..48e0b846d 100644 --- a/cudamapper/src/overlapper.cpp +++ b/cudamapper/src/overlapper.cpp @@ -136,11 +136,11 @@ void Overlapper::post_process_overlaps(std::vector& overlaps, const boo { const auto num_overlaps = get_size(overlaps); bool in_fuse = false; - int fused_target_start; - int fused_query_start; - int fused_target_end; - int fused_query_end; - int num_residues = 0; + int fused_target_start = 0; + int fused_query_start = 0; + int fused_target_end = 0; + int fused_query_end = 0; + int num_residues = 0; Overlap prev_overlap; std::vector drop_overlap_mask; if (drop_fused_overlaps) @@ -306,7 +306,6 @@ void Overlapper::rescue_overlap_ends(std::vector& overlaps, const std::int32_t extension, const float required_similarity) { - auto reverse_overlap = [](cudamapper::Overlap& overlap, std::uint32_t target_sequence_length) { overlap.relative_strand = overlap.relative_strand == RelativeStrand::Forward ? RelativeStrand::Reverse : RelativeStrand::Forward; position_in_read_t start_tmp = overlap.target_start_position_in_read_; @@ -350,7 +349,7 @@ void Overlapper::rescue_overlap_ends(std::vector& overlaps, while (rescue_rounds < max_rescue_rounds) { - details::overlapper::extend_overlap_by_sequence_similarity(overlap, query_view, target_view, 100, 0.9); + details::overlapper::extend_overlap_by_sequence_similarity(overlap, query_view, target_view, extension, required_similarity); ++rescue_rounds; if (overlap.query_end_position_in_read_ == prev_query_start && overlap.query_end_position_in_read_ == prev_query_end && diff --git a/cudamapper/src/utils.cpp b/cudamapper/src/utils.cpp index 24bcb9817..3e6a2e6eb 100644 --- a/cudamapper/src/utils.cpp +++ b/cudamapper/src/utils.cpp @@ -156,13 +156,13 @@ static inline void encode_seq(bam1_t* const alignment, const std::string& seq) 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; - const size_t len = seq.length(); + const int32_t len = get_size(seq); // encode bases as nibbles and write them into the alignment struct 2 bases at a time // 4 high bits are first base, 4 low bits are second base uint8_t* bam_seq_start = bam_get_seq(alignment); - int i = 0; - for (i; (i + 1) < len; i += 2) + int32_t i = 0; + for (; (i + 1) < len; i += 2) { *bam_seq_start++ = (L[static_cast(seq[i])] << 4) + L[static_cast(seq[i + 1])]; } @@ -214,7 +214,6 @@ void print_sam(const std::vector& overlaps, const int64_t number_of_overlaps_to_print = get_size(overlaps); for (int64_t i = 0; i < number_of_overlaps_to_print; ++i) { - const std::string& query_read_name = query_parser.get_sequence_by_id(overlaps[i].query_read_id_).name; const std::string& target_read_name = target_parser.get_sequence_by_id(overlaps[i].target_read_id_).name; const std::string length = std::to_string(target_parser.get_sequence_by_id(overlaps[i].target_read_id_).seq.length()); @@ -305,7 +304,11 @@ void print_sam(const std::vector& overlaps, // TODO: write AUX data if available std::lock_guard lg(write_output_mutex); - int out = sam_write1(file.get(), header.get(), alignment.get()); + result = sam_write1(file.get(), header.get(), alignment.get()); + if (result < 0) + { + fprintf(stderr, "print_sam: could not write alignment"); + } } return; From fd99970e651bab0d854ca509a581686310162324 Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Fri, 6 Nov 2020 12:25:05 -0800 Subject: [PATCH 235/281] Address comments --- cudamapper/CMakeLists.txt | 2 +- cudamapper/src/utils.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cudamapper/CMakeLists.txt b/cudamapper/CMakeLists.txt index 16463c21f..7846d38ce 100644 --- a/cudamapper/CMakeLists.txt +++ b/cudamapper/CMakeLists.txt @@ -79,7 +79,7 @@ endif() target_compile_options(${MODULE_NAME} PRIVATE -Werror -Wall -Wextra) if (gw_optimize_for_native_cpu) - target_compile_options(cudapoa PRIVATE -march=native) + target_compile_options(${MODULE_NAME} PRIVATE -march=native) endif() add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) diff --git a/cudamapper/src/utils.cpp b/cudamapper/src/utils.cpp index 3e6a2e6eb..a59d43667 100644 --- a/cudamapper/src/utils.cpp +++ b/cudamapper/src/utils.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -221,7 +222,7 @@ void print_sam(const std::vector& overlaps, "CL", NULL); if (result < 0) { - fprintf(stderr, "print_sam: could not add header value"); + std::cerr << "print_sam: could not add header value" << std::endl; } } @@ -238,14 +239,14 @@ void print_sam(const std::vector& overlaps, } if (result < 0) { - fprintf(stderr, "print_sam: could not add PG header line"); + std::cerr << "print_sam: could not add PG header line" << std::endl; } { result = sam_hdr_write(file.get(), header.get()); if (result < 0) { - fprintf(stderr, "print_sam: could not add PG header line"); + std::cerr << "print_sam: could not add PG header line" << std::endl; } } @@ -307,7 +308,7 @@ void print_sam(const std::vector& overlaps, result = sam_write1(file.get(), header.get(), alignment.get()); if (result < 0) { - fprintf(stderr, "print_sam: could not write alignment"); + std::cerr << "print_sam: could not write alignment" << std::endl; } } From c85792fe446ccd8aa02efa20e5859fe95f1da743 Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Mon, 9 Nov 2020 15:54:34 -0800 Subject: [PATCH 236/281] throw runtime_error if alignment cannot be written --- cudamapper/src/utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudamapper/src/utils.cpp b/cudamapper/src/utils.cpp index a59d43667..52224f620 100644 --- a/cudamapper/src/utils.cpp +++ b/cudamapper/src/utils.cpp @@ -308,7 +308,7 @@ void print_sam(const std::vector& overlaps, result = sam_write1(file.get(), header.get(), alignment.get()); if (result < 0) { - std::cerr << "print_sam: could not write alignment" << std::endl; + throw std::runtime_error("ERROR, print_sam: could not write alignment"); } } From 0f7540a5486eaaa1448ffae6ebccf2a1faa8f130 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 12 Nov 2020 16:01:50 -0800 Subject: [PATCH 237/281] [cudapoa] multi-batch device allocator WAR --- cudapoa/benchmarks/multi_batch.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cudapoa/benchmarks/multi_batch.hpp b/cudapoa/benchmarks/multi_batch.hpp index 194a38a33..d58645747 100644 --- a/cudapoa/benchmarks/multi_batch.hpp +++ b/cudapoa/benchmarks/multi_batch.hpp @@ -51,11 +51,12 @@ class MultiBatch size_t total = 0, free = 0; cudaSetDevice(0); cudaMemGetInfo(&free, &total); + DefaultDeviceAllocator allocator = create_default_device_allocator(0.901*free); int64_t mem_per_batch = 0.9 * free / num_batches_; for (int32_t batch = 0; batch < num_batches_; batch++) { streams_.emplace_back(make_cuda_stream()); - batches_.emplace_back(create_batch(0, streams_.back().get(), mem_per_batch, + batches_.emplace_back(create_batch(0, streams_.back().get(), allocator, mem_per_batch, OutputType::consensus, batch_size, -8, -6, 8)); From 3a63d406da80734ae1d9898ca9844710ffd76486 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Thu, 12 Nov 2020 16:05:11 -0800 Subject: [PATCH 238/281] [cudapoa] make format --- cudapoa/benchmarks/multi_batch.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cudapoa/benchmarks/multi_batch.hpp b/cudapoa/benchmarks/multi_batch.hpp index d58645747..a8204542d 100644 --- a/cudapoa/benchmarks/multi_batch.hpp +++ b/cudapoa/benchmarks/multi_batch.hpp @@ -51,8 +51,8 @@ class MultiBatch size_t total = 0, free = 0; cudaSetDevice(0); cudaMemGetInfo(&free, &total); - DefaultDeviceAllocator allocator = create_default_device_allocator(0.901*free); - int64_t mem_per_batch = 0.9 * free / num_batches_; + DefaultDeviceAllocator allocator = create_default_device_allocator(0.901 * free); + int64_t mem_per_batch = 0.9 * free / num_batches_; for (int32_t batch = 0; batch < num_batches_; batch++) { streams_.emplace_back(make_cuda_stream()); From 2c2cc48a9e7daf2419514f100e3dae7cd2d65712 Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Thu, 12 Nov 2020 18:20:34 -0800 Subject: [PATCH 239/281] local variables int to position_in_read_t and use UINT32_MAX --- cudamapper/src/overlapper.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cudamapper/src/overlapper.cpp b/cudamapper/src/overlapper.cpp index 48e0b846d..bff86eb99 100644 --- a/cudamapper/src/overlapper.cpp +++ b/cudamapper/src/overlapper.cpp @@ -136,10 +136,10 @@ void Overlapper::post_process_overlaps(std::vector& overlaps, const boo { const auto num_overlaps = get_size(overlaps); bool in_fuse = false; - int fused_target_start = 0; - int fused_query_start = 0; - int fused_target_end = 0; - int fused_query_end = 0; + position_in_read_t fused_target_start = UINT32_MAX; + position_in_read_t fused_query_start = UINT32_MAX; + position_in_read_t fused_target_end = UINT32_MAX; + position_in_read_t fused_query_end = UINT32_MAX; int num_residues = 0; Overlap prev_overlap; std::vector drop_overlap_mask; From 3dd61736072c76ca94daf36936909e946bbaf9bd Mon Sep 17 00:00:00 2001 From: nvvishanthi Date: Fri, 13 Nov 2020 12:15:58 -0800 Subject: [PATCH 240/281] use numeric_limits instead of MAX macros --- cudamapper/src/overlapper.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cudamapper/src/overlapper.cpp b/cudamapper/src/overlapper.cpp index bff86eb99..b6632f52a 100644 --- a/cudamapper/src/overlapper.cpp +++ b/cudamapper/src/overlapper.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include "cudamapper_utils.hpp" #include "overlapper_triggered.hpp" @@ -134,13 +135,13 @@ namespace cudamapper void Overlapper::post_process_overlaps(std::vector& overlaps, const bool drop_fused_overlaps) { - const auto num_overlaps = get_size(overlaps); - bool in_fuse = false; - position_in_read_t fused_target_start = UINT32_MAX; - position_in_read_t fused_query_start = UINT32_MAX; - position_in_read_t fused_target_end = UINT32_MAX; - position_in_read_t fused_query_end = UINT32_MAX; - int num_residues = 0; + const auto num_overlaps = get_size(overlaps); + bool in_fuse = false; + position_in_read_t fused_target_start = std::numeric_limits::max(); + position_in_read_t fused_query_start = std::numeric_limits::max(); + position_in_read_t fused_target_end = std::numeric_limits::max(); + position_in_read_t fused_query_end = std::numeric_limits::max(); + int num_residues = 0; Overlap prev_overlap; std::vector drop_overlap_mask; if (drop_fused_overlaps) From 8c3fdc2f4598d321deeb4a363e86e2a709d3afab Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 13 Nov 2020 12:55:55 -0800 Subject: [PATCH 241/281] [common] Round up buffer size allocated by DeviceAllocator --- .../utils/device_preallocated_allocator.cuh | 33 ++++++++++++------- cudapoa/benchmarks/multi_batch.hpp | 2 +- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index a21f3a976..bd0cd1d85 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -64,13 +64,13 @@ public: /// Allocates the buffer /// \param buffer_size DevicePreallocatedAllocator(size_t buffer_size) - : buffer_size_(buffer_size) - , buffer_ptr_(create_buffer(buffer_size)) + : buffer_size_(roundup_allocation(buffer_size)) + , buffer_ptr_(create_buffer(buffer_size_)) { - assert(buffer_size > 0); + assert(buffer_size_ > 0); MemoryBlock whole_memory_block; whole_memory_block.begin = 0; - whole_memory_block.size = buffer_size; + whole_memory_block.size = buffer_size_; free_blocks_.push_back(whole_memory_block); } @@ -174,14 +174,7 @@ private: } // ** All allocations should be alligned with 256 bytes - // The easiest way to do this is to make all allocation request sizes divisible by 256 - if ((bytes_needed & 0xFF) != 0) - { - // bytes needed not divisible by 256, increase it to the next value divisible by 256 - bytes_needed = bytes_needed + (0x100 - (bytes_needed & 0xFF)); - } - assert((bytes_needed & 0xFF) == 0); - + bytes_needed = roundup_allocation(bytes_needed); // ** look for first free block that can fit requested size auto block_to_get_memory_from_iter = std::find_if(std::begin(free_blocks_), std::end(free_blocks_), @@ -337,6 +330,22 @@ private: return cudaSuccess; } + /// \brief Rounds up requested size to align with allocation boundaries (256B) + /// \param bytes_requested + /// \return bytes needed for aligning with 256B boundary + inline static size_t roundup_allocation(size_t requested_bytes) + { + // ** All allocations should be aligned with 256 bytes + // The easiest way to do this is to make all allocation request sizes divisible by 256 + if ((requested_bytes & 0xFF) != 0) + { + // bytes needed not divisible by 256, increase it to the next value divisible by 256 + requested_bytes = requested_bytes + (0x100 - (requested_bytes & 0xFF)); + } + assert((requested_bytes & 0xFF) == 0); + return requested_bytes; + } + /// buffer size const size_t buffer_size_; /// preallocated buffer diff --git a/cudapoa/benchmarks/multi_batch.hpp b/cudapoa/benchmarks/multi_batch.hpp index a8204542d..96e257f73 100644 --- a/cudapoa/benchmarks/multi_batch.hpp +++ b/cudapoa/benchmarks/multi_batch.hpp @@ -51,7 +51,7 @@ class MultiBatch size_t total = 0, free = 0; cudaSetDevice(0); cudaMemGetInfo(&free, &total); - DefaultDeviceAllocator allocator = create_default_device_allocator(0.901 * free); + DefaultDeviceAllocator allocator = create_default_device_allocator(0.9 * free); int64_t mem_per_batch = 0.9 * free / num_batches_; for (int32_t batch = 0; batch < num_batches_; batch++) { From 235189203bf4ca90895c54e38c972fc4c3a28608 Mon Sep 17 00:00:00 2001 From: Ashutosh Tadkase Date: Fri, 13 Nov 2020 12:56:54 -0800 Subject: [PATCH 242/281] [common] Make format --- .../genomeworks/utils/device_preallocated_allocator.cuh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index bd0cd1d85..4b4ba9715 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -64,8 +64,8 @@ public: /// Allocates the buffer /// \param buffer_size DevicePreallocatedAllocator(size_t buffer_size) - : buffer_size_(roundup_allocation(buffer_size)) - , buffer_ptr_(create_buffer(buffer_size_)) + : buffer_size_(roundup_allocation(buffer_size)) + , buffer_ptr_(create_buffer(buffer_size_)) { assert(buffer_size_ > 0); MemoryBlock whole_memory_block; From 0341326da810945218bbf800c59c029616bc60aa Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Mon, 23 Nov 2020 14:57:58 +0100 Subject: [PATCH 243/281] [common] DevicePreallocatedAllocator allocates exactly the needed amount of memory --- .../utils/device_preallocated_allocator.cuh | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 4b4ba9715..1a1d4ac67 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -64,7 +64,7 @@ public: /// Allocates the buffer /// \param buffer_size DevicePreallocatedAllocator(size_t buffer_size) - : buffer_size_(roundup_allocation(buffer_size)) + : buffer_size_(buffer_size) , buffer_ptr_(create_buffer(buffer_size_)) { assert(buffer_size_ > 0); @@ -173,8 +173,6 @@ private: return cudaErrorMemoryAllocation; } - // ** All allocations should be alligned with 256 bytes - bytes_needed = roundup_allocation(bytes_needed); // ** look for first free block that can fit requested size auto block_to_get_memory_from_iter = std::find_if(std::begin(free_blocks_), std::end(free_blocks_), @@ -192,8 +190,12 @@ private: bytes_needed, associated_streams}; + // Allocations are aligned to 256B. new_memory_block's size is exactly bytes_needed, but the remaining + // memory block should start at byte divisible by 256 + const size_t rounded_up_bytes = roundup_allocation(bytes_needed); + // ** reduce the size of the block the memory is going to be taken from - if (block_to_get_memory_from_iter->size == bytes_needed) + if (block_to_get_memory_from_iter->size <= rounded_up_bytes) { // this memory block is completely used, remove it free_blocks_.erase(block_to_get_memory_from_iter); @@ -201,8 +203,8 @@ private: else { // there will still be some memory left, update free block size - block_to_get_memory_from_iter->begin += new_memory_block.size; - block_to_get_memory_from_iter->size -= new_memory_block.size; + block_to_get_memory_from_iter->begin += rounded_up_bytes; + block_to_get_memory_from_iter->size -= rounded_up_bytes; } // ** add new used memory block to the list of used blocks @@ -251,8 +253,18 @@ private: GW_CU_ABORT_ON_ERR(cudaStreamSynchronize(associated_stream)); } + // ** find actual block size + // Allocations are aligned by 256B. block_to_be_freed_iter->size is the exact number of requested bytes, but the block + // is actually allocated that-value-rounded-up-to-the-next-number-divisible-by-256 bytes. + // One exception is the block that goes into the last 256-divisible block of allocated buffer. In that case actually + // allocated memory goes up to the end of the buffer, even in buffer's size if not divisible by 256. In this case number_of_bytes + // is not divisible by 256 but the lenght from the beginning of the block until the end of the buffer + const size_t blocks_last_byte_index = block_to_be_freed_iter->begin + block_to_be_freed_iter->size; + assert(blocks_last_byte_index <= buffer_size_); + const bool block_ends_in_buffers_last_block = roundup_allocation(buffer_size_) - blocks_last_byte_index < 256; + const size_t number_of_bytes = block_ends_in_buffers_last_block ? buffer_size_ - block_to_be_freed_iter->begin : roundup_allocation(block_to_be_freed_iter->size); + // ** remove memory block from the list of used memory blocks - const size_t number_of_bytes = block_to_be_freed_iter->size; used_blocks_.erase(block_to_be_freed_iter); // ** add the block back the list of free blocks (and merge with any neighbouring free blocks) From c7887b5b411c4475a81e23e08792489d0143d649 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Mon, 23 Nov 2020 15:43:38 +0100 Subject: [PATCH 244/281] [common] bytes_needed in DevicePreallocatedAllocator can be const --- .../utils/device_preallocated_allocator.cuh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 1a1d4ac67..49d2f6a6d 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -163,7 +163,7 @@ private: /// \param associated_streams on deallocation this memory block is guaranteed to live at least until all previously scheduled work in these streams has finished /// \return cudaSuccess if allocation was successful, cudaErrorMemoryAllocation otherwise cudaError_t allocate_memory_block(void** ptr, - size_t bytes_needed, + const size_t bytes_needed, const std::vector& associated_streams) { *ptr = nullptr; @@ -345,17 +345,18 @@ private: /// \brief Rounds up requested size to align with allocation boundaries (256B) /// \param bytes_requested /// \return bytes needed for aligning with 256B boundary - inline static size_t roundup_allocation(size_t requested_bytes) + inline static size_t roundup_allocation(const size_t requested_bytes) { + size_t rounded_up_value = requested_bytes; // ** All allocations should be aligned with 256 bytes // The easiest way to do this is to make all allocation request sizes divisible by 256 if ((requested_bytes & 0xFF) != 0) { // bytes needed not divisible by 256, increase it to the next value divisible by 256 - requested_bytes = requested_bytes + (0x100 - (requested_bytes & 0xFF)); + rounded_up_value = requested_bytes + (0x100 - (requested_bytes & 0xFF)); } - assert((requested_bytes & 0xFF) == 0); - return requested_bytes; + assert((rounded_up_value & 0xFF) == 0); + return rounded_up_value; } /// buffer size From aae3a0056a7dbadddf7d836ec0780015a896d54d Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Mon, 23 Nov 2020 18:28:33 +0100 Subject: [PATCH 245/281] [common] Added round_up() to mathutils.hpp --- .../utils/device_preallocated_allocator.cuh | 24 +++----------- .../genomeworks/utils/mathutils.hpp | 31 +++++++++++++++++++ 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 49d2f6a6d..3cb2bee61 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -23,6 +23,7 @@ #include #include +#include namespace claraparabricks { @@ -192,7 +193,7 @@ private: // Allocations are aligned to 256B. new_memory_block's size is exactly bytes_needed, but the remaining // memory block should start at byte divisible by 256 - const size_t rounded_up_bytes = roundup_allocation(bytes_needed); + const size_t rounded_up_bytes = round_up(bytes_needed, 256); // ** reduce the size of the block the memory is going to be taken from if (block_to_get_memory_from_iter->size <= rounded_up_bytes) @@ -261,8 +262,8 @@ private: // is not divisible by 256 but the lenght from the beginning of the block until the end of the buffer const size_t blocks_last_byte_index = block_to_be_freed_iter->begin + block_to_be_freed_iter->size; assert(blocks_last_byte_index <= buffer_size_); - const bool block_ends_in_buffers_last_block = roundup_allocation(buffer_size_) - blocks_last_byte_index < 256; - const size_t number_of_bytes = block_ends_in_buffers_last_block ? buffer_size_ - block_to_be_freed_iter->begin : roundup_allocation(block_to_be_freed_iter->size); + const bool block_ends_in_buffers_last_block = round_up(buffer_size_, 256) - blocks_last_byte_index < 256; + const size_t number_of_bytes = block_ends_in_buffers_last_block ? buffer_size_ - block_to_be_freed_iter->begin : round_up(block_to_be_freed_iter->size, 256); // ** remove memory block from the list of used memory blocks used_blocks_.erase(block_to_be_freed_iter); @@ -342,23 +343,6 @@ private: return cudaSuccess; } - /// \brief Rounds up requested size to align with allocation boundaries (256B) - /// \param bytes_requested - /// \return bytes needed for aligning with 256B boundary - inline static size_t roundup_allocation(const size_t requested_bytes) - { - size_t rounded_up_value = requested_bytes; - // ** All allocations should be aligned with 256 bytes - // The easiest way to do this is to make all allocation request sizes divisible by 256 - if ((requested_bytes & 0xFF) != 0) - { - // bytes needed not divisible by 256, increase it to the next value divisible by 256 - rounded_up_value = requested_bytes + (0x100 - (requested_bytes & 0xFF)); - } - assert((rounded_up_value & 0xFF) == 0); - return rounded_up_value; - } - /// buffer size const size_t buffer_size_; /// preallocated buffer diff --git a/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp b/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp index 3035d8ec9..4590431ff 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp @@ -78,6 +78,37 @@ std::int32_t int_floor_log2(T val) return power; } +/// @brief Rounds up a number to the next number divisible by the given denominator. If the number is already divisible by the denominator it remains the same +/// @param val number to round up +/// @param roundup_denominator +/// @tparam Integer val' type, has to be integer +template +__host__ __device__ Integer round_up(const Integer val, + const uint32_t roundup_denominator) +{ + static_assert(std::is_integral::value, "Expected an integer"); + assert(roundup_denominator > 0); + + const Integer remainder = val % roundup_denominator; + + if (remainder == 0) + { + return val; + } + + if (val > 0) + { + // for value 11 and denomintor 4 remainder is 3 so 11 - 3 + 4 = 8 + 4 = 12 + return val - remainder + roundup_denominator; + } + else + { + // remainder is negative is this case, i.e. for value -11 and denominator 4 remainder is -3, + // so -11 - (-3) = -11 + 3 = -8 + return val - remainder; + } +} + } // namespace genomeworks } // namespace claraparabricks From a99758ca7710b4fe86b57799a700928a80fa6fe2 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Mon, 23 Nov 2020 18:43:07 +0100 Subject: [PATCH 246/281] [common] Alignment in DevicePreallocatedAllocator declared in one place --- .../utils/device_preallocated_allocator.cuh | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 3cb2bee61..9a0966026 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -191,9 +191,9 @@ private: bytes_needed, associated_streams}; - // Allocations are aligned to 256B. new_memory_block's size is exactly bytes_needed, but the remaining - // memory block should start at byte divisible by 256 - const size_t rounded_up_bytes = round_up(bytes_needed, 256); + // Allocations are aligned to alignment_ bytes. new_memory_block's size is exactly bytes_needed, but the remaining + // memory block should start at byte divisible by alignment_ + const size_t rounded_up_bytes = round_up(bytes_needed, alignment_); // ** reduce the size of the block the memory is going to be taken from if (block_to_get_memory_from_iter->size <= rounded_up_bytes) @@ -255,15 +255,15 @@ private: } // ** find actual block size - // Allocations are aligned by 256B. block_to_be_freed_iter->size is the exact number of requested bytes, but the block - // is actually allocated that-value-rounded-up-to-the-next-number-divisible-by-256 bytes. - // One exception is the block that goes into the last 256-divisible block of allocated buffer. In that case actually - // allocated memory goes up to the end of the buffer, even in buffer's size if not divisible by 256. In this case number_of_bytes - // is not divisible by 256 but the lenght from the beginning of the block until the end of the buffer + // Allocations are aligned by alignment_ bytes. block_to_be_freed_iter->size is the exact number of requested bytes, but the block + // is actually allocated that-value-rounded-up-to-the-next-number-divisible-by-alignment_ bytes. + // One exception is the block that goes into the last alignment_-divisible block of allocated buffer. In that case actually + // allocated memory goes up to the end of the buffer, even in buffer's size if not divisible by alignment_. In this case number_of_bytes + // is not divisible by alignment_ but the lenght from the beginning of the block until the end of the buffer const size_t blocks_last_byte_index = block_to_be_freed_iter->begin + block_to_be_freed_iter->size; assert(blocks_last_byte_index <= buffer_size_); - const bool block_ends_in_buffers_last_block = round_up(buffer_size_, 256) - blocks_last_byte_index < 256; - const size_t number_of_bytes = block_ends_in_buffers_last_block ? buffer_size_ - block_to_be_freed_iter->begin : round_up(block_to_be_freed_iter->size, 256); + const bool block_ends_in_buffers_last_block = round_up(buffer_size_, alignment_) - blocks_last_byte_index < alignment_; + const size_t number_of_bytes = block_ends_in_buffers_last_block ? buffer_size_ - block_to_be_freed_iter->begin : round_up(block_to_be_freed_iter->size, alignment_); // ** remove memory block from the list of used memory blocks used_blocks_.erase(block_to_be_freed_iter); @@ -354,6 +354,9 @@ private: std::list free_blocks_; /// list of block in use, sorted by memory block beginning location std::list used_blocks_; + + /// number of bytes to align allocations to + static const uint32_t alignment_ = 256; // if changing alignment update comments as well }; } // namespace genomeworks From a1cf800e70bdce59b2b83be53beb94cc52e14715 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Mon, 23 Nov 2020 18:55:40 +0100 Subject: [PATCH 247/281] [common] Improved comments for DevicePreallocatedAllocator --- .../genomeworks/utils/device_preallocated_allocator.cuh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 9a0966026..37e2dde70 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -33,7 +33,7 @@ namespace genomeworks /// \brief Allocator that preallocates one big buffer of device memory and assigns sections of it to allocation requests /// Allocator allocates one big buffer of device memory during constructor and keeps it until destruction. -/// For every allocation request it linearly scans the preallocated memory and assigns first buffer of it that is big enough. +/// For every allocation request it linearly scans the preallocated buffer and assigns first section of it that is big enough. /// /// For example imagine 100000 bytes buffer has been preallocated and that sections between (10000, 19999), (30000, 39999) /// and (60000, 79999) bytes have already been assigned: @@ -48,8 +48,8 @@ namespace genomeworks /// 0 --- 10000 --- 20000 --- 30000 --- 40000 ------ 55000 - 60000 --- 70000 --- 80000 --- 90000 --- 100000 /// | FREE | USED | FREE | USED | USED | FREE | USED | FREE | /// -/// Uppon returning a memory allocation its gets merged with any neighboring free arey, so if section (10000, 19999) is -/// returned the memory will look like this: +/// Uppon deallocation deallocated memory gets merged with neighboring free blocks, if any, so if section (10000, 19999) is +/// deallocated the memory will look like this: /// /// 0 ---------------------- 30000 --- 40000 ------ 55000 - 60000 --- 70000 --- 80000 --- 90000 --- 100000 /// | FREE | USED | USED | FREE | USED | FREE | @@ -57,7 +57,7 @@ namespace genomeworks /// Allocator aligns memory by 256 bytes /// /// Note that this allocator heavily relies on iterating over lists of free and used memory and should thus -/// not be used for cases when there are millions of allocation at a time +/// not be used when there are many allocations and deallocations class DevicePreallocatedAllocator { public: From 98253189442b2e011f6c067362ff11d374cff226 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Tue, 24 Nov 2020 14:34:47 +0100 Subject: [PATCH 248/281] [common] More explicitly setting ptr to null on error in DevicePreallocatedAllocator::allocate_memory_block() --- .../genomeworks/utils/device_preallocated_allocator.cuh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 37e2dde70..e7ecfe8ff 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -167,8 +167,6 @@ private: const size_t bytes_needed, const std::vector& associated_streams) { - *ptr = nullptr; - if (free_blocks_.empty()) { return cudaErrorMemoryAllocation; @@ -183,6 +181,7 @@ private: if (block_to_get_memory_from_iter == std::end(free_blocks_)) { + *ptr = nullptr; return cudaErrorMemoryAllocation; } From 1caff070755b7efa7762f97f9aa63495535c1c11 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Tue, 24 Nov 2020 14:51:42 +0100 Subject: [PATCH 249/281] [common] DevicePreallocatedAllocator::free_memory_block() actually returns cudaErrorInvalidValue for invalid pointers --- .../utils/device_preallocated_allocator.cuh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index e7ecfe8ff..9990a491c 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -106,7 +106,7 @@ public: /// \brief deallocates memory (returns its part of buffer to the list of free parts) /// This function blocks until all work on associated_stream is done /// \param ptr - /// \return error status + /// \return cudaSuccess if deallocation was successful, cudaErrorInvalidValue otherwise cudaError_t DeviceFree(void* ptr) { cudaError_t status = cudaSuccess; @@ -227,7 +227,7 @@ private: /// \brief returns the block starting at pointer /// This function blocks until all work on associated_streams is done /// \param pointer pointer at the begining of the block to be freed - /// \return error status + /// \return cudaSuccess if deallocation was successful, cudaErrorInvalidValue otherwise cudaError_t free_memory_block(void* pointer) { assert(static_cast(pointer) >= buffer_ptr_.get()); @@ -240,7 +240,12 @@ private: [block_start](const MemoryBlock& memory_block) { return memory_block.begin == block_start; }); - assert(block_to_be_freed_iter != std::end(used_blocks_)); + + // * return error is pointer is not valid + if (block_to_be_freed_iter == std::end(used_blocks_)) + { + return cudaErrorInvalidValue; + } // ** wait for all work on associated_streams to finish before freeing up this memory block assert(!block_to_be_freed_iter->associated_streams.empty()); From 7cf3a48f52a1d43b65955867016738296a37fa02 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Tue, 24 Nov 2020 15:11:47 +0100 Subject: [PATCH 250/281] [common] Added tests for DevicePreallocatedAllocator --- common/base/tests/CMakeLists.txt | 1 + .../tests/Test_DevicePreallocatedAllocator.cu | 280 ++++++++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 common/base/tests/Test_DevicePreallocatedAllocator.cu diff --git a/common/base/tests/CMakeLists.txt b/common/base/tests/CMakeLists.txt index c6c50a8c3..1ed37b7d2 100644 --- a/common/base/tests/CMakeLists.txt +++ b/common/base/tests/CMakeLists.txt @@ -20,6 +20,7 @@ set(TARGET_NAME gwbasetests) set(SOURCES main.cpp + Test_DevicePreallocatedAllocator.cu Test_UtilsCudasort.cu Test_UtilsThreadsafeContainers.cpp TestGraph.cpp diff --git a/common/base/tests/Test_DevicePreallocatedAllocator.cu b/common/base/tests/Test_DevicePreallocatedAllocator.cu new file mode 100644 index 000000000..19c4ca8dd --- /dev/null +++ b/common/base/tests/Test_DevicePreallocatedAllocator.cu @@ -0,0 +1,280 @@ +/* +* Copyright 2019-2020 NVIDIA CORPORATION. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "gtest/gtest.h" + +#include + +#include +#include +#include + +namespace claraparabricks +{ + +namespace genomeworks +{ + +TEST(TestDevicePreallocatedAllocator, allocations_do_not_overlap) +{ + CudaStream cuda_stream = make_cuda_stream(); + + std::vector cuda_streams; + cuda_streams.push_back(cuda_stream.get()); + + DevicePreallocatedAllocator allocator(2000); + // 0 - 1999: free + + cudaError status; + + void* pointer_from_0_to_999_actually_to_1023 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_0_to_999_actually_to_1023, 1000, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1023: taken + // 1024 - 1999: free + + void* pointer_from_1024_to_1523_actually_to_1535 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1024_to_1523_actually_to_1535, 500, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1535: taken + // 1536 - 1999: free + ASSERT_EQ(static_cast(pointer_from_1024_to_1523_actually_to_1535) - static_cast(pointer_from_0_to_999_actually_to_1023), 1024); + + void* pointer_from_1536_to_1537_actually_to_1791 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1536_to_1537_actually_to_1791, 2, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1791: taken + // 1792 - 1999: free + ASSERT_EQ(static_cast(pointer_from_1536_to_1537_actually_to_1791) - static_cast(pointer_from_1024_to_1523_actually_to_1535), 512); + + void* pointer_from_1792_to_1999_actually_to_1999 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1792_to_1999_actually_to_1999, 208, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1999: taken + ASSERT_EQ(static_cast(pointer_from_1792_to_1999_actually_to_1999) - static_cast(pointer_from_1536_to_1537_actually_to_1791), 256); + + status = allocator.DeviceFree(pointer_from_1792_to_1999_actually_to_1999); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_1536_to_1537_actually_to_1791); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_1024_to_1523_actually_to_1535); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_0_to_999_actually_to_1023); + ASSERT_EQ(status, cudaSuccess); +} + +TEST(TestDevicePreallocatedAllocator, memory_correctly_deallocated) +{ + CudaStream cuda_stream = make_cuda_stream(); + + std::vector cuda_streams; + cuda_streams.push_back(cuda_stream.get()); + + DevicePreallocatedAllocator allocator(2000); + // 0 - 1999: free + + cudaError status; + + void* pointer_from_0_to_999_actually_to_1023 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_0_to_999_actually_to_1023, 1000, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1023: taken + // 1024 - 19999: free + + void* pointer_from_1024_to_1523_actually_to_1535 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1024_to_1523_actually_to_1535, 500, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1535: taken + // 1536 - 19999: free + ASSERT_EQ(static_cast(pointer_from_1024_to_1523_actually_to_1535) - static_cast(pointer_from_0_to_999_actually_to_1023), 1024); + + void* pointer_from_1536_to_1537_actually_to_1791 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1536_to_1537_actually_to_1791, 2, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1791: taken + // 1792 - 19999: free + ASSERT_EQ(static_cast(pointer_from_1536_to_1537_actually_to_1791) - static_cast(pointer_from_1024_to_1523_actually_to_1535), 512); + + status = allocator.DeviceFree(pointer_from_1024_to_1523_actually_to_1535); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1023: taken + // 1024 - 1535: free + // 1536 - 1791: taken + // 1792 - 1999: free + + void* pointer_from_1024_to_1027_actually_1279 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1024_to_1027_actually_1279, 4, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1279: taken + // 1280 - 1535: free + // 1535 - 1791: taken + // 1792 - 1999: free + ASSERT_EQ(static_cast(pointer_from_1024_to_1027_actually_1279) - static_cast(pointer_from_0_to_999_actually_to_1023), 1024); + + void* pointer_from_1280_to_1300_actually_1535 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1280_to_1300_actually_1535, 21, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1791: taken + // 1792 - 1999: free + ASSERT_EQ(static_cast(pointer_from_1280_to_1300_actually_1535) - static_cast(pointer_from_1024_to_1027_actually_1279), 256); + + void* pointer_from_1792_to_1800_actually_1999 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1792_to_1800_actually_1999, 9, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1999: taken + ASSERT_EQ(static_cast(pointer_from_1792_to_1800_actually_1999) - static_cast(pointer_from_1536_to_1537_actually_to_1791), 256); + + status = allocator.DeviceFree(pointer_from_1280_to_1300_actually_1535); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_1024_to_1027_actually_1279); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_1536_to_1537_actually_to_1791); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_0_to_999_actually_to_1023); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1999: free + + void* pointer_from_0_to_199_actually_255 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_0_to_199_actually_255, 200, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 255: taken + // 256 - 1999: free + + void* pointer_from_256_to_260_actually_511 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_256_to_260_actually_511, 5, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 511: take + // 512 - 1999: free + ASSERT_EQ(static_cast(pointer_from_256_to_260_actually_511) - static_cast(pointer_from_0_to_199_actually_255), 256); + + void* pointer_from_512_to_515_actually_767 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_512_to_515_actually_767, 4, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 767: take + // 768 - 1999: free + ASSERT_EQ(static_cast(pointer_from_512_to_515_actually_767) - static_cast(pointer_from_256_to_260_actually_511), 256); + + status = allocator.DeviceFree(pointer_from_256_to_260_actually_511); + ASSERT_EQ(status, cudaSuccess); + // 0 - 255: taken + // 256 - 511: free + // 512 - 767: taken + // 768 - 1999: free + + void* pointer_from_768_to_1067_actually_1279 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_768_to_1067_actually_1279, 300, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 255: taken + // 256 - 511: free + // 512 - 1279: taken + // 1280 - 1999: free + ASSERT_EQ(static_cast(pointer_from_768_to_1067_actually_1279) - static_cast(pointer_from_512_to_515_actually_767), 256); + + void* pointer_from_256_to_270_actually_511 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_256_to_270_actually_511, 15, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1279: taken + // 1280 - 1999: free + ASSERT_EQ(static_cast(pointer_from_256_to_270_actually_511) - static_cast(pointer_from_0_to_199_actually_255), 256); + + void* pointer_from_1280_to_1290_actually_1535 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1280_to_1290_actually_1535, 11, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + // 0 - 1535: taken + // 1536 - 1999: free + ASSERT_EQ(static_cast(pointer_from_1280_to_1290_actually_1535) - static_cast(pointer_from_768_to_1067_actually_1279), 512); + + status = allocator.DeviceFree(pointer_from_1280_to_1290_actually_1535); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_768_to_1067_actually_1279); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_512_to_515_actually_767); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_256_to_270_actually_511); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_0_to_199_actually_255); + ASSERT_EQ(status, cudaSuccess); +} + +TEST(TestDevicePreallocatedAllocator, no_memory_left) +{ + CudaStream cuda_stream = make_cuda_stream(); + + std::vector cuda_streams; + cuda_streams.push_back(cuda_stream.get()); + + DevicePreallocatedAllocator allocator(2000); + // 0 - 1999: free + + cudaError status; + + void* pointer_from_0_to_1499_actually_1535 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_0_to_1499_actually_1535, 1500, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + ASSERT_NE(pointer_from_0_to_1499_actually_1535, nullptr); + // 0 - 1535: taken + // 1536 - 1999: free + + void* pointer_from_1536_to_2000_actually_error = pointer_from_0_to_1499_actually_1535; // initially set to some value to make sure allocator.DeviceAllocate() sets it to nullptr if allocation was not successful + status = allocator.DeviceAllocate(&pointer_from_1536_to_2000_actually_error, 465, cuda_streams); + ASSERT_EQ(status, cudaErrorMemoryAllocation); + ASSERT_EQ(pointer_from_1536_to_2000_actually_error, nullptr); + // 0 - 1535: taken + // 1536 - 1999: free + + void* pointer_from_1536_to_1999_actually_1999 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1536_to_1999_actually_1999, 464, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + ASSERT_NE(pointer_from_1536_to_1999_actually_1999, nullptr); + // 0 - 1999: taken + + status = allocator.DeviceFree(pointer_from_1536_to_1999_actually_1999); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_0_to_1499_actually_1535); + ASSERT_EQ(status, cudaSuccess); +} + +TEST(TestDevicePreallocatedAllocator, deallocating_invalid_pointer) +{ + CudaStream cuda_stream = make_cuda_stream(); + + std::vector cuda_streams; + cuda_streams.push_back(cuda_stream.get()); + + DevicePreallocatedAllocator allocator(2000); + + cudaError status; + + void* valid_ptr = nullptr; + status = allocator.DeviceAllocate(&valid_ptr, 1500, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + ASSERT_NE(valid_ptr, nullptr); + + void* invalid_ptr = static_cast(static_cast(valid_ptr) + 10); + status = allocator.DeviceFree(invalid_ptr); + ASSERT_EQ(status, cudaErrorInvalidValue); + + void* null_ptr = nullptr; + status = allocator.DeviceFree(null_ptr); + ASSERT_EQ(status, cudaSuccess); + + status = allocator.DeviceFree(valid_ptr); + ASSERT_EQ(status, cudaSuccess); +} + +} // namespace genomeworks + +} // namespace claraparabricks From ff72180511369503d63692df5cddd586f82abdc6 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 26 Nov 2020 10:40:36 +0100 Subject: [PATCH 251/281] [common] Improved comments and fixed typos in DevicePreallocatedAllocator and Test_DevicePreallocatedAllocator --- .../utils/device_preallocated_allocator.cuh | 16 ++++++++-------- .../tests/Test_DevicePreallocatedAllocator.cu | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 9990a491c..f10599922 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -48,7 +48,7 @@ namespace genomeworks /// 0 --- 10000 --- 20000 --- 30000 --- 40000 ------ 55000 - 60000 --- 70000 --- 80000 --- 90000 --- 100000 /// | FREE | USED | FREE | USED | USED | FREE | USED | FREE | /// -/// Uppon deallocation deallocated memory gets merged with neighboring free blocks, if any, so if section (10000, 19999) is +/// Upon deallocation deallocated memory gets merged with neighboring free blocks, if any, so if section (10000, 19999) is /// deallocated the memory will look like this: /// /// 0 ---------------------- 30000 --- 40000 ------ 55000 - 60000 --- 70000 --- 80000 --- 90000 --- 100000 @@ -190,8 +190,8 @@ private: bytes_needed, associated_streams}; - // Allocations are aligned to alignment_ bytes. new_memory_block's size is exactly bytes_needed, but the remaining - // memory block should start at byte divisible by alignment_ + // Allocations are aligned to alignment_ bytes. new_memory_block's size is exactly bytes_needed, but the part of + // the original memory block which remais unallocated should start at byte divisible by alignment_ const size_t rounded_up_bytes = round_up(bytes_needed, alignment_); // ** reduce the size of the block the memory is going to be taken from @@ -224,10 +224,10 @@ private: return cudaSuccess; } - /// \brief returns the block starting at pointer + /// \brief releases the block starting at pointer /// This function blocks until all work on associated_streams is done /// \param pointer pointer at the begining of the block to be freed - /// \return cudaSuccess if deallocation was successful, cudaErrorInvalidValue otherwise + /// \return cudaSuccess if release was successful, cudaErrorInvalidValue otherwise cudaError_t free_memory_block(void* pointer) { assert(static_cast(pointer) >= buffer_ptr_.get()); @@ -241,7 +241,7 @@ private: return memory_block.begin == block_start; }); - // * return error is pointer is not valid + // * return error if pointer is not valid if (block_to_be_freed_iter == std::end(used_blocks_)) { return cudaErrorInvalidValue; @@ -262,8 +262,8 @@ private: // Allocations are aligned by alignment_ bytes. block_to_be_freed_iter->size is the exact number of requested bytes, but the block // is actually allocated that-value-rounded-up-to-the-next-number-divisible-by-alignment_ bytes. // One exception is the block that goes into the last alignment_-divisible block of allocated buffer. In that case actually - // allocated memory goes up to the end of the buffer, even in buffer's size if not divisible by alignment_. In this case number_of_bytes - // is not divisible by alignment_ but the lenght from the beginning of the block until the end of the buffer + // allocated memory goes up to the end of the buffer, even if buffer's size is not divisible by alignment_. In this case number_of_bytes + // is not divisible by alignment_ but the length from the beginning of the block until the end of the buffer const size_t blocks_last_byte_index = block_to_be_freed_iter->begin + block_to_be_freed_iter->size; assert(blocks_last_byte_index <= buffer_size_); const bool block_ends_in_buffers_last_block = round_up(buffer_size_, alignment_) - blocks_last_byte_index < alignment_; diff --git a/common/base/tests/Test_DevicePreallocatedAllocator.cu b/common/base/tests/Test_DevicePreallocatedAllocator.cu index 19c4ca8dd..688630151 100644 --- a/common/base/tests/Test_DevicePreallocatedAllocator.cu +++ b/common/base/tests/Test_DevicePreallocatedAllocator.cu @@ -92,20 +92,20 @@ TEST(TestDevicePreallocatedAllocator, memory_correctly_deallocated) status = allocator.DeviceAllocate(&pointer_from_0_to_999_actually_to_1023, 1000, cuda_streams); ASSERT_EQ(status, cudaSuccess); // 0 - 1023: taken - // 1024 - 19999: free + // 1024 - 1999: free void* pointer_from_1024_to_1523_actually_to_1535 = nullptr; status = allocator.DeviceAllocate(&pointer_from_1024_to_1523_actually_to_1535, 500, cuda_streams); ASSERT_EQ(status, cudaSuccess); // 0 - 1535: taken - // 1536 - 19999: free + // 1536 - 1999: free ASSERT_EQ(static_cast(pointer_from_1024_to_1523_actually_to_1535) - static_cast(pointer_from_0_to_999_actually_to_1023), 1024); void* pointer_from_1536_to_1537_actually_to_1791 = nullptr; status = allocator.DeviceAllocate(&pointer_from_1536_to_1537_actually_to_1791, 2, cuda_streams); ASSERT_EQ(status, cudaSuccess); // 0 - 1791: taken - // 1792 - 19999: free + // 1792 - 1999: free ASSERT_EQ(static_cast(pointer_from_1536_to_1537_actually_to_1791) - static_cast(pointer_from_1024_to_1523_actually_to_1535), 512); status = allocator.DeviceFree(pointer_from_1024_to_1523_actually_to_1535); From b0e5b00658561eb7608b428d23bd4443aa9c62f9 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 26 Nov 2020 10:57:10 +0100 Subject: [PATCH 252/281] [common] Moved DevicePreallocatedAllocator into detils namespace --- .../claraparabricks/genomeworks/utils/allocator.hpp | 4 ++-- .../claraparabricks/genomeworks/utils/device_buffer.hpp | 2 +- .../genomeworks/utils/device_preallocated_allocator.cuh | 5 +++++ common/base/tests/Test_DevicePreallocatedAllocator.cu | 8 ++++---- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp index c2d0d0e4c..a448f9022 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/allocator.hpp @@ -157,7 +157,7 @@ class CudaMallocAllocator /// \brief A simple caching allocator for device memory allocations /// \tparam T -/// \tparam MemoryResource resource that does actual allocation, e.g. cudautils::DevicePreallocatedAllocator +/// \tparam MemoryResource resource that does actual allocation, e.g. cudautils::details::DevicePreallocatedAllocator template class CachingDeviceAllocator { @@ -320,7 +320,7 @@ class CachingDeviceAllocator }; #ifdef GW_ENABLE_CACHING_ALLOCATOR -using DefaultDeviceAllocator = CachingDeviceAllocator; +using DefaultDeviceAllocator = CachingDeviceAllocator; #else using DefaultDeviceAllocator = CudaMallocAllocator; #endif diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_buffer.hpp b/common/base/include/claraparabricks/genomeworks/utils/device_buffer.hpp index 83c9a48f2..90d8a4db1 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_buffer.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/device_buffer.hpp @@ -215,7 +215,7 @@ class buffer template #ifdef GW_ENABLE_CACHING_ALLOCATOR -using device_buffer = details::buffer>; +using device_buffer = details::buffer>; #else using device_buffer = details::buffer>; #endif diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index f10599922..bca7099e4 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -31,6 +31,9 @@ namespace claraparabricks namespace genomeworks { +namespace details +{ + /// \brief Allocator that preallocates one big buffer of device memory and assigns sections of it to allocation requests /// Allocator allocates one big buffer of device memory during constructor and keeps it until destruction. /// For every allocation request it linearly scans the preallocated buffer and assigns first section of it that is big enough. @@ -363,6 +366,8 @@ private: static const uint32_t alignment_ = 256; // if changing alignment update comments as well }; +} // namespace details + } // namespace genomeworks } // namespace claraparabricks diff --git a/common/base/tests/Test_DevicePreallocatedAllocator.cu b/common/base/tests/Test_DevicePreallocatedAllocator.cu index 688630151..3f1280b4c 100644 --- a/common/base/tests/Test_DevicePreallocatedAllocator.cu +++ b/common/base/tests/Test_DevicePreallocatedAllocator.cu @@ -35,7 +35,7 @@ TEST(TestDevicePreallocatedAllocator, allocations_do_not_overlap) std::vector cuda_streams; cuda_streams.push_back(cuda_stream.get()); - DevicePreallocatedAllocator allocator(2000); + details::DevicePreallocatedAllocator allocator(2000); // 0 - 1999: free cudaError status; @@ -83,7 +83,7 @@ TEST(TestDevicePreallocatedAllocator, memory_correctly_deallocated) std::vector cuda_streams; cuda_streams.push_back(cuda_stream.get()); - DevicePreallocatedAllocator allocator(2000); + details::DevicePreallocatedAllocator allocator(2000); // 0 - 1999: free cudaError status; @@ -216,7 +216,7 @@ TEST(TestDevicePreallocatedAllocator, no_memory_left) std::vector cuda_streams; cuda_streams.push_back(cuda_stream.get()); - DevicePreallocatedAllocator allocator(2000); + details::DevicePreallocatedAllocator allocator(2000); // 0 - 1999: free cudaError status; @@ -254,7 +254,7 @@ TEST(TestDevicePreallocatedAllocator, deallocating_invalid_pointer) std::vector cuda_streams; cuda_streams.push_back(cuda_stream.get()); - DevicePreallocatedAllocator allocator(2000); + details::DevicePreallocatedAllocator allocator(2000); cudaError status; From f39e352f98e1edc0cc4001c26a09a0b52f3bfffe Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 26 Nov 2020 11:56:42 +0100 Subject: [PATCH 253/281] [common] Simplified finfing block size in DevicePreallocatedAllocator::free_memory_block() --- .../genomeworks/utils/device_preallocated_allocator.cuh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index bca7099e4..ea792367c 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -267,10 +267,7 @@ private: // One exception is the block that goes into the last alignment_-divisible block of allocated buffer. In that case actually // allocated memory goes up to the end of the buffer, even if buffer's size is not divisible by alignment_. In this case number_of_bytes // is not divisible by alignment_ but the length from the beginning of the block until the end of the buffer - const size_t blocks_last_byte_index = block_to_be_freed_iter->begin + block_to_be_freed_iter->size; - assert(blocks_last_byte_index <= buffer_size_); - const bool block_ends_in_buffers_last_block = round_up(buffer_size_, alignment_) - blocks_last_byte_index < alignment_; - const size_t number_of_bytes = block_ends_in_buffers_last_block ? buffer_size_ - block_to_be_freed_iter->begin : round_up(block_to_be_freed_iter->size, alignment_); + const size_t number_of_bytes = std::min(round_up(block_to_be_freed_iter->size, alignment_), buffer_size_ - block_to_be_freed_iter->begin); // ** remove memory block from the list of used memory blocks used_blocks_.erase(block_to_be_freed_iter); From 26da05898e1bef33a200a2aedae049c66b961328 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 26 Nov 2020 11:59:52 +0100 Subject: [PATCH 254/281] [common] Renamed roundup() into roundup_next_multiple() --- .../genomeworks/utils/device_preallocated_allocator.cuh | 4 ++-- .../include/claraparabricks/genomeworks/utils/mathutils.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index ea792367c..7b5be697d 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -195,7 +195,7 @@ private: // Allocations are aligned to alignment_ bytes. new_memory_block's size is exactly bytes_needed, but the part of // the original memory block which remais unallocated should start at byte divisible by alignment_ - const size_t rounded_up_bytes = round_up(bytes_needed, alignment_); + const size_t rounded_up_bytes = roundup_next_multiple(bytes_needed, alignment_); // ** reduce the size of the block the memory is going to be taken from if (block_to_get_memory_from_iter->size <= rounded_up_bytes) @@ -267,7 +267,7 @@ private: // One exception is the block that goes into the last alignment_-divisible block of allocated buffer. In that case actually // allocated memory goes up to the end of the buffer, even if buffer's size is not divisible by alignment_. In this case number_of_bytes // is not divisible by alignment_ but the length from the beginning of the block until the end of the buffer - const size_t number_of_bytes = std::min(round_up(block_to_be_freed_iter->size, alignment_), buffer_size_ - block_to_be_freed_iter->begin); + const size_t number_of_bytes = std::min(roundup_next_multiple(block_to_be_freed_iter->size, alignment_), buffer_size_ - block_to_be_freed_iter->begin); // ** remove memory block from the list of used memory blocks used_blocks_.erase(block_to_be_freed_iter); diff --git a/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp b/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp index 4590431ff..798ee53e0 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp @@ -83,8 +83,8 @@ std::int32_t int_floor_log2(T val) /// @param roundup_denominator /// @tparam Integer val' type, has to be integer template -__host__ __device__ Integer round_up(const Integer val, - const uint32_t roundup_denominator) +__host__ __device__ Integer roundup_next_multiple(const Integer val, + const uint32_t roundup_denominator) { static_assert(std::is_integral::value, "Expected an integer"); assert(roundup_denominator > 0); From a9e15baff1fd58d09f3a50a888a11ad854c3605e Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 26 Nov 2020 12:33:24 +0100 Subject: [PATCH 255/281] [common] roundup_next_multiple's roundup_denominator is now int32_t --- .../genomeworks/utils/device_preallocated_allocator.cuh | 4 ++-- .../claraparabricks/genomeworks/utils/mathutils.hpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 7b5be697d..86423b44f 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -194,7 +194,7 @@ private: associated_streams}; // Allocations are aligned to alignment_ bytes. new_memory_block's size is exactly bytes_needed, but the part of - // the original memory block which remais unallocated should start at byte divisible by alignment_ + // the original memory block which remains unallocated should start at byte divisible by alignment_ const size_t rounded_up_bytes = roundup_next_multiple(bytes_needed, alignment_); // ** reduce the size of the block the memory is going to be taken from @@ -360,7 +360,7 @@ private: std::list used_blocks_; /// number of bytes to align allocations to - static const uint32_t alignment_ = 256; // if changing alignment update comments as well + static const int32_t alignment_ = 256; // if changing alignment update comments as well }; } // namespace details diff --git a/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp b/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp index 798ee53e0..ffda2a155 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp @@ -80,14 +80,14 @@ std::int32_t int_floor_log2(T val) /// @brief Rounds up a number to the next number divisible by the given denominator. If the number is already divisible by the denominator it remains the same /// @param val number to round up -/// @param roundup_denominator -/// @tparam Integer val' type, has to be integer +/// @param roundup_denominator has to be positive +/// @tparam Integer has to be integer template __host__ __device__ Integer roundup_next_multiple(const Integer val, - const uint32_t roundup_denominator) + int32_t roundup_denominator) { static_assert(std::is_integral::value, "Expected an integer"); - assert(roundup_denominator > 0); + assert(roundup_denominator >= 0); const Integer remainder = val % roundup_denominator; From e2493a49d901ba63fde70c2f2a025271811084e8 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Thu, 26 Nov 2020 13:11:50 +0100 Subject: [PATCH 256/281] [common] roundup_next_multiple()'s roundup_denominator has to be positive and not non-negative --- .../include/claraparabricks/genomeworks/utils/mathutils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp b/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp index ffda2a155..c7be754a6 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp +++ b/common/base/include/claraparabricks/genomeworks/utils/mathutils.hpp @@ -87,7 +87,7 @@ __host__ __device__ Integer roundup_next_multiple(const Integer val, int32_t roundup_denominator) { static_assert(std::is_integral::value, "Expected an integer"); - assert(roundup_denominator >= 0); + assert(roundup_denominator > 0); const Integer remainder = val % roundup_denominator; From 94f05a88096811516edafd636c3fd0c91c8a8e00 Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 26 Nov 2020 13:58:15 -0500 Subject: [PATCH 257/281] [cudapoa] added a couple of tests for verifying static and adaptive banded NW against full-alignment --- cudapoa/src/cudapoa_nw.cuh | 68 +++++----- cudapoa/src/cudapoa_nw_banded.cuh | 116 +++++++++++++++++ cudapoa/tests/Test_CudapoaGenerateMSA2.cu | 3 +- cudapoa/tests/Test_CudapoaNW.cu | 148 +++++++++++++++++++++- 4 files changed, 298 insertions(+), 37 deletions(-) diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index 76bdc4474..d8cfa0232 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -453,6 +453,8 @@ __device__ __forceinline__ return aligned_nodes; } +// global kernel used in testing, hence uses int16_t for SizeT and ScoreT, +// may need to change if test inputs change to long reads template __global__ void runNeedlemanWunschKernel(uint8_t* nodes, SizeT* graph, @@ -472,22 +474,22 @@ __global__ void runNeedlemanWunschKernel(uint8_t* nodes, int32_t match_score, SizeT* aligned_nodes) { - *aligned_nodes = runNeedlemanWunsch(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_length, - scores, - scores_width, - alignment_graph, - alignment_read, - gap_score, - mismatch_score, - match_score); + *aligned_nodes = runNeedlemanWunsch(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + scores_width, + alignment_graph, + alignment_read, + gap_score, + mismatch_score, + match_score); } // Host function that calls the kernel @@ -510,23 +512,23 @@ void runNW(uint8_t* nodes, int32_t match_score, SizeT* aligned_nodes) { - runNeedlemanWunschKernel<<<1, 64>>>(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_length, - scores, - scores_width, - alignment_graph, - alignment_read, - gap_score, - mismatch_score, - match_score, - aligned_nodes); + runNeedlemanWunschKernel<<<1, CUDAPOA_THREADS_PER_BLOCK>>>(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + scores_width, + alignment_graph, + alignment_read, + gap_score, + mismatch_score, + match_score, + aligned_nodes); GW_CU_CHECK_ERR(cudaPeekAtLastError()); } diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 1b4218277..e60b09cfe 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -556,6 +556,122 @@ __device__ __forceinline__ return aligned_nodes; } +// global kernel used in testing, hence uses int16_t for SizeT and ScoreT, +// may need to change if test inputs change to long reads +template +__global__ void runNeedlemanWunschBandedKernel(uint8_t* nodes, + SizeT* graph, + SizeT* node_id_to_pos, + int32_t graph_count, + uint16_t* incoming_edge_count, + SizeT* incoming_edges, + uint16_t* outgoing_edge_count, + uint8_t* read, + int32_t read_length, + int16_t* scores, + int32_t scores_width, + int32_t max_nodes_per_graph, + SizeT* alignment_graph, + SizeT* alignment_read, + int32_t band_width, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, + SizeT* aligned_nodes, + bool adaptive) +{ + float banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_width); + + if (adaptive) + { + *aligned_nodes = runNeedlemanWunschBanded(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + band_width, + gap_score, + mismatch_score, + match_score, + 0); + } + else + { + *aligned_nodes = runNeedlemanWunschBanded(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + band_width, + gap_score, + mismatch_score, + match_score, + 0); + } +} + +// Host function that calls the kernel +template +void runNWbanded(uint8_t* nodes, + SizeT* graph, + SizeT* node_id_to_pos, + int32_t graph_count, + uint16_t* incoming_edge_count, + SizeT* incoming_edges, + uint16_t* outgoing_edge_count, + uint8_t* read, + int32_t read_length, + int16_t* scores, + int32_t scores_width, + int32_t max_nodes_per_graph, + SizeT* alignment_graph, + SizeT* alignment_read, + int32_t band_width, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, + SizeT* aligned_nodes, + bool adaptive) +{ + runNeedlemanWunschBandedKernel<<<1, CUDAPOA_BANDED_THREADS_PER_BLOCK>>>(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + scores_width, + max_nodes_per_graph, + alignment_graph, + alignment_read, + band_width, + gap_score, + mismatch_score, + match_score, + aligned_nodes, + adaptive); + GW_CU_CHECK_ERR(cudaPeekAtLastError()); +} + } // namespace cudapoa } // namespace genomeworks diff --git a/cudapoa/tests/Test_CudapoaGenerateMSA2.cu b/cudapoa/tests/Test_CudapoaGenerateMSA2.cu index 07d767c1c..2567c4c06 100644 --- a/cudapoa/tests/Test_CudapoaGenerateMSA2.cu +++ b/cudapoa/tests/Test_CudapoaGenerateMSA2.cu @@ -117,9 +117,8 @@ TEST_F(MSATest, CudapoaMSA) auto spoa_msa = spoa_generate_multiple_sequence_alignments(sequences); #ifndef SPOA_ACCURATE - for (int32_t i = 0; i < spoa_msa.size(); i++) + for (size_t i = 0; i < spoa_msa.size(); i++) { - std::string msa = cudapoa_msa[0][i]; msa.erase(std::remove(msa.begin(), msa.end(), '-'), msa.end()); ASSERT_EQ(msa, sequences[i]); diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index da6d8fbeb..c9235d317 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -20,6 +20,7 @@ #include //GW_CU_CHECK_ERR #include //array_to_string #include //get_size +#include #include "gtest/gtest.h" @@ -185,7 +186,7 @@ std::vector getNWTestCases() return test_cases; } -// host function for calling the kernel to test topsort device function. +// host function for calling the kernel to test full-band NW device function. NWAnswer testNW(const BasicNW& obj) { typedef int16_t SizeT; @@ -251,7 +252,7 @@ NWAnswer testNW(const BasicNW& obj) read, read_count, scores, - BatchConfig().matrix_sequence_dimension, + batch_size.matrix_sequence_dimension, alignment_graph, alignment_read, gap_score, @@ -304,6 +305,149 @@ TEST_P(NWTest, TestNWCorrectness) INSTANTIATE_TEST_SUITE_P(TestNW, NWTest, ValuesIn(getNWTestCases())); +//--------------------------------------------------------------------------------------- + +// host function for calling the kernel to test static/adaptive-band NW device functions +NWAnswer testNWbanded(const BasicNW& obj, bool adaptive) +{ + typedef int16_t SizeT; + + //declare device buffer + uint8_t* nodes; + SizeT* graph; + SizeT* node_id_to_pos; + SizeT graph_count; //local + uint16_t* incoming_edge_count; + SizeT* incoming_edges; + uint16_t* outgoing_edge_count; + SizeT* outgoing_edges; + uint8_t* read; + uint16_t read_count; //local + int16_t* scores; + SizeT* alignment_graph; + SizeT* alignment_read; + int32_t gap_score; + int32_t mismatch_score; + int32_t match_score; + SizeT* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) + BatchConfig batch_size(1024 /*max_sequence_size*/, 2 /*max_sequences_per_poa*/, 128 /*= band_width*/, + adaptive ? BandMode::adaptive_band : BandMode::static_band /*band-mode*/); + + //allocate unified memory so they can be accessed by both host and device. + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&read, batch_size.max_sequence_size * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_nodes_per_graph * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&aligned_nodes, sizeof(SizeT))); + + //initialize all 'count' buffers + memset((void**)incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset((void**)outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset((void**)node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(SizeT)); + memset((void**)scores, 0, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t)); + + //calculate edge counts on host + obj.get_graph_buffers(incoming_edges, incoming_edge_count, + outgoing_edges, outgoing_edge_count, + nodes, &graph_count, + graph, node_id_to_pos); + obj.get_read_buffers(read, &read_count); + gap_score = BasicNW::gap_score_; + mismatch_score = BasicNW::mismatch_score_; + match_score = BasicNW::match_score_; + + //call the host wrapper of nw kernel + runNWbanded(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_count, + scores, + batch_size.matrix_sequence_dimension, + batch_size.max_nodes_per_graph, + alignment_graph, + alignment_read, + batch_size.alignment_band_width, + gap_score, + mismatch_score, + match_score, + aligned_nodes, + adaptive); + + GW_CU_CHECK_ERR(cudaDeviceSynchronize()); + + //input and output buffers are the same ones in unified memory, so the results are updated in place + //results are stored in alignment_graph and alignment_read; return string representation of those + auto res = std::make_pair(genomeworks::stringutils::array_to_string(alignment_graph, *aligned_nodes, ","), + genomeworks::stringutils::array_to_string(alignment_read, *aligned_nodes, ",")); + + GW_CU_CHECK_ERR(cudaFree(nodes)); + GW_CU_CHECK_ERR(cudaFree(graph)); + GW_CU_CHECK_ERR(cudaFree(node_id_to_pos)); + GW_CU_CHECK_ERR(cudaFree(incoming_edges)); + GW_CU_CHECK_ERR(cudaFree(incoming_edge_count)); + GW_CU_CHECK_ERR(cudaFree(outgoing_edges)); + GW_CU_CHECK_ERR(cudaFree(outgoing_edge_count)); + GW_CU_CHECK_ERR(cudaFree(scores)); + GW_CU_CHECK_ERR(cudaFree(alignment_graph)); + GW_CU_CHECK_ERR(cudaFree(read)); + GW_CU_CHECK_ERR(cudaFree(alignment_read)); + GW_CU_CHECK_ERR(cudaFree(aligned_nodes)); + + return res; +} + +class NWbandedTest : public ::testing::Test +{ +public: + BasicNW* nw; //({'A', 'A', 'A', 'A'}, {0, 1, 2, 3}, {{1}, {2}, {3}, {}}, {'A', 'A', 'T', 'A'}); +public: + NWbandedTest(){}; + void SetUp() + { + // initialize nw graph and read with the following data + std::string nodes_str = "TTTAACCTAATAAATCAGTGAAGATTTAAAATATGATAATTATTGATTTTGGTGAGAGTGCAAAGAAATTTGTTACCCTCATAAGCTGAGCAGACAGATAAGATAGAAAAACAGAAGATAGAATATTAAAACCATGATAGGTACAGACTGAAAAATTCTTGGATAAATATTAAAATTTAGGCTTTAGTAGTAGATTGATGACTGTGAGGAAAAAGGATGTCCAATTGTTGAGTGACATGTAGAATGCCTTAAAATAATTTTACACGTCACTGAAAGCTATATTTATATTCAGGAAGGATATATCCCAGTCATGATTTTCTTAATAAGTTGCCCCATTTTCCAAGTTTAGCTAATTAACATTTATGTCTTCTATAATCAGGAATAGTCATTAACTGACACAGAAACAATTGGAAGCATATGTAGCCAAAAACATAAAAATTATTGCATCCAAATAATGATAAAGTAAAATATTAAAAAATATAGTCTTCTAAAT"; + std::string read_str = "TTTCACCTAGAAAATCAGTGAAGATTTAACAAAAAAAAAAAAAAAAAAAAAAAAATATTGATAATTATTGATTTTGGTGAGAGTGCAAAGCAATTGGCTACCCTCATAAGCTGAGCAGAAGATAAGATAGACAACAGAAGATAGAATAGTTAAACCATGATAGGTACAGACTGCAAAAAAATTCGATAAATATTAAAATTTAGGGCTTTAGTATATATTGATGACTGAGAAAAATCGTGATGTGCAATTGTGCGTGACATGTAGAATTGCCTTAAATAAAATTTAATCTGTCACTGAAGCTATATTTATATTCAGGAAGGATATATCCCAGTCATTGCTTTTCTTAATAAGTGCCCATGTTCCAAGTTTAGCCTAATTAAAAACTTTATGTCTTCTATATCAGAATAGTCATTAATGCACAGAAACAATTTGCGAAGGCATTATGTAGCAAAAACATAAAAAATTATTGCAGCCAAATAATGAATAAAAGTAACACAATCATTTAAAAAAATTATTATGTACTTCTAAAC"; + // extract data to build BasicNW + std::vector nodes(nodes_str.begin(), nodes_str.end()); + std::vector sorted_graph(nodes.size()); + std::iota(sorted_graph.begin(), sorted_graph.end(), 0); + SizeTVec2D outgoing_edges(nodes.size()); + for (size_t i = 0; i < outgoing_edges.size() - 1; i++) + { + outgoing_edges[i].push_back(i + 1); + } + std::vector read(read_str.begin(), read_str.end()); + // + nw = new BasicNW(nodes, sorted_graph, outgoing_edges, read); + } + + void TearDown() { delete nw; } +}; + +TEST_F(NWbandedTest, NWSaticBandvsFull) +{ + auto str1 = testNW(*nw).second; + auto str2 = testNWbanded(*nw, false).second; + EXPECT_EQ(testNW(*nw).second, testNWbanded(*nw, false).second); +} + +TEST_F(NWbandedTest, NWAdaptiveBandvsFull) +{ + EXPECT_EQ(testNW(*nw).second, testNWbanded(*nw, true).second); +} + } // namespace cudapoa } // namespace genomeworks From e8fe6340bc0850db232ce8130cfbaa36f1db755f Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 26 Nov 2020 14:47:59 -0500 Subject: [PATCH 258/281] [cudapoa] slightly modified banded NW test to check both alignment graph and alignment read --- cudapoa/tests/Test_CudapoaNW.cu | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index c9235d317..530cf092f 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -411,9 +411,9 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive) class NWbandedTest : public ::testing::Test { public: - BasicNW* nw; //({'A', 'A', 'A', 'A'}, {0, 1, 2, 3}, {{1}, {2}, {3}, {}}, {'A', 'A', 'T', 'A'}); + BasicNW* nw; + public: - NWbandedTest(){}; void SetUp() { // initialize nw graph and read with the following data @@ -429,7 +429,7 @@ public: outgoing_edges[i].push_back(i + 1); } std::vector read(read_str.begin(), read_str.end()); - // + // setup nw nw = new BasicNW(nodes, sorted_graph, outgoing_edges, read); } @@ -438,14 +438,22 @@ public: TEST_F(NWbandedTest, NWSaticBandvsFull) { - auto str1 = testNW(*nw).second; - auto str2 = testNWbanded(*nw, false).second; - EXPECT_EQ(testNW(*nw).second, testNWbanded(*nw, false).second); + auto full_alignment_results = testNW(*nw); + auto static_banded_results = testNWbanded(*nw, false); + // verify alignment_graph + EXPECT_EQ(full_alignment_results.first, static_banded_results.first); + // verify alignment_read + EXPECT_EQ(full_alignment_results.second, static_banded_results.second); } TEST_F(NWbandedTest, NWAdaptiveBandvsFull) { - EXPECT_EQ(testNW(*nw).second, testNWbanded(*nw, true).second); + auto full_alignment_results = testNW(*nw); + auto adaptive_banded_results = testNWbanded(*nw, true); + // verify alignment_graph + EXPECT_EQ(full_alignment_results.first, adaptive_banded_results.first); + // verify alignment_read + EXPECT_EQ(full_alignment_results.second, adaptive_banded_results.second); } } // namespace cudapoa From 15c7946057640ac35da8449988d001b9cd00a750 Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 26 Nov 2020 19:59:49 -0500 Subject: [PATCH 259/281] [cudapoa] added adaptive/static banded with traceback buffer for NW tests --- cudapoa/src/cudapoa_nw_tb_banded.cuh | 128 +++++++++++++++++++++++++++ cudapoa/tests/Test_CudapoaNW.cu | 117 ++++++++++++++++++------ 2 files changed, 218 insertions(+), 27 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index 120487163..ac739ec51 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -642,6 +642,134 @@ __device__ __forceinline__ return aligned_nodes; } +// global kernel used in testing, hence uses int16_t for SizeT and ScoreT, +// may need to change if test inputs change to long reads +template +__global__ void runNeedlemanWunschBandedTBKernel(uint8_t* nodes, + SizeT* graph, + SizeT* node_id_to_pos, + int32_t graph_count, + uint16_t* incoming_edge_count, + SizeT* incoming_edges, + uint16_t* outgoing_edge_count, + uint8_t* read, + int32_t read_length, + int16_t* scores, + int16_t* traceback, + int32_t scores_width, + int32_t max_nodes_per_graph, + SizeT* alignment_graph, + SizeT* alignment_read, + int32_t band_width, + int32_t score_matrix_height, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, + SizeT* aligned_nodes, + bool adaptive) +{ + float banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_width); + + if (adaptive) + { + *aligned_nodes = runNeedlemanWunschBandedTraceback(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + band_width, + score_matrix_height, + gap_score, + mismatch_score, + match_score, + 0); + } + else + { + *aligned_nodes = runNeedlemanWunschBandedTraceback(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + band_width, + score_matrix_height, + gap_score, + mismatch_score, + match_score, + 0); + } +} + +// Host function that calls the kernel +template +void runNWbandedTB(uint8_t* nodes, + SizeT* graph, + SizeT* node_id_to_pos, + int32_t graph_count, + uint16_t* incoming_edge_count, + SizeT* incoming_edges, + uint16_t* outgoing_edge_count, + uint8_t* read, + int32_t read_length, + int16_t* scores, + int16_t* traceback, + int32_t scores_width, + int32_t max_nodes_per_graph, + SizeT* alignment_graph, + SizeT* alignment_read, + int32_t band_width, + int32_t score_matrix_height, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, + SizeT* aligned_nodes, + bool adaptive) +{ + runNeedlemanWunschBandedTBKernel<<<1, CUDAPOA_BANDED_THREADS_PER_BLOCK>>>(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + traceback, + scores_width, + max_nodes_per_graph, + alignment_graph, + alignment_read, + band_width, + score_matrix_height, + gap_score, + mismatch_score, + match_score, + aligned_nodes, + adaptive); + GW_CU_CHECK_ERR(cudaPeekAtLastError()); +} + } // namespace cudapoa } // namespace genomeworks diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index 530cf092f..e97e1b46e 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -307,8 +307,8 @@ INSTANTIATE_TEST_SUITE_P(TestNW, NWTest, ValuesIn(getNWTestCases())); //--------------------------------------------------------------------------------------- -// host function for calling the kernel to test static/adaptive-band NW device functions -NWAnswer testNWbanded(const BasicNW& obj, bool adaptive) +// host function for calling the kernels to test static/adaptive-band NW with/without traceback buffer +NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) { typedef int16_t SizeT; @@ -324,14 +324,17 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive) uint8_t* read; uint16_t read_count; //local int16_t* scores; + int16_t* traces; SizeT* alignment_graph; SizeT* alignment_read; int32_t gap_score; int32_t mismatch_score; int32_t match_score; SizeT* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) - BatchConfig batch_size(1024 /*max_sequence_size*/, 2 /*max_sequences_per_poa*/, 128 /*= band_width*/, - adaptive ? BandMode::adaptive_band : BandMode::static_band /*band-mode*/); + BandMode band_mode = traceback ? (adaptive ? BandMode::adaptive_band_traceback : BandMode::static_band_traceback) + : (adaptive ? BandMode::adaptive_band : BandMode::static_band); + BatchConfig batch_size(1024 /*max_sequence_size*/, 2 /*max_sequences_per_poa*/, + 128 /*= band_width*/, band_mode); //allocate unified memory so they can be accessed by both host and device. GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); @@ -341,11 +344,19 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive) GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&read, batch_size.max_sequence_size * sizeof(uint8_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_nodes_per_graph * sizeof(SizeT))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&aligned_nodes, sizeof(SizeT))); + if (traceback) + { + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_banded_pred_distance * batch_size.matrix_sequence_dimension * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&traces, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); + } + else + { + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); + } //initialize all 'count' buffers memset((void**)incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); @@ -363,27 +374,55 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive) mismatch_score = BasicNW::mismatch_score_; match_score = BasicNW::match_score_; - //call the host wrapper of nw kernel - runNWbanded(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_count, - scores, - batch_size.matrix_sequence_dimension, - batch_size.max_nodes_per_graph, - alignment_graph, - alignment_read, - batch_size.alignment_band_width, - gap_score, - mismatch_score, - match_score, - aligned_nodes, - adaptive); + //call the host wrapper of nw kernels + if (traceback) + { + runNWbandedTB(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_count, + scores, + traces, + batch_size.matrix_sequence_dimension, + batch_size.max_nodes_per_graph, + alignment_graph, + alignment_read, + batch_size.alignment_band_width, + batch_size.max_banded_pred_distance, + gap_score, + mismatch_score, + match_score, + aligned_nodes, + adaptive); + } + else + { + runNWbanded(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_count, + scores, + batch_size.matrix_sequence_dimension, + batch_size.max_nodes_per_graph, + alignment_graph, + alignment_read, + batch_size.alignment_band_width, + gap_score, + mismatch_score, + match_score, + aligned_nodes, + adaptive); + } GW_CU_CHECK_ERR(cudaDeviceSynchronize()); @@ -404,6 +443,10 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive) GW_CU_CHECK_ERR(cudaFree(read)); GW_CU_CHECK_ERR(cudaFree(alignment_read)); GW_CU_CHECK_ERR(cudaFree(aligned_nodes)); + if (traceback) + { + GW_CU_CHECK_ERR(cudaFree(traces)); + } return res; } @@ -448,7 +491,7 @@ TEST_F(NWbandedTest, NWSaticBandvsFull) TEST_F(NWbandedTest, NWAdaptiveBandvsFull) { - auto full_alignment_results = testNW(*nw); + auto full_alignment_results = testNW(*nw); auto adaptive_banded_results = testNWbanded(*nw, true); // verify alignment_graph EXPECT_EQ(full_alignment_results.first, adaptive_banded_results.first); @@ -456,6 +499,26 @@ TEST_F(NWbandedTest, NWAdaptiveBandvsFull) EXPECT_EQ(full_alignment_results.second, adaptive_banded_results.second); } +TEST_F(NWbandedTest, NWSaticBandTracebackvsFull) +{ + auto full_alignment_results = testNW(*nw); + auto static_banded_tb_results = testNWbanded(*nw, false, true); + // verify alignment_graph + EXPECT_EQ(full_alignment_results.first, static_banded_tb_results.first); + // verify alignment_read + EXPECT_EQ(full_alignment_results.second, static_banded_tb_results.second); +} + +TEST_F(NWbandedTest, NWAdaptiveBandTracebackvsFull) +{ + auto full_alignment_results = testNW(*nw); + auto adaptive_banded_tb_results = testNWbanded(*nw, true, true); + // verify alignment_graph + EXPECT_EQ(full_alignment_results.first, adaptive_banded_tb_results.first); + // verify alignment_read + EXPECT_EQ(full_alignment_results.second, adaptive_banded_tb_results.second); +} + } // namespace cudapoa } // namespace genomeworks From b2361c68856f91b044decb6153e6f995ccf58e75 Mon Sep 17 00:00:00 2001 From: Milos Maric Date: Fri, 27 Nov 2020 10:19:25 +0100 Subject: [PATCH 260/281] [common] Added missing set-to-nullptr to DevicePreallocatedAllocator::free_memory_block() and added tests --- .../utils/device_preallocated_allocator.cuh | 1 + .../tests/Test_DevicePreallocatedAllocator.cu | 46 ++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh index 86423b44f..70b352164 100644 --- a/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh +++ b/common/base/include/claraparabricks/genomeworks/utils/device_preallocated_allocator.cuh @@ -172,6 +172,7 @@ private: { if (free_blocks_.empty()) { + *ptr = nullptr; return cudaErrorMemoryAllocation; } diff --git a/common/base/tests/Test_DevicePreallocatedAllocator.cu b/common/base/tests/Test_DevicePreallocatedAllocator.cu index 3f1280b4c..d73444cb3 100644 --- a/common/base/tests/Test_DevicePreallocatedAllocator.cu +++ b/common/base/tests/Test_DevicePreallocatedAllocator.cu @@ -209,7 +209,7 @@ TEST(TestDevicePreallocatedAllocator, memory_correctly_deallocated) ASSERT_EQ(status, cudaSuccess); } -TEST(TestDevicePreallocatedAllocator, no_memory_left) +TEST(TestDevicePreallocatedAllocator, not_enough_memory_left) { CudaStream cuda_stream = make_cuda_stream(); @@ -247,6 +247,42 @@ TEST(TestDevicePreallocatedAllocator, no_memory_left) ASSERT_EQ(status, cudaSuccess); } +TEST(TestDevicePreallocatedAllocator, no_memory_left) +{ + CudaStream cuda_stream = make_cuda_stream(); + + std::vector cuda_streams; + cuda_streams.push_back(cuda_stream.get()); + + details::DevicePreallocatedAllocator allocator(2000); + // 0 - 1999: free + + cudaError status; + + void* pointer_from_0_to_1499_actually_1535 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_0_to_1499_actually_1535, 1500, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + ASSERT_NE(pointer_from_0_to_1499_actually_1535, nullptr); + // 0 - 1535: taken + // 1536 - 1999: free + + void* pointer_from_1536_to_1999_actually_1999 = nullptr; + status = allocator.DeviceAllocate(&pointer_from_1536_to_1999_actually_1999, 464, cuda_streams); + ASSERT_EQ(status, cudaSuccess); + ASSERT_NE(pointer_from_1536_to_1999_actually_1999, nullptr); + // 0 - 1999: taken + + void* pointer_to_unsuccessful_allocation = pointer_from_1536_to_1999_actually_1999; // set it to some value to make sure it gets reset to nullptr + status = allocator.DeviceAllocate(&pointer_to_unsuccessful_allocation, 1, cuda_streams); + ASSERT_EQ(status, cudaErrorMemoryAllocation); + ASSERT_EQ(pointer_to_unsuccessful_allocation, nullptr); + + status = allocator.DeviceFree(pointer_from_1536_to_1999_actually_1999); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(pointer_from_0_to_1499_actually_1535); + ASSERT_EQ(status, cudaSuccess); +} + TEST(TestDevicePreallocatedAllocator, deallocating_invalid_pointer) { CudaStream cuda_stream = make_cuda_stream(); @@ -271,8 +307,16 @@ TEST(TestDevicePreallocatedAllocator, deallocating_invalid_pointer) status = allocator.DeviceFree(null_ptr); ASSERT_EQ(status, cudaSuccess); + // deallocating nullptr does nothing, but is a success + status = allocator.DeviceFree(nullptr); + ASSERT_EQ(status, cudaSuccess); + status = allocator.DeviceFree(valid_ptr); ASSERT_EQ(status, cudaSuccess); + + // deallocating previously deallocated pointer results in error + status = allocator.DeviceFree(valid_ptr); // pointer not valid anymore + ASSERT_EQ(status, cudaErrorInvalidValue); } } // namespace genomeworks From 9b540b6afa1f77d5d6fb68d3eb60b3eccb4fc26d Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 27 Nov 2020 12:24:26 -0500 Subject: [PATCH 261/281] [cudapoa] got rid of SizeT in Test_CudapoaNW.cu --- cudapoa/src/cudapoa_nw.cuh | 2 + cudapoa/src/cudapoa_nw_banded.cuh | 2 + cudapoa/src/cudapoa_nw_tb_banded.cuh | 2 + cudapoa/tests/Test_CudapoaNW.cu | 174 +++++++++++++-------------- 4 files changed, 91 insertions(+), 89 deletions(-) diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index d8cfa0232..55179d72f 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -512,6 +512,8 @@ void runNW(uint8_t* nodes, int32_t match_score, SizeT* aligned_nodes) { + static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); + runNeedlemanWunschKernel<<<1, CUDAPOA_THREADS_PER_BLOCK>>>(nodes, graph, node_id_to_pos, diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index e60b09cfe..829e4264d 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -649,6 +649,8 @@ void runNWbanded(uint8_t* nodes, SizeT* aligned_nodes, bool adaptive) { + static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); + runNeedlemanWunschBandedKernel<<<1, CUDAPOA_BANDED_THREADS_PER_BLOCK>>>(nodes, graph, node_id_to_pos, diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index ac739ec51..cba8d0025 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -745,6 +745,8 @@ void runNWbandedTB(uint8_t* nodes, SizeT* aligned_nodes, bool adaptive) { + static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); + runNeedlemanWunschBandedTBKernel<<<1, CUDAPOA_BANDED_THREADS_PER_BLOCK>>>(nodes, graph, node_id_to_pos, diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index e97e1b46e..13afb7f5d 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -42,7 +42,7 @@ public: const static int16_t match_score_ = 8; public: - BasicNW(std::vector nodes, std::vector sorted_graph, SizeTVec2D outgoing_edges, + BasicNW(std::vector nodes, std::vector sorted_graph, SizeTVec2D outgoing_edges, std::vector read) : graph_(nodes, sorted_graph, outgoing_edges) , read_(read) @@ -52,10 +52,10 @@ public: BasicNW() = delete; - void get_graph_buffers(SizeT* incoming_edges, uint16_t* incoming_edge_count, - SizeT* outgoing_edges, uint16_t* outgoing_edge_count, - uint8_t* nodes, SizeT* node_count, - SizeT* graph, SizeT* node_id_to_pos) const + void get_graph_buffers(int16_t* incoming_edges, uint16_t* incoming_edge_count, + int16_t* outgoing_edges, uint16_t* outgoing_edge_count, + uint8_t* nodes, int16_t* node_count, + int16_t* graph, int16_t* node_id_to_pos) const { graph_.get_edges(incoming_edges, incoming_edge_count, outgoing_edges, outgoing_edge_count); graph_.get_nodes(nodes, node_count); @@ -189,46 +189,44 @@ std::vector getNWTestCases() // host function for calling the kernel to test full-band NW device function. NWAnswer testNW(const BasicNW& obj) { - typedef int16_t SizeT; - //declare device buffer uint8_t* nodes; - SizeT* graph; - SizeT* node_id_to_pos; - SizeT graph_count; //local + int16_t* graph; + int16_t* node_id_to_pos; + int16_t graph_count; //local uint16_t* incoming_edge_count; - SizeT* incoming_edges; + int16_t* incoming_edges; uint16_t* outgoing_edge_count; - SizeT* outgoing_edges; + int16_t* outgoing_edges; uint8_t* read; uint16_t read_count; //local int16_t* scores; - SizeT* alignment_graph; - SizeT* alignment_read; + int16_t* alignment_graph; + int16_t* alignment_read; int32_t gap_score; int32_t mismatch_score; int32_t match_score; - SizeT* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) + int16_t* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) BatchConfig batch_size; //default max_sequence_size = 1024, max_sequences_per_poa = 100 //allocate unified memory so they can be accessed by both host and device. GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&read, batch_size.max_sequence_size * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&aligned_nodes, sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&aligned_nodes, sizeof(int16_t))); //initialize all 'count' buffers memset((void**)incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); memset((void**)outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(SizeT)); + memset((void**)node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(int16_t)); memset((void**)scores, 0, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t)); //calculate edge counts on host @@ -242,30 +240,30 @@ NWAnswer testNW(const BasicNW& obj) match_score = BasicNW::match_score_; //call the host wrapper of nw kernel - runNW(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_count, - scores, - batch_size.matrix_sequence_dimension, - alignment_graph, - alignment_read, - gap_score, - mismatch_score, - match_score, - aligned_nodes); + runNW(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_count, + scores, + batch_size.matrix_sequence_dimension, + alignment_graph, + alignment_read, + gap_score, + mismatch_score, + match_score, + aligned_nodes); GW_CU_CHECK_ERR(cudaDeviceSynchronize()); //input and output buffers are the same ones in unified memory, so the results are updated in place //results are stored in alignment_graph and alignment_read; return string representation of those - auto res = std::make_pair(genomeworks::stringutils::array_to_string(alignment_graph, *aligned_nodes, ","), - genomeworks::stringutils::array_to_string(alignment_read, *aligned_nodes, ",")); + auto res = std::make_pair(genomeworks::stringutils::array_to_string(alignment_graph, *aligned_nodes, ","), + genomeworks::stringutils::array_to_string(alignment_read, *aligned_nodes, ",")); GW_CU_CHECK_ERR(cudaFree(nodes)); GW_CU_CHECK_ERR(cudaFree(graph)); @@ -310,27 +308,25 @@ INSTANTIATE_TEST_SUITE_P(TestNW, NWTest, ValuesIn(getNWTestCases())); // host function for calling the kernels to test static/adaptive-band NW with/without traceback buffer NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) { - typedef int16_t SizeT; - //declare device buffer uint8_t* nodes; - SizeT* graph; - SizeT* node_id_to_pos; - SizeT graph_count; //local + int16_t* graph; + int16_t* node_id_to_pos; + int16_t graph_count; //local uint16_t* incoming_edge_count; - SizeT* incoming_edges; + int16_t* incoming_edges; uint16_t* outgoing_edge_count; - SizeT* outgoing_edges; + int16_t* outgoing_edges; uint8_t* read; uint16_t read_count; //local int16_t* scores; int16_t* traces; - SizeT* alignment_graph; - SizeT* alignment_read; + int16_t* alignment_graph; + int16_t* alignment_read; int32_t gap_score; int32_t mismatch_score; int32_t match_score; - SizeT* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) + int16_t* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) BandMode band_mode = traceback ? (adaptive ? BandMode::adaptive_band_traceback : BandMode::static_band_traceback) : (adaptive ? BandMode::adaptive_band : BandMode::static_band); BatchConfig batch_size(1024 /*max_sequence_size*/, 2 /*max_sequences_per_poa*/, @@ -338,16 +334,16 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) //allocate unified memory so they can be accessed by both host and device. GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&read, batch_size.max_sequence_size * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&aligned_nodes, sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&aligned_nodes, sizeof(int16_t))); if (traceback) { GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_banded_pred_distance * batch_size.matrix_sequence_dimension * sizeof(int16_t))); @@ -361,7 +357,7 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) //initialize all 'count' buffers memset((void**)incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); memset((void**)outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(SizeT)); + memset((void**)node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(int16_t)); memset((void**)scores, 0, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t)); //calculate edge counts on host @@ -377,7 +373,32 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) //call the host wrapper of nw kernels if (traceback) { - runNWbandedTB(nodes, + runNWbandedTB(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_count, + scores, + traces, + batch_size.matrix_sequence_dimension, + batch_size.max_nodes_per_graph, + alignment_graph, + alignment_read, + batch_size.alignment_band_width, + batch_size.max_banded_pred_distance, + gap_score, + mismatch_score, + match_score, + aligned_nodes, + adaptive); + } + else + { + runNWbanded(nodes, graph, node_id_to_pos, graph_count, @@ -387,49 +408,24 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) read, read_count, scores, - traces, batch_size.matrix_sequence_dimension, batch_size.max_nodes_per_graph, alignment_graph, alignment_read, batch_size.alignment_band_width, - batch_size.max_banded_pred_distance, gap_score, mismatch_score, match_score, aligned_nodes, adaptive); } - else - { - runNWbanded(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_count, - scores, - batch_size.matrix_sequence_dimension, - batch_size.max_nodes_per_graph, - alignment_graph, - alignment_read, - batch_size.alignment_band_width, - gap_score, - mismatch_score, - match_score, - aligned_nodes, - adaptive); - } GW_CU_CHECK_ERR(cudaDeviceSynchronize()); //input and output buffers are the same ones in unified memory, so the results are updated in place //results are stored in alignment_graph and alignment_read; return string representation of those - auto res = std::make_pair(genomeworks::stringutils::array_to_string(alignment_graph, *aligned_nodes, ","), - genomeworks::stringutils::array_to_string(alignment_read, *aligned_nodes, ",")); + auto res = std::make_pair(genomeworks::stringutils::array_to_string(alignment_graph, *aligned_nodes, ","), + genomeworks::stringutils::array_to_string(alignment_read, *aligned_nodes, ",")); GW_CU_CHECK_ERR(cudaFree(nodes)); GW_CU_CHECK_ERR(cudaFree(graph)); @@ -464,7 +460,7 @@ public: std::string read_str = "TTTCACCTAGAAAATCAGTGAAGATTTAACAAAAAAAAAAAAAAAAAAAAAAAAATATTGATAATTATTGATTTTGGTGAGAGTGCAAAGCAATTGGCTACCCTCATAAGCTGAGCAGAAGATAAGATAGACAACAGAAGATAGAATAGTTAAACCATGATAGGTACAGACTGCAAAAAAATTCGATAAATATTAAAATTTAGGGCTTTAGTATATATTGATGACTGAGAAAAATCGTGATGTGCAATTGTGCGTGACATGTAGAATTGCCTTAAATAAAATTTAATCTGTCACTGAAGCTATATTTATATTCAGGAAGGATATATCCCAGTCATTGCTTTTCTTAATAAGTGCCCATGTTCCAAGTTTAGCCTAATTAAAAACTTTATGTCTTCTATATCAGAATAGTCATTAATGCACAGAAACAATTTGCGAAGGCATTATGTAGCAAAAACATAAAAAATTATTGCAGCCAAATAATGAATAAAAGTAACACAATCATTTAAAAAAATTATTATGTACTTCTAAAC"; // extract data to build BasicNW std::vector nodes(nodes_str.begin(), nodes_str.end()); - std::vector sorted_graph(nodes.size()); + std::vector sorted_graph(nodes.size()); std::iota(sorted_graph.begin(), sorted_graph.end(), 0); SizeTVec2D outgoing_edges(nodes.size()); for (size_t i = 0; i < outgoing_edges.size() - 1; i++) From 54b1dda3e03b82cefb7af5dd61ff5b38348ca9b9 Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 27 Nov 2020 13:28:05 -0500 Subject: [PATCH 262/281] [cudapoa] removing SizeT in other test files and adding static_assert to host functions used in tests to verify SizeT is int16_t --- cudapoa/src/cudapoa_add_alignment.cuh | 40 ++++----- cudapoa/src/cudapoa_generate_consensus.cuh | 2 + cudapoa/src/cudapoa_topsort.cuh | 2 + cudapoa/tests/Test_CudapoaAddAlignment.cu | 71 ++++++++-------- .../tests/Test_CudapoaGenerateConsensus.cu | 80 +++++++++--------- cudapoa/tests/Test_CudapoaNW.cu | 84 +++++++++---------- cudapoa/tests/Test_CudapoaTopSort.cu | 42 +++++----- cudapoa/tests/basic_graph.hpp | 34 ++++---- cudapoa/tests/sorted_graph.hpp | 16 ++-- 9 files changed, 187 insertions(+), 184 deletions(-) diff --git a/cudapoa/src/cudapoa_add_alignment.cuh b/cudapoa/src/cudapoa_add_alignment.cuh index 92b3f9f72..326f78e57 100644 --- a/cudapoa/src/cudapoa_add_alignment.cuh +++ b/cudapoa/src/cudapoa_add_alignment.cuh @@ -352,25 +352,27 @@ void addAlignment(uint8_t* nodes, uint32_t max_sequences_per_poa, uint32_t max_limit_nodes_per_window) { - addAlignmentKernel<<<1, 1>>>(nodes, - node_count, - node_alignments, node_alignment_count, - incoming_edges, incoming_edge_count, - outgoing_edges, outgoing_edge_count, - incoming_edge_w, - alignment_length, - graph, - alignment_graph, - read, - alignment_read, - node_coverage_counts, - base_weights, - sequence_begin_nodes_ids, - outgoing_edges_coverage, - outgoing_edges_coverage_count, - s, - max_sequences_per_poa, - max_limit_nodes_per_window); + static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); + + addAlignmentKernel<<<1, 1>>>(nodes, + node_count, + node_alignments, node_alignment_count, + incoming_edges, incoming_edge_count, + outgoing_edges, outgoing_edge_count, + incoming_edge_w, + alignment_length, + graph, + alignment_graph, + read, + alignment_read, + node_coverage_counts, + base_weights, + sequence_begin_nodes_ids, + outgoing_edges_coverage, + outgoing_edges_coverage_count, + s, + max_sequences_per_poa, + max_limit_nodes_per_window); GW_CU_CHECK_ERR(cudaPeekAtLastError()); } diff --git a/cudapoa/src/cudapoa_generate_consensus.cuh b/cudapoa/src/cudapoa_generate_consensus.cuh index ceffe64af..723592a28 100644 --- a/cudapoa/src/cudapoa_generate_consensus.cuh +++ b/cudapoa/src/cudapoa_generate_consensus.cuh @@ -410,6 +410,8 @@ void generateConsensusTestHost(uint8_t* nodes, uint16_t* node_alignment_count, int32_t max_limit_consensus_size) { + static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); + generateConsensusTestKernel<<<1, 1>>>(nodes, node_count, graph, diff --git a/cudapoa/src/cudapoa_topsort.cuh b/cudapoa/src/cudapoa_topsort.cuh index 2982525ea..7bd789267 100644 --- a/cudapoa/src/cudapoa_topsort.cuh +++ b/cudapoa/src/cudapoa_topsort.cuh @@ -225,6 +225,8 @@ void runTopSort(SizeT* sorted_poa, uint16_t* outgoing_edge_count, uint16_t* local_incoming_edge_count) { + static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); + // calls the topsort kernel on 1 thread runTopSortKernel<<<1, 1>>>(sorted_poa, sorted_poa_node_map, diff --git a/cudapoa/tests/Test_CudapoaAddAlignment.cu b/cudapoa/tests/Test_CudapoaAddAlignment.cu index 84e4471ae..2dd1e3b6b 100644 --- a/cudapoa/tests/Test_CudapoaAddAlignment.cu +++ b/cudapoa/tests/Test_CudapoaAddAlignment.cu @@ -36,9 +36,9 @@ namespace cudapoa class BasicAlignment { public: - BasicAlignment(std::vector nodes, SizeTVec2D outgoing_edges, - SizeTVec2D node_alignments, std::vector node_coverage_counts, - std::vector read, std::vector base_weights, std::vector alignment_graph, std::vector alignment_read) + BasicAlignment(std::vector nodes, Int16Vec2D outgoing_edges, + Int16Vec2D node_alignments, std::vector node_coverage_counts, + std::vector read, std::vector base_weights, std::vector alignment_graph, std::vector alignment_read) : graph(nodes, outgoing_edges, node_alignments, node_coverage_counts) , read_(read) , alignment_graph_(alignment_graph) @@ -47,7 +47,7 @@ public: //do nothing for now } - void get_alignments(SizeT* alignment_graph, SizeT* alignment_read, SizeT* alignment_length) const + void get_alignments(int16_t* alignment_graph, int16_t* alignment_read, int16_t* alignment_length) const { for (int i = 0; i < get_size(alignment_graph_); i++) { @@ -73,10 +73,10 @@ public: } } - void get_graph_buffers(SizeT* incoming_edges, uint16_t* incoming_edge_count, - SizeT* outgoing_edges, uint16_t* outgoing_edge_count, - uint8_t* nodes, SizeT* node_count, - SizeT* node_alignments, uint16_t* node_alignment_count, + void get_graph_buffers(int16_t* incoming_edges, uint16_t* incoming_edge_count, + int16_t* outgoing_edges, uint16_t* outgoing_edge_count, + uint8_t* nodes, int16_t* node_count, + int16_t* node_alignments, uint16_t* node_alignment_count, uint16_t* node_coverage_counts) const { if (!graph.is_complete()) @@ -89,7 +89,7 @@ public: graph.get_node_coverage_counts(node_coverage_counts); } - void get_alignment_buffers(SizeT* alignment_graph, SizeT* alignment_read, SizeT* alignment_length, + void get_alignment_buffers(int16_t* alignment_graph, int16_t* alignment_read, int16_t* alignment_length, uint8_t* read, int8_t* base_weights) const { get_alignments(alignment_graph, alignment_read, alignment_length); @@ -101,8 +101,8 @@ protected: BasicGraph graph; std::vector read_; std::vector base_weights_; - std::vector alignment_graph_; - std::vector alignment_read_; + std::vector alignment_graph_; + std::vector alignment_read_; }; typedef std::pair AddAlginmentTestPair; @@ -123,7 +123,7 @@ std::vector getAddAlignmentTestCases() * \ / * A */ - BasicGraph ans_1(SizeTVec2D({{}, {0}, {1}, {2, 4}, {1}})); + BasicGraph ans_1(Int16Vec2D({{}, {0}, {1}, {2, 4}, {1}})); BasicAlignment ali_1({'A', 'A', 'A', 'A'}, //nodes {{}, {0}, {1}, {2}}, //outgoing_edges {{}, {}, {}, {}}, //node_alignments @@ -143,7 +143,7 @@ std::vector getAddAlignmentTestCases() * final graph A — T — C — G — A * */ - BasicGraph ans_2(SizeTVec2D({{}, {0}, {1}, {2}, {3}})); + BasicGraph ans_2(Int16Vec2D({{}, {0}, {1}, {2}, {3}})); BasicAlignment ali_2({'A', 'T', 'C', 'G'}, //nodes {{}, {0}, {1}, {2}}, //outgoing_edges {{}, {}, {}, {}}, //node_alignments @@ -167,7 +167,7 @@ std::vector getAddAlignmentTestCases() * \ / * A */ - BasicGraph ans_3(SizeTVec2D({{}, {0}, {1, 4, 5}, {2}, {0}, {0}})); + BasicGraph ans_3(Int16Vec2D({{}, {0}, {1, 4, 5}, {2}, {0}, {0}})); BasicAlignment ali_3({'A', 'A', 'C', 'G', 'C'}, //nodes {{}, {0}, {1, 4}, {2}, {0}}, //outgoing_edges {{}, {}, {}, {}}, //node_alignments @@ -188,7 +188,7 @@ std::vector getAddAlignmentTestCases() * \_____________/ * */ - BasicGraph ans_4(SizeTVec2D({{}, {0}, {1}, {2}, {3, 0}})); + BasicGraph ans_4(Int16Vec2D({{}, {0}, {1}, {2}, {3, 0}})); BasicAlignment ali_4({'A', 'T', 'T', 'G', 'A'}, //nodes {{}, {0}, {1}, {2}, {3}}, //outgoing_edges {{}, {}, {}, {}}, //node_alignments @@ -213,7 +213,7 @@ std::vector getAddAlignmentTestCases() * T * */ - BasicGraph ans_5(SizeTVec2D({{}, {0}, {1}, {2, 6, 7}, {3}, {0}, {5}, {5}})); + BasicGraph ans_5(Int16Vec2D({{}, {0}, {1}, {2, 6, 7}, {3}, {0}, {5}, {5}})); BasicAlignment ali_5({'A', 'T', 'G', 'T', 'A', 'C', 'A'}, //nodes {{}, {0}, {1}, {2, 6}, {3}, {0}, {5}}, //outgoing_edges {{}, {}, {}, {}}, //node_alignments @@ -232,26 +232,25 @@ std::vector getAddAlignmentTestCases() // host function for calling the kernel to test topsort device function. BasicGraph testAddAlignment(const BasicAlignment& obj) { - typedef int16_t SizeT; //declare device buffer uint8_t* nodes; - SizeT* node_count; - SizeT* node_alignments; + int16_t* node_count; + int16_t* node_alignments; uint16_t* node_alignment_count; - SizeT* incoming_edges; + int16_t* incoming_edges; uint16_t* incoming_edge_count; - SizeT* outgoing_edges; + int16_t* outgoing_edges; uint16_t* outgoing_edge_count; uint16_t* incoming_edge_w; uint16_t* outgoing_edge_w; - SizeT* alignment_length; - SizeT* graph; - SizeT* alignment_graph; + int16_t* alignment_length; + int16_t* graph; + int16_t* alignment_graph; uint8_t* read; int8_t* base_weights; - SizeT* alignment_read; + int16_t* alignment_read; uint16_t* node_coverage_counts; - SizeT* sequence_begin_nodes_ids; + int16_t* sequence_begin_nodes_ids; uint16_t* outgoing_edges_coverage; uint16_t* outgoing_edges_coverage_count; uint16_t s = 0; @@ -259,23 +258,23 @@ BasicGraph testAddAlignment(const BasicAlignment& obj) //allocate unified memory so they can be accessed by both host and device. GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_count, sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignments, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_count, sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignments, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignment_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_w, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_w, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_length, sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_sequence_size * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_length, sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_sequence_size * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&read, batch_size.max_sequence_size * sizeof(uint8_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&base_weights, batch_size.max_sequence_size * sizeof(int8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_sequence_size * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_coverage_counts, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&sequence_begin_nodes_ids, batch_size.max_sequences_per_poa * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_sequence_size * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_coverage_counts, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&sequence_begin_nodes_ids, batch_size.max_sequences_per_poa * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges_coverage, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * batch_size.max_sequences_per_poa * sizeof(uint16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges_coverage_count, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); diff --git a/cudapoa/tests/Test_CudapoaGenerateConsensus.cu b/cudapoa/tests/Test_CudapoaGenerateConsensus.cu index d212c1258..486f14bd8 100644 --- a/cudapoa/tests/Test_CudapoaGenerateConsensus.cu +++ b/cudapoa/tests/Test_CudapoaGenerateConsensus.cu @@ -35,20 +35,20 @@ class BasicGenerateConsensus { public: - BasicGenerateConsensus(std::vector nodes, std::vector sorted_graph, SizeTVec2D node_alignments, - SizeTVec2D outgoing_edges, std::vector node_coverage_counts, Uint16Vec2D outgoing_edge_w) + BasicGenerateConsensus(std::vector nodes, std::vector sorted_graph, Int16Vec2D node_alignments, + Int16Vec2D outgoing_edges, std::vector node_coverage_counts, Uint16Vec2D outgoing_edge_w) : graph_(nodes, sorted_graph, node_alignments, node_coverage_counts, outgoing_edges) , outgoing_edge_w_(outgoing_edge_w) , outgoing_edges_(outgoing_edges) { } - void get_graph_buffers(uint8_t* nodes, SizeT* node_count, - SizeT* sorted_poa, SizeT* node_id_to_pos, - SizeT* incoming_edges, uint16_t* incoming_edge_count, - SizeT* outgoing_edges, uint16_t* outgoing_edge_count, + void get_graph_buffers(uint8_t* nodes, int16_t* node_count, + int16_t* sorted_poa, int16_t* node_id_to_pos, + int16_t* incoming_edges, uint16_t* incoming_edge_count, + int16_t* outgoing_edges, uint16_t* outgoing_edge_count, uint16_t* incoming_edge_w, uint16_t* node_coverage_counts, - SizeT* node_alignments, uint16_t* node_alignment_count) const + int16_t* node_alignments, uint16_t* node_alignment_count) const { graph_.get_nodes(nodes, node_count); graph_.get_sorted_graph(sorted_poa); @@ -66,7 +66,7 @@ public: { for (int j = 0; j < get_size(outgoing_edges[i]); j++) { - SizeT to_node = outgoing_edges[i][j]; + int16_t to_node = outgoing_edges[i][j]; incoming_edge_w[to_node * CUDAPOA_MAX_NODE_EDGES + i] = outgoing_edge_w_[i][j]; } } @@ -74,7 +74,7 @@ public: protected: SortedGraph graph_; - SizeTVec2D outgoing_edges_; + Int16Vec2D outgoing_edges_; Uint16Vec2D outgoing_edge_w_; }; @@ -169,20 +169,20 @@ std::string testGenerateConsensus(const BasicGenerateConsensus& obj) { //declare device buffer uint8_t* nodes; - SizeT* node_count; - SizeT* graph; - SizeT* node_id_to_pos; - SizeT* incoming_edges; + int16_t* node_count; + int16_t* graph; + int16_t* node_id_to_pos; + int16_t* incoming_edges; uint16_t* incoming_edge_count; - SizeT* outgoing_edges; + int16_t* outgoing_edges; uint16_t* outgoing_edge_count; uint16_t* incoming_edge_w; uint16_t* node_coverage_counts; - SizeT* node_alignments; + int16_t* node_alignments; uint16_t* node_alignment_count; //buffers that don't need initialization - SizeT* predecessors; + int16_t* predecessors; int32_t* scores; uint8_t* consensus; uint16_t* coverage; @@ -192,19 +192,19 @@ std::string testGenerateConsensus(const BasicGenerateConsensus& obj) //allocate unified memory so they can be accessed by both host and device. GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_count, sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_count, sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_w, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_coverage_counts, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignments, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignments, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignment_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&predecessors, batch_size.max_nodes_per_graph * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&predecessors, batch_size.max_nodes_per_graph * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_nodes_per_graph * sizeof(int32_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&consensus, batch_size.max_consensus_size * sizeof(uint8_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&coverage, batch_size.max_consensus_size * sizeof(uint16_t))); @@ -224,23 +224,23 @@ std::string testGenerateConsensus(const BasicGenerateConsensus& obj) node_alignments, node_alignment_count); // call the host wrapper of topsort kernel - generateConsensusTestHost(nodes, - *node_count, - graph, - node_id_to_pos, - incoming_edges, - incoming_edge_count, - outgoing_edges, - outgoing_edge_count, - incoming_edge_w, - predecessors, - scores, - consensus, - coverage, - node_coverage_counts, - node_alignments, - node_alignment_count, - batch_size.max_consensus_size); + generateConsensusTestHost(nodes, + *node_count, + graph, + node_id_to_pos, + incoming_edges, + incoming_edge_count, + outgoing_edges, + outgoing_edge_count, + incoming_edge_w, + predecessors, + scores, + consensus, + coverage, + node_coverage_counts, + node_alignments, + node_alignment_count, + batch_size.max_consensus_size); GW_CU_CHECK_ERR(cudaDeviceSynchronize()); diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index 13afb7f5d..12409c5c4 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -42,7 +42,7 @@ public: const static int16_t match_score_ = 8; public: - BasicNW(std::vector nodes, std::vector sorted_graph, SizeTVec2D outgoing_edges, + BasicNW(std::vector nodes, std::vector sorted_graph, Int16Vec2D outgoing_edges, std::vector read) : graph_(nodes, sorted_graph, outgoing_edges) , read_(read) @@ -374,50 +374,50 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) if (traceback) { runNWbandedTB(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_count, - scores, - traces, - batch_size.matrix_sequence_dimension, - batch_size.max_nodes_per_graph, - alignment_graph, - alignment_read, - batch_size.alignment_band_width, - batch_size.max_banded_pred_distance, - gap_score, - mismatch_score, - match_score, - aligned_nodes, - adaptive); + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_count, + scores, + traces, + batch_size.matrix_sequence_dimension, + batch_size.max_nodes_per_graph, + alignment_graph, + alignment_read, + batch_size.alignment_band_width, + batch_size.max_banded_pred_distance, + gap_score, + mismatch_score, + match_score, + aligned_nodes, + adaptive); } else { runNWbanded(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_count, - scores, - batch_size.matrix_sequence_dimension, - batch_size.max_nodes_per_graph, - alignment_graph, - alignment_read, - batch_size.alignment_band_width, - gap_score, - mismatch_score, - match_score, - aligned_nodes, - adaptive); + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_count, + scores, + batch_size.matrix_sequence_dimension, + batch_size.max_nodes_per_graph, + alignment_graph, + alignment_read, + batch_size.alignment_band_width, + gap_score, + mismatch_score, + match_score, + aligned_nodes, + adaptive); } GW_CU_CHECK_ERR(cudaDeviceSynchronize()); @@ -462,7 +462,7 @@ public: std::vector nodes(nodes_str.begin(), nodes_str.end()); std::vector sorted_graph(nodes.size()); std::iota(sorted_graph.begin(), sorted_graph.end(), 0); - SizeTVec2D outgoing_edges(nodes.size()); + Int16Vec2D outgoing_edges(nodes.size()); for (size_t i = 0; i < outgoing_edges.size() - 1; i++) { outgoing_edges[i].push_back(i + 1); diff --git a/cudapoa/tests/Test_CudapoaTopSort.cu b/cudapoa/tests/Test_CudapoaTopSort.cu index 2e11de733..2cee8fac6 100644 --- a/cudapoa/tests/Test_CudapoaTopSort.cu +++ b/cudapoa/tests/Test_CudapoaTopSort.cu @@ -34,7 +34,7 @@ namespace cudapoa { // alias for a test case (answer, graph) -typedef std::pair TopSortTestPair; +typedef std::pair TopSortTestPair; using ::testing::TestWithParam; using ::testing::ValuesIn; @@ -45,15 +45,15 @@ std::vector getTopSortTestCases() std::vector test_cases; - SizeTVec2D outgoing_edges_1 = {{}, {}, {3}, {1}, {0, 1}, {0, 2}}; + Int16Vec2D outgoing_edges_1 = {{}, {}, {3}, {1}, {0, 1}, {0, 2}}; std::string answer_1 = "4-5-0-2-3-1"; test_cases.emplace_back(answer_1, outgoing_edges_1); - SizeTVec2D outgoing_edges_2 = {{1, 3}, {2, 3}, {3, 4, 5}, {4, 5}, {5}, {}}; + Int16Vec2D outgoing_edges_2 = {{1, 3}, {2, 3}, {3, 4, 5}, {4, 5}, {5}, {}}; std::string answer_2 = "0-1-2-3-4-5"; test_cases.emplace_back(answer_2, outgoing_edges_2); - SizeTVec2D outgoing_edges_3 = {{}, {}, {3}, {1}, {0, 1, 7}, {0, 2}, {4}, {5}}; + Int16Vec2D outgoing_edges_3 = {{}, {}, {3}, {1}, {0, 1, 7}, {0, 2}, {4}, {5}}; std::string answer_3 = "6-4-7-5-0-2-3-1"; test_cases.emplace_back(answer_3, outgoing_edges_3); @@ -63,23 +63,23 @@ std::vector getTopSortTestCases() } // host function for calling the kernel to test topsort device function. -std::string testTopSortDeviceUtil(SizeT node_count, SizeTVec2D outgoing_edges_vec) +std::string testTopSortDeviceUtil(int16_t node_count, Int16Vec2D outgoing_edges_vec) { //declare device buffer - SizeT* sorted_poa; - SizeT* sorted_poa_node_map; + int16_t* sorted_poa; + int16_t* sorted_poa_node_map; uint16_t* incoming_edge_count; - SizeT* outgoing_edges; + int16_t* outgoing_edges; uint16_t* outgoing_edge_count; uint16_t* local_incoming_edge_count; size_t graph_size = node_count * sizeof(uint16_t); //allocate unified memory so they can be accessed by both host and device. - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&sorted_poa, node_count * sizeof(SizeT))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&sorted_poa_node_map, node_count * sizeof(SizeT))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&sorted_poa, node_count * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&sorted_poa_node_map, node_count * sizeof(int16_t))); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, graph_size)); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, node_count * sizeof(SizeT) * CUDAPOA_MAX_NODE_EDGES)); + GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, node_count * sizeof(int16_t) * CUDAPOA_MAX_NODE_EDGES)); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, graph_size)); GW_CU_CHECK_ERR(cudaMallocManaged((void**)&local_incoming_edge_count, graph_size)); @@ -89,7 +89,7 @@ std::string testTopSortDeviceUtil(SizeT node_count, SizeTVec2D outgoing_edges_ve //calculate edge counts on host - SizeT out_node; + int16_t out_node; for (int i = 0; i < node_count; i++) { outgoing_edge_count[i] = get_size(outgoing_edges_vec[i]); @@ -103,17 +103,17 @@ std::string testTopSortDeviceUtil(SizeT node_count, SizeTVec2D outgoing_edges_ve } // call the host wrapper of topsort kernel - runTopSort(sorted_poa, - sorted_poa_node_map, - node_count, - incoming_edge_count, - outgoing_edges, - outgoing_edge_count, - local_incoming_edge_count); + runTopSort(sorted_poa, + sorted_poa_node_map, + node_count, + incoming_edge_count, + outgoing_edges, + outgoing_edge_count, + local_incoming_edge_count); GW_CU_CHECK_ERR(cudaDeviceSynchronize()); - std::string res = genomeworks::stringutils::array_to_string(sorted_poa, node_count); + std::string res = genomeworks::stringutils::array_to_string(sorted_poa, node_count); GW_CU_CHECK_ERR(cudaFree(sorted_poa)); GW_CU_CHECK_ERR(cudaFree(sorted_poa_node_map)); @@ -130,7 +130,7 @@ class TopSortDeviceUtilTest : public TestWithParam public: void SetUp() {} - std::string runTopSortDevice(SizeTVec2D outgoing_edges) + std::string runTopSortDevice(Int16Vec2D outgoing_edges) { return testTopSortDeviceUtil(get_size(outgoing_edges), outgoing_edges); } diff --git a/cudapoa/tests/basic_graph.hpp b/cudapoa/tests/basic_graph.hpp index 08bcd4b52..35527a420 100644 --- a/cudapoa/tests/basic_graph.hpp +++ b/cudapoa/tests/basic_graph.hpp @@ -31,17 +31,15 @@ namespace genomeworks namespace cudapoa { -// in all tests, SizeT can be considered int16_t -typedef int16_t SizeT; // alias for the 2d vector graph representation -typedef std::vector> SizeTVec2D; +typedef std::vector> Int16Vec2D; typedef std::vector>> Uint16Vec3D; typedef std::vector> Uint16Vec2D; class BasicGraph { public: - BasicGraph(std::vector nodes, SizeTVec2D outgoing_edges, SizeTVec2D node_alignments, std::vector node_coverage_counts, Uint16Vec3D outgoing_edges_coverage = {}) + BasicGraph(std::vector nodes, Int16Vec2D outgoing_edges, Int16Vec2D node_alignments, std::vector node_coverage_counts, Uint16Vec3D outgoing_edges_coverage = {}) : nodes_(nodes) , outgoing_edges_(outgoing_edges) , node_alignments_(node_alignments) @@ -52,20 +50,20 @@ class BasicGraph node_count_ = get_size(nodes_); } - BasicGraph(SizeT* outgoing_edges, uint16_t* outgoing_edge_count, SizeT node_count) + BasicGraph(int16_t* outgoing_edges, uint16_t* outgoing_edge_count, int16_t node_count) { graph_complete_ = false; outgoing_edges_ = edges_to_graph(outgoing_edges, outgoing_edge_count, node_count); } - BasicGraph(SizeTVec2D outgoing_edges) + BasicGraph(Int16Vec2D outgoing_edges) { graph_complete_ = false; outgoing_edges_ = outgoing_edges; node_count_ = get_size(outgoing_edges); } - BasicGraph(std::vector nodes, SizeTVec2D outgoing_edges) + BasicGraph(std::vector nodes, Int16Vec2D outgoing_edges) : BasicGraph(outgoing_edges) { nodes_ = nodes; @@ -74,10 +72,10 @@ class BasicGraph BasicGraph() = delete; //fill in the edge-related pointers based on stored graph - void get_edges(SizeT* incoming_edges, uint16_t* incoming_edge_count, - SizeT* outgoing_edges, uint16_t* outgoing_edge_count) const + void get_edges(int16_t* incoming_edges, uint16_t* incoming_edge_count, + int16_t* outgoing_edges, uint16_t* outgoing_edge_count) const { - SizeT out_node; + int16_t out_node; for (int i = 0; i < node_count_; i++) { outgoing_edge_count[i] = get_size(outgoing_edges_[i]); @@ -92,7 +90,7 @@ class BasicGraph } } //fill in the nodes and node_count pointer - void get_nodes(uint8_t* nodes, SizeT* node_count) const + void get_nodes(uint8_t* nodes, int16_t* node_count) const { for (int i = 0; i < get_size(nodes_); i++) { @@ -101,9 +99,9 @@ class BasicGraph *node_count = node_count_; } //fill in the node_alignments and node_alignment_count pointers - void get_node_alignments(SizeT* node_alignments, uint16_t* node_alignment_count) const + void get_node_alignments(int16_t* node_alignments, uint16_t* node_alignment_count) const { - SizeT aligned_node; + int16_t aligned_node; for (int i = 0; i < get_size(node_alignments_); i++) { for (int j = 0; j < get_size(node_alignments_[i]); j++) @@ -124,9 +122,9 @@ class BasicGraph } // convert results from outgoing_edges to Uint16Vec2D graph - SizeTVec2D edges_to_graph(SizeT* outgoing_edges, uint16_t* outgoing_edge_count, uint16_t node_count) + Int16Vec2D edges_to_graph(int16_t* outgoing_edges, uint16_t* outgoing_edge_count, uint16_t node_count) { - SizeTVec2D graph(node_count); + Int16Vec2D graph(node_count); for (uint16_t i = 0; i < node_count; i++) { for (uint16_t j = 0; j < outgoing_edge_count[i]; j++) @@ -165,14 +163,14 @@ class BasicGraph return this->outgoing_edges_ == rhs.outgoing_edges_; } - const SizeTVec2D& get_outgoing_edges() const { return outgoing_edges_; } + const Int16Vec2D& get_outgoing_edges() const { return outgoing_edges_; } protected: bool graph_complete_; std::vector nodes_; - SizeTVec2D outgoing_edges_; //this uniquely represents the graph structure; equality of BasicGraph is based on this member. + Int16Vec2D outgoing_edges_; //this uniquely represents the graph structure; equality of BasicGraph is based on this member. Uint16Vec3D outgoing_edges_coverage_; - SizeTVec2D node_alignments_; + Int16Vec2D node_alignments_; std::vector node_coverage_counts_; uint16_t node_count_; }; diff --git a/cudapoa/tests/sorted_graph.hpp b/cudapoa/tests/sorted_graph.hpp index b398667ee..95271ab73 100644 --- a/cudapoa/tests/sorted_graph.hpp +++ b/cudapoa/tests/sorted_graph.hpp @@ -36,16 +36,16 @@ class SortedGraph : public BasicGraph { public: - SortedGraph(std::vector nodes, std::vector sorted_graph, SizeTVec2D outgoing_edges) + SortedGraph(std::vector nodes, std::vector sorted_graph, Int16Vec2D outgoing_edges) : BasicGraph(nodes, outgoing_edges) , sorted_graph_(sorted_graph) { // do nothing for now } - SortedGraph(std::vector nodes, std::vector sorted_graph, - SizeTVec2D node_alignments, std::vector node_coverage_counts, - SizeTVec2D outgoing_edges, Uint16Vec3D outgoing_edges_coverage = {}) + SortedGraph(std::vector nodes, std::vector sorted_graph, + Int16Vec2D node_alignments, std::vector node_coverage_counts, + Int16Vec2D outgoing_edges, Uint16Vec3D outgoing_edges_coverage = {}) : BasicGraph(nodes, outgoing_edges, node_alignments, node_coverage_counts, outgoing_edges_coverage) , sorted_graph_(sorted_graph) { @@ -53,16 +53,16 @@ class SortedGraph : public BasicGraph } SortedGraph() = delete; - void get_node_id_to_pos(SizeT* node_id_to_pos) const + void get_node_id_to_pos(int16_t* node_id_to_pos) const { for (int32_t pos = 0; pos < get_size(sorted_graph_); pos++) { int32_t id = sorted_graph_[pos]; - node_id_to_pos[id] = static_cast(pos); + node_id_to_pos[id] = static_cast(pos); } } - void get_sorted_graph(SizeT* graph) const + void get_sorted_graph(int16_t* graph) const { for (int i = 0; i < get_size(sorted_graph_); i++) { @@ -71,7 +71,7 @@ class SortedGraph : public BasicGraph } protected: - std::vector sorted_graph_; + std::vector sorted_graph_; }; } // namespace cudapoa From 672b13d9bbedf05dd5255ef1af733ce2a82facd6 Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 27 Nov 2020 14:19:30 -0500 Subject: [PATCH 263/281] [cudapoa] addressing PR reviews --- cudapoa/src/cudapoa_nw_banded.cuh | 96 +++++------ cudapoa/tests/Test_CudapoaAddAlignment.cu | 48 +++--- .../tests/Test_CudapoaGenerateConsensus.cu | 42 ++--- cudapoa/tests/Test_CudapoaNW.cu | 154 +++++++++--------- cudapoa/tests/Test_CudapoaTopSort.cu | 16 +- 5 files changed, 174 insertions(+), 182 deletions(-) diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index 829e4264d..aaed72ad9 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -558,7 +558,7 @@ __device__ __forceinline__ // global kernel used in testing, hence uses int16_t for SizeT and ScoreT, // may need to change if test inputs change to long reads -template +template __global__ void runNeedlemanWunschBandedKernel(uint8_t* nodes, SizeT* graph, SizeT* node_id_to_pos, @@ -577,14 +577,11 @@ __global__ void runNeedlemanWunschBandedKernel(uint8_t* nodes, int32_t gap_score, int32_t mismatch_score, int32_t match_score, - SizeT* aligned_nodes, - bool adaptive) + SizeT* aligned_nodes) { float banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_width); - if (adaptive) - { - *aligned_nodes = runNeedlemanWunschBanded(nodes, + *aligned_nodes = runNeedlemanWunschBanded(nodes, graph, node_id_to_pos, graph_count, @@ -602,28 +599,6 @@ __global__ void runNeedlemanWunschBandedKernel(uint8_t* nodes, mismatch_score, match_score, 0); - } - else - { - *aligned_nodes = runNeedlemanWunschBanded(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_length, - scores, - banded_buffer_size, - alignment_graph, - alignment_read, - band_width, - gap_score, - mismatch_score, - match_score, - 0); - } } // Host function that calls the kernel @@ -651,26 +626,51 @@ void runNWbanded(uint8_t* nodes, { static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); - runNeedlemanWunschBandedKernel<<<1, CUDAPOA_BANDED_THREADS_PER_BLOCK>>>(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_length, - scores, - scores_width, - max_nodes_per_graph, - alignment_graph, - alignment_read, - band_width, - gap_score, - mismatch_score, - match_score, - aligned_nodes, - adaptive); + if (adaptive) + { + runNeedlemanWunschBandedKernel<<<1, CUDAPOA_BANDED_THREADS_PER_BLOCK>>>(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + scores_width, + max_nodes_per_graph, + alignment_graph, + alignment_read, + band_width, + gap_score, + mismatch_score, + match_score, + aligned_nodes); + } + else + { + runNeedlemanWunschBandedKernel<<<1, CUDAPOA_BANDED_THREADS_PER_BLOCK>>>(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + scores_width, + max_nodes_per_graph, + alignment_graph, + alignment_read, + band_width, + gap_score, + mismatch_score, + match_score, + aligned_nodes); + } + GW_CU_CHECK_ERR(cudaPeekAtLastError()); } diff --git a/cudapoa/tests/Test_CudapoaAddAlignment.cu b/cudapoa/tests/Test_CudapoaAddAlignment.cu index 2dd1e3b6b..69818a786 100644 --- a/cudapoa/tests/Test_CudapoaAddAlignment.cu +++ b/cudapoa/tests/Test_CudapoaAddAlignment.cu @@ -257,32 +257,32 @@ BasicGraph testAddAlignment(const BasicAlignment& obj) BatchConfig batch_size; // default max_sequence_size = 1024, max_sequences_per_poa = 100 //allocate unified memory so they can be accessed by both host and device. - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_count, sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignments, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignment_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_w, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_w, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_length, sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_sequence_size * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&read, batch_size.max_sequence_size * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&base_weights, batch_size.max_sequence_size * sizeof(int8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_sequence_size * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_coverage_counts, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&sequence_begin_nodes_ids, batch_size.max_sequences_per_poa * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges_coverage, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * batch_size.max_sequences_per_poa * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges_coverage_count, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_count, sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_alignments, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_alignment_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edge_w, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edge_w, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&alignment_length, sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&alignment_graph, batch_size.max_sequence_size * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&read, batch_size.max_sequence_size * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&base_weights, batch_size.max_sequence_size * sizeof(int8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&alignment_read, batch_size.max_sequence_size * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_coverage_counts, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&sequence_begin_nodes_ids, batch_size.max_sequences_per_poa * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edges_coverage, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * batch_size.max_sequences_per_poa * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edges_coverage_count, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); //initialize all 'count' buffers - memset((void**)node_alignment_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)node_coverage_counts, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(node_alignment_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(node_coverage_counts, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); //calculate edge counts on host //3 buffers are disregarded because they don't affect correctness -- incoming_edge_w, outgoing_edge_w, graph diff --git a/cudapoa/tests/Test_CudapoaGenerateConsensus.cu b/cudapoa/tests/Test_CudapoaGenerateConsensus.cu index 486f14bd8..2670279ae 100644 --- a/cudapoa/tests/Test_CudapoaGenerateConsensus.cu +++ b/cudapoa/tests/Test_CudapoaGenerateConsensus.cu @@ -191,29 +191,29 @@ std::string testGenerateConsensus(const BasicGenerateConsensus& obj) BatchConfig batch_size; //allocate unified memory so they can be accessed by both host and device. - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_count, sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_w, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_coverage_counts, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignments, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_alignment_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&predecessors, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_nodes_per_graph * sizeof(int32_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&consensus, batch_size.max_consensus_size * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&coverage, batch_size.max_consensus_size * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_count, sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edge_w, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_coverage_counts, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_alignments, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_ALIGNMENTS * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_alignment_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + + GW_CU_CHECK_ERR(cudaMallocManaged(&predecessors, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&scores, batch_size.max_nodes_per_graph * sizeof(int32_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&consensus, batch_size.max_consensus_size * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&coverage, batch_size.max_consensus_size * sizeof(uint16_t))); //initialize all 'count' buffers - memset((void**)incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)node_coverage_counts, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)node_alignment_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(node_coverage_counts, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(node_alignment_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); //calculate edge counts on host obj.get_graph_buffers(nodes, node_count, diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index 12409c5c4..8ae7220a4 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -190,44 +190,41 @@ std::vector getNWTestCases() NWAnswer testNW(const BasicNW& obj) { //declare device buffer - uint8_t* nodes; - int16_t* graph; - int16_t* node_id_to_pos; - int16_t graph_count; //local - uint16_t* incoming_edge_count; - int16_t* incoming_edges; - uint16_t* outgoing_edge_count; - int16_t* outgoing_edges; - uint8_t* read; - uint16_t read_count; //local - int16_t* scores; - int16_t* alignment_graph; - int16_t* alignment_read; - int32_t gap_score; - int32_t mismatch_score; - int32_t match_score; - int16_t* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) - BatchConfig batch_size; //default max_sequence_size = 1024, max_sequences_per_poa = 100 + uint8_t* nodes = nullptr; + int16_t* graph = nullptr; + int16_t* node_id_to_pos = nullptr; + int16_t graph_count = 0; //local + uint16_t* incoming_edge_count = nullptr; + int16_t* incoming_edges = nullptr; + uint16_t* outgoing_edge_count = nullptr; + int16_t* outgoing_edges = nullptr; + uint8_t* read = nullptr; + uint16_t read_count = 0; //local + int16_t* scores = nullptr; + int16_t* alignment_graph = nullptr; + int16_t* alignment_read = nullptr; + int16_t* aligned_nodes = nullptr; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) + BatchConfig batch_size; //default max_sequence_size = 1024, max_sequences_per_poa = 100 //allocate unified memory so they can be accessed by both host and device. - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&read, batch_size.max_sequence_size * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&aligned_nodes, sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&scores, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&alignment_graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&read, batch_size.max_sequence_size * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&alignment_read, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&aligned_nodes, sizeof(int16_t))); //initialize all 'count' buffers - memset((void**)incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(int16_t)); - memset((void**)scores, 0, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t)); + memset(incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(int16_t)); + memset(scores, 0, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t)); //calculate edge counts on host obj.get_graph_buffers(incoming_edges, incoming_edge_count, @@ -235,9 +232,9 @@ NWAnswer testNW(const BasicNW& obj) nodes, &graph_count, graph, node_id_to_pos); obj.get_read_buffers(read, &read_count); - gap_score = BasicNW::gap_score_; - mismatch_score = BasicNW::mismatch_score_; - match_score = BasicNW::match_score_; + int32_t gap_score = BasicNW::gap_score_; + int32_t mismatch_score = BasicNW::mismatch_score_; + int32_t match_score = BasicNW::match_score_; //call the host wrapper of nw kernel runNW(nodes, @@ -309,56 +306,53 @@ INSTANTIATE_TEST_SUITE_P(TestNW, NWTest, ValuesIn(getNWTestCases())); NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) { //declare device buffer - uint8_t* nodes; - int16_t* graph; - int16_t* node_id_to_pos; - int16_t graph_count; //local - uint16_t* incoming_edge_count; - int16_t* incoming_edges; - uint16_t* outgoing_edge_count; - int16_t* outgoing_edges; - uint8_t* read; - uint16_t read_count; //local - int16_t* scores; - int16_t* traces; - int16_t* alignment_graph; - int16_t* alignment_read; - int32_t gap_score; - int32_t mismatch_score; - int32_t match_score; - int16_t* aligned_nodes; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) - BandMode band_mode = traceback ? (adaptive ? BandMode::adaptive_band_traceback : BandMode::static_band_traceback) + uint8_t* nodes = nullptr; + int16_t* graph = nullptr; + int16_t* node_id_to_pos = nullptr; + int16_t graph_count = 0; //local + uint16_t* incoming_edge_count = nullptr; + int16_t* incoming_edges = nullptr; + uint16_t* outgoing_edge_count = nullptr; + int16_t* outgoing_edges = nullptr; + uint8_t* read = nullptr; + uint16_t read_count = 0; //local + int16_t* scores = nullptr; + int16_t* traces = nullptr; + int16_t* alignment_graph = nullptr; + int16_t* alignment_read = nullptr; + int16_t* aligned_nodes = nullptr; //local; to store num of nodes aligned (length of alignment_graph and alignment_read) + BandMode band_mode = traceback ? (adaptive ? BandMode::adaptive_band_traceback : BandMode::static_band_traceback) : (adaptive ? BandMode::adaptive_band : BandMode::static_band); BatchConfig batch_size(1024 /*max_sequence_size*/, 2 /*max_sequences_per_poa*/, 128 /*= band_width*/, band_mode); //allocate unified memory so they can be accessed by both host and device. - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&read, batch_size.max_sequence_size * sizeof(uint8_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&alignment_read, batch_size.max_nodes_per_graph * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&aligned_nodes, sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&nodes, batch_size.max_nodes_per_graph * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&node_id_to_pos, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edges, batch_size.max_nodes_per_graph * CUDAPOA_MAX_NODE_EDGES * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edge_count, batch_size.max_nodes_per_graph * sizeof(uint16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&alignment_graph, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&read, batch_size.max_sequence_size * sizeof(uint8_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&alignment_read, batch_size.max_nodes_per_graph * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&aligned_nodes, sizeof(int16_t))); if (traceback) { - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_banded_pred_distance * batch_size.matrix_sequence_dimension * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&traces, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&scores, batch_size.max_banded_pred_distance * batch_size.matrix_sequence_dimension * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&traces, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); } else { - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&scores, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&scores, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t))); } //initialize all 'count' buffers - memset((void**)incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); - memset((void**)node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(int16_t)); - memset((void**)scores, 0, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t)); + memset(incoming_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(outgoing_edge_count, 0, batch_size.max_nodes_per_graph * sizeof(uint16_t)); + memset(node_id_to_pos, 0, batch_size.max_nodes_per_graph * sizeof(int16_t)); + memset(scores, 0, batch_size.max_nodes_per_graph * batch_size.matrix_sequence_dimension * sizeof(int16_t)); //calculate edge counts on host obj.get_graph_buffers(incoming_edges, incoming_edge_count, @@ -366,9 +360,9 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) nodes, &graph_count, graph, node_id_to_pos); obj.get_read_buffers(read, &read_count); - gap_score = BasicNW::gap_score_; - mismatch_score = BasicNW::mismatch_score_; - match_score = BasicNW::match_score_; + int32_t gap_score = BasicNW::gap_score_; + int32_t mismatch_score = BasicNW::mismatch_score_; + int32_t match_score = BasicNW::match_score_; //call the host wrapper of nw kernels if (traceback) @@ -450,7 +444,7 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) class NWbandedTest : public ::testing::Test { public: - BasicNW* nw; + std::unique_ptr nw; public: void SetUp() @@ -469,10 +463,8 @@ public: } std::vector read(read_str.begin(), read_str.end()); // setup nw - nw = new BasicNW(nodes, sorted_graph, outgoing_edges, read); + nw = std::make_unique(nodes, sorted_graph, outgoing_edges, read); } - - void TearDown() { delete nw; } }; TEST_F(NWbandedTest, NWSaticBandvsFull) diff --git a/cudapoa/tests/Test_CudapoaTopSort.cu b/cudapoa/tests/Test_CudapoaTopSort.cu index 2cee8fac6..ce05df1a8 100644 --- a/cudapoa/tests/Test_CudapoaTopSort.cu +++ b/cudapoa/tests/Test_CudapoaTopSort.cu @@ -76,16 +76,16 @@ std::string testTopSortDeviceUtil(int16_t node_count, Int16Vec2D outgoing_edges_ size_t graph_size = node_count * sizeof(uint16_t); //allocate unified memory so they can be accessed by both host and device. - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&sorted_poa, node_count * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&sorted_poa_node_map, node_count * sizeof(int16_t))); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&incoming_edge_count, graph_size)); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edges, node_count * sizeof(int16_t) * CUDAPOA_MAX_NODE_EDGES)); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&outgoing_edge_count, graph_size)); - GW_CU_CHECK_ERR(cudaMallocManaged((void**)&local_incoming_edge_count, graph_size)); + GW_CU_CHECK_ERR(cudaMallocManaged(&sorted_poa, node_count * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&sorted_poa_node_map, node_count * sizeof(int16_t))); + GW_CU_CHECK_ERR(cudaMallocManaged(&incoming_edge_count, graph_size)); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edges, node_count * sizeof(int16_t) * CUDAPOA_MAX_NODE_EDGES)); + GW_CU_CHECK_ERR(cudaMallocManaged(&outgoing_edge_count, graph_size)); + GW_CU_CHECK_ERR(cudaMallocManaged(&local_incoming_edge_count, graph_size)); //initialize incoming_edge_count & local_incoming_edge_count - memset((void**)incoming_edge_count, 0, graph_size); - memset((void**)local_incoming_edge_count, 0, graph_size); + memset(incoming_edge_count, 0, graph_size); + memset(local_incoming_edge_count, 0, graph_size); //calculate edge counts on host From bfc349c79c01c91f7f2b9aa4d9b2bc982473dfd0 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 30 Nov 2020 16:20:27 -0500 Subject: [PATCH 264/281] [cudapoa] addressing PR reviews --- cudapoa/src/cudapoa_generate_consensus.cuh | 2 - cudapoa/src/cudapoa_kernels.cuh | 10 ++--- cudapoa/src/cudapoa_nw.cuh | 4 +- cudapoa/src/cudapoa_nw_banded.cuh | 16 +++---- cudapoa/src/cudapoa_nw_tb_banded.cuh | 10 ++--- cudapoa/tests/Test_CudapoaAddAlignment.cu | 42 +++++++++---------- .../tests/Test_CudapoaGenerateConsensus.cu | 32 +++++++------- cudapoa/tests/Test_CudapoaNW.cu | 8 ++-- cudapoa/tests/Test_CudapoaTopSort.cu | 14 +++---- cudapoa/tests/sorted_graph.hpp | 4 +- 10 files changed, 70 insertions(+), 72 deletions(-) diff --git a/cudapoa/src/cudapoa_generate_consensus.cuh b/cudapoa/src/cudapoa_generate_consensus.cuh index 723592a28..ceffe64af 100644 --- a/cudapoa/src/cudapoa_generate_consensus.cuh +++ b/cudapoa/src/cudapoa_generate_consensus.cuh @@ -410,8 +410,6 @@ void generateConsensusTestHost(uint8_t* nodes, uint16_t* node_alignment_count, int32_t max_limit_consensus_size) { - static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); - generateConsensusTestKernel<<<1, 1>>>(nodes, node_count, graph, diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index e42a3f95d..a4dbe6e76 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -73,8 +73,8 @@ namespace cudapoa * @param[in] mismatch_score Score for finding a mismatch in alignment * @param[in] match_score Score for finding a match in alignment */ -template -__launch_bounds__(TRACEBACK ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS : GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_64_REGISTERS) +template +__launch_bounds__(Traceback ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS : GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_64_REGISTERS) __global__ void generatePOAKernel(uint8_t* consensus_d, uint8_t* sequences_d, int8_t* base_weights_d, @@ -142,7 +142,7 @@ __launch_bounds__(TRACEBACK ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS TraceT* traceback = traceback_d; // only used in traceback int32_t scores_width = window_details_d[window_idx].scores_width; // only used in non-traceback - if (TRACEBACK) + if (Traceback) { // buffer size for scores, in traceback we only need to store part of the scores matrix banded_buffer_size = static_cast(max_pred_distance) * static_cast(scores_matrix_width); @@ -267,7 +267,7 @@ __launch_bounds__(TRACEBACK ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS // Run Needleman-Wunsch alignment between graph and new sequence. SizeT alignment_length; - if (TRACEBACK) + if (Traceback) { // Adaptive band with traceback ------------------------------------------------------------------------ if (BM == BandMode::adaptive_band_traceback && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) @@ -459,7 +459,7 @@ __launch_bounds__(TRACEBACK ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS } return; } - if (TRACEBACK) + if (Traceback) { if (alignment_length == CUDAPOA_KERNEL_NW_TRACEBACK_BUFFER_FAILED) { diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index 55179d72f..81944696a 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -474,6 +474,8 @@ __global__ void runNeedlemanWunschKernel(uint8_t* nodes, int32_t match_score, SizeT* aligned_nodes) { + static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); + *aligned_nodes = runNeedlemanWunsch(nodes, graph, node_id_to_pos, @@ -512,8 +514,6 @@ void runNW(uint8_t* nodes, int32_t match_score, SizeT* aligned_nodes) { - static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); - runNeedlemanWunschKernel<<<1, CUDAPOA_THREADS_PER_BLOCK>>>(nodes, graph, node_id_to_pos, diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index aaed72ad9..ca609a92d 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -177,7 +177,7 @@ __device__ __forceinline__ void initialize_band(ScoreT* scores, template + bool Adaptive = true> __device__ __forceinline__ int32_t runNeedlemanWunschBanded(SeqT* nodes, @@ -210,7 +210,7 @@ __device__ __forceinline__ // Set band-width based on scores matrix aspect ratio //--------------------------------------------------------- - if (ADAPTIVE) + if (Adaptive) { if (gradient > 1.1) // ad-hoc rule 1.a { @@ -236,7 +236,7 @@ __device__ __forceinline__ // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 int32_t band_shift = band_width / 2; - if (ADAPTIVE) + if (Adaptive) { // rerun code is defined in backtracking loop from previous alignment try // SHIFT_ADAPTIVE_BAND_TO_LEFT means traceback path was too close to the left bound of band @@ -439,7 +439,7 @@ __device__ __forceinline__ // Check if move is diagonal. if (i != 0 && j != 0) { - if (ADAPTIVE) + if (Adaptive) { // no need to request rerun if (a) it's not the first run, (b) band_width == CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH already if (rerun == 0 && band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) @@ -558,7 +558,7 @@ __device__ __forceinline__ // global kernel used in testing, hence uses int16_t for SizeT and ScoreT, // may need to change if test inputs change to long reads -template +template __global__ void runNeedlemanWunschBandedKernel(uint8_t* nodes, SizeT* graph, SizeT* node_id_to_pos, @@ -579,9 +579,11 @@ __global__ void runNeedlemanWunschBandedKernel(uint8_t* nodes, int32_t match_score, SizeT* aligned_nodes) { + static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); + float banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_width); - *aligned_nodes = runNeedlemanWunschBanded(nodes, + *aligned_nodes = runNeedlemanWunschBanded(nodes, graph, node_id_to_pos, graph_count, @@ -624,8 +626,6 @@ void runNWbanded(uint8_t* nodes, SizeT* aligned_nodes, bool adaptive) { - static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); - if (adaptive) { runNeedlemanWunschBandedKernel<<<1, CUDAPOA_BANDED_THREADS_PER_BLOCK>>>(nodes, diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index cba8d0025..2a2f0e15d 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -265,7 +265,7 @@ template + bool Adaptive = true> __device__ __forceinline__ int32_t runNeedlemanWunschBandedTraceback(SeqT* nodes, @@ -303,7 +303,7 @@ __device__ __forceinline__ // band_shift defines distance of band_start from the scores matrix diagonal, ad-hoc rule 4 int32_t band_shift = band_width / 2; - if (ADAPTIVE) + if (Adaptive) { // rerun code is defined in backtracking loop from previous alignment try // SHIFT_ADAPTIVE_BAND_TO_LEFT means traceback path was too close to the left bound of band @@ -600,7 +600,7 @@ __device__ __forceinline__ i -= trace; j--; - if (ADAPTIVE) + if (Adaptive) { // no need to request rerun if (a) it's not the first run, (b) band_width == CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH already if (rerun == 0 && band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) @@ -668,6 +668,8 @@ __global__ void runNeedlemanWunschBandedTBKernel(uint8_t* nodes, SizeT* aligned_nodes, bool adaptive) { + static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); + float banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_width); if (adaptive) @@ -745,8 +747,6 @@ void runNWbandedTB(uint8_t* nodes, SizeT* aligned_nodes, bool adaptive) { - static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); - runNeedlemanWunschBandedTBKernel<<<1, CUDAPOA_BANDED_THREADS_PER_BLOCK>>>(nodes, graph, node_id_to_pos, diff --git a/cudapoa/tests/Test_CudapoaAddAlignment.cu b/cudapoa/tests/Test_CudapoaAddAlignment.cu index 69818a786..6a807cd62 100644 --- a/cudapoa/tests/Test_CudapoaAddAlignment.cu +++ b/cudapoa/tests/Test_CudapoaAddAlignment.cu @@ -233,27 +233,27 @@ std::vector getAddAlignmentTestCases() BasicGraph testAddAlignment(const BasicAlignment& obj) { //declare device buffer - uint8_t* nodes; - int16_t* node_count; - int16_t* node_alignments; - uint16_t* node_alignment_count; - int16_t* incoming_edges; - uint16_t* incoming_edge_count; - int16_t* outgoing_edges; - uint16_t* outgoing_edge_count; - uint16_t* incoming_edge_w; - uint16_t* outgoing_edge_w; - int16_t* alignment_length; - int16_t* graph; - int16_t* alignment_graph; - uint8_t* read; - int8_t* base_weights; - int16_t* alignment_read; - uint16_t* node_coverage_counts; - int16_t* sequence_begin_nodes_ids; - uint16_t* outgoing_edges_coverage; - uint16_t* outgoing_edges_coverage_count; - uint16_t s = 0; + uint8_t* nodes = nullptr; + int16_t* node_count = nullptr; + int16_t* node_alignments = nullptr; + uint16_t* node_alignment_count = nullptr; + int16_t* incoming_edges = nullptr; + uint16_t* incoming_edge_count = nullptr; + int16_t* outgoing_edges = nullptr; + uint16_t* outgoing_edge_count = nullptr; + uint16_t* incoming_edge_w = nullptr; + uint16_t* outgoing_edge_w = nullptr; + int16_t* alignment_length = nullptr; + int16_t* graph = nullptr; + int16_t* alignment_graph = nullptr; + uint8_t* read = nullptr; + int8_t* base_weights = nullptr; + int16_t* alignment_read = nullptr; + uint16_t* node_coverage_counts = nullptr; + int16_t* sequence_begin_nodes_ids = nullptr; + uint16_t* outgoing_edges_coverage = nullptr; + uint16_t* outgoing_edges_coverage_count = nullptr; + uint16_t s = 0; BatchConfig batch_size; // default max_sequence_size = 1024, max_sequences_per_poa = 100 //allocate unified memory so they can be accessed by both host and device. diff --git a/cudapoa/tests/Test_CudapoaGenerateConsensus.cu b/cudapoa/tests/Test_CudapoaGenerateConsensus.cu index 2670279ae..bb884a308 100644 --- a/cudapoa/tests/Test_CudapoaGenerateConsensus.cu +++ b/cudapoa/tests/Test_CudapoaGenerateConsensus.cu @@ -168,24 +168,24 @@ std::vector getGenerateConsensusTestCases() std::string testGenerateConsensus(const BasicGenerateConsensus& obj) { //declare device buffer - uint8_t* nodes; - int16_t* node_count; - int16_t* graph; - int16_t* node_id_to_pos; - int16_t* incoming_edges; - uint16_t* incoming_edge_count; - int16_t* outgoing_edges; - uint16_t* outgoing_edge_count; - uint16_t* incoming_edge_w; - uint16_t* node_coverage_counts; - int16_t* node_alignments; - uint16_t* node_alignment_count; + uint8_t* nodes = nullptr; + int16_t* node_count = nullptr; + int16_t* graph = nullptr; + int16_t* node_id_to_pos = nullptr; + int16_t* incoming_edges = nullptr; + uint16_t* incoming_edge_count = nullptr; + int16_t* outgoing_edges = nullptr; + uint16_t* outgoing_edge_count = nullptr; + uint16_t* incoming_edge_w = nullptr; + uint16_t* node_coverage_counts = nullptr; + int16_t* node_alignments = nullptr; + uint16_t* node_alignment_count = nullptr; //buffers that don't need initialization - int16_t* predecessors; - int32_t* scores; - uint8_t* consensus; - uint16_t* coverage; + int16_t* predecessors = nullptr; + int32_t* scores = nullptr; + uint8_t* consensus = nullptr; + uint16_t* coverage = nullptr; //default data size limits BatchConfig batch_size; diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index 8ae7220a4..e92d60e7a 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -259,8 +259,8 @@ NWAnswer testNW(const BasicNW& obj) //input and output buffers are the same ones in unified memory, so the results are updated in place //results are stored in alignment_graph and alignment_read; return string representation of those - auto res = std::make_pair(genomeworks::stringutils::array_to_string(alignment_graph, *aligned_nodes, ","), - genomeworks::stringutils::array_to_string(alignment_read, *aligned_nodes, ",")); + auto res = std::make_pair(genomeworks::stringutils::array_to_string(alignment_graph, *aligned_nodes, ","), + genomeworks::stringutils::array_to_string(alignment_read, *aligned_nodes, ",")); GW_CU_CHECK_ERR(cudaFree(nodes)); GW_CU_CHECK_ERR(cudaFree(graph)); @@ -418,8 +418,8 @@ NWAnswer testNWbanded(const BasicNW& obj, bool adaptive, bool traceback = false) //input and output buffers are the same ones in unified memory, so the results are updated in place //results are stored in alignment_graph and alignment_read; return string representation of those - auto res = std::make_pair(genomeworks::stringutils::array_to_string(alignment_graph, *aligned_nodes, ","), - genomeworks::stringutils::array_to_string(alignment_read, *aligned_nodes, ",")); + auto res = std::make_pair(genomeworks::stringutils::array_to_string(alignment_graph, *aligned_nodes, ","), + genomeworks::stringutils::array_to_string(alignment_read, *aligned_nodes, ",")); GW_CU_CHECK_ERR(cudaFree(nodes)); GW_CU_CHECK_ERR(cudaFree(graph)); diff --git a/cudapoa/tests/Test_CudapoaTopSort.cu b/cudapoa/tests/Test_CudapoaTopSort.cu index ce05df1a8..6867a3789 100644 --- a/cudapoa/tests/Test_CudapoaTopSort.cu +++ b/cudapoa/tests/Test_CudapoaTopSort.cu @@ -66,12 +66,12 @@ std::vector getTopSortTestCases() std::string testTopSortDeviceUtil(int16_t node_count, Int16Vec2D outgoing_edges_vec) { //declare device buffer - int16_t* sorted_poa; - int16_t* sorted_poa_node_map; - uint16_t* incoming_edge_count; - int16_t* outgoing_edges; - uint16_t* outgoing_edge_count; - uint16_t* local_incoming_edge_count; + int16_t* sorted_poa = nullptr; + int16_t* sorted_poa_node_map = nullptr; + uint16_t* incoming_edge_count = nullptr; + int16_t* outgoing_edges = nullptr; + uint16_t* outgoing_edge_count = nullptr; + uint16_t* local_incoming_edge_count = nullptr; size_t graph_size = node_count * sizeof(uint16_t); @@ -113,7 +113,7 @@ std::string testTopSortDeviceUtil(int16_t node_count, Int16Vec2D outgoing_edges_ GW_CU_CHECK_ERR(cudaDeviceSynchronize()); - std::string res = genomeworks::stringutils::array_to_string(sorted_poa, node_count); + std::string res = genomeworks::stringutils::array_to_string(sorted_poa, node_count); GW_CU_CHECK_ERR(cudaFree(sorted_poa)); GW_CU_CHECK_ERR(cudaFree(sorted_poa_node_map)); diff --git a/cudapoa/tests/sorted_graph.hpp b/cudapoa/tests/sorted_graph.hpp index 95271ab73..2dbfd190d 100644 --- a/cudapoa/tests/sorted_graph.hpp +++ b/cudapoa/tests/sorted_graph.hpp @@ -55,10 +55,10 @@ class SortedGraph : public BasicGraph void get_node_id_to_pos(int16_t* node_id_to_pos) const { - for (int32_t pos = 0; pos < get_size(sorted_graph_); pos++) + for (int16_t pos = 0; pos < get_size(sorted_graph_); pos++) { int32_t id = sorted_graph_[pos]; - node_id_to_pos[id] = static_cast(pos); + node_id_to_pos[id] = pos; } } From e64424fbdce4037dfffba30ee8dc34fbdde4544b Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 30 Nov 2020 16:59:47 -0500 Subject: [PATCH 265/281] [cudapoa] addressing PR reviews --- cudapoa/src/cudapoa_kernels.cuh | 212 +++++++++++++-------------- cudapoa/src/cudapoa_nw.cuh | 64 ++++---- cudapoa/src/cudapoa_nw_banded.cuh | 72 ++++----- cudapoa/src/cudapoa_nw_tb_banded.cuh | 124 ++++++++-------- cudapoa/tests/Test_CudapoaNW.cu | 4 +- 5 files changed, 238 insertions(+), 238 deletions(-) diff --git a/cudapoa/src/cudapoa_kernels.cuh b/cudapoa/src/cudapoa_kernels.cuh index a4dbe6e76..e8a115bca 100644 --- a/cudapoa/src/cudapoa_kernels.cuh +++ b/cudapoa/src/cudapoa_kernels.cuh @@ -272,57 +272,31 @@ __launch_bounds__(Traceback ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS // Adaptive band with traceback ------------------------------------------------------------------------ if (BM == BandMode::adaptive_band_traceback && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) { - alignment_length = runNeedlemanWunschBandedTraceback(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - traceback, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - max_pred_distance, - gap_score, - mismatch_score, - match_score, - 0); + alignment_length = needlemanWunschBandedTraceback(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + max_pred_distance, + gap_score, + mismatch_score, + match_score, + 0); __syncwarp(); if (alignment_length == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_RIGHT) { // rerun with extended and shifted band-width - alignment_length = runNeedlemanWunschBandedTraceback(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - traceback, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - max_pred_distance, - gap_score, - mismatch_score, - match_score, - alignment_length); - __syncwarp(); - } - } - // Static band with traceback -------------------------------------------------------------------------- - else if (BM == BandMode::static_band_traceback || (BM == BandMode::adaptive_band_traceback && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) - { - alignment_length = runNeedlemanWunschBandedTraceback(nodes, + alignment_length = needlemanWunschBandedTraceback(nodes, sorted_poa, node_id_to_pos, sequence_lengths[0], @@ -342,6 +316,32 @@ __launch_bounds__(Traceback ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS mismatch_score, match_score, alignment_length); + __syncwarp(); + } + } + // Static band with traceback -------------------------------------------------------------------------- + else if (BM == BandMode::static_band_traceback || (BM == BandMode::adaptive_band_traceback && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) + { + alignment_length = needlemanWunschBandedTraceback(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + max_pred_distance, + gap_score, + mismatch_score, + match_score, + alignment_length); __syncwarp(); } } @@ -351,54 +351,30 @@ __launch_bounds__(Traceback ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS if (BM == BandMode::adaptive_band && static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH) { // run in adaptive mode only if static_band_width < CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH - alignment_length = runNeedlemanWunschBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score, - 0); + alignment_length = needlemanWunschBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + 0); __syncwarp(); if (alignment_length == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_LEFT || alignment_length == CUDAPOA_SHIFT_ADAPTIVE_BAND_TO_RIGHT) { // rerun with extended and shifted band-width - alignment_length = runNeedlemanWunschBanded(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - banded_buffer_size, - alignment_graph, - alignment_read, - static_band_width, - gap_score, - mismatch_score, - match_score, - alignment_length); - __syncwarp(); - } - } - // Static band --------------------------------------------------------------------------------------- - else if (BM == BandMode::static_band || (BM == BandMode::adaptive_band && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) - { - alignment_length = runNeedlemanWunschBanded(nodes, + alignment_length = needlemanWunschBanded(nodes, sorted_poa, node_id_to_pos, sequence_lengths[0], @@ -416,27 +392,51 @@ __launch_bounds__(Traceback ? GW_POA_KERNELS_MAX_THREADS_PER_BLOCK_72_REGISTERS mismatch_score, match_score, alignment_length); + __syncwarp(); + } + } + // Static band --------------------------------------------------------------------------------------- + else if (BM == BandMode::static_band || (BM == BandMode::adaptive_band && static_band_width >= CUDAPOA_MAX_ADAPTIVE_BAND_WIDTH)) + { + alignment_length = needlemanWunschBanded(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + static_band_width, + gap_score, + mismatch_score, + match_score, + alignment_length); __syncwarp(); } // Full band ------------------------------------------------------------------------------------------- else if (BM == BandMode::full_band) { - alignment_length = runNeedlemanWunsch(nodes, - sorted_poa, - node_id_to_pos, - sequence_lengths[0], - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - sequence, - seq_len, - scores, - scores_width, - alignment_graph, - alignment_read, - gap_score, - mismatch_score, - match_score); + alignment_length = needlemanWunsch(nodes, + sorted_poa, + node_id_to_pos, + sequence_lengths[0], + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + sequence, + seq_len, + scores, + scores_width, + alignment_graph, + alignment_read, + gap_score, + mismatch_score, + match_score); __syncwarp(); } } diff --git a/cudapoa/src/cudapoa_nw.cuh b/cudapoa/src/cudapoa_nw.cuh index 81944696a..4ab09ffa1 100644 --- a/cudapoa/src/cudapoa_nw.cuh +++ b/cudapoa/src/cudapoa_nw.cuh @@ -152,22 +152,22 @@ template __device__ __forceinline__ int32_t - runNeedlemanWunsch(SeqT* nodes, - SizeT* graph, - SizeT* node_id_to_pos, - int32_t graph_count, - uint16_t* incoming_edge_count, - SizeT* incoming_edges, - uint16_t* outgoing_edge_count, - SeqT* read, - int32_t read_length, - ScoreT* scores, - int32_t scores_width, - SizeT* alignment_graph, - SizeT* alignment_read, - int32_t gap_score, - int32_t mismatch_score, - int32_t match_score) + needlemanWunsch(SeqT* nodes, + SizeT* graph, + SizeT* node_id_to_pos, + int32_t graph_count, + uint16_t* incoming_edge_count, + SizeT* incoming_edges, + uint16_t* outgoing_edge_count, + SeqT* read, + int32_t read_length, + ScoreT* scores, + int32_t scores_width, + SizeT* alignment_graph, + SizeT* alignment_read, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score) { static_assert(CPT == 4, "implementation currently supports only 4 cells per thread"); @@ -476,22 +476,22 @@ __global__ void runNeedlemanWunschKernel(uint8_t* nodes, { static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); - *aligned_nodes = runNeedlemanWunsch(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_length, - scores, - scores_width, - alignment_graph, - alignment_read, - gap_score, - mismatch_score, - match_score); + *aligned_nodes = needlemanWunsch(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + scores_width, + alignment_graph, + alignment_read, + gap_score, + mismatch_score, + match_score); } // Host function that calls the kernel diff --git a/cudapoa/src/cudapoa_nw_banded.cuh b/cudapoa/src/cudapoa_nw_banded.cuh index ca609a92d..c754e1661 100644 --- a/cudapoa/src/cudapoa_nw_banded.cuh +++ b/cudapoa/src/cudapoa_nw_banded.cuh @@ -180,24 +180,24 @@ template __device__ __forceinline__ int32_t - runNeedlemanWunschBanded(SeqT* nodes, - SizeT* graph, - SizeT* node_id_to_pos, - int32_t graph_count, - uint16_t* incoming_edge_count, - SizeT* incoming_edges, - uint16_t* outgoing_edge_count, - SeqT* read, - int32_t read_length, - ScoreT* scores, - float max_buffer_size, - SizeT* alignment_graph, - SizeT* alignment_read, - int32_t band_width, - int32_t gap_score, - int32_t mismatch_score, - int32_t match_score, - int32_t rerun) + needlemanWunschBanded(SeqT* nodes, + SizeT* graph, + SizeT* node_id_to_pos, + int32_t graph_count, + uint16_t* incoming_edge_count, + SizeT* incoming_edges, + uint16_t* outgoing_edge_count, + SeqT* read, + int32_t read_length, + ScoreT* scores, + float max_buffer_size, + SizeT* alignment_graph, + SizeT* alignment_read, + int32_t band_width, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, + int32_t rerun) { const ScoreT min_score_value = numeric_limits::min() / 2; @@ -583,24 +583,24 @@ __global__ void runNeedlemanWunschBandedKernel(uint8_t* nodes, float banded_buffer_size = static_cast(max_nodes_per_graph) * static_cast(scores_width); - *aligned_nodes = runNeedlemanWunschBanded(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_length, - scores, - banded_buffer_size, - alignment_graph, - alignment_read, - band_width, - gap_score, - mismatch_score, - match_score, - 0); + *aligned_nodes = needlemanWunschBanded(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + banded_buffer_size, + alignment_graph, + alignment_read, + band_width, + gap_score, + mismatch_score, + match_score, + 0); } // Host function that calls the kernel diff --git a/cudapoa/src/cudapoa_nw_tb_banded.cuh b/cudapoa/src/cudapoa_nw_tb_banded.cuh index 2a2f0e15d..38ca64b0b 100644 --- a/cudapoa/src/cudapoa_nw_tb_banded.cuh +++ b/cudapoa/src/cudapoa_nw_tb_banded.cuh @@ -268,26 +268,26 @@ template __device__ __forceinline__ int32_t - runNeedlemanWunschBandedTraceback(SeqT* nodes, - SizeT* graph, - SizeT* node_id_to_pos, - int32_t graph_count, - uint16_t* incoming_edge_count, - SizeT* incoming_edges, - uint16_t* outgoing_edge_count, - SeqT* read, - int32_t read_length, - ScoreT* scores, - TraceT* traceback, - float max_buffer_size, - SizeT* alignment_graph, - SizeT* alignment_read, - int32_t band_width, - int32_t score_matrix_height, - int32_t gap_score, - int32_t mismatch_score, - int32_t match_score, - int32_t rerun) + needlemanWunschBandedTraceback(SeqT* nodes, + SizeT* graph, + SizeT* node_id_to_pos, + int32_t graph_count, + uint16_t* incoming_edge_count, + SizeT* incoming_edges, + uint16_t* outgoing_edge_count, + SeqT* read, + int32_t read_length, + ScoreT* scores, + TraceT* traceback, + float max_buffer_size, + SizeT* alignment_graph, + SizeT* alignment_read, + int32_t band_width, + int32_t score_matrix_height, + int32_t gap_score, + int32_t mismatch_score, + int32_t match_score, + int32_t rerun) { const ScoreT min_score_value = numeric_limits::min() / 2; @@ -674,51 +674,51 @@ __global__ void runNeedlemanWunschBandedTBKernel(uint8_t* nodes, if (adaptive) { - *aligned_nodes = runNeedlemanWunschBandedTraceback(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_length, - scores, - traceback, - banded_buffer_size, - alignment_graph, - alignment_read, - band_width, - score_matrix_height, - gap_score, - mismatch_score, - match_score, - 0); + *aligned_nodes = needlemanWunschBandedTraceback(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + band_width, + score_matrix_height, + gap_score, + mismatch_score, + match_score, + 0); } else { - *aligned_nodes = runNeedlemanWunschBandedTraceback(nodes, - graph, - node_id_to_pos, - graph_count, - incoming_edge_count, - incoming_edges, - outgoing_edge_count, - read, - read_length, - scores, - traceback, - banded_buffer_size, - alignment_graph, - alignment_read, - band_width, - score_matrix_height, - gap_score, - mismatch_score, - match_score, - 0); + *aligned_nodes = needlemanWunschBandedTraceback(nodes, + graph, + node_id_to_pos, + graph_count, + incoming_edge_count, + incoming_edges, + outgoing_edge_count, + read, + read_length, + scores, + traceback, + banded_buffer_size, + alignment_graph, + alignment_read, + band_width, + score_matrix_height, + gap_score, + mismatch_score, + match_score, + 0); } } diff --git a/cudapoa/tests/Test_CudapoaNW.cu b/cudapoa/tests/Test_CudapoaNW.cu index e92d60e7a..2d5a56481 100644 --- a/cudapoa/tests/Test_CudapoaNW.cu +++ b/cudapoa/tests/Test_CudapoaNW.cu @@ -467,7 +467,7 @@ public: } }; -TEST_F(NWbandedTest, NWSaticBandvsFull) +TEST_F(NWbandedTest, NWStaticBandvsFull) { auto full_alignment_results = testNW(*nw); auto static_banded_results = testNWbanded(*nw, false); @@ -487,7 +487,7 @@ TEST_F(NWbandedTest, NWAdaptiveBandvsFull) EXPECT_EQ(full_alignment_results.second, adaptive_banded_results.second); } -TEST_F(NWbandedTest, NWSaticBandTracebackvsFull) +TEST_F(NWbandedTest, NWStaticBandTracebackvsFull) { auto full_alignment_results = testNW(*nw); auto static_banded_tb_results = testNWbanded(*nw, false, true); From c209460343152a6bea96cb113f6dd35e5ecde495 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 30 Nov 2020 19:31:29 -0500 Subject: [PATCH 266/281] [cudapoa] defined decode_error() to generate proper error message --- .../genomeworks/cudapoa/cudapoa.hpp | 7 ++++ cudapoa/src/cudapoa.cpp | 33 ++++++++++++++++ cudapoa/src/cudapoa_batch.cuh | 38 ++----------------- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp index 5df1e9249..2fc483117 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp @@ -16,6 +16,8 @@ #pragma once +#include + namespace claraparabricks { @@ -45,6 +47,11 @@ enum StatusType exceeded_maximum_predecessor_distance }; +/// Generate corresponding error message for a given error type +/// \param [in] error_type input error code +/// \return corresponding error message +std::string decode_error(StatusType error_type); + /// Banding mode used in Needleman-Wunsch algorithm /// - full_band performs computations on full scores matrix, highest accuracy /// - static_band performs computations on a fixed band along scores matrix diagonal, fastest implementation diff --git a/cudapoa/src/cudapoa.cpp b/cudapoa/src/cudapoa.cpp index 9699c663d..7c889d0cf 100644 --- a/cudapoa/src/cudapoa.cpp +++ b/cudapoa/src/cudapoa.cpp @@ -34,6 +34,39 @@ StatusType Init() return StatusType::success; } +std::string decode_error(StatusType error_type) +{ + std::string error_message; + switch (error_type) + { + case genomeworks::cudapoa::StatusType::node_count_exceeded_maximum_graph_size: + error_message = "Kernel Error:: Node count exceeded maximum nodes per graph in batch"; + break; + case genomeworks::cudapoa::StatusType::edge_count_exceeded_maximum_graph_size: + error_message = "Kernel Error:: Edge count exceeded maximum edges per graph in batch"; + break; + case genomeworks::cudapoa::StatusType::seq_len_exceeded_maximum_nodes_per_window: + error_message = "Kernel Error:: Sequence length exceeded maximum nodes per window in batch"; + break; + case genomeworks::cudapoa::StatusType::loop_count_exceeded_upper_bound: + error_message = "Kernel Error:: Loop count exceeded upper bound in nw algorithm in batch"; + break; + case genomeworks::cudapoa::StatusType::exceeded_adaptive_banded_matrix_size: + error_message = "Kernel Error:: Band width set for adaptive matrix allocation is too small in batch"; + break; + case genomeworks::cudapoa::StatusType::exceeded_maximum_sequence_size: + error_message = "Kernel Error:: Consensus/MSA sequence size exceeded max sequence size in batch"; + break; + case genomeworks::cudapoa::StatusType::exceeded_maximum_predecessor_distance: + error_message = "Kernel Error:: Set value for maximum predecessor distance in traceback NW is too small"; + break; + default: + error_message = "Kernel Error:: Unknown error in batch"; + break; + } + return error_message; +} + } // namespace cudapoa } // namespace genomeworks diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 690d1628e..3fd5887ad 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -440,41 +440,9 @@ protected: void decode_cudapoa_kernel_error(genomeworks::cudapoa::StatusType error_type, std::vector& output_status) { - switch (error_type) - { - case genomeworks::cudapoa::StatusType::node_count_exceeded_maximum_graph_size: - GW_LOG_WARN("Kernel Error:: Node count exceeded maximum nodes per graph in batch {}\n", bid_); - output_status.emplace_back(error_type); - break; - case genomeworks::cudapoa::StatusType::edge_count_exceeded_maximum_graph_size: - GW_LOG_WARN("Kernel Error:: Edge count exceeded maximum edges per graph in batch {}\n", bid_); - output_status.emplace_back(error_type); - break; - case genomeworks::cudapoa::StatusType::seq_len_exceeded_maximum_nodes_per_window: - GW_LOG_WARN("Kernel Error:: Sequence length exceeded maximum nodes per window in batch {}\n", bid_); - output_status.emplace_back(error_type); - break; - case genomeworks::cudapoa::StatusType::loop_count_exceeded_upper_bound: - GW_LOG_WARN("Kernel Error:: Loop count exceeded upper bound in nw algorithm in batch {}\n", bid_); - output_status.emplace_back(error_type); - break; - case genomeworks::cudapoa::StatusType::exceeded_adaptive_banded_matrix_size: - GW_LOG_WARN("Kernel Error:: Band width set for adaptive matrix allocation is too small in batch {}\n", bid_); - output_status.emplace_back(error_type); - break; - case genomeworks::cudapoa::StatusType::exceeded_maximum_sequence_size: - GW_LOG_WARN("Kernel Error:: Consensus/MSA sequence size exceeded max sequence size in batch {}\n", bid_); - output_status.emplace_back(error_type); - break; - case genomeworks::cudapoa::StatusType::exceeded_maximum_predecessor_distance: - GW_LOG_WARN("Kernel Error:: Set value for maximum predecessor distance in traceback NW is too small {}\n", bid_); - output_status.emplace_back(error_type); - break; - default: - GW_LOG_WARN("Kernel Error:: Unknown error in batch {}\n", bid_); - output_status.emplace_back(error_type); - break; - } + std::string error_message = decode_error(error_type) + " {}\n"; + GW_LOG_WARN(error_message.c_str(), bid_); + output_status.emplace_back(error_type); } // Add new partial order alignment to batch. From 5e4f217c3443e67a8ae45eadc8a4a3d450bf521a Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 1 Dec 2020 11:12:50 -0500 Subject: [PATCH 267/281] [cudapoa] improved error messages in decode_error() --- .../genomeworks/cudapoa/cudapoa.hpp | 5 +- cudapoa/src/cudapoa.cpp | 54 ++++++++++++------- cudapoa/src/cudapoa_batch.cuh | 5 +- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp index 2fc483117..1caaae56e 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp @@ -49,8 +49,9 @@ enum StatusType /// Generate corresponding error message for a given error type /// \param [in] error_type input error code -/// \return corresponding error message -std::string decode_error(StatusType error_type); +/// \param [out] error_message corresponding error message +/// \param [out] error_hint possible hint to resolve the error +void decode_error(StatusType error_type, std::string& error_message, std::string& error_hint); /// Banding mode used in Needleman-Wunsch algorithm /// - full_band performs computations on full scores matrix, highest accuracy diff --git a/cudapoa/src/cudapoa.cpp b/cudapoa/src/cudapoa.cpp index 7c889d0cf..216227cd6 100644 --- a/cudapoa/src/cudapoa.cpp +++ b/cudapoa/src/cudapoa.cpp @@ -34,37 +34,55 @@ StatusType Init() return StatusType::success; } -std::string decode_error(StatusType error_type) +void decode_error(StatusType error_type, std::string& error_message, std::string& error_hint) { - std::string error_message; switch (error_type) { - case genomeworks::cudapoa::StatusType::node_count_exceeded_maximum_graph_size: - error_message = "Kernel Error:: Node count exceeded maximum nodes per graph in batch"; + case StatusType::exceeded_maximum_poas: + error_message = "Kernel Error: Number of groups per batch exceeded maximum POAs"; + error_hint = "Suggestion : Evaluate maximum number of groups per batch using BatchBlock::estimate_max_poas()"; break; - case genomeworks::cudapoa::StatusType::edge_count_exceeded_maximum_graph_size: - error_message = "Kernel Error:: Edge count exceeded maximum edges per graph in batch"; + case StatusType::exceeded_maximum_sequence_size: + error_message = "Kernel Error: Input read length or output consensus/MSA sequence length exceeded max sequence size"; + error_hint = "Suggestion : Check BatchConfig::max_sequence_size and BatchConfig::max_consensus_size, increase if necessary"; break; - case genomeworks::cudapoa::StatusType::seq_len_exceeded_maximum_nodes_per_window: - error_message = "Kernel Error:: Sequence length exceeded maximum nodes per window in batch"; + case StatusType::exceeded_maximum_sequences_per_poa: + error_message = "Kernel Error: Exceeded maximum number of reads per POA"; + error_hint = "Suggestion : Check BatchConfig::max_sequences_per_poa and increase if necessary"; break; - case genomeworks::cudapoa::StatusType::loop_count_exceeded_upper_bound: - error_message = "Kernel Error:: Loop count exceeded upper bound in nw algorithm in batch"; + case StatusType::node_count_exceeded_maximum_graph_size: + error_message = "Kernel Error: Node count exceeded maximum nodes per POA graph"; + error_hint = "Suggestion : Check BatchConfig::max_nodes_per_graph and increase if necessary"; break; - case genomeworks::cudapoa::StatusType::exceeded_adaptive_banded_matrix_size: - error_message = "Kernel Error:: Band width set for adaptive matrix allocation is too small in batch"; + case StatusType::edge_count_exceeded_maximum_graph_size: + error_message = "Kernel Error: Edge count exceeded maximum edges per graph"; + error_hint = "Suggestion : Check default value of CUDAPOA_MAX_NODE_EDGES, note that increasing this macro would increase memory usage per POA"; break; - case genomeworks::cudapoa::StatusType::exceeded_maximum_sequence_size: - error_message = "Kernel Error:: Consensus/MSA sequence size exceeded max sequence size in batch"; + case StatusType::exceeded_adaptive_banded_matrix_size: + error_message = "Kernel Error: Allocated buffer for score/traceback matrix in adaptive banding is not large enough"; + error_hint = "Suggestion : Check BatchConfig::matrix_sequence_dimension and increase if necessary"; break; - case genomeworks::cudapoa::StatusType::exceeded_maximum_predecessor_distance: - error_message = "Kernel Error:: Set value for maximum predecessor distance in traceback NW is too small"; + case StatusType::loop_count_exceeded_upper_bound: + error_message = "Kernel Error: Traceback in Needleman-Wunsch algorithm failed"; + error_hint = "Suggestion : You may retry with a different banding mode"; + break; + case StatusType::output_type_unavailable: + error_message = "Kernel Error: Output type not available"; + error_hint = "Suggestion : Check MSA/Consensus selection for output type"; + break; + case StatusType::exceeded_maximum_predecessor_distance: + error_message = "Kernel Error: Set value for maximum predecessor distance in Needleman-Wunsch algorithm with traceback buffer is not large enough"; + error_hint = "Suggestion : Check BatchConfig::max_banded_pred_distance and increase if necessary"; + break; + case StatusType::generic_error: + error_message = "Kernel Error: Unknown error"; + error_hint = ""; break; default: - error_message = "Kernel Error:: Unknown error in batch"; + error_message = "Kernel Error: Unknown error"; + error_hint = ""; break; } - return error_message; } } // namespace cudapoa diff --git a/cudapoa/src/cudapoa_batch.cuh b/cudapoa/src/cudapoa_batch.cuh index 3fd5887ad..70d7ee956 100644 --- a/cudapoa/src/cudapoa_batch.cuh +++ b/cudapoa/src/cudapoa_batch.cuh @@ -440,7 +440,10 @@ protected: void decode_cudapoa_kernel_error(genomeworks::cudapoa::StatusType error_type, std::vector& output_status) { - std::string error_message = decode_error(error_type) + " {}\n"; + std::string error_message; + std::string error_hint; + decode_error(error_type, error_message, error_hint); + error_message = error_message + " in batch {}\n" + error_hint; GW_LOG_WARN(error_message.c_str(), bid_); output_status.emplace_back(error_type); } From 6fb1760633b20f9170b1cb6ea200ffa5c04e5db5 Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 1 Dec 2020 11:27:55 -0500 Subject: [PATCH 268/281] [cudapoa] removed obsolete seq_len_exceeded_maximum_nodes_per_window error code; update python code --- .../claraparabricks/genomeworks/cudapoa/cudapoa.hpp | 5 ++--- cudapoa/src/cudapoa.cpp | 8 ++++---- pygenomeworks/genomeworks/cudapoa/cudapoa.pxd | 3 ++- pygenomeworks/genomeworks/cudapoa/cudapoa.pyx | 6 ++++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp index 1caaae56e..24f632c5c 100644 --- a/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp +++ b/cudapoa/include/claraparabricks/genomeworks/cudapoa/cudapoa.hpp @@ -40,11 +40,10 @@ enum StatusType node_count_exceeded_maximum_graph_size, edge_count_exceeded_maximum_graph_size, exceeded_adaptive_banded_matrix_size, - seq_len_exceeded_maximum_nodes_per_window, + exceeded_maximum_predecessor_distance, loop_count_exceeded_upper_bound, output_type_unavailable, - generic_error, - exceeded_maximum_predecessor_distance + generic_error }; /// Generate corresponding error message for a given error type diff --git a/cudapoa/src/cudapoa.cpp b/cudapoa/src/cudapoa.cpp index 216227cd6..d239c1235 100644 --- a/cudapoa/src/cudapoa.cpp +++ b/cudapoa/src/cudapoa.cpp @@ -62,6 +62,10 @@ void decode_error(StatusType error_type, std::string& error_message, std::string error_message = "Kernel Error: Allocated buffer for score/traceback matrix in adaptive banding is not large enough"; error_hint = "Suggestion : Check BatchConfig::matrix_sequence_dimension and increase if necessary"; break; + case StatusType::exceeded_maximum_predecessor_distance: + error_message = "Kernel Error: Set value for maximum predecessor distance in Needleman-Wunsch algorithm with traceback buffer is not large enough"; + error_hint = "Suggestion : Check BatchConfig::max_banded_pred_distance and increase if necessary"; + break; case StatusType::loop_count_exceeded_upper_bound: error_message = "Kernel Error: Traceback in Needleman-Wunsch algorithm failed"; error_hint = "Suggestion : You may retry with a different banding mode"; @@ -70,10 +74,6 @@ void decode_error(StatusType error_type, std::string& error_message, std::string error_message = "Kernel Error: Output type not available"; error_hint = "Suggestion : Check MSA/Consensus selection for output type"; break; - case StatusType::exceeded_maximum_predecessor_distance: - error_message = "Kernel Error: Set value for maximum predecessor distance in Needleman-Wunsch algorithm with traceback buffer is not large enough"; - error_hint = "Suggestion : Check BatchConfig::max_banded_pred_distance and increase if necessary"; - break; case StatusType::generic_error: error_message = "Kernel Error: Unknown error"; error_hint = ""; diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd b/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd index eff1b0310..5f7f5bcbc 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pxd @@ -47,7 +47,8 @@ cdef extern from "claraparabricks/genomeworks/cudapoa/cudapoa.hpp" namespace "cl exceeded_maximum_sequences_per_poa node_count_exceeded_maximum_graph_size edge_count_exceeded_maximum_graph_size - seq_len_exceeded_maximum_nodes_per_window + exceeded_adaptive_banded_matrix_size + exceeded_maximum_predecessor_distance loop_count_exceeded_upper_bound output_type_unavailable generic_error diff --git a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx index c78a638a7..8d52f5061 100644 --- a/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx +++ b/pygenomeworks/genomeworks/cudapoa/cudapoa.pyx @@ -52,8 +52,10 @@ def status_to_str(status): return "node_count_exceeded_maximum_graph_size" elif status == cudapoa.edge_count_exceeded_maximum_graph_size: return "edge_count_exceeded_maximum_graph_size" - elif status == cudapoa.seq_len_exceeded_maximum_nodes_per_window: - return "seq_len_exceeded_maximum_nodes_per_window" + elif status == cudapoa.exceeded_adaptive_banded_matrix_size: + return "exceeded_adaptive_banded_matrix_size" + elif status == cudapoa.exceeded_maximum_predecessor_distance: + return "exceeded_maximum_predecessor_distance" elif status == cudapoa.loop_count_exceeded_upper_bound: return "loop_count_exceeded_upper_bound" elif status == cudapoa.output_type_unavailable: From b80d7dfed43ea6cee25eafe6de7fae2c20a0aee3 Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 1 Dec 2020 13:13:52 -0500 Subject: [PATCH 269/281] [cudapoa] updated error message output in cudapoa-bin and sample-cudapoa --- cudapoa/samples/sample_cudapoa.cpp | 29 ++++++++++++++++++++++++----- cudapoa/src/main.cpp | 29 ++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/cudapoa/samples/sample_cudapoa.cpp b/cudapoa/samples/sample_cudapoa.cpp index 116995f9e..773e8ee17 100644 --- a/cudapoa/samples/sample_cudapoa.cpp +++ b/cudapoa/samples/sample_cudapoa.cpp @@ -64,6 +64,7 @@ std::unique_ptr initialize_batch(bool msa, const BatchConfig& batch_size) void process_batch(Batch* batch, bool msa_flag, bool print, std::vector& list_of_group_ids, int id_offset) { batch->generate_poa(); + std::string error_message, error_hint; StatusType status = StatusType::success; if (msa_flag) @@ -75,14 +76,20 @@ void process_batch(Batch* batch, bool msa_flag, bool print, std::vector status = batch->get_msa(msa, output_status); if (status != StatusType::success) { - std::cerr << "Could not generate MSA for batch : " << status << std::endl; + decode_error(status, error_message, error_hint); + std::cerr << "Could not generate MSA for batch : " << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; } for (int32_t g = 0; g < get_size(msa); g++) { if (output_status[g] != StatusType::success) { - std::cerr << "Error generating MSA for POA group " << list_of_group_ids[g + id_offset] << ". Error type " << output_status[g] << std::endl; + decode_error(output_status[g], error_message, error_hint); + std::cerr << "Error generating MSA for POA group " << list_of_group_ids[g + id_offset] << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; } else { @@ -106,14 +113,20 @@ void process_batch(Batch* batch, bool msa_flag, bool print, std::vector status = batch->get_consensus(consensus, coverage, output_status); if (status != StatusType::success) { - std::cerr << "Could not generate consensus for batch : " << status << std::endl; + decode_error(status, error_message, error_hint); + std::cerr << "Could not generate consensus for batch : " << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; } for (int32_t g = 0; g < get_size(consensus); g++) { if (output_status[g] != StatusType::success) { - std::cerr << "Error generating consensus for POA group " << list_of_group_ids[g + id_offset] << ". Error type " << output_status[g] << std::endl; + decode_error(output_status[g], error_message, error_hint); + std::cerr << "Error generating consensus for POA group " << list_of_group_ids[g + id_offset] << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; } else { @@ -213,6 +226,9 @@ int main(int argc, char** argv) } } + // for error code message + std::string error_message, error_hint; + // analyze the POA groups and create a minimal set of batches to process them all std::vector list_of_batch_sizes; std::vector> list_of_groups_per_batch; @@ -304,7 +320,10 @@ int main(int argc, char** argv) if (status != StatusType::exceeded_maximum_poas && status != StatusType::success) { - std::cout << "Could not add POA group " << batch_group_ids[i] << " to batch " << b << ". Error code " << status << std::endl; + decode_error(status, error_message, error_hint); + std::cerr << "Could not add POA group " << batch_group_ids[i] << " to batch " << b << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; i++; } } diff --git a/cudapoa/src/main.cpp b/cudapoa/src/main.cpp index e68d0d2f6..2765492e5 100644 --- a/cudapoa/src/main.cpp +++ b/cudapoa/src/main.cpp @@ -67,6 +67,7 @@ std::unique_ptr initialize_batch(int32_t mismatch_score, void process_batch(Batch* batch, bool msa_flag, bool print, std::vector& list_of_group_ids, int id_offset) { batch->generate_poa(); + std::string error_message, error_hint; StatusType status = StatusType::success; if (msa_flag) @@ -78,14 +79,20 @@ void process_batch(Batch* batch, bool msa_flag, bool print, std::vector status = batch->get_msa(msa, output_status); if (status != StatusType::success) { - std::cerr << "Could not generate MSA for batch : " << status << std::endl; + decode_error(status, error_message, error_hint); + std::cerr << "Could not generate MSA for batch : " << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; } for (int32_t g = 0; g < get_size(msa); g++) { if (output_status[g] != StatusType::success) { - std::cerr << "Error generating MSA for POA group " << list_of_group_ids[g + id_offset] << ". Error type " << output_status[g] << std::endl; + decode_error(output_status[g], error_message, error_hint); + std::cerr << "Error generating MSA for POA group " << list_of_group_ids[g + id_offset] << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; } else { @@ -109,14 +116,20 @@ void process_batch(Batch* batch, bool msa_flag, bool print, std::vector status = batch->get_consensus(consensus, coverage, output_status); if (status != StatusType::success) { - std::cerr << "Could not generate consensus for batch : " << status << std::endl; + decode_error(status, error_message, error_hint); + std::cerr << "Could not generate consensus for batch : " << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; } for (int32_t g = 0; g < get_size(consensus); g++) { if (output_status[g] != StatusType::success) { - std::cerr << "Error generating consensus for POA group " << list_of_group_ids[g + id_offset] << ". Error type " << output_status[g] << std::endl; + decode_error(output_status[g], error_message, error_hint); + std::cerr << "Error generating consensus for POA group " << list_of_group_ids[g + id_offset] << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; } else { @@ -158,6 +171,9 @@ int main(int argc, char* argv[]) } } + // for error code message + std::string error_message, error_hint; + // Create a vector of POA groups based on windows std::vector poa_groups(windows.size()); for (int32_t i = 0; i < get_size(windows); ++i) @@ -289,7 +305,10 @@ int main(int argc, char* argv[]) if (status != StatusType::exceeded_maximum_poas && status != StatusType::success) { - std::cerr << "Could not add POA group " << batch_group_ids[i] << " to batch " << b << ". Error code " << status << std::endl; + decode_error(status, error_message, error_hint); + std::cerr << "Could not add POA group " << batch_group_ids[i] << " to batch " << b << std::endl; + std::cerr << error_message << std::endl + << error_hint << std::endl; i++; } } From 8d8ecde99fbc9194333aff6dc5bb697a52e33767 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 2 Dec 2020 10:12:48 -0500 Subject: [PATCH 270/281] [cudapoa] added a couple of missing fixes re static_assert --- cudapoa/src/cudapoa_add_alignment.cuh | 2 -- cudapoa/src/cudapoa_topsort.cuh | 2 -- 2 files changed, 4 deletions(-) diff --git a/cudapoa/src/cudapoa_add_alignment.cuh b/cudapoa/src/cudapoa_add_alignment.cuh index 326f78e57..03c0e14a5 100644 --- a/cudapoa/src/cudapoa_add_alignment.cuh +++ b/cudapoa/src/cudapoa_add_alignment.cuh @@ -352,8 +352,6 @@ void addAlignment(uint8_t* nodes, uint32_t max_sequences_per_poa, uint32_t max_limit_nodes_per_window) { - static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); - addAlignmentKernel<<<1, 1>>>(nodes, node_count, node_alignments, node_alignment_count, diff --git a/cudapoa/src/cudapoa_topsort.cuh b/cudapoa/src/cudapoa_topsort.cuh index 7bd789267..2982525ea 100644 --- a/cudapoa/src/cudapoa_topsort.cuh +++ b/cudapoa/src/cudapoa_topsort.cuh @@ -225,8 +225,6 @@ void runTopSort(SizeT* sorted_poa, uint16_t* outgoing_edge_count, uint16_t* local_incoming_edge_count) { - static_assert(std::is_same::value, "This function only accepts int16_t as SizeT."); - // calls the topsort kernel on 1 thread runTopSortKernel<<<1, 1>>>(sorted_poa, sorted_poa_node_map, From cb4142215648cdfc813e0aa30a2992d1b80335ec Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 2 Dec 2020 17:09:47 -0500 Subject: [PATCH 271/281] [cudapoa] minor fix --- cudapoa/src/cudapoa.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cudapoa/src/cudapoa.cpp b/cudapoa/src/cudapoa.cpp index d239c1235..d970b8789 100644 --- a/cudapoa/src/cudapoa.cpp +++ b/cudapoa/src/cudapoa.cpp @@ -75,11 +75,11 @@ void decode_error(StatusType error_type, std::string& error_message, std::string error_hint = "Suggestion : Check MSA/Consensus selection for output type"; break; case StatusType::generic_error: - error_message = "Kernel Error: Unknown error"; + error_message = "Unknown error"; error_hint = ""; break; default: - error_message = "Kernel Error: Unknown error"; + error_message = "Unknown error"; error_hint = ""; break; } From 9f5ed3d305e31d7b1123a355329b8bd3d62a14bd Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Thu, 3 Dec 2020 09:35:35 -0500 Subject: [PATCH 272/281] [cudapoa] throw error for unknown error type --- cudapoa/src/cudapoa.cpp | 42 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/cudapoa/src/cudapoa.cpp b/cudapoa/src/cudapoa.cpp index d970b8789..c16b08d73 100644 --- a/cudapoa/src/cudapoa.cpp +++ b/cudapoa/src/cudapoa.cpp @@ -39,49 +39,47 @@ void decode_error(StatusType error_type, std::string& error_message, std::string switch (error_type) { case StatusType::exceeded_maximum_poas: - error_message = "Kernel Error: Number of groups per batch exceeded maximum POAs"; - error_hint = "Suggestion : Evaluate maximum number of groups per batch using BatchBlock::estimate_max_poas()"; + error_message = "Kernel Error: Number of groups per batch exceeded maximum POAs."; + error_hint = "Suggestion : Evaluate maximum number of groups per batch using BatchBlock::estimate_max_poas()."; break; case StatusType::exceeded_maximum_sequence_size: - error_message = "Kernel Error: Input read length or output consensus/MSA sequence length exceeded max sequence size"; - error_hint = "Suggestion : Check BatchConfig::max_sequence_size and BatchConfig::max_consensus_size, increase if necessary"; + error_message = "Kernel Error: Input read length or output consensus/MSA sequence length exceeded max sequence size."; + error_hint = "Suggestion : Check BatchConfig::max_sequence_size and BatchConfig::max_consensus_size, increase if necessary."; break; case StatusType::exceeded_maximum_sequences_per_poa: - error_message = "Kernel Error: Exceeded maximum number of reads per POA"; - error_hint = "Suggestion : Check BatchConfig::max_sequences_per_poa and increase if necessary"; + error_message = "Kernel Error: Exceeded maximum number of reads per POA."; + error_hint = "Suggestion : Check BatchConfig::max_sequences_per_poa and increase if necessary."; break; case StatusType::node_count_exceeded_maximum_graph_size: - error_message = "Kernel Error: Node count exceeded maximum nodes per POA graph"; - error_hint = "Suggestion : Check BatchConfig::max_nodes_per_graph and increase if necessary"; + error_message = "Kernel Error: Node count exceeded maximum nodes per POA graph."; + error_hint = "Suggestion : Check BatchConfig::max_nodes_per_graph and increase if necessary."; break; case StatusType::edge_count_exceeded_maximum_graph_size: - error_message = "Kernel Error: Edge count exceeded maximum edges per graph"; - error_hint = "Suggestion : Check default value of CUDAPOA_MAX_NODE_EDGES, note that increasing this macro would increase memory usage per POA"; + error_message = "Kernel Error: Edge count exceeded maximum edges per graph."; + error_hint = "Suggestion : Check default value of CUDAPOA_MAX_NODE_EDGES, note that increasing this macro would increase memory usage per POA."; break; case StatusType::exceeded_adaptive_banded_matrix_size: - error_message = "Kernel Error: Allocated buffer for score/traceback matrix in adaptive banding is not large enough"; - error_hint = "Suggestion : Check BatchConfig::matrix_sequence_dimension and increase if necessary"; + error_message = "Kernel Error: Allocated buffer for score/traceback matrix in adaptive banding is not large enough."; + error_hint = "Suggestion : Check BatchConfig::matrix_sequence_dimension and increase if necessary."; break; case StatusType::exceeded_maximum_predecessor_distance: - error_message = "Kernel Error: Set value for maximum predecessor distance in Needleman-Wunsch algorithm with traceback buffer is not large enough"; - error_hint = "Suggestion : Check BatchConfig::max_banded_pred_distance and increase if necessary"; + error_message = "Kernel Error: Set value for maximum predecessor distance in Needleman-Wunsch algorithm with traceback buffer is not large enough."; + error_hint = "Suggestion : Check BatchConfig::max_banded_pred_distance and increase if necessary."; break; case StatusType::loop_count_exceeded_upper_bound: - error_message = "Kernel Error: Traceback in Needleman-Wunsch algorithm failed"; - error_hint = "Suggestion : You may retry with a different banding mode"; + error_message = "Kernel Error: Traceback in Needleman-Wunsch algorithm failed."; + error_hint = "Suggestion : You may retry with a different banding mode."; break; case StatusType::output_type_unavailable: - error_message = "Kernel Error: Output type not available"; - error_hint = "Suggestion : Check MSA/Consensus selection for output type"; + error_message = "Kernel Error: Output type not available."; + error_hint = "Suggestion : Check MSA/Consensus selection for output type."; break; case StatusType::generic_error: - error_message = "Unknown error"; + error_message = "Unknown error."; error_hint = ""; break; default: - error_message = "Unknown error"; - error_hint = ""; - break; + throw std::runtime_error("Unknown error type detected."); } } From a5547d4bd7dac792db9ae9e8f62096b908058ddc Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 3 Dec 2020 09:43:35 -0500 Subject: [PATCH 273/281] [cmake] changed compute version in cudaaligner and cudapoa to -arch=compute_60, updated readme --- README.md | 11 ++++++----- cudaaligner/CMakeLists.txt | 4 ++-- cudapoa/CMakeLists.txt | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fb8b027ba..94629a78f 100644 --- a/README.md +++ b/README.md @@ -109,11 +109,12 @@ Minimum requirements - 1. Ubuntu 16.04 or Ubuntu 18.04 2. CUDA 9.0+ (official instructions for installing CUDA are available [here](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html)) -3. gcc/g++ 5.4.0+ / 7.x.x -4. Python 3.6.7+ -5. CMake (>= 3.10.2) -6. autoconf (required to output SAM/BAM files) -7. automake (required to output SAM/BAM files) +3. GPU generation Pascal and later (compute capability >= 6.0) +4. gcc/g++ 5.4.0+ / 7.x.x +5. Python 3.6.7+ +6. CMake (>= 3.10.2) +7. autoconf (required to output SAM/BAM files) +8. automake (required to output SAM/BAM files) ## GenomeWorks Setup diff --git a/cudaaligner/CMakeLists.txt b/cudaaligner/CMakeLists.txt index d2a2f5712..9799e5b71 100644 --- a/cudaaligner/CMakeLists.txt +++ b/cudaaligner/CMakeLists.txt @@ -20,9 +20,9 @@ set(MODULE_NAME cudaaligner) # Project specific NVCC flags if((CUDA_VERSION_MAJOR GREATER 10) OR (CUDA_VERSION_MAJOR EQUAL 10 AND CUDA_VERSION_MINOR GREATER 0)) -set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14 --expt-relaxed-constexpr") +set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14 -arch=compute_60 --expt-relaxed-constexpr") else() -set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14") +set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14 -arch=compute_60") endif() message(STATUS "nvcc flags for ${MODULE_NAME}: ${CUDA_NVCC_FLAGS}") diff --git a/cudapoa/CMakeLists.txt b/cudapoa/CMakeLists.txt index 53505f523..73cc37ad7 100644 --- a/cudapoa/CMakeLists.txt +++ b/cudapoa/CMakeLists.txt @@ -38,7 +38,7 @@ endif() # Project specific NVCC flags, --expt-relaxed-constexpr is being added to allow using numeric_limits inside device kernels. # Using in CUDA 10.1+ to avoid compilation issues with CUDA <= 10.0. -set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14") +set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++14 -arch=compute_60") if(gw_cuda_after_10_0) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} --expt-relaxed-constexpr") endif() From 60756b29c43161850c2563ddf04732e768babf7f Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Thu, 3 Dec 2020 09:43:50 -0500 Subject: [PATCH 274/281] [cudamapper] fix substr api call in kmerization function --- cudamapper/src/cudamapper_utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudamapper/src/cudamapper_utils.cpp b/cudamapper/src/cudamapper_utils.cpp index a7b8466fb..269ef5080 100644 --- a/cudamapper/src/cudamapper_utils.cpp +++ b/cudamapper/src/cudamapper_utils.cpp @@ -46,7 +46,7 @@ std::vector split_into_kmers(const gw_string_view_t& s, const for (std::size_t i = 0; i < kmer_count; i += stride) { - kmers.push_back(s.substr(i, i + kmer_size)); + kmers.push_back(s.substr(i, kmer_size)); } return kmers; } From f1ac5937ed5e021ed5c5bece523bd86270d04929 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Thu, 3 Dec 2020 14:05:22 -0500 Subject: [PATCH 275/281] [docs] refactor docs to cleanup repo look --- README.md | 77 ++----------------------------------- cudaaligner/CMakeLists.txt | 1 + cudaaligner/README.md | 13 +++++++ cudaextender/CMakeLists.txt | 1 + cudaextender/README.md | 8 ++-- cudamapper/CMakeLists.txt | 1 + cudamapper/README.md | 30 +++++++++++++++ cudapoa/CMakeLists.txt | 1 + cudapoa/README.md | 21 ++++++++++ 9 files changed, 76 insertions(+), 77 deletions(-) create mode 100644 cudaaligner/README.md create mode 100644 cudamapper/README.md create mode 100644 cudapoa/README.md diff --git a/README.md b/README.md index fb8b027ba..f6a59b8d3 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@ GenomeWorks is a GPU-accelerated library for biological sequence analysis. This For more detailed API documentation please refer to the [documentation](#enable-doc-generation). * Modules - * [cudamapper](#cudamapper) - CUDA-accelerated sequence to sequence mapping - * [cudapoa](#cudapoa) - CUDA-accelerated partial order alignment - * [cudaaligner](#cudaaligner) - CUDA-accelerated pairwise sequence alignment - * [cudaextender](#cudaextender) - CUDA-accelerated seed extension + * [cudamapper](cudamapper/README.md) - CUDA-accelerated sequence to sequence mapping + * [cudapoa](cudapoa/README.md) - CUDA-accelerated partial order alignment + * [cudaaligner](cudaaligner/README.md) - CUDA-accelerated pairwise sequence alignment + * [cudaextender](cudaextender/README.md) - CUDA-accelerated seed extension * Setup GenomeWorks * [Clone GenomeWorks](#clone-genomeworks) * [System Requirements](#system-requirements) @@ -17,75 +17,6 @@ For more detailed API documentation please refer to the [documentation](#enable- * [Python API](#genomeworks-python-api) * [Development Support](#development-support) -### cudamapper - -The `cudamapper` package provides minimizer-based GPU-accelerated approximate mapping. - -#### Tool - *cudamapper* - -`cudamapper` is an end-to-end command line to for sequence to sequence mapping. `cudamapper` outputs -mappings in the PAF format and is currently optimised for all-vs-all long read (ONT, Pacific Biosciences) sequences. - -To run all-vs all overlaps use the following command: - -`cudamapper in.fasta in.fasta` - -A query fasta can be mapped to a reference as follows: - -`cudamapper query.fasta target.fasta` - -To access more information about running cudamapper, run `cudamapper --help`. - -#### Library - *libcudamapper.so* - -* `Indexer` module to generate an index of minimizers from a list of sequences. -* `Matcher` module to find locations of matching pairs of minimizers between sequences using minimizer indices. -* `Overlapper` module to generate overlaps from sequence of minimizer matches generated by matcher. - -#### Sample - *sample_cudamapper* - -A prototypical binary highlighting the usage of `libcudamapper.so` APIs (indexer, matcher and overlapper) and -techniques to tie them into an application. - -### cudapoa - -The `cudapoa` package provides a GPU-accelerated implementation of the [Partial Order Alignment](https://simpsonlab.github.io/2015/05/01/understanding-poa/) -algorithm. It is heavily influenced by [SPOA](https://github.com/rvaser/spoa) and in many cases can be considered a GPU-accelerated replacement. Features include: - -#### Tool - *cudapoa* - -A command line tool for generating consensus and MSA from a list of `fasta`/`fastq` files. The tool -is built on top of `libcudapoa.so` and showcases optimization strategies for writing high performance -applications with `libcudapoa.so`. - -#### Library - *libcudapoa.so* - -* Generation of consensus sequences -* Generation of multi-sequence alignments (MSAs) -* Custom adaptive band implementation of POA -* Support for long and short read sequences - -#### Sample - *sample_cudapoa* - -A prototypical binary to showcase the use of `libcudapoa.so` APIs. - -### cudaaligner - -The `cudaaligner` package provides GPU-accelerated global alignment. Features include: - -#### Library - *libcudaaligner.so* - -* Short and long read support -* Banded implementation with configurable band width for flexible performance and accuracy trade-off - -#### Sample - *sample_cudaaligner* - -A prototypical binary to showcase the use of `libcudaaligner.so` APIs. - -### cudaextender -The `cudaextender` package provides GPU-accelerated seed-extension. Details can be found in -the package's readme. - ## Clone GenomeWorks ### Latest released version diff --git a/cudaaligner/CMakeLists.txt b/cudaaligner/CMakeLists.txt index d2a2f5712..3e75d8e7f 100644 --- a/cudaaligner/CMakeLists.txt +++ b/cudaaligner/CMakeLists.txt @@ -65,6 +65,7 @@ target_include_directories(${MODULE_NAME} target_compile_options(${MODULE_NAME} PRIVATE -Werror) add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) +add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/README.md) # Add tests folder add_subdirectory(tests) diff --git a/cudaaligner/README.md b/cudaaligner/README.md new file mode 100644 index 000000000..8ccdec9ad --- /dev/null +++ b/cudaaligner/README.md @@ -0,0 +1,13 @@ +# cudaaligner + +The `cudaaligner` package provides GPU-accelerated global alignment. Features include: + +## Library - libcudaaligner.so + +* Short and long read support +* Banded implementation with configurable band width for flexible performance and accuracy trade-off + +## Sample - sample_cudaaligner + +A prototypical binary to showcase the use of `libcudaaligner.so` APIs. + diff --git a/cudaextender/CMakeLists.txt b/cudaextender/CMakeLists.txt index adf51a0ff..8c00a1147 100644 --- a/cudaextender/CMakeLists.txt +++ b/cudaextender/CMakeLists.txt @@ -59,6 +59,7 @@ target_include_directories(${MODULE_NAME} ) add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) +add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/README.md) install(TARGETS ${MODULE_NAME} COMPONENT gwlogging diff --git a/cudaextender/README.md b/cudaextender/README.md index 078eb5376..5c049e025 100644 --- a/cudaextender/README.md +++ b/cudaextender/README.md @@ -7,21 +7,21 @@ Currently this module implements the ungapped X-drop algorithm, adapted from [SegAlign's](https://github.com/gsneha26/SegAlign) Ungapped Extender authored by Sneha Goenka (gsneha@stanford.edu) and Yatish Turakhia (yturakhi@uscs.edu). -### Encoded Input +## Encoded Input `cudaextender` expects the input strands to be encoded as integer sequences. This encoding scheme is documented here: [utils.hpp](include/claraparabricks/genomeworks/cudaextender/utils.hpp) file. The provided `encode_sequence()` helper function will encode the input strands on CPU with the expected scheme. -### API +## API `cudaextender` provides host and device pointer APIs to enable ease of integration with other producer/consumer modules. The user is expected to handle all memory transactions and device sychronizations for the device pointer API. The host pointer API abstracts those operations away. Both APIs are documented here: [extender.hpp](include/claraparabricks/genomeworks/cudaextender/extender.hpp) -### Library - *libcudaextender.so* +## Library - libcudaextender.so Features: * Ungapped X-Drop extension -### Sample - *[sample_cudaextender.cpp](samples/sample_cudaextender.cpp)* +## Sample - sample_cudaextender Protoype to show the usage of host and device pointer APIs on FASTA sequences. diff --git a/cudamapper/CMakeLists.txt b/cudamapper/CMakeLists.txt index 7846d38ce..22868ecd3 100644 --- a/cudamapper/CMakeLists.txt +++ b/cudamapper/CMakeLists.txt @@ -83,6 +83,7 @@ if (gw_optimize_for_native_cpu) endif() add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) +add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/README.md) cuda_add_executable(${MODULE_NAME}-bin src/main.cu diff --git a/cudamapper/README.md b/cudamapper/README.md new file mode 100644 index 000000000..19b6fd68a --- /dev/null +++ b/cudamapper/README.md @@ -0,0 +1,30 @@ +# cudamapper + +The `cudamapper` package provides minimizer-based GPU-accelerated approximate mapping. + +## Tool - cudamapper + +`cudamapper` is an end-to-end command line to for sequence to sequence mapping. `cudamapper` outputs +mappings in the PAF format and is currently optimised for all-vs-all long read (ONT, Pacific Biosciences) sequences. + +To run all-vs all overlaps use the following command: + +`cudamapper in.fasta in.fasta` + +A query fasta can be mapped to a reference as follows: + +`cudamapper query.fasta target.fasta` + +To access more information about running cudamapper, run `cudamapper --help`. + +## Library - libcudamapper.so + +* `Indexer` module to generate an index of minimizers from a list of sequences. +* `Matcher` module to find locations of matching pairs of minimizers between sequences using minimizer indices. +* `Overlapper` module to generate overlaps from sequence of minimizer matches generated by matcher. + +## Sample - sample_cudamapper + +A prototypical binary highlighting the usage of `libcudamapper.so` APIs (indexer, matcher and overlapper) and +techniques to tie them into an application. + diff --git a/cudapoa/CMakeLists.txt b/cudapoa/CMakeLists.txt index 53505f523..4ef5331a5 100644 --- a/cudapoa/CMakeLists.txt +++ b/cudapoa/CMakeLists.txt @@ -74,6 +74,7 @@ target_include_directories(${MODULE_NAME} ) add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) +add_doxygen_source_dir(${CMAKE_CURRENT_SOURCE_DIR}/README.md) add_executable(${MODULE_NAME}-bin src/main.cpp diff --git a/cudapoa/README.md b/cudapoa/README.md new file mode 100644 index 000000000..d7b2e9ca6 --- /dev/null +++ b/cudapoa/README.md @@ -0,0 +1,21 @@ +# CUDAPOA + +The `cudapoa` package provides a GPU-accelerated implementation of the [Partial Order Alignment](https://simpsonlab.github.io/2015/05/01/understanding-poa/) +algorithm. It is heavily influenced by [SPOA](https://github.com/rvaser/spoa) and in many cases can be considered a GPU-accelerated replacement. Features include: + +## Tool - cudapoa + +A command line tool for generating consensus and MSA from a list of `fasta`/`fastq` files. The tool +is built on top of `libcudapoa.so` and showcases optimization strategies for writing high performance +applications with `libcudapoa.so`. + +## Library - libcudapoa.so + +* Generation of consensus sequences +* Generation of multi-sequence alignments (MSAs) +* Custom adaptive band implementation of POA +* Support for long and short read sequences + +## Sample - sample_cudapoa + +A prototypical binary to showcase the use of `libcudapoa.so` APIs. From 04f784550402151439ad08847491c76a6e4036ec Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Thu, 3 Dec 2020 14:15:13 -0500 Subject: [PATCH 276/281] [docs] added links to include folder and sample --- cudaaligner/README.md | 8 +++++--- cudaextender/README.md | 9 +++++---- cudamapper/README.md | 10 ++++++---- cudapoa/README.md | 10 ++++++---- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/cudaaligner/README.md b/cudaaligner/README.md index 8ccdec9ad..d642d6ad5 100644 --- a/cudaaligner/README.md +++ b/cudaaligner/README.md @@ -2,12 +2,14 @@ The `cudaaligner` package provides GPU-accelerated global alignment. Features include: -## Library - libcudaaligner.so +## Library +Built as `libcudaaligner.[so|a]`. * Short and long read support * Banded implementation with configurable band width for flexible performance and accuracy trade-off -## Sample - sample_cudaaligner +APIs documented in [include](include/claraparabricks/genomeworks/cudaaligner) folder. -A prototypical binary to showcase the use of `libcudaaligner.so` APIs. +## Sample +[sample_cudaaligner](samples/sample_cudaaligner.cpp) - A prototypical binary to showcase the use of `libcudaaligner.so` APIs. diff --git a/cudaextender/README.md b/cudaextender/README.md index 5c049e025..9e718df4d 100644 --- a/cudaextender/README.md +++ b/cudaextender/README.md @@ -19,9 +19,10 @@ producer/consumer modules. The user is expected to handle all memory transaction sychronizations for the device pointer API. The host pointer API abstracts those operations away. Both APIs are documented here: [extender.hpp](include/claraparabricks/genomeworks/cudaextender/extender.hpp) -## Library - libcudaextender.so -Features: +## Library +Built as `libcudaextender.[so|a]` + * Ungapped X-Drop extension -## Sample - sample_cudaextender -Protoype to show the usage of host and device pointer APIs on FASTA sequences. +## Sample +[sample_cudaextender](samples/sample_cudaextender.cpp) - Protoype to show the usage of host and device pointer APIs on FASTA sequences. diff --git a/cudamapper/README.md b/cudamapper/README.md index 19b6fd68a..5b2cdbd8b 100644 --- a/cudamapper/README.md +++ b/cudamapper/README.md @@ -2,7 +2,7 @@ The `cudamapper` package provides minimizer-based GPU-accelerated approximate mapping. -## Tool - cudamapper +## Tool `cudamapper` is an end-to-end command line to for sequence to sequence mapping. `cudamapper` outputs mappings in the PAF format and is currently optimised for all-vs-all long read (ONT, Pacific Biosciences) sequences. @@ -17,14 +17,16 @@ A query fasta can be mapped to a reference as follows: To access more information about running cudamapper, run `cudamapper --help`. -## Library - libcudamapper.so +## Library +Built as `libcudamapper.[so|a]` * `Indexer` module to generate an index of minimizers from a list of sequences. * `Matcher` module to find locations of matching pairs of minimizers between sequences using minimizer indices. * `Overlapper` module to generate overlaps from sequence of minimizer matches generated by matcher. -## Sample - sample_cudamapper +APIs documented in [include](include/claraparabricks/genomeworks/cudamapper) folder. -A prototypical binary highlighting the usage of `libcudamapper.so` APIs (indexer, matcher and overlapper) and +## Sample +[sample_cudamapper](samples/sample_cudamapper.cpp) - A prototypical binary highlighting the usage of `libcudamapper.so` APIs (indexer, matcher and overlapper) and techniques to tie them into an application. diff --git a/cudapoa/README.md b/cudapoa/README.md index d7b2e9ca6..e6aafb6c0 100644 --- a/cudapoa/README.md +++ b/cudapoa/README.md @@ -3,19 +3,21 @@ The `cudapoa` package provides a GPU-accelerated implementation of the [Partial Order Alignment](https://simpsonlab.github.io/2015/05/01/understanding-poa/) algorithm. It is heavily influenced by [SPOA](https://github.com/rvaser/spoa) and in many cases can be considered a GPU-accelerated replacement. Features include: -## Tool - cudapoa +## Tool A command line tool for generating consensus and MSA from a list of `fasta`/`fastq` files. The tool is built on top of `libcudapoa.so` and showcases optimization strategies for writing high performance applications with `libcudapoa.so`. -## Library - libcudapoa.so +## Library +Built as `libcudapoa.[so|a]` * Generation of consensus sequences * Generation of multi-sequence alignments (MSAs) * Custom adaptive band implementation of POA * Support for long and short read sequences -## Sample - sample_cudapoa +APIs documented in [include](include/claraparabricks/genomeworks/cudapoa) folder. -A prototypical binary to showcase the use of `libcudapoa.so` APIs. +## Sample +[sample_cudapoa](samples/sample_cudapoa.cpp) - A prototypical binary to showcase the use of `libcudapoa.so` APIs. From 3c08b431485c3db777aafc876f5688d00d878a1f Mon Sep 17 00:00:00 2001 From: Andreas Hehn Date: Wed, 2 Dec 2020 11:09:33 +0100 Subject: [PATCH 277/281] [cudaaligner] Fixed a bug in approximate alignment backtrace --- cudaaligner/src/myers_gpu.cu | 25 +++++--- .../tests/Test_ApproximateBandedMyers.cpp | 62 +++++++++++++++++++ 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/cudaaligner/src/myers_gpu.cu b/cudaaligner/src/myers_gpu.cu index e03d332fd..c18a3f2b6 100644 --- a/cudaaligner/src/myers_gpu.cu +++ b/cudaaligner/src/myers_gpu.cu @@ -425,19 +425,25 @@ __device__ int32_t myers_backtrace_banded(int8_t* path, device_matrix_view= 0); + assert(diagonal_end >= diagonal_begin); + assert(diagonal_end >= 2); // this should only break if target_size == 0 - which is not valid input. + int32_t i = band_width; int32_t j = target_size; const WordType last_entry_mask = band_width % word_size != 0 ? (WordType(1) << (band_width % word_size)) - 1 : ~WordType(0); - nw_score_t myscore = score((i - 1) / word_size, j); // row 0 is implicit, NW matrix is shifted by i -> i-1 (see get_myers_score) - int32_t pos = 0; + const nw_score_t last_diagonal_score = diagonal_end < 2 ? out_of_band : get_myers_score(1, diagonal_end - 2, pv, mv, score, last_entry_mask) + 2; + nw_score_t myscore = score((i - 1) / word_size, j); // row 0 is implicit, NW matrix is shifted by i -> i-1, i.e. i \in [1,band_width] for get_myers_score. (see get_myers_score) + int32_t pos = 0; while (j >= diagonal_end) { - int8_t r = 0; - nw_score_t const above = i <= 1 ? j : get_myers_score(i - 1, j, pv, mv, score, last_entry_mask); - nw_score_t const diag = i <= 1 ? j - 1 : get_myers_score(i - 1, j - 1, pv, mv, score, last_entry_mask); - nw_score_t const left = get_myers_score(i, j - 1, pv, mv, score, last_entry_mask); + int8_t r = 0; + // Worst case for the implicit top row (i == 0) of the bottom right block of the NW is the last diagonal entry on the same row + (j - diagonal_end) * indel cost. + nw_score_t const above = i <= 1 ? (last_diagonal_score + j - diagonal_end) : get_myers_score(i - 1, j, pv, mv, score, last_entry_mask); + nw_score_t const diag = i <= 1 ? (last_diagonal_score + j - 1 - diagonal_end) : get_myers_score(i - 1, j - 1, pv, mv, score, last_entry_mask); + nw_score_t const left = i < 1 ? (last_diagonal_score + j - 1 - diagonal_end) : get_myers_score(i, j - 1, pv, mv, score, last_entry_mask); if (left + 1 == myscore) { r = static_cast(AlignmentState::insertion); @@ -452,6 +458,7 @@ __device__ int32_t myers_backtrace_banded(int8_t* path, device_matrix_view(AlignmentState::match) : static_cast(AlignmentState::mismatch)); myscore = diag; --i; @@ -463,9 +470,10 @@ __device__ int32_t myers_backtrace_banded(int8_t* path, device_matrix_view= diagonal_begin) { int8_t r = 0; - nw_score_t const above = i <= 1 ? j : get_myers_score(i - 1, j, pv, mv, score, last_entry_mask); + nw_score_t const above = i <= 1 ? out_of_band : get_myers_score(i - 1, j, pv, mv, score, last_entry_mask); nw_score_t const diag = i <= 0 ? j - 1 : get_myers_score(i, j - 1, pv, mv, score, last_entry_mask); nw_score_t const left = i >= band_width ? out_of_band : get_myers_score(i + 1, j - 1, pv, mv, score, last_entry_mask); + // out-of-band cases: diag always preferrable, since worst-case-(above|left) - myscore >= diag - myscore always holds. if (left + 1 == myscore) { r = static_cast(AlignmentState::insertion); @@ -481,6 +489,7 @@ __device__ int32_t myers_backtrace_banded(int8_t* path, device_matrix_view(AlignmentState::match) : static_cast(AlignmentState::mismatch)); myscore = diag; --j; @@ -494,6 +503,7 @@ __device__ int32_t myers_backtrace_banded(int8_t* path, device_matrix_view band_width ? out_of_band : get_myers_score(i, j - 1, pv, mv, score, last_entry_mask); + // out-of-band cases: diag always preferrable, since worst-case-(above|left) - myscore >= diag - myscore always holds. if (left + 1 == myscore) { r = static_cast(AlignmentState::insertion); @@ -508,6 +518,7 @@ __device__ int32_t myers_backtrace_banded(int8_t* path, device_matrix_view(AlignmentState::match) : static_cast(AlignmentState::mismatch)); myscore = diag; --i; diff --git a/cudaaligner/tests/Test_ApproximateBandedMyers.cpp b/cudaaligner/tests/Test_ApproximateBandedMyers.cpp index daa585697..e9c6fd161 100644 --- a/cudaaligner/tests/Test_ApproximateBandedMyers.cpp +++ b/cudaaligner/tests/Test_ApproximateBandedMyers.cpp @@ -56,8 +56,70 @@ class TestApproximateBandedMyers : public ::testing::TestWithParam { }; +void implicit_new_entries_test_impl(const std::string& query, const std::string& target, const std::string& expected_cigar) +{ + using namespace claraparabricks::genomeworks::cudaaligner; + using namespace claraparabricks::genomeworks; + const int32_t max_bw = 7; + DefaultDeviceAllocator allocator = create_default_device_allocator(); + std::unique_ptr aligner = std::make_unique(-1, + max_bw, + allocator, + nullptr, + 0); + ASSERT_EQ(StatusType::success, aligner->add_alignment(query.c_str(), query.length(), target.c_str(), target.length())) + << "Could not add alignment to aligner"; + aligner->align_all(); + aligner->sync_alignments(); + const std::vector>& alignments = aligner->get_alignments(); + ASSERT_EQ(get_size(alignments), 1); + ASSERT_EQ(alignments[0]->get_status(), StatusType::success); + ASSERT_EQ(alignments[0]->is_optimal(), false); + ASSERT_EQ(alignments[0]->convert_to_cigar(), expected_cigar); +} + } // namespace +TEST(TestApproximateBandedMyersStatic, ImplicitNWEntries1) +{ + // The banded Myers implementation uses implicit entries for the 0-th row of the NW matrix (and the band), + // which is assumed to be the worst case value. This does not pose a problem on the top left block and + // the diagonal band, but on the lower right block there are cases where the backtrace runs through this worst case + // - which is technically still part of the band. This tests this specific corner case. + + // Band of the NW matrix: (* = tested implicit 0-row entry) + // * * + // A A C C G G T T A A A A C C C C G G G G G T T A A A C G G T T + // A 1 0 1 2 3 4 5 A 5 C 5 C 5 G 5 G 5 T 5 T 5 A 5 A 5 C 5 C 5 G 5 G 5 T 6 T 7 A 8 A 9 C 10 C 11 C 12 12 13 14 15 16 + // A 2 1 0 1 2 3 4 C 4 C 4 G 4 G 4 T 4 T 4 A 4 A 4 C 4 C 4 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 13 12 13 14 15 + // C 3 2 1 0 1 2 3 C 3 G 3 G 3 T 3 T 3 A 3 A 3 C 3 C 3 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 G 12 13 13 12 13 14 + // C 4 3 2 1 0 1 2 G 2 G 2 T 2 T 2 A 2 A 2 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 13 T 13 13 14 13 12 13 + // G 5 4 3 2 1 0 1 G 1 T 1 T 1 A 1 A 1 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 12 T 12 T 13 14 14 14 13 12 + // G 6 5 4 3 2 1 0 T 0 T 0 A 0 A 0 C 1 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 10 G 11 T 11 T 11 T 12 T 13 14 15 15 14 13 + // T 7 6 5 4 3 2 1 T 1 A 1 A 1 C 1 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 10 G 10 T 10 T 10 T 11 T 12 T 13 14 15 16 15 14 + + implicit_new_entries_test_impl("AACCGGTTAACCGGTTAACCGGTTTT", + "AACCGGTTAAAACCCCGGGGGTTAAACGGTT", + "10M2I2M2I7M3I5M2D"); +} + +TEST(TestApproximateBandedMyersStatic, ImplicitNWEntries2) +{ + // Band of the NW matrix: (* = tested implicit 0-row entry) + // * + // A A C C G G T T A A A A C C C C G G G G G T T A A C C G G T T + // A 1 0 1 2 3 4 5 6 A 6 C 6 C 6 G 6 G 6 T 6 T 6 A 6 A 6 C 6 C 6 G 6 G 6 T 7 T 7 A 8 A 8 C 9 C 9 10 11 12 13 14 + // A 2 1 0 1 2 3 4 5 C 5 C 5 G 5 G 5 T 5 T 5 A 5 A 5 C 5 C 5 G 5 G 5 T 6 T 7 A 8 A 9 C 9 C 10 C 9 9 10 11 12 13 + // C 3 2 1 0 1 2 3 4 C 4 G 4 G 4 T 4 T 4 A 4 A 4 C 4 C 4 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 10 10 9 10 11 12 + // C 4 3 2 1 0 1 2 3 G 3 G 3 T 3 T 3 A 3 A 3 C 3 C 3 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 G 11 11 10 9 10 11 + // G 5 4 3 2 1 0 1 2 G 2 T 2 T 2 A 2 A 2 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 13 T 12 12 11 10 9 10 + // G 6 5 4 3 2 1 0 1 T 1 T 1 A 1 A 1 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 12 T 13 T 13 13 12 11 10 9 + // T 7 6 5 4 3 2 1 0 T 0 A 0 A 0 C 1 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 10 G 11 T 11 T 12 T 13 T 14 14 13 12 11 10 + implicit_new_entries_test_impl("AACCGGTTAACCGGTTAACCGGTTT", + "AACCGGTTAAAACCCCGGGGGTTAACCGGTT", + "10M2I2M2I3M2I3M1I6M1D"); +} + TEST_P(TestApproximateBandedMyers, EditDistanceMonotonicallyDecreasesWithBandWidth) { using namespace claraparabricks::genomeworks::cudaaligner; From 714a2dd6338c272a5339b6b0c6c17f356f3f0c65 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Thu, 3 Dec 2020 16:31:25 -0500 Subject: [PATCH 278/281] [docs] improve organization of docs for cudapoa/cudaaligner --- cudaaligner/README.md | 4 ++-- cudaextender/README.md | 20 +++++++++----------- cudamapper/README.md | 27 +++++++++++++-------------- cudapoa/README.md | 15 ++++++++------- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/cudaaligner/README.md b/cudaaligner/README.md index d642d6ad5..270eb02fb 100644 --- a/cudaaligner/README.md +++ b/cudaaligner/README.md @@ -1,6 +1,6 @@ # cudaaligner -The `cudaaligner` package provides GPU-accelerated global alignment. Features include: +The `cudaaligner` package provides GPU-accelerated global alignment. ## Library Built as `libcudaaligner.[so|a]`. @@ -11,5 +11,5 @@ Built as `libcudaaligner.[so|a]`. APIs documented in [include](include/claraparabricks/genomeworks/cudaaligner) folder. ## Sample -[sample_cudaaligner](samples/sample_cudaaligner.cpp) - A prototypical binary to showcase the use of `libcudaaligner.so` APIs. +[sample_cudaaligner](samples/sample_cudaaligner.cpp) - A prototypical binary to showcase the use of `libcudaaligner` APIs. diff --git a/cudaextender/README.md b/cudaextender/README.md index 9e718df4d..6d03d61d2 100644 --- a/cudaextender/README.md +++ b/cudaextender/README.md @@ -1,28 +1,26 @@ # cudaextender -## Overview This package implements CUDA-accelerated seed-extension algorithms that use seed positions in encoded input strands to extend and compute the alignment between the strands. Currently this module implements the ungapped X-drop algorithm, adapted from [SegAlign's](https://github.com/gsneha26/SegAlign) Ungapped Extender authored by Sneha Goenka (gsneha@stanford.edu) and Yatish Turakhia (yturakhi@uscs.edu). -## Encoded Input -`cudaextender` expects the input strands to be encoded as integer sequences. -This encoding scheme is documented here: [utils.hpp](include/claraparabricks/genomeworks/cudaextender/utils.hpp) -file. The provided `encode_sequence()` helper function will encode the input strands on CPU with -the expected scheme. +## Library +Built as `libcudaextender.[so|a]` + +* Ungapped X-Drop extension -## API `cudaextender` provides host and device pointer APIs to enable ease of integration with other producer/consumer modules. The user is expected to handle all memory transactions and device sychronizations for the device pointer API. The host pointer API abstracts those operations away. Both APIs are documented here: [extender.hpp](include/claraparabricks/genomeworks/cudaextender/extender.hpp) -## Library -Built as `libcudaextender.[so|a]` - -* Ungapped X-Drop extension +### Encoded Input +`cudaextender` expects the input strands to be encoded as integer sequences. +This encoding scheme is documented here: [utils.hpp](include/claraparabricks/genomeworks/cudaextender/utils.hpp) +file. The provided `encode_sequence()` helper function will encode the input strands on CPU with +the expected scheme. ## Sample [sample_cudaextender](samples/sample_cudaextender.cpp) - Protoype to show the usage of host and device pointer APIs on FASTA sequences. diff --git a/cudamapper/README.md b/cudamapper/README.md index 5b2cdbd8b..85fe8be9c 100644 --- a/cudamapper/README.md +++ b/cudamapper/README.md @@ -2,6 +2,19 @@ The `cudamapper` package provides minimizer-based GPU-accelerated approximate mapping. +## Library +Built as `libcudamapper.[so|a]` + +* `Indexer` module to generate an index of minimizers from a list of sequences. +* `Matcher` module to find locations of matching pairs of minimizers between sequences using minimizer indices. +* `Overlapper` module to generate overlaps from sequence of minimizer matches generated by matcher. + +APIs documented in [include](include/claraparabricks/genomeworks/cudamapper) folder. + +## Sample +[sample_cudamapper](samples/sample_cudamapper.cpp) - A prototypical binary highlighting the usage of `libcudamapper` APIs (indexer, matcher and overlapper) and +techniques to tie them into an application. + ## Tool `cudamapper` is an end-to-end command line to for sequence to sequence mapping. `cudamapper` outputs @@ -16,17 +29,3 @@ A query fasta can be mapped to a reference as follows: `cudamapper query.fasta target.fasta` To access more information about running cudamapper, run `cudamapper --help`. - -## Library -Built as `libcudamapper.[so|a]` - -* `Indexer` module to generate an index of minimizers from a list of sequences. -* `Matcher` module to find locations of matching pairs of minimizers between sequences using minimizer indices. -* `Overlapper` module to generate overlaps from sequence of minimizer matches generated by matcher. - -APIs documented in [include](include/claraparabricks/genomeworks/cudamapper) folder. - -## Sample -[sample_cudamapper](samples/sample_cudamapper.cpp) - A prototypical binary highlighting the usage of `libcudamapper.so` APIs (indexer, matcher and overlapper) and -techniques to tie them into an application. - diff --git a/cudapoa/README.md b/cudapoa/README.md index e6aafb6c0..ffe8b86c0 100644 --- a/cudapoa/README.md +++ b/cudapoa/README.md @@ -3,12 +3,6 @@ The `cudapoa` package provides a GPU-accelerated implementation of the [Partial Order Alignment](https://simpsonlab.github.io/2015/05/01/understanding-poa/) algorithm. It is heavily influenced by [SPOA](https://github.com/rvaser/spoa) and in many cases can be considered a GPU-accelerated replacement. Features include: -## Tool - -A command line tool for generating consensus and MSA from a list of `fasta`/`fastq` files. The tool -is built on top of `libcudapoa.so` and showcases optimization strategies for writing high performance -applications with `libcudapoa.so`. - ## Library Built as `libcudapoa.[so|a]` @@ -20,4 +14,11 @@ Built as `libcudapoa.[so|a]` APIs documented in [include](include/claraparabricks/genomeworks/cudapoa) folder. ## Sample -[sample_cudapoa](samples/sample_cudapoa.cpp) - A prototypical binary to showcase the use of `libcudapoa.so` APIs. +[sample_cudapoa](samples/sample_cudapoa.cpp) - A prototypical binary to showcase the use of `libcudapoa` APIs. + +## Tool + +A command line tool for generating consensus and MSA from a list of `fasta`/`fastq` files. The tool +is built on top of `libcudapoa` and showcases optimization strategies for writing high performance +applications with `libcudapoa`. + From 301c6418296a975a7c81cffb077bf9f285b38219 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Thu, 3 Dec 2020 17:06:28 -0500 Subject: [PATCH 279/281] [docs] update cudapoa string --- cudapoa/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudapoa/README.md b/cudapoa/README.md index ffe8b86c0..7b42694eb 100644 --- a/cudapoa/README.md +++ b/cudapoa/README.md @@ -1,7 +1,7 @@ # CUDAPOA The `cudapoa` package provides a GPU-accelerated implementation of the [Partial Order Alignment](https://simpsonlab.github.io/2015/05/01/understanding-poa/) -algorithm. It is heavily influenced by [SPOA](https://github.com/rvaser/spoa) and in many cases can be considered a GPU-accelerated replacement. Features include: +algorithm. It is heavily influenced by [SPOA](https://github.com/rvaser/spoa) and in many cases can be considered a GPU-accelerated replacement. ## Library Built as `libcudapoa.[so|a]` From 04185fc49f35784d4c9d0b76b0dfd5e8d64b18b7 Mon Sep 17 00:00:00 2001 From: Joyjit Daw Date: Thu, 3 Dec 2020 18:50:04 -0500 Subject: [PATCH 280/281] [cudapoa] add missing include in src file --- cudapoa/src/cudapoa.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cudapoa/src/cudapoa.cpp b/cudapoa/src/cudapoa.cpp index c16b08d73..1069250c3 100644 --- a/cudapoa/src/cudapoa.cpp +++ b/cudapoa/src/cudapoa.cpp @@ -17,6 +17,8 @@ #include #include +#include + namespace claraparabricks { From 7bab9e59a3866a33d7627b364566b515e648bc6c Mon Sep 17 00:00:00 2001 From: Andreas Hehn Date: Fri, 4 Dec 2020 12:16:05 +0100 Subject: [PATCH 281/281] [cudaaligner] improved explanatory comment of new tests --- .../tests/Test_ApproximateBandedMyers.cpp | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/cudaaligner/tests/Test_ApproximateBandedMyers.cpp b/cudaaligner/tests/Test_ApproximateBandedMyers.cpp index e9c6fd161..2492d24b9 100644 --- a/cudaaligner/tests/Test_ApproximateBandedMyers.cpp +++ b/cudaaligner/tests/Test_ApproximateBandedMyers.cpp @@ -87,16 +87,21 @@ TEST(TestApproximateBandedMyersStatic, ImplicitNWEntries1) // the diagonal band, but on the lower right block there are cases where the backtrace runs through this worst case // - which is technically still part of the band. This tests this specific corner case. - // Band of the NW matrix: (* = tested implicit 0-row entry) - // * * - // A A C C G G T T A A A A C C C C G G G G G T T A A A C G G T T - // A 1 0 1 2 3 4 5 A 5 C 5 C 5 G 5 G 5 T 5 T 5 A 5 A 5 C 5 C 5 G 5 G 5 T 6 T 7 A 8 A 9 C 10 C 11 C 12 12 13 14 15 16 - // A 2 1 0 1 2 3 4 C 4 C 4 G 4 G 4 T 4 T 4 A 4 A 4 C 4 C 4 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 13 12 13 14 15 - // C 3 2 1 0 1 2 3 C 3 G 3 G 3 T 3 T 3 A 3 A 3 C 3 C 3 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 G 12 13 13 12 13 14 - // C 4 3 2 1 0 1 2 G 2 G 2 T 2 T 2 A 2 A 2 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 13 T 13 13 14 13 12 13 - // G 5 4 3 2 1 0 1 G 1 T 1 T 1 A 1 A 1 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 12 T 12 T 13 14 14 14 13 12 - // G 6 5 4 3 2 1 0 T 0 T 0 A 0 A 0 C 1 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 10 G 11 T 11 T 11 T 12 T 13 14 15 15 14 13 - // T 7 6 5 4 3 2 1 T 1 A 1 A 1 C 1 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 10 G 10 T 10 T 10 T 11 T 12 T 13 14 15 16 15 14 + // * = tested implicit 0-row entry + // Band of the NW matrix: + // top left block of NW m.| diagonal band (each column shifted by one row) | bottom right block + // NW_(i,j), index i of first shown row: + // 1 1 1 1 1 1 1 | 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | 20 20 20 20 20 20 + // + // A A C C G G | T T A A A A C C C C G G G G G T T A A | A C G G T T + // row 0 (implicit): | * | * + // A 1 0 1 2 3 4 5 |A 5 C 5 C 5 G 5 G 5 T 5 T 5 A 5 A 5 C 5 C 5 G 5 G 5 T 6 T 7 A 8 A 9 C 10 C 11 |C 12 12 13 14 15 16 + // A 2 1 0 1 2 3 4 |C 4 C 4 G 4 G 4 T 4 T 4 A 4 A 4 C 4 C 4 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 |G 12 13 12 13 14 15 + // C 3 2 1 0 1 2 3 |C 3 G 3 G 3 T 3 T 3 A 3 A 3 C 3 C 3 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 |G 12 13 13 12 13 14 + // C 4 3 2 1 0 1 2 |G 2 G 2 T 2 T 2 A 2 A 2 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 13 |T 13 13 14 13 12 13 + // G 5 4 3 2 1 0 1 |G 1 T 1 T 1 A 1 A 1 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 12 T 12 |T 13 14 14 14 13 12 + // G 6 5 4 3 2 1 0 |T 0 T 0 A 0 A 0 C 1 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 10 G 11 T 11 T 11 T 12 |T 13 14 15 15 14 13 + // T 7 6 5 4 3 2 1 |T 1 A 1 A 1 C 1 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 10 G 10 T 10 T 10 T 11 T 12 |T 13 14 15 16 15 14 implicit_new_entries_test_impl("AACCGGTTAACCGGTTAACCGGTTTT", "AACCGGTTAAAACCCCGGGGGTTAAACGGTT", @@ -105,16 +110,21 @@ TEST(TestApproximateBandedMyersStatic, ImplicitNWEntries1) TEST(TestApproximateBandedMyersStatic, ImplicitNWEntries2) { - // Band of the NW matrix: (* = tested implicit 0-row entry) - // * - // A A C C G G T T A A A A C C C C G G G G G T T A A C C G G T T - // A 1 0 1 2 3 4 5 6 A 6 C 6 C 6 G 6 G 6 T 6 T 6 A 6 A 6 C 6 C 6 G 6 G 6 T 7 T 7 A 8 A 8 C 9 C 9 10 11 12 13 14 - // A 2 1 0 1 2 3 4 5 C 5 C 5 G 5 G 5 T 5 T 5 A 5 A 5 C 5 C 5 G 5 G 5 T 6 T 7 A 8 A 9 C 9 C 10 C 9 9 10 11 12 13 - // C 3 2 1 0 1 2 3 4 C 4 G 4 G 4 T 4 T 4 A 4 A 4 C 4 C 4 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 10 10 9 10 11 12 - // C 4 3 2 1 0 1 2 3 G 3 G 3 T 3 T 3 A 3 A 3 C 3 C 3 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 G 11 11 10 9 10 11 - // G 5 4 3 2 1 0 1 2 G 2 T 2 T 2 A 2 A 2 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 13 T 12 12 11 10 9 10 - // G 6 5 4 3 2 1 0 1 T 1 T 1 A 1 A 1 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 12 T 13 T 13 13 12 11 10 9 - // T 7 6 5 4 3 2 1 0 T 0 A 0 A 0 C 1 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 10 G 11 T 11 T 12 T 13 T 14 14 13 12 11 10 + // * = tested implicit 0-row entry + // Band of the NW matrix: + // top left block of NW mat, | diagonal band (each column shifted by one row) | bottom right block + // NW_(i,j), index i of first shown row: + // 1 1 1 1 1 1 1 1 | 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | 19 19 19 19 19 19 + // + // A A C C G G T | T A A A A C C C C G G G G G T T A A | C C G G T T + // row 0 (implicit): | * | + // A 1 0 1 2 3 4 5 6 |A 6 C 6 C 6 G 6 G 6 T 6 T 6 A 6 A 6 C 6 C 6 G 6 G 6 T 7 T 7 A 8 A 8 C 9 |C 9 10 11 12 13 14 + // A 2 1 0 1 2 3 4 5 |C 5 C 5 G 5 G 5 T 5 T 5 A 5 A 5 C 5 C 5 G 5 G 5 T 6 T 7 A 8 A 9 C 9 C 10 |C 9 9 10 11 12 13 + // C 3 2 1 0 1 2 3 4 |C 4 G 4 G 4 T 4 T 4 A 4 A 4 C 4 C 4 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 |G 10 10 9 10 11 12 + // C 4 3 2 1 0 1 2 3 |G 3 G 3 T 3 T 3 A 3 A 3 C 3 C 3 G 4 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 |G 11 11 10 9 10 11 + // G 5 4 3 2 1 0 1 2 |G 2 T 2 T 2 A 2 A 2 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 13 |T 12 12 11 10 9 10 + // G 6 5 4 3 2 1 0 1 |T 1 T 1 A 1 A 1 C 2 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 11 G 12 T 12 T 13 |T 13 13 12 11 10 9 + // T 7 6 5 4 3 2 1 0 |T 0 A 0 A 0 C 1 C 2 G 3 G 4 T 5 T 6 A 7 A 8 C 9 C 10 G 10 G 11 T 11 T 12 T 13 |T 14 14 13 12 11 10 implicit_new_entries_test_impl("AACCGGTTAACCGGTTAACCGGTTT", "AACCGGTTAAAACCCCGGGGGTTAACCGGTT", "10M2I2M2I3M2I3M1I6M1D");