@@ -53,6 +53,149 @@ struct PrefetchRange {
5353 : start_offset(start_offset), end_offset(end_offset) {}
5454
5555 PrefetchRange () : start_offset(0 ), end_offset(0 ) {}
56+
57+ bool operator ==(const PrefetchRange& other) const {
58+ return (start_offset == other.start_offset ) && (end_offset == other.end_offset );
59+ }
60+
61+ bool operator !=(const PrefetchRange& other) const { return !(*this == other); }
62+
63+ PrefetchRange span (const PrefetchRange& other) const {
64+ return {std::min (start_offset, other.end_offset ), std::max (start_offset, other.end_offset )};
65+ }
66+ PrefetchRange seq_span (const PrefetchRange& other) const {
67+ return {start_offset, other.end_offset };
68+ }
69+
70+ // Ranges needs to be sorted.
71+ static std::vector<PrefetchRange>
merge_adjacent_seq_ranges (
72+ const std::vector<PrefetchRange>& seq_ranges,
int64_t max_merge_distance_bytes,
73+ int64_t once_max_read_bytes) {
74+ if (seq_ranges.empty ()) {
75+ return {};
76+ }
77+ // Merge overlapping ranges
78+ std::vector<PrefetchRange> result;
79+ PrefetchRange last = seq_ranges.front ();
80+ for (size_t i = 1 ; i < seq_ranges.size (); ++i) {
81+ PrefetchRange current = seq_ranges[i];
82+ PrefetchRange merged = last.seq_span (current);
83+ if (merged.end_offset <= once_max_read_bytes + merged.start_offset &&
84+ last.end_offset + max_merge_distance_bytes >= current.start_offset ) {
85+ last = merged;
86+ } else {
87+ result.push_back (last);
88+ last = current;
89+ }
90+ }
91+ result.push_back (last);
92+ return result;
93+ }
94+ };
95+
96+ class RangeFinder {
97+ public:
98+ virtual ~RangeFinder () = default ;
99+ virtual Status get_range_for (int64_t desired_offset, io::PrefetchRange& result_range) = 0;
100+ virtual size_t get_max_range_size () const = 0;
101+ };
102+
103+ class LinearProbeRangeFinder : public RangeFinder {
104+ public:
105+ LinearProbeRangeFinder (std::vector<io::PrefetchRange>&& ranges) : _ranges(std::move(ranges)) {}
106+
107+ Status get_range_for (int64_t desired_offset, io::PrefetchRange& result_range) override ;
108+
109+ size_t get_max_range_size () const override {
110+ size_t max_range_size = 0 ;
111+ for (const auto & range : _ranges) {
112+ max_range_size = std::max (max_range_size, range.end_offset - range.start_offset );
113+ }
114+ return max_range_size;
115+ }
116+
117+ ~LinearProbeRangeFinder () override = default ;
118+
119+ private:
120+ std::vector<io::PrefetchRange> _ranges;
121+ size_t index {0 };
122+ };
123+
124+ /* *
125+ * The reader provides a solution to read one range at a time. You can customize RangeFinder to meet your scenario.
126+ * For me, since there will be tiny stripes when reading orc files, in order to reduce the requests to hdfs,
127+ * I first merge the access to the orc files to be read (of course there is a problem of read amplification,
128+ * but in my scenario, compared with reading hdfs multiple times, it is faster to read more data on hdfs at one time),
129+ * and then because the actual reading of orc files is in order from front to back, I provide LinearProbeRangeFinder.
130+ */
131+ class RangeCacheFileReader : public io ::FileReader {
132+ struct RangeCacheReaderStatistics {
133+ int64_t request_io = 0 ;
134+ int64_t request_bytes = 0 ;
135+ int64_t request_time = 0 ;
136+ int64_t read_to_cache_time = 0 ;
137+ int64_t cache_refresh_count = 0 ;
138+ int64_t read_to_cache_bytes = 0 ;
139+ };
140+
141+ public:
142+ RangeCacheFileReader (RuntimeProfile* profile, io::FileReaderSPtr inner_reader,
143+ std::shared_ptr<RangeFinder> range_finder);
144+
145+ ~RangeCacheFileReader () override = default ;
146+
147+ Status close () override {
148+ if (!_closed) {
149+ _closed = true ;
150+ }
151+ return Status::OK ();
152+ }
153+
154+ const io::Path& path () const override { return _inner_reader->path (); }
155+
156+ size_t size () const override { return _size; }
157+
158+ bool closed () const override { return _closed; }
159+
160+ std::shared_ptr<io::FileSystem> fs () const override { return _inner_reader->fs (); }
161+
162+ protected:
163+ Status read_at_impl (size_t offset, Slice result, size_t * bytes_read,
164+ const IOContext* io_ctx) override ;
165+
166+ void _collect_profile_before_close () override ;
167+
168+ private:
169+ RuntimeProfile* _profile = nullptr ;
170+ io::FileReaderSPtr _inner_reader;
171+ std::shared_ptr<RangeFinder> _range_finder;
172+
173+ OwnedSlice _cache;
174+ int64_t _current_start_offset = -1 ;
175+
176+ size_t _size;
177+ bool _closed = false ;
178+
179+ RuntimeProfile::Counter* _request_io = nullptr ;
180+ RuntimeProfile::Counter* _request_bytes = nullptr ;
181+ RuntimeProfile::Counter* _request_time = nullptr ;
182+ RuntimeProfile::Counter* _read_to_cache_time = nullptr ;
183+ RuntimeProfile::Counter* _cache_refresh_count = nullptr ;
184+ RuntimeProfile::Counter* _read_to_cache_bytes = nullptr ;
185+ RangeCacheReaderStatistics _cache_statistics;
186+ /* *
187+ * `RangeCacheFileReader`:
188+ * 1. `CacheRefreshCount`: how many IOs are merged
189+ * 2. `ReadToCacheBytes`: how much data is actually read after merging
190+ * 3. `ReadToCacheTime`: how long it takes to read data after merging
191+ * 4. `RequestBytes`: how many bytes does the apache-orc library actually need to read the orc file
192+ * 5. `RequestIO`: how many times the apache-orc library calls this read interface
193+ * 6. `RequestTime`: how long it takes the apache-orc library to call this read interface
194+ *
195+ * It should be noted that `RangeCacheFileReader` is a wrapper of the reader that actually reads data,such as
196+ * the hdfs reader, so strictly speaking, `CacheRefreshCount` is not equal to how many IOs are initiated to hdfs,
197+ * because each time the hdfs reader is requested, the hdfs reader may not be able to read all the data at once.
198+ */
56199};
57200
58201/* *
0 commit comments