-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'origin/main' into complex-type-support-…
…for-gte
- Loading branch information
Showing
300 changed files
with
8,985 additions
and
2,648 deletions.
There are no files selected for viewing
Validating CODEOWNERS rules …
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| # Copyright (c) Facebook, Inc. and its affiliates. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| # The CODEOWNERS file allows us to setup file based rules that will automatically | ||
| # request a review from owners on PRs with changes to matching files. | ||
| # We currently do not enforce these reviews as required so it's only a tool | ||
| # for more granular notifications at the moment. For example component maintainers | ||
| # can set a rule so that they are pinged on changes to the sections of the | ||
| # codebase that are relevant for their component. | ||
|
|
||
| # Only users that have write access to the repo can be added as owners. | ||
| # See the official docs for more details on syntax and precedence of rules: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#example-of-a-codeowners-file | ||
|
|
||
| # Build & CI | ||
| CMake/ @assignUser @majetideepak | ||
| *.cmake @assignUser @majetideepak | ||
| **/CMakeLists.txt @assignUser @majetideepak | ||
| scripts/ @assignUser @majetideepak | ||
| .github/ @assignUser @majetideepak | ||
|
|
||
| # Parquet | ||
| velox/dwio/parquet/ @majetideepak | ||
|
|
||
| # Storage Adapters | ||
| velox/connectors/hive/storage_adapters/ @majetideepak | ||
|
|
||
| # Connectors | ||
| velox/connectors/ @majetideepak | ||
|
|
||
| # Caching | ||
| velox/common/caching/ @majetideepak |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| /* | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #include "velox/common/base/AdmissionController.h" | ||
|
|
||
| #include "velox/common/base/Exceptions.h" | ||
| #include "velox/common/base/StatsReporter.h" | ||
| #include "velox/common/time/Timer.h" | ||
|
|
||
| namespace facebook::velox::common { | ||
|
|
||
| void AdmissionController::accept(uint64_t resourceUnits) { | ||
| ContinueFuture future; | ||
| uint64_t updatedValue = 0; | ||
| VELOX_CHECK_LE( | ||
| resourceUnits, | ||
| config_.maxLimit, | ||
| "A single request cannot exceed the max limit"); | ||
| { | ||
| std::lock_guard<std::mutex> l(mu_); | ||
| if (unitsUsed_ + resourceUnits > config_.maxLimit) { | ||
| auto [unblockPromise, unblockFuture] = makeVeloxContinuePromiseContract(); | ||
| Request req; | ||
| req.unitsRequested = resourceUnits; | ||
| req.promise = std::move(unblockPromise); | ||
| queue_.push_back(std::move(req)); | ||
| future = std::move(unblockFuture); | ||
| } else { | ||
| updatedValue = unitsUsed_ += resourceUnits; | ||
| } | ||
| } | ||
| if (!future.valid()) { | ||
| // Only upadate if there was no wait, as the releasing thread is responsible | ||
| // for updating the metric. | ||
| if (!config_.resourceUsageAvgMetric.empty()) { | ||
| RECORD_METRIC_VALUE(config_.resourceUsageAvgMetric, updatedValue); | ||
| } | ||
| return; | ||
| } | ||
| if (!config_.resourceQueuedCountMetric.empty()) { | ||
| RECORD_METRIC_VALUE(config_.resourceQueuedCountMetric); | ||
| } | ||
| uint64_t waitTimeUs{0}; | ||
| { | ||
| MicrosecondTimer timer(&waitTimeUs); | ||
| future.wait(); | ||
| } | ||
| if (!config_.resourceQueuedTimeMsHistogramMetric.empty()) { | ||
| RECORD_HISTOGRAM_METRIC_VALUE( | ||
| config_.resourceQueuedTimeMsHistogramMetric, waitTimeUs / 1'000); | ||
| } | ||
| } | ||
|
|
||
| void AdmissionController::release(uint64_t resourceUnits) { | ||
| uint64_t updatedValue = 0; | ||
| { | ||
| std::lock_guard<std::mutex> l(mu_); | ||
| VELOX_CHECK_LE( | ||
| resourceUnits, | ||
| unitsUsed_, | ||
| "Cannot release more units than have been acquired"); | ||
| unitsUsed_ -= resourceUnits; | ||
| while (!queue_.empty()) { | ||
| auto& request = queue_.front(); | ||
| if (unitsUsed_ + request.unitsRequested > config_.maxLimit) { | ||
| break; | ||
| } | ||
| unitsUsed_ += request.unitsRequested; | ||
| request.promise.setValue(); | ||
| queue_.pop_front(); | ||
| } | ||
| updatedValue = unitsUsed_; | ||
| } | ||
| if (!config_.resourceUsageAvgMetric.empty()) { | ||
| RECORD_METRIC_VALUE(config_.resourceUsageAvgMetric, updatedValue); | ||
| } | ||
| } | ||
| } // namespace facebook::velox::common |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| /* | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| #pragma once | ||
|
|
||
| #include <deque> | ||
| #include <mutex> | ||
| #include "velox/common/future/VeloxPromise.h" | ||
|
|
||
| namespace facebook::velox::common { | ||
|
|
||
| /// A generic admission controller that can be used to limit the number of | ||
| /// resources in use and can log metrics like resource usage, queued count, | ||
| /// queued wait times. When a calling thread's request for resources surpasses | ||
| /// the set limit, it will be placed in a FIFO queue. The thread must then wait | ||
| /// until sufficient resources are freed by other threads, addressing all | ||
| /// preceding requests in the queue, before its own request can be granted. | ||
| class AdmissionController { | ||
| public: | ||
| struct Config { | ||
| /// The maximum number of resource units that can be used at any given time. | ||
| /// Set to a default value of max unit64 to signify unlimited limit. | ||
| uint64_t maxLimit{std::numeric_limits<uint64_t>::max()}; | ||
| /// The metric name for resource usage. If not set, it will not be reported. | ||
| /// Should be a registered as a average metric. | ||
| std::string resourceUsageAvgMetric; | ||
| /// The metric name for resource queued count. If not set, it will not be | ||
| /// reported. Should be a registered as a count metric. | ||
| std::string resourceQueuedCountMetric; | ||
| /// The metric name for resource queued wait time. If not set, it will not | ||
| /// be reported. Should be a registered as a histogram metric. | ||
| std::string resourceQueuedTimeMsHistogramMetric; | ||
| }; | ||
| explicit AdmissionController(const Config& config) : config_(config) {} | ||
|
|
||
| // Accept can block until sufficient resources are freed by other threads. | ||
| void accept(uint64_t resourceUnits); | ||
| void release(uint64_t resourceUnits); | ||
|
|
||
| uint64_t currentResourceUsage() const { | ||
| std::lock_guard<std::mutex> l(mu_); | ||
| return unitsUsed_; | ||
| } | ||
|
|
||
| private: | ||
| struct Request { | ||
| uint64_t unitsRequested; | ||
| ContinuePromise promise; | ||
| }; | ||
| Config config_; | ||
| mutable std::mutex mu_; | ||
| uint64_t unitsUsed_{0}; | ||
| std::deque<Request> queue_; | ||
| }; | ||
| } // namespace facebook::velox::common |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.