Merge pull request #3 from influxdata/mgattozzi/pro

feat: merge the influxdb source code into pro

.cargo/config

@@ -0,0 +1,25 @@
+[build]
+# enable tokio-console and some other goodies
+rustflags = [
+    "--cfg", "tokio_unstable",
+]
+
+# sparse protocol opt-in
+# See https://blog.rust-lang.org/2023/03/09/Rust-1.68.0.html#cargos-sparse-protocol
+[registries.crates-io]
+protocol = "sparse"
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = [
+    # see above
+    "--cfg", "tokio_unstable",
+    # Faster linker.
+    "-C", "link-arg=-fuse-ld=lld",
+    # Fix `perf` as suggested by https://github.com/flamegraph-rs/flamegraph/blob/2d19a162df4066f37d58d5471634f0bd9f0f4a62/README.md?plain=1#L18
+    # Also see https://bugs.chromium.org/p/chromium/issues/detail?id=919499#c16
+    "-C", "link-arg=-Wl,--no-rosegment",
+    # Enable all features supported by CPUs more recent than haswell (2013)
+    "-C", "target-cpu=haswell",
+    # Enable framepointers because profiling and debugging is a nightmare w/o it and it is generally not considered a performance advantage on modern x86_64 CPUs.
+    "-C", "force-frame-pointers=yes",
+]

.circleci/config.yml

@@ -0,0 +1,284 @@
+---
+# CI Overview
+# -----------
+#
+# Every commit:
+#
+# The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries
+#
+# On main if all these checks pass it will then additionally compile in "release" mode and
+# publish a docker image to quay.io/influxdb/influxdb3:$COMMIT_SHA
+#
+# Manually trigger build and push of container image for a branch:
+#
+# Navigate to https://app.circleci.com/pipelines/github/influxdata/influxdb?branch=<branch-name> (<- change this!)
+# Then:
+#
+# - Click "Run Pipeline" in the top-right
+# - Expand "Add Parameters"
+# - Add a "boolean" parameter called "release_branch" with the value true
+# - Click "Run Pipeline"
+#
+# You can also do this using the CircleCI API:
+#
+# Using `xh`:
+#
+#   # e.g. using 'xh' (https://github.com/ducaale/xh)
+#   $ xh -a '<your personal circleCI token>:' POST \
+#       https://circleci.com/api/v2/project/github/influxdata/influxdb/pipeline \
+#       parameters:='{"release_branch": true}' branch=chore/ci-tidy-up
+#
+# ...or equivalent with `curl`:
+#   $ curl -XPOST -H "Content-Type: application/json" -H "Circle-Token: <your personal circleCI token>" \
+#       -d '{"parameters": {"release_branch": true}, "branch": "chore/ci-tidy-up"}' \
+#       https://circleci.com/api/v2/project/github/influxdata/influxdb/pipeline
+
+version: 2.1
+
+commands:
+  rust_components:
+    description: Verify installed components
+    steps:
+      - run:
+          name: Verify installed components
+          command: |
+            rustup --version
+            rustup show
+            cargo fmt --version
+            cargo clippy --version
+
+  login_to_gcloud:
+    steps:
+      - run:
+          name: Login to gcloud
+          command: |
+            echo "${GCLOUD_SERVICE_ACCOUNT_KEY}" >/tmp/gcloud.json
+            gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file /tmp/gcloud.json --quiet
+            rm -f /tmp/gcloud.json
+            gcloud auth configure-docker us-docker.pkg.dev
+
+jobs:
+  fmt:
+    docker:
+      - image: quay.io/influxdb/rust:ci
+    environment:
+      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
+      CARGO_INCREMENTAL: "0"
+      # Disable full debug symbol generation to speed up CI build
+      # "1" means line tables only, which is useful for panic tracebacks.
+      CARGO_PROFILE_DEV_DEBUG: "1"
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+    steps:
+      - checkout
+      - rust_components
+      - run:
+          name: Rust fmt
+          command: cargo fmt --all -- --check
+  lint:
+    docker:
+      - image: quay.io/influxdb/rust:ci
+    environment:
+      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
+      CARGO_INCREMENTAL: "0"
+      # Disable full debug symbol generation to speed up CI build
+      # "1" means line tables only, which is useful for panic tracebacks.
+      CARGO_PROFILE_DEV_DEBUG: "1"
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+    steps:
+      - checkout
+      - rust_components
+      - run:
+          name: Clippy
+          command: cargo clippy --all-targets --all-features --workspace -- -D warnings
+      - run:
+          name: Yamllint
+          command: yamllint --config-file .circleci/yamllint.yml --strict .
+  cargo_audit:
+    docker:
+      - image: quay.io/influxdb/rust:ci
+    environment:
+      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
+      CARGO_INCREMENTAL: "0"
+      # Disable full debug symbol generation to speed up CI build
+      # "1" means line tables only, which is useful for panic tracebacks.
+      CARGO_PROFILE_DEV_DEBUG: "1"
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+    steps:
+      - checkout
+      - rust_components
+      - run:
+          name: Install cargo-deny
+          command: cargo install cargo-deny --locked
+      - run:
+          name: cargo-deny Checks
+          command: cargo deny check -s
+  doc:
+    docker:
+      - image: quay.io/influxdb/rust:ci
+    resource_class: large # use of a smaller executor runs out of memory
+    environment:
+      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
+      CARGO_INCREMENTAL: "0"
+      # Disable full debug symbol generation to speed up CI build
+      # "1" means line tables only, which is useful for panic tracebacks.
+      CARGO_PROFILE_DEV_DEBUG: "1"
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+      # Turn warnings into errors
+      RUSTDOCFLAGS: "-D warnings -A rustdoc::private-intra-doc-links"
+    steps:
+      - checkout
+      - rust_components
+      - run:
+          name: Cargo doc
+          command: cargo doc --document-private-items --no-deps --workspace
+      - run:
+          name: Compress Docs
+          command: tar -cvzf rustdoc.tar.gz target/doc/
+      - store_artifacts:
+          path: rustdoc.tar.gz
+
+  # Run all tests
+  test:
+    docker:
+      - image: quay.io/influxdb/rust:ci
+    resource_class: 2xlarge+ # use of a smaller executor tends crashes on link
+    environment:
+      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
+      CARGO_INCREMENTAL: "0"
+      # Disable full debug symbol generation to speed up CI build
+      # "1" means line tables only, which is useful for panic tracebacks.
+      CARGO_PROFILE_DEV_DEBUG: "1"
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+      RUST_BACKTRACE: "1"
+    steps:
+      - checkout
+      - rust_components
+      - run:
+          name: cargo test --workspace
+          command: cargo test --workspace
+
+  # end to end tests with Heappy (heap profiling enabled)
+  test_heappy:
+    docker:
+      - image: quay.io/influxdb/rust:ci
+    resource_class: xlarge # use of a smaller executor tends crashes on link
+    environment:
+      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
+      CARGO_INCREMENTAL: "0"
+      # Disable full debug symbol generation to speed up CI build
+      # "1" means line tables only, which is useful for panic tracebacks.
+      CARGO_PROFILE_DEV_DEBUG: "1"
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+      RUST_BACKTRACE: "1"
+    steps:
+      - checkout
+      - rust_components
+      - run:
+          name: cargo test --no-default-features --features=heappy --workspace
+          command: cargo test --no-default-features --features=heappy --workspace
+
+  # Build a dev binary.
+  #
+  # Compiles a binary with the default ("dev") cargo profile from the influxdb3 source
+  # using the latest ci_image (influxdb/rust) and ensures various targets compile successfully
+  build_dev:
+    docker:
+      - image: quay.io/influxdb/rust:ci
+    resource_class: 2xlarge # use of a smaller executor tends crashes on link
+    environment:
+      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
+      CARGO_INCREMENTAL: "0"
+      # Disable full debug symbol generation to speed up CI build
+      # "1" means line tables only, which is useful for panic tracebacks.
+      CARGO_PROFILE_DEV_DEBUG: "1"
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+      # The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on
+      # avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit.
+      CARGO_BUILD_JOBS: "12"
+    steps:
+      - checkout
+      - rust_components
+      - run:
+          name: Cargo build
+          command: cargo build --workspace
+      - run:
+          name: Check benches compile
+          command: cargo check --workspace --benches
+      - run:
+          name: Check extra features (like prod image)
+          command: cargo check --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof"
+
+  # Compile cargo "release" profile binaries for influxdb3 & data generator
+  #
+  # Uses the latest ci_image (influxdb/rust) to build a release binary and
+  # copies it to a minimal container image based upon `rust:slim-buster`. This
+  # minimal image is then pushed to `quay.io/influxdb/influxdb3:${BRANCH}`.
+  build_release:
+    # need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a
+    # kinda small node)
+    machine:
+      image: default
+    resource_class: 2xlarge # CPU bound, so make it fast
+    steps:
+      - checkout
+      - run:
+          name: Cargo release build
+          command: |
+            COMMIT_SHA="$(git rev-parse HEAD)"
+
+            .circleci/docker_build_release.bash \
+              "influxdb3" \
+              "aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" \
+              "quay.io/influxdb/influxdb3:$COMMIT_SHA"
+
+            mkdir /tmp/images
+            docker save quay.io/influxdb/influxdb3:"$COMMIT_SHA" | gzip > /tmp/images/influxdb3.tar.gz
+          # linking might take a while and doesn't produce CLI output
+          no_output_timeout: 30m
+      - store_artifacts:
+          path: /tmp/images
+      - persist_to_workspace:
+          root: /tmp/images
+          paths:
+            - "*.tar.gz"
+
+parameters:
+  release_branch:
+    description: "Build and push container image for non-main branch"
+    type: boolean
+    default: false
+
+workflows:
+  version: 2
+
+  # CI for all pull requests.
+  ci:
+    when:
+      and:
+        - not: << pipeline.parameters.release_branch >>
+    jobs:
+      - fmt
+      - lint
+      - cargo_audit
+      - test
+      - test_heappy
+      - build_dev
+      - doc
+      - build_release:
+          filters:
+            branches:
+              only: main
+
+  # Force build and push of container image for non-main branch.
+  # See instructions at the top of this file
+  release_branch:
+    when: << pipeline.parameters.release_branch >>
+    jobs:
+      - build_release

.circleci/docker_build_release.bash

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly PACKAGE="$1"
+readonly FEATURES="$2"
+readonly TAG="$3"
+
+RUST_VERSION="$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)"
+COMMIT_SHA="$(git rev-parse HEAD)"
+COMMIT_TS="$(env TZ=UTC0 git show --quiet --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" HEAD)"
+NOW="$(date --utc --iso-8601=seconds)"
+REPO_URL="https://github.com/influxdata/influxdb_iox"
+
+exec docker buildx build \
+  --build-arg CARGO_INCREMENTAL="no" \
+  --build-arg CARGO_NET_GIT_FETCH_WITH_CLI="true" \
+  --build-arg FEATURES="$FEATURES" \
+  --build-arg RUST_VERSION="$RUST_VERSION" \
+  --build-arg PACKAGE="$PACKAGE" \
+  --label org.opencontainers.image.created="$NOW" \
+  --label org.opencontainers.image.url="$REPO_URL" \
+  --label org.opencontainers.image.revision="$COMMIT_SHA" \
+  --label org.opencontainers.image.vendor="InfluxData Inc." \
+  --label org.opencontainers.image.title="InfluxDB IOx, '$PACKAGE'" \
+  --label org.opencontainers.image.description="InfluxDB IOx production image for package '$PACKAGE'" \
+  --label com.influxdata.image.commit-date="$COMMIT_TS" \
+  --label com.influxdata.image.package="$PACKAGE" \
+  --progress plain \
+  --tag "$TAG" \
+  .

.circleci/yamllint.yml

@@ -0,0 +1,3 @@
+rules:
+  truthy:
+    check-keys: false

.editorconfig

@@ -0,0 +1,13 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+
+[{Dockerfile*,*.proto}]
+indent_size = 2
+
+[{*.rs,*.toml,*.sh,*.bash}]
+indent_size = 4

.gitattributes

@@ -0,0 +1,4 @@
+generated_types/protos/google/ linguist-generated=true
+generated_types/protos/grpc/ linguist-generated=true
+generated_types/src/wal_generated.rs linguist-generated=true
+trace_exporters/src/thrift/ linguist-generated=true

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# This file allows teams or people to be assigned as
+# automatic code-reviewers for files or directories.
+#
+# Here is information about how to configure this file:
+# https://help.github.com/en/articles/about-code-owners