diff --git a/app/models/event.rb b/app/models/event.rb index 1179ba1f622..6d035a3ed0b 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -8,7 +8,6 @@ class Event < EventsRecord include OrganizationTimezone belongs_to :organization - belongs_to :quantified_event, optional: true validates :transaction_id, presence: true, uniqueness: { scope: %i[organization_id external_subscription_id] } validates :code, presence: true diff --git a/app/services/billable_metrics/aggregation_factory.rb b/app/services/billable_metrics/aggregation_factory.rb index 76d057aa819..e4d5484f0c9 100644 --- a/app/services/billable_metrics/aggregation_factory.rb +++ b/app/services/billable_metrics/aggregation_factory.rb @@ -2,16 +2,10 @@ module BillableMetrics class AggregationFactory - CLICKHOUSE_READY = %i[count_agg latest_agg max_agg sum_agg weighted_sum_agg].freeze - class << self def supports_clickhouse? ENV['LAGO_CLICKHOUSE_ENABLED'].present? end - - def clickhouse_ready?(aggregation_type) - CLICKHOUSE_READY.include?(aggregation_type.to_sym) - end end def self.new_instance(charge:, current_usage: false, **attributes) @@ -19,8 +13,7 @@ def self.new_instance(charge:, current_usage: false, **attributes) if !charge.pay_in_advance? && supports_clickhouse? && - charge.billable_metric.organization.clickhouse_aggregation? && - clickhouse_ready?(charge.billable_metric.aggregation_type) + charge.billable_metric.organization.clickhouse_aggregation? event_store = Events::Stores::ClickhouseStore end diff --git a/app/services/billable_metrics/aggregations/unique_count_service.rb b/app/services/billable_metrics/aggregations/unique_count_service.rb index 7bf7e827752..6e7f419a468 100644 --- a/app/services/billable_metrics/aggregations/unique_count_service.rb +++ b/app/services/billable_metrics/aggregations/unique_count_service.rb @@ -162,19 +162,21 @@ def base_scope quantified_events = QuantifiedEvent .where(organization_id: billable_metric.organization_id) .where(billable_metric_id: billable_metric.id) + .where(external_subscription_id: subscription.external_id) - quantified_events = if billable_metric.recurring? - quantified_events.where(external_subscription_id: subscription.external_id) - else - scope = Event.where(external_subscription_id: subscription.external_id) - .where('quantified_event_id IS NOT NULL') - .where(timestamp: subscription.started_at..) - - scope = scope.where(timestamp: ...subscription.terminated_at) if subscription.terminated? - - quantified_event_ids = scope.pluck('DISTINCT(quantified_event_id)') + unless billable_metric.recurring? + store = event_store_class.new( + code: billable_metric.code, + subscription:, + boundaries: { + from_datetime: subscription.started_at, + to_datetime: subscription.terminated_at, + }, + group:, + ) + store.aggregation_property = billable_metric.field_name - quantified_events.where(id: quantified_event_ids) + quantified_events = quantified_events.where(external_id: store.events_values) end return quantified_events unless group diff --git a/app/services/events/post_process_service.rb b/app/services/events/post_process_service.rb index 1424a83ad6f..733f2974d63 100644 --- a/app/services/events/post_process_service.rb +++ b/app/services/events/post_process_service.rb @@ -108,10 +108,6 @@ def should_handle_quantified_event? def handle_quantified_event service_result = quantified_event_service.call service_result.raise_if_error! - - # TODO: Remove this relation - event.quantified_event_id = service_result.quantified_event&.id - event.save! end def handle_pay_in_advance diff --git a/app/services/events/stores/clickhouse_store.rb b/app/services/events/stores/clickhouse_store.rb index adde5a7f848..bbbdb652076 100644 --- a/app/services/events/stores/clickhouse_store.rb +++ b/app/services/events/stores/clickhouse_store.rb @@ -10,12 +10,12 @@ class ClickhouseStore < BaseStore # and should be deduplicated depending on the aggregation logic def events(force_from: false) scope = ::Clickhouse::EventsRaw.where(external_subscription_id: subscription.external_id) - .where('events_raw.timestamp <= ?', to_datetime) .where(code:) .order(timestamp: :asc) # TODO: check how to deal with this since events are not stored forever in clickhouse scope = scope.where('events_raw.timestamp >= ?', from_datetime) if force_from || use_from_boundary + scope = scope.where('events_raw.timestamp <= ?', to_datetime) if to_datetime scope = scope.where(numeric_condition) if numeric_property return scope unless group diff --git a/app/services/events/stores/postgres_store.rb b/app/services/events/stores/postgres_store.rb index 270265f68ba..bb8a0db3649 100644 --- a/app/services/events/stores/postgres_store.rb +++ b/app/services/events/stores/postgres_store.rb @@ -5,11 +5,11 @@ module Stores class PostgresStore < BaseStore def events(force_from: false) scope = Event.where(external_subscription_id: subscription.external_id) - .to_datetime(to_datetime) .where(code:) .order(timestamp: :asc) scope = scope.from_datetime(from_datetime) if force_from || use_from_boundary + scope = scope.to_datetime(to_datetime) if to_datetime if numeric_property scope = scope.where(presence_condition) diff --git a/db/clickhouse_schema.rb b/db/clickhouse_schema.rb index ed54af695c6..7f3bf2b1c6e 100644 --- a/db/clickhouse_schema.rb +++ b/db/clickhouse_schema.rb @@ -1,44 +1,45 @@ -# This file is auto-generated from the current state of the database. Instead -# of editing this file, please use the migrations feature of Active Record to -# incrementally modify your database, and then regenerate this schema definition. -# -# This file is the source Rails uses to define your schema when running `rails -# clickhouse:schema:load`. When creating a new database, `rails clickhouse:schema:load` tends to -# be faster and is potentially less error prone than running all of your -# migrations from scratch. Old migrations may fail to apply correctly if those -# migrations use external dependencies or application code. -# -# It's strongly recommended that you check this file into your version control system. - -ClickhouseActiverecord::Schema.define(version: 2023_10_30_163703) do - - # TABLE: events_raw - # SQL: CREATE TABLE default.events_raw ( `organization_id` String, `external_customer_id` String, `external_subscription_id` String, `transaction_id` String, `timestamp` DateTime, `code` String, `properties` Map(String, String) ) ENGINE = MergeTree PRIMARY KEY (organization_id, external_subscription_id, code, toStartOfDay(timestamp)) ORDER BY (organization_id, external_subscription_id, code, toStartOfDay(timestamp)) TTL timestamp TO VOLUME 'hot', timestamp + toIntervalDay(90) TO VOLUME 'cold' SETTINGS storage_policy = 'hot_cold', index_granularity = 8192 - create_table "events_raw", id: false, options: "MergeTree PRIMARY KEY (organization_id, external_subscription_id, code, toStartOfDay(timestamp)) ORDER BY (organization_id, external_subscription_id, code, toStartOfDay(timestamp)) TTL timestamp TO VOLUME 'hot', timestamp + toIntervalDay(90) TO VOLUME 'cold' SETTINGS storage_policy = 'hot_cold', index_granularity = 8192", force: :cascade do |t| - t.string "organization_id", null: false - t.string "external_customer_id", null: false - t.string "external_subscription_id", null: false - t.string "transaction_id", null: false - t.datetime "timestamp", precision: nil, null: false - t.string "code", null: false - t.map "properties", key_type: "string", value_type: "string", null: false - end - - # TABLE: events_raw_queue - # SQL: CREATE TABLE default.events_raw_queue ( `organization_id` String, `external_customer_id` String, `external_subscription_id` String, `transaction_id` String, `timestamp` DateTime, `code` String, `properties` String ) ENGINE = Kafka SETTINGS kafka_broker_list = '*****', kafka_topic_list = 'events-raw', kafka_group_name = 'clickhouse', kafka_format = 'JSONEachRow' - create_table "events_raw_queue", id: false, options: "Kafka SETTINGS kafka_broker_list = '*****', kafka_topic_list = 'events-raw', kafka_group_name = 'clickhouse', kafka_format = 'JSONEachRow'", force: :cascade do |t| - t.string "organization_id", null: false - t.string "external_customer_id", null: false - t.string "external_subscription_id", null: false - t.string "transaction_id", null: false - t.datetime "timestamp", precision: nil, null: false - t.string "code", null: false - t.string "properties", null: false - end - - # TABLE: events_raw_mv - # SQL: CREATE MATERIALIZED VIEW default.events_raw_mv TO default.events_raw ( `organization_id` String, `external_customer_id` String, `external_subscription_id` String, `transaction_id` String, `timestamp` DateTime, `code` String, `properties` Map(String, String) ) AS SELECT organization_id, external_customer_id, external_subscription_id, transaction_id, timestamp, code, CAST(JSONExtractKeysAndValuesRaw(properties), 'Map(String, String)') AS properties FROM default.events_raw_queue - create_table "events_raw_mv", view: true, materialized: true, id: false, as: "SELECT organization_id, external_customer_id, external_subscription_id, transaction_id, timestamp, code, CAST(JSONExtractKeysAndValuesRaw(properties), 'Map(String, String)') AS properties FROM default.events_raw_queue", force: :cascade do |t| - end - -end +*****#***** *****T*****h*****i*****s***** *****f*****i*****l*****e***** *****i*****s***** *****a*****u*****t*****o*****-*****g*****e*****n*****e*****r*****a*****t*****e*****d***** *****f*****r*****o*****m***** *****t*****h*****e***** *****c*****u*****r*****r*****e*****n*****t***** *****s*****t*****a*****t*****e***** *****o*****f***** *****t*****h*****e***** *****d*****a*****t*****a*****b*****a*****s*****e*****.***** *****I*****n*****s*****t*****e*****a*****d***** +*****#***** *****o*****f***** *****e*****d*****i*****t*****i*****n*****g***** *****t*****h*****i*****s***** *****f*****i*****l*****e*****,***** *****p*****l*****e*****a*****s*****e***** *****u*****s*****e***** *****t*****h*****e***** *****m*****i*****g*****r*****a*****t*****i*****o*****n*****s***** *****f*****e*****a*****t*****u*****r*****e***** *****o*****f***** *****A*****c*****t*****i*****v*****e***** *****R*****e*****c*****o*****r*****d***** *****t*****o***** +*****#***** *****i*****n*****c*****r*****e*****m*****e*****n*****t*****a*****l*****l*****y***** *****m*****o*****d*****i*****f*****y***** *****y*****o*****u*****r***** *****d*****a*****t*****a*****b*****a*****s*****e*****,***** *****a*****n*****d***** *****t*****h*****e*****n***** *****r*****e*****g*****e*****n*****e*****r*****a*****t*****e***** *****t*****h*****i*****s***** *****s*****c*****h*****e*****m*****a***** *****d*****e*****f*****i*****n*****i*****t*****i*****o*****n*****.***** +*****#***** +*****#***** *****T*****h*****i*****s***** *****f*****i*****l*****e***** *****i*****s***** *****t*****h*****e***** *****s*****o*****u*****r*****c*****e***** *****R*****a*****i*****l*****s***** *****u*****s*****e*****s***** *****t*****o***** *****d*****e*****f*****i*****n*****e***** *****y*****o*****u*****r***** *****s*****c*****h*****e*****m*****a***** *****w*****h*****e*****n***** *****r*****u*****n*****n*****i*****n*****g***** *****`*****r*****a*****i*****l*****s***** +*****#***** *****c*****l*****i*****c*****k*****h*****o*****u*****s*****e*****:*****s*****c*****h*****e*****m*****a*****:*****l*****o*****a*****d*****`*****.***** *****W*****h*****e*****n***** *****c*****r*****e*****a*****t*****i*****n*****g***** *****a***** *****n*****e*****w***** *****d*****a*****t*****a*****b*****a*****s*****e*****,***** *****`*****r*****a*****i*****l*****s***** *****c*****l*****i*****c*****k*****h*****o*****u*****s*****e*****:*****s*****c*****h*****e*****m*****a*****:*****l*****o*****a*****d*****`***** *****t*****e*****n*****d*****s***** *****t*****o***** +*****#***** *****b*****e***** *****f*****a*****s*****t*****e*****r***** *****a*****n*****d***** *****i*****s***** *****p*****o*****t*****e*****n*****t*****i*****a*****l*****l*****y***** *****l*****e*****s*****s***** *****e*****r*****r*****o*****r***** *****p*****r*****o*****n*****e***** *****t*****h*****a*****n***** *****r*****u*****n*****n*****i*****n*****g***** *****a*****l*****l***** *****o*****f***** *****y*****o*****u*****r***** +*****#***** *****m*****i*****g*****r*****a*****t*****i*****o*****n*****s***** *****f*****r*****o*****m***** *****s*****c*****r*****a*****t*****c*****h*****.***** *****O*****l*****d***** *****m*****i*****g*****r*****a*****t*****i*****o*****n*****s***** *****m*****a*****y***** *****f*****a*****i*****l***** *****t*****o***** *****a*****p*****p*****l*****y***** *****c*****o*****r*****r*****e*****c*****t*****l*****y***** *****i*****f***** *****t*****h*****o*****s*****e***** +*****#***** *****m*****i*****g*****r*****a*****t*****i*****o*****n*****s***** *****u*****s*****e***** *****e*****x*****t*****e*****r*****n*****a*****l***** *****d*****e*****p*****e*****n*****d*****e*****n*****c*****i*****e*****s***** *****o*****r***** *****a*****p*****p*****l*****i*****c*****a*****t*****i*****o*****n***** *****c*****o*****d*****e*****.***** +*****#***** +*****#***** *****I*****t*****'*****s***** *****s*****t*****r*****o*****n*****g*****l*****y***** *****r*****e*****c*****o*****m*****m*****e*****n*****d*****e*****d***** *****t*****h*****a*****t***** *****y*****o*****u***** *****c*****h*****e*****c*****k***** *****t*****h*****i*****s***** *****f*****i*****l*****e***** *****i*****n*****t*****o***** *****y*****o*****u*****r***** *****v*****e*****r*****s*****i*****o*****n***** *****c*****o*****n*****t*****r*****o*****l***** *****s*****y*****s*****t*****e*****m*****.***** +***** +*****C*****l*****i*****c*****k*****h*****o*****u*****s*****e*****A*****c*****t*****i*****v*****e*****r*****e*****c*****o*****r*****d*****:*****:*****S*****c*****h*****e*****m*****a*****.*****d*****e*****f*****i*****n*****e*****(*****v*****e*****r*****s*****i*****o*****n*****:***** *****2*****0*****2*****3*****_*****1*****0*****_*****3*****0*****_*****1*****6*****3*****7*****0*****3*****)***** *****d*****o***** +***** +***** ***** *****#***** *****T*****A*****B*****L*****E*****:***** *****e*****v*****e*****n*****t*****s*****_*****r*****a*****w***** +***** ***** *****#***** *****S*****Q*****L*****:***** *****C*****R*****E*****A*****T*****E***** *****T*****A*****B*****L*****E***** *****d*****e*****f*****a*****u*****l*****t*****.*****e*****v*****e*****n*****t*****s*****_*****r*****a*****w***** *****(***** *****`*****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****e*****x*****t*****e*****r*****n*****a*****l*****_*****c*****u*****s*****t*****o*****m*****e*****r*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****t*****r*****a*****n*****s*****a*****c*****t*****i*****o*****n*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****t*****i*****m*****e*****s*****t*****a*****m*****p*****`***** *****D*****a*****t*****e*****T*****i*****m*****e*****,***** *****`*****c*****o*****d*****e*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****p*****r*****o*****p*****e*****r*****t*****i*****e*****s*****`***** *****M*****a*****p*****(*****S*****t*****r*****i*****n*****g*****,***** *****S*****t*****r*****i*****n*****g*****)***** *****)***** *****E*****N*****G*****I*****N*****E***** *****=***** *****M*****e*****r*****g*****e*****T*****r*****e*****e***** *****P*****R*****I*****M*****A*****R*****Y***** *****K*****E*****Y***** *****(*****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****,***** *****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****,***** *****c*****o*****d*****e*****,***** *****t*****o*****S*****t*****a*****r*****t*****O*****f*****D*****a*****y*****(*****t*****i*****m*****e*****s*****t*****a*****m*****p*****)*****)***** *****O*****R*****D*****E*****R***** *****B*****Y***** *****(*****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****,***** *****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****,***** *****c*****o*****d*****e*****,***** *****t*****o*****S*****t*****a*****r*****t*****O*****f*****D*****a*****y*****(*****t*****i*****m*****e*****s*****t*****a*****m*****p*****)*****)***** *****T*****T*****L***** *****t*****i*****m*****e*****s*****t*****a*****m*****p***** *****T*****O***** *****V*****O*****L*****U*****M*****E***** *****'*****h*****o*****t*****'*****,***** *****t*****i*****m*****e*****s*****t*****a*****m*****p***** *****+***** *****t*****o*****I*****n*****t*****e*****r*****v*****a*****l*****D*****a*****y*****(*****9*****0*****)***** *****T*****O***** *****V*****O*****L*****U*****M*****E***** *****'*****c*****o*****l*****d*****'***** *****S*****E*****T*****T*****I*****N*****G*****S***** *****s*****t*****o*****r*****a*****g*****e*****_*****p*****o*****l*****i*****c*****y***** *****=***** *****'*****h*****o*****t*****_*****c*****o*****l*****d*****'*****,***** *****i*****n*****d*****e*****x*****_*****g*****r*****a*****n*****u*****l*****a*****r*****i*****t*****y***** *****=***** *****8*****1*****9*****2***** +***** ***** *****c*****r*****e*****a*****t*****e*****_*****t*****a*****b*****l*****e***** *****"*****e*****v*****e*****n*****t*****s*****_*****r*****a*****w*****"*****,***** *****i*****d*****:***** *****f*****a*****l*****s*****e*****,***** *****o*****p*****t*****i*****o*****n*****s*****:***** *****"*****M*****e*****r*****g*****e*****T*****r*****e*****e***** *****P*****R*****I*****M*****A*****R*****Y***** *****K*****E*****Y***** *****(*****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****,***** *****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****,***** *****c*****o*****d*****e*****,***** *****t*****o*****S*****t*****a*****r*****t*****O*****f*****D*****a*****y*****(*****t*****i*****m*****e*****s*****t*****a*****m*****p*****)*****)***** *****O*****R*****D*****E*****R***** *****B*****Y***** *****(*****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****,***** *****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****,***** *****c*****o*****d*****e*****,***** *****t*****o*****S*****t*****a*****r*****t*****O*****f*****D*****a*****y*****(*****t*****i*****m*****e*****s*****t*****a*****m*****p*****)*****)***** *****T*****T*****L***** *****t*****i*****m*****e*****s*****t*****a*****m*****p***** *****T*****O***** *****V*****O*****L*****U*****M*****E***** *****'*****h*****o*****t*****'*****,***** *****t*****i*****m*****e*****s*****t*****a*****m*****p***** *****+***** *****t*****o*****I*****n*****t*****e*****r*****v*****a*****l*****D*****a*****y*****(*****9*****0*****)***** *****T*****O***** *****V*****O*****L*****U*****M*****E***** *****'*****c*****o*****l*****d*****'***** *****S*****E*****T*****T*****I*****N*****G*****S***** *****s*****t*****o*****r*****a*****g*****e*****_*****p*****o*****l*****i*****c*****y***** *****=***** *****'*****h*****o*****t*****_*****c*****o*****l*****d*****'*****,***** *****i*****n*****d*****e*****x*****_*****g*****r*****a*****n*****u*****l*****a*****r*****i*****t*****y***** *****=***** *****8*****1*****9*****2*****"*****,***** *****f*****o*****r*****c*****e*****:***** *****:*****c*****a*****s*****c*****a*****d*****e***** *****d*****o***** *****|*****t*****|***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****e*****x*****t*****e*****r*****n*****a*****l*****_*****c*****u*****s*****t*****o*****m*****e*****r*****_*****i*****d*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****t*****r*****a*****n*****s*****a*****c*****t*****i*****o*****n*****_*****i*****d*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****d*****a*****t*****e*****t*****i*****m*****e***** *****"*****t*****i*****m*****e*****s*****t*****a*****m*****p*****"*****,***** *****p*****r*****e*****c*****i*****s*****i*****o*****n*****:***** *****n*****i*****l*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****c*****o*****d*****e*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****m*****a*****p***** *****"*****p*****r*****o*****p*****e*****r*****t*****i*****e*****s*****"*****,***** *****k*****e*****y*****_*****t*****y*****p*****e*****:***** *****"*****s*****t*****r*****i*****n*****g*****"*****,***** *****v*****a*****l*****u*****e*****_*****t*****y*****p*****e*****:***** *****"*****s*****t*****r*****i*****n*****g*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** *****e*****n*****d***** +***** +***** ***** *****#***** *****T*****A*****B*****L*****E*****:***** *****e*****v*****e*****n*****t*****s*****_*****r*****a*****w*****_*****q*****u*****e*****u*****e***** +***** ***** *****#***** *****S*****Q*****L*****:***** *****C*****R*****E*****A*****T*****E***** *****T*****A*****B*****L*****E***** *****d*****e*****f*****a*****u*****l*****t*****.*****e*****v*****e*****n*****t*****s*****_*****r*****a*****w*****_*****q*****u*****e*****u*****e***** *****(***** *****`*****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****e*****x*****t*****e*****r*****n*****a*****l*****_*****c*****u*****s*****t*****o*****m*****e*****r*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****t*****r*****a*****n*****s*****a*****c*****t*****i*****o*****n*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****t*****i*****m*****e*****s*****t*****a*****m*****p*****`***** *****D*****a*****t*****e*****T*****i*****m*****e*****,***** *****`*****c*****o*****d*****e*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****p*****r*****o*****p*****e*****r*****t*****i*****e*****s*****`***** *****S*****t*****r*****i*****n*****g***** *****)***** *****E*****N*****G*****I*****N*****E***** *****=***** *****K*****a*****f*****k*****a***** *****S*****E*****T*****T*****I*****N*****G*****S***** *****k*****a*****f*****k*****a*****_*****b*****r*****o*****k*****e*****r*****_*****l*****i*****s*****t***** *****=***** *****'***********************************'*****,***** *****k*****a*****f*****k*****a*****_*****t*****o*****p*****i*****c*****_*****l*****i*****s*****t***** *****=***** *****'*****e*****v*****e*****n*****t*****s*****-*****r*****a*****w*****'*****,***** *****k*****a*****f*****k*****a*****_*****g*****r*****o*****u*****p*****_*****n*****a*****m*****e***** *****=***** *****'*****c*****l*****i*****c*****k*****h*****o*****u*****s*****e*****'*****,***** *****k*****a*****f*****k*****a*****_*****f*****o*****r*****m*****a*****t***** *****=***** *****'*****J*****S*****O*****N*****E*****a*****c*****h*****R*****o*****w*****'***** +***** ***** *****c*****r*****e*****a*****t*****e*****_*****t*****a*****b*****l*****e***** *****"*****e*****v*****e*****n*****t*****s*****_*****r*****a*****w*****_*****q*****u*****e*****u*****e*****"*****,***** *****i*****d*****:***** *****f*****a*****l*****s*****e*****,***** *****o*****p*****t*****i*****o*****n*****s*****:***** *****"*****K*****a*****f*****k*****a***** *****S*****E*****T*****T*****I*****N*****G*****S***** *****k*****a*****f*****k*****a*****_*****b*****r*****o*****k*****e*****r*****_*****l*****i*****s*****t***** *****=***** *****'***********************************'*****,***** *****k*****a*****f*****k*****a*****_*****t*****o*****p*****i*****c*****_*****l*****i*****s*****t***** *****=***** *****'*****e*****v*****e*****n*****t*****s*****-*****r*****a*****w*****'*****,***** *****k*****a*****f*****k*****a*****_*****g*****r*****o*****u*****p*****_*****n*****a*****m*****e***** *****=***** *****'*****c*****l*****i*****c*****k*****h*****o*****u*****s*****e*****'*****,***** *****k*****a*****f*****k*****a*****_*****f*****o*****r*****m*****a*****t***** *****=***** *****'*****J*****S*****O*****N*****E*****a*****c*****h*****R*****o*****w*****'*****"*****,***** *****f*****o*****r*****c*****e*****:***** *****:*****c*****a*****s*****c*****a*****d*****e***** *****d*****o***** *****|*****t*****|***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****e*****x*****t*****e*****r*****n*****a*****l*****_*****c*****u*****s*****t*****o*****m*****e*****r*****_*****i*****d*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****t*****r*****a*****n*****s*****a*****c*****t*****i*****o*****n*****_*****i*****d*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****d*****a*****t*****e*****t*****i*****m*****e***** *****"*****t*****i*****m*****e*****s*****t*****a*****m*****p*****"*****,***** *****p*****r*****e*****c*****i*****s*****i*****o*****n*****:***** *****n*****i*****l*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****c*****o*****d*****e*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** ***** ***** *****t*****.*****s*****t*****r*****i*****n*****g***** *****"*****p*****r*****o*****p*****e*****r*****t*****i*****e*****s*****"*****,***** *****n*****u*****l*****l*****:***** *****f*****a*****l*****s*****e***** +***** ***** *****e*****n*****d***** +***** +***** ***** *****#***** *****T*****A*****B*****L*****E*****:***** *****e*****v*****e*****n*****t*****s*****_*****r*****a*****w*****_*****m*****v***** +***** ***** *****#***** *****S*****Q*****L*****:***** *****C*****R*****E*****A*****T*****E***** *****M*****A*****T*****E*****R*****I*****A*****L*****I*****Z*****E*****D***** *****V*****I*****E*****W***** *****d*****e*****f*****a*****u*****l*****t*****.*****e*****v*****e*****n*****t*****s*****_*****r*****a*****w*****_*****m*****v***** *****T*****O***** *****d*****e*****f*****a*****u*****l*****t*****.*****e*****v*****e*****n*****t*****s*****_*****r*****a*****w***** *****(***** *****`*****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****e*****x*****t*****e*****r*****n*****a*****l*****_*****c*****u*****s*****t*****o*****m*****e*****r*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****t*****r*****a*****n*****s*****a*****c*****t*****i*****o*****n*****_*****i*****d*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****t*****i*****m*****e*****s*****t*****a*****m*****p*****`***** *****D*****a*****t*****e*****T*****i*****m*****e*****,***** *****`*****c*****o*****d*****e*****`***** *****S*****t*****r*****i*****n*****g*****,***** *****`*****p*****r*****o*****p*****e*****r*****t*****i*****e*****s*****`***** *****M*****a*****p*****(*****S*****t*****r*****i*****n*****g*****,***** *****S*****t*****r*****i*****n*****g*****)***** *****)***** *****A*****S***** *****S*****E*****L*****E*****C*****T***** *****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****,***** *****e*****x*****t*****e*****r*****n*****a*****l*****_*****c*****u*****s*****t*****o*****m*****e*****r*****_*****i*****d*****,***** *****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****,***** *****t*****r*****a*****n*****s*****a*****c*****t*****i*****o*****n*****_*****i*****d*****,***** *****t*****i*****m*****e*****s*****t*****a*****m*****p*****,***** *****c*****o*****d*****e*****,***** *****C*****A*****S*****T*****(*****J*****S*****O*****N*****E*****x*****t*****r*****a*****c*****t*****K*****e*****y*****s*****A*****n*****d*****V*****a*****l*****u*****e*****s*****R*****a*****w*****(*****p*****r*****o*****p*****e*****r*****t*****i*****e*****s*****)*****,***** *****'*****M*****a*****p*****(*****S*****t*****r*****i*****n*****g*****,***** *****S*****t*****r*****i*****n*****g*****)*****'*****)***** *****A*****S***** *****p*****r*****o*****p*****e*****r*****t*****i*****e*****s***** *****F*****R*****O*****M***** *****d*****e*****f*****a*****u*****l*****t*****.*****e*****v*****e*****n*****t*****s*****_*****r*****a*****w*****_*****q*****u*****e*****u*****e***** +***** ***** *****c*****r*****e*****a*****t*****e*****_*****t*****a*****b*****l*****e***** *****"*****e*****v*****e*****n*****t*****s*****_*****r*****a*****w*****_*****m*****v*****"*****,***** *****v*****i*****e*****w*****:***** *****t*****r*****u*****e*****,***** *****m*****a*****t*****e*****r*****i*****a*****l*****i*****z*****e*****d*****:***** *****t*****r*****u*****e*****,***** *****i*****d*****:***** *****f*****a*****l*****s*****e*****,***** *****a*****s*****:***** *****"*****S*****E*****L*****E*****C*****T***** *****o*****r*****g*****a*****n*****i*****z*****a*****t*****i*****o*****n*****_*****i*****d*****,***** *****e*****x*****t*****e*****r*****n*****a*****l*****_*****c*****u*****s*****t*****o*****m*****e*****r*****_*****i*****d*****,***** *****e*****x*****t*****e*****r*****n*****a*****l*****_*****s*****u*****b*****s*****c*****r*****i*****p*****t*****i*****o*****n*****_*****i*****d*****,***** *****t*****r*****a*****n*****s*****a*****c*****t*****i*****o*****n*****_*****i*****d*****,***** *****t*****i*****m*****e*****s*****t*****a*****m*****p*****,***** *****c*****o*****d*****e*****,***** *****C*****A*****S*****T*****(*****J*****S*****O*****N*****E*****x*****t*****r*****a*****c*****t*****K*****e*****y*****s*****A*****n*****d*****V*****a*****l*****u*****e*****s*****R*****a*****w*****(*****p*****r*****o*****p*****e*****r*****t*****i*****e*****s*****)*****,***** *****'*****M*****a*****p*****(*****S*****t*****r*****i*****n*****g*****,***** *****S*****t*****r*****i*****n*****g*****)*****'*****)***** *****A*****S***** *****p*****r*****o*****p*****e*****r*****t*****i*****e*****s***** *****F*****R*****O*****M***** *****d*****e*****f*****a*****u*****l*****t*****.*****e*****v*****e*****n*****t*****s*****_*****r*****a*****w*****_*****q*****u*****e*****u*****e*****"*****,***** *****f*****o*****r*****c*****e*****:***** *****:*****c*****a*****s*****c*****a*****d*****e***** *****d*****o***** *****|*****t*****|***** +***** ***** *****e*****n*****d***** +***** +*****e*****n*****d***** +***** diff --git a/db/migrate/20231205153156_remove_events_quantified_events_relation.rb b/db/migrate/20231205153156_remove_events_quantified_events_relation.rb new file mode 100644 index 00000000000..43de7b668c5 --- /dev/null +++ b/db/migrate/20231205153156_remove_events_quantified_events_relation.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class RemoveEventsQuantifiedEventsRelation < ActiveRecord::Migration[7.0] + def up + remove_column :events, :quantified_event_id + end + + def down + add_column :events, :quantified_event_id, :uuid + add_index :events, :quantified_event_id + end +end diff --git a/db/schema.rb b/db/schema.rb index 9384a42c8e0..e5de7628e90 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -355,7 +355,6 @@ t.jsonb "metadata", default: {}, null: false t.uuid "subscription_id" t.datetime "deleted_at" - t.uuid "quantified_event_id" t.string "external_customer_id" t.string "external_subscription_id" t.index ["customer_id"], name: "index_events_on_customer_id" @@ -366,7 +365,6 @@ t.index ["organization_id", "external_subscription_id", "transaction_id"], name: "index_unique_transaction_id", unique: true t.index ["organization_id"], name: "index_events_on_organization_id" t.index ["properties"], name: "index_events_on_properties", opclass: :jsonb_path_ops, using: :gin - t.index ["quantified_event_id"], name: "index_events_on_quantified_event_id" t.index ["subscription_id", "code", "timestamp"], name: "index_events_on_subscription_id_and_code_and_timestamp", where: "(deleted_at IS NULL)" t.index ["subscription_id"], name: "index_events_on_subscription_id" end diff --git a/spec/services/billable_metrics/aggregations/unique_count_service_spec.rb b/spec/services/billable_metrics/aggregations/unique_count_service_spec.rb index 1f63381828a..13e4a74e89c 100644 --- a/spec/services/billable_metrics/aggregations/unique_count_service_spec.rb +++ b/spec/services/billable_metrics/aggregations/unique_count_service_spec.rb @@ -60,16 +60,18 @@ let(:unique_count_event) do create( :event, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: added_at, - quantified_event:, + properties: { unique_id: quantified_event.external_id }, ) end let(:quantified_event) do create( :quantified_event, + organization:, added_at:, removed_at:, external_subscription_id: subscription.external_id, @@ -86,6 +88,7 @@ let(:new_quantified_event) do create( :quantified_event, + organization:, added_at: from_datetime + 10.days, removed_at:, external_subscription_id: subscription.external_id, @@ -95,11 +98,12 @@ let(:new_unique_count_event) do create( :event, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 10.days, - quantified_event: new_quantified_event, + properties: { unique_id: new_quantified_event.external_id }, ) end @@ -123,16 +127,18 @@ let(:new_unique_count_event) do create( :event, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 10.days, - quantified_event: new_quantified_event, + properties: { unique_id: new_quantified_event.external_id }, ) end let(:new_quantified_event) do create( :quantified_event, + organization:, added_at: from_datetime + 10.days, removed_at:, external_subscription_id: subscription.external_id, @@ -275,14 +281,13 @@ let(:previous_event) do create( :event, - organization:, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 5.days, - quantified_event: previous_quantified_event, properties: { - unique_id: '000', + unique_id: previous_quantified_event.external_id, }, ) end @@ -335,21 +340,22 @@ end context 'when event is given' do - let(:properties) { { unique_id: '111' } } + let(:properties) { { unique_id: new_quantified_event.external_id } } let(:pay_in_advance_event) do create( :event, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 10.days, properties:, - quantified_event: new_quantified_event, ) end let(:new_quantified_event) do create( :quantified_event, + organization:, added_at: from_datetime + 10.days, removed_at:, external_subscription_id: subscription.external_id, @@ -393,13 +399,13 @@ let(:previous_event) do create( :event, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 5.days, - quantified_event: previous_quantified_event, properties: { - unique_id: '000', + unique_id: previous_quantified_event.external_id, }, ) end @@ -407,6 +413,7 @@ let(:previous_quantified_event) do create( :quantified_event, + organization:, added_at: from_datetime + 5.days, removed_at:, external_id: '000', @@ -446,9 +453,8 @@ external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 5.days, - quantified_event: previous_quantified_event, properties: { - unique_id: '000', + unique_id: previous_quantified_event.external_id, }, ) end diff --git a/spec/services/billable_metrics/prorated_aggregations/unique_count_service_spec.rb b/spec/services/billable_metrics/prorated_aggregations/unique_count_service_spec.rb index 384a28ea5dc..80d975f5a66 100644 --- a/spec/services/billable_metrics/prorated_aggregations/unique_count_service_spec.rb +++ b/spec/services/billable_metrics/prorated_aggregations/unique_count_service_spec.rb @@ -263,13 +263,13 @@ let(:previous_event) do create( :event, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 5.days, - quantified_event: previous_quantified_event, properties: { - unique_id: '000', + unique_id: previous_quantified_event.external_id, }, ) end @@ -318,16 +318,16 @@ end context 'when event is given' do - let(:properties) { { unique_id: '111' } } + let(:properties) { { unique_id: new_quantified_event.external_id } } let(:pay_in_advance_event) do create( :event, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 10.days, properties:, - quantified_event: new_quantified_event, ) end @@ -360,24 +360,18 @@ let(:previous_event) do create( :event, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 5.days, - quantified_event: previous_quantified_event, - properties: { - unique_id: '000', - }, - metadata: { - current_aggregation: '7', - max_aggregation: '7', - max_aggregation_with_proration: '5.8', - }, + properties: { unique_id: previous_quantified_event.external_id }, ) end let(:previous_quantified_event) do create( :quantified_event, + organization:, added_at: from_datetime + 5.days, removed_at:, external_id: '000', @@ -397,13 +391,13 @@ let(:previous_event) do create( :event, + organization_id: organization.id, code: billable_metric.code, external_customer_id: customer.external_id, external_subscription_id: subscription.external_id, timestamp: from_datetime + 5.days, - quantified_event: previous_quantified_event, properties: { - unique_id: '000', + unique_id: previous_quantified_event.external_id, }, ) end @@ -411,6 +405,7 @@ let(:previous_quantified_event) do create( :quantified_event, + organization:, added_at: from_datetime + 5.days, removed_at:, external_id: '000', diff --git a/spec/services/events/post_process_service_spec.rb b/spec/services/events/post_process_service_spec.rb index c516eba7987..f961755f267 100644 --- a/spec/services/events/post_process_service_spec.rb +++ b/spec/services/events/post_process_service_spec.rb @@ -21,7 +21,7 @@ let(:event) do create( :event, - organization:, + organization_id: organization.id, external_customer_id:, external_subscription_id:, timestamp:, @@ -108,7 +108,7 @@ } end - it 'creates an association with a quantified event' do + it 'creates a quantified event' do result = nil aggregate_failures do @@ -116,7 +116,6 @@ .to change(QuantifiedEvent, :count).by(1) expect(result).to be_success - expect(event.reload.quantified_event).to be_present end end end diff --git a/spec/services/fees/charge_service_spec.rb b/spec/services/fees/charge_service_spec.rb index a23d3ca417f..8415f0b70be 100644 --- a/spec/services/fees/charge_service_spec.rb +++ b/spec/services/fees/charge_service_spec.rb @@ -421,67 +421,73 @@ let(:event1) do create( :event, + organization_id: organization.id, code: charge.billable_metric.code, - customer: subscription.customer, - subscription:, + external_customer_id: subscription.customer.external_id, + external_subscription_id: subscription.external_id, timestamp: DateTime.parse('2022-03-16'), - quantified_event: quantified_event1, - properties: { region: 'usa', foo_bar: 12 }, + properties: { region: 'usa', foo_bar: quantified_event1.external_id }, ) end let(:quantified_event1) do create( :quantified_event, + organization_id: organization.id, added_at: DateTime.parse('2022-03-16'), removed_at: nil, external_id: '12', external_subscription_id: subscription.external_id, billable_metric: charge.billable_metric, properties: { region: 'usa', foo_bar: 12 }, + group: usa, ) end let(:event2) do create( :event, + organization_id: organization.id, code: charge.billable_metric.code, - customer: subscription.customer, - subscription:, + external_customer_id: subscription.customer.external_id, + external_subscription_id: subscription.external_id, timestamp: DateTime.parse('2022-03-16'), - quantified_event: quantified_event2, - properties: { region: 'europe', foo_bar: 10 }, + properties: { region: 'europe', foo_bar: quantified_event2.external_id }, ) end let(:quantified_event2) do create( :quantified_event, + organization_id: organization.id, added_at: DateTime.parse('2022-03-16'), removed_at: nil, external_id: '10', external_subscription_id: subscription.external_id, billable_metric: charge.billable_metric, properties: { region: 'europe', foo_bar: 10 }, + group: europe, ) end let(:event3) do create( :event, + organization_id: organization.id, code: charge.billable_metric.code, - customer: subscription.customer, - subscription:, + external_customer_id: subscription.customer.external_id, + external_subscription_id: subscription.external_id, timestamp: DateTime.parse('2022-03-16'), - quantified_event: quantified_event3, - properties: { country: 'france', foo_bar: 5 }, + properties: { country: 'france', foo_bar: quantified_event3.external_id }, ) end let(:quantified_event3) do create( :quantified_event, + organization_id: organization.id, added_at: DateTime.parse('2022-03-16'), removed_at: nil, external_id: '5', external_subscription_id: subscription.external_id, billable_metric: charge.billable_metric, properties: { country: 'france', foo_bar: 5 }, + group: france, ) end