Skip to content

Commit

Permalink
refactor: Drop JSON debugging columns from Kingdom Spanner DB
Browse files Browse the repository at this point in the history
  • Loading branch information
SanjayVas committed Sep 13, 2024
1 parent 7876bc3 commit 7d68a80
Show file tree
Hide file tree
Showing 28 changed files with 64 additions and 52 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ class DataProviderReader : SpannerReader<DataProviderReader.Result>() {
DataProviders.DataProviderId,
DataProviders.ExternalDataProviderId,
DataProviders.DataProviderDetails,
DataProviders.DataProviderDetailsJson,
DataProviderCertificates.ExternalDataProviderCertificateId,
Certificates.CertificateId,
Certificates.SubjectKeyIdentifier,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,6 @@ class ExchangeReader : SpannerReader<ExchangeReader.Result>() {
"Exchanges.Date",
"Exchanges.State",
"Exchanges.ExchangeDetails",
"Exchanges.ExchangeDetailsJson",
"RecurringExchanges.ExternalRecurringExchangeId",
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ class ExchangeStepAttemptReader : SpannerReader<ExchangeStepAttemptReader.Result
ExchangeStepAttempts.AttemptIndex,
ExchangeStepAttempts.State,
ExchangeStepAttempts.ExchangeStepAttemptDetails,
ExchangeStepAttempts.ExchangeStepAttemptDetailsJson,
RecurringExchanges.ExternalRecurringExchangeId
FROM ExchangeStepAttempts
JOIN RecurringExchanges USING (RecurringExchangeId)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ class MeasurementConsumerReader : SpannerReader<MeasurementConsumerReader.Result
MeasurementConsumers.MeasurementConsumerId,
MeasurementConsumers.ExternalMeasurementConsumerId,
MeasurementConsumers.MeasurementConsumerDetails,
MeasurementConsumers.MeasurementConsumerDetailsJson,
MeasurementConsumerCertificates.ExternalMeasurementConsumerCertificateId,
Certificates.SubjectKeyIdentifier,
Certificates.NotValidBefore,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ class RecurringExchangeReader(recurringExchangesIndex: Index = Index.NONE) :
"RecurringExchanges.State",
"RecurringExchanges.NextExchangeDate",
"RecurringExchanges.RecurringExchangeDetails",
"RecurringExchanges.RecurringExchangeDetailsJson",
"DataProviders.ExternalDataProviderId",
"ModelProviders.ExternalModelProviderId",
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import org.wfanet.measurement.gcloud.common.toCloudDate
import org.wfanet.measurement.gcloud.common.toGcloudTimestamp
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.gcloud.spanner.statement
import org.wfanet.measurement.internal.kingdom.ClaimReadyExchangeStepRequest
import org.wfanet.measurement.internal.kingdom.ExchangeStep
Expand Down Expand Up @@ -128,7 +127,6 @@ class ClaimReadyExchangeStep(
set("ExpirationTime" to (now + DEFAULT_EXPIRATION_DURATION).toGcloudTimestamp())

set("ExchangeStepAttemptDetails" to details)
setJson("ExchangeStepAttemptDetailsJson" to details)
}

return attemptIndex
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import org.wfanet.measurement.gcloud.spanner.AsyncDatabaseClient
import org.wfanet.measurement.gcloud.spanner.bufferTo
import org.wfanet.measurement.gcloud.spanner.insertMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.Certificate
import org.wfanet.measurement.kingdom.deploy.common.DuchyIds
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.CertSubjectKeyIdAlreadyExistsException
Expand Down Expand Up @@ -144,6 +143,5 @@ fun Certificate.toInsertMutation(internalId: InternalId): Mutation {
set("NotValidAfter" to notValidAfter.toGcloudTimestamp())
set("RevocationState" to revocationState)
set("CertificateDetails" to details)
setJson("CertificateDetailsJson" to details)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import org.wfanet.measurement.common.identity.InternalId
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.bufferTo
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.DataProvider
import org.wfanet.measurement.internal.kingdom.copy
import org.wfanet.measurement.kingdom.deploy.common.DuchyIds
Expand All @@ -39,7 +38,6 @@ class CreateDataProvider(private val dataProvider: DataProvider) :
set("PublicKeyCertificateId" to internalCertificateId)
set("ExternalDataProviderId" to externalDataProviderId)
set("DataProviderDetails" to dataProvider.details)
setJson("DataProviderDetailsJson" to dataProvider.details)
}

for (externalDuchyId in dataProvider.requiredExternalDuchyIdsList) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.common.identity.InternalId
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.CreateEventGroupRequest
import org.wfanet.measurement.internal.kingdom.EventGroup
import org.wfanet.measurement.internal.kingdom.copy
Expand Down Expand Up @@ -83,7 +82,6 @@ class CreateEventGroup(private val request: CreateEventGroupRequest) :
set("UpdateTime" to Value.COMMIT_TIMESTAMP)
if (request.eventGroup.hasDetails()) {
set("EventGroupDetails" to request.eventGroup.details)
setJson("EventGroupDetailsJson" to request.eventGroup.details)
}
set("State" to EventGroup.State.ACTIVE)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.common.identity.InternalId
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.EventGroupMetadataDescriptor
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.DataProviderNotFoundException
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorAlreadyExistsWithTypeException
Expand Down Expand Up @@ -85,7 +84,6 @@ class CreateEventGroupMetadataDescriptor(
}

set("DescriptorDetails" to eventGroupMetadataDescriptor.details)
setJson("DescriptorDetailsJson" to eventGroupMetadataDescriptor.details)
}

for (protobufTypeName in protobufTypeNames) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.gcloud.common.toCloudDate
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.Exchange
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.RecurringExchangeNotFoundException
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.RecurringExchangeReader
Expand All @@ -45,7 +44,6 @@ class CreateExchange(private val exchange: Exchange) : SimpleSpannerWriter<Excha
set("Date" to exchange.date.toCloudDate())
set("State" to INITIAL_STATE)
set("ExchangeDetails" to exchange.details)
setJson("ExchangeDetailsJson" to exchange.details)
}

return exchange.toBuilder().apply { state = INITIAL_STATE }.build()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import org.wfanet.measurement.gcloud.spanner.bind
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.Exchange
import org.wfanet.measurement.internal.kingdom.ExchangeDetails
import org.wfanet.measurement.internal.kingdom.ExchangeStep
Expand Down Expand Up @@ -129,7 +128,6 @@ class CreateExchangesAndSteps(
set("Date" to date.toCloudDate())
set("State" to Exchange.State.ACTIVE)
set("ExchangeDetails" to exchangeDetails)
setJson("ExchangeDetailsJson" to exchangeDetails)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.bufferTo
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.Account
import org.wfanet.measurement.internal.kingdom.MeasurementConsumer
import org.wfanet.measurement.internal.kingdom.copy
Expand Down Expand Up @@ -78,7 +77,6 @@ class CreateMeasurementConsumer(
set("PublicKeyCertificateId" to internalCertificateId)
set("ExternalMeasurementConsumerId" to externalMeasurementConsumerId)
set("MeasurementConsumerDetails" to measurementConsumer.details)
setJson("MeasurementConsumerDetailsJson" to measurementConsumer.details)
}

val externalMeasurementConsumerCertificateId = idGenerator.generateExternalId()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import org.wfanet.measurement.gcloud.spanner.appendClause
import org.wfanet.measurement.gcloud.spanner.bind
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.ComputationParticipant
import org.wfanet.measurement.internal.kingdom.ComputationParticipantDetails
import org.wfanet.measurement.internal.kingdom.CreateMeasurementRequest
Expand Down Expand Up @@ -330,7 +329,6 @@ class CreateMeasurements(private val requests: List<CreateMeasurementRequest>) :
set("CertificateId" to measurementConsumerCertificateId)
set("State" to initialMeasurementState)
set("MeasurementDetails" to createMeasurementRequest.measurement.details)
setJson("MeasurementDetailsJson" to createMeasurementRequest.measurement.details)
set("CreateTime" to Value.COMMIT_TIMESTAMP)
set("UpdateTime" to Value.COMMIT_TIMESTAMP)
}
Expand All @@ -349,7 +347,6 @@ class CreateMeasurements(private val requests: List<CreateMeasurementRequest>) :
set("UpdateTime" to Value.COMMIT_TIMESTAMP)
set("State" to ComputationParticipant.State.CREATED)
set("ParticipantDetails" to participantDetails)
setJson("ParticipantDetailsJson" to participantDetails)
}
}

Expand Down Expand Up @@ -417,7 +414,6 @@ class CreateMeasurements(private val requests: List<CreateMeasurementRequest>) :
set("State" to initialRequisitionState)
fulfillingDuchyId?.let { set("FulfillingDuchyId" to it) }
set("RequisitionDetails" to details)
setJson("RequisitionDetailsJson" to details)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.gcloud.common.toCloudDate
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.RecurringExchange
import org.wfanet.measurement.internal.kingdom.copy
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.DataProviderNotFoundException
Expand Down Expand Up @@ -59,7 +58,6 @@ class CreateRecurringExchange(private val recurringExchange: RecurringExchange)
set("State" to INITIAL_STATE)
set("NextExchangeDate" to recurringExchange.nextExchangeDate.toCloudDate())
set("RecurringExchangeDetails" to recurringExchange.details)
setJson("RecurringExchangeDetailsJson" to recurringExchange.details)
}

return recurringExchange.copy {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import com.google.cloud.spanner.Value
import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.DeleteEventGroupRequest
import org.wfanet.measurement.internal.kingdom.EventGroup
import org.wfanet.measurement.internal.kingdom.EventGroupDetails
Expand Down Expand Up @@ -60,7 +59,6 @@ class DeleteEventGroup(private val request: DeleteEventGroupRequest) :
set("MeasurementConsumerCertificateId" to null as Long?)
set("UpdateTime" to Value.COMMIT_TIMESTAMP)
set("EventGroupDetails" to null as EventGroupDetails?)
setJson("EventGroupDetailsJson" to null as EventGroupDetails?)
set("State" to EventGroup.State.DELETED)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,7 @@ import org.wfanet.measurement.gcloud.common.toCloudDate
import org.wfanet.measurement.gcloud.spanner.appendClause
import org.wfanet.measurement.gcloud.spanner.bind
import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation
import org.wfanet.measurement.gcloud.spanner.getProtoEnum
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.gcloud.spanner.statement
import org.wfanet.measurement.internal.kingdom.Exchange
import org.wfanet.measurement.internal.kingdom.ExchangeStep
Expand Down Expand Up @@ -233,7 +231,6 @@ class FinishExchangeStepAttempt(
set("AttemptIndex" to attemptNumber.toLong())
set("State" to state)
set("ExchangeStepAttemptDetails" to details)
setJson("ExchangeStepAttemptDetailsJson" to details)
}

return exchangeStepAttempt.copy {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.common.identity.InternalId
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.DuchyMeasurementLogEntryDetails
import org.wfanet.measurement.internal.kingdom.Measurement
import org.wfanet.measurement.internal.kingdom.MeasurementLogEntryDetails
Expand All @@ -37,7 +36,6 @@ internal fun SpannerWriter.TransactionScope.insertMeasurementLogEntry(
set("MeasurementId" to measurementId)
set("CreateTime" to Value.COMMIT_TIMESTAMP)
set("MeasurementLogDetails" to logDetails)
setJson("MeasurementLogDetailsJson" to logDetails)
}
}

Expand Down Expand Up @@ -74,7 +72,6 @@ internal fun SpannerWriter.TransactionScope.insertDuchyMeasurementLogEntry(
set("DuchyId" to duchyId)
set("ExternalComputationLogEntryId" to externalComputationLogEntryId)
set("DuchyMeasurementLogDetails" to logDetails)
setJson("DuchyMeasurementLogDetailsJson" to logDetails)
}

return externalComputationLogEntryId
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner.writers
import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.DataProvider
import org.wfanet.measurement.internal.kingdom.ReplaceDataAvailabilityIntervalRequest
import org.wfanet.measurement.internal.kingdom.copy
Expand All @@ -43,7 +42,6 @@ class ReplaceDataAvailabilityInterval(private val request: ReplaceDataAvailabili
transactionContext.bufferUpdateMutation("DataProviders") {
set("DataProviderId" to dataProviderId)
set("DataProviderDetails" to updatedDetails)
setJson("DataProviderDetailsJson" to updatedDetails)
}

return dataProvider.copy { details = updatedDetails }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner.writers
import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.DataProvider
import org.wfanet.measurement.internal.kingdom.DataProviderDetails
import org.wfanet.measurement.internal.kingdom.ReplaceDataProviderCapabilitiesRequest
Expand All @@ -41,7 +40,6 @@ class ReplaceDataProviderCapabilities(private val request: ReplaceDataProviderCa
transactionContext.bufferUpdateMutation("DataProviders") {
set("DataProviderId" to dataProviderResult.dataProviderId)
set("DataProviderDetails" to updatedDetails)
setJson("DataProviderDetailsJson" to updatedDetails)
}

return dataProviderResult.dataProvider.copy { details = updatedDetails }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import org.wfanet.measurement.gcloud.spanner.bind
import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation
import org.wfanet.measurement.gcloud.spanner.getProtoMessage
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.gcloud.spanner.statement
import org.wfanet.measurement.internal.kingdom.ComputationParticipant
import org.wfanet.measurement.internal.kingdom.ComputationParticipantDetails
Expand Down Expand Up @@ -171,7 +170,6 @@ class SetParticipantRequisitionParams(private val request: SetParticipantRequisi
set("UpdateTime" to Value.COMMIT_TIMESTAMP)
set("State" to nextState)
set("ParticipantDetails" to participantDetails)
setJson("ParticipantDetailsJson" to participantDetails)
}

val otherComputationParticipants: List<ComputationParticipantResult> =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import com.google.cloud.spanner.Value
import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.EventGroup
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupInvalidArgsException
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupNotFoundException
Expand Down Expand Up @@ -70,7 +69,6 @@ class UpdateEventGroup(private val eventGroup: EventGroup) :
set("ProvidedEventGroupId" to providedEventGroupId)
set("UpdateTime" to Value.COMMIT_TIMESTAMP)
set("EventGroupDetails" to eventGroup.details)
setJson("EventGroupDetailsJson" to eventGroup.details)
}

return eventGroup
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import org.wfanet.measurement.common.identity.ExternalId
import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation
import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.internal.kingdom.EventGroupMetadataDescriptor
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorAlreadyExistsWithTypeException
import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorNotFoundException
Expand Down Expand Up @@ -69,7 +68,6 @@ class UpdateEventGroupMetadataDescriptor(
)

set("DescriptorDetails" to eventGroupMetadataDescriptor.details)
setJson("DescriptorDetailsJson" to eventGroupMetadataDescriptor.details)
}

transactionContext.buffer(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import com.google.cloud.spanner.Value
import org.wfanet.measurement.common.identity.InternalId
import org.wfanet.measurement.gcloud.spanner.bufferTo
import org.wfanet.measurement.gcloud.spanner.set
import org.wfanet.measurement.gcloud.spanner.setJson
import org.wfanet.measurement.gcloud.spanner.updateMutation
import org.wfanet.measurement.internal.kingdom.Measurement
import org.wfanet.measurement.internal.kingdom.MeasurementDetails
Expand All @@ -40,7 +39,6 @@ internal fun SpannerWriter.TransactionScope.updateMeasurementState(
set("UpdateTime" to Value.COMMIT_TIMESTAMP)
if (details != null) {
set("MeasurementDetails" to details)
setJson("MeasurementDetailsJson" to details)
}
}
.bufferTo(transactionContext)
Expand Down
Loading

0 comments on commit 7d68a80

Please sign in to comment.