[batch] Add deduped id column to resources table

batch/batch/driver/billing_manager.py

@@ -113,6 +113,15 @@ async def _refresh_resources_from_retail_prices(self, prices: List[Price]):
         @transaction(self.db)
         async def insert_or_update(tx):
             if resource_updates:
+                last_resource_id = await tx.execute_and_fetchone(
+                    '''
+SELECT COALESCE(MAX(resource_id), 0) AS last_resource_id
+FROM resources
+FOR UPDATE
+'''
+                )
+                last_resource_id = last_resource_id['last_resource_id']
+
                 await tx.execute_many(
                     '''
 INSERT INTO `resources` (resource, rate)
@@ -121,6 +130,15 @@ async def insert_or_update(tx):
                     resource_updates,
                 )
 
+                await tx.execute_update(
+                    '''
+UPDATE resources
+SET deduped_resource_id = resource_id
+WHERE resource_id > %s AND deduped_resource_id IS NULL
+''',
+                    (last_resource_id,),
+                )
+
             if product_version_updates:
                 await tx.execute_many(
                     '''

batch/sql/add-deduped-resource-ids-to-resources.sql

@@ -0,0 +1,3 @@
+ALTER TABLE resources ADD COLUMN deduped_resource_id INT DEFAULT NULL, ALGORITHM=INSTANT;
+CREATE INDEX `resources_deduped_resource_id` ON `resources` (`deduped_resource_id`);
+

batch/sql/estimated-current.sql

@@ -9,8 +9,10 @@ CREATE TABLE IF NOT EXISTS `resources` (
   `resource` VARCHAR(100) NOT NULL,
   `rate` DOUBLE NOT NULL,
   `resource_id` INT AUTO_INCREMENT UNIQUE NOT NULL,
+  `deduped_resource_id` INT DEFAULT NULL,
   PRIMARY KEY (`resource`)
 ) ENGINE = InnoDB;
+CREATE INDEX `resources_deduped_resource_id` ON `resources` (`deduped_resource_id`);
 
 CREATE TABLE IF NOT EXISTS `latest_product_versions` (
   `product` VARCHAR(100) NOT NULL,

build.yaml

@@ -2036,6 +2036,9 @@ steps:
       - name: cleanup-old-billing-tables
         script: /io/sql/cleanup-old-billing-tables.sql
         online: true
+      - name: add-deduped-resource-ids-to-resources
+        script: /io/sql/add-deduped-resource-ids-to-resources.sql
+        online: true
     inputs:
       - from: /repo/batch/sql
         to: /io/sql