From e6988496fd2be7410c32eaa8a14a22055e1686b7 Mon Sep 17 00:00:00 2001 From: Stephen Foulkes Date: Wed, 23 Jan 2013 09:22:35 -0600 Subject: [PATCH] Fix up the DBS3Uploader and modify it to support an upload only mode. Kill off SQLite in DBS3Buffer and DBSBuffer. --- .../WMComponent/DBS3Buffer/DBSBufferBlock.py | 49 +++--- .../WMComponent/DBS3Buffer/DBSBufferUtil.py | 23 +-- .../WMComponent/DBS3Buffer/DBSUploadPoller.py | 141 ++++++++---------- .../WMComponent/DBS3Buffer/MySQL/Create.py | 13 +- .../DBS3Buffer/MySQL/GetOpenBlocks.py | 21 ++- .../DBS3Buffer/MySQL/LoadBlocks.py | 31 ++-- .../DBS3Buffer/MySQL/UpdateBlocks.py | 19 ++- .../WMComponent/DBS3Buffer/Oracle/Create.py | 3 +- .../DBS3Buffer/Oracle/LoadBlocks.py | 13 ++ .../DBS3Buffer/Oracle/UpdateBlocks.py | 13 ++ .../DBS3Buffer/SQLite/AlgoDatasetAssoc.py | 35 ----- .../DBS3Buffer/SQLite/CountBlocks.py | 18 --- .../DBS3Buffer/SQLite/CountFiles.py | 18 --- .../WMComponent/DBS3Buffer/SQLite/Create.py | 127 ---------------- .../DBS3Buffer/SQLite/DBSBufferFiles/Add.py | 20 --- .../SQLite/DBSBufferFiles/AddCKType.py | 16 -- .../SQLite/DBSBufferFiles/AddChecksum.py | 22 --- .../SQLite/DBSBufferFiles/AddChecksumByLFN.py | 19 --- .../SQLite/DBSBufferFiles/AddLocation.py | 42 ------ .../SQLite/DBSBufferFiles/AddRunLumi.py | 28 ---- .../DBSBufferFiles/BulkHeritageParent.py | 19 --- .../SQLite/DBSBufferFiles/Delete.py | 24 --- .../SQLite/DBSBufferFiles/Exists.py | 20 --- .../DBSBufferFiles/ExistsForAccountant.py | 18 --- .../SQLite/DBSBufferFiles/GetBlock.py | 19 --- .../SQLite/DBSBufferFiles/GetByID.py | 24 --- .../SQLite/DBSBufferFiles/GetByLFN.py | 24 --- .../SQLite/DBSBufferFiles/GetChecksum.py | 17 --- .../SQLite/DBSBufferFiles/GetChildren.py | 14 -- .../SQLite/DBSBufferFiles/GetLocation.py | 24 --- .../SQLite/DBSBufferFiles/GetParentStatus.py | 14 -- .../SQLite/DBSBufferFiles/GetParents.py | 24 --- .../SQLite/DBSBufferFiles/GetRunLumiFile.py | 24 --- .../SQLite/DBSBufferFiles/HeritageLFNChild.py | 15 -- .../DBSBufferFiles/HeritageLFNParent.py | 15 -- .../DBSBufferFiles/LoadBulkFilesByID.py | 19 --- .../SQLite/DBSBufferFiles/SetBlock.py | 19 --- .../SQLite/DBSBufferFiles/SetLocation.py | 19 --- .../SQLite/DBSBufferFiles/SetLocationByLFN.py | 17 --- .../SQLite/DBSBufferFiles/SetStatus.py | 19 --- .../SQLite/DBSBufferFiles/__init__.py | 9 -- .../WMComponent/DBS3Buffer/SQLite/Destroy.py | 37 ----- .../DBS3Buffer/SQLite/FindDASToUpload.py | 21 --- .../DBS3Buffer/SQLite/GetBlockFromDataset.py | 18 --- .../DBS3Buffer/SQLite/GetOpenBlocks.py | 17 --- .../WMComponent/DBS3Buffer/SQLite/ListAlgo.py | 20 --- .../DBS3Buffer/SQLite/ListAlgoDatasetAssoc.py | 20 --- .../DBS3Buffer/SQLite/ListDataset.py | 20 --- .../DBS3Buffer/SQLite/LoadBlocksByDAS.py | 18 --- .../DBS3Buffer/SQLite/LoadDBSFilesByDAS.py | 20 --- .../WMComponent/DBS3Buffer/SQLite/NewAlgo.py | 30 ---- .../DBS3Buffer/SQLite/NewDataset.py | 22 --- .../DBS3Buffer/SQLite/SetBlockStatus.py | 26 ---- .../WMComponent/DBS3Buffer/SQLite/Status.py | 14 -- .../DBS3Buffer/SQLite/UpdateAlgo.py | 23 --- .../SQLite/UpdateAlgoDatasetAssoc.py | 19 --- .../WMComponent/DBS3Buffer/SQLite/__init__.py | 6 - src/python/WMComponent/DBSBuffer/DBSBuffer.py | 35 ----- .../Database/SQLite/AlgoDatasetAssoc.py | 35 ----- .../DBSBuffer/Database/SQLite/CountBlocks.py | 18 --- .../DBSBuffer/Database/SQLite/CountFiles.py | 18 --- .../DBSBuffer/Database/SQLite/Create.py | 127 ---------------- .../Database/SQLite/DBSBufferFiles/Add.py | 20 --- .../SQLite/DBSBufferFiles/AddCKType.py | 16 -- .../SQLite/DBSBufferFiles/AddChecksum.py | 22 --- .../SQLite/DBSBufferFiles/AddChecksumByLFN.py | 19 --- .../SQLite/DBSBufferFiles/AddLocation.py | 42 ------ .../SQLite/DBSBufferFiles/AddRunLumi.py | 28 ---- .../DBSBufferFiles/BulkHeritageParent.py | 19 --- .../Database/SQLite/DBSBufferFiles/Delete.py | 24 --- .../Database/SQLite/DBSBufferFiles/Exists.py | 20 --- .../DBSBufferFiles/ExistsForAccountant.py | 18 --- .../SQLite/DBSBufferFiles/GetBlock.py | 19 --- .../Database/SQLite/DBSBufferFiles/GetByID.py | 24 --- .../SQLite/DBSBufferFiles/GetByLFN.py | 24 --- .../SQLite/DBSBufferFiles/GetChecksum.py | 17 --- .../SQLite/DBSBufferFiles/GetChildren.py | 14 -- .../SQLite/DBSBufferFiles/GetLocation.py | 24 --- .../SQLite/DBSBufferFiles/GetParentStatus.py | 14 -- .../SQLite/DBSBufferFiles/GetParents.py | 24 --- .../SQLite/DBSBufferFiles/GetRunLumiFile.py | 24 --- .../SQLite/DBSBufferFiles/HeritageLFNChild.py | 15 -- .../DBSBufferFiles/HeritageLFNParent.py | 15 -- .../DBSBufferFiles/LoadBulkFilesByID.py | 19 --- .../SQLite/DBSBufferFiles/SetBlock.py | 19 --- .../SQLite/DBSBufferFiles/SetLocation.py | 19 --- .../SQLite/DBSBufferFiles/SetLocationByLFN.py | 17 --- .../SQLite/DBSBufferFiles/SetStatus.py | 19 --- .../SQLite/DBSBufferFiles/__init__.py | 9 -- .../DBSBuffer/Database/SQLite/Destroy.py | 37 ----- .../Database/SQLite/GetBlockFromDataset.py | 18 --- .../Database/SQLite/GetOpenBlocks.py | 17 --- .../DBSBuffer/Database/SQLite/ListAlgo.py | 20 --- .../Database/SQLite/ListAlgoDatasetAssoc.py | 20 --- .../DBSBuffer/Database/SQLite/ListDataset.py | 20 --- .../DBSBuffer/Database/SQLite/NewAlgo.py | 30 ---- .../DBSBuffer/Database/SQLite/NewDataset.py | 22 --- .../DBSBuffer/Database/SQLite/Status.py | 14 -- .../DBSBuffer/Database/SQLite/UpdateAlgo.py | 23 --- .../Database/SQLite/UpdateAlgoDatasetAssoc.py | 19 --- .../DBSBuffer/Database/SQLite/__init__.py | 6 - .../WMComponent/DBSBuffer/DefaultConfig.py | 27 ---- .../WMComponent_t/DBS3Buffer_t/DBSUpload_t.py | 106 ++++--------- 103 files changed, 201 insertions(+), 2354 deletions(-) create mode 100644 src/python/WMComponent/DBS3Buffer/Oracle/LoadBlocks.py create mode 100644 src/python/WMComponent/DBS3Buffer/Oracle/UpdateBlocks.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/AlgoDatasetAssoc.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/CountBlocks.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/CountFiles.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/Create.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Add.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddCKType.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddChecksum.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddChecksumByLFN.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddLocation.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddRunLumi.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/BulkHeritageParent.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Delete.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Exists.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/ExistsForAccountant.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetBlock.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetByID.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetByLFN.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetChecksum.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetChildren.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetLocation.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetParentStatus.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetParents.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetRunLumiFile.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/HeritageLFNChild.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/HeritageLFNParent.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/LoadBulkFilesByID.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetBlock.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetLocation.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetLocationByLFN.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetStatus.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/__init__.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/Destroy.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/FindDASToUpload.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/GetBlockFromDataset.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/GetOpenBlocks.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/ListAlgo.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/ListAlgoDatasetAssoc.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/ListDataset.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/LoadBlocksByDAS.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/LoadDBSFilesByDAS.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/NewAlgo.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/NewDataset.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/SetBlockStatus.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/Status.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/UpdateAlgo.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/UpdateAlgoDatasetAssoc.py delete mode 100644 src/python/WMComponent/DBS3Buffer/SQLite/__init__.py delete mode 100644 src/python/WMComponent/DBSBuffer/DBSBuffer.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/AlgoDatasetAssoc.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/CountBlocks.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/CountFiles.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/Create.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Add.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddCKType.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddChecksum.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddChecksumByLFN.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddLocation.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddRunLumi.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/BulkHeritageParent.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Delete.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Exists.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/ExistsForAccountant.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetBlock.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetByID.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetByLFN.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetChecksum.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetChildren.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetLocation.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetParentStatus.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetParents.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetRunLumiFile.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/HeritageLFNChild.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/HeritageLFNParent.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/LoadBulkFilesByID.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetBlock.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetLocation.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetLocationByLFN.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetStatus.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/__init__.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/Destroy.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/GetBlockFromDataset.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/GetOpenBlocks.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/ListAlgo.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/ListAlgoDatasetAssoc.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/ListDataset.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/NewAlgo.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/NewDataset.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/Status.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/UpdateAlgo.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/UpdateAlgoDatasetAssoc.py delete mode 100644 src/python/WMComponent/DBSBuffer/Database/SQLite/__init__.py delete mode 100644 src/python/WMComponent/DBSBuffer/DefaultConfig.py diff --git a/src/python/WMComponent/DBS3Buffer/DBSBufferBlock.py b/src/python/WMComponent/DBS3Buffer/DBSBufferBlock.py index ed558ea141..bbc233a257 100644 --- a/src/python/WMComponent/DBS3Buffer/DBSBufferBlock.py +++ b/src/python/WMComponent/DBS3Buffer/DBSBufferBlock.py @@ -43,16 +43,13 @@ def __init__(self, name, location, das): self.data = {'dataset_conf_list': [], # List of dataset configurations - 'file_conf_list': [], # List of files, with the configuration for each + 'file_conf_list': [], # List of files, the configuration for each 'files': [], # List of file objects 'block': {}, # Dict of block info - 'block_parent_list': [], # List of block parents 'processing_era': {}, # Dict of processing era info - 'ds_parent_list': [], # List of parent datasets 'acquisition_era': {}, # Dict of acquisition era information 'primds': {}, # Dict of primary dataset info 'dataset': {}, # Dict of processed dataset info - 'physics_group_name': {}, # Physics Group Name 'file_parent_list': []} # List of file parents self.files = [] @@ -68,6 +65,10 @@ def __init__(self, name, location, das): self.data['block']['origin_site_name'] = location self.data['block']['open_for_writing'] = 0 # If we're sending a block, it better be open + self.data['block']['create_by'] = "WMAgent" + self.data['block']['creation_date'] = int(time.time()) + self.data['block']['block_size'] = 0 + self.data['block']['file_count'] = 0 return @@ -89,9 +90,6 @@ def addFile(self, dbsFile): Add a DBSBufferFile object to our block """ - - - if dbsFile['id'] in [x['id'] for x in self.files]: msg = "Duplicate file inserted into DBSBlock: %i\n" % (dbsFile['id']) msg += "Ignoring this file for now!\n" @@ -103,12 +101,21 @@ def addFile(self, dbsFile): return self.files.append(dbsFile) + self.data['block']['block_size'] += dbsFile['size'] + self.data['block']['file_count'] += 1 + # Assemble information for the file itself fileDict = {} fileDict['file_type'] = 'EDM' fileDict['logical_file_name'] = dbsFile['lfn'] fileDict['file_size'] = dbsFile['size'] fileDict['event_count'] = dbsFile['events'] + fileDict['adler32'] = "NOTSET" + fileDict['md5'] = "NOTSET" + fileDict['last_modified_by'] = "WMAgent" + fileDict['last_modification_date'] = int(time.time()) + fileDict['auto_cross_section'] = 0.0 + # Do the checksums for cktype in dbsFile['checksums'].keys(): cksum = dbsFile['checksums'][cktype] @@ -198,14 +205,15 @@ def addDatasetParent(self, parent): self.data['ds_parent_list'].append({'parent_dataset': parent}) return - def setProcessingVer(self, era): + def setProcessingVer(self, procVer): """ - _setProcessingEra_ + _setProcessingVer_ - Set the block's processing era + Set the block's processing version. """ - - self.data['processing_era']['processing_version'] = era + self.data["processing_era"]["processing_version"] = procVer + self.data["processing_era"]["create_by"] = "WMAgent" + self.data["processing_era"]["description"] = "" return def setAcquisitionEra(self, era, date = 123456789): @@ -214,7 +222,6 @@ def setAcquisitionEra(self, era, date = 123456789): Set the acquisition era for the block """ - self.data['acquisition_era']['acquisition_era_name'] = era self.data['acquisition_era']['start_date'] = date return @@ -235,11 +242,8 @@ def hasDataset(self): Check and see if the dataset has been properly set """ - return self.data['dataset'].get('dataset', False) - - def setDataset(self, datasetName, primaryType, datasetType, physicsGroup = None, overwrite = False, valid = 1): """ @@ -248,7 +252,6 @@ def setDataset(self, datasetName, primaryType, Set all the information concerning a single dataset, including the primary, processed and tier info """ - if self.hasDataset() and not overwrite: # Do nothing, we already have a dataset return @@ -276,7 +279,8 @@ def setDataset(self, datasetName, primaryType, # Do the primary dataset self.data['primds']['primary_ds_name'] = primary self.data['primds']['primary_ds_type'] = primaryType - + self.data['primds']['create_by'] = "WMAgent" + self.data['primds']['creation_date'] = int(time.time()) # Do the processed self.data['dataset']['physics_group_name'] = physicsGroup @@ -285,6 +289,11 @@ def setDataset(self, datasetName, primaryType, self.data['dataset']['dataset_access_type'] = datasetType self.data['dataset']['dataset'] = datasetName + # Add misc meta data. + self.data['dataset']['create_by'] = "WMAgent" + self.data['dataset']['last_modified_by'] = "WMAgent" + self.data['dataset']['creation_date'] = int(time.time()) + self.data['dataset']['last_modification_date'] = int(time.time()) return @@ -383,5 +392,9 @@ def FillFromDBSBuffer(self, blockInfo): self.startTime = blockInfo.get('creation_date') self.inBuff = True + if 'status' in blockInfo.keys(): + self.status = blockInfo['status'] + del blockInfo['status'] + for key in blockInfo.keys(): self.data['block'][key] = blockInfo.get(key) diff --git a/src/python/WMComponent/DBS3Buffer/DBSBufferUtil.py b/src/python/WMComponent/DBS3Buffer/DBSBufferUtil.py index 9361c42bda..555625a377 100644 --- a/src/python/WMComponent/DBS3Buffer/DBSBufferUtil.py +++ b/src/python/WMComponent/DBS3Buffer/DBSBufferUtil.py @@ -2,16 +2,12 @@ """ _DBSBufferUtil_ -APIs related to using the DBSBuffer - +APIs related to using the DBSBuffer. """ - - - import logging -import os import threading + from WMCore.WMFactory import WMFactory from WMCore.DAOFactory import DAOFactory from WMComponent.DBS3Buffer.DBSBufferFile import DBSBufferFile @@ -208,18 +204,17 @@ def findUploadableDAS(self): - def findOpenBlocks(self): + def findOpenBlocks(self, dbs3OnlyUpload = False): """ _findOpenBlocks_ This should find all blocks. """ - myThread = threading.currentThread() existingTransaction = self.beginTransaction() openBlocks = self.daoFactory(classname = "GetOpenBlocks") - result = openBlocks.execute(conn = self.getDBConn(), + result = openBlocks.execute(dbs3OnlyUpload, conn = self.getDBConn(), transaction=self.existingTransaction()) self.commitTransaction(existingTransaction) @@ -250,7 +245,7 @@ def loadBlocksByDAS(self, das): - def loadBlocks(self, blocknames): + def loadBlocks(self, blocknames, dbs3UploadOnly): """ _loadBlocksByDAS_ @@ -266,12 +261,11 @@ def loadBlocks(self, blocknames): existingTransaction = self.beginTransaction() findBlocks = self.daoFactory(classname = "LoadBlocks") - result = findBlocks.execute(blocknames = blocknames, + result = findBlocks.execute(blocknames, dbs3UploadOnly, conn = self.getDBConn(), transaction=self.existingTransaction()) self.commitTransaction(existingTransaction) - return result @@ -321,13 +315,12 @@ def findUploadableFilesByDAS(self, das): return dbsFiles - def updateBlocks(self, blocks): + def updateBlocks(self, blocks, dbs3UploadOnly = False): """ _updateBlocks_ Update the blocks in DBSBuffer """ - if len(blocks) < 1: # Nothing to do return @@ -336,7 +329,7 @@ def updateBlocks(self, blocks): existingTransaction = self.beginTransaction() updateBlock = self.daoFactory(classname = "UpdateBlocks") - updateBlock.execute(blocks = blocks, conn = self.getDBConn(), + updateBlock.execute(blocks, dbs3UploadOnly, conn = self.getDBConn(), transaction=self.existingTransaction()) self.commitTransaction(existingTransaction) return diff --git a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py index 2e2a0d6c6e..1cb8c54e53 100644 --- a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py +++ b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py @@ -155,8 +155,6 @@ def uploadWorker(input, results, dbsUrl): return - - class DBSUploadException(WMException): """ Holds the exception info for @@ -200,8 +198,9 @@ def __init__(self, config, dbsconfig = None): self.nProc = getattr(self.config.DBSUpload, 'nProcesses', 4) self.wait = getattr(self.config.DBSUpload, 'dbsWaitTime', 1) self.nTries = getattr(self.config.DBSUpload, 'dbsNTries', 300) - self.physicsGroup = getattr(self.config.DBSUpload, 'physicsGroup', 'DBS3Test') - self.blockCount = 0 + self.dbs3UploadOnly = getattr(self.config.DBSUpload, 'dbs3UploadOnly', False) + self.physicsGroup = getattr(self.config.DBSUpload, 'physicsGroup', 'DBS3Test') + self.blockCount = 0 # List of blocks currently in processing self.queuedBlocks = [] @@ -251,10 +250,8 @@ def __del__(self): Trigger a close of connections if necessary """ self.close() - #BaseWorkerThread.__del__(self) return - def close(self): """ _close_ @@ -311,13 +308,18 @@ def algorithm(self, parameters = None): Then add blocks to DBS Then mark blocks as done in DBSBuffer """ - try: logging.info("Starting the DBSUpload Polling Cycle") self.loadBlocks() - self.loadFiles() - self.checkTimeout() - self.inputBlocks() + + # The following two functions will actually place new files into + # blocks. In DBS3 upload mode we rely on something else to do that + # for us and will skip this step. + if not self.dbs3UploadOnly: + self.loadFiles() + self.checkTimeout() + + self.inputBlocks() self.retrieveBlocks() except WMException: raise @@ -335,7 +337,7 @@ def loadBlocks(self): Find all blocks; make sure they're in the cache """ - openBlocks = self.dbsUtil.findOpenBlocks() + openBlocks = self.dbsUtil.findOpenBlocks(self.dbs3UploadOnly) logging.info("These are the openblocks: %s" % openBlocks) # Load them if we don't have them @@ -347,7 +349,7 @@ def loadBlocks(self): # Now load the blocks try: - loadedBlocks = self.dbsUtil.loadBlocks(blocknames = blocksToLoad) + loadedBlocks = self.dbsUtil.loadBlocks(blocksToLoad, self.dbs3UploadOnly) logging.info("Loaded blocks: %s" % loadedBlocks) except WMException: raise @@ -386,12 +388,10 @@ def loadBlocks(self): # Add to the cache self.addNewBlock(block = block) - # All blocks should now be loaded and present # in both the block cache (which has all the info) # and the dasCache (which is a list of name pointers # to the keys in the block cache). - return @@ -399,13 +399,9 @@ def loadFiles(self): """ _loadFiles_ - Load all files that need to be loaded. - - I will do this by DAS for now to break - the monstrous calls down into smaller chunks. + Load all files that need to be loaded. I will do this by DAS for now to + break the monstrous calls down into smaller chunks. """ - - # Grab all the Dataset-Algo combindations dasList = self.dbsUtil.findUploadableDAS() @@ -440,11 +436,6 @@ def loadFiles(self): # Sort the files and blocks by location fileDict = sortListByKey(input = loadedFiles, key = 'locations') - # Now we have both files and blocks - # We need a sorting algorithm of sorts... - - - # Now add each file for location in fileDict.keys(): files = fileDict.get(location) @@ -463,7 +454,7 @@ def loadFiles(self): location = location, das = dasID) # Add the era info currentBlock.setAcquisitionEra(era = dasInfo['AcquisitionEra']) - currentBlock.setProcessingVer(era = dasInfo['ProcessingVer']) + currentBlock.setProcessingVer(procVer = dasInfo['ProcessingVer']) self.addNewBlock(block = currentBlock) dasBlocks.append(currentBlock.getName()) @@ -486,7 +477,7 @@ def loadFiles(self): location = location, das = dasID) currentBlock.setAcquisitionEra(era = dasInfo['AcquisitionEra']) - currentBlock.setProcessingVer(era = dasInfo['ProcessingVer']) + currentBlock.setProcessingVer(procVer = dasInfo['ProcessingVer']) # Now deal with the file currentBlock.addFile(dbsFile = newFile) @@ -503,15 +494,15 @@ def loadFiles(self): return - def checkTimeout(self): """ - Check all blocks for a timeout + _checkTimeout_ + Loop all Open blocks and mark them as Pending if they have timed out. """ for block in self.blockCache.values(): - if block.status == 'Open' and block.getTime() > self.maxBlockTime: - block.status = 'Pending' + if block.status == "Open" and block.getTime() > self.maxBlockTime: + block.status = "Pending" self.blockCache[block.getName()] = block def addNewBlock(self, block): @@ -590,46 +581,47 @@ def getBlock(self, newFile, dasBlocks, location, das): dasBlocks.append(blockname) return newBlock - - def inputBlocks(self): """ - _processBlocks_ - - Process the blocks that have new files in them. - 1) Put them into DBSBuffer in state 'Pending' - 2) Upload them to DBS + _inputBlocks_ + + Loop through all of the "active" blocks and sort them so we can act + appropriately on them. Everything will be sorted based on the + following: + Queued - Block is already being acted on by another process. We just + ignore it. + Pending, not in DBSBuffer - Block that has been closed and needs to + be injected into DBS and also written to DBSBuffer. We'll do both. + Pending, in DBSBuffer - Block has been closed and written to + DBSBuffer. We just need to inject it into DBS. + Open, not in DBSBuffer - Newly created block that needs to be written + not DBSBuffer. + Open, in DBSBuffer - Newly created block that has already been + written to DBSBuffer. We don't have to do anything with it. """ - - myThread = threading.currentThread() - # We want to run this over all pending blocks - blocks = [] - blockForDBSBuffer = [] - updateBlocks = [] + createInDBS = [] + createInDBSBuffer = [] + updateInDBSBuffer = [] for block in self.blockCache.values(): if block.getName() in self.queuedBlocks: - logging.error("Skipping queuedBlock %s" % block.getName()) + # Block is already being dealt with by another process. We'll + # ignore it here. continue if block.status == 'Pending': - blocks.append(block) - # Either this is a new block - # or it's one we need to update + # All pending blocks need to be injected into DBS. + createInDBS.append(block) + + # If this is a new block it needs to be added to DBSBuffer + # otherwise it just needs to be updated in DBSBuffer. if not block.inBuff: - blockForDBSBuffer.append(block) + createInDBSBuffer.append(block) else: - updateBlocks.append(block) + updateInDBSBuffer.append(block) if block.status == 'Open' and not block.inBuff: - # Then this is a new block we need to - # add to the buffer - # but not process - blockForDBSBuffer.append(block) - - - if len(blocks) < 1: - # Nothing to do - return + # New block that needs to be added to DBSBuffer. + createInDBSBuffer.append(block) # Build the pool if it was closed if len(self.pool) == 0: @@ -638,8 +630,8 @@ def inputBlocks(self): # First handle new and updated blocks try: myThread.transaction.begin() - self.dbsUtil.createBlocks(blocks = blockForDBSBuffer) - self.dbsUtil.updateBlocks(blocks = updateBlocks) + self.dbsUtil.createBlocks(blocks = createInDBSBuffer) + self.dbsUtil.updateBlocks(blocks = updateInDBSBuffer) myThread.transaction.commit() except WMException: myThread.transaction.rollback() @@ -648,16 +640,17 @@ def inputBlocks(self): msg = "Unhandled exception while writing new blocks into DBSBuffer\n" msg += str(ex) logging.error(msg) - logging.debug("Blocks for DBSBuffer: %s\n" % blockForDBSBuffer) - logging.debug("Blocks for Update: %s\n" % updateBlocks) + logging.debug("Blocks for DBSBuffer: %s\n" % createInDBSBuffer) + logging.debug("Blocks for Update: %s\n" % updateInDBSBuffer) myThread.transaction.rollback() raise DBSUploadException(msg) - # List blocks as inBuff - for block in blockForDBSBuffer: + # Update block status in the block cache. Mark the blocks that we have + # added to DBSBuffer as being in DBSBuffer. + for block in createInDBSBuffer: self.blockCache.get(block.getName()).inBuff = True - # Now put in the files that we just added. + # Record new file/block associations in DBSBuffer. try: myThread.transaction.begin() self.dbsUtil.setBlockFiles(binds = self.filesToUpdate) @@ -674,10 +667,8 @@ def inputBlocks(self): myThread.transaction.rollback() raise DBSUploadException(msg) - - # Now that things are in DBSBuffer, we can put them in DBS - - for block in blocks: + # Finally upload blocks to DBS. + for block in createInDBS: if len(block.files) < 1: # What are we doing? logging.debug("Skipping empty block") @@ -700,7 +691,6 @@ def replaceKeys(block, oldKey, newKey): replaceKeys(block, 'CreationDate', 'creation_date') replaceKeys(block, 'NumberOfFiles', 'file_count') replaceKeys(block, 'location', 'origin_site_name') - block.data['block']['block_size'] = float(block.data['block']['block_size']) for key in ['insertedFiles', 'newFiles', 'DatasetAlgo', 'file_count', 'block_size', 'origin_site_name', 'creation_date', 'open', 'Name']: if key in block.data.keys(): @@ -717,14 +707,9 @@ def replaceKeys(block, oldKey, newKey): f.close() self.queuedBlocks.append(block.getName()) - - # And all work is in and we're done for now - return - - def retrieveBlocks(self): """ _retrieveBlocks_ @@ -774,7 +759,7 @@ def retrieveBlocks(self): try: myThread.transaction.begin() - self.dbsUtil.updateBlocks(blocks = loadedBlocks) + self.dbsUtil.updateBlocks(loadedBlocks, self.dbs3UploadOnly) myThread.transaction.commit() except WMException: myThread.transaction.rollback() @@ -787,7 +772,6 @@ def retrieveBlocks(self): myThread.transaction.rollback() raise DBSUploadException(msg) - for block in loadedBlocks: # Clean things up name = block.getName() @@ -800,6 +784,5 @@ def retrieveBlocks(self): if len(self.pool) > 0: self.close() - # And we're done return diff --git a/src/python/WMComponent/DBS3Buffer/MySQL/Create.py b/src/python/WMComponent/DBS3Buffer/MySQL/Create.py index b9afc8a4a0..8e46694806 100644 --- a/src/python/WMComponent/DBS3Buffer/MySQL/Create.py +++ b/src/python/WMComponent/DBS3Buffer/MySQL/Create.py @@ -1,10 +1,9 @@ """ -_Create_DBSBuffer_ +_Create_DBS3Buffer_ Implementation of Create_DBSBuffer for MySQL. """ -import logging import threading from WMCore.Database.DBCreator import DBCreator @@ -45,7 +44,7 @@ def __init__(self, logger = None, dbi = None, params = None): app_fam varchar(100), pset_hash varchar(700), config_content LONGTEXT, - in_dbs int, + in_dbs int, primary key(ID), unique (app_name, app_ver, app_fam, pset_hash) ) ENGINE=InnoDB""" @@ -56,7 +55,7 @@ def __init__(self, logger = None, dbi = None, params = None): id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, algo_id BIGINT UNSIGNED, dataset_id BIGINT UNSIGNED, - in_dbs INTEGER DEFAULT 0, + in_dbs INTEGER DEFAULT 0, primary key(id), FOREIGN KEY (algo_id) REFERENCES dbsbuffer_algo(id) ON DELETE CASCADE, @@ -80,7 +79,7 @@ def __init__(self, logger = None, dbi = None, params = None): events INTEGER, dataset_algo BIGINT UNSIGNED not null, block_id BIGINT UNSIGNED, - status varchar(20), + status VARCHAR(20), in_phedex INTEGER DEFAULT 0, workflow INTEGER, LastModificationDate BIGINT, @@ -124,6 +123,7 @@ def __init__(self, logger = None, dbi = None, params = None): location INTEGER NOT NULL, create_time INTEGER, status VARCHAR(20), + status3 VARCHAR(20) DEFAULT 'Pending', UNIQUE(blockname, location))ENGINE=InnoDB""" self.create["11dbsbuffer_checksum_type"] = \ @@ -143,9 +143,6 @@ def __init__(self, logger = None, dbi = None, params = None): FOREIGN KEY (fileid) REFERENCES dbsbuffer_file(id) ON DELETE CASCADE) ENGINE=InnoDB""" - - - checksumTypes = ['cksum', 'adler32', 'md5'] for i in checksumTypes: checksumTypeQuery = """INSERT INTO dbsbuffer_checksum_type (type) VALUES ('%s') diff --git a/src/python/WMComponent/DBS3Buffer/MySQL/GetOpenBlocks.py b/src/python/WMComponent/DBS3Buffer/MySQL/GetOpenBlocks.py index c1b5fb3832..fcfc631571 100644 --- a/src/python/WMComponent/DBS3Buffer/MySQL/GetOpenBlocks.py +++ b/src/python/WMComponent/DBS3Buffer/MySQL/GetOpenBlocks.py @@ -5,17 +5,24 @@ MySQL implementation of DBSBufferFiles.GetBlock """ - - - from WMCore.Database.DBFormatter import DBFormatter class GetOpenBlocks(DBFormatter): sql = """SELECT DISTINCT blockname AS blockname, create_time AS create_time FROM dbsbuffer_block - WHERE status = 'Open' OR status = 'Pending' """ + WHERE status = 'Open' OR status = 'Pending'""" + + # Find only blocks that have been closed and uploaded to DBS2 but are still + # waiting to be uploaded to DBS3. + sql3 = """SELECT DISTINCT blockname AS blockname, create_time AS create_time + FROM dbsbuffer_block + WHERE status = 'InDBS' AND (status3 = 'Open' OR status3 = 'Pending')""" - def execute(self, conn = None, transaction = False): - result = self.dbi.processData(self.sql, {}, - conn = conn, transaction = transaction) + def execute(self, dbs3OnlyUpload, conn = None, transaction = False): + if dbs3OnlyUpload: + result = self.dbi.processData(self.sql3, {}, conn = conn, + transaction = transaction) + else: + result = self.dbi.processData(self.sql, {}, conn = conn, + transaction = transaction) return self.formatDict(result) diff --git a/src/python/WMComponent/DBS3Buffer/MySQL/LoadBlocks.py b/src/python/WMComponent/DBS3Buffer/MySQL/LoadBlocks.py index 94ffd836f3..9da6528714 100644 --- a/src/python/WMComponent/DBS3Buffer/MySQL/LoadBlocks.py +++ b/src/python/WMComponent/DBS3Buffer/MySQL/LoadBlocks.py @@ -12,16 +12,18 @@ class LoadBlocks(DBFormatter): sql = """SELECT DISTINCT dbb.blockname as blockname, dbb.create_time as create_time, + dbb.status AS status, dbb.status3 AS status3, (SELECT COUNT(*) FROM dbsbuffer_file dbf WHERE dbf.block_id = dbb.id) AS nFiles, (SELECT SUM(dbf2.filesize) FROM dbsbuffer_file dbf2 WHERE dbf2.block_id = dbb.id) AS blocksize, dbl.se_name AS location, dbf3.dataset_algo AS das - FROM dbsbuffer_block dbb - INNER JOIN dbsbuffer_file dbf3 ON dbf3.block_id = dbb.id - INNER JOIN dbsbuffer_location dbl ON dbl.id = dbb.location - WHERE dbb.blockname = :blockname""" - - - def format(self, result): + FROM dbsbuffer_block dbb + INNER JOIN dbsbuffer_file dbf3 ON + dbf3.block_id = dbb.id + INNER JOIN dbsbuffer_location dbl ON + dbl.id = dbb.location + WHERE dbb.blockname = :blockname""" + + def format(self, result, dbs3UploadOnly): tmpList = self.formatDict(result) blockList = [] for tmp in tmpList: @@ -33,21 +35,24 @@ def format(self, result): final['origin_site_name'] = tmp['location'] final['DatasetAlgo'] = tmp['das'] final['open_for_writing'] = 1 + + if dbs3UploadOnly: + final['status'] = tmp['status3'] + else: + final['status'] = tmp['status'] + blockList.append(final) return blockList - - def execute(self, blocknames, conn = None, transaction = False): + def execute(self, blocknames, dbs3UploadOnly, conn = None, transaction = False): """ Take a list of blocknames and use them to load the blocks. """ - binds = [] for name in blocknames: binds.append({'blockname': name}) - result = self.dbi.processData(self.sql, binds, - conn = conn, + result = self.dbi.processData(self.sql, binds, conn = conn, transaction = transaction) - return self.format(result) + return self.format(result, dbs3UploadOnly) diff --git a/src/python/WMComponent/DBS3Buffer/MySQL/UpdateBlocks.py b/src/python/WMComponent/DBS3Buffer/MySQL/UpdateBlocks.py index de06ef7ba1..78db08d6fa 100644 --- a/src/python/WMComponent/DBS3Buffer/MySQL/UpdateBlocks.py +++ b/src/python/WMComponent/DBS3Buffer/MySQL/UpdateBlocks.py @@ -1,32 +1,28 @@ #!/usr/bin/env python """ -_DBSBuffer.SetBlockStatus_ +_DBS3Buffer.SetBlockStatus_ Create new block in dbsbuffer_block Update file to reflect block information """ - - - import threading -import exceptions from WMCore.Database.DBFormatter import DBFormatter class UpdateBlocks(DBFormatter): - sql = """UPDATE dbsbuffer_block SET status = :status, create_time = :time, location = (SELECT dbl.id FROM dbsbuffer_location dbl WHERE dbl.se_name = :location) WHERE blockname = :block""" + sql3 = """UPDATE dbsbuffer_block SET status3 = :status WHERE blockname = :block""" def __init__(self, logger = None, dbi = None): myThread = threading.currentThread() DBFormatter.__init__(self, myThread.logger, myThread.dbi) - def execute(self, blocks, conn = None, transaction = False): + def execute(self, blocks, dbs3UploadOnly = False, conn = None, transaction = False): """ _execute_ @@ -38,8 +34,11 @@ def execute(self, blocks, conn = None, transaction = False): bindVars.append({"block": block.getName(), "location": block.getLocation(), "status": block.status, "time": block.getStartTime()}) - self.dbi.processData(self.sql, bindVars, conn = conn, - transaction = transaction) - + if dbs3UploadOnly: + self.dbi.processData(self.sql3, bindVars, conn = conn, + transaction = transaction) + else: + self.dbi.processData(self.sql, bindVars, conn = conn, + transaction = transaction) return diff --git a/src/python/WMComponent/DBS3Buffer/Oracle/Create.py b/src/python/WMComponent/DBS3Buffer/Oracle/Create.py index 39214fff90..cff75ca608 100644 --- a/src/python/WMComponent/DBS3Buffer/Oracle/Create.py +++ b/src/python/WMComponent/DBS3Buffer/Oracle/Create.py @@ -184,7 +184,8 @@ def __init__(self, logger = None, dbi = None, params = None): blockname VARCHAR(250) NOT NULL, location INTEGER NOT NULL, create_time INTEGER, - status VARCHAR(20)) %s""" % tablespaceTable + status VARCHAR(20), + status3 VARCHAR(20) DEFAULT 'Pending') %s""" % tablespaceTable self.create["10dbsbuffer_block_seq"] = \ """CREATE SEQUENCE dbsbuffer_block_seq diff --git a/src/python/WMComponent/DBS3Buffer/Oracle/LoadBlocks.py b/src/python/WMComponent/DBS3Buffer/Oracle/LoadBlocks.py new file mode 100644 index 0000000000..88ae7a7875 --- /dev/null +++ b/src/python/WMComponent/DBS3Buffer/Oracle/LoadBlocks.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +""" +_LoadBlocks_ + +Oracle implementation of DBS3Buffer.LoadBlocks +""" + +from WMComponent.DBSBuffer.Database.MySQL.LoadBlocks import LoadBlocks as MySQLLoadBlocks + +class LoadBlocks(MySQLLoadBlocks): + """ + Identical to MySQL version. + """ diff --git a/src/python/WMComponent/DBS3Buffer/Oracle/UpdateBlocks.py b/src/python/WMComponent/DBS3Buffer/Oracle/UpdateBlocks.py new file mode 100644 index 0000000000..0e2bbd106d --- /dev/null +++ b/src/python/WMComponent/DBS3Buffer/Oracle/UpdateBlocks.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +""" +_UpdateBlocks_ + +Oracle implementation of DBS3Buffer.UpdateBlocks +""" + +from WMComponent.DBSBuffer.Database.MySQL.UpdateBlocks import UpdateBlocks as MySQLUpdateBlocks + +class UpdateBlocks(MySQLUpdateBlocks): + """ + Identical to MySQL version. + """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/AlgoDatasetAssoc.py b/src/python/WMComponent/DBS3Buffer/SQLite/AlgoDatasetAssoc.py deleted file mode 100644 index 763eacf344..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/AlgoDatasetAssoc.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -""" -_AlgoDatasetAssoc_ - -SQLite implementation of DBSBuffer.AlgoDatasetAssoc -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.AlgoDatasetAssoc import AlgoDatasetAssoc as MySQLAlgoDatasetAssoc - -class AlgoDatasetAssoc(MySQLAlgoDatasetAssoc): - """ - _AlgoDatasetAssoc_ - - Associate an algorithm to a dataset and return the ID of the association. - """ - sql = """INSERT INTO dbsbuffer_algo_dataset_assoc (algo_id, dataset_id) - SELECT (SELECT id FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash) AS algo_id, - (SELECT id FROM dbsbuffer_dataset WHERE path = :path) AS dataset_id - WHERE NOT EXISTS - (SELECT * FROM dbsbuffer_algo_dataset_assoc WHERE algo_id = - (SELECT id FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash) AND dataset_id = - (SELECT id FROM dbsbuffer_dataset WHERE path = :path))""" - - idsql = """SELECT id FROM dbsbuffer_algo_dataset_assoc WHERE algo_id = - (SELECT id FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash) AND dataset_id = - (SELECT id FROM dbsbuffer_dataset WHERE path = :path)""" diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/CountBlocks.py b/src/python/WMComponent/DBS3Buffer/SQLite/CountBlocks.py deleted file mode 100644 index 7640520248..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/CountBlocks.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -_CountBlocks_ - -SQLite implementation of DBSBuffer.CountBlocks -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.CountBlocks import CountBlocks as MySQLCountBlocks - -class CountBlocks(MySQLCountBlocks): - """ - _CountBlocks_ - - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/CountFiles.py b/src/python/WMComponent/DBS3Buffer/SQLite/CountFiles.py deleted file mode 100644 index 9980e18677..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/CountFiles.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -_CountFiles_ - -SQLite implementation of DBSBuffer.CountFiles -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.CountFiles import CountFiles as MySQLCountFiles - -class CountFiles(MySQLCountFiles): - """ - _CountFiles_ - - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/Create.py b/src/python/WMComponent/DBS3Buffer/SQLite/Create.py deleted file mode 100644 index da08ae391a..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/Create.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -_Create_DBSBuffer_ - -Implementation of Create_DBSBuffer for SQLite -""" - - - - -import logging -import threading - -from WMCore.Database.DBCreator import DBCreator - -class Create(DBCreator): - - def __init__(self): - """ - _init_ - - Call the base class's constructor and create all necessary tables, - constraints and inserts. - """ - myThread = threading.currentThread() - DBCreator.__init__(self, myThread.logger, myThread.dbi) - - self.create["01dbsbuffer_dataset"] = \ - """CREATE TABLE dbsbuffer_dataset - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - path VARCHAR(500) UNIQUE NOT NULL - )""" - - self.create["02dbsbuffer_algo"] = \ - """CREATE TABLE dbsbuffer_algo - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - app_name VARCHAR(100), - app_ver VARCHAR(100), - app_fam VARCHAR(100), - pset_hash VARCHAR(700), - config_content LONGTEXT, - in_dbs INTEGER DEFAULT 0, - UNIQUE(app_name, app_ver, app_fam, pset_hash) - )""" - - self.create["03dbsbuffer_algo_dataset_assoc"] = \ - """CREATE TABLE dbsbuffer_algo_dataset_assoc - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - algo_id BIGINT UNSIGNED, - dataset_id BIGINT UNSINGED, - unmigrated_files BIGINT UNSIGNED DEFAULT 0, - in_dbs INTEGER DEFAULT 0, - FOREIGN KEY (algo_id) REFERENCES dbsbuffer_algo(id) - ON DELETE CASCADE, - FOREIGN KEY (dataset_id) REFERENCES dbsbuffer_dataset(id) - ON DELETE CASCADE - )""" - - self.create["04dbsbuffer_file"] = \ - """CREATE TABLE dbsbuffer_file ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - lfn VARCHAR(255) NOT NULL, - filesize BIGINT, - events INTEGER, - cksum BIGINT UNSIGNED, - dataset_algo INTEGER NOT NULL, - block_id INTEGER, - status varchar(20), - LastModificationDate BIGINT)""" - - self.create["05dbsbuffer_file_parent"] = \ - """CREATE TABLE dbsbuffer_file_parent ( - child INTEGER NOT NULL, - parent INTEGER NOT NULL, - FOREIGN KEY (child) references dbsbuffer_file(id) - ON DELETE CASCADE, - FOREIGN KEY (parent) references dbsbuffer_file(id), - UNIQUE(child, parent))""" - - self.create["06dbsbuffer_file_runlumi_map"] = \ - """CREATE TABLE dbsbuffer_file_runlumi_map ( - filename INTEGER NOT NULL, - run INTEGER NOT NULL, - lumi INTEGER NOT NULL, - FOREIGN KEY (filename) references dbsbuffer_file(id) - ON DELETE CASCADE)""" - - self.create["07dbsbuffer_location"] = \ - """CREATE TABLE dbsbuffer_location ( - id INTEGER PRIMARY KEY, - se_name VARCHAR(255) NOT NULL, - UNIQUE(se_name))""" - - self.create["08dbsbuffer_file_location"] = \ - """CREATE TABLE dbsbuffer_file_location ( - filename INTEGER NOT NULL, - location INTEGER NOT NULL, - UNIQUE(filename, location), - FOREIGN KEY(filename) REFERENCES dbsbuffer_file(id) - ON DELETE CASCADE, - FOREIGN KEY(location) REFERENCES dbsbuffer_location(id) - ON DELETE CASCADE)""" - - self.create["10dbsbuffer_block"] = \ - """CREATE TABLE dbsbuffer_block ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - blockname VARCHAR(250) NOT NULL, - location INTEGER NOT NULL, - create_time INTEGER, - status VARCHAR(20), - UNIQUE(blockname, location) - )""" - - self.create["11dbsbuffer_checksum_type"] = \ - """CREATE TABLE dbsbuffer_checksum_type ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type VARCHAR(255) )""" - - - self.create["12dbsbuffer_file_checksums"] = \ - """CREATE TABLE dbsbuffer_file_checksums ( - fileid INTEGER, - typeid INTEGER, - cksum VARCHAR(100), - UNIQUE(fileid, typeid))""" diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Add.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Add.py deleted file mode 100644 index 8e7f0e9020..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Add.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of AddFile - -""" - - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.Add import Add as MySQLAdd - -class Add(MySQLAdd): - """ - SQLite implementation of AddFile - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddCKType.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddCKType.py deleted file mode 100644 index 03c0512202..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddCKType.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python - -""" -SQLite implementation of AddCKType -""" - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.AddCKType import AddCKType as MySQLAddCKType - -class AddCKType(MySQLAddCKType): - """ - Identical to MySQL version - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddChecksum.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddChecksum.py deleted file mode 100644 index 47898c8ab2..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddChecksum.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python - -""" -SQLite implementation of AddChecksum -""" - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.AddChecksum import AddChecksum as MySQLAddChecksum - -class AddChecksum(MySQLAddChecksum): - """ - Add a Checksum - - """ - - sql = """INSERT INTO dbsbuffer_file_checksums (fileid, typeid, cksum) - SELECT :fileid, (SELECT id FROM dbsbuffer_checksum_type WHERE type = :cktype), :cksum - WHERE NOT EXISTS (SELECT fileid FROM dbsbuffer_file_checksums WHERE - fileid = :fileid AND typeid = (SELECT id FROM dbsbuffer_checksum_type WHERE type = :cktype))""" diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddChecksumByLFN.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddChecksumByLFN.py deleted file mode 100644 index 0efbffe3b6..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddChecksumByLFN.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python - -""" -Oracle implementation of AddChecksumByLFN -""" - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.AddChecksumByLFN import AddChecksumByLFN as MySQLAddChecksumByLFN - -class AddChecksumByLFN(MySQLAddChecksumByLFN): - """ - - Add Checksums using lfn as key - - """ - sql = MySQLAddChecksumByLFN.sql.replace('FROM dual', '') diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddLocation.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddLocation.py deleted file mode 100644 index c9f3eced31..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddLocation.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -""" -_AddLocation_ - -SQLite implementation of DBSBufferFiles.AddLocation -""" - -from WMCore.Database.DBFormatter import DBFormatter -from WMCore.Database.Transaction import Transaction - -class AddLocation(DBFormatter): - sql = """INSERT INTO dbsbuffer_location (se_name) - SELECT :location AS se_name WHERE NOT EXISTS - (SELECT se_name FROM dbsbuffer_location WHERE se_name = :location)""" - - existsSQL = """SELECT se_name, id FROM dbsbuffer_location - WHERE se_name = :location""" - - def execute(self, siteName, conn = None, transaction = False): - - if type(siteName) == str: - binds = {"location": siteName} - else: - binds = [] - for aLocation in siteName: - binds.append({"location": aLocation}) - - myTransaction = Transaction(self.dbi) - myTransaction.begin() - - nameMap = {} - self.dbi.processData(self.sql, binds, conn = conn, - transaction = transaction) - results = self.dbi.processData(self.existsSQL, binds, - conn = myTransaction.conn, - transaction = True) - results = self.format(results) - for result in results: - nameMap[result[0]] = int(result[1]) - - myTransaction.commit() - return nameMap diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddRunLumi.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddRunLumi.py deleted file mode 100644 index 55b8cbe3ac..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/AddRunLumi.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of AddRunLumi - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.AddRunLumi import AddRunLumi as MySQLAddRunLumi - -class AddRunLumi(MySQLAddRunLumi): - """ - - SQLite implementation of AddRunLumi - - """ - - sql = """INSERT INTO dbsbuffer_file_runlumi_map (filename, run, lumi) - SELECT id, :run, :lumi from dbsbuffer_file - where lfn = :lfn""" - - def GetUpdateAddDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/BulkHeritageParent.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/BulkHeritageParent.py deleted file mode 100644 index 6ac3589e59..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/BulkHeritageParent.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_HeritageLFNParent_ - -SQLite implementation of DBSBufferFiles.HeritageLFNParent -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.BulkHeritageParent import BulkHeritageParent as MySQLBulkHeritageParent - - -class BulkHeritageParent(DBFormatter): - """ - Commit parentage information in bulk - - - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Delete.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Delete.py deleted file mode 100644 index 5f984dee60..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Delete.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of Delete - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.Delete import Delete as MySQLDelete - -class Delete(MySQLDelete): - """ - - SQLite implementation of Delete - - """ - - def GetUpdateDeleteDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Exists.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Exists.py deleted file mode 100644 index d86a8e6cc8..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/Exists.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" -_Exists_ - -SQLite implementation of Files.Exists -""" - -__all__ = [] - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.Exists import Exists as MySQLExists - -class Exists(MySQLExists): - """ - _Exists_ - - SQLite implementation of Files.Exists - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/ExistsForAccountant.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/ExistsForAccountant.py deleted file mode 100644 index 7d0b2168b2..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/ExistsForAccountant.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -_ExistsForAccountant_ - -SQLite implementation of Files.ExistsForAccountant -""" - -__all__ = [] - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.ExistsForAccountant import ExistsForAccountant as MySQLExistsForAccountant - -class ExistsForAccountant(MySQLExistsForAccountant): - """ - This is highly specialized. You shouldn't confuse it with - a normal Exists DAO - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetBlock.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetBlock.py deleted file mode 100644 index eddbced6ce..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetBlock.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_GetBlock_ - -SQLite implementation of DBSBufferFiles.GetBlock -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetBlock import GetBlock as MySQLGetBlock - -class GetBlock(MySQLGetBlock): - """ - _GetBlock_ - - SQLite implementation of DBSBufferFiles.GetBlock - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetByID.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetByID.py deleted file mode 100644 index 8a093905f0..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetByID.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetByID - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetByID import GetByID as MySQLGetByID - -class GetByID(MySQLGetByID): - """ - - SQLite implementation of GetByID - - """ - - def GetUpdateGetByIDDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetByLFN.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetByLFN.py deleted file mode 100644 index 1b3cdbf4a6..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetByLFN.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetByLFN - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetByLFN import GetByLFN as MySQLGetByLFN - -class GetByLFN(MySQLGetByLFN): - """ - - SQLite implementation of GetByLFN - - """ - - def GetUpdateGetByLFNDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetChecksum.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetChecksum.py deleted file mode 100644 index 2504141e23..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetChecksum.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python - -""" -SQLite implementation of GetChecksum -""" - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetChecksum import GetChecksum as MySQLGetChecksum - -class GetChecksum(MySQLGetChecksum): - """ - Identical to MySQL Version - - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetChildren.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetChildren.py deleted file mode 100644 index 1a78a9f26e..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetChildren.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -""" -_GetChildren_ - -SQLite implementation of DBSBufferFile.GetChildren -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetChildren import GetChildren as MySQLGetChildren - -class GetChildren(MySQLGetChildren): - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetLocation.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetLocation.py deleted file mode 100644 index d9b6f94e78..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetLocation.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetLocation - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetLocation import GetLocation as MySQLGetLocation - -class GetLocation(MySQLGetLocation): - """ - - SQLite implementation of GetLocation - - """ - - def GetUpdateGetLocationDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetParentStatus.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetParentStatus.py deleted file mode 100644 index 2b6a2ca222..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetParentStatus.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -""" -_GetParentStatus_ - -SQLite implementation of DBSBufferFile.GetParentStatus -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetParentStatus import GetParentStatus as MySQLGetParentStatus - -class GetParentStatus(MySQLGetParentStatus): - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetParents.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetParents.py deleted file mode 100644 index dd9f06867c..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetParents.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetParents - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetParents import GetParents as MySQLGetParents - -class GetParents(MySQLGetParents): - """ - - SQLite implementation of GetParents - - """ - - def GetUpdateGetParentsDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetRunLumiFile.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetRunLumiFile.py deleted file mode 100644 index 7c81d48e81..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/GetRunLumiFile.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetRunLumiFile - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetRunLumiFile import GetRunLumiFile as MySQLGetRunLumiFile - -class GetRunLumiFile(MySQLGetRunLumiFile): - """ - - SQLite implementation of GetRunLumiFile - - """ - - def GetUpdateGetRunLumiFileDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/HeritageLFNChild.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/HeritageLFNChild.py deleted file mode 100644 index f1be08e3cc..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/HeritageLFNChild.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -""" -_HeritageLFNChild_ - -SQLite implementation of DBSBufferFiles.HeritageLFNChild -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.HeritageLFNChild import \ - HeritageLFNChild as MySQLHeritageLFNChild - -class HeritageLFNChild(MySQLHeritageLFNChild): - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/HeritageLFNParent.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/HeritageLFNParent.py deleted file mode 100644 index f62f6f8d53..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/HeritageLFNParent.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -""" -_HeritageLFNParent_ - -SQLite implementation of DBSBufferFiles.HeritageLFNParent -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.HeritageLFNParent import \ - HeritageLFNParent as MySQLHeritageLFNParent - -class HeritageLFNParent(MySQLHeritageLFNParent): - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/LoadBulkFilesByID.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/LoadBulkFilesByID.py deleted file mode 100644 index 932f59f406..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/LoadBulkFilesByID.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_GetByID_ - -SQLite implementation of DBSBufferFiles.GetByID -""" - - - - -import logging - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles import LoadBulkFilesByID as MySQLLoadBulkFilesByID - -class LoadBulkFilesByID(MySQLLoadBulkFilesByID): - """ - Same as MySQL - - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetBlock.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetBlock.py deleted file mode 100644 index 3ecb2053f4..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetBlock.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_SetBlock - -SQLite implementation of DBSBufferFiles.SetBlock -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.SetBlock import SetBlock as MySQLSetBlock - -class SetBlock(MySQLSetBlock): - """ - _SetBlock_ - - SQLite implementation of SetBlock - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetLocation.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetLocation.py deleted file mode 100644 index 8801aec374..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetLocation.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of SetLocation - -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.SetLocation import SetLocation as MySQLSetLocation - -class SetLocation(MySQLSetLocation): - """ - - SQLite implementation of SetLocation - - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetLocationByLFN.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetLocationByLFN.py deleted file mode 100644 index 1e9286a426..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetLocationByLFN.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -""" -_SetLocationByLFN_ - -SQLite implementation of DBSBuffer.SetLocationByLFN -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.SetLocationByLFN import SetLocationByLFN as MySQLSetLocationByLFN - -class SetLocationByLFN(MySQLSetLocationByLFN): - """ - Set the location of files using lfn as the key - - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetStatus.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetStatus.py deleted file mode 100644 index 7acdfbee41..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/SetStatus.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_SetStatus_ - -SQLite implementation of DBSBufferFiles.SetStatus -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.SetStatus import SetStatus as MySQLSetStatus - -class SetStatus(MySQLSetStatus): - """ - _SetStatus_ - - SQLite implementation of SetStatus - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/__init__.py b/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/__init__.py deleted file mode 100644 index d85419a1ba..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/DBSBufferFiles/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python -""" -_WMBS_ - -Workload Management Book-keeping Service - - -""" -__all__ = [] diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/Destroy.py b/src/python/WMComponent/DBS3Buffer/SQLite/Destroy.py deleted file mode 100644 index 6df5d20573..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/Destroy.py +++ /dev/null @@ -1,37 +0,0 @@ -#/usr/bin/env python -""" -_Destroy_ - -Implementation of DBSBuffer.Destroy for SQLite -""" - - - - -import threading - -from WMCore.Database.DBCreator import DBCreator - -class Destroy(DBCreator): - def __init__(self): - """ - _init_ - - Call the base class's constructor and create all necessary tables, - constraints and inserts. - """ - myThread = threading.currentThread() - DBCreator.__init__(self, myThread.logger, myThread.dbi) - - - self.delete["11dbsbuffer_block"] = "DROP TABLE dbsbuffer_block" - self.delete["10dbsbuffer_dataset"] = "DROP TABLE dbsbuffer_dataset" - self.delete["09dbsbuffer_algo"] = "DROP TABLE dbsbuffer_algo" - self.delete["08dbsbuffer_algo_dataset_assoc"] = "DROP TABLE dbsbuffer_algo_dataset_assoc" - self.delete["07dbsbuffer_file"] = "DROP TABLE dbsbuffer_file" - self.delete["06dbsbuffer_file_parent"] = "DROP TABLE dbsbuffer_file_parent" - self.delete["05dbsbuffer_file_runlumi_map"] = "DROP TABLE dbsbuffer_file_runlumi_map" - self.delete["04dbsbuffer_location"] = "DROP TABLE dbsbuffer_location" - self.delete["03dbsbuffer_file_location"] = "DROP TABLE dbsbuffer_file_location" - self.delete["02dbsbuffer_checksum_type"] = "DROP TABLE dbsbuffer_checksum_type" - self.delete["01dbsbuffer_file_checksums"] = "DROP TABLE dbsbuffer_file_checksums" diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/FindDASToUpload.py b/src/python/WMComponent/DBS3Buffer/SQLite/FindDASToUpload.py deleted file mode 100644 index 3eaa07fe0d..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/FindDASToUpload.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python - -""" -This code should load the necessary information regarding -dataset-algo combinations from the DBSBuffer. - -SQLite version - -""" - - - - -from WMComponent.DBSUpload.Database.MySQL.FindDASToUpload import FindDASToUpload as MySQLFindDASToUpload - - -class FindDASToUpload(MySQLFindDASToUpload): - """ - Find Uploadable DAS - - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/GetBlockFromDataset.py b/src/python/WMComponent/DBS3Buffer/SQLite/GetBlockFromDataset.py deleted file mode 100644 index 02fbf4eee3..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/GetBlockFromDataset.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -_ListDataset_ - -SQLite implementation of DBSBuffer.GetBlockFromDataset -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.GetBlockFromDataset import GetBlockFromDataset as MySQLGetBlockFromDataset - -class GetBlockFromDataset(MySQLGetBlockFromDataset): - """ - _GetBlockFromDataset_ - - Identical to MySQL version - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/GetOpenBlocks.py b/src/python/WMComponent/DBS3Buffer/SQLite/GetOpenBlocks.py deleted file mode 100644 index b003da649a..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/GetOpenBlocks.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -""" -_GetOpenBlocks_ - -SQLite implementation of DBSBuffer.GetOpenBlocks -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.GetOpenBlocks import GetOpenBlocks as MySQLGetOpenBlocks - -class GetOpenBlocks(MySQLGetOpenBlocks): - """ - Identical to MySQL version - - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/ListAlgo.py b/src/python/WMComponent/DBS3Buffer/SQLite/ListAlgo.py deleted file mode 100644 index 092b521d64..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/ListAlgo.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" -_ListAlgo_ - -SQLite implementation of DBSBuffer.ListAlgo -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.ListAlgo import ListAlgo as MySQLListAlgo - -class ListAlgo(MySQLListAlgo): - """ - _ListAlgo_ - - Retrieve information about an algorithm from the DBSBuffer. This is mainly - used by the unit tests to verify that the NewAlgo DAO is working correctly. - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/ListAlgoDatasetAssoc.py b/src/python/WMComponent/DBS3Buffer/SQLite/ListAlgoDatasetAssoc.py deleted file mode 100644 index 1da1de1897..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/ListAlgoDatasetAssoc.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" -_ListAlgoDatasetAssoc_ - -SQLite implementation of DBSBuffer.ListAlgoDatasetAssoc -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.ListAlgoDatasetAssoc import ListAlgoDatasetAssoc as MySQLListAlgoDatasetAssoc - -class ListAlgoDatasetAssoc(MySQLListAlgoDatasetAssoc): - """ - _ListAlgoDatasetAssoc_ - - Retrieve information about a dataset/algorthim association in the DBSBuffer. - This is mostly used by the unit tests. - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/ListDataset.py b/src/python/WMComponent/DBS3Buffer/SQLite/ListDataset.py deleted file mode 100644 index 42dea3e724..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/ListDataset.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" -_ListDataset_ - -SQLite implementation of DBSBuffer.ListDataset -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.ListDataset import ListDataset as MySQLListDataset - -class ListDataset(MySQLListDataset): - """ - _ListDataset_ - - Retrieve information about a dataset in the DBSBuffer. This is mostly used - by the unit tests. - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/LoadBlocksByDAS.py b/src/python/WMComponent/DBS3Buffer/SQLite/LoadBlocksByDAS.py deleted file mode 100644 index 8fc7794002..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/LoadBlocksByDAS.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -_LoadBlocksByDAS_ - -SQLite implementation of LoadBlocksByDAS -""" - - - - -from WMComponent.DBSUpload.Database.MySQL.LoadBlocksByDAS import LoadBlocksByDAS as MySQLLoadBlocksByDAS - -class LoadBlocksByDAS(MySQLLoadBlocksByDAS): - """ - _LoadBlocksByDAS_ - - Untested; same as MySQL - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/LoadDBSFilesByDAS.py b/src/python/WMComponent/DBS3Buffer/SQLite/LoadDBSFilesByDAS.py deleted file mode 100644 index 6a3eacba08..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/LoadDBSFilesByDAS.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" -_LoadDBSFilesByDAS_ - -SQLite implementation of LoadDBSFilesByDAS -""" - - - - -import logging - -from WMComponent.DBSUpload.Database.MySQL.LoadDBSFilesByDAS import LoadDBSFilesByDAS as MySQLLoadDBSFilesByDAS - -class LoadDBSFilesByDAS(MySQLLoadDBSFilesByDAS): - """ - _LoadDBSFilesByDAS_ - - SQLite implementation, untested - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/NewAlgo.py b/src/python/WMComponent/DBS3Buffer/SQLite/NewAlgo.py deleted file mode 100644 index 681bbe4fa5..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/NewAlgo.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -""" -_NewAlgo_ - -SQLite implementation of DBSBuffer.NewAlgo -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.NewAlgo import NewAlgo as MySQLNewAlgo - -class NewAlgo(MySQLNewAlgo): - """ - _NewAlgo_ - - Add a new algorithm to the DBSBuffer. This will do nothing if an algorithm - with the given parameters already exists. - """ - sql = """INSERT INTO dbsbuffer_algo (app_name, app_ver, app_fam, pset_hash, - config_content, in_dbs) - SELECT :app_name, :app_ver, :app_fam, :pset_hash, - :config_content, 0 WHERE NOT EXISTS - (SELECT * FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash)""" - - existsSQL = """SELECT id FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash""" diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/NewDataset.py b/src/python/WMComponent/DBS3Buffer/SQLite/NewDataset.py deleted file mode 100644 index 20b0aef1fa..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/NewDataset.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -""" -_NewDataset_ - -SQLite implementation of DBSBuffer.NewDataset -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.NewDataset import NewDataset as MySQLNewDataset - -class NewDataset(MySQLNewDataset): - """ - _NewDataset_ - - Add a new dataset to DBS Buffer - """ - sql = """INSERT INTO dbsbuffer_dataset (path) - SELECT :path WHERE NOT EXISTS - (SELECT * FROM dbsbuffer_dataset WHERE path = :path)""" - existsSQL = "SELECT id FROM dbsbuffer_dataset WHERE path = :path" diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/SetBlockStatus.py b/src/python/WMComponent/DBS3Buffer/SQLite/SetBlockStatus.py deleted file mode 100644 index 6dd7492ec5..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/SetBlockStatus.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -""" -_SetBlockStatus_ - -Create a new block in dbsbuffer_block. -""" - - - - -import threading -import exceptions - -from WMComponent.DBSUpload.Database.MySQL.SetBlockStatus import SetBlockStatus as MySQLSetBlockStatus - - -class SetBlockStatus(MySQLSetBlockStatus): - """ - _SetBlockStatus_ - - """ - sql = """INSERT INTO dbsbuffer_block (blockname, location) - SELECT :block, (SELECT id FROM dbsbuffer_location WHERE se_name = :location) - WHERE NOT EXISTS (SELECT blockname FROM dbsbuffer_block WHERE blockname = :block - and location = (SELECT id FROM dbsbuffer_location WHERE se_name = :location)) - """ diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/Status.py b/src/python/WMComponent/DBS3Buffer/SQLite/Status.py deleted file mode 100644 index a8ad1bc232..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/Status.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -""" -_Status_ - -SQLite implementation of DBSBuffer.Status -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.Status import Status as MySQLStatus - -class Status(MySQLStatus): - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/UpdateAlgo.py b/src/python/WMComponent/DBS3Buffer/SQLite/UpdateAlgo.py deleted file mode 100644 index f2146c485a..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/UpdateAlgo.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -""" -_DBSBuffer.UpdateAlgo_ - -Add PSetHash to Algo in DBS Buffer - -""" - - - - -import threading - -from WMComponent.DBSBuffer.Database.MySQL.UpdateAlgo import UpdateAlgo as MySQLUpdateAlgo - -class UpdateAlgo(MySQLUpdateAlgo): - """ - _DBSBuffer.UpdateAlgo_ - - Add PSetHash to Algo in DBS Buffer - - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/UpdateAlgoDatasetAssoc.py b/src/python/WMComponent/DBS3Buffer/SQLite/UpdateAlgoDatasetAssoc.py deleted file mode 100644 index d1eee1605a..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/UpdateAlgoDatasetAssoc.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_UpdateAlgoDatasetAssoc_ - -SQLite implementation of DBSBuffer.UpdateAlgoDatasetAssoc -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.UpdateAlgoDatasetAssoc import UpdateAlgoDatasetAssoc as MySQLUpdateAlgoDatasetAssoc - -class UpdateAlgoDatasetAssoc(MySQLUpdateAlgoDatasetAssoc): - """ - _UpdateAlgoDatasetAssoc_ - - Update the in_dbs column for a particular algo/dataset association. - """ - pass diff --git a/src/python/WMComponent/DBS3Buffer/SQLite/__init__.py b/src/python/WMComponent/DBS3Buffer/SQLite/__init__.py deleted file mode 100644 index 90690013b5..0000000000 --- a/src/python/WMComponent/DBS3Buffer/SQLite/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -""" -__init__ - -""" -__all__ = [] diff --git a/src/python/WMComponent/DBSBuffer/DBSBuffer.py b/src/python/WMComponent/DBSBuffer/DBSBuffer.py deleted file mode 100644 index a6295dfe48..0000000000 --- a/src/python/WMComponent/DBSBuffer/DBSBuffer.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -#pylint: disable-msg=E1101,E1103,C0103,R0902 - -""" -Performs bulk DBS File insertion by buffering, subscribes to job-completion event. -""" - - - - -import logging - -# harness class that encapsulates the basic component logic. -from WMCore.Agent.Harness import Harness -# we do not import failure handlers as they are dynamicly -# loaded from the config file. -from WMCore.WMFactory import WMFactory - - -#['__call__', '__doc__', '__init__', '__module__', '__str__', 'config', 'handleMessage', 'initInThread', 'initialization', 'logState', 'postInitialization', 'preInitialization', 'prepareToStart', 'publishItem', 'startComponent'] - -class DBSBuffer(Harness): - - def __init__(self, config): - # call the base class - Harness.__init__(self, config) - print "DBSBuffer.__init__" - - def preInitialization(self): - print "DBSBuffer.preInitialization" - - # use a factory to dynamically load handlers. - factory = WMFactory('generic') - self.messages['JobSuccess'] = \ - factory.loadObject(self.config.DBSBuffer.jobSuccessHandler, self) diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/AlgoDatasetAssoc.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/AlgoDatasetAssoc.py deleted file mode 100644 index 763eacf344..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/AlgoDatasetAssoc.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -""" -_AlgoDatasetAssoc_ - -SQLite implementation of DBSBuffer.AlgoDatasetAssoc -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.AlgoDatasetAssoc import AlgoDatasetAssoc as MySQLAlgoDatasetAssoc - -class AlgoDatasetAssoc(MySQLAlgoDatasetAssoc): - """ - _AlgoDatasetAssoc_ - - Associate an algorithm to a dataset and return the ID of the association. - """ - sql = """INSERT INTO dbsbuffer_algo_dataset_assoc (algo_id, dataset_id) - SELECT (SELECT id FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash) AS algo_id, - (SELECT id FROM dbsbuffer_dataset WHERE path = :path) AS dataset_id - WHERE NOT EXISTS - (SELECT * FROM dbsbuffer_algo_dataset_assoc WHERE algo_id = - (SELECT id FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash) AND dataset_id = - (SELECT id FROM dbsbuffer_dataset WHERE path = :path))""" - - idsql = """SELECT id FROM dbsbuffer_algo_dataset_assoc WHERE algo_id = - (SELECT id FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash) AND dataset_id = - (SELECT id FROM dbsbuffer_dataset WHERE path = :path)""" diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/CountBlocks.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/CountBlocks.py deleted file mode 100644 index 7640520248..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/CountBlocks.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -_CountBlocks_ - -SQLite implementation of DBSBuffer.CountBlocks -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.CountBlocks import CountBlocks as MySQLCountBlocks - -class CountBlocks(MySQLCountBlocks): - """ - _CountBlocks_ - - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/CountFiles.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/CountFiles.py deleted file mode 100644 index 9980e18677..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/CountFiles.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -_CountFiles_ - -SQLite implementation of DBSBuffer.CountFiles -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.CountFiles import CountFiles as MySQLCountFiles - -class CountFiles(MySQLCountFiles): - """ - _CountFiles_ - - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/Create.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/Create.py deleted file mode 100644 index da08ae391a..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/Create.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -_Create_DBSBuffer_ - -Implementation of Create_DBSBuffer for SQLite -""" - - - - -import logging -import threading - -from WMCore.Database.DBCreator import DBCreator - -class Create(DBCreator): - - def __init__(self): - """ - _init_ - - Call the base class's constructor and create all necessary tables, - constraints and inserts. - """ - myThread = threading.currentThread() - DBCreator.__init__(self, myThread.logger, myThread.dbi) - - self.create["01dbsbuffer_dataset"] = \ - """CREATE TABLE dbsbuffer_dataset - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - path VARCHAR(500) UNIQUE NOT NULL - )""" - - self.create["02dbsbuffer_algo"] = \ - """CREATE TABLE dbsbuffer_algo - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - app_name VARCHAR(100), - app_ver VARCHAR(100), - app_fam VARCHAR(100), - pset_hash VARCHAR(700), - config_content LONGTEXT, - in_dbs INTEGER DEFAULT 0, - UNIQUE(app_name, app_ver, app_fam, pset_hash) - )""" - - self.create["03dbsbuffer_algo_dataset_assoc"] = \ - """CREATE TABLE dbsbuffer_algo_dataset_assoc - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - algo_id BIGINT UNSIGNED, - dataset_id BIGINT UNSINGED, - unmigrated_files BIGINT UNSIGNED DEFAULT 0, - in_dbs INTEGER DEFAULT 0, - FOREIGN KEY (algo_id) REFERENCES dbsbuffer_algo(id) - ON DELETE CASCADE, - FOREIGN KEY (dataset_id) REFERENCES dbsbuffer_dataset(id) - ON DELETE CASCADE - )""" - - self.create["04dbsbuffer_file"] = \ - """CREATE TABLE dbsbuffer_file ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - lfn VARCHAR(255) NOT NULL, - filesize BIGINT, - events INTEGER, - cksum BIGINT UNSIGNED, - dataset_algo INTEGER NOT NULL, - block_id INTEGER, - status varchar(20), - LastModificationDate BIGINT)""" - - self.create["05dbsbuffer_file_parent"] = \ - """CREATE TABLE dbsbuffer_file_parent ( - child INTEGER NOT NULL, - parent INTEGER NOT NULL, - FOREIGN KEY (child) references dbsbuffer_file(id) - ON DELETE CASCADE, - FOREIGN KEY (parent) references dbsbuffer_file(id), - UNIQUE(child, parent))""" - - self.create["06dbsbuffer_file_runlumi_map"] = \ - """CREATE TABLE dbsbuffer_file_runlumi_map ( - filename INTEGER NOT NULL, - run INTEGER NOT NULL, - lumi INTEGER NOT NULL, - FOREIGN KEY (filename) references dbsbuffer_file(id) - ON DELETE CASCADE)""" - - self.create["07dbsbuffer_location"] = \ - """CREATE TABLE dbsbuffer_location ( - id INTEGER PRIMARY KEY, - se_name VARCHAR(255) NOT NULL, - UNIQUE(se_name))""" - - self.create["08dbsbuffer_file_location"] = \ - """CREATE TABLE dbsbuffer_file_location ( - filename INTEGER NOT NULL, - location INTEGER NOT NULL, - UNIQUE(filename, location), - FOREIGN KEY(filename) REFERENCES dbsbuffer_file(id) - ON DELETE CASCADE, - FOREIGN KEY(location) REFERENCES dbsbuffer_location(id) - ON DELETE CASCADE)""" - - self.create["10dbsbuffer_block"] = \ - """CREATE TABLE dbsbuffer_block ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - blockname VARCHAR(250) NOT NULL, - location INTEGER NOT NULL, - create_time INTEGER, - status VARCHAR(20), - UNIQUE(blockname, location) - )""" - - self.create["11dbsbuffer_checksum_type"] = \ - """CREATE TABLE dbsbuffer_checksum_type ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type VARCHAR(255) )""" - - - self.create["12dbsbuffer_file_checksums"] = \ - """CREATE TABLE dbsbuffer_file_checksums ( - fileid INTEGER, - typeid INTEGER, - cksum VARCHAR(100), - UNIQUE(fileid, typeid))""" diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Add.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Add.py deleted file mode 100644 index 8e7f0e9020..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Add.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of AddFile - -""" - - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.Add import Add as MySQLAdd - -class Add(MySQLAdd): - """ - SQLite implementation of AddFile - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddCKType.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddCKType.py deleted file mode 100644 index 03c0512202..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddCKType.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python - -""" -SQLite implementation of AddCKType -""" - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.AddCKType import AddCKType as MySQLAddCKType - -class AddCKType(MySQLAddCKType): - """ - Identical to MySQL version - """ diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddChecksum.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddChecksum.py deleted file mode 100644 index 47898c8ab2..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddChecksum.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python - -""" -SQLite implementation of AddChecksum -""" - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.AddChecksum import AddChecksum as MySQLAddChecksum - -class AddChecksum(MySQLAddChecksum): - """ - Add a Checksum - - """ - - sql = """INSERT INTO dbsbuffer_file_checksums (fileid, typeid, cksum) - SELECT :fileid, (SELECT id FROM dbsbuffer_checksum_type WHERE type = :cktype), :cksum - WHERE NOT EXISTS (SELECT fileid FROM dbsbuffer_file_checksums WHERE - fileid = :fileid AND typeid = (SELECT id FROM dbsbuffer_checksum_type WHERE type = :cktype))""" diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddChecksumByLFN.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddChecksumByLFN.py deleted file mode 100644 index 0efbffe3b6..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddChecksumByLFN.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python - -""" -Oracle implementation of AddChecksumByLFN -""" - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.AddChecksumByLFN import AddChecksumByLFN as MySQLAddChecksumByLFN - -class AddChecksumByLFN(MySQLAddChecksumByLFN): - """ - - Add Checksums using lfn as key - - """ - sql = MySQLAddChecksumByLFN.sql.replace('FROM dual', '') diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddLocation.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddLocation.py deleted file mode 100644 index c9f3eced31..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddLocation.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -""" -_AddLocation_ - -SQLite implementation of DBSBufferFiles.AddLocation -""" - -from WMCore.Database.DBFormatter import DBFormatter -from WMCore.Database.Transaction import Transaction - -class AddLocation(DBFormatter): - sql = """INSERT INTO dbsbuffer_location (se_name) - SELECT :location AS se_name WHERE NOT EXISTS - (SELECT se_name FROM dbsbuffer_location WHERE se_name = :location)""" - - existsSQL = """SELECT se_name, id FROM dbsbuffer_location - WHERE se_name = :location""" - - def execute(self, siteName, conn = None, transaction = False): - - if type(siteName) == str: - binds = {"location": siteName} - else: - binds = [] - for aLocation in siteName: - binds.append({"location": aLocation}) - - myTransaction = Transaction(self.dbi) - myTransaction.begin() - - nameMap = {} - self.dbi.processData(self.sql, binds, conn = conn, - transaction = transaction) - results = self.dbi.processData(self.existsSQL, binds, - conn = myTransaction.conn, - transaction = True) - results = self.format(results) - for result in results: - nameMap[result[0]] = int(result[1]) - - myTransaction.commit() - return nameMap diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddRunLumi.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddRunLumi.py deleted file mode 100644 index 55b8cbe3ac..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/AddRunLumi.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of AddRunLumi - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.AddRunLumi import AddRunLumi as MySQLAddRunLumi - -class AddRunLumi(MySQLAddRunLumi): - """ - - SQLite implementation of AddRunLumi - - """ - - sql = """INSERT INTO dbsbuffer_file_runlumi_map (filename, run, lumi) - SELECT id, :run, :lumi from dbsbuffer_file - where lfn = :lfn""" - - def GetUpdateAddDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/BulkHeritageParent.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/BulkHeritageParent.py deleted file mode 100644 index 6ac3589e59..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/BulkHeritageParent.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_HeritageLFNParent_ - -SQLite implementation of DBSBufferFiles.HeritageLFNParent -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.BulkHeritageParent import BulkHeritageParent as MySQLBulkHeritageParent - - -class BulkHeritageParent(DBFormatter): - """ - Commit parentage information in bulk - - - """ diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Delete.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Delete.py deleted file mode 100644 index 5f984dee60..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Delete.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of Delete - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.Delete import Delete as MySQLDelete - -class Delete(MySQLDelete): - """ - - SQLite implementation of Delete - - """ - - def GetUpdateDeleteDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Exists.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Exists.py deleted file mode 100644 index d86a8e6cc8..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/Exists.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" -_Exists_ - -SQLite implementation of Files.Exists -""" - -__all__ = [] - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.Exists import Exists as MySQLExists - -class Exists(MySQLExists): - """ - _Exists_ - - SQLite implementation of Files.Exists - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/ExistsForAccountant.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/ExistsForAccountant.py deleted file mode 100644 index 7d0b2168b2..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/ExistsForAccountant.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -_ExistsForAccountant_ - -SQLite implementation of Files.ExistsForAccountant -""" - -__all__ = [] - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.ExistsForAccountant import ExistsForAccountant as MySQLExistsForAccountant - -class ExistsForAccountant(MySQLExistsForAccountant): - """ - This is highly specialized. You shouldn't confuse it with - a normal Exists DAO - """ diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetBlock.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetBlock.py deleted file mode 100644 index eddbced6ce..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetBlock.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_GetBlock_ - -SQLite implementation of DBSBufferFiles.GetBlock -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetBlock import GetBlock as MySQLGetBlock - -class GetBlock(MySQLGetBlock): - """ - _GetBlock_ - - SQLite implementation of DBSBufferFiles.GetBlock - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetByID.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetByID.py deleted file mode 100644 index 8a093905f0..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetByID.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetByID - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetByID import GetByID as MySQLGetByID - -class GetByID(MySQLGetByID): - """ - - SQLite implementation of GetByID - - """ - - def GetUpdateGetByIDDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetByLFN.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetByLFN.py deleted file mode 100644 index 1b3cdbf4a6..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetByLFN.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetByLFN - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetByLFN import GetByLFN as MySQLGetByLFN - -class GetByLFN(MySQLGetByLFN): - """ - - SQLite implementation of GetByLFN - - """ - - def GetUpdateGetByLFNDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetChecksum.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetChecksum.py deleted file mode 100644 index 2504141e23..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetChecksum.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python - -""" -SQLite implementation of GetChecksum -""" - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetChecksum import GetChecksum as MySQLGetChecksum - -class GetChecksum(MySQLGetChecksum): - """ - Identical to MySQL Version - - """ diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetChildren.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetChildren.py deleted file mode 100644 index 1a78a9f26e..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetChildren.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -""" -_GetChildren_ - -SQLite implementation of DBSBufferFile.GetChildren -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetChildren import GetChildren as MySQLGetChildren - -class GetChildren(MySQLGetChildren): - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetLocation.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetLocation.py deleted file mode 100644 index d9b6f94e78..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetLocation.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetLocation - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetLocation import GetLocation as MySQLGetLocation - -class GetLocation(MySQLGetLocation): - """ - - SQLite implementation of GetLocation - - """ - - def GetUpdateGetLocationDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetParentStatus.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetParentStatus.py deleted file mode 100644 index 2b6a2ca222..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetParentStatus.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -""" -_GetParentStatus_ - -SQLite implementation of DBSBufferFile.GetParentStatus -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetParentStatus import GetParentStatus as MySQLGetParentStatus - -class GetParentStatus(MySQLGetParentStatus): - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetParents.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetParents.py deleted file mode 100644 index dd9f06867c..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetParents.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetParents - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetParents import GetParents as MySQLGetParents - -class GetParents(MySQLGetParents): - """ - - SQLite implementation of GetParents - - """ - - def GetUpdateGetParentsDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetRunLumiFile.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetRunLumiFile.py deleted file mode 100644 index 7c81d48e81..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/GetRunLumiFile.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of GetRunLumiFile - -""" - - - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.GetRunLumiFile import GetRunLumiFile as MySQLGetRunLumiFile - -class GetRunLumiFile(MySQLGetRunLumiFile): - """ - - SQLite implementation of GetRunLumiFile - - """ - - def GetUpdateGetRunLumiFileDialect(self): - - return 'SQLite' diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/HeritageLFNChild.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/HeritageLFNChild.py deleted file mode 100644 index f1be08e3cc..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/HeritageLFNChild.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -""" -_HeritageLFNChild_ - -SQLite implementation of DBSBufferFiles.HeritageLFNChild -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.HeritageLFNChild import \ - HeritageLFNChild as MySQLHeritageLFNChild - -class HeritageLFNChild(MySQLHeritageLFNChild): - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/HeritageLFNParent.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/HeritageLFNParent.py deleted file mode 100644 index f62f6f8d53..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/HeritageLFNParent.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -""" -_HeritageLFNParent_ - -SQLite implementation of DBSBufferFiles.HeritageLFNParent -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.HeritageLFNParent import \ - HeritageLFNParent as MySQLHeritageLFNParent - -class HeritageLFNParent(MySQLHeritageLFNParent): - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/LoadBulkFilesByID.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/LoadBulkFilesByID.py deleted file mode 100644 index 932f59f406..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/LoadBulkFilesByID.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_GetByID_ - -SQLite implementation of DBSBufferFiles.GetByID -""" - - - - -import logging - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles import LoadBulkFilesByID as MySQLLoadBulkFilesByID - -class LoadBulkFilesByID(MySQLLoadBulkFilesByID): - """ - Same as MySQL - - """ diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetBlock.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetBlock.py deleted file mode 100644 index 3ecb2053f4..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetBlock.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_SetBlock - -SQLite implementation of DBSBufferFiles.SetBlock -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.SetBlock import SetBlock as MySQLSetBlock - -class SetBlock(MySQLSetBlock): - """ - _SetBlock_ - - SQLite implementation of SetBlock - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetLocation.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetLocation.py deleted file mode 100644 index 8801aec374..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetLocation.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" - -SQLite implementation of SetLocation - -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.SetLocation import SetLocation as MySQLSetLocation - -class SetLocation(MySQLSetLocation): - """ - - SQLite implementation of SetLocation - - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetLocationByLFN.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetLocationByLFN.py deleted file mode 100644 index 1e9286a426..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetLocationByLFN.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -""" -_SetLocationByLFN_ - -SQLite implementation of DBSBuffer.SetLocationByLFN -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.SetLocationByLFN import SetLocationByLFN as MySQLSetLocationByLFN - -class SetLocationByLFN(MySQLSetLocationByLFN): - """ - Set the location of files using lfn as the key - - """ diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetStatus.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetStatus.py deleted file mode 100644 index 7acdfbee41..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/SetStatus.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_SetStatus_ - -SQLite implementation of DBSBufferFiles.SetStatus -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.DBSBufferFiles.SetStatus import SetStatus as MySQLSetStatus - -class SetStatus(MySQLSetStatus): - """ - _SetStatus_ - - SQLite implementation of SetStatus - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/__init__.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/__init__.py deleted file mode 100644 index d85419a1ba..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/DBSBufferFiles/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python -""" -_WMBS_ - -Workload Management Book-keeping Service - - -""" -__all__ = [] diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/Destroy.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/Destroy.py deleted file mode 100644 index 6df5d20573..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/Destroy.py +++ /dev/null @@ -1,37 +0,0 @@ -#/usr/bin/env python -""" -_Destroy_ - -Implementation of DBSBuffer.Destroy for SQLite -""" - - - - -import threading - -from WMCore.Database.DBCreator import DBCreator - -class Destroy(DBCreator): - def __init__(self): - """ - _init_ - - Call the base class's constructor and create all necessary tables, - constraints and inserts. - """ - myThread = threading.currentThread() - DBCreator.__init__(self, myThread.logger, myThread.dbi) - - - self.delete["11dbsbuffer_block"] = "DROP TABLE dbsbuffer_block" - self.delete["10dbsbuffer_dataset"] = "DROP TABLE dbsbuffer_dataset" - self.delete["09dbsbuffer_algo"] = "DROP TABLE dbsbuffer_algo" - self.delete["08dbsbuffer_algo_dataset_assoc"] = "DROP TABLE dbsbuffer_algo_dataset_assoc" - self.delete["07dbsbuffer_file"] = "DROP TABLE dbsbuffer_file" - self.delete["06dbsbuffer_file_parent"] = "DROP TABLE dbsbuffer_file_parent" - self.delete["05dbsbuffer_file_runlumi_map"] = "DROP TABLE dbsbuffer_file_runlumi_map" - self.delete["04dbsbuffer_location"] = "DROP TABLE dbsbuffer_location" - self.delete["03dbsbuffer_file_location"] = "DROP TABLE dbsbuffer_file_location" - self.delete["02dbsbuffer_checksum_type"] = "DROP TABLE dbsbuffer_checksum_type" - self.delete["01dbsbuffer_file_checksums"] = "DROP TABLE dbsbuffer_file_checksums" diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/GetBlockFromDataset.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/GetBlockFromDataset.py deleted file mode 100644 index 02fbf4eee3..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/GetBlockFromDataset.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -_ListDataset_ - -SQLite implementation of DBSBuffer.GetBlockFromDataset -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.GetBlockFromDataset import GetBlockFromDataset as MySQLGetBlockFromDataset - -class GetBlockFromDataset(MySQLGetBlockFromDataset): - """ - _GetBlockFromDataset_ - - Identical to MySQL version - """ diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/GetOpenBlocks.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/GetOpenBlocks.py deleted file mode 100644 index b003da649a..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/GetOpenBlocks.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -""" -_GetOpenBlocks_ - -SQLite implementation of DBSBuffer.GetOpenBlocks -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.GetOpenBlocks import GetOpenBlocks as MySQLGetOpenBlocks - -class GetOpenBlocks(MySQLGetOpenBlocks): - """ - Identical to MySQL version - - """ diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/ListAlgo.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/ListAlgo.py deleted file mode 100644 index 092b521d64..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/ListAlgo.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" -_ListAlgo_ - -SQLite implementation of DBSBuffer.ListAlgo -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.ListAlgo import ListAlgo as MySQLListAlgo - -class ListAlgo(MySQLListAlgo): - """ - _ListAlgo_ - - Retrieve information about an algorithm from the DBSBuffer. This is mainly - used by the unit tests to verify that the NewAlgo DAO is working correctly. - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/ListAlgoDatasetAssoc.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/ListAlgoDatasetAssoc.py deleted file mode 100644 index 1da1de1897..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/ListAlgoDatasetAssoc.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" -_ListAlgoDatasetAssoc_ - -SQLite implementation of DBSBuffer.ListAlgoDatasetAssoc -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.ListAlgoDatasetAssoc import ListAlgoDatasetAssoc as MySQLListAlgoDatasetAssoc - -class ListAlgoDatasetAssoc(MySQLListAlgoDatasetAssoc): - """ - _ListAlgoDatasetAssoc_ - - Retrieve information about a dataset/algorthim association in the DBSBuffer. - This is mostly used by the unit tests. - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/ListDataset.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/ListDataset.py deleted file mode 100644 index 42dea3e724..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/ListDataset.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -""" -_ListDataset_ - -SQLite implementation of DBSBuffer.ListDataset -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.ListDataset import ListDataset as MySQLListDataset - -class ListDataset(MySQLListDataset): - """ - _ListDataset_ - - Retrieve information about a dataset in the DBSBuffer. This is mostly used - by the unit tests. - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/NewAlgo.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/NewAlgo.py deleted file mode 100644 index 681bbe4fa5..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/NewAlgo.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -""" -_NewAlgo_ - -SQLite implementation of DBSBuffer.NewAlgo -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.NewAlgo import NewAlgo as MySQLNewAlgo - -class NewAlgo(MySQLNewAlgo): - """ - _NewAlgo_ - - Add a new algorithm to the DBSBuffer. This will do nothing if an algorithm - with the given parameters already exists. - """ - sql = """INSERT INTO dbsbuffer_algo (app_name, app_ver, app_fam, pset_hash, - config_content, in_dbs) - SELECT :app_name, :app_ver, :app_fam, :pset_hash, - :config_content, 0 WHERE NOT EXISTS - (SELECT * FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash)""" - - existsSQL = """SELECT id FROM dbsbuffer_algo WHERE app_name = :app_name AND - app_ver = :app_ver AND app_fam = :app_fam AND - pset_hash = :pset_hash""" diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/NewDataset.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/NewDataset.py deleted file mode 100644 index 20b0aef1fa..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/NewDataset.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -""" -_NewDataset_ - -SQLite implementation of DBSBuffer.NewDataset -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.NewDataset import NewDataset as MySQLNewDataset - -class NewDataset(MySQLNewDataset): - """ - _NewDataset_ - - Add a new dataset to DBS Buffer - """ - sql = """INSERT INTO dbsbuffer_dataset (path) - SELECT :path WHERE NOT EXISTS - (SELECT * FROM dbsbuffer_dataset WHERE path = :path)""" - existsSQL = "SELECT id FROM dbsbuffer_dataset WHERE path = :path" diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/Status.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/Status.py deleted file mode 100644 index a8ad1bc232..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/Status.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -""" -_Status_ - -SQLite implementation of DBSBuffer.Status -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.Status import Status as MySQLStatus - -class Status(MySQLStatus): - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/UpdateAlgo.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/UpdateAlgo.py deleted file mode 100644 index f2146c485a..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/UpdateAlgo.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -""" -_DBSBuffer.UpdateAlgo_ - -Add PSetHash to Algo in DBS Buffer - -""" - - - - -import threading - -from WMComponent.DBSBuffer.Database.MySQL.UpdateAlgo import UpdateAlgo as MySQLUpdateAlgo - -class UpdateAlgo(MySQLUpdateAlgo): - """ - _DBSBuffer.UpdateAlgo_ - - Add PSetHash to Algo in DBS Buffer - - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/UpdateAlgoDatasetAssoc.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/UpdateAlgoDatasetAssoc.py deleted file mode 100644 index d1eee1605a..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/UpdateAlgoDatasetAssoc.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -""" -_UpdateAlgoDatasetAssoc_ - -SQLite implementation of DBSBuffer.UpdateAlgoDatasetAssoc -""" - - - - -from WMComponent.DBSBuffer.Database.MySQL.UpdateAlgoDatasetAssoc import UpdateAlgoDatasetAssoc as MySQLUpdateAlgoDatasetAssoc - -class UpdateAlgoDatasetAssoc(MySQLUpdateAlgoDatasetAssoc): - """ - _UpdateAlgoDatasetAssoc_ - - Update the in_dbs column for a particular algo/dataset association. - """ - pass diff --git a/src/python/WMComponent/DBSBuffer/Database/SQLite/__init__.py b/src/python/WMComponent/DBSBuffer/Database/SQLite/__init__.py deleted file mode 100644 index 90690013b5..0000000000 --- a/src/python/WMComponent/DBSBuffer/Database/SQLite/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -""" -__init__ - -""" -__all__ = [] diff --git a/src/python/WMComponent/DBSBuffer/DefaultConfig.py b/src/python/WMComponent/DBSBuffer/DefaultConfig.py deleted file mode 100644 index 93757ef8f9..0000000000 --- a/src/python/WMComponent/DBSBuffer/DefaultConfig.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -#pylint: disable-msg=E1101,E1103,C0103,R0902 -""" -Defines default config values for DBSBuffer specific -parameters. -""" -__all__ = [] - - - - -from WMCore.Agent.Configuration import Configuration - -config = Configuration() -config.component_("DBSBuffer") -#The log level of the component. -config.DBSBuffer.logLevel = 'INFO' -#The namespace of the buffer. Used to load the module as daemon -config.DBSBuffer.namespace = 'WMComponent.DBSBuffer.DBSBuffer' -# maximum number of threads we want to deal -# with messages per pool. -config.DBSBuffer.maxThreads = 1 -# -# JobSuccess Handler -# -config.DBSBuffer.jobSuccessHandler = \ - 'WMComponent.DBSBuffer.Handler.JobSuccess' diff --git a/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py b/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py index 800c9be558..37b1b824d3 100644 --- a/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py +++ b/test/python/WMComponent_t/DBS3Buffer_t/DBSUpload_t.py @@ -1,18 +1,12 @@ #!/usr/bin/env python #pylint: disable-msg=E1101, W6501, W0142, C0103, W0401, E1103 # W0401: I am not going to import all those functions by hand - - """ DBSUpload test TestDBSUpload module and the harness """ - - - - import os import threading import time @@ -26,6 +20,7 @@ from WMCore.DAOFactory import DAOFactory from WMCore.Services.UUID import makeUUID from WMCore.DataStructs.Run import Run +from WMCore.Services.UUID import makeUUID from WMCore.Agent.Configuration import Configuration from WMCore.Agent.HeartbeatAPI import HeartbeatAPI @@ -77,19 +72,14 @@ def tearDown(self): tearDown function for unittest """ - self.testInit.clearDatabase() - def getConfig(self): """ _getConfig_ - This creates the actual config file used by the component - + This creates the actual config file used by the component. """ - - config = Configuration() #First the general stuff @@ -106,68 +96,63 @@ def getConfig(self): config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") - config.component_("DBSUpload") config.DBSUpload.pollInterval = 10 config.DBSUpload.logLevel = 'DEBUG' config.DBSUpload.DBSBlockMaxFiles = 1 config.DBSUpload.DBSBlockMaxTime = 2 config.DBSUpload.DBSBlockMaxSize = 999999999999 - config.DBSUpload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter' - #config.DBSUpload.dbsUrl = 'https://localhost:1443/dbs/prod/global/DBSWriter' + #config.DBSUpload.dbsUrl = "https://cmsweb-testbed.cern.ch/dbs/dev/global/DBSWriter" + #config.DBSUpload.dbsUrl = "https://dbs3-dev01.cern.ch/dbs/prod/global/DBSWriter" + config.DBSUpload.dbsUrl = "https://localhost:1443/dbs/dev/global/DBSWriter" config.DBSUpload.namespace = 'WMComponent.DBS3Buffer.DBSUpload' config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBSUpload.nProcesses = 1 config.DBSUpload.dbsWaitTime = 0.1 - - return config - def getFiles(self, name, tier, nFiles = 12, site = "malpaquet", nLumis = 1): """ - Create some quick dummy test files - - + _getFiles_ + + Create some dummy test files. """ - files = [] + (acqEra, procVer) = name.split("-") + baseLFN = "/store/data/%s/Cosmics/RECO/%s/000/143/316/" % (acqEra, procVer) for f in range(nFiles): - testFile = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size = 1024, - events = 20, checksums = {'cksum': 1}) - testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", + testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, + events = 20, checksums = {"cksum": 1}) + testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") - testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) + testFile.setDatasetPath("/Cosmics/%s-%s/RECO" % (acqEra, procVer)) lumis = [] for i in range(nLumis): lumis.append((f * 1000000) + i) testFile.addRun(Run( 1, *lumis)) - testFile.setAcquisitionEra(name.split('-')[0]) + testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("0") - testFile.setGlobalTag("Weird") + testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation(site) files.append(testFile) - - testFileChild = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-child.root' %(name, site), size = 1024, - events = 10, checksums = {'cksum': 1}) - testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", - appFam = "RECO", psetHash = "GIBBERISH", + baseLFN = "/store/data/%s/Cosmics/RAW-RECO/%s/000/143/316/" % (acqEra, procVer) + testFileChild = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", + size = 1024, events = 10, checksums = {'cksum': 1}) + testFileChild.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", + appFam = "RAW-RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") - testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name)) + testFileChild.setDatasetPath("/Cosmics/%s-%s/RAW-RECO" %(acqEra, procVer)) testFileChild.addRun(Run( 1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x['lfn'] for x in files]) - - return files - @attr('integration') def testA_basicFunction(self): """ @@ -193,16 +178,13 @@ def testA_basicFunction(self): dbsUploader.close() raise - - name = "ThisIsATest_%s" % (makeUUID()) + name = "ThisIsATest%s" % (int(time.time())) tier = "RECO" nFiles = 12 name = name.replace('-', '_') name = '%s-v0' % name files = self.getFiles(name = name, tier = tier, nFiles = nFiles) - datasetPath = '/%s/%s/%s' % (name, name, tier) - shortPath = '/%s/%s' % (name, name) - + datasetPath = "/Cosmics/%s/%s" % (name, tier) try: dbsUploader.algorithm() @@ -214,16 +196,13 @@ def testA_basicFunction(self): # Now look in DBS try: - print "Should have just tried with name %s" % name - result = dbsApi.listPrimaryDatasets(primary_ds_name = name) - self.assertEqual(len(result), 1) - self.assertEqual(result[0]['primary_ds_name'], name) result = dbsApi.listDatasets(dataset = datasetPath, detail = True, dataset_access_type = 'PRODUCTION') self.assertEqual(len(result), 1) - self.assertEqual(result[0]['data_tier_name'], u'RECO') + self.assertEqual(result[0]['data_tier_name'], 'RECO') self.assertEqual(result[0]['processing_version'], 0) self.assertEqual(result[0]['acquisition_era_name'], name.split('-')[0]) + result = dbsApi.listFiles(dataset=datasetPath) self.assertEqual(len(result), 11) except: @@ -259,38 +238,5 @@ def testA_basicFunction(self): return - @attr('integration') - def testB_DONOTUSE(self): - return - config = self.getConfig() - config.DBSUpload.DBSBlockMaxFiles = 1 - config.DBSUpload.copyBlock = True - - name = "ThisIsATest_%s" % (makeUUID()) - tier = "RECO" - nFiles = 10 - name = name.replace('-', '_') - name = '%s-v0' % name - files = self.getFiles(name = name, tier = tier, nFiles = nFiles, nLumis = 3) - datasetPath = '/%s/%s/%s' % (name, name, tier) - - from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller - dbsUploader = DBSUploadPoller(config = config) - dbsUtil = DBSBufferUtil() - from dbs.apis.dbsClient import DbsApi - dbsApi = DbsApi(url = config.DBSUpload.dbsUrl) - - # This should do nothing - # Just making sure we don't crash - try: - dbsUploader.algorithm() - except: - dbsUploader.close() - raise - - - - - if __name__ == '__main__': unittest.main()