Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RSE gateway endpoints #336

Merged
merged 5 commits into from
Sep 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions src/lib/core/dto/rse-dto.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { BaseDTO, BaseStreamableDTO } from "@/lib/sdk/dto";
import { RSE } from "../entity/rucio";
import { RSE, RSEAttribute, RSEProtocol } from "@/lib/core/entity/rucio";

/**
* The Data Transfer Object for the ListRSEsEndpoint which contains the stream
Expand All @@ -9,4 +9,16 @@ export interface ListRSEsDTO extends BaseStreamableDTO {}
/**
* Data Transfer Object for GET RSE Endpoint
*/
export interface RSEDTO extends BaseDTO, RSE {}
export interface RSEDTO extends BaseDTO, RSE {}

/**
* Data Transfer Object for GET RSE Protocols Endpoint
*/
export interface RSEProtocolDTO extends BaseDTO {
protocols: RSEProtocol[]
}

/**
* Data Transfer Object for GET RSE Attributes Endpoint
*/
export interface RSEAttributeDTO extends RSEAttribute, BaseDTO {}
6 changes: 3 additions & 3 deletions src/lib/core/entity/rucio.ts
Original file line number Diff line number Diff line change
Expand Up @@ -216,12 +216,12 @@ export type RSEProtocol = {
read: number,
write: number,
delete: number,
tpc: number,
tpc?: number, // TODO: what is this?
tpcwrite: number,
tpcread: number,
},
updated_at: DateISO,
created_at: DateISO,
updated_at?: DateISO, // TODO: rucio does not provide this
created_at?: DateISO, // TODO: rucio does not provide this
}

export type RSEAttribute = {
Expand Down
30 changes: 23 additions & 7 deletions src/lib/core/port/secondary/rse-gateway-output-port.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,33 @@
import { ListRSEsDTO, RSEDTO } from "../../dto/rse-dto";
import { ListRSEsDTO, RSEAttributeDTO, RSEDTO, RSEProtocolDTO } from "@/lib/core/dto/rse-dto";

export default interface RSEGatewayOutputPort {

/**
* Streams the RSEs for the given RSE Expression.
* Gets the details of the given RSE.
* @param rucioAuthToken A valid Rucio Auth Token.
* @param rseExpression The RSE Expression to list RSEs for.
* @param rseName The RSE to get.
*/
listRSEs(rucioAuthToken: string, rseExpression: string): Promise<ListRSEsDTO>
getRSE(rucioAuthToken: string, rseName: string): Promise<RSEDTO>

/**
* Gets the RSE for the given RSE name.
* Lists all supported protocols for a given RSE.
* @param rucioAuthToken A valid Rucio Auth Token.
* @param rseName The RSE name to get.
* @param rseName The RSE to list protocols for.
*/
getRSE(rucioAuthToken: string, rseName: string): Promise<RSEDTO>
getRSEProtocols(rucioAuthToken: string, rseName: string): Promise<RSEProtocolDTO>

/**
* Get all supported attributes for a given RSE.
* @param rucioAuthToken A valid Rucio Auth Token.
* @param rseName The RSE to list attributes for.
*/
getRSEAttributes(rucioAuthToken: string, rseName: string): Promise<RSEAttributeDTO>

/**
* Streams the RSEs for the given RSE Expression.
* @param rucioAuthToken A valid Rucio Auth Token.
* @param rseExpression The RSE Expression to list RSEs for.
*/
listRSEs(rucioAuthToken: string, rseExpression: string): Promise<ListRSEsDTO>

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { RSEAttributeDTO } from "@/lib/core/dto/rse-dto";
import { BaseEndpoint } from "@/lib/sdk/gateway-endpoints";
import { HTTPRequest } from "@/lib/sdk/http";
import { Response } from "node-fetch";

export default class GetRSEAttributesEndpoint extends BaseEndpoint<RSEAttributeDTO> {
constructor(
private rucioAuthToken: string,
private rseName: string,
) {
super();
}

async initialize(): Promise<void> {
await super.initialize();
this.url = `${this.rucioHost}/rses/${this.rseName}/attr`;
const request: HTTPRequest = {
method: 'GET',
url: this.url,
headers: {
'X-Rucio-Auth-Token': this.rucioAuthToken,
'Content-Type': 'application/json',
},
body: null,
params: undefined
}
this.request = request;
this.initialized = true;
}

async reportErrors(statusCode: number, response: Response): Promise<RSEAttributeDTO | undefined> {
const errorMessage = await response.json()
const error = {
status: 'error',
errorCode: statusCode,
errorMessage: errorMessage,
errorName: 'Rucio Server Error',
errorType: 'gateway_endpoint_error',
}
return error as RSEAttributeDTO
}

createDTO(data: Object): RSEAttributeDTO {
return {
status: 'success',
...data
} as RSEAttributeDTO
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import { RSEProtocolDTO } from "@/lib/core/dto/rse-dto";
import { RSEProtocol } from "@/lib/core/entity/rucio";
import { BaseEndpoint } from "@/lib/sdk/gateway-endpoints";
import { HTTPRequest } from "@/lib/sdk/http";
import { Response } from "node-fetch";
import { convertToRSEDTO, covertToRSEProtocol, TRucioRSEProtocol } from "../rse-gateway-utils";

export default class GetRSEProtocolsEndpoint extends BaseEndpoint<RSEProtocolDTO> {
constructor(
private rucioAuthToken: string,
private rseName: string,
){
super()
}

async initialize(): Promise<void> {
await super.initialize()
this.url = `${this.rucioHost}/rses/${this.rseName}/protocols`
const request: HTTPRequest = {
method: 'GET',
url: this.url,
headers: {
'X-Rucio-Auth-Token': this.rucioAuthToken,
'Content-Type': 'application/json',
},
body: null,
params: undefined
}
this.request = request
this.initialized = true
}

async reportErrors(statusCode: number, response: Response): Promise<RSEProtocolDTO | undefined> {
const errorMessage = await response.json()
const error: RSEProtocolDTO = {
status: 'error',
errorCode: statusCode,
errorMessage: errorMessage,
errorName: 'Rucio Server Error',
errorType: 'gateway_endpoint_error',
protocols: []
}
return error
}

createDTO(data: Object): RSEProtocolDTO {
const protocols: TRucioRSEProtocol[] = data as TRucioRSEProtocol[]
const rseProtocols: RSEProtocol[] = []
const rseName = this.rseName
protocols.map((protocol: TRucioRSEProtocol) => {
const rseProtocol = covertToRSEProtocol(protocol, rseName)
rseProtocols.push(rseProtocol)
})

const rseDTO: RSEProtocolDTO = {
status: 'success',
protocols: rseProtocols
}
return rseDTO

}

}
57 changes: 55 additions & 2 deletions src/lib/infrastructure/gateway/rse-gateway/rse-gateway-utils.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { RSEDTO } from '@/lib/core/dto/rse-dto';
import { RSEType } from '@/lib/core/entity/rucio';
import { RSEDTO, RSEProtocolDTO } from '@/lib/core/dto/rse-dto';
import { RSEProtocol, RSEType } from '@/lib/core/entity/rucio';
/**
* Represents the data returned by Rucio Server for a RSE.
*/
Expand Down Expand Up @@ -31,6 +31,27 @@ export type TRucioRSE = {
updated_at: string;
}

export type TRucioRSEProtocol = {
domains: {
[key: string]: {
delete: number;
read: number;
third_party_copy_read?: number;
third_party_copy_write?: number;
write: number;
};
};
extended_attributes: {
space_token: string;
web_service_path: string;
} | null;
hostname: string;
impl: string;
port: number;
prefix: string;
scheme: string;
}

/**
* Converts a string representing the RSE Type to a {@link RSEType} enum value.
* @param rseType A string representing the RSE Type
Expand Down Expand Up @@ -62,4 +83,36 @@ export function convertToRSEDTO(rse: TRucioRSE): RSEDTO {
staging_area: rse.staging_area,
}
return dto
}

/**
* Converts rucio server RSEProtocol to a {@link RSEProtocol} object.
* @param protocol The RSE Protocol of type {@link TRucioRSEProtocol} to convert to a DTO
* @param rseName The name of the RSE for which the protocols are being retrieved
* @returns A {@link RSEProtocol} object
*/
export function covertToRSEProtocol(protocol: TRucioRSEProtocol, rseName: string): RSEProtocol {
const domains = protocol.domains
const rseProtocol: RSEProtocol = {
rseid: rseName,
scheme: protocol.scheme,
hostname: protocol.hostname,
port: protocol.port,
prefix: protocol.prefix,
impl: protocol.impl,
priorities_lan: {
read: domains.lan.read || 0,
write: domains.lan.write || 0,
delete: domains.lan.delete || 0,
},
priorities_wan: {
read: domains.wan.read || 0,
write: domains.wan.write || 0,
delete: domains.wan.delete || 0,
tpcread: domains.wan.third_party_copy_read || 0,
tpcwrite: domains.wan.third_party_copy_write || 0,
},

}
return rseProtocol
}
28 changes: 22 additions & 6 deletions src/lib/infrastructure/gateway/rse-gateway/rse-gateway.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,31 @@
import { ListRSEsDTO, RSEDTO } from "@/lib/core/dto/rse-dto";
import { ListRSEsDTO, RSEAttributeDTO, RSEDTO, RSEProtocolDTO } from "@/lib/core/dto/rse-dto";
import RSEGatewayOutputPort from "@/lib/core/port/secondary/rse-gateway-output-port";
import { injectable } from "inversify";
import ListRSEsEndpoint from "./endpoints/list-rses-endpoint";
import GetRSEEndpoint from "./endpoints/get-rse-endpoint";
import GetRSEAttributesEndpoint from "./endpoints/get-rse-attributes-endpoint";
import GetRSEProtocolsEndpoint from "./endpoints/get-rse-protocols-endpoint";

@injectable()
export default class RSEGateway implements RSEGatewayOutputPort {
async getRSE(rucioAuthToken: string, rseName: string): Promise<RSEDTO> {
const endpoint = new GetRSEEndpoint(rucioAuthToken, rseName)
const dto = await endpoint.fetch()
return dto
}

getRSEProtocols(rucioAuthToken: string, rseName: string): Promise<RSEProtocolDTO> {
const endpoint = new GetRSEProtocolsEndpoint(rucioAuthToken, rseName)
const dto = endpoint.fetch()
return dto
}

async getRSEAttributes(rucioAuthToken: string, rseName: string): Promise<RSEAttributeDTO> {
const endpoint = new GetRSEAttributesEndpoint(rucioAuthToken, rseName)
const dto = await endpoint.fetch()
return dto
}

async listRSEs(rucioAuthToken: string, rseExpression: string): Promise<ListRSEsDTO> {
const endpoint: ListRSEsEndpoint = new ListRSEsEndpoint(rucioAuthToken, rseExpression)
const errorDTO = await endpoint.fetch()
Expand All @@ -18,9 +38,5 @@ export default class RSEGateway implements RSEGatewayOutputPort {
}
return Promise.resolve(errorDTO)
}
async getRSE(rucioAuthToken: string, rseName: string): Promise<RSEDTO> {
const endpoint = new GetRSEEndpoint(rucioAuthToken, rseName)
const dto = await endpoint.fetch()
return dto
}

}
49 changes: 49 additions & 0 deletions test/gateway/rse/rse-gateway-get-rse-attributes.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import RSEGatewayOutputPort from "@/lib/core/port/secondary/rse-gateway-output-port";
import appContainer from "@/lib/infrastructure/ioc/container-config";
import GATEWAYS from "@/lib/infrastructure/ioc/ioc-symbols-gateway";
import MockRucioServerFactory, { MockEndpoint } from "test/fixtures/rucio-server";

describe('RSEGateway GET RSE Attributes Endpoint Tests', () => {
beforeEach(() => {
fetchMock.doMock();
const getRSEAttributesMockEndpoint: MockEndpoint = {
url: `${MockRucioServerFactory.RUCIO_HOST}/rses/MOCK3/attr`,
method: 'GET',
includes: '/rses/MOCK3/attr',
response: {
status: 200,
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
"ISP": "CERN- LHC",
"MOCK3": true,
"continent": "EU",
"country_name": "Switzerland",
"region_code": "07",
"time_zone": "Europe/Zurich"
})
}
}

MockRucioServerFactory.createMockRucioServer(true, [getRSEAttributesMockEndpoint]);
})

afterEach(() => {
fetchMock.dontMock();
})

test('it should get RSE attributes', async () => {
const rseGateway: RSEGatewayOutputPort = appContainer.get<RSEGatewayOutputPort>(GATEWAYS.RSE)
const rseAttributes = await rseGateway.getRSEAttributes(MockRucioServerFactory.VALID_RUCIO_TOKEN, 'MOCK3')
expect(rseAttributes).toEqual({
status: 'success',
ISP: "CERN- LHC",
MOCK3: true,
continent: "EU",
country_name: "Switzerland",
region_code: "07",
time_zone: "Europe/Zurich"
})
})
})
Loading