From ce2ff41898813fe8d570f5722816aca98ae409bd Mon Sep 17 00:00:00 2001 From: Dorian De Rosa Date: Mon, 24 Jul 2023 11:45:57 +0200 Subject: [PATCH] feat(formation): transformation fiches formations initiales --- .env.example | 1 + .env.test | 1 + apps/cli/src/command/transform.command.ts | 5 + .../configuration/configuration.ts | 2 +- apps/formations-initiales/src/index.ts | 7 +- .../application-service/index.ts | 35 ++++ .../transformer-flux-onisep.usecase.ts | 22 +++ .../domain/model/1jeune1solution/index.ts | 13 ++ .../src/transformation/domain/model/flux.ts | 15 ++ .../domain/model/onisep/index.ts | 16 ++ .../formations-initiales.repository.ts | 7 + .../onisep/convertir.domain-service.ts | 20 ++ .../configuration/configuration.ts | 83 ++++++++ .../configuration/log.decorator.ts | 28 +++ .../configuration/logger-strategy.ts | 36 ++++ .../infrastructure/gateway/index.ts | 81 ++++++++ .../minio-formations-initiales.repository.ts | 117 +++++++++++ .../transformation/infrastructure/index.ts | 39 ++++ .../transform-flow-onisep.sub-command.ts | 32 +++ .../transformer-flux-onisep.usecase.test.ts | 65 ++++++ ...io-formations-initiales.repository.test.ts | 185 ++++++++++++++++++ 21 files changed, 806 insertions(+), 4 deletions(-) create mode 100644 apps/formations-initiales/src/transformation/application-service/index.ts create mode 100644 apps/formations-initiales/src/transformation/application-service/transformer-flux-onisep.usecase.ts create mode 100644 apps/formations-initiales/src/transformation/domain/model/1jeune1solution/index.ts create mode 100644 apps/formations-initiales/src/transformation/domain/model/flux.ts create mode 100644 apps/formations-initiales/src/transformation/domain/model/onisep/index.ts create mode 100644 apps/formations-initiales/src/transformation/domain/service/formations-initiales.repository.ts create mode 100644 apps/formations-initiales/src/transformation/domain/service/onisep/convertir.domain-service.ts create mode 100644 apps/formations-initiales/src/transformation/infrastructure/configuration/configuration.ts create mode 100644 apps/formations-initiales/src/transformation/infrastructure/configuration/log.decorator.ts create mode 100644 apps/formations-initiales/src/transformation/infrastructure/configuration/logger-strategy.ts create mode 100644 apps/formations-initiales/src/transformation/infrastructure/gateway/index.ts create mode 100644 apps/formations-initiales/src/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository.ts create mode 100644 apps/formations-initiales/src/transformation/infrastructure/index.ts create mode 100644 apps/formations-initiales/src/transformation/infrastructure/sub-command/transform-flow-onisep.sub-command.ts create mode 100644 apps/formations-initiales/test/transformation/application-service/transformer-flux-onisep.usecase.test.ts create mode 100644 apps/formations-initiales/test/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository.test.ts diff --git a/.env.example b/.env.example index 48ed9fc54..a1ee558aa 100644 --- a/.env.example +++ b/.env.example @@ -40,6 +40,7 @@ FORMATIONS_INITIALES_ONISEP_DIRECTORY_NAME= FORMATIONS_INITIALES_ONISEP_FLUX_URL= FORMATIONS_INITIALES_ONISEP_RAW_FILE_EXTENSION= FORMATIONS_INITIALES_EXTRACT_LOG_LEVEL= +FORMATIONS_INITIALES_MINIO_RAW_BUCKET_NAME= FORMATIONS_INITIALES_MINIO_RAW_BUCKET_NAME= diff --git a/.env.test b/.env.test index 7dc028fcf..5204c8e5a 100644 --- a/.env.test +++ b/.env.test @@ -40,6 +40,7 @@ FORMATIONS_INITIALES_ONISEP_DIRECTORY_NAME=onisep FORMATIONS_INITIALES_ONISEP_FLUX_URL=https://some.url.com/onisep FORMATIONS_INITIALES_ONISEP_RAW_FILE_EXTENSION=.xml FORMATIONS_INITIALES_EXTRACT_LOG_LEVEL=debug +FORMATIONS_INITIALES_MINIO_RAW_BUCKET_NAME=formations-initiales-raw FORMATIONS_INITIALES_MINIO_RAW_BUCKET_NAME=formations-initiales-raw diff --git a/apps/cli/src/command/transform.command.ts b/apps/cli/src/command/transform.command.ts index 9514e4525..1b1804fc0 100644 --- a/apps/cli/src/command/transform.command.ts +++ b/apps/cli/src/command/transform.command.ts @@ -4,6 +4,10 @@ import { TransformFlowTousMobilisesSubCommand, } from "@evenements/src/transformation/infrastructure/sub-command/transform-flow-tous-mobilises.sub-command"; +import { + TransformFlowOnisepSubCommand, +} from "@formations-initiales/src/transformation/infrastructure/sub-command/transform-flow-onisep.sub-command"; + import { TransformFlowImmojeuneSubCommand, } from "@logements/src/transformation/infrastructure/sub-command/transform-flow-immojeune.sub-command"; @@ -30,6 +34,7 @@ import { TransformFlowJobteaserSubCommand, TransformFlowStagefrCompressedSubCommand, TransformFlowStagefrUncompressedSubCommand, + TransformFlowOnisepSubCommand, ], }) export class TransformCommand extends CommandRunner { diff --git a/apps/evenements/src/transformation/infrastructure/configuration/configuration.ts b/apps/evenements/src/transformation/infrastructure/configuration/configuration.ts index b409caa52..b22172247 100644 --- a/apps/evenements/src/transformation/infrastructure/configuration/configuration.ts +++ b/apps/evenements/src/transformation/infrastructure/configuration/configuration.ts @@ -49,7 +49,7 @@ export class ConfigurationFactory extends ConfigurationValidator { const DEFAULT_HISTORY_DIRECTORY_NAME = "history"; return { - CONTEXT: "evenements/extraction", + CONTEXT: "evenements/transformation", DOMAINE: "Évènements", FLOWS: [ getOrError("EVENTS_TOUS_MOBILISES_NAME"), diff --git a/apps/formations-initiales/src/index.ts b/apps/formations-initiales/src/index.ts index 32e918290..7b6456237 100644 --- a/apps/formations-initiales/src/index.ts +++ b/apps/formations-initiales/src/index.ts @@ -1,10 +1,11 @@ import { Module } from "@nestjs/common"; -import { Extraction } from "./extraction"; +import { Extraction } from "@formations-initiales/src/extraction"; +import { Transformation } from "@formations-initiales/src/transformation/infrastructure"; @Module({ - imports: [Extraction], - exports: [Extraction], + imports: [Extraction, Transformation], + exports: [Extraction, Transformation], }) export class FormationsInitiales { } diff --git a/apps/formations-initiales/src/transformation/application-service/index.ts b/apps/formations-initiales/src/transformation/application-service/index.ts new file mode 100644 index 000000000..82647999e --- /dev/null +++ b/apps/formations-initiales/src/transformation/application-service/index.ts @@ -0,0 +1,35 @@ +import { Module } from "@nestjs/common"; + +import { + TransformerFluxOnisep, +} from "@formations-initiales/src/transformation/application-service/transformer-flux-onisep.usecase"; +import { + FormationsInitialesRepository, +} from "@formations-initiales/src/transformation/domain/service/formations-initiales.repository"; +import { Convertir } from "@formations-initiales/src/transformation/domain/service/onisep/convertir.domain-service"; +import { Gateways } from "@formations-initiales/src/transformation/infrastructure/gateway"; + +import { Shared } from "@shared/src"; + +@Module({ + imports: [Gateways, Shared], + providers: [{ + provide: Convertir, + inject: [], + useFactory: (): Convertir => { + return new Convertir(); + }, + }, { + provide: TransformerFluxOnisep, + inject: [ + "FormationsInitialesRepository", + Convertir, + ], + useFactory: (formationsInitialesRepository: FormationsInitialesRepository, convertirDomainService: Convertir): TransformerFluxOnisep => { + return new TransformerFluxOnisep(formationsInitialesRepository, convertirDomainService); + }, + }], + exports: [TransformerFluxOnisep], +}) +export class Usecases { +} diff --git a/apps/formations-initiales/src/transformation/application-service/transformer-flux-onisep.usecase.ts b/apps/formations-initiales/src/transformation/application-service/transformer-flux-onisep.usecase.ts new file mode 100644 index 000000000..879343d27 --- /dev/null +++ b/apps/formations-initiales/src/transformation/application-service/transformer-flux-onisep.usecase.ts @@ -0,0 +1,22 @@ +import { FluxTransformation } from "@formations-initiales/src/transformation/domain/model/flux"; +import { Onisep } from "@formations-initiales/src/transformation/domain/model/onisep"; +import { + FormationsInitialesRepository, +} from "@formations-initiales/src/transformation/domain/service/formations-initiales.repository"; +import { Convertir } from "@formations-initiales/src/transformation/domain/service/onisep/convertir.domain-service"; + +import { Usecase } from "@shared/src/application-service/usecase"; + +export class TransformerFluxOnisep implements Usecase { + constructor( + private readonly formationsInitialesRepository: FormationsInitialesRepository, + private readonly convertir: Convertir, + ) { + } + + public async executer(flux: FluxTransformation): Promise { + const contenuDuFlux = await this.formationsInitialesRepository.recuperer(flux); + + await this.formationsInitialesRepository.sauvegarder(this.convertir.depuisOnisep(contenuDuFlux), flux); + } +} diff --git a/apps/formations-initiales/src/transformation/domain/model/1jeune1solution/index.ts b/apps/formations-initiales/src/transformation/domain/model/1jeune1solution/index.ts new file mode 100644 index 000000000..cdd4a3aa9 --- /dev/null +++ b/apps/formations-initiales/src/transformation/domain/model/1jeune1solution/index.ts @@ -0,0 +1,13 @@ +export namespace UnJeuneUneSolution { + export type FormationInitiale = { + identifiant: string + intitule: string + duree: string + certification: string + niveauEtudesVise: string + description: string + attendusParcoursup: string + conditionsAcces: string + poursuiteEtudes: string + } +} diff --git a/apps/formations-initiales/src/transformation/domain/model/flux.ts b/apps/formations-initiales/src/transformation/domain/model/flux.ts new file mode 100644 index 000000000..251a879ed --- /dev/null +++ b/apps/formations-initiales/src/transformation/domain/model/flux.ts @@ -0,0 +1,15 @@ +import { Flux } from "@shared/src/domain/model/flux"; + +export class FluxTransformation extends Flux { + constructor( + nom: string, + public readonly dossierHistorisation: string, + public readonly extensionFichierBrut: string, + public readonly extensionFichierTransforme: string + ) { + super(nom, extensionFichierBrut); + this.dossierHistorisation = dossierHistorisation; + this.extensionFichierBrut = extensionFichierBrut; + this.extensionFichierTransforme = extensionFichierTransforme; + } +} diff --git a/apps/formations-initiales/src/transformation/domain/model/onisep/index.ts b/apps/formations-initiales/src/transformation/domain/model/onisep/index.ts new file mode 100644 index 000000000..b39032d78 --- /dev/null +++ b/apps/formations-initiales/src/transformation/domain/model/onisep/index.ts @@ -0,0 +1,16 @@ +export namespace Onisep { + export type Contenu = { + formations: { formation: Array } + } + export type Formation = { + identifiant: string + libelle_complet: string + duree_formation: string + niveau_certification: string + niveau_etudes: { libelle: string } + descriptif_format_court: string + attendus: string + descriptif_acces: string + descriptif_poursuite_etudes: string + } +} diff --git a/apps/formations-initiales/src/transformation/domain/service/formations-initiales.repository.ts b/apps/formations-initiales/src/transformation/domain/service/formations-initiales.repository.ts new file mode 100644 index 000000000..b4ae1a423 --- /dev/null +++ b/apps/formations-initiales/src/transformation/domain/service/formations-initiales.repository.ts @@ -0,0 +1,7 @@ +import { UnJeuneUneSolution } from "@formations-initiales/src/transformation/domain/model/1jeune1solution"; +import { FluxTransformation } from "@formations-initiales/src/transformation/domain/model/flux"; + +export interface FormationsInitialesRepository { + recuperer(flux: FluxTransformation): Promise; + sauvegarder(formationInitiales: Array, flux: FluxTransformation): Promise; +} diff --git a/apps/formations-initiales/src/transformation/domain/service/onisep/convertir.domain-service.ts b/apps/formations-initiales/src/transformation/domain/service/onisep/convertir.domain-service.ts new file mode 100644 index 000000000..fd20756cf --- /dev/null +++ b/apps/formations-initiales/src/transformation/domain/service/onisep/convertir.domain-service.ts @@ -0,0 +1,20 @@ +import { UnJeuneUneSolution } from "@formations-initiales//src/transformation/domain/model/1jeune1solution"; +import { Onisep } from "@formations-initiales/src/transformation/domain/model/onisep"; + +export class Convertir { + public depuisOnisep(formations: Onisep.Contenu): Array { + return formations.formations.formation.map((formation) => { + return { + identifiant: formation.identifiant, + intitule: formation.libelle_complet, + duree: formation.duree_formation, + certification: formation.niveau_certification, + niveauEtudesVise: formation.niveau_etudes.libelle, + description: formation.descriptif_format_court, + attendusParcoursup: formation.attendus, + conditionsAcces: formation.descriptif_acces, + poursuiteEtudes: formation.descriptif_poursuite_etudes, + }; + }); + } +} diff --git a/apps/formations-initiales/src/transformation/infrastructure/configuration/configuration.ts b/apps/formations-initiales/src/transformation/infrastructure/configuration/configuration.ts new file mode 100644 index 000000000..962567e81 --- /dev/null +++ b/apps/formations-initiales/src/transformation/infrastructure/configuration/configuration.ts @@ -0,0 +1,83 @@ +import { + ConfigurationValidator, + Environment, + SentryConfiguration, +} from "@shared/src/infrastructure/configuration/configuration"; +import { Domaine, LogLevel } from "@shared/src/infrastructure/configuration/logger"; + +type MinioConfiguration = { + ACCESS_KEY: string + HISTORY_DIRECTORY_NAME: string + PORT: number + RAW_BUCKET_NAME: string + SECRET_KEY: string + URL: string + TRANSFORMED_BUCKET_NAME: string +} + +type TaskConfiguration = { + DIRECTORY_NAME: string + FLUX_URL: string + NAME: string + RAW_FILE_EXTENSION: string + TRANSFORMED_FILE_EXTENSION: string +} + +export type Configuration = { + CONTEXT: string + DOMAINE: Domaine + FLOWS: Array + ONISEP: TaskConfiguration + LOGGER_LOG_LEVEL: LogLevel + MINIO: MinioConfiguration + NODE_ENV: Environment + SENTRY: SentryConfiguration + TEMPORARY_DIRECTORY_PATH: string +} + +export class ConfigurationFactory extends ConfigurationValidator { + public static createRoot(): { formationsInitialesTransformation: Configuration } { + return { formationsInitialesTransformation: ConfigurationFactory.create() }; + } + + public static create(): Configuration { + const { getOrError, getOrDefault } = ConfigurationFactory; + const DEFAULT_RAW_BUCKET_NAME = "raw"; + const DEFAULT_MINIO_PORT = "9000"; + const DEFAULT_ONISEP_NAME = "onisep"; + const DEFAULT_LOG_LEVEL = "debug"; + const DEFAULT_HISTORY_DIRECTORY_NAME = "history"; + + return { + CONTEXT: "formations-initiales/transformation", + DOMAINE: "Formations initiales", + FLOWS: [ + getOrError("FORMATIONS_INITIALES_ONISEP_NAME"), + ], + ONISEP: { + DIRECTORY_NAME: getOrDefault("FORMATIONS_INITIALES_ONISEP_DIRECTORY_NAME", DEFAULT_ONISEP_NAME), + FLUX_URL: getOrError("FORMATIONS_INITIALES_ONISEP_FLUX_URL"), + NAME: getOrDefault("FORMATIONS_INITIALES_ONISEP_NAME", DEFAULT_ONISEP_NAME), + RAW_FILE_EXTENSION: getOrDefault("FORMATIONS_INITIALES_ONISEP_RAW_FILE_EXTENSION", "xml"), + TRANSFORMED_FILE_EXTENSION: getOrDefault("FORMATIONS_INITIALES_ONISEP_TRANSFORMED_FILE_EXTENSION", "json"), + }, + LOGGER_LOG_LEVEL: getOrDefault("LOGGER_LOG_LEVEL", DEFAULT_LOG_LEVEL) as LogLevel, + MINIO: { + ACCESS_KEY: getOrError("MINIO_ACCESS_KEY"), + HISTORY_DIRECTORY_NAME: getOrDefault("MINIO_HISTORY_DIRECTORY_NAME", DEFAULT_HISTORY_DIRECTORY_NAME), + PORT: Number(getOrDefault("MINIO_PORT", DEFAULT_MINIO_PORT)), + RAW_BUCKET_NAME: getOrDefault("EVENTS_MINIO_RAW_BUCKET_NAME", DEFAULT_RAW_BUCKET_NAME), + SECRET_KEY: getOrError("MINIO_SECRET_KEY"), + URL: getOrError("MINIO_URL"), + TRANSFORMED_BUCKET_NAME: getOrError("EVENTS_MINIO_TRANSFORMED_BUCKET_NAME"), + }, + NODE_ENV: getOrError("NODE_ENV") as Environment, + SENTRY: { + DSN: getOrError("SENTRY_DSN"), + PROJECT: getOrError("npm_package_name"), + RELEASE: getOrError("npm_package_version"), + }, + TEMPORARY_DIRECTORY_PATH: getOrError("TEMPORARY_DIRECTORY_PATH"), + }; + } +} diff --git a/apps/formations-initiales/src/transformation/infrastructure/configuration/log.decorator.ts b/apps/formations-initiales/src/transformation/infrastructure/configuration/log.decorator.ts new file mode 100644 index 000000000..8cf95c480 --- /dev/null +++ b/apps/formations-initiales/src/transformation/infrastructure/configuration/log.decorator.ts @@ -0,0 +1,28 @@ +import { + ConfigurationFactory, +} from "@formations-initiales/src/transformation/infrastructure/configuration/configuration"; +import { + FormationsInitialesTransformationLoggerStrategy, +} from "@formations-initiales/src/transformation/infrastructure/configuration/logger-strategy"; + +const configuration = ConfigurationFactory.create(); +const loggerStrategy = new FormationsInitialesTransformationLoggerStrategy(configuration); + +export function CommandLog(flowName: string): (target: unknown, propertyKey: string, descriptor: PropertyDescriptor) => PropertyDescriptor { + return function (target: unknown, propertyKey: string, descriptor: TypedPropertyDescriptor<() => Promise>): PropertyDescriptor { + const originalMethod = descriptor.value; + + descriptor.value = async function (...args: []): Promise { + try { + loggerStrategy.get(flowName).info(`Starting to transform [${flowName}] flow`); + await originalMethod.apply(this, args); + } catch (e) { + loggerStrategy.get(flowName).fatal({ msg: (e).message, extra: { stack: (e).stack } }); + } finally { + loggerStrategy.get(flowName).info(`End of transforming from [${flowName}] flow`); + } + }; + + return descriptor; + }; +} diff --git a/apps/formations-initiales/src/transformation/infrastructure/configuration/logger-strategy.ts b/apps/formations-initiales/src/transformation/infrastructure/configuration/logger-strategy.ts new file mode 100644 index 000000000..8b9c813ea --- /dev/null +++ b/apps/formations-initiales/src/transformation/infrastructure/configuration/logger-strategy.ts @@ -0,0 +1,36 @@ +import { Configuration } from "@formations-initiales/src/transformation/infrastructure/configuration/configuration"; + +import { + Logger, + LoggerFactory, + LoggerStrategy, + LoggerStrategyError, +} from "@shared/src/infrastructure/configuration/logger"; + +export class FormationsInitialesTransformationLoggerStrategy implements LoggerStrategy { + private readonly loggers: Map; + + constructor(configuration: Configuration) { + const loggerFactory = new LoggerFactory( + configuration.SENTRY.DSN, + configuration.SENTRY.PROJECT, + configuration.SENTRY.RELEASE, + configuration.NODE_ENV, + configuration.CONTEXT, + configuration.LOGGER_LOG_LEVEL, + configuration.DOMAINE, + ); + this.loggers = new Map(); + configuration.FLOWS.forEach((flow) => { + this.loggers.set(flow, loggerFactory.create({ name: flow })); + }); + } + + get(flowName: string): Logger { + const logger = this.loggers.get(flowName); + if (logger) { + return logger; + } + throw new LoggerStrategyError(flowName); + } +} diff --git a/apps/formations-initiales/src/transformation/infrastructure/gateway/index.ts b/apps/formations-initiales/src/transformation/infrastructure/gateway/index.ts new file mode 100644 index 000000000..019a7611f --- /dev/null +++ b/apps/formations-initiales/src/transformation/infrastructure/gateway/index.ts @@ -0,0 +1,81 @@ +import { Module } from "@nestjs/common"; +import { ConfigModule, ConfigService } from "@nestjs/config"; + +import { XMLParser } from "fast-xml-parser"; +import { Client } from "minio"; + +import { + FormationsInitialesRepository, +} from "@formations-initiales/src/transformation/domain/service/formations-initiales.repository"; +import { + Configuration, + ConfigurationFactory, +} from "@formations-initiales/src/transformation/infrastructure/configuration/configuration"; +import { + FormationsInitialesTransformationLoggerStrategy, +} from "@formations-initiales/src/transformation/infrastructure/configuration/logger-strategy"; +import { + MinioFormationsInitialesRepository, +} from "@formations-initiales/src/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository"; + +import { Shared } from "@shared/src"; +import { DateService } from "@shared/src/domain/service/date.service"; +import { FileSystemClient } from "@shared/src/infrastructure/gateway/common/node-file-system.client"; +import { ContentParser, XmlContentParser } from "@shared/src/infrastructure/gateway/content.parser"; +import { UuidGenerator } from "@shared/src/infrastructure/gateway/uuid.generator"; + +@Module({ + imports: [ + ConfigModule.forRoot({ + load: [ConfigurationFactory.createRoot], + envFilePath: process.env.NODE_ENV === "test" ? ".env.test" : ".env", + }), + Shared, + ], + providers: [{ + provide: FormationsInitialesTransformationLoggerStrategy, + inject: [ConfigService], + useFactory: (configurationService: ConfigService): FormationsInitialesTransformationLoggerStrategy => { + const configuration = configurationService.get("formationsInitialesTransformation"); + return new FormationsInitialesTransformationLoggerStrategy(configuration); + }, + }, + { + provide: "ContentParser", + useValue: new XmlContentParser(new XMLParser({ trimValues: true })), + }, + { + provide: "FormationsInitialesRepository", + inject: [ + ConfigService, + Client, + "FileSystemClient", + "UuidGenerator", + DateService, + FormationsInitialesTransformationLoggerStrategy, + "ContentParser", + ], + useFactory: ( + configurationService: ConfigService, + minioClient: Client, + fileSystemClient: FileSystemClient, + uuidGenerator: UuidGenerator, + dateService: DateService, + loggerStrategy: FormationsInitialesTransformationLoggerStrategy, + contentParser: ContentParser, + ): FormationsInitialesRepository => { + return new MinioFormationsInitialesRepository( + configurationService.get("formationsInitialesTransformation"), + minioClient, + fileSystemClient, + uuidGenerator, + dateService, + loggerStrategy, + contentParser, + ); + }, + }], + exports: ["FormationsInitialesRepository"], +}) +export class Gateways { +} diff --git a/apps/formations-initiales/src/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository.ts b/apps/formations-initiales/src/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository.ts new file mode 100644 index 000000000..10b0290c7 --- /dev/null +++ b/apps/formations-initiales/src/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository.ts @@ -0,0 +1,117 @@ +import { Client } from "minio"; + +import { UnJeuneUneSolution } from "@formations-initiales/src/transformation/domain/model/1jeune1solution"; +import { FluxTransformation } from "@formations-initiales/src/transformation/domain/model/flux"; +import { + FormationsInitialesRepository, +} from "@formations-initiales/src/transformation/domain/service/formations-initiales.repository"; +import { Configuration } from "@formations-initiales/src/transformation/infrastructure/configuration/configuration"; + +import { DateService } from "@shared/src/domain/service/date.service"; +import { LoggerStrategy } from "@shared/src/infrastructure/configuration/logger"; +import { FileSystemClient } from "@shared/src/infrastructure/gateway/common/node-file-system.client"; +import { ContentParser } from "@shared/src/infrastructure/gateway/content.parser"; +import { EcritureFluxErreur, RecupererContenuErreur } from "@shared/src/infrastructure/gateway/flux.erreur"; +import { UuidGenerator } from "@shared/src/infrastructure/gateway/uuid.generator"; + +export class MinioFormationsInitialesRepository implements FormationsInitialesRepository { + private static readonly PATH_SEPARATOR: string = "/"; + private static readonly LATEST_FILE_NAME: string = "latest"; + private static readonly JSON_REPLACER: null = null; + private static readonly JSON_INDENTATION: number = 2; + + constructor( + private readonly configuration: Configuration, + private readonly minioClient: Client, + private readonly fileSystemClient: FileSystemClient, + private readonly uuidGenerator: UuidGenerator, + private readonly dateService: DateService, + private readonly loggerStrategy: LoggerStrategy, + private readonly contentParser: ContentParser, + ) { + } + + public async recuperer(flux: FluxTransformation): Promise { + this.loggerStrategy.get(flux.nom).info(`Starting to pull flow ${flux.nom}`); + const fileNameToPull = this.getFileNameToFetch(flux); + const localFileNameIncludingPath = this.configuration.TEMPORARY_DIRECTORY_PATH.concat(this.generateFileName()); + + try { + await this.minioClient.fGetObject( + this.configuration.MINIO.RAW_BUCKET_NAME, + fileNameToPull, + localFileNameIncludingPath, + ); + const fileContent = await this.fileSystemClient.read(localFileNameIncludingPath); + const t = await this.contentParser.parse(fileContent); + return t; + } catch (e) { + throw new RecupererContenuErreur(); + } finally { + await this.fileSystemClient.delete(localFileNameIncludingPath); + this.loggerStrategy.get(flux.nom).info(`Flow ${flux.nom} pulled`); + } + } + + public async sauvegarder(formationsInitiales: Array, flux: FluxTransformation): Promise { + this.loggerStrategy.get(flux.nom).info(`Starting to push flow ${flux.nom}`); + const temporaryFileName = this.generateFileName(); + const localFileNameIncludingPath = this.configuration.TEMPORARY_DIRECTORY_PATH.concat(temporaryFileName); + + try { + await this.fileSystemClient.write(localFileNameIncludingPath, this.toReadableJson(formationsInitiales)); + await this.saveFiles(flux, localFileNameIncludingPath); + } catch (e) { + throw new EcritureFluxErreur(flux.nom); + } finally { + await this.fileSystemClient.delete(localFileNameIncludingPath); + this.loggerStrategy.get(flux.nom).info(`Flow ${flux.nom} pushed`); + } + } + + private toReadableJson(formationsInitiales: Array): string { + return JSON.stringify(formationsInitiales, MinioFormationsInitialesRepository.JSON_REPLACER, MinioFormationsInitialesRepository.JSON_INDENTATION); + } + + private async saveFiles(flow: FluxTransformation, localFileNameIncludingPath: string): Promise { + await this.saveHistoryFile(flow, localFileNameIncludingPath); + await this.saveLatestFile(flow, localFileNameIncludingPath); + } + + private async saveHistoryFile(flux: FluxTransformation, localFileNameIncludingPath: string): Promise { + const historyFileName = this.createHistoryFileName(flux); + await this.minioClient.fPutObject(this.configuration.MINIO.TRANSFORMED_BUCKET_NAME, historyFileName, localFileNameIncludingPath); + } + + private async saveLatestFile(flux: FluxTransformation, localFileNameIncludingPath: string): Promise { + const latestFileName = this.createLatestFileName(flux); + await this.minioClient.fPutObject(this.configuration.MINIO.TRANSFORMED_BUCKET_NAME, latestFileName, localFileNameIncludingPath); + } + + private createHistoryFileName(flux: FluxTransformation): string { + return flux.nom + .concat(MinioFormationsInitialesRepository.PATH_SEPARATOR) + .concat(flux.dossierHistorisation) + .concat(MinioFormationsInitialesRepository.PATH_SEPARATOR) + .concat(this.dateService.maintenant().toISOString()) + .concat(flux.extensionFichierTransforme); + } + + private createLatestFileName(flux: FluxTransformation): string { + return flux.nom + .concat(MinioFormationsInitialesRepository.PATH_SEPARATOR) + .concat(MinioFormationsInitialesRepository.LATEST_FILE_NAME) + .concat(flux.extensionFichierTransforme); + } + + private getFileNameToFetch(flow: FluxTransformation): string { + return flow.nom + .concat(MinioFormationsInitialesRepository.PATH_SEPARATOR) + .concat(MinioFormationsInitialesRepository.LATEST_FILE_NAME) + .concat(flow.extensionFichierBrut); + } + + private generateFileName(): string { + return this.uuidGenerator.generate(); + } +} diff --git a/apps/formations-initiales/src/transformation/infrastructure/index.ts b/apps/formations-initiales/src/transformation/infrastructure/index.ts new file mode 100644 index 000000000..f7c0eeece --- /dev/null +++ b/apps/formations-initiales/src/transformation/infrastructure/index.ts @@ -0,0 +1,39 @@ +import { Module } from "@nestjs/common"; +import { ConfigModule, ConfigService } from "@nestjs/config"; + +import { Usecases } from "@formations-initiales/src/transformation/application-service"; +import { + TransformerFluxOnisep, +} from "@formations-initiales/src/transformation/application-service/transformer-flux-onisep.usecase"; +import { + ConfigurationFactory, +} from "@formations-initiales/src/transformation/infrastructure/configuration/configuration"; +import { + TransformFlowOnisepSubCommand, +} from "@formations-initiales/src/transformation/infrastructure/sub-command/transform-flow-onisep.sub-command"; + +@Module({ + imports: [ + ConfigModule.forRoot({ + load: [ConfigurationFactory.createRoot], + envFilePath: process.env.NODE_ENV === "test" ? ".env.test" : ".env", + }), + Usecases, + ], + providers: [{ + provide: TransformFlowOnisepSubCommand, + inject: [ConfigService, TransformerFluxOnisep], + useFactory: ( + configurationService: ConfigService, + transformerFluxOnisep: TransformerFluxOnisep, + ): TransformFlowOnisepSubCommand => { + return new TransformFlowOnisepSubCommand( + transformerFluxOnisep, + configurationService.get("formationsInitialesTransformation"), + ); + }, + }], + exports: [TransformFlowOnisepSubCommand], +}) +export class Transformation { +} diff --git a/apps/formations-initiales/src/transformation/infrastructure/sub-command/transform-flow-onisep.sub-command.ts b/apps/formations-initiales/src/transformation/infrastructure/sub-command/transform-flow-onisep.sub-command.ts new file mode 100644 index 000000000..7f9be4783 --- /dev/null +++ b/apps/formations-initiales/src/transformation/infrastructure/sub-command/transform-flow-onisep.sub-command.ts @@ -0,0 +1,32 @@ +import { CommandRunner, SubCommand } from "nest-commander"; + +import { + TransformerFluxOnisep, +} from "@formations-initiales/src/transformation/application-service/transformer-flux-onisep.usecase"; +import { FluxTransformation } from "@formations-initiales/src/transformation/domain/model/flux"; +import { Configuration } from "@formations-initiales/src/transformation/infrastructure/configuration/configuration"; +import { CommandLog } from "@formations-initiales/src/transformation/infrastructure/configuration/log.decorator"; + +@SubCommand({ name: TransformFlowOnisepSubCommand.FLOW_NAME }) +export class TransformFlowOnisepSubCommand extends CommandRunner { + private static readonly FLOW_NAME = "onisep"; + + constructor( + private readonly usecase: TransformerFluxOnisep, + private readonly configuration: Configuration, + ) { + super(); + } + + @CommandLog(TransformFlowOnisepSubCommand.FLOW_NAME) + public async run(): Promise { + await this.usecase.executer( + new FluxTransformation( + this.configuration.ONISEP.NAME, + this.configuration.MINIO.HISTORY_DIRECTORY_NAME, + this.configuration.ONISEP.RAW_FILE_EXTENSION, + this.configuration.ONISEP.TRANSFORMED_FILE_EXTENSION, + ), + ); + } +} diff --git a/apps/formations-initiales/test/transformation/application-service/transformer-flux-onisep.usecase.test.ts b/apps/formations-initiales/test/transformation/application-service/transformer-flux-onisep.usecase.test.ts new file mode 100644 index 000000000..8b46e9515 --- /dev/null +++ b/apps/formations-initiales/test/transformation/application-service/transformer-flux-onisep.usecase.test.ts @@ -0,0 +1,65 @@ +import { expect, sinon, StubbedType, stubInterface } from "@test/library"; + +import { + TransformerFluxOnisep, +} from "@formations-initiales/src/transformation/application-service/transformer-flux-onisep.usecase"; +import { FluxTransformation } from "@formations-initiales/src/transformation/domain/model/flux"; +import { Onisep } from "@formations-initiales/src/transformation/domain/model/onisep"; +import { + FormationsInitialesRepository, +} from "@formations-initiales/src/transformation/domain/service/formations-initiales.repository"; +import { Convertir } from "@formations-initiales/src/transformation/domain/service/onisep/convertir.domain-service"; + +describe("TransformerFluxOnisepUseCase", () => { + let usecase: TransformerFluxOnisep; + let repo: StubbedType; + let flux: FluxTransformation; + + const aFluxOnisep: Onisep.Contenu = { + formations: { + formation: [ + { + identifiant: "identifiant", + libelle_complet: "libelle_complet", + duree_formation: "duree_formation", + niveau_certification: "niveau_certification", + niveau_etudes: { libelle: "niveau_etudes" }, + descriptif_format_court: "descriptif_format_court", + attendus: "attendus", + descriptif_acces: "descriptif_acces", + descriptif_poursuite_etudes: "descriptif_poursuite_etudes", + }, + ], + }, + }; + + beforeEach(() => { + repo = stubInterface(sinon); + flux = new FluxTransformation("onisep", "history", ".xml", ".json"); + + usecase = new TransformerFluxOnisep(repo, new Convertir()); + }); + + context("Lorsque je récupère le flux provenant de onisep", () => { + beforeEach(() => { + repo.recuperer.resolves(aFluxOnisep); + }); + + it("je retourne une liste de formations initiales d'un jeune une solution", async () => { + await usecase.executer(flux); + + expect(repo.sauvegarder).to.have.been.calledWithMatch([{ + identifiant: "identifiant", + intitule: "libelle_complet", + duree: "duree_formation", + certification: "niveau_certification", + niveauEtudesVise: "niveau_etudes", + description: "descriptif_format_court", + attendusParcoursup: "attendus", + conditionsAcces: "descriptif_acces", + poursuiteEtudes: "descriptif_poursuite_etudes", + }], + flux); + }); + }); +}); diff --git a/apps/formations-initiales/test/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository.test.ts b/apps/formations-initiales/test/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository.test.ts new file mode 100644 index 000000000..d4233cc68 --- /dev/null +++ b/apps/formations-initiales/test/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository.test.ts @@ -0,0 +1,185 @@ +import { Client } from "minio"; + +import { expect, sinon, StubbedClass, StubbedType, stubClass, stubInterface } from "@test/library"; + +import { UnJeuneUneSolution } from "@formations-initiales/src/transformation/domain/model/1jeune1solution"; +import { FluxTransformation } from "@formations-initiales/src/transformation/domain/model/flux"; +import { Configuration } from "@formations-initiales/src/transformation/infrastructure/configuration/configuration"; +import { + MinioFormationsInitialesRepository, +} from "@formations-initiales/src/transformation/infrastructure/gateway/repository/minio-formations-initiales.repository"; + +import { DateService } from "@shared/src/domain/service/date.service"; +import { Logger, LoggerStrategy } from "@shared/src/infrastructure/configuration/logger"; +import { FileSystemClient } from "@shared/src/infrastructure/gateway/common/node-file-system.client"; +import { ContentParser } from "@shared/src/infrastructure/gateway/content.parser"; +import { EcritureFluxErreur, RecupererContenuErreur } from "@shared/src/infrastructure/gateway/flux.erreur"; +import { UuidGenerator } from "@shared/src/infrastructure/gateway/uuid.generator"; + +let localFileNameIncludingPath: string; +let formationsInitiales: Array; +let fileContent: string; +let latestFileNameIncludingPath: string; +let historyFileNameIncludingPath: string; + +let configuration: StubbedType; +let fileSystemClient: StubbedType; +let loggerStrategy: StubbedType; +let logger: StubbedType; +let flux: FluxTransformation; +let uuidClient: StubbedType; +let minioStub: StubbedClass; +let dateService: StubbedClass; +let minioFormationsInitialesRepository: MinioFormationsInitialesRepository; +let contentParser: StubbedType; + +describe("MinioFormationsInitialesRepositoryTest", () => { + beforeEach(() => { + latestFileNameIncludingPath = "source/latest.json"; + historyFileNameIncludingPath = "source/history/2022-01-01T00:00:00.000Z.json"; + + fileContent = ""; + flux = new FluxTransformation("source", "history", ".xml", ".json"); + + formationsInitiales = [ + { + identifiant: "identifiant", + intitule: "libelle_complet", + duree: "duree_formation", + certification: "niveau_certification", + niveauEtudesVise: "niveau_etudes", + description: "descriptif_format_court", + attendusParcoursup: "attendus", + conditionsAcces: "descriptif_acces", + poursuiteEtudes: "descriptif_poursuite_etudes", + }, + ]; + + localFileNameIncludingPath = "./tmp/d184b5b1-75ad-44f0-8fe7-7c55208bf26c"; + + minioStub = stubClass(Client); + + loggerStrategy = stubInterface(sinon); + logger = stubInterface(sinon); + loggerStrategy.get.returns(logger); + + configuration = stubInterface(sinon); + configuration.MINIO.RAW_BUCKET_NAME = "raw"; + configuration.MINIO.TRANSFORMED_BUCKET_NAME = "json"; + configuration.TEMPORARY_DIRECTORY_PATH = "./tmp/"; + + fileSystemClient = stubInterface(sinon); + uuidClient = stubInterface(sinon); + dateService = stubClass(DateService); + dateService.maintenant.returns(new Date("2022-01-01T00:00:00.000Z")); + contentParser = stubInterface(sinon); + + uuidClient.generate.returns("d184b5b1-75ad-44f0-8fe7-7c55208bf26c"); + minioFormationsInitialesRepository = new MinioFormationsInitialesRepository( + configuration, + minioStub, + fileSystemClient, + uuidClient, + dateService, + loggerStrategy, + contentParser, + ); + }); + + describe("recuperer", () => { + context("Lorsque je récupère le contenu d'un fichier", () => { + beforeEach(() => { + configuration = stubInterface(sinon); + configuration.MINIO.RAW_BUCKET_NAME = "raw"; + + uuidClient.generate.returns("f278702a-ea1f-445b-a58a-37ee58892175"); + localFileNameIncludingPath = "./tmp/f278702a-ea1f-445b-a58a-37ee58892175"; + + minioStub.fGetObject.resolves(); + fileSystemClient.read.resolves(fileContent); + contentParser.parse.resolves(formationsInitiales); + }); + + it("je récupère le contenu du fichier", async () => { + const result = await minioFormationsInitialesRepository.recuperer( + flux, + ); + + expect(result).to.eql(formationsInitiales); + expect(uuidClient.generate).to.have.been.calledOnce; + expect(minioStub.fGetObject).to.have.been.calledOnce; + expect(minioStub.fGetObject).to.have.been.calledWith( + configuration.MINIO.RAW_BUCKET_NAME, + "source/latest.xml", + localFileNameIncludingPath + ); + expect(fileSystemClient.read).to.have.been.calledOnce; + expect(fileSystemClient.read).to.have.been.calledWith(localFileNameIncludingPath); + expect(fileSystemClient.delete).to.have.been.calledOnce; + expect(fileSystemClient.delete).to.have.been.calledWith(localFileNameIncludingPath); }); + }); + + context("Lorsque je ne réussis pas à lire le contenu d'un fichier", () => { + beforeEach(() => { + configuration = stubInterface(sinon); + configuration.MINIO.RAW_BUCKET_NAME = "raw"; + + uuidClient.generate.returns("f278702a-ea1f-445b-a58a-37ee58892175"); + localFileNameIncludingPath = "./tmp/f278702a-ea1f-445b-a58a-37ee58892175"; + + minioStub.fGetObject.rejects(new Error("Oops! Something went wrong !")); + }); + + it("je lance une erreur", async () => { + await expect(minioFormationsInitialesRepository.recuperer( + new FluxTransformation("source", "history", ".xml", ".json"), + )).to.be.rejectedWith( + RecupererContenuErreur, + "Une erreur de lecture ou de parsing est survenue lors de la récupération du contenu", + ); + }); + }); + }); + + describe("sauvegarder", () => { + context("Lorsque j'écris le contenu d'un fichier qui existe bien et qu'il est bien nommé dans un dossier racine existant", () => { + it("j'écris le contenu du fichier", async () => { + await minioFormationsInitialesRepository.sauvegarder(formationsInitiales, flux); + + expect(uuidClient.generate).to.have.been.calledOnce; + + expect(fileSystemClient.write).to.have.been.calledOnce; + expect(fileSystemClient.write.getCall(0).args[0]).to.eql(localFileNameIncludingPath); + expect(JSON.parse(fileSystemClient.write.getCall(0).args[1] as string)).to.have.deep.members(formationsInitiales); + + expect(minioStub.fPutObject).to.have.been.calledTwice; + expect(minioStub.fPutObject.firstCall.args).to.have.deep.members([ + configuration.MINIO.TRANSFORMED_BUCKET_NAME, + historyFileNameIncludingPath, + localFileNameIncludingPath, + ]); + expect(minioStub.fPutObject.secondCall.args).to.have.deep.members([ + configuration.MINIO.TRANSFORMED_BUCKET_NAME, + latestFileNameIncludingPath, + localFileNameIncludingPath, + ]); + + expect(fileSystemClient.delete).to.have.been.calledOnce; + expect(fileSystemClient.delete).to.have.been.calledWith(localFileNameIncludingPath); + }); + }); + + context("Lorsque je n'arrive pas à écrire le fichier chez moi", () => { + beforeEach(() => { + fileSystemClient.write.rejects(new Error("Oops! Something went wrong !")); + }); + + it("je lance une erreur", async () => { + await expect(minioFormationsInitialesRepository.sauvegarder(formationsInitiales, flux)).to.be.rejectedWith( + EcritureFluxErreur, + "Le flux source n'a pas été extrait car une erreur d'écriture est survenue", + ); + }); + }); + }); +});