From 153498777a0114de9e371f927bffec313125d452 Mon Sep 17 00:00:00 2001 From: Dwayne Collins Date: Fri, 9 Jul 2021 01:11:11 -0400 Subject: [PATCH] Implement PR#36 from upstream Implement changes as listed PR#36 (which has not yet been merged into upstream repo). https://github.com/Islandora-Devops/migrate_7x_claw/pull/36 --- composer.json | 1 - migrate_7x_claw.info.yml | 2 +- ...grate_plus.migration.islandora_7x_tags.yml | 2 + ...migrate_plus.migration.islandora_files.yml | 232 +----------------- ...islandora_migrate_7x_claw_feature.info.yml | 2 +- src/Form/MIGRATE7XCLAWSettingsForm.php | 4 +- .../data_parser/AuthenticatedXml.php | 11 + 7 files changed, 22 insertions(+), 232 deletions(-) diff --git a/composer.json b/composer.json index 9b92f4e..3bd998f 100644 --- a/composer.json +++ b/composer.json @@ -5,7 +5,6 @@ "homepage": "https://github.com/Islandora-Devops/migrate_7x_claw", "support": { "issues": "https://github.com/Islandora-CLAW/CLAW/issues", - "irc": "irc://irc.freenode.org/islandora", "source": "https://github.com/Islandora-Devops/migrate_7x_claw" }, "license": "GPL-2.0-or-later", diff --git a/migrate_7x_claw.info.yml b/migrate_7x_claw.info.yml index 9445b93..b669123 100644 --- a/migrate_7x_claw.info.yml +++ b/migrate_7x_claw.info.yml @@ -2,7 +2,7 @@ type: module name: Migrate Islandora 7.x to CLAW description: 'Migration setup for migrating from an Islandora 7.x instance to CLAW.' package: Islandora -core: 8.x +core_version_requirement: ^8 || ^9 dependencies: - drupal:migrate - migrate_plus:migrate_plus diff --git a/modules/islandora_migrate_7x_claw_feature/config/install/migrate_plus.migration.islandora_7x_tags.yml b/modules/islandora_migrate_7x_claw_feature/config/install/migrate_plus.migration.islandora_7x_tags.yml index a3cc204..dead82a 100644 --- a/modules/islandora_migrate_7x_claw_feature/config/install/migrate_plus.migration.islandora_7x_tags.yml +++ b/modules/islandora_migrate_7x_claw_feature/config/install/migrate_plus.migration.islandora_7x_tags.yml @@ -15,6 +15,8 @@ source: plugin: csv path: modules/contrib/migrate_7x_claw/migrate/tags.csv header_row_count: 1 + ids: + - vid keys: - external_uri process: diff --git a/modules/islandora_migrate_7x_claw_feature/config/install/migrate_plus.migration.islandora_files.yml b/modules/islandora_migrate_7x_claw_feature/config/install/migrate_plus.migration.islandora_files.yml index 859772c..4f772d6 100644 --- a/modules/islandora_migrate_7x_claw_feature/config/install/migrate_plus.migration.islandora_files.yml +++ b/modules/islandora_migrate_7x_claw_feature/config/install/migrate_plus.migration.islandora_files.yml @@ -78,11 +78,8 @@ process: application/acad: dwg application/arj: arj application/base64: mm - application/base64: mme application/binhex4: hqx - application/binhex: hqx application/book: boo - application/book: book application/cdf: cdf application/clariscad: ccad application/commonground: dp @@ -92,17 +89,6 @@ process: application/ecmascript: js application/envoy: evy application/excel: xl - application/excel: xla - application/excel: xlb - application/excel: xlc - application/excel: xld - application/excel: xlk - application/excel: xll - application/excel: xlm - application/excel: xls - application/excel: xlt - application/excel: xlv - application/excel: xlw application/fractals: fif application/freeloader: frl application/futuresplash: spl @@ -112,7 +98,6 @@ process: application/hta: hta application/i-deas: unv application/iges: iges - application/iges: igs application/inf: inf application/java-byte-code: class application/java: class @@ -128,74 +113,40 @@ process: application/mbedlet: mbd application/mcad: mcd application/mime: aps - application/mspowerpoint: pot - application/mspowerpoint: pps application/mspowerpoint: ppt - application/mspowerpoint: ppz application/msword: doc - application/msword: dot - application/msword: w6w - application/msword: wiz - application/msword: word - application/mswrite: wri + application/mswrite: msw application/netmc: mcp - application/octet-stream: a - application/octet-stream: arc - application/octet-stream: arj - application/octet-stream: bin - application/octet-stream: com - application/octet-stream: dump application/octet-stream: exe - application/octet-stream: lha - application/octet-stream: lhx - application/octet-stream: lzh - application/octet-stream: lzx - application/octet-stream: o - application/octet-stream: psd - application/octet-stream: saveme - application/octet-stream: uu - application/octet-stream: zoo application/oda: oda application/pdf: pdf application/pkcs-12: p12 application/pkcs-crl: crl application/pkcs10: p10 application/pkcs7-mime: p7c - application/pkcs7-mime: p7m application/pkcs7-signature: p7s application/pkix-cert: cer - application/pkix-cert: crt application/pkix-crl: crl application/plain: text - application/postscript: ai application/postscript: eps - application/postscript: ps application/powerpoint: ppt application/pro_eng: part - application/pro_eng: prt application/rdf+xml: xml application/ringing-tones: rng application/rtf: rtf - application/rtf: rtx application/sdp: sdp application/sea: sea application/set: set application/sla: stl application/smil: smi - application/smil: smil application/solids: sol application/sounder: sdr application/step: step - application/step: stp application/streamingmedia: ssm application/toolbook: tbk application/vda: vda application/vocaltec-media-desc: vmd - application/vocaltec-media-file: vmf application/wordperfect: wp - application/wordperfect: wp5 - application/wordperfect: wp6 - application/wordperfect: wpd application/x-123: wk1 application/x-aim: aim application/x-authorware-bin: aab @@ -204,58 +155,35 @@ process: application/x-bcpio: bcpio application/x-binary: bin application/x-binhex40: hqx - application/x-bsh: bsh application/x-bsh: sh - application/x-bsh: shar - application/x-bzip2: boz application/x-bzip2: bz2 application/x-bzip: bz application/x-cdf: cdf application/x-cdlink: vcd - application/x-chat: cha application/x-chat: chat application/x-cmu-raster: ras application/x-cocoa: cco application/x-compactpro: cpt application/x-compress: z - application/x-compressed: gz - application/x-compressed: tgz - application/x-compressed: z application/x-compressed: zip - application/x-conference: nsc application/x-cpio: cpio application/x-cpt: cpt application/x-csh: csh application/x-deepv: deepv - application/x-director: dcr application/x-director: dir - application/x-director: dxr application/x-dvi: dvi application/x-elc: elc application/x-envoy: env - application/x-envoy: evy application/x-esrehber: es application/x-excel: xla - application/x-excel: xlb - application/x-excel: xlc - application/x-excel: xld - application/x-excel: xlk - application/x-excel: xll - application/x-excel: xlm - application/x-excel: xls - application/x-excel: xlt - application/x-excel: xlv - application/x-excel: xlw application/x-frame: mif application/x-freelance: pre application/x-gsp: gsp application/x-gss: gss application/x-gtar: gtar - application/x-gzip: gz application/x-gzip: gzip application/x-hdf: hdf application/x-helpfile: help - application/x-helpfile: hlp application/x-httpd-imap: imap application/x-ima: ima application/x-internett-signup: ins @@ -265,12 +193,8 @@ process: application/x-java-commerce: jcm application/x-javascript: js application/x-koan: skd - application/x-koan: skm - application/x-koan: skp - application/x-koan: skt application/x-ksh: ksh application/x-latex: latex - application/x-latex: ltx application/x-lha: lha application/x-lisp: lsp application/x-livescreen: ivy @@ -282,20 +206,16 @@ process: application/x-macbinary: bin application/x-mathcad: mcd application/x-meme: mm - application/x-midi: mid application/x-midi: midi application/x-mif: mif application/x-mix-transfer: nix application/x-mplayer2: asx application/x-msexcel: xla - application/x-msexcel: xls - application/x-msexcel: xlw application/x-mspowerpoint: ppt application/x-navi-animation: ani application/x-navidoc: nvd application/x-navimap: map application/x-navistyle: stl - application/x-netcdf: cdf application/x-netcdf: nc application/x-newton-compatible-pkg: pkg application/x-nokia-9000-communicator-add-on-software: aos @@ -303,7 +223,6 @@ process: application/x-omcdatamaker: omcd application/x-omcregerator: omcr application/x-pagemaker: pm4 - application/x-pagemaker: pm5 application/x-pcl: pcl application/x-pixclscript: plx application/x-pkcs10: p10 @@ -311,14 +230,10 @@ process: application/x-pkcs7-certificates: spc application/x-pkcs7-certreqresp: p7r application/x-pkcs7-mime: p7c - application/x-pkcs7-mime: p7m application/x-pkcs7-signature: p7a application/x-pointplus: css application/x-portable-anymap: pnm application/x-project: mpc - application/x-project: mpt - application/x-project: mpv - application/x-project: mpx application/x-qpro: wb1 application/x-rtf: rtf application/x-sdp: sdp @@ -326,71 +241,46 @@ process: application/x-seelogo: sl application/x-sh: sh application/x-shar: sh - application/x-shar: shar application/x-shockwave-flash: swf application/x-sit: sit application/x-sprite: spr - application/x-sprite: sprite application/x-stuffit: sit application/x-sv4cpio: sv4cpio application/x-sv4crc: sv4crc application/x-tar: tar application/x-tbook: sbk - application/x-tbook: tbk application/x-tcl: tcl application/x-tex: tex application/x-texinfo: texi - application/x-texinfo: texinfo application/x-troff-man: man application/x-troff-me: me application/x-troff-ms: ms application/x-troff-msvideo: avi application/x-troff: roff - application/x-troff: t - application/x-troff: tr application/x-ustar: ustar application/x-visio: vsd - application/x-visio: vst - application/x-visio: vsw application/x-vrml: vrml application/x-wais-source: src - application/x-wais-source: wsrc application/x-winhelp: hlp application/x-wintalk: wtk application/x-world: svr - application/x-world: wrl application/x-wpwin: wpd application/x-wri: wri application/x-x509-ca-cert: cer - application/x-x509-ca-cert: crt - application/x-x509-ca-cert: der application/x-x509-user-cert: crt application/x-zip-compressed: zip application/xml: xml application/zip: zip audio/aac: aac - audio/aiff: aif - audio/aiff: aifc audio/aiff: aiff audio/basic: au - audio/basic: snd audio/it: it audio/make: funk - audio/make: my - audio/make: pfunk audio/mid: rmi - audio/midi: kar - audio/midi: mid audio/midi: midi audio/mod: mod audio/mpeg3: mp3 audio/mpeg: m2a - audio/mpeg: mp2 - audio/mpeg: mp3 - audio/mpeg: mpa - audio/mpeg: mpg - audio/mpeg: mpga - audio/nspaudio: la audio/nspaudio: lma audio/s3m: s3m audio/tsp-audio: tsi @@ -400,84 +290,52 @@ process: audio/wav: wav audio/x-adpcm: snd audio/x-aiff: aif - audio/x-aiff: aifc - audio/x-aiff: aiff audio/x-au: au audio/x-gsm: gsd - audio/x-gsm: gsm audio/x-jam: jam audio/x-liveaudio: lam audio/x-mid: mid - audio/x-mid: midi - audio/x-midi: mid audio/x-midi: midi audio/x-mod: mod audio/x-mpeg-3: mp3 audio/x-mpeg: mp2 audio/x-mpequrl: m3u - audio/x-nspaudio: la audio/x-nspaudio: lma audio/x-pn-realaudio-plugin: ra - audio/x-pn-realaudio-plugin: rmp - audio/x-pn-realaudio-plugin: rpm audio/x-pn-realaudio: ra - audio/x-pn-realaudio: ram - audio/x-pn-realaudio: rm - audio/x-pn-realaudio: rmm - audio/x-pn-realaudio: rmp audio/x-psid: sid audio/x-realaudio: ra audio/x-twinvq-plugin: vqe - audio/x-twinvq-plugin: vql audio/x-twinvq: vqf audio/x-voc: voc audio/x-wav: wav audio/xm: xm chemical/x-pdb: pdb - chemical/x-pdb: xyz drawing/x-dwf (old): dwf i-world/i-vrml: ivr image/bmp: bm - image/bmp: bmp - image/cmu-raster: ras image/cmu-raster: rast image/fif: fif image/florian: flo - image/florian: turbot image/g3fax: g3 image/gif: gif image/ief: ief - image/ief: iefs image/jp2: jp2 image/jpeg: jfif - image/jpeg: jfif-tbnl - image/jpeg: jpe - image/jpeg: jpeg - image/jpeg: jpg image/jpg: jpg image/jutvision: jut image/naplps: nap - image/naplps: naplps image/pict: pic - image/pict: pict - image/pjpeg: jfif - image/pjpeg: jpe image/pjpeg: jpeg - image/pjpeg: jpg image/png: png - image/png: x-png image/tif: tiff - image/tiff: tif image/tiff: tiff image/vasa: mcf image/x-cmu-raster: ras image/x-dwg: dwg - image/x-dwg: dxf - image/x-dwg: svf image/x-icon: ico image/x-jg: art image/x-jps: jps - image/x-niff: nif image/x-niff: niff image/x-pcx: pcx image/x-pict: pct @@ -486,114 +344,55 @@ process: image/x-portable-graymap: pgm image/x-portable-greymap: pgm image/x-portable-pixmap: ppm - image/x-quicktime: qif - image/x-quicktime: qti image/x-quicktime: qtif image/x-rgb: rgb image/x-tiff: tif - image/x-tiff: tiff image/x-windows-bmp: bmp image/x-xbitmap: xbm image/x-xbm: xbm - image/x-xpixmap: pm image/x-xpixmap: xpm image/x-xwd: xwd image/x-xwindowdump: xwd image/xbm: xbm image/xpm: xpm - message/rfc822: mht - message/rfc822: mhtml message/rfc822: mime model/iges: iges - model/iges: igs model/vrml: vrml - model/vrml: wrl - model/vrml: wrz model/x-pov: pov multipart/x-gzip: gzip multipart/x-ustar: ustar multipart/x-zip: zip - music/crescendo: mid music/crescendo: midi music/x-karaoke: kar paleovu/x-pv: pvu text/asp: asp text/css: css text/ecmascript: js - text/html: acgi - text/html: htm text/html: html - text/html: htmls - text/html: htx - text/html: shtml text/javascript: js text/mcf: mcf text/pascal: pas - text/plain: c - text/plain: c++ - text/plain: cc - text/plain: com - text/plain: conf - text/plain: cxx - text/plain: def - text/plain: f - text/plain: f90 - text/plain: for - text/plain: g - text/plain: h - text/plain: hh - text/plain: idc - text/plain: jav - text/plain: java - text/plain: list - text/plain: log - text/plain: lst - text/plain: m - text/plain: mar - text/plain: pl - text/plain: sdml - text/plain: text text/plain: txt - text/richtext: rt text/richtext: rtf - text/richtext: rtx text/scriplet: wsc - text/sgml: sgm text/sgml: sgml text/tab-separated-values: tsv - text/uri-list: uni - text/uri-list: unis - text/uri-list: uri text/uri-list: uris text/webviewhtml: htt text/x-asm: asm - text/x-asm: s text/x-audiosoft-intra: aip text/x-c: c - text/x-c: cc - text/x-c: cpp text/x-component: htc text/x-fortran: f - text/x-fortran: f77 - text/x-fortran: f90 - text/x-fortran: for - text/x-h: h text/x-h: hh - text/x-java-source: jav text/x-java-source: java text/x-la-asf: lsx text/x-m: m text/x-pascal: p text/x-script: hlb text/x-server-parsed-html: shtml - text/x-server-parsed-html: ssi text/x-setext: etx - text/x-sgml: sgm text/x-sgml: sgml - text/x-speech: spc - text/x-speech: talk - text/x-uil: uil - text/x-uuencode: uu text/x-uuencode: uue text/x-vcalendar: vcs text/xml: xml @@ -605,56 +404,36 @@ process: video/gl: gl video/mp4: mp4 video/mpeg: m1v - video/mpeg: m2v - video/mpeg: mp2 - video/mpeg: mp3 - video/mpeg: mpa - video/mpeg: mpe - video/mpeg: mpeg - video/mpeg: mpg video/msvideo: avi - video/quicktime: moov video/quicktime: mov - video/quicktime: qt video/vdo: vdo video/vivo: viv - video/vivo: vivo video/vosaic: vos video/x-amt-demorun: xdr video/x-amt-showrun: xsr video/x-atomic3d-feature: fmf video/x-dl: dl video/x-dv: dif - video/x-dv: dv video/x-fli: fli video/x-gl: gl video/x-isvideo: isu video/x-matroska: mkv video/x-motion-jpeg: mjpg - video/x-mpeg: mp2 video/x-mpeg: mp3 video/x-mpeq2a: mp2 video/x-ms-asf-plugin: asx video/x-ms-asf: asf - video/x-ms-asf: asx video/x-msvideo: avi video/x-qtc: qtc video/x-scm: scm video/x-sgi-movie: movie - video/x-sgi-movie: mv windows/metafile: wmf www/mime: mime x-conference/x-cooltalk: ice - x-music/x-midi: mid x-music/x-midi: midi x-world/x-3dmf: 3dm - x-world/x-3dmf: 3dmf - x-world/x-3dmf: qd3 - x-world/x-3dmf: qd3d x-world/x-svr: svr x-world/x-vrml: vrml - x-world/x-vrml: wrl - x-world/x-vrml: wrz x-world/x-vrt: vrt xgl/drawing: xgz xgl/movie: xmz @@ -675,11 +454,10 @@ process: source: - '@remote_path' - '@local_path' - settings: - authentication: - plugin: basic - username: fedoraAdmin - password: fedoraAdmin + guzzle_options: + auth: + - fedoraAdmin + - fedoraAdmin uid: constants/creator_uid destination: plugin: 'entity:file' diff --git a/modules/islandora_migrate_7x_claw_feature/islandora_migrate_7x_claw_feature.info.yml b/modules/islandora_migrate_7x_claw_feature/islandora_migrate_7x_claw_feature.info.yml index ab3b7b5..8f1927c 100644 --- a/modules/islandora_migrate_7x_claw_feature/islandora_migrate_7x_claw_feature.info.yml +++ b/modules/islandora_migrate_7x_claw_feature/islandora_migrate_7x_claw_feature.info.yml @@ -1,7 +1,7 @@ name: 'Migrate 7x Claw Feature' description: 'Migration configuration for moving to Islandora CLAW from 7.x' type: module -core: 8.x +core_version_requirement: ^8 || ^9 dependencies: - migrate_plus - migrate_7x_claw diff --git a/src/Form/MIGRATE7XCLAWSettingsForm.php b/src/Form/MIGRATE7XCLAWSettingsForm.php index c7b09c7..c058a85 100644 --- a/src/Form/MIGRATE7XCLAWSettingsForm.php +++ b/src/Form/MIGRATE7XCLAWSettingsForm.php @@ -125,7 +125,6 @@ public function submitForm(array &$form, FormStateInterface $form_state) { $islandora_objects_config->set('source.authentication.username', $form_state->getValue('oldfedoraUsername')); $islandora_person_config->set('source.authentication.username', $form_state->getValue('oldfedoraUsername')); $islandora_subject_config->set('source.authentication.username', $form_state->getValue('oldfedoraUsername')); - $islandora_files_config->set('process.uri.settings.authentication.username', $form_state->getValue('oldfedoraUsername')); if (!$form_state->getValue('oldfedorapsswd') == '') { $config->set('oldfedorapsswd', $form_state->getValue('oldfedorapsswd')); @@ -138,7 +137,8 @@ public function submitForm(array &$form, FormStateInterface $form_state) { $islandora_objects_config->set('source.authentication.password', $form_state->getValue('oldfedorapsswd')); $islandora_person_config->set('source.authentication.password', $form_state->getValue('oldfedorapsswd')); $islandora_subject_config->set('source.authentication.password', $form_state->getValue('oldfedorapsswd')); - $islandora_files_config->set('process.uri.settings.authentication.password', $form_state->getValue('oldfedorapsswd')); + // File needs a special authentication configuration to pass to Guzzle. + $islandora_files_config->set('process.uri.guzzle_options.auth', [$form_state->getValue('oldfedoraUsername'), $form_state->getValue('oldfedorapsswd')]); } $config->set('solr-endpoint-url', $form_state->getValue('solr-endpoint-url')); diff --git a/src/Plugin/migrate_plus/data_parser/AuthenticatedXml.php b/src/Plugin/migrate_plus/data_parser/AuthenticatedXml.php index 775dc0a..f36131f 100644 --- a/src/Plugin/migrate_plus/data_parser/AuthenticatedXml.php +++ b/src/Plugin/migrate_plus/data_parser/AuthenticatedXml.php @@ -57,6 +57,17 @@ protected function fetchNextRow() { $this->currentItem[$field_name] = reset($values); } } + + // Make the PID available as a field in the migration + // This facilitates migrate_lookup, needed to migrate data + // into paragraphs. + if (empty($currentItem['PID'])) { + $pid_matches = []; + preg_match('/\/objects\/(.*?)\/datastreams/', $this->urls[$this->activeUrl], $pid_matches); + if (!empty($pid_matches[1])) { + $this->currentItem['PID'] = $pid_matches[1]; + } + } } }