From 9416735e7ad478569cdee1ffc2777fd58aa56f46 Mon Sep 17 00:00:00 2001 From: Ana Trisovic Date: Wed, 6 Jan 2021 14:35:54 -0500 Subject: [PATCH 1/7] add more mime types based on use frequency --- .../propertyFiles/MimeTypeFacets.properties | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/main/java/propertyFiles/MimeTypeFacets.properties b/src/main/java/propertyFiles/MimeTypeFacets.properties index 54c5e01d317..2f91ca042b4 100644 --- a/src/main/java/propertyFiles/MimeTypeFacets.properties +++ b/src/main/java/propertyFiles/MimeTypeFacets.properties @@ -5,6 +5,7 @@ application/pdf=Document image/pdf=Document text/pdf=Document application/x-pdf=Document +application/cnt=Document application/msword=Document application/vnd.ms-excel=Document application/vnd.openxmlformats-officedocument.spreadsheetml.sheet=Document @@ -12,14 +13,17 @@ application/vnd.ms-powerpoint=Document application/vnd.openxmlformats-officedocument.presentationml.presentation=Document application/vnd.openxmlformats-officedocument.wordprocessingml.document=Document application/vnd.oasis.opendocument.spreadsheet=Document +application/vnd.ms-excel.sheet.macroenabled.12=Document # Text text/plain=Text +text/x-log=Text text/html=Text application/x-tex=Text text/x-tex=Text text/markdown=Text text/x-markdown=Text text/x-r-markdown=Text +text/x-rst=Text application/rtf=Text text/rtf=Text text/richtext=Text @@ -28,16 +32,24 @@ application/xml=Text text/xml=Text # Code text/x-c=Code +text/x-c++src=Code text/css=Code +text/x-objcsrc=Code +application/java-vm=Code +text/x-java-source=Code text/javascript=Code application/javascript=Code application/x-javascript=Code +text/x-perl-script=Code text/x-matlab=Code text/x-mathematica=Code text/php=Code +text/x-fortran=Code +text/x-pascal=Code text/x-python=Code text/x-python-script=Code text/x-r-source=Code +text/x-sh=Code application/x-sh=Code application/x-shellscript=Code application/x-sql=Code @@ -53,12 +65,17 @@ application/x-spss-syntax=Code text/x-sas-syntax=Code application/x-sas-syntax=Code type/x-r-syntax=Code +application/postscript=Code +application/vnd.wolfram.mathematica.package=Code +application/vnd.wolfram.mathematica=Code # Ingested text/tab-separated-values=Tabular Data # Data text/tsv=Data text/comma-separated-values=Data +text/x-comma-separated-values=Data text/csv=Data +text/x-vcard=Data text/x-fixed-field=Data application/x-rlang-transport=Data application/x-R-2=Data @@ -81,6 +98,7 @@ application/x-sas-data=Data application/x-sas-catalog=Data application/x-sas-log=Data application/x-sas-output=Data +application/x-r-data=Data application/softgrid-do=Data application/x-dvn-csvspss-zip=Data application/x-dvn-tabddi-zip=Data @@ -100,7 +118,12 @@ application/netcdf=Data application/x-netcdf=Data application/vnd.lotus-notes=Data application/x-nsdstat=Data +application/vnd.flographit=Data application/vnd.realvnc.bed=Data +application/vnd.ms-pki.stl=Data +application/vnd.isac.fcs=Data +application/java-serialized-object=Data +chemical/x-xyz=Data # FITS image/fits=FITS application/fits=FITS @@ -112,6 +135,8 @@ application/sbn=Shape application/sbx=Shape application/shp=Shape application/shx=Shape +application/x-esri-shape=Shape +application/vnd.google-earth.kml+xml=Shape application/zipped-shapefile=Shape # Archive application/zip=Archive @@ -122,15 +147,18 @@ application/x-bzip2=Archive application/vnd.google-earth.kmz=Archive application/gzip=Archive application/x-gzip=Archive +application/x-gzip-compressed=Archive application/rar=Archive application/x-rar=Archive application/x-rar-compressed=Archive application/tar=Archive application/x-tar=Archive +application/x-compressed=Archive application/x-compressed-tar=Archive application/x-7z-compressed=Archive application/x-xz=Archive application/warc=Archive +application/x-iso9660-image=Archive # Image image/gif=Image image/jpeg=Image @@ -151,8 +179,13 @@ image/tiff=Image image/bmp=Image image/x-xbitmap=Image image/RAW=Image +image/raw=Image +application/x-tgif=Image image/x-xpixmap=Image image/x-xwindowdump=Image +application/photoshop=Image +image/vnd.adobe.photoshop=Image +application/x-photoshop=Image # (anything else that looks like image/* will also be indexed as facet type "Image") # Audio audio/x-aiff=Audio From 18ab2c1ec8854f03dac7210e037a9b14d5e7e470 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 8 Jan 2021 09:56:20 -0500 Subject: [PATCH 2/7] adding release note --- doc/release-notes/7502-more-mime-types.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/release-notes/7502-more-mime-types.md diff --git a/doc/release-notes/7502-more-mime-types.md b/doc/release-notes/7502-more-mime-types.md new file mode 100644 index 00000000000..8fc3f33f92f --- /dev/null +++ b/doc/release-notes/7502-more-mime-types.md @@ -0,0 +1,4 @@ +## Upgrade Steps + +- Redetect file types using the Redetect File Types API: +https://guides.dataverse.org/en/latest/api/native-api.html#redetect-file-type \ No newline at end of file From 0ad2266a6d91feb78b55d576739e50bf8dbb307a Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Tue, 12 Jan 2021 10:53:28 -0500 Subject: [PATCH 3/7] Added new file types based on new mime types --- .../propertyFiles/MimeTypeDisplay.properties | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 29407ccda40..6899f9b4518 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -5,6 +5,7 @@ application/pdf=Adobe PDF image/pdf=Adobe PDF text/pdf=Adobe PDF application/x-pdf=Adobe PDF +application/cnt=CNT application/msword=MS Word application/vnd.ms-excel=MS Excel Spreadsheet application/vnd.openxmlformats-officedocument.spreadsheetml.sheet=MS Excel Spreadsheet @@ -12,8 +13,10 @@ application/vnd.ms-powerpoint=MS Powerpoint application/vnd.openxmlformats-officedocument.presentationml.presentation=MS Powerpoint application/vnd.openxmlformats-officedocument.wordprocessingml.document=MS Word application/vnd.oasis.opendocument.spreadsheet=OpenOffice Spreadsheet +application/vnd.ms-excel.sheet.macroenabled.12=MS Excel Spreadsheet # Text text/plain=Plain Text +text/x-log=Application Log text/html=HTML application/x-tex=LaTeX text/x-tex=LaTeX @@ -21,6 +24,7 @@ text/markdown=Markdown Text text/x-markdown=Markdown Text text/x-r-markdown=R Markdown Text application/rtf=Rich Text Format +text/x-rst=reStructuredText text/rtf=Rich Text Format text/richtext=Rich Text Format text/turtle=Turtle RDF @@ -28,16 +32,26 @@ application/xml=XML text/xml=XML # Code text/x-c=C++ Source +text/x-c++src=C++ Source text/css=Cascading Style Sheet +text/x-fortran=Fortran Source Code +application/java-vm=Java Class +text/x-java-source=Java Source Code text/javascript=Javascript Code application/javascript=Javascript Code application/x-javascript=Javascript Code text/x-matlab=MATLAB Source Code text/x-mathematica=Mathematica Input +text/x-objcsrc=Objective-C Source Code +text/x-pascal=Pascal Source Code +text/x-perl=Perl Script +text/x-perl-script=Perl Script text/php=PHP Source Code +application/postscript=Postscript text/x-python=Python Source Code text/x-python-script=Python Source Code text/x-r-source=R Source Code +text/x-sh=Shell Script application/x-sh=Shell Script application/x-shellscript=Shell Script application/x-sql=SQL Code @@ -57,13 +71,18 @@ application/x-spss-sps=SPSS Script Syntax text/x-sas-syntax=SAS Syntax application/x-sas-syntax=SAS Syntax type/x-r-syntax=R Syntax +application/vnd.wolfram.mathematica.package=Mathematica Package +application/vnd.wolfram.mathematica=Mathematica Package # Ingested Tabular Data text/tab-separated-values=Tab-Delimited # RawData text/tsv=Tab-Separated Values text/comma-separated-values=Comma Separated Values +text/x-comma-separated-values=Comma Separated Values text/csv=Comma Separated Values text/x-fixed-field=Fixed Field Text Data +application/vnd.flographit=FloGraphIt Media +application/x-r-data=R Data application/x-rlang-transport=R Data application/x-R-2=R Binary application/x-stata=Stata Binary @@ -94,6 +113,7 @@ application/matlab-mat=MATLAB Data application/x-matlab-data=MATLAB Data application/x-matlab-figure=MATLAB Figure application/x-matlab-workspace=MATLAB Workspace +text/x-vcard=Virtual Contact File application/x-xfig=MATLAB Figure application/x-msaccess=MS Access application/netcdf=Network Common Data Form @@ -101,6 +121,10 @@ application/x-netcdf=Network Common Data Form application/vnd.lotus-notes=Notes Storage Facility application/x-nsdstat=NSDstat application/vnd.realvnc.bed=PLINK Binary +application/vnd.ms-pki.stl=Data +application/vnd.isac.fcs=Data +application/java-serialized-object=Java Serialized Object +chemical/x-xyz=Co-Ordinate Animation # FITS image/fits=FITS application/fits=FITS @@ -112,7 +136,9 @@ application/sbn=ESRI Spatial Index application/sbx=ESRI Spatial Index application/shp=Shape application/shx=Shape -application/zipped-shapefile=Shape +application/x-esri-shape=ESRI Shapefile +application/vnd.google-earth.kml+xml=Keyhole Markup Language +application/zipped-shapefile=Zipped Shapefiles # Archive application/zip=ZIP Archive application/x-zip-compressed=ZIP Archive @@ -122,15 +148,18 @@ application/x-bzip2=Bzip Archive application/vnd.google-earth.kmz=Google Earth Archive application/gzip=Gzip Archive application/x-gzip=Gzip Archive +application/x-gzip-compressed=Gzip Archive application/rar=RAR Archive application/x-rar=RAR Archive application/x-rar-compressed=RAR Archive application/tar=TAR Archive application/x-tar=TAR Archive +application/x-compressed=Compressed Archive application/x-compressed-tar=TAR Archive application/x-7z-compressed=7Z Archive application/x-xz=XZ Archive application/warc=Web Archive +application/x-iso9660-image=Optical Disc Image # Image image/gif=GIF Image image/jpeg=JPEG Image @@ -151,8 +180,13 @@ image/tiff=TIFF Image image/bmp=Bitmap Image image/x-xbitmap=Bitmap Image image/RAW=Bitmap Image +image/raw=Bitmap Image +application/x-tgif=TGIF File image/x-xpixmap=Pixmap Image image/x-xwindowdump=X Windows Dump +application/photoshop=Photoshop Image +image/vnd.adobe.photoshop=Photoshop Image +application/x-photoshop=Photoshop Image # Audio audio/x-aiff=AIFF Audio audio/mp3=MP3 Audio @@ -177,4 +211,4 @@ text/xml-graphml=GraphML Network Data # Other application/octet-stream=Unknown # Dataverse-specific -application/vnd.dataverse.file-package=Dataverse Package \ No newline at end of file +application/vnd.dataverse.file-package=Dataverse Package From 944776c04ddbde0134b41f680d3d89550ce2a96b Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Tue, 12 Jan 2021 11:04:38 -0500 Subject: [PATCH 4/7] Missed a couple of data file types --- src/main/java/propertyFiles/MimeTypeDisplay.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 6899f9b4518..8b07a87832d 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -121,8 +121,8 @@ application/x-netcdf=Network Common Data Form application/vnd.lotus-notes=Notes Storage Facility application/x-nsdstat=NSDstat application/vnd.realvnc.bed=PLINK Binary -application/vnd.ms-pki.stl=Data -application/vnd.isac.fcs=Data +application/vnd.ms-pki.stl=STL Format +application/vnd.isac.fcs=FCS Data application/java-serialized-object=Java Serialized Object chemical/x-xyz=Co-Ordinate Animation # FITS From 0e21c22c0374919f5e87c430abd59b6e3239a1d0 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Tue, 12 Jan 2021 15:11:29 -0500 Subject: [PATCH 5/7] Update 7502-more-mime-types.md Updated release notes to specify reindex rather than reidentify. --- doc/release-notes/7502-more-mime-types.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/7502-more-mime-types.md b/doc/release-notes/7502-more-mime-types.md index 8fc3f33f92f..e5a7f7c95ed 100644 --- a/doc/release-notes/7502-more-mime-types.md +++ b/doc/release-notes/7502-more-mime-types.md @@ -1,4 +1,7 @@ ## Upgrade Steps -- Redetect file types using the Redetect File Types API: -https://guides.dataverse.org/en/latest/api/native-api.html#redetect-file-type \ No newline at end of file +In addition to mapping friendly names to these file types, the types are further mapped to aggregate file types facets on the homepage. This requires them to be reindexed to appear in the facet count. + +Kick off full reindex + +http://guides.dataverse.org/en/4.20/admin/solr-search-index.html From f3ab76c02597b1847971a7c48c83a2f8e13e4324 Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Tue, 12 Jan 2021 15:17:14 -0500 Subject: [PATCH 6/7] Revisions from Ana - thanks! --- src/main/java/propertyFiles/MimeTypeDisplay.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 8b07a87832d..614ecb9b12a 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -5,7 +5,7 @@ application/pdf=Adobe PDF image/pdf=Adobe PDF text/pdf=Adobe PDF application/x-pdf=Adobe PDF -application/cnt=CNT +application/cnt=Windows Help Contents File application/msword=MS Word application/vnd.ms-excel=MS Excel Spreadsheet application/vnd.openxmlformats-officedocument.spreadsheetml.sheet=MS Excel Spreadsheet @@ -71,8 +71,8 @@ application/x-spss-sps=SPSS Script Syntax text/x-sas-syntax=SAS Syntax application/x-sas-syntax=SAS Syntax type/x-r-syntax=R Syntax -application/vnd.wolfram.mathematica.package=Mathematica Package -application/vnd.wolfram.mathematica=Mathematica Package +application/vnd.wolfram.mathematica.package=Wolfram Mathematica Code +application/vnd.wolfram.mathematica=Wolfram Mathematica Code # Ingested Tabular Data text/tab-separated-values=Tab-Delimited # RawData From 7cc0ea642403e9971863167ca033736140abb20a Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Tue, 12 Jan 2021 15:19:42 -0500 Subject: [PATCH 7/7] Update 7502-more-mime-types.md Tweak based on Leonid's input --- doc/release-notes/7502-more-mime-types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7502-more-mime-types.md b/doc/release-notes/7502-more-mime-types.md index e5a7f7c95ed..0c57dc3e389 100644 --- a/doc/release-notes/7502-more-mime-types.md +++ b/doc/release-notes/7502-more-mime-types.md @@ -1,6 +1,6 @@ ## Upgrade Steps -In addition to mapping friendly names to these file types, the types are further mapped to aggregate file types facets on the homepage. This requires them to be reindexed to appear in the facet count. +In addition to mapping friendly names to these file types, the types are further mapped to aggregate file types facets on the homepage. A full reindex is required for the facets to be refreshed. Kick off full reindex