From 1c659c6ae8c02c51b83bddab176f36be966a6221 Mon Sep 17 00:00:00 2001 From: FLIO Date: Wed, 11 Jan 2023 15:38:13 +0100 Subject: [PATCH 1/6] feature(#21426) Added in the export module the possibility to choose between utf-8 and iso format --- ....modules.php => export_csviso.modules.php} | 20 +- .../modules/export/export_csvutf8.modules.php | 379 ++++++++++++++++++ 2 files changed, 386 insertions(+), 13 deletions(-) rename htdocs/core/modules/export/{export_csv.modules.php => export_csviso.modules.php} (94%) create mode 100644 htdocs/core/modules/export/export_csvutf8.modules.php diff --git a/htdocs/core/modules/export/export_csv.modules.php b/htdocs/core/modules/export/export_csviso.modules.php similarity index 94% rename from htdocs/core/modules/export/export_csv.modules.php rename to htdocs/core/modules/export/export_csviso.modules.php index 26f20378b23ec..00759d3de55e1 100644 --- a/htdocs/core/modules/export/export_csv.modules.php +++ b/htdocs/core/modules/export/export_csviso.modules.php @@ -74,9 +74,8 @@ public function __construct($db) } $this->escape = '"'; $this->enclosure = '"'; - - $this->id = 'csv'; // Same value then xxx in file name export_xxx.modules.php - $this->label = 'CSV'; // Label of driver + $this->id = 'csviso'; // Same value then xxx in file name export_xxx.modules.php + $this->label = 'CSV ISO-8859-1'; // Label of driver $this->desc = $langs->trans("CSVFormatDesc", $this->separator, $this->enclosure, $this->escape); $this->extension = 'csv'; // Extension for generated file by this driver $this->picto = 'mime/other'; // Picto @@ -215,11 +214,9 @@ public function write_title($array_export_fields_label, $array_selected_sorted, // phpcs:enable global $conf; - if (!empty($conf->global->EXPORT_CSV_FORCE_CHARSET)) { - $outputlangs->charset_output = $conf->global->EXPORT_CSV_FORCE_CHARSET; - } else { - $outputlangs->charset_output = 'ISO-8859-1'; - } + $outputlangs->charset_output = 'ISO-8859-1'; + $conf->global->EXPORT_CSV_FORCE_CHARSET = 'ISO-8859-1'; + $selectlabel = array(); foreach ($array_selected_sorted as $code => $value) { @@ -256,11 +253,8 @@ public function write_record($array_selected_sorted, $objp, $outputlangs, $array // phpcs:enable global $conf; - if (!empty($conf->global->EXPORT_CSV_FORCE_CHARSET)) { - $outputlangs->charset_output = $conf->global->EXPORT_CSV_FORCE_CHARSET; - } else { - $outputlangs->charset_output = 'ISO-8859-1'; - } + $outputlangs->charset_output = 'ISO-8859-1'; + $conf->global->EXPORT_CSV_FORCE_CHARSET = 'ISO-8859-1'; $this->col = 0; diff --git a/htdocs/core/modules/export/export_csvutf8.modules.php b/htdocs/core/modules/export/export_csvutf8.modules.php new file mode 100644 index 0000000000000..75958fb7b3534 --- /dev/null +++ b/htdocs/core/modules/export/export_csvutf8.modules.php @@ -0,0 +1,379 @@ + + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** + * \file htdocs/core/modules/export/export_csv.modules.php + * \ingroup export + * \brief File of class to build exports with CSV format + */ + +require_once DOL_DOCUMENT_ROOT.'/core/modules/export/modules_export.php'; + +// avoid timeout for big export +set_time_limit(0); + +/** + * Class to build export files with format CSV + */ +class ExportCsvUtf8 extends ModeleExports +{ + /** + * @var string ID ex: csv, tsv, excel... + */ + public $id; + + /** + * @var string export files label + */ + public $label; + + public $extension; + + /** + * Dolibarr version of the loaded document + * @var string + */ + public $version = 'dolibarr'; + + public $label_lib; + + public $version_lib; + + public $separator; + + public $handle; // Handle fichier + + + /** + * Constructor + * + * @param DoliDB $db Database handler + */ + public function __construct($db) + { + global $conf, $langs; + $this->db = $db; + + $this->separator = ','; + if (!empty($conf->global->EXPORT_CSV_SEPARATOR_TO_USE)) { + $this->separator = $conf->global->EXPORT_CSV_SEPARATOR_TO_USE; + } + $this->escape = '"'; + $this->enclosure = '"'; + + $this->id = 'csvutf8'; // Same value then xxx in file name export_xxx.modules.php + $this->label = 'CSV UTF-8'; // Label of driver + $this->desc = $langs->trans("CSVFormatDesc", $this->separator, $this->enclosure, $this->escape); + $this->extension = 'csv'; // Extension for generated file by this driver + $this->picto = 'mime/other'; // Picto + $this->version = '1.32'; // Driver version + + // If driver use an external library, put its name here + $this->label_lib = 'Dolibarr'; + $this->version_lib = DOL_VERSION; + } + + /** + * getDriverId + * + * @return string + */ + public function getDriverId() + { + return $this->id; + } + + /** + * getDriverLabel + * + * @return string Return driver label + */ + public function getDriverLabel() + { + return $this->label; + } + + /** + * getDriverDesc + * + * @return string + */ + public function getDriverDesc() + { + return $this->desc; + } + + /** + * getDriverExtension + * + * @return string + */ + public function getDriverExtension() + { + return $this->extension; + } + + /** + * getDriverVersion + * + * @return string + */ + public function getDriverVersion() + { + return $this->version; + } + + /** + * getLabelLabel + * + * @return string + */ + public function getLibLabel() + { + return $this->label_lib; + } + + /** + * getLibVersion + * + * @return string + */ + public function getLibVersion() + { + return $this->version_lib; + } + + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Open output file + * + * @param string $file Path of filename to generate + * @param Translate $outputlangs Output language object + * @return int <0 if KO, >=0 if OK + */ + public function open_file($file, $outputlangs) + { + // phpcs:enable + global $langs; + + dol_syslog("ExportCsv::open_file file=".$file); + + $ret = 1; + + $outputlangs->load("exports"); + $this->handle = fopen($file, "wt"); + if (!$this->handle) { + $langs->load("errors"); + $this->error = $langs->trans("ErrorFailToCreateFile", $file); + $ret = -1; + } + + return $ret; + } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output header into file + * + * @param Translate $outputlangs Output language object + * @return int <0 if KO, >0 if OK + */ + public function write_header($outputlangs) + { + // phpcs:enable + return 0; + } + + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output title line into file + * + * @param array $array_export_fields_label Array with list of label of fields + * @param array $array_selected_sorted Array with list of field to export + * @param Translate $outputlangs Object lang to translate values + * @param array $array_types Array with types of fields + * @return int <0 if KO, >0 if OK + */ + public function write_title($array_export_fields_label, $array_selected_sorted, $outputlangs, $array_types) + { + // phpcs:enable + global $conf; + + $outputlangs->charset_output = 'UTF-8'; + // var_dump($outputlangs->charset_output);exit; + // var_dump($conf->global->EXPORT_CSV_FORCE_CHARSET, "here");exit; + $selectlabel = array(); + + foreach ($array_selected_sorted as $code => $value) { + $newvalue = $outputlangs->transnoentities($array_export_fields_label[$code]); // newvalue is now $outputlangs->charset_output encoded + $newvalue = $this->csvClean($newvalue, $outputlangs->charset_output); + + fwrite($this->handle, $newvalue.$this->separator); + $typefield = isset($array_types[$code]) ? $array_types[$code] : ''; + + if (preg_match('/^Select:/i', $typefield) && $typefield = substr($typefield, 7)) { + $selectlabel[$code."_label"] = $newvalue."_label"; + } + } + foreach ($selectlabel as $key => $value) { + fwrite($this->handle, $value.$this->separator); + } + fwrite($this->handle, "\n"); + return 0; + } + + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output record line into file + * + * @param array $array_selected_sorted Array with list of field to export + * @param resource $objp A record from a fetch with all fields from select + * @param Translate $outputlangs Object lang to translate values + * @param array $array_types Array with types of fields + * @return int <0 if KO, >0 if OK + */ + public function write_record($array_selected_sorted, $objp, $outputlangs, $array_types) + { + // phpcs:enable + global $conf; + + $outputlangs->charset_output = 'UTF-8'; + // var_dump($conf->global->EXPORT_CSV_FORCE_CHARSET, "here");exit; + $this->col = 0; + + $reg = array(); + $selectlabelvalues = array(); + foreach ($array_selected_sorted as $code => $value) { + if (strpos($code, ' as ') == 0) { + $alias = str_replace(array('.', '-', '(', ')'), '_', $code); + } else { + $alias = substr($code, strpos($code, ' as ') + 4); + } + if (empty($alias)) { + dol_print_error('', 'Bad value for field with key='.$code.'. Try to redefine export.'); + } + + $newvalue = $outputlangs->convToOutputCharset($objp->$alias); // objp->$alias must be utf8 encoded as any var in memory // newvalue is now $outputlangs->charset_output encoded + $typefield = isset($array_types[$code]) ? $array_types[$code] : ''; + + // Translation newvalue + if (preg_match('/^\((.*)\)$/i', $newvalue, $reg)) { + $newvalue = $outputlangs->transnoentities($reg[1]); + } + + // Clean data and add encloser if required (depending on value of USE_STRICT_CSV_RULES) + $newvalue = $this->csvClean($newvalue, $outputlangs->charset_output); + + if (preg_match('/^Select:/i', $typefield) && $typefield = substr($typefield, 7)) { + $array = jsonOrUnserialize($typefield); + if (is_array($array) && !empty($newvalue)) { + $array = $array['options']; + $selectlabelvalues[$code."_label"] = $array[$newvalue]; + } else { + $selectlabelvalues[$code."_label"] = ""; + } + } + + fwrite($this->handle, $newvalue.$this->separator); + $this->col++; + } + foreach ($selectlabelvalues as $key => $value) { + fwrite($this->handle, $value.$this->separator); + $this->col++; + } + + fwrite($this->handle, "\n"); + return 0; + } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output footer into file + * + * @param Translate $outputlangs Output language object + * @return int <0 if KO, >0 if OK + */ + public function write_footer($outputlangs) + { + // phpcs:enable + return 0; + } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Close file handle + * + * @return int <0 if KO, >0 if OK + */ + public function close_file() + { + // phpcs:enable + fclose($this->handle); + return 0; + } + + + /** + * Clean a cell to respect rules of CSV file cells + * Note: It uses $this->separator + * Note: We keep this function public to be able to test + * + * @param string $newvalue String to clean + * @param string $charset Input AND Output character set + * @return string Value cleaned + */ + public function csvClean($newvalue, $charset) + { + global $conf; + $addquote = 0; + + // Rule Dolibarr: No HTML + //print $charset.' '.$newvalue."\n"; + //$newvalue=dol_string_nohtmltag($newvalue,0,$charset); + $newvalue = dol_htmlcleanlastbr($newvalue); + //print $charset.' '.$newvalue."\n"; + + // Rule 1 CSV: No CR, LF in cells (except if USE_STRICT_CSV_RULES is 1, we can keep record as it is but we must add quotes) + $oldvalue = $newvalue; + $newvalue = str_replace("\r", '', $newvalue); + $newvalue = str_replace("\n", '\n', $newvalue); + if (!empty($conf->global->USE_STRICT_CSV_RULES) && $oldvalue != $newvalue) { + // If we must use enclusure on text with CR/LF) + if ($conf->global->USE_STRICT_CSV_RULES == 1) { + // If we use strict CSV rules (original value must remain but we add quote) + $newvalue = $oldvalue; + } + $addquote = 1; + } + + // Rule 2 CSV: If value contains ", we must escape with ", and add " + if (preg_match('/"/', $newvalue)) { + $addquote = 1; + $newvalue = str_replace('"', '""', $newvalue); + } + + // Rule 3 CSV: If value contains separator, we must add " + if (preg_match('/'.$this->separator.'/', $newvalue)) { + $addquote = 1; + } + + return ($addquote ? '"' : '').$newvalue.($addquote ? '"' : ''); + } +} From c5d3474e10b87d7cccf8546c12a06da49a5a8054 Mon Sep 17 00:00:00 2001 From: FLIO Date: Tue, 17 Jan 2023 18:25:49 +0100 Subject: [PATCH 2/6] Create class module export csv for extend csv utf-8 and iso --- .../modules/export/export_csviso.modules.php | 328 +---------------- .../modules/export/export_csvutf8.modules.php | 328 +---------------- .../core/modules/export/module_export_csv.php | 348 ++++++++++++++++++ 3 files changed, 362 insertions(+), 642 deletions(-) create mode 100644 htdocs/core/modules/export/module_export_csv.php diff --git a/htdocs/core/modules/export/export_csviso.modules.php b/htdocs/core/modules/export/export_csviso.modules.php index 00759d3de55e1..b477868c09c8d 100644 --- a/htdocs/core/modules/export/export_csviso.modules.php +++ b/htdocs/core/modules/export/export_csviso.modules.php @@ -21,43 +21,17 @@ * \brief File of class to build exports with CSV format */ -require_once DOL_DOCUMENT_ROOT.'/core/modules/export/modules_export.php'; +require_once DOL_DOCUMENT_ROOT.'/core/modules/export/module_export_csv.php'; // avoid timeout for big export set_time_limit(0); /** - * Class to build export files with format CSV + * Class to build export files with format CSV iso */ -class ExportCsv extends ModeleExports +class ExportCsvIso extends ExportCsv { - /** - * @var string ID ex: csv, tsv, excel... - */ - public $id; - - /** - * @var string export files label - */ - public $label; - - public $extension; - - /** - * Dolibarr version of the loaded document - * @var string - */ - public $version = 'dolibarr'; - - public $label_lib; - - public $version_lib; - - public $separator; - - public $handle; // Handle fichier - - + /** * Constructor * @@ -72,6 +46,9 @@ public function __construct($db) if (!empty($conf->global->EXPORT_CSV_SEPARATOR_TO_USE)) { $this->separator = $conf->global->EXPORT_CSV_SEPARATOR_TO_USE; } + + $conf->global->EXPORT_CSV_FORCE_CHARSET = 'ISO-8859-1'; + $this->escape = '"'; $this->enclosure = '"'; $this->id = 'csviso'; // Same value then xxx in file name export_xxx.modules.php @@ -85,295 +62,4 @@ public function __construct($db) $this->label_lib = 'Dolibarr'; $this->version_lib = DOL_VERSION; } - - /** - * getDriverId - * - * @return string - */ - public function getDriverId() - { - return $this->id; - } - - /** - * getDriverLabel - * - * @return string Return driver label - */ - public function getDriverLabel() - { - return $this->label; - } - - /** - * getDriverDesc - * - * @return string - */ - public function getDriverDesc() - { - return $this->desc; - } - - /** - * getDriverExtension - * - * @return string - */ - public function getDriverExtension() - { - return $this->extension; - } - - /** - * getDriverVersion - * - * @return string - */ - public function getDriverVersion() - { - return $this->version; - } - - /** - * getLabelLabel - * - * @return string - */ - public function getLibLabel() - { - return $this->label_lib; - } - - /** - * getLibVersion - * - * @return string - */ - public function getLibVersion() - { - return $this->version_lib; - } - - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Open output file - * - * @param string $file Path of filename to generate - * @param Translate $outputlangs Output language object - * @return int <0 if KO, >=0 if OK - */ - public function open_file($file, $outputlangs) - { - // phpcs:enable - global $langs; - - dol_syslog("ExportCsv::open_file file=".$file); - - $ret = 1; - - $outputlangs->load("exports"); - $this->handle = fopen($file, "wt"); - if (!$this->handle) { - $langs->load("errors"); - $this->error = $langs->trans("ErrorFailToCreateFile", $file); - $ret = -1; - } - - return $ret; - } - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Output header into file - * - * @param Translate $outputlangs Output language object - * @return int <0 if KO, >0 if OK - */ - public function write_header($outputlangs) - { - // phpcs:enable - return 0; - } - - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Output title line into file - * - * @param array $array_export_fields_label Array with list of label of fields - * @param array $array_selected_sorted Array with list of field to export - * @param Translate $outputlangs Object lang to translate values - * @param array $array_types Array with types of fields - * @return int <0 if KO, >0 if OK - */ - public function write_title($array_export_fields_label, $array_selected_sorted, $outputlangs, $array_types) - { - // phpcs:enable - global $conf; - - $outputlangs->charset_output = 'ISO-8859-1'; - $conf->global->EXPORT_CSV_FORCE_CHARSET = 'ISO-8859-1'; - - $selectlabel = array(); - - foreach ($array_selected_sorted as $code => $value) { - $newvalue = $outputlangs->transnoentities($array_export_fields_label[$code]); // newvalue is now $outputlangs->charset_output encoded - $newvalue = $this->csvClean($newvalue, $outputlangs->charset_output); - - fwrite($this->handle, $newvalue.$this->separator); - $typefield = isset($array_types[$code]) ? $array_types[$code] : ''; - - if (preg_match('/^Select:/i', $typefield) && $typefield = substr($typefield, 7)) { - $selectlabel[$code."_label"] = $newvalue."_label"; - } - } - foreach ($selectlabel as $key => $value) { - fwrite($this->handle, $value.$this->separator); - } - fwrite($this->handle, "\n"); - return 0; - } - - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Output record line into file - * - * @param array $array_selected_sorted Array with list of field to export - * @param resource $objp A record from a fetch with all fields from select - * @param Translate $outputlangs Object lang to translate values - * @param array $array_types Array with types of fields - * @return int <0 if KO, >0 if OK - */ - public function write_record($array_selected_sorted, $objp, $outputlangs, $array_types) - { - // phpcs:enable - global $conf; - - $outputlangs->charset_output = 'ISO-8859-1'; - $conf->global->EXPORT_CSV_FORCE_CHARSET = 'ISO-8859-1'; - - $this->col = 0; - - $reg = array(); - $selectlabelvalues = array(); - foreach ($array_selected_sorted as $code => $value) { - if (strpos($code, ' as ') == 0) { - $alias = str_replace(array('.', '-', '(', ')'), '_', $code); - } else { - $alias = substr($code, strpos($code, ' as ') + 4); - } - if (empty($alias)) { - dol_print_error('', 'Bad value for field with key='.$code.'. Try to redefine export.'); - } - - $newvalue = $outputlangs->convToOutputCharset($objp->$alias); // objp->$alias must be utf8 encoded as any var in memory // newvalue is now $outputlangs->charset_output encoded - $typefield = isset($array_types[$code]) ? $array_types[$code] : ''; - - // Translation newvalue - if (preg_match('/^\((.*)\)$/i', $newvalue, $reg)) { - $newvalue = $outputlangs->transnoentities($reg[1]); - } - - // Clean data and add encloser if required (depending on value of USE_STRICT_CSV_RULES) - $newvalue = $this->csvClean($newvalue, $outputlangs->charset_output); - - if (preg_match('/^Select:/i', $typefield) && $typefield = substr($typefield, 7)) { - $array = jsonOrUnserialize($typefield); - if (is_array($array) && !empty($newvalue)) { - $array = $array['options']; - $selectlabelvalues[$code."_label"] = $array[$newvalue]; - } else { - $selectlabelvalues[$code."_label"] = ""; - } - } - - fwrite($this->handle, $newvalue.$this->separator); - $this->col++; - } - foreach ($selectlabelvalues as $key => $value) { - fwrite($this->handle, $value.$this->separator); - $this->col++; - } - - fwrite($this->handle, "\n"); - return 0; - } - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Output footer into file - * - * @param Translate $outputlangs Output language object - * @return int <0 if KO, >0 if OK - */ - public function write_footer($outputlangs) - { - // phpcs:enable - return 0; - } - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Close file handle - * - * @return int <0 if KO, >0 if OK - */ - public function close_file() - { - // phpcs:enable - fclose($this->handle); - return 0; - } - - - /** - * Clean a cell to respect rules of CSV file cells - * Note: It uses $this->separator - * Note: We keep this function public to be able to test - * - * @param string $newvalue String to clean - * @param string $charset Input AND Output character set - * @return string Value cleaned - */ - public function csvClean($newvalue, $charset) - { - global $conf; - $addquote = 0; - - // Rule Dolibarr: No HTML - //print $charset.' '.$newvalue."\n"; - //$newvalue=dol_string_nohtmltag($newvalue,0,$charset); - $newvalue = dol_htmlcleanlastbr($newvalue); - //print $charset.' '.$newvalue."\n"; - - // Rule 1 CSV: No CR, LF in cells (except if USE_STRICT_CSV_RULES is 1, we can keep record as it is but we must add quotes) - $oldvalue = $newvalue; - $newvalue = str_replace("\r", '', $newvalue); - $newvalue = str_replace("\n", '\n', $newvalue); - if (!empty($conf->global->USE_STRICT_CSV_RULES) && $oldvalue != $newvalue) { - // If we must use enclusure on text with CR/LF) - if ($conf->global->USE_STRICT_CSV_RULES == 1) { - // If we use strict CSV rules (original value must remain but we add quote) - $newvalue = $oldvalue; - } - $addquote = 1; - } - - // Rule 2 CSV: If value contains ", we must escape with ", and add " - if (preg_match('/"/', $newvalue)) { - $addquote = 1; - $newvalue = str_replace('"', '""', $newvalue); - } - - // Rule 3 CSV: If value contains separator, we must add " - if (preg_match('/'.$this->separator.'/', $newvalue)) { - $addquote = 1; - } - - return ($addquote ? '"' : '').$newvalue.($addquote ? '"' : ''); - } } diff --git a/htdocs/core/modules/export/export_csvutf8.modules.php b/htdocs/core/modules/export/export_csvutf8.modules.php index 75958fb7b3534..f8b254cabe9a2 100644 --- a/htdocs/core/modules/export/export_csvutf8.modules.php +++ b/htdocs/core/modules/export/export_csvutf8.modules.php @@ -21,43 +21,17 @@ * \brief File of class to build exports with CSV format */ -require_once DOL_DOCUMENT_ROOT.'/core/modules/export/modules_export.php'; +require_once DOL_DOCUMENT_ROOT.'/core/modules/export/module_export_csv.php'; // avoid timeout for big export set_time_limit(0); /** - * Class to build export files with format CSV + * Class to build export files with format CSV utf-8 */ -class ExportCsvUtf8 extends ModeleExports +class ExportCsvUtf8 extends ExportCsv { - /** - * @var string ID ex: csv, tsv, excel... - */ - public $id; - - /** - * @var string export files label - */ - public $label; - - public $extension; - - /** - * Dolibarr version of the loaded document - * @var string - */ - public $version = 'dolibarr'; - - public $label_lib; - - public $version_lib; - - public $separator; - - public $handle; // Handle fichier - - + /** * Constructor * @@ -72,9 +46,11 @@ public function __construct($db) if (!empty($conf->global->EXPORT_CSV_SEPARATOR_TO_USE)) { $this->separator = $conf->global->EXPORT_CSV_SEPARATOR_TO_USE; } + + $conf->global->EXPORT_CSV_FORCE_CHARSET = 'UTF-8'; + $this->escape = '"'; $this->enclosure = '"'; - $this->id = 'csvutf8'; // Same value then xxx in file name export_xxx.modules.php $this->label = 'CSV UTF-8'; // Label of driver $this->desc = $langs->trans("CSVFormatDesc", $this->separator, $this->enclosure, $this->escape); @@ -86,294 +62,4 @@ public function __construct($db) $this->label_lib = 'Dolibarr'; $this->version_lib = DOL_VERSION; } - - /** - * getDriverId - * - * @return string - */ - public function getDriverId() - { - return $this->id; - } - - /** - * getDriverLabel - * - * @return string Return driver label - */ - public function getDriverLabel() - { - return $this->label; - } - - /** - * getDriverDesc - * - * @return string - */ - public function getDriverDesc() - { - return $this->desc; - } - - /** - * getDriverExtension - * - * @return string - */ - public function getDriverExtension() - { - return $this->extension; - } - - /** - * getDriverVersion - * - * @return string - */ - public function getDriverVersion() - { - return $this->version; - } - - /** - * getLabelLabel - * - * @return string - */ - public function getLibLabel() - { - return $this->label_lib; - } - - /** - * getLibVersion - * - * @return string - */ - public function getLibVersion() - { - return $this->version_lib; - } - - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Open output file - * - * @param string $file Path of filename to generate - * @param Translate $outputlangs Output language object - * @return int <0 if KO, >=0 if OK - */ - public function open_file($file, $outputlangs) - { - // phpcs:enable - global $langs; - - dol_syslog("ExportCsv::open_file file=".$file); - - $ret = 1; - - $outputlangs->load("exports"); - $this->handle = fopen($file, "wt"); - if (!$this->handle) { - $langs->load("errors"); - $this->error = $langs->trans("ErrorFailToCreateFile", $file); - $ret = -1; - } - - return $ret; - } - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Output header into file - * - * @param Translate $outputlangs Output language object - * @return int <0 if KO, >0 if OK - */ - public function write_header($outputlangs) - { - // phpcs:enable - return 0; - } - - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Output title line into file - * - * @param array $array_export_fields_label Array with list of label of fields - * @param array $array_selected_sorted Array with list of field to export - * @param Translate $outputlangs Object lang to translate values - * @param array $array_types Array with types of fields - * @return int <0 if KO, >0 if OK - */ - public function write_title($array_export_fields_label, $array_selected_sorted, $outputlangs, $array_types) - { - // phpcs:enable - global $conf; - - $outputlangs->charset_output = 'UTF-8'; - // var_dump($outputlangs->charset_output);exit; - // var_dump($conf->global->EXPORT_CSV_FORCE_CHARSET, "here");exit; - $selectlabel = array(); - - foreach ($array_selected_sorted as $code => $value) { - $newvalue = $outputlangs->transnoentities($array_export_fields_label[$code]); // newvalue is now $outputlangs->charset_output encoded - $newvalue = $this->csvClean($newvalue, $outputlangs->charset_output); - - fwrite($this->handle, $newvalue.$this->separator); - $typefield = isset($array_types[$code]) ? $array_types[$code] : ''; - - if (preg_match('/^Select:/i', $typefield) && $typefield = substr($typefield, 7)) { - $selectlabel[$code."_label"] = $newvalue."_label"; - } - } - foreach ($selectlabel as $key => $value) { - fwrite($this->handle, $value.$this->separator); - } - fwrite($this->handle, "\n"); - return 0; - } - - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Output record line into file - * - * @param array $array_selected_sorted Array with list of field to export - * @param resource $objp A record from a fetch with all fields from select - * @param Translate $outputlangs Object lang to translate values - * @param array $array_types Array with types of fields - * @return int <0 if KO, >0 if OK - */ - public function write_record($array_selected_sorted, $objp, $outputlangs, $array_types) - { - // phpcs:enable - global $conf; - - $outputlangs->charset_output = 'UTF-8'; - // var_dump($conf->global->EXPORT_CSV_FORCE_CHARSET, "here");exit; - $this->col = 0; - - $reg = array(); - $selectlabelvalues = array(); - foreach ($array_selected_sorted as $code => $value) { - if (strpos($code, ' as ') == 0) { - $alias = str_replace(array('.', '-', '(', ')'), '_', $code); - } else { - $alias = substr($code, strpos($code, ' as ') + 4); - } - if (empty($alias)) { - dol_print_error('', 'Bad value for field with key='.$code.'. Try to redefine export.'); - } - - $newvalue = $outputlangs->convToOutputCharset($objp->$alias); // objp->$alias must be utf8 encoded as any var in memory // newvalue is now $outputlangs->charset_output encoded - $typefield = isset($array_types[$code]) ? $array_types[$code] : ''; - - // Translation newvalue - if (preg_match('/^\((.*)\)$/i', $newvalue, $reg)) { - $newvalue = $outputlangs->transnoentities($reg[1]); - } - - // Clean data and add encloser if required (depending on value of USE_STRICT_CSV_RULES) - $newvalue = $this->csvClean($newvalue, $outputlangs->charset_output); - - if (preg_match('/^Select:/i', $typefield) && $typefield = substr($typefield, 7)) { - $array = jsonOrUnserialize($typefield); - if (is_array($array) && !empty($newvalue)) { - $array = $array['options']; - $selectlabelvalues[$code."_label"] = $array[$newvalue]; - } else { - $selectlabelvalues[$code."_label"] = ""; - } - } - - fwrite($this->handle, $newvalue.$this->separator); - $this->col++; - } - foreach ($selectlabelvalues as $key => $value) { - fwrite($this->handle, $value.$this->separator); - $this->col++; - } - - fwrite($this->handle, "\n"); - return 0; - } - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Output footer into file - * - * @param Translate $outputlangs Output language object - * @return int <0 if KO, >0 if OK - */ - public function write_footer($outputlangs) - { - // phpcs:enable - return 0; - } - - // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps - /** - * Close file handle - * - * @return int <0 if KO, >0 if OK - */ - public function close_file() - { - // phpcs:enable - fclose($this->handle); - return 0; - } - - - /** - * Clean a cell to respect rules of CSV file cells - * Note: It uses $this->separator - * Note: We keep this function public to be able to test - * - * @param string $newvalue String to clean - * @param string $charset Input AND Output character set - * @return string Value cleaned - */ - public function csvClean($newvalue, $charset) - { - global $conf; - $addquote = 0; - - // Rule Dolibarr: No HTML - //print $charset.' '.$newvalue."\n"; - //$newvalue=dol_string_nohtmltag($newvalue,0,$charset); - $newvalue = dol_htmlcleanlastbr($newvalue); - //print $charset.' '.$newvalue."\n"; - - // Rule 1 CSV: No CR, LF in cells (except if USE_STRICT_CSV_RULES is 1, we can keep record as it is but we must add quotes) - $oldvalue = $newvalue; - $newvalue = str_replace("\r", '', $newvalue); - $newvalue = str_replace("\n", '\n', $newvalue); - if (!empty($conf->global->USE_STRICT_CSV_RULES) && $oldvalue != $newvalue) { - // If we must use enclusure on text with CR/LF) - if ($conf->global->USE_STRICT_CSV_RULES == 1) { - // If we use strict CSV rules (original value must remain but we add quote) - $newvalue = $oldvalue; - } - $addquote = 1; - } - - // Rule 2 CSV: If value contains ", we must escape with ", and add " - if (preg_match('/"/', $newvalue)) { - $addquote = 1; - $newvalue = str_replace('"', '""', $newvalue); - } - - // Rule 3 CSV: If value contains separator, we must add " - if (preg_match('/'.$this->separator.'/', $newvalue)) { - $addquote = 1; - } - - return ($addquote ? '"' : '').$newvalue.($addquote ? '"' : ''); - } } diff --git a/htdocs/core/modules/export/module_export_csv.php b/htdocs/core/modules/export/module_export_csv.php new file mode 100644 index 0000000000000..442c7cb628f25 --- /dev/null +++ b/htdocs/core/modules/export/module_export_csv.php @@ -0,0 +1,348 @@ + + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** + * \file htdocs/core/modules/export/export_csv.modules.php + * \ingroup export + * \brief File of class to build exports with CSV format + */ + +require_once DOL_DOCUMENT_ROOT.'/core/modules/export/modules_export.php'; + +// avoid timeout for big export +set_time_limit(0); + +/** + * Class to build export files with format CSV + */ +class ExportCsv extends ModeleExports +{ + /** + * @var string ID ex: csv, tsv, excel... + */ + public $id; + + /** + * @var string export files label + */ + public $label; + + public $extension; + + /** + * Dolibarr version of the loaded document + * @var string + */ + public $version = 'dolibarr'; + + public $label_lib; + + public $version_lib; + + public $separator; + + public $handle; // Handle fichier + + /** + * getDriverId + * + * @return string + */ + public function getDriverId() + { + return $this->id; + } + + /** + * getDriverLabel + * + * @return string Return driver label + */ + public function getDriverLabel() + { + return $this->label; + } + + /** + * getDriverDesc + * + * @return string + */ + public function getDriverDesc() + { + return $this->desc; + } + + /** + * getDriverExtension + * + * @return string + */ + public function getDriverExtension() + { + return $this->extension; + } + + /** + * getDriverVersion + * + * @return string + */ + public function getDriverVersion() + { + return $this->version; + } + + /** + * getLabelLabel + * + * @return string + */ + public function getLibLabel() + { + return $this->label_lib; + } + + /** + * getLibVersion + * + * @return string + */ + public function getLibVersion() + { + return $this->version_lib; + } + + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Open output file + * + * @param string $file Path of filename to generate + * @param Translate $outputlangs Output language object + * @return int <0 if KO, >=0 if OK + */ + public function open_file($file, $outputlangs) + { + // phpcs:enable + global $langs; + + dol_syslog("ExportCsv::open_file file=".$file); + + $ret = 1; + + $outputlangs->load("exports"); + $this->handle = fopen($file, "wt"); + if (!$this->handle) { + $langs->load("errors"); + $this->error = $langs->trans("ErrorFailToCreateFile", $file); + $ret = -1; + } + + return $ret; + } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output header into file + * + * @param Translate $outputlangs Output language object + * @return int <0 if KO, >0 if OK + */ + public function write_header($outputlangs) + { + // phpcs:enable + return 0; + } + + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output title line into file + * + * @param array $array_export_fields_label Array with list of label of fields + * @param array $array_selected_sorted Array with list of field to export + * @param Translate $outputlangs Object lang to translate values + * @param array $array_types Array with types of fields + * @return int <0 if KO, >0 if OK + */ + public function write_title($array_export_fields_label, $array_selected_sorted, $outputlangs, $array_types) + { + // phpcs:enable + global $conf; + + $outputlangs->charset_output = $conf->global->EXPORT_CSV_FORCE_CHARSET; + + $selectlabel = array(); + + foreach ($array_selected_sorted as $code => $value) { + $newvalue = $outputlangs->transnoentities($array_export_fields_label[$code]); // newvalue is now $outputlangs->charset_output encoded + $newvalue = $this->csvClean($newvalue, $outputlangs->charset_output); + + fwrite($this->handle, $newvalue.$this->separator); + $typefield = isset($array_types[$code]) ? $array_types[$code] : ''; + + if (preg_match('/^Select:/i', $typefield) && $typefield = substr($typefield, 7)) { + $selectlabel[$code."_label"] = $newvalue."_label"; + } + } + foreach ($selectlabel as $key => $value) { + fwrite($this->handle, $value.$this->separator); + } + fwrite($this->handle, "\n"); + return 0; + } + + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output record line into file + * + * @param array $array_selected_sorted Array with list of field to export + * @param resource $objp A record from a fetch with all fields from select + * @param Translate $outputlangs Object lang to translate values + * @param array $array_types Array with types of fields + * @return int <0 if KO, >0 if OK + */ + public function write_record($array_selected_sorted, $objp, $outputlangs, $array_types) + { + // phpcs:enable + global $conf; + + $outputlangs->charset_output = $conf->global->EXPORT_CSV_FORCE_CHARSET; + + $this->col = 0; + + $reg = array(); + $selectlabelvalues = array(); + foreach ($array_selected_sorted as $code => $value) { + if (strpos($code, ' as ') == 0) { + $alias = str_replace(array('.', '-', '(', ')'), '_', $code); + } else { + $alias = substr($code, strpos($code, ' as ') + 4); + } + if (empty($alias)) { + dol_print_error('', 'Bad value for field with key='.$code.'. Try to redefine export.'); + } + + $newvalue = $outputlangs->convToOutputCharset($objp->$alias); // objp->$alias must be utf8 encoded as any var in memory // newvalue is now $outputlangs->charset_output encoded + $typefield = isset($array_types[$code]) ? $array_types[$code] : ''; + + // Translation newvalue + if (preg_match('/^\((.*)\)$/i', $newvalue, $reg)) { + $newvalue = $outputlangs->transnoentities($reg[1]); + } + + // Clean data and add encloser if required (depending on value of USE_STRICT_CSV_RULES) + $newvalue = $this->csvClean($newvalue, $outputlangs->charset_output); + + if (preg_match('/^Select:/i', $typefield) && $typefield = substr($typefield, 7)) { + $array = jsonOrUnserialize($typefield); + if (is_array($array) && !empty($newvalue)) { + $array = $array['options']; + $selectlabelvalues[$code."_label"] = $array[$newvalue]; + } else { + $selectlabelvalues[$code."_label"] = ""; + } + } + + fwrite($this->handle, $newvalue.$this->separator); + $this->col++; + } + foreach ($selectlabelvalues as $key => $value) { + fwrite($this->handle, $value.$this->separator); + $this->col++; + } + + fwrite($this->handle, "\n"); + return 0; + } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output footer into file + * + * @param Translate $outputlangs Output language object + * @return int <0 if KO, >0 if OK + */ + public function write_footer($outputlangs) + { + // phpcs:enable + return 0; + } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Close file handle + * + * @return int <0 if KO, >0 if OK + */ + public function close_file() + { + // phpcs:enable + fclose($this->handle); + return 0; + } + + + /** + * Clean a cell to respect rules of CSV file cells + * Note: It uses $this->separator + * Note: We keep this function public to be able to test + * + * @param string $newvalue String to clean + * @param string $charset Input AND Output character set + * @return string Value cleaned + */ + public function csvClean($newvalue, $charset) + { + global $conf; + $addquote = 0; + + // Rule Dolibarr: No HTML + //print $charset.' '.$newvalue."\n"; + //$newvalue=dol_string_nohtmltag($newvalue,0,$charset); + $newvalue = dol_htmlcleanlastbr($newvalue); + //print $charset.' '.$newvalue."\n"; + + // Rule 1 CSV: No CR, LF in cells (except if USE_STRICT_CSV_RULES is 1, we can keep record as it is but we must add quotes) + $oldvalue = $newvalue; + $newvalue = str_replace("\r", '', $newvalue); + $newvalue = str_replace("\n", '\n', $newvalue); + if (!empty($conf->global->USE_STRICT_CSV_RULES) && $oldvalue != $newvalue) { + // If we must use enclusure on text with CR/LF) + if ($conf->global->USE_STRICT_CSV_RULES == 1) { + // If we use strict CSV rules (original value must remain but we add quote) + $newvalue = $oldvalue; + } + $addquote = 1; + } + + // Rule 2 CSV: If value contains ", we must escape with ", and add " + if (preg_match('/"/', $newvalue)) { + $addquote = 1; + $newvalue = str_replace('"', '""', $newvalue); + } + + // Rule 3 CSV: If value contains separator, we must add " + if (preg_match('/'.$this->separator.'/', $newvalue)) { + $addquote = 1; + } + + return ($addquote ? '"' : '').$newvalue.($addquote ? '"' : ''); + } +} From 28d0f492f487922e59a042ff36142da37b9d4477 Mon Sep 17 00:00:00 2001 From: FLIO Date: Tue, 17 Jan 2023 18:26:19 +0100 Subject: [PATCH 3/6] Create class module export csv --- htdocs/core/modules/export/export_csviso.modules.php | 2 +- htdocs/core/modules/export/export_csvutf8.modules.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/htdocs/core/modules/export/export_csviso.modules.php b/htdocs/core/modules/export/export_csviso.modules.php index b477868c09c8d..573dbdcadd04d 100644 --- a/htdocs/core/modules/export/export_csviso.modules.php +++ b/htdocs/core/modules/export/export_csviso.modules.php @@ -31,7 +31,7 @@ */ class ExportCsvIso extends ExportCsv { - + /** * Constructor * diff --git a/htdocs/core/modules/export/export_csvutf8.modules.php b/htdocs/core/modules/export/export_csvutf8.modules.php index f8b254cabe9a2..13000c00e2595 100644 --- a/htdocs/core/modules/export/export_csvutf8.modules.php +++ b/htdocs/core/modules/export/export_csvutf8.modules.php @@ -31,7 +31,7 @@ */ class ExportCsvUtf8 extends ExportCsv { - + /** * Constructor * From c8c998044b3ada2e4d0f4a7c312af34d2324d6c8 Mon Sep 17 00:00:00 2001 From: FLIO Date: Tue, 7 Feb 2023 12:13:34 +0100 Subject: [PATCH 4/6] feature renamed the module_export_csv file and edited the iso and utf files to override the write_title and write_record functions to initialize EXPORT_CSV_FORCE_CHARSET --- ..._export_csv.php => export_csv.modules.php} | 0 .../modules/export/export_csviso.modules.php | 38 +++++++++++++++++- .../modules/export/export_csvutf8.modules.php | 40 +++++++++++++++++-- 3 files changed, 74 insertions(+), 4 deletions(-) rename htdocs/core/modules/export/{module_export_csv.php => export_csv.modules.php} (100%) diff --git a/htdocs/core/modules/export/module_export_csv.php b/htdocs/core/modules/export/export_csv.modules.php similarity index 100% rename from htdocs/core/modules/export/module_export_csv.php rename to htdocs/core/modules/export/export_csv.modules.php diff --git a/htdocs/core/modules/export/export_csviso.modules.php b/htdocs/core/modules/export/export_csviso.modules.php index 573dbdcadd04d..ad40e7ae807e3 100644 --- a/htdocs/core/modules/export/export_csviso.modules.php +++ b/htdocs/core/modules/export/export_csviso.modules.php @@ -21,7 +21,7 @@ * \brief File of class to build exports with CSV format */ -require_once DOL_DOCUMENT_ROOT.'/core/modules/export/module_export_csv.php'; +require_once DOL_DOCUMENT_ROOT.'/core/modules/export/export_csv.modules.php'; // avoid timeout for big export set_time_limit(0); @@ -62,4 +62,40 @@ public function __construct($db) $this->label_lib = 'Dolibarr'; $this->version_lib = DOL_VERSION; } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output title line into file + * + * @param array $array_export_fields_label Array with list of label of fields + * @param array $array_selected_sorted Array with list of field to export + * @param Translate $outputlangs Object lang to translate values + * @param array $array_types Array with types of fields + * @return int <0 if KO, >0 if OK + */ + public function write_title($array_export_fields_label, $array_selected_sorted, $outputlangs, $array_types) + { + global $conf; + $conf->global->EXPORT_CSV_FORCE_CHARSET = 'ISO-8859-1'; + + parent::write_title($array_export_fields_label, $array_selected_sorted, $outputlangs, $array_types); + } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output record line into file + * + * @param array $array_selected_sorted Array with list of field to export + * @param resource $objp A record from a fetch with all fields from select + * @param Translate $outputlangs Object lang to translate values + * @param array $array_types Array with types of fields + * @return int <0 if KO, >0 if OK + */ + public function write_record($array_selected_sorted, $objp, $outputlangs, $array_types) + { + global $conf; + $conf->global->EXPORT_CSV_FORCE_CHARSET = 'ISO-8859-1'; + + parent::write_record($array_selected_sorted, $objp, $outputlangs, $array_types); + } } diff --git a/htdocs/core/modules/export/export_csvutf8.modules.php b/htdocs/core/modules/export/export_csvutf8.modules.php index 13000c00e2595..bc6a7d70d492e 100644 --- a/htdocs/core/modules/export/export_csvutf8.modules.php +++ b/htdocs/core/modules/export/export_csvutf8.modules.php @@ -21,7 +21,7 @@ * \brief File of class to build exports with CSV format */ -require_once DOL_DOCUMENT_ROOT.'/core/modules/export/module_export_csv.php'; +require_once DOL_DOCUMENT_ROOT.'/core/modules/export/export_csv.modules.php'; // avoid timeout for big export set_time_limit(0); @@ -47,8 +47,6 @@ public function __construct($db) $this->separator = $conf->global->EXPORT_CSV_SEPARATOR_TO_USE; } - $conf->global->EXPORT_CSV_FORCE_CHARSET = 'UTF-8'; - $this->escape = '"'; $this->enclosure = '"'; $this->id = 'csvutf8'; // Same value then xxx in file name export_xxx.modules.php @@ -62,4 +60,40 @@ public function __construct($db) $this->label_lib = 'Dolibarr'; $this->version_lib = DOL_VERSION; } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output title line into file + * + * @param array $array_export_fields_label Array with list of label of fields + * @param array $array_selected_sorted Array with list of field to export + * @param Translate $outputlangs Object lang to translate values + * @param array $array_types Array with types of fields + * @return int <0 if KO, >0 if OK + */ + public function write_title($array_export_fields_label, $array_selected_sorted, $outputlangs, $array_types) + { + global $conf; + $conf->global->EXPORT_CSV_FORCE_CHARSET = 'UTF-8'; + + parent::write_title($array_export_fields_label, $array_selected_sorted, $outputlangs, $array_types); + } + + // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps + /** + * Output record line into file + * + * @param array $array_selected_sorted Array with list of field to export + * @param resource $objp A record from a fetch with all fields from select + * @param Translate $outputlangs Object lang to translate values + * @param array $array_types Array with types of fields + * @return int <0 if KO, >0 if OK + */ + public function write_record($array_selected_sorted, $objp, $outputlangs, $array_types) + { + global $conf; + + $conf->global->EXPORT_CSV_FORCE_CHARSET = 'UTF-8'; + parent::write_record($array_selected_sorted, $objp, $outputlangs, $array_types); + } } From c000eeeecf18c5fdb5890e1f1332270c703c4c5c Mon Sep 17 00:00:00 2001 From: FLIO Date: Thu, 16 Feb 2023 01:00:39 +0100 Subject: [PATCH 5/6] feature (#21426) edit ExportTest for resolve error and add test for utf8 --- test/phpunit/ExportTest.php | 102 ++++++++++++++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/test/phpunit/ExportTest.php b/test/phpunit/ExportTest.php index cf2f0520908c7..0a0a1c867ba4c 100644 --- a/test/phpunit/ExportTest.php +++ b/test/phpunit/ExportTest.php @@ -149,6 +149,94 @@ protected function tearDown(): void print __METHOD__."\n"; } + /** + * Other tests + * + * @return void + */ + public function testExportCsvUtf() + { + global $conf,$user,$langs,$db; + + $model='csvutf8'; + + $conf->global->EXPORT_CSV_SEPARATOR_TO_USE = ','; + print 'EXPORT_CSV_SEPARATOR_TO_USE = '.$conf->global->EXPORT_CSV_SEPARATOR_TO_USE; + + // Creation of class to export using model ExportXXX + $dir = DOL_DOCUMENT_ROOT . "/core/modules/export/"; + $file = "export_".$model.".modules.php"; + $classname = "Export".$model; + require_once $dir.$file; + $objmodel = new $classname($db); + + // First test without option USE_STRICT_CSV_RULES + unset($conf->global->USE_STRICT_CSV_RULES); + + $valtotest='A simple string'; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, 'A simple string'); + + $valtotest='A string with , and ; inside'; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, '"A string with , and ; inside"', 'Error in csvClean for '.$file); + + $valtotest='A string with " inside'; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, '"A string with "" inside"'); + + $valtotest='A string with " inside and '."\r\n".' carriage returns'; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, '"A string with "" inside and \n carriage returns"'); + + $valtotest='A string with html
content
inside
'."\n"; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, '"A string with html
content
inside"'); + + // Same tests with strict mode + $conf->global->USE_STRICT_CSV_RULES = 1; + + $valtotest='A simple string'; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, 'A simple string'); + + $valtotest='A string with , and ; inside'; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, '"A string with , and ; inside"'); + + $valtotest='A string with " inside'; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, '"A string with "" inside"'); + + $valtotest='A string with " inside and '."\r\n".' carriage returns'; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, "\"A string with \"\" inside and \r\n carriage returns\""); + + $valtotest='A string with html
content
inside
'."\n"; + print __METHOD__." valtotest=".$valtotest."\n"; + $result = $objmodel->csvClean($valtotest, $langs->charset_output); + print __METHOD__." result=".$result."\n"; + $this->assertEquals($result, '"A string with html
content
inside"'); + } + /** * Other tests @@ -159,7 +247,7 @@ public function testExportOther() { global $conf,$user,$langs,$db; - $model='csv'; + $model='csviso'; $conf->global->EXPORT_CSV_SEPARATOR_TO_USE = ','; print 'EXPORT_CSV_SEPARATOR_TO_USE = '.$conf->global->EXPORT_CSV_SEPARATOR_TO_USE; @@ -263,7 +351,15 @@ public function testExportPersonalizedExport() dol_mkdir($conf->export->dir_temp); - $model='csv'; + $model='csviso'; + + // Build export file + print "Process build_file for model = ".$model."\n"; + $result=$objexport->build_file($user, $model, $datatoexport, $array_selected, array(), $sql); + $expectedresult = 1; + $this->assertEquals($expectedresult, $result, 'Error in CSV export'); + + $model='csvutf8'; // Build export file print "Process build_file for model = ".$model."\n"; @@ -353,7 +449,7 @@ public function testExportModulesDatasets() { global $conf,$user,$langs,$db; - $model='csv'; + $model='csviso'; $filterdatatoexport=''; //$filterdatatoexport=''; From cfe6ebfeee54fc313bb6ae58504ec03e377c6385 Mon Sep 17 00:00:00 2001 From: FLIO Date: Wed, 22 Feb 2023 10:29:47 +0100 Subject: [PATCH 6/6] fix (#21426) edit file name of export_csv.modules because it adds an export type when choosing the export type --- htdocs/core/modules/export/export_csviso.modules.php | 2 +- htdocs/core/modules/export/export_csvutf8.modules.php | 2 +- .../export/{export_csv.modules.php => exports_csv.modules.php} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename htdocs/core/modules/export/{export_csv.modules.php => exports_csv.modules.php} (100%) diff --git a/htdocs/core/modules/export/export_csviso.modules.php b/htdocs/core/modules/export/export_csviso.modules.php index ad40e7ae807e3..29a9e24e76acf 100644 --- a/htdocs/core/modules/export/export_csviso.modules.php +++ b/htdocs/core/modules/export/export_csviso.modules.php @@ -21,7 +21,7 @@ * \brief File of class to build exports with CSV format */ -require_once DOL_DOCUMENT_ROOT.'/core/modules/export/export_csv.modules.php'; +require_once DOL_DOCUMENT_ROOT.'/core/modules/export/exports_csv.modules.php'; // avoid timeout for big export set_time_limit(0); diff --git a/htdocs/core/modules/export/export_csvutf8.modules.php b/htdocs/core/modules/export/export_csvutf8.modules.php index bc6a7d70d492e..f517dd00839dc 100644 --- a/htdocs/core/modules/export/export_csvutf8.modules.php +++ b/htdocs/core/modules/export/export_csvutf8.modules.php @@ -21,7 +21,7 @@ * \brief File of class to build exports with CSV format */ -require_once DOL_DOCUMENT_ROOT.'/core/modules/export/export_csv.modules.php'; +require_once DOL_DOCUMENT_ROOT.'/core/modules/export/exports_csv.modules.php'; // avoid timeout for big export set_time_limit(0); diff --git a/htdocs/core/modules/export/export_csv.modules.php b/htdocs/core/modules/export/exports_csv.modules.php similarity index 100% rename from htdocs/core/modules/export/export_csv.modules.php rename to htdocs/core/modules/export/exports_csv.modules.php