diff --git a/conf/toolbox-ext.xml b/conf/toolbox-ext.xml index 57518c1..3bab0c2 100644 --- a/conf/toolbox-ext.xml +++ b/conf/toolbox-ext.xml @@ -1,6 +1,6 @@ excel - application + request nl.isaac.dotcms.excelreader.viewtool.ExcelReaderTool diff --git a/lib/SuperCSV-with_src-1.52.jar b/lib/SuperCSV-with_src-1.52.jar new file mode 100644 index 0000000..e17802b Binary files /dev/null and b/lib/SuperCSV-with_src-1.52.jar differ diff --git a/src/nl/isaac/dotcms/excelreader/shared/CacheGroupHandler.java b/src/nl/isaac/dotcms/excelreader/shared/CacheGroupHandler.java index eeedbd5..f21d0c8 100644 --- a/src/nl/isaac/dotcms/excelreader/shared/CacheGroupHandler.java +++ b/src/nl/isaac/dotcms/excelreader/shared/CacheGroupHandler.java @@ -22,58 +22,58 @@ * * @param */ -public class CacheGroupHandler { +public class CacheGroupHandler { private String groupName; - protected ItemHandler itemHandler; + protected ItemHandler itemHandler; - public CacheGroupHandler(String groupName, ItemHandler itemHandler) { + public CacheGroupHandler(String groupName, ItemHandler itemHandler) { this.groupName = groupName; this.itemHandler = itemHandler; } - public T get(String key) { + public V get(K key) { DotCacheAdministrator cache = CacheLocator.getCacheAdministrator(); Object o = null; if(!itemHandler.isChanged(key)) { try { - o = cache.get(key, groupName); + o = cache.get(key.toString(), groupName); } catch (DotCacheException e) { Logger.info(this.getClass(), String.format("DotCacheException for Group '%s', key '%s', message: %s", groupName, key, e.getMessage())); } } if(o == null) { - T t = itemHandler.get(key); + V t = itemHandler.get(key); put(key, t); return t; } else { - return (T)o; + return (V)o; } } - public void put(String key, T t) { + public void put(K key, V t) { DotCacheAdministrator cache = CacheLocator.getCacheAdministrator(); - cache.put(key, t, groupName); + cache.put(key.toString(), t, groupName); } /** * Updates the given key by calling the itemhandler's get method */ - public void updateWithItemHandler(String key) { + public void updateWithItemHandler(K key) { remove(key); put(key, itemHandler.get(key)); } - public void remove(String key) { + public void remove(K key) { DotCacheAdministrator cache = CacheLocator.getCacheAdministrator(); - cache.remove(key, groupName); + cache.remove(key.toString(), groupName); } public void fillInitialCache() { removeAll(); - Map initialCache = itemHandler.getInitialCache(); - for(Entry entry: initialCache.entrySet()) { + Map initialCache = itemHandler.getInitialCache(); + for(Entry entry: initialCache.entrySet()) { put(entry.getKey(), entry.getValue()); } } diff --git a/src/nl/isaac/dotcms/excelreader/shared/ItemHandler.java b/src/nl/isaac/dotcms/excelreader/shared/ItemHandler.java index 974c13c..3136d05 100644 --- a/src/nl/isaac/dotcms/excelreader/shared/ItemHandler.java +++ b/src/nl/isaac/dotcms/excelreader/shared/ItemHandler.java @@ -17,8 +17,8 @@ * * @param */ -public interface ItemHandler { - public T get(String key); - public boolean isChanged(String key); - public Map getInitialCache(); +public interface ItemHandler { + public V get(K key); + public boolean isChanged(K key); + public Map getInitialCache(); } \ No newline at end of file diff --git a/src/nl/isaac/dotcms/excelreader/shared/RequestUtil.java b/src/nl/isaac/dotcms/excelreader/shared/RequestUtil.java new file mode 100644 index 0000000..5c26136 --- /dev/null +++ b/src/nl/isaac/dotcms/excelreader/shared/RequestUtil.java @@ -0,0 +1,36 @@ +package nl.isaac.dotcms.excelreader.shared; + +import javax.servlet.http.HttpServletRequest; + +import com.dotmarketing.util.WebKeys; + +public class RequestUtil { + public static boolean isLiveMode(HttpServletRequest request) { + return !(isEditMode(request) || isPreviewMode(request)); + } + + public static boolean isEditMode(HttpServletRequest request) { + Object EDIT_MODE_SESSION = request.getSession().getAttribute(com.dotmarketing.util.WebKeys.EDIT_MODE_SESSION); + if(EDIT_MODE_SESSION != null) { + return Boolean.valueOf(EDIT_MODE_SESSION.toString()); + } + return false; + } + + public static boolean isPreviewMode(HttpServletRequest request) { + Object PREVIEW_MODE_SESSION = request.getSession().getAttribute(com.dotmarketing.util.WebKeys.PREVIEW_MODE_SESSION); + if(PREVIEW_MODE_SESSION != null) { + return Boolean.valueOf(PREVIEW_MODE_SESSION.toString()); + } + return false; + } + + public static Integer getLanguage(HttpServletRequest request) { + return (Integer)request.getSession().getAttribute(WebKeys.HTMLPAGE_LANGUAGE); + } + + public static Integer getSelectedLanguage(HttpServletRequest request) { + return (Integer)request.getSession().getAttribute(WebKeys.LANGUAGE); + } + +} diff --git a/src/nl/isaac/dotcms/excelreader/util/ExcelReaderCacheGroupHandler.java b/src/nl/isaac/dotcms/excelreader/util/ExcelReaderCacheGroupHandler.java index 50b6f19..0a02653 100644 --- a/src/nl/isaac/dotcms/excelreader/util/ExcelReaderCacheGroupHandler.java +++ b/src/nl/isaac/dotcms/excelreader/util/ExcelReaderCacheGroupHandler.java @@ -18,7 +18,7 @@ * @author xander * */ -public class ExcelReaderCacheGroupHandler extends CacheGroupHandler>> { +public class ExcelReaderCacheGroupHandler extends CacheGroupHandler>> { private static ExcelReaderCacheGroupHandler cache; private ExcelReaderCacheGroupHandler() { diff --git a/src/nl/isaac/dotcms/excelreader/util/ExcelReaderDotCMSFileKey.java b/src/nl/isaac/dotcms/excelreader/util/ExcelReaderDotCMSFileKey.java new file mode 100644 index 0000000..aae7cc5 --- /dev/null +++ b/src/nl/isaac/dotcms/excelreader/util/ExcelReaderDotCMSFileKey.java @@ -0,0 +1,39 @@ +package nl.isaac.dotcms.excelreader.util; + +import com.dotmarketing.beans.Host; + +public class ExcelReaderDotCMSFileKey extends ExcelReaderFileKey { + private final Host host; + private final boolean live; + + public ExcelReaderDotCMSFileKey(String path, Host host, boolean live) { + super(path); + this.host = host; + this.live = live; + } + + public Host getHost() { + return host; + } + + public boolean isLive() { + return live; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof ExcelReaderDotCMSFileKey) { + ExcelReaderDotCMSFileKey key = (ExcelReaderDotCMSFileKey)obj; + return super.equals(key) && + key.getHost().getHostname().equals(getHost().getHostname()) && + (key.isLive() == isLive()); + } + return false; + } + + @Override + public String toString() { + return super.toString() + "_" + getHost().getHostname() + "_" + isLive(); + } + +} diff --git a/src/nl/isaac/dotcms/excelreader/util/ExcelReaderFileKey.java b/src/nl/isaac/dotcms/excelreader/util/ExcelReaderFileKey.java new file mode 100644 index 0000000..12df813 --- /dev/null +++ b/src/nl/isaac/dotcms/excelreader/util/ExcelReaderFileKey.java @@ -0,0 +1,32 @@ +package nl.isaac.dotcms.excelreader.util; + +public class ExcelReaderFileKey { + private final String path; + + public ExcelReaderFileKey(String path) { + this.path = path; + } + + public String getPath() { + return path; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof ExcelReaderFileKey) { + return ((ExcelReaderFileKey)obj).getPath().equals(getPath()); + } + return false; + } + + @Override + public int hashCode() { + return getPath().hashCode(); + } + + @Override + public String toString() { + return getPath(); + } + +} diff --git a/src/nl/isaac/dotcms/excelreader/util/ExcelReaderItemHandler.java b/src/nl/isaac/dotcms/excelreader/util/ExcelReaderItemHandler.java index 520a8be..5a5c9e3 100644 --- a/src/nl/isaac/dotcms/excelreader/util/ExcelReaderItemHandler.java +++ b/src/nl/isaac/dotcms/excelreader/util/ExcelReaderItemHandler.java @@ -8,13 +8,19 @@ * @copyright Copyright (c) 2011 ISAAC Software Solutions B.V. (http://www.isaac.nl) */ +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.portlets.files.factories.FileFactory; import com.dotmarketing.util.Logger; import nl.isaac.dotcms.excelreader.shared.ItemHandler; @@ -25,17 +31,25 @@ * @author xander */ -public class ExcelReaderItemHandler implements ItemHandler>> { - private Map lastModDates = new HashMap(); +public class ExcelReaderItemHandler implements ItemHandler>> { + private Map lastModDates = new HashMap(); /** * @return whether the key has changed since it was last requested */ - public boolean isChanged(String key) { + public boolean isChanged(ExcelReaderFileKey key) { try { if(lastModDates.containsKey(key)) { - File file = new File(key); - return file == null || !lastModDates.get(key).equals(file.lastModified()); + if(key instanceof ExcelReaderDotCMSFileKey) { + //get from dotCMS + ExcelReaderDotCMSFileKey dotcmsKey = (ExcelReaderDotCMSFileKey)key; + com.dotmarketing.portlets.files.model.File file = FileFactory.getFileByURI(dotcmsKey.getPath(), dotcmsKey.getHost(), dotcmsKey.isLive()); + return file == null || !lastModDates.get(key).equals(file.getModDate().getTime()); + } else { + //get from the file system + File file = new File(key.getPath()); + return file == null || !lastModDates.get(key).equals(file.lastModified()); + } } else { return true; } @@ -48,26 +62,55 @@ public boolean isChanged(String key) { /** * @return an excel sheet as a List of Maps. The key is the location of the file (on the harddisk) */ - public List> get(String key) { + public List> get(ExcelReaderFileKey key) { Calendar start = Calendar.getInstance(); + InputStream is = null; + File file = null; + com.dotmarketing.portlets.files.model.File dotcmsFile = null; try { - File file = new File(key); - FileInputStream fis = new FileInputStream(file); + if(key instanceof ExcelReaderDotCMSFileKey) { + //get from dotCMS + ExcelReaderDotCMSFileKey dotcmsKey = (ExcelReaderDotCMSFileKey)key; + dotcmsFile = FileFactory.getFileByURI(dotcmsKey.getPath(), dotcmsKey.getHost(), dotcmsKey.isLive()); + is = new ByteArrayInputStream(FileFactory.getFileData(dotcmsFile)); + } else { + //get from the file system + file = new File(key.getPath()); + is = new FileInputStream(file); + } + DefaultRowStrategy strategy = new DefaultRowStrategy(); ExcelUtilStatus status = new ExcelUtilStatus(); - ExcelUtil.executeStrategyOnExcelSheet(fis, strategy, status); + ExcelUtil.executeStrategyOnExcelSheet(is, strategy, status); Logger.info(this.getClass(), "Reading of excel '" + key + "' took " + (Calendar.getInstance().getTimeInMillis() - start.getTimeInMillis()) + "ms"); - lastModDates.put(key, file.lastModified()); + + if(key instanceof ExcelReaderDotCMSFileKey) { + lastModDates.put(key, dotcmsFile.getModDate().getTime()); + } else { + lastModDates.put(key, file.lastModified()); + } + for(Entry entry: status.getMapWithRowNumbersAndExceptions().entrySet()) { + Logger.info(this.getClass(), "Row " + entry.getKey() + " has error: " + entry.getValue().getMessage()); + } + Logger.info(this.getClass(), "Successfully read " + status.getNumberOfImportedRows() + " row and skipped " + status.getNumberOfFailedRows()); return strategy.getData(); } catch (Throwable t) { Logger.error(this.getClass(), "Error while reading excel", t); return null; + } finally { + if(is != null) { + try { + is.close(); + } catch (IOException e) { + Logger.warn(this.getClass(), "Unable to close file connection to " + key.getPath()); + } + } } } - - public Map>> getInitialCache() { - return new HashMap>>(); + @Override + public Map>> getInitialCache() { + return new HashMap>>(); } } diff --git a/src/nl/isaac/dotcms/excelreader/util/ExcelUtil.java b/src/nl/isaac/dotcms/excelreader/util/ExcelUtil.java index 84df166..6416ac7 100644 --- a/src/nl/isaac/dotcms/excelreader/util/ExcelUtil.java +++ b/src/nl/isaac/dotcms/excelreader/util/ExcelUtil.java @@ -8,8 +8,11 @@ * @copyright Copyright (c) 2011 ISAAC Software Solutions B.V. (http://www.isaac.nl) */ +import java.io.BufferedInputStream; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -17,13 +20,15 @@ import java.util.Map; import java.util.Map.Entry; -import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.supercsv.io.CsvMapReader; +import org.supercsv.io.ICsvMapReader; +import org.supercsv.prefs.CsvPreference; import com.dotmarketing.util.Logger; @@ -39,10 +44,14 @@ public class ExcelUtil { * @return an ExcelUtilStatus containing info about the executed rows * @throws IOException when there's a problem with the excel sheet file */ - public static void executeStrategyOnExcelSheet(FileInputStream fis, RowStrategy rowStrategy, ExcelUtilStatus status) throws IOException { + public static void executeStrategyOnExcelSheet(InputStream is, RowStrategy rowStrategy, ExcelUtilStatus status) throws IOException { + //The bufferedInputStream is used, because the WorkbookFactory reads first 8 bytes of data before it go's to the exception. + //Without this inputStream the first 8 characters of the first columnname will be removed. + BufferedInputStream bis = new BufferedInputStream(is); + bis.mark(32); Map headerMapping = new HashMap(); try { - Workbook workbook = WorkbookFactory.create(fis); + Workbook workbook = WorkbookFactory.create(bis); Sheet sheet = workbook.getSheetAt(0); status.setTotalNumberOfRows(sheet.getLastRowNum()); Iterator rowIterator = sheet.rowIterator(); @@ -61,10 +70,31 @@ public static void executeStrategyOnExcelSheet(FileInputStream fis, RowStrategy } } } - } catch(InvalidFormatException e) { - Logger.error(ExcelUtil.class, "Can't read Excel file", e); + } catch(Exception e) { + bis.reset(); + //TODO: catch the exceptions from the workbookfactory: IllegalArgumentException + IllegalFormatException + Logger.warn(ExcelUtil.class, "Can't read Excel file, trying csv format: " + e.getMessage()); + ICsvMapReader inFile = new CsvMapReader(new InputStreamReader(bis), CsvPreference.EXCEL_PREFERENCE); + try { + final String[] header = inFile.getCSVHeader(true); + Map stringMap; + while((stringMap = inFile.read(header)) != null) { + Map row = getStringMapAsMap(stringMap); + try { + rowStrategy.executeRow(row); + status.addSuccesfulRow(); + } catch (Exception e2) { + status.addFailedRowWithException(e2); + } + } + } catch (IOException ioe) { + Logger.error(ExcelUtil.class, "Can't read excel file as CSV", ioe); + Logger.error(ExcelUtil.class, "Original WorkbookFactory excel reader error:", e); + } finally { + inFile.close(); + } } finally { - fis.close(); + bis.close(); } status.setFinished(); @@ -123,6 +153,17 @@ public static Map getRowAsMap(Row row, Map head return rowMap; } + /** + * @return convert a Map to a Map (for the CSV reader) + */ + private static Map getStringMapAsMap(Map stringMap) { + Map objectMap = new HashMap(); + for(Entry entry: stringMap.entrySet()) { + objectMap.put(entry.getKey(), entry.getValue()); + } + return objectMap; + } + /** * A strategy that can be performed on a given row. * An exception can be thrown when there's something wrong with the data. diff --git a/src/nl/isaac/dotcms/excelreader/viewtool/ExcelReaderTool.java b/src/nl/isaac/dotcms/excelreader/viewtool/ExcelReaderTool.java index c4a3869..fbab793 100644 --- a/src/nl/isaac/dotcms/excelreader/viewtool/ExcelReaderTool.java +++ b/src/nl/isaac/dotcms/excelreader/viewtool/ExcelReaderTool.java @@ -15,9 +15,23 @@ import java.util.List; import java.util.Map; +import javax.servlet.http.HttpServletRequest; + +import nl.isaac.dotcms.excelreader.shared.RequestUtil; import nl.isaac.dotcms.excelreader.util.ExcelReaderCacheGroupHandler; +import nl.isaac.dotcms.excelreader.util.ExcelReaderDotCMSFileKey; +import nl.isaac.dotcms.excelreader.util.ExcelReaderFileKey; +import org.apache.velocity.tools.view.context.ViewContext; import org.apache.velocity.tools.view.tools.ViewTool; + +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.business.web.WebAPILocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import com.liferay.portal.PortalException; +import com.liferay.portal.SystemException; /** * A ViewTool to get the information out of Excel files * @@ -25,16 +39,22 @@ * */ public class ExcelReaderTool implements ViewTool { + private HttpServletRequest request; public void init(Object arg0) { - + ViewContext context = (ViewContext) arg0; + this.request = context.getRequest(); } /** * Get the information of an Excel file as a list of maps. */ - public static List> readExcel(String file) { - return ExcelReaderCacheGroupHandler.getInstance().get(file); + public List> readExcel(String file) { + return ExcelReaderCacheGroupHandler.getInstance().get(new ExcelReaderFileKey(file)); + } + + public List> readExcelFromDotCMS(String url) { + return ExcelReaderCacheGroupHandler.getInstance().get(new ExcelReaderDotCMSFileKey(url, getCurrentHost(), isLive())); } /** @@ -81,4 +101,23 @@ public int compare(Map m1, Map m2) { } } + + public Host getCurrentHost() { + try { + return WebAPILocator.getHostWebAPI().getCurrentHost(request); + } catch (SystemException e) { + throw new RuntimeException("Error while getting host", e); + } catch (PortalException e) { + throw new RuntimeException("Error while getting host", e); + } catch (DotDataException e) { + throw new RuntimeException("Error while getting host", e); + } catch (DotSecurityException e) { + throw new RuntimeException("Error while getting host", e); + } + } + + public boolean isLive() { + return RequestUtil.isLiveMode(request); + } + }