diff --git a/core/app.py b/core/app.py
index c43763fd..fc36787b 100644
--- a/core/app.py
+++ b/core/app.py
@@ -26,11 +26,13 @@
from .util import cmp_value, fix_surrogate_encoding
from . import directories, results, export, fs, prioritize
from .ignore import IgnoreList
+from .exclude import ExcludeDict as ExcludeList
from .scanner import ScanType
from .gui.deletion_options import DeletionOptions
from .gui.details_panel import DetailsPanel
from .gui.directory_tree import DirectoryTree
from .gui.ignore_list_dialog import IgnoreListDialog
+from .gui.exclude_list_dialog import ExcludeListDialogCore
from .gui.problem_dialog import ProblemDialog
from .gui.stats_label import StatsLabel
@@ -137,7 +139,8 @@ def __init__(self, view):
os.makedirs(self.appdata)
self.app_mode = AppMode.Standard
self.discarded_file_count = 0
- self.directories = directories.Directories()
+ self.exclude_list = ExcludeList()
+ self.directories = directories.Directories(self.exclude_list)
self.results = results.Results(self)
self.ignore_list = IgnoreList()
# In addition to "app-level" options, this dictionary also holds options that will be
@@ -155,6 +158,7 @@ def __init__(self, view):
self.directory_tree = DirectoryTree(self)
self.problem_dialog = ProblemDialog(self)
self.ignore_list_dialog = IgnoreListDialog(self)
+ self.exclude_list_dialog = ExcludeListDialogCore(self)
self.stats_label = StatsLabel(self)
self.result_table = None
self.deletion_options = DeletionOptions()
@@ -587,6 +591,9 @@ def load(self):
p = op.join(self.appdata, "ignore_list.xml")
self.ignore_list.load_from_xml(p)
self.ignore_list_dialog.refresh()
+ p = op.join(self.appdata, "exclude_list.xml")
+ self.exclude_list.load_from_xml(p)
+ self.exclude_list_dialog.refresh()
def load_directories(self, filepath):
# Clear out previous entries
@@ -779,6 +786,8 @@ def save(self):
self.directories.save_to_file(op.join(self.appdata, "last_directories.xml"))
p = op.join(self.appdata, "ignore_list.xml")
self.ignore_list.save_to_xml(p)
+ p = op.join(self.appdata, "exclude_list.xml")
+ self.exclude_list.save_to_xml(p)
self.notify("save_session")
def save_as(self, filename):
diff --git a/core/directories.py b/core/directories.py
index 66ba8163..7cd103fc 100644
--- a/core/directories.py
+++ b/core/directories.py
@@ -54,10 +54,11 @@ class Directories:
"""
# ---Override
- def __init__(self):
+ def __init__(self, exclude_list=None):
self._dirs = []
# {path: state}
self.states = {}
+ self._exclude_list = exclude_list
def __contains__(self, path):
for p in self._dirs:
@@ -76,39 +77,62 @@ def __len__(self):
# ---Private
def _default_state_for_path(self, path):
+ # New logic with regex filters
+ if self._exclude_list is not None and self._exclude_list.mark_count > 0:
+ # We iterate even if we only have one item here
+ for denied_path_re in self._exclude_list.compiled:
+ if denied_path_re.match(str(path.name)):
+ return DirectoryState.Excluded
+ # return # We still use the old logic to force state on hidden dirs
# Override this in subclasses to specify the state of some special folders.
- if path.name.startswith("."): # hidden
+ if path.name.startswith("."):
return DirectoryState.Excluded
def _get_files(self, from_path, fileclasses, j):
for root, dirs, files in os.walk(str(from_path)):
j.check_if_cancelled()
- root = Path(root)
- state = self.get_state(root)
+ rootPath = Path(root)
+ state = self.get_state(rootPath)
if state == DirectoryState.Excluded:
# Recursively get files from folders with lots of subfolder is expensive. However, there
# might be a subfolder in this path that is not excluded. What we want to do is to skim
# through self.states and see if we must continue, or we can stop right here to save time
- if not any(p[: len(root)] == root for p in self.states):
+ if not any(p[: len(rootPath)] == rootPath for p in self.states):
del dirs[:]
try:
if state != DirectoryState.Excluded:
- found_files = [
- fs.get_file(root + f, fileclasses=fileclasses) for f in files
- ]
+ # Old logic
+ if self._exclude_list is None or not self._exclude_list.mark_count:
+ found_files = [fs.get_file(rootPath + f, fileclasses=fileclasses) for f in files]
+ else:
+ found_files = []
+ # print(f"len of files: {len(files)} {files}")
+ for f in files:
+ found = False
+ for expr in self._exclude_list.compiled_files:
+ if expr.match(f):
+ found = True
+ break
+ if not found:
+ for expr in self._exclude_list.compiled_paths:
+ if expr.match(root + os.sep + f):
+ found = True
+ break
+ if not found:
+ found_files.append(fs.get_file(rootPath + f, fileclasses=fileclasses))
found_files = [f for f in found_files if f is not None]
# In some cases, directories can be considered as files by dupeGuru, which is
# why we have this line below. In fact, there only one case: Bundle files under
# OS X... In other situations, this forloop will do nothing.
for d in dirs[:]:
- f = fs.get_file(root + d, fileclasses=fileclasses)
+ f = fs.get_file(rootPath + d, fileclasses=fileclasses)
if f is not None:
found_files.append(f)
dirs.remove(d)
logging.debug(
"Collected %d files in folder %s",
len(found_files),
- str(from_path),
+ str(rootPath),
)
for file in found_files:
file.is_ref = state == DirectoryState.Reference
@@ -194,8 +218,14 @@ def get_state(self, path):
if path in self.states:
return self.states[path]
state = self._default_state_for_path(path) or DirectoryState.Normal
+ # Save non-default states in cache, necessary for _get_files()
+ if state != DirectoryState.Normal:
+ self.states[path] = state
+ return state
+
prevlen = 0
# we loop through the states to find the longest matching prefix
+ # if the parent has a state in cache, return that state
for p, s in self.states.items():
if p.is_parent_of(path) and len(p) > prevlen:
prevlen = len(p)
diff --git a/core/exclude.py b/core/exclude.py
new file mode 100644
index 00000000..29b00e6b
--- /dev/null
+++ b/core/exclude.py
@@ -0,0 +1,499 @@
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from .markable import Markable
+from xml.etree import ElementTree as ET
+# TODO: perhaps use regex module for better Unicode support? https://pypi.org/project/regex/
+# also https://pypi.org/project/re2/
+# TODO update the Result list with newly added regexes if possible
+import re
+from os import sep
+import logging
+import functools
+from hscommon.util import FileOrPath
+from hscommon.plat import ISWINDOWS
+import time
+
+default_regexes = [r"^thumbs\.db$", # Obsolete after WindowsXP
+ r"^desktop\.ini$", # Windows metadata
+ r"^\.DS_Store$", # MacOS metadata
+ r"^\.Trash\-.*", # Linux trash directories
+ r"^\$Recycle\.Bin$", # Windows
+ r"^\..*", # Hidden files on Unix-like
+ ]
+# These are too broad
+forbidden_regexes = [r".*", r"\/.*", r".*\/.*", r".*\\\\.*", r".*\..*"]
+
+
+def timer(func):
+ @functools.wraps(func)
+ def wrapper_timer(*args):
+ start = time.perf_counter_ns()
+ value = func(*args)
+ end = time.perf_counter_ns()
+ print(f"DEBUG: func {func.__name__!r} took {end - start} ns.")
+ return value
+ return wrapper_timer
+
+
+def memoize(func):
+ func.cache = dict()
+
+ @functools.wraps(func)
+ def _memoize(*args):
+ if args not in func.cache:
+ func.cache[args] = func(*args)
+ return func.cache[args]
+ return _memoize
+
+
+class AlreadyThereException(Exception):
+ """Expression already in the list"""
+ def __init__(self, arg="Expression is already in excluded list."):
+ super().__init__(arg)
+
+
+class ExcludeList(Markable):
+ """A list of lists holding regular expression strings and the compiled re.Pattern"""
+
+ # Used to filter out directories and files that we would rather avoid scanning.
+ # The list() class allows us to preserve item order without too much hassle.
+ # The downside is we have to compare strings every time we look for an item in the list
+ # since we use regex strings as keys.
+ # If _use_union is True, the compiled regexes will be combined into one single
+ # Pattern instead of separate Patterns which may or may not give better
+ # performance compared to looping through each Pattern individually.
+
+ # ---Override
+ def __init__(self, union_regex=True):
+ Markable.__init__(self)
+ self._use_union = union_regex
+ # list([str regex, bool iscompilable, re.error exception, Pattern compiled], ...)
+ self._excluded = []
+ self._excluded_compiled = set()
+ self._dirty = True
+
+ def __iter__(self):
+ """Iterate in order."""
+ for item in self._excluded:
+ regex = item[0]
+ yield self.is_marked(regex), regex
+
+ def __contains__(self, item):
+ return self.isExcluded(item)
+
+ def __len__(self):
+ """Returns the total number of regexes regardless of mark status."""
+ return len(self._excluded)
+
+ def __getitem__(self, key):
+ """Returns the list item corresponding to key."""
+ for item in self._excluded:
+ if item[0] == key:
+ return item
+ raise KeyError(f"Key {key} is not in exclusion list.")
+
+ def __setitem__(self, key, value):
+ # TODO if necessary
+ pass
+
+ def __delitem__(self, key):
+ # TODO if necessary
+ pass
+
+ def get_compiled(self, key):
+ """Returns the (precompiled) Pattern for key"""
+ return self.__getitem__(key)[3]
+
+ def is_markable(self, regex):
+ return self._is_markable(regex)
+
+ def _is_markable(self, regex):
+ """Return the cached result of "compilable" property"""
+ for item in self._excluded:
+ if item[0] == regex:
+ return item[1]
+ return False # should not be necessary, the regex SHOULD be in there
+
+ def _did_mark(self, regex):
+ self._add_compiled(regex)
+
+ def _did_unmark(self, regex):
+ self._remove_compiled(regex)
+
+ def _add_compiled(self, regex):
+ self._dirty = True
+ if self._use_union:
+ return
+ for item in self._excluded:
+ # FIXME probably faster to just rebuild the set from the compiled instead of comparing strings
+ if item[0] == regex:
+ # no need to test if already present since it's a set()
+ self._excluded_compiled.add(item[3])
+ break
+
+ def _remove_compiled(self, regex):
+ self._dirty = True
+ if self._use_union:
+ return
+ for item in self._excluded_compiled:
+ if regex in item.pattern:
+ self._excluded_compiled.remove(item)
+ break
+
+ # @timer
+ @memoize
+ def _do_compile(self, expr):
+ try:
+ return re.compile(expr)
+ except Exception as e:
+ raise(e)
+
+ # @timer
+ # @memoize # probably not worth memoizing this one if we memoize the above
+ def compile_re(self, regex):
+ compiled = None
+ try:
+ compiled = self._do_compile(regex)
+ except Exception as e:
+ return False, e, compiled
+ return True, None, compiled
+
+ def error(self, regex):
+ """Return the compilation error Exception for regex.
+ It should have a "msg" attr."""
+ for item in self._excluded:
+ if item[0] == regex:
+ return item[2]
+
+ def build_compiled_caches(self, union=False):
+ if not union:
+ self._cached_compiled_files =\
+ [x for x in self._excluded_compiled if not has_sep(x.pattern)]
+ self._cached_compiled_paths =\
+ [x for x in self._excluded_compiled if has_sep(x.pattern)]
+ return
+ marked_count = [x for marked, x in self if marked]
+ # If there is no item, the compiled Pattern will be '' and match everything!
+ if not marked_count:
+ self._cached_compiled_union_all = []
+ self._cached_compiled_union_files = []
+ self._cached_compiled_union_paths = []
+ else:
+ # HACK returned as a tuple to get a free iterator and keep interface
+ # the same regardless of whether the client asked for union or not
+ self._cached_compiled_union_all =\
+ (re.compile('|'.join(marked_count)),)
+ files_marked = [x for x in marked_count if not has_sep(x)]
+ if not files_marked:
+ self._cached_compiled_union_files = tuple()
+ else:
+ self._cached_compiled_union_files =\
+ (re.compile('|'.join(files_marked)),)
+ paths_marked = [x for x in marked_count if has_sep(x)]
+ if not paths_marked:
+ self._cached_compiled_union_paths = tuple()
+ else:
+ self._cached_compiled_union_paths =\
+ (re.compile('|'.join(paths_marked)),)
+
+ @property
+ def compiled(self):
+ """Should be used by other classes to retrieve the up-to-date list of patterns."""
+ if self._use_union:
+ if self._dirty:
+ self.build_compiled_caches(True)
+ self._dirty = False
+ return self._cached_compiled_union_all
+ return self._excluded_compiled
+
+ @property
+ def compiled_files(self):
+ """When matching against filenames only, we probably won't be seeing any
+ directory separator, so we filter out regexes with os.sep in them.
+ The interface should be expected to be a generator, even if it returns only
+ one item (one Pattern in the union case)."""
+ if self._dirty:
+ self.build_compiled_caches(True if self._use_union else False)
+ self._dirty = False
+ return self._cached_compiled_union_files if self._use_union\
+ else self._cached_compiled_files
+
+ @property
+ def compiled_paths(self):
+ """Returns patterns with only separators in them, for more precise filtering."""
+ if self._dirty:
+ self.build_compiled_caches(True if self._use_union else False)
+ self._dirty = False
+ return self._cached_compiled_union_paths if self._use_union\
+ else self._cached_compiled_paths
+
+ # ---Public
+ def add(self, regex, forced=False):
+ """This interface should throw exceptions if there is an error during
+ regex compilation"""
+ if self.isExcluded(regex):
+ # This exception should never be ignored
+ raise AlreadyThereException()
+ if regex in forbidden_regexes:
+ raise Exception("Forbidden (dangerous) expression.")
+
+ iscompilable, exception, compiled = self.compile_re(regex)
+ if not iscompilable and not forced:
+ # This exception can be ignored, but taken into account
+ # to avoid adding to compiled set
+ raise exception
+ else:
+ self._do_add(regex, iscompilable, exception, compiled)
+
+ def _do_add(self, regex, iscompilable, exception, compiled):
+ # We need to insert at the top
+ self._excluded.insert(0, [regex, iscompilable, exception, compiled])
+
+ @property
+ def marked_count(self):
+ """Returns the number of marked regexes only."""
+ return len([x for marked, x in self if marked])
+
+ def isExcluded(self, regex):
+ for item in self._excluded:
+ if regex == item[0]:
+ return True
+ return False
+
+ def remove(self, regex):
+ for item in self._excluded:
+ if item[0] == regex:
+ self._excluded.remove(item)
+ self._remove_compiled(regex)
+
+ def rename(self, regex, newregex):
+ if regex == newregex:
+ return
+ found = False
+ was_marked = False
+ is_compilable = False
+ for item in self._excluded:
+ if item[0] == regex:
+ found = True
+ was_marked = self.is_marked(regex)
+ is_compilable, exception, compiled = self.compile_re(newregex)
+ # We overwrite the found entry
+ self._excluded[self._excluded.index(item)] =\
+ [newregex, is_compilable, exception, compiled]
+ self._remove_compiled(regex)
+ break
+ if not found:
+ return
+ if is_compilable and was_marked:
+ # Not marked by default when added, add it back
+ self.mark(newregex)
+
+ # def change_index(self, regex, new_index):
+ # """Internal list must be a list, not dict."""
+ # item = self._excluded.pop(regex)
+ # self._excluded.insert(new_index, item)
+
+ def restore_defaults(self):
+ for _, regex in self:
+ if regex not in default_regexes:
+ self.unmark(regex)
+ for default_regex in default_regexes:
+ if not self.isExcluded(default_regex):
+ self.add(default_regex)
+ self.mark(default_regex)
+
+ def load_from_xml(self, infile):
+ """Loads the ignore list from a XML created with save_to_xml.
+
+ infile can be a file object or a filename.
+ """
+ try:
+ root = ET.parse(infile).getroot()
+ except Exception as e:
+ logging.warning(f"Error while loading {infile}: {e}")
+ self.restore_defaults()
+ return e
+
+ marked = set()
+ exclude_elems = (e for e in root if e.tag == "exclude")
+ for exclude_item in exclude_elems:
+ regex_string = exclude_item.get("regex")
+ if not regex_string:
+ continue
+ try:
+ # "forced" avoids compilation exceptions and adds anyway
+ self.add(regex_string, forced=True)
+ except AlreadyThereException:
+ logging.error(f"Regex \"{regex_string}\" \
+loaded from XML was already present in the list.")
+ continue
+ if exclude_item.get("marked") == "y":
+ marked.add(regex_string)
+
+ for item in marked:
+ self.mark(item)
+
+ def save_to_xml(self, outfile):
+ """Create a XML file that can be used by load_from_xml.
+ outfile can be a file object or a filename."""
+ root = ET.Element("exclude_list")
+ # reversed in order to keep order of entries when reloading from xml later
+ for item in reversed(self._excluded):
+ exclude_node = ET.SubElement(root, "exclude")
+ exclude_node.set("regex", str(item[0]))
+ exclude_node.set("marked", ("y" if self.is_marked(item[0]) else "n"))
+ tree = ET.ElementTree(root)
+ with FileOrPath(outfile, "wb") as fp:
+ tree.write(fp, encoding="utf-8")
+
+
+class ExcludeDict(ExcludeList):
+ """Exclusion list holding a set of regular expressions as keys, the compiled
+ Pattern, compilation error and compilable boolean as values."""
+ # Implemntation around a dictionary instead of a list, which implies
+ # to keep the index of each string-key as its sub-element and keep it updated
+ # whenever insert/remove is done.
+
+ def __init__(self, union_regex=False):
+ Markable.__init__(self)
+ self._use_union = union_regex
+ # { "regex string":
+ # {
+ # "index": int,
+ # "compilable": bool,
+ # "error": str,
+ # "compiled": Pattern or None
+ # }
+ # }
+ self._excluded = {}
+ self._excluded_compiled = set()
+ self._dirty = True
+
+ def __iter__(self):
+ """Iterate in order."""
+ for regex in ordered_keys(self._excluded):
+ yield self.is_marked(regex), regex
+
+ def __getitem__(self, key):
+ """Returns the dict item correponding to key"""
+ return self._excluded.__getitem__(key)
+
+ def get_compiled(self, key):
+ """Returns the compiled item for key"""
+ return self.__getitem__(key).get("compiled")
+
+ def is_markable(self, regex):
+ return self._is_markable(regex)
+
+ def _is_markable(self, regex):
+ """Return the cached result of "compilable" property"""
+ exists = self._excluded.get(regex)
+ if exists:
+ return exists.get("compilable")
+ return False
+
+ def _add_compiled(self, regex):
+ self._dirty = True
+ if self._use_union:
+ return
+ try:
+ self._excluded_compiled.add(self._excluded[regex]["compiled"])
+ except Exception as e:
+ logging.warning(f"Exception while adding regex {regex} to compiled set: {e}")
+ return
+
+ def is_compilable(self, regex):
+ """Returns the cached "compilable" value"""
+ return self._excluded[regex]["compilable"]
+
+ def error(self, regex):
+ """Return the compilation error message for regex string"""
+ return self._excluded.get(regex).get("error")
+
+ # ---Public
+ def _do_add(self, regex, iscompilable, exception, compiled):
+ # We always insert at the top, so index should be 0
+ # and other indices should be pushed by one
+ for value in self._excluded.values():
+ value["index"] += 1
+ self._excluded[regex] = {
+ "index": 0,
+ "compilable": iscompilable,
+ "error": exception,
+ "compiled": compiled
+ }
+
+ def isExcluded(self, regex):
+ if regex in self._excluded.keys():
+ return True
+ return False
+
+ def remove(self, regex):
+ old_value = self._excluded.pop(regex)
+ # Bring down all indices which where above it
+ index = old_value["index"]
+ if index == len(self._excluded) - 1: # we start at 0...
+ # Old index was at the end, no need to update other indices
+ self._remove_compiled(regex)
+ return
+
+ for value in self._excluded.values():
+ if value.get("index") > old_value["index"]:
+ value["index"] -= 1
+ self._remove_compiled(regex)
+
+ def rename(self, regex, newregex):
+ if regex == newregex or regex not in self._excluded.keys():
+ return
+ was_marked = self.is_marked(regex)
+ previous = self._excluded.pop(regex)
+ iscompilable, error, compiled = self.compile_re(newregex)
+ self._excluded[newregex] = {
+ "index": previous["index"],
+ "compilable": iscompilable,
+ "error": error,
+ "compiled": compiled
+ }
+ self._remove_compiled(regex)
+ if was_marked and iscompilable:
+ self.mark(newregex)
+
+ def save_to_xml(self, outfile):
+ """Create a XML file that can be used by load_from_xml.
+
+ outfile can be a file object or a filename.
+ """
+ root = ET.Element("exclude_list")
+ # reversed in order to keep order of entries when reloading from xml later
+ reversed_list = []
+ for key in ordered_keys(self._excluded):
+ reversed_list.append(key)
+ for item in reversed(reversed_list):
+ exclude_node = ET.SubElement(root, "exclude")
+ exclude_node.set("regex", str(item))
+ exclude_node.set("marked", ("y" if self.is_marked(item) else "n"))
+ tree = ET.ElementTree(root)
+ with FileOrPath(outfile, "wb") as fp:
+ tree.write(fp, encoding="utf-8")
+
+
+def ordered_keys(_dict):
+ """Returns an iterator over the keys of dictionary sorted by "index" key"""
+ if not len(_dict):
+ return
+ list_of_items = []
+ for item in _dict.items():
+ list_of_items.append(item)
+ list_of_items.sort(key=lambda x: x[1].get("index"))
+ for item in list_of_items:
+ yield item[0]
+
+
+if ISWINDOWS:
+ def has_sep(x):
+ return '\\' + sep in x
+else:
+ def has_sep(x):
+ return sep in x
diff --git a/core/gui/exclude_list_dialog.py b/core/gui/exclude_list_dialog.py
new file mode 100644
index 00000000..c6409ef7
--- /dev/null
+++ b/core/gui/exclude_list_dialog.py
@@ -0,0 +1,70 @@
+# Created On: 2012/03/13
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+# from hscommon.trans import tr
+from .exclude_list_table import ExcludeListTable
+import logging
+
+
+class ExcludeListDialogCore:
+ def __init__(self, app):
+ self.app = app
+ self.exclude_list = self.app.exclude_list # Markable from exclude.py
+ self.exclude_list_table = ExcludeListTable(self, app) # GUITable, this is the "model"
+
+ def restore_defaults(self):
+ self.exclude_list.restore_defaults()
+ self.refresh()
+
+ def refresh(self):
+ self.exclude_list_table.refresh()
+
+ def remove_selected(self):
+ for row in self.exclude_list_table.selected_rows:
+ self.exclude_list_table.remove(row)
+ self.exclude_list.remove(row.regex)
+ self.refresh()
+
+ def rename_selected(self, newregex):
+ """Renames the selected regex to ``newregex``.
+ If there's more than one selected row, the first one is used.
+ :param str newregex: The regex to rename the row's regex to.
+ """
+ try:
+ r = self.exclude_list_table.selected_rows[0]
+ self.exclude_list.rename(r.regex, newregex)
+ self.refresh()
+ return True
+ except Exception as e:
+ logging.warning(f"Error while renaming regex to {newregex}: {e}")
+ return False
+
+ def add(self, regex):
+ try:
+ self.exclude_list.add(regex)
+ except Exception as e:
+ raise(e)
+ self.exclude_list.mark(regex)
+ self.exclude_list_table.add(regex)
+
+ def test_string(self, test_string):
+ """Sets property on row to highlight if its regex matches test_string supplied."""
+ matched = False
+ for row in self.exclude_list_table.rows:
+ if self.exclude_list.get_compiled(row.regex).match(test_string):
+ matched = True
+ row.highlight = True
+ else:
+ row.highlight = False
+ return matched
+
+ def reset_rows_highlight(self):
+ for row in self.exclude_list_table.rows:
+ row.highlight = False
+
+ def show(self):
+ self.view.show()
diff --git a/core/gui/exclude_list_table.py b/core/gui/exclude_list_table.py
new file mode 100644
index 00000000..8875d330
--- /dev/null
+++ b/core/gui/exclude_list_table.py
@@ -0,0 +1,98 @@
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from .base import DupeGuruGUIObject
+from hscommon.gui.table import GUITable, Row
+from hscommon.gui.column import Column, Columns
+from hscommon.trans import trget
+tr = trget("ui")
+
+
+class ExcludeListTable(GUITable, DupeGuruGUIObject):
+ COLUMNS = [
+ Column("marked", ""),
+ Column("regex", tr("Regular Expressions"))
+ ]
+
+ def __init__(self, exclude_list_dialog, app):
+ GUITable.__init__(self)
+ DupeGuruGUIObject.__init__(self, app)
+ self.columns = Columns(self)
+ self.dialog = exclude_list_dialog
+
+ def rename_selected(self, newname):
+ row = self.selected_row
+ if row is None:
+ return False
+ row._data = None
+ return self.dialog.rename_selected(newname)
+
+ # --- Virtual
+ def _do_add(self, regex):
+ """(Virtual) Creates a new row, adds it in the table.
+ Returns ``(row, insert_index)``."""
+ # Return index 0 to insert at the top
+ return ExcludeListRow(self, self.dialog.exclude_list.is_marked(regex), regex), 0
+
+ def _do_delete(self):
+ self.dalog.exclude_list.remove(self.selected_row.regex)
+
+ # --- Override
+ def add(self, regex):
+ row, insert_index = self._do_add(regex)
+ self.insert(insert_index, row)
+ self.view.refresh()
+
+ def _fill(self):
+ for enabled, regex in self.dialog.exclude_list:
+ self.append(ExcludeListRow(self, enabled, regex))
+
+ def refresh(self, refresh_view=True):
+ """Override to avoid keeping previous selection in case of multiple rows
+ selected previously."""
+ self.cancel_edits()
+ del self[:]
+ self._fill()
+ if refresh_view:
+ self.view.refresh()
+
+
+class ExcludeListRow(Row):
+ def __init__(self, table, enabled, regex):
+ Row.__init__(self, table)
+ self._app = table.app
+ self._data = None
+ self.enabled = str(enabled)
+ self.regex = str(regex)
+ self.highlight = False
+
+ @property
+ def data(self):
+ if self._data is None:
+ self._data = {"marked": self.enabled, "regex": self.regex}
+ return self._data
+
+ @property
+ def markable(self):
+ return self._app.exclude_list.is_markable(self.regex)
+
+ @property
+ def marked(self):
+ return self._app.exclude_list.is_marked(self.regex)
+
+ @marked.setter
+ def marked(self, value):
+ if value:
+ self._app.exclude_list.mark(self.regex)
+ else:
+ self._app.exclude_list.unmark(self.regex)
+
+ @property
+ def error(self):
+ # This assumes error() returns an Exception()
+ message = self._app.exclude_list.error(self.regex)
+ if hasattr(message, "msg"):
+ return self._app.exclude_list.error(self.regex).msg
+ else:
+ return message # Exception object
diff --git a/core/gui/ignore_list_dialog.py b/core/gui/ignore_list_dialog.py
index 1590c837..cd3a0996 100644
--- a/core/gui/ignore_list_dialog.py
+++ b/core/gui/ignore_list_dialog.py
@@ -17,7 +17,7 @@ class IgnoreListDialog:
def __init__(self, app):
self.app = app
self.ignore_list = self.app.ignore_list
- self.ignore_list_table = IgnoreListTable(self)
+ self.ignore_list_table = IgnoreListTable(self) # GUITable
def clear(self):
if not self.ignore_list:
diff --git a/core/tests/directories_test.py b/core/tests/directories_test.py
index 05d814b2..8a5ddcdb 100644
--- a/core/tests/directories_test.py
+++ b/core/tests/directories_test.py
@@ -12,6 +12,7 @@
from pytest import raises
from hscommon.path import Path
from hscommon.testutil import eq_
+from hscommon.plat import ISWINDOWS
from ..fs import File
from ..directories import (
@@ -20,6 +21,7 @@
AlreadyThereError,
InvalidPathError,
)
+from ..exclude import ExcludeList, ExcludeDict
def create_fake_fs(rootpath):
@@ -341,3 +343,200 @@ def _default_state_for_path(self, path):
d.set_state(p1["foobar"], DirectoryState.Normal)
eq_(d.get_state(p1["foobar"]), DirectoryState.Normal)
eq_(len(list(d.get_files())), 2)
+
+
+class TestExcludeList():
+ def setup_method(self, method):
+ self.d = Directories(exclude_list=ExcludeList(union_regex=False))
+
+ def get_files_and_expect_num_result(self, num_result):
+ """Calls get_files(), get the filenames only, print for debugging.
+ num_result is how many files are expected as a result."""
+ print(f"EXCLUDED REGEX: paths {self.d._exclude_list.compiled_paths} \
+files: {self.d._exclude_list.compiled_files} all: {self.d._exclude_list.compiled}")
+ files = list(self.d.get_files())
+ files = [file.name for file in files]
+ print(f"FINAL FILES {files}")
+ eq_(len(files), num_result)
+ return files
+
+ def test_exclude_recycle_bin_by_default(self, tmpdir):
+ regex = r"^.*Recycle\.Bin$"
+ self.d._exclude_list.add(regex)
+ self.d._exclude_list.mark(regex)
+ p1 = Path(str(tmpdir))
+ p1["$Recycle.Bin"].mkdir()
+ p1["$Recycle.Bin"]["subdir"].mkdir()
+ self.d.add_path(p1)
+ eq_(self.d.get_state(p1["$Recycle.Bin"]), DirectoryState.Excluded)
+ # By default, subdirs should be excluded too, but this can be overriden separately
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdir"]), DirectoryState.Excluded)
+ self.d.set_state(p1["$Recycle.Bin"]["subdir"], DirectoryState.Normal)
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdir"]), DirectoryState.Normal)
+
+ def test_exclude_refined(self, tmpdir):
+ regex1 = r"^\$Recycle\.Bin$"
+ self.d._exclude_list.add(regex1)
+ self.d._exclude_list.mark(regex1)
+ p1 = Path(str(tmpdir))
+ p1["$Recycle.Bin"].mkdir()
+ p1["$Recycle.Bin"]["somefile.png"].open("w").close()
+ p1["$Recycle.Bin"]["some_unwanted_file.jpg"].open("w").close()
+ p1["$Recycle.Bin"]["subdir"].mkdir()
+ p1["$Recycle.Bin"]["subdir"]["somesubdirfile.png"].open("w").close()
+ p1["$Recycle.Bin"]["subdir"]["unwanted_subdirfile.gif"].open("w").close()
+ p1["$Recycle.Bin"]["subdar"].mkdir()
+ p1["$Recycle.Bin"]["subdar"]["somesubdarfile.jpeg"].open("w").close()
+ p1["$Recycle.Bin"]["subdar"]["unwanted_subdarfile.png"].open("w").close()
+ self.d.add_path(p1["$Recycle.Bin"])
+
+ # Filter should set the default state to Excluded
+ eq_(self.d.get_state(p1["$Recycle.Bin"]), DirectoryState.Excluded)
+ # The subdir should inherit its parent state
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdir"]), DirectoryState.Excluded)
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdar"]), DirectoryState.Excluded)
+ # Override a child path's state
+ self.d.set_state(p1["$Recycle.Bin"]["subdir"], DirectoryState.Normal)
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdir"]), DirectoryState.Normal)
+ # Parent should keep its default state, and the other child too
+ eq_(self.d.get_state(p1["$Recycle.Bin"]), DirectoryState.Excluded)
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdar"]), DirectoryState.Excluded)
+ # print(f"get_folders(): {[x for x in self.d.get_folders()]}")
+
+ # only the 2 files directly under the Normal directory
+ files = self.get_files_and_expect_num_result(2)
+ assert "somefile.png" not in files
+ assert "some_unwanted_file.jpg" not in files
+ assert "somesubdarfile.jpeg" not in files
+ assert "unwanted_subdarfile.png" not in files
+ assert "somesubdirfile.png" in files
+ assert "unwanted_subdirfile.gif" in files
+ # Overriding the parent should enable all children
+ self.d.set_state(p1["$Recycle.Bin"], DirectoryState.Normal)
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdar"]), DirectoryState.Normal)
+ # all files there
+ files = self.get_files_and_expect_num_result(6)
+ assert "somefile.png" in files
+ assert "some_unwanted_file.jpg" in files
+
+ # This should still filter out files under directory, despite the Normal state
+ regex2 = r".*unwanted.*"
+ self.d._exclude_list.add(regex2)
+ self.d._exclude_list.mark(regex2)
+ files = self.get_files_and_expect_num_result(3)
+ assert "somefile.png" in files
+ assert "some_unwanted_file.jpg" not in files
+ assert "unwanted_subdirfile.gif" not in files
+ assert "unwanted_subdarfile.png" not in files
+
+ if ISWINDOWS:
+ regex3 = r".*Recycle\.Bin\\.*unwanted.*subdirfile.*"
+ else:
+ regex3 = r".*Recycle\.Bin\/.*unwanted.*subdirfile.*"
+ self.d._exclude_list.rename(regex2, regex3)
+ assert self.d._exclude_list.error(regex3) is None
+ # print(f"get_folders(): {[x for x in self.d.get_folders()]}")
+ # Directory shouldn't change its state here, unless explicitely done by user
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdir"]), DirectoryState.Normal)
+ files = self.get_files_and_expect_num_result(5)
+ assert "unwanted_subdirfile.gif" not in files
+ assert "unwanted_subdarfile.png" in files
+
+ # using end of line character should only filter the directory, or file ending with subdir
+ regex4 = r".*subdir$"
+ self.d._exclude_list.rename(regex3, regex4)
+ assert self.d._exclude_list.error(regex4) is None
+ p1["$Recycle.Bin"]["subdar"]["file_ending_with_subdir"].open("w").close()
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdir"]), DirectoryState.Excluded)
+ files = self.get_files_and_expect_num_result(4)
+ assert "file_ending_with_subdir" not in files
+ assert "somesubdarfile.jpeg" in files
+ assert "somesubdirfile.png" not in files
+ assert "unwanted_subdirfile.gif" not in files
+ self.d.set_state(p1["$Recycle.Bin"]["subdir"], DirectoryState.Normal)
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdir"]), DirectoryState.Normal)
+ # print(f"get_folders(): {[x for x in self.d.get_folders()]}")
+ files = self.get_files_and_expect_num_result(6)
+ assert "file_ending_with_subdir" not in files
+ assert "somesubdirfile.png" in files
+ assert "unwanted_subdirfile.gif" in files
+
+ regex5 = r".*subdir.*"
+ self.d._exclude_list.rename(regex4, regex5)
+ # Files containing substring should be filtered
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["subdir"]), DirectoryState.Normal)
+ # The path should not match, only the filename, the "subdir" in the directory name shouldn't matter
+ p1["$Recycle.Bin"]["subdir"]["file_which_shouldnt_match"].open("w").close()
+ files = self.get_files_and_expect_num_result(5)
+ assert "somesubdirfile.png" not in files
+ assert "unwanted_subdirfile.gif" not in files
+ assert "file_ending_with_subdir" not in files
+ assert "file_which_shouldnt_match" in files
+
+ def test_japanese_unicode(self, tmpdir):
+ p1 = Path(str(tmpdir))
+ p1["$Recycle.Bin"].mkdir()
+ p1["$Recycle.Bin"]["somerecycledfile.png"].open("w").close()
+ p1["$Recycle.Bin"]["some_unwanted_file.jpg"].open("w").close()
+ p1["$Recycle.Bin"]["subdir"].mkdir()
+ p1["$Recycle.Bin"]["subdir"]["過去白濁物語~]_カラー.jpg"].open("w").close()
+ p1["$Recycle.Bin"]["思叫物語"].mkdir()
+ p1["$Recycle.Bin"]["思叫物語"]["なししろ会う前"].open("w").close()
+ p1["$Recycle.Bin"]["思叫物語"]["堂~ロ"].open("w").close()
+ self.d.add_path(p1["$Recycle.Bin"])
+ regex3 = r".*物語.*"
+ self.d._exclude_list.add(regex3)
+ self.d._exclude_list.mark(regex3)
+ # print(f"get_folders(): {[x for x in self.d.get_folders()]}")
+ eq_(self.d.get_state(p1["$Recycle.Bin"]["思叫物語"]), DirectoryState.Excluded)
+ files = self.get_files_and_expect_num_result(2)
+ assert "過去白濁物語~]_カラー.jpg" not in files
+ assert "なししろ会う前" not in files
+ assert "堂~ロ" not in files
+ # using end of line character should only filter that directory, not affecting its files
+ regex4 = r".*物語$"
+ self.d._exclude_list.rename(regex3, regex4)
+ assert self.d._exclude_list.error(regex4) is None
+ self.d.set_state(p1["$Recycle.Bin"]["思叫物語"], DirectoryState.Normal)
+ files = self.get_files_and_expect_num_result(5)
+ assert "過去白濁物語~]_カラー.jpg" in files
+ assert "なししろ会う前" in files
+ assert "堂~ロ" in files
+
+ def test_get_state_returns_excluded_for_hidden_directories_and_files(self, tmpdir):
+ # This regex only work for files, not paths
+ regex = r"^\..*$"
+ self.d._exclude_list.add(regex)
+ self.d._exclude_list.mark(regex)
+ p1 = Path(str(tmpdir))
+ p1["foobar"].mkdir()
+ p1["foobar"][".hidden_file.txt"].open("w").close()
+ p1["foobar"][".hidden_dir"].mkdir()
+ p1["foobar"][".hidden_dir"]["foobar.jpg"].open("w").close()
+ p1["foobar"][".hidden_dir"][".hidden_subfile.png"].open("w").close()
+ self.d.add_path(p1["foobar"])
+ # It should not inherit its parent's state originally
+ eq_(self.d.get_state(p1["foobar"][".hidden_dir"]), DirectoryState.Excluded)
+ self.d.set_state(p1["foobar"][".hidden_dir"], DirectoryState.Normal)
+ # The files should still be filtered
+ files = self.get_files_and_expect_num_result(1)
+ eq_(len(self.d._exclude_list.compiled_paths), 0)
+ eq_(len(self.d._exclude_list.compiled_files), 1)
+ assert ".hidden_file.txt" not in files
+ assert ".hidden_subfile.png" not in files
+ assert "foobar.jpg" in files
+
+
+class TestExcludeDict(TestExcludeList):
+ def setup_method(self, method):
+ self.d = Directories(exclude_list=ExcludeDict(union_regex=False))
+
+
+class TestExcludeListunion(TestExcludeList):
+ def setup_method(self, method):
+ self.d = Directories(exclude_list=ExcludeList(union_regex=True))
+
+
+class TestExcludeDictunion(TestExcludeList):
+ def setup_method(self, method):
+ self.d = Directories(exclude_list=ExcludeDict(union_regex=True))
diff --git a/core/tests/exclude_test.py b/core/tests/exclude_test.py
new file mode 100644
index 00000000..de5e46c5
--- /dev/null
+++ b/core/tests/exclude_test.py
@@ -0,0 +1,282 @@
+# Copyright 2016 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+import io
+# import os.path as op
+
+from xml.etree import ElementTree as ET
+
+# from pytest import raises
+from hscommon.testutil import eq_
+from hscommon.plat import ISWINDOWS
+
+from .base import DupeGuru
+from ..exclude import ExcludeList, ExcludeDict, default_regexes, AlreadyThereException
+
+from re import error
+
+
+# Two slightly different implementations here, one around a list of lists,
+# and another around a dictionary.
+
+
+class TestCaseListXMLLoading:
+ def setup_method(self, method):
+ self.exclude_list = ExcludeList()
+
+ def test_load_non_existant_file(self):
+ # Loads the pre-defined regexes
+ self.exclude_list.load_from_xml("non_existant.xml")
+ eq_(len(default_regexes), len(self.exclude_list))
+ # they should also be marked by default
+ eq_(len(default_regexes), self.exclude_list.marked_count)
+
+ def test_save_to_xml(self):
+ f = io.BytesIO()
+ self.exclude_list.save_to_xml(f)
+ f.seek(0)
+ doc = ET.parse(f)
+ root = doc.getroot()
+ eq_("exclude_list", root.tag)
+
+ def test_save_and_load(self, tmpdir):
+ e1 = ExcludeList()
+ e2 = ExcludeList()
+ eq_(len(e1), 0)
+ e1.add(r"one")
+ e1.mark(r"one")
+ e1.add(r"two")
+ tmpxml = str(tmpdir.join("exclude_testunit.xml"))
+ e1.save_to_xml(tmpxml)
+ e2.load_from_xml(tmpxml)
+ # We should have the default regexes
+ assert r"one" in e2
+ assert r"two" in e2
+ eq_(len(e2), 2)
+ eq_(e2.marked_count, 1)
+
+ def test_load_xml_with_garbage_and_missing_elements(self):
+ root = ET.Element("foobar") # The root element shouldn't matter
+ exclude_node = ET.SubElement(root, "bogus")
+ exclude_node.set("regex", "None")
+ exclude_node.set("marked", "y")
+
+ exclude_node = ET.SubElement(root, "exclude")
+ exclude_node.set("regex", "one")
+ # marked field invalid
+ exclude_node.set("markedddd", "y")
+
+ exclude_node = ET.SubElement(root, "exclude")
+ exclude_node.set("regex", "two")
+ # missing marked field
+
+ exclude_node = ET.SubElement(root, "exclude")
+ exclude_node.set("regex", "three")
+ exclude_node.set("markedddd", "pazjbjepo")
+
+ f = io.BytesIO()
+ tree = ET.ElementTree(root)
+ tree.write(f, encoding="utf-8")
+ f.seek(0)
+ self.exclude_list.load_from_xml(f)
+ print(f"{[x for x in self.exclude_list]}")
+ # only the two "exclude" nodes should be added,
+ eq_(3, len(self.exclude_list))
+ # None should be marked
+ eq_(0, self.exclude_list.marked_count)
+
+
+class TestCaseDictXMLLoading(TestCaseListXMLLoading):
+ def setup_method(self, method):
+ self.exclude_list = ExcludeDict()
+
+
+class TestCaseListEmpty:
+ def setup_method(self, method):
+ self.app = DupeGuru()
+ self.app.exclude_list = ExcludeList(union_regex=False)
+ self.exclude_list = self.app.exclude_list
+
+ def test_add_mark_and_remove_regex(self):
+ regex1 = r"one"
+ regex2 = r"two"
+ self.exclude_list.add(regex1)
+ assert(regex1 in self.exclude_list)
+ self.exclude_list.add(regex2)
+ self.exclude_list.mark(regex1)
+ self.exclude_list.mark(regex2)
+ eq_(len(self.exclude_list), 2)
+ eq_(len(self.exclude_list.compiled), 2)
+ compiled_files = [x for x in self.exclude_list.compiled_files]
+ eq_(len(compiled_files), 2)
+ self.exclude_list.remove(regex2)
+ assert(regex2 not in self.exclude_list)
+ eq_(len(self.exclude_list), 1)
+
+ def test_add_duplicate(self):
+ self.exclude_list.add(r"one")
+ eq_(1 , len(self.exclude_list))
+ try:
+ self.exclude_list.add(r"one")
+ except Exception:
+ pass
+ eq_(1 , len(self.exclude_list))
+
+ def test_add_not_compilable(self):
+ # Trying to add a non-valid regex should not work and raise exception
+ regex = r"one))"
+ try:
+ self.exclude_list.add(regex)
+ except Exception as e:
+ # Make sure we raise a re.error so that the interface can process it
+ eq_(type(e), error)
+ added = self.exclude_list.mark(regex)
+ eq_(added, False)
+ eq_(len(self.exclude_list), 0)
+ eq_(len(self.exclude_list.compiled), 0)
+ compiled_files = [x for x in self.exclude_list.compiled_files]
+ eq_(len(compiled_files), 0)
+
+ def test_force_add_not_compilable(self):
+ """Used when loading from XML for example"""
+ regex = r"one))"
+ try:
+ self.exclude_list.add(regex, forced=True)
+ except Exception as e:
+ # Should not get an exception here unless it's a duplicate regex
+ raise e
+ marked = self.exclude_list.mark(regex)
+ eq_(marked, False) # can't be marked since not compilable
+ eq_(len(self.exclude_list), 1)
+ eq_(len(self.exclude_list.compiled), 0)
+ compiled_files = [x for x in self.exclude_list.compiled_files]
+ eq_(len(compiled_files), 0)
+ # adding a duplicate
+ regex = r"one))"
+ try:
+ self.exclude_list.add(regex, forced=True)
+ except Exception as e:
+ # we should have this exception, and it shouldn't be added
+ assert type(e) is AlreadyThereException
+ eq_(len(self.exclude_list), 1)
+ eq_(len(self.exclude_list.compiled), 0)
+
+ def test_rename_regex(self):
+ regex = r"one"
+ self.exclude_list.add(regex)
+ self.exclude_list.mark(regex)
+ regex_renamed = r"one))"
+ # Not compilable, can't be marked
+ self.exclude_list.rename(regex, regex_renamed)
+ assert regex not in self.exclude_list
+ assert regex_renamed in self.exclude_list
+ eq_(self.exclude_list.is_marked(regex_renamed), False)
+ self.exclude_list.mark(regex_renamed)
+ eq_(self.exclude_list.is_marked(regex_renamed), False)
+ regex_renamed_compilable = r"two"
+ self.exclude_list.rename(regex_renamed, regex_renamed_compilable)
+ assert regex_renamed_compilable in self.exclude_list
+ eq_(self.exclude_list.is_marked(regex_renamed), False)
+ self.exclude_list.mark(regex_renamed_compilable)
+ eq_(self.exclude_list.is_marked(regex_renamed_compilable), True)
+ eq_(len(self.exclude_list), 1)
+ # Should still be marked after rename
+ regex_compilable = r"three"
+ self.exclude_list.rename(regex_renamed_compilable, regex_compilable)
+ eq_(self.exclude_list.is_marked(regex_compilable), True)
+
+ def test_restore_default(self):
+ """Only unmark previously added regexes and mark the pre-defined ones"""
+ regex = r"one"
+ self.exclude_list.add(regex)
+ self.exclude_list.mark(regex)
+ self.exclude_list.restore_defaults()
+ eq_(len(default_regexes), self.exclude_list.marked_count)
+ # added regex shouldn't be marked
+ eq_(self.exclude_list.is_marked(regex), False)
+ # added regex shouldn't be in compiled list either
+ compiled = [x for x in self.exclude_list.compiled]
+ assert regex not in compiled
+ # Only default regexes marked and in compiled list
+ for re in default_regexes:
+ assert self.exclude_list.is_marked(re)
+ found = False
+ for compiled_re in compiled:
+ if compiled_re.pattern == re:
+ found = True
+ if not found:
+ raise(Exception(f"Default RE {re} not found in compiled list."))
+ continue
+ eq_(len(default_regexes), len(self.exclude_list.compiled))
+
+
+class TestCaseDictEmpty(TestCaseListEmpty):
+ """Same, but with dictionary implementation"""
+ def setup_method(self, method):
+ self.app = DupeGuru()
+ self.app.exclude_list = ExcludeDict(union_regex=False)
+ self.exclude_list = self.app.exclude_list
+
+
+def split_union(pattern_object):
+ """Returns list of strings for each union pattern"""
+ return [x for x in pattern_object.pattern.split("|")]
+
+
+class TestCaseCompiledList():
+ """Test consistency between union or and separate versions."""
+ def setup_method(self, method):
+ self.e_separate = ExcludeList(union_regex=False)
+ self.e_separate.restore_defaults()
+ self.e_union = ExcludeList(union_regex=True)
+ self.e_union.restore_defaults()
+
+ def test_same_number_of_expressions(self):
+ # We only get one union Pattern item in a tuple, which is made of however many parts
+ eq_(len(split_union(self.e_union.compiled[0])), len(default_regexes))
+ # We get as many as there are marked items
+ eq_(len(self.e_separate.compiled), len(default_regexes))
+ exprs = split_union(self.e_union.compiled[0])
+ # We should have the same number and the same expressions
+ eq_(len(exprs), len(self.e_separate.compiled))
+ for expr in self.e_separate.compiled:
+ assert expr.pattern in exprs
+
+ def test_compiled_files(self):
+ # is path separator checked properly to yield the output
+ if ISWINDOWS:
+ regex1 = r"test\\one\\sub"
+ else:
+ regex1 = r"test/one/sub"
+ self.e_separate.add(regex1)
+ self.e_separate.mark(regex1)
+ self.e_union.add(regex1)
+ self.e_union.mark(regex1)
+ separate_compiled_dirs = self.e_separate.compiled
+ separate_compiled_files = [x for x in self.e_separate.compiled_files]
+ # HACK we need to call compiled property FIRST to generate the cache
+ union_compiled_dirs = self.e_union.compiled
+ # print(f"type: {type(self.e_union.compiled_files[0])}")
+ # A generator returning only one item... ugh
+ union_compiled_files = [x for x in self.e_union.compiled_files][0]
+ print(f"compiled files: {union_compiled_files}")
+ # Separate should give several plus the one added
+ eq_(len(separate_compiled_dirs), len(default_regexes) + 1)
+ # regex1 shouldn't be in the "files" version
+ eq_(len(separate_compiled_files), len(default_regexes))
+ # Only one Pattern returned, which when split should be however many + 1
+ eq_(len(split_union(union_compiled_dirs[0])), len(default_regexes) + 1)
+ # regex1 shouldn't be here either
+ eq_(len(split_union(union_compiled_files)), len(default_regexes))
+
+
+class TestCaseCompiledDict(TestCaseCompiledList):
+ """Test the dictionary version"""
+ def setup_method(self, method):
+ self.e_separate = ExcludeDict(union_regex=False)
+ self.e_separate.restore_defaults()
+ self.e_union = ExcludeDict(union_regex=True)
+ self.e_union.restore_defaults()
diff --git a/images/dialog-error.png b/images/dialog-error.png
new file mode 100644
index 00000000..625c7ff8
Binary files /dev/null and b/images/dialog-error.png differ
diff --git a/qt/app.py b/qt/app.py
index 4ed70b54..574b6a21 100644
--- a/qt/app.py
+++ b/qt/app.py
@@ -27,6 +27,7 @@
from .directories_dialog import DirectoriesDialog
from .problem_dialog import ProblemDialog
from .ignore_list_dialog import IgnoreListDialog
+from .exclude_list_dialog import ExcludeListDialog
from .deletion_options import DeletionOptions
from .se.details_dialog import DetailsDialog as DetailsDialogStandard
from .me.details_dialog import DetailsDialog as DetailsDialogMusic
@@ -86,11 +87,17 @@ def _setup(self):
"IgnoreListDialog",
parent=self.main_window,
model=self.model.ignore_list_dialog)
- self.ignoreListDialog.accepted.connect(self.main_window.onDialogAccepted)
+
+ self.excludeListDialog = self.main_window.createPage(
+ "ExcludeListDialog",
+ app=self,
+ parent=self.main_window,
+ model=self.model.exclude_list_dialog)
else:
self.ignoreListDialog = IgnoreListDialog(
- parent=parent_window, model=self.model.ignore_list_dialog
- )
+ parent=parent_window, model=self.model.ignore_list_dialog)
+ self.excludeDialog = ExcludeListDialog(
+ app=self, parent=parent_window, model=self.model.exclude_list_dialog)
self.deletionOptions = DeletionOptions(
parent=parent_window,
@@ -130,6 +137,7 @@ def _setupActions(self):
tr("Clear Picture Cache"),
self.clearPictureCacheTriggered,
),
+ ("actionExcludeList", "", "", tr("Exclusion Filters"), self.excludeListTriggered),
("actionShowHelp", "F1", "", tr("dupeGuru Help"), self.showHelpTriggered),
("actionAbout", "", "", tr("About dupeGuru"), self.showAboutBoxTriggered),
(
@@ -223,6 +231,10 @@ def show_details(self):
def showResultsWindow(self):
if self.resultWindow is not None:
if self.use_tabs:
+ if self.main_window.indexOfWidget(self.resultWindow) < 0:
+ self.main_window.addTab(
+ self.resultWindow, "Results", switch=True)
+ return
self.main_window.showTab(self.resultWindow)
else:
self.resultWindow.show()
@@ -267,19 +279,25 @@ def clearPictureCacheTriggered(self):
def ignoreListTriggered(self):
if self.use_tabs:
- # Fetch the index in the TabWidget or the StackWidget (depends on class):
- index = self.main_window.indexOfWidget(self.ignoreListDialog)
- if index < 0:
- # we have not instantiated and populated it in their internal list yet
- index = self.main_window.addTab(
- self.ignoreListDialog, "Ignore List", switch=True)
- # if not self.main_window.tabWidget.isTabVisible(index):
- self.main_window.setTabVisible(index, True)
- self.main_window.setCurrentIndex(index)
- return
- else:
+ self.showTriggeredTabbedDialog(self.ignoreListDialog, "Ignore List")
+ else: # floating windows
self.model.ignore_list_dialog.show()
+ def excludeListTriggered(self):
+ if self.use_tabs:
+ self.showTriggeredTabbedDialog(self.excludeListDialog, "Exclusion Filters")
+ else: # floating windows
+ self.model.exclude_list_dialog.show()
+
+ def showTriggeredTabbedDialog(self, dialog, desc_string):
+ """Add tab for dialog, name the tab with desc_string, then show it."""
+ index = self.main_window.indexOfWidget(dialog)
+ # Create the tab if it doesn't exist already
+ if index < 0: # or (not dialog.isVisible() and not self.main_window.isTabVisible(index)):
+ index = self.main_window.addTab(dialog, desc_string, switch=True)
+ # Show the tab for that widget
+ self.main_window.setCurrentIndex(index)
+
def openDebugLogTriggered(self):
debugLogPath = op.join(self.model.appdata, "debug.log")
desktop.open_path(debugLogPath)
@@ -344,15 +362,15 @@ def create_results_window(self):
# or simply delete it on close which is probably cleaner:
self.details_dialog.setAttribute(Qt.WA_DeleteOnClose)
self.details_dialog.close()
- # self.details_dialog.setParent(None) # seems unnecessary
+ # if we don't do the following, Qt will crash when we recreate the Results dialog
+ self.details_dialog.setParent(None)
if self.resultWindow is not None:
self.resultWindow.close()
- self.resultWindow.setParent(None)
+ # This is better for tabs, as it takes care of duplicate items in menu bar
+ self.resultWindow.deleteLater() if self.use_tabs else self.resultWindow.setParent(None)
if self.use_tabs:
self.resultWindow = self.main_window.createPage(
"ResultWindow", parent=self.main_window, app=self)
- self.main_window.addTab(
- self.resultWindow, "Results", switch=False)
else: # We don't use a tab widget, regular floating QMainWindow
self.resultWindow = ResultWindow(self.directories_dialog, self)
self.directories_dialog._updateActionsState()
diff --git a/qt/dg.qrc b/qt/dg.qrc
index 760f2a85..7b2846bf 100644
--- a/qt/dg.qrc
+++ b/qt/dg.qrc
@@ -10,5 +10,6 @@
\
+Directores will also have their default state set to Excluded \
+in the Directories tab if their name happen to match one of the regular expressions.
\
+For each file collected two tests are perfomed on each of them to determine whether or not to filter them out:
\
+
.*My\\sPictures\\\\.*\\.png
C:\\\\User\\My Pictures\\test.png