harvest

#!/usr/bin/env zsh

# Copyright (C) 2014-2022 Dyne.org Foundation

# Harvest is designed, written and maintained by Denis "Jaromil" Roio

# This source code is free software; you can redistribute it and/or
# modify it under the terms of the GNU Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This source code is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  Please refer
# to the GNU Public License for more details.
#
# You should have received a copy of the GNU Public License along with
# this source code; if not, write to: Free Software Foundation, Inc.,
# 675 Mass Ave, Cambridge, MA 02139, USA.

harvest_version=0.6
harvest_release_date="Nov/2022"
harvest_csv_separator=';'

# export HARVEST_PREFIX=$HOME/devel/harvest
R=${HARVEST_PREFIX:-/usr/local/share/harvest}

DEBUG=${DEBUG:-0}

# {{{ ZUPER

##########################
typeset -aU vars
typeset -aU arrs
typeset -aU maps

typeset -aU funs

vars=(DEBUG QUIET LOG)
arrs=(req freq)

vars+=(zuper_version)
zuper_version=0.4

# zsh extensions
# zmodload zsh/regex
# zmodload zsh/system
# zmodload zsh/mapfile

# {{{ Messaging

# Messaging function with pretty coloring
autoload colors
colors

vars+=(last_act last_func last_notice)

function _msg() {
    local msg="$2"
	local i
    command -v gettext 1>/dev/null 2>/dev/null && msg="$(gettext -s "$2")"
    for i in {3..${#}}; do
        msg=${(S)msg//::$(($i - 2))*::/$*[$i]}
    done

    local command="print -P"
    local progname="$fg[magenta]${PROGRAM##*/}$reset_color"
    local message="$fg_bold[normal]$fg_no_bold[normal]$msg$reset_color"
    local -i returncode

    case "$1" in
        inline)
            command+=" -n"; pchars=" > "; pcolor="yellow"
            ;;
        message)
            last_act="$msg"
            pchars=" . "; pcolor="white"; message="$fg_no_bold[$pcolor]$msg$reset_color"
            ;;
        verbose)
            last_func="$msg"
            pchars="[D]"; pcolor="blue"
            ;;
        success)
            last_notice="$msg"
            pchars="(*)"; pcolor="green"; message="$fg_no_bold[$pcolor]$msg$reset_color"
            ;;
        warning)
            pchars="[W]"; pcolor="yellow"; message="$fg_no_bold[$pcolor]$msg$reset_color"
            ;;
        failure)
            pchars="[E]"; pcolor="red"; message="$fg_no_bold[$pcolor]$msg$reset_color"
            returncode=1
            ;;
        print)
            progname=""
            ;;
        *)
            pchars="[F]"; pcolor="red"
            message="Developer oops!  Usage: _msg MESSAGE_TYPE \"MESSAGE_CONTENT\""
            returncode=127
            zerr
            ;;
    esac
    ${=command} "${progname} $fg_bold[$pcolor]$pchars$reset_color ${message}$color[reset_color]" >&2

    # write the log if its configured
    [[ "$LOG" = "" ]] || {
        touch $LOG || return $?
        ${=command} "${progname} $fg_bold[$pcolor]$pchars$reset_color ${message}$color[reset_color]" >> $LOG
    }

    return $returncode
}

function _message say act() {
    local notice="message"
    [[ "$1" = "-n" ]] && shift && notice="inline"
    [[ $QUIET = 1 ]] || _msg "$notice" $@
    return 0
}

function _verbose xxx func() {
    [[ $DEBUG = 1 ]] && _msg verbose $@
    return 0
}

function _success yes notice() {
    [[ $QUIET = 1 ]] || _msg success $@
    return 0
}

function _warning no warn warning() {
    [[ $QUIET = 1 ]] || _msg warning $@
    return 0
}

function _failure fatal die error() {
    #    typeset -i exitcode=${exitv:-1}
    [[ $QUIET = 1 ]] || _msg failure $@
    return 1
}

function _print() {
    [[ $QUIET = 1 ]] || _msg print $@
    return 0
}

# }}} Messaging

# {{{ Debugging

fn() {
    fun="$@"
    req=()
    freq=()
    func "$fun"
}

zerr() {
    error "error in: ${fun:-$last_notice}"
    [[ "$last_func"   = "" ]] || warn "called in: $last_func"
    [[ "$last_act"    = "" ]] || warn "called in: $last_act"
    [[ "$last_notice" = "" ]] || warn "called in: $last_notice"
    # [[ "$fun"         = "" ]] || warn "called in: $fun"
    TRAPEXIT() {
        error "error reported, operation aborted."
    }
    return 1
}


function ckreq reqck() {
    err=0
    for v in $req; do
        [[ "${(P)v}" = "" ]] && {
            warn "${fun[(ws: :)1]}(): required setting is blank: $v"
            err=1
        }
    done

    [[ $err = 1 ]] && return $err

    for f in $freq; do
        # exists and has size greater than zero
        [[ -s $f ]] || {
            warn "required file empty: $f"
            err=1
        }
    done
    [[ $err == 1 ]] && zerr
    return $err
}

# dump all variables, arrays and maps declared as global in zuper
# do not print out what is empty
zdump() {
    fn zdump
    [[ ${#vars} -gt 0 ]] && {
        print "Global variables:"
        for _v in $vars; do
			_c=${(P)_v}
			[[ "$_c" = "" ]] ||
				print " $_v = \t $_c"
        done
    }
    [[ ${#arrs} -gt 0 ]] && {
        print "Global arrays:"
        for _a in $arrs; do
			_c=${(P)_a}
			[[ "$_c" = "" ]] ||
				print " $_a \t ( ${(P)_a} )"
        done
    }
    [[ ${#maps} -gt 0 ]] && {
        print "Global maps:"
        for _m in $maps; do
			[[ "${(Pv)_m}" = "" ]] || {
				print " $_m [key] \t ( ${(Pk)_m} )"
				print " $_m [val] \t ( ${(Pv)_m} )"
			}
        done
    }
}

# handy wrappers for throw/catch execution of blocks where we need the
# program to exit on any error (non-zero) returned by any function
throw() { function TRAPZERR() { zerr; return 1 } }
catch() { function TRAPZERR() { } }

##########################
# Endgame handling

arrs+=(destruens)

# Trap functions for the endgame event
# TRAPINT()  { endgame INT;   return $? }
# TRAPEXIT() { endgame EXIT;  return $? }
TRAPHUP()  { endgame HUP;   return $? }
TRAPQUIT() { endgame QUIT;  return $? }
TRAPABRT() { endgame ABORT; return $? }
TRAPKILL() { endgame KILL;  return $? }
# TRAPPIPE() { endgame PIPE;  return $? }
TRAPTERM() { endgame TERM;  return $? }
TRAPSTOP() { endgame STOP;  return $? }
# TRAPZERR() { func "function returns non-zero." }


funs+=(__test_fn)

__test_fn(){
    echo "foo"
}

function zuper_end endgame() {
    fn "endgame $*"

    # execute all no matter what
    TRAPZERR() { }

    # process registered destructors
    for d in $destruens; do
        fn "destructor: $d"
        $d
    done

    # unset all the variables included  in "vars"
    for v in $vars; do
        unset $v
    done

    # unset all the assoc-arrays included  in "arrs"
    for a in $arrs; do
        unset $a
    done

    # unset all the maps included  in "maps"
    for m in $maps; do
        unset $m
    done

    ## We should also undefine the core zuper functions to make it
    ## really idempotent. I have added an array "funs" which contains
    ## the names of the functions to be undefined by endgame/zuper_end
    ## FIXME!!!! The only "registered" function so far is __test_fn,
    ## but if we like this we should register all the core zuper
    ## functions as soon as they are declared
    for f in $funs; do
        unfunction $f
    done
    unset maps
    unset arrs
    unset vars
    unset funs

    return 0
}

## This function should reinitialise zuper and all the variables
# zuper_restart(){
#     endgame
#     source zuper
# }


# Use this to make sure endgame() is called at exit.
# unlike TRAPEXIT, the zshexit() hook is not called when functions exit.
function zuper.exit zshexit() { endgame EXIT; return $? }

# }}} Debugging

vars+=(files numfiles dirs)
vars+=(ext video_factor image_factor audio_factor code_factor text_factor other_factor web_factor slide_factor sheet_factor archiv_factor)

arrs+=(alldirs)

maps+=(video audio code text image other web slide sheet archiv)
maps+=(totals)

vars+=(file_info_date file_info_size dir_info_date dir_info_size)

case `uname` in
  Darwin)
    STAT_OPTS="-f %Sm -t %F"
    ;;
  Linux)
    STAT_OPTS='-c %y'
    ;;
  FreeBSD)
    STAT_OPTS="-f %Sm -t %F"
    ;;
esac


# initialize globals only after sourcing everything
# since zlibs may contain more variable declarations
for _v in $vars; do
    typeset -h $_v
done
for _a in $arrs; do
    typeset -aU $_a
done
for _m in $maps; do
    typeset -A $_m
done

# reset defaults
DEBUG=${DEBUG:-0}
QUIET=${QUIET:-0}
LOG=${LOG:-""}
SEP=${harvest_csv_separator}
req=()
freq=()
last_act=()
last_func=()
last_notice=()
tmpfiles=()
config_section=()
config_section_type=${config_section_type:-org-mode}

func "Zuper $zuper_version initialized"
func "${#vars} global variables registered"
func "${#arrs} global arrays registered"
func "${#maps} global maps registered"

################
## END OF ZUPER
## BEGIN HARVEST
################

LOCAL_ARCH=`uname -s`
# avoid aliases to different ls programs
LS=`which ls`

function get-file-info() {
	local ls
	case ${LOCAL_ARCH:l} in
		linux)
			ls=`$LS -l --time-style '+%Y-%m-%d' "$1" | tr -s ' '`
			;;
		freebsd|darwin)
			ls=`$LS -l -D '%Y-%m-%d' "$1" | tr -s ' '`
			;;
	esac
	print - "$ls" | cut -d ' ' -f 6 | read file_info_date
	print - "$ls" | cut -d ' ' -f 5 | read file_info_size
}

function get-dir-info() {
	local ls
	case ${LOCAL_ARCH:l} in
		linux)
			ls=`$LS -d -l --time-style '+%Y-%m-%d' "$1" | tr -s ' '`
			;;
		freebsd|darwin)
			ls=`$LS -d -l -D '%Y-%m-%d' "$1" | tr -s ' '`
			;;
	esac
	print - "$ls" | cut -d ' ' -f 6 | read dir_info_date
#	print - "$ls" | cut -d ' ' -f 5 | read file_info_size
}

function file-extension-parser() {
   ext=${1##*.}
   case ${ext:l} in
	   7z|a|apk|ar|bz2|cab|cpio|deb|dmg|egg|gz|iso|jar|lha|mar|pak|pea|rar|rpm|s7z|shar|tar|tbz2|tgz|tlz|war|whl|xpi|xz|zip|zipx)
		   print - "archive"
		   ;;
	   aac|aiff|ape|au|flac|gsm|it|m3u|m4a|mid|mod|mp3|mpa|ogg|pls|ra|s3m|sid|wav|wma|xm)
		   print - "audio"
		   ;;
	   azw|azw1|azw3|azw4|azw6|cbr|cbz|epub|mobi)
		   print - "book"
		   ;;
	   1.ada|2.ada|ada|adb|ads|asm|asp|aspx|bas|c|c++|cbl|cc|class|clj|cob|cpp|cs|cxx|d|diff|e|el|f|f77|f90|for|fth|ftn|go|groovy|h|hh|hpp|hs|htm|html|hxx|inc|java|js|jsp|jsx|kt|lhs|lisp|lua|m|m4|nim|patch|php|php3|php4|php5|phtml|pl|po|pp|py|r|rb|rs|s|scala|swg|swift|v|vb|vcxproj|xcodeproj|xml)
																												   print - "code"
																												   ;;
																												   bash|bat|bin|command|crx|csh|exe|fish|ksh|msi|sh|zsh)
																													   print - "exec"
																													   ;;
																												   eot|otf|ttf|woff|woff2)
																													   print - "font"
																													   ;;
																												   3dm|3ds|ai|bmp|dds|dwg|dxf|eps|gif|gpx|jpeg|jpg|kml|kmz|max|png|ps|psd|svg|tga|thm|tif|tiff|webp|xcf)
																													   print - "image"
																													   ;;
																												   csv|ics|ods|vcf|xls|xlsx)
																													   print - "sheet"
																													   ;;
																												   odp|ppt)
																													   print - "slide"
																													   ;;
																												   doc|docx|ebook|log|md|msg|odt|org|pages|pdf|rst|rtf|tex|txt|wpd|wps)
																													   print - "text"
																													   ;;
																												   3g2|3gp|aaf|asf|avchd|avi|drc|flv|m2v|m4p|m4v|mkv|mng|mov|mp2|mp4|mpe|mpeg|mpg|mpv|mxf|nsv|ogm|ogv|qt|rm|rmvb|roq|srt|svi|vob|webm|wmv|yuv)
																													   print - "video"
																													   ;;
																												   css|less|scss|wasm)
																													   print - "web"
																													   ;;
																												   *)
																													   print - "other"
																													   ;;
	   esac
}

# fuzzy thresholds
#
# this is the most important section to tune the selection: the higher
# the values the more file of that type need to be present in a
# directory to classify it with their own type. In other words a lower
# number makes the type "dominant".
video_factor=1  #  minimum video files to increase the video factor
audio_factor=3  #  minimum audio files to increase the audio factor
text_factor=4   #  minimum text  files to increase the text factor
image_factor=10 #  minimum image files to increase the image factor
other_factor=20 #  minimum other files to increase the other factor
code_factor=5   #  minimum code  files to increase the code factor
web_factor=10
slide_factor=2
sheet_factor=3
archiv_factor=10

notice "Harvest $harvest_version - manage large collections of files and directories"

# counts all totals using a basic fuzzy logic algo and prints results
fuzzycount() {
    fn fuzzycount $*
    arg=$1
    req=(arg)
    ckreq || return 1

    # this redefines the priority order
    choice=(video audio image text web slide sheet archiv code other)

    # succesful match means an entry is greater than all others
    match=$(( ${#choice} - 1 ))

    # reset totals
    totals[video]=0
    totals[audio]=0
    totals[other]=0
    totals[image]=0
    totals[code]=0
    totals[text]=0
	totals[web]=0
	totals[slide]=0
	totals[sheet]=0
	totals[archiv]=0

    [[ "$video[$arg]" = "" ]] && video[$arg]=0
    [[ "$audio[$arg]" = "" ]] && audio[$arg]=0
    [[ "$other[$arg]" = "" ]] && other[$arg]=0
    [[ "$image[$arg]" = "" ]] && image[$arg]=0
    [[ "$code[$arg]"  = "" ]] &&  code[$arg]=0
    [[ "$text[$arg]"  = "" ]] &&  text[$arg]=0
    [[ "$web[$arg]"  = "" ]] &&    web[$arg]=0
    [[ "$slide[$arg]"  = "" ]] && slide[$arg]=0
    [[ "$sheet[$arg]"  = "" ]] && sheet[$arg]=0
    [[ "$archiv[$arg]"  = "" ]] && archiv[$arg]=0

    # compute a very, very simple linear fuzzy logic for each
    (( ${#video} )) && totals[video]=$(( $video[$arg] / $video_factor ))
    (( ${#audio} )) && totals[audio]=$(( $audio[$arg] / $audio_factor ))
    (( ${#other} )) && totals[other]=$(( $other[$arg] / $other_factor ))
    (( ${#image} )) && totals[image]=$(( $image[$arg] / $image_factor ))
    (( ${#code}  )) &&  totals[code]=$((  $code[$arg] / $code_factor  ))
    (( ${#text}  )) &&  totals[text]=$((  $text[$arg] / $text_factor  ))
    (( ${#web}  )) &&  totals[web]=$((  $web[$arg] / $web_factor  ))
    (( ${#slide}  )) &&  totals[slide]=$((  $slide[$arg] / $slide_factor  ))
    (( ${#sheet}  )) &&  totals[sheet]=$((  $sheet[$arg] / $sheet_factor  ))
    (( ${#archiv}  )) &&  totals[archiv]=$((  $archiv[$arg] / $archiv_factor  ))

    for t in $choice; do
        count=0
        for o in ${(k)totals}; do
            # don't compare with self
            [[ "$t" = "$o" ]] && continue

            (( ${totals[$t]} > ${totals[$o]} )) && (( count++ ))
        done
        (( $count == $match )) && {
			# get the year and print out directories here
			get-dir-info "$arg"
			print - "dir${SEP}$t${SEP}$dir_info_date${SEP}$arg"
			return 0 }
    done
	get-file-info "$arg"
	print - "dir${SEP}other${SEP}$dir_info_date${SEP}$arg"
    return 0
}

analyse_files() {
	act "analysing files..."
	local base
	realpath "$1" | read base
	local prev=""
	local numfiles=0
	local tmp=`mktemp`
	find "$base" -maxdepth 1 -type f >$tmp
	[[ $? != 0 ]] && { zuper.exit; rm -f $tmp; exit 1 }
	for i in ${(f)"$(cat $tmp | tail -n +2)"}; do
		file_path="$i"
		[[ -r "$file_path" ]] || continue
		file-extension-parser "$file_path" | read file_type
		get-file-info "$file_path"
		print -n - "file${SEP}$file_type${SEP}$file_info_date${SEP}$file_path${SEP}$file_info_size\n"
		numfiles=$(( $numfiles + 1 ))
	done
	rm -f $tmp
}

analyse_dirs() {
	prev=""
	realpath "$2" | read base
	for i in ${(f)1}; do
		file_path="$i"
		file-extension-parser "$file_path" | read file_type
		# func "$file_type \t $base"
		[[ "$base" = "" ]] && continue;
		case "$file_type" in
			video) (( ++video[$base] )) ;;
			audio) (( ++audio[$base] )) ;;
			image) (( ++image[$base] )) ;;
			code)  (( ++code[$base] ))  ;;
			text)  (( ++text[$base] ))  ;;
			slide)  (( ++slide[$base] ))  ;;
			web)  (( ++web[$base] ))  ;;
			sheet)  (( ++sheet[$base] ))  ;;
			archiv)  (( ++archiv[$base] ))  ;;
			*)     (( ++other[$base] ))
				   # func "${(r:5:)other[$base]} other :: $base"
				   ;;
		esac
		[[ "$prev" = "$base" ]] || {
			prev="$base" }
		alldirs+=($base)
	done
}

getattr() {
    a=$1; v=$2
    shift 3
    getfattr -n $a $* |
		awk "BEGIN  {
      RS=\"# file: \"
      FS=\"\n\"
    }/=\"$v\"/ { print \$1 }"
}

# return here if sourcing as lib
[[ "$1" == "source" ]] && {
	zuper.exit
	return 0
}

# main argument: harvest will parse all filenames found in incoming
command="${1:-scan}"

bmarks=$HOME/.config/gtk-3.0/bookmarks

case "${command:l}" in
	scan)
		target="${2:-$PWD}/"
		depth="${3:-2}"
		act   "scanning folder: $target (depth $depth)"
		analyse_files "$target"
		act "scanning sub-folders..."
		local tmp=`mktemp`
		find $target -maxdepth 1 -type d >$tmp
		[[ $? != 0 ]] && { zuper.exit; rm $tmp; exit 1 }
		local tmploop=`mktemp`
		for d in ${(f)"$(cat $tmp | tail -n +2)"}; do
			# TODO: make depth configurable
			find $d -maxdepth $depth >$tmploop
			[[ $? != 0 ]] && { zuper.exit; rm -f $tmp $tmploop; exit 1 }
			analyse_dirs "`cat $tmploop`" $d
		done
		for o in $alldirs; do
			fuzzycount "$o"
		done
		rm -f $tmp $tmploop
		notice "${numfiles} files and ${#alldirs} folders scanned in `basename $target`"
		;;

	# using attr (getfattr / setfattr)
	attr)
		if command -v setfattr >/dev/null; then
			act "Set harvest.type extended attribute on harvested files from stdin"
			for i in ${(f)"$(cat)"}; do
				fname=${i[(ws:;:)4]}
				ftype=${i[(ws:;:)2]}
				[[ -r "$fname" ]] && {
					setfattr -n "harvest.type" -v "$ftype" "$fname"
				}
			done
		else
			error "setfattr not found"
		fi
		;;

	tmsu)
		command -v tmsu >/dev/null || {
			error "TMSU not installed"
			zuper.exit; return 1
		}
		root="${1:-$PWD}"
		act "tagging with tmsu: $root"
		func "tmsu init -v -D $root"
		tmsu init -v -D $root
		tmsu config \
			 autoCreateTags=yes \
			 autoCreateValues=no \
			 directoryFingerprintAlgorithm=none \
			 fileFingerprintAlgorithm=none \
			 reportDuplicates=yes
		for i in ${(f)"$(cat)"}; do
			fname=${i[(ws:;:)4]}
			[[ -r "$fname" ]] || continue
			ftype=${i[(ws:;:)2]}
			fyear=${i[(ws:;:)3]}
			# act "tmsu tag $fname $ftype"
			tmsu tag "$fname" "$ftype"
			numfiles=$(( $numfiles + 1 ))
		done
		notice "Files tagged in TMSU: $numfiles"
		;;

	mount)
		command -v tmsu >/dev/null || {
			error "TMSU not installed"
			zuper.exit; return 1
		}
		root="${2:-$PWD}"
		act "Mounting TMSU in: $root"
		[[ -r $root/.tmsu/db ]] || {
			error "tmsu tags not initialised, use 'harvest tmsu'"
			zuper.exit; return 1 }
		[[ -d $root/.mnt/tags ]] || {
			act "Mounting tagged filesystem in $root/.mnt"
			mkdir -p $root/.mnt
			tmsu mount -D $root/.tmsu/db $root/.mnt || {
				error "Error mounting tagged filesystem using tmsu"
				zuper.exit; return 1 }
		}
		# setup gtk bookmarks too (for sidebar in pcmanfm, thunar etc.)
		[[ -r $bmarks~harvest ]] || {
			mv $bmarks $bmarks~harvest
			for i in ${(f)"$($LS $root/.mnt/tags)"}; do
				print "file://$root/.mnt/tags/$i" >> $bmarks
			done
		}
		act "$root tagged filesytem ready: $root"
		print - "$root/.mnt"
		;;

	umount|unmount)
		command -v tmsu >/dev/null || {
			error "tmsu command not found"
			zuper.exit; return 1 }
		for i in ${(f)"$(mount | awk '
/fuse.pathfs.pathInode/ { print $3 }')"}; do
			act "umount $i"
			tmsu umount "$i"
			if [[ $? = 0 ]]; then
				act "$i tagged filesystem unmounted"
				[[ -r $bmarks~harvest ]] &&
					mv $bmarks~harvest $bmarks
				unset harvest
			else
				error "$i error unmounting tagged filesystem"
				break
			fi
		done
		;;
esac

zuper.exit; return 0