dynamic_flow_algorithm.py

# -*- coding: utf-8 -*-

"""
/***************************************************************************
 DynamicFlow
                                 A QGIS plugin
 Dynamic is a qgis plugin to estimate the spatio-temporal 3D gradient flow from the point observation of the attributes values such as aggregated cell-phone mobility data. 
 Generated by Plugin Builder: http://g-sherman.github.io/Qgis-Plugin-Builder/
                              -------------------
        begin                : 2024-03-28
        copyright            : (C) 2024 by Bo-Cheng Lin, Chen-Yu Liu, Ta-Chien Chan
        email                : tachien@geohealth.tw

 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""

__author__ = 'Bo-Cheng Lin, Chen-Yu Liu, Ta-Chien Chan'
__date__ = '2024-03-28'
__copyright__ = '(C) 2024 by Bo-Cheng Lin, Chen-Yu Liu, Ta-Chien Chan'

# This will get replaced with a git SHA1 when you do a git archive

__revision__ = '$Format:%H$'

import os
from qgis.core import (QgsProcessing,
                       QgsFeatureSink,
                       QgsProcessingAlgorithm,
                       QgsProcessingParameterFeatureSource,
                       QgsProcessingParameterFeatureSink)
from qgis.PyQt.QtCore import QCoreApplication, QVariant
from qgis.PyQt.QtGui import QIcon
from qgis.core import (
    QgsProcessing,
    QgsFeatureSink,
    QgsProcessingException,
    QgsProcessingAlgorithm,
    QgsProcessingParameterFeatureSink,
    QgsProcessingParameterField,
    QgsProcessingParameterBoolean,
    QgsProcessingParameterNumber,
    QgsFields,
    QgsField,
    QgsFeature,
    QgsProcessingParameterFile,
    QgsWkbTypes,
    QgsPointXY, 
    QgsGeometry
)
import numpy as np
import datetime
import geopandas as gpd
from qgis import processing        


class DynamicFlowAlgorithm(QgsProcessingAlgorithm):
    VARIABLES = {
        'ID': 'ID',
        'INPUT': 'INPUT',
        'ACCMASKSIZE': 'ACCMASKSIZE',
        'OUTPUT': 'OUTPUT',
        'OUTPUTVCOMP': 'OUTPUTVCOMP',
        'OUTPUTINITVEC': 'OUTPUTINITVEC',
        'OUTPUTVCOMP_CHECKBOX': 'OUTPUTVCOMP_CHECKBOX',
        'OUTPUTINITVEC_CHECKBOX': 'OUTPUTINITVEC_CHECKBOX',
        'VARS': 'VARS'
    }

    def initAlgorithm(self, config):

        self.addParameter(
            QgsProcessingParameterFile(
                name = self.VARIABLES['INPUT'],
                description = 'Open shp file containing geographical coordinates',
                defaultValue=None,
                optional = False,
                fileFilter='SHP Files (*.shp)', 
            )
          )
        
        self.addParameter(
            QgsProcessingParameterField(
                self.VARIABLES['ID'],
                self.tr('ID Var'),
                allowMultiple = False,
                parentLayerParameterName = self.VARIABLES['INPUT']
            )
        )
        
        self.addParameter(
            QgsProcessingParameterField(
                self.VARIABLES['VARS'],
                self.tr('Variables'),
                allowMultiple = True,
                parentLayerParameterName = self.VARIABLES['INPUT']
            )
        )
        
        self.addParameter(
            QgsProcessingParameterNumber(
                self.VARIABLES['ACCMASKSIZE'],
                self.tr("Accumulated Mask Size (>= 3 Odd Integer)"),
                defaultValue = 3,
                minValue = 3
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                self.VARIABLES['OUTPUTINITVEC_CHECKBOX'],
                self.tr('Save Initial Vector File(.csv)')
            )
        )
        
        self.addParameter(
            QgsProcessingParameterBoolean(
                self.VARIABLES['OUTPUTVCOMP_CHECKBOX'],
                self.tr('Save Vector Components File(.csv)')
            )
        )
        
        self.addParameter(
            QgsProcessingParameterFeatureSink(
                self.VARIABLES['OUTPUT'],
                self.tr('Output layer')
            )
        )

    def processAlgorithm(self, parameters, context, feedback):
        self.results = {}
        self.source = self.parameterAsSource(parameters, self.VARIABLES['INPUT'], context)
        self.check_valid_input(parameters)
        flow_estimate_process = DynamicFlow(parameters, context, feedback, self.VARIABLES)
        self.gdf_flow, self.gdf_vcomp, self.gdf_accflow = flow_estimate_process.main_process()
        feedback.setProgress(90)
        # self.save_process_result(parameters, context)
        headers=[col for col in self.gdf_accflow.columns]
        headers.remove("centroid")
        fieldlist=QgsFields()
        fieldlist.append(QgsField(headers[0],QVariant.Int))
        for name in headers[1:]:
            fieldlist.append(QgsField(name,QVariant.Double))
        
        (sink, dest_id) = self.parameterAsSink(
            parameters,
            self.VARIABLES['OUTPUT'],
            context,
            fieldlist,
            QgsWkbTypes.Point, 
            self.source.sourceCrs()
        )
        if sink is None:
            raise QgsProcessingException(self.invalidSinkError(parameters, self.VARIABLES['OUTPUT']))
        if not os.access(dest_id, os.W_OK) and parameters[self.VARIABLES['OUTPUTINITVEC_CHECKBOX']] and parameters[self.VARIABLES['OUTPUTVCOMP_CHECKBOX']]: 
            raise QgsProcessingException('The specified output folder is not writable. Please choose a different destination.')
        
        of_path = dest_id
        of_type = of_path.split(".")[-1] # gpkg, shp, ...
        for i in self.gdf_accflow.index.to_list():
            featur=QgsFeature()
            newrow=self.gdf_accflow[headers].iloc[i].tolist()
            featur.setAttributes(newrow)
            point = QgsPointXY(self.gdf_accflow.iloc[i].centroid.x, self.gdf_accflow.iloc[i].centroid.y)
            featur.setGeometry(QgsGeometry.fromPointXY(point))
            sink.addFeature(featur, QgsFeatureSink.FastInsert)
            self.results[self.VARIABLES['OUTPUT']] = dest_id

        self.dest_id=dest_id
        feedback.setProgress(95)
        # if output initial vector (gdf_flow) is True
        if parameters[self.VARIABLES['OUTPUTINITVEC_CHECKBOX']]:
            of_initvec_path = of_path.replace(f".{of_type}", "_initvec.csv")
            self.gdf_flow.to_csv(of_initvec_path)
            
        # if output vector component (gdf_vcomp) is True
        if parameters[self.VARIABLES['OUTPUTVCOMP_CHECKBOX']]:
            of_vcomp_path = of_path.replace(f".{of_type}", "_vcomp.csv")
            self.gdf_vcomp.to_csv(of_vcomp_path)
        feedback.setProgress(100)
        return self.results

    def name(self):
        return 'Dynamic Flow'

    def displayName(self):
        return self.tr(self.name())

    def group(self):
        return self.tr(self.groupId())

    def groupId(self):
        return ''

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)
    
    def icon(self):
        return QIcon(os.path.join(os.path.dirname(__file__), '.', 'icons', 'icon.svg'))

    def createInstance(self):
        return DynamicFlowAlgorithm()
    
    def check_valid_input(self, parameters):
        if parameters[self.VARIABLES['ACCMASKSIZE']] % 2 == 0: 
            raise QgsProcessingException('Accumulated Mask Size must be an Odd Integer')
        if self.source is None:
            raise QgsProcessingException(self.invalidSourceError(parameters, self.VARIABLES['INPUT']))

class DynamicFlow:
    def __init__(self, parameters, context, feedback, VARIABLES):
        self.parameters = parameters
        self.context = context
        self.feedback = feedback
        self.VARIABLES = VARIABLES
        self.gdf, self.timestamps, self.uid, _, self.ofname, self.ofname2 = self._get_layer_variable()
        self.gdf_flow, self.gdf_vcomp, self.gdf_accflow = self._create_empty_gdfs()
        self.Xmin = self.gdf.total_bounds[0] #xmin
        self.Ymin = self.gdf.total_bounds[1] #ymin
        self.Xmax = self.gdf.total_bounds[2] #xmax
        self.Ymax = self.gdf.total_bounds[3] #ymax
        
        # grid size
        self.grid_size_x = int(self.gdf['geometry'][0].bounds[2] - self.gdf['geometry'][0].bounds[0]) + 1
        #grid_size_y = int(gdf['geometry'][0].bounds[3] - gdf['geometry'][0].bounds[1]) + 1

        self.Xi_range = int((self.Xmax - self.Xmin)/self.grid_size_x) + 1
        self.Yi_range = int((self.Ymax - self.Ymin)/self.grid_size_x) + 1
    
    def _get_layer_variable(self):
    
        gdf = gpd.read_file(self.parameters[self.VARIABLES['INPUT']])
        assert gdf is not None
        timestamps = len(self.parameters[self.VARIABLES['VARS']])
        uid = self.parameters[self.VARIABLES['ID']]

        ifname = os.path.splitext(self.parameters[self.VARIABLES['INPUT']])[0]
        #建立參考時間的輸出檔名
        currenttime = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
        ofname = ifname + '_flow_' + currenttime + '.shp'
        ofname2 = ifname + '_accflow_' + currenttime + '.shp'
        
        return gdf, timestamps, uid, ifname, ofname, ofname2
    
    def _create_empty_gdfs(self):
        gdf_flow = []
        self.gdf['centroid'] = self.gdf.centroid
        gdf_flow = self.gdf[[self.uid,'centroid']]
        gdf_flow = gpd.GeoDataFrame(gdf_flow, geometry="centroid")

        gdf_vcomp = []
        gdf_vcomp = self.gdf[[self.uid,'centroid']]
        gdf_vcomp = gpd.GeoDataFrame(gdf_vcomp, geometry="centroid")
        
        gdf_accflow = []
        gdf_accflow = self.gdf[[self.uid,'centroid']]
        gdf_accflow = gpd.GeoDataFrame(gdf_accflow, geometry="centroid")
        timestamp_field_acc = 'azi'
        gradient_field_acc = 'gradient'
        gdf_accflow.loc[:, timestamp_field_acc] = 0.0
        gdf_accflow.loc[:, gradient_field_acc] = 0.0
        
        return gdf_flow, gdf_vcomp, gdf_accflow
        
    def _flow_estimate_process(self):
        # create empty matrix & gridid
        Matrix = [[[0 for y in range(self.Yi_range)] for x in range(self.Xi_range)] for t in range(self.timestamps)]
        Gridid = [[0 for y in range(self.Yi_range)] for x in range(self.Xi_range)]
        # create empty fx, fy, ft, fg
        fx = [[[0 for y in range(self.Yi_range)] for x in range(self.Xi_range)] for t in range(self.timestamps)]
        fy = [[[0 for y in range(self.Yi_range)] for x in range(self.Xi_range)] for t in range(self.timestamps)]
        ft = [[[0 for y in range(self.Yi_range)] for x in range(self.Xi_range)] for t in range(self.timestamps)]
        fg = [[[0 for y in range(self.Yi_range)] for x in range(self.Xi_range)] for t in range(self.timestamps)] 
        self.feedback.setProgress(55)
        #3D array
        for index, grid in self.gdf.iterrows():
            x = round(grid.geometry.centroid.x,3)
            y = round(grid.geometry.centroid.y,3)
            xi = int((x - self.Xmin)/self.grid_size_x)
            yi = int((y - self.Ymin)/self.grid_size_x)

            Gridid[xi][yi] = grid[self.parameters[self.VARIABLES['ID']]] #grid.YKR_ID

            for j in range(0,self.timestamps):
                var = self.parameters[self.VARIABLES['VARS']][j]
                Matrix[j][xi][yi] = grid[var]
        self.feedback.setProgress(60)         
        for xi in range(0,self.Xi_range):
            for yi in range(0,self.Yi_range):
                if xi>0 and (xi+1)<self.Xi_range and yi>0 and (yi+1)<self.Yi_range:
                    for j in range(1,self.timestamps-1): # 扣除頭尾
                        vx = 0
                        vy = 0
                        vt = 0
                        vw = 0
                        #sobel
                        vx = 1*(Matrix[j+1][xi+1][yi+1]-Matrix[j+1][xi-1][yi+1])+2*(Matrix[j+1][xi+1][yi+0]-Matrix[j+1][xi-1][yi+0]) \
                            + 1*(Matrix[j+1][xi+1][yi-1]-Matrix[j+1][xi-1][yi-1])+2*(Matrix[j+0][xi+1][yi+1]-Matrix[j+0][xi-1][yi+1]) \
                            + 4*(Matrix[j+0][xi+1][yi+0]-Matrix[j+0][xi-1][yi+0])+2*(Matrix[j+0][xi+1][yi-1]-Matrix[j+0][xi-1][yi-1]) \
                            + 1*(Matrix[j-1][xi+1][yi+1]-Matrix[j-1][xi-1][yi+1])+2*(Matrix[j-1][xi+1][yi+0]-Matrix[j-1][xi-1][yi+0]) \
                            + 1*(Matrix[j-1][xi+1][yi-1]-Matrix[j-1][xi-1][yi-1])

                        vy = 1*(Matrix[j+1][xi+1][yi+1]-Matrix[j+1][xi+1][yi-1])+2*(Matrix[j+1][xi+0][yi+1]-Matrix[j+1][xi+0][yi-1]) \
                            + 1*(Matrix[j+1][xi-1][yi+1]-Matrix[j+1][xi-1][yi-1])+2*(Matrix[j+0][xi+1][yi+1]-Matrix[j+0][xi+1][yi-1]) \
                            + 4*(Matrix[j+0][xi+0][yi+1]-Matrix[j+0][xi+0][yi-1])+2*(Matrix[j+0][xi-1][yi+1]-Matrix[j+0][xi-1][yi-1]) \
                            + 1*(Matrix[j-1][xi+1][yi+1]-Matrix[j-1][xi+1][yi-1])+2*(Matrix[j-1][xi+0][yi+1]-Matrix[j-1][xi-0][yi-1]) \
                            + 1*(Matrix[j-1][xi-1][yi+1]-Matrix[j-1][xi-1][yi-1])

                        vt = 1*(Matrix[j+1][xi+1][yi+1]-Matrix[j-1][xi+1][yi+1])+2*(Matrix[j+1][xi+1][yi+0]-Matrix[j-1][xi+1][yi+0]) \
                            + 1*(Matrix[j+1][xi+1][yi-1]-Matrix[j-1][xi+1][yi-1])+2*(Matrix[j+1][xi+0][yi+1]-Matrix[j-1][xi+0][yi+1]) \
                            + 4*(Matrix[j+1][xi+0][yi+0]-Matrix[j-1][xi+0][yi+0])+2*(Matrix[j+1][xi+0][yi-1]-Matrix[j-1][xi+0][yi-1]) \
                            + 1*(Matrix[j+1][xi-1][yi+1]-Matrix[j-1][xi-1][yi+1])+2*(Matrix[j+1][xi-1][yi+0]-Matrix[j-1][xi-1][yi+0]) \
                            + 1*(Matrix[j+1][xi-1][yi-1]-Matrix[j-1][xi-1][yi-1])

                        gval = np.sqrt(vx**2 + vy**2 + vt**2)

                        if gval > 0:
                            fx[j][xi][yi] = vx / gval 
                            fy[j][xi][yi] = vy / gval
                            ft[j][xi][yi] = vt / gval
                            fg[j][xi][yi] = gval
                        else:
                            fx[j][xi][yi] = 0
                            fy[j][xi][yi] = 0
                            ft[j][xi][yi] = 0
                            fg[j][xi][yi] = 0
        self.FXYTG = [fx, fy, ft, fg]
        self.feedback.setProgress(65)
        for index, grid in self.gdf.iterrows():
            x = round(grid.geometry.centroid.x,3)
            y = round(grid.geometry.centroid.y,3)

            xi = int((x - self.Xmin)/self.grid_size_x)
            yi = int((y - self.Ymin)/self.grid_size_x)

            if xi>0 and (xi+1)<self.Xi_range and yi>0 and (yi+1)<self.Yi_range:
                for j in range(1,self.timestamps-1):
                    tfield_name = 'T' + str(j+1)
                    gfield_name = 'G' + str(j+1)

                    vxfield = 'v_' + str(j+1) + '_x'
                    vyfield = 'v_' + str(j+1) + '_y'
                    vzfield = 'v_' + str(j+1) + '_z'

                    ax = fx[j][xi][yi]
                    ay = fy[j][xi][yi]
                    at = ft[j][xi][yi]

                    initial_bearing = np.arctan2(ax, ay)
                    initial_bearing = np.degrees(initial_bearing)
                    azi = (initial_bearing + 360) % 360
                    cazi = (azi + 180) % 360 if at < 0 else azi

                    self.gdf_flow.at[index, tfield_name] = cazi
                    self.gdf_flow.at[index, gfield_name] = fg[j][xi][yi]
                    self.gdf_vcomp.at[index, vxfield] = ax
                    self.gdf_vcomp.at[index, vyfield] = ay
                    self.gdf_vcomp.at[index, vzfield] = at
        self.gdf_flow.fillna(0, inplace=True)
        self.feedback.setProgress(70)            
        
    def _accflow_process(self) -> None: 
        
        r_grid_userdef = self.parameters[self.VARIABLES['ACCMASKSIZE']]
        r_grid   = int((r_grid_userdef - 1)/2)
        
        timestamp_field_acc = 'azi'
        gradient_field_acc = 'gradient'
        fx, fy, ft, fg = self.FXYTG
        
        for index, grid in self.gdf.iterrows():
            x = round(grid.geometry.centroid.x,3)
            y = round(grid.geometry.centroid.y,3)

            xi = int((x - self.Xmin)/self.grid_size_x)
            yi = int((y - self.Ymin)/self.grid_size_x)
            ax = 0
            ay = 0
            at = 0
            #ag = 0
            count = 0

            for j in range(1,self.timestamps-1):
                #ag = ag + Matrix[j][xi][yi]
                for m in range(-r_grid, r_grid+1):# spatial
                    for n in range(-r_grid, r_grid+1):
                        xii = xi+m
                        yii = yi+n

                        #weighted
                        dist_power2 = m*m + n*n
                        w = np.exp(-dist_power2/(1.2*1.2*(r_grid+1)*(r_grid+1)))

                        if xii>=0 and xii<self.Xi_range and yii>=0 and yii<self.Yi_range:
                            ax = ax + fx[j][xii][yii] * w * fg[j][xii][yii]
                            ay = ay + fy[j][xii][yii] * w * fg[j][xii][yii]
                            at = at + ft[j][xii][yii] * w * fg[j][xii][yii]

                        count = count + 1*w
  
            if count != 0:
                ax = ax / count # normalized
                ay = ay / count
                at = at / count

                agval = np.sqrt(ax**2 + ay**2 + at**2)

                # normalized
                ax = ax / agval 
                ay = ay / agval
                at = at / agval

                initial_bearing = np.arctan2(ax, ay)
                initial_bearing = np.degrees(initial_bearing)
                azi = (initial_bearing + 360) % 360
                cazi = (azi + 180) % 360 if at < 0 else azi

                self.gdf_accflow.at[index, timestamp_field_acc] = cazi
                self.gdf_accflow.at[index, gradient_field_acc] = agval
    
    def main_process(self):
        self.feedback.setProgress(10)
        self._flow_estimate_process()
        self.feedback.setProgress(50)
        self._accflow_process()
        self.feedback.setProgress(80)
        return self.gdf_flow, self.gdf_vcomp, self.gdf_accflow