Slovak Games audio+text embeddings_final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## First step \n",
    "## Download the slovak games corpus to a specific location\n",
    "## Later, execute the code in sequence \n",
    "## This program reads each textgrid file and creates a new words file with three columns specifying start, end and words uttered by the speaker.\n",
    "## The program reads the .TextGrid file from each folder and creates a new words file with the extension .words\n",
    "## For instance, s01.objects.1.A.words.TextGrid will be processed, and the new file will be created on the same path with s01.objects.1.A.words\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import glob\n",
    "import os.path\n",
    "import pandas as pd\n",
    "import re\n",
    "import textgrid\n",
    "#Install textgrid from https://github.com/kylebgorman/textgrid\n",
    "\n",
    "#Specify the path of the dataset\n",
    "list_of_files = glob.glob(r'D:\\Jay\\sk-games-zip\\**\\*.TextGrid',recursive=True)\n",
    "\n",
    "for file_name in list_of_files:    \n",
    "    out_name=file_name[:-9]\n",
    "    tgrid = textgrid.read_textgrid(file_name)\n",
    "    df1=pd.DataFrame(tgrid)\n",
    "    df1.drop('tier', axis=1, inplace=True)\n",
    "    df1.to_csv(out_name, index=False,header=False,sep=' ')\n",
    "    del tgrid, df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Second step \n",
    "## The program reads each words file generated in the first step and creates a new words file with additional information about \"speaker\" and \"filename\".\n",
    "## The program reads the .words file from each folder and creates a new words file with the extension .words_with_speaker\n",
    "## For instance, s01.objects.1.A.words will be processed, and the new file will be created on the same path with s01.objects.1.A.words_with_speaker\n",
    "\n",
    "#Program to add speaker and filename column in each word file\n",
    "import os\n",
    "import sys\n",
    "import glob\n",
    "import os.path\n",
    "import pandas as pd\n",
    "import re\n",
    "\n",
    "list_of_files = glob.glob(r'D:\\Jay\\sk-games-zip\\data\\**\\*.words',recursive=True) \n",
    "\n",
    "for file_name in list_of_files:    \n",
    "    out_name=file_name+'_with_speaker'\n",
    "    csv_input = pd.read_csv(file_name,delimiter=' ',header=None)\n",
    "    csv_input['Speaker'] = file_name[-7]\n",
    "    csv_input['filename'] = file_name\n",
    "    csv_input.to_csv(out_name, index=False,header=False,sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Third step \n",
    "## The program reads each words_with_speaker file processed in the second step and combines the text files of both speakers in each session.\n",
    "## For instance, s01.objects.1.A.words_with_speaker and s01.objects.1.B.words_with_speaker will be read and combined. \n",
    "## The output will be saved as s01.objects.1.A.words_with_speaker_txt\n",
    "## The program also converts each wav file sample rate to 16khz\n",
    "## For instance, s01.objects.1.A.wav will be transformed to s01.objects.1.A.wav with sample rate 16k\n",
    "\n",
    "#Program to merge Speaker text files and convert flac files\n",
    "\n",
    "\n",
    "import os\n",
    "import fnmatch\n",
    "import sys\n",
    "import subprocess\n",
    "\n",
    "#Specify path of dataset\n",
    "OLD_BASE = r'D:\\Jay\\sk-games-zip\\data'\n",
    "\n",
    "#Create the empty folder where ouput needs to be saved\n",
    "#Specify path of output folder\n",
    "NEW_BASE = r'D:\\Jay\\sk-games-zip\\jkdata'\n",
    "\n",
    "def merge_files(infiles):\n",
    "    output=os.path.basename(infiles[0])+'_txt'\n",
    "    outfiletxt = os.path.join(new_dir, output)\n",
    "    with open(outfiletxt, 'wb') as fo:\n",
    "        for infile in infiles:\n",
    "            with open(infile, 'rb') as fi:\n",
    "                fo.write(fi.read())\n",
    "\n",
    "def convert_files(infiles):\n",
    "    output_channel1=os.path.basename(infiles[0][:-3])+'wav'\n",
    "    output_channel2=os.path.basename(infiles[1][:-3])+'wav'\n",
    "    outfile1 = os.path.join(new_dir, output_channel1)\n",
    "    outfile2 = os.path.join(new_dir, output_channel2)\n",
    "    #Specify path of ffmpeg for converting audio files\n",
    "    ffp=r\"C:\\ffmpeg-master-latest-win64-gpl\\bin\\ffmpeg\"\n",
    "    cmd2wav1 = ffp+' -i ' + infiles[0] + ' ' + \"-ar 16000\" + ' ' + outfile1\n",
    "    cmd2wav2 = ffp+' -i ' + infiles[1] + ' ' + \"-ar 16000\" + ' ' + outfile2\n",
    "    print(cmd2wav1)\n",
    "    print(cmd2wav2)\n",
    "    subprocess.call(cmd2wav1, shell=True)\n",
    "    subprocess.call(cmd2wav2, shell=True)\n",
    "\n",
    "for (dirpath, dirnames, filenames) in os.walk(OLD_BASE):\n",
    "    base, tail = os.path.split(dirpath)\n",
    "    if base != OLD_BASE: continue  # Don't operate on OLD_BASE, only children directories\n",
    "\n",
    "    # Build infiles list for flac objects\n",
    "    object_flac = sorted([os.path.join(dirpath, filename) for filename in filenames if fnmatch.fnmatch(filename,\"*objects*.wav\")])\n",
    "    # Build infiles list for words objects\n",
    "    object_text = sorted([os.path.join(dirpath, filename) for filename in filenames if fnmatch.fnmatch(filename,\"*objects*.words_with_speaker\")])\n",
    "\n",
    "    # Create output directory\n",
    "    new_dir =  os.path.join(NEW_BASE, tail)\n",
    "    os.mkdir(new_dir)  # This will raise an OSError if the directory already exists    \n",
    "\n",
    "    # Merge\n",
    "    convert_files(object_flac)\n",
    "    merge_files(object_text)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fourth step \n",
    "## The program reads each concatenated words_with_speaker_txt file processed in the third step and further processes it in the following way.\n",
    "## Each word consecutively spoken by a speaker is merged and converted into a turn.\n",
    "## Similarly, start and end times are adjusted accordingly\n",
    "## For instance, s01.objects.1.A.words_with_speaker_txt will be read and s01.objects.1.txt will be generated in same folder. \n",
    "\n",
    "#Program to extract turns\n",
    "import os\n",
    "import sys\n",
    "import glob\n",
    "import os.path\n",
    "import pandas as pd\n",
    "import re\n",
    "\n",
    "#Add path for output files saved in second step\n",
    "list_of_files = glob.glob(r'D:\\Jay\\sk-games-zip\\jkdata\\**\\*.words_with_speaker_txt',recursive=True) \n",
    "\n",
    "for file_name in list_of_files:    \n",
    "    out_name=file_name[:-24]+'txt'\n",
    "    headers= ['start','end','words','Speaker','filename']\n",
    "    csv_input = pd.read_csv(file_name,delimiter=' ')\n",
    "    csv_input.columns = headers\n",
    "    cgc2=csv_input.sort_values(['start'])    \n",
    "    cgc3=cgc2[cgc2[\"words\"].str.contains(\"#\")==False]\n",
    "    cgc3[['start', 'end']] = cgc3[['start', 'end']].astype(str)\n",
    "    #Merge consecutive turns of same speaker\n",
    "    cgc3['obj1_count'] = (cgc3['Speaker'].ne(cgc3['Speaker'].shift())).cumsum()\n",
    "    df3=cgc3.groupby('obj1_count').agg(lambda x: ' '.join(x))\n",
    "    #Adjust start and end time\n",
    "    df3['start'] = df3['start'].map(lambda x: x.split(\" \")[0])\n",
    "    df3['end'] = df3['end'].map(lambda x: x.split(\" \")[-1])\n",
    "    #Add speaker information\n",
    "    df3['Speaker'] = df3['Speaker'].map(lambda x: x.split(\" \")[-1])\n",
    "    df3['filename'] = df3['filename'].map(lambda x: x.split(\" \")[-1])\n",
    "    df3['filename'] = df3['filename'].str.replace('words','flac')\n",
    "    #Add session information\n",
    "    df3['session']=re.search('(\\d\\d)',file_name)[0]\n",
    "    if df3.iloc[-1]['Speaker'] == df3.iloc[0]['Speaker']:\n",
    "        df4=df3.drop(df3.index[len(df3)-1])\n",
    "        df4.to_csv(out_name, index=False,header=False,sep='\\t')\n",
    "        del df4\n",
    "    else:\n",
    "        df3.to_csv(out_name, index=False,header=False,sep='\\t')\n",
    "    del csv_input,cgc2,cgc3,df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fifth step \n",
    "## Program to delete words_with_speaker_txt file\n",
    "## The program deletes all the words_with_speaker_txt files generated in third step\n",
    "## The program allows better readability of the generated output\n",
    "## After executing the program, Each session folder will have two wav files and one txt file\n",
    "## For instance, s01.objects.1.A.wav, s01.objects.1.B.wav and s01.objects.1.txt\n",
    "import glob\n",
    "\n",
    "#Delete files with unwanted extension\n",
    "# Please specify path of the folder \n",
    "for f in glob.glob(r'D:\\Jay\\sk-games-zip\\jkdata\\**\\*.words_with_speaker_txt',recursive=True):\n",
    "    os.remove(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sixth step \n",
    "## Program to extract semantic and auditory embeddings from each turn\n",
    "## The program reads each text file generated in the fourth step and extracts TRILL vectors and semantic embeddings from each turn\n",
    "## The program saves embeddings in pkl file format\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "from transformers import BertTokenizer, BertModel\n",
    "import torch\n",
    "from functools import reduce\n",
    "import csv\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import random\n",
    "import subprocess\n",
    "import re\n",
    "import glob\n",
    "import os,sys\n",
    "import numpy as np\n",
    "import soundfile as sf\n",
    "import wave\n",
    "import json\n",
    "import tensorflow as tf1\n",
    "import tensorflow_hub as hub\n",
    "# Import TF 2.X and make sure we're running eager.\n",
    "import tensorflow.compat.v2 as tf\n",
    "#import tensorflow as tf\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "modulev3=None\n",
    "modulev3_graph=None\n",
    "import pickle\n",
    "tf.enable_v2_behavior()\n",
    "assert tf.executing_eagerly()\n",
    "   \n",
    "##Function to load TRILL vector model    \n",
    "def get_TRILLv3_signal(signal,samplerate):\n",
    "    global modulev3\n",
    "    if modulev3==None:\n",
    "        #Specify the path of TRILL vector model\n",
    "        print('******************\\nLoading model ...\\n******************')    \n",
    "        modulev3 = hub.load(r'D:\\Jay\\columbia-games-corpus\\trill_extraction_v2\\v3')\n",
    "    \n",
    "    \n",
    "    max_int16 = 2**15\n",
    "    chunks_cnt=int(signal.shape[0]/(samplerate*10.0))#10 seconds max in chunk\n",
    "    if chunks_cnt==0:\n",
    "        chunks=[signal]\n",
    "    else:\n",
    "        chunks=np.array_split(signal, chunks_cnt)\n",
    "    \n",
    "    trillv3_emb_all=np.empty(shape=(0,512))\n",
    "    \n",
    "    for chunk in chunks:\n",
    "        trillv3 = modulev3(samples=chunk, sample_rate=samplerate)\n",
    "        trillv3_emb = trillv3['embedding']\n",
    "        trillv3_emb_all=np.concatenate((trillv3_emb_all, trillv3_emb))\n",
    "\n",
    "    trillv3_emb_avg = np.mean(trillv3_emb_all, axis=0, keepdims=False)\n",
    "\n",
    "    return (trillv3_emb_avg.tolist())    \n",
    "\n",
    "##Function to check if audio files are in proper format \n",
    "def check_wav_format(wav_file, start, end):\n",
    "    wf = wave.open(wav_file)\n",
    "    nchannels, sampwidth, framerate, nframes, comptype, compname = wf.getparams()\n",
    "    print(nchannels, sampwidth, framerate, nframes, comptype, compname)\n",
    "    wav_length = float(nframes) / float(framerate)\n",
    "    print(wav_length)\n",
    "    if nchannels!=1:\n",
    "        print('Error: Incoming audio file has more then 1 channel')\n",
    "        return(-1) \n",
    "\t\t\t\t\n",
    "    if framerate!=16000:\n",
    "        print('Error: Incoming audio file sampling frequency must be 16000')\n",
    "        return(-2)\n",
    "\t\t\t\t\n",
    "    if sampwidth!=2:\n",
    "        print('Error: Incoming audio file sample width must be 16 bit')\n",
    "        return(-3)\n",
    "\n",
    "    if wav_length<end:\n",
    "        print('Error: The duration of the audio file is shorter than the required end time')\n",
    "        return(-4)\n",
    "\t\t\n",
    "    if start<0.0:\n",
    "        print('Error: start time is lower then 0.0')\n",
    "        return(-5)\n",
    "    \n",
    "    if start>=end:\n",
    "        print('Error: start time is larger then end time')\n",
    "        return(-6)    \n",
    "\t\t\n",
    "\t\t\t\t\n",
    "    return(framerate)\n",
    "\n",
    "##Function to extract TRILL embeddings from specific file name with start and end time as parameters  \n",
    "def get_TRILLv3_audiofile_from_to(wav_file,start,end):\n",
    "    print('get_TRILLv3_signal:',wav_file,start,end)\n",
    "    samplerate=check_wav_format(wav_file, start, end)\n",
    "    if samplerate<0:\n",
    "        return(null)    \n",
    "    startsample=int(start*samplerate)\n",
    "    endsample=int(end*samplerate)\n",
    "    signal, samplerate = sf.read(wav_file,start=startsample, stop=endsample)\n",
    "    print(len(signal),samplerate)    \n",
    "\n",
    "    trill=get_TRILLv3_signal(signal,samplerate)\n",
    "    return(trill)\n",
    "\n",
    "##Specify input path from where text files needs to be read\n",
    "#Specify path for files generated in fourth step\n",
    "path = 'D:\\\\Jay\\\\sk-games-zip\\\\jkdata\\\\'\n",
    "\n",
    "#Specify path where auditory embeddings needs to be saved\n",
    "audio_embedding_path = 'D:\\\\Jay\\\\sk-games-zip\\\\jkdata\\\\embedding\\\\audio\\\\'\n",
    "\n",
    "#Specify path where semantic embeddings needs to be saved\n",
    "text_embedding_path = 'D:\\\\Jay\\\\sk-games-zip\\\\jkdata\\\\embedding\\\\text\\\\'\n",
    "\n",
    "#Initialize transformer model for semantic feature extraction\n",
    "model = SentenceTransformer('kinit/slovakbert-sts-stsb')\n",
    "\n",
    "all_files = os.listdir(path)\n",
    "\n",
    "for root, dirs, files in os.walk(path):\n",
    "    for file in files:\n",
    "        if file.endswith('.txt'):\n",
    "            with open(os.path.join(root, file), 'r', encoding=\"utf8\") as f:\n",
    "                out_audio_embedding= os.path.join(audio_embedding_path, os.path.basename(file))\n",
    "                out_text_embedding= os.path.join(text_embedding_path, os.path.basename(file))\n",
    "                sentenceembeddings=[]\n",
    "                audioembeddings = []\n",
    "                sentence_embeddings_tensor = []\n",
    "                text = f.readlines()\n",
    "                for line in text:\n",
    "                    n1=line.split('\\t')[0]\n",
    "                    n2=line.split('\\t')[1]\n",
    "                    n3=line.split('\\t')[4].replace('flac', 'wav').replace('data', 'jkdata')\n",
    "                    sentence=line.split('\\t')[2]\n",
    "                    x=get_TRILLv3_audiofile_from_to(n3,float(n1),float(n2))\n",
    "                    sentenceembeddings.append(sentence)\n",
    "                    audioembeddings.append(x)\n",
    "                sentence_embeddings_tensor = model.encode(sentenceembeddings, convert_to_tensor=False)\n",
    "                sentence_embeddings_tensor=sentence_embeddings_tensor.tolist()\n",
    "                with open(out_audio_embedding+'.pkl', 'wb') as f:\n",
    "                    pickle.dump(audioembeddings, f)\n",
    "                with open(out_text_embedding+'.pkl', 'wb') as f:\n",
    "                    pickle.dump(sentence_embeddings_tensor, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Seventh step \n",
    "## Program to measure entrainment distance using semantic and auditory embeddings\n",
    "## The program reads pkl file generated in the FSixth step and measures adjacent and non-adjacent scores at semantic and acoustic levels\n",
    "## The program also measures self score of both the speakers at the semantic and acoustic level\n",
    "## The program generates a text file for each session with the following column information\n",
    "### Same_pair_text [adjacent scores semantic] Not_Same_Pair_text [non-adjacent scores semantic]\n",
    "### Same_pair_audio [adjacent scores acoustic]  Not_Same_Pair_audio [non-adjacent scores acoustic]  \n",
    "### Speaker1_text_self_distance [Self score semantic Speaker 1] Speaker2_text_self_distance [Self score semantic Speaker 2]\n",
    "### Speaker1_audio_self_distance    [Self score acoustic Speaker 1] Speaker2_audio_self_distance [Self score acoustic Speaker 2]\n",
    "\n",
    "import pandas as pd\n",
    "import csv\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sentence_transformers import util\n",
    "import random\n",
    "import subprocess\n",
    "import re\n",
    "import glob\n",
    "import os,sys\n",
    "import numpy as np\n",
    "import pickle\n",
    "\n",
    "random.seed(223)\n",
    "#Define path of embedding files for acoustic and semantic features generated in Sixth step \n",
    "audio_embedding_path = 'D:\\\\Jay\\\\sk-games-zip\\\\jkdata\\\\embedding\\\\audio\\\\'\n",
    "text_embedding_path = 'D:\\\\Jay\\\\sk-games-zip\\\\jkdata\\\\embedding\\\\text\\\\'\n",
    "\n",
    "#Specify the output path where output needs to be saved\n",
    "output_path = 'D:\\\\Jay\\\\sk-games-zip\\\\jkdata\\\\output\\\\'\n",
    "\n",
    "for audiofile,textfile in zip(os.listdir(audio_embedding_path),os.listdir(text_embedding_path)):\n",
    "        if audiofile.endswith(\".pkl\") & textfile.endswith(\".pkl\"):\n",
    "            current_audiofile = os.path.join(audio_embedding_path,audiofile)\n",
    "            current_textfile = os.path.join(text_embedding_path,textfile)\n",
    "            output_filename= os.path.splitext(os.path.join(output_path, os.path.basename(audiofile)))[0]\n",
    "            #Open both audio and semantic pickle files\n",
    "            with open(current_audiofile, 'rb') as a, open(current_textfile, 'rb') as b:\n",
    "                audio_embeddings_tensor=pickle.load(a)\n",
    "                sentence_embeddings_tensor=pickle.load(b)\n",
    "                sentence_embedding_random = []\n",
    "                audio_embedding_random = []\n",
    "                text_speaker1=[]\n",
    "                text_speaker2=[]\n",
    "                audio_speaker1=[]\n",
    "                audio_speaker2=[]\n",
    "                same_pairs_text = []\n",
    "                not_samepairs_text_array = []\n",
    "                not_samepairs_text=[]\n",
    "                same_pairs_audio = []\n",
    "                not_samepairs_audio_array = []\n",
    "                not_samepairs_audio=[]\n",
    "                \n",
    "                for i in range(0, len(sentence_embeddings_tensor)-1):\n",
    "                    #Condition for random turns to exclude current and next adjacent turn  \n",
    "                    sentence_embedding_random = [item for item in sentence_embeddings_tensor if np.logical_and(\n",
    "                                                 item!=sentence_embeddings_tensor[i] , item != sentence_embeddings_tensor[i+1])]\n",
    "                    audio_embedding_random = [item for item in audio_embeddings_tensor if np.logical_and(\n",
    "                                                 item!=audio_embeddings_tensor[i] , item != audio_embeddings_tensor[i+1])]\n",
    "\n",
    "                    if(i%2==0):\n",
    "                        #Measure Adjacent score for even turns\n",
    "                        spt=util.cos_sim(sentence_embeddings_tensor[i],sentence_embeddings_tensor[i+1]).item()\n",
    "                        same_pairs_text.append(spt)\n",
    "                        spa=util.cos_sim(audio_embeddings_tensor[i],audio_embeddings_tensor[i+1]).item()\n",
    "                        same_pairs_audio.append(spa)\n",
    "                        \n",
    "                        for x in range(10):\n",
    "                            #Measure Non-adjacent score for odd turns\n",
    "                            #Here, mean Non-adjacent score is measured on 10 random turns\n",
    "                            nspta=util.cos_sim(sentence_embeddings_tensor[i],sentence_embedding_random[random.randrange(1,len(sentence_embedding_random),2)]).item()\n",
    "                            not_samepairs_text_array.append(nspta)\n",
    "                            nspaa=util.cos_sim(audio_embeddings_tensor[i],audio_embedding_random[random.randrange(1,len(audio_embedding_random),2)]).item()\n",
    "                            not_samepairs_audio_array.append(nspaa)\n",
    "\n",
    "                        nspt = np.mean(not_samepairs_text_array)\n",
    "                        nspa = np.mean(not_samepairs_audio_array)\n",
    "\n",
    "                        not_samepairs_text_array = []\n",
    "                        not_samepairs_audio_array = []\n",
    "\n",
    "                        not_samepairs_text.append(nspt)                       \n",
    "                        not_samepairs_audio.append(nspa)\n",
    "\n",
    "                    elif(i%2!=0):\n",
    "                        #Measure Adjacent score for odd turns\n",
    "                        spt=util.cos_sim(sentence_embeddings_tensor[i],sentence_embeddings_tensor[i+1]).item()\n",
    "                        same_pairs_text.append(spt)\n",
    "                        spa=util.cos_sim(audio_embeddings_tensor[i],audio_embeddings_tensor[i+1]).item()\n",
    "                        same_pairs_audio.append(spa)\n",
    "                        \n",
    "                        for x in range(10):\n",
    "                            #Measure Non-adjacent score for odd turns\n",
    "                            #Here, mean Non-adjacent score is measured on 10 random turns\n",
    "                            nspta=util.cos_sim(sentence_embeddings_tensor[i],sentence_embedding_random[random.randrange(0,len(sentence_embedding_random),2)]).item()\n",
    "                            not_samepairs_text_array.append(nspta)\n",
    "                            nspaa=util.cos_sim(audio_embeddings_tensor[i],audio_embedding_random[random.randrange(0,len(audio_embedding_random),2)]).item()\n",
    "                            not_samepairs_audio_array.append(nspaa)\n",
    "\n",
    "                        nspt = np.mean(not_samepairs_text_array)\n",
    "                        nspa = np.mean(not_samepairs_audio_array)\n",
    "\n",
    "                        not_samepairs_text_array = []\n",
    "                        not_samepairs_audio_array = []\n",
    "\n",
    "                        not_samepairs_text.append(nspt)                       \n",
    "                        not_samepairs_audio.append(nspa)\n",
    "\n",
    "                for n in range(0, len(sentence_embeddings_tensor)-2):\n",
    "                    #Measure self-scores\n",
    "                    if(n%2==0):\n",
    "                        same_spkr1_text=util.cos_sim(sentence_embeddings_tensor[n],sentence_embeddings_tensor[n+2]).item()\n",
    "                        text_speaker1.append(same_spkr1_text)\n",
    "                        same_spkr1_audio=util.cos_sim(audio_embeddings_tensor[n],audio_embeddings_tensor[n+2]).item()\n",
    "                        audio_speaker1.append(same_spkr1_audio)\n",
    "                        \n",
    "                    elif(n%2!=0):\n",
    "                        same_spkr2_text=util.cos_sim(sentence_embeddings_tensor[n],sentence_embeddings_tensor[n+2]).item()\n",
    "                        text_speaker2.append(same_spkr2_text)\n",
    "                        same_spkr2_audio=util.cos_sim(audio_embeddings_tensor[n],audio_embeddings_tensor[n+2]).item()\n",
    "                        audio_speaker2.append(same_spkr2_audio)\n",
    "                        \n",
    "                df = pd.DataFrame(data=None)\n",
    "                df['Same_pair_text'] = pd.Series(same_pairs_text)\n",
    "                df['Not_Same_Pair_text'] = pd.Series(not_samepairs_text)\n",
    "                df['Same_pair_audio'] = pd.Series(same_pairs_audio)\n",
    "                df['Not_Same_Pair_audio'] = pd.Series(not_samepairs_audio)\n",
    "\n",
    "                df['Speaker1_text_self_distance'] = pd.Series(text_speaker1)\n",
    "                df['Speaker2_text_self_distance'] = pd.Series(text_speaker2)\n",
    "                df['Speaker1_audio_self_distance'] = pd.Series(audio_speaker1)\n",
    "                df['Speaker2_audio_self_distance'] = pd.Series(audio_speaker2)\n",
    "\n",
    "                df.insert(0, 'Turn_number', range(1, 1 + len(df)))\n",
    "                df.to_csv(output_filename, index=False,sep='\\t')\n",
    "                same_pairs_text = []\n",
    "                not_samepairs_text_array = []\n",
    "                not_samepairs_text=[]\n",
    "                same_pairs_audio = []\n",
    "                not_samepairs_audio_array = []\n",
    "                not_samepairs_audio=[]\n",
    "                sentence_embedding_random = []\n",
    "                audio_embedding_random = []\n",
    "                text_speaker1 = []\n",
    "                text_speaker2 = []\n",
    "                audio_speaker1 = []\n",
    "                audio_speaker2 = []\n",
    "                del df,spt,nspta,nspt,spa,nspaa,nspa,same_spkr1_text,same_spkr1_audio,same_spkr2_text,same_spkr2_audio\n",
    "\n",
    "         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Eight step \n",
    "## Program to measure proximity, convergence and syncrony using adjacent, non-adjacent and self-score obtained in Seventh step\n",
    "## The program reads text file generated in Seventh step and statistically analyzes the data using different methods \n",
    "## For instance, Pearson's correlation for measuring convergence\n",
    "## The program generates text file with following column information for each session\n",
    "\n",
    "### Relationship between semantic and acoutic embeddings at adjacent turns\n",
    "## Relationship-bw-both-Pearson rho\t Relationship-bw-both-Pearson p-value\tRelationship-bw-both-Pearson p-value_adj\t\n",
    "\n",
    "### Proximity at semantic level\n",
    "## Proximity-t-value_text\tProximity-p-value_text\tProximity-p-value_text_adj\t\n",
    "### Proximity at acoutic level\n",
    "## Proximity-t-value_audio\tProximity-p-value_audio\tProximity-p-value_audio_adj\t\n",
    "\n",
    "### Convergence at semantic level\n",
    "## Convergence-Pearson rho_text\tConvergence-Pearson p-value_text\tConvergence-Pearson p-value_text_adj\t\n",
    "### Convergence at acoustic level\n",
    "## Convergence-Pearson rho_audio\tConvergence-Pearson p-value_audio\tConvergence-Pearson p-value_audio_adj\t\n",
    "\n",
    "### Syncrony at semantic level\n",
    "## Syncrony-Pearson rho_text\tSyncrony-Pearson p-value_text\tSyncrony-Pearson p-value_text_adj\t\n",
    "### Syncrony at acoustic level\n",
    "## Syncrony-Pearson rho_audio\tSyncrony-Pearson p-value_audio\tSyncrony-Pearson p-value_audio_adj\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy\n",
    "import pingouin as pg\n",
    "from scipy.stats import ttest_ind\n",
    "from math import sqrt\n",
    "import sys\n",
    "from scipy import stats\n",
    "import os\n",
    "import sys\n",
    "import glob\n",
    "import os.path\n",
    "import pandas as pd\n",
    "from statsmodels.stats.multitest import multipletests\n",
    "\n",
    "#Specify path of input files generated at seventh step\n",
    "list_of_files = glob.glob(r'D:\\Jay\\sk-games-zip\\jkdata\\output\\*.txt',recursive=True)\n",
    "\n",
    "#Specify output path\n",
    "output_path = r'D:\\Jay\\sk-games-zip\\jkdata\\statsoutput'\n",
    "\n",
    "for file_name in list_of_files:\n",
    "    out_name= os.path.join(output_path, os.path.basename(file_name))\n",
    "    csv_input = pd.read_csv(file_name, delimiter='\\t')\n",
    "    \n",
    "    #For relationship between audio and text\n",
    "    s,p = stats.pearsonr(csv_input['Same_pair_audio'], csv_input['Same_pair_text'])\n",
    "    csv_input['Relationship-bw-both-Pearson rho'] = s\n",
    "    csv_input['Relationship-bw-both-Pearson p-value'] = p\n",
    "    #Here, Bonferroni correction is applied. Since there were 9 sessions we divided the significance value (0.05) by nr. of sessions\n",
    "    csv_input['Relationship-bw-both-Pearson p-value_adj'] = csv_input['Relationship-bw-both-Pearson p-value'].lt(0.05/9)\n",
    "  \n",
    "    #Proximity\n",
    "    proximity_text=stats.ttest_rel(csv_input['Same_pair_text'], csv_input['Not_Same_Pair_text'])\n",
    "    csv_input['Proximity-t-value_text']= proximity_text.statistic\n",
    "    csv_input['Proximity-p-value_text']= proximity_text.pvalue\n",
    "    #Bonferroni correction\n",
    "    csv_input['Proximity-p-value_text_adj'] = csv_input['Proximity-p-value_text'].lt(0.05/9)\n",
    "\n",
    "\n",
    "    proximity_audio=stats.ttest_rel(csv_input['Same_pair_audio'], csv_input['Not_Same_Pair_audio'])\n",
    "    csv_input['Proximity-t-value_audio']= proximity_audio.statistic\n",
    "    csv_input['Proximity-p-value_audio']= proximity_audio.pvalue\n",
    "    #Bonferroni correction\n",
    "    csv_input['Proximity-p-value_audio_adj'] = csv_input['Proximity-p-value_audio'].lt(0.05/9)\n",
    "    \n",
    "    #Convergence\n",
    "    convtextr,convtextp = stats.pearsonr(csv_input['Same_pair_text'], csv_input['Turn_number'])\n",
    "    csv_input['Convergence-Pearson rho_text'] = convtextr\n",
    "    csv_input['Convergence-Pearson p-value_text'] = convtextp\n",
    "    #Bonferroni correction\n",
    "    csv_input['Convergence-Pearson p-value_text_adj'] = csv_input['Convergence-Pearson p-value_text'].lt(0.05/9)\n",
    "\n",
    "    convaudior,convaudiop = stats.pearsonr(csv_input['Same_pair_audio'], csv_input['Turn_number'])\n",
    "    csv_input['Convergence-Pearson rho_audio'] = convaudior\n",
    "    csv_input['Convergence-Pearson p-value_audio'] = convaudiop\n",
    "    #Bonferroni correction\n",
    "    csv_input['Convergence-Pearson p-value_audio_adj'] = csv_input['Convergence-Pearson p-value_audio'].lt(0.05/9)\n",
    "\n",
    "    #Synchrony\n",
    "    csv_input.dropna(inplace=True)\n",
    "    synctextr,synctextp = stats.pearsonr(csv_input['Speaker1_text_self_distance'], csv_input['Speaker2_text_self_distance'])\n",
    "    csv_input['Syncrony-Pearson rho_text'] = synctextr\n",
    "    csv_input['Syncrony-Pearson p-value_text'] = synctextp\n",
    "    #Bonferroni correction\n",
    "    csv_input['Syncrony-Pearson p-value_text_adj'] = csv_input['Syncrony-Pearson p-value_text'].lt(0.05/9)\n",
    "\n",
    "    syncaudior,syncaudiop = stats.pearsonr(csv_input['Speaker1_audio_self_distance'], csv_input['Speaker2_audio_self_distance'])\n",
    "    csv_input['Syncrony-Pearson rho_audio'] = syncaudior\n",
    "    csv_input['Syncrony-Pearson p-value_audio'] = syncaudiop\n",
    "    #Bonferroni correction\n",
    "    csv_input['Syncrony-Pearson p-value_audio_adj'] = csv_input['Syncrony-Pearson p-value_audio'].lt(0.05/9)\n",
    " \n",
    "\n",
    "    csv_input.drop(['Same_pair_text', 'Not_Same_Pair_text', 'Same_pair_audio', 'Not_Same_Pair_audio',\n",
    "                'Speaker1_text_self_distance','Speaker2_text_self_distance','Turn_number','Speaker1_audio_self_distance',\n",
    "                'Speaker2_audio_self_distance'], axis=1,inplace=True)\n",
    "    csv_input = csv_input.loc[[0]]\n",
    "    csv_input.to_csv(out_name, index=False,sep='\\t')\n",
    "    del csv_input,s,p,proximity_text,convtextr,convtextp,synctextr,synctextp,proximity_audio,convaudior,convaudiop,syncaudior,syncaudiop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Final step \n",
    "## Program to merge output files generated at seventh step\n",
    "# The program concatenates statistical output of each inidividual sessions into one file\n",
    "\n",
    "import pandas as pd\n",
    "import glob\n",
    "\n",
    "#Specify Input path\n",
    "list_of_files = glob.glob(r'D:\\Jay\\sk-games-zip\\jkdata\\statsoutput\\*.txt',recursive=True)\n",
    "\n",
    "#Specify Output path\n",
    "out_name = r'D:\\Jay\\sk-games-zip\\jkdata\\statsoutput\\merged_stats.txt'\n",
    "li = []\n",
    "\n",
    "for filename in list_of_files:\n",
    "    df = pd.read_csv(filename, index_col=None, header=0, delimiter='\\t')\n",
    "    basename = os.path.basename(filename)\n",
    "    file_name = os.path.splitext(basename)[0]\n",
    "    df['File_name'] = file_name\n",
    "    #Re-arrange columns with filename as first column\n",
    "    cols = df.columns.tolist()\n",
    "    cols = cols[-1:] + cols[:-1]\n",
    "    #Define new arranged column to data-frame\n",
    "    df = df[cols]\n",
    "    #Append the dataframe to list\n",
    "    li.append(df)\n",
    "frame = pd.concat(li, axis=0, ignore_index=True) \n",
    "\n",
    "frame.to_csv(out_name, index=False,sep='\\t')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "43e249bda334b009252ca11161b502852fd620d93fbb243f1a14cfe6570157ba"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}