SubjectAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' \n",
    "June 16 2023 \n",
    "\n",
    "@author: Grady Robbins\n",
    "\n",
    "A Jupyter notebook which analyzes the Cool Neighbors subjects. Contains a cross match, an evaluation of the target's type accuracies, \n",
    "and an evaluation of all targets 'yes' votes for each type returning desired subjects\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Cross-Match\n",
    "\n",
    "A script to determine if candidates are present in a subject set using a csv of IDs to check.\n",
    "\n",
    "Subject ID is cross referenced with original subject set to find RA and DEC.\n",
    "\n",
    "Then RA and DEC are searched for in the csv file. If present, the candidate information is returned.\n",
    "'''\n",
    "\n",
    "import csv\n",
    "import re\n",
    "import numpy as np\n",
    "\n",
    "def limit_RA_DEC(RA,DEC,CharacterCount):\n",
    "    '''\n",
    "    This function limits the RA and DEC values to be x characters in length for subject matching.\n",
    "    parameters:\n",
    "    RA - list of float DEC values, should be in degrees\n",
    "    DEC - list of float DEC values, should be in degrees\n",
    "    returns:\n",
    "    --------------------------------\n",
    "    RA - shortened list of float RA values in degrees\n",
    "    DEC - shortened list of float DEC values in degrees\n",
    "    '''\n",
    "    for k in range(len(RA)):\n",
    "        RAlist = [*str(RA[k])]\n",
    "        DEClist = [*str(DEC[k])]\n",
    "        RA[k] = ''.join(RAlist[0:CharacterCount])\n",
    "        DEC[k] = ''.join(DEClist[0:CharacterCount])\n",
    "    return RA,DEC\n",
    "\n",
    "def FindSubjectCoords(filename = str):\n",
    "    '''\n",
    "    This function finds the RA and DEC coordinates for all subjects in a subject csv file.\n",
    "\n",
    "    Parameters:\n",
    "    filename - path to subject csv file\n",
    "    \n",
    "    Returns:\n",
    "    --------------------------------\n",
    "    RA - list of str RA values in degrees\n",
    "    DEC - list of str DEC values in degrees\n",
    "    '''\n",
    "    RA_subjects = []\n",
    "    DEC_subjects = []\n",
    "    with open(filename, 'r', newline='') as file: #load in file\n",
    "        for line in file:\n",
    "            line_list = re.split(',', line)\n",
    "            for k in range(len(line_list)):# iterate each line and return RA and DEC for all subjects\n",
    "                new_ra_list = []\n",
    "                new_dec_list = []\n",
    "                if '\"RA\"\":\"\"' in line_list[k]:\n",
    "                    ra_temp_list = [*line_list[k]]\n",
    "                    for l in range(len(ra_temp_list)):\n",
    "                        if ra_temp_list[l].isnumeric() or ra_temp_list[l] == '.':\n",
    "                            new_ra_list.append(ra_temp_list[l])\n",
    "                    RA_subjects.append(''.join(new_ra_list))\n",
    "                if '\"DEC\"\":\"\"' in line_list[k]:\n",
    "                    dec_temp_list = [*line_list[k]]\n",
    "                    for l in range(len(dec_temp_list)):\n",
    "                        if dec_temp_list[l].isnumeric() or dec_temp_list[l] == '.':\n",
    "                            new_dec_list.append(dec_temp_list[l])\n",
    "                    DEC_subjects.append(''.join(new_dec_list))\n",
    "    return RA_subjects, DEC_subjects\n",
    "\n",
    "def FindTargetIDs(filename = str):\n",
    "    '''\n",
    "    This function finds target IDs for cross matching from csv containing only subject IDs\n",
    "    \n",
    "    Parameters:\n",
    "    filename - path to candidate csv file\n",
    "    \n",
    "    Returns:\n",
    "    --------------------------------\n",
    "    target_IDs - a list of target IDs to match with a larger subject set\n",
    "    '''\n",
    "    target_IDs = []\n",
    "    with open(filename, 'r') as file:\n",
    "        for line in file:\n",
    "            target_IDs.append(str(int(line)))\n",
    "    return target_IDs\n",
    "\n",
    "def FindCandidateCoords(larger_candidate_filename = str, target_IDs = list):\n",
    "    '''\n",
    "    This function grabs the RA and DEC coordinates for all candidates in a subject source csv file using subject ID.\n",
    "    \n",
    "    Parameters:\n",
    "    larger_candidate_filename - path to larger candidate csv with all metadata\n",
    "    candidate_IDs - a list of subject IDs to match with a larger subject set    \n",
    "    Returns:\n",
    "    --------------------------------\n",
    "    RA - list of str RA values in degrees\n",
    "    DEC - list of str DEC values in degrees\n",
    "    '''\n",
    "    RA = []\n",
    "    DEC = []\n",
    "    with open(larger_candidate_filename, 'r', newline='') as file:\n",
    "            for line in file:\n",
    "                line_list = re.split(',', line)\n",
    "                for target_ID in target_IDs:\n",
    "                    if target_ID == line_list[0]:\n",
    "                        RA.append(line_list[1])\n",
    "                        DEC.append(line_list[2])\n",
    "    print(len(RA), 'candidates to check for presence')\n",
    "    return RA, DEC\n",
    "\n",
    "def MatchRADEC(RA_candidates,DEC_candidates, RA_subjects, DEC_subjects, target_IDs = None):\n",
    "    '''\n",
    "    This function matches the RA and DEC coordinates for all candidates to a larger subject set and prints the matching RA, DEC, and subject IDs.\n",
    "    \n",
    "    Parameters:\n",
    "    RA_candidates - list of str RA values in degrees for smaller candidate set\n",
    "    DEC_candidates - list of str DEC values in degrees for smaller candidate set\n",
    "    RA_subjects - list of str RA values in degrees for larger subject set\n",
    "    DEC_subjects - list of str DEC values in degrees for larger subject set\n",
    "    target_IDs - a list of subject IDs to print in event of a match\n",
    "    Returns:\n",
    "    --------------------------------\n",
    "    None\n",
    "    '''\n",
    "    cross_match = 0\n",
    "    for k in range(len(RA_candidates)):\n",
    "        if k in 500*np.arange(0,1001): #print progress\n",
    "            print('step',k)\n",
    "        for l in range(len(RA_subjects)): #iterate through subjects, if a subject is present in both files print subject data\n",
    "            if RA_candidates[k] in RA_subjects[l] and DEC_candidates[k] in DEC_subjects[l]:\n",
    "                if target_IDs is not None:\n",
    "                    print('present RA, DEC, subject_ID:',RA_candidates[k],DEC_candidates[k],target_IDs[k])\n",
    "                else:\n",
    "                    print('present RA, DEC:',RA_candidates[k],DEC_candidates[k])\n",
    "                cross_match +=1\n",
    "    print(cross_match,'total matching IDs')\n",
    "\n",
    "#use functions and change filenames\n",
    "RAsubjects, DECsubjects = FindSubjectCoords(r'Subjects.csv') # Zooniverse subjects file\n",
    "targetIDs = FindTargetIDs(r'CandidateIDs.csv') # list of the subject IDs to compare, can comment out if not needed\n",
    "RAcandidates, DECcandidates = FindCandidateCoords(r'TargetFile.csv', targetIDs) # csv of all target (RA, DEC, target type), can delete targetIDs if needed\n",
    "\n",
    "print(len(RAsubjects),'total subjects present')\n",
    "print(len(RAcandidates),'total candidates present')\n",
    "\n",
    "MatchRADEC(RAcandidates,DECcandidates, RAsubjects, DECsubjects, targetIDs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Subject Results\n",
    "\n",
    "A script to find subjects from Zooniverse classification file and target file which have movement above a specified threshold.\n",
    "\n",
    "Returns target type CSVs with movement ratio, RA, DEC, Zooniverse subject link, and WiseView link titled as 'typexresults.csv' where x is type.\n",
    "\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import re\n",
    "import csv\n",
    "import copy\n",
    "from alive_progress import alive_bar; import time\n",
    "\n",
    "#load in coordinates, classifications, and targets\n",
    "ClassificationsFile = r'Launch_Classifications6_29_23.csv'\n",
    "\n",
    "Types = ['1','4','8','16','32']#define values for type of target, here 2^(0,1,2,3,4,5) used\n",
    "acceptance_threshold = 4 # define minimum number of movement votes for a target to be recorded\n",
    "\n",
    "#Define dictionaries\n",
    "\n",
    "CoordinateMovement = {Types[0]:{'RA': [], 'DEC': [], 'zoolink': [], 'bywlink': [], 'movement': [], 'total': []},\n",
    "                      Types[1]:{'RA': [], 'DEC': [], 'zoolink': [], 'bywlink': [], 'movement': [], 'total': []},\n",
    "                      Types[2]:{'RA': [], 'DEC': [], 'zoolink': [], 'bywlink': [], 'movement': [], 'total': []},\n",
    "                      Types[3]:{'RA': [], 'DEC': [], 'zoolink': [], 'bywlink': [], 'movement': [], 'total': []},\n",
    "                      Types[4]:{'RA': [], 'DEC': [], 'zoolink': [], 'bywlink': [], 'movement': [], 'total': []}}\n",
    "\n",
    "def returnfloat(string):\n",
    "    number = []\n",
    "    for i in (string):\n",
    "        if i.isdigit() or i == '.' or i == '-':\n",
    "            number.append(i)\n",
    "    return float(''.join(number))\n",
    "\n",
    "#separate data by type and remove bad IDs\n",
    "\n",
    "bad_ID = [] # these IDs should not be counted, as the targets are not verified\n",
    "ClassificationText = open(ClassificationsFile, 'r')\n",
    "\n",
    "ClassificationCounter = -1\n",
    "#get all data for each type and separate them in dictionaries\n",
    "for line in ClassificationText:\n",
    "    ClassificationCounter +=1\n",
    "count=-1\n",
    "ClassificationText = open(ClassificationsFile, 'r')\n",
    "with alive_bar(ClassificationCounter) as bar:\n",
    "    for line in ClassificationText:\n",
    "        bar()\n",
    "        count += 1\n",
    "        if count == 0:\n",
    "            continue\n",
    "        itemlist = re.split(',',line)\n",
    "        for item in itemlist:\n",
    "            if 'RA\"\":' in item:\n",
    "                RA = returnfloat(item)\n",
    "            if 'DEC\"\":' in item:\n",
    "                DEC = returnfloat(item)\n",
    "            if '#BITMASK\"\":' in item:\n",
    "                Type = str(int(returnfloat(item)))\n",
    "            if '\"\"ID\"\":\"\"' in item:\n",
    "                ID = str(int(returnfloat(item)))\n",
    "                if ID in bad_ID:\n",
    "                    continue\n",
    "        if str(RA) in CoordinateMovement[Type]['RA'] and str(DEC) in CoordinateMovement[Type]['DEC']:\n",
    "            for i in range(len(CoordinateMovement[Type]['RA'])):\n",
    "                if RA == CoordinateMovement[Type]['RA']:\n",
    "                    RA_index = i\n",
    "        else:\n",
    "            bywlink = ('http://byw.tools/wiseview#ra='+str(RA)+'&dec='+str(DEC)+'&size=176&band=3&speed=20&minbright=-50.0000&maxbright=500.0000&window=0.5&diff_window=1&linear=1&color=&zoom=9&border=0&gaia=1&invert=1&maxdyr=0&scandir=0&neowise=0&diff=0&outer_epochs=0&unique_window=1&smooth_scan=0&shift=0&pmra=0&pmdec=0&synth_a=0&synth_a_sub=0&synth_a_ra=&synth_a_dec=&synth_a_w1=&synth_a_w2=&synth_a_pmra=0&synth_a_pmdec=0&synth_a_mjd=&synth_b=0&synth_b_sub=0&synth_b_ra=&synth_b_dec=&synth_b_w1=&synth_b_w2=&synth_b_pmra=0&synth_b_pmdec=0&synth_b_mjd=')\n",
    "            zoolink = ('https://www.zooniverse.org/projects/coolneighbors/backyard-worlds-cool-neighbors/talk/subjects/'+str(ID))\n",
    "            CoordinateMovement[Type]['RA'].append(str(RA))\n",
    "            CoordinateMovement[Type]['DEC'].append(str(DEC))\n",
    "            CoordinateMovement[Type]['bywlink'].append(bywlink)\n",
    "            CoordinateMovement[Type]['zoolink'].append(zoolink)\n",
    "            CoordinateMovement[Type]['movement'].append(0)\n",
    "            CoordinateMovement[Type]['total'].append(0)\n",
    "            RA_index = -1\n",
    "        if '\"Yes\"' in line:\n",
    "            CoordinateMovement[Type]['movement'][RA_index] += 1\n",
    "            CoordinateMovement[Type]['total'][RA_index] += 1\n",
    "        if '\"No\"' in line:\n",
    "            CoordinateMovement[Type]['total'][RA_index] += 1\n",
    "        \n",
    "for i in Types:\n",
    "    with open(r'type'+str(i)+'results.csv','w', newline = '') as file:\n",
    "        writer = csv.writer(file)\n",
    "        for k in range(len(CoordinateMovement[i]['bywlink'])):\n",
    "            if CoordinateMovement[i]['total'][k] != 0:\n",
    "                if CoordinateMovement[i]['movement'][k] >= acceptance_threshold:\n",
    "                    writer.writerow([(CoordinateMovement[i]['movement'][k])/(CoordinateMovement[i]['total'][k]),CoordinateMovement[i]['RA'][k],CoordinateMovement[i]['DEC'][k],CoordinateMovement[Type]['zoolink'][k],CoordinateMovement[i]['bywlink'][k]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Type Evaluation\n",
    "\n",
    "This script analyzes total movement ratio of subjects separated by type. Useful for larger reviews, not for individual classification.\n",
    "\n",
    "Can also specify specific subject IDs that should not be counted in results\n",
    "\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import math\n",
    "import re\n",
    "import csv\n",
    "from alive_progress import alive_bar; import time\n",
    "\n",
    "Types = ['1','4','8','16','32'] # enter type values here\n",
    "\n",
    "#load in file\n",
    "ClassificationFile = r'Classifications_File'\n",
    "\n",
    "ClassificationText = open(ClassificationFile, \"r\")\n",
    "\n",
    "\n",
    "typeRA = {Types[0]:[],Types[1]:[],Types[2]:[],Types[3]:[],Types[4]:[]}\n",
    "typeDEC = {Types[0]:[],Types[1]:[],Types[2]:[],Types[3]:[],Types[4]:[]}\n",
    "typedata = {Types[0]:[],Types[1]:[],Types[2]:[],Types[3]:[],Types[4]:[]}\n",
    "\n",
    "#generate searchable patterns for each target by type\n",
    "patterntype = {Types[0]:'#BITMASK\"\":\"\"'+Types[0]+'\"',Types[1]:'\"#BITMASK\"\":\"\"'+Types[1]+'\"',Types[2]:'\"#BITMASK\"\":\"\"'+Types[2]+'\"',Types[3]:'\"#BITMASK\"\":\"\"'+Types[3]+'\"',Types[4]:'\"#BITMASK\"\":\"\"'+Types[4]+'\"'}\n",
    "\n",
    "#read in data and search for specific type\n",
    "\n",
    "ClassificationCounter = -1\n",
    "move_count = {Types[0]:0,Types[1]:0,Types[2]:0,Types[3]:0,Types[4]:0}\n",
    "nomove_count = {Types[0]:0,Types[1]:0,Types[2]:0,Types[3]:0,Types[4]:0}\n",
    "\n",
    "for line in ClassificationText:\n",
    "    ClassificationCounter +=1\n",
    "with alive_bar(ClassificationCounter*len(Types)) as bar:\n",
    "    for i in Types:\n",
    "        ClassificationText = open(ClassificationFile,'r')\n",
    "        for line in ClassificationText:\n",
    "            time.sleep(.00001)\n",
    "            bar()\n",
    "            if re.search(patterntype[i], line):\n",
    "                if \"Yes\" in line:\n",
    "                    move_count[i] +=1\n",
    "                elif \"No\" in line:\n",
    "                    nomove_count[i] +=1\n",
    "        ClassificationText.close()\n",
    "        print('there were',move_count[i],'movement decisions and',nomove_count[i],'nonmovement decisions on type',i,'targets, giving a movement ratio of', (move_count[i])/(move_count[i]+nomove_count[i])*100, '%')\n",
    "\n",
    "print('Classifications Read')\n",
    "\n",
    "print('there are ',move_count[Types[0]]+move_count[Types[1]]+move_count[Types[2]]+move_count[Types[3]]+move_count[Types[4]]+nomove_count[Types[0]]+nomove_count[Types[1]]+nomove_count[Types[2]]+nomove_count[Types[3]]+nomove_count[Types[4]],'classifications counted and',ClassificationCounter,'total classifications.')\n"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}