processCCD_image.py

import matplotlib.pyplot as plt
import numpy as np
import scipy as sp
from scipy import stats
from scipy.optimize import curve_fit
import scipy.interpolate as intp
import robust
import zipfile
import os, pyfits

class CCD:
    """CCD RIXS image with associated processing and plotting

    Data objects:
    -----------
            CCD.fname_list      List of file names to be read
            CCD.fname_BG        String path to background image
            CCD.exclude         2 element list. Only use columns >exclude[0] & <exclude[1] 
            CCD.raw_images      List of 2D arrays with raw CCD images
            CCD.raw_BGimages    List of 2D arrays with background CCD images
            CCD.images          List of 2D arrays with processed CCD images
            CCD.BGimages        List of 2D arrays with processed CCD BGimages
            CCD.specs           List of 1D RIXS spectra Each spectrum is [x, y] list of numpy arrays
            CCD.BGspecs         List of 1D RIXS spectra ----------------------"---------------------
            CCD.raw_spectrum    Sum of all specs [x [in pixels], y, e] list of numpy arrays
            CCD.spectrum        Sum of all specs [x [in energy], y, e] list of numpy arrays
            CCD.background      Sum of all BGspecs [x [in pixels], y, e] list of numpy arrays
            CCD.correlations    List of correlation functions
 
            CCD.photon_E        Energy of incident photons. Used for electrons -> photon intensity conversion
            CCD.curvature       polynominal describing curvature
            CCD.poly_order      Order of polynominal describing Curvature
                                order = 3 imples x**0 + x**1 + x**2
            CCD.binpix          Number of pixels to bin together in curvature determination
            
            info_               prefix info denotes stuff related to curvature determination 
            CCD.info_x          Centers of bins for cuvature determination in pixel units
            CCD.info_peaks      Peaks in correlation functions used to get curvature
            CCD.info_corr       correlation function matrix
            CCD.info_shifted    CCD image after shifting by curvature
            CCD.info_ref        The reference spectrum generated by the left hand edge
                                of the image
                                
            CCD.shifts          Numbers of pixels the spectra were shifted in CCD.correlate() 
                                proceedure
            
            CCD.file_header     The raw text extracted out of the image file
            CCD.file_dictionary Python dictionary created from CCD.file_header
                                
    Processes:
    -----------            
            CCD.clean()         reject cosmic rays above threshold
            CCD.clean_std()     reject cosmic rays depending on deviation 
                                for mean values in curvature-corrected row trajectory
            CCD.sub_backgrounds() subtract BGimages from images
                                At this point the signal is converted from electrons to
                                photons
            CCD.plot()          plot CCD image
            CCD.get_curvature() determine curvature
            CCD.get_specs()     bin CCD images into 1D using current curvature
            CCD.get_BGspecs()   bin CCD BGimages into 1D using current curvature
            CCD.correlate()     Shift the pixel/E values overlapp all subsequent 
                                spectra with the first spectrum
            CCD.sum_specs()     sum 1D specs together into CCD.spectrum ...
                                and sum 1D BGspecs togetehr into CCD.specground
            CCD.bin_points()    Bin points together
            
    Internal Processes:
    -------------------
            CCD._gendark()      Generate the dark image when none is specified.
                                This is computed from the average of the edge of the image.
            CCD._clean_std()    get image array with cosmic rays removed 
                                based on deviation from mean
            CCD._get_header()   Reads the header out of the CCD file and writes
                                it into CCD.file_header & CCD.file_dictionary.
                                This is designed for SLS data. It probably needs 
                                to be changed to other facilities.
            CCD._verbose        
    """ 

    def __init__(self, fname_list=[], photon_E=930., poly_order=3, binpix=8, 
                 fileout='test.dat', fname_list_BG =[], exclude=[None, None],
                 verbose=True):
        """ Initiate class
        loading raw data from list of strings describing filesnames. 
        Various values (defined above) are initiated to
        defaults unless otherwise specified
        """
        self.binpix = binpix
        self.poly_order = poly_order
        self.photon_E = photon_E
        self.fname_list = fname_list
        self.fname_BG = fname_list_BG
        self.exclude = exclude
        self._verbose = verbose
        self.shifts = []
        self.fileout = fileout
        
        self.raw_images = []
        for fname in fname_list:
            if self._verbose:
                print "Loading {}".format(fname)
            if fname[-4:] == 'fits':
                fitsobj = pyfits.open(fname)
                self.raw_images.append(fitsobj[2].data)
            else:
                try:
                    self.raw_images.append(plt.imread(fname + '.tif'))
                    self._get_header(fname + '.tif')
                except IOError:
                    try:
                        z = zipfile.ZipFile(fname + '.zip')
                        z.extract(z.namelist()[0], path=os.path.dirname(fname))
                        self.raw_images.append(plt.imread(fname + '.tif'))
                        self._get_header(fname + '.tif')
                        zout = zipfile.ZipFile(fname + '.zip', "w")
                        zout.close()
                    except IOError:
                        print "Didn't find {}!".format(fname)
                        raise IOError

        if len(fname_list_BG) == 1:
            fname_list_BG = fname_list_BG * len(fname_list)
        self.raw_BGimages = []
        for fname_BG in fname_list_BG:
            if self._verbose:
                print "Loading background {}".format(fname_BG)
            if fname_BG[-4:] == 'fits':
                fitsobj = pyfits.open(fname_BG)
                self.raw_BGimages.append(fitsobj[2].data)
            else:
                try:
                    self.raw_BGimages.append(plt.imread(fname_BG + '.tif'))
                except IOError:
                    try:
                        z = zipfile.ZipFile(fname_BG + '.zip')
                        z.extract(z.namelist()[0], path=os.path.dirname(fname_BG))
                        self.raw_BGimages.append(plt.imread(fname_BG + '.tif'))
                        zout = zipfile.ZipFile(fname_BG + '.zip', "w")
                        zout.close()
                    except IOError:
                        print "Didn't find background {}!".format(fname_BG)
                        raise IOError
       
        if self.raw_BGimages == []:
            if self._verbose:
                    print "Generating the dark images, as no file was specified"
            for M in self.raw_images:
                self.raw_BGimages.append(self._gen_dark(M))
        
################ PROCESSES FOR DATA ##################
   
    def clean(self, thHIGH):
        """ Remove background.
        values above thHIGH are set to the mean value of the image
        """
        
        self.images = []
        for image in self.raw_images:
            clean_image = np.copy(image)
            meanimage = np.mean(image[image < thHIGH])
            clean_image[clean_image > thHIGH] = meanimage
            self.images.append(clean_image)
            
            changed_pixels = np.sum(image != clean_image) / ( len(image.flat)  + 0.0) # force float
            if self._verbose:
                print "Image: {:.2f} % of pixels rejected".format(changed_pixels*100)
     
        self.BGimages = []
        for BGimage in self.raw_BGimages:
            clean_BGimage = np.copy(BGimage)
            meanBGimage = np.mean(BGimage[BGimage < thHIGH])
            clean_BGimage[clean_BGimage > thHIGH] = meanBGimage
            self.BGimages.append(clean_BGimage)

            changed_pixels = np.sum(BGimage != clean_BGimage) / ( len(BGimage.flat)  + 0.0) # force float
            if self._verbose:
                print "Background: {:.1f} % of pixels rejected".format(changed_pixels*100)

    def clean_old(self, thHIGH):
        """ Remove background and convert from electrons to photons 
        gainregion  = indlow:indhigh defining energy gain rows on array
        e.g. 0:500        
        and set values above thHIGH to 0
        Images are still in units of electrons
        """
        
        self.images = []
        for image in self.raw_images:             
            clean_image = np.copy(image)
            meanimage = np.mean(image[image < thHIGH])
            clean_image[clean_image > thHIGH] = meanimage
            self.images.append(clean_image)
            
            changed_pixels = np.sum(image != clean_image) / ( len(image.flat)  + 0.0) # force float
            if self._verbose:
                print "Image: {:.1f} % of pixels rejected".format(changed_pixels*100)
     
        self.BGimages = []
        for BGimage in self.raw_BGimages: 
            clean_BGimage = np.copy(BGimage)
            meanBGimage = np.mean(BGimage[BGimage < thHIGH])
            clean_BGimage[clean_BGimage > thHIGH] = meanBGimage
            self.BGimages.append(clean_BGimage)

            changed_pixels = np.sum(BGimage != clean_BGimage) / ( len(BGimage.flat)  + 0.0) # force float
            if self._verbose:
                print "Background: {:.1f} % of pixels rejected".format(changed_pixels*100)                      
    
    def clean_std_new(self, noise):
        """
        Removes background based on the average intensity at each energy loss
        and its standard deviation. This process is iterated len(noise) times
        as the presence of a large count in a pixel screws up the average and
        standart deviation. Images are still in units of electrons.
        """
        self.images = []
        for image in self.raw_images:
            clean_image = self._clean_std_new(image, noise)
           
            self.images.append(clean_image)
            
            changed_pixels = np.sum(image != clean_image) / ( len(image.flat)  + 0.0) # force float
            if self._verbose:
                print "Image: {:.1f} % of pixels rejected".format(changed_pixels*100)

        
        self.BGimages = []
        for BGimage in self.raw_BGimages:
            clean_BGimage = self._clean_std_new(BGimage, noise)

            self.BGimages.append(clean_BGimage)
            
            changed_pixels = np.sum(BGimage != clean_BGimage) / ( len(BGimage.flat)  + 0.0) # force float
            if self._verbose:
                print "Background: {:.1f} % of pixels rejected".format(changed_pixels*100)    
    
    def clean_std(self, noise):
        """
        Removes background based on the average intensity at each energy loss
        and its standard deviation. Images are still in units of electrons at the point.
        """
        self.images = []
        for image in self.raw_images:
            clean_image = self._clean_std(image, noise)
            e_per_ph = self.photon_E /  3.65
            clean_image = clean_image / e_per_ph;
            self.images.append(clean_image)
            
            changed_pixels = np.sum(image != clean_image) / ( len(image.flat)  + 0.0) # force float
            if self._verbose:
                print "Image: {:.1f} % of pixels rejected".format(changed_pixels*100)

        
        self.BGimages = []
        for BGimage in self.raw_BGimages:
            clean_BGimage = self._clean_std(BGimage, noise)
            self.BGimages.append(clean_BGimage)
            
            changed_pixels = np.sum(BGimage != clean_BGimage) / ( len(BGimage.flat)  + 0.0) # force float
            if self._verbose:
                print "Background: {:.1f} % of pixels rejected".format(changed_pixels*100)

    
    def sub_backgrounds(self):
        """ Subtract backgrounds and convert from electrons to photons"""
        for i in range(len(self.images)):
            e_per_ph = self.photon_E /  3.65
            self.images[i] = (self.images[i] - self.BGimages[i])/e_per_ph

    def get_curvature(self, index=0):
        """ Determine the curvature of the CCD image specified by index
        The data are binned in columns self.binpix pixels wide
        a correlation function is calculated using the central set of pixels
        as a reference. The peaks of each binned column is then fit by a polynominal """
        M = self.images[index]
        #M = M - np.mean(M[0:50, :]) # need to have zeros at the top and bottom 
        #                              of the image 
        x = self.binpix/2 + self.binpix * np.arange(np.floor(np.shape(M)[1]/self.binpix))
        keep = np.all((x> self.exclude[0]+self.binpix/2 , x < self.exclude[1]-self.binpix/2), axis=0)
        x = x[keep]
        
        M_corr = np.zeros((np.shape(M)[0], np.shape(M)[1]/self.binpix))
        peaks = []
        
        cenpix = 0 
        ref = np.sum(M[:,(x[cenpix]-self.binpix):(x[cenpix]+self.binpix)], axis=1)
        for i in range(0, int(np.shape(M)[1]/self.binpix)):
            indices = i*self.binpix + np.array(range(self.binpix))
            curr = np.sum(M[:,indices], axis=1)
            ycorr = np.correlate(curr, ref, mode='same')
            M_corr[:,i] = ycorr
            peaks.append(np.argmax(ycorr))
        
        peaks = np.array(peaks) + 0.0
        # exclude values        
        peaks = peaks[keep]
        
        self.curvature = robust.polyfit(x, peaks, self.poly_order, iterMax=25)
        self.info_x = x
        self.info_peaks = peaks
        self.info_corr = M_corr
        self.info_ref = ref
        
    def offset_curvature(self, offset='Find Peak'):
        """ The constant in the curvature defined in self.curvature and self.curvature_info
        is abitrary.  This offsets it for convenient plotting. If no argument is given the offset is set to the
        peak in the spectrum"""
        if offset=='Find Peak':
            currentOffset = np.polyval(self.curvature, self.info_x[0])
            currentPeak = np.argmax(self.info_ref)
            offset  = currentPeak - currentOffset
        self.curvature[-1:] += offset
        self.info_peaks += offset
        if self.verbose:
            print "Curvature was offst by {:.2f}".format(offset)
           
###################### BINNING AND SUMMING #################

    def get_specs(self):
        """ Extract the spectra using the predefined curvature"""
        # Generate specs from images
        y = np.arange(np.shape(self.images[0])[0])  # a column
        p = np.hstack((self.curvature[:-1], 0))
                
        self.specs = []
        M_shifted = np.zeros(np.shape(self.images[0]))  # will be filled with image with columns shifted to cancel the curvature
        for image in self.images:
            for col in range(np.shape(image)[1]):
                M_shifted[:, col] = np.interp(y, y - np.polyval(p, col), image[:, col],
                                           left=np.NaN, right=np.NaN)
            I = np.sum(M_shifted[:,self.exclude[0]:self.exclude[1]], axis=1)
            inds = ~np.logical_or(np.isnan(I), np.isnan(I))
            self.specs.append([y.transpose()[inds], I.transpose()[inds]])
  
        self.info_shifted = M_shifted
    
        # Generate BGspecs from background images
        x = np.arange(np.shape(self.BGimages[0])[0])
        p = np.hstack((self.curvature[:-1], 0))
        
        self.BGspecs = []
        M_shifted = np.zeros(np.shape(self.BGimages[0]))  # will be filled with image with columns shifted to cancel the curvature
        for BGimage in self.BGimages:
            for col in range(np.shape(BGimage)[1]):
                M_shifted[:, col] = np.interp(x, x - np.polyval(p, col), BGimage[:, col],
                                           left=np.NaN, right=np.NaN)
            y = np.sum(M_shifted[:,self.exclude[0]:self.exclude[1]], axis=1)
            inds = ~np.logical_or(np.isnan(x), np.isnan(y))
            self.BGspecs.append([x.transpose()[inds], y.transpose()[inds]])
            
    def get_specs_error(self):
        """ Extract the single spectra using the predefined curvature. Calculate error based
        on the standard variation across the image"""
        # Generate specs from images
        y = np.arange(np.shape(self.images[0])[0])  # a column
        p = np.hstack((self.curvature[:-1], 0))
                
        self.specs = []
        M_shifted = np.zeros(np.shape(self.images[0]))  # will be filled with image with columns shifted to cancel the curvature
        for image in self.images:
            for col in range(np.shape(image)[1]):
                M_shifted[:, col] = np.interp(y, y - np.polyval(p, col), image[:, col],
                                           left=np.NaN, right=np.NaN)
            I = np.sum(M_shifted[:,self.exclude[0]:self.exclude[1]], axis=1)
            
            if (np.abs(self.exclude[0]-self.exclude[1])+1) < len(M_shifted[0,:]):
                num_points = np.abs(self.exclude[0]-self.exclude[1])+1
            else:
                num_points = len(M_shifted[0,:])
            
            std = np.sqrt(num_points)*np.std(M_shifted[:,self.exclude[0]:self.exclude[1]], axis=1)

            inds = ~np.logical_or(np.isnan(I), np.isnan(I))
            self.specs.append([y.transpose()[inds], I.transpose()[inds], std.transpose()[inds]])
  
        self.info_shifted = M_shifted
    
        # Generate BGspecs from background images
        x = np.arange(np.shape(self.BGimages[0])[0])
        p = np.hstack((self.curvature[:-1], 0))
        
        self.BGspecs = []
        M_shifted = np.zeros(np.shape(self.BGimages[0]))  # will be filled with image with columns shifted to cancel the curvature
        for BGimage in self.BGimages:
            for col in range(np.shape(BGimage)[1]):
                M_shifted[:, col] = np.interp(x, x - np.polyval(p, col), BGimage[:, col],
                                           left=np.NaN, right=np.NaN)
            y = np.sum(M_shifted[:,self.exclude[0]:self.exclude[1]], axis=1)
            inds = ~np.logical_or(np.isnan(x), np.isnan(y))
            self.BGspecs.append([x.transpose()[inds], y.transpose()[inds]])


    def correlate_specs(self):
        """ determine x shift referenced to the first spectrum
        x values are shifted by this value"""
        self.shifts.append(0)
        dx = 0.1
        xfine = np.arange(np.min(self.specs[0][0]), np.max(self.specs[0][0]), dx) # 10 times oversampling
        ref = sp.interp(xfine, self.specs[0][0], self.specs[0][1])
        self.correlations = []
        self.correlations.append(sp.correlate(ref, 
                                            ref, mode='full'))
        for i in range(1, len(self.specs)):
            currfine = sp.interp(xfine, self.specs[i][0], self.specs[i][1])
            self.correlations.append(sp.correlate(ref, currfine,  mode='full'))
        for i in range(1, len(self.specs)):
            shift = np.argmax(self.correlations[i]) - np.argmax(self.correlations[0])
            shift = shift * dx           
            self.specs[i][0] = self.specs[i][0] + shift
            self.shifts.append(shift)
            if self._verbose:
                print "spectrum {:d} shifted by {:.2f}".format(i, shift)


    def sum_specs(self):
        """ Sum the specs into one spectrum. And the bgspecs into background
        Calculating error from standard deviation"""
        # specs
        x = self.specs[0][0]
        if len(self.specs) == 1:
            y = self.specs[0][1]
            e = y*0.0
        else:
            YY = self.specs[0][1]
            for i in range(1, len(self.specs)):
                funcy = intp.interp1d(self.specs[i][0], self.specs[i][1], kind='linear', 
                            bounds_error=False, fill_value=np.NaN)
                YY = np.column_stack((YY, funcy(x)))
            y = np.sum(YY, axis=1)
            e = np.std(YY, axis=1)
        
        inds = ~np.logical_or(np.isnan(x), np.isnan(y), np.isnan(e)),        
        self.spectrum = [x[inds], y[inds], e[inds]]
        
        # BGspecs
        x = self.BGspecs[0][0]
        if len(self.BGspecs) == 1:
            y = self.BGspecs[0][1]
            e = y*0.0
        else:
            YY = self.BGspecs[0][1]
            for i in range(1, len(self.BGspecs)):
                funcy = intp.interp1d(self.BGspecs[i][0], self.BGspecs[i][1], kind='linear', 
                            bounds_error=False, fill_value=np.NaN)
                YY = np.column_stack((YY, funcy(x)))
            y = np.sum(YY, axis=1)
            e = np.std(YY, axis=1)
        
        inds = ~np.logical_or(np.isnan(x), np.isnan(y), np.isnan(e)),        
        self.background = [x[inds], y[inds], e[inds]]

    def bin_points(self, dx, statistic = 'mean'):
        """ Bin points together in intervals of dx
        """
        binedges = np.arange(-dx/2 + np.min(self.spectrum[0]),
                             np.max(self.spectrum[0]) +dx/2 , dx)
        xnew = (binedges[1:] + binedges[0:-1])/2
        ynew, _, _ = stats.binned_statistic(self.spectrum[0], self.spectrum[1],
                                            statistic=statistic, bins=binedges)
        def quadrature_func(x):
            return np.sqrt(np.sum(x**2)) / len(x)
        enew, _, _ = stats.binned_statistic(self.spectrum[0], self.spectrum[2], 
                                            statistic=quadrature_func, bins=binedges)
        self.spectrum = [xnew, ynew, enew]       

    def calibrate(self, elastic_pixel, E_per_pix):
        """ Convert the spectrum into energy by specifying the elastic pixel and
        energy perpixel"""
        self.spectrum[0] = (elastic_pixel - self.spectrum[0]) *  E_per_pix
        self.background[0] = (elastic_pixel - self.background[0]) *  E_per_pix

#    def calibrate_poly(self, elastic_pixel, p):
#        """ Convert the spectrum into energy by specifying the elastic pixel and
#        energy perpixel"""
#        self.spectrum[0] = (elastic_pixel - self.spectrum[0]) *  E_per_pix 
#        self.spectrum[0] = np.polyv

    def fit_elastic(self, cen=None, sigma=None, I0=None, offset=None, x_window=[-np.inf, np.inf]):
        """ 
        Fit gaussian to the spectrum in order to determine the pixel correponding to the elastic line.
        x_window is a list that cuts the spectrum between x_window[0] and x_window[1]        
        cen=, sigma=None, I0=None, offset=None
        """
        def gauss(x, cen, sigma, I0, offset):
            return I0 * np.exp(-(x-cen)**2/(2*sigma**2)) + offset

        keep_inds = (self.spectrum[0] > x_window[0]) & (self.spectrum[0] < x_window[1])
        x = self.spectrum[0][keep_inds]
        y = self.spectrum[1][keep_inds]
        
        if cen == None:
            try:
                index = np.argmax(y)
                cen = x[index]
            except ValueError:
                print 'Failed to initialize peak center!'
                return -100
        if sigma == None:
            sigma = (np.max(x) - np.min(x)) / 4
        if I0 == None:
            I0 = np.max(y)
        if offset == None:
            offset = np.min(y)
        
        #print "cen = {}, sigma = {}, I0 = {}, offset = {}".format(cen, sigma, I0, offset)
        try:        
            popt, pcov = curve_fit(gauss, x, y, p0=[cen, sigma, I0, offset])    
        except RuntimeError:
            print 'Fit failed!'
            return -100.0
            
        if self._verbose:
                print "Guess values: cen = {:.2f}, sigma = {:.2f}, I0 = {:.2f}, offset = {:.2f}".format(cen, sigma, I0, offset)
                print "Fit values: cen = {:.2f}, sigma = {:.2f}, I0 = {:.2f}, offset = {:.2f}".format(*popt)
        return popt[0]

    def shift_e(self, func = 'gauss', guess = -1, var = 9999):
        """ 
        Shifts the energy loss spectra to align the elastic peak to the first
        scan of self.specs. Outputs the position that all scans were aligned to
        be an input for self.calibrate.
        N.B. 
        """
        i = 0
        for spec in self.specs:        

            gmax = np.max(spec[1][:])
            data = np.copy(spec)
            if guess < 0:
                ind = int(spec[0][np.where(spec[1][:] == gmax)])
            else:
                ind = guess
                avg = np.average(data[1][0:int(guess-var)])

                for j in range (len(spec[1])):
                    if data[0][j] < (guess-var) or data[0][j] > (guess+var):
                        #data[1][j] = 0.0
                        data[1][j] = avg                      
                        
            cnte = spec[1][ind-100]
            
            
            #print gmax, ind, cnte
            
            if func == 'gauss':
                def gauss(x,a,x0,sigma):
                    return a*np.exp(-(x-x0)**2/(2*sigma**2)) 
                
                try:
                    popt,pcov = curve_fit(gauss,data[0],data[1],p0=[gmax,ind,1])
                    
                    #plt.figure()
                    #plt.plot(data[0], data[1], '.')  
                    #plt.plot(data[0], gauss(data[0], popt[0], popt[1], popt[2]))
                    #plt.show()
                    
                    if i == 0:
                        x_first = popt[1]
                    
                    if self._verbose:
                        print 'Center of spec #{:d}: Guess = {:0.2f}, Fit = {:0.2f}'.format(i+1, ind, popt[1])
                    
                except RuntimeError:
                    print 'Failed to fit spec #{:d}. Using guess = {:0.2f}'.format(i+1, ind)
                    popt[1] = ind
                    
                if np.abs(guess - popt[1]) > var:
                    print "Fitted value looks wrong. Using guess = {:0.2f}".format(ind)
                    popt[1] = ind
                    
                spec[0] = spec[0] + (x_first - popt[1])
                self.specs[i] = spec
                i = i+1
                
            if func == 'slit':
                def slit(x,a,x0,k,w,cnte):
                    y = np.empty((len(x)))
                    y[:] = 0.0
                    ind1 = np.abs(2*k*(x-x0+w/2)) > np.log(np.finfo(np.float64).max) #Looks for the maximum exponential that it can calculate!
                    ind2 = np.abs(2*k*(x-x0-w/2)) > np.log(np.finfo(np.float64).max)
                    index = -(ind1 + ind2)
                    y[index] = a*(1/(1+np.exp(-2*k*(x[index]-x0+w/2)))-1/(1+np.exp(-2*k*(x[index]-x0-w/2)))) + cnte
                    return y
                
                popt,pcov = curve_fit(slit,spec[0],spec[1],p0=[gmax,ind,1,10,cnte])
                
                if i == 0:
                    x_first = popt[1]
                    
                xlow = spec[0][0]
                xhigh = spec[0][len(spec[0])-1]
                    
                xnew = spec[0] + (x_first - popt[1])
                
                f = intp.interp1d(xnew, spec[1], kind='linear', 
                            bounds_error=False, fill_value=0.0)
                
                spec[1] = f(spec[0])
                
                #spec[0]= spec[0] + (x_first - popt[1])
                self.specs[i] = spec
                i = i+1
        
        return x_first
                

###################### OUTPUTTING DATA #####################

    def write_file(self):
        """Write text file""" 
        header = "Files\n"
        for fname in self.fname_list:
            header += fname + "\n"
        header += "Curvature " + str(self.curvature) + '\n'
        header += "Shifts" + str(self.shifts) + '\n'
        header += "Exclusion" + str(self.exclude) + '\n'
        header += "########################\n"
        header += "pixel \t phonons \t error \n"
        header += self.file_header + "\n"
        M = np.column_stack((self.spectrum[0], self.spectrum[1],
                             self.spectrum[2]))
        np.savetxt(self.fileout, M, header=header)

###################### PLOTTING FUNCTIONS #####################

    def plot_curvature(self):
        """ Plot the peaks in the correlation function and the fit defining the 
        curvature
        """
        plt.plot(self.info_x, self.info_peaks, 'b.')
        hoizPixels = np.shape(self.images[0])[1]
        rowofCCD = np.arange(hoizPixels)
        plt.plot(rowofCCD, np.polyval(self.curvature, rowofCCD), 'r-')
    
    def plot_image(self, index = 0, percentlow=1, percenthigh=99, **kwargs):
        """ Plot the specified CCD image, as chosen via index.
        **kwargs are passed to plt.imshow
        intensity limits are set as percentages unless vmin or vmax are passed as kwargs"""
        if ('vmin' or 'vmax') in kwargs.keys():
            plt.imshow(self.images[index], cmap=plt.cm.Greys_r, 
                       aspect='auto', interpolation='none', **kwargs)
        else:    
            plt.imshow(self.images[index], vmin=np.percentile(self.images[index],percentlow),
                   vmax=np.percentile(self.images[index], percenthigh),
                    cmap=plt.cm.Greys_r, aspect='auto', interpolation='none', **kwargs)
        plt.colorbar()
        
    def plot_raw_image(self, index = 0, percentlow=1, percenthigh=99, **kwargs):
        """ Plot the specified raw CCD image, as chosen via index.
        **kwargs are passed to plt.imshow
        intensity limits are set as percentages unless vmin or vmax are passed as kwargs"""
        if ('vmin' or 'vmax') in kwargs.keys():
            plt.imshow(self.raw_images[index], cmap=plt.cm.Greys_r, 
                       aspect='auto', interpolation='none', **kwargs)
        else:    
            plt.imshow(self.raw_images[index], vmin=np.percentile(self.raw_images[index],percentlow),
                   vmax=np.percentile(self.raw_images[index], percenthigh),
                    cmap=plt.cm.Greys_r, aspect='auto', interpolation='none', **kwargs)
        plt.colorbar()
    
    def plot_hist(self, index=0, bins=10, **kwargs):
        """ Plot histrogram of specified CCD image, as chosed via index
        bins = the number of bins to be used this function can be used to 
        examine the statistics of the spectrum. **kwargs are pass to plt.hist"""
        plt.hist(self.images[index].ravel(), bins=bins, log=True, **kwargs)
        plt.xlabel('Number of photons')
        plt.ylabel('Number of pixels')
        
    def plot_specs(self, index=[], **kwargs):
        """ Plot all spectra. **kwargs are passed to plt.plot"""
        if index==[]:
            for spec in self.specs:
                plt.plot(spec[0], spec[1], '.-', **kwargs)
        else:
            plt.plot(self.specs[index][0], self.specs[index][1], '.-', **kwargs)

    def plot_BGimages(self, index=[], offset = 0.0, **kwargs):
        """ Plot all spectra. **kwargs are passed to plt.plot"""
        if index==[]:
            for BGspec in self.BGspecs:
                plt.plot(BGspec[0], BGspec[1] + offset, '.-', **kwargs)
        else:
            plt.plot(self.BGspecs[index][0], self.BGspecs[index][1] + offset, '.-', **kwargs)

    def plot_spectrum(self, offset=0.0, **kwargs):
        """ Plot the summed spectrum using errorbar. **kwargs are passed to plt.plot"""
        
        plt.errorbar(self.spectrum[0], self.spectrum[1]+ offset,
                     self.spectrum[2], fmt='.-', **kwargs)

################ INTERNAL PROCESSES ##########
    def _gen_dark(self, M, start_row_index=200, end_row_index=500):
        """ Generate image for background subtraction without real dark image
        Data is taken between row start_row_index and end_row_index
        the 50% percentile is taken to minimize sensitivity to spikes
        """
        return M*0 + np.percentile(M[start_row_index:end_row_index,:], 50)
     
    def _clean_std(self, M, noise):
        """ Clean cosmic rays based on horizontal, curvature corrected, rows.
        For each row values < or > than mean -+ (noise*std) are rejected"""
        p = self.curvature[:-1] + [0.]
        y = y = np.arange(M.shape[0]) # a column 
        
        # Create curvature corrected array
        M_shifted = np.zeros(np.shape(M)) 
        for col in range(np.shape(M_shifted)[1]):
            shift = np.round(np.polyval(p, col))
            interpfunc = sp.interpolate.interp1d(y - shift, M[:, col], kind='nearest', bounds_error=False, fill_value=np.inf) # inf will be removed by threshold
            M_shifted[:, col] = interpfunc(y)
        
        # Apply thresholding 
        M_shifted_cleaned = np.zeros(np.shape(M))
        
        for row_ind in range(np.shape(M_shifted_cleaned)[0]):
            row = np.copy(M_shifted[row_ind, :])
            
            excluded_row = np.copy(row[self.exclude[0]:self.exclude[1]])
            mean = np.mean(excluded_row[np.isfinite(excluded_row)])
            std = np.std(excluded_row[np.isfinite(excluded_row)])
            
           # print " Improve handing of NaNs"
            indlow = row < (mean - noise*std)
            indhigh = row > (mean + noise*std)
           # print mean, std
            row[indlow] = mean
            row[indhigh] = mean
            row[np.isnan(row)] = mean
            M_shifted_cleaned[row_ind, :] = row
        
        # Undo curvature correction
        M_cleaned = np.zeros(np.shape(M)) 
        for col in range(np.shape(M_cleaned)[1]):
            shift = np.round(np.polyval(p, col))
            interpfunc = sp.interpolate.interp1d(y + shift, M_shifted_cleaned[:, col],
                                                 kind='nearest', bounds_error=False, fill_value=np.NaN)                           
            M_cleaned[:, col] = interpfunc(y)
        
        # Use original pixels where row is cut due to curvature
        inds = np.isnan(M_cleaned)
        M_cleaned[inds] = M[inds]
    
        changed_pixels = np.sum(M != M_cleaned) / ( len(M.flat)  + 0.0) # force float
        print "{0} % of pixels rejected".format(changed_pixels*100)
        
        return M_cleaned

    def _clean_std_new(self, M, noise):
        """ Clean cosmic rays based on horizontal, curvature corrected, rows.
        The cleaning iterates len(noise) times to account for the influence of
        the large cosmic rays count that inflates the mean. For each row values
        < or > than mean -+ (noise*std) are rejected"""
        
        p = self.curvature[:-1] + [0.]
        y = y = np.arange(M.shape[0]) # a column
        mean = 0.0
        
        # Create curvature corrected array
        M_shifted = np.zeros(np.shape(M)) 
        for col in range(np.shape(M_shifted)[1]):
            shift = np.round(np.polyval(p, col))
            interpfunc = sp.interpolate.interp1d(y - shift, M[:, col], kind='nearest', bounds_error=False, fill_value=np.inf) # inf will be removed by threshold
            M_shifted[:, col] = interpfunc(y)
        
        # Apply thresholding 
        M_shifted_cleaned = np.zeros(np.shape(M))
        for row_ind in range(np.shape(M_shifted_cleaned)[0]):
            row = np.copy(M_shifted[row_ind, :])
            
            for clean in noise:
                excluded_row = np.copy(row[self.exclude[0]:self.exclude[1]]) 
                
                #If all excluded_row are not finite, then this row is equal to
                #the previous mean. If it's the first row, then mean = 0.0
                if all(test == False for test in np.isfinite(excluded_row)) == True: 
                    row[:] = mean
                else:
                    mean = np.mean(excluded_row[np.isfinite(excluded_row)])
                    std = np.std(excluded_row[np.isfinite(excluded_row)])
                    indlow = row < (mean - clean*std)
                    indhigh = row > (mean + clean*std)
                    row[indlow] = mean
                    row[indhigh] = mean
                    row[np.isnan(row)] = mean
                
            M_shifted_cleaned[row_ind, :] = row
            
        # Undo curvature correction
        M_cleaned = np.zeros(np.shape(M)) 
        for col in range(np.shape(M_cleaned)[1]):
            shift = np.round(np.polyval(p, col))
            interpfunc = sp.interpolate.interp1d(y + shift, M_shifted_cleaned[:, col],
                                                 kind='nearest', bounds_error=False, fill_value=np.NaN)                           
            M_cleaned[:, col] = interpfunc(y)
        
        # Use original pixels where row is cut due to curvature
        inds = np.isnan(M_cleaned)
        M_cleaned[inds] = M[inds]
        
        return M_cleaned
        
    def _get_header(self, tiff_filename):
        """ Extract the header information and write into a text file
        self.file_header & and dictionary self.file_dictionary
        
        self.file_header gets written into the output file
        N.B. at SLS files seem to randomly change whether 
        """
        fid = open(tiff_filename)
        alltxt = fid.read()
        start = alltxt.find('[sample]')
        if start != -1:
            self.file_header = alltxt[start:].lower()
    
            self.file_dictionary = {}
            section_header = ''
            for line in self.file_header.splitlines():
                try:
                    variable, valuestr = line.split(' = ')
                    self.file_dictionary.update({section_header + variable.lower(): float(valuestr)})
                except ValueError:
                    section_header = line.replace('[','').replace(']','') + '_'
        else:
            if self._verbose:
                print "No ADRESS style header found."
            self.file_header = ''
            self.file_dictionary =''