# $Copyright:
# ----------------------------------------------------------------
# This confidential and proprietary software may be used only as
# authorised by a licensing agreement from ARM Limited
#  (C) COPYRIGHT 2013-2014 ARM Limited
#       ALL RIGHTS RESERVED
# The entire notice above must be reproduced on all authorised
# copies and copies may only be made to the extent permitted
# by a licensing agreement from ARM Limited.
# ----------------------------------------------------------------
# File:        latency.py
# ----------------------------------------------------------------
# $
#

#from __future__ import division
import os
import sys
import argparse
import time
import csv
import shutil
#import threading
#import errno
import tempfile
import msvcrt
import subprocess
import numpy as np

try:
    import pandas as pd
except ImportError:
    pd = None
    sys.stderr.write('ERROR: Pandas Python Package Required.\n')
    sys.exit(0)
	

"""
Generates latency data and statistics for and Android application

"""

VSYNC_INTERVAL = 16666667L
DEFAULT_JANK_THRESHOLD = 10000 #record all janks
DEFAULT_CADENCE = 2.00

def parse_arguments(args=None):
    if args is None:
        args = sys.argv[1:]

    parser = argparse.ArgumentParser(description=
                                     "Captures SurfaceFlinger data from a device & generates two CSV data files " +
                                     "containing FPS & Jank Statistics, Time Based Data & Raw Data.")
    parser.add_argument('filename', metavar='FILENAME', help='Output File Name')
    parser.add_argument('--c', dest='cadence', type=float, default=DEFAULT_CADENCE, help='Cadence of data grabber in seconds')
    parser.add_argument('--j', dest='jank_threshold', type=int, default=DEFAULT_JANK_THRESHOLD, help='Threshold above which to ignore Janks - default = No Janks Ignored')
    
    args = parser.parse_args(args)
    if not args.filename:
        sys.stderr.write('ERROR: must specify a File Name.\n')
        sys.exit(0)
    else:
        return args

def RecordContexts(out_file, cadence, keep, ignore):
    
    sf_type = 1 #default SurfaceFlinger type is 1

    #First capture idle processes to ignore in final analysis
    try:
        fd, tmp_file = tempfile.mkstemp()
        
        print "Identifying Idle Processes...\n"
        proc = subprocess.Popen("adb shell dumpsys SurfaceFlinger --latency", stdout=subprocess.PIPE)
        wfh = os.fdopen(fd, 'wb')
        try:
            wfh.write(proc.communicate()[0])
        finally:
            wfh.close()
        
        #This removes all data except context names
        with open(tmp_file) as fh:
            text = fh.read().replace('\r\n', '\n').replace('\r', '\n')
            for line in text.split('\n'):
                line = line.strip()
                if line:    
                    items = line.split('\t')     
                    if len(items) != 3 and items[0].isdigit() == False:
                        ignore.append(line)
        os.unlink(tmp_file)
    except Exception, e:
        print e.message, e.args
        os.unlink(tmp_file)
        return

    #check if type 2 SurfaceFlinger use different method
    if len(ignore) == 0:
        sf_type = 2
        proc = subprocess.Popen("adb shell dumpsys SurfaceFlinger --list", stdout=subprocess.PIPE)
        text = proc.communicate()[0].replace('\r\n', '\n').replace('\r', '\n')
        for line in text.split('\n'):
            if line:    
                if line not in ignore:
                    ignore.append(line)

    print "Ignoring Contexts:"
    for line in ignore:
        print line
    
    #Now capture application data
    try:
        fd, tmp_file = tempfile.mkstemp()
        
        try:
            print "\nStart your App\nPress any key after App is closed"
            wfh = os.fdopen(fd, 'wb')
            cadence = cadence - 0.1 #remove loop time so more accurate
            if sf_type == 1:
                while msvcrt.kbhit() != True:
                    proc = subprocess.Popen("adb shell dumpsys SurfaceFlinger --latency", stdout=subprocess.PIPE)
                    wfh.write(proc.communicate()[0])
                    time.sleep(cadence)
            else:
                while msvcrt.kbhit() != True:
                    proc2 = subprocess.Popen("adb shell dumpsys SurfaceFlinger --list", stdout=subprocess.PIPE)
                    text = proc2.communicate()[0].replace('\r\n', '\n').replace('\r', '\n')
                    for line in text.split('\n'):
                        if line:    
                            if line not in keep and line not in ignore:
                                keep.append(line)
                    for line in keep:
                        proc = subprocess.Popen("adb shell dumpsys SurfaceFlinger --latency " + line, stdout=subprocess.PIPE)
                        wfh.write(line + '\r\n')                        
                        wfh.write(proc.communicate()[0])
                    time.sleep(cadence - (0.1 * len(keep)))
                    del keep[:]
                
        finally:
            wfh.close()

        #Determine which contexts to keep
        with open(tmp_file) as fh:
            text = fh.read().replace('\r\n', '\n').replace('\r', '\n')
            for line in text.split('\n'):
                line = line.strip()
                if line:    
                    items = line.split('\t')     
                    if len(items) != 3 and items[0].isdigit() == False:
                        if items[0] not in ignore and items[0] not in keep:
                            keep.append(line)

        #Copy sanitised data from temporary file to output file
        o_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), out_file + '-raw.txt')
        if os.path.isdir(os.path.dirname(o_file)) == False:
            os.mkdir(os.path.dirname(o_file))
        shutil.copy(tmp_file, o_file)
        os.unlink(tmp_file)
        
    except Exception, e:
        print e.message, e.args
        os.unlink(tmp_file)
        return    

def AnalyzeContexts(in_file, jank_threshold, keep, ignore):
    
    print "\nAnalyzing Data for:"
    for line in keep:
        print line
    print "Please Wait..."
    
    #Separate data for contexts we are keeping
    try:
        surface_flinger_raw = []
        current_context = None
        
        for c in keep:
            surface_flinger_raw.append(list())
        
        i_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), in_file + '-raw.txt')
        with open(i_file) as fh:
            text = fh.read()
            for line in text.split('\n'):
                line = line.strip()
                #Do we have some data
                if line:
                    parts = line.split('\t')
                    #time data found
                    if len(parts) == 3 and current_context != None:
                        if parts[0] != '0':
                            surface_flinger_raw[keep.index(current_context)].append(tuple(parts))
                    #Context or VSYNC data found
                    elif len(parts) == 1:
                        #Context found
                        if parts[0].isdigit() == False:
                            if parts[0] in keep:
                                current_context = parts[0]
                            else:
                                current_context = None
                        else:
                            continue
                    #Some bad data
                    else:
                        continue
      
    except Exception, e:
        print e.message, e.args
        return

    #merge into single container and remove duplicates
    surface_flinger = []
    i=0
    for c in surface_flinger_raw:
        context = keep[i]
        for d in c:
            desired, actual, ready = map(long, d)
            surface_flinger.append((desired, actual, ready, context))
        i+=1
    surface_flinger.sort()
    
    #create full raw data set
    df = pd.DataFrame(surface_flinger, columns=['desired_present_time','actual_present_time','frame_ready_time','Context'])
    df.sort(['actual_present_time'], inplace=True)
    df.drop_duplicates(cols='actual_present_time', inplace=True)
    df.drop_duplicates(cols='desired_present_time', inplace=True) #Android 4.4 (or just Nexus7) has a bug
    df.index = range(0,len(df))
    df.insert(3, 'relative_time', (df.actual_present_time.astype(float) - df.actual_present_time[0].astype(float)) / 1000000000.0)
    df.relative_time.iloc[0] = 0.0 #assume zero time
    df.insert(4, 'present_duration', df.actual_present_time - df.actual_present_time.shift(1))
    df.present_duration[0] = VSYNC_INTERVAL #assume it took 1 vsync
    df.present_duration = df.present_duration.astype(long)
    df.insert(5, 'vsyncs_raw', (df.present_duration.astype(float)/VSYNC_INTERVAL).apply(lambda x: round(x, 2)))
    df.insert(6, 'vsyncs', df.vsyncs_raw.apply(lambda x: round(x, 0)))
    df.vsyncs = df.vsyncs.astype(long)
    df.insert(7, 'fps', 60 / df.vsyncs)
    df.insert(8, 'janks_raw', abs(df.vsyncs - df.vsyncs.shift(1)))
    df.janks_raw[0] = 0
    df.janks_raw = df.janks_raw.astype(long)

    #ignore janks above threshold
    def ValidJank(j):
        if j > 0 and j <= jank_threshold:
            return 1
        else:
            return 0
    df.insert(9, 'janks', (df.janks_raw.apply(ValidJank)))
    
    #write raw values to file - overwrites original FILENAME-raw.txt
    o_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), in_file + '-raw.txt')
    with open(o_file, 'w') as wfh:
        writer = csv.writer(wfh)
        writer.writerow(list(df.columns.values))
        writer.writerows(df.values)

    #generate time based (frames & janks per second) data
    dfs = pd.DataFrame(np.zeros(shape=(abs(df.relative_time.iloc[-1])+2,3)), columns=['Time(s)','FPS','JPS'], dtype=int)
    dfs['Time(s)'] = range(len(dfs.index))
    t = 1
    r = 1
    j = 0
    for row in df.relative_time:
        jank = df.janks.iloc[j] > 0
        j+=1
        while row >= t:
            t+=1
            r+=1
        dfs.FPS.iloc[r] +=1
        if jank:
            dfs.JPS.iloc[r] +=1
            
    #create statistics and distribution lists
    runtime = round(df.relative_time.iloc[-1], 2)
    num_frames = len(df.index)
    #fps_median = round(dfs.FPS.median(),1)
    fps_dist = pd.Series(df.fps).value_counts()
    #fps_mode = fps_dist.index.values[0]
    #try:
    #    frames_at_vsync = fps_dist[60] #assume vsync = VSYNC_INTERVAL
    #except:
    #    frames_at_vsync = 0
    #frames_at_mode = fps_dist.iloc[0]
    fps_dist = fps_dist.sort_index(ascending=False)
    
    jank_dist = pd.Series(df.janks).value_counts().sort_index()
    try:
        num_janks = jank_dist[1]
    except:
        num_janks = 0
        
    jank_raw_dist = pd.Series(df.janks_raw).value_counts().sort_index()
    jank_raw_dist = jank_raw_dist.drop(jank_raw_dist.index[0])
    statistics = [('Run Time', runtime),
                  ('Total Frames', num_frames),
                  ('FPS', round(float(num_frames)/runtime, 2)),
                  #('FPS (Median)', fps_median),
                  #('FPS (Mode)', fps_mode),
                  #('Frames at MODE', frames_at_mode),
                  #('Frames Not at MODE', num_frames-frames_at_mode),
                  #('% Frames at MODE', round(float(frames_at_mode)/(float(num_frames)/100.0), 2)),
                  #('Frames at vSync', frames_at_vsync),
                  #('Frames not at vSync', num_frames-frames_at_vsync),
                  #('% Frames at vSync', round(float(frames_at_vsync)/(float(num_frames)/100.0), 2)),
                  ('Jank Limit', jank_threshold),
                  ('No. Janks', num_janks),
                  ('JPS', round(float(num_janks)/runtime, 2)),
                  ('Janks %', round(float(num_janks)/(float(num_frames)/100.0), 2))]
    
    #write statistics, distribution data & time-based data to file
    while len(fps_dist) < len(jank_raw_dist):
        fps_dist = fps_dist.append(pd.Series((0), index=[-1]))
    while len(jank_raw_dist) < len(fps_dist):
        jank_raw_dist = jank_raw_dist.append(pd.Series((0), index=[-1]))
    dist_list = pd.DataFrame(np.zeros(shape=(len(fps_dist), 1)), columns=['tmp'], dtype=int)
    dist_list['FPS'] = pd.Series(fps_dist.index.values)
    dist_list['FPS-Dist'] = pd.Series(fps_dist.values)
    dist_list['Janks'] = pd.Series(jank_raw_dist.index.values)
    dist_list['Janks-Dist'] = pd.Series(jank_raw_dist.values)
    dist_list = dist_list.drop('tmp', 1)
    
    o_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), in_file + '-stats.txt')
    with open(o_file, 'w') as wfh:
        writer = csv.writer(wfh)
        writer.writerow([in_file])
        writer.writerow(['Stats'])
        writer.writerows(statistics)
        writer.writerow([' '])
        writer.writerow(['Dist'])
        writer.writerow(list(dist_list.columns.values))
        writer.writerows(dist_list.values)
        writer.writerow([' '])
        writer.writerow(['Graphs'])
        writer.writerow(list(dfs.columns.values))
        writer.writerows(dfs.values)

    print "\nAnalysis Complete"

if __name__ == "__main__":

    ignore_contexts = []
    keep_contexts = []
    
    #ignore_contexts = ['DimSurface', 'com.android.systemui.ImageWallpaper', 'com.teslacoilsw.launcher/com.android.launcher2.Launcher', 'DimAnimator', 'StatusBar']
    #keep_contexts = ['SurfaceView', 'com.rovio.angrybirds/com.rovio.fusion.App', 'Starting com.rovio.angrybirds']
    #sys.argv.append('test')
    #sys.argv.append('--c')
    #sys.argv.append('40')
    #print sys.argv
    
    args = parse_arguments()
    
    RecordContexts(out_file=args.filename, cadence=args.cadence, keep=keep_contexts, ignore=ignore_contexts)
    AnalyzeContexts(in_file=args.filename, jank_threshold=args.jank_threshold, keep=keep_contexts, ignore=ignore_contexts)