Matplotlib, SciPy, and Pandas

#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 30 08:37:33 2018

Description: This script demonstrates the basic use of the Pandas package, in particular
the filtering, groupby, aggregation, and plotting functionality. 

Python 2.7

@author: ks
"""

import pandas as pd
import os
from matplotlib import pyplot as plt 
import calendar
from __future__ import print


def read_rawdata(data_file, header_file, ozoneday_col):
    """
    Reads raw data from data_file into a pandas DataFrame with column values
    assigned based on values read from header_file.
    An additional label (ozonday_col) that is not provided by header_file
    is appended to label the last column in the DataFrame.
    Returns the raw data DataFrame object.
    """
    # check if files exist
    if not os.path.isfile(data_file) or not os.path.isfile(header_file):
        return None
    
    # read file with column headers
    datainfo = pd.read_csv(
        header_file,                    
        header=None,                     # this file does not have any column headers
        skiprows=range(2),               # skip first two rows; they contain additional annotations that we dont need
        names=['column name','data type'],    # set column labels
        delimiter=':')                   # the ':' separates the column name from the data type in each row

    # fix column labels by adding missing label for 'Ozone day' to columnnames
    columnnames = list(datainfo['column name'])
    columnnames.append(ozoneday_col)

    # read data file and set column names
    data = pd.read_csv(
        data_file,              # name of txt file to open
        header=None,            # this file does not have any column headers
        names=columnnames,      # set column names
        delimiter=',')          # the ',' separates the column name from the data type in each row
    return data


def convertColumnValues(data, date_col, ozoneday_col):
    """ 
    Returns copy of DataFrame object with conversion of date column values 
    to datetime64 and ozone day colum values to boolean.
    - data : DataFrame object
    - date_col: string that specifies the column with date values
    - ozoneday_col: string that specifies the column with ozone classification
    """
    converted = data
    converted[date_col] = converted[date_col].apply(pd.to_datetime)
    converted[converted.columns[1:]] = converted[converted.columns[1:]].apply(pd.to_numeric, errors='coerce')
    converted[ozoneday_col] = converted[ozoneday_col].astype('bool')
    return converted

    
def ozonedays_per_year(data, date_col, ozoneday_col):
    """ 
    Returns DataFrame with number of ozone days per year.
    - data : DataFrame object
    - date_col: string that specifies the column with date values
    - ozoneday_col: string that specifies the column with ozone classification
    """
    groupparam = [data[date_col].dt.year]
    grouped_od_column = data.groupby(groupparam)[ozoneday_col]
    odays_per_year = grouped_od_column.sum()
    
    # convert resulting DataSeries to DataFrame and rename index to 'Year'
    df = odays_per_year.to_frame(ozoneday_col)
    df.index.names = ['Year']

    # rename column with sum values
    df.rename(index=str, columns={ozoneday_col:'# of ozone days'}, inplace=True)
    return df


def ozonedays_by_month(data, date_col, ozoneday_col):
    """ 
    Returns DataFrame with total number of ozone days (value) by calendar month.
    - data : DataFrame object
    - date_col: string that specifies the column with date values
    - ozoneday_col: string that specifies the column with ozone classification
    """
    # group data and use .sum aggregate function
    groupparam = [data[date_col].dt.month]#dt.to_period('M')
    grouped_od_column = data.groupby(groupparam)[ozoneday_col]
    odays_by_month = grouped_od_column.sum()
    
    # Convert resulting DataSeries to DataFrame and set index to reflect month names rather than month numbers
    df = odays_by_month.to_frame(ozoneday_col)#, index=monthidx)
    monthidx = [calendar.month_abbr[x] for x in range(13)]
    df['Month'] = pd.Series(monthidx) # create new column 'Month' with month labels
    df.set_index('Month', inplace=True, drop=True)
    
    # Rename column with sum values
    df.rename(index=str, columns={ozoneday_col:'# of ozone days'}, inplace=True)
    return df


def precipitation_analysis(data, precp_col, precp_thr, ozoneday_col):
    """
    Returns the GroupBy object and a legend dictinary after peforming the grouping 
    by precipitation value and ozone classification.
    - data : DataFrame object
    - precp_col : string that specifies the column with precipitation data
    - precp_thr : float value used as threshold to categorize precipitation data 
    - ozoneday_col : string that specifies the column with ozone classification
    """ 
    # group data by precp threshold and ozone day classification --> 4 groups
    groupparam = [data[precp_col] > precp_thr,ozoneday_col]
    results = data.groupby(groupparam)
    
    # the group names in results are (bool, bool) tuples 
    # create a dictionary we can use as legend for the four groups
    legenddict = {}
    legenddict[(False,False)]='Precp <= %.2f, no ozone day' % precp_thr
    legenddict[(False,True)]='Precp <= %.2f, ozone day'  % precp_thr
    legenddict[(True,False)]='Precp > %.2f, no ozone day'  % precp_thr
    legenddict[(True,True)]='Precp > %.2f, ozone day'  % precp_thr
    return results, legenddict


def print_ozonedays(title, ozonedays):
    """
    Prints a title string and the content of the ozonedays object.
    """
    print("\n%s" % title)
    # convert the float values to integer-like output
    print(ozonedays.to_string(header=True, index=True, float_format=lambda x: '%.0f' % x))
    
    
def print_precpdata(groupeddata, group_labels):
    """
    Creates and prints the summary statistics for a Pandas GroupBy object.
    - groupeddata : The data object obtained by a groupby operation
    - group_labels : labels for precipitation and ozone day categories
    """
    # define windspeed categorie and stats we're interested in 
    windspeed_cat = ["WSR%d" %ws for ws in range(10)]
    stats = ['min', 'max', 'mean', 'std', 'median','count']
    # iterate over all windspeed categrories, pull out of groupeddata, calculate stats
    for windspeed in windspeed_cat:
        print("-------------------\n\n",windspeed)
        ws_data = groupeddata[windspeed].aggregate(stats)
        ws_data.index.names = group_labels
        print(groupeddata[windspeed].aggregate(stats))
    

def plot_ozonedays(ozonedays, title):
    """
    Createas a bar plot of a DataFrame object
    - ozonedays : a DataFrame object with the data to plot
    - title : string object to use as title for the plot
    """
    # create a new figure and axis object
    fig, ax = plt.subplots()
    ozonedays.plot.bar(title=title, ax=ax)
    xlabel = ozonedays.index.names[0]
    # xlabel is unicode string, so clean up by ignoring special characters
    xlabel = xlabel.encode('ascii','ignore')
    ax.set_xlabel(xlabel)
    ax.set_ylabel('days')

    
def plot_scatter(datagroups, title, xlabel, ylabel, legenddict):
    """
    Createas a scatter plot of values in two columns from multiple data groups 
    - datagroups : GroupBy object obtained via Pandas groupbby function
    - title : title for the plot
    - xlabel : label for the x-axis
    - ylabel : label for the y-axis
    - legenddict : dictionary that maps group names to meaningful legend labels
    """
    fig,ax = plt.subplots()
    for name,group in datagroups:
        plt.scatter(group[xlabel], group[ylabel], s=10, label=legenddict[name])
    ax.set_title(title)
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.legend(loc='upper left', bbox_to_anchor=(1,1))
    plt.show()    
    

    
if __name__ == '__main__':
    data_file = 'eighthr.data.txt'
    header_file = 'eighthr.names.txt'
    date_label = 'Date'
    ozoneday_label = 'Ozone day'
    pthreshold = 0.5
    
    data = read_rawdata(data_file, header_file, ozoneday_label)
    if (data is not None):
        # analyze data
        data = convertColumnValues(data, date_label, ozoneday_label)
        odays_per_year = ozonedays_per_year(data, date_label, ozoneday_label)
        ocount_by_month = ozonedays_by_month(data, date_label, ozoneday_label)
        precp_groups, legenddict = precipitation_analysis(data, 'Precp', pthreshold, ozoneday_label)

        # print data as tables
        print_ozonedays('Ozone days per year:', odays_per_year)
        print_ozonedays('Aggregate ozone days by month', ocount_by_month)
        print_precpdata(precp_groups, ['Precp > %.2f' % pthreshold, ozoneday_label])
        
        # plot data
        plot_ozonedays(odays_per_year, 'Ozone days per year')
        plot_ozonedays(ocount_by_month, 'Aggregate ozone days by month')
        plot_scatter(precp_groups, 'Windspeed versus Temperature', 'T0', 'WSR0', legenddict)
    else:
        print("Missing data (%s) and/or header (%) file. Data cannot be analyzed." % (data_file, header_file))