import sys
import os
sys.path.append(os.path.dirname(os.getcwd()))

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
import sys
import blog
%matplotlib inline
import my_utils
blog.set_blog_style()

df_io = pd.read_csv('./data.csv',index_col=0,parse_dates=['date_'])
df_io.head()

df = df_io.copy().sort_values('date_').set_index('date_').drop(columns='val_updated')
df.head()

df_io\
.copy()\
.sort_values('date_')\
.set_index('date_')\
.drop(columns='val_updated')\
.head()

# This block will result in an error
df_io\
.copy()\ # This is an inline comment
# This is a regular comment
.sort_values('date_')\
# .set_index('date_')\
.drop(columns='val_updated')\ 
.head()

# This block will result in an error
df_io\
.copy()\
.sort_values('date_')\
.set_index('date_')\
.drop(columns='val_updated')\ 
.head()

# We can split these value inside ()
users = (134856, 195373, 295817, 294003, 262166, 121066, 129678, 307120,
         258759, 277922, 220794, 192312, 318486, 314631, 306448, 297059,206892, 
         169046, 181703, 146200, 199876, 247904, 250884, 282989, 234280, 202520,
         138064, 133577, 301053, 242157)

# Penny Drop: We can also Split here
df = (df_io
        .copy() # This is an inline comment
        # This is a regular comment
        .sort_values('date_')
        .set_index('date_')
        .drop(columns='val_updated')   
)

df.head()

df.event_type.map(lambda x: x+3).head()

# A dict is also a calleble
df['event_type'] = df.event_type.map({
    1:'A',
    5:'B',
    7:'C'
})
df.head()

bad = df.copy()
bad['day'] = bad.index.date
bad['hour'] = bad.index.hour
(bad
.groupby(['day','hour'])
.count()
)

df.resample('H').count() # H is for Hour

rs = df.resample('10T').count()
# T is for Minute, and pandas understands 10 T, it will also under stand 11T if you wonder
rs.head()

# Take only timestamp in the hour of 21:00.
rs.loc['2018-10-09 21',:]

# Take all time stamps berfore 18:31
rs.loc[:'2018-10-09 18:31',:]

fig, ax = plt.subplots()
rs.plot(ax=ax,linestyle='--')
(rs
 .rolling(6)
 .mean()
 .rename(columns = {'event_type':'rolling mean'})
 .plot(ax=ax)
)

rs.expanding(6).mean().rename(columns = {'event_type':'expanding mean'}).plot(ax=ax)
rs.ewm(6).mean().rename(columns = {'event_type':'ewm mean'}).plot(ax=ax)
plt.show()

fig, ax = plt.subplots()
rs.plot(ax=ax,linestyle='--')
(rs
 .rolling(6)
 .apply(lambda x: np.power(np.product(x),1/len(x)), raw=True )
 .rename(columns = {'event_type':'Rolling Geometric Mean'})
 .plot(ax=ax)
)
plt.show()

per_event = (df
             .groupby('event_type')
             .resample('15T')
             .apply('count')
             .rename(columns={'event_type':'amount'})
            )
per_event.head()

pt = pd.pivot_table(per_event,values = 'amount',columns='event_type',index='date_')
pt.head()

pt = per_event.unstack('event_type')
pt.columns = pt.columns.droplevel() # Unstack creates a multiindex on columns
pt.head()

pt.stack().head()

stack_back = (pt
              .stack()
              .to_frame('amount') # Turn Series to DF without calling the DF constructor
              .swaplevel() # Swaps the levels of the index
              .sort_index() # Sort by index, makes so much sense yet I used to do .reset_index().sort_values()
             )
stack_back.head()

stack_back.equals(per_event)

per_event.sort_values(by=['amount']).head(10)

per_event.sort_index().head(7)

per_event.sort_index(level=1).head(7)

per_event.sort_values(['amount','event_type'], ascending=(False, True)).head(10)

cl = pt.copy()
lb = 5
ub = 12
# Needed A loop of 3 lines
for col in ['A','B','C']:
    cl['clipped_{}'.format(col)] = cl[col]
    cl.loc[cl[col] < lb,'clipped_{}'.format(col)] = lb
    cl.loc[cl[col] > ub,'clipped_{}'.format(col)] = ub
my_utils.plot_clipped(cl) # my_utils can be found in the github repo

cl = pt.copy()
# Beutiful One Liner
cl[['clipped_A','clipped_B','clipped_C']] = cl.clip(5,12)
my_utils.plot_clipped(cl) # my_utils can be found in the github repo

etypes = list('ABCDZ') # New columns

# Define a date range - Pandas will automatically make this into an index
idx = pd.date_range(start='2018-10-09 16:00:00',end='2018-10-09 23:59:00',freq='15T')
type(idx)

pt.reindex(idx, columns=etypes, fill_value=0).head()

### Let's put this in a function - This will help us later.
def get_all_types_and_timestamps(df, min_date='2018-10-09 16:00:00',
                                 max_date='2018-10-09 23:59:00', etypes=list('ABCDZ')):
    ret = df.copy()
    time_idx = pd.date_range(start=min_date,end=max_date,freq='15T')
    # Indices work like set. This is a good practive so we don't override our intended index
    idx = ret.index.union(time_idx)
    etypes = df.columns.union(set(etypes))
    ret = ret.reindex(idx, columns=etypes, fill_value=0)
    return ret

pt.assign(mean_all = pt.mean(axis=1)).head()

def do_something(df, col='A', n = 200):
    ret = df.copy()
    # A dataframe is mutable, if you don't copy it first, this is prone to many errors.
    # I always copy when I enter a function, even if I'm sure it shouldn't change anything.
    ret[col] = ret[col] + n
    return ret

do_something(do_something(do_something(pt), 'B', 100), 'C',500).head()

(pt
  .pipe(do_something)
  .pipe(do_something, col='B', n=100)
  .pipe(do_something, col='C', n=500)  
  .head(5))

df = (pd
      .read_csv    ('./data.csv', index_col=0, parse_dates=['date_'])
      .assign      (event_type=lambda df: df.event_type.map({1: 'A', 5: 'B', 7: 'C'}))
      .sort_values ('date_')
      .set_index   ('date_')
      .drop        (columns='val_updated')
      .groupby     ('event_type')
      .resample    ('15T')
      .apply       ('count')
      .rename      (columns={'event_type': 'amount'})
      .unstack     ('event_type')
      .pipe        (my_utils.remove_multi_index)
      .pipe        (get_all_types_and_timestamps) # Remember this from before?
      .assign      (mean_event=lambda df: df.mean(axis=1))
      .loc         [:, ['mean_event']]
      .pipe        (my_utils.make_sliding_time_windows, steps_back=6)
      # Imagine last pipe is an api. Not sure how it's implemented, but the name is meaningful
      .dropna      ()
      )

df.head()