# imports
import requests
from datascience import *
import matplotlib.pyplot as plt
import datetime
import folium
import time
import json
import os
from branca.colormap import linear
import branca.colormap
import pandas as pd
%matplotlib inline

sf_police = os.path.join('Police_Department_Sample_Incidents.csv')
df = pd.read_csv(sf_police)
df

#ts = Table(data.labels)
import pandas as pd
from sodapy import Socrata

# Unauthenticated client only works with public data sets. Note 'None'
# in place of application token, and no username or password:
client = Socrata("data.sfgov.org", None)

# Example authenticated client (needed for non-public datasets):
# client = Socrata(data.sfgov.org,
#                  MyAppToken,
#                  userame="user@example.com",
#                  password="AFakePassword")

# First 2000 results, returned as JSON from API / converted to Python list of
# dictionaries by sodapy.
results = client.get("tmnf-yvry", limit=2000)

# Convert to pandas DataFrame
results_df = pd.DataFrame.from_records(results)

import pandas as pd
df = pd.DataFrame(recs)
df.head()

data = Table.from_df(df.drop('location', axis=1))
data

min(df['date'])

# making a table out of our json
# DO NOT USE 
#data = Table.from_records(json_response)
#data.show(3)

data['y'] = data['y'].astype('float')
data['x'] = data['x'].astype('float')

# creating a year column from the first four characters of the 'date' column
data['year'] = pd.DatetimeIndex(df['date']).year
data

agg_on_year = data.group('year')
agg_on_year.show()

agg_on_year.plot('year', 'count')

agg_by_crime = data.where('year', 2017).group('category')
agg_by_crime['proportion'] = [count / sum(agg_by_crime.column('count')) for count in agg_by_crime.column('count')]
agg_by_crime.sort('proportion', descending=True)

agg_by_crime.sort('count', descending=True).barh('category', 'proportion')

data.group('dayofweek').barh('dayofweek')

# making an hour column that can be grouped on
data['hour'] = [int(t[:2]) for t in data['time']]

data.group('hour').bar('hour')

twentyeighteen = data.where('year' == 2017).sample(1000)
twentyeighteen['y'] = twentyeighteen['y'].astype('float')
twentyeighteen['x'] = twentyeighteen['x'].astype('float')

mp = folium.Map(location=[37.7749, -122.4194])
for coords in list(zip(twentyeighteen['y'], twentyeighteen['x'])):
    folium.Marker(
        location=coords
    ).add_to(mp)
mp

from folium.plugins import HeatMap

mp = folium.Map(location=[37.7749, -122.4194])
HeatMap(list(zip(twentyeighteen['y'], twentyeighteen['x']))).add_to(mp)
mp

#import requests
#r = requests.get(url='https://data.sfgov.org/resource/xfcw-9evu.json')

sf_neighborhoods = os.path.join('SF Find Neighborhoods.geojson')
geo_json_data = json.load(open(sf_neighborhoods))

m = folium.Map([37.7749, -122.4194], zoom_start = 12)
m

# might be too big bc won't display
m = folium.Map(
    location=[37.7749, -122.4194], zoom_start = 12
)

folium.GeoJson(geo_json_data
).add_to(m)
m

twentyeighteen = twentyeighteen.to_df()

import geopandas as gpd
import shapely
shapely.speedups.enable()

twentyeighteen_spatial_points = gpd.GeoDataFrame(twentyeighteen.drop(['x', 'y'], axis=1),
                       crs={'init': 'epsg:4326'},
                       geometry=twentyeighteen.apply(lambda row: shapely.geometry.Point((row.x, row.y)), axis=1))

sf_polygons = gpd.GeoDataFrame.from_features(geo_json_data['features'])
sf_polygons.crs = {'init' :'epsg:4326'}

sf_spatial = gpd.sjoin(sf_polygons, twentyeighteen_spatial_points, how="inner", op="intersects")

crime_neighborhood = pd.DataFrame(sf_spatial).reset_index()
crime_neighborhood.head(5)

crime_neighborhood_agg = crime_neighborhood.groupby('name').size().reset_index()
crime_neighborhood_agg.head(5)

crime_neighborhood_agg.columns = ['neighborhood', 'crimes']
crime_neighborhood_agg.head(5)

m = folium.Map(
    location=[37.7749, -122.4194], zoom_start = 12
)

m.choropleth(
    geo_data=geo_json_data,
    data=crime_neighborhood_agg,
    columns=['neighborhood', 'crimes'],
    key_on='feature.properties.name',
    fill_color='OrRd',
    threshold_scale=[10, 60, 100, 140],
    highlight=True
    )

m

def append_and_follow(t, link, n):
    if n == 0:
        print('Next link (if you want to continue)', link)
        return t
    time.sleep(3)
    print(n)
    r = requests.get(link)
    js = r.json()
    return append_and_follow(t.append(Table.from_records(js['value'])), js['@odata.nextLink'], n-1)

starter = Table(('__id', 'address', 'category', 'date', 'dayofweek', 'descript', 'incidntnum', 
                 'location', 'pddistrict', 'pdid', 'resolution', 'time', 'x', 'y'))
starting_url = 'https://data.sfgov.org/api/odata/v4/tmnf-yvry'

c = append_and_follow(starter, starting_url, 6)
c