Updates to NE

DATA-README.md

@@ -519,9 +519,9 @@ http://ctrp3.ctdata.org/rawdata/
 
 ### Nebraska
 
-**Original format**: MDB
+**Original format**: MDB, excel (xlsx)
 
-**Time period**: 2002-2014
+**Time period**: 2002-2016
 
 **Columns with no data**:
 - `stop_time`
@@ -540,7 +540,7 @@ http://ctrp3.ctdata.org/rawdata/
 - The original data was aggregated. It was grouped by stop reason, outcome and whether there was a search separately. Therefore, it is not possible to cross tabulate them together. We only use the last grouping.
 - State and local stops are mixed together, but identifiable by the `dept_lvl` field.
 - The data is by quarter, not by day. So all stop_dates are the first date of the quarter.
-- For state patrol stops, there is a strange jump (Q1) and then dip (Q2–4) in the data for 2012. It looks like for 2012 all stops are recorded as happening in the first quarter.
+- For state patrol stops, there is a strange jump (Q1) and then dip (Q2–4) in the data for 2012. It looks like for 2012 most stops are recorded as happening in the first quarter.
 
 **Extra fields**: none
 

src/processing/states/NE.R

@@ -8,31 +8,68 @@ change_path(this_state)
 # Read in and combine data
 print(sprintf("[%s] reading in the data", this_state))
 # Read mappings
-d_dept <- read_csv("tbl_ORI.csv") %>% select(DeptID=ID, dept=ORI_Description, dept_lvl=ORITYPEID, county=AgencyCounty)
+d_dept <- read_csv("tbl_ORI.csv") %>% select(DeptID=ID, dept=ORI_Description, dept_lvl=ORITYPEID, county=AgencyCounty) %>%
+  mutate(dept=str_replace_all(dept, '\\.', ''))
 d_time <- read_csv("tbl_Quarters.csv") %>% select(TimeID=QuarterID, TimeStart=`Actual Quarter Date Start`)
 d_repo <- read_csv("tbl_Reports.csv") %>% select(ReportID, Race, Reason=TopicDescription, Outcome=DetailDescription)
-# Read data
+# Read first dataset
 d <- read_csv("tbl_Quarter_ID.csv") %>%
   select(TimeID=QuarterID, DeptID=ID, ReportID, n=Value) %>%
   inner_join(d_time, by='TimeID') %>%
   inner_join(d_dept, by='DeptID') %>%
   inner_join(d_repo, by='ReportID') %>%
   select(-TimeID, -DeptID, -ReportID) %>%
+  # Note : grouping by Reason leads to different sum totals per (Time, Deptartment, Race) group.
+  # We only look at the 'search' grouping for this script.
+  filter(Reason=='Searches') %>%
+  # extract date
+  mutate(date=as.Date(substr(TimeStart, 1, 8), format='%m/%d/%y')) %>%
+  select(date, dept_lvl, dept, county, Race, Outcome, n)
+
+# Read in newer Excel-based data
+# new agencies
+d_dept_new <- read_csv("Nebraska_Agency.csv") %>%
+  rename(dept=AgencyName, Agency_Cd=AgencyCd) %>%
+  mutate(dept=str_replace_all(dept, '\\.', '')) %>%
+  left_join(d_dept, by='dept') # get agency info from old dataset
+# new data
+d_new <- read_csv("Nebraska_Traffic-Stop-2015-2016.csv") %>%
+  # convert to quarter
+  mutate(
+    stop_month=str_pad(((Racial_Profile_Quarter-1)*3)+1, 2, 'left', '0'),
+    date=as.Date(paste(Racial_Profile_Year, stop_month, '01', sep='-'))
+  ) %>%
+  # join in agencies
+  left_join(d_dept_new, by="Agency_Cd") %>%
+  # turn into long data format
+  select(date, dept, dept_lvl, county,
+           Search_Conducted_White, Search_Not_Conducted_White,
+           Search_Conducted_Black, Search_Not_Conducted_Black,
+           Search_Conducted_Hispanic, Search_Not_Conducted_Hispanic,
+           Search_Conducted_NatAmerican, Search_Not_Conducted_NatAmerican,
+           Search_Conducted_Asian, Search_Not_Conducted_Asian,
+           Search_Conducted_Other, Search_Not_Conducted_Other
+        ) %>%
+  gather(type, n, -date:-county) %>%
+  mutate(
+  	Outcome=str_replace(str_extract(type, ".*(?=_)"), '_', ' '),
+  	Race=substr(type, nchar(Outcome)+2, nchar(type))
+  ) %>%
+  select(date, dept_lvl, dept, county, Race, Outcome, n)
+
+# combine the two
+d <- rbind(d, d_new) %>%
   # Remove Federal agencies, Other and Private agencies, respectively
   filter(!dept_lvl %in% c(4, 7, 12))
 
-# Note : grouping by Reason leads to different sum totals per (Time, Deptartment, Race) group.
-# We only look at the 'search' grouping for this script.
-d <- d %>% filter(Reason=='Searches')
-
 # Value dictionaries
-race_keys <- c("Asian / Pacific Islander", "Native American - Alaskan")
-race_vals <- c("Asian", "Other")
+race_keys <- c("Asian / Pacific Islander", "Native American - Alaskan", "NatAmerican")
+race_vals <- c("Asian", "Other", "Other")
 
 # Rename and extract columns
 print(sprintf("[%s] extracting columns", this_state))
 d$state                 <- this_state
-d$stop_date             <- make_date(substr(d$TimeStart, 1, 8), format='%m/%d/%y')
+d$stop_date             <- make_date(d$date)
 d$stop_time             <- NA  # not included
 d$id                    <- make_row_id(d)
 d$location_raw          <- ifelse(d$county %in% c('Inactive', 'NSP and Other', 'Private'), NA, d$county)