diff --git a/DATA-README.md b/DATA-README.md index 5faa982..b46fb86 100644 --- a/DATA-README.md +++ b/DATA-README.md @@ -519,9 +519,9 @@ http://ctrp3.ctdata.org/rawdata/ ### Nebraska -**Original format**: MDB +**Original format**: MDB, excel (xlsx) -**Time period**: 2002-2014 +**Time period**: 2002-2016 **Columns with no data**: - `stop_time` @@ -540,7 +540,7 @@ http://ctrp3.ctdata.org/rawdata/ - The original data was aggregated. It was grouped by stop reason, outcome and whether there was a search separately. Therefore, it is not possible to cross tabulate them together. We only use the last grouping. - State and local stops are mixed together, but identifiable by the `dept_lvl` field. - The data is by quarter, not by day. So all stop_dates are the first date of the quarter. -- For state patrol stops, there is a strange jump (Q1) and then dip (Q2–4) in the data for 2012. It looks like for 2012 all stops are recorded as happening in the first quarter. +- For state patrol stops, there is a strange jump (Q1) and then dip (Q2–4) in the data for 2012. It looks like for 2012 most stops are recorded as happening in the first quarter. **Extra fields**: none diff --git a/src/processing/states/NE.R b/src/processing/states/NE.R index 1afa2dd..4952a6e 100755 --- a/src/processing/states/NE.R +++ b/src/processing/states/NE.R @@ -8,31 +8,68 @@ change_path(this_state) # Read in and combine data print(sprintf("[%s] reading in the data", this_state)) # Read mappings -d_dept <- read_csv("tbl_ORI.csv") %>% select(DeptID=ID, dept=ORI_Description, dept_lvl=ORITYPEID, county=AgencyCounty) +d_dept <- read_csv("tbl_ORI.csv") %>% select(DeptID=ID, dept=ORI_Description, dept_lvl=ORITYPEID, county=AgencyCounty) %>% + mutate(dept=str_replace_all(dept, '\\.', '')) d_time <- read_csv("tbl_Quarters.csv") %>% select(TimeID=QuarterID, TimeStart=`Actual Quarter Date Start`) d_repo <- read_csv("tbl_Reports.csv") %>% select(ReportID, Race, Reason=TopicDescription, Outcome=DetailDescription) -# Read data +# Read first dataset d <- read_csv("tbl_Quarter_ID.csv") %>% select(TimeID=QuarterID, DeptID=ID, ReportID, n=Value) %>% inner_join(d_time, by='TimeID') %>% inner_join(d_dept, by='DeptID') %>% inner_join(d_repo, by='ReportID') %>% select(-TimeID, -DeptID, -ReportID) %>% + # Note : grouping by Reason leads to different sum totals per (Time, Deptartment, Race) group. + # We only look at the 'search' grouping for this script. + filter(Reason=='Searches') %>% + # extract date + mutate(date=as.Date(substr(TimeStart, 1, 8), format='%m/%d/%y')) %>% + select(date, dept_lvl, dept, county, Race, Outcome, n) + +# Read in newer Excel-based data +# new agencies +d_dept_new <- read_csv("Nebraska_Agency.csv") %>% + rename(dept=AgencyName, Agency_Cd=AgencyCd) %>% + mutate(dept=str_replace_all(dept, '\\.', '')) %>% + left_join(d_dept, by='dept') # get agency info from old dataset +# new data +d_new <- read_csv("Nebraska_Traffic-Stop-2015-2016.csv") %>% + # convert to quarter + mutate( + stop_month=str_pad(((Racial_Profile_Quarter-1)*3)+1, 2, 'left', '0'), + date=as.Date(paste(Racial_Profile_Year, stop_month, '01', sep='-')) + ) %>% + # join in agencies + left_join(d_dept_new, by="Agency_Cd") %>% + # turn into long data format + select(date, dept, dept_lvl, county, + Search_Conducted_White, Search_Not_Conducted_White, + Search_Conducted_Black, Search_Not_Conducted_Black, + Search_Conducted_Hispanic, Search_Not_Conducted_Hispanic, + Search_Conducted_NatAmerican, Search_Not_Conducted_NatAmerican, + Search_Conducted_Asian, Search_Not_Conducted_Asian, + Search_Conducted_Other, Search_Not_Conducted_Other + ) %>% + gather(type, n, -date:-county) %>% + mutate( + Outcome=str_replace(str_extract(type, ".*(?=_)"), '_', ' '), + Race=substr(type, nchar(Outcome)+2, nchar(type)) + ) %>% + select(date, dept_lvl, dept, county, Race, Outcome, n) + +# combine the two +d <- rbind(d, d_new) %>% # Remove Federal agencies, Other and Private agencies, respectively filter(!dept_lvl %in% c(4, 7, 12)) -# Note : grouping by Reason leads to different sum totals per (Time, Deptartment, Race) group. -# We only look at the 'search' grouping for this script. -d <- d %>% filter(Reason=='Searches') - # Value dictionaries -race_keys <- c("Asian / Pacific Islander", "Native American - Alaskan") -race_vals <- c("Asian", "Other") +race_keys <- c("Asian / Pacific Islander", "Native American - Alaskan", "NatAmerican") +race_vals <- c("Asian", "Other", "Other") # Rename and extract columns print(sprintf("[%s] extracting columns", this_state)) d$state <- this_state -d$stop_date <- make_date(substr(d$TimeStart, 1, 8), format='%m/%d/%y') +d$stop_date <- make_date(d$date) d$stop_time <- NA # not included d$id <- make_row_id(d) d$location_raw <- ifelse(d$county %in% c('Inactive', 'NSP and Other', 'Private'), NA, d$county)