2024 modelling data update

.github/workflows/lint.yaml

@@ -23,6 +23,10 @@ jobs:
           # list of changed files within `super-linter`
           fetch-depth: 0
 
+      - name: Disable renv
+        shell: bash
+        run: rm etl/.Rprofile
+
       - name: Lint
         uses: github/super-linter@v6
         env:

etl/renv.lock

@@ -1,6 +1,6 @@
 {
   "R": {
-    "Version": "4.4.2",
+    "Version": "4.4.1",
     "Repositories": [
       {
         "Name": "CRAN",
@@ -2808,6 +2808,19 @@
       ],
       "Hash": "ad57b543f7c3fca05213ba78ff63df9b"
     },
+    "sfarrow": {
+      "Package": "sfarrow",
+      "Version": "0.4.1",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "arrow",
+        "dplyr",
+        "jsonlite",
+        "sf"
+      ],
+      "Hash": "b320f164b1d7bb7e4582b841e22d15a0"
+    },
     "sfheaders": {
       "Package": "sfheaders",
       "Version": "0.4.4",

etl/scripts-ccao-data-raw-us-east-1/housing/housing-ihs_index.R

@@ -26,13 +26,13 @@ remote_file <- file.path(output_bucket, paste0("ihs_price_index_data.parquet"))
 
 # Grab the data, clean it just a bit, and write if it doesn't already exist
 data.frame(t(
-  openxlsx::read.xlsx(most_recent_ihs_data_url, sheet = 2) %>%
-    dplyr::select(-c("X2", "X3", "X4"))
+  read.xlsx(most_recent_ihs_data_url, sheet = 2) %>%
+    select(-c("X2", "X3", "X4"))
 )) %>%
   # Names and columns are kind of a mess after the transpose,
   # shift up first row, shift over column names
-  janitor::row_to_names(1) %>%
-  dplyr::mutate(puma = rownames(.)) %>%
-  dplyr::relocate(puma, .before = "YEARQ") %>%
-  dplyr::rename(name = "YEARQ") %>%
-  arrow::write_parquet(remote_file)
+  row_to_names(1) %>%
+  mutate(puma = rownames(.)) %>%
+  relocate(puma, .before = "YEARQ") %>%
+  rename(name = "YEARQ") %>%
+  write_parquet(remote_file)

etl/scripts-ccao-data-raw-us-east-1/sale/sale-foreclosure.R

@@ -16,7 +16,7 @@ output_bucket <- file.path(AWS_S3_RAW_BUCKET, "sale", "foreclosure")
 files <- list.files("O:/CCAODATA/data/foreclosures", recursive = TRUE)
 
 # Function to retrieve data and write to S3
-read_write <- function(x) {
+walk(files, \(x) {
   output_dest <- file.path(output_bucket, glue(parse_number(x), ".parquet"))
 
   if (!object_exists(output_dest)) {
@@ -28,7 +28,4 @@ read_write <- function(x) {
     ) %>%
       write_parquet(output_dest)
   }
-}
-
-# Apply function to foreclosure data
-walk(files, read_write)
+})

etl/scripts-ccao-data-raw-us-east-1/sale/sale-mydec.R

@@ -22,7 +22,7 @@ files <- xml2::read_html(
   str_subset("ptax203")
 
 # Function to scrape IDOR data and write to S3
-down_up <- function(x) {
+walk(files, \(x) {
   year <- str_extract(x, pattern = "[0-9]{4}")
 
   if (
@@ -44,8 +44,4 @@ down_up <- function(x) {
     readr::read_delim(list.files(tmp2, full.names = TRUE), delim = "\t") %>%
       write_parquet(file.path(output_bucket, glue("{year}.parquet")))
   }
-}
-
-
-# Apply function to foreclosure data
-walk(files, down_up)
+})

etl/scripts-ccao-data-raw-us-east-1/spatial/spatial-access.R

@@ -12,6 +12,8 @@ output_bucket <- file.path(AWS_S3_RAW_BUCKET, "spatial", "access")
 # List APIs from city site
 sources_list <- bind_rows(list(
   # INDUSTRIAL CORRIDORS
+  # See https://data.cityofchicago.org/Community-Economic-Development/IndustrialCorridor_Jan2013/3tu3-iesz/about_data # nolint
+  # for more information
   "ind_2013" = c(
     "source" = "https://data.cityofchicago.org/api/geospatial/",
     "api_url" = "e6xh-nr8w?method=export&format=GeoJSON",

etl/scripts-ccao-data-raw-us-east-1/spatial/spatial-ccao.R

@@ -11,19 +11,17 @@ output_bucket <- file.path(AWS_S3_RAW_BUCKET, "spatial", "ccao")
 
 # Read privileges for the this drive location are limited.
 # Contact Cook County GIS if permissions need to be changed.
-file_path <- "//gisemcv1.ccounty.com/ArchiveServices/"
+file_path <- "//gisemcv1.ccounty.com/ArchiveServices/" # nolint
 
-sources_list <- bind_rows(list(
+sources_list <- data.frame(
   # NEIGHBORHOOD
-  "neighborhood" = c(
-    "url" = paste0(
-      "https://gitlab.com/ccao-data-science---modeling/packages/ccao",
-      "/-/raw/master/data-raw/nbhd_shp.geojson"
-    ),
-    "boundary" = "neighborhood",
-    "year" = "2021"
-  )
-))
+  "url" = paste0(
+    "https://gitlab.com/ccao-data-science---modeling/packages/ccao",
+    "/-/raw/master/data-raw/nbhd_shp.geojson"
+  ),
+  "boundary" = "neighborhood",
+  "year" = "2021"
+)
 
 # Function to call referenced API, pull requested data, and write it to S3
 pwalk(sources_list, function(...) {
@@ -45,6 +43,7 @@ gdb_files <- data.frame("path" = list.files(file_path, full.names = TRUE)) %>%
   filter(
     str_detect(path, "Current", negate = TRUE) &
       str_detect(path, "20") &
+      # We detect parcel GDBs, but will extract the township layer
       str_detect(path, "Parcels")
   )
 

etl/scripts-ccao-data-raw-us-east-1/spatial/spatial-political.R

@@ -12,7 +12,7 @@ output_bucket <- file.path(AWS_S3_RAW_BUCKET, "spatial", "political")
 
 # Read privileges for the this drive location are limited.
 # Contact Cook County GIS if permissions need to be changed.
-file_path <- "//gisemcv1.ccounty.com/ArchiveServices/"
+file_path <- "//gisemcv1.ccounty.com/ArchiveServices/" # nolint
 
 sources_list <- bind_rows(list(
   # BOARD OF REVIEW
@@ -23,7 +23,7 @@ sources_list <- bind_rows(list(
     "year" = "2012"
   ),
   "bor_2023" = c(
-    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/",
+    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/", # nolint
     "api_url" = "10/query?outFields=*&where=1%3D1&f=geojson",
     "boundary" = "board_of_review_district",
     "year" = "2023"
@@ -37,7 +37,7 @@ sources_list <- bind_rows(list(
     "year" = "2012"
   ),
   "cmd_2023" = c(
-    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/",
+    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/", # nolint
     "api_url" = "9/query?outFields=*&where=1%3D1&f=geojson",
     "boundary" = "commissioner_district",
     "year" = "2023"
@@ -51,7 +51,7 @@ sources_list <- bind_rows(list(
     "year" = "2010"
   ),
   "cnd_2023" = c(
-    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/",
+    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/", # nolint
     "api_url" = "13/query?outFields=*&where=1%3D1&f=geojson",
     "boundary" = "congressional_district",
     "year" = "2023"
@@ -65,7 +65,7 @@ sources_list <- bind_rows(list(
     "year" = "2012"
   ),
   "jsd_2022" = c(
-    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/",
+    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/", # nolint
     "api_url" = "5/query?outFields=*&where=1%3D1&f=geojson",
     "boundary" = "judicial_district",
     "year" = "2022"
@@ -79,7 +79,7 @@ sources_list <- bind_rows(list(
     "year" = "2010"
   ),
   "str_2023" = c(
-    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/",
+    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/", # nolint
     "api_url" = "11/query?outFields=*&where=1%3D1&f=geojson",
     "boundary" = "state_representative_district",
     "year" = "2023"
@@ -93,7 +93,7 @@ sources_list <- bind_rows(list(
     "year" = "2010"
   ),
   "sts_2023" = c(
-    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/",
+    "source" = "https://gis.cookcountyil.gov/traditional/rest/services/politicalBoundary/MapServer/", # nolint
     "api_url" = "12/query?outFields=*&where=1%3D1&f=geojson",
     "boundary" = "state_senate_district",
     "year" = "2023"
@@ -150,20 +150,19 @@ pwalk(sources_list, function(...) {
 # MUNICIPALITY
 
 # Paths for all relevant geodatabases
-gdb_files <- data.frame("path" = list.files(file_path, full.names = TRUE)) %>%
+data.frame("path" = list.files(file_path, full.names = TRUE)) %>%
   filter(
     str_detect(path, "Current", negate = TRUE) &
       str_detect(path, "20") &
       str_detect(path, "Admin")
-  )
-
-# Function to call referenced API, pull requested data, and write it to S3
-pwalk(gdb_files, function(...) {
-  df <- tibble::tibble(...)
-  county_gdb_to_s3(
-    s3_bucket_uri = output_bucket,
-    dir_name = "municipality",
-    file_path = df$path,
-    layer = "MuniTaxDist"
-  )
-})
+  ) %>%
+  # Function to call referenced API, pull requested data, and write it to S3
+  pwalk(function(...) {
+    df <- tibble::tibble(...)
+    county_gdb_to_s3(
+      s3_bucket_uri = output_bucket,
+      dir_name = "municipality",
+      file_path = df$path,
+      layer = "MuniTaxDist"
+    )
+  })

etl/scripts-ccao-data-raw-us-east-1/spatial/spatial-school.R

@@ -121,6 +121,12 @@ sources_list <- bind_rows(list(
     "boundary" = "cps_attendance_elementary",
     "year" = "2023-2024"
   ),
+  "attendance_ele_2025" = c(
+    "source" = "https://data.cityofchicago.org/api/geospatial/",
+    "api_url" = "5ihw-cbdn?method=export&format=GeoJSON",
+    "boundary" = "cps_attendance_elementary",
+    "year" = "2024-2025"
+  ),
 
   # CPS ATTENDANCE - SECONDARY
   "attendance_sec_0607" = c(
@@ -231,6 +237,12 @@ sources_list <- bind_rows(list(
     "boundary" = "cps_attendance_secondary",
     "year" = "2023-2024"
   ),
+  "attendance_sec_2025" = c(
+    "source" = "https://data.cityofchicago.org/api/geospatial/",
+    "api_url" = "4kfz-zr3a?method=export&format=GeoJSON",
+    "boundary" = "cps_attendance_secondary",
+    "year" = "2024-2025"
+  ),
 
   # LOCATION
   "locations_all_21" = c(

etl/scripts-ccao-data-raw-us-east-1/spatial/spatial-transit.R

@@ -20,15 +20,25 @@ options(timeout = max(300, getOption("timeout")))
 cta_feed_dates_list <- c(
   "2015-10-29", "2016-09-30", "2017-10-22", "2018-10-06",
   "2019-10-04", "2020-10-10", "2021-10-09", "2022-10-20",
-  "2023-10-04"
+  "2023-10-04", "2024-10-17"
 )
 
 # If missing feed on S3, download and remove .htm file (causes errors)
 # then rezip and upload
 get_cta_feed <- function(feed_date) {
-  feed_url <- paste0(
-    "https://transitfeeds.com/p/chicago-transit-authority/165/",
-    str_remove_all(feed_date, "-"), "/download"
+  feed_url <- ifelse(
+    substr(feed_date, 1, 4) <= "2023",
+    paste0(
+      "https://transitfeeds.com/p/chicago-transit-authority/165/",
+      str_remove_all(feed_date, "-"), "/download"
+    ),
+    paste0(
+      "https://files.mobilitydatabase.org/mdb-389/mdb-389-",
+      str_remove_all(feed_date, "-"),
+      "0023/mdb-389-",
+      str_remove_all(feed_date, "-"),
+      "0023.zip"
+    )
   )
   s3_uri <- file.path(output_path, "cta", paste0(feed_date, "-gtfs.zip"))
 
@@ -55,14 +65,26 @@ walk(cta_feed_dates_list, get_cta_feed)
 metra_feed_dates_list <- c(
   "2015-10-30", "2016-09-30", "2017-10-21", "2018-10-05",
   "2019-10-04", "2020-10-10", "2021-10-08", "2022-10-21",
-  "2023-10-14"
+  "2023-10-14", "2024-04-22"
 )
 
 get_metra_feed <- function(feed_date) {
-  feed_url <- paste0(
-    "https://transitfeeds.com/p/metra/169/",
-    str_remove_all(feed_date, "-"), "/download"
+
+  feed_url <- ifelse(
+    substr(feed_date, 1, 4) <= "2023",
+    paste0(
+      "https://transitfeeds.com/p/metra/169/",
+      str_remove_all(feed_date, "-"), "/download"
+    ),
+    paste0(
+      "https://files.mobilitydatabase.org/mdb-1187/mdb-1187-",
+      str_remove_all(feed_date, "-"),
+      "0016/mdb-1187-",
+      str_remove_all(feed_date, "-"),
+      "0016.zip"
+    )
   )
+
   s3_uri <- file.path(output_path, "metra", paste0(feed_date, "-gtfs.zip"))
 
   if (!aws.s3::object_exists(s3_uri)) {
@@ -80,7 +102,8 @@ walk(metra_feed_dates_list, get_metra_feed)
 ##### Pace #####
 pace_feed_dates_list <- c(
   "2015-10-16", "2016-10-15", "2017-10-16", "2018-10-17",
-  "2019-10-22", "2020-09-23", "2021-03-15", "2023-09-24"
+  "2019-10-22", "2020-09-23", "2021-03-15", "2023-09-24",
+  "2024-02-07"
 )
 
 get_pace_feed <- function(feed_date) {