index.Rmd

---
title: "Data Science News"
description: |
  A newsfeed of data science, tech, startups, and coding articles from around the web.
site: distill::distill_website
---

**Last updated `r format(lubridate::now("US/Pacific"), '%a, %b %d %I:%M %p')`** PST.

_This is news re-imagined_. Not really. It's simply an RSS news aggregator compiled into a few, interactive tables. Each table can be filtered by author (using `Feed` search), searched using regular expresssions (table search), or sorted by date or post. 

```{r setup, include=FALSE}
start_time = Sys.time()
knitr::opts_chunk$set(echo = FALSE, warning = FALSE, message = FALSE, layout='l-page')
# setwd("~/github/dsnews")
# renv::init()
# renv::activate()
# install.packages('install.load')
# distill needs to be installed, even if not loaded
# install.packages('distill')
install.load::install_load('tidyRSS', 'dplyr', 'readr', 'tidyverse', 'kableExtra', 'aRxiv', 'lubridate', 'magrittr', 'crayon', 'DT', 'rvest', 'stringr', 'crosstalk')

# colors for pander palette
# ggthemes::palette_pander(10)
# "#56B4E9" "#009E73" "#F0E442" "#0072B2" "#D55E00" "#CC79A7" "#999999" "#E69F00"
gray = "#999999"
orange = "#E69F00"
lt_blue ="#56B4E9"
green = "#009E73"
yellow = "#F0E442"
dk_blue = "#0072B2"
dk_orange = "#D55E00"

# other sources:
# twitter: top tweets
# todo: arxiv output

# Learn more about creating websites with Distill at:
# https://rstudio.github.io/distill/website.html

# Learn more about publishing to GitHub Pages at:
# https://rstudio.github.io/distill/publish_website.html#github-pages


```

```{r feeds}

# https://github.com/kilimchoi/engineering-blogs
# https://github.com/kilimchoi/engineering-blogs/blob/master/engineering_blogs.opml

# to fix:
# https://bair.berkeley.edu/blog/feed
# https://engineering.foursquare.com/feed
# http://engineering.shopify.com/blogs/engineering.atom
# https://instagram-engineering.com/feed

feeds = list(
  'engineering'= c(
    'openai'= 'https://blog.openai.com/rss/',
    'fb'= 'https://research.fb.com/feed/',
    'airbnb-ds'= 'https://medium.com/feed/airbnb-engineering/tagged/data-science',
    'ggl_ai'= 'http://feeds.feedburner.com/blogspot/gJZg',
    'instacart'= ' https://tech.instacart.com/feed',
    'Google Developers'= 'http://feeds.feedburner.com/GDBcode',
    'Google Open Source'= 'http://feeds.feedburner.com/GoogleOpenSourceBlog',
    'fb_code'= 'https://code.fb.com/feed/',
    'uber_ai'= 'https://eng.uber.com/tag/uber-ai-labs/feed',
    'uber_eng'= 'https://eng.uber.com/feed',
    'netflix_tech'= 'https://medium.com/feed/netflix-techblog',
    'pinterest'= 'https://medium.com/feed/@Pinterest_Engineering',
    'sebrash'= 'https://sebastianraschka.com/rss_feed.xml',
    'zillow'= 'https://www.zillow.com/tech/feed/',
    'acing-ai' = 'https://medium.com/feed/acing-ai',
    # https://dropbox.tech/machine-learning/feed
    # 'doordash' = 'https://doordash.engineering/category/data-science-and-machine-learning/feed/'
    'dropbox' = 'https://dropbox.tech/feed', 
    'doordash' = 'https://doordash.engineering/feed/',
    'yelp' = 'https://engineeringblog.yelp.com/feed.xml',
    # 'foursquare' = 'https://engineering.foursquare.com/feed',
    'squarespace'='https://engineering.squarespace.com/blog?format=RSS',
    # 'shopify' = 'http://engineering.shopify.com/blogs/engineering.atom',
    'microsoft'='https://devblogs.microsoft.com/python/feed/',
    'expedia'='https://medium.com/feed/expedia-group-tech',
    # 'instagram'='https://instagram-engineering.com/feed',
    'quickbooks'='https://quickbooks-engineering.intuit.com/feed',
    'kickstarter'='https://kickstarter.engineering/feed',
    'lyft'='https://eng.lyft.com/feed',
    'medium'='https://medium.com/feed/medium-eng',
    'slack'='https://slack.engineering/feed/',
    'spotify'='https://engineering.atspotify.com/feed/',
    'strava'='https://medium.com/feed/strava-engineering',
    'twitter'='https://blog.twitter.com/engineering/feed',
    'blackrock'='https://medium.com/feed/blackrock-engineering',
    'capitalone'='https://medium.com/feed/capital-one-tech'
  ),
  'tutorials'= c(
    # 'databricks'= 'https://databricks.com/feed',
    'datacamp'= 'https://www.datacamp.com/community/rss.xml',
    'ml_mast'= 'https://machinelearningmastery.com/blog/feed/',
    'twrds'= 'https://towardsdatascience.com/feed/',
    'devto'= 'https://dev.to/feed',
    'r-bloggers'='https://feeds.feedburner.com/RBloggers'
  ),
  'world'= c(
    'gnews'= 'https://news.google.com/news/rss/?hl=en&ned=us&gl=US&ned=us&gl=US',
    'espn'= 'http://www.espn.com/espn/rss/news',
    'Science'= 'http://feeds.reuters.com/reuters/scienceNews',
    'TopNews'= 'http://feeds.reuters.com/reuters/topNews',
    'World News'= 'http://feeds.reuters.com/Reuters/worldNews',
    'Sports News'= 'http://feeds.reuters.com/reuters/sportsNews',
    'BBC'= 'http://feeds.bbci.co.uk/news/video_and_audio/news_front_page/rss.xml',
    'BBC US'= 'http://feeds.bbci.co.uk/news/video_and_audio/news_front_page/rss.xml?edition=us',
    'BBC International'= 'http://feeds.bbci.co.uk/news/rss.xml?edition=int',
    'Snopes'= 'https://www.snopes.com/feed/'
  ),
  'tech'= c(
    'economist' = 'https://www.economist.com/science-and-technology/rss.xml',
    'dataconomy' = 'https://dataconomy.com/feed/',
    'mit'= 'https://www.technologyreview.com/stories.rss',
    'fc'= 'https://www.fastcompany.com/technology/rss',
    # 'reuters'= 'http://feeds.reuters.com/reuters/technologyNews',
    'bbc'= 'http://feeds.bbci.co.uk/news/video_and_audio/technology/rss.xml',
    'tc'= 'https://techcrunch.com/startups/feed/',
    'vb'= 'https://venturebeat.com/feed/'
  ),
  # https://www.economist.com/rss
  'economist' = c(
    'game theory' = 'https://www.economist.com/game-theory/rss.xml',
    'graphic' = 'https://www.economist.com/graphic-detail/rss.xml',
    'leaders' = 'https://www.economist.com/leaders/rss.xml',
    'business' = 'https://www.economist.com/business/rss.xml',
    'economist' = 'https://www.economist.com/science-and-technology/rss.xml',
    'finance-econ' = 'https://www.economist.com/finance-and-economics/rss.xml',
    'arts' = 'https://www.economist.com/books-and-arts/rss.xml',
    'indicators' = 'https://www.economist.com/economic-and-financial-indicators/rss.xml'
  ),
  'general' = c(
    # move these to "news"
    'datafloq'='https://datafloq.com/read/feed.rss',
    'kdd'='http://feeds.feedburner.com/kdnuggets-data-mining-analytics',
    'smartdata' = 'https://www.smartdatacollective.com/feed/',
    'insidebigdata' = 'https://insidebigdata.com/feed/'
  ),
  'dataviz' = c(
    'economist' = 'https://www.economist.com/graphic-detail/rss.xml',
    'flowingdata' = 'https://flowingdata.com/feed',
    'infoisbeaut' = 'http://feeds.feedburner.com/InformationIsBeautiful'
  ),
  'startups'= c(
    'tim_ferris'= 'https://tim.blog/feed/',
    'avc'= 'http://feeds.feedburner.com/avc',
    'andrew_chen'= 'https://andrewchen.co/feed/',
    'ycombinator'= 'https://blog.ycombinator.com/feed/',
    'A Horowitz'= 'https://a16z.com/feed/',
    'AVC'= 'https://avc.com/feed/',
    # 'Sam Altman'= 'http://blog.samaltman.com/posts.atom',
    'seth_godin' = 'https://seths.blog/feed/'
  ),
  'podcasts'=c(
    'talkpython'= 'https://talkpython.fm/episodes/rss',
    'data_eng_pod'='https://www.dataengineeringpodcast.com/feed/',
    'dataframed'='https://www.datacamp.com/community/rss.xml',
    'freakonomics'='http://feeds.feedburner.com/freakonomicsradio'
  ),
  'religious'= c(
    'lds'= 'https://www.mormonnewsroom.org/rss'
  )
)

```


```{r}
# df = tidyfeed('https://blog.openai.com/rss/')
# df = tidyfeed('https://www.datatau.com/rss/')

loop_feeds <- function(feed) {
  df = data.frame()
  for (f in feed){
    message(blue('Starting:', f))
    # try-catch: https://stackoverflow.com/a/12195574
    df_feed = tryCatch({
      tidyfeed(f)
    }, error=function(cond) {
      message(red$bold("URL does not seem to exist:", f, cond))
      return(NA)
    },
    warning=function(cond) {
      message(cat(red$bold("URL caused a warning:", f)))
      message("Here's the original warning message:")
      message(cond)
      # Choose a return value in case of warning
      return(NULL)
    }, finally={
      message(green("Processed URL:", f))
    }
    )
    # keep relevant names
    
    # print(names(df)); print(names(df_feed))
    if (is.data.frame(df_feed)){
      message(dim(df_feed), paste(names(df_feed), collapse = ', '))
      for (col in names(df)) {
        if (!(col %in% names(df_feed))) {df_feed[col] = NA}
      }
      # standardize the names - different feeds have different names
      if ('entry_title' %in% names(df_feed)) {df_feed$item_title = df_feed$entry_title}
      if ('entry_link' %in% names(df_feed)) {df_feed$item_link = df_feed$entry_link}
      if ('entry_content' %in% names(df_feed)) {df_feed$item_description = df_feed$entry_title}
      if ('entry_last_updated' %in% names(df_feed)) {df_feed$item_pub_date = df_feed$entry_last_updated}
      
      df <- df_feed %>% 
        dplyr::select(feed_title, feed_link,
               item_title, item_link, item_description, item_pub_date) %>%
        rbind(df, .)
    } else {
      cat('feed did not work: ', red$bold$underline(f))
    }
  }
  return(df)
}

process_feed <- function(df){
  df %>% 
    distinct() %>% # remove duplicated queries
    arrange(desc(item_pub_date)) %>%
    mutate(
      # add + 1 because I was getting dates as -1
      date_diff = round(difftime(Sys.Date() + 1, item_pub_date, unit='days')),
      date_diff = ifelse(date_diff == 0, "", paste0('<span style="color:gray">\n(-', round(date_diff), 'd)</span>')),
      blank_date = paste0('<p style="display:none">', item_pub_date, '</p>'),
      date_fmt = paste0(
        blank_date,
        format(item_pub_date, '%a %m/%d'),
        date_diff
      ),
      blog_title = cell_spec(feed_title, "html", link=feed_link, new_tab=T),
      # blog = paste0('<a href="', feed_title,  '" color="#fff" background-color:"#999999">'), 
      # blog = cell_spec(feed_title, "html",  link=feed_link,  color="#fff", underline=F, background=gray),
      linked_title = cell_spec(item_title, 'html', link=item_link, new_tab=T),
      post = paste(
        ifelse(
          is.na(item_description) | item_description == item_title, linked_title, 
          paste0(linked_title, 
                 '<details><summary>', 
                 substring(item_description, 1, 100),
                 '</summary>',
                 substring(item_description, 101, 500),
                 '</details>')
        )
      ),
      blank_rand = paste0('<p style="display:none">', runif(n()), '</p>'),
      # post = paste0(blank_date, post, " | ", date_fmt, " | ", blog)
      # post = paste(blog, cell_spec(date, 'html', link=item_link), item_description)
      # blog = paste0(blank_date, blog_title, '<br>', date_fmt),
      blog = paste0(blank_date, date_fmt),
      post = paste0(blank_rand, post)
    ) %>%
    mutate(rand=runif(n())) %>%
    dplyr::select(blog, post, blog_title, feed_title, rand) 
}
# df1_p <- process_feed(df1)
# make_dt(df1_p)

make_dt <- function(df) {
  datatable(
    df, 
    escape=F,
    rownames=F,
    colnames = c('Date', 'Post'),
    # class='compact',
    # extensions='Scroller',
    extensions=c('FixedHeader', 'RowGroup'),
    # search isn't necessary as global search is good enough
    # filter='top',
    options = list(
      # enable regular expressions search
      search = list(regex = TRUE, caseInsensitive = TRUE), 
      # hide random columns
      columnDefs = list(
        list(
          visible=FALSE, targets=c(2, 3, 4)
        )
        #, # extensions = SearchPanes
        # list(
        #   searchPanes = list(show = TRUE), targets = 2
        # )
      ),
      rowGroup=list(dataSrc=2),
      pageLength = 7,
      dom = 'flptr', #iP
      # pageLength = 1000,
      # dom = 'ft',
      lengthMenu = c(7, 50, 10000),
      fixedHeader=T,
      # deferRender=TRUE,
      # scrollY=300,
      # scroller=TRUE,
      
      # enforce column width
      # autoWidth = TRUE,
      # columnDefs = list(list(width = '50px', targets = c(0, 1))),
      
      # background color
      initComplete = JS(
        "function(settings, json) {",
        paste0("$(this.api().table().header()).css({'background-color': '", dk_blue, "', 'color': '#fff'});"),
        "}"
      )
    )
  )
}
# make_dt(df1_p)

# TODO: try out kable for neat columns (better mobile?)
# kable(list(x, x, x), "html", escape = FALSE) %>%
# kable_styling(bootstrap_options = c("hover", "condensed"))


# https://rstudio.github.io/crosstalk/
my_bscols <- function(sd){
  return(
    bscols(
      # filter_slider("rand", label="Random article filterer", 
      #               sd, column=~rand, step=0.01,
      #               min=0, max=1),
      filter_select("feed", label="Feed", sd, group=~feed_title)
    )
  )
}
```



# Tech + Engineering Blogs

```{r techeng}
all_df = data.frame()
df1 <- loop_feeds(feeds$engineering) 
df2 <- process_feed(df1) 
sd <- SharedData$new(df2)
my_bscols(sd)
make_dt(sd)
all_df = rbind(all_df, df1)
all_df['topic'] = '#tech #engineering'
```


# Tutorials

```{r tutorials, layout='l-page'}
# add datatau
df1 <- loop_feeds(feeds$tutorials) 

# TODO: scrape deepai.org and arXiv
# 
# read_html("http://www.deepai.org") %>%
#   # html_nodes(".card") %>%
#   html_nodes("#title") %>%
#   html_attr("href")
#   html_text
#   # html_nodes("#trending")
#   html_attrs()
#   # html_text()
# 
# x <- read_html("http://www.deepai.org") %>%
#   html_nodes('.card')
#   
# read_html("http://www.deepai.org") %>%
#   html_nodes(".title , .title a") %>%
#   html_attr('href')
#   html_text
#   


try({
  datatau <- read_html("http://www.datatau.com/rssxx") %>% 
    html_text() %>% 
    str_split("]]") %>%
    .[[1]] %>%
    str_match(., ">(.*)(https://.*)(http[s]?://.*)") %>% 
    as.data.frame() %>%
    dplyr::select(V2, V3) %>%
    rename(item_title=V2, item_link=V3) %>%
    drop_na() %>%
    as_tibble() %>%
    mutate(feed_title='DataTau', 
           feed_link='http://www.datatau.com', 
           item_pub_date = now())
  for (c in names(df1)) {if (!(c %in% names(datatau))) datatau[c] = NA}
  df1 <- df1 %>% rbind(datatau)
})

df2 <- process_feed(df1)
# produce table
sd <- SharedData$new(df2)
my_bscols(sd)
make_dt(sd)

df1['topic'] = '#datascience #tutorials'
all_df = rbind(all_df, df1)
```

# Data Viz

```{r dataviz}
df1 <- loop_feeds(feeds$dataviz) 
df2 <- process_feed(df1) 
sd <- SharedData$new(df2)
my_bscols(sd)
make_dt(sd)

df1['topic'] = '#dataviz'
all_df = rbind(all_df, df1)
```

# Tech news
```{r tech, layout='l-page'}
df1 <- loop_feeds(feeds$tech) 
df2 <- process_feed(df1) 
sd <- SharedData$new(df2)
my_bscols(sd)
make_dt(sd)
df1['topic'] = '#tech #news'
all_df = rbind(all_df, df1)
```

# Startup

```{r startup, layout='l-page'}
df1 <- loop_feeds(feeds$startup) 
df2 <- process_feed(df1) 
sd <- SharedData$new(df2)
my_bscols(sd)
make_dt(sd)
# df1['topic'] = 'startups'
# all_df = rbind(all_df, df1)
```

# Podcasts

```{r podcasts, layout='l-page'}
df1 <- loop_feeds(feeds$podcasts) 
df2 <- process_feed(df1) 
sd <- SharedData$new(df2)
my_bscols(sd)
make_dt(sd)
df1 %<>%
  mutate(topic = case_when(
    str_detect(feed_link, 'freak') ~ "#economics #podcast",
    TRUE ~ "#datascience #ds #podcast"
  ))
all_df = rbind(all_df, df1)
```


# Full Table of All DS news

This table combines all the news feeds above it into one, searchable table.

```{r all}
sd <- SharedData$new(process_feed(all_df))
my_bscols(sd)
make_dt(sd)
# save out 24 random articls
all_df %>%
  group_by(topic) %>%
  do(sample_n(., size=4)) %>%
  head(24) %>%
  write_csv(., 'output/random_articles.csv')
```


# Other

## Economist

```{r economist}
df1 <- loop_feeds(feeds$economist)
df2 <- process_feed(df1)
sd <- SharedData$new(df2)
my_bscols(sd)
make_dt(sd)

```

## World News

```{r world, layout='l-page'}
df1 <- loop_feeds(feeds$world) 
df2 <- process_feed(df1) 
sd <- SharedData$new(df2)
my_bscols(sd)
make_dt(sd)
```

_Page compile time: `r Sys.time() - start_time`_.