diff --git a/scripts/la-metro-crontask b/scripts/la-metro-crontask index 0b71262..8e03858 100644 --- a/scripts/la-metro-crontask +++ b/scripts/la-metro-crontask @@ -2,7 +2,7 @@ APPDIR=/home/datamade/scrapers-us-municipal PUPADIR=/home/datamade/.virtualenvs/opencivicdata/bin/pupa -# --- NIGHTLY --- +# --- SUNDAY THROUGH THURSDAY & SATURDAY --- # UTC: 12:05 am # CST: 6:05 pm @@ -10,7 +10,7 @@ PUPADIR=/home/datamade/.virtualenvs/opencivicdata/bin/pupa # Full scrape -5 0 * * * datamade /usr/bin/flock /tmp/metrobills.lock /usr/bin/flock /tmp/metroevents.lock $APPDIR/scripts/lametro/full-scrape.sh >> /tmp/lametro.log +5 0 * * 0-4,6 datamade /usr/bin/flock /tmp/metrobills.lock /usr/bin/flock /tmp/metroevents.lock $APPDIR/scripts/lametro/full-scrape.sh >> /tmp/lametro.log # --- SUNDAY THROUGH THURSDAY --- @@ -49,6 +49,14 @@ PUPADIR=/home/datamade/.virtualenvs/opencivicdata/bin/pupa 30,45 0-5 * * 6 datamade /usr/bin/flock -n /tmp/metroevents.lock -c "WINDOW=1 $APPDIR/scripts/lametro/windowed-event-scrape.sh" >> /tmp/lametro.log 35,50 0-5 * * 6 datamade /usr/bin/flock -n /tmp/metrobills.lock -c "WINDOW=1 $APPDIR/scripts/lametro/windowed-bill-scrape.sh" >> /tmp/lametro.log +# UTC: 12:05 am +# CST: 6:05 pm +# CDT: 7:05 pm + +# Person scrape, in lieu of nightly full scrape + +5 0 * * 5 datamade $APPDIR/scripts/lametro/person-scrape.sh >> /tmp/lametro.log + # --- SATURDAY --- # UTC: 6:00 am to 11:50 pm Saturday diff --git a/scripts/lametro/person-scrape.sh b/scripts/lametro/person-scrape.sh new file mode 100644 index 0000000..8d05d23 --- /dev/null +++ b/scripts/lametro/person-scrape.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -e + +exec 2>&1 + +cd $APPDIR +$PUPADIR update --datadir=/cache/people/_data/ lametro --scrape people +SHARED_DB=True DATABASE_URL=postgis://datamade@3.93.9.229/lametro $PUPADIR update --datadir=/cache/people/_data/ lametro --import people +SHARED_DB=True DATABASE_URL=postgis://datamade@3.93.9.229/lametro_staging $PUPADIR update --datadir=/cache/people/_data/ lametro --import people +$PUPADIR update --datadir=/cache/people/_data/ lametro --import people