diff --git a/.env.local b/.env.local deleted file mode 100644 index 4f069f9aa..000000000 --- a/.env.local +++ /dev/null @@ -1,24 +0,0 @@ -DJANGO_SECRET_KEY=replacethiswithsomethingsecret -DATABASE_URL=postgis://postgres:postgres@postgres:5432/lametro -SEARCH_URL=http://elasticsearch:9200 -DEBUG=True -DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0 -SHOW_TEST_EVENTS=False -MERGE_HOST=https://datamade-metro-pdf-merger-testing.s3.amazonaws.com/ -MERGE_ENDPOINT=http://host.docker.internal:8080/api/experimental/dags/make_packet/dag_runs -FLUSH_KEY=super secret junk -REFRESH_KEY=something very secret -API_KEY=test api key -SMART_LOGIC_KEY=smartlogic api key -SMART_LOGIC_ENVIRONMENT=0ef5d755-1f43-4a7e-8b06-7591bed8d453 -ANALYTICS_TRACKING_CODE= -REMOTE_ANALYTICS_FOLDER= -SENTRY_DSN= -LOCAL_DOCKER=True -AWS_KEY= -AWS_SECRET= -RECAPTCHA_PUBLIC_KEY= -RECAPTCHA_PRIVATE_KEY= -GOOGLE_API_KEY= -AWS_STORAGE_BUCKET_NAME="la-metro-headshots-staging" -GOOGLE_SERVICE_ACCT_API_KEY= diff --git a/.env.local.example b/.env.local.example new file mode 100644 index 000000000..e4f8566f6 --- /dev/null +++ b/.env.local.example @@ -0,0 +1,11 @@ +SMART_LOGIC_KEY= +ANALYTICS_TRACKING_CODE= +REMOTE_ANALYTICS_FOLDER= +SENTRY_DSN= +AWS_S3_ACCESS_KEY_ID= +AWS_S3_SECRET_ACCESS_KEY= +AWS_STORAGE_BUCKET_NAME=la-metro-headshots-staging +RECAPTCHA_PUBLIC_KEY= +RECAPTCHA_PRIVATE_KEY= +GOOGLE_API_KEY= +GOOGLE_SERVICE_ACCT_API_KEY= diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6705f6281..080743c6c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,4 +64,5 @@ jobs: run: | flake8 . black --check . + cp .env.local.example .env.local pytest -sv diff --git a/.gitignore b/.gitignore index e1ccf481d..9be953be0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,15 @@ # Byte-compiled / optimized / DLL files .DS_Store -# Settings -settings_deployment.py - # Collected Static Files static/** **/static/images/ocd-person/*.jpg **/static/pdf/agenda-*.pdf -solr-4.10.4/** - **/__pycache__ .pytest_cache/ downloads -/keyrings/live/pubring.gpg~ -/keyrings/live/pubring.kbx~ -/keyrings/live/secring.gpg merged_pdfs/* debug.log @@ -27,15 +19,8 @@ debug.log *.csv !historical_events.csv !lametro_divisions.csv -.gnupg/ *_bk/ lametro/secrets.py -/configs/settings_deployment.upgrade.py - -/configs/upgrade-config.conf -/configs/upgrade.conf.nginx -/configs/upgrade.conf.supervisor -.env -.venv +.env.local diff --git a/Dockerfile b/Dockerfile index d73949415..1328e1347 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,3 @@ -FROM ubuntu:20.04 as builder - -# Clone and build Blackbox -RUN apt-get update && \ - apt-get install -y build-essential git-core && \ - git clone https://github.com/StackExchange/blackbox.git && \ - cd blackbox && \ - make copy-install - FROM python:3.10-slim-bullseye LABEL maintainer "DataMade " @@ -14,7 +5,7 @@ RUN apt-get update && \ apt-get install -y libpq-dev gcc gdal-bin gnupg && \ apt-get install -y libxml2-dev libxslt1-dev antiword unrtf poppler-utils \ tesseract-ocr flac ffmpeg lame libmad0 libsox-fmt-mp3 \ - sox libjpeg-dev swig libpulse-dev curl && \ + sox libjpeg-dev swig libpulse-dev curl git && \ apt-get clean && \ rm -rf /var/cache/apt/* /var/lib/apt/lists/* @@ -28,11 +19,6 @@ RUN pip install pip==24.0 && \ COPY . /app -# Copy Blackbox executables from builder stage -COPY --from=builder /usr/local/bin/blackbox* /usr/local/bin/ -COPY --from=builder /usr/local/bin/_blackbox* /usr/local/bin/ -COPY --from=builder /usr/local/bin/_stack_lib.sh /usr/local/bin/ - RUN DJANGO_SETTINGS_MODULE=councilmatic.minimal_settings python manage.py collectstatic ENTRYPOINT ["/app/docker-entrypoint.sh"] diff --git a/Dockerfile.dev b/Dockerfile.dev index 79a29a788..2420a532c 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,5 +1,8 @@ FROM ghcr.io/metro-records/la-metro-councilmatic:main +RUN apt-get update && \ + apt-get install -y git + COPY ./requirements.txt /app/requirements.txt RUN pip install pip==24.0 && \ pip install --no-cache-dir -r requirements.txt diff --git a/README.md b/README.md index e170f3ff6..50e33cca4 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Metro Board Reports is a member of the [Councilmatic family](https://www.council ## Setup -These days, we run apps in containers for local development. More on that [here](https://github.com/datamade/how-to/docker/local-development.md). Prefer to run the app locally? See the [legacy setup instructions](https://github.com/datamade/la-metro-councilmatic/blob/b8bc14f6d90f1b05e24b5076b1bfcd5e0d37527a/README.md). +These days, we run apps in containers for local development. More on that [here](https://github.com/datamade/how-to/docker/local-development.md). ### Set-up pre-commit @@ -38,37 +38,24 @@ to set up the git hooks. Since hooks are run locally, you can modify which scripts are run before each commit by modifying `.pre-commit-config.yaml`. -### Get the API key +### Get the Legistar API key There should be an entry in the DataMade LastPass account called 'LA Metro - secrets.py.' Copy its contents into a file called `secrets.py` and place it in `lametro/`. -### Generate the deployment settings - -Run `cp councilmatic/settings_deployment.py.example councilmatic/settings_deployment.py`. - ### Install OS level dependencies: * [Docker](https://www.docker.com/get-started) -### Run the application - -```bash -docker-compose up -d -``` - -Note that you can omit the `-d` flag to follow the application and service logs. If you prefer a quieter environment, you can view one log stream at a time with `docker-compose logs -f SERVICE_NAME`, where `SERVICE_NAME` is the name of one of the services defined in `docker-compose.yml`, e.g., `app`, `postgres`, etc. - -When the command exits (`-d`) or your logs indicate that your app is up and running, visit http://localhost:8001 to visit your shiny, new local application! - ### Load in the data The Metro app ingests updated data from the Legistar API several times an hour. -To import data into your local instance, first decrypt the bundled `secrets.py` file, so you can -retrieve information about private bills. +To import data into your local instance, copy DataMade's Legistar API token from +Bitwarden (search `LA Metro Legistar API Token`). Next, create a copy of the secrets +file and paste in the token: -``` -blackbox_decrypt_file lametro/secrets.py.gpg +```bash +cp lametro/secrets.py.example lametro/secrets.py ``` Then, simply run: @@ -80,19 +67,33 @@ docker-compose run --rm scrapers This may take a few minutes to an hour, depending on the volume of recent updates. -Once it's finished, head over to http://localhost:8001 to view your shiny new app! +### Run the application + +First, create your own local env file: + +```bash +cp .env.local.example .env.local +``` + +Next, bring up the app: + +```bash +docker-compose up +``` + +When your logs indicate that your app is up and running, visit http://localhost:8001 to visit your shiny, new local application! ### Optional: Populate the search index If you wish to use search in your local install, you need a SmartLogic API -key. Initiated DataMade staff may decrypt application secrets for use: +key. Initiated DataMade staff may retrieve values for the `SMART_LOGIC_ENVIRONMENT` +and `SMART_LOGIC_KEY` environment variables from Heroku: ```bash -blackbox_cat configs/settings_deployment.staging.py +heroku config:get SMART_LOGIC_ENVIRONMENT SMART_LOGIC_KEY -a la-metro-councilmatic-staging ``` -Grab the `SMARTLOGIC_API_KEY` value from the decrypted settings, and swap it -into your local `councilmatic/settings_deployment.py` file. +Paste these values into your `.env.local` file. Then, run the `refresh_guid` management command to grab the appropriate classifications for topics in the database. @@ -109,8 +110,7 @@ Haystack. docker-compose run --rm app python manage.py update_index ``` -When the command exits, your search index has been filled. (You can view the -Solr admin panel at http://localhost:8987/solr.) +When the command exits, your search index has been filled. ## Running arbitrary scrapes Occasionally, after a while without running an event scrape, you may find that your local app is broken. If this happens, make sure you have events in your database that are scheduled for the future, as the app queries for upcoming events in order to render the landing page. @@ -135,133 +135,21 @@ It's sometimes helpful to make sure you have a specific bill in your database fo docker-compose run --rm scrapers pupa update lametro bills matter_ids= --rpm=0 ``` -## Making changes to the Solr schema - -Did you make a change to the schema file that Solr uses to make its magic (`solr_configs/conf/schema.xml`)? Did you add a new field or adjust how Solr indexes data? If so, you need to take a few steps – locally and on the server. - -### Local development - -First, remove your Solr container. - -```bash -# Remove your existing Metro containers and the volume containing your Solr data -docker-compose down -docker volume rm la-metro-councilmatic_lametro-solr-data - -# Build the containers anew -docker-compose up -d -``` - -Then, rebuild your index. - -```bash -python manage.py refresh_guid # Run if you made a change to facets based on topics -docker-compose run --rm app python manage.py rebuild_index --batch-size=50 -``` - -### On the Server - -The Dockerized versions of Solr on the server need your attention, too. Perform -the following steps first on staging, then – after confirming that everything -is working as expected – on production. - -1. Deploy your changes to the appropriate environment (staging or production). - - To deploy to staging, merge the relevant PR into `main`. - - To deploy to production, [create and push a tag](https://github.com/datamade/deploy-a-site/blob/master/How-to-deploy-with-continuous-deployment.md#3-deploy-to-production). - -2. Shell into the server, and `cd` into the relevant project directory. - ```bash - ssh ubuntu@boardagendas.metro.net - - # Staging project directory: lametro-staging - # Production project directory: lametro - cd /home/datamade/${PROJECT_DIRECTORY} - ``` - -3. Remove and restart the Solr container. - ```bash - # Staging Solr container: lametro-staging-solr - # Production Solr container: lametro-production-solr - sudo docker stop ${SOLR_CONTAINER} - sudo docker rm ${SOLR_CONTAINER} - - sudo docker-compose -f docker-compose.deployment.yml up -d ${SOLR_CONTAINER} - ``` - -4. Solr will only apply changes to the schema and config upon core creation, so -consult the Solr logs to confirm the core was remade. - ```bash - # Staging Solr service: solr-staging - # Production Solr service: solr-production - sudo docker-compose -f docker-compose.deployment.yml logs -f ${SOLR_SERVICE} - ``` - - You should see logs resembling this: - - ```bash - Attaching to ${SOLR_CONTAINER} - Executing /opt/docker-solr/scripts/solr-create -c ${SOLR_CORE} -d /la-metro-councilmatic_configs - Running solr in the background. Logs are in /opt/solr/server/logs - Waiting up to 180 seconds to see Solr running on port 8983 [\] - Started Solr server on port 8983 (pid=64). Happy searching! - - Solr is running on http://localhost:8983 - Creating core with: -c ${SOLR_CORE} -d /la-metro-councilmatic_configs - INFO - 2020-11-18 13:57:09.874; org.apache.solr.util.configuration.SSLCredentialProviderFactory; Processing SSL Credential Provider chain: env;sysprop - - Created new core '${SOLR_CORE}' <---- IMPORTANT MESSAGE - Checking core - ``` - - If you see something like "Skipping core creation", you need to perform the - additional step of recreating the Solr core. - - ```bash - # Staging Solr core: lametro-staging - # Production Solr core: lametro - sudo docker exec ${SOLR_CONTAINER} solr delete -c ${SOLR_CORE} - sudo docker exec ${SOLR_CONTAINER} solr-create -c ${SOLR_CORE} -d /la-metro-councilmatic_configs - ``` - - Note that we remove and recreate the core, rather than the blunt force - option of removing the Docker volume containg the Solr data, because the - staging and production Solr containers use the same volume, so removing it - would wipe out both indexes at once. - -5. Switch to the `datamade` user. - ```bash - sudo su - datamade - ``` - -6. Rebuild the index: - ```bash - # Staging and production virtual environments are named after the corresponding project directory - source ~/.virtualenvs/${PROJECT_DIRECTORY}/bin/activate - python manage.py refresh_guid # Run if you made a change to facets based on topics - python manage.py rebuild_index --batch-size=50 - ``` - -Nice! The production server should have the newly edited schema and freshly -built index, ready to search, filter, and facet. - ## Connecting to AWS S3 for development If you want to use the S3 bucket, you’ll need the AWS S3 API keys. This can be found by running: - ```bash - blackbox_cat configs/settings_deployment.staging.py - ``` +```bash +heroku config:get AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY -a la-metro-councilmatic-staging +``` -Grab the values for the `AWS_S3_ACCESS_KEY_ID` and the `AWS_S3_SECRET_ACCESS_KEY`. Then, find/create your `.env` file in the `councilmatic/` folder and paste in your values, naming them `ACCESS_KEY` and `SECRET_KEY` respectively. +Grab the values for the `AWS_ACCESS_KEY_ID` and the `AWS_SECRET_ACCESS_KEY` and add them to your `.env.local` file. Now you should be able to start uploading some files! ## Adding a new board member Hooray! A new member has been elected or appointed to the Board of Directors. -Metro will provide a headshot and bio for the new member. There are a few -changes you need to make so they appear correctly on the site. - -**N.b., these changes can be made in any order.** +There are a few changes you need to make so they appear correctly on the site. ### Update the scraper @@ -275,14 +163,6 @@ new member that were created without a post. Person.objects.get(family_name='').memberships.filter(organization__name='Board of Directors', post__isnull=True).delete() ``` -### Update the Metro app - -- Add the new member's headshot to the `lametro/static/images/manual-headshots` -directory. **Be sure to follow the naming convention `${given_name}-${family_name}.jpg`, all lowercase with punctuation stripped.** -- Add the new member's bio to the `MEMBER_BIOS` object in `councilmatic/settings_jurisdiction.py`, again **following the `${given_name}-${family_name}.jpg` naming convention.** - - Example: https://github.com/datamade/la-metro-councilmatic/pull/686 - - Tip: Replace newlines in the provided bio with `

`. - ### Check your work To confirm your changes worked, run the app locally and confirm the following: @@ -295,10 +175,6 @@ listing and confirm the new member is listed with the correct post, e.g., scraper (e.g., does the member's name as it appears in the API match the key you added?), that your changes to the scraper have been deployed, and that a person scrape has been run since the deployment. -- View the new member's detail page and confirm that their headshot and bio -appear as expected, and without any formatting issues. - -If everything looks good, you can deploy to staging, check again, then push the changes to the live site. ## A note on tests diff --git a/app.json b/app.json index 8ad634e74..bb2d15e35 100644 --- a/app.json +++ b/app.json @@ -7,9 +7,135 @@ "size": "basic" } }, + "env": { + "DJANGO_SECRET_KEY": { + "description": "A key for Django operations requiring a crytographic signature.", + "required": true + }, + "DJANGO_DEBUG": { + "required": true + }, + "DJANGO_ALLOWED_HOSTS": { + "required": true + }, + "DATABASE_URL": { + "description": "URI for PostGIS database. Provisioned in deployment by Heroku.", + "required": true + }, + "SEARCH_URL": { + "description": "URI for ElasticSearch instance. Provisioned in deployment by Heroku.", + "required": true + }, + "SHOW_TEST_EVENTS": { + "description": "Whether to show test events on the website.", + "required": true + }, + "MERGE_HOST": { + "description": "URL for S3 Bucket containing merged document packets.", + "required": true + }, + "MERGE_ENDPOINT": { + "description": "Airflow API endpoint to trigger packet merge.", + "required": true + }, + "FLUSH_KEY": { + "description": "Key to clear cache.", + "required": true + }, + "REFRESH_KEY": { + "description": "Key to trigger refresh_guid management command.", + "required": true + }, + "API_KEY": { + "description": "Key to access API endpoints.", + "required": true + }, + "SMART_LOGIC_ENVIRONMENT": { + "description": "Provide GUID and API key for SmartLogic instance to enable tag classification and search autocomplete.", + "required": true + }, + "SMART_LOGIC_KEY": { + "description": "Provide GUID and API key for SmartLogic instance to enable tag classification and search autocomplete.", + "required": true + }, + "ANALYTICS_TRACKING_CODE": { + "description": "Provide ID to enable Google Analytics.", + "required": false + }, + "SENTRY_DSN": { + "description": "Provide DSN to enable Sentry logging.", + "required": false + }, + "AWS_S3_ACCESS_KEY_ID": { + "description": "Provide AWS IAM access key ID and secret access key to enable headshot uploads to S3.", + "required": false + }, + "AWS_S3_SECRET_ACCESS_KEY": { + "description": "Provide AWS IAM access key ID and secret access key to enable headshot uploads to S3.", + "required": false + }, + "AWS_STORAGE_BUCKET_NAME": { + "description": "Bucket to upload headshots.", + "required": false + }, + "RECAPTCHA_PUBLIC_KEY": { + "description": "Provide public and private keys to enable ReCAPTCHA.", + "required": false + }, + "RECAPTCHA_PRIVATE_KEY": { + "description": "Provide public and private keys to enable ReCAPTCHA.", + "required": false + }, + "REMOTE_ANALYTICS_FOLDER": { + "description": "Google Drive location to upload monthly tag analytics reports.", + "required": false + }, + "GOOGLE_SERVICE_ACCT_API_KEY": { + "description": "Provider Google service account API key to enable tag analytics reporting.", + "required": false + }, + "GOOGLE_API_KEY": { + "description": "Provide Google API key to enable map rendering.", + "required": false + } + }, "environments": { "review": { - "addons": ["heroku-postgresql:mini"] + "env": { + "DJANGO_SECRET_KEY": { + "value": "hecatomb kaput rustle roisterer division literacy" + }, + "DJANGO_DEBUG": { + "value": "True" + }, + "DJANGO_ALLOWED_HOSTS": { + "value": ".herokuapp.com" + }, + "FLUSH_KEY": { + "value": "flushitallaway" + }, + "REFRESH_KEY": { + "value": "hittherefreshbutton" + }, + "API_KEY": { + "value": "testapikey" + }, + "SHOW_TEST_EVENTS": { + "value": "True" + }, + "SMART_LOGIC_ENVIRONMENT": { + "value": "d3807554-347e-4091-90ea-f107a906aaff" + }, + "MERGE_HOST": { + "value": "https://datamade-metro-pdf-merger-testing.s3.amazonaws.com/" + }, + "AWS_STORAGE_BUCKET_NAME": { + "value": "la-metro-headshots-staging" + }, + "LOCAL_DOCKER": { + "value": "False" + } + } } }, "buildpacks": [], diff --git a/configs/test_settings_deployment.py b/configs/test_settings_deployment.py deleted file mode 100644 index d8e4c3fc1..000000000 --- a/configs/test_settings_deployment.py +++ /dev/null @@ -1,115 +0,0 @@ -# These are all the settings that are specific to a deployment -import os -import sys - -import dj_database_url - - -ALLOWED_HOSTS = [ - "localhost", - "127.0.0.1", - "0.0.0.0", -] - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = "testing secrets" - -# SECURITY WARNING: don't run with debug turned on in production! -# Set this to True while you are developing -DEBUG = True - -# Database -# https://docs.djangoproject.com/en/1.8/ref/settings/#databases -if os.getenv("DATABASE_URL"): - DATABASES = { - "default": dj_database_url.config(default=os.environ["DATABASE_URL"]), - } -else: - DATABASES = { - "default": { - "ENGINE": "django.contrib.gis.db.backends.postgis", - "NAME": "lametro", - "USER": "postgres", - "PASSWORD": "postgres", - "HOST": "localhost", - "PORT": 5432, - }, - } - -HAYSTACK_CONNECTIONS = { - "default": { - "ENGINE": "haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine", - "URL": "http://elasticsearch:9200", - "INDEX_NAME": "lametro", - "SILENTLY_FAIL": False, - "BATCH_SIZE": 10, - } -} -HAYSTACK_SIGNAL_PROCESSOR = "haystack.signals.RealtimeSignalProcessor" -HAYSTACK_IDENTIFIER_METHOD = "lametro.utils.get_identifier" - -# Remember to run python manage.py createcachetable so this will work! -# developers, set your BACKEND to 'django.core.cache.backends.dummy.DummyCache' -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.db.DatabaseCache", - "LOCATION": "councilmatic_cache", - } -} - -MEDIA_ROOT = "/media" - -# Set this to flush the cache at /flush-cache/{FLUSH_KEY} -FLUSH_KEY = "super secret junk" -API_KEY = "test api key" - -# Set this to allow Disqus comments to render -DISQUS_SHORTNAME = None - -HEADSHOT_PATH = os.path.join(os.path.dirname(__file__), ".." "/lametro/static/images/") - -SHOW_TEST_EVENTS = False - -AWS_STORAGE_BUCKET_NAME = "la-metro-headshots-staging" - -SMART_LOGIC_KEY = "smartlogic api key" -SMART_LOGIC_ENVIRONMENT = "0ef5d755-1f43-4a7e-8b06-7591bed8d453" - -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "console": { - "format": "[%(asctime)s][%(levelname)s] %(name)s " - "%(filename)s:%(funcName)s:%(lineno)d | %(message)s", - "datefmt": "%H:%M:%S", - }, - }, - "handlers": { - "console": { - "level": "INFO", - "class": "logging.StreamHandler", - "formatter": "console", - "stream": sys.stdout, - }, - }, - "root": {"handlers": ["console"], "level": "INFO"}, - "loggers": { - "lametro": { - "handlers": ["console"], - "propagate": True, - }, - }, -} - -# Allow some html tags to render in markdown. Mainly for alerts -MARKDOWNIFY = { - "default": { - "WHITELIST_TAGS": [ - "br", - "strong", - "em", - "a", - ] - } -} diff --git a/councilmatic/custom_storage.py b/councilmatic/custom_storage.py deleted file mode 100644 index 9b4bb9654..000000000 --- a/councilmatic/custom_storage.py +++ /dev/null @@ -1,6 +0,0 @@ -from storages.backends.s3boto3 import S3Boto3Storage -from django.conf import settings - - -class MediaStorage(S3Boto3Storage): - bucket_name = settings.AWS_STORAGE_BUCKET_NAME diff --git a/councilmatic/settings.py b/councilmatic/settings.py index 3d5e46870..9cdc823b1 100644 --- a/councilmatic/settings.py +++ b/councilmatic/settings.py @@ -8,8 +8,33 @@ env = environ.Env( # Set default values - DEBUG=(bool, False), - LOCAL_DOCKER=(bool, False), + LOCAL_DOCKER=(bool, True), + DJANGO_SECRET_KEY=(str, "replacethiswithsomethingsecret"), + DJANGO_DEBUG=(bool, True), + DJANGO_ALLOWED_HOSTS=(list, ["localhost", "127.0.0.1", "0.0.0.0"]), + DATABASE_URL=(str, "postgis://postgres:postgres@postgres:5432/lametro"), + SEARCH_URL=(str, "http://elasticsearch:9200"), + SHOW_TEST_EVENTS=(bool, False), + MERGE_HOST=(str, "https://datamade-metro-pdf-merger-testing.s3.amazonaws.com/"), + MERGE_ENDPOINT=( + str, + "http://host.docker.internal:8080/api/experimental/dags/make_packet/dag_runs", + ), + FLUSH_KEY=(str, "super secret junk"), + REFRESH_KEY=(str, "something very secret"), + API_KEY=(str, "test api key"), + SMART_LOGIC_ENVIRONMENT=(str, "d3807554-347e-4091-90ea-f107a906aaff"), + SMART_LOGIC_KEY=(str, ""), + ANALYTICS_TRACKING_CODE=(str, ""), + SENTRY_DSN=(str, ""), + AWS_S3_ACCESS_KEY_ID=(str, ""), + AWS_S3_SECRET_ACCESS_KEY=(str, ""), + AWS_STORAGE_BUCKET_NAME=(str, ""), + RECAPTCHA_PUBLIC_KEY=(str, ""), + RECAPTCHA_PRIVATE_KEY=(str, ""), + REMOTE_ANALYTICS_FOLDER=(str, ""), + GOOGLE_SERVICE_ACCT_API_KEY=(str, ""), + GOOGLE_API_KEY=(str, ""), ) # Core Django Settings @@ -19,7 +44,7 @@ environ.Env.read_env(os.path.join(BASE_DIR, ".env.local")) SECRET_KEY = env("DJANGO_SECRET_KEY") -DEBUG = env.bool("DEBUG") +DEBUG = env.bool("DJANGO_DEBUG") SHOW_TEST_EVENTS = env.bool("SHOW_TEST_EVENTS") ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS") @@ -185,11 +210,23 @@ GOOGLE_API_KEY = env("GOOGLE_API_KEY") # - AWS -AWS_KEY = env("AWS_KEY") -AWS_SECRET = env("AWS_SECRET") +AWS_S3_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID") +AWS_S3_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY") +AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME") + +if AWS_S3_ACCESS_KEY_ID and AWS_S3_SECRET_ACCESS_KEY: + print( + f"AWS configured. Uploading and retrieving headshots from {AWS_STORAGE_BUCKET_NAME}..." + ) + from django.core.files.storage import get_storage_class -# SITE CONFIG -HEADSHOT_PATH = os.path.join(os.path.dirname(__file__), ".." "/lametro/static/images/") + S3Storage = get_storage_class("storages.backends.s3boto3.S3Boto3Storage") + + AWS_QUERYSTRING_AUTH = False + COUNCILMATIC_HEADSHOT_STORAGE_BACKEND = S3Storage() + +else: + print("AWS not configured. Defaulting to local storage...") # LOGGING SENTRY_DSN = env("SENTRY_DSN") @@ -259,7 +296,6 @@ def custom_sampler(ctx): }, } -AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME") # Allow some html tags to render in markdown. Mainly for alerts MARKDOWNIFY = { diff --git a/councilmatic/settings_deployment.py.example b/councilmatic/settings_deployment.py.example deleted file mode 100644 index 1b76b074d..000000000 --- a/councilmatic/settings_deployment.py.example +++ /dev/null @@ -1,142 +0,0 @@ -# These are all the settings that are specific to a deployment - -import os -import sys - -ALLOWED_HOSTS = [ - 'localhost', - '127.0.0.1', - '0.0.0.0', -] - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'replacethiswithsomethingsecret' - -# SECURITY WARNING: don't run with debug turned on in production! -# Set this to True while you are developing -DEBUG = True - -SENTRY_DSN = "" -SENTRY_ENVIRONMENT = "dev" - -# Database -# https://docs.djangoproject.com/en/1.8/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'lametro', - 'USER': 'postgres', - 'HOST': 'postgres', - 'PASSWORD': '', - 'PORT': 5432, - } -} - -HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine', - 'URL': 'http://elasticsearch:9200', - 'INDEX_NAME': 'lametro', - 'SILENTLY_FAIL': False, - 'BATCH_SIZE': 10, - }, -} - -# Remember to run python manage.py createcachetable so this will work! -# Developers, set your BACKEND to 'django.core.cache.backends.dummy.DummyCache' to run the app locally -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - 'LOCATION': 'councilmatic_cache', - } -} - -MEDIA_ROOT = '/media' - -# Set this to flush the cache at /flush-cache/{FLUSH_KEY} -FLUSH_KEY = 'super secret junk' - -# Set this to allow updating of /refresh-guid/{REFRESH_KEY} -REFRESH_KEY = 'something very secret' - -# Set this to allow access to /object-counts/{API_KEY} -API_KEY = 'test key' - -# Set this to allow Disqus comments to render -DISQUS_SHORTNAME = None - -HEADSHOT_PATH = os.path.join(os.path.dirname(__file__), '..' - '/lametro/static/images/') - -EXTRA_APPS = () - -# Use standard logging module to catch errors in import_data (which uses a 'logger') - -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'console': { - 'format': '[%(asctime)s][%(levelname)s] %(name)s ' - '%(filename)s:%(funcName)s:%(lineno)d | %(message)s', - 'datefmt': '%H:%M:%S', - }, - }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'console', - 'stream': sys.stdout - }, - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG' - }, - 'loggers': { - 'lametro': { - 'handlers': ['console'], - 'propagate': False, - }, - } -} - -# Set to False in production! -SHOW_TEST_EVENTS = True - -import socket -hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) -INTERNAL_IPS = [ip[:-1] + '1' for ip in ips] + ['127.0.0.1', '10.0.2.2'] - -SMART_LOGIC_KEY = 'smartlogic api key' -SMART_LOGIC_ENVIRONMENT = 'd3807554-347e-4091-90ea-f107a906aaff' - -# Populate and uncomment to add live reCAPTCHA calls (defaults to always valid test calls) -# RECAPTCHA_PUBLIC_KEY = '' -# RECAPTCHA_PRIVATE_KEY = '' - -# Django-storages S3 settings -# Uncomment and set secret values in a .env file as documented in the README to enable -# remote storage -# AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") -# AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") -AWS_STORAGE_BUCKET_NAME = "la-metro-headshots-staging" - -MERGE_HOST = 'https://datamade-metro-pdf-merger-testing.s3.amazonaws.com/' - -# Assumes a locally running instance of the dashboard -MERGE_ENDPOINT = 'http://host.docker.internal:8080/api/experimental/dags/make_packet/dag_runs' - -# Allow some html tags to render in markdown. Mainly for alerts -MARKDOWNIFY = { - "default": { - "WHITELIST_TAGS": [ - "br", - "strong", - "em", - "a", - ] - } -} diff --git a/docker-compose.deployment.yml b/docker-compose.deployment.yml deleted file mode 100644 index d9474ff30..000000000 --- a/docker-compose.deployment.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: '3.3' -services: - solr-production: - image: solr:8.8 - container_name: solr-production - volumes: - - ./solr_configs:/la-metro-councilmatic_configs - - lametro-solr-data:/var/solr/data/ - command: sh -c 'solr-create -c lametro -d /la-metro-councilmatic_configs' - ports: - - '0.0.0.0:8985:8983' - environment: - - SOLR_LOG_LEVEL=ERROR - - SOLR_HEAP=1g - restart: on-failure - solr-staging: - image: solr:8.8 - container_name: solr-staging - volumes: - - ./solr_configs:/la-metro-councilmatic_configs - - lametro-solr-data:/var/solr/data/ - command: sh -c 'solr-create -c lametro-staging -d /la-metro-councilmatic_configs' - ports: - - '0.0.0.0:8986:8983' - environment: - SOLR_LOG_LEVEL: ERROR - restart: on-failure - # TODO: Remove post upgrade - solr-upgrade: - image: solr:8.8 - container_name: solr-upgrade - volumes: - - ./solr_configs:/la-metro-councilmatic_configs - - lametro-solr-data:/var/solr/data/ - command: sh -c 'solr-create -c lametro-staging -d /la-metro-councilmatic_configs' - ports: - - '0.0.0.0:8986:8983' - environment: - - SOLR_LOG_LEVEL=ERROR - restart: on-failure -volumes: - lametro-solr-data: - name: lametro-solr-data diff --git a/docker-compose.yml b/docker-compose.yml index b3ba56ed9..ea8f69521 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,6 @@ services: app: env_file: - .env.local - #image: ghcr.io/datamade/la-metro-councilmatic:hcg-upgrade build: dockerfile: Dockerfile.dev context: . @@ -43,7 +42,6 @@ services: ports: - 32006:5432 - elasticsearch: image: elasticsearch:7.14.2 container_name: lametro-councilmatic-elasticsearch diff --git a/lametro/forms.py b/lametro/forms.py index dba110ab7..7fdc7f6e2 100644 --- a/lametro/forms.py +++ b/lametro/forms.py @@ -160,7 +160,7 @@ class PersonHeadshotForm(forms.ModelForm): def __init__(self, *args, **kwargs): super(PersonHeadshotForm, self).__init__(*args, **kwargs) - self.fields["image"].widget.attrs.update( + self.fields["headshot"].widget.attrs.update( { "required": "True", } @@ -168,7 +168,7 @@ def __init__(self, *args, **kwargs): class Meta: model = LAMetroPerson - fields = ["image"] + fields = ["headshot"] class PersonBioForm(forms.ModelForm): diff --git a/lametro/models.py b/lametro/models.py index f738b518a..a3833731d 100644 --- a/lametro/models.py +++ b/lametro/models.py @@ -454,11 +454,7 @@ def ceo(cls): @property def headshot_url(self): if self.headshot: - # Assigning this to image_url would make 'return static(image_url)' - # at the end of this property concatenate 'static' to the url. - # Returning here solves that. - if "headshot_placeholder.png" not in self.headshot.path: - return self.headshot + return self.headshot.url file_directory = os.path.dirname(__file__) absolute_file_directory = os.path.abspath(file_directory) @@ -470,11 +466,7 @@ def headshot_url(self): ) if Path(manual_headshot).exists(): - image_url = "images/manual-headshots/" + filename - - elif self.headshot: - image_url = self.headshot.url - + image_url = f"images/manual-headshots/{filename}" else: image_url = "images/headshot_placeholder.png" diff --git a/lametro/secrets.py.example b/lametro/secrets.py.example new file mode 100644 index 000000000..e69de29bb diff --git a/lametro/secrets.py.gpg b/lametro/secrets.py.gpg deleted file mode 100644 index d32985184..000000000 Binary files a/lametro/secrets.py.gpg and /dev/null differ diff --git a/lametro/views.py b/lametro/views.py index b6205d1df..39382e513 100644 --- a/lametro/views.py +++ b/lametro/views.py @@ -74,7 +74,6 @@ from councilmatic.settings_jurisdiction import MEMBER_BIOS, BILL_STATUS_DESCRIPTIONS from councilmatic.settings import PIC_BASE_URL -from councilmatic.custom_storage import MediaStorage from opencivicdata.legislative.models import EventDocument @@ -668,7 +667,7 @@ def post(self, request, *args, **kwargs): self.get_context_data(form=form, headshot_error=error) ) - person.headshot = self.get_file_url(request, file_obj) + person.headshot = file_obj person.save() cache.clear() @@ -685,22 +684,6 @@ def validate_file(self, file): return True return False - def get_file_url(self, request, file): - # Save file in bucket and return the resulting url - - file_dir_within_bucket = "user_upload_files/{username}".format( - username=request.user - ) - - # create full file path - file_path_within_bucket = os.path.join(file_dir_within_bucket, file.name) - - media_storage = MediaStorage() - - media_storage.save(file_path_within_bucket, file) - file_url = media_storage.url(file_path_within_bucket) - return file_url - def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) person = context["person"] diff --git a/requirements.txt b/requirements.txt index 497635730..a224819c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -django-councilmatic[convert_docs]>=3.0 +django-councilmatic[convert_docs]>=3.3 https://github.com/opencivicdata/python-legistar-scraper/zipball/master Django>=3.2,<3.3 psycopg2-binary>=2.8