diff --git a/README.md b/README.md index 15750ec..352cf58 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ For the following registrys: - [ ] [Ireland](http://www.ubss.org.uk/search_irishcaves.php) # Download -~~To download the output from this repository go to the releases page and download the file for the region you are interested in.~~ +To download the output from this repository go to the releases page and download the file for the region you are interested in. ## Accuracy All efforts have been made to ensure accuracy of the output files but not every registry entry is acurate to begin with. Not every registry entry lists WGS84 or NGR which means it has to be derived from the other if availaible which can introduce an error. If you chose to use this data for navigation its not my fault if you end up wandering around on leck fell lost in the mist! @@ -33,8 +33,8 @@ Docker is used manage the python enviroment. 1. First build the docker image `docker-compose build` 1. Then run the spiders with `docker-compose up` -To run scrapy commands run `docker-compose run scrapy scrapy `. +To run scrapy commands run `docker-compose run scrapy scrapy ` such as docker-compose run scrapy scrapy crawl ` to crawl a single registry. To have access to the containers shell run `docker-compose run scrapy /bin/bash` -You may need to reset the owner and permissions of files created by docker/scrapy using `sudo chmod` & `sudo chown` +You may need to reset the owner and permissions of files created by docker/scrapy using `sudo chmod` & `sudo chown` on linux. diff --git a/app/start.py b/app/start.py index 42cb02f..ceb6674 100644 --- a/app/start.py +++ b/app/start.py @@ -1,6 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess -from ukcavegis.spiders import dcaregistry, mendipregistry, scotlandregistry, cnccregistry +from ukcavegis.spiders import dcaregistry, mendipregistry, scotlandregistry, cnccregistry, cccregistry, fodccagregistry, ducregistry from scrapy.utils.project import get_project_settings process = CrawlerProcess(get_project_settings()) @@ -8,7 +8,7 @@ process.crawl(mendipregistry.MendipRegistry) process.crawl(scotlandregistry.ScotlandRegistry) process.crawl(cnccregistry.CnccRegistry) -process.crawl(cnccregistry.DucRegistry) -process.crawl(cnccregistry.FodccagRegistry) -process.crawl(cnccregistry.CccRegistry) +process.crawl(ducregistry.DucRegistry) +process.crawl(fodccagregistry.FodccagRegistry) +process.crawl(cccregistry.CccRegistry) process.start()