Skip to content
This repository has been archived by the owner on Jun 10, 2024. It is now read-only.

add phantomjs proxy support #692

Merged
merged 1 commit into from
Apr 18, 2017
Merged

add phantomjs proxy support #692

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pyspider/fetcher/phantomjs_fetcher.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,12 @@ if (system.args.length !== 2) {

// create and set page
var page = webpage.create();
if (fetch.proxy) {
if (fetch.proxy.indexOf('://') == -1){
fetch.proxy = 'http://' + fetch.proxy
}
page.setProxy(fetch.proxy);
}
page.onConsoleMessage = function(msg) {
console.log('console: ' + msg);
};
Expand Down
4 changes: 0 additions & 4 deletions pyspider/libs/base_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,10 +329,6 @@ def _crawl(self, url, **kwargs):

if self.is_debugger():
task = self.task_join_crawl_config(task, self.crawl_config)
if task['fetch'].get('proxy', False) and task['fetch'].get('fetch_type', None) in ('js', 'phantomjs') \
and not hasattr(self, '_proxy_warning'):
self.logger.warning('phantomjs does not support specify proxy from script, use phantomjs args instead')
self._proxy_warning = True

cache_key = "%(project)s:%(taskid)s" % task
if cache_key not in self._follows_keys:
Expand Down