Skip to content

Commit

Permalink
Merge pull request #31 from MikeMeliz/refactoring
Browse files Browse the repository at this point in the history 
Refactoring & Improvements
  • Loading branch information
@MikeMeliz
MikeMeliz authored Oct 31, 2024
2 parents 246d9aa + 63c1e3c commit 568a859
Show file tree
Hide file tree
Showing 10 changed files with 119 additions and 204 deletions.
109 changes: 4 additions & 105 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
# Project Specific
output/*
!output/.gitkeep

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
Expand All @@ -26,12 +27,6 @@ share/python-wheels/
*.egg
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt
Expand All @@ -51,74 +46,6 @@ coverage.xml
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version

# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock

# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock

# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
Expand All @@ -128,33 +55,5 @@ ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
15 changes: 9 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,12 @@ $ torcrawl -v -u http://www.github.com/ -c -d 2 -p 2
## Installation

### Easy Installation with pip:
*Comming soon..*
*Coming soon...*

### Manual Installation:
1. **Clone this repository**:<br>
`git clone https://github.com/MikeMeliz/TorCrawl.py.git`
2. **Install dependecies**:<br>
2. **Install dependencies**:<br>
`pip install -r requirements.txt`
3. **Install and Start TOR Service**:
1. **Debian/Ubuntu**: <br>
Expand All @@ -82,12 +82,12 @@ $ torcrawl -v -u http://www.github.com/ -c -d 2 -p 2
-f |--folder| The directory which will contain the generated files
**Extract**: | |
-e |--extract| Extract page's code to terminal or file (Default: Terminal)
-i |--input filename| Input file with URL(s) (seperated by line)
-i |--input filename| Input file with URL(s) (separated by line)
-o |--output [filename]| Output page(s) to file(s) (for one page)
-y |--yara | Perform yara keyword search:<br>h = search entire html object,<br>t = search only text
**Crawl**: | |
-c |--crawl| Crawl website (Default output on website/links.txt)
-d |--cdepth| Set depth of crawler's travel (Default: 1)
-d |--depth| Set depth of crawler's travel (Default: 1)
-p |--pause| Seconds of pause between requests (Default: 0)
-l |--log| Log file with visited URLs and their response code

Expand Down Expand Up @@ -134,8 +134,7 @@ $ python torcrawl.py -i links.txt


### As Crawler:
Crawl the links of the webpage without the use of TOR,
also show verbose output (really helpfull):
Crawl the links of the webpage without the use of TOR, also show verbose output (really helpful):

```shell
$ python torcrawl.py -v -w -u http://www.github.com/ -c
Expand Down Expand Up @@ -216,6 +215,10 @@ Feel free to contribute on this project! Just fork it, make any change on your f

## Changelog
```shell
v1.32:
* Removed 1 second default pause between requests
* Several improvements on results
* Improved logs
v1.31:
* Fixed Input Link NoneType Error
* Fixed name mismatch
Expand Down
26 changes: 17 additions & 9 deletions modules/checker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,22 @@ def folder(website, verbose):
:param website: String - URL of website to crawl.
:param verbose: Boolean - Logging level.
:return: String 'out_path' - Path of the output folder.
:return: String 'output_folder' - Path of the output folder.
"""
out_path = website
if not os.path.exists(out_path):
os.makedirs(out_path)
parsed = urlparse(website)
if parsed.scheme != '':
output_folder = "output/" + urlparse(website).netloc
else:
output_folder = "output/" + website
if not os.path.exists(output_folder):
try:
os.makedirs(output_folder)
except FileExistsError:
if verbose:
print(f"## Folder exists already: {website}")
if verbose:
print(f"## Folder created: {out_path}")
return out_path
print(f"## Folder created: {website}")
return output_folder


def check_tor(verbose):
Expand Down Expand Up @@ -87,11 +95,11 @@ def check_ip():
""" Checks users IP from external resource.
:return: None or HTTPError
"""
addr = 'https://api.ipify.org/?format=json'
api_address = 'https://api.ipify.org/?format=json'
try:
my_ip = load(urlopen(addr))['ip']
my_ip = load(urlopen(api_address))['ip']
print(f'## Your IP: {my_ip}')
except HTTPError as err:
error = sys.exc_info()[0]
print(f"Error: {error} \n## IP cannot be obtained. \n## Is {addr} up? "
print(f"Error: {error} \n## IP cannot be obtained. \n## Is {api_address} up? "
f"\n## HTTPError: {err}")
73 changes: 39 additions & 34 deletions modules/crawler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def excludes(self, link):
:param link: String
:return: Boolean
"""
now = datetime.datetime.now().strftime("%Y%m%d")
now = datetime.datetime.now().strftime("%y%m%d")

# BUG: For NoneType Exceptions, got to find a solution here
if link is None:
Expand All @@ -36,31 +36,33 @@ def excludes(self, link):
return True
# External links
elif link.startswith('http') and not link.startswith(self.website):
file_path = self.out_path + '/' + now + '_extlinks.txt'
with open(file_path, 'w+', encoding='UTF-8') as lst_file:
file_path = self.out_path + '/' + now + '_ext-links.txt'
with open(file_path, 'a+', encoding='UTF-8') as lst_file:
lst_file.write(str(link) + '\n')
return True
# Telephone Number
elif link.startswith('tel:'):
file_path = self.out_path + '/' + now + '_telephones.txt'
with open(file_path, 'w+', encoding='UTF-8') as lst_file:
with open(file_path, 'a+', encoding='UTF-8') as lst_file:
lst_file.write(str(link) + '\n')
return True
# Mails
elif link.startswith('mailto:'):
file_path = self.out_path + '/' + now + '_mails.txt'
with open(file_path, 'w+', encoding='UTF-8') as lst_file:
with open(file_path, 'a+', encoding='UTF-8') as lst_file:
lst_file.write(str(link) + '\n')
return True
# Type of files
elif re.search('^.*\\.(pdf|jpg|jpeg|png|gif|doc)$', link,
re.IGNORECASE):
elif re.search('^.*\\.(pdf|jpg|jpeg|png|gif|doc)$', link, re.IGNORECASE):
file_path = self.out_path + '/' + now + '_files.txt'
with open(file_path, 'a+', encoding='UTF-8') as lst_file:
lst_file.write(str(link) + '\n')
return True

def canonical(self, link):
""" Canonicalization of the link.
:param link: String
:param link: String: URL(s)
:return: String 'final_link': parsed canonical url.
"""
# Already formatted
Expand All @@ -83,6 +85,20 @@ def canonical(self, link):
final_link = self.website + "/" + link
return final_link

def write_log(self, log):
log_path = self.out_path + '/crawler.log'
now = datetime.datetime.now()

if self.logs is True:
open(log_path, 'a+')
if self.logs is True and os.access(log_path, os.W_OK) is False:
print(f"## Unable to write to {self.out_path}/log.txt - Exiting")
sys.exit(2)
with open(log_path, 'a+', encoding='UTF-8') as log_file:
log_file.write(str(now) + " [crawler.py] " + log)
log_file.close()


def crawl(self):
""" Core of the crawler.
:return: List (ord_lst) - List of crawled links.
Expand All @@ -91,11 +107,6 @@ def crawl(self):
ord_lst = []
ord_lst.insert(0, self.website)
ord_lst_ind = 0
log_path = self.out_path + '/log.txt'

if self.logs is True and os.access(log_path, os.W_OK) is False:
print(f"## Unable to write to {self.out_path}/log.txt - Exiting")
sys.exit(2)

print(f"## Crawler started from {self.website} with "
f"{str(self.c_depth)} depth crawl, and {str(self.c_pause)} "
Expand All @@ -113,27 +124,23 @@ def crawl(self):
if item is not None:
html_page = urllib.request.urlopen(item)
except (HTTPError, URLError) as error:
print('## ERROR: Domain or link seems to be '
'unreachable. Add -v to see the verbose error.'
'Or write the full URL at -u argument!')
if self.verbose: print(error)
self.write_log(f"[INFO] ERROR: Domain or link seems to be unreachable: {str(item)} | "
f"Message: {error}\n")
continue
else:
try:
html_page = urllib.request.urlopen(self.website)
ord_lst_ind += 1
except (HTTPError, URLError) as error:
print('## ERROR: Domain or link seems to be '
'unreachable. Add -v to see the verbose error.'
'Or write the full URL at -u argument!')
if self.verbose: print(error)
self.write_log(f"[INFO] ERROR: Domain or link seems to be unreachable: {str(item)} | "
f"Message: {error}\n")
ord_lst_ind += 1
continue

try:
soup = BeautifulSoup(html_page, features="html.parser")
except TypeError as err:
print(f"## Soup Error Encountered:: could to parse "
except TypeError:
print(f"## Soup Error Encountered:: couldn't parse "
f"ord_list # {ord_lst_ind}::{ord_lst[ord_lst_ind]}")
continue

Expand All @@ -159,7 +166,7 @@ def crawl(self):
if ver_link is not None:
lst.add(ver_link)

# TODO: For non-formal links, using RegEx
# TODO: For non-formal links, using RegEx, should be an additional parameter, and all patterns to be stored in a file
# url_pattern = r'/(?:(?:https?|ftp|file):\/\/|www\.|ftp\.)(?:\([-A-Z0-9+&@#\/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#\/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#\/%=~_|$?!:,.]*\)|[A-Z0-9+&@#\/%=~_|$])/igm'
# html_content = urllib.request.urlopen(self.website).read().decode('utf-8')

Expand All @@ -180,21 +187,19 @@ def crawl(self):
ord_lst = ord_lst + list(set(lst))
ord_lst = list(set(ord_lst))

# Keeps logs for every webpage visited.
page_code = html_page.status
url_visited = f"[{str(page_code)}] {str(item)} \n"
self.write_log("[INFO] Parsed: " + url_visited)

if self.verbose:
sys.stdout.write("-- Results: " + str(len(ord_lst)) + "\r")
sys.stdout.write(" -- Results: " + str(len(ord_lst)) + "\r")
sys.stdout.flush()

# Pause time.
if (ord_lst.index(item) != len(ord_lst) - 1) and \
float(self.c_pause) > 0:
# Add Pause time between each iteration
if (ord_lst.index(item) != len(ord_lst) - 1) and float(self.c_pause) > 0:
time.sleep(float(self.c_pause))

# Keeps logs for every webpage visited.
if self.logs:
it_code = html_page.getcode()
with open(log_path, 'w+', encoding='UTF-8') as log_file:
log_file.write(f"[{str(it_code)}] {str(item)} \n")

print(f"## Step {str(index + 1)} completed "
f"with: {str(len(ord_lst))} result(s)")

Expand Down
Loading

0 comments on commit 568a859

Please sign in to comment.