Merge pull request #4 from ecoron/0.4.2

start adding CsvWriter, add logo

serpscrap/__init__.py

@@ -1,7 +1,8 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 from serpscrap.config import Config
-from serpscrap.serpscrap import SerpScrap
-from serpscrap.urlscrape import UrlScrape
+from serpscrap.csv_writer import CsvWriter
 from serpscrap.markovi import Markovi
+from serpscrap.serpscrap import SerpScrap
 from serpscrap.tfidf import TfIdf
+from serpscrap.urlscrape import UrlScrape

serpscrap/config.py

@@ -32,7 +32,7 @@ class Config():
         'output_filename': None,
         # 'print_results': 'all',
         'scrape_urls': True,
-        'url_threads': 3,
+        'url_threads': 6,
         'log_level': 'INFO',
         'num_workers': 1,
         'num_results_per_page': 10,

serpscrap/csv_writer.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+import csv
+import traceback
+
+
+class CsvWriter():
+
+    def write(self, file_name, my_dict):
+        try:
+            with open(file_name, 'w', encoding='utf-8') as f:
+                w = csv.DictWriter(f, my_dict[0].keys(), dialect='excel')
+                w.writeheader()
+                for row in my_dict[1:]:
+                    w.writerow(row)
+        except Exception:
+            print(traceback.print_exc())
+            raise Exception

serpscrap/serpscrap.py

@@ -10,6 +10,7 @@
 import chardet
 from scrapcore.core import Core
 from serpscrap.config import Config
+from serpscrap.csv_writer import CsvWriter
 from serpscrap.urlscrape import UrlScrape
 
 
@@ -88,6 +89,13 @@ def run(self):
                         results[index].update(result_url)
         return results if isinstance(results, list) else [results]
 
+    def as_csv(self, file_path):
+        writer = CsvWriter()
+        index = 0
+        result = self.run()
+        writer.write(file_path + '0' + str(index) + '.csv', result)
+
+
     def scrap_serps(self):
         """call scrap method and append serp results to list
         Returns

setup.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 from setuptools import setup, find_packages
 
-version = '0.4.1'
+version = '0.4.2'
 
 
 setup(