make proposers and withdrawers the same format (for assembly_id < 19)… #31

bills/main.py

@@ -5,20 +5,20 @@
 import specific
 import pdf
 
-assembly_s, assembly_e = 17, 19 # start, end id of assembly
+assembly_s, assembly_e = 19, 19 # start, end id of assembly
 bill_s, bill_e = None, None     # start, end number of bill
 
 for a in range(assembly_s, assembly_e+1):
     print '\n# Assembly %d' % a
 
     print '## Get meta data'
-    npages = meta.get_npages(a)
-    meta.get_html(a, npages)
-    meta.html2csv(a, npages)
+#    npages = meta.get_npages(a)
+#   meta.get_html(a, npages)
+#   meta.html2csv(a, npages)
 
     print '## Get specific data'
     specific.get_html(a, range=(bill_s, bill_e))
     specific.html2json(a, range=(bill_s, bill_e))
 
     print '## Get pdfs'
-    pdf.get_pdf(a, range=(bill_s, bill_e))
+#    pdf.get_pdf(a, range=(bill_s, bill_e))

bills/meta/html.py

@@ -5,6 +5,8 @@
 import re
 import sys
 import math
+import traceback
+import time
 
 import gevent
 from gevent import monkey; monkey.patch_all()
@@ -41,7 +43,9 @@ def get_page(baseurl, page, directory, npages):
 
             is_first = True
             while is_first or 'TEXTAREA ID="MSG" STYLE="display:none"' in doc:
-                doc = utils.get_webpage_text(url)
+                #doc = utils.get_webpage_text(url)
+                postarg  = {'AGE_FROM':assembly_id, 'AGE_TO':assembly_id, 'PAGE':page, 'PAGE_SIZE':PAGE_SIZE}
+                doc = utils.post_webpage_text(BASEURL['list'], postarg)
                 is_first = False
 
             with open(fn, 'w') as f:
@@ -58,8 +62,15 @@ def get_page(baseurl, page, directory, npages):
 
     #
     print 'Downloading:'
-    jobs = [gevent.spawn(get_page, baseurl, page, directory, npages)\
-            for page in range(1, npages+1)]
-    gevent.joinall(jobs)
-
+    for page in range(1, npages+1):
+        trial = 0;
+        while (trial < 7):
+            trial = trial + 1
+            try:
+                get_page(baseurl, page, directory, npages)
+            except:
+                traceback.print_exc(file=sys.stderr)
+                time.sleep(3)
+                continue
+            break
     return npages

bills/safe.py

@@ -0,0 +1,102 @@
+#! /usr/bin/python2.7
+# -*- coding: utf-8 -*-
+import meta
+import specific
+import pdf
+import sys,traceback
+import time
+
+from settings import BASEURL, DIR, PAGE_SIZE, SESSION, HTML_FIELDS, X
+import utils
+
+import os
+import re
+import lxml
+import pandas as pd
+
+
+assembly_s, assembly_e = 19, 19 # start, end id of assembly
+bill_s, bill_e = None, None     # start, end number of bill
+get_meta_data = False
+
+def safe_get_page (assembly_id, bill_id, link_id, has_summaries):
+    for field in HTML_FIELDS:
+        utils.check_dir('%s/%s' % (DIR[field], assembly_id))
+
+    # Let's get html
+    for field in HTML_FIELDS[1:3]:
+        specific.get_page(assembly_id, bill_id, link_id, field)
+
+    specific.get_specifics(assembly_id, bill_id, link_id)
+    specific.get_summaries(assembly_id, bill_id, link_id, has_summaries)
+
+
+def safe_parse_page (assembly_id, bill_id, meta):
+    jsondir = '%s/%s' % (DIR['data'], assembly_id)
+    utils.check_dir(jsondir)
+    specific.parse_page(assembly_id, bill_id, meta, jsondir);
+
+
+def safe_get_parse_page (assembly_id, range=(None, None)):
+    is_err = True
+
+    # Read meta (csv) file
+    metafile = '%s/%d.csv' % (DIR['meta'], assembly_id)
+    print metafile
+    metacsv= pd.read_csv(metafile, dtype={'bill_id': object, 'link_id': object, 'has_summaries': object})
+
+    # Get all ids
+    for idx in range(len(metacsv['bill_id'])):
+        bill_id = metacsv['bill_id'][idx];
+        link_id = metacsv['link_id'][idx];
+        has_summaries = metacsv['has_summaries'][idx];
+
+        sys.stdout.write('%s' % bill_id)
+        sys.stdout.flush()
+
+        for trial in range(7):
+            try:
+                safe_get_page(assembly_id, bill_id, link_id, has_summaries);
+                sys.stdout.write('G')
+                sys.stdout.flush()
+
+                safe_parse_page(assembly_id, bill_id, metacsv);
+                sys.stdout.write('P')
+                sys.stdout.flush()
+                is_err = False
+            except:
+                sys.stdout.write('X')
+                sys.stdout.flush()
+                is_err = True
+                traceback.print_exc(file=sys.stderr)
+                time.sleep(3)
+                continue
+            break
+
+        sys.stdout.write('\t')
+        sys.stdout.flush()
+
+        #Still error? Write log!
+        if is_err:
+            link_id, has_summaries = metadata[bill_id]
+            sys.stderr.write('%s ' % bill_id)
+            sys.stderr.write('http://likms.assembly.go.kr/bill/jsp/BillDetail.jsp?bill_id=%s Failed\n' % link_id)
+            #sys.exit(-1)
+
+def main():
+    for a in range(assembly_s, assembly_e+1):
+        print '\n# Assembly %d' % a
+
+        if get_meta_data: 
+            print '## Get meta data'
+            npages = meta.get_npages(a)
+            meta.get_html(a, npages)
+            meta.html2csv(a, npages)
+
+        print '## Get specific data'
+        safe_get_parse_page(a)
+
+        print '## Get pdfs'
+        #pdf.get_pdf(a, range=(bill_s, bill_e))
+
+main()

bills/specific/__init__.py

@@ -1,2 +1,2 @@
-from get_html import get_html
-from html2json import html2json
+from get_html import get_html, get_page, get_specifics, get_summaries, get_metadata
+from html2json import html2json, parse_page

bills/specific/html2json.py

@@ -221,7 +221,13 @@ def extract_proposers(assembly_id, bill_id):
 def extract_withdrawers(assembly_id, bill_id):
     fn = '%s/%s/%s.html' % (DIR['withdrawers'], assembly_id, bill_id)
     page = utils.read_webpage(fn)
-    return utils.get_elems(page, X['withdrawers'])
+    elems = utils.get_elems(page, X['withdrawers'])
+    if assembly_id < 19:
+        return elems
+    else:
+        key = ['name_kr', 'party', 'name_cn']
+        values = [filter(None, re.split('[\(/\)]', e)) for e in elems]
+        return [{k: v for k, v in zip(key, value)} for value in values]
 
 def include(meta, bill_id, attr):
     value = list(meta.ix[meta['bill_id']==str(bill_id), attr])[0]

bills/utils.py

@@ -12,6 +12,8 @@
     'Referer': 'http://likms.assembly.go.kr/bill/jsp/BillSearchResult.jsp',
 }
 
+s = requests.Session()
+
 def check_dir(directory):
     if not os.path.exists(directory):
         os.makedirs(directory)
@@ -25,7 +27,7 @@ def get_elem_texts(page, x):
 
 def get_webpage(url, outp):
     try:
-        r = requests.get(url, headers=HEADERS, stream=True)
+        r = s.get(url, headers=HEADERS, stream=True)
         assert r.ok
     except (requests.exceptions.RequestException, AssertionError) as e:
         import sys
@@ -39,7 +41,12 @@ def get_webpage(url, outp):
             f.write(block)
 
 def get_webpage_text(url):
-    r = requests.get(url, headers=HEADERS)
+    r = s.get(url, headers=HEADERS)
+    return r.content
+
+
+def post_webpage_text(url, args):
+    r = s.post(url, data=args, headers=HEADERS)
     return r.content
 
 def read_json(fname):
@@ -57,4 +64,4 @@ def read_webpage(filename):
 def write_json(data, fn):
     with open(fn, 'w') as f:
         json.dump(data, f, indent=2)
-    print 'Data written to ' + fn
+    #print 'Data written to ' + fn