Skip to content

Commit

Permalink
[scrap] council_id 정수로 변환, import __init__과 basic에 몰아주기
Browse files Browse the repository at this point in the history
  • Loading branch information
Re-st committed Nov 8, 2023
1 parent e510d26 commit b5aa631
Show file tree
Hide file tree
Showing 16 changed files with 970 additions and 1,687 deletions.
17 changes: 14 additions & 3 deletions scrap/local_councils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,17 @@
각 기초의회들의 크롤링 코드를 모아놓은 패키지입니다.
광역자치단체 별로 폴더를 만들어서 관리합니다.
"""
from .daejeon import *
from .ulsan import *
from .basic import *
import re
from urllib.parse import urlparse
from typing import List
from scrap.utils.types import Councilor, ScrapResult, ScrapBasicArgument
from scrap.utils.requests import get_soup
from scrap.utils.types import CouncilType
from scrap.utils.utils import getPartyList

def returncouncilors(cid, councilors):
return ScrapResult(
council_id=cid,
council_type=CouncilType.LOCAL_COUNCIL,
councilors=councilors,
)
14 changes: 2 additions & 12 deletions scrap/local_councils/basic.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
from urllib.parse import urlparse

from scrap.utils.types import CouncilType, Councilor, ScrapResult, ScrapBasicArgument
from scrap.utils.requests import get_soup
from scrap.utils.utils import getPartyList
from scrap.local_councils import *
import re
import requests
import copy
Expand All @@ -11,7 +7,6 @@
party_keywords = getPartyList()
party_keywords.append("무소속")


def find(soup, element, class_):
if class_ is None:
return soup.find(element)
Expand Down Expand Up @@ -146,7 +141,6 @@ def get_party_easy(profile, wrapper_element, wrapper_class_, wrapper_txt, url):
assert party is not None
return party


def scrap_basic(url, cid, args: ScrapBasicArgument, encoding="utf-8") -> ScrapResult:
"""의원 상세약력 스크랩
:param url: 의원 목록 사이트 url
Expand Down Expand Up @@ -193,11 +187,7 @@ def scrap_basic(url, cid, args: ScrapBasicArgument, encoding="utf-8") -> ScrapRe
raise RuntimeError("[basic.py] 의원 정당을 가져오는데 실패했습니다. 이유: " + str(e))
councilors.append(Councilor(name=name, party=party))

return ScrapResult(
council_id=str(cid),
council_type=CouncilType.LOCAL_COUNCIL,
councilors=councilors,
)
return returncouncilors(cid, councilors)


if __name__ == "__main__":
Expand Down
Loading

0 comments on commit b5aa631

Please sign in to comment.