-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #36 from miha42-github/V3.0.0--dev
V3.0.0 dev
- Loading branch information
Showing
37 changed files
with
1,417 additions
and
1,444 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| name: Monthly company_dns build | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
|
|
||
| jobs: | ||
| build: | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| repository: 'miha42-github/company_dns' | ||
| ref: 'main' | ||
|
|
||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v1 | ||
|
|
||
| - name: Login to GitHub Container Registry | ||
| uses: docker/login-action@v1 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.repository_owner }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| - name: Build and push | ||
| uses: docker/build-push-action@v2 | ||
| with: | ||
| context: . | ||
| push: true | ||
| tags: ghcr.io/${{ github.repository }}/company_dns:latest |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,9 @@ | ||
| *.gz | ||
| *.db | ||
| .db_exists | ||
| db.exists | ||
| __pycache__ | ||
| .DS_Store | ||
| test.py | ||
| .cache_exists | ||
| form_* | ||
| cache.exists | ||
| form_* | ||
| edgar_data |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| # Use an official Python runtime as a parent image | ||
| FROM python:3.12-slim | ||
|
|
||
| # Set the working directory in the container to /app | ||
| WORKDIR /app | ||
|
|
||
| # Add the current directory contents into the container at /app | ||
| ADD . /app | ||
|
|
||
| # Install curl and create directory | ||
| RUN apt-get update && apt-get install -y curl && mkdir -p /app/edgar_data | ||
| # RUN apk --no-cache add curl && mkdir -p /app/edgar_data | ||
|
|
||
| # Install any needed packages specified in requirements.txt | ||
| RUN pip install --no-cache-dir -r requirements.txt | ||
|
|
||
| # Run makedb.py to create the database cache | ||
| RUN python makedb.py | ||
|
|
||
| # Make port 80 available to the world outside this container | ||
| EXPOSE 8000 | ||
|
|
||
| # Run the command to start the application | ||
| CMD ["python", "company_dns.py"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,14 +1,16 @@ | ||
| [db_control] | ||
| DB_NAME = company_dns.db | ||
| CACHE_EXISTS = .cache_exists | ||
| DB_EXISTS = .db_exists | ||
|
|
||
| [edgar_data] | ||
| EDGAR_DATA_DIR = ./edgar_data | ||
| ALL_FORMS = form_all.tab | ||
| CACHE_EXISTS = cache.exists | ||
|
|
||
| [sic_data] | ||
| SIC_DATA_DIR = ./sic_data | ||
| DIVISIONS = divisions.csv | ||
| MAJOR_GROUPS = major-groups.csv | ||
| INDUSTRY_GROUPS = industry-groups.csv | ||
| SIC_CODES = sic-codes.csv | ||
|
|
||
| [db_control] | ||
| DB_NAME = company_dns.db | ||
| DB_EXISTS = db.exists | ||
| DB_PATH = ./ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,200 @@ | ||
| from starlette.applications import Starlette | ||
| from starlette.responses import JSONResponse, RedirectResponse | ||
| from starlette.routing import Route | ||
| from starlette.routing import Mount | ||
| from starlette.staticfiles import StaticFiles | ||
| from starlette.middleware import Middleware | ||
| from starlette.middleware.cors import CORSMiddleware | ||
| from starlette.middleware.base import BaseHTTPMiddleware | ||
| import uvicorn | ||
| import logging | ||
|
|
||
| from lib.sic import SICQueries | ||
| from lib.edgar import EdgarQueries | ||
| from lib.wikipedia import WikipediaQueries | ||
| from lib.firmographics import GeneralQueries | ||
|
|
||
| # -------------------------------------------------------------- # | ||
| # BEGIN: Standard Idustry Classification (SIC) database cache functions | ||
| async def sic_description(request): | ||
| return _handle_request(request, sq, sq.get_all_sic_by_name, 'sic_desc') | ||
|
|
||
| async def sic_code(request): | ||
| return _handle_request(request, sq, sq.get_all_sic_by_no, 'sic_code') | ||
|
|
||
| async def division_code(request): | ||
| return _handle_request(request, sq, sq.get_division_desc_by_id, 'division_code') | ||
|
|
||
| async def industry_code(request): | ||
| return _handle_request(request, sq, sq.get_all_industry_group_by_no, 'industry_code') | ||
|
|
||
| async def major_code(request): | ||
| return _handle_request(request, sq, sq.get_all_major_group_by_no, 'major_code') | ||
| # END: Standard Idustry Classification (SIC) database cache functions | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| # -------------------------------------------------------------- # | ||
| # BEGIN: EDGAR dabase cache functions | ||
| async def edgar_detail(request): | ||
| return _handle_request(request, eq, eq.get_all_details, 'company_name') | ||
|
|
||
| async def edgar_summary(request): | ||
| return _handle_request(request, eq, eq.get_all_details, 'company_name', firmographics=False) | ||
|
|
||
| async def edgar_ciks(request): | ||
| return _handle_request(request, eq, eq.get_all_ciks, 'company_name') | ||
|
|
||
| async def edgar_firmographics(request): | ||
| return _handle_request(request, eq, eq.get_firmographics, 'cik_no') | ||
| # END: EDGAR dabase cache functions | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| # -------------------------------------------------------------- # | ||
| # BEGIN: Wikipedia functions | ||
| async def wikipedia_firmographics(request): | ||
| return _handle_request(request, wq, wq.get_firmographics, 'company_name') | ||
| # END: Wikipedia functions | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| # -------------------------------------------------------------- # | ||
| # BEGIN: General query functions | ||
| async def general_query(request): | ||
| try: | ||
| gq.query = request.path_params['company_name'] | ||
| # Log the query request as a debug message | ||
| logger.debug(f'Querying for general data for {request.path_params["company_name"]}') | ||
| company_wiki_data = gq.get_firmographics_wikipedia() | ||
| general_company_data = gq.merge_data(company_wiki_data['data'], company_wiki_data['data']['cik']) | ||
| # Call check_status_and_return to check the status of the data and return the data or an error message | ||
| checked_data = _check_status_and_return(general_company_data, request.path_params['company_name']) | ||
| if 'error' in checked_data: | ||
| return JSONResponse(checked_data, status_code=checked_data['code']) | ||
| return JSONResponse(checked_data) | ||
| except Exception as e: | ||
| logger.error(f'Error: {e}') | ||
| general_company_data = {'error': 'A general or code error has occured', 'code': 500} | ||
| return JSONResponse(general_company_data, status_code=general_company_data['code']) | ||
| # END: General query functions | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| # -------------------------------------------------------------- # | ||
| # BEGIN: Helper functions | ||
| def _check_status_and_return(data, resource_name): | ||
| if data.get('code') != 200: | ||
| # Log the error message | ||
| logger.error(f'Data for resource {resource_name} not found') | ||
| # Return an error message that the data was not found using the resource name | ||
| return {'error': f'Data for resource {resource_name} not found', 'code': 404} | ||
| return data | ||
|
|
||
| def _prepare_logging(log_level=logging.DEBUG): | ||
| logging.basicConfig(format='%(levelname)s:\t%(asctime)s [module: %(name)s] %(message)s', level=log_level) | ||
| return logging.getLogger(__file__) | ||
|
|
||
| def _handle_request(request, handler, func, path_param, *args, **kwargs): | ||
| handler.query = request.path_params.get(path_param) | ||
| data = func(*args, **kwargs) | ||
| checked_data = _check_status_and_return(data, path_param) | ||
| if 'error' in checked_data: | ||
| return JSONResponse(checked_data, status_code=checked_data['code']) | ||
| return JSONResponse(data) | ||
| # END: Helper functions | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| # -------------------------------------------------------------- # | ||
| # BEGIN: Define query objects | ||
| global sq | ||
| sq = SICQueries() | ||
|
|
||
| global eq | ||
| eq = EdgarQueries() | ||
|
|
||
| global wq | ||
| wq = WikipediaQueries() | ||
|
|
||
| global gq | ||
| gq = GeneralQueries() | ||
| # END: Define query objects | ||
| # -------------------------------------------------------------- # | ||
|
|
||
|
|
||
| # -------------------------------------------------------------- # | ||
| # BEGIN: Define the Starlette app | ||
| class CatchAllMiddleware(BaseHTTPMiddleware): | ||
| async def dispatch(self, request, call_next): | ||
| response = await call_next(request) | ||
| if response.status_code == 404: | ||
| return RedirectResponse(url='/help') | ||
| return response | ||
|
|
||
| middleware = [ | ||
| Middleware(CatchAllMiddleware), | ||
| Middleware(CORSMiddleware, allow_origins=['*']) | ||
| ] | ||
|
|
||
| global logger | ||
| logger = _prepare_logging() | ||
|
|
||
| app = Starlette(debug=True, middleware=middleware, routes=[ | ||
| # -------------------------------------------------------------- # | ||
| # SIC endpoints for V2.0 | ||
| Route('/V2.0/sic/description/{sic_desc}', sic_description), | ||
| Route('/V2.0/sic/code/{sic_code}', sic_code), | ||
| Route('/V2.0/sic/division/{division_code}', division_code), | ||
| Route('/V2.0/sic/industry/{industry_code}', industry_code), | ||
| Route('/V2.0/sic/major/{major_code}', major_code), | ||
|
|
||
| # SIC endpoints for V3.0 | ||
| Route('/V3.0/na/sic/description/{sic_desc}', sic_description), | ||
| Route('/V3.0/na/sic/code/{sic_code}', sic_code), | ||
| Route('/V3.0/na/sic/division/{division_code}', division_code), | ||
| Route('/V3.0/na/sic/industry/{industry_code}', industry_code), | ||
| Route('/V3.0/na/sic/major/{major_code}', major_code), | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| # -------------------------------------------------------------- # | ||
| # EDGAR endpoints for V2.0 | ||
| Route('/V2.0/companies/edgar/detail/{company_name}', edgar_detail), | ||
| Route('/V2.0/companies/edgar/summary/{company_name}', edgar_summary), | ||
| Route('/V2.0/companies/edgar/ciks/{company_name}', edgar_ciks), | ||
| Route('/V2.0/company/edgar/firmographics/{cik_no}', edgar_firmographics), | ||
|
|
||
| # EDGAR endpoints for V3.0 | ||
| Route('/V3.0/na/companies/edgar/detail/{company_name}', edgar_detail), | ||
| Route('/V3.0/na/companies/edgar/summary/{company_name}', edgar_summary), | ||
| Route('/V3.0/na/companies/edgar/ciks/{company_name}', edgar_ciks), | ||
| Route('/V3.0/na/company/edgar/firmographics/{cik_no}', edgar_firmographics), | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| # -------------------------------------------------------------- # | ||
| # Wikipedia endpoints for V2.0 | ||
| Route('/V2.0/company/wikipedia/firmographics/{company_name}', wikipedia_firmographics), | ||
|
|
||
| # Wikipedia endpoints for V3.0 | ||
| Route('/V3.0/global/company/wikipedia/firmographics/{company_name}', wikipedia_firmographics), | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| # -------------------------------------------------------------- # | ||
| # General query endpoint for V2.0 | ||
| Route('/V2.0/company/merged/firmographics/{company_name}', general_query), | ||
|
|
||
| # General query endpoint for V3.0 | ||
| Route('/V3.0/global/company/merged/firmographics/{company_name}', general_query), | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| # Serve the local directory ./html at the /help | ||
| Mount('/help', app=StaticFiles(directory='html', html=True)), | ||
|
|
||
| # Catch-all route which redirects to /help | ||
| # Route("/{path:path}", endpoint=lambda _: RedirectResponse(url='/help'), methods=["GET"]), | ||
|
|
||
|
|
||
| ]) | ||
| # END: Define the Starlette app | ||
| # -------------------------------------------------------------- # | ||
|
|
||
| if __name__ == "__main__": | ||
| try: | ||
| uvicorn.run(app, host='0.0.0.0', port=8000, log_level="debug", lifespan='off') | ||
| except KeyboardInterrupt: | ||
| logger.info("Server was shut down by the user.") |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.