Merge pull request #42 from vtuber-plan/dev

Version 0.3.4

.gitignore

@@ -165,3 +165,9 @@ cython_debug/
 /repos/
 /logs/
 /cache/
+
+*.key
+*.csr
+*.pem
+*.crt
+

.vscode/tasks.json

@@ -14,12 +14,12 @@
         {
             "label": "docker: build main",
             "type": "shell",
-            "command": "docker build -t xiaoyao9184/olah:main -f ./docker/build@source/dockerfile .",
+            "command": "docker build -t xiahan2019/olah:main -f ./docker/build@source/dockerfile .",
         },
         {
             "label": "docker: build 0.3.3",
             "type": "shell",
-            "command": "docker build -t xiaoyao9184/olah:0.3.3 -f ./docker/build@pypi/dockerfile .",
+            "command": "docker build -t xiahan2019/olah:0.3.3 -f ./docker/build@pypi/dockerfile .",
         },
         {
             "label": "huggingface-cli: download bert-base-uncased",

docker/up/docker-compose.yml

@@ -2,7 +2,7 @@
 
 services:
   olah:
-    image: xiaoyao9184/olah:lastet
+    image: xiahan2019/olah:lastet
     container_name: olah
     command: -c /app/configs.toml
     ports:

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "olah"
-version = "0.3.3"
+version = "0.3.4"
 description = "Self-hosted lightweight huggingface mirror."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -29,10 +29,8 @@ dev = ["black==24.10.0", "pylint==3.3.1", "pytest==8.3.3"]
 olah-cli = "olah.server:cli"
 
 [tool.setuptools.packages.find]
+where = ["src"]
 exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]
 
-[tool.setuptools.package-data]
-static = ["*.html"]
-
 [tool.wheel]
 exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]

olah/constants.py → src/olah/constants.py

@@ -5,11 +5,15 @@
 # license that can be found in the LICENSE file or at
 # https://opensource.org/licenses/MIT.
 
+import os
+
+
 WORKER_API_TIMEOUT = 15
 CHUNK_SIZE = 4096
 LFS_FILE_BLOCK = 64 * 1024 * 1024
 
 DEFAULT_LOGGER_DIR = "./logs"
+OLAH_CODE_DIR = os.path.dirname(os.path.abspath(__file__))
 
 ORIGINAL_LOC = "oriloc"
 

olah/proxy/commits.py → src/olah/proxy/commits.py

@@ -82,7 +82,7 @@ async def commits_generator(
 
     org_repo = get_org_repo(org, repo)
     # save
-    repos_path = app.app_settings.config.repos_path
+    repos_path = app.state.app_settings.config.repos_path
     save_dir = os.path.join(
         repos_path, f"api/{repo_type}/{org_repo}/commits/{commit}"
     )
@@ -93,7 +93,7 @@ async def commits_generator(
 
     org_repo = get_org_repo(org, repo)
     commits_url = urljoin(
-        app.app_settings.config.hf_url_base(),
+        app.state.app_settings.config.hf_url_base(),
         f"/api/{repo_type}/{org_repo}/commits/{commit}",
     )
     # proxy

olah/proxy/files.py → src/olah/proxy/files.py

@@ -267,7 +267,7 @@ async def _file_chunk_head(
     allow_cache: bool,
     file_size: int,
 ):
-    if not app.app_settings.config.offline:
+    if not app.state.app_settings.config.offline:
         async with client.stream(
             method=method,
             url=url,
@@ -333,17 +333,17 @@ async def _file_realtime_stream(
             )
         else:
             hf_url = urljoin(
-                app.app_settings.config.hf_lfs_url_base(), get_url_tail(clean_url)
+                app.state.app_settings.config.hf_lfs_url_base(), get_url_tail(clean_url)
             )
     else:
         if urlparse(url).netloc in [
-            app.app_settings.config.hf_netloc,
-            app.app_settings.config.hf_lfs_netloc,
+            app.state.app_settings.config.hf_netloc,
+            app.state.app_settings.config.hf_lfs_netloc,
         ]:
             hf_url = url
         else:
             hf_url = urljoin(
-                app.app_settings.config.hf_lfs_url_base(), get_url_tail(url)
+                app.state.app_settings.config.hf_lfs_url_base(), get_url_tail(url)
             )
 
     request_headers = {k: v for k, v in request.headers.items()}
@@ -409,7 +409,7 @@ async def _file_realtime_stream(
     etag = await _resource_etag(
         hf_url=hf_url,
         authorization=request.headers.get("authorization", None),
-        offline=app.app_settings.config.offline,
+        offline=app.state.app_settings.config.offline,
     )
     response_headers["etag"] = etag
 
@@ -466,7 +466,7 @@ async def file_get_generator(
 ):
     org_repo = get_org_repo(org, repo)
     # save
-    repos_path = app.app_settings.config.repos_path
+    repos_path = app.state.app_settings.config.repos_path
     head_path = os.path.join(
         repos_path, f"heads/{repo_type}/{org_repo}/resolve/{commit}/{file_path}"
     )
@@ -482,12 +482,12 @@ async def file_get_generator(
     # proxy
     if repo_type == "models":
         url = urljoin(
-            app.app_settings.config.hf_url_base(),
+            app.state.app_settings.config.hf_url_base(),
             f"/{org_repo}/resolve/{commit}/{file_path}",
         )
     else:
         url = urljoin(
-            app.app_settings.config.hf_url_base(),
+            app.state.app_settings.config.hf_url_base(),
             f"/{repo_type}/{org_repo}/resolve/{commit}/{file_path}",
         )
     return _file_realtime_stream(
@@ -520,7 +520,7 @@ async def cdn_file_get_generator(
 
     org_repo = get_org_repo(org, repo)
     # save
-    repos_path = app.app_settings.config.repos_path
+    repos_path = app.state.app_settings.config.repos_path
     head_path = os.path.join(
         repos_path, f"heads/{repo_type}/{org_repo}/cdn/{file_hash}"
     )
@@ -535,10 +535,10 @@ async def cdn_file_get_generator(
 
     # proxy
     # request_url = urlparse(str(request.url))
-    # if request_url.netloc == app.app_settings.config.hf_lfs_netloc:
-    #     redirected_url = urljoin(app.app_settings.config.mirror_lfs_url_base(), get_url_tail(request_url))
+    # if request_url.netloc == app.state.app_settings.config.hf_lfs_netloc:
+    #     redirected_url = urljoin(app.state.app_settings.config.mirror_lfs_url_base(), get_url_tail(request_url))
     # else:
-    #     redirected_url = urljoin(app.app_settings.config.mirror_url_base(), get_url_tail(request_url))
+    #     redirected_url = urljoin(app.state.app_settings.config.mirror_url_base(), get_url_tail(request_url))
 
     return _file_realtime_stream(
         app=app,

olah/proxy/lfs.py → src/olah/proxy/lfs.py

@@ -17,7 +17,7 @@ async def lfs_head_generator(
     app, dir1: str, dir2: str, hash_repo: str, hash_file: str, request: Request
 ):
     # save
-    repos_path = app.app_settings.config.repos_path
+    repos_path = app.state.app_settings.config.repos_path
     head_path = os.path.join(
         repos_path, f"lfs/heads/{dir1}/{dir2}/{hash_repo}/{hash_file}"
     )
@@ -47,7 +47,7 @@ async def lfs_get_generator(
     app, dir1: str, dir2: str, hash_repo: str, hash_file: str, request: Request
 ):
     # save
-    repos_path = app.app_settings.config.repos_path
+    repos_path = app.state.app_settings.config.repos_path
     head_path = os.path.join(
         repos_path, f"lfs/heads/{dir1}/{dir2}/{hash_repo}/{hash_file}"
     )

olah/proxy/meta.py → src/olah/proxy/meta.py

@@ -78,7 +78,7 @@ async def meta_generator(
 
     org_repo = get_org_repo(org, repo)
     # save
-    repos_path = app.app_settings.config.repos_path
+    repos_path = app.state.app_settings.config.repos_path
     save_dir = os.path.join(
         repos_path, f"api/{repo_type}/{org_repo}/revision/{commit}"
     )
@@ -90,7 +90,7 @@ async def meta_generator(
 
     org_repo = get_org_repo(org, repo)
     meta_url = urljoin(
-        app.app_settings.config.hf_url_base(),
+        app.state.app_settings.config.hf_url_base(),
         f"/api/{repo_type}/{org_repo}/revision/{commit}",
     )
     # proxy

olah/proxy/pathsinfo.py → src/olah/proxy/pathsinfo.py

@@ -74,7 +74,7 @@ async def pathsinfo_generator(
 
     org_repo = get_org_repo(org, repo)
     # save
-    repos_path = app.app_settings.config.repos_path
+    repos_path = app.state.app_settings.config.repos_path
 
     final_content = []
     for path in paths:
@@ -89,7 +89,7 @@ async def pathsinfo_generator(
 
         org_repo = get_org_repo(org, repo)
         pathsinfo_url = urljoin(
-            app.app_settings.config.hf_url_base(),
+            app.state.app_settings.config.hf_url_base(),
             f"/api/{repo_type}/{org_repo}/paths-info/{commit}",
         )
         # proxy

olah/proxy/tree.py → src/olah/proxy/tree.py

@@ -84,7 +84,7 @@ async def tree_generator(
 
     org_repo = get_org_repo(org, repo)
     # save
-    repos_path = app.app_settings.config.repos_path
+    repos_path = app.state.app_settings.config.repos_path
     save_dir = os.path.join(
         repos_path, f"api/{repo_type}/{org_repo}/tree/{commit}/{path}"
     )
@@ -95,7 +95,7 @@ async def tree_generator(
 
     org_repo = get_org_repo(org, repo)
     tree_url = urljoin(
-        app.app_settings.config.hf_url_base(),
+        app.state.app_settings.config.hf_url_base(),
         f"/api/{repo_type}/{org_repo}/tree/{commit}/{path}",
     )
     # proxy