Merge pull request #141 from MrIbrahem/update

Update

mass/radio/st3sort/count_files/b.py → fix_mass/count_files/b.py

@@ -48,4 +48,4 @@
 if "del" in sys.argv:
     for x in errors:
         os.remove(studies_urls_to_files_dir / x)
-        print(f"delete {studies_urls_to_files_dir / x}")
+        print(f"delete {studies_urls_to_files_dir / x}")

mass/radio/st3sort/count_files/do.py → fix_mass/count_files/do.py

@@ -50,6 +50,7 @@ def main(ids_tab):
         studies = [study.split("/")[-1] for study in va["studies"]]
         # ---
         if not studies:
+            printe.output(f"!!! studies not found: {caseId=}.")
             continue
         # ---
         tab.append({"caseId": caseId, "title": title, "studies": studies})

fix_mass/fix_sets/__init__.py

@@ -1,7 +1,7 @@
 """
 
-python3 core8/pwb.py fix_mass/radio/cases_in_ids
-python3 core8/pwb.py fix_mass/radio/to_work
-python3 core8/pwb.py fix_mass/radio/st3/start3 get:500
+python3 core8/pwb.py mass/radio/cases_in_ids
+python3 core8/pwb.py mass/radio/to_work
+python3 core8/pwb.py mass/radio/st3/start3 get:500
 
 """

fix_mass/fix_sets/bots/__init__.py

@@ -1,7 +1,7 @@
 """
 
-python3 core8/pwb.py fix_mass/radio/cases_in_ids
-python3 core8/pwb.py fix_mass/radio/to_work
-python3 core8/pwb.py fix_mass/radio/st3/start3 get:500
+python3 core8/pwb.py mass/radio/cases_in_ids
+python3 core8/pwb.py mass/radio/to_work
+python3 core8/pwb.py mass/radio/st3/start3 get:500
 
 """

fix_mass/fix_sets/bots/find_from_url.py

@@ -0,0 +1,66 @@
+"""
+
+from fix_mass.fix_sets.bots.find_from_url import find_file_name_from_url
+
+"""
+import jsonlines
+# import os
+from pathlib import Path
+from newapi.ncc_page import NEW_API
+from newapi import printe
+
+
+api_new = NEW_API("www", family="nccommons")
+api_new.Login_to_wiki()
+
+jsons_dir = Path(__file__).parent.parent / "jsons"
+
+url_to_file_file = jsons_dir / "find_from_url.jsonl"
+
+if not url_to_file_file.exists():
+    url_to_file_file.write_text('{"url": "", "file_name": ""}')
+
+data = jsonlines.open(url_to_file_file)
+data = {d["url"]: d["file_name"] for d in data}
+
+
+def append_data(url, file_name):
+    data[url] = file_name
+    # ---
+    with jsonlines.open(url_to_file_file, mode="a") as writer:
+        writer.write({"url": url, "file_name": file_name})
+
+
+def get_from_api(url):
+    # ---
+    params = {"action": "upload", "format": "json", "filename": "Wiki.jpg", "url": url, "stash": 1, "formatversion": "2"}
+    # ---
+    # { "upload": { "result": "Warning", "warnings": { "duplicate": [ "Angiodysplasia_-_cecal_active_bleed_(Radiopaedia_168775-136954_Coronal_91).jpeg" ] }, "filekey": "1b00hc5unqxw.olk8pi.13.", "sessionkey": "1b00hc5unqxw.olk8pi.13." } }
+    # ---
+    data = api_new.post_params(params)
+    # ---
+    duplicate = data.get("upload", {}).get("warnings", {}).get("duplicate", [])
+    # ---
+    if not duplicate:
+        return ""
+    # ---
+    du = "File:" + duplicate[0]
+    du = du.replace("_", " ")
+    # ---
+    printe.output(f"find_file_name_from_url: {du}")
+    # ---
+    return du
+
+
+def find_file_name_from_url(url):
+    na = ""
+    if url in data:
+        printe.output(f"find_file_name_from_url: {data[url]}")
+        return data[url]
+    # ---
+    na = get_from_api(url)
+    # ---
+    if na:
+        append_data(url, na)
+    # ---
+    return na

fix_mass/fix_sets/bots/get_img_info.py

@@ -5,10 +5,10 @@
 from fix_mass.fix_sets.bots.get_img_info import one_img_info
 
 """
-import sys
+# import sys
 import re
 import json
-import os
+# import os
 from pathlib import Path
 
 from newapi import printe
@@ -23,13 +23,17 @@
 
 
 def dump_st(data, file):
-    
+
     with open(file, "w", encoding="utf-8") as f:
         json.dump(data, f, ensure_ascii=False, indent=2)
         printe.output(f"<<green>> write {len(data)} to file: {file}")
 
 
-def gt_img_info(titles, id_to_url={}):
+def gt_img_info(titles, id_to_url=None):
+    # ---
+    if not id_to_url:
+        id_to_url = {}
+    # ---
     # ---
     titles = [titles] if not isinstance(titles, list) else titles
     # ---
@@ -137,7 +141,7 @@ def one_img_info(title, study_id, json_data):
     id_to_url = {}
     # ---
     for x in json_data:
-        for n, image in enumerate(x["images"], start=1):
+        for _, image in enumerate(x["images"], start=1):
             id_to_url[str(image["id"])] = image["public_filename"]
     # ---
     info = gt_img_info(title, id_to_url)

fix_mass/fix_sets/bots/set_text.py

@@ -6,6 +6,9 @@
 """
 from newapi import printe
 from fix_mass.fix_sets.bots.has_url import has_url_append
+from fix_mass.fix_sets.bots.find_from_url import find_file_name_from_url
+from fix_mass.fix_sets.lists.sf_infos import from_sf_infos  # from_sf_infos(url, study_id)
+
 
 def make_text(modality, files, set_title, leen):
     # ---
@@ -29,7 +32,7 @@ def make_text(modality, files, set_title, leen):
 
 def make_text_one_study(json_data, data, study_title, study_id):
     # ---
-    url_to_file    = {v["img_url"]: x for x, v in data.items()}
+    url_to_file = {v["img_url"]: x for x, v in data.items()}
     img_id_to_file = {str(v["img_id"]): x for x, v in data.items()}
     # ---
     to_move = {}
@@ -43,14 +46,12 @@ def make_text_one_study(json_data, data, study_title, study_id):
     for x in json_data:
         # ---
         modality = x["modality"]
-        images   = x["images"]
+        images = x["images"]
         # ---
         ty = modality
         # ---
         # print(f"modality: {modality}, images: {len(images)}")
         # ---
-        files = {}
-        # ---
         # sort images by position key
         # images = sorted(images, key=lambda x: x["position"])
         # ---
@@ -71,20 +72,23 @@ def make_text_one_study(json_data, data, study_title, study_id):
             # ---
             if not file_name:
                 file_name = img_id_to_file.get(str(img_id))
-                # print(f"img_id_to_file file_name: {file_name}")
+            # ---
+            if not file_name:
+                file_name = from_sf_infos(public_filename, study_id)
+            # ---
+            if not file_name:
+                file_name = find_file_name_from_url(public_filename)
+            # ---
+            if file_name and not file_name.startswith("File:"):
+                file_name = "File:" + file_name
             # ---
             if not file_name:
                 noo += 1
                 file_name = public_filename
             # ---
             numb = len(to_move[ty]) + 1
             # ---
-            # files[numb] = file_name
             to_move[ty][numb] = file_name
-            # ---
-        # ---
-        # # ---
-        # to_move[ty].update(files)
     # ---
     print(f"noo: {noo}")
     # ---

fix_mass/fix_sets/bots/set_text2.py

@@ -0,0 +1,116 @@
+"""
+
+from fix_mass.fix_sets.bots.set_text import make_text_one_study
+from fix_mass.fix_sets.bots.done import studies_done_append
+
+"""
+from newapi import printe
+from fix_mass.fix_sets.bots.has_url import has_url_append
+from fix_mass.fix_sets.bots.find_from_url import find_file_name_from_url
+from fix_mass.fix_sets.lists.sf_infos import from_sf_infos  # from_sf_infos(url, study_id)
+
+
+def make_text_one_study(json_data, data, study_title, study_id):
+    # ---
+    url_to_file = {v["img_url"]: x for x, v in data.items()}
+    # ---
+    to_move = {}
+    # ---
+    modalities = set([x["modality"] for x in json_data])
+    # ---
+    printe.output(f"modalities: {modalities}")
+    # ---
+    noo = 0
+    # ---
+    urlls = {}
+    # ---
+    texts = {}
+    # ---
+    for x in json_data:
+        # ---
+        modality = x["modality"]
+        images = x["images"]
+        # ---
+        ty = modality
+        # ---
+        # print(f"modality: {modality}, images: {len(images)}")
+        # ---
+        # sort images by position key
+        images = sorted(images, key=lambda x: x["position"])
+        # ---
+        for _n, image in enumerate(images, start=1):
+            # ---
+            plane_projection = image["plane_projection"]
+            aux_modality = image["aux_modality"]
+            # ---
+            # if len(modalities) == 1 and plane_projection:
+            ty = plane_projection
+            # ---
+            if aux_modality:
+                ty = f"{plane_projection} {aux_modality}"
+            # ---
+            if ty not in texts:
+                texts[ty] = ""
+            # ---
+            if ty not in to_move:
+                to_move[ty] = {}
+            # ---
+            public_filename = image["public_filename"]
+            # ---
+            texts[ty] += f"|{public_filename}|\n"
+            # ---
+            file_name = ""
+            # ---
+            # file_name = url_to_file.get(public_filename)
+            # # ---
+            # if not file_name:
+            #     file_name = from_sf_infos(public_filename, study_id)
+            # ---
+            if not file_name:
+                file_name = find_file_name_from_url(public_filename)
+            # ---
+            if file_name and not file_name.startswith("File:"):
+                file_name = "File:" + file_name
+            # ---
+            if file_name:
+                urlls[public_filename] = file_name
+            else:
+                noo += 1
+                file_name = public_filename
+            # ---
+            numb = len(to_move[ty]) + 1
+            # ---
+            to_move[ty][numb] = file_name
+    # ---
+    print(f"noo: {noo}")
+    # ---
+    text = ""
+    # ---
+    study_title2 = study_title
+    # ---
+    for ty, txt in texts.copy().items():
+        for url, file_name in urlls.items():
+            txt = txt.replace(url, file_name)
+        # ---
+        texts[ty] = txt
+    # ---
+    # sum all files in to_move
+    all_files = sum([len(x) for x in to_move.values()])
+    # ---
+    if all_files == len(to_move):
+        printe.output("len to_move == all_files")
+        has_url_append(study_id)
+        return text, to_move
+    # ---
+    for ty, files in to_move.items():
+        # ---
+        print(f"ty: {ty}, files: {len(files)}")
+        # ---
+        text += f"== {ty} ==\n"
+        text += "{{Imagestack\n|width=850\n"
+        text += f"|title={study_title2}\n|align=centre\n|loop=no\n"
+        text += texts[ty].strip()
+        text += "\n}}\n"
+        # ---
+    # ---
+    return text, to_move