Improve importer tests

src/com/eldrix/hermes/importer.clj

@@ -136,6 +136,17 @@
               (log/debug "Processing cancelled (output channel closed)")
               (throw (InterruptedException. "process cancelled")))))))))
 
+(defn import-file
+  "Import a SNOMED file, returning a map containing :type :headings :parser 
+  and :data as per [[process-file]]. This is designed only for testing and
+  development purposes."
+  [f]
+  (let [ch (a/chan)]
+    (a/thread
+      (process-file f ch)
+      (a/close! ch))
+    (a/<!! ch)))
+
 (s/fdef load-snomed-files
   :args (s/cat :files (s/coll-of :info.snomed/ReleaseFile)
                :opts (s/keys* :opt-un [::nthreads ::batch-size])))

test/resources/example-snapshot/Terminology/sct2_Description_Snapshot-en_INT_20230131.txt

@@ -3,3 +3,5 @@ id	effectiveTime	active	moduleId	conceptId	languageCode	typeId	term	caseSignific
 102018	20170731	1	900000000000207008	126814004	en	900000000000013009	Neoplasm of junctional region of epiglottis	900000000000448009
 103011	20170731	1	900000000000207008	126815003	en	900000000000013009	Neoplasm of lateral wall of oropharynx	900000000000448009
 104017	20170731	1	900000000000207008	126816002	en	900000000000013009	Neoplasm of posterior wall of oropharynx	900000000000448009
+21377010        20020131        0       900000000000207008      12418009        en      900000000000013009      Transfer of "walking" tube graft        900000000000020002
+28578012        20020131        0       900000000000207008      16878007        en      900000000000013009      Blood group antibody rh"        900000000000020002

test/src/com/eldrix/hermes/importer_test.clj

@@ -2,76 +2,52 @@
   (:require
    [clojure.core.async :as async]
    [clojure.java.io :as io]
-   [clojure.spec.gen.alpha :as gen]
    [clojure.test :refer [deftest is testing]]
-   [com.eldrix.hermes.importer :as importer]
-   [com.eldrix.hermes.rf2 :as rf2])
+   [com.eldrix.hermes.importer :as imp])
   (:import (java.time LocalDate)))
 
-(deftest parse-filename
+(deftest parse-filenames
   (testing "nil filename"
-    (is (nil? (importer/parse-filename nil))))
+    (is (nil? (imp/parse-filename nil))))
   (testing "concept filename as string"
-    (let [{:keys [format version-date content-subtype type country-code identifier component]} (importer/parse-filename "sct2_Concept_Snapshot_INT_20230131.txt")]
+    (let [{fmt :format t :type :keys [version-date country-code identifier component]} (imp/parse-filename "sct2_Concept_Snapshot_INT_20230131.txt")]
       (is (= "Concept" component))
       (is (= "INT" country-code))
       (is (= :info.snomed/Concept identifier))
-      (is (= "2" format))
-      (is (= "sct" type))
+      (is (= "2" fmt))
+      (is (= "sct" t))
       (is (= (LocalDate/of 2023 1 31) version-date))))
   (testing "description filename as URL"
-    (let [{:keys [identifier]} (importer/parse-filename (java.net.URL. "file://Terminology/sct2_Description_Snapshot-en_INT_20230131.txt"))]
+    (let [{:keys [identifier]} (imp/parse-filename (java.net.URL. "file://Terminology/sct2_Description_Snapshot-en_INT_20230131.txt"))]
       (is (= :info.snomed/Description identifier))))
   (testing "relationship concrete values filename as file"
-    (let [{:keys [identifier]} (importer/parse-filename (io/file "./Terminology/sct2_RelationshipConcreteValues_Snapshot_INT_20230131.txt"))]
+    (let [{:keys [identifier]} (imp/parse-filename (io/file "./Terminology/sct2_RelationshipConcreteValues_Snapshot_INT_20230131.txt"))]
       (is (= :info.snomed/RelationshipConcreteValues identifier)))))
 
-(defn import-file
-  "Import a SNOMED file"
-  [f]
-  (let [ch (async/chan)]
-    (async/thread
-      (importer/process-file f ch)
-      (async/close! ch))
-    (async/<!! ch)))
-
-(deftest import-concepts
-  (let [{:keys [type parser headings data]} (import-file (io/resource "example-snapshot/Terminology/sct2_Concept_Snapshot_INT_20230131.txt"))]
-    (is (= :info.snomed/Concept type))))
+(deftest import-terminology
+  (testing "concepts"
+    (let [{t :type} (imp/import-file (io/resource "example-snapshot/Terminology/sct2_Concept_Snapshot_INT_20230131.txt"))]
+      (is (= :info.snomed/Concept t))))
+  (testing "descriptions"
+    (let [{t :type} (imp/import-file (io/resource "example-snapshot/Terminology/sct2_Description_Snapshot-en_INT_20230131.txt"))]
+      (is (= :info.snomed/Description t)))))
 
 (deftest import-refset
-  (let [{:keys [type parser headings data] :as f} (import-file (io/resource "example-snapshot/Refset/Map/der2_iisssccRefset_ExtendedMapSnapshot_INT_20230131.txt"))]
-    (is (= :info.snomed/ExtendedMapRefset type))
+  (let [{t :type, headings :headings} (imp/import-file (io/resource "example-snapshot/Refset/Map/der2_iisssccRefset_ExtendedMapSnapshot_INT_20230131.txt"))]
+    (is (= :info.snomed/ExtendedMapRefset t))
     (is (= ["id" "effectiveTime" "active" "moduleId" "refsetId" "referencedComponentId"
             "mapGroup" "mapPriority" "mapRule" "mapAdvice" "mapTarget"
             "correlationId" "mapCategoryId"] headings))))
 
 (deftest import-custom-refset-nil-values
-  (let [{:keys [type parser headings data] :as f} (import-file  (io/resource "example-snapshot/Refset/Map/der2_ssRefset_SimpleMapWithDescriptionSnapshot_12345_20241021.txt"))]
-    (is (= :info.snomed/SimpleMapRefset type))
+  (let [{t :type :keys [headings data]} (imp/import-file (io/resource "example-snapshot/Refset/Map/der2_ssRefset_SimpleMapWithDescriptionSnapshot_12345_20241021.txt"))]
+    (is (= :info.snomed/SimpleMapRefset t))
     (is (= ["id" "effectiveTime" "active" "moduleId" "refsetId" "referencedComponentId"
             "mapTarget" "mapTargetDescription"] headings))
     (is (= ["000d91ce-aae4-4f9e-ad06-51be576becd6" "20241021"
             "1" "195941000112101" "22671000001102"
             "1434181000001106" "ABC01256Q" ""] (first data))
         "Empty last column should be returned as empty string")))
 
-(comment
-  (require '[clojure.data.csv :as csv])
-  (csv/read-csv "hi\tthere\tand\thow\tare\tyou?\t" :separator \tab)
-  (def f (io/resource "example-snapshot/Terminology/sct2_Concept_Snapshot_INT_20230131.txt"))
-  (type f)
-  (io/as-file f)
-  (importer/parse-filename "sct2_Concept.txt")
-  (importer/parse-filename (java.net.URL. "https://wibble.com/sct_Concept_Snapshot_INT_20230131.txt"))
-  (importer/parse-filename f)
-  (importer/parse-filename nil)
-  (def ch (async/chan))
-  (async/thread
-    (importer/process-file f ch)
-    (async/close! ch))
-  (def ch (importer/load-snomed (io/resource "example-snapshot/")))
-  (async/<!! ch)
 
-  (gen/sample (rf2/gen-simple-map-refset {:fields [""]})))