Merge branch 'main' of github.com:pkiraly/metadata-qa-marc into main

README.md

@@ -2,7 +2,7 @@
 
 QA catalogue reads MARC files (in binary MARC, MARCXML or other serialization formats), analyses different aspects of quality, and saves the results into CSV files. These CSV files could be used in different context, we provide a lightweight, web-based [user interface](#user-interface) for that.
 
-![Output sample](https://pbs.twimg.com/media/FHJvPCYXEAMEi_N?format=jpg&name=medium)
+![Output sample](https://github.com/pkiraly/metadata-qa-marc-web/raw/gh-pages/img/issues-v1.gif)
 Screenshot from the web UI of the QA cataloge
 
 * For more info

common-script

@@ -243,28 +243,28 @@ do_sqlite() {
   printf "%s %s> php scripts/sqlite/normalize-issue-details.php ${OUTPUT_DIR} &> ${PREFIX}/sqlite.log\n" $(date +"%F %T")
   php scripts/sqlite/normalize-issue-details.php ${OUTPUT_DIR} &> ${PREFIX}/sqlite.log
 
-  # printf "%s %s> Rscript scripts/sqlite/create-database.R ${OUTPUT_DIR} &>> ${PREFIX}/sqlite.log\n" $(date +"%F %T")
-  # Rscript scripts/sqlite/create-database.R ${OUTPUT_DIR} &>> ${PREFIX}/sqlite.log
-
-  # printf "%s %s> sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite < scripts/sqlite/modify-tables.sql &>> ${PREFIX}/sqlite.log\n" $(date +"%F %T")
-  # sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite < scripts/sqlite/modify-tables.sql &>> ${PREFIX}/sqlite.log
-
-  # alternative way
   printf "%s %s> delete\n" $(date +"%F %T")
   if [[ -e ${OUTPUT_DIR}/qa_catalogue.sqlite ]]; then
     rm ${OUTPUT_DIR}/qa_catalogue.sqlite
   fi
   printf "%s %s> create\n" $(date +"%F %T")
   sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite < scripts/sqlite/qa_catalogue.sqlite.sql
-  # php scripts/sqlite/import-issue-summary.php ${OUTPUT_DIR}
+
+  printf "%s %s> create importable files\n" $(date +"%F %T")
+  tail -n +2 ${OUTPUT_DIR}/issue-details-normalized.csv > ${OUTPUT_DIR}/issue-details-normalized_noheader.csv
+  tail -n +2 ${OUTPUT_DIR}/issue-summary.csv > ${OUTPUT_DIR}/issue-summary_noheader.csv
+
   printf "%s %s> import\n" $(date +"%F %T")
   sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite << EOF
-.separator ,
-.import --skip 1 ${OUTPUT_DIR}/issue-details-normalized.csv issue_details
-.import --skip 1 ${OUTPUT_DIR}/issue-summary.csv issue_summary
+.mode csv
+.import ${OUTPUT_DIR}/issue-details-normalized_noheader.csv issue_details
+.import ${OUTPUT_DIR}/issue-summary_noheader.csv issue_summary
 EOF
 
-  # .import ${OUTPUT_DIR}/issue-summary.csv issue_summary
+  printf "%s %s> delete importable files\n" $(date +"%F %T")
+  rm ${OUTPUT_DIR}/issue-details-normalized_noheader.csv
+  rm ${OUTPUT_DIR}/issue-summary_noheader.csv
+
   printf "%s %s> index\n" $(date +"%F %T")
   sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite < scripts/sqlite/modify-tables.sql &>> ${PREFIX}/sqlite.log
 }

scripts/sqlite/qa_catalogue.sqlite.sql

@@ -1,41 +1,18 @@
 BEGIN TRANSACTION;
--- CREATE TABLE IF NOT EXISTS "issue_groups" (
--- 	"categoryId"	INT,
--- 	"typeId"	INT,
--- 	"path"	TEXT,
--- 	"variants"	,
--- 	"records"	,
--- 	"instances"
--- );
 CREATE TABLE IF NOT EXISTS "issue_summary" (
-	"id"	INTEGER,
-	"MarcPath"	TEXT,
-	"categoryId"	INTEGER,
-	"typeId"	INTEGER,
-	"type"	TEXT,
-	"message"	TEXT,
-	"url"	TEXT,
-	"instances"	INTEGER,
-	"records"	INTEGER
+  "id"         INTEGER,
+  "MarcPath"   TEXT,
+  "categoryId" INTEGER,
+  "typeId"     INTEGER,
+  "type"       TEXT,
+  "message"    TEXT,
+  "url"        TEXT,
+  "instances"  INTEGER,
+  "records"    INTEGER
 );
 CREATE TABLE IF NOT EXISTS "issue_details" (
-	"id"	TEXT,
-	"errorId"	INTEGER,
-	"instances"	INTEGER
+  "id"         TEXT,
+  "errorId"    INTEGER,
+  "instances"  INTEGER
 );
--- CREATE INDEX IF NOT EXISTS "errorId" ON "issue_details" (
--- 	"errorId"
--- );
--- CREATE INDEX IF NOT EXISTS "recordId" ON "issue_details" (
--- 	"recordId"
--- );
--- CREATE INDEX IF NOT EXISTS "id" ON "issue_summary" (
--- 	"id"
--- );
--- CREATE INDEX IF NOT EXISTS "categoryId" ON "issue_summary" (
--- 	"categoryId"
--- );
--- CREATE INDEX IF NOT EXISTS "typeId" ON "issue_summary" (
--- 	"typeId"
--- );
 COMMIT;