Cleanup, rework CI

.gitlab-ci.yml

@@ -63,8 +63,11 @@ bld:api-test:
 
 deploy-checks:
     stage: prerelease
+    variables:
+        PACKAGE_NAME: modbampy
     script:
         - !reference [.check, argp-c-version]
+        - !reference [.check, python-version]
         - !reference [.check, changelog]
     rules:
         - if: '$CI_COMMIT_TAG =~ /^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$/'
@@ -109,38 +112,52 @@ bld:py-sdist:
             - dist/*.tar.gz
 
 .many-linux: &many-linux-def
-  image: quay.io/pypa/manylinux2010_x86_64
-  stage: prerelease
-  artifacts:
-      paths:
-          - wheelhouse/*.whl
-
-bld:py-6:
-    <<: *many-linux-def
+    stage: prerelease
+    variables:
+        DO_COUNT_TEST: 1
     script:
-        - ./build-wheels.sh . 6
+        - echo "Building a Python ${PYWHEEL} wheel on manylinux_${FLAVOUR}"
+        - ./build-wheels.sh . ${PYWHEEL}
+    artifacts:
+        paths:
+            - wheelhouse-final/*.whl
+    only:
+        - tags
 
-bld:py-7:
-    <<: *many-linux-def
-    script:
-        - ./build-wheels.sh . 7
 
-bld:py-8:
-    <<: *many-linux-def
-    script:
-        - ./build-wheels.sh . 8
+make-wheels-2010:
+    extends: .many-linux
+    image: "quay.io/pypa/manylinux2010_x86_64"
+    parallel:
+        matrix:
+            - PYWHEEL: [7, 8]
+              FLAVOUR: ["2010"]
+
+
+make-wheels-2014:
+    extends: .many-linux
+    image: "quay.io/pypa/manylinux2014_x86_64"
+    parallel:
+        matrix:
+            - PYWHEEL: [7, 8, 9]
+              FLAVOUR: ["2014"]
+
+
+make-wheels-2_24:
+    extends: .many-linux
+    image: "quay.io/pypa/manylinux_2_24_x86_64"
+    parallel:
+        matrix:
+            - PYWHEEL: [8, 9, 10]
+              FLAVOUR: ["2_24"]
 
-bld:py-9:
-    <<: *many-linux-def
-    script:
-        - ./build-wheels.sh . 9
 
 deploy:pypi:
     stage: release
     script:
         - *minimal-python
         - make pypi_build/bin/activate
         - source pypi_build/bin/activate
-        - twine upload --non-interactive dist/modbampy*.tar.gz wheelhouse/modbampy*.whl
+        - twine upload --non-interactive dist/modbampy*.tar.gz wheelhouse-final/modbampy*.whl
     rules:
         - if: '$CI_COMMIT_TAG =~ /^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$/'

Makefile

@@ -89,7 +89,7 @@ clean: clean_obj clean_htslib
 .PHONY: mem_check
 mem_check: modbam2bed
 	$(VALGRIND) --error-exitcode=1 --tool=memcheck --leak-check=full --show-leak-kinds=all -s \
-		./modbam2bed -b 0.66 -a 0.33 -t 2 -r ecoli1 test_data/ecoli.fasta.gz test_data/400ecoli.bam test_data/400ecoli.bam > /dev/null
+		./modbam2bed --threshold 0.66 -t 2 -r ecoli1 test_data/ecoli.fasta.gz test_data/400ecoli.bam test_data/400ecoli.bam > /dev/null
 
 
 .PHONY: test_api

build-wheels.sh

@@ -10,13 +10,18 @@ shift
 echo "Changing cwd to ${workdir}"
 cd ${workdir}
 
-yum install -y zlib-devel bzip2 bzip2-devel xz-devel curl-devel openssl-devel ncurses-devel
+# some many linux containers are centos-based, others are debian!
+if [ -f /etc/centos-release ]; then
+    yum install -y zlib-devel bzip2 bzip2-devel xz-devel curl-devel openssl-devel ncurses-devel
+else
+    apt update
+    apt install -y zlib1g-dev libbz2-dev liblzma-dev libncurses5-dev libcurl4-gnutls-dev libssl-dev libffi-dev
+fi
 
 # downgrade autoconf to work more nicely with htslib
 curl -L -O http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz
 tar zxf autoconf-2.69.tar.gz
 cd autoconf-2.69
-yum install -y openssl-devel
 ./configure
 make && make install
 cd ..
@@ -33,7 +38,7 @@ ls /opt/python/
 
 # Compile wheels
 for minor in $@; do
-    if [[ "${minor}" == "8" ]]  || [[ "${minor}" == "9" ]]; then
+    if [[ "${minor}" == "8" ]]  || [[ "${minor}" == "9" ]] || [[ "${minor}" == "10" ]]; then
         PYBIN="/opt/python/cp3${minor}-cp3${minor}/bin"
     else
         PYBIN="/opt/python/cp3${minor}-cp3${minor}m/bin"
@@ -55,7 +60,7 @@ unset LD_LIBRARY_PATH
 
 ## Install packages
 for minor in $@; do
-    if [[ "${minor}" == "8" || "${minor}" == "9" ]]; then
+    if [[ "${minor}" == "8" ]]  || [[ "${minor}" == "9" ]] || [[ "${minor}" == "10" ]]; then
         PYBIN="/opt/python/cp3${minor}-cp3${minor}/bin"
     else
         PYBIN="/opt/python/cp3${minor}-cp3${minor}m/bin"
@@ -65,4 +70,5 @@ for minor in $@; do
     "${PYBIN}"/modbampy --pileup test_data/400ecoli.bam ecoli1 105000 105100
 done
 
-cd wheelhouse && ls | grep -v "${PACKAGE_NAME}.*manylinux" | xargs rm
+mkdir wheelhouse-final
+cp wheelhouse/${PACKAGE_FILE_NAME}*manylinux* wheelhouse-final

modbampy/__init__.py

@@ -8,7 +8,7 @@
 import libmodbampy
 
 # remember to bump version in src/version.h too
-__version__ = "0.8.0"
+__version__ = "0.9.0"
 ffi = libmodbampy.ffi
 libbam = libmodbampy.lib
 
@@ -143,8 +143,8 @@ def reads(
     def pileup(
             self, chrom, start, end,
             read_group=None, tag_name=None, tag_value=None,
-            low_threshold=0.33, high_threshold=0.66, mod_base="m",
-            max_depth=None, canon_base=None, combine=False):
+            low_threshold=0.33, high_threshold=0.66, threshold=0.66,
+            mod_base="m", max_depth=None, canon_base=None, combine=False):
         """Create a base count matrix.
 
         :param chrom: reference sequence from BAM.
@@ -153,10 +153,7 @@ def pileup(
         :param read group: read group of read to return.
         :param tag_name: read tag to check during read filtering.
         :param tag_value: tag value for reads to keep.
-        :param low_threshold: threshold below which a base is determined as
-            not the modified base.
-        :param high_threshold: threshold above which a base is determined
-            to be the modified base.
+        :param threshold: probability filter threshold for excluding calls from counts.
         :param mod_base: ChEBI code of modified base to examine.
         :param max_depth: maximum read depth to examine.
         :param canon_base: canonical base corresponding to `mod_base`.
@@ -168,8 +165,9 @@ def pileup(
         for thresh in (low_threshold, high_threshold):
             if thresh < 0.0 or thresh > 1.0:
                 raise ValueError("Thresholds should be in (0,1).")
-        low_threshold, high_threshold = (
-            int(x * 255.0) for x in (low_threshold, high_threshold))
+        threshold = int(threshold * 255.0)
+        # C code currently uses high_threshold as the only threshold
+        high_threshold = threshold
         read_group, tag_name, tag_value = _tidy_args(
             read_group, tag_name, tag_value)
 
@@ -182,7 +180,7 @@ def pileup(
         plp_data = libbam.calculate_pileup(
             fsets, chrom.encode(), start, end,
             read_group, tag_name, tag_value,
-            low_threshold, high_threshold, mod_base.struct,
+            threshold, mod_base.struct,
             combine, max_depth)
         # TODO: check for NULL
 

src/args.c

@@ -104,7 +104,7 @@ static error_t parse_opt (int key, char *arg, struct argp_state *state) {
             if (thresh < 0 || thresh > 1.0) {
                 argp_error (state, "Threshold parameter must be in (0,1), got %s", arg);
             }
-            arguments->highthreshold = (int)(thresh * 255);
+            arguments->threshold = (int)(thresh * 255);
             break;
         case 'm':
             for (size_t i = 0; i < n_mod_bases; ++i) {
@@ -215,8 +215,7 @@ arguments_t parse_arguments(int argc, char** argv) {
     arguments_t args;
     args.mod_base = default_mod_base;
     args.combine = false;
-    args.lowthreshold = -1;
-    args.highthreshold = (int)(0.66 * MAX_QUAL);
+    args.threshold = (int)(0.66 * MAX_QUAL);
     args.bam = NULL;
     args.ref = NULL;
     args.region = NULL;
@@ -252,10 +251,6 @@ arguments_t parse_arguments(int argc, char** argv) {
         fprintf(stderr, "ERROR: If --haplotype is given neither of --tag_name or --tag_value should be provided.\n");
         exit(1);
     }
-    if (args.highthreshold < args.lowthreshold) {
-        fprintf(stderr, "ERROR: --highthreshold must be larger than --lowthreshold\n");
-        exit(1);
-    }
     if (strncmp("5mC", args.mod_base.abbrev, 3) == 0 || strncmp("5hmC", args.mod_base.abbrev, 4)) {
         fprintf(stderr,
 "WARNING: You have specified either 5mC or 5hmC as a modified base.\n\

src/args.h

@@ -21,8 +21,7 @@ typedef struct arguments {
     bool extended;
     bool accumulated;
     int threads;
-    int lowthreshold;
-    int highthreshold;
+    int threshold;
     char* prefix;
     bool pileup;
     int hts_maxcnt;

src/counts.c

@@ -455,8 +455,7 @@ bool query_mod_subtag(hts_base_mod_state *state, int qtype, int qcanonical, char
  *  @param read_group by which to filter alignments.
  *  @param tag_name by which to filter alignments.
  *  @param tag_value associated with tag_name
- *  @param lowthreshold highest probability to call base as canonical.
- *  @param highthreshold lowest probablity to call base as modified.
+ *  @param threshold probability filter for excluding calls from counts.
  *  @param mb BAM code for modified base to report. (e.g. h for 5hmC), or a ChEBI code.
  *  @param combine combine all modified bases corresponding to same canonical base as mb
  *  @returns a pileup data pointer.
@@ -467,7 +466,7 @@ bool query_mod_subtag(hts_base_mod_state *state, int qtype, int qcanonical, char
 plp_data calculate_pileup(
         const set_fsets *fsets, const char *chr, int start, int end,
         const char *read_group, const char tag_name[2], const int tag_value,
-        int lowthreshold, int highthreshold, mod_base mb, bool combine, int max_depth) {
+        int threshold, mod_base mb, bool combine, int max_depth) {
 
     static bool shown_second_strand_warning = false;
 
@@ -610,16 +609,16 @@ plp_data calculate_pileup(
                         // we found some mods, lets not worry about funny mixes
                         // of calls and no calls i.e. were assuming we have a call
                         // for all the mods present (implicit non-mod doesn't matter here therefore).
-                        if (canon_score > highthreshold) { // implied canon score 
+                        if (canon_score > threshold) { // implied canon score 
                             base_i = num2countbase[bam_is_rev(p->b) ? base_j + 16 : base_j];
                         }
                         else if (best_mod == our_mod) { // the mod requested
-                            base_i = (best_score > highthreshold) ?
+                            base_i = (best_score > threshold) ?
                                 (bam_is_rev(p->b) ? rev_mod : fwd_mod) :
                                 (bam_is_rev(p->b) ? rev_filt : fwd_filt);
                         }
                         else { // some other mod in the family
-                            base_i = (best_score > highthreshold) ?
+                            base_i = (best_score > threshold) ?
                                 (bam_is_rev(p->b) ? rev_in_family : fwd_in_family) :  // either mod or other depending on combine
                                 (bam_is_rev(p->b) ? rev_filt : fwd_filt);
                         }

src/counts.h

@@ -168,6 +168,6 @@ void print_bedmethyl(
 plp_data calculate_pileup(
     const set_fsets *fsets, const char *chr, int start, int end,
     const char *read_group, const char tag_name[2], const int tag_value,
-    int lowthreshold, int highthreshold, mod_base mb, bool combine, int max_depth);
+    int threshold, mod_base mb, bool combine, int max_depth);
 
 #endif

src/modbam2bed.c

@@ -32,7 +32,7 @@ void *pileup_worker(void *arg) {
     plp_data pileup = calculate_pileup(
         files, j.chr, j.start, j.end,
         j.args.read_group, j.args.tag_name, j.args.tag_value,
-        j.args.lowthreshold, j.args.highthreshold, j.args.mod_base, j.args.combine,
+        j.args.threshold, j.args.mod_base, j.args.combine,
         j.args.hts_maxcnt);
     destroy_filesets(files);
     free(arg);
@@ -57,7 +57,7 @@ void process_region(arguments_t args, const char *chr, int start, int end, char
     plp_data pileup = calculate_pileup(
         args.bam, chr, start, end,
         args.read_group, args.tag_name, args.tag_value,
-        args.lowthreshold, args.highthreshold, args.mod_base, args.combine,
+        args.threshold, args.mod_base, args.combine,
         args.hts_maxcnt);
     if (pileup == NULL) return;