Remove intermediate _clustered/_consensus directories/files from pixel and cell clustering pipeline

ark/phenotyping/cell_consensus_cluster.R

@@ -2,14 +2,13 @@
 # defined as the mean counts of each SOM pixel/meta cluster across all cell SOM clusters in each fov
 # (m x n table, where m is the number of cell SOM/meta clusters and n is the number of pixel SOM/meta clusters).
 
-# Usage: Rscript {pixelClusterCol} {maxK} {cap} {cellClusterPath} {clusterAvgPath} {cellConsensusPath} {clustToMeta} {seed}
+# Usage: Rscript {pixelClusterCol} {maxK} {cap} {cellMatPath} {clusterAvgPath} {clustToMeta} {seed}
 
 # - pixelClusterCol: the prefix of the columns defining pixel SOM/meta cluster counts per cell
 # - maxK: number of consensus clusters
 # - cap: maximum z-score cutoff
-# - cellClusterPath: path to the cell-level data containing the counts of each SOM pixel/meta clusters per cell, labeled with cell SOM clusters
+# - cellMatPath: path to the cell-level data containing the counts of each SOM pixel/meta clusters per cell, labeled with cell SOM clusters
 # - clusterAvgPath: path to the averaged cell data table (as defined above)
-# - cellConsensusPath: path to file where the cell consensus cluster results will be written
 # - clustToMeta: path to file where the SOM cluster to meta cluster mapping will be written
 # - seed: random factor
 
@@ -29,20 +28,17 @@ maxK <- strtoi(args[2])
 # get z-score scaling factor
 cap <- strtoi(args[3])
 
-# get the cell cluster path
-cellClusterPath <- args[4]
+# get the path to the cell data (with SOM labels)
+cellMatPath <- args[4]
 
 # get path to the averaged cluster data
 clusterAvgPath <- args[5]
 
-# get consensus cluster write path
-cellConsensusPath <- args[6]
-
 # get the clust to meta write path
-clustToMeta <- args[7]
+clustToMeta <- args[6]
 
 # set the random seed
-seed <- strtoi(args[8])
+seed <- strtoi(args[7])
 set.seed(seed)
 
 print("Reading cluster averaged data")
@@ -66,9 +62,9 @@ names(som_to_meta_map) <- clusterAvgs$cell_som_cluster
 
 # append cell_meta_cluster to data
 print("Writing consensus clustering")
-cellClusterData <- arrow::read_feather(cellClusterPath)
+cellClusterData <- arrow::read_feather(cellMatPath)
 cellClusterData$cell_meta_cluster <- som_to_meta_map[as.character(cellClusterData$cell_som_cluster)]
-arrow::write_feather(as.data.table(cellClusterData), cellConsensusPath)
+arrow::write_feather(as.data.table(cellClusterData), cellMatPath)
 
 # save the mapping from cell_som_cluster to cell_meta_cluster
 print("Writing SOM to meta cluster mapping table")

ark/phenotyping/pixel_consensus_cluster.R

@@ -1,14 +1,13 @@
 # Runs consensus clustering on the pixel data averaged across all channels
 
-# Usage: Rscript {fovs} {markers} {maxK} {cap} {pixelClusterDir} {clusterAvgPath} {pixelMatConsensus} {clustToMeta} {seed}
+# Usage: Rscript {fovs} {markers} {maxK} {cap} {pixelMatDir} {clusterAvgPath} {clustToMeta} {seed}
 
 # - fovs: list of fovs to cluster
 # - markers: list of channel columns to use
 # - maxK: number of consensus clusters
 # - cap: max z-score cutoff
-# - pixelClusterDir: path to the pixel data with SOM clusters
+# - pixelMatDir: path to the pixel data with SOM clusters
 # - clusterAvgPath: path to the averaged cluster data
-# - pixelMatConsensus: path to file where the consensus cluster results will be written
 # - clustToMeta: path to file where the SOM cluster to meta cluster mapping will be written
 # - seed: random factor
 
@@ -31,20 +30,17 @@ maxK <- strtoi(args[3])
 # get z-score scaling factor
 cap <- strtoi(args[4])
 
-# get path to the clustered pixel data
-pixelClusterDir <- args[5]
+# get path to the pixel data
+pixelMatDir <- args[5]
 
 # get path to the averaged channel data
 clusterAvgPath <- args[6]
 
-# get consensus clustered write path
-pixelMatConsensus <- args[7]
-
 # get the clust to meta write path
-clustToMeta <- args[8]
+clustToMeta <- args[7]
 
 # set the random seed
-seed <- strtoi(args[9])
+seed <- strtoi(args[8])
 set.seed(seed)
 
 # read cluster averaged data
@@ -70,15 +66,14 @@ print("Writing consensus clustering results")
 for (i in 1:length(fovs)) {
     # read in pixel data, we'll need the cluster column for mapping
     fileName <- file.path(fovs[i], "feather", fsep=".")
-    matPath <- file.path(pixelClusterDir, fileName)
+    matPath <- file.path(pixelMatDir, fileName)
     fovPixelData <- arrow::read_feather(matPath)
 
     # assign hierarchical cluster labels
     fovPixelData$pixel_meta_cluster <- som_to_meta_map[as.character(fovPixelData$pixel_som_cluster)]
 
-    # write consensus clustered data
-    clusterPath <- file.path(pixelMatConsensus, fileName)
-    arrow::write_feather(as.data.table(fovPixelData), clusterPath)
+    # write consensus clustered data, overwrite original data with the same data with meta cluster label
+    arrow::write_feather(as.data.table(fovPixelData), matPath)
 
     # print an update every 10 fovs
     if (i %% 10 == 0) {

ark/phenotyping/run_cell_som.R

@@ -1,10 +1,10 @@
 # Assigns cluster labels to cell data using a trained SOM weights matrix
 
-# Usage: Rscript run_cell_som.R {clusterCountsNormPath} {cellWeightsPath} {cellClusterNormPath}
+# Usage: Rscript run_cell_som.R {clusterCountsNormPath} {cellWeightsPath} {cellMatNormPath}
 
 # - clusterCountsNormPath: path to file with counts of unique cells (rows) by unique pixel SOM/meta clusters (columns), with counts normalized by cell size
 # - cellWeightsPath: path to the SOM weights file
-# - cellClusterNormPath: the path to write the normalized pixel SOM/meta cluster count data (normalized) with cell SOM labelss. This will be used for consensus clustering.
+# - cellMatNormPath: the path to write the normalized pixel SOM/meta cluster count data (normalized) with cell SOM labelss. This will be used for consensus clustering.
 
 library(arrow)
 library(data.table)
@@ -19,8 +19,8 @@ clusterCountsPath <- args[1]
 # get the weights write path
 cellWeightsPath <- args[2]
 
-# get the cluster write path (normalized)
-cellClusterPathNorm <- args[3]
+# get the data write path (normalized)
+cellMatPathNorm <- args[3]
 
 # read the cluster counts data (norm)
 print("Reading the cluster counts data")
@@ -62,4 +62,4 @@ clusterCountsNorm$cell_som_cluster <- as.integer(clusters[,1])
 
 # write to feather
 print("Writing clustered data")
-arrow::write_feather(as.data.table(clusterCountsNorm), cellClusterPathNorm)
+arrow::write_feather(as.data.table(clusterCountsNorm), cellMatPathNorm)

ark/phenotyping/run_pixel_som.R

@@ -1,12 +1,11 @@
 # Assigns cluster labels to pixel data using a trained SOM weights matrix
 
-# Usage: Rscript run_pixel_som.R {fovs} {pixelMatDir} {normValsPath} {pixelWeightsPath} {pixelClusterDir}
+# Usage: Rscript run_pixel_som.R {fovs} {pixelMatDir} {normValsPath} {pixelWeightsPath}
 
 # - fovs: list of fovs to cluster
 # - pixelMatDir: path to directory containing the complete pixel data
 # - normValsPath: path to the 99.9% normalization values file (created during preprocessing)
 # - pixelWeightsPath: path to the SOM weights file
-# - pixelClusterDir: path to directory where the clustered data will be written to
 
 library(arrow)
 library(data.table)
@@ -27,9 +26,6 @@ normValsPath <- args[3]
 # get path to the weights
 pixelWeightsPath <- args[4]
 
-# get the cluster write path directory
-pixelClusterDir <- args[5]
-
 # read the weights
 somWeights <- as.matrix(arrow::read_feather(pixelWeightsPath))
 
@@ -64,9 +60,8 @@ for (i in 1:length(fovs)) {
     # assign cluster labels column to pixel data
     fovPixelData$pixel_som_cluster <- as.integer(clusters[,1])
 
-    # write to feather
-    clusterPath <- file.path(pixelClusterDir, fileName)
-    arrow::write_feather(as.data.table(fovPixelData), clusterPath)
+    # write to feather, overwrite original data with the same data with SOM cluster label
+    arrow::write_feather(as.data.table(fovPixelData), matPath)
 
     # print an update every 10 fovs
     # TODO: find a way to capture sprintf to the console