-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathrrun-e-crf-batch.sh
executable file
·96 lines (77 loc) · 5.67 KB
/
rrun-e-crf-batch.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#!/usr/local/bin/bash
source ./rrun.env.sh
SCRIPT=./scripts/pipelines/concepts/run_crf.sh
SENTENCE_PIPELINE_KEY="SENTENCE_SEGMENTATION"
SENTENCE_PIPELINE_VERSION="recent"
AUGMENTED_SENTENCE_PIPELINE_KEY="DOC_TEXT_AUGMENTATION"
AUGMENTED_SENTENCE_PIPELINE_VERSION="recent"
OUTPUT_PIPELINE_VERSION="0.3.0"
CRAFT_SERVICE_URL=$CRAFT_CRF_SERVICE_URL_PROD
NLMDISEASE_SERVICE_URL=$NLMDISEASE_CRF_SERVICE_URL_PROD
# SUBSET_PREFIX=PUBMED_SUB_
# MAX_SUBSET_INDEX=36
SUBSET_PREFIX=PMC_SUBSET_
MAX_SUBSET_INDEX=41
OVERWRITE=YES
OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION="null"
# # use the below to run a single collection
# COLLECTION="PMC_SUBSET_39"
# $SCRIPT $CRAFT_SERVICE_URL $NLMDISEASE_SERVICE_URL $SENTENCE_PIPELINE_KEY $SENTENCE_PIPELINE_VERSION $AUGMENTED_SENTENCE_PIPELINE_KEY $AUGMENTED_SENTENCE_PIPELINE_VERSION $PROJECT_ID $COLLECTION $OVERWRITE ${STAGE_LOCATION} ${TEMP_LOCATION} $OUTPUT_PIPELINE_VERSION $OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION $JAR_VERSION &> "./logs/crf-${COLLECTION}.log" &
# sleep 3600
# COLLECTION="PMC_SUBSET_41"
# $SCRIPT $CRAFT_SERVICE_URL $NLMDISEASE_SERVICE_URL $SENTENCE_PIPELINE_KEY $SENTENCE_PIPELINE_VERSION $AUGMENTED_SENTENCE_PIPELINE_KEY $AUGMENTED_SENTENCE_PIPELINE_VERSION $PROJECT_ID $COLLECTION $OVERWRITE ${STAGE_LOCATION} ${TEMP_LOCATION} $OUTPUT_PIPELINE_VERSION $OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION $JAR_VERSION &> "./logs/crf-${COLLECTION}.log" &
# for INDEX in $(seq 31 1 $MAX_SUBSET_INDEX)
# do
# ind=$(($INDEX + 0))
# if (( ind <= $MAX_SUBSET_INDEX)); then
# echo "Starting crf pipeline... ${ind} $(date)"
# COLLECTION="${SUBSET_PREFIX}${ind}"
# $SCRIPT $CRAFT_SERVICE_URL $NLMDISEASE_SERVICE_URL $SENTENCE_PIPELINE_KEY $SENTENCE_PIPELINE_VERSION $AUGMENTED_SENTENCE_PIPELINE_KEY $AUGMENTED_SENTENCE_PIPELINE_VERSION $PROJECT_ID $COLLECTION $OVERWRITE ${STAGE_LOCATION} ${TEMP_LOCATION} $OUTPUT_PIPELINE_VERSION $OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION $JAR_VERSION &> "./logs/crf-${COLLECTION}.log" &
# sleep 3600 # I found that each run takes ~1.5 hours - and that if you overlap them then the cloud run instance count stays relatively high. We were running into an issue where sometimes the cloud run instance count would get set to a low level (~20s) for some runs causing them to take 5-9 hours instead of 1.5.
# fi
# # wait
# done
# for INDEX in $(seq 21 4 $MAX_SUBSET_INDEX)
# do
# ind=$(($INDEX + 0))
# if (( ind <= $MAX_SUBSET_INDEX)); then
# echo "Starting crf pipeline... ${ind} $(date)"
# COLLECTION="${SUBSET_PREFIX}${ind}"
# $SCRIPT $CRAFT_SERVICE_URL $NLMDISEASE_SERVICE_URL $SENTENCE_PIPELINE_KEY $SENTENCE_PIPELINE_VERSION $AUGMENTED_SENTENCE_PIPELINE_KEY $AUGMENTED_SENTENCE_PIPELINE_VERSION $PROJECT_ID $COLLECTION $OVERWRITE ${STAGE_LOCATION} ${TEMP_LOCATION} $OUTPUT_PIPELINE_VERSION $OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION $JAR_VERSION &> "./logs/crf-${COLLECTION}.log" &
# sleep 120
# fi
# ind=$(($INDEX + 1))
# if (( ind <= $MAX_SUBSET_INDEX)); then
# echo "Starting crf pipeline... ${ind} $(date)"
# COLLECTION="${SUBSET_PREFIX}${ind}"
# $SCRIPT $CRAFT_SERVICE_URL $NLMDISEASE_SERVICE_URL $SENTENCE_PIPELINE_KEY $SENTENCE_PIPELINE_VERSION $AUGMENTED_SENTENCE_PIPELINE_KEY $AUGMENTED_SENTENCE_PIPELINE_VERSION $PROJECT_ID $COLLECTION $OVERWRITE ${STAGE_LOCATION} ${TEMP_LOCATION} $OUTPUT_PIPELINE_VERSION $OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION $JAR_VERSION &> "./logs/crf-${COLLECTION}.log" &
# sleep 120
# fi
# ind=$(($INDEX + 2))
# if (( ind <= $MAX_SUBSET_INDEX)); then
# echo "Starting crf pipeline... ${ind} $(date)"
# COLLECTION="${SUBSET_PREFIX}${ind}"
# $SCRIPT $CRAFT_SERVICE_URL $NLMDISEASE_SERVICE_URL $SENTENCE_PIPELINE_KEY $SENTENCE_PIPELINE_VERSION $AUGMENTED_SENTENCE_PIPELINE_KEY $AUGMENTED_SENTENCE_PIPELINE_VERSION $PROJECT_ID $COLLECTION $OVERWRITE ${STAGE_LOCATION} ${TEMP_LOCATION} $OUTPUT_PIPELINE_VERSION $OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION $JAR_VERSION &> "./logs/crf-${COLLECTION}.log" &
# sleep 120
# fi
# ind=$(($INDEX + 3))
# if (( ind <= $MAX_SUBSET_INDEX)); then
# echo "Starting crf pipeline... ${ind} $(date)"
# COLLECTION="${SUBSET_PREFIX}${ind}"
# $SCRIPT $CRAFT_SERVICE_URL $NLMDISEASE_SERVICE_URL $SENTENCE_PIPELINE_KEY $SENTENCE_PIPELINE_VERSION $AUGMENTED_SENTENCE_PIPELINE_KEY $AUGMENTED_SENTENCE_PIPELINE_VERSION $PROJECT_ID $COLLECTION $OVERWRITE ${STAGE_LOCATION} ${TEMP_LOCATION} $OUTPUT_PIPELINE_VERSION $OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION $JAR_VERSION &> "./logs/crf-${COLLECTION}.log" &
# fi
# wait
# done
# ---------------------------------------------
# use the below to do a redo-crf run where the status entities have been
# tagged by the collection-assignment pipeline. In this case, the status
# entities will belong to a specific collection, e.g., REDO_CRF_20230915,
# however the documents will not be tagged with that collection. In order to
# run the redo, we set OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION to an appropriate
# collection for the documents, e.g., PUBMED or PMCOA. This way, the
# documents are not constrained to "REDO_CRF_20230915" since that would result
# in no documents being returned. The tradeoff is that more documents will be
# initially found than needed.
OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION=PMCOA
COLLECTION="REDO_2_CRF_PMOA_20231207"
$SCRIPT $CRAFT_SERVICE_URL $NLMDISEASE_SERVICE_URL $SENTENCE_PIPELINE_KEY $SENTENCE_PIPELINE_VERSION $AUGMENTED_SENTENCE_PIPELINE_KEY $AUGMENTED_SENTENCE_PIPELINE_VERSION $PROJECT_ID $COLLECTION $OVERWRITE ${STAGE_LOCATION} ${TEMP_LOCATION} $OUTPUT_PIPELINE_VERSION $OPTIONAL_DOCUMENT_SPECIFIC_COLLECTION $JAR_VERSION &> "./logs/crf-${COLLECTION}.log" &