From d0fd86b68fb4f23f99165b5511259988735e54d6 Mon Sep 17 00:00:00 2001
From: Daniel Scholl <dascholl@microsoft.com>
Date: Tue, 24 Sep 2024 14:37:21 -0500
Subject: [PATCH] Changed CSV stuff.

---
 bicep/modules/blade_common.bicep             |   4 +
 bicep/modules/script-share-csvdag/README.md  |   5 +
 bicep/modules/script-share-csvdag/main.bicep |  51 +++++++-
 bicep/modules/script-share-csvdag/script.sh  | 122 ++++++++++++++-----
 dags/csv_parser.zip                          | Bin 10061 -> 0 bytes
 dags/test_fetch_remote.py                    |  39 ------
 dags/test_pip_packages.py                    |  36 ------
 7 files changed, 146 insertions(+), 111 deletions(-)
 create mode 100644 bicep/modules/script-share-csvdag/README.md
 delete mode 100644 dags/csv_parser.zip
 delete mode 100644 dags/test_fetch_remote.py
 delete mode 100644 dags/test_pip_packages.py

diff --git a/bicep/modules/blade_common.bicep b/bicep/modules/blade_common.bicep
index b9eabeed..dc830147 100644
--- a/bicep/modules/blade_common.bicep
+++ b/bicep/modules/blade_common.bicep
@@ -446,6 +446,10 @@ module csvDagShareUpload './script-share-csvdag/main.bicep' = {
     shareName: 'airflow-dags'
     filename: 'airflowdags'
     fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/-/archive/master/csv-parser-master.tar.gz'
+    keyVaultUrl: keyvault.outputs.uri
+    insightsKey: insights.outputs.instrumentationKey
+    clientId: applicationClientId
+    clientSecret: applicationClientSecret
     useExistingManagedIdentity: true
     managedIdentityName: deploymentScriptIdentity
     existingManagedIdentitySubId: subscription().subscriptionId
diff --git a/bicep/modules/script-share-csvdag/README.md b/bicep/modules/script-share-csvdag/README.md
new file mode 100644
index 00000000..dcc80c14
--- /dev/null
+++ b/bicep/modules/script-share-csvdag/README.md
@@ -0,0 +1,5 @@
+# NOTE
+
+This module is tightly coupled to the csv-parser dag. It is used to upload a file to a blob storage account and then execute a script on the file.
+
+It shouldn't be done this way and we have to move this to a kubernetesjob that can run a python script and just copy into a pvc mount.
\ No newline at end of file
diff --git a/bicep/modules/script-share-csvdag/main.bicep b/bicep/modules/script-share-csvdag/main.bicep
index 6e857e7f..bf45aa1c 100644
--- a/bicep/modules/script-share-csvdag/main.bicep
+++ b/bicep/modules/script-share-csvdag/main.bicep
@@ -44,6 +44,17 @@ param initialScriptDelay string = '30s'
 @description('When the script resource is cleaned up')
 param cleanupPreference string = 'OnSuccess'
 
+@description('Keyvault url')
+param keyVaultUrl string
+
+@description('App Insights Instrumentation Key')
+param insightsKey string
+
+@description('Client Id for the service principal')
+param clientId string
+
+@description('Client Secret for the service principal')
+param clientSecret string
 
 resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' existing = {
   name: storageAccountName
@@ -69,16 +80,48 @@ resource rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(
   }
 }
 
-var searchAndReplace = [ 
+var findAndReplace = [ 
   {
     find: '{| DAG_NAME |}'
     replace: 'csv-parser'
   }
   {
     find: '{| DOCKER_IMAGE |}'
-    replace: 'msosdu.azurecr.io/csv-parser-msi:v5'
+    replace: 'community.opengroup.org:5555/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/csv-parser-v0-27-0-azure-1:60747714ac490be0defe8f3e821497b3cce03390'
+  }
+  {
+    find: '{| NAMESPACE |}'
+    replace: 'airflow'
+  }
+  {
+    find: '{| K8S_POD_OPERATOR_KWARGS or {} |}'
+    replace: {
+      labels: {
+        aadpodidbinding: 'osdu-identity'
+      }
+      annotations: {
+        'sidecar.istio.io/inject': 'false'
+      }
+    }
+  }
+  {
+    find: '{| ENV_VARS or {} |}'
+    replace: {
+      storage_service_endpoint: 'http://storage.osdu-core.svc.cluster.local/api/storage/v2'
+      schema_service_endpoint: 'http://schema.osdu-core.svc.cluster.local/api/schema-service/v1'
+      search_service_endpoint: 'http://search.osdu-core.svc.cluster.local/api/search/v2'
+      partition_service_endpoint: 'http://partition.osdu-core.svc.cluster.local/api/partition/v1'
+      unit_service_endpoint: 'http://unit.osdu-core.svc.cluster.local/api/unit/v2/unit/symbol'
+      file_service_endpoint: 'http://file.osdu-core.svc.cluster.local/api/file/v2'
+      KEYVAULT_URI: keyVaultUrl
+      appinsights_key: insightsKey
+      azure_paas_podidentity_isEnabled: 'false'
+      AZURE_TENANT_ID: subscription().tenantId
+      AZURE_CLIENT_ID: clientId
+      AZURE_CLIENT_SECRET: clientSecret
+      aad_client_id: clientId
+    }
   }
-
 ]
 
 resource uploadFile 'Microsoft.Resources/deploymentScripts@2023-08-01' = {
@@ -102,7 +145,7 @@ resource uploadFile 'Microsoft.Resources/deploymentScripts@2023-08-01' = {
       { name: 'URL', value: fileurl }
       { name: 'SHARE', value: shareName }
       { name: 'initialDelay', value: initialScriptDelay }
-      { name: 'SEARCH_AND_REPLACE', value: string(searchAndReplace) }
+      { name: 'SEARCH_AND_REPLACE', value: string(findAndReplace) }
     ]
     scriptContent: loadTextContent('script.sh')
     cleanupPreference: cleanupPreference
diff --git a/bicep/modules/script-share-csvdag/script.sh b/bicep/modules/script-share-csvdag/script.sh
index f1578e1b..ebab9ba1 100644
--- a/bicep/modules/script-share-csvdag/script.sh
+++ b/bicep/modules/script-share-csvdag/script.sh
@@ -22,14 +22,18 @@ set -e
 # 
 # The SEARCH_AND_REPLACE variable is required for the script to perform the find/replace operations.
 
-
+# Ensure necessary packages are installed
+apk add --no-cache curl zip jq
 
 echo "Waiting on Identity RBAC replication (${initialDelay})"
 sleep "${initialDelay}"
-apk add --no-cache curl zip
+
+echo "###########################"
+echo "${SEARCH_AND_REPLACE}"
+echo "###########################"
 
 # Download the source code and extract it.
-url_basename=$(basename ${URL})
+url_basename=$(basename "${URL}")
 echo "Derived filename from URL: ${url_basename}"
 echo "Downloading file from ${URL} to ${url_basename}"
 curl -so "${url_basename}" "${URL}"
@@ -37,40 +41,94 @@ echo "Extracting tar.gz archive..."
 mkdir -p extracted_files
 tar -xzf "${url_basename}" --strip-components=1 -C extracted_files
 
-
-# Find and Replace.
+# Process the replacements
 csv_file="extracted_files/${FILE}/csv_ingestion_all_steps.py"
+output_file="extracted_files/${FILE}/csv-parser.py"
+
 if [ -f "${csv_file}" ]; then
     echo "Processing ${csv_file} file"
 
-    # Escape patterns for sed
-    escape_sed_pattern() {
-        printf '%s' "$1" | sed 's/[\/&]/\\&/g; s/[][$.*^]/\\&/g'
-    }
-    escape_sed_replacement() {
-        printf '%s' "$1" | sed 's/[\/&]/\\&/g'
-    }
-
-    # Create sed script from search and replace JSON
-    sed_script_file="sed_script.sed"
-
-    echo "${SEARCH_AND_REPLACE}" | jq -c '.[]' | while IFS= read -r item; do
-        find=$(echo "$item" | jq -r '.find')
-        replace=$(echo "$item" | jq -r '.replace')
-
-        find_escaped=$(escape_sed_pattern "$find")
-        replace_escaped=$(escape_sed_replacement "$replace")
-
-        echo "find: ${find_escaped}"
-        echo "replace: ${replace_escaped}"
-
-        echo "s/${find_escaped}/${replace_escaped}/g" >> "$sed_script_file"
+    # Number of replacements
+    num_replacements=$(echo "${SEARCH_AND_REPLACE}" | jq '. | length')
+
+    # Initialize arrays
+    declare -a finds
+    declare -a replaces
+    declare -a replace_types
+
+    # Build arrays
+    for (( idx=0; idx<${num_replacements}; idx++ )); do
+        finds[$idx]=$(echo "${SEARCH_AND_REPLACE}" | jq -r ".[$idx].find")
+        replace_type=$(echo "${SEARCH_AND_REPLACE}" | jq -r ".[$idx].replace | type")
+        replace_types[$idx]=$replace_type
+        if [ "$replace_type" == "string" ]; then
+            replaces[$idx]=$(echo "${SEARCH_AND_REPLACE}" | jq -r ".[$idx].replace")
+        else
+            replaces[$idx]=$(echo "${SEARCH_AND_REPLACE}" | jq -c ".[$idx].replace")
+        fi
     done
 
-    echo "Running sed script:"
-    cat "$sed_script_file"
-    sed -f "$sed_script_file" "$csv_file" > "extracted_files/${FILE}/csv-parser.py"
-    rm "$sed_script_file"
+    # Empty the output file
+    > "$output_file"
+
+    # Read the input file line by line
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        replaced=0
+        # For each 'find'/'replace' pair
+        for idx in "${!finds[@]}"; do
+            find_placeholder="${finds[$idx]}"
+            replace_value="${replaces[$idx]}"
+            replace_type="${replace_types[$idx]}"
+
+            if [[ "$line" == *"$find_placeholder"* ]]; then
+                # Line contains the placeholder
+
+                if [ "$replace_type" == "object" ]; then
+                    # 'replace_value' is a JSON object
+
+                    # Split the line at the placeholder
+                    line_before_placeholder="${line%%$find_placeholder*}"
+                    line_after_placeholder="${line#*$find_placeholder}"
+
+                    # Get the indentation of the line up to the placeholder
+                    leading_spaces=$(echo "$line_before_placeholder" | sed -n 's/^\(\s*\).*$/\1/p')
+
+                    # Format the JSON with jq
+                    formatted_json=$(echo "$replace_value" | jq '.')
+
+                    # Indent the JSON
+                    indented_json=$(echo "$formatted_json" | sed "s/^/${leading_spaces}/")
+
+                    # Output the line before the placeholder (excluding placeholder)
+                    echo -n "$line_before_placeholder" >> "$output_file"
+
+                    # Output the indented JSON
+                    echo "$indented_json" >> "$output_file"
+
+                    # Output the rest of the line after the placeholder, if any
+                    if [ -n "$line_after_placeholder" ]; then
+                        echo "$line_after_placeholder" >> "$output_file"
+                    fi
+                else
+                    # 'replace_value' is a string
+
+                    # Replace the placeholder in the line
+                    replaced_line="${line//$find_placeholder/$replace_value}"
+
+                    # Output the modified line
+                    echo "$replaced_line" >> "$output_file"
+                fi
+                replaced=1
+                break  # Skip checking other placeholders for this line
+            fi
+        done
+        if [[ $replaced -eq 0 ]]; then
+            # Line did not contain any placeholder
+            echo "$line" >> "$output_file"
+        fi
+    done < "$csv_file"
+
+    # Remove the original file
     rm "$csv_file"
 fi
 
@@ -83,4 +141,4 @@ zip -r "${current_dir}/${zip_filename}" .
 cd - || exit 1
 
 az storage file upload -s "${SHARE}" --source "${zip_filename}" -onone
-echo "Zip file ${zip_filename} uploaded to file share ${SHARE}."
+echo "Zip file ${zip_filename} uploaded to file share ${SHARE}."
\ No newline at end of file
diff --git a/dags/csv_parser.zip b/dags/csv_parser.zip
deleted file mode 100644
index db866e7058c630dccd42153006f1f177086021f7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 10061
zcmbtaWmp{9wr$*@ad&rjhv4pRL4pMLAVGqLpdmQHgL~ry4er4kcXtRDcx2|zojWsm
z^X7i9`m5@yAHCK-`<&i;*I8Rt0SX!s@Z&(U;MM!(=3f^efC%8|YT<5T?&@LUWai>(
z>B6d|i2#6NE;zONak_gU0U)3+Apihqtf$|oJ|X@1h6iAI^XhHZhe>I}007je007Dp
z(h~^tuOKW=-oL=~4V+r_Kb?PqDbrbWT;l`29yLL?B(>JDZxG%!j@31awrVcK0_G)N
z6%TMkKsaIwRuk*j?LGQ)j=%-xA0F;{wR>&xl4jz^gyqMlq^Wixk0uY+7&5G{9lyyB
z6;QBvJ)=HZn%C!Y$Tt=30#e*GM=0UpKBH(@$i1t=l0FS#buB22lfOgz5=7sD{FN#R
zp$H!oS`sRp*q`_OgFwDAqB^7T*!5~$@tbar!J<{9OqZHV%(^@&TX-!-qY@?;Ax4{;
zXu1V&-BnO_f%W?Z-L-`Yg8Fnchx{oJY?4#7fRIn-#o~+J5EU5_!~y}~dW{Z&7u3RL
z{Fo6m>appzP~ogxQ>AM`i~~S703N6-WVPrjE_~TQX-7Wu-a|j4GU^O#%oCI^I)^!Y
z_Fi1Rdc=<~L^`PwCz5#9el`*xBy39YjuT~4g?RvHEnU5v2gW_#SuG&xyJ&kO?lv2T
z%!KR2Qu^~`))Z%6E@nLF#?qz462w&rKjxd_05>OkqA$-8s5>xK_k~3%BLRpdPTn(S
zDoiXunM#Jagx!T@cdv;?PZMLIFdQpp>_;<h;zXHeRHVs;RPFWxA>B^z@g9PPp*rM=
z;aj)7`)yOGQ;Sey@Y&<BQokE~eTYm+7(s{0;H<CNpx;l|$DFSu(s12=N7Dgw!;xl~
z&~ZO4=-Rpi=AsIg<0C|-HK#&mJrTr<a>5Tu3jFTOO=NKEjt8U7)R4aZijzdT0oBSw
z5J^THI&#Y-Ok-g*x1Wh>{EgxkF7Hw+RC$q!><e!Ztf0aVGJ*5>vJkkkc!y#>6zeMj
z&-|*E(zi7srd?TRf=isFLuWcHb?4?)H^D*Y;|!xak>~`BV{?mgS62aIwA8Jggmmf`
z5K1GTI@<Tj%X7EI#QM3Chr8yo1y|1P_4cKk#!wFs0rk*)m18ewG=(y~gzJmGpK-kM
zxkSEi^$X#rdnQJ#%8*_Lw=xf+7G!;Et;h+x@K^)QNNLH?YODn}D?DJ_-W|$F&p}hk
zN~erV|F!)s4Q@ha8ZJ&C(P4SYbIY$;-kE2!u<fJRJibsULl9I|eKc7Fse&|>S3rGN
zmPR(OJ`lTi1-Oh%<TLp;(0glN>7dsJf?eI8!93N&X2j0W^W4XehPWcT;&Tz0VoAr|
zY&$XMXiLL9Pz}3cxz-f@*r2lV!?Cv!fg`=6*zulYEb}%}$FRYC+++E3=s@shas8Y`
z0m#3!s{VrV(daw8ZOA6Wy$9h!6om^XVGe`DJeGtl5f&M4YQ?m40$d$)c88k)(P&tb
zNiKgRXiRiPDl}T+Fs~c4&>hbiekQx!MTOzC%`wX|bbX~lO}2cS*^ZhPa+;4~tia$T
z)GM4MDs)w4C-fTe6$z@9ncK4*i(2JpX2uHqL~l;!{k%-(Z&xGF^L_F}FG?eC%a^=m
z$d-f+S1H|ki^2qhIL-NEyy=Z9v6|;OZ!IZXRw!1Vhh)wn7MdTmwu7IA#P;ySsLMj}
zhHi~=kNL7R8~VPeV$P?ImRmw{E0J-S=EMG|d%XYc6S9m?-)=oGHDQIPx>%KjU#GM6
zJgGuQx<-=i%Nk3eD=V#o7RdXsbz(vt1=17qLjM5di`LvVtvb`APRnn_f%CavZ<H$n
z!o;?@tiEsc3*Pnb94!ihF@**S&lf9RALs&$An)3amghw2Xg6!TUbO`~z57bn0KOk?
zPn(R<?9eCkEx9{HH|@Q<KFx9|&N}F|?}wR*Eqx^TroUtdqi{NfxFiGvGxp6oH6A+t
zcq)QF^J?@XcrIv>(ZS#8+#XR2W>k1)R{pjQp%bZi6n&7b^zoPA{PQr6_meR1^wRx>
zjf3m|8SZ~YJT)ZvL*gL(38IOKjf0JwiOFxlzo%lz%Jk<s<maFiKLP#COh+e67c)0U
zmwy@j1Dou>!KVBh?Ekmn%zp#^yW#%@&96E3htsVf9zwSf0DvkQ0D$8k(YXJZgx^^I
zb1r`3FA#x|;{wl_-)orc9NnT%bd3NQL@1GqRGP4rvAhN@6(q?l>hiK0?TLu^4EvR*
zOf0ERLG3cbegq9=9LL#pv*lFiw@iXpaqG7zVi8t{1sW=E#ImP-B~mL~f@%atJHW-e
zE9~X*A+#Z4Xs4GNpUC6FYuVp3H;xo0-50k_7wt8+z-YgI*H^xOH80j?lNPA+ZfAKi
z(K_@U$<deB(&P|EZI1?Z?{1807sv0*!D(s5jdL$A(B5Y=1nhinim-EL-BaK^f9PtC
z>9?w(w`HNeThI)SW2_5NWZ@x%9UVpDcS1JS&D04<-LAsA6tS!(zLPLy<G-LE8qBu6
zRt3EyxlykTyH_T<+(&C^i#s_EYlgc|ti*k7PC74b2=w;dfcNc|<6*9OmO^g!U6FZ|
zg`4y!?+dD!t|LqGD!S>A3T_r@S_^|2fu50K_RZi}DUTOR<Hg7_qH*Mss19E5$w+h|
z5ug#y4M>NpGzZe+Ot+G}_@+g!-JW3o1y?O9c&tDQwL7VU4_c%rsn}|{2mKbv>E9)S
z(n9(y#ZvG65FS*Lr>Q*uN$5dd#8KTy_m!<g!}eu(z(`BA8t=4wX^(>U?(B&EI5&RW
zphGuMhVANeRD2}_^Q<k*&in-A6z0bU!<?b5e4T)iOI5{dx!JY1wXtS6{j>qT2m4<A
zT4xq$Rmo`LF!?6UWBS;PbYODS=PT5te5Oa`TdRIrNElBAS**L*7(u6TO0QZrNhEXX
zUEOG$18NGW0m_t>`^$QBCTVx?_qJ~13rIso`M?-GeRbJpI2S{S5fLS%K+&Vu8d+j=
z)i7mOr8Z&M;!E#8JC8Y;xI1&akPzhs*PA}3A*$<<pN}3o8pt34jm<gnKvl(NIrwOJ
zSrpT+3`(<<vS|`E`7DrvMm<;J+?f$)m-bRmea0Z#%3s)Qq1Qrbcx08KP4x~0yY(rp
zSkz(3Z5$<6@nxZ{oQMU4n$Jwyu{gpoAr~KyWs`dYTKgwU<4gBLpe5t0y<DkKax;<B
zQY2L?{67dQao20Zh)xam6pmdWaKO1QUFv>7?nPT^ig#0fiN?%0Ms&de$NqJ-{)~z?
z%37=j2g|hseJ63u5LdnBL6Yuc=2{20sPm%PW;Pz$NqQNb!Oiov#i%dcN=?WK+%vFd
z!wrm%um+e^WwY%SOlKdq5)&|WRs<M4>Lxt=;yvEi>+5UW@K1(5G;GPBgQa8{n>g;7
z%AT8tvxe-zUa2_yg1_0nj;7if&)<qHyoxS&@Gjz%E{;dML%Q3SAbx8b5L*!ST@|Ii
z=JZN+xZj`<^`P6OI5io#;F5#eHOtF%%Y_pX%aLk_+smp6fq4?GC&`G^K-9Vm^ZxDd
zFcs|S^DYe<2~$wooW{4r(k!AJzTmlvnDQgGN7G@12g#RKClpD$%S_oPNt{%;)7^Z;
zD3B`9Euzx6C!ww@=qG7~#{suf>#1)LtmW@@rPVm#GG87AmwIbii!QlYLwc*d>Q3Ox
z^g?A0_*8*0i_E-1ysX#MG#^)-!+4DGLUK^3eA(A;5*$h8&VU~gK?l`Ldd-HECS=k;
zH}B;<ZtgK|H1!_q5KDI)p%2yrJ1E^~6S~&XocbW)Ad|w{(cu~WP|)Fr<`ko6ITuQo
zZ<8e3Qa@b}uhiOMJQ8S5s$aj0F%zQZ4^N9Qoz*Elw4QD)^PLy60-DR3_p@12#&sI(
zOv%cIQ4>mjnv6KNQTKT?xDE5+O80#uYJ!Ly!#CgcnL@r;fz3TqkK2Tlb_Q3D<HeyR
zMlz-+8TQaVuK8=$KFqUf23TGhmI+35SG}2{7%SPs<}&3}a0w_9LGuzmNZ0AZfi3Fg
z($yx9CHUB+4P0(SpUjGD;jZ6_S;M$JQdu1xOPgPEoP6+1?KBm;zGWK*%l`mJ2o)tv
z|1PtCeYL5qwNKh6PalE9p=-&y2$>^i6JSMQS@Bx&o*sFmo?rX+)j9Ni(gp80pLe&B
zw~_!mrmkbvv838Nuz!{`HTUWQ%(-kL^anN>-vvb3@>!gTQ}2o4()<vp(voZh_h`l!
z<iwhzhDXmDp+99tqsA2L$?eV<Nq-Y$IrI-F#JaUmvam=pbG1gzvnsWV%BP!uRpoN<
z7Lws%u=4(WRi@mB4w6JjWgD2w$EDyD${`e<6w?%qL*j<T$oPTT7p5NQFXe)juhg_4
ztzkaZiwVqJrS12gPgXfIz#O!<5sBU@q@M72Sn)q*lCsF^&<~Nfbs||cu<g|!D?Q|t
zLdL1TRWtx_mJ<1>+?(i*KTyq80ovr}YFEEUk*H!<(l6EJyE;{&t!HXnGbQu)7#L*i
zaWbnKYMFN0Hn?0LB_~R_T~mrQzV}unj^my5Yk-z-8gW64y3`IU5$Nj+P++c=YRwIW
zjx3n(D6ljCSXWdo)-ex~v1uJkLGgCiRw~*j<{MR%U*hM}`LLi_Lmlfta#GNl6tzk3
z^KsXD_+^=Z@-)I%a9s74+Ba2t(n8QCQ)O`wP!T~ov0!?n=+#&8v#rns2IE>oH={_b
zw)`_=yLJ`1v`yb{hX_1(q@bHp0-T(&Wg{WN!ssF`CsAq?;0isNy=hxbu8tS&!^gn;
z1855ie{GD3S6)_#9lfLZr|MyuAHM~7ig~+PA4@HGDHmTGcFpZO>^?&1J3I~<mYIUA
zOzy_9;c1xsDx7SNM!ZtK4rAFCZF%l(Iepep7+E+@1}CP}x<kcv(j*mn?lrC9+&8Q1
z!$j5=bo6%<D&6Wx=_!E|t6gKbTtya(zU(*MUSv#`HU*F%Q!%VPLq~yey$;80Qd7{&
zn+rd%)SmJN%dQrUNrp7yE)!LhQGgZI8b3SKt9&ind!urkY$%gro*S+Xc>1&6-%c^|
zA%4Cc$5cLQJV;Q7)mXM3i%<jwCfVx^hDHM&mKT~qOvOEy@lCzrhxYO7`;__t>*C$j
znJOETn>}A|zbYC`E!N{U4L-kG#i=lDKbahuMpyLY>@i>sI>a|gXG-WI(!cU!@n4Uh
zsgz5NO@p`Kiv;z7qS(J}Vu%vO^4nghhXoQs?{SEN+OB06z$OJEMK)WOM%d#{;ySx_
z{%q%Z0QQHGX}!jmqy!iKE4Y&`00A6zp5ZmqgXhMiaOAFj5g;<jqVO5_mMA0j;yI?$
zv9t)w;&lR<^~%?$8t<w0k?Z8OZmoKiRf+~&sZK)DDg;rLNrsf7^-3%3UIt;dhKyj7
z%Im><@s}mV8yLrTZ+h+IeGV0P9nP_=-A?~f$aJ8tn>}!oj(;Mp=dEaTMmUS^n~y?T
zd%E8-@Wa({AC4)R=5=0UK$9sl7Wst>e(i_6(E;v%IbPYrR&Mh$DM!fKz<hIFHn6_c
z_WQQYkCIYF4p!6?763581^@*AQAz1y`O?hI#?ism+Q!NCr?n;dUu*G1T?0oj50?M8
z3bNWT`HUP77#?I;j#>q<8C@*S;FSe5B&8ba3%v<hh6uaswqqh=D)soU!*r@?+nuhW
zq#6+(Ds;G7iy~HcbrztZqe2V&UTCc{GT&v8M4ZX$L5;A1)@RiUzP|UR*4%0W2=nfj
z<g9v9clvUS=OA3Bgm&qArsfi`V6Wd4Pvlu>L`iCmblE|b>Enf>z-gN$m@lw;*^`6X
zT|r^DVc(;9bJzn^Z8C;w9IzMZxSlFpc{`!z-}Z3LuUDomrIb(gx-ykviRUcSbD|Od
zPK0)UDzbh1u$x&Zn3QVNyGtvxH-1V>-U9qKs4PO00;Wl`WgK@l;Zx4ywxCC)sEK+N
zjEV8*1>v>&H-`5T<>1gV(5w4Cl7sMsmkm}794?*TQyMc3^#vtzUZ6U8wMp>2pgd`)
zAP?S2OHUG$)}W;-4Lu{8+7eCDWSKa|XHL;Az%EzLoEhqdV$~f<t+w;d{g8r>{pGvf
z5;^D~0=xA&PmMLeN4=cnWZgEBSBp?aoEcq5bVFJ#TlYsz8I=<`K%A$^Fu&ugcbJ9?
zvHInZ>_lvz1RtuYUu$N@{pE;)H~z+0GrMAjc9MNcBqiTnYgGJs#D&*Q6#Gz7uPp+v
zrm|EI*Z#-$3~;XQAni7%XbKL#a+t4apO0FvM#;#W;ON7`HU7-dT}}Q5qk+|#n^ud}
zJj#h&XB32|t+h+S8#2`^Iuc8m55$bF)|<v-^BzYz(rJ#APWIQLRg&-Hhs$*6u{G_-
zuim-f0GEK%7e1xdI0l&U&a%6q*^!;Cq1G&&lx<t5=fpSfikuE$-O#zzRvKi&anqGS
z2sQgliJQ-gY|9c^^Qs7@)vD6?Ya$@4cC>xA!NQHZ6{w!-PK9O1myIX}wiF7VX+^0X
zU1K#XAvV6hjE9FMx0ZGLiVKs#L9m@<!fG8M+T`Z<-p2;tFc0R!tUL`-QuS*_ivK#F
ztoA5w8Dpy&NC%0YKNI8pt_w3<&gtx}Sa&vGg2)Z!PD6F^GU0IO!5uMIuk{TZ>YDfa
zOvhSww-;+GeC$B;8%Ae%ABY}cGB&YF#YG!kaP+(ewy0o$I))x6#bc_RXCLZb(5^Zc
zK&}^wMKK~HJLw@Rqc%8URKa*9c7lw-<5S9#V>WaJY<B4)`uzKK$64{>B;ZK=>KkC~
zT=n(v{tUhM`E75%SF+B{y9g-u08I7>F6?xve#OZG)=E)R$m@ZT<-Run-FhcWn41^W
z-Wz+F)}qn0$RbiLyVMT-n_RUt{B{?oT&lBA`<PN9JC7&+x~Z5ov@|iL_G%YFLQBCN
zvvUE9bq+#yQ3r_bhlC^X7Da`<I4L6`RV6(2>-3OUEnDi14un`1Ua6EuJt&KkdXx)s
z{L&W_9%8zWut9ku#!57Uy)9^|r}J#jZX3n`rVePR!ZGAM$N2g<Tc2K%5|;W`;&xWU
z31_PEsGhQ+QXiv6_Ait-M-T{y&}CKGfcLVjPTmAK#3;?Ydv!rrY~egvvbVNPC%5Hl
z$CGuSej283Tab3;POQ8eE#dbuLsne6<wan{2{Kj9;BYqrK5N+BkjUrBQSRpZbaTh5
zcV1O5_C7NQyKg#?3E2Z2+M}+7oDZ?y@KhVDt&K|d=5R-xr1T)!KQj9ANZa&1mWL_8
zJQnkdnKtw@kr2ctM4b4X?5rVS@yVRQv~Q(Ay1iO~ce=D`O7Rn0$M8m-hGr&^ph_1p
z?l6P8(FM|;8`{sRVT2;&4U5Ud46f-YE!t#-GxEhwq98-%AAP$nCgRr9Q)-*o=r;@#
z4|JYAEieRfO{$0);^fLB_`{9X)Kni6$#5O^u=_VP;Bs9H;r4NUsUj9OT@BuG=J`hH
zhQQF}%9g>|af<`DQtZ}m(OYEl=~4}IER$@Y$iIN*6Wy41>NOt#<~Rexprp&IJ+eZN
z^N5cS5b=5|AWDWU_A_Wuw3Cbd&Kfx_aB6UHfpKM+iMakemBd9<O8-%76V|vKw$t%V
zH|Ys6A<UyKcW31A%GKfF0YP)~YU>4r&UrO43lnwoe!a8XB8HcEOujzE)l5YZ^J`Ow
zVpa&0(fBX+Y(AkYnpJW<x!w7%YF#1QxN6WS$cUO~pa83r(9%QV)-&>Oj!C4=9Xw9*
zZ4r8Y7rVK`1zMclHyUS5u>_?&ub5)v6X;6YLT+)YtKf4UVg8uw2n=1K)1Pu3H4Xs4
z{*Q8<xy#Qp+)s8-%J9GrV4jOb6Y)Z{L;;z7L}WZjuh|cEV(eMWsfYA*(9$`?EF48~
zni#x&|4MKfB17=Nh;4t-N$q~&Y(skkZ<8a-E`1|ojbV#kEN*r5&&PQj8)$rY8>8Cb
z<U(E|YXWTrq{+Y_4<nyKg1US^yV_4T(~R1-V!49a#`R|crY5pxbhwH}wdn{oXT6c%
z!JH@?58p|drM${5J>f^o77iO`%d{UL>{qzi7SD{9ss*PVAUC>tzkQ6Hvel~Bv~$UJ
zV`Gf5XByFAglt=;v0A}zi7j3WN-G}VwyU?xv8=H5P_9+83$W{g;Wk8keWp6-=RSz~
zaEYxYi1KK>xU{YzKH^VJns2)OLHNN9?bU`9e|SoKR5Dqc+gg9mejFsFJAm?)!gx@0
z049kB^wir(UVyX~Em@SmEArmDkxen~DsL2zB6HVXEbiMek_;$1Pcc+jEk&-3jm~uK
z;37fe8vHY$!BzX-5*w-hWPosTEbtnssQZid*4&pQ1&oPMLxN&tyMgMV$hdY@Kcu+H
zP((tS(eph^NUs}rvW8sym~e!Jkiva1kLUKgC93SVR0ZfY2Lr^M-mou7Q%m_s+TfE$
zfnQan3rU>hF5g>3mHWXQ43KVq_{>LgBBw!D6{ZO#F;&9z(N@dXGw9g>X_tWQccI{|
z@?^6w2vwWRH6H@>2%3;|Cnda_`KUgh$#^R=X@TJ45ecmQkow*(q;ni&FV?7lSziuf
z1rrTSZ|vC(AXQi4(!kRi0%>lAw>Sq|FP<ML*2xxTlTAZp5D7)1bT1>pn}Vweti-_>
zui*5@_dlRB5K!brD31}Wl~%bzqEHMQ(>`TAyL$pX&0W0{!dSvSG*i=<^GP#TW=A~u
zErSWO`|Rm}<rTJ0?p>^}EBgCP45-kIBnw!zOql-0*ERFr1zJ>CL5KNUJENDL$wKTj
zhl(F%rY%f`orUM2uhJ8u?oE5`<}U7(w`GRutR8&60U~A^$mS;j%MA-&MeAeF7n+Ti
zfeztuRHYCIyt$}0<*TFY%>%JIS0D>*V-fdfiX9}j)A(n!f-M^I#oM|P*6I#L5#@<)
zWqI!^!a<A#PID@>lqdDn1bercX@vXe`vuHH9)3_=6fEhOHJ0rvVQ-%o=mc3L`5AGR
zake#xEud70&%S#ZI)0^S8Regve>BCHj-w$MWG?UAbyX*g0bxUwwFVsKjx}H*uvX@a
zK&qYtatWvPGH2@*&$OE>3-Fe_*+k_;;K$+g)eV{BNNR-oz<<*xWb$Phz521pC<=D<
zwq+q72x&mMe?o4u$M2jMQud8sdrtr?fPRV<k5-w*(gG>1qy65gzugEg;W>IGQ7-OA
zt2+^g8(4E^&DpcsJzQG0rY~Q^3!P(y4jAFezk>rHJrzo$RjK4R3&_!k9&GXv&2rJ!
zmvH51>vGf^jLk=Avs{d^fO{F~<K62bp44BCom>C)UbNHBuwh*o&d0vR4!j6^muRBG
zV`iSOWcf|}LS4aeu}Q?CdX(9~%$`6`=Iv0bkO=nCA}a3!qNv~}XD18jP%3lW%u9pb
zxxn>@bSC%QtuC6h(dL_7n5Q_5E()#a)}2gWvx-Q_RhTF&^4<Ee8bjSfMaka<Y?Lrm
z68GgfMvtm(23G1E^SIG!S)15O>h<MoOcOL34X!aAlnUq8y0-|?>KSmVHjb7hyU+03
zCzy%41bJqRgH;F(!XhJy2bWv?TWK7PA6rzFcga4xjgTER!yD&=2<tZ`PVew&r1SRs
z*Q^mR@XMJLDC%ROOPvkgfYwhDD8)MXsUnx+`}Z}|)(R(LRG_+9)s7ED-=igcj+n<x
zl+o0iX{Gqa+dv_?f<hL86uNjKS(n)xSyO3HkRm+YU&|kr53}Fv6uF#Oj1y`>*BB@s
z@u?xwaG%J>B>%Ljobh6(CrMl%`UtT#TC@f$VL#z6idwp==<R{CJ!_?_U*)+JJikcN
zdHF)0Pv9KdZO~KLp`~Deqa)d(-@NQ#GrB<nzj<db<-|Ss31pW+5GtklijwoLwZhEF
zod;ZFQk|3RS*cfYhUB)8IbA-g7d-mae^E{LEVq>&LE%D<(aLM1Iml!qqXy{A*=GQ*
zU3gYc_Q|T!!CU-oo3$z^xP-Izpf6ojHzI*klxyTcz>O&wjO+hm1H`>1qe*7)>ALM3
zndYe@4ir4BoCE4Ow4q}7a**DOkK|MQL8BsEuCUfO0dc<Muk^pe{hkJYX#RnME;_SM
zx;-==0D%9KG|2Ky!$jjrU9tQ-@&7Pc39*w67_vMW1jDP>G=$wE4ipZ=1u}=LmOf4J
znIVaiLZ;{#&Ovd`aqy~v8;uQbiXz19eflT>j!cq^#hTijOV;|Wvx#{Ol=6*@!OMXw
z5WytKoJegYYSG6}Z#GN;p8O3k2kvF%oW<-6P{Nn|@@8dMOlQ1j8zJNK6yN;U!1A1T
z-Ut~J_=naia7~e0*_K2iQG$fDSgy67Br8;K)*SJq8k-(j$N3+T$o=sYa7(@rjRt0k
zFWP+<o*rrxK@?<xl)E?M`@%T-1IryB_#EEhkFLrTe<cO}m#fHs;;P>SsekA)e$*$I
zNj<*766a!}_k(dTm0E>13;;<YKHt@FW{H-jc#T6(7Qz7kkRV;r7nQ+9q5{zM*H`()
zV<TR2J=Sq!IJw5=ft2zG&bp+=6;nAy#M9h@FZY^0-A;MveK6Q>hyqH$lM8firM|oM
zLm}_Rok&N^M0LZq_styJ)p(aHOuWM|pPV#CpD`YN<(5@qO`_kk+SO3wEWs}0Bj9_{
zFL93&OgD4!NWV1lC=pKDGx1z|qX7+?In{W7Kbdpeh0)g)U3Lwvi=D*#2X5J50MO6r
zsm_9cB!T+5-UtLdt+QVaAmCB_$IV}xOkjR(GWk<y$gimXTZ_rBs87u(fV@EYr|X{@
zPM#otG@Set^5=%){lU=fr(Tvn8~WRHYia(S!9UkC3IB}_0MrHYKb4_>M*o8(_(bqS
z*8Cgu|E$7N{F~s1pQC?5@SFL+>487w)Bm1_^4~l^2D0}zJpa!DG5>)`2JWvO`|E)I
zG3Eazs{X#J>H^FE5#?Vc*1z-V@4SE9dHYXZ2#kL+)W6w>`)3CH@pb4w4LE#y`19)g
ztFQm^a`b=G{x$vlO8X-!<ON=k{+0GWBI^_UPrJ~5gq~mB`UC!tA^Sh!Ny+~T|J$hk
X0S(O4N(TUtpFV5Q0Du<7kEj0uoaaXu

diff --git a/dags/test_fetch_remote.py b/dags/test_fetch_remote.py
deleted file mode 100644
index 84054ecb..00000000
--- a/dags/test_fetch_remote.py
+++ /dev/null
@@ -1,39 +0,0 @@
-"""
-Description:
-    This DAG tests the Airflow's ability to communicate with remote servers.
-"""
-
-# import modules and functions
-from datetime import datetime, timedelta
-
-from airflow import DAG
-from airflow.operators.bash_operator import BashOperator
-
-# default DAG arguments
-default_args = {
-    "owner": "OSDU",
-    "retries": 1,
-    "retry_delay": timedelta(minutes=5)
-}
-
-# set DAG
-dag = DAG(
-    "test_fetch_remote",
-    default_args = default_args,
-    description = "A simple DAG to test remote data fetching",
-    schedule_interval = "@daily",
-    start_date = datetime(2024, 1, 1),
-    catchup = False
-)
-
-
-
-# set tasks
-fetch_task = BashOperator(
-    task_id = "fetch_task",
-    bash_command = "curl --location 'https://postman-echo.com/get?foo1=bar1&foo2=bar2' ", # list services, processes and python packages
-    dag = dag
-)
-
-# run tasks
-fetch_task
\ No newline at end of file
diff --git a/dags/test_pip_packages.py b/dags/test_pip_packages.py
deleted file mode 100644
index e3e20b47..00000000
--- a/dags/test_pip_packages.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from airflow import DAG
-from airflow.operators.python_operator import PythonOperator
-from datetime import datetime
-import subprocess
-
-# Define a function to get the list of installed pip packages
-def list_installed_packages():
-    # Run the pip freeze command to get installed packages
-    installed_packages = subprocess.check_output(['pip', 'freeze']).decode('utf-8')
-    
-    # Log the output to Airflow logs
-    print("Installed pip packages:")
-    print(installed_packages)
-
-# Define the DAG
-default_args = {
-    'owner': 'airflow',
-    'start_date': datetime(2023, 1, 1),
-    'retries': 1
-}
-
-with DAG(
-    dag_id='list_pip_packages',
-    default_args=default_args,
-    schedule_interval=None,
-    catchup=False
-) as dag:
-
-    # Define the task using PythonOperator
-    check_pip_packages = PythonOperator(
-        task_id='check_pip_packages',
-        python_callable=list_installed_packages
-    )
-
-    # Set task dependencies (if needed)
-    check_pip_packages