From a0873dd81305ec4b2541d1f24e8553e0aa10408f Mon Sep 17 00:00:00 2001 From: kacurez Date: Fri, 6 Sep 2024 17:30:57 +0200 Subject: [PATCH 1/5] add support for debug mode --- src/keboola/facebook/extractor/core.clj | 5 +++++ src/keboola/http/recording.clj | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/keboola/facebook/extractor/core.clj b/src/keboola/facebook/extractor/core.clj index 875d4f0..7ad99b8 100644 --- a/src/keboola/facebook/extractor/core.clj +++ b/src/keboola/facebook/extractor/core.clj @@ -11,6 +11,7 @@ [keboola.facebook.extractor.query :as query] [keboola.facebook.extractor.sync-actions :as sync-actions] [keboola.http.client :refer [fb-requests-count]] + [keboola.http.recording :refer [turn-log-responses-on]] [keboola.utils.json-to-csv :as csv] [slingshot.slingshot :refer [throw+ try+]])) @@ -64,6 +65,10 @@ app-access-token (docker-config/app-access-token datadir) out-dir-path (docker-config/out-dir-path datadir) credentials (docker-config/user-credentials datadir)] + (when (:debug-mode parameters) + (log "Running in debug mode") + (sync-actions/disable-log-token) + (turn-log-responses-on)) (cond (empty? credentials) (docker-runtime/user-error "Missing facebook credentials") (empty? (docker-config/get-fb-token credentials)) (docker-runtime/user-error "Missing facebook token")) diff --git a/src/keboola/http/recording.clj b/src/keboola/http/recording.clj index 72cad63..e554edf 100644 --- a/src/keboola/http/recording.clj +++ b/src/keboola/http/recording.clj @@ -4,12 +4,16 @@ (def recording (atom '())) (def do-recording? (atom false)) +(def do-log-responses? (atom false)) + (defn reset-recording [] (reset! recording '())) (defn turn-recording-on [] (reset! do-recording? true)) (defn turn-recording-off [] (reset! do-recording? false)) +(defn turn-log-responses-on [] (reset! do-log-responses? true)) + (def VALID-CHARS (map char (concat (range 48 58) ; 0-9 (range 66 91) ; A-Z @@ -36,8 +40,20 @@ (defn anonymize-map [m] (postwalk anonymize-item m)) + +(defn replace-token-by-regexp [item] + (if (string? item) + (clojure.string/replace item #"access_token=[^&]*" "access_token=TOKEN") + item)) + (defn record-request [response method url request-rest] - #_(println (pr-str request-rest)) + (when @do-log-responses? + (let [result-map {:method method + :url url + :response {:status (:status response) :body (:body response)}} + result (postwalk replace-token-by-regexp result-map)] + (println (generate-string result {:pretty true}))) + (Thread/sleep 350)) (if @do-recording? (let [request-base {:method method :address url} request (merge request-base (apply hash-map request-rest)) From 613d8794f7db28b2157bf1877541ca9ce8e84bc7 Mon Sep 17 00:00:00 2001 From: kacurez Date: Fri, 6 Sep 2024 17:49:54 +0200 Subject: [PATCH 2/5] replace also access_token key in the request body --- src/keboola/http/recording.clj | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/keboola/http/recording.clj b/src/keboola/http/recording.clj index e554edf..5a58a2a 100644 --- a/src/keboola/http/recording.clj +++ b/src/keboola/http/recording.clj @@ -42,9 +42,16 @@ (defn replace-token-by-regexp [item] - (if (string? item) + (cond + (string? item) (clojure.string/replace item #"access_token=[^&]*" "access_token=TOKEN") - item)) + + (map? item) + (if (contains? item :access_token) + (assoc item :access_token "TOKEN") + item) + + :else item)) (defn record-request [response method url request-rest] (when @do-log-responses? From cd45fea7e5262ab509dfe6e7960afd61781d34f1 Mon Sep 17 00:00:00 2001 From: kacurez Date: Mon, 9 Sep 2024 15:00:09 +0200 Subject: [PATCH 3/5] improve async polling --- src/keboola/facebook/api/exponential_backoff.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/keboola/facebook/api/exponential_backoff.clj b/src/keboola/facebook/api/exponential_backoff.clj index 46db157..1c5ad3b 100644 --- a/src/keboola/facebook/api/exponential_backoff.clj +++ b/src/keboola/facebook/api/exponential_backoff.clj @@ -1,9 +1,9 @@ (ns keboola.facebook.api.exponential-backoff (:require [keboola.docker.runtime :as runtime])) -(def time-slot-ms 100) -(def truncate 5) -(def MAX_WAIT_TIME (* 1000 60 60 10)) +(def time-slot-ms 1000) +(def truncate 6) ; sleep for 64 seconds at most (2^6) +(def MAX_WAIT_TIME (* 1000 60 60 24)) ; poll for 24 hours at most (defn with-exp-backoff [action!] (loop [c 0 From 9bdb410cad0f6149cf7ea0978c6346017e555b8d Mon Sep 17 00:00:00 2001 From: kacurez Date: Mon, 9 Sep 2024 16:10:27 +0200 Subject: [PATCH 4/5] log poll response --- src/keboola/facebook/api/request.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/keboola/facebook/api/request.clj b/src/keboola/facebook/api/request.clj index 62eaafa..504a6df 100644 --- a/src/keboola/facebook/api/request.clj +++ b/src/keboola/facebook/api/request.clj @@ -2,7 +2,7 @@ (:require [clojure.spec.alpha :as s] [keboola.facebook.api.exponential-backoff :refer [with-exp-backoff try-3-times]] [keboola.facebook.api.parser :as parser] - [keboola.docker.runtime :refer [log-strings app-error log-error]] + [keboola.docker.runtime :refer [log-strings app-error log-error log]] [slingshot.slingshot :refer [try+ throw+]] [keboola.http.client :as client] [clojure.string :as string] @@ -218,6 +218,7 @@ (let [status (-> poll-response :body :async_status) completed? (= status "Job Completed") failed? (some (partial = status) ["Job Failed" "Job Skipped"])] + (log (:body poll-response)) (cond (not status) (runtime/app-error (str "Polling failed with unknown status" (:body poll-response))) failed? (runtime/user-error (str "Polling failed with status:" status (:body poll-response))) From 7e5d7001ce8decd70cfc4443ff474b39ca1b85b2 Mon Sep 17 00:00:00 2001 From: kacurez Date: Tue, 10 Sep 2024 09:11:40 +0200 Subject: [PATCH 5/5] improve logging --- src/keboola/docker/runtime.clj | 8 +++++++- src/keboola/facebook/api/request.clj | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/keboola/docker/runtime.clj b/src/keboola/docker/runtime.clj index 58fe602..b8dc624 100644 --- a/src/keboola/docker/runtime.clj +++ b/src/keboola/docker/runtime.clj @@ -22,7 +22,13 @@ (defn log-strings [& strings] - (apply println strings)) + (apply println (map + #(if (map? %) + ; pretty print maps + (generate-string % {:pretty true}) + ; else just print + %) + strings))) (defn log [what] (println what)) diff --git a/src/keboola/facebook/api/request.clj b/src/keboola/facebook/api/request.clj index 504a6df..cc52bf2 100644 --- a/src/keboola/facebook/api/request.clj +++ b/src/keboola/facebook/api/request.clj @@ -2,7 +2,7 @@ (:require [clojure.spec.alpha :as s] [keboola.facebook.api.exponential-backoff :refer [with-exp-backoff try-3-times]] [keboola.facebook.api.parser :as parser] - [keboola.docker.runtime :refer [log-strings app-error log-error log]] + [keboola.docker.runtime :refer [log-strings app-error log-error]] [slingshot.slingshot :refer [try+ throw+]] [keboola.http.client :as client] [clojure.string :as string] @@ -218,7 +218,7 @@ (let [status (-> poll-response :body :async_status) completed? (= status "Job Completed") failed? (some (partial = status) ["Job Failed" "Job Skipped"])] - (log (:body poll-response)) + (log-strings "Facebook API async insights job state:" (:body poll-response)) (cond (not status) (runtime/app-error (str "Polling failed with unknown status" (:body poll-response))) failed? (runtime/user-error (str "Polling failed with status:" status (:body poll-response)))