add support for debug mode

src/keboola/docker/runtime.clj

@@ -22,7 +22,13 @@
 
 
 (defn log-strings [& strings]
-  (apply println strings))
+  (apply println (map
+                  #(if (map? %)
+                     ; pretty print maps
+                     (generate-string % {:pretty true})
+                     ; else just print
+                     %)
+                  strings)))
 
 (defn log [what]
   (println what))

src/keboola/facebook/api/exponential_backoff.clj

@@ -1,9 +1,9 @@
 (ns keboola.facebook.api.exponential-backoff
   (:require [keboola.docker.runtime :as runtime]))
 
-(def time-slot-ms 100)
-(def truncate 5)
-(def MAX_WAIT_TIME (* 1000 60 60 10))
+(def time-slot-ms 1000)
+(def truncate 6) ; sleep for 64 seconds at most (2^6)
+(def MAX_WAIT_TIME (* 1000 60 60 24)) ; poll for 24 hours at most
 
 (defn with-exp-backoff [action!]
   (loop [c 0

src/keboola/facebook/api/request.clj

@@ -218,6 +218,7 @@
   (let [status (-> poll-response :body :async_status)
         completed? (= status "Job Completed")
         failed? (some (partial = status)  ["Job Failed" "Job Skipped"])]
+    (log-strings "Facebook API async insights job state:" (:body poll-response))
     (cond
       (not status) (runtime/app-error (str "Polling failed with unknown status" (:body poll-response)))
       failed?  (runtime/user-error (str "Polling failed with status:" status (:body poll-response)))

src/keboola/facebook/extractor/core.clj

@@ -11,6 +11,7 @@
             [keboola.facebook.extractor.query :as query]
             [keboola.facebook.extractor.sync-actions :as sync-actions]
             [keboola.http.client :refer [fb-requests-count]]
+            [keboola.http.recording :refer [turn-log-responses-on]]
             [keboola.utils.json-to-csv :as csv]
             [slingshot.slingshot :refer [throw+ try+]]))
 
@@ -64,6 +65,10 @@
         app-access-token (docker-config/app-access-token datadir)
         out-dir-path (docker-config/out-dir-path datadir)
         credentials (docker-config/user-credentials datadir)]
+    (when (:debug-mode parameters)
+      (log "Running in debug mode")
+      (sync-actions/disable-log-token)
+      (turn-log-responses-on))
     (cond
       (empty? credentials) (docker-runtime/user-error "Missing facebook credentials")
       (empty? (docker-config/get-fb-token credentials)) (docker-runtime/user-error "Missing facebook token"))

src/keboola/http/recording.clj

@@ -4,12 +4,16 @@
 
 (def recording (atom '()))
 (def do-recording? (atom false))
+(def do-log-responses? (atom false))
+
 
 (defn reset-recording [] (reset! recording '()))
 
 (defn turn-recording-on [] (reset! do-recording? true))
 (defn turn-recording-off [] (reset! do-recording? false))
 
+(defn turn-log-responses-on [] (reset! do-log-responses? true))
+
 (def VALID-CHARS
   (map char (concat (range 48 58) ; 0-9
                     (range 66 91) ; A-Z
@@ -36,8 +40,27 @@
 (defn anonymize-map [m]
   (postwalk anonymize-item m))
 
+
+(defn replace-token-by-regexp [item]
+  (cond
+    (string? item)
+    (clojure.string/replace item #"access_token=[^&]*" "access_token=TOKEN")
+
+    (map? item)
+    (if (contains? item :access_token)
+      (assoc item :access_token "TOKEN")
+      item)
+
+    :else item))
+
 (defn record-request [response method url request-rest]
-  #_(println (pr-str request-rest))
+  (when @do-log-responses?
+    (let [result-map  {:method method
+                       :url url
+                       :response {:status (:status response) :body (:body response)}}
+          result (postwalk replace-token-by-regexp result-map)]
+      (println (generate-string result {:pretty true})))
+    (Thread/sleep 350))
   (if @do-recording?
     (let [request-base {:method method :address url}
           request (merge request-base (apply hash-map request-rest))