Merge pull request #22 from keboola/deps-update

development update and refinements

.gitignore

@@ -4,3 +4,5 @@
 /.nrepl-history
 /.nrepl-port
 /src/keboola/facebook/api/sandbox.clj
+/.fbtokens-env
+*tern-port

Dockerfile

@@ -1,23 +1,11 @@
-FROM openjdk:8
+FROM clojure:boot-2.7.2-alpine
 MAINTAINER <tomas.kacur@keboola.com>
 
-# download&install boot
-RUN wget -O /usr/bin/boot https://github.com/boot-clj/boot-bin/releases/download/latest/boot.sh \
-&& chmod +x /usr/bin/boot
+# ENV BOOT_JVM_OPTIONS=-Xmx256m
+ENV BOOT_CLOJURE_VERSION=1.9.0-beta1
 
-# Setup ENV
-ENV JAVA_HOME /opt/java
-ENV BOOT_HOME /.boot
-ENV BOOT_AS_ROOT yes
-ENV BOOT_LOCAL_REPO /m2
-ENV BOOT_JVM_OPTIONS=-Xmx256m
-ENV BOOT_CLOJURE_VERSION=1.9.0-alpha14
-ENV BOOT_VERSION=2.7.2
-
-# download & install deps, cache REPL and web deps
-RUN boot repl -e '(System/exit 0)' && rm -rf target
 ADD . /code
 WORKDIR /code
-RUN boot build-insights
+RUN boot build
 EXPOSE 1111
-ENTRYPOINT ["java", "-jar", "-Xmx1g","targetinsights/exinsights-0.0.1.jar", "-d" , "/data/"]
+ENTRYPOINT ["java", "-jar", "-Xmx1g","target/ex-fb-graph-api-1.0.jar", "-d" , "/data/"]

Makefile

@@ -2,6 +2,9 @@
 # Makefile commands to manage and simplify development
 
 DATADIR = ${PWD}/tmp/
+FILTER =".*"
+-include .fbtokens-env
+export
 
 #starts clojure repl in a docker container
 repl:
@@ -11,13 +14,13 @@ repl:
 bash:
 	docker-compose run --rm --entrypoint /bin/bash app
 
-# builds java app jar file - result is targetinsights/project.jar file
+# builds java app jar file - result is targe/project.jar file
 build-jar:
-	boot build-insights
+	boot build
 
 # build docker image from dockerfile
 build-image:
-	docker build -t keboola/ex-facebook-insights .
+	docker build -t keboola/ex-facebook-graph-api .
 
 # runs extractor docker container - docker-runner command
 run-docker:
@@ -26,15 +29,17 @@ run-docker:
 # runs extractor a compiled jar file as java app. Built via boot build
 # command or make build-jar command.
 run-jar:
-	java -Xmx1g -jar targetinsights/exinsights-0.0.1.jar -d $(DATADIR)
+	java -Xmx1g -jar target/ex-fb-graph-api-1.0.jar -d $(DATADIR)
 
 # runs extractor directly from boot ie runs as clojure program
 run-boot:
-	boot run-insights --args "-d $(DATADIR)"
+	boot run-extractor --args "-d $(DATADIR)"
+regenerate-snapshots:
+	boot regenerate-snapshots "-f$(FILTER)"
 
 docker-test:
 	docker-compose run app test
 
 # runs jar with visual vm profiler. Needs visualVM to be running with Start up profiler plugin
 run-jar-agent:
-	java -Xmx256m -agentpath:/Applications/VisualVM.app/Contents/Resources/visualvm/profiler/lib/deployed/jdk16/mac/libprofilerinterface.jnilib=/Applications/VisualVM.app/Contents/Resources/visualvm/profiler/lib,5140 -jar targetinsights/project.jar -d tmp/
+	java -Xmx256m -agentpath:/Applications/VisualVM.app/Contents/Resources/visualvm/profiler/lib/deployed/jdk16/mac/libprofilerinterface.jnilib=/Applications/VisualVM.app/Contents/Resources/visualvm/profiler/lib,5140 -jar target/ex-fb-graph-api-1.0.jar -d tmp/

build.boot

@@ -2,60 +2,63 @@
 (set-env!
  :source-paths #{"src" "test"}
  :dependencies '[
-                 [org.clojure/clojure "1.9.0-alpha14"]
-                 [cheshire "5.6.3"]
-                 [clj-http "3.3.0"]
+                 [org.clojure/clojure "1.9.0-beta1"]
+                 [cheshire "5.8.0"]
+                 [clj-http "3.7.0"]
                  [clojure-csv/clojure-csv "2.0.1"]
                  [de.ubercode.clostache/clostache "1.4.0"]
                  [org.clojure/tools.cli "0.3.5"]
-                 [semantic-csv "0.1.0"]
-                 [org.clojure/data.csv "0.1.3"]
+                 [semantic-csv "0.2.0"]
+                 [org.clojure/data.csv "0.1.4"]
                  [org.clojure/test.check "0.9.0"]
-                 [org.clojure/core.async "0.2.395"]
+                 [org.clojure/core.async "0.3.443"]
                  [adzerk/boot-test "1.2.0" :scope "test"]
                  [clj-http-fake "1.0.3"]
                  [slingshot "0.12.2"]
-                 [clj-time "0.12.2"]])
+                 [clj-time "0.14.0"]])
 
-(require '[keboola.facebook.insights-extractor.core])
-(require '[keboola.regression-tests.core])
+(require '[keboola.facebook.extractor.core])
+(require '[keboola.snapshots.core])
 (require '[adzerk.boot-test :refer :all])
 
-(deftask run-insights
-  "run insights extractor"
+(deftask run-extractor
+  "run extractor"
   [x args VAL  str "arguments string for main- function"]
   (if-not args
     (do (boot.util/fail "arguments string x is requried. ")
         (*usage*)))
-  ((resolve 'keboola.facebook.insights-extractor.core/-main) args))
+  ((resolve 'keboola.facebook.extractor.core/-main) args))
 
 (deftask generate-test
   "given data dir with config.json, this task runs extraktor,
-record api calls, create regression tests with recrded api calls and compare result dirs"
-  [d data VAL str "name of directory in test/keboola/regresion_tests containing config.json"
+record api calls, create snapshot tests with recrded api calls and compare result dirs"
+  [d data VAL str "name of directory in test/keboola/snapshots containing config.json"
    s skip-token    bool "skip token anonymization in config.json"]
   (if-not data
     (do (boot.util/fail "arguments string d is requried. ")
         (*usage*)))
-  ((resolve 'keboola.regression-tests.core/generate-test) data (not skip-token)))
+  ((resolve 'keboola.snapshots.core/generate-test) data (not skip-token)))
 
-(deftask build-insights
-  "Builds an uberjar of insights extractor that can be run with java -jar"
+(deftask regenerate-snapshots [f dirfilter VAL str "regexp to filter dirs to process"]
+  ((resolve 'keboola.snapshots.core/regenerate-all-snapshot-dirs) dirfilter))
+
+(deftask build
+  "Builds an uberjar extractor that can be run with java -jar"
   []
   (comp
    (aot :all true)
-   (pom :project 'exinsights
-        :version "0.0.1")
+   (pom :project 'ex-fb-graph-api
+        :version "1.0")
    (uber)
-   (jar :main 'keboola.facebook.insights-extractor.core)
-   (target :dir #{"targetinsights"})))
+   (jar :main 'keboola.facebook.extractor.core)
+   (target :dir #{"target"})))
 
 (deftask start-docker-repl
   "run repl server on 1111 port"
   []
   (require 'boot.repl)
   (swap! boot.repl/*default-dependencies*
-         concat '[[cider/cider-nrepl "0.15.0-SNAPSHOT"]])
+         concat '[[cider/cider-nrepl "0.15.1"]])
   (swap! boot.repl/*default-middleware*
          conj 'cider.nrepl/cider-middleware)
   (repl :bind "0.0.0.0" :port 1111)

src/keboola/docker/config.clj

@@ -1,7 +1,7 @@
 (ns keboola.docker.config
   (:require [cheshire.core :refer [parse-string]]
             [clojure.string :refer [trim]]))
-            
+
 
 (def default-dir "/data/")
 
@@ -17,7 +17,7 @@
          file-content (slurp (trim (str dirpath "config.json")))]
      (parse-string file-content true))))
 
-(def load-config (memoize load-config-once))
+(def load-config load-config-once)
 
 (defn mkdirp [path]
   (let [dir (java.io.File. path)]
@@ -31,15 +31,15 @@
   ([datadir]
    (let [result (mkdirp (str (check-path (trim datadir)) "out/tables/"))]
      result)))
-     
-  
+
+
 (defn config [& datadir]
   (apply load-config datadir))
 
 
 (defn parameters [& datadir]
   (:parameters (apply load-config datadir)))
-  
+
 
 (defn app-access-token [& datadir]
   (let

src/keboola/facebook/api/parser.clj

@@ -1,5 +1,5 @@
 (ns keboola.facebook.api.parser
-  (:require   [clojure.spec :as s]
+  (:require   [clojure.spec.alpha :as s]
               [keboola.facebook.api.specs :as ds]
               [keboola.docker.runtime :refer [app-error]]
               [clj-time.core :as t]

src/keboola/facebook/api/request.clj

@@ -1,5 +1,5 @@
 (ns keboola.facebook.api.request
-  (:require   [clojure.spec :as s]
+  (:require   [clojure.spec.alpha :as s]
               [keboola.facebook.api.parser :as parser]
               [keboola.docker.runtime :refer [log-strings app-error log-error]]
               [slingshot.slingshot :refer [try+ throw+]]
@@ -117,14 +117,14 @@
               :fb-graph-node (:fb-graph-node params)
               :name (:table-name params)
               :data new-response}]
-              
+
             (contains? new-response (keyword ex-account-id))
             [{
               :parent-id ex-account-id
               :fb-graph-node top-node
               :name top-node
               :data new-response}]
-              
+
             :else
             (app-error (str "Unknown page structure:" (keys new-response) "next-page" next-page-url (dissoc params :body-data :response)))))))
 
@@ -162,7 +162,7 @@
         response (make-get-request full-url query-params)
         response-body (:body response)
         sanitized-path (keyword (string/replace path #"/" "_"))]
-        
+
     (log-strings "calling" full-url "with" preparsed-fields ids preparsed-since preparsed-until)
     (if (some? ids)
       (mapcat
@@ -174,7 +174,7 @@
           :table-name "page"
           :body-data [(if (not-empty path) {sanitized-path (second %)} (second %))]
           :response response-body})
-          
+
        response-body)
        ;else - no ids response
       (page-and-collect
@@ -185,7 +185,7 @@
           :table-name "page"
           :body-data [(if (not-empty path) {sanitized-path response-body} response-body)]
           :response (if (not-empty path) {sanitized-path response-body} response-body)}))))
-        
+
 
 (defn- collect-result [response api-fn]
   (lazy-seq
@@ -203,7 +203,7 @@
     (collect-result
      (:body (request-fn full-url))
      request-fn)))
-     
+
 
 (defn get-accounts [access-token & {:keys [version]}]
   (apply concat (get-request access-token "me/accounts" :version version)))

src/keboola/facebook/api/specs.clj

@@ -1,5 +1,5 @@
 (ns keboola.facebook.api.specs
-  (:require [clojure.spec :as s]
+  (:require [clojure.spec.alpha :as s]
             [clojure.string :refer [blank?]]
             [clojure.test.check.generators :as gen]))
 

src/keboola/facebook/insights_extractor/core.clj → src/keboola/facebook/extractor/core.clj

@@ -1,20 +1,23 @@
-(ns keboola.facebook.insights-extractor.core
+(ns keboola.facebook.extractor.core
   (:gen-class)
-  (:require [clojure.tools.cli :refer [parse-opts]]
+  (:require [clojure.string :as string]
+            [clojure.tools.cli :refer [parse-opts]]
             [keboola.docker.config :as docker-config]
+            [keboola.docker.runtime
+             :as
+             docker-runtime
+             :refer
+             [app-error log log-error-and-exit log-strings user-error]]
+            [keboola.facebook.extractor.query :as query]
+            [keboola.facebook.extractor.sync-actions :as sync-actions]
             [keboola.http.client :refer [fb-requests-count]]
-            [keboola.docker.runtime :as docker-runtime :refer [log-error-and-exit user-error app-error]]
-            [keboola.facebook.insights-extractor.query :as query]
-            [keboola.facebook.insights-extractor.sync-actions :as sync-actions]
             [keboola.utils.json-to-csv :as csv]
-            [keboola.docker.runtime :refer [log log-strings]]
-            [slingshot.slingshot :refer [try+ throw+]]
-            [clojure.string :as string]))
+            [slingshot.slingshot :refer [throw+ try+]]))
 
 (def cli-options  [["-d" "--dataDir path" "Path to data directory e.g. /data"]])
 
 (defn usage [options-summary]
-  (->> ["Keboola Facebook Insights Extractor"
+  (->> ["Keboola Facebook Graph Api Extractor"
         "Usage: program-name options"
         "Options:"
         options-summary]

src/keboola/facebook/insights_extractor/output.clj → src/keboola/facebook/extractor/output.clj

@@ -1,4 +1,4 @@
-(ns keboola.facebook.insights-extractor.output
+(ns keboola.facebook.extractor.output
   (:require [keboola.utils.json-to-csv :as csv]
             [keboola.docker.runtime :as runtime]
             [keboola.docker.config :refer [mkdirp]]

src/keboola/facebook/insights_extractor/query.clj → src/keboola/facebook/extractor/query.clj

@@ -1,10 +1,10 @@
-(ns keboola.facebook.insights-extractor.query
+(ns keboola.facebook.extractor.query
   (:gen-class)
-  (:require [keboola.facebook.api.request :as request]
-            [keboola.facebook.insights-extractor.output :as output]
-            [keboola.docker.runtime :as runtime]
+  (:require [clojure.core.async :as async]
             [clojure.string :as s]
-            [clojure.core.async :as async]))
+            [keboola.docker.runtime :as runtime]
+            [keboola.facebook.api.request :as request]
+            [keboola.facebook.extractor.output :as output]))
 
 (defn- run-and-write [token out-dir prefix query version]
   (let [nested-data (request/nested-request token query :version version)

src/keboola/facebook/insights_extractor/sync_actions.clj → src/keboola/facebook/extractor/sync_actions.clj

@@ -1,4 +1,4 @@
-(ns keboola.facebook.insights-extractor.sync-actions
+(ns keboola.facebook.extractor.sync-actions
   (:require [keboola.facebook.api.request :as request]
             [cheshire.core :refer [generate-string]]
             [slingshot.slingshot :refer [try+ throw+]]

src/keboola/http/recording.clj

@@ -1,7 +1,7 @@
 (ns keboola.http.recording
   (:require [clojure.walk :refer [postwalk postwalk-demo]]
             [cheshire.core :refer [generate-string]]))
-            
+
 
 (def recording (atom '()))
 (def do-recording? (atom false))
@@ -61,14 +61,15 @@
 (defn prepare-recording [token]
   (apply str (mapcat (fn [r]
                        (let [request (postwalk #(replace-token % token) (:request r))
-                             response (postwalk #(replace-token % token) (:response r))]
+                             response (postwalk #(replace-token % token) (:response r))
+                             shaved-response (select-keys response [:status :body])
+                             ]
                          [(pprint request)
-                          (str "(fn [req]" (pprint response) ")")]))
+                          (str "(fn [req]" (pprint shaved-response) ")")]))
                      @recording)))
 
 (defn save-current-recording [path namespace-name token-to-replace]
   (let [ns-str (str "(ns " namespace-name ")\n")
         recording-str (str "\n{\n" (prepare-recording token-to-replace) "\n}")]
     (with-open [w (clojure.java.io/writer path)]
       (.write w (str ns-str (str "(def recorded " recording-str ")"))))))
-

src/keboola/utils/json_to_csv.clj

@@ -2,9 +2,10 @@
   (:require [semantic-csv.core :as csv]
             [clojure.data.csv :as cd-csv]
             [clojure.set :refer [rename-keys]]
-            [clojure.spec :as s]
+            [clojure.spec.alpha :as s
+             ]
             [clojure.string]))
-            
+
 ; (mapcat #(conj '(:aa) %) (filter #(.contains (name %) "-") (keys {:a-id 2 :b 2}))) => (:a-id :aa)
 
 (s/fdef replace-dash

test/keboola/facebook/api/parser_test.clj

@@ -1,7 +1,7 @@
 (ns keboola.facebook.api.parser-test
   (:require [keboola.facebook.api.parser :as sut]
             [clojure.test :refer :all]
-            [clojure.spec :as s]
+            [clojure.spec.alpha :as s]
             [keboola.test-utils.core :refer [test-and-check]]
             [clojure.test.check.clojure-test :refer [defspec]]
             [clojure.test.check.properties :as prop]))