Skip to content

Commit

Permalink
[backport] [serdes-v1] can now dump/load entity_id (metabase#39535)
Browse files Browse the repository at this point in the history
  • Loading branch information
piranha authored Mar 4, 2024
1 parent cb5726b commit a900c85
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 15 deletions.
28 changes: 15 additions & 13 deletions enterprise/backend/src/metabase_enterprise/serialization/cmd.clj
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
[clojure.java.io :as io]
[metabase-enterprise.serialization.dump :as dump]
[metabase-enterprise.serialization.load :as load]
[metabase-enterprise.serialization.serialize :as serialize]
[metabase-enterprise.serialization.v2.entity-ids :as v2.entity-ids]
[metabase-enterprise.serialization.v2.extract :as v2.extract]
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
Expand Down Expand Up @@ -163,7 +164,7 @@

(defn v1-dump!
"Legacy Metabase app data dump"
[path {:keys [state user] :or {state :active} :as opts}]
[path {:keys [state user include-entity-id] :or {state :active} :as opts}]
(log/info (trs "BEGIN DUMP to {0} via user {1}" path user))
(mdb/setup-db!)
(check-premium-token!)
Expand All @@ -188,18 +189,19 @@
(t2/select Metric :table_id [:in (map :id tables)] {:order-by [[:id :asc]]})
(t2/select Metric))
collections (select-collections users state)]
(dump/dump! path
databases
tables
(mapcat field/with-values (u/batches-of 32000 fields))
metrics
(select-segments-in-tables tables state)
collections
(select-entities-in-collections NativeQuerySnippet collections state)
(select-entities-in-collections Card collections state)
(select-entities-in-collections Dashboard collections state)
(select-entities-in-collections Pulse collections state)
users))
(binding [serialize/*include-entity-id* (boolean include-entity-id)]
(dump/dump! path
databases
tables
(mapcat field/with-values (u/batches-of 32000 fields))
metrics
(select-segments-in-tables tables state)
collections
(select-entities-in-collections NativeQuerySnippet collections state)
(select-entities-in-collections Card collections state)
(select-entities-in-collections Dashboard collections state)
(select-entities-in-collections Pulse collections state)
users)))
(dump/dump-settings! path)
(dump/dump-dimensions! path)
(log/info (trs "END DUMP to {0} via user {1}" path user)))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@
;; version 2 - start adding namespace portion to /collections/ paths
2)

(def ^:dynamic *include-entity-id*
"If entity_id should be included in v1 serialization dump"
false)

(def ^:private ^{:arglists '([form])} mbql-entity-reference?
"Is given form an MBQL entity reference?"
(partial mbql.normalize/is-clause? #{:field :field-id :fk-> :metric :segment}))
Expand Down Expand Up @@ -103,7 +107,8 @@
[entity]
(cond-> (dissoc entity :id :creator_id :created_at :updated_at :db_id :location
:dashboard_id :fields_hash :personal_owner_id :made_public_by_id :collection_id
:pulse_id :result_metadata :entity_id :action_id)
:pulse_id :result_metadata :action_id)
(not *include-entity-id*) (dissoc :entity_id)
(some #(instance? % entity) (map type [Metric Field Segment])) (dissoc :table_id)))

(defmulti ^:private serialize-one
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,3 +163,42 @@
(throw (ex-info "Do not call me!" {})))]
(is (thrown-with-msg? Exception #"Destination path is not writeable: "
(cmd/export dump-dir))))))))

(deftest entity-id-dump&load-test
(let [entity-ids* (atom {})
eid-map #(into {} (map (juxt :name :entity_id) %))]
(testing "--include-entity-id should include entity ids in serialization"
(premium-features-test/with-premium-features #{:serialization}
(ts/with-random-dump-dir [dump-dir "serialization"]
(ts/with-dbs [source-db dest-db]
(testing "create 2 questions in a dashboard"
(ts/with-db source-db
(let [db (ts/create! Database)
dash (ts/create! Dashboard)
c1 (ts/create! Card {:name "card1"
:database_id (:id db)
:dataset_query {:database (:id db), :type :native, :native {:query "SELECT 1;"}}})
c2 (ts/create! Card {:name "card2"
:database_id (:id db)
:dataset_query {:database (:id db), :type :native, :native {:query "SELECT 1;"}}})
_ (ts/create! DashboardCard {:dashboard_id (:id dash) :card_id (:id c1)})
_ (ts/create! DashboardCard {:dashboard_id (:id dash) :card_id (:id c2)})]
(testing "initial dump"
(is (nil? (cmd/dump dump-dir))))
(testing "storing original entity ids"
(is (reset! entity-ids* (eid-map [c1 c2])))))))
(testing "initial load"
(ts/with-db dest-db
(is (some? (ts/create! User, :is_superuser true)))
(is (nil? (cmd/load dump-dir "--on-error" "abort")))
(testing "verify that entities got their own entity_id"
(is (not= @entity-ids*
(eid-map (t2/select Card)))))))
(testing "creating dump with entity ids included"
(ts/with-db source-db
(is (nil? (cmd/dump dump-dir "--include-entity-id")))))
(testing "loading dump with entity ids will overwrite new entity ids with original ones"
(ts/with-db dest-db
(is (nil? (cmd/load dump-dir "--on-error" "abort" "--mode" "update")))
(is (= @entity-ids*
(eid-map (t2/select Card))))))))))))
3 changes: 2 additions & 1 deletion src/metabase/cmd.clj
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,8 @@
:default :all
:default-desc "all"
:parse-fn mbql.u/normalize-token
:validate [#{:active :all} "Must be 'active' or 'all'"]]]}
:validate [#{:active :all} "Must be 'active' or 'all'"]]
[nil "--include-entity-id" "Include entity_id property in all dumped entities. Default: false."]]}
[path & options]
(log/warn (u/colorize :red (trs "''dump'' is deprecated and will be removed in a future release. Please migrate to ''export''.")))
(call-enterprise 'metabase-enterprise.serialization.cmd/v1-dump! path (get-parsed-options #'dump options)))
Expand Down

0 comments on commit a900c85

Please sign in to comment.