Merge pull request #68 from chronicle-app/use-chronicle-core

Move models and serialization to separate chronicle-core gem
chronicle-app · Aug 13, 2023 · df97cbd · df97cbd
2 parents 10169d9 + 846394a
commit df97cbd
Show file tree

Hide file tree

Showing 24 changed files with 24 additions and 347 deletions.
diff --git a/chronicle-etl.gemspec b/chronicle-etl.gemspec
@@ -38,6 +38,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 2.7"
 
   spec.add_dependency "activesupport", "~> 7.0"
+  spec.add_dependency "chronicle-core", "~> 0.2.1"
   spec.add_dependency "chronic_duration", "~> 0.10.6"
   spec.add_dependency "colorize", "~> 0.8.1"
   spec.add_dependency "gems", ">= 1"

diff --git a/lib/chronicle/etl.rb b/lib/chronicle/etl.rb
@@ -1,3 +1,5 @@
+require 'chronicle/schema'
+
 require_relative 'etl/registry/registry'
 require_relative 'etl/authorizer'
 require_relative 'etl/config'
@@ -9,14 +11,8 @@
 require_relative 'etl/job_logger'
 require_relative 'etl/job'
 require_relative 'etl/logger'
-require_relative 'etl/models/activity'
-require_relative 'etl/models/attachment'
-require_relative 'etl/models/base'
-require_relative 'etl/models/raw'
-require_relative 'etl/models/entity'
 require_relative 'etl/runner'
 require_relative 'etl/secrets'
-require_relative 'etl/serializers/serializer'
 require_relative 'etl/utils/binary_attachments'
 require_relative 'etl/utils/hash_utilities'
 require_relative 'etl/utils/text_recognition'

diff --git a/lib/chronicle/etl/exceptions.rb b/lib/chronicle/etl/exceptions.rb
@@ -51,8 +51,6 @@ class ProviderConnectorNotAvailableError < ConnectorNotAvailableError; end
 
     class ExtractionError < Error; end
 
-    class SerializationError < Error; end
-
     class TransformationError < Error; end
 
     class UntransformableRecordError < TransformationError; end

diff --git a/lib/chronicle/etl/loaders/json_loader.rb b/lib/chronicle/etl/loaders/json_loader.rb
@@ -72,7 +72,8 @@ def finish
 
       # TODO: implement this
       def serializer
-        @config.serializer || Chronicle::ETL::RawSerializer
+        require 'chronicle/serialization'
+        @config.serializer || Chronicle::Serialization::HashSerializer
       end
     end
   end

diff --git a/lib/chronicle/etl/loaders/rest_loader.rb b/lib/chronicle/etl/loaders/rest_loader.rb
@@ -1,6 +1,7 @@
 require 'net/http'
 require 'uri'
 require 'json'
+require 'chronicle/serialization'
 
 module Chronicle
   module ETL
@@ -14,7 +15,7 @@ class RestLoader < Chronicle::ETL::Loader
       setting :access_token
 
       def load(record)
-        payload = Chronicle::ETL::JSONAPISerializer.serialize(record)
+        payload = Chronicle::Serialization::JSONAPISerializer.serialize(record)
         # have the outer data key that json-api expects
         payload = { data: payload } unless payload[:data]
 

diff --git a/lib/chronicle/etl/loaders/table_loader.rb b/lib/chronicle/etl/loaders/table_loader.rb
@@ -11,7 +11,7 @@ class TableLoader < Chronicle::ETL::Loader
 
       setting :truncate_values_at, default: 40
       setting :table_renderer, default: :basic
-      setting :fields_exclude, default: ['lids', 'type']
+      setting :fields_exclude, default: ['type']
       setting :header_row, default: true
 
       def load(record)

diff --git a/lib/chronicle/etl/models/activity.rb b/lib/chronicle/etl/models/activity.rb
diff --git a/lib/chronicle/etl/models/attachment.rb b/lib/chronicle/etl/models/attachment.rb
diff --git a/lib/chronicle/etl/models/base.rb b/lib/chronicle/etl/models/base.rb
diff --git a/lib/chronicle/etl/models/entity.rb b/lib/chronicle/etl/models/entity.rb
diff --git a/lib/chronicle/etl/models/raw.rb b/lib/chronicle/etl/models/raw.rb
diff --git a/lib/chronicle/etl/registry/plugins.rb b/lib/chronicle/etl/registry/plugins.rb
@@ -107,7 +107,7 @@ def self.exists?(name)
         # All versions of all plugins currently installed
         def self.installed_gemspecs
           # TODO: add check for chronicle-etl dependency
-          Gem::Specification.filter { |s| s.name.match(/^chronicle-/) && s.name != "chronicle-etl" }
+          Gem::Specification.filter { |s| s.name.match(/^chronicle-/) && s.name != "chronicle-etl" && s.name != "chronicle-core" }
         end
 
         # Latest version of each installed plugin

diff --git a/lib/chronicle/etl/runner.rb b/lib/chronicle/etl/runner.rb
@@ -88,8 +88,8 @@ def process_extraction(extraction)
     new_objects = [transformer.transform].flatten
 
     # raise an error unless all new_objects are a Base
-    unless new_objects.all? { |r| r.is_a?(Chronicle::ETL::Models::Base) || r.is_a?(Chronicle::ETL::Models::Raw) }
-      raise(Chronicle::ETL::RunnerError, "Expected transformer to output a Chronicle ETL Model")
+    unless new_objects.all? { |r| r.is_a?(Chronicle::Schema::Base) }
+      raise(Chronicle::ETL::RunnerError, "Expected transformer to output a Chronicle Schema model")
     end
 
     Chronicle::ETL::Logger.debug(tty_log_transformation(transformer))

diff --git a/lib/chronicle/etl/serializers/jsonapi_serializer.rb b/lib/chronicle/etl/serializers/jsonapi_serializer.rb
diff --git a/lib/chronicle/etl/serializers/raw_serializer.rb b/lib/chronicle/etl/serializers/raw_serializer.rb
diff --git a/lib/chronicle/etl/serializers/serializer.rb b/lib/chronicle/etl/serializers/serializer.rb