From 2c9d318fb4307756e7b7c32bc5f366eed1d5c372 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Thu, 28 Nov 2024 10:29:02 +0200 Subject: [PATCH] Implement fetching of most positive/negative/replied/retweeted tweets per slug --- lib/sanbase/social_data/tweet.ex | 81 +++++++++++++ .../graphql/resolvers/social_data_resolver.ex | 8 ++ .../schema/queries/social_data_queries.ex | 12 ++ .../graphql/schema/types/social_data_types.ex | 27 +++++ .../billing/query_access_level_test.exs | 7 +- .../graphql/social_data/most_tweets_test.exs | 113 ++++++++++++++++++ 6 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 lib/sanbase/social_data/tweet.ex create mode 100644 test/sanbase_web/graphql/social_data/most_tweets_test.exs diff --git a/lib/sanbase/social_data/tweet.ex b/lib/sanbase/social_data/tweet.ex new file mode 100644 index 000000000..36033e38b --- /dev/null +++ b/lib/sanbase/social_data/tweet.ex @@ -0,0 +1,81 @@ +defmodule Sanbase.SocialData.Tweet do + require Mockery.Macro + @tweet_types [:most_positive, :most_negative, :most_retweets, :most_replies] + @tweet_type_mapping %{ + most_positive: :sentiment_pos, + most_negative: :sentiment_neg, + most_retweets: :retweet, + most_replies: :reply + } + @recv_timeout 25_000 + def get_most_tweets(%{} = selector, type, from, to, size) do + slugs = (Map.get(selector, :slug) || Map.get(selector, :slugs)) |> List.wrap() + + tweets_request(slugs, type, from, to, size) + |> handle_tweets_response() + end + + defp handle_tweets_response({:ok, %HTTPoison.Response{status_code: 200, body: json_body}}) do + case Jason.decode(json_body) do + {:ok, %{"data" => data}} -> + {:ok, decode_tweets_data(data)} + + _ -> + {:error, "Malformed response fetching tweets"} + end + end + + defp handle_tweets_response({:ok, %HTTPoison.Response{status_code: status}}) do + {:error, "Error status #{status} fetching tweets"} + end + + defp decode_tweets_data(data_map) when is_map(data_map) do + data_map + |> Enum.map(fn {slug, json_list} -> + list = Jason.decode!(json_list) + + tweets = + Enum.map(list, fn map -> + %{ + text: Map.fetch!(map, "text"), + screen_name: Map.fetch!(map, "screen_name"), + datetime: + Map.fetch!(map, "timestamp") + |> NaiveDateTime.from_iso8601!() + |> DateTime.from_naive!("Etc/UTC"), + replies_count: Map.fetch!(map, "reply"), + sentiment_positive: Map.fetch!(map, "sentiment_pos"), + sentiment_negative: Map.fetch!(map, "sentiment_neg"), + retweets_count: Map.fetch!(map, "retweet") + } + end) + + %{slug: slug, tweets: tweets} + end) + end + + defp tweets_request(slugs, type, from, to, size) + when type in @tweet_types and is_list(slugs) do + url = Path.join([metrics_hub_url(), "fetch_documents"]) + + options = [ + recv_timeout: @recv_timeout, + params: [ + {"slugs", slugs |> List.wrap() |> Enum.join(",")}, + {"from_timestamp", from |> DateTime.truncate(:second) |> DateTime.to_iso8601()}, + {"to_timestamp", to |> DateTime.truncate(:second) |> DateTime.to_iso8601()}, + {"size", size}, + {"source", "twitter"}, + {"most_type", Map.fetch!(@tweet_type_mapping, type)} + ] + ] + + http_client().get(url, [], options) + end + + defp http_client, do: Mockery.Macro.mockable(HTTPoison) + + defp metrics_hub_url() do + Sanbase.Utils.Config.module_get(Sanbase.SocialData, :metricshub_url) + end +end diff --git a/lib/sanbase_web/graphql/resolvers/social_data_resolver.ex b/lib/sanbase_web/graphql/resolvers/social_data_resolver.ex index ed2b31d63..0e555170b 100644 --- a/lib/sanbase_web/graphql/resolvers/social_data_resolver.ex +++ b/lib/sanbase_web/graphql/resolvers/social_data_resolver.ex @@ -9,6 +9,14 @@ defmodule SanbaseWeb.Graphql.Resolvers.SocialDataResolver do @context_words_default_size 10 + def get_most_tweets( + _root, + %{selector: selector, from: from, to: to, size: size, tweet_type: type}, + _resolution + ) do + SocialData.Tweet.get_most_tweets(selector, type, from, to, size) + end + def get_metric_spike_explanations( _root, %{metric: metric, slug: slug, from: from, to: to}, diff --git a/lib/sanbase_web/graphql/schema/queries/social_data_queries.ex b/lib/sanbase_web/graphql/schema/queries/social_data_queries.ex index 7ff30f377..336eae1b9 100644 --- a/lib/sanbase_web/graphql/schema/queries/social_data_queries.ex +++ b/lib/sanbase_web/graphql/schema/queries/social_data_queries.ex @@ -8,6 +8,18 @@ defmodule SanbaseWeb.Graphql.Schema.SocialDataQueries do alias SanbaseWeb.Graphql.Resolvers.SocialDataResolver object :social_data_queries do + field :get_most_tweets, list_of(:slug_tweets_object) do + meta(access: :free) + + arg(:selector, non_null(:selector_slug_or_slug_input_object)) + arg(:from, non_null(:datetime)) + arg(:to, non_null(:datetime)) + arg(:size, non_null(:integer)) + arg(:tweet_type, non_null(:most_tweet_type)) + + cache_resolve(&SocialDataResolver.get_most_tweets/3) + end + field :get_metric_spike_explanations, list_of(:metric_spike_explanation) do meta(access: :free) diff --git a/lib/sanbase_web/graphql/schema/types/social_data_types.ex b/lib/sanbase_web/graphql/schema/types/social_data_types.ex index 96d69c002..3c5cef39c 100644 --- a/lib/sanbase_web/graphql/schema/types/social_data_types.ex +++ b/lib/sanbase_web/graphql/schema/types/social_data_types.ex @@ -4,6 +4,13 @@ defmodule SanbaseWeb.Graphql.SocialDataTypes do alias SanbaseWeb.Graphql.Resolvers.SocialDataResolver import SanbaseWeb.Graphql.Cache, only: [cache_resolve: 1] + enum :most_tweet_type do + value(:most_positive) + value(:most_negative) + value(:most_retweets) + value(:most_replies) + end + enum :trending_word_type_filter do value(:project) value(:non_project) @@ -30,6 +37,26 @@ defmodule SanbaseWeb.Graphql.SocialDataTypes do value(:telegram_discussion_overview) end + input_object :selector_slug_or_slug_input_object do + field(:slug, :string) + field(:slugs, list_of(:string)) + end + + object :slug_tweets_object do + field(:slug, non_null(:string)) + field(:tweets, list_of(:tweet)) + end + + object :tweet do + field(:datetime, non_null(:datetime)) + field(:text, non_null(:string)) + field(:screen_name, non_null(:string)) + field(:sentiment_positive, :float) + field(:sentiment_negative, :float) + field(:replies_count, :integer) + field(:retweets_count, :integer) + end + object :metric_spike_explanation do field(:spike_start_datetime, non_null(:datetime)) field(:spike_end_datetime, non_null(:datetime)) diff --git a/test/sanbase/billing/query_access_level_test.exs b/test/sanbase/billing/query_access_level_test.exs index 455b9ff7e..730d4e2e5 100644 --- a/test/sanbase/billing/query_access_level_test.exs +++ b/test/sanbase/billing/query_access_level_test.exs @@ -96,6 +96,7 @@ defmodule Sanbase.Billing.QueryAccessLevelTest do :get_metric, :get_metric_spike_explanations, :get_most_recent, + :get_most_tweets, :get_most_used, :get_most_voted, :get_nft_collection_by_contract, @@ -188,17 +189,17 @@ defmodule Sanbase.Billing.QueryAccessLevelTest do expected_restricted_queries = [ - :get_latest_metric_data, :gas_used, + :get_latest_metric_data, :get_project_trending_history, - :get_word_trending_history, :get_trending_words, + :get_word_trending_history, :miners_balance, :percent_of_token_supply_on_exchanges, :realtime_top_holders, :top_exchanges_by_balance, - :top_holders_percent_of_total_supply, :top_holders, + :top_holders_percent_of_total_supply, :word_context, :word_trend_score, :words_context, diff --git a/test/sanbase_web/graphql/social_data/most_tweets_test.exs b/test/sanbase_web/graphql/social_data/most_tweets_test.exs new file mode 100644 index 000000000..a869a6071 --- /dev/null +++ b/test/sanbase_web/graphql/social_data/most_tweets_test.exs @@ -0,0 +1,113 @@ +defmodule SanbaseWeb.Graphql.MostTweetsSocialDataTest do + use SanbaseWeb.ConnCase, async: false + + import Sanbase.Factory + import SanbaseWeb.Graphql.TestHelpers + + setup do + %{user: user} = insert(:subscription_pro_sanbase, user: insert(:user)) + conn = setup_jwt_auth(build_conn(), user) + + %{conn: conn} + end + + test "successfully fetch tweets", context do + body = metricshub_data() |> Jason.encode!() + resp = %HTTPoison.Response{status_code: 200, body: body} + + Sanbase.Mock.prepare_mock2(&HTTPoison.get/3, {:ok, resp}) + |> Sanbase.Mock.run_with_mocks(fn -> + # Get the top 2 most positive tweets per slug in the given time range + query = """ + { + getMostTweets( + tweetType: MOST_POSITIVE + selector: { slugs: ["bitcoin", "ethereum"] } + from: "2024-11-25T00:00:00Z" + to: "2024-11-28T00:00:00Z" + size: 2){ + slug + tweets{ + datetime + text + screenName + sentimentPositive + sentimentNegative + repliesCount + retweetsCount + } + } + } + """ + + result = + context.conn + |> post("/graphql", query_skeleton(query)) + |> json_response(200) + |> get_in(["data", "getMostTweets"]) + + assert %{ + "slug" => "bitcoin", + "tweets" => [ + %{ + "datetime" => "2024-11-26T18:21:30Z", + "repliesCount" => 0, + "retweetsCount" => 0, + "screenName" => "take_gains", + "sentimentNegative" => 0.0171372827, + "sentimentPositive" => 0.9828627173, + "text" => + "Whipsaw wick completed, lets continue. \n\n$BTC https://t.co/e8mH8RUKjO" + }, + %{ + "datetime" => "2024-11-26T19:07:51Z", + "repliesCount" => 0, + "retweetsCount" => 0, + "screenName" => "IIICapital", + "sentimentNegative" => 0.0222001588, + "sentimentPositive" => 0.9777998412, + "text" => + "Incredible podcast with two of the sharpest minds in bitcoin.\n\nThank you both for the time @Excellion and @dhruvbansal.\n\nTune in and comment whether you think bitcoin was an invention or discovery!" + } + ] + } in result + + assert %{ + "slug" => "ethereum", + "tweets" => [ + %{ + "datetime" => "2024-11-26T17:07:36Z", + "repliesCount" => 0, + "retweetsCount" => 0, + "screenName" => "koeppelmann", + "sentimentNegative" => 0.0269591219, + "sentimentPositive" => 0.9730408781, + "text" => + "Thanks for hosting this debate @laurashin! While I think Justin and I share a similar vision of what Ethereum should ideally become in a couple of years, he thinks we are on track for it - I believe decisive action is needed now to achieve that vision." + }, + %{ + "datetime" => "2024-11-27T14:00:12Z", + "repliesCount" => 0, + "retweetsCount" => 1, + "screenName" => "AerodromeFi", + "sentimentNegative" => 0.0277438289, + "sentimentPositive" => 0.9722561711, + "text" => + "New Launch Alert ✈️\n\nA big welcome to @doge_eth_gov who have launched an $DOGE - $WETH pool on Aerodrome.\n\nBridge $DOGE from Ethereum mainnet to @base, powered by @axelar: https://t.co/wejMHuq62H\n\nLiquidity has been added and LP rewards incoming. https://t.co/A5lpoDZakD" + } + ] + } in result + end) + end + + defp metricshub_data() do + %{ + "data" => %{ + "bitcoin" => + "[{\"screen_name\":\"take_gains\",\"text\":\"Whipsaw wick completed, lets continue. \\n\\n$BTC https:\\/\\/t.co\\/e8mH8RUKjO\",\"reply\":0,\"retweet\":0,\"timestamp\":\"2024-11-26T18:21:30\",\"sentiment_neg\":0.0171372827,\"sentiment_pos\":0.9828627173},{\"screen_name\":\"IIICapital\",\"text\":\"Incredible podcast with two of the sharpest minds in bitcoin.\\n\\nThank you both for the time @Excellion and @dhruvbansal.\\n\\nTune in and comment whether you think bitcoin was an invention or discovery!\",\"reply\":0,\"retweet\":0,\"timestamp\":\"2024-11-26T19:07:51\",\"sentiment_neg\":0.0222001588,\"sentiment_pos\":0.9777998412}]", + "ethereum" => + "[{\"screen_name\":\"koeppelmann\",\"text\":\"Thanks for hosting this debate @laurashin! While I think Justin and I share a similar vision of what Ethereum should ideally become in a couple of years, he thinks we are on track for it - I believe decisive action is needed now to achieve that vision.\",\"reply\":0,\"retweet\":0,\"timestamp\":\"2024-11-26T17:07:36\",\"sentiment_neg\":0.0269591219,\"sentiment_pos\":0.9730408781},{\"screen_name\":\"AerodromeFi\",\"text\":\"New Launch Alert \\u2708\\ufe0f\\n\\nA big welcome to @doge_eth_gov who have launched an $DOGE - $WETH pool on Aerodrome.\\n\\nBridge $DOGE from Ethereum mainnet to @base, powered by @axelar: https:\\/\\/t.co\\/wejMHuq62H\\n\\nLiquidity has been added and LP rewards incoming. https:\\/\\/t.co\\/A5lpoDZakD\",\"reply\":0,\"retweet\":1,\"timestamp\":\"2024-11-27T14:00:12\",\"sentiment_neg\":0.0277438289,\"sentiment_pos\":0.9722561711}]" + } + } + end +end