From 1b1522fb693894772cb6ba0502d13e65faaa65a0 Mon Sep 17 00:00:00 2001 From: Mikkel Kroman Date: Fri, 27 Dec 2024 17:21:34 +0100 Subject: [PATCH 1/4] tldr: Don't print empty lines --- scripts/tldr.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/tldr.rb b/scripts/tldr.rb index 0c57a49..d51ae62 100644 --- a/scripts/tldr.rb +++ b/scripts/tldr.rb @@ -86,8 +86,8 @@ def initialize return channel.say(format('Could not summarize contents')) if data.nil? || data.empty? markdown = data['markdown'] - markdown.each_line do |line| - channel.say(format(line.strip)) + markdown.each_line.lazy.map(&:strip).each do |line| + channel.say(format(line)) unless line.empty? end rescue Error => e channel.say(format("Error:\x0f #{e}")) @@ -102,7 +102,6 @@ def initialize channel.say(format("Error:\x0f #{e.message}")) end - # This is an argument parser for the `.tldr` command. ArgumentParser = Optimist::Parser.new do banner <<~BANNER From 43108d29fad851b805085be21bf83c37cc984560 Mon Sep 17 00:00:00 2001 From: Mikkel Kroman Date: Fri, 27 Dec 2024 18:05:07 +0100 Subject: [PATCH 2/4] tldr: Use anthropic claude to summarize the summary --- scripts/tldr.rb | 66 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/scripts/tldr.rb b/scripts/tldr.rb index d51ae62..2a13273 100644 --- a/scripts/tldr.rb +++ b/scripts/tldr.rb @@ -53,6 +53,24 @@ 'Chinese (simplified)' => 'ZH', 'Chinese (traditional)' => 'ZH-HANT' }.freeze + RESUMMARIZE_PROMPT = <<-PROMPT + You will be given a long summary of an article or news story. Your task is to create a shorter, more concise summary that captures the essential information. + + Here is the long summary: + + {{LONG_SUMMARY}} + + + To create the shorter summary: + 1. Identify the main topic or event described in the long summary. + 2. Determine the key actors or parties involved. + 3. Highlight the most important facts or developments. + 4. Include any significant outcomes or implications. + + Your summary should be approximately 1 sentence long. Aim for clarity and brevity while maintaining the core message of the original summary. + + Write the summary using a tag. + PROMPT class Error < StandardError; end class RequestError < Error; end @@ -66,6 +84,9 @@ def initialize @token = @config['token'] || ENV.fetch('KAGI_SESSION_TOKEN', nil) raise "`token' (or KAGI_SESSION_TOKEN) is not set" unless @token + @anthropic_api_key = @config['anthropic_api_key'] || ENV.fetch('ANTHROPIC_API_KEY', nil) + raise "`anthropic_api_key' (or ANTHROPIC_API_KEY) is not set" unless @anthropic_api_key + @http = HTTPX.with_timeout(total_timeout: 60) end @@ -86,7 +107,9 @@ def initialize return channel.say(format('Could not summarize contents')) if data.nil? || data.empty? markdown = data['markdown'] - markdown.each_line.lazy.map(&:strip).each do |line| + summary = resummarize(markdown).wait + + summary.to_s.each_line.lazy.map(&:strip).each do |line| channel.say(format(line)) unless line.empty? end rescue Error => e @@ -131,6 +154,47 @@ def summarize(url, summary_type: 'summary', target_language: nil) end end + def resummarize(summary) + Async do + response = request_resummary(summary).wait + + json = response&.json + return unless json&.key?('content') + + content = json['content'] + text_content = content.filter { |c| c['type'] == 'text' } + text = text_content.map { |c| c['text'] }.join($INPUT_RECORD_SEPARATOR) + + if text =~ %r{(.*?)}mi + Regexp.last_match(1) + else + text + end + end + end + + # Send a request to have Claude resummarize the summary. + def request_resummary(summary) + Async do + prompt = RESUMMARIZE_PROMPT.gsub('{{LONG_SUMMARY}}', summary) + request = { + 'model' => 'claude-3-5-haiku-latest', + 'max_tokens' => 1024, + 'messages' => [ + { 'role' => 'user', 'content' => [{ type: 'text', text: prompt }] } + ] + } + headers = { + 'anthropic-version' => '2023-06-01', + 'x-api-key' => @anthropic_api_key, + 'content-type' => 'application/json' + } + response = @http.post('https://api.anthropic.com/v1/messages', json: request, headers:) + response.raise_for_status + response + end + end + def process_summary_response(response) raise UnexpectedResponseError, "unexpected http status: #{response.status}" unless response.status == 200 From 16581c202bed23497b4de529d0f8acd17c86cc9a Mon Sep 17 00:00:00 2001 From: Mikkel Kroman Date: Fri, 27 Dec 2024 18:12:55 +0100 Subject: [PATCH 3/4] Bump dependencies --- Gemfile.lock | 171 +++++++++++++++++++++++++++------------------------ 1 file changed, 91 insertions(+), 80 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5eb3713..8219aa3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -26,51 +26,52 @@ PATH GEM remote: https://rubygems.org/ specs: - activesupport (7.1.3.2) + activesupport (7.2.2.1) base64 + benchmark (>= 0.3) bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) - addressable (2.8.6) - public_suffix (>= 2.0.2, < 6.0) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) ast (2.4.2) - async (2.10.1) - console (~> 1.10) + async (2.21.1) + console (~> 1.29) fiber-annotation - io-event (~> 1.5, >= 1.5.1) - timers (~> 4.1) - async-io (1.42.0) + io-event (~> 1.6, >= 1.6.5) + async-io (1.43.2) async aws-eventstream (1.3.0) - aws-partitions (1.905.0) - aws-sdk-core (3.191.5) + aws-partitions (1.1029.0) + aws-sdk-core (3.214.0) aws-eventstream (~> 1, >= 1.3.0) - aws-partitions (~> 1, >= 1.651.0) - aws-sigv4 (~> 1.8) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.78.0) - aws-sdk-core (~> 3, >= 3.191.0) - aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.146.1) - aws-sdk-core (~> 3, >= 3.191.0) + aws-sdk-kms (1.96.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-s3 (1.176.1) + aws-sdk-core (~> 3, >= 3.210.0) aws-sdk-kms (~> 1) - aws-sigv4 (~> 1.8) - aws-sigv4 (1.8.0) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.10.1) aws-eventstream (~> 1, >= 1.0.2) backport (1.2.0) base64 (0.2.0) - benchmark (0.3.0) - bigdecimal (3.1.7) - concurrent-ruby (1.2.3) + benchmark (0.4.0) + bigdecimal (3.1.9) + concurrent-ruby (1.3.4) connection_pool (2.4.1) - console (1.23.6) + console (1.29.2) fiber-annotation - fiber-local + fiber-local (~> 1.1) json deep_merge (1.2.2) diff-lcs (1.5.1) @@ -80,46 +81,52 @@ GEM i18n drb (2.2.1) e2mmap (0.1.0) - faraday (2.9.0) - faraday-net_http (>= 2.0, < 3.2) - faraday-net_http (3.1.0) - net-http + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.0) + net-http (>= 0.5.0) fiber-annotation (0.2.0) - fiber-local (1.0.0) + fiber-local (1.1.0) + fiber-storage + fiber-storage (1.0.0) fuzzy_match (2.1.0) hashie (5.0.0) htmlentities (4.3.4) - http-2-next (1.0.3) - http-cookie (1.0.5) + http-2 (1.0.2) + http-cookie (1.0.8) domain_name (~> 0.5) - httpx (1.2.3) - http-2-next (>= 1.0.3) - i18n (1.14.4) + httpx (1.4.0) + http-2 (>= 1.0.0) + i18n (1.14.6) concurrent-ruby (~> 1.0) - io-event (1.5.1) + io-event (1.7.5) ircparser (1.0.0) - jaro_winkler (1.5.6) + jaro_winkler (1.6.0) jmespath (1.6.2) - json (2.7.1) - jwt (2.8.1) + json (2.9.1) + jwt (2.10.1) base64 - kramdown (2.4.0) - rexml + kramdown (2.5.1) + rexml (>= 3.3.9) kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) language_server-protocol (3.17.0.3) - mime-types (3.5.2) + logger (1.6.4) + mime-types (3.6.0) + logger mime-types-data (~> 3.2015) - mime-types-data (3.2024.0305) - minitest (5.22.3) + mime-types-data (3.2024.1203) + minitest (5.25.4) multi_json (1.15.0) - multi_xml (0.6.0) - mutex_m (0.2.0) + multi_xml (0.7.1) + bigdecimal (~> 3.1) net-dns (0.20.0) - net-http (0.4.1) + net-http (0.6.0) uri netrc (0.11.0) - nokogiri (1.16.3-x86_64-linux) + nokogiri (1.18.0-x86_64-linux-gnu) racc (~> 1.4) oauth2 (2.0.9) faraday (>= 0.17.3, < 3.0) @@ -128,8 +135,9 @@ GEM rack (>= 1.2, < 4) snaky_hash (~> 2.0) version_gem (~> 1.1) - oj (3.16.3) + oj (3.16.8) bigdecimal (>= 3.0) + ostruct (>= 0.2) omniauth (2.1.2) hashie (>= 3.4.6) rack (>= 2.2.3) @@ -137,46 +145,48 @@ GEM omniauth-oauth2 (1.8.0) oauth2 (>= 1.4, < 3) omniauth (~> 2.0) - optimist (3.1.0) - parallel (1.24.0) - parser (3.3.0.5) + optimist (3.2.0) + ostruct (0.6.1) + parallel (1.26.3) + parser (3.3.6.0) ast (~> 2.4.1) racc - public_suffix (5.0.4) - racc (1.7.3) - rack (3.0.10) - rack-protection (4.0.0) + public_suffix (6.0.1) + racc (1.8.1) + rack (3.1.8) + rack-protection (4.1.1) base64 (>= 0.1.0) + logger (>= 1.6.0) rack (>= 3.0.0, < 4) rainbow (3.1.1) rbs (2.8.4) - regexp_parser (2.9.0) + regexp_parser (2.10.0) rest-client (2.0.2) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) reverse_markdown (2.1.1) nokogiri - rexml (3.2.6) - rspotify (2.12.0) + rexml (3.4.0) + rspotify (2.12.4) addressable (~> 2.8.0) omniauth-oauth2 (>= 1.6) rest-client (~> 2.0.2) - rubocop (1.62.1) + rubocop (1.69.2) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.31.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.36.2, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.37.0) + parser (>= 3.3.1.0) ruby-progressbar (1.13.0) - semantic_logger (4.15.0) + securerandom (0.4.1) + semantic_logger (4.16.1) concurrent-ruby (~> 1.0) snaky_hash (2.0.1) hashie @@ -197,20 +207,21 @@ GEM thor (~> 1.0) tilt (~> 2.0) yard (~> 0.9, >= 0.9.24) - thor (1.3.1) - tilt (2.3.0) - timers (4.3.5) + thor (1.3.2) + tilt (2.5.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - unicode-categories (1.9.0) - unicode-display_width (2.5.0) - unicode-name (1.12.0) - unicode-types (~> 1.9) - unicode-sequence_name (1.13.0) - unicode-types (1.9.0) - uri (0.13.0) + unicode-categories (1.10.0) + unicode-display_width (3.1.3) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) + unicode-name (1.13.5) + unicode-types (~> 1.10) + unicode-sequence_name (1.15.3) + unicode-types (1.10.0) + uri (1.0.2) version_gem (1.1.4) - yard (0.9.36) + yard (0.9.37) PLATFORMS x86_64-linux From a081a149c07cc66737a52b7c08281dfa94599ac1 Mon Sep 17 00:00:00 2001 From: Mikkel Kroman Date: Fri, 27 Dec 2024 18:22:14 +0100 Subject: [PATCH 4/4] Bundle gems with more recent bundler version --- Dockerfile | 2 +- Gemfile.lock | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5bf28c2..7217817 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_IMAGE=ruby:3.3.0-alpine +ARG RUBY_IMAGE=ruby:3.4.1-alpine FROM ${RUBY_IMAGE} AS base diff --git a/Gemfile.lock b/Gemfile.lock index 8219aa3..0b2e01d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -126,8 +126,22 @@ GEM net-http (0.6.0) uri netrc (0.11.0) + nokogiri (1.18.0-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.0-aarch64-linux-musl) + racc (~> 1.4) + nokogiri (1.18.0-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.0-arm-linux-musl) + racc (~> 1.4) + nokogiri (1.18.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.0-x86_64-darwin) + racc (~> 1.4) nokogiri (1.18.0-x86_64-linux-gnu) racc (~> 1.4) + nokogiri (1.18.0-x86_64-linux-musl) + racc (~> 1.4) oauth2 (2.0.9) faraday (>= 0.17.3, < 3.0) jwt (>= 1.0, < 3.0) @@ -224,7 +238,14 @@ GEM yard (0.9.37) PLATFORMS - x86_64-linux + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES activesupport (~> 7.1) @@ -248,4 +269,4 @@ DEPENDENCIES unicode-sequence_name (~> 1.13) BUNDLED WITH - 2.3.6 + 2.6.2