Have NetKAN bot collect download counts

lib/App/KSP_CKAN/DownloadCounts.pm

@@ -0,0 +1,203 @@
+package App::KSP_CKAN::DownloadCounts;
+
+use v5.010;
+use strict;
+use warnings;
+use autodie;
+use Method::Signatures 20140224;
+use Carp qw( croak );
+use File::chdir;
+use File::Slurper qw(read_text write_text);
+use File::Basename qw(basename);
+use Try::Tiny;
+use JSON;
+use File::Path qw(mkpath);
+use App::KSP_CKAN::Tools::Git;
+use Moo;
+use namespace::clean;
+
+# ABSTRACT: NetKAN Download Counter
+
+# VERSION: Generated by DZP::OurPkg:Version
+# (Actually generated by copying Status.pm)
+
+=head1 SYNOPSIS
+
+  use App::KSP_CKAN::DownloadCounts;
+
+  my $status = App::KSP_CKAN::DownloadCounts->new(
+    config => $config,
+  );
+
+=head1 DESCRIPTION
+
+Gets download counts from public hosting APIs and exports them to a JSON file.
+
+=cut
+
+my $Ref = sub {
+  croak("auth isn't a 'App::KSP_CKAN::Tools::Config' object!") unless $_[0]->DOES("App::KSP_CKAN::Tools::Config");
+};
+
+my $Meta = sub {
+  croak("ckan-meta isn't a 'App::KSP_CKAN::Tools::Git' object!") unless $_[0]->DOES("App::KSP_CKAN::Tools::Git");
+};
+
+has 'config'        => ( is => 'ro', required => 1, isa => $Ref );
+has 'ckan_meta'     => ( is => 'ro', required => 1, isa => $Meta );
+
+has '_http'         => ( is => 'ro', lazy => 1, builder => 1 );
+has '_data'         => ( is => 'ro', lazy => 1, builder => 1 );
+has '_json'         => ( is => 'ro', lazy => 1, builder => 1 );
+has '_NetKAN'       => ( is => 'ro', lazy => 1, builder => 1 );
+has '_output_file'  => ( is => 'ro', lazy => 1, builder => 1 );
+
+method _build__http {
+  return HTTP::Tiny->new(timeout => 15);
+}
+
+method _build__json {
+  return JSON->new->allow_blessed(1)->convert_blessed(1);
+}
+
+method _build__NetKAN {
+  return App::KSP_CKAN::Tools::Git->new(
+    remote => $self->config->NetKAN,
+    local  => $self->config->working,
+    clean  => 1,
+  );
+}
+
+method _build__output_file {
+  return $self->config->working
+    . '/' . $self->ckan_meta->working
+    . '/download_counts.json';
+}
+
+method _build__data {
+  return $self->_json->decode('{}');
+}
+
+method _get_count_from_spacedock($id) {
+  try {
+    my $sd_json = $self->_json->decode(
+      $self->_http->get("https://spacedock.info/api/mod/$id")->{content}
+    );
+    return $sd_json->{downloads};
+  };
+}
+
+method _get_count_from_github($id) {
+  if (my ($user, $proj) = $id =~ m{^([^/]+)/([^/]+)}) {
+    try {
+      my $token   = $self->config->GH_token;
+      my %headers = ( 'Authorization' => "token $token" );
+      my %options = ( 'headers'       => \%headers      );
+      my $gh_json = $self->_json->decode(
+        $self->_http->get("https://api.github.com/repos/$user/$proj/releases", \%options)->{content}
+      );
+      my $sum = 0;
+      foreach my $rel (@{$gh_json}) {
+        foreach my $asset (@{$rel->{assets}}) {
+          if (defined($asset->{download_count})) {
+            $sum += $asset->{download_count};
+          }
+        }
+      }
+      return $sum;
+    };
+  }
+}
+
+method _get_count_from_curse($id) {
+  try {
+    my $curse_json = $self->_json->decode(
+      $self->_http->get(
+        ($id =~ m{^\d+$})
+          # Numeric ID, use old URL format
+          ? "https://api.cfwidget.com/project/$id"
+          # Non numeric ID, use new URL format
+          : "https://api.cfwidget.com/kerbal/ksp-mods/$id"
+      )->{content}
+    );
+    return $curse_json->{downloads}->{total};
+  };
+}
+
+method _get_count_from_netkan_json($netkan_json) {
+  my $kref = $netkan_json->{'$kref'};
+  return unless defined($kref);
+  if (my ($kref_kind, $kref_id) = $kref =~ m{^#/ckan/([^/]+)/(.+)}) {
+    if ($kref_kind eq 'netkan') {
+      return $self->_get_count_from_url($kref_id);
+    } elsif ($kref_kind eq 'spacedock') {
+      return $self->_get_count_from_spacedock($kref_id);
+    } elsif ($kref_kind eq 'github') {
+      return $self->_get_count_from_github($kref_id);
+    } elsif ($kref_kind eq 'curse') {
+      return $self->_get_count_from_curse($kref_id);
+    }
+  }
+}
+
+method _get_count_from_url($url) {
+  try {
+    return $self->_get_count_from_netkan_json(
+      $self->_json->decode(
+        $self->_http->get($url)->{content}
+      )
+    );
+  };
+}
+
+method _get_count_from_file($file) {
+  try {
+    return $self->_get_count_from_netkan_json(
+      $self->_json->decode(
+        read_text($file)
+      )
+    );
+  };
+}
+
+=method get_counts
+
+  $download_counts->get_counts
+
+Gets download counts from public hosting APIs.
+
+=cut
+
+method get_counts {
+  # Chdir to NetKAN repo root
+  local $CWD = $self->config->working . '/' . $self->_NetKAN->working;
+  # Loop over the .netkan files
+  foreach my $file (glob('NetKAN/*.netkan')) {
+    # Get this module's download count
+    my $count = $self->_get_count_from_file($file);
+    next unless defined($count);
+    if ($count > 0) {
+      my $identifier = basename($file, '.netkan');
+      $self->_data->{$identifier} = $count;
+    }
+  }
+}
+
+=method write_json
+
+  download_counts->write_json
+
+Writes our download counts file out to disk.
+
+=cut
+
+method write_json {
+  write_text($self->_output_file, $self->_json->encode($self->_data));
+  $self->ckan_meta->add($self->_output_file);
+  $self->ckan_meta->commit(
+    file    => $self->_output_file,
+    message => "NetKAN updated download counts",
+  );
+}
+
+1;

lib/App/KSP_CKAN/NetKAN.pm

@@ -8,6 +8,7 @@ use Method::Signatures 20140224;
 use File::chdir;
 use Carp qw( croak );
 use App::KSP_CKAN::Status;
+use App::KSP_CKAN::DownloadCounts;
 use App::KSP_CKAN::Tools::Http;
 use App::KSP_CKAN::Tools::Git;
 use App::KSP_CKAN::Tools::NetKAN;
@@ -115,6 +116,18 @@ method _inflate_all(:$rescan = 1) {
   return;
 }
 
+# Calculate the download counts and save them to CKAN-meta/download_counts.json
+# Works for mods with a $kref on Curse, GitHub, or SpaceDock.
+# Expected to take a few minutes.
+method _update_download_counts() {
+  my $counter = App::KSP_CKAN::DownloadCounts->new(
+    config    => $self->config,
+    ckan_meta => $self->_CKAN_meta,
+  );
+  $counter->get_counts;
+  $counter->write_json;
+}
+
 method _push {
   $self->_CKAN_meta->pull(ours => 1);
   $self->_CKAN_meta->push;
@@ -131,6 +144,7 @@ it into CKAN-meta (or whichever repository is configured)
 method full_index {
   $self->_mirror_files;
   $self->_inflate_all;
+  $self->_update_download_counts;
   if ( ! $self->is_debug() ) {
     $self->_push;
     $self->_status->write_json;