Merge pull request #366 from rusq/i228-emoji-users

Get info on emoji users

README.md

@@ -211,7 +211,12 @@ suggested a feature, helped to reproduce, or spent time chatting with me on
 the Telegram or Slack to help to understand the issue and tested the proposed
 solution.
 
-Also, I'd like to thank all those who made a donation to support the project:
+Also, I'd like to thank current sponsors:
+
+- [<img class="avatar avatar-user" src="https://avatars.githubusercontent.com/u/9138285?s=60&amp;v=4" width="30" height="30" alt="@malsatin">](https://github.com/malsatin) @malsatin
+
+And everyone who made a donation to support the project in the past and keep
+supporting the project:
 
 - Vivek R.
 - Fabian I.

cmd/slackdump/internal/diag/edge.go

@@ -3,6 +3,7 @@ package diag
 import (
 	"context"
 	"encoding/json"
+	"log/slog"
 	"os"
 
 	"github.com/rusq/slackdump/v3/auth"
@@ -49,12 +50,30 @@ func runEdge(ctx context.Context, cmd *base.Command, args []string) error {
 	defer cl.Close()
 	lg.Info("connected")
 
-	lg.Info("*** Search for Channels test ***")
-	channels, err := cl.SearchChannels(ctx, "")
-	if err != nil {
-		return err
+	// lg.Info("*** Search for Channels test ***")
+	// channels, err := cl.SearchChannels(ctx, "")
+	// if err != nil {
+	// 	return err
+	// }
+	// if err := save("channels.json", channels); err != nil {
+	// 	return err
+	// }
+
+	lg.Info("*** AdminEmojiList test ***")
+	var allEmoji edge.EmojiResult
+
+	var iter = 0
+	for res, err := range cl.AdminEmojiList(ctx) {
+		if err != nil {
+			return err
+		}
+		slog.Info("got emojis", "count", len(res.Emoji), "disabled", len(res.DisabledEmoji), "iter", iter)
+		iter++
+		allEmoji.Emoji = append(allEmoji.Emoji, res.Emoji...)
+		allEmoji.DisabledEmoji = append(allEmoji.DisabledEmoji, res.DisabledEmoji...)
 	}
-	if err := save("channels.json", channels); err != nil {
+
+	if err := save("emoji.json", allEmoji); err != nil {
 		return err
 	}
 

cmd/slackdump/internal/diag/tools.go

@@ -20,7 +20,7 @@ Tools command contains different tools, running which may be requested if you op
 	PrintFlags:  false,
 	RequireAuth: false,
 	Commands: []*base.Command{
-		// cmdEdge,
+		cmdEdge,
 		dmdEncrypt,
 		dmdEzTest,
 		dmdInfo,

cmd/slackdump/internal/emoji/assets/emoji.md

@@ -0,0 +1,64 @@
+# Emoji Command
+This command allows you to download all the custom emojis from the Slack
+workspace.
+
+There are two modes of operation:
+- **Standard**: download only the names and URLs of the custom emojis;
+- **Full**: Download all the custom emojis from the workspace.
+
+Full mode is approx. 2.3 times slower than the standard mode, but it provides
+more information about the custom emojis.
+
+In both modes:
+- aliases are skipped, as they just point to the main emoji;
+- emoji files and saves in the "emojis" directory within the archive directory
+  or ZIP file.
+
+
+## Standard Mode
+In this mode, the command uses the standard Slack API that returns a mapping
+of the custom emoji names to their URLs, including the standard Slack emojis.
+
+The output is a JSON file with the following structure:
+```json
+{
+  "emoji_name": "emoji_url",
+  // ...
+}
+```
+
+## Full Mode
+In this mode, the command uses Slack Client API to download all information
+about the custom emojis.  This includes:
+- the emoji name;
+- the URL of the emoji image;
+- the user display name of the user who created the emoji and their ID;
+- the date when the emoji was created;
+- it's aliases;
+- team ID;
+- user's avatar hash.
+
+NOTE: This API endpoint is not documented by Slack, and it's not guaranteed to
+be stable.  The command uses the undocumented API endpoint to download the
+information about the custom emojis.
+
+It is slower than the standard mode, but slackdump does it's best to do things
+in parallel to speed up the process.
+
+The output is a JSON file with the following structure:
+
+```json
+{
+  "emoji_name": {
+    "name": "emoji_name",
+    "url": "emoji_url",
+    "team": "team_id",
+    "user_id": "user_id",
+    "created": 1670466722,
+    "user_display_name": "user_name",
+    "aliases": ["alias1", "alias2"],
+    "avatar": "avatar_hash"
+  },
+  // ...
+}
+```

cmd/slackdump/internal/emoji/emoji.go

@@ -2,28 +2,41 @@ package emoji
 
 import (
 	"context"
+	_ "embed"
 	"fmt"
+	"io"
+	"log/slog"
+	"sync"
+	"time"
 
 	"github.com/rusq/fsadapter"
+	"github.com/schollz/progressbar/v3"
+
 	"github.com/rusq/slackdump/v3"
+	"github.com/rusq/slackdump/v3/auth"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/emoji/emojidl"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base"
+	"github.com/rusq/slackdump/v3/internal/edge"
 )
 
+//go:embed assets/emoji.md
+var emojiMD string
+
 var CmdEmoji = &base.Command{
 	Run:         run,
 	UsageLine:   "slackdump emoji [flags]",
-	Short:       "download workspace emojis",
-	Long:        "", // TODO: add long description
-	FlagMask:    cfg.OmitDownloadFlag | cfg.OmitConfigFlag,
+	Short:       "download custom workspace emojis",
+	Long:        emojiMD, // TODO: add long description
+	FlagMask:    cfg.OmitDownloadFlag | cfg.OmitConfigFlag | cfg.OmitChunkCacheFlag | cfg.OmitUserCacheFlag,
 	RequireAuth: true,
 	PrintFlags:  true,
 }
 
 type options struct {
 	ignoreErrors bool
+	full         bool
 }
 
 // emoji specific flags
@@ -34,6 +47,7 @@ var cmdFlags = options{
 func init() {
 	CmdEmoji.Wizard = wizard
 	CmdEmoji.Flag.BoolVar(&cmdFlags.ignoreErrors, "ignore-errors", true, "ignore download errors (skip failed emojis)")
+	CmdEmoji.Flag.BoolVar(&cmdFlags.full, "full", false, "fetch emojis using Edge API to get full emoji information, including usernames")
 }
 
 func run(ctx context.Context, cmd *base.Command, args []string) error {
@@ -43,13 +57,65 @@ func run(ctx context.Context, cmd *base.Command, args []string) error {
 	}
 	defer fsa.Close()
 
+	prov, err := auth.FromContext(ctx)
+	if err != nil {
+		base.SetExitStatus(base.SApplicationError)
+		return err
+	}
+
+	start := time.Now()
+	r, cl := statusReporter()
+	defer cl.Close()
+	if cmdFlags.full {
+		err = runEdge(ctx, fsa, prov, r)
+	} else {
+		err = runLegacy(ctx, fsa, r)
+	}
+	cl.Close()
+	if err != nil {
+		base.SetExitStatus(base.SApplicationError)
+		return err
+	}
+
+	slog.InfoContext(ctx, "Emojis downloaded", "dir", cfg.Output, "took", time.Since(start).String())
+	return nil
+}
+
+func statusReporter() (emojidl.StatusFunc, io.Closer) {
+	pb := progressbar.NewOptions(0,
+		progressbar.OptionSetDescription("Downloading emojis"),
+		progressbar.OptionClearOnFinish(),
+		progressbar.OptionShowCount(),
+	)
+	var once sync.Once
+	return func(name string, total, count int) {
+		once.Do(func() {
+			pb.ChangeMax(total)
+		})
+		pb.Add(1)
+	}, pb
+
+}
+
+func runLegacy(ctx context.Context, fsa fsadapter.FS, cb emojidl.StatusFunc) error {
 	sess, err := bootstrap.SlackdumpSession(ctx, slackdump.WithFilesystem(fsa))
 	if err != nil {
 		base.SetExitStatus(base.SApplicationError)
-		return fmt.Errorf("application error: %s", err)
+		return err
+	}
+
+	return emojidl.DlFS(ctx, sess, fsa, cmdFlags.ignoreErrors, cb)
+}
+
+func runEdge(ctx context.Context, fsa fsadapter.FS, prov auth.Provider, cb emojidl.StatusFunc) error {
+	sess, err := edge.New(ctx, prov)
+	if err != nil {
+		base.SetExitStatus(base.SApplicationError)
+		return err
 	}
+	defer sess.Close()
 
-	if err := emojidl.DlFS(ctx, sess, fsa, cmdFlags.ignoreErrors); err != nil {
+	if err := emojidl.DlEdgeFS(ctx, sess, fsa, cmdFlags.ignoreErrors, cb); err != nil {
 		base.SetExitStatus(base.SApplicationError)
 		return fmt.Errorf("application error: %s", err)
 	}