diff --git a/cmd/podsync/main.go b/cmd/podsync/main.go index 72816cd1..039a1494 100644 --- a/cmd/podsync/main.go +++ b/cmd/podsync/main.go @@ -13,13 +13,12 @@ import ( "github.com/robfig/cron/v3" log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" + "gopkg.in/natefinch/lumberjack.v2" "github.com/mxpv/podsync/pkg/config" "github.com/mxpv/podsync/pkg/db" "github.com/mxpv/podsync/pkg/fs" "github.com/mxpv/podsync/pkg/ytdl" - - "gopkg.in/natefinch/lumberjack.v2" ) type Opts struct { @@ -107,7 +106,7 @@ func main() { log.WithError(err).Fatal("failed to open database") } - storage, err := fs.NewLocal(cfg.Server.DataDir, cfg.Server.Hostname) + storage, err := fs.NewLocal(cfg.Server.DataDir) if err != nil { log.WithError(err).Fatal("failed to open storage") } @@ -178,7 +177,7 @@ func main() { }) // Run web server - srv := NewServer(cfg) + srv := NewServer(cfg, storage) group.Go(func() error { log.Infof("running listener at %s", srv.Addr) diff --git a/cmd/podsync/server.go b/cmd/podsync/server.go index 5ab4fc5c..2e49090a 100644 --- a/cmd/podsync/server.go +++ b/cmd/podsync/server.go @@ -13,24 +13,26 @@ type Server struct { http.Server } -func NewServer(cfg *config.Config) *Server { +func NewServer(cfg *config.Config, storage http.FileSystem) *Server { port := cfg.Server.Port if port == 0 { port = 8080 } + bindAddress := cfg.Server.BindAddress if bindAddress == "*" { bindAddress = "" } + srv := Server{} srv.Addr = fmt.Sprintf("%s:%d", bindAddress, port) log.Debugf("using address: %s:%s", bindAddress, srv.Addr) - fs := http.FileServer(http.Dir(cfg.Server.DataDir)) - path := cfg.Server.Path - http.Handle(fmt.Sprintf("/%s", path), fs) - log.Debugf("handle path: /%s", path) + fileServer := http.FileServer(storage) + + log.Debugf("handle path: /%s", cfg.Server.Path) + http.Handle(fmt.Sprintf("/%s", cfg.Server.Path), fileServer) return &srv } diff --git a/cmd/podsync/updater.go b/cmd/podsync/updater.go index a548ea28..54bdf9ce 100644 --- a/cmd/podsync/updater.go +++ b/cmd/podsync/updater.go @@ -203,7 +203,7 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed) // Limit the number of episodes downloaded at once pageSize-- - if pageSize <= 0 { + if pageSize < 0 { return nil } @@ -235,7 +235,7 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed) ) // Check whether episode already exists - size, err := u.fs.Size(ctx, feedID, episodeName) + size, err := fs.Size(u.fs, fmt.Sprintf("%s/%s", feedID, episodeName)) if err == nil { logger.Infof("episode %q already exists on disk", episode.ID) @@ -283,7 +283,7 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed) } logger.Debug("copying file") - fileSize, err := u.fs.Create(ctx, feedID, episodeName, tempFile) + fileSize, err := u.fs.Create(ctx, fmt.Sprintf("%s/%s", feedID, episodeName), tempFile) tempFile.Close() if err != nil { logger.WithError(err).Error("failed to copy file") @@ -316,7 +316,7 @@ func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error { // Build iTunes XML feed with data received from builder log.Debug("building iTunes podcast feed") - podcast, err := feed.Build(ctx, f, feedConfig, u.fs) + podcast, err := feed.Build(ctx, f, feedConfig, u.config.Server.Hostname) if err != nil { return err } @@ -326,7 +326,7 @@ func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error { xmlName = fmt.Sprintf("%s.xml", feedConfig.ID) ) - if _, err := u.fs.Create(ctx, "", xmlName, reader); err != nil { + if _, err := u.fs.Create(ctx, xmlName, reader); err != nil { return errors.Wrap(err, "failed to upload new XML feed") } @@ -336,7 +336,7 @@ func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error { func (u *Updater) buildOPML(ctx context.Context) error { // Build OPML with data received from builder log.Debug("building podcast OPML") - opml, err := feed.BuildOPML(ctx, u.config, u.db, u.fs) + opml, err := feed.BuildOPML(ctx, u.config, u.db, u.config.Server.Hostname) if err != nil { return err } @@ -346,7 +346,7 @@ func (u *Updater) buildOPML(ctx context.Context) error { xmlName = fmt.Sprintf("%s.opml", "podsync") ) - if _, err := u.fs.Create(ctx, "", xmlName, reader); err != nil { + if _, err := u.fs.Create(ctx, xmlName, reader); err != nil { return errors.Wrap(err, "failed to upload OPML") } @@ -388,7 +388,12 @@ func (u *Updater) cleanup(ctx context.Context, feedConfig *config.Feed) error { for _, episode := range list[count:] { logger.WithField("episode_id", episode.ID).Infof("deleting %q", episode.Title) - if err := u.fs.Delete(ctx, feedConfig.ID, feed.EpisodeName(feedConfig, episode)); err != nil { + var ( + episodeName = feed.EpisodeName(feedConfig, episode) + path = fmt.Sprintf("%s/%s", feedConfig.ID, episodeName) + ) + + if err := u.fs.Delete(ctx, path); err != nil { result = multierror.Append(result, errors.Wrapf(err, "failed to delete episode: %s", episode.ID)) continue } diff --git a/pkg/feed/deps.go b/pkg/feed/deps.go index 4c568aa9..df47b070 100644 --- a/pkg/feed/deps.go +++ b/pkg/feed/deps.go @@ -11,7 +11,3 @@ import ( type feedProvider interface { GetFeed(ctx context.Context, feedID string) (*model.Feed, error) } - -type urlProvider interface { - URL(ctx context.Context, ns string, fileName string) (string, error) -} diff --git a/pkg/feed/deps_mock_test.go b/pkg/feed/deps_mock_test.go index e7dec7d1..c6771bf6 100644 --- a/pkg/feed/deps_mock_test.go +++ b/pkg/feed/deps_mock_test.go @@ -6,35 +6,36 @@ package feed import ( context "context" + reflect "reflect" + gomock "github.com/golang/mock/gomock" model "github.com/mxpv/podsync/pkg/model" - reflect "reflect" ) -// MockfeedProvider is a mock of feedProvider interface +// MockfeedProvider is a mock of feedProvider interface. type MockfeedProvider struct { ctrl *gomock.Controller recorder *MockfeedProviderMockRecorder } -// MockfeedProviderMockRecorder is the mock recorder for MockfeedProvider +// MockfeedProviderMockRecorder is the mock recorder for MockfeedProvider. type MockfeedProviderMockRecorder struct { mock *MockfeedProvider } -// NewMockfeedProvider creates a new mock instance +// NewMockfeedProvider creates a new mock instance. func NewMockfeedProvider(ctrl *gomock.Controller) *MockfeedProvider { mock := &MockfeedProvider{ctrl: ctrl} mock.recorder = &MockfeedProviderMockRecorder{mock} return mock } -// EXPECT returns an object that allows the caller to indicate expected use +// EXPECT returns an object that allows the caller to indicate expected use. func (m *MockfeedProvider) EXPECT() *MockfeedProviderMockRecorder { return m.recorder } -// GetFeed mocks base method +// GetFeed mocks base method. func (m *MockfeedProvider) GetFeed(ctx context.Context, feedID string) (*model.Feed, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetFeed", ctx, feedID) @@ -43,46 +44,8 @@ func (m *MockfeedProvider) GetFeed(ctx context.Context, feedID string) (*model.F return ret0, ret1 } -// GetFeed indicates an expected call of GetFeed +// GetFeed indicates an expected call of GetFeed. func (mr *MockfeedProviderMockRecorder) GetFeed(ctx, feedID interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeed", reflect.TypeOf((*MockfeedProvider)(nil).GetFeed), ctx, feedID) } - -// MockurlProvider is a mock of urlProvider interface -type MockurlProvider struct { - ctrl *gomock.Controller - recorder *MockurlProviderMockRecorder -} - -// MockurlProviderMockRecorder is the mock recorder for MockurlProvider -type MockurlProviderMockRecorder struct { - mock *MockurlProvider -} - -// NewMockurlProvider creates a new mock instance -func NewMockurlProvider(ctrl *gomock.Controller) *MockurlProvider { - mock := &MockurlProvider{ctrl: ctrl} - mock.recorder = &MockurlProviderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockurlProvider) EXPECT() *MockurlProviderMockRecorder { - return m.recorder -} - -// URL mocks base method -func (m *MockurlProvider) URL(ctx context.Context, ns, fileName string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "URL", ctx, ns, fileName) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// URL indicates an expected call of URL -func (mr *MockurlProviderMockRecorder) URL(ctx, ns, fileName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "URL", reflect.TypeOf((*MockurlProvider)(nil).URL), ctx, ns, fileName) -} diff --git a/pkg/feed/opml.go b/pkg/feed/opml.go index c98aee00..597b5e47 100644 --- a/pkg/feed/opml.go +++ b/pkg/feed/opml.go @@ -3,6 +3,7 @@ package feed import ( "context" "fmt" + "strings" "github.com/gilliek/go-opml/opml" "github.com/pkg/errors" @@ -12,7 +13,7 @@ import ( "github.com/mxpv/podsync/pkg/model" ) -func BuildOPML(ctx context.Context, config *config.Config, db feedProvider, provider urlProvider) (string, error) { +func BuildOPML(ctx context.Context, config *config.Config, db feedProvider, hostname string) (string, error) { doc := opml.OPML{Version: "1.0"} doc.Head = opml.Head{Title: "Podsync feeds"} doc.Body = opml.Body{} @@ -31,16 +32,11 @@ func BuildOPML(ctx context.Context, config *config.Config, db feedProvider, prov continue } - downloadURL, err := provider.URL(ctx, "", fmt.Sprintf("%s.xml", feed.ID)) - if err != nil { - return "", errors.Wrapf(err, "failed to get feed URL for %q", feed.ID) - } - outline := opml.Outline{ Title: f.Title, Text: f.Description, Type: "rss", - XMLURL: downloadURL, + XMLURL: fmt.Sprintf("%s/%s.xml", strings.TrimRight(hostname, "/"), feed.ID), } doc.Body.Outlines = append(doc.Body.Outlines, outline) diff --git a/pkg/feed/opml_test.go b/pkg/feed/opml_test.go index 1697676c..d5d4ab7d 100644 --- a/pkg/feed/opml_test.go +++ b/pkg/feed/opml_test.go @@ -25,9 +25,6 @@ func TestBuildOPML(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - urlMock := NewMockurlProvider(ctrl) - urlMock.EXPECT().URL(gomock.Any(), "", "1.xml").Return("https://url/1.xml", nil) - dbMock := NewMockfeedProvider(ctrl) dbMock.EXPECT().GetFeed(gomock.Any(), "1").Return(&model.Feed{Title: "1", Description: "desc"}, nil) @@ -37,7 +34,7 @@ func TestBuildOPML(t *testing.T) { }, } - out, err := BuildOPML(context.Background(), &cfg, dbMock, urlMock) + out, err := BuildOPML(context.Background(), &cfg, dbMock, "https://url/") assert.NoError(t, err) assert.Equal(t, expected, out) } diff --git a/pkg/feed/xml.go b/pkg/feed/xml.go index 40769e36..ac05312f 100644 --- a/pkg/feed/xml.go +++ b/pkg/feed/xml.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "strconv" + "strings" "time" itunes "github.com/eduncan911/podcast" @@ -30,7 +31,7 @@ func (p timeSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func Build(ctx context.Context, feed *model.Feed, cfg *config.Feed, provider urlProvider) (*itunes.Podcast, error) { +func Build(_ctx context.Context, feed *model.Feed, cfg *config.Feed, hostname string) (*itunes.Podcast, error) { const ( podsyncGenerator = "Podsync generator (support us at https://github.com/mxpv/podsync)" defaultCategory = "TV & Film" @@ -125,11 +126,10 @@ func Build(ctx context.Context, feed *model.Feed, cfg *config.Feed, provider url enclosureType = itunes.MP3 } - episodeName := EpisodeName(cfg, episode) - downloadURL, err := provider.URL(ctx, cfg.ID, episodeName) - if err != nil { - return nil, errors.Wrapf(err, "failed to obtain download URL for: %s", episodeName) - } + var ( + episodeName = EpisodeName(cfg, episode) + downloadURL = fmt.Sprintf("%s/%s/%s", strings.TrimRight(hostname, "/"), cfg.ID, episodeName) + ) item.AddEnclosure(downloadURL, enclosureType, episode.Size) diff --git a/pkg/feed/xml_test.go b/pkg/feed/xml_test.go index 39e60817..eba47237 100644 --- a/pkg/feed/xml_test.go +++ b/pkg/feed/xml_test.go @@ -4,27 +4,31 @@ import ( "context" "testing" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - + itunes "github.com/eduncan911/podcast" "github.com/mxpv/podsync/pkg/config" "github.com/mxpv/podsync/pkg/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestBuildXML(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - urlMock := NewMockurlProvider(ctrl) - - feed := model.Feed{} + feed := model.Feed{ + Episodes: []*model.Episode{ + { + ID: "1", + Status: model.EpisodeDownloaded, + Title: "title", + Description: "description", + }, + }, + } cfg := config.Feed{ + ID: "test", Custom: config.Custom{Description: "description", Category: "Technology", Subcategories: []string{"Gadgets", "Podcasting"}}, } - out, err := Build(context.Background(), &feed, &cfg, urlMock) + out, err := Build(context.Background(), &feed, &cfg, "http://localhost/") assert.NoError(t, err) assert.EqualValues(t, "description", out.Description) @@ -33,7 +37,13 @@ func TestBuildXML(t *testing.T) { require.Len(t, out.ICategories, 1) category := out.ICategories[0] assert.EqualValues(t, "Technology", category.Text) + require.Len(t, category.ICategories, 2) assert.EqualValues(t, "Gadgets", category.ICategories[0].Text) assert.EqualValues(t, "Podcasting", category.ICategories[1].Text) + + require.Len(t, out.Items, 1) + require.NotNil(t, out.Items[0].Enclosure) + assert.EqualValues(t, out.Items[0].Enclosure.URL, "http://localhost/test/1.mp4") + assert.EqualValues(t, out.Items[0].Enclosure.Type, itunes.MP4) } diff --git a/pkg/fs/local.go b/pkg/fs/local.go index 12d58275..8a7734f9 100644 --- a/pkg/fs/local.go +++ b/pkg/fs/local.go @@ -2,91 +2,54 @@ package fs import ( "context" - "fmt" "io" + "net/http" "os" "path/filepath" - "strings" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) +// Local implements local file storage type Local struct { - hostname string - rootDir string + rootDir string } -func NewLocal(rootDir string, hostname string) (*Local, error) { - if hostname == "" { - return nil, errors.New("hostname can't be empty") - } +func NewLocal(rootDir string) (*Local, error) { + return &Local{rootDir: rootDir}, nil +} - hostname = strings.TrimSuffix(hostname, "/") - if !strings.HasPrefix(hostname, "http") { - hostname = fmt.Sprintf("http://%s", hostname) - } +func (l *Local) Open(name string) (http.File, error) { + path := filepath.Join(l.rootDir, name) + return os.Open(path) +} - return &Local{rootDir: rootDir, hostname: hostname}, nil +func (l *Local) Delete(_ctx context.Context, name string) error { + path := filepath.Join(l.rootDir, name) + return os.Remove(path) } -func (l *Local) Create(ctx context.Context, ns string, fileName string, reader io.Reader) (int64, error) { +func (l *Local) Create(_ctx context.Context, name string, reader io.Reader) (int64, error) { var ( - logger = log.WithField("episode_id", fileName) - feedDir = filepath.Join(l.rootDir, ns) + logger = log.WithField("name", name) + path = filepath.Join(l.rootDir, name) ) - if err := os.MkdirAll(feedDir, 0755); err != nil { - return 0, errors.Wrapf(err, "failed to create a directory for the feed: %s", feedDir) - } - - logger.Debugf("creating directory: %s", feedDir) - if err := os.MkdirAll(feedDir, 0755); err != nil { - return 0, errors.Wrapf(err, "failed to create feed dir: %s", feedDir) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return 0, errors.Wrapf(err, "failed to mkdir: %s", path) } - var ( - episodePath = filepath.Join(l.rootDir, ns, fileName) - ) - - logger.Debugf("copying to: %s", episodePath) - written, err := l.copyFile(reader, episodePath) + logger.Infof("creating file: %s", path) + written, err := l.copyFile(reader, path) if err != nil { return 0, errors.Wrap(err, "failed to copy file") } - logger.Debugf("copied %d bytes", written) + logger.Debugf("written %d bytes", written) return written, nil } -func (l *Local) Delete(ctx context.Context, ns string, fileName string) error { - path := filepath.Join(l.rootDir, ns, fileName) - return os.Remove(path) -} - -func (l *Local) Size(ctx context.Context, ns string, fileName string) (int64, error) { - path := filepath.Join(l.rootDir, ns, fileName) - - stat, err := os.Stat(path) - if err == nil { - return stat.Size(), nil - } - - return 0, err -} - -func (l *Local) URL(ctx context.Context, ns string, fileName string) (string, error) { - if _, err := l.Size(ctx, ns, fileName); err != nil { - return "", errors.Wrap(err, "failed to check whether file exists") - } - - if ns == "" { - return fmt.Sprintf("%s/%s", l.hostname, fileName), nil - } - - return fmt.Sprintf("%s/%s/%s", l.hostname, ns, fileName), nil -} - func (l *Local) copyFile(source io.Reader, destinationPath string) (int64, error) { dest, err := os.Create(destinationPath) if err != nil { diff --git a/pkg/fs/local_test.go b/pkg/fs/local_test.go index 33a4135a..67bcb092 100644 --- a/pkg/fs/local_test.go +++ b/pkg/fs/local_test.go @@ -17,13 +17,9 @@ var ( ) func TestNewLocal(t *testing.T) { - local, err := NewLocal("", "localhost") + local, err := NewLocal("") assert.NoError(t, err) - assert.Equal(t, "http://localhost", local.hostname) - - local, err = NewLocal("", "https://localhost:8080/") - assert.NoError(t, err) - assert.Equal(t, "https://localhost:8080", local.hostname) + assert.NotNil(t, local) } func TestLocal_Create(t *testing.T) { @@ -32,10 +28,10 @@ func TestLocal_Create(t *testing.T) { defer os.RemoveAll(tmpDir) - stor, err := NewLocal(tmpDir, "localhost") + stor, err := NewLocal(tmpDir) assert.NoError(t, err) - written, err := stor.Create(testCtx, "1", "test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) + written, err := stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) assert.NoError(t, err) assert.EqualValues(t, 5, written) @@ -50,22 +46,22 @@ func TestLocal_Size(t *testing.T) { defer os.RemoveAll(tmpDir) - stor, err := NewLocal(tmpDir, "localhost") + stor, err := NewLocal(tmpDir) assert.NoError(t, err) - _, err = stor.Create(testCtx, "1", "test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) + _, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) assert.NoError(t, err) - sz, err := stor.Size(testCtx, "1", "test") + sz, err := Size(stor, "1/test") assert.NoError(t, err) assert.EqualValues(t, 5, sz) } func TestLocal_NoSize(t *testing.T) { - stor, err := NewLocal("", "localhost") + stor, err := NewLocal("") assert.NoError(t, err) - _, err = stor.Size(testCtx, "1", "test") + _, err = Size(stor, "1/test") assert.True(t, os.IsNotExist(err)) } @@ -75,39 +71,22 @@ func TestLocal_Delete(t *testing.T) { defer os.RemoveAll(tmpDir) - stor, err := NewLocal(tmpDir, "localhost") + stor, err := NewLocal(tmpDir) assert.NoError(t, err) - _, err = stor.Create(testCtx, "1", "test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) + _, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) assert.NoError(t, err) - err = stor.Delete(testCtx, "1", "test") + err = stor.Delete(testCtx, "1/test") assert.NoError(t, err) - _, err = stor.Size(testCtx, "1", "test") + _, err = Size(stor, "1/test") assert.True(t, os.IsNotExist(err)) _, err = os.Stat(filepath.Join(tmpDir, "1", "test")) assert.True(t, os.IsNotExist(err)) } -func TestLocal_URL(t *testing.T) { - tmpDir, err := ioutil.TempDir("", "podsync-local-stor-") - require.NoError(t, err) - - defer os.RemoveAll(tmpDir) - - stor, err := NewLocal(tmpDir, "localhost") - assert.NoError(t, err) - - _, err = stor.Create(testCtx, "1", "test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) - assert.NoError(t, err) - - url, err := stor.URL(testCtx, "1", "test") - assert.NoError(t, err) - assert.EqualValues(t, "http://localhost/1/test", url) -} - func TestLocal_copyFile(t *testing.T) { reader := bytes.NewReader([]byte{1, 2, 4}) diff --git a/pkg/fs/storage.go b/pkg/fs/storage.go index a157bcf2..65402940 100644 --- a/pkg/fs/storage.go +++ b/pkg/fs/storage.go @@ -3,18 +3,33 @@ package fs import ( "context" "io" + "net/http" ) +// Storage is a file system interface to host downloaded episodes and feeds. type Storage interface { + // FileSystem must be implemented to in order to pass Storage interface to HTTP file server. + http.FileSystem + // Create will create a new file from reader - Create(ctx context.Context, ns string, fileName string, reader io.Reader) (int64, error) + Create(ctx context.Context, name string, reader io.Reader) (int64, error) // Delete deletes the file - Delete(ctx context.Context, ns string, fileName string) error + Delete(ctx context.Context, name string) error +} + +// Size returns storage object's size in bytes. +func Size(storage http.FileSystem, name string) (int64, error) { + file, err := storage.Open(name) + if err != nil { + return 0, err + } + defer file.Close() - // Size returns the size of a file in bytes - Size(ctx context.Context, ns string, fileName string) (int64, error) + stat, err := file.Stat() + if err != nil { + return 0, err + } - // URL will generate a download link for a file - URL(ctx context.Context, ns string, fileName string) (string, error) + return stat.Size(), nil }