* Make project to be go module

  * Make code compilable
* Fix README.md examples to be runnable
* Fix all open file and create directory permissions

.gitignore

@@ -20,3 +20,4 @@ debug/
 test/
 MQL4/
 go-duka
+go-duka.exe

README.md

@@ -18,14 +18,14 @@ go build
 Get January (2019) data for EURUSD and convert to FXT/MT4
 
 ```
-./go-duka -symbol EURUSD -format fxt -start "2018-01-01" -end "2008-01-31"
+./go-duka -symbol EURUSD -format fxt -start "2018-01-01" -end "2018-01-31"
 ```
 
 Get 2018 data for XAUUSD and convert to FXT, and then to HST
 
 ```
-./go-duka -symbol EURUSD -format fxt -start "2018-01-01" -end "2008-12-31"
-./go-duka -symbol EURUSD -format hst -start "2018-01-01" -end "2008-12-31"
+./go-duka -symbol EURUSD -format fxt -start "2018-01-01" -end "2018-12-31"
+./go-duka -symbol EURUSD -format hst -start "2018-01-01" -end "2018-12-31"
 ```
 
 

dukapp.go → app/dukapp.go

@@ -1,28 +1,43 @@
-package main
+package app
 
 import (
-	"errors"
 	"fmt"
+	"github.com/pkg/errors"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 
-	"./bi5"
-	"./core"
-	"./csv"
-	"./fxt4"
-	"./hst"
-	"./misc"
+	"ed-fx/go-duka/bi5"
+	"ed-fx/go-duka/core"
+	"ed-fx/go-duka/csv"
+	"ed-fx/go-duka/fxt4"
+	"ed-fx/go-duka/hst"
+	"ed-fx/go-duka/misc"
 )
 
 var (
 	log             = misc.NewLogger("App", 2)
 	supportsFormats = []string{"csv", "fxt", "hst"}
 )
 
+type ArgsList struct {
+	Verbose bool
+	Header  bool
+	Local   bool
+	Spread  uint
+	Model   uint
+	Dump    string
+	Symbol  string
+	Output  string
+	Format  string
+	Period  string
+	Start   string
+	End     string
+}
+
 // DukaApp used to download source tick data
 //
 type DukaApp struct {
@@ -47,7 +62,7 @@ type AppOption struct {
 
 // ParseOption parse input command line
 //
-func ParseOption(args argsList) (*AppOption, error) {
+func ParseOption(args ArgsList) (*AppOption, error) {
 	var err error
 	opt := AppOption{
 		CsvHeader: args.Header,
@@ -77,16 +92,7 @@ func ParseOption(args argsList) (*AppOption, error) {
 		}
 		opt.Format = format
 	}
-	if opt.Start, err = time.ParseInLocation("2006-01-02", args.Start, time.UTC); err != nil {
-		err = fmt.Errorf("invalid start parameter")
-		return nil, err
-	}
-	if opt.End, err = time.ParseInLocation("2006-01-02", args.End, time.UTC); err != nil {
-		err = fmt.Errorf("invalid end parameter")
-		return nil, nil
-	}
-	if opt.End.Unix() <= opt.Start.Unix() {
-		err = fmt.Errorf("invalid end parameter which shouldn't early then start")
+	if err = handleTimeArguments(args, &opt); err != nil {
 		return nil, err
 	}
 	if opt.Folder, err = filepath.Abs(args.Output); err != nil {
@@ -110,6 +116,22 @@ func ParseOption(args argsList) (*AppOption, error) {
 	return &opt, nil
 }
 
+func handleTimeArguments(args ArgsList, opt *AppOption) (err error) {
+	if opt.Start, err = time.ParseInLocation("2006-01-02", args.Start, time.UTC); err != nil {
+		err = errors.Wrap(err, "invalid start parameter")
+		return
+	}
+	if opt.End, err = time.ParseInLocation("2006-01-02", args.End, time.UTC); err != nil {
+		err = fmt.Errorf("invalid end parameter")
+		return
+	}
+	if opt.End.Before(opt.Start) || opt.End.Equal(opt.Start) {
+		err = fmt.Errorf("invalid end parameter which shouldn't early then start")
+		return
+	}
+	return
+}
+
 // NewOutputs create timeframe instance
 //
 func NewOutputs(opt *AppOption) []core.Converter {
@@ -172,7 +194,7 @@ func (app *DukaApp) Execute() error {
 	// Create an output directory
 	//
 	if _, err := os.Stat(opt.Folder); os.IsNotExist(err) {
-		if err = os.MkdirAll(opt.Folder, 666); err != nil {
+		if err = os.MkdirAll(opt.Folder, 0770); err != nil {
 			log.Error("Create folder (%s) failed: %v.", opt.Folder, err)
 			return err
 		}

dukapp_test.go → app/dukapp_test.go

@@ -1,20 +1,19 @@
-package main
+package app
 
 import (
 	"fmt"
 	"testing"
 
-	clog "github.com/go-clog/clog"
+	"github.com/go-clog/clog"
 )
 
 func TestDukaApp(t *testing.T) {
-	args := argsList{
+	args := ArgsList{
 		Verbose: true,
 		Header:  true,
 		Spread:  20,
 		Model:   0,
 		Symbol:  "EURUSD",
-		Output:  "f:\\00",
 		Format:  "csv",
 		Period:  "M1",
 		Start:   "2017-01-01",
@@ -31,7 +30,7 @@ func TestDukaApp(t *testing.T) {
 	fmt.Printf("    Symbol: %s\n", opt.Symbol)
 	fmt.Printf("    Spread: %d\n", opt.Spread)
 	fmt.Printf("      Mode: %d\n", opt.Mode)
-	fmt.Printf(" Timeframe: %d\n", opt.Timeframe)
+	//fmt.Printf(" Timeframe: %d\n", opt.Timeframe)
 	fmt.Printf("    Format: %s\n", opt.Format)
 	fmt.Printf(" CsvHeader: %t\n", opt.CsvHeader)
 	fmt.Printf(" StartDate: %s\n", opt.Start.Format("2006-01-02:15H"))

bi5/bi5.go

@@ -11,8 +11,8 @@ import (
 	"path/filepath"
 	"time"
 
-	"../core"
-	"../misc"
+	"ed-fx/go-duka/core"
+	"ed-fx/go-duka/misc"
 	"github.com/kjk/lzma"
 )
 
@@ -39,7 +39,7 @@ type Bi5 struct {
 // New create an bi5 saver
 func New(day time.Time, symbol, dest string) *Bi5 {
 	y, m, d := day.Date()
-	dir := fmt.Sprintf("%s/%04d/%02d/%02d", symbol, y, m-1, d)
+	dir := fmt.Sprintf("%s/%04d/%02d/%02d", symbol, y, m, d)
 
 	return &Bi5{
 		dest:   filepath.Join(dest, dir),
@@ -142,7 +142,7 @@ func (b *Bi5) Download() ([]byte, error) {
 
 	year, month, day := b.dayH.Date()
 	// !! 注意: month - 1
-	link := fmt.Sprintf(core.DukaTmplURL, b.symbol, year, month-1, day, b.dayH.Hour())
+	link := fmt.Sprintf(core.DukaTmplURL, b.symbol, year, month, day, b.dayH.Hour())
 
 	if data, err = httpDownld.Download(link); err != nil {
 		log.Error("%s %s download failed: %v.", b.symbol, b.dayH.Format("2006-01-02:15H"), err)

core/fetch.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 	"time"
 
-	"../misc"
+	"ed-fx/go-duka/misc"
 )
 
 const (

csv/csv.go

@@ -7,8 +7,8 @@ import (
 	"path/filepath"
 	"time"
 
-	"../core"
-	"../misc"
+	"ed-fx/go-duka/core"
+	"ed-fx/go-duka/misc"
 )
 
 var (
@@ -76,7 +76,7 @@ func (c *CsvDump) worker() error {
 		ext)
 
 	fpath := filepath.Join(c.dest, fname)
-	f, err := os.OpenFile(fpath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 666)
+	f, err := os.OpenFile(fpath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666)
 	if err != nil {
 		log.Error("Failed to create file %s, error %v.", fpath, err)
 		return err

csv/csv_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	"../bi5"
+	"ed-fx/go-duka/bi5"
 )
 
 func TestCloseChan(t *testing.T) {

fxt4/fxt4.go

@@ -3,21 +3,20 @@ package fxt4
 import (
 	"bufio"
 	"bytes"
+	"ed-fx/go-duka/misc"
 	"encoding/binary"
 	"fmt"
 	"io"
-	"log"
 	"math"
 	"os"
 	"path/filepath"
 
-	"../core"
-	//"github.com/iizotop/go-duka/misc"
+	"ed-fx/go-duka/core"
 )
 
-//var (
-//	log = misc.NewLogger("FXT", 3)
-//)
+var (
+	log = misc.NewLogger("FXT", 3)
+)
 
 // FxtFile define fxt file format
 //
@@ -64,12 +63,12 @@ func NewFxtFile(timeframe, spread, model uint32, dest, symbol string) *FxtFile {
 func (f *FxtFile) worker() error {
 	defer func() {
 		close(f.chClose)
-		log.Printf("M%d Saved Bar: %d, Ticks: %d.", f.timeframe, f.barCount, f.tickCount)
+		log.Info("M%d Saved Bar: %d, Ticks: %d.", f.timeframe, f.barCount, f.tickCount)
 	}()
 
 	fxt, err := os.OpenFile(f.fpath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666)
 	if err != nil {
-		log.Printf("Create file %s failed: %v.", f.fpath, err)
+		log.Error("Create file %s failed: %v.", f.fpath, err)
 		return err
 	}
 
@@ -80,31 +79,31 @@ func (f *FxtFile) worker() error {
 	// convert FXT header
 	//
 	if err = binary.Write(bu, binary.LittleEndian, f.header); err != nil {
-		log.Printf("Convert FXT header failed: %v.", err)
+		log.Error("Convert FXT header failed: %v.", err)
 		return err
 	}
 	// write FXT file
 	if _, err := fxt.Write(bu.Bytes()); err != nil {
-		log.Printf("Write FXT header failed: %v.", err)
+		log.Error("Write FXT header failed: %v.", err)
 		return err
 	}
 
 	for tick := range f.chTicks {
 
 		if tick.BarTimestamp > uint64(tick.TickTimestamp) {
-			log.Printf("Tick(%v)", tick)
+			log.Info("Tick(%v)", tick)
 		}
 
 		bu.Reset()
 		//
 		//  write tick data
 		//
 		if err = binary.Write(bu, binary.LittleEndian, tick); err != nil {
-			log.Printf("Pack tick failed: %v.", err)
+			log.Error("Pack tick failed: %v.", err)
 			break
 		}
 		if _, err = fxt.Write(bu.Bytes()); err != nil {
-			log.Printf("Write fxt tick (%x) failed: %v.", bu.Bytes(), err)
+			log.Error("Write fxt tick (%x) failed: %v.", bu.Bytes(), err)
 			break
 		}
 
@@ -159,7 +158,7 @@ func (f *FxtFile) adjustHeader() error {
 
 	fxt, err := os.OpenFile(f.fpath, os.O_RDWR, 0666)
 	if err != nil {
-		log.Printf("Open file %s failed: %v.", f.fpath, err)
+		log.Error("Open file %s failed: %v.", f.fpath, err)
 		return err
 	}
 	defer fxt.Close()
@@ -181,11 +180,11 @@ func (f *FxtFile) adjustHeader() error {
 			_, err = fxt.Write(bu.Bytes())
 		}
 		if err != nil {
-			log.Printf("Adjust FXT header 1 failed: %v.", err)
+			log.Error("Adjust FXT header 1 failed: %v.", err)
 			return err
 		}
 	} else {
-		log.Printf("File seek 1 failed: %v.", err)
+		log.Error("File seek 1 failed: %v.", err)
 		return err
 	}
 
@@ -203,11 +202,11 @@ func (f *FxtFile) adjustHeader() error {
 			_, err = fxt.Write(bu.Bytes())
 		}
 		if err != nil {
-			log.Printf("Adjust FXT header 2 failed: %v.", err)
+			log.Error("Adjust FXT header 2 failed: %v.", err)
 			return err
 		}
 	} else {
-		log.Printf("File seek 2 failed: %v.", err)
+		log.Error("File seek 2 failed: %v.", err)
 		return err
 	}
 
@@ -225,22 +224,22 @@ func (f *FxtFile) Finish() error {
 func DumpFile(fname string, header bool, w io.Writer) {
 	fh, err := os.OpenFile(fname, os.O_RDONLY, 0666)
 	if err != nil {
-		log.Printf("Open fxt file failed: %v.", err)
+		log.Error("Open fxt file failed: %v.", err)
 		return
 	}
 	defer fh.Close()
 
 	bs := make([]byte, headerSize)
 	n, err := fh.Read(bs[:])
 	if err != nil || n != headerSize {
-		log.Printf("Read fxt header failed: %v.", err)
+		log.Error("Read fxt header failed: %v.", err)
 		return
 	}
 
 	var h FXTHeader
 	err = binary.Read(bytes.NewBuffer(bs[:]), binary.LittleEndian, &h)
 	if err != nil {
-		log.Printf("Decode fxt header failed: %v.", err)
+		log.Error("Decode fxt header failed: %v.", err)
 		return
 	}
 
@@ -264,14 +263,14 @@ func DumpFile(fname string, header bool, w io.Writer) {
 		}
 
 		if n != tickSize || err != nil {
-			log.Printf("Read tick data failed: %v.", err)
+			log.Error("Read tick data failed: %v.", err)
 			break
 		}
 
 		var tick FxtTick
 		err = binary.Read(bytes.NewBuffer(tickBs[:]), binary.LittleEndian, &tick)
 		if err != nil {
-			log.Printf("Decode tick data failed: %v.", err)
+			log.Error("Decode tick data failed: %v.", err)
 			break
 		}