gocryptotrader/database/repository/candle/candle.go

package candle

import (
	"context"
	"database/sql"
	"encoding/csv"
	"fmt"
	"io"
	"math"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gofrs/uuid"
	"github.com/thrasher-corp/gocryptotrader/database"
	modelPSQL "github.com/thrasher-corp/gocryptotrader/database/models/postgres"
	modelSQLite "github.com/thrasher-corp/gocryptotrader/database/models/sqlite3"
	"github.com/thrasher-corp/gocryptotrader/database/repository"
	"github.com/thrasher-corp/gocryptotrader/database/repository/exchange"
	"github.com/thrasher-corp/gocryptotrader/log"
	"github.com/thrasher-corp/sqlboiler/boil"
	"github.com/thrasher-corp/sqlboiler/queries/qm"
	"github.com/volatiletech/null"
)

// Series returns candle data
func Series(exchangeName, base, quote string, interval int64, asset string, start, end time.Time) (out Item, err error) {
	if exchangeName == "" || base == "" || quote == "" || asset == "" || interval <= 0 {
		return out, errInvalidInput
	}

	queries := []qm.QueryMod{
		qm.Where("base = ?", strings.ToUpper(base)),
		qm.Where("quote = ?", strings.ToUpper(quote)),
		qm.Where("interval = ?", interval),
		qm.Where("asset = ?", strings.ToLower(asset)),
		qm.OrderBy("timestamp"),
	}

	exchangeUUID, errS := exchange.UUIDByName(exchangeName)
	if errS != nil {
		return out, errS
	}
	queries = append(queries, qm.Where("exchange_name_id = ?", exchangeUUID.String()))
	if repository.GetSQLDialect() == database.DBSQLite3 {
		queries = append(queries, qm.Where("timestamp between ? and ?", start.UTC().Format(time.RFC3339), end.UTC().Format(time.RFC3339)))
		retCandle, errC := modelSQLite.Candles(queries...).All(context.TODO(), database.DB.SQL)
		if errC != nil {
			return out, errC
		}
		for x := range retCandle {
			t, errT := time.Parse(time.RFC3339, retCandle[x].Timestamp)
			if errT != nil {
				return out, errT
			}
			out.Candles = append(out.Candles, Candle{
				Timestamp:        t,
				Open:             retCandle[x].Open,
				High:             retCandle[x].High,
				Low:              retCandle[x].Low,
				Close:            retCandle[x].Close,
				Volume:           retCandle[x].Volume,
				SourceJobID:      retCandle[x].SourceJobID.String,
				ValidationJobID:  retCandle[x].ValidationJobID.String,
				ValidationIssues: retCandle[x].ValidationIssues.String,
			})
		}
	} else {
		queries = append(queries, qm.Where("timestamp between ? and ?", start.UTC(), end.UTC()))
		retCandle, errC := modelPSQL.Candles(queries...).All(context.TODO(), database.DB.SQL)
		if errC != nil {
			return out, errC
		}

		for x := range retCandle {
			out.Candles = append(out.Candles, Candle{
				Timestamp:        retCandle[x].Timestamp,
				Open:             retCandle[x].Open,
				High:             retCandle[x].High,
				Low:              retCandle[x].Low,
				Close:            retCandle[x].Close,
				Volume:           retCandle[x].Volume,
				SourceJobID:      retCandle[x].SourceJobID.String,
				ValidationJobID:  retCandle[x].ValidationJobID.String,
				ValidationIssues: retCandle[x].ValidationIssues.String,
			})
		}
	}
	if len(out.Candles) < 1 {
		return out, fmt.Errorf("%w: %s %s %s %v %s", ErrNoCandleDataFound, exchangeName, base, quote, interval, asset)
	}

	out.ExchangeID = exchangeName
	out.Interval = interval
	out.Base = base
	out.Quote = quote
	out.Asset = asset
	return out, err
}

// DeleteCandles will delete all existing matching candles
func DeleteCandles(in *Item) (int64, error) {
	if database.DB.SQL == nil {
		return 0, database.ErrDatabaseSupportDisabled
	}
	if len(in.Candles) < 1 {
		return 0, errNoCandleData
	}

	ctx := context.TODO()
	queries := []qm.QueryMod{
		qm.Where("base = ?", strings.ToUpper(in.Base)),
		qm.Where("quote = ?", strings.ToUpper(in.Quote)),
		qm.Where("interval = ?", in.Interval),
		qm.Where("asset = ?", strings.ToLower(in.Asset)),
		qm.Where("exchange_name_id = ?", in.ExchangeID),
	}
	if repository.GetSQLDialect() == database.DBSQLite3 {
		queries = append(queries, qm.Where("timestamp between ? and ?", in.Candles[0].Timestamp.UTC().Format(time.RFC3339), in.Candles[len(in.Candles)-1].Timestamp.UTC().Format(time.RFC3339)))
		return deleteSQLite(ctx, queries)
	}

	queries = append(queries, qm.Where("timestamp between ? and ?", in.Candles[0].Timestamp.UTC(), in.Candles[len(in.Candles)-1].Timestamp.UTC()))
	return deletePostgres(ctx, queries)
}

func deleteSQLite(ctx context.Context, queries []qm.QueryMod) (int64, error) {
	retCandle, err := modelSQLite.Candles(queries...).All(ctx, database.DB.SQL)
	if err != nil {
		return 0, err
	}
	var tx *sql.Tx
	tx, err = database.DB.SQL.BeginTx(ctx, nil)
	if err != nil {
		return 0, err
	}
	var totalDeleted int64
	totalDeleted, err = retCandle.DeleteAll(ctx, tx)
	if err != nil {
		errRB := tx.Rollback()
		if errRB != nil {
			log.Errorln(log.DatabaseMgr, errRB)
		}
		return 0, err
	}

	err = tx.Commit()
	if err != nil {
		return 0, err
	}
	return totalDeleted, nil
}

func deletePostgres(ctx context.Context, queries []qm.QueryMod) (int64, error) {
	retCandle, err := modelPSQL.Candles(queries...).All(ctx, database.DB.SQL)
	if err != nil {
		return 0, err
	}
	var tx *sql.Tx
	tx, err = database.DB.SQL.BeginTx(ctx, nil)
	if err != nil {
		return 0, err
	}
	var totalDeleted int64
	totalDeleted, err = retCandle.DeleteAll(ctx, tx)
	if err != nil {
		errRB := tx.Rollback()
		if errRB != nil {
			log.Errorln(log.DatabaseMgr, errRB)
		}
		return 0, err
	}

	err = tx.Commit()
	if err != nil {
		return 0, err
	}
	return totalDeleted, nil
}

// Insert series of candles
func Insert(in *Item) (uint64, error) {
	if database.DB.SQL == nil {
		return 0, database.ErrDatabaseSupportDisabled
	}

	if len(in.Candles) < 1 {
		return 0, errNoCandleData
	}

	ctx := context.TODO()
	tx, err := database.DB.SQL.BeginTx(ctx, nil)
	if err != nil {
		return 0, err
	}

	var totalInserted uint64
	if repository.GetSQLDialect() == database.DBSQLite3 {
		totalInserted, err = insertSQLite(ctx, tx, in)
	} else {
		totalInserted, err = insertPostgresSQL(ctx, tx, in)
	}
	if err != nil {
		errRB := tx.Rollback()
		if errRB != nil {
			log.Errorln(log.DatabaseMgr, errRB)
		}
		return 0, err
	}

	err = tx.Commit()
	if err != nil {
		return 0, err
	}
	return totalInserted, nil
}

func insertSQLite(ctx context.Context, tx *sql.Tx, in *Item) (uint64, error) {
	var totalInserted uint64
	for x := range in.Candles {
		var tempCandle = modelSQLite.Candle{
			ExchangeNameID: in.ExchangeID,
			Base:           strings.ToUpper(in.Base),
			Quote:          strings.ToUpper(in.Quote),
			Interval:       strconv.FormatInt(in.Interval, 10),
			Asset:          strings.ToLower(in.Asset),
			Timestamp:      in.Candles[x].Timestamp.UTC().Format(time.RFC3339),
			Open:           in.Candles[x].Open,
			High:           in.Candles[x].High,
			Low:            in.Candles[x].Low,
			Close:          in.Candles[x].Close,
			Volume:         in.Candles[x].Volume,
		}
		tempUUID, err := uuid.NewV4()
		if err != nil {
			return 0, err
		}
		tempCandle.ID = tempUUID.String()
		tempCandle.ValidationJobID = null.String{String: in.Candles[x].ValidationJobID, Valid: in.Candles[x].ValidationJobID != ""}
		tempCandle.ValidationIssues = null.String{String: in.Candles[x].ValidationIssues, Valid: in.Candles[x].ValidationIssues != ""}
		tempCandle.SourceJobID = null.String{String: in.Candles[x].SourceJobID, Valid: in.Candles[x].SourceJobID != ""}
		err = tempCandle.Insert(ctx, tx, boil.Infer())
		if err != nil {
			return 0, err
		}
		if totalInserted < math.MaxUint64 {
			totalInserted++
		}
	}
	return totalInserted, nil
}

func insertPostgresSQL(ctx context.Context, tx *sql.Tx, in *Item) (uint64, error) {
	var totalInserted uint64
	for x := range in.Candles {
		var tempCandle = modelPSQL.Candle{
			ExchangeNameID: in.ExchangeID,
			Base:           strings.ToUpper(in.Base),
			Quote:          strings.ToUpper(in.Quote),
			Interval:       in.Interval,
			Asset:          strings.ToLower(in.Asset),
			Timestamp:      in.Candles[x].Timestamp,
			Open:           in.Candles[x].Open,
			High:           in.Candles[x].High,
			Low:            in.Candles[x].Low,
			Close:          in.Candles[x].Close,
			Volume:         in.Candles[x].Volume,
		}
		tempCandle.ValidationJobID = null.String{String: in.Candles[x].ValidationJobID, Valid: in.Candles[x].ValidationJobID != ""}
		tempCandle.ValidationIssues = null.String{String: in.Candles[x].ValidationIssues, Valid: in.Candles[x].ValidationIssues != ""}
		tempCandle.SourceJobID = null.String{String: in.Candles[x].SourceJobID, Valid: in.Candles[x].SourceJobID != ""}
		err := tempCandle.Upsert(ctx, tx, true, []string{"timestamp", "exchange_name_id", "base", "quote", "interval", "asset"}, boil.Infer(), boil.Infer())
		if err != nil {
			return 0, err
		}
		if totalInserted < math.MaxUint64 {
			totalInserted++
		}
	}
	return totalInserted, nil
}

// InsertFromCSV load a CSV list of candle data and insert into database
func InsertFromCSV(exchangeName, base, quote string, interval int64, asset, file string) (uint64, error) {
	csvFile, err := os.Open(file)
	if err != nil {
		return 0, err
	}

	defer func() {
		err = csvFile.Close()
		if err != nil {
			log.Errorln(log.Global, err)
		}
	}()

	csvData := csv.NewReader(csvFile)

	exchangeUUID, err := exchange.UUIDByName(exchangeName)
	if err != nil {
		return 0, err
	}

	tempCandle := &Item{
		ExchangeID: exchangeUUID.String(),
		Base:       base,
		Quote:      quote,
		Interval:   interval,
		Asset:      asset,
	}

	for {
		row, errCSV := csvData.Read()
		if errCSV != nil {
			if errCSV == io.EOF {
				break
			}
			return 0, errCSV
		}

		tempTick := Candle{}
		v, errParse := strconv.ParseInt(row[0], 10, 32)
		if errParse != nil {
			return 0, errParse
		}
		tempTick.Timestamp = time.Unix(v, 0).UTC()
		if tempTick.Timestamp.IsZero() {
			err = fmt.Errorf("invalid timestamp received on row %v", row)
			break
		}

		tempTick.Volume, err = strconv.ParseFloat(row[1], 64)
		if err != nil {
			break
		}

		tempTick.Open, err = strconv.ParseFloat(row[2], 64)
		if err != nil {
			break
		}

		tempTick.High, err = strconv.ParseFloat(row[3], 64)
		if err != nil {
			break
		}

		tempTick.Low, err = strconv.ParseFloat(row[4], 64)
		if err != nil {
			break
		}

		tempTick.Close, err = strconv.ParseFloat(row[5], 64)
		if err != nil {
			break
		}
		tempCandle.Candles = append(tempCandle.Candles, tempTick)
	}
	if err != nil {
		return 0, err
	}

	return Insert(tempCandle)
}