Files
gocryptotrader/engine/ntp_manager.go
Scott 197ef2df21 Feature: Data history manager engine subsystem (#693)
* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange
2021-07-01 16:21:48 +10:00

206 lines
5.4 KiB
Go

package engine
import (
"encoding/binary"
"fmt"
"net"
"sync/atomic"
"time"
"github.com/thrasher-corp/gocryptotrader/config"
"github.com/thrasher-corp/gocryptotrader/log"
)
// setupNTPManager creates a new NTP manager
func setupNTPManager(cfg *config.NTPClientConfig, loggingEnabled bool) (*ntpManager, error) {
if cfg == nil {
return nil, errNilConfig
}
if cfg.AllowedNegativeDifference == nil ||
cfg.AllowedDifference == nil {
return nil, errNilConfigValues
}
return &ntpManager{
shutdown: make(chan struct{}),
level: int64(cfg.Level),
allowedDifference: *cfg.AllowedDifference,
allowedNegativeDifference: *cfg.AllowedNegativeDifference,
pools: cfg.Pool,
checkInterval: defaultNTPCheckInterval,
retryLimit: defaultRetryLimit,
loggingEnabled: loggingEnabled,
}, nil
}
// IsRunning safely checks whether the subsystem is running
func (m *ntpManager) IsRunning() bool {
if m == nil {
return false
}
return atomic.LoadInt32(&m.started) == 1
}
// Start runs the subsystem
func (m *ntpManager) Start() error {
if m == nil {
return fmt.Errorf("ntp manager %w", ErrNilSubsystem)
}
if !atomic.CompareAndSwapInt32(&m.started, 0, 1) {
return fmt.Errorf("NTP manager %w", ErrSubSystemAlreadyStarted)
}
if m.level == 0 && m.loggingEnabled {
// Sometimes the NTP client can have transient issues due to UDP, try
// the default retry limits before giving up
check:
for i := 0; i < m.retryLimit; i++ {
err := m.processTime()
switch err {
case nil:
break check
case ErrSubSystemNotStarted:
log.Debugln(log.TimeMgr, "NTP manager: User disabled NTP prompts. Exiting.")
atomic.CompareAndSwapInt32(&m.started, 1, 0)
return nil
default:
if i == m.retryLimit-1 {
return err
}
}
}
}
if m.level != 1 {
atomic.CompareAndSwapInt32(&m.started, 1, 0)
return errNTPManagerDisabled
}
m.shutdown = make(chan struct{})
go m.run()
log.Debugf(log.TimeMgr, "NTP manager %s", MsgSubSystemStarted)
return nil
}
// Stop attempts to shutdown the subsystem
func (m *ntpManager) Stop() error {
if m == nil {
return fmt.Errorf("ntp manager %w", ErrNilSubsystem)
}
if atomic.LoadInt32(&m.started) == 0 {
return fmt.Errorf("NTP manager %w", ErrSubSystemNotStarted)
}
defer func() {
log.Debugf(log.TimeMgr, "NTP manager %s", MsgSubSystemShutdown)
atomic.CompareAndSwapInt32(&m.started, 1, 0)
}()
log.Debugf(log.TimeMgr, "NTP manager %s", MsgSubSystemShuttingDown)
close(m.shutdown)
return nil
}
// continuously checks the internet connection at intervals
func (m *ntpManager) run() {
t := time.NewTicker(m.checkInterval)
defer func() {
t.Stop()
}()
for {
select {
case <-m.shutdown:
return
case <-t.C:
err := m.processTime()
if err != nil {
log.Error(log.TimeMgr, err)
}
}
}
}
// FetchNTPTime returns the time from defined NTP pools
func (m *ntpManager) FetchNTPTime() (time.Time, error) {
if m == nil {
return time.Time{}, fmt.Errorf("ntp manager %w", ErrNilSubsystem)
}
if atomic.LoadInt32(&m.started) == 0 {
return time.Time{}, fmt.Errorf("NTP manager %w", ErrSubSystemNotStarted)
}
return m.checkTimeInPools(), nil
}
// processTime determines the difference between system time and NTP time
// to discover discrepancies
func (m *ntpManager) processTime() error {
if atomic.LoadInt32(&m.started) == 0 {
return fmt.Errorf("NTP manager %w", ErrSubSystemNotStarted)
}
NTPTime, err := m.FetchNTPTime()
if err != nil {
return err
}
currentTime := time.Now()
diff := NTPTime.Sub(currentTime)
configNTPTime := m.allowedDifference
negDiff := m.allowedNegativeDifference
configNTPNegativeTime := -negDiff
if diff > configNTPTime || diff < configNTPNegativeTime {
log.Warnf(log.TimeMgr, "NTP manager: Time out of sync (NTP): %v | (time.Now()): %v | (Difference): %v | (Allowed): +%v / %v\n",
NTPTime,
currentTime,
diff,
configNTPTime,
configNTPNegativeTime)
}
return nil
}
// checkTimeInPools returns local based on ntp servers provided timestamp
// if no server can be reached will return local time in UTC()
func (m *ntpManager) checkTimeInPools() time.Time {
for i := range m.pools {
con, err := net.DialTimeout("udp", m.pools[i], 5*time.Second)
if err != nil {
log.Warnf(log.TimeMgr, "Unable to connect to hosts %v attempting next", m.pools[i])
continue
}
if err = con.SetDeadline(time.Now().Add(5 * time.Second)); err != nil {
log.Warnf(log.TimeMgr, "Unable to SetDeadline. Error: %s\n", err)
err = con.Close()
if err != nil {
log.Error(log.TimeMgr, err)
}
continue
}
req := &ntpPacket{Settings: 0x1B}
if err = binary.Write(con, binary.BigEndian, req); err != nil {
log.Warnf(log.TimeMgr, "Unable to write. Error: %s\n", err)
err = con.Close()
if err != nil {
log.Error(log.TimeMgr, err)
}
continue
}
rsp := &ntpPacket{}
if err = binary.Read(con, binary.BigEndian, rsp); err != nil {
log.Warnf(log.TimeMgr, "Unable to read. Error: %s\n", err)
err = con.Close()
if err != nil {
log.Error(log.TimeMgr, err)
}
continue
}
secs := float64(rsp.TxTimeSec) - 2208988800
nanos := (int64(rsp.TxTimeFrac) * 1e9) >> 32
err = con.Close()
if err != nil {
log.Error(log.TimeMgr, err)
}
return time.Unix(int64(secs), nanos)
}
log.Warnln(log.TimeMgr, "No valid NTP servers found, using current system time")
return time.Now().UTC()
}