gocryptotrader/exchanges/mock/recording.go

package mock

import (
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"slices"
	"strings"
	"sync"

	"github.com/thrasher-corp/gocryptotrader/common"
	"github.com/thrasher-corp/gocryptotrader/common/file"
	"github.com/thrasher-corp/gocryptotrader/encoding/json"
)

// defaultDataSliceLimit the mock slice data size limit to a default of 5
const defaultDataSliceLimit = 5

// HTTPResponse defines expected response from the end point including request
// data for pathing on the VCR server
type HTTPResponse struct {
	Data        json.RawMessage     `json:"data"`
	QueryString string              `json:"queryString"`
	BodyParams  string              `json:"bodyParams"`
	Headers     map[string][]string `json:"headers"`
}

// HTTPRecord will record the request and response to a default JSON file for
// mocking purposes
// mockDataSliceLimit defaults to 5
func HTTPRecord(res *http.Response, service string, respContents []byte, mockDataSliceLimit int) error {
	if res == nil {
		return errors.New("http.Response cannot be nil")
	}

	if res.Request == nil {
		return errors.New("http.Request cannot be nil")
	}

	if res.Request.Method == "" {
		return errors.New("request method not supplied")
	}

	if service == "" {
		return errors.New("service not supplied cannot access correct mock file")
	}
	service = strings.ToLower(service)
	if mockDataSliceLimit == 0 {
		mockDataSliceLimit = defaultDataSliceLimit
	}
	outputFilePath := filepath.Join(DefaultDirectory, service, service+".json")
	_, err := os.Stat(outputFilePath)
	if err != nil {
		if os.IsExist(err) {
			return err
		}
		// check alternative path to add compatibility with /internal/testing/exchange/exchange.go MockHTTPInstance
		outputFilePath = filepath.Join("..", service, "testdata", "http.json")
		_, err = os.Stat(outputFilePath)
		if err != nil {
			return err
		}
	}

	contents, err := os.ReadFile(outputFilePath)
	if err != nil {
		return err
	}

	var m VCRMock
	err = json.Unmarshal(contents, &m)
	if err != nil {
		return err
	}

	if m.Routes == nil {
		m.Routes = make(map[string]map[string][]HTTPResponse)
	}

	items, err := getExcludedItems()
	if err != nil {
		return err
	}

	var httpResponse HTTPResponse
	cleanedContents, err := CheckResponsePayload(respContents, items, mockDataSliceLimit)
	if err != nil {
		return err
	}

	err = json.Unmarshal(cleanedContents, &httpResponse.Data)
	if err != nil {
		return err
	}

	var body string
	if res.Request.GetBody != nil {
		bodycopy, bodyErr := res.Request.GetBody()
		if bodyErr != nil {
			return bodyErr
		}
		payload, bodyErr := io.ReadAll(bodycopy)
		if bodyErr != nil {
			return bodyErr
		}
		body = string(payload)
	}

	switch res.Request.Header.Get(contentType) {
	case applicationURLEncoded:
		vals, urlErr := url.ParseQuery(body)
		if urlErr != nil {
			return urlErr
		}

		httpResponse.BodyParams = GetFilteredURLVals(vals, items)
	case textPlain:
		payload := res.Request.Header.Get("X-Gemini-Payload")
		j, dErr := base64.StdEncoding.DecodeString(payload)
		if dErr != nil {
			return dErr
		}

		httpResponse.BodyParams = string(j)

	default:
		httpResponse.BodyParams = body
	}

	httpResponse.Headers = GetFilteredHeader(res, items)
	httpResponse.QueryString = GetFilteredURLVals(res.Request.URL.Query(), items)

	_, ok := m.Routes[res.Request.URL.Path]
	if !ok {
		m.Routes[res.Request.URL.Path] = make(map[string][]HTTPResponse)
		m.Routes[res.Request.URL.Path][res.Request.Method] = []HTTPResponse{httpResponse}
	} else {
		mockResponses, ok := m.Routes[res.Request.URL.Path][res.Request.Method]
		if !ok {
			m.Routes[res.Request.URL.Path][res.Request.Method] = []HTTPResponse{httpResponse}
		} else {
			switch res.Request.Method { // Based off method - check add or replace
			case http.MethodGet:
				for i := range mockResponses {
					mockQuery, urlErr := url.ParseQuery(mockResponses[i].QueryString)
					if urlErr != nil {
						return urlErr
					}

					if MatchURLVals(mockQuery, res.Request.URL.Query()) {
						mockResponses = slices.Delete(mockResponses, i, i+1)
						break
					}
				}

			case http.MethodPost:
				for i := range mockResponses {
					cType, ok := mockResponses[i].Headers[contentType]

					jCType := strings.Join(cType, "")
					var found bool
					switch jCType {
					case applicationURLEncoded:
						respQueryVals, urlErr := url.ParseQuery(body)
						if urlErr != nil {
							return urlErr
						}

						mockRespVals, urlErr := url.ParseQuery(mockResponses[i].BodyParams)
						if urlErr != nil {
							return urlErr
						}

						if MatchURLVals(respQueryVals, mockRespVals) {
							// if found will delete instance and overwrite with new
							// data
							mockResponses = slices.Delete(mockResponses, i, i+1)
							found = true
						}

					case applicationJSON, textPlain:
						reqVals, jErr := DeriveURLValsFromJSONMap([]byte(body))
						if jErr != nil {
							return jErr
						}

						mockVals, jErr := DeriveURLValsFromJSONMap([]byte(mockResponses[i].BodyParams))
						if jErr != nil {
							return jErr
						}

						if MatchURLVals(reqVals, mockVals) {
							// if found will delete instance and overwrite with new
							// data
							mockResponses = slices.Delete(mockResponses, i, i+1)
							found = true
						}
					case "":
						if !ok {
							// Assume query params are used
							mockQuery, urlErr := url.ParseQuery(mockResponses[i].QueryString)
							if urlErr != nil {
								return urlErr
							}

							if MatchURLVals(mockQuery, res.Request.URL.Query()) {
								// if found will delete instance and overwrite with new data
								mockResponses = slices.Delete(mockResponses, i, i+1)
								found = true
							}

							break
						}

						fallthrough
					default:
						return fmt.Errorf("unhandled content type %s", jCType)
					}
					if found {
						break
					}
				}

			default:
				return fmt.Errorf("unhandled request method %s", res.Request.Method)
			}

			m.Routes[res.Request.URL.Path][res.Request.Method] = append(mockResponses, httpResponse)
		}
	}

	payload, err := json.MarshalIndent(m, "", " ")
	if err != nil {
		return err
	}

	return file.Write(outputFilePath, payload)
}

// GetFilteredHeader filters excluded http headers for insertion into a mock
// test file
func GetFilteredHeader(res *http.Response, items Exclusion) http.Header {
	for i := range items.Headers {
		if res.Request.Header.Get(items.Headers[i]) != "" {
			res.Request.Header.Set(items.Headers[i], "")
		}
	}
	return res.Request.Header
}

// GetFilteredURLVals filters excluded url value variables for insertion into a
// mock test file
func GetFilteredURLVals(vals url.Values, items Exclusion) string {
	for key := range vals {
		for i := range items.Variables {
			if strings.EqualFold(items.Variables[i], key) {
				vals.Set(key, "")
			}
		}
	}
	return vals.Encode()
}

// CheckResponsePayload checks to see if there are any response body variables
// that should not be there.
func CheckResponsePayload(data []byte, items Exclusion, mockDataSliceLimit int) ([]byte, error) {
	var intermediary any
	if err := json.Unmarshal(data, &intermediary); err != nil {
		return nil, err
	}

	payload, err := CheckJSON(intermediary, &items, mockDataSliceLimit)
	if err != nil {
		return nil, err
	}

	return json.MarshalIndent(payload, "", " ")
}

// Reflection consts
const (
	Float64 = "float64"
	Slice   = "slice"
	String  = "string"
	Bool    = "bool"
	Invalid = "invalid"
)

// CheckJSON recursively parses json data to retract keywords, quite intensive.
func CheckJSON(data any, excluded *Exclusion, limit int) (any, error) {
	if value, ok := data.([]any); ok {
		var sData []any
		for i := range value {
			switch subvalue := value[i].(type) {
			case []any, map[string]any:
				checkedData, err := CheckJSON(subvalue, excluded, limit)
				if err != nil {
					return nil, err
				}
				sData = append(sData, checkedData)
				if limit > 0 && len(sData) >= limit {
					return sData, nil
				}
			default:
				// Primitive value doesn't need exclusions applied, e.g. float64 or string
				sData = append(sData, subvalue)
			}
		}
		return sData, nil
	}

	conv, err := json.Marshal(data)
	if err != nil {
		return nil, err
	}

	var contextValue map[string]any
	err = json.Unmarshal(conv, &contextValue)
	if err != nil {
		return nil, err
	}

	if len(contextValue) == 0 {
		// Nil for some reason, should error out before in json.Unmarshal
		return contextValue, nil
	}

	for key, val := range contextValue {
		switch reflect.ValueOf(val).Kind().String() {
		case String:
			if IsExcluded(key, excluded.Variables) {
				contextValue[key] = "" // Zero val string
			}
		case Float64:
			if IsExcluded(key, excluded.Variables) {
				contextValue[key] = 0.0 // Zero val float
			}
		case Slice:
			slice, ok := val.([]any)
			if !ok {
				return nil, common.GetTypeAssertError("[]any", val)
			}
			switch {
			case len(slice) == 0:
				// Empty slice found
				contextValue[key] = slice
			case IsExcluded(key, excluded.Variables):
				contextValue[key] = nil // Zero val slice
			default:
				contextValue[key], err = CheckJSON(slice, excluded, limit)
				if err != nil {
					return nil, err
				}
			}
		case Bool, Invalid: // Skip these bad boys for now
		default:
			// Recursively check map data
			contextValue[key], err = CheckJSON(val, excluded, limit)
			if err != nil {
				return nil, err
			}
		}
	}

	return contextValue, nil
}

// IsExcluded cross references the key with the excluded variables
func IsExcluded(key string, excludedVars []string) bool {
	for i := range excludedVars {
		if strings.EqualFold(key, excludedVars[i]) {
			return true
		}
	}
	return false
}

var (
	excludedList  Exclusion
	m             sync.Mutex
	set           bool
	exclusionFile = DefaultDirectory + "exclusion.json"
)

var defaultExcludedHeaders = []string{
	"Key",
	"X-Mbx-Apikey",
	"Rest-Key",
	"Apiauth-Key",
	"X-Bapi-Api-Key",
}

var defaultExcludedVariables = []string{
	"bsb",
	"user",
	"name",
	"real_name",
	"receiver_name",
	"account_number",
	"username",
	"apiKey",
}

// Exclusion defines a list of items to be excluded from the main mock output
// this attempts a catch all approach and needs to be updated per exchange basis
type Exclusion struct {
	Headers   []string `json:"headers"`
	Variables []string `json:"variables"`
}

// getExcludedItems checks to see if the variable is in the exclusion list as to
// not display secure items in mock file generator output
func getExcludedItems() (Exclusion, error) {
	m.Lock()
	defer m.Unlock()
	if !set {
		f, err := os.ReadFile(exclusionFile)
		if err != nil {
			if !strings.Contains(err.Error(), "no such file or directory") {
				return excludedList, err
			}

			excludedList.Headers = defaultExcludedHeaders
			excludedList.Variables = defaultExcludedVariables

			data, mErr := json.MarshalIndent(excludedList, "", " ")
			if mErr != nil {
				return excludedList, mErr
			}

			mErr = os.WriteFile(exclusionFile, data, os.ModePerm)
			if mErr != nil {
				return excludedList, mErr
			}
		} else {
			err = json.Unmarshal(f, &excludedList)
			if err != nil {
				return excludedList, err
			}

			if len(excludedList.Headers) == 0 || len(excludedList.Variables) == 0 {
				return excludedList, errors.New("exclusion list does not have names")
			}
		}

		set = true
	}

	return excludedList, nil
}