CI: Add macOS, Windows x64 and Linux arm64 test support (#1422)

* CI: Add macOS, Windows and Linux arm64 support

* Modify arm64 job name to be inline with the others

* linter: Prevent cache from causing "Cannot open: File exists" issues

* Use setup-go and setup-node's inbuilt caching

.appveyor.yml

@@ -1,71 +0,0 @@
-build: off
-
-image: Visual Studio 2022 # See https://www.appveyor.com/docs/build-environment/#build-worker-images
-
-clone_folder: c:\gopath\src\github.com\thrasher-corp\gocryptotrader
-
-cache:
-  - '%APPDATA%\npm-cache'
-  - '%GOPATH%\pkg\mod'
-  - '%LOCALAPPDATA%\go-build'
-  - c:\gopath\src\github.com\thrasher-corp\gocryptotrader\web\node_modules
-
-environment:
-  GOPATH: c:\gopath
-  GO111MODULE: on
-  NODEJS_VER: 10.15.3
-  APPVEYOR_SAVE_CACHE_ON_ERROR: true
-  POSTGRES_PATH: C:\Program Files\PostgreSQL\9.6
-  PGUSER: postgres
-  PGPASSWORD: Password12!
-  POSTGRES_ENV_POSTGRES_USER: postgres
-  POSTGRES_ENV_POSTGRES_PASSWORD: Password12!
-  POSTGRES_ENV_POSTGRES_DB: gct_dev_ci
-  PSQL_USER: postgres
-  PSQL_HOST: localhost
-  PSQL_PASS: Password12!
-  PSQL_DBNAME: gct_dev_ci
-  PSQL_SSLMODE: disable
-  PSQL_SKIPSQLCMD: true
-  PSQL_TESTDBNAME: gct_dev_ci
-
-stack: go 1.17.8 # this is not actually used on Windows images
-
-services:
-  - postgresql96
-
-init:
-  - set PATH=%POSTGRES_PATH%\bin;%PATH%
-
-install:
-  - set Path=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%Path%
-  - ps: Install-Product node $env:NODEJS_VER
-  - cd c:\gopath\src\github.com\thrasher-corp\gocryptotrader\web
-  - npm install
-
-build_script:
-  - createdb gct_dev_ci
-
-before_test:
-  - cd c:\gopath\src\github.com\thrasher-corp\gocryptotrader
-  - go env
-  - go version
-  - go install
-
-test_script:
-  # test back-end
-  - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.0
-  - '%GOPATH%\bin\golangci-lint.exe run --verbose'
-  - ps: >-
-      if($env:APPVEYOR_SCHEDULED_BUILD -eq 'true') {
-          go test -race ./... -tags=mock_test_off
-      }else {
-          go test -race ./...
-      }
-  
-  # test front-end
-  - node --version
-  - npm --version
-  - cd c:\gopath\src\github.com\thrasher-corp\gocryptotrader\web
-  - npm run lint
-  - npm run build

.github/PULL_REQUEST_TEMPLATE.md

@@ -30,5 +30,5 @@ also consider improving test coverage whilst working on a certain feature or pac
 - [ ] I have made corresponding changes to the documentation and regenerated documentation via the documentation tool
 - [ ] My changes generate no new warnings
 - [ ] I have added tests that prove my fix is effective or that my feature works
-- [ ] New and existing unit tests pass locally and on Github Actions/AppVeyor with my changes
+- [ ] New and existing unit tests pass locally and on Github Actions with my changes
 - [ ] Any dependent changes have been merged and published in downstream modules

.github/workflows/arm64.Dockerfile

@@ -0,0 +1,12 @@
+FROM arm64v8/golang:1.21
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+
+COPY . /app
+
+RUN go build
+
+CMD ["go", "test", "./..."]

.github/workflows/linter.yml

@@ -9,7 +9,8 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version: '1.21.x'
+          cache: false
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.54.0
+          version: v1.54.0

.github/workflows/tests.yml

@@ -4,21 +4,33 @@ env:
   GO_VERSION: 1.21.x
 jobs:
   backend-psql:
-    name: GoCryptoTrader back-end with PSQL
+    name: GoCryptoTrader back-end
-    runs-on: ubuntu-latest
+    strategy:
-    services:
+      fail-fast: false
-      postgres:
+      matrix:
-        image: postgres
+        include:
-        env:
+          - os: ubuntu-latest
-          POSTGRES_DB: gct_dev_ci
+            goarch: amd64
-          POSTGRES_PASSWORD: postgres
+            psql: true
-        options: >-
+            skip_wrapper_tests: false
-          --health-cmd pg_isready
+          - os: ubuntu-latest
-          --health-interval 10s
+            goarch: 386
-          --health-timeout 5s
+            psql: true
-          --health-retries 5
+            skip_wrapper_tests: true
-        ports:
+          - os: macos-latest
-          - 5432:5432
+            goarch: amd64
+            psql: true
+            skip_wrapper_tests: true
+          - os: macos-13 # beta
+            goarch: amd64
+            psql: false 
+            skip_wrapper_tests: true
+          - os: windows-latest
+            goarch: amd64
+            psql: true
+            skip_wrapper_tests: true
+
+    runs-on: ${{ matrix.os }}
 
     steps:
     - name: Cancel previous workflow runs
@@ -26,6 +38,25 @@ jobs:
       with:
         access_token: ${{ github.token }}
 
+    - name: Setup Postgres
+      if: matrix.psql == true
+      uses: ikalnytskyi/action-setup-postgres@v4
+      with:
+        database: gct_dev_ci
+      id: postgres
+
+    - name: Set up Postgres environment
+      if: matrix.psql == true
+      run: |
+        echo "PSQL_USER=postgres" >> $GITHUB_ENV
+        echo "PSQL_PASS=postgres" >> $GITHUB_ENV
+        echo "PSQL_HOST=localhost" >> $GITHUB_ENV
+        echo "PSQL_DBNAME=gct_dev_ci" >> $GITHUB_ENV
+        echo "PSQL_TESTDBNAME=gct_dev_ci" >> $GITHUB_ENV
+        echo "PSQL_SSLMODE=disable" >> $GITHUB_ENV
+        echo "PSQL_SKIPSQLCMD=true" >> $GITHUB_ENV
+      shell: bash
+
     - name: Checkout code
       uses: actions/checkout@v4
 
@@ -34,45 +65,38 @@ jobs:
       with:
         go-version: ${{ env.GO_VERSION }}
 
-    - name: Cache go modules  
+    - name: Skip additional wrapper CI tests
-      uses: actions/cache@v3
+      if: matrix.skip_wrapper_tests == true
-      with:
+      run: |
-        path: |
+        echo "SKIP_WRAPPER_CI_TESTS=true" >> $GITHUB_ENV
-          ~/go/pkg/mod
+      shell: bash
-          ~/.cache/go-build
+
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+    - name: Additional steps for 386 architecture
-        restore-keys: ${{ runner.os }}-go-
+      if: matrix.goarch == '386'
+      run: |
+        echo "CGO_ENABLED=1" >> $GITHUB_ENV
+        sudo apt-get update
+        sudo apt-get install gcc-multilib g++-multilib
+      shell: bash
 
     - name: Test
-      run: go test -race -coverprofile coverage.txt -covermode atomic ./...
+      run: |
+        unset PGSERVICEFILE
+        if [ "${{ matrix.goarch }}" = "386" ]; then
+          go test -coverprofile coverage.txt -covermode atomic ./...
+        else
+          go test -race -coverprofile coverage.txt -covermode atomic ./...
+        fi
+      shell: bash
       env:
-        PSQL_USER: postgres
+        GOARCH: ${{ matrix.goarch }}
-        PSQL_PASS: postgres
-        PSQL_HOST: localhost
-        PSQL_DBNAME: gct_dev_ci
-        PSQL_SKIPSQLCMD: true
-        PSQL_TESTDBNAME: gct_dev_ci
-        PSQL_SSLMODE: disable
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
 
-  backend-32bit:
+  backend-arm64:
-    name: GoCryptoTrader back-end 32-bit with PSQL
+    name: GoCryptoTrader back-end (ubuntu-latest, arm64, false, true)
     runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_DB: gct_dev_ci
-          POSTGRES_PASSWORD: postgres
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
 
     steps:
     - name: Cancel previous workflow runs
@@ -82,43 +106,38 @@ jobs:
 
     - name: Checkout code
       uses: actions/checkout@v4
-
+      
-    - name: Setup Go
+    - name: Set up QEMU 
-      uses: actions/setup-go@v5
+      uses: docker/setup-qemu-action@v3
       with:
-        go-version: ${{ env.GO_VERSION }}
+        platforms: linux/arm64
 
-    - name: Cache go modules  
+    - name: Set up Docker buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Cache Docker layers
       uses: actions/cache@v3
       with:
-        path: |
+        path: /tmp/.docker-buildx-cache
-          ~/go/pkg/mod
+        key: ${{ runner.os }}-docker-buildx-${{ hashFiles('./.github/workflows/arm64.Dockerfile') }}-${{ github.sha }}
-          ~/.cache/go-build
+        restore-keys: |
-        key: ${{ runner.os }}-go-386-${{ hashFiles('**/go.sum') }}
+          ${{ runner.os }}-docker-buildx-${{ hashFiles('./.github/workflows/arm64.Dockerfile') }}-
-        restore-keys: ${{ runner.os }}-go-386-
+          ${{ runner.os }}-docker-buildx-
-
+    
-    - name: Update apt-get
+    - name: Build Docker image
-      run: sudo apt-get update
+      run: |
-
+        docker buildx build \
-    - name: Install gcc-multilib
+          --load \
-      run: sudo apt-get install gcc-multilib
+          --platform linux/arm64 \
-
+          --cache-to=type=local,dest=/tmp/.docker-buildx-cache,mode=max \
-    - name: Test
+          --cache-from=type=local,src=/tmp/.docker-buildx-cache \
-      run: go test -coverprofile coverage.txt -covermode atomic ./...
+          --tag gct-backend-arm64 \
-      env:
+          -f ./.github/workflows/arm64.Dockerfile \
-        PSQL_USER: postgres
+          .
-        PSQL_PASS: postgres
+    
-        PSQL_HOST: localhost
+    - name: Run Docker image
-        PSQL_DBNAME: gct_dev_ci
+      run: |
-        PSQL_SKIPSQLCMD: true
+        docker run --platform linux/arm64 --env SKIP_WRAPPER_CI_TESTS=true --env CI=true --rm gct-backend-arm64
-        PSQL_TESTDBNAME: gct_dev_ci
-        PSQL_SSLMODE: disable
-        GOARCH: 386
-        CGO_ENABLED: 1
-        CGO_CFLAGS: -fno-stack-protector
-
-    - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
 
   frontend:
     name: GoCryptoTrader front-end
@@ -135,15 +154,8 @@ jobs:
       uses: actions/setup-node@v4
       with:
         node-version: '10.8.x'
-
+        cache: 'npm'
-    - name: Cache Node.js modules
+        cache-dependency-path: web/package-lock.json
-      uses: actions/cache@v3
-      with:
-        path: ~/.npm
-        key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.OS }}-node-
-          ${{ runner.OS }}-
 
     - name: Build
       run: | 

.travis.yml

@@ -1,99 +0,0 @@
-matrix:
-  include:
-    - language: node_js
-      name: 'GoCryptoTrader [front-end]'
-      node_js:
-        - '10'
-        - '8'
-        - '6'
-      before_install:
-        - cd web/
-      install:
-        - npm install
-      script:
-        - npm run lint
-        - npm run build
-
-    - language: go
-      dist: xenial
-      name: 'GoCryptoTrader [back-end] [linux] [64-bit]'
-      go:
-        - 1.17.x
-      env:
-        - GO111MODULE=on
-        - PSQL_USER=postgres
-        - PSQL_HOST=localhost
-        - PSQL_DBNAME=gct_dev_ci
-        - PSQL_SKIPSQLCMD=true
-        - PSQL_TESTDBNAME=gct_dev_ci
-      install: true
-      cache:
-        directories:
-          - $GOPATH/pkg/mod
-      services:
-        - postgresql
-      before_script:
-        - psql -c 'create database gct_dev_ci;' -U postgres
-      script:
-        - make check
-      after_success:
-        - bash <(curl -s https://codecov.io/bash)
-
-    - language: go
-      dist: xenial
-      name: 'GoCryptoTrader [back-end] [linux] [32-bit]'
-      go:
-        - 1.17.x
-      env:
-        - GO111MODULE=on
-        - NO_RACE_TEST=1
-        - PSQL_USER=postgres
-        - PSQL_HOST=localhost
-        - PSQL_DBNAME=gct_dev_ci
-        - PSQL_SKIPSQLCMD=true
-        - PSQL_TESTDBNAME=gct_dev_ci
-      install: true
-      cache:
-        directories:
-          - $GOPATH/pkg/mod
-      services:
-        - postgresql
-      before_script:
-        - psql -c 'create database gct_dev_ci;' -U postgres
-      script:
-        - export GOARCH=386
-        - export CGO_ENABLED=1
-        - sudo apt-get update
-        - sudo apt-get install gcc-multilib
-        - make test
-      after_success:
-        - bash <(curl -s https://codecov.io/bash)
-
-    - language: go
-      os: osx
-      name: 'GoCryptoTrader [back-end] [darwin]'
-      go:
-        - 1.17.x
-      env:
-        - GO111MODULE=on
-        - PSQL_USER=postgres
-        - PSQL_HOST=localhost
-        - PSQL_DBNAME=gct_dev_ci
-        - PSQL_SSLMODE=disable
-        - PSQL_SKIPSQLCMD=true
-        - PSQL_TESTDBNAME=gct_dev_ci
-        - MACOSX_DEPLOYMENT_TARGET=10.15
-      install: true
-      cache:
-        directories:
-          - $GOPATH/pkg/mod
-      before_install:
-        - rm -rf /usr/local/var/postgres
-        - initdb /usr/local/var/postgres
-        - pg_ctl start --pgdata /usr/local/var/postgres
-        - createuser -s postgres
-        - psql -c 'create database gct_dev_ci;' -U postgres
-      script:
-        - make check
-      after_success:
-        - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -4,7 +4,6 @@ LINTPKG = github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.0
 LINTBIN = $(GOPATH)/bin/golangci-lint
 GCTLISTENPORT=9050
 GCTPROFILERLISTENPORT=8085
-CRON = $(TRAVIS_EVENT_TYPE)
 DRIVER ?= psql
 RACE_FLAG := $(if $(NO_RACE_TEST),,-race)
 CONFIG_FLAG = $(if $(CONFIG),-config $(CONFIG),)
@@ -24,11 +23,7 @@ linter:
 check: linter test
 
 test:
-ifeq ($(CRON), cron)
-	go test $(RACE_FLAG) -tags=mock_test_off -coverprofile=coverage.txt -covermode=atomic  ./...
-else
 	go test $(RACE_FLAG) -coverprofile=coverage.txt -covermode=atomic  ./...
-endif
 
 build:
 	go build $(LDFLAGS)

cmd/exchange_wrapper_standards/exchange_wrapper_standards_test.go

@@ -30,7 +30,7 @@ import (
 
 func TestMain(m *testing.M) {
 	// only run testing suite for one CI/CD environment
-	if isAppVeyor() || is32BitJob() {
+	if skipAdditionalWrapperCITests() {
 		return
 	}
 	request.MaxRequestJobs = 200
@@ -754,16 +754,9 @@ Rsd80LrBCVI8ctzrvYRFSugC`
 }
 
 func isCITest() bool {
-	ci := os.Getenv("CI")
+	return os.Getenv("CI") == "true"
-	return ci == "true" /* github actions */ || ci == "True" /* appveyor */
 }
 
-func isAppVeyor() bool {
+func skipAdditionalWrapperCITests() bool {
-	ci := os.Getenv("APPVEYOR")
+	return os.Getenv("SKIP_WRAPPER_CI_TESTS") == "true"
-	return ci == "True"
-}
-
-func is32BitJob() bool {
-	ci := os.Getenv("GITHUB_JOB")
-	return ci == "backend-32bit"
 }

codecov.yml

@@ -1,19 +1,19 @@
 codecov: 
-    require_ci_to_pass: true
+  require_ci_to_pass: true
-  comment: 
+comment: 
-    behavior: default
+  behavior: default
-    layout: "reach,diff,flags,tree"
+  layout: "reach,diff,flags,tree"
-    require_changes: false
+  require_changes: false
-  coverage: 
+coverage: 
-    precision: 2
+  precision: 2
-    range: "70...100"
+  range: "70...100"
-    round: down
+  round: down
-  github_checks: 
+github_checks: 
-    annotations: false
+  annotations: false
-  parsers: 
+parsers: 
-    gcov: 
+  gcov: 
-      branch_detection: 
+    branch_detection: 
-        conditional: true
+      conditional: true
-        loop: true
+      loop: true
-        macro: false
+      macro: false
-        method: false
+      method: false

codelingo.yaml

@@ -1,39 +0,0 @@
-tenets:
-  # Import effective Go bundle manually
-  - import: codelingo/effective-go/avoid-annotations-in-comments
-  - import: codelingo/effective-go/comment-first-word-as-subject
-  - import: codelingo/effective-go/good-package-name
-  - import: codelingo/effective-go/single-method-interface-name
-  - import: codelingo/effective-go/underscores-in-name
-  - import: codelingo/effective-go/unnecessary-else
-  - import: codelingo/code-review-comments/declare-empty-slice 
-  - import: codelingo/effective-go/defer-close-file
-  # - import: codelingo/effective-go/comment-first-word-when-empty # this has been disabled temporarily
-  - name:  missing-stop-ticker
-    actions:
-      codelingo/review:
-        comment: Add `defer {{varName}}.Stop()` to stop the ticker and release associated resources.
-    query: |
-      import codelingo/ast/go
-
-      go.func_decl(depth = any):
-        @review comment
-        go.assign_stmt(depth = any):
-          go.lhs:
-            go.ident:
-              sibling_order == 0
-              name as varName
-          go.rhs:
-            go.call_expr:
-              go.selector_expr:
-                go.ident:
-                  name == "time"
-                go.ident:
-                  name == "NewTicker"
-        exclude:
-          go.call_expr(depth = any):
-            go.selector_expr:
-              go.ident:
-                name ==  varName
-              go.ident:
-                name == "Stop"

database/testhelpers/test_helpers.go

@@ -2,7 +2,6 @@ package testhelpers
 
 import (
 	"database/sql"
-	"os"
 	"path/filepath"
 	"reflect"
 
@@ -26,38 +25,6 @@ var (
 
 // GetConnectionDetails returns connection details for CI or test db instances
 func GetConnectionDetails() *database.Config {
-	_, exists := os.LookupEnv("TRAVIS")
-	if exists {
-		return &database.Config{
-			Enabled: true,
-			Driver:  "postgres",
-			ConnectionDetails: drivers.ConnectionDetails{
-				Host:     "localhost",
-				Port:     5432,
-				Username: "postgres",
-				Password: "",
-				Database: "gct_dev_ci",
-				SSLMode:  "",
-			},
-		}
-	}
-
-	_, exists = os.LookupEnv("APPVEYOR")
-	if exists {
-		return &database.Config{
-			Enabled: true,
-			Driver:  "postgres",
-			ConnectionDetails: drivers.ConnectionDetails{
-				Host:     "localhost",
-				Port:     5432,
-				Username: "postgres",
-				Password: "Password12!",
-				Database: "gct_dev_ci",
-				SSLMode:  "",
-			},
-		}
-	}
-
 	return &database.Config{
 		Enabled:           true,
 		Driver:            "postgres",

engine/engine_test.go

@@ -20,8 +20,7 @@ var blockedCIExchanges = []string{
 }
 
 func isCITest() bool {
-	ci := os.Getenv("CI")
+	return os.Getenv("CI") == "true"
-	return ci == "true" /* github actions */ || ci == "True" /* appveyor */
 }
 
 func TestLoadConfigWithSettings(t *testing.T) {