package main import ( "context" "errors" "flag" "fmt" "log" "net/http" "net/url" "os" "path/filepath" "slices" "sort" "strconv" "strings" "text/template" "time" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/common/file" "github.com/thrasher-corp/gocryptotrader/core" "github.com/thrasher-corp/gocryptotrader/encoding/json" "golang.org/x/text/cases" "golang.org/x/text/language" ) const ( // DefaultRepo is the main example repository DefaultRepo = "https://api.github.com/repos/thrasher-corp/gocryptotrader" // GithubAPIEndpoint allows the program to query your repository // contributor list GithubAPIEndpoint = "/contributors" // LicenseFile defines a license file LicenseFile = "LICENSE" // ContributorFile defines contributor file ContributorFile = "CONTRIBUTORS" defaultGithubAPIPerPageLimit = 100 ) var ( // DefaultExcludedDirectories defines the basic directory exclusion list for GCT DefaultExcludedDirectories = []string{ ".github", ".git", "node_modules", ".vscode", ".idea", "cmd_templates", "common_templates", "communications_templates", "config_templates", "currency_templates", "events_templates", "exchanges_templates", "portfolio_templates", "root_templates", "sub_templates", "testdata_templates", "tools_templates", "web_templates", } // global flag for verbosity verbose bool // current tool directory to specify working templates toolDir string // exposes root directory if outside of document tool directory repoDir string // is a broken down version of the documentation tool dir for cross platform // checking ref = []string{"gocryptotrader", "cmd", "documentation"} engineFolder = "engine" githubToken = os.Getenv("GITHUB_TOKEN") // Overridden by the ghtoken flag when set ) // Contributor defines an account associated with this code base by doing // contributions type Contributor struct { Login string `json:"login"` URL string `json:"html_url"` Contributions int `json:"contributions"` } // ghError defines a GitHub error response type ghError struct { Message string `json:"message"` Status string `json:"status"` } // Config defines the running config to deploy documentation across a github // repository including exclusion lists for files and directories type Config struct { GithubRepo string `json:"githubRepo"` Exclusions Exclusions `json:"exclusionList"` RootReadme bool `json:"rootReadmeActive"` LicenseFile bool `json:"licenseFileActive"` ContributorFile bool `json:"contributorFileActive"` } // Exclusions defines the exclusion list so documents are not generated type Exclusions struct { Files []string `json:"Files"` Directories []string `json:"Directories"` } // DocumentationDetails defines parameters to update documentation type DocumentationDetails struct { Directories []string Tmpl *template.Template Contributors []Contributor Config *Config } // Attributes defines specific documentation attributes when a template is // executed type Attributes struct { Name string Contributors []Contributor NameURL string Year int CapitalName string DonationAddress string } func main() { flag.BoolVar(&verbose, "v", false, "Verbose output") flag.StringVar(&toolDir, "tooldir", "", "Pass in the documentation tool directory if outside tool folder") flag.StringVar(&githubToken, "ghtoken", githubToken, "Github authentication token to use when fetching the contributors list") flag.Parse() wd, err := os.Getwd() if err != nil { fmt.Println("Documentation tool error cannot get working dir:", err) os.Exit(1) } if strings.Contains(wd, filepath.Join(ref...)) { rootDir := filepath.Dir(filepath.Dir(wd)) repoDir = rootDir toolDir = wd } else { if toolDir == "" { fmt.Println("Please set documentation tool directory via the tooldir flag if working outside of tool directory") os.Exit(1) } repoDir = wd } fmt.Print(core.Banner) fmt.Println("This will update and regenerate documentation for the different packages in your repo.") fmt.Println() if verbose { fmt.Println("Fetching configuration...") } config, err := GetConfiguration() if err != nil { log.Fatalf("Documentation Generation Tool - GetConfiguration error %s", err) } if verbose { fmt.Println("Fetching project directory tree...") } dirList, err := GetProjectDirectoryTree(&config) if err != nil { log.Fatalf("Documentation Generation Tool - GetProjectDirectoryTree error %s", err) } var contributors []Contributor if config.ContributorFile { if verbose { fmt.Println("Fetching repository contributor list...") } contributors, err = GetContributorList(context.TODO(), config.GithubRepo, verbose) if err != nil { log.Fatalf("Documentation Generation Tool - GetContributorList error: %s", err) } // Github API missing/deleted user contributors contributors = append(contributors, []Contributor{ // idoall's contributors were forked and merged, so his contributions // aren't automatically retrievable { Login: "idoall", URL: "https://github.com/idoall", Contributions: 1, }, { Login: "starit", URL: "https://github.com/starit", Contributions: 1, }, }...) sort.Slice(contributors, func(i, j int) bool { return contributors[i].Contributions > contributors[j].Contributions }) if verbose { fmt.Println("Contributor List Fetched") for i := range contributors { fmt.Println(contributors[i].Login) } } } else { fmt.Println("Contributor list file disabled skipping fetching details") } if verbose { fmt.Println("Fetching template files...") } tmpl, err := GetTemplateFiles() if err != nil { log.Fatalf("Documentation Generation Tool - GetTemplateFiles error %s", err) } if verbose { fmt.Println("All core systems fetched, updating documentation...") } UpdateDocumentation(DocumentationDetails{ dirList, tmpl, contributors, &config, }) fmt.Println("\nDocumentation Generation Tool - Finished") } // GetConfiguration retrieves the documentation configuration func GetConfiguration() (Config, error) { var c Config configFilePath := filepath.Join(toolDir, "config.json") if file.Exists(configFilePath) { config, err := os.ReadFile(configFilePath) if err != nil { return c, err } err = json.Unmarshal(config, &c) if err != nil { return c, err } if c.GithubRepo == "" { return c, errors.New("repository not set in config.json file, please change") } return c, nil } fmt.Println("Creating configuration file, please check to add a different github repository path and change preferences") // Set default params for configuration c.GithubRepo = DefaultRepo c.ContributorFile = true c.LicenseFile = true c.RootReadme = true c.Exclusions.Directories = DefaultExcludedDirectories data, err := json.MarshalIndent(c, "", " ") if err != nil { return c, err } if err := os.WriteFile(configFilePath, data, file.DefaultPermissionOctal); err != nil { return c, err } return c, nil } // GetProjectDirectoryTree uses filepath walk functions to get each individual // directory name and path to match templates with func GetProjectDirectoryTree(c *Config) ([]string, error) { var directoryData []string if c.RootReadme { // Projects root README.md directoryData = append(directoryData, repoDir) } if c.LicenseFile { // Standard license file directoryData = append(directoryData, filepath.Join(repoDir, LicenseFile)) } if c.ContributorFile { // Standard contributor file directoryData = append(directoryData, filepath.Join(repoDir, ContributorFile)) } walkFn := func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { // Bypass what is contained in config.json directory exclusion if slices.Contains(c.Exclusions.Directories, info.Name()) { if verbose { fmt.Println("Excluding Directory:", info.Name()) } return filepath.SkipDir } // Don't append parent directory if strings.EqualFold(info.Name(), "..") { return nil } directoryData = append(directoryData, path) } return nil } return directoryData, filepath.Walk(repoDir, walkFn) } // GetTemplateFiles parses and returns all template files in the documentation // tree func GetTemplateFiles() (*template.Template, error) { tmpl := template.New("") walkFn := func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { if path == "." || path == ".." { return nil } var tmplExt *template.Template tmplExt, err = tmpl.ParseGlob(filepath.Join(path, "*.tmpl")) if err != nil { fmt.Println(err) if strings.Contains(err.Error(), "pattern matches no files") { return nil } return err } tmpl = tmplExt return filepath.SkipDir } return nil } return tmpl, filepath.Walk(toolDir, walkFn) } // GetContributorList fetches a list of contributors from the Github API endpoint func GetContributorList(ctx context.Context, repo string, verbose bool) ([]Contributor, error) { var contributors []Contributor vals := url.Values{} vals.Set("per_page", strconv.Itoa(defaultGithubAPIPerPageLimit)) headers := make(map[string]string) if githubToken != "" { headers["Authorization"] = "Bearer " + githubToken fmt.Println("Using GitHub token for authentication") } for page := 1; ; page++ { vals.Set("page", strconv.Itoa(page)) contents, err := common.SendHTTPRequest(ctx, http.MethodGet, common.EncodeURLValues(repo+GithubAPIEndpoint, vals), headers, nil, verbose) if err != nil { return nil, err } var g ghError if err := json.Unmarshal(contents, &g); err == nil && g.Message != "" { return nil, fmt.Errorf("GitHub error message: %q Status: %s", g.Message, g.Status) } var resp []Contributor if err := json.Unmarshal(contents, &resp); err != nil { return nil, err } contributors = append(contributors, resp...) if len(resp) < defaultGithubAPIPerPageLimit { return contributors, nil } } } // GetDocumentationAttributes returns specific attributes for a file template func GetDocumentationAttributes(packageName string, contributors []Contributor) Attributes { return Attributes{ Name: GetPackageName(packageName, false), Contributors: contributors, NameURL: GetGoDocURL(packageName), Year: time.Now().Year(), CapitalName: GetPackageName(packageName, true), DonationAddress: core.BitcoinDonationAddress, } } // GetPackageName returns the package name after cleaning path as a string func GetPackageName(name string, capital bool) string { newStrings := strings.Split(name, " ") var i int if len(newStrings) > 1 { // retrieve the latest spacing to define the most childish package name i = len(newStrings) - 1 } if capital { return cases.Title(language.English).String(strings.ReplaceAll(newStrings[i], "_", " ")) } return newStrings[i] } // GetGoDocURL returns a string for godoc package names func GetGoDocURL(name string) string { if strings.Contains(name, " ") { return strings.Join(strings.Split(name, " "), "/") } if name == "testdata" || name == "tools" || name == ContributorFile || name == LicenseFile { return "" } return name } // UpdateDocumentation generates or updates readme/documentation files across // the codebase func UpdateDocumentation(details DocumentationDetails) { for i := range details.Directories { cutSet := details.Directories[i][len(repoDir):] if cutSet != "" && cutSet[0] == os.PathSeparator { cutSet = cutSet[1:] } data := strings.Split(cutSet, string(os.PathSeparator)) var temp []string for x := range data { if data[x] == ".." { continue } if data[x] == "" { break } temp = append(temp, data[x]) } var name string if len(temp) == 0 { name = "root" } else { name = strings.Join(temp, " ") } if slices.Contains(details.Config.Exclusions.Files, name) { if verbose { fmt.Println("Excluding file:", name) } continue } if strings.Contains(name, engineFolder) { d, err := os.ReadDir(details.Directories[i]) if err != nil { fmt.Println("Excluding file:", err) } for x := range d { nameSplit := strings.Split(d[x].Name(), ".go") engineTemplateName := engineFolder + " " + nameSplit[0] if details.Tmpl.Lookup(engineTemplateName) == nil { fmt.Printf("Template not found for path %s create new template with {{define \"%s\" -}} TEMPLATE HERE {{end}}\n", details.Directories[i], name) continue } err = runTemplate(details, filepath.Join(details.Directories[i], nameSplit[0]+".md"), engineTemplateName) if err != nil { fmt.Println(err) } } continue } if details.Tmpl.Lookup(name) == nil { fmt.Printf("Template not found for path %s create new template with {{define \"%s\" -}} TEMPLATE HERE {{end}}\n", details.Directories[i], name) continue } var mainPath string switch name { case LicenseFile, ContributorFile: mainPath = details.Directories[i] default: mainPath = filepath.Join(details.Directories[i], "README.md") } if err := runTemplate(details, mainPath, name); err != nil { log.Println(err) continue } } } func runTemplate(details DocumentationDetails, mainPath, name string) error { err := os.Remove(mainPath) if err != nil && !os.IsNotExist(err) { return err } f, err := os.Create(mainPath) if err != nil { return err } defer func(f *os.File) { err := f.Close() if err != nil { log.Printf("could not close file %s: %v", mainPath, err) } }(f) attr := GetDocumentationAttributes(name, details.Contributors) return details.Tmpl.ExecuteTemplate(f, name, attr) }