PageRenderTime 50ms CodeModel.GetById 19ms RepoModel.GetById 1ms app.codeStats 0ms

/vendor/github.com/Azure/azure-sdk-for-go/tools/generator/program.go

https://gitlab.com/unofficial-mirrors/openshift-origin
Go | 345 lines | 239 code | 51 blank | 55 comment | 40 complexity | 89eb8ce5c5c36aaf0b1b31a811c3aafc MD5 | raw file
  1. // Copyright 2017 Microsoft Corporation and contributors. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. // http://www.apache.org/licenses/LICENSE-2.0
  7. //
  8. // Unless required by applicable law or agreed to in writing, software
  9. // distributed under the License is distributed on an "AS IS" BASIS,
  10. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. //
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // generator scours the github.com/Azure/azure-rest-api-specs repository in search for
  15. // REST functionality that has Go configuration, then generates packages accordingly.
  16. // This tool was developed with the intention that it be an internal tool for the
  17. // Azure-SDK-for-Go team, but the usage of this package for public scenarios is in no
  18. // way prohibited. For example, have a fork of https://github.com/Azure/autorest.go?
  19. // Generate out and SDK with the same shape as our SDK, but using your patterns using
  20. // this tool.
  21. //
  22. // Given that this code was developed as an internal tool, troubles with it that do not
  23. // pertain to our usage of it may be slow to be fixed. Pull Requests are welcome though,
  24. // feel free to contribute.
  25. package main
  26. import (
  27. "bytes"
  28. "flag"
  29. "fmt"
  30. "io/ioutil"
  31. "log"
  32. "os"
  33. "os/exec"
  34. "path"
  35. "path/filepath"
  36. "regexp"
  37. "strings"
  38. "time"
  39. "github.com/marstr/collection"
  40. )
  41. const (
  42. defaultGenVer = "@microsoft.azure/autorest.go@v2.1.62"
  43. )
  44. var (
  45. targetFiles collection.Enumerable
  46. packageVersion string
  47. outputBase string
  48. logDirBase string
  49. autorestVer string
  50. errLog *log.Logger
  51. statusLog *log.Logger
  52. dryRun bool
  53. )
  54. // version should be set by the linker when compiling this program by providing the following arguments:
  55. // -X main.version=<version>
  56. //
  57. // If installing the generator in your machine, that means running the following the command:
  58. // go install -ldflags "-X main.version=<version>"
  59. //
  60. // The reason this is not controlled in source, is to allow for the git commit SHA1 to be used as the
  61. // version of the string. To retrieve the currently checked-out git commit SHA1, you can use the command:
  62. // git rev-parse HEAD
  63. //
  64. // If this value is set, it is recommended that it be the value of the SHA1 git commit identifier for the
  65. // source that was used at build time.
  66. var version string
  67. func main() {
  68. start := time.Now()
  69. type generationTuple struct {
  70. fileName string
  71. packageName string
  72. outputFolder string
  73. }
  74. literateFiles := collection.Where(targetFiles, func(subject interface{}) bool {
  75. return strings.EqualFold(path.Base(subject.(string)), "README.md")
  76. })
  77. tuples := collection.SelectMany(literateFiles,
  78. // The following function compiles the regexp which finds Go related package tags in a literate file, and creates a collection.Unfolder.
  79. // This function has been declared this way so that the relatively expensive act of compiling a regexp is only done once.
  80. func() collection.Unfolder {
  81. const patternText = "```" + `\s+yaml\s+\$\(\s*tag\s*\)\s*==\s*'([\d\w\-\.]+)'\s*&&\s*\$\(go\)[\w\d\-\s:]*\s+output-folder:\s+\$\(go-sdk-folder\)([\w\d\-_\\/\.]+)\s+[\w\d\-\s:]*` + "```"
  82. goConfigPattern := regexp.MustCompile(patternText)
  83. // This function is a collection.Unfolder which takes a literate file as a path, and retrieves all configuration which applies to a package tag and Go.
  84. return func(subject interface{}) collection.Enumerator {
  85. results := make(chan interface{})
  86. go func() {
  87. defer close(results)
  88. targetContents, err := ioutil.ReadFile(subject.(string))
  89. if err != nil {
  90. errLog.Printf("Skipping %q because: %v", subject.(string), err)
  91. return
  92. }
  93. matches := goConfigPattern.FindAllStringSubmatch(string(targetContents), -1)
  94. if len(matches) == 0 {
  95. statusLog.Printf("Skipping %q because there were no package tags with go configuration found.", subject.(string))
  96. } else {
  97. for _, submatch := range matches {
  98. results <- generationTuple{
  99. fileName: subject.(string),
  100. packageName: normalizePath(submatch[1]),
  101. outputFolder: normalizePath(submatch[2]),
  102. }
  103. }
  104. }
  105. }()
  106. return results
  107. }
  108. }())
  109. if dryRun {
  110. for entry := range tuples.Enumerate(nil) {
  111. tuple := entry.(generationTuple)
  112. fmt.Printf("%q in %q to %q\n", tuple.packageName, tuple.fileName, tuple.outputFolder)
  113. }
  114. } else {
  115. var generatedCount, formattedCount, builtCount, vettedCount uint
  116. done := make(chan struct{})
  117. // Call AutoRest for each of the tags in each of the literate files, generating a Go package for calling that service.
  118. generated := tuples.Enumerate(done).ParallelSelect(func(subject interface{}) interface{} {
  119. tuple := subject.(generationTuple)
  120. args := []string{
  121. tuple.fileName,
  122. "--go",
  123. fmt.Sprintf("--go-sdk-folder='%s'", outputBase),
  124. "--verbose",
  125. "--tag=" + tuple.packageName,
  126. "--use=" + autorestVer,
  127. }
  128. if packageVersion != "" {
  129. args = append(args, fmt.Sprintf("--package-version='%s'", packageVersion))
  130. args = append(args, fmt.Sprintf("--user-agent='Azure-SDK-For-Go/%s services'", packageVersion))
  131. }
  132. logFileLoc := filepath.Join(logDirBase, tuple.outputFolder)
  133. err := os.MkdirAll(logFileLoc, os.ModePerm)
  134. if err != nil {
  135. errLog.Printf("Could not create log directory %q", logFileLoc)
  136. return nil
  137. }
  138. logFile, err := os.Create(filepath.Join(logFileLoc, "autorestLog.txt"))
  139. if err != nil {
  140. errLog.Printf("Could not create log file %q for AutoRest generating from: %q in %q", logFileLoc, tuple.packageName, tuple.fileName)
  141. return nil
  142. }
  143. commandText := new(bytes.Buffer)
  144. fmt.Fprint(commandText, "Executing Command: \"")
  145. fmt.Fprint(commandText, "autorest ")
  146. for _, a := range args {
  147. fmt.Fprint(commandText, a)
  148. fmt.Fprint(commandText, " ")
  149. }
  150. commandText.Truncate(commandText.Len() - 1)
  151. fmt.Fprint(commandText, `"`)
  152. fmt.Fprintln(logFile, commandText.String())
  153. genProc := exec.Command("autorest", args...)
  154. genProc.Stdout = logFile
  155. genProc.Stderr = logFile
  156. err = genProc.Run()
  157. if err != nil {
  158. fmt.Fprintln(logFile, "Autorest Exectution Error: ", err)
  159. return nil
  160. }
  161. generatedCount++
  162. return path.Join(outputBase, tuple.outputFolder)
  163. }).Where(isntNil)
  164. // Take all of the generated packages and reformat them to adhere to Go's guidelines.
  165. formatted := generated.Select(func(subject interface{}) (result interface{}) {
  166. err := exec.Command("gofmt", "-w", subject.(string)).Run()
  167. if err == nil {
  168. formattedCount++
  169. result = subject
  170. } else {
  171. errLog.Printf("Failed to format: %q", subject.(string))
  172. }
  173. return
  174. }).Where(isntNil)
  175. // Build all of the packages as a sanity check.
  176. built := formatted.Select(func(subject interface{}) (result interface{}) {
  177. pkgName := strings.TrimPrefix(trimGoPath(subject.(string)), "/src/")
  178. err := exec.Command("go", "build", pkgName).Run()
  179. if err == nil {
  180. builtCount++
  181. return pkgName
  182. }
  183. errLog.Printf("Failed to build: %q", pkgName)
  184. return nil
  185. }).Where(isntNil)
  186. vetted := built.Select(func(subject interface{}) interface{} {
  187. err := exec.Command("go", "vet", subject.(string)).Run()
  188. if err == nil {
  189. vettedCount++
  190. return subject
  191. }
  192. errLog.Printf("Failed to vet: %q", subject.(string))
  193. return nil
  194. }).Where(isntNil)
  195. // Turn the crank. This loop forces evaluation of the chain of enumerators that were built up
  196. // in the code above.
  197. for range vetted {
  198. // Intenionally Left Blank
  199. }
  200. fmt.Println("Execution Time: ", time.Now().Sub(start))
  201. fmt.Println("Generated: ", generatedCount)
  202. fmt.Println("Formatted: ", formattedCount)
  203. fmt.Println("Built: ", builtCount)
  204. fmt.Println("Vetted: ", vettedCount)
  205. close(done)
  206. }
  207. }
  208. func init() {
  209. var useRecursive, useStatus bool
  210. const automaticLogDirValue = "temp"
  211. const logDirUsage = "The root directory where all logs can be found. If the value `" + automaticLogDirValue + "` is supplied, a randomly named directory will be generated in the $TEMP directory."
  212. flag.BoolVar(&useRecursive, "r", false, "Recursively traverses the directories specified looking for literate files.")
  213. flag.StringVar(&outputBase, "o", getDefaultOutputBase(), "The root directory to use for the output of generated files. i.e. The value to be treated as the go-sdk-folder when AutoRest is called.")
  214. flag.BoolVar(&useStatus, "v", false, "Print status messages as generation takes place.")
  215. flag.BoolVar(&dryRun, "p", false, "Preview which packages would be generated instead of actaully calling autorest.")
  216. flag.StringVar(&packageVersion, "version", "", "The version that should be stamped on this SDK. This should be a semver.")
  217. flag.StringVar(&logDirBase, "l", getDefaultOutputBase(), logDirUsage)
  218. flag.StringVar(&autorestVer, "a", defaultGenVer, "The version of the AutoRest Go code generator to use, defaults to `"+defaultGenVer+"`.")
  219. // Override the default usage message, printing to stderr as the default one would.
  220. flag.Usage = func() {
  221. fmt.Fprintln(os.Stderr, "generator [options] [target Literate files]...")
  222. flag.PrintDefaults()
  223. }
  224. flag.Parse()
  225. statusWriter := ioutil.Discard
  226. if useStatus {
  227. statusWriter = os.Stdout
  228. }
  229. statusLog = log.New(statusWriter, "[STATUS] ", 0)
  230. errLog = log.New(os.Stderr, "[ERROR] ", 0)
  231. if logDirBase == automaticLogDirValue {
  232. var err error
  233. logDirBase, err = ioutil.TempDir("", "az-go-sdk-logs")
  234. logDirBase = normalizePath(logDirBase)
  235. if err == nil {
  236. statusLog.Print("Generation logs can be found at: ", logDirBase)
  237. } else {
  238. errLog.Print("Logging disabled. Could not create directory: ", logDirBase)
  239. }
  240. }
  241. targetFiles = collection.AsEnumerable(flag.Args())
  242. targetFiles = collection.SelectMany(targetFiles, func(subject interface{}) collection.Enumerator {
  243. cast, ok := subject.(string)
  244. if !ok {
  245. return collection.Empty.Enumerate(nil)
  246. }
  247. pathInfo, err := os.Stat(cast)
  248. if err != nil {
  249. return collection.Empty.Enumerate(nil)
  250. }
  251. if pathInfo.IsDir() {
  252. traverser := collection.Directory{
  253. Location: cast,
  254. Options: collection.DirectoryOptionsExcludeDirectories,
  255. }
  256. if useRecursive {
  257. traverser.Options |= collection.DirectoryOptionsRecursive
  258. }
  259. return traverser.Enumerate(nil)
  260. }
  261. return collection.AsEnumerable(cast).Enumerate(nil)
  262. })
  263. targetFiles = collection.Select(targetFiles, func(subject interface{}) interface{} {
  264. return normalizePath(subject.(string))
  265. })
  266. }
  267. // goPath returns the value of the environement variable GOPATH at the beginning of this
  268. // program's execution. The actual enviroment variable is only queried once, before any calls
  269. // are made to this function.
  270. var goPath = func() func() string {
  271. val := normalizePath(os.Getenv("GOPATH"))
  272. return func() string {
  273. return val
  274. }
  275. }()
  276. // getDefaultOutputBase returns the default location of the Azure-SDK-for-Go on your filesystem.
  277. func getDefaultOutputBase() string {
  278. return normalizePath(path.Join(goPath(), "src", "github.com", "Azure", "azure-sdk-for-go"))
  279. }
  280. // trimGoPath operates like strings.TrimPrefix, where the prefix is always the value of GOPATH in the
  281. // environment in which this program is being executed.
  282. func trimGoPath(location string) string {
  283. return strings.TrimPrefix(normalizePath(location), goPath())
  284. }
  285. // normalizePath ensures that a path is expressed using forward slashes for sanity when working
  286. // with Go Standard Library functions that seem to expect this.
  287. func normalizePath(location string) (result string) {
  288. result = strings.Replace(location, `\`, "/", -1)
  289. return
  290. }
  291. // isntNil is a simple `collection.Predicate` which filters out `nil` objects from an Enumerator.
  292. func isntNil(subject interface{}) bool {
  293. return subject != nil
  294. }