The chatops bot of aventer
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

app.go 7.9KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. package main
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "io/ioutil"
  6. "net/http"
  7. _ "net/http/pprof"
  8. "path/filepath"
  9. "./api"
  10. "./api/handlers"
  11. "./clients"
  12. "./database"
  13. "./polling"
  14. _ "./realms/github"
  15. _ "./services/aws"
  16. _ "./services/echo"
  17. _ "./services/gitea"
  18. _ "./services/github"
  19. _ "./services/invoice"
  20. _ "./services/nlp"
  21. _ "./services/pentest"
  22. _ "./services/travisci"
  23. _ "./services/wekan"
  24. "./types"
  25. "git.aventer.biz/AVENTER/util"
  26. "github.com/matrix-org/dugong"
  27. _ "github.com/mattn/go-sqlite3"
  28. "github.com/prometheus/client_golang/prometheus"
  29. log "github.com/sirupsen/logrus"
  30. yaml "gopkg.in/yaml.v2"
  31. )
  32. // BindAddress is the Bind Address of the bot
  33. var BindAddress string
  34. // DatabaseType is by default sqlite3
  35. var DatabaseType string
  36. // DatabaseURL is the url of the database :-)
  37. var DatabaseURL string
  38. // BaseURL is the url format of the database query
  39. var BaseURL string
  40. // ConfigFile is the bots config file in yaml format
  41. var ConfigFile string
  42. // LogDir is the directory where the bot will log in
  43. var LogDir string
  44. // MinVersion is the BuildVersion Number
  45. var MinVersion string
  46. // loadFromConfig loads a config file and returns a ConfigFile
  47. func loadFromConfig(db *database.ServiceDB, configFilePath string) (*api.ConfigFile, error) {
  48. // ::Horrible hacks ahead::
  49. // The config is represented as YAML, and we want to convert that into NEB types.
  50. // However, NEB types make liberal use of json.RawMessage which the YAML parser
  51. // doesn't like. We can't implement MarshalYAML/UnmarshalYAML as a custom type easily
  52. // because YAML is insane and supports numbers as keys. The YAML parser therefore has the
  53. // generic form of map[interface{}]interface{} - but the JSON parser doesn't know
  54. // how to parse that.
  55. //
  56. // The hack that follows gets around this by type asserting all parsed YAML keys as
  57. // strings then re-encoding/decoding as JSON. That is:
  58. // YAML bytes -> map[interface]interface -> map[string]interface -> JSON bytes -> NEB types
  59. // Convert to YAML bytes
  60. contents, err := ioutil.ReadFile(configFilePath)
  61. if err != nil {
  62. return nil, err
  63. }
  64. // Convert to map[interface]interface
  65. var cfg map[interface{}]interface{}
  66. if err = yaml.Unmarshal(contents, &cfg); err != nil {
  67. return nil, fmt.Errorf("Failed to unmarshal YAML: %s", err)
  68. }
  69. // Convert to map[string]interface
  70. dict := convertKeysToStrings(cfg)
  71. // Convert to JSON bytes
  72. b, err := json.Marshal(dict)
  73. if err != nil {
  74. return nil, fmt.Errorf("Failed to marshal config as JSON: %s", err)
  75. }
  76. // Finally, Convert to NEB types
  77. var c api.ConfigFile
  78. if err := json.Unmarshal(b, &c); err != nil {
  79. return nil, fmt.Errorf("Failed to convert to config file: %s", err)
  80. }
  81. // sanity check (at least 1 client and 1 service)
  82. if len(c.Clients) == 0 || len(c.Services) == 0 {
  83. return nil, fmt.Errorf("At least 1 client and 1 service must be specified")
  84. }
  85. return &c, nil
  86. }
  87. func convertKeysToStrings(iface interface{}) interface{} {
  88. obj, isObj := iface.(map[interface{}]interface{})
  89. if isObj {
  90. strObj := make(map[string]interface{})
  91. for k, v := range obj {
  92. strObj[k.(string)] = convertKeysToStrings(v) // handle nested objects
  93. }
  94. return strObj
  95. }
  96. arr, isArr := iface.([]interface{})
  97. if isArr {
  98. for i := range arr {
  99. arr[i] = convertKeysToStrings(arr[i]) // handle nested objects
  100. }
  101. return arr
  102. }
  103. return iface // base type like string or number
  104. }
  105. func insertServicesFromConfig(clis *clients.Clients, serviceReqs []api.ConfigureServiceRequest) error {
  106. for i, s := range serviceReqs {
  107. if err := s.Check(); err != nil {
  108. return fmt.Errorf("config: Service[%d] : %s", i, err)
  109. }
  110. service, err := types.CreateService(s.ID, s.Type, s.UserID, s.Config)
  111. if err != nil {
  112. return fmt.Errorf("config: Service[%d] : %s", i, err)
  113. }
  114. // Fetch the client for this service and register/poll
  115. c, err := clis.Client(s.UserID)
  116. if err != nil {
  117. return fmt.Errorf("config: Service[%d] : %s", i, err)
  118. }
  119. if err = service.Register(nil, c); err != nil {
  120. return fmt.Errorf("config: Service[%d] : %s", i, err)
  121. }
  122. if _, err := database.GetServiceDB().StoreService(service); err != nil {
  123. return fmt.Errorf("config: Service[%d] : %s", i, err)
  124. }
  125. service.PostRegister(nil)
  126. }
  127. return nil
  128. }
  129. func loadDatabase(databaseType, databaseURL, configYAML string) (*database.ServiceDB, error) {
  130. if configYAML != "" {
  131. databaseType = "sqlite3"
  132. databaseURL = ":memory:?_busy_timeout=5000"
  133. }
  134. db, err := database.Open(databaseType, databaseURL)
  135. if err == nil {
  136. database.SetServiceDB(db) // set singleton
  137. }
  138. return db, err
  139. }
  140. func setup(mux *http.ServeMux, matrixClient *http.Client) {
  141. err := types.BaseURL(BaseURL)
  142. if err != nil {
  143. log.WithError(err).Panic("Failed to get base url")
  144. }
  145. db, err := loadDatabase(DatabaseType, DatabaseURL, ConfigFile)
  146. if err != nil {
  147. log.WithError(err).Panic("Failed to open database")
  148. }
  149. // Populate the database from the config file if one was supplied.
  150. var cfg *api.ConfigFile
  151. if ConfigFile != "" {
  152. if cfg, err = loadFromConfig(db, ConfigFile); err != nil {
  153. log.WithError(err).WithField("config_file", ConfigFile).Panic("Failed to load config file")
  154. }
  155. if err := db.InsertFromConfig(cfg); err != nil {
  156. log.WithError(err).Panic("Failed to persist config data into in-memory DB")
  157. }
  158. log.Info("Inserted ", len(cfg.Clients), " clients")
  159. log.Info("Inserted ", len(cfg.Realms), " realms")
  160. log.Info("Inserted ", len(cfg.Sessions), " sessions")
  161. }
  162. clients := clients.New(db, matrixClient)
  163. if err := clients.Start(); err != nil {
  164. log.WithError(err).Panic("Failed to start up clients")
  165. }
  166. // Handle non-admin paths for normal NEB functioning
  167. mux.Handle("/metrics", prometheus.Handler())
  168. mux.Handle("/test", prometheus.InstrumentHandler("test", util.MakeJSONAPI(&handlers.Heartbeat{})))
  169. wh := handlers.NewWebhook(db, clients)
  170. mux.HandleFunc("/services/hooks/", prometheus.InstrumentHandlerFunc("webhookHandler", util.Protect(wh.Handle)))
  171. rh := &handlers.RealmRedirect{db}
  172. mux.HandleFunc("/realms/redirects/", prometheus.InstrumentHandlerFunc("realmRedirectHandler", util.Protect(rh.Handle)))
  173. // Read exclusively from the config file if one was supplied.
  174. // Otherwise, add HTTP listeners for new Services/Sessions/Clients/etc.
  175. if ConfigFile != "" {
  176. if err := insertServicesFromConfig(clients, cfg.Services); err != nil {
  177. log.WithError(err).Panic("Failed to insert services")
  178. }
  179. log.Info("Inserted ", len(cfg.Services), " services")
  180. } else {
  181. mux.Handle("/admin/getService", prometheus.InstrumentHandler("getService", util.MakeJSONAPI(&handlers.GetService{db})))
  182. mux.Handle("/admin/getSession", prometheus.InstrumentHandler("getSession", util.MakeJSONAPI(&handlers.GetSession{db})))
  183. mux.Handle("/admin/configureClient", prometheus.InstrumentHandler("configureClient", util.MakeJSONAPI(&handlers.ConfigureClient{clients})))
  184. mux.Handle("/admin/configureService", prometheus.InstrumentHandler("configureService", util.MakeJSONAPI(handlers.NewConfigureService(db, clients))))
  185. mux.Handle("/admin/configureAuthRealm", prometheus.InstrumentHandler("configureAuthRealm", util.MakeJSONAPI(&handlers.ConfigureAuthRealm{db})))
  186. mux.Handle("/admin/requestAuthSession", prometheus.InstrumentHandler("requestAuthSession", util.MakeJSONAPI(&handlers.RequestAuthSession{db})))
  187. mux.Handle("/admin/removeAuthSession", prometheus.InstrumentHandler("removeAuthSession", util.MakeJSONAPI(&handlers.RemoveAuthSession{db})))
  188. }
  189. polling.SetClients(clients)
  190. if err := polling.Start(); err != nil {
  191. log.WithError(err).Panic("Failed to start polling")
  192. }
  193. }
  194. func main() {
  195. if LogDir != "" {
  196. log.AddHook(dugong.NewFSHook(
  197. filepath.Join(LogDir, "avbot.log"),
  198. &log.TextFormatter{
  199. TimestampFormat: "2006-01-02 15:04:05.000000",
  200. DisableColors: true,
  201. DisableTimestamp: false,
  202. DisableSorting: false,
  203. }, &dugong.DailyRotationSchedule{GZip: false},
  204. ))
  205. }
  206. log.Infof("GO-AVBOT build %s (%s %s %s %s %s %s)", MinVersion, BindAddress, BaseURL, DatabaseType, DatabaseURL, LogDir, ConfigFile)
  207. setup(http.DefaultServeMux, http.DefaultClient)
  208. log.Fatal(http.ListenAndServe(BindAddress, nil))
  209. }