PageRenderTime 57ms CodeModel.GetById 24ms RepoModel.GetById 0ms app.codeStats 0ms

/daisy.go

https://gitlab.com/jramb/daisy
Go | 1524 lines | 1207 code | 132 blank | 185 comment | 277 complexity | 821095bec203107be4c785ee5a519293 MD5 | raw file
  1. package main
  2. /*
  3. 2016 by Jörg Ramb
  4. */
  5. import (
  6. "bufio"
  7. "bytes"
  8. "crypto/sha256"
  9. "encoding/base64"
  10. "encoding/gob"
  11. "encoding/hex"
  12. "encoding/json"
  13. "flag"
  14. "fmt"
  15. "io"
  16. "io/ioutil"
  17. "log"
  18. "net"
  19. "net/http"
  20. "net/url"
  21. "os"
  22. "os/signal"
  23. "path"
  24. "regexp"
  25. "runtime"
  26. "strconv"
  27. "strings"
  28. "time"
  29. "goji.io"
  30. "goji.io/pat"
  31. "golang.org/x/net/context"
  32. "github.com/spf13/viper"
  33. // "github.com/boltdb/bolt"
  34. "github.com/coreos/bbolt"
  35. // go get github.com/mattn/go-sqlite3
  36. //_ "github.com/mattn/go-sqlite3"
  37. // "github.com/satori/go.uuid"
  38. "github.com/josselin-c/go.uuid"
  39. // "github.com/google/uuid"
  40. //"github.com/pborman/uuid"
  41. //"github.com/nu7hatch/gouuid"
  42. )
  43. //var isoDateTimeFormat = time.RFC3339
  44. const isoDateTimeFormat = "2006-01-02 15:04:05"
  45. const contentTypeJSON = "application/json"
  46. const daisyVersion = "2018.07.15.2"
  47. //const storePath = "store" + string(os.PathSeparator)
  48. var storePath string //= viper.GetString("store")
  49. var logsPath string //= viper.GetString("logs")
  50. const pathMode = 0700
  51. // ba7eaa41-9321-4895-a0e9-fb365612163e
  52. var uuidRegexp = regexp.MustCompile(`^[[:xdigit:]]{8}-?[[:xdigit:]]{4}-?[[:xdigit:]]{4}-?[[:xdigit:]]{12}$`)
  53. var sourcePathRegexp = regexp.MustCompile("^/source(?:/([^/]+)(?:/(.*))?)?")
  54. //var filenameRegexp = regexp.MustCompile(`^[\w\-\. ]+$`) // \w = [0-9a-zA-Z_].
  55. //var sha256re = regexp.MustCompile(`[0-9a-f]{64}`)
  56. const (
  57. bucketAccesses = "ACCESSES"
  58. bucketSources = "SOURCES"
  59. bucketLinkSA = "SOURCE-ACCESSES-LINK"
  60. bucketLinkHS = "HASH-SOURCE-LINK"
  61. )
  62. // Counters is a setup of profiling values at a specific time
  63. type Counters struct {
  64. LastUpdate time.Time
  65. TimeDiff string
  66. Sources int
  67. Accesses int
  68. Fetches int
  69. Errors int
  70. }
  71. // DaisyDB is the type combining a collection of values related to a single instance of a Daisy DB
  72. type DaisyDB struct {
  73. DB *bolt.DB `json:"-"`
  74. dbNeedsBackup bool
  75. ServerStartTime time.Time
  76. Uptime string
  77. DBStats bolt.Stats
  78. DBStatsDiff bolt.Stats
  79. previousCounters Counters
  80. CurrentCounters Counters
  81. CountersDiff Counters
  82. }
  83. var gDB *DaisyDB
  84. var (
  85. requestLog *os.File
  86. errorLog *os.File
  87. urlPathPrefix string
  88. uuidStyle string
  89. setupAPIKey string
  90. )
  91. // Sub calculates a difference between two Counters
  92. func (from Counters) Sub(sub Counters) Counters {
  93. diff := from
  94. if !sub.LastUpdate.IsZero() {
  95. diff.TimeDiff = from.LastUpdate.Sub(sub.LastUpdate).String()
  96. }
  97. diff.Sources = from.Sources - sub.Sources
  98. diff.Accesses = from.Accesses - sub.Accesses
  99. diff.Fetches = from.Fetches - sub.Fetches
  100. diff.Errors = from.Errors - sub.Errors
  101. return diff
  102. }
  103. func enc(b []byte) string {
  104. return hex.EncodeToString(b)
  105. }
  106. func encMaybe(b *[]byte) *string {
  107. if b != nil {
  108. s := enc(*b)
  109. return &s
  110. }
  111. return nil
  112. }
  113. func dec(s string) []byte {
  114. b, _ := hex.DecodeString(strings.Trim(s, " "))
  115. return b
  116. }
  117. func decMaybe(s *string) *[]byte {
  118. if s != nil {
  119. b := dec(*s)
  120. return &b
  121. }
  122. return nil
  123. }
  124. func uuidEnc(u uuid.UUID) string {
  125. if uuidStyle == "full" {
  126. return u.String()
  127. }
  128. return strings.Trim(base64.URLEncoding.EncodeToString(u.Bytes()), "=")
  129. }
  130. func uuidDec(s string) (u uuid.UUID) {
  131. var err error
  132. if uuidRegexp.MatchString(s) {
  133. u, err = uuid.FromString(s)
  134. } else {
  135. var ub []byte
  136. ub, err = base64.URLEncoding.WithPadding(base64.NoPadding).DecodeString(s)
  137. u = uuid.FromBytesOrNil(ub)
  138. }
  139. if err != nil {
  140. panic(err)
  141. }
  142. return u
  143. }
  144. func errCheck(err error, msg string) {
  145. }
  146. func constructPath(sum []byte) string {
  147. p := path.Join(storePath, enc(sum[:1]))
  148. os.MkdirAll(p, pathMode)
  149. return path.Join(p, enc(sum))
  150. }
  151. func constructLogPath(sourceID uuid.UUID) string {
  152. sourceIDstr := strings.ToUpper(enc(sourceID[:]))
  153. sourceLogPath := path.Join(logsPath, "sourcelogs", sourceIDstr[:2])
  154. os.MkdirAll(sourceLogPath, pathMode)
  155. return path.Join(sourceLogPath, sourceIDstr+".log")
  156. }
  157. // JSONAHandler is a JSON representation of an access handler
  158. type JSONAHandler struct {
  159. AccessID string `json:"access"`
  160. SourceID string `json:"-"`
  161. URLPath string `json:"url-path"`
  162. FetchCount int `json:"fetch-count"`
  163. StartDate *string `json:"start-date,omitempty"`
  164. EndDate *string `json:"end-date,omitempty"`
  165. }
  166. // JSONSource is the JSON representation of a source
  167. type JSONSource struct {
  168. SourceID string `json:"source"`
  169. Binary bool `json:"binary"`
  170. MimeType *string `json:"mime-type,omitempty"`
  171. FileName *string `json:"file-name,omitempty"`
  172. URL *string `json:"url,omitempty"`
  173. StartDate *string `json:"start-date,omitempty"`
  174. EndDate *string `json:"end-date,omitempty"`
  175. APIKey *string `json:"apikey,omitempty"`
  176. Sha256 *string `json:"sha256,omitempty"`
  177. NumHandlers int `json:"num-handlers"`
  178. AccessHandlers []JSONAHandler `json:"access-handlers"`
  179. }
  180. // DBAHandler is the internal DB representation of an access handler
  181. type DBAHandler struct {
  182. AccessID uuid.UUID // this is the key anyway
  183. SourceID uuid.UUID `json:"source"`
  184. FetchCount int `json:"fetch-count"`
  185. StartDate *time.Time `json:"start-time,omitempty"`
  186. EndDate *time.Time `json:"end-time,omitempty"`
  187. }
  188. // DBSource is the internal DB representation of a source
  189. type DBSource struct {
  190. SourceID uuid.UUID
  191. Binary bool `json:"binary"`
  192. MimeType *string `json:"mine-type"`
  193. FileName *string `json:"file-name,omitempty"`
  194. URL *string `json:"url,omitempty"`
  195. StartDate *time.Time `json:"start-time,omitempty"`
  196. EndDate *time.Time `json:"end-time,omitempty"`
  197. Sha256 *[]byte `json:"sha256,omitempty"`
  198. NumHandlers int `json:"num-handlers"`
  199. AccessHandlers []DBAHandler `json:"access-handlers"`
  200. APIKey *string `json:"apikey"`
  201. }
  202. func parseDate(s *string) *time.Time {
  203. if s == nil || *s == "" {
  204. return nil
  205. }
  206. sdate, err := time.ParseInLocation(isoDateTimeFormat, *s, time.Local)
  207. if err != nil {
  208. sdate, err = time.ParseInLocation("2006-01-02", *s, time.Local)
  209. if err != nil {
  210. sdate, err = time.ParseInLocation(time.RFC3339Nano, *s, time.Local)
  211. if err != nil {
  212. sdate, err = time.ParseInLocation(time.RFC3339, *s, time.Local)
  213. }
  214. }
  215. }
  216. if err != nil {
  217. panic(err)
  218. }
  219. return &sdate
  220. }
  221. func renderDate(d *time.Time) *string {
  222. if d == nil {
  223. return nil
  224. }
  225. s := d.Format(isoDateTimeFormat)
  226. return &s
  227. }
  228. func renderDateOrEmpty(d *time.Time) string {
  229. strPtr := renderDate(d)
  230. if strPtr == nil {
  231. return ""
  232. }
  233. return *strPtr
  234. }
  235. func renderStringOrEmpty(s *string) string {
  236. if s == nil {
  237. return ""
  238. }
  239. return *s
  240. }
  241. // DBStruct converts a JSON representation to a DB representation
  242. func (s *JSONSource) DBStruct() *DBSource {
  243. if s == nil {
  244. return nil
  245. }
  246. sdate := parseDate(s.StartDate)
  247. edate := parseDate(s.EndDate)
  248. allHandlers := make([]DBAHandler, 0, len(s.AccessHandlers))
  249. for _, ah := range s.AccessHandlers {
  250. dbah := ah.DBStruct()
  251. if dbah != nil {
  252. allHandlers = append(allHandlers, *dbah)
  253. }
  254. }
  255. return &DBSource{
  256. Binary: s.Binary,
  257. MimeType: s.MimeType,
  258. FileName: s.FileName,
  259. URL: s.URL,
  260. StartDate: sdate,
  261. EndDate: edate,
  262. Sha256: decMaybe(s.Sha256),
  263. NumHandlers: s.NumHandlers,
  264. AccessHandlers: allHandlers,
  265. }
  266. }
  267. // JSONStruct converts a DB representation to a JSON representation
  268. func (a *DBAHandler) JSONStruct() *JSONAHandler {
  269. if a == nil {
  270. return nil
  271. }
  272. return &JSONAHandler{
  273. AccessID: uuidEnc(a.AccessID),
  274. SourceID: uuidEnc(a.SourceID),
  275. URLPath: urlPathPrefix + uuidEnc(a.AccessID),
  276. FetchCount: a.FetchCount,
  277. StartDate: renderDate(a.StartDate),
  278. EndDate: renderDate(a.EndDate),
  279. }
  280. }
  281. func listAccessHandlersToJSON(ahl []DBAHandler) []JSONAHandler {
  282. allHandlers := make([]JSONAHandler, 0, 10)
  283. for _, ah := range ahl {
  284. jah := ah.JSONStruct()
  285. if jah != nil {
  286. allHandlers = append(allHandlers, *jah)
  287. }
  288. }
  289. return allHandlers
  290. }
  291. // JSONStruct converts a DB representation to a JSON representation
  292. func (s *DBSource) JSONStruct() *JSONSource {
  293. if s == nil {
  294. return nil
  295. }
  296. return &JSONSource{
  297. SourceID: uuidEnc(s.SourceID), // explicit uuid.String() conversion
  298. Binary: s.Binary,
  299. MimeType: s.MimeType,
  300. FileName: s.FileName,
  301. URL: s.URL,
  302. StartDate: renderDate(s.StartDate),
  303. EndDate: renderDate(s.EndDate),
  304. Sha256: encMaybe(s.Sha256),
  305. NumHandlers: s.NumHandlers,
  306. AccessHandlers: listAccessHandlersToJSON(s.AccessHandlers),
  307. }
  308. }
  309. // DBStruct converts a JSON representation to a DB representation
  310. func (a *JSONAHandler) DBStruct() *DBAHandler {
  311. if a == nil {
  312. return nil
  313. }
  314. sdate := parseDate(a.StartDate)
  315. edate := parseDate(a.EndDate)
  316. return &DBAHandler{
  317. AccessID: uuidDec(a.AccessID),
  318. SourceID: uuidDec(a.SourceID),
  319. FetchCount: a.FetchCount,
  320. StartDate: sdate,
  321. EndDate: edate,
  322. }
  323. }
  324. func initDB(db *bolt.DB) error {
  325. log.Println("Initializing database", db)
  326. err := db.Update(func(tx *bolt.Tx) error {
  327. bucketNames := []string{
  328. bucketSources,
  329. bucketAccesses,
  330. bucketLinkSA,
  331. bucketLinkHS,
  332. }
  333. for _, bn := range bucketNames {
  334. if _, err := tx.CreateBucketIfNotExists([]byte(bn)); err != nil {
  335. return err
  336. }
  337. }
  338. return nil
  339. })
  340. return err
  341. }
  342. func databaseDump(db *DaisyDB, filename string) error {
  343. if filename == "-" {
  344. return nil
  345. }
  346. return db.DB.View(func(tx *bolt.Tx) error {
  347. src := tx.Bucket([]byte(bucketSources))
  348. var dumpFileName string
  349. if filename == "" {
  350. dumpFileName = "db_dump_" + time.Now().Format("2006-01-02_15-04-05") + ".tsv"
  351. } else {
  352. dumpFileName = filename
  353. }
  354. dumpFile, err := os.OpenFile(path.Join(storePath, dumpFileName), os.O_WRONLY|os.O_CREATE, 0600)
  355. if err != nil {
  356. return err
  357. }
  358. defer dumpFile.Close()
  359. src.ForEach(func(k, v []byte) error {
  360. s, _ := decodeDBSource(v)
  361. fmt.Fprintln(dumpFile, strings.Join([]string{
  362. "S",
  363. uuidEnc(s.SourceID),
  364. renderDateOrEmpty(s.StartDate),
  365. renderDateOrEmpty(s.EndDate),
  366. renderStringOrEmpty(encMaybe(s.Sha256)),
  367. renderStringOrEmpty(s.MimeType),
  368. renderStringOrEmpty(s.FileName),
  369. renderStringOrEmpty(s.URL),
  370. }, "\t"))
  371. allA, _ := dbGetAllAccesses(tx, s.SourceID)
  372. for _, a := range allA {
  373. //fmt.Println("%+v",a)
  374. fmt.Fprintln(dumpFile, strings.Join([]string{
  375. "A",
  376. uuidEnc(a.SourceID), // maybe not needed
  377. uuidEnc(a.AccessID),
  378. strconv.Itoa(a.FetchCount),
  379. renderDateOrEmpty(a.StartDate),
  380. renderDateOrEmpty(a.EndDate),
  381. }, "\t"))
  382. }
  383. //fmt.Fprintf(dumpFile, "S\t%s\t%s\n", uuidEnc(s.SourceID), "hej")
  384. return nil
  385. })
  386. log.Println("Database dumped to", dumpFile.Name())
  387. return nil
  388. })
  389. }
  390. func backupDB(db *DaisyDB) string {
  391. if !db.dbNeedsBackup {
  392. return ""
  393. }
  394. err := databaseDump(db, viper.GetString("dump-file"))
  395. if err != nil {
  396. fmt.Println("Error during databaseDump", err)
  397. }
  398. // rename the old backup (paranoid? backup of backup!)
  399. fname := path.Join(storePath, viper.GetString("dbfile")+".backup")
  400. // keep three generations (TODO?: parameter controlled?)
  401. err = os.Rename(fname+".1", fname+".2") // ignore errors
  402. err = os.Rename(fname, fname+".1") // ignore errors
  403. f, err := os.Create(fname)
  404. if err != nil {
  405. log.Fatal(err)
  406. }
  407. defer f.Close()
  408. w := bufio.NewWriter(f)
  409. defer w.Flush()
  410. // write the backup
  411. db.dbNeedsBackup = false
  412. err = db.DB.View(func(tx *bolt.Tx) error {
  413. n, err := tx.WriteTo(w)
  414. log.Printf("Backup wrote %d bytes\n", n)
  415. return err
  416. })
  417. if err != nil {
  418. return err.Error()
  419. }
  420. return " backup " + fname
  421. }
  422. func decodeDBSource(b []byte) (*DBSource, error) {
  423. if b != nil {
  424. var s DBSource
  425. decoder := gob.NewDecoder(bytes.NewBuffer(b))
  426. err := decoder.Decode(&s)
  427. if err != nil {
  428. return nil, err
  429. }
  430. return &s, nil
  431. }
  432. return nil, nil
  433. }
  434. func decodeDBAHandler(b []byte) (*DBAHandler, error) {
  435. if b != nil {
  436. var a DBAHandler
  437. decoder := gob.NewDecoder(bytes.NewBuffer(b))
  438. err := decoder.Decode(&a)
  439. if err != nil {
  440. return nil, err
  441. }
  442. return &a, nil
  443. }
  444. return nil, nil
  445. }
  446. // Bytes returns a []byte of a DB object
  447. func (s DBSource) Bytes() []byte {
  448. var stream bytes.Buffer
  449. encoder := gob.NewEncoder(&stream)
  450. s.AccessHandlers = nil // do not save list of AHs!
  451. encoder.Encode(s)
  452. return stream.Bytes()
  453. }
  454. // Bytes returns a []byte of a DB object
  455. func (a DBAHandler) Bytes() []byte {
  456. var stream bytes.Buffer
  457. enc := gob.NewEncoder(&stream)
  458. enc.Encode(a)
  459. return stream.Bytes()
  460. }
  461. func dbUpsertSource(tx *bolt.Tx, what string, s DBSource) error {
  462. var err error
  463. b := tx.Bucket([]byte(bucketSources))
  464. sBytes := s.Bytes()
  465. if err = b.Put(s.SourceID.Bytes(), sBytes); err != nil {
  466. return err
  467. }
  468. gDB.dbNeedsBackup = true
  469. if s.Sha256 != nil {
  470. dbLinkHashSource(tx, *s.Sha256, s.SourceID, s.Binary)
  471. }
  472. return err
  473. }
  474. func dbIncrementAccess(aid uuid.UUID) error {
  475. // Update would serialize, but will not be called multiple times
  476. err := gDB.DB.Batch(func(tx *bolt.Tx) error {
  477. var err error
  478. b := tx.Bucket([]byte(bucketAccesses))
  479. ah, err := decodeDBAHandler(b.Get(aid[:]))
  480. if err != nil {
  481. return err
  482. }
  483. if ah != nil {
  484. ah.FetchCount = ah.FetchCount + 1
  485. err = b.Put(aid[:], ah.Bytes())
  486. gDB.dbNeedsBackup = true
  487. }
  488. return err
  489. })
  490. return err
  491. }
  492. func dbUpsertAccess(tx *bolt.Tx, what string, a DBAHandler) error {
  493. b := tx.Bucket([]byte(bucketAccesses))
  494. err := b.Put(a.AccessID.Bytes(), a.Bytes())
  495. gDB.dbNeedsBackup = true
  496. return err
  497. }
  498. func dbGetAccess(tx *bolt.Tx, aid uuid.UUID) (*DBAHandler, error) {
  499. b := tx.Bucket([]byte(bucketAccesses))
  500. return decodeDBAHandler(b.Get(aid[:]))
  501. }
  502. func dbLinkSourceAccess(tx *bolt.Tx, s uuid.UUID, a uuid.UUID) error {
  503. b := tx.Bucket([]byte(bucketLinkSA))
  504. joined := bytes.Join([][]byte{s.Bytes(), a.Bytes()}, nil)
  505. err := b.Put(joined, nil) // no data stored on the link
  506. gDB.dbNeedsBackup = true
  507. return err
  508. }
  509. func countPrefixes(c *bolt.Cursor, prefix []byte) int {
  510. cnt := 0
  511. for k, _ := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, _ = c.Next() {
  512. cnt = cnt + 1
  513. }
  514. return cnt
  515. }
  516. func countNodes(b *bolt.Bucket) int {
  517. cnt := 0
  518. b.ForEach(func(k, v []byte) error {
  519. cnt++
  520. return nil
  521. })
  522. return cnt
  523. }
  524. func countObjects(db *DaisyDB) {
  525. db.DB.View(func(tx *bolt.Tx) error {
  526. db.CurrentCounters.Sources = countNodes(tx.Bucket([]byte(bucketSources)))
  527. db.CurrentCounters.Accesses = countNodes(tx.Bucket([]byte(bucketAccesses)))
  528. db.previousCounters = db.CurrentCounters
  529. return nil
  530. })
  531. }
  532. /*
  533. func checkExistsKey(bucket string, key []byte) bool {
  534. var existing bool
  535. err := gDB.DB.View(func(tx *bolt.Tx) error {
  536. b := tx.Bucket([]byte(bucket))
  537. result := b.Get(key)
  538. existing = (result != nil)
  539. return nil
  540. })
  541. if err != nil {
  542. panic(err)
  543. }
  544. return existing
  545. }
  546. */
  547. func dbLinkHashSource(tx *bolt.Tx, hash []byte, s uuid.UUID, addIt bool) error {
  548. b := tx.Bucket([]byte(bucketLinkHS))
  549. joined := bytes.Join([][]byte{hash, s.Bytes()}, nil)
  550. if addIt {
  551. return b.Put(joined, nil) // nil is ok
  552. }
  553. err := b.Delete(joined)
  554. return err //b.Delete(joined)
  555. }
  556. func dbGetAllAccesses(tx *bolt.Tx, sid uuid.UUID) ([]DBAHandler, error) {
  557. allHandlers := make([]DBAHandler, 0, 10)
  558. c := tx.Bucket([]byte(bucketLinkSA)).Cursor()
  559. b := tx.Bucket([]byte(bucketAccesses))
  560. for k, _ := c.Seek(sid.Bytes()); bytes.HasPrefix(k, sid.Bytes()); k, _ = c.Next() {
  561. ah, err := decodeDBAHandler(b.Get(k[16:])) // aid is the second part of the key
  562. if err != nil {
  563. return nil, err
  564. }
  565. allHandlers = append(allHandlers, *ah)
  566. }
  567. return allHandlers, nil
  568. }
  569. func dbGetSource(tx *bolt.Tx, sid uuid.UUID) (*DBSource, error) {
  570. var s *DBSource
  571. b := tx.Bucket([]byte(bucketSources))
  572. var err error
  573. s, err = decodeDBSource(b.Get(sid.Bytes()))
  574. if err != nil {
  575. return nil, err
  576. }
  577. if s != nil {
  578. s.AccessHandlers, err = dbGetAllAccesses(tx, sid)
  579. }
  580. return s, err
  581. }
  582. func deleteFile(hash []byte) error {
  583. return os.Remove(constructPath(hash))
  584. }
  585. func expireBinaries(db *DaisyDB) string {
  586. now := time.Now().AddDate(0, 0, -viper.GetInt("retention-days"))
  587. deletedFiles := 0
  588. err := db.DB.Batch(func(tx *bolt.Tx) error {
  589. hs := tx.Bucket([]byte(bucketLinkHS)).Cursor()
  590. src := tx.Bucket([]byte(bucketSources))
  591. // go through all
  592. //for sid, source := src.First(); sid != nil ; sid, source := src.Next() {
  593. src.ForEach(func(k, v []byte) error {
  594. s, _ := decodeDBSource(v)
  595. //fmt.Println("checking", uuidEnc(s.SourceID))
  596. if s.Binary && s.EndDate != nil && s.Sha256 != nil && now.After(*s.EndDate) {
  597. s.Binary = false
  598. copies := countPrefixes(hs, *s.Sha256) // need to check before, because deletions are delayed??
  599. // updating the source will remove the hash-link as well
  600. if err := dbUpsertSource(tx, "update", *s); err != nil {
  601. return err
  602. }
  603. //fmt.Println("checking ", enc(*s.Sha256), copies)
  604. //if no HS left then delete binary
  605. if copies <= 1 { // is 1 if there is only (this) instance
  606. fmt.Println("DELETING FILE:", enc(*s.Sha256))
  607. deletedFiles = deletedFiles + 1
  608. deleteFile(*s.Sha256)
  609. sourceLog(s.SourceID, nil, "deleted binary")
  610. }
  611. }
  612. return nil
  613. })
  614. fmt.Println("Deleted files:", deletedFiles)
  615. return nil
  616. })
  617. if err != nil {
  618. return err.Error()
  619. }
  620. return ""
  621. }
  622. func newAccessHandler(tx *bolt.Tx, sid uuid.UUID) (DBAHandler, error) {
  623. var a DBAHandler
  624. a.AccessID = uuid.Must(uuid.NewV4())
  625. a.SourceID = sid
  626. a.FetchCount = 0
  627. //a.URLPath = "/g/" + a.AccessID
  628. dbUpsertAccess(tx, "insert", a) // was: go ...
  629. gDB.CurrentCounters.Accesses++
  630. dbLinkSourceAccess(tx, sid, a.AccessID)
  631. return a, nil
  632. }
  633. func extendAccessHandlers(tx *bolt.Tx, s *DBSource) error {
  634. //if s.AccessHandlers == nil {
  635. //newHandlers := make([]DBAHandler, 0, s.NumHandlers)
  636. //s.AccessHandlers = newHandlers
  637. //}
  638. for i := len(s.AccessHandlers); i < s.NumHandlers; i++ {
  639. a, err := newAccessHandler(tx, s.SourceID)
  640. if err != nil {
  641. return err
  642. }
  643. newHandlers := append(s.AccessHandlers, a)
  644. s.AccessHandlers = newHandlers
  645. }
  646. return nil
  647. }
  648. func genericHandler(w http.ResponseWriter, r *http.Request) {
  649. fmt.Fprintf(w, "r.URL.Path=%s\n", r.URL.Path)
  650. }
  651. func getSourceOrPanic(tx *bolt.Tx, sourceID uuid.UUID) *DBSource {
  652. s, err := dbGetSource(tx, sourceID)
  653. if s == nil {
  654. panic(fmt.Errorf("No such source found"))
  655. }
  656. if err != nil {
  657. panic(err)
  658. }
  659. return s
  660. }
  661. func getAccessOrPanic(tx *bolt.Tx, accessID uuid.UUID) *DBAHandler {
  662. a, err := dbGetAccess(tx, accessID)
  663. if a == nil {
  664. panic(fmt.Errorf("No such access handler found: %s", uuidEnc(accessID)))
  665. }
  666. if err != nil {
  667. panic(err)
  668. }
  669. return a
  670. }
  671. func verifyActiveDate(from *time.Time, to *time.Time) bool {
  672. now := time.Now()
  673. if from != nil && now.Before(*from) {
  674. return false
  675. }
  676. if to != nil && now.After(*to) {
  677. return false
  678. }
  679. return true
  680. }
  681. func verifyActiveDateStr(startDate *string, endDate *string) bool {
  682. from := parseDate(startDate)
  683. to := parseDate(endDate)
  684. return verifyActiveDate(from, to)
  685. }
  686. func printStackTrace(err error) {
  687. if viper.GetBool("stacktrace") {
  688. buf := make([]byte, 10240)
  689. runtime.Stack(buf, false)
  690. log.Printf("%s\n%s\n", err, buf)
  691. }
  692. }
  693. func decodeRemoteAddr(r *http.Request) string {
  694. strp := regexp.MustCompile(":[0-9]+$") // strip :port
  695. for _, h := range []string{"X-Forwarded-For", "X-Real-Ip"} {
  696. addresses := strings.Split(r.Header.Get(h), ",")
  697. // march from right to left until we get a public address
  698. // that will be the address right before our proxy.
  699. for i := len(addresses) - 1; i >= 0; i-- {
  700. ip := strings.TrimSpace(addresses[i])
  701. // header can contain spaces too, strip those out.
  702. realIP := net.ParseIP(ip)
  703. if !realIP.IsGlobalUnicast() {
  704. // bad address, go to next
  705. continue
  706. }
  707. return strp.ReplaceAllLiteralString(ip, "")
  708. }
  709. }
  710. return strp.ReplaceAllLiteralString(r.RemoteAddr, "")
  711. /* // old version, did not handle ipv6
  712. xff := r.Header.Get("X-Forwarded-For")
  713. ra := r.RemoteAddr // might be ipv6: [::1]:47016!
  714. ips := strings.Split(ra, ":")
  715. ra = ips[0]
  716. if xff != "" {
  717. ips = strings.Split(xff, ", ")
  718. return fmt.Sprintf("%s(%s)", ips[0], ra)
  719. } else {
  720. return ra
  721. }
  722. */
  723. }
  724. func logError(w http.ResponseWriter, r *http.Request, err error) {
  725. gDB.CurrentCounters.Errors++
  726. t := time.Now().Format(isoDateTimeFormat)
  727. fmt.Fprintf(errorLog, "%s %s %s %s: %s\n", t, decodeRemoteAddr(r), r.Method, r.URL.Path, err)
  728. fmt.Fprintln(w, err.Error())
  729. printStackTrace(err)
  730. errorLog.Sync()
  731. }
  732. func fetchHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  733. //accessIDStr := r.URL.Path[len(viper.GetString("fetch-prefix")):]
  734. accessIDStr := pat.Param(ctx, "accessid") //r.URL.Path
  735. accessID := uuidDec(accessIDStr)
  736. var ah *DBAHandler
  737. var s *DBSource
  738. err := gDB.DB.View(func(tx *bolt.Tx) error {
  739. ah = getAccessOrPanic(tx, accessID)
  740. /*
  741. ah, err = dbGetAccess(tx, accessID)
  742. if err != nil {
  743. return err
  744. }
  745. */
  746. if !verifyActiveDate(ah.StartDate, ah.EndDate) {
  747. return fmt.Errorf("Access ID not active")
  748. }
  749. s = getSourceOrPanic(tx, ah.SourceID)
  750. if !verifyActiveDate(s.StartDate, s.EndDate) {
  751. return fmt.Errorf("Source not active")
  752. }
  753. if s.URL == nil && s.Sha256 == nil {
  754. return fmt.Errorf("Source is missing Sha256")
  755. }
  756. return nil
  757. })
  758. if err != nil {
  759. logError(w, r, err)
  760. //sourceLog(ah.SourceID, r, err.Error())
  761. return
  762. }
  763. gDB.CurrentCounters.Fetches++
  764. go dbIncrementAccess(ah.AccessID) // offline, will be in own goroutine
  765. if s.URL != nil {
  766. http.Redirect(w, r, *s.URL, http.StatusSeeOther)
  767. sourceLog(ah.SourceID, r, "redirected: "+*s.URL)
  768. } else {
  769. err = sendRawFile(w, *s.Sha256, s.MimeType, s.FileName) // view does not block? so ok to do this in tx
  770. if err != nil {
  771. logError(w, r, err)
  772. sourceLog(ah.SourceID, r, err.Error())
  773. return
  774. }
  775. sourceLog(ah.SourceID, r, "fetched")
  776. }
  777. }
  778. func simpleFileName(s string) string {
  779. reg, err := regexp.Compile("[^A-Za-z0-9.-]")
  780. if err != nil {
  781. return "file.xxx"
  782. }
  783. return reg.ReplaceAllString(s, "_")
  784. }
  785. func sendRawFile(w http.ResponseWriter, sha []byte, mimeType *string, fileName *string) error {
  786. f, err := os.Open(constructPath(sha))
  787. if err != nil {
  788. return err
  789. }
  790. defer f.Close()
  791. if mimeType != nil {
  792. w.Header().Set("Content-Type", *mimeType)
  793. // w.Header().Set("Content-Type", "application/octet-stream") //forces the save as dialog
  794. }
  795. //Disposition makes the browser to offer save-as with the given file name, instead of displaying it
  796. if fileName != nil /*&& filenameRegexp.MatchString(*filename)*/ {
  797. w.Header().Set("Content-Disposition", "attachment; "+
  798. "filename=\""+simpleFileName(*fileName)+"\"; "+
  799. "filename*=UTF-8''"+strings.Replace(url.QueryEscape(*fileName), "+", "%20", -1))
  800. // w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+url.QueryEscape(*fileName))
  801. // w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+*fileName)
  802. }
  803. io.Copy(w, f)
  804. return nil
  805. }
  806. func rawHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  807. var sum string = r.URL.Path[len("/raw/"):]
  808. sha := dec(sum)
  809. //fmt.Println(sum)
  810. if err := sendRawFile(w, sha, nil, nil); err != nil {
  811. logError(w, r, err)
  812. }
  813. }
  814. /* Catch "panic" and generate a "BadRequest" */
  815. func catchPanic(ctx context.Context, w http.ResponseWriter, r *http.Request, sendDetails bool) {
  816. if err := recover(); err != nil {
  817. t := time.Now().Format(isoDateTimeFormat)
  818. fmt.Fprintf(errorLog, "%s %s %s %s: %s\n", t, decodeRemoteAddr(r), r.Method, r.URL.Path, err)
  819. gDB.CurrentCounters.Errors++
  820. if sendDetails {
  821. http.Error(w, fmt.Sprintf("%s", err), http.StatusInternalServerError)
  822. } else {
  823. http.Error(w, "Internal error", http.StatusInternalServerError)
  824. }
  825. if viper.GetBool("stacktrace") {
  826. buf := make([]byte, 1024)
  827. runtime.Stack(buf, false)
  828. log.Printf("%s\n%s\n", err, buf)
  829. }
  830. //panic(err)
  831. }
  832. }
  833. func logRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, started time.Time) {
  834. //println(r.Method, r.URL.Path)
  835. now := time.Now()
  836. t := now.Format(isoDateTimeFormat)
  837. //if _, ok := *w.(ResponseWriterProxy); ok {
  838. fmt.Fprintf(requestLog, "%s %s %s %s (%s)\n",
  839. t, decodeRemoteAddr(r), r.Method, r.URL.Path, now.Sub(started))
  840. //} else { // normal ResponseWriter, no more info
  841. //fmt.Fprintf(requestLog, "%s %s %s %s\n", t, decodeRemoteAddr(r), r.Method, r.URL.Path)
  842. //}
  843. requestLog.Sync()
  844. }
  845. //http://grokbase.com/t/gg/golang-nuts/154v03yk9y/go-nuts-http-response-status-in-a-http-handler
  846. func logging(h goji.Handler) goji.Handler {
  847. return goji.HandlerFunc(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  848. //xw := modwriter{ResponseWriter: w} // create a wrapping
  849. started := time.Now()
  850. defer logRequest(ctx, w, r, started)
  851. defer catchPanic(ctx, w, r, true)
  852. h.ServeHTTPC(ctx, w, r)
  853. })
  854. }
  855. func sourceLog(sourceID uuid.UUID, r *http.Request, logStr string) {
  856. go func() {
  857. logFileName := constructLogPath(sourceID)
  858. // log.Println("sourceLog", logFileName, logStr)
  859. var logStrFinal string
  860. if r != nil {
  861. logStrFinal = fmt.Sprintf("%s | %s | %s | %s", decodeRemoteAddr(r), r.Method, r.URL.Path, logStr)
  862. } else {
  863. logStrFinal = logStr
  864. }
  865. logFile, err := os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
  866. if err != nil {
  867. log.Println("Source log error:", uuidEnc(sourceID), logStrFinal)
  868. } else {
  869. // fmt.Println(time.Now().Format(isoDateTimeFormat), "|", logStrFinal)
  870. fmt.Fprintln(logFile, time.Now().Format(isoDateTimeFormat), "|", logStrFinal)
  871. logFile.Close()
  872. }
  873. }()
  874. }
  875. func refreshStats(db *DaisyDB) {
  876. //gDB.DBStats = gDB.DB.Stats()
  877. stats := db.DB.Stats()
  878. gDB.DBStatsDiff = stats.Sub(&gDB.DBStats)
  879. gDB.DBStats = stats
  880. }
  881. func wuiHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  882. switch r.Method {
  883. case "GET":
  884. switch pat.Param(ctx, "cmd") {
  885. case "stats":
  886. //refreshStats(gDB)
  887. now := time.Now().Round(time.Second)
  888. gDB.CurrentCounters.LastUpdate = now
  889. gDB.Uptime = now.Sub(gDB.ServerStartTime).String()
  890. w.Header().Set("Content-Type", contentTypeJSON)
  891. if b, err := json.Marshal(gDB); err != nil {
  892. log.Println("Marshal failed for stats:", err)
  893. w.WriteHeader(http.StatusInternalServerError)
  894. } else {
  895. var out bytes.Buffer
  896. json.Indent(&out, b, "", " ")
  897. out.WriteTo(w) // returns n, err
  898. }
  899. default:
  900. logError(w, r, fmt.Errorf("Invalid wui request %s -> %s", r.URL.Path, pat.Param(ctx, "cmd")))
  901. }
  902. }
  903. }
  904. func sourcePostNewHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  905. var result JSONSource
  906. var inputJ JSONSource
  907. var input *DBSource
  908. var current *DBSource
  909. b, err := ioutil.ReadAll(r.Body)
  910. //fmt.Printf("body: %s\n", b)
  911. err = json.Unmarshal(b, &inputJ)
  912. if inputJ.APIKey == nil {
  913. logError(w, r, fmt.Errorf("Unauthorized access"))
  914. return
  915. }
  916. if setupAPIKey != "" && strings.Compare(*inputJ.APIKey, setupAPIKey) != 0 {
  917. logError(w, r, fmt.Errorf("Unauthorized access: %s", *inputJ.APIKey))
  918. return
  919. }
  920. input = inputJ.DBStruct()
  921. if err != nil {
  922. panic(err)
  923. }
  924. // create a new source
  925. current = input // start with what the input gave us
  926. current.Binary = false
  927. current.SourceID = uuid.Must(uuid.NewV4())
  928. if current.StartDate == nil {
  929. now := time.Now()
  930. current.StartDate = &now
  931. }
  932. if current.EndDate == nil {
  933. nowLater := current.StartDate.AddDate(0, viper.GetInt("default-end-months"), -1)
  934. current.EndDate = &nowLater
  935. }
  936. //if current.FileName != nil && !filenameRegexp.MatchString(current.FileName) {
  937. //logError(w, r, fmt.Errorf("Invalid filename: "+current.FileName))
  938. //return
  939. //}
  940. // either FileName and MimeType or URL must be given
  941. if current.URL == nil && (current.FileName == nil || current.MimeType == nil) {
  942. logError(w, r, fmt.Errorf("Either url or file-name and mime-type must be given"))
  943. return
  944. }
  945. err = gDB.DB.Batch(func(tx *bolt.Tx) error {
  946. err = dbUpsertSource(tx, "insert", *current)
  947. sourceLog(current.SourceID, r, "created")
  948. gDB.CurrentCounters.Sources++
  949. if err != nil {
  950. return err
  951. }
  952. return extendAccessHandlers(tx, current)
  953. })
  954. if err != nil {
  955. logError(w, r, err)
  956. }
  957. result = *current.JSONStruct()
  958. w.Header().Set("Content-Type", contentTypeJSON)
  959. b, err = json.Marshal(result)
  960. if err != nil {
  961. log.Println("Marshal failed", result, err)
  962. w.WriteHeader(http.StatusInternalServerError)
  963. }
  964. var out bytes.Buffer
  965. json.Indent(&out, b, "", " ")
  966. out.WriteTo(w) // returns n, err
  967. }
  968. func sourceGetBinaryHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  969. sourceIDStr := pat.Param(ctx, "sourceid")
  970. err := gDB.DB.View(func(tx *bolt.Tx) error {
  971. source := getSourceOrPanic(tx, uuidDec(sourceIDStr))
  972. if source.Sha256 != nil {
  973. return sendRawFile(w, *source.Sha256, source.MimeType, source.FileName)
  974. }
  975. return fmt.Errorf("Not a binary")
  976. })
  977. if err != nil {
  978. logError(w, r, err)
  979. }
  980. }
  981. func sourceGetLog(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  982. sourceIDStr := pat.Param(ctx, "sourceid")
  983. sourceID := uuidDec(sourceIDStr)
  984. logFile, err := os.Open(constructLogPath(sourceID))
  985. // fmt.Println("sourceGetLog", logFile.Name())
  986. if err != nil {
  987. logError(w, r, err)
  988. w.WriteHeader(http.StatusInternalServerError)
  989. return
  990. }
  991. w.Header().Set("content-Type", "text/plain")
  992. io.Copy(w, logFile)
  993. }
  994. func sourceGetHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  995. var result JSONSource
  996. var current *DBSource
  997. sourceIDStr := pat.Param(ctx, "sourceid")
  998. var sourceID uuid.UUID
  999. if sourceIDStr != "" {
  1000. sourceID = uuidDec(sourceIDStr)
  1001. }
  1002. err := gDB.DB.View(func(tx *bolt.Tx) error {
  1003. current = getSourceOrPanic(tx, sourceID)
  1004. return nil
  1005. })
  1006. if err != nil {
  1007. logError(w, r, err)
  1008. }
  1009. result = *current.JSONStruct()
  1010. w.Header().Set("Content-Type", contentTypeJSON)
  1011. b, err := json.Marshal(result)
  1012. if err != nil {
  1013. log.Println("Marshal failed", result, err)
  1014. w.WriteHeader(http.StatusInternalServerError)
  1015. }
  1016. var out bytes.Buffer
  1017. json.Indent(&out, b, "", " ")
  1018. out.WriteTo(w) // returns n, err
  1019. }
  1020. func sourcePostHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  1021. var result JSONSource
  1022. var inputJ JSONSource
  1023. var input *DBSource
  1024. var current *DBSource
  1025. sourceIDStr := pat.Param(ctx, "sourceid")
  1026. var sourceID uuid.UUID
  1027. if sourceIDStr != "" {
  1028. sourceID = uuidDec(sourceIDStr)
  1029. }
  1030. b, err := ioutil.ReadAll(r.Body)
  1031. //fmt.Printf("body: %s\n", b)
  1032. err = json.Unmarshal(b, &inputJ)
  1033. if inputJ.APIKey == nil {
  1034. logError(w, r, fmt.Errorf("Unauthorized access"))
  1035. return
  1036. }
  1037. if setupAPIKey != "" && strings.Compare(*inputJ.APIKey, setupAPIKey) != 0 {
  1038. logError(w, r, fmt.Errorf("Unauthorized access: %s", *inputJ.APIKey))
  1039. return
  1040. }
  1041. input = inputJ.DBStruct()
  1042. if err != nil {
  1043. panic(err)
  1044. }
  1045. err = gDB.DB.Batch(func(tx *bolt.Tx) error {
  1046. current = getSourceOrPanic(tx, sourceID)
  1047. if input.NumHandlers > 500 {
  1048. return fmt.Errorf("Too many access handlers: %d", input.NumHandlers)
  1049. }
  1050. if current.NumHandlers < input.NumHandlers {
  1051. current.NumHandlers = input.NumHandlers
  1052. }
  1053. if input.StartDate != nil {
  1054. current.StartDate = input.StartDate
  1055. }
  1056. if input.EndDate != nil {
  1057. current.EndDate = input.EndDate
  1058. }
  1059. if len(input.AccessHandlers) != 0 {
  1060. for _, ah := range input.AccessHandlers {
  1061. a := getAccessOrPanic(tx, ah.AccessID)
  1062. if ah.StartDate != nil {
  1063. a.StartDate = ah.StartDate
  1064. }
  1065. if ah.EndDate != nil {
  1066. a.EndDate = ah.EndDate
  1067. }
  1068. err = dbUpsertAccess(tx, "update", *a)
  1069. if err != nil {
  1070. logError(w, r, fmt.Errorf("Could not update access object %s", uuidEnc(ah.AccessID)))
  1071. }
  1072. }
  1073. }
  1074. err = dbUpsertSource(tx, "update", *current)
  1075. if err != nil {
  1076. panic(fmt.Sprintf("Could not update source"))
  1077. }
  1078. sourceLog(current.SourceID, r, "updated")
  1079. return extendAccessHandlers(tx, current)
  1080. })
  1081. if err != nil {
  1082. logError(w, r, err)
  1083. }
  1084. result = *current.JSONStruct()
  1085. w.Header().Set("Content-Type", contentTypeJSON)
  1086. b, err = json.Marshal(result)
  1087. if err != nil {
  1088. log.Println("Marshal failed", result, err)
  1089. w.WriteHeader(http.StatusInternalServerError)
  1090. }
  1091. var out bytes.Buffer
  1092. json.Indent(&out, b, "", " ")
  1093. out.WriteTo(w) // returns n, err
  1094. }
  1095. func calculateHashForFile(fileName string) []byte {
  1096. f, err := os.Open(fileName)
  1097. if err != nil {
  1098. panic(err)
  1099. }
  1100. defer f.Close()
  1101. hash := sha256.New()
  1102. io.Copy(hash, f)
  1103. return hash.Sum(nil) // []byte
  1104. }
  1105. func sourcePutBinaryHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  1106. var result JSONSource
  1107. var current *DBSource
  1108. sourceIDStr := pat.Param(ctx, "sourceid")
  1109. var sourceID uuid.UUID
  1110. if sourceIDStr != "" {
  1111. sourceID = uuidDec(sourceIDStr)
  1112. }
  1113. //NOTE: PUT always needs client provided IDs and always overwrites (send full contents).
  1114. //Hence the only useful/allowed usage here is uploading binaries!
  1115. f, err := ioutil.TempFile(storePath, "tmp") // + "hej"
  1116. if err != nil {
  1117. logError(w, r, err)
  1118. return
  1119. }
  1120. tmpFileName := f.Name()
  1121. _, err = io.Copy(f, r.Body)
  1122. if err != nil {
  1123. logError(w, r, err)
  1124. return
  1125. }
  1126. f.Close()
  1127. md := calculateHashForFile(tmpFileName)
  1128. //mdSum := enc(md)
  1129. err = gDB.DB.Batch(func(tx *bolt.Tx) error {
  1130. current = getSourceOrPanic(tx, sourceID)
  1131. if current.Sha256 == nil {
  1132. current.Sha256 = &md
  1133. }
  1134. if bytes.Equal(md, *current.Sha256) {
  1135. //fmt.Printf("Sum: %v\n", mdSum)
  1136. err := os.Rename(tmpFileName, constructPath(md))
  1137. if err != nil {
  1138. return err
  1139. }
  1140. //fmt.Printf("## Renamed: %s to %s\n", tmpFileName, constructPath(md))
  1141. // define binary as uploaded to the result (and update the database accordingly)
  1142. current.Binary = true
  1143. sourceLog(current.SourceID, r, "updated")
  1144. return dbUpsertSource(tx, "update", *current)
  1145. }
  1146. fmt.Printf("Removing " + tmpFileName)
  1147. os.Remove(tmpFileName)
  1148. //w.WriteHeader(http.StatusBadRequest)
  1149. return fmt.Errorf("Missmatch SHA256 sum, expected %s", *current.Sha256)
  1150. })
  1151. if err != nil {
  1152. logError(w, r, err)
  1153. }
  1154. result = *current.JSONStruct()
  1155. w.Header().Set("Content-Type", contentTypeJSON)
  1156. b, err := json.Marshal(result)
  1157. if err != nil {
  1158. log.Println("Marshal failed", result, err)
  1159. w.WriteHeader(http.StatusInternalServerError)
  1160. }
  1161. var out bytes.Buffer
  1162. json.Indent(&out, b, "", " ")
  1163. out.WriteTo(w)
  1164. }
  1165. // upload logic (CURRENTLY NOT USED)
  1166. /*
  1167. func formUpload(ctx context.Context, w http.ResponseWriter, r *http.Request) {
  1168. fmt.Println("method:", r.Method)
  1169. //if r.Method == "GET" {
  1170. //crutime := time.Now().Unix()
  1171. //h := md5.New()
  1172. //io.WriteString(h, strconv.FormatInt(crutime, 10))
  1173. //token := fmt.Sprintf("%x", h.Sum(nil))
  1174. //t, _ := template.ParseFiles("upload.gtpl")
  1175. //t.Execute(w, token)
  1176. //} else {
  1177. r.ParseMultipartForm(32 << 20)
  1178. file, handler, err := r.FormFile("binary")
  1179. if err != nil {
  1180. log.Fatal(err)
  1181. return
  1182. }
  1183. fmt.Println("binary=", file)
  1184. defer file.Close()
  1185. //fmt.Fprintf(w, "Header: %v", handler.Header)
  1186. fmt.Printf("Header: %#v\n", handler.Header)
  1187. fmt.Printf("Disposition: %v\n", handler.Header["Content-Disposition"][0])
  1188. fmt.Printf("Content Type: %v\n", handler.Header["Content-Type"][0])
  1189. f, err := os.OpenFile("./test/"+handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
  1190. if err != nil {
  1191. log.Fatal(err)
  1192. return
  1193. }
  1194. defer f.Close()
  1195. io.Copy(f, file)
  1196. u4 := uuid.NewV4()
  1197. fmt.Fprintf(w, `{"source":"%s"}\n`, u4)
  1198. //}
  1199. }
  1200. */
  1201. /*
  1202. This function is executed as a go routine.
  1203. It is a simple scheduler for periodic work, such as backup.
  1204. */
  1205. func periodicExecution(db *DaisyDB) {
  1206. time.Sleep(5 * time.Second)
  1207. for {
  1208. log.Println("Periodic execution start")
  1209. log.Print(backupDB(db))
  1210. log.Println("Expire binaries")
  1211. log.Print(expireBinaries(db))
  1212. log.Println("Periodic execution ends")
  1213. nextInterval := viper.GetDuration("backup-interval")
  1214. //log.Println("Next execution: "+nextInterval)
  1215. time.Sleep(nextInterval) //8 * time.Hour)
  1216. }
  1217. }
  1218. func pulseExecution(db *DaisyDB) {
  1219. //previousStats := db.DB.Stats()
  1220. for {
  1221. log.Println("Pulse")
  1222. db.CurrentCounters.LastUpdate = time.Now().Round(time.Second)
  1223. db.CountersDiff = db.CurrentCounters.Sub(db.previousCounters)
  1224. db.previousCounters = db.CurrentCounters
  1225. nextInterval := viper.GetDuration("pulse-interval")
  1226. //log.Println("Next execution: "+nextInterval)
  1227. refreshStats(db)
  1228. //stats := db.DB.Stats()
  1229. //diff := stats.Sub(&previousStats)
  1230. ////json.NewEncoder(os.Stdout).Encode(diff)
  1231. //db.DBStats = stats
  1232. time.Sleep(nextInterval)
  1233. }
  1234. }
  1235. func catchCtrlC(db *bolt.DB) {
  1236. c := make(chan os.Signal, 1)
  1237. signal.Notify(c, os.Interrupt)
  1238. go func() {
  1239. for sig := range c {
  1240. // sig is a ^C, handle it
  1241. log.Printf("Caught interrupt %#v\n", sig)
  1242. db.Close()
  1243. log.Println("Database closed. Exiting.")
  1244. os.Exit(0)
  1245. }
  1246. }()
  1247. }
  1248. func openLogFiles() {
  1249. requestLogFname := "request.log"
  1250. errorLogFname := "error.log"
  1251. var err error
  1252. os.Mkdir(logsPath, pathMode)
  1253. requestLog, err = os.OpenFile(path.Join(logsPath, requestLogFname), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
  1254. if err != nil {
  1255. log.Fatal(err)
  1256. }
  1257. errorLog, err = os.OpenFile(path.Join(logsPath, errorLogFname), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
  1258. if err != nil {
  1259. log.Fatal(err)
  1260. }
  1261. }
  1262. func listenHTTP() {
  1263. //if *APIKey == "" {
  1264. //log.Println("*** You need to give an APIkey!")
  1265. //flag.PrintDefaults()
  1266. //return
  1267. //}
  1268. portStr := strconv.Itoa(viper.GetInt("port"))
  1269. prefix := viper.GetString("prefix")
  1270. fetchPrefix := viper.GetString("fetch-prefix")
  1271. uuidStyle = viper.GetString("uuid-style")
  1272. setupAPIKey = viper.GetString("api-key")
  1273. log.Println("Port: " + portStr)
  1274. log.Println("ServerURL: " + viper.GetString("server"))
  1275. log.Println("PathPrefix: " + prefix)
  1276. log.Println("uuidStyle: " + uuidStyle)
  1277. mux := goji.NewMux()
  1278. //http.HandleFunc(prefix+"/source/", http.StripPrefix(prefix, sourceHandler))
  1279. //http.HandleFunc(prefix+"/source/", sourceHandler)
  1280. //http.HandleFunc(prefix+"/access/", accessHandler)
  1281. //http.HandleFunc(prefix+fetchPrefix, fetchHandler)
  1282. //http.HandleFunc(prefix+"/upload", formUpload)
  1283. //mux.HandleFuncC(pat.Get(prefix+"/wui/:cmd"), wuiHandler)
  1284. mux.UseC(logging) // middleware logging
  1285. mux.HandleFuncC(pat.Get(prefix+"/wui/:cmd"), wuiHandler)
  1286. mux.HandleFuncC(pat.Post(prefix+"/source/"), sourcePostNewHandler)
  1287. mux.HandleFuncC(pat.Get(prefix+"/source/:sourceid"), sourceGetHandler)
  1288. mux.HandleFuncC(pat.Get(prefix+"/source/:sourceid/log"), sourceGetLog)
  1289. mux.HandleFuncC(pat.Post(prefix+"/source/:sourceid"), sourcePostHandler)
  1290. mux.HandleFuncC(pat.Put(prefix+"/source/:sourceid/binary"), sourcePutBinaryHandler)
  1291. mux.HandleFuncC(pat.Get(prefix+"/source/:sourceid/binary"), sourceGetBinaryHandler)
  1292. mux.HandleFuncC(pat.Get(prefix+fetchPrefix+":accessid"), fetchHandler)
  1293. mux.HandleFuncC(pat.Get(prefix+"/raw/*"), rawHandler)
  1294. mux.Handle(pat.Get(prefix+"/*"), http.FileServer(http.Dir("static")))
  1295. //mux.HandleC(pat.Get(prefix+"/"), logging("static", goji.FileServer(http.Dir("static"))))
  1296. //err := http.ListenAndServeTLS(":10443", "cert.pem", "key.pem", nil)
  1297. //if err := http.ListenAndServe(":"+portStr, nil); err != nil {
  1298. if err := http.ListenAndServe(":"+portStr, mux); err != nil {
  1299. log.Fatal(err)
  1300. }
  1301. }
  1302. func prepareDatabase() (*DaisyDB, error) {
  1303. os.Mkdir(storePath, pathMode)
  1304. var db DaisyDB
  1305. dbFileName := path.Join(storePath, viper.GetString("dbfile"))
  1306. log.Println("Opening DB: " + dbFileName)
  1307. lDB, err := bolt.Open(dbFileName, 0600, &bolt.Options{Timeout: 5 * time.Second})
  1308. db.ServerStartTime = time.Now().Round(time.Second)
  1309. db.DB = lDB
  1310. if err := initDB(db.DB); err != nil {
  1311. return nil, err
  1312. }
  1313. db.DBStats = db.DB.Stats()
  1314. countObjects(&db)
  1315. if err != nil {
  1316. return nil, fmt.Errorf("Opening %s: %s", dbFileName, err)
  1317. }
  1318. catchCtrlC(db.DB)
  1319. return &db, nil
  1320. }
  1321. func setupViper() {
  1322. viper.SetConfigName("daisy") // name of config file (without extension)
  1323. viper.AddConfigPath(".") //current
  1324. viper.SetDefault("port", 8090)
  1325. viper.SetDefault("uuid-style", "base64")
  1326. viper.SetDefault("fetch-prefix", "/g/")
  1327. viper.SetDefault("dump-file", "db_dump_current.tsv")
  1328. viper.SetDefault("backup-interval", time.Duration(8*time.Hour))
  1329. viper.SetDefault("pulse-interval", time.Duration(5*time.Minute))
  1330. viper.SetDefault("store", "store")
  1331. viper.SetDefault("logs", "logs")
  1332. viper.SetDefault("dbfile", "daisy.bolt.db")
  1333. viper.SetDefault("default-end-months", 3)
  1334. viper.SetDefault("retention-days", 7)
  1335. viper.SetDefault("stacktrace", false)
  1336. if err := viper.ReadInConfig(); err == nil {
  1337. log.Println("Using config file: " + viper.ConfigFileUsed())
  1338. } else {
  1339. panic(fmt.Errorf("could not load config file"))
  1340. }
  1341. urlPathPrefix = viper.GetString("server") + viper.GetString("prefix") + viper.GetString("fetch-prefix")
  1342. storePath = viper.GetString("store")
  1343. logsPath = viper.GetString("logs")
  1344. }
  1345. func main() {
  1346. log.Println("--== Daisy ==--")
  1347. log.Println("2016 J Ramb")
  1348. log.Println("Version:", daisyVersion)
  1349. flag.Parse()
  1350. args := flag.Args()
  1351. //if cfgFile != "" { // enable ability to specify config file via flag
  1352. //viper.SetConfigFile(cfgFile)
  1353. //}
  1354. setupViper()
  1355. openLogFiles()
  1356. var err error
  1357. if gDB, err = prepareDatabase(); err != nil {
  1358. log.Fatal(err)
  1359. }
  1360. defer func() {
  1361. gDB.DB.Close()
  1362. log.Println("Database closed")
  1363. }()
  1364. var command string
  1365. if len(args) > 0 {
  1366. command = args[0]
  1367. } else {
  1368. command = "server" // default
  1369. }
  1370. log.Println("Command line command:", command)
  1371. switch command {
  1372. case "server":
  1373. gDB.dbNeedsBackup = true
  1374. go periodicExecution(gDB) // simple periodic execution
  1375. go pulseExecution(gDB) // simple periodic execution
  1376. listenHTTP()
  1377. case "backup":
  1378. {
  1379. gDB.dbNeedsBackup = true
  1380. log.Println(backupDB(gDB))
  1381. }
  1382. case "dump":
  1383. err = databaseDump(gDB, "")
  1384. if err != nil {
  1385. log.Println(err)
  1386. }
  1387. default:
  1388. log.Println("Invalid cmd:", command)
  1389. }
  1390. }