replace zxq.co/ripple/hanayo

vendor/github.com/osuripple/cheesegull/housekeeper/binarystate.go

@@ -0,0 +1,113 @@
+package housekeeper
+
+import (
+	"encoding"
+	"errors"
+	"io"
+)
+
+const cachedBeatmapBinSize = 8 + 1 + 15 + 15 + 8
+
+func b2i(b bool) byte {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+func marshalBinaryCopy(dst []byte, t encoding.BinaryMarshaler) {
+	b, _ := t.MarshalBinary()
+	copy(dst, b)
+}
+
+// copied from binary.BigEndian
+func putUint64(b []byte, v uint64) {
+	_ = b[7] // boundary check
+	b[0] = byte(v >> 56)
+	b[1] = byte(v >> 48)
+	b[2] = byte(v >> 40)
+	b[3] = byte(v >> 32)
+	b[4] = byte(v >> 24)
+	b[5] = byte(v >> 16)
+	b[6] = byte(v >> 8)
+	b[7] = byte(v)
+}
+
+func writeBeatmaps(w io.Writer, c []*CachedBeatmap) error {
+	_, err := w.Write(append([]byte("CGBIN001"), cachedBeatmapBinSize))
+	if err != nil {
+		return err
+	}
+	for _, b := range c {
+		if b == nil || !b.isDownloaded {
+			continue
+		}
+		enc := make([]byte, cachedBeatmapBinSize)
+		putUint64(enc[:8], uint64(b.ID))
+		enc[8] = b2i(b.NoVideo)
+		marshalBinaryCopy(enc[9:24], b.LastUpdate)
+		marshalBinaryCopy(enc[24:39], b.lastRequested)
+		putUint64(enc[39:47], b.fileSize)
+		_, err := w.Write(enc)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// copied from binary.BigEndian
+func readUint64(b []byte) uint64 {
+	_ = b[7] // bounds check hint to compiler; see golang.org/issue/14808
+	return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 |
+		uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56
+
+}
+
+func readCachedBeatmap(b []byte) *CachedBeatmap {
+	m := &CachedBeatmap{}
+	m.ID = int(readUint64(b[:8]))
+	m.NoVideo = b[8] == 1
+	(&m.LastUpdate).UnmarshalBinary(b[9:24])
+	(&m.lastRequested).UnmarshalBinary(b[24:39])
+	m.fileSize = readUint64(b[39:47])
+	m.isDownloaded = true
+	return m
+}
+
+func readBeatmaps(r io.Reader) ([]*CachedBeatmap, error) {
+	b := make([]byte, 8)
+	_, err := r.Read(b)
+	if err != nil {
+		return nil, err
+	}
+	if string(b) != "CGBIN001" {
+		return nil, errors.New("unknown cgbin version")
+	}
+
+	b = make([]byte, 1)
+	_, err = r.Read(b)
+	if err != nil {
+		return nil, err
+	}
+
+	bmLength := b[0]
+	if bmLength == 0 {
+		return nil, nil
+	}
+	b = make([]byte, bmLength)
+	beatmaps := make([]*CachedBeatmap, 0, 50)
+
+	for {
+		read, err := r.Read(b)
+		switch {
+		case err == io.EOF:
+			return beatmaps, nil
+		case err != nil:
+			return nil, err
+		case byte(read) != bmLength:
+			return beatmaps, nil
+		}
+		beatmaps = append(beatmaps, readCachedBeatmap(b))
+	}
+}

vendor/github.com/osuripple/cheesegull/housekeeper/binarystate_test.go

@@ -0,0 +1,79 @@
+package housekeeper
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+	"time"
+)
+
+var testBeatmaps = []*CachedBeatmap{
+	&CachedBeatmap{
+		isDownloaded: true,
+	},
+	&CachedBeatmap{
+		ID:           851,
+		NoVideo:      true,
+		isDownloaded: true,
+	},
+	&CachedBeatmap{
+		ID:           1337777,
+		fileSize:     58111,
+		isDownloaded: true,
+	},
+	&CachedBeatmap{
+		ID:            851,
+		LastUpdate:    time.Date(2017, 9, 21, 11, 11, 50, 0, time.UTC),
+		lastRequested: time.Date(2017, 9, 21, 22, 11, 50, 0, time.UTC),
+		isDownloaded:  true,
+	},
+}
+
+func TestEncodeDecode(t *testing.T) {
+	buf := &bytes.Buffer{}
+
+	start := time.Now()
+	err := writeBeatmaps(buf, testBeatmaps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("Write took %v", time.Since(start))
+
+	start = time.Now()
+	readBMs, err := readBeatmaps(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("Read took %v", time.Since(start))
+
+	if !reflect.DeepEqual(readBMs, testBeatmaps) {
+		t.Fatalf("original %v read %v", testBeatmaps, readBMs)
+	}
+}
+
+func BenchmarkWriteBinaryState(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		writeBeatmaps(fakeWriter{}, testBeatmaps)
+	}
+}
+
+func BenchmarkReadBinaryState(b *testing.B) {
+	buf := &bytes.Buffer{}
+	err := writeBeatmaps(buf, testBeatmaps)
+	if err != nil {
+		b.Fatal(err)
+	}
+	bufBytes := buf.Bytes()
+	bReader := bytes.NewReader(bufBytes)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		readBeatmaps(bReader)
+		bReader.Reset(bufBytes)
+	}
+}
+
+type fakeWriter struct{}
+
+func (fakeWriter) Write(b []byte) (int, error) {
+	return len(b), nil
+}

vendor/github.com/osuripple/cheesegull/housekeeper/housekeeper.go

@@ -0,0 +1,188 @@
+// Package housekeeper manages the local cache of CheeseGull, by always keeping
+// track of the local state of the cache and keeping it to a low amount.
+package housekeeper
+
+import (
+	"log"
+	"os"
+	"sort"
+	"sync"
+
+	raven "github.com/getsentry/raven-go"
+)
+
+// House manages the state of the cached beatmaps in the local filesystem.
+type House struct {
+	MaxSize     uint64
+	state       []*CachedBeatmap
+	stateMutex  sync.RWMutex
+	requestChan chan struct{}
+	// set to non-nil to avoid calling os.Remove on the files to remove, and
+	// place them here instead.
+	dryRun []*CachedBeatmap
+}
+
+// New creates a new house, initialised with the default values.
+func New() *House {
+	return &House{
+		MaxSize:     1024 * 1024 * 1024 * 10, // 10 gigs
+		requestChan: make(chan struct{}, 1),
+	}
+}
+
+// scheduleCleanup enschedules a housekeeping request if one isn't already
+// present.
+func (h *House) scheduleCleanup() {
+	select {
+	case h.requestChan <- struct{}{}:
+		// carry on
+	default:
+		// carry on
+	}
+}
+
+// StartCleaner starts the process that will do the necessary housekeeping
+// every time a cleanup is scheduled with scheduleCleanup.
+func (h *House) StartCleaner() {
+	go func() {
+		for {
+			<-h.requestChan
+			h.cleanUp()
+		}
+	}()
+}
+
+func (h *House) cleanUp() {
+	log.Println("[C] Running cleanup")
+
+	toRemove := h.mapsToRemove()
+
+	f, err := os.Create("cgbin.db")
+	if err != nil {
+		logError(err)
+		return
+	}
+
+	// build new state by removing from it the beatmaps from toRemove
+	h.stateMutex.Lock()
+	newState := make([]*CachedBeatmap, 0, len(h.state))
+StateLoop:
+	for _, b := range h.state {
+		for _, r := range toRemove {
+			if r.ID == b.ID && r.NoVideo == b.NoVideo {
+				continue StateLoop
+			}
+		}
+		newState = append(newState, b)
+	}
+	h.state = newState
+	err = writeBeatmaps(f, h.state)
+	h.stateMutex.Unlock()
+
+	f.Close()
+	if err != nil {
+		logError(err)
+		return
+	}
+
+	if h.dryRun != nil {
+		h.dryRun = toRemove
+		return
+	}
+
+	for _, b := range toRemove {
+		err := os.Remove(b.fileName())
+		switch {
+		case err == nil, os.IsNotExist(err):
+			// silently ignore
+		default:
+			logError(err)
+		}
+	}
+}
+
+func (h *House) mapsToRemove() []*CachedBeatmap {
+	totalSize, removable := h.stateSizeAndRemovableMaps()
+
+	if totalSize <= h.MaxSize {
+		// no clean up needed, our totalSize has still not gotten over the
+		// threshold
+		return nil
+	}
+
+	sortByLastRequested(removable)
+
+	removeBytes := int(totalSize - h.MaxSize)
+	var toRemove []*CachedBeatmap
+	for _, b := range removable {
+		toRemove = append(toRemove, b)
+		fSize := b.FileSize()
+		removeBytes -= int(fSize)
+		if removeBytes <= 0 {
+			break
+		}
+	}
+
+	return toRemove
+}
+
+// i hate verbose names myself, but it was very hard to come up with something
+// even as short as this.
+func (h *House) stateSizeAndRemovableMaps() (totalSize uint64, removable []*CachedBeatmap) {
+	h.stateMutex.RLock()
+	for _, b := range h.state {
+		if !b.IsDownloaded() {
+			continue
+		}
+		fSize := b.FileSize()
+		totalSize += fSize
+		if fSize == 0 {
+			continue
+		}
+		removable = append(removable, b)
+	}
+	h.stateMutex.RUnlock()
+	return
+}
+
+func sortByLastRequested(b []*CachedBeatmap) {
+	sort.Slice(b, func(i, j int) bool {
+		b[i].mtx.RLock()
+		b[j].mtx.RLock()
+		r := b[i].lastRequested.Before(b[j].lastRequested)
+		b[i].mtx.RUnlock()
+		b[j].mtx.RUnlock()
+		return r
+	})
+}
+
+// LoadState attempts to load the state from cgbin.db
+func (h *House) LoadState() error {
+	f, err := os.Open("cgbin.db")
+	switch {
+	case os.IsNotExist(err):
+		return nil
+	case err != nil:
+		return err
+	}
+	defer f.Close()
+
+	h.stateMutex.Lock()
+	h.state, err = readBeatmaps(f)
+	h.stateMutex.Unlock()
+
+	return err
+}
+
+var envSentryDSN = os.Getenv("SENTRY_DSN")
+
+// logError attempts to log an error to Sentry, as well as stdout.
+func logError(err error) {
+	if err == nil {
+		return
+	}
+	if envSentryDSN != "" {
+		raven.CaptureError(err, nil)
+	}
+	log.Println(err)
+}

vendor/github.com/osuripple/cheesegull/housekeeper/housekeeper_test.go

@@ -0,0 +1,129 @@
+package housekeeper
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestCleanupOneMap(t *testing.T) {
+	expectRemove := []*CachedBeatmap{
+		&CachedBeatmap{
+			ID:            1,
+			lastRequested: time.Date(2017, 4, 5, 15, 5, 3, 0, time.UTC),
+			fileSize:      15,
+			isDownloaded:  true,
+		},
+	}
+	expectRemain := []*CachedBeatmap{
+		&CachedBeatmap{
+			ID:            2,
+			lastRequested: time.Date(2017, 4, 10, 15, 5, 3, 0, time.UTC),
+			fileSize:      15,
+			isDownloaded:  true,
+		},
+		&CachedBeatmap{
+			ID:            3,
+			lastRequested: time.Date(2017, 4, 15, 15, 5, 3, 0, time.UTC),
+			fileSize:      15,
+			isDownloaded:  true,
+		},
+		&CachedBeatmap{
+			ID:            4,
+			lastRequested: time.Date(2017, 4, 20, 15, 5, 3, 0, time.UTC),
+			fileSize:      15,
+			isDownloaded:  true,
+		},
+	}
+
+	h := New()
+	h.MaxSize = 50
+	h.state = append(expectRemain, expectRemove...)
+	h.dryRun = make([]*CachedBeatmap, 0)
+
+	start := time.Now()
+	h.cleanUp()
+	t.Log("cleanup took", time.Since(start))
+
+	if !reflect.DeepEqual(expectRemain, h.state) {
+		t.Errorf("Want %v got %v", expectRemain, h.state)
+	}
+	if !reflect.DeepEqual(expectRemove, h.dryRun) {
+		t.Errorf("Want %v got %v", expectRemove, h.dryRun)
+	}
+}
+
+func TestCleanupNoMaps(t *testing.T) {
+	expectRemove := []*CachedBeatmap{}
+	expectRemain := []*CachedBeatmap{
+		&CachedBeatmap{
+			ID:            1,
+			lastRequested: time.Date(2017, 4, 10, 15, 5, 3, 0, time.UTC),
+			fileSize:      10,
+			isDownloaded:  true,
+		},
+	}
+
+	h := New()
+	h.MaxSize = 10
+	h.state = append(expectRemain, expectRemove...)
+	h.dryRun = make([]*CachedBeatmap, 0)
+
+	start := time.Now()
+	h.cleanUp()
+	t.Log("cleanup took", time.Since(start))
+
+	if !reflect.DeepEqual(expectRemain, h.state) {
+		t.Errorf("Want %v got %v", expectRemain, h.state)
+	}
+	if !reflect.DeepEqual(expectRemove, h.dryRun) {
+		t.Errorf("Want %v got %v", expectRemove, h.dryRun)
+	}
+}
+
+func TestCleanupEmptyBeatmaps(t *testing.T) {
+	expectRemove := []*CachedBeatmap{
+		&CachedBeatmap{
+			ID:            1,
+			lastRequested: time.Date(2017, 4, 10, 15, 5, 3, 0, time.UTC),
+			fileSize:      10,
+			isDownloaded:  true,
+		},
+	}
+	expectRemain := []*CachedBeatmap{
+		&CachedBeatmap{
+			ID:            2,
+			lastRequested: time.Date(2017, 4, 5, 15, 5, 3, 0, time.UTC),
+			fileSize:      0,
+			isDownloaded:  true,
+		},
+		&CachedBeatmap{
+			ID:            3,
+			lastRequested: time.Date(2017, 4, 4, 15, 5, 3, 0, time.UTC),
+			fileSize:      0,
+			isDownloaded:  true,
+		},
+		&CachedBeatmap{
+			ID:            4,
+			lastRequested: time.Date(2017, 4, 3, 15, 5, 3, 0, time.UTC),
+			fileSize:      0,
+			isDownloaded:  true,
+		},
+	}
+
+	h := New()
+	h.MaxSize = 5
+	h.state = append(expectRemain, expectRemove...)
+	h.dryRun = make([]*CachedBeatmap, 0)
+
+	start := time.Now()
+	h.cleanUp()
+	t.Log("cleanup took", time.Since(start))
+
+	if !reflect.DeepEqual(expectRemain, h.state) {
+		t.Errorf("Want %v got %v", expectRemain, h.state)
+	}
+	if !reflect.DeepEqual(expectRemove, h.dryRun) {
+		t.Errorf("Want %v got %v", expectRemove, h.dryRun)
+	}
+}

vendor/github.com/osuripple/cheesegull/housekeeper/state.go

@@ -0,0 +1,158 @@
+package housekeeper
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// CachedBeatmap represents a beatmap that is held in the cache of CheeseGull.
+type CachedBeatmap struct {
+	ID         int
+	NoVideo    bool
+	LastUpdate time.Time
+
+	lastRequested time.Time
+
+	fileSize     uint64
+	isDownloaded bool
+	mtx          sync.RWMutex
+	waitGroup    sync.WaitGroup
+}
+
+// File opens the File of the beatmap from the filesystem.
+func (c *CachedBeatmap) File() (*os.File, error) {
+	return os.Open(c.fileName())
+}
+
+// CreateFile creates the File of the beatmap in the filesystem, and returns it
+// in write mode.
+func (c *CachedBeatmap) CreateFile() (*os.File, error) {
+	return os.Create(c.fileName())
+}
+
+func (c *CachedBeatmap) fileName() string {
+	n := ""
+	if c.NoVideo {
+		n = "n"
+	}
+	return "data/" + strconv.Itoa(c.ID) + n + ".osz"
+}
+
+// IsDownloaded checks whether the beatmap has been downloaded.
+func (c *CachedBeatmap) IsDownloaded() bool {
+	c.mtx.RLock()
+	i := c.isDownloaded
+	c.mtx.RUnlock()
+	return i
+}
+
+// FileSize returns the FileSize of c.
+func (c *CachedBeatmap) FileSize() uint64 {
+	c.mtx.RLock()
+	i := c.fileSize
+	c.mtx.RUnlock()
+	return i
+}
+
+// MustBeDownloaded will check whether the beatmap is downloaded.
+// If it is not, it will wait for it to become downloaded.
+func (c *CachedBeatmap) MustBeDownloaded() {
+	if c.IsDownloaded() {
+		return
+	}
+	c.waitGroup.Wait()
+}
+
+// DownloadCompleted must be called once the beatmap has finished downloading.
+func (c *CachedBeatmap) DownloadCompleted(fileSize uint64, parentHouse *House) {
+	c.mtx.Lock()
+	c.fileSize = fileSize
+	c.isDownloaded = true
+	c.mtx.Unlock()
+	c.waitGroup.Done()
+	parentHouse.scheduleCleanup()
+}
+
+// SetLastRequested changes the last requested time.
+func (c *CachedBeatmap) SetLastRequested(t time.Time) {
+	c.mtx.Lock()
+	c.lastRequested = t
+	c.mtx.Unlock()
+}
+
+func (c *CachedBeatmap) String() string {
+	return fmt.Sprintf("{ID: %d NoVideo: %t LastUpdate: %v}", c.ID, c.NoVideo, c.LastUpdate)
+}
+
+// AcquireBeatmap attempts to add a new CachedBeatmap to the state.
+// In order to add a new CachedBeatmap to the state, one must not already exist
+// in the state with the same ID, NoVideo and LastUpdate. In case one is already
+// found, this is returned, alongside with false. If LastUpdate is newer than
+// that of the beatmap stored in the state, then the beatmap in the state's
+// downloaded status is switched back to false and the LastUpdate is changed.
+// true is also returned, indicating that the caller now has the burden of
+// downloading the beatmap.
+//
+// In the case the cachedbeatmap has not been stored in the state, then
+// it is added to the state and, like the case where LastUpdated has been
+// changed, true is returned, indicating that the caller must now download the
+// beatmap.
+//
+// If you're confused attempting to read this, let me give you an example:
+//
+//   A: Yo, is this beatmap cached?
+//   B: Yes, yes it is! Here you go with the information about it. No need to do
+//      anything else.
+//      ----
+//   A: Yo, got this beatmap updated 2 hours ago. Have you got it cached?
+//   B: Ah, I'm afraid that I only have the version updated 10 hours ago.
+//      Mind downloading the updated version for me?
+//      ----
+//   A: Yo, is this beatmap cached?
+//   B: Nope, I didn't know it existed before you told me. I've recorded its
+//      info now, but jokes on you, you now have to actually download it.
+//      Chop chop!
+func (h *House) AcquireBeatmap(c *CachedBeatmap) (*CachedBeatmap, bool) {
+	if c == nil {
+		return nil, false
+	}
+
+	h.stateMutex.RLock()
+	for _, b := range h.state {
+		// if the id or novideo is different, then all is good and we
+		// can proceed with the next element.
+		if b.ID != c.ID || b.NoVideo != c.NoVideo {
+			continue
+		}
+		// unlocking because in either branch, we will return.
+		h.stateMutex.RUnlock()
+
+		// if c is not newer than b, then just return.
+		if !b.LastUpdate.Before(c.LastUpdate) {
+			return b, false
+		}
+
+		b.LastUpdate = c.LastUpdate
+		return b, true
+	}
+	h.stateMutex.RUnlock()
+
+	// c was not present in our state: we need to add it.
+
+	// we need to recreate the CachedBeatmap: this way we can be sure the zero
+	// is set for the unexported fields.
+	n := &CachedBeatmap{
+		ID:         c.ID,
+		NoVideo:    c.NoVideo,
+		LastUpdate: c.LastUpdate,
+	}
+	n.waitGroup.Add(1)
+	h.stateMutex.Lock()
+	h.state = append(h.state, n)
+	h.stateMutex.Unlock()
+
+	return n, true
+}