Move to fasthttp for improved performance

This commit is contained in:
Morgan Bazalgette
2017-02-02 13:40:28 +01:00
parent ace2fded7e
commit 85e6dc7e5e
26 changed files with 448 additions and 380 deletions

View File

@@ -1,17 +1,11 @@
// Package internals has methods that suit none of the API packages.
package internals
import (
"github.com/gin-gonic/gin"
)
import "github.com/valyala/fasthttp"
type statusResponse struct {
Status int `json:"status"`
}
var statusResp = []byte(`{ "status": 1 }`)
// Status is used for checking the API is up by the ripple website, on the status page.
func Status(c *gin.Context) {
c.JSON(200, statusResponse{
Status: 1,
})
func Status(c *fasthttp.RequestCtx) {
c.Write(statusResp)
}

View File

@@ -1,54 +1,43 @@
package app
import (
"crypto/md5"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"regexp"
"strings"
"unsafe"
"github.com/valyala/fasthttp"
"zxq.co/ripple/rippleapi/common"
"github.com/gin-gonic/gin"
)
// Method wraps an API method to a HandlerFunc.
func Method(f func(md common.MethodData) common.CodeMessager, privilegesNeeded ...int) gin.HandlerFunc {
return func(c *gin.Context) {
func Method(f func(md common.MethodData) common.CodeMessager, privilegesNeeded ...int) fasthttp.RequestHandler {
return func(c *fasthttp.RequestCtx) {
initialCaretaker(c, f, privilegesNeeded...)
}
}
func initialCaretaker(c *gin.Context, f func(md common.MethodData) common.CodeMessager, privilegesNeeded ...int) {
rateLimiter()
func initialCaretaker(c *fasthttp.RequestCtx, f func(md common.MethodData) common.CodeMessager, privilegesNeeded ...int) {
var doggoTags []string
data, err := ioutil.ReadAll(c.Request.Body)
if err != nil {
c.Error(err)
}
token := ""
qa := c.Request.URI().QueryArgs()
var token string
switch {
case c.Request.Header.Get("X-Ripple-Token") != "":
token = c.Request.Header.Get("X-Ripple-Token")
case c.Query("token") != "":
token = c.Query("token")
case c.Query("k") != "":
token = c.Query("k")
case len(c.Request.Header.Peek("X-Ripple-Token")) > 0:
token = string(c.Request.Header.Peek("X-Ripple-Token"))
case len(qa.Peek("token")) > 0:
token = string(qa.Peek("token"))
case len(qa.Peek("k")) > 0:
token = string(qa.Peek("k"))
default:
token, _ = c.Cookie("rt")
token = string(c.Request.Header.Cookie("rt"))
}
c.Set("token", fmt.Sprintf("%x", md5.Sum([]byte(token))))
md := common.MethodData{
DB: db,
RequestData: data,
C: c,
Doggo: doggo,
R: red,
DB: db,
Ctx: c,
Doggo: doggo,
R: red,
}
if token != "" {
tokenReal, exists := GetTokenFull(token, db)
@@ -58,25 +47,8 @@ func initialCaretaker(c *gin.Context, f func(md common.MethodData) common.CodeMe
}
}
var ip string
if requestIP, _, err := net.SplitHostPort(strings.TrimSpace(c.Request.RemoteAddr)); err != nil {
panic(err)
} else {
// if requestIP is not 127.0.0.1, means no reverse proxy is being used => direct request.
if requestIP != "127.0.0.1" {
ip = requestIP
}
}
// means we're using reverse-proxy, so X-Real-IP
if ip == "" {
ip = c.ClientIP()
}
// requests from hanayo should not be rate limited.
if !(c.Request.Header.Get("H-Key") == cf.HanayoKey && c.Request.UserAgent() == "hanayo") {
perUserRequestLimiter(md.ID(), c.ClientIP())
} else {
// log into datadog that this is an hanayo request
if b2s(c.Request.Header.Peek("H-Key")) == cf.HanayoKey && b2s(c.UserAgent()) == "hanayo" {
doggoTags = append(doggoTags, "hanayo")
}
@@ -89,21 +61,22 @@ func initialCaretaker(c *gin.Context, f func(md common.MethodData) common.CodeMe
}
}
if missingPrivileges != 0 {
c.IndentedJSON(401, common.SimpleResponse(401, "You don't have the privilege(s): "+common.Privileges(missingPrivileges).String()+"."))
c.SetStatusCode(401)
mkjson(c, common.SimpleResponse(401, "You don't have the privilege(s): "+common.Privileges(missingPrivileges).String()+"."))
return
}
resp := f(md)
if md.HasQuery("pls200") {
c.Writer.WriteHeader(200)
c.SetStatusCode(200)
} else {
c.Writer.WriteHeader(resp.GetCode())
c.SetStatusCode(resp.GetCode())
}
if md.HasQuery("callback") {
c.Header("Content-Type", "application/javascript; charset=utf-8")
c.Response.Header.Add("Content-Type", "application/javascript; charset=utf-8")
} else {
c.Header("Content-Type", "application/json; charset=utf-8")
c.Response.Header.Add("Content-Type", "application/json; charset=utf-8")
}
mkjson(c, resp)
@@ -113,22 +86,31 @@ func initialCaretaker(c *gin.Context, f func(md common.MethodData) common.CodeMe
var callbackJSONP = regexp.MustCompile(`^[a-zA-Z_\$][a-zA-Z0-9_\$]*$`)
// mkjson auto indents json, and wraps json into a jsonp callback if specified by the request.
// then writes to the gin.Context the data.
func mkjson(c *gin.Context, data interface{}) {
// then writes to the RequestCtx the data.
func mkjson(c *fasthttp.RequestCtx, data interface{}) {
exported, err := json.MarshalIndent(data, "", "\t")
if err != nil {
c.Error(err)
fmt.Println(err)
exported = []byte(`{ "code": 500, "message": "something has gone really really really really really really wrong." }`)
}
cb := c.Query("callback")
cb := string(c.URI().QueryArgs().Peek("callback"))
willcb := cb != "" &&
len(cb) < 100 &&
callbackJSONP.MatchString(cb)
if willcb {
c.Writer.Write([]byte("/**/ typeof " + cb + " === 'function' && " + cb + "("))
c.Write([]byte("/**/ typeof " + cb + " === 'function' && " + cb + "("))
}
c.Writer.Write(exported)
c.Write(exported)
if willcb {
c.Writer.Write([]byte(");"))
c.Write([]byte(");"))
}
}
// b2s converts byte slice to a string without memory allocation.
// See https://groups.google.com/forum/#!msg/Golang-Nuts/ENgbUzYvCuU/90yGx7GUAgAJ .
//
// Note it may break if string and/or slice header will change
// in the future go versions.
func b2s(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}

View File

@@ -4,32 +4,32 @@ import (
"strconv"
"strings"
"zxq.co/ripple/rippleapi/common"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"github.com/thehowl/go-osuapi"
"github.com/valyala/fasthttp"
"zxq.co/ripple/rippleapi/common"
)
// GetBeatmap retrieves general beatmap information.
func GetBeatmap(c *gin.Context, db *sqlx.DB) {
func GetBeatmap(c *fasthttp.RequestCtx, db *sqlx.DB) {
var whereClauses []string
var params []interface{}
limit := strconv.Itoa(common.InString(1, c.Query("limit"), 500, 500))
limit := strconv.Itoa(common.InString(1, query(c, "limit"), 500, 500))
// since value is not stored, silently ignore
if c.Query("s") != "" {
if query(c, "s") != "" {
whereClauses = append(whereClauses, "beatmaps.beatmapset_id = ?")
params = append(params, c.Query("s"))
params = append(params, query(c, "s"))
}
if c.Query("b") != "" {
if query(c, "b") != "" {
whereClauses = append(whereClauses, "beatmaps.beatmap_id = ?")
params = append(params, c.Query("b"))
params = append(params, query(c, "b"))
// b is unique, so change limit to 1
limit = "1"
}
// creator is not stored, silently ignore u and type
if c.Query("m") != "" {
m := genmode(c.Query("m"))
if query(c, "m") != "" {
m := genmode(query(c, "m"))
if m == "std" {
// Since STD beatmaps are converted, all of the diffs must be != 0
for _, i := range modes {
@@ -37,14 +37,14 @@ func GetBeatmap(c *gin.Context, db *sqlx.DB) {
}
} else {
whereClauses = append(whereClauses, "beatmaps.difficulty_"+m+" != 0")
if c.Query("a") == "1" {
if query(c, "a") == "1" {
whereClauses = append(whereClauses, "beatmaps.difficulty_std = 0")
}
}
}
if c.Query("h") != "" {
if query(c, "h") != "" {
whereClauses = append(whereClauses, "beatmaps.beatmap_md5 = ?")
params = append(params, c.Query("h"))
params = append(params, query(c, "h"))
}
where := strings.Join(whereClauses, " AND ")
@@ -61,8 +61,8 @@ func GetBeatmap(c *gin.Context, db *sqlx.DB) {
FROM beatmaps `+where+" ORDER BY id DESC LIMIT "+limit,
params...)
if err != nil {
c.Error(err)
c.JSON(200, defaultResponse)
common.Err(c, err)
json(c, 200, defaultResponse)
return
}
@@ -82,7 +82,7 @@ FROM beatmaps `+where+" ORDER BY id DESC LIMIT "+limit,
&rawLastUpdate,
)
if err != nil {
c.Error(err)
common.Err(c, err)
continue
}
bm.TotalLength = bm.HitLength
@@ -103,7 +103,7 @@ FROM beatmaps `+where+" ORDER BY id DESC LIMIT "+limit,
bms = append(bms, bm)
}
c.JSON(200, bms)
json(c, 200, bms)
}
var rippleToOsuRankedStatus = map[int]osuapi.ApprovedStatus{

View File

@@ -2,12 +2,12 @@ package peppy
import (
"database/sql"
_json "encoding/json"
"strconv"
"zxq.co/ripple/rippleapi/common"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"github.com/valyala/fasthttp"
"zxq.co/ripple/rippleapi/common"
)
var modes = []string{"std", "taiko", "ctb", "mania"}
@@ -30,25 +30,25 @@ func rankable(m string) bool {
return x == 0 || x == 3
}
func genUser(c *gin.Context, db *sqlx.DB) (string, string) {
func genUser(c *fasthttp.RequestCtx, db *sqlx.DB) (string, string) {
var whereClause string
var p string
// used in second case of switch
s, err := strconv.Atoi(c.Query("u"))
s, err := strconv.Atoi(query(c, "u"))
switch {
// We know for sure that it's an username.
case c.Query("type") == "string":
case query(c, "type") == "string":
whereClause = "users.username_safe = ?"
p = common.SafeUsername(c.Query("u"))
p = common.SafeUsername(query(c, "u"))
// It could be an user ID, so we look for an user with that username first.
case err == nil:
err = db.QueryRow("SELECT id FROM users WHERE id = ? LIMIT 1", s).Scan(&p)
if err == sql.ErrNoRows {
// If no user with that userID were found, assume username.
whereClause = "users.username_safe = ?"
p = common.SafeUsername(c.Query("u"))
p = common.SafeUsername(query(c, "u"))
} else {
// An user with that userID was found. Thus it's an userID.
whereClause = "users.id = ?"
@@ -56,7 +56,20 @@ func genUser(c *gin.Context, db *sqlx.DB) (string, string) {
// u contains letters, so it's an username.
default:
whereClause = "users.username_safe = ?"
p = common.SafeUsername(c.Query("u"))
p = common.SafeUsername(query(c, "u"))
}
return whereClause, p
}
func query(c *fasthttp.RequestCtx, s string) string {
return string(c.QueryArgs().Peek(s))
}
func json(c *fasthttp.RequestCtx, code int, data interface{}) {
c.SetStatusCode(code)
d, err := _json.Marshal(data)
if err != nil {
panic(err)
}
c.Write(d)
}

View File

@@ -2,11 +2,11 @@
package peppy
import (
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"github.com/valyala/fasthttp"
)
// GetMatch retrieves general match information.
func GetMatch(c *gin.Context, db *sqlx.DB) {
c.JSON(200, defaultResponse)
func GetMatch(c *fasthttp.RequestCtx, db *sqlx.DB) {
json(c, 200, defaultResponse)
}

View File

@@ -7,43 +7,43 @@ import (
"zxq.co/ripple/rippleapi/common"
"zxq.co/x/getrank"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"github.com/valyala/fasthttp"
"gopkg.in/thehowl/go-osuapi.v1"
"zxq.co/x/getrank"
)
// GetScores retrieve information about the top 100 scores of a specified beatmap.
func GetScores(c *gin.Context, db *sqlx.DB) {
if c.Query("b") == "" {
c.JSON(200, defaultResponse)
func GetScores(c *fasthttp.RequestCtx, db *sqlx.DB) {
if query(c, "b") == "" {
json(c, 200, defaultResponse)
return
}
var beatmapMD5 string
err := db.Get(&beatmapMD5, "SELECT beatmap_md5 FROM beatmaps WHERE beatmap_id = ? LIMIT 1", c.Query("b"))
err := db.Get(&beatmapMD5, "SELECT beatmap_md5 FROM beatmaps WHERE beatmap_id = ? LIMIT 1", query(c, "b"))
switch {
case err == sql.ErrNoRows:
c.JSON(200, defaultResponse)
json(c, 200, defaultResponse)
return
case err != nil:
c.Error(err)
c.JSON(200, defaultResponse)
common.Err(c, err)
json(c, 200, defaultResponse)
return
}
var sb = "scores.score"
if rankable(c.Query("m")) {
if rankable(query(c, "m")) {
sb = "scores.pp"
}
var (
extraWhere string
extraParams []interface{}
)
if c.Query("u") != "" {
if query(c, "u") != "" {
w, p := genUser(c, db)
extraWhere = "AND " + w
extraParams = append(extraParams, p)
}
mods := common.Int(c.Query("mods"))
mods := common.Int(query(c, "mods"))
rows, err := db.Query(`
SELECT
scores.id, scores.score, users.username, scores.300_count, scores.100_count,
@@ -58,11 +58,11 @@ WHERE scores.completed = '3'
AND scores.play_mode = ?
AND scores.mods & ? = ?
`+extraWhere+`
ORDER BY `+sb+` DESC LIMIT `+strconv.Itoa(common.InString(1, c.Query("limit"), 100, 50)),
append([]interface{}{beatmapMD5, genmodei(c.Query("m")), mods, mods}, extraParams...)...)
ORDER BY `+sb+` DESC LIMIT `+strconv.Itoa(common.InString(1, query(c, "limit"), 100, 50)),
append([]interface{}{beatmapMD5, genmodei(query(c, "m")), mods, mods}, extraParams...)...)
if err != nil {
c.Error(err)
c.JSON(200, defaultResponse)
common.Err(c, err)
json(c, 200, defaultResponse)
return
}
var results []osuapi.GSScore
@@ -82,17 +82,17 @@ ORDER BY `+sb+` DESC LIMIT `+strconv.Itoa(common.InString(1, c.Query("limit"), 1
)
if err != nil {
if err != sql.ErrNoRows {
c.Error(err)
common.Err(c, err)
}
continue
}
s.FullCombo = osuapi.OsuBool(fullcombo)
s.Mods = osuapi.Mods(mods)
s.Date = osuapi.MySQLDate(date)
s.Rank = strings.ToUpper(getrank.GetRank(osuapi.Mode(genmodei(c.Query("m"))), s.Mods,
s.Rank = strings.ToUpper(getrank.GetRank(osuapi.Mode(genmodei(query(c, "m"))), s.Mods,
accuracy, s.Count300, s.Count100, s.Count50, s.CountMiss))
results = append(results, s)
}
c.JSON(200, results)
json(c, 200, results)
return
}

View File

@@ -5,23 +5,24 @@ import (
"database/sql"
"fmt"
"zxq.co/ripple/ocl"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"github.com/thehowl/go-osuapi"
"github.com/valyala/fasthttp"
"zxq.co/ripple/ocl"
"zxq.co/ripple/rippleapi/common"
)
// GetUser retrieves general user information.
func GetUser(c *gin.Context, db *sqlx.DB) {
if c.Query("u") == "" {
c.JSON(200, defaultResponse)
func GetUser(c *fasthttp.RequestCtx, db *sqlx.DB) {
if query(c, "u") == "" {
json(c, 200, defaultResponse)
return
}
var user osuapi.User
whereClause, p := genUser(c, db)
whereClause = "WHERE " + whereClause
mode := genmode(c.Query("m"))
mode := genmode(query(c, "m"))
var lbpos *int
err := db.QueryRow(fmt.Sprintf(
@@ -43,9 +44,9 @@ func GetUser(c *gin.Context, db *sqlx.DB) {
&user.Country,
)
if err != nil {
c.JSON(200, defaultResponse)
json(c, 200, defaultResponse)
if err != sql.ErrNoRows {
c.Error(err)
common.Err(c, err)
}
return
}
@@ -54,5 +55,5 @@ func GetUser(c *gin.Context, db *sqlx.DB) {
}
user.Level = ocl.GetLevelPrecise(user.TotalScore)
c.JSON(200, []osuapi.User{user})
json(c, 200, []osuapi.User{user})
}

View File

@@ -4,32 +4,32 @@ import (
"fmt"
"strings"
"github.com/jmoiron/sqlx"
"github.com/valyala/fasthttp"
"gopkg.in/thehowl/go-osuapi.v1"
"zxq.co/ripple/rippleapi/common"
"zxq.co/x/getrank"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"gopkg.in/thehowl/go-osuapi.v1"
)
// GetUserRecent retrieves an user's recent scores.
func GetUserRecent(c *gin.Context, db *sqlx.DB) {
getUserX(c, db, "ORDER BY scores.time DESC", common.InString(1, c.Query("limit"), 50, 10))
func GetUserRecent(c *fasthttp.RequestCtx, db *sqlx.DB) {
getUserX(c, db, "ORDER BY scores.time DESC", common.InString(1, query(c, "limit"), 50, 10))
}
// GetUserBest retrieves an user's best scores.
func GetUserBest(c *gin.Context, db *sqlx.DB) {
func GetUserBest(c *fasthttp.RequestCtx, db *sqlx.DB) {
var sb string
if rankable(c.Query("m")) {
if rankable(query(c, "m")) {
sb = "scores.pp"
} else {
sb = "scores.score"
}
getUserX(c, db, "AND completed = '3' ORDER BY "+sb+" DESC", common.InString(1, c.Query("limit"), 100, 10))
getUserX(c, db, "AND completed = '3' ORDER BY "+sb+" DESC", common.InString(1, query(c, "limit"), 100, 10))
}
func getUserX(c *gin.Context, db *sqlx.DB, orderBy string, limit int) {
func getUserX(c *fasthttp.RequestCtx, db *sqlx.DB, orderBy string, limit int) {
whereClause, p := genUser(c, db)
query := fmt.Sprintf(
sqlQuery := fmt.Sprintf(
`SELECT
beatmaps.beatmap_id, scores.score, scores.max_combo,
scores.300_count, scores.100_count, scores.50_count,
@@ -44,11 +44,11 @@ func getUserX(c *gin.Context, db *sqlx.DB, orderBy string, limit int) {
LIMIT %d`, whereClause, orderBy, limit,
)
scores := make([]osuapi.GUSScore, 0, limit)
m := genmodei(c.Query("m"))
rows, err := db.Query(query, p, m)
m := genmodei(query(c, "m"))
rows, err := db.Query(sqlQuery, p, m)
if err != nil {
c.JSON(200, defaultResponse)
c.Error(err)
json(c, 200, defaultResponse)
common.Err(c, err)
return
}
for rows.Next() {
@@ -68,8 +68,8 @@ func getUserX(c *gin.Context, db *sqlx.DB, orderBy string, limit int) {
&curscore.PP, &acc,
)
if err != nil {
c.JSON(200, defaultResponse)
c.Error(err)
json(c, 200, defaultResponse)
common.Err(c, err)
return
}
if bid == nil {
@@ -91,5 +91,5 @@ func getUserX(c *gin.Context, db *sqlx.DB, orderBy string, limit int) {
))
scores = append(scores, curscore)
}
c.JSON(200, scores)
json(c, 200, scores)
}

View File

@@ -1,16 +1,13 @@
package app
import (
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"github.com/valyala/fasthttp"
)
// PeppyMethod generates a method for the peppyapi
func PeppyMethod(a func(c *gin.Context, db *sqlx.DB)) gin.HandlerFunc {
return func(c *gin.Context) {
rateLimiter()
perUserRequestLimiter(0, c.ClientIP())
func PeppyMethod(a func(c *fasthttp.RequestCtx, db *sqlx.DB)) fasthttp.RequestHandler {
return func(c *fasthttp.RequestCtx) {
doggo.Incr("requests.peppy", nil, 1)
// I have no idea how, but I manged to accidentally string the first 4

View File

@@ -1,36 +0,0 @@
package app
import (
"strconv"
"time"
"zxq.co/ripple/rippleapi/limit"
)
const reqsPerSecond = 5000
const sleepTime = time.Second / reqsPerSecond
var limiter = make(chan struct{}, reqsPerSecond)
func setUpLimiter() {
for i := 0; i < reqsPerSecond; i++ {
limiter <- struct{}{}
}
go func() {
for {
limiter <- struct{}{}
time.Sleep(sleepTime)
}
}()
}
func rateLimiter() {
<-limiter
}
func perUserRequestLimiter(uid int, ip string) {
if uid == 0 {
limit.Request("ip:"+ip, 200)
} else {
limit.Request("user:"+strconv.Itoa(uid), 3000)
}
}

View File

@@ -4,9 +4,8 @@ import (
"fmt"
"github.com/DataDog/datadog-go/statsd"
fhr "github.com/buaazp/fasthttprouter"
"github.com/getsentry/raven-go"
"github.com/gin-gonic/contrib/gzip"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"github.com/serenize/snaker"
"gopkg.in/redis.v5"
@@ -29,7 +28,7 @@ var commonClusterfucks = map[string]string{
}
// Start begins taking HTTP connections.
func Start(conf common.Conf, dbO *sqlx.DB) *gin.Engine {
func Start(conf common.Conf, dbO *sqlx.DB) *fhr.Router {
db = dbO
cf = conf
@@ -40,10 +39,10 @@ func Start(conf common.Conf, dbO *sqlx.DB) *gin.Engine {
return snaker.CamelToSnake(s)
})
setUpLimiter()
r := gin.Default()
r.Use(gzip.Gzip(gzip.DefaultCompression))
r := fhr.New()
// TODO: add back gzip
// TODO: add logging
// TODO: add sentry panic recovering
// sentry
if conf.SentryDSN != "" {
@@ -52,7 +51,8 @@ func Start(conf common.Conf, dbO *sqlx.DB) *gin.Engine {
if err != nil {
fmt.Println(err)
} else {
r.Use(Recovery(ravenClient, false))
// r.Use(Recovery(ravenClient, false))
common.RavenClient = ravenClient
}
}
@@ -63,9 +63,9 @@ func Start(conf common.Conf, dbO *sqlx.DB) *gin.Engine {
fmt.Println(err)
}
doggo.Namespace = "api."
r.Use(func(c *gin.Context) {
doggo.Incr("requests", nil, 1)
})
// r.Use(func(c *gin.Context) {
// doggo.Incr("requests", nil, 1)
// })
// redis
red = redis.NewClient(&redis.Options{
@@ -77,94 +77,93 @@ func Start(conf common.Conf, dbO *sqlx.DB) *gin.Engine {
// token updater
go tokenUpdater(db)
api := r.Group("/api")
// peppyapi
{
p := api.Group("/")
{
p.GET("/get_user", PeppyMethod(peppy.GetUser))
p.GET("/get_match", PeppyMethod(peppy.GetMatch))
p.GET("/get_user_recent", PeppyMethod(peppy.GetUserRecent))
p.GET("/get_user_best", PeppyMethod(peppy.GetUserBest))
p.GET("/get_scores", PeppyMethod(peppy.GetScores))
p.GET("/get_beatmaps", PeppyMethod(peppy.GetBeatmap))
}
gv1 := api.Group("/v1")
{
gv1.POST("/tokens", Method(v1.TokenNewPOST))
gv1.POST("/tokens/new", Method(v1.TokenNewPOST))
gv1.POST("/tokens/self/delete", Method(v1.TokenSelfDeletePOST))
// Auth-free API endpoints (public data)
gv1.GET("/ping", Method(v1.PingGET))
gv1.GET("/surprise_me", Method(v1.SurpriseMeGET))
gv1.GET("/doc", Method(v1.DocGET))
gv1.GET("/doc/content", Method(v1.DocContentGET))
gv1.GET("/doc/rules", Method(v1.DocRulesGET))
gv1.GET("/users", Method(v1.UsersGET))
gv1.GET("/users/whatid", Method(v1.UserWhatsTheIDGET))
gv1.GET("/users/full", Method(v1.UserFullGET))
gv1.GET("/users/userpage", Method(v1.UserUserpageGET))
gv1.GET("/users/lookup", Method(v1.UserLookupGET))
gv1.GET("/users/scores/best", Method(v1.UserScoresBestGET))
gv1.GET("/users/scores/recent", Method(v1.UserScoresRecentGET))
gv1.GET("/badges", Method(v1.BadgesGET))
gv1.GET("/beatmaps", Method(v1.BeatmapGET))
gv1.GET("/leaderboard", Method(v1.LeaderboardGET))
gv1.GET("/tokens", Method(v1.TokenGET))
gv1.GET("/users/self", Method(v1.UserSelfGET))
gv1.GET("/tokens/self", Method(v1.TokenSelfGET))
gv1.GET("/blog/posts", Method(v1.BlogPostsGET))
gv1.GET("/scores", Method(v1.ScoresGET))
gv1.GET("/beatmaps/rank_requests/status", Method(v1.BeatmapRankRequestsStatusGET))
// ReadConfidential privilege required
gv1.GET("/friends", Method(v1.FriendsGET, common.PrivilegeReadConfidential))
gv1.GET("/friends/with", Method(v1.FriendsWithGET, common.PrivilegeReadConfidential))
gv1.GET("/users/self/donor_info", Method(v1.UsersSelfDonorInfoGET, common.PrivilegeReadConfidential))
gv1.GET("/users/self/favourite_mode", Method(v1.UsersSelfFavouriteModeGET, common.PrivilegeReadConfidential))
gv1.GET("/users/self/settings", Method(v1.UsersSelfSettingsGET, common.PrivilegeReadConfidential))
// Write privilege required
gv1.POST("/friends/add", Method(v1.FriendsAddPOST, common.PrivilegeWrite))
gv1.POST("/friends/del", Method(v1.FriendsDelPOST, common.PrivilegeWrite))
gv1.POST("/users/self/settings", Method(v1.UsersSelfSettingsPOST, common.PrivilegeWrite))
gv1.POST("/users/self/userpage", Method(v1.UserSelfUserpagePOST, common.PrivilegeWrite))
gv1.POST("/beatmaps/rank_requests", Method(v1.BeatmapRankRequestsSubmitPOST, common.PrivilegeWrite))
// Admin: beatmap
gv1.POST("/beatmaps/set_status", Method(v1.BeatmapSetStatusPOST, common.PrivilegeBeatmap))
gv1.GET("/beatmaps/ranked_frozen_full", Method(v1.BeatmapRankedFrozenFullGET, common.PrivilegeBeatmap))
// Admin: user managing
gv1.POST("/users/manage/set_allowed", Method(v1.UserManageSetAllowedPOST, common.PrivilegeManageUser))
// M E T A
// E T "wow thats so meta"
// T E -- the one who said "wow thats so meta"
// A T E M
gv1.GET("/meta/restart", Method(v1.MetaRestartGET, common.PrivilegeAPIMeta))
gv1.GET("/meta/kill", Method(v1.MetaKillGET, common.PrivilegeAPIMeta))
gv1.GET("/meta/up_since", Method(v1.MetaUpSinceGET, common.PrivilegeAPIMeta))
gv1.GET("/meta/update", Method(v1.MetaUpdateGET, common.PrivilegeAPIMeta))
// User Managing + meta
gv1.POST("/tokens/fix_privileges", Method(v1.TokenFixPrivilegesPOST,
common.PrivilegeManageUser, common.PrivilegeAPIMeta))
// in the new osu-web, the old endpoints are also in /v1 it seems. So /shrug
gv1.GET("/get_user", PeppyMethod(peppy.GetUser))
gv1.GET("/get_match", PeppyMethod(peppy.GetMatch))
gv1.GET("/get_user_recent", PeppyMethod(peppy.GetUserRecent))
gv1.GET("/get_user_best", PeppyMethod(peppy.GetUserBest))
gv1.GET("/get_scores", PeppyMethod(peppy.GetScores))
gv1.GET("/get_beatmaps", PeppyMethod(peppy.GetBeatmap))
}
api.GET("/status", internals.Status)
r.GET("/api/get_user", PeppyMethod(peppy.GetUser))
r.GET("/api/get_match", PeppyMethod(peppy.GetMatch))
r.GET("/api/get_user_recent", PeppyMethod(peppy.GetUserRecent))
r.GET("/api/get_user_best", PeppyMethod(peppy.GetUserBest))
r.GET("/api/get_scores", PeppyMethod(peppy.GetScores))
r.GET("/api/get_beatmaps", PeppyMethod(peppy.GetBeatmap))
}
r.NoRoute(v1.Handle404)
// v1 API
{
r.POST("/api/v1/tokens", Method(v1.TokenNewPOST))
r.POST("/api/v1/tokens/new", Method(v1.TokenNewPOST))
r.POST("/api/v1/tokens/self/delete", Method(v1.TokenSelfDeletePOST))
// Auth-free API endpoints (public data)
r.GET("/api/v1/ping", Method(v1.PingGET))
r.GET("/api/v1/surprise_me", Method(v1.SurpriseMeGET))
r.GET("/api/v1/doc", Method(v1.DocGET))
r.GET("/api/v1/doc/content", Method(v1.DocContentGET))
r.GET("/api/v1/doc/rules", Method(v1.DocRulesGET))
r.GET("/api/v1/users", Method(v1.UsersGET))
r.GET("/api/v1/users/whatid", Method(v1.UserWhatsTheIDGET))
r.GET("/api/v1/users/full", Method(v1.UserFullGET))
r.GET("/api/v1/users/userpage", Method(v1.UserUserpageGET))
r.GET("/api/v1/users/lookup", Method(v1.UserLookupGET))
r.GET("/api/v1/users/scores/best", Method(v1.UserScoresBestGET))
r.GET("/api/v1/users/scores/recent", Method(v1.UserScoresRecentGET))
r.GET("/api/v1/badges", Method(v1.BadgesGET))
r.GET("/api/v1/beatmaps", Method(v1.BeatmapGET))
r.GET("/api/v1/leaderboard", Method(v1.LeaderboardGET))
r.GET("/api/v1/tokens", Method(v1.TokenGET))
r.GET("/api/v1/users/self", Method(v1.UserSelfGET))
r.GET("/api/v1/tokens/self", Method(v1.TokenSelfGET))
r.GET("/api/v1/blog/posts", Method(v1.BlogPostsGET))
r.GET("/api/v1/scores", Method(v1.ScoresGET))
r.GET("/api/v1/beatmaps/rank_requests/status", Method(v1.BeatmapRankRequestsStatusGET))
// ReadConfidential privilege required
r.GET("/api/v1/friends", Method(v1.FriendsGET, common.PrivilegeReadConfidential))
r.GET("/api/v1/friends/with", Method(v1.FriendsWithGET, common.PrivilegeReadConfidential))
r.GET("/api/v1/users/self/donor_info", Method(v1.UsersSelfDonorInfoGET, common.PrivilegeReadConfidential))
r.GET("/api/v1/users/self/favourite_mode", Method(v1.UsersSelfFavouriteModeGET, common.PrivilegeReadConfidential))
r.GET("/api/v1/users/self/settings", Method(v1.UsersSelfSettingsGET, common.PrivilegeReadConfidential))
// Write privilege required
r.POST("/api/v1/friends/add", Method(v1.FriendsAddPOST, common.PrivilegeWrite))
r.POST("/api/v1/friends/del", Method(v1.FriendsDelPOST, common.PrivilegeWrite))
r.POST("/api/v1/users/self/settings", Method(v1.UsersSelfSettingsPOST, common.PrivilegeWrite))
r.POST("/api/v1/users/self/userpage", Method(v1.UserSelfUserpagePOST, common.PrivilegeWrite))
r.POST("/api/v1/beatmaps/rank_requests", Method(v1.BeatmapRankRequestsSubmitPOST, common.PrivilegeWrite))
// Admin: beatmap
r.POST("/api/v1/beatmaps/set_status", Method(v1.BeatmapSetStatusPOST, common.PrivilegeBeatmap))
r.GET("/api/v1/beatmaps/ranked_frozen_full", Method(v1.BeatmapRankedFrozenFullGET, common.PrivilegeBeatmap))
// Admin: user managing
r.POST("/api/v1/users/manage/set_allowed", Method(v1.UserManageSetAllowedPOST, common.PrivilegeManageUser))
// M E T A
// E T "wow thats so meta"
// T E -- the one who said "wow thats so meta"
// A T E M
r.GET("/api/v1/meta/restart", Method(v1.MetaRestartGET, common.PrivilegeAPIMeta))
r.GET("/api/v1/meta/kill", Method(v1.MetaKillGET, common.PrivilegeAPIMeta))
r.GET("/api/v1/meta/up_since", Method(v1.MetaUpSinceGET, common.PrivilegeAPIMeta))
r.GET("/api/v1/meta/update", Method(v1.MetaUpdateGET, common.PrivilegeAPIMeta))
// User Managing + meta
r.POST("/api/v1/tokens/fix_privileges", Method(v1.TokenFixPrivilegesPOST,
common.PrivilegeManageUser, common.PrivilegeAPIMeta))
}
// in the new osu-web, the old endpoints are also in /v1 it seems. So /shrug
{
r.GET("/api/v1/get_user", PeppyMethod(peppy.GetUser))
r.GET("/api/v1/get_match", PeppyMethod(peppy.GetMatch))
r.GET("/api/v1/get_user_recent", PeppyMethod(peppy.GetUserRecent))
r.GET("/api/v1/get_user_best", PeppyMethod(peppy.GetUserBest))
r.GET("/api/v1/get_scores", PeppyMethod(peppy.GetScores))
r.GET("/api/v1/get_beatmaps", PeppyMethod(peppy.GetBeatmap))
}
r.GET("/api/status", internals.Status)
r.NotFound = v1.Handle404
return r
}

View File

@@ -1,8 +1,10 @@
package v1
import (
"encoding/json"
"github.com/valyala/fasthttp"
"zxq.co/ripple/rippleapi/common"
"github.com/gin-gonic/gin"
)
type response404 struct {
@@ -11,12 +13,17 @@ type response404 struct {
}
// Handle404 handles requests with no implemented handlers.
func Handle404(c *gin.Context) {
c.Header("X-Real-404", "yes")
c.IndentedJSON(404, response404{
func Handle404(c *fasthttp.RequestCtx) {
c.Response.Header.Add("X-Real-404", "yes")
data, err := json.MarshalIndent(response404{
ResponseBase: common.ResponseBase{
Code: 404,
},
Cats: surpriseMe(),
})
}, "", "\t")
if err != nil {
panic(err)
}
c.SetStatusCode(404)
c.Write(data)
}

View File

@@ -2,8 +2,6 @@ package v1
import (
"database/sql"
"fmt"
"net/url"
"zxq.co/ripple/rippleapi/common"
)
@@ -51,10 +49,10 @@ type beatmapSetStatusData struct {
// the beatmap ranked status is frozen. Or freezed. Freezed best meme 2k16
func BeatmapSetStatusPOST(md common.MethodData) common.CodeMessager {
var req beatmapSetStatusData
md.RequestData.Unmarshal(&req)
md.Unmarshal(&req)
var miss []string
if req.BeatmapsetID == 0 && req.BeatmapID == 0 {
if req.BeatmapsetID <= 0 && req.BeatmapID <= 0 {
miss = append(miss, "beatmapset_id or beatmap_id")
}
if len(miss) != 0 {
@@ -84,32 +82,14 @@ func BeatmapSetStatusPOST(md common.MethodData) common.CodeMessager {
SET ranked = ?, ranked_status_freezed = ?
WHERE beatmapset_id = ?`, req.RankedStatus, req.Frozen, param)
var x = make(map[string]interface{}, 1)
if req.BeatmapID != 0 {
x["bb"] = req.BeatmapID
if req.BeatmapID > 0 {
md.Ctx.Request.URI().QueryArgs().SetUint("bb", req.BeatmapID)
} else {
x["s"] = req.BeatmapsetID
md.Ctx.Request.URI().QueryArgs().SetUint("s", req.BeatmapsetID)
}
md.C.Request.URL = genURL(x)
return getMultipleBeatmaps(md)
}
func genURL(d map[string]interface{}) *url.URL {
var s string
for k, v := range d {
if s != "" {
s += "&"
}
s += k + "=" + url.QueryEscape(fmt.Sprintf("%v", v))
}
u := new(url.URL)
if len(d) == 0 {
return u
}
u.RawQuery = s
return u
}
// BeatmapGET retrieves a beatmap.
func BeatmapGET(md common.MethodData) common.CodeMessager {
beatmapID := common.Int(md.Query("b"))

View File

@@ -78,7 +78,7 @@ type submitRequestData struct {
// BeatmapRankRequestsSubmitPOST submits a new beatmap for ranking approval.
func BeatmapRankRequestsSubmitPOST(md common.MethodData) common.CodeMessager {
var d submitRequestData
err := md.RequestData.Unmarshal(&d)
err := md.Unmarshal(&d)
if err != nil {
return ErrBadJSON
}
@@ -91,9 +91,6 @@ func BeatmapRankRequestsSubmitPOST(md common.MethodData) common.CodeMessager {
if !limit.NonBlockingRequest("rankrequest:u:"+strconv.Itoa(md.ID()), 5) {
return common.SimpleResponse(429, "You may only try to request 5 beatmaps per minute.")
}
if !limit.NonBlockingRequest("rankrequest:ip:"+md.C.ClientIP(), 8) {
return common.SimpleResponse(429, "You may only try to request 8 beatmaps per minute from the same IP.")
}
// find out from BeatmapRankRequestsStatusGET if we can submit beatmaps.
statusRaw := BeatmapRankRequestsStatusGET(md)

View File

@@ -134,7 +134,7 @@ func FriendsAddPOST(md common.MethodData) common.CodeMessager {
var u struct {
User int `json:"user"`
}
md.RequestData.Unmarshal(&u)
md.Unmarshal(&u)
return addFriend(md, u.User)
}
@@ -183,7 +183,7 @@ func FriendsDelPOST(md common.MethodData) common.CodeMessager {
var u struct {
User int `json:"user"`
}
md.RequestData.Unmarshal(&u)
md.Unmarshal(&u)
return delFriend(md, u.User)
}

View File

@@ -14,7 +14,7 @@ type setAllowedData struct {
// UserManageSetAllowedPOST allows to set the allowed status of an user.
func UserManageSetAllowedPOST(md common.MethodData) common.CodeMessager {
data := setAllowedData{}
if err := md.RequestData.Unmarshal(&data); err != nil {
if err := md.Unmarshal(&data); err != nil {
return ErrBadJSON
}
if data.Allowed < 0 || data.Allowed > 2 {

View File

@@ -62,7 +62,7 @@ type userSettingsData struct {
// UsersSelfSettingsPOST allows to modify information about the current user.
func UsersSelfSettingsPOST(md common.MethodData) common.CodeMessager {
var d userSettingsData
md.RequestData.Unmarshal(&d)
md.Unmarshal(&d)
// input sanitisation
*d.UsernameAKA = common.SanitiseString(*d.UsernameAKA)

View File

@@ -8,10 +8,10 @@ import (
"github.com/jmoiron/sqlx"
"golang.org/x/crypto/bcrypt"
"zxq.co/ripple/rippleapi/common"
"zxq.co/ripple/rippleapi/limit"
"zxq.co/ripple/schiavolib"
"golang.org/x/crypto/bcrypt"
)
type tokenNewInData struct {
@@ -37,7 +37,7 @@ type tokenNewResponse struct {
func TokenNewPOST(md common.MethodData) common.CodeMessager {
var r tokenNewResponse
data := tokenNewInData{}
err := md.RequestData.Unmarshal(&data)
err := md.Unmarshal(&data)
if err != nil {
return ErrBadJSON
}
@@ -80,7 +80,7 @@ func TokenNewPOST(md common.MethodData) common.CodeMessager {
}
privileges := common.UserPrivileges(privilegesRaw)
if !limit.NonBlockingRequest(fmt.Sprintf("loginattempt:%d:%s", r.ID, md.C.ClientIP()), 5) {
if !limit.NonBlockingRequest(fmt.Sprintf("loginattempt:%d:%s", r.ID, md.ClientIP()), 5) {
return common.SimpleResponse(429, "You've made too many login attempts. Try again later.")
}

View File

@@ -5,9 +5,9 @@ import (
"database/sql"
"strconv"
"strings"
"unicode"
"github.com/jmoiron/sqlx"
"zxq.co/ripple/ocl"
"zxq.co/ripple/rippleapi/common"
)
@@ -71,8 +71,7 @@ type userPutsMultiUserData struct {
}
func userPutsMulti(md common.MethodData) common.CodeMessager {
q := md.C.Request.URL.Query()
pm := md.Ctx.Request.URI().QueryArgs().PeekMulti
// query composition
wh := common.
Where("users.username_safe = ?", common.SafeUsername(md.Query("nname"))).
@@ -83,10 +82,10 @@ func userPutsMulti(md common.MethodData) common.CodeMessager {
Where("users_stats.country = ?", md.Query("country")).
Where("users_stats.username_aka = ?", md.Query("name_aka")).
Where("privileges_groups.name = ?", md.Query("privilege_group")).
In("users.id", q["ids"]...).
In("users.username_safe", safeUsernameBulk(q["names"])...).
In("users_stats.username_aka", q["names_aka"]...).
In("users_stats.country", q["countries"]...)
In("users.id", pm("ids")...).
In("users.username_safe", safeUsernameBulk(pm("names"))...).
In("users_stats.username_aka", pm("names_aka")...).
In("users_stats.country", pm("countries")...)
var extraJoin string
if md.Query("privilege_group") != "" {
@@ -130,13 +129,19 @@ func userPutsMulti(md common.MethodData) common.CodeMessager {
// UserSelfGET is a shortcut for /users/id/self. (/users/self)
func UserSelfGET(md common.MethodData) common.CodeMessager {
md.C.Request.URL.RawQuery = "id=self&" + md.C.Request.URL.RawQuery
md.Ctx.Request.URI().SetQueryString("id=self")
return UsersGET(md)
}
func safeUsernameBulk(us []string) []string {
for i, u := range us {
us[i] = common.SafeUsername(u)
func safeUsernameBulk(us [][]byte) [][]byte {
for _, u := range us {
for idx, v := range u {
if v == ' ' {
u[idx] = '_'
continue
}
u[idx] = byte(unicode.ToLower(rune(v)))
}
}
return us
}
@@ -341,7 +346,7 @@ func UserSelfUserpagePOST(md common.MethodData) common.CodeMessager {
var d struct {
Data *string `json:"data"`
}
md.RequestData.Unmarshal(&d)
md.Unmarshal(&d)
if d.Data == nil {
return ErrMissingField("data")
}
@@ -350,7 +355,7 @@ func UserSelfUserpagePOST(md common.MethodData) common.CodeMessager {
if err != nil {
md.Err(err)
}
md.C.Request.URL.RawQuery += "&id=" + strconv.Itoa(md.ID())
md.Ctx.URI().SetQueryString("id=self")
return UserUserpageGET(md)
}