mirror of
https://github.com/weaveworks/scope.git
synced 2026-08-18 03:46:45 +00:00
Add vendored dependancies
This commit is contained in:
+13
@@ -0,0 +1,13 @@
|
||||
include $(GOROOT)/src/Make.inc
|
||||
|
||||
TARG=bitbucket.org/ww/goautoneg
|
||||
GOFILES=autoneg.go
|
||||
|
||||
include $(GOROOT)/src/Make.pkg
|
||||
|
||||
format:
|
||||
gofmt -w *.go
|
||||
|
||||
docs:
|
||||
gomake clean
|
||||
godoc ${TARG} > README.txt
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
PACKAGE
|
||||
|
||||
package goautoneg
|
||||
import "bitbucket.org/ww/goautoneg"
|
||||
|
||||
HTTP Content-Type Autonegotiation.
|
||||
|
||||
The functions in this package implement the behaviour specified in
|
||||
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
|
||||
|
||||
Copyright (c) 2011, Open Knowledge Foundation Ltd.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
Neither the name of the Open Knowledge Foundation Ltd. nor the
|
||||
names of its contributors may be used to endorse or promote
|
||||
products derived from this software without specific prior written
|
||||
permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
FUNCTIONS
|
||||
|
||||
func Negotiate(header string, alternatives []string) (content_type string)
|
||||
Negotiate the most appropriate content_type given the accept header
|
||||
and a list of alternatives.
|
||||
|
||||
func ParseAccept(header string) (accept []Accept)
|
||||
Parse an Accept Header string returning a sorted list
|
||||
of clauses
|
||||
|
||||
|
||||
TYPES
|
||||
|
||||
type Accept struct {
|
||||
Type, SubType string
|
||||
Q float32
|
||||
Params map[string]string
|
||||
}
|
||||
Structure to represent a clause in an HTTP Accept Header
|
||||
|
||||
|
||||
SUBDIRECTORIES
|
||||
|
||||
.hg
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
/*
|
||||
HTTP Content-Type Autonegotiation.
|
||||
|
||||
The functions in this package implement the behaviour specified in
|
||||
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
|
||||
|
||||
Copyright (c) 2011, Open Knowledge Foundation Ltd.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
Neither the name of the Open Knowledge Foundation Ltd. nor the
|
||||
names of its contributors may be used to endorse or promote
|
||||
products derived from this software without specific prior written
|
||||
permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
*/
|
||||
package goautoneg
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Structure to represent a clause in an HTTP Accept Header
|
||||
type Accept struct {
|
||||
Type, SubType string
|
||||
Q float64
|
||||
Params map[string]string
|
||||
}
|
||||
|
||||
// For internal use, so that we can use the sort interface
|
||||
type accept_slice []Accept
|
||||
|
||||
func (accept accept_slice) Len() int {
|
||||
slice := []Accept(accept)
|
||||
return len(slice)
|
||||
}
|
||||
|
||||
func (accept accept_slice) Less(i, j int) bool {
|
||||
slice := []Accept(accept)
|
||||
ai, aj := slice[i], slice[j]
|
||||
if ai.Q > aj.Q {
|
||||
return true
|
||||
}
|
||||
if ai.Type != "*" && aj.Type == "*" {
|
||||
return true
|
||||
}
|
||||
if ai.SubType != "*" && aj.SubType == "*" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (accept accept_slice) Swap(i, j int) {
|
||||
slice := []Accept(accept)
|
||||
slice[i], slice[j] = slice[j], slice[i]
|
||||
}
|
||||
|
||||
// Parse an Accept Header string returning a sorted list
|
||||
// of clauses
|
||||
func ParseAccept(header string) (accept []Accept) {
|
||||
parts := strings.Split(header, ",")
|
||||
accept = make([]Accept, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part := strings.Trim(part, " ")
|
||||
|
||||
a := Accept{}
|
||||
a.Params = make(map[string]string)
|
||||
a.Q = 1.0
|
||||
|
||||
mrp := strings.Split(part, ";")
|
||||
|
||||
media_range := mrp[0]
|
||||
sp := strings.Split(media_range, "/")
|
||||
a.Type = strings.Trim(sp[0], " ")
|
||||
|
||||
switch {
|
||||
case len(sp) == 1 && a.Type == "*":
|
||||
a.SubType = "*"
|
||||
case len(sp) == 2:
|
||||
a.SubType = strings.Trim(sp[1], " ")
|
||||
default:
|
||||
continue
|
||||
}
|
||||
|
||||
if len(mrp) == 1 {
|
||||
accept = append(accept, a)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, param := range mrp[1:] {
|
||||
sp := strings.SplitN(param, "=", 2)
|
||||
if len(sp) != 2 {
|
||||
continue
|
||||
}
|
||||
token := strings.Trim(sp[0], " ")
|
||||
if token == "q" {
|
||||
a.Q, _ = strconv.ParseFloat(sp[1], 32)
|
||||
} else {
|
||||
a.Params[token] = strings.Trim(sp[1], " ")
|
||||
}
|
||||
}
|
||||
|
||||
accept = append(accept, a)
|
||||
}
|
||||
|
||||
slice := accept_slice(accept)
|
||||
sort.Sort(slice)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Negotiate the most appropriate content_type given the accept header
|
||||
// and a list of alternatives.
|
||||
func Negotiate(header string, alternatives []string) (content_type string) {
|
||||
asp := make([][]string, 0, len(alternatives))
|
||||
for _, ctype := range alternatives {
|
||||
asp = append(asp, strings.SplitN(ctype, "/", 2))
|
||||
}
|
||||
for _, clause := range ParseAccept(header) {
|
||||
for i, ctsp := range asp {
|
||||
if clause.Type == ctsp[0] && clause.SubType == ctsp[1] {
|
||||
content_type = alternatives[i]
|
||||
return
|
||||
}
|
||||
if clause.Type == ctsp[0] && clause.SubType == "*" {
|
||||
content_type = alternatives[i]
|
||||
return
|
||||
}
|
||||
if clause.Type == "*" && clause.SubType == "*" {
|
||||
content_type = alternatives[i]
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package goautoneg
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var chrome = "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"
|
||||
|
||||
func TestParseAccept(t *testing.T) {
|
||||
alternatives := []string{"text/html", "image/png"}
|
||||
content_type := Negotiate(chrome, alternatives)
|
||||
if content_type != "image/png" {
|
||||
t.Errorf("got %s expected image/png", content_type)
|
||||
}
|
||||
|
||||
alternatives = []string{"text/html", "text/plain", "text/n3"}
|
||||
content_type = Negotiate(chrome, alternatives)
|
||||
if content_type != "text/html" {
|
||||
t.Errorf("got %s expected text/html", content_type)
|
||||
}
|
||||
|
||||
alternatives = []string{"text/n3", "text/plain"}
|
||||
content_type = Negotiate(chrome, alternatives)
|
||||
if content_type != "text/plain" {
|
||||
t.Errorf("got %s expected text/plain", content_type)
|
||||
}
|
||||
|
||||
alternatives = []string{"text/n3", "application/rdf+xml"}
|
||||
content_type = Negotiate(chrome, alternatives)
|
||||
if content_type != "text/n3" {
|
||||
t.Errorf("got %s expected text/n3", content_type)
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
Copyright (c) 2013, Martin Angers
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
# Ghost
|
||||
|
||||
Ghost is a web development library loosely inspired by node's [Connect library][connect]. It provides a number of simple, single-responsibility HTTP handlers that can be combined to build a full-featured web server, and a generic template engine integration interface.
|
||||
|
||||
It stays close to the metal, not abstracting Go's standard library away. As a matter of fact, any stdlib handler can be used with Ghost's handlers, they simply are `net/http.Handler`'s.
|
||||
|
||||
## Installation and documentation
|
||||
|
||||
`go get github.com/PuerkitoBio/ghost`
|
||||
|
||||
[API reference][godoc]
|
||||
|
||||
*Status* : Still under development, things will change.
|
||||
|
||||
## Example
|
||||
|
||||
See the /ghostest directory for a complete working example of a website built with Ghost. It shows all handlers and template support of Ghost.
|
||||
|
||||
## Handlers
|
||||
|
||||
Ghost offers the following handlers:
|
||||
|
||||
* BasicAuthHandler : basic authentication support.
|
||||
* ContextHandler : key-value map provider for the duration of the request.
|
||||
* FaviconHandler : simple and efficient favicon renderer.
|
||||
* GZIPHandler : gzip-compresser for the body of the response.
|
||||
* LogHandler : fully customizable request logger.
|
||||
* PanicHandler : panic-catching handler to control the error response.
|
||||
* SessionHandler : store-agnostic server-side session provider.
|
||||
* StaticHandler : convenience handler that wraps a call to `net/http.ServeFile`.
|
||||
|
||||
Two stores are provided for the session persistence, `MemoryStore`, an in-memory map that is not suited for production environment, and `RedisStore`, a more robust and scalable [redigo][]-based Redis store. Because of the generic `SessionStore` interface, custom stores can easily be created as needed.
|
||||
|
||||
The `handlers` package also offers the `ChainableHandler` interface, which supports combining HTTP handlers in a sequential fashion, and the `ChainHandlers()` function that creates a new handler from the sequential combination of any number of handlers.
|
||||
|
||||
As a convenience, all functions that take a `http.Handler` as argument also have a corresponding function with the `Func` suffix that take a `http.HandlerFunc` instead as argument. This saves the type-cast when a simple handler function is passed (for example, `SessionHandler()` and `SessionHandlerFunc()`).
|
||||
|
||||
### Handlers Design
|
||||
|
||||
The HTTP handlers such as Basic Auth and Context need to store some state information to provide their functionality. Instead of using variables and a mutex to control shared access, Ghost augments the `http.ResponseWriter` interface that is part of the Handler's `ServeHTTP()` function signature. Because this instance is unique for each request and is not shared, there is no locking involved to access the state information.
|
||||
|
||||
However, when combining such handlers, Ghost needs a way to move through the chain of augmented ResponseWriters. This is why these *augmented writers* need to implement the `WrapWriter` interface. A single method is required, `WrappedWriter() http.ResponseWriter`, which returns the wrapped ResponseWriter.
|
||||
|
||||
And to get back a specific augmented writer, the `GetResponseWriter()` function is provided. It takes a ResponseWriter and a predicate function as argument, and returns the requested specific writer using the *comma-ok* pattern. Example, for the session writer:
|
||||
|
||||
```Go
|
||||
func getSessionWriter(w http.ResponseWriter) (*sessResponseWriter, bool) {
|
||||
ss, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
|
||||
_, ok := tst.(*sessResponseWriter)
|
||||
return ok
|
||||
})
|
||||
if ok {
|
||||
return ss.(*sessResponseWriter), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
```
|
||||
|
||||
Ghost does not provide a muxer, there are already many great ones available, but I would recommend Go's native `http.ServeMux` or [pat][] because it has great features and plays well with Ghost's design. Gorilla's muxer is very popular, but since it depends on Gorilla's (mutex-based) context provider, this is redundant with Ghost's context.
|
||||
|
||||
## Templates
|
||||
|
||||
Ghost supports the following template engines:
|
||||
|
||||
* Go's native templates (needs work, at the moment does not work with nested templates)
|
||||
* [Amber][]
|
||||
|
||||
TODO : Go's mustache implementation.
|
||||
|
||||
### Templates Design
|
||||
|
||||
The template engines can be registered much in the same way as database drivers, just by importing for side effects (using `_ "import/path"`). The `init()` function of the template engine's package registers the template compiler with the correct file extension, and the engine can be used.
|
||||
|
||||
## License
|
||||
|
||||
The [BSD 3-Clause license][lic].
|
||||
|
||||
[connect]: https://github.com/senchalabs/connect
|
||||
[godoc]: http://godoc.org/github.com/PuerkitoBio/ghost
|
||||
[lic]: http://opensource.org/licenses/BSD-3-Clause
|
||||
[redigo]: https://github.com/garyburd/redigo
|
||||
[pat]: https://github.com/bmizerany/pat
|
||||
[amber]: https://github.com/eknkc/amber
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package ghost
|
||||
|
||||
import (
|
||||
"log"
|
||||
)
|
||||
|
||||
// Logging function, defaults to Go's native log.Printf function. The idea to use
|
||||
// this instead of a *log.Logger struct is that it can be set to any of log.{Printf,Fatalf, Panicf},
|
||||
// but also to more flexible userland loggers like SeeLog (https://github.com/cihub/seelog).
|
||||
// It could be set, for example, to SeeLog's Debugf function. Any function with the
|
||||
// signature func(fmt string, params ...interface{}).
|
||||
var LogFn = log.Printf
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
<html>
|
||||
<head>
|
||||
<title>Ghost Test</title>
|
||||
<link type="text/css" rel="stylesheet" href="/public/styles.css">
|
||||
<link type="text/css" rel="stylesheet" href="/public/bootstrap-combined.min.css">
|
||||
</head>
|
||||
<body>
|
||||
<h1>Welcome to Ghost Test</h1>
|
||||
<img src="/public/logo.png" alt="peace" />
|
||||
<ol>
|
||||
<li><a href="/session">Session</a></li>
|
||||
<li><a href="/session/auth">Authenticated Session</a></li>
|
||||
<li><a href="/context">Chained Context</a></li>
|
||||
<li><a href="/panic">Panic</a></li>
|
||||
<li><a href="/public/styles.css">Styles.css</a></li>
|
||||
<li><a href="/public/jquery-2.0.0.min.js">JQuery</a></li>
|
||||
<li><a href="/public/logo.png">Logo</a></li>
|
||||
</ol>
|
||||
|
||||
<script src="/public/jquery-2.0.0.min.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
// Ghostest is an interactive end-to-end Web site application to test
|
||||
// the ghost packages. It serves the following URLs, with the specified
|
||||
// features (handlers):
|
||||
//
|
||||
// / : panic;log;gzip;static; -> serve file index.html
|
||||
// /public/styles.css : panic;log;gzip;StripPrefix;FileServer; -> serve directory public/
|
||||
// /public/script.js : panic;log;gzip;StripPrefix;FileServer; -> serve directory public/
|
||||
// /public/logo.pn : panic;log;gzip;StripPrefix;FileServer; -> serve directory public/
|
||||
// /session : panic;log;gzip;session;context;Custom; -> serve dynamic Go template
|
||||
// /session/auth : panic;log;gzip;session;context;basicAuth;Custom; -> serve dynamic template
|
||||
// /panic : panic;log;gzip;Custom; -> panics
|
||||
// /context : panic;log;gzip;context;Custom1;Custom2; -> serve dynamic Amber template
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/ghost/handlers"
|
||||
"github.com/PuerkitoBio/ghost/templates"
|
||||
_ "github.com/PuerkitoBio/ghost/templates/amber"
|
||||
_ "github.com/PuerkitoBio/ghost/templates/gotpl"
|
||||
"github.com/bmizerany/pat"
|
||||
)
|
||||
|
||||
const (
|
||||
sessionPageTitle = "Session Page"
|
||||
sessionPageAuthTitle = "Authenticated Session Page"
|
||||
sessionPageKey = "txt"
|
||||
contextPageKey = "time"
|
||||
sessionExpiration = 10 // Session expires after 10 seconds
|
||||
)
|
||||
|
||||
var (
|
||||
// Create the common session store and secret
|
||||
memStore = handlers.NewMemoryStore(1)
|
||||
secret = "testimony of the ancients"
|
||||
)
|
||||
|
||||
// The struct used to pass data to the session template.
|
||||
type sessionPageInfo struct {
|
||||
SessionID string
|
||||
Title string
|
||||
Text string
|
||||
}
|
||||
|
||||
// Authenticate the Basic Auth credentials.
|
||||
func authenticate(u, p string) (interface{}, bool) {
|
||||
if u == "user" && p == "pwd" {
|
||||
return u + p, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Handle the session page requests.
|
||||
func sessionPageRenderer(w handlers.GhostWriter, r *http.Request) {
|
||||
var (
|
||||
txt interface{}
|
||||
data sessionPageInfo
|
||||
title string
|
||||
)
|
||||
|
||||
ssn := w.Session()
|
||||
if r.Method == "GET" {
|
||||
txt = ssn.Data[sessionPageKey]
|
||||
} else {
|
||||
txt = r.FormValue(sessionPageKey)
|
||||
ssn.Data[sessionPageKey] = txt
|
||||
}
|
||||
if r.URL.Path == "/session/auth" {
|
||||
title = sessionPageAuthTitle
|
||||
} else {
|
||||
title = sessionPageTitle
|
||||
}
|
||||
if txt != nil {
|
||||
data = sessionPageInfo{ssn.ID(), title, txt.(string)}
|
||||
} else {
|
||||
data = sessionPageInfo{ssn.ID(), title, "[nil]"}
|
||||
}
|
||||
err := templates.Render("templates/session.tmpl", w, data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare the context value for the chained handlers context page.
|
||||
func setContext(w handlers.GhostWriter, r *http.Request) {
|
||||
w.Context()[contextPageKey] = time.Now().String()
|
||||
}
|
||||
|
||||
// Retrieve the context value and render the chained handlers context page.
|
||||
func renderContextPage(w handlers.GhostWriter, r *http.Request) {
|
||||
err := templates.Render("templates/amber/context.amber",
|
||||
w, &struct{ Val string }{w.Context()[contextPageKey].(string)})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare the web server and kick it off.
|
||||
func main() {
|
||||
// Blank the default logger's prefixes
|
||||
log.SetFlags(0)
|
||||
|
||||
// Compile the dynamic templates (native Go templates and Amber
|
||||
// templates are both registered via the for-side-effects-only imports)
|
||||
err := templates.CompileDir("./templates/")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Set the simple routes for static files
|
||||
mux := pat.New()
|
||||
mux.Get("/", handlers.StaticFileHandler("./index.html"))
|
||||
mux.Get("/public/", http.StripPrefix("/public/", http.FileServer(http.Dir("./public/"))))
|
||||
|
||||
// Set the more complex routes for session handling and dynamic page (same
|
||||
// handler is used for both GET and POST).
|
||||
ssnOpts := handlers.NewSessionOptions(memStore, secret)
|
||||
ssnOpts.CookieTemplate.MaxAge = sessionExpiration
|
||||
hSsn := handlers.SessionHandler(
|
||||
handlers.ContextHandlerFunc(
|
||||
handlers.GhostHandlerFunc(sessionPageRenderer),
|
||||
1),
|
||||
ssnOpts)
|
||||
mux.Get("/session", hSsn)
|
||||
mux.Post("/session", hSsn)
|
||||
|
||||
hAuthSsn := handlers.BasicAuthHandler(hSsn, authenticate, "")
|
||||
mux.Get("/session/auth", hAuthSsn)
|
||||
mux.Post("/session/auth", hAuthSsn)
|
||||
|
||||
// Set the handler for the chained context route
|
||||
mux.Get("/context", handlers.ContextHandler(handlers.ChainHandlerFuncs(
|
||||
handlers.GhostHandlerFunc(setContext),
|
||||
handlers.GhostHandlerFunc(renderContextPage)),
|
||||
1))
|
||||
|
||||
// Set the panic route, which simply panics
|
||||
mux.Get("/panic", http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
panic("explicit panic")
|
||||
}))
|
||||
|
||||
// Combine the top level handlers, that wrap around the muxer.
|
||||
// Panic is the outermost, so that any panic is caught and responded to with a code 500.
|
||||
// Log is next, so that every request is logged along with the URL, status code and response time.
|
||||
// GZIP is then applied, so that content is compressed.
|
||||
// Finally, the muxer finds the specific handler that applies to the route.
|
||||
h := handlers.FaviconHandler(
|
||||
handlers.PanicHandler(
|
||||
handlers.LogHandler(
|
||||
handlers.GZIPHandler(
|
||||
mux,
|
||||
nil),
|
||||
handlers.NewLogOptions(nil, handlers.Ltiny)),
|
||||
nil),
|
||||
"./public/favicon.ico",
|
||||
48*time.Hour)
|
||||
|
||||
// Assign the combined handler to the server.
|
||||
http.Handle("/", h)
|
||||
|
||||
// Start it up.
|
||||
if err := http.ListenAndServe(":9000", nil); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
+873
File diff suppressed because one or more lines are too long
BIN
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
+6
File diff suppressed because one or more lines are too long
BIN
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
+3
@@ -0,0 +1,3 @@
|
||||
body {
|
||||
background-color: silver;
|
||||
}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
!!! 5
|
||||
html
|
||||
head
|
||||
title Chained Context
|
||||
link[type="text/css"][rel="stylesheet"][href="/public/bootstrap-combined.min.css"]
|
||||
body
|
||||
h1 Chained Context
|
||||
h2 Value found: #{Val}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
<html>
|
||||
<head>
|
||||
<title>{{ .Title }}</title>
|
||||
<link type="text/css" rel="stylesheet" href="/public/styles.css">
|
||||
<link type="text/css" rel="stylesheet" href="/public/bootstrap-combined.min.css">
|
||||
</head>
|
||||
<body>
|
||||
<h1>Session: {{ .SessionID }}</h1>
|
||||
<ol>
|
||||
<li><a href="/">Home</a></li>
|
||||
<li><a href="/session">Session</a></li>
|
||||
<li><a href="/session/auth">Authenticated Session</a></li>
|
||||
<li><a href="/context">Chained Context</a></li>
|
||||
<li><a href="/panic">Panic</a></li>
|
||||
<li><a href="/public/styles.css">Styles.css</a></li>
|
||||
<li><a href="/public/jquery-2.0.0.min.js">JQuery</a></li>
|
||||
<li><a href="/public/logo.png">Logo</a></li>
|
||||
</ol>
|
||||
<h2>Current Value: {{ .Text }}</h2>
|
||||
<form method="POST">
|
||||
<input type="text" name="txt" placeholder="some value to save to session"></input>
|
||||
<button type="submit">Submit</button>
|
||||
</form>
|
||||
|
||||
<script src="/public/jquery-2.0.0.min.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
package handlers
|
||||
|
||||
// Inspired by node.js' Connect library implementation of the basicAuth middleware.
|
||||
// https://github.com/senchalabs/connect
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Internal writer that keeps track of the currently authenticated user.
|
||||
type userResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
user interface{}
|
||||
userName string
|
||||
}
|
||||
|
||||
// Implement the WrapWriter interface.
|
||||
func (this *userResponseWriter) WrappedWriter() http.ResponseWriter {
|
||||
return this.ResponseWriter
|
||||
}
|
||||
|
||||
// Writes an unauthorized response to the client, specifying the expected authentication
|
||||
// information.
|
||||
func Unauthorized(w http.ResponseWriter, realm string) {
|
||||
w.Header().Set("Www-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm))
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
w.Write([]byte("Unauthorized"))
|
||||
}
|
||||
|
||||
// Writes a bad request response to the client, with an optional message.
|
||||
func BadRequest(w http.ResponseWriter, msg string) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
if msg == "" {
|
||||
msg = "Bad Request"
|
||||
}
|
||||
w.Write([]byte(msg))
|
||||
}
|
||||
|
||||
// BasicAuthHandlerFunc is the same as BasicAuthHandler, it is just a convenience
|
||||
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
|
||||
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
|
||||
func BasicAuthHandlerFunc(h http.HandlerFunc,
|
||||
authFn func(string, string) (interface{}, bool), realm string) http.HandlerFunc {
|
||||
return BasicAuthHandler(h, authFn, realm)
|
||||
}
|
||||
|
||||
// Returns a Basic Authentication handler, protecting the wrapped handler from
|
||||
// being accessed if the authentication function is not successful.
|
||||
func BasicAuthHandler(h http.Handler,
|
||||
authFn func(string, string) (interface{}, bool), realm string) http.HandlerFunc {
|
||||
|
||||
if realm == "" {
|
||||
realm = "Authorization Required"
|
||||
}
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
// Self-awareness
|
||||
if _, ok := GetUser(w); ok {
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
authInfo := r.Header.Get("Authorization")
|
||||
if authInfo == "" {
|
||||
// No authorization info, return 401
|
||||
Unauthorized(w, realm)
|
||||
return
|
||||
}
|
||||
parts := strings.Split(authInfo, " ")
|
||||
if len(parts) != 2 {
|
||||
BadRequest(w, "Bad authorization header")
|
||||
return
|
||||
}
|
||||
scheme := parts[0]
|
||||
creds, err := base64.StdEncoding.DecodeString(parts[1])
|
||||
if err != nil {
|
||||
BadRequest(w, "Bad credentials encoding")
|
||||
return
|
||||
}
|
||||
index := bytes.Index(creds, []byte(":"))
|
||||
if scheme != "Basic" || index < 0 {
|
||||
BadRequest(w, "Bad authorization header")
|
||||
return
|
||||
}
|
||||
user, pwd := string(creds[:index]), string(creds[index+1:])
|
||||
udata, ok := authFn(user, pwd)
|
||||
if ok {
|
||||
// Save user data and continue
|
||||
uw := &userResponseWriter{w, udata, user}
|
||||
h.ServeHTTP(uw, r)
|
||||
} else {
|
||||
Unauthorized(w, realm)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return the currently authenticated user. This is the same data that was returned
|
||||
// by the authentication function passed to BasicAuthHandler.
|
||||
func GetUser(w http.ResponseWriter) (interface{}, bool) {
|
||||
usr, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
|
||||
_, ok := tst.(*userResponseWriter)
|
||||
return ok
|
||||
})
|
||||
if ok {
|
||||
return usr.(*userResponseWriter).user, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Return the currently authenticated user name. This is the user name that was
|
||||
// authenticated for the current request.
|
||||
func GetUserName(w http.ResponseWriter) (string, bool) {
|
||||
usr, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
|
||||
_, ok := tst.(*userResponseWriter)
|
||||
return ok
|
||||
})
|
||||
if ok {
|
||||
return usr.(*userResponseWriter).userName, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUnauth(t *testing.T) {
|
||||
h := BasicAuthHandler(StaticFileHandler("./testdata/script.js"), func(u, pwd string) (interface{}, bool) {
|
||||
if u == "me" && pwd == "you" {
|
||||
return u, true
|
||||
}
|
||||
return nil, false
|
||||
}, "foo")
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusUnauthorized, res.StatusCode, t)
|
||||
assertHeader("Www-Authenticate", `Basic realm="foo"`, res, t)
|
||||
}
|
||||
|
||||
func TestGzippedAuth(t *testing.T) {
|
||||
h := GZIPHandler(BasicAuthHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
usr, ok := GetUser(w)
|
||||
if assertTrue(ok, "expected authenticated user, got false", t) {
|
||||
assertTrue(usr.(string) == "meyou", fmt.Sprintf("expected user data to be 'meyou', got '%s'", usr), t)
|
||||
}
|
||||
usr, ok = GetUserName(w)
|
||||
if assertTrue(ok, "expected authenticated user name, got false", t) {
|
||||
assertTrue(usr == "me", fmt.Sprintf("expected user name to be 'me', got '%s'", usr), t)
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Write([]byte(usr.(string)))
|
||||
}), func(u, pwd string) (interface{}, bool) {
|
||||
if u == "me" && pwd == "you" {
|
||||
return u + pwd, true
|
||||
}
|
||||
return nil, false
|
||||
}, ""), nil)
|
||||
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
req, err := http.NewRequest("GET", "http://me:you@"+s.URL[7:], nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertGzippedBody([]byte("me"), res, t)
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// ChainableHandler is a valid Handler interface, and adds the possibility to
|
||||
// chain other handlers.
|
||||
type ChainableHandler interface {
|
||||
http.Handler
|
||||
Chain(http.Handler) ChainableHandler
|
||||
ChainFunc(http.HandlerFunc) ChainableHandler
|
||||
}
|
||||
|
||||
// Default implementation of a simple ChainableHandler
|
||||
type chainHandler struct {
|
||||
http.Handler
|
||||
}
|
||||
|
||||
func (this *chainHandler) ChainFunc(h http.HandlerFunc) ChainableHandler {
|
||||
return this.Chain(h)
|
||||
}
|
||||
|
||||
// Implementation of the ChainableHandler interface, calls the chained handler
|
||||
// after the current one (sequential).
|
||||
func (this *chainHandler) Chain(h http.Handler) ChainableHandler {
|
||||
return &chainHandler{
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Add the chained handler after the call to this handler
|
||||
this.ServeHTTP(w, r)
|
||||
h.ServeHTTP(w, r)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// Convert a standard http handler to a chainable handler interface.
|
||||
func NewChainableHandler(h http.Handler) ChainableHandler {
|
||||
return &chainHandler{
|
||||
h,
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to chain multiple handler functions in a single call.
|
||||
func ChainHandlerFuncs(h ...http.HandlerFunc) ChainableHandler {
|
||||
return &chainHandler{
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
for _, v := range h {
|
||||
v(w, r)
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to chain multiple handlers in a single call.
|
||||
func ChainHandlers(h ...http.Handler) ChainableHandler {
|
||||
return &chainHandler{
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
for _, v := range h {
|
||||
v.ServeHTTP(w, r)
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestChaining(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
a := func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('a')
|
||||
}
|
||||
b := func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('b')
|
||||
}
|
||||
c := func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('c')
|
||||
}
|
||||
f := NewChainableHandler(http.HandlerFunc(a)).Chain(http.HandlerFunc(b)).Chain(http.HandlerFunc(c))
|
||||
f.ServeHTTP(nil, nil)
|
||||
|
||||
if buf.String() != "abc" {
|
||||
t.Errorf("expected 'abc', got %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainingWithHelperFunc(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
a := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('a')
|
||||
})
|
||||
b := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('b')
|
||||
})
|
||||
c := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('c')
|
||||
})
|
||||
d := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('d')
|
||||
})
|
||||
f := ChainHandlers(a, b, c, d)
|
||||
f.ServeHTTP(nil, nil)
|
||||
|
||||
if buf.String() != "abcd" {
|
||||
t.Errorf("expected 'abcd', got %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainingMixed(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
a := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('a')
|
||||
})
|
||||
b := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('b')
|
||||
})
|
||||
c := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('c')
|
||||
})
|
||||
d := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
buf.WriteRune('d')
|
||||
})
|
||||
f := NewChainableHandler(a).Chain(ChainHandlers(b, c)).Chain(d)
|
||||
f.ServeHTTP(nil, nil)
|
||||
|
||||
if buf.String() != "abcd" {
|
||||
t.Errorf("expected 'abcd', got %s", buf.String())
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Structure that holds the context map and exposes the ResponseWriter interface.
|
||||
type contextResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
m map[interface{}]interface{}
|
||||
}
|
||||
|
||||
// Implement the WrapWriter interface.
|
||||
func (this *contextResponseWriter) WrappedWriter() http.ResponseWriter {
|
||||
return this.ResponseWriter
|
||||
}
|
||||
|
||||
// ContextHandlerFunc is the same as ContextHandler, it is just a convenience
|
||||
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
|
||||
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
|
||||
func ContextHandlerFunc(h http.HandlerFunc, cap int) http.HandlerFunc {
|
||||
return ContextHandler(h, cap)
|
||||
}
|
||||
|
||||
// ContextHandler gives a context storage that lives only for the duration of
|
||||
// the request, with no locking involved.
|
||||
func ContextHandler(h http.Handler, cap int) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := GetContext(w); ok {
|
||||
// Self-awareness, context handler is already set up
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Create the context-providing ResponseWriter replacement.
|
||||
ctxw := &contextResponseWriter{
|
||||
w,
|
||||
make(map[interface{}]interface{}, cap),
|
||||
}
|
||||
// Call the wrapped handler with the context-aware writer
|
||||
h.ServeHTTP(ctxw, r)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to retrieve the context map from the ResponseWriter interface.
|
||||
func GetContext(w http.ResponseWriter) (map[interface{}]interface{}, bool) {
|
||||
ctxw, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
|
||||
_, ok := tst.(*contextResponseWriter)
|
||||
return ok
|
||||
})
|
||||
if ok {
|
||||
return ctxw.(*contextResponseWriter).m, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestContext(t *testing.T) {
|
||||
key := "key"
|
||||
val := 10
|
||||
body := "this is the output"
|
||||
|
||||
h2 := wrappedHandler(t, key, val, body)
|
||||
// Create the context handler with a wrapped handler
|
||||
h := ContextHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, _ := GetContext(w)
|
||||
assertTrue(ctx != nil, "expected context to be non-nil", t)
|
||||
assertTrue(len(ctx) == 0, fmt.Sprintf("expected context to be empty, got %d", len(ctx)), t)
|
||||
ctx[key] = val
|
||||
h2.ServeHTTP(w, r)
|
||||
}), 2)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
// First call
|
||||
res, err := http.DefaultClient.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
res.Body.Close()
|
||||
// Second call, context should be cleaned at start
|
||||
res, err = http.DefaultClient.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertBody([]byte(body), res, t)
|
||||
}
|
||||
|
||||
func TestWrappedContext(t *testing.T) {
|
||||
key := "key"
|
||||
val := 10
|
||||
body := "this is the output"
|
||||
|
||||
h2 := wrappedHandler(t, key, val, body)
|
||||
h := ContextHandler(LogHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, _ := GetContext(w)
|
||||
if !assertTrue(ctx != nil, "expected context to be non-nil", t) {
|
||||
panic("ctx is nil")
|
||||
}
|
||||
assertTrue(len(ctx) == 0, fmt.Sprintf("expected context to be empty, got %d", len(ctx)), t)
|
||||
ctx[key] = val
|
||||
h2.ServeHTTP(w, r)
|
||||
}), NewLogOptions(nil, "%s", "url")), 2)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.DefaultClient.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertBody([]byte(body), res, t)
|
||||
}
|
||||
|
||||
func wrappedHandler(t *testing.T, k, v interface{}, body string) http.Handler {
|
||||
return http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, _ := GetContext(w)
|
||||
ac := ctx[k]
|
||||
assertTrue(ac == v, fmt.Sprintf("expected value to be %v, got %v", v, ac), t)
|
||||
|
||||
// Actually write something
|
||||
_, err := w.Write([]byte(body))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
// Package handlers define reusable handler components that focus on offering
|
||||
// a single well-defined feature. Note that any http.Handler implementation
|
||||
// can be used with Ghost's chainable or wrappable handlers design.
|
||||
//
|
||||
// Go's standard library provides a number of such useful handlers in net/http:
|
||||
//
|
||||
// - FileServer(http.FileSystem)
|
||||
// - NotFoundHandler()
|
||||
// - RedirectHandler(string, int)
|
||||
// - StripPrefix(string, http.Handler)
|
||||
// - TimeoutHandler(http.Handler, time.Duration, string)
|
||||
//
|
||||
// This package adds the following list of handlers:
|
||||
//
|
||||
// - BasicAuthHandler(http.Handler, func(string, string) (interface{}, bool), string)
|
||||
// a Basic Authentication handler.
|
||||
// - ContextHandler(http.Handler, int) : a volatile storage map valid only
|
||||
// for the duration of the request, with no locking required.
|
||||
// - FaviconHandler(http.Handler, string, time.Duration) : an efficient favicon
|
||||
// handler.
|
||||
// - GZIPHandler(http.Handler) : compress the content of the body if the client
|
||||
// accepts gzip compression.
|
||||
// - LogHandler(http.Handler, *LogOptions) : customizable request logger.
|
||||
// - PanicHandler(http.Handler) : handle panics gracefully so that the client
|
||||
// receives a response (status code 500).
|
||||
// - SessionHandler(http.Handler, *SessionOptions) : a cookie-based, store-agnostic
|
||||
// persistent session handler.
|
||||
// - StaticFileHandler(string) : serve the contents of a specific file.
|
||||
package handlers
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/ghost"
|
||||
)
|
||||
|
||||
// FaviconHandlerFunc is the same as FaviconHandler, it is just a convenience
|
||||
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
|
||||
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
|
||||
func FaviconHandlerFunc(h http.HandlerFunc, path string, maxAge time.Duration) http.HandlerFunc {
|
||||
return FaviconHandler(h, path, maxAge)
|
||||
}
|
||||
|
||||
// Efficient favicon handler, mostly a port of node's Connect library implementation
|
||||
// of the favicon middleware.
|
||||
// https://github.com/senchalabs/connect
|
||||
func FaviconHandler(h http.Handler, path string, maxAge time.Duration) http.HandlerFunc {
|
||||
var buf []byte
|
||||
var hash string
|
||||
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
var err error
|
||||
if r.URL.Path == "/favicon.ico" {
|
||||
if buf == nil {
|
||||
// Read from file and cache
|
||||
ghost.LogFn("ghost.favicon : serving from %s", path)
|
||||
buf, err = ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
ghost.LogFn("ghost.favicon : error reading file : %s", err)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
hash = hashContent(buf)
|
||||
}
|
||||
writeHeaders(w.Header(), buf, maxAge, hash)
|
||||
writeBody(w, r, buf)
|
||||
} else {
|
||||
h.ServeHTTP(w, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Write the content of the favicon, or respond with a 404 not found
|
||||
// in case of error (hardly a critical error).
|
||||
func writeBody(w http.ResponseWriter, r *http.Request, buf []byte) {
|
||||
_, err := w.Write(buf)
|
||||
if err != nil {
|
||||
ghost.LogFn("ghost.favicon : error writing response : %s", err)
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// Correctly set the http headers.
|
||||
func writeHeaders(hdr http.Header, buf []byte, maxAge time.Duration, hash string) {
|
||||
hdr.Set("Content-Type", "image/x-icon")
|
||||
hdr.Set("Content-Length", strconv.Itoa(len(buf)))
|
||||
hdr.Set("Etag", hash)
|
||||
hdr.Set("Cache-Control", "public, max-age="+strconv.Itoa(int(maxAge.Seconds())))
|
||||
}
|
||||
|
||||
// Get the MD5 hash of the content.
|
||||
func hashContent(buf []byte) string {
|
||||
h := md5.New()
|
||||
return string(h.Sum(buf))
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFavicon(t *testing.T) {
|
||||
s := httptest.NewServer(FaviconHandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("ok"))
|
||||
}, "./testdata/favicon.ico", time.Second))
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL + "/favicon.ico")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertHeader("Content-Type", "image/x-icon", res, t)
|
||||
assertHeader("Cache-Control", "public, max-age=1", res, t)
|
||||
assertHeader("Content-Length", "1406", res, t)
|
||||
}
|
||||
|
||||
func TestFaviconInvalidPath(t *testing.T) {
|
||||
s := httptest.NewServer(FaviconHandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("ok"))
|
||||
}, "./testdata/xfavicon.ico", time.Second))
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL + "/favicon.ico")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
assertStatus(http.StatusNotFound, res.StatusCode, t)
|
||||
}
|
||||
|
||||
func TestFaviconFromCache(t *testing.T) {
|
||||
s := httptest.NewServer(FaviconHandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("ok"))
|
||||
}, "./testdata/favicon.ico", time.Second))
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL + "/favicon.ico")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
// Rename the file temporarily
|
||||
err = os.Rename("./testdata/favicon.ico", "./testdata/xfavicon.ico")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer os.Rename("./testdata/xfavicon.ico", "./testdata/favicon.ico")
|
||||
res, err = http.Get(s.URL + "/favicon.ico")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertHeader("Content-Type", "image/x-icon", res, t)
|
||||
assertHeader("Cache-Control", "public, max-age=1", res, t)
|
||||
assertHeader("Content-Length", "1406", res, t)
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Interface giving easy access to the most common augmented features.
|
||||
type GhostWriter interface {
|
||||
http.ResponseWriter
|
||||
UserName() string
|
||||
User() interface{}
|
||||
Context() map[interface{}]interface{}
|
||||
Session() *Session
|
||||
}
|
||||
|
||||
// Internal implementation of the GhostWriter interface.
|
||||
type ghostWriter struct {
|
||||
http.ResponseWriter
|
||||
userName string
|
||||
user interface{}
|
||||
ctx map[interface{}]interface{}
|
||||
ssn *Session
|
||||
}
|
||||
|
||||
func (this *ghostWriter) UserName() string {
|
||||
return this.userName
|
||||
}
|
||||
|
||||
func (this *ghostWriter) User() interface{} {
|
||||
return this.user
|
||||
}
|
||||
|
||||
func (this *ghostWriter) Context() map[interface{}]interface{} {
|
||||
return this.ctx
|
||||
}
|
||||
|
||||
func (this *ghostWriter) Session() *Session {
|
||||
return this.ssn
|
||||
}
|
||||
|
||||
// Convenience handler that wraps a custom function with direct access to the
|
||||
// authenticated user, context and session on the writer.
|
||||
func GhostHandlerFunc(h func(w GhostWriter, r *http.Request)) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if gw, ok := getGhostWriter(w); ok {
|
||||
// Self-awareness
|
||||
h(gw, r)
|
||||
return
|
||||
}
|
||||
uid, _ := GetUserName(w)
|
||||
usr, _ := GetUser(w)
|
||||
ctx, _ := GetContext(w)
|
||||
ssn, _ := GetSession(w)
|
||||
gw := &ghostWriter{
|
||||
w,
|
||||
uid,
|
||||
usr,
|
||||
ctx,
|
||||
ssn,
|
||||
}
|
||||
h(gw, r)
|
||||
}
|
||||
}
|
||||
|
||||
// Check the writer chain to find a ghostWriter.
|
||||
func getGhostWriter(w http.ResponseWriter) (*ghostWriter, bool) {
|
||||
gw, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
|
||||
_, ok := tst.(*ghostWriter)
|
||||
return ok
|
||||
})
|
||||
if ok {
|
||||
return gw.(*ghostWriter), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Thanks to Andrew Gerrand for inspiration:
|
||||
// https://groups.google.com/d/msg/golang-nuts/eVnTcMwNVjM/4vYU8id9Q2UJ
|
||||
//
|
||||
// Also, node's Connect library implementation of the compress middleware:
|
||||
// https://github.com/senchalabs/connect/blob/master/lib/middleware/compress.js
|
||||
//
|
||||
// And StackOverflow's explanation of Vary: Accept-Encoding header:
|
||||
// http://stackoverflow.com/questions/7848796/what-does-varyaccept-encoding-mean
|
||||
|
||||
// Internal gzipped writer that satisfies both the (body) writer in gzipped format,
|
||||
// and maintains the rest of the ResponseWriter interface for header manipulation.
|
||||
type gzipResponseWriter struct {
|
||||
io.Writer
|
||||
http.ResponseWriter
|
||||
r *http.Request // Keep a hold of the Request, for the filter function
|
||||
filtered bool // Has the request been run through the filter function?
|
||||
dogzip bool // Should we do GZIP compression for this request?
|
||||
filterFn func(http.ResponseWriter, *http.Request) bool
|
||||
}
|
||||
|
||||
// Make sure the filter function is applied.
|
||||
func (w *gzipResponseWriter) applyFilter() {
|
||||
if !w.filtered {
|
||||
if w.dogzip = w.filterFn(w, w.r); w.dogzip {
|
||||
setGzipHeaders(w.Header())
|
||||
}
|
||||
w.filtered = true
|
||||
}
|
||||
}
|
||||
|
||||
// Unambiguous Write() implementation (otherwise both ResponseWriter and Writer
|
||||
// want to claim this method).
|
||||
func (w *gzipResponseWriter) Write(b []byte) (int, error) {
|
||||
w.applyFilter()
|
||||
if w.dogzip {
|
||||
// Write compressed
|
||||
return w.Writer.Write(b)
|
||||
}
|
||||
// Write uncompressed
|
||||
return w.ResponseWriter.Write(b)
|
||||
}
|
||||
|
||||
// Intercept the WriteHeader call to correctly set the GZIP headers.
|
||||
func (w *gzipResponseWriter) WriteHeader(code int) {
|
||||
w.applyFilter()
|
||||
w.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
// Implement WrapWriter interface
|
||||
func (w *gzipResponseWriter) WrappedWriter() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
|
||||
var (
|
||||
defaultFilterTypes = [...]string{
|
||||
"text",
|
||||
"javascript",
|
||||
"json",
|
||||
}
|
||||
)
|
||||
|
||||
// Default filter to check if the response should be GZIPped.
|
||||
// By default, all text (html, css, xml, ...), javascript and json
|
||||
// content types are candidates for GZIP.
|
||||
func defaultFilter(w http.ResponseWriter, r *http.Request) bool {
|
||||
hdr := w.Header()
|
||||
for _, tp := range defaultFilterTypes {
|
||||
ok := HeaderMatch(hdr, "Content-Type", HmContains, tp)
|
||||
if ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GZIPHandlerFunc is the same as GZIPHandler, it is just a convenience
|
||||
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
|
||||
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
|
||||
func GZIPHandlerFunc(h http.HandlerFunc, filterFn func(http.ResponseWriter, *http.Request) bool) http.HandlerFunc {
|
||||
return GZIPHandler(h, filterFn)
|
||||
}
|
||||
|
||||
// Gzip compression HTTP handler. If the client supports it, it compresses the response
|
||||
// written by the wrapped handler. The filter function is called when the response is about
|
||||
// to be written to determine if compression should be applied. If this argument is nil,
|
||||
// the default filter will GZIP only content types containing /json|text|javascript/.
|
||||
func GZIPHandler(h http.Handler, filterFn func(http.ResponseWriter, *http.Request) bool) http.HandlerFunc {
|
||||
if filterFn == nil {
|
||||
filterFn = defaultFilter
|
||||
}
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := getGzipWriter(w); ok {
|
||||
// Self-awareness, gzip handler is already set up
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
hdr := w.Header()
|
||||
setVaryHeader(hdr)
|
||||
|
||||
// Do nothing on a HEAD request
|
||||
if r.Method == "HEAD" {
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if !acceptsGzip(r.Header) {
|
||||
// No gzip support from the client, return uncompressed
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Prepare a gzip response container
|
||||
gz := gzip.NewWriter(w)
|
||||
gzw := &gzipResponseWriter{
|
||||
Writer: gz,
|
||||
ResponseWriter: w,
|
||||
r: r,
|
||||
filterFn: filterFn,
|
||||
}
|
||||
h.ServeHTTP(gzw, r)
|
||||
// Iff the handler completed successfully (no panic) and GZIP was indeed used, close the gzip writer,
|
||||
// which seems to generate a Write to the underlying writer.
|
||||
if gzw.dogzip {
|
||||
gz.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add the vary by "accept-encoding" header if it is not already set.
|
||||
func setVaryHeader(hdr http.Header) {
|
||||
if !HeaderMatch(hdr, "Vary", HmContains, "accept-encoding") {
|
||||
hdr.Add("Vary", "Accept-Encoding")
|
||||
}
|
||||
}
|
||||
|
||||
// Checks if the client accepts GZIP-encoded responses.
|
||||
func acceptsGzip(hdr http.Header) bool {
|
||||
ok := HeaderMatch(hdr, "Accept-Encoding", HmContains, "gzip")
|
||||
if !ok {
|
||||
ok = HeaderMatch(hdr, "Accept-Encoding", HmEquals, "*")
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
func setGzipHeaders(hdr http.Header) {
|
||||
// The content-type will be explicitly set somewhere down the path of handlers
|
||||
hdr.Set("Content-Encoding", "gzip")
|
||||
hdr.Del("Content-Length")
|
||||
}
|
||||
|
||||
// Helper function to retrieve the gzip writer.
|
||||
func getGzipWriter(w http.ResponseWriter) (*gzipResponseWriter, bool) {
|
||||
gz, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
|
||||
_, ok := tst.(*gzipResponseWriter)
|
||||
return ok
|
||||
})
|
||||
if ok {
|
||||
return gz.(*gzipResponseWriter), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGzipped(t *testing.T) {
|
||||
body := "This is the body"
|
||||
headers := []string{"gzip", "*", "gzip, deflate, sdch"}
|
||||
|
||||
h := GZIPHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, err := w.Write([]byte(body))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}), nil)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
for _, hdr := range headers {
|
||||
t.Logf("running with Accept-Encoding header %s", hdr)
|
||||
req, err := http.NewRequest("GET", s.URL, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", hdr)
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertHeader("Content-Encoding", "gzip", res, t)
|
||||
assertGzippedBody([]byte(body), res, t)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoGzip(t *testing.T) {
|
||||
body := "This is the body"
|
||||
|
||||
h := GZIPHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, err := w.Write([]byte(body))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}), nil)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
req, err := http.NewRequest("GET", s.URL, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertHeader("Content-Encoding", "", res, t)
|
||||
assertBody([]byte(body), res, t)
|
||||
}
|
||||
|
||||
func TestGzipOuterPanic(t *testing.T) {
|
||||
msg := "ko"
|
||||
|
||||
h := PanicHandler(
|
||||
GZIPHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
panic(msg)
|
||||
}), nil), nil)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
req, err := http.NewRequest("GET", s.URL, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusInternalServerError, res.StatusCode, t)
|
||||
assertHeader("Content-Encoding", "", res, t)
|
||||
assertBody([]byte(msg+"\n"), res, t)
|
||||
}
|
||||
|
||||
func TestNoGzipOnFilter(t *testing.T) {
|
||||
body := "This is the body"
|
||||
|
||||
h := GZIPHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "x/x")
|
||||
_, err := w.Write([]byte(body))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}), nil)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
req, err := http.NewRequest("GET", s.URL, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertHeader("Content-Encoding", "", res, t)
|
||||
assertBody([]byte(body), res, t)
|
||||
}
|
||||
|
||||
func TestNoGzipOnCustomFilter(t *testing.T) {
|
||||
body := "This is the body"
|
||||
|
||||
h := GZIPHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, err := w.Write([]byte(body))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}), func(w http.ResponseWriter, r *http.Request) bool {
|
||||
return false
|
||||
})
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
req, err := http.NewRequest("GET", s.URL, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertHeader("Content-Encoding", "", res, t)
|
||||
assertBody([]byte(body), res, t)
|
||||
}
|
||||
|
||||
func TestGzipOnCustomFilter(t *testing.T) {
|
||||
body := "This is the body"
|
||||
|
||||
h := GZIPHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "x/x")
|
||||
_, err := w.Write([]byte(body))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}), func(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
})
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
req, err := http.NewRequest("GET", s.URL, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertHeader("Content-Encoding", "gzip", res, t)
|
||||
assertGzippedBody([]byte(body), res, t)
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Kind of match to apply to the header check.
|
||||
type HeaderMatchType int
|
||||
|
||||
const (
|
||||
HmEquals HeaderMatchType = iota
|
||||
HmStartsWith
|
||||
HmEndsWith
|
||||
HmContains
|
||||
)
|
||||
|
||||
// Check if the specified header matches the test string, applying the header match type
|
||||
// specified.
|
||||
func HeaderMatch(hdr http.Header, nm string, matchType HeaderMatchType, test string) bool {
|
||||
// First get the header value
|
||||
val := hdr[http.CanonicalHeaderKey(nm)]
|
||||
if len(val) == 0 {
|
||||
return false
|
||||
}
|
||||
// Prepare the match test
|
||||
test = strings.ToLower(test)
|
||||
for _, v := range val {
|
||||
v = strings.Trim(strings.ToLower(v), " \n\t")
|
||||
switch matchType {
|
||||
case HmEquals:
|
||||
if v == test {
|
||||
return true
|
||||
}
|
||||
case HmStartsWith:
|
||||
if strings.HasPrefix(v, test) {
|
||||
return true
|
||||
}
|
||||
case HmEndsWith:
|
||||
if strings.HasSuffix(v, test) {
|
||||
return true
|
||||
}
|
||||
case HmContains:
|
||||
if strings.Contains(v, test) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
package handlers
|
||||
|
||||
// Inspired by node's Connect library implementation of the logging middleware
|
||||
// https://github.com/senchalabs/connect
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/ghost"
|
||||
)
|
||||
|
||||
const (
|
||||
// Predefined logging formats that can be passed as format string.
|
||||
Ldefault = "_default_"
|
||||
Lshort = "_short_"
|
||||
Ltiny = "_tiny_"
|
||||
)
|
||||
|
||||
var (
|
||||
// Token parser for request and response headers
|
||||
rxHeaders = regexp.MustCompile(`^(req|res)\[([^\]]+)\]$`)
|
||||
|
||||
// Lookup table for predefined formats
|
||||
predefFormats = map[string]struct {
|
||||
fmt string
|
||||
toks []string
|
||||
}{
|
||||
Ldefault: {
|
||||
`%s - - [%s] "%s %s HTTP/%s" %d %s "%s" "%s"`,
|
||||
[]string{"remote-addr", "date", "method", "url", "http-version", "status", "res[Content-Length]", "referrer", "user-agent"},
|
||||
},
|
||||
Lshort: {
|
||||
`%s - %s %s HTTP/%s %d %s - %.3f s`,
|
||||
[]string{"remote-addr", "method", "url", "http-version", "status", "res[Content-Length]", "response-time"},
|
||||
},
|
||||
Ltiny: {
|
||||
`%s %s %d %s - %.3f s`,
|
||||
[]string{"method", "url", "status", "res[Content-Length]", "response-time"},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// Augmented ResponseWriter implementation that captures the status code for the logger.
|
||||
type statusResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
code int
|
||||
oriURL string
|
||||
}
|
||||
|
||||
// Intercept the WriteHeader call to save the status code.
|
||||
func (this *statusResponseWriter) WriteHeader(code int) {
|
||||
this.code = code
|
||||
this.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
// Intercept the Write call to save the default status code.
|
||||
func (this *statusResponseWriter) Write(data []byte) (int, error) {
|
||||
if this.code == 0 {
|
||||
this.code = http.StatusOK
|
||||
}
|
||||
return this.ResponseWriter.Write(data)
|
||||
}
|
||||
|
||||
// Implement the WrapWriter interface.
|
||||
func (this *statusResponseWriter) WrappedWriter() http.ResponseWriter {
|
||||
return this.ResponseWriter
|
||||
}
|
||||
|
||||
// LogHandler options
|
||||
type LogOptions struct {
|
||||
LogFn func(string, ...interface{}) // Defaults to ghost.LogFn if nil
|
||||
Format string
|
||||
Tokens []string
|
||||
CustomTokens map[string]func(http.ResponseWriter, *http.Request) string
|
||||
Immediate bool
|
||||
DateFormat string
|
||||
}
|
||||
|
||||
// Create a new LogOptions struct. The DateFormat defaults to time.RFC3339.
|
||||
func NewLogOptions(l func(string, ...interface{}), ft string, tok ...string) *LogOptions {
|
||||
return &LogOptions{
|
||||
LogFn: l,
|
||||
Format: ft,
|
||||
Tokens: tok,
|
||||
CustomTokens: make(map[string]func(http.ResponseWriter, *http.Request) string),
|
||||
DateFormat: time.RFC3339,
|
||||
}
|
||||
}
|
||||
|
||||
// LogHandlerFunc is the same as LogHandler, it is just a convenience
|
||||
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
|
||||
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
|
||||
func LogHandlerFunc(h http.HandlerFunc, opts *LogOptions) http.HandlerFunc {
|
||||
return LogHandler(h, opts)
|
||||
}
|
||||
|
||||
// Create a log handler for every request it receives.
|
||||
func LogHandler(h http.Handler, opts *LogOptions) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := getStatusWriter(w); ok {
|
||||
// Self-awareness, logging handler already set up
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Save the response start time
|
||||
st := time.Now()
|
||||
// Call the wrapped handler, with the augmented ResponseWriter to handle the status code
|
||||
stw := &statusResponseWriter{w, 0, ""}
|
||||
|
||||
// Log immediately if requested, otherwise on exit
|
||||
if opts.Immediate {
|
||||
logRequest(stw, r, st, opts)
|
||||
} else {
|
||||
// Store original URL, may get modified by handlers (i.e. StripPrefix)
|
||||
stw.oriURL = r.URL.String()
|
||||
defer logRequest(stw, r, st, opts)
|
||||
}
|
||||
h.ServeHTTP(stw, r)
|
||||
}
|
||||
}
|
||||
|
||||
func getIpAddress(r *http.Request) string {
|
||||
hdr := r.Header
|
||||
hdrRealIp := hdr.Get("X-Real-Ip")
|
||||
hdrForwardedFor := hdr.Get("X-Forwarded-For")
|
||||
if hdrRealIp == "" && hdrForwardedFor == "" {
|
||||
return r.RemoteAddr
|
||||
}
|
||||
if hdrForwardedFor != "" {
|
||||
// X-Forwarded-For is potentially a list of addresses separated with ","
|
||||
part := strings.Split(hdrForwardedFor, ",")[0]
|
||||
return strings.TrimSpace(part) + ":0"
|
||||
}
|
||||
return hdrRealIp
|
||||
}
|
||||
|
||||
// Check if the specified token is a predefined one, and if so return its current value.
|
||||
func getPredefinedTokenValue(t string, w *statusResponseWriter, r *http.Request,
|
||||
st time.Time, opts *LogOptions) (interface{}, bool) {
|
||||
|
||||
switch t {
|
||||
case "http-version":
|
||||
return fmt.Sprintf("%d.%d", r.ProtoMajor, r.ProtoMinor), true
|
||||
case "response-time":
|
||||
return time.Now().Sub(st).Seconds(), true
|
||||
case "remote-addr":
|
||||
return getIpAddress(r), true
|
||||
case "date":
|
||||
return time.Now().Format(opts.DateFormat), true
|
||||
case "method":
|
||||
return r.Method, true
|
||||
case "url":
|
||||
if w.oriURL != "" {
|
||||
return w.oriURL, true
|
||||
}
|
||||
return r.URL.String(), true
|
||||
case "referrer", "referer":
|
||||
return r.Referer(), true
|
||||
case "user-agent":
|
||||
return r.UserAgent(), true
|
||||
case "status":
|
||||
return w.code, true
|
||||
}
|
||||
|
||||
// Handle special cases for header
|
||||
mtch := rxHeaders.FindStringSubmatch(t)
|
||||
if len(mtch) > 2 {
|
||||
if mtch[1] == "req" {
|
||||
return r.Header.Get(mtch[2]), true
|
||||
} else {
|
||||
// This only works for headers explicitly set via the Header() map of
|
||||
// the writer, not those added by the http package under the covers.
|
||||
return w.Header().Get(mtch[2]), true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Do the actual logging.
|
||||
func logRequest(w *statusResponseWriter, r *http.Request, st time.Time, opts *LogOptions) {
|
||||
var (
|
||||
fn func(string, ...interface{})
|
||||
ok bool
|
||||
format string
|
||||
toks []string
|
||||
)
|
||||
|
||||
// If no specific log function, use the default one from the ghost package
|
||||
if opts.LogFn == nil {
|
||||
fn = ghost.LogFn
|
||||
} else {
|
||||
fn = opts.LogFn
|
||||
}
|
||||
|
||||
// If this is a predefined format, use it instead
|
||||
if v, ok := predefFormats[opts.Format]; ok {
|
||||
format = v.fmt
|
||||
toks = v.toks
|
||||
} else {
|
||||
format = opts.Format
|
||||
toks = opts.Tokens
|
||||
}
|
||||
args := make([]interface{}, len(toks))
|
||||
for i, t := range toks {
|
||||
if args[i], ok = getPredefinedTokenValue(t, w, r, st, opts); !ok {
|
||||
if f, ok := opts.CustomTokens[t]; ok && f != nil {
|
||||
args[i] = f(w, r)
|
||||
} else {
|
||||
args[i] = "?"
|
||||
}
|
||||
}
|
||||
}
|
||||
fn(format, args...)
|
||||
}
|
||||
|
||||
// Helper function to retrieve the status writer.
|
||||
func getStatusWriter(w http.ResponseWriter) (*statusResponseWriter, bool) {
|
||||
st, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
|
||||
_, ok := tst.(*statusResponseWriter)
|
||||
return ok
|
||||
})
|
||||
if ok {
|
||||
return st.(*statusResponseWriter), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type testCase struct {
|
||||
tok string
|
||||
fmt string
|
||||
rx *regexp.Regexp
|
||||
}
|
||||
|
||||
func TestLog(t *testing.T) {
|
||||
log.SetFlags(0)
|
||||
now := time.Now()
|
||||
|
||||
formats := []testCase{
|
||||
testCase{"remote-addr",
|
||||
"%s",
|
||||
regexp.MustCompile(`^127\.0\.0\.1:\d+\n$`),
|
||||
},
|
||||
testCase{"date",
|
||||
"%s",
|
||||
regexp.MustCompile(`^` + fmt.Sprintf("%04d-%02d-%02d", now.Year(), now.Month(), now.Day()) + `\n$`),
|
||||
},
|
||||
testCase{"method",
|
||||
"%s",
|
||||
regexp.MustCompile(`^GET\n$`),
|
||||
},
|
||||
testCase{"url",
|
||||
"%s",
|
||||
regexp.MustCompile(`^/\n$`),
|
||||
},
|
||||
testCase{"http-version",
|
||||
"%s",
|
||||
regexp.MustCompile(`^1\.1\n$`),
|
||||
},
|
||||
testCase{"status",
|
||||
"%d",
|
||||
regexp.MustCompile(`^200\n$`),
|
||||
},
|
||||
testCase{"referer",
|
||||
"%s",
|
||||
regexp.MustCompile(`^http://www\.test\.com\n$`),
|
||||
},
|
||||
testCase{"referrer",
|
||||
"%s",
|
||||
regexp.MustCompile(`^http://www\.test\.com\n$`),
|
||||
},
|
||||
testCase{"user-agent",
|
||||
"%s",
|
||||
regexp.MustCompile(`^Go \d+\.\d+ package http\n$`),
|
||||
},
|
||||
testCase{"bidon",
|
||||
"%s",
|
||||
regexp.MustCompile(`^\?\n$`),
|
||||
},
|
||||
testCase{"response-time",
|
||||
"%.3f",
|
||||
regexp.MustCompile(`^0\.1\d\d\n$`),
|
||||
},
|
||||
testCase{"req[Accept-Encoding]",
|
||||
"%s",
|
||||
regexp.MustCompile(`^gzip\n$`),
|
||||
},
|
||||
testCase{"res[blah]",
|
||||
"%s",
|
||||
regexp.MustCompile(`^$`),
|
||||
},
|
||||
testCase{"tiny",
|
||||
Ltiny,
|
||||
regexp.MustCompile(`^GET / 200 - 0\.1\d\d s\n$`),
|
||||
},
|
||||
testCase{"short",
|
||||
Lshort,
|
||||
regexp.MustCompile(`^127\.0\.0\.1:\d+ - GET / HTTP/1\.1 200 - 0\.1\d\d s\n$`),
|
||||
},
|
||||
testCase{"default",
|
||||
Ldefault,
|
||||
regexp.MustCompile(`^127\.0\.0\.1:\d+ - - \[\d{4}-\d{2}-\d{2}\] "GET / HTTP/1\.1" 200 "http://www\.test\.com" "Go \d+\.\d+ package http"\n$`),
|
||||
},
|
||||
testCase{"res[Content-Type]",
|
||||
"%s",
|
||||
regexp.MustCompile(`^text/plain\n$`),
|
||||
},
|
||||
}
|
||||
for _, tc := range formats {
|
||||
testLogCase(tc, t)
|
||||
}
|
||||
}
|
||||
|
||||
func testLogCase(tc testCase, t *testing.T) {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
log.SetOutput(buf)
|
||||
opts := NewLogOptions(log.Printf, tc.fmt, tc.tok)
|
||||
opts.DateFormat = "2006-01-02"
|
||||
h := LogHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(200)
|
||||
w.Write([]byte("body"))
|
||||
}), opts)
|
||||
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
t.Logf("running %s...", tc.tok)
|
||||
req, err := http.NewRequest("GET", s.URL, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Referer", "http://www.test.com")
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
ac := buf.String()
|
||||
assertTrue(tc.rx.MatchString(ac), fmt.Sprintf("expected log to match '%s', got '%s'", tc.rx.String(), ac), t)
|
||||
}
|
||||
|
||||
func TestForwardedFor(t *testing.T) {
|
||||
rx := regexp.MustCompile(`^1\.1\.1\.1:0 - - \[\d{4}-\d{2}-\d{2}\] "GET / HTTP/1\.1" 200 "http://www\.test\.com" "Go \d+\.\d+ package http"\n$`)
|
||||
|
||||
buf := bytes.NewBuffer(nil)
|
||||
log.SetOutput(buf)
|
||||
opts := NewLogOptions(log.Printf, Ldefault)
|
||||
opts.DateFormat = "2006-01-02"
|
||||
|
||||
h := LogHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(200)
|
||||
w.Write([]byte("body"))
|
||||
}), opts)
|
||||
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
t.Logf("running ForwardedFor...")
|
||||
req, err := http.NewRequest("GET", s.URL, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Referer", "http://www.test.com")
|
||||
req.Header.Set("X-Forwarded-For", "1.1.1.1")
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
ac := buf.String()
|
||||
assertTrue(rx.MatchString(ac), fmt.Sprintf("expected log to match '%s', got '%s'", rx.String(), ac), t)
|
||||
}
|
||||
|
||||
func TestImmediate(t *testing.T) {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
log.SetFlags(0)
|
||||
log.SetOutput(buf)
|
||||
opts := NewLogOptions(nil, Ltiny)
|
||||
opts.Immediate = true
|
||||
h := LogHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
w.WriteHeader(200)
|
||||
w.Write([]byte("body"))
|
||||
}), opts)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
ac := buf.String()
|
||||
// Since it is Immediate logging, status is still 0 and response time is less than 100ms
|
||||
rx := regexp.MustCompile(`GET / 0 - 0\.0\d\d s\n`)
|
||||
assertTrue(rx.MatchString(ac), fmt.Sprintf("expected log to match '%s', got '%s'", rx.String(), ac), t)
|
||||
}
|
||||
|
||||
func TestCustom(t *testing.T) {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
log.SetFlags(0)
|
||||
log.SetOutput(buf)
|
||||
opts := NewLogOptions(nil, "%s %s", "method", "custom")
|
||||
opts.CustomTokens["custom"] = func(w http.ResponseWriter, r *http.Request) string {
|
||||
return "toto"
|
||||
}
|
||||
|
||||
h := LogHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
w.WriteHeader(200)
|
||||
w.Write([]byte("body"))
|
||||
}), opts)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
ac := buf.String()
|
||||
rx := regexp.MustCompile(`GET toto`)
|
||||
assertTrue(rx.MatchString(ac), fmt.Sprintf("expected log to match '%s', got '%s'", rx.String(), ac), t)
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Augmented response writer to hold the panic data (can be anything, not necessarily an error
|
||||
// interface).
|
||||
type errResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
perr interface{}
|
||||
}
|
||||
|
||||
// Implement the WrapWriter interface.
|
||||
func (this *errResponseWriter) WrappedWriter() http.ResponseWriter {
|
||||
return this.ResponseWriter
|
||||
}
|
||||
|
||||
// PanicHandlerFunc is the same as PanicHandler, it is just a convenience
|
||||
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
|
||||
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
|
||||
func PanicHandlerFunc(h http.HandlerFunc, errH http.HandlerFunc) http.HandlerFunc {
|
||||
return PanicHandler(h, errH)
|
||||
}
|
||||
|
||||
// Calls the wrapped handler and on panic calls the specified error handler. If the error handler is nil,
|
||||
// responds with a 500 error message.
|
||||
func PanicHandler(h http.Handler, errH http.Handler) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
if errH != nil {
|
||||
ew := &errResponseWriter{w, err}
|
||||
errH.ServeHTTP(ew, r)
|
||||
} else {
|
||||
http.Error(w, fmt.Sprintf("%s", err), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Call the protected handler
|
||||
h.ServeHTTP(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to retrieve the panic error, if any.
|
||||
func GetPanicError(w http.ResponseWriter) (interface{}, bool) {
|
||||
er, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
|
||||
_, ok := tst.(*errResponseWriter)
|
||||
return ok
|
||||
})
|
||||
if ok {
|
||||
return er.(*errResponseWriter).perr, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPanic(t *testing.T) {
|
||||
h := PanicHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
panic("test")
|
||||
}), nil)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusInternalServerError, res.StatusCode, t)
|
||||
}
|
||||
|
||||
func TestNoPanic(t *testing.T) {
|
||||
h := PanicHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
}), nil)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
}
|
||||
|
||||
func TestPanicCustom(t *testing.T) {
|
||||
h := PanicHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
panic("ok")
|
||||
}),
|
||||
http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
err, ok := GetPanicError(w)
|
||||
if !ok {
|
||||
panic("no panic error found")
|
||||
}
|
||||
w.WriteHeader(501)
|
||||
w.Write([]byte(err.(string)))
|
||||
}))
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(501, res.StatusCode, t)
|
||||
assertBody([]byte("ok"), res, t)
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/garyburd/redigo/redis"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNoKeyPrefix = errors.New("cannot get session keys without a key prefix")
|
||||
)
|
||||
|
||||
type RedisStoreOptions struct {
|
||||
Network string
|
||||
Address string
|
||||
ConnectTimeout time.Duration
|
||||
ReadTimeout time.Duration
|
||||
WriteTimeout time.Duration
|
||||
Database int // Redis database to use for session keys
|
||||
KeyPrefix string // If set, keys will be KeyPrefix:SessionID (semicolon added)
|
||||
BrowserSessServerTTL time.Duration // Defaults to 2 days
|
||||
}
|
||||
|
||||
type RedisStore struct {
|
||||
opts *RedisStoreOptions
|
||||
conn redis.Conn
|
||||
}
|
||||
|
||||
// Create a redis session store with the specified options.
|
||||
func NewRedisStore(opts *RedisStoreOptions) *RedisStore {
|
||||
var err error
|
||||
rs := &RedisStore{opts, nil}
|
||||
rs.conn, err = redis.DialTimeout(opts.Network, opts.Address, opts.ConnectTimeout,
|
||||
opts.ReadTimeout, opts.WriteTimeout)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return rs
|
||||
}
|
||||
|
||||
// Get the session from the store.
|
||||
func (this *RedisStore) Get(id string) (*Session, error) {
|
||||
key := id
|
||||
if this.opts.KeyPrefix != "" {
|
||||
key = this.opts.KeyPrefix + ":" + id
|
||||
}
|
||||
b, err := redis.Bytes(this.conn.Do("GET", key))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var sess Session
|
||||
err = json.Unmarshal(b, &sess)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sess, nil
|
||||
}
|
||||
|
||||
// Save the session into the store.
|
||||
func (this *RedisStore) Set(sess *Session) error {
|
||||
b, err := json.Marshal(sess)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := sess.ID()
|
||||
if this.opts.KeyPrefix != "" {
|
||||
key = this.opts.KeyPrefix + ":" + sess.ID()
|
||||
}
|
||||
ttl := sess.MaxAge()
|
||||
if ttl == 0 {
|
||||
// Browser session, set to specified TTL
|
||||
ttl = this.opts.BrowserSessServerTTL
|
||||
if ttl == 0 {
|
||||
ttl = 2 * 24 * time.Hour // Default to 2 days
|
||||
}
|
||||
}
|
||||
_, err = this.conn.Do("SETEX", key, int(ttl.Seconds()), b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete the session from the store.
|
||||
func (this *RedisStore) Delete(id string) error {
|
||||
key := id
|
||||
if this.opts.KeyPrefix != "" {
|
||||
key = this.opts.KeyPrefix + ":" + id
|
||||
}
|
||||
_, err := this.conn.Do("DEL", key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clear all sessions from the store. Requires the use of a key
|
||||
// prefix in the store options, otherwise the method refuses to delete all keys.
|
||||
func (this *RedisStore) Clear() error {
|
||||
vals, err := this.getSessionKeys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(vals) > 0 {
|
||||
this.conn.Send("MULTI")
|
||||
for _, v := range vals {
|
||||
this.conn.Send("DEL", v)
|
||||
}
|
||||
_, err = this.conn.Do("EXEC")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the number of session keys in the store. Requires the use of a
|
||||
// key prefix in the store options, otherwise returns -1 (cannot tell
|
||||
// session keys from other keys).
|
||||
func (this *RedisStore) Len() int {
|
||||
vals, err := this.getSessionKeys()
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return len(vals)
|
||||
}
|
||||
|
||||
func (this *RedisStore) getSessionKeys() ([]interface{}, error) {
|
||||
if this.opts.KeyPrefix != "" {
|
||||
return redis.Values(this.conn.Do("KEYS", this.opts.KeyPrefix+":*"))
|
||||
}
|
||||
return nil, ErrNoKeyPrefix
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// This interface can be implemented by an augmented ResponseWriter, so that
|
||||
// it doesn't hide other augmented writers in the chain.
|
||||
type WrapWriter interface {
|
||||
http.ResponseWriter
|
||||
WrappedWriter() http.ResponseWriter
|
||||
}
|
||||
|
||||
// Helper function to retrieve a specific ResponseWriter.
|
||||
func GetResponseWriter(w http.ResponseWriter,
|
||||
predicate func(http.ResponseWriter) bool) (http.ResponseWriter, bool) {
|
||||
|
||||
for {
|
||||
// Check if this writer is the one we're looking for
|
||||
if w != nil && predicate(w) {
|
||||
return w, true
|
||||
}
|
||||
// If it is a WrapWriter, move back the chain of wrapped writers
|
||||
ww, ok := w.(WrapWriter)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
w = ww.WrappedWriter()
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type baseWriter struct{}
|
||||
|
||||
func (b *baseWriter) Write(data []byte) (int, error) { return 0, nil }
|
||||
func (b *baseWriter) WriteHeader(code int) {}
|
||||
func (b *baseWriter) Header() http.Header { return nil }
|
||||
|
||||
func TestNilWriter(t *testing.T) {
|
||||
rw, ok := GetResponseWriter(nil, func(w http.ResponseWriter) bool {
|
||||
return true
|
||||
})
|
||||
assertTrue(rw == nil, "expected nil, got non-nil", t)
|
||||
assertTrue(!ok, "expected false, got true", t)
|
||||
}
|
||||
|
||||
func TestBaseWriter(t *testing.T) {
|
||||
bw := &baseWriter{}
|
||||
rw, ok := GetResponseWriter(bw, func(w http.ResponseWriter) bool {
|
||||
return true
|
||||
})
|
||||
assertTrue(rw == bw, fmt.Sprintf("expected %#v, got %#v", bw, rw), t)
|
||||
assertTrue(ok, "expected true, got false", t)
|
||||
}
|
||||
|
||||
func TestWrappedWriter(t *testing.T) {
|
||||
bw := &baseWriter{}
|
||||
ctx := &contextResponseWriter{bw, nil}
|
||||
rw, ok := GetResponseWriter(ctx, func(w http.ResponseWriter) bool {
|
||||
_, ok := w.(*baseWriter)
|
||||
return ok
|
||||
})
|
||||
assertTrue(rw == bw, fmt.Sprintf("expected %#v, got %#v", bw, rw), t)
|
||||
assertTrue(ok, "expected true, got false", t)
|
||||
}
|
||||
|
||||
func TestWrappedNotFoundWriter(t *testing.T) {
|
||||
bw := &baseWriter{}
|
||||
ctx := &contextResponseWriter{bw, nil}
|
||||
rw, ok := GetResponseWriter(ctx, func(w http.ResponseWriter) bool {
|
||||
_, ok := w.(*statusResponseWriter)
|
||||
return ok
|
||||
})
|
||||
assertTrue(rw == nil, fmt.Sprintf("expected nil, got %#v", rw), t)
|
||||
assertTrue(!ok, "expected false, got true", t)
|
||||
}
|
||||
+321
@@ -0,0 +1,321 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/ghost"
|
||||
"github.com/gorilla/securecookie"
|
||||
"github.com/nu7hatch/gouuid"
|
||||
)
|
||||
|
||||
const defaultCookieName = "ghost.sid"
|
||||
|
||||
var (
|
||||
ErrSessionSecretMissing = errors.New("session secret is missing")
|
||||
ErrNoSessionID = errors.New("session ID could not be generated")
|
||||
)
|
||||
|
||||
// The Session holds the data map that persists for the duration of the session.
|
||||
// The information stored in this map should be marshalable for the target Session store
|
||||
// format (i.e. json, sql, gob, etc. depending on how the store persists the data).
|
||||
type Session struct {
|
||||
isNew bool // keep private, not saved to JSON, will be false once read from the store
|
||||
internalSession
|
||||
}
|
||||
|
||||
// Use a separate private struct to hold the private fields of the Session,
|
||||
// although those fields are exposed (public). This is a trick to simplify
|
||||
// JSON encoding.
|
||||
type internalSession struct {
|
||||
Data map[string]interface{} // JSON cannot marshal a map[interface{}]interface{}
|
||||
ID string
|
||||
Created time.Time
|
||||
MaxAge time.Duration
|
||||
}
|
||||
|
||||
// Create a new Session instance. It panics in the unlikely event that a new random ID cannot be generated.
|
||||
func newSession(maxAge int) *Session {
|
||||
uid, err := uuid.NewV4()
|
||||
if err != nil {
|
||||
panic(ErrNoSessionID)
|
||||
}
|
||||
return &Session{
|
||||
true, // is new
|
||||
internalSession{
|
||||
make(map[string]interface{}),
|
||||
uid.String(),
|
||||
time.Now(),
|
||||
time.Duration(maxAge) * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Gets the ID of the session.
|
||||
func (ø *Session) ID() string {
|
||||
return ø.internalSession.ID
|
||||
}
|
||||
|
||||
// Get the max age duration
|
||||
func (ø *Session) MaxAge() time.Duration {
|
||||
return ø.internalSession.MaxAge
|
||||
}
|
||||
|
||||
// Get the creation time of the session.
|
||||
func (ø *Session) Created() time.Time {
|
||||
return ø.internalSession.Created
|
||||
}
|
||||
|
||||
// Is this a new Session (created by the current request)
|
||||
func (ø *Session) IsNew() bool {
|
||||
return ø.isNew
|
||||
}
|
||||
|
||||
// TODO : Resets the max age property of the session to its original value (sliding expiration).
|
||||
func (ø *Session) resetMaxAge() {
|
||||
}
|
||||
|
||||
// Marshal the session to JSON.
|
||||
func (ø *Session) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(ø.internalSession)
|
||||
}
|
||||
|
||||
// Unmarshal the JSON into the internal session struct.
|
||||
func (ø *Session) UnmarshalJSON(b []byte) error {
|
||||
return json.Unmarshal(b, &ø.internalSession)
|
||||
}
|
||||
|
||||
// Options object for the session handler. It specified the Session store to use for
|
||||
// persistence, the template for the session cookie (name, path, maxage, etc.),
|
||||
// whether or not the proxy should be trusted to determine if the connection is secure,
|
||||
// and the required secret to sign the session cookie.
|
||||
type SessionOptions struct {
|
||||
Store SessionStore
|
||||
CookieTemplate http.Cookie
|
||||
TrustProxy bool
|
||||
Secret string
|
||||
}
|
||||
|
||||
// Create a new SessionOptions struct, using default cookie and proxy values.
|
||||
func NewSessionOptions(store SessionStore, secret string) *SessionOptions {
|
||||
return &SessionOptions{
|
||||
Store: store,
|
||||
Secret: secret,
|
||||
}
|
||||
}
|
||||
|
||||
// The augmented ResponseWriter struct for the session handler. It holds the current
|
||||
// Session object and Session store, as well as flags and function to send the actual
|
||||
// session cookie at the end of the request.
|
||||
type sessResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
sess *Session
|
||||
sessStore SessionStore
|
||||
sessSent bool
|
||||
sendCookieFn func()
|
||||
}
|
||||
|
||||
// Implement the WrapWriter interface.
|
||||
func (ø *sessResponseWriter) WrappedWriter() http.ResponseWriter {
|
||||
return ø.ResponseWriter
|
||||
}
|
||||
|
||||
// Intercept the Write() method to add the Set-Cookie header before it's too late.
|
||||
func (ø *sessResponseWriter) Write(data []byte) (int, error) {
|
||||
if !ø.sessSent {
|
||||
ø.sendCookieFn()
|
||||
ø.sessSent = true
|
||||
}
|
||||
return ø.ResponseWriter.Write(data)
|
||||
}
|
||||
|
||||
// Intercept the WriteHeader() method to add the Set-Cookie header before it's too late.
|
||||
func (ø *sessResponseWriter) WriteHeader(code int) {
|
||||
if !ø.sessSent {
|
||||
ø.sendCookieFn()
|
||||
ø.sessSent = true
|
||||
}
|
||||
ø.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
// SessionHandlerFunc is the same as SessionHandler, it is just a convenience
|
||||
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
|
||||
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
|
||||
func SessionHandlerFunc(h http.HandlerFunc, opts *SessionOptions) http.HandlerFunc {
|
||||
return SessionHandler(h, opts)
|
||||
}
|
||||
|
||||
// Create a Session handler to offer the Session behaviour to the specified handler.
|
||||
func SessionHandler(h http.Handler, opts *SessionOptions) http.HandlerFunc {
|
||||
// Make sure the required cookie fields are set
|
||||
if opts.CookieTemplate.Name == "" {
|
||||
opts.CookieTemplate.Name = defaultCookieName
|
||||
}
|
||||
if opts.CookieTemplate.Path == "" {
|
||||
opts.CookieTemplate.Path = "/"
|
||||
}
|
||||
// Secret is required
|
||||
if opts.Secret == "" {
|
||||
panic(ErrSessionSecretMissing)
|
||||
}
|
||||
|
||||
// Return the actual handler
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := getSessionWriter(w); ok {
|
||||
// Self-awareness
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
if strings.Index(r.URL.Path, opts.CookieTemplate.Path) != 0 {
|
||||
// Session does not apply to this path
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Create a new Session or retrieve the existing session based on the
|
||||
// session cookie received.
|
||||
var sess *Session
|
||||
var ckSessId string
|
||||
exCk, err := r.Cookie(opts.CookieTemplate.Name)
|
||||
if err != nil {
|
||||
sess = newSession(opts.CookieTemplate.MaxAge)
|
||||
ghost.LogFn("ghost.session : error getting session cookie : %s", err)
|
||||
} else {
|
||||
ckSessId, err = parseSignedCookie(exCk, opts.Secret)
|
||||
if err != nil {
|
||||
sess = newSession(opts.CookieTemplate.MaxAge)
|
||||
ghost.LogFn("ghost.session : error parsing signed cookie : %s", err)
|
||||
} else if ckSessId == "" {
|
||||
sess = newSession(opts.CookieTemplate.MaxAge)
|
||||
ghost.LogFn("ghost.session : no existing session ID")
|
||||
} else {
|
||||
// Get the session
|
||||
sess, err = opts.Store.Get(ckSessId)
|
||||
if err != nil {
|
||||
sess = newSession(opts.CookieTemplate.MaxAge)
|
||||
ghost.LogFn("ghost.session : error getting session from store : %s", err)
|
||||
} else if sess == nil {
|
||||
sess = newSession(opts.CookieTemplate.MaxAge)
|
||||
ghost.LogFn("ghost.session : nil session")
|
||||
}
|
||||
}
|
||||
}
|
||||
// Save the original hash of the session, used to compare if the contents
|
||||
// have changed during the handling of the request, so that it has to be
|
||||
// saved to the stored.
|
||||
oriHash := hash(sess)
|
||||
|
||||
// Create the augmented ResponseWriter.
|
||||
srw := &sessResponseWriter{w, sess, opts.Store, false, func() {
|
||||
// This function is called when the header is about to be written, so that
|
||||
// the session cookie is correctly set.
|
||||
|
||||
// Check if the connection is secure
|
||||
proto := strings.Trim(strings.ToLower(r.Header.Get("X-Forwarded-Proto")), " ")
|
||||
tls := r.TLS != nil || (strings.HasPrefix(proto, "https") && opts.TrustProxy)
|
||||
if opts.CookieTemplate.Secure && !tls {
|
||||
ghost.LogFn("ghost.session : secure cookie on a non-secure connection, cookie not sent")
|
||||
return
|
||||
}
|
||||
if !sess.IsNew() {
|
||||
// If this is not a new session, no need to send back the cookie
|
||||
// TODO : Handle expires?
|
||||
return
|
||||
}
|
||||
|
||||
// Send the session cookie
|
||||
ck := opts.CookieTemplate
|
||||
ck.Value = sess.ID()
|
||||
err := signCookie(&ck, opts.Secret)
|
||||
if err != nil {
|
||||
ghost.LogFn("ghost.session : error signing cookie : %s", err)
|
||||
return
|
||||
}
|
||||
http.SetCookie(w, &ck)
|
||||
}}
|
||||
|
||||
// Call wrapped handler
|
||||
h.ServeHTTP(srw, r)
|
||||
|
||||
// TODO : Expiration management? srw.sess.resetMaxAge()
|
||||
// Do not save if content is the same, unless session is new (to avoid
|
||||
// creating a new session and sending a cookie on each successive request).
|
||||
if newHash := hash(sess); !sess.IsNew() && oriHash == newHash && newHash != 0 {
|
||||
// No changes to the session, no need to save
|
||||
ghost.LogFn("ghost.session : no changes to save to store")
|
||||
return
|
||||
}
|
||||
err = opts.Store.Set(sess)
|
||||
if err != nil {
|
||||
ghost.LogFn("ghost.session : error saving session to store : %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to retrieve the session for the current request.
|
||||
func GetSession(w http.ResponseWriter) (*Session, bool) {
|
||||
ss, ok := getSessionWriter(w)
|
||||
if ok {
|
||||
return ss.sess, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Helper function to retrieve the session store
|
||||
func GetSessionStore(w http.ResponseWriter) (SessionStore, bool) {
|
||||
ss, ok := getSessionWriter(w)
|
||||
if ok {
|
||||
return ss.sessStore, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Internal helper function to retrieve the session writer object.
|
||||
func getSessionWriter(w http.ResponseWriter) (*sessResponseWriter, bool) {
|
||||
ss, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
|
||||
_, ok := tst.(*sessResponseWriter)
|
||||
return ok
|
||||
})
|
||||
if ok {
|
||||
return ss.(*sessResponseWriter), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Parse a signed cookie and return the cookie value
|
||||
func parseSignedCookie(ck *http.Cookie, secret string) (string, error) {
|
||||
var val string
|
||||
|
||||
sck := securecookie.New([]byte(secret), nil)
|
||||
err := sck.Decode(ck.Name, ck.Value, &val)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// Sign the specified cookie's value
|
||||
func signCookie(ck *http.Cookie, secret string) error {
|
||||
sck := securecookie.New([]byte(secret), nil)
|
||||
enc, err := sck.Encode(ck.Name, ck.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ck.Value = enc
|
||||
return nil
|
||||
}
|
||||
|
||||
// Compute a CRC32 hash of the session's JSON-encoded contents.
|
||||
func hash(s *Session) uint32 {
|
||||
data, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
ghost.LogFn("ghost.session : error hash : %s", err)
|
||||
return 0 // 0 is always treated as "modified" session content
|
||||
}
|
||||
return crc32.ChecksumIEEE(data)
|
||||
}
|
||||
+258
@@ -0,0 +1,258 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
store SessionStore
|
||||
secret = "butchered at birth"
|
||||
)
|
||||
|
||||
func TestSession(t *testing.T) {
|
||||
stores := map[string]SessionStore{
|
||||
"memory": NewMemoryStore(1),
|
||||
"redis": NewRedisStore(&RedisStoreOptions{
|
||||
Network: "tcp",
|
||||
Address: ":6379",
|
||||
Database: 1,
|
||||
KeyPrefix: "sess",
|
||||
}),
|
||||
}
|
||||
for k, v := range stores {
|
||||
t.Logf("testing session with %s store\n", k)
|
||||
store = v
|
||||
t.Log("SessionExists")
|
||||
testSessionExists(t)
|
||||
t.Log("SessionPersists")
|
||||
testSessionPersists(t)
|
||||
t.Log("SessionExpires")
|
||||
testSessionExpires(t)
|
||||
t.Log("SessionBeforeExpires")
|
||||
testSessionBeforeExpires(t)
|
||||
t.Log("PanicIfNoSecret")
|
||||
testPanicIfNoSecret(t)
|
||||
t.Log("InvalidPath")
|
||||
testInvalidPath(t)
|
||||
t.Log("ValidSubPath")
|
||||
testValidSubPath(t)
|
||||
t.Log("SecureOverHttp")
|
||||
testSecureOverHttp(t)
|
||||
}
|
||||
}
|
||||
|
||||
func setupTest(f func(w http.ResponseWriter, r *http.Request), ckPath string, secure bool, maxAge int) *httptest.Server {
|
||||
opts := NewSessionOptions(store, secret)
|
||||
if ckPath != "" {
|
||||
opts.CookieTemplate.Path = ckPath
|
||||
}
|
||||
opts.CookieTemplate.Secure = secure
|
||||
opts.CookieTemplate.MaxAge = maxAge
|
||||
h := SessionHandler(http.HandlerFunc(f), opts)
|
||||
return httptest.NewServer(h)
|
||||
}
|
||||
|
||||
func doRequest(u string, newJar bool) *http.Response {
|
||||
var err error
|
||||
if newJar {
|
||||
http.DefaultClient.Jar, err = cookiejar.New(new(cookiejar.Options))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
res, err := http.Get(u)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func testSessionExists(t *testing.T) {
|
||||
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
|
||||
ssn, ok := GetSession(w)
|
||||
if assertTrue(ok, "expected session to be non-nil, got nil", t) {
|
||||
ssn.Data["foo"] = "bar"
|
||||
assertTrue(ssn.Data["foo"] == "bar", fmt.Sprintf("expected ssn[foo] to be 'bar', got %v", ssn.Data["foo"]), t)
|
||||
}
|
||||
w.Write([]byte("ok"))
|
||||
}, "", false, 0)
|
||||
defer s.Close()
|
||||
|
||||
res := doRequest(s.URL, true)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertBody([]byte("ok"), res, t)
|
||||
assertTrue(len(res.Cookies()) == 1, fmt.Sprintf("expected response to have 1 cookie, got %d", len(res.Cookies())), t)
|
||||
}
|
||||
|
||||
func testSessionPersists(t *testing.T) {
|
||||
cnt := 0
|
||||
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
|
||||
ssn, ok := GetSession(w)
|
||||
if !ok {
|
||||
panic("session not found!")
|
||||
}
|
||||
if cnt == 0 {
|
||||
ssn.Data["foo"] = "bar"
|
||||
w.Write([]byte("ok"))
|
||||
cnt++
|
||||
} else {
|
||||
w.Write([]byte(ssn.Data["foo"].(string)))
|
||||
}
|
||||
}, "", false, 0)
|
||||
defer s.Close()
|
||||
|
||||
// 1st call, set the session value
|
||||
res := doRequest(s.URL, true)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertBody([]byte("ok"), res, t)
|
||||
|
||||
// 2nd call, get the session value
|
||||
res = doRequest(s.URL, false)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertBody([]byte("bar"), res, t)
|
||||
assertTrue(len(res.Cookies()) == 0, fmt.Sprintf("expected 2nd response to have 0 cookie, got %d", len(res.Cookies())), t)
|
||||
}
|
||||
|
||||
func testSessionExpires(t *testing.T) {
|
||||
cnt := 0
|
||||
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
|
||||
ssn, ok := GetSession(w)
|
||||
if !ok {
|
||||
panic("session not found!")
|
||||
}
|
||||
if cnt == 0 {
|
||||
w.Write([]byte(ssn.ID()))
|
||||
cnt++
|
||||
} else {
|
||||
w.Write([]byte(ssn.ID()))
|
||||
}
|
||||
}, "", false, 1) // Expire in 1 second
|
||||
defer s.Close()
|
||||
|
||||
// 1st call, set the session value
|
||||
res := doRequest(s.URL, true)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
id1, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
res.Body.Close()
|
||||
time.Sleep(1001 * time.Millisecond)
|
||||
|
||||
// 2nd call, get the session value
|
||||
res = doRequest(s.URL, false)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
id2, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
res.Body.Close()
|
||||
sid1, sid2 := string(id1), string(id2)
|
||||
assertTrue(len(res.Cookies()) == 1, fmt.Sprintf("expected 2nd response to have 1 cookie, got %d", len(res.Cookies())), t)
|
||||
assertTrue(sid1 != sid2, "expected session IDs to be different, got same", t)
|
||||
}
|
||||
|
||||
func testSessionBeforeExpires(t *testing.T) {
|
||||
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
|
||||
ssn, ok := GetSession(w)
|
||||
if !ok {
|
||||
panic("session not found!")
|
||||
}
|
||||
w.Write([]byte(ssn.ID()))
|
||||
}, "", false, 1) // Expire in 1 second
|
||||
defer s.Close()
|
||||
|
||||
// 1st call, set the session value
|
||||
res := doRequest(s.URL, true)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
id1, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
res.Body.Close()
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
// 2nd call, get the session value
|
||||
res = doRequest(s.URL, false)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
id2, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
res.Body.Close()
|
||||
sid1, sid2 := string(id1), string(id2)
|
||||
assertTrue(len(res.Cookies()) == 0, fmt.Sprintf("expected 2nd response to have no cookie, got %d", len(res.Cookies())), t)
|
||||
assertTrue(sid1 == sid2, "expected session IDs to be the same, got different", t)
|
||||
}
|
||||
|
||||
func testPanicIfNoSecret(t *testing.T) {
|
||||
defer assertPanic(t)
|
||||
SessionHandler(http.NotFoundHandler(), NewSessionOptions(nil, ""))
|
||||
}
|
||||
|
||||
func testInvalidPath(t *testing.T) {
|
||||
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, ok := GetSession(w)
|
||||
assertTrue(!ok, "expected session to be nil, got non-nil", t)
|
||||
w.Write([]byte("ok"))
|
||||
}, "/foo", false, 0)
|
||||
defer s.Close()
|
||||
|
||||
res := doRequest(s.URL, true)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertBody([]byte("ok"), res, t)
|
||||
assertTrue(len(res.Cookies()) == 0, fmt.Sprintf("expected response to have no cookie, got %d", len(res.Cookies())), t)
|
||||
}
|
||||
|
||||
func testValidSubPath(t *testing.T) {
|
||||
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, ok := GetSession(w)
|
||||
assertTrue(ok, "expected session to be non-nil, got nil", t)
|
||||
w.Write([]byte("ok"))
|
||||
}, "/foo", false, 0)
|
||||
defer s.Close()
|
||||
|
||||
res := doRequest(s.URL+"/foo/bar", true)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertBody([]byte("ok"), res, t)
|
||||
assertTrue(len(res.Cookies()) == 1, fmt.Sprintf("expected response to have 1 cookie, got %d", len(res.Cookies())), t)
|
||||
}
|
||||
|
||||
func testSecureOverHttp(t *testing.T) {
|
||||
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, ok := GetSession(w)
|
||||
assertTrue(ok, "expected session to be non-nil, got nil", t)
|
||||
w.Write([]byte("ok"))
|
||||
}, "", true, 0)
|
||||
defer s.Close()
|
||||
|
||||
res := doRequest(s.URL, true)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertBody([]byte("ok"), res, t)
|
||||
assertTrue(len(res.Cookies()) == 0, fmt.Sprintf("expected response to have no cookie, got %d", len(res.Cookies())), t)
|
||||
}
|
||||
|
||||
// TODO : commented, certificate problem
|
||||
func xtestSecureOverHttps(t *testing.T) {
|
||||
opts := NewSessionOptions(store, secret)
|
||||
opts.CookieTemplate.Secure = true
|
||||
h := SessionHandler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
_, ok := GetSession(w)
|
||||
assertTrue(ok, "expected session to be non-nil, got nil", t)
|
||||
w.Write([]byte("ok"))
|
||||
}), opts)
|
||||
s := httptest.NewTLSServer(h)
|
||||
defer s.Close()
|
||||
|
||||
res := doRequest(s.URL, true)
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertBody([]byte("ok"), res, t)
|
||||
assertTrue(len(res.Cookies()) == 1, fmt.Sprintf("expected response to have 1 cookie, got %d", len(res.Cookies())), t)
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SessionStore interface, must be implemented by any store to be used
|
||||
// for session storage.
|
||||
type SessionStore interface {
|
||||
Get(id string) (*Session, error) // Get the session from the store
|
||||
Set(sess *Session) error // Save the session in the store
|
||||
Delete(id string) error // Delete the session from the store
|
||||
Clear() error // Delete all sessions from the store
|
||||
Len() int // Get the number of sessions in the store
|
||||
}
|
||||
|
||||
// In-memory implementation of a session store. Not recommended for production
|
||||
// use.
|
||||
type MemoryStore struct {
|
||||
l sync.RWMutex
|
||||
m map[string]*Session
|
||||
capc int
|
||||
}
|
||||
|
||||
// Create a new memory store.
|
||||
func NewMemoryStore(capc int) *MemoryStore {
|
||||
m := &MemoryStore{}
|
||||
m.capc = capc
|
||||
m.newMap()
|
||||
return m
|
||||
}
|
||||
|
||||
// Get the number of sessions saved in the store.
|
||||
func (this *MemoryStore) Len() int {
|
||||
return len(this.m)
|
||||
}
|
||||
|
||||
// Get the requested session from the store.
|
||||
func (this *MemoryStore) Get(id string) (*Session, error) {
|
||||
this.l.RLock()
|
||||
defer this.l.RUnlock()
|
||||
return this.m[id], nil
|
||||
}
|
||||
|
||||
// Save the session to the store.
|
||||
func (this *MemoryStore) Set(sess *Session) error {
|
||||
this.l.Lock()
|
||||
defer this.l.Unlock()
|
||||
this.m[sess.ID()] = sess
|
||||
if sess.IsNew() {
|
||||
// Since the memory store doesn't marshal to a string without the isNew, if it is left
|
||||
// to true, it will stay true forever.
|
||||
sess.isNew = false
|
||||
// Expire in the given time. If the maxAge is 0 (which means browser-session lifetime),
|
||||
// expire in a reasonable delay, 2 days. The weird case of a negative maxAge will
|
||||
// cause the immediate Delete call.
|
||||
wait := sess.MaxAge()
|
||||
if wait == 0 {
|
||||
wait = 2 * 24 * time.Hour
|
||||
}
|
||||
go func() {
|
||||
// Clear the session after the specified delay
|
||||
<-time.After(wait)
|
||||
this.Delete(sess.ID())
|
||||
}()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete the specified session ID from the store.
|
||||
func (this *MemoryStore) Delete(id string) error {
|
||||
this.l.Lock()
|
||||
defer this.l.Unlock()
|
||||
delete(this.m, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clear all sessions from the store.
|
||||
func (this *MemoryStore) Clear() error {
|
||||
this.l.Lock()
|
||||
defer this.l.Unlock()
|
||||
this.newMap()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Re-create the internal map, dropping all existing sessions.
|
||||
func (this *MemoryStore) newMap() {
|
||||
this.m = make(map[string]*Session, this.capc)
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// StaticFileHandler, unlike net/http.FileServer, serves the contents of a specific
|
||||
// file when it is called.
|
||||
func StaticFileHandler(path string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestServeFile(t *testing.T) {
|
||||
h := StaticFileHandler("./testdata/styles.css")
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
res, err := http.Get(s.URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertHeader("Content-Type", "text/css; charset=utf-8", res, t)
|
||||
assertHeader("Content-Encoding", "", res, t)
|
||||
assertBody([]byte(`* {
|
||||
background-color: white;
|
||||
}`), res, t)
|
||||
}
|
||||
|
||||
func TestGzippedFile(t *testing.T) {
|
||||
h := GZIPHandler(StaticFileHandler("./testdata/styles.css"), nil)
|
||||
s := httptest.NewServer(h)
|
||||
defer s.Close()
|
||||
|
||||
req, err := http.NewRequest("GET", s.URL, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", "*")
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
assertStatus(http.StatusOK, res.StatusCode, t)
|
||||
assertHeader("Content-Encoding", "gzip", res, t)
|
||||
assertHeader("Content-Type", "text/css; charset=utf-8", res, t)
|
||||
assertGzippedBody([]byte(`* {
|
||||
background-color: white;
|
||||
}`), res, t)
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
+1
@@ -0,0 +1 @@
|
||||
var a = 0;
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
* {
|
||||
background-color: white;
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func assertTrue(cond bool, msg string, t *testing.T) bool {
|
||||
if !cond {
|
||||
t.Error(msg)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func assertStatus(ex, ac int, t *testing.T) {
|
||||
if ex != ac {
|
||||
t.Errorf("expected status code to be %d, got %d", ex, ac)
|
||||
}
|
||||
}
|
||||
|
||||
func assertBody(ex []byte, res *http.Response, t *testing.T) {
|
||||
buf, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if !bytes.Equal(ex, buf) {
|
||||
t.Errorf("expected body to be '%s' (%d), got '%s' (%d)", ex, len(ex), buf, len(buf))
|
||||
}
|
||||
}
|
||||
|
||||
func assertGzippedBody(ex []byte, res *http.Response, t *testing.T) {
|
||||
gr, err := gzip.NewReader(res.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
buf := bytes.NewBuffer(nil)
|
||||
_, err = io.Copy(buf, gr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if !bytes.Equal(ex, buf.Bytes()) {
|
||||
t.Errorf("expected unzipped body to be '%s' (%d), got '%s' (%d)", ex, len(ex), buf.Bytes(), buf.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func assertHeader(hName, ex string, res *http.Response, t *testing.T) {
|
||||
hVal, ok := res.Header[hName]
|
||||
if (!ok || len(hVal) == 0) && len(ex) > 0 {
|
||||
t.Errorf("expected header %s to be %s, was not set", hName, ex)
|
||||
} else if len(hVal) > 0 && hVal[0] != ex {
|
||||
t.Errorf("expected header %s to be %s, got %s", hName, ex, hVal)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPanic(t *testing.T) {
|
||||
if err := recover(); err == nil {
|
||||
t.Error("expected a panic, got none")
|
||||
}
|
||||
}
|
||||
+203
File diff suppressed because one or more lines are too long
+38
@@ -0,0 +1,38 @@
|
||||
package amber
|
||||
|
||||
import (
|
||||
"github.com/PuerkitoBio/ghost/templates"
|
||||
"github.com/eknkc/amber"
|
||||
)
|
||||
|
||||
// The template compiler for Amber templates.
|
||||
type AmberCompiler struct {
|
||||
Options amber.Options
|
||||
c *amber.Compiler
|
||||
}
|
||||
|
||||
// Create a new Amber compiler with the specified Amber-specific options.
|
||||
func NewAmberCompiler(opts amber.Options) *AmberCompiler {
|
||||
return &AmberCompiler{
|
||||
opts,
|
||||
nil,
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation of the TemplateCompiler interface.
|
||||
func (this *AmberCompiler) Compile(f string) (templates.Templater, error) {
|
||||
// amber.CompileFile creates a new compiler each time. To limit the number
|
||||
// of allocations, reuse a compiler.
|
||||
if this.c == nil {
|
||||
this.c = amber.New()
|
||||
}
|
||||
this.c.Options = this.Options
|
||||
if err := this.c.ParseFile(f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return this.c.Compile()
|
||||
}
|
||||
|
||||
func init() {
|
||||
templates.Register(".amber", NewAmberCompiler(amber.DefaultOptions))
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package gotpl
|
||||
|
||||
import (
|
||||
"html/template"
|
||||
|
||||
"github.com/PuerkitoBio/ghost/templates"
|
||||
)
|
||||
|
||||
// The template compiler for native Go templates.
|
||||
type GoTemplateCompiler struct{}
|
||||
|
||||
// Implementation of the TemplateCompiler interface.
|
||||
func (this *GoTemplateCompiler) Compile(f string) (templates.Templater, error) {
|
||||
return template.ParseFiles(f)
|
||||
}
|
||||
|
||||
func init() {
|
||||
templates.Register(".tmpl", new(GoTemplateCompiler))
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
package templates
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/PuerkitoBio/ghost"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrTemplateNotExist = errors.New("template does not exist")
|
||||
ErrDirNotExist = errors.New("directory does not exist")
|
||||
|
||||
compilers = make(map[string]TemplateCompiler)
|
||||
|
||||
// The mutex guards the templaters map
|
||||
mu sync.RWMutex
|
||||
templaters = make(map[string]Templater)
|
||||
)
|
||||
|
||||
// Defines the interface that the template compiler must return. The Go native
|
||||
// templates implement this interface.
|
||||
type Templater interface {
|
||||
Execute(wr io.Writer, data interface{}) error
|
||||
}
|
||||
|
||||
// The interface that a template engine must implement to be used by Ghost.
|
||||
type TemplateCompiler interface {
|
||||
Compile(fileName string) (Templater, error)
|
||||
}
|
||||
|
||||
// TODO : How to manage Go nested templates?
|
||||
// TODO : Support Go's port of the mustache template?
|
||||
|
||||
// Register a template compiler for the specified extension. Extensions are case-sensitive.
|
||||
// The extension must start with a dot (it is compared to the result of path.Ext() on a
|
||||
// given file name).
|
||||
//
|
||||
// Registering is not thread-safe. Compilers should be registered before the http server
|
||||
// is started.
|
||||
// Compiling templates, on the other hand, is thread-safe.
|
||||
func Register(ext string, c TemplateCompiler) {
|
||||
if c == nil {
|
||||
panic("ghost: Register TemplateCompiler is nil")
|
||||
}
|
||||
if _, dup := compilers[ext]; dup {
|
||||
panic("ghost: Register called twice for extension " + ext)
|
||||
}
|
||||
compilers[ext] = c
|
||||
}
|
||||
|
||||
// Compile all templates that have a matching compiler (based on their extension) in the
|
||||
// specified directory.
|
||||
func CompileDir(dir string) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
return filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
|
||||
if fi == nil {
|
||||
return ErrDirNotExist
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
err = compileTemplate(path, dir)
|
||||
if err != nil {
|
||||
ghost.LogFn("ghost.templates : error compiling template %s : %s", path, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// Compile a single template file, using the specified base directory. The base
|
||||
// directory is used to set the name of the template (the part of the path relative to this
|
||||
// base directory is used as the name of the template).
|
||||
func Compile(path, base string) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
return compileTemplate(path, base)
|
||||
}
|
||||
|
||||
// Compile the specified template file if there is a matching compiler.
|
||||
func compileTemplate(p, base string) error {
|
||||
ext := path.Ext(p)
|
||||
c, ok := compilers[ext]
|
||||
// Ignore file if no template compiler exist for this extension
|
||||
if ok {
|
||||
t, err := c.Compile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := filepath.Rel(base, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ghost.LogFn("ghost.templates : storing template for file %s", key)
|
||||
templaters[key] = t
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Execute the template.
|
||||
func Execute(tplName string, w io.Writer, data interface{}) error {
|
||||
mu.RLock()
|
||||
t, ok := templaters[tplName]
|
||||
mu.RUnlock()
|
||||
if !ok {
|
||||
return ErrTemplateNotExist
|
||||
}
|
||||
return t.Execute(w, data)
|
||||
}
|
||||
|
||||
// Render is the same as Execute, except that it takes a http.ResponseWriter
|
||||
// instead of a generic io.Writer, and sets the Content-Type to text/html.
|
||||
func Render(tplName string, w http.ResponseWriter, data interface{}) (err error) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
defer func() {
|
||||
if err != nil {
|
||||
w.Header().Del("Content-Type")
|
||||
}
|
||||
}()
|
||||
return Execute(tplName, w, data)
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
package quantile
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkInsertTargeted(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
|
||||
s := NewTargeted(Targets)
|
||||
b.ResetTimer()
|
||||
for i := float64(0); i < float64(b.N); i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInsertTargetedSmallEpsilon(b *testing.B) {
|
||||
s := NewTargeted(TargetsSmallEpsilon)
|
||||
b.ResetTimer()
|
||||
for i := float64(0); i < float64(b.N); i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInsertBiased(b *testing.B) {
|
||||
s := NewLowBiased(0.01)
|
||||
b.ResetTimer()
|
||||
for i := float64(0); i < float64(b.N); i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInsertBiasedSmallEpsilon(b *testing.B) {
|
||||
s := NewLowBiased(0.0001)
|
||||
b.ResetTimer()
|
||||
for i := float64(0); i < float64(b.N); i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkQuery(b *testing.B) {
|
||||
s := NewTargeted(Targets)
|
||||
for i := float64(0); i < 1e6; i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
b.ResetTimer()
|
||||
n := float64(b.N)
|
||||
for i := float64(0); i < n; i++ {
|
||||
s.Query(i / n)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkQuerySmallEpsilon(b *testing.B) {
|
||||
s := NewTargeted(TargetsSmallEpsilon)
|
||||
for i := float64(0); i < 1e6; i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
b.ResetTimer()
|
||||
n := float64(b.N)
|
||||
for i := float64(0); i < n; i++ {
|
||||
s.Query(i / n)
|
||||
}
|
||||
}
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
// +build go1.1
|
||||
|
||||
package quantile_test
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/beorn7/perks/quantile"
|
||||
)
|
||||
|
||||
func Example_simple() {
|
||||
ch := make(chan float64)
|
||||
go sendFloats(ch)
|
||||
|
||||
// Compute the 50th, 90th, and 99th percentile.
|
||||
q := quantile.NewTargeted(map[float64]float64{
|
||||
0.50: 0.005,
|
||||
0.90: 0.001,
|
||||
0.99: 0.0001,
|
||||
})
|
||||
for v := range ch {
|
||||
q.Insert(v)
|
||||
}
|
||||
|
||||
fmt.Println("perc50:", q.Query(0.50))
|
||||
fmt.Println("perc90:", q.Query(0.90))
|
||||
fmt.Println("perc99:", q.Query(0.99))
|
||||
fmt.Println("count:", q.Count())
|
||||
// Output:
|
||||
// perc50: 5
|
||||
// perc90: 16
|
||||
// perc99: 223
|
||||
// count: 2388
|
||||
}
|
||||
|
||||
func Example_mergeMultipleStreams() {
|
||||
// Scenario:
|
||||
// We have multiple database shards. On each shard, there is a process
|
||||
// collecting query response times from the database logs and inserting
|
||||
// them into a Stream (created via NewTargeted(0.90)), much like the
|
||||
// Simple example. These processes expose a network interface for us to
|
||||
// ask them to serialize and send us the results of their
|
||||
// Stream.Samples so we may Merge and Query them.
|
||||
//
|
||||
// NOTES:
|
||||
// * These sample sets are small, allowing us to get them
|
||||
// across the network much faster than sending the entire list of data
|
||||
// points.
|
||||
//
|
||||
// * For this to work correctly, we must supply the same quantiles
|
||||
// a priori the process collecting the samples supplied to NewTargeted,
|
||||
// even if we do not plan to query them all here.
|
||||
ch := make(chan quantile.Samples)
|
||||
getDBQuerySamples(ch)
|
||||
q := quantile.NewTargeted(map[float64]float64{0.90: 0.001})
|
||||
for samples := range ch {
|
||||
q.Merge(samples)
|
||||
}
|
||||
fmt.Println("perc90:", q.Query(0.90))
|
||||
}
|
||||
|
||||
func Example_window() {
|
||||
// Scenario: We want the 90th, 95th, and 99th percentiles for each
|
||||
// minute.
|
||||
|
||||
ch := make(chan float64)
|
||||
go sendStreamValues(ch)
|
||||
|
||||
tick := time.NewTicker(1 * time.Minute)
|
||||
q := quantile.NewTargeted(map[float64]float64{
|
||||
0.90: 0.001,
|
||||
0.95: 0.0005,
|
||||
0.99: 0.0001,
|
||||
})
|
||||
for {
|
||||
select {
|
||||
case t := <-tick.C:
|
||||
flushToDB(t, q.Samples())
|
||||
q.Reset()
|
||||
case v := <-ch:
|
||||
q.Insert(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sendStreamValues(ch chan float64) {
|
||||
// Use your imagination
|
||||
}
|
||||
|
||||
func flushToDB(t time.Time, samples quantile.Samples) {
|
||||
// Use your imagination
|
||||
}
|
||||
|
||||
// This is a stub for the above example. In reality this would hit the remote
|
||||
// servers via http or something like it.
|
||||
func getDBQuerySamples(ch chan quantile.Samples) {}
|
||||
|
||||
func sendFloats(ch chan<- float64) {
|
||||
f, err := os.Open("exampledata.txt")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
b := sc.Bytes()
|
||||
v, err := strconv.ParseFloat(string(b), 64)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
ch <- v
|
||||
}
|
||||
if sc.Err() != nil {
|
||||
log.Fatal(sc.Err())
|
||||
}
|
||||
close(ch)
|
||||
}
|
||||
+2388
File diff suppressed because it is too large
Load Diff
+292
@@ -0,0 +1,292 @@
|
||||
// Package quantile computes approximate quantiles over an unbounded data
|
||||
// stream within low memory and CPU bounds.
|
||||
//
|
||||
// A small amount of accuracy is traded to achieve the above properties.
|
||||
//
|
||||
// Multiple streams can be merged before calling Query to generate a single set
|
||||
// of results. This is meaningful when the streams represent the same type of
|
||||
// data. See Merge and Samples.
|
||||
//
|
||||
// For more detailed information about the algorithm used, see:
|
||||
//
|
||||
// Effective Computation of Biased Quantiles over Data Streams
|
||||
//
|
||||
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
|
||||
package quantile
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Sample holds an observed value and meta information for compression. JSON
|
||||
// tags have been added for convenience.
|
||||
type Sample struct {
|
||||
Value float64 `json:",string"`
|
||||
Width float64 `json:",string"`
|
||||
Delta float64 `json:",string"`
|
||||
}
|
||||
|
||||
// Samples represents a slice of samples. It implements sort.Interface.
|
||||
type Samples []Sample
|
||||
|
||||
func (a Samples) Len() int { return len(a) }
|
||||
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
|
||||
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
|
||||
type invariant func(s *stream, r float64) float64
|
||||
|
||||
// NewLowBiased returns an initialized Stream for low-biased quantiles
|
||||
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
|
||||
// error guarantees can still be given even for the lower ranks of the data
|
||||
// distribution.
|
||||
//
|
||||
// The provided epsilon is a relative error, i.e. the true quantile of a value
|
||||
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
|
||||
//
|
||||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
|
||||
// properties.
|
||||
func NewLowBiased(epsilon float64) *Stream {
|
||||
ƒ := func(s *stream, r float64) float64 {
|
||||
return 2 * epsilon * r
|
||||
}
|
||||
return newStream(ƒ)
|
||||
}
|
||||
|
||||
// NewHighBiased returns an initialized Stream for high-biased quantiles
|
||||
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
|
||||
// error guarantees can still be given even for the higher ranks of the data
|
||||
// distribution.
|
||||
//
|
||||
// The provided epsilon is a relative error, i.e. the true quantile of a value
|
||||
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
|
||||
//
|
||||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
|
||||
// properties.
|
||||
func NewHighBiased(epsilon float64) *Stream {
|
||||
ƒ := func(s *stream, r float64) float64 {
|
||||
return 2 * epsilon * (s.n - r)
|
||||
}
|
||||
return newStream(ƒ)
|
||||
}
|
||||
|
||||
// NewTargeted returns an initialized Stream concerned with a particular set of
|
||||
// quantile values that are supplied a priori. Knowing these a priori reduces
|
||||
// space and computation time. The targets map maps the desired quantiles to
|
||||
// their absolute errors, i.e. the true quantile of a value returned by a query
|
||||
// is guaranteed to be within (Quantile±Epsilon).
|
||||
//
|
||||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
|
||||
func NewTargeted(targets map[float64]float64) *Stream {
|
||||
ƒ := func(s *stream, r float64) float64 {
|
||||
var m = math.MaxFloat64
|
||||
var f float64
|
||||
for quantile, epsilon := range targets {
|
||||
if quantile*s.n <= r {
|
||||
f = (2 * epsilon * r) / quantile
|
||||
} else {
|
||||
f = (2 * epsilon * (s.n - r)) / (1 - quantile)
|
||||
}
|
||||
if f < m {
|
||||
m = f
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
return newStream(ƒ)
|
||||
}
|
||||
|
||||
// Stream computes quantiles for a stream of float64s. It is not thread-safe by
|
||||
// design. Take care when using across multiple goroutines.
|
||||
type Stream struct {
|
||||
*stream
|
||||
b Samples
|
||||
sorted bool
|
||||
}
|
||||
|
||||
func newStream(ƒ invariant) *Stream {
|
||||
x := &stream{ƒ: ƒ}
|
||||
return &Stream{x, make(Samples, 0, 500), true}
|
||||
}
|
||||
|
||||
// Insert inserts v into the stream.
|
||||
func (s *Stream) Insert(v float64) {
|
||||
s.insert(Sample{Value: v, Width: 1})
|
||||
}
|
||||
|
||||
func (s *Stream) insert(sample Sample) {
|
||||
s.b = append(s.b, sample)
|
||||
s.sorted = false
|
||||
if len(s.b) == cap(s.b) {
|
||||
s.flush()
|
||||
}
|
||||
}
|
||||
|
||||
// Query returns the computed qth percentiles value. If s was created with
|
||||
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
|
||||
// will return an unspecified result.
|
||||
func (s *Stream) Query(q float64) float64 {
|
||||
if !s.flushed() {
|
||||
// Fast path when there hasn't been enough data for a flush;
|
||||
// this also yields better accuracy for small sets of data.
|
||||
l := len(s.b)
|
||||
if l == 0 {
|
||||
return 0
|
||||
}
|
||||
i := int(float64(l) * q)
|
||||
if i > 0 {
|
||||
i -= 1
|
||||
}
|
||||
s.maybeSort()
|
||||
return s.b[i].Value
|
||||
}
|
||||
s.flush()
|
||||
return s.stream.query(q)
|
||||
}
|
||||
|
||||
// Merge merges samples into the underlying streams samples. This is handy when
|
||||
// merging multiple streams from separate threads, database shards, etc.
|
||||
//
|
||||
// ATTENTION: This method is broken and does not yield correct results. The
|
||||
// underlying algorithm is not capable of merging streams correctly.
|
||||
func (s *Stream) Merge(samples Samples) {
|
||||
sort.Sort(samples)
|
||||
s.stream.merge(samples)
|
||||
}
|
||||
|
||||
// Reset reinitializes and clears the list reusing the samples buffer memory.
|
||||
func (s *Stream) Reset() {
|
||||
s.stream.reset()
|
||||
s.b = s.b[:0]
|
||||
}
|
||||
|
||||
// Samples returns stream samples held by s.
|
||||
func (s *Stream) Samples() Samples {
|
||||
if !s.flushed() {
|
||||
return s.b
|
||||
}
|
||||
s.flush()
|
||||
return s.stream.samples()
|
||||
}
|
||||
|
||||
// Count returns the total number of samples observed in the stream
|
||||
// since initialization.
|
||||
func (s *Stream) Count() int {
|
||||
return len(s.b) + s.stream.count()
|
||||
}
|
||||
|
||||
func (s *Stream) flush() {
|
||||
s.maybeSort()
|
||||
s.stream.merge(s.b)
|
||||
s.b = s.b[:0]
|
||||
}
|
||||
|
||||
func (s *Stream) maybeSort() {
|
||||
if !s.sorted {
|
||||
s.sorted = true
|
||||
sort.Sort(s.b)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stream) flushed() bool {
|
||||
return len(s.stream.l) > 0
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
n float64
|
||||
l []Sample
|
||||
ƒ invariant
|
||||
}
|
||||
|
||||
func (s *stream) reset() {
|
||||
s.l = s.l[:0]
|
||||
s.n = 0
|
||||
}
|
||||
|
||||
func (s *stream) insert(v float64) {
|
||||
s.merge(Samples{{v, 1, 0}})
|
||||
}
|
||||
|
||||
func (s *stream) merge(samples Samples) {
|
||||
// TODO(beorn7): This tries to merge not only individual samples, but
|
||||
// whole summaries. The paper doesn't mention merging summaries at
|
||||
// all. Unittests show that the merging is inaccurate. Find out how to
|
||||
// do merges properly.
|
||||
var r float64
|
||||
i := 0
|
||||
for _, sample := range samples {
|
||||
for ; i < len(s.l); i++ {
|
||||
c := s.l[i]
|
||||
if c.Value > sample.Value {
|
||||
// Insert at position i.
|
||||
s.l = append(s.l, Sample{})
|
||||
copy(s.l[i+1:], s.l[i:])
|
||||
s.l[i] = Sample{
|
||||
sample.Value,
|
||||
sample.Width,
|
||||
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
|
||||
// TODO(beorn7): How to calculate delta correctly?
|
||||
}
|
||||
i++
|
||||
goto inserted
|
||||
}
|
||||
r += c.Width
|
||||
}
|
||||
s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
|
||||
i++
|
||||
inserted:
|
||||
s.n += sample.Width
|
||||
r += sample.Width
|
||||
}
|
||||
s.compress()
|
||||
}
|
||||
|
||||
func (s *stream) count() int {
|
||||
return int(s.n)
|
||||
}
|
||||
|
||||
func (s *stream) query(q float64) float64 {
|
||||
t := math.Ceil(q * s.n)
|
||||
t += math.Ceil(s.ƒ(s, t) / 2)
|
||||
p := s.l[0]
|
||||
var r float64
|
||||
for _, c := range s.l[1:] {
|
||||
r += p.Width
|
||||
if r+c.Width+c.Delta > t {
|
||||
return p.Value
|
||||
}
|
||||
p = c
|
||||
}
|
||||
return p.Value
|
||||
}
|
||||
|
||||
func (s *stream) compress() {
|
||||
if len(s.l) < 2 {
|
||||
return
|
||||
}
|
||||
x := s.l[len(s.l)-1]
|
||||
xi := len(s.l) - 1
|
||||
r := s.n - 1 - x.Width
|
||||
|
||||
for i := len(s.l) - 2; i >= 0; i-- {
|
||||
c := s.l[i]
|
||||
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
|
||||
x.Width += c.Width
|
||||
s.l[xi] = x
|
||||
// Remove element at i.
|
||||
copy(s.l[i:], s.l[i+1:])
|
||||
s.l = s.l[:len(s.l)-1]
|
||||
xi -= 1
|
||||
} else {
|
||||
x = c
|
||||
xi = i
|
||||
}
|
||||
r -= c.Width
|
||||
}
|
||||
}
|
||||
|
||||
func (s *stream) samples() Samples {
|
||||
samples := make(Samples, len(s.l))
|
||||
copy(samples, s.l)
|
||||
return samples
|
||||
}
|
||||
+188
@@ -0,0 +1,188 @@
|
||||
package quantile
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var (
|
||||
Targets = map[float64]float64{
|
||||
0.01: 0.001,
|
||||
0.10: 0.01,
|
||||
0.50: 0.05,
|
||||
0.90: 0.01,
|
||||
0.99: 0.001,
|
||||
}
|
||||
TargetsSmallEpsilon = map[float64]float64{
|
||||
0.01: 0.0001,
|
||||
0.10: 0.001,
|
||||
0.50: 0.005,
|
||||
0.90: 0.001,
|
||||
0.99: 0.0001,
|
||||
}
|
||||
LowQuantiles = []float64{0.01, 0.1, 0.5}
|
||||
HighQuantiles = []float64{0.99, 0.9, 0.5}
|
||||
)
|
||||
|
||||
const RelativeEpsilon = 0.01
|
||||
|
||||
func verifyPercsWithAbsoluteEpsilon(t *testing.T, a []float64, s *Stream) {
|
||||
sort.Float64s(a)
|
||||
for quantile, epsilon := range Targets {
|
||||
n := float64(len(a))
|
||||
k := int(quantile * n)
|
||||
lower := int((quantile - epsilon) * n)
|
||||
if lower < 1 {
|
||||
lower = 1
|
||||
}
|
||||
upper := int(math.Ceil((quantile + epsilon) * n))
|
||||
if upper > len(a) {
|
||||
upper = len(a)
|
||||
}
|
||||
w, min, max := a[k-1], a[lower-1], a[upper-1]
|
||||
if g := s.Query(quantile); g < min || g > max {
|
||||
t.Errorf("q=%f: want %v [%f,%f], got %v", quantile, w, min, max, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyLowPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
|
||||
sort.Float64s(a)
|
||||
for _, qu := range LowQuantiles {
|
||||
n := float64(len(a))
|
||||
k := int(qu * n)
|
||||
|
||||
lowerRank := int((1 - RelativeEpsilon) * qu * n)
|
||||
upperRank := int(math.Ceil((1 + RelativeEpsilon) * qu * n))
|
||||
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
|
||||
if g := s.Query(qu); g < min || g > max {
|
||||
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyHighPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
|
||||
sort.Float64s(a)
|
||||
for _, qu := range HighQuantiles {
|
||||
n := float64(len(a))
|
||||
k := int(qu * n)
|
||||
|
||||
lowerRank := int((1 - (1+RelativeEpsilon)*(1-qu)) * n)
|
||||
upperRank := int(math.Ceil((1 - (1-RelativeEpsilon)*(1-qu)) * n))
|
||||
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
|
||||
if g := s.Query(qu); g < min || g > max {
|
||||
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func populateStream(s *Stream) []float64 {
|
||||
a := make([]float64, 0, 1e5+100)
|
||||
for i := 0; i < cap(a); i++ {
|
||||
v := rand.NormFloat64()
|
||||
// Add 5% asymmetric outliers.
|
||||
if i%20 == 0 {
|
||||
v = v*v + 1
|
||||
}
|
||||
s.Insert(v)
|
||||
a = append(a, v)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func TestTargetedQuery(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s := NewTargeted(Targets)
|
||||
a := populateStream(s)
|
||||
verifyPercsWithAbsoluteEpsilon(t, a, s)
|
||||
}
|
||||
|
||||
func TestLowBiasedQuery(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s := NewLowBiased(RelativeEpsilon)
|
||||
a := populateStream(s)
|
||||
verifyLowPercsWithRelativeEpsilon(t, a, s)
|
||||
}
|
||||
|
||||
func TestHighBiasedQuery(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s := NewHighBiased(RelativeEpsilon)
|
||||
a := populateStream(s)
|
||||
verifyHighPercsWithRelativeEpsilon(t, a, s)
|
||||
}
|
||||
|
||||
// BrokenTestTargetedMerge is broken, see Merge doc comment.
|
||||
func BrokenTestTargetedMerge(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s1 := NewTargeted(Targets)
|
||||
s2 := NewTargeted(Targets)
|
||||
a := populateStream(s1)
|
||||
a = append(a, populateStream(s2)...)
|
||||
s1.Merge(s2.Samples())
|
||||
verifyPercsWithAbsoluteEpsilon(t, a, s1)
|
||||
}
|
||||
|
||||
// BrokenTestLowBiasedMerge is broken, see Merge doc comment.
|
||||
func BrokenTestLowBiasedMerge(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s1 := NewLowBiased(RelativeEpsilon)
|
||||
s2 := NewLowBiased(RelativeEpsilon)
|
||||
a := populateStream(s1)
|
||||
a = append(a, populateStream(s2)...)
|
||||
s1.Merge(s2.Samples())
|
||||
verifyLowPercsWithRelativeEpsilon(t, a, s2)
|
||||
}
|
||||
|
||||
// BrokenTestHighBiasedMerge is broken, see Merge doc comment.
|
||||
func BrokenTestHighBiasedMerge(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s1 := NewHighBiased(RelativeEpsilon)
|
||||
s2 := NewHighBiased(RelativeEpsilon)
|
||||
a := populateStream(s1)
|
||||
a = append(a, populateStream(s2)...)
|
||||
s1.Merge(s2.Samples())
|
||||
verifyHighPercsWithRelativeEpsilon(t, a, s2)
|
||||
}
|
||||
|
||||
func TestUncompressed(t *testing.T) {
|
||||
q := NewTargeted(Targets)
|
||||
for i := 100; i > 0; i-- {
|
||||
q.Insert(float64(i))
|
||||
}
|
||||
if g := q.Count(); g != 100 {
|
||||
t.Errorf("want count 100, got %d", g)
|
||||
}
|
||||
// Before compression, Query should have 100% accuracy.
|
||||
for quantile := range Targets {
|
||||
w := quantile * 100
|
||||
if g := q.Query(quantile); g != w {
|
||||
t.Errorf("want %f, got %f", w, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUncompressedSamples(t *testing.T) {
|
||||
q := NewTargeted(map[float64]float64{0.99: 0.001})
|
||||
for i := 1; i <= 100; i++ {
|
||||
q.Insert(float64(i))
|
||||
}
|
||||
if g := q.Samples().Len(); g != 100 {
|
||||
t.Errorf("want count 100, got %d", g)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUncompressedOne(t *testing.T) {
|
||||
q := NewTargeted(map[float64]float64{0.99: 0.01})
|
||||
q.Insert(3.14)
|
||||
if g := q.Query(0.90); g != 3.14 {
|
||||
t.Error("want PI, got", g)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaults(t *testing.T) {
|
||||
if g := NewTargeted(map[float64]float64{0.99: 0.001}).Query(0.99); g != 0 {
|
||||
t.Errorf("want 0, got %f", g)
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015 Jun Kimura
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
# GCache [](https://app.wercker.com/project/bykey/1471b6c9cbc9ebbd15f8f9fe8f71ac67)[](https://godoc.org/github.com/bluele/gcache)
|
||||
|
||||
Cache library for golang. It supports expirable Cache, LFU, LRU and ARC.
|
||||
|
||||
## Features
|
||||
|
||||
* Supports expirable Cache, LFU, LRU and ARC.
|
||||
|
||||
* Goroutine safe.
|
||||
|
||||
* Supports event handlers which evict and add entry. (Optional)
|
||||
|
||||
* Automatically load cache if it doesn't exists. (Optional)
|
||||
|
||||
## Install
|
||||
|
||||
```
|
||||
$ go get github.com/bluele/gcache
|
||||
```
|
||||
|
||||
## Example
|
||||
|
||||
### Manually set a key-value pair.
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/bluele/gcache"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
gc := gcache.New(20).
|
||||
LRU().
|
||||
Build()
|
||||
gc.Set("key", "ok")
|
||||
value, err := gc.Get("key")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println("Get:", value)
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
Get: ok
|
||||
```
|
||||
|
||||
### Automatically load value
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/bluele/gcache"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
gc := gcache.New(20).
|
||||
LRU().
|
||||
LoaderFunc(func(key interface{}) (interface{}, error) {
|
||||
return "ok", nil
|
||||
}).
|
||||
Build()
|
||||
value, err := gc.Get("key")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println("Get:", value)
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
Get: ok
|
||||
```
|
||||
|
||||
## Cache Algorithm
|
||||
|
||||
* Least-Frequently Used (LFU)
|
||||
|
||||
Discards the least frequently used items first.
|
||||
|
||||
```go
|
||||
func main() {
|
||||
// size: 10
|
||||
gc := gcache.New(10).
|
||||
LFU().
|
||||
Build()
|
||||
gc.Set("key", "value")
|
||||
}
|
||||
```
|
||||
|
||||
* Least Recently Used (LRU)
|
||||
|
||||
Discards the least recently used items first.
|
||||
|
||||
```go
|
||||
func main() {
|
||||
// size: 10
|
||||
gc := gcache.New(10).
|
||||
LRU().
|
||||
Build()
|
||||
gc.Set("key", "value")
|
||||
}
|
||||
```
|
||||
|
||||
* Adaptive Replacement Cache (ARC)
|
||||
|
||||
Constantly balances between LRU and LFU, to improve the combined result.
|
||||
|
||||
detail: http://en.wikipedia.org/wiki/Adaptive_replacement_cache
|
||||
|
||||
```go
|
||||
func main() {
|
||||
// size: 10
|
||||
gc := gcache.New(10).
|
||||
ARC().
|
||||
Build()
|
||||
gc.Set("key", "value")
|
||||
}
|
||||
```
|
||||
|
||||
* SimpleCache (Default)
|
||||
|
||||
SimpleCache has no clear priority for evict cache. It depends on key-value map order.
|
||||
|
||||
```go
|
||||
func main() {
|
||||
// size: 10
|
||||
gc := gcache.New(10).Build()
|
||||
gc.Set("key", "value")
|
||||
v, err := gc.Get("key")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Loading Cache
|
||||
|
||||
If specified `LoaderFunc`, values are automatically loaded by the cache, and are stored in the cache until either evicted or manually invalidated.
|
||||
|
||||
```go
|
||||
func main() {
|
||||
gc := gcache.New(10).
|
||||
LRU().
|
||||
LoaderFunc(func(key interface{}) (interface{}, error) {
|
||||
return "value", nil
|
||||
}).
|
||||
Build()
|
||||
v, _ := gc.Get("key")
|
||||
// output: "value"
|
||||
fmt.Println(v)
|
||||
}
|
||||
```
|
||||
|
||||
GCache coordinates cache fills such that only one load in one process of an entire replicated set of processes populates the cache, then multiplexes the loaded value to all callers.
|
||||
|
||||
## Expirable cache
|
||||
|
||||
```go
|
||||
func main() {
|
||||
// LRU cache, size: 10, expiration: after a hour
|
||||
gc := gcache.New(10).
|
||||
LRU().
|
||||
Expiration(time.Hour).
|
||||
Build()
|
||||
}
|
||||
```
|
||||
|
||||
## Event handlers
|
||||
|
||||
### Evicted handler
|
||||
|
||||
Event handler for evict the entry.
|
||||
|
||||
```go
|
||||
func main() {
|
||||
gc := gcache.New(2).
|
||||
EvictedFunc(func(key, value interface{}) {
|
||||
fmt.Println("evicted key:", key)
|
||||
}).
|
||||
Build()
|
||||
for i := 0; i < 3; i++ {
|
||||
gc.Set(i, i*i)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
evicted key: 0
|
||||
```
|
||||
|
||||
### Added handler
|
||||
|
||||
Event handler for add the entry.
|
||||
|
||||
```go
|
||||
func main() {
|
||||
gc := gcache.New(2).
|
||||
AddedFunc(func(key, value interface{}) {
|
||||
fmt.Println("added key:", key)
|
||||
}).
|
||||
Build()
|
||||
for i := 0; i < 3; i++ {
|
||||
gc.Set(i, i*i)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
added key: 0
|
||||
added key: 1
|
||||
added key: 2
|
||||
```
|
||||
|
||||
# Author
|
||||
|
||||
**Jun Kimura**
|
||||
|
||||
* <http://github.com/bluele>
|
||||
* <junkxdev@gmail.com>
|
||||
+331
@@ -0,0 +1,331 @@
|
||||
package gcache
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Constantly balances between LRU and LFU, to improve the combined result.
|
||||
type ARC struct {
|
||||
baseCache
|
||||
items map[interface{}]*arcItem
|
||||
|
||||
part int
|
||||
t1 *arcList
|
||||
t2 *arcList
|
||||
b1 *arcList
|
||||
b2 *arcList
|
||||
}
|
||||
|
||||
func newARC(cb *CacheBuilder) *ARC {
|
||||
c := &ARC{
|
||||
items: make(map[interface{}]*arcItem),
|
||||
t1: newARCList(),
|
||||
t2: newARCList(),
|
||||
b1: newARCList(),
|
||||
b2: newARCList(),
|
||||
}
|
||||
buildCache(&c.baseCache, cb)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *ARC) replace(key interface{}) {
|
||||
var old interface{}
|
||||
if (c.t1.Len() > 0 && c.b2.Has(key) && c.t1.Len() == c.part) || (c.t1.Len() > c.part) {
|
||||
old = c.t1.RemoveTail()
|
||||
c.b1.PushFront(old)
|
||||
} else {
|
||||
old = c.t2.RemoveTail()
|
||||
c.b2.PushFront(old)
|
||||
}
|
||||
item, ok := c.items[old]
|
||||
if ok {
|
||||
delete(c.items, old)
|
||||
if c.evictedFunc != nil {
|
||||
go (*c.evictedFunc)(item.key, item.value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ARC) Set(key, value interface{}) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.set(key, value)
|
||||
}
|
||||
|
||||
func (c *ARC) set(key, value interface{}) (interface{}, error) {
|
||||
item, ok := c.items[key]
|
||||
if ok {
|
||||
item.value = value
|
||||
} else {
|
||||
item = &arcItem{
|
||||
key: key,
|
||||
value: value,
|
||||
}
|
||||
c.items[key] = item
|
||||
}
|
||||
|
||||
if c.expiration != nil {
|
||||
t := time.Now().Add(*c.expiration)
|
||||
item.expiration = &t
|
||||
}
|
||||
|
||||
if elt := c.b1.Lookup(key); elt != nil {
|
||||
c.part = minInt(c.size, c.part+maxInt(c.b2.Len()/c.b1.Len(), 1))
|
||||
c.replace(key)
|
||||
c.b1.Remove(key, elt)
|
||||
c.t2.PushFront(key)
|
||||
return item, nil
|
||||
}
|
||||
|
||||
if elt := c.b2.Lookup(key); elt != nil {
|
||||
c.part = maxInt(0, c.part-maxInt(c.b1.Len()/c.b2.Len(), 1))
|
||||
c.replace(key)
|
||||
c.b2.Remove(key, elt)
|
||||
c.t2.PushFront(key)
|
||||
return item, nil
|
||||
}
|
||||
|
||||
if c.t1.Len()+c.b1.Len() == c.size {
|
||||
if c.t1.Len() < c.size {
|
||||
c.b1.RemoveTail()
|
||||
c.replace(key)
|
||||
} else {
|
||||
pop := c.t1.RemoveTail()
|
||||
item, ok := c.items[pop]
|
||||
if ok {
|
||||
delete(c.items, pop)
|
||||
if c.evictedFunc != nil {
|
||||
go (*c.evictedFunc)(item.key, item.value)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
total := c.t1.Len() + c.b1.Len() + c.t2.Len() + c.b2.Len()
|
||||
if total >= c.size {
|
||||
if total == (2 * c.size) {
|
||||
c.b2.RemoveTail()
|
||||
}
|
||||
c.replace(key)
|
||||
}
|
||||
}
|
||||
|
||||
c.t1.PushFront(key)
|
||||
|
||||
if c.addedFunc != nil {
|
||||
go (*c.addedFunc)(key, value)
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// Get a value from cache pool using key if it exists. If not exists and it has LoaderFunc, it will generate the value using you have specified LoaderFunc method returns value.
|
||||
func (c *ARC) Get(key interface{}) (interface{}, error) {
|
||||
rl := false
|
||||
c.mu.RLock()
|
||||
if elt := c.t1.Lookup(key); elt != nil {
|
||||
c.mu.RUnlock()
|
||||
rl = true
|
||||
c.mu.Lock()
|
||||
c.t1.Remove(key, elt)
|
||||
item := c.items[key]
|
||||
if !item.IsExpired(nil) {
|
||||
c.t2.PushFront(key)
|
||||
c.mu.Unlock()
|
||||
return item.value, nil
|
||||
}
|
||||
c.b2.PushFront(key)
|
||||
if c.evictedFunc != nil {
|
||||
go (*c.evictedFunc)(key, elt.Value)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
if elt := c.t2.Lookup(key); elt != nil {
|
||||
c.mu.RUnlock()
|
||||
rl = true
|
||||
c.mu.Lock()
|
||||
item := c.items[key]
|
||||
if !item.IsExpired(nil) {
|
||||
c.t2.MoveToFront(elt)
|
||||
c.mu.Unlock()
|
||||
return item.value, nil
|
||||
}
|
||||
c.t2.Remove(key, elt)
|
||||
c.b2.PushFront(key)
|
||||
if c.evictedFunc != nil {
|
||||
go (*c.evictedFunc)(key, elt.Value)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
if !rl {
|
||||
c.mu.RUnlock()
|
||||
}
|
||||
|
||||
if c.loaderFunc == nil {
|
||||
return nil, NotFoundKeyError
|
||||
}
|
||||
|
||||
item, err := c.load(key, func(v interface{}, e error) (interface{}, error) {
|
||||
if e == nil {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.set(key, v)
|
||||
}
|
||||
return nil, e
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return item.(*arcItem).value, nil
|
||||
}
|
||||
|
||||
// Remove removes the provided key from the cache.
|
||||
func (c *ARC) Remove(key interface{}) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.remove(key)
|
||||
}
|
||||
|
||||
func (c *ARC) remove(key interface{}) bool {
|
||||
if elt := c.t1.Lookup(key); elt != nil {
|
||||
v := elt.Value.(*arcItem).value
|
||||
c.t1.Remove(key, elt)
|
||||
if c.evictedFunc != nil {
|
||||
go (*c.evictedFunc)(key, v)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
if elt := c.t2.Lookup(key); elt != nil {
|
||||
v := elt.Value.(*arcItem).value
|
||||
c.t2.Remove(key, elt)
|
||||
if c.evictedFunc != nil {
|
||||
go (*c.evictedFunc)(key, v)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Keys returns a slice of the keys in the cache.
|
||||
func (c *ARC) Keys() []interface{} {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
keys := []interface{}{}
|
||||
for key := range c.items {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Len returns the number of items in the cache.
|
||||
func (c *ARC) Len() int {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
return len(c.items)
|
||||
}
|
||||
|
||||
// Purge is used to completely clear the cache
|
||||
func (c *ARC) Purge() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.items = make(map[interface{}]*arcItem)
|
||||
c.t1 = newARCList()
|
||||
c.t2 = newARCList()
|
||||
c.b1 = newARCList()
|
||||
c.b2 = newARCList()
|
||||
}
|
||||
|
||||
func (c *ARC) gc() {
|
||||
now := time.Now()
|
||||
keys := []interface{}{}
|
||||
c.mu.RLock()
|
||||
for k, item := range c.items {
|
||||
if item.IsExpired(&now) {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
c.mu.RUnlock()
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
for _, k := range keys {
|
||||
c.remove(k)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// returns boolean value whether this item is expired or not.
|
||||
func (it *arcItem) IsExpired(now *time.Time) bool {
|
||||
if it.expiration == nil {
|
||||
return false
|
||||
}
|
||||
if now == nil {
|
||||
t := time.Now()
|
||||
now = &t
|
||||
}
|
||||
return it.expiration.Before(*now)
|
||||
}
|
||||
|
||||
type arcList struct {
|
||||
l *list.List
|
||||
keys map[interface{}]*list.Element
|
||||
}
|
||||
|
||||
type arcItem struct {
|
||||
key interface{}
|
||||
value interface{}
|
||||
expiration *time.Time
|
||||
}
|
||||
|
||||
func newARCList() *arcList {
|
||||
return &arcList{
|
||||
l: list.New(),
|
||||
keys: make(map[interface{}]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
func (al *arcList) Has(key interface{}) bool {
|
||||
_, ok := al.keys[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (al *arcList) Lookup(key interface{}) *list.Element {
|
||||
elt := al.keys[key]
|
||||
return elt
|
||||
}
|
||||
|
||||
func (al *arcList) MoveToFront(elt *list.Element) {
|
||||
al.l.MoveToFront(elt)
|
||||
}
|
||||
|
||||
func (al *arcList) PushFront(key interface{}) {
|
||||
elt := al.l.PushFront(key)
|
||||
al.keys[key] = elt
|
||||
}
|
||||
|
||||
func (al *arcList) Remove(key interface{}, elt *list.Element) {
|
||||
delete(al.keys, key)
|
||||
al.l.Remove(elt)
|
||||
}
|
||||
|
||||
func (al *arcList) RemoveTail() interface{} {
|
||||
elt := al.l.Back()
|
||||
al.l.Remove(elt)
|
||||
|
||||
key := elt.Value
|
||||
delete(al.keys, key)
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func (al *arcList) Len() int {
|
||||
return al.l.Len()
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
package gcache_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/bluele/gcache"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func buildARCache(size int) gcache.Cache {
|
||||
return gcache.New(size).
|
||||
ARC().
|
||||
EvictedFunc(evictedFuncForARC).
|
||||
Build()
|
||||
}
|
||||
|
||||
func buildLoadingARCache(size int) gcache.Cache {
|
||||
return gcache.New(size).
|
||||
ARC().
|
||||
LoaderFunc(loader).
|
||||
EvictedFunc(evictedFuncForARC).
|
||||
Build()
|
||||
}
|
||||
|
||||
func evictedFuncForARC(key, value interface{}) {
|
||||
fmt.Printf("[ARC] Key:%v Value:%v will evicted.\n", key, value)
|
||||
}
|
||||
|
||||
func TestARCGet(t *testing.T) {
|
||||
size := 1000
|
||||
gc := buildARCache(size)
|
||||
testSetCache(t, gc, size)
|
||||
testGetCache(t, gc, size)
|
||||
}
|
||||
|
||||
func TestLoadingARCGet(t *testing.T) {
|
||||
size := 1000
|
||||
numbers := 1000
|
||||
testGetCache(t, buildLoadingARCache(size), numbers)
|
||||
}
|
||||
|
||||
func TestARCLength(t *testing.T) {
|
||||
gc := buildLoadingARCache(1000)
|
||||
gc.Get("test1")
|
||||
gc.Get("test2")
|
||||
length := gc.Len()
|
||||
expectedLength := 2
|
||||
if gc.Len() != expectedLength {
|
||||
t.Errorf("Expected length is %v, not %v", length, expectedLength)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARCEvictItem(t *testing.T) {
|
||||
cacheSize := 10
|
||||
numbers := 11
|
||||
gc := buildLoadingARCache(cacheSize)
|
||||
|
||||
for i := 0; i < numbers; i++ {
|
||||
_, err := gc.Get(fmt.Sprintf("Key-%d", i))
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
package gcache
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/bluele/gcache/singleflight"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
TYPE_SIMPLE = "simple"
|
||||
TYPE_LRU = "lru"
|
||||
TYPE_LFU = "lfu"
|
||||
TYPE_ARC = "arc"
|
||||
)
|
||||
|
||||
var NotFoundKeyError = errors.New("Not found error.")
|
||||
|
||||
type Cache interface {
|
||||
Set(interface{}, interface{})
|
||||
Get(interface{}) (interface{}, error)
|
||||
Remove(interface{}) bool
|
||||
Purge()
|
||||
Keys() []interface{}
|
||||
Len() int
|
||||
gc()
|
||||
}
|
||||
|
||||
type baseCache struct {
|
||||
size int
|
||||
loaderFunc *LoaderFunc
|
||||
evictedFunc *EvictedFunc
|
||||
addedFunc *AddedFunc
|
||||
expiration *time.Duration
|
||||
mu sync.RWMutex
|
||||
loadGroup singleflight.Group
|
||||
}
|
||||
|
||||
type LoaderFunc func(interface{}) (interface{}, error)
|
||||
|
||||
type EvictedFunc func(interface{}, interface{})
|
||||
|
||||
type AddedFunc func(interface{}, interface{})
|
||||
|
||||
type CacheBuilder struct {
|
||||
tp string
|
||||
size int
|
||||
loaderFunc *LoaderFunc
|
||||
evictedFunc *EvictedFunc
|
||||
addedFunc *AddedFunc
|
||||
expiration *time.Duration
|
||||
gcInterval *time.Duration
|
||||
}
|
||||
|
||||
func New(size int) *CacheBuilder {
|
||||
if size <= 0 {
|
||||
panic("gcache: size <= 0")
|
||||
}
|
||||
return &CacheBuilder{
|
||||
tp: TYPE_SIMPLE,
|
||||
size: size,
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) LoaderFunc(loaderFunc LoaderFunc) *CacheBuilder {
|
||||
cb.loaderFunc = &loaderFunc
|
||||
return cb
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) EnableGC(interval time.Duration) *CacheBuilder {
|
||||
cb.gcInterval = &interval
|
||||
return cb
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) EvictType(tp string) *CacheBuilder {
|
||||
cb.tp = tp
|
||||
return cb
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) Simple() *CacheBuilder {
|
||||
return cb.EvictType(TYPE_SIMPLE)
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) LRU() *CacheBuilder {
|
||||
return cb.EvictType(TYPE_LRU)
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) LFU() *CacheBuilder {
|
||||
return cb.EvictType(TYPE_LFU)
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) ARC() *CacheBuilder {
|
||||
return cb.EvictType(TYPE_ARC)
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) EvictedFunc(evictedFunc EvictedFunc) *CacheBuilder {
|
||||
cb.evictedFunc = &evictedFunc
|
||||
return cb
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) AddedFunc(addedFunc AddedFunc) *CacheBuilder {
|
||||
cb.addedFunc = &addedFunc
|
||||
return cb
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) Expiration(expiration time.Duration) *CacheBuilder {
|
||||
cb.expiration = &expiration
|
||||
return cb
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) Build() Cache {
|
||||
cache := cb.build()
|
||||
if cb.gcInterval != nil {
|
||||
go func() {
|
||||
t := time.NewTicker(*cb.gcInterval)
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
go cache.gc()
|
||||
}
|
||||
}
|
||||
t.Stop()
|
||||
}()
|
||||
}
|
||||
return cache
|
||||
}
|
||||
|
||||
func (cb *CacheBuilder) build() Cache {
|
||||
switch cb.tp {
|
||||
case TYPE_SIMPLE:
|
||||
return newSimpleCache(cb)
|
||||
case TYPE_LRU:
|
||||
return newLRUCache(cb)
|
||||
case TYPE_LFU:
|
||||
return newLFUCache(cb)
|
||||
case TYPE_ARC:
|
||||
return newARC(cb)
|
||||
default:
|
||||
panic("gcache: Unknown type " + cb.tp)
|
||||
}
|
||||
}
|
||||
|
||||
func buildCache(c *baseCache, cb *CacheBuilder) {
|
||||
c.size = cb.size
|
||||
c.loaderFunc = cb.loaderFunc
|
||||
c.expiration = cb.expiration
|
||||
c.addedFunc = cb.addedFunc
|
||||
c.evictedFunc = cb.evictedFunc
|
||||
}
|
||||
|
||||
// load a new value using by specified key.
|
||||
func (c *baseCache) load(key interface{}, cb func(interface{}, error) (interface{}, error)) (interface{}, error) {
|
||||
v, err := c.loadGroup.Do(key, func() (interface{}, error) {
|
||||
return cb((*c.loaderFunc)(key))
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/bluele/gcache"
|
||||
)
|
||||
|
||||
func main() {
|
||||
gc := gcache.New(10).
|
||||
LFU().
|
||||
LoaderFunc(func(key interface{}) (interface{}, error) {
|
||||
return fmt.Sprintf("%v-value", key), nil
|
||||
}).
|
||||
Build()
|
||||
|
||||
v, err := gc.Get("key")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(v)
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/bluele/gcache"
|
||||
)
|
||||
|
||||
func main() {
|
||||
gc := gcache.New(10).
|
||||
LFU().
|
||||
Build()
|
||||
gc.Set("key", "ok")
|
||||
|
||||
v, err := gc.Get("key")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println("value:", v)
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package gcache_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/bluele/gcache"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func loader(key interface{}) (interface{}, error) {
|
||||
return fmt.Sprintf("valueFor%s", key), nil
|
||||
}
|
||||
|
||||
func testSetCache(t *testing.T, gc gcache.Cache, numbers int) {
|
||||
for i := 0; i < numbers; i++ {
|
||||
key := fmt.Sprintf("Key-%d", i)
|
||||
value, err := loader(key)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
gc.Set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
func testGetCache(t *testing.T, gc gcache.Cache, numbers int) {
|
||||
for i := 0; i < numbers; i++ {
|
||||
key := fmt.Sprintf("Key-%d", i)
|
||||
v, err := gc.Get(key)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
expectedV, _ := loader(key)
|
||||
if v != expectedV {
|
||||
t.Errorf("Expected value is %v, not %v", expectedV, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
package gcache
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Discards the least frequently used items first.
|
||||
type LFUCache struct {
|
||||
baseCache
|
||||
items map[interface{}]*lfuItem
|
||||
freqList *list.List // list for freqEntry
|
||||
}
|
||||
|
||||
func newLFUCache(cb *CacheBuilder) *LFUCache {
|
||||
c := &LFUCache{}
|
||||
buildCache(&c.baseCache, cb)
|
||||
|
||||
c.freqList = list.New()
|
||||
c.items = make(map[interface{}]*lfuItem, c.size+1)
|
||||
c.freqList.PushFront(&freqEntry{
|
||||
freq: 0,
|
||||
items: make(map[*lfuItem]byte),
|
||||
})
|
||||
return c
|
||||
}
|
||||
|
||||
// set a new key-value pair
|
||||
func (c *LFUCache) Set(key, value interface{}) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.set(key, value)
|
||||
}
|
||||
|
||||
func (c *LFUCache) set(key, value interface{}) (*lfuItem, error) {
|
||||
// Check for existing item
|
||||
item, ok := c.items[key]
|
||||
if ok {
|
||||
item.value = value
|
||||
} else {
|
||||
// Verify size not exceeded
|
||||
if len(c.items) >= c.size {
|
||||
c.evict(1)
|
||||
}
|
||||
item = &lfuItem{
|
||||
key: key,
|
||||
value: value,
|
||||
freqElement: nil,
|
||||
}
|
||||
el := c.freqList.Front()
|
||||
fe := el.Value.(*freqEntry)
|
||||
fe.items[item] = 1
|
||||
|
||||
item.freqElement = el
|
||||
c.items[key] = item
|
||||
}
|
||||
|
||||
if c.expiration != nil {
|
||||
t := time.Now().Add(*c.expiration)
|
||||
item.expiration = &t
|
||||
}
|
||||
|
||||
if c.addedFunc != nil {
|
||||
go (*c.addedFunc)(key, value)
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// Get a value from cache pool using key if it exists.
|
||||
// If it dose not exists key and has LoaderFunc,
|
||||
// generate a value using `LoaderFunc` method returns value.
|
||||
func (c *LFUCache) Get(key interface{}) (interface{}, error) {
|
||||
c.mu.RLock()
|
||||
item, ok := c.items[key]
|
||||
c.mu.RUnlock()
|
||||
|
||||
if ok {
|
||||
if !item.IsExpired(nil) {
|
||||
c.mu.Lock()
|
||||
c.increment(item)
|
||||
c.mu.Unlock()
|
||||
return item.value, nil
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.removeItem(item)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
if c.loaderFunc == nil {
|
||||
return nil, NotFoundKeyError
|
||||
}
|
||||
|
||||
it, err := c.load(key, func(v interface{}, e error) (interface{}, error) {
|
||||
if e == nil {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.set(key, v)
|
||||
}
|
||||
return nil, e
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
li := it.(*lfuItem)
|
||||
c.increment(li)
|
||||
return li.value, nil
|
||||
}
|
||||
|
||||
func (c *LFUCache) increment(item *lfuItem) {
|
||||
currentFreqElement := item.freqElement
|
||||
currentFreqEntry := currentFreqElement.Value.(*freqEntry)
|
||||
nextFreq := currentFreqEntry.freq + 1
|
||||
delete(currentFreqEntry.items, item)
|
||||
|
||||
nextFreqElement := currentFreqElement.Next()
|
||||
if nextFreqElement == nil {
|
||||
nextFreqElement = c.freqList.InsertAfter(&freqEntry{
|
||||
freq: nextFreq,
|
||||
items: make(map[*lfuItem]byte),
|
||||
}, currentFreqElement)
|
||||
}
|
||||
nextFreqElement.Value.(*freqEntry).items[item] = 1
|
||||
item.freqElement = nextFreqElement
|
||||
}
|
||||
|
||||
// evict removes the least frequence item from the cache.
|
||||
func (c *LFUCache) evict(count int) {
|
||||
entry := c.freqList.Front()
|
||||
for i := 0; i < count; {
|
||||
if entry == nil {
|
||||
return
|
||||
} else {
|
||||
for item, _ := range entry.Value.(*freqEntry).items {
|
||||
if i >= count {
|
||||
return
|
||||
}
|
||||
c.removeItem(item)
|
||||
i++
|
||||
}
|
||||
entry = entry.Next()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Removes the provided key from the cache.
|
||||
func (c *LFUCache) Remove(key interface{}) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.remove(key)
|
||||
}
|
||||
|
||||
func (c *LFUCache) remove(key interface{}) bool {
|
||||
if item, ok := c.items[key]; ok {
|
||||
c.removeItem(item)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// removeElement is used to remove a given list element from the cache
|
||||
func (c *LFUCache) removeItem(item *lfuItem) {
|
||||
delete(c.items, item.key)
|
||||
delete(item.freqElement.Value.(*freqEntry).items, item)
|
||||
if c.evictedFunc != nil {
|
||||
go (*c.evictedFunc)(item.key, item.value)
|
||||
}
|
||||
}
|
||||
|
||||
// Returns a slice of the keys in the cache.
|
||||
func (c *LFUCache) Keys() []interface{} {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
keys := make([]interface{}, len(c.items))
|
||||
i := 0
|
||||
for k := range c.items {
|
||||
keys[i] = k
|
||||
i++
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
// Returns the number of items in the cache.
|
||||
func (c *LFUCache) Len() int {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
return len(c.items)
|
||||
}
|
||||
|
||||
// Completely clear the cache
|
||||
func (c *LFUCache) Purge() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.freqList = list.New()
|
||||
c.items = make(map[interface{}]*lfuItem, c.size)
|
||||
}
|
||||
|
||||
// evict all expired entry
|
||||
func (c *LFUCache) gc() {
|
||||
now := time.Now()
|
||||
keys := []interface{}{}
|
||||
c.mu.RLock()
|
||||
for k, item := range c.items {
|
||||
if item.IsExpired(&now) {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
c.mu.RUnlock()
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
for _, k := range keys {
|
||||
c.remove(k)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
type freqEntry struct {
|
||||
freq uint
|
||||
items map[*lfuItem]byte
|
||||
}
|
||||
|
||||
type lfuItem struct {
|
||||
key interface{}
|
||||
value interface{}
|
||||
freqElement *list.Element
|
||||
expiration *time.Time
|
||||
}
|
||||
|
||||
// returns boolean value whether this item is expired or not.
|
||||
func (it *lfuItem) IsExpired(now *time.Time) bool {
|
||||
if it.expiration == nil {
|
||||
return false
|
||||
}
|
||||
if now == nil {
|
||||
t := time.Now()
|
||||
now = &t
|
||||
}
|
||||
return it.expiration.Before(*now)
|
||||
}
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
package gcache_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/bluele/gcache"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func evictedFuncForLFU(key, value interface{}) {
|
||||
fmt.Printf("[LFU] Key:%v Value:%v will evicted.\n", key, value)
|
||||
}
|
||||
|
||||
func buildLFUCache(size int) gcache.Cache {
|
||||
return gcache.New(size).
|
||||
LFU().
|
||||
EvictedFunc(evictedFuncForLFU).
|
||||
Expiration(time.Second).
|
||||
Build()
|
||||
}
|
||||
|
||||
func buildLoadingLFUCache(size int, loader gcache.LoaderFunc) gcache.Cache {
|
||||
return gcache.New(size).
|
||||
LFU().
|
||||
LoaderFunc(loader).
|
||||
EvictedFunc(evictedFuncForLFU).
|
||||
Expiration(time.Second).
|
||||
Build()
|
||||
}
|
||||
|
||||
func TestLFUGet(t *testing.T) {
|
||||
size := 1000
|
||||
numbers := 1000
|
||||
|
||||
gc := buildLoadingLFUCache(size, loader)
|
||||
testSetCache(t, gc, numbers)
|
||||
testGetCache(t, gc, numbers)
|
||||
}
|
||||
|
||||
func TestLoadingLFUGet(t *testing.T) {
|
||||
size := 1000
|
||||
numbers := 1000
|
||||
|
||||
gc := buildLoadingLFUCache(size, loader)
|
||||
testGetCache(t, gc, numbers)
|
||||
}
|
||||
|
||||
func TestLFULength(t *testing.T) {
|
||||
gc := buildLoadingLFUCache(1000, loader)
|
||||
gc.Get("test1")
|
||||
gc.Get("test2")
|
||||
length := gc.Len()
|
||||
expectedLength := 2
|
||||
if gc.Len() != expectedLength {
|
||||
t.Errorf("Expected length is %v, not %v", length, expectedLength)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLFUEvictItem(t *testing.T) {
|
||||
cacheSize := 10
|
||||
numbers := 11
|
||||
gc := buildLoadingLFUCache(cacheSize, loader)
|
||||
|
||||
for i := 0; i < numbers; i++ {
|
||||
_, err := gc.Get(fmt.Sprintf("Key-%d", i))
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
package gcache
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Discards the least recently used items first.
|
||||
type LRUCache struct {
|
||||
baseCache
|
||||
items map[interface{}]*list.Element
|
||||
evictList *list.List
|
||||
}
|
||||
|
||||
func newLRUCache(cb *CacheBuilder) *LRUCache {
|
||||
c := &LRUCache{}
|
||||
buildCache(&c.baseCache, cb)
|
||||
|
||||
c.evictList = list.New()
|
||||
c.items = make(map[interface{}]*list.Element, c.size+1)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *LRUCache) set(key, value interface{}) (interface{}, error) {
|
||||
// Check for existing item
|
||||
var item *lruItem
|
||||
if it, ok := c.items[key]; ok {
|
||||
c.evictList.MoveToFront(it)
|
||||
item = it.Value.(*lruItem)
|
||||
item.value = value
|
||||
} else {
|
||||
// Verify size not exceeded
|
||||
if c.evictList.Len() >= c.size {
|
||||
c.evict(1)
|
||||
}
|
||||
item = &lruItem{
|
||||
key: key,
|
||||
value: value,
|
||||
}
|
||||
c.items[key] = c.evictList.PushFront(item)
|
||||
}
|
||||
|
||||
if c.expiration != nil {
|
||||
t := time.Now().Add(*c.expiration)
|
||||
item.expiration = &t
|
||||
}
|
||||
|
||||
if c.addedFunc != nil {
|
||||
go (*c.addedFunc)(key, value)
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// set a new key-value pair
|
||||
func (c *LRUCache) Set(key, value interface{}) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.set(key, value)
|
||||
}
|
||||
|
||||
// Get a value from cache pool using key if it exists.
|
||||
// If it dose not exists key and has LoaderFunc,
|
||||
// generate a value using `LoaderFunc` method returns value.
|
||||
func (c *LRUCache) Get(key interface{}) (interface{}, error) {
|
||||
c.mu.RLock()
|
||||
item, ok := c.items[key]
|
||||
c.mu.RUnlock()
|
||||
|
||||
if ok {
|
||||
it := item.Value.(*lruItem)
|
||||
if !it.IsExpired(nil) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return it.value, nil
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.removeElement(item)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
if c.loaderFunc == nil {
|
||||
return nil, NotFoundKeyError
|
||||
}
|
||||
|
||||
it, err := c.load(key, func(v interface{}, e error) (interface{}, error) {
|
||||
if e == nil {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.set(key, v)
|
||||
}
|
||||
return nil, e
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return it.(*lruItem).value, nil
|
||||
}
|
||||
|
||||
// evict removes the oldest item from the cache.
|
||||
func (c *LRUCache) evict(count int) {
|
||||
for i := 0; i < count; i++ {
|
||||
ent := c.evictList.Back()
|
||||
if ent == nil {
|
||||
return
|
||||
} else {
|
||||
c.removeElement(ent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Removes the provided key from the cache.
|
||||
func (c *LRUCache) Remove(key interface{}) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.remove(key)
|
||||
}
|
||||
|
||||
func (c *LRUCache) remove(key interface{}) bool {
|
||||
if ent, ok := c.items[key]; ok {
|
||||
c.removeElement(ent)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *LRUCache) removeElement(e *list.Element) {
|
||||
c.evictList.Remove(e)
|
||||
entry := e.Value.(*lruItem)
|
||||
delete(c.items, entry.key)
|
||||
if c.evictedFunc != nil {
|
||||
entry := e.Value.(*lruItem)
|
||||
go (*c.evictedFunc)(entry.key, entry.value)
|
||||
}
|
||||
}
|
||||
|
||||
// Returns a slice of the keys in the cache.
|
||||
func (c *LRUCache) Keys() []interface{} {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
keys := make([]interface{}, len(c.items))
|
||||
i := 0
|
||||
for k := range c.items {
|
||||
keys[i] = k
|
||||
i++
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
// Returns the number of items in the cache.
|
||||
func (c *LRUCache) Len() int {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
return len(c.items)
|
||||
}
|
||||
|
||||
// Completely clear the cache
|
||||
func (c *LRUCache) Purge() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.evictList = list.New()
|
||||
c.items = make(map[interface{}]*list.Element, c.size)
|
||||
}
|
||||
|
||||
// evict all expired entry
|
||||
func (c *LRUCache) gc() {
|
||||
now := time.Now()
|
||||
keys := []interface{}{}
|
||||
c.mu.RLock()
|
||||
for k, item := range c.items {
|
||||
if item.Value.(*lruItem).IsExpired(&now) {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
c.mu.RUnlock()
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
for _, k := range keys {
|
||||
c.remove(k)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
type lruItem struct {
|
||||
key interface{}
|
||||
value interface{}
|
||||
expiration *time.Time
|
||||
}
|
||||
|
||||
// returns boolean value whether this item is expired or not.
|
||||
func (it *lruItem) IsExpired(now *time.Time) bool {
|
||||
if it.expiration == nil {
|
||||
return false
|
||||
}
|
||||
if now == nil {
|
||||
t := time.Now()
|
||||
now = &t
|
||||
}
|
||||
return it.expiration.Before(*now)
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
package gcache_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/bluele/gcache"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func evictedFuncForLRU(key, value interface{}) {
|
||||
fmt.Printf("[LRU] Key:%v Value:%v will evicted.\n", key, value)
|
||||
}
|
||||
|
||||
func buildLRUCache(size int) gcache.Cache {
|
||||
return gcache.New(size).
|
||||
LRU().
|
||||
EvictedFunc(evictedFuncForLRU).
|
||||
Expiration(time.Second).
|
||||
Build()
|
||||
}
|
||||
|
||||
func buildLoadingLRUCache(size int, loader gcache.LoaderFunc) gcache.Cache {
|
||||
return gcache.New(size).
|
||||
LRU().
|
||||
LoaderFunc(loader).
|
||||
EvictedFunc(evictedFuncForLRU).
|
||||
Expiration(time.Second).
|
||||
Build()
|
||||
}
|
||||
|
||||
func TestLRUGet(t *testing.T) {
|
||||
size := 1000
|
||||
gc := buildLRUCache(size)
|
||||
testSetCache(t, gc, size)
|
||||
testGetCache(t, gc, size)
|
||||
}
|
||||
|
||||
func TestLoadingLRUGet(t *testing.T) {
|
||||
size := 1000
|
||||
gc := buildLoadingLRUCache(size, loader)
|
||||
testGetCache(t, gc, size)
|
||||
}
|
||||
|
||||
func TestLRULength(t *testing.T) {
|
||||
gc := buildLoadingLRUCache(1000, loader)
|
||||
gc.Get("test1")
|
||||
gc.Get("test2")
|
||||
length := gc.Len()
|
||||
expectedLength := 2
|
||||
if length != expectedLength {
|
||||
t.Errorf("Expected length is %v, not %v", length, expectedLength)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLRUEvictItem(t *testing.T) {
|
||||
cacheSize := 10
|
||||
numbers := 11
|
||||
gc := buildLoadingLRUCache(cacheSize, loader)
|
||||
|
||||
for i := 0; i < numbers; i++ {
|
||||
_, err := gc.Get(fmt.Sprintf("Key-%d", i))
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
package gcache
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// SimpleCache has no clear priority for evict cache. It depends on key-value map order.
|
||||
type SimpleCache struct {
|
||||
baseCache
|
||||
items map[interface{}]*simpleItem
|
||||
}
|
||||
|
||||
func newSimpleCache(cb *CacheBuilder) *SimpleCache {
|
||||
c := &SimpleCache{}
|
||||
buildCache(&c.baseCache, cb)
|
||||
|
||||
c.items = make(map[interface{}]*simpleItem, c.size)
|
||||
return c
|
||||
}
|
||||
|
||||
// set a new key-value pair
|
||||
func (c *SimpleCache) Set(key, value interface{}) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.set(key, value)
|
||||
}
|
||||
|
||||
func (c *SimpleCache) set(key, value interface{}) (interface{}, error) {
|
||||
// Check for existing item
|
||||
item, ok := c.items[key]
|
||||
if ok {
|
||||
item.value = value
|
||||
} else {
|
||||
// Verify size not exceeded
|
||||
if len(c.items) >= c.size {
|
||||
c.evict(1)
|
||||
}
|
||||
item = &simpleItem{
|
||||
value: value,
|
||||
}
|
||||
c.items[key] = item
|
||||
}
|
||||
|
||||
if c.expiration != nil {
|
||||
t := time.Now().Add(*c.expiration)
|
||||
item.expiration = &t
|
||||
}
|
||||
|
||||
if c.addedFunc != nil {
|
||||
go (*c.addedFunc)(key, value)
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// Get a value from cache pool using key if it exists.
|
||||
// If it dose not exists key and has LoaderFunc,
|
||||
// generate a value using `LoaderFunc` method returns value.
|
||||
func (c *SimpleCache) Get(key interface{}) (interface{}, error) {
|
||||
c.mu.RLock()
|
||||
item, ok := c.items[key]
|
||||
c.mu.RUnlock()
|
||||
if ok {
|
||||
if !item.IsExpired(nil) {
|
||||
return item.value, nil
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.remove(key)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
if c.loaderFunc == nil {
|
||||
return nil, NotFoundKeyError
|
||||
}
|
||||
|
||||
it, err := c.load(key, func(v interface{}, e error) (interface{}, error) {
|
||||
if e == nil {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.set(key, v)
|
||||
}
|
||||
return nil, e
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return it.(*simpleItem).value, nil
|
||||
}
|
||||
|
||||
func (c *SimpleCache) evict(count int) {
|
||||
now := time.Now()
|
||||
current := 0
|
||||
for key, item := range c.items {
|
||||
if current >= count {
|
||||
return
|
||||
}
|
||||
if item.expiration == nil || now.After(*item.expiration) {
|
||||
defer c.remove(key)
|
||||
current += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Removes the provided key from the cache.
|
||||
func (c *SimpleCache) Remove(key interface{}) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.remove(key)
|
||||
}
|
||||
|
||||
func (c *SimpleCache) remove(key interface{}) bool {
|
||||
item, ok := c.items[key]
|
||||
if ok {
|
||||
delete(c.items, key)
|
||||
if c.evictedFunc != nil {
|
||||
go (*c.evictedFunc)(key, item.value)
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns a slice of the keys in the cache.
|
||||
func (c *SimpleCache) Keys() []interface{} {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
keys := make([]interface{}, len(c.items))
|
||||
i := 0
|
||||
for k := range c.items {
|
||||
keys[i] = k
|
||||
i++
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
// Returns the number of items in the cache.
|
||||
func (c *SimpleCache) Len() int {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
return len(c.items)
|
||||
}
|
||||
|
||||
// Completely clear the cache
|
||||
func (c *SimpleCache) Purge() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.items = make(map[interface{}]*simpleItem, c.size)
|
||||
}
|
||||
|
||||
// evict all expired entry
|
||||
func (c *SimpleCache) gc() {
|
||||
now := time.Now()
|
||||
keys := []interface{}{}
|
||||
c.mu.RLock()
|
||||
for k, item := range c.items {
|
||||
if item.IsExpired(&now) {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
c.mu.RUnlock()
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
for _, k := range keys {
|
||||
c.remove(k)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
type simpleItem struct {
|
||||
value interface{}
|
||||
expiration *time.Time
|
||||
}
|
||||
|
||||
// returns boolean value whether this item is expired or not.
|
||||
func (si *simpleItem) IsExpired(now *time.Time) bool {
|
||||
if si.expiration == nil {
|
||||
return false
|
||||
}
|
||||
if now == nil {
|
||||
t := time.Now()
|
||||
now = &t
|
||||
}
|
||||
return si.expiration.Before(*now)
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
package gcache_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
gcache "github.com/bluele/gcache"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func buildSimpleCache(size int) gcache.Cache {
|
||||
return gcache.New(size).
|
||||
Simple().
|
||||
EvictedFunc(evictedFuncForSimple).
|
||||
Build()
|
||||
}
|
||||
|
||||
func buildLoadingSimpleCache(size int, loader gcache.LoaderFunc) gcache.Cache {
|
||||
return gcache.New(size).
|
||||
LoaderFunc(loader).
|
||||
Simple().
|
||||
EvictedFunc(evictedFuncForSimple).
|
||||
Build()
|
||||
}
|
||||
|
||||
func evictedFuncForSimple(key, value interface{}) {
|
||||
fmt.Printf("[Simple] Key:%v Value:%v will evicted.\n", key, value)
|
||||
}
|
||||
|
||||
func TestSimpleGet(t *testing.T) {
|
||||
size := 1000
|
||||
gc := buildSimpleCache(size)
|
||||
testSetCache(t, gc, size)
|
||||
testGetCache(t, gc, size)
|
||||
}
|
||||
|
||||
func TestLoadingSimpleGet(t *testing.T) {
|
||||
size := 1000
|
||||
numbers := 1000
|
||||
testGetCache(t, buildLoadingSimpleCache(size, loader), numbers)
|
||||
}
|
||||
|
||||
func TestSimpleLength(t *testing.T) {
|
||||
gc := buildLoadingSimpleCache(1000, loader)
|
||||
gc.Get("test1")
|
||||
gc.Get("test2")
|
||||
length := gc.Len()
|
||||
expectedLength := 2
|
||||
if length != expectedLength {
|
||||
t.Errorf("Expected length is %v, not %v", length, expectedLength)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimpleEvictItem(t *testing.T) {
|
||||
cacheSize := 10
|
||||
numbers := 11
|
||||
gc := buildLoadingSimpleCache(cacheSize, loader)
|
||||
|
||||
for i := 0; i < numbers; i++ {
|
||||
_, err := gc.Get(fmt.Sprintf("Key-%d", i))
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
Copyright 2012 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package singleflight provides a duplicate function call suppression
|
||||
// mechanism.
|
||||
package singleflight
|
||||
|
||||
import "sync"
|
||||
|
||||
// call is an in-flight or completed Do call
|
||||
type call struct {
|
||||
wg sync.WaitGroup
|
||||
val interface{}
|
||||
err error
|
||||
}
|
||||
|
||||
// Group represents a class of work and forms a namespace in which
|
||||
// units of work can be executed with duplicate suppression.
|
||||
type Group struct {
|
||||
mu sync.Mutex // protects m
|
||||
m map[interface{}]*call // lazily initialized
|
||||
}
|
||||
|
||||
// Do executes and returns the results of the given function, making
|
||||
// sure that only one execution is in-flight for a given key at a
|
||||
// time. If a duplicate comes in, the duplicate caller waits for the
|
||||
// original to complete and receives the same results.
|
||||
func (g *Group) Do(key interface{}, fn func() (interface{}, error)) (interface{}, error) {
|
||||
g.mu.Lock()
|
||||
if g.m == nil {
|
||||
g.m = make(map[interface{}]*call)
|
||||
}
|
||||
if c, ok := g.m[key]; ok {
|
||||
g.mu.Unlock()
|
||||
c.wg.Wait()
|
||||
return c.val, c.err
|
||||
}
|
||||
c := new(call)
|
||||
c.wg.Add(1)
|
||||
g.m[key] = c
|
||||
g.mu.Unlock()
|
||||
|
||||
c.val, c.err = fn()
|
||||
c.wg.Done()
|
||||
|
||||
g.mu.Lock()
|
||||
delete(g.m, key)
|
||||
g.mu.Unlock()
|
||||
|
||||
return c.val, c.err
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
Copyright 2012 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package singleflight
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDo(t *testing.T) {
|
||||
var g Group
|
||||
v, err := g.Do("key", func() (interface{}, error) {
|
||||
return "bar", nil
|
||||
})
|
||||
if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want {
|
||||
t.Errorf("Do = %v; want %v", got, want)
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("Do error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoErr(t *testing.T) {
|
||||
var g Group
|
||||
someErr := errors.New("Some error")
|
||||
v, err := g.Do("key", func() (interface{}, error) {
|
||||
return nil, someErr
|
||||
})
|
||||
if err != someErr {
|
||||
t.Errorf("Do error = %v; want someErr", err)
|
||||
}
|
||||
if v != nil {
|
||||
t.Errorf("unexpected non-nil value %#v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoDupSuppress(t *testing.T) {
|
||||
var g Group
|
||||
c := make(chan string)
|
||||
var calls int32
|
||||
fn := func() (interface{}, error) {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
return <-c, nil
|
||||
}
|
||||
|
||||
const n = 10
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < n; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
v, err := g.Do("key", fn)
|
||||
if err != nil {
|
||||
t.Errorf("Do error: %v", err)
|
||||
}
|
||||
if v.(string) != "bar" {
|
||||
t.Errorf("got %q; want %q", v, "bar")
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond) // let goroutines above block
|
||||
c <- "bar"
|
||||
wg.Wait()
|
||||
if got := atomic.LoadInt32(&calls); got != 1 {
|
||||
t.Errorf("number of calls = %d; want 1", got)
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package gcache
|
||||
|
||||
func minInt(x, y int) int {
|
||||
if x < y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
|
||||
func maxInt(x, y int) int {
|
||||
if x > y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
box: wercker/golang
|
||||
|
||||
build:
|
||||
steps:
|
||||
# Sets the go workspace and places you package
|
||||
# at the right place in the workspace tree
|
||||
- setup-go-workspace
|
||||
|
||||
# Test the project
|
||||
- script:
|
||||
name: go test
|
||||
code: |
|
||||
cd $WERCKER_SOURCE_DIR
|
||||
go version
|
||||
go test
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
This Source Code Form is subject to the terms of the Mozilla Public License,
|
||||
v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain
|
||||
one at http://mozilla.org/MPL/2.0/.
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
# GoCertifi: SSL Certificates for Golang
|
||||
|
||||
This Go package contains a CA bundle that you can reference in your Go code.
|
||||
This is useful for systems that do not have CA bundles that Golang can find
|
||||
itself, or where a uniform set of CAs is valuable.
|
||||
|
||||
This is the same CA bundle that ships with the
|
||||
[Python Requests](https://github.com/kennethreitz/requests) library, and is a
|
||||
Golang specific port of [certifi](https://github.com/kennethreitz/certifi). The
|
||||
CA bundle is derived from Mozilla's canonical set.
|
||||
|
||||
## Usage
|
||||
|
||||
You can use the `gocertifi` package as follows:
|
||||
|
||||
```go
|
||||
import "github.com/certifi/gocertifi"
|
||||
|
||||
cert_pool, err := gocertifi.CACerts()
|
||||
```
|
||||
|
||||
You can use the returned `*x509.CertPool` as part of an HTTP transport, for example:
|
||||
|
||||
```go
|
||||
import (
|
||||
"net/http"
|
||||
"crypto/tls"
|
||||
)
|
||||
|
||||
// Setup an HTTP client with a custom transport
|
||||
transport := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{RootCAs: cert_pool},
|
||||
}
|
||||
client := &http.Client{Transport: transport}
|
||||
|
||||
// Make an HTTP request using our custom transport
|
||||
resp, err := client.Get("https://example.com")
|
||||
```
|
||||
|
||||
## Detailed Documentation
|
||||
|
||||
Import as follows:
|
||||
|
||||
```go
|
||||
import "github.com/certifi/gocertifi"
|
||||
```
|
||||
|
||||
### Errors
|
||||
|
||||
```go
|
||||
var ErrParseFailed = errors.New("gocertifi: error when parsing certificates")
|
||||
```
|
||||
|
||||
### Functions
|
||||
|
||||
```go
|
||||
func CACerts() (*x509.CertPool, error)
|
||||
```
|
||||
CACerts builds an X.509 certificate pool containing the Mozilla CA Certificate
|
||||
bundle. Returns nil on error along with an appropriate error code.
|
||||
+5251
File diff suppressed because it is too large
Load Diff
+10
@@ -0,0 +1,10 @@
|
||||
package gocertifi
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestGetCerts(t *testing.T) {
|
||||
cert_pool, err := CACerts()
|
||||
if (cert_pool == nil) || (err != nil) {
|
||||
t.Errorf("Failed to return the certificates.")
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
from invoke import task, run
|
||||
import requests
|
||||
|
||||
@task
|
||||
def update():
|
||||
r = requests.get('https://mkcert.org/generate/')
|
||||
r.raise_for_status()
|
||||
certs = r.content
|
||||
|
||||
with open('certifi.go', 'rb') as f:
|
||||
file = f.read()
|
||||
|
||||
file = file.split('`\n')
|
||||
assert len(file) == 3
|
||||
file[1] = certs
|
||||
|
||||
run("rm certifi.go")
|
||||
|
||||
with open('certifi.go', 'wb') as f:
|
||||
f.write('`\n'.join(file))
|
||||
Generated
Vendored
+13
@@ -0,0 +1,13 @@
|
||||
include $(GOROOT)/src/Make.inc
|
||||
|
||||
TARG=bitbucket.org/ww/goautoneg
|
||||
GOFILES=autoneg.go
|
||||
|
||||
include $(GOROOT)/src/Make.pkg
|
||||
|
||||
format:
|
||||
gofmt -w *.go
|
||||
|
||||
docs:
|
||||
gomake clean
|
||||
godoc ${TARG} > README.txt
|
||||
Generated
Vendored
+67
@@ -0,0 +1,67 @@
|
||||
PACKAGE
|
||||
|
||||
package goautoneg
|
||||
import "bitbucket.org/ww/goautoneg"
|
||||
|
||||
HTTP Content-Type Autonegotiation.
|
||||
|
||||
The functions in this package implement the behaviour specified in
|
||||
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
|
||||
|
||||
Copyright (c) 2011, Open Knowledge Foundation Ltd.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
Neither the name of the Open Knowledge Foundation Ltd. nor the
|
||||
names of its contributors may be used to endorse or promote
|
||||
products derived from this software without specific prior written
|
||||
permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
FUNCTIONS
|
||||
|
||||
func Negotiate(header string, alternatives []string) (content_type string)
|
||||
Negotiate the most appropriate content_type given the accept header
|
||||
and a list of alternatives.
|
||||
|
||||
func ParseAccept(header string) (accept []Accept)
|
||||
Parse an Accept Header string returning a sorted list
|
||||
of clauses
|
||||
|
||||
|
||||
TYPES
|
||||
|
||||
type Accept struct {
|
||||
Type, SubType string
|
||||
Q float32
|
||||
Params map[string]string
|
||||
}
|
||||
Structure to represent a clause in an HTTP Accept Header
|
||||
|
||||
|
||||
SUBDIRECTORIES
|
||||
|
||||
.hg
|
||||
Generated
Vendored
+162
@@ -0,0 +1,162 @@
|
||||
/*
|
||||
HTTP Content-Type Autonegotiation.
|
||||
|
||||
The functions in this package implement the behaviour specified in
|
||||
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
|
||||
|
||||
Copyright (c) 2011, Open Knowledge Foundation Ltd.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
Neither the name of the Open Knowledge Foundation Ltd. nor the
|
||||
names of its contributors may be used to endorse or promote
|
||||
products derived from this software without specific prior written
|
||||
permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
*/
|
||||
package goautoneg
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Structure to represent a clause in an HTTP Accept Header
|
||||
type Accept struct {
|
||||
Type, SubType string
|
||||
Q float64
|
||||
Params map[string]string
|
||||
}
|
||||
|
||||
// For internal use, so that we can use the sort interface
|
||||
type accept_slice []Accept
|
||||
|
||||
func (accept accept_slice) Len() int {
|
||||
slice := []Accept(accept)
|
||||
return len(slice)
|
||||
}
|
||||
|
||||
func (accept accept_slice) Less(i, j int) bool {
|
||||
slice := []Accept(accept)
|
||||
ai, aj := slice[i], slice[j]
|
||||
if ai.Q > aj.Q {
|
||||
return true
|
||||
}
|
||||
if ai.Type != "*" && aj.Type == "*" {
|
||||
return true
|
||||
}
|
||||
if ai.SubType != "*" && aj.SubType == "*" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (accept accept_slice) Swap(i, j int) {
|
||||
slice := []Accept(accept)
|
||||
slice[i], slice[j] = slice[j], slice[i]
|
||||
}
|
||||
|
||||
// Parse an Accept Header string returning a sorted list
|
||||
// of clauses
|
||||
func ParseAccept(header string) (accept []Accept) {
|
||||
parts := strings.Split(header, ",")
|
||||
accept = make([]Accept, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part := strings.Trim(part, " ")
|
||||
|
||||
a := Accept{}
|
||||
a.Params = make(map[string]string)
|
||||
a.Q = 1.0
|
||||
|
||||
mrp := strings.Split(part, ";")
|
||||
|
||||
media_range := mrp[0]
|
||||
sp := strings.Split(media_range, "/")
|
||||
a.Type = strings.Trim(sp[0], " ")
|
||||
|
||||
switch {
|
||||
case len(sp) == 1 && a.Type == "*":
|
||||
a.SubType = "*"
|
||||
case len(sp) == 2:
|
||||
a.SubType = strings.Trim(sp[1], " ")
|
||||
default:
|
||||
continue
|
||||
}
|
||||
|
||||
if len(mrp) == 1 {
|
||||
accept = append(accept, a)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, param := range mrp[1:] {
|
||||
sp := strings.SplitN(param, "=", 2)
|
||||
if len(sp) != 2 {
|
||||
continue
|
||||
}
|
||||
token := strings.Trim(sp[0], " ")
|
||||
if token == "q" {
|
||||
a.Q, _ = strconv.ParseFloat(sp[1], 32)
|
||||
} else {
|
||||
a.Params[token] = strings.Trim(sp[1], " ")
|
||||
}
|
||||
}
|
||||
|
||||
accept = append(accept, a)
|
||||
}
|
||||
|
||||
slice := accept_slice(accept)
|
||||
sort.Sort(slice)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Negotiate the most appropriate content_type given the accept header
|
||||
// and a list of alternatives.
|
||||
func Negotiate(header string, alternatives []string) (content_type string) {
|
||||
asp := make([][]string, 0, len(alternatives))
|
||||
for _, ctype := range alternatives {
|
||||
asp = append(asp, strings.SplitN(ctype, "/", 2))
|
||||
}
|
||||
for _, clause := range ParseAccept(header) {
|
||||
for i, ctsp := range asp {
|
||||
if clause.Type == ctsp[0] && clause.SubType == ctsp[1] {
|
||||
content_type = alternatives[i]
|
||||
return
|
||||
}
|
||||
if clause.Type == ctsp[0] && clause.SubType == "*" {
|
||||
content_type = alternatives[i]
|
||||
return
|
||||
}
|
||||
if clause.Type == "*" && clause.SubType == "*" {
|
||||
content_type = alternatives[i]
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
Generated
Vendored
+33
@@ -0,0 +1,33 @@
|
||||
package goautoneg
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var chrome = "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"
|
||||
|
||||
func TestParseAccept(t *testing.T) {
|
||||
alternatives := []string{"text/html", "image/png"}
|
||||
content_type := Negotiate(chrome, alternatives)
|
||||
if content_type != "image/png" {
|
||||
t.Errorf("got %s expected image/png", content_type)
|
||||
}
|
||||
|
||||
alternatives = []string{"text/html", "text/plain", "text/n3"}
|
||||
content_type = Negotiate(chrome, alternatives)
|
||||
if content_type != "text/html" {
|
||||
t.Errorf("got %s expected text/html", content_type)
|
||||
}
|
||||
|
||||
alternatives = []string{"text/n3", "text/plain"}
|
||||
content_type = Negotiate(chrome, alternatives)
|
||||
if content_type != "text/plain" {
|
||||
t.Errorf("got %s expected text/plain", content_type)
|
||||
}
|
||||
|
||||
alternatives = []string{"text/n3", "application/rdf+xml"}
|
||||
content_type = Negotiate(chrome, alternatives)
|
||||
if content_type != "text/n3" {
|
||||
t.Errorf("got %s expected text/n3", content_type)
|
||||
}
|
||||
}
|
||||
Generated
Vendored
+63
@@ -0,0 +1,63 @@
|
||||
package quantile
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkInsertTargeted(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
|
||||
s := NewTargeted(Targets)
|
||||
b.ResetTimer()
|
||||
for i := float64(0); i < float64(b.N); i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInsertTargetedSmallEpsilon(b *testing.B) {
|
||||
s := NewTargeted(TargetsSmallEpsilon)
|
||||
b.ResetTimer()
|
||||
for i := float64(0); i < float64(b.N); i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInsertBiased(b *testing.B) {
|
||||
s := NewLowBiased(0.01)
|
||||
b.ResetTimer()
|
||||
for i := float64(0); i < float64(b.N); i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkInsertBiasedSmallEpsilon(b *testing.B) {
|
||||
s := NewLowBiased(0.0001)
|
||||
b.ResetTimer()
|
||||
for i := float64(0); i < float64(b.N); i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkQuery(b *testing.B) {
|
||||
s := NewTargeted(Targets)
|
||||
for i := float64(0); i < 1e6; i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
b.ResetTimer()
|
||||
n := float64(b.N)
|
||||
for i := float64(0); i < n; i++ {
|
||||
s.Query(i / n)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkQuerySmallEpsilon(b *testing.B) {
|
||||
s := NewTargeted(TargetsSmallEpsilon)
|
||||
for i := float64(0); i < 1e6; i++ {
|
||||
s.Insert(i)
|
||||
}
|
||||
b.ResetTimer()
|
||||
n := float64(b.N)
|
||||
for i := float64(0); i < n; i++ {
|
||||
s.Query(i / n)
|
||||
}
|
||||
}
|
||||
vendor/github.com/coreos/etcd/Godeps/_workspace/src/github.com/beorn7/perks/quantile/example_test.go
Generated
Vendored
+121
@@ -0,0 +1,121 @@
|
||||
// +build go1.1
|
||||
|
||||
package quantile_test
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/beorn7/perks/quantile"
|
||||
)
|
||||
|
||||
func Example_simple() {
|
||||
ch := make(chan float64)
|
||||
go sendFloats(ch)
|
||||
|
||||
// Compute the 50th, 90th, and 99th percentile.
|
||||
q := quantile.NewTargeted(map[float64]float64{
|
||||
0.50: 0.005,
|
||||
0.90: 0.001,
|
||||
0.99: 0.0001,
|
||||
})
|
||||
for v := range ch {
|
||||
q.Insert(v)
|
||||
}
|
||||
|
||||
fmt.Println("perc50:", q.Query(0.50))
|
||||
fmt.Println("perc90:", q.Query(0.90))
|
||||
fmt.Println("perc99:", q.Query(0.99))
|
||||
fmt.Println("count:", q.Count())
|
||||
// Output:
|
||||
// perc50: 5
|
||||
// perc90: 16
|
||||
// perc99: 223
|
||||
// count: 2388
|
||||
}
|
||||
|
||||
func Example_mergeMultipleStreams() {
|
||||
// Scenario:
|
||||
// We have multiple database shards. On each shard, there is a process
|
||||
// collecting query response times from the database logs and inserting
|
||||
// them into a Stream (created via NewTargeted(0.90)), much like the
|
||||
// Simple example. These processes expose a network interface for us to
|
||||
// ask them to serialize and send us the results of their
|
||||
// Stream.Samples so we may Merge and Query them.
|
||||
//
|
||||
// NOTES:
|
||||
// * These sample sets are small, allowing us to get them
|
||||
// across the network much faster than sending the entire list of data
|
||||
// points.
|
||||
//
|
||||
// * For this to work correctly, we must supply the same quantiles
|
||||
// a priori the process collecting the samples supplied to NewTargeted,
|
||||
// even if we do not plan to query them all here.
|
||||
ch := make(chan quantile.Samples)
|
||||
getDBQuerySamples(ch)
|
||||
q := quantile.NewTargeted(map[float64]float64{0.90: 0.001})
|
||||
for samples := range ch {
|
||||
q.Merge(samples)
|
||||
}
|
||||
fmt.Println("perc90:", q.Query(0.90))
|
||||
}
|
||||
|
||||
func Example_window() {
|
||||
// Scenario: We want the 90th, 95th, and 99th percentiles for each
|
||||
// minute.
|
||||
|
||||
ch := make(chan float64)
|
||||
go sendStreamValues(ch)
|
||||
|
||||
tick := time.NewTicker(1 * time.Minute)
|
||||
q := quantile.NewTargeted(map[float64]float64{
|
||||
0.90: 0.001,
|
||||
0.95: 0.0005,
|
||||
0.99: 0.0001,
|
||||
})
|
||||
for {
|
||||
select {
|
||||
case t := <-tick.C:
|
||||
flushToDB(t, q.Samples())
|
||||
q.Reset()
|
||||
case v := <-ch:
|
||||
q.Insert(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sendStreamValues(ch chan float64) {
|
||||
// Use your imagination
|
||||
}
|
||||
|
||||
func flushToDB(t time.Time, samples quantile.Samples) {
|
||||
// Use your imagination
|
||||
}
|
||||
|
||||
// This is a stub for the above example. In reality this would hit the remote
|
||||
// servers via http or something like it.
|
||||
func getDBQuerySamples(ch chan quantile.Samples) {}
|
||||
|
||||
func sendFloats(ch chan<- float64) {
|
||||
f, err := os.Open("exampledata.txt")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
b := sc.Bytes()
|
||||
v, err := strconv.ParseFloat(string(b), 64)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
ch <- v
|
||||
}
|
||||
if sc.Err() != nil {
|
||||
log.Fatal(sc.Err())
|
||||
}
|
||||
close(ch)
|
||||
}
|
||||
vendor/github.com/coreos/etcd/Godeps/_workspace/src/github.com/beorn7/perks/quantile/exampledata.txt
Generated
Vendored
+2388
File diff suppressed because it is too large
Load Diff
Generated
Vendored
+292
@@ -0,0 +1,292 @@
|
||||
// Package quantile computes approximate quantiles over an unbounded data
|
||||
// stream within low memory and CPU bounds.
|
||||
//
|
||||
// A small amount of accuracy is traded to achieve the above properties.
|
||||
//
|
||||
// Multiple streams can be merged before calling Query to generate a single set
|
||||
// of results. This is meaningful when the streams represent the same type of
|
||||
// data. See Merge and Samples.
|
||||
//
|
||||
// For more detailed information about the algorithm used, see:
|
||||
//
|
||||
// Effective Computation of Biased Quantiles over Data Streams
|
||||
//
|
||||
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
|
||||
package quantile
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Sample holds an observed value and meta information for compression. JSON
|
||||
// tags have been added for convenience.
|
||||
type Sample struct {
|
||||
Value float64 `json:",string"`
|
||||
Width float64 `json:",string"`
|
||||
Delta float64 `json:",string"`
|
||||
}
|
||||
|
||||
// Samples represents a slice of samples. It implements sort.Interface.
|
||||
type Samples []Sample
|
||||
|
||||
func (a Samples) Len() int { return len(a) }
|
||||
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
|
||||
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
|
||||
type invariant func(s *stream, r float64) float64
|
||||
|
||||
// NewLowBiased returns an initialized Stream for low-biased quantiles
|
||||
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
|
||||
// error guarantees can still be given even for the lower ranks of the data
|
||||
// distribution.
|
||||
//
|
||||
// The provided epsilon is a relative error, i.e. the true quantile of a value
|
||||
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
|
||||
//
|
||||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
|
||||
// properties.
|
||||
func NewLowBiased(epsilon float64) *Stream {
|
||||
ƒ := func(s *stream, r float64) float64 {
|
||||
return 2 * epsilon * r
|
||||
}
|
||||
return newStream(ƒ)
|
||||
}
|
||||
|
||||
// NewHighBiased returns an initialized Stream for high-biased quantiles
|
||||
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
|
||||
// error guarantees can still be given even for the higher ranks of the data
|
||||
// distribution.
|
||||
//
|
||||
// The provided epsilon is a relative error, i.e. the true quantile of a value
|
||||
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
|
||||
//
|
||||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
|
||||
// properties.
|
||||
func NewHighBiased(epsilon float64) *Stream {
|
||||
ƒ := func(s *stream, r float64) float64 {
|
||||
return 2 * epsilon * (s.n - r)
|
||||
}
|
||||
return newStream(ƒ)
|
||||
}
|
||||
|
||||
// NewTargeted returns an initialized Stream concerned with a particular set of
|
||||
// quantile values that are supplied a priori. Knowing these a priori reduces
|
||||
// space and computation time. The targets map maps the desired quantiles to
|
||||
// their absolute errors, i.e. the true quantile of a value returned by a query
|
||||
// is guaranteed to be within (Quantile±Epsilon).
|
||||
//
|
||||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
|
||||
func NewTargeted(targets map[float64]float64) *Stream {
|
||||
ƒ := func(s *stream, r float64) float64 {
|
||||
var m = math.MaxFloat64
|
||||
var f float64
|
||||
for quantile, epsilon := range targets {
|
||||
if quantile*s.n <= r {
|
||||
f = (2 * epsilon * r) / quantile
|
||||
} else {
|
||||
f = (2 * epsilon * (s.n - r)) / (1 - quantile)
|
||||
}
|
||||
if f < m {
|
||||
m = f
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
return newStream(ƒ)
|
||||
}
|
||||
|
||||
// Stream computes quantiles for a stream of float64s. It is not thread-safe by
|
||||
// design. Take care when using across multiple goroutines.
|
||||
type Stream struct {
|
||||
*stream
|
||||
b Samples
|
||||
sorted bool
|
||||
}
|
||||
|
||||
func newStream(ƒ invariant) *Stream {
|
||||
x := &stream{ƒ: ƒ}
|
||||
return &Stream{x, make(Samples, 0, 500), true}
|
||||
}
|
||||
|
||||
// Insert inserts v into the stream.
|
||||
func (s *Stream) Insert(v float64) {
|
||||
s.insert(Sample{Value: v, Width: 1})
|
||||
}
|
||||
|
||||
func (s *Stream) insert(sample Sample) {
|
||||
s.b = append(s.b, sample)
|
||||
s.sorted = false
|
||||
if len(s.b) == cap(s.b) {
|
||||
s.flush()
|
||||
}
|
||||
}
|
||||
|
||||
// Query returns the computed qth percentiles value. If s was created with
|
||||
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
|
||||
// will return an unspecified result.
|
||||
func (s *Stream) Query(q float64) float64 {
|
||||
if !s.flushed() {
|
||||
// Fast path when there hasn't been enough data for a flush;
|
||||
// this also yields better accuracy for small sets of data.
|
||||
l := len(s.b)
|
||||
if l == 0 {
|
||||
return 0
|
||||
}
|
||||
i := int(float64(l) * q)
|
||||
if i > 0 {
|
||||
i -= 1
|
||||
}
|
||||
s.maybeSort()
|
||||
return s.b[i].Value
|
||||
}
|
||||
s.flush()
|
||||
return s.stream.query(q)
|
||||
}
|
||||
|
||||
// Merge merges samples into the underlying streams samples. This is handy when
|
||||
// merging multiple streams from separate threads, database shards, etc.
|
||||
//
|
||||
// ATTENTION: This method is broken and does not yield correct results. The
|
||||
// underlying algorithm is not capable of merging streams correctly.
|
||||
func (s *Stream) Merge(samples Samples) {
|
||||
sort.Sort(samples)
|
||||
s.stream.merge(samples)
|
||||
}
|
||||
|
||||
// Reset reinitializes and clears the list reusing the samples buffer memory.
|
||||
func (s *Stream) Reset() {
|
||||
s.stream.reset()
|
||||
s.b = s.b[:0]
|
||||
}
|
||||
|
||||
// Samples returns stream samples held by s.
|
||||
func (s *Stream) Samples() Samples {
|
||||
if !s.flushed() {
|
||||
return s.b
|
||||
}
|
||||
s.flush()
|
||||
return s.stream.samples()
|
||||
}
|
||||
|
||||
// Count returns the total number of samples observed in the stream
|
||||
// since initialization.
|
||||
func (s *Stream) Count() int {
|
||||
return len(s.b) + s.stream.count()
|
||||
}
|
||||
|
||||
func (s *Stream) flush() {
|
||||
s.maybeSort()
|
||||
s.stream.merge(s.b)
|
||||
s.b = s.b[:0]
|
||||
}
|
||||
|
||||
func (s *Stream) maybeSort() {
|
||||
if !s.sorted {
|
||||
s.sorted = true
|
||||
sort.Sort(s.b)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stream) flushed() bool {
|
||||
return len(s.stream.l) > 0
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
n float64
|
||||
l []Sample
|
||||
ƒ invariant
|
||||
}
|
||||
|
||||
func (s *stream) reset() {
|
||||
s.l = s.l[:0]
|
||||
s.n = 0
|
||||
}
|
||||
|
||||
func (s *stream) insert(v float64) {
|
||||
s.merge(Samples{{v, 1, 0}})
|
||||
}
|
||||
|
||||
func (s *stream) merge(samples Samples) {
|
||||
// TODO(beorn7): This tries to merge not only individual samples, but
|
||||
// whole summaries. The paper doesn't mention merging summaries at
|
||||
// all. Unittests show that the merging is inaccurate. Find out how to
|
||||
// do merges properly.
|
||||
var r float64
|
||||
i := 0
|
||||
for _, sample := range samples {
|
||||
for ; i < len(s.l); i++ {
|
||||
c := s.l[i]
|
||||
if c.Value > sample.Value {
|
||||
// Insert at position i.
|
||||
s.l = append(s.l, Sample{})
|
||||
copy(s.l[i+1:], s.l[i:])
|
||||
s.l[i] = Sample{
|
||||
sample.Value,
|
||||
sample.Width,
|
||||
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
|
||||
// TODO(beorn7): How to calculate delta correctly?
|
||||
}
|
||||
i++
|
||||
goto inserted
|
||||
}
|
||||
r += c.Width
|
||||
}
|
||||
s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
|
||||
i++
|
||||
inserted:
|
||||
s.n += sample.Width
|
||||
r += sample.Width
|
||||
}
|
||||
s.compress()
|
||||
}
|
||||
|
||||
func (s *stream) count() int {
|
||||
return int(s.n)
|
||||
}
|
||||
|
||||
func (s *stream) query(q float64) float64 {
|
||||
t := math.Ceil(q * s.n)
|
||||
t += math.Ceil(s.ƒ(s, t) / 2)
|
||||
p := s.l[0]
|
||||
var r float64
|
||||
for _, c := range s.l[1:] {
|
||||
r += p.Width
|
||||
if r+c.Width+c.Delta > t {
|
||||
return p.Value
|
||||
}
|
||||
p = c
|
||||
}
|
||||
return p.Value
|
||||
}
|
||||
|
||||
func (s *stream) compress() {
|
||||
if len(s.l) < 2 {
|
||||
return
|
||||
}
|
||||
x := s.l[len(s.l)-1]
|
||||
xi := len(s.l) - 1
|
||||
r := s.n - 1 - x.Width
|
||||
|
||||
for i := len(s.l) - 2; i >= 0; i-- {
|
||||
c := s.l[i]
|
||||
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
|
||||
x.Width += c.Width
|
||||
s.l[xi] = x
|
||||
// Remove element at i.
|
||||
copy(s.l[i:], s.l[i+1:])
|
||||
s.l = s.l[:len(s.l)-1]
|
||||
xi -= 1
|
||||
} else {
|
||||
x = c
|
||||
xi = i
|
||||
}
|
||||
r -= c.Width
|
||||
}
|
||||
}
|
||||
|
||||
func (s *stream) samples() Samples {
|
||||
samples := make(Samples, len(s.l))
|
||||
copy(samples, s.l)
|
||||
return samples
|
||||
}
|
||||
Generated
Vendored
+188
@@ -0,0 +1,188 @@
|
||||
package quantile
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var (
|
||||
Targets = map[float64]float64{
|
||||
0.01: 0.001,
|
||||
0.10: 0.01,
|
||||
0.50: 0.05,
|
||||
0.90: 0.01,
|
||||
0.99: 0.001,
|
||||
}
|
||||
TargetsSmallEpsilon = map[float64]float64{
|
||||
0.01: 0.0001,
|
||||
0.10: 0.001,
|
||||
0.50: 0.005,
|
||||
0.90: 0.001,
|
||||
0.99: 0.0001,
|
||||
}
|
||||
LowQuantiles = []float64{0.01, 0.1, 0.5}
|
||||
HighQuantiles = []float64{0.99, 0.9, 0.5}
|
||||
)
|
||||
|
||||
const RelativeEpsilon = 0.01
|
||||
|
||||
func verifyPercsWithAbsoluteEpsilon(t *testing.T, a []float64, s *Stream) {
|
||||
sort.Float64s(a)
|
||||
for quantile, epsilon := range Targets {
|
||||
n := float64(len(a))
|
||||
k := int(quantile * n)
|
||||
lower := int((quantile - epsilon) * n)
|
||||
if lower < 1 {
|
||||
lower = 1
|
||||
}
|
||||
upper := int(math.Ceil((quantile + epsilon) * n))
|
||||
if upper > len(a) {
|
||||
upper = len(a)
|
||||
}
|
||||
w, min, max := a[k-1], a[lower-1], a[upper-1]
|
||||
if g := s.Query(quantile); g < min || g > max {
|
||||
t.Errorf("q=%f: want %v [%f,%f], got %v", quantile, w, min, max, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyLowPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
|
||||
sort.Float64s(a)
|
||||
for _, qu := range LowQuantiles {
|
||||
n := float64(len(a))
|
||||
k := int(qu * n)
|
||||
|
||||
lowerRank := int((1 - RelativeEpsilon) * qu * n)
|
||||
upperRank := int(math.Ceil((1 + RelativeEpsilon) * qu * n))
|
||||
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
|
||||
if g := s.Query(qu); g < min || g > max {
|
||||
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyHighPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
|
||||
sort.Float64s(a)
|
||||
for _, qu := range HighQuantiles {
|
||||
n := float64(len(a))
|
||||
k := int(qu * n)
|
||||
|
||||
lowerRank := int((1 - (1+RelativeEpsilon)*(1-qu)) * n)
|
||||
upperRank := int(math.Ceil((1 - (1-RelativeEpsilon)*(1-qu)) * n))
|
||||
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
|
||||
if g := s.Query(qu); g < min || g > max {
|
||||
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func populateStream(s *Stream) []float64 {
|
||||
a := make([]float64, 0, 1e5+100)
|
||||
for i := 0; i < cap(a); i++ {
|
||||
v := rand.NormFloat64()
|
||||
// Add 5% asymmetric outliers.
|
||||
if i%20 == 0 {
|
||||
v = v*v + 1
|
||||
}
|
||||
s.Insert(v)
|
||||
a = append(a, v)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func TestTargetedQuery(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s := NewTargeted(Targets)
|
||||
a := populateStream(s)
|
||||
verifyPercsWithAbsoluteEpsilon(t, a, s)
|
||||
}
|
||||
|
||||
func TestLowBiasedQuery(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s := NewLowBiased(RelativeEpsilon)
|
||||
a := populateStream(s)
|
||||
verifyLowPercsWithRelativeEpsilon(t, a, s)
|
||||
}
|
||||
|
||||
func TestHighBiasedQuery(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s := NewHighBiased(RelativeEpsilon)
|
||||
a := populateStream(s)
|
||||
verifyHighPercsWithRelativeEpsilon(t, a, s)
|
||||
}
|
||||
|
||||
// BrokenTestTargetedMerge is broken, see Merge doc comment.
|
||||
func BrokenTestTargetedMerge(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s1 := NewTargeted(Targets)
|
||||
s2 := NewTargeted(Targets)
|
||||
a := populateStream(s1)
|
||||
a = append(a, populateStream(s2)...)
|
||||
s1.Merge(s2.Samples())
|
||||
verifyPercsWithAbsoluteEpsilon(t, a, s1)
|
||||
}
|
||||
|
||||
// BrokenTestLowBiasedMerge is broken, see Merge doc comment.
|
||||
func BrokenTestLowBiasedMerge(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s1 := NewLowBiased(RelativeEpsilon)
|
||||
s2 := NewLowBiased(RelativeEpsilon)
|
||||
a := populateStream(s1)
|
||||
a = append(a, populateStream(s2)...)
|
||||
s1.Merge(s2.Samples())
|
||||
verifyLowPercsWithRelativeEpsilon(t, a, s2)
|
||||
}
|
||||
|
||||
// BrokenTestHighBiasedMerge is broken, see Merge doc comment.
|
||||
func BrokenTestHighBiasedMerge(t *testing.T) {
|
||||
rand.Seed(42)
|
||||
s1 := NewHighBiased(RelativeEpsilon)
|
||||
s2 := NewHighBiased(RelativeEpsilon)
|
||||
a := populateStream(s1)
|
||||
a = append(a, populateStream(s2)...)
|
||||
s1.Merge(s2.Samples())
|
||||
verifyHighPercsWithRelativeEpsilon(t, a, s2)
|
||||
}
|
||||
|
||||
func TestUncompressed(t *testing.T) {
|
||||
q := NewTargeted(Targets)
|
||||
for i := 100; i > 0; i-- {
|
||||
q.Insert(float64(i))
|
||||
}
|
||||
if g := q.Count(); g != 100 {
|
||||
t.Errorf("want count 100, got %d", g)
|
||||
}
|
||||
// Before compression, Query should have 100% accuracy.
|
||||
for quantile := range Targets {
|
||||
w := quantile * 100
|
||||
if g := q.Query(quantile); g != w {
|
||||
t.Errorf("want %f, got %f", w, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUncompressedSamples(t *testing.T) {
|
||||
q := NewTargeted(map[float64]float64{0.99: 0.001})
|
||||
for i := 1; i <= 100; i++ {
|
||||
q.Insert(float64(i))
|
||||
}
|
||||
if g := q.Samples().Len(); g != 100 {
|
||||
t.Errorf("want count 100, got %d", g)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUncompressedOne(t *testing.T) {
|
||||
q := NewTargeted(map[float64]float64{0.99: 0.01})
|
||||
q.Insert(3.14)
|
||||
if g := q.Query(0.90); g != 3.14 {
|
||||
t.Error("want PI, got", g)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaults(t *testing.T) {
|
||||
if g := NewTargeted(map[float64]float64{0.99: 0.001}).Query(0.99); g != 0 {
|
||||
t.Errorf("want 0, got %f", g)
|
||||
}
|
||||
}
|
||||
Generated
Vendored
+20
@@ -0,0 +1,20 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2013 Ben Johnson
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
Generated
Vendored
+54
@@ -0,0 +1,54 @@
|
||||
TEST=.
|
||||
BENCH=.
|
||||
COVERPROFILE=/tmp/c.out
|
||||
BRANCH=`git rev-parse --abbrev-ref HEAD`
|
||||
COMMIT=`git rev-parse --short HEAD`
|
||||
GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
|
||||
|
||||
default: build
|
||||
|
||||
bench:
|
||||
go test -v -test.run=NOTHINCONTAINSTHIS -test.bench=$(BENCH)
|
||||
|
||||
# http://cloc.sourceforge.net/
|
||||
cloc:
|
||||
@cloc --not-match-f='Makefile|_test.go' .
|
||||
|
||||
cover: fmt
|
||||
go test -coverprofile=$(COVERPROFILE) -test.run=$(TEST) $(COVERFLAG) .
|
||||
go tool cover -html=$(COVERPROFILE)
|
||||
rm $(COVERPROFILE)
|
||||
|
||||
cpuprofile: fmt
|
||||
@go test -c
|
||||
@./bolt.test -test.v -test.run=$(TEST) -test.cpuprofile cpu.prof
|
||||
|
||||
# go get github.com/kisielk/errcheck
|
||||
errcheck:
|
||||
@echo "=== errcheck ==="
|
||||
@errcheck github.com/boltdb/bolt
|
||||
|
||||
fmt:
|
||||
@go fmt ./...
|
||||
|
||||
get:
|
||||
@go get -d ./...
|
||||
|
||||
build: get
|
||||
@mkdir -p bin
|
||||
@go build -ldflags=$(GOLDFLAGS) -a -o bin/bolt ./cmd/bolt
|
||||
|
||||
test: fmt
|
||||
@go get github.com/stretchr/testify/assert
|
||||
@echo "=== TESTS ==="
|
||||
@go test -v -cover -test.run=$(TEST)
|
||||
@echo ""
|
||||
@echo ""
|
||||
@echo "=== CLI ==="
|
||||
@go test -v -test.run=$(TEST) ./cmd/bolt
|
||||
@echo ""
|
||||
@echo ""
|
||||
@echo "=== RACE DETECTOR ==="
|
||||
@go test -v -race -test.run="TestSimulate_(100op|1000op)"
|
||||
|
||||
.PHONY: bench cloc cover cpuprofile fmt memprofile test
|
||||
Generated
Vendored
+621
@@ -0,0 +1,621 @@
|
||||
Bolt [](https://drone.io/github.com/boltdb/bolt/latest) [](https://coveralls.io/r/boltdb/bolt?branch=master) [](https://godoc.org/github.com/boltdb/bolt) 
|
||||
====
|
||||
|
||||
Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] and
|
||||
the [LMDB project][lmdb]. The goal of the project is to provide a simple,
|
||||
fast, and reliable database for projects that don't require a full database
|
||||
server such as Postgres or MySQL.
|
||||
|
||||
Since Bolt is meant to be used as such a low-level piece of functionality,
|
||||
simplicity is key. The API will be small and only focus on getting values
|
||||
and setting values. That's it.
|
||||
|
||||
[hyc_symas]: https://twitter.com/hyc_symas
|
||||
[lmdb]: http://symas.com/mdb/
|
||||
|
||||
|
||||
## Project Status
|
||||
|
||||
Bolt is stable and the API is fixed. Full unit test coverage and randomized
|
||||
black box testing are used to ensure database consistency and thread safety.
|
||||
Bolt is currently in high-load production environments serving databases as
|
||||
large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed
|
||||
services every day.
|
||||
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Installing
|
||||
|
||||
To start using Bolt, install Go and run `go get`:
|
||||
|
||||
```sh
|
||||
$ go get github.com/boltdb/bolt/...
|
||||
```
|
||||
|
||||
This will retrieve the library and install the `bolt` command line utility into
|
||||
your `$GOBIN` path.
|
||||
|
||||
|
||||
### Opening a database
|
||||
|
||||
The top-level object in Bolt is a `DB`. It is represented as a single file on
|
||||
your disk and represents a consistent snapshot of your data.
|
||||
|
||||
To open your database, simply use the `bolt.Open()` function:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/boltdb/bolt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Open the my.db data file in your current directory.
|
||||
// It will be created if it doesn't exist.
|
||||
db, err := bolt.Open("my.db", 0600, nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
Please note that Bolt obtains a file lock on the data file so multiple processes
|
||||
cannot open the same database at the same time. Opening an already open Bolt
|
||||
database will cause it to hang until the other process closes it. To prevent
|
||||
an indefinite wait you can pass a timeout option to the `Open()` function:
|
||||
|
||||
```go
|
||||
db, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second})
|
||||
```
|
||||
|
||||
|
||||
### Transactions
|
||||
|
||||
Bolt allows only one read-write transaction at a time but allows as many
|
||||
read-only transactions as you want at a time. Each transaction has a consistent
|
||||
view of the data as it existed when the transaction started.
|
||||
|
||||
Individual transactions and all objects created from them (e.g. buckets, keys)
|
||||
are not thread safe. To work with data in multiple goroutines you must start
|
||||
a transaction for each one or use locking to ensure only one goroutine accesses
|
||||
a transaction at a time. Creating transaction from the `DB` is thread safe.
|
||||
|
||||
Read-only transactions and read-write transactions should not depend on one
|
||||
another and generally shouldn't be opened simultaneously in the same goroutine.
|
||||
This can cause a deadlock as the read-write transaction needs to periodically
|
||||
re-map the data file but it cannot do so while a read-only transaction is open.
|
||||
|
||||
|
||||
#### Read-write transactions
|
||||
|
||||
To start a read-write transaction, you can use the `DB.Update()` function:
|
||||
|
||||
```go
|
||||
err := db.Update(func(tx *bolt.Tx) error {
|
||||
...
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
Inside the closure, you have a consistent view of the database. You commit the
|
||||
transaction by returning `nil` at the end. You can also rollback the transaction
|
||||
at any point by returning an error. All database operations are allowed inside
|
||||
a read-write transaction.
|
||||
|
||||
Always check the return error as it will report any disk failures that can cause
|
||||
your transaction to not complete. If you return an error within your closure
|
||||
it will be passed through.
|
||||
|
||||
|
||||
#### Read-only transactions
|
||||
|
||||
To start a read-only transaction, you can use the `DB.View()` function:
|
||||
|
||||
```go
|
||||
err := db.View(func(tx *bolt.Tx) error {
|
||||
...
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
You also get a consistent view of the database within this closure, however,
|
||||
no mutating operations are allowed within a read-only transaction. You can only
|
||||
retrieve buckets, retrieve values, and copy the database within a read-only
|
||||
transaction.
|
||||
|
||||
|
||||
#### Batch read-write transactions
|
||||
|
||||
Each `DB.Update()` waits for disk to commit the writes. This overhead
|
||||
can be minimized by combining multiple updates with the `DB.Batch()`
|
||||
function:
|
||||
|
||||
```go
|
||||
err := db.Batch(func(tx *bolt.Tx) error {
|
||||
...
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
Concurrent Batch calls are opportunistically combined into larger
|
||||
transactions. Batch is only useful when there are multiple goroutines
|
||||
calling it.
|
||||
|
||||
The trade-off is that `Batch` can call the given
|
||||
function multiple times, if parts of the transaction fail. The
|
||||
function must be idempotent and side effects must take effect only
|
||||
after a successful return from `DB.Batch()`.
|
||||
|
||||
For example: don't display messages from inside the function, instead
|
||||
set variables in the enclosing scope:
|
||||
|
||||
```go
|
||||
var id uint64
|
||||
err := db.Batch(func(tx *bolt.Tx) error {
|
||||
// Find last key in bucket, decode as bigendian uint64, increment
|
||||
// by one, encode back to []byte, and add new key.
|
||||
...
|
||||
id = newValue
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return ...
|
||||
}
|
||||
fmt.Println("Allocated ID %d", id)
|
||||
```
|
||||
|
||||
|
||||
#### Managing transactions manually
|
||||
|
||||
The `DB.View()` and `DB.Update()` functions are wrappers around the `DB.Begin()`
|
||||
function. These helper functions will start the transaction, execute a function,
|
||||
and then safely close your transaction if an error is returned. This is the
|
||||
recommended way to use Bolt transactions.
|
||||
|
||||
However, sometimes you may want to manually start and end your transactions.
|
||||
You can use the `Tx.Begin()` function directly but _please_ be sure to close the
|
||||
transaction.
|
||||
|
||||
```go
|
||||
// Start a writable transaction.
|
||||
tx, err := db.Begin(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// Use the transaction...
|
||||
_, err := tx.CreateBucket([]byte("MyBucket"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Commit the transaction and check for error.
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
The first argument to `DB.Begin()` is a boolean stating if the transaction
|
||||
should be writable.
|
||||
|
||||
|
||||
### Using buckets
|
||||
|
||||
Buckets are collections of key/value pairs within the database. All keys in a
|
||||
bucket must be unique. You can create a bucket using the `DB.CreateBucket()`
|
||||
function:
|
||||
|
||||
```go
|
||||
db.Update(func(tx *bolt.Tx) error {
|
||||
b, err := tx.CreateBucket([]byte("MyBucket"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create bucket: %s", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
You can also create a bucket only if it doesn't exist by using the
|
||||
`Tx.CreateBucketIfNotExists()` function. It's a common pattern to call this
|
||||
function for all your top-level buckets after you open your database so you can
|
||||
guarantee that they exist for future transactions.
|
||||
|
||||
To delete a bucket, simply call the `Tx.DeleteBucket()` function.
|
||||
|
||||
|
||||
### Using key/value pairs
|
||||
|
||||
To save a key/value pair to a bucket, use the `Bucket.Put()` function:
|
||||
|
||||
```go
|
||||
db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("MyBucket"))
|
||||
err := b.Put([]byte("answer"), []byte("42"))
|
||||
return err
|
||||
})
|
||||
```
|
||||
|
||||
This will set the value of the `"answer"` key to `"42"` in the `MyBucket`
|
||||
bucket. To retrieve this value, we can use the `Bucket.Get()` function:
|
||||
|
||||
```go
|
||||
db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("MyBucket"))
|
||||
v := b.Get([]byte("answer"))
|
||||
fmt.Printf("The answer is: %s\n", v)
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
The `Get()` function does not return an error because its operation is
|
||||
guarenteed to work (unless there is some kind of system failure). If the key
|
||||
exists then it will return its byte slice value. If it doesn't exist then it
|
||||
will return `nil`. It's important to note that you can have a zero-length value
|
||||
set to a key which is different than the key not existing.
|
||||
|
||||
Use the `Bucket.Delete()` function to delete a key from the bucket.
|
||||
|
||||
Please note that values returned from `Get()` are only valid while the
|
||||
transaction is open. If you need to use a value outside of the transaction
|
||||
then you must use `copy()` to copy it to another byte slice.
|
||||
|
||||
|
||||
### Iterating over keys
|
||||
|
||||
Bolt stores its keys in byte-sorted order within a bucket. This makes sequential
|
||||
iteration over these keys extremely fast. To iterate over keys we'll use a
|
||||
`Cursor`:
|
||||
|
||||
```go
|
||||
db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("MyBucket"))
|
||||
c := b.Cursor()
|
||||
|
||||
for k, v := c.First(); k != nil; k, v = c.Next() {
|
||||
fmt.Printf("key=%s, value=%s\n", k, v)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
The cursor allows you to move to a specific point in the list of keys and move
|
||||
forward or backward through the keys one at a time.
|
||||
|
||||
The following functions are available on the cursor:
|
||||
|
||||
```
|
||||
First() Move to the first key.
|
||||
Last() Move to the last key.
|
||||
Seek() Move to a specific key.
|
||||
Next() Move to the next key.
|
||||
Prev() Move to the previous key.
|
||||
```
|
||||
|
||||
When you have iterated to the end of the cursor then `Next()` will return `nil`.
|
||||
You must seek to a position using `First()`, `Last()`, or `Seek()` before
|
||||
calling `Next()` or `Prev()`. If you do not seek to a position then these
|
||||
functions will return `nil`.
|
||||
|
||||
|
||||
#### Prefix scans
|
||||
|
||||
To iterate over a key prefix, you can combine `Seek()` and `bytes.HasPrefix()`:
|
||||
|
||||
```go
|
||||
db.View(func(tx *bolt.Tx) error {
|
||||
c := tx.Bucket([]byte("MyBucket")).Cursor()
|
||||
|
||||
prefix := []byte("1234")
|
||||
for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
|
||||
fmt.Printf("key=%s, value=%s\n", k, v)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
#### Range scans
|
||||
|
||||
Another common use case is scanning over a range such as a time range. If you
|
||||
use a sortable time encoding such as RFC3339 then you can query a specific
|
||||
date range like this:
|
||||
|
||||
```go
|
||||
db.View(func(tx *bolt.Tx) error {
|
||||
// Assume our events bucket has RFC3339 encoded time keys.
|
||||
c := tx.Bucket([]byte("Events")).Cursor()
|
||||
|
||||
// Our time range spans the 90's decade.
|
||||
min := []byte("1990-01-01T00:00:00Z")
|
||||
max := []byte("2000-01-01T00:00:00Z")
|
||||
|
||||
// Iterate over the 90's.
|
||||
for k, v := c.Seek(min); k != nil && bytes.Compare(k, max) <= 0; k, v = c.Next() {
|
||||
fmt.Printf("%s: %s\n", k, v)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
|
||||
#### ForEach()
|
||||
|
||||
You can also use the function `ForEach()` if you know you'll be iterating over
|
||||
all the keys in a bucket:
|
||||
|
||||
```go
|
||||
db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("MyBucket"))
|
||||
b.ForEach(func(k, v []byte) error {
|
||||
fmt.Printf("key=%s, value=%s\n", k, v)
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
|
||||
### Nested buckets
|
||||
|
||||
You can also store a bucket in a key to create nested buckets. The API is the
|
||||
same as the bucket management API on the `DB` object:
|
||||
|
||||
```go
|
||||
func (*Bucket) CreateBucket(key []byte) (*Bucket, error)
|
||||
func (*Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error)
|
||||
func (*Bucket) DeleteBucket(key []byte) error
|
||||
```
|
||||
|
||||
|
||||
### Database backups
|
||||
|
||||
Bolt is a single file so it's easy to backup. You can use the `Tx.WriteTo()`
|
||||
function to write a consistent view of the database to a writer. If you call
|
||||
this from a read-only transaction, it will perform a hot backup and not block
|
||||
your other database reads and writes. It will also use `O_DIRECT` when available
|
||||
to prevent page cache trashing.
|
||||
|
||||
One common use case is to backup over HTTP so you can use tools like `cURL` to
|
||||
do database backups:
|
||||
|
||||
```go
|
||||
func BackupHandleFunc(w http.ResponseWriter, req *http.Request) {
|
||||
err := db.View(func(tx *bolt.Tx) error {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="my.db"`)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(int(tx.Size())))
|
||||
_, err := tx.WriteTo(w)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Then you can backup using this command:
|
||||
|
||||
```sh
|
||||
$ curl http://localhost/backup > my.db
|
||||
```
|
||||
|
||||
Or you can open your browser to `http://localhost/backup` and it will download
|
||||
automatically.
|
||||
|
||||
If you want to backup to another file you can use the `Tx.CopyFile()` helper
|
||||
function.
|
||||
|
||||
|
||||
### Statistics
|
||||
|
||||
The database keeps a running count of many of the internal operations it
|
||||
performs so you can better understand what's going on. By grabbing a snapshot
|
||||
of these stats at two points in time we can see what operations were performed
|
||||
in that time range.
|
||||
|
||||
For example, we could start a goroutine to log stats every 10 seconds:
|
||||
|
||||
```go
|
||||
go func() {
|
||||
// Grab the initial stats.
|
||||
prev := db.Stats()
|
||||
|
||||
for {
|
||||
// Wait for 10s.
|
||||
time.Sleep(10 * time.Second)
|
||||
|
||||
// Grab the current stats and diff them.
|
||||
stats := db.Stats()
|
||||
diff := stats.Sub(&prev)
|
||||
|
||||
// Encode stats to JSON and print to STDERR.
|
||||
json.NewEncoder(os.Stderr).Encode(diff)
|
||||
|
||||
// Save stats for the next loop.
|
||||
prev = stats
|
||||
}
|
||||
}()
|
||||
```
|
||||
|
||||
It's also useful to pipe these stats to a service such as statsd for monitoring
|
||||
or to provide an HTTP endpoint that will perform a fixed-length sample.
|
||||
|
||||
|
||||
### Read-Only Mode
|
||||
|
||||
Sometimes it is useful to create a shared, read-only Bolt database. To this,
|
||||
set the `Options.ReadOnly` flag when opening your database. Read-only mode
|
||||
uses a shared lock to allow multiple processes to read from the database but
|
||||
it will block any processes from opening the database in read-write mode.
|
||||
|
||||
```go
|
||||
db, err := bolt.Open("my.db", 0666, &bolt.Options{ReadOnly: true})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Resources
|
||||
|
||||
For more information on getting started with Bolt, check out the following articles:
|
||||
|
||||
* [Intro to BoltDB: Painless Performant Persistence](http://npf.io/2014/07/intro-to-boltdb-painless-performant-persistence/) by [Nate Finch](https://github.com/natefinch).
|
||||
* [Bolt -- an embedded key/value database for Go](https://www.progville.com/go/bolt-embedded-db-golang/) by Progville
|
||||
|
||||
|
||||
## Comparison with other databases
|
||||
|
||||
### Postgres, MySQL, & other relational databases
|
||||
|
||||
Relational databases structure data into rows and are only accessible through
|
||||
the use of SQL. This approach provides flexibility in how you store and query
|
||||
your data but also incurs overhead in parsing and planning SQL statements. Bolt
|
||||
accesses all data by a byte slice key. This makes Bolt fast to read and write
|
||||
data by key but provides no built-in support for joining values together.
|
||||
|
||||
Most relational databases (with the exception of SQLite) are standalone servers
|
||||
that run separately from your application. This gives your systems
|
||||
flexibility to connect multiple application servers to a single database
|
||||
server but also adds overhead in serializing and transporting data over the
|
||||
network. Bolt runs as a library included in your application so all data access
|
||||
has to go through your application's process. This brings data closer to your
|
||||
application but limits multi-process access to the data.
|
||||
|
||||
|
||||
### LevelDB, RocksDB
|
||||
|
||||
LevelDB and its derivatives (RocksDB, HyperLevelDB) are similar to Bolt in that
|
||||
they are libraries bundled into the application, however, their underlying
|
||||
structure is a log-structured merge-tree (LSM tree). An LSM tree optimizes
|
||||
random writes by using a write ahead log and multi-tiered, sorted files called
|
||||
SSTables. Bolt uses a B+tree internally and only a single file. Both approaches
|
||||
have trade offs.
|
||||
|
||||
If you require a high random write throughput (>10,000 w/sec) or you need to use
|
||||
spinning disks then LevelDB could be a good choice. If your application is
|
||||
read-heavy or does a lot of range scans then Bolt could be a good choice.
|
||||
|
||||
One other important consideration is that LevelDB does not have transactions.
|
||||
It supports batch writing of key/values pairs and it supports read snapshots
|
||||
but it will not give you the ability to do a compare-and-swap operation safely.
|
||||
Bolt supports fully serializable ACID transactions.
|
||||
|
||||
|
||||
### LMDB
|
||||
|
||||
Bolt was originally a port of LMDB so it is architecturally similar. Both use
|
||||
a B+tree, have ACID semantics with fully serializable transactions, and support
|
||||
lock-free MVCC using a single writer and multiple readers.
|
||||
|
||||
The two projects have somewhat diverged. LMDB heavily focuses on raw performance
|
||||
while Bolt has focused on simplicity and ease of use. For example, LMDB allows
|
||||
several unsafe actions such as direct writes for the sake of performance. Bolt
|
||||
opts to disallow actions which can leave the database in a corrupted state. The
|
||||
only exception to this in Bolt is `DB.NoSync`.
|
||||
|
||||
There are also a few differences in API. LMDB requires a maximum mmap size when
|
||||
opening an `mdb_env` whereas Bolt will handle incremental mmap resizing
|
||||
automatically. LMDB overloads the getter and setter functions with multiple
|
||||
flags whereas Bolt splits these specialized cases into their own functions.
|
||||
|
||||
|
||||
## Caveats & Limitations
|
||||
|
||||
It's important to pick the right tool for the job and Bolt is no exception.
|
||||
Here are a few things to note when evaluating and using Bolt:
|
||||
|
||||
* Bolt is good for read intensive workloads. Sequential write performance is
|
||||
also fast but random writes can be slow. You can add a write-ahead log or
|
||||
[transaction coalescer](https://github.com/boltdb/coalescer) in front of Bolt
|
||||
to mitigate this issue.
|
||||
|
||||
* Bolt uses a B+tree internally so there can be a lot of random page access.
|
||||
SSDs provide a significant performance boost over spinning disks.
|
||||
|
||||
* Try to avoid long running read transactions. Bolt uses copy-on-write so
|
||||
old pages cannot be reclaimed while an old transaction is using them.
|
||||
|
||||
* Byte slices returned from Bolt are only valid during a transaction. Once the
|
||||
transaction has been committed or rolled back then the memory they point to
|
||||
can be reused by a new page or can be unmapped from virtual memory and you'll
|
||||
see an `unexpected fault address` panic when accessing it.
|
||||
|
||||
* Be careful when using `Bucket.FillPercent`. Setting a high fill percent for
|
||||
buckets that have random inserts will cause your database to have very poor
|
||||
page utilization.
|
||||
|
||||
* Use larger buckets in general. Smaller buckets causes poor page utilization
|
||||
once they become larger than the page size (typically 4KB).
|
||||
|
||||
* Bulk loading a lot of random writes into a new bucket can be slow as the
|
||||
page will not split until the transaction is committed. Randomly inserting
|
||||
more than 100,000 key/value pairs into a single new bucket in a single
|
||||
transaction is not advised.
|
||||
|
||||
* Bolt uses a memory-mapped file so the underlying operating system handles the
|
||||
caching of the data. Typically, the OS will cache as much of the file as it
|
||||
can in memory and will release memory as needed to other processes. This means
|
||||
that Bolt can show very high memory usage when working with large databases.
|
||||
However, this is expected and the OS will release memory as needed. Bolt can
|
||||
handle databases much larger than the available physical RAM.
|
||||
|
||||
* The data structures in the Bolt database are memory mapped so the data file
|
||||
will be endian specific. This means that you cannot copy a Bolt file from a
|
||||
little endian machine to a big endian machine and have it work. For most
|
||||
users this is not a concern since most modern CPUs are little endian.
|
||||
|
||||
* Because of the way pages are laid out on disk, Bolt cannot truncate data files
|
||||
and return free pages back to the disk. Instead, Bolt maintains a free list
|
||||
of unused pages within its data file. These free pages can be reused by later
|
||||
transactions. This works well for many use cases as databases generally tend
|
||||
to grow. However, it's important to note that deleting large chunks of data
|
||||
will not allow you to reclaim that space on disk.
|
||||
|
||||
For more information on page allocation, [see this comment][page-allocation].
|
||||
|
||||
[page-allocation]: https://github.com/boltdb/bolt/issues/308#issuecomment-74811638
|
||||
|
||||
|
||||
## Other Projects Using Bolt
|
||||
|
||||
Below is a list of public, open source projects that use Bolt:
|
||||
|
||||
* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard.
|
||||
* [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside.
|
||||
* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb.
|
||||
* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics.
|
||||
* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects.
|
||||
* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday.
|
||||
* [ChainStore](https://github.com/nulayer/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations.
|
||||
* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite.
|
||||
* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin".
|
||||
* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka.
|
||||
* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed.
|
||||
* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt.
|
||||
* [photosite/session](http://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site.
|
||||
* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage.
|
||||
* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters.
|
||||
* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend.
|
||||
* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend.
|
||||
* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server.
|
||||
* [SkyDB](https://github.com/skydb/sky) - Behavioral analytics database.
|
||||
* [Seaweed File System](https://github.com/chrislusf/weed-fs) - Highly scalable distributed key~file system with O(1) disk read.
|
||||
* [InfluxDB](http://influxdb.com) - Scalable datastore for metrics, events, and real-time analytics.
|
||||
* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data.
|
||||
* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system.
|
||||
* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware.
|
||||
* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistant, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs.
|
||||
* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems.
|
||||
|
||||
If you are using Bolt in a project please send a pull request to add it to the list.
|
||||
Generated
Vendored
+138
@@ -0,0 +1,138 @@
|
||||
package bolt
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Batch calls fn as part of a batch. It behaves similar to Update,
|
||||
// except:
|
||||
//
|
||||
// 1. concurrent Batch calls can be combined into a single Bolt
|
||||
// transaction.
|
||||
//
|
||||
// 2. the function passed to Batch may be called multiple times,
|
||||
// regardless of whether it returns error or not.
|
||||
//
|
||||
// This means that Batch function side effects must be idempotent and
|
||||
// take permanent effect only after a successful return is seen in
|
||||
// caller.
|
||||
//
|
||||
// The maximum batch size and delay can be adjusted with DB.MaxBatchSize
|
||||
// and DB.MaxBatchDelay, respectively.
|
||||
//
|
||||
// Batch is only useful when there are multiple goroutines calling it.
|
||||
func (db *DB) Batch(fn func(*Tx) error) error {
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
db.batchMu.Lock()
|
||||
if (db.batch == nil) || (db.batch != nil && len(db.batch.calls) >= db.MaxBatchSize) {
|
||||
// There is no existing batch, or the existing batch is full; start a new one.
|
||||
db.batch = &batch{
|
||||
db: db,
|
||||
}
|
||||
db.batch.timer = time.AfterFunc(db.MaxBatchDelay, db.batch.trigger)
|
||||
}
|
||||
db.batch.calls = append(db.batch.calls, call{fn: fn, err: errCh})
|
||||
if len(db.batch.calls) >= db.MaxBatchSize {
|
||||
// wake up batch, it's ready to run
|
||||
go db.batch.trigger()
|
||||
}
|
||||
db.batchMu.Unlock()
|
||||
|
||||
err := <-errCh
|
||||
if err == trySolo {
|
||||
err = db.Update(fn)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type call struct {
|
||||
fn func(*Tx) error
|
||||
err chan<- error
|
||||
}
|
||||
|
||||
type batch struct {
|
||||
db *DB
|
||||
timer *time.Timer
|
||||
start sync.Once
|
||||
calls []call
|
||||
}
|
||||
|
||||
// trigger runs the batch if it hasn't already been run.
|
||||
func (b *batch) trigger() {
|
||||
b.start.Do(b.run)
|
||||
}
|
||||
|
||||
// run performs the transactions in the batch and communicates results
|
||||
// back to DB.Batch.
|
||||
func (b *batch) run() {
|
||||
b.db.batchMu.Lock()
|
||||
b.timer.Stop()
|
||||
// Make sure no new work is added to this batch, but don't break
|
||||
// other batches.
|
||||
if b.db.batch == b {
|
||||
b.db.batch = nil
|
||||
}
|
||||
b.db.batchMu.Unlock()
|
||||
|
||||
retry:
|
||||
for len(b.calls) > 0 {
|
||||
var failIdx = -1
|
||||
err := b.db.Update(func(tx *Tx) error {
|
||||
for i, c := range b.calls {
|
||||
if err := safelyCall(c.fn, tx); err != nil {
|
||||
failIdx = i
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if failIdx >= 0 {
|
||||
// take the failing transaction out of the batch. it's
|
||||
// safe to shorten b.calls here because db.batch no longer
|
||||
// points to us, and we hold the mutex anyway.
|
||||
c := b.calls[failIdx]
|
||||
b.calls[failIdx], b.calls = b.calls[len(b.calls)-1], b.calls[:len(b.calls)-1]
|
||||
// tell the submitter re-run it solo, continue with the rest of the batch
|
||||
c.err <- trySolo
|
||||
continue retry
|
||||
}
|
||||
|
||||
// pass success, or bolt internal errors, to all callers
|
||||
for _, c := range b.calls {
|
||||
if c.err != nil {
|
||||
c.err <- err
|
||||
}
|
||||
}
|
||||
break retry
|
||||
}
|
||||
}
|
||||
|
||||
// trySolo is a special sentinel error value used for signaling that a
|
||||
// transaction function should be re-run. It should never be seen by
|
||||
// callers.
|
||||
var trySolo = errors.New("batch function returned an error and should be re-run solo")
|
||||
|
||||
type panicked struct {
|
||||
reason interface{}
|
||||
}
|
||||
|
||||
func (p panicked) Error() string {
|
||||
if err, ok := p.reason.(error); ok {
|
||||
return err.Error()
|
||||
}
|
||||
return fmt.Sprintf("panic: %v", p.reason)
|
||||
}
|
||||
|
||||
func safelyCall(fn func(*Tx) error, tx *Tx) (err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
err = panicked{p}
|
||||
}
|
||||
}()
|
||||
return fn(tx)
|
||||
}
|
||||
Generated
Vendored
+170
@@ -0,0 +1,170 @@
|
||||
package bolt_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/fnv"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"
|
||||
)
|
||||
|
||||
func validateBatchBench(b *testing.B, db *TestDB) {
|
||||
var rollback = errors.New("sentinel error to cause rollback")
|
||||
validate := func(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte("bench"))
|
||||
h := fnv.New32a()
|
||||
buf := make([]byte, 4)
|
||||
for id := uint32(0); id < 1000; id++ {
|
||||
binary.LittleEndian.PutUint32(buf, id)
|
||||
h.Reset()
|
||||
h.Write(buf[:])
|
||||
k := h.Sum(nil)
|
||||
v := bucket.Get(k)
|
||||
if v == nil {
|
||||
b.Errorf("not found id=%d key=%x", id, k)
|
||||
continue
|
||||
}
|
||||
if g, e := v, []byte("filler"); !bytes.Equal(g, e) {
|
||||
b.Errorf("bad value for id=%d key=%x: %s != %q", id, k, g, e)
|
||||
}
|
||||
if err := bucket.Delete(k); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// should be empty now
|
||||
c := bucket.Cursor()
|
||||
for k, v := c.First(); k != nil; k, v = c.Next() {
|
||||
b.Errorf("unexpected key: %x = %q", k, v)
|
||||
}
|
||||
return rollback
|
||||
}
|
||||
if err := db.Update(validate); err != nil && err != rollback {
|
||||
b.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDBBatchAutomatic(b *testing.B) {
|
||||
db := NewTestDB()
|
||||
defer db.Close()
|
||||
db.MustCreateBucket([]byte("bench"))
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for round := 0; round < 1000; round++ {
|
||||
wg.Add(1)
|
||||
|
||||
go func(id uint32) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
|
||||
h := fnv.New32a()
|
||||
buf := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(buf, id)
|
||||
h.Write(buf[:])
|
||||
k := h.Sum(nil)
|
||||
insert := func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("bench"))
|
||||
return b.Put(k, []byte("filler"))
|
||||
}
|
||||
if err := db.Batch(insert); err != nil {
|
||||
b.Error(err)
|
||||
return
|
||||
}
|
||||
}(uint32(round))
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
validateBatchBench(b, db)
|
||||
}
|
||||
|
||||
func BenchmarkDBBatchSingle(b *testing.B) {
|
||||
db := NewTestDB()
|
||||
defer db.Close()
|
||||
db.MustCreateBucket([]byte("bench"))
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for round := 0; round < 1000; round++ {
|
||||
wg.Add(1)
|
||||
go func(id uint32) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
|
||||
h := fnv.New32a()
|
||||
buf := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(buf, id)
|
||||
h.Write(buf[:])
|
||||
k := h.Sum(nil)
|
||||
insert := func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("bench"))
|
||||
return b.Put(k, []byte("filler"))
|
||||
}
|
||||
if err := db.Update(insert); err != nil {
|
||||
b.Error(err)
|
||||
return
|
||||
}
|
||||
}(uint32(round))
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
validateBatchBench(b, db)
|
||||
}
|
||||
|
||||
func BenchmarkDBBatchManual10x100(b *testing.B) {
|
||||
db := NewTestDB()
|
||||
defer db.Close()
|
||||
db.MustCreateBucket([]byte("bench"))
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for major := 0; major < 10; major++ {
|
||||
wg.Add(1)
|
||||
go func(id uint32) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
|
||||
insert100 := func(tx *bolt.Tx) error {
|
||||
h := fnv.New32a()
|
||||
buf := make([]byte, 4)
|
||||
for minor := uint32(0); minor < 100; minor++ {
|
||||
binary.LittleEndian.PutUint32(buf, uint32(id*100+minor))
|
||||
h.Reset()
|
||||
h.Write(buf[:])
|
||||
k := h.Sum(nil)
|
||||
b := tx.Bucket([]byte("bench"))
|
||||
if err := b.Put(k, []byte("filler")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := db.Update(insert100); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}(uint32(major))
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
validateBatchBench(b, db)
|
||||
}
|
||||
Generated
Vendored
+148
@@ -0,0 +1,148 @@
|
||||
package bolt_test
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
|
||||
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"
|
||||
)
|
||||
|
||||
// Set this to see how the counts are actually updated.
|
||||
const verbose = false
|
||||
|
||||
// Counter updates a counter in Bolt for every URL path requested.
|
||||
type counter struct {
|
||||
db *bolt.DB
|
||||
}
|
||||
|
||||
func (c counter) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
|
||||
// Communicates the new count from a successful database
|
||||
// transaction.
|
||||
var result uint64
|
||||
|
||||
increment := func(tx *bolt.Tx) error {
|
||||
b, err := tx.CreateBucketIfNotExists([]byte("hits"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := []byte(req.URL.String())
|
||||
// Decode handles key not found for us.
|
||||
count := decode(b.Get(key)) + 1
|
||||
b.Put(key, encode(count))
|
||||
// All good, communicate new count.
|
||||
result = count
|
||||
return nil
|
||||
}
|
||||
if err := c.db.Batch(increment); err != nil {
|
||||
http.Error(rw, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
|
||||
if verbose {
|
||||
log.Printf("server: %s: %d", req.URL.String(), result)
|
||||
}
|
||||
|
||||
rw.Header().Set("Content-Type", "application/octet-stream")
|
||||
fmt.Fprintf(rw, "%d\n", result)
|
||||
}
|
||||
|
||||
func client(id int, base string, paths []string) error {
|
||||
// Process paths in random order.
|
||||
rng := rand.New(rand.NewSource(int64(id)))
|
||||
permutation := rng.Perm(len(paths))
|
||||
|
||||
for i := range paths {
|
||||
path := paths[permutation[i]]
|
||||
resp, err := http.Get(base + path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
buf, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if verbose {
|
||||
log.Printf("client: %s: %s", path, buf)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ExampleDB_Batch() {
|
||||
// Open the database.
|
||||
db, _ := bolt.Open(tempfile(), 0666, nil)
|
||||
defer os.Remove(db.Path())
|
||||
defer db.Close()
|
||||
|
||||
// Start our web server
|
||||
count := counter{db}
|
||||
srv := httptest.NewServer(count)
|
||||
defer srv.Close()
|
||||
|
||||
// Decrease the batch size to make things more interesting.
|
||||
db.MaxBatchSize = 3
|
||||
|
||||
// Get every path multiple times concurrently.
|
||||
const clients = 10
|
||||
paths := []string{
|
||||
"/foo",
|
||||
"/bar",
|
||||
"/baz",
|
||||
"/quux",
|
||||
"/thud",
|
||||
"/xyzzy",
|
||||
}
|
||||
errors := make(chan error, clients)
|
||||
for i := 0; i < clients; i++ {
|
||||
go func(id int) {
|
||||
errors <- client(id, srv.URL, paths)
|
||||
}(i)
|
||||
}
|
||||
// Check all responses to make sure there's no error.
|
||||
for i := 0; i < clients; i++ {
|
||||
if err := <-errors; err != nil {
|
||||
fmt.Printf("client error: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Check the final result
|
||||
db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("hits"))
|
||||
c := b.Cursor()
|
||||
for k, v := c.First(); k != nil; k, v = c.Next() {
|
||||
fmt.Printf("hits to %s: %d\n", k, decode(v))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Output:
|
||||
// hits to /bar: 10
|
||||
// hits to /baz: 10
|
||||
// hits to /foo: 10
|
||||
// hits to /quux: 10
|
||||
// hits to /thud: 10
|
||||
// hits to /xyzzy: 10
|
||||
}
|
||||
|
||||
// encode marshals a counter.
|
||||
func encode(n uint64) []byte {
|
||||
buf := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(buf, n)
|
||||
return buf
|
||||
}
|
||||
|
||||
// decode unmarshals a counter. Nil buffers are decoded as 0.
|
||||
func decode(buf []byte) uint64 {
|
||||
if buf == nil {
|
||||
return 0
|
||||
}
|
||||
return binary.BigEndian.Uint64(buf)
|
||||
}
|
||||
Generated
Vendored
+167
@@ -0,0 +1,167 @@
|
||||
package bolt_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"
|
||||
)
|
||||
|
||||
// Ensure two functions can perform updates in a single batch.
|
||||
func TestDB_Batch(t *testing.T) {
|
||||
db := NewTestDB()
|
||||
defer db.Close()
|
||||
db.MustCreateBucket([]byte("widgets"))
|
||||
|
||||
// Iterate over multiple updates in separate goroutines.
|
||||
n := 2
|
||||
ch := make(chan error)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
ch <- db.Batch(func(tx *bolt.Tx) error {
|
||||
return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{})
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Check all responses to make sure there's no error.
|
||||
for i := 0; i < n; i++ {
|
||||
if err := <-ch; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure data is correct.
|
||||
db.MustView(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("widgets"))
|
||||
for i := 0; i < n; i++ {
|
||||
if v := b.Get(u64tob(uint64(i))); v == nil {
|
||||
t.Errorf("key not found: %d", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func TestDB_Batch_Panic(t *testing.T) {
|
||||
db := NewTestDB()
|
||||
defer db.Close()
|
||||
|
||||
var sentinel int
|
||||
var bork = &sentinel
|
||||
var problem interface{}
|
||||
var err error
|
||||
|
||||
// Execute a function inside a batch that panics.
|
||||
func() {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
problem = p
|
||||
}
|
||||
}()
|
||||
err = db.Batch(func(tx *bolt.Tx) error {
|
||||
panic(bork)
|
||||
})
|
||||
}()
|
||||
|
||||
// Verify there is no error.
|
||||
if g, e := err, error(nil); g != e {
|
||||
t.Fatalf("wrong error: %v != %v", g, e)
|
||||
}
|
||||
// Verify the panic was captured.
|
||||
if g, e := problem, bork; g != e {
|
||||
t.Fatalf("wrong error: %v != %v", g, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDB_BatchFull(t *testing.T) {
|
||||
db := NewTestDB()
|
||||
defer db.Close()
|
||||
db.MustCreateBucket([]byte("widgets"))
|
||||
|
||||
const size = 3
|
||||
// buffered so we never leak goroutines
|
||||
ch := make(chan error, size)
|
||||
put := func(i int) {
|
||||
ch <- db.Batch(func(tx *bolt.Tx) error {
|
||||
return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{})
|
||||
})
|
||||
}
|
||||
|
||||
db.MaxBatchSize = size
|
||||
// high enough to never trigger here
|
||||
db.MaxBatchDelay = 1 * time.Hour
|
||||
|
||||
go put(1)
|
||||
go put(2)
|
||||
|
||||
// Give the batch a chance to exhibit bugs.
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// not triggered yet
|
||||
select {
|
||||
case <-ch:
|
||||
t.Fatalf("batch triggered too early")
|
||||
default:
|
||||
}
|
||||
|
||||
go put(3)
|
||||
|
||||
// Check all responses to make sure there's no error.
|
||||
for i := 0; i < size; i++ {
|
||||
if err := <-ch; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure data is correct.
|
||||
db.MustView(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("widgets"))
|
||||
for i := 1; i <= size; i++ {
|
||||
if v := b.Get(u64tob(uint64(i))); v == nil {
|
||||
t.Errorf("key not found: %d", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func TestDB_BatchTime(t *testing.T) {
|
||||
db := NewTestDB()
|
||||
defer db.Close()
|
||||
db.MustCreateBucket([]byte("widgets"))
|
||||
|
||||
const size = 1
|
||||
// buffered so we never leak goroutines
|
||||
ch := make(chan error, size)
|
||||
put := func(i int) {
|
||||
ch <- db.Batch(func(tx *bolt.Tx) error {
|
||||
return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{})
|
||||
})
|
||||
}
|
||||
|
||||
db.MaxBatchSize = 1000
|
||||
db.MaxBatchDelay = 0
|
||||
|
||||
go put(1)
|
||||
|
||||
// Batch must trigger by time alone.
|
||||
|
||||
// Check all responses to make sure there's no error.
|
||||
for i := 0; i < size; i++ {
|
||||
if err := <-ch; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure data is correct.
|
||||
db.MustView(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("widgets"))
|
||||
for i := 1; i <= size; i++ {
|
||||
if v := b.Get(u64tob(uint64(i))); v == nil {
|
||||
t.Errorf("key not found: %d", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
Generated
Vendored
+7
@@ -0,0 +1,7 @@
|
||||
package bolt
|
||||
|
||||
// maxMapSize represents the largest mmap size supported by Bolt.
|
||||
const maxMapSize = 0x7FFFFFFF // 2GB
|
||||
|
||||
// maxAllocSize is the size used when creating array pointers.
|
||||
const maxAllocSize = 0xFFFFFFF
|
||||
Generated
Vendored
+7
@@ -0,0 +1,7 @@
|
||||
package bolt
|
||||
|
||||
// maxMapSize represents the largest mmap size supported by Bolt.
|
||||
const maxMapSize = 0xFFFFFFFFFFFF // 256TB
|
||||
|
||||
// maxAllocSize is the size used when creating array pointers.
|
||||
const maxAllocSize = 0x7FFFFFFF
|
||||
Generated
Vendored
+7
@@ -0,0 +1,7 @@
|
||||
package bolt
|
||||
|
||||
// maxMapSize represents the largest mmap size supported by Bolt.
|
||||
const maxMapSize = 0x7FFFFFFF // 2GB
|
||||
|
||||
// maxAllocSize is the size used when creating array pointers.
|
||||
const maxAllocSize = 0xFFFFFFF
|
||||
Generated
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
package bolt
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
)
|
||||
|
||||
var odirect = syscall.O_DIRECT
|
||||
|
||||
// fdatasync flushes written data to a file descriptor.
|
||||
func fdatasync(db *DB) error {
|
||||
return syscall.Fdatasync(int(db.file.Fd()))
|
||||
}
|
||||
Generated
Vendored
+29
@@ -0,0 +1,29 @@
|
||||
package bolt
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
msAsync = 1 << iota // perform asynchronous writes
|
||||
msSync // perform synchronous writes
|
||||
msInvalidate // invalidate cached data
|
||||
)
|
||||
|
||||
var odirect int
|
||||
|
||||
func msync(db *DB) error {
|
||||
_, _, errno := syscall.Syscall(syscall.SYS_MSYNC, uintptr(unsafe.Pointer(db.data)), uintptr(db.datasz), msInvalidate)
|
||||
if errno != 0 {
|
||||
return errno
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fdatasync(db *DB) error {
|
||||
if db.data != nil {
|
||||
return msync(db)
|
||||
}
|
||||
return db.file.Sync()
|
||||
}
|
||||
Generated
Vendored
+36
@@ -0,0 +1,36 @@
|
||||
package bolt_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// assert fails the test if the condition is false.
|
||||
func assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
|
||||
if !condition {
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
|
||||
tb.FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// ok fails the test if an err is not nil.
|
||||
func ok(tb testing.TB, err error) {
|
||||
if err != nil {
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error())
|
||||
tb.FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
// equals fails the test if exp is not equal to act.
|
||||
func equals(tb testing.TB, exp, act interface{}) {
|
||||
if !reflect.DeepEqual(exp, act) {
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
|
||||
tb.FailNow()
|
||||
}
|
||||
}
|
||||
Generated
Vendored
+100
@@ -0,0 +1,100 @@
|
||||
// +build !windows,!plan9,!solaris
|
||||
|
||||
package bolt
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// flock acquires an advisory lock on a file descriptor.
|
||||
func flock(f *os.File, exclusive bool, timeout time.Duration) error {
|
||||
var t time.Time
|
||||
for {
|
||||
// If we're beyond our timeout then return an error.
|
||||
// This can only occur after we've attempted a flock once.
|
||||
if t.IsZero() {
|
||||
t = time.Now()
|
||||
} else if timeout > 0 && time.Since(t) > timeout {
|
||||
return ErrTimeout
|
||||
}
|
||||
flag := syscall.LOCK_SH
|
||||
if exclusive {
|
||||
flag = syscall.LOCK_EX
|
||||
}
|
||||
|
||||
// Otherwise attempt to obtain an exclusive lock.
|
||||
err := syscall.Flock(int(f.Fd()), flag|syscall.LOCK_NB)
|
||||
if err == nil {
|
||||
return nil
|
||||
} else if err != syscall.EWOULDBLOCK {
|
||||
return err
|
||||
}
|
||||
|
||||
// Wait for a bit and try again.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// funlock releases an advisory lock on a file descriptor.
|
||||
func funlock(f *os.File) error {
|
||||
return syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||
}
|
||||
|
||||
// mmap memory maps a DB's data file.
|
||||
func mmap(db *DB, sz int) error {
|
||||
// Truncate and fsync to ensure file size metadata is flushed.
|
||||
// https://github.com/boltdb/bolt/issues/284
|
||||
if !db.NoGrowSync && !db.readOnly {
|
||||
if err := db.file.Truncate(int64(sz)); err != nil {
|
||||
return fmt.Errorf("file resize error: %s", err)
|
||||
}
|
||||
if err := db.file.Sync(); err != nil {
|
||||
return fmt.Errorf("file sync error: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Map the data file to memory.
|
||||
b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Advise the kernel that the mmap is accessed randomly.
|
||||
if err := madvise(b, syscall.MADV_RANDOM); err != nil {
|
||||
return fmt.Errorf("madvise: %s", err)
|
||||
}
|
||||
|
||||
// Save the original byte slice and convert to a byte array pointer.
|
||||
db.dataref = b
|
||||
db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0]))
|
||||
db.datasz = sz
|
||||
return nil
|
||||
}
|
||||
|
||||
// munmap unmaps a DB's data file from memory.
|
||||
func munmap(db *DB) error {
|
||||
// Ignore the unmap if we have no mapped data.
|
||||
if db.dataref == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unmap using the original byte slice.
|
||||
err := syscall.Munmap(db.dataref)
|
||||
db.dataref = nil
|
||||
db.data = nil
|
||||
db.datasz = 0
|
||||
return err
|
||||
}
|
||||
|
||||
// NOTE: This function is copied from stdlib because it is not available on darwin.
|
||||
func madvise(b []byte, advice int) (err error) {
|
||||
_, _, e1 := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), uintptr(advice))
|
||||
if e1 != 0 {
|
||||
err = e1
|
||||
}
|
||||
return
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user