Add vendored dependancies

This commit is contained in:
Tom Wilkie
2015-10-24 11:19:50 +01:00
committed by Tom Wilkie
parent e8c96a0242
commit bb755979b1
2437 changed files with 703083 additions and 0 deletions
+13
View File
@@ -0,0 +1,13 @@
include $(GOROOT)/src/Make.inc
TARG=bitbucket.org/ww/goautoneg
GOFILES=autoneg.go
include $(GOROOT)/src/Make.pkg
format:
gofmt -w *.go
docs:
gomake clean
godoc ${TARG} > README.txt
+67
View File
@@ -0,0 +1,67 @@
PACKAGE
package goautoneg
import "bitbucket.org/ww/goautoneg"
HTTP Content-Type Autonegotiation.
The functions in this package implement the behaviour specified in
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
Copyright (c) 2011, Open Knowledge Foundation Ltd.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
Neither the name of the Open Knowledge Foundation Ltd. nor the
names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
FUNCTIONS
func Negotiate(header string, alternatives []string) (content_type string)
Negotiate the most appropriate content_type given the accept header
and a list of alternatives.
func ParseAccept(header string) (accept []Accept)
Parse an Accept Header string returning a sorted list
of clauses
TYPES
type Accept struct {
Type, SubType string
Q float32
Params map[string]string
}
Structure to represent a clause in an HTTP Accept Header
SUBDIRECTORIES
.hg
+162
View File
@@ -0,0 +1,162 @@
/*
HTTP Content-Type Autonegotiation.
The functions in this package implement the behaviour specified in
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
Copyright (c) 2011, Open Knowledge Foundation Ltd.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
Neither the name of the Open Knowledge Foundation Ltd. nor the
names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package goautoneg
import (
"sort"
"strconv"
"strings"
)
// Structure to represent a clause in an HTTP Accept Header
type Accept struct {
Type, SubType string
Q float64
Params map[string]string
}
// For internal use, so that we can use the sort interface
type accept_slice []Accept
func (accept accept_slice) Len() int {
slice := []Accept(accept)
return len(slice)
}
func (accept accept_slice) Less(i, j int) bool {
slice := []Accept(accept)
ai, aj := slice[i], slice[j]
if ai.Q > aj.Q {
return true
}
if ai.Type != "*" && aj.Type == "*" {
return true
}
if ai.SubType != "*" && aj.SubType == "*" {
return true
}
return false
}
func (accept accept_slice) Swap(i, j int) {
slice := []Accept(accept)
slice[i], slice[j] = slice[j], slice[i]
}
// Parse an Accept Header string returning a sorted list
// of clauses
func ParseAccept(header string) (accept []Accept) {
parts := strings.Split(header, ",")
accept = make([]Accept, 0, len(parts))
for _, part := range parts {
part := strings.Trim(part, " ")
a := Accept{}
a.Params = make(map[string]string)
a.Q = 1.0
mrp := strings.Split(part, ";")
media_range := mrp[0]
sp := strings.Split(media_range, "/")
a.Type = strings.Trim(sp[0], " ")
switch {
case len(sp) == 1 && a.Type == "*":
a.SubType = "*"
case len(sp) == 2:
a.SubType = strings.Trim(sp[1], " ")
default:
continue
}
if len(mrp) == 1 {
accept = append(accept, a)
continue
}
for _, param := range mrp[1:] {
sp := strings.SplitN(param, "=", 2)
if len(sp) != 2 {
continue
}
token := strings.Trim(sp[0], " ")
if token == "q" {
a.Q, _ = strconv.ParseFloat(sp[1], 32)
} else {
a.Params[token] = strings.Trim(sp[1], " ")
}
}
accept = append(accept, a)
}
slice := accept_slice(accept)
sort.Sort(slice)
return
}
// Negotiate the most appropriate content_type given the accept header
// and a list of alternatives.
func Negotiate(header string, alternatives []string) (content_type string) {
asp := make([][]string, 0, len(alternatives))
for _, ctype := range alternatives {
asp = append(asp, strings.SplitN(ctype, "/", 2))
}
for _, clause := range ParseAccept(header) {
for i, ctsp := range asp {
if clause.Type == ctsp[0] && clause.SubType == ctsp[1] {
content_type = alternatives[i]
return
}
if clause.Type == ctsp[0] && clause.SubType == "*" {
content_type = alternatives[i]
return
}
if clause.Type == "*" && clause.SubType == "*" {
content_type = alternatives[i]
return
}
}
}
return
}
+33
View File
@@ -0,0 +1,33 @@
package goautoneg
import (
"testing"
)
var chrome = "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"
func TestParseAccept(t *testing.T) {
alternatives := []string{"text/html", "image/png"}
content_type := Negotiate(chrome, alternatives)
if content_type != "image/png" {
t.Errorf("got %s expected image/png", content_type)
}
alternatives = []string{"text/html", "text/plain", "text/n3"}
content_type = Negotiate(chrome, alternatives)
if content_type != "text/html" {
t.Errorf("got %s expected text/html", content_type)
}
alternatives = []string{"text/n3", "text/plain"}
content_type = Negotiate(chrome, alternatives)
if content_type != "text/plain" {
t.Errorf("got %s expected text/plain", content_type)
}
alternatives = []string{"text/n3", "application/rdf+xml"}
content_type = Negotiate(chrome, alternatives)
if content_type != "text/n3" {
t.Errorf("got %s expected text/n3", content_type)
}
}
+12
View File
@@ -0,0 +1,12 @@
Copyright (c) 2013, Martin Angers
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+83
View File
@@ -0,0 +1,83 @@
# Ghost
Ghost is a web development library loosely inspired by node's [Connect library][connect]. It provides a number of simple, single-responsibility HTTP handlers that can be combined to build a full-featured web server, and a generic template engine integration interface.
It stays close to the metal, not abstracting Go's standard library away. As a matter of fact, any stdlib handler can be used with Ghost's handlers, they simply are `net/http.Handler`'s.
## Installation and documentation
`go get github.com/PuerkitoBio/ghost`
[API reference][godoc]
*Status* : Still under development, things will change.
## Example
See the /ghostest directory for a complete working example of a website built with Ghost. It shows all handlers and template support of Ghost.
## Handlers
Ghost offers the following handlers:
* BasicAuthHandler : basic authentication support.
* ContextHandler : key-value map provider for the duration of the request.
* FaviconHandler : simple and efficient favicon renderer.
* GZIPHandler : gzip-compresser for the body of the response.
* LogHandler : fully customizable request logger.
* PanicHandler : panic-catching handler to control the error response.
* SessionHandler : store-agnostic server-side session provider.
* StaticHandler : convenience handler that wraps a call to `net/http.ServeFile`.
Two stores are provided for the session persistence, `MemoryStore`, an in-memory map that is not suited for production environment, and `RedisStore`, a more robust and scalable [redigo][]-based Redis store. Because of the generic `SessionStore` interface, custom stores can easily be created as needed.
The `handlers` package also offers the `ChainableHandler` interface, which supports combining HTTP handlers in a sequential fashion, and the `ChainHandlers()` function that creates a new handler from the sequential combination of any number of handlers.
As a convenience, all functions that take a `http.Handler` as argument also have a corresponding function with the `Func` suffix that take a `http.HandlerFunc` instead as argument. This saves the type-cast when a simple handler function is passed (for example, `SessionHandler()` and `SessionHandlerFunc()`).
### Handlers Design
The HTTP handlers such as Basic Auth and Context need to store some state information to provide their functionality. Instead of using variables and a mutex to control shared access, Ghost augments the `http.ResponseWriter` interface that is part of the Handler's `ServeHTTP()` function signature. Because this instance is unique for each request and is not shared, there is no locking involved to access the state information.
However, when combining such handlers, Ghost needs a way to move through the chain of augmented ResponseWriters. This is why these *augmented writers* need to implement the `WrapWriter` interface. A single method is required, `WrappedWriter() http.ResponseWriter`, which returns the wrapped ResponseWriter.
And to get back a specific augmented writer, the `GetResponseWriter()` function is provided. It takes a ResponseWriter and a predicate function as argument, and returns the requested specific writer using the *comma-ok* pattern. Example, for the session writer:
```Go
func getSessionWriter(w http.ResponseWriter) (*sessResponseWriter, bool) {
ss, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
_, ok := tst.(*sessResponseWriter)
return ok
})
if ok {
return ss.(*sessResponseWriter), true
}
return nil, false
}
```
Ghost does not provide a muxer, there are already many great ones available, but I would recommend Go's native `http.ServeMux` or [pat][] because it has great features and plays well with Ghost's design. Gorilla's muxer is very popular, but since it depends on Gorilla's (mutex-based) context provider, this is redundant with Ghost's context.
## Templates
Ghost supports the following template engines:
* Go's native templates (needs work, at the moment does not work with nested templates)
* [Amber][]
TODO : Go's mustache implementation.
### Templates Design
The template engines can be registered much in the same way as database drivers, just by importing for side effects (using `_ "import/path"`). The `init()` function of the template engine's package registers the template compiler with the correct file extension, and the engine can be used.
## License
The [BSD 3-Clause license][lic].
[connect]: https://github.com/senchalabs/connect
[godoc]: http://godoc.org/github.com/PuerkitoBio/ghost
[lic]: http://opensource.org/licenses/BSD-3-Clause
[redigo]: https://github.com/garyburd/redigo
[pat]: https://github.com/bmizerany/pat
[amber]: https://github.com/eknkc/amber
+12
View File
@@ -0,0 +1,12 @@
package ghost
import (
"log"
)
// Logging function, defaults to Go's native log.Printf function. The idea to use
// this instead of a *log.Logger struct is that it can be set to any of log.{Printf,Fatalf, Panicf},
// but also to more flexible userland loggers like SeeLog (https://github.com/cihub/seelog).
// It could be set, for example, to SeeLog's Debugf function. Any function with the
// signature func(fmt string, params ...interface{}).
var LogFn = log.Printf
+22
View File
@@ -0,0 +1,22 @@
<html>
<head>
<title>Ghost Test</title>
<link type="text/css" rel="stylesheet" href="/public/styles.css">
<link type="text/css" rel="stylesheet" href="/public/bootstrap-combined.min.css">
</head>
<body>
<h1>Welcome to Ghost Test</h1>
<img src="/public/logo.png" alt="peace" />
<ol>
<li><a href="/session">Session</a></li>
<li><a href="/session/auth">Authenticated Session</a></li>
<li><a href="/context">Chained Context</a></li>
<li><a href="/panic">Panic</a></li>
<li><a href="/public/styles.css">Styles.css</a></li>
<li><a href="/public/jquery-2.0.0.min.js">JQuery</a></li>
<li><a href="/public/logo.png">Logo</a></li>
</ol>
<script src="/public/jquery-2.0.0.min.js"></script>
</body>
</html>
+169
View File
@@ -0,0 +1,169 @@
// Ghostest is an interactive end-to-end Web site application to test
// the ghost packages. It serves the following URLs, with the specified
// features (handlers):
//
// / : panic;log;gzip;static; -> serve file index.html
// /public/styles.css : panic;log;gzip;StripPrefix;FileServer; -> serve directory public/
// /public/script.js : panic;log;gzip;StripPrefix;FileServer; -> serve directory public/
// /public/logo.pn : panic;log;gzip;StripPrefix;FileServer; -> serve directory public/
// /session : panic;log;gzip;session;context;Custom; -> serve dynamic Go template
// /session/auth : panic;log;gzip;session;context;basicAuth;Custom; -> serve dynamic template
// /panic : panic;log;gzip;Custom; -> panics
// /context : panic;log;gzip;context;Custom1;Custom2; -> serve dynamic Amber template
package main
import (
"log"
"net/http"
"time"
"github.com/PuerkitoBio/ghost/handlers"
"github.com/PuerkitoBio/ghost/templates"
_ "github.com/PuerkitoBio/ghost/templates/amber"
_ "github.com/PuerkitoBio/ghost/templates/gotpl"
"github.com/bmizerany/pat"
)
const (
sessionPageTitle = "Session Page"
sessionPageAuthTitle = "Authenticated Session Page"
sessionPageKey = "txt"
contextPageKey = "time"
sessionExpiration = 10 // Session expires after 10 seconds
)
var (
// Create the common session store and secret
memStore = handlers.NewMemoryStore(1)
secret = "testimony of the ancients"
)
// The struct used to pass data to the session template.
type sessionPageInfo struct {
SessionID string
Title string
Text string
}
// Authenticate the Basic Auth credentials.
func authenticate(u, p string) (interface{}, bool) {
if u == "user" && p == "pwd" {
return u + p, true
}
return nil, false
}
// Handle the session page requests.
func sessionPageRenderer(w handlers.GhostWriter, r *http.Request) {
var (
txt interface{}
data sessionPageInfo
title string
)
ssn := w.Session()
if r.Method == "GET" {
txt = ssn.Data[sessionPageKey]
} else {
txt = r.FormValue(sessionPageKey)
ssn.Data[sessionPageKey] = txt
}
if r.URL.Path == "/session/auth" {
title = sessionPageAuthTitle
} else {
title = sessionPageTitle
}
if txt != nil {
data = sessionPageInfo{ssn.ID(), title, txt.(string)}
} else {
data = sessionPageInfo{ssn.ID(), title, "[nil]"}
}
err := templates.Render("templates/session.tmpl", w, data)
if err != nil {
panic(err)
}
}
// Prepare the context value for the chained handlers context page.
func setContext(w handlers.GhostWriter, r *http.Request) {
w.Context()[contextPageKey] = time.Now().String()
}
// Retrieve the context value and render the chained handlers context page.
func renderContextPage(w handlers.GhostWriter, r *http.Request) {
err := templates.Render("templates/amber/context.amber",
w, &struct{ Val string }{w.Context()[contextPageKey].(string)})
if err != nil {
panic(err)
}
}
// Prepare the web server and kick it off.
func main() {
// Blank the default logger's prefixes
log.SetFlags(0)
// Compile the dynamic templates (native Go templates and Amber
// templates are both registered via the for-side-effects-only imports)
err := templates.CompileDir("./templates/")
if err != nil {
panic(err)
}
// Set the simple routes for static files
mux := pat.New()
mux.Get("/", handlers.StaticFileHandler("./index.html"))
mux.Get("/public/", http.StripPrefix("/public/", http.FileServer(http.Dir("./public/"))))
// Set the more complex routes for session handling and dynamic page (same
// handler is used for both GET and POST).
ssnOpts := handlers.NewSessionOptions(memStore, secret)
ssnOpts.CookieTemplate.MaxAge = sessionExpiration
hSsn := handlers.SessionHandler(
handlers.ContextHandlerFunc(
handlers.GhostHandlerFunc(sessionPageRenderer),
1),
ssnOpts)
mux.Get("/session", hSsn)
mux.Post("/session", hSsn)
hAuthSsn := handlers.BasicAuthHandler(hSsn, authenticate, "")
mux.Get("/session/auth", hAuthSsn)
mux.Post("/session/auth", hAuthSsn)
// Set the handler for the chained context route
mux.Get("/context", handlers.ContextHandler(handlers.ChainHandlerFuncs(
handlers.GhostHandlerFunc(setContext),
handlers.GhostHandlerFunc(renderContextPage)),
1))
// Set the panic route, which simply panics
mux.Get("/panic", http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
panic("explicit panic")
}))
// Combine the top level handlers, that wrap around the muxer.
// Panic is the outermost, so that any panic is caught and responded to with a code 500.
// Log is next, so that every request is logged along with the URL, status code and response time.
// GZIP is then applied, so that content is compressed.
// Finally, the muxer finds the specific handler that applies to the route.
h := handlers.FaviconHandler(
handlers.PanicHandler(
handlers.LogHandler(
handlers.GZIPHandler(
mux,
nil),
handlers.NewLogOptions(nil, handlers.Ltiny)),
nil),
"./public/favicon.ico",
48*time.Hour)
// Assign the combined handler to the server.
http.Handle("/", h)
// Start it up.
if err := http.ListenAndServe(":9000", nil); err != nil {
panic(err)
}
}
File diff suppressed because one or more lines are too long
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

File diff suppressed because one or more lines are too long
Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

+3
View File
@@ -0,0 +1,3 @@
body {
background-color: silver;
}
@@ -0,0 +1,8 @@
!!! 5
html
head
title Chained Context
link[type="text/css"][rel="stylesheet"][href="/public/bootstrap-combined.min.css"]
body
h1 Chained Context
h2 Value found: #{Val}
+28
View File
@@ -0,0 +1,28 @@
<html>
<head>
<title>{{ .Title }}</title>
<link type="text/css" rel="stylesheet" href="/public/styles.css">
<link type="text/css" rel="stylesheet" href="/public/bootstrap-combined.min.css">
</head>
<body>
<h1>Session: {{ .SessionID }}</h1>
<ol>
<li><a href="/">Home</a></li>
<li><a href="/session">Session</a></li>
<li><a href="/session/auth">Authenticated Session</a></li>
<li><a href="/context">Chained Context</a></li>
<li><a href="/panic">Panic</a></li>
<li><a href="/public/styles.css">Styles.css</a></li>
<li><a href="/public/jquery-2.0.0.min.js">JQuery</a></li>
<li><a href="/public/logo.png">Logo</a></li>
</ol>
<h2>Current Value: {{ .Text }}</h2>
<form method="POST">
<input type="text" name="txt" placeholder="some value to save to session"></input>
<button type="submit">Submit</button>
</form>
<script src="/public/jquery-2.0.0.min.js"></script>
</body>
</html>
+123
View File
@@ -0,0 +1,123 @@
package handlers
// Inspired by node.js' Connect library implementation of the basicAuth middleware.
// https://github.com/senchalabs/connect
import (
"bytes"
"encoding/base64"
"fmt"
"net/http"
"strings"
)
// Internal writer that keeps track of the currently authenticated user.
type userResponseWriter struct {
http.ResponseWriter
user interface{}
userName string
}
// Implement the WrapWriter interface.
func (this *userResponseWriter) WrappedWriter() http.ResponseWriter {
return this.ResponseWriter
}
// Writes an unauthorized response to the client, specifying the expected authentication
// information.
func Unauthorized(w http.ResponseWriter, realm string) {
w.Header().Set("Www-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm))
w.WriteHeader(http.StatusUnauthorized)
w.Write([]byte("Unauthorized"))
}
// Writes a bad request response to the client, with an optional message.
func BadRequest(w http.ResponseWriter, msg string) {
w.WriteHeader(http.StatusBadRequest)
if msg == "" {
msg = "Bad Request"
}
w.Write([]byte(msg))
}
// BasicAuthHandlerFunc is the same as BasicAuthHandler, it is just a convenience
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
func BasicAuthHandlerFunc(h http.HandlerFunc,
authFn func(string, string) (interface{}, bool), realm string) http.HandlerFunc {
return BasicAuthHandler(h, authFn, realm)
}
// Returns a Basic Authentication handler, protecting the wrapped handler from
// being accessed if the authentication function is not successful.
func BasicAuthHandler(h http.Handler,
authFn func(string, string) (interface{}, bool), realm string) http.HandlerFunc {
if realm == "" {
realm = "Authorization Required"
}
return func(w http.ResponseWriter, r *http.Request) {
// Self-awareness
if _, ok := GetUser(w); ok {
h.ServeHTTP(w, r)
return
}
authInfo := r.Header.Get("Authorization")
if authInfo == "" {
// No authorization info, return 401
Unauthorized(w, realm)
return
}
parts := strings.Split(authInfo, " ")
if len(parts) != 2 {
BadRequest(w, "Bad authorization header")
return
}
scheme := parts[0]
creds, err := base64.StdEncoding.DecodeString(parts[1])
if err != nil {
BadRequest(w, "Bad credentials encoding")
return
}
index := bytes.Index(creds, []byte(":"))
if scheme != "Basic" || index < 0 {
BadRequest(w, "Bad authorization header")
return
}
user, pwd := string(creds[:index]), string(creds[index+1:])
udata, ok := authFn(user, pwd)
if ok {
// Save user data and continue
uw := &userResponseWriter{w, udata, user}
h.ServeHTTP(uw, r)
} else {
Unauthorized(w, realm)
}
}
}
// Return the currently authenticated user. This is the same data that was returned
// by the authentication function passed to BasicAuthHandler.
func GetUser(w http.ResponseWriter) (interface{}, bool) {
usr, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
_, ok := tst.(*userResponseWriter)
return ok
})
if ok {
return usr.(*userResponseWriter).user, true
}
return nil, false
}
// Return the currently authenticated user name. This is the user name that was
// authenticated for the current request.
func GetUserName(w http.ResponseWriter) (string, bool) {
usr, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
_, ok := tst.(*userResponseWriter)
return ok
})
if ok {
return usr.(*userResponseWriter).userName, true
}
return "", false
}
+62
View File
@@ -0,0 +1,62 @@
package handlers
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
)
func TestUnauth(t *testing.T) {
h := BasicAuthHandler(StaticFileHandler("./testdata/script.js"), func(u, pwd string) (interface{}, bool) {
if u == "me" && pwd == "you" {
return u, true
}
return nil, false
}, "foo")
s := httptest.NewServer(h)
defer s.Close()
res, err := http.Get(s.URL)
if err != nil {
panic(err)
}
assertStatus(http.StatusUnauthorized, res.StatusCode, t)
assertHeader("Www-Authenticate", `Basic realm="foo"`, res, t)
}
func TestGzippedAuth(t *testing.T) {
h := GZIPHandler(BasicAuthHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
usr, ok := GetUser(w)
if assertTrue(ok, "expected authenticated user, got false", t) {
assertTrue(usr.(string) == "meyou", fmt.Sprintf("expected user data to be 'meyou', got '%s'", usr), t)
}
usr, ok = GetUserName(w)
if assertTrue(ok, "expected authenticated user name, got false", t) {
assertTrue(usr == "me", fmt.Sprintf("expected user name to be 'me', got '%s'", usr), t)
}
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte(usr.(string)))
}), func(u, pwd string) (interface{}, bool) {
if u == "me" && pwd == "you" {
return u + pwd, true
}
return nil, false
}, ""), nil)
s := httptest.NewServer(h)
defer s.Close()
req, err := http.NewRequest("GET", "http://me:you@"+s.URL[7:], nil)
if err != nil {
panic(err)
}
req.Header.Set("Accept-Encoding", "gzip")
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertGzippedBody([]byte("me"), res, t)
}
+63
View File
@@ -0,0 +1,63 @@
package handlers
import (
"net/http"
)
// ChainableHandler is a valid Handler interface, and adds the possibility to
// chain other handlers.
type ChainableHandler interface {
http.Handler
Chain(http.Handler) ChainableHandler
ChainFunc(http.HandlerFunc) ChainableHandler
}
// Default implementation of a simple ChainableHandler
type chainHandler struct {
http.Handler
}
func (this *chainHandler) ChainFunc(h http.HandlerFunc) ChainableHandler {
return this.Chain(h)
}
// Implementation of the ChainableHandler interface, calls the chained handler
// after the current one (sequential).
func (this *chainHandler) Chain(h http.Handler) ChainableHandler {
return &chainHandler{
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Add the chained handler after the call to this handler
this.ServeHTTP(w, r)
h.ServeHTTP(w, r)
}),
}
}
// Convert a standard http handler to a chainable handler interface.
func NewChainableHandler(h http.Handler) ChainableHandler {
return &chainHandler{
h,
}
}
// Helper function to chain multiple handler functions in a single call.
func ChainHandlerFuncs(h ...http.HandlerFunc) ChainableHandler {
return &chainHandler{
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for _, v := range h {
v(w, r)
}
}),
}
}
// Helper function to chain multiple handlers in a single call.
func ChainHandlers(h ...http.Handler) ChainableHandler {
return &chainHandler{
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for _, v := range h {
v.ServeHTTP(w, r)
}
}),
}
}
+73
View File
@@ -0,0 +1,73 @@
package handlers
import (
"bytes"
"net/http"
"testing"
)
func TestChaining(t *testing.T) {
var buf bytes.Buffer
a := func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('a')
}
b := func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('b')
}
c := func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('c')
}
f := NewChainableHandler(http.HandlerFunc(a)).Chain(http.HandlerFunc(b)).Chain(http.HandlerFunc(c))
f.ServeHTTP(nil, nil)
if buf.String() != "abc" {
t.Errorf("expected 'abc', got %s", buf.String())
}
}
func TestChainingWithHelperFunc(t *testing.T) {
var buf bytes.Buffer
a := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('a')
})
b := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('b')
})
c := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('c')
})
d := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('d')
})
f := ChainHandlers(a, b, c, d)
f.ServeHTTP(nil, nil)
if buf.String() != "abcd" {
t.Errorf("expected 'abcd', got %s", buf.String())
}
}
func TestChainingMixed(t *testing.T) {
var buf bytes.Buffer
a := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('a')
})
b := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('b')
})
c := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('c')
})
d := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf.WriteRune('d')
})
f := NewChainableHandler(a).Chain(ChainHandlers(b, c)).Chain(d)
f.ServeHTTP(nil, nil)
if buf.String() != "abcd" {
t.Errorf("expected 'abcd', got %s", buf.String())
}
}
+55
View File
@@ -0,0 +1,55 @@
package handlers
import (
"net/http"
)
// Structure that holds the context map and exposes the ResponseWriter interface.
type contextResponseWriter struct {
http.ResponseWriter
m map[interface{}]interface{}
}
// Implement the WrapWriter interface.
func (this *contextResponseWriter) WrappedWriter() http.ResponseWriter {
return this.ResponseWriter
}
// ContextHandlerFunc is the same as ContextHandler, it is just a convenience
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
func ContextHandlerFunc(h http.HandlerFunc, cap int) http.HandlerFunc {
return ContextHandler(h, cap)
}
// ContextHandler gives a context storage that lives only for the duration of
// the request, with no locking involved.
func ContextHandler(h http.Handler, cap int) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if _, ok := GetContext(w); ok {
// Self-awareness, context handler is already set up
h.ServeHTTP(w, r)
return
}
// Create the context-providing ResponseWriter replacement.
ctxw := &contextResponseWriter{
w,
make(map[interface{}]interface{}, cap),
}
// Call the wrapped handler with the context-aware writer
h.ServeHTTP(ctxw, r)
}
}
// Helper function to retrieve the context map from the ResponseWriter interface.
func GetContext(w http.ResponseWriter) (map[interface{}]interface{}, bool) {
ctxw, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
_, ok := tst.(*contextResponseWriter)
return ok
})
if ok {
return ctxw.(*contextResponseWriter).m, true
}
return nil, false
}
+83
View File
@@ -0,0 +1,83 @@
package handlers
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
)
func TestContext(t *testing.T) {
key := "key"
val := 10
body := "this is the output"
h2 := wrappedHandler(t, key, val, body)
// Create the context handler with a wrapped handler
h := ContextHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
ctx, _ := GetContext(w)
assertTrue(ctx != nil, "expected context to be non-nil", t)
assertTrue(len(ctx) == 0, fmt.Sprintf("expected context to be empty, got %d", len(ctx)), t)
ctx[key] = val
h2.ServeHTTP(w, r)
}), 2)
s := httptest.NewServer(h)
defer s.Close()
// First call
res, err := http.DefaultClient.Get(s.URL)
if err != nil {
panic(err)
}
res.Body.Close()
// Second call, context should be cleaned at start
res, err = http.DefaultClient.Get(s.URL)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertBody([]byte(body), res, t)
}
func TestWrappedContext(t *testing.T) {
key := "key"
val := 10
body := "this is the output"
h2 := wrappedHandler(t, key, val, body)
h := ContextHandler(LogHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
ctx, _ := GetContext(w)
if !assertTrue(ctx != nil, "expected context to be non-nil", t) {
panic("ctx is nil")
}
assertTrue(len(ctx) == 0, fmt.Sprintf("expected context to be empty, got %d", len(ctx)), t)
ctx[key] = val
h2.ServeHTTP(w, r)
}), NewLogOptions(nil, "%s", "url")), 2)
s := httptest.NewServer(h)
defer s.Close()
res, err := http.DefaultClient.Get(s.URL)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertBody([]byte(body), res, t)
}
func wrappedHandler(t *testing.T, k, v interface{}, body string) http.Handler {
return http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
ctx, _ := GetContext(w)
ac := ctx[k]
assertTrue(ac == v, fmt.Sprintf("expected value to be %v, got %v", v, ac), t)
// Actually write something
_, err := w.Write([]byte(body))
if err != nil {
panic(err)
}
})
}
+29
View File
@@ -0,0 +1,29 @@
// Package handlers define reusable handler components that focus on offering
// a single well-defined feature. Note that any http.Handler implementation
// can be used with Ghost's chainable or wrappable handlers design.
//
// Go's standard library provides a number of such useful handlers in net/http:
//
// - FileServer(http.FileSystem)
// - NotFoundHandler()
// - RedirectHandler(string, int)
// - StripPrefix(string, http.Handler)
// - TimeoutHandler(http.Handler, time.Duration, string)
//
// This package adds the following list of handlers:
//
// - BasicAuthHandler(http.Handler, func(string, string) (interface{}, bool), string)
// a Basic Authentication handler.
// - ContextHandler(http.Handler, int) : a volatile storage map valid only
// for the duration of the request, with no locking required.
// - FaviconHandler(http.Handler, string, time.Duration) : an efficient favicon
// handler.
// - GZIPHandler(http.Handler) : compress the content of the body if the client
// accepts gzip compression.
// - LogHandler(http.Handler, *LogOptions) : customizable request logger.
// - PanicHandler(http.Handler) : handle panics gracefully so that the client
// receives a response (status code 500).
// - SessionHandler(http.Handler, *SessionOptions) : a cookie-based, store-agnostic
// persistent session handler.
// - StaticFileHandler(string) : serve the contents of a specific file.
package handlers
+71
View File
@@ -0,0 +1,71 @@
package handlers
import (
"crypto/md5"
"io/ioutil"
"net/http"
"strconv"
"time"
"github.com/PuerkitoBio/ghost"
)
// FaviconHandlerFunc is the same as FaviconHandler, it is just a convenience
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
func FaviconHandlerFunc(h http.HandlerFunc, path string, maxAge time.Duration) http.HandlerFunc {
return FaviconHandler(h, path, maxAge)
}
// Efficient favicon handler, mostly a port of node's Connect library implementation
// of the favicon middleware.
// https://github.com/senchalabs/connect
func FaviconHandler(h http.Handler, path string, maxAge time.Duration) http.HandlerFunc {
var buf []byte
var hash string
return func(w http.ResponseWriter, r *http.Request) {
var err error
if r.URL.Path == "/favicon.ico" {
if buf == nil {
// Read from file and cache
ghost.LogFn("ghost.favicon : serving from %s", path)
buf, err = ioutil.ReadFile(path)
if err != nil {
ghost.LogFn("ghost.favicon : error reading file : %s", err)
http.NotFound(w, r)
return
}
hash = hashContent(buf)
}
writeHeaders(w.Header(), buf, maxAge, hash)
writeBody(w, r, buf)
} else {
h.ServeHTTP(w, r)
}
}
}
// Write the content of the favicon, or respond with a 404 not found
// in case of error (hardly a critical error).
func writeBody(w http.ResponseWriter, r *http.Request, buf []byte) {
_, err := w.Write(buf)
if err != nil {
ghost.LogFn("ghost.favicon : error writing response : %s", err)
http.NotFound(w, r)
}
}
// Correctly set the http headers.
func writeHeaders(hdr http.Header, buf []byte, maxAge time.Duration, hash string) {
hdr.Set("Content-Type", "image/x-icon")
hdr.Set("Content-Length", strconv.Itoa(len(buf)))
hdr.Set("Etag", hash)
hdr.Set("Cache-Control", "public, max-age="+strconv.Itoa(int(maxAge.Seconds())))
}
// Get the MD5 hash of the content.
func hashContent(buf []byte) string {
h := md5.New()
return string(h.Sum(buf))
}
+72
View File
@@ -0,0 +1,72 @@
package handlers
import (
"net/http"
"net/http/httptest"
"os"
"testing"
"time"
)
func TestFavicon(t *testing.T) {
s := httptest.NewServer(FaviconHandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("ok"))
}, "./testdata/favicon.ico", time.Second))
defer s.Close()
res, err := http.Get(s.URL + "/favicon.ico")
if err != nil {
panic(err)
}
defer res.Body.Close()
assertStatus(http.StatusOK, res.StatusCode, t)
assertHeader("Content-Type", "image/x-icon", res, t)
assertHeader("Cache-Control", "public, max-age=1", res, t)
assertHeader("Content-Length", "1406", res, t)
}
func TestFaviconInvalidPath(t *testing.T) {
s := httptest.NewServer(FaviconHandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("ok"))
}, "./testdata/xfavicon.ico", time.Second))
defer s.Close()
res, err := http.Get(s.URL + "/favicon.ico")
if err != nil {
panic(err)
}
defer res.Body.Close()
assertStatus(http.StatusNotFound, res.StatusCode, t)
}
func TestFaviconFromCache(t *testing.T) {
s := httptest.NewServer(FaviconHandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("ok"))
}, "./testdata/favicon.ico", time.Second))
defer s.Close()
res, err := http.Get(s.URL + "/favicon.ico")
if err != nil {
panic(err)
}
defer res.Body.Close()
// Rename the file temporarily
err = os.Rename("./testdata/favicon.ico", "./testdata/xfavicon.ico")
if err != nil {
panic(err)
}
defer os.Rename("./testdata/xfavicon.ico", "./testdata/favicon.ico")
res, err = http.Get(s.URL + "/favicon.ico")
if err != nil {
panic(err)
}
defer res.Body.Close()
assertStatus(http.StatusOK, res.StatusCode, t)
assertHeader("Content-Type", "image/x-icon", res, t)
assertHeader("Cache-Control", "public, max-age=1", res, t)
assertHeader("Content-Length", "1406", res, t)
}
+75
View File
@@ -0,0 +1,75 @@
package handlers
import (
"net/http"
)
// Interface giving easy access to the most common augmented features.
type GhostWriter interface {
http.ResponseWriter
UserName() string
User() interface{}
Context() map[interface{}]interface{}
Session() *Session
}
// Internal implementation of the GhostWriter interface.
type ghostWriter struct {
http.ResponseWriter
userName string
user interface{}
ctx map[interface{}]interface{}
ssn *Session
}
func (this *ghostWriter) UserName() string {
return this.userName
}
func (this *ghostWriter) User() interface{} {
return this.user
}
func (this *ghostWriter) Context() map[interface{}]interface{} {
return this.ctx
}
func (this *ghostWriter) Session() *Session {
return this.ssn
}
// Convenience handler that wraps a custom function with direct access to the
// authenticated user, context and session on the writer.
func GhostHandlerFunc(h func(w GhostWriter, r *http.Request)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if gw, ok := getGhostWriter(w); ok {
// Self-awareness
h(gw, r)
return
}
uid, _ := GetUserName(w)
usr, _ := GetUser(w)
ctx, _ := GetContext(w)
ssn, _ := GetSession(w)
gw := &ghostWriter{
w,
uid,
usr,
ctx,
ssn,
}
h(gw, r)
}
}
// Check the writer chain to find a ghostWriter.
func getGhostWriter(w http.ResponseWriter) (*ghostWriter, bool) {
gw, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
_, ok := tst.(*ghostWriter)
return ok
})
if ok {
return gw.(*ghostWriter), true
}
return nil, false
}
+168
View File
@@ -0,0 +1,168 @@
package handlers
import (
"compress/gzip"
"io"
"net/http"
)
// Thanks to Andrew Gerrand for inspiration:
// https://groups.google.com/d/msg/golang-nuts/eVnTcMwNVjM/4vYU8id9Q2UJ
//
// Also, node's Connect library implementation of the compress middleware:
// https://github.com/senchalabs/connect/blob/master/lib/middleware/compress.js
//
// And StackOverflow's explanation of Vary: Accept-Encoding header:
// http://stackoverflow.com/questions/7848796/what-does-varyaccept-encoding-mean
// Internal gzipped writer that satisfies both the (body) writer in gzipped format,
// and maintains the rest of the ResponseWriter interface for header manipulation.
type gzipResponseWriter struct {
io.Writer
http.ResponseWriter
r *http.Request // Keep a hold of the Request, for the filter function
filtered bool // Has the request been run through the filter function?
dogzip bool // Should we do GZIP compression for this request?
filterFn func(http.ResponseWriter, *http.Request) bool
}
// Make sure the filter function is applied.
func (w *gzipResponseWriter) applyFilter() {
if !w.filtered {
if w.dogzip = w.filterFn(w, w.r); w.dogzip {
setGzipHeaders(w.Header())
}
w.filtered = true
}
}
// Unambiguous Write() implementation (otherwise both ResponseWriter and Writer
// want to claim this method).
func (w *gzipResponseWriter) Write(b []byte) (int, error) {
w.applyFilter()
if w.dogzip {
// Write compressed
return w.Writer.Write(b)
}
// Write uncompressed
return w.ResponseWriter.Write(b)
}
// Intercept the WriteHeader call to correctly set the GZIP headers.
func (w *gzipResponseWriter) WriteHeader(code int) {
w.applyFilter()
w.ResponseWriter.WriteHeader(code)
}
// Implement WrapWriter interface
func (w *gzipResponseWriter) WrappedWriter() http.ResponseWriter {
return w.ResponseWriter
}
var (
defaultFilterTypes = [...]string{
"text",
"javascript",
"json",
}
)
// Default filter to check if the response should be GZIPped.
// By default, all text (html, css, xml, ...), javascript and json
// content types are candidates for GZIP.
func defaultFilter(w http.ResponseWriter, r *http.Request) bool {
hdr := w.Header()
for _, tp := range defaultFilterTypes {
ok := HeaderMatch(hdr, "Content-Type", HmContains, tp)
if ok {
return true
}
}
return false
}
// GZIPHandlerFunc is the same as GZIPHandler, it is just a convenience
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
func GZIPHandlerFunc(h http.HandlerFunc, filterFn func(http.ResponseWriter, *http.Request) bool) http.HandlerFunc {
return GZIPHandler(h, filterFn)
}
// Gzip compression HTTP handler. If the client supports it, it compresses the response
// written by the wrapped handler. The filter function is called when the response is about
// to be written to determine if compression should be applied. If this argument is nil,
// the default filter will GZIP only content types containing /json|text|javascript/.
func GZIPHandler(h http.Handler, filterFn func(http.ResponseWriter, *http.Request) bool) http.HandlerFunc {
if filterFn == nil {
filterFn = defaultFilter
}
return func(w http.ResponseWriter, r *http.Request) {
if _, ok := getGzipWriter(w); ok {
// Self-awareness, gzip handler is already set up
h.ServeHTTP(w, r)
return
}
hdr := w.Header()
setVaryHeader(hdr)
// Do nothing on a HEAD request
if r.Method == "HEAD" {
h.ServeHTTP(w, r)
return
}
if !acceptsGzip(r.Header) {
// No gzip support from the client, return uncompressed
h.ServeHTTP(w, r)
return
}
// Prepare a gzip response container
gz := gzip.NewWriter(w)
gzw := &gzipResponseWriter{
Writer: gz,
ResponseWriter: w,
r: r,
filterFn: filterFn,
}
h.ServeHTTP(gzw, r)
// Iff the handler completed successfully (no panic) and GZIP was indeed used, close the gzip writer,
// which seems to generate a Write to the underlying writer.
if gzw.dogzip {
gz.Close()
}
}
}
// Add the vary by "accept-encoding" header if it is not already set.
func setVaryHeader(hdr http.Header) {
if !HeaderMatch(hdr, "Vary", HmContains, "accept-encoding") {
hdr.Add("Vary", "Accept-Encoding")
}
}
// Checks if the client accepts GZIP-encoded responses.
func acceptsGzip(hdr http.Header) bool {
ok := HeaderMatch(hdr, "Accept-Encoding", HmContains, "gzip")
if !ok {
ok = HeaderMatch(hdr, "Accept-Encoding", HmEquals, "*")
}
return ok
}
func setGzipHeaders(hdr http.Header) {
// The content-type will be explicitly set somewhere down the path of handlers
hdr.Set("Content-Encoding", "gzip")
hdr.Del("Content-Length")
}
// Helper function to retrieve the gzip writer.
func getGzipWriter(w http.ResponseWriter) (*gzipResponseWriter, bool) {
gz, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
_, ok := tst.(*gzipResponseWriter)
return ok
})
if ok {
return gz.(*gzipResponseWriter), true
}
return nil, false
}
+178
View File
@@ -0,0 +1,178 @@
package handlers
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestGzipped(t *testing.T) {
body := "This is the body"
headers := []string{"gzip", "*", "gzip, deflate, sdch"}
h := GZIPHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, err := w.Write([]byte(body))
if err != nil {
panic(err)
}
}), nil)
s := httptest.NewServer(h)
defer s.Close()
for _, hdr := range headers {
t.Logf("running with Accept-Encoding header %s", hdr)
req, err := http.NewRequest("GET", s.URL, nil)
if err != nil {
panic(err)
}
req.Header.Set("Accept-Encoding", hdr)
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertHeader("Content-Encoding", "gzip", res, t)
assertGzippedBody([]byte(body), res, t)
}
}
func TestNoGzip(t *testing.T) {
body := "This is the body"
h := GZIPHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, err := w.Write([]byte(body))
if err != nil {
panic(err)
}
}), nil)
s := httptest.NewServer(h)
defer s.Close()
req, err := http.NewRequest("GET", s.URL, nil)
if err != nil {
panic(err)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertHeader("Content-Encoding", "", res, t)
assertBody([]byte(body), res, t)
}
func TestGzipOuterPanic(t *testing.T) {
msg := "ko"
h := PanicHandler(
GZIPHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
panic(msg)
}), nil), nil)
s := httptest.NewServer(h)
defer s.Close()
req, err := http.NewRequest("GET", s.URL, nil)
if err != nil {
panic(err)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusInternalServerError, res.StatusCode, t)
assertHeader("Content-Encoding", "", res, t)
assertBody([]byte(msg+"\n"), res, t)
}
func TestNoGzipOnFilter(t *testing.T) {
body := "This is the body"
h := GZIPHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "x/x")
_, err := w.Write([]byte(body))
if err != nil {
panic(err)
}
}), nil)
s := httptest.NewServer(h)
defer s.Close()
req, err := http.NewRequest("GET", s.URL, nil)
if err != nil {
panic(err)
}
req.Header.Set("Accept-Encoding", "gzip")
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertHeader("Content-Encoding", "", res, t)
assertBody([]byte(body), res, t)
}
func TestNoGzipOnCustomFilter(t *testing.T) {
body := "This is the body"
h := GZIPHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, err := w.Write([]byte(body))
if err != nil {
panic(err)
}
}), func(w http.ResponseWriter, r *http.Request) bool {
return false
})
s := httptest.NewServer(h)
defer s.Close()
req, err := http.NewRequest("GET", s.URL, nil)
if err != nil {
panic(err)
}
req.Header.Set("Accept-Encoding", "gzip")
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertHeader("Content-Encoding", "", res, t)
assertBody([]byte(body), res, t)
}
func TestGzipOnCustomFilter(t *testing.T) {
body := "This is the body"
h := GZIPHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "x/x")
_, err := w.Write([]byte(body))
if err != nil {
panic(err)
}
}), func(w http.ResponseWriter, r *http.Request) bool {
return true
})
s := httptest.NewServer(h)
defer s.Close()
req, err := http.NewRequest("GET", s.URL, nil)
if err != nil {
panic(err)
}
req.Header.Set("Accept-Encoding", "gzip")
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertHeader("Content-Encoding", "gzip", res, t)
assertGzippedBody([]byte(body), res, t)
}
+50
View File
@@ -0,0 +1,50 @@
package handlers
import (
"net/http"
"strings"
)
// Kind of match to apply to the header check.
type HeaderMatchType int
const (
HmEquals HeaderMatchType = iota
HmStartsWith
HmEndsWith
HmContains
)
// Check if the specified header matches the test string, applying the header match type
// specified.
func HeaderMatch(hdr http.Header, nm string, matchType HeaderMatchType, test string) bool {
// First get the header value
val := hdr[http.CanonicalHeaderKey(nm)]
if len(val) == 0 {
return false
}
// Prepare the match test
test = strings.ToLower(test)
for _, v := range val {
v = strings.Trim(strings.ToLower(v), " \n\t")
switch matchType {
case HmEquals:
if v == test {
return true
}
case HmStartsWith:
if strings.HasPrefix(v, test) {
return true
}
case HmEndsWith:
if strings.HasSuffix(v, test) {
return true
}
case HmContains:
if strings.Contains(v, test) {
return true
}
}
}
return false
}
+231
View File
@@ -0,0 +1,231 @@
package handlers
// Inspired by node's Connect library implementation of the logging middleware
// https://github.com/senchalabs/connect
import (
"fmt"
"net/http"
"regexp"
"strings"
"time"
"github.com/PuerkitoBio/ghost"
)
const (
// Predefined logging formats that can be passed as format string.
Ldefault = "_default_"
Lshort = "_short_"
Ltiny = "_tiny_"
)
var (
// Token parser for request and response headers
rxHeaders = regexp.MustCompile(`^(req|res)\[([^\]]+)\]$`)
// Lookup table for predefined formats
predefFormats = map[string]struct {
fmt string
toks []string
}{
Ldefault: {
`%s - - [%s] "%s %s HTTP/%s" %d %s "%s" "%s"`,
[]string{"remote-addr", "date", "method", "url", "http-version", "status", "res[Content-Length]", "referrer", "user-agent"},
},
Lshort: {
`%s - %s %s HTTP/%s %d %s - %.3f s`,
[]string{"remote-addr", "method", "url", "http-version", "status", "res[Content-Length]", "response-time"},
},
Ltiny: {
`%s %s %d %s - %.3f s`,
[]string{"method", "url", "status", "res[Content-Length]", "response-time"},
},
}
)
// Augmented ResponseWriter implementation that captures the status code for the logger.
type statusResponseWriter struct {
http.ResponseWriter
code int
oriURL string
}
// Intercept the WriteHeader call to save the status code.
func (this *statusResponseWriter) WriteHeader(code int) {
this.code = code
this.ResponseWriter.WriteHeader(code)
}
// Intercept the Write call to save the default status code.
func (this *statusResponseWriter) Write(data []byte) (int, error) {
if this.code == 0 {
this.code = http.StatusOK
}
return this.ResponseWriter.Write(data)
}
// Implement the WrapWriter interface.
func (this *statusResponseWriter) WrappedWriter() http.ResponseWriter {
return this.ResponseWriter
}
// LogHandler options
type LogOptions struct {
LogFn func(string, ...interface{}) // Defaults to ghost.LogFn if nil
Format string
Tokens []string
CustomTokens map[string]func(http.ResponseWriter, *http.Request) string
Immediate bool
DateFormat string
}
// Create a new LogOptions struct. The DateFormat defaults to time.RFC3339.
func NewLogOptions(l func(string, ...interface{}), ft string, tok ...string) *LogOptions {
return &LogOptions{
LogFn: l,
Format: ft,
Tokens: tok,
CustomTokens: make(map[string]func(http.ResponseWriter, *http.Request) string),
DateFormat: time.RFC3339,
}
}
// LogHandlerFunc is the same as LogHandler, it is just a convenience
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
func LogHandlerFunc(h http.HandlerFunc, opts *LogOptions) http.HandlerFunc {
return LogHandler(h, opts)
}
// Create a log handler for every request it receives.
func LogHandler(h http.Handler, opts *LogOptions) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if _, ok := getStatusWriter(w); ok {
// Self-awareness, logging handler already set up
h.ServeHTTP(w, r)
return
}
// Save the response start time
st := time.Now()
// Call the wrapped handler, with the augmented ResponseWriter to handle the status code
stw := &statusResponseWriter{w, 0, ""}
// Log immediately if requested, otherwise on exit
if opts.Immediate {
logRequest(stw, r, st, opts)
} else {
// Store original URL, may get modified by handlers (i.e. StripPrefix)
stw.oriURL = r.URL.String()
defer logRequest(stw, r, st, opts)
}
h.ServeHTTP(stw, r)
}
}
func getIpAddress(r *http.Request) string {
hdr := r.Header
hdrRealIp := hdr.Get("X-Real-Ip")
hdrForwardedFor := hdr.Get("X-Forwarded-For")
if hdrRealIp == "" && hdrForwardedFor == "" {
return r.RemoteAddr
}
if hdrForwardedFor != "" {
// X-Forwarded-For is potentially a list of addresses separated with ","
part := strings.Split(hdrForwardedFor, ",")[0]
return strings.TrimSpace(part) + ":0"
}
return hdrRealIp
}
// Check if the specified token is a predefined one, and if so return its current value.
func getPredefinedTokenValue(t string, w *statusResponseWriter, r *http.Request,
st time.Time, opts *LogOptions) (interface{}, bool) {
switch t {
case "http-version":
return fmt.Sprintf("%d.%d", r.ProtoMajor, r.ProtoMinor), true
case "response-time":
return time.Now().Sub(st).Seconds(), true
case "remote-addr":
return getIpAddress(r), true
case "date":
return time.Now().Format(opts.DateFormat), true
case "method":
return r.Method, true
case "url":
if w.oriURL != "" {
return w.oriURL, true
}
return r.URL.String(), true
case "referrer", "referer":
return r.Referer(), true
case "user-agent":
return r.UserAgent(), true
case "status":
return w.code, true
}
// Handle special cases for header
mtch := rxHeaders.FindStringSubmatch(t)
if len(mtch) > 2 {
if mtch[1] == "req" {
return r.Header.Get(mtch[2]), true
} else {
// This only works for headers explicitly set via the Header() map of
// the writer, not those added by the http package under the covers.
return w.Header().Get(mtch[2]), true
}
}
return nil, false
}
// Do the actual logging.
func logRequest(w *statusResponseWriter, r *http.Request, st time.Time, opts *LogOptions) {
var (
fn func(string, ...interface{})
ok bool
format string
toks []string
)
// If no specific log function, use the default one from the ghost package
if opts.LogFn == nil {
fn = ghost.LogFn
} else {
fn = opts.LogFn
}
// If this is a predefined format, use it instead
if v, ok := predefFormats[opts.Format]; ok {
format = v.fmt
toks = v.toks
} else {
format = opts.Format
toks = opts.Tokens
}
args := make([]interface{}, len(toks))
for i, t := range toks {
if args[i], ok = getPredefinedTokenValue(t, w, r, st, opts); !ok {
if f, ok := opts.CustomTokens[t]; ok && f != nil {
args[i] = f(w, r)
} else {
args[i] = "?"
}
}
}
fn(format, args...)
}
// Helper function to retrieve the status writer.
func getStatusWriter(w http.ResponseWriter) (*statusResponseWriter, bool) {
st, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
_, ok := tst.(*statusResponseWriter)
return ok
})
if ok {
return st.(*statusResponseWriter), true
}
return nil, false
}
+217
View File
@@ -0,0 +1,217 @@
package handlers
import (
"bytes"
"fmt"
"log"
"net/http"
"net/http/httptest"
"regexp"
"testing"
"time"
)
type testCase struct {
tok string
fmt string
rx *regexp.Regexp
}
func TestLog(t *testing.T) {
log.SetFlags(0)
now := time.Now()
formats := []testCase{
testCase{"remote-addr",
"%s",
regexp.MustCompile(`^127\.0\.0\.1:\d+\n$`),
},
testCase{"date",
"%s",
regexp.MustCompile(`^` + fmt.Sprintf("%04d-%02d-%02d", now.Year(), now.Month(), now.Day()) + `\n$`),
},
testCase{"method",
"%s",
regexp.MustCompile(`^GET\n$`),
},
testCase{"url",
"%s",
regexp.MustCompile(`^/\n$`),
},
testCase{"http-version",
"%s",
regexp.MustCompile(`^1\.1\n$`),
},
testCase{"status",
"%d",
regexp.MustCompile(`^200\n$`),
},
testCase{"referer",
"%s",
regexp.MustCompile(`^http://www\.test\.com\n$`),
},
testCase{"referrer",
"%s",
regexp.MustCompile(`^http://www\.test\.com\n$`),
},
testCase{"user-agent",
"%s",
regexp.MustCompile(`^Go \d+\.\d+ package http\n$`),
},
testCase{"bidon",
"%s",
regexp.MustCompile(`^\?\n$`),
},
testCase{"response-time",
"%.3f",
regexp.MustCompile(`^0\.1\d\d\n$`),
},
testCase{"req[Accept-Encoding]",
"%s",
regexp.MustCompile(`^gzip\n$`),
},
testCase{"res[blah]",
"%s",
regexp.MustCompile(`^$`),
},
testCase{"tiny",
Ltiny,
regexp.MustCompile(`^GET / 200 - 0\.1\d\d s\n$`),
},
testCase{"short",
Lshort,
regexp.MustCompile(`^127\.0\.0\.1:\d+ - GET / HTTP/1\.1 200 - 0\.1\d\d s\n$`),
},
testCase{"default",
Ldefault,
regexp.MustCompile(`^127\.0\.0\.1:\d+ - - \[\d{4}-\d{2}-\d{2}\] "GET / HTTP/1\.1" 200 "http://www\.test\.com" "Go \d+\.\d+ package http"\n$`),
},
testCase{"res[Content-Type]",
"%s",
regexp.MustCompile(`^text/plain\n$`),
},
}
for _, tc := range formats {
testLogCase(tc, t)
}
}
func testLogCase(tc testCase, t *testing.T) {
buf := bytes.NewBuffer(nil)
log.SetOutput(buf)
opts := NewLogOptions(log.Printf, tc.fmt, tc.tok)
opts.DateFormat = "2006-01-02"
h := LogHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
time.Sleep(100 * time.Millisecond)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(200)
w.Write([]byte("body"))
}), opts)
s := httptest.NewServer(h)
defer s.Close()
t.Logf("running %s...", tc.tok)
req, err := http.NewRequest("GET", s.URL, nil)
if err != nil {
panic(err)
}
req.Header.Set("Referer", "http://www.test.com")
req.Header.Set("Accept-Encoding", "gzip")
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
ac := buf.String()
assertTrue(tc.rx.MatchString(ac), fmt.Sprintf("expected log to match '%s', got '%s'", tc.rx.String(), ac), t)
}
func TestForwardedFor(t *testing.T) {
rx := regexp.MustCompile(`^1\.1\.1\.1:0 - - \[\d{4}-\d{2}-\d{2}\] "GET / HTTP/1\.1" 200 "http://www\.test\.com" "Go \d+\.\d+ package http"\n$`)
buf := bytes.NewBuffer(nil)
log.SetOutput(buf)
opts := NewLogOptions(log.Printf, Ldefault)
opts.DateFormat = "2006-01-02"
h := LogHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
time.Sleep(100 * time.Millisecond)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(200)
w.Write([]byte("body"))
}), opts)
s := httptest.NewServer(h)
defer s.Close()
t.Logf("running ForwardedFor...")
req, err := http.NewRequest("GET", s.URL, nil)
if err != nil {
panic(err)
}
req.Header.Set("Referer", "http://www.test.com")
req.Header.Set("X-Forwarded-For", "1.1.1.1")
req.Header.Set("Accept-Encoding", "gzip")
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
ac := buf.String()
assertTrue(rx.MatchString(ac), fmt.Sprintf("expected log to match '%s', got '%s'", rx.String(), ac), t)
}
func TestImmediate(t *testing.T) {
buf := bytes.NewBuffer(nil)
log.SetFlags(0)
log.SetOutput(buf)
opts := NewLogOptions(nil, Ltiny)
opts.Immediate = true
h := LogHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
time.Sleep(100 * time.Millisecond)
w.WriteHeader(200)
w.Write([]byte("body"))
}), opts)
s := httptest.NewServer(h)
defer s.Close()
res, err := http.Get(s.URL)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
ac := buf.String()
// Since it is Immediate logging, status is still 0 and response time is less than 100ms
rx := regexp.MustCompile(`GET / 0 - 0\.0\d\d s\n`)
assertTrue(rx.MatchString(ac), fmt.Sprintf("expected log to match '%s', got '%s'", rx.String(), ac), t)
}
func TestCustom(t *testing.T) {
buf := bytes.NewBuffer(nil)
log.SetFlags(0)
log.SetOutput(buf)
opts := NewLogOptions(nil, "%s %s", "method", "custom")
opts.CustomTokens["custom"] = func(w http.ResponseWriter, r *http.Request) string {
return "toto"
}
h := LogHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
time.Sleep(100 * time.Millisecond)
w.WriteHeader(200)
w.Write([]byte("body"))
}), opts)
s := httptest.NewServer(h)
defer s.Close()
res, err := http.Get(s.URL)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
ac := buf.String()
rx := regexp.MustCompile(`GET toto`)
assertTrue(rx.MatchString(ac), fmt.Sprintf("expected log to match '%s', got '%s'", rx.String(), ac), t)
}
+57
View File
@@ -0,0 +1,57 @@
package handlers
import (
"fmt"
"net/http"
)
// Augmented response writer to hold the panic data (can be anything, not necessarily an error
// interface).
type errResponseWriter struct {
http.ResponseWriter
perr interface{}
}
// Implement the WrapWriter interface.
func (this *errResponseWriter) WrappedWriter() http.ResponseWriter {
return this.ResponseWriter
}
// PanicHandlerFunc is the same as PanicHandler, it is just a convenience
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
func PanicHandlerFunc(h http.HandlerFunc, errH http.HandlerFunc) http.HandlerFunc {
return PanicHandler(h, errH)
}
// Calls the wrapped handler and on panic calls the specified error handler. If the error handler is nil,
// responds with a 500 error message.
func PanicHandler(h http.Handler, errH http.Handler) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err != nil {
if errH != nil {
ew := &errResponseWriter{w, err}
errH.ServeHTTP(ew, r)
} else {
http.Error(w, fmt.Sprintf("%s", err), http.StatusInternalServerError)
}
}
}()
// Call the protected handler
h.ServeHTTP(w, r)
}
}
// Helper function to retrieve the panic error, if any.
func GetPanicError(w http.ResponseWriter) (interface{}, bool) {
er, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
_, ok := tst.(*errResponseWriter)
return ok
})
if ok {
return er.(*errResponseWriter).perr, true
}
return nil, false
}
+62
View File
@@ -0,0 +1,62 @@
package handlers
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestPanic(t *testing.T) {
h := PanicHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
panic("test")
}), nil)
s := httptest.NewServer(h)
defer s.Close()
res, err := http.Get(s.URL)
if err != nil {
panic(err)
}
assertStatus(http.StatusInternalServerError, res.StatusCode, t)
}
func TestNoPanic(t *testing.T) {
h := PanicHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
}), nil)
s := httptest.NewServer(h)
defer s.Close()
res, err := http.Get(s.URL)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
}
func TestPanicCustom(t *testing.T) {
h := PanicHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
panic("ok")
}),
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
err, ok := GetPanicError(w)
if !ok {
panic("no panic error found")
}
w.WriteHeader(501)
w.Write([]byte(err.(string)))
}))
s := httptest.NewServer(h)
defer s.Close()
res, err := http.Get(s.URL)
if err != nil {
panic(err)
}
assertStatus(501, res.StatusCode, t)
assertBody([]byte("ok"), res, t)
}
+135
View File
@@ -0,0 +1,135 @@
package handlers
import (
"encoding/json"
"errors"
"time"
"github.com/garyburd/redigo/redis"
)
var (
ErrNoKeyPrefix = errors.New("cannot get session keys without a key prefix")
)
type RedisStoreOptions struct {
Network string
Address string
ConnectTimeout time.Duration
ReadTimeout time.Duration
WriteTimeout time.Duration
Database int // Redis database to use for session keys
KeyPrefix string // If set, keys will be KeyPrefix:SessionID (semicolon added)
BrowserSessServerTTL time.Duration // Defaults to 2 days
}
type RedisStore struct {
opts *RedisStoreOptions
conn redis.Conn
}
// Create a redis session store with the specified options.
func NewRedisStore(opts *RedisStoreOptions) *RedisStore {
var err error
rs := &RedisStore{opts, nil}
rs.conn, err = redis.DialTimeout(opts.Network, opts.Address, opts.ConnectTimeout,
opts.ReadTimeout, opts.WriteTimeout)
if err != nil {
panic(err)
}
return rs
}
// Get the session from the store.
func (this *RedisStore) Get(id string) (*Session, error) {
key := id
if this.opts.KeyPrefix != "" {
key = this.opts.KeyPrefix + ":" + id
}
b, err := redis.Bytes(this.conn.Do("GET", key))
if err != nil {
return nil, err
}
var sess Session
err = json.Unmarshal(b, &sess)
if err != nil {
return nil, err
}
return &sess, nil
}
// Save the session into the store.
func (this *RedisStore) Set(sess *Session) error {
b, err := json.Marshal(sess)
if err != nil {
return err
}
key := sess.ID()
if this.opts.KeyPrefix != "" {
key = this.opts.KeyPrefix + ":" + sess.ID()
}
ttl := sess.MaxAge()
if ttl == 0 {
// Browser session, set to specified TTL
ttl = this.opts.BrowserSessServerTTL
if ttl == 0 {
ttl = 2 * 24 * time.Hour // Default to 2 days
}
}
_, err = this.conn.Do("SETEX", key, int(ttl.Seconds()), b)
if err != nil {
return err
}
return nil
}
// Delete the session from the store.
func (this *RedisStore) Delete(id string) error {
key := id
if this.opts.KeyPrefix != "" {
key = this.opts.KeyPrefix + ":" + id
}
_, err := this.conn.Do("DEL", key)
if err != nil {
return err
}
return nil
}
// Clear all sessions from the store. Requires the use of a key
// prefix in the store options, otherwise the method refuses to delete all keys.
func (this *RedisStore) Clear() error {
vals, err := this.getSessionKeys()
if err != nil {
return err
}
if len(vals) > 0 {
this.conn.Send("MULTI")
for _, v := range vals {
this.conn.Send("DEL", v)
}
_, err = this.conn.Do("EXEC")
if err != nil {
return err
}
}
return nil
}
// Get the number of session keys in the store. Requires the use of a
// key prefix in the store options, otherwise returns -1 (cannot tell
// session keys from other keys).
func (this *RedisStore) Len() int {
vals, err := this.getSessionKeys()
if err != nil {
return -1
}
return len(vals)
}
func (this *RedisStore) getSessionKeys() ([]interface{}, error) {
if this.opts.KeyPrefix != "" {
return redis.Values(this.conn.Do("KEYS", this.opts.KeyPrefix+":*"))
}
return nil, ErrNoKeyPrefix
}
+30
View File
@@ -0,0 +1,30 @@
package handlers
import (
"net/http"
)
// This interface can be implemented by an augmented ResponseWriter, so that
// it doesn't hide other augmented writers in the chain.
type WrapWriter interface {
http.ResponseWriter
WrappedWriter() http.ResponseWriter
}
// Helper function to retrieve a specific ResponseWriter.
func GetResponseWriter(w http.ResponseWriter,
predicate func(http.ResponseWriter) bool) (http.ResponseWriter, bool) {
for {
// Check if this writer is the one we're looking for
if w != nil && predicate(w) {
return w, true
}
// If it is a WrapWriter, move back the chain of wrapped writers
ww, ok := w.(WrapWriter)
if !ok {
return nil, false
}
w = ww.WrappedWriter()
}
}
+52
View File
@@ -0,0 +1,52 @@
package handlers
import (
"fmt"
"net/http"
"testing"
)
type baseWriter struct{}
func (b *baseWriter) Write(data []byte) (int, error) { return 0, nil }
func (b *baseWriter) WriteHeader(code int) {}
func (b *baseWriter) Header() http.Header { return nil }
func TestNilWriter(t *testing.T) {
rw, ok := GetResponseWriter(nil, func(w http.ResponseWriter) bool {
return true
})
assertTrue(rw == nil, "expected nil, got non-nil", t)
assertTrue(!ok, "expected false, got true", t)
}
func TestBaseWriter(t *testing.T) {
bw := &baseWriter{}
rw, ok := GetResponseWriter(bw, func(w http.ResponseWriter) bool {
return true
})
assertTrue(rw == bw, fmt.Sprintf("expected %#v, got %#v", bw, rw), t)
assertTrue(ok, "expected true, got false", t)
}
func TestWrappedWriter(t *testing.T) {
bw := &baseWriter{}
ctx := &contextResponseWriter{bw, nil}
rw, ok := GetResponseWriter(ctx, func(w http.ResponseWriter) bool {
_, ok := w.(*baseWriter)
return ok
})
assertTrue(rw == bw, fmt.Sprintf("expected %#v, got %#v", bw, rw), t)
assertTrue(ok, "expected true, got false", t)
}
func TestWrappedNotFoundWriter(t *testing.T) {
bw := &baseWriter{}
ctx := &contextResponseWriter{bw, nil}
rw, ok := GetResponseWriter(ctx, func(w http.ResponseWriter) bool {
_, ok := w.(*statusResponseWriter)
return ok
})
assertTrue(rw == nil, fmt.Sprintf("expected nil, got %#v", rw), t)
assertTrue(!ok, "expected false, got true", t)
}
+321
View File
@@ -0,0 +1,321 @@
package handlers
import (
"encoding/json"
"errors"
"hash/crc32"
"net/http"
"strings"
"time"
"github.com/PuerkitoBio/ghost"
"github.com/gorilla/securecookie"
"github.com/nu7hatch/gouuid"
)
const defaultCookieName = "ghost.sid"
var (
ErrSessionSecretMissing = errors.New("session secret is missing")
ErrNoSessionID = errors.New("session ID could not be generated")
)
// The Session holds the data map that persists for the duration of the session.
// The information stored in this map should be marshalable for the target Session store
// format (i.e. json, sql, gob, etc. depending on how the store persists the data).
type Session struct {
isNew bool // keep private, not saved to JSON, will be false once read from the store
internalSession
}
// Use a separate private struct to hold the private fields of the Session,
// although those fields are exposed (public). This is a trick to simplify
// JSON encoding.
type internalSession struct {
Data map[string]interface{} // JSON cannot marshal a map[interface{}]interface{}
ID string
Created time.Time
MaxAge time.Duration
}
// Create a new Session instance. It panics in the unlikely event that a new random ID cannot be generated.
func newSession(maxAge int) *Session {
uid, err := uuid.NewV4()
if err != nil {
panic(ErrNoSessionID)
}
return &Session{
true, // is new
internalSession{
make(map[string]interface{}),
uid.String(),
time.Now(),
time.Duration(maxAge) * time.Second,
},
}
}
// Gets the ID of the session.
func (ø *Session) ID() string {
return ø.internalSession.ID
}
// Get the max age duration
func (ø *Session) MaxAge() time.Duration {
return ø.internalSession.MaxAge
}
// Get the creation time of the session.
func (ø *Session) Created() time.Time {
return ø.internalSession.Created
}
// Is this a new Session (created by the current request)
func (ø *Session) IsNew() bool {
return ø.isNew
}
// TODO : Resets the max age property of the session to its original value (sliding expiration).
func (ø *Session) resetMaxAge() {
}
// Marshal the session to JSON.
func (ø *Session) MarshalJSON() ([]byte, error) {
return json.Marshal(ø.internalSession)
}
// Unmarshal the JSON into the internal session struct.
func (ø *Session) UnmarshalJSON(b []byte) error {
return json.Unmarshal(b, &ø.internalSession)
}
// Options object for the session handler. It specified the Session store to use for
// persistence, the template for the session cookie (name, path, maxage, etc.),
// whether or not the proxy should be trusted to determine if the connection is secure,
// and the required secret to sign the session cookie.
type SessionOptions struct {
Store SessionStore
CookieTemplate http.Cookie
TrustProxy bool
Secret string
}
// Create a new SessionOptions struct, using default cookie and proxy values.
func NewSessionOptions(store SessionStore, secret string) *SessionOptions {
return &SessionOptions{
Store: store,
Secret: secret,
}
}
// The augmented ResponseWriter struct for the session handler. It holds the current
// Session object and Session store, as well as flags and function to send the actual
// session cookie at the end of the request.
type sessResponseWriter struct {
http.ResponseWriter
sess *Session
sessStore SessionStore
sessSent bool
sendCookieFn func()
}
// Implement the WrapWriter interface.
func (ø *sessResponseWriter) WrappedWriter() http.ResponseWriter {
return ø.ResponseWriter
}
// Intercept the Write() method to add the Set-Cookie header before it's too late.
func (ø *sessResponseWriter) Write(data []byte) (int, error) {
if !ø.sessSent {
ø.sendCookieFn()
ø.sessSent = true
}
return ø.ResponseWriter.Write(data)
}
// Intercept the WriteHeader() method to add the Set-Cookie header before it's too late.
func (ø *sessResponseWriter) WriteHeader(code int) {
if !ø.sessSent {
ø.sendCookieFn()
ø.sessSent = true
}
ø.ResponseWriter.WriteHeader(code)
}
// SessionHandlerFunc is the same as SessionHandler, it is just a convenience
// signature that accepts a func(http.ResponseWriter, *http.Request) instead of
// a http.Handler interface. It saves the boilerplate http.HandlerFunc() cast.
func SessionHandlerFunc(h http.HandlerFunc, opts *SessionOptions) http.HandlerFunc {
return SessionHandler(h, opts)
}
// Create a Session handler to offer the Session behaviour to the specified handler.
func SessionHandler(h http.Handler, opts *SessionOptions) http.HandlerFunc {
// Make sure the required cookie fields are set
if opts.CookieTemplate.Name == "" {
opts.CookieTemplate.Name = defaultCookieName
}
if opts.CookieTemplate.Path == "" {
opts.CookieTemplate.Path = "/"
}
// Secret is required
if opts.Secret == "" {
panic(ErrSessionSecretMissing)
}
// Return the actual handler
return func(w http.ResponseWriter, r *http.Request) {
if _, ok := getSessionWriter(w); ok {
// Self-awareness
h.ServeHTTP(w, r)
return
}
if strings.Index(r.URL.Path, opts.CookieTemplate.Path) != 0 {
// Session does not apply to this path
h.ServeHTTP(w, r)
return
}
// Create a new Session or retrieve the existing session based on the
// session cookie received.
var sess *Session
var ckSessId string
exCk, err := r.Cookie(opts.CookieTemplate.Name)
if err != nil {
sess = newSession(opts.CookieTemplate.MaxAge)
ghost.LogFn("ghost.session : error getting session cookie : %s", err)
} else {
ckSessId, err = parseSignedCookie(exCk, opts.Secret)
if err != nil {
sess = newSession(opts.CookieTemplate.MaxAge)
ghost.LogFn("ghost.session : error parsing signed cookie : %s", err)
} else if ckSessId == "" {
sess = newSession(opts.CookieTemplate.MaxAge)
ghost.LogFn("ghost.session : no existing session ID")
} else {
// Get the session
sess, err = opts.Store.Get(ckSessId)
if err != nil {
sess = newSession(opts.CookieTemplate.MaxAge)
ghost.LogFn("ghost.session : error getting session from store : %s", err)
} else if sess == nil {
sess = newSession(opts.CookieTemplate.MaxAge)
ghost.LogFn("ghost.session : nil session")
}
}
}
// Save the original hash of the session, used to compare if the contents
// have changed during the handling of the request, so that it has to be
// saved to the stored.
oriHash := hash(sess)
// Create the augmented ResponseWriter.
srw := &sessResponseWriter{w, sess, opts.Store, false, func() {
// This function is called when the header is about to be written, so that
// the session cookie is correctly set.
// Check if the connection is secure
proto := strings.Trim(strings.ToLower(r.Header.Get("X-Forwarded-Proto")), " ")
tls := r.TLS != nil || (strings.HasPrefix(proto, "https") && opts.TrustProxy)
if opts.CookieTemplate.Secure && !tls {
ghost.LogFn("ghost.session : secure cookie on a non-secure connection, cookie not sent")
return
}
if !sess.IsNew() {
// If this is not a new session, no need to send back the cookie
// TODO : Handle expires?
return
}
// Send the session cookie
ck := opts.CookieTemplate
ck.Value = sess.ID()
err := signCookie(&ck, opts.Secret)
if err != nil {
ghost.LogFn("ghost.session : error signing cookie : %s", err)
return
}
http.SetCookie(w, &ck)
}}
// Call wrapped handler
h.ServeHTTP(srw, r)
// TODO : Expiration management? srw.sess.resetMaxAge()
// Do not save if content is the same, unless session is new (to avoid
// creating a new session and sending a cookie on each successive request).
if newHash := hash(sess); !sess.IsNew() && oriHash == newHash && newHash != 0 {
// No changes to the session, no need to save
ghost.LogFn("ghost.session : no changes to save to store")
return
}
err = opts.Store.Set(sess)
if err != nil {
ghost.LogFn("ghost.session : error saving session to store : %s", err)
}
}
}
// Helper function to retrieve the session for the current request.
func GetSession(w http.ResponseWriter) (*Session, bool) {
ss, ok := getSessionWriter(w)
if ok {
return ss.sess, true
}
return nil, false
}
// Helper function to retrieve the session store
func GetSessionStore(w http.ResponseWriter) (SessionStore, bool) {
ss, ok := getSessionWriter(w)
if ok {
return ss.sessStore, true
}
return nil, false
}
// Internal helper function to retrieve the session writer object.
func getSessionWriter(w http.ResponseWriter) (*sessResponseWriter, bool) {
ss, ok := GetResponseWriter(w, func(tst http.ResponseWriter) bool {
_, ok := tst.(*sessResponseWriter)
return ok
})
if ok {
return ss.(*sessResponseWriter), true
}
return nil, false
}
// Parse a signed cookie and return the cookie value
func parseSignedCookie(ck *http.Cookie, secret string) (string, error) {
var val string
sck := securecookie.New([]byte(secret), nil)
err := sck.Decode(ck.Name, ck.Value, &val)
if err != nil {
return "", err
}
return val, nil
}
// Sign the specified cookie's value
func signCookie(ck *http.Cookie, secret string) error {
sck := securecookie.New([]byte(secret), nil)
enc, err := sck.Encode(ck.Name, ck.Value)
if err != nil {
return err
}
ck.Value = enc
return nil
}
// Compute a CRC32 hash of the session's JSON-encoded contents.
func hash(s *Session) uint32 {
data, err := json.Marshal(s)
if err != nil {
ghost.LogFn("ghost.session : error hash : %s", err)
return 0 // 0 is always treated as "modified" session content
}
return crc32.ChecksumIEEE(data)
}
+258
View File
@@ -0,0 +1,258 @@
package handlers
import (
"fmt"
"io/ioutil"
"net/http"
"net/http/cookiejar"
"net/http/httptest"
"testing"
"time"
)
var (
store SessionStore
secret = "butchered at birth"
)
func TestSession(t *testing.T) {
stores := map[string]SessionStore{
"memory": NewMemoryStore(1),
"redis": NewRedisStore(&RedisStoreOptions{
Network: "tcp",
Address: ":6379",
Database: 1,
KeyPrefix: "sess",
}),
}
for k, v := range stores {
t.Logf("testing session with %s store\n", k)
store = v
t.Log("SessionExists")
testSessionExists(t)
t.Log("SessionPersists")
testSessionPersists(t)
t.Log("SessionExpires")
testSessionExpires(t)
t.Log("SessionBeforeExpires")
testSessionBeforeExpires(t)
t.Log("PanicIfNoSecret")
testPanicIfNoSecret(t)
t.Log("InvalidPath")
testInvalidPath(t)
t.Log("ValidSubPath")
testValidSubPath(t)
t.Log("SecureOverHttp")
testSecureOverHttp(t)
}
}
func setupTest(f func(w http.ResponseWriter, r *http.Request), ckPath string, secure bool, maxAge int) *httptest.Server {
opts := NewSessionOptions(store, secret)
if ckPath != "" {
opts.CookieTemplate.Path = ckPath
}
opts.CookieTemplate.Secure = secure
opts.CookieTemplate.MaxAge = maxAge
h := SessionHandler(http.HandlerFunc(f), opts)
return httptest.NewServer(h)
}
func doRequest(u string, newJar bool) *http.Response {
var err error
if newJar {
http.DefaultClient.Jar, err = cookiejar.New(new(cookiejar.Options))
if err != nil {
panic(err)
}
}
res, err := http.Get(u)
if err != nil {
panic(err)
}
return res
}
func testSessionExists(t *testing.T) {
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
ssn, ok := GetSession(w)
if assertTrue(ok, "expected session to be non-nil, got nil", t) {
ssn.Data["foo"] = "bar"
assertTrue(ssn.Data["foo"] == "bar", fmt.Sprintf("expected ssn[foo] to be 'bar', got %v", ssn.Data["foo"]), t)
}
w.Write([]byte("ok"))
}, "", false, 0)
defer s.Close()
res := doRequest(s.URL, true)
assertStatus(http.StatusOK, res.StatusCode, t)
assertBody([]byte("ok"), res, t)
assertTrue(len(res.Cookies()) == 1, fmt.Sprintf("expected response to have 1 cookie, got %d", len(res.Cookies())), t)
}
func testSessionPersists(t *testing.T) {
cnt := 0
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
ssn, ok := GetSession(w)
if !ok {
panic("session not found!")
}
if cnt == 0 {
ssn.Data["foo"] = "bar"
w.Write([]byte("ok"))
cnt++
} else {
w.Write([]byte(ssn.Data["foo"].(string)))
}
}, "", false, 0)
defer s.Close()
// 1st call, set the session value
res := doRequest(s.URL, true)
assertStatus(http.StatusOK, res.StatusCode, t)
assertBody([]byte("ok"), res, t)
// 2nd call, get the session value
res = doRequest(s.URL, false)
assertStatus(http.StatusOK, res.StatusCode, t)
assertBody([]byte("bar"), res, t)
assertTrue(len(res.Cookies()) == 0, fmt.Sprintf("expected 2nd response to have 0 cookie, got %d", len(res.Cookies())), t)
}
func testSessionExpires(t *testing.T) {
cnt := 0
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
ssn, ok := GetSession(w)
if !ok {
panic("session not found!")
}
if cnt == 0 {
w.Write([]byte(ssn.ID()))
cnt++
} else {
w.Write([]byte(ssn.ID()))
}
}, "", false, 1) // Expire in 1 second
defer s.Close()
// 1st call, set the session value
res := doRequest(s.URL, true)
assertStatus(http.StatusOK, res.StatusCode, t)
id1, err := ioutil.ReadAll(res.Body)
if err != nil {
panic(err)
}
res.Body.Close()
time.Sleep(1001 * time.Millisecond)
// 2nd call, get the session value
res = doRequest(s.URL, false)
assertStatus(http.StatusOK, res.StatusCode, t)
id2, err := ioutil.ReadAll(res.Body)
if err != nil {
panic(err)
}
res.Body.Close()
sid1, sid2 := string(id1), string(id2)
assertTrue(len(res.Cookies()) == 1, fmt.Sprintf("expected 2nd response to have 1 cookie, got %d", len(res.Cookies())), t)
assertTrue(sid1 != sid2, "expected session IDs to be different, got same", t)
}
func testSessionBeforeExpires(t *testing.T) {
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
ssn, ok := GetSession(w)
if !ok {
panic("session not found!")
}
w.Write([]byte(ssn.ID()))
}, "", false, 1) // Expire in 1 second
defer s.Close()
// 1st call, set the session value
res := doRequest(s.URL, true)
assertStatus(http.StatusOK, res.StatusCode, t)
id1, err := ioutil.ReadAll(res.Body)
if err != nil {
panic(err)
}
res.Body.Close()
time.Sleep(500 * time.Millisecond)
// 2nd call, get the session value
res = doRequest(s.URL, false)
assertStatus(http.StatusOK, res.StatusCode, t)
id2, err := ioutil.ReadAll(res.Body)
if err != nil {
panic(err)
}
res.Body.Close()
sid1, sid2 := string(id1), string(id2)
assertTrue(len(res.Cookies()) == 0, fmt.Sprintf("expected 2nd response to have no cookie, got %d", len(res.Cookies())), t)
assertTrue(sid1 == sid2, "expected session IDs to be the same, got different", t)
}
func testPanicIfNoSecret(t *testing.T) {
defer assertPanic(t)
SessionHandler(http.NotFoundHandler(), NewSessionOptions(nil, ""))
}
func testInvalidPath(t *testing.T) {
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
_, ok := GetSession(w)
assertTrue(!ok, "expected session to be nil, got non-nil", t)
w.Write([]byte("ok"))
}, "/foo", false, 0)
defer s.Close()
res := doRequest(s.URL, true)
assertStatus(http.StatusOK, res.StatusCode, t)
assertBody([]byte("ok"), res, t)
assertTrue(len(res.Cookies()) == 0, fmt.Sprintf("expected response to have no cookie, got %d", len(res.Cookies())), t)
}
func testValidSubPath(t *testing.T) {
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
_, ok := GetSession(w)
assertTrue(ok, "expected session to be non-nil, got nil", t)
w.Write([]byte("ok"))
}, "/foo", false, 0)
defer s.Close()
res := doRequest(s.URL+"/foo/bar", true)
assertStatus(http.StatusOK, res.StatusCode, t)
assertBody([]byte("ok"), res, t)
assertTrue(len(res.Cookies()) == 1, fmt.Sprintf("expected response to have 1 cookie, got %d", len(res.Cookies())), t)
}
func testSecureOverHttp(t *testing.T) {
s := setupTest(func(w http.ResponseWriter, r *http.Request) {
_, ok := GetSession(w)
assertTrue(ok, "expected session to be non-nil, got nil", t)
w.Write([]byte("ok"))
}, "", true, 0)
defer s.Close()
res := doRequest(s.URL, true)
assertStatus(http.StatusOK, res.StatusCode, t)
assertBody([]byte("ok"), res, t)
assertTrue(len(res.Cookies()) == 0, fmt.Sprintf("expected response to have no cookie, got %d", len(res.Cookies())), t)
}
// TODO : commented, certificate problem
func xtestSecureOverHttps(t *testing.T) {
opts := NewSessionOptions(store, secret)
opts.CookieTemplate.Secure = true
h := SessionHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
_, ok := GetSession(w)
assertTrue(ok, "expected session to be non-nil, got nil", t)
w.Write([]byte("ok"))
}), opts)
s := httptest.NewTLSServer(h)
defer s.Close()
res := doRequest(s.URL, true)
assertStatus(http.StatusOK, res.StatusCode, t)
assertBody([]byte("ok"), res, t)
assertTrue(len(res.Cookies()) == 1, fmt.Sprintf("expected response to have 1 cookie, got %d", len(res.Cookies())), t)
}
+90
View File
@@ -0,0 +1,90 @@
package handlers
import (
"sync"
"time"
)
// SessionStore interface, must be implemented by any store to be used
// for session storage.
type SessionStore interface {
Get(id string) (*Session, error) // Get the session from the store
Set(sess *Session) error // Save the session in the store
Delete(id string) error // Delete the session from the store
Clear() error // Delete all sessions from the store
Len() int // Get the number of sessions in the store
}
// In-memory implementation of a session store. Not recommended for production
// use.
type MemoryStore struct {
l sync.RWMutex
m map[string]*Session
capc int
}
// Create a new memory store.
func NewMemoryStore(capc int) *MemoryStore {
m := &MemoryStore{}
m.capc = capc
m.newMap()
return m
}
// Get the number of sessions saved in the store.
func (this *MemoryStore) Len() int {
return len(this.m)
}
// Get the requested session from the store.
func (this *MemoryStore) Get(id string) (*Session, error) {
this.l.RLock()
defer this.l.RUnlock()
return this.m[id], nil
}
// Save the session to the store.
func (this *MemoryStore) Set(sess *Session) error {
this.l.Lock()
defer this.l.Unlock()
this.m[sess.ID()] = sess
if sess.IsNew() {
// Since the memory store doesn't marshal to a string without the isNew, if it is left
// to true, it will stay true forever.
sess.isNew = false
// Expire in the given time. If the maxAge is 0 (which means browser-session lifetime),
// expire in a reasonable delay, 2 days. The weird case of a negative maxAge will
// cause the immediate Delete call.
wait := sess.MaxAge()
if wait == 0 {
wait = 2 * 24 * time.Hour
}
go func() {
// Clear the session after the specified delay
<-time.After(wait)
this.Delete(sess.ID())
}()
}
return nil
}
// Delete the specified session ID from the store.
func (this *MemoryStore) Delete(id string) error {
this.l.Lock()
defer this.l.Unlock()
delete(this.m, id)
return nil
}
// Clear all sessions from the store.
func (this *MemoryStore) Clear() error {
this.l.Lock()
defer this.l.Unlock()
this.newMap()
return nil
}
// Re-create the internal map, dropping all existing sessions.
func (this *MemoryStore) newMap() {
this.m = make(map[string]*Session, this.capc)
}
+13
View File
@@ -0,0 +1,13 @@
package handlers
import (
"net/http"
)
// StaticFileHandler, unlike net/http.FileServer, serves the contents of a specific
// file when it is called.
func StaticFileHandler(path string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, path)
}
}
+46
View File
@@ -0,0 +1,46 @@
package handlers
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestServeFile(t *testing.T) {
h := StaticFileHandler("./testdata/styles.css")
s := httptest.NewServer(h)
defer s.Close()
res, err := http.Get(s.URL)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertHeader("Content-Type", "text/css; charset=utf-8", res, t)
assertHeader("Content-Encoding", "", res, t)
assertBody([]byte(`* {
background-color: white;
}`), res, t)
}
func TestGzippedFile(t *testing.T) {
h := GZIPHandler(StaticFileHandler("./testdata/styles.css"), nil)
s := httptest.NewServer(h)
defer s.Close()
req, err := http.NewRequest("GET", s.URL, nil)
if err != nil {
panic(err)
}
req.Header.Set("Accept-Encoding", "*")
res, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
assertStatus(http.StatusOK, res.StatusCode, t)
assertHeader("Content-Encoding", "gzip", res, t)
assertHeader("Content-Type", "text/css; charset=utf-8", res, t)
assertGzippedBody([]byte(`* {
background-color: white;
}`), res, t)
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

+1
View File
@@ -0,0 +1 @@
var a = 0;
+3
View File
@@ -0,0 +1,3 @@
* {
background-color: white;
}
+68
View File
@@ -0,0 +1,68 @@
package handlers
import (
"bytes"
"compress/gzip"
"io"
"io/ioutil"
"net/http"
"testing"
)
func assertTrue(cond bool, msg string, t *testing.T) bool {
if !cond {
t.Error(msg)
return false
}
return true
}
func assertStatus(ex, ac int, t *testing.T) {
if ex != ac {
t.Errorf("expected status code to be %d, got %d", ex, ac)
}
}
func assertBody(ex []byte, res *http.Response, t *testing.T) {
buf, err := ioutil.ReadAll(res.Body)
if err != nil {
panic(err)
}
defer res.Body.Close()
if !bytes.Equal(ex, buf) {
t.Errorf("expected body to be '%s' (%d), got '%s' (%d)", ex, len(ex), buf, len(buf))
}
}
func assertGzippedBody(ex []byte, res *http.Response, t *testing.T) {
gr, err := gzip.NewReader(res.Body)
if err != nil {
panic(err)
}
defer res.Body.Close()
buf := bytes.NewBuffer(nil)
_, err = io.Copy(buf, gr)
if err != nil {
panic(err)
}
if !bytes.Equal(ex, buf.Bytes()) {
t.Errorf("expected unzipped body to be '%s' (%d), got '%s' (%d)", ex, len(ex), buf.Bytes(), buf.Len())
}
}
func assertHeader(hName, ex string, res *http.Response, t *testing.T) {
hVal, ok := res.Header[hName]
if (!ok || len(hVal) == 0) && len(ex) > 0 {
t.Errorf("expected header %s to be %s, was not set", hName, ex)
} else if len(hVal) > 0 && hVal[0] != ex {
t.Errorf("expected header %s to be %s, got %s", hName, ex, hVal)
}
}
func assertPanic(t *testing.T) {
if err := recover(); err == nil {
t.Error("expected a panic, got none")
}
}
+203
View File
File diff suppressed because one or more lines are too long
+38
View File
@@ -0,0 +1,38 @@
package amber
import (
"github.com/PuerkitoBio/ghost/templates"
"github.com/eknkc/amber"
)
// The template compiler for Amber templates.
type AmberCompiler struct {
Options amber.Options
c *amber.Compiler
}
// Create a new Amber compiler with the specified Amber-specific options.
func NewAmberCompiler(opts amber.Options) *AmberCompiler {
return &AmberCompiler{
opts,
nil,
}
}
// Implementation of the TemplateCompiler interface.
func (this *AmberCompiler) Compile(f string) (templates.Templater, error) {
// amber.CompileFile creates a new compiler each time. To limit the number
// of allocations, reuse a compiler.
if this.c == nil {
this.c = amber.New()
}
this.c.Options = this.Options
if err := this.c.ParseFile(f); err != nil {
return nil, err
}
return this.c.Compile()
}
func init() {
templates.Register(".amber", NewAmberCompiler(amber.DefaultOptions))
}
+19
View File
@@ -0,0 +1,19 @@
package gotpl
import (
"html/template"
"github.com/PuerkitoBio/ghost/templates"
)
// The template compiler for native Go templates.
type GoTemplateCompiler struct{}
// Implementation of the TemplateCompiler interface.
func (this *GoTemplateCompiler) Compile(f string) (templates.Templater, error) {
return template.ParseFiles(f)
}
func init() {
templates.Register(".tmpl", new(GoTemplateCompiler))
}
+129
View File
@@ -0,0 +1,129 @@
package templates
import (
"errors"
"io"
"net/http"
"os"
"path"
"path/filepath"
"sync"
"github.com/PuerkitoBio/ghost"
)
var (
ErrTemplateNotExist = errors.New("template does not exist")
ErrDirNotExist = errors.New("directory does not exist")
compilers = make(map[string]TemplateCompiler)
// The mutex guards the templaters map
mu sync.RWMutex
templaters = make(map[string]Templater)
)
// Defines the interface that the template compiler must return. The Go native
// templates implement this interface.
type Templater interface {
Execute(wr io.Writer, data interface{}) error
}
// The interface that a template engine must implement to be used by Ghost.
type TemplateCompiler interface {
Compile(fileName string) (Templater, error)
}
// TODO : How to manage Go nested templates?
// TODO : Support Go's port of the mustache template?
// Register a template compiler for the specified extension. Extensions are case-sensitive.
// The extension must start with a dot (it is compared to the result of path.Ext() on a
// given file name).
//
// Registering is not thread-safe. Compilers should be registered before the http server
// is started.
// Compiling templates, on the other hand, is thread-safe.
func Register(ext string, c TemplateCompiler) {
if c == nil {
panic("ghost: Register TemplateCompiler is nil")
}
if _, dup := compilers[ext]; dup {
panic("ghost: Register called twice for extension " + ext)
}
compilers[ext] = c
}
// Compile all templates that have a matching compiler (based on their extension) in the
// specified directory.
func CompileDir(dir string) error {
mu.Lock()
defer mu.Unlock()
return filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
if fi == nil {
return ErrDirNotExist
}
if !fi.IsDir() {
err = compileTemplate(path, dir)
if err != nil {
ghost.LogFn("ghost.templates : error compiling template %s : %s", path, err)
return err
}
}
return nil
})
}
// Compile a single template file, using the specified base directory. The base
// directory is used to set the name of the template (the part of the path relative to this
// base directory is used as the name of the template).
func Compile(path, base string) error {
mu.Lock()
defer mu.Unlock()
return compileTemplate(path, base)
}
// Compile the specified template file if there is a matching compiler.
func compileTemplate(p, base string) error {
ext := path.Ext(p)
c, ok := compilers[ext]
// Ignore file if no template compiler exist for this extension
if ok {
t, err := c.Compile(p)
if err != nil {
return err
}
key, err := filepath.Rel(base, p)
if err != nil {
return err
}
ghost.LogFn("ghost.templates : storing template for file %s", key)
templaters[key] = t
}
return nil
}
// Execute the template.
func Execute(tplName string, w io.Writer, data interface{}) error {
mu.RLock()
t, ok := templaters[tplName]
mu.RUnlock()
if !ok {
return ErrTemplateNotExist
}
return t.Execute(w, data)
}
// Render is the same as Execute, except that it takes a http.ResponseWriter
// instead of a generic io.Writer, and sets the Content-Type to text/html.
func Render(tplName string, w http.ResponseWriter, data interface{}) (err error) {
w.Header().Set("Content-Type", "text/html")
defer func() {
if err != nil {
w.Header().Del("Content-Type")
}
}()
return Execute(tplName, w, data)
}
+63
View File
@@ -0,0 +1,63 @@
package quantile
import (
"testing"
)
func BenchmarkInsertTargeted(b *testing.B) {
b.ReportAllocs()
s := NewTargeted(Targets)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}
func BenchmarkInsertTargetedSmallEpsilon(b *testing.B) {
s := NewTargeted(TargetsSmallEpsilon)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}
func BenchmarkInsertBiased(b *testing.B) {
s := NewLowBiased(0.01)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}
func BenchmarkInsertBiasedSmallEpsilon(b *testing.B) {
s := NewLowBiased(0.0001)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}
func BenchmarkQuery(b *testing.B) {
s := NewTargeted(Targets)
for i := float64(0); i < 1e6; i++ {
s.Insert(i)
}
b.ResetTimer()
n := float64(b.N)
for i := float64(0); i < n; i++ {
s.Query(i / n)
}
}
func BenchmarkQuerySmallEpsilon(b *testing.B) {
s := NewTargeted(TargetsSmallEpsilon)
for i := float64(0); i < 1e6; i++ {
s.Insert(i)
}
b.ResetTimer()
n := float64(b.N)
for i := float64(0); i < n; i++ {
s.Query(i / n)
}
}
+121
View File
@@ -0,0 +1,121 @@
// +build go1.1
package quantile_test
import (
"bufio"
"fmt"
"log"
"os"
"strconv"
"time"
"github.com/beorn7/perks/quantile"
)
func Example_simple() {
ch := make(chan float64)
go sendFloats(ch)
// Compute the 50th, 90th, and 99th percentile.
q := quantile.NewTargeted(map[float64]float64{
0.50: 0.005,
0.90: 0.001,
0.99: 0.0001,
})
for v := range ch {
q.Insert(v)
}
fmt.Println("perc50:", q.Query(0.50))
fmt.Println("perc90:", q.Query(0.90))
fmt.Println("perc99:", q.Query(0.99))
fmt.Println("count:", q.Count())
// Output:
// perc50: 5
// perc90: 16
// perc99: 223
// count: 2388
}
func Example_mergeMultipleStreams() {
// Scenario:
// We have multiple database shards. On each shard, there is a process
// collecting query response times from the database logs and inserting
// them into a Stream (created via NewTargeted(0.90)), much like the
// Simple example. These processes expose a network interface for us to
// ask them to serialize and send us the results of their
// Stream.Samples so we may Merge and Query them.
//
// NOTES:
// * These sample sets are small, allowing us to get them
// across the network much faster than sending the entire list of data
// points.
//
// * For this to work correctly, we must supply the same quantiles
// a priori the process collecting the samples supplied to NewTargeted,
// even if we do not plan to query them all here.
ch := make(chan quantile.Samples)
getDBQuerySamples(ch)
q := quantile.NewTargeted(map[float64]float64{0.90: 0.001})
for samples := range ch {
q.Merge(samples)
}
fmt.Println("perc90:", q.Query(0.90))
}
func Example_window() {
// Scenario: We want the 90th, 95th, and 99th percentiles for each
// minute.
ch := make(chan float64)
go sendStreamValues(ch)
tick := time.NewTicker(1 * time.Minute)
q := quantile.NewTargeted(map[float64]float64{
0.90: 0.001,
0.95: 0.0005,
0.99: 0.0001,
})
for {
select {
case t := <-tick.C:
flushToDB(t, q.Samples())
q.Reset()
case v := <-ch:
q.Insert(v)
}
}
}
func sendStreamValues(ch chan float64) {
// Use your imagination
}
func flushToDB(t time.Time, samples quantile.Samples) {
// Use your imagination
}
// This is a stub for the above example. In reality this would hit the remote
// servers via http or something like it.
func getDBQuerySamples(ch chan quantile.Samples) {}
func sendFloats(ch chan<- float64) {
f, err := os.Open("exampledata.txt")
if err != nil {
log.Fatal(err)
}
sc := bufio.NewScanner(f)
for sc.Scan() {
b := sc.Bytes()
v, err := strconv.ParseFloat(string(b), 64)
if err != nil {
log.Fatal(err)
}
ch <- v
}
if sc.Err() != nil {
log.Fatal(sc.Err())
}
close(ch)
}
File diff suppressed because it is too large Load Diff
+292
View File
@@ -0,0 +1,292 @@
// Package quantile computes approximate quantiles over an unbounded data
// stream within low memory and CPU bounds.
//
// A small amount of accuracy is traded to achieve the above properties.
//
// Multiple streams can be merged before calling Query to generate a single set
// of results. This is meaningful when the streams represent the same type of
// data. See Merge and Samples.
//
// For more detailed information about the algorithm used, see:
//
// Effective Computation of Biased Quantiles over Data Streams
//
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile
import (
"math"
"sort"
)
// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
Value float64 `json:",string"`
Width float64 `json:",string"`
Delta float64 `json:",string"`
}
// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample
func (a Samples) Len() int { return len(a) }
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
type invariant func(s *stream, r float64) float64
// NewLowBiased returns an initialized Stream for low-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the lower ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewLowBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * r
}
return newStream(ƒ)
}
// NewHighBiased returns an initialized Stream for high-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the higher ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewHighBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * (s.n - r)
}
return newStream(ƒ)
}
// NewTargeted returns an initialized Stream concerned with a particular set of
// quantile values that are supplied a priori. Knowing these a priori reduces
// space and computation time. The targets map maps the desired quantiles to
// their absolute errors, i.e. the true quantile of a value returned by a query
// is guaranteed to be within (Quantile±Epsilon).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewTargeted(targets map[float64]float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
var m = math.MaxFloat64
var f float64
for quantile, epsilon := range targets {
if quantile*s.n <= r {
f = (2 * epsilon * r) / quantile
} else {
f = (2 * epsilon * (s.n - r)) / (1 - quantile)
}
if f < m {
m = f
}
}
return m
}
return newStream(ƒ)
}
// Stream computes quantiles for a stream of float64s. It is not thread-safe by
// design. Take care when using across multiple goroutines.
type Stream struct {
*stream
b Samples
sorted bool
}
func newStream(ƒ invariant) *Stream {
x := &stream{ƒ: ƒ}
return &Stream{x, make(Samples, 0, 500), true}
}
// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
s.insert(Sample{Value: v, Width: 1})
}
func (s *Stream) insert(sample Sample) {
s.b = append(s.b, sample)
s.sorted = false
if len(s.b) == cap(s.b) {
s.flush()
}
}
// Query returns the computed qth percentiles value. If s was created with
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
// will return an unspecified result.
func (s *Stream) Query(q float64) float64 {
if !s.flushed() {
// Fast path when there hasn't been enough data for a flush;
// this also yields better accuracy for small sets of data.
l := len(s.b)
if l == 0 {
return 0
}
i := int(float64(l) * q)
if i > 0 {
i -= 1
}
s.maybeSort()
return s.b[i].Value
}
s.flush()
return s.stream.query(q)
}
// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads, database shards, etc.
//
// ATTENTION: This method is broken and does not yield correct results. The
// underlying algorithm is not capable of merging streams correctly.
func (s *Stream) Merge(samples Samples) {
sort.Sort(samples)
s.stream.merge(samples)
}
// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
s.stream.reset()
s.b = s.b[:0]
}
// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
if !s.flushed() {
return s.b
}
s.flush()
return s.stream.samples()
}
// Count returns the total number of samples observed in the stream
// since initialization.
func (s *Stream) Count() int {
return len(s.b) + s.stream.count()
}
func (s *Stream) flush() {
s.maybeSort()
s.stream.merge(s.b)
s.b = s.b[:0]
}
func (s *Stream) maybeSort() {
if !s.sorted {
s.sorted = true
sort.Sort(s.b)
}
}
func (s *Stream) flushed() bool {
return len(s.stream.l) > 0
}
type stream struct {
n float64
l []Sample
ƒ invariant
}
func (s *stream) reset() {
s.l = s.l[:0]
s.n = 0
}
func (s *stream) insert(v float64) {
s.merge(Samples{{v, 1, 0}})
}
func (s *stream) merge(samples Samples) {
// TODO(beorn7): This tries to merge not only individual samples, but
// whole summaries. The paper doesn't mention merging summaries at
// all. Unittests show that the merging is inaccurate. Find out how to
// do merges properly.
var r float64
i := 0
for _, sample := range samples {
for ; i < len(s.l); i++ {
c := s.l[i]
if c.Value > sample.Value {
// Insert at position i.
s.l = append(s.l, Sample{})
copy(s.l[i+1:], s.l[i:])
s.l[i] = Sample{
sample.Value,
sample.Width,
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
// TODO(beorn7): How to calculate delta correctly?
}
i++
goto inserted
}
r += c.Width
}
s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
i++
inserted:
s.n += sample.Width
r += sample.Width
}
s.compress()
}
func (s *stream) count() int {
return int(s.n)
}
func (s *stream) query(q float64) float64 {
t := math.Ceil(q * s.n)
t += math.Ceil(s.ƒ(s, t) / 2)
p := s.l[0]
var r float64
for _, c := range s.l[1:] {
r += p.Width
if r+c.Width+c.Delta > t {
return p.Value
}
p = c
}
return p.Value
}
func (s *stream) compress() {
if len(s.l) < 2 {
return
}
x := s.l[len(s.l)-1]
xi := len(s.l) - 1
r := s.n - 1 - x.Width
for i := len(s.l) - 2; i >= 0; i-- {
c := s.l[i]
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
x.Width += c.Width
s.l[xi] = x
// Remove element at i.
copy(s.l[i:], s.l[i+1:])
s.l = s.l[:len(s.l)-1]
xi -= 1
} else {
x = c
xi = i
}
r -= c.Width
}
}
func (s *stream) samples() Samples {
samples := make(Samples, len(s.l))
copy(samples, s.l)
return samples
}
+188
View File
@@ -0,0 +1,188 @@
package quantile
import (
"math"
"math/rand"
"sort"
"testing"
)
var (
Targets = map[float64]float64{
0.01: 0.001,
0.10: 0.01,
0.50: 0.05,
0.90: 0.01,
0.99: 0.001,
}
TargetsSmallEpsilon = map[float64]float64{
0.01: 0.0001,
0.10: 0.001,
0.50: 0.005,
0.90: 0.001,
0.99: 0.0001,
}
LowQuantiles = []float64{0.01, 0.1, 0.5}
HighQuantiles = []float64{0.99, 0.9, 0.5}
)
const RelativeEpsilon = 0.01
func verifyPercsWithAbsoluteEpsilon(t *testing.T, a []float64, s *Stream) {
sort.Float64s(a)
for quantile, epsilon := range Targets {
n := float64(len(a))
k := int(quantile * n)
lower := int((quantile - epsilon) * n)
if lower < 1 {
lower = 1
}
upper := int(math.Ceil((quantile + epsilon) * n))
if upper > len(a) {
upper = len(a)
}
w, min, max := a[k-1], a[lower-1], a[upper-1]
if g := s.Query(quantile); g < min || g > max {
t.Errorf("q=%f: want %v [%f,%f], got %v", quantile, w, min, max, g)
}
}
}
func verifyLowPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
sort.Float64s(a)
for _, qu := range LowQuantiles {
n := float64(len(a))
k := int(qu * n)
lowerRank := int((1 - RelativeEpsilon) * qu * n)
upperRank := int(math.Ceil((1 + RelativeEpsilon) * qu * n))
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
if g := s.Query(qu); g < min || g > max {
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
}
}
}
func verifyHighPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
sort.Float64s(a)
for _, qu := range HighQuantiles {
n := float64(len(a))
k := int(qu * n)
lowerRank := int((1 - (1+RelativeEpsilon)*(1-qu)) * n)
upperRank := int(math.Ceil((1 - (1-RelativeEpsilon)*(1-qu)) * n))
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
if g := s.Query(qu); g < min || g > max {
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
}
}
}
func populateStream(s *Stream) []float64 {
a := make([]float64, 0, 1e5+100)
for i := 0; i < cap(a); i++ {
v := rand.NormFloat64()
// Add 5% asymmetric outliers.
if i%20 == 0 {
v = v*v + 1
}
s.Insert(v)
a = append(a, v)
}
return a
}
func TestTargetedQuery(t *testing.T) {
rand.Seed(42)
s := NewTargeted(Targets)
a := populateStream(s)
verifyPercsWithAbsoluteEpsilon(t, a, s)
}
func TestLowBiasedQuery(t *testing.T) {
rand.Seed(42)
s := NewLowBiased(RelativeEpsilon)
a := populateStream(s)
verifyLowPercsWithRelativeEpsilon(t, a, s)
}
func TestHighBiasedQuery(t *testing.T) {
rand.Seed(42)
s := NewHighBiased(RelativeEpsilon)
a := populateStream(s)
verifyHighPercsWithRelativeEpsilon(t, a, s)
}
// BrokenTestTargetedMerge is broken, see Merge doc comment.
func BrokenTestTargetedMerge(t *testing.T) {
rand.Seed(42)
s1 := NewTargeted(Targets)
s2 := NewTargeted(Targets)
a := populateStream(s1)
a = append(a, populateStream(s2)...)
s1.Merge(s2.Samples())
verifyPercsWithAbsoluteEpsilon(t, a, s1)
}
// BrokenTestLowBiasedMerge is broken, see Merge doc comment.
func BrokenTestLowBiasedMerge(t *testing.T) {
rand.Seed(42)
s1 := NewLowBiased(RelativeEpsilon)
s2 := NewLowBiased(RelativeEpsilon)
a := populateStream(s1)
a = append(a, populateStream(s2)...)
s1.Merge(s2.Samples())
verifyLowPercsWithRelativeEpsilon(t, a, s2)
}
// BrokenTestHighBiasedMerge is broken, see Merge doc comment.
func BrokenTestHighBiasedMerge(t *testing.T) {
rand.Seed(42)
s1 := NewHighBiased(RelativeEpsilon)
s2 := NewHighBiased(RelativeEpsilon)
a := populateStream(s1)
a = append(a, populateStream(s2)...)
s1.Merge(s2.Samples())
verifyHighPercsWithRelativeEpsilon(t, a, s2)
}
func TestUncompressed(t *testing.T) {
q := NewTargeted(Targets)
for i := 100; i > 0; i-- {
q.Insert(float64(i))
}
if g := q.Count(); g != 100 {
t.Errorf("want count 100, got %d", g)
}
// Before compression, Query should have 100% accuracy.
for quantile := range Targets {
w := quantile * 100
if g := q.Query(quantile); g != w {
t.Errorf("want %f, got %f", w, g)
}
}
}
func TestUncompressedSamples(t *testing.T) {
q := NewTargeted(map[float64]float64{0.99: 0.001})
for i := 1; i <= 100; i++ {
q.Insert(float64(i))
}
if g := q.Samples().Len(); g != 100 {
t.Errorf("want count 100, got %d", g)
}
}
func TestUncompressedOne(t *testing.T) {
q := NewTargeted(map[float64]float64{0.99: 0.01})
q.Insert(3.14)
if g := q.Query(0.90); g != 3.14 {
t.Error("want PI, got", g)
}
}
func TestDefaults(t *testing.T) {
if g := NewTargeted(map[float64]float64{0.99: 0.001}).Query(0.99); g != 0 {
t.Errorf("want 0, got %f", g)
}
}
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2015 Jun Kimura
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
+224
View File
@@ -0,0 +1,224 @@
# GCache [![wercker status](https://app.wercker.com/status/1471b6c9cbc9ebbd15f8f9fe8f71ac67/s "wercker status")](https://app.wercker.com/project/bykey/1471b6c9cbc9ebbd15f8f9fe8f71ac67)[![GoDoc](https://godoc.org/github.com/bluele/gcache?status.png)](https://godoc.org/github.com/bluele/gcache)
Cache library for golang. It supports expirable Cache, LFU, LRU and ARC.
## Features
* Supports expirable Cache, LFU, LRU and ARC.
* Goroutine safe.
* Supports event handlers which evict and add entry. (Optional)
* Automatically load cache if it doesn't exists. (Optional)
## Install
```
$ go get github.com/bluele/gcache
```
## Example
### Manually set a key-value pair.
```go
package main
import (
"github.com/bluele/gcache"
"fmt"
)
func main() {
gc := gcache.New(20).
LRU().
Build()
gc.Set("key", "ok")
value, err := gc.Get("key")
if err != nil {
panic(err)
}
fmt.Println("Get:", value)
}
```
```
Get: ok
```
### Automatically load value
```go
package main
import (
"github.com/bluele/gcache"
"fmt"
)
func main() {
gc := gcache.New(20).
LRU().
LoaderFunc(func(key interface{}) (interface{}, error) {
return "ok", nil
}).
Build()
value, err := gc.Get("key")
if err != nil {
panic(err)
}
fmt.Println("Get:", value)
}
```
```
Get: ok
```
## Cache Algorithm
* Least-Frequently Used (LFU)
Discards the least frequently used items first.
```go
func main() {
// size: 10
gc := gcache.New(10).
LFU().
Build()
gc.Set("key", "value")
}
```
* Least Recently Used (LRU)
Discards the least recently used items first.
```go
func main() {
// size: 10
gc := gcache.New(10).
LRU().
Build()
gc.Set("key", "value")
}
```
* Adaptive Replacement Cache (ARC)
Constantly balances between LRU and LFU, to improve the combined result.
detail: http://en.wikipedia.org/wiki/Adaptive_replacement_cache
```go
func main() {
// size: 10
gc := gcache.New(10).
ARC().
Build()
gc.Set("key", "value")
}
```
* SimpleCache (Default)
SimpleCache has no clear priority for evict cache. It depends on key-value map order.
```go
func main() {
// size: 10
gc := gcache.New(10).Build()
gc.Set("key", "value")
v, err := gc.Get("key")
if err != nil {
panic(err)
}
}
```
## Loading Cache
If specified `LoaderFunc`, values are automatically loaded by the cache, and are stored in the cache until either evicted or manually invalidated.
```go
func main() {
gc := gcache.New(10).
LRU().
LoaderFunc(func(key interface{}) (interface{}, error) {
return "value", nil
}).
Build()
v, _ := gc.Get("key")
// output: "value"
fmt.Println(v)
}
```
GCache coordinates cache fills such that only one load in one process of an entire replicated set of processes populates the cache, then multiplexes the loaded value to all callers.
## Expirable cache
```go
func main() {
// LRU cache, size: 10, expiration: after a hour
gc := gcache.New(10).
LRU().
Expiration(time.Hour).
Build()
}
```
## Event handlers
### Evicted handler
Event handler for evict the entry.
```go
func main() {
gc := gcache.New(2).
EvictedFunc(func(key, value interface{}) {
fmt.Println("evicted key:", key)
}).
Build()
for i := 0; i < 3; i++ {
gc.Set(i, i*i)
}
}
```
```
evicted key: 0
```
### Added handler
Event handler for add the entry.
```go
func main() {
gc := gcache.New(2).
AddedFunc(func(key, value interface{}) {
fmt.Println("added key:", key)
}).
Build()
for i := 0; i < 3; i++ {
gc.Set(i, i*i)
}
}
```
```
added key: 0
added key: 1
added key: 2
```
# Author
**Jun Kimura**
* <http://github.com/bluele>
* <junkxdev@gmail.com>
+331
View File
@@ -0,0 +1,331 @@
package gcache
import (
"container/list"
"time"
)
// Constantly balances between LRU and LFU, to improve the combined result.
type ARC struct {
baseCache
items map[interface{}]*arcItem
part int
t1 *arcList
t2 *arcList
b1 *arcList
b2 *arcList
}
func newARC(cb *CacheBuilder) *ARC {
c := &ARC{
items: make(map[interface{}]*arcItem),
t1: newARCList(),
t2: newARCList(),
b1: newARCList(),
b2: newARCList(),
}
buildCache(&c.baseCache, cb)
return c
}
func (c *ARC) replace(key interface{}) {
var old interface{}
if (c.t1.Len() > 0 && c.b2.Has(key) && c.t1.Len() == c.part) || (c.t1.Len() > c.part) {
old = c.t1.RemoveTail()
c.b1.PushFront(old)
} else {
old = c.t2.RemoveTail()
c.b2.PushFront(old)
}
item, ok := c.items[old]
if ok {
delete(c.items, old)
if c.evictedFunc != nil {
go (*c.evictedFunc)(item.key, item.value)
}
}
}
func (c *ARC) Set(key, value interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.set(key, value)
}
func (c *ARC) set(key, value interface{}) (interface{}, error) {
item, ok := c.items[key]
if ok {
item.value = value
} else {
item = &arcItem{
key: key,
value: value,
}
c.items[key] = item
}
if c.expiration != nil {
t := time.Now().Add(*c.expiration)
item.expiration = &t
}
if elt := c.b1.Lookup(key); elt != nil {
c.part = minInt(c.size, c.part+maxInt(c.b2.Len()/c.b1.Len(), 1))
c.replace(key)
c.b1.Remove(key, elt)
c.t2.PushFront(key)
return item, nil
}
if elt := c.b2.Lookup(key); elt != nil {
c.part = maxInt(0, c.part-maxInt(c.b1.Len()/c.b2.Len(), 1))
c.replace(key)
c.b2.Remove(key, elt)
c.t2.PushFront(key)
return item, nil
}
if c.t1.Len()+c.b1.Len() == c.size {
if c.t1.Len() < c.size {
c.b1.RemoveTail()
c.replace(key)
} else {
pop := c.t1.RemoveTail()
item, ok := c.items[pop]
if ok {
delete(c.items, pop)
if c.evictedFunc != nil {
go (*c.evictedFunc)(item.key, item.value)
}
}
}
} else {
total := c.t1.Len() + c.b1.Len() + c.t2.Len() + c.b2.Len()
if total >= c.size {
if total == (2 * c.size) {
c.b2.RemoveTail()
}
c.replace(key)
}
}
c.t1.PushFront(key)
if c.addedFunc != nil {
go (*c.addedFunc)(key, value)
}
return item, nil
}
// Get a value from cache pool using key if it exists. If not exists and it has LoaderFunc, it will generate the value using you have specified LoaderFunc method returns value.
func (c *ARC) Get(key interface{}) (interface{}, error) {
rl := false
c.mu.RLock()
if elt := c.t1.Lookup(key); elt != nil {
c.mu.RUnlock()
rl = true
c.mu.Lock()
c.t1.Remove(key, elt)
item := c.items[key]
if !item.IsExpired(nil) {
c.t2.PushFront(key)
c.mu.Unlock()
return item.value, nil
}
c.b2.PushFront(key)
if c.evictedFunc != nil {
go (*c.evictedFunc)(key, elt.Value)
}
c.mu.Unlock()
}
if elt := c.t2.Lookup(key); elt != nil {
c.mu.RUnlock()
rl = true
c.mu.Lock()
item := c.items[key]
if !item.IsExpired(nil) {
c.t2.MoveToFront(elt)
c.mu.Unlock()
return item.value, nil
}
c.t2.Remove(key, elt)
c.b2.PushFront(key)
if c.evictedFunc != nil {
go (*c.evictedFunc)(key, elt.Value)
}
c.mu.Unlock()
}
if !rl {
c.mu.RUnlock()
}
if c.loaderFunc == nil {
return nil, NotFoundKeyError
}
item, err := c.load(key, func(v interface{}, e error) (interface{}, error) {
if e == nil {
c.mu.Lock()
defer c.mu.Unlock()
return c.set(key, v)
}
return nil, e
})
if err != nil {
return nil, err
}
return item.(*arcItem).value, nil
}
// Remove removes the provided key from the cache.
func (c *ARC) Remove(key interface{}) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.remove(key)
}
func (c *ARC) remove(key interface{}) bool {
if elt := c.t1.Lookup(key); elt != nil {
v := elt.Value.(*arcItem).value
c.t1.Remove(key, elt)
if c.evictedFunc != nil {
go (*c.evictedFunc)(key, v)
}
return true
}
if elt := c.t2.Lookup(key); elt != nil {
v := elt.Value.(*arcItem).value
c.t2.Remove(key, elt)
if c.evictedFunc != nil {
go (*c.evictedFunc)(key, v)
}
return true
}
return false
}
// Keys returns a slice of the keys in the cache.
func (c *ARC) Keys() []interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
keys := []interface{}{}
for key := range c.items {
keys = append(keys, key)
}
return keys
}
// Len returns the number of items in the cache.
func (c *ARC) Len() int {
c.mu.RLock()
defer c.mu.RUnlock()
return len(c.items)
}
// Purge is used to completely clear the cache
func (c *ARC) Purge() {
c.mu.Lock()
defer c.mu.Unlock()
c.items = make(map[interface{}]*arcItem)
c.t1 = newARCList()
c.t2 = newARCList()
c.b1 = newARCList()
c.b2 = newARCList()
}
func (c *ARC) gc() {
now := time.Now()
keys := []interface{}{}
c.mu.RLock()
for k, item := range c.items {
if item.IsExpired(&now) {
keys = append(keys, k)
}
}
c.mu.RUnlock()
if len(keys) == 0 {
return
}
c.mu.Lock()
for _, k := range keys {
c.remove(k)
}
c.mu.Unlock()
}
// returns boolean value whether this item is expired or not.
func (it *arcItem) IsExpired(now *time.Time) bool {
if it.expiration == nil {
return false
}
if now == nil {
t := time.Now()
now = &t
}
return it.expiration.Before(*now)
}
type arcList struct {
l *list.List
keys map[interface{}]*list.Element
}
type arcItem struct {
key interface{}
value interface{}
expiration *time.Time
}
func newARCList() *arcList {
return &arcList{
l: list.New(),
keys: make(map[interface{}]*list.Element),
}
}
func (al *arcList) Has(key interface{}) bool {
_, ok := al.keys[key]
return ok
}
func (al *arcList) Lookup(key interface{}) *list.Element {
elt := al.keys[key]
return elt
}
func (al *arcList) MoveToFront(elt *list.Element) {
al.l.MoveToFront(elt)
}
func (al *arcList) PushFront(key interface{}) {
elt := al.l.PushFront(key)
al.keys[key] = elt
}
func (al *arcList) Remove(key interface{}, elt *list.Element) {
delete(al.keys, key)
al.l.Remove(elt)
}
func (al *arcList) RemoveTail() interface{} {
elt := al.l.Back()
al.l.Remove(elt)
key := elt.Value
delete(al.keys, key)
return key
}
func (al *arcList) Len() int {
return al.l.Len()
}
+63
View File
@@ -0,0 +1,63 @@
package gcache_test
import (
"fmt"
"github.com/bluele/gcache"
"testing"
)
func buildARCache(size int) gcache.Cache {
return gcache.New(size).
ARC().
EvictedFunc(evictedFuncForARC).
Build()
}
func buildLoadingARCache(size int) gcache.Cache {
return gcache.New(size).
ARC().
LoaderFunc(loader).
EvictedFunc(evictedFuncForARC).
Build()
}
func evictedFuncForARC(key, value interface{}) {
fmt.Printf("[ARC] Key:%v Value:%v will evicted.\n", key, value)
}
func TestARCGet(t *testing.T) {
size := 1000
gc := buildARCache(size)
testSetCache(t, gc, size)
testGetCache(t, gc, size)
}
func TestLoadingARCGet(t *testing.T) {
size := 1000
numbers := 1000
testGetCache(t, buildLoadingARCache(size), numbers)
}
func TestARCLength(t *testing.T) {
gc := buildLoadingARCache(1000)
gc.Get("test1")
gc.Get("test2")
length := gc.Len()
expectedLength := 2
if gc.Len() != expectedLength {
t.Errorf("Expected length is %v, not %v", length, expectedLength)
}
}
func TestARCEvictItem(t *testing.T) {
cacheSize := 10
numbers := 11
gc := buildLoadingARCache(cacheSize)
for i := 0; i < numbers; i++ {
_, err := gc.Get(fmt.Sprintf("Key-%d", i))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
}
}
+160
View File
@@ -0,0 +1,160 @@
package gcache
import (
"errors"
"github.com/bluele/gcache/singleflight"
"sync"
"time"
)
const (
TYPE_SIMPLE = "simple"
TYPE_LRU = "lru"
TYPE_LFU = "lfu"
TYPE_ARC = "arc"
)
var NotFoundKeyError = errors.New("Not found error.")
type Cache interface {
Set(interface{}, interface{})
Get(interface{}) (interface{}, error)
Remove(interface{}) bool
Purge()
Keys() []interface{}
Len() int
gc()
}
type baseCache struct {
size int
loaderFunc *LoaderFunc
evictedFunc *EvictedFunc
addedFunc *AddedFunc
expiration *time.Duration
mu sync.RWMutex
loadGroup singleflight.Group
}
type LoaderFunc func(interface{}) (interface{}, error)
type EvictedFunc func(interface{}, interface{})
type AddedFunc func(interface{}, interface{})
type CacheBuilder struct {
tp string
size int
loaderFunc *LoaderFunc
evictedFunc *EvictedFunc
addedFunc *AddedFunc
expiration *time.Duration
gcInterval *time.Duration
}
func New(size int) *CacheBuilder {
if size <= 0 {
panic("gcache: size <= 0")
}
return &CacheBuilder{
tp: TYPE_SIMPLE,
size: size,
}
}
func (cb *CacheBuilder) LoaderFunc(loaderFunc LoaderFunc) *CacheBuilder {
cb.loaderFunc = &loaderFunc
return cb
}
func (cb *CacheBuilder) EnableGC(interval time.Duration) *CacheBuilder {
cb.gcInterval = &interval
return cb
}
func (cb *CacheBuilder) EvictType(tp string) *CacheBuilder {
cb.tp = tp
return cb
}
func (cb *CacheBuilder) Simple() *CacheBuilder {
return cb.EvictType(TYPE_SIMPLE)
}
func (cb *CacheBuilder) LRU() *CacheBuilder {
return cb.EvictType(TYPE_LRU)
}
func (cb *CacheBuilder) LFU() *CacheBuilder {
return cb.EvictType(TYPE_LFU)
}
func (cb *CacheBuilder) ARC() *CacheBuilder {
return cb.EvictType(TYPE_ARC)
}
func (cb *CacheBuilder) EvictedFunc(evictedFunc EvictedFunc) *CacheBuilder {
cb.evictedFunc = &evictedFunc
return cb
}
func (cb *CacheBuilder) AddedFunc(addedFunc AddedFunc) *CacheBuilder {
cb.addedFunc = &addedFunc
return cb
}
func (cb *CacheBuilder) Expiration(expiration time.Duration) *CacheBuilder {
cb.expiration = &expiration
return cb
}
func (cb *CacheBuilder) Build() Cache {
cache := cb.build()
if cb.gcInterval != nil {
go func() {
t := time.NewTicker(*cb.gcInterval)
for {
select {
case <-t.C:
go cache.gc()
}
}
t.Stop()
}()
}
return cache
}
func (cb *CacheBuilder) build() Cache {
switch cb.tp {
case TYPE_SIMPLE:
return newSimpleCache(cb)
case TYPE_LRU:
return newLRUCache(cb)
case TYPE_LFU:
return newLFUCache(cb)
case TYPE_ARC:
return newARC(cb)
default:
panic("gcache: Unknown type " + cb.tp)
}
}
func buildCache(c *baseCache, cb *CacheBuilder) {
c.size = cb.size
c.loaderFunc = cb.loaderFunc
c.expiration = cb.expiration
c.addedFunc = cb.addedFunc
c.evictedFunc = cb.evictedFunc
}
// load a new value using by specified key.
func (c *baseCache) load(key interface{}, cb func(interface{}, error) (interface{}, error)) (interface{}, error) {
v, err := c.loadGroup.Do(key, func() (interface{}, error) {
return cb((*c.loaderFunc)(key))
})
if err != nil {
return nil, err
}
return v, nil
}
+21
View File
@@ -0,0 +1,21 @@
package main
import (
"fmt"
"github.com/bluele/gcache"
)
func main() {
gc := gcache.New(10).
LFU().
LoaderFunc(func(key interface{}) (interface{}, error) {
return fmt.Sprintf("%v-value", key), nil
}).
Build()
v, err := gc.Get("key")
if err != nil {
panic(err)
}
fmt.Println(v)
}
+19
View File
@@ -0,0 +1,19 @@
package main
import (
"fmt"
"github.com/bluele/gcache"
)
func main() {
gc := gcache.New(10).
LFU().
Build()
gc.Set("key", "ok")
v, err := gc.Get("key")
if err != nil {
panic(err)
}
fmt.Println("value:", v)
}
+37
View File
@@ -0,0 +1,37 @@
package gcache_test
import (
"fmt"
"github.com/bluele/gcache"
"testing"
)
func loader(key interface{}) (interface{}, error) {
return fmt.Sprintf("valueFor%s", key), nil
}
func testSetCache(t *testing.T, gc gcache.Cache, numbers int) {
for i := 0; i < numbers; i++ {
key := fmt.Sprintf("Key-%d", i)
value, err := loader(key)
if err != nil {
t.Error(err)
return
}
gc.Set(key, value)
}
}
func testGetCache(t *testing.T, gc gcache.Cache, numbers int) {
for i := 0; i < numbers; i++ {
key := fmt.Sprintf("Key-%d", i)
v, err := gc.Get(key)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
expectedV, _ := loader(key)
if v != expectedV {
t.Errorf("Expected value is %v, not %v", expectedV, v)
}
}
}
+248
View File
@@ -0,0 +1,248 @@
package gcache
import (
"container/list"
"time"
)
// Discards the least frequently used items first.
type LFUCache struct {
baseCache
items map[interface{}]*lfuItem
freqList *list.List // list for freqEntry
}
func newLFUCache(cb *CacheBuilder) *LFUCache {
c := &LFUCache{}
buildCache(&c.baseCache, cb)
c.freqList = list.New()
c.items = make(map[interface{}]*lfuItem, c.size+1)
c.freqList.PushFront(&freqEntry{
freq: 0,
items: make(map[*lfuItem]byte),
})
return c
}
// set a new key-value pair
func (c *LFUCache) Set(key, value interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.set(key, value)
}
func (c *LFUCache) set(key, value interface{}) (*lfuItem, error) {
// Check for existing item
item, ok := c.items[key]
if ok {
item.value = value
} else {
// Verify size not exceeded
if len(c.items) >= c.size {
c.evict(1)
}
item = &lfuItem{
key: key,
value: value,
freqElement: nil,
}
el := c.freqList.Front()
fe := el.Value.(*freqEntry)
fe.items[item] = 1
item.freqElement = el
c.items[key] = item
}
if c.expiration != nil {
t := time.Now().Add(*c.expiration)
item.expiration = &t
}
if c.addedFunc != nil {
go (*c.addedFunc)(key, value)
}
return item, nil
}
// Get a value from cache pool using key if it exists.
// If it dose not exists key and has LoaderFunc,
// generate a value using `LoaderFunc` method returns value.
func (c *LFUCache) Get(key interface{}) (interface{}, error) {
c.mu.RLock()
item, ok := c.items[key]
c.mu.RUnlock()
if ok {
if !item.IsExpired(nil) {
c.mu.Lock()
c.increment(item)
c.mu.Unlock()
return item.value, nil
}
c.mu.Lock()
c.removeItem(item)
c.mu.Unlock()
}
if c.loaderFunc == nil {
return nil, NotFoundKeyError
}
it, err := c.load(key, func(v interface{}, e error) (interface{}, error) {
if e == nil {
c.mu.Lock()
defer c.mu.Unlock()
return c.set(key, v)
}
return nil, e
})
if err != nil {
return nil, err
}
c.mu.Lock()
defer c.mu.Unlock()
li := it.(*lfuItem)
c.increment(li)
return li.value, nil
}
func (c *LFUCache) increment(item *lfuItem) {
currentFreqElement := item.freqElement
currentFreqEntry := currentFreqElement.Value.(*freqEntry)
nextFreq := currentFreqEntry.freq + 1
delete(currentFreqEntry.items, item)
nextFreqElement := currentFreqElement.Next()
if nextFreqElement == nil {
nextFreqElement = c.freqList.InsertAfter(&freqEntry{
freq: nextFreq,
items: make(map[*lfuItem]byte),
}, currentFreqElement)
}
nextFreqElement.Value.(*freqEntry).items[item] = 1
item.freqElement = nextFreqElement
}
// evict removes the least frequence item from the cache.
func (c *LFUCache) evict(count int) {
entry := c.freqList.Front()
for i := 0; i < count; {
if entry == nil {
return
} else {
for item, _ := range entry.Value.(*freqEntry).items {
if i >= count {
return
}
c.removeItem(item)
i++
}
entry = entry.Next()
}
}
}
// Removes the provided key from the cache.
func (c *LFUCache) Remove(key interface{}) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.remove(key)
}
func (c *LFUCache) remove(key interface{}) bool {
if item, ok := c.items[key]; ok {
c.removeItem(item)
return true
}
return false
}
// removeElement is used to remove a given list element from the cache
func (c *LFUCache) removeItem(item *lfuItem) {
delete(c.items, item.key)
delete(item.freqElement.Value.(*freqEntry).items, item)
if c.evictedFunc != nil {
go (*c.evictedFunc)(item.key, item.value)
}
}
// Returns a slice of the keys in the cache.
func (c *LFUCache) Keys() []interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
keys := make([]interface{}, len(c.items))
i := 0
for k := range c.items {
keys[i] = k
i++
}
return keys
}
// Returns the number of items in the cache.
func (c *LFUCache) Len() int {
c.mu.RLock()
defer c.mu.RUnlock()
return len(c.items)
}
// Completely clear the cache
func (c *LFUCache) Purge() {
c.mu.Lock()
defer c.mu.Unlock()
c.freqList = list.New()
c.items = make(map[interface{}]*lfuItem, c.size)
}
// evict all expired entry
func (c *LFUCache) gc() {
now := time.Now()
keys := []interface{}{}
c.mu.RLock()
for k, item := range c.items {
if item.IsExpired(&now) {
keys = append(keys, k)
}
}
c.mu.RUnlock()
if len(keys) == 0 {
return
}
c.mu.Lock()
for _, k := range keys {
c.remove(k)
}
c.mu.Unlock()
}
type freqEntry struct {
freq uint
items map[*lfuItem]byte
}
type lfuItem struct {
key interface{}
value interface{}
freqElement *list.Element
expiration *time.Time
}
// returns boolean value whether this item is expired or not.
func (it *lfuItem) IsExpired(now *time.Time) bool {
if it.expiration == nil {
return false
}
if now == nil {
t := time.Now()
now = &t
}
return it.expiration.Before(*now)
}
+70
View File
@@ -0,0 +1,70 @@
package gcache_test
import (
"fmt"
"github.com/bluele/gcache"
"testing"
"time"
)
func evictedFuncForLFU(key, value interface{}) {
fmt.Printf("[LFU] Key:%v Value:%v will evicted.\n", key, value)
}
func buildLFUCache(size int) gcache.Cache {
return gcache.New(size).
LFU().
EvictedFunc(evictedFuncForLFU).
Expiration(time.Second).
Build()
}
func buildLoadingLFUCache(size int, loader gcache.LoaderFunc) gcache.Cache {
return gcache.New(size).
LFU().
LoaderFunc(loader).
EvictedFunc(evictedFuncForLFU).
Expiration(time.Second).
Build()
}
func TestLFUGet(t *testing.T) {
size := 1000
numbers := 1000
gc := buildLoadingLFUCache(size, loader)
testSetCache(t, gc, numbers)
testGetCache(t, gc, numbers)
}
func TestLoadingLFUGet(t *testing.T) {
size := 1000
numbers := 1000
gc := buildLoadingLFUCache(size, loader)
testGetCache(t, gc, numbers)
}
func TestLFULength(t *testing.T) {
gc := buildLoadingLFUCache(1000, loader)
gc.Get("test1")
gc.Get("test2")
length := gc.Len()
expectedLength := 2
if gc.Len() != expectedLength {
t.Errorf("Expected length is %v, not %v", length, expectedLength)
}
}
func TestLFUEvictItem(t *testing.T) {
cacheSize := 10
numbers := 11
gc := buildLoadingLFUCache(cacheSize, loader)
for i := 0; i < numbers; i++ {
_, err := gc.Get(fmt.Sprintf("Key-%d", i))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
}
}
+207
View File
@@ -0,0 +1,207 @@
package gcache
import (
"container/list"
"time"
)
// Discards the least recently used items first.
type LRUCache struct {
baseCache
items map[interface{}]*list.Element
evictList *list.List
}
func newLRUCache(cb *CacheBuilder) *LRUCache {
c := &LRUCache{}
buildCache(&c.baseCache, cb)
c.evictList = list.New()
c.items = make(map[interface{}]*list.Element, c.size+1)
return c
}
func (c *LRUCache) set(key, value interface{}) (interface{}, error) {
// Check for existing item
var item *lruItem
if it, ok := c.items[key]; ok {
c.evictList.MoveToFront(it)
item = it.Value.(*lruItem)
item.value = value
} else {
// Verify size not exceeded
if c.evictList.Len() >= c.size {
c.evict(1)
}
item = &lruItem{
key: key,
value: value,
}
c.items[key] = c.evictList.PushFront(item)
}
if c.expiration != nil {
t := time.Now().Add(*c.expiration)
item.expiration = &t
}
if c.addedFunc != nil {
go (*c.addedFunc)(key, value)
}
return item, nil
}
// set a new key-value pair
func (c *LRUCache) Set(key, value interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.set(key, value)
}
// Get a value from cache pool using key if it exists.
// If it dose not exists key and has LoaderFunc,
// generate a value using `LoaderFunc` method returns value.
func (c *LRUCache) Get(key interface{}) (interface{}, error) {
c.mu.RLock()
item, ok := c.items[key]
c.mu.RUnlock()
if ok {
it := item.Value.(*lruItem)
if !it.IsExpired(nil) {
c.mu.Lock()
defer c.mu.Unlock()
return it.value, nil
}
c.mu.Lock()
c.removeElement(item)
c.mu.Unlock()
}
if c.loaderFunc == nil {
return nil, NotFoundKeyError
}
it, err := c.load(key, func(v interface{}, e error) (interface{}, error) {
if e == nil {
c.mu.Lock()
defer c.mu.Unlock()
return c.set(key, v)
}
return nil, e
})
if err != nil {
return nil, err
}
return it.(*lruItem).value, nil
}
// evict removes the oldest item from the cache.
func (c *LRUCache) evict(count int) {
for i := 0; i < count; i++ {
ent := c.evictList.Back()
if ent == nil {
return
} else {
c.removeElement(ent)
}
}
}
// Removes the provided key from the cache.
func (c *LRUCache) Remove(key interface{}) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.remove(key)
}
func (c *LRUCache) remove(key interface{}) bool {
if ent, ok := c.items[key]; ok {
c.removeElement(ent)
return true
}
return false
}
func (c *LRUCache) removeElement(e *list.Element) {
c.evictList.Remove(e)
entry := e.Value.(*lruItem)
delete(c.items, entry.key)
if c.evictedFunc != nil {
entry := e.Value.(*lruItem)
go (*c.evictedFunc)(entry.key, entry.value)
}
}
// Returns a slice of the keys in the cache.
func (c *LRUCache) Keys() []interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
keys := make([]interface{}, len(c.items))
i := 0
for k := range c.items {
keys[i] = k
i++
}
return keys
}
// Returns the number of items in the cache.
func (c *LRUCache) Len() int {
c.mu.RLock()
defer c.mu.RUnlock()
return len(c.items)
}
// Completely clear the cache
func (c *LRUCache) Purge() {
c.mu.Lock()
defer c.mu.Unlock()
c.evictList = list.New()
c.items = make(map[interface{}]*list.Element, c.size)
}
// evict all expired entry
func (c *LRUCache) gc() {
now := time.Now()
keys := []interface{}{}
c.mu.RLock()
for k, item := range c.items {
if item.Value.(*lruItem).IsExpired(&now) {
keys = append(keys, k)
}
}
c.mu.RUnlock()
if len(keys) == 0 {
return
}
c.mu.Lock()
for _, k := range keys {
c.remove(k)
}
c.mu.Unlock()
}
type lruItem struct {
key interface{}
value interface{}
expiration *time.Time
}
// returns boolean value whether this item is expired or not.
func (it *lruItem) IsExpired(now *time.Time) bool {
if it.expiration == nil {
return false
}
if now == nil {
t := time.Now()
now = &t
}
return it.expiration.Before(*now)
}
+66
View File
@@ -0,0 +1,66 @@
package gcache_test
import (
"fmt"
"github.com/bluele/gcache"
"testing"
"time"
)
func evictedFuncForLRU(key, value interface{}) {
fmt.Printf("[LRU] Key:%v Value:%v will evicted.\n", key, value)
}
func buildLRUCache(size int) gcache.Cache {
return gcache.New(size).
LRU().
EvictedFunc(evictedFuncForLRU).
Expiration(time.Second).
Build()
}
func buildLoadingLRUCache(size int, loader gcache.LoaderFunc) gcache.Cache {
return gcache.New(size).
LRU().
LoaderFunc(loader).
EvictedFunc(evictedFuncForLRU).
Expiration(time.Second).
Build()
}
func TestLRUGet(t *testing.T) {
size := 1000
gc := buildLRUCache(size)
testSetCache(t, gc, size)
testGetCache(t, gc, size)
}
func TestLoadingLRUGet(t *testing.T) {
size := 1000
gc := buildLoadingLRUCache(size, loader)
testGetCache(t, gc, size)
}
func TestLRULength(t *testing.T) {
gc := buildLoadingLRUCache(1000, loader)
gc.Get("test1")
gc.Get("test2")
length := gc.Len()
expectedLength := 2
if length != expectedLength {
t.Errorf("Expected length is %v, not %v", length, expectedLength)
}
}
func TestLRUEvictItem(t *testing.T) {
cacheSize := 10
numbers := 11
gc := buildLoadingLRUCache(cacheSize, loader)
for i := 0; i < numbers; i++ {
_, err := gc.Get(fmt.Sprintf("Key-%d", i))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
}
}
+191
View File
@@ -0,0 +1,191 @@
package gcache
import (
"time"
)
// SimpleCache has no clear priority for evict cache. It depends on key-value map order.
type SimpleCache struct {
baseCache
items map[interface{}]*simpleItem
}
func newSimpleCache(cb *CacheBuilder) *SimpleCache {
c := &SimpleCache{}
buildCache(&c.baseCache, cb)
c.items = make(map[interface{}]*simpleItem, c.size)
return c
}
// set a new key-value pair
func (c *SimpleCache) Set(key, value interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.set(key, value)
}
func (c *SimpleCache) set(key, value interface{}) (interface{}, error) {
// Check for existing item
item, ok := c.items[key]
if ok {
item.value = value
} else {
// Verify size not exceeded
if len(c.items) >= c.size {
c.evict(1)
}
item = &simpleItem{
value: value,
}
c.items[key] = item
}
if c.expiration != nil {
t := time.Now().Add(*c.expiration)
item.expiration = &t
}
if c.addedFunc != nil {
go (*c.addedFunc)(key, value)
}
return item, nil
}
// Get a value from cache pool using key if it exists.
// If it dose not exists key and has LoaderFunc,
// generate a value using `LoaderFunc` method returns value.
func (c *SimpleCache) Get(key interface{}) (interface{}, error) {
c.mu.RLock()
item, ok := c.items[key]
c.mu.RUnlock()
if ok {
if !item.IsExpired(nil) {
return item.value, nil
}
c.mu.Lock()
c.remove(key)
c.mu.Unlock()
}
if c.loaderFunc == nil {
return nil, NotFoundKeyError
}
it, err := c.load(key, func(v interface{}, e error) (interface{}, error) {
if e == nil {
c.mu.Lock()
defer c.mu.Unlock()
return c.set(key, v)
}
return nil, e
})
if err != nil {
return nil, err
}
return it.(*simpleItem).value, nil
}
func (c *SimpleCache) evict(count int) {
now := time.Now()
current := 0
for key, item := range c.items {
if current >= count {
return
}
if item.expiration == nil || now.After(*item.expiration) {
defer c.remove(key)
current += 1
}
}
}
// Removes the provided key from the cache.
func (c *SimpleCache) Remove(key interface{}) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.remove(key)
}
func (c *SimpleCache) remove(key interface{}) bool {
item, ok := c.items[key]
if ok {
delete(c.items, key)
if c.evictedFunc != nil {
go (*c.evictedFunc)(key, item.value)
}
return true
}
return false
}
// Returns a slice of the keys in the cache.
func (c *SimpleCache) Keys() []interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
keys := make([]interface{}, len(c.items))
i := 0
for k := range c.items {
keys[i] = k
i++
}
return keys
}
// Returns the number of items in the cache.
func (c *SimpleCache) Len() int {
c.mu.RLock()
defer c.mu.RUnlock()
return len(c.items)
}
// Completely clear the cache
func (c *SimpleCache) Purge() {
c.mu.Lock()
defer c.mu.Unlock()
c.items = make(map[interface{}]*simpleItem, c.size)
}
// evict all expired entry
func (c *SimpleCache) gc() {
now := time.Now()
keys := []interface{}{}
c.mu.RLock()
for k, item := range c.items {
if item.IsExpired(&now) {
keys = append(keys, k)
}
}
c.mu.RUnlock()
if len(keys) == 0 {
return
}
c.mu.Lock()
for _, k := range keys {
c.remove(k)
}
c.mu.Unlock()
}
type simpleItem struct {
value interface{}
expiration *time.Time
}
// returns boolean value whether this item is expired or not.
func (si *simpleItem) IsExpired(now *time.Time) bool {
if si.expiration == nil {
return false
}
if now == nil {
t := time.Now()
now = &t
}
return si.expiration.Before(*now)
}
+63
View File
@@ -0,0 +1,63 @@
package gcache_test
import (
"fmt"
gcache "github.com/bluele/gcache"
"testing"
)
func buildSimpleCache(size int) gcache.Cache {
return gcache.New(size).
Simple().
EvictedFunc(evictedFuncForSimple).
Build()
}
func buildLoadingSimpleCache(size int, loader gcache.LoaderFunc) gcache.Cache {
return gcache.New(size).
LoaderFunc(loader).
Simple().
EvictedFunc(evictedFuncForSimple).
Build()
}
func evictedFuncForSimple(key, value interface{}) {
fmt.Printf("[Simple] Key:%v Value:%v will evicted.\n", key, value)
}
func TestSimpleGet(t *testing.T) {
size := 1000
gc := buildSimpleCache(size)
testSetCache(t, gc, size)
testGetCache(t, gc, size)
}
func TestLoadingSimpleGet(t *testing.T) {
size := 1000
numbers := 1000
testGetCache(t, buildLoadingSimpleCache(size, loader), numbers)
}
func TestSimpleLength(t *testing.T) {
gc := buildLoadingSimpleCache(1000, loader)
gc.Get("test1")
gc.Get("test2")
length := gc.Len()
expectedLength := 2
if length != expectedLength {
t.Errorf("Expected length is %v, not %v", length, expectedLength)
}
}
func TestSimpleEvictItem(t *testing.T) {
cacheSize := 10
numbers := 11
gc := buildLoadingSimpleCache(cacheSize, loader)
for i := 0; i < numbers; i++ {
_, err := gc.Get(fmt.Sprintf("Key-%d", i))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
}
}
+64
View File
@@ -0,0 +1,64 @@
/*
Copyright 2012 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package singleflight provides a duplicate function call suppression
// mechanism.
package singleflight
import "sync"
// call is an in-flight or completed Do call
type call struct {
wg sync.WaitGroup
val interface{}
err error
}
// Group represents a class of work and forms a namespace in which
// units of work can be executed with duplicate suppression.
type Group struct {
mu sync.Mutex // protects m
m map[interface{}]*call // lazily initialized
}
// Do executes and returns the results of the given function, making
// sure that only one execution is in-flight for a given key at a
// time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (g *Group) Do(key interface{}, fn func() (interface{}, error)) (interface{}, error) {
g.mu.Lock()
if g.m == nil {
g.m = make(map[interface{}]*call)
}
if c, ok := g.m[key]; ok {
g.mu.Unlock()
c.wg.Wait()
return c.val, c.err
}
c := new(call)
c.wg.Add(1)
g.m[key] = c
g.mu.Unlock()
c.val, c.err = fn()
c.wg.Done()
g.mu.Lock()
delete(g.m, key)
g.mu.Unlock()
return c.val, c.err
}
+85
View File
@@ -0,0 +1,85 @@
/*
Copyright 2012 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package singleflight
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestDo(t *testing.T) {
var g Group
v, err := g.Do("key", func() (interface{}, error) {
return "bar", nil
})
if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want {
t.Errorf("Do = %v; want %v", got, want)
}
if err != nil {
t.Errorf("Do error = %v", err)
}
}
func TestDoErr(t *testing.T) {
var g Group
someErr := errors.New("Some error")
v, err := g.Do("key", func() (interface{}, error) {
return nil, someErr
})
if err != someErr {
t.Errorf("Do error = %v; want someErr", err)
}
if v != nil {
t.Errorf("unexpected non-nil value %#v", v)
}
}
func TestDoDupSuppress(t *testing.T) {
var g Group
c := make(chan string)
var calls int32
fn := func() (interface{}, error) {
atomic.AddInt32(&calls, 1)
return <-c, nil
}
const n = 10
var wg sync.WaitGroup
for i := 0; i < n; i++ {
wg.Add(1)
go func() {
v, err := g.Do("key", fn)
if err != nil {
t.Errorf("Do error: %v", err)
}
if v.(string) != "bar" {
t.Errorf("got %q; want %q", v, "bar")
}
wg.Done()
}()
}
time.Sleep(100 * time.Millisecond) // let goroutines above block
c <- "bar"
wg.Wait()
if got := atomic.LoadInt32(&calls); got != 1 {
t.Errorf("number of calls = %d; want 1", got)
}
}
+15
View File
@@ -0,0 +1,15 @@
package gcache
func minInt(x, y int) int {
if x < y {
return x
}
return y
}
func maxInt(x, y int) int {
if x > y {
return x
}
return y
}
+15
View File
@@ -0,0 +1,15 @@
box: wercker/golang
build:
steps:
# Sets the go workspace and places you package
# at the right place in the workspace tree
- setup-go-workspace
# Test the project
- script:
name: go test
code: |
cd $WERCKER_SOURCE_DIR
go version
go test
+3
View File
@@ -0,0 +1,3 @@
This Source Code Form is subject to the terms of the Mozilla Public License,
v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain
one at http://mozilla.org/MPL/2.0/.
+60
View File
@@ -0,0 +1,60 @@
# GoCertifi: SSL Certificates for Golang
This Go package contains a CA bundle that you can reference in your Go code.
This is useful for systems that do not have CA bundles that Golang can find
itself, or where a uniform set of CAs is valuable.
This is the same CA bundle that ships with the
[Python Requests](https://github.com/kennethreitz/requests) library, and is a
Golang specific port of [certifi](https://github.com/kennethreitz/certifi). The
CA bundle is derived from Mozilla's canonical set.
## Usage
You can use the `gocertifi` package as follows:
```go
import "github.com/certifi/gocertifi"
cert_pool, err := gocertifi.CACerts()
```
You can use the returned `*x509.CertPool` as part of an HTTP transport, for example:
```go
import (
"net/http"
"crypto/tls"
)
// Setup an HTTP client with a custom transport
transport := &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: cert_pool},
}
client := &http.Client{Transport: transport}
// Make an HTTP request using our custom transport
resp, err := client.Get("https://example.com")
```
## Detailed Documentation
Import as follows:
```go
import "github.com/certifi/gocertifi"
```
### Errors
```go
var ErrParseFailed = errors.New("gocertifi: error when parsing certificates")
```
### Functions
```go
func CACerts() (*x509.CertPool, error)
```
CACerts builds an X.509 certificate pool containing the Mozilla CA Certificate
bundle. Returns nil on error along with an appropriate error code.
+5251
View File
File diff suppressed because it is too large Load Diff
+10
View File
@@ -0,0 +1,10 @@
package gocertifi
import "testing"
func TestGetCerts(t *testing.T) {
cert_pool, err := CACerts()
if (cert_pool == nil) || (err != nil) {
t.Errorf("Failed to return the certificates.")
}
}
+20
View File
@@ -0,0 +1,20 @@
from invoke import task, run
import requests
@task
def update():
r = requests.get('https://mkcert.org/generate/')
r.raise_for_status()
certs = r.content
with open('certifi.go', 'rb') as f:
file = f.read()
file = file.split('`\n')
assert len(file) == 3
file[1] = certs
run("rm certifi.go")
with open('certifi.go', 'wb') as f:
f.write('`\n'.join(file))
@@ -0,0 +1,13 @@
include $(GOROOT)/src/Make.inc
TARG=bitbucket.org/ww/goautoneg
GOFILES=autoneg.go
include $(GOROOT)/src/Make.pkg
format:
gofmt -w *.go
docs:
gomake clean
godoc ${TARG} > README.txt
@@ -0,0 +1,67 @@
PACKAGE
package goautoneg
import "bitbucket.org/ww/goautoneg"
HTTP Content-Type Autonegotiation.
The functions in this package implement the behaviour specified in
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
Copyright (c) 2011, Open Knowledge Foundation Ltd.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
Neither the name of the Open Knowledge Foundation Ltd. nor the
names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
FUNCTIONS
func Negotiate(header string, alternatives []string) (content_type string)
Negotiate the most appropriate content_type given the accept header
and a list of alternatives.
func ParseAccept(header string) (accept []Accept)
Parse an Accept Header string returning a sorted list
of clauses
TYPES
type Accept struct {
Type, SubType string
Q float32
Params map[string]string
}
Structure to represent a clause in an HTTP Accept Header
SUBDIRECTORIES
.hg
@@ -0,0 +1,162 @@
/*
HTTP Content-Type Autonegotiation.
The functions in this package implement the behaviour specified in
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
Copyright (c) 2011, Open Knowledge Foundation Ltd.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
Neither the name of the Open Knowledge Foundation Ltd. nor the
names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package goautoneg
import (
"sort"
"strconv"
"strings"
)
// Structure to represent a clause in an HTTP Accept Header
type Accept struct {
Type, SubType string
Q float64
Params map[string]string
}
// For internal use, so that we can use the sort interface
type accept_slice []Accept
func (accept accept_slice) Len() int {
slice := []Accept(accept)
return len(slice)
}
func (accept accept_slice) Less(i, j int) bool {
slice := []Accept(accept)
ai, aj := slice[i], slice[j]
if ai.Q > aj.Q {
return true
}
if ai.Type != "*" && aj.Type == "*" {
return true
}
if ai.SubType != "*" && aj.SubType == "*" {
return true
}
return false
}
func (accept accept_slice) Swap(i, j int) {
slice := []Accept(accept)
slice[i], slice[j] = slice[j], slice[i]
}
// Parse an Accept Header string returning a sorted list
// of clauses
func ParseAccept(header string) (accept []Accept) {
parts := strings.Split(header, ",")
accept = make([]Accept, 0, len(parts))
for _, part := range parts {
part := strings.Trim(part, " ")
a := Accept{}
a.Params = make(map[string]string)
a.Q = 1.0
mrp := strings.Split(part, ";")
media_range := mrp[0]
sp := strings.Split(media_range, "/")
a.Type = strings.Trim(sp[0], " ")
switch {
case len(sp) == 1 && a.Type == "*":
a.SubType = "*"
case len(sp) == 2:
a.SubType = strings.Trim(sp[1], " ")
default:
continue
}
if len(mrp) == 1 {
accept = append(accept, a)
continue
}
for _, param := range mrp[1:] {
sp := strings.SplitN(param, "=", 2)
if len(sp) != 2 {
continue
}
token := strings.Trim(sp[0], " ")
if token == "q" {
a.Q, _ = strconv.ParseFloat(sp[1], 32)
} else {
a.Params[token] = strings.Trim(sp[1], " ")
}
}
accept = append(accept, a)
}
slice := accept_slice(accept)
sort.Sort(slice)
return
}
// Negotiate the most appropriate content_type given the accept header
// and a list of alternatives.
func Negotiate(header string, alternatives []string) (content_type string) {
asp := make([][]string, 0, len(alternatives))
for _, ctype := range alternatives {
asp = append(asp, strings.SplitN(ctype, "/", 2))
}
for _, clause := range ParseAccept(header) {
for i, ctsp := range asp {
if clause.Type == ctsp[0] && clause.SubType == ctsp[1] {
content_type = alternatives[i]
return
}
if clause.Type == ctsp[0] && clause.SubType == "*" {
content_type = alternatives[i]
return
}
if clause.Type == "*" && clause.SubType == "*" {
content_type = alternatives[i]
return
}
}
}
return
}
@@ -0,0 +1,33 @@
package goautoneg
import (
"testing"
)
var chrome = "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"
func TestParseAccept(t *testing.T) {
alternatives := []string{"text/html", "image/png"}
content_type := Negotiate(chrome, alternatives)
if content_type != "image/png" {
t.Errorf("got %s expected image/png", content_type)
}
alternatives = []string{"text/html", "text/plain", "text/n3"}
content_type = Negotiate(chrome, alternatives)
if content_type != "text/html" {
t.Errorf("got %s expected text/html", content_type)
}
alternatives = []string{"text/n3", "text/plain"}
content_type = Negotiate(chrome, alternatives)
if content_type != "text/plain" {
t.Errorf("got %s expected text/plain", content_type)
}
alternatives = []string{"text/n3", "application/rdf+xml"}
content_type = Negotiate(chrome, alternatives)
if content_type != "text/n3" {
t.Errorf("got %s expected text/n3", content_type)
}
}
@@ -0,0 +1,63 @@
package quantile
import (
"testing"
)
func BenchmarkInsertTargeted(b *testing.B) {
b.ReportAllocs()
s := NewTargeted(Targets)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}
func BenchmarkInsertTargetedSmallEpsilon(b *testing.B) {
s := NewTargeted(TargetsSmallEpsilon)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}
func BenchmarkInsertBiased(b *testing.B) {
s := NewLowBiased(0.01)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}
func BenchmarkInsertBiasedSmallEpsilon(b *testing.B) {
s := NewLowBiased(0.0001)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}
func BenchmarkQuery(b *testing.B) {
s := NewTargeted(Targets)
for i := float64(0); i < 1e6; i++ {
s.Insert(i)
}
b.ResetTimer()
n := float64(b.N)
for i := float64(0); i < n; i++ {
s.Query(i / n)
}
}
func BenchmarkQuerySmallEpsilon(b *testing.B) {
s := NewTargeted(TargetsSmallEpsilon)
for i := float64(0); i < 1e6; i++ {
s.Insert(i)
}
b.ResetTimer()
n := float64(b.N)
for i := float64(0); i < n; i++ {
s.Query(i / n)
}
}
@@ -0,0 +1,121 @@
// +build go1.1
package quantile_test
import (
"bufio"
"fmt"
"log"
"os"
"strconv"
"time"
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/beorn7/perks/quantile"
)
func Example_simple() {
ch := make(chan float64)
go sendFloats(ch)
// Compute the 50th, 90th, and 99th percentile.
q := quantile.NewTargeted(map[float64]float64{
0.50: 0.005,
0.90: 0.001,
0.99: 0.0001,
})
for v := range ch {
q.Insert(v)
}
fmt.Println("perc50:", q.Query(0.50))
fmt.Println("perc90:", q.Query(0.90))
fmt.Println("perc99:", q.Query(0.99))
fmt.Println("count:", q.Count())
// Output:
// perc50: 5
// perc90: 16
// perc99: 223
// count: 2388
}
func Example_mergeMultipleStreams() {
// Scenario:
// We have multiple database shards. On each shard, there is a process
// collecting query response times from the database logs and inserting
// them into a Stream (created via NewTargeted(0.90)), much like the
// Simple example. These processes expose a network interface for us to
// ask them to serialize and send us the results of their
// Stream.Samples so we may Merge and Query them.
//
// NOTES:
// * These sample sets are small, allowing us to get them
// across the network much faster than sending the entire list of data
// points.
//
// * For this to work correctly, we must supply the same quantiles
// a priori the process collecting the samples supplied to NewTargeted,
// even if we do not plan to query them all here.
ch := make(chan quantile.Samples)
getDBQuerySamples(ch)
q := quantile.NewTargeted(map[float64]float64{0.90: 0.001})
for samples := range ch {
q.Merge(samples)
}
fmt.Println("perc90:", q.Query(0.90))
}
func Example_window() {
// Scenario: We want the 90th, 95th, and 99th percentiles for each
// minute.
ch := make(chan float64)
go sendStreamValues(ch)
tick := time.NewTicker(1 * time.Minute)
q := quantile.NewTargeted(map[float64]float64{
0.90: 0.001,
0.95: 0.0005,
0.99: 0.0001,
})
for {
select {
case t := <-tick.C:
flushToDB(t, q.Samples())
q.Reset()
case v := <-ch:
q.Insert(v)
}
}
}
func sendStreamValues(ch chan float64) {
// Use your imagination
}
func flushToDB(t time.Time, samples quantile.Samples) {
// Use your imagination
}
// This is a stub for the above example. In reality this would hit the remote
// servers via http or something like it.
func getDBQuerySamples(ch chan quantile.Samples) {}
func sendFloats(ch chan<- float64) {
f, err := os.Open("exampledata.txt")
if err != nil {
log.Fatal(err)
}
sc := bufio.NewScanner(f)
for sc.Scan() {
b := sc.Bytes()
v, err := strconv.ParseFloat(string(b), 64)
if err != nil {
log.Fatal(err)
}
ch <- v
}
if sc.Err() != nil {
log.Fatal(sc.Err())
}
close(ch)
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,292 @@
// Package quantile computes approximate quantiles over an unbounded data
// stream within low memory and CPU bounds.
//
// A small amount of accuracy is traded to achieve the above properties.
//
// Multiple streams can be merged before calling Query to generate a single set
// of results. This is meaningful when the streams represent the same type of
// data. See Merge and Samples.
//
// For more detailed information about the algorithm used, see:
//
// Effective Computation of Biased Quantiles over Data Streams
//
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile
import (
"math"
"sort"
)
// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
Value float64 `json:",string"`
Width float64 `json:",string"`
Delta float64 `json:",string"`
}
// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample
func (a Samples) Len() int { return len(a) }
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
type invariant func(s *stream, r float64) float64
// NewLowBiased returns an initialized Stream for low-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the lower ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewLowBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * r
}
return newStream(ƒ)
}
// NewHighBiased returns an initialized Stream for high-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the higher ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewHighBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * (s.n - r)
}
return newStream(ƒ)
}
// NewTargeted returns an initialized Stream concerned with a particular set of
// quantile values that are supplied a priori. Knowing these a priori reduces
// space and computation time. The targets map maps the desired quantiles to
// their absolute errors, i.e. the true quantile of a value returned by a query
// is guaranteed to be within (Quantile±Epsilon).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewTargeted(targets map[float64]float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
var m = math.MaxFloat64
var f float64
for quantile, epsilon := range targets {
if quantile*s.n <= r {
f = (2 * epsilon * r) / quantile
} else {
f = (2 * epsilon * (s.n - r)) / (1 - quantile)
}
if f < m {
m = f
}
}
return m
}
return newStream(ƒ)
}
// Stream computes quantiles for a stream of float64s. It is not thread-safe by
// design. Take care when using across multiple goroutines.
type Stream struct {
*stream
b Samples
sorted bool
}
func newStream(ƒ invariant) *Stream {
x := &stream{ƒ: ƒ}
return &Stream{x, make(Samples, 0, 500), true}
}
// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
s.insert(Sample{Value: v, Width: 1})
}
func (s *Stream) insert(sample Sample) {
s.b = append(s.b, sample)
s.sorted = false
if len(s.b) == cap(s.b) {
s.flush()
}
}
// Query returns the computed qth percentiles value. If s was created with
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
// will return an unspecified result.
func (s *Stream) Query(q float64) float64 {
if !s.flushed() {
// Fast path when there hasn't been enough data for a flush;
// this also yields better accuracy for small sets of data.
l := len(s.b)
if l == 0 {
return 0
}
i := int(float64(l) * q)
if i > 0 {
i -= 1
}
s.maybeSort()
return s.b[i].Value
}
s.flush()
return s.stream.query(q)
}
// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads, database shards, etc.
//
// ATTENTION: This method is broken and does not yield correct results. The
// underlying algorithm is not capable of merging streams correctly.
func (s *Stream) Merge(samples Samples) {
sort.Sort(samples)
s.stream.merge(samples)
}
// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
s.stream.reset()
s.b = s.b[:0]
}
// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
if !s.flushed() {
return s.b
}
s.flush()
return s.stream.samples()
}
// Count returns the total number of samples observed in the stream
// since initialization.
func (s *Stream) Count() int {
return len(s.b) + s.stream.count()
}
func (s *Stream) flush() {
s.maybeSort()
s.stream.merge(s.b)
s.b = s.b[:0]
}
func (s *Stream) maybeSort() {
if !s.sorted {
s.sorted = true
sort.Sort(s.b)
}
}
func (s *Stream) flushed() bool {
return len(s.stream.l) > 0
}
type stream struct {
n float64
l []Sample
ƒ invariant
}
func (s *stream) reset() {
s.l = s.l[:0]
s.n = 0
}
func (s *stream) insert(v float64) {
s.merge(Samples{{v, 1, 0}})
}
func (s *stream) merge(samples Samples) {
// TODO(beorn7): This tries to merge not only individual samples, but
// whole summaries. The paper doesn't mention merging summaries at
// all. Unittests show that the merging is inaccurate. Find out how to
// do merges properly.
var r float64
i := 0
for _, sample := range samples {
for ; i < len(s.l); i++ {
c := s.l[i]
if c.Value > sample.Value {
// Insert at position i.
s.l = append(s.l, Sample{})
copy(s.l[i+1:], s.l[i:])
s.l[i] = Sample{
sample.Value,
sample.Width,
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
// TODO(beorn7): How to calculate delta correctly?
}
i++
goto inserted
}
r += c.Width
}
s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
i++
inserted:
s.n += sample.Width
r += sample.Width
}
s.compress()
}
func (s *stream) count() int {
return int(s.n)
}
func (s *stream) query(q float64) float64 {
t := math.Ceil(q * s.n)
t += math.Ceil(s.ƒ(s, t) / 2)
p := s.l[0]
var r float64
for _, c := range s.l[1:] {
r += p.Width
if r+c.Width+c.Delta > t {
return p.Value
}
p = c
}
return p.Value
}
func (s *stream) compress() {
if len(s.l) < 2 {
return
}
x := s.l[len(s.l)-1]
xi := len(s.l) - 1
r := s.n - 1 - x.Width
for i := len(s.l) - 2; i >= 0; i-- {
c := s.l[i]
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
x.Width += c.Width
s.l[xi] = x
// Remove element at i.
copy(s.l[i:], s.l[i+1:])
s.l = s.l[:len(s.l)-1]
xi -= 1
} else {
x = c
xi = i
}
r -= c.Width
}
}
func (s *stream) samples() Samples {
samples := make(Samples, len(s.l))
copy(samples, s.l)
return samples
}
@@ -0,0 +1,188 @@
package quantile
import (
"math"
"math/rand"
"sort"
"testing"
)
var (
Targets = map[float64]float64{
0.01: 0.001,
0.10: 0.01,
0.50: 0.05,
0.90: 0.01,
0.99: 0.001,
}
TargetsSmallEpsilon = map[float64]float64{
0.01: 0.0001,
0.10: 0.001,
0.50: 0.005,
0.90: 0.001,
0.99: 0.0001,
}
LowQuantiles = []float64{0.01, 0.1, 0.5}
HighQuantiles = []float64{0.99, 0.9, 0.5}
)
const RelativeEpsilon = 0.01
func verifyPercsWithAbsoluteEpsilon(t *testing.T, a []float64, s *Stream) {
sort.Float64s(a)
for quantile, epsilon := range Targets {
n := float64(len(a))
k := int(quantile * n)
lower := int((quantile - epsilon) * n)
if lower < 1 {
lower = 1
}
upper := int(math.Ceil((quantile + epsilon) * n))
if upper > len(a) {
upper = len(a)
}
w, min, max := a[k-1], a[lower-1], a[upper-1]
if g := s.Query(quantile); g < min || g > max {
t.Errorf("q=%f: want %v [%f,%f], got %v", quantile, w, min, max, g)
}
}
}
func verifyLowPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
sort.Float64s(a)
for _, qu := range LowQuantiles {
n := float64(len(a))
k := int(qu * n)
lowerRank := int((1 - RelativeEpsilon) * qu * n)
upperRank := int(math.Ceil((1 + RelativeEpsilon) * qu * n))
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
if g := s.Query(qu); g < min || g > max {
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
}
}
}
func verifyHighPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
sort.Float64s(a)
for _, qu := range HighQuantiles {
n := float64(len(a))
k := int(qu * n)
lowerRank := int((1 - (1+RelativeEpsilon)*(1-qu)) * n)
upperRank := int(math.Ceil((1 - (1-RelativeEpsilon)*(1-qu)) * n))
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
if g := s.Query(qu); g < min || g > max {
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
}
}
}
func populateStream(s *Stream) []float64 {
a := make([]float64, 0, 1e5+100)
for i := 0; i < cap(a); i++ {
v := rand.NormFloat64()
// Add 5% asymmetric outliers.
if i%20 == 0 {
v = v*v + 1
}
s.Insert(v)
a = append(a, v)
}
return a
}
func TestTargetedQuery(t *testing.T) {
rand.Seed(42)
s := NewTargeted(Targets)
a := populateStream(s)
verifyPercsWithAbsoluteEpsilon(t, a, s)
}
func TestLowBiasedQuery(t *testing.T) {
rand.Seed(42)
s := NewLowBiased(RelativeEpsilon)
a := populateStream(s)
verifyLowPercsWithRelativeEpsilon(t, a, s)
}
func TestHighBiasedQuery(t *testing.T) {
rand.Seed(42)
s := NewHighBiased(RelativeEpsilon)
a := populateStream(s)
verifyHighPercsWithRelativeEpsilon(t, a, s)
}
// BrokenTestTargetedMerge is broken, see Merge doc comment.
func BrokenTestTargetedMerge(t *testing.T) {
rand.Seed(42)
s1 := NewTargeted(Targets)
s2 := NewTargeted(Targets)
a := populateStream(s1)
a = append(a, populateStream(s2)...)
s1.Merge(s2.Samples())
verifyPercsWithAbsoluteEpsilon(t, a, s1)
}
// BrokenTestLowBiasedMerge is broken, see Merge doc comment.
func BrokenTestLowBiasedMerge(t *testing.T) {
rand.Seed(42)
s1 := NewLowBiased(RelativeEpsilon)
s2 := NewLowBiased(RelativeEpsilon)
a := populateStream(s1)
a = append(a, populateStream(s2)...)
s1.Merge(s2.Samples())
verifyLowPercsWithRelativeEpsilon(t, a, s2)
}
// BrokenTestHighBiasedMerge is broken, see Merge doc comment.
func BrokenTestHighBiasedMerge(t *testing.T) {
rand.Seed(42)
s1 := NewHighBiased(RelativeEpsilon)
s2 := NewHighBiased(RelativeEpsilon)
a := populateStream(s1)
a = append(a, populateStream(s2)...)
s1.Merge(s2.Samples())
verifyHighPercsWithRelativeEpsilon(t, a, s2)
}
func TestUncompressed(t *testing.T) {
q := NewTargeted(Targets)
for i := 100; i > 0; i-- {
q.Insert(float64(i))
}
if g := q.Count(); g != 100 {
t.Errorf("want count 100, got %d", g)
}
// Before compression, Query should have 100% accuracy.
for quantile := range Targets {
w := quantile * 100
if g := q.Query(quantile); g != w {
t.Errorf("want %f, got %f", w, g)
}
}
}
func TestUncompressedSamples(t *testing.T) {
q := NewTargeted(map[float64]float64{0.99: 0.001})
for i := 1; i <= 100; i++ {
q.Insert(float64(i))
}
if g := q.Samples().Len(); g != 100 {
t.Errorf("want count 100, got %d", g)
}
}
func TestUncompressedOne(t *testing.T) {
q := NewTargeted(map[float64]float64{0.99: 0.01})
q.Insert(3.14)
if g := q.Query(0.90); g != 3.14 {
t.Error("want PI, got", g)
}
}
func TestDefaults(t *testing.T) {
if g := NewTargeted(map[float64]float64{0.99: 0.001}).Query(0.99); g != 0 {
t.Errorf("want 0, got %f", g)
}
}
@@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2013 Ben Johnson
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,54 @@
TEST=.
BENCH=.
COVERPROFILE=/tmp/c.out
BRANCH=`git rev-parse --abbrev-ref HEAD`
COMMIT=`git rev-parse --short HEAD`
GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
default: build
bench:
go test -v -test.run=NOTHINCONTAINSTHIS -test.bench=$(BENCH)
# http://cloc.sourceforge.net/
cloc:
@cloc --not-match-f='Makefile|_test.go' .
cover: fmt
go test -coverprofile=$(COVERPROFILE) -test.run=$(TEST) $(COVERFLAG) .
go tool cover -html=$(COVERPROFILE)
rm $(COVERPROFILE)
cpuprofile: fmt
@go test -c
@./bolt.test -test.v -test.run=$(TEST) -test.cpuprofile cpu.prof
# go get github.com/kisielk/errcheck
errcheck:
@echo "=== errcheck ==="
@errcheck github.com/boltdb/bolt
fmt:
@go fmt ./...
get:
@go get -d ./...
build: get
@mkdir -p bin
@go build -ldflags=$(GOLDFLAGS) -a -o bin/bolt ./cmd/bolt
test: fmt
@go get github.com/stretchr/testify/assert
@echo "=== TESTS ==="
@go test -v -cover -test.run=$(TEST)
@echo ""
@echo ""
@echo "=== CLI ==="
@go test -v -test.run=$(TEST) ./cmd/bolt
@echo ""
@echo ""
@echo "=== RACE DETECTOR ==="
@go test -v -race -test.run="TestSimulate_(100op|1000op)"
.PHONY: bench cloc cover cpuprofile fmt memprofile test
@@ -0,0 +1,621 @@
Bolt [![Build Status](https://drone.io/github.com/boltdb/bolt/status.png)](https://drone.io/github.com/boltdb/bolt/latest) [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.png?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.png)](https://godoc.org/github.com/boltdb/bolt) ![Version](http://img.shields.io/badge/version-1.0-green.png)
====
Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] and
the [LMDB project][lmdb]. The goal of the project is to provide a simple,
fast, and reliable database for projects that don't require a full database
server such as Postgres or MySQL.
Since Bolt is meant to be used as such a low-level piece of functionality,
simplicity is key. The API will be small and only focus on getting values
and setting values. That's it.
[hyc_symas]: https://twitter.com/hyc_symas
[lmdb]: http://symas.com/mdb/
## Project Status
Bolt is stable and the API is fixed. Full unit test coverage and randomized
black box testing are used to ensure database consistency and thread safety.
Bolt is currently in high-load production environments serving databases as
large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed
services every day.
## Getting Started
### Installing
To start using Bolt, install Go and run `go get`:
```sh
$ go get github.com/boltdb/bolt/...
```
This will retrieve the library and install the `bolt` command line utility into
your `$GOBIN` path.
### Opening a database
The top-level object in Bolt is a `DB`. It is represented as a single file on
your disk and represents a consistent snapshot of your data.
To open your database, simply use the `bolt.Open()` function:
```go
package main
import (
"log"
"github.com/boltdb/bolt"
)
func main() {
// Open the my.db data file in your current directory.
// It will be created if it doesn't exist.
db, err := bolt.Open("my.db", 0600, nil)
if err != nil {
log.Fatal(err)
}
defer db.Close()
...
}
```
Please note that Bolt obtains a file lock on the data file so multiple processes
cannot open the same database at the same time. Opening an already open Bolt
database will cause it to hang until the other process closes it. To prevent
an indefinite wait you can pass a timeout option to the `Open()` function:
```go
db, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second})
```
### Transactions
Bolt allows only one read-write transaction at a time but allows as many
read-only transactions as you want at a time. Each transaction has a consistent
view of the data as it existed when the transaction started.
Individual transactions and all objects created from them (e.g. buckets, keys)
are not thread safe. To work with data in multiple goroutines you must start
a transaction for each one or use locking to ensure only one goroutine accesses
a transaction at a time. Creating transaction from the `DB` is thread safe.
Read-only transactions and read-write transactions should not depend on one
another and generally shouldn't be opened simultaneously in the same goroutine.
This can cause a deadlock as the read-write transaction needs to periodically
re-map the data file but it cannot do so while a read-only transaction is open.
#### Read-write transactions
To start a read-write transaction, you can use the `DB.Update()` function:
```go
err := db.Update(func(tx *bolt.Tx) error {
...
return nil
})
```
Inside the closure, you have a consistent view of the database. You commit the
transaction by returning `nil` at the end. You can also rollback the transaction
at any point by returning an error. All database operations are allowed inside
a read-write transaction.
Always check the return error as it will report any disk failures that can cause
your transaction to not complete. If you return an error within your closure
it will be passed through.
#### Read-only transactions
To start a read-only transaction, you can use the `DB.View()` function:
```go
err := db.View(func(tx *bolt.Tx) error {
...
return nil
})
```
You also get a consistent view of the database within this closure, however,
no mutating operations are allowed within a read-only transaction. You can only
retrieve buckets, retrieve values, and copy the database within a read-only
transaction.
#### Batch read-write transactions
Each `DB.Update()` waits for disk to commit the writes. This overhead
can be minimized by combining multiple updates with the `DB.Batch()`
function:
```go
err := db.Batch(func(tx *bolt.Tx) error {
...
return nil
})
```
Concurrent Batch calls are opportunistically combined into larger
transactions. Batch is only useful when there are multiple goroutines
calling it.
The trade-off is that `Batch` can call the given
function multiple times, if parts of the transaction fail. The
function must be idempotent and side effects must take effect only
after a successful return from `DB.Batch()`.
For example: don't display messages from inside the function, instead
set variables in the enclosing scope:
```go
var id uint64
err := db.Batch(func(tx *bolt.Tx) error {
// Find last key in bucket, decode as bigendian uint64, increment
// by one, encode back to []byte, and add new key.
...
id = newValue
return nil
})
if err != nil {
return ...
}
fmt.Println("Allocated ID %d", id)
```
#### Managing transactions manually
The `DB.View()` and `DB.Update()` functions are wrappers around the `DB.Begin()`
function. These helper functions will start the transaction, execute a function,
and then safely close your transaction if an error is returned. This is the
recommended way to use Bolt transactions.
However, sometimes you may want to manually start and end your transactions.
You can use the `Tx.Begin()` function directly but _please_ be sure to close the
transaction.
```go
// Start a writable transaction.
tx, err := db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
// Use the transaction...
_, err := tx.CreateBucket([]byte("MyBucket"))
if err != nil {
return err
}
// Commit the transaction and check for error.
if err := tx.Commit(); err != nil {
return err
}
```
The first argument to `DB.Begin()` is a boolean stating if the transaction
should be writable.
### Using buckets
Buckets are collections of key/value pairs within the database. All keys in a
bucket must be unique. You can create a bucket using the `DB.CreateBucket()`
function:
```go
db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucket([]byte("MyBucket"))
if err != nil {
return fmt.Errorf("create bucket: %s", err)
}
return nil
})
```
You can also create a bucket only if it doesn't exist by using the
`Tx.CreateBucketIfNotExists()` function. It's a common pattern to call this
function for all your top-level buckets after you open your database so you can
guarantee that they exist for future transactions.
To delete a bucket, simply call the `Tx.DeleteBucket()` function.
### Using key/value pairs
To save a key/value pair to a bucket, use the `Bucket.Put()` function:
```go
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("MyBucket"))
err := b.Put([]byte("answer"), []byte("42"))
return err
})
```
This will set the value of the `"answer"` key to `"42"` in the `MyBucket`
bucket. To retrieve this value, we can use the `Bucket.Get()` function:
```go
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("MyBucket"))
v := b.Get([]byte("answer"))
fmt.Printf("The answer is: %s\n", v)
return nil
})
```
The `Get()` function does not return an error because its operation is
guarenteed to work (unless there is some kind of system failure). If the key
exists then it will return its byte slice value. If it doesn't exist then it
will return `nil`. It's important to note that you can have a zero-length value
set to a key which is different than the key not existing.
Use the `Bucket.Delete()` function to delete a key from the bucket.
Please note that values returned from `Get()` are only valid while the
transaction is open. If you need to use a value outside of the transaction
then you must use `copy()` to copy it to another byte slice.
### Iterating over keys
Bolt stores its keys in byte-sorted order within a bucket. This makes sequential
iteration over these keys extremely fast. To iterate over keys we'll use a
`Cursor`:
```go
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("MyBucket"))
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
fmt.Printf("key=%s, value=%s\n", k, v)
}
return nil
})
```
The cursor allows you to move to a specific point in the list of keys and move
forward or backward through the keys one at a time.
The following functions are available on the cursor:
```
First() Move to the first key.
Last() Move to the last key.
Seek() Move to a specific key.
Next() Move to the next key.
Prev() Move to the previous key.
```
When you have iterated to the end of the cursor then `Next()` will return `nil`.
You must seek to a position using `First()`, `Last()`, or `Seek()` before
calling `Next()` or `Prev()`. If you do not seek to a position then these
functions will return `nil`.
#### Prefix scans
To iterate over a key prefix, you can combine `Seek()` and `bytes.HasPrefix()`:
```go
db.View(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte("MyBucket")).Cursor()
prefix := []byte("1234")
for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
fmt.Printf("key=%s, value=%s\n", k, v)
}
return nil
})
```
#### Range scans
Another common use case is scanning over a range such as a time range. If you
use a sortable time encoding such as RFC3339 then you can query a specific
date range like this:
```go
db.View(func(tx *bolt.Tx) error {
// Assume our events bucket has RFC3339 encoded time keys.
c := tx.Bucket([]byte("Events")).Cursor()
// Our time range spans the 90's decade.
min := []byte("1990-01-01T00:00:00Z")
max := []byte("2000-01-01T00:00:00Z")
// Iterate over the 90's.
for k, v := c.Seek(min); k != nil && bytes.Compare(k, max) <= 0; k, v = c.Next() {
fmt.Printf("%s: %s\n", k, v)
}
return nil
})
```
#### ForEach()
You can also use the function `ForEach()` if you know you'll be iterating over
all the keys in a bucket:
```go
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("MyBucket"))
b.ForEach(func(k, v []byte) error {
fmt.Printf("key=%s, value=%s\n", k, v)
return nil
})
return nil
})
```
### Nested buckets
You can also store a bucket in a key to create nested buckets. The API is the
same as the bucket management API on the `DB` object:
```go
func (*Bucket) CreateBucket(key []byte) (*Bucket, error)
func (*Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error)
func (*Bucket) DeleteBucket(key []byte) error
```
### Database backups
Bolt is a single file so it's easy to backup. You can use the `Tx.WriteTo()`
function to write a consistent view of the database to a writer. If you call
this from a read-only transaction, it will perform a hot backup and not block
your other database reads and writes. It will also use `O_DIRECT` when available
to prevent page cache trashing.
One common use case is to backup over HTTP so you can use tools like `cURL` to
do database backups:
```go
func BackupHandleFunc(w http.ResponseWriter, req *http.Request) {
err := db.View(func(tx *bolt.Tx) error {
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Disposition", `attachment; filename="my.db"`)
w.Header().Set("Content-Length", strconv.Itoa(int(tx.Size())))
_, err := tx.WriteTo(w)
return err
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
```
Then you can backup using this command:
```sh
$ curl http://localhost/backup > my.db
```
Or you can open your browser to `http://localhost/backup` and it will download
automatically.
If you want to backup to another file you can use the `Tx.CopyFile()` helper
function.
### Statistics
The database keeps a running count of many of the internal operations it
performs so you can better understand what's going on. By grabbing a snapshot
of these stats at two points in time we can see what operations were performed
in that time range.
For example, we could start a goroutine to log stats every 10 seconds:
```go
go func() {
// Grab the initial stats.
prev := db.Stats()
for {
// Wait for 10s.
time.Sleep(10 * time.Second)
// Grab the current stats and diff them.
stats := db.Stats()
diff := stats.Sub(&prev)
// Encode stats to JSON and print to STDERR.
json.NewEncoder(os.Stderr).Encode(diff)
// Save stats for the next loop.
prev = stats
}
}()
```
It's also useful to pipe these stats to a service such as statsd for monitoring
or to provide an HTTP endpoint that will perform a fixed-length sample.
### Read-Only Mode
Sometimes it is useful to create a shared, read-only Bolt database. To this,
set the `Options.ReadOnly` flag when opening your database. Read-only mode
uses a shared lock to allow multiple processes to read from the database but
it will block any processes from opening the database in read-write mode.
```go
db, err := bolt.Open("my.db", 0666, &bolt.Options{ReadOnly: true})
if err != nil {
log.Fatal(err)
}
```
## Resources
For more information on getting started with Bolt, check out the following articles:
* [Intro to BoltDB: Painless Performant Persistence](http://npf.io/2014/07/intro-to-boltdb-painless-performant-persistence/) by [Nate Finch](https://github.com/natefinch).
* [Bolt -- an embedded key/value database for Go](https://www.progville.com/go/bolt-embedded-db-golang/) by Progville
## Comparison with other databases
### Postgres, MySQL, & other relational databases
Relational databases structure data into rows and are only accessible through
the use of SQL. This approach provides flexibility in how you store and query
your data but also incurs overhead in parsing and planning SQL statements. Bolt
accesses all data by a byte slice key. This makes Bolt fast to read and write
data by key but provides no built-in support for joining values together.
Most relational databases (with the exception of SQLite) are standalone servers
that run separately from your application. This gives your systems
flexibility to connect multiple application servers to a single database
server but also adds overhead in serializing and transporting data over the
network. Bolt runs as a library included in your application so all data access
has to go through your application's process. This brings data closer to your
application but limits multi-process access to the data.
### LevelDB, RocksDB
LevelDB and its derivatives (RocksDB, HyperLevelDB) are similar to Bolt in that
they are libraries bundled into the application, however, their underlying
structure is a log-structured merge-tree (LSM tree). An LSM tree optimizes
random writes by using a write ahead log and multi-tiered, sorted files called
SSTables. Bolt uses a B+tree internally and only a single file. Both approaches
have trade offs.
If you require a high random write throughput (>10,000 w/sec) or you need to use
spinning disks then LevelDB could be a good choice. If your application is
read-heavy or does a lot of range scans then Bolt could be a good choice.
One other important consideration is that LevelDB does not have transactions.
It supports batch writing of key/values pairs and it supports read snapshots
but it will not give you the ability to do a compare-and-swap operation safely.
Bolt supports fully serializable ACID transactions.
### LMDB
Bolt was originally a port of LMDB so it is architecturally similar. Both use
a B+tree, have ACID semantics with fully serializable transactions, and support
lock-free MVCC using a single writer and multiple readers.
The two projects have somewhat diverged. LMDB heavily focuses on raw performance
while Bolt has focused on simplicity and ease of use. For example, LMDB allows
several unsafe actions such as direct writes for the sake of performance. Bolt
opts to disallow actions which can leave the database in a corrupted state. The
only exception to this in Bolt is `DB.NoSync`.
There are also a few differences in API. LMDB requires a maximum mmap size when
opening an `mdb_env` whereas Bolt will handle incremental mmap resizing
automatically. LMDB overloads the getter and setter functions with multiple
flags whereas Bolt splits these specialized cases into their own functions.
## Caveats & Limitations
It's important to pick the right tool for the job and Bolt is no exception.
Here are a few things to note when evaluating and using Bolt:
* Bolt is good for read intensive workloads. Sequential write performance is
also fast but random writes can be slow. You can add a write-ahead log or
[transaction coalescer](https://github.com/boltdb/coalescer) in front of Bolt
to mitigate this issue.
* Bolt uses a B+tree internally so there can be a lot of random page access.
SSDs provide a significant performance boost over spinning disks.
* Try to avoid long running read transactions. Bolt uses copy-on-write so
old pages cannot be reclaimed while an old transaction is using them.
* Byte slices returned from Bolt are only valid during a transaction. Once the
transaction has been committed or rolled back then the memory they point to
can be reused by a new page or can be unmapped from virtual memory and you'll
see an `unexpected fault address` panic when accessing it.
* Be careful when using `Bucket.FillPercent`. Setting a high fill percent for
buckets that have random inserts will cause your database to have very poor
page utilization.
* Use larger buckets in general. Smaller buckets causes poor page utilization
once they become larger than the page size (typically 4KB).
* Bulk loading a lot of random writes into a new bucket can be slow as the
page will not split until the transaction is committed. Randomly inserting
more than 100,000 key/value pairs into a single new bucket in a single
transaction is not advised.
* Bolt uses a memory-mapped file so the underlying operating system handles the
caching of the data. Typically, the OS will cache as much of the file as it
can in memory and will release memory as needed to other processes. This means
that Bolt can show very high memory usage when working with large databases.
However, this is expected and the OS will release memory as needed. Bolt can
handle databases much larger than the available physical RAM.
* The data structures in the Bolt database are memory mapped so the data file
will be endian specific. This means that you cannot copy a Bolt file from a
little endian machine to a big endian machine and have it work. For most
users this is not a concern since most modern CPUs are little endian.
* Because of the way pages are laid out on disk, Bolt cannot truncate data files
and return free pages back to the disk. Instead, Bolt maintains a free list
of unused pages within its data file. These free pages can be reused by later
transactions. This works well for many use cases as databases generally tend
to grow. However, it's important to note that deleting large chunks of data
will not allow you to reclaim that space on disk.
For more information on page allocation, [see this comment][page-allocation].
[page-allocation]: https://github.com/boltdb/bolt/issues/308#issuecomment-74811638
## Other Projects Using Bolt
Below is a list of public, open source projects that use Bolt:
* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard.
* [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside.
* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb.
* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics.
* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects.
* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday.
* [ChainStore](https://github.com/nulayer/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations.
* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite.
* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin".
* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka.
* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed.
* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt.
* [photosite/session](http://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site.
* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage.
* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters.
* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend.
* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend.
* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server.
* [SkyDB](https://github.com/skydb/sky) - Behavioral analytics database.
* [Seaweed File System](https://github.com/chrislusf/weed-fs) - Highly scalable distributed key~file system with O(1) disk read.
* [InfluxDB](http://influxdb.com) - Scalable datastore for metrics, events, and real-time analytics.
* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data.
* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system.
* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware.
* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistant, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs.
* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems.
If you are using Bolt in a project please send a pull request to add it to the list.
@@ -0,0 +1,138 @@
package bolt
import (
"errors"
"fmt"
"sync"
"time"
)
// Batch calls fn as part of a batch. It behaves similar to Update,
// except:
//
// 1. concurrent Batch calls can be combined into a single Bolt
// transaction.
//
// 2. the function passed to Batch may be called multiple times,
// regardless of whether it returns error or not.
//
// This means that Batch function side effects must be idempotent and
// take permanent effect only after a successful return is seen in
// caller.
//
// The maximum batch size and delay can be adjusted with DB.MaxBatchSize
// and DB.MaxBatchDelay, respectively.
//
// Batch is only useful when there are multiple goroutines calling it.
func (db *DB) Batch(fn func(*Tx) error) error {
errCh := make(chan error, 1)
db.batchMu.Lock()
if (db.batch == nil) || (db.batch != nil && len(db.batch.calls) >= db.MaxBatchSize) {
// There is no existing batch, or the existing batch is full; start a new one.
db.batch = &batch{
db: db,
}
db.batch.timer = time.AfterFunc(db.MaxBatchDelay, db.batch.trigger)
}
db.batch.calls = append(db.batch.calls, call{fn: fn, err: errCh})
if len(db.batch.calls) >= db.MaxBatchSize {
// wake up batch, it's ready to run
go db.batch.trigger()
}
db.batchMu.Unlock()
err := <-errCh
if err == trySolo {
err = db.Update(fn)
}
return err
}
type call struct {
fn func(*Tx) error
err chan<- error
}
type batch struct {
db *DB
timer *time.Timer
start sync.Once
calls []call
}
// trigger runs the batch if it hasn't already been run.
func (b *batch) trigger() {
b.start.Do(b.run)
}
// run performs the transactions in the batch and communicates results
// back to DB.Batch.
func (b *batch) run() {
b.db.batchMu.Lock()
b.timer.Stop()
// Make sure no new work is added to this batch, but don't break
// other batches.
if b.db.batch == b {
b.db.batch = nil
}
b.db.batchMu.Unlock()
retry:
for len(b.calls) > 0 {
var failIdx = -1
err := b.db.Update(func(tx *Tx) error {
for i, c := range b.calls {
if err := safelyCall(c.fn, tx); err != nil {
failIdx = i
return err
}
}
return nil
})
if failIdx >= 0 {
// take the failing transaction out of the batch. it's
// safe to shorten b.calls here because db.batch no longer
// points to us, and we hold the mutex anyway.
c := b.calls[failIdx]
b.calls[failIdx], b.calls = b.calls[len(b.calls)-1], b.calls[:len(b.calls)-1]
// tell the submitter re-run it solo, continue with the rest of the batch
c.err <- trySolo
continue retry
}
// pass success, or bolt internal errors, to all callers
for _, c := range b.calls {
if c.err != nil {
c.err <- err
}
}
break retry
}
}
// trySolo is a special sentinel error value used for signaling that a
// transaction function should be re-run. It should never be seen by
// callers.
var trySolo = errors.New("batch function returned an error and should be re-run solo")
type panicked struct {
reason interface{}
}
func (p panicked) Error() string {
if err, ok := p.reason.(error); ok {
return err.Error()
}
return fmt.Sprintf("panic: %v", p.reason)
}
func safelyCall(fn func(*Tx) error, tx *Tx) (err error) {
defer func() {
if p := recover(); p != nil {
err = panicked{p}
}
}()
return fn(tx)
}
@@ -0,0 +1,170 @@
package bolt_test
import (
"bytes"
"encoding/binary"
"errors"
"hash/fnv"
"sync"
"testing"
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"
)
func validateBatchBench(b *testing.B, db *TestDB) {
var rollback = errors.New("sentinel error to cause rollback")
validate := func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte("bench"))
h := fnv.New32a()
buf := make([]byte, 4)
for id := uint32(0); id < 1000; id++ {
binary.LittleEndian.PutUint32(buf, id)
h.Reset()
h.Write(buf[:])
k := h.Sum(nil)
v := bucket.Get(k)
if v == nil {
b.Errorf("not found id=%d key=%x", id, k)
continue
}
if g, e := v, []byte("filler"); !bytes.Equal(g, e) {
b.Errorf("bad value for id=%d key=%x: %s != %q", id, k, g, e)
}
if err := bucket.Delete(k); err != nil {
return err
}
}
// should be empty now
c := bucket.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
b.Errorf("unexpected key: %x = %q", k, v)
}
return rollback
}
if err := db.Update(validate); err != nil && err != rollback {
b.Error(err)
}
}
func BenchmarkDBBatchAutomatic(b *testing.B) {
db := NewTestDB()
defer db.Close()
db.MustCreateBucket([]byte("bench"))
b.ResetTimer()
for i := 0; i < b.N; i++ {
start := make(chan struct{})
var wg sync.WaitGroup
for round := 0; round < 1000; round++ {
wg.Add(1)
go func(id uint32) {
defer wg.Done()
<-start
h := fnv.New32a()
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, id)
h.Write(buf[:])
k := h.Sum(nil)
insert := func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("bench"))
return b.Put(k, []byte("filler"))
}
if err := db.Batch(insert); err != nil {
b.Error(err)
return
}
}(uint32(round))
}
close(start)
wg.Wait()
}
b.StopTimer()
validateBatchBench(b, db)
}
func BenchmarkDBBatchSingle(b *testing.B) {
db := NewTestDB()
defer db.Close()
db.MustCreateBucket([]byte("bench"))
b.ResetTimer()
for i := 0; i < b.N; i++ {
start := make(chan struct{})
var wg sync.WaitGroup
for round := 0; round < 1000; round++ {
wg.Add(1)
go func(id uint32) {
defer wg.Done()
<-start
h := fnv.New32a()
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, id)
h.Write(buf[:])
k := h.Sum(nil)
insert := func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("bench"))
return b.Put(k, []byte("filler"))
}
if err := db.Update(insert); err != nil {
b.Error(err)
return
}
}(uint32(round))
}
close(start)
wg.Wait()
}
b.StopTimer()
validateBatchBench(b, db)
}
func BenchmarkDBBatchManual10x100(b *testing.B) {
db := NewTestDB()
defer db.Close()
db.MustCreateBucket([]byte("bench"))
b.ResetTimer()
for i := 0; i < b.N; i++ {
start := make(chan struct{})
var wg sync.WaitGroup
for major := 0; major < 10; major++ {
wg.Add(1)
go func(id uint32) {
defer wg.Done()
<-start
insert100 := func(tx *bolt.Tx) error {
h := fnv.New32a()
buf := make([]byte, 4)
for minor := uint32(0); minor < 100; minor++ {
binary.LittleEndian.PutUint32(buf, uint32(id*100+minor))
h.Reset()
h.Write(buf[:])
k := h.Sum(nil)
b := tx.Bucket([]byte("bench"))
if err := b.Put(k, []byte("filler")); err != nil {
return err
}
}
return nil
}
if err := db.Update(insert100); err != nil {
b.Fatal(err)
}
}(uint32(major))
}
close(start)
wg.Wait()
}
b.StopTimer()
validateBatchBench(b, db)
}
@@ -0,0 +1,148 @@
package bolt_test
import (
"encoding/binary"
"fmt"
"io/ioutil"
"log"
"math/rand"
"net/http"
"net/http/httptest"
"os"
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"
)
// Set this to see how the counts are actually updated.
const verbose = false
// Counter updates a counter in Bolt for every URL path requested.
type counter struct {
db *bolt.DB
}
func (c counter) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
// Communicates the new count from a successful database
// transaction.
var result uint64
increment := func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("hits"))
if err != nil {
return err
}
key := []byte(req.URL.String())
// Decode handles key not found for us.
count := decode(b.Get(key)) + 1
b.Put(key, encode(count))
// All good, communicate new count.
result = count
return nil
}
if err := c.db.Batch(increment); err != nil {
http.Error(rw, err.Error(), 500)
return
}
if verbose {
log.Printf("server: %s: %d", req.URL.String(), result)
}
rw.Header().Set("Content-Type", "application/octet-stream")
fmt.Fprintf(rw, "%d\n", result)
}
func client(id int, base string, paths []string) error {
// Process paths in random order.
rng := rand.New(rand.NewSource(int64(id)))
permutation := rng.Perm(len(paths))
for i := range paths {
path := paths[permutation[i]]
resp, err := http.Get(base + path)
if err != nil {
return err
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if verbose {
log.Printf("client: %s: %s", path, buf)
}
}
return nil
}
func ExampleDB_Batch() {
// Open the database.
db, _ := bolt.Open(tempfile(), 0666, nil)
defer os.Remove(db.Path())
defer db.Close()
// Start our web server
count := counter{db}
srv := httptest.NewServer(count)
defer srv.Close()
// Decrease the batch size to make things more interesting.
db.MaxBatchSize = 3
// Get every path multiple times concurrently.
const clients = 10
paths := []string{
"/foo",
"/bar",
"/baz",
"/quux",
"/thud",
"/xyzzy",
}
errors := make(chan error, clients)
for i := 0; i < clients; i++ {
go func(id int) {
errors <- client(id, srv.URL, paths)
}(i)
}
// Check all responses to make sure there's no error.
for i := 0; i < clients; i++ {
if err := <-errors; err != nil {
fmt.Printf("client error: %v", err)
return
}
}
// Check the final result
db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("hits"))
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
fmt.Printf("hits to %s: %d\n", k, decode(v))
}
return nil
})
// Output:
// hits to /bar: 10
// hits to /baz: 10
// hits to /foo: 10
// hits to /quux: 10
// hits to /thud: 10
// hits to /xyzzy: 10
}
// encode marshals a counter.
func encode(n uint64) []byte {
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, n)
return buf
}
// decode unmarshals a counter. Nil buffers are decoded as 0.
func decode(buf []byte) uint64 {
if buf == nil {
return 0
}
return binary.BigEndian.Uint64(buf)
}
@@ -0,0 +1,167 @@
package bolt_test
import (
"testing"
"time"
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"
)
// Ensure two functions can perform updates in a single batch.
func TestDB_Batch(t *testing.T) {
db := NewTestDB()
defer db.Close()
db.MustCreateBucket([]byte("widgets"))
// Iterate over multiple updates in separate goroutines.
n := 2
ch := make(chan error)
for i := 0; i < n; i++ {
go func(i int) {
ch <- db.Batch(func(tx *bolt.Tx) error {
return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{})
})
}(i)
}
// Check all responses to make sure there's no error.
for i := 0; i < n; i++ {
if err := <-ch; err != nil {
t.Fatal(err)
}
}
// Ensure data is correct.
db.MustView(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("widgets"))
for i := 0; i < n; i++ {
if v := b.Get(u64tob(uint64(i))); v == nil {
t.Errorf("key not found: %d", i)
}
}
return nil
})
}
func TestDB_Batch_Panic(t *testing.T) {
db := NewTestDB()
defer db.Close()
var sentinel int
var bork = &sentinel
var problem interface{}
var err error
// Execute a function inside a batch that panics.
func() {
defer func() {
if p := recover(); p != nil {
problem = p
}
}()
err = db.Batch(func(tx *bolt.Tx) error {
panic(bork)
})
}()
// Verify there is no error.
if g, e := err, error(nil); g != e {
t.Fatalf("wrong error: %v != %v", g, e)
}
// Verify the panic was captured.
if g, e := problem, bork; g != e {
t.Fatalf("wrong error: %v != %v", g, e)
}
}
func TestDB_BatchFull(t *testing.T) {
db := NewTestDB()
defer db.Close()
db.MustCreateBucket([]byte("widgets"))
const size = 3
// buffered so we never leak goroutines
ch := make(chan error, size)
put := func(i int) {
ch <- db.Batch(func(tx *bolt.Tx) error {
return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{})
})
}
db.MaxBatchSize = size
// high enough to never trigger here
db.MaxBatchDelay = 1 * time.Hour
go put(1)
go put(2)
// Give the batch a chance to exhibit bugs.
time.Sleep(10 * time.Millisecond)
// not triggered yet
select {
case <-ch:
t.Fatalf("batch triggered too early")
default:
}
go put(3)
// Check all responses to make sure there's no error.
for i := 0; i < size; i++ {
if err := <-ch; err != nil {
t.Fatal(err)
}
}
// Ensure data is correct.
db.MustView(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("widgets"))
for i := 1; i <= size; i++ {
if v := b.Get(u64tob(uint64(i))); v == nil {
t.Errorf("key not found: %d", i)
}
}
return nil
})
}
func TestDB_BatchTime(t *testing.T) {
db := NewTestDB()
defer db.Close()
db.MustCreateBucket([]byte("widgets"))
const size = 1
// buffered so we never leak goroutines
ch := make(chan error, size)
put := func(i int) {
ch <- db.Batch(func(tx *bolt.Tx) error {
return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{})
})
}
db.MaxBatchSize = 1000
db.MaxBatchDelay = 0
go put(1)
// Batch must trigger by time alone.
// Check all responses to make sure there's no error.
for i := 0; i < size; i++ {
if err := <-ch; err != nil {
t.Fatal(err)
}
}
// Ensure data is correct.
db.MustView(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("widgets"))
for i := 1; i <= size; i++ {
if v := b.Get(u64tob(uint64(i))); v == nil {
t.Errorf("key not found: %d", i)
}
}
return nil
})
}
@@ -0,0 +1,7 @@
package bolt
// maxMapSize represents the largest mmap size supported by Bolt.
const maxMapSize = 0x7FFFFFFF // 2GB
// maxAllocSize is the size used when creating array pointers.
const maxAllocSize = 0xFFFFFFF
@@ -0,0 +1,7 @@
package bolt
// maxMapSize represents the largest mmap size supported by Bolt.
const maxMapSize = 0xFFFFFFFFFFFF // 256TB
// maxAllocSize is the size used when creating array pointers.
const maxAllocSize = 0x7FFFFFFF
@@ -0,0 +1,7 @@
package bolt
// maxMapSize represents the largest mmap size supported by Bolt.
const maxMapSize = 0x7FFFFFFF // 2GB
// maxAllocSize is the size used when creating array pointers.
const maxAllocSize = 0xFFFFFFF
@@ -0,0 +1,12 @@
package bolt
import (
"syscall"
)
var odirect = syscall.O_DIRECT
// fdatasync flushes written data to a file descriptor.
func fdatasync(db *DB) error {
return syscall.Fdatasync(int(db.file.Fd()))
}
@@ -0,0 +1,29 @@
package bolt
import (
"syscall"
"unsafe"
)
const (
msAsync = 1 << iota // perform asynchronous writes
msSync // perform synchronous writes
msInvalidate // invalidate cached data
)
var odirect int
func msync(db *DB) error {
_, _, errno := syscall.Syscall(syscall.SYS_MSYNC, uintptr(unsafe.Pointer(db.data)), uintptr(db.datasz), msInvalidate)
if errno != 0 {
return errno
}
return nil
}
func fdatasync(db *DB) error {
if db.data != nil {
return msync(db)
}
return db.file.Sync()
}
@@ -0,0 +1,36 @@
package bolt_test
import (
"fmt"
"path/filepath"
"reflect"
"runtime"
"testing"
)
// assert fails the test if the condition is false.
func assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
if !condition {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
tb.FailNow()
}
}
// ok fails the test if an err is not nil.
func ok(tb testing.TB, err error) {
if err != nil {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error())
tb.FailNow()
}
}
// equals fails the test if exp is not equal to act.
func equals(tb testing.TB, exp, act interface{}) {
if !reflect.DeepEqual(exp, act) {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
tb.FailNow()
}
}
@@ -0,0 +1,100 @@
// +build !windows,!plan9,!solaris
package bolt
import (
"fmt"
"os"
"syscall"
"time"
"unsafe"
)
// flock acquires an advisory lock on a file descriptor.
func flock(f *os.File, exclusive bool, timeout time.Duration) error {
var t time.Time
for {
// If we're beyond our timeout then return an error.
// This can only occur after we've attempted a flock once.
if t.IsZero() {
t = time.Now()
} else if timeout > 0 && time.Since(t) > timeout {
return ErrTimeout
}
flag := syscall.LOCK_SH
if exclusive {
flag = syscall.LOCK_EX
}
// Otherwise attempt to obtain an exclusive lock.
err := syscall.Flock(int(f.Fd()), flag|syscall.LOCK_NB)
if err == nil {
return nil
} else if err != syscall.EWOULDBLOCK {
return err
}
// Wait for a bit and try again.
time.Sleep(50 * time.Millisecond)
}
}
// funlock releases an advisory lock on a file descriptor.
func funlock(f *os.File) error {
return syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
}
// mmap memory maps a DB's data file.
func mmap(db *DB, sz int) error {
// Truncate and fsync to ensure file size metadata is flushed.
// https://github.com/boltdb/bolt/issues/284
if !db.NoGrowSync && !db.readOnly {
if err := db.file.Truncate(int64(sz)); err != nil {
return fmt.Errorf("file resize error: %s", err)
}
if err := db.file.Sync(); err != nil {
return fmt.Errorf("file sync error: %s", err)
}
}
// Map the data file to memory.
b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
return err
}
// Advise the kernel that the mmap is accessed randomly.
if err := madvise(b, syscall.MADV_RANDOM); err != nil {
return fmt.Errorf("madvise: %s", err)
}
// Save the original byte slice and convert to a byte array pointer.
db.dataref = b
db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0]))
db.datasz = sz
return nil
}
// munmap unmaps a DB's data file from memory.
func munmap(db *DB) error {
// Ignore the unmap if we have no mapped data.
if db.dataref == nil {
return nil
}
// Unmap using the original byte slice.
err := syscall.Munmap(db.dataref)
db.dataref = nil
db.data = nil
db.datasz = 0
return err
}
// NOTE: This function is copied from stdlib because it is not available on darwin.
func madvise(b []byte, advice int) (err error) {
_, _, e1 := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), uintptr(advice))
if e1 != 0 {
err = e1
}
return
}

Some files were not shown because too many files have changed in this diff Show More