Exemplu de program de administrare pentru baza de date bolt

Categorii: Programare

19-Apr-2015 17:08 - 564 vizionari

Go este foarte usor de utilizat, dovada ca exista peste 270 de aplicatii de servere de web scrise in Go, peste 700 de solutii de afisare mesaje in log scrise in Go (utile la monitorizarea evolutiei programului in timp), peste 90 de programe scrise in Go si asociate cu emulatoare (de procesor, de consola de jocuri, etc) …

Bolt este o biblioteca de functii (nu un server, nu o aplicatie separata) scrisa in Go si care ofera o solutie robusta de stocare a datelor sub forma cheie-valoare.

Pentru ca sistemul Bolt este proiectat sa fie simplu, setul de comenzi (API) este final si implementarea este incheiata.
Proiectul Bolt nu este abandonat, dar este considerat o solutie bine testata si maturizata, cu baze de date foarte mari folosite in productie de ceva timp.

Curios fiind de performantele bolt, am creat o interfata accesibila prin web de afisare, editare si populare a continutului unei bazei de date bolt.

Bolt poate lucra cu baze de date foarte mari si poate stoca intr-o cheie valori de pana la 4 tera de date.
Bolt lucreaza bine cu baze de date care depasesc memoria fizica (incarcand portiuni din baza de date, nu toata baza odata) si este limitat numai la manevarea cheilor de o anumita marime (cred ca la 1/10 din memoria RAM).
Pe un calculator cu 8 Giga de RAM am facut manevre pe o baza de date de pana la 24 de Giga (crescuta in timpul testelor) cu chei de pana la 900 mega (fisiere iso si filme) si peste 900 de mega, de cele mai multe ori, primesc eroare de memorie.

Sursa Bolt la github
Documentatia Bolt pe godoc.org
Intro to BoltDB: Painless Performant Persistence
Bolt — an embedded key/value database for Go

Pentru afisarea bucketelor (un fel de nume al colectiilor cheie-voaloare) si umplerea lor cu date, inclusiv upload din browserul de web, stergerea bucketelor si a cheilor din interiorul lor si pentru editarea continutului cheilor am creat un mic server de web scris in go.

Programul server are paginile de web (template) definite in cod pentru simplificarea utilizarii:


/*

webserver as bolt admin

*/

package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/boltdb/bolt"
	"github.com/gorilla/mux"
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"runtime/debug"
	"strconv"
	"time"
)

var (
	db              *bolt.DB
	cnt             uint64 = 0
	dbfile          string
	useSSL          bool
	tcpPORT         int
	tmplBucketsView *template.Template
	tmplBucketView  *template.Template
	tmplEditKey     *template.Template
)

type dict map[string]interface{}

func indexHandler(rw http.ResponseWriter, r *http.Request) {
	buckets := []dict{}
	title := "List of buckets"
	err := db.View(func(tx *bolt.Tx) error {
		c := tx.Cursor()
		for k, _ := c.First(); k != nil; k, _ = c.Next() {
			b := tx.Bucket(k)
			stat := b.Stats()
			if stat.KeyN < 100 {
				buckets = Extend(buckets, dict{"name": string(k),
					"size": stat.KeyN, "buckets": stat.BucketN - 1, "showLink": true})
			} else {
				buckets = Extend(buckets, dict{"name": string(k),
					"size": stat.KeyN, "buckets": stat.BucketN - 1, "showLink": false})
			}
		}
		return nil
	})
	TestError("view db", err)
	context := dict{"title": title, "buckets": buckets}
	err = tmplBucketsView.Execute(rw, context)
	TestError("index view", err)
}

func addBucket(rw http.ResponseWriter, r *http.Request) {
	bucketName := r.FormValue("bucket")
	keyName := r.FormValue("key")
	keyValue := r.FormValue("value")
	if len(bucketName) == 0 {
		fmt.Fprintln(rw, "bucket name not defined")
		return
	}
	var err error
	err = db.Batch(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
		TestError("create bucket", err)
		if len(keyName) != 0 {
			if len(keyValue) == 0 {
				//create sub bucket - not implemented in this program
				//bucket.CreateBucketIfNotExists([]byte(keyName))
			} else {
				bucket.Put([]byte(keyName), []byte(keyValue))
			}
		}
		return nil
	})
	TestError("create bucket", err)
	url := fmt.Sprintf("/view/%s", bucketName)
	http.Redirect(rw, r, url, http.StatusMovedPermanently)
}

func viewBucket(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	bucketName := vars["bucket"]
	title := fmt.Sprintf("View bucket: %s", bucketName)
	keys := []dict{}
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucketName))
		b.ForEach(func(k, v []byte) error {
			var sizeM = fmt.Sprintf("%.2f", float64(len(v))/1e6)
			if len(v) < 100 {
				keys = Extend(keys, dict{"name": string(k),
					"value": string(v), "showValue": true, "sizeM": sizeM})
			} else {
				//value is not added in dict
				keys = Extend(keys, dict{"name": string(k),
					"showValue": false, "sizeM": sizeM})
			}
			return nil
		})
		return nil
	})
	TestError("view db", err)
	context := dict{"title": title, "keys": keys, "bucket": bucketName}
	err = tmplBucketView.Execute(rw, context)
	TestError("index view", err)
}

func deleteBucket(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	bucketName := vars["bucket"]
	err := db.Update(func(tx *bolt.Tx) error {
		return tx.DeleteBucket([]byte(bucketName))
	})
	TestError("delete bucket", err)
	http.Redirect(rw, r, "/", http.StatusOK)
}

func deleteKey(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	bucketName := vars["bucket"]
	keyName := vars["key"]
	err := db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
		TestError("create buk in delete", err)
		return bucket.Delete([]byte(keyName))
	})
	TestError("delete key", err)
	url := fmt.Sprintf("/view/%s", bucketName)
	http.Redirect(rw, r, url, http.StatusOK)
}

func editKey(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	bucketName := vars["bucket"]
	keyName := vars["key"]
	keyValue := r.FormValue("value")
	if len(keyValue) == 0 {
		//show edit page
		err := db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucketName))
			keyValue = string(b.Get([]byte(keyName)))
			return nil
		})
		TestError("view db", err)
		title := fmt.Sprintf("Edit key '%s' of bucket '%s'", keyName, bucketName)
		context := dict{"title": title, "bucket": bucketName, "key": keyName, "value": keyValue}
		err = tmplEditKey.Execute(rw, context)
		TestError("index view", err)
	} else {
		//save key
		err := db.Update(func(tx *bolt.Tx) error {
			bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
			TestError("create buk in edit key", err)
			return bucket.Put([]byte(keyName), []byte(keyValue))
		})
		TestError("update key", err)
		url := fmt.Sprintf("/view/%s", bucketName)
		http.Redirect(rw, r, url, http.StatusMovedPermanently)
	}
}

func uploadFile(rw http.ResponseWriter, r *http.Request) {
	bucketName := r.FormValue("bucket")
	file, header, err := r.FormFile("filename")
	if err != nil {
		fmt.Fprintln(rw, err)
		return
	}
	defer file.Close()
	if len(bucketName) == 0 {
		fmt.Fprintln(rw, "bucket name not defined")
		return
	}
	fmt.Println("\nUploading file", header.Filename)
	var mem runtime.MemStats
	//HeapSys will remain constant if uploading same batch of files
	//runtime.ReadMemStats(&mem)
	//fmt.Printf("Alloc=%dM TotalAlloc=%dM HeapAlloc=%dM HeapSys=%dM\n", mem.Alloc/1e6, mem.TotalAlloc/1e6, mem.HeapAlloc/1e6, mem.HeapSys/1e6)
	var filesize int
	t1 := time.Now().UnixNano()
	keyName := header.Filename
	err = db.Batch(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
		TestError("create bucket", err)
		var keyValue []byte
		keyValue, err = ioutil.ReadAll(file)
		filesize = len(keyValue)
		TestError("read uploaded file", err)
		err = bucket.Put([]byte(keyName), keyValue)
		TestError("put uploaded file", err)
		//keyValue = nil //that is not needed
		return nil
	})
	TestError("bolt update db in upload file", err)
	//runtime.ReadMemStats(&mem)
	//fmt.Printf("Alloc=%dM TotalAlloc=%dM HeapAlloc=%dM HeapSys=%dM\n", mem.Alloc/1e6, mem.TotalAlloc/1e6, mem.HeapAlloc/1e6, mem.HeapSys/1e6)
	//debug.FreeOSMemory() //also this is not necessary
	runtime.ReadMemStats(&mem)
	fmt.Printf("Alloc=%dM TotalAlloc=%dM HeapAlloc=%dM HeapSys=%dM\n", mem.Alloc/1e6, mem.TotalAlloc/1e6, mem.HeapAlloc/1e6, mem.HeapSys/1e6)
	url := fmt.Sprintf("/view/%s", bucketName)
	http.Redirect(rw, r, url, http.StatusMovedPermanently)
	t2 := time.Now().UnixNano()
	durata := float32(t2-t1) / 1e9
	fmt.Printf("Upload %.2fMB in %.1fs, rate is %.2f MB/s\n", float32(filesize)/1e6, durata, float32(filesize)/1e6/durata)
	if false { //this, if activated, will shutdown webserver in one second
		go func() {
			time.Sleep(time.Second * 1)
			os.Exit(0)
		}()
	}
}

func downloadKey(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	bucketName := vars["bucket"]
	keyName := vars["key"]
	var keyValue []byte
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucketName))
		keyValue = b.Get([]byte(keyName))
		return nil
	})
	TestError("view db get download key", err)
	content := fmt.Sprintf(`attachment; filename="%s"`, keyName)
	rw.Header().Set("Content-disposition", content)
	//rw.Header().Set("Content-Type", http.DetectContentType(keyValue))
	rw.Header().Set("Content-Length", strconv.Itoa(int(len(keyValue))))
	rw.Write(keyValue)
}

func backupDB(rw http.ResponseWriter, r *http.Request) {
	t1 := time.Now().UnixNano()
	bakfilename := fmt.Sprintf("%s.bak", dbfile)
	err := db.View(func(tx *bolt.Tx) error {
		f, err := os.Create(bakfilename)
		TestError("create bak", err)
		defer f.Close()
		w := bufio.NewWriter(f)
		tx.WriteTo(w)
		w.Flush()
		return err
	})
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	} else {
		t2 := time.Now().UnixNano()
		durata := float32(t2 - t1) / 1e9
		fmt.Fprintf(rw, "Backup db ok, filename=%s, durata: %.2f s,", bakfilename, durata)
	}
}

func downloadBackupDB(w http.ResponseWriter, req *http.Request) {
	t1 := time.Now().UnixNano()
	err := db.View(func(tx *bolt.Tx) error {
		w.Header().Set("Content-Type", "application/octet-stream")
		w.Header().Set("Content-Disposition", `attachment; filename="my.db"`)
		w.Header().Set("Content-Length", strconv.Itoa(int(tx.Size())))
		_, err := tx.WriteTo(w)
		return err
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else {
		t2 := time.Now().UnixNano()
		durata := float32(t2 - t1) / 1e9
		//print on console server
		fmt.Printf("Download bak ok, durata: %.2f s,", durata)
	}
}

func saveAs(rw http.ResponseWriter, req *http.Request) {
	var newfilename = fmt.Sprintf("%s.new", dbfile)
	newdb, err1 := bolt.Open(newfilename, 0644, &bolt.Options{Timeout: 1 * time.Second})
	TestError("open new db", err1)
	defer newdb.Close()
	t1 := time.Now().UnixNano()
	err2 := db.View(func(tx *bolt.Tx) error {
		c := tx.Cursor()
		for bucketName, _ := c.First(); bucketName != nil; bucketName, _ = c.Next() {
			err3 := newdb.Batch(func(tx *bolt.Tx) error {
				newbucket, err4 := tx.CreateBucketIfNotExists([]byte(bucketName))
				TestError("saveas new bucket", err4)
				//fmt.Println("create new bucket", string(bucketName))
				err5 := db.View(func(tx *bolt.Tx) error {
					oldbuket := tx.Bucket([]byte(bucketName))
					oldbuket.ForEach(func(k, v []byte) error {
						err6 := newbucket.Put(k, v)
						TestError("update kv new bucket", err6)
						//fmt.Println(" SET ", string(k))
						//debug.FreeOSMemory()
						return nil
					})
					return nil
				})
				TestError("view old db", err5)
				return nil
			})
			TestError("saveas create new bucket", err3)
		}
		return nil
	})
	TestError("db save as", err2)
	t2 := time.Now().UnixNano()
	durata := float32(t2 - t1) / 1e9
	fmt.Fprintf(rw, "Save as ok, file=%s, durata: %.2f s,", newfilename, durata)
	debug.FreeOSMemory()
}

func init() {
	var err error

	//flags
	flag.StringVar(&dbfile, "d", "bolt.db", "Database file")
	flag.BoolVar(&useSSL, "s", false, "Use https instead of http")
	flag.IntVar(&tcpPORT, "p", 8080, "Webserver port, should be 443 pentru https, anything else for http")

	//templates
	tmplBucketsView, err = template.New("bucketsview").Parse(strBucketsView)
	TestError("template", err)
	tmplBucketsView.Parse(strHeader)
	tmplBucketsView.Parse(strFooter)

	tmplBucketView, err = template.New("bucketview").Parse(strBucketView)
	TestError("template", err)
	tmplBucketView.Parse(strHeader)
	tmplBucketView.Parse(strFooter)

	tmplEditKey, err = template.New("editkey").Parse(strEditKey)
	TestError("template", err)
	tmplEditKey.Parse(strHeader)
	tmplEditKey.Parse(strFooter)
}

func main() {
	fmt.Println("Start server")
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	var err error
	db, err = bolt.Open(dbfile, 0644, &bolt.Options{Timeout: 1 * time.Second})
	TestError("open db", err)
	defer db.Close()

	r := mux.NewRouter()
	r.HandleFunc("/", indexHandler)
	r.HandleFunc("/view/{bucket}", viewBucket)
	r.HandleFunc("/add", addBucket)
	r.HandleFunc("/deleteb/{bucket}", deleteBucket)
	r.HandleFunc("/deletek/{bucket}/{key}", deleteKey)
	r.HandleFunc("/edit/{bucket}/{key}", editKey)
	r.HandleFunc("/upload", uploadFile)
	r.HandleFunc("/download/{bucket}/{key}", downloadKey)
	r.HandleFunc("/bak", backupDB)
	r.HandleFunc("/downloadbak", downloadBackupDB)
	r.HandleFunc("/saveasnew", saveAs)
	http.Handle("/", r)
	httpAddr := fmt.Sprintf("localhost:%d", tcpPORT)
	if useSSL {
		fmt.Println("Start https server,", httpAddr)
		//For windows
		exec.Command("cmd", "/c start https://"+httpAddr).Run()
		TestError("webserver", http.ListenAndServeTLS(httpAddr, "server.pem", "server.key", nil))
	} else {
		fmt.Println("Start http server,", httpAddr)
		//For windows
		exec.Command("cmd", "/c start http://"+httpAddr).Run()
		TestError("webserver", http.ListenAndServe(httpAddr, nil))
	}
}

func TestError(msg string, err error) {
	if err != nil {
		fmt.Println(msg)
		panic(err)
	}
}

func Extend(slice []dict, element dict) []dict {
	n := len(slice)
	if n == cap(slice) {
		// Slice is full; must grow.
		// We double its size and add 1, so if the size is zero we still grow.
		newSlice := make([]dict, len(slice), 2*len(slice)+1)
		copy(newSlice, slice)
		slice = newSlice
	}
	slice = slice[0 : n+1]
	slice[n] = element
	return slice
}

var (
	strBucketsView = `{{template "header" .}}
<h1>{{ .title }}</h1>
<div align="right">
<a href="/saveasnew">Save as new db</a> - <a href="/bak">Backup to local file</a> - <a href="/downloadbak">Download snapshot of db</a>
</div>
<br>
<form method="POST" action="/add">
<input type="text" name="bucket" value=""><input type="submit" value="Add bit bucket"><br></form>
<ul>
{{ range .buckets}}
<li>{{if .showLink}}<a href="/view/{{.name}}">{{.name}}</a>{{end}}
{{if not .showLink}}{{.name}}{{end}}
 - [<a href="/deleteb/{{.name}}" title="delete bucket">X</a>] - {{.size}} keys.</li>
{{end}}
</ul>
{{template "footer" .}}
`
	strBucketView = `{{template "header" .}}
<h1>{{ .title }}</h1>
<a href="/">Home</a><br>

<table>
<tr><th></th><th>Name</th><th>Size</th><th>Content</th><th></th></tr>
{{$bb := .bucket}}
{{ range .keys}}
<tr>
<td>{{if .showValue}}<a href="/edit/{{$bb}}/{{.name}}">Edit</a>{{end}}</td>
<td>{{.name}}</td>
<td>{{.sizeM}}M</td>
<td>
{{if .showValue}}{{.value}}{{end}}
{{if not .showValue}}<a href="/download/{{$bb}}/{{.name}}">Download as binary</a>{{end}}
</td>
<td><a href="/deletek/{{$bb}}/{{.name}}" title="delete key">Delete</a></td>
</tr>
{{end}}
</table>
<hr>
<form method="POST" action="/upload" enctype="multipart/form-data">
<input type="hidden" name="bucket" value="{{.bucket}}">
File: <input type="file" name="filename" id="filename"> <input type="submit" value="Upload">
</form>
<form method="POST" action="/add">
<input type="hidden" name="bucket" value="{{.bucket}}">
Add key:<input type="text" name="key" value=""> <input type="submit" value="Add key/value to bit bucket"><br>
Add value:<br><textarea cols="60" rows="5" name="value"></textarea>
</form>

{{template "footer" .}}
`

	strEditKey = `{{template "header" .}}
<h1>{{ .title }}</h1>
<a href="/">Home</a> - <a href="/view/{{.bucket}}">{{.bucket}}</a><br>
<form method="POST" action="/edit/{{.bucket}}/{{.key}}">
<input type="submit" value="Update"><br>
<textarea cols="60" rows="5" name="value">{{.value}}</textarea>
</form>
{{template "footer" .}}
`

	strHeader = `{{define "header"}}<!DOCTYPE html>
<html lang="en">
<head>
	<meta charset="utf-8">
	<meta name="description" content="Project in Go using net/http as a web admin for bolt">
	<meta name="keywords" content="go, golang, microframework, mvc">
	<title>{{.title}}</title>
	<style>
	html {
	background-color: #fff;
	padding: 20px;
	font-size: 16px;
	font-family: Verdana, Geneva, sans-serif;
	}
	a { text-decoration: none; }
    a:link, a:visited { color: #27f;}
    a:hover { color: red;}
	tr:nth-child(even) {background: #CEF}
	tr:nth-child(odd) {background: #CFF}
	</style>
</head>
<body>	
{{end}}`

	strFooter = `{{define "footer"}}
<hr>
	<div id="footer" align="right">
Bolt admin server
	</div>
  </body>
</html>
{{end}}`
)



Ultimele pagini: RSS

Alte adrese de Internet

Categorii

Istoric



Contorizari incepand cu 9 iunie 2014:
Flag Counter

Atentie: Continutul acestui server reprezinta ideile mele si acestea pot fi gresite.