Camlistore branch, master, updated. 71f1646fdfbcfbfdc4cb6081e2ed46224a08ce82
Camlistore-git
The branch, master has been updated
via 71f1646fdfbcfbfdc4cb6081e2ed46224a08ce82 (commit)
via 1a14a121d04f6f5d58d62fa4ea72ff6c619d588b (commit)
via 962d65877d16d21da352e8850c4f179f55b015dc (commit)
via 73a7329401227d0fde8cd6b088c41b4961a7b2a4 (commit)
via 916c2cdaad3e6da77b38331c5a793e5b8d4ca309 (commit)
via b078f4f88ce1488e02a12e92a45e2ada913c9c39 (commit)
via 9f8aae9d0aea8a95fc02a030a6f758886fd22365 (commit)
from 127b23ad947b89359bdeffe0029e2257710dc203 (commit)
- Log -----------------------------------------------------------------
commit 71f1646fdfbcfbfdc4cb6081e2ed46224a08ce82
Author: Brad Fitzpatrick <br...@danga.com>
Date: Thu May 5 18:27:19 2011 -0700
more UI js/html work, for signatures.
commit 1a14a121d04f6f5d58d62fa4ea72ff6c619d588b
Author: Brad Fitzpatrick <br...@danga.com>
Date: Thu May 5 17:15:32 2011 -0700
improve jsonsign discovery handler
commit 962d65877d16d21da352e8850c4f179f55b015dc
Author: Brad Fitzpatrick <br...@danga.com>
Date: Thu May 5 16:41:00 2011 -0700
some sig discovery work
commit 73a7329401227d0fde8cd6b088c41b4961a7b2a4
Author: Brad Fitzpatrick <br...@danga.com>
Date: Thu May 5 16:31:19 2011 -0700
blobref memory store
commit 916c2cdaad3e6da77b38331c5a793e5b8d4ca309
Author: Brad Fitzpatrick <br...@danga.com>
Date: Thu May 5 16:31:09 2011 -0700
disable fonts to speed up slide development
commit b078f4f88ce1488e02a12e92a45e2ada913c9c39
Author: Brad Fitzpatrick <br...@danga.com>
Date: Thu May 5 15:50:16 2011 -0700
more slides
commit 9f8aae9d0aea8a95fc02a030a6f758886fd22365
Author: Brad Fitzpatrick <br...@danga.com>
Date: Thu May 5 15:06:49 2011 -0700
more camlistore slide work
-----------------------------------------------------------------------
Summary of changes:
lib/go/camli/blobref/fetcher.go | 31 +-
lib/go/camli/blobserver/handlers/get.go | 26 +-
lib/go/camli/httputil/httputil.go | 16 +
server/go/camlistored/camlistored.go | 2 +-
server/go/camlistored/sig.go | 63 +++-
server/go/camlistored/ui/index.html | 25 +
server/go/camlistored/ui/ui.js | 56 ++-
.../2011-05-07-Camlistore-Sao-Paolo/blobjects.png | Bin 0 -> 136514 bytes
talks/2011-05-07-Camlistore-Sao-Paolo/diagrams.odp | Bin 0 -> 15790 bytes
talks/2011-05-07-Camlistore-Sao-Paolo/fsbackup.png | Bin 0 -> 26577 bytes
talks/2011-05-07-Camlistore-Sao-Paolo/index.html | 534 ++++++++++++--------
talks/2011-05-07-Camlistore-Sao-Paolo/repl.png | Bin 0 -> 36057 bytes
talks/2011-05-07-Camlistore-Sao-Paolo/slides.js | 4 +-
13 files changed, 512 insertions(+), 245 deletions(-)
create mode 100644 talks/2011-05-07-Camlistore-Sao-Paolo/blobjects.png
create mode 100644 talks/2011-05-07-Camlistore-Sao-Paolo/diagrams.odp
create mode 100644 talks/2011-05-07-Camlistore-Sao-Paolo/fsbackup.png
create mode 100644 talks/2011-05-07-Camlistore-Sao-Paolo/repl.png
Diff:
diff --git a/lib/go/camli/blobref/fetcher.go b/lib/go/camli/blobref/fetcher.go
index f5dfcfe..fb15978 100644
--- a/lib/go/camli/blobref/fetcher.go
+++ b/lib/go/camli/blobref/fetcher.go
@@ -20,9 +20,12 @@ import (
"crypto"
"fmt"
"io"
+ "io/ioutil"
"log"
"os"
"path/filepath"
+ "strings"
+ "sync"
)
var _ = log.Printf
@@ -113,18 +116,36 @@ func (df *DirFetcher) Fetch(b *BlobRef) (file ReadSeekCloser, size int64, err os
// MemoryStore stores blobs in memory and is a Fetcher and
// StreamingFetcher. Its zero value is usable.
type MemoryStore struct {
- m map[string][]byte
+ lk sync.Mutex
+ m map[string]string
}
-func (s *MemoryStore) AddBlob(hashtype crypto.Hash, data []byte) os.Error {
+func (s *MemoryStore) AddBlob(hashtype crypto.Hash, data string) (*BlobRef, os.Error) {
if hashtype != crypto.SHA1 {
- return os.NewError("blobref: unsupported hash type")
+ return nil, os.NewError("blobref: unsupported hash type")
}
hash := hashtype.New()
- hash.Write(data)
+ hash.Write([]byte(data))
bstr := fmt.Sprintf("sha1-%x", hash.Sum())
+ s.lk.Lock()
+ defer s.lk.Unlock()
+ if s.m == nil {
+ s.m = make(map[string]string)
+ }
s.m[bstr] = data
log.Printf("added %s", bstr)
- return nil
+ return Parse(bstr), nil
}
+func (s *MemoryStore) FetchStreaming(b *BlobRef) (file io.ReadCloser, size int64, err os.Error) {
+ s.lk.Lock()
+ defer s.lk.Unlock()
+ if s.m == nil {
+ return nil, 0, os.ENOENT
+ }
+ str, ok := s.m[b.String()]
+ if !ok {
+ return nil, 0, os.ENOENT
+ }
+ return ioutil.NopCloser(strings.NewReader(str)), int64(len(str)), nil
+}
diff --git a/lib/go/camli/blobserver/handlers/get.go b/lib/go/camli/blobserver/handlers/get.go
index e139a4f..b5b6b93 100644
--- a/lib/go/camli/blobserver/handlers/get.go
+++ b/lib/go/camli/blobserver/handlers/get.go
@@ -38,14 +38,20 @@ import (
var kGetPattern *regexp.Regexp = regexp.MustCompile(`/camli/([a-z0-9]+)-([a-f0-9]+)$`)
+type GetHandler struct {
+ Fetcher blobref.StreamingFetcher
+ AllowGlobalAccess bool
+}
+
func CreateGetHandler(fetcher blobref.StreamingFetcher) func(http.ResponseWriter, *http.Request) {
+ gh := &GetHandler{Fetcher: fetcher}
return func(conn http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/camli/sha1-deadbeef00000000000000000000000000000000" {
// Test handler.
simulatePrematurelyClosedConnection(conn, req)
return
}
- handleGet(conn, req, fetcher)
+ gh.ServeHTTP(conn, req)
}
}
@@ -57,27 +63,27 @@ func sendUnauthorized(conn http.ResponseWriter) {
fmt.Fprintf(conn, "<h1>Unauthorized</h1>")
}
-func handleGet(conn http.ResponseWriter, req *http.Request, fetcher blobref.StreamingFetcher) {
+func (h *GetHandler) ServeHTTP(conn http.ResponseWriter, req *http.Request) {
blobRef := blobFromUrlPath(req.URL.Path)
if blobRef == nil {
- httputil.BadRequestError(conn, "Malformed GET URL.")
+ http.Error(conn, "Malformed GET URL.", 400)
return
}
switch {
- case auth.IsAuthorized(req):
- serveBlobRef(conn, req, blobRef, fetcher)
+ case h.AllowGlobalAccess || auth.IsAuthorized(req):
+ serveBlobRef(conn, req, blobRef, h.Fetcher)
case auth.TriedAuthorization(req):
log.Printf("Attempted authorization failed on %s", req.URL)
sendUnauthorized(conn)
default:
- handleGetViaSharing(conn, req, blobRef, fetcher)
+ handleGetViaSharing(conn, req, blobRef, h.Fetcher)
}
}
// serveBlobRef sends 'blobref' to 'conn' as directed by the Range header in 'req'
func serveBlobRef(conn http.ResponseWriter, req *http.Request,
- blobRef *blobref.BlobRef, fetcher blobref.StreamingFetcher) {
+blobRef *blobref.BlobRef, fetcher blobref.StreamingFetcher) {
file, size, err := fetcher.FetchStreaming(blobRef)
switch err {
@@ -180,7 +186,7 @@ func serveBlobRef(conn http.ResponseWriter, req *http.Request,
// Unauthenticated user. Be paranoid.
func handleGetViaSharing(conn http.ResponseWriter, req *http.Request,
- blobRef *blobref.BlobRef, fetcher blobref.StreamingFetcher) {
+blobRef *blobref.BlobRef, fetcher blobref.StreamingFetcher) {
viaPathOkay := false
startTime := time.Nanoseconds()
@@ -307,5 +313,5 @@ func simulatePrematurelyClosedConnection(conn http.ResponseWriter, req *http.Req
flusher.Flush()
}
wrc, _, _ := hj.Hijack()
- wrc.Close() // without sending final chunk; should be an error for the client
-}
\ No newline at end of file
+ wrc.Close() // without sending final chunk; should be an error for the client
+}
diff --git a/lib/go/camli/httputil/httputil.go b/lib/go/camli/httputil/httputil.go
index 6c63edb..b913e6c 100644
--- a/lib/go/camli/httputil/httputil.go
+++ b/lib/go/camli/httputil/httputil.go
@@ -22,6 +22,7 @@ import (
"json"
"os"
"log"
+ "strings"
)
func BadRequestError(conn http.ResponseWriter, errorMessage string) {
@@ -46,3 +47,18 @@ func ReturnJson(conn http.ResponseWriter, data interface{}) {
conn.Write(bytes)
conn.Write([]byte("\n"))
}
+
+type PrefixHandler struct {
+ Prefix string
+ Handler http.Handler
+}
+
+func (p *PrefixHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+ if !strings.HasPrefix(req.URL.Path, p.Prefix) {
+ http.Error(rw, "Inconfigured PrefixHandler", 500)
+ return
+ }
+ req.Header.Set("X-PrefixHandler-PathBase", p.Prefix)
+ req.Header.Set("X-PrefixHandler-PathSuffix", req.URL.Path[len(p.Prefix):])
+ p.Handler.ServeHTTP(rw, req)
+}
diff --git a/server/go/camlistored/camlistored.go b/server/go/camlistored/camlistored.go
index 02beb5a..0b6d280 100644
--- a/server/go/camlistored/camlistored.go
+++ b/server/go/camlistored/camlistored.go
@@ -273,7 +273,7 @@ func configFileMain() {
prefix, err)
}
createdHandlers[prefix] = h
- ws.Handle(prefix, h)
+ ws.Handle(prefix, &httputil.PrefixHandler{prefix, h})
}
switch {
case handlerType == "search":
diff --git a/server/go/camlistored/sig.go b/server/go/camlistored/sig.go
index 707dc19..4e682f6 100644
--- a/server/go/camlistored/sig.go
+++ b/server/go/camlistored/sig.go
@@ -18,6 +18,7 @@ package main
import (
"bytes"
+ "crypto"
"crypto/openpgp"
"crypto/openpgp/armor"
"fmt"
@@ -28,6 +29,8 @@ import (
"strings"
"camli/blobref"
+ "camli/blobserver/handlers"
+ "camli/httputil"
"camli/jsonconfig"
"camli/jsonsign"
)
@@ -44,8 +47,12 @@ type JSONSignHandler struct {
// of the longer forms.
keyId string
+ pubKeyBlobRef *blobref.BlobRef
pubKeyFetcher blobref.StreamingFetcher
+ pubKeyBlobRefServeSuffix string // "camli/sha1-xxxx"
+ pubKeyHandler http.Handler
+
entity *openpgp.Entity
}
@@ -98,19 +105,58 @@ func createJSONSignHandler(conf jsonconfig.Obj) (http.Handler, os.Error) {
wc.Close()
log.Printf("got key: %s", buf.String())
- h.pubKeyFetcher = nil // TODO
+ ms := new(blobref.MemoryStore)
+ h.pubKeyBlobRef, err = ms.AddBlob(crypto.SHA1, buf.String())
+ if err != nil {
+ return nil, err
+ }
+ h.pubKeyFetcher = ms
+
+ h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
+ h.pubKeyHandler = &handlers.GetHandler{
+ Fetcher: ms,
+ AllowGlobalAccess: true, // just public keys
+ }
return h, nil
}
func (h *JSONSignHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+ base := req.Header.Get("X-PrefixHandler-PathBase")
+ subPath := req.Header.Get("X-PrefixHandler-PathSuffix")
switch req.Method {
+ case "GET":
+ switch subPath {
+ case "":
+ http.Redirect(rw, req, base + "camli/sig/discovery", http.StatusFound)
+ return
+ case h.pubKeyBlobRefServeSuffix:
+ h.pubKeyHandler.ServeHTTP(rw, req)
+ return
+ case "camli/sig/sign":
+ fallthrough
+ case "camli/sig/verify":
+ http.Error(rw, "POST required", 400)
+ return
+ case "camli/sig/discovery":
+ m := map[string]interface{}{
+ "publicKeyId": h.keyId,
+ "signHandler": base + "camli/sig/sign",
+ "verifyHandler": base + "camli/sig/verify",
+ }
+ if h.pubKeyBlobRef != nil {
+ m["publicKeyBlobRef"] = h.pubKeyBlobRef.String()
+ m["publicKey"] = base + h.pubKeyBlobRefServeSuffix
+ }
+ httputil.ReturnJson(rw, m)
+ return
+ }
case "POST":
- switch {
- case strings.HasSuffix(req.URL.Path, "/camli/sig/sign"):
+ switch subPath {
+ case "camli/sig/sign":
h.handleSign(rw, req)
return
- case strings.HasSuffix(req.URL.Path, "/camli/sig/verify"):
+ case "camli/sig/verify":
h.handleVerify(rw, req)
return
}
@@ -125,9 +171,16 @@ func (h *JSONSignHandler) handleVerify(rw http.ResponseWriter, req *http.Request
func (h *JSONSignHandler) handleSign(rw http.ResponseWriter, req *http.Request) {
req.ParseForm()
+ badReq := func(s string) {
+ http.Error(rw, s, http.StatusBadRequest)
+ log.Printf("bad request: %s", s)
+ return
+ }
+ // TODO: SECURITY: auth
+
jsonStr := req.FormValue("json")
if jsonStr == "" {
- http.Error(rw, "Missing json parameter", http.StatusBadRequest)
+ badReq(rw, "Missing json parameter", http.StatusBadRequest)
return
}
if len(jsonStr) > kMaxJsonLength {
diff --git a/server/go/camlistored/ui/index.html b/server/go/camlistored/ui/index.html
index e896a37..8f98503 100644
--- a/server/go/camlistored/ui/index.html
+++ b/server/go/camlistored/ui/index.html
@@ -8,10 +8,35 @@
<h1>Camlistored UI</h1>
<form>
+ <h2>Root Discovery</h2>
<p><input type="button" id="discobtn" onclick="discover()" value="Do Discovery" /></p>
<div id="discores" style="border: 2px solid gray">(discovery results)</div>
+
+
+ <h2>Signing</h2>
+ <p><input type="button" id="sigdiscobtn" onclick="discoverJsonSign()" value="Do jsonSign discovery" /></p>
+ <div id="sigdiscores" style="border: 2px solid gray">(jsonsign discovery results)</div>
+
+ <table>
+ <tr align='left'>
+ <th>JSON blob to sign: <input type='button' id='addkeyref' onclick='addKeyRef()' value="Add keyref"/></th>
+ <th></th>
+ <th>Signed blob:</th>
+ </tr>
+ <tr>
+ <td><textarea id='clearjson' rows=10 cols=40>{"camliVersion": 1,
+ "camliType": "whatever",
+ "foo": "bar"
+}</textarea></td>
+ <td valign='middle'><input type='button' id='sign' onclick='doSign()' value="Sign >>" /></td>
+ <td><textarea id="signedjson" rows=10 cols=40 readonly></textarea></td>
+ </tr>
+ </table>
+
+ <h2>Search</h2>
<p><input type="button" id="searchbtn" onclick="search()" value="Do Search" /></p>
<div id="searchres" style="border: 2px solid gray">(search results)</div>
+
</form>
</body>
diff --git a/server/go/camlistored/ui/ui.js b/server/go/camlistored/ui/ui.js
index 4401045..91c6ece 100644
--- a/server/go/camlistored/ui/ui.js
+++ b/server/go/camlistored/ui/ui.js
@@ -38,8 +38,62 @@ function discover() {
xhr.send();
}
+var sigdisco = null;
+
+function discoverJsonSign() {
+ var xhr = new XMLHttpRequest();
+ xhr.onreadystatechange = function() {
+ if (xhr.readyState != 4) { return; }
+ if (xhr.status != 200) {
+ console.log("no status 200; got " + xhr.status);
+ return;
+ }
+ sigdisco = JSON.parse(xhr.responseText);
+ document.getElementById("sigdiscores").innerHTML = JSON.stringify(sigdisco);
+ };
+ xhr.open("GET", disco.jsonSignRoot + "/camli/sig/discovery", true);
+ xhr.send();
+}
+
+function addKeyRef() {
+ if (!sigdisco) {
+ alert("must do jsonsign discovery first");
+ return;
+ }
+ clearta = document.getElementById("clearjson");
+ var j;
+ try {
+ j = JSON.parse(clearta.value);
+ } catch (x) {
+ alert(x);
+ return
+ }
+ j.camliSigner = sigdisco.publicKeyBlobRef;
+ clearta.value = JSON.stringify(j);
+}
+
+function doSign() {
+ if (!sigdisco) {
+ alert("must do jsonsign discovery first");
+ return;
+ }
+ clearta = document.getElementById("clearjson");
+
+ var xhr = new XMLHttpRequest();
+ xhr.onreadystatechange = function() {
+ if (xhr.readyState != 4) { return; }
+ if (xhr.status != 200) {
+ alert("got status " + xhr.status)
+ return;
+ }
+ document.getElementById("signedjson").value = xhr.responseText;
+ };
+ xhr.open("POST", sigdisco.signHandler, true)
+ xhr.send("json=" + encodeURIComponent(clearta.vale));
+}
+
function search() {
- var xhr = new XMLHttpRequest();
+ var xhr = new XMLHttpRequest();
xhr.onreadystatechange = function() {
if (xhr.readyState != 4) { return; }
if (xhr.status != 200) {
diff --git a/talks/2011-05-07-Camlistore-Sao-Paolo/blobjects.png b/talks/2011-05-07-Camlistore-Sao-Paolo/blobjects.png
new file mode 100644
index 0000000..7ede091
Binary files /dev/null and b/talks/2011-05-07-Camlistore-Sao-Paolo/blobjects.png differ
diff --git a/talks/2011-05-07-Camlistore-Sao-Paolo/diagrams.odp b/talks/2011-05-07-Camlistore-Sao-Paolo/diagrams.odp
new file mode 100644
index 0000000..629da12
Binary files /dev/null and b/talks/2011-05-07-Camlistore-Sao-Paolo/diagrams.odp differ
diff --git a/talks/2011-05-07-Camlistore-Sao-Paolo/fsbackup.png b/talks/2011-05-07-Camlistore-Sao-Paolo/fsbackup.png
new file mode 100644
index 0000000..8c6d240
Binary files /dev/null and b/talks/2011-05-07-Camlistore-Sao-Paolo/fsbackup.png differ
diff --git a/talks/2011-05-07-Camlistore-Sao-Paolo/index.html b/talks/2011-05-07-Camlistore-Sao-Paolo/index.html
index c290736..0e7b039 100755
--- a/talks/2011-05-07-Camlistore-Sao-Paolo/index.html
+++ b/talks/2011-05-07-Camlistore-Sao-Paolo/index.html
@@ -164,7 +164,7 @@
Motivation (cont)
</h3>
<ul>
- <li>I still want to solve the Decentralized Social Network Problem
+ <li>I still want to help solve the Decentralized Social Network Problem
<ul>
<li>protocols, not companies</li>
<li>gmail, hotmail: hosted versions of SMTP, IMAP</li>
@@ -194,17 +194,27 @@
</h3>
<ul>
<li>one private dumping ground to store anything</li>
- <li><i><b>your</b></i> private repo, for life</li>
- <li>backups, filesystems, objects, my website, ...</li>
+ <li>backups, filesystems, objects, photos, likes, bookmarks, shares, my website, ...</li>
<li>live backup my phone</li>
<li>live replicate / sync my dumping group between my house & laptop & Amazon & Google</li>
- <li>give out access to / sub-trees to friends or the world</li>
<li>web UI (ala gmail, docs.google.com, etc) or FUSE filesystem</li>
<li>Easy for end-users; powerful for dorks</li>
</ul>
</article>
<article>
+ <h3>
+ Security Model
+ </h3>
+ <ul>
+ <li><i><b>your</b></i> private repo, for life</li>
+ <li>everything private by default</li>
+ <li>grant access to specific objects/trees with friends or the world</li>
+ <li>web UI or CLI tools let you share</li>
+ </ul>
+ </article>
+
+ <article>
<h1 class='center'>
So what's with the silly name?
</h1>
@@ -275,210 +285,358 @@
</ul>
</article>
+ <article>
+ <h2>
+ From the bottom up...
+ </h2>
+ </article>
+
+ <article>
+ <h2>
+ Blob Server
+ </h2>
+ </article>
+
+ <article>
+ <h3>Blob Server: how dumb it is</h3>
+ <ul>
+ <li>"Blob" == zero or more bytes. <i class='red'>no</i> meta-data</li>
+ <li>private operations, to owner of data only:</li>
+ <ul>
+ <li class='green'>get(blobref) → blob</li>
+ <li class='green'>stat(blobref+) → [(blobref, size), ...]</li>
+ <li class='green'>put(blobref, blob)</li>
+ <li class='green'>enumerate(..) → [(blobref, size)...] (sorted by blobref)</li>
+ </ul>
+ <li>no public (non-owner) access</li>
+ <li>HTTP interface: <tt>GET /camli/sha1-xxxxxxx HTTP/1.1</tt></li>
+ <li><span class='green'>delete(blobref)</span> is disabled by default, privileged op for GC or replication queues only</li>
+ </ul>
+ </article>
+
+ <article>
+ <h3>Blob Server: seriously, no metadata</h3>
+ <ul>
+ <li>no filenames</li>
+ <li>no "mime types"</li>
+ <li>no "{create,mod,access} time"</li>
+ <li>size is implicit</li>
+ <li>blob: just some bytes</li>
+ <li>metadata? layers above.</li>
+ </ul>
+ </article>
+
+ <article>
+ <h1 class='center'>
+ Uh, what can you do with that?
+ </h1>
+ </article>
+
+ <article>
+ <h3>Uh, what can you do with that?</h3>
+ <ul>
+ <li>with just a blob server?</li>
+ <li>not much</li>
+ <li>but let's start with an easy example...</li>
+ </ul>
+ </article>
+
+ <article>
+ <h1 class='center'>
+ Filesystem Backups
+ </h1>
+ </article>
+
+ <article>
+ <h3>Filesystem Backups</h3>
+ <ul>
+ <li>previous project: brackup
+ <ul>
+ <li>good: Perl, slide/dice/encrypt S3 backup, content-addressed, good iterative backups</li>
+ <li>bad: large (several MB) "backup manifest" text files
+ </ul>
+ </li>
+
+ <li>fossil/venti, git, etc: directories content-addressed by content of their children, hash trees, etc</li>
+ <li>git: "tree objects", "commmit objects", etc</li>
+ <li>Camlistore: "schema blobs"</li>
+ </ul>
+ </article>
+
+ <article>
+ <h3>Schema: how to model your content</h3>
+ <ul>
+ <li>Camlistore defines <i>one possible</i> schema</li>
+ <li>but blobserver doesn't know about it all</li>
+ <li>tools generate schema,</li>
+ <li>indexer + search understand the schema.</li>
+ </ul>
+ </article>
+
+ <article>
+ <h3>Schema Blobs</h3>
+ <ul>
+ <li>so if all blobs are just dumb blobs of bytes with no metadata,</li>
+ <li>how do you store metadata?</li>
+ <li>as blobs themselves!</li>
+ </ul>
+ </article>
+
+ <article>
+ <h3>Minimal Schema Blob</h3>
+
+ <section>
+ <pre>{
+ "camliVersion": 1,
+ "camliType": "whatever"
+}</pre>
+ </section>
+ <p>Whitespace doesn't matter. Just must be valid JSON in its
+entirety. Use whatever JSON libraries you've got.</p>
+
+ <p>That one is named<br/><tt class='smaller'>sha1-19e851fe3eb3d1f3d9d1cefe9f92c6f3c7d754f6</tt></p>
+ <p>or perhaps: <tt class='smaller'>sha512-2c6746aba012337aaf113fd63c24d994a0703d33eb5d6ed58859e45dc4e02dcf<br/>dae5c4d46c5c757fb85d5aff342245fe4edb780c028a6f3c994c1295236c931e</tt></p>
+
+ </article>
+
<!-- END -->
<article>
- <p>
- This is a slide with just text. This is a slide with just text.
- This is a slide with just text. This is a slide with just text.
- This is a slide with just text. This is a slide with just text.
- </p>
- <p>
- There is more text just underneath.
- </p>
+ <h3>Schema blob; type "file"</h3>
+ <section><pre>{"camliVersion": 1,
+ <span style='background: #fff'>"camliType": "file",</span>
+ "fileName": "foo.dat",
+ "unixPermission": "0644",
+ ...,
+ "size": 6000133,
+ "contentParts": [
+ {"blobRef": "sha1-...dead", "size": 111},
+ {"blobRef": "sha1-...beef", "size": 5000000, "offset": 492 },
+ {"size": 1000000},
+ {"blobRef": "digalg-blobref", "size": 22},
+ ]
+}</pre></section>
+ </article>
+
+ <article>
+ <h3>Schema blob; type "directory"</h3>
+ <section><pre>{"camliVersion": 1,
+ <span style='background: #fff'>"camliType": "directory",</span>
+ "fileName": "foodir",
+ "unixPermission": "0755",
+ ...,
+ "entries": <span style='background: #fff'>"sha1-c3764bc2138338d5e2936def18ff8cc9cda38455"</span>
+}</pre></section>
+ </article>
+
+ <article>
+ <h3>Schema blob; type "static-set"</h3>
+ <section><pre>{"camliVersion": 1,
+ <span style='background: #fff'>"camliType": "static-set",</span>
+"members": [
+ "sha1-xxxxxxxxxxxx",
+ "sha1-xxxxxxxxxxxx",
+ "sha1-xxxxxxxxxxxx",
+ "sha1-xxxxxxxxxxxx",
+ "sha1-xxxxxxxxxxxx",
+ "sha1-xxxxxxxxxxxx",
+ ]
+}</pre></section>
</article>
<article>
<h3>
- Simple slide with header and text
+ Backup a directory...
</h3>
- <p>
- This is a slide with just text. This is a slide with just text.
- This is a slide with just text. This is a slide with just text.
- This is a slide with just text. This is a slide with just text.
- </p>
- <p>
- There is more text just underneath with a <code>code sample: 5px</code>.
- </p>
+ <section><pre>$ camput --file $HOME
+sha1-8659a52f726588dc44d38dfb22d84a4da2902fed</pre></section>
+
+<p>(like git/hg/fossil, that identifier represents everything down.)</p>
+
+<p>Iterative backups are cheap, easy identifier to share, etc</p>
+
+<p>But how will you remember that identifier? (later)</p>
</article>
- <article class='smaller'>
+ <article>
<h3>
- Simple slide with header and text (small font)
+ But what about mutable data?
</h3>
- <p>
- This is a slide with just text. This is a slide with just text.
- This is a slide with just text. This is a slide with just text.
- This is a slide with just text. This is a slide with just text.
- </p>
- <p>
- There is more text just underneath with a <code>code sample: 5px</code>.
- </p>
+ <ul>
+ <li>immutable data is easy to represent & reference</li>
+ <ul>
+ <li><tt class='smaller'>sha1-8659a52f726588dc44d38dfb22d84a4da2902fed</tt> is an immutable snapshot</li>
+ </ul>
+ <li>how to represent mutable data in an immutable, content-addressed world?</li>
+ <li>how to share a reference to a mutable object when changing an object mutates its name?</li>
+ </ul>
</article>
<article>
+ <h1 class='center'>
+ Objects & "Permanodes"
+ </h1>
+ </article>
+
+ <article>
<h3>
- Slide with bullet points and a longer title, just because we
- can make it longer
+ Terminology
</h3>
<ul>
- <li>
- Use this template to create your presentation
- </li>
- <li>
- Use the provided color palette, box and arrow graphics, and
- chart styles
- </li>
- <li>
- Instructions are provided to assist you in using this
- presentation template effectively
- </li>
- <li>
- At all times strive to maintain Google's corporate look and feel
- </li>
+ <li><span class='red'>blob</span>: just dumb, immutable series of bytes</li>
+ <li><span class='red'>schema blob</span>: a blob that's a valid JSON object w/ camliVersion & camliType</li>
+ <li><span class='red'>signed schema blob</span> aka "<span class='red'>claim</span>": a schema blob with an embedded OpenPGP signature</li>
+ <li><span class='red'>object</span>: something mutable. represented as an anchor "<span class='blue'>permanode</span>" + a set of mutations (<span class='blue'>claims</span>)</li>
+ <li><span class='red'>permanode</span>: a stable reference. an anchor. just a <span class='blue'>signed schema blob</span>, but of almost no content...</li>
</ul>
</article>
<article>
<h3>
- Slide with bullet points that builds
+ Permanode
</h3>
- <ul class="nobuild">
- <li>
- This is an example of a list
- </li>
- <li>
- The list items fade in
- </li>
- <li>
- Last one!
- </li>
- </ul>
+ <section><pre><span style='font-weight: bold' class='blue'>$ camput --permanode</span>
+sha1-ea799271abfbf85d8e22e4577f15f704c8349026
+
+<span style='font-weight: bold' class="blue">$ camget sha1-ea799271abfbf85d8e22e4577f15f704c8349026</span>
+<span style="background: #ff7">{"camliVersion": 1,
+ "camliSigner": "sha1-c4da9d771661563a27704b91b67989e7ea1e50b8",
+ <span style='font-weight: bold'>"camliType": "permanode"</span>,
+ "random": "oj)r}$Wa/[J|XQThNdhE"</span>
+,"camliSig":"iQEcBAABAgAGBQJNRxceAAoJEGjzeDN/6vt8ihIH/Aov7FRIq4dODAPWGDwqL
+1X9Ko2ZtSSO1lwHxCQVdCMquDtAdI3387fDlEG/ALoT/LhmtXQgYTt8QqDxVdu
+EK1or6/jqo3RMQ8tTgZ+rW2cj9f3Q/dg7el0Ngoq03hyYXdo3whxCH2x0jajSt4RCc
+gdXN6XmLlOgD/LVQEJ303Du1OhCvKX1A40BIdwe1zxBc5zkLmoa8rClAlHdqwo
+gxYFY4cwFm+jJM5YhSPemNrDe8W7KT6r0oA7SVfOan1NbIQUel65xwIZBD0ah
+CXBx6WXvfId6AdiahnbZiBup1fWSzxeeW7Y2/RQwv5IZ8UgfBqRHvnxcbNmScrzl
+p3V3ZoY"}</pre></section>
- <div class="nobuild">
- <p>Any element with child nodes can build.</p>
- <p>It doesn't have to be a list.</p>
- </div>
</article>
- <article class='smaller'>
+ <article>
<h3>
- Slide with bullet points (small font)
+ Backup a directory...
</h3>
+ <section><pre><span style='font-weight: bold'>$ camput --file $HOME</span>
+sha1-8659a52f726588dc44d38dfb22d84a4da2902fed
+<span style='font-weight: bold'>$ camput --permanode --file $HOME</span>
+sha1-ea799271abfbf85d8e22e4577f15f704c8349026
+<span style='font-weight: bold'>$ camput --permanode --name="Brad's home directory" --file $HOME</span>
+sha1-ea799271abfbf85d8e22e4577f15f704c8349026</pre></section>
<ul>
- <li>
- Use this template to create your presentation
- <li>
- Use the provided color palette, box and arrow graphics, and
- chart styles
- <li>
- Instructions are provided to assist you in using this
- presentation template effectively
- <li>
- At all times strive to maintain Google's corporate look and feel
+ <li>all the file data blobs, file/dir schema blobs,</li>
+ <li>a new permanode, owned by you</li>
+ <li>a mutation: permanode's content attribute == directory root</li>
+ <li>a mutation: permanode's name attribute == "Brad's home directory"</li>
</ul>
</article>
+ <article class='fill'>
+ <p><img src="fsbackup.png" height="100%"/></p>
+ </article>
+
<article>
- <h3>
- Slide with a table
- </h3>
-
- <table>
- <tr>
- <th>
- Name
- <th>
- Occupation
- <tr>
- <td>
- Luke Mahé
- <td>
- V.P. of Keepin’ It Real
- <tr>
- <td>
- Marcin Wichary
- <td>
- The Michael Bay of Doodles
- </table>
+ <h1 class='center'>
+ Modeling non-filesystem objects
+ </h1>
</article>
-
- <article class='smaller'>
+
+ <article>
+ <h3>Example: a photo gallery</h3>
+ <ul>
+ <li>Photos are objects</li>
+ <li>Galleries (sets) are objects</li>
+ <li>Photos are members of galleries</li>
+ <li>Photos & galleries have attributes (single-valued: "title", multi-valued: "tag")</li>
+ <li>Photos might be updated over time:
+ <ul>
+ <li>EXIF GPS updated, cropping, white balance</li>
+ <li>don't want to break links!</li>
+ </ul>
+ </li>
+ </ul>
+ </article>
+
+ <article class='fill'>
+ <p><img src="blobjects.png" width="100%"/></p>
+ </article>
+
+ <article>
+ <h1 class='center'>
+ How to make sense of that?
+ </h1>
+ </article>
+
+ <article>
+ <h1 class='center'>
+ Indexing & Search
+ </h1>
+ </article>
+
+ <article>
<h3>
- Slide with a table (smaller text)
+ Indexing: summary
</h3>
-
- <table>
- <tr>
- <th>
- Name
- <th>
- Occupation
- <tr>
- <td>
- Luke Mahé
- <td>
- V.P. of Keepin’ It Real
- <tr>
- <td>
- Marcin Wichary
- <td>
- The Michael Bay of Doodles
- </table>
+ <p style='margin-top: 2em'>For each blob, build an index of:
+ <ul>
+ <li>directed graph of inter-blob references</li>
+ <li>(permanode, time) => resolved attributes</li>
+ <li>(permanode, time) => set memberships</li>
+ <li>etc...</li>
+ </ul>
</article>
-
+
<article>
<h3>
- Styles
+ Indexing & Replication
</h3>
<ul>
- <li>
- <span class='red'>class="red"</span>
- <li>
- <span class='blue'>class="blue"</span>
- <li>
- <span class='green'>class="green"</span>
- <li>
- <span class='yellow'>class="yellow"</span>
- <li>
- <span class='black'>class="black"</span>
- <li>
- <span class='white'>class="white"</span>
- <li>
- <b>bold</b> and <i>italic</i>
+ <li>indexing is real-time, no polling</li>
+ <li>MySQL index speaks the blob server protocol</li>
+ <li>just replicated to MySQL (etc) just like Amazon S3 (etc)</li>
</ul>
+ <center><img src='repl.png' /></center>
</article>
-
+
<article>
- <h2>
- Segue slide
- </h2>
+ <h3>
+ Search
+ </h3>
+ <ul>
+ <li>Permanodes created by $who, sorted by date desc, type "photo", tagged "funny"</li>
+ <li>My recent backups with attribute "hostname" == "camlistore.org",</l>
+ <li>All friends' galleries in which this photo appears,</li>
+ <li>etc...</li>
+ </ul>
+ <p>...similar to your email, or docs.google.com. "My stuff" or "My bookmarks".</p>
</article>
<article>
<h3>
- Slide with an image
+ Privacy Model
</h3>
- <p>
- <img style='height: 500px' src='images/example-graph.png'>
- </p>
- <div class='source'>
- Source: Sergey Brin
- </div>
+ <ul>
+ <li>all your blobs & objects & searches are private</li>
+ <li>nothing is public by default</li>
+ </ul>
</article>
- <article>
+ <article>
+ <h1>
+ What if you want to share with friends, or globally publish something?
+ </h1>
+ </article>
+
+ <article>
<h3>
- Slide with an image (centered)
+ Sharing & Share Blobs
</h3>
- <p>
- <img class='centered' style='height: 500px' src='images/example-graph.png'>
- </p>
- <div class='source'>
- Source: Larry Page
- </div>
+ <ul>
+ <li>the act of sharing involves creating a new <span class='red'>share claim</span>, just another blob, signed.</li>
+ </ul>
</article>
+
<article class='fill'>
<h3>
Image filling the slide (with optional header)
@@ -487,76 +645,10 @@
<img src='images/example-cat.jpg'>
</p>
<div class='source white'>
- Source: Eric Schmidt
+ Cat
</div>
</article>
- <article>
- <h3>
- This slide has some code
- </h3>
- <section>
- <pre>
-<script type='text/javascript'>
- // Say hello world until the user starts questioning
- // the meaningfulness of their existence.
- function helloWorld(world) {
- for (var i = 42; --i >= 0;) {
- alert('Hello ' + String(world));
- }
- }
-</script>
-<style>
- p { color: pink }
- b { color: blue }
- u { color: 'umber' }
-</style>
-</pre>
- </section>
- </article>
-
- <article class='smaller'>
- <h3>
- This slide has some code (small font)
- </h3>
- <section>
- <pre>
-<script type='text/javascript'>
- // Say hello world until the user starts questioning
- // the meaningfulness of their existence.
- function helloWorld(world) {
- for (var i = 42; --i >= 0;) {
- alert('Hello ' + String(world));
- }
- }
-</script>
-<style>
- p { color: pink }
- b { color: blue }
- u { color: 'umber' }
-</style>
-</pre>
- </section>
- </article>
-
- <article>
- <q>
- The best way to predict the future is to invent it.
- </q>
- <div class='author'>
- Alan Kay
- </div>
- </article>
-
- <article class='smaller'>
- <q>
- A distributed system is one in which the failure of a computer
- you didn’t even know existed can render your own computer unusable.
- </q>
- <div class='author'>
- Leslie Lamport
- </div>
- </article>
<article class='nobackground'>
<h3>
@@ -583,8 +675,8 @@
</h3>
<ul>
- <li>
- <a href='http://www.google.com'>google.com</a>
+ <li>Brad Fitzpatrick, br...@danga.com</li>
+ <li>camlistore.org</li>
</ul>
</article>
diff --git a/talks/2011-05-07-Camlistore-Sao-Paolo/repl.png b/talks/2011-05-07-Camlistore-Sao-Paolo/repl.png
new file mode 100644
index 0000000..81ed94e
Binary files /dev/null and b/talks/2011-05-07-Camlistore-Sao-Paolo/repl.png differ
diff --git a/talks/2011-05-07-Camlistore-Sao-Paolo/slides.js b/talks/2011-05-07-Camlistore-Sao-Paolo/slides.js
index eed1cdb..7f8ce9b 100644
--- a/talks/2011-05-07-Camlistore-Sao-Paolo/slides.js
+++ b/talks/2011-05-07-Camlistore-Sao-Paolo/slides.js
@@ -546,8 +546,8 @@ function addFontStyle() {
var el = document.createElement('link');
el.rel = 'stylesheet';
el.type = 'text/css';
- el.href = 'http://fonts.googleapis.com/css?family=' +
- 'Open+Sans:regular,semibold,italic,italicsemibold|Droid+Sans+Mono';
+// el.href = 'http://fonts.googleapis.com/css?family=' +
+// 'Open+Sans:regular,semibold,italic,italicsemibold|Droid+Sans+Mono';
document.body.appendChild(el);
};
hooks/post-receive
--
Camlistore