From 5b4469fcb7f074bbb72e7970094abfee9cdfa037 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Wed, 31 Mar 2021 11:59:13 +0200 Subject: [PATCH] Use BARE for internal message IDs This allows to have shorter and more future-proof IDs. This also guarantees the IDs will only use reasonable ASCII characters (no spaces), removing the need to encode them for PING/PONG tokens. --- downstream.go | 13 ++----- go.mod | 1 + go.sum | 11 +++--- msgstore.go | 84 ++++++++++++++++++++++++++++++++++++++-------- msgstore_fs.go | 61 +++++++++++++++++++++------------ msgstore_memory.go | 23 ++++++++----- 6 files changed, 135 insertions(+), 58 deletions(-) diff --git a/downstream.go b/downstream.go index 3fa4ccc..74e69d8 100644 --- a/downstream.go +++ b/downstream.go @@ -350,7 +350,7 @@ func (dc *downstreamConn) advanceMessageWithID(msg *irc.Message, id string) { // ackMsgID acknowledges that a message has been received. func (dc *downstreamConn) ackMsgID(id string) { - netID, entity, _, err := parseMsgID(id) + netID, entity, err := parseMsgID(id, nil) if err != nil { dc.logger.Printf("failed to ACK message ID %q: %v", id, err) return @@ -365,7 +365,7 @@ func (dc *downstreamConn) ackMsgID(id string) { } func (dc *downstreamConn) sendPing(msgID string) { - token := "soju-msgid-" + base64.RawURLEncoding.EncodeToString([]byte(msgID)) + token := "soju-msgid-" + msgID dc.SendMessage(&irc.Message{ Command: "PING", Params: []string{token}, @@ -377,14 +377,7 @@ func (dc *downstreamConn) handlePong(token string) { dc.logger.Printf("received unrecognized PONG token %q", token) return } - token = strings.TrimPrefix(token, "soju-msgid-") - b, err := base64.RawURLEncoding.DecodeString(token) - if err != nil { - dc.logger.Printf("received malformed PONG token: %v", err) - return - } - msgID := string(b) - + msgID := strings.TrimPrefix(token, "soju-msgid-") dc.ackMsgID(msgID) } diff --git a/go.mod b/go.mod index 8451d6f..9dae53d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( git.sr.ht/~emersion/go-scfg v0.0.0-20201019143924-142a8aa629fc + git.sr.ht/~sircmpwn/go-bare v0.0.0-20210331145808-46f9b5e5bcf9 github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/klauspost/compress v1.11.7 // indirect diff --git a/go.sum b/go.sum index 5560c15..c96db4e 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ git.sr.ht/~emersion/go-scfg v0.0.0-20201019143924-142a8aa629fc h1:51BD67xFX+bozd3ZRuOUfalrhx4/nQSh6A9lI08rYOk= git.sr.ht/~emersion/go-scfg v0.0.0-20201019143924-142a8aa629fc/go.mod h1:t+Ww6SR24yYnXzEWiNlOY0AFo5E9B73X++10lrSpp4U= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +git.sr.ht/~sircmpwn/getopt v0.0.0-20191230200459-23622cc906b3/go.mod h1:wMEGFFFNuPos7vHmWXfszqImLppbc0wEhh6JBfJIUgw= +git.sr.ht/~sircmpwn/go-bare v0.0.0-20210331145808-46f9b5e5bcf9 h1:GpgMhmmlPgaKOSU3WnoaSpZGWgnprcS+ss2w9SchYu4= +git.sr.ht/~sircmpwn/go-bare v0.0.0-20210331145808-46f9b5e5bcf9/go.mod h1:BVJwbDfVjCjoFiKrhkei6NdGcZYpkDkdyCdg1ukytRA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -36,7 +38,6 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -56,8 +57,9 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= @@ -72,7 +74,6 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY= golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -89,5 +90,7 @@ gopkg.in/irc.v3 v3.1.4/go.mod h1:shO2gz8+PVeS+4E6GAny88Z0YVVQSxQghdrMVGQsR9s= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= diff --git a/msgstore.go b/msgstore.go index 18cd0c9..36541cc 100644 --- a/msgstore.go +++ b/msgstore.go @@ -1,11 +1,12 @@ package soju import ( + "bytes" + "encoding/base64" "fmt" - "strconv" - "strings" "time" + "git.sr.ht/~sircmpwn/go-bare" "gopkg.in/irc.v3" ) @@ -29,18 +30,73 @@ type chatHistoryMessageStore interface { LoadAfterTime(network *network, entity string, t time.Time, limit int) ([]*irc.Message, error) } -func formatMsgID(netID int64, entity, extra string) string { - return fmt.Sprintf("%v %v %v", netID, entity, extra) +type msgIDType uint + +const ( + msgIDNone msgIDType = iota + msgIDMemory + msgIDFS +) + +const msgIDVersion uint = 0 + +type msgIDHeader struct { + Version uint + Network bare.Int + Target string + Type msgIDType } -func parseMsgID(s string) (netID int64, entity, extra string, err error) { - l := strings.SplitN(s, " ", 3) - if len(l) != 3 { - return 0, "", "", fmt.Errorf("invalid message ID %q: expected 3 fields", s) - } - netID, err = strconv.ParseInt(l[0], 10, 64) - if err != nil { - return 0, "", "", fmt.Errorf("invalid message ID %q: %v", s, err) - } - return netID, l[1], l[2], nil +type msgIDBody interface { + msgIDType() msgIDType +} + +func formatMsgID(netID int64, target string, body msgIDBody) string { + var buf bytes.Buffer + w := bare.NewWriter(&buf) + + header := msgIDHeader{ + Version: msgIDVersion, + Network: bare.Int(netID), + Target: target, + Type: body.msgIDType(), + } + if err := bare.MarshalWriter(w, &header); err != nil { + panic(err) + } + if err := bare.MarshalWriter(w, body); err != nil { + panic(err) + } + return base64.RawURLEncoding.EncodeToString(buf.Bytes()) +} + +func parseMsgID(s string, body msgIDBody) (netID int64, target string, err error) { + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + return 0, "", fmt.Errorf("invalid internal message ID: %v", err) + } + + r := bare.NewReader(bytes.NewReader(b)) + + var header msgIDHeader + if err := bare.UnmarshalBareReader(r, &header); err != nil { + return 0, "", fmt.Errorf("invalid internal message ID: %v", err) + } + + if header.Version != msgIDVersion { + return 0, "", fmt.Errorf("invalid internal message ID: got version %v, want %v", header.Version, msgIDVersion) + } + + if body != nil { + typ := body.msgIDType() + if header.Type != typ { + return 0, "", fmt.Errorf("invalid internal message ID: got type %v, want %v", header.Type, typ) + } + + if err := bare.UnmarshalBareReader(r, body); err != nil { + return 0, "", fmt.Errorf("invalid internal message ID: %v", err) + } + } + + return int64(header.Network), header.Target, nil } diff --git a/msgstore_fs.go b/msgstore_fs.go index fdeb91c..1518473 100644 --- a/msgstore_fs.go +++ b/msgstore_fs.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "git.sr.ht/~sircmpwn/go-bare" "gopkg.in/irc.v3" ) @@ -16,6 +17,45 @@ const fsMessageStoreMaxTries = 100 var escapeFilename = strings.NewReplacer("/", "-", "\\", "-") +type date struct { + Year, Month, Day int +} + +func newDate(t time.Time) date { + year, month, day := t.Date() + return date{year, int(month), day} +} + +func (d date) Time() time.Time { + return time.Date(d.Year, time.Month(d.Month), d.Day, 0, 0, 0, 0, time.Local) +} + +type fsMsgID struct { + Date date + Offset bare.Int +} + +func (fsMsgID) msgIDType() msgIDType { + return msgIDFS +} + +func parseFSMsgID(s string) (netID int64, entity string, t time.Time, offset int64, err error) { + var id fsMsgID + netID, entity, err = parseMsgID(s, &id) + if err != nil { + return 0, "", time.Time{}, 0, err + } + return netID, entity, id.Date.Time(), int64(id.Offset), nil +} + +func formatFSMsgID(netID int64, entity string, t time.Time, offset int64) string { + id := fsMsgID{ + Date: newDate(t), + Offset: bare.Int(offset), + } + return formatMsgID(netID, entity, &id) +} + // fsMessageStore is a per-user on-disk store for IRC messages. type fsMessageStore struct { root string @@ -36,27 +76,6 @@ func (ms *fsMessageStore) logPath(network *network, entity string, t time.Time) return filepath.Join(ms.root, escapeFilename.Replace(network.GetName()), escapeFilename.Replace(entity), filename) } -func parseFSMsgID(s string) (netID int64, entity string, t time.Time, offset int64, err error) { - netID, entity, extra, err := parseMsgID(s) - if err != nil { - return 0, "", time.Time{}, 0, err - } - - var year, month, day int - _, err = fmt.Sscanf(extra, "%04d-%02d-%02d %d", &year, &month, &day, &offset) - if err != nil { - return 0, "", time.Time{}, 0, fmt.Errorf("invalid message ID %q: %v", s, err) - } - t = time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local) - return netID, entity, t, offset, nil -} - -func formatFSMsgID(netID int64, entity string, t time.Time, offset int64) string { - year, month, day := t.Date() - extra := fmt.Sprintf("%04d-%02d-%02d %d", year, month, day, offset) - return formatMsgID(netID, entity, extra) -} - // nextMsgID queries the message ID for the next message to be written to f. func nextFSMsgID(network *network, entity string, t time.Time, f *os.File) (string, error) { offset, err := f.Seek(0, io.SeekEnd) diff --git a/msgstore_memory.go b/msgstore_memory.go index 0c16cee..bb3f6a7 100644 --- a/msgstore_memory.go +++ b/msgstore_memory.go @@ -2,29 +2,34 @@ package soju import ( "fmt" - "strconv" "time" + "git.sr.ht/~sircmpwn/go-bare" "gopkg.in/irc.v3" ) const messageRingBufferCap = 4096 +type memoryMsgID struct { + Seq bare.Uint +} + +func (memoryMsgID) msgIDType() msgIDType { + return msgIDMemory +} + func parseMemoryMsgID(s string) (netID int64, entity string, seq uint64, err error) { - netID, entity, extra, err := parseMsgID(s) + var id memoryMsgID + netID, entity, err = parseMsgID(s, &id) if err != nil { return 0, "", 0, err } - seq, err = strconv.ParseUint(extra, 10, 64) - if err != nil { - return 0, "", 0, fmt.Errorf("failed to parse message ID %q: %v", s, err) - } - return netID, entity, seq, nil + return netID, entity, uint64(id.Seq), nil } func formatMemoryMsgID(netID int64, entity string, seq uint64) string { - extra := strconv.FormatUint(seq, 10) - return formatMsgID(netID, entity, extra) + id := memoryMsgID{bare.Uint(seq)} + return formatMsgID(netID, entity, &id) } type ringBufferKey struct {