soju/logger.go

314 lines
7.5 KiB
Go
Raw Normal View History

package soju
import (
Add support for downstream CHATHISTORY This adds support for the WIP (at the time of this commit) draft/chathistory extension, based on the draft at [1] and the additional comments at [2]. This gets the history by parsing the chat logs, and is therefore only enabled when the logs are enabled and the log path is configured. Getting the history only from the logs adds some restrictions: - we cannot get history by msgid (those are not logged) - we cannot get the users masks (maybe they could be inferred from the JOIN etc, but it is not worth the effort and would not work every time) The regular soju network history is not sent to clients that support draft/chathistory, so that they can fetch what they need by manually calling CHATHISTORY. The only supported command is BEFORE for now, because that is the only required command for an app that offers an "infinite history scrollback" feature. Regarding implementation, rather than reading the file from the end in reverse, we simply start from the beginning of each log file, store each PRIVMSG into a ring, then add the last lines of that ring into the history we'll return later. The message parsing implementation must be kept somewhat fast because an app could potentially request thousands of messages in several files. Here we are using simple sscanf and indexOf rather than regexps. In case some log files do not contain any message (for example because the user had not joined a channel at that time), we try up to a 100 days of empty log files before giving up. [1]: https://github.com/prawnsalad/ircv3-specifications/pull/3/files [2]: https://github.com/ircv3/ircv3-specifications/pull/393/files#r350210018
2020-05-21 23:59:57 +01:00
"bufio"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"gopkg.in/irc.v3"
)
const messageLoggerMaxTries = 100
type messageLogger struct {
network *network
entity string
path string
file *os.File
}
func newMessageLogger(network *network, entity string) *messageLogger {
return &messageLogger{
network: network,
entity: entity,
}
}
func logPath(network *network, entity string, t time.Time) string {
user := network.user
srv := user.srv
// TODO: handle/forbid network/entity names with illegal path characters
year, month, day := t.Date()
filename := fmt.Sprintf("%04d-%02d-%02d.log", year, month, day)
return filepath.Join(srv.LogPath, user.Username, network.GetName(), entity, filename)
}
func (ml *messageLogger) Append(msg *irc.Message) error {
s := formatMessage(msg)
if s == "" {
return nil
}
var t time.Time
if tag, ok := msg.Tags["time"]; ok {
var err error
t, err = time.Parse(serverTimeLayout, string(tag))
if err != nil {
return fmt.Errorf("failed to parse message time tag: %v", err)
}
t = t.In(time.Local)
} else {
t = time.Now()
}
2020-04-03 19:48:23 +01:00
// TODO: enforce maximum open file handles (LRU cache of file handles)
// TODO: handle non-monotonic clock behaviour
path := logPath(ml.network, ml.entity, t)
if ml.path != path {
if ml.file != nil {
ml.file.Close()
}
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0700); err != nil {
return fmt.Errorf("failed to create logs directory %q: %v", dir, err)
}
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return fmt.Errorf("failed to open log file %q: %v", path, err)
}
ml.path = path
ml.file = f
}
_, err := fmt.Fprintf(ml.file, "[%02d:%02d:%02d] %s\n", t.Hour(), t.Minute(), t.Second(), s)
if err != nil {
return fmt.Errorf("failed to log message to %q: %v", ml.path, err)
}
return nil
}
func (ml *messageLogger) Close() error {
if ml.file == nil {
return nil
}
return ml.file.Close()
}
// formatMessage formats a message log line. It assumes a well-formed IRC
// message.
func formatMessage(msg *irc.Message) string {
switch strings.ToUpper(msg.Command) {
case "NICK":
return fmt.Sprintf("*** %s is now known as %s", msg.Prefix.Name, msg.Params[0])
case "JOIN":
return fmt.Sprintf("*** Joins: %s (%s@%s)", msg.Prefix.Name, msg.Prefix.User, msg.Prefix.Host)
case "PART":
var reason string
if len(msg.Params) > 1 {
reason = msg.Params[1]
}
return fmt.Sprintf("*** Parts: %s (%s@%s) (%s)", msg.Prefix.Name, msg.Prefix.User, msg.Prefix.Host, reason)
case "KICK":
nick := msg.Params[1]
var reason string
if len(msg.Params) > 2 {
reason = msg.Params[2]
}
return fmt.Sprintf("*** %s was kicked by %s (%s)", nick, msg.Prefix.Name, reason)
case "QUIT":
var reason string
if len(msg.Params) > 0 {
reason = msg.Params[0]
}
return fmt.Sprintf("*** Quits: %s (%s@%s) (%s)", msg.Prefix.Name, msg.Prefix.User, msg.Prefix.Host, reason)
case "TOPIC":
var topic string
if len(msg.Params) > 1 {
topic = msg.Params[1]
}
return fmt.Sprintf("*** %s changes topic to '%s'", msg.Prefix.Name, topic)
case "MODE":
return fmt.Sprintf("*** %s sets mode: %s", msg.Prefix.Name, strings.Join(msg.Params[1:], " "))
case "NOTICE":
return fmt.Sprintf("-%s- %s", msg.Prefix.Name, msg.Params[1])
case "PRIVMSG":
2020-08-17 14:01:53 +01:00
if cmd, params, ok := parseCTCPMessage(msg); ok && cmd == "ACTION" {
return fmt.Sprintf("* %s %s", msg.Prefix.Name, params)
} else {
return fmt.Sprintf("<%s> %s", msg.Prefix.Name, msg.Params[1])
}
default:
return ""
}
}
Add support for downstream CHATHISTORY This adds support for the WIP (at the time of this commit) draft/chathistory extension, based on the draft at [1] and the additional comments at [2]. This gets the history by parsing the chat logs, and is therefore only enabled when the logs are enabled and the log path is configured. Getting the history only from the logs adds some restrictions: - we cannot get history by msgid (those are not logged) - we cannot get the users masks (maybe they could be inferred from the JOIN etc, but it is not worth the effort and would not work every time) The regular soju network history is not sent to clients that support draft/chathistory, so that they can fetch what they need by manually calling CHATHISTORY. The only supported command is BEFORE for now, because that is the only required command for an app that offers an "infinite history scrollback" feature. Regarding implementation, rather than reading the file from the end in reverse, we simply start from the beginning of each log file, store each PRIVMSG into a ring, then add the last lines of that ring into the history we'll return later. The message parsing implementation must be kept somewhat fast because an app could potentially request thousands of messages in several files. Here we are using simple sscanf and indexOf rather than regexps. In case some log files do not contain any message (for example because the user had not joined a channel at that time), we try up to a 100 days of empty log files before giving up. [1]: https://github.com/prawnsalad/ircv3-specifications/pull/3/files [2]: https://github.com/ircv3/ircv3-specifications/pull/393/files#r350210018
2020-05-21 23:59:57 +01:00
func parseMessage(line, entity string, ref time.Time) (*irc.Message, time.Time, error) {
var hour, minute, second int
_, err := fmt.Sscanf(line, "[%02d:%02d:%02d] ", &hour, &minute, &second)
if err != nil {
return nil, time.Time{}, err
}
line = line[11:]
2020-08-17 14:01:53 +01:00
var cmd, sender, text string
2020-08-17 12:55:22 +01:00
if strings.HasPrefix(line, "<") {
cmd = "PRIVMSG"
2020-08-17 14:01:53 +01:00
parts := strings.SplitN(line[1:], "> ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
sender, text = parts[0], parts[1]
2020-08-17 12:55:22 +01:00
} else if strings.HasPrefix(line, "-") {
cmd = "NOTICE"
2020-08-17 14:01:53 +01:00
parts := strings.SplitN(line[1:], "- ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
sender, text = parts[0], parts[1]
} else if strings.HasPrefix(line, "* ") {
cmd = "PRIVMSG"
parts := strings.SplitN(line[2:], " ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
sender, text = parts[0], "\x01ACTION "+parts[1]+"\x01"
2020-08-17 12:55:22 +01:00
} else {
return nil, time.Time{}, nil
}
2020-08-17 12:55:22 +01:00
year, month, day := ref.Date()
t := time.Date(year, month, day, hour, minute, second, 0, time.Local)
msg := &irc.Message{
Tags: map[string]irc.TagValue{
"time": irc.TagValue(t.UTC().Format(serverTimeLayout)),
},
2020-07-22 11:16:01 +01:00
Prefix: &irc.Prefix{Name: sender},
2020-08-17 12:55:22 +01:00
Command: cmd,
Params: []string{entity, text},
}
return msg, t, nil
}
func parseMessagesBefore(network *network, entity string, ref time.Time, limit int) ([]*irc.Message, error) {
path := logPath(network, entity, ref)
Add support for downstream CHATHISTORY This adds support for the WIP (at the time of this commit) draft/chathistory extension, based on the draft at [1] and the additional comments at [2]. This gets the history by parsing the chat logs, and is therefore only enabled when the logs are enabled and the log path is configured. Getting the history only from the logs adds some restrictions: - we cannot get history by msgid (those are not logged) - we cannot get the users masks (maybe they could be inferred from the JOIN etc, but it is not worth the effort and would not work every time) The regular soju network history is not sent to clients that support draft/chathistory, so that they can fetch what they need by manually calling CHATHISTORY. The only supported command is BEFORE for now, because that is the only required command for an app that offers an "infinite history scrollback" feature. Regarding implementation, rather than reading the file from the end in reverse, we simply start from the beginning of each log file, store each PRIVMSG into a ring, then add the last lines of that ring into the history we'll return later. The message parsing implementation must be kept somewhat fast because an app could potentially request thousands of messages in several files. Here we are using simple sscanf and indexOf rather than regexps. In case some log files do not contain any message (for example because the user had not joined a channel at that time), we try up to a 100 days of empty log files before giving up. [1]: https://github.com/prawnsalad/ircv3-specifications/pull/3/files [2]: https://github.com/ircv3/ircv3-specifications/pull/393/files#r350210018
2020-05-21 23:59:57 +01:00
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
defer f.Close()
historyRing := make([]*irc.Message, limit)
cur := 0
sc := bufio.NewScanner(f)
for sc.Scan() {
msg, t, err := parseMessage(sc.Text(), entity, ref)
Add support for downstream CHATHISTORY This adds support for the WIP (at the time of this commit) draft/chathistory extension, based on the draft at [1] and the additional comments at [2]. This gets the history by parsing the chat logs, and is therefore only enabled when the logs are enabled and the log path is configured. Getting the history only from the logs adds some restrictions: - we cannot get history by msgid (those are not logged) - we cannot get the users masks (maybe they could be inferred from the JOIN etc, but it is not worth the effort and would not work every time) The regular soju network history is not sent to clients that support draft/chathistory, so that they can fetch what they need by manually calling CHATHISTORY. The only supported command is BEFORE for now, because that is the only required command for an app that offers an "infinite history scrollback" feature. Regarding implementation, rather than reading the file from the end in reverse, we simply start from the beginning of each log file, store each PRIVMSG into a ring, then add the last lines of that ring into the history we'll return later. The message parsing implementation must be kept somewhat fast because an app could potentially request thousands of messages in several files. Here we are using simple sscanf and indexOf rather than regexps. In case some log files do not contain any message (for example because the user had not joined a channel at that time), we try up to a 100 days of empty log files before giving up. [1]: https://github.com/prawnsalad/ircv3-specifications/pull/3/files [2]: https://github.com/ircv3/ircv3-specifications/pull/393/files#r350210018
2020-05-21 23:59:57 +01:00
if err != nil {
return nil, err
} else if msg == nil {
Add support for downstream CHATHISTORY This adds support for the WIP (at the time of this commit) draft/chathistory extension, based on the draft at [1] and the additional comments at [2]. This gets the history by parsing the chat logs, and is therefore only enabled when the logs are enabled and the log path is configured. Getting the history only from the logs adds some restrictions: - we cannot get history by msgid (those are not logged) - we cannot get the users masks (maybe they could be inferred from the JOIN etc, but it is not worth the effort and would not work every time) The regular soju network history is not sent to clients that support draft/chathistory, so that they can fetch what they need by manually calling CHATHISTORY. The only supported command is BEFORE for now, because that is the only required command for an app that offers an "infinite history scrollback" feature. Regarding implementation, rather than reading the file from the end in reverse, we simply start from the beginning of each log file, store each PRIVMSG into a ring, then add the last lines of that ring into the history we'll return later. The message parsing implementation must be kept somewhat fast because an app could potentially request thousands of messages in several files. Here we are using simple sscanf and indexOf rather than regexps. In case some log files do not contain any message (for example because the user had not joined a channel at that time), we try up to a 100 days of empty log files before giving up. [1]: https://github.com/prawnsalad/ircv3-specifications/pull/3/files [2]: https://github.com/ircv3/ircv3-specifications/pull/393/files#r350210018
2020-05-21 23:59:57 +01:00
continue
} else if !t.Before(ref) {
Add support for downstream CHATHISTORY This adds support for the WIP (at the time of this commit) draft/chathistory extension, based on the draft at [1] and the additional comments at [2]. This gets the history by parsing the chat logs, and is therefore only enabled when the logs are enabled and the log path is configured. Getting the history only from the logs adds some restrictions: - we cannot get history by msgid (those are not logged) - we cannot get the users masks (maybe they could be inferred from the JOIN etc, but it is not worth the effort and would not work every time) The regular soju network history is not sent to clients that support draft/chathistory, so that they can fetch what they need by manually calling CHATHISTORY. The only supported command is BEFORE for now, because that is the only required command for an app that offers an "infinite history scrollback" feature. Regarding implementation, rather than reading the file from the end in reverse, we simply start from the beginning of each log file, store each PRIVMSG into a ring, then add the last lines of that ring into the history we'll return later. The message parsing implementation must be kept somewhat fast because an app could potentially request thousands of messages in several files. Here we are using simple sscanf and indexOf rather than regexps. In case some log files do not contain any message (for example because the user had not joined a channel at that time), we try up to a 100 days of empty log files before giving up. [1]: https://github.com/prawnsalad/ircv3-specifications/pull/3/files [2]: https://github.com/ircv3/ircv3-specifications/pull/393/files#r350210018
2020-05-21 23:59:57 +01:00
break
}
historyRing[cur%limit] = msg
Add support for downstream CHATHISTORY This adds support for the WIP (at the time of this commit) draft/chathistory extension, based on the draft at [1] and the additional comments at [2]. This gets the history by parsing the chat logs, and is therefore only enabled when the logs are enabled and the log path is configured. Getting the history only from the logs adds some restrictions: - we cannot get history by msgid (those are not logged) - we cannot get the users masks (maybe they could be inferred from the JOIN etc, but it is not worth the effort and would not work every time) The regular soju network history is not sent to clients that support draft/chathistory, so that they can fetch what they need by manually calling CHATHISTORY. The only supported command is BEFORE for now, because that is the only required command for an app that offers an "infinite history scrollback" feature. Regarding implementation, rather than reading the file from the end in reverse, we simply start from the beginning of each log file, store each PRIVMSG into a ring, then add the last lines of that ring into the history we'll return later. The message parsing implementation must be kept somewhat fast because an app could potentially request thousands of messages in several files. Here we are using simple sscanf and indexOf rather than regexps. In case some log files do not contain any message (for example because the user had not joined a channel at that time), we try up to a 100 days of empty log files before giving up. [1]: https://github.com/prawnsalad/ircv3-specifications/pull/3/files [2]: https://github.com/ircv3/ircv3-specifications/pull/393/files#r350210018
2020-05-21 23:59:57 +01:00
cur++
}
if sc.Err() != nil {
return nil, sc.Err()
}
n := limit
if cur < limit {
n = cur
}
start := (cur - n + limit) % limit
if start+n <= limit { // ring doesnt wrap
return historyRing[start : start+n], nil
} else { // ring wraps
history := make([]*irc.Message, n)
r := copy(history, historyRing[start:])
copy(history[r:], historyRing[:n-r])
return history, nil
}
}
func parseMessagesAfter(network *network, entity string, ref time.Time, limit int) ([]*irc.Message, error) {
path := logPath(network, entity, ref)
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
defer f.Close()
var history []*irc.Message
sc := bufio.NewScanner(f)
for sc.Scan() && len(history) < limit {
msg, t, err := parseMessage(sc.Text(), entity, ref)
if err != nil {
return nil, err
} else if msg == nil || !t.After(ref) {
continue
}
history = append(history, msg)
}
if sc.Err() != nil {
return nil, sc.Err()
}
return history, nil
}
func loadHistoryBeforeTime(network *network, entity string, t time.Time, limit int) ([]*irc.Message, error) {
history := make([]*irc.Message, limit)
remaining := limit
tries := 0
for remaining > 0 && tries < messageLoggerMaxTries {
buf, err := parseMessagesBefore(network, entity, t, remaining)
if err != nil {
return nil, err
}
if len(buf) == 0 {
tries++
} else {
tries = 0
}
copy(history[remaining-len(buf):], buf)
remaining -= len(buf)
year, month, day := t.Date()
t = time.Date(year, month, day, 0, 0, 0, 0, t.Location()).Add(-1)
}
return history[remaining:], nil
}
func loadHistoryAfterTime(network *network, entity string, t time.Time, limit int) ([]*irc.Message, error) {
var history []*irc.Message
remaining := limit
tries := 0
now := time.Now()
for remaining > 0 && tries < messageLoggerMaxTries && t.Before(now) {
buf, err := parseMessagesAfter(network, entity, t, remaining)
if err != nil {
return nil, err
}
if len(buf) == 0 {
tries++
} else {
tries = 0
}
history = append(history, buf...)
remaining -= len(buf)
year, month, day := t.Date()
t = time.Date(year, month, day+1, 0, 0, 0, 0, t.Location())
}
return history, nil
}