добавить заголовок seen для mirror-to

README.md

@@ -1,12 +1,8 @@
 
-    Rspamd plugin for CommuniGate Pro 5.x, 6.x
+    Rspamd helper for CommuniGate Pro 5.x, 6.x
 
-    Installation and usage instructions can be found at
+    Copyright (C) 2017-2023 Andrey Igoshin <ai@vsu.ru>
-    http://www.communigate.com/CGPMcAfee/#Integrate
+    Version 1.5.0
-
-
-    Copyright (C) 2017-2022 Andrey Igoshin <ai@vsu.ru>
-    Version 1.4.0
 
     https://git.vsu.ru/ai/rspamd-cgp
 
@@ -16,9 +12,9 @@
             Authentication Identifier (default CommuniGate Pro Main Domain)
       -host string
             Rspamd host to connect (default "localhost:11333")
+      -mirror-discard
+            Mirror then discard selected messages
       -mirror-to string
             Mirror selected messages to email
-      -reject-score float
-            Reject score to discard (default +Inf)
       -timeout duration
             Rspamd request timeout (default 15s)

build

@@ -2,6 +2,10 @@
 
 export GOPATH="${HOME}/src/rspamd-cgp"
 
+# если на целевой ОС не совпадает glibc, то собираем без зависимостей.
+# результирующий файл, возможно, получится медленнее и большего размера.
+export CGO_ENABLED=0
+
 if   [ "$1" == "fmt" ]; then
     go fmt $*
 elif [ "$1" == "tidy" ]; then

cgp/cgp.go

@@ -2,14 +2,23 @@ package cgp
 
 import (
 	"bufio"
+	"bytes"
+	"crypto/sha256"
 	"fmt"
+	"io"
 	"os"
 	"regexp"
 	"strconv"
 	"strings"
 	"syscall"
+	"unicode"
 )
 
+const recvHdr = "Received:"
+const seenHdr = "X-Rspamd-Seen:"
+const subjHdr = "Subject:"
+const submitDir = "Submitted"
+
 var MainDomain string
 var reMD *regexp.Regexp
 var reSELF *regexp.Regexp
@@ -52,7 +61,7 @@ func Intf(seq int, ver string) {
 	Putline("%d INTF %d\n", seq, protocol)
 }
 
-func Message(filename string) (from string, rcpts []string, auth string, ip string, qid int, body []byte, err error) {
+func Message(filename string) (from string, rcpts []string, auth string, ip string, qid int, body []byte, seen bool, err error) {
 
 	qid, err = strconv.Atoi((filename)[strings.LastIndexByte(filename, '/')+1 : strings.LastIndexByte(filename, '.')])
 	if err != nil {
@@ -68,7 +77,9 @@ func Message(filename string) (from string, rcpts []string, auth string, ip stri
 	var line []byte
 	var pos int64
 
-	for m := bufio.NewReader(h); ; {
+	m := bufio.NewReader(h)
+
+	for {
 
 		line, err = m.ReadSlice('\n')
 		if err != nil {
@@ -101,7 +112,14 @@ func Message(filename string) (from string, rcpts []string, auth string, ip stri
 		}
 	}
 
-	rcpts = uniqueNonEmptyElementsOf(rcpts)
+	seen, err = isSeen(m)
+	if err != nil {
+		return
+	}
+
+	if seen {
+		return
+	}
 
 	fi, err := h.Stat()
 	if err != nil {
@@ -119,6 +137,8 @@ func Message(filename string) (from string, rcpts []string, auth string, ip stri
 		return
 	}
 
+	rcpts = uniqueNonEmptyElementsOf(rcpts)
+
 	if from == "" || len(rcpts) == 0 || n < len(body) {
 		err = fmt.Errorf("cgp.Message() error: from='%s', len(to)=%d, auth='%s' ip='%s', size=%d/%d", from, len(rcpts), auth, ip, len(body), n)
 	}
@@ -126,26 +146,39 @@ func Message(filename string) (from string, rcpts []string, auth string, ip stri
 	return
 }
 
-func MirrorTo(seq int, to []string, headers []string) {
+func MirrorTo(seq int, qid int, to []string, headers []string, body []byte, mirrorDiscard bool) {
-
-	hdrs := replaceSpecChars(strings.Join(headers, "\n"))
 
 	if protocol >= 4 {
 
 		if len(to) > 0 {
 
+			seenHdr, err := makeSeen(body)
+			if err != nil {
+				Failure(seq, qid, err)
+				return
+			}
+
+			headers = append(headers, seenHdr)
+			hdrs := replaceSpecChars(strings.Join(headers, "\n"))
+
 			mirrorTo := []string{}
 			for _, m := range to {
 				mirrorTo = append(mirrorTo, fmt.Sprintf("MIRRORTO \"%s\"", m))
 			}
 
-			Putline("%d %s ADDHEADER \"%s\" OK\n", seq, strings.Join(mirrorTo, " "), hdrs)
+			if mirrorDiscard {
+				Putline("%d ADDHEADER \"%s\" %s DISCARD\n", seq, hdrs, strings.Join(mirrorTo, " "))
+			} else {
+				Putline("%d ADDHEADER \"%s\" %s OK\n", seq, hdrs, strings.Join(mirrorTo, " "))
+			}
 
 		} else {
+			hdrs := replaceSpecChars(strings.Join(headers, "\n"))
 			Putline("%d ADDHEADER \"%s\" OK\n", seq, hdrs)
 		}
 
 	} else {
+		hdrs := replaceSpecChars(strings.Join(headers, "\n"))
 		Putline("%d ADDHEADER \"%s\"\n", seq, hdrs)
 	}
 }
@@ -154,6 +187,11 @@ func Ok(seq int) {
 	Putline("%d OK\n", seq)
 }
 
+func OkSeen(seq, qid int) {
+	Putline("* %d [%d]: Already seen by Rspamd\n", seq, qid)
+	Putline("%d OK\n", seq)
+}
+
 func Putline(format string, a ...interface{}) {
 	s := fmt.Sprintf(format, a...)
 	syscall.Write(int(os.Stdout.Fd()), []byte(s))
@@ -163,6 +201,302 @@ func Reject(seq int) {
 	Putline("%d REJECT Try again later\n", seq)
 }
 
+func RewriteSubject(seq int, headers []string, subject string, qid int, from string, rcpts []string, body []byte) {
+
+	var err error
+	var firstRecv bool = true
+	var line []byte
+	var m *bufio.Reader
+	var hdr string
+
+	filename := fmt.Sprintf("%s/%drs.sub", submitDir, qid)
+	filetemp := strings.Replace(filename, "sub", "tmp", 1)
+
+	fh, err := os.Create(filetemp)
+	if err != nil {
+		goto fin
+	}
+	defer fh.Close()
+
+	_, err = fh.WriteString(strings.Join([]string{"Return-Path: ", from, "\n"}, ""))
+	if err != nil {
+		goto fin
+	}
+
+	for _, rcpt := range rcpts {
+		_, err = fh.WriteString(strings.Join([]string{"Envelope-To: ", rcpt, "\n"}, ""))
+		if err != nil {
+			goto fin
+		}
+	}
+
+	for _, hdr = range headers {
+		_, err = fh.WriteString(hdr)
+		if err != nil {
+			goto fin
+		}
+	}
+
+	m = bufio.NewReader(bytes.NewReader(body))
+
+	for {
+
+		hdr, err = getHeader(m)
+		if err == io.EOF {
+			err = nil
+			if len(hdr) == 0 {
+				break
+			}
+		}
+		if err != nil {
+			goto fin
+		}
+
+		if hdr == "\n" {
+			// конец заголовка
+			_, err = fh.WriteString(hdr)
+			if err != nil {
+				goto fin
+			}
+			break
+		}
+
+		if firstRecv && strings.HasPrefix(hdr, recvHdr) {
+
+			sum := fmt.Sprintf("%x", sha256.Sum224([]byte(nospace(hdr))))
+
+			_, err = fh.WriteString(strings.Join([]string{seenHdr, " ", sum, "\n"}, ""))
+			if err != nil {
+				goto fin
+			}
+
+			_, err = fh.WriteString(hdr)
+			if err != nil {
+				goto fin
+			}
+
+			firstRecv = false
+			continue
+		}
+
+		if strings.HasPrefix(hdr, subjHdr) {
+
+			_, err = fh.WriteString(strings.Join([]string{subjHdr, " ", subject, "\n"}, ""))
+			if err != nil {
+				goto fin
+			}
+
+			continue
+		}
+
+		_, err = fh.WriteString(hdr)
+		if err != nil {
+			goto fin
+		}
+	}
+
+	for {
+		line, err = m.ReadSlice('\n')
+		if err == io.EOF {
+			err = nil
+			break
+		}
+		if err != nil {
+			goto fin
+		}
+
+		_, err = fh.Write(line)
+		if err != nil {
+			goto fin
+		}
+	}
+
+	if err = fh.Close(); err != nil {
+		goto fin
+	}
+
+	err = os.Rename(filetemp, filename)
+
+fin:
+	if err != nil {
+		Failure(seq, qid, err)
+	} else {
+		Discard(seq)
+	}
+}
+
+func getHeader(m *bufio.Reader) (hdr string, err error) {
+
+	var c byte
+	var line []byte
+
+	var b strings.Builder
+	b.Grow(384)
+
+	for {
+		c, err = m.ReadByte()
+		if err == io.EOF {
+			if c == 0 {
+				break
+			} else {
+				err = nil
+			}
+		}
+		if err != nil {
+			return
+		}
+
+		if b.Len() == 0 {
+
+			if c == ' ' || c == '\t' {
+				err = m.UnreadByte()
+				if err != nil {
+					return
+				}
+				err = fmt.Errorf("bad header")
+				return
+
+			} else if c == '\n' {
+
+				b.WriteByte(c)
+				break
+
+			} else {
+
+				b.WriteByte(c)
+
+				line, err = m.ReadSlice('\n')
+				if err == io.EOF {
+					err = nil
+				}
+				if err != nil {
+					return
+				}
+
+				b.Write(line)
+			}
+
+		} else {
+
+			if c == ' ' || c == '\t' {
+
+				b.WriteByte(c)
+
+				line, err = m.ReadSlice('\n')
+				if err == io.EOF {
+					err = nil
+				}
+				if err != nil {
+					return
+				}
+
+				b.Write(line)
+
+			} else {
+				err = m.UnreadByte()
+				if err != nil {
+					return
+				}
+				break
+			}
+		}
+	}
+
+	hdr = b.String()
+
+	return
+}
+
+func isSeen(m *bufio.Reader) (seen bool, err error) {
+
+	var found bool
+	var seenSum string
+	var hdr string
+
+	for {
+
+		hdr, err = getHeader(m)
+		if err == io.EOF {
+			err = nil
+			if len(hdr) == 0 {
+				break
+			}
+		}
+		if err != nil {
+			return
+		}
+
+		if hdr == "\n" {
+			// конец RFC5322 заголовка
+			break
+		}
+
+		if !found {
+
+			if seenSum, found = strings.CutPrefix(hdr, seenHdr); found {
+				seenSum = strings.TrimSpace(seenSum)
+			}
+
+		} else if strings.HasPrefix(hdr, recvHdr) {
+
+			sum := fmt.Sprintf("%x", sha256.Sum224([]byte(nospace(hdr))))
+			if seenSum == sum {
+				seen = true
+			}
+
+			break
+		}
+	}
+
+	return
+}
+
+func makeSeen(body []byte) (seenhdr string, err error) {
+
+	var hdr string
+
+	m := bufio.NewReader(bytes.NewReader(body))
+
+	for {
+
+		hdr, err = getHeader(m)
+		if err == io.EOF {
+			err = nil
+			if len(hdr) == 0 {
+				break
+			}
+		}
+		if err != nil {
+			return
+		}
+
+		if hdr == "\n" {
+			break
+		}
+
+		if strings.HasPrefix(hdr, recvHdr) {
+			seenhdr = fmt.Sprintf("%s %x", seenHdr, sha256.Sum224([]byte(nospace(hdr))))
+			break
+		}
+	}
+
+	return
+}
+
+func nospace(s string) string {
+
+	var b strings.Builder
+	b.Grow(len(s))
+
+	for _, c := range s {
+		if !unicode.IsSpace(c) {
+			b.WriteRune(c)
+		}
+	}
+
+	return b.String()
+}
+
 func replaceSpecChars(msg string) string {
 
 	var sb strings.Builder

config/config.go

@@ -2,18 +2,17 @@ package config
 
 import (
 	"flag"
-	"math"
 	"strings"
 	"time"
 )
 
 type Config struct {
-	AuthservId  string
+	AuthservId    string
-	Debug       bool
+	Debug         bool
-	Host        string
+	Host          string
-	MirrorTo    []string
+	MirrorDiscard bool
-	RejectScore float64
+	MirrorTo      []string
-	Timeout     time.Duration
+	Timeout       time.Duration
 }
 
 func New() *Config {
@@ -24,8 +23,8 @@ func New() *Config {
 
 	flag.StringVar(&config.AuthservId, "authserv-id", "", "Authentication Identifier (default CommuniGate Pro Main Domain)")
 	flag.StringVar(&config.Host, "host", "localhost:11333", "Rspamd host to connect")
+	flag.BoolVar(&config.MirrorDiscard, "mirror-discard", false, "Mirror then discard selected messages")
 	flag.StringVar(&mirrorTo, "mirror-to", "", "Mirror selected messages to email")
-	flag.Float64Var(&config.RejectScore, "reject-score", math.Inf(1), "Reject score to discard")
 	flag.DurationVar(&config.Timeout, "timeout", 15*time.Second, "Rspamd request timeout")
 	flag.BoolVar(&config.Debug, "debug", false, "Export debug information (for developers)")
 
@@ -35,6 +34,10 @@ func New() *Config {
 		config.MirrorTo = strings.Split(strings.ReplaceAll(mirrorTo, " ", ""), ",")
 	}
 
+	if config.MirrorDiscard && len(config.MirrorTo) == 0 {
+		config.MirrorDiscard = false
+	}
+
 	if config.Timeout < time.Second {
 		config.Timeout *= time.Second
 	}

rspamc/rspamc.go

@@ -16,7 +16,7 @@ import (
 	"git.vsu.ru/ai/rspamd-cgp/config"
 )
 
-var (
+const (
 	headerJunkG string = "X-Junk-Score: [XX]"
 	headerJunkA string = "X-Junk-Score: [XXXX]"
 	headerJunkR string = "X-Junk-Score: [XXXXXXXXXX]"
@@ -24,8 +24,8 @@ var (
 
 var authservId string
 var client *http.Client
+var mirrorDiscard bool
 var mirrorTo []string
-var rejectScore float64
 var host string
 var debug bool
 
@@ -39,8 +39,8 @@ func init() {
 		authservId = cgp.MainDomain
 	}
 
+	mirrorDiscard = config.MirrorDiscard
 	mirrorTo = config.MirrorTo
-	rejectScore = config.RejectScore
 	host = "http://" + config.Host + "/checkv2"
 	debug = config.Debug
 
@@ -61,12 +61,17 @@ func printResponse(v any) {
 
 func Scan(seq int, filename string) {
 
-	from, rcpts, auth, ip, qid, body, err := cgp.Message(filename)
+	from, rcpts, auth, ip, qid, body, seen, err := cgp.Message(filename)
 	if err != nil {
 		cgp.Failure(seq, qid, err)
 		return
 	}
 
+	if seen {
+		cgp.OkSeen(seq, qid)
+		return
+	}
+
 	req, err := http.NewRequest("POST", host, bytes.NewReader(body))
 	if err != nil {
 		cgp.Failure(seq, qid, err)
@@ -100,23 +105,23 @@ func Scan(seq int, filename string) {
 		return
 	}
 
-	var js map[string]interface{}
+	var res map[string]interface{}
-	if err := json.Unmarshal(rbody, &js); err != nil {
+	if err := json.Unmarshal(rbody, &res); err != nil {
 		cgp.Failure(seq, qid, err)
 		return
 	}
 
 	if debug {
-		printResponse(js)
+		printResponse(res)
 	}
 
 	var headers []string
 
-	if _, ok := js["dkim-signature"]; ok {
+	if _, ok := res["dkim-signature"]; ok {
-		headers = append(headers, strings.Join([]string{"DKIM-Signature: ", js["dkim-signature"].(string)}, ""))
+		headers = append(headers, strings.Join([]string{"DKIM-Signature: ", res["dkim-signature"].(string)}, ""))
 	}
 
-	if milter, ok := js["milter"]; ok {
+	if milter, ok := res["milter"]; ok {
 		if hdrs, ok := milter.(map[string]interface{})["add_headers"]; ok {
 			if reflect.TypeOf(hdrs).String() == "map[string]interface {}" {
 				for h, vh := range hdrs.(map[string]interface{}) {
@@ -130,11 +135,10 @@ func Scan(seq int, filename string) {
 		}
 	}
 
-	action := js["action"]
+	action := res["action"]
-	score := (js["score"]).(float64)
 
 	cgp.Putline("* %d [%d]: Action: %s; Score: %.2f/%.2f; Time elapsed: %.3fs\n",
-		seq, qid, action, score, js["required_score"], js["time_real"])
+		seq, qid, action, res["score"], res["required_score"], res["time_real"])
 
 	switch action {
 	case "no action":
@@ -148,19 +152,18 @@ func Scan(seq int, filename string) {
 		cgp.Discard(seq)
 
 	case "quarantine":
-		cgp.MirrorTo(seq, mirrorTo, append(headers, headerJunkR))
+		cgp.MirrorTo(seq, qid, mirrorTo, append(headers, headerJunkR), body, mirrorDiscard)
 
 	case "reject":
-		if score >= rejectScore {
+		cgp.AddHeader(seq, append(headers, headerJunkR))
-			cgp.Putline("* %d [%d]: Action set to discard due to Score(%.2f) >= rejectScore(%d)\n",
-				seq, qid, score, rejectScore)
-			cgp.Discard(seq)
-		} else {
-			cgp.AddHeader(seq, append(headers, headerJunkR))
-		}
 
 	case "rewrite subject":
-		fallthrough
+		if subject, ok := res["subject"]; ok {
+			cgp.RewriteSubject(seq, append(headers, headerJunkA), subject.(string), qid, from, rcpts, body)
+		} else {
+			cgp.AddHeader(seq, append(headers, headerJunkA))
+		}
+
 	case "add header":
 		cgp.AddHeader(seq, append(headers, headerJunkA))