Merge pull request #6 from pankajsoni19/develop

Weighted sender for smtp+messenger

README.md

@@ -9,49 +9,56 @@ Visit [listmonk.app](https://listmonk.app) for more info.
 I tagged it to db schema version, and will follow that as semver.
 
 ##### Long running campaigns
-* When creating campaign choose run type as `Event Subscription`
-* Supported via
-	* Modify subscriber list memberships -> `/api/subscribers/lists`
-	* Create a new subscriber -> `/api/subscribers`
-* If no new message, worker will sleep for 1 minute before querying
+
+- When creating campaign choose run type as `Event Subscription`
+- Supported via
+  - Modify subscriber list memberships -> `/api/subscribers/lists`
+  - Create a new subscriber -> `/api/subscribers`
+- If no new message, worker will sleep for 1 minute before querying
+
+##### Weighted From
+
+- From is moved into smtp, messenger config.
+- Sample -> `One <[email protected]>,10,Two <[email protected]>,5`. This will send based on weights assigned. Can be phone number for messenger.
 
 ##### Per Campaign smtp/messenger
 
-* In `Settings`>`SMTP` config specify a name.
-* In create/update campaign specify same name. 
-* It will use that messenger to send outbound event.
-* Specify a `default`, if none specified picks first. It is used to send system alert emails.
+- In `Settings`>`SMTP` config specify a name.
+- In create/update campaign specify same name.
+- It will use that messenger to send outbound event.
+- Specify a `default`, if none specified picks first. It is used to send system alert emails.
 
 ##### Query list by exact name
 
-* `/api/lists` specify `?name=` for exact lookup. Specify multiple times to search multiple
+- `/api/lists` specify `?name=` for exact lookup. Specify multiple times to search multiple
 
 ##### Per Campaign `Sliding window limit`
 
-* Config added to create/update campaign UI
-* Config is removed from `Settings`>`Performance`. On upgrade this is copied to all campaigns.
-* Allows setting limit per campaign per endpoint.
+- Config added to create/update campaign UI
+- Config is removed from `Settings`>`Performance`. On upgrade this is copied to all campaigns.
+- Allows setting limit per campaign per endpoint.
 
 ##### Cloning of lists
 
-* Cloning action button on lists page. It copies all config, subscribers to the new list.
+- Cloning action button on lists page. It copies all config, subscribers to the new list.
 
 ##### Bug fixes
 
-* Campaign pause/stop. 
-	* Though stopped on UI, it used to run in bg over the last fetched subscriber list and loop stops quite late.
-	* Server config change forces a restart so the delta of unprocessed subscribers get lost.
+- Campaign pause/stop.
+  - Though stopped on UI, it used to run in bg over the last fetched subscriber list and loop stops quite late.
+  - Server config change forces a restart so the delta of unprocessed subscribers get lost.
 
 ##### Default data
 
-* Skips default data creation for list, template, campaigns on install.
-* check_updates is false
+- Skips default data creation for list, template, campaigns on install.
+- check_updates is false
 
 ## Installation
 
 Assumes we are on debian
 
 ### Via docker
+
 ```bash
 docker pull pankaj19soni/listmonk:latest
 
@@ -67,10 +74,10 @@ docker compose up -d
 
 ### Requirements
 
-* yarn 2
-* nodejs 20
-* golang latest
-* postgres >= 12
+- yarn 2
+- nodejs 20
+- golang latest
+- postgres >= 12
 
 ### Build container
 
@@ -104,28 +111,30 @@ make dist
 docker build -t YOUR-TAG .
 ```
 
-## Upgrade 
+## Upgrade
 
-* Take db backup
-* Stop previous container/binary.
-* Update compose file to point to new image or point to new binary and start
+- Take db backup
+- Stop previous container/binary.
+- Update compose file to point to new image or point to new binary and start
 
 Visit `http://localhost:9000`
 
-__________________
+---
 
 ### Binary
+
 - `./listmonk --new-config` to generate config.toml. Edit it.
 - `./listmonk --install` to setup the Postgres DB (or `--upgrade` to upgrade an existing DB. Upgrades are idempotent and running them multiple times have no side effects).
 - Run `./listmonk` and visit `http://localhost:9000`
 
 See [installation docs](https://listmonk.app/docs/installation)
-__________________
 
+---
 
 ## Developers
-listmonk is free and open source software licensed under AGPLv3. If you are interested in contributing, refer to the [developer setup](https://listmonk.app/docs/developer-setup). The backend is written in Go and the frontend is Vue with Buefy for UI. 
 
+listmonk is free and open source software licensed under AGPLv3. If you are interested in contributing, refer to the [developer setup](https://listmonk.app/docs/developer-setup). The backend is written in Go and the frontend is Vue with Buefy for UI.
 
 ## License
+
 listmonk is licensed under the AGPL v3 license.

cmd/admin.go

@@ -13,7 +13,6 @@ import (
 
 type serverConfig struct {
 	RootURL       string          `json:"root_url"`
-	FromEmail     string          `json:"from_email"`
 	Messengers    []string        `json:"messengers"`
 	Langs         []i18nLang      `json:"langs"`
 	Lang          string          `json:"lang"`
@@ -32,7 +31,6 @@ func handleGetServerConfig(c echo.Context) error {
 
 	out := serverConfig{
 		RootURL:       app.constants.RootURL,
-		FromEmail:     app.constants.FromEmail,
 		Lang:          app.constants.Lang,
 		Permissions:   app.constants.PermissionsRaw,
 		HasLegacyUser: app.constants.HasLegacyUser,

cmd/campaigns.go

@@ -211,7 +211,7 @@ func handleCreateCampaign(c echo.Context) error {
 		o.ContentType = models.CampaignContentTypeRichtext
 	}
 	if o.Messenger == "" {
-		o.Messenger = "email"
+		o.Messenger = app.defaultMessenger.Name()
 	}
 
 	// Validate.
@@ -473,7 +473,6 @@ func handleTestCampaign(c echo.Context) error {
 	// Override certain values from the DB with incoming values.
 	camp.Name = req.Name
 	camp.Subject = req.Subject
-	camp.FromEmail = req.FromEmail
 	camp.Body = req.Body
 	camp.AltBody = req.AltBody
 	camp.Messenger = req.Messenger
@@ -565,14 +564,6 @@ func sendTestMessage(sub models.Subscriber, camp *models.Campaign, app *App) err
 
 // validateCampaignFields validates incoming campaign field values.
 func validateCampaignFields(c campaignReq, app *App) (campaignReq, error) {
-	if c.FromEmail == "" {
-		c.FromEmail = app.constants.FromEmail
-	} else if !regexFromAddress.Match([]byte(c.FromEmail)) {
-		if _, err := app.importer.SanitizeEmail(c.FromEmail); err != nil {
-			return c, errors.New(app.i18n.T("campaigns.fieldInvalidFromEmail"))
-		}
-	}
-
 	if !strHasLen(c.Name, 1, stdInputMaxLen) {
 		return c, errors.New(app.i18n.T("campaigns.fieldInvalidName"))
 	}

cmd/init.go

@@ -61,7 +61,6 @@ type constants struct {
 	LogoURL                       string   `koanf:"logo_url"`
 	FaviconURL                    string   `koanf:"favicon_url"`
 	LoginURL                      string   `koanf:"login_url"`
-	FromEmail                     string   `koanf:"from_email"`
 	NotifyEmails                  []string `koanf:"notify_emails"`
 	EnablePublicSubPage           bool     `koanf:"enable_public_subscription_page"`
 	EnablePublicArchive           bool     `koanf:"enable_public_archive"`
@@ -495,7 +494,6 @@ func initCampaignManager(q *models.Queries, cs *constants, app *App) *manager.Ma
 		Concurrency:        ko.Int("app.concurrency"),
 		MessageRate:        ko.Int("app.message_rate"),
 		MaxSendErrors:      ko.Int("app.max_send_errors"),
-		FromEmail:          cs.FromEmail,
 		IndividualTracking: ko.Bool("privacy.individual_tracking"),
 		UnsubURL:           cs.UnsubURL,
 		OptinURL:           cs.OptinURL,

cmd/notifications.go

@@ -43,7 +43,6 @@ func (app *App) sendNotification(toEmails []string, subject, tplName string, dat
 
 	m := models.Message{}
 	m.ContentType = app.notifTpls.contentType
-	m.From = app.constants.FromEmail
 	m.To = toEmails
 	m.Subject = subject
 	m.Body = body

cmd/public.go

@@ -600,7 +600,6 @@ func handleSelfExportSubscriberData(c echo.Context) error {
 	const fname = "data.json"
 	if err := app.defaultMessenger.Push(models.Message{
 		ContentType: app.notifTpls.contentType,
-		From:        app.constants.FromEmail,
 		To:          []string{data.Email},
 		Subject:     subject,
 		Body:        body,

cmd/settings.go

@@ -310,7 +310,6 @@ func handleTestSMTPSettings(c echo.Context) error {
 
 	m := models.Message{}
 	m.ContentType = app.notifTpls.contentType
-	m.From = app.constants.FromEmail
 	m.To = []string{to}
 	m.Subject = app.i18n.T("settings.smtp.testConnection")
 	m.Body = b.Bytes()

cmd/subscribers.go

@@ -3,7 +3,6 @@ package main
 import (
 	"encoding/csv"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -33,16 +32,6 @@ type subQueryReq struct {
 	All                bool   `json:"all"`
 }
 
-// subProfileData represents a subscriber's collated data in JSON
-// for export.
-type subProfileData struct {
-	Email         string          `db:"email" json:"-"`
-	Profile       json.RawMessage `db:"profile" json:"profile,omitempty"`
-	Subscriptions json.RawMessage `db:"subscriptions" json:"subscriptions,omitempty"`
-	CampaignViews json.RawMessage `db:"campaign_views" json:"campaign_views,omitempty"`
-	LinkClicks    json.RawMessage `db:"link_clicks" json:"link_clicks,omitempty"`
-}
-
 // subOptin contains the data that's passed to the double opt-in e-mail template.
 type subOptin struct {
 	models.Subscriber
@@ -59,10 +48,6 @@ var (
 		UUID:    dummyUUID,
 		Attribs: models.JSON{"city": "Bengaluru"},
 	}
-
-	subQuerySortFields = []string{"email", "name", "created_at", "updated_at"}
-
-	errSubscriberExists = errors.New("subscriber already exists")
 )
 
 // handleGetSubscriber handles the retrieval of a single subscriber by ID.
@@ -174,7 +159,7 @@ loop:
 		if err != nil {
 			return err
 		}
-		if out == nil || len(out) == 0 {
+		if len(out) == 0 {
 			break
 		}
 
@@ -421,7 +406,7 @@ func handleDeleteSubscribers(c echo.Context) error {
 		}
 		if len(i) == 0 {
 			return echo.NewHTTPError(http.StatusBadRequest,
-				app.i18n.Ts("subscribers.errorNoIDs", "error", err.Error()))
+				app.i18n.Ts("subscribers.errorNoIDs", "error"))
 		}
 		subIDs = i
 	}

cmd/templates.go

@@ -101,7 +101,6 @@ func handlePreviewTemplate(c echo.Context) error {
 			UUID:         dummyUUID,
 			Name:         app.i18n.T("templates.dummyName"),
 			Subject:      app.i18n.T("templates.dummySubject"),
-			FromEmail:    "[email protected]",
 			TemplateBody: tpl.Body,
 			Body:         dummyTpl,
 		}

cmd/tx.go

@@ -124,7 +124,6 @@ func handleSendTxMessage(c echo.Context) error {
 		msg := models.Message{}
 		msg.Subscriber = sub
 		msg.To = []string{sub.Email}
-		msg.From = m.FromEmail
 		msg.Subject = m.Subject
 		msg.ContentType = m.ContentType
 		msg.Messenger = m.Messenger
@@ -193,10 +192,6 @@ func validateTxMessage(m models.TxMessage, app *App) (models.TxMessage, error) {
 		}
 	}
 
-	if m.FromEmail == "" {
-		m.FromEmail = app.constants.FromEmail
-	}
-
 	if !app.manager.HasMessenger(m.Messenger) {
 		return m, echo.NewHTTPError(http.StatusBadRequest, app.i18n.Ts("campaigns.fieldInvalidMessenger", "name", m.Messenger))
 	}

cmd/upgrade.go

@@ -43,6 +43,7 @@ var migList = []migFunc{
 	{"v5.0.0", migrations.V5_0_0},
 	{"v5.1.0", migrations.V5_1_0},
 	{"v5.2.0", migrations.V5_2_0},
+	{"v5.3.0", migrations.V5_3_0},
 }
 
 // upgrade upgrades the database to the current version by running SQL migration files